locomotivecms 4.0.3 → 4.1.0.rc1

data/app/models/locomotive/concerns/content_type/import.rb

@@ -0,0 +1,124 @@
+module Locomotive
+  module Concerns
+    module ContentType
+      module Import
+
+        extend ActiveSupport::Concern
+
+        included do
+          field :import_enabled, type: Boolean, default: false
+          field :raw_import_state, type: Hash, default: nil
+        end
+
+        def import_state
+          @import_state ||= ImportState.new(raw_import_state)
+        end
+
+        def can_import?          
+          import_enabled? && import_state.can_start?
+        end
+
+        def importing?
+          import_state.running?
+        end
+
+        def import_status
+          import_state.status
+        end
+
+        def start_import(total:)
+          change_import_state({
+            'status' => 'in_progress', 'total_rows' => total
+          }, true)
+        end
+
+        def finish_import
+          change_import_state({ 'status' => 'done' })
+        end
+
+        def cancel_import(error_message)
+          change_import_state({ 'status' => 'canceled', 'error' => error_message })
+        end
+
+        def on_imported_row(index, row_status)
+          change_import_state({
+            row_status.to_s => import_state.rows_count(row_status.to_s) + 1,
+            'failed_ids' => import_state.failed_rows_ids + (row_status.to_s == 'failed' ? [index] : []),
+          })
+        end
+
+        def change_import_state(attributes, clear_state = false)
+          new_attributes = ((clear_state ? nil : raw_import_state) || {}).merge(attributes)
+          update(raw_import_state: new_attributes.merge('updated_at' => Time.zone.now)).tap do
+            @import_state = nil # reset the state object
+          end
+        end
+
+        class ImportState
+
+          def initialize(raw_state)
+            @raw_state = raw_state || { 'status' => 'ready' }
+          end
+
+          def can_start?
+            !running?
+          end
+
+          def running?
+            status == :in_progress
+          end
+
+          def status
+            @raw_state['status'].to_sym
+          end
+
+          def total_rows
+            @raw_state['total_rows']
+          end
+          
+          def processed_rows_count
+            created_rows_count + updated_rows_count + failed_rows_count
+          end
+
+          def rows_count(topic)
+            @raw_state[topic] || 0
+          end
+
+          def created_rows_count
+            rows_count('created')
+          end
+
+          def updated_rows_count
+            rows_count('updated')
+          end
+
+          def failed_rows_count
+            rows_count('failed')
+          end
+
+          def failed_rows_ids
+            @raw_state['failed_ids'] || []
+          end
+        end        
+      end
+    end
+  end
+
+  class ContentEntryImport
+    include ActiveModel::Model
+    
+    attr_accessor :file, :col_sep, :quote_char
+    
+    validates_each :file do |record, attr, value|
+      record.errors.add attr, :blank if value.blank?
+    end
+
+    def options
+      { col_sep: col_sep || ',', quote_char: quote_char || "\"" }
+    end
+
+    def file?
+      file.present?
+    end
+  end
+end

data/app/models/locomotive/concerns/site/metafields.rb

@@ -41,6 +41,21 @@ module Locomotive
           end
         end
 
+        def cast_metafields(namespace)
+          return nil if namespace.blank? || !has_metafields?
+
+          schema = self.metafields_schema.find { |s| s['name'] == namespace }
+          values = self.metafields[namespace]
+
+          return nil if schema.blank? || values.blank?
+
+          values.map do |name, value|
+            field = schema['fields'].find { |f| f['name'] == name }
+            next unless field
+            [name, cast_metafield_value(field, value)]
+          end.compact.to_h
+        end
+
         protected
 
         def _metafields_schema_schema
@@ -75,6 +90,17 @@ module Locomotive
           }
         end
 
+        def cast_metafield_value(field, value)
+          case field['type']
+          when 'boolean'
+            ['1', 'true', true].include?(value)
+          when 'integer'
+            Integer(value)
+          else
+            value
+          end
+        end
+
       end
 
     end

data/app/models/locomotive/content_asset.rb

@@ -1,12 +1,13 @@
 module Locomotive
   class ContentAsset
 
-    include Locomotive::Mongoid::Document
+    include Locomotive::Mongoid::Document    
 
     ## extensions ##
     include Concerns::Asset::Types
     include Concerns::Asset::Vignette
     include Concerns::Asset::Checksum
+    include ActionView::Helpers::NumberHelper
 
     ## fields ##
     field :content_type,  type: String
@@ -20,6 +21,7 @@ module Locomotive
 
     ## validations ##
     validates_presence_of :source
+    validate :maximum_file_size
 
     ## behaviours ##
     mount_uploader :source, ContentAssetUploader, mount_on: :source_filename
@@ -27,6 +29,7 @@ module Locomotive
     ## scopes ##
     scope :ordered, -> { order_by(created_at: :desc) }
     scope :by_filename, ->(query) { where(source_filename: /.*#{query}.*/i) }
+    scope :by_exact_filename, ->(filename) { where(source_filename: filename) }
 
     ## methods ##
 
@@ -39,9 +42,13 @@ module Locomotive
 
     def as_json(options = nil)
       super.merge(
-        thumbnail_url: self.big_vignette_url
+        thumbnail_url: self.big_vignette_url,
+        size_to_human: number_to_human_size(self.size)
       )
     end
 
+    def maximum_file_size
+      errors.add(:source, :maximum_file_size_exceeded) if size_changed? && size > site.maximum_uploaded_file_size
+    end
   end
 end

data/app/models/locomotive/content_type.rb

@@ -15,6 +15,7 @@ module Locomotive
     include Concerns::ContentType::ClassHelpers
     include Concerns::ContentType::PublicSubmissionTitleTemplate
     include Concerns::ContentType::FilterFields
+    include Concerns::ContentType::Import
 
     ## fields ##
     field :name

data/app/models/locomotive/section.rb

@@ -76,7 +76,7 @@ module Locomotive
             properties: {
               id:       { type: 'string' },
               label:    { '$ref': '#/definitions/locale_string' },
-              type:     { enum: ['text', 'image_picker', 'checkbox', 'select', 'url', 'radio', 'content_type', 'hint', 'integer'] },
+              type:     { enum: ['text', 'image_picker', 'asset_picker', 'checkbox', 'select', 'url', 'radio', 'content_type', 'content_entry', 'hint', 'integer'] },
               default:  {}
             },
             required: [:id, :type]
@@ -129,6 +129,8 @@ module Locomotive
           presets:          { type: 'array', items: { '$ref': '#/definitions/preset' } },
           blocks:           { type: 'array', items: { '$ref': '#/definitions/blocks' } },
           max_blocks:       { type: 'integer' },
+          blocks_display:   { enum: ['list', 'tree'] },
+          max_blocks_depth: { type: 'integer' },
           default:          {
             type: 'object',
             properties: {

data/app/models/locomotive/site.rb

@@ -18,6 +18,8 @@ module Locomotive
     ## fields ##
     field :name
     field :robots_txt
+    field :maximum_uploaded_file_size, type: ::Integer, default: Locomotive.config.default_maximum_uploaded_file_size
+    field :overwrite_same_content_assets, type: Boolean, default: false
 
     mount_uploader :picture, PictureUploader, validate_integrity: true
 

data/app/policies/locomotive/content_entry_policy.rb

@@ -23,7 +23,6 @@ module Locomotive
 
     def destroy?
       site_staff? && !membership.visitor?
-    end
-
+    end    
   end
 end

data/app/policies/locomotive/content_type_policy.rb

@@ -25,5 +25,8 @@ module Locomotive
       site_admin_or_designer? || !@resource.hidden?
     end
 
+    def import?
+      @resource.import_enabled?      
+    end
   end
 end

data/app/policies/locomotive/site_policy.rb

@@ -55,16 +55,18 @@ module Locomotive
 
     def permitted_attributes
       plain = [
-        :name, :handle, :picture, :remove_picture, :seo_title, :meta_keywords, :meta_description, :robots_txt,
+        :name, :handle, :picture, :remove_picture, :seo_title, :meta_keywords, :meta_description, :robots_txt, :maximum_uploaded_file_size,
         :timezone_name, :timezone,
         :cache_enabled, :cache_control, :cache_vary,
         :asset_host, :redirect_to_first_domain, :redirect_to_https,
-        :private_access, :password, :prefix_default_locale, :bypass_browser_locale
+        :private_access, :password, :prefix_default_locale, :bypass_browser_locale,
+        :overwrite_same_content_assets,
+        :permitted_params_from_policy
       ]
       hash  = { domains: [], locales: [], url_redirections: [] }
 
       if persisted? && !update_advanced?
-        plain -= [:timezone_name, :timezone, :robots_txt, :cache_enabled, :cache_control, :cache_vary, :prefix_default_locale, :bypass_browser_locale, :asset_host]
+        plain -= [:timezone_name, :timezone, :robots_txt, :cache_enabled, :cache_control, :cache_vary, :prefix_default_locale, :bypass_browser_locale, :asset_host, :maximum_uploaded_file_size]
         hash.delete(:locales)
         hash.delete(:url_redirections)
       end

data/app/services/locomotive/content_asset_service.rb

@@ -13,13 +13,15 @@ module Locomotive
         .page(options[:page] || 1).per(options[:per_page])
     end
 
+    def create(params)
+      create_or_update(params)
+    end
+
     def bulk_create(list)
       list = list.values if list.is_a?(Hash)
 
-      assets = list.map do |params|
-        site.content_assets.create(params)
-      end
-
+      assets = list.map { |params| create_or_update(params) }
+        
       valid_assets = assets.map { |a| a.errors.empty? ? { name: a.source_filename, url: a.source.url, image: a.image?, id: a._id } : nil }.compact
       track_activity 'content_asset.created_bulk', parameters: { assets: valid_assets } unless valid_assets.empty?
 
@@ -32,5 +34,26 @@ module Locomotive
       end
     end
 
+    private
+
+    def create_or_update(params)
+      return site.content_assets.create(params) unless site.overwrite_same_content_assets?
+
+      asset = site.content_assets.build(params)
+      filename = asset.source.filename
+      
+      existing_asset = site.content_assets.by_exact_filename(filename).first      
+
+      return asset.tap { asset.save } unless existing_asset       
+
+      existing_asset.tap do 
+        # can't find out another way to replace the file when it has the same filename        
+        existing_asset.destroy
+
+        # very important to keep the same asset id
+        asset._id = existing_asset._id
+        asset.save
+      end      
+    end
   end
 end

data/app/services/locomotive/content_entry_import_service.rb

@@ -0,0 +1,107 @@
+require 'csv'
+
+module Locomotive
+  class ContentEntryImportService < Struct.new(:content_type)
+
+    class WrongImportFileException < StandardError; end
+
+    def async_import(file, csv_options = nil)
+      csv_asset = content_assets.create(source: file)
+      Locomotive::ImportContentEntryJob.perform_later(
+        content_type.id.to_s,
+        csv_asset.id.to_s,
+        csv_options
+      )
+    end
+
+    def import(csv_asset_id, csv_options = nil)
+      begin
+        csv = load_csv(csv_asset_id, csv_options)
+      rescue WrongImportFileException => e
+        content_type.cancel_import(e.message)
+        return false
+      end 
+
+      content_type.start_import(total: csv.count)
+      import_rows(csv)
+      content_type.finish_import
+
+      content_assets.destroy_all(id: csv_asset_id)
+    end
+
+    def cancel(reason)
+      content_type.cancel_import(reason)
+    end
+
+    private
+
+    def import_rows(csv)
+      csv.each_with_index do |row, index|
+        entry = content_type.entries.where(_slug: row['_slug']).first || content_type.entries.build
+        import_row(row, index, entry)
+      end
+    end
+
+    def import_row(row, index, entry)
+      is_new_entry = !entry.persisted?
+      entry.attributes = attributes_from_row(row)
+      
+      if entry.save
+        content_type.on_imported_row(index, is_new_entry ? :created : :updated) 
+      else
+        content_type.on_imported_row(index, :failed)
+      end
+    rescue Exception => e
+      Rails.logger.error e.message # don't hide the exception
+      Rails.logger.error e.backtrace.join("\n")
+      content_type.on_imported_row(index, :failed)
+    end
+
+    def attributes_from_row(row)
+      attributes = {}
+      content_type.entries_custom_fields.each do |field|
+        next if row[field.name].blank?
+        name, value = transform_attribute(field, row[field.name])
+        attributes[name] = value
+      end
+      attributes
+    end
+
+    def load_csv(csv_asset_id, csv_options = nil)
+      csv_options = { headers: true, col_sep: ';', quote_char: "\"" }.merge(csv_options || {})
+      asset = content_assets.where(id: csv_asset_id).first
+      raise 'The CSV file doesn\'t exist anymore' unless asset
+      CSV.parse(asset.source.read, csv_options)
+    rescue Exception => e
+      raise WrongImportFileException.new e.message
+    end
+    
+    def transform_attribute(field, value)
+      case field.type
+      when 'string'
+        field.name =~ /_asset_url$/ ? 
+        [field.name, content_assets.where(source_filename: value).order_by(:created_at.desc).first&.source&.url] :
+        [field.name, value]
+      when 'belongs_to'
+        [field.name, fetch_entry_ids(field.class_name, value).first]
+      when 'many_to_many'
+        ["#{field.name.singularize}_ids", fetch_entry_ids(field.class_name, value.split(','))]
+      else
+        [field.name, value]
+      end
+    end
+
+    def content_assets
+      content_type.site.content_assets
+    end
+
+    def fetch_entry_ids(class_name, ids_or_slugs)
+      return [] if ids_or_slugs.blank?
+
+      ids_or_slugs  = [*ids_or_slugs]
+      klass         = class_name.constantize
+
+      klass.by_ids_or_slugs(ids_or_slugs).pluck(:_id)
+    end
+  end
+end

data/app/services/locomotive/content_entry_service.rb

@@ -173,7 +173,7 @@ module Locomotive
         { :_id.in   => self.content_type.public_submission_accounts || [] },
         { :email.in => emails || [] }
       ).each do |account|
-        Locomotive::Notifications.new_content_entry(account, entry).deliver
+        Locomotive::Notifications.new_content_entry(site, account, entry).deliver
       end
     end
 

data/app/uploaders/locomotive/picture_uploader.rb

@@ -4,7 +4,7 @@ module Locomotive
   class PictureUploader < BaseUploader
 
     def extension_whitelist
-      %w(jpg jpeg gif png)
+      %w(jpg jpeg gif png tiff)
     end
 
     def image?

data/app/uploaders/locomotive/theme_asset_uploader.rb

@@ -10,7 +10,7 @@ module Locomotive
     end
 
     def extension_whitelist
-      %w(jpg jpeg gif png css js swf flv mp4 eot svg svgz ttf ttc woff woff2 otf ico htc map html cur txt xml json ogv)
+      %w(jpg jpeg gif png css js swf flv mp4 eot svg svgz ttf ttc woff woff2 otf ico htc map html cur txt xml json ogv webm)
     end
 
     def apply_content_type_exception(value)

data/app/views/locomotive/content_assets/_dropzone.html.slim

@@ -1,9 +1,13 @@
 .instructions
-  i.fa.fa-cloud-upload
+  i.fas.fa-cloud-upload-alt
   br
   = t('.instructions').html_safe
   |  
   = link_to t('.browse'), '#', class: 'upload'
+  br
+  br
+  i
+    = t('.maximum_file_size', maximum: number_to_human_size(current_site.maximum_uploaded_file_size)).html_safe
 
 = form_tag('', class: 'hide') do
   = file_field_tag :source, multiple: true

data/app/views/locomotive/content_entries/index.html.slim

@@ -41,6 +41,14 @@
         |  
         = t('.new')
 
+    - if @content_type.import_enabled? 
+      |  
+      |  
+      = link_to new_content_entry_import_path(current_site, @content_type.slug), class: 'btn btn-primary btn-sm' do
+        i.fa.fa-file-import
+        |  
+        = t('.import')
+
 - if @content_type.groupable?
   .row.list-groups
     .nav-container.nav-container--scroll

data/app/views/locomotive/content_entry_imports/new.html.slim

@@ -0,0 +1,23 @@
+- title t('.title', type: @content_type.name.capitalize)
+
+- content_for :actions do
+  .main-action
+    = link_to content_entry_import_path(current_site, @content_type.slug), class: 'btn btn-primary btn-sm' do
+      i.fas.fa-tachometer-alt
+      |  
+      = t('.show')
+
+    |  
+    |  
+
+    = link_to t(:back, scope: 'locomotive.content_entries.shared').html_safe, content_entries_path(current_site, @content_type.slug), class: 'btn btn-sm btn-default'
+
+= locomotive_form_for @import, url: content_entry_import_path(current_site, @content_type.slug) do |f|
+
+  = f.inputs :information do
+    = f.input :file, as: :file
+
+    = f.input :col_sep
+    = f.input :quote_char
+  
+  = f.actions back_url: content_entries_path(current_site, @content_type.slug), use_stored_location: true

data/app/views/locomotive/content_entry_imports/show.html.slim

@@ -0,0 +1,54 @@
+- title t('.title', type: @content_type.name.capitalize)
+
+- content_for :actions do
+  .main-action
+    - if @content_type.importing?
+      = link_to content_entry_import_path(current_site, @content_type.slug), class: 'btn btn-primary btn-sm' do
+        i.fas.fa-sync
+        |  
+        = t('.refresh')
+
+      |  
+      |  
+
+      = link_to content_entry_import_path(current_site,  @content_type.slug), class: 'btn btn-danger btn-sm', method: :delete, data: { confirm: t('locomotive.messages.confirm') } do
+        i.far.fa-trash-alt
+        |  
+        = t('.cancel')
+
+    |  
+    |  
+
+    = link_to t(:back, scope: 'locomotive.content_entries.shared').html_safe, content_entries_path(current_site, @content_type.slug), class: 'btn btn-sm btn-default'
+
+br
+br
+br
+br
+
+.row
+  .col-md-3
+    .metric
+      h2= t('.report.status')
+      h3= t(".statuses.#{@content_type.import_status}")
+
+  .col-md-3
+    .metric
+      h2= t('.report.created_rows')
+      h3= @content_type.import_state.created_rows_count
+
+  .col-md-3
+    .metric
+      h2= t('.report.updated_rows')
+      h3= @content_type.import_state.updated_rows_count
+
+  .col-md-3
+    .metric
+      h2= t('.report.failed_rows')
+      h3= @content_type.import_state.failed_rows_count
+
+br
+br
+
+pre
+  code= @content_type.raw_import_state

data/app/views/locomotive/current_site/form/_advanced.html.slim

@@ -9,6 +9,8 @@
 
   = f.input :robots_txt, as: :code, wrapper_html: { class: 'small' }
 
+  = f.input :maximum_uploaded_file_size
+
   = f.inputs :cache_enabled do
     = f.input :cache_enabled, as: :toggle
     = f.input :cache_control, as: :string, wrapper_html: { class: "#{'hide' unless f.object.cache_enabled?}" }

data/app/views/locomotive/page_content/edit.html.erb

@@ -11,6 +11,7 @@
 
     window.Locomotive.i18n = <%= h I18n.t('locomotive.editor').to_json.html_safe %>;
 
+    window.Locomotive.maximum_uploaded_file_size = <%= current_site.maximum_uploaded_file_size %>;
   </script>
 
   <%= javascript_include_tag  'locomotive/editor' %>

data/config/locales/editor.en.yml

@@ -35,9 +35,19 @@ en:
           remove_button: "Remove"
           crop_button: "Crop"
 
+        asset_picker:
+          select_button: "Select"
+          change_button: "Change"
+          remove_button: "Remove"
+
         content_type:
           show: "Show"
 
+        content_entry:
+          select_button: "Select"
+          change_button: "Change"
+          remove_button: "Remove"
+
       views:
         action_bar:
           header:
@@ -54,6 +64,12 @@ en:
             seo: "SEO"
 
         pickers:
+          assets:
+            title: "Assets"
+            loading: "Loading the assets. Please wait!"
+            add: "+ add"
+            upload_in_progress: "Uploading..."
+            search_placeholder: "Name of your asset"
           images:
             title: "Images"
             loading: "Loading the images. Please wait!"
@@ -82,9 +98,15 @@ en:
             email:
               label: Email address
               placeholder: "name@example.com"
+          content_entry:
+            title: "Pick an entry"
+            input:
+              label: Instance
+              placeholder: Type the label of the instance
 
         preview:
           view: "View"
+          errorMessage: "The page couldn't be loaded. Please contact the developer of the site."
 
         sections:
           edit: