decidim-admin 0.25.2 → 0.26.0.rc1

data/lib/decidim/admin/form_builder.rb

@@ -63,7 +63,14 @@ module Decidim
 
       # Calls Decidim::FormBuilder#editor with default options for admin.
       def editor(name, options = {})
-        super(name, { toolbar: :full, lines: 25 }.merge(options))
+        super(
+          name,
+          {
+            toolbar: :full,
+            lines: 25,
+            editor_images: true
+          }.merge(options)
+        )
       end
     end
   end

data/lib/decidim/admin/import/creator.rb

@@ -11,6 +11,33 @@ module Decidim
       # in order to parse relevant fields. Every import should specify their
       # own creator or this default will be used.
       class Creator
+        class << self
+          # Retuns the resource class to be created with the provided data.
+          def resource_klass
+            raise NotImplementedError, "#{self.class.name} does not define resource class"
+          end
+
+          # Returns the verifier class to be used to ensure the data is valid
+          # for the import.
+          def verifier_klass
+            Decidim::Admin::Import::Verifier
+          end
+
+          def required_headers
+            []
+          end
+
+          def localize_headers(header, locales)
+            @localize_headers ||= begin
+              localize_headers = []
+              locales.each do |locale|
+                localize_headers << "#{header}/#{locale}".to_sym
+              end
+              localize_headers
+            end
+          end
+        end
+
         attr_reader :data
 
         # Initializes the creator with a resource.
@@ -18,15 +45,10 @@ module Decidim
         # data - The data hash to parse.
         # context - The context needed by the producer
         def initialize(data, context = nil)
-          @data = data.except(:id, "id")
+          @data = data
           @context = context
         end
 
-        # Retuns the resource class to be created with the provided data.
-        def self.resource_klass
-          raise NotImplementedError, "#{self.class.name} does not define resource class"
-        end
-
         # Can be used to convert the data hash to the resource attributes in
         # case the data hash to be imported has different column names than the
         # resource object to be created of it.
@@ -50,7 +72,9 @@ module Decidim
           resource.save!
         end
 
-        private
+        protected
+
+        attr_reader :context
 
         def resource
           raise NotImplementedError, "#{self.class.name} does not define resource"
@@ -65,14 +89,10 @@ module Decidim
         # Returns the hash including locale-imported_data pairs. eg. {en: "Heading", ca: "Cap", es: "Bóveda"}
         #
         def locale_hasher(field, locales)
-          return data[field.to_sym] if data.has_key?(field.to_sym)
-
           hash = {}
           locales.each do |locale|
             parsed = data[:"#{field}/#{locale}"]
-            next if parsed.nil?
-
-            hash[locale] = parsed
+            hash[locale] = parsed unless parsed.nil?
           end
           hash
         end

data/lib/decidim/admin/import/importer.rb

@@ -10,17 +10,24 @@ module Decidim
       # You can also use the ImporterFactory class to create an Importer
       # instance.
       class Importer
+        delegate :errors, to: :verifier
+
         # Public: Initializes an Importer.
         #
         # file   - A file with the data to be imported.
         # reader - A Reader to be used to read the data from the file.
-        # creator - A Creator to be used during the import.
+        # creator - A Creator class to be used during the import.
         # context - A hash including component specific data.
         def initialize(file:, reader: Readers::Base, creator: Creator, context: nil)
           @file = file
           @reader = reader
           @creator = creator
           @context = context
+          @data_headers = []
+        end
+
+        def verify
+          verifier.valid?
         end
 
         # Import data and create resources
@@ -40,27 +47,38 @@ module Decidim
           @collection ||= collection_data.map { |item| creator.new(item, context) }
         end
 
-        # Returns array of all resource indexes where validations fail.
-        def invalid_lines
-          @invalid_lines ||= check_invalid_lines(prepare)
+        def invalid_file?
+          collection.blank?
+        rescue Decidim::Admin::Import::InvalidFileError
+          true
         end
 
         private
 
-        attr_reader :file, :reader, :creator, :context
+        attr_reader :file, :reader, :creator, :context, :data_headers
+
+        def verifier
+          # Prepare needs to be called so that data headers become available.
+          data = prepare
+          @verifier ||= creator.verifier_klass.new(
+            headers: data_headers.map(&:to_s),
+            data: data,
+            reader: reader,
+            context: context
+          )
+        end
 
         def collection_data
           return @collection_data if @collection_data
 
           @collection_data = []
-          data_headers = []
           reader.new(file).read_rows do |rowdata, index|
             if index.zero?
-              data_headers = rowdata.map(&:to_sym)
+              @data_headers = rowdata.map { |d| d.to_s.to_sym }
             else
               @collection_data << Hash[
                 rowdata.each_with_index.map do |val, ind|
-                  [data_headers[ind], val]
+                  [@data_headers[ind], val]
                 end
               ]
             end
@@ -69,12 +87,12 @@ module Decidim
           @collection_data
         end
 
-        def check_invalid_lines(imported_data)
-          invalid_lines = []
-          imported_data.each_with_index do |record, index|
-            invalid_lines << index + 1 unless record.valid?
-          end
-          invalid_lines
+        def component
+          context[:current_component]
+        end
+
+        def available_locales
+          @available_locales ||= component.participatory_space.organization.available_locales
         end
       end
     end

data/lib/decidim/admin/import/readers/base.rb

@@ -7,6 +7,13 @@ module Decidim
         # Abstract class with a very naive default implementation. Each importable
         # file type should have it's own reader.
         class Base
+          # Defines which index of the records defines the first line of actual
+          # data. E.g. with spreadsheet formats, the first row contains column
+          # name information.
+          def self.first_data_index
+            0
+          end
+
           def initialize(file)
             @file = file
           end
@@ -17,10 +24,26 @@ module Decidim
           # which can be later used to map the data to correct attributes.
           #
           # This needs to be implemented by the extending classes.
+          #
+          # Returns an array of the import data where the first row should
+          # contain the columns.
           def read_rows
             raise NotImplementedError
           end
 
+          # The example_file should produce an example data file for the user to
+          # download and take example from to produce their import files. The
+          # data provided for the example file generation should be the same as
+          # what is returned by the read_rows method.
+          #
+          # _data - An array of data to produce the file from
+          #
+          # Returns an IO stream that can be saved to a file or sent to the
+          # browser to produce the import file.
+          def example_file(_data)
+            raise NotImplementedError
+          end
+
           protected
 
           attr_reader :file

data/lib/decidim/admin/import/readers/csv.rb

@@ -11,11 +11,26 @@ module Decidim
         class CSV < Base
           MIME_TYPE = "text/csv"
 
+          def self.first_data_index
+            1
+          end
+
           def read_rows
             ::CSV.read(file, col_sep: ";").each_with_index do |row, index|
               yield row, index
             end
           end
+
+          # Returns a StringIO
+          def example_file(data)
+            csv_data = ::CSV.generate(col_sep: ";") do |csv|
+              data.each do |row|
+                csv << row
+              end
+            end
+
+            ::StringIO.new(csv_data)
+          end
         end
       end
     end

data/lib/decidim/admin/import/readers/json.rb

@@ -13,9 +13,79 @@ module Decidim
 
           def read_rows
             json_string = File.read(file)
-            ::JSON.parse(json_string).each_with_index do |row, index|
-              yield row.keys, index if index.zero?
-              yield row.values, index + 1
+            columns = []
+            data = ::JSON.parse(json_string)
+            data.each_with_index do |row, index|
+              row = flat_hash(row)
+              if index.zero?
+                columns = row.keys
+                yield columns.map(&:to_s), index
+              end
+
+              values = columns.map { |c| row[c] }
+              last_present = values.rindex { |v| !v.nil? }
+              if last_present
+                yield values[0..last_present], index + 1
+              else
+                yield [], index + 1
+              end
+            end
+          rescue ::JSON::ParserError
+            raise Decidim::Admin::Import::InvalidFileError, "The provided JSON file is not valid"
+          end
+
+          # Returns a StringIO
+          def example_file(data)
+            columns = data.shift
+            json_data = data.map do |row|
+              deep_hash(
+                columns.each_with_index.map { |col, ind| [col, row[ind]] }.to_h
+              )
+            end
+
+            ::StringIO.new(::JSON.pretty_generate(json_data))
+          end
+
+          private
+
+          # Converts e.g. structure as follows:
+          # { title: { en: => "Foo", es: => "Bar" } }
+          #
+          # Into:
+          # { "title/en": "Foo", "title/es": "Bar" }
+          def flat_hash(data)
+            {}.tap do |final|
+              data.each do |key, value|
+                if value.is_a?(Hash)
+                  flat_hash(value).each do |subkey, subvalue|
+                    final["#{key}/#{subkey}".to_sym] = subvalue
+                  end
+                else
+                  final[key.to_sym] = value
+                end
+              end
+            end
+          end
+
+          # Converts e.g. structure as follows:
+          # { "title/en": "Foo", "title/es": "Bar" }
+          #
+          # Into:
+          # { title: { en: "Foo", es: "Bar" } }
+          def deep_hash(data)
+            {}.tap do |final|
+              data.each do |key, value|
+                keyparts = key.to_s.split("/")
+                current = final
+                while (keypart = keyparts.shift&.to_sym)
+                  if keyparts.any?
+                    current[keypart] ||= {}
+                    current = current[keypart]
+                  else
+                    current[keypart] = value
+                  end
+                end
+              end
             end
           end
         end

data/lib/decidim/admin/import/readers/xlsx.rb

@@ -11,12 +11,36 @@ module Decidim
         class XLSX < Base
           MIME_TYPE = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
 
+          def self.first_data_index
+            1
+          end
+
           def read_rows
             workbook = RubyXL::Parser.parse(file)
             sheet = workbook.worksheets[0]
             sheet.each_with_index do |row, index|
-              yield row.cells.map(&:value), index
+              if row
+                yield row.cells.map { |c| c && c.value }, index
+              else
+                yield [], index
+              end
+            end
+          rescue Zip::Error
+            raise Decidim::Admin::Import::InvalidFileError, "The provided XLSX file is not valid"
+          end
+
+          # Returns a StringIO
+          def example_file(data)
+            workbook = RubyXL::Workbook.new
+            sheet = workbook.worksheets[0]
+
+            data.each_with_index do |row, rowi|
+              row.each_with_index do |col, coli|
+                sheet.add_cell(rowi, coli, col)
+              end
             end
+
+            workbook.stream
           end
         end
       end

data/lib/decidim/admin/import/readers.rb

@@ -13,8 +13,8 @@ module Decidim
         # keys: are used for dynamic help text on admin form.
         # values: are used to validate the file format of imported document.
         ACCEPTED_MIME_TYPES = {
-          json: Readers::JSON::MIME_TYPE,
           csv: Readers::CSV::MIME_TYPE,
+          json: Readers::JSON::MIME_TYPE,
           xlsx: Readers::XLSX::MIME_TYPE
         }.freeze
 
@@ -33,6 +33,12 @@ module Decidim
 
           nil
         end
+
+        def self.search_by_file_extension(extension)
+          return unless ACCEPTED_MIME_TYPES.has_key?(extension.to_sym)
+
+          search_by_mime_type(ACCEPTED_MIME_TYPES[extension.to_sym])
+        end
       end
     end
   end

data/lib/decidim/admin/import/verifier.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+module Decidim
+  module Admin
+    module Import
+      # This is the default verifier class that verifies the import data is
+      # valid before starting the import process. It makes sure the data is in
+      # correct format, contains the correct data headers, etc.
+      #
+      # Individual importers can extend this class to customize the verification
+      # process.
+      class Verifier
+        include ActiveModel::Validations
+
+        validate :validate_headers
+        validate :validate_data, if: -> { errors.blank? }
+
+        # Public: Initializes an Importer.
+        #
+        # headers - An array of the data headers for the import.
+        # data - An array of the generated data records to be imported.
+        # reader - A Reader class that was used to read the raw data.
+        # context - A hash including component specific data.
+        def initialize(headers:, data:, reader:, context: nil)
+          @headers = headers
+          @data = data
+          @reader = reader
+          @context = context
+        end
+
+        protected
+
+        attr_reader :headers, :data, :reader, :context
+
+        def validate_headers
+          if missing_headers.any?
+            message = [
+              I18n.t(
+                "decidim.admin.imports.data_errors.missing_headers.message",
+                count: missing_headers.count,
+                columns: humanize_list(missing_headers)
+              ),
+              I18n.t("decidim.admin.imports.data_errors.missing_headers.detail")
+            ].join(" ")
+
+            errors.add(:headers, message)
+          end
+
+          return unless duplicate_headers.any?
+
+          message = [
+            I18n.t(
+              "decidim.admin.imports.data_errors.duplicate_headers.message",
+              count: duplicate_headers.count,
+              columns: humanize_list(duplicate_headers)
+            ),
+            I18n.t("decidim.admin.imports.data_errors.duplicate_headers.detail")
+          ].join(" ")
+
+          errors.add(:headers, message)
+        end
+
+        def validate_data
+          return if invalid_indexes.empty?
+
+          indexes = humanize_indexes(invalid_indexes, reader.first_data_index)
+          message =
+            if reader.first_data_index.zero?
+              # If the data starts from index zero we don't want to say to the
+              # user that there are errors on "rows". We want to refer to record
+              # numbers instead. This is the case e.g. with JSON data format.
+              [
+                I18n.t(
+                  "decidim.admin.imports.data_errors.invalid_indexes.records.message",
+                  count: invalid_indexes.count,
+                  indexes: indexes
+                ),
+                I18n.t("decidim.admin.imports.data_errors.invalid_indexes.records.detail")
+              ].join(" ")
+            else
+              [
+                I18n.t(
+                  "decidim.admin.imports.data_errors.invalid_indexes.lines.message",
+                  count: invalid_indexes.count,
+                  indexes: indexes
+                ),
+                I18n.t("decidim.admin.imports.data_errors.invalid_indexes.lines.detail")
+              ].join(" ")
+            end
+
+          errors.add(:data, message)
+        end
+
+        def available_locales
+          @available_locales ||= context[:current_organization]&.available_locales || I18n.available_locales.map(&:to_s)
+        end
+
+        def default_locale
+          @default_locale ||= context[:current_organization]&.default_locale || I18n.default_locale.to_s
+        end
+
+        # Individual verifier classes can extend this to provide their required
+        # headers.
+        #
+        # Returns an array of required headers.
+        def required_headers
+          []
+        end
+
+        def required_localized_headers(name)
+          ["#{name}/#{default_locale}"]
+        end
+
+        def missing_headers
+          @missing_headers ||= [].tap do |array|
+            required_headers.each do |required|
+              array << required unless headers.include?(required)
+            end
+          end
+        end
+
+        def duplicate_headers
+          @duplicate_headers ||= headers.select { |e| headers.count(e) > 1 }.uniq
+        end
+
+        # Returns array of all resource indexes where validations fail.
+        def invalid_indexes
+          @invalid_indexes ||= [].tap do |indexes|
+            data.each_with_index do |record, index|
+              indexes << index unless valid_record?(record)
+            end
+          end
+        end
+
+        # Validates the record and allows individual verifiers to customize the
+        # validation logic by overriding this method.
+        #
+        # Returns a boolean indicating whether the record to be imported is
+        # valid.
+        def valid_record?(record)
+          record.valid?
+        end
+
+        # Humanizes the index numbers so that it is understandable for humans.
+        # Index zero becomes one and the indexes are included in a single
+        # string with the last item separated with "and". For instance, for
+        # indexes [1, 2, 3] the message would be "1, 2 and 3".
+        #
+        # Returns a String.
+        def humanize_indexes(indexes, start_index)
+          # Humans don't start counting from zero and this message is shown
+          # for humans. This also takes the data start index into account.
+          indexes = indexes.map { |i| i + start_index + 1 }
+
+          humanize_list(indexes)
+        end
+
+        def humanize_list(list)
+          if list.count > 1
+            last = list.pop
+            "#{list.join(", ")} #{I18n.t("decidim.admin.imports.and")} #{last}"
+          else
+            list.join
+          end
+        end
+      end
+    end
+  end
+end

data/lib/decidim/admin/import.rb

@@ -7,6 +7,9 @@ module Decidim
       autoload :Importer, "decidim/admin/import/importer"
       autoload :Creator, "decidim/admin/import/creator"
       autoload :Readers, "decidim/admin/import/readers"
+      autoload :Verifier, "decidim/admin/import/verifier"
+
+      class InvalidFileError < StandardError; end
     end
   end
 end