imp_exp 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b947ce095504a9bc4f41b60310670a169259363d84af4518b281b099590770fa
+  data.tar.gz: d959c36041d86fee8c74190434ccc2d23f93259b29ce627ceaabc72a002454eb
+SHA512:
+  metadata.gz: 361153041533b315f3e2277366511738aade7726a802652b19da6a54590f3f2d36b4bd6ff7a13e28fd1690c13791e6e8e3bb6355e425ac299d6f829fe3ddee71
+  data.tar.gz: 4129a1910c806240b85dfbb1d5bf0e0cf07afe8321d23b47dac4da37da9361ee2189ee2427c6122147302553e2f9571ec17cc040f40518f6a652a29d35e8829b

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright 2023 Nicolas Florentin
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,59 @@
+# ImpExp
+
+## Fonctionnement
+
+Features:
+- Export de données brutes (ImpExp::Exporters::Base) et d'un fichier xlsx (ImpExp::Exporters::Xlsx)
+- Import de données brutes (ImpExp::Importers::Base) et depuis un fichier xlsx (ImpExp::Importers::Xlsx) ainsi que les fichiers attachés (facultatifs)
+- Exporter des fichiers (ImpExp::Exporters::Files)
+
+Configuration:
+- un fichier de configuration par modèle (= un onglet dans un classeur xlsx)/ La configuration d'un modèle liste les informations exportés et comment elles doivent être exportées (`serializers`)
+- les imports qui listent les modèles à exporter pour chaque type d'export (voir exemple ImpExp::Imports::Account)
+- les exports qui listent les modèles à exporter pour chaque type d'export (voir exemple ImpExp::Exports::Account)
+
+Extensions:
+- Des serializers peuvent être ajoutés, ils sont des services simples implémentant une méthod `load` et une méthode `dump` permettant de sérialiser et déserialiser une donnée d'un modèle (= une cellule d'une feuille de classeur)
+- On peut ajouter un service de validation pour l'import Xlsx (voir exemple ImpExp::AccountSheetValidator), cette validation est bloquante pour l'import
+- On peut ajouter un service de validation des données brutes (voir exemple ImpExp::DummyDataValidator), cette validation est bloquante pour l'import
+- On peut facilement rajouter un format d'export ou d'import en s'inspirant de ImpExp::Exporters::Xlsx et ImpExp::Importers::Xlsx, en héritant de ImpExp::Exporters::Base et ImpExp::Importers::Base
+
+Autres:
+- la gem définit un service `Utils::XlsxReader` permettant d'abstraire la lecture d'un xlsx (et switcher de librairie facilement)
+- la gem définit un service `PingObserver`, qui permet d'implémenter le pattern observer pour les imports donc suivre l'avancement de l'import
+
+Les fichiers les plus importants à comprendre sont :
+- `imp_exp/models/base.rb` : classe parente de tous les modèles
+- `imp_exp/importers/base.rb` : service contenant la logique d'import
+
+## Bon à savoir pour les développeurs
+
+### Générer un export xlsx à partir des fixtures
+
+```bash
+cd test/dummy
+RAILS_ENV=test bundle exec rake imp_exp:generate_account_export
+```
+
+### Charger les fixtures dans la base
+
+```bash
+cd test/dummy
+FIXTURES_PATH=../fixtures RAILS_ENV=test bundle exec rails db:fixtures:load
+```
+
+### Créer des nouveaux modèles pour l'app dummy
+
+Lorsqu'on créé des nouveaux modèles pour la base dummy, les fichiers (tests et fixtures) sont générés au mauvaise endroit (dans le dossier dummy),
+il faut les déplacement manuellement dans le répertoire parent (imp_exp/test).
+
+### Exécuter les tests
+
+```bash
+  cd imp_exp
+  bin/test
+```
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+require "bundler/setup"
+
+require "bundler/gem_tasks"

data/app/services/imp_exp/data_validators/base.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module DataValidators
+    class Base
+      attr_reader :active_record_class, :columns, :row_offset
+
+      def initialize(model, row_offset = 0)
+        @active_record_class = model.active_record_class
+        @columns = model.active_record_class.columns.index_by(&:name).with_indifferent_access
+        @row_offset = row_offset
+      end
+
+      def call(attributes = [])
+        errors = []
+        attributes.each_with_index do |attrs, i|
+          attrs.each do |attr_name, attr_value|
+            column = columns[attr_name]
+
+            next if column&.array?
+
+            column_type = column&.type || :string
+
+            enums = active_record_class.defined_enums[column&.name]
+
+            if enums.present?
+              possible_values = enums.keys + enums.values
+              unless attr_value.in? possible_values
+                errors << error_hash(row_offset + i + 1, attr_name, attr_value, possible_values)
+              end
+            else
+              unless valid?(attr_value, column_type)
+                errors << error_hash(row_offset + i + 1, attr_name, attr_value, column_type)
+              end
+            end
+          end
+        end
+
+        errors
+      end
+
+      def valid?(attribute_value, column_type)
+        return true if attribute_value.to_s.blank?
+
+        case column_type
+        when :string
+          return true
+        when :boolean
+          return false unless valid_boolean?(attribute_value)
+        when :bigint, :integer
+          return false unless valid_integer?(attribute_value)
+        when :decimal, :float
+          return false unless valid_decimal?(attribute_value)
+        when :date, :datetime
+          return false unless valid_datetime?(attribute_value)
+        when :time
+          return false unless valid_time?(attribute_value)
+        end
+
+        true
+      end
+
+      private
+
+      def error_hash(line, column, value, expected)
+        { line: line, column: column.to_s, value: value, expected: expected }
+      end
+
+      def valid_boolean?(value)
+        value.in? [true, false, "true", "false", "t", "f", 0, 1, "0", "1"]
+      end
+
+      def valid_integer?(value)
+        value.is_a?(Integer) || (value.is_a?(String) && value.to_i.to_s == value)
+      end
+
+      def valid_decimal?(value)
+        BigDecimal(value.to_s)
+      rescue ArgumentError
+        false
+      end
+
+      def valid_datetime?(value)
+        value.is_a?(Date) || value.is_a?(Time) || value.is_a?(DateTime)
+      end
+
+      def valid_time?(value)
+        Time.zone.parse(value.to_s) != nil
+      end
+    end
+  end
+end

data/app/services/imp_exp/exporters/base.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Exporters
+    class Base
+      attr_reader :scoping_parent, :models, :logger
+
+      def initialize(scoping_parent, models, logger: nil)
+        @scoping_parent = scoping_parent
+        @models = models
+        @logger = logger || Logger.new('log/exports.log', 1)
+      end
+
+      def call
+        logger.info "Starting export #{self.class.name} for #{scoping_parent.model_name.singular} " \
+                    "#{scoping_parent.name} (#{scoping_parent.id})"
+        begin
+          export_result = yield
+          logger.info "Export completed\n---------"
+          export_result
+        rescue StandardError => e
+          logger.error "Export interrupted with error : #{e.class.name} #{e.message}"
+          raise e
+        end
+      end
+    end
+  end
+end

data/app/services/imp_exp/exporters/files.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'zip'
+
+module ImpExp
+  module Exporters
+    class Files < Base
+      def call
+        super do
+          tmp_file = Tempfile.new("exporters_files")
+          zip = Zip::OutputStream.write_buffer(tmp_file) do |output_stream|
+            models.each do |model|
+              files = model.files(scoping_parent)
+              next if files.empty?
+
+              files.each do |active_storage_file|
+                full_path = "#{model.files_directory_path}/#{model.filename(active_storage_file)}"
+                add_entry_to_zip(output_stream, active_storage_file, full_path)
+              end
+            end
+          end
+          zip.flush
+          tmp_file.rewind
+          { zip_stream: tmp_file }
+        end
+      end
+
+      private
+
+      def add_entry_to_zip(output_stream, active_storage_file, full_path)
+        output_stream.put_next_entry(full_path)
+        output_stream.write active_storage_file.download
+      rescue ActiveStorage::FileNotFoundError
+        record = active_storage_file.record
+        logger.error "File not found for entity #{record.class} #{record.id} " \
+                     "attribute #{active_storage_file.name}"
+        logger.info "Continuing..."
+      end
+    end
+  end
+end

data/app/services/imp_exp/exporters/xlsx.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'axlsx'
+
+module ImpExp
+  module Exporters
+    class Xlsx < Base
+      def call
+        super do
+          rows_count = 0
+          package = Axlsx::Package.new
+          models.each do |model|
+            worksheet = package.workbook.add_worksheet(name: model.model_name)
+            worksheet.add_row model.headers
+            worksheet.add_row model.headers_translated
+            model.rows(scoping_parent).each do |row|
+              worksheet.add_row row, style: row_style(worksheet, model)
+              rows_count += 1
+            end
+          end
+          { xlsx_stream: package.to_stream, rows_count: rows_count }
+        end
+      end
+
+      private
+
+      def row_style(worksheet, model)
+        {
+          datetime: worksheet.styles.add_style(num_fmt: Axlsx::NUM_FMT_YYYYMMDDHHMMSS)
+        }.values_at(*model.attribute_types)
+      end
+    end
+  end
+end

data/app/services/imp_exp/importers/base.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "observer"
+
+module ImpExp
+  module Importers
+    class Base
+      include Observable
+
+      attr_reader :scoping_parent, :models, :logger, :exception_notifier, :errors, :imported
+
+      ROW_OFFSET = 2
+
+      def initialize(scoping_parent, models, logger: nil, exception_notifier: nil)
+        @scoping_parent = scoping_parent
+        @models = models
+        @logger = logger || Logger.new('log/imports.log', 1)
+        @exception_notifier = exception_notifier
+      end
+
+      def call(data, files_directory_path = nil)
+        @errors = []
+        @imported = data.keys.map(&:model_name).index_with { 0 }
+        logger.info "Starting import #{self.class.name} for #{scoping_parent.model_name.singular} " \
+                    "#{scoping_parent.name} (#{scoping_parent.id})"
+        begin
+          ping(stage: 'validation')
+          import(data, files_directory_path)
+          logger.info "Import completed\n---------"
+          import_result
+        rescue StandardError => e
+          logger.error "Import interrupted with error : #{e.class.name} #{e.message}:"
+          logger.error e.backtrace.join("\n")
+
+          raise e if raise_rescued_standard_error?
+
+          exception_notifier&.call(e)
+          import_result.merge(exception_raised: "#{e.class.name} #{e.message}")
+        end
+      end
+
+      def import(data, files_directory_path)
+        data.each do |model, model_data|
+          ping(forced: true, stage: 'import', model_name: model.model_name)
+          import_model_data(model, model_data, files_directory_path)
+        end
+      end
+
+      def import_result
+        { imported: imported, import_errors: errors }
+      end
+
+      def import_model_data(model, model_data, files_directory_path)
+        model_data.each_with_index do |row, i|
+          line = ROW_OFFSET + i + 1
+          model_name = model.model_name
+
+          relation_attributes = model.relation_names.index_with do |relation_name|
+            relation_attribute(model, relation_name, row[relation_name])
+          end
+
+          relation_attributes = process_errors(relation_attributes, model, line, row.to_s)
+
+          attrs = attributes_from_row(row, model).merge!(relation_attributes)
+
+          record = find_or_initialize_record(row.merge(relation_attributes), scoping_parent, model)
+
+          upsert_no_record_error(row, model, line) && next unless record
+
+          assign_attributes(record, attrs, model_name, line, row)
+
+          record.assign_attributes(model.extra_attributes_imported_at_runtime(scoping_parent, record, row))
+
+          serialize_attributes_if_needed(record, row, model)
+
+          attach_files(record, files_directory_path, model, row, line) if files_directory_path.present?
+
+          if record.save(context: :import)
+            @imported[model.model_name] += 1
+            next
+          end
+
+          upsert_error_messages(model_name, line, row.to_s, record.errors.full_messages)
+        end
+      end
+
+      private
+
+      def raise_rescued_standard_error?
+        Rails.env.test? || Rails.env.development?
+      end
+
+      def attributes_from_row(row, model)
+        row.slice(*model.attribute_to_import_names)
+      end
+
+      def find_or_initialize_record(attributes, scoping_parent, model)
+        record = model.find_record(attributes, scoping_parent)
+        return if !record && model.strict_find_by_attribute && attributes[model.strict_find_by_attribute.to_s].present?
+
+        record || model.active_record_class.new
+      end
+
+      def upsert_no_record_error(row, model, line)
+        conditions = row.slice(model.strict_find_by_attribute).map { |k, v| "#{k}=#{v}" }.join(' ')
+        message = I18n.t('imp_exp.importer.record_not_found', model_name: model.active_record_class.model_name,
+                                                              conditions: conditions)
+        upsert_error_messages(model.model_name, line, row.to_s, message)
+      end
+
+      def assign_attributes(record, attributes, model_name, line, row)
+        record.assign_attributes(attributes)
+      rescue ActiveRecord::RecordNotSaved => e
+        upsert_error_messages(model_name, line, row.to_s, e.message)
+      end
+
+      def relation_attribute(model, relation_name, relation_datum)
+        serializer = model.relation_options(relation_name)[:serializer]
+        serializer.load(relation_datum, model: model, relation_name: relation_name, scoping_parent: scoping_parent)
+      end
+
+      def attach_files(record, files_directory_path, model, row, line)
+        model.file_attribute_names.each do |file_attribute_name|
+          file_path = row[file_attribute_name]
+          next if file_path.blank?
+
+          full_file_path = File.join(files_directory_path, file_path.strip)
+          begin
+            file = File.open(full_file_path)
+            record.send(file_attribute_name).attach(io: file, filename: File.basename(file_path))
+          rescue Errno::ENOENT
+            error_msg = I18n.t('imp_exp.importer.file_not_found', file_path: file_path)
+            upsert_error_messages(model.model_name, line, row.to_s, error_msg)
+          end
+        end
+      end
+
+      def process_errors(relation_attributes, model, line, row_str)
+        relation_attributes.transform_values do |data|
+          if data.is_a?(Array)
+            data.delete_if do |datum|
+              if data_is_an_error?(datum)
+                upsert_error_messages(model.model_name, line, row_str, datum.message)
+                true
+              end
+            end
+          elsif data_is_an_error?(data)
+            upsert_error_messages(model.model_name, line, row_str, data.message)
+            nil
+          else
+            data
+          end
+        end
+      end
+
+      def data_is_an_error?(data)
+        data.is_a?(ImpExp::Serializers::Errors::Base)
+      end
+
+      def upsert_error_messages(model_name, line, row_str, messages)
+        error = @errors.find { |e| e[:model_name] == model_name && e[:line] == line }
+        if error
+          error[:messages] = error[:messages] + Array(messages)
+        else
+          @errors << { model_name: model_name, line: line, row: row_str, messages: Array(messages) }
+        end
+      end
+
+      def serialize_attributes_if_needed(record, row, model)
+        model.attribute_to_import_names.each do |attribute_name|
+          serializer = model.attribute_options(attribute_name)[:serializer]
+          serializer && record.assign_attributes(attribute_name => serializer.load(row[attribute_name],
+                                                                                   model: model,
+                                                                                   record: record,
+                                                                                   scoping_parent: scoping_parent))
+        end
+      end
+
+      def ping(stage: 'import', model_name: nil, forced: false)
+        if forced
+          changed
+          notify_observers({ stage: stage, model_name: model_name })
+        elsif @last_ping.nil? || @last_ping + ::Import::PING_TIMEOUT < Time.zone.now
+          changed
+          notify_observers({ stage: stage, model_name: model_name })
+          @last_ping = Time.zone.now
+        end
+      end
+    end
+  end
+end

data/app/services/imp_exp/importers/xlsx.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Importers
+    class Xlsx < Base
+      attr_reader :xlsx_reader, :format_errors, :data_validation_errors,
+                  :extra_format_validator_klass, :extra_data_validator_klass
+
+      # rubocop : disable Metrics/ParameterLists
+      def initialize(scoping_parent, models, logger: nil, extra_format_validator_klass: nil, exception_notifier: nil,
+                     extra_data_validator_klass: nil)
+        # rubocop : enable Metrics/ParameterLists
+        super(scoping_parent, models, logger: logger, exception_notifier: exception_notifier)
+        @extra_format_validator_klass = extra_format_validator_klass
+        @extra_data_validator_klass = extra_data_validator_klass
+        @format_errors = []
+        @data_validation_errors = []
+      end
+
+      def call(xlsx_stream, files_directory_path = nil)
+        ping(forced: true, stage: "validation")
+        logger.info "validating xlsx import file"
+
+        @xlsx_reader = ImpExp::Utils::XlsxReader.new(xlsx_stream)
+
+        logger.info "validating sheets..."
+        validate_presence_of_sheets(xlsx_reader.sheet_names)
+
+        logger.info "validating columns..."
+        validate_presence_of_columns(xlsx_reader)
+
+        format_errors.concat extra_format_validator_klass.new(self).call if extra_format_validator_klass
+
+        logger.info "validating data..."
+        validate_data(xlsx_reader)
+
+        import_result = if format_errors.none? && data_validation_errors.none?
+                          data = models.index_with { |model| xlsx_reader.sheet_data(model.model_name) }
+                          super(data, files_directory_path)
+                        else
+                          { imported: {}, import_errors: [] }
+                        end
+
+        { format_errors: format_errors, data_validation_errors: data_validation_errors }.merge(import_result)
+      end
+
+      private
+
+      def validate_presence_of_sheets(sheet_names)
+        expected_model_names = models.map(&:model_name).sort
+        return unless sheet_names.sort != expected_model_names
+
+        format_errors << { name: :invalid_sheets,
+                           expected_sheets: expected_model_names,
+                           in_file_sheets: sheet_names.sort }
+      end
+
+      def validate_presence_of_columns(xlsx_reader)
+        models.each do |model|
+          sheet_headers = xlsx_reader.sheet_header_names(model.model_name)
+          model_headers = model.headers
+          next if sheet_headers.sort == model_headers.sort
+
+          format_errors << { name: :invalid_columns, model_name: model.model_name,
+                             expected_columns: model_headers.map(&:to_s),
+                             in_file_columns: sheet_headers.map(&:to_s) }
+        end
+      end
+
+      def validate_data(xlsx_reader)
+        models.select { |m| m.model_name.in? xlsx_reader.sheet_names }.each do |model|
+          data = xlsx_reader.sheet_data(model.model_name)
+          errors = ImpExp::DataValidators::Base.new(model, ROW_OFFSET).call(data)
+          errors.concat extra_data_validator_klass.new(model, ROW_OFFSET).call(data) if extra_data_validator_klass
+          data_validation_errors.concat(errors.map { |err| err.merge(model_name: model.model_name) }) if errors.any?
+        end
+        data_validation_errors.flatten
+      end
+    end
+  end
+end

data/app/services/imp_exp/models/base.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Models
+    class Base
+      def active_record_class
+        "::#{self.class.name.demodulize}".constantize
+      end
+
+      def find_record(attrs, scoping_parent)
+        scoped(scoping_parent).find_by(code: attrs[:code])
+      end
+
+      def strict_find_by_attribute
+        nil
+      end
+
+      def attributes
+        raise NotImplementedError
+      end
+
+      def attribute_names
+        attributes.map(&:first)
+      end
+
+      def attribute_types
+        attribute_names.map { active_record_class.column_for_attribute(_1)&.type }
+      end
+
+      def attribute_to_import_names
+        attribute_names.reject { |attribute_name| attribute_options(attribute_name)[:imported] == false }
+      end
+
+      def extra_attributes_imported_at_runtime(scoping_parent, _record, _row)
+        scoping_parent_fk = scoping_parent_foreign_key_column_name(scoping_parent)
+        if scoping_parent && active_record_class.attribute_names.include?(scoping_parent_fk)
+          return { scoping_parent_fk.to_sym => scoping_parent.id }
+        end
+
+        {}
+      end
+
+      def file_attributes
+        []
+      end
+
+      def file_attribute_names
+        file_attributes.map(&:first)
+      end
+
+      def relations
+        []
+      end
+
+      def relation_names
+        relations.map(&:first)
+      end
+
+      def scoping_parent_foreign_key_column_name(scoping_parent)
+        "#{scoping_parent.model_name.singular}_id"
+      end
+
+      def scoped(scoping_parent)
+        active_record_class.rewhere({ scoping_parent_foreign_key_column_name(scoping_parent) => scoping_parent.id })
+      end
+
+      def preload_associations(collection)
+        collection
+      end
+
+      def order(collection)
+        collection.order(:created_at)
+      end
+
+      def model_name
+        self.class.name.demodulize
+      end
+
+      def relation_options(relation_name)
+        relations.find { |relation| relation[0] == relation_name.to_sym }&.slice(1) || {}
+      end
+
+      def attribute_options(attribute_name)
+        attributes.find { |attribute| attribute[0] == attribute_name.to_sym }&.slice(1) || {}
+      end
+
+      def headers_translated
+        (attributes + relations).map do |name, opts = {}|
+          i18n_path = opts[:i18n_path].presence || "#{active_record_class.model_name.i18n_key}.#{name}"
+          key = "activerecord.attributes.#{i18n_path}"
+          I18n.t(key, raise: true)
+        end
+      end
+
+      def headers
+        attribute_names + relation_names + file_attribute_names
+      end
+
+      def rows(scoping_parent)
+        rows = []
+        preload_associations(order(scoped(scoping_parent))).each do |record|
+          data = record.slice(attribute_names)
+          attribute_names.each do |attribute_name|
+            serializer = attribute_options(attribute_name)[:serializer]
+            data[attribute_name] = serializer.dump(record, attribute_name) if serializer
+          end
+          data = data.values
+          relation_names.each do |relation_name|
+            serializer = relation_options(relation_name)[:serializer]
+            data << serializer.dump(record, relation_name)
+          end
+          data.concat(file_attribute_names.map do |file_attribute_name|
+            file = record.send(file_attribute_name)
+            "#{files_directory_path}/#{filename(file)}" if file.attached?
+          end)
+          rows << data
+        end
+        rows
+      end
+
+      def files(scoping_parent)
+        scoped(scoping_parent).flat_map do |record|
+          record.slice(file_attribute_names).values.select(&:attached?)
+        end
+      end
+
+      def files_directory_path
+        active_record_class.model_name.element
+      end
+
+      def filename(active_storage_file)
+        "#{active_storage_file.name}-#{active_storage_file.record.code}#{file_extension(active_storage_file)}"
+      end
+
+      private
+
+      def file_extension(active_storage_file)
+        active_storage_file.filename.extension_with_delimiter.presence ||
+          ".#{Marcel::EXTENSIONS.invert[active_storage_file.content_type]}"
+      end
+    end
+  end
+end

data/app/services/imp_exp/ping_observer.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module ImpExp
+  class PingObserver
+    attr_reader :import_service, :active_record_import
+
+    def initialize(import_service, active_record_import)
+      @import_service = import_service
+      @active_record_import = active_record_import
+    end
+
+    def setup
+      import_service.importers.each do |importer|
+        importer.add_observer(self)
+      end
+    end
+
+    def update(data_hash)
+      active_record_import.ping(**data_hash)
+    end
+  end
+end

data/app/services/imp_exp/serializers/array_like.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class ArrayLike
+      SEPARATOR = "|"
+
+      def dump(record, attribute_name)
+        record.send(attribute_name)&.join(SEPARATOR)
+      end
+
+      # rubocop: disable Lint/UnusedMethodArgument
+      def load(data, model: nil, record: nil, scoping_parent: nil)
+        return [] if data.blank?
+
+        data.to_s.split(SEPARATOR).map(&:strip)
+      end
+      # rubocop: enable Lint/UnusedMethodArgument
+    end
+  end
+end

data/app/services/imp_exp/serializers/errors/base.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# rubocop : disable Lint/EmptyClass
+module ImpExp
+  module Serializers
+    module Errors
+      class Base
+      end
+    end
+  end
+end
+# rubocop : enable Lint/EmptyClass

data/app/services/imp_exp/serializers/errors/record_not_found.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    module Errors
+      class RecordNotFound < Base
+        attr_reader :active_record_class, :find_args
+
+        def initialize(active_record_class, find_args)
+          super()
+          @active_record_class = active_record_class
+          @active_record_class = @active_record_class.constantize if @active_record_class.is_a?(String)
+          @find_args = find_args&.symbolize_keys
+        end
+
+        def message
+          conditions = find_args.map { |k, v| "#{k}=#{v}" }.join(' ')
+          I18n.t('imp_exp.importer.record_not_found', model_name: active_record_class.model_name,
+                                                      conditions: conditions)
+        end
+      end
+    end
+  end
+end

data/app/services/imp_exp/serializers/errors/wrong_format.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    module Errors
+      class WrongFormat < Base
+        attr_reader :raw_message
+
+        def initialize(raw_message = nil)
+          super()
+          @raw_message = raw_message
+        end
+
+        def message
+          I18n.t('imp_exp.importer.wrong_format', message: raw_message)
+        end
+      end
+    end
+  end
+end

data/app/services/imp_exp/serializers/polymorphic_relation_one_to_one.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class PolymorphicRelationOneToOne
+      attr_reader :column, :relation_classes
+
+      def initialize(column, relation_classes)
+        @column = column
+        @relation_classes = relation_classes
+      end
+
+      def dump(record, relation_name)
+        relation = record.send(relation_name)
+        value = relation&.send(column)
+
+        "#{relation.model_name.name}|#{value}" if relation && value.present?
+      end
+
+      # rubocop: disable Lint/UnusedMethodArgument
+      def load(data, scoping_parent:, model: nil, relation_name: nil)
+        return if data.blank?
+
+        relation_class, value = data.strip.split("|")
+
+        return if relation_class.blank? || value.blank? || !relation_class.in?(relation_classes)
+
+        record = ImpExp::Utils::RelationScoper.call(relation_class.constantize, relation_class, scoping_parent)
+                                              .find_by({ column => value })
+        return record if record
+
+        ImpExp::Serializers::Errors::RecordNotFound.new(relation_class, { column => value })
+      end
+      # rubocop: enable Lint/UnusedMethodArgument
+    end
+  end
+end

data/app/services/imp_exp/serializers/relation_one_to_many.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class RelationOneToMany
+      SEPARATOR = ','
+
+      attr_reader :columns, :class_name
+
+      def initialize(*columns, class_name: nil)
+        @columns = columns
+        @class_name = class_name
+      end
+
+      def dump(record, relation_name)
+        records = record.send(relation_name)
+        records.map do |r|
+          columns.map do |column|
+            r.send(column)
+          end.join(COMPOSITE_IDENTIFIER_SEPARATOR)
+        end
+        .join(SEPARATOR)
+      end
+
+      def load(data, model:, relation_name:, scoping_parent:)
+        values_array = data.to_s.split(SEPARATOR).map do |chunk|
+          chunk.strip.split(COMPOSITE_IDENTIFIER_SEPARATOR).map(&:strip)
+        end
+
+        relation_class = class_name || relation_class_from_reflection(model, relation_name)
+        relation_class = relation_class.to_s.constantize
+
+        if values_array.empty?
+          records = relation_class.none
+        else
+          records = relation_class.where(columns.zip(values_array[0]).to_h)
+        end
+
+        values_array.uniq!
+
+        values_array[1..]&.each do |values|
+          records = records.or(relation_class.where(columns.zip(values).to_h))
+        end
+
+        records = Utils::RelationScoper.call(records, relation_class, scoping_parent)
+
+        values_array.map do |values|
+          record = records.find { |r| r.slice(columns).values.map(&:to_s) == values }
+          record || ImpExp::Serializers::Errors::RecordNotFound.new(relation_class, columns.zip(values).to_h)
+        end
+      end
+
+      private
+
+      def relation_class_from_reflection(model, relation_name)
+        reflection = model.active_record_class.reflections[relation_name.to_s]
+        reflection.options[:class_name] || reflection.name.to_s.classify
+      end
+    end
+  end
+end

data/app/services/imp_exp/serializers/relation_one_to_one.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class RelationOneToOne
+      attr_reader :columns
+
+      def initialize(*columns)
+        @columns = columns
+      end
+
+      def dump(record, relation_name)
+        columns.map do |column|
+          record.send(relation_name)&.send(column)
+        end.join(COMPOSITE_IDENTIFIER_SEPARATOR)
+      end
+
+      def load(data, model:, relation_name:, scoping_parent:)
+        return if data.blank?
+
+        relation_class = model.active_record_class.reflections[relation_name.to_s].options[:class_name]
+        relation_class ||= relation_name.to_s.classify
+        relation_class = relation_class.constantize
+
+        data = data.to_s.split(COMPOSITE_IDENTIFIER_SEPARATOR).map(&:strip)
+
+        conditions = columns.zip(data).to_h
+
+        query_scoped = Utils::RelationScoper.call(relation_class.all, relation_class, scoping_parent)
+
+        record = query_scoped.find_by(conditions)
+        return record if record
+
+        ImpExp::Serializers::Errors::RecordNotFound.new(relation_class, conditions)
+      end
+    end
+  end
+end

data/app/services/imp_exp/serializers/time.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class Time
+      def dump(record, attribute_name)
+        record.send(attribute_name)&.strftime("%H:%M:%S")
+      end
+
+      # rubocop: disable Lint/UnusedMethodArgument
+      def load(data, model: nil, record: nil, scoping_parent: nil)
+        data&.strip
+      end
+      # rubocop: enable Lint/UnusedMethodArgument
+    end
+  end
+end

data/app/services/imp_exp/serializers/to_stable_uuid.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class ToStableUuid
+      # ce serializer permet de transformer un uuid en un autre uuid
+      # de manière stable, c'est à dire qu'appeler 2 fois le serializer
+      # avec le même argument (même uuid) donnera toujours le même uuid en
+      # résultat du dump
+
+      def dump(record, attribute_name)
+        value = record.send(attribute_name)
+        return if value.blank?
+
+        Digest::MD5.hexdigest(value).gsub(/(.{8})(.{4})(.{4})(.{4})(.{12})/, '\1-\2-\3-\4-\5')
+      end
+
+      # rubocop: disable Lint/UnusedMethodArgument
+      def load(data, model: nil, record: nil, scoping_parent: nil)
+        data&.strip
+      end
+      # rubocop: enable Lint/UnusedMethodArgument
+    end
+  end
+end

data/app/services/imp_exp/serializers/whole_records.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    class WholeRecords
+      attr_reader :columns
+
+      def initialize(columns)
+        @columns = columns
+      end
+
+      def dump(record, relation_name)
+        record.send(relation_name).map do |associated_records|
+          associated_records.slice(columns)
+        end.to_json
+      end
+
+      # rubocop: disable Lint/UnusedMethodArgument
+      def load(data, model:, relation_name:, scoping_parent: nil)
+        return [] if data.blank?
+
+        begin
+          data = JSON.parse(data)
+        rescue JSON::ParserError => e
+          message = e.message.gsub("859: ", "") # Ruby 3.1 + 3.2 compatibility
+          return [ImpExp::Serializers::Errors::WrongFormat.new("#{relation_name} error | #{message}")]
+        end
+
+        unless data.is_a?(Array) && data.all? { |datum| datum.is_a?(Hash) }
+          return [ImpExp::Serializers::Errors::WrongFormat.new("#{relation_name} should be an array of hashes")]
+        end
+
+        data = data.map(&:deep_symbolize_keys)
+
+        relation_class = model.active_record_class.reflections[relation_name.to_s].options[:class_name]
+        relation_class ||= relation_name.to_s.classify
+        relation_class = relation_class.constantize
+
+        data.map do |hash|
+          relation_class.new(hash.slice(*columns))
+        end
+      end
+      # rubocop: enable Lint/UnusedMethodArgument
+    end
+  end
+end

data/app/services/imp_exp/service.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module ImpExp
+  class Service
+    def self.call(...)
+      new(...).call
+    end
+  end
+end

data/app/services/imp_exp/utils/relation_scoper.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Utils
+    class RelationScoper < Service
+      attr_reader :records, :relation_class, :scoping_parent
+
+      def initialize(records, relation_class, scoping_parent)
+        super()
+        @records = records
+        @relation_class = relation_class
+        @scoping_parent = scoping_parent
+      end
+
+      def call
+        records.where(id: imp_exp_model.scoped(scoping_parent))
+      end
+
+      private
+
+      def imp_exp_model
+        "ImpExp::Models::#{relation_class}".constantize.new
+      end
+    end
+  end
+end

data/app/services/imp_exp/utils/xlsx_reader.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require 'xsv'
+
+module ImpExp
+  module Utils
+    class XlsxReader
+      attr_reader :workbook, :skip_rows, :parse_headers
+
+      delegate :sheets, to: :workbook
+
+      def initialize(xlsx_stream, skip_rows: 1, parse_headers: true)
+        @workbook = Xsv.open(xlsx_stream, parse_headers: parse_headers)
+        @parse_headers = parse_headers
+        @skip_rows = skip_rows
+      end
+
+      def sheet_names
+        sheets.map(&:name)
+      end
+
+      def sheet_by_name(sheet_name)
+        workbook.sheets_by_name(sheet_name)[0]
+      end
+
+      def sheet_header_names(sheet_name)
+        return [] unless parse_headers
+
+        sheet = sheet_by_name(sheet_name)
+        return [] unless sheet
+
+        sheet[0].keys.compact.map(&:to_sym)
+      end
+
+      def sheet_row_values(sheet_name, row_nb)
+        sheet = sheet_by_name(sheet_name)
+        return [] unless sheet
+
+        data = sheet[row_nb + skip_rows]
+        return data unless parse_headers
+
+        data.values
+      end
+
+      def sheet_data(sheet_name)
+        data = sheet_by_name(sheet_name).entries[skip_rows..]
+
+        return [] if data.nil?
+
+        if parse_headers
+          data.map { |h| h.delete_if { |k, _v| k.nil? }.with_indifferent_access }
+              .delete_if { |h| h.values.all?(&:blank?) }
+        else
+          data.delete_if { |a| a.all?(&:blank?) }
+        end
+      end
+    end
+  end
+end

data/config/locales/en.yml

@@ -0,0 +1,10 @@
+en:
+  imp_exp:
+    importer:
+      invalid_columns: "The %{model_name} sheet does not contain the expected columns. Expected columns: %{expected_columns} Columns in the file: %{in_file_columns}."
+      invalid_sheets: "The file does not contain the expected sheets. Expected sheets: %{expected_sheets} Sheets in the file: %{in_file_sheets}."
+      record_not_found: "%{model_name} with %{conditions} conditions has not been found."
+      wrong_format: "Format error: %{message}."
+      model_errors: "%{model_name} sheet - Row %{line}: %{messages}"
+      data_validation_error: "%{model_name} sheet - Row %{line} - Column \"%{column}\": value \"%{value}\" incorrect, expected value: %{expected}"
+      file_not_found: "%{file_path} file not found."

data/config/locales/fr.yml

@@ -0,0 +1,10 @@
+fr:
+  imp_exp:
+    importer:
+      invalid_columns: "La feuille %{model_name} ne contient pas les colonnes attendues. Colonnes attendues : %{expected_columns}. Colonnes dans le fichier : %{in_file_columns}."
+      invalid_sheets: "Le fichier ne contient pas les feuilles attendues. Feuilles attendues : %{expected_sheets}. Feuilles dans le fichier : %{in_file_sheets}."
+      record_not_found: "%{model_name} avec les conditions %{conditions} n'a pas été trouvé."
+      wrong_format: "Erreur de format : %{message}."
+      model_errors: "Feuille %{model_name} - Ligne %{line} : %{messages}"
+      data_validation_error: "Feuille %{model_name} - Ligne %{line} - Colonne \"%{column}\" : valeur \"%{value}\" incorrecte, valeur attendue : %{expected}"
+      file_not_found: "Fichier %{file_path} non trouvé."

data/lib/imp_exp/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module ImpExp
+  VERSION = "1.0.0"
+end

data/lib/imp_exp.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module ImpExp
+  module Serializers
+    COMPOSITE_IDENTIFIER_SEPARATOR = '|'
+  end
+
+  class Engine < ::Rails::Engine
+  end
+end

data/lib/tasks/imp_exp.rake

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+namespace :imp_exp do
+  task generate_account_export: :environment do
+    xlsx_stream = ImpExp::Exports::Account.new(Account.find_by!(code: "C1")).call[:xlsx_stream]
+    File.binwrite('../../account_export.xlsx', xlsx_stream.read)
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: imp_exp
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Nicolas Florentin
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-10-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: caxlsx
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.4'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 7.0.5
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.3.2
+- !ruby/object:Gem::Dependency
+  name: xsv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Import data from xlsx into a rails app and export data to xlsx from a
+  rails app
+email:
+- nicolas@sleede.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- MIT-LICENSE
+- README.md
+- Rakefile
+- app/services/imp_exp/data_validators/base.rb
+- app/services/imp_exp/exporters/base.rb
+- app/services/imp_exp/exporters/files.rb
+- app/services/imp_exp/exporters/xlsx.rb
+- app/services/imp_exp/importers/base.rb
+- app/services/imp_exp/importers/xlsx.rb
+- app/services/imp_exp/models/base.rb
+- app/services/imp_exp/ping_observer.rb
+- app/services/imp_exp/serializers/array_like.rb
+- app/services/imp_exp/serializers/errors/base.rb
+- app/services/imp_exp/serializers/errors/record_not_found.rb
+- app/services/imp_exp/serializers/errors/wrong_format.rb
+- app/services/imp_exp/serializers/polymorphic_relation_one_to_one.rb
+- app/services/imp_exp/serializers/relation_one_to_many.rb
+- app/services/imp_exp/serializers/relation_one_to_one.rb
+- app/services/imp_exp/serializers/time.rb
+- app/services/imp_exp/serializers/to_stable_uuid.rb
+- app/services/imp_exp/serializers/whole_records.rb
+- app/services/imp_exp/service.rb
+- app/services/imp_exp/utils/relation_scoper.rb
+- app/services/imp_exp/utils/xlsx_reader.rb
+- config/locales/en.yml
+- config/locales/fr.yml
+- lib/imp_exp.rb
+- lib/imp_exp/version.rb
+- lib/tasks/imp_exp.rake
+homepage:
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Import data from xlsx into a rails app and export data to xlsx from a rails
+  app
+test_files: []