data_porter 0.5.0 → 0.9.0

data/lib/data_porter/components/shared/pagination.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module Components
+    module Shared
+      class Pagination < Base
+        def initialize(page:, total_pages:, base_url:)
+          super()
+          @page = page
+          @total_pages = total_pages
+          @base_url = base_url
+        end
+
+        def view_template
+          return if @total_pages <= 1
+
+          nav(class: "dp-pagination") do
+            render_prev
+            render_indicator
+            render_next
+          end
+        end
+
+        private
+
+        def render_prev
+          if @page > 1
+            a(href: page_url(@page - 1), class: "dp-pagination__btn") { "\u2190 Previous" }
+          else
+            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { "\u2190 Previous" }
+          end
+        end
+
+        def render_indicator
+          span(class: "dp-pagination__info") { "Page #{@page} of #{@total_pages}" }
+        end
+
+        def render_next
+          if @page < @total_pages
+            a(href: page_url(@page + 1), class: "dp-pagination__btn") { "Next \u2192" }
+          else
+            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { "Next \u2192" }
+          end
+        end
+
+        def page_url(number)
+          separator = @base_url.include?("?") ? "&" : "?"
+          "#{@base_url}#{separator}page=#{number}#records"
+        end
+      end
+    end
+  end
+end

data/lib/data_porter/components.rb

@@ -3,6 +3,7 @@
 require_relative "components/base"
 require_relative "components/shared/status_badge"
 require_relative "components/shared/failure_alert"
+require_relative "components/shared/pagination"
 require_relative "components/preview/table"
 require_relative "components/preview/summary_cards"
 require_relative "components/preview/results_summary"

data/lib/data_porter/dsl/param.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module DSL
+    VALID_PARAM_TYPES = %i[select text number hidden].freeze
+
+    Param = Struct.new(:name, :type, :required, :label, :collection, :default, :options, keyword_init: true) do
+      def initialize(**attrs)
+        validate_type!(attrs[:type] || :text)
+        extra = attrs.except(:name, :type, :required, :label, :collection, :default, :options)
+        apply_defaults!(attrs, extra)
+        super(**attrs.slice(:name, :type, :required, :label, :collection, :default, :options))
+      end
+
+      private
+
+      def apply_defaults!(attrs, extra)
+        attrs[:name] = attrs[:name].to_sym
+        attrs[:type] = (attrs[:type] || :text).to_sym
+        attrs[:required] = attrs.fetch(:required, false)
+        attrs[:label] ||= attrs[:name].to_s.humanize
+        attrs[:options] = extra
+      end
+
+      def validate_type!(type)
+        return if VALID_PARAM_TYPES.include?(type.to_sym)
+
+        raise ArgumentError, "invalid param type: #{type}. Must be one of: #{VALID_PARAM_TYPES.join(", ")}"
+      end
+    end
+  end
+end

data/lib/data_porter/engine.rb

@@ -10,6 +10,10 @@ module DataPorter
           data_porter/application.css
           data_porter/turbo.min.js
           data_porter/stimulus.min.js
+          data_porter/mapping_controller.js
+          data_porter/template_form_controller.js
+          data_porter/progress_controller.js
+          data_porter/import_form_controller.js
         ]
       end
     end

data/lib/data_porter/orchestrator/dry_runner.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class Orchestrator
+    module DryRunner
+      private
+
+      def run_dry_run_records
+        records = @data_import.records
+        importable = records.select(&:importable?)
+        context = build_context
+
+        importable.each do |record|
+          dry_run_record(record, context)
+        end
+
+        @data_import.records_will_change!
+        @data_import.update!(records: records)
+      end
+
+      def dry_run_record(record, context)
+        @target.persist(record, context: context)
+        record.dry_run_passed = true
+      rescue StandardError => e
+        record.dry_run_passed = false
+        record.add_error(e.message)
+      end
+    end
+  end
+end

data/lib/data_porter/orchestrator/importer.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class Orchestrator
+    module Importer
+      private
+
+      def import_records
+        importable = @data_import.importable_records
+        context = build_context
+        results = { created: 0, errored: 0 }
+        total = importable.size
+
+        importable.each_with_index do |record, index|
+          persist_record(record, context, results)
+          broadcast_progress(index + 1, total)
+        end
+
+        @data_import.update!(status: :completed)
+        @broadcaster.success
+        results
+      end
+
+      def persist_record(record, context, results)
+        @target.persist(record, context: context)
+        results[:created] += 1
+      rescue StandardError => e
+        record.add_error(e.message)
+        @target.on_error(record, e, context: context)
+        results[:errored] += 1
+      end
+
+      def update_import_report(results)
+        report = @data_import.report || StoreModels::Report.new
+        report.imported_count = results[:created]
+        report.errored_count = results[:errored]
+        @data_import.update!(report: report)
+      end
+    end
+  end
+end

data/lib/data_porter/orchestrator/record_builder.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class Orchestrator
+    module RecordBuilder
+      private
+
+      def build_records
+        source = build_source
+        raw_rows = source.fetch
+        columns = @target.class._columns || []
+        validator = RecordValidator.new(columns)
+
+        raw_rows.each_with_index.map do |row, index|
+          build_record(row, index, columns, validator)
+        end
+      end
+
+      def build_record(row, index, columns, validator)
+        record = StoreModels::ImportRecord.new(
+          line_number: index + 1,
+          data: extract_data(row, columns)
+        )
+        record = @target.transform(record)
+        @target.validate(record)
+        validator.validate(record)
+        record.determine_status!
+        record
+      end
+
+      def extract_data(row, columns)
+        columns.each_with_object({}) do |col, hash|
+          hash[col.name] = row[col.name] || row[col.name.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/data_porter/orchestrator.rb

@@ -1,10 +1,18 @@
 # frozen_string_literal: true
 
+require_relative "orchestrator/record_builder"
+require_relative "orchestrator/importer"
+require_relative "orchestrator/dry_runner"
+
 module DataPorter
   class Orchestrator
+    include RecordBuilder
+    include Importer
+    include DryRunner
+
     def initialize(data_import, content: nil)
       @data_import = data_import
-      @target = data_import.target_class.new
+      @target = build_target(data_import)
       @broadcaster = Broadcaster.new(data_import.id)
       @source_options = { content: content }.compact
     end
@@ -48,6 +56,13 @@ module DataPorter
 
     private
 
+    def build_target(data_import)
+      target = data_import.target_class.new
+      config = data_import.config || {}
+      target.import_params = config["import_params"] || {}
+      target
+    end
+
     def build_source
       @data_import.source_class.new(@data_import, **@source_options)
     end
@@ -58,88 +73,6 @@ module DataPorter
       @data_import.update!(config: config)
     end
 
-    def build_records
-      source = build_source
-      raw_rows = source.fetch
-      columns = @target.class._columns || []
-      validator = RecordValidator.new(columns)
-
-      raw_rows.each_with_index.map do |row, index|
-        build_record(row, index, columns, validator)
-      end
-    end
-
-    def build_record(row, index, columns, validator)
-      record = StoreModels::ImportRecord.new(
-        line_number: index + 1,
-        data: extract_data(row, columns)
-      )
-      record = @target.transform(record)
-      @target.validate(record)
-      validator.validate(record)
-      record.determine_status!
-      record
-    end
-
-    def extract_data(row, columns)
-      columns.each_with_object({}) do |col, hash|
-        hash[col.name] = row[col.name] || row[col.name.to_s]
-      end
-    end
-
-    def run_dry_run_records
-      records = @data_import.records
-      importable = records.select(&:importable?)
-      context = build_context
-
-      importable.each do |record|
-        dry_run_record(record, context)
-      end
-
-      @data_import.records_will_change!
-      @data_import.update!(records: records)
-    end
-
-    def dry_run_record(record, context)
-      @target.persist(record, context: context)
-      record.dry_run_passed = true
-    rescue StandardError => e
-      record.dry_run_passed = false
-      record.add_error(e.message)
-    end
-
-    def import_records
-      importable = @data_import.importable_records
-      context = build_context
-      results = { created: 0, errored: 0 }
-      total = importable.size
-
-      importable.each_with_index do |record, index|
-        persist_record(record, context, results)
-        broadcast_progress(index + 1, total)
-      end
-
-      @data_import.update!(status: :completed)
-      @broadcaster.success
-      results
-    end
-
-    def persist_record(record, context, results)
-      @target.persist(record, context: context)
-      results[:created] += 1
-    rescue StandardError => e
-      record.add_error(e.message)
-      @target.on_error(record, e, context: context)
-      results[:errored] += 1
-    end
-
-    def update_import_report(results)
-      report = @data_import.report || StoreModels::Report.new
-      report.imported_count = results[:created]
-      report.errored_count = results[:errored]
-      @data_import.update!(report: report)
-    end
-
     def build_report
       summary = @data_import.records_summary
       report = StoreModels::Report.new(

data/lib/data_porter/registry.rb

@@ -21,7 +21,8 @@ module DataPorter
             key: key,
             label: klass._label,
             icon: klass._icon,
-            sources: klass._sources || DataPorter.configuration.enabled_sources
+            sources: klass._sources || DataPorter.configuration.enabled_sources,
+            params: serialize_params(klass._params)
           }
         end
       end
@@ -33,6 +34,25 @@ module DataPorter
       def clear
         @targets = {}
       end
+
+      private
+
+      def serialize_params(params)
+        return [] unless params
+
+        params.map { |p| serialize_param(p) }
+      end
+
+      def serialize_param(param)
+        {
+          name: param.name,
+          type: param.type,
+          required: param.required,
+          label: param.label,
+          default: param.default,
+          collection: param.collection&.call
+        }.compact
+      end
     end
   end
 end

data/lib/data_porter/target.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "dsl/column"
+require_relative "dsl/param"
 require_relative "dsl/api_config"
 
 module DataPorter
@@ -8,7 +9,7 @@ module DataPorter
     class << self
       attr_reader :_label, :_model_name, :_icon, :_sources,
                   :_columns, :_csv_mappings, :_dedup_keys, :_json_root,
-                  :_api_config, :_dry_run_enabled
+                  :_api_config, :_dry_run_enabled, :_params
 
       def label(value)
         @_label = value
@@ -62,6 +63,15 @@ module DataPorter
         @_dry_run_enabled = true
       end
 
+      def params(&)
+        @_params = []
+        instance_eval(&)
+      end
+
+      def param(name, **)
+        @_params << DSL::Param.new(name: name, **)
+      end
+
       private
 
       def auto_register
@@ -72,6 +82,12 @@ module DataPorter
       end
     end
 
+    attr_accessor :import_params
+
+    def initialize
+      @import_params = {}
+    end
+
     def transform(record)
       record
     end

data/lib/data_porter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataPorter
-  VERSION = "0.5.0"
+  VERSION = "0.9.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_porter
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.9.0
 platform: ruby
 authors:
 - Seryl Lounis
@@ -101,7 +101,11 @@ files:
 - README.md
 - ROADMAP.md
 - Rakefile
+- app/assets/javascripts/data_porter/import_form_controller.js
+- app/assets/javascripts/data_porter/mapping_controller.js
+- app/assets/javascripts/data_porter/progress_controller.js
 - app/assets/javascripts/data_porter/stimulus.min.js
+- app/assets/javascripts/data_porter/template_form_controller.js
 - app/assets/javascripts/data_porter/turbo.min.js
 - app/assets/stylesheets/data_porter/alerts.css
 - app/assets/stylesheets/data_porter/application.css
@@ -115,11 +119,11 @@ files:
 - app/assets/stylesheets/data_porter/progress.css
 - app/assets/stylesheets/data_porter/table.css
 - app/channels/data_porter/import_channel.rb
+- app/controllers/data_porter/concerns/import_validation.rb
+- app/controllers/data_porter/concerns/mapping_management.rb
+- app/controllers/data_porter/concerns/record_pagination.rb
 - app/controllers/data_porter/imports_controller.rb
 - app/controllers/data_porter/mapping_templates_controller.rb
-- app/javascript/data_porter/mapping_controller.js
-- app/javascript/data_porter/progress_controller.js
-- app/javascript/data_porter/template_form_controller.js
 - app/jobs/data_porter/dry_run_job.rb
 - app/jobs/data_porter/extract_headers_job.rb
 - app/jobs/data_porter/import_job.rb
@@ -137,6 +141,7 @@ files:
 - config/routes.rb
 - docs/CONFIGURATION.md
 - docs/MAPPING.md
+- docs/ROADMAP.md
 - docs/SOURCES.md
 - docs/TARGETS.md
 - docs/screenshots/index-with-previewing.jpg
@@ -156,12 +161,17 @@ files:
 - lib/data_porter/components/preview/table.rb
 - lib/data_porter/components/progress/bar.rb
 - lib/data_porter/components/shared/failure_alert.rb
+- lib/data_porter/components/shared/pagination.rb
 - lib/data_porter/components/shared/status_badge.rb
 - lib/data_porter/configuration.rb
 - lib/data_porter/dsl/api_config.rb
 - lib/data_porter/dsl/column.rb
+- lib/data_porter/dsl/param.rb
 - lib/data_porter/engine.rb
 - lib/data_porter/orchestrator.rb
+- lib/data_porter/orchestrator/dry_runner.rb
+- lib/data_porter/orchestrator/importer.rb
+- lib/data_porter/orchestrator/record_builder.rb
 - lib/data_porter/record_validator.rb
 - lib/data_porter/registry.rb
 - lib/data_porter/sources.rb