data_porter 2.0.0 → 2.3.0

data/lib/data_porter/components/preview/summary_cards.rb

@@ -11,10 +11,11 @@ module DataPorter
 
         def view_template
           div(class: "dp-summary-cards") do
-            card("dp-card--complete", @report.complete_count, "Ready")
-            card("dp-card--partial", @report.partial_count, "Incomplete")
-            card("dp-card--missing", @report.missing_count, "Missing")
-            card("dp-card--duplicate", @report.duplicate_count, "Duplicates")
+            card("dp-card--complete", @report.complete_count, I18n.t("data_porter.components.summary_cards.ready"))
+            card("dp-card--partial", @report.partial_count, I18n.t("data_porter.components.summary_cards.incomplete"))
+            card("dp-card--missing", @report.missing_count, I18n.t("data_porter.components.summary_cards.missing"))
+            card("dp-card--duplicate", @report.duplicate_count,
+                 I18n.t("data_porter.components.summary_cards.duplicates"))
           end
         end
 

data/lib/data_porter/components/preview/table.rb

@@ -24,10 +24,10 @@ module DataPorter
         def render_header
           thead do
             tr do
-              th { "#" }
-              th { "Status" }
+              th { I18n.t("data_porter.components.table.line_number") }
+              th { I18n.t("data_porter.components.table.status") }
               @columns.each { |col| th { col.label } }
-              th { "Errors" }
+              th { I18n.t("data_porter.components.table.errors") }
             end
           end
         end

data/lib/data_porter/components/progress/bar.rb

@@ -21,7 +21,7 @@ module DataPorter
 
         def render_label
           div(class: "dp-progress-label") do
-            span(data_data_porter__progress_target: "label") { "Waiting..." }
+            span(data_data_porter__progress_target: "label") { I18n.t("data_porter.components.progress.waiting") }
           end
         end
 
@@ -36,11 +36,18 @@ module DataPorter
         def stimulus_attrs
           attrs = {
             data_controller: "data-porter--progress",
-            data_data_porter__progress_id_value: @import_id.to_s
+            data_data_porter__progress_id_value: @import_id.to_s,
+            data_data_porter__progress_labels_value: progress_labels.to_json
           }
           attrs[:data_data_porter__progress_url_value] = @status_url if @status_url
           attrs
         end
+
+        def progress_labels
+          %w[pending extracting_headers parsing importing dry_running processing].index_with do |key|
+            I18n.t("data_porter.components.progress.#{key}")
+          end
+        end
       end
     end
   end

data/lib/data_porter/components/shared/pagination.rb

@@ -24,22 +24,26 @@ module DataPorter
         private
 
         def render_prev
+          label = "\u2190 #{I18n.t("data_porter.components.pagination.previous")}"
           if @page > 1
-            a(href: page_url(@page - 1), class: "dp-pagination__btn") { "\u2190 Previous" }
+            a(href: page_url(@page - 1), class: "dp-pagination__btn") { label }
           else
-            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { "\u2190 Previous" }
+            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { label }
           end
         end
 
         def render_indicator
-          span(class: "dp-pagination__info") { "Page #{@page} of #{@total_pages}" }
+          span(class: "dp-pagination__info") do
+            I18n.t("data_porter.components.pagination.page_info", page: @page, total: @total_pages)
+          end
         end
 
         def render_next
+          label = "#{I18n.t("data_porter.components.pagination.next")} \u2192"
           if @page < @total_pages
-            a(href: page_url(@page + 1), class: "dp-pagination__btn") { "Next \u2192" }
+            a(href: page_url(@page + 1), class: "dp-pagination__btn") { label }
           else
-            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { "Next \u2192" }
+            span(class: "dp-pagination__btn dp-pagination__btn--disabled") { label }
           end
         end
 

data/lib/data_porter/components/shared/status_badge.rb

@@ -10,7 +10,9 @@ module DataPorter
         end
 
         def view_template
-          span(class: "dp-badge dp-badge--#{@status}") { @status.capitalize }
+          span(class: "dp-badge dp-badge--#{@status}") do
+            I18n.t("data_porter.components.status_badge.#{@status}", default: @status.capitalize)
+          end
         end
       end
     end

data/lib/data_porter/configuration.rb

@@ -13,7 +13,8 @@ module DataPorter
                   :purge_after,
                   :max_file_size,
                   :max_records,
-                  :transaction_mode
+                  :transaction_mode,
+                  :webhook_secret
 
     def initialize
       @parent_controller = "ActionController::Base"
@@ -28,6 +29,7 @@ module DataPorter
       @max_file_size = 10.megabytes
       @max_records = 10_000
       @transaction_mode = :per_record
+      @webhook_secret = nil
     end
   end
 end

data/lib/data_porter/dsl/column.rb

@@ -2,13 +2,14 @@
 
 module DataPorter
   module DSL
-    Column = Struct.new(:name, :type, :required, :label, :options, keyword_init: true) do
-      def initialize(name:, type: :string, required: false, label: nil, **options)
+    Column = Struct.new(:name, :type, :required, :label, :transform, :options, keyword_init: true) do
+      def initialize(name:, type: :string, required: false, label: nil, transform: [], **options)
         super(
           name: name.to_sym,
           type: type.to_sym,
           required: required,
           label: label || name.to_s.humanize,
+          transform: Array(transform),
           options: options
         )
       end

data/lib/data_porter/dsl/webhook.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module DSL
+    VALID_WEBHOOK_EVENTS = %i[started completed failed parsed].freeze
+
+    Webhook = Struct.new(:url, :events, :headers, :payload, keyword_init: true) do
+      def initialize(url:, events: VALID_WEBHOOK_EVENTS, headers: {}, payload: nil)
+        validate_url!(url)
+        events = events.map(&:to_sym)
+        validate_events!(events)
+        super
+      end
+
+      private
+
+      def validate_url!(url)
+        raise ArgumentError, "url is required" if url.nil? || url.to_s.strip.empty?
+      end
+
+      def validate_events!(events)
+        events.each do |event|
+          next if VALID_WEBHOOK_EVENTS.include?(event)
+
+          raise ArgumentError,
+                "invalid webhook event: #{event}. Must be one of: #{VALID_WEBHOOK_EVENTS.join(", ")}"
+        end
+      end
+    end
+  end
+end

data/lib/data_porter/engine.rb

@@ -4,6 +4,10 @@ module DataPorter
   class Engine < ::Rails::Engine
     isolate_namespace DataPorter
 
+    initializer "data_porter.i18n" do
+      config.i18n.load_path += Dir[root.join("config/locales/**/*.yml")]
+    end
+
     initializer "data_porter.assets.precompile" do |app|
       if app.config.respond_to?(:assets)
         app.config.assets.precompile += %w[

data/lib/data_porter/orchestrator/importer.rb

@@ -45,6 +45,7 @@ module DataPorter
       def finalize_import(results)
         @data_import.update!(status: :completed)
         @broadcaster.success
+        WebhookNotifier.notify(@data_import, "import.completed")
         results
       end
 

data/lib/data_porter/orchestrator/record_builder.rb

@@ -22,7 +22,7 @@ module DataPorter
         return unless max
         return if count <= max
 
-        raise Error, "File contains #{count} records, exceeds maximum of #{max}"
+        raise Error, I18n.t("data_porter.errors.max_records", count: count, max: max)
       end
 
       def build_record(row, index, columns, validator)
@@ -30,6 +30,7 @@ module DataPorter
           line_number: index + 1,
           data: extract_data(row, columns)
         )
+        ColumnTransformer.apply_all(record, columns)
         record = @target.transform(record)
         @target.validate(record)
         validator.validate(record)

data/lib/data_porter/orchestrator.rb

@@ -32,12 +32,14 @@ module DataPorter
       records = build_records
       @data_import.update!(records: records, status: :previewing)
       build_report
+      WebhookNotifier.notify(@data_import, "import.parsed")
     rescue StandardError => e
       handle_failure(e)
     end
 
     def import!
       @data_import.importing!
+      WebhookNotifier.notify(@data_import, "import.started")
       results = import_records
       update_import_report(results)
       @target.after_import(results, context: build_context)
@@ -102,6 +104,7 @@ module DataPorter
       )
       @data_import.update!(status: :failed, report: report)
       @broadcaster.failure(error.message)
+      WebhookNotifier.notify(@data_import, "import.failed")
     end
   end
 end

data/lib/data_porter/record_validator.rb

@@ -19,14 +19,14 @@ module DataPorter
     def validate_required(record, col, value)
       return unless col.required && value.to_s.strip.empty?
 
-      record.add_error("#{col.label} is required")
+      record.add_error(I18n.t("data_porter.errors.required", label: col.label))
     end
 
     def validate_type(record, col, value)
       return if value.to_s.strip.empty?
       return if TypeValidator.valid?(value, col.type, col.options)
 
-      record.add_error("#{col.label}: invalid #{col.type}")
+      record.add_error(I18n.t("data_porter.errors.invalid_type", label: col.label, type: col.type))
     end
   end
 end

data/lib/data_porter/target.rb

@@ -3,13 +3,14 @@
 require_relative "dsl/column"
 require_relative "dsl/param"
 require_relative "dsl/api_config"
+require_relative "dsl/webhook"
 
 module DataPorter
   class Target
     class << self
       attr_reader :_label, :_model_name, :_icon, :_sources,
                   :_columns, :_csv_mappings, :_dedup_keys, :_json_root,
-                  :_api_config, :_dry_run_enabled, :_params
+                  :_api_config, :_dry_run_enabled, :_params, :_webhooks
 
       def label(value)
         @_label = value
@@ -72,6 +73,15 @@ module DataPorter
         @_params << DSL::Param.new(name: name, **)
       end
 
+      def webhooks(&)
+        @_webhooks = []
+        instance_eval(&)
+      end
+
+      def webhook(url, **)
+        @_webhooks << DSL::Webhook.new(url: url, **)
+      end
+
       private
 
       def auto_register

data/lib/data_porter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataPorter
-  VERSION = "2.0.0"
+  VERSION = "2.3.0"
 end

data/lib/data_porter/webhook_notifier.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module WebhookNotifier
+    EVENT_MAP = {
+      "import.started" => :started,
+      "import.completed" => :completed,
+      "import.failed" => :failed,
+      "import.parsed" => :parsed
+    }.freeze
+
+    module_function
+
+    def notify(data_import, event)
+      webhooks = resolve_webhooks(data_import)
+      return if webhooks.nil? || webhooks.empty?
+
+      event_sym = EVENT_MAP.fetch(event)
+      matching = webhooks.select { |w| w.events.include?(event_sym) }
+      return if matching.empty?
+
+      matching.each do |webhook|
+        payload = build_payload(data_import, event, webhook)
+        DataPorter::WebhookJob.perform_later(webhook.url, payload, webhook.headers.dup)
+      end
+    end
+
+    def resolve_webhooks(data_import)
+      data_import.target_class._webhooks
+    end
+
+    def build_payload(data_import, event, webhook)
+      data = default_payload(data_import, event)
+      data = webhook.payload.call(data_import, event, data) if webhook.payload
+      data.to_json
+    end
+
+    def default_payload(data_import, event)
+      {
+        "event" => event,
+        "timestamp" => Time.current.iso8601,
+        "import" => import_data(data_import, event)
+      }
+    end
+
+    def import_data(data_import, event)
+      base = base_import_data(data_import)
+      merge_event_data(base, data_import, event)
+    end
+
+    def base_import_data(data_import)
+      {
+        "id" => data_import.id,
+        "target_key" => data_import.target_key,
+        "source_type" => data_import.source_type
+      }
+    end
+
+    def merge_event_data(base, data_import, event)
+      case event
+      when "import.completed" then merge_completed(base, data_import)
+      when "import.failed" then merge_failed(base, data_import)
+      when "import.parsed" then merge_parsed(base, data_import)
+      when "import.started" then merge_started(base, data_import)
+      else base
+      end
+    end
+
+    def merge_completed(base, data_import)
+      report = data_import.report
+      base.merge(
+        "imported_count" => report&.imported_count.to_i,
+        "errored_count" => report&.errored_count.to_i
+      )
+    end
+
+    def merge_failed(base, data_import)
+      message = first_error_message(data_import.report)
+      base.merge("error_message" => message)
+    end
+
+    def first_error_message(report)
+      errors = report&.error_reports
+      errors&.first&.message
+    end
+
+    def merge_parsed(base, data_import)
+      report = data_import.report
+      base.merge(
+        "records_count" => report&.records_count.to_i,
+        "complete_count" => report&.complete_count.to_i,
+        "partial_count" => report&.partial_count.to_i
+      )
+    end
+
+    def merge_started(base, data_import)
+      base.merge("records_count" => data_import.records.size)
+    end
+  end
+end

data/lib/data_porter.rb

@@ -9,6 +9,7 @@ end
 require_relative "data_porter/version"
 require_relative "data_porter/configuration"
 require_relative "data_porter/type_validator"
+require_relative "data_porter/column_transformer"
 require_relative "data_porter/store_models/error"
 require_relative "data_porter/store_models/report"
 require_relative "data_porter/store_models/import_record"
@@ -17,6 +18,7 @@ require_relative "data_porter/registry"
 require_relative "data_porter/sources"
 require_relative "data_porter/record_validator"
 require_relative "data_porter/broadcaster"
+require_relative "data_porter/webhook_notifier"
 require_relative "data_porter/orchestrator"
 require_relative "data_porter/rejects_csv_builder"
 require_relative "data_porter/components"

data/lib/generators/data_porter/install/templates/initializer.rb

@@ -36,4 +36,9 @@ DataPorter.configure do |config|
   # Auto-purge completed/failed imports older than this duration.
   # Set to nil to disable auto-purge. Run `rake data_porter:purge` manually or via cron.
   # config.purge_after = 60.days
+
+  # HMAC-SHA256 secret for signing webhook payloads.
+  # When set, every webhook request includes an X-DataPorter-Signature header.
+  # Set to nil to disable signing (default).
+  # config.webhook_secret = ENV["DATA_PORTER_WEBHOOK_SECRET"]
 end

data/lib/generators/data_porter/locale/locale_generator.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require "rails/generators"
+
+module DataPorter
+  module Generators
+    class LocaleGenerator < Rails::Generators::Base
+      source_root File.expand_path("../../../../config/locales", __dir__)
+
+      argument :locale, type: :string, default: "en"
+
+      def copy_locale_file
+        source = source_file
+        destination = "config/locales/data_porter.#{locale}.yml"
+
+        copy_file(source, destination)
+        gsub_file(destination, /^#{source_locale}:/, "#{locale}:") unless locale == source_locale
+      end
+
+      def show_instructions
+        say ""
+        say "DataPorter locale file created: config/locales/data_porter.#{locale}.yml", :green
+        say ""
+        say "Next steps:"
+        say "  1. Translate the values in the generated file"
+        say "  2. Set your default locale in config/application.rb:"
+        say "       config.i18n.default_locale = :#{locale}"
+        say ""
+      end
+
+      private
+
+      def source_file
+        File.exist?(File.join(self.class.source_root, "#{locale}.yml")) ? "#{locale}.yml" : "en.yml"
+      end
+
+      def source_locale
+        source_file.delete_suffix(".yml")
+      end
+    end
+  end
+end

data/mkdocs.yml

@@ -0,0 +1,98 @@
+site_name: DataPorter
+site_description: A mountable Rails engine for data import workflows
+site_url: https://seryllns.github.io/data_porter
+repo_url: https://github.com/SerylLns/data_porter
+repo_name: SerylLns/data_porter
+edit_uri: edit/main/docs/
+
+theme:
+  name: material
+  palette:
+    - media: "(prefers-color-scheme: light)"
+      scheme: default
+      primary: indigo
+      accent: indigo
+      toggle:
+        icon: material/brightness-7
+        name: Switch to dark mode
+    - media: "(prefers-color-scheme: dark)"
+      scheme: slate
+      primary: indigo
+      accent: indigo
+      toggle:
+        icon: material/brightness-4
+        name: Switch to light mode
+  font:
+    text: Inter
+    code: JetBrains Mono
+  icon:
+    repo: fontawesome/brands/github
+    logo: material/database-import
+  features:
+    - navigation.sections
+    - navigation.expand
+    - navigation.top
+    - navigation.indexes
+    - navigation.footer
+    - search.highlight
+    - search.suggest
+    - content.code.copy
+    - content.code.annotate
+    - content.tabs.link
+    - toc.follow
+
+plugins:
+  - search
+  - exclude:
+      glob:
+        - blog/*
+        - blog_part_2/*
+        - V1_PLAN.md
+        - dev_to.md
+        - reddit_post.md
+
+markdown_extensions:
+  - admonition
+  - pymdownx.details
+  - pymdownx.superfences
+  - pymdownx.highlight:
+      anchor_linenums: true
+      line_spans: __span
+      pygments_lang_class: true
+  - pymdownx.inlinehilite
+  - pymdownx.tabbed:
+      alternate_style: true
+  - pymdownx.emoji:
+      emoji_index: !!python/name:material.extensions.emoji.twemoji
+      emoji_generator: !!python/name:material.extensions.emoji.to_svg
+  - pymdownx.snippets:
+      base_path: ["."]
+  - attr_list
+  - md_in_html
+  - tables
+  - toc:
+      permalink: true
+
+extra:
+  social:
+    - icon: fontawesome/brands/github
+      link: https://github.com/SerylLns/data_porter
+    - icon: fontawesome/brands/dev
+      link: https://dev.to/seryllns
+  generator: false
+
+extra_css:
+  - stylesheets/extra.css
+
+nav:
+  - Home: index.md
+  - Getting Started: getting-started.md
+  - Reference:
+      - Configuration: CONFIGURATION.md
+      - Targets: TARGETS.md
+      - Sources: SOURCES.md
+      - Column Mapping: MAPPING.md
+      - Routes: routes.md
+  - Roadmap: ROADMAP.md
+  - Changelog: changelog.md
+  - Contributing: contributing.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_porter
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.3.0
 platform: ruby
 authors:
 - Seryl Lounis
@@ -129,6 +129,7 @@ files:
 - app/jobs/data_porter/extract_headers_job.rb
 - app/jobs/data_porter/import_job.rb
 - app/jobs/data_porter/parse_job.rb
+- app/jobs/data_porter/webhook_job.rb
 - app/models/data_porter/data_import.rb
 - app/models/data_porter/mapping_template.rb
 - app/views/data_porter/imports/index.html.erb
@@ -139,10 +140,12 @@ files:
 - app/views/data_porter/mapping_templates/index.html.erb
 - app/views/data_porter/mapping_templates/new.html.erb
 - app/views/layouts/data_porter/application.html.erb
-- bookmarklet.md
+- config/locales/en.yml
+- config/locales/fr.yml
 - config/routes.rb
 - lib/data_porter.rb
 - lib/data_porter/broadcaster.rb
+- lib/data_porter/column_transformer.rb
 - lib/data_porter/components.rb
 - lib/data_porter/components/base.rb
 - lib/data_porter/components/mapping/column_row.rb
@@ -159,6 +162,7 @@ files:
 - lib/data_porter/dsl/api_config.rb
 - lib/data_porter/dsl/column.rb
 - lib/data_porter/dsl/param.rb
+- lib/data_porter/dsl/webhook.rb
 - lib/data_porter/engine.rb
 - lib/data_porter/orchestrator.rb
 - lib/data_porter/orchestrator/dry_runner.rb
@@ -179,13 +183,16 @@ files:
 - lib/data_porter/target.rb
 - lib/data_porter/type_validator.rb
 - lib/data_porter/version.rb
+- lib/data_porter/webhook_notifier.rb
 - lib/generators/data_porter/install/install_generator.rb
 - lib/generators/data_porter/install/templates/create_data_porter_imports.rb.erb
 - lib/generators/data_porter/install/templates/create_data_porter_mapping_templates.rb.erb
 - lib/generators/data_porter/install/templates/initializer.rb
+- lib/generators/data_porter/locale/locale_generator.rb
 - lib/generators/data_porter/target/target_generator.rb
 - lib/generators/data_porter/target/templates/target.rb.tt
 - lib/tasks/data_porter.rake
+- mkdocs.yml
 - sig/data_porter.rbs
 homepage: https://github.com/SerylLns/data_porter
 licenses:
@@ -194,7 +201,7 @@ metadata:
   homepage_uri: https://github.com/SerylLns/data_porter
   source_code_uri: https://github.com/SerylLns/data_porter
   changelog_uri: https://github.com/SerylLns/data_porter/blob/main/CHANGELOG.md
-  documentation_uri: https://github.com/SerylLns/data_porter#readme
+  documentation_uri: https://seryllns.github.io/data_porter/
   bug_tracker_uri: https://github.com/SerylLns/data_porter/issues
   rubygems_mcp_server_uri: https://rubygems.org/gems/data_porter
   rubygems_mfa_required: 'true'