data_porter 2.1.1 → 2.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '04931dd3e74ae9a12b2225ddb1c5467022458013d49fb35a07282121296ec22b'
-  data.tar.gz: 4d0d04002509ae87358375f0df64bec0d8c134201131ead1a1a07460c9e19297
+  metadata.gz: 14b97e1d544f1169ed2e19763f1ae4ae1d15373cd809f5c6ddb46ebe286cc804
+  data.tar.gz: 5e7fd19fdf843d6f7721ef91d3147197d06d453771990408354a38d8758f647b
 SHA512:
-  metadata.gz: 11ba16dcc818425722fc20e1a919833e4c4f77def52c07b46068707a1e511bb045bf7801edfb9b3186750fbf242a633d526e81e6b1bf3cd4b54b806c47e701aa
-  data.tar.gz: 4c99d6a1c4d04e4cee197199db9dd3517bc7d44b283e511a7289e46703ddf3d69f91a8cd7e4fb4f43ce5e8fdaaccbe805b5a649846d1c5e096a7803eafb07442
+  metadata.gz: b23a128b472327079631531f020fda16ea7399bb6c2f09437c8976ddf738800e1738c9afb9167db359c662d02e29ca7994bc7709d97ecb4a46e57df8e691a02f
+  data.tar.gz: 1bb9f3cc08b7fee3678638cfb047e191c495447afc93db7115c5a6c2914851f127ed9bf4ab7c44d110cc2585ae2de0fd9ee67d6a469082e9ede3d71315a9655a

data/CHANGELOG.md

@@ -5,6 +5,26 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [2.3.0] - 2026-02-20
+
+### Added
+
+- **Webhooks** -- Per-target HTTP callbacks on import lifecycle events (`import.started`, `import.parsed`, `import.completed`, `import.failed`). Declarative DSL via `webhooks do webhook(url, events:, headers:, payload:) end`. HMAC-SHA256 request signing via `config.webhook_secret`. Async delivery via `WebhookJob` (fire-and-forget, 10s timeout). Custom payload lambdas and per-webhook headers supported
+
+### Changed
+
+- 508 RSpec examples (up from 466), 0 failures
+
+## [2.2.0] - 2026-02-20
+
+### Added
+
+- **Column transformers** -- Declarative per-column transformation pipeline via `transform: [:strip, :downcase]` in the columns DSL. Applied automatically before the target's `transform` method. Ships with 9 built-in transformers (`strip`, `downcase`, `upcase`, `titleize`, `normalize_phone`, `parse_date`, `parse_boolean`, `parse_integer`, `parse_decimal`). Custom transformers via `DataPorter::ColumnTransformer.register(:name) { |v| ... }`
+
+### Changed
+
+- 466 RSpec examples (up from 438), 0 failures
+
 ## [2.1.1] - 2026-02-20
 
 ### Fixed

data/ROADMAP.md

@@ -2,28 +2,6 @@
 
 ## Next
 
-### Column transformers
-
-Built-in transformation pipeline applied per-column before the target's `transform` method. Declarative DSL in the target:
-
-```ruby
-columns do
-  column :email, type: :string, transform: [:strip, :downcase]
-  column :phone, type: :string, transform: [:strip, :normalize_phone]
-  column :born_on, type: :date, transform: [:parse_date]
-end
-```
-
-Ships with common transformers (`strip`, `downcase`, `titleize`, `normalize_phone`, `parse_date`). Custom transformers via a registry.
-
-### Webhooks
-
-HTTP callbacks on import lifecycle events (started, completed, failed). Configurable per-target with URL, headers, and payload template. Enables integration with Slack notifications, CI pipelines, or external dashboards.
-
----
-
-## Planned
-
 ### Bulk import
 
 High-volume import support using `insert_all` / `upsert_all` for batch persistence. Opt-in per target to bypass per-record `persist` calls, enabling 10-100x throughput for simple create/upsert scenarios. Configurable batch size, with fallback to per-record mode on conflict.

data/app/jobs/data_porter/webhook_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require "net/http"
+require "openssl"
+
+module DataPorter
+  class WebhookJob < ActiveJob::Base
+    queue_as { DataPorter.configuration.queue_name }
+
+    def perform(url, payload_json, headers = {})
+      uri = URI.parse(url)
+      request = build_request(uri, payload_json, headers)
+      execute_request(uri, request)
+    end
+
+    private
+
+    def build_request(uri, payload_json, headers)
+      request = Net::HTTP::Post.new(uri.request_uri)
+      request["Content-Type"] = "application/json"
+      headers.each { |key, value| request[key] = value }
+      sign_request(request, payload_json)
+      request.body = payload_json
+      request
+    end
+
+    def sign_request(request, payload_json)
+      secret = DataPorter.configuration.webhook_secret
+      return unless secret
+
+      digest = OpenSSL::HMAC.hexdigest("SHA256", secret, payload_json)
+      request["X-DataPorter-Signature"] = "sha256=#{digest}"
+    end
+
+    def execute_request(uri, request)
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = uri.scheme == "https"
+      http.open_timeout = 10
+      http.read_timeout = 10
+      http.request(request)
+    rescue StandardError => e
+      Rails.logger.error("[DataPorter] Webhook delivery failed: #{e.message}")
+    end
+  end
+end

data/lib/data_porter/column_transformer.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module ColumnTransformer
+    BUILT_IN = {
+      strip: :strip.to_proc,
+      downcase: :downcase.to_proc,
+      upcase: :upcase.to_proc,
+      titleize: :titleize.to_proc,
+      normalize_phone: ->(value) { value.gsub(/[\s\-().]+/, "") },
+      parse_date: ->(value) { Date.parse(value).iso8601 rescue value }, # rubocop:disable Style/RescueModifier
+      parse_boolean: ->(value) { %w[true 1 yes oui].include?(value.downcase) ? "true" : "false" },
+      parse_integer: ->(value) { Float(value, exception: false) ? value.to_f.to_i.to_s : value },
+      parse_decimal: ->(value) { Float(value, exception: false)&.to_s || value }
+    }.freeze
+
+    def self.apply(value, transformer_name)
+      return value if value.nil?
+
+      transformer = BUILT_IN[transformer_name] || custom_transformers[transformer_name]
+      raise Error, "Unknown transformer: #{transformer_name}" unless transformer
+
+      transformer.call(value.to_s)
+    end
+
+    def self.register(name, &block)
+      custom_transformers[name.to_sym] = block
+    end
+
+    def self.apply_all(record, columns)
+      columns.each do |col|
+        next if col.transform.empty?
+
+        key = resolve_key(record.data, col.name)
+        next unless key
+
+        col.transform.each do |t|
+          record.data[key] = apply(record.data[key], t)
+        end
+      end
+    end
+
+    def self.resolve_key(data, name)
+      return name.to_s if data.key?(name.to_s)
+      return name if data.key?(name)
+
+      nil
+    end
+
+    def self.custom_transformers
+      @custom_transformers ||= {}
+    end
+
+    private_class_method :custom_transformers, :resolve_key
+  end
+end

data/lib/data_porter/configuration.rb

@@ -13,7 +13,8 @@ module DataPorter
                   :purge_after,
                   :max_file_size,
                   :max_records,
-                  :transaction_mode
+                  :transaction_mode,
+                  :webhook_secret
 
     def initialize
       @parent_controller = "ActionController::Base"
@@ -28,6 +29,7 @@ module DataPorter
       @max_file_size = 10.megabytes
       @max_records = 10_000
       @transaction_mode = :per_record
+      @webhook_secret = nil
     end
   end
 end

data/lib/data_porter/dsl/column.rb

@@ -2,13 +2,14 @@
 
 module DataPorter
   module DSL
-    Column = Struct.new(:name, :type, :required, :label, :options, keyword_init: true) do
-      def initialize(name:, type: :string, required: false, label: nil, **options)
+    Column = Struct.new(:name, :type, :required, :label, :transform, :options, keyword_init: true) do
+      def initialize(name:, type: :string, required: false, label: nil, transform: [], **options)
         super(
           name: name.to_sym,
           type: type.to_sym,
           required: required,
           label: label || name.to_s.humanize,
+          transform: Array(transform),
           options: options
         )
       end

data/lib/data_porter/dsl/webhook.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module DSL
+    VALID_WEBHOOK_EVENTS = %i[started completed failed parsed].freeze
+
+    Webhook = Struct.new(:url, :events, :headers, :payload, keyword_init: true) do
+      def initialize(url:, events: VALID_WEBHOOK_EVENTS, headers: {}, payload: nil)
+        validate_url!(url)
+        events = events.map(&:to_sym)
+        validate_events!(events)
+        super
+      end
+
+      private
+
+      def validate_url!(url)
+        raise ArgumentError, "url is required" if url.nil? || url.to_s.strip.empty?
+      end
+
+      def validate_events!(events)
+        events.each do |event|
+          next if VALID_WEBHOOK_EVENTS.include?(event)
+
+          raise ArgumentError,
+                "invalid webhook event: #{event}. Must be one of: #{VALID_WEBHOOK_EVENTS.join(", ")}"
+        end
+      end
+    end
+  end
+end

data/lib/data_porter/orchestrator/importer.rb

@@ -45,6 +45,7 @@ module DataPorter
       def finalize_import(results)
         @data_import.update!(status: :completed)
         @broadcaster.success
+        WebhookNotifier.notify(@data_import, "import.completed")
         results
       end
 

data/lib/data_porter/orchestrator/record_builder.rb

@@ -30,6 +30,7 @@ module DataPorter
           line_number: index + 1,
           data: extract_data(row, columns)
         )
+        ColumnTransformer.apply_all(record, columns)
         record = @target.transform(record)
         @target.validate(record)
         validator.validate(record)

data/lib/data_porter/orchestrator.rb

@@ -32,12 +32,14 @@ module DataPorter
       records = build_records
       @data_import.update!(records: records, status: :previewing)
       build_report
+      WebhookNotifier.notify(@data_import, "import.parsed")
     rescue StandardError => e
       handle_failure(e)
     end
 
     def import!
       @data_import.importing!
+      WebhookNotifier.notify(@data_import, "import.started")
       results = import_records
       update_import_report(results)
       @target.after_import(results, context: build_context)
@@ -102,6 +104,7 @@ module DataPorter
       )
       @data_import.update!(status: :failed, report: report)
       @broadcaster.failure(error.message)
+      WebhookNotifier.notify(@data_import, "import.failed")
     end
   end
 end

data/lib/data_porter/target.rb

@@ -3,13 +3,14 @@
 require_relative "dsl/column"
 require_relative "dsl/param"
 require_relative "dsl/api_config"
+require_relative "dsl/webhook"
 
 module DataPorter
   class Target
     class << self
       attr_reader :_label, :_model_name, :_icon, :_sources,
                   :_columns, :_csv_mappings, :_dedup_keys, :_json_root,
-                  :_api_config, :_dry_run_enabled, :_params
+                  :_api_config, :_dry_run_enabled, :_params, :_webhooks
 
       def label(value)
         @_label = value
@@ -72,6 +73,15 @@ module DataPorter
         @_params << DSL::Param.new(name: name, **)
       end
 
+      def webhooks(&)
+        @_webhooks = []
+        instance_eval(&)
+      end
+
+      def webhook(url, **)
+        @_webhooks << DSL::Webhook.new(url: url, **)
+      end
+
       private
 
       def auto_register

data/lib/data_porter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataPorter
-  VERSION = "2.1.1"
+  VERSION = "2.3.0"
 end

data/lib/data_porter/webhook_notifier.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module DataPorter
+  module WebhookNotifier
+    EVENT_MAP = {
+      "import.started" => :started,
+      "import.completed" => :completed,
+      "import.failed" => :failed,
+      "import.parsed" => :parsed
+    }.freeze
+
+    module_function
+
+    def notify(data_import, event)
+      webhooks = resolve_webhooks(data_import)
+      return if webhooks.nil? || webhooks.empty?
+
+      event_sym = EVENT_MAP.fetch(event)
+      matching = webhooks.select { |w| w.events.include?(event_sym) }
+      return if matching.empty?
+
+      matching.each do |webhook|
+        payload = build_payload(data_import, event, webhook)
+        DataPorter::WebhookJob.perform_later(webhook.url, payload, webhook.headers.dup)
+      end
+    end
+
+    def resolve_webhooks(data_import)
+      data_import.target_class._webhooks
+    end
+
+    def build_payload(data_import, event, webhook)
+      data = default_payload(data_import, event)
+      data = webhook.payload.call(data_import, event, data) if webhook.payload
+      data.to_json
+    end
+
+    def default_payload(data_import, event)
+      {
+        "event" => event,
+        "timestamp" => Time.current.iso8601,
+        "import" => import_data(data_import, event)
+      }
+    end
+
+    def import_data(data_import, event)
+      base = base_import_data(data_import)
+      merge_event_data(base, data_import, event)
+    end
+
+    def base_import_data(data_import)
+      {
+        "id" => data_import.id,
+        "target_key" => data_import.target_key,
+        "source_type" => data_import.source_type
+      }
+    end
+
+    def merge_event_data(base, data_import, event)
+      case event
+      when "import.completed" then merge_completed(base, data_import)
+      when "import.failed" then merge_failed(base, data_import)
+      when "import.parsed" then merge_parsed(base, data_import)
+      when "import.started" then merge_started(base, data_import)
+      else base
+      end
+    end
+
+    def merge_completed(base, data_import)
+      report = data_import.report
+      base.merge(
+        "imported_count" => report&.imported_count.to_i,
+        "errored_count" => report&.errored_count.to_i
+      )
+    end
+
+    def merge_failed(base, data_import)
+      message = first_error_message(data_import.report)
+      base.merge("error_message" => message)
+    end
+
+    def first_error_message(report)
+      errors = report&.error_reports
+      errors&.first&.message
+    end
+
+    def merge_parsed(base, data_import)
+      report = data_import.report
+      base.merge(
+        "records_count" => report&.records_count.to_i,
+        "complete_count" => report&.complete_count.to_i,
+        "partial_count" => report&.partial_count.to_i
+      )
+    end
+
+    def merge_started(base, data_import)
+      base.merge("records_count" => data_import.records.size)
+    end
+  end
+end

data/lib/data_porter.rb

@@ -9,6 +9,7 @@ end
 require_relative "data_porter/version"
 require_relative "data_porter/configuration"
 require_relative "data_porter/type_validator"
+require_relative "data_porter/column_transformer"
 require_relative "data_porter/store_models/error"
 require_relative "data_porter/store_models/report"
 require_relative "data_porter/store_models/import_record"
@@ -17,6 +18,7 @@ require_relative "data_porter/registry"
 require_relative "data_porter/sources"
 require_relative "data_porter/record_validator"
 require_relative "data_porter/broadcaster"
+require_relative "data_porter/webhook_notifier"
 require_relative "data_porter/orchestrator"
 require_relative "data_porter/rejects_csv_builder"
 require_relative "data_porter/components"

data/lib/generators/data_porter/install/templates/initializer.rb

@@ -36,4 +36,9 @@ DataPorter.configure do |config|
   # Auto-purge completed/failed imports older than this duration.
   # Set to nil to disable auto-purge. Run `rake data_porter:purge` manually or via cron.
   # config.purge_after = 60.days
+
+  # HMAC-SHA256 secret for signing webhook payloads.
+  # When set, every webhook request includes an X-DataPorter-Signature header.
+  # Set to nil to disable signing (default).
+  # config.webhook_secret = ENV["DATA_PORTER_WEBHOOK_SECRET"]
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_porter
 version: !ruby/object:Gem::Version
-  version: 2.1.1
+  version: 2.3.0
 platform: ruby
 authors:
 - Seryl Lounis
@@ -129,6 +129,7 @@ files:
 - app/jobs/data_porter/extract_headers_job.rb
 - app/jobs/data_porter/import_job.rb
 - app/jobs/data_porter/parse_job.rb
+- app/jobs/data_porter/webhook_job.rb
 - app/models/data_porter/data_import.rb
 - app/models/data_porter/mapping_template.rb
 - app/views/data_porter/imports/index.html.erb
@@ -144,6 +145,7 @@ files:
 - config/routes.rb
 - lib/data_porter.rb
 - lib/data_porter/broadcaster.rb
+- lib/data_porter/column_transformer.rb
 - lib/data_porter/components.rb
 - lib/data_porter/components/base.rb
 - lib/data_porter/components/mapping/column_row.rb
@@ -160,6 +162,7 @@ files:
 - lib/data_porter/dsl/api_config.rb
 - lib/data_porter/dsl/column.rb
 - lib/data_porter/dsl/param.rb
+- lib/data_porter/dsl/webhook.rb
 - lib/data_porter/engine.rb
 - lib/data_porter/orchestrator.rb
 - lib/data_porter/orchestrator/dry_runner.rb
@@ -180,6 +183,7 @@ files:
 - lib/data_porter/target.rb
 - lib/data_porter/type_validator.rb
 - lib/data_porter/version.rb
+- lib/data_porter/webhook_notifier.rb
 - lib/generators/data_porter/install/install_generator.rb
 - lib/generators/data_porter/install/templates/create_data_porter_imports.rb.erb
 - lib/generators/data_porter/install/templates/create_data_porter_mapping_templates.rb.erb