data_porter 2.4.0 → 2.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b393c618ddb47334a61a62de03796211188b58367cad71bfc8574874208a34db
-  data.tar.gz: 9eaa6b01ea0127bd22fa8557cccc34f47070e58991512f66996df6f9b789e8e3
+  metadata.gz: 99b0792c8088d9d2e55826f898fcfa99d872bea179f41d352b1e4aaff9ab14d8
+  data.tar.gz: 77c539c666176f992747f2d326a9ce8284cb19426b5f1ae470a317f7dfd740f1
 SHA512:
-  metadata.gz: 9f5e73c6eb11dccd143496f4147c941d2729a17643b10eed46f092e4615a2a861d1359b6993dadddaa7ab256672646a25b93f73f77bf238b1b4543fb7c92eb26
-  data.tar.gz: 3f2c4be7e4dbe828a57bd81709c64b5c592c71aabaae0e604643216455c2c3bfc7ef74d14b78241658104c996125495587cfff73301c5a8f918abca64aca8e4f
+  metadata.gz: b863d64f885f55ba6f530ce99173fef55ca473f74c7b0fc6736890bd770583949cf2101389c0635707ea38ceede63aebc529fc1a1e5990c76bf5c0750de318d6
+  data.tar.gz: 392ebe242c8ae947b37961bc8d59373a3d9f6a84730f0c946cd35af0a621ecc1a1cfb8b09b361f3a74fcc826c098fad317a4350d9b213206197406d88f763a01

data/CHANGELOG.md

@@ -5,6 +5,28 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [Unreleased]
+
+### Added
+
+- **Auto-map heuristics** -- Smart column suggestions that pre-fill mapping selects when CSV/XLSX headers match target fields by exact name or built-in synonym (e.g. "E-mail Address" → email, "fname" → first_name). Supports per-column custom synonyms via `synonyms:` keyword in column DSL. Fallback chain: saved mapping > code-defined > auto-map > empty
+
+## [2.5.1] - 2026-02-21
+
+### Fixed
+
+- Display target icon (from `icon` DSL) in the imports index table and show page title/details. Previously the icon was stored in the Registry but never rendered in the UI
+
+## [2.5.0] - 2026-02-21
+
+### Added
+
+- **Bulk import mode** -- Opt-in per-target batch persistence via `bulk_mode batch_size: 500, on_conflict: :retry_per_record`. Uses `insert_all` by default (with auto-injected timestamps) for 10-100x throughput on simple create scenarios. Custom batch logic via `persist_batch` override. Configurable conflict strategy: `:retry_per_record` (default) retries failed batches record-by-record, `:fail_batch` marks entire batch as errored. Progress broadcasts per batch instead of per record
+
+### Changed
+
+- 551 RSpec examples (up from 540), 0 failures
+
 ## [2.4.0] - 2026-02-21
 
 ### Added

data/ROADMAP.md

@@ -2,10 +2,6 @@
 
 ## Next
 
-### Bulk import
-
-High-volume import support using `insert_all` / `upsert_all` for batch persistence. Opt-in per target to bypass per-record `persist` calls, enabling 10-100x throughput for simple create/upsert scenarios. Configurable batch size, with fallback to per-record mode on conflict.
-
 ### Update & diff mode
 
 Support update (upsert) imports alongside create-only. Given a `deduplicate_by` key, detect existing records and show a diff preview: new records, changed fields (highlighted), unchanged rows. User confirms which changes to apply. Enables recurring data sync workflows.
@@ -34,10 +30,6 @@ Headless REST API for programmatic imports:
 - Auth via `config.api_authenticate` lambda (API key or Bearer token)
 - Reuses existing job pipeline (parse, import, dry run)
 
-### Auto-map heuristics
-
-Smart column mapping suggestions using tokenized header matching and synonym dictionaries. When a CSV has "E-mail Address", auto-suggest mapping to `:email`. Built-in synonyms for common patterns (phone → phone_number, first name → first_name). Configurable synonym lists per target.
-
 ---
 
 ## Ideas

data/app/controllers/data_porter/concerns/mapping_management.rb

@@ -27,7 +27,21 @@ module DataPorter
         saved = @import.config&.dig("column_mapping")
         return saved if saved.present?
 
-        (target._csv_mappings || {}).transform_values(&:to_s)
+        code_mapping = (target._csv_mappings || {}).transform_values(&:to_s)
+        return code_mapping if code_mapping.present?
+
+        auto_map_suggestions(target)
+      end
+
+      def auto_map_suggestions(target)
+        columns = target._columns || []
+        return {} if columns.empty? || @file_headers.empty?
+
+        custom = columns.each_with_object({}) do |col, hash|
+          hash[col.name] = col.synonyms if col.synonyms.any?
+        end
+
+        AutoMapper.new(@file_headers, columns.map(&:name), custom_synonyms: custom).call
       end
 
       def save_column_mapping

data/app/views/data_porter/imports/index.html.erb

@@ -28,7 +28,7 @@
         <% @imports.each do |import| %>
           <tr>
             <td><%= import.id %></td>
-            <td><%= import.target_key %></td>
+            <td><% target_cls = import.target_class rescue nil %><% if target_cls&._icon.present? %><i class="<%= target_cls._icon %>"></i> <% end %><%= target_cls&._label || import.target_key %></td>
             <td><%= import.source_type %></td>
             <td><%= raw DataPorter::Components::Shared::StatusBadge.new(status: import.status).call %></td>
             <td><%= import.created_at&.strftime("%Y-%m-%d %H:%M") %></td>

data/app/views/data_porter/imports/show.html.erb

@@ -4,7 +4,7 @@
       <%= link_to t("data_porter.imports.back_to_imports"), imports_path, class: "dp-btn dp-btn--secondary" %>
     </div>
     <h1 class="dp-title">
-      <%= t("data_porter.imports.show_title", target: @target._label, id: @import.id) %>
+      <% if @target._icon.present? %><i class="<%= @target._icon %>"></i> <% end %><%= t("data_porter.imports.show_title", target: @target._label, id: @import.id) %>
     </h1>
     <%= raw DataPorter::Components::Shared::StatusBadge.new(status: @import.status).call %>
   </div>
@@ -12,7 +12,7 @@
   <div class="dp-import-details">
     <dl class="dp-details-grid">
       <dt><%= t("data_porter.imports.details.target") %></dt>
-      <dd><%= @target._label %></dd>
+      <dd><% if @target._icon.present? %><i class="<%= @target._icon %>"></i> <% end %><%= @target._label %></dd>
       <dt><%= t("data_porter.imports.details.source") %></dt>
       <dd><%= @import.source_type.upcase %></dd>
       <% if @import.file.attached? %>

data/app/views/layouts/data_porter/application.html.erb

@@ -6,6 +6,7 @@
   <title>DataPorter</title>
   <%= csrf_meta_tags %>
   <%= stylesheet_link_tag "data_porter/application" %>
+  <link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.5.1/css/all.min.css" integrity="sha512-DTOQO9RWCH3ppGqcWaEA1BIZOC6xxalwEsw9c2QQeAIftl+Vegovlnee1c9QX4TctnWMn13TZye+giMm8e2LwA==" crossorigin="anonymous" referrerpolicy="no-referrer" />
   <script type="importmap">
     {
       "imports": {

data/lib/data_porter/auto_mapper.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class AutoMapper
+    SYNONYMS = {
+      email: %w[email e_mail email_address e_mail_address courriel mail],
+      first_name: %w[first_name firstname fname first prenom],
+      last_name: %w[last_name lastname lname last nom],
+      name: %w[name full_name fullname nom_complet],
+      phone_number: %w[phone_number phone tel telephone mobile cell],
+      address: %w[address addr street adresse],
+      city: %w[city ville town],
+      zip_code: %w[zip_code zip postal_code postcode code_postal],
+      country: %w[country pays nation],
+      company: %w[company company_name organization organisation entreprise societe],
+      title: %w[title job_title position titre poste],
+      description: %w[description desc notes],
+      quantity: %w[quantity qty amount],
+      price: %w[price unit_price prix montant],
+      date: %w[date created_at updated_at],
+      status: %w[status state statut etat],
+      id: %w[id identifier external_id ref reference]
+    }.freeze
+
+    def initialize(headers, target_columns, custom_synonyms: {})
+      @headers = headers
+      @target_columns = target_columns.map(&:to_s)
+      @custom_synonyms = custom_synonyms
+    end
+
+    def call
+      used = Set.new
+      @headers.each_with_object({}) do |header, mapping|
+        match = find_match(header, used)
+        used.add(match) if match
+        mapping[header] = match || ""
+      end
+    end
+
+    private
+
+    def find_match(header, used)
+      normalized = normalize(header)
+      return nil if normalized.empty?
+
+      exact_match(normalized, used) || synonym_match(normalized, used)
+    end
+
+    def exact_match(normalized, used)
+      @target_columns.find { |col| col == normalized && !used.include?(col) }
+    end
+
+    def synonym_match(normalized, used)
+      lookup_table[normalized]&.find { |col| !used.include?(col) }
+    end
+
+    def lookup_table
+      @lookup_table ||= build_lookup_table
+    end
+
+    def build_lookup_table
+      table = Hash.new { |h, k| h[k] = [] }
+      merged_synonyms.each do |column, synonyms|
+        col_name = column.to_s
+        next unless @target_columns.include?(col_name)
+
+        synonyms.each { |syn| table[syn] << col_name }
+      end
+      table
+    end
+
+    def merged_synonyms
+      result = SYNONYMS.transform_values(&:dup)
+      @custom_synonyms.each do |column, syns|
+        key = column.to_sym
+        result[key] = (result.fetch(key, []) + syns.map { |s| normalize(s) }).uniq
+      end
+      result
+    end
+
+    def normalize(header)
+      return "" if header.nil?
+
+      header.to_s.strip.downcase.gsub(/[\s-]+/, "_").gsub(/[^a-z0-9_]/, "")
+    end
+  end
+end

data/lib/data_porter/dsl/column.rb

@@ -2,14 +2,15 @@
 
 module DataPorter
   module DSL
-    Column = Struct.new(:name, :type, :required, :label, :transform, :options, keyword_init: true) do
-      def initialize(name:, type: :string, required: false, label: nil, transform: [], **options)
+    Column = Struct.new(:name, :type, :required, :label, :transform, :synonyms, :options, keyword_init: true) do
+      def initialize(name:, type: :string, required: false, label: nil, transform: [], synonyms: [], **options)
         super(
           name: name.to_sym,
           type: type.to_sym,
           required: required,
           label: label || name.to_s.humanize,
           transform: Array(transform),
+          synonyms: Array(synonyms),
           options: options
         )
       end

data/lib/data_porter/orchestrator/bulk_importer.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class Orchestrator
+    module BulkImporter
+      private
+
+      def import_bulk
+        importable = @data_import.importable_records
+        context = build_context
+        config = @target.class._bulk_config
+        results = { created: 0, errored: 0 }
+        total = importable.size
+        processed = 0
+
+        importable.each_slice(config[:batch_size]) do |batch|
+          persist_batch_with_fallback(batch, context, config, results)
+          processed += batch.size
+          broadcast_progress(processed, total)
+        end
+
+        finalize_import(results)
+      end
+
+      def persist_batch_with_fallback(batch, context, config, results)
+        @target.persist_batch(batch, context: context)
+        results[:created] += batch.size
+      rescue StandardError => e
+        handle_batch_failure(batch, context, config, results, e)
+      end
+
+      def handle_batch_failure(batch, context, config, results, error)
+        if config[:on_conflict] == :fail_batch
+          fail_batch(batch, results, error)
+        else
+          retry_per_record(batch, context, results)
+        end
+      end
+
+      def fail_batch(batch, results, error)
+        batch.each do |record|
+          record.add_error(error.message)
+        end
+        results[:errored] += batch.size
+      end
+
+      def retry_per_record(batch, context, results)
+        batch.each do |record|
+          persist_record(record, context, results)
+        end
+      end
+    end
+  end
+end

data/lib/data_porter/orchestrator/importer.rb

@@ -6,7 +6,9 @@ module DataPorter
       private
 
       def import_records
-        if DataPorter.configuration.transaction_mode == :all
+        if @target.class._bulk_config
+          import_bulk
+        elsif DataPorter.configuration.transaction_mode == :all
           import_all_or_nothing
         else
           import_per_record

data/lib/data_porter/orchestrator.rb

@@ -2,12 +2,14 @@
 
 require_relative "orchestrator/record_builder"
 require_relative "orchestrator/importer"
+require_relative "orchestrator/bulk_importer"
 require_relative "orchestrator/dry_runner"
 
 module DataPorter
   class Orchestrator
     include RecordBuilder
     include Importer
+    include BulkImporter
     include DryRunner
 
     def initialize(data_import, content: nil)

data/lib/data_porter/target.rb

@@ -10,7 +10,8 @@ module DataPorter
     class << self
       attr_reader :_label, :_model_name, :_icon, :_sources,
                   :_columns, :_csv_mappings, :_dedup_keys, :_json_root,
-                  :_api_config, :_dry_run_enabled, :_params, :_webhooks
+                  :_api_config, :_dry_run_enabled, :_params, :_webhooks,
+                  :_bulk_config
 
       def label(value)
         @_label = value
@@ -82,6 +83,10 @@ module DataPorter
         @_webhooks << DSL::Webhook.new(url: url, **)
       end
 
+      def bulk_mode(batch_size: 500, on_conflict: :retry_per_record)
+        @_bulk_config = { batch_size: batch_size, on_conflict: on_conflict }
+      end
+
       private
 
       def auto_register
@@ -108,6 +113,16 @@ module DataPorter
       raise NotImplementedError
     end
 
+    def persist_batch(records, context: nil) # rubocop:disable Lint/UnusedMethodArgument
+      raise Error, "model_name is required for default persist_batch" unless self.class._model_name
+
+      now = Time.current
+      model_class = self.class._model_name.constantize
+      model_class.insert_all(
+        records.map { |r| r.data.merge("created_at" => now, "updated_at" => now) }
+      )
+    end
+
     def after_import(_results, context:); end
 
     def on_error(_record, _error, context:); end

data/lib/data_porter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataPorter
-  VERSION = "2.4.0"
+  VERSION = "2.5.1"
 end

data/lib/data_porter.rb

@@ -20,6 +20,7 @@ require_relative "data_porter/record_validator"
 require_relative "data_porter/broadcaster"
 require_relative "data_porter/webhook_notifier"
 require_relative "data_porter/orchestrator"
+require_relative "data_porter/auto_mapper"
 require_relative "data_porter/rejects_csv_builder"
 require_relative "data_porter/components"
 require_relative "data_porter/engine"

data/lib/generators/data_porter/install/templates/initializer.rb

@@ -37,6 +37,10 @@ DataPorter.configure do |config|
   # Set to nil to disable auto-purge. Run `rake data_porter:purge` manually or via cron.
   # config.purge_after = 60.days
 
+  # Bulk import: enable per-target via `bulk_mode` in your Target class.
+  # Uses insert_all for 10-100x throughput on large imports.
+  # See docs/ADVANCED.md for configuration options.
+
   # HMAC-SHA256 secret for signing webhook payloads.
   # When set, every webhook request includes an X-DataPorter-Signature header.
   # Set to nil to disable signing (default).

data/lib/generators/data_porter/target/templates/target.rb.tt

@@ -5,6 +5,7 @@ class <%= target_class_name %> < DataPorter::Target
   model_name "<%= model_name %>"
   icon "fas fa-file-import"
   sources <%= target_sources %>
+  # bulk_mode batch_size: 500, on_conflict: :retry_per_record
 <% if parsed_columns.any? %>
 
   columns do

data/mkdocs.yml

@@ -92,7 +92,9 @@ nav:
       - Targets: TARGETS.md
       - Sources: SOURCES.md
       - Column Mapping: MAPPING.md
+      - Views & Theming: VIEWS.md
       - Routes: routes.md
+  - Advanced: ADVANCED.md
   - Roadmap: ROADMAP.md
   - Changelog: changelog.md
   - Contributing: contributing.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_porter
 version: !ruby/object:Gem::Version
-  version: 2.4.0
+  version: 2.5.1
 platform: ruby
 authors:
 - Seryl Lounis
@@ -146,6 +146,7 @@ files:
 - config/locales/fr.yml
 - config/routes.rb
 - lib/data_porter.rb
+- lib/data_porter/auto_mapper.rb
 - lib/data_porter/broadcaster.rb
 - lib/data_porter/column_transformer.rb
 - lib/data_porter/components.rb
@@ -167,6 +168,7 @@ files:
 - lib/data_porter/dsl/webhook.rb
 - lib/data_porter/engine.rb
 - lib/data_porter/orchestrator.rb
+- lib/data_porter/orchestrator/bulk_importer.rb
 - lib/data_porter/orchestrator/dry_runner.rb
 - lib/data_porter/orchestrator/importer.rb
 - lib/data_porter/orchestrator/record_builder.rb