data_porter 2.5.1 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99b0792c8088d9d2e55826f898fcfa99d872bea179f41d352b1e4aaff9ab14d8
-  data.tar.gz: 77c539c666176f992747f2d326a9ce8284cb19426b5f1ae470a317f7dfd740f1
+  metadata.gz: 955a886124d8ff2f1da4f23e725a52caec3bbc635450f058e3bbce81f4b898f5
+  data.tar.gz: 6f1f0be41999d105c7558b7192f3f61d79f935431ce7b99bede5753d69be9ce3
 SHA512:
-  metadata.gz: b863d64f885f55ba6f530ce99173fef55ca473f74c7b0fc6736890bd770583949cf2101389c0635707ea38ceede63aebc529fc1a1e5990c76bf5c0750de318d6
-  data.tar.gz: 392ebe242c8ae947b37961bc8d59373a3d9f6a84730f0c946cd35af0a621ecc1a1cfb8b09b361f3a74fcc826c098fad317a4350d9b213206197406d88f763a01
+  metadata.gz: 399c87e6daa56196ae96525ca75b27a464a109463d3f4ebf5361738a93cc6385b30e37e588cab45c3a2218b99596dabcd9a48b0a9856eac17902907a9f4f9a34
+  data.tar.gz: f423073b27fc407cc94ecc8e11693d40da5210387129f428105ac4973d7b7be034a59b8161c2a4da3c99acc7e4ec1b13d4828c54336e12d441044549b6243151

data/CHANGELOG.md

@@ -11,6 +11,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - **Auto-map heuristics** -- Smart column suggestions that pre-fill mapping selects when CSV/XLSX headers match target fields by exact name or built-in synonym (e.g. "E-mail Address" → email, "fname" → first_name). Supports per-column custom synonyms via `synonyms:` keyword in column DSL. Fallback chain: saved mapping > code-defined > auto-map > empty
 
+## [2.6.0] - 2026-02-21
+
+### Added
+
+- **Resume on failure** -- When an import fails mid-way (crash, timeout, exception), resume from the last successful record instead of re-importing from scratch. Progress checkpoints stored in the existing `config` JSONB column alongside `broadcast_progress` — zero additional DB operations or migrations. Works with both per-record and bulk import modes
+- `resumable?` predicate on `DataImport` — returns `true` when a failed import has a checkpoint with processed records
+- Resume button in the failed import UI (primary action), with Retry demoted to secondary
+- `POST :resume` route on the imports controller
+
+### Fixed
+
+- `handle_failure` now preserves existing report data (parsed counts, partial results) instead of creating a new empty report
+- `parse!` now clears stale checkpoint and progress data from previous import attempts
+
+### Changed
+
+- 574 RSpec examples (up from 551), 0 failures
+
 ## [2.5.1] - 2026-02-21
 
 ### Fixed

data/README.md

@@ -103,6 +103,8 @@ pending -> parsing -> previewing -> importing -> completed
 
 **[Full documentation on GitHub Pages](https://seryllns.github.io/data_porter/)**
 
+> **Build series**: Want to see how DataPorter was built step by step? [Building DataPorter on dev.to](https://dev.to/seryllns_/series/35813) -- 30 parts covering architecture, TDD, and every feature from first commit to production.
+
 | Topic | Description |
 |---|---|
 | [Configuration](docs/CONFIGURATION.md) | All options, authentication, context builder, real-time updates |

data/ROADMAP.md

@@ -6,10 +6,6 @@
 
 Support update (upsert) imports alongside create-only. Given a `deduplicate_by` key, detect existing records and show a diff preview: new records, changed fields (highlighted), unchanged rows. User confirms which changes to apply. Enables recurring data sync workflows.
 
-### Resume / retry on failure
-
-If an import fails mid-way (timeout, crash, transient error), resume from the last successful record instead of restarting from scratch. Track a checkpoint index in the report. Critical for large imports (5k+ records) where re-processing everything is not acceptable.
-
 ### API pagination
 
 Support paginated API sources. The current API source does a single GET, which works for small datasets but not for APIs returning thousands of records across multiple pages. Support offset, cursor, and link-header pagination strategies via `api_config`:

data/app/controllers/data_porter/imports_controller.rb

@@ -10,7 +10,7 @@ module DataPorter
     layout "data_porter/application"
 
     before_action :set_import, only: %i[show parse confirm cancel dry_run update_mapping
-                                        status export_rejects destroy back_to_mapping]
+                                        status export_rejects destroy back_to_mapping resume]
     before_action :load_targets, only: %i[index new create]
 
     def index
@@ -69,6 +69,12 @@ module DataPorter
       redirect_to import_path(@import)
     end
 
+    def resume
+      @import.update!(status: :pending)
+      DataPorter::ImportJob.perform_later(@import.id)
+      redirect_to import_path(@import)
+    end
+
     def dry_run
       @import.update!(status: :pending)
       DataPorter::DryRunJob.perform_later(@import.id)

data/app/models/data_porter/data_import.rb

@@ -53,12 +53,16 @@ module DataPorter
       records.group_by(&:status).transform_values(&:count)
     end
 
+    def resumable?
+      failed? && config&.dig("checkpoint", "processed").to_i.positive?
+    end
+
     def reset_to_mapping!
       update!(
         status: :mapping,
         records: [],
         report: StoreModels::Report.new,
-        config: (config || {}).except("progress")
+        config: (config || {}).except("progress", "checkpoint")
       )
     end
 

data/app/views/data_porter/imports/show.html.erb

@@ -103,8 +103,12 @@
   <% if @import.failed? %>
     <%= raw DataPorter::Components::Shared::FailureAlert.new(report: @import.report).call %>
     <div class="dp-actions">
+      <% if @import.resumable? %>
+        <%= button_to t("data_porter.imports.resume"), resume_import_path(@import),
+              method: :post, class: "dp-btn dp-btn--primary" %>
+      <% end %>
       <%= button_to t("data_porter.imports.retry"), parse_import_path(@import),
-            method: :post, class: "dp-btn dp-btn--primary" %>
+            method: :post, class: "dp-btn dp-btn--secondary" %>
       <%= button_to t("data_porter.imports.delete"), import_path(@import),
             method: :delete, class: "dp-btn dp-btn--danger",
             data: { turbo_confirm: t("data_porter.imports.delete_confirm") } %>

data/config/locales/en.yml

@@ -10,6 +10,7 @@ en:
       delete: "Delete"
       delete_confirm: "Delete this import?"
       retry: "Retry"
+      resume: "Resume"
       start_import: "Start Import"
       confirm_import: "Confirm Import"
       dry_run: "Dry Run"

data/config/locales/fr.yml

@@ -10,6 +10,7 @@ fr:
       delete: "Supprimer"
       delete_confirm: "Supprimer cet import ?"
       retry: "Réessayer"
+      resume: "Reprendre"
       start_import: "Lancer l'import"
       confirm_import: "Confirmer l'import"
       dry_run: "Essai à blanc"

data/config/routes.rb

@@ -10,6 +10,7 @@ DataPorter::Engine.routes.draw do
       post :cancel
       post :back_to_mapping
       post :dry_run
+      post :resume
       patch :update_mapping
       get :status
       get :export_rejects

data/lib/data_porter/orchestrator/bulk_importer.rb

@@ -7,46 +7,54 @@ module DataPorter
 
       def import_bulk
         importable = @data_import.importable_records
-        context = build_context
-        config = @target.class._bulk_config
-        results = { created: 0, errored: 0 }
-        total = importable.size
-        processed = 0
-
-        importable.each_slice(config[:batch_size]) do |batch|
-          persist_batch_with_fallback(batch, context, config, results)
-          processed += batch.size
-          broadcast_progress(processed, total)
-        end
+        checkpoint = load_checkpoint
+        @bulk_state = build_bulk_state(importable, checkpoint)
+
+        process_batches(importable.drop(checkpoint[:processed]))
+        finalize_import(@bulk_state[:results])
+      end
+
+      def build_bulk_state(importable, checkpoint)
+        {
+          context: build_context,
+          bulk_config: @target.class._bulk_config,
+          results: seed_results(checkpoint),
+          total: importable.size,
+          processed: checkpoint[:processed]
+        }
+      end
 
-        finalize_import(results)
+      def process_batches(records)
+        records.each_slice(@bulk_state[:bulk_config][:batch_size]) do |batch|
+          persist_batch_with_fallback(batch)
+          @bulk_state[:processed] += batch.size
+          broadcast_progress(@bulk_state[:processed], @bulk_state[:total], results: @bulk_state[:results])
+        end
       end
 
-      def persist_batch_with_fallback(batch, context, config, results)
-        @target.persist_batch(batch, context: context)
-        results[:created] += batch.size
+      def persist_batch_with_fallback(batch)
+        @target.persist_batch(batch, context: @bulk_state[:context])
+        @bulk_state[:results][:created] += batch.size
       rescue StandardError => e
-        handle_batch_failure(batch, context, config, results, e)
+        handle_batch_failure(batch, e)
       end
 
-      def handle_batch_failure(batch, context, config, results, error)
-        if config[:on_conflict] == :fail_batch
-          fail_batch(batch, results, error)
+      def handle_batch_failure(batch, error)
+        if @bulk_state[:bulk_config][:on_conflict] == :fail_batch
+          fail_batch(batch, error)
         else
-          retry_per_record(batch, context, results)
+          retry_per_record(batch)
         end
       end
 
-      def fail_batch(batch, results, error)
-        batch.each do |record|
-          record.add_error(error.message)
-        end
-        results[:errored] += batch.size
+      def fail_batch(batch, error)
+        batch.each { |record| record.add_error(error.message) }
+        @bulk_state[:results][:errored] += batch.size
       end
 
-      def retry_per_record(batch, context, results)
+      def retry_per_record(batch)
         batch.each do |record|
-          persist_record(record, context, results)
+          persist_record(record, @bulk_state[:context], @bulk_state[:results])
         end
       end
     end

data/lib/data_porter/orchestrator/importer.rb

@@ -18,12 +18,14 @@ module DataPorter
       def import_per_record
         importable = @data_import.importable_records
         context = build_context
-        results = { created: 0, errored: 0 }
+        checkpoint = load_checkpoint
+        results = seed_results(checkpoint)
+        remaining = importable.drop(checkpoint[:processed])
         total = importable.size
 
-        importable.each_with_index do |record, index|
+        remaining.each_with_index do |record, index|
           persist_record(record, context, results)
-          broadcast_progress(index + 1, total)
+          broadcast_progress(checkpoint[:processed] + index + 1, total, results: results)
         end
 
         finalize_import(results)
@@ -45,6 +47,7 @@ module DataPorter
       end
 
       def finalize_import(results)
+        clear_checkpoint
         @data_import.update!(status: :completed)
         @broadcaster.success
         WebhookNotifier.notify(@data_import, "import.completed")
@@ -66,6 +69,25 @@ module DataPorter
         report.errored_count = results[:errored]
         @data_import.update!(report: report)
       end
+
+      def load_checkpoint
+        cp = @data_import.config&.dig("checkpoint") || {}
+        {
+          processed: cp["processed"].to_i,
+          created: cp["created"].to_i,
+          errored: cp["errored"].to_i
+        }
+      end
+
+      def seed_results(checkpoint)
+        { created: checkpoint[:created], errored: checkpoint[:errored] }
+      end
+
+      def clear_checkpoint
+        config = @data_import.config || {}
+        config.delete("checkpoint")
+        @data_import.update_column(:config, config)
+      end
     end
   end
 end

data/lib/data_porter/orchestrator.rb

@@ -32,6 +32,7 @@ module DataPorter
     def parse!
       @data_import.parsing!
       records = build_records
+      clear_stale_import_data
       @data_import.update!(records: records, status: :previewing)
       build_report
       WebhookNotifier.notify(@data_import, "import.parsed")
@@ -92,18 +93,36 @@ module DataPorter
       DataPorter.configuration.context_builder&.call(@data_import)
     end
 
-    def broadcast_progress(current, total)
-      percentage = ((current.to_f / total) * 100).round
+    def broadcast_progress(current, total, results: nil)
       config = @data_import.config || {}
-      config["progress"] = { "current" => current, "total" => total, "percentage" => percentage }
+      config["progress"] = { "current" => current, "total" => total, "percentage" => pct(current, total) }
+      save_checkpoint(config, current, results) if results
       @data_import.update_column(:config, config)
       @broadcaster.progress(current, total)
     end
 
+    def pct(current, total)
+      ((current.to_f / total) * 100).round
+    end
+
+    def save_checkpoint(config, processed, results)
+      config["checkpoint"] = {
+        "processed" => processed,
+        "created" => results[:created],
+        "errored" => results[:errored]
+      }
+    end
+
+    def clear_stale_import_data
+      config = @data_import.config || {}
+      config.delete("checkpoint")
+      config.delete("progress")
+      @data_import.config = config
+    end
+
     def handle_failure(error)
-      report = StoreModels::Report.new(
-        error_reports: [StoreModels::Error.new(message: error.message)]
-      )
+      report = @data_import.report || StoreModels::Report.new
+      report.error_reports = [StoreModels::Error.new(message: error.message)]
       @data_import.update!(status: :failed, report: report)
       @broadcaster.failure(error.message)
       WebhookNotifier.notify(@data_import, "import.failed")

data/lib/data_porter/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DataPorter
-  VERSION = "2.5.1"
+  VERSION = "2.6.0"
 end

data/mkdocs.yml

@@ -94,7 +94,7 @@ nav:
       - Column Mapping: MAPPING.md
       - Views & Theming: VIEWS.md
       - Routes: routes.md
-  - Advanced: ADVANCED.md
+      - Advanced: ADVANCED.md
   - Roadmap: ROADMAP.md
   - Changelog: changelog.md
   - Contributing: contributing.md

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_porter
 version: !ruby/object:Gem::Version
-  version: 2.5.1
+  version: 2.6.0
 platform: ruby
 authors:
 - Seryl Lounis