data_porter 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. checksums.yaml +7 -0
  2. data/.claude/commands/blog-status.md +10 -0
  3. data/.claude/commands/blog.md +109 -0
  4. data/.claude/commands/task-done.md +27 -0
  5. data/.claude/commands/tm/add-dependency.md +58 -0
  6. data/.claude/commands/tm/add-subtask.md +79 -0
  7. data/.claude/commands/tm/add-task.md +81 -0
  8. data/.claude/commands/tm/analyze-complexity.md +124 -0
  9. data/.claude/commands/tm/analyze-project.md +100 -0
  10. data/.claude/commands/tm/auto-implement-tasks.md +100 -0
  11. data/.claude/commands/tm/command-pipeline.md +80 -0
  12. data/.claude/commands/tm/complexity-report.md +120 -0
  13. data/.claude/commands/tm/convert-task-to-subtask.md +74 -0
  14. data/.claude/commands/tm/expand-all-tasks.md +52 -0
  15. data/.claude/commands/tm/expand-task.md +52 -0
  16. data/.claude/commands/tm/fix-dependencies.md +82 -0
  17. data/.claude/commands/tm/help.md +101 -0
  18. data/.claude/commands/tm/init-project-quick.md +49 -0
  19. data/.claude/commands/tm/init-project.md +53 -0
  20. data/.claude/commands/tm/install-taskmaster.md +118 -0
  21. data/.claude/commands/tm/learn.md +106 -0
  22. data/.claude/commands/tm/list-tasks-by-status.md +42 -0
  23. data/.claude/commands/tm/list-tasks-with-subtasks.md +30 -0
  24. data/.claude/commands/tm/list-tasks.md +46 -0
  25. data/.claude/commands/tm/next-task.md +69 -0
  26. data/.claude/commands/tm/parse-prd-with-research.md +51 -0
  27. data/.claude/commands/tm/parse-prd.md +52 -0
  28. data/.claude/commands/tm/project-status.md +67 -0
  29. data/.claude/commands/tm/quick-install-taskmaster.md +23 -0
  30. data/.claude/commands/tm/remove-all-subtasks.md +94 -0
  31. data/.claude/commands/tm/remove-dependency.md +65 -0
  32. data/.claude/commands/tm/remove-subtask.md +87 -0
  33. data/.claude/commands/tm/remove-subtasks.md +89 -0
  34. data/.claude/commands/tm/remove-task.md +110 -0
  35. data/.claude/commands/tm/setup-models.md +52 -0
  36. data/.claude/commands/tm/show-task.md +85 -0
  37. data/.claude/commands/tm/smart-workflow.md +58 -0
  38. data/.claude/commands/tm/sync-readme.md +120 -0
  39. data/.claude/commands/tm/tm-main.md +147 -0
  40. data/.claude/commands/tm/to-cancelled.md +58 -0
  41. data/.claude/commands/tm/to-deferred.md +50 -0
  42. data/.claude/commands/tm/to-done.md +47 -0
  43. data/.claude/commands/tm/to-in-progress.md +39 -0
  44. data/.claude/commands/tm/to-pending.md +35 -0
  45. data/.claude/commands/tm/to-review.md +43 -0
  46. data/.claude/commands/tm/update-single-task.md +122 -0
  47. data/.claude/commands/tm/update-task.md +75 -0
  48. data/.claude/commands/tm/update-tasks-from-id.md +111 -0
  49. data/.claude/commands/tm/validate-dependencies.md +72 -0
  50. data/.claude/commands/tm/view-models.md +52 -0
  51. data/.env.example +12 -0
  52. data/.mcp.json +24 -0
  53. data/.taskmaster/CLAUDE.md +435 -0
  54. data/.taskmaster/config.json +44 -0
  55. data/.taskmaster/docs/prd.txt +2044 -0
  56. data/.taskmaster/state.json +6 -0
  57. data/.taskmaster/tasks/task_001.md +19 -0
  58. data/.taskmaster/tasks/task_002.md +19 -0
  59. data/.taskmaster/tasks/task_003.md +19 -0
  60. data/.taskmaster/tasks/task_004.md +19 -0
  61. data/.taskmaster/tasks/task_005.md +19 -0
  62. data/.taskmaster/tasks/task_006.md +19 -0
  63. data/.taskmaster/tasks/task_007.md +19 -0
  64. data/.taskmaster/tasks/task_008.md +19 -0
  65. data/.taskmaster/tasks/task_009.md +19 -0
  66. data/.taskmaster/tasks/task_010.md +19 -0
  67. data/.taskmaster/tasks/task_011.md +19 -0
  68. data/.taskmaster/tasks/task_012.md +19 -0
  69. data/.taskmaster/tasks/task_013.md +19 -0
  70. data/.taskmaster/tasks/task_014.md +19 -0
  71. data/.taskmaster/tasks/task_015.md +19 -0
  72. data/.taskmaster/tasks/task_016.md +19 -0
  73. data/.taskmaster/tasks/task_017.md +19 -0
  74. data/.taskmaster/tasks/task_018.md +19 -0
  75. data/.taskmaster/tasks/task_019.md +19 -0
  76. data/.taskmaster/tasks/task_020.md +19 -0
  77. data/.taskmaster/tasks/tasks.json +299 -0
  78. data/.taskmaster/templates/example_prd.txt +47 -0
  79. data/.taskmaster/templates/example_prd_rpg.txt +511 -0
  80. data/CHANGELOG.md +29 -0
  81. data/CLAUDE.md +65 -0
  82. data/CODE_OF_CONDUCT.md +10 -0
  83. data/CONTRIBUTING.md +49 -0
  84. data/LICENSE +21 -0
  85. data/README.md +463 -0
  86. data/Rakefile +12 -0
  87. data/app/assets/stylesheets/data_porter/application.css +646 -0
  88. data/app/channels/data_porter/import_channel.rb +10 -0
  89. data/app/controllers/data_porter/imports_controller.rb +68 -0
  90. data/app/javascript/data_porter/progress_controller.js +33 -0
  91. data/app/jobs/data_porter/dry_run_job.rb +12 -0
  92. data/app/jobs/data_porter/import_job.rb +12 -0
  93. data/app/jobs/data_porter/parse_job.rb +12 -0
  94. data/app/models/data_porter/data_import.rb +49 -0
  95. data/app/views/data_porter/imports/index.html.erb +142 -0
  96. data/app/views/data_porter/imports/new.html.erb +88 -0
  97. data/app/views/data_porter/imports/show.html.erb +49 -0
  98. data/config/database.yml +3 -0
  99. data/config/routes.rb +12 -0
  100. data/docs/SPEC.md +2012 -0
  101. data/docs/UI.md +32 -0
  102. data/docs/blog/001-why-build-a-data-import-engine.md +166 -0
  103. data/docs/blog/002-scaffolding-a-rails-engine.md +188 -0
  104. data/docs/blog/003-configuration-dsl.md +222 -0
  105. data/docs/blog/004-store-model-jsonb.md +237 -0
  106. data/docs/blog/005-target-dsl.md +284 -0
  107. data/docs/blog/006-parsing-csv-sources.md +300 -0
  108. data/docs/blog/007-orchestrator.md +247 -0
  109. data/docs/blog/008-actioncable-stimulus.md +376 -0
  110. data/docs/blog/009-phlex-ui-components.md +446 -0
  111. data/docs/blog/010-controllers-routing.md +374 -0
  112. data/docs/blog/011-generators.md +364 -0
  113. data/docs/blog/012-json-api-sources.md +323 -0
  114. data/docs/blog/013-testing-rails-engine.md +618 -0
  115. data/docs/blog/014-dry-run.md +307 -0
  116. data/docs/blog/015-publishing-retro.md +264 -0
  117. data/docs/blog/016-erb-view-templates.md +431 -0
  118. data/docs/blog/017-showcase-final-retro.md +220 -0
  119. data/docs/blog/BACKLOG.md +8 -0
  120. data/docs/blog/SERIES.md +154 -0
  121. data/docs/screenshots/index-with-previewing.jpg +0 -0
  122. data/docs/screenshots/index.jpg +0 -0
  123. data/docs/screenshots/modal-new-import.jpg +0 -0
  124. data/docs/screenshots/preview.jpg +0 -0
  125. data/lib/data_porter/broadcaster.rb +29 -0
  126. data/lib/data_porter/components/base.rb +10 -0
  127. data/lib/data_porter/components/failure_alert.rb +20 -0
  128. data/lib/data_porter/components/preview_table.rb +54 -0
  129. data/lib/data_porter/components/progress_bar.rb +33 -0
  130. data/lib/data_porter/components/results_summary.rb +19 -0
  131. data/lib/data_porter/components/status_badge.rb +16 -0
  132. data/lib/data_porter/components/summary_cards.rb +30 -0
  133. data/lib/data_porter/components.rb +14 -0
  134. data/lib/data_porter/configuration.rb +25 -0
  135. data/lib/data_porter/dsl/api_config.rb +25 -0
  136. data/lib/data_porter/dsl/column.rb +17 -0
  137. data/lib/data_porter/engine.rb +15 -0
  138. data/lib/data_porter/orchestrator.rb +141 -0
  139. data/lib/data_porter/record_validator.rb +32 -0
  140. data/lib/data_porter/registry.rb +33 -0
  141. data/lib/data_porter/sources/api.rb +49 -0
  142. data/lib/data_porter/sources/base.rb +35 -0
  143. data/lib/data_porter/sources/csv.rb +43 -0
  144. data/lib/data_porter/sources/json.rb +45 -0
  145. data/lib/data_porter/sources.rb +20 -0
  146. data/lib/data_porter/store_models/error.rb +13 -0
  147. data/lib/data_porter/store_models/import_record.rb +52 -0
  148. data/lib/data_porter/store_models/report.rb +21 -0
  149. data/lib/data_porter/target.rb +89 -0
  150. data/lib/data_porter/type_validator.rb +46 -0
  151. data/lib/data_porter/version.rb +5 -0
  152. data/lib/data_porter.rb +32 -0
  153. data/lib/generators/data_porter/install/install_generator.rb +33 -0
  154. data/lib/generators/data_porter/install/templates/create_data_porter_imports.rb.erb +21 -0
  155. data/lib/generators/data_porter/install/templates/initializer.rb +30 -0
  156. data/lib/generators/data_porter/target/target_generator.rb +44 -0
  157. data/lib/generators/data_porter/target/templates/target.rb.tt +20 -0
  158. data/sig/data_porter.rbs +4 -0
  159. metadata +274 -0
data/docs/SPEC.md ADDED
@@ -0,0 +1,2012 @@
1
+ # DataPorter - Rails Engine pour l'import de donnees
2
+
3
+ > Gem Rails mountable qui fournit un workflow complet d'import de donnees en 3 etapes :
4
+ > **Upload/Config -> Preview visuel -> Validation & Creation**
5
+
6
+ ---
7
+
8
+ ## Table des matieres
9
+
10
+ - [Probleme](#probleme)
11
+ - [Solution](#solution)
12
+ - [Architecture](#architecture)
13
+ - [Installation (cote app hote)](#installation-cote-app-hote)
14
+ - [Usage : definir un Target](#usage--definir-un-target)
15
+ - [DSL Target en detail](#dsl-target-en-detail)
16
+ - [Sources supportees](#sources-supportees)
17
+ - [Architecture interne de la gem](#architecture-interne-de-la-gem)
18
+ - [Engine](#engine)
19
+ - [Modele DataImport](#modele-dataimport)
20
+ - [State Machine](#state-machine)
21
+ - [Orchestrator](#orchestrator)
22
+ - [Registry & Auto-discovery](#registry--auto-discovery)
23
+ - [StoreModels (JSONB)](#storemodels-jsonb)
24
+ - [Sources](#sources-1)
25
+ - [Controllers](#controllers)
26
+ - [Vues & Preview](#vues--preview)
27
+ - [Jobs (ActiveJob)](#jobs-activejob)
28
+ - [ActionCable & Progress](#actioncable--progress)
29
+ - [Multi-tenancy](#multi-tenancy)
30
+ - [Generators](#generators)
31
+ - [Override & Extension](#override--extension)
32
+ - [Comparaison avec maintenance_tasks](#comparaison-avec-maintenance_tasks)
33
+ - [Structure fichiers de la gem](#structure-fichiers-de-la-gem)
34
+ - [Dry Run (v0.2)](#dry-run-v02)
35
+ - [Rollback (v0.3 / v0.4)](#rollback-v03--v04)
36
+ - [Roadmap](#roadmap)
37
+
38
+ ---
39
+
40
+ ## Probleme
41
+
42
+ Dans beaucoup d'apps Rails, on a besoin d'importer de la donnee via differents moyens (CSV, JSON, API/scraping) pour creer des instances de modeles. Le pattern se repete a chaque fois :
43
+
44
+ 1. Uploader ou recuperer les donnees
45
+ 2. Les parser et les valider
46
+ 3. Les afficher pour verification humaine
47
+ 4. Les persister en base
48
+
49
+ Ce workflow est reimplemente a chaque fois, pour chaque modele, sans mutualisation.
50
+
51
+ ## Solution
52
+
53
+ **DataPorter** est un Rails Engine mountable qui factorise tout le workflow d'import. La gem fournit l'infrastructure (modele, state machine, jobs, UI, progress) et l'app hote ne definit que la partie metier : quels modeles importer et comment les creer.
54
+
55
+ ```
56
+ 1 fichier Target = 1 type d'import operationnel
57
+ ```
58
+
59
+ ---
60
+
61
+ ## Architecture
62
+
63
+ ```
64
+ +---------------------------------------------+
65
+ | GEM : DataPorter |
66
+ | |
67
+ | - Rails Engine (mountable, isolate_ns) |
68
+ | - Modele DataImport + migration generator |
69
+ | - State machine (enum) |
70
+ | - Sources : CSV, JSON, API |
71
+ | - StoreModels : ImportRecord, Error, Report|
72
+ | - Orchestrator |
73
+ | - Workers (ActiveJob) |
74
+ | - ActionCable channel + broadcaster |
75
+ | - UI : liste, form, preview, progress |
76
+ | - DSL pour declarer des Targets |
77
+ | - Registry (auto-discovery) |
78
+ | - Generators (install, target) |
79
+ +---------------------------------------------+
80
+ ^
81
+ | mount + configure + define targets
82
+ |
83
+ +---------------------------------------------+
84
+ | APP HOTE |
85
+ | |
86
+ | - config/initializers/data_porter.rb |
87
+ | - app/importers/guests_target.rb |
88
+ | - app/importers/stays_target.rb |
89
+ | - app/importers/vendors_target.rb |
90
+ | - Auth/layout (parent controller) |
91
+ | - Styling override (optionnel) |
92
+ +---------------------------------------------+
93
+ ```
94
+
95
+ ---
96
+
97
+ ## Installation (cote app hote)
98
+
99
+ ### Gemfile
100
+
101
+ ```ruby
102
+ gem 'data_porter', '~> 0.1'
103
+ ```
104
+
105
+ ### Generator
106
+
107
+ ```bash
108
+ rails generate data_porter:install
109
+ rails db:migrate
110
+ ```
111
+
112
+ Le generator cree :
113
+
114
+ - La migration `data_porter_imports`
115
+ - `config/initializers/data_porter.rb`
116
+ - Le dossier `app/importers/`
117
+ - La route mount dans `config/routes.rb`
118
+
119
+ ### Configuration
120
+
121
+ ```ruby
122
+ # config/initializers/data_porter.rb
123
+ DataPorter.configure do |config|
124
+ # Controller parent dont herite le controller de la gem
125
+ # Permet d'heriter l'auth, les layouts, les helpers
126
+ config.parent_controller = 'AdminController'
127
+
128
+ # Backend de jobs (la gem utilise ActiveJob, agnostique du backend)
129
+ config.queue_name = :imports
130
+
131
+ # Service ActiveStorage pour stocker les fichiers uploades
132
+ config.storage_service = :local
133
+
134
+ # Prefix ActionCable
135
+ config.cable_channel_prefix = 'data_porter'
136
+
137
+ # Builder de contexte : injecte des donnees metier dans les targets
138
+ # Le block recoit le controller courant
139
+ config.context_builder = ->(controller) {
140
+ OpenStruct.new(
141
+ hotel: controller.current_user.hotel,
142
+ user: controller.current_user
143
+ )
144
+ }
145
+
146
+ # Nombre max de records affichables en preview (pagination au-dela)
147
+ config.preview_limit = 500
148
+
149
+ # Activer/desactiver les sources disponibles
150
+ config.enabled_sources = [:csv, :json, :api]
151
+ end
152
+ ```
153
+
154
+ ### Routes
155
+
156
+ ```ruby
157
+ # config/routes.rb
158
+ mount DataPorter::Engine, at: '/admin/imports'
159
+
160
+ # Ou derriere un scope authentifie
161
+ authenticate :user, ->(u) { u.admin? } do
162
+ mount DataPorter::Engine, at: '/admin/imports'
163
+ end
164
+ ```
165
+
166
+ ---
167
+
168
+ ## Usage : definir un Target
169
+
170
+ Un Target = un type d'import. Un fichier, une classe, c'est tout.
171
+
172
+ ### Exemple simple : Guests
173
+
174
+ ```ruby
175
+ # app/importers/guests_target.rb
176
+ class GuestsTarget < DataPorter::Target
177
+ label "Guests"
178
+ model Guest
179
+ icon "fas fa-users"
180
+
181
+ sources :csv, :json
182
+
183
+ columns do
184
+ column :first_name, type: :string, required: true
185
+ column :last_name, type: :string, required: true
186
+ column :email, type: :email
187
+ column :phone, type: :phone
188
+ column :language, type: :string, in: %w[fr en es de it]
189
+ column :date_of_birth, type: :date, format: '%d/%m/%Y'
190
+ end
191
+
192
+ csv_mapping do
193
+ map "Prenom" => :first_name
194
+ map "Nom" => :last_name
195
+ map "E-mail" => :email
196
+ map "Telephone" => :phone
197
+ map "Langue" => :language
198
+ map "Date naissance" => :date_of_birth
199
+ end
200
+
201
+ deduplicate_by :email, :phone
202
+
203
+ def persist(record, context:)
204
+ Guest.create!(
205
+ hotel: context.hotel,
206
+ **record.attributes
207
+ )
208
+ end
209
+ end
210
+ ```
211
+
212
+ ### Exemple avec source API : Scraper Airbnb
213
+
214
+ ```ruby
215
+ # app/importers/airbnb_stays_target.rb
216
+ class AirbnbStaysTarget < DataPorter::Target
217
+ label "Stays (Airbnb Scraper)"
218
+ model Stay
219
+ icon "fas fa-bed"
220
+
221
+ sources :api
222
+
223
+ api_config do
224
+ endpoint ->(params) { "https://scraper.internal/api/stays?housing_id=#{params[:housing_id]}" }
225
+ headers { { 'Authorization' => "Bearer #{ENV['SCRAPER_TOKEN']}" } }
226
+ response_root :stays
227
+ end
228
+
229
+ params do
230
+ param :housing_id, type: :integer, required: true, label: "Housing"
231
+ end
232
+
233
+ columns do
234
+ column :external_confirmation_code, type: :string, required: true
235
+ column :guest_name, type: :string, required: true
236
+ column :check_in, type: :date, required: true
237
+ column :check_out, type: :date, required: true
238
+ column :total_price, type: :decimal
239
+ column :nb_guests, type: :integer
240
+ column :source, type: :string
241
+ end
242
+
243
+ deduplicate_by :external_confirmation_code
244
+
245
+ def persist(record, context:)
246
+ Stays::Upsert.call(
247
+ hotel: context.hotel,
248
+ housing_id: record.housing_id,
249
+ **record.attributes
250
+ )
251
+ end
252
+ end
253
+ ```
254
+
255
+ ### Exemple JSON : Vendors
256
+
257
+ ```ruby
258
+ # app/importers/vendors_target.rb
259
+ class VendorsTarget < DataPorter::Target
260
+ label "Vendors"
261
+ model Vendor
262
+ icon "fas fa-store"
263
+
264
+ sources :csv, :json
265
+
266
+ columns do
267
+ column :name, type: :string, required: true
268
+ column :email, type: :email
269
+ column :phone, type: :phone
270
+ column :address, type: :string
271
+ column :category, type: :string, in: %w[cleaning maintenance catering transport]
272
+ column :website, type: :url
273
+ end
274
+
275
+ # Hook optionnel : transformation apres parsing, avant preview
276
+ def transform(record)
277
+ record.data[:name] = record.data[:name]&.titleize
278
+ record.data[:category] = record.data[:category]&.downcase
279
+ record
280
+ end
281
+
282
+ # Hook optionnel : validation metier custom
283
+ def validate(record)
284
+ if record.data[:email].blank? && record.data[:phone].blank?
285
+ record.add_warning("Ni email ni telephone renseigne")
286
+ end
287
+ end
288
+
289
+ def persist(record, context:)
290
+ Vendor.create!(hotel: context.hotel, **record.attributes)
291
+ end
292
+ end
293
+ ```
294
+
295
+ ### Generator de target
296
+
297
+ ```bash
298
+ rails generate data_porter:target vendors name:string:required email:email phone:phone
299
+ ```
300
+
301
+ Genere le squelette dans `app/importers/vendors_target.rb`.
302
+
303
+ ---
304
+
305
+ ## DSL Target en detail
306
+
307
+ ### `label(string)`
308
+
309
+ Nom affiche dans l'UI.
310
+
311
+ ### `model(class)`
312
+
313
+ Classe ActiveRecord cible. Utilise pour l'affichage et la detection automatique des colonnes (optionnel).
314
+
315
+ ### `icon(string)`
316
+
317
+ Classe CSS d'icone (FontAwesome). Affiche dans la liste des targets.
318
+
319
+ ### `sources(*types)`
320
+
321
+ Types de sources acceptes pour ce target. Valeurs : `:csv`, `:json`, `:api`.
322
+
323
+ ### `columns(&block)`
324
+
325
+ Definition des colonnes attendues. Chaque colonne a :
326
+
327
+ | Option | Type | Description |
328
+ | ---------- | ------- | ----------------------------------------------------------------------------------------------- |
329
+ | `type` | Symbol | `:string`, `:integer`, `:decimal`, `:date`, `:datetime`, `:email`, `:phone`, `:url`, `:boolean` |
330
+ | `required` | Boolean | Champ obligatoire (default: false) |
331
+ | `in` | Array | Valeurs autorisees |
332
+ | `format` | String | Format de parsing (pour les dates) |
333
+ | `label` | String | Label affiche en preview (default: humanize du nom) |
334
+
335
+ ### `csv_mapping(&block)`
336
+
337
+ Mapping entre les headers CSV et les noms de colonnes internes.
338
+ Si absent, la gem tente un matching automatique (parameterize + fuzzy).
339
+
340
+ ### `deduplicate_by(*keys)`
341
+
342
+ Colonnes utilisees pour detecter les doublons avec les records existants en base.
343
+ En preview, les doublons sont marques avec le status `:duplicate`.
344
+
345
+ ### `params(&block)`
346
+
347
+ Parametres additionnels demandes a l'utilisateur au moment du lancement.
348
+ Utilise pour les sources API (ex: housing_id) ou pour filtrer les donnees.
349
+
350
+ ### `api_config(&block)`
351
+
352
+ Configuration specifique a la source API :
353
+
354
+ - `endpoint` : URL ou lambda recevant les params
355
+ - `headers` : Hash ou lambda
356
+ - `response_root` : Cle JSON contenant le tableau de records
357
+
358
+ ### Hooks overridables
359
+
360
+ | Hook | Moment | Usage |
361
+ | ----------------------------------- | -------------------------------- | ------------------------- |
362
+ | `transform(record)` | Apres parsing, avant preview | Normalisation des donnees |
363
+ | `validate(record)` | Apres transform, avant preview | Validation metier custom |
364
+ | `persist(record, context:)` | A l'import final | Creation en base |
365
+ | `after_import(results, context:)` | Apres l'import complet | Notifications, logs, etc. |
366
+ | `on_error(record, error, context:)` | Si persist echoue pour un record | Gestion d'erreur custom |
367
+
368
+ ---
369
+
370
+ ## Sources supportees
371
+
372
+ ### CSV
373
+
374
+ ```
375
+ Upload d'un fichier CSV via ActiveStorage.
376
+ Headers mappes aux colonnes via csv_mapping ou matching auto.
377
+ Options CSV Ruby configurables (separateur, encoding, skip_lines...).
378
+ ```
379
+
380
+ ### JSON
381
+
382
+ ```
383
+ Upload d'un fichier JSON ou saisie en textarea.
384
+ Attend un tableau d'objets a la racine ou sous une cle configurable.
385
+ Les cles des objets sont mappees aux colonnes.
386
+ ```
387
+
388
+ ### API
389
+
390
+ ```
391
+ Appel HTTP a un endpoint configure dans api_config.
392
+ Les params du target sont injectes dans l'URL.
393
+ La reponse JSON est parsee comme un tableau de records.
394
+ Ideal pour les scrapers et les APIs internes.
395
+ ```
396
+
397
+ ---
398
+
399
+ ## Architecture interne de la gem
400
+
401
+ ### Engine
402
+
403
+ ```ruby
404
+ # lib/data_porter/engine.rb
405
+ module DataPorter
406
+ class Engine < ::Rails::Engine
407
+ isolate_namespace DataPorter
408
+
409
+ initializer 'data_porter.assets' do |app|
410
+ if app.config.respond_to?(:assets)
411
+ app.config.assets.precompile += %w[
412
+ data_porter/application.css
413
+ data_porter/application.js
414
+ ]
415
+ end
416
+ end
417
+
418
+ initializer 'data_porter.active_storage' do
419
+ ActiveSupport.on_load(:active_storage_blob) do
420
+ # config storage
421
+ end
422
+ end
423
+
424
+ config.to_prepare do
425
+ Dir[Rails.root.join('app/importers/*_target.rb')].each { |f| require f }
426
+ end
427
+ end
428
+ end
429
+ ```
430
+
431
+ ### Modele DataImport
432
+
433
+ ```ruby
434
+ # app/models/data_porter/data_import.rb
435
+ module DataPorter
436
+ class DataImport < ActiveRecord::Base
437
+ self.table_name = 'data_porter_imports'
438
+
439
+ belongs_to :user, polymorphic: true
440
+
441
+ has_one_attached :file
442
+
443
+ enum :status, {
444
+ pending: 0,
445
+ parsing: 1,
446
+ previewing: 2,
447
+ importing: 3,
448
+ completed: 4,
449
+ failed: 5
450
+ }
451
+
452
+ attribute :records, DataPorter::StoreModels::ImportRecord.to_array_type, default: []
453
+ attribute :report, DataPorter::StoreModels::Report.to_type
454
+
455
+ serialize :config, coder: JSON
456
+
457
+ validates :target_key, presence: true
458
+ validates :source_type, presence: true, inclusion: { in: %w[csv json api] }
459
+
460
+ def target_class
461
+ DataPorter::Registry.find(target_key)
462
+ end
463
+
464
+ def source_class
465
+ DataPorter::Sources.resolve(source_type)
466
+ end
467
+
468
+ def previewable?
469
+ previewing? && records.any?
470
+ end
471
+
472
+ def importable_records
473
+ records.select { |r| r.status == 'complete' }
474
+ end
475
+
476
+ def records_summary
477
+ records.group_by(&:status).transform_values(&:count)
478
+ end
479
+ end
480
+ end
481
+ ```
482
+
483
+ #### Migration
484
+
485
+ ```ruby
486
+ # db/migrate/xxx_create_data_porter_imports.rb
487
+ class CreateDataPorterImports < ActiveRecord::Migration[7.2]
488
+ def change
489
+ create_table :data_porter_imports do |t|
490
+ t.string :target_key, null: false
491
+ t.string :source_type, null: false, default: 'csv'
492
+ t.integer :status, null: false, default: 0
493
+ t.jsonb :records, null: false, default: []
494
+ t.jsonb :report, null: false, default: {}
495
+ t.jsonb :config, null: false, default: {}
496
+
497
+ t.references :user, polymorphic: true, null: false
498
+
499
+ t.timestamps
500
+ end
501
+
502
+ add_index :data_porter_imports, :status
503
+ add_index :data_porter_imports, :target_key
504
+ end
505
+ end
506
+ ```
507
+
508
+ ### State Machine
509
+
510
+ ```
511
+ pending -----> parsing -----> previewing -----> importing -----> completed
512
+ | |
513
+ +--------------> failed <--------+
514
+ ```
515
+
516
+ | Status | Description | Declencheur |
517
+ | ------------ | ------------------------------------------------- | ------------------------ |
518
+ | `pending` | Import cree, en attente de lancement | `create` |
519
+ | `parsing` | Source en cours de parsing (async) | `ParseJob` |
520
+ | `previewing` | Donnees parsees, en attente de validation humaine | Fin du parsing |
521
+ | `importing` | Creation des records en cours (async) | Confirmation utilisateur |
522
+ | `completed` | Tous les records importes | Fin de l'import |
523
+ | `failed` | Erreur fatale pendant parsing ou import | Exception |
524
+
525
+ ### Orchestrator
526
+
527
+ Le chef d'orchestre qui coordonne source + target.
528
+
529
+ ```ruby
530
+ # lib/data_porter/orchestrator.rb
531
+ module DataPorter
532
+ class Orchestrator
533
+ def initialize(data_import)
534
+ @data_import = data_import
535
+ @source = data_import.source_class.new(data_import)
536
+ @target = data_import.target_class.new
537
+ end
538
+
539
+ def parse!
540
+ @data_import.parsing!
541
+ raw_rows = @source.fetch
542
+ records = raw_rows.each_with_index.map do |row, index|
543
+ record = DataPorter::StoreModels::ImportRecord.new(
544
+ line_number: index + 1,
545
+ data: @target.class._columns.each_with_object({}) do |col, hash|
546
+ hash[col.name] = row[col.name] || row[col.name.to_s]
547
+ end
548
+ )
549
+ record = @target.transform(record)
550
+ @target.validate(record)
551
+ record.run_validations!(@target.class._columns, @target.class._dedup_keys)
552
+ broadcast_progress(index + 1, raw_rows.size)
553
+ record
554
+ end
555
+
556
+ @data_import.update!(records: records, status: :previewing)
557
+ build_report
558
+ broadcast_success
559
+ rescue StandardError => e
560
+ handle_failure(e)
561
+ end
562
+
563
+ def import!
564
+ @data_import.importing!
565
+ importable = @data_import.importable_records
566
+ context = build_context
567
+
568
+ results = { created: 0, skipped: 0, errored: 0 }
569
+
570
+ importable.each_with_index do |record, index|
571
+ @target.persist(record, context: context)
572
+ record.status = 'imported'
573
+ results[:created] += 1
574
+ broadcast_progress(index + 1, importable.size)
575
+ rescue StandardError => e
576
+ record.status = 'error'
577
+ record.errors_list << DataPorter::StoreModels::Error.new(message: e.message)
578
+ @target.on_error(record, e, context: context)
579
+ results[:errored] += 1
580
+ end
581
+
582
+ @data_import.update!(status: :completed)
583
+ @target.after_import(results, context: context)
584
+ broadcast_success
585
+ rescue StandardError => e
586
+ handle_failure(e)
587
+ end
588
+
589
+ private
590
+
591
+ def broadcast_progress(current, total)
592
+ DataPorter::Broadcaster.new(@data_import.id).progress(current, total)
593
+ end
594
+
595
+ def broadcast_success
596
+ DataPorter::Broadcaster.new(@data_import.id).success
597
+ end
598
+
599
+ def handle_failure(error)
600
+ report = DataPorter::StoreModels::Report.new(
601
+ error_reports: [DataPorter::StoreModels::Error.new(message: error.message)]
602
+ )
603
+ @data_import.update!(status: :failed, report: report)
604
+ DataPorter::Broadcaster.new(@data_import.id).failure(error.message)
605
+ end
606
+
607
+ def build_context
608
+ DataPorter.configuration.context_builder&.call(@data_import)
609
+ end
610
+
611
+ def build_report
612
+ summary = @data_import.records_summary
613
+ report = DataPorter::StoreModels::Report.new(
614
+ records_count: @data_import.records.size,
615
+ complete_count: summary['complete'] || 0,
616
+ partial_count: summary['partial'] || 0,
617
+ missing_count: summary['missing'] || 0,
618
+ duplicate_count: summary['duplicate'] || 0
619
+ )
620
+ @data_import.update!(report: report)
621
+ end
622
+ end
623
+ end
624
+ ```
625
+
626
+ ### Registry & Auto-discovery
627
+
628
+ ```ruby
629
+ # lib/data_porter/registry.rb
630
+ module DataPorter
631
+ class Registry
632
+ class << self
633
+ def targets
634
+ @targets ||= {}
635
+ end
636
+
637
+ def register(key, klass)
638
+ targets[key.to_s] = klass
639
+ end
640
+
641
+ def find(key)
642
+ targets[key.to_s] || raise(TargetNotFound, "Target '#{key}' not found")
643
+ end
644
+
645
+ def available
646
+ targets.map { |key, klass| { key: key, label: klass._label, icon: klass._icon } }
647
+ end
648
+
649
+ def refresh!
650
+ @targets = nil
651
+ DataPorter::Target.descendants.each do |klass|
652
+ next if klass._label.nil?
653
+ register(klass._label.parameterize, klass)
654
+ end
655
+ end
656
+ end
657
+ end
658
+ end
659
+ ```
660
+
661
+ ### StoreModels (JSONB)
662
+
663
+ Donnees typees stockees en JSONB dans `data_porter_imports.records`.
664
+
665
+ ```ruby
666
+ # lib/data_porter/store_models/import_record.rb
667
+ module DataPorter
668
+ module StoreModels
669
+ class ImportRecord
670
+ include StoreModel::Model
671
+
672
+ attribute :line_number, :integer
673
+ attribute :status, :string, default: 'pending'
674
+ attribute :data, :json, default: {}
675
+ attribute :errors_list, DataPorter::StoreModels::Error.to_array_type, default: []
676
+ attribute :warnings, DataPorter::StoreModels::Error.to_array_type, default: []
677
+ attribute :target_id, :integer
678
+
679
+ STATUSES = %w[pending complete partial missing duplicate imported error].freeze
680
+
681
+ def complete? = status == 'complete'
682
+ def importable? = status == 'complete'
683
+
684
+ def add_error(message)
685
+ errors_list << DataPorter::StoreModels::Error.new(message: message)
686
+ end
687
+
688
+ def add_warning(message)
689
+ warnings << DataPorter::StoreModels::Error.new(message: message)
690
+ end
691
+
692
+ def attributes
693
+ data.symbolize_keys.compact
694
+ end
695
+
696
+ def run_validations!(columns, dedup_keys)
697
+ validate_required_columns!(columns)
698
+ validate_types!(columns)
699
+ validate_inclusions!(columns)
700
+ check_duplicates!(dedup_keys)
701
+ determine_status!
702
+ end
703
+
704
+ private
705
+
706
+ def validate_required_columns!(columns)
707
+ columns.select(&:required).each do |col|
708
+ if data[col.name].blank?
709
+ add_error("#{col.label} is required")
710
+ end
711
+ end
712
+ end
713
+
714
+ def validate_types!(columns)
715
+ columns.each do |col|
716
+ value = data[col.name]
717
+ next if value.blank?
718
+ unless TypeValidator.valid?(value, col.type, col.options)
719
+ add_error("#{col.label}: invalid #{col.type}")
720
+ end
721
+ end
722
+ end
723
+
724
+ def validate_inclusions!(columns)
725
+ columns.select { |c| c.options[:in] }.each do |col|
726
+ value = data[col.name]
727
+ next if value.blank?
728
+ unless col.options[:in].include?(value)
729
+ add_error("#{col.label}: must be one of #{col.options[:in].join(', ')}")
730
+ end
731
+ end
732
+ end
733
+
734
+ def check_duplicates!(dedup_keys)
735
+ return if dedup_keys.blank?
736
+ # la logique de dedup est geree par l'orchestrator
737
+ # qui a acces a la base et aux autres records du batch
738
+ end
739
+
740
+ def determine_status!
741
+ if errors_list.any? { |e| e.message.include?('is required') }
742
+ self.status = 'missing'
743
+ elsif errors_list.any?
744
+ self.status = 'partial'
745
+ else
746
+ self.status = 'complete'
747
+ end
748
+ end
749
+ end
750
+ end
751
+ end
752
+ ```
753
+
754
+ ```ruby
755
+ # lib/data_porter/store_models/error.rb
756
+ module DataPorter
757
+ module StoreModels
758
+ class Error
759
+ include StoreModel::Model
760
+
761
+ attribute :message, :string
762
+ end
763
+ end
764
+ end
765
+ ```
766
+
767
+ ```ruby
768
+ # lib/data_porter/store_models/report.rb
769
+ module DataPorter
770
+ module StoreModels
771
+ class Report
772
+ include StoreModel::Model
773
+
774
+ attribute :records_count, :integer, default: 0
775
+ attribute :complete_count, :integer, default: 0
776
+ attribute :partial_count, :integer, default: 0
777
+ attribute :missing_count, :integer, default: 0
778
+ attribute :duplicate_count, :integer, default: 0
779
+ attribute :imported_count, :integer, default: 0
780
+ attribute :errored_count, :integer, default: 0
781
+ attribute :error_reports, DataPorter::StoreModels::Error.to_array_type, default: []
782
+ end
783
+ end
784
+ end
785
+ ```
786
+
787
+ ### Sources
788
+
789
+ ```ruby
790
+ # lib/data_porter/sources/base.rb
791
+ module DataPorter
792
+ module Sources
793
+ class Base
794
+ def initialize(data_import)
795
+ @data_import = data_import
796
+ @target_class = data_import.target_class
797
+ end
798
+
799
+ def fetch
800
+ raise NotImplementedError
801
+ end
802
+
803
+ private
804
+
805
+ def apply_csv_mapping(row)
806
+ mappings = @target_class._csv_mappings || {}
807
+ if mappings.any?
808
+ mappings.each_with_object({}) do |(header, column), hash|
809
+ hash[column] = row[header]
810
+ end
811
+ else
812
+ row.to_h.transform_keys { |k| k.parameterize(separator: '_').to_sym }
813
+ end
814
+ end
815
+ end
816
+ end
817
+ end
818
+ ```
819
+
820
+ ```ruby
821
+ # lib/data_porter/sources/csv.rb
822
+ module DataPorter
823
+ module Sources
824
+ class Csv < Base
825
+ def fetch
826
+ file = @data_import.file
827
+ csv_options = @target_class._csv_options || { headers: true }
828
+
829
+ rows = []
830
+ ::CSV.parse(URI.parse(file.url).open, **csv_options) do |row|
831
+ rows << apply_csv_mapping(row)
832
+ end
833
+ rows
834
+ end
835
+ end
836
+ end
837
+ end
838
+ ```
839
+
840
+ ```ruby
841
+ # lib/data_porter/sources/json.rb
842
+ module DataPorter
843
+ module Sources
844
+ class Json < Base
845
+ def fetch
846
+ raw = if @data_import.file.attached?
847
+ @data_import.file.download
848
+ else
849
+ @data_import.config['raw_json']
850
+ end
851
+
852
+ parsed = JSON.parse(raw)
853
+ root = @target_class._json_root
854
+ records = root ? parsed.dig(*root.split('.')) : parsed
855
+
856
+ Array.wrap(records).map do |hash|
857
+ hash.transform_keys { |k| k.parameterize(separator: '_').to_sym }
858
+ end
859
+ end
860
+ end
861
+ end
862
+ end
863
+ ```
864
+
865
+ ```ruby
866
+ # lib/data_porter/sources/api.rb
867
+ module DataPorter
868
+ module Sources
869
+ class Api < Base
870
+ def fetch
871
+ api = @target_class._api_config
872
+ params = @data_import.config.symbolize_keys
873
+
874
+ url = api.endpoint.is_a?(Proc) ? api.endpoint.call(params) : api.endpoint
875
+ headers = api.headers.is_a?(Proc) ? api.headers.call : api.headers
876
+
877
+ response = Net::HTTP.get(URI(url), headers)
878
+ parsed = JSON.parse(response)
879
+
880
+ root = api.response_root
881
+ records = root ? parsed[root.to_s] : parsed
882
+
883
+ Array.wrap(records).map do |hash|
884
+ hash.transform_keys { |k| k.parameterize(separator: '_').to_sym }
885
+ end
886
+ end
887
+ end
888
+ end
889
+ end
890
+ ```
891
+
892
+ ### Controllers
893
+
894
+ ```ruby
895
+ # app/controllers/data_porter/imports_controller.rb
896
+ module DataPorter
897
+ class ImportsController < DataPorter.configuration.parent_controller.constantize
898
+ before_action :set_import, only: [:show, :parse, :confirm, :cancel]
899
+
900
+ def index
901
+ @imports = DataPorter::DataImport.order(created_at: :desc).page(params[:page])
902
+ end
903
+
904
+ def new
905
+ @import = DataPorter::DataImport.new
906
+ @targets = DataPorter::Registry.available
907
+ end
908
+
909
+ def create
910
+ @import = DataPorter::DataImport.new(import_params)
911
+ @import.user = current_user
912
+ @import.status = :pending
913
+
914
+ if @import.save
915
+ DataPorter::ParseJob.perform_later(@import.id)
916
+ redirect_to import_path(@import)
917
+ else
918
+ @targets = DataPorter::Registry.available
919
+ render :new
920
+ end
921
+ end
922
+
923
+ # GET /imports/:id
924
+ # Affiche le progress (si parsing/importing) ou le preview (si previewing)
925
+ def show
926
+ @target = @import.target_class
927
+ @records = @import.records
928
+ @grouped = @records.group_by(&:status)
929
+ end
930
+
931
+ # POST /imports/:id/parse
932
+ # Relance le parsing (si echec)
933
+ def parse
934
+ @import.update!(status: :pending)
935
+ DataPorter::ParseJob.perform_later(@import.id)
936
+ redirect_to import_path(@import)
937
+ end
938
+
939
+ # POST /imports/:id/confirm
940
+ # Lance l'import des records valides
941
+ def confirm
942
+ DataPorter::ImportJob.perform_later(@import.id)
943
+ redirect_to import_path(@import)
944
+ end
945
+
946
+ # POST /imports/:id/cancel
947
+ def cancel
948
+ @import.update!(status: :failed)
949
+ redirect_to imports_path
950
+ end
951
+
952
+ private
953
+
954
+ def set_import
955
+ @import = DataPorter::DataImport.find(params[:id])
956
+ end
957
+
958
+ def import_params
959
+ params.require(:data_import).permit(:target_key, :source_type, :file, config: {})
960
+ end
961
+ end
962
+ end
963
+ ```
964
+
965
+ ### Vues & Preview
966
+
967
+ Le coeur de la valeur ajoutee : un tableau de preview auto-genere a partir des colonnes du Target.
968
+
969
+ #### Layout
970
+
971
+ ```erb
972
+ <%# app/views/layouts/data_porter/application.html.erb %>
973
+ <%= render template: DataPorter.configuration.layout || 'layouts/application' %>
974
+ ```
975
+
976
+ L'engine herite du layout de l'app hote via le `parent_controller`.
977
+
978
+ #### Page `show` (Preview)
979
+
980
+ ```erb
981
+ <%# app/views/data_porter/imports/show.html.erb %>
982
+
983
+ <div data-controller="data-porter--progress"
984
+ data-data-porter--progress-id-value="<%= @import.id %>">
985
+
986
+ <%# === HEADER === %>
987
+ <h1><%= @target._label %> Import #<%= @import.id %></h1>
988
+ <p>Status: <span class="badge badge-<%= status_color(@import.status) %>">
989
+ <%= @import.status.humanize %>
990
+ </span></p>
991
+
992
+ <%# === PROGRESS BAR (visible pendant parsing/importing) === %>
993
+ <% if @import.parsing? || @import.importing? %>
994
+ <div class="progress">
995
+ <div class="progress-bar" data-data-porter--progress-target="bar" style="width: 0%">
996
+ <span data-data-porter--progress-target="text">0%</span>
997
+ </div>
998
+ </div>
999
+ <% end %>
1000
+
1001
+ <%# === SUMMARY (visible apres parsing) === %>
1002
+ <% if @import.previewable? || @import.completed? %>
1003
+ <div class="summary-cards">
1004
+ <div class="card card-complete">
1005
+ <strong><%= @grouped['complete']&.size || 0 %></strong> Ready
1006
+ </div>
1007
+ <div class="card card-partial">
1008
+ <strong><%= @grouped['partial']&.size || 0 %></strong> Incomplete
1009
+ </div>
1010
+ <div class="card card-missing">
1011
+ <strong><%= @grouped['missing']&.size || 0 %></strong> Missing
1012
+ </div>
1013
+ <div class="card card-duplicate">
1014
+ <strong><%= @grouped['duplicate']&.size || 0 %></strong> Duplicates
1015
+ </div>
1016
+ </div>
1017
+ <% end %>
1018
+
1019
+ <%# === PREVIEW TABLE (visible en previewing) === %>
1020
+ <% if @import.previewable? %>
1021
+ <table class="table">
1022
+ <thead>
1023
+ <tr>
1024
+ <th>#</th>
1025
+ <th>Status</th>
1026
+ <% @target._columns.each do |col| %>
1027
+ <th><%= col.label %></th>
1028
+ <% end %>
1029
+ <th>Errors</th>
1030
+ </tr>
1031
+ </thead>
1032
+ <tbody>
1033
+ <% @records.each do |record| %>
1034
+ <tr class="row-<%= record.status %>">
1035
+ <td><%= record.line_number %></td>
1036
+ <td><%= status_icon(record.status) %></td>
1037
+ <% @target._columns.each do |col| %>
1038
+ <td><%= record.data[col.name] %></td>
1039
+ <% end %>
1040
+ <td class="errors">
1041
+ <%= record.errors_list.map(&:message).join(', ') %>
1042
+ </td>
1043
+ </tr>
1044
+ <% end %>
1045
+ </tbody>
1046
+ </table>
1047
+
1048
+ <%# === ACTIONS === %>
1049
+ <div class="actions">
1050
+ <%= link_to "Cancel", cancel_import_path(@import),
1051
+ method: :post, class: "btn btn-outline-secondary" %>
1052
+ <%= link_to "Import #{@import.importable_records.size} records",
1053
+ confirm_import_path(@import),
1054
+ method: :post, class: "btn btn-primary",
1055
+ data: { confirm: "Import #{@import.importable_records.size} records?" } %>
1056
+ </div>
1057
+ <% end %>
1058
+
1059
+ <%# === RESULTS (visible apres import) === %>
1060
+ <% if @import.completed? %>
1061
+ <div class="results">
1062
+ <p>Created: <%= @import.report.imported_count %></p>
1063
+ <p>Errors: <%= @import.report.errored_count %></p>
1064
+ </div>
1065
+ <% end %>
1066
+
1067
+ <%# === FAILURE === %>
1068
+ <% if @import.failed? %>
1069
+ <div class="alert alert-danger">
1070
+ <% @import.report&.error_reports&.each do |err| %>
1071
+ <p><%= err.message %></p>
1072
+ <% end %>
1073
+ </div>
1074
+ <%= link_to "Retry", parse_import_path(@import), method: :post, class: "btn btn-warning" %>
1075
+ <% end %>
1076
+ </div>
1077
+ ```
1078
+
1079
+ ### Jobs (ActiveJob)
1080
+
1081
+ ```ruby
1082
+ # app/jobs/data_porter/parse_job.rb
1083
+ module DataPorter
1084
+ class ParseJob < ActiveJob::Base
1085
+ queue_as { DataPorter.configuration.queue_name }
1086
+
1087
+ def perform(import_id)
1088
+ data_import = DataPorter::DataImport.find(import_id)
1089
+ DataPorter::Orchestrator.new(data_import).parse!
1090
+ end
1091
+ end
1092
+ end
1093
+ ```
1094
+
1095
+ ```ruby
1096
+ # app/jobs/data_porter/import_job.rb
1097
+ module DataPorter
1098
+ class ImportJob < ActiveJob::Base
1099
+ queue_as { DataPorter.configuration.queue_name }
1100
+
1101
+ def perform(import_id)
1102
+ data_import = DataPorter::DataImport.find(import_id)
1103
+ DataPorter::Orchestrator.new(data_import).import!
1104
+ end
1105
+ end
1106
+ end
1107
+ ```
1108
+
1109
+ ### ActionCable & Progress
1110
+
1111
+ ```ruby
1112
+ # lib/data_porter/broadcaster.rb
1113
+ module DataPorter
1114
+ class Broadcaster
1115
+ def initialize(import_id)
1116
+ @channel = "#{DataPorter.configuration.cable_channel_prefix}/imports/#{import_id}"
1117
+ end
1118
+
1119
+ def progress(current, total)
1120
+ percentage = ((current.to_f / total) * 100).round
1121
+ broadcast(status: :processing, percentage: percentage, current: current, total: total)
1122
+ end
1123
+
1124
+ def success
1125
+ broadcast(status: :success)
1126
+ end
1127
+
1128
+ def failure(message)
1129
+ broadcast(status: :failure, error: message)
1130
+ end
1131
+
1132
+ private
1133
+
1134
+ def broadcast(message)
1135
+ ActionCable.server.broadcast(@channel, message)
1136
+ end
1137
+ end
1138
+ end
1139
+ ```
1140
+
1141
+ ```javascript
1142
+ // app/javascript/data_porter/progress_controller.js
1143
+ import { Controller } from "@hotwired/stimulus";
1144
+ import { createConsumer } from "@rails/actioncable";
1145
+
1146
+ export default class extends Controller {
1147
+ static targets = ["bar", "text"];
1148
+ static values = { id: Number };
1149
+
1150
+ connect() {
1151
+ this.subscription = createConsumer().subscriptions.create(
1152
+ { channel: "DataPorter::ImportChannel", id: this.idValue },
1153
+ {
1154
+ received: (data) => {
1155
+ if (data.status === "processing") {
1156
+ this.updateProgress(data.percentage);
1157
+ } else {
1158
+ window.location.reload();
1159
+ }
1160
+ },
1161
+ },
1162
+ );
1163
+ }
1164
+
1165
+ updateProgress(percentage) {
1166
+ if (this.hasBarTarget) {
1167
+ this.barTarget.style.width = `${percentage}%`;
1168
+ this.textTarget.textContent = `${percentage}%`;
1169
+ }
1170
+ }
1171
+
1172
+ disconnect() {
1173
+ this.subscription?.unsubscribe();
1174
+ }
1175
+ }
1176
+ ```
1177
+
1178
+ ---
1179
+
1180
+ ## Multi-tenancy
1181
+
1182
+ La gem est agnostique du concept de tenant. Le scoping se fait via le `context_builder` :
1183
+
1184
+ ```ruby
1185
+ # config/initializers/data_porter.rb
1186
+ DataPorter.configure do |config|
1187
+ config.context_builder = ->(controller) {
1188
+ OpenStruct.new(
1189
+ hotel: controller.current_user.hotel,
1190
+ user: controller.current_user
1191
+ )
1192
+ }
1193
+ end
1194
+ ```
1195
+
1196
+ Le `context` est passe a chaque appel `persist` :
1197
+
1198
+ ```ruby
1199
+ def persist(record, context:)
1200
+ Guest.create!(hotel: context.hotel, **record.attributes)
1201
+ end
1202
+ ```
1203
+
1204
+ Pour filtrer les imports affiches par tenant, override le controller :
1205
+
1206
+ ```ruby
1207
+ # Dans l'app hote
1208
+ DataPorter::ImportsController.class_eval do
1209
+ def index
1210
+ @imports = DataPorter::DataImport
1211
+ .joins(:user)
1212
+ .where(users: { hotel_id: current_user.hotel_id })
1213
+ .order(created_at: :desc)
1214
+ .page(params[:page])
1215
+ end
1216
+ end
1217
+ ```
1218
+
1219
+ Ou via le `parent_controller` qui expose un `scope_imports` :
1220
+
1221
+ ```ruby
1222
+ DataPorter.configure do |config|
1223
+ config.scope = ->(relation, controller) {
1224
+ relation.where(user: controller.current_user)
1225
+ }
1226
+ end
1227
+ ```
1228
+
1229
+ ---
1230
+
1231
+ ## Generators
1232
+
1233
+ ### `data_porter:install`
1234
+
1235
+ ```bash
1236
+ rails generate data_porter:install
1237
+ ```
1238
+
1239
+ Cree :
1240
+
1241
+ - `db/migrate/xxx_create_data_porter_imports.rb`
1242
+ - `config/initializers/data_porter.rb`
1243
+ - `app/importers/` (dossier vide)
1244
+ - Ajoute `mount DataPorter::Engine, at: '/imports'` dans `config/routes.rb`
1245
+
1246
+ ### `data_porter:target NAME [columns...]`
1247
+
1248
+ ```bash
1249
+ rails generate data_porter:target guests first_name:string:required last_name:string:required email:email phone:phone
1250
+ ```
1251
+
1252
+ Cree :
1253
+
1254
+ - `app/importers/guests_target.rb` avec le squelette pre-rempli
1255
+
1256
+ ---
1257
+
1258
+ ## Override & Extension
1259
+
1260
+ ### Vues
1261
+
1262
+ Les vues de l'engine sont overridables dans l'app hote :
1263
+
1264
+ ```
1265
+ app/views/data_porter/imports/index.html.erb -> override la liste
1266
+ app/views/data_porter/imports/show.html.erb -> override le preview
1267
+ app/views/data_porter/imports/new.html.erb -> override le formulaire
1268
+ ```
1269
+
1270
+ ### CSS
1271
+
1272
+ La gem fournit des classes CSS minimales. Override via :
1273
+
1274
+ ```css
1275
+ /* app/assets/stylesheets/data_porter_overrides.css */
1276
+ .data-porter .row-complete {
1277
+ background: #f0fff0;
1278
+ }
1279
+ .data-porter .row-partial {
1280
+ background: #fffdf0;
1281
+ }
1282
+ .data-porter .row-missing {
1283
+ background: #fff0f0;
1284
+ }
1285
+ ```
1286
+
1287
+ ### Controller
1288
+
1289
+ Heritage via `parent_controller` pour l'auth et le layout.
1290
+ Override individuel via `class_eval` ou monkey-patching classique.
1291
+
1292
+ ### Custom Source
1293
+
1294
+ ```ruby
1295
+ # lib/data_porter/sources/google_sheets.rb
1296
+ module DataPorter
1297
+ module Sources
1298
+ class GoogleSheets < Base
1299
+ def fetch
1300
+ spreadsheet_id = @data_import.config['spreadsheet_id']
1301
+ # ... Google Sheets API logic ...
1302
+ rows.map { |row| map_row(row) }
1303
+ end
1304
+ end
1305
+ end
1306
+ end
1307
+
1308
+ # config/initializers/data_porter.rb
1309
+ DataPorter.configure do |config|
1310
+ config.enabled_sources = [:csv, :json, :api, :google_sheets]
1311
+ end
1312
+ ```
1313
+
1314
+ ---
1315
+
1316
+ ## Comparaison avec maintenance_tasks
1317
+
1318
+ | Aspect | maintenance_tasks | DataPorter |
1319
+ | ------------------- | ----------------------------- | ----------------------------------------------------- |
1320
+ | Purpose | One-off maintenance scripts | Data import workflow |
1321
+ | Definition | `Task` (collection + process) | `Target` (columns + persist) |
1322
+ | Data source | CSV or ActiveRecord | CSV, JSON, API (extensible) |
1323
+ | Preview | Non | Oui (etat `previewing`) |
1324
+ | UI auto-generee | Form params | Tableau colonnes dynamiques |
1325
+ | Auto-discovery | `Task.descendants` | `Target.descendants` |
1326
+ | Validation visuelle | Non | Oui (complete/partial/missing) |
1327
+ | Progress realtime | Non | Oui (ActionCable) |
1328
+ | Multi-step | Non (fire & forget) | Oui (parse -> preview -> import) |
1329
+ | Parent controller | Configurable | Configurable |
1330
+ | ActiveJob | Oui | Oui |
1331
+ | State machine | Run/Pause/Cancel | Pending/Parsing/Previewing/Importing/Completed/Failed |
1332
+ | Throttling | Oui | Non (pas necessaire) |
1333
+ | Gem dependencies | Minimal | store_model, actioncable |
1334
+
1335
+ ---
1336
+
1337
+ ## Structure fichiers de la gem
1338
+
1339
+ ```
1340
+ data_porter/
1341
+ app/
1342
+ channels/
1343
+ data_porter/
1344
+ import_channel.rb
1345
+ controllers/
1346
+ data_porter/
1347
+ imports_controller.rb
1348
+ jobs/
1349
+ data_porter/
1350
+ parse_job.rb
1351
+ import_job.rb
1352
+ models/
1353
+ data_porter/
1354
+ data_import.rb
1355
+ views/
1356
+ data_porter/
1357
+ imports/
1358
+ index.html.erb
1359
+ new.html.erb
1360
+ show.html.erb
1361
+ _form.html.erb
1362
+ _preview_table.html.erb
1363
+ _progress.html.erb
1364
+ _summary.html.erb
1365
+ assets/
1366
+ stylesheets/
1367
+ data_porter/
1368
+ application.css
1369
+ javascript/
1370
+ data_porter/
1371
+ progress_controller.js
1372
+ config/
1373
+ routes.rb
1374
+ db/
1375
+ migrate/
1376
+ create_data_porter_imports.rb
1377
+ lib/
1378
+ data_porter.rb
1379
+ data_porter/
1380
+ configuration.rb
1381
+ engine.rb
1382
+ orchestrator.rb
1383
+ registry.rb
1384
+ broadcaster.rb
1385
+ target.rb
1386
+ type_validator.rb
1387
+ version.rb
1388
+ sources/
1389
+ base.rb
1390
+ csv.rb
1391
+ json.rb
1392
+ api.rb
1393
+ store_models/
1394
+ import_record.rb
1395
+ error.rb
1396
+ report.rb
1397
+ dsl/
1398
+ columns_dsl.rb
1399
+ csv_mapping_dsl.rb
1400
+ api_config_dsl.rb
1401
+ params_dsl.rb
1402
+ generators/
1403
+ data_porter/
1404
+ install_generator.rb
1405
+ target_generator.rb
1406
+ templates/
1407
+ initializer.rb.tt
1408
+ target.rb.tt
1409
+ spec/
1410
+ ...
1411
+ data_porter.gemspec
1412
+ Gemfile
1413
+ README.md
1414
+ LICENSE
1415
+ ```
1416
+
1417
+ ---
1418
+
1419
+ ## Dry Run (v0.2)
1420
+
1421
+ Le dry-run permet de simuler l'import sans rien persister. Il comble le gap entre le preview
1422
+ (validation colonnes/types) et l'import reel (validation base de donnees).
1423
+
1424
+ ### Pourquoi c'est essentiel
1425
+
1426
+ Le preview valide au niveau **colonnes** (types, required, inclusion). Le dry-run valide au
1427
+ niveau **base de donnees** :
1428
+
1429
+ | Couche de validation | Preview | Dry Run |
1430
+ | ----------------------------------- | ------- | ------- |
1431
+ | Champs requis | oui | oui |
1432
+ | Types (email, date...) | oui | oui |
1433
+ | Contraintes DB (unique index, FK) | **non** | **oui** |
1434
+ | Validations ActiveRecord du model | **non** | **oui** |
1435
+ | Callbacks `before_create` | **non** | **oui** |
1436
+ | Interactions entre records du batch | **non** | **oui** |
1437
+
1438
+ ### Principe
1439
+
1440
+ Envelopper l'import dans une transaction ActiveRecord et la rollback systematiquement.
1441
+ Les erreurs DB sont capturees et ajoutees aux records, le tableau preview se met a jour.
1442
+
1443
+ ### Nouveau status
1444
+
1445
+ ```
1446
+ pending -> parsing -> previewing -> dry_running -> previewing -> importing -> completed
1447
+ ^ |
1448
+ +----------------------------+
1449
+ (retour au preview avec erreurs DB enrichies)
1450
+ ```
1451
+
1452
+ `dry_running` est un status transitoire. Apres le dry-run, l'import revient en `previewing`
1453
+ avec les records enrichis des erreurs DB.
1454
+
1455
+ ### DSL
1456
+
1457
+ ```ruby
1458
+ class GuestsTarget < DataPorter::Target
1459
+ label "Guests"
1460
+ model Guest
1461
+ dry_run_enabled
1462
+
1463
+ # ...
1464
+ end
1465
+ ```
1466
+
1467
+ `dry_run_enabled` active le bouton "Test Import" dans l'UI pour ce target.
1468
+ Par defaut, le dry-run est desactive (pas tous les targets en ont besoin).
1469
+
1470
+ ### Orchestrator
1471
+
1472
+ ```ruby
1473
+ # lib/data_porter/orchestrator.rb
1474
+ module DataPorter
1475
+ class Orchestrator
1476
+ def dry_run!
1477
+ @data_import.dry_running!
1478
+ context = build_context
1479
+ importable = @data_import.importable_records
1480
+
1481
+ ActiveRecord::Base.transaction do
1482
+ importable.each_with_index do |record, index|
1483
+ instance = @target.persist(record, context: context)
1484
+ record.target_id = instance.id
1485
+ record.status = 'complete'
1486
+ record.dry_run_passed = true
1487
+ broadcast_progress(index + 1, importable.size)
1488
+ rescue ActiveRecord::RecordInvalid => e
1489
+ record.status = 'partial'
1490
+ e.record.errors.full_messages.each { |msg| record.add_error(msg) }
1491
+ rescue ActiveRecord::RecordNotUnique => e
1492
+ record.status = 'partial'
1493
+ record.add_error(extract_unique_violation_message(e))
1494
+ rescue StandardError => e
1495
+ record.status = 'error'
1496
+ record.add_error(e.message)
1497
+ end
1498
+
1499
+ raise ActiveRecord::Rollback
1500
+ end
1501
+
1502
+ reset_target_ids!(importable)
1503
+ @data_import.update!(status: :previewing)
1504
+ build_report
1505
+ broadcast_success
1506
+ rescue StandardError => e
1507
+ handle_failure(e)
1508
+ end
1509
+
1510
+ private
1511
+
1512
+ def reset_target_ids!(records)
1513
+ records.each { |r| r.target_id = nil }
1514
+ end
1515
+
1516
+ def extract_unique_violation_message(error)
1517
+ match = error.message.match(/Key \((.+?)\)=\((.+?)\) already exists/)
1518
+ if match
1519
+ "#{match[1]}: '#{match[2]}' already exists"
1520
+ else
1521
+ "Duplicate record detected"
1522
+ end
1523
+ end
1524
+ end
1525
+ end
1526
+ ```
1527
+
1528
+ ### Job
1529
+
1530
+ ```ruby
1531
+ # app/jobs/data_porter/dry_run_job.rb
1532
+ module DataPorter
1533
+ class DryRunJob < ActiveJob::Base
1534
+ queue_as { DataPorter.configuration.queue_name }
1535
+
1536
+ def perform(import_id)
1537
+ data_import = DataPorter::DataImport.find(import_id)
1538
+ DataPorter::Orchestrator.new(data_import).dry_run!
1539
+ end
1540
+ end
1541
+ end
1542
+ ```
1543
+
1544
+ ### Controller
1545
+
1546
+ ```ruby
1547
+ # Ajout dans ImportsController
1548
+ def dry_run
1549
+ DataPorter::DryRunJob.perform_later(@import.id)
1550
+ redirect_to import_path(@import)
1551
+ end
1552
+ ```
1553
+
1554
+ ### Migration (ajout au modele DataImport)
1555
+
1556
+ ```ruby
1557
+ # Ajouter dry_running au enum status
1558
+ enum :status, {
1559
+ pending: 0,
1560
+ parsing: 1,
1561
+ previewing: 2,
1562
+ importing: 3,
1563
+ completed: 4,
1564
+ failed: 5,
1565
+ dry_running: 6
1566
+ }
1567
+ ```
1568
+
1569
+ ### StoreModel ImportRecord (ajout)
1570
+
1571
+ ```ruby
1572
+ attribute :dry_run_passed, :boolean, default: false
1573
+ ```
1574
+
1575
+ ### UI : boutons preview
1576
+
1577
+ Deux boutons sur la page preview quand le target a `dry_run_enabled` :
1578
+
1579
+ ```erb
1580
+ <% if @import.previewable? %>
1581
+ <div class="actions">
1582
+ <%= link_to "Cancel", cancel_import_path(@import),
1583
+ method: :post, class: "btn btn-outline-secondary" %>
1584
+
1585
+ <% if @target.class._dry_run_enabled %>
1586
+ <%= link_to "Test Import (dry-run)",
1587
+ dry_run_import_path(@import),
1588
+ method: :post, class: "btn btn-outline-primary" %>
1589
+ <% end %>
1590
+
1591
+ <%= link_to "Import #{@import.importable_records.size} records",
1592
+ confirm_import_path(@import),
1593
+ method: :post, class: "btn btn-primary",
1594
+ data: { confirm: "Import #{@import.importable_records.size} records?" } %>
1595
+ </div>
1596
+ <% end %>
1597
+ ```
1598
+
1599
+ Apres le dry-run, les records enrichis affichent un indicateur supplementaire :
1600
+
1601
+ ```erb
1602
+ <% if record.dry_run_passed %>
1603
+ <span class="badge badge-success" title="DB validation passed">DB OK</span>
1604
+ <% end %>
1605
+ ```
1606
+
1607
+ ### Workflow utilisateur
1608
+
1609
+ ```
1610
+ 1. Upload CSV
1611
+ 2. Preview : voit 142 records "complete", 3 "partial", 5 "missing"
1612
+ 3. Clique "Test Import (dry-run)"
1613
+ 4. Progress bar (parsing async)
1614
+ 5. Retour au preview : 138 records "complete + DB OK", 4 nouveaux "partial"
1615
+ (ex: "email: 'john@doe.com' already exists", "housing_id: 999 doesn't exist")
1616
+ 6. Corrige son CSV et relance, OU confirme l'import des 138 valides
1617
+ ```
1618
+
1619
+ ---
1620
+
1621
+ ## Rollback (v0.3 / v0.4)
1622
+
1623
+ Le rollback permet d'annuler un import termine en supprimant les records crees.
1624
+
1625
+ ### Principe
1626
+
1627
+ Chaque `ImportRecord` stocke le `target_id` du record cree en base apres l'import.
1628
+ Le rollback utilise ces IDs pour identifier et supprimer les records.
1629
+
1630
+ ### Niveaux de rollback
1631
+
1632
+ #### v0.3 — Rollback simple
1633
+
1634
+ Suppression directe des records crees, avec un time window configurable.
1635
+
1636
+ #### v0.4 — Rollback garde
1637
+
1638
+ Verifications de securite avant suppression : records modifies depuis l'import,
1639
+ dependances creees, side-effects irreversibles.
1640
+
1641
+ ### DSL
1642
+
1643
+ ```ruby
1644
+ class GuestsTarget < DataPorter::Target
1645
+ label "Guests"
1646
+ model Guest
1647
+ rollback_enabled
1648
+ rollback_window 2.hours
1649
+ rollback_strategy :destroy
1650
+
1651
+ def rollbackable?(record, context:)
1652
+ instance = Guest.find_by(id: record.target_id)
1653
+ return false if instance.nil?
1654
+ return false if instance.updated_at > context.import_completed_at
1655
+ true
1656
+ end
1657
+ end
1658
+ ```
1659
+
1660
+ | Option DSL | Type | Description |
1661
+ | ------------------- | -------- | ----------------------------------------------------------------- |
1662
+ | `rollback_enabled` | flag | Active le rollback pour ce target |
1663
+ | `rollback_window` | Duration | Duree pendant laquelle le rollback est possible (default: 1.hour) |
1664
+ | `rollback_strategy` | Symbol | `:destroy`, `:delete`, `:soft_delete`, `:flag` |
1665
+ | `rollbackable?` | Hook | Verification custom par record avant rollback |
1666
+
1667
+ ### Strategies de rollback
1668
+
1669
+ ```ruby
1670
+ # lib/data_porter/rollback_strategies.rb
1671
+ module DataPorter
1672
+ module RollbackStrategies
1673
+ STRATEGIES = {
1674
+ destroy: ->(model, ids) {
1675
+ model.where(id: ids).destroy_all
1676
+ },
1677
+
1678
+ delete: ->(model, ids) {
1679
+ model.where(id: ids).delete_all
1680
+ },
1681
+
1682
+ soft_delete: ->(model, ids) {
1683
+ model.where(id: ids).update_all(discarded_at: Time.current)
1684
+ },
1685
+
1686
+ flag: ->(model, ids) {
1687
+ model.where(id: ids).update_all(import_rolled_back: true)
1688
+ }
1689
+ }.freeze
1690
+
1691
+ def self.execute(strategy, model, ids)
1692
+ handler = STRATEGIES[strategy] || raise(UnknownStrategy, "Unknown strategy: #{strategy}")
1693
+ handler.call(model, ids)
1694
+ end
1695
+ end
1696
+ end
1697
+ ```
1698
+
1699
+ | Strategie | Effet | Callbacks | Reversible | Quand l'utiliser |
1700
+ | -------------- | ------------------------ | ------------------------------- | ---------- | ------------------------------------------------- |
1701
+ | `:destroy` | `destroy_all` | Oui (after_destroy, dependents) | Non | Default, quand les callbacks sont importants |
1702
+ | `:delete` | `delete_all` | Non (SQL direct) | Non | Performance, pas de side-effects |
1703
+ | `:soft_delete` | Set `discarded_at` | Non | Oui | Avec discard gem, quand on veut pouvoir restaurer |
1704
+ | `:flag` | Set `import_rolled_back` | Non | Oui | Quand on veut garder les records mais les marquer |
1705
+
1706
+ ### Orchestrator
1707
+
1708
+ ```ruby
1709
+ # lib/data_porter/orchestrator.rb
1710
+ module DataPorter
1711
+ class Orchestrator
1712
+ def rollback!
1713
+ validate_rollback_allowed!
1714
+ @data_import.rolling_back!
1715
+
1716
+ target_class = @target.class
1717
+ model = target_class._model
1718
+ strategy = target_class._rollback_strategy || :destroy
1719
+ context = build_rollback_context
1720
+
1721
+ imported_records = @data_import.records.select { |r| r.status == 'imported' && r.target_id.present? }
1722
+ rollbackable, skipped = partition_rollbackable(imported_records, context)
1723
+
1724
+ rollbackable_ids = rollbackable.map(&:target_id)
1725
+
1726
+ ActiveRecord::Base.transaction do
1727
+ DataPorter::RollbackStrategies.execute(strategy, model, rollbackable_ids)
1728
+ end
1729
+
1730
+ rollbackable.each { |r| r.status = 'rolled_back' }
1731
+ skipped.each { |r| r.status = 'rollback_skipped' }
1732
+
1733
+ @data_import.update!(status: :rolled_back)
1734
+ build_rollback_report(rollbackable, skipped)
1735
+ broadcast_success
1736
+ rescue StandardError => e
1737
+ handle_failure(e)
1738
+ end
1739
+
1740
+ private
1741
+
1742
+ def validate_rollback_allowed!
1743
+ target_class = @target.class
1744
+ raise RollbackNotEnabled, "Rollback not enabled for #{target_class._label}" unless target_class._rollback_enabled
1745
+
1746
+ window = target_class._rollback_window || 1.hour
1747
+ if @data_import.completed_at && @data_import.completed_at < window.ago
1748
+ raise RollbackWindowExpired,
1749
+ "Rollback window expired (#{window.inspect} after completion)"
1750
+ end
1751
+ end
1752
+
1753
+ def partition_rollbackable(records, context)
1754
+ rollbackable = []
1755
+ skipped = []
1756
+
1757
+ records.each do |record|
1758
+ if @target.rollbackable?(record, context: context)
1759
+ rollbackable << record
1760
+ else
1761
+ skipped << record
1762
+ record.add_warning("Skipped: record modified or deleted since import")
1763
+ end
1764
+ end
1765
+
1766
+ [rollbackable, skipped]
1767
+ end
1768
+
1769
+ def build_rollback_context
1770
+ OpenStruct.new(
1771
+ import_completed_at: @data_import.updated_at,
1772
+ **(@data_import.config || {}).symbolize_keys
1773
+ )
1774
+ end
1775
+
1776
+ def build_rollback_report(rollbackable, skipped)
1777
+ report = @data_import.report || DataPorter::StoreModels::Report.new
1778
+ report.rolled_back_count = rollbackable.size
1779
+ report.rollback_skipped_count = skipped.size
1780
+ @data_import.update!(report: report)
1781
+ end
1782
+ end
1783
+ end
1784
+ ```
1785
+
1786
+ ### Modele DataImport (ajouts)
1787
+
1788
+ ```ruby
1789
+ enum :status, {
1790
+ pending: 0,
1791
+ parsing: 1,
1792
+ previewing: 2,
1793
+ importing: 3,
1794
+ completed: 4,
1795
+ failed: 5,
1796
+ dry_running: 6,
1797
+ rolling_back: 7,
1798
+ rolled_back: 8
1799
+ }
1800
+
1801
+ def rollback_available?
1802
+ return false unless completed?
1803
+
1804
+ target = target_class
1805
+ return false unless target._rollback_enabled
1806
+
1807
+ window = target._rollback_window || 1.hour
1808
+ updated_at > window.ago
1809
+ end
1810
+
1811
+ def rollback_expires_at
1812
+ return nil unless rollback_available?
1813
+
1814
+ window = target_class._rollback_window || 1.hour
1815
+ updated_at + window
1816
+ end
1817
+ ```
1818
+
1819
+ ### StoreModel ImportRecord (ajouts)
1820
+
1821
+ ```ruby
1822
+ STATUSES = %w[
1823
+ pending complete partial missing duplicate
1824
+ imported error
1825
+ rolled_back rollback_skipped
1826
+ ].freeze
1827
+ ```
1828
+
1829
+ ### StoreModel Report (ajouts)
1830
+
1831
+ ```ruby
1832
+ attribute :rolled_back_count, :integer, default: 0
1833
+ attribute :rollback_skipped_count, :integer, default: 0
1834
+ ```
1835
+
1836
+ ### Controller
1837
+
1838
+ ```ruby
1839
+ def rollback
1840
+ if @import.rollback_available?
1841
+ DataPorter::RollbackJob.perform_later(@import.id)
1842
+ redirect_to import_path(@import)
1843
+ else
1844
+ redirect_to import_path(@import), alert: "Rollback not available"
1845
+ end
1846
+ end
1847
+ ```
1848
+
1849
+ ### Job
1850
+
1851
+ ```ruby
1852
+ # app/jobs/data_porter/rollback_job.rb
1853
+ module DataPorter
1854
+ class RollbackJob < ActiveJob::Base
1855
+ queue_as { DataPorter.configuration.queue_name }
1856
+
1857
+ def perform(import_id)
1858
+ data_import = DataPorter::DataImport.find(import_id)
1859
+ DataPorter::Orchestrator.new(data_import).rollback!
1860
+ end
1861
+ end
1862
+ end
1863
+ ```
1864
+
1865
+ ### UI : page completed avec rollback
1866
+
1867
+ ```erb
1868
+ <% if @import.completed? %>
1869
+ <div class="results">
1870
+ <p>Created: <%= @import.report.imported_count %></p>
1871
+ <p>Errors: <%= @import.report.errored_count %></p>
1872
+ </div>
1873
+
1874
+ <% if @import.rollback_available? %>
1875
+ <div class="rollback-section">
1876
+ <p>
1877
+ Rollback available for
1878
+ <strong><%= distance_of_time_in_words_to_now(@import.rollback_expires_at) %></strong>
1879
+ </p>
1880
+
1881
+ <% modified_count = @import.records.count { |r|
1882
+ r.status == 'imported' && r.target_id &&
1883
+ @target.class._model.find_by(id: r.target_id)&.updated_at.to_i > @import.updated_at.to_i
1884
+ } %>
1885
+
1886
+ <% if modified_count > 0 %>
1887
+ <div class="alert alert-warning">
1888
+ <strong><%= modified_count %></strong> record(s) modified since import — will be skipped
1889
+ </div>
1890
+ <% end %>
1891
+
1892
+ <%= link_to "Rollback #{@import.records.count { |r| r.status == 'imported' } - modified_count} records",
1893
+ rollback_import_path(@import),
1894
+ method: :post, class: "btn btn-danger",
1895
+ data: { confirm: "This will delete the imported records. Continue?" } %>
1896
+ </div>
1897
+ <% end %>
1898
+ <% end %>
1899
+
1900
+ <% if @import.rolled_back? %>
1901
+ <div class="results">
1902
+ <p>Rolled back: <%= @import.report.rolled_back_count %></p>
1903
+ <p>Skipped: <%= @import.report.rollback_skipped_count %></p>
1904
+ </div>
1905
+ <% end %>
1906
+ ```
1907
+
1908
+ ### State machine complete (v0.4)
1909
+
1910
+ ```
1911
+ +-> dry_running -+
1912
+ | |
1913
+ pending -> parsing -> previewing ---+ +-> previewing
1914
+ ^ | |
1915
+ | +----------------------+
1916
+ |
1917
+ +--- (re-upload / retry)
1918
+
1919
+ previewing -> importing -> completed -> rolling_back -> rolled_back
1920
+ |
1921
+ +-> failed
1922
+
1923
+ parsing -> failed
1924
+ importing -> failed
1925
+ rolling_back -> failed
1926
+ ```
1927
+
1928
+ ### Limites documentees
1929
+
1930
+ Le rollback ne peut PAS annuler :
1931
+
1932
+ - Les emails/notifications envoyees par les callbacks
1933
+ - Les webhooks declenches
1934
+ - Les fichiers uploades via ActiveStorage dans `persist`
1935
+ - Les modifications faites par d'autres utilisateurs sur les records importes
1936
+ - Les records dont les associations ont ete supprimees (FK violations)
1937
+
1938
+ Ces limites doivent etre clairement affichees dans l'UI avant confirmation du rollback.
1939
+
1940
+ ### Configuration globale
1941
+
1942
+ ```ruby
1943
+ # config/initializers/data_porter.rb
1944
+ DataPorter.configure do |config|
1945
+ config.default_rollback_strategy = :destroy
1946
+ config.default_rollback_window = 1.hour
1947
+ config.rollback_strategies = {
1948
+ # Ajout de strategies custom
1949
+ archive: ->(model, ids) {
1950
+ model.where(id: ids).update_all(archived: true, archived_at: Time.current)
1951
+ }
1952
+ }
1953
+ end
1954
+ ```
1955
+
1956
+ ### Structure fichiers supplementaires
1957
+
1958
+ ```
1959
+ lib/
1960
+ data_porter/
1961
+ rollback_strategies.rb
1962
+ app/
1963
+ jobs/
1964
+ data_porter/
1965
+ dry_run_job.rb
1966
+ rollback_job.rb
1967
+ ```
1968
+
1969
+ ---
1970
+
1971
+ ## Roadmap
1972
+
1973
+ ### v0.1 - MVP
1974
+
1975
+ - Modele DataImport + migration
1976
+ - Source CSV
1977
+ - Orchestrator (parse + import)
1978
+ - Preview table auto-generee
1979
+ - DSL Target (columns, csv_mapping, persist)
1980
+ - Jobs ActiveJob
1981
+ - ActionCable progress
1982
+ - Generator install + target
1983
+ - Tests RSpec
1984
+
1985
+ ### v0.2 - Dry Run + Sources
1986
+
1987
+ - **Dry-run mode** (transaction + rollback, enrichissement des erreurs DB)
1988
+ - DSL `dry_run_enabled`
1989
+ - Source JSON (file upload + raw text)
1990
+ - Source API (HTTP client + config)
1991
+ - Deduplication visuelle
1992
+
1993
+ ### v0.3 - Rollback simple + UX
1994
+
1995
+ - **Rollback simple** (destroy des records crees via target_id)
1996
+ - DSL `rollback_enabled`, `rollback_window`, `rollback_strategy`
1997
+ - Strategies built-in : destroy, delete, soft_delete, flag
1998
+ - Rollback time window avec countdown UI
1999
+ - Filtres/tri dans le preview
2000
+ - Export des erreurs en CSV
2001
+ - Pagination du preview
2002
+
2003
+ ### v0.4 - Rollback garde + Avance
2004
+
2005
+ - **Rollback garde** : hook `rollbackable?`, detection records modifies
2006
+ - Strategies de rollback custom (plugin)
2007
+ - Affichage pre-rollback des records modifies/non-rollbackables
2008
+ - Custom sources (plugin system)
2009
+ - Batch persist (bulk insert)
2010
+ - Webhook notifications (fin d'import)
2011
+ - Import scheduling (cron-like pour les sources API)
2012
+ - Row-level accept/reject en preview