data_porter 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.claude/commands/blog-status.md +10 -0
- data/.claude/commands/blog.md +109 -0
- data/.claude/commands/task-done.md +27 -0
- data/.claude/commands/tm/add-dependency.md +58 -0
- data/.claude/commands/tm/add-subtask.md +79 -0
- data/.claude/commands/tm/add-task.md +81 -0
- data/.claude/commands/tm/analyze-complexity.md +124 -0
- data/.claude/commands/tm/analyze-project.md +100 -0
- data/.claude/commands/tm/auto-implement-tasks.md +100 -0
- data/.claude/commands/tm/command-pipeline.md +80 -0
- data/.claude/commands/tm/complexity-report.md +120 -0
- data/.claude/commands/tm/convert-task-to-subtask.md +74 -0
- data/.claude/commands/tm/expand-all-tasks.md +52 -0
- data/.claude/commands/tm/expand-task.md +52 -0
- data/.claude/commands/tm/fix-dependencies.md +82 -0
- data/.claude/commands/tm/help.md +101 -0
- data/.claude/commands/tm/init-project-quick.md +49 -0
- data/.claude/commands/tm/init-project.md +53 -0
- data/.claude/commands/tm/install-taskmaster.md +118 -0
- data/.claude/commands/tm/learn.md +106 -0
- data/.claude/commands/tm/list-tasks-by-status.md +42 -0
- data/.claude/commands/tm/list-tasks-with-subtasks.md +30 -0
- data/.claude/commands/tm/list-tasks.md +46 -0
- data/.claude/commands/tm/next-task.md +69 -0
- data/.claude/commands/tm/parse-prd-with-research.md +51 -0
- data/.claude/commands/tm/parse-prd.md +52 -0
- data/.claude/commands/tm/project-status.md +67 -0
- data/.claude/commands/tm/quick-install-taskmaster.md +23 -0
- data/.claude/commands/tm/remove-all-subtasks.md +94 -0
- data/.claude/commands/tm/remove-dependency.md +65 -0
- data/.claude/commands/tm/remove-subtask.md +87 -0
- data/.claude/commands/tm/remove-subtasks.md +89 -0
- data/.claude/commands/tm/remove-task.md +110 -0
- data/.claude/commands/tm/setup-models.md +52 -0
- data/.claude/commands/tm/show-task.md +85 -0
- data/.claude/commands/tm/smart-workflow.md +58 -0
- data/.claude/commands/tm/sync-readme.md +120 -0
- data/.claude/commands/tm/tm-main.md +147 -0
- data/.claude/commands/tm/to-cancelled.md +58 -0
- data/.claude/commands/tm/to-deferred.md +50 -0
- data/.claude/commands/tm/to-done.md +47 -0
- data/.claude/commands/tm/to-in-progress.md +39 -0
- data/.claude/commands/tm/to-pending.md +35 -0
- data/.claude/commands/tm/to-review.md +43 -0
- data/.claude/commands/tm/update-single-task.md +122 -0
- data/.claude/commands/tm/update-task.md +75 -0
- data/.claude/commands/tm/update-tasks-from-id.md +111 -0
- data/.claude/commands/tm/validate-dependencies.md +72 -0
- data/.claude/commands/tm/view-models.md +52 -0
- data/.env.example +12 -0
- data/.mcp.json +24 -0
- data/.taskmaster/CLAUDE.md +435 -0
- data/.taskmaster/config.json +44 -0
- data/.taskmaster/docs/prd.txt +2044 -0
- data/.taskmaster/state.json +6 -0
- data/.taskmaster/tasks/task_001.md +19 -0
- data/.taskmaster/tasks/task_002.md +19 -0
- data/.taskmaster/tasks/task_003.md +19 -0
- data/.taskmaster/tasks/task_004.md +19 -0
- data/.taskmaster/tasks/task_005.md +19 -0
- data/.taskmaster/tasks/task_006.md +19 -0
- data/.taskmaster/tasks/task_007.md +19 -0
- data/.taskmaster/tasks/task_008.md +19 -0
- data/.taskmaster/tasks/task_009.md +19 -0
- data/.taskmaster/tasks/task_010.md +19 -0
- data/.taskmaster/tasks/task_011.md +19 -0
- data/.taskmaster/tasks/task_012.md +19 -0
- data/.taskmaster/tasks/task_013.md +19 -0
- data/.taskmaster/tasks/task_014.md +19 -0
- data/.taskmaster/tasks/task_015.md +19 -0
- data/.taskmaster/tasks/task_016.md +19 -0
- data/.taskmaster/tasks/task_017.md +19 -0
- data/.taskmaster/tasks/task_018.md +19 -0
- data/.taskmaster/tasks/task_019.md +19 -0
- data/.taskmaster/tasks/task_020.md +19 -0
- data/.taskmaster/tasks/tasks.json +299 -0
- data/.taskmaster/templates/example_prd.txt +47 -0
- data/.taskmaster/templates/example_prd_rpg.txt +511 -0
- data/CHANGELOG.md +29 -0
- data/CLAUDE.md +65 -0
- data/CODE_OF_CONDUCT.md +10 -0
- data/CONTRIBUTING.md +49 -0
- data/LICENSE +21 -0
- data/README.md +463 -0
- data/Rakefile +12 -0
- data/app/assets/stylesheets/data_porter/application.css +646 -0
- data/app/channels/data_porter/import_channel.rb +10 -0
- data/app/controllers/data_porter/imports_controller.rb +68 -0
- data/app/javascript/data_porter/progress_controller.js +33 -0
- data/app/jobs/data_porter/dry_run_job.rb +12 -0
- data/app/jobs/data_porter/import_job.rb +12 -0
- data/app/jobs/data_porter/parse_job.rb +12 -0
- data/app/models/data_porter/data_import.rb +49 -0
- data/app/views/data_porter/imports/index.html.erb +142 -0
- data/app/views/data_porter/imports/new.html.erb +88 -0
- data/app/views/data_porter/imports/show.html.erb +49 -0
- data/config/database.yml +3 -0
- data/config/routes.rb +12 -0
- data/docs/SPEC.md +2012 -0
- data/docs/UI.md +32 -0
- data/docs/blog/001-why-build-a-data-import-engine.md +166 -0
- data/docs/blog/002-scaffolding-a-rails-engine.md +188 -0
- data/docs/blog/003-configuration-dsl.md +222 -0
- data/docs/blog/004-store-model-jsonb.md +237 -0
- data/docs/blog/005-target-dsl.md +284 -0
- data/docs/blog/006-parsing-csv-sources.md +300 -0
- data/docs/blog/007-orchestrator.md +247 -0
- data/docs/blog/008-actioncable-stimulus.md +376 -0
- data/docs/blog/009-phlex-ui-components.md +446 -0
- data/docs/blog/010-controllers-routing.md +374 -0
- data/docs/blog/011-generators.md +364 -0
- data/docs/blog/012-json-api-sources.md +323 -0
- data/docs/blog/013-testing-rails-engine.md +618 -0
- data/docs/blog/014-dry-run.md +307 -0
- data/docs/blog/015-publishing-retro.md +264 -0
- data/docs/blog/016-erb-view-templates.md +431 -0
- data/docs/blog/017-showcase-final-retro.md +220 -0
- data/docs/blog/BACKLOG.md +8 -0
- data/docs/blog/SERIES.md +154 -0
- data/docs/screenshots/index-with-previewing.jpg +0 -0
- data/docs/screenshots/index.jpg +0 -0
- data/docs/screenshots/modal-new-import.jpg +0 -0
- data/docs/screenshots/preview.jpg +0 -0
- data/lib/data_porter/broadcaster.rb +29 -0
- data/lib/data_porter/components/base.rb +10 -0
- data/lib/data_porter/components/failure_alert.rb +20 -0
- data/lib/data_porter/components/preview_table.rb +54 -0
- data/lib/data_porter/components/progress_bar.rb +33 -0
- data/lib/data_porter/components/results_summary.rb +19 -0
- data/lib/data_porter/components/status_badge.rb +16 -0
- data/lib/data_porter/components/summary_cards.rb +30 -0
- data/lib/data_porter/components.rb +14 -0
- data/lib/data_porter/configuration.rb +25 -0
- data/lib/data_porter/dsl/api_config.rb +25 -0
- data/lib/data_porter/dsl/column.rb +17 -0
- data/lib/data_porter/engine.rb +15 -0
- data/lib/data_porter/orchestrator.rb +141 -0
- data/lib/data_porter/record_validator.rb +32 -0
- data/lib/data_porter/registry.rb +33 -0
- data/lib/data_porter/sources/api.rb +49 -0
- data/lib/data_porter/sources/base.rb +35 -0
- data/lib/data_porter/sources/csv.rb +43 -0
- data/lib/data_porter/sources/json.rb +45 -0
- data/lib/data_porter/sources.rb +20 -0
- data/lib/data_porter/store_models/error.rb +13 -0
- data/lib/data_porter/store_models/import_record.rb +52 -0
- data/lib/data_porter/store_models/report.rb +21 -0
- data/lib/data_porter/target.rb +89 -0
- data/lib/data_porter/type_validator.rb +46 -0
- data/lib/data_porter/version.rb +5 -0
- data/lib/data_porter.rb +32 -0
- data/lib/generators/data_porter/install/install_generator.rb +33 -0
- data/lib/generators/data_porter/install/templates/create_data_porter_imports.rb.erb +21 -0
- data/lib/generators/data_porter/install/templates/initializer.rb +30 -0
- data/lib/generators/data_porter/target/target_generator.rb +44 -0
- data/lib/generators/data_porter/target/templates/target.rb.tt +20 -0
- data/sig/data_porter.rbs +4 -0
- metadata +274 -0
data/docs/SPEC.md
ADDED
|
@@ -0,0 +1,2012 @@
|
|
|
1
|
+
# DataPorter - Rails Engine pour l'import de donnees
|
|
2
|
+
|
|
3
|
+
> Gem Rails mountable qui fournit un workflow complet d'import de donnees en 3 etapes :
|
|
4
|
+
> **Upload/Config -> Preview visuel -> Validation & Creation**
|
|
5
|
+
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
## Table des matieres
|
|
9
|
+
|
|
10
|
+
- [Probleme](#probleme)
|
|
11
|
+
- [Solution](#solution)
|
|
12
|
+
- [Architecture](#architecture)
|
|
13
|
+
- [Installation (cote app hote)](#installation-cote-app-hote)
|
|
14
|
+
- [Usage : definir un Target](#usage--definir-un-target)
|
|
15
|
+
- [DSL Target en detail](#dsl-target-en-detail)
|
|
16
|
+
- [Sources supportees](#sources-supportees)
|
|
17
|
+
- [Architecture interne de la gem](#architecture-interne-de-la-gem)
|
|
18
|
+
- [Engine](#engine)
|
|
19
|
+
- [Modele DataImport](#modele-dataimport)
|
|
20
|
+
- [State Machine](#state-machine)
|
|
21
|
+
- [Orchestrator](#orchestrator)
|
|
22
|
+
- [Registry & Auto-discovery](#registry--auto-discovery)
|
|
23
|
+
- [StoreModels (JSONB)](#storemodels-jsonb)
|
|
24
|
+
- [Sources](#sources-1)
|
|
25
|
+
- [Controllers](#controllers)
|
|
26
|
+
- [Vues & Preview](#vues--preview)
|
|
27
|
+
- [Jobs (ActiveJob)](#jobs-activejob)
|
|
28
|
+
- [ActionCable & Progress](#actioncable--progress)
|
|
29
|
+
- [Multi-tenancy](#multi-tenancy)
|
|
30
|
+
- [Generators](#generators)
|
|
31
|
+
- [Override & Extension](#override--extension)
|
|
32
|
+
- [Comparaison avec maintenance_tasks](#comparaison-avec-maintenance_tasks)
|
|
33
|
+
- [Structure fichiers de la gem](#structure-fichiers-de-la-gem)
|
|
34
|
+
- [Dry Run (v0.2)](#dry-run-v02)
|
|
35
|
+
- [Rollback (v0.3 / v0.4)](#rollback-v03--v04)
|
|
36
|
+
- [Roadmap](#roadmap)
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
40
|
+
## Probleme
|
|
41
|
+
|
|
42
|
+
Dans beaucoup d'apps Rails, on a besoin d'importer de la donnee via differents moyens (CSV, JSON, API/scraping) pour creer des instances de modeles. Le pattern se repete a chaque fois :
|
|
43
|
+
|
|
44
|
+
1. Uploader ou recuperer les donnees
|
|
45
|
+
2. Les parser et les valider
|
|
46
|
+
3. Les afficher pour verification humaine
|
|
47
|
+
4. Les persister en base
|
|
48
|
+
|
|
49
|
+
Ce workflow est reimplemente a chaque fois, pour chaque modele, sans mutualisation.
|
|
50
|
+
|
|
51
|
+
## Solution
|
|
52
|
+
|
|
53
|
+
**DataPorter** est un Rails Engine mountable qui factorise tout le workflow d'import. La gem fournit l'infrastructure (modele, state machine, jobs, UI, progress) et l'app hote ne definit que la partie metier : quels modeles importer et comment les creer.
|
|
54
|
+
|
|
55
|
+
```
|
|
56
|
+
1 fichier Target = 1 type d'import operationnel
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
---
|
|
60
|
+
|
|
61
|
+
## Architecture
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
+---------------------------------------------+
|
|
65
|
+
| GEM : DataPorter |
|
|
66
|
+
| |
|
|
67
|
+
| - Rails Engine (mountable, isolate_ns) |
|
|
68
|
+
| - Modele DataImport + migration generator |
|
|
69
|
+
| - State machine (enum) |
|
|
70
|
+
| - Sources : CSV, JSON, API |
|
|
71
|
+
| - StoreModels : ImportRecord, Error, Report|
|
|
72
|
+
| - Orchestrator |
|
|
73
|
+
| - Workers (ActiveJob) |
|
|
74
|
+
| - ActionCable channel + broadcaster |
|
|
75
|
+
| - UI : liste, form, preview, progress |
|
|
76
|
+
| - DSL pour declarer des Targets |
|
|
77
|
+
| - Registry (auto-discovery) |
|
|
78
|
+
| - Generators (install, target) |
|
|
79
|
+
+---------------------------------------------+
|
|
80
|
+
^
|
|
81
|
+
| mount + configure + define targets
|
|
82
|
+
|
|
|
83
|
+
+---------------------------------------------+
|
|
84
|
+
| APP HOTE |
|
|
85
|
+
| |
|
|
86
|
+
| - config/initializers/data_porter.rb |
|
|
87
|
+
| - app/importers/guests_target.rb |
|
|
88
|
+
| - app/importers/stays_target.rb |
|
|
89
|
+
| - app/importers/vendors_target.rb |
|
|
90
|
+
| - Auth/layout (parent controller) |
|
|
91
|
+
| - Styling override (optionnel) |
|
|
92
|
+
+---------------------------------------------+
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
---
|
|
96
|
+
|
|
97
|
+
## Installation (cote app hote)
|
|
98
|
+
|
|
99
|
+
### Gemfile
|
|
100
|
+
|
|
101
|
+
```ruby
|
|
102
|
+
gem 'data_porter', '~> 0.1'
|
|
103
|
+
```
|
|
104
|
+
|
|
105
|
+
### Generator
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
rails generate data_porter:install
|
|
109
|
+
rails db:migrate
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
Le generator cree :
|
|
113
|
+
|
|
114
|
+
- La migration `data_porter_imports`
|
|
115
|
+
- `config/initializers/data_porter.rb`
|
|
116
|
+
- Le dossier `app/importers/`
|
|
117
|
+
- La route mount dans `config/routes.rb`
|
|
118
|
+
|
|
119
|
+
### Configuration
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
# config/initializers/data_porter.rb
|
|
123
|
+
DataPorter.configure do |config|
|
|
124
|
+
# Controller parent dont herite le controller de la gem
|
|
125
|
+
# Permet d'heriter l'auth, les layouts, les helpers
|
|
126
|
+
config.parent_controller = 'AdminController'
|
|
127
|
+
|
|
128
|
+
# Backend de jobs (la gem utilise ActiveJob, agnostique du backend)
|
|
129
|
+
config.queue_name = :imports
|
|
130
|
+
|
|
131
|
+
# Service ActiveStorage pour stocker les fichiers uploades
|
|
132
|
+
config.storage_service = :local
|
|
133
|
+
|
|
134
|
+
# Prefix ActionCable
|
|
135
|
+
config.cable_channel_prefix = 'data_porter'
|
|
136
|
+
|
|
137
|
+
# Builder de contexte : injecte des donnees metier dans les targets
|
|
138
|
+
# Le block recoit le controller courant
|
|
139
|
+
config.context_builder = ->(controller) {
|
|
140
|
+
OpenStruct.new(
|
|
141
|
+
hotel: controller.current_user.hotel,
|
|
142
|
+
user: controller.current_user
|
|
143
|
+
)
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
# Nombre max de records affichables en preview (pagination au-dela)
|
|
147
|
+
config.preview_limit = 500
|
|
148
|
+
|
|
149
|
+
# Activer/desactiver les sources disponibles
|
|
150
|
+
config.enabled_sources = [:csv, :json, :api]
|
|
151
|
+
end
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### Routes
|
|
155
|
+
|
|
156
|
+
```ruby
|
|
157
|
+
# config/routes.rb
|
|
158
|
+
mount DataPorter::Engine, at: '/admin/imports'
|
|
159
|
+
|
|
160
|
+
# Ou derriere un scope authentifie
|
|
161
|
+
authenticate :user, ->(u) { u.admin? } do
|
|
162
|
+
mount DataPorter::Engine, at: '/admin/imports'
|
|
163
|
+
end
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
---
|
|
167
|
+
|
|
168
|
+
## Usage : definir un Target
|
|
169
|
+
|
|
170
|
+
Un Target = un type d'import. Un fichier, une classe, c'est tout.
|
|
171
|
+
|
|
172
|
+
### Exemple simple : Guests
|
|
173
|
+
|
|
174
|
+
```ruby
|
|
175
|
+
# app/importers/guests_target.rb
|
|
176
|
+
class GuestsTarget < DataPorter::Target
|
|
177
|
+
label "Guests"
|
|
178
|
+
model Guest
|
|
179
|
+
icon "fas fa-users"
|
|
180
|
+
|
|
181
|
+
sources :csv, :json
|
|
182
|
+
|
|
183
|
+
columns do
|
|
184
|
+
column :first_name, type: :string, required: true
|
|
185
|
+
column :last_name, type: :string, required: true
|
|
186
|
+
column :email, type: :email
|
|
187
|
+
column :phone, type: :phone
|
|
188
|
+
column :language, type: :string, in: %w[fr en es de it]
|
|
189
|
+
column :date_of_birth, type: :date, format: '%d/%m/%Y'
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
csv_mapping do
|
|
193
|
+
map "Prenom" => :first_name
|
|
194
|
+
map "Nom" => :last_name
|
|
195
|
+
map "E-mail" => :email
|
|
196
|
+
map "Telephone" => :phone
|
|
197
|
+
map "Langue" => :language
|
|
198
|
+
map "Date naissance" => :date_of_birth
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
deduplicate_by :email, :phone
|
|
202
|
+
|
|
203
|
+
def persist(record, context:)
|
|
204
|
+
Guest.create!(
|
|
205
|
+
hotel: context.hotel,
|
|
206
|
+
**record.attributes
|
|
207
|
+
)
|
|
208
|
+
end
|
|
209
|
+
end
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
### Exemple avec source API : Scraper Airbnb
|
|
213
|
+
|
|
214
|
+
```ruby
|
|
215
|
+
# app/importers/airbnb_stays_target.rb
|
|
216
|
+
class AirbnbStaysTarget < DataPorter::Target
|
|
217
|
+
label "Stays (Airbnb Scraper)"
|
|
218
|
+
model Stay
|
|
219
|
+
icon "fas fa-bed"
|
|
220
|
+
|
|
221
|
+
sources :api
|
|
222
|
+
|
|
223
|
+
api_config do
|
|
224
|
+
endpoint ->(params) { "https://scraper.internal/api/stays?housing_id=#{params[:housing_id]}" }
|
|
225
|
+
headers { { 'Authorization' => "Bearer #{ENV['SCRAPER_TOKEN']}" } }
|
|
226
|
+
response_root :stays
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
params do
|
|
230
|
+
param :housing_id, type: :integer, required: true, label: "Housing"
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
columns do
|
|
234
|
+
column :external_confirmation_code, type: :string, required: true
|
|
235
|
+
column :guest_name, type: :string, required: true
|
|
236
|
+
column :check_in, type: :date, required: true
|
|
237
|
+
column :check_out, type: :date, required: true
|
|
238
|
+
column :total_price, type: :decimal
|
|
239
|
+
column :nb_guests, type: :integer
|
|
240
|
+
column :source, type: :string
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
deduplicate_by :external_confirmation_code
|
|
244
|
+
|
|
245
|
+
def persist(record, context:)
|
|
246
|
+
Stays::Upsert.call(
|
|
247
|
+
hotel: context.hotel,
|
|
248
|
+
housing_id: record.housing_id,
|
|
249
|
+
**record.attributes
|
|
250
|
+
)
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
```
|
|
254
|
+
|
|
255
|
+
### Exemple JSON : Vendors
|
|
256
|
+
|
|
257
|
+
```ruby
|
|
258
|
+
# app/importers/vendors_target.rb
|
|
259
|
+
class VendorsTarget < DataPorter::Target
|
|
260
|
+
label "Vendors"
|
|
261
|
+
model Vendor
|
|
262
|
+
icon "fas fa-store"
|
|
263
|
+
|
|
264
|
+
sources :csv, :json
|
|
265
|
+
|
|
266
|
+
columns do
|
|
267
|
+
column :name, type: :string, required: true
|
|
268
|
+
column :email, type: :email
|
|
269
|
+
column :phone, type: :phone
|
|
270
|
+
column :address, type: :string
|
|
271
|
+
column :category, type: :string, in: %w[cleaning maintenance catering transport]
|
|
272
|
+
column :website, type: :url
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Hook optionnel : transformation apres parsing, avant preview
|
|
276
|
+
def transform(record)
|
|
277
|
+
record.data[:name] = record.data[:name]&.titleize
|
|
278
|
+
record.data[:category] = record.data[:category]&.downcase
|
|
279
|
+
record
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
# Hook optionnel : validation metier custom
|
|
283
|
+
def validate(record)
|
|
284
|
+
if record.data[:email].blank? && record.data[:phone].blank?
|
|
285
|
+
record.add_warning("Ni email ni telephone renseigne")
|
|
286
|
+
end
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
def persist(record, context:)
|
|
290
|
+
Vendor.create!(hotel: context.hotel, **record.attributes)
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
### Generator de target
|
|
296
|
+
|
|
297
|
+
```bash
|
|
298
|
+
rails generate data_porter:target vendors name:string:required email:email phone:phone
|
|
299
|
+
```
|
|
300
|
+
|
|
301
|
+
Genere le squelette dans `app/importers/vendors_target.rb`.
|
|
302
|
+
|
|
303
|
+
---
|
|
304
|
+
|
|
305
|
+
## DSL Target en detail
|
|
306
|
+
|
|
307
|
+
### `label(string)`
|
|
308
|
+
|
|
309
|
+
Nom affiche dans l'UI.
|
|
310
|
+
|
|
311
|
+
### `model(class)`
|
|
312
|
+
|
|
313
|
+
Classe ActiveRecord cible. Utilise pour l'affichage et la detection automatique des colonnes (optionnel).
|
|
314
|
+
|
|
315
|
+
### `icon(string)`
|
|
316
|
+
|
|
317
|
+
Classe CSS d'icone (FontAwesome). Affiche dans la liste des targets.
|
|
318
|
+
|
|
319
|
+
### `sources(*types)`
|
|
320
|
+
|
|
321
|
+
Types de sources acceptes pour ce target. Valeurs : `:csv`, `:json`, `:api`.
|
|
322
|
+
|
|
323
|
+
### `columns(&block)`
|
|
324
|
+
|
|
325
|
+
Definition des colonnes attendues. Chaque colonne a :
|
|
326
|
+
|
|
327
|
+
| Option | Type | Description |
|
|
328
|
+
| ---------- | ------- | ----------------------------------------------------------------------------------------------- |
|
|
329
|
+
| `type` | Symbol | `:string`, `:integer`, `:decimal`, `:date`, `:datetime`, `:email`, `:phone`, `:url`, `:boolean` |
|
|
330
|
+
| `required` | Boolean | Champ obligatoire (default: false) |
|
|
331
|
+
| `in` | Array | Valeurs autorisees |
|
|
332
|
+
| `format` | String | Format de parsing (pour les dates) |
|
|
333
|
+
| `label` | String | Label affiche en preview (default: humanize du nom) |
|
|
334
|
+
|
|
335
|
+
### `csv_mapping(&block)`
|
|
336
|
+
|
|
337
|
+
Mapping entre les headers CSV et les noms de colonnes internes.
|
|
338
|
+
Si absent, la gem tente un matching automatique (parameterize + fuzzy).
|
|
339
|
+
|
|
340
|
+
### `deduplicate_by(*keys)`
|
|
341
|
+
|
|
342
|
+
Colonnes utilisees pour detecter les doublons avec les records existants en base.
|
|
343
|
+
En preview, les doublons sont marques avec le status `:duplicate`.
|
|
344
|
+
|
|
345
|
+
### `params(&block)`
|
|
346
|
+
|
|
347
|
+
Parametres additionnels demandes a l'utilisateur au moment du lancement.
|
|
348
|
+
Utilise pour les sources API (ex: housing_id) ou pour filtrer les donnees.
|
|
349
|
+
|
|
350
|
+
### `api_config(&block)`
|
|
351
|
+
|
|
352
|
+
Configuration specifique a la source API :
|
|
353
|
+
|
|
354
|
+
- `endpoint` : URL ou lambda recevant les params
|
|
355
|
+
- `headers` : Hash ou lambda
|
|
356
|
+
- `response_root` : Cle JSON contenant le tableau de records
|
|
357
|
+
|
|
358
|
+
### Hooks overridables
|
|
359
|
+
|
|
360
|
+
| Hook | Moment | Usage |
|
|
361
|
+
| ----------------------------------- | -------------------------------- | ------------------------- |
|
|
362
|
+
| `transform(record)` | Apres parsing, avant preview | Normalisation des donnees |
|
|
363
|
+
| `validate(record)` | Apres transform, avant preview | Validation metier custom |
|
|
364
|
+
| `persist(record, context:)` | A l'import final | Creation en base |
|
|
365
|
+
| `after_import(results, context:)` | Apres l'import complet | Notifications, logs, etc. |
|
|
366
|
+
| `on_error(record, error, context:)` | Si persist echoue pour un record | Gestion d'erreur custom |
|
|
367
|
+
|
|
368
|
+
---
|
|
369
|
+
|
|
370
|
+
## Sources supportees
|
|
371
|
+
|
|
372
|
+
### CSV
|
|
373
|
+
|
|
374
|
+
```
|
|
375
|
+
Upload d'un fichier CSV via ActiveStorage.
|
|
376
|
+
Headers mappes aux colonnes via csv_mapping ou matching auto.
|
|
377
|
+
Options CSV Ruby configurables (separateur, encoding, skip_lines...).
|
|
378
|
+
```
|
|
379
|
+
|
|
380
|
+
### JSON
|
|
381
|
+
|
|
382
|
+
```
|
|
383
|
+
Upload d'un fichier JSON ou saisie en textarea.
|
|
384
|
+
Attend un tableau d'objets a la racine ou sous une cle configurable.
|
|
385
|
+
Les cles des objets sont mappees aux colonnes.
|
|
386
|
+
```
|
|
387
|
+
|
|
388
|
+
### API
|
|
389
|
+
|
|
390
|
+
```
|
|
391
|
+
Appel HTTP a un endpoint configure dans api_config.
|
|
392
|
+
Les params du target sont injectes dans l'URL.
|
|
393
|
+
La reponse JSON est parsee comme un tableau de records.
|
|
394
|
+
Ideal pour les scrapers et les APIs internes.
|
|
395
|
+
```
|
|
396
|
+
|
|
397
|
+
---
|
|
398
|
+
|
|
399
|
+
## Architecture interne de la gem
|
|
400
|
+
|
|
401
|
+
### Engine
|
|
402
|
+
|
|
403
|
+
```ruby
|
|
404
|
+
# lib/data_porter/engine.rb
|
|
405
|
+
module DataPorter
|
|
406
|
+
class Engine < ::Rails::Engine
|
|
407
|
+
isolate_namespace DataPorter
|
|
408
|
+
|
|
409
|
+
initializer 'data_porter.assets' do |app|
|
|
410
|
+
if app.config.respond_to?(:assets)
|
|
411
|
+
app.config.assets.precompile += %w[
|
|
412
|
+
data_porter/application.css
|
|
413
|
+
data_porter/application.js
|
|
414
|
+
]
|
|
415
|
+
end
|
|
416
|
+
end
|
|
417
|
+
|
|
418
|
+
initializer 'data_porter.active_storage' do
|
|
419
|
+
ActiveSupport.on_load(:active_storage_blob) do
|
|
420
|
+
# config storage
|
|
421
|
+
end
|
|
422
|
+
end
|
|
423
|
+
|
|
424
|
+
config.to_prepare do
|
|
425
|
+
Dir[Rails.root.join('app/importers/*_target.rb')].each { |f| require f }
|
|
426
|
+
end
|
|
427
|
+
end
|
|
428
|
+
end
|
|
429
|
+
```
|
|
430
|
+
|
|
431
|
+
### Modele DataImport
|
|
432
|
+
|
|
433
|
+
```ruby
|
|
434
|
+
# app/models/data_porter/data_import.rb
|
|
435
|
+
module DataPorter
|
|
436
|
+
class DataImport < ActiveRecord::Base
|
|
437
|
+
self.table_name = 'data_porter_imports'
|
|
438
|
+
|
|
439
|
+
belongs_to :user, polymorphic: true
|
|
440
|
+
|
|
441
|
+
has_one_attached :file
|
|
442
|
+
|
|
443
|
+
enum :status, {
|
|
444
|
+
pending: 0,
|
|
445
|
+
parsing: 1,
|
|
446
|
+
previewing: 2,
|
|
447
|
+
importing: 3,
|
|
448
|
+
completed: 4,
|
|
449
|
+
failed: 5
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
attribute :records, DataPorter::StoreModels::ImportRecord.to_array_type, default: []
|
|
453
|
+
attribute :report, DataPorter::StoreModels::Report.to_type
|
|
454
|
+
|
|
455
|
+
serialize :config, coder: JSON
|
|
456
|
+
|
|
457
|
+
validates :target_key, presence: true
|
|
458
|
+
validates :source_type, presence: true, inclusion: { in: %w[csv json api] }
|
|
459
|
+
|
|
460
|
+
def target_class
|
|
461
|
+
DataPorter::Registry.find(target_key)
|
|
462
|
+
end
|
|
463
|
+
|
|
464
|
+
def source_class
|
|
465
|
+
DataPorter::Sources.resolve(source_type)
|
|
466
|
+
end
|
|
467
|
+
|
|
468
|
+
def previewable?
|
|
469
|
+
previewing? && records.any?
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
def importable_records
|
|
473
|
+
records.select { |r| r.status == 'complete' }
|
|
474
|
+
end
|
|
475
|
+
|
|
476
|
+
def records_summary
|
|
477
|
+
records.group_by(&:status).transform_values(&:count)
|
|
478
|
+
end
|
|
479
|
+
end
|
|
480
|
+
end
|
|
481
|
+
```
|
|
482
|
+
|
|
483
|
+
#### Migration
|
|
484
|
+
|
|
485
|
+
```ruby
|
|
486
|
+
# db/migrate/xxx_create_data_porter_imports.rb
|
|
487
|
+
class CreateDataPorterImports < ActiveRecord::Migration[7.2]
|
|
488
|
+
def change
|
|
489
|
+
create_table :data_porter_imports do |t|
|
|
490
|
+
t.string :target_key, null: false
|
|
491
|
+
t.string :source_type, null: false, default: 'csv'
|
|
492
|
+
t.integer :status, null: false, default: 0
|
|
493
|
+
t.jsonb :records, null: false, default: []
|
|
494
|
+
t.jsonb :report, null: false, default: {}
|
|
495
|
+
t.jsonb :config, null: false, default: {}
|
|
496
|
+
|
|
497
|
+
t.references :user, polymorphic: true, null: false
|
|
498
|
+
|
|
499
|
+
t.timestamps
|
|
500
|
+
end
|
|
501
|
+
|
|
502
|
+
add_index :data_porter_imports, :status
|
|
503
|
+
add_index :data_porter_imports, :target_key
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
### State Machine
|
|
509
|
+
|
|
510
|
+
```
|
|
511
|
+
pending -----> parsing -----> previewing -----> importing -----> completed
|
|
512
|
+
| |
|
|
513
|
+
+--------------> failed <--------+
|
|
514
|
+
```
|
|
515
|
+
|
|
516
|
+
| Status | Description | Declencheur |
|
|
517
|
+
| ------------ | ------------------------------------------------- | ------------------------ |
|
|
518
|
+
| `pending` | Import cree, en attente de lancement | `create` |
|
|
519
|
+
| `parsing` | Source en cours de parsing (async) | `ParseJob` |
|
|
520
|
+
| `previewing` | Donnees parsees, en attente de validation humaine | Fin du parsing |
|
|
521
|
+
| `importing` | Creation des records en cours (async) | Confirmation utilisateur |
|
|
522
|
+
| `completed` | Tous les records importes | Fin de l'import |
|
|
523
|
+
| `failed` | Erreur fatale pendant parsing ou import | Exception |
|
|
524
|
+
|
|
525
|
+
### Orchestrator
|
|
526
|
+
|
|
527
|
+
Le chef d'orchestre qui coordonne source + target.
|
|
528
|
+
|
|
529
|
+
```ruby
|
|
530
|
+
# lib/data_porter/orchestrator.rb
|
|
531
|
+
module DataPorter
|
|
532
|
+
class Orchestrator
|
|
533
|
+
def initialize(data_import)
|
|
534
|
+
@data_import = data_import
|
|
535
|
+
@source = data_import.source_class.new(data_import)
|
|
536
|
+
@target = data_import.target_class.new
|
|
537
|
+
end
|
|
538
|
+
|
|
539
|
+
def parse!
|
|
540
|
+
@data_import.parsing!
|
|
541
|
+
raw_rows = @source.fetch
|
|
542
|
+
records = raw_rows.each_with_index.map do |row, index|
|
|
543
|
+
record = DataPorter::StoreModels::ImportRecord.new(
|
|
544
|
+
line_number: index + 1,
|
|
545
|
+
data: @target.class._columns.each_with_object({}) do |col, hash|
|
|
546
|
+
hash[col.name] = row[col.name] || row[col.name.to_s]
|
|
547
|
+
end
|
|
548
|
+
)
|
|
549
|
+
record = @target.transform(record)
|
|
550
|
+
@target.validate(record)
|
|
551
|
+
record.run_validations!(@target.class._columns, @target.class._dedup_keys)
|
|
552
|
+
broadcast_progress(index + 1, raw_rows.size)
|
|
553
|
+
record
|
|
554
|
+
end
|
|
555
|
+
|
|
556
|
+
@data_import.update!(records: records, status: :previewing)
|
|
557
|
+
build_report
|
|
558
|
+
broadcast_success
|
|
559
|
+
rescue StandardError => e
|
|
560
|
+
handle_failure(e)
|
|
561
|
+
end
|
|
562
|
+
|
|
563
|
+
def import!
|
|
564
|
+
@data_import.importing!
|
|
565
|
+
importable = @data_import.importable_records
|
|
566
|
+
context = build_context
|
|
567
|
+
|
|
568
|
+
results = { created: 0, skipped: 0, errored: 0 }
|
|
569
|
+
|
|
570
|
+
importable.each_with_index do |record, index|
|
|
571
|
+
@target.persist(record, context: context)
|
|
572
|
+
record.status = 'imported'
|
|
573
|
+
results[:created] += 1
|
|
574
|
+
broadcast_progress(index + 1, importable.size)
|
|
575
|
+
rescue StandardError => e
|
|
576
|
+
record.status = 'error'
|
|
577
|
+
record.errors_list << DataPorter::StoreModels::Error.new(message: e.message)
|
|
578
|
+
@target.on_error(record, e, context: context)
|
|
579
|
+
results[:errored] += 1
|
|
580
|
+
end
|
|
581
|
+
|
|
582
|
+
@data_import.update!(status: :completed)
|
|
583
|
+
@target.after_import(results, context: context)
|
|
584
|
+
broadcast_success
|
|
585
|
+
rescue StandardError => e
|
|
586
|
+
handle_failure(e)
|
|
587
|
+
end
|
|
588
|
+
|
|
589
|
+
private
|
|
590
|
+
|
|
591
|
+
def broadcast_progress(current, total)
|
|
592
|
+
DataPorter::Broadcaster.new(@data_import.id).progress(current, total)
|
|
593
|
+
end
|
|
594
|
+
|
|
595
|
+
def broadcast_success
|
|
596
|
+
DataPorter::Broadcaster.new(@data_import.id).success
|
|
597
|
+
end
|
|
598
|
+
|
|
599
|
+
def handle_failure(error)
|
|
600
|
+
report = DataPorter::StoreModels::Report.new(
|
|
601
|
+
error_reports: [DataPorter::StoreModels::Error.new(message: error.message)]
|
|
602
|
+
)
|
|
603
|
+
@data_import.update!(status: :failed, report: report)
|
|
604
|
+
DataPorter::Broadcaster.new(@data_import.id).failure(error.message)
|
|
605
|
+
end
|
|
606
|
+
|
|
607
|
+
def build_context
|
|
608
|
+
DataPorter.configuration.context_builder&.call(@data_import)
|
|
609
|
+
end
|
|
610
|
+
|
|
611
|
+
def build_report
|
|
612
|
+
summary = @data_import.records_summary
|
|
613
|
+
report = DataPorter::StoreModels::Report.new(
|
|
614
|
+
records_count: @data_import.records.size,
|
|
615
|
+
complete_count: summary['complete'] || 0,
|
|
616
|
+
partial_count: summary['partial'] || 0,
|
|
617
|
+
missing_count: summary['missing'] || 0,
|
|
618
|
+
duplicate_count: summary['duplicate'] || 0
|
|
619
|
+
)
|
|
620
|
+
@data_import.update!(report: report)
|
|
621
|
+
end
|
|
622
|
+
end
|
|
623
|
+
end
|
|
624
|
+
```
|
|
625
|
+
|
|
626
|
+
### Registry & Auto-discovery
|
|
627
|
+
|
|
628
|
+
```ruby
|
|
629
|
+
# lib/data_porter/registry.rb
|
|
630
|
+
module DataPorter
|
|
631
|
+
class Registry
|
|
632
|
+
class << self
|
|
633
|
+
def targets
|
|
634
|
+
@targets ||= {}
|
|
635
|
+
end
|
|
636
|
+
|
|
637
|
+
def register(key, klass)
|
|
638
|
+
targets[key.to_s] = klass
|
|
639
|
+
end
|
|
640
|
+
|
|
641
|
+
def find(key)
|
|
642
|
+
targets[key.to_s] || raise(TargetNotFound, "Target '#{key}' not found")
|
|
643
|
+
end
|
|
644
|
+
|
|
645
|
+
def available
|
|
646
|
+
targets.map { |key, klass| { key: key, label: klass._label, icon: klass._icon } }
|
|
647
|
+
end
|
|
648
|
+
|
|
649
|
+
def refresh!
|
|
650
|
+
@targets = nil
|
|
651
|
+
DataPorter::Target.descendants.each do |klass|
|
|
652
|
+
next if klass._label.nil?
|
|
653
|
+
register(klass._label.parameterize, klass)
|
|
654
|
+
end
|
|
655
|
+
end
|
|
656
|
+
end
|
|
657
|
+
end
|
|
658
|
+
end
|
|
659
|
+
```
|
|
660
|
+
|
|
661
|
+
### StoreModels (JSONB)
|
|
662
|
+
|
|
663
|
+
Donnees typees stockees en JSONB dans `data_porter_imports.records`.
|
|
664
|
+
|
|
665
|
+
```ruby
|
|
666
|
+
# lib/data_porter/store_models/import_record.rb
|
|
667
|
+
module DataPorter
|
|
668
|
+
module StoreModels
|
|
669
|
+
class ImportRecord
|
|
670
|
+
include StoreModel::Model
|
|
671
|
+
|
|
672
|
+
attribute :line_number, :integer
|
|
673
|
+
attribute :status, :string, default: 'pending'
|
|
674
|
+
attribute :data, :json, default: {}
|
|
675
|
+
attribute :errors_list, DataPorter::StoreModels::Error.to_array_type, default: []
|
|
676
|
+
attribute :warnings, DataPorter::StoreModels::Error.to_array_type, default: []
|
|
677
|
+
attribute :target_id, :integer
|
|
678
|
+
|
|
679
|
+
STATUSES = %w[pending complete partial missing duplicate imported error].freeze
|
|
680
|
+
|
|
681
|
+
def complete? = status == 'complete'
|
|
682
|
+
def importable? = status == 'complete'
|
|
683
|
+
|
|
684
|
+
def add_error(message)
|
|
685
|
+
errors_list << DataPorter::StoreModels::Error.new(message: message)
|
|
686
|
+
end
|
|
687
|
+
|
|
688
|
+
def add_warning(message)
|
|
689
|
+
warnings << DataPorter::StoreModels::Error.new(message: message)
|
|
690
|
+
end
|
|
691
|
+
|
|
692
|
+
def attributes
|
|
693
|
+
data.symbolize_keys.compact
|
|
694
|
+
end
|
|
695
|
+
|
|
696
|
+
def run_validations!(columns, dedup_keys)
|
|
697
|
+
validate_required_columns!(columns)
|
|
698
|
+
validate_types!(columns)
|
|
699
|
+
validate_inclusions!(columns)
|
|
700
|
+
check_duplicates!(dedup_keys)
|
|
701
|
+
determine_status!
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
private
|
|
705
|
+
|
|
706
|
+
def validate_required_columns!(columns)
|
|
707
|
+
columns.select(&:required).each do |col|
|
|
708
|
+
if data[col.name].blank?
|
|
709
|
+
add_error("#{col.label} is required")
|
|
710
|
+
end
|
|
711
|
+
end
|
|
712
|
+
end
|
|
713
|
+
|
|
714
|
+
def validate_types!(columns)
|
|
715
|
+
columns.each do |col|
|
|
716
|
+
value = data[col.name]
|
|
717
|
+
next if value.blank?
|
|
718
|
+
unless TypeValidator.valid?(value, col.type, col.options)
|
|
719
|
+
add_error("#{col.label}: invalid #{col.type}")
|
|
720
|
+
end
|
|
721
|
+
end
|
|
722
|
+
end
|
|
723
|
+
|
|
724
|
+
def validate_inclusions!(columns)
|
|
725
|
+
columns.select { |c| c.options[:in] }.each do |col|
|
|
726
|
+
value = data[col.name]
|
|
727
|
+
next if value.blank?
|
|
728
|
+
unless col.options[:in].include?(value)
|
|
729
|
+
add_error("#{col.label}: must be one of #{col.options[:in].join(', ')}")
|
|
730
|
+
end
|
|
731
|
+
end
|
|
732
|
+
end
|
|
733
|
+
|
|
734
|
+
def check_duplicates!(dedup_keys)
|
|
735
|
+
return if dedup_keys.blank?
|
|
736
|
+
# la logique de dedup est geree par l'orchestrator
|
|
737
|
+
# qui a acces a la base et aux autres records du batch
|
|
738
|
+
end
|
|
739
|
+
|
|
740
|
+
def determine_status!
|
|
741
|
+
if errors_list.any? { |e| e.message.include?('is required') }
|
|
742
|
+
self.status = 'missing'
|
|
743
|
+
elsif errors_list.any?
|
|
744
|
+
self.status = 'partial'
|
|
745
|
+
else
|
|
746
|
+
self.status = 'complete'
|
|
747
|
+
end
|
|
748
|
+
end
|
|
749
|
+
end
|
|
750
|
+
end
|
|
751
|
+
end
|
|
752
|
+
```
|
|
753
|
+
|
|
754
|
+
```ruby
|
|
755
|
+
# lib/data_porter/store_models/error.rb
|
|
756
|
+
module DataPorter
|
|
757
|
+
module StoreModels
|
|
758
|
+
class Error
|
|
759
|
+
include StoreModel::Model
|
|
760
|
+
|
|
761
|
+
attribute :message, :string
|
|
762
|
+
end
|
|
763
|
+
end
|
|
764
|
+
end
|
|
765
|
+
```
|
|
766
|
+
|
|
767
|
+
```ruby
|
|
768
|
+
# lib/data_porter/store_models/report.rb
|
|
769
|
+
module DataPorter
|
|
770
|
+
module StoreModels
|
|
771
|
+
class Report
|
|
772
|
+
include StoreModel::Model
|
|
773
|
+
|
|
774
|
+
attribute :records_count, :integer, default: 0
|
|
775
|
+
attribute :complete_count, :integer, default: 0
|
|
776
|
+
attribute :partial_count, :integer, default: 0
|
|
777
|
+
attribute :missing_count, :integer, default: 0
|
|
778
|
+
attribute :duplicate_count, :integer, default: 0
|
|
779
|
+
attribute :imported_count, :integer, default: 0
|
|
780
|
+
attribute :errored_count, :integer, default: 0
|
|
781
|
+
attribute :error_reports, DataPorter::StoreModels::Error.to_array_type, default: []
|
|
782
|
+
end
|
|
783
|
+
end
|
|
784
|
+
end
|
|
785
|
+
```
|
|
786
|
+
|
|
787
|
+
### Sources
|
|
788
|
+
|
|
789
|
+
```ruby
|
|
790
|
+
# lib/data_porter/sources/base.rb
|
|
791
|
+
module DataPorter
|
|
792
|
+
module Sources
|
|
793
|
+
class Base
|
|
794
|
+
def initialize(data_import)
|
|
795
|
+
@data_import = data_import
|
|
796
|
+
@target_class = data_import.target_class
|
|
797
|
+
end
|
|
798
|
+
|
|
799
|
+
def fetch
|
|
800
|
+
raise NotImplementedError
|
|
801
|
+
end
|
|
802
|
+
|
|
803
|
+
private
|
|
804
|
+
|
|
805
|
+
def apply_csv_mapping(row)
|
|
806
|
+
mappings = @target_class._csv_mappings || {}
|
|
807
|
+
if mappings.any?
|
|
808
|
+
mappings.each_with_object({}) do |(header, column), hash|
|
|
809
|
+
hash[column] = row[header]
|
|
810
|
+
end
|
|
811
|
+
else
|
|
812
|
+
row.to_h.transform_keys { |k| k.parameterize(separator: '_').to_sym }
|
|
813
|
+
end
|
|
814
|
+
end
|
|
815
|
+
end
|
|
816
|
+
end
|
|
817
|
+
end
|
|
818
|
+
```
|
|
819
|
+
|
|
820
|
+
```ruby
|
|
821
|
+
# lib/data_porter/sources/csv.rb
|
|
822
|
+
module DataPorter
|
|
823
|
+
module Sources
|
|
824
|
+
class Csv < Base
|
|
825
|
+
def fetch
|
|
826
|
+
file = @data_import.file
|
|
827
|
+
csv_options = @target_class._csv_options || { headers: true }
|
|
828
|
+
|
|
829
|
+
rows = []
|
|
830
|
+
::CSV.parse(URI.parse(file.url).open, **csv_options) do |row|
|
|
831
|
+
rows << apply_csv_mapping(row)
|
|
832
|
+
end
|
|
833
|
+
rows
|
|
834
|
+
end
|
|
835
|
+
end
|
|
836
|
+
end
|
|
837
|
+
end
|
|
838
|
+
```
|
|
839
|
+
|
|
840
|
+
```ruby
|
|
841
|
+
# lib/data_porter/sources/json.rb
|
|
842
|
+
module DataPorter
|
|
843
|
+
module Sources
|
|
844
|
+
class Json < Base
|
|
845
|
+
def fetch
|
|
846
|
+
raw = if @data_import.file.attached?
|
|
847
|
+
@data_import.file.download
|
|
848
|
+
else
|
|
849
|
+
@data_import.config['raw_json']
|
|
850
|
+
end
|
|
851
|
+
|
|
852
|
+
parsed = JSON.parse(raw)
|
|
853
|
+
root = @target_class._json_root
|
|
854
|
+
records = root ? parsed.dig(*root.split('.')) : parsed
|
|
855
|
+
|
|
856
|
+
Array.wrap(records).map do |hash|
|
|
857
|
+
hash.transform_keys { |k| k.parameterize(separator: '_').to_sym }
|
|
858
|
+
end
|
|
859
|
+
end
|
|
860
|
+
end
|
|
861
|
+
end
|
|
862
|
+
end
|
|
863
|
+
```
|
|
864
|
+
|
|
865
|
+
```ruby
|
|
866
|
+
# lib/data_porter/sources/api.rb
|
|
867
|
+
module DataPorter
|
|
868
|
+
module Sources
|
|
869
|
+
class Api < Base
|
|
870
|
+
def fetch
|
|
871
|
+
api = @target_class._api_config
|
|
872
|
+
params = @data_import.config.symbolize_keys
|
|
873
|
+
|
|
874
|
+
url = api.endpoint.is_a?(Proc) ? api.endpoint.call(params) : api.endpoint
|
|
875
|
+
headers = api.headers.is_a?(Proc) ? api.headers.call : api.headers
|
|
876
|
+
|
|
877
|
+
response = Net::HTTP.get(URI(url), headers)
|
|
878
|
+
parsed = JSON.parse(response)
|
|
879
|
+
|
|
880
|
+
root = api.response_root
|
|
881
|
+
records = root ? parsed[root.to_s] : parsed
|
|
882
|
+
|
|
883
|
+
Array.wrap(records).map do |hash|
|
|
884
|
+
hash.transform_keys { |k| k.parameterize(separator: '_').to_sym }
|
|
885
|
+
end
|
|
886
|
+
end
|
|
887
|
+
end
|
|
888
|
+
end
|
|
889
|
+
end
|
|
890
|
+
```
|
|
891
|
+
|
|
892
|
+
### Controllers
|
|
893
|
+
|
|
894
|
+
```ruby
|
|
895
|
+
# app/controllers/data_porter/imports_controller.rb
|
|
896
|
+
module DataPorter
|
|
897
|
+
class ImportsController < DataPorter.configuration.parent_controller.constantize
|
|
898
|
+
before_action :set_import, only: [:show, :parse, :confirm, :cancel]
|
|
899
|
+
|
|
900
|
+
def index
|
|
901
|
+
@imports = DataPorter::DataImport.order(created_at: :desc).page(params[:page])
|
|
902
|
+
end
|
|
903
|
+
|
|
904
|
+
def new
|
|
905
|
+
@import = DataPorter::DataImport.new
|
|
906
|
+
@targets = DataPorter::Registry.available
|
|
907
|
+
end
|
|
908
|
+
|
|
909
|
+
def create
|
|
910
|
+
@import = DataPorter::DataImport.new(import_params)
|
|
911
|
+
@import.user = current_user
|
|
912
|
+
@import.status = :pending
|
|
913
|
+
|
|
914
|
+
if @import.save
|
|
915
|
+
DataPorter::ParseJob.perform_later(@import.id)
|
|
916
|
+
redirect_to import_path(@import)
|
|
917
|
+
else
|
|
918
|
+
@targets = DataPorter::Registry.available
|
|
919
|
+
render :new
|
|
920
|
+
end
|
|
921
|
+
end
|
|
922
|
+
|
|
923
|
+
# GET /imports/:id
|
|
924
|
+
# Affiche le progress (si parsing/importing) ou le preview (si previewing)
|
|
925
|
+
def show
|
|
926
|
+
@target = @import.target_class
|
|
927
|
+
@records = @import.records
|
|
928
|
+
@grouped = @records.group_by(&:status)
|
|
929
|
+
end
|
|
930
|
+
|
|
931
|
+
# POST /imports/:id/parse
|
|
932
|
+
# Relance le parsing (si echec)
|
|
933
|
+
def parse
|
|
934
|
+
@import.update!(status: :pending)
|
|
935
|
+
DataPorter::ParseJob.perform_later(@import.id)
|
|
936
|
+
redirect_to import_path(@import)
|
|
937
|
+
end
|
|
938
|
+
|
|
939
|
+
# POST /imports/:id/confirm
|
|
940
|
+
# Lance l'import des records valides
|
|
941
|
+
def confirm
|
|
942
|
+
DataPorter::ImportJob.perform_later(@import.id)
|
|
943
|
+
redirect_to import_path(@import)
|
|
944
|
+
end
|
|
945
|
+
|
|
946
|
+
# POST /imports/:id/cancel
|
|
947
|
+
def cancel
|
|
948
|
+
@import.update!(status: :failed)
|
|
949
|
+
redirect_to imports_path
|
|
950
|
+
end
|
|
951
|
+
|
|
952
|
+
private
|
|
953
|
+
|
|
954
|
+
def set_import
|
|
955
|
+
@import = DataPorter::DataImport.find(params[:id])
|
|
956
|
+
end
|
|
957
|
+
|
|
958
|
+
def import_params
|
|
959
|
+
params.require(:data_import).permit(:target_key, :source_type, :file, config: {})
|
|
960
|
+
end
|
|
961
|
+
end
|
|
962
|
+
end
|
|
963
|
+
```
|
|
964
|
+
|
|
965
|
+
### Vues & Preview
|
|
966
|
+
|
|
967
|
+
Le coeur de la valeur ajoutee : un tableau de preview auto-genere a partir des colonnes du Target.
|
|
968
|
+
|
|
969
|
+
#### Layout
|
|
970
|
+
|
|
971
|
+
```erb
|
|
972
|
+
<%# app/views/layouts/data_porter/application.html.erb %>
|
|
973
|
+
<%= render template: DataPorter.configuration.layout || 'layouts/application' %>
|
|
974
|
+
```
|
|
975
|
+
|
|
976
|
+
L'engine herite du layout de l'app hote via le `parent_controller`.
|
|
977
|
+
|
|
978
|
+
#### Page `show` (Preview)
|
|
979
|
+
|
|
980
|
+
```erb
|
|
981
|
+
<%# app/views/data_porter/imports/show.html.erb %>
|
|
982
|
+
|
|
983
|
+
<div data-controller="data-porter--progress"
|
|
984
|
+
data-data-porter--progress-id-value="<%= @import.id %>">
|
|
985
|
+
|
|
986
|
+
<%# === HEADER === %>
|
|
987
|
+
<h1><%= @target._label %> Import #<%= @import.id %></h1>
|
|
988
|
+
<p>Status: <span class="badge badge-<%= status_color(@import.status) %>">
|
|
989
|
+
<%= @import.status.humanize %>
|
|
990
|
+
</span></p>
|
|
991
|
+
|
|
992
|
+
<%# === PROGRESS BAR (visible pendant parsing/importing) === %>
|
|
993
|
+
<% if @import.parsing? || @import.importing? %>
|
|
994
|
+
<div class="progress">
|
|
995
|
+
<div class="progress-bar" data-data-porter--progress-target="bar" style="width: 0%">
|
|
996
|
+
<span data-data-porter--progress-target="text">0%</span>
|
|
997
|
+
</div>
|
|
998
|
+
</div>
|
|
999
|
+
<% end %>
|
|
1000
|
+
|
|
1001
|
+
<%# === SUMMARY (visible apres parsing) === %>
|
|
1002
|
+
<% if @import.previewable? || @import.completed? %>
|
|
1003
|
+
<div class="summary-cards">
|
|
1004
|
+
<div class="card card-complete">
|
|
1005
|
+
<strong><%= @grouped['complete']&.size || 0 %></strong> Ready
|
|
1006
|
+
</div>
|
|
1007
|
+
<div class="card card-partial">
|
|
1008
|
+
<strong><%= @grouped['partial']&.size || 0 %></strong> Incomplete
|
|
1009
|
+
</div>
|
|
1010
|
+
<div class="card card-missing">
|
|
1011
|
+
<strong><%= @grouped['missing']&.size || 0 %></strong> Missing
|
|
1012
|
+
</div>
|
|
1013
|
+
<div class="card card-duplicate">
|
|
1014
|
+
<strong><%= @grouped['duplicate']&.size || 0 %></strong> Duplicates
|
|
1015
|
+
</div>
|
|
1016
|
+
</div>
|
|
1017
|
+
<% end %>
|
|
1018
|
+
|
|
1019
|
+
<%# === PREVIEW TABLE (visible en previewing) === %>
|
|
1020
|
+
<% if @import.previewable? %>
|
|
1021
|
+
<table class="table">
|
|
1022
|
+
<thead>
|
|
1023
|
+
<tr>
|
|
1024
|
+
<th>#</th>
|
|
1025
|
+
<th>Status</th>
|
|
1026
|
+
<% @target._columns.each do |col| %>
|
|
1027
|
+
<th><%= col.label %></th>
|
|
1028
|
+
<% end %>
|
|
1029
|
+
<th>Errors</th>
|
|
1030
|
+
</tr>
|
|
1031
|
+
</thead>
|
|
1032
|
+
<tbody>
|
|
1033
|
+
<% @records.each do |record| %>
|
|
1034
|
+
<tr class="row-<%= record.status %>">
|
|
1035
|
+
<td><%= record.line_number %></td>
|
|
1036
|
+
<td><%= status_icon(record.status) %></td>
|
|
1037
|
+
<% @target._columns.each do |col| %>
|
|
1038
|
+
<td><%= record.data[col.name] %></td>
|
|
1039
|
+
<% end %>
|
|
1040
|
+
<td class="errors">
|
|
1041
|
+
<%= record.errors_list.map(&:message).join(', ') %>
|
|
1042
|
+
</td>
|
|
1043
|
+
</tr>
|
|
1044
|
+
<% end %>
|
|
1045
|
+
</tbody>
|
|
1046
|
+
</table>
|
|
1047
|
+
|
|
1048
|
+
<%# === ACTIONS === %>
|
|
1049
|
+
<div class="actions">
|
|
1050
|
+
<%= link_to "Cancel", cancel_import_path(@import),
|
|
1051
|
+
method: :post, class: "btn btn-outline-secondary" %>
|
|
1052
|
+
<%= link_to "Import #{@import.importable_records.size} records",
|
|
1053
|
+
confirm_import_path(@import),
|
|
1054
|
+
method: :post, class: "btn btn-primary",
|
|
1055
|
+
data: { confirm: "Import #{@import.importable_records.size} records?" } %>
|
|
1056
|
+
</div>
|
|
1057
|
+
<% end %>
|
|
1058
|
+
|
|
1059
|
+
<%# === RESULTS (visible apres import) === %>
|
|
1060
|
+
<% if @import.completed? %>
|
|
1061
|
+
<div class="results">
|
|
1062
|
+
<p>Created: <%= @import.report.imported_count %></p>
|
|
1063
|
+
<p>Errors: <%= @import.report.errored_count %></p>
|
|
1064
|
+
</div>
|
|
1065
|
+
<% end %>
|
|
1066
|
+
|
|
1067
|
+
<%# === FAILURE === %>
|
|
1068
|
+
<% if @import.failed? %>
|
|
1069
|
+
<div class="alert alert-danger">
|
|
1070
|
+
<% @import.report&.error_reports&.each do |err| %>
|
|
1071
|
+
<p><%= err.message %></p>
|
|
1072
|
+
<% end %>
|
|
1073
|
+
</div>
|
|
1074
|
+
<%= link_to "Retry", parse_import_path(@import), method: :post, class: "btn btn-warning" %>
|
|
1075
|
+
<% end %>
|
|
1076
|
+
</div>
|
|
1077
|
+
```
|
|
1078
|
+
|
|
1079
|
+
### Jobs (ActiveJob)
|
|
1080
|
+
|
|
1081
|
+
```ruby
|
|
1082
|
+
# app/jobs/data_porter/parse_job.rb
|
|
1083
|
+
module DataPorter
|
|
1084
|
+
class ParseJob < ActiveJob::Base
|
|
1085
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
1086
|
+
|
|
1087
|
+
def perform(import_id)
|
|
1088
|
+
data_import = DataPorter::DataImport.find(import_id)
|
|
1089
|
+
DataPorter::Orchestrator.new(data_import).parse!
|
|
1090
|
+
end
|
|
1091
|
+
end
|
|
1092
|
+
end
|
|
1093
|
+
```
|
|
1094
|
+
|
|
1095
|
+
```ruby
|
|
1096
|
+
# app/jobs/data_porter/import_job.rb
|
|
1097
|
+
module DataPorter
|
|
1098
|
+
class ImportJob < ActiveJob::Base
|
|
1099
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
1100
|
+
|
|
1101
|
+
def perform(import_id)
|
|
1102
|
+
data_import = DataPorter::DataImport.find(import_id)
|
|
1103
|
+
DataPorter::Orchestrator.new(data_import).import!
|
|
1104
|
+
end
|
|
1105
|
+
end
|
|
1106
|
+
end
|
|
1107
|
+
```
|
|
1108
|
+
|
|
1109
|
+
### ActionCable & Progress
|
|
1110
|
+
|
|
1111
|
+
```ruby
|
|
1112
|
+
# lib/data_porter/broadcaster.rb
|
|
1113
|
+
module DataPorter
|
|
1114
|
+
class Broadcaster
|
|
1115
|
+
def initialize(import_id)
|
|
1116
|
+
@channel = "#{DataPorter.configuration.cable_channel_prefix}/imports/#{import_id}"
|
|
1117
|
+
end
|
|
1118
|
+
|
|
1119
|
+
def progress(current, total)
|
|
1120
|
+
percentage = ((current.to_f / total) * 100).round
|
|
1121
|
+
broadcast(status: :processing, percentage: percentage, current: current, total: total)
|
|
1122
|
+
end
|
|
1123
|
+
|
|
1124
|
+
def success
|
|
1125
|
+
broadcast(status: :success)
|
|
1126
|
+
end
|
|
1127
|
+
|
|
1128
|
+
def failure(message)
|
|
1129
|
+
broadcast(status: :failure, error: message)
|
|
1130
|
+
end
|
|
1131
|
+
|
|
1132
|
+
private
|
|
1133
|
+
|
|
1134
|
+
def broadcast(message)
|
|
1135
|
+
ActionCable.server.broadcast(@channel, message)
|
|
1136
|
+
end
|
|
1137
|
+
end
|
|
1138
|
+
end
|
|
1139
|
+
```
|
|
1140
|
+
|
|
1141
|
+
```javascript
|
|
1142
|
+
// app/javascript/data_porter/progress_controller.js
|
|
1143
|
+
import { Controller } from "@hotwired/stimulus";
|
|
1144
|
+
import { createConsumer } from "@rails/actioncable";
|
|
1145
|
+
|
|
1146
|
+
export default class extends Controller {
|
|
1147
|
+
static targets = ["bar", "text"];
|
|
1148
|
+
static values = { id: Number };
|
|
1149
|
+
|
|
1150
|
+
connect() {
|
|
1151
|
+
this.subscription = createConsumer().subscriptions.create(
|
|
1152
|
+
{ channel: "DataPorter::ImportChannel", id: this.idValue },
|
|
1153
|
+
{
|
|
1154
|
+
received: (data) => {
|
|
1155
|
+
if (data.status === "processing") {
|
|
1156
|
+
this.updateProgress(data.percentage);
|
|
1157
|
+
} else {
|
|
1158
|
+
window.location.reload();
|
|
1159
|
+
}
|
|
1160
|
+
},
|
|
1161
|
+
},
|
|
1162
|
+
);
|
|
1163
|
+
}
|
|
1164
|
+
|
|
1165
|
+
updateProgress(percentage) {
|
|
1166
|
+
if (this.hasBarTarget) {
|
|
1167
|
+
this.barTarget.style.width = `${percentage}%`;
|
|
1168
|
+
this.textTarget.textContent = `${percentage}%`;
|
|
1169
|
+
}
|
|
1170
|
+
}
|
|
1171
|
+
|
|
1172
|
+
disconnect() {
|
|
1173
|
+
this.subscription?.unsubscribe();
|
|
1174
|
+
}
|
|
1175
|
+
}
|
|
1176
|
+
```
|
|
1177
|
+
|
|
1178
|
+
---
|
|
1179
|
+
|
|
1180
|
+
## Multi-tenancy
|
|
1181
|
+
|
|
1182
|
+
La gem est agnostique du concept de tenant. Le scoping se fait via le `context_builder` :
|
|
1183
|
+
|
|
1184
|
+
```ruby
|
|
1185
|
+
# config/initializers/data_porter.rb
|
|
1186
|
+
DataPorter.configure do |config|
|
|
1187
|
+
config.context_builder = ->(controller) {
|
|
1188
|
+
OpenStruct.new(
|
|
1189
|
+
hotel: controller.current_user.hotel,
|
|
1190
|
+
user: controller.current_user
|
|
1191
|
+
)
|
|
1192
|
+
}
|
|
1193
|
+
end
|
|
1194
|
+
```
|
|
1195
|
+
|
|
1196
|
+
Le `context` est passe a chaque appel `persist` :
|
|
1197
|
+
|
|
1198
|
+
```ruby
|
|
1199
|
+
def persist(record, context:)
|
|
1200
|
+
Guest.create!(hotel: context.hotel, **record.attributes)
|
|
1201
|
+
end
|
|
1202
|
+
```
|
|
1203
|
+
|
|
1204
|
+
Pour filtrer les imports affiches par tenant, override le controller :
|
|
1205
|
+
|
|
1206
|
+
```ruby
|
|
1207
|
+
# Dans l'app hote
|
|
1208
|
+
DataPorter::ImportsController.class_eval do
|
|
1209
|
+
def index
|
|
1210
|
+
@imports = DataPorter::DataImport
|
|
1211
|
+
.joins(:user)
|
|
1212
|
+
.where(users: { hotel_id: current_user.hotel_id })
|
|
1213
|
+
.order(created_at: :desc)
|
|
1214
|
+
.page(params[:page])
|
|
1215
|
+
end
|
|
1216
|
+
end
|
|
1217
|
+
```
|
|
1218
|
+
|
|
1219
|
+
Ou via le `parent_controller` qui expose un `scope_imports` :
|
|
1220
|
+
|
|
1221
|
+
```ruby
|
|
1222
|
+
DataPorter.configure do |config|
|
|
1223
|
+
config.scope = ->(relation, controller) {
|
|
1224
|
+
relation.where(user: controller.current_user)
|
|
1225
|
+
}
|
|
1226
|
+
end
|
|
1227
|
+
```
|
|
1228
|
+
|
|
1229
|
+
---
|
|
1230
|
+
|
|
1231
|
+
## Generators
|
|
1232
|
+
|
|
1233
|
+
### `data_porter:install`
|
|
1234
|
+
|
|
1235
|
+
```bash
|
|
1236
|
+
rails generate data_porter:install
|
|
1237
|
+
```
|
|
1238
|
+
|
|
1239
|
+
Cree :
|
|
1240
|
+
|
|
1241
|
+
- `db/migrate/xxx_create_data_porter_imports.rb`
|
|
1242
|
+
- `config/initializers/data_porter.rb`
|
|
1243
|
+
- `app/importers/` (dossier vide)
|
|
1244
|
+
- Ajoute `mount DataPorter::Engine, at: '/imports'` dans `config/routes.rb`
|
|
1245
|
+
|
|
1246
|
+
### `data_porter:target NAME [columns...]`
|
|
1247
|
+
|
|
1248
|
+
```bash
|
|
1249
|
+
rails generate data_porter:target guests first_name:string:required last_name:string:required email:email phone:phone
|
|
1250
|
+
```
|
|
1251
|
+
|
|
1252
|
+
Cree :
|
|
1253
|
+
|
|
1254
|
+
- `app/importers/guests_target.rb` avec le squelette pre-rempli
|
|
1255
|
+
|
|
1256
|
+
---
|
|
1257
|
+
|
|
1258
|
+
## Override & Extension
|
|
1259
|
+
|
|
1260
|
+
### Vues
|
|
1261
|
+
|
|
1262
|
+
Les vues de l'engine sont overridables dans l'app hote :
|
|
1263
|
+
|
|
1264
|
+
```
|
|
1265
|
+
app/views/data_porter/imports/index.html.erb -> override la liste
|
|
1266
|
+
app/views/data_porter/imports/show.html.erb -> override le preview
|
|
1267
|
+
app/views/data_porter/imports/new.html.erb -> override le formulaire
|
|
1268
|
+
```
|
|
1269
|
+
|
|
1270
|
+
### CSS
|
|
1271
|
+
|
|
1272
|
+
La gem fournit des classes CSS minimales. Override via :
|
|
1273
|
+
|
|
1274
|
+
```css
|
|
1275
|
+
/* app/assets/stylesheets/data_porter_overrides.css */
|
|
1276
|
+
.data-porter .row-complete {
|
|
1277
|
+
background: #f0fff0;
|
|
1278
|
+
}
|
|
1279
|
+
.data-porter .row-partial {
|
|
1280
|
+
background: #fffdf0;
|
|
1281
|
+
}
|
|
1282
|
+
.data-porter .row-missing {
|
|
1283
|
+
background: #fff0f0;
|
|
1284
|
+
}
|
|
1285
|
+
```
|
|
1286
|
+
|
|
1287
|
+
### Controller
|
|
1288
|
+
|
|
1289
|
+
Heritage via `parent_controller` pour l'auth et le layout.
|
|
1290
|
+
Override individuel via `class_eval` ou monkey-patching classique.
|
|
1291
|
+
|
|
1292
|
+
### Custom Source
|
|
1293
|
+
|
|
1294
|
+
```ruby
|
|
1295
|
+
# lib/data_porter/sources/google_sheets.rb
|
|
1296
|
+
module DataPorter
|
|
1297
|
+
module Sources
|
|
1298
|
+
class GoogleSheets < Base
|
|
1299
|
+
def fetch
|
|
1300
|
+
spreadsheet_id = @data_import.config['spreadsheet_id']
|
|
1301
|
+
# ... Google Sheets API logic ...
|
|
1302
|
+
rows.map { |row| map_row(row) }
|
|
1303
|
+
end
|
|
1304
|
+
end
|
|
1305
|
+
end
|
|
1306
|
+
end
|
|
1307
|
+
|
|
1308
|
+
# config/initializers/data_porter.rb
|
|
1309
|
+
DataPorter.configure do |config|
|
|
1310
|
+
config.enabled_sources = [:csv, :json, :api, :google_sheets]
|
|
1311
|
+
end
|
|
1312
|
+
```
|
|
1313
|
+
|
|
1314
|
+
---
|
|
1315
|
+
|
|
1316
|
+
## Comparaison avec maintenance_tasks
|
|
1317
|
+
|
|
1318
|
+
| Aspect | maintenance_tasks | DataPorter |
|
|
1319
|
+
| ------------------- | ----------------------------- | ----------------------------------------------------- |
|
|
1320
|
+
| Purpose | One-off maintenance scripts | Data import workflow |
|
|
1321
|
+
| Definition | `Task` (collection + process) | `Target` (columns + persist) |
|
|
1322
|
+
| Data source | CSV or ActiveRecord | CSV, JSON, API (extensible) |
|
|
1323
|
+
| Preview | Non | Oui (etat `previewing`) |
|
|
1324
|
+
| UI auto-generee | Form params | Tableau colonnes dynamiques |
|
|
1325
|
+
| Auto-discovery | `Task.descendants` | `Target.descendants` |
|
|
1326
|
+
| Validation visuelle | Non | Oui (complete/partial/missing) |
|
|
1327
|
+
| Progress realtime | Non | Oui (ActionCable) |
|
|
1328
|
+
| Multi-step | Non (fire & forget) | Oui (parse -> preview -> import) |
|
|
1329
|
+
| Parent controller | Configurable | Configurable |
|
|
1330
|
+
| ActiveJob | Oui | Oui |
|
|
1331
|
+
| State machine | Run/Pause/Cancel | Pending/Parsing/Previewing/Importing/Completed/Failed |
|
|
1332
|
+
| Throttling | Oui | Non (pas necessaire) |
|
|
1333
|
+
| Gem dependencies | Minimal | store_model, actioncable |
|
|
1334
|
+
|
|
1335
|
+
---
|
|
1336
|
+
|
|
1337
|
+
## Structure fichiers de la gem
|
|
1338
|
+
|
|
1339
|
+
```
|
|
1340
|
+
data_porter/
|
|
1341
|
+
app/
|
|
1342
|
+
channels/
|
|
1343
|
+
data_porter/
|
|
1344
|
+
import_channel.rb
|
|
1345
|
+
controllers/
|
|
1346
|
+
data_porter/
|
|
1347
|
+
imports_controller.rb
|
|
1348
|
+
jobs/
|
|
1349
|
+
data_porter/
|
|
1350
|
+
parse_job.rb
|
|
1351
|
+
import_job.rb
|
|
1352
|
+
models/
|
|
1353
|
+
data_porter/
|
|
1354
|
+
data_import.rb
|
|
1355
|
+
views/
|
|
1356
|
+
data_porter/
|
|
1357
|
+
imports/
|
|
1358
|
+
index.html.erb
|
|
1359
|
+
new.html.erb
|
|
1360
|
+
show.html.erb
|
|
1361
|
+
_form.html.erb
|
|
1362
|
+
_preview_table.html.erb
|
|
1363
|
+
_progress.html.erb
|
|
1364
|
+
_summary.html.erb
|
|
1365
|
+
assets/
|
|
1366
|
+
stylesheets/
|
|
1367
|
+
data_porter/
|
|
1368
|
+
application.css
|
|
1369
|
+
javascript/
|
|
1370
|
+
data_porter/
|
|
1371
|
+
progress_controller.js
|
|
1372
|
+
config/
|
|
1373
|
+
routes.rb
|
|
1374
|
+
db/
|
|
1375
|
+
migrate/
|
|
1376
|
+
create_data_porter_imports.rb
|
|
1377
|
+
lib/
|
|
1378
|
+
data_porter.rb
|
|
1379
|
+
data_porter/
|
|
1380
|
+
configuration.rb
|
|
1381
|
+
engine.rb
|
|
1382
|
+
orchestrator.rb
|
|
1383
|
+
registry.rb
|
|
1384
|
+
broadcaster.rb
|
|
1385
|
+
target.rb
|
|
1386
|
+
type_validator.rb
|
|
1387
|
+
version.rb
|
|
1388
|
+
sources/
|
|
1389
|
+
base.rb
|
|
1390
|
+
csv.rb
|
|
1391
|
+
json.rb
|
|
1392
|
+
api.rb
|
|
1393
|
+
store_models/
|
|
1394
|
+
import_record.rb
|
|
1395
|
+
error.rb
|
|
1396
|
+
report.rb
|
|
1397
|
+
dsl/
|
|
1398
|
+
columns_dsl.rb
|
|
1399
|
+
csv_mapping_dsl.rb
|
|
1400
|
+
api_config_dsl.rb
|
|
1401
|
+
params_dsl.rb
|
|
1402
|
+
generators/
|
|
1403
|
+
data_porter/
|
|
1404
|
+
install_generator.rb
|
|
1405
|
+
target_generator.rb
|
|
1406
|
+
templates/
|
|
1407
|
+
initializer.rb.tt
|
|
1408
|
+
target.rb.tt
|
|
1409
|
+
spec/
|
|
1410
|
+
...
|
|
1411
|
+
data_porter.gemspec
|
|
1412
|
+
Gemfile
|
|
1413
|
+
README.md
|
|
1414
|
+
LICENSE
|
|
1415
|
+
```
|
|
1416
|
+
|
|
1417
|
+
---
|
|
1418
|
+
|
|
1419
|
+
## Dry Run (v0.2)
|
|
1420
|
+
|
|
1421
|
+
Le dry-run permet de simuler l'import sans rien persister. Il comble le gap entre le preview
|
|
1422
|
+
(validation colonnes/types) et l'import reel (validation base de donnees).
|
|
1423
|
+
|
|
1424
|
+
### Pourquoi c'est essentiel
|
|
1425
|
+
|
|
1426
|
+
Le preview valide au niveau **colonnes** (types, required, inclusion). Le dry-run valide au
|
|
1427
|
+
niveau **base de donnees** :
|
|
1428
|
+
|
|
1429
|
+
| Couche de validation | Preview | Dry Run |
|
|
1430
|
+
| ----------------------------------- | ------- | ------- |
|
|
1431
|
+
| Champs requis | oui | oui |
|
|
1432
|
+
| Types (email, date...) | oui | oui |
|
|
1433
|
+
| Contraintes DB (unique index, FK) | **non** | **oui** |
|
|
1434
|
+
| Validations ActiveRecord du model | **non** | **oui** |
|
|
1435
|
+
| Callbacks `before_create` | **non** | **oui** |
|
|
1436
|
+
| Interactions entre records du batch | **non** | **oui** |
|
|
1437
|
+
|
|
1438
|
+
### Principe
|
|
1439
|
+
|
|
1440
|
+
Envelopper l'import dans une transaction ActiveRecord et la rollback systematiquement.
|
|
1441
|
+
Les erreurs DB sont capturees et ajoutees aux records, le tableau preview se met a jour.
|
|
1442
|
+
|
|
1443
|
+
### Nouveau status
|
|
1444
|
+
|
|
1445
|
+
```
|
|
1446
|
+
pending -> parsing -> previewing -> dry_running -> previewing -> importing -> completed
|
|
1447
|
+
^ |
|
|
1448
|
+
+----------------------------+
|
|
1449
|
+
(retour au preview avec erreurs DB enrichies)
|
|
1450
|
+
```
|
|
1451
|
+
|
|
1452
|
+
`dry_running` est un status transitoire. Apres le dry-run, l'import revient en `previewing`
|
|
1453
|
+
avec les records enrichis des erreurs DB.
|
|
1454
|
+
|
|
1455
|
+
### DSL
|
|
1456
|
+
|
|
1457
|
+
```ruby
|
|
1458
|
+
class GuestsTarget < DataPorter::Target
|
|
1459
|
+
label "Guests"
|
|
1460
|
+
model Guest
|
|
1461
|
+
dry_run_enabled
|
|
1462
|
+
|
|
1463
|
+
# ...
|
|
1464
|
+
end
|
|
1465
|
+
```
|
|
1466
|
+
|
|
1467
|
+
`dry_run_enabled` active le bouton "Test Import" dans l'UI pour ce target.
|
|
1468
|
+
Par defaut, le dry-run est desactive (pas tous les targets en ont besoin).
|
|
1469
|
+
|
|
1470
|
+
### Orchestrator
|
|
1471
|
+
|
|
1472
|
+
```ruby
|
|
1473
|
+
# lib/data_porter/orchestrator.rb
|
|
1474
|
+
module DataPorter
|
|
1475
|
+
class Orchestrator
|
|
1476
|
+
def dry_run!
|
|
1477
|
+
@data_import.dry_running!
|
|
1478
|
+
context = build_context
|
|
1479
|
+
importable = @data_import.importable_records
|
|
1480
|
+
|
|
1481
|
+
ActiveRecord::Base.transaction do
|
|
1482
|
+
importable.each_with_index do |record, index|
|
|
1483
|
+
instance = @target.persist(record, context: context)
|
|
1484
|
+
record.target_id = instance.id
|
|
1485
|
+
record.status = 'complete'
|
|
1486
|
+
record.dry_run_passed = true
|
|
1487
|
+
broadcast_progress(index + 1, importable.size)
|
|
1488
|
+
rescue ActiveRecord::RecordInvalid => e
|
|
1489
|
+
record.status = 'partial'
|
|
1490
|
+
e.record.errors.full_messages.each { |msg| record.add_error(msg) }
|
|
1491
|
+
rescue ActiveRecord::RecordNotUnique => e
|
|
1492
|
+
record.status = 'partial'
|
|
1493
|
+
record.add_error(extract_unique_violation_message(e))
|
|
1494
|
+
rescue StandardError => e
|
|
1495
|
+
record.status = 'error'
|
|
1496
|
+
record.add_error(e.message)
|
|
1497
|
+
end
|
|
1498
|
+
|
|
1499
|
+
raise ActiveRecord::Rollback
|
|
1500
|
+
end
|
|
1501
|
+
|
|
1502
|
+
reset_target_ids!(importable)
|
|
1503
|
+
@data_import.update!(status: :previewing)
|
|
1504
|
+
build_report
|
|
1505
|
+
broadcast_success
|
|
1506
|
+
rescue StandardError => e
|
|
1507
|
+
handle_failure(e)
|
|
1508
|
+
end
|
|
1509
|
+
|
|
1510
|
+
private
|
|
1511
|
+
|
|
1512
|
+
def reset_target_ids!(records)
|
|
1513
|
+
records.each { |r| r.target_id = nil }
|
|
1514
|
+
end
|
|
1515
|
+
|
|
1516
|
+
def extract_unique_violation_message(error)
|
|
1517
|
+
match = error.message.match(/Key \((.+?)\)=\((.+?)\) already exists/)
|
|
1518
|
+
if match
|
|
1519
|
+
"#{match[1]}: '#{match[2]}' already exists"
|
|
1520
|
+
else
|
|
1521
|
+
"Duplicate record detected"
|
|
1522
|
+
end
|
|
1523
|
+
end
|
|
1524
|
+
end
|
|
1525
|
+
end
|
|
1526
|
+
```
|
|
1527
|
+
|
|
1528
|
+
### Job
|
|
1529
|
+
|
|
1530
|
+
```ruby
|
|
1531
|
+
# app/jobs/data_porter/dry_run_job.rb
|
|
1532
|
+
module DataPorter
|
|
1533
|
+
class DryRunJob < ActiveJob::Base
|
|
1534
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
1535
|
+
|
|
1536
|
+
def perform(import_id)
|
|
1537
|
+
data_import = DataPorter::DataImport.find(import_id)
|
|
1538
|
+
DataPorter::Orchestrator.new(data_import).dry_run!
|
|
1539
|
+
end
|
|
1540
|
+
end
|
|
1541
|
+
end
|
|
1542
|
+
```
|
|
1543
|
+
|
|
1544
|
+
### Controller
|
|
1545
|
+
|
|
1546
|
+
```ruby
|
|
1547
|
+
# Ajout dans ImportsController
|
|
1548
|
+
def dry_run
|
|
1549
|
+
DataPorter::DryRunJob.perform_later(@import.id)
|
|
1550
|
+
redirect_to import_path(@import)
|
|
1551
|
+
end
|
|
1552
|
+
```
|
|
1553
|
+
|
|
1554
|
+
### Migration (ajout au modele DataImport)
|
|
1555
|
+
|
|
1556
|
+
```ruby
|
|
1557
|
+
# Ajouter dry_running au enum status
|
|
1558
|
+
enum :status, {
|
|
1559
|
+
pending: 0,
|
|
1560
|
+
parsing: 1,
|
|
1561
|
+
previewing: 2,
|
|
1562
|
+
importing: 3,
|
|
1563
|
+
completed: 4,
|
|
1564
|
+
failed: 5,
|
|
1565
|
+
dry_running: 6
|
|
1566
|
+
}
|
|
1567
|
+
```
|
|
1568
|
+
|
|
1569
|
+
### StoreModel ImportRecord (ajout)
|
|
1570
|
+
|
|
1571
|
+
```ruby
|
|
1572
|
+
attribute :dry_run_passed, :boolean, default: false
|
|
1573
|
+
```
|
|
1574
|
+
|
|
1575
|
+
### UI : boutons preview
|
|
1576
|
+
|
|
1577
|
+
Deux boutons sur la page preview quand le target a `dry_run_enabled` :
|
|
1578
|
+
|
|
1579
|
+
```erb
|
|
1580
|
+
<% if @import.previewable? %>
|
|
1581
|
+
<div class="actions">
|
|
1582
|
+
<%= link_to "Cancel", cancel_import_path(@import),
|
|
1583
|
+
method: :post, class: "btn btn-outline-secondary" %>
|
|
1584
|
+
|
|
1585
|
+
<% if @target.class._dry_run_enabled %>
|
|
1586
|
+
<%= link_to "Test Import (dry-run)",
|
|
1587
|
+
dry_run_import_path(@import),
|
|
1588
|
+
method: :post, class: "btn btn-outline-primary" %>
|
|
1589
|
+
<% end %>
|
|
1590
|
+
|
|
1591
|
+
<%= link_to "Import #{@import.importable_records.size} records",
|
|
1592
|
+
confirm_import_path(@import),
|
|
1593
|
+
method: :post, class: "btn btn-primary",
|
|
1594
|
+
data: { confirm: "Import #{@import.importable_records.size} records?" } %>
|
|
1595
|
+
</div>
|
|
1596
|
+
<% end %>
|
|
1597
|
+
```
|
|
1598
|
+
|
|
1599
|
+
Apres le dry-run, les records enrichis affichent un indicateur supplementaire :
|
|
1600
|
+
|
|
1601
|
+
```erb
|
|
1602
|
+
<% if record.dry_run_passed %>
|
|
1603
|
+
<span class="badge badge-success" title="DB validation passed">DB OK</span>
|
|
1604
|
+
<% end %>
|
|
1605
|
+
```
|
|
1606
|
+
|
|
1607
|
+
### Workflow utilisateur
|
|
1608
|
+
|
|
1609
|
+
```
|
|
1610
|
+
1. Upload CSV
|
|
1611
|
+
2. Preview : voit 142 records "complete", 3 "partial", 5 "missing"
|
|
1612
|
+
3. Clique "Test Import (dry-run)"
|
|
1613
|
+
4. Progress bar (parsing async)
|
|
1614
|
+
5. Retour au preview : 138 records "complete + DB OK", 4 nouveaux "partial"
|
|
1615
|
+
(ex: "email: 'john@doe.com' already exists", "housing_id: 999 doesn't exist")
|
|
1616
|
+
6. Corrige son CSV et relance, OU confirme l'import des 138 valides
|
|
1617
|
+
```
|
|
1618
|
+
|
|
1619
|
+
---
|
|
1620
|
+
|
|
1621
|
+
## Rollback (v0.3 / v0.4)
|
|
1622
|
+
|
|
1623
|
+
Le rollback permet d'annuler un import termine en supprimant les records crees.
|
|
1624
|
+
|
|
1625
|
+
### Principe
|
|
1626
|
+
|
|
1627
|
+
Chaque `ImportRecord` stocke le `target_id` du record cree en base apres l'import.
|
|
1628
|
+
Le rollback utilise ces IDs pour identifier et supprimer les records.
|
|
1629
|
+
|
|
1630
|
+
### Niveaux de rollback
|
|
1631
|
+
|
|
1632
|
+
#### v0.3 — Rollback simple
|
|
1633
|
+
|
|
1634
|
+
Suppression directe des records crees, avec un time window configurable.
|
|
1635
|
+
|
|
1636
|
+
#### v0.4 — Rollback garde
|
|
1637
|
+
|
|
1638
|
+
Verifications de securite avant suppression : records modifies depuis l'import,
|
|
1639
|
+
dependances creees, side-effects irreversibles.
|
|
1640
|
+
|
|
1641
|
+
### DSL
|
|
1642
|
+
|
|
1643
|
+
```ruby
|
|
1644
|
+
class GuestsTarget < DataPorter::Target
|
|
1645
|
+
label "Guests"
|
|
1646
|
+
model Guest
|
|
1647
|
+
rollback_enabled
|
|
1648
|
+
rollback_window 2.hours
|
|
1649
|
+
rollback_strategy :destroy
|
|
1650
|
+
|
|
1651
|
+
def rollbackable?(record, context:)
|
|
1652
|
+
instance = Guest.find_by(id: record.target_id)
|
|
1653
|
+
return false if instance.nil?
|
|
1654
|
+
return false if instance.updated_at > context.import_completed_at
|
|
1655
|
+
true
|
|
1656
|
+
end
|
|
1657
|
+
end
|
|
1658
|
+
```
|
|
1659
|
+
|
|
1660
|
+
| Option DSL | Type | Description |
|
|
1661
|
+
| ------------------- | -------- | ----------------------------------------------------------------- |
|
|
1662
|
+
| `rollback_enabled` | flag | Active le rollback pour ce target |
|
|
1663
|
+
| `rollback_window` | Duration | Duree pendant laquelle le rollback est possible (default: 1.hour) |
|
|
1664
|
+
| `rollback_strategy` | Symbol | `:destroy`, `:delete`, `:soft_delete`, `:flag` |
|
|
1665
|
+
| `rollbackable?` | Hook | Verification custom par record avant rollback |
|
|
1666
|
+
|
|
1667
|
+
### Strategies de rollback
|
|
1668
|
+
|
|
1669
|
+
```ruby
|
|
1670
|
+
# lib/data_porter/rollback_strategies.rb
|
|
1671
|
+
module DataPorter
|
|
1672
|
+
module RollbackStrategies
|
|
1673
|
+
STRATEGIES = {
|
|
1674
|
+
destroy: ->(model, ids) {
|
|
1675
|
+
model.where(id: ids).destroy_all
|
|
1676
|
+
},
|
|
1677
|
+
|
|
1678
|
+
delete: ->(model, ids) {
|
|
1679
|
+
model.where(id: ids).delete_all
|
|
1680
|
+
},
|
|
1681
|
+
|
|
1682
|
+
soft_delete: ->(model, ids) {
|
|
1683
|
+
model.where(id: ids).update_all(discarded_at: Time.current)
|
|
1684
|
+
},
|
|
1685
|
+
|
|
1686
|
+
flag: ->(model, ids) {
|
|
1687
|
+
model.where(id: ids).update_all(import_rolled_back: true)
|
|
1688
|
+
}
|
|
1689
|
+
}.freeze
|
|
1690
|
+
|
|
1691
|
+
def self.execute(strategy, model, ids)
|
|
1692
|
+
handler = STRATEGIES[strategy] || raise(UnknownStrategy, "Unknown strategy: #{strategy}")
|
|
1693
|
+
handler.call(model, ids)
|
|
1694
|
+
end
|
|
1695
|
+
end
|
|
1696
|
+
end
|
|
1697
|
+
```
|
|
1698
|
+
|
|
1699
|
+
| Strategie | Effet | Callbacks | Reversible | Quand l'utiliser |
|
|
1700
|
+
| -------------- | ------------------------ | ------------------------------- | ---------- | ------------------------------------------------- |
|
|
1701
|
+
| `:destroy` | `destroy_all` | Oui (after_destroy, dependents) | Non | Default, quand les callbacks sont importants |
|
|
1702
|
+
| `:delete` | `delete_all` | Non (SQL direct) | Non | Performance, pas de side-effects |
|
|
1703
|
+
| `:soft_delete` | Set `discarded_at` | Non | Oui | Avec discard gem, quand on veut pouvoir restaurer |
|
|
1704
|
+
| `:flag` | Set `import_rolled_back` | Non | Oui | Quand on veut garder les records mais les marquer |
|
|
1705
|
+
|
|
1706
|
+
### Orchestrator
|
|
1707
|
+
|
|
1708
|
+
```ruby
|
|
1709
|
+
# lib/data_porter/orchestrator.rb
|
|
1710
|
+
module DataPorter
|
|
1711
|
+
class Orchestrator
|
|
1712
|
+
def rollback!
|
|
1713
|
+
validate_rollback_allowed!
|
|
1714
|
+
@data_import.rolling_back!
|
|
1715
|
+
|
|
1716
|
+
target_class = @target.class
|
|
1717
|
+
model = target_class._model
|
|
1718
|
+
strategy = target_class._rollback_strategy || :destroy
|
|
1719
|
+
context = build_rollback_context
|
|
1720
|
+
|
|
1721
|
+
imported_records = @data_import.records.select { |r| r.status == 'imported' && r.target_id.present? }
|
|
1722
|
+
rollbackable, skipped = partition_rollbackable(imported_records, context)
|
|
1723
|
+
|
|
1724
|
+
rollbackable_ids = rollbackable.map(&:target_id)
|
|
1725
|
+
|
|
1726
|
+
ActiveRecord::Base.transaction do
|
|
1727
|
+
DataPorter::RollbackStrategies.execute(strategy, model, rollbackable_ids)
|
|
1728
|
+
end
|
|
1729
|
+
|
|
1730
|
+
rollbackable.each { |r| r.status = 'rolled_back' }
|
|
1731
|
+
skipped.each { |r| r.status = 'rollback_skipped' }
|
|
1732
|
+
|
|
1733
|
+
@data_import.update!(status: :rolled_back)
|
|
1734
|
+
build_rollback_report(rollbackable, skipped)
|
|
1735
|
+
broadcast_success
|
|
1736
|
+
rescue StandardError => e
|
|
1737
|
+
handle_failure(e)
|
|
1738
|
+
end
|
|
1739
|
+
|
|
1740
|
+
private
|
|
1741
|
+
|
|
1742
|
+
def validate_rollback_allowed!
|
|
1743
|
+
target_class = @target.class
|
|
1744
|
+
raise RollbackNotEnabled, "Rollback not enabled for #{target_class._label}" unless target_class._rollback_enabled
|
|
1745
|
+
|
|
1746
|
+
window = target_class._rollback_window || 1.hour
|
|
1747
|
+
if @data_import.completed_at && @data_import.completed_at < window.ago
|
|
1748
|
+
raise RollbackWindowExpired,
|
|
1749
|
+
"Rollback window expired (#{window.inspect} after completion)"
|
|
1750
|
+
end
|
|
1751
|
+
end
|
|
1752
|
+
|
|
1753
|
+
def partition_rollbackable(records, context)
|
|
1754
|
+
rollbackable = []
|
|
1755
|
+
skipped = []
|
|
1756
|
+
|
|
1757
|
+
records.each do |record|
|
|
1758
|
+
if @target.rollbackable?(record, context: context)
|
|
1759
|
+
rollbackable << record
|
|
1760
|
+
else
|
|
1761
|
+
skipped << record
|
|
1762
|
+
record.add_warning("Skipped: record modified or deleted since import")
|
|
1763
|
+
end
|
|
1764
|
+
end
|
|
1765
|
+
|
|
1766
|
+
[rollbackable, skipped]
|
|
1767
|
+
end
|
|
1768
|
+
|
|
1769
|
+
def build_rollback_context
|
|
1770
|
+
OpenStruct.new(
|
|
1771
|
+
import_completed_at: @data_import.updated_at,
|
|
1772
|
+
**(@data_import.config || {}).symbolize_keys
|
|
1773
|
+
)
|
|
1774
|
+
end
|
|
1775
|
+
|
|
1776
|
+
def build_rollback_report(rollbackable, skipped)
|
|
1777
|
+
report = @data_import.report || DataPorter::StoreModels::Report.new
|
|
1778
|
+
report.rolled_back_count = rollbackable.size
|
|
1779
|
+
report.rollback_skipped_count = skipped.size
|
|
1780
|
+
@data_import.update!(report: report)
|
|
1781
|
+
end
|
|
1782
|
+
end
|
|
1783
|
+
end
|
|
1784
|
+
```
|
|
1785
|
+
|
|
1786
|
+
### Modele DataImport (ajouts)
|
|
1787
|
+
|
|
1788
|
+
```ruby
|
|
1789
|
+
enum :status, {
|
|
1790
|
+
pending: 0,
|
|
1791
|
+
parsing: 1,
|
|
1792
|
+
previewing: 2,
|
|
1793
|
+
importing: 3,
|
|
1794
|
+
completed: 4,
|
|
1795
|
+
failed: 5,
|
|
1796
|
+
dry_running: 6,
|
|
1797
|
+
rolling_back: 7,
|
|
1798
|
+
rolled_back: 8
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
def rollback_available?
|
|
1802
|
+
return false unless completed?
|
|
1803
|
+
|
|
1804
|
+
target = target_class
|
|
1805
|
+
return false unless target._rollback_enabled
|
|
1806
|
+
|
|
1807
|
+
window = target._rollback_window || 1.hour
|
|
1808
|
+
updated_at > window.ago
|
|
1809
|
+
end
|
|
1810
|
+
|
|
1811
|
+
def rollback_expires_at
|
|
1812
|
+
return nil unless rollback_available?
|
|
1813
|
+
|
|
1814
|
+
window = target_class._rollback_window || 1.hour
|
|
1815
|
+
updated_at + window
|
|
1816
|
+
end
|
|
1817
|
+
```
|
|
1818
|
+
|
|
1819
|
+
### StoreModel ImportRecord (ajouts)
|
|
1820
|
+
|
|
1821
|
+
```ruby
|
|
1822
|
+
STATUSES = %w[
|
|
1823
|
+
pending complete partial missing duplicate
|
|
1824
|
+
imported error
|
|
1825
|
+
rolled_back rollback_skipped
|
|
1826
|
+
].freeze
|
|
1827
|
+
```
|
|
1828
|
+
|
|
1829
|
+
### StoreModel Report (ajouts)
|
|
1830
|
+
|
|
1831
|
+
```ruby
|
|
1832
|
+
attribute :rolled_back_count, :integer, default: 0
|
|
1833
|
+
attribute :rollback_skipped_count, :integer, default: 0
|
|
1834
|
+
```
|
|
1835
|
+
|
|
1836
|
+
### Controller
|
|
1837
|
+
|
|
1838
|
+
```ruby
|
|
1839
|
+
def rollback
|
|
1840
|
+
if @import.rollback_available?
|
|
1841
|
+
DataPorter::RollbackJob.perform_later(@import.id)
|
|
1842
|
+
redirect_to import_path(@import)
|
|
1843
|
+
else
|
|
1844
|
+
redirect_to import_path(@import), alert: "Rollback not available"
|
|
1845
|
+
end
|
|
1846
|
+
end
|
|
1847
|
+
```
|
|
1848
|
+
|
|
1849
|
+
### Job
|
|
1850
|
+
|
|
1851
|
+
```ruby
|
|
1852
|
+
# app/jobs/data_porter/rollback_job.rb
|
|
1853
|
+
module DataPorter
|
|
1854
|
+
class RollbackJob < ActiveJob::Base
|
|
1855
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
1856
|
+
|
|
1857
|
+
def perform(import_id)
|
|
1858
|
+
data_import = DataPorter::DataImport.find(import_id)
|
|
1859
|
+
DataPorter::Orchestrator.new(data_import).rollback!
|
|
1860
|
+
end
|
|
1861
|
+
end
|
|
1862
|
+
end
|
|
1863
|
+
```
|
|
1864
|
+
|
|
1865
|
+
### UI : page completed avec rollback
|
|
1866
|
+
|
|
1867
|
+
```erb
|
|
1868
|
+
<% if @import.completed? %>
|
|
1869
|
+
<div class="results">
|
|
1870
|
+
<p>Created: <%= @import.report.imported_count %></p>
|
|
1871
|
+
<p>Errors: <%= @import.report.errored_count %></p>
|
|
1872
|
+
</div>
|
|
1873
|
+
|
|
1874
|
+
<% if @import.rollback_available? %>
|
|
1875
|
+
<div class="rollback-section">
|
|
1876
|
+
<p>
|
|
1877
|
+
Rollback available for
|
|
1878
|
+
<strong><%= distance_of_time_in_words_to_now(@import.rollback_expires_at) %></strong>
|
|
1879
|
+
</p>
|
|
1880
|
+
|
|
1881
|
+
<% modified_count = @import.records.count { |r|
|
|
1882
|
+
r.status == 'imported' && r.target_id &&
|
|
1883
|
+
@target.class._model.find_by(id: r.target_id)&.updated_at.to_i > @import.updated_at.to_i
|
|
1884
|
+
} %>
|
|
1885
|
+
|
|
1886
|
+
<% if modified_count > 0 %>
|
|
1887
|
+
<div class="alert alert-warning">
|
|
1888
|
+
<strong><%= modified_count %></strong> record(s) modified since import — will be skipped
|
|
1889
|
+
</div>
|
|
1890
|
+
<% end %>
|
|
1891
|
+
|
|
1892
|
+
<%= link_to "Rollback #{@import.records.count { |r| r.status == 'imported' } - modified_count} records",
|
|
1893
|
+
rollback_import_path(@import),
|
|
1894
|
+
method: :post, class: "btn btn-danger",
|
|
1895
|
+
data: { confirm: "This will delete the imported records. Continue?" } %>
|
|
1896
|
+
</div>
|
|
1897
|
+
<% end %>
|
|
1898
|
+
<% end %>
|
|
1899
|
+
|
|
1900
|
+
<% if @import.rolled_back? %>
|
|
1901
|
+
<div class="results">
|
|
1902
|
+
<p>Rolled back: <%= @import.report.rolled_back_count %></p>
|
|
1903
|
+
<p>Skipped: <%= @import.report.rollback_skipped_count %></p>
|
|
1904
|
+
</div>
|
|
1905
|
+
<% end %>
|
|
1906
|
+
```
|
|
1907
|
+
|
|
1908
|
+
### State machine complete (v0.4)
|
|
1909
|
+
|
|
1910
|
+
```
|
|
1911
|
+
+-> dry_running -+
|
|
1912
|
+
| |
|
|
1913
|
+
pending -> parsing -> previewing ---+ +-> previewing
|
|
1914
|
+
^ | |
|
|
1915
|
+
| +----------------------+
|
|
1916
|
+
|
|
|
1917
|
+
+--- (re-upload / retry)
|
|
1918
|
+
|
|
1919
|
+
previewing -> importing -> completed -> rolling_back -> rolled_back
|
|
1920
|
+
|
|
|
1921
|
+
+-> failed
|
|
1922
|
+
|
|
1923
|
+
parsing -> failed
|
|
1924
|
+
importing -> failed
|
|
1925
|
+
rolling_back -> failed
|
|
1926
|
+
```
|
|
1927
|
+
|
|
1928
|
+
### Limites documentees
|
|
1929
|
+
|
|
1930
|
+
Le rollback ne peut PAS annuler :
|
|
1931
|
+
|
|
1932
|
+
- Les emails/notifications envoyees par les callbacks
|
|
1933
|
+
- Les webhooks declenches
|
|
1934
|
+
- Les fichiers uploades via ActiveStorage dans `persist`
|
|
1935
|
+
- Les modifications faites par d'autres utilisateurs sur les records importes
|
|
1936
|
+
- Les records dont les associations ont ete supprimees (FK violations)
|
|
1937
|
+
|
|
1938
|
+
Ces limites doivent etre clairement affichees dans l'UI avant confirmation du rollback.
|
|
1939
|
+
|
|
1940
|
+
### Configuration globale
|
|
1941
|
+
|
|
1942
|
+
```ruby
|
|
1943
|
+
# config/initializers/data_porter.rb
|
|
1944
|
+
DataPorter.configure do |config|
|
|
1945
|
+
config.default_rollback_strategy = :destroy
|
|
1946
|
+
config.default_rollback_window = 1.hour
|
|
1947
|
+
config.rollback_strategies = {
|
|
1948
|
+
# Ajout de strategies custom
|
|
1949
|
+
archive: ->(model, ids) {
|
|
1950
|
+
model.where(id: ids).update_all(archived: true, archived_at: Time.current)
|
|
1951
|
+
}
|
|
1952
|
+
}
|
|
1953
|
+
end
|
|
1954
|
+
```
|
|
1955
|
+
|
|
1956
|
+
### Structure fichiers supplementaires
|
|
1957
|
+
|
|
1958
|
+
```
|
|
1959
|
+
lib/
|
|
1960
|
+
data_porter/
|
|
1961
|
+
rollback_strategies.rb
|
|
1962
|
+
app/
|
|
1963
|
+
jobs/
|
|
1964
|
+
data_porter/
|
|
1965
|
+
dry_run_job.rb
|
|
1966
|
+
rollback_job.rb
|
|
1967
|
+
```
|
|
1968
|
+
|
|
1969
|
+
---
|
|
1970
|
+
|
|
1971
|
+
## Roadmap
|
|
1972
|
+
|
|
1973
|
+
### v0.1 - MVP
|
|
1974
|
+
|
|
1975
|
+
- Modele DataImport + migration
|
|
1976
|
+
- Source CSV
|
|
1977
|
+
- Orchestrator (parse + import)
|
|
1978
|
+
- Preview table auto-generee
|
|
1979
|
+
- DSL Target (columns, csv_mapping, persist)
|
|
1980
|
+
- Jobs ActiveJob
|
|
1981
|
+
- ActionCable progress
|
|
1982
|
+
- Generator install + target
|
|
1983
|
+
- Tests RSpec
|
|
1984
|
+
|
|
1985
|
+
### v0.2 - Dry Run + Sources
|
|
1986
|
+
|
|
1987
|
+
- **Dry-run mode** (transaction + rollback, enrichissement des erreurs DB)
|
|
1988
|
+
- DSL `dry_run_enabled`
|
|
1989
|
+
- Source JSON (file upload + raw text)
|
|
1990
|
+
- Source API (HTTP client + config)
|
|
1991
|
+
- Deduplication visuelle
|
|
1992
|
+
|
|
1993
|
+
### v0.3 - Rollback simple + UX
|
|
1994
|
+
|
|
1995
|
+
- **Rollback simple** (destroy des records crees via target_id)
|
|
1996
|
+
- DSL `rollback_enabled`, `rollback_window`, `rollback_strategy`
|
|
1997
|
+
- Strategies built-in : destroy, delete, soft_delete, flag
|
|
1998
|
+
- Rollback time window avec countdown UI
|
|
1999
|
+
- Filtres/tri dans le preview
|
|
2000
|
+
- Export des erreurs en CSV
|
|
2001
|
+
- Pagination du preview
|
|
2002
|
+
|
|
2003
|
+
### v0.4 - Rollback garde + Avance
|
|
2004
|
+
|
|
2005
|
+
- **Rollback garde** : hook `rollbackable?`, detection records modifies
|
|
2006
|
+
- Strategies de rollback custom (plugin)
|
|
2007
|
+
- Affichage pre-rollback des records modifies/non-rollbackables
|
|
2008
|
+
- Custom sources (plugin system)
|
|
2009
|
+
- Batch persist (bulk insert)
|
|
2010
|
+
- Webhook notifications (fin d'import)
|
|
2011
|
+
- Import scheduling (cron-like pour les sources API)
|
|
2012
|
+
- Row-level accept/reject en preview
|