data_porter 0.5.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -0
- data/README.md +12 -6
- data/app/assets/javascripts/data_porter/import_form_controller.js +126 -0
- data/app/assets/stylesheets/data_porter/table.css +45 -0
- data/app/controllers/data_porter/concerns/import_validation.rb +47 -0
- data/app/controllers/data_porter/concerns/mapping_management.rb +43 -0
- data/app/controllers/data_porter/concerns/record_pagination.rb +19 -0
- data/app/controllers/data_porter/imports_controller.rb +14 -48
- data/app/views/data_porter/imports/index.html.erb +59 -115
- data/app/views/data_porter/imports/new.html.erb +51 -2
- data/app/views/data_porter/imports/show.html.erb +10 -0
- data/app/views/layouts/data_porter/application.html.erb +17 -146
- data/docs/CONFIGURATION.md +28 -6
- data/docs/ROADMAP.md +28 -0
- data/docs/TARGETS.md +54 -3
- data/lib/data_porter/components/shared/pagination.rb +53 -0
- data/lib/data_porter/components.rb +1 -0
- data/lib/data_porter/dsl/param.rb +32 -0
- data/lib/data_porter/engine.rb +4 -0
- data/lib/data_porter/orchestrator/dry_runner.rb +30 -0
- data/lib/data_porter/orchestrator/importer.rb +41 -0
- data/lib/data_porter/orchestrator/record_builder.rb +38 -0
- data/lib/data_porter/orchestrator.rb +16 -83
- data/lib/data_porter/registry.rb +21 -1
- data/lib/data_porter/target.rb +17 -1
- data/lib/data_porter/version.rb +1 -1
- metadata +14 -4
- /data/app/{javascript → assets/javascripts}/data_porter/mapping_controller.js +0 -0
- /data/app/{javascript → assets/javascripts}/data_porter/progress_controller.js +0 -0
- /data/app/{javascript → assets/javascripts}/data_porter/template_form_controller.js +0 -0
|
@@ -1,9 +1,12 @@
|
|
|
1
|
-
<div class="data-porter"
|
|
1
|
+
<div class="data-porter" data-controller="data-porter--import-form"
|
|
2
|
+
data-data-porter--import-form-sources-value="<%= @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json %>"
|
|
3
|
+
data-data-porter--import-form-params-value="<%= @targets.map { |t| [t[:key], t[:params]] }.to_h.to_json %>"
|
|
4
|
+
data-action="keydown@document->data-porter--import-form#closeModal">
|
|
2
5
|
<div class="dp-header">
|
|
3
6
|
<h1 class="dp-title">Imports</h1>
|
|
4
7
|
<div class="dp-header__actions">
|
|
5
8
|
<%= link_to "Mapping Templates", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
|
|
6
|
-
<button type="button" class="dp-btn dp-btn--primary"
|
|
9
|
+
<button type="button" class="dp-btn dp-btn--primary" data-action="data-porter--import-form#openModal">
|
|
7
10
|
New Import
|
|
8
11
|
</button>
|
|
9
12
|
</div>
|
|
@@ -45,127 +48,68 @@
|
|
|
45
48
|
<div class="dp-empty-state">
|
|
46
49
|
<div class="dp-empty-state__icon">📦</div>
|
|
47
50
|
<p class="dp-empty-state__text">No imports yet</p>
|
|
48
|
-
<button type="button" class="dp-btn dp-btn--primary"
|
|
51
|
+
<button type="button" class="dp-btn dp-btn--primary" data-action="data-porter--import-form#openModal">
|
|
49
52
|
Create your first import
|
|
50
53
|
</button>
|
|
51
54
|
</div>
|
|
52
55
|
<% end %>
|
|
53
|
-
</div>
|
|
54
|
-
|
|
55
|
-
<div id="dp-modal" class="dp-modal">
|
|
56
|
-
<div class="dp-modal__backdrop" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')"></div>
|
|
57
|
-
<div class="dp-modal__content">
|
|
58
|
-
<div class="dp-modal__header">
|
|
59
|
-
<h2 class="dp-modal__title">New Import</h2>
|
|
60
|
-
<button type="button" class="dp-modal__close" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">×</button>
|
|
61
|
-
</div>
|
|
62
|
-
|
|
63
|
-
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
|
|
64
|
-
<div class="dp-field">
|
|
65
|
-
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
66
|
-
<%= f.select :target_key,
|
|
67
|
-
@targets.map { |t| [t[:label], t[:key]] },
|
|
68
|
-
{ prompt: "Select a target..." },
|
|
69
|
-
id: "dp-target-select",
|
|
70
|
-
class: "dp-select",
|
|
71
|
-
data: { sources: @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json } %>
|
|
72
|
-
</div>
|
|
73
|
-
|
|
74
|
-
<div class="dp-field">
|
|
75
|
-
<%= f.label :source_type, "Source Type", class: "dp-label" %>
|
|
76
|
-
<%= f.select :source_type,
|
|
77
|
-
DataPorter.configuration.enabled_sources.map { |s| [s.to_s.upcase, s] },
|
|
78
|
-
{ prompt: "Select source type..." },
|
|
79
|
-
id: "dp-source-select",
|
|
80
|
-
class: "dp-select" %>
|
|
81
|
-
</div>
|
|
82
|
-
|
|
83
|
-
<div id="dp-file-field" class="dp-field">
|
|
84
|
-
<%= f.label :file, "File", class: "dp-label" %>
|
|
85
|
-
<label class="dp-dropzone" id="dp-dropzone">
|
|
86
|
-
<input type="file" name="data_import[file]" id="dp-file-input" class="dp-dropzone__input" />
|
|
87
|
-
<div class="dp-dropzone__content">
|
|
88
|
-
<div class="dp-dropzone__icon">📄</div>
|
|
89
|
-
<span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
|
|
90
|
-
<span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
|
|
91
|
-
</div>
|
|
92
|
-
<div class="dp-dropzone__selected" id="dp-file-name" style="display: none;"></div>
|
|
93
|
-
</label>
|
|
94
|
-
</div>
|
|
95
56
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
57
|
+
<div id="dp-modal" class="dp-modal" data-data-porter--import-form-target="modal">
|
|
58
|
+
<div class="dp-modal__backdrop" data-action="click->data-porter--import-form#closeModalClick"></div>
|
|
59
|
+
<div class="dp-modal__content">
|
|
60
|
+
<div class="dp-modal__header">
|
|
61
|
+
<h2 class="dp-modal__title">New Import</h2>
|
|
62
|
+
<button type="button" class="dp-modal__close" data-action="data-porter--import-form#closeModalClick">×</button>
|
|
99
63
|
</div>
|
|
100
|
-
<% end %>
|
|
101
|
-
</div>
|
|
102
|
-
</div>
|
|
103
|
-
|
|
104
|
-
<script>
|
|
105
|
-
(function() {
|
|
106
|
-
var targetSelect = document.getElementById("dp-target-select");
|
|
107
|
-
var sourceSelect = document.getElementById("dp-source-select");
|
|
108
|
-
var fileField = document.getElementById("dp-file-field");
|
|
109
|
-
var fileInput = document.getElementById("dp-file-input");
|
|
110
|
-
var dropzone = document.getElementById("dp-dropzone");
|
|
111
|
-
var fileName = document.getElementById("dp-file-name");
|
|
112
64
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
}
|
|
65
|
+
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
|
|
66
|
+
<div class="dp-field">
|
|
67
|
+
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
68
|
+
<%= f.select :target_key,
|
|
69
|
+
@targets.map { |t| [t[:label], t[:key]] },
|
|
70
|
+
{ prompt: "Select a target..." },
|
|
71
|
+
class: "dp-select",
|
|
72
|
+
data: {
|
|
73
|
+
data_porter__import_form_target: "targetSelect",
|
|
74
|
+
action: "data-porter--import-form#filterSources"
|
|
75
|
+
} %>
|
|
76
|
+
</div>
|
|
126
77
|
|
|
127
|
-
|
|
128
|
-
targetSelect.addEventListener("change", filterSources);
|
|
129
|
-
}
|
|
78
|
+
<div data-data-porter--import-form-target="paramsContainer"></div>
|
|
130
79
|
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
80
|
+
<div class="dp-field">
|
|
81
|
+
<%= f.label :source_type, "Source Type", class: "dp-label" %>
|
|
82
|
+
<%= f.select :source_type,
|
|
83
|
+
DataPorter.configuration.enabled_sources.map { |s| [s.to_s.upcase, s] },
|
|
84
|
+
{ prompt: "Select source type..." },
|
|
85
|
+
class: "dp-select",
|
|
86
|
+
data: {
|
|
87
|
+
data_porter__import_form_target: "sourceSelect",
|
|
88
|
+
action: "data-porter--import-form#toggleFileField"
|
|
89
|
+
} %>
|
|
90
|
+
</div>
|
|
136
91
|
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
92
|
+
<div class="dp-field" data-data-porter--import-form-target="fileField">
|
|
93
|
+
<%= f.label :file, "File", class: "dp-label" %>
|
|
94
|
+
<label class="dp-dropzone" data-data-porter--import-form-target="dropzone"
|
|
95
|
+
data-action="dragover->data-porter--import-form#dragover dragleave->data-porter--import-form#dragleave drop->data-porter--import-form#drop">
|
|
96
|
+
<input type="file" name="data_import[file]" class="dp-dropzone__input"
|
|
97
|
+
data-data-porter--import-form-target="fileInput"
|
|
98
|
+
data-action="data-porter--import-form#handleFile" />
|
|
99
|
+
<div class="dp-dropzone__content">
|
|
100
|
+
<div class="dp-dropzone__icon">📄</div>
|
|
101
|
+
<span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
|
|
102
|
+
<span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
|
|
103
|
+
</div>
|
|
104
|
+
<div class="dp-dropzone__selected" data-data-porter--import-form-target="fileName" style="display: none;"></div>
|
|
105
|
+
</label>
|
|
106
|
+
</div>
|
|
146
107
|
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
dropzone.addEventListener("drop", function(e) {
|
|
156
|
-
e.preventDefault();
|
|
157
|
-
this.classList.remove("dp-dropzone--dragover");
|
|
158
|
-
if (e.dataTransfer.files.length > 0) {
|
|
159
|
-
fileInput.files = e.dataTransfer.files;
|
|
160
|
-
fileInput.dispatchEvent(new Event("change"));
|
|
161
|
-
}
|
|
162
|
-
});
|
|
163
|
-
}
|
|
164
|
-
|
|
165
|
-
document.addEventListener("keydown", function(e) {
|
|
166
|
-
if (e.key === "Escape") {
|
|
167
|
-
document.getElementById("dp-modal").classList.remove("dp-modal--open");
|
|
168
|
-
}
|
|
169
|
-
});
|
|
170
|
-
})();
|
|
171
|
-
</script>
|
|
108
|
+
<div class="dp-modal__footer">
|
|
109
|
+
<%= f.submit "Start Import", class: "dp-btn dp-btn--primary" %>
|
|
110
|
+
<button type="button" class="dp-btn dp-btn--secondary" data-action="data-porter--import-form#closeModalClick">Cancel</button>
|
|
111
|
+
</div>
|
|
112
|
+
<% end %>
|
|
113
|
+
</div>
|
|
114
|
+
</div>
|
|
115
|
+
</div>
|
|
@@ -19,9 +19,14 @@
|
|
|
19
19
|
{ prompt: "Select a target..." },
|
|
20
20
|
id: "dp-target-select-new",
|
|
21
21
|
class: "dp-select",
|
|
22
|
-
data: {
|
|
22
|
+
data: {
|
|
23
|
+
sources: @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json,
|
|
24
|
+
params: @targets.map { |t| [t[:key], t[:params]] }.to_h.to_json
|
|
25
|
+
} %>
|
|
23
26
|
</div>
|
|
24
27
|
|
|
28
|
+
<div id="dp-params-container" class="dp-params-container"></div>
|
|
29
|
+
|
|
25
30
|
<div class="dp-field">
|
|
26
31
|
<%= f.label :source_type, "Source Type", class: "dp-label" %>
|
|
27
32
|
<%= f.select :source_type,
|
|
@@ -59,6 +64,47 @@
|
|
|
59
64
|
var fileInput = document.getElementById("dp-file-input-new");
|
|
60
65
|
var dropzone = document.getElementById("dp-dropzone-new");
|
|
61
66
|
var fileName = document.getElementById("dp-file-name-new");
|
|
67
|
+
var paramsContainer = document.getElementById("dp-params-container");
|
|
68
|
+
var paramsData = JSON.parse((targetSelect && targetSelect.dataset.params) || "{}");
|
|
69
|
+
|
|
70
|
+
function renderParams() {
|
|
71
|
+
if (!paramsContainer || !targetSelect) return;
|
|
72
|
+
paramsContainer.innerHTML = "";
|
|
73
|
+
var defs = paramsData[targetSelect.value] || [];
|
|
74
|
+
defs.forEach(function(p) {
|
|
75
|
+
var div = document.createElement("div");
|
|
76
|
+
div.className = "dp-field";
|
|
77
|
+
var label = document.createElement("label");
|
|
78
|
+
label.className = "dp-label";
|
|
79
|
+
label.textContent = p.label + (p.required ? " *" : "");
|
|
80
|
+
div.appendChild(label);
|
|
81
|
+
var input;
|
|
82
|
+
if (p.type === "select" && p.collection) {
|
|
83
|
+
input = document.createElement("select");
|
|
84
|
+
input.className = "dp-select";
|
|
85
|
+
var blank = document.createElement("option");
|
|
86
|
+
blank.value = "";
|
|
87
|
+
blank.textContent = "Select...";
|
|
88
|
+
input.appendChild(blank);
|
|
89
|
+
p.collection.forEach(function(opt) {
|
|
90
|
+
var o = document.createElement("option");
|
|
91
|
+
o.textContent = opt[0];
|
|
92
|
+
o.value = opt[1];
|
|
93
|
+
if (p["default"] && String(opt[1]) === String(p["default"])) o.selected = true;
|
|
94
|
+
input.appendChild(o);
|
|
95
|
+
});
|
|
96
|
+
} else {
|
|
97
|
+
input = document.createElement("input");
|
|
98
|
+
input.className = "dp-input";
|
|
99
|
+
input.type = p.type === "number" ? "number" : (p.type === "hidden" ? "hidden" : "text");
|
|
100
|
+
if (p["default"]) input.value = p["default"];
|
|
101
|
+
}
|
|
102
|
+
input.name = "data_import[config][import_params][" + p.name + "]";
|
|
103
|
+
if (p.required) input.required = true;
|
|
104
|
+
div.appendChild(input);
|
|
105
|
+
paramsContainer.appendChild(div);
|
|
106
|
+
});
|
|
107
|
+
}
|
|
62
108
|
|
|
63
109
|
function filterSources() {
|
|
64
110
|
if (!targetSelect || !sourceSelect) return;
|
|
@@ -75,7 +121,10 @@
|
|
|
75
121
|
}
|
|
76
122
|
|
|
77
123
|
if (targetSelect) {
|
|
78
|
-
targetSelect.addEventListener("change",
|
|
124
|
+
targetSelect.addEventListener("change", function() {
|
|
125
|
+
filterSources();
|
|
126
|
+
renderParams();
|
|
127
|
+
});
|
|
79
128
|
}
|
|
80
129
|
|
|
81
130
|
if (sourceSelect) {
|
|
@@ -45,11 +45,16 @@
|
|
|
45
45
|
<% end %>
|
|
46
46
|
|
|
47
47
|
<% if @import.previewing? %>
|
|
48
|
+
<div id="records">
|
|
48
49
|
<%= raw DataPorter::Components::Preview::SummaryCards.new(report: @import.report).call %>
|
|
49
50
|
<%= raw DataPorter::Components::Preview::Table.new(
|
|
50
51
|
columns: @target._columns,
|
|
51
52
|
records: @records
|
|
52
53
|
).call %>
|
|
54
|
+
</div>
|
|
55
|
+
<%= raw DataPorter::Components::Shared::Pagination.new(
|
|
56
|
+
page: @page, total_pages: @total_pages, base_url: import_path(@import)
|
|
57
|
+
).call %>
|
|
53
58
|
|
|
54
59
|
<div class="dp-actions">
|
|
55
60
|
<%= button_to confirm_import_path(@import), method: :post, class: "dp-btn dp-btn--primary", data: { dp_submit: true } do %>
|
|
@@ -69,10 +74,15 @@
|
|
|
69
74
|
<% duration = @import.updated_at && @import.created_at ? distance_of_time_in_words(@import.created_at, @import.updated_at) : nil %>
|
|
70
75
|
<%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report, duration: duration).call %>
|
|
71
76
|
<% if @records.any? %>
|
|
77
|
+
<div id="records">
|
|
72
78
|
<%= raw DataPorter::Components::Preview::Table.new(
|
|
73
79
|
columns: @target._columns,
|
|
74
80
|
records: @records
|
|
75
81
|
).call %>
|
|
82
|
+
</div>
|
|
83
|
+
<%= raw DataPorter::Components::Shared::Pagination.new(
|
|
84
|
+
page: @page, total_pages: @total_pages, base_url: import_path(@import)
|
|
85
|
+
).call %>
|
|
76
86
|
<% end %>
|
|
77
87
|
<div class="dp-actions">
|
|
78
88
|
<%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--primary" %>
|
|
@@ -10,159 +10,30 @@
|
|
|
10
10
|
{
|
|
11
11
|
"imports": {
|
|
12
12
|
"@hotwired/turbo": "<%= asset_path('data_porter/turbo.min.js') %>",
|
|
13
|
-
"@hotwired/stimulus": "<%= asset_path('data_porter/stimulus.min.js') %>"
|
|
13
|
+
"@hotwired/stimulus": "<%= asset_path('data_porter/stimulus.min.js') %>",
|
|
14
|
+
"data_porter/mapping_controller": "<%= asset_path('data_porter/mapping_controller.js') %>",
|
|
15
|
+
"data_porter/template_form_controller": "<%= asset_path('data_porter/template_form_controller.js') %>",
|
|
16
|
+
"data_porter/progress_controller": "<%= asset_path('data_porter/progress_controller.js') %>",
|
|
17
|
+
"data_porter/import_form_controller": "<%= asset_path('data_porter/import_form_controller.js') %>"
|
|
14
18
|
}
|
|
15
19
|
}
|
|
16
20
|
</script>
|
|
17
|
-
</head>
|
|
18
|
-
<body>
|
|
19
|
-
<%= yield %>
|
|
20
|
-
|
|
21
21
|
<script type="module">
|
|
22
22
|
import "@hotwired/turbo"
|
|
23
|
-
import { Application
|
|
23
|
+
import { Application } from "@hotwired/stimulus"
|
|
24
|
+
import MappingController from "data_porter/mapping_controller"
|
|
25
|
+
import TemplateFormController from "data_porter/template_form_controller"
|
|
26
|
+
import ProgressController from "data_porter/progress_controller"
|
|
27
|
+
import ImportFormController from "data_porter/import_form_controller"
|
|
24
28
|
|
|
25
29
|
const application = Application.start()
|
|
26
|
-
|
|
27
|
-
application.register("data-porter--
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
connect() { this.validate() }
|
|
32
|
-
|
|
33
|
-
loadTemplate(event) {
|
|
34
|
-
const option = event.target.selectedOptions[0]
|
|
35
|
-
if (!option || !option.dataset.mapping) return
|
|
36
|
-
const mapping = JSON.parse(option.dataset.mapping)
|
|
37
|
-
this.columnSelectTargets.forEach(select => {
|
|
38
|
-
const header = select.name.match(/\[(.+)\]/)?.[1]
|
|
39
|
-
select.value = (header && mapping[header]) ? mapping[header] : ""
|
|
40
|
-
})
|
|
41
|
-
if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
|
|
42
|
-
this.validate()
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
onChange() { this.validate() }
|
|
46
|
-
|
|
47
|
-
validate() {
|
|
48
|
-
this.validateRequired()
|
|
49
|
-
this.validateDuplicates()
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
validateRequired() {
|
|
53
|
-
if (!this.hasRequiredWarningTarget) return
|
|
54
|
-
const selected = new Set(this.columnSelectTargets.map(s => s.value).filter(v => v !== ""))
|
|
55
|
-
const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
|
|
56
|
-
if (missing.length > 0) {
|
|
57
|
-
this.requiredWarningTarget.textContent = "Required fields not mapped: " + missing.map(c => c.label).join(", ")
|
|
58
|
-
this.requiredWarningTarget.style.display = ""
|
|
59
|
-
} else {
|
|
60
|
-
this.requiredWarningTarget.style.display = "none"
|
|
61
|
-
}
|
|
62
|
-
}
|
|
63
|
-
|
|
64
|
-
validateDuplicates() {
|
|
65
|
-
const counts = {}
|
|
66
|
-
this.columnSelectTargets.forEach(select => {
|
|
67
|
-
if (select.value === "") return
|
|
68
|
-
counts[select.value] = (counts[select.value] || 0) + 1
|
|
69
|
-
})
|
|
70
|
-
const duplicates = new Set(Object.keys(counts).filter(k => counts[k] > 1))
|
|
71
|
-
this.columnSelectTargets.forEach(select => {
|
|
72
|
-
const row = select.closest(".dp-mapping-row")
|
|
73
|
-
if (!row) return
|
|
74
|
-
row.classList.toggle("dp-mapping-row--duplicate", select.value !== "" && duplicates.has(select.value))
|
|
75
|
-
})
|
|
76
|
-
if (this.hasDuplicateWarningTarget) {
|
|
77
|
-
if (duplicates.size > 0) {
|
|
78
|
-
this.duplicateWarningTarget.textContent = "Duplicate mappings detected for: " + [...duplicates].join(", ")
|
|
79
|
-
this.duplicateWarningTarget.style.display = ""
|
|
80
|
-
} else {
|
|
81
|
-
this.duplicateWarningTarget.style.display = "none"
|
|
82
|
-
}
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
})
|
|
86
|
-
|
|
87
|
-
application.register("data-porter--template-form", class extends Controller {
|
|
88
|
-
static targets = ["pairsContainer", "fieldSelect"]
|
|
89
|
-
static values = { columns: Object }
|
|
90
|
-
|
|
91
|
-
targetChanged(event) {
|
|
92
|
-
const columns = this.columnsValue[event.target.value] || []
|
|
93
|
-
this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
|
|
94
|
-
}
|
|
95
|
-
|
|
96
|
-
addPair() {
|
|
97
|
-
const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
|
|
98
|
-
const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
|
|
99
|
-
const pair = document.createElement("div")
|
|
100
|
-
pair.className = "dp-mapping-pair"
|
|
101
|
-
pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
|
|
102
|
-
pair.innerHTML = this.pairHTML(columns)
|
|
103
|
-
this.pairsContainerTarget.appendChild(pair)
|
|
104
|
-
}
|
|
105
|
-
|
|
106
|
-
updateOptions(select, columns) {
|
|
107
|
-
const current = select.value
|
|
108
|
-
select.innerHTML = '<option value="">Select a field...</option>'
|
|
109
|
-
columns.forEach(([label, name]) => {
|
|
110
|
-
const opt = document.createElement("option")
|
|
111
|
-
opt.value = name
|
|
112
|
-
opt.textContent = label
|
|
113
|
-
if (name === current) opt.selected = true
|
|
114
|
-
select.appendChild(opt)
|
|
115
|
-
})
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
pairHTML(columns) {
|
|
119
|
-
const options = columns.map(([label, name]) => '<option value="' + name + '">' + label + "</option>").join("")
|
|
120
|
-
return '<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />' +
|
|
121
|
-
'<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">' +
|
|
122
|
-
'<option value="">Select a field...</option>' + options + "</select>"
|
|
123
|
-
}
|
|
124
|
-
})
|
|
125
|
-
|
|
126
|
-
application.register("data-porter--progress", class extends Controller {
|
|
127
|
-
static targets = ["bar", "text", "label"]
|
|
128
|
-
static values = { id: Number, url: String }
|
|
129
|
-
|
|
130
|
-
connect() {
|
|
131
|
-
this.poll()
|
|
132
|
-
}
|
|
133
|
-
|
|
134
|
-
poll() {
|
|
135
|
-
this.timer = setInterval(() => this.fetchStatus(), 1000)
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
async fetchStatus() {
|
|
139
|
-
try {
|
|
140
|
-
const response = await fetch(this.urlValue)
|
|
141
|
-
const data = await response.json()
|
|
142
|
-
this.updateLabel(data.status)
|
|
143
|
-
if (data.progress && data.progress.percentage !== undefined) {
|
|
144
|
-
this.barTarget.style.width = data.progress.percentage + "%"
|
|
145
|
-
this.textTarget.textContent = data.progress.percentage + "%"
|
|
146
|
-
}
|
|
147
|
-
if (!["pending", "importing", "parsing", "dry_running", "extracting_headers"].includes(data.status)) {
|
|
148
|
-
clearInterval(this.timer)
|
|
149
|
-
this.barTarget.style.width = "100%"
|
|
150
|
-
this.textTarget.textContent = "100%"
|
|
151
|
-
setTimeout(() => Turbo.visit(window.location.href, { action: "replace" }), 500)
|
|
152
|
-
}
|
|
153
|
-
} catch (e) {}
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
updateLabel(status) {
|
|
157
|
-
if (!this.hasLabelTarget) return
|
|
158
|
-
var labels = { pending: "Waiting...", extracting_headers: "Extracting headers...", parsing: "Parsing records...", importing: "Importing...", dry_running: "Dry run..." }
|
|
159
|
-
this.labelTarget.textContent = labels[status] || "Processing..."
|
|
160
|
-
}
|
|
161
|
-
|
|
162
|
-
disconnect() {
|
|
163
|
-
if (this.timer) clearInterval(this.timer)
|
|
164
|
-
}
|
|
165
|
-
})
|
|
30
|
+
application.register("data-porter--mapping", MappingController)
|
|
31
|
+
application.register("data-porter--template-form", TemplateFormController)
|
|
32
|
+
application.register("data-porter--progress", ProgressController)
|
|
33
|
+
application.register("data-porter--import-form", ImportFormController)
|
|
166
34
|
</script>
|
|
35
|
+
</head>
|
|
36
|
+
<body>
|
|
37
|
+
<%= yield %>
|
|
167
38
|
</body>
|
|
168
39
|
</html>
|
data/docs/CONFIGURATION.md
CHANGED
|
@@ -28,6 +28,10 @@ DataPorter.configure do |config|
|
|
|
28
28
|
|
|
29
29
|
# Enabled source types.
|
|
30
30
|
config.enabled_sources = %i[csv json api xlsx]
|
|
31
|
+
|
|
32
|
+
# Auto-purge completed/failed imports older than this duration.
|
|
33
|
+
# Set to nil to disable. Run `rake data_porter:purge` manually or via cron.
|
|
34
|
+
config.purge_after = 60.days
|
|
31
35
|
end
|
|
32
36
|
```
|
|
33
37
|
|
|
@@ -42,6 +46,7 @@ end
|
|
|
42
46
|
| `context_builder` | `nil` | Lambda receiving the controller, returns context passed to target methods |
|
|
43
47
|
| `preview_limit` | `500` | Max records shown in the preview step |
|
|
44
48
|
| `enabled_sources` | `%i[csv json api xlsx]` | Source types available in the UI |
|
|
49
|
+
| `purge_after` | `60.days` | Auto-purge completed/failed imports older than this duration |
|
|
45
50
|
|
|
46
51
|
## Authentication
|
|
47
52
|
|
|
@@ -68,14 +73,31 @@ config.context_builder = ->(controller) {
|
|
|
68
73
|
|
|
69
74
|
The returned object is available as `context` in all target instance methods.
|
|
70
75
|
|
|
71
|
-
## Real-time
|
|
76
|
+
## Real-time progress
|
|
72
77
|
|
|
73
|
-
DataPorter
|
|
78
|
+
DataPorter tracks import progress via JSON polling. The Stimulus progress controller polls `GET /imports/:id/status` every second and updates an animated progress bar.
|
|
74
79
|
|
|
80
|
+
The status endpoint returns:
|
|
81
|
+
|
|
82
|
+
```json
|
|
83
|
+
{
|
|
84
|
+
"status": "importing",
|
|
85
|
+
"progress": { "current": 42, "total": 100, "percentage": 42 }
|
|
86
|
+
}
|
|
75
87
|
```
|
|
76
|
-
#{cable_channel_prefix}/imports/#{import_id}
|
|
77
|
-
```
|
|
78
88
|
|
|
79
|
-
|
|
89
|
+
No ActionCable or WebSocket configuration required -- it works out of the box with any deployment.
|
|
90
|
+
|
|
91
|
+
## Auto-purge
|
|
92
|
+
|
|
93
|
+
Old completed/failed imports can be cleaned up automatically:
|
|
94
|
+
|
|
95
|
+
```bash
|
|
96
|
+
# Run manually
|
|
97
|
+
bin/rails data_porter:purge
|
|
98
|
+
|
|
99
|
+
# Or schedule via cron (e.g. with whenever or solid_queue)
|
|
100
|
+
# Removes imports older than purge_after (default: 60 days)
|
|
101
|
+
```
|
|
80
102
|
|
|
81
|
-
|
|
103
|
+
Attached files are purged from ActiveStorage along with the import record.
|
data/docs/ROADMAP.md
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# Roadmap
|
|
2
|
+
|
|
3
|
+
## v1.0 — Production-ready
|
|
4
|
+
|
|
5
|
+
The goal is a gem that handles real-world imports reliably at scale.
|
|
6
|
+
|
|
7
|
+
### ~~1. Records pagination~~ DONE
|
|
8
|
+
|
|
9
|
+
Implemented in v0.6.0. Preview and completed pages are paginated (50 per page).
|
|
10
|
+
Controller limits records loaded via `RecordPagination` concern.
|
|
11
|
+
|
|
12
|
+
### ~~2. Import params~~ DONE
|
|
13
|
+
|
|
14
|
+
Implemented in v0.9.0. Targets declare `params` with a DSL (`:select`, `:text`,
|
|
15
|
+
`:number`, `:hidden`). Values stored in `config["import_params"]`, accessible
|
|
16
|
+
via `import_params` in all target instance methods. See [Targets docs](TARGETS.md#params--).
|
|
17
|
+
|
|
18
|
+
---
|
|
19
|
+
|
|
20
|
+
## v2+ (future)
|
|
21
|
+
|
|
22
|
+
- Scoped imports (filter index by user/tenant)
|
|
23
|
+
- Webhooks / callbacks on import completion
|
|
24
|
+
- Batch persist (`insert_all` support)
|
|
25
|
+
- Resume / partial retry
|
|
26
|
+
- Scheduled imports (recurring API source)
|
|
27
|
+
- i18n
|
|
28
|
+
- Dashboard stats
|
data/docs/TARGETS.md
CHANGED
|
@@ -5,20 +5,23 @@ Targets are plain Ruby classes in `app/importers/` that inherit from `DataPorter
|
|
|
5
5
|
## Generator
|
|
6
6
|
|
|
7
7
|
```bash
|
|
8
|
-
bin/rails generate data_porter:target ModelName column:type[:required] ...
|
|
8
|
+
bin/rails generate data_porter:target ModelName column:type[:required] ... [--sources csv xlsx]
|
|
9
9
|
```
|
|
10
10
|
|
|
11
11
|
Examples:
|
|
12
12
|
|
|
13
13
|
```bash
|
|
14
|
-
bin/rails generate data_porter:target User email:string:required name:string age:integer
|
|
15
|
-
bin/rails generate data_porter:target Product name:string price:decimal
|
|
14
|
+
bin/rails generate data_porter:target User email:string:required name:string age:integer --sources csv xlsx
|
|
15
|
+
bin/rails generate data_porter:target Product name:string price:decimal --sources csv
|
|
16
|
+
bin/rails generate data_porter:target Order order_number:string total:decimal
|
|
16
17
|
```
|
|
17
18
|
|
|
18
19
|
Column format: `name:type[:required]`
|
|
19
20
|
|
|
20
21
|
Supported types: `string`, `integer`, `decimal`, `boolean`, `date`.
|
|
21
22
|
|
|
23
|
+
The `--sources` option specifies which source types the target accepts (default: `csv`). The UI will only show these sources when the target is selected.
|
|
24
|
+
|
|
22
25
|
## Class-level DSL
|
|
23
26
|
|
|
24
27
|
```ruby
|
|
@@ -52,6 +55,12 @@ class OrderTarget < DataPorter::Target
|
|
|
52
55
|
deduplicate_by :order_number
|
|
53
56
|
|
|
54
57
|
dry_run_enabled
|
|
58
|
+
|
|
59
|
+
params do
|
|
60
|
+
param :warehouse_id, type: :select, label: "Warehouse", required: true,
|
|
61
|
+
collection: -> { Warehouse.pluck(:name, :id) }
|
|
62
|
+
param :currency, type: :text, default: "USD"
|
|
63
|
+
end
|
|
55
64
|
end
|
|
56
65
|
```
|
|
57
66
|
|
|
@@ -120,8 +129,50 @@ deduplicate_by :first_name, :last_name
|
|
|
120
129
|
|
|
121
130
|
Enables dry run mode for this target. A "Dry Run" button appears in the preview step. Dry run executes the full import pipeline (transform, validate, persist) inside a rolled-back transaction, giving a validation report without modifying the database.
|
|
122
131
|
|
|
132
|
+
### `params { ... }`
|
|
133
|
+
|
|
134
|
+
Declares extra form fields shown when this target is selected in the import form. Values are stored in `config["import_params"]` and accessible via `import_params` in all instance methods.
|
|
135
|
+
|
|
136
|
+
```ruby
|
|
137
|
+
params do
|
|
138
|
+
param :hotel_id, type: :select, label: "Hotel", required: true,
|
|
139
|
+
collection: -> { Hotel.pluck(:name, :id) }
|
|
140
|
+
param :currency, type: :text, label: "Currency", default: "EUR"
|
|
141
|
+
param :batch_size, type: :number, label: "Batch Size", default: "100"
|
|
142
|
+
param :tenant_id, type: :hidden, default: "abc123"
|
|
143
|
+
end
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
Each param accepts:
|
|
147
|
+
|
|
148
|
+
| Parameter | Type | Default | Description |
|
|
149
|
+
|---|---|---|---|
|
|
150
|
+
| `name` | Symbol | (required) | Param identifier |
|
|
151
|
+
| `type` | Symbol | `:text` | One of `:select`, `:text`, `:number`, `:hidden` |
|
|
152
|
+
| `required` | Boolean | `false` | Validated on import creation, shown with `*` in the form |
|
|
153
|
+
| `label` | String | Humanized name | Display label in the form |
|
|
154
|
+
| `default` | String | `nil` | Pre-filled value in the form |
|
|
155
|
+
| `collection` | Lambda | `nil` | For `:select` type -- returns `[[label, value], ...]` |
|
|
156
|
+
|
|
157
|
+
Collection lambdas are evaluated when the form loads, not at boot time. This ensures fresh data (e.g., newly created hotels appear immediately).
|
|
158
|
+
|
|
123
159
|
## Instance Methods
|
|
124
160
|
|
|
161
|
+
### `import_params`
|
|
162
|
+
|
|
163
|
+
Returns a hash of the import params values set by the user in the form. Available in all instance methods (`persist`, `transform`, `validate`, `after_import`, `on_error`). Defaults to `{}` when no params are declared.
|
|
164
|
+
|
|
165
|
+
```ruby
|
|
166
|
+
def persist(record, context:)
|
|
167
|
+
Guest.create!(
|
|
168
|
+
record.attributes.merge(
|
|
169
|
+
hotel_id: import_params["hotel_id"],
|
|
170
|
+
currency: import_params["currency"]
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
end
|
|
174
|
+
```
|
|
175
|
+
|
|
125
176
|
Override these in your target to customize behavior.
|
|
126
177
|
|
|
127
178
|
### `transform(record)`
|