data_porter 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +62 -1
- data/README.md +63 -386
- data/ROADMAP.md +89 -0
- data/app/assets/javascripts/data_porter/stimulus.min.js +2 -0
- data/app/assets/javascripts/data_porter/turbo.min.js +29 -0
- data/app/assets/stylesheets/data_porter/alerts.css +25 -0
- data/app/assets/stylesheets/data_porter/application.css +12 -646
- data/app/assets/stylesheets/data_porter/badges.css +73 -0
- data/app/assets/stylesheets/data_porter/base.css +56 -0
- data/app/assets/stylesheets/data_porter/cards.css +60 -0
- data/app/assets/stylesheets/data_porter/layout.css +128 -0
- data/app/assets/stylesheets/data_porter/mapping.css +79 -0
- data/app/assets/stylesheets/data_porter/modal.css +49 -0
- data/app/assets/stylesheets/data_porter/preview.css +24 -0
- data/app/assets/stylesheets/data_porter/progress.css +37 -0
- data/app/assets/stylesheets/data_porter/table.css +45 -0
- data/app/controllers/data_porter/imports_controller.rb +74 -10
- data/app/controllers/data_porter/mapping_templates_controller.rb +85 -0
- data/app/javascript/data_porter/mapping_controller.js +86 -0
- data/app/javascript/data_porter/progress_controller.js +1 -1
- data/app/javascript/data_porter/template_form_controller.js +46 -0
- data/app/jobs/data_porter/extract_headers_job.rb +12 -0
- data/app/models/data_porter/data_import.rb +8 -2
- data/app/models/data_porter/mapping_template.rb +15 -0
- data/app/views/data_porter/imports/index.html.erb +9 -8
- data/app/views/data_porter/imports/new.html.erb +10 -4
- data/app/views/data_porter/imports/show.html.erb +41 -13
- data/app/views/data_porter/mapping_templates/_form.html.erb +40 -0
- data/app/views/data_porter/mapping_templates/edit.html.erb +11 -0
- data/app/views/data_porter/mapping_templates/index.html.erb +42 -0
- data/app/views/data_porter/mapping_templates/new.html.erb +11 -0
- data/app/views/layouts/data_porter/application.html.erb +162 -0
- data/config/routes.rb +3 -0
- data/docs/CONFIGURATION.md +81 -0
- data/docs/MAPPING.md +44 -0
- data/docs/SOURCES.md +94 -0
- data/docs/TARGETS.md +176 -0
- data/docs/screenshots/mapping.jpg +0 -0
- data/lib/data_porter/components/mapping/column_row.rb +52 -0
- data/lib/data_porter/components/mapping/form.rb +127 -0
- data/lib/data_porter/components/mapping/template_select.rb +35 -0
- data/lib/data_porter/components/preview/results_summary.rb +21 -0
- data/lib/data_porter/components/preview/summary_cards.rb +32 -0
- data/lib/data_porter/components/preview/table.rb +56 -0
- data/lib/data_porter/components/progress/bar.rb +35 -0
- data/lib/data_porter/components/shared/failure_alert.rb +22 -0
- data/lib/data_porter/components/shared/status_badge.rb +18 -0
- data/lib/data_porter/components.rb +9 -6
- data/lib/data_porter/configuration.rb +1 -1
- data/lib/data_porter/engine.rb +7 -1
- data/lib/data_porter/orchestrator.rb +21 -1
- data/lib/data_porter/sources/base.rb +18 -3
- data/lib/data_porter/sources/csv.rb +5 -0
- data/lib/data_porter/sources/xlsx.rb +76 -0
- data/lib/data_porter/sources.rb +3 -1
- data/lib/data_porter/version.rb +1 -1
- data/lib/generators/data_porter/install/install_generator.rb +4 -0
- data/lib/generators/data_porter/install/templates/create_data_porter_mapping_templates.rb.erb +16 -0
- data/lib/generators/data_porter/install/templates/initializer.rb +1 -1
- metadata +72 -135
- data/.claude/commands/blog-status.md +0 -10
- data/.claude/commands/blog.md +0 -109
- data/.claude/commands/task-done.md +0 -27
- data/.claude/commands/tm/add-dependency.md +0 -58
- data/.claude/commands/tm/add-subtask.md +0 -79
- data/.claude/commands/tm/add-task.md +0 -81
- data/.claude/commands/tm/analyze-complexity.md +0 -124
- data/.claude/commands/tm/analyze-project.md +0 -100
- data/.claude/commands/tm/auto-implement-tasks.md +0 -100
- data/.claude/commands/tm/command-pipeline.md +0 -80
- data/.claude/commands/tm/complexity-report.md +0 -120
- data/.claude/commands/tm/convert-task-to-subtask.md +0 -74
- data/.claude/commands/tm/expand-all-tasks.md +0 -52
- data/.claude/commands/tm/expand-task.md +0 -52
- data/.claude/commands/tm/fix-dependencies.md +0 -82
- data/.claude/commands/tm/help.md +0 -101
- data/.claude/commands/tm/init-project-quick.md +0 -49
- data/.claude/commands/tm/init-project.md +0 -53
- data/.claude/commands/tm/install-taskmaster.md +0 -118
- data/.claude/commands/tm/learn.md +0 -106
- data/.claude/commands/tm/list-tasks-by-status.md +0 -42
- data/.claude/commands/tm/list-tasks-with-subtasks.md +0 -30
- data/.claude/commands/tm/list-tasks.md +0 -46
- data/.claude/commands/tm/next-task.md +0 -69
- data/.claude/commands/tm/parse-prd-with-research.md +0 -51
- data/.claude/commands/tm/parse-prd.md +0 -52
- data/.claude/commands/tm/project-status.md +0 -67
- data/.claude/commands/tm/quick-install-taskmaster.md +0 -23
- data/.claude/commands/tm/remove-all-subtasks.md +0 -94
- data/.claude/commands/tm/remove-dependency.md +0 -65
- data/.claude/commands/tm/remove-subtask.md +0 -87
- data/.claude/commands/tm/remove-subtasks.md +0 -89
- data/.claude/commands/tm/remove-task.md +0 -110
- data/.claude/commands/tm/setup-models.md +0 -52
- data/.claude/commands/tm/show-task.md +0 -85
- data/.claude/commands/tm/smart-workflow.md +0 -58
- data/.claude/commands/tm/sync-readme.md +0 -120
- data/.claude/commands/tm/tm-main.md +0 -147
- data/.claude/commands/tm/to-cancelled.md +0 -58
- data/.claude/commands/tm/to-deferred.md +0 -50
- data/.claude/commands/tm/to-done.md +0 -47
- data/.claude/commands/tm/to-in-progress.md +0 -39
- data/.claude/commands/tm/to-pending.md +0 -35
- data/.claude/commands/tm/to-review.md +0 -43
- data/.claude/commands/tm/update-single-task.md +0 -122
- data/.claude/commands/tm/update-task.md +0 -75
- data/.claude/commands/tm/update-tasks-from-id.md +0 -111
- data/.claude/commands/tm/validate-dependencies.md +0 -72
- data/.claude/commands/tm/view-models.md +0 -52
- data/.env.example +0 -12
- data/.mcp.json +0 -24
- data/.taskmaster/CLAUDE.md +0 -435
- data/.taskmaster/config.json +0 -44
- data/.taskmaster/docs/prd.txt +0 -2044
- data/.taskmaster/state.json +0 -6
- data/.taskmaster/tasks/task_001.md +0 -19
- data/.taskmaster/tasks/task_002.md +0 -19
- data/.taskmaster/tasks/task_003.md +0 -19
- data/.taskmaster/tasks/task_004.md +0 -19
- data/.taskmaster/tasks/task_005.md +0 -19
- data/.taskmaster/tasks/task_006.md +0 -19
- data/.taskmaster/tasks/task_007.md +0 -19
- data/.taskmaster/tasks/task_008.md +0 -19
- data/.taskmaster/tasks/task_009.md +0 -19
- data/.taskmaster/tasks/task_010.md +0 -19
- data/.taskmaster/tasks/task_011.md +0 -19
- data/.taskmaster/tasks/task_012.md +0 -19
- data/.taskmaster/tasks/task_013.md +0 -19
- data/.taskmaster/tasks/task_014.md +0 -19
- data/.taskmaster/tasks/task_015.md +0 -19
- data/.taskmaster/tasks/task_016.md +0 -19
- data/.taskmaster/tasks/task_017.md +0 -19
- data/.taskmaster/tasks/task_018.md +0 -19
- data/.taskmaster/tasks/task_019.md +0 -19
- data/.taskmaster/tasks/task_020.md +0 -19
- data/.taskmaster/tasks/tasks.json +0 -299
- data/.taskmaster/templates/example_prd.txt +0 -47
- data/.taskmaster/templates/example_prd_rpg.txt +0 -511
- data/CLAUDE.md +0 -65
- data/config/database.yml +0 -3
- data/docs/SPEC.md +0 -2012
- data/docs/UI.md +0 -32
- data/docs/blog/001-why-build-a-data-import-engine.md +0 -166
- data/docs/blog/002-scaffolding-a-rails-engine.md +0 -188
- data/docs/blog/003-configuration-dsl.md +0 -222
- data/docs/blog/004-store-model-jsonb.md +0 -237
- data/docs/blog/005-target-dsl.md +0 -284
- data/docs/blog/006-parsing-csv-sources.md +0 -300
- data/docs/blog/007-orchestrator.md +0 -247
- data/docs/blog/008-actioncable-stimulus.md +0 -376
- data/docs/blog/009-phlex-ui-components.md +0 -446
- data/docs/blog/010-controllers-routing.md +0 -374
- data/docs/blog/011-generators.md +0 -364
- data/docs/blog/012-json-api-sources.md +0 -323
- data/docs/blog/013-testing-rails-engine.md +0 -618
- data/docs/blog/014-dry-run.md +0 -307
- data/docs/blog/015-publishing-retro.md +0 -264
- data/docs/blog/016-erb-view-templates.md +0 -431
- data/docs/blog/017-showcase-final-retro.md +0 -220
- data/docs/blog/BACKLOG.md +0 -8
- data/docs/blog/SERIES.md +0 -154
- data/lib/data_porter/components/failure_alert.rb +0 -20
- data/lib/data_porter/components/preview_table.rb +0 -54
- data/lib/data_porter/components/progress_bar.rb +0 -33
- data/lib/data_porter/components/results_summary.rb +0 -19
- data/lib/data_porter/components/status_badge.rb +0 -16
- data/lib/data_porter/components/summary_cards.rb +0 -30
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Controller } from "@hotwired/stimulus"
|
|
2
|
+
|
|
3
|
+
export default class extends Controller {
|
|
4
|
+
static targets = ["columnSelect", "requiredWarning", "duplicateWarning", "saveTemplate"]
|
|
5
|
+
static values = { requiredColumns: Array }
|
|
6
|
+
|
|
7
|
+
connect() {
|
|
8
|
+
this.validate()
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
loadTemplate(event) {
|
|
12
|
+
const option = event.target.selectedOptions[0]
|
|
13
|
+
if (!option || !option.dataset.mapping) return
|
|
14
|
+
|
|
15
|
+
const mapping = JSON.parse(option.dataset.mapping)
|
|
16
|
+
this.columnSelectTargets.forEach(select => {
|
|
17
|
+
const header = select.name.match(/\[(.+)\]/)?.[1]
|
|
18
|
+
if (header && mapping[header]) {
|
|
19
|
+
select.value = mapping[header]
|
|
20
|
+
} else {
|
|
21
|
+
select.value = ""
|
|
22
|
+
}
|
|
23
|
+
})
|
|
24
|
+
if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
|
|
25
|
+
this.validate()
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
onChange() {
|
|
29
|
+
this.validate()
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
validate() {
|
|
33
|
+
this.validateRequired()
|
|
34
|
+
this.validateDuplicates()
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
validateRequired() {
|
|
38
|
+
if (!this.hasRequiredWarningTarget) return
|
|
39
|
+
|
|
40
|
+
const selected = new Set(
|
|
41
|
+
this.columnSelectTargets.map(s => s.value).filter(v => v !== "")
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
|
|
45
|
+
|
|
46
|
+
if (missing.length > 0) {
|
|
47
|
+
const names = missing.map(c => c.label).join(", ")
|
|
48
|
+
this.requiredWarningTarget.textContent = `Required fields not mapped: ${names}`
|
|
49
|
+
this.requiredWarningTarget.style.display = ""
|
|
50
|
+
} else {
|
|
51
|
+
this.requiredWarningTarget.style.display = "none"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
validateDuplicates() {
|
|
56
|
+
const counts = {}
|
|
57
|
+
this.columnSelectTargets.forEach(select => {
|
|
58
|
+
if (select.value === "") return
|
|
59
|
+
counts[select.value] = (counts[select.value] || 0) + 1
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
const duplicates = new Set(
|
|
63
|
+
Object.keys(counts).filter(k => counts[k] > 1)
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
this.columnSelectTargets.forEach(select => {
|
|
67
|
+
const row = select.closest(".dp-mapping-row")
|
|
68
|
+
if (!row) return
|
|
69
|
+
|
|
70
|
+
if (select.value !== "" && duplicates.has(select.value)) {
|
|
71
|
+
row.classList.add("dp-mapping-row--duplicate")
|
|
72
|
+
} else {
|
|
73
|
+
row.classList.remove("dp-mapping-row--duplicate")
|
|
74
|
+
}
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
if (this.hasDuplicateWarningTarget) {
|
|
78
|
+
if (duplicates.size > 0) {
|
|
79
|
+
this.duplicateWarningTarget.textContent = `Duplicate mappings detected for: ${[...duplicates].join(", ")}`
|
|
80
|
+
this.duplicateWarningTarget.style.display = ""
|
|
81
|
+
} else {
|
|
82
|
+
this.duplicateWarningTarget.style.display = "none"
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Controller } from "@hotwired/stimulus"
|
|
2
|
+
|
|
3
|
+
export default class extends Controller {
|
|
4
|
+
static targets = ["pairsContainer", "fieldSelect"]
|
|
5
|
+
static values = { columns: Object }
|
|
6
|
+
|
|
7
|
+
targetChanged(event) {
|
|
8
|
+
const targetKey = event.target.value
|
|
9
|
+
const columns = this.columnsValue[targetKey] || []
|
|
10
|
+
this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
addPair() {
|
|
14
|
+
const container = this.pairsContainerTarget
|
|
15
|
+
const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
|
|
16
|
+
const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
|
|
17
|
+
|
|
18
|
+
const pair = document.createElement("div")
|
|
19
|
+
pair.className = "dp-mapping-pair"
|
|
20
|
+
pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
|
|
21
|
+
pair.innerHTML = this.pairHTML(columns)
|
|
22
|
+
container.appendChild(pair)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
updateOptions(select, columns) {
|
|
26
|
+
const current = select.value
|
|
27
|
+
select.innerHTML = '<option value="">Select a field...</option>'
|
|
28
|
+
columns.forEach(([label, name]) => {
|
|
29
|
+
const opt = document.createElement("option")
|
|
30
|
+
opt.value = name
|
|
31
|
+
opt.textContent = label
|
|
32
|
+
if (name === current) opt.selected = true
|
|
33
|
+
select.appendChild(opt)
|
|
34
|
+
})
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
pairHTML(columns) {
|
|
38
|
+
const options = columns.map(([label, name]) =>
|
|
39
|
+
`<option value="${name}">${label}</option>`
|
|
40
|
+
).join("")
|
|
41
|
+
|
|
42
|
+
return `<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />` +
|
|
43
|
+
`<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">` +
|
|
44
|
+
`<option value="">Select a field...</option>${options}</select>`
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DataPorter
|
|
4
|
+
class ExtractHeadersJob < ActiveJob::Base
|
|
5
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
6
|
+
|
|
7
|
+
def perform(import_id)
|
|
8
|
+
data_import = DataImport.find(import_id)
|
|
9
|
+
Orchestrator.new(data_import).extract_headers!
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -15,7 +15,9 @@ module DataPorter
|
|
|
15
15
|
importing: 3,
|
|
16
16
|
completed: 4,
|
|
17
17
|
failed: 5,
|
|
18
|
-
dry_running: 6
|
|
18
|
+
dry_running: 6,
|
|
19
|
+
extracting_headers: 7,
|
|
20
|
+
mapping: 8
|
|
19
21
|
}
|
|
20
22
|
|
|
21
23
|
attribute :records, StoreModels::ImportRecord.to_array_type, default: -> { [] }
|
|
@@ -24,7 +26,7 @@ module DataPorter
|
|
|
24
26
|
attribute :config, :json, default: -> { {} }
|
|
25
27
|
|
|
26
28
|
validates :target_key, presence: true
|
|
27
|
-
validates :source_type, presence: true, inclusion: { in: %w[csv json api] }
|
|
29
|
+
validates :source_type, presence: true, inclusion: { in: %w[csv json api xlsx] }
|
|
28
30
|
|
|
29
31
|
def target_class
|
|
30
32
|
Registry.find(target_key)
|
|
@@ -45,5 +47,9 @@ module DataPorter
|
|
|
45
47
|
def records_summary
|
|
46
48
|
records.group_by(&:status).transform_values(&:count)
|
|
47
49
|
end
|
|
50
|
+
|
|
51
|
+
def file_based?
|
|
52
|
+
%w[csv xlsx].include?(source_type)
|
|
53
|
+
end
|
|
48
54
|
end
|
|
49
55
|
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DataPorter
|
|
4
|
+
class MappingTemplate < ActiveRecord::Base
|
|
5
|
+
self.table_name = "data_porter_mapping_templates"
|
|
6
|
+
|
|
7
|
+
attribute :mapping, :json, default: -> { {} }
|
|
8
|
+
|
|
9
|
+
validates :target_key, presence: true
|
|
10
|
+
validates :name, presence: true, uniqueness: { scope: :target_key }
|
|
11
|
+
validates :mapping, presence: true
|
|
12
|
+
|
|
13
|
+
scope :for_target, ->(key) { where(target_key: key) }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
5
3
|
<h1 class="dp-title">Imports</h1>
|
|
6
|
-
<
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
<div class="dp-header__actions">
|
|
5
|
+
<%= link_to "Mapping Templates", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
|
|
6
|
+
<button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
|
|
7
|
+
New Import
|
|
8
|
+
</button>
|
|
9
|
+
</div>
|
|
9
10
|
</div>
|
|
10
11
|
|
|
11
12
|
<% if @imports.any? %>
|
|
@@ -26,7 +27,7 @@
|
|
|
26
27
|
<td><%= import.id %></td>
|
|
27
28
|
<td><%= import.target_key %></td>
|
|
28
29
|
<td><%= import.source_type %></td>
|
|
29
|
-
<td><%= raw DataPorter::Components::StatusBadge.new(status: import.status).call %></td>
|
|
30
|
+
<td><%= raw DataPorter::Components::Shared::StatusBadge.new(status: import.status).call %></td>
|
|
30
31
|
<td><%= import.created_at&.strftime("%Y-%m-%d %H:%M") %></td>
|
|
31
32
|
<td><%= link_to "View", import_path(import), class: "dp-link" %></td>
|
|
32
33
|
</tr>
|
|
@@ -52,7 +53,7 @@
|
|
|
52
53
|
<button type="button" class="dp-modal__close" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">×</button>
|
|
53
54
|
</div>
|
|
54
55
|
|
|
55
|
-
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true do |f| %>
|
|
56
|
+
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
|
|
56
57
|
<div class="dp-field">
|
|
57
58
|
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
58
59
|
<%= f.select :target_key,
|
|
@@ -77,7 +78,7 @@
|
|
|
77
78
|
<div class="dp-dropzone__content">
|
|
78
79
|
<div class="dp-dropzone__icon">📄</div>
|
|
79
80
|
<span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
|
|
80
|
-
<span class="dp-dropzone__hint">CSV or
|
|
81
|
+
<span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
|
|
81
82
|
</div>
|
|
82
83
|
<div class="dp-dropzone__selected" id="dp-file-name" style="display: none;"></div>
|
|
83
84
|
</label>
|
|
@@ -1,11 +1,17 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
5
3
|
<h1 class="dp-title">New Import</h1>
|
|
6
4
|
</div>
|
|
7
5
|
|
|
8
|
-
|
|
6
|
+
<% if @import.errors.any? %>
|
|
7
|
+
<div class="dp-alert dp-alert--danger">
|
|
8
|
+
<% @import.errors.full_messages.each do |msg| %>
|
|
9
|
+
<p><%= msg %></p>
|
|
10
|
+
<% end %>
|
|
11
|
+
</div>
|
|
12
|
+
<% end %>
|
|
13
|
+
|
|
14
|
+
<%= form_with model: @import, url: imports_path, class: "dp-form", multipart: true, data: { turbo: false } do |f| %>
|
|
9
15
|
<div class="dp-field">
|
|
10
16
|
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
11
17
|
<%= f.select :target_key,
|
|
@@ -30,7 +36,7 @@
|
|
|
30
36
|
<div class="dp-dropzone__content">
|
|
31
37
|
<div class="dp-dropzone__icon">📄</div>
|
|
32
38
|
<span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
|
|
33
|
-
<span class="dp-dropzone__hint">CSV or
|
|
39
|
+
<span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
|
|
34
40
|
</div>
|
|
35
41
|
<div class="dp-dropzone__selected" id="dp-file-name-new" style="display: none;"></div>
|
|
36
42
|
</label>
|
|
@@ -1,20 +1,52 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
3
|
+
<div class="dp-header__actions">
|
|
4
|
+
<%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--secondary" %>
|
|
5
|
+
</div>
|
|
5
6
|
<h1 class="dp-title">
|
|
6
7
|
<%= @target._label %> Import #<%= @import.id %>
|
|
7
8
|
</h1>
|
|
8
|
-
<%= raw DataPorter::Components::StatusBadge.new(status: @import.status).call %>
|
|
9
|
+
<%= raw DataPorter::Components::Shared::StatusBadge.new(status: @import.status).call %>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div class="dp-import-details">
|
|
13
|
+
<dl class="dp-details-grid">
|
|
14
|
+
<dt>Target</dt>
|
|
15
|
+
<dd><%= @target._label %></dd>
|
|
16
|
+
<dt>Source</dt>
|
|
17
|
+
<dd><%= @import.source_type.upcase %></dd>
|
|
18
|
+
<% if @import.file.attached? %>
|
|
19
|
+
<dt>File</dt>
|
|
20
|
+
<dd><%= @import.file.filename %></dd>
|
|
21
|
+
<% end %>
|
|
22
|
+
<dt>Created</dt>
|
|
23
|
+
<dd><%= @import.created_at&.strftime("%Y-%m-%d %H:%M") %></dd>
|
|
24
|
+
<% if @import.report.records_count.positive? %>
|
|
25
|
+
<dt>Records</dt>
|
|
26
|
+
<dd><%= @import.report.records_count %></dd>
|
|
27
|
+
<% end %>
|
|
28
|
+
</dl>
|
|
9
29
|
</div>
|
|
10
30
|
|
|
11
|
-
<% if @import.parsing? || @import.importing? || @import.dry_running? %>
|
|
12
|
-
<%= raw DataPorter::Components::
|
|
31
|
+
<% if @import.parsing? || @import.importing? || @import.dry_running? || @import.extracting_headers? %>
|
|
32
|
+
<%= raw DataPorter::Components::Progress::Bar.new(import_id: @import.id).call %>
|
|
33
|
+
<% end %>
|
|
34
|
+
|
|
35
|
+
<% if @import.mapping? %>
|
|
36
|
+
<%= raw DataPorter::Components::Mapping::Form.new(
|
|
37
|
+
import: @import,
|
|
38
|
+
action_url: update_mapping_import_path(@import),
|
|
39
|
+
csrf_token: form_authenticity_token,
|
|
40
|
+
file_headers: @file_headers,
|
|
41
|
+
target_columns: @target_columns,
|
|
42
|
+
templates: @templates,
|
|
43
|
+
default_mapping: @default_mapping
|
|
44
|
+
).call %>
|
|
13
45
|
<% end %>
|
|
14
46
|
|
|
15
47
|
<% if @import.previewing? %>
|
|
16
|
-
<%= raw DataPorter::Components::SummaryCards.new(report: @import.report).call %>
|
|
17
|
-
<%= raw DataPorter::Components::
|
|
48
|
+
<%= raw DataPorter::Components::Preview::SummaryCards.new(report: @import.report).call %>
|
|
49
|
+
<%= raw DataPorter::Components::Preview::Table.new(
|
|
18
50
|
columns: @target._columns,
|
|
19
51
|
records: @records
|
|
20
52
|
).call %>
|
|
@@ -32,18 +64,14 @@
|
|
|
32
64
|
<% end %>
|
|
33
65
|
|
|
34
66
|
<% if @import.completed? %>
|
|
35
|
-
<%= raw DataPorter::Components::ResultsSummary.new(report: @import.report).call %>
|
|
67
|
+
<%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report).call %>
|
|
36
68
|
<% end %>
|
|
37
69
|
|
|
38
70
|
<% if @import.failed? %>
|
|
39
|
-
<%= raw DataPorter::Components::FailureAlert.new(report: @import.report).call %>
|
|
71
|
+
<%= raw DataPorter::Components::Shared::FailureAlert.new(report: @import.report).call %>
|
|
40
72
|
<div class="dp-actions">
|
|
41
73
|
<%= button_to "Retry", parse_import_path(@import),
|
|
42
74
|
method: :post, class: "dp-btn dp-btn--primary" %>
|
|
43
75
|
</div>
|
|
44
76
|
<% end %>
|
|
45
|
-
|
|
46
|
-
<div class="dp-nav">
|
|
47
|
-
<%= link_to "Back to imports", imports_path, class: "dp-link" %>
|
|
48
|
-
</div>
|
|
49
77
|
</div>
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
<%= form_with model: template, url: template.persisted? ? mapping_template_path(template) : mapping_templates_path, class: "dp-form", method: template.persisted? ? :patch : :post, data: { controller: "data-porter--template-form", "data-porter--template-form-columns-value": target_columns_map.to_json } do |f| %>
|
|
2
|
+
<div class="dp-field">
|
|
3
|
+
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
4
|
+
<%= f.select :target_key,
|
|
5
|
+
targets.map { |t| [t[:label], t[:key]] },
|
|
6
|
+
{ prompt: "Select a target..." },
|
|
7
|
+
class: "dp-select",
|
|
8
|
+
data: { action: "change->data-porter--template-form#targetChanged" } %>
|
|
9
|
+
</div>
|
|
10
|
+
|
|
11
|
+
<div class="dp-field">
|
|
12
|
+
<%= f.label :name, "Template Name", class: "dp-label" %>
|
|
13
|
+
<%= f.text_field :name, class: "dp-select", placeholder: "e.g. French Headers" %>
|
|
14
|
+
</div>
|
|
15
|
+
|
|
16
|
+
<div class="dp-field">
|
|
17
|
+
<label class="dp-label">Column Mappings</label>
|
|
18
|
+
<div id="dp-mapping-pairs" data-data-porter--template-form-target="pairsContainer">
|
|
19
|
+
<% (template.mapping.presence || { "" => "" }).each do |header, field| %>
|
|
20
|
+
<div class="dp-mapping-pair" style="display: flex; gap: 0.5rem; margin-bottom: 0.5rem;">
|
|
21
|
+
<input type="text" name="mapping_template[mapping_keys][]" value="<%= header %>" placeholder="File header" class="dp-select" style="flex: 1;" />
|
|
22
|
+
<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">
|
|
23
|
+
<option value="">Select a field...</option>
|
|
24
|
+
<% if template.target_key.present? && target_columns_map[template.target_key.to_s].present? %>
|
|
25
|
+
<% target_columns_map[template.target_key.to_s].each do |label, name| %>
|
|
26
|
+
<option value="<%= name %>" <%= "selected" if name == field.to_s %>><%= label %></option>
|
|
27
|
+
<% end %>
|
|
28
|
+
<% end %>
|
|
29
|
+
</select>
|
|
30
|
+
</div>
|
|
31
|
+
<% end %>
|
|
32
|
+
</div>
|
|
33
|
+
<button type="button" class="dp-btn dp-btn--secondary" style="margin-top: 0.5rem;" data-action="data-porter--template-form#addPair">+ Add Mapping</button>
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
<div class="dp-actions">
|
|
37
|
+
<%= f.submit template.persisted? ? "Update Template" : "Create Template", class: "dp-btn dp-btn--primary" %>
|
|
38
|
+
<%= link_to "Cancel", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
|
|
39
|
+
</div>
|
|
40
|
+
<% end %>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<div class="data-porter">
|
|
2
|
+
<div class="dp-header">
|
|
3
|
+
<h1 class="dp-title">Edit Mapping Template</h1>
|
|
4
|
+
</div>
|
|
5
|
+
|
|
6
|
+
<%= render "form", template: @template, targets: @targets, target_columns_map: @target_columns_map %>
|
|
7
|
+
|
|
8
|
+
<div class="dp-nav">
|
|
9
|
+
<%= link_to "Back to templates", mapping_templates_path, class: "dp-link" %>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
<div class="data-porter">
|
|
2
|
+
<div class="dp-header">
|
|
3
|
+
<div class="dp-header__actions">
|
|
4
|
+
<%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--secondary" %>
|
|
5
|
+
</div>
|
|
6
|
+
<h1 class="dp-title">Mapping Templates</h1>
|
|
7
|
+
<%= link_to "New Template", new_mapping_template_path, class: "dp-btn dp-btn--primary" %>
|
|
8
|
+
</div>
|
|
9
|
+
|
|
10
|
+
<% if @grouped.any? %>
|
|
11
|
+
<% @grouped.each do |target_key, templates| %>
|
|
12
|
+
<h2 style="font-size: 1.125rem; font-weight: 600; margin: 1.5rem 0 0.75rem; color: var(--dp-gray-700);"><%= target_key.titleize %></h2>
|
|
13
|
+
<table class="dp-table">
|
|
14
|
+
<thead>
|
|
15
|
+
<tr>
|
|
16
|
+
<th>Name</th>
|
|
17
|
+
<th>Mappings</th>
|
|
18
|
+
<th></th>
|
|
19
|
+
</tr>
|
|
20
|
+
</thead>
|
|
21
|
+
<tbody>
|
|
22
|
+
<% templates.each do |template| %>
|
|
23
|
+
<tr>
|
|
24
|
+
<td><%= template.name %></td>
|
|
25
|
+
<td><%= template.mapping.size %> columns</td>
|
|
26
|
+
<td style="display: flex; gap: 0.5rem; align-items: center;">
|
|
27
|
+
<%= link_to "Edit", edit_mapping_template_path(template), class: "dp-btn dp-btn--secondary" %>
|
|
28
|
+
<%= button_to "Delete", mapping_template_path(template), method: :delete, class: "dp-btn dp-btn--danger" %>
|
|
29
|
+
</td>
|
|
30
|
+
</tr>
|
|
31
|
+
<% end %>
|
|
32
|
+
</tbody>
|
|
33
|
+
</table>
|
|
34
|
+
<% end %>
|
|
35
|
+
<% else %>
|
|
36
|
+
<div class="dp-empty-state">
|
|
37
|
+
<div class="dp-empty-state__icon">📋</div>
|
|
38
|
+
<p class="dp-empty-state__text">No mapping templates yet</p>
|
|
39
|
+
<%= link_to "Create your first template", new_mapping_template_path, class: "dp-btn dp-btn--primary" %>
|
|
40
|
+
</div>
|
|
41
|
+
<% end %>
|
|
42
|
+
</div>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<div class="data-porter">
|
|
2
|
+
<div class="dp-header">
|
|
3
|
+
<h1 class="dp-title">New Mapping Template</h1>
|
|
4
|
+
</div>
|
|
5
|
+
|
|
6
|
+
<%= render "form", template: @template, targets: @targets, target_columns_map: @target_columns_map %>
|
|
7
|
+
|
|
8
|
+
<div class="dp-nav">
|
|
9
|
+
<%= link_to "Back to templates", mapping_templates_path, class: "dp-link" %>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
<!DOCTYPE html>
|
|
2
|
+
<html>
|
|
3
|
+
<head>
|
|
4
|
+
<meta charset="UTF-8">
|
|
5
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
|
+
<title>DataPorter</title>
|
|
7
|
+
<%= csrf_meta_tags %>
|
|
8
|
+
<%= stylesheet_link_tag "data_porter/application" %>
|
|
9
|
+
<script type="importmap">
|
|
10
|
+
{
|
|
11
|
+
"imports": {
|
|
12
|
+
"@hotwired/turbo": "<%= asset_path('data_porter/turbo.min.js') %>",
|
|
13
|
+
"@hotwired/stimulus": "<%= asset_path('data_porter/stimulus.min.js') %>"
|
|
14
|
+
}
|
|
15
|
+
}
|
|
16
|
+
</script>
|
|
17
|
+
</head>
|
|
18
|
+
<body>
|
|
19
|
+
<%= yield %>
|
|
20
|
+
|
|
21
|
+
<script type="module">
|
|
22
|
+
import "@hotwired/turbo"
|
|
23
|
+
import { Application, Controller } from "@hotwired/stimulus"
|
|
24
|
+
|
|
25
|
+
const application = Application.start()
|
|
26
|
+
|
|
27
|
+
application.register("data-porter--mapping", class extends Controller {
|
|
28
|
+
static targets = ["columnSelect", "requiredWarning", "duplicateWarning", "saveTemplate"]
|
|
29
|
+
static values = { requiredColumns: Array }
|
|
30
|
+
|
|
31
|
+
connect() { this.validate() }
|
|
32
|
+
|
|
33
|
+
loadTemplate(event) {
|
|
34
|
+
const option = event.target.selectedOptions[0]
|
|
35
|
+
if (!option || !option.dataset.mapping) return
|
|
36
|
+
const mapping = JSON.parse(option.dataset.mapping)
|
|
37
|
+
this.columnSelectTargets.forEach(select => {
|
|
38
|
+
const header = select.name.match(/\[(.+)\]/)?.[1]
|
|
39
|
+
select.value = (header && mapping[header]) ? mapping[header] : ""
|
|
40
|
+
})
|
|
41
|
+
if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
|
|
42
|
+
this.validate()
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
onChange() { this.validate() }
|
|
46
|
+
|
|
47
|
+
validate() {
|
|
48
|
+
this.validateRequired()
|
|
49
|
+
this.validateDuplicates()
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
validateRequired() {
|
|
53
|
+
if (!this.hasRequiredWarningTarget) return
|
|
54
|
+
const selected = new Set(this.columnSelectTargets.map(s => s.value).filter(v => v !== ""))
|
|
55
|
+
const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
|
|
56
|
+
if (missing.length > 0) {
|
|
57
|
+
this.requiredWarningTarget.textContent = "Required fields not mapped: " + missing.map(c => c.label).join(", ")
|
|
58
|
+
this.requiredWarningTarget.style.display = ""
|
|
59
|
+
} else {
|
|
60
|
+
this.requiredWarningTarget.style.display = "none"
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
validateDuplicates() {
|
|
65
|
+
const counts = {}
|
|
66
|
+
this.columnSelectTargets.forEach(select => {
|
|
67
|
+
if (select.value === "") return
|
|
68
|
+
counts[select.value] = (counts[select.value] || 0) + 1
|
|
69
|
+
})
|
|
70
|
+
const duplicates = new Set(Object.keys(counts).filter(k => counts[k] > 1))
|
|
71
|
+
this.columnSelectTargets.forEach(select => {
|
|
72
|
+
const row = select.closest(".dp-mapping-row")
|
|
73
|
+
if (!row) return
|
|
74
|
+
row.classList.toggle("dp-mapping-row--duplicate", select.value !== "" && duplicates.has(select.value))
|
|
75
|
+
})
|
|
76
|
+
if (this.hasDuplicateWarningTarget) {
|
|
77
|
+
if (duplicates.size > 0) {
|
|
78
|
+
this.duplicateWarningTarget.textContent = "Duplicate mappings detected for: " + [...duplicates].join(", ")
|
|
79
|
+
this.duplicateWarningTarget.style.display = ""
|
|
80
|
+
} else {
|
|
81
|
+
this.duplicateWarningTarget.style.display = "none"
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
application.register("data-porter--template-form", class extends Controller {
|
|
88
|
+
static targets = ["pairsContainer", "fieldSelect"]
|
|
89
|
+
static values = { columns: Object }
|
|
90
|
+
|
|
91
|
+
targetChanged(event) {
|
|
92
|
+
const columns = this.columnsValue[event.target.value] || []
|
|
93
|
+
this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
addPair() {
|
|
97
|
+
const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
|
|
98
|
+
const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
|
|
99
|
+
const pair = document.createElement("div")
|
|
100
|
+
pair.className = "dp-mapping-pair"
|
|
101
|
+
pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
|
|
102
|
+
pair.innerHTML = this.pairHTML(columns)
|
|
103
|
+
this.pairsContainerTarget.appendChild(pair)
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
updateOptions(select, columns) {
|
|
107
|
+
const current = select.value
|
|
108
|
+
select.innerHTML = '<option value="">Select a field...</option>'
|
|
109
|
+
columns.forEach(([label, name]) => {
|
|
110
|
+
const opt = document.createElement("option")
|
|
111
|
+
opt.value = name
|
|
112
|
+
opt.textContent = label
|
|
113
|
+
if (name === current) opt.selected = true
|
|
114
|
+
select.appendChild(opt)
|
|
115
|
+
})
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
pairHTML(columns) {
|
|
119
|
+
const options = columns.map(([label, name]) => '<option value="' + name + '">' + label + "</option>").join("")
|
|
120
|
+
return '<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />' +
|
|
121
|
+
'<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">' +
|
|
122
|
+
'<option value="">Select a field...</option>' + options + "</select>"
|
|
123
|
+
}
|
|
124
|
+
})
|
|
125
|
+
|
|
126
|
+
application.register("data-porter--progress", class extends Controller {
|
|
127
|
+
static targets = ["bar", "text"]
|
|
128
|
+
static values = { id: Number }
|
|
129
|
+
|
|
130
|
+
async connect() {
|
|
131
|
+
try {
|
|
132
|
+
const { createConsumer } = await import("@rails/actioncable")
|
|
133
|
+
this.subscription = createConsumer().subscriptions.create(
|
|
134
|
+
{ channel: "DataPorter::ImportChannel", id: this.idValue },
|
|
135
|
+
{
|
|
136
|
+
received: (data) => {
|
|
137
|
+
if (data.status === "processing") {
|
|
138
|
+
this.barTarget.style.width = data.percentage + "%"
|
|
139
|
+
this.textTarget.textContent = data.percentage + "%"
|
|
140
|
+
} else {
|
|
141
|
+
Turbo.visit(window.location.href)
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
)
|
|
146
|
+
} catch (e) {
|
|
147
|
+
this.pollForChanges()
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
pollForChanges() {
|
|
152
|
+
this.timer = setInterval(() => Turbo.visit(window.location.href), 3000)
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
disconnect() {
|
|
156
|
+
if (this.subscription) this.subscription.unsubscribe()
|
|
157
|
+
if (this.timer) clearInterval(this.timer)
|
|
158
|
+
}
|
|
159
|
+
})
|
|
160
|
+
</script>
|
|
161
|
+
</body>
|
|
162
|
+
</html>
|