data_porter 0.2.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +73 -0
- data/README.md +60 -393
- data/ROADMAP.md +30 -12
- data/app/assets/javascripts/data_porter/stimulus.min.js +2 -0
- data/app/assets/javascripts/data_porter/turbo.min.js +29 -0
- data/app/assets/stylesheets/data_porter/alerts.css +83 -0
- data/app/assets/stylesheets/data_porter/application.css +12 -646
- data/app/assets/stylesheets/data_porter/badges.css +73 -0
- data/app/assets/stylesheets/data_porter/base.css +56 -0
- data/app/assets/stylesheets/data_porter/cards.css +60 -0
- data/app/assets/stylesheets/data_porter/layout.css +147 -0
- data/app/assets/stylesheets/data_porter/mapping.css +79 -0
- data/app/assets/stylesheets/data_porter/modal.css +49 -0
- data/app/assets/stylesheets/data_porter/preview.css +24 -0
- data/app/assets/stylesheets/data_porter/progress.css +62 -0
- data/app/assets/stylesheets/data_porter/table.css +51 -0
- data/app/controllers/data_porter/imports_controller.rb +96 -10
- data/app/controllers/data_porter/mapping_templates_controller.rb +85 -0
- data/app/javascript/data_porter/mapping_controller.js +86 -0
- data/app/javascript/data_porter/progress_controller.js +29 -20
- data/app/javascript/data_porter/template_form_controller.js +46 -0
- data/app/jobs/data_porter/extract_headers_job.rb +12 -0
- data/app/models/data_porter/data_import.rb +12 -1
- data/app/models/data_porter/mapping_template.rb +15 -0
- data/app/views/data_porter/imports/index.html.erb +38 -9
- data/app/views/data_porter/imports/new.html.erb +31 -4
- data/app/views/data_porter/imports/show.html.erb +74 -17
- data/app/views/data_porter/mapping_templates/_form.html.erb +40 -0
- data/app/views/data_porter/mapping_templates/edit.html.erb +11 -0
- data/app/views/data_porter/mapping_templates/index.html.erb +42 -0
- data/app/views/data_porter/mapping_templates/new.html.erb +11 -0
- data/app/views/layouts/data_porter/application.html.erb +168 -0
- data/config/routes.rb +5 -1
- data/docs/CONFIGURATION.md +81 -0
- data/docs/MAPPING.md +44 -0
- data/docs/SOURCES.md +94 -0
- data/docs/TARGETS.md +176 -0
- data/docs/screenshots/mapping.jpg +0 -0
- data/lib/data_porter/components/mapping/column_row.rb +52 -0
- data/lib/data_porter/components/mapping/form.rb +127 -0
- data/lib/data_porter/components/mapping/template_select.rb +35 -0
- data/lib/data_porter/components/preview/results_summary.rb +60 -0
- data/lib/data_porter/components/preview/summary_cards.rb +32 -0
- data/lib/data_porter/components/preview/table.rb +56 -0
- data/lib/data_porter/components/progress/bar.rb +47 -0
- data/lib/data_porter/components/shared/failure_alert.rb +22 -0
- data/lib/data_porter/components/shared/status_badge.rb +18 -0
- data/lib/data_porter/components.rb +9 -6
- data/lib/data_porter/configuration.rb +3 -1
- data/lib/data_porter/engine.rb +7 -1
- data/lib/data_porter/orchestrator.rb +35 -2
- data/lib/data_porter/registry.rb +6 -1
- data/lib/data_porter/sources/base.rb +18 -3
- data/lib/data_porter/sources/csv.rb +5 -0
- data/lib/data_porter/sources/xlsx.rb +8 -0
- data/lib/data_porter/version.rb +1 -1
- data/lib/generators/data_porter/install/install_generator.rb +4 -0
- data/lib/generators/data_porter/install/templates/create_data_porter_mapping_templates.rb.erb +16 -0
- data/lib/generators/data_porter/install/templates/initializer.rb +5 -1
- data/lib/generators/data_porter/target/target_generator.rb +5 -0
- data/lib/generators/data_porter/target/templates/target.rb.tt +1 -1
- data/lib/tasks/data_porter.rake +9 -0
- metadata +62 -39
- data/lib/data_porter/components/failure_alert.rb +0 -20
- data/lib/data_porter/components/preview_table.rb +0 -54
- data/lib/data_porter/components/progress_bar.rb +0 -33
- data/lib/data_porter/components/results_summary.rb +0 -19
- data/lib/data_porter/components/status_badge.rb +0 -16
- data/lib/data_porter/components/summary_cards.rb +0 -30
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DataPorter
|
|
4
|
+
class MappingTemplatesController < DataPorter.configuration.parent_controller.constantize
|
|
5
|
+
layout "data_porter/application"
|
|
6
|
+
|
|
7
|
+
before_action :set_template, only: %i[edit update destroy]
|
|
8
|
+
|
|
9
|
+
def index
|
|
10
|
+
@templates = MappingTemplate.order(:target_key, :name)
|
|
11
|
+
@grouped = @templates.group_by(&:target_key)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def new
|
|
15
|
+
@template = MappingTemplate.new
|
|
16
|
+
load_form_data
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def create
|
|
20
|
+
@template = MappingTemplate.new(template_params)
|
|
21
|
+
|
|
22
|
+
if @template.save
|
|
23
|
+
redirect_to mapping_templates_path
|
|
24
|
+
else
|
|
25
|
+
load_form_data
|
|
26
|
+
render :new, status: :unprocessable_entity
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def edit
|
|
31
|
+
load_form_data
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def update
|
|
35
|
+
if @template.update(template_params)
|
|
36
|
+
redirect_to mapping_templates_path
|
|
37
|
+
else
|
|
38
|
+
load_form_data
|
|
39
|
+
render :edit, status: :unprocessable_entity
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def destroy
|
|
44
|
+
@template.destroy
|
|
45
|
+
redirect_to mapping_templates_path
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def set_template
|
|
51
|
+
@template = MappingTemplate.find(params[:id])
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def template_params
|
|
55
|
+
permitted = params.require(:mapping_template).permit(
|
|
56
|
+
:target_key, :name, mapping: {}, mapping_keys: [], mapping_values: []
|
|
57
|
+
)
|
|
58
|
+
build_mapping_from_arrays(permitted)
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def build_mapping_from_arrays(raw)
|
|
62
|
+
keys = raw.delete(:mapping_keys)
|
|
63
|
+
values = raw.delete(:mapping_values)
|
|
64
|
+
return raw unless keys && values
|
|
65
|
+
|
|
66
|
+
mapping = keys.zip(values).reject { |k, _| k.blank? }.to_h
|
|
67
|
+
raw.merge(mapping: mapping)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def load_form_data
|
|
71
|
+
@targets = Registry.available
|
|
72
|
+
@target_columns_map = build_target_columns_map
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def build_target_columns_map
|
|
76
|
+
columns = {}
|
|
77
|
+
@targets.each do |t|
|
|
78
|
+
target = Registry.find(t[:key])
|
|
79
|
+
cols = target._columns || []
|
|
80
|
+
columns[t[:key].to_s] = cols.map { |c| [c.label, c.name.to_s] }
|
|
81
|
+
end
|
|
82
|
+
columns
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { Controller } from "@hotwired/stimulus"
|
|
2
|
+
|
|
3
|
+
export default class extends Controller {
|
|
4
|
+
static targets = ["columnSelect", "requiredWarning", "duplicateWarning", "saveTemplate"]
|
|
5
|
+
static values = { requiredColumns: Array }
|
|
6
|
+
|
|
7
|
+
connect() {
|
|
8
|
+
this.validate()
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
loadTemplate(event) {
|
|
12
|
+
const option = event.target.selectedOptions[0]
|
|
13
|
+
if (!option || !option.dataset.mapping) return
|
|
14
|
+
|
|
15
|
+
const mapping = JSON.parse(option.dataset.mapping)
|
|
16
|
+
this.columnSelectTargets.forEach(select => {
|
|
17
|
+
const header = select.name.match(/\[(.+)\]/)?.[1]
|
|
18
|
+
if (header && mapping[header]) {
|
|
19
|
+
select.value = mapping[header]
|
|
20
|
+
} else {
|
|
21
|
+
select.value = ""
|
|
22
|
+
}
|
|
23
|
+
})
|
|
24
|
+
if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
|
|
25
|
+
this.validate()
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
onChange() {
|
|
29
|
+
this.validate()
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
validate() {
|
|
33
|
+
this.validateRequired()
|
|
34
|
+
this.validateDuplicates()
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
validateRequired() {
|
|
38
|
+
if (!this.hasRequiredWarningTarget) return
|
|
39
|
+
|
|
40
|
+
const selected = new Set(
|
|
41
|
+
this.columnSelectTargets.map(s => s.value).filter(v => v !== "")
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
|
|
45
|
+
|
|
46
|
+
if (missing.length > 0) {
|
|
47
|
+
const names = missing.map(c => c.label).join(", ")
|
|
48
|
+
this.requiredWarningTarget.textContent = `Required fields not mapped: ${names}`
|
|
49
|
+
this.requiredWarningTarget.style.display = ""
|
|
50
|
+
} else {
|
|
51
|
+
this.requiredWarningTarget.style.display = "none"
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
validateDuplicates() {
|
|
56
|
+
const counts = {}
|
|
57
|
+
this.columnSelectTargets.forEach(select => {
|
|
58
|
+
if (select.value === "") return
|
|
59
|
+
counts[select.value] = (counts[select.value] || 0) + 1
|
|
60
|
+
})
|
|
61
|
+
|
|
62
|
+
const duplicates = new Set(
|
|
63
|
+
Object.keys(counts).filter(k => counts[k] > 1)
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
this.columnSelectTargets.forEach(select => {
|
|
67
|
+
const row = select.closest(".dp-mapping-row")
|
|
68
|
+
if (!row) return
|
|
69
|
+
|
|
70
|
+
if (select.value !== "" && duplicates.has(select.value)) {
|
|
71
|
+
row.classList.add("dp-mapping-row--duplicate")
|
|
72
|
+
} else {
|
|
73
|
+
row.classList.remove("dp-mapping-row--duplicate")
|
|
74
|
+
}
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
if (this.hasDuplicateWarningTarget) {
|
|
78
|
+
if (duplicates.size > 0) {
|
|
79
|
+
this.duplicateWarningTarget.textContent = `Duplicate mappings detected for: ${[...duplicates].join(", ")}`
|
|
80
|
+
this.duplicateWarningTarget.style.display = ""
|
|
81
|
+
} else {
|
|
82
|
+
this.duplicateWarningTarget.style.display = "none"
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -1,33 +1,42 @@
|
|
|
1
1
|
import { Controller } from "@hotwired/stimulus"
|
|
2
|
-
import { createConsumer } from "@rails/actioncable"
|
|
3
2
|
|
|
4
3
|
export default class extends Controller {
|
|
5
|
-
static targets = ["bar", "text"]
|
|
6
|
-
static values = { id: Number }
|
|
4
|
+
static targets = ["bar", "text", "label"]
|
|
5
|
+
static values = { id: Number, url: String }
|
|
7
6
|
|
|
8
7
|
connect() {
|
|
9
|
-
this.
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
8
|
+
this.poll()
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
poll() {
|
|
12
|
+
this.timer = setInterval(() => this.fetchStatus(), 1000)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
async fetchStatus() {
|
|
16
|
+
try {
|
|
17
|
+
const response = await fetch(this.urlValue)
|
|
18
|
+
const data = await response.json()
|
|
19
|
+
this.updateLabel(data.status)
|
|
20
|
+
if (data.progress && data.progress.percentage !== undefined) {
|
|
21
|
+
this.barTarget.style.width = data.progress.percentage + "%"
|
|
22
|
+
this.textTarget.textContent = data.progress.percentage + "%"
|
|
23
|
+
}
|
|
24
|
+
if (!["pending", "importing", "parsing", "dry_running", "extracting_headers"].includes(data.status)) {
|
|
25
|
+
clearInterval(this.timer)
|
|
26
|
+
this.barTarget.style.width = "100%"
|
|
27
|
+
this.textTarget.textContent = "100%"
|
|
28
|
+
setTimeout(() => Turbo.visit(window.location.href, { action: "replace" }), 500)
|
|
19
29
|
}
|
|
20
|
-
)
|
|
30
|
+
} catch (e) {}
|
|
21
31
|
}
|
|
22
32
|
|
|
23
|
-
|
|
24
|
-
if (this.
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
}
|
|
33
|
+
updateLabel(status) {
|
|
34
|
+
if (!this.hasLabelTarget) return
|
|
35
|
+
var labels = { pending: "Waiting...", extracting_headers: "Extracting headers...", parsing: "Parsing records...", importing: "Importing...", dry_running: "Dry run..." }
|
|
36
|
+
this.labelTarget.textContent = labels[status] || "Processing..."
|
|
28
37
|
}
|
|
29
38
|
|
|
30
39
|
disconnect() {
|
|
31
|
-
this.
|
|
40
|
+
if (this.timer) clearInterval(this.timer)
|
|
32
41
|
}
|
|
33
42
|
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import { Controller } from "@hotwired/stimulus"
|
|
2
|
+
|
|
3
|
+
export default class extends Controller {
|
|
4
|
+
static targets = ["pairsContainer", "fieldSelect"]
|
|
5
|
+
static values = { columns: Object }
|
|
6
|
+
|
|
7
|
+
targetChanged(event) {
|
|
8
|
+
const targetKey = event.target.value
|
|
9
|
+
const columns = this.columnsValue[targetKey] || []
|
|
10
|
+
this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
addPair() {
|
|
14
|
+
const container = this.pairsContainerTarget
|
|
15
|
+
const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
|
|
16
|
+
const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
|
|
17
|
+
|
|
18
|
+
const pair = document.createElement("div")
|
|
19
|
+
pair.className = "dp-mapping-pair"
|
|
20
|
+
pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
|
|
21
|
+
pair.innerHTML = this.pairHTML(columns)
|
|
22
|
+
container.appendChild(pair)
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
updateOptions(select, columns) {
|
|
26
|
+
const current = select.value
|
|
27
|
+
select.innerHTML = '<option value="">Select a field...</option>'
|
|
28
|
+
columns.forEach(([label, name]) => {
|
|
29
|
+
const opt = document.createElement("option")
|
|
30
|
+
opt.value = name
|
|
31
|
+
opt.textContent = label
|
|
32
|
+
if (name === current) opt.selected = true
|
|
33
|
+
select.appendChild(opt)
|
|
34
|
+
})
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
pairHTML(columns) {
|
|
38
|
+
const options = columns.map(([label, name]) =>
|
|
39
|
+
`<option value="${name}">${label}</option>`
|
|
40
|
+
).join("")
|
|
41
|
+
|
|
42
|
+
return `<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />` +
|
|
43
|
+
`<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">` +
|
|
44
|
+
`<option value="">Select a field...</option>${options}</select>`
|
|
45
|
+
}
|
|
46
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DataPorter
|
|
4
|
+
class ExtractHeadersJob < ActiveJob::Base
|
|
5
|
+
queue_as { DataPorter.configuration.queue_name }
|
|
6
|
+
|
|
7
|
+
def perform(import_id)
|
|
8
|
+
data_import = DataImport.find(import_id)
|
|
9
|
+
Orchestrator.new(data_import).extract_headers!
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
@@ -15,7 +15,9 @@ module DataPorter
|
|
|
15
15
|
importing: 3,
|
|
16
16
|
completed: 4,
|
|
17
17
|
failed: 5,
|
|
18
|
-
dry_running: 6
|
|
18
|
+
dry_running: 6,
|
|
19
|
+
extracting_headers: 7,
|
|
20
|
+
mapping: 8
|
|
19
21
|
}
|
|
20
22
|
|
|
21
23
|
attribute :records, StoreModels::ImportRecord.to_array_type, default: -> { [] }
|
|
@@ -23,6 +25,11 @@ module DataPorter
|
|
|
23
25
|
|
|
24
26
|
attribute :config, :json, default: -> { {} }
|
|
25
27
|
|
|
28
|
+
scope :purgeable, lambda {
|
|
29
|
+
where(status: %i[completed failed])
|
|
30
|
+
.where(created_at: ...DataPorter.configuration.purge_after.ago)
|
|
31
|
+
}
|
|
32
|
+
|
|
26
33
|
validates :target_key, presence: true
|
|
27
34
|
validates :source_type, presence: true, inclusion: { in: %w[csv json api xlsx] }
|
|
28
35
|
|
|
@@ -45,5 +52,9 @@ module DataPorter
|
|
|
45
52
|
def records_summary
|
|
46
53
|
records.group_by(&:status).transform_values(&:count)
|
|
47
54
|
end
|
|
55
|
+
|
|
56
|
+
def file_based?
|
|
57
|
+
%w[csv xlsx].include?(source_type)
|
|
58
|
+
end
|
|
48
59
|
end
|
|
49
60
|
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module DataPorter
|
|
4
|
+
class MappingTemplate < ActiveRecord::Base
|
|
5
|
+
self.table_name = "data_porter_mapping_templates"
|
|
6
|
+
|
|
7
|
+
attribute :mapping, :json, default: -> { {} }
|
|
8
|
+
|
|
9
|
+
validates :target_key, presence: true
|
|
10
|
+
validates :name, presence: true, uniqueness: { scope: :target_key }
|
|
11
|
+
validates :mapping, presence: true
|
|
12
|
+
|
|
13
|
+
scope :for_target, ->(key) { where(target_key: key) }
|
|
14
|
+
end
|
|
15
|
+
end
|
|
@@ -1,11 +1,12 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
5
3
|
<h1 class="dp-title">Imports</h1>
|
|
6
|
-
<
|
|
7
|
-
|
|
8
|
-
|
|
4
|
+
<div class="dp-header__actions">
|
|
5
|
+
<%= link_to "Mapping Templates", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
|
|
6
|
+
<button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
|
|
7
|
+
New Import
|
|
8
|
+
</button>
|
|
9
|
+
</div>
|
|
9
10
|
</div>
|
|
10
11
|
|
|
11
12
|
<% if @imports.any? %>
|
|
@@ -26,9 +27,16 @@
|
|
|
26
27
|
<td><%= import.id %></td>
|
|
27
28
|
<td><%= import.target_key %></td>
|
|
28
29
|
<td><%= import.source_type %></td>
|
|
29
|
-
<td><%= raw DataPorter::Components::StatusBadge.new(status: import.status).call %></td>
|
|
30
|
+
<td><%= raw DataPorter::Components::Shared::StatusBadge.new(status: import.status).call %></td>
|
|
30
31
|
<td><%= import.created_at&.strftime("%Y-%m-%d %H:%M") %></td>
|
|
31
|
-
<td
|
|
32
|
+
<td class="dp-table__actions">
|
|
33
|
+
<%= link_to "View", import_path(import), class: "dp-btn dp-btn--sm dp-btn--secondary" %>
|
|
34
|
+
<% if import.completed? || import.failed? %>
|
|
35
|
+
<%= button_to "Delete", import_path(import),
|
|
36
|
+
method: :delete, class: "dp-btn dp-btn--sm dp-btn--danger",
|
|
37
|
+
data: { turbo_confirm: "Delete this import?" } %>
|
|
38
|
+
<% end %>
|
|
39
|
+
</td>
|
|
32
40
|
</tr>
|
|
33
41
|
<% end %>
|
|
34
42
|
</tbody>
|
|
@@ -52,13 +60,15 @@
|
|
|
52
60
|
<button type="button" class="dp-modal__close" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">×</button>
|
|
53
61
|
</div>
|
|
54
62
|
|
|
55
|
-
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true do |f| %>
|
|
63
|
+
<%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
|
|
56
64
|
<div class="dp-field">
|
|
57
65
|
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
58
66
|
<%= f.select :target_key,
|
|
59
67
|
@targets.map { |t| [t[:label], t[:key]] },
|
|
60
68
|
{ prompt: "Select a target..." },
|
|
61
|
-
|
|
69
|
+
id: "dp-target-select",
|
|
70
|
+
class: "dp-select",
|
|
71
|
+
data: { sources: @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json } %>
|
|
62
72
|
</div>
|
|
63
73
|
|
|
64
74
|
<div class="dp-field">
|
|
@@ -93,12 +103,31 @@
|
|
|
93
103
|
|
|
94
104
|
<script>
|
|
95
105
|
(function() {
|
|
106
|
+
var targetSelect = document.getElementById("dp-target-select");
|
|
96
107
|
var sourceSelect = document.getElementById("dp-source-select");
|
|
97
108
|
var fileField = document.getElementById("dp-file-field");
|
|
98
109
|
var fileInput = document.getElementById("dp-file-input");
|
|
99
110
|
var dropzone = document.getElementById("dp-dropzone");
|
|
100
111
|
var fileName = document.getElementById("dp-file-name");
|
|
101
112
|
|
|
113
|
+
function filterSources() {
|
|
114
|
+
if (!targetSelect || !sourceSelect) return;
|
|
115
|
+
var sourcesMap = JSON.parse(targetSelect.dataset.sources || "{}");
|
|
116
|
+
var allowed = sourcesMap[targetSelect.value];
|
|
117
|
+
var options = sourceSelect.options;
|
|
118
|
+
for (var i = 1; i < options.length; i++) {
|
|
119
|
+
options[i].style.display = allowed && allowed.indexOf(options[i].value) === -1 ? "none" : "";
|
|
120
|
+
}
|
|
121
|
+
if (allowed && sourceSelect.selectedIndex > 0 && allowed.indexOf(sourceSelect.value) === -1) {
|
|
122
|
+
sourceSelect.selectedIndex = 0;
|
|
123
|
+
fileField.style.display = "";
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (targetSelect) {
|
|
128
|
+
targetSelect.addEventListener("change", filterSources);
|
|
129
|
+
}
|
|
130
|
+
|
|
102
131
|
if (sourceSelect) {
|
|
103
132
|
sourceSelect.addEventListener("change", function() {
|
|
104
133
|
fileField.style.display = this.value === "api" ? "none" : "";
|
|
@@ -1,17 +1,25 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
5
3
|
<h1 class="dp-title">New Import</h1>
|
|
6
4
|
</div>
|
|
7
5
|
|
|
8
|
-
|
|
6
|
+
<% if @import.errors.any? %>
|
|
7
|
+
<div class="dp-alert dp-alert--danger">
|
|
8
|
+
<% @import.errors.full_messages.each do |msg| %>
|
|
9
|
+
<p><%= msg %></p>
|
|
10
|
+
<% end %>
|
|
11
|
+
</div>
|
|
12
|
+
<% end %>
|
|
13
|
+
|
|
14
|
+
<%= form_with model: @import, url: imports_path, class: "dp-form", multipart: true, data: { turbo: false } do |f| %>
|
|
9
15
|
<div class="dp-field">
|
|
10
16
|
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
11
17
|
<%= f.select :target_key,
|
|
12
18
|
@targets.map { |t| [t[:label], t[:key]] },
|
|
13
19
|
{ prompt: "Select a target..." },
|
|
14
|
-
|
|
20
|
+
id: "dp-target-select-new",
|
|
21
|
+
class: "dp-select",
|
|
22
|
+
data: { sources: @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json } %>
|
|
15
23
|
</div>
|
|
16
24
|
|
|
17
25
|
<div class="dp-field">
|
|
@@ -45,12 +53,31 @@
|
|
|
45
53
|
|
|
46
54
|
<script>
|
|
47
55
|
(function() {
|
|
56
|
+
var targetSelect = document.getElementById("dp-target-select-new");
|
|
48
57
|
var sourceSelect = document.getElementById("dp-source-select-new");
|
|
49
58
|
var fileField = document.getElementById("dp-file-field-new");
|
|
50
59
|
var fileInput = document.getElementById("dp-file-input-new");
|
|
51
60
|
var dropzone = document.getElementById("dp-dropzone-new");
|
|
52
61
|
var fileName = document.getElementById("dp-file-name-new");
|
|
53
62
|
|
|
63
|
+
function filterSources() {
|
|
64
|
+
if (!targetSelect || !sourceSelect) return;
|
|
65
|
+
var sourcesMap = JSON.parse(targetSelect.dataset.sources || "{}");
|
|
66
|
+
var allowed = sourcesMap[targetSelect.value];
|
|
67
|
+
var options = sourceSelect.options;
|
|
68
|
+
for (var i = 1; i < options.length; i++) {
|
|
69
|
+
options[i].style.display = allowed && allowed.indexOf(options[i].value) === -1 ? "none" : "";
|
|
70
|
+
}
|
|
71
|
+
if (allowed && sourceSelect.selectedIndex > 0 && allowed.indexOf(sourceSelect.value) === -1) {
|
|
72
|
+
sourceSelect.selectedIndex = 0;
|
|
73
|
+
fileField.style.display = "";
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
if (targetSelect) {
|
|
78
|
+
targetSelect.addEventListener("change", filterSources);
|
|
79
|
+
}
|
|
80
|
+
|
|
54
81
|
if (sourceSelect) {
|
|
55
82
|
sourceSelect.addEventListener("change", function() {
|
|
56
83
|
fileField.style.display = this.value === "api" ? "none" : "";
|
|
@@ -1,30 +1,64 @@
|
|
|
1
|
-
<%= stylesheet_link_tag "data_porter/application" %>
|
|
2
|
-
|
|
3
1
|
<div class="data-porter">
|
|
4
2
|
<div class="dp-header">
|
|
3
|
+
<div class="dp-header__actions">
|
|
4
|
+
<%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--secondary" %>
|
|
5
|
+
</div>
|
|
5
6
|
<h1 class="dp-title">
|
|
6
7
|
<%= @target._label %> Import #<%= @import.id %>
|
|
7
8
|
</h1>
|
|
8
|
-
<%= raw DataPorter::Components::StatusBadge.new(status: @import.status).call %>
|
|
9
|
+
<%= raw DataPorter::Components::Shared::StatusBadge.new(status: @import.status).call %>
|
|
10
|
+
</div>
|
|
11
|
+
|
|
12
|
+
<div class="dp-import-details">
|
|
13
|
+
<dl class="dp-details-grid">
|
|
14
|
+
<dt>Target</dt>
|
|
15
|
+
<dd><%= @target._label %></dd>
|
|
16
|
+
<dt>Source</dt>
|
|
17
|
+
<dd><%= @import.source_type.upcase %></dd>
|
|
18
|
+
<% if @import.file.attached? %>
|
|
19
|
+
<dt>File</dt>
|
|
20
|
+
<dd><%= @import.file.filename %></dd>
|
|
21
|
+
<% end %>
|
|
22
|
+
<dt>Created</dt>
|
|
23
|
+
<dd><%= @import.created_at&.strftime("%Y-%m-%d %H:%M") %></dd>
|
|
24
|
+
<% if @import.report.records_count.positive? %>
|
|
25
|
+
<dt>Records</dt>
|
|
26
|
+
<dd><%= @import.report.records_count %></dd>
|
|
27
|
+
<% end %>
|
|
28
|
+
</dl>
|
|
9
29
|
</div>
|
|
10
30
|
|
|
11
|
-
<% if @import.parsing? || @import.importing? || @import.dry_running? %>
|
|
12
|
-
<%= raw DataPorter::Components::
|
|
31
|
+
<% if @import.pending? || @import.parsing? || @import.importing? || @import.dry_running? || @import.extracting_headers? %>
|
|
32
|
+
<%= raw DataPorter::Components::Progress::Bar.new(import_id: @import.id, status_url: status_import_path(@import)).call %>
|
|
33
|
+
<% end %>
|
|
34
|
+
|
|
35
|
+
<% if @import.mapping? %>
|
|
36
|
+
<%= raw DataPorter::Components::Mapping::Form.new(
|
|
37
|
+
import: @import,
|
|
38
|
+
action_url: update_mapping_import_path(@import),
|
|
39
|
+
csrf_token: form_authenticity_token,
|
|
40
|
+
file_headers: @file_headers,
|
|
41
|
+
target_columns: @target_columns,
|
|
42
|
+
templates: @templates,
|
|
43
|
+
default_mapping: @default_mapping
|
|
44
|
+
).call %>
|
|
13
45
|
<% end %>
|
|
14
46
|
|
|
15
47
|
<% if @import.previewing? %>
|
|
16
|
-
<%= raw DataPorter::Components::SummaryCards.new(report: @import.report).call %>
|
|
17
|
-
<%= raw DataPorter::Components::
|
|
48
|
+
<%= raw DataPorter::Components::Preview::SummaryCards.new(report: @import.report).call %>
|
|
49
|
+
<%= raw DataPorter::Components::Preview::Table.new(
|
|
18
50
|
columns: @target._columns,
|
|
19
51
|
records: @records
|
|
20
52
|
).call %>
|
|
21
53
|
|
|
22
54
|
<div class="dp-actions">
|
|
23
|
-
<%= button_to
|
|
24
|
-
|
|
55
|
+
<%= button_to confirm_import_path(@import), method: :post, class: "dp-btn dp-btn--primary", data: { dp_submit: true } do %>
|
|
56
|
+
Confirm Import
|
|
57
|
+
<% end %>
|
|
25
58
|
<% if @target._dry_run_enabled %>
|
|
26
|
-
<%= button_to
|
|
27
|
-
|
|
59
|
+
<%= button_to dry_run_import_path(@import), method: :post, class: "dp-btn dp-btn--secondary", data: { dp_submit: true } do %>
|
|
60
|
+
Dry Run
|
|
61
|
+
<% end %>
|
|
28
62
|
<% end %>
|
|
29
63
|
<%= button_to "Cancel", cancel_import_path(@import),
|
|
30
64
|
method: :post, class: "dp-btn dp-btn--danger" %>
|
|
@@ -32,18 +66,41 @@
|
|
|
32
66
|
<% end %>
|
|
33
67
|
|
|
34
68
|
<% if @import.completed? %>
|
|
35
|
-
|
|
69
|
+
<% duration = @import.updated_at && @import.created_at ? distance_of_time_in_words(@import.created_at, @import.updated_at) : nil %>
|
|
70
|
+
<%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report, duration: duration).call %>
|
|
71
|
+
<% if @records.any? %>
|
|
72
|
+
<%= raw DataPorter::Components::Preview::Table.new(
|
|
73
|
+
columns: @target._columns,
|
|
74
|
+
records: @records
|
|
75
|
+
).call %>
|
|
76
|
+
<% end %>
|
|
77
|
+
<div class="dp-actions">
|
|
78
|
+
<%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--primary" %>
|
|
79
|
+
<%= button_to "Delete", import_path(@import),
|
|
80
|
+
method: :delete, class: "dp-btn dp-btn--danger",
|
|
81
|
+
data: { turbo_confirm: "Delete this import?" } %>
|
|
82
|
+
</div>
|
|
36
83
|
<% end %>
|
|
37
84
|
|
|
38
85
|
<% if @import.failed? %>
|
|
39
|
-
<%= raw DataPorter::Components::FailureAlert.new(report: @import.report).call %>
|
|
86
|
+
<%= raw DataPorter::Components::Shared::FailureAlert.new(report: @import.report).call %>
|
|
40
87
|
<div class="dp-actions">
|
|
41
88
|
<%= button_to "Retry", parse_import_path(@import),
|
|
42
89
|
method: :post, class: "dp-btn dp-btn--primary" %>
|
|
90
|
+
<%= button_to "Delete", import_path(@import),
|
|
91
|
+
method: :delete, class: "dp-btn dp-btn--danger",
|
|
92
|
+
data: { turbo_confirm: "Delete this import?" } %>
|
|
43
93
|
</div>
|
|
44
94
|
<% end %>
|
|
45
|
-
|
|
46
|
-
<div class="dp-nav">
|
|
47
|
-
<%= link_to "Back to imports", imports_path, class: "dp-link" %>
|
|
48
|
-
</div>
|
|
49
95
|
</div>
|
|
96
|
+
|
|
97
|
+
<script>
|
|
98
|
+
(function() {
|
|
99
|
+
document.querySelectorAll("[data-dp-submit]").forEach(function(btn) {
|
|
100
|
+
btn.closest("form").addEventListener("submit", function() {
|
|
101
|
+
btn.disabled = true;
|
|
102
|
+
btn.innerHTML = '<span class="dp-spinner"></span>Processing...';
|
|
103
|
+
});
|
|
104
|
+
});
|
|
105
|
+
})();
|
|
106
|
+
</script>
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
<%= form_with model: template, url: template.persisted? ? mapping_template_path(template) : mapping_templates_path, class: "dp-form", method: template.persisted? ? :patch : :post, data: { controller: "data-porter--template-form", "data-porter--template-form-columns-value": target_columns_map.to_json } do |f| %>
|
|
2
|
+
<div class="dp-field">
|
|
3
|
+
<%= f.label :target_key, "Target", class: "dp-label" %>
|
|
4
|
+
<%= f.select :target_key,
|
|
5
|
+
targets.map { |t| [t[:label], t[:key]] },
|
|
6
|
+
{ prompt: "Select a target..." },
|
|
7
|
+
class: "dp-select",
|
|
8
|
+
data: { action: "change->data-porter--template-form#targetChanged" } %>
|
|
9
|
+
</div>
|
|
10
|
+
|
|
11
|
+
<div class="dp-field">
|
|
12
|
+
<%= f.label :name, "Template Name", class: "dp-label" %>
|
|
13
|
+
<%= f.text_field :name, class: "dp-select", placeholder: "e.g. French Headers" %>
|
|
14
|
+
</div>
|
|
15
|
+
|
|
16
|
+
<div class="dp-field">
|
|
17
|
+
<label class="dp-label">Column Mappings</label>
|
|
18
|
+
<div id="dp-mapping-pairs" data-data-porter--template-form-target="pairsContainer">
|
|
19
|
+
<% (template.mapping.presence || { "" => "" }).each do |header, field| %>
|
|
20
|
+
<div class="dp-mapping-pair" style="display: flex; gap: 0.5rem; margin-bottom: 0.5rem;">
|
|
21
|
+
<input type="text" name="mapping_template[mapping_keys][]" value="<%= header %>" placeholder="File header" class="dp-select" style="flex: 1;" />
|
|
22
|
+
<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">
|
|
23
|
+
<option value="">Select a field...</option>
|
|
24
|
+
<% if template.target_key.present? && target_columns_map[template.target_key.to_s].present? %>
|
|
25
|
+
<% target_columns_map[template.target_key.to_s].each do |label, name| %>
|
|
26
|
+
<option value="<%= name %>" <%= "selected" if name == field.to_s %>><%= label %></option>
|
|
27
|
+
<% end %>
|
|
28
|
+
<% end %>
|
|
29
|
+
</select>
|
|
30
|
+
</div>
|
|
31
|
+
<% end %>
|
|
32
|
+
</div>
|
|
33
|
+
<button type="button" class="dp-btn dp-btn--secondary" style="margin-top: 0.5rem;" data-action="data-porter--template-form#addPair">+ Add Mapping</button>
|
|
34
|
+
</div>
|
|
35
|
+
|
|
36
|
+
<div class="dp-actions">
|
|
37
|
+
<%= f.submit template.persisted? ? "Update Template" : "Create Template", class: "dp-btn dp-btn--primary" %>
|
|
38
|
+
<%= link_to "Cancel", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
|
|
39
|
+
</div>
|
|
40
|
+
<% end %>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<div class="data-porter">
|
|
2
|
+
<div class="dp-header">
|
|
3
|
+
<h1 class="dp-title">Edit Mapping Template</h1>
|
|
4
|
+
</div>
|
|
5
|
+
|
|
6
|
+
<%= render "form", template: @template, targets: @targets, target_columns_map: @target_columns_map %>
|
|
7
|
+
|
|
8
|
+
<div class="dp-nav">
|
|
9
|
+
<%= link_to "Back to templates", mapping_templates_path, class: "dp-link" %>
|
|
10
|
+
</div>
|
|
11
|
+
</div>
|