data_porter 0.2.0 → 0.4.0

data/app/javascript/data_porter/mapping_controller.js

@@ -0,0 +1,86 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["columnSelect", "requiredWarning", "duplicateWarning", "saveTemplate"]
+  static values = { requiredColumns: Array }
+
+  connect() {
+    this.validate()
+  }
+
+  loadTemplate(event) {
+    const option = event.target.selectedOptions[0]
+    if (!option || !option.dataset.mapping) return
+
+    const mapping = JSON.parse(option.dataset.mapping)
+    this.columnSelectTargets.forEach(select => {
+      const header = select.name.match(/\[(.+)\]/)?.[1]
+      if (header && mapping[header]) {
+        select.value = mapping[header]
+      } else {
+        select.value = ""
+      }
+    })
+    if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
+    this.validate()
+  }
+
+  onChange() {
+    this.validate()
+  }
+
+  validate() {
+    this.validateRequired()
+    this.validateDuplicates()
+  }
+
+  validateRequired() {
+    if (!this.hasRequiredWarningTarget) return
+
+    const selected = new Set(
+      this.columnSelectTargets.map(s => s.value).filter(v => v !== "")
+    )
+
+    const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
+
+    if (missing.length > 0) {
+      const names = missing.map(c => c.label).join(", ")
+      this.requiredWarningTarget.textContent = `Required fields not mapped: ${names}`
+      this.requiredWarningTarget.style.display = ""
+    } else {
+      this.requiredWarningTarget.style.display = "none"
+    }
+  }
+
+  validateDuplicates() {
+    const counts = {}
+    this.columnSelectTargets.forEach(select => {
+      if (select.value === "") return
+      counts[select.value] = (counts[select.value] || 0) + 1
+    })
+
+    const duplicates = new Set(
+      Object.keys(counts).filter(k => counts[k] > 1)
+    )
+
+    this.columnSelectTargets.forEach(select => {
+      const row = select.closest(".dp-mapping-row")
+      if (!row) return
+
+      if (select.value !== "" && duplicates.has(select.value)) {
+        row.classList.add("dp-mapping-row--duplicate")
+      } else {
+        row.classList.remove("dp-mapping-row--duplicate")
+      }
+    })
+
+    if (this.hasDuplicateWarningTarget) {
+      if (duplicates.size > 0) {
+        this.duplicateWarningTarget.textContent = `Duplicate mappings detected for: ${[...duplicates].join(", ")}`
+        this.duplicateWarningTarget.style.display = ""
+      } else {
+        this.duplicateWarningTarget.style.display = "none"
+      }
+    }
+  }
+}

data/app/javascript/data_porter/progress_controller.js

@@ -13,7 +13,7 @@ export default class extends Controller {
           if (data.status === "processing") {
             this.updateProgress(data.percentage)
           } else {
-            window.location.reload()
+            Turbo.visit(window.location.href)
           }
         }
       }

data/app/javascript/data_porter/template_form_controller.js

@@ -0,0 +1,46 @@
+import { Controller } from "@hotwired/stimulus"
+
+export default class extends Controller {
+  static targets = ["pairsContainer", "fieldSelect"]
+  static values = { columns: Object }
+
+  targetChanged(event) {
+    const targetKey = event.target.value
+    const columns = this.columnsValue[targetKey] || []
+    this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
+  }
+
+  addPair() {
+    const container = this.pairsContainerTarget
+    const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
+    const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
+
+    const pair = document.createElement("div")
+    pair.className = "dp-mapping-pair"
+    pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
+    pair.innerHTML = this.pairHTML(columns)
+    container.appendChild(pair)
+  }
+
+  updateOptions(select, columns) {
+    const current = select.value
+    select.innerHTML = '<option value="">Select a field...</option>'
+    columns.forEach(([label, name]) => {
+      const opt = document.createElement("option")
+      opt.value = name
+      opt.textContent = label
+      if (name === current) opt.selected = true
+      select.appendChild(opt)
+    })
+  }
+
+  pairHTML(columns) {
+    const options = columns.map(([label, name]) =>
+      `<option value="${name}">${label}</option>`
+    ).join("")
+
+    return `<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />` +
+      `<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">` +
+      `<option value="">Select a field...</option>${options}</select>`
+  }
+}

data/app/jobs/data_porter/extract_headers_job.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class ExtractHeadersJob < ActiveJob::Base
+    queue_as { DataPorter.configuration.queue_name }
+
+    def perform(import_id)
+      data_import = DataImport.find(import_id)
+      Orchestrator.new(data_import).extract_headers!
+    end
+  end
+end

data/app/models/data_porter/data_import.rb

@@ -15,7 +15,9 @@ module DataPorter
       importing: 3,
       completed: 4,
       failed: 5,
-      dry_running: 6
+      dry_running: 6,
+      extracting_headers: 7,
+      mapping: 8
     }
 
     attribute :records, StoreModels::ImportRecord.to_array_type, default: -> { [] }
@@ -45,5 +47,9 @@ module DataPorter
     def records_summary
       records.group_by(&:status).transform_values(&:count)
     end
+
+    def file_based?
+      %w[csv xlsx].include?(source_type)
+    end
   end
 end

data/app/models/data_porter/mapping_template.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module DataPorter
+  class MappingTemplate < ActiveRecord::Base
+    self.table_name = "data_porter_mapping_templates"
+
+    attribute :mapping, :json, default: -> { {} }
+
+    validates :target_key, presence: true
+    validates :name, presence: true, uniqueness: { scope: :target_key }
+    validates :mapping, presence: true
+
+    scope :for_target, ->(key) { where(target_key: key) }
+  end
+end

data/app/views/data_porter/imports/index.html.erb

@@ -1,11 +1,12 @@
-<%= stylesheet_link_tag "data_porter/application" %>
-
 <div class="data-porter">
   <div class="dp-header">
     <h1 class="dp-title">Imports</h1>
-    <button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
-      New Import
-    </button>
+    <div class="dp-header__actions">
+      <%= link_to "Mapping Templates", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
+      <button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
+        New Import
+      </button>
+    </div>
   </div>
 
   <% if @imports.any? %>
@@ -26,7 +27,7 @@
             <td><%= import.id %></td>
             <td><%= import.target_key %></td>
             <td><%= import.source_type %></td>
-            <td><%= raw DataPorter::Components::StatusBadge.new(status: import.status).call %></td>
+            <td><%= raw DataPorter::Components::Shared::StatusBadge.new(status: import.status).call %></td>
             <td><%= import.created_at&.strftime("%Y-%m-%d %H:%M") %></td>
             <td><%= link_to "View", import_path(import), class: "dp-link" %></td>
           </tr>
@@ -52,7 +53,7 @@
       <button type="button" class="dp-modal__close" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">&times;</button>
     </div>
 
-    <%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true do |f| %>
+    <%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
       <div class="dp-field">
         <%= f.label :target_key, "Target", class: "dp-label" %>
         <%= f.select :target_key,

data/app/views/data_porter/imports/new.html.erb

@@ -1,11 +1,17 @@
-<%= stylesheet_link_tag "data_porter/application" %>
-
 <div class="data-porter">
   <div class="dp-header">
     <h1 class="dp-title">New Import</h1>
   </div>
 
-  <%= form_with model: @import, url: imports_path, class: "dp-form", multipart: true do |f| %>
+  <% if @import.errors.any? %>
+    <div class="dp-alert dp-alert--danger">
+      <% @import.errors.full_messages.each do |msg| %>
+        <p><%= msg %></p>
+      <% end %>
+    </div>
+  <% end %>
+
+  <%= form_with model: @import, url: imports_path, class: "dp-form", multipart: true, data: { turbo: false } do |f| %>
     <div class="dp-field">
       <%= f.label :target_key, "Target", class: "dp-label" %>
       <%= f.select :target_key,

data/app/views/data_porter/imports/show.html.erb

@@ -1,20 +1,52 @@
-<%= stylesheet_link_tag "data_porter/application" %>
-
 <div class="data-porter">
   <div class="dp-header">
+    <div class="dp-header__actions">
+      <%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--secondary" %>
+    </div>
     <h1 class="dp-title">
       <%= @target._label %> Import #<%= @import.id %>
     </h1>
-    <%= raw DataPorter::Components::StatusBadge.new(status: @import.status).call %>
+    <%= raw DataPorter::Components::Shared::StatusBadge.new(status: @import.status).call %>
+  </div>
+
+  <div class="dp-import-details">
+    <dl class="dp-details-grid">
+      <dt>Target</dt>
+      <dd><%= @target._label %></dd>
+      <dt>Source</dt>
+      <dd><%= @import.source_type.upcase %></dd>
+      <% if @import.file.attached? %>
+        <dt>File</dt>
+        <dd><%= @import.file.filename %></dd>
+      <% end %>
+      <dt>Created</dt>
+      <dd><%= @import.created_at&.strftime("%Y-%m-%d %H:%M") %></dd>
+      <% if @import.report.records_count.positive? %>
+        <dt>Records</dt>
+        <dd><%= @import.report.records_count %></dd>
+      <% end %>
+    </dl>
   </div>
 
-  <% if @import.parsing? || @import.importing? || @import.dry_running? %>
-    <%= raw DataPorter::Components::ProgressBar.new(import_id: @import.id).call %>
+  <% if @import.parsing? || @import.importing? || @import.dry_running? || @import.extracting_headers? %>
+    <%= raw DataPorter::Components::Progress::Bar.new(import_id: @import.id).call %>
+  <% end %>
+
+  <% if @import.mapping? %>
+    <%= raw DataPorter::Components::Mapping::Form.new(
+          import: @import,
+          action_url: update_mapping_import_path(@import),
+          csrf_token: form_authenticity_token,
+          file_headers: @file_headers,
+          target_columns: @target_columns,
+          templates: @templates,
+          default_mapping: @default_mapping
+        ).call %>
   <% end %>
 
   <% if @import.previewing? %>
-    <%= raw DataPorter::Components::SummaryCards.new(report: @import.report).call %>
-    <%= raw DataPorter::Components::PreviewTable.new(
+    <%= raw DataPorter::Components::Preview::SummaryCards.new(report: @import.report).call %>
+    <%= raw DataPorter::Components::Preview::Table.new(
           columns: @target._columns,
           records: @records
         ).call %>
@@ -32,18 +64,14 @@
   <% end %>
 
   <% if @import.completed? %>
-    <%= raw DataPorter::Components::ResultsSummary.new(report: @import.report).call %>
+    <%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report).call %>
   <% end %>
 
   <% if @import.failed? %>
-    <%= raw DataPorter::Components::FailureAlert.new(report: @import.report).call %>
+    <%= raw DataPorter::Components::Shared::FailureAlert.new(report: @import.report).call %>
     <div class="dp-actions">
       <%= button_to "Retry", parse_import_path(@import),
             method: :post, class: "dp-btn dp-btn--primary" %>
     </div>
   <% end %>
-
-  <div class="dp-nav">
-    <%= link_to "Back to imports", imports_path, class: "dp-link" %>
-  </div>
 </div>

data/app/views/data_porter/mapping_templates/_form.html.erb

@@ -0,0 +1,40 @@
+<%= form_with model: template, url: template.persisted? ? mapping_template_path(template) : mapping_templates_path, class: "dp-form", method: template.persisted? ? :patch : :post, data: { controller: "data-porter--template-form", "data-porter--template-form-columns-value": target_columns_map.to_json } do |f| %>
+  <div class="dp-field">
+    <%= f.label :target_key, "Target", class: "dp-label" %>
+    <%= f.select :target_key,
+          targets.map { |t| [t[:label], t[:key]] },
+          { prompt: "Select a target..." },
+          class: "dp-select",
+          data: { action: "change->data-porter--template-form#targetChanged" } %>
+  </div>
+
+  <div class="dp-field">
+    <%= f.label :name, "Template Name", class: "dp-label" %>
+    <%= f.text_field :name, class: "dp-select", placeholder: "e.g. French Headers" %>
+  </div>
+
+  <div class="dp-field">
+    <label class="dp-label">Column Mappings</label>
+    <div id="dp-mapping-pairs" data-data-porter--template-form-target="pairsContainer">
+      <% (template.mapping.presence || { "" => "" }).each do |header, field| %>
+        <div class="dp-mapping-pair" style="display: flex; gap: 0.5rem; margin-bottom: 0.5rem;">
+          <input type="text" name="mapping_template[mapping_keys][]" value="<%= header %>" placeholder="File header" class="dp-select" style="flex: 1;" />
+          <select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">
+            <option value="">Select a field...</option>
+            <% if template.target_key.present? && target_columns_map[template.target_key.to_s].present? %>
+              <% target_columns_map[template.target_key.to_s].each do |label, name| %>
+                <option value="<%= name %>" <%= "selected" if name == field.to_s %>><%= label %></option>
+              <% end %>
+            <% end %>
+          </select>
+        </div>
+      <% end %>
+    </div>
+    <button type="button" class="dp-btn dp-btn--secondary" style="margin-top: 0.5rem;" data-action="data-porter--template-form#addPair">+ Add Mapping</button>
+  </div>
+
+  <div class="dp-actions">
+    <%= f.submit template.persisted? ? "Update Template" : "Create Template", class: "dp-btn dp-btn--primary" %>
+    <%= link_to "Cancel", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
+  </div>
+<% end %>

data/app/views/data_porter/mapping_templates/edit.html.erb

@@ -0,0 +1,11 @@
+<div class="data-porter">
+  <div class="dp-header">
+    <h1 class="dp-title">Edit Mapping Template</h1>
+  </div>
+
+  <%= render "form", template: @template, targets: @targets, target_columns_map: @target_columns_map %>
+
+  <div class="dp-nav">
+    <%= link_to "Back to templates", mapping_templates_path, class: "dp-link" %>
+  </div>
+</div>

data/app/views/data_porter/mapping_templates/index.html.erb

@@ -0,0 +1,42 @@
+<div class="data-porter">
+  <div class="dp-header">
+    <div class="dp-header__actions">
+      <%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--secondary" %>
+    </div>
+    <h1 class="dp-title">Mapping Templates</h1>
+    <%= link_to "New Template", new_mapping_template_path, class: "dp-btn dp-btn--primary" %>
+  </div>
+
+  <% if @grouped.any? %>
+    <% @grouped.each do |target_key, templates| %>
+      <h2 style="font-size: 1.125rem; font-weight: 600; margin: 1.5rem 0 0.75rem; color: var(--dp-gray-700);"><%= target_key.titleize %></h2>
+      <table class="dp-table">
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Mappings</th>
+            <th></th>
+          </tr>
+        </thead>
+        <tbody>
+          <% templates.each do |template| %>
+            <tr>
+              <td><%= template.name %></td>
+              <td><%= template.mapping.size %> columns</td>
+              <td style="display: flex; gap: 0.5rem; align-items: center;">
+                <%= link_to "Edit", edit_mapping_template_path(template), class: "dp-btn dp-btn--secondary" %>
+                <%= button_to "Delete", mapping_template_path(template), method: :delete, class: "dp-btn dp-btn--danger" %>
+              </td>
+            </tr>
+          <% end %>
+        </tbody>
+      </table>
+    <% end %>
+  <% else %>
+    <div class="dp-empty-state">
+      <div class="dp-empty-state__icon">&#128203;</div>
+      <p class="dp-empty-state__text">No mapping templates yet</p>
+      <%= link_to "Create your first template", new_mapping_template_path, class: "dp-btn dp-btn--primary" %>
+    </div>
+  <% end %>
+</div>

data/app/views/data_porter/mapping_templates/new.html.erb

@@ -0,0 +1,11 @@
+<div class="data-porter">
+  <div class="dp-header">
+    <h1 class="dp-title">New Mapping Template</h1>
+  </div>
+
+  <%= render "form", template: @template, targets: @targets, target_columns_map: @target_columns_map %>
+
+  <div class="dp-nav">
+    <%= link_to "Back to templates", mapping_templates_path, class: "dp-link" %>
+  </div>
+</div>

data/app/views/layouts/data_porter/application.html.erb

@@ -0,0 +1,162 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>DataPorter</title>
+  <%= csrf_meta_tags %>
+  <%= stylesheet_link_tag "data_porter/application" %>
+  <script type="importmap">
+    {
+      "imports": {
+        "@hotwired/turbo": "<%= asset_path('data_porter/turbo.min.js') %>",
+        "@hotwired/stimulus": "<%= asset_path('data_porter/stimulus.min.js') %>"
+      }
+    }
+  </script>
+</head>
+<body>
+  <%= yield %>
+
+  <script type="module">
+    import "@hotwired/turbo"
+    import { Application, Controller } from "@hotwired/stimulus"
+
+    const application = Application.start()
+
+    application.register("data-porter--mapping", class extends Controller {
+      static targets = ["columnSelect", "requiredWarning", "duplicateWarning", "saveTemplate"]
+      static values = { requiredColumns: Array }
+
+      connect() { this.validate() }
+
+      loadTemplate(event) {
+        const option = event.target.selectedOptions[0]
+        if (!option || !option.dataset.mapping) return
+        const mapping = JSON.parse(option.dataset.mapping)
+        this.columnSelectTargets.forEach(select => {
+          const header = select.name.match(/\[(.+)\]/)?.[1]
+          select.value = (header && mapping[header]) ? mapping[header] : ""
+        })
+        if (this.hasSaveTemplateTarget) this.saveTemplateTarget.style.display = "none"
+        this.validate()
+      }
+
+      onChange() { this.validate() }
+
+      validate() {
+        this.validateRequired()
+        this.validateDuplicates()
+      }
+
+      validateRequired() {
+        if (!this.hasRequiredWarningTarget) return
+        const selected = new Set(this.columnSelectTargets.map(s => s.value).filter(v => v !== ""))
+        const missing = this.requiredColumnsValue.filter(c => !selected.has(c.name))
+        if (missing.length > 0) {
+          this.requiredWarningTarget.textContent = "Required fields not mapped: " + missing.map(c => c.label).join(", ")
+          this.requiredWarningTarget.style.display = ""
+        } else {
+          this.requiredWarningTarget.style.display = "none"
+        }
+      }
+
+      validateDuplicates() {
+        const counts = {}
+        this.columnSelectTargets.forEach(select => {
+          if (select.value === "") return
+          counts[select.value] = (counts[select.value] || 0) + 1
+        })
+        const duplicates = new Set(Object.keys(counts).filter(k => counts[k] > 1))
+        this.columnSelectTargets.forEach(select => {
+          const row = select.closest(".dp-mapping-row")
+          if (!row) return
+          row.classList.toggle("dp-mapping-row--duplicate", select.value !== "" && duplicates.has(select.value))
+        })
+        if (this.hasDuplicateWarningTarget) {
+          if (duplicates.size > 0) {
+            this.duplicateWarningTarget.textContent = "Duplicate mappings detected for: " + [...duplicates].join(", ")
+            this.duplicateWarningTarget.style.display = ""
+          } else {
+            this.duplicateWarningTarget.style.display = "none"
+          }
+        }
+      }
+    })
+
+    application.register("data-porter--template-form", class extends Controller {
+      static targets = ["pairsContainer", "fieldSelect"]
+      static values = { columns: Object }
+
+      targetChanged(event) {
+        const columns = this.columnsValue[event.target.value] || []
+        this.fieldSelectTargets.forEach(select => this.updateOptions(select, columns))
+      }
+
+      addPair() {
+        const targetKey = this.element.querySelector("[name='mapping_template[target_key]']")?.value
+        const columns = targetKey ? (this.columnsValue[targetKey] || []) : []
+        const pair = document.createElement("div")
+        pair.className = "dp-mapping-pair"
+        pair.style.cssText = "display: flex; gap: 0.5rem; margin-bottom: 0.5rem;"
+        pair.innerHTML = this.pairHTML(columns)
+        this.pairsContainerTarget.appendChild(pair)
+      }
+
+      updateOptions(select, columns) {
+        const current = select.value
+        select.innerHTML = '<option value="">Select a field...</option>'
+        columns.forEach(([label, name]) => {
+          const opt = document.createElement("option")
+          opt.value = name
+          opt.textContent = label
+          if (name === current) opt.selected = true
+          select.appendChild(opt)
+        })
+      }
+
+      pairHTML(columns) {
+        const options = columns.map(([label, name]) => '<option value="' + name + '">' + label + "</option>").join("")
+        return '<input type="text" name="mapping_template[mapping_keys][]" placeholder="File header" class="dp-select" style="flex: 1;" />' +
+          '<select name="mapping_template[mapping_values][]" class="dp-select" style="flex: 1;" data-data-porter--template-form-target="fieldSelect">' +
+          '<option value="">Select a field...</option>' + options + "</select>"
+      }
+    })
+
+    application.register("data-porter--progress", class extends Controller {
+      static targets = ["bar", "text"]
+      static values = { id: Number }
+
+      async connect() {
+        try {
+          const { createConsumer } = await import("@rails/actioncable")
+          this.subscription = createConsumer().subscriptions.create(
+            { channel: "DataPorter::ImportChannel", id: this.idValue },
+            {
+              received: (data) => {
+                if (data.status === "processing") {
+                  this.barTarget.style.width = data.percentage + "%"
+                  this.textTarget.textContent = data.percentage + "%"
+                } else {
+                  Turbo.visit(window.location.href)
+                }
+              }
+            }
+          )
+        } catch (e) {
+          this.pollForChanges()
+        }
+      }
+
+      pollForChanges() {
+        this.timer = setInterval(() => Turbo.visit(window.location.href), 3000)
+      }
+
+      disconnect() {
+        if (this.subscription) this.subscription.unsubscribe()
+        if (this.timer) clearInterval(this.timer)
+      }
+    })
+  </script>
+</body>
+</html>

data/config/routes.rb

@@ -7,6 +7,9 @@ DataPorter::Engine.routes.draw do
       post :confirm
       post :cancel
       post :dry_run
+      patch :update_mapping
     end
   end
+
+  resources :mapping_templates, except: :show
 end