data_porter 0.4.0 → 0.9.0

data/app/controllers/data_porter/imports_controller.rb

@@ -2,9 +2,13 @@
 
 module DataPorter
   class ImportsController < DataPorter.configuration.parent_controller.constantize
+    include Concerns::ImportValidation
+    include Concerns::MappingManagement
+    include Concerns::RecordPagination
+
     layout "data_porter/application"
 
-    before_action :set_import, only: %i[show parse confirm cancel dry_run update_mapping]
+    before_action :set_import, only: %i[show parse confirm cancel dry_run update_mapping status destroy]
     before_action :load_targets, only: %i[index new create]
 
     def index
@@ -18,7 +22,7 @@ module DataPorter
     def create
       build_import
 
-      if valid_file_presence? && @import.save
+      if valid_source_for_target? && valid_file_presence? && valid_import_params? && @import.save
         enqueue_after_create
         redirect_to import_path(@import)
       else
@@ -30,6 +34,7 @@ module DataPorter
       @target = @import.target_class
       @records = @import.records
       @grouped = @records.group_by(&:status)
+      paginate_records
       load_mapping_data if @import.mapping?
     end
 
@@ -47,6 +52,7 @@ module DataPorter
     end
 
     def confirm
+      @import.update!(status: :pending)
       DataPorter::ImportJob.perform_later(@import.id)
       redirect_to import_path(@import)
     end
@@ -57,10 +63,22 @@ module DataPorter
     end
 
     def dry_run
+      @import.update!(status: :pending)
       DataPorter::DryRunJob.perform_later(@import.id)
       redirect_to import_path(@import)
     end
 
+    def status
+      progress = @import.config["progress"] || {}
+      render json: { status: @import.status, progress: progress }
+    end
+
+    def destroy
+      @import.file.purge if @import.file.attached?
+      @import.destroy!
+      redirect_to imports_path
+    end
+
     private
 
     def set_import
@@ -78,15 +96,17 @@ module DataPorter
     end
 
     def import_params
-      params.require(:data_import).permit(:target_key, :source_type, :file, config: {})
+      permitted = params.require(:data_import).permit(:target_key, :source_type, :file, config: {})
+      merge_import_params(permitted)
     end
 
-    def valid_file_presence?
-      return true unless %w[csv json xlsx].include?(@import.source_type)
-      return true if @import.file.attached?
+    def merge_import_params(permitted)
+      nested = params.dig(:data_import, :config, :import_params)
+      return permitted unless nested
 
-      @import.errors.add(:file, "must be attached for #{@import.source_type.upcase} imports")
-      false
+      config = permitted[:config] || {}
+      config["import_params"] = nested.permit!.to_h
+      permitted.merge(config: config)
     end
 
     def enqueue_after_create
@@ -96,37 +116,5 @@ module DataPorter
         DataPorter::ParseJob.perform_later(@import.id)
       end
     end
-
-    def load_mapping_data
-      target = @import.target_class
-      columns = target._columns || []
-      @file_headers = @import.config["file_headers"] || []
-      @target_columns = columns.map { |c| [c.label, c.name.to_s, c.required] }
-      @default_mapping = (target._csv_mappings || {}).transform_values(&:to_s)
-      @templates = load_templates
-    end
-
-    def load_templates
-      return [] unless defined?(DataPorter::MappingTemplate)
-
-      DataPorter::MappingTemplate.for_target(@import.target_key)
-    end
-
-    def save_column_mapping
-      mapping = params.require(:column_mapping).permit!.to_h
-      merged = (@import.config || {}).merge("column_mapping" => mapping)
-      @import.update!(config: merged, status: :pending)
-    end
-
-    def save_template_if_requested
-      return unless params[:save_template] == "1"
-      return unless defined?(DataPorter::MappingTemplate)
-
-      mapping = params.require(:column_mapping).permit!.to_h
-      DataPorter::MappingTemplate.find_or_initialize_by(
-        target_key: @import.target_key,
-        name: params[:template_name].presence || "Default"
-      ).update!(mapping: mapping)
-    end
   end
 end

data/app/models/data_porter/data_import.rb

@@ -25,6 +25,11 @@ module DataPorter
 
     attribute :config, :json, default: -> { {} }
 
+    scope :purgeable, lambda {
+      where(status: %i[completed failed])
+        .where(created_at: ...DataPorter.configuration.purge_after.ago)
+    }
+
     validates :target_key, presence: true
     validates :source_type, presence: true, inclusion: { in: %w[csv json api xlsx] }
 

data/app/views/data_porter/imports/index.html.erb

@@ -1,9 +1,12 @@
-<div class="data-porter">
+<div class="data-porter" data-controller="data-porter--import-form"
+     data-data-porter--import-form-sources-value="<%= @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json %>"
+     data-data-porter--import-form-params-value="<%= @targets.map { |t| [t[:key], t[:params]] }.to_h.to_json %>"
+     data-action="keydown@document->data-porter--import-form#closeModal">
   <div class="dp-header">
     <h1 class="dp-title">Imports</h1>
     <div class="dp-header__actions">
       <%= link_to "Mapping Templates", mapping_templates_path, class: "dp-btn dp-btn--secondary" %>
-      <button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
+      <button type="button" class="dp-btn dp-btn--primary" data-action="data-porter--import-form#openModal">
         New Import
       </button>
     </div>
@@ -29,7 +32,14 @@
             <td><%= import.source_type %></td>
             <td><%= raw DataPorter::Components::Shared::StatusBadge.new(status: import.status).call %></td>
             <td><%= import.created_at&.strftime("%Y-%m-%d %H:%M") %></td>
-            <td><%= link_to "View", import_path(import), class: "dp-link" %></td>
+            <td class="dp-table__actions">
+              <%= link_to "View", import_path(import), class: "dp-btn dp-btn--sm dp-btn--secondary" %>
+              <% if import.completed? || import.failed? %>
+                <%= button_to "Delete", import_path(import),
+                      method: :delete, class: "dp-btn dp-btn--sm dp-btn--danger",
+                      data: { turbo_confirm: "Delete this import?" } %>
+              <% end %>
+            </td>
           </tr>
         <% end %>
       </tbody>
@@ -38,106 +48,68 @@
     <div class="dp-empty-state">
       <div class="dp-empty-state__icon">&#128230;</div>
       <p class="dp-empty-state__text">No imports yet</p>
-      <button type="button" class="dp-btn dp-btn--primary" onclick="document.getElementById('dp-modal').classList.add('dp-modal--open')">
+      <button type="button" class="dp-btn dp-btn--primary" data-action="data-porter--import-form#openModal">
         Create your first import
       </button>
     </div>
   <% end %>
-</div>
-
-<div id="dp-modal" class="dp-modal">
-  <div class="dp-modal__backdrop" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')"></div>
-  <div class="dp-modal__content">
-    <div class="dp-modal__header">
-      <h2 class="dp-modal__title">New Import</h2>
-      <button type="button" class="dp-modal__close" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">&times;</button>
-    </div>
-
-    <%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
-      <div class="dp-field">
-        <%= f.label :target_key, "Target", class: "dp-label" %>
-        <%= f.select :target_key,
-              @targets.map { |t| [t[:label], t[:key]] },
-              { prompt: "Select a target..." },
-              class: "dp-select" %>
-      </div>
-
-      <div class="dp-field">
-        <%= f.label :source_type, "Source Type", class: "dp-label" %>
-        <%= f.select :source_type,
-              DataPorter.configuration.enabled_sources.map { |s| [s.to_s.upcase, s] },
-              { prompt: "Select source type..." },
-              id: "dp-source-select",
-              class: "dp-select" %>
-      </div>
 
-      <div id="dp-file-field" class="dp-field">
-        <%= f.label :file, "File", class: "dp-label" %>
-        <label class="dp-dropzone" id="dp-dropzone">
-          <input type="file" name="data_import[file]" id="dp-file-input" class="dp-dropzone__input" />
-          <div class="dp-dropzone__content">
-            <div class="dp-dropzone__icon">&#128196;</div>
-            <span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
-            <span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
-          </div>
-          <div class="dp-dropzone__selected" id="dp-file-name" style="display: none;"></div>
-        </label>
+  <div id="dp-modal" class="dp-modal" data-data-porter--import-form-target="modal">
+    <div class="dp-modal__backdrop" data-action="click->data-porter--import-form#closeModalClick"></div>
+    <div class="dp-modal__content">
+      <div class="dp-modal__header">
+        <h2 class="dp-modal__title">New Import</h2>
+        <button type="button" class="dp-modal__close" data-action="data-porter--import-form#closeModalClick">&times;</button>
       </div>
 
-      <div class="dp-modal__footer">
-        <%= f.submit "Start Import", class: "dp-btn dp-btn--primary" %>
-        <button type="button" class="dp-btn dp-btn--secondary" onclick="document.getElementById('dp-modal').classList.remove('dp-modal--open')">Cancel</button>
-      </div>
-    <% end %>
-  </div>
-</div>
-
-<script>
-  (function() {
-    var sourceSelect = document.getElementById("dp-source-select");
-    var fileField = document.getElementById("dp-file-field");
-    var fileInput = document.getElementById("dp-file-input");
-    var dropzone = document.getElementById("dp-dropzone");
-    var fileName = document.getElementById("dp-file-name");
+      <%= form_with model: DataPorter::DataImport.new, url: imports_path, class: "dp-modal__body", multipart: true, data: { turbo: false } do |f| %>
+        <div class="dp-field">
+          <%= f.label :target_key, "Target", class: "dp-label" %>
+          <%= f.select :target_key,
+                @targets.map { |t| [t[:label], t[:key]] },
+                { prompt: "Select a target..." },
+                class: "dp-select",
+                data: {
+                  data_porter__import_form_target: "targetSelect",
+                  action: "data-porter--import-form#filterSources"
+                } %>
+        </div>
 
-    if (sourceSelect) {
-      sourceSelect.addEventListener("change", function() {
-        fileField.style.display = this.value === "api" ? "none" : "";
-      });
-    }
+        <div data-data-porter--import-form-target="paramsContainer"></div>
 
-    if (fileInput) {
-      fileInput.addEventListener("change", function() {
-        if (this.files.length > 0) {
-          fileName.textContent = this.files[0].name;
-          fileName.style.display = "";
-          dropzone.classList.add("dp-dropzone--has-file");
-        }
-      });
-    }
+        <div class="dp-field">
+          <%= f.label :source_type, "Source Type", class: "dp-label" %>
+          <%= f.select :source_type,
+                DataPorter.configuration.enabled_sources.map { |s| [s.to_s.upcase, s] },
+                { prompt: "Select source type..." },
+                class: "dp-select",
+                data: {
+                  data_porter__import_form_target: "sourceSelect",
+                  action: "data-porter--import-form#toggleFileField"
+                } %>
+        </div>
 
-    if (dropzone) {
-      dropzone.addEventListener("dragover", function(e) {
-        e.preventDefault();
-        this.classList.add("dp-dropzone--dragover");
-      });
-      dropzone.addEventListener("dragleave", function() {
-        this.classList.remove("dp-dropzone--dragover");
-      });
-      dropzone.addEventListener("drop", function(e) {
-        e.preventDefault();
-        this.classList.remove("dp-dropzone--dragover");
-        if (e.dataTransfer.files.length > 0) {
-          fileInput.files = e.dataTransfer.files;
-          fileInput.dispatchEvent(new Event("change"));
-        }
-      });
-    }
+        <div class="dp-field" data-data-porter--import-form-target="fileField">
+          <%= f.label :file, "File", class: "dp-label" %>
+          <label class="dp-dropzone" data-data-porter--import-form-target="dropzone"
+                 data-action="dragover->data-porter--import-form#dragover dragleave->data-porter--import-form#dragleave drop->data-porter--import-form#drop">
+            <input type="file" name="data_import[file]" class="dp-dropzone__input"
+                   data-data-porter--import-form-target="fileInput"
+                   data-action="data-porter--import-form#handleFile" />
+            <div class="dp-dropzone__content">
+              <div class="dp-dropzone__icon">&#128196;</div>
+              <span class="dp-dropzone__text">Drop your file here or <strong>browse</strong></span>
+              <span class="dp-dropzone__hint">CSV, JSON, or XLSX files accepted</span>
+            </div>
+            <div class="dp-dropzone__selected" data-data-porter--import-form-target="fileName" style="display: none;"></div>
+          </label>
+        </div>
 
-    document.addEventListener("keydown", function(e) {
-      if (e.key === "Escape") {
-        document.getElementById("dp-modal").classList.remove("dp-modal--open");
-      }
-    });
-  })();
-</script>
+        <div class="dp-modal__footer">
+          <%= f.submit "Start Import", class: "dp-btn dp-btn--primary" %>
+          <button type="button" class="dp-btn dp-btn--secondary" data-action="data-porter--import-form#closeModalClick">Cancel</button>
+        </div>
+      <% end %>
+    </div>
+  </div>
+</div>

data/app/views/data_porter/imports/new.html.erb

@@ -17,9 +17,16 @@
       <%= f.select :target_key,
             @targets.map { |t| [t[:label], t[:key]] },
             { prompt: "Select a target..." },
-            class: "dp-select" %>
+            id: "dp-target-select-new",
+            class: "dp-select",
+            data: {
+              sources: @targets.map { |t| [t[:key], t[:sources]] }.to_h.to_json,
+              params: @targets.map { |t| [t[:key], t[:params]] }.to_h.to_json
+            } %>
     </div>
 
+    <div id="dp-params-container" class="dp-params-container"></div>
+
     <div class="dp-field">
       <%= f.label :source_type, "Source Type", class: "dp-label" %>
       <%= f.select :source_type,
@@ -51,11 +58,74 @@
 
 <script>
   (function() {
+    var targetSelect = document.getElementById("dp-target-select-new");
     var sourceSelect = document.getElementById("dp-source-select-new");
     var fileField = document.getElementById("dp-file-field-new");
     var fileInput = document.getElementById("dp-file-input-new");
     var dropzone = document.getElementById("dp-dropzone-new");
     var fileName = document.getElementById("dp-file-name-new");
+    var paramsContainer = document.getElementById("dp-params-container");
+    var paramsData = JSON.parse((targetSelect && targetSelect.dataset.params) || "{}");
+
+    function renderParams() {
+      if (!paramsContainer || !targetSelect) return;
+      paramsContainer.innerHTML = "";
+      var defs = paramsData[targetSelect.value] || [];
+      defs.forEach(function(p) {
+        var div = document.createElement("div");
+        div.className = "dp-field";
+        var label = document.createElement("label");
+        label.className = "dp-label";
+        label.textContent = p.label + (p.required ? " *" : "");
+        div.appendChild(label);
+        var input;
+        if (p.type === "select" && p.collection) {
+          input = document.createElement("select");
+          input.className = "dp-select";
+          var blank = document.createElement("option");
+          blank.value = "";
+          blank.textContent = "Select...";
+          input.appendChild(blank);
+          p.collection.forEach(function(opt) {
+            var o = document.createElement("option");
+            o.textContent = opt[0];
+            o.value = opt[1];
+            if (p["default"] && String(opt[1]) === String(p["default"])) o.selected = true;
+            input.appendChild(o);
+          });
+        } else {
+          input = document.createElement("input");
+          input.className = "dp-input";
+          input.type = p.type === "number" ? "number" : (p.type === "hidden" ? "hidden" : "text");
+          if (p["default"]) input.value = p["default"];
+        }
+        input.name = "data_import[config][import_params][" + p.name + "]";
+        if (p.required) input.required = true;
+        div.appendChild(input);
+        paramsContainer.appendChild(div);
+      });
+    }
+
+    function filterSources() {
+      if (!targetSelect || !sourceSelect) return;
+      var sourcesMap = JSON.parse(targetSelect.dataset.sources || "{}");
+      var allowed = sourcesMap[targetSelect.value];
+      var options = sourceSelect.options;
+      for (var i = 1; i < options.length; i++) {
+        options[i].style.display = allowed && allowed.indexOf(options[i].value) === -1 ? "none" : "";
+      }
+      if (allowed && sourceSelect.selectedIndex > 0 && allowed.indexOf(sourceSelect.value) === -1) {
+        sourceSelect.selectedIndex = 0;
+        fileField.style.display = "";
+      }
+    }
+
+    if (targetSelect) {
+      targetSelect.addEventListener("change", function() {
+        filterSources();
+        renderParams();
+      });
+    }
 
     if (sourceSelect) {
       sourceSelect.addEventListener("change", function() {

data/app/views/data_porter/imports/show.html.erb

@@ -28,8 +28,8 @@
     </dl>
   </div>
 
-  <% if @import.parsing? || @import.importing? || @import.dry_running? || @import.extracting_headers? %>
-    <%= raw DataPorter::Components::Progress::Bar.new(import_id: @import.id).call %>
+  <% if @import.pending? || @import.parsing? || @import.importing? || @import.dry_running? || @import.extracting_headers? %>
+    <%= raw DataPorter::Components::Progress::Bar.new(import_id: @import.id, status_url: status_import_path(@import)).call %>
   <% end %>
 
   <% if @import.mapping? %>
@@ -45,18 +45,25 @@
   <% end %>
 
   <% if @import.previewing? %>
+    <div id="records">
     <%= raw DataPorter::Components::Preview::SummaryCards.new(report: @import.report).call %>
     <%= raw DataPorter::Components::Preview::Table.new(
           columns: @target._columns,
           records: @records
         ).call %>
+    </div>
+    <%= raw DataPorter::Components::Shared::Pagination.new(
+          page: @page, total_pages: @total_pages, base_url: import_path(@import)
+        ).call %>
 
     <div class="dp-actions">
-      <%= button_to "Confirm", confirm_import_path(@import),
-            method: :post, class: "dp-btn dp-btn--primary" %>
+      <%= button_to confirm_import_path(@import), method: :post, class: "dp-btn dp-btn--primary", data: { dp_submit: true } do %>
+        Confirm Import
+      <% end %>
       <% if @target._dry_run_enabled %>
-        <%= button_to "Dry Run", dry_run_import_path(@import),
-              method: :post, class: "dp-btn dp-btn--secondary" %>
+        <%= button_to dry_run_import_path(@import), method: :post, class: "dp-btn dp-btn--secondary", data: { dp_submit: true } do %>
+          Dry Run
+        <% end %>
       <% end %>
       <%= button_to "Cancel", cancel_import_path(@import),
             method: :post, class: "dp-btn dp-btn--danger" %>
@@ -64,7 +71,25 @@
   <% end %>
 
   <% if @import.completed? %>
-    <%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report).call %>
+    <% duration = @import.updated_at && @import.created_at ? distance_of_time_in_words(@import.created_at, @import.updated_at) : nil %>
+    <%= raw DataPorter::Components::Preview::ResultsSummary.new(report: @import.report, duration: duration).call %>
+    <% if @records.any? %>
+      <div id="records">
+      <%= raw DataPorter::Components::Preview::Table.new(
+            columns: @target._columns,
+            records: @records
+          ).call %>
+      </div>
+      <%= raw DataPorter::Components::Shared::Pagination.new(
+            page: @page, total_pages: @total_pages, base_url: import_path(@import)
+          ).call %>
+    <% end %>
+    <div class="dp-actions">
+      <%= link_to "Back to imports", imports_path, class: "dp-btn dp-btn--primary" %>
+      <%= button_to "Delete", import_path(@import),
+            method: :delete, class: "dp-btn dp-btn--danger",
+            data: { turbo_confirm: "Delete this import?" } %>
+    </div>
   <% end %>
 
   <% if @import.failed? %>
@@ -72,6 +97,20 @@
     <div class="dp-actions">
       <%= button_to "Retry", parse_import_path(@import),
             method: :post, class: "dp-btn dp-btn--primary" %>
+      <%= button_to "Delete", import_path(@import),
+            method: :delete, class: "dp-btn dp-btn--danger",
+            data: { turbo_confirm: "Delete this import?" } %>
     </div>
   <% end %>
 </div>
+
+<script>
+  (function() {
+    document.querySelectorAll("[data-dp-submit]").forEach(function(btn) {
+      btn.closest("form").addEventListener("submit", function() {
+        btn.disabled = true;
+        btn.innerHTML = '<span class="dp-spinner"></span>Processing...';
+      });
+    });
+  })();
+</script>