ixport 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7010d4338968ec32338a7f02a446a59a091487b0860f955c090f5e180b61d65a
-  data.tar.gz: 82f2bf80f348f9a308a0c3674efaf62441c0b4865cfe867e599243861b5227fe
+  metadata.gz: b40a581b96b5e012197ee7a185f08891e4580fb7f2bc824ddd5de3789ccb9e02
+  data.tar.gz: 9f6e9bde4f0590cfac3524df4695bff2ee132ad47cd8d66938b919fbf5ec2c67
 SHA512:
-  metadata.gz: 363acc8c4d91440e255240436d96196607831fd98ebcab6d90fc3c817b94609aa5a2978ac16aab41eb3b6fd4371f9d565367eec0bb08c32596e351c647fdb683
-  data.tar.gz: cda9b8842119709c211008e76820185f0d36544b831d718b8bfa743f0b681b4a9e79919b341b3a73acf0c600b98d15472a72632861b5d6a51635089bea0d935b
+  metadata.gz: 6f3dde1eac2e6e592a49fcf9049c822efa56e70468cbe1666329c8be35beef75e94e03091786bf94f8f0f141619e48a338fa99ecc1b349f6e547461569a7d12d
+  data.tar.gz: 0af9d0471ea9d3e62a0476e1e6ef9a7fa8d9bff8050e90576d7d226e53c620bf90f62261930e52a43a9115b3a97b4b5d0daa450b999d618c3576f2a61bd1db35

data/README.md

@@ -81,7 +81,7 @@ To match on several fields, name them all, for example `match_on :happened_on, :
 
 From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file is not UTF-8 text, when its quoting is broken (the message names the line where reading stopped), when it has no header row, or, in an app with no background runner, when it has more rows than `config.inline_row_limit`. A byte order mark at the start of the file is ignored, so the first column keeps its name.
 
-After uploading, the person matches each column of the file's header row to one of the importer's fields, by its label, or skips it. Saving the mapping moves them on to the preview.
+After uploading, the person matches each column of the file's header row to one of the importer's fields, by its label, or skips it. Saving the mapping moves them on to the preview, and going back to the mapping page shows the choices already made.
 
 The preview shows how many rows will add a record, update one or be refused, before anything is saved. It lists each row that will update a record and each refused row, by its row number in the file, with the reason the record gives for a refusal. A row is refused when the record it builds is not valid. Viewing the preview saves nothing, and the person can go back from it to change the mapping.
 
@@ -89,7 +89,7 @@ The preview shows how many rows will add a record, update one or be refused, bef
 
 The person runs the import from the preview. With no background runner configured, it runs inside the request. Each row that passes the record's validations is saved, updating the record it matches or adding a new one. Each row that fails them is refused and kept with its row number and reason, and the record it matched is left unchanged.
 
-The import's result page then shows how many rows were added and how many updated, and lists each refused row. It shows the same thing on every later visit. A finished import offers no Run button, its preview leads to its result page, and running it again adds or changes nothing.
+The import's result page then shows its status, how many rows the file had, how many were added and how many updated, and lists each refused row. It shows the same thing on every later visit. A finished import offers no Run button, its preview leads to its result page, and running it again adds or changes nothing.
 
 An import belongs to the person and account that uploaded it, and anyone else gets not found when they open it.
 

data/app/controllers/ixport/application_controller.rb

@@ -13,7 +13,8 @@ module Ixport
 
     def find_import
       @import = Import.find_by(id: params[:import_id] || params[:id], person: ixport_person, account: ixport_account)
-      head :not_found unless @import && allowed?(Ixport.registration(@import.key))
+      registration = @import && Ixport.registrations[@import.key.to_sym]
+      head :not_found unless registration && allowed?(registration)
     end
 
     def ixport_person

data/app/controllers/ixport/export_templates_controller.rb

@@ -10,8 +10,12 @@ module Ixport
     end
 
     def create
-      ExportTemplate.create!(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
-      redirect_to exports_path
+      template = ExportTemplate.new(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
+      return redirect_to exports_path if template.save
+
+      @columns = @registration.exporter.constantize.columns
+      @name_error = template.errors[:name].first
+      render :new, status: :unprocessable_entity
     end
 
     def show
@@ -24,8 +28,12 @@ module Ixport
     end
 
     def update
-      @template.update!(name: params[:name], columns: known_columns(@template.registration))
-      redirect_to exports_path
+      return redirect_to exports_path if @template.update(name: params[:name], columns: known_columns(@template.registration))
+
+      @columns = @template.registration.exporter.constantize.columns
+      @name_error = @template.errors[:name].first
+      @template.restore_attributes
+      render :edit, status: :unprocessable_entity
     end
 
     def destroy
@@ -37,7 +45,8 @@ module Ixport
 
     def find_template
       @template = ExportTemplate.find_by(id: params[:id], account: ixport_account)
-      head :not_found unless @template && allowed?(@template.registration)
+      registration = @template && Ixport.export_registrations[@template.key.to_sym]
+      head :not_found unless registration && allowed?(registration)
     end
 
     def find_chosen_export

data/app/controllers/ixport/runs_controller.rb

@@ -7,15 +7,28 @@ module Ixport
     def create
       runner = Ixport.background_runner
       return run_in_request unless runner
-      return redirect_to import_path(@import) unless @import.status == "pending"
+      return redirect_to import_path(@import) unless claim_pending_import
 
-      @import.update!(status: "running")
-      runner.enqueue(@import)
-      redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
+      hand_over(runner)
     end
 
     private
 
+    def claim_pending_import
+      claimed = Import.where(id: @import.id, status: "pending").update_all(status: "running", updated_at: Time.current) == 1
+      @import.reload if claimed
+      claimed
+    end
+
+    def hand_over(runner)
+      runner.enqueue(@import)
+      redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
+    rescue StandardError => error
+      Rails.error.report(error)
+      @import.update!(status: "pending")
+      redirect_to import_preview_path(@import), flash: { ixport_not_started: true }
+    end
+
     def run_in_request
       @import.run!
       redirect_to import_path(@import)

data/app/models/ixport/export_template.rb

@@ -7,6 +7,8 @@ module Ixport
     belongs_to :person, polymorphic: true
     belongs_to :account, polymorphic: true, optional: true
 
+    validates :name, presence: { message: "Give the template a name." }
+
     def registration
       Ixport.export_registration(key)
     end

data/app/models/ixport/import.rb

@@ -12,11 +12,15 @@ module Ixport
     has_one_attached :file
     has_many :refusals, -> { order(:row_number) }, class_name: "Ixport::ImportRefusal", dependent: :destroy
 
+    STALLED_AFTER = 1.hour
+
     validate :incoming_file_is_usable, on: :create
 
-    def run!(reporter: NullReporter.new)
+    attr_reader :run_error
+
+    def run!(reporter: NullReporter.new, report_failure: true)
       with_lock do
-        walk_and_save(reporter) if status.in?(%w[pending running])
+        walk_and_save(reporter, report_failure) if status.in?(%w[pending running])
       end
     end
 
@@ -28,6 +32,10 @@ module Ixport
       status == "running"
     end
 
+    def stalled?
+      running? && updated_at < STALLED_AFTER.ago
+    end
+
     def failed?
       status == "failed"
     end
@@ -54,6 +62,10 @@ module Ixport
       Ixport.registration(key).importer.constantize
     end
 
+    def row_count
+      [ CSV.parse(contents).size - 1, 0 ].max
+    end
+
     def headers
       CSV.parse_line(contents) || []
     end
@@ -68,7 +80,7 @@ module Ixport
 
     private
 
-    def walk_and_save(reporter)
+    def walk_and_save(reporter, report_failure)
       counts = { add: 0, update: 0 }
       rows = self.rows
       reporter.set_total(rows.size)
@@ -79,7 +91,8 @@ module Ixport
       end
       update!(status: "finished", added_count: counts[:add], updated_count: counts[:update])
     rescue StandardError => error
-      Rails.error.report(error)
+      @run_error = error
+      Rails.error.report(error) if report_failure
       update!(status: "failed", error_message: error.message, added_count: counts[:add], updated_count: counts[:update])
     end
 

data/app/models/ixport/wallflower_task_runner.rb

@@ -6,9 +6,9 @@ module Ixport
 
     def call(task)
       import = Import.find(task.params["import_id"])
-      Ixport.run_import(import, reporter: task)
+      import.run!(reporter: task, report_failure: false)
       task.link_result(ixport.import_path(import))
-      raise ImportFailed, import.error_message if import.failed?
+      raise(import.run_error || ImportFailed.new(import.error_message)) if import.failed?
     end
   end
 end

data/app/views/ixport/export_templates/edit.html.erb

@@ -1,7 +1,7 @@
 <%= ui_form_page(title: "Edit #{@template.name}", subtitle: @template.registration.title, back_url: exports_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_form(action: export_template_path(@template), method: :patch) do %>
-    <%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name) %>
+    <%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name, errors: @name_error.present? ? [tag.span(@name_error, data: { name_error: true })] : []) %>
     <% column_options = @columns.map { |column| [column.label, column.key] } %>
     <% @columns.each_index do |index| %>
       <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None", value: @template.columns[index]) %>

data/app/views/ixport/export_templates/new.html.erb

@@ -1,7 +1,7 @@
 <%= ui_form_page(title: "New #{@registration.title} template", back_url: exports_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_form(action: export_templates_path(@registration.key)) do %>
-    <%= ui_form_field(attribute: "name", label: "Template name", required: true) %>
+    <%= ui_form_field(attribute: "name", label: "Template name", required: true, errors: @name_error.present? ? [tag.span(@name_error, data: { name_error: true })] : []) %>
     <% column_options = @columns.map { |column| [column.label, column.key] } %>
     <% @columns.each_index do |index| %>
       <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None") %>

data/app/views/ixport/imports/show.html.erb

@@ -1,4 +1,6 @@
-<% heading = if @import.running?
+<% heading = if @import.stalled?
+                "Import stalled"
+              elsif @import.running?
                 "Import running"
               elsif @import.failed?
                 "Import failed"
@@ -8,16 +10,21 @@
 <%= ui_show_page(title: heading, back_url: root_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_page_header(title: heading) %>
-  <% if @import.running? %>
+  <% if @import.stalled? %>
+    <%= ui_alert(type: :error, message: tag.span("This import has stopped making progress.", data: { import_stalled: true })) %>
+    <p>Rows saved before it stopped stay saved.</p>
+  <% elsif @import.running? %>
     <%= ui_alert(type: :info, message: tag.span("This import is running.", data: { import_running: true })) %>
   <% else %>
     <% if @import.failed? %>
       <%= ui_alert(type: :error, message: tag.span(@import.error_message, data: { import_error: true })) %>
       <p>Rows saved before it stopped stay saved.</p>
     <% end %>
-    <%= ui_grid(cols: { default: 1, sm: 2 }) do %>
+    <%= ui_badge(label: tag.span(@import.status.humanize, data: { import_status: true }), variant: @import.failed? ? :danger : :success) %>
+    <%= ui_grid(cols: { default: 1, sm: 3 }) do %>
       <%= ui_stat_card(label: "Added", value: tag.span(@import.added_count, data: { result_count: "added" })) %>
       <%= ui_stat_card(label: "Updated", value: tag.span(@import.updated_count, data: { result_count: "updated" })) %>
+      <%= ui_stat_card(label: "Rows in file", value: tag.span(@import.row_count, data: { result_count: "total" })) %>
     <% end %>
     <% if @import.refusals.any? %>
       <%= ui_section(title: "Refused") do %>

data/app/views/ixport/mappings/show.html.erb

@@ -1,10 +1,14 @@
 <%= ui_form_page(title: "Match columns", back_url: root_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_form(action: import_mapping_path(@import), method: :patch) do %>
-    <% @import.headers.each_with_index do |header, index| %>
-      <%= tag.div(data: { column: true }) do %>
-        <%= tag.span(header, data: { column_name: true }) %>
-        <%= ui_select(name: "mapping[#{index}]", options: @fields.map { |field| [field.label, field.key] }, include_blank: "Skip") %>
+    <%= ui_grid(cols: { default: 1 }, gap: :md) do %>
+      <% @import.headers.each_with_index do |header, index| %>
+        <%= tag.div(data: { column: true }) do %>
+          <%= ui_grid(cols: { default: 1, sm: 2 }, gap: :sm) do %>
+            <%= tag.span(header, data: { column_name: true }) %>
+            <%= ui_select(name: "mapping[#{index}]", options: @fields.map { |field| [field.label, field.key] }, selected: @import.mapping&.dig(index.to_s), include_blank: "Skip") %>
+          <% end %>
+        <% end %>
       <% end %>
     <% end %>
     <%= ui_button(label: "Save mapping") %>

data/app/views/ixport/previews/show.html.erb

@@ -1,6 +1,9 @@
 <%= ui_show_page(title: "Preview", back_url: import_mapping_path(@import)) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_page_header(title: "Preview") %>
+  <% if flash[:ixport_not_started].present? %>
+    <%= ui_alert(type: :error, message: tag.span("This import did not start. Run it again.", data: { import_not_started: true })) %>
+  <% end %>
   <%= ui_grid(cols: { default: 1, sm: 3 }) do %>
     <%= ui_stat_card(label: "Will add", value: tag.span(@counts[:add], data: { count: "add" })) %>
     <%= ui_stat_card(label: "Will update", value: tag.span(@counts[:update], data: { count: "update" })) %>

data/lib/ixport/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ixport
-  VERSION = "0.3.1"
+  VERSION = "0.3.3"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ixport
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Tyler Schneider