ixport 0.3.2 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 03be028cedf3161974f8143be44d6f31d79cc669e0388d266bc787d12a9dcd9e
-  data.tar.gz: e7aec59ef0d4d8288a4062e713f0ef87eb7bc23f3bbda19db8e638eabc2872ad
+  metadata.gz: 317e30ac3c1fe5057cfe6ce21402e6e8e73cb4076966fee0b030f67eed212da8
+  data.tar.gz: 41f70d3512416fb810524d1defe824d7b1d550918542abda43b437a0a5ee93c3
 SHA512:
-  metadata.gz: 2afc07d8cba3ef5d3272d5e74d4981ffd882c4dafe969d70af85dd85947049acea43356b804b8afc7bc4f3a918027f3a6aecebe990c774686cef254420f3873f
-  data.tar.gz: eb1aba5fb3eecb5071f3ce89fd847eea67d0e79bf6270f639fc0d0c24bb92c3216e0ce8924a191622b07c07afae1af004312523db31dcf89436f527ba0897731
+  metadata.gz: dca598f882beb292b8159f43bac75cd00d29cff9ad7d85ef15fbb5d090fde8048def9377cc2e25880de9ac0ab7d7ff20d09e029198752ba0556dc5ea0026aac3
+  data.tar.gz: 816ee37b957e2adfc0734aab39a35a48da427a614ec923a5a248034af8f3b77d95b17e2bb170c94a5171e1efa0929daeaf46b130617dd9586267387165c5a047

data/README.md

@@ -77,6 +77,39 @@ Each field's key is the record's attribute name. ixport looks up every match val
 
 To match on several fields, name them all, for example `match_on :happened_on, :amount, :name`.
 
+## Importing into a chosen record
+
+An import can add its rows to one record the app chooses, such as questions uploaded into one quiz. The app names that record in the link to the upload page, as its Global ID:
+
+```erb
+<%= link_to "Import questions", ixport.new_import_path(key: :quiz_questions, target: @quiz.to_global_id) %>
+```
+
+The import keeps that record through upload, mapping, preview and run, and its importer reaches it as `target`:
+
+```ruby
+class QuizQuestionImporter < Ixport::Importer
+  field :prompt, label: "Question"
+  match_on :prompt
+
+  def records
+    target.questions
+  end
+end
+```
+
+A link naming a record that does not exist answers not found. Anyone can edit the link, so the registration's `allow:` rule decides which records a person may import into, as described under [Who may import or export](#who-may-import-or-export). An import opened with no record has a `target` of `nil` and works as before. Run `bin/rails generate ixport:update` after updating, for the migration that lets an import keep its record.
+
+## Refusing a whole file
+
+An import can be registered to save nothing from a file in which any row is refused, so a partial import never leaves half of a quiz's questions added:
+
+```ruby
+Ixport.register_import(:quiz_questions, title: "Quiz questions", importer: "QuizQuestionImporter", refuse_whole_file: true)
+```
+
+While any row is refused, its preview offers no Run import button and says to fix the refused rows in the file and upload it again. A run of such a file, inside the request or in the background, saves no row, records each refused row and counts no row added or updated. When every row is valid, every row is saved. If the run stops on an error, every row it saved is undone and it counts no row added or updated. Imports registered without `refuse_whole_file` work as before.
+
 ## Uploading and mapping a file
 
 From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file is not UTF-8 text, when its quoting is broken (the message names the line where reading stopped), when it has no header row, or, in an app with no background runner, when it has more rows than `config.inline_row_limit`. A byte order mark at the start of the file is ignored, so the first column keeps its name.
@@ -164,6 +197,15 @@ Ixport.register_import :products, title: "Products", importer: "ProductImporter"
   allow: ->(person, account) { ProductImportPolicy.new(person, account).create? }
 ```
 
+A rule that takes a third argument also receives the import's chosen record, or `nil` when there is none, and returns false to refuse it:
+
+```ruby
+Ixport.register_import :quiz_questions, title: "Quiz questions", importer: "QuizQuestionImporter",
+  allow: ->(person, account, quiz) { quiz.nil? || quiz.account == account }
+```
+
+A link or upload naming a record the rule refuses answers not found, and so does every page of an import whose record it refuses. A rule that takes two arguments is called as before.
+
 A person the rule refuses does not see that import or export listed, nor their past imports of it, and every one of its pages and downloads answers not found. A registration with no rule is open to every signed-in person.
 
 ## Claude Code agents

data/app/controllers/ixport/application_controller.rb

@@ -7,13 +7,14 @@ module Ixport
 
     private
 
-    def allowed?(registration)
-      registration.allows?(ixport_person, ixport_account)
+    def allowed?(registration, target = nil)
+      registration.allows?(ixport_person, ixport_account, target)
     end
 
     def find_import
       @import = Import.find_by(id: params[:import_id] || params[:id], person: ixport_person, account: ixport_account)
-      head :not_found unless @import && allowed?(Ixport.registration(@import.key))
+      registration = @import && Ixport.registrations[@import.key.to_sym]
+      head :not_found unless registration && allowed?(registration, @import.target)
     end
 
     def ixport_person

data/app/controllers/ixport/export_templates_controller.rb

@@ -10,8 +10,12 @@ module Ixport
     end
 
     def create
-      ExportTemplate.create!(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
-      redirect_to exports_path
+      template = ExportTemplate.new(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
+      return redirect_to exports_path if template.save
+
+      @columns = @registration.exporter.constantize.columns
+      @name_error = template.errors[:name].first
+      render :new, status: :unprocessable_entity
     end
 
     def show
@@ -24,8 +28,12 @@ module Ixport
     end
 
     def update
-      @template.update!(name: params[:name], columns: known_columns(@template.registration))
-      redirect_to exports_path
+      return redirect_to exports_path if @template.update(name: params[:name], columns: known_columns(@template.registration))
+
+      @columns = @template.registration.exporter.constantize.columns
+      @name_error = @template.errors[:name].first
+      @template.restore_attributes
+      render :edit, status: :unprocessable_entity
     end
 
     def destroy
@@ -37,7 +45,8 @@ module Ixport
 
     def find_template
       @template = ExportTemplate.find_by(id: params[:id], account: ixport_account)
-      head :not_found unless @template && allowed?(@template.registration)
+      registration = @template && Ixport.export_registrations[@template.key.to_sym]
+      head :not_found unless registration && allowed?(registration)
     end
 
     def find_chosen_export

data/app/controllers/ixport/imports_controller.rb

@@ -4,6 +4,7 @@ module Ixport
   class ImportsController < ApplicationController
     before_action :find_registration, only: %i[new create]
     before_action :find_import, only: :show
+    before_action :find_fields, only: %i[new create]
 
     def index
       @registrations = Ixport.registrations.values.select { |registration| allowed?(registration) }
@@ -18,7 +19,7 @@ module Ixport
     end
 
     def create
-      import = Import.new(key: @registration.key, person: ixport_person, account: ixport_account, file: params[:file])
+      import = Import.new(key: @registration.key, person: ixport_person, account: ixport_account, target: @target, file: params[:file])
       if import.save
         redirect_to import_mapping_path(import)
       else
@@ -31,7 +32,18 @@ module Ixport
 
     def find_registration
       @registration = Ixport.registrations[params[:key].to_s.to_sym]
-      head :not_found unless @registration && allowed?(@registration)
+      @target = chosen_target
+      head :not_found unless @registration && (@target || params[:target].blank?) && allowed?(@registration, @target)
+    end
+
+    def find_fields
+      @fields = @registration.importer.constantize.fields
+    end
+
+    def chosen_target
+      GlobalID::Locator.locate(params[:target]) if params[:target].present?
+    rescue ActiveRecord::RecordNotFound, NameError
+      nil
     end
   end
 end

data/app/controllers/ixport/previews_controller.rb

@@ -11,6 +11,7 @@ module Ixport
       @counts = { add: 0, update: 0, refused: 0 }.merge(rows.group_by(&:outcome).transform_values(&:size))
       @update_rows = rows.select { |row| row.outcome == :update }
       @refused_rows = rows.select { |row| row.outcome == :refused }
+      @whole_file_refused = @import.whole_file_refused?(rows)
     end
   end
 end

data/app/controllers/ixport/runs_controller.rb

@@ -7,15 +7,28 @@ module Ixport
     def create
       runner = Ixport.background_runner
       return run_in_request unless runner
-      return redirect_to import_path(@import) unless @import.status == "pending"
+      return redirect_to import_path(@import) unless claim_pending_import
 
-      @import.update!(status: "running")
-      runner.enqueue(@import)
-      redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
+      hand_over(runner)
     end
 
     private
 
+    def claim_pending_import
+      claimed = Import.where(id: @import.id, status: "pending").update_all(status: "running", updated_at: Time.current) == 1
+      @import.reload if claimed
+      claimed
+    end
+
+    def hand_over(runner)
+      runner.enqueue(@import)
+      redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
+    rescue StandardError => error
+      Rails.error.report(error)
+      @import.update!(status: "pending")
+      redirect_to import_preview_path(@import), flash: { ixport_not_started: true }
+    end
+
     def run_in_request
       @import.run!
       redirect_to import_path(@import)

data/app/models/ixport/export_template.rb

@@ -7,6 +7,8 @@ module Ixport
     belongs_to :person, polymorphic: true
     belongs_to :account, polymorphic: true, optional: true
 
+    validates :name, presence: { message: "Give the template a name." }
+
     def registration
       Ixport.export_registration(key)
     end

data/app/models/ixport/import.rb

@@ -8,15 +8,20 @@ module Ixport
 
     belongs_to :person, polymorphic: true
     belongs_to :account, polymorphic: true, optional: true
+    belongs_to :target, polymorphic: true, optional: true
 
     has_one_attached :file
     has_many :refusals, -> { order(:row_number) }, class_name: "Ixport::ImportRefusal", dependent: :destroy
 
+    STALLED_AFTER = 1.hour
+
     validate :incoming_file_is_usable, on: :create
 
-    def run!(reporter: NullReporter.new)
+    attr_reader :run_error
+
+    def run!(reporter: NullReporter.new, report_failure: true)
       with_lock do
-        walk_and_save(reporter) if status.in?(%w[pending running])
+        walk_and_save(reporter, report_failure) if status.in?(%w[pending running])
       end
     end
 
@@ -28,6 +33,10 @@ module Ixport
       status == "running"
     end
 
+    def stalled?
+      running? && updated_at < STALLED_AFTER.ago
+    end
+
     def failed?
       status == "failed"
     end
@@ -46,8 +55,12 @@ module Ixport
       RowWalk.new(self, importer).rows
     end
 
+    def whole_file_refused?(rows = self.rows)
+      Ixport.registration(key).refuse_whole_file && rows.any? { |row| row.outcome == :refused }
+    end
+
     def importer
-      importer_class.new(person: person, account: account)
+      importer_class.new(person: person, account: account, target: target)
     end
 
     def importer_class
@@ -72,21 +85,40 @@ module Ixport
 
     private
 
-    def walk_and_save(reporter)
+    def walk_and_save(reporter, report_failure)
       counts = { add: 0, update: 0 }
-      rows = self.rows
+      rows = rows_to_walk
       reporter.set_total(rows.size)
-      rows.each do |row|
-        save_row(row, reporter)
-        counts[row.outcome] += 1 if counts.key?(row.outcome)
-        reporter.advance
+      saving_together(counts) do
+        rows.each do |row|
+          save_row(row, reporter)
+          counts[row.outcome] += 1 if counts.key?(row.outcome)
+          reporter.advance
+        end
       end
       update!(status: "finished", added_count: counts[:add], updated_count: counts[:update])
     rescue StandardError => error
-      Rails.error.report(error)
+      @run_error = error
+      Rails.error.report(error) if report_failure
       update!(status: "failed", error_message: error.message, added_count: counts[:add], updated_count: counts[:update])
     end
 
+    def saving_together(counts, &block)
+      return yield unless Ixport.registration(key).refuse_whole_file
+
+      begin
+        self.class.transaction(requires_new: true, &block)
+      rescue StandardError
+        counts.transform_values! { 0 }
+        raise
+      end
+    end
+
+    def rows_to_walk
+      rows = self.rows
+      whole_file_refused?(rows) ? rows.select { |row| row.outcome == :refused } : rows
+    end
+
     def save_row(row, reporter)
       if row.outcome == :refused
         refusals.create!(row_number: row.number, reason: row.reason)

data/app/models/ixport/wallflower_task_runner.rb

@@ -6,9 +6,9 @@ module Ixport
 
     def call(task)
       import = Import.find(task.params["import_id"])
-      Ixport.run_import(import, reporter: task)
+      import.run!(reporter: task, report_failure: false)
       task.link_result(ixport.import_path(import))
-      raise ImportFailed, import.error_message if import.failed?
+      raise(import.run_error || ImportFailed.new(import.error_message)) if import.failed?
     end
   end
 end

data/app/views/ixport/export_templates/edit.html.erb

@@ -1,7 +1,7 @@
 <%= ui_form_page(title: "Edit #{@template.name}", subtitle: @template.registration.title, back_url: exports_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_form(action: export_template_path(@template), method: :patch) do %>
-    <%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name) %>
+    <%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name, errors: @name_error.present? ? [tag.span(@name_error, data: { name_error: true })] : []) %>
     <% column_options = @columns.map { |column| [column.label, column.key] } %>
     <% @columns.each_index do |index| %>
       <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None", value: @template.columns[index]) %>

data/app/views/ixport/export_templates/new.html.erb

@@ -1,7 +1,7 @@
 <%= ui_form_page(title: "New #{@registration.title} template", back_url: exports_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_form(action: export_templates_path(@registration.key)) do %>
-    <%= ui_form_field(attribute: "name", label: "Template name", required: true) %>
+    <%= ui_form_field(attribute: "name", label: "Template name", required: true, errors: @name_error.present? ? [tag.span(@name_error, data: { name_error: true })] : []) %>
     <% column_options = @columns.map { |column| [column.label, column.key] } %>
     <% @columns.each_index do |index| %>
       <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None") %>

data/app/views/ixport/imports/new.html.erb

@@ -3,8 +3,16 @@
   <% if @error.present? %>
     <%= ui_alert(type: :error, message: tag.span(@error, data: { upload_error: true })) %>
   <% end %>
+  <%= ui_section(title: "Fields this import expects") do %>
+    <ul>
+      <% @fields.each do |field| %>
+        <%= tag.li(field.label, data: { expected_field: true }) %>
+      <% end %>
+    </ul>
+  <% end %>
   <%= ui_form(action: imports_path, multipart: true) do %>
     <%= hidden_field_tag "key", @registration.key %>
+    <%= hidden_field_tag "target", @target.to_global_id if @target %>
     <%= ui_file_upload(name: "file", label: "CSV file", accept: ".csv") %>
     <%= ui_button(label: "Upload") %>
   <% end %>

data/app/views/ixport/imports/show.html.erb

@@ -1,4 +1,6 @@
-<% heading = if @import.running?
+<% heading = if @import.stalled?
+                "Import stalled"
+              elsif @import.running?
                 "Import running"
               elsif @import.failed?
                 "Import failed"
@@ -8,7 +10,10 @@
 <%= ui_show_page(title: heading, back_url: root_path) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_page_header(title: heading) %>
-  <% if @import.running? %>
+  <% if @import.stalled? %>
+    <%= ui_alert(type: :error, message: tag.span("This import has stopped making progress.", data: { import_stalled: true })) %>
+    <p>Rows saved before it stopped stay saved.</p>
+  <% elsif @import.running? %>
     <%= ui_alert(type: :info, message: tag.span("This import is running.", data: { import_running: true })) %>
   <% else %>
     <% if @import.failed? %>

data/app/views/ixport/previews/show.html.erb

@@ -1,6 +1,9 @@
 <%= ui_show_page(title: "Preview", back_url: import_mapping_path(@import)) %>
 <%= ui_page(max_width: :md) do %>
   <%= ui_page_header(title: "Preview") %>
+  <% if flash[:ixport_not_started].present? %>
+    <%= ui_alert(type: :error, message: tag.span("This import did not start. Run it again.", data: { import_not_started: true })) %>
+  <% end %>
   <%= ui_grid(cols: { default: 1, sm: 3 }) do %>
     <%= ui_stat_card(label: "Will add", value: tag.span(@counts[:add], data: { count: "add" })) %>
     <%= ui_stat_card(label: "Will update", value: tag.span(@counts[:update], data: { count: "update" })) %>
@@ -29,7 +32,11 @@
       </ul>
     <% end %>
   <% end %>
-  <%= ui_form(action: import_run_path(@import)) do %>
-    <%= ui_button(label: "Run import") %>
+  <% if @whole_file_refused %>
+    <%= ui_alert(type: :error, message: tag.span("This import saves no row while any row is refused. Fix the refused rows in the file and upload it again.", data: { whole_file_refused: true })) %>
+  <% else %>
+    <%= ui_form(action: import_run_path(@import)) do %>
+      <%= ui_button(label: "Run import") %>
+    <% end %>
   <% end %>
 <% end %>

data/lib/generators/ixport/install/install_generator.rb

@@ -18,6 +18,7 @@ module Ixport
         migration_template "create_ixport_export_templates.rb.erb", "db/migrate/create_ixport_export_templates.rb"
         migration_template "add_wallflower_task_to_ixport_imports.rb.erb", "db/migrate/add_wallflower_task_to_ixport_imports.rb"
         migration_template "add_error_message_to_ixport_imports.rb.erb", "db/migrate/add_error_message_to_ixport_imports.rb"
+        migration_template "add_target_to_ixport_imports.rb.erb", "db/migrate/add_target_to_ixport_imports.rb"
       end
     end
   end

data/lib/generators/ixport/install/templates/add_target_to_ixport_imports.rb.erb

@@ -0,0 +1,5 @@
+class AddTargetToIxportImports < ActiveRecord::Migration[<%= ActiveRecord::Migration.current_version %>]
+  def change
+    add_reference :ixport_imports, :target, polymorphic: true, null: true
+  end
+end

data/lib/generators/ixport/update/update_generator.rb

@@ -23,6 +23,10 @@ module Ixport
       def copy_error_message_migration
         migration_template "add_error_message_to_ixport_imports.rb.erb", "db/migrate/add_error_message_to_ixport_imports.rb"
       end
+
+      def copy_target_migration
+        migration_template "add_target_to_ixport_imports.rb.erb", "db/migrate/add_target_to_ixport_imports.rb"
+      end
     end
   end
 end

data/lib/ixport/importer.rb

@@ -4,11 +4,12 @@ module Ixport
   class Importer
     Field = Struct.new(:key, :label, keyword_init: true)
 
-    attr_reader :person, :account
+    attr_reader :person, :account, :target
 
-    def initialize(person:, account:)
+    def initialize(person:, account:, target: nil)
       @person = person
       @account = account
+      @target = target
     end
 
     def self.field(key, label:)

data/lib/ixport/registrations.rb

@@ -2,19 +2,22 @@
 
 module Ixport
   module AccessRule
-    def allows?(person, account)
-      allow.nil? || allow.call(person, account)
+    def allows?(person, account, target = nil)
+      return true if allow.nil?
+      return allow.call(person, account) if allow.arity == 2
+
+      allow.call(person, account, target)
     end
   end
 
-  Registration = Struct.new(:key, :title, :importer, :allow, keyword_init: true) { include AccessRule }
+  Registration = Struct.new(:key, :title, :importer, :allow, :refuse_whole_file, keyword_init: true) { include AccessRule }
   ExportRegistration = Struct.new(:key, :title, :exporter, :chosen, :allow, keyword_init: true) { include AccessRule }
 
   class MissingImporter < StandardError; end
   class MissingExporter < StandardError; end
 
-  def self.register_import(key, title:, importer:, allow: nil)
-    registrations[key.to_sym] = Registration.new(key: key.to_sym, title: title, importer: importer.to_s, allow: allow)
+  def self.register_import(key, title:, importer:, allow: nil, refuse_whole_file: false)
+    registrations[key.to_sym] = Registration.new(key: key.to_sym, title: title, importer: importer.to_s, allow: allow, refuse_whole_file: refuse_whole_file)
   end
 
   def self.registration(key)

data/lib/ixport/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Ixport
-  VERSION = "0.3.2"
+  VERSION = "0.4.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ixport
 version: !ruby/object:Gem::Version
-  version: 0.3.2
+  version: 0.4.0
 platform: ruby
 authors:
 - Tyler Schneider
@@ -118,6 +118,7 @@ files:
 - config/routes.rb
 - lib/generators/ixport/install/install_generator.rb
 - lib/generators/ixport/install/templates/add_error_message_to_ixport_imports.rb.erb
+- lib/generators/ixport/install/templates/add_target_to_ixport_imports.rb.erb
 - lib/generators/ixport/install/templates/add_wallflower_task_to_ixport_imports.rb.erb
 - lib/generators/ixport/install/templates/create_ixport_export_templates.rb.erb
 - lib/generators/ixport/install/templates/create_ixport_import_refusals.rb.erb