ixport 0.1.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +92 -6
  3. data/app/controllers/ixport/application_controller.rb +6 -2
  4. data/app/controllers/ixport/export_templates_controller.rb +52 -0
  5. data/app/controllers/ixport/exports_controller.rb +18 -0
  6. data/app/controllers/ixport/imports_controller.rb +4 -3
  7. data/app/controllers/ixport/previews_controller.rb +2 -2
  8. data/app/controllers/ixport/runs_controller.rb +13 -1
  9. data/app/helpers/ixport/imports_helper.rb +13 -0
  10. data/app/models/ixport/csv_writer.rb +19 -0
  11. data/app/models/ixport/export_template.rb +19 -0
  12. data/app/models/ixport/import.rb +71 -19
  13. data/app/models/ixport/null_reporter.rb +11 -0
  14. data/app/models/ixport/row_walk.rb +15 -6
  15. data/app/models/ixport/wallflower_runner.rb +10 -0
  16. data/app/models/ixport/wallflower_task_runner.rb +14 -0
  17. data/app/views/ixport/export_templates/edit.html.erb +14 -0
  18. data/app/views/ixport/export_templates/new.html.erb +11 -0
  19. data/app/views/ixport/exports/index.html.erb +23 -0
  20. data/app/views/ixport/imports/index.html.erb +17 -0
  21. data/app/views/ixport/imports/show.html.erb +30 -15
  22. data/config/routes.rb +4 -0
  23. data/lib/generators/ixport/install/install_generator.rb +3 -0
  24. data/lib/generators/ixport/install/templates/add_error_message_to_ixport_imports.rb.erb +5 -0
  25. data/lib/generators/ixport/install/templates/add_wallflower_task_to_ixport_imports.rb.erb +6 -0
  26. data/lib/generators/ixport/install/templates/create_ixport_export_templates.rb.erb +12 -0
  27. data/lib/generators/ixport/update/update_generator.rb +28 -0
  28. data/lib/ixport/background_runner.rb +23 -0
  29. data/lib/ixport/configuration.rb +3 -1
  30. data/lib/ixport/engine.rb +4 -1
  31. data/lib/ixport/exporter.rb +27 -0
  32. data/lib/ixport/importer.rb +7 -3
  33. data/lib/ixport/registrations.rb +29 -3
  34. data/lib/ixport/version.rb +1 -1
  35. data/lib/ixport.rb +2 -0
  36. data/the_local/agents/ixport-develop.md +235 -37
  37. data/the_local/agents/ixport-info.md +61 -26
  38. data/the_local/agents/ixport-install.md +66 -38
  39. data/the_local/interface.yml +37 -1
  40. metadata +18 -1
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 81097e7e2778b75a3e0a8e036151b7cc6627c50bd929f86129a78c429083cb38
4
- data.tar.gz: 5d16aede3fdae08b99f79afe2db7933b5b171ee92efa6c4998224269a5742191
3
+ metadata.gz: 571077183f90149d06c934dbfc5de2eedeaa972747a6449911af9f823a073d53
4
+ data.tar.gz: 98a736581d838a51b59111457fcf37a5f428ff7a918ac97bb985a49a73b4f039
5
5
  SHA512:
6
- metadata.gz: d874135a600df60b6c2c98a56d41160d171a132eada93fad35de5494792afc256fc63132a905e57577653025b0b2b69404548d9fd6024613e55057a6f752126d
7
- data.tar.gz: a690f23002ff143c8bc1a2906b213df52f358e6e40d6405e3a7686906de653d1d8af5d90441ff3d11b2b62c79337a367acdaba46a8ec487abeeebdf4d702b419
6
+ metadata.gz: d36d12873d7e4daefa43065e7778fa839e71507ba9cb93d34065b8754382cf1a73b153893770cfa2d4e2caf7bc9053e36959be40f387095e6d5cfe1bc4f6aecc
7
+ data.tar.gz: 3ed3b12877932c4229294da54a59050dee026b887cb754e6072506e443b619265967da83b18d84ed889699a422aa782d4b278ebe8d13aacd47f01deb63e9218b
data/README.md CHANGED
@@ -1,10 +1,10 @@
1
1
  # ixport
2
2
 
3
- CSV import and export for Rails apps. An app registers the kinds of CSV a person can import and the fields each one maps to.
3
+ CSV import and export for Rails apps. An app registers the kinds of CSV a person can import and the fields each one maps to, and the CSV downloads a person can export.
4
4
 
5
5
  ## Installation
6
6
 
7
- Add the gem and run its install generator, which copies the migrations for its imports table and the table of rows an import refused:
7
+ Add the gem and run its install generator, which copies the migrations for its tables:
8
8
 
9
9
  ```ruby
10
10
  gem "ixport"
@@ -16,6 +16,13 @@ bin/rails generate ixport:install
16
16
  bin/rails db:migrate
17
17
  ```
18
18
 
19
+ After updating the gem, run its update generator to copy any migrations added since it was installed, then migrate:
20
+
21
+ ```bash
22
+ bin/rails generate ixport:update
23
+ bin/rails db:migrate
24
+ ```
25
+
19
26
  Mount the engine at the path its pages should live under:
20
27
 
21
28
  ```ruby
@@ -33,9 +40,13 @@ Ixport.configure do |config|
33
40
  config.current_account_method = :current_account # returns the account an import belongs to
34
41
  config.layout = "application" # the layout ixport's pages are drawn in
35
42
  config.inline_row_limit = 1_000 # the most rows an import may have when it runs inside the request
43
+ config.background_runner = nil # :wallflower, or the name of the app's own runner class
44
+ config.base_controller = "::ApplicationController" # the controller ixport's pages inherit, with its checks and helpers
36
45
  end
37
46
  ```
38
47
 
48
+ `config.base_controller` is read when ixport's controllers load, so set it in an initializer. Name a controller such as an admin area's base controller to give ixport's pages that area's access checks.
49
+
39
50
  ## Mapped imports and importers
40
51
 
41
52
  A mapped import is a kind of CSV a person can import into one of the app's records. Register each one in an initializer with its key, its title and the name of its importer class:
@@ -48,7 +59,7 @@ An app whose registered importer class does not exist fails to boot, with a mess
48
59
 
49
60
  The engine's root page lists every registered import by its title, each linking to its upload page.
50
61
 
51
- An importer lists the fields a person maps the file's columns to, each with the label the person sees. It names the one field rows are matched on, and the records a row may match, which it reaches through the `person` and `account` the import is for:
62
+ An importer lists the fields a person maps the file's columns to, each with the label the person sees. It names the field or fields rows are matched on, and the records a row may match, which it reaches through the `person` and `account` the import is for:
52
63
 
53
64
  ```ruby
54
65
  class ProductImporter < Ixport::Importer
@@ -62,11 +73,13 @@ class ProductImporter < Ixport::Importer
62
73
  end
63
74
  ```
64
75
 
65
- Each field's key is the record's attribute name. ixport looks up every match value in the file with one query against `records`. A row whose match value matches one of those records updates it, and a row that matches nothing adds a new record built from `records`. A row whose match value is blank, or whose file has no column mapped to the match field, always adds a record.
76
+ Each field's key is the record's attribute name. ixport looks up every match value in the file with one query against `records`. A row whose match values equal one of those records on every match field updates it, and a row that matches nothing adds a new record built from `records`. Values are compared as the record's own types, so `12.50` in a file matches `12.5` stored on a record. A row with a blank value in any match field, or whose file has no column mapped to one, always adds a record.
77
+
78
+ To match on several fields, name them all, for example `match_on :happened_on, :amount, :name`.
66
79
 
67
80
  ## Uploading and mapping a file
68
81
 
69
- From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file has no header row or has more rows than `config.inline_row_limit`.
82
+ From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file is not UTF-8 text, when its quoting is broken (the message names the line where reading stopped), when it has no header row, or, in an app with no background runner, when it has more rows than `config.inline_row_limit`. A byte order mark at the start of the file is ignored, so the first column keeps its name.
70
83
 
71
84
  After uploading, the person matches each column of the file's header row to one of the importer's fields, by its label, or skips it. Saving the mapping moves them on to the preview.
72
85
 
@@ -74,12 +87,85 @@ The preview shows how many rows will add a record, update one or be refused, bef
74
87
 
75
88
  ## Running an import
76
89
 
77
- The person runs the import from the preview, and it runs inside the request. Each row that passes the record's validations is saved, updating the record it matches or adding a new one. Each row that fails them is refused and kept with its row number and reason, and the record it matched is left unchanged.
90
+ The person runs the import from the preview. With no background runner configured, it runs inside the request. Each row that passes the record's validations is saved, updating the record it matches or adding a new one. Each row that fails them is refused and kept with its row number and reason, and the record it matched is left unchanged.
78
91
 
79
92
  The import's result page then shows how many rows were added and how many updated, and lists each refused row. It shows the same thing on every later visit. A finished import offers no Run button, its preview leads to its result page, and running it again adds or changes nothing.
80
93
 
81
94
  An import belongs to the person and account that uploaded it, and anyone else gets not found when they open it.
82
95
 
96
+ An import whose run raises an error stops there and is marked failed. Its page shows the error's message and how many rows were added and updated before it stopped, and those rows stay saved. The error is reported through `Rails.error`. Through wallflower, the task fails with the same message and still links to the import's page.
97
+
98
+ The engine's root page also lists the person's imports in the current account, newest first. Each shows its title, file name, status and date, and its added, updated and refused counts once it has run. Each opens the page for the step the import has reached: its mapping, its preview, or its result.
99
+
100
+ ## Running imports in the background
101
+
102
+ With `config.background_runner` set, every import runs through the runner, whatever its size, and the inline row limit does not apply. Having wallflower loaded does not set it.
103
+
104
+ Set it to `:wallflower` to run imports as wallflower tasks. Running an import then starts a task and takes the person to the task page, which shows its progress, the rows it changed and refused, and a link to the import's result page when it finishes. The host must mount wallflower under its own name. Run `bin/rails generate ixport:update` after updating, for the migration that links an import to its task.
105
+
106
+ Set it to the name of the app's own runner class to run imports another way. ixport hands each import to the class's `enqueue`, and the runner later runs it with `Ixport.run_import`. Until then the import's page shows it as running:
107
+
108
+ ```ruby
109
+ class ImportRunner
110
+ def self.enqueue(import)
111
+ RunImportJob.perform_later(import.id)
112
+ end
113
+ end
114
+
115
+ class RunImportJob < ApplicationJob
116
+ def perform(import_id)
117
+ Ixport.run_import(Ixport::Import.find(import_id))
118
+ end
119
+ end
120
+ ```
121
+
122
+ ## Fixed exports
123
+
124
+ A fixed export is a CSV download whose columns the app chooses. Register each one in an initializer with its key, its title and the name of its exporter class:
125
+
126
+ ```ruby
127
+ Ixport.register_export :products, title: "Products", exporter: "ProductExporter"
128
+ ```
129
+
130
+ An app whose registered exporter class does not exist fails to boot, with a message naming the class.
131
+
132
+ An exporter lists its columns, each with the label that heads it in the CSV. A column can come from a related record by naming the association it comes through. `records` returns the records to export, reached through the `person` and `account` the download is for:
133
+
134
+ ```ruby
135
+ class ProductExporter < Ixport::Exporter
136
+ column :sku, label: "SKU"
137
+ column :price_cents, label: "Price"
138
+ column :name, label: "Category", through: :category
139
+
140
+ def records
141
+ account.products.includes(:category)
142
+ end
143
+ end
144
+ ```
145
+
146
+ The engine's exports page, at `exports` under the mount path, lists every registered export by its title, each linking to its download. The download's header row is the exporter's column labels in order, followed by one row per record.
147
+
148
+ ## Chosen exports and templates
149
+
150
+ An export registered with `chosen: true` lets a person pick its columns instead of taking all of them. It uses the same kind of exporter:
151
+
152
+ ```ruby
153
+ Ixport.register_export :products, title: "Products", exporter: "ProductExporter", chosen: true
154
+ ```
155
+
156
+ On the exports page, a chosen export lists its saved templates and offers New template. A template has a name and the exporter's columns picked in order, and downloading it gives a CSV of only those columns. Every member of an account sees, downloads, edits and deletes the account's templates, and another account's templates answer not found. A template whose column the exporter no longer lists still downloads, without that column.
157
+
158
+ ## Who may import or export
159
+
160
+ Any import or export registration can take an `allow:` rule. It receives the signed-in person and the current account, and returns true to allow them. A rule can call the app's own Pundit policy:
161
+
162
+ ```ruby
163
+ Ixport.register_import :products, title: "Products", importer: "ProductImporter",
164
+ allow: ->(person, account) { ProductImportPolicy.new(person, account).create? }
165
+ ```
166
+
167
+ A person the rule refuses does not see that import or export listed, nor their past imports of it, and every one of its pages and downloads answers not found. A registration with no rule is open to every signed-in person.
168
+
83
169
  ## Claude Code agents
84
170
 
85
171
  ixport ships its own Claude Code agents through [the_local](https://github.com/DYB-Development/the_local). An app that lists ixport in its Gemfile and runs `bundle exec the_local install` gets them in `.claude/agents/`.
@@ -1,15 +1,19 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Ixport
4
- class ApplicationController < ::ApplicationController
4
+ class ApplicationController < Ixport.configuration.base_controller.constantize
5
5
  before_action { send(Ixport.configuration.authentication_method) }
6
6
  layout -> { Ixport.configuration.layout }
7
7
 
8
8
  private
9
9
 
10
+ def allowed?(registration)
11
+ registration.allows?(ixport_person, ixport_account)
12
+ end
13
+
10
14
  def find_import
11
15
  @import = Import.find_by(id: params[:import_id] || params[:id], person: ixport_person, account: ixport_account)
12
- head :not_found unless @import
16
+ head :not_found unless @import && allowed?(Ixport.registration(@import.key))
13
17
  end
14
18
 
15
19
  def ixport_person
@@ -0,0 +1,52 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class ExportTemplatesController < ApplicationController
5
+ before_action :find_chosen_export, only: %i[new create]
6
+ before_action :find_template, only: %i[show edit update destroy]
7
+
8
+ def new
9
+ @columns = @registration.exporter.constantize.columns
10
+ end
11
+
12
+ def create
13
+ ExportTemplate.create!(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
14
+ redirect_to exports_path
15
+ end
16
+
17
+ def show
18
+ exporter = @template.registration.exporter.constantize.new(person: ixport_person, account: ixport_account)
19
+ send_data CsvWriter.new(exporter, @template.picked_columns).to_csv, filename: "#{@template.name.parameterize}.csv", type: "text/csv"
20
+ end
21
+
22
+ def edit
23
+ @columns = @template.registration.exporter.constantize.columns
24
+ end
25
+
26
+ def update
27
+ @template.update!(name: params[:name], columns: known_columns(@template.registration))
28
+ redirect_to exports_path
29
+ end
30
+
31
+ def destroy
32
+ @template.destroy!
33
+ redirect_to exports_path
34
+ end
35
+
36
+ private
37
+
38
+ def find_template
39
+ @template = ExportTemplate.find_by(id: params[:id], account: ixport_account)
40
+ head :not_found unless @template && allowed?(@template.registration)
41
+ end
42
+
43
+ def find_chosen_export
44
+ @registration = Ixport.export_registrations[params[:export_id].to_s.to_sym]
45
+ head :not_found unless @registration&.chosen && allowed?(@registration)
46
+ end
47
+
48
+ def known_columns(registration)
49
+ Array(params[:columns]).compact_blank & registration.exporter.constantize.columns.map { |column| column.key.to_s }
50
+ end
51
+ end
52
+ end
@@ -0,0 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class ExportsController < ApplicationController
5
+ def index
6
+ @registrations = Ixport.export_registrations.values.select { |registration| allowed?(registration) }
7
+ @templates = ExportTemplate.where(account: ixport_account).order(:name).group_by(&:key)
8
+ end
9
+
10
+ def show
11
+ registration = Ixport.export_registrations[params[:id].to_sym]
12
+ return head :not_found unless registration && allowed?(registration)
13
+
14
+ exporter = registration.exporter.constantize.new(person: ixport_person, account: ixport_account)
15
+ send_data CsvWriter.new(exporter).to_csv, filename: "#{registration.key}.csv", type: "text/csv"
16
+ end
17
+ end
18
+ end
@@ -6,14 +6,15 @@ module Ixport
6
6
  before_action :find_import, only: :show
7
7
 
8
8
  def index
9
- @registrations = Ixport.registrations.values
9
+ @registrations = Ixport.registrations.values.select { |registration| allowed?(registration) }
10
+ @imports = Import.where(person: ixport_person, account: ixport_account, key: @registrations.map { |registration| registration.key.to_s }).order(created_at: :desc).with_attached_file.includes(:refusals)
10
11
  end
11
12
 
12
13
  def new
13
14
  end
14
15
 
15
16
  def show
16
- redirect_to import_preview_path(@import) unless @import.finished?
17
+ redirect_to import_preview_path(@import) if @import.status == "pending"
17
18
  end
18
19
 
19
20
  def create
@@ -30,7 +31,7 @@ module Ixport
30
31
 
31
32
  def find_registration
32
33
  @registration = Ixport.registrations[params[:key].to_s.to_sym]
33
- head :not_found unless @registration
34
+ head :not_found unless @registration && allowed?(@registration)
34
35
  end
35
36
  end
36
37
  end
@@ -5,9 +5,9 @@ module Ixport
5
5
  before_action :find_import
6
6
 
7
7
  def show
8
- return redirect_to import_path(@import) if @import.finished?
8
+ return redirect_to import_path(@import) unless @import.status == "pending"
9
9
 
10
- rows = RowWalk.new(@import, @import.importer_class.new(person: ixport_person, account: ixport_account)).rows
10
+ rows = @import.rows
11
11
  @counts = { add: 0, update: 0, refused: 0 }.merge(rows.group_by(&:outcome).transform_values(&:size))
12
12
  @update_rows = rows.select { |row| row.outcome == :update }
13
13
  @refused_rows = rows.select { |row| row.outcome == :refused }
@@ -5,7 +5,19 @@ module Ixport
5
5
  before_action :find_import
6
6
 
7
7
  def create
8
- @import.run!(@import.importer_class.new(person: ixport_person, account: ixport_account))
8
+ runner = Ixport.background_runner
9
+ return run_in_request unless runner
10
+ return redirect_to import_path(@import) unless @import.status == "pending"
11
+
12
+ @import.update!(status: "running")
13
+ runner.enqueue(@import)
14
+ redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
15
+ end
16
+
17
+ private
18
+
19
+ def run_in_request
20
+ @import.run!
9
21
  redirect_to import_path(@import)
10
22
  end
11
23
  end
@@ -0,0 +1,13 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ module ImportsHelper
5
+ def import_step_path(import)
6
+ case import.step
7
+ when :result then import_path(import)
8
+ when :preview then import_preview_path(import)
9
+ else import_mapping_path(import)
10
+ end
11
+ end
12
+ end
13
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "csv"
4
+
5
+ module Ixport
6
+ class CsvWriter
7
+ def initialize(exporter, columns = exporter.class.columns)
8
+ @exporter = exporter
9
+ @columns = columns
10
+ end
11
+
12
+ def to_csv
13
+ CSV.generate do |csv|
14
+ csv << @columns.map(&:label)
15
+ @exporter.records.each { |record| csv << @columns.map { |column| @exporter.value(record, column) } }
16
+ end
17
+ end
18
+ end
19
+ end
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class ExportTemplate < ApplicationRecord
5
+ self.table_name = "ixport_export_templates"
6
+
7
+ belongs_to :person, polymorphic: true
8
+ belongs_to :account, polymorphic: true, optional: true
9
+
10
+ def registration
11
+ Ixport.export_registration(key)
12
+ end
13
+
14
+ def picked_columns
15
+ exporter_columns = registration.exporter.constantize.columns.index_by { |column| column.key.to_s }
16
+ columns.filter_map { |key| exporter_columns[key] }
17
+ end
18
+ end
19
+ end
@@ -12,12 +12,11 @@ module Ixport
12
12
  has_one_attached :file
13
13
  has_many :refusals, -> { order(:row_number) }, class_name: "Ixport::ImportRefusal", dependent: :destroy
14
14
 
15
- validate :file_has_header_row, on: :create
16
- validate :file_within_inline_row_limit, on: :create
15
+ validate :incoming_file_is_usable, on: :create
17
16
 
18
- def run!(importer)
17
+ def run!(reporter: NullReporter.new)
19
18
  with_lock do
20
- walk_and_save(importer) unless finished?
19
+ walk_and_save(reporter) if status.in?(%w[pending running])
21
20
  end
22
21
  end
23
22
 
@@ -25,36 +24,89 @@ module Ixport
25
24
  status == "finished"
26
25
  end
27
26
 
27
+ def running?
28
+ status == "running"
29
+ end
30
+
31
+ def failed?
32
+ status == "failed"
33
+ end
34
+
35
+ def step
36
+ return :result if finished?
37
+
38
+ mapping.present? ? :preview : :mapping
39
+ end
40
+
41
+ def title
42
+ Ixport.registration(key).title
43
+ end
44
+
45
+ def rows
46
+ RowWalk.new(self, importer).rows
47
+ end
48
+
49
+ def importer
50
+ importer_class.new(person: person, account: account)
51
+ end
52
+
28
53
  def importer_class
29
54
  Ixport.registration(key).importer.constantize
30
55
  end
31
56
 
32
57
  def headers
33
- CSV.parse_line(file.download) || []
58
+ CSV.parse_line(contents) || []
59
+ end
60
+
61
+ def contents
62
+ self.class.readable(file.download)
63
+ end
64
+
65
+ def self.readable(text)
66
+ text.to_s.dup.force_encoding(Encoding::UTF_8).delete_prefix("\uFEFF")
34
67
  end
35
68
 
36
69
  private
37
70
 
38
- def walk_and_save(importer)
39
- rows = RowWalk.new(self, importer).rows
71
+ def walk_and_save(reporter)
72
+ counts = { add: 0, update: 0 }
73
+ rows = self.rows
74
+ reporter.set_total(rows.size)
40
75
  rows.each do |row|
41
- if row.outcome == :refused
42
- refusals.create!(row_number: row.number, reason: row.reason)
43
- else
44
- row.record.save!
45
- end
76
+ save_row(row, reporter)
77
+ counts[row.outcome] += 1 if counts.key?(row.outcome)
78
+ reporter.advance
46
79
  end
47
- update!(status: "finished", added_count: rows.count { |row| row.outcome == :add }, updated_count: rows.count { |row| row.outcome == :update })
80
+ update!(status: "finished", added_count: counts[:add], updated_count: counts[:update])
81
+ rescue StandardError => error
82
+ Rails.error.report(error)
83
+ update!(status: "failed", error_message: error.message, added_count: counts[:add], updated_count: counts[:update])
48
84
  end
49
85
 
50
- def file_has_header_row
51
- errors.add(:file, "This file has no header row.") if CSV.parse_line(incoming_file_contents).blank?
86
+ def save_row(row, reporter)
87
+ if row.outcome == :refused
88
+ refusals.create!(row_number: row.number, reason: row.reason)
89
+ reporter.refuse(label: "Row #{row.number}", reason: row.reason)
90
+ else
91
+ self.class.transaction(requires_new: true) { row.record.save! }
92
+ end
93
+ end
94
+
95
+ def incoming_file_is_usable
96
+ text = incoming_file_contents
97
+ return errors.add(:file, "This file is not UTF-8 text. Save it as UTF-8 and upload it again.") unless text.valid_encoding?
98
+
99
+ rows = CSV.parse(text)
100
+ return errors.add(:file, "This file has no header row.") if rows.first.blank?
101
+
102
+ check_inline_row_limit(rows.size - 1) unless Ixport.configuration.background_runner
103
+ rescue CSV::MalformedCSVError => error
104
+ errors.add(:file, "This file could not be read past line #{error.line_number}. Check the quotes on that line.")
52
105
  end
53
106
 
54
- def file_within_inline_row_limit
107
+ def check_inline_row_limit(row_count)
55
108
  limit = Ixport.configuration.inline_row_limit
56
- rows = [ CSV.parse(incoming_file_contents).size - 1, 0 ].max
57
- errors.add(:file, "This file has #{rows} rows, and the most an import can have is #{limit}.") if rows > limit
109
+ errors.add(:file, "This file has #{row_count} rows, and the most an import can have is #{limit}.") if row_count > limit
58
110
  end
59
111
 
60
112
  def incoming_file_contents
@@ -62,7 +114,7 @@ module Ixport
62
114
  io = attachable.is_a?(Hash) ? attachable[:io] : attachable
63
115
  return "" unless io
64
116
 
65
- io.read.tap { io.rewind }.to_s
117
+ self.class.readable(io.read.tap { io.rewind })
66
118
  end
67
119
  end
68
120
  end
@@ -0,0 +1,11 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class NullReporter
5
+ def set_total(_total); end
6
+
7
+ def advance; end
8
+
9
+ def refuse(label:, reason:); end
10
+ end
11
+ end
@@ -19,7 +19,7 @@ module Ixport
19
19
 
20
20
  def row_for(cells, number)
21
21
  values = values_of(cells)
22
- existing = existing_records[values[match_field]]
22
+ existing = existing_records[match_key(values)]
23
23
  record = existing ? existing.tap { |found| found.assign_attributes(values) } : @importer.records.new(values)
24
24
  return Row.new(number: number, record: record, outcome: :refused, reason: record.errors.full_messages.to_sentence) unless record.valid?
25
25
 
@@ -27,21 +27,30 @@ module Ixport
27
27
  end
28
28
 
29
29
  def data_rows
30
- @data_rows ||= CSV.parse(@import.file.download).drop(1)
30
+ @data_rows ||= CSV.parse(@import.contents).drop(1)
31
31
  end
32
32
 
33
33
  def values_of(cells)
34
34
  @import.mapping.to_h { |column, field| [ field.to_sym, cells[column.to_i] ] }
35
35
  end
36
36
 
37
- def match_field
38
- @importer.class.match_field
37
+ def match_fields
38
+ @importer.class.match_fields
39
+ end
40
+
41
+ def match_key(values)
42
+ key = match_fields.map { |field| cast(field, values[field]) }
43
+ key unless key.any?(&:blank?)
44
+ end
45
+
46
+ def cast(field, value)
47
+ @importer.records.klass.type_for_attribute(field.to_s).cast(value)
39
48
  end
40
49
 
41
50
  def existing_records
42
51
  @existing_records ||= begin
43
- match_values = data_rows.map { |cells| values_of(cells)[match_field] }.compact_blank
44
- @importer.records.where(match_field => match_values).index_by { |record| record.public_send(match_field).to_s }
52
+ candidates = match_fields.to_h { |field| [ field, data_rows.map { |cells| cast(field, values_of(cells)[field]) }.compact_blank ] }
53
+ @importer.records.where(candidates).index_by { |record| match_fields.map { |field| record.public_send(field) } }
45
54
  end
46
55
  end
47
56
  end
@@ -0,0 +1,10 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class WallflowerRunner
5
+ def self.enqueue(import)
6
+ task = Wallflower.start(kind: :ixport_import, person: import.person, account: import.account, params: { "import_id" => import.id })
7
+ import.update!(wallflower_task_id: task.id)
8
+ end
9
+ end
10
+ end
@@ -0,0 +1,14 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Ixport
4
+ class WallflowerTaskRunner
5
+ include Rails.application.routes.mounted_helpers
6
+
7
+ def call(task)
8
+ import = Import.find(task.params["import_id"])
9
+ Ixport.run_import(import, reporter: task)
10
+ task.link_result(ixport.import_path(import))
11
+ raise ImportFailed, import.error_message if import.failed?
12
+ end
13
+ end
14
+ end
@@ -0,0 +1,14 @@
1
+ <%= ui_form_page(title: "Edit #{@template.name}", subtitle: @template.registration.title, back_url: exports_path) %>
2
+ <%= ui_page(max_width: :md) do %>
3
+ <%= ui_form(action: export_template_path(@template), method: :patch) do %>
4
+ <%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name) %>
5
+ <% column_options = @columns.map { |column| [column.label, column.key] } %>
6
+ <% @columns.each_index do |index| %>
7
+ <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None", value: @template.columns[index]) %>
8
+ <% end %>
9
+ <%= ui_button(label: "Save template") %>
10
+ <% end %>
11
+ <%= ui_action_menu do %>
12
+ <%= ui_action_menu_item(label: "Delete", href: export_template_path(@template), method: :delete) %>
13
+ <% end %>
14
+ <% end %>
@@ -0,0 +1,11 @@
1
+ <%= ui_form_page(title: "New #{@registration.title} template", back_url: exports_path) %>
2
+ <%= ui_page(max_width: :md) do %>
3
+ <%= ui_form(action: export_templates_path(@registration.key)) do %>
4
+ <%= ui_form_field(attribute: "name", label: "Template name", required: true) %>
5
+ <% column_options = @columns.map { |column| [column.label, column.key] } %>
6
+ <% @columns.each_index do |index| %>
7
+ <%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None") %>
8
+ <% end %>
9
+ <%= ui_button(label: "Save template") %>
10
+ <% end %>
11
+ <% end %>
@@ -0,0 +1,23 @@
1
+ <%= ui_page(max_width: :md) do %>
2
+ <h1>Exports</h1>
3
+ <% if @registrations.empty? %>
4
+ <%= ui_alert(message: "No exports are set up yet.") %>
5
+ <% else %>
6
+ <% @registrations.each do |registration| %>
7
+ <% if registration.chosen %>
8
+ <div data-export><%= registration.title %></div>
9
+ <ul>
10
+ <% Array(@templates[registration.key.to_s]).each do |template| %>
11
+ <li>
12
+ <span data-export-template><%= link_to template.name, export_template_path(template) %></span>
13
+ <%= link_to "Edit", edit_export_template_path(template) %>
14
+ </li>
15
+ <% end %>
16
+ </ul>
17
+ <%= ui_button(label: "New template", href: new_export_template_path(registration.key), variant: :secondary, size: :sm) %>
18
+ <% else %>
19
+ <div data-export><%= link_to registration.title, export_path(registration.key) %></div>
20
+ <% end %>
21
+ <% end %>
22
+ <% end %>
23
+ <% end %>