ixport 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +85 -4
- data/app/controllers/ixport/application_controller.rb +5 -1
- data/app/controllers/ixport/export_templates_controller.rb +52 -0
- data/app/controllers/ixport/exports_controller.rb +18 -0
- data/app/controllers/ixport/imports_controller.rb +4 -3
- data/app/controllers/ixport/previews_controller.rb +2 -2
- data/app/controllers/ixport/runs_controller.rb +13 -1
- data/app/helpers/ixport/imports_helper.rb +13 -0
- data/app/models/ixport/csv_writer.rb +19 -0
- data/app/models/ixport/export_template.rb +19 -0
- data/app/models/ixport/import.rb +71 -19
- data/app/models/ixport/null_reporter.rb +11 -0
- data/app/models/ixport/row_walk.rb +1 -1
- data/app/models/ixport/wallflower_runner.rb +10 -0
- data/app/models/ixport/wallflower_task_runner.rb +14 -0
- data/app/views/ixport/export_templates/edit.html.erb +14 -0
- data/app/views/ixport/export_templates/new.html.erb +11 -0
- data/app/views/ixport/exports/index.html.erb +23 -0
- data/app/views/ixport/imports/index.html.erb +17 -0
- data/app/views/ixport/imports/show.html.erb +30 -15
- data/config/routes.rb +4 -0
- data/lib/generators/ixport/install/install_generator.rb +3 -0
- data/lib/generators/ixport/install/templates/add_error_message_to_ixport_imports.rb.erb +5 -0
- data/lib/generators/ixport/install/templates/add_wallflower_task_to_ixport_imports.rb.erb +6 -0
- data/lib/generators/ixport/install/templates/create_ixport_export_templates.rb.erb +12 -0
- data/lib/generators/ixport/update/update_generator.rb +28 -0
- data/lib/ixport/background_runner.rb +23 -0
- data/lib/ixport/configuration.rb +2 -1
- data/lib/ixport/engine.rb +4 -1
- data/lib/ixport/exporter.rb +27 -0
- data/lib/ixport/registrations.rb +29 -3
- data/lib/ixport/version.rb +1 -1
- data/lib/ixport.rb +2 -0
- data/the_local/agents/ixport-develop.md +224 -29
- data/the_local/agents/ixport-info.md +55 -22
- data/the_local/agents/ixport-install.md +63 -38
- data/the_local/interface.yml +36 -1
- metadata +18 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 48f9af7983bb1c63e4df9cf1c02681f6a0aba4f40a7bf86fe7cfbfe072cf82d2
|
|
4
|
+
data.tar.gz: 816d85ef4796bc08057d087d8ef67c96366a2d3f3726680be4b645479fbdc703
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 9592b85531796f33f18e3610de6574ff1530d64cc8b692161e54621589a2ef52ed1e80e40ace9622522bd402e96548e710e16e18db35ab1b018a30d56d972171
|
|
7
|
+
data.tar.gz: 744e6ff530c9a5b1b79bdcdc8f6fb3388c4c8f329948b90f5cddea19bf0c40a85db7a3c10caa8653719326878b67ee32711f92b35b7d43c15300dd4e6aca755a
|
data/README.md
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
# ixport
|
|
2
2
|
|
|
3
|
-
CSV import and export for Rails apps. An app registers the kinds of CSV a person can import and the fields each one maps to.
|
|
3
|
+
CSV import and export for Rails apps. An app registers the kinds of CSV a person can import and the fields each one maps to, and the CSV downloads a person can export.
|
|
4
4
|
|
|
5
5
|
## Installation
|
|
6
6
|
|
|
7
|
-
Add the gem and run its install generator, which copies the migrations for its
|
|
7
|
+
Add the gem and run its install generator, which copies the migrations for its tables:
|
|
8
8
|
|
|
9
9
|
```ruby
|
|
10
10
|
gem "ixport"
|
|
@@ -16,6 +16,13 @@ bin/rails generate ixport:install
|
|
|
16
16
|
bin/rails db:migrate
|
|
17
17
|
```
|
|
18
18
|
|
|
19
|
+
After updating the gem, run its update generator to copy any migrations added since it was installed, then migrate:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
bin/rails generate ixport:update
|
|
23
|
+
bin/rails db:migrate
|
|
24
|
+
```
|
|
25
|
+
|
|
19
26
|
Mount the engine at the path its pages should live under:
|
|
20
27
|
|
|
21
28
|
```ruby
|
|
@@ -33,6 +40,7 @@ Ixport.configure do |config|
|
|
|
33
40
|
config.current_account_method = :current_account # returns the account an import belongs to
|
|
34
41
|
config.layout = "application" # the layout ixport's pages are drawn in
|
|
35
42
|
config.inline_row_limit = 1_000 # the most rows an import may have when it runs inside the request
|
|
43
|
+
config.background_runner = nil # :wallflower, or the name of the app's own runner class
|
|
36
44
|
end
|
|
37
45
|
```
|
|
38
46
|
|
|
@@ -66,7 +74,7 @@ Each field's key is the record's attribute name. ixport looks up every match val
|
|
|
66
74
|
|
|
67
75
|
## Uploading and mapping a file
|
|
68
76
|
|
|
69
|
-
From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file has no header row or has more rows than `config.inline_row_limit`.
|
|
77
|
+
From the engine's root page, a person picks a registered import and uploads a CSV for it. The upload is refused, with a message on the upload page, when the file is not UTF-8 text, when its quoting is broken (the message names the line where reading stopped), when it has no header row, or, in an app with no background runner, when it has more rows than `config.inline_row_limit`. A byte order mark at the start of the file is ignored, so the first column keeps its name.
|
|
70
78
|
|
|
71
79
|
After uploading, the person matches each column of the file's header row to one of the importer's fields, by its label, or skips it. Saving the mapping moves them on to the preview.
|
|
72
80
|
|
|
@@ -74,12 +82,85 @@ The preview shows how many rows will add a record, update one or be refused, bef
|
|
|
74
82
|
|
|
75
83
|
## Running an import
|
|
76
84
|
|
|
77
|
-
The person runs the import from the preview,
|
|
85
|
+
The person runs the import from the preview. With no background runner configured, it runs inside the request. Each row that passes the record's validations is saved, updating the record it matches or adding a new one. Each row that fails them is refused and kept with its row number and reason, and the record it matched is left unchanged.
|
|
78
86
|
|
|
79
87
|
The import's result page then shows how many rows were added and how many updated, and lists each refused row. It shows the same thing on every later visit. A finished import offers no Run button, its preview leads to its result page, and running it again adds or changes nothing.
|
|
80
88
|
|
|
81
89
|
An import belongs to the person and account that uploaded it, and anyone else gets not found when they open it.
|
|
82
90
|
|
|
91
|
+
An import whose run raises an error stops there and is marked failed. Its page shows the error's message and how many rows were added and updated before it stopped, and those rows stay saved. The error is reported through `Rails.error`. Through wallflower, the task fails with the same message and still links to the import's page.
|
|
92
|
+
|
|
93
|
+
The engine's root page also lists the person's imports in the current account, newest first. Each shows its title, file name, status and date, and its added, updated and refused counts once it has run. Each opens the page for the step the import has reached: its mapping, its preview, or its result.
|
|
94
|
+
|
|
95
|
+
## Running imports in the background
|
|
96
|
+
|
|
97
|
+
With `config.background_runner` set, every import runs through the runner, whatever its size, and the inline row limit does not apply. Having wallflower loaded does not set it.
|
|
98
|
+
|
|
99
|
+
Set it to `:wallflower` to run imports as wallflower tasks. Running an import then starts a task and takes the person to the task page, which shows its progress, the rows it changed and refused, and a link to the import's result page when it finishes. The host must mount wallflower under its own name. Run `bin/rails generate ixport:update` after updating, for the migration that links an import to its task.
|
|
100
|
+
|
|
101
|
+
Set it to the name of the app's own runner class to run imports another way. ixport hands each import to the class's `enqueue`, and the runner later runs it with `Ixport.run_import`. Until then the import's page shows it as running:
|
|
102
|
+
|
|
103
|
+
```ruby
|
|
104
|
+
class ImportRunner
|
|
105
|
+
def self.enqueue(import)
|
|
106
|
+
RunImportJob.perform_later(import.id)
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
class RunImportJob < ApplicationJob
|
|
111
|
+
def perform(import_id)
|
|
112
|
+
Ixport.run_import(Ixport::Import.find(import_id))
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
## Fixed exports
|
|
118
|
+
|
|
119
|
+
A fixed export is a CSV download whose columns the app chooses. Register each one in an initializer with its key, its title and the name of its exporter class:
|
|
120
|
+
|
|
121
|
+
```ruby
|
|
122
|
+
Ixport.register_export :products, title: "Products", exporter: "ProductExporter"
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
An app whose registered exporter class does not exist fails to boot, with a message naming the class.
|
|
126
|
+
|
|
127
|
+
An exporter lists its columns, each with the label that heads it in the CSV. A column can come from a related record by naming the association it comes through. `records` returns the records to export, reached through the `person` and `account` the download is for:
|
|
128
|
+
|
|
129
|
+
```ruby
|
|
130
|
+
class ProductExporter < Ixport::Exporter
|
|
131
|
+
column :sku, label: "SKU"
|
|
132
|
+
column :price_cents, label: "Price"
|
|
133
|
+
column :name, label: "Category", through: :category
|
|
134
|
+
|
|
135
|
+
def records
|
|
136
|
+
account.products.includes(:category)
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
```
|
|
140
|
+
|
|
141
|
+
The engine's exports page, at `exports` under the mount path, lists every registered export by its title, each linking to its download. The download's header row is the exporter's column labels in order, followed by one row per record.
|
|
142
|
+
|
|
143
|
+
## Chosen exports and templates
|
|
144
|
+
|
|
145
|
+
An export registered with `chosen: true` lets a person pick its columns instead of taking all of them. It uses the same kind of exporter:
|
|
146
|
+
|
|
147
|
+
```ruby
|
|
148
|
+
Ixport.register_export :products, title: "Products", exporter: "ProductExporter", chosen: true
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
On the exports page, a chosen export lists its saved templates and offers New template. A template has a name and the exporter's columns picked in order, and downloading it gives a CSV of only those columns. Every member of an account sees, downloads, edits and deletes the account's templates, and another account's templates answer not found. A template whose column the exporter no longer lists still downloads, without that column.
|
|
152
|
+
|
|
153
|
+
## Who may import or export
|
|
154
|
+
|
|
155
|
+
Any import or export registration can take an `allow:` rule. It receives the signed-in person and the current account, and returns true to allow them. A rule can call the app's own Pundit policy:
|
|
156
|
+
|
|
157
|
+
```ruby
|
|
158
|
+
Ixport.register_import :products, title: "Products", importer: "ProductImporter",
|
|
159
|
+
allow: ->(person, account) { ProductImportPolicy.new(person, account).create? }
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
A person the rule refuses does not see that import or export listed, nor their past imports of it, and every one of its pages and downloads answers not found. A registration with no rule is open to every signed-in person.
|
|
163
|
+
|
|
83
164
|
## Claude Code agents
|
|
84
165
|
|
|
85
166
|
ixport ships its own Claude Code agents through [the_local](https://github.com/DYB-Development/the_local). An app that lists ixport in its Gemfile and runs `bundle exec the_local install` gets them in `.claude/agents/`.
|
|
@@ -7,9 +7,13 @@ module Ixport
|
|
|
7
7
|
|
|
8
8
|
private
|
|
9
9
|
|
|
10
|
+
def allowed?(registration)
|
|
11
|
+
registration.allows?(ixport_person, ixport_account)
|
|
12
|
+
end
|
|
13
|
+
|
|
10
14
|
def find_import
|
|
11
15
|
@import = Import.find_by(id: params[:import_id] || params[:id], person: ixport_person, account: ixport_account)
|
|
12
|
-
head :not_found unless @import
|
|
16
|
+
head :not_found unless @import && allowed?(Ixport.registration(@import.key))
|
|
13
17
|
end
|
|
14
18
|
|
|
15
19
|
def ixport_person
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
class ExportTemplatesController < ApplicationController
|
|
5
|
+
before_action :find_chosen_export, only: %i[new create]
|
|
6
|
+
before_action :find_template, only: %i[show edit update destroy]
|
|
7
|
+
|
|
8
|
+
def new
|
|
9
|
+
@columns = @registration.exporter.constantize.columns
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def create
|
|
13
|
+
ExportTemplate.create!(key: @registration.key, name: params[:name], columns: known_columns(@registration), person: ixport_person, account: ixport_account)
|
|
14
|
+
redirect_to exports_path
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def show
|
|
18
|
+
exporter = @template.registration.exporter.constantize.new(person: ixport_person, account: ixport_account)
|
|
19
|
+
send_data CsvWriter.new(exporter, @template.picked_columns).to_csv, filename: "#{@template.name.parameterize}.csv", type: "text/csv"
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def edit
|
|
23
|
+
@columns = @template.registration.exporter.constantize.columns
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def update
|
|
27
|
+
@template.update!(name: params[:name], columns: known_columns(@template.registration))
|
|
28
|
+
redirect_to exports_path
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def destroy
|
|
32
|
+
@template.destroy!
|
|
33
|
+
redirect_to exports_path
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def find_template
|
|
39
|
+
@template = ExportTemplate.find_by(id: params[:id], account: ixport_account)
|
|
40
|
+
head :not_found unless @template && allowed?(@template.registration)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def find_chosen_export
|
|
44
|
+
@registration = Ixport.export_registrations[params[:export_id].to_s.to_sym]
|
|
45
|
+
head :not_found unless @registration&.chosen && allowed?(@registration)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def known_columns(registration)
|
|
49
|
+
Array(params[:columns]).compact_blank & registration.exporter.constantize.columns.map { |column| column.key.to_s }
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
class ExportsController < ApplicationController
|
|
5
|
+
def index
|
|
6
|
+
@registrations = Ixport.export_registrations.values.select { |registration| allowed?(registration) }
|
|
7
|
+
@templates = ExportTemplate.where(account: ixport_account).order(:name).group_by(&:key)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def show
|
|
11
|
+
registration = Ixport.export_registrations[params[:id].to_sym]
|
|
12
|
+
return head :not_found unless registration && allowed?(registration)
|
|
13
|
+
|
|
14
|
+
exporter = registration.exporter.constantize.new(person: ixport_person, account: ixport_account)
|
|
15
|
+
send_data CsvWriter.new(exporter).to_csv, filename: "#{registration.key}.csv", type: "text/csv"
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
@@ -6,14 +6,15 @@ module Ixport
|
|
|
6
6
|
before_action :find_import, only: :show
|
|
7
7
|
|
|
8
8
|
def index
|
|
9
|
-
@registrations = Ixport.registrations.values
|
|
9
|
+
@registrations = Ixport.registrations.values.select { |registration| allowed?(registration) }
|
|
10
|
+
@imports = Import.where(person: ixport_person, account: ixport_account, key: @registrations.map { |registration| registration.key.to_s }).order(created_at: :desc).with_attached_file.includes(:refusals)
|
|
10
11
|
end
|
|
11
12
|
|
|
12
13
|
def new
|
|
13
14
|
end
|
|
14
15
|
|
|
15
16
|
def show
|
|
16
|
-
redirect_to import_preview_path(@import)
|
|
17
|
+
redirect_to import_preview_path(@import) if @import.status == "pending"
|
|
17
18
|
end
|
|
18
19
|
|
|
19
20
|
def create
|
|
@@ -30,7 +31,7 @@ module Ixport
|
|
|
30
31
|
|
|
31
32
|
def find_registration
|
|
32
33
|
@registration = Ixport.registrations[params[:key].to_s.to_sym]
|
|
33
|
-
head :not_found unless @registration
|
|
34
|
+
head :not_found unless @registration && allowed?(@registration)
|
|
34
35
|
end
|
|
35
36
|
end
|
|
36
37
|
end
|
|
@@ -5,9 +5,9 @@ module Ixport
|
|
|
5
5
|
before_action :find_import
|
|
6
6
|
|
|
7
7
|
def show
|
|
8
|
-
return redirect_to import_path(@import)
|
|
8
|
+
return redirect_to import_path(@import) unless @import.status == "pending"
|
|
9
9
|
|
|
10
|
-
rows =
|
|
10
|
+
rows = @import.rows
|
|
11
11
|
@counts = { add: 0, update: 0, refused: 0 }.merge(rows.group_by(&:outcome).transform_values(&:size))
|
|
12
12
|
@update_rows = rows.select { |row| row.outcome == :update }
|
|
13
13
|
@refused_rows = rows.select { |row| row.outcome == :refused }
|
|
@@ -5,7 +5,19 @@ module Ixport
|
|
|
5
5
|
before_action :find_import
|
|
6
6
|
|
|
7
7
|
def create
|
|
8
|
-
|
|
8
|
+
runner = Ixport.background_runner
|
|
9
|
+
return run_in_request unless runner
|
|
10
|
+
return redirect_to import_path(@import) unless @import.status == "pending"
|
|
11
|
+
|
|
12
|
+
@import.update!(status: "running")
|
|
13
|
+
runner.enqueue(@import)
|
|
14
|
+
redirect_to @import.wallflower_task_id ? wallflower.task_path(@import.wallflower_task_id) : import_path(@import)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
private
|
|
18
|
+
|
|
19
|
+
def run_in_request
|
|
20
|
+
@import.run!
|
|
9
21
|
redirect_to import_path(@import)
|
|
10
22
|
end
|
|
11
23
|
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
module ImportsHelper
|
|
5
|
+
def import_step_path(import)
|
|
6
|
+
case import.step
|
|
7
|
+
when :result then import_path(import)
|
|
8
|
+
when :preview then import_preview_path(import)
|
|
9
|
+
else import_mapping_path(import)
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "csv"
|
|
4
|
+
|
|
5
|
+
module Ixport
|
|
6
|
+
class CsvWriter
|
|
7
|
+
def initialize(exporter, columns = exporter.class.columns)
|
|
8
|
+
@exporter = exporter
|
|
9
|
+
@columns = columns
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
def to_csv
|
|
13
|
+
CSV.generate do |csv|
|
|
14
|
+
csv << @columns.map(&:label)
|
|
15
|
+
@exporter.records.each { |record| csv << @columns.map { |column| @exporter.value(record, column) } }
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
class ExportTemplate < ApplicationRecord
|
|
5
|
+
self.table_name = "ixport_export_templates"
|
|
6
|
+
|
|
7
|
+
belongs_to :person, polymorphic: true
|
|
8
|
+
belongs_to :account, polymorphic: true, optional: true
|
|
9
|
+
|
|
10
|
+
def registration
|
|
11
|
+
Ixport.export_registration(key)
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def picked_columns
|
|
15
|
+
exporter_columns = registration.exporter.constantize.columns.index_by { |column| column.key.to_s }
|
|
16
|
+
columns.filter_map { |key| exporter_columns[key] }
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
end
|
data/app/models/ixport/import.rb
CHANGED
|
@@ -12,12 +12,11 @@ module Ixport
|
|
|
12
12
|
has_one_attached :file
|
|
13
13
|
has_many :refusals, -> { order(:row_number) }, class_name: "Ixport::ImportRefusal", dependent: :destroy
|
|
14
14
|
|
|
15
|
-
validate :
|
|
16
|
-
validate :file_within_inline_row_limit, on: :create
|
|
15
|
+
validate :incoming_file_is_usable, on: :create
|
|
17
16
|
|
|
18
|
-
def run!(
|
|
17
|
+
def run!(reporter: NullReporter.new)
|
|
19
18
|
with_lock do
|
|
20
|
-
walk_and_save(
|
|
19
|
+
walk_and_save(reporter) if status.in?(%w[pending running])
|
|
21
20
|
end
|
|
22
21
|
end
|
|
23
22
|
|
|
@@ -25,36 +24,89 @@ module Ixport
|
|
|
25
24
|
status == "finished"
|
|
26
25
|
end
|
|
27
26
|
|
|
27
|
+
def running?
|
|
28
|
+
status == "running"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def failed?
|
|
32
|
+
status == "failed"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def step
|
|
36
|
+
return :result if finished?
|
|
37
|
+
|
|
38
|
+
mapping.present? ? :preview : :mapping
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def title
|
|
42
|
+
Ixport.registration(key).title
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def rows
|
|
46
|
+
RowWalk.new(self, importer).rows
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def importer
|
|
50
|
+
importer_class.new(person: person, account: account)
|
|
51
|
+
end
|
|
52
|
+
|
|
28
53
|
def importer_class
|
|
29
54
|
Ixport.registration(key).importer.constantize
|
|
30
55
|
end
|
|
31
56
|
|
|
32
57
|
def headers
|
|
33
|
-
CSV.parse_line(
|
|
58
|
+
CSV.parse_line(contents) || []
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def contents
|
|
62
|
+
self.class.readable(file.download)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.readable(text)
|
|
66
|
+
text.to_s.dup.force_encoding(Encoding::UTF_8).delete_prefix("\uFEFF")
|
|
34
67
|
end
|
|
35
68
|
|
|
36
69
|
private
|
|
37
70
|
|
|
38
|
-
def walk_and_save(
|
|
39
|
-
|
|
71
|
+
def walk_and_save(reporter)
|
|
72
|
+
counts = { add: 0, update: 0 }
|
|
73
|
+
rows = self.rows
|
|
74
|
+
reporter.set_total(rows.size)
|
|
40
75
|
rows.each do |row|
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
row.record.save!
|
|
45
|
-
end
|
|
76
|
+
save_row(row, reporter)
|
|
77
|
+
counts[row.outcome] += 1 if counts.key?(row.outcome)
|
|
78
|
+
reporter.advance
|
|
46
79
|
end
|
|
47
|
-
update!(status: "finished", added_count:
|
|
80
|
+
update!(status: "finished", added_count: counts[:add], updated_count: counts[:update])
|
|
81
|
+
rescue StandardError => error
|
|
82
|
+
Rails.error.report(error)
|
|
83
|
+
update!(status: "failed", error_message: error.message, added_count: counts[:add], updated_count: counts[:update])
|
|
48
84
|
end
|
|
49
85
|
|
|
50
|
-
def
|
|
51
|
-
|
|
86
|
+
def save_row(row, reporter)
|
|
87
|
+
if row.outcome == :refused
|
|
88
|
+
refusals.create!(row_number: row.number, reason: row.reason)
|
|
89
|
+
reporter.refuse(label: "Row #{row.number}", reason: row.reason)
|
|
90
|
+
else
|
|
91
|
+
self.class.transaction(requires_new: true) { row.record.save! }
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def incoming_file_is_usable
|
|
96
|
+
text = incoming_file_contents
|
|
97
|
+
return errors.add(:file, "This file is not UTF-8 text. Save it as UTF-8 and upload it again.") unless text.valid_encoding?
|
|
98
|
+
|
|
99
|
+
rows = CSV.parse(text)
|
|
100
|
+
return errors.add(:file, "This file has no header row.") if rows.first.blank?
|
|
101
|
+
|
|
102
|
+
check_inline_row_limit(rows.size - 1) unless Ixport.configuration.background_runner
|
|
103
|
+
rescue CSV::MalformedCSVError => error
|
|
104
|
+
errors.add(:file, "This file could not be read past line #{error.line_number}. Check the quotes on that line.")
|
|
52
105
|
end
|
|
53
106
|
|
|
54
|
-
def
|
|
107
|
+
def check_inline_row_limit(row_count)
|
|
55
108
|
limit = Ixport.configuration.inline_row_limit
|
|
56
|
-
rows
|
|
57
|
-
errors.add(:file, "This file has #{rows} rows, and the most an import can have is #{limit}.") if rows > limit
|
|
109
|
+
errors.add(:file, "This file has #{row_count} rows, and the most an import can have is #{limit}.") if row_count > limit
|
|
58
110
|
end
|
|
59
111
|
|
|
60
112
|
def incoming_file_contents
|
|
@@ -62,7 +114,7 @@ module Ixport
|
|
|
62
114
|
io = attachable.is_a?(Hash) ? attachable[:io] : attachable
|
|
63
115
|
return "" unless io
|
|
64
116
|
|
|
65
|
-
io.read.tap { io.rewind }
|
|
117
|
+
self.class.readable(io.read.tap { io.rewind })
|
|
66
118
|
end
|
|
67
119
|
end
|
|
68
120
|
end
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
class WallflowerRunner
|
|
5
|
+
def self.enqueue(import)
|
|
6
|
+
task = Wallflower.start(kind: :ixport_import, person: import.person, account: import.account, params: { "import_id" => import.id })
|
|
7
|
+
import.update!(wallflower_task_id: task.id)
|
|
8
|
+
end
|
|
9
|
+
end
|
|
10
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ixport
|
|
4
|
+
class WallflowerTaskRunner
|
|
5
|
+
include Rails.application.routes.mounted_helpers
|
|
6
|
+
|
|
7
|
+
def call(task)
|
|
8
|
+
import = Import.find(task.params["import_id"])
|
|
9
|
+
Ixport.run_import(import, reporter: task)
|
|
10
|
+
task.link_result(ixport.import_path(import))
|
|
11
|
+
raise ImportFailed, import.error_message if import.failed?
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
end
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
<%= ui_form_page(title: "Edit #{@template.name}", subtitle: @template.registration.title, back_url: exports_path) %>
|
|
2
|
+
<%= ui_page(max_width: :md) do %>
|
|
3
|
+
<%= ui_form(action: export_template_path(@template), method: :patch) do %>
|
|
4
|
+
<%= ui_form_field(attribute: "name", label: "Template name", required: true, value: @template.name) %>
|
|
5
|
+
<% column_options = @columns.map { |column| [column.label, column.key] } %>
|
|
6
|
+
<% @columns.each_index do |index| %>
|
|
7
|
+
<%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None", value: @template.columns[index]) %>
|
|
8
|
+
<% end %>
|
|
9
|
+
<%= ui_button(label: "Save template") %>
|
|
10
|
+
<% end %>
|
|
11
|
+
<%= ui_action_menu do %>
|
|
12
|
+
<%= ui_action_menu_item(label: "Delete", href: export_template_path(@template), method: :delete) %>
|
|
13
|
+
<% end %>
|
|
14
|
+
<% end %>
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
<%= ui_form_page(title: "New #{@registration.title} template", back_url: exports_path) %>
|
|
2
|
+
<%= ui_page(max_width: :md) do %>
|
|
3
|
+
<%= ui_form(action: export_templates_path(@registration.key)) do %>
|
|
4
|
+
<%= ui_form_field(attribute: "name", label: "Template name", required: true) %>
|
|
5
|
+
<% column_options = @columns.map { |column| [column.label, column.key] } %>
|
|
6
|
+
<% @columns.each_index do |index| %>
|
|
7
|
+
<%= ui_form_field(attribute: "columns[]", label: "Column #{index + 1}", type: :select, options: column_options, include_blank: "None") %>
|
|
8
|
+
<% end %>
|
|
9
|
+
<%= ui_button(label: "Save template") %>
|
|
10
|
+
<% end %>
|
|
11
|
+
<% end %>
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
<%= ui_page(max_width: :md) do %>
|
|
2
|
+
<h1>Exports</h1>
|
|
3
|
+
<% if @registrations.empty? %>
|
|
4
|
+
<%= ui_alert(message: "No exports are set up yet.") %>
|
|
5
|
+
<% else %>
|
|
6
|
+
<% @registrations.each do |registration| %>
|
|
7
|
+
<% if registration.chosen %>
|
|
8
|
+
<div data-export><%= registration.title %></div>
|
|
9
|
+
<ul>
|
|
10
|
+
<% Array(@templates[registration.key.to_s]).each do |template| %>
|
|
11
|
+
<li>
|
|
12
|
+
<span data-export-template><%= link_to template.name, export_template_path(template) %></span>
|
|
13
|
+
<%= link_to "Edit", edit_export_template_path(template) %>
|
|
14
|
+
</li>
|
|
15
|
+
<% end %>
|
|
16
|
+
</ul>
|
|
17
|
+
<%= ui_button(label: "New template", href: new_export_template_path(registration.key), variant: :secondary, size: :sm) %>
|
|
18
|
+
<% else %>
|
|
19
|
+
<div data-export><%= link_to registration.title, export_path(registration.key) %></div>
|
|
20
|
+
<% end %>
|
|
21
|
+
<% end %>
|
|
22
|
+
<% end %>
|
|
23
|
+
<% end %>
|
|
@@ -7,4 +7,21 @@
|
|
|
7
7
|
<div data-import><%= link_to registration.title, new_import_path(key: registration.key) %></div>
|
|
8
8
|
<% end %>
|
|
9
9
|
<% end %>
|
|
10
|
+
<% if @imports.any? %>
|
|
11
|
+
<%= ui_section(title: "Your imports") do %>
|
|
12
|
+
<% @imports.each do |import| %>
|
|
13
|
+
<div data-past-import>
|
|
14
|
+
<%= link_to import_step_path(import), data: { step_link: true } do %><span data-import-title><%= import.title %></span><% end %>
|
|
15
|
+
<span data-file-name><%= import.file.filename.to_s %></span>
|
|
16
|
+
<span data-status><%= import.status.capitalize %></span>
|
|
17
|
+
<span data-made-on><%= import.created_at.strftime("%b %-d, %Y") %></span>
|
|
18
|
+
<% if import.finished? %>
|
|
19
|
+
<span>Added <span data-added><%= import.added_count %></span></span>
|
|
20
|
+
<span>Updated <span data-updated><%= import.updated_count %></span></span>
|
|
21
|
+
<span>Refused <span data-refused><%= import.refusals.size %></span></span>
|
|
22
|
+
<% end %>
|
|
23
|
+
</div>
|
|
24
|
+
<% end %>
|
|
25
|
+
<% end %>
|
|
26
|
+
<% end %>
|
|
10
27
|
<% end %>
|
|
@@ -1,20 +1,35 @@
|
|
|
1
|
-
|
|
1
|
+
<% heading = if @import.running?
|
|
2
|
+
"Import running"
|
|
3
|
+
elsif @import.failed?
|
|
4
|
+
"Import failed"
|
|
5
|
+
else
|
|
6
|
+
"Import finished"
|
|
7
|
+
end %>
|
|
8
|
+
<%= ui_show_page(title: heading, back_url: root_path) %>
|
|
2
9
|
<%= ui_page(max_width: :md) do %>
|
|
3
|
-
<%= ui_page_header(title:
|
|
4
|
-
|
|
5
|
-
<%=
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
10
|
+
<%= ui_page_header(title: heading) %>
|
|
11
|
+
<% if @import.running? %>
|
|
12
|
+
<%= ui_alert(type: :info, message: tag.span("This import is running.", data: { import_running: true })) %>
|
|
13
|
+
<% else %>
|
|
14
|
+
<% if @import.failed? %>
|
|
15
|
+
<%= ui_alert(type: :error, message: tag.span(@import.error_message, data: { import_error: true })) %>
|
|
16
|
+
<p>Rows saved before it stopped stay saved.</p>
|
|
17
|
+
<% end %>
|
|
18
|
+
<%= ui_grid(cols: { default: 1, sm: 2 }) do %>
|
|
19
|
+
<%= ui_stat_card(label: "Added", value: tag.span(@import.added_count, data: { result_count: "added" })) %>
|
|
20
|
+
<%= ui_stat_card(label: "Updated", value: tag.span(@import.updated_count, data: { result_count: "updated" })) %>
|
|
21
|
+
<% end %>
|
|
22
|
+
<% if @import.refusals.any? %>
|
|
23
|
+
<%= ui_section(title: "Refused") do %>
|
|
24
|
+
<ul>
|
|
25
|
+
<% @import.refusals.each do |refusal| %>
|
|
26
|
+
<%= tag.li(data: { refused_row: true }) do %>
|
|
27
|
+
Row <%= tag.span(refusal.row_number, data: { row_number: true }) %>
|
|
28
|
+
<%= tag.span(refusal.reason, data: { reason: true }) %>
|
|
29
|
+
<% end %>
|
|
15
30
|
<% end %>
|
|
16
|
-
|
|
17
|
-
|
|
31
|
+
</ul>
|
|
32
|
+
<% end %>
|
|
18
33
|
<% end %>
|
|
19
34
|
<% end %>
|
|
20
35
|
<% end %>
|