importu 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.editorconfig +15 -0
- data/.github/workflows/ci.yml +48 -0
- data/.gitignore +4 -0
- data/.rspec +1 -0
- data/.rubocop.yml +311 -0
- data/.simplecov +14 -0
- data/.yardstick.yml +36 -0
- data/Appraisals +22 -0
- data/CHANGELOG.md +51 -0
- data/CONTRIBUTING.md +86 -0
- data/Gemfile +5 -1
- data/LICENSE +21 -0
- data/README.md +435 -52
- data/Rakefile +71 -0
- data/UPGRADING.md +188 -0
- data/gemfiles/rails_7_2.gemfile +11 -0
- data/gemfiles/rails_7_2.gemfile.lock +268 -0
- data/gemfiles/rails_8_0.gemfile +11 -0
- data/gemfiles/rails_8_0.gemfile.lock +271 -0
- data/gemfiles/rails_8_1.gemfile +11 -0
- data/gemfiles/rails_8_1.gemfile.lock +269 -0
- data/gemfiles/standalone.gemfile +8 -0
- data/gemfiles/standalone.gemfile.lock +197 -0
- data/importu.gemspec +41 -22
- data/lib/importu/backends/active_record.rb +171 -0
- data/lib/importu/backends/middleware/duplicate_manager_proxy.rb +41 -0
- data/lib/importu/backends/middleware/enforce_allowed_actions.rb +52 -0
- data/lib/importu/backends/middleware.rb +11 -0
- data/lib/importu/backends.rb +103 -0
- data/lib/importu/config_dsl.rb +381 -0
- data/lib/importu/converter_context.rb +94 -0
- data/lib/importu/converters.rb +119 -64
- data/lib/importu/definition.rb +23 -0
- data/lib/importu/duplicate_manager.rb +88 -0
- data/lib/importu/exceptions.rb +135 -4
- data/lib/importu/importer.rb +183 -96
- data/lib/importu/record.rb +138 -102
- data/lib/importu/sources/csv.rb +122 -0
- data/lib/importu/sources/json.rb +106 -0
- data/lib/importu/sources/ruby.rb +46 -0
- data/lib/importu/sources/xml.rb +133 -0
- data/lib/importu/sources.rb +13 -0
- data/lib/importu/summary.rb +277 -0
- data/lib/importu/version.rb +3 -1
- data/lib/importu.rb +45 -9
- data/spec/fixtures/books-duplicates/README.md +7 -0
- data/spec/fixtures/books-duplicates/infile.csv +7 -0
- data/spec/fixtures/books-duplicates/model.json +23 -0
- data/spec/fixtures/books-duplicates/summary.json +10 -0
- data/spec/fixtures/books-valid/README.md +13 -0
- data/spec/fixtures/books-valid/infile.csv +4 -0
- data/spec/fixtures/books-valid/infile.json +23 -0
- data/spec/fixtures/books-valid/infile.xml +21 -0
- data/spec/fixtures/books-valid/model.json +23 -0
- data/spec/fixtures/books-valid/record.json +26 -0
- data/spec/fixtures/books-valid/summary.json +8 -0
- data/spec/fixtures/source-empty-file/infile.csv +0 -0
- data/spec/fixtures/source-empty-file/infile.json +0 -0
- data/spec/fixtures/source-empty-file/infile.xml +0 -0
- data/spec/fixtures/source-empty-records/infile.csv +3 -0
- data/spec/fixtures/source-empty-records/infile.json +1 -0
- data/spec/fixtures/source-empty-records/infile.xml +6 -0
- data/spec/fixtures/source-malformed/infile.csv +1 -0
- data/spec/fixtures/source-malformed/infile.json +1 -0
- data/spec/fixtures/source-malformed/infile.xml +3 -0
- data/spec/fixtures/source-no-records/infile.csv +1 -0
- data/spec/fixtures/source-no-records/infile.json +1 -0
- data/spec/fixtures/source-no-records/infile.xml +3 -0
- data/spec/lib/importu/backends/active_record_spec.rb +150 -0
- data/spec/lib/importu/backends/middleware/duplicate_manager_proxy_spec.rb +70 -0
- data/spec/lib/importu/backends/middleware/enforce_allowed_actions_spec.rb +70 -0
- data/spec/lib/importu/backends_spec.rb +170 -0
- data/spec/lib/importu/converters_spec.rb +184 -141
- data/spec/lib/importu/definition_spec.rb +248 -0
- data/spec/lib/importu/duplicate_manager_spec.rb +92 -0
- data/spec/lib/importu/exceptions_spec.rb +69 -16
- data/spec/lib/importu/import_context_spec.rb +199 -0
- data/spec/lib/importu/importer_spec.rb +95 -0
- data/spec/lib/importu/integration_spec.rb +221 -0
- data/spec/lib/importu/record_spec.rb +130 -80
- data/spec/lib/importu/sources/csv_spec.rb +29 -0
- data/spec/lib/importu/sources/importer_source_examples.rb +175 -0
- data/spec/lib/importu/sources/json_spec.rb +29 -0
- data/spec/lib/importu/sources/ruby_spec.rb +102 -0
- data/spec/lib/importu/sources/xml_spec.rb +70 -0
- data/spec/lib/importu/summary_spec.rb +186 -0
- data/spec/spec_helper.rb +91 -7
- data/spec/support/active_record.rb +20 -0
- data/spec/support/book_importer.rb +31 -0
- data/spec/support/dummy_backend.rb +50 -0
- data/spec/support/fixtures_helper.rb +43 -0
- data/spec/support/matchers/delegate_matcher.rb +14 -8
- metadata +173 -100
- data/lib/importu/core_ext/array/deep_freeze.rb +0 -7
- data/lib/importu/core_ext/deep_freeze.rb +0 -3
- data/lib/importu/core_ext/hash/deep_freeze.rb +0 -7
- data/lib/importu/core_ext/object/deep_freeze.rb +0 -6
- data/lib/importu/core_ext.rb +0 -3
- data/lib/importu/dsl.rb +0 -127
- data/lib/importu/importer/csv.rb +0 -52
- data/lib/importu/importer/json.rb +0 -45
- data/lib/importu/importer/xml.rb +0 -55
- data/spec/factories/importer.rb +0 -12
- data/spec/factories/importer_record.rb +0 -13
- data/spec/factories/json_importer.rb +0 -14
- data/spec/factories/xml_importer.rb +0 -12
- data/spec/lib/importu/dsl_spec.rb +0 -26
- data/spec/lib/importu/importer/json_spec.rb +0 -37
- data/spec/lib/importu/importer/xml_spec.rb +0 -14
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
PATH
|
|
2
|
+
remote: ..
|
|
3
|
+
specs:
|
|
4
|
+
importu (0.1.0)
|
|
5
|
+
bigdecimal
|
|
6
|
+
csv
|
|
7
|
+
nokogiri (>= 1.13)
|
|
8
|
+
|
|
9
|
+
GEM
|
|
10
|
+
remote: https://rubygems.org/
|
|
11
|
+
specs:
|
|
12
|
+
appraisal (2.5.0)
|
|
13
|
+
bundler
|
|
14
|
+
rake
|
|
15
|
+
thor (>= 0.14.0)
|
|
16
|
+
ast (2.4.3)
|
|
17
|
+
bigdecimal (4.0.1)
|
|
18
|
+
csv (3.3.5)
|
|
19
|
+
date (3.5.1)
|
|
20
|
+
diff-lcs (1.6.2)
|
|
21
|
+
docile (1.4.1)
|
|
22
|
+
erb (6.0.1)
|
|
23
|
+
io-console (0.8.2)
|
|
24
|
+
irb (1.16.0)
|
|
25
|
+
pp (>= 0.6.0)
|
|
26
|
+
rdoc (>= 4.0.0)
|
|
27
|
+
reline (>= 0.4.2)
|
|
28
|
+
json (2.18.0)
|
|
29
|
+
language_server-protocol (3.17.0.5)
|
|
30
|
+
lint_roller (1.1.0)
|
|
31
|
+
nokogiri (1.19.0-aarch64-linux-gnu)
|
|
32
|
+
racc (~> 1.4)
|
|
33
|
+
nokogiri (1.19.0-aarch64-linux-musl)
|
|
34
|
+
racc (~> 1.4)
|
|
35
|
+
nokogiri (1.19.0-arm-linux-gnu)
|
|
36
|
+
racc (~> 1.4)
|
|
37
|
+
nokogiri (1.19.0-arm-linux-musl)
|
|
38
|
+
racc (~> 1.4)
|
|
39
|
+
nokogiri (1.19.0-arm64-darwin)
|
|
40
|
+
racc (~> 1.4)
|
|
41
|
+
nokogiri (1.19.0-x86_64-darwin)
|
|
42
|
+
racc (~> 1.4)
|
|
43
|
+
nokogiri (1.19.0-x86_64-linux-gnu)
|
|
44
|
+
racc (~> 1.4)
|
|
45
|
+
nokogiri (1.19.0-x86_64-linux-musl)
|
|
46
|
+
racc (~> 1.4)
|
|
47
|
+
parallel (1.27.0)
|
|
48
|
+
parser (3.3.10.1)
|
|
49
|
+
ast (~> 2.4.1)
|
|
50
|
+
racc
|
|
51
|
+
pp (0.6.3)
|
|
52
|
+
prettyprint
|
|
53
|
+
prettyprint (0.2.0)
|
|
54
|
+
prism (1.9.0)
|
|
55
|
+
psych (5.3.1)
|
|
56
|
+
date
|
|
57
|
+
stringio
|
|
58
|
+
racc (1.8.1)
|
|
59
|
+
rainbow (3.1.1)
|
|
60
|
+
rake (13.3.1)
|
|
61
|
+
rdoc (7.1.0)
|
|
62
|
+
erb
|
|
63
|
+
psych (>= 4.0.0)
|
|
64
|
+
tsort
|
|
65
|
+
redcarpet (3.6.1)
|
|
66
|
+
regexp_parser (2.11.3)
|
|
67
|
+
reline (0.6.3)
|
|
68
|
+
io-console (~> 0.5)
|
|
69
|
+
rspec (3.13.2)
|
|
70
|
+
rspec-core (~> 3.13.0)
|
|
71
|
+
rspec-expectations (~> 3.13.0)
|
|
72
|
+
rspec-mocks (~> 3.13.0)
|
|
73
|
+
rspec-core (3.13.6)
|
|
74
|
+
rspec-support (~> 3.13.0)
|
|
75
|
+
rspec-expectations (3.13.5)
|
|
76
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
77
|
+
rspec-support (~> 3.13.0)
|
|
78
|
+
rspec-mocks (3.13.7)
|
|
79
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
|
80
|
+
rspec-support (~> 3.13.0)
|
|
81
|
+
rspec-support (3.13.6)
|
|
82
|
+
rubocop (1.84.0)
|
|
83
|
+
json (~> 2.3)
|
|
84
|
+
language_server-protocol (~> 3.17.0.2)
|
|
85
|
+
lint_roller (~> 1.1.0)
|
|
86
|
+
parallel (~> 1.10)
|
|
87
|
+
parser (>= 3.3.0.2)
|
|
88
|
+
rainbow (>= 2.2.2, < 4.0)
|
|
89
|
+
regexp_parser (>= 2.9.3, < 3.0)
|
|
90
|
+
rubocop-ast (>= 1.49.0, < 2.0)
|
|
91
|
+
ruby-progressbar (~> 1.7)
|
|
92
|
+
unicode-display_width (>= 2.4.0, < 4.0)
|
|
93
|
+
rubocop-ast (1.49.0)
|
|
94
|
+
parser (>= 3.3.7.2)
|
|
95
|
+
prism (~> 1.7)
|
|
96
|
+
rubocop-performance (1.26.1)
|
|
97
|
+
lint_roller (~> 1.1)
|
|
98
|
+
rubocop (>= 1.75.0, < 2.0)
|
|
99
|
+
rubocop-ast (>= 1.47.1, < 2.0)
|
|
100
|
+
ruby-progressbar (1.13.0)
|
|
101
|
+
simplecov (0.22.0)
|
|
102
|
+
docile (~> 1.1)
|
|
103
|
+
simplecov-html (~> 0.11)
|
|
104
|
+
simplecov_json_formatter (~> 0.1)
|
|
105
|
+
simplecov-html (0.13.2)
|
|
106
|
+
simplecov_json_formatter (0.1.4)
|
|
107
|
+
stringio (3.2.0)
|
|
108
|
+
thor (1.5.0)
|
|
109
|
+
tsort (0.2.0)
|
|
110
|
+
unicode-display_width (3.2.0)
|
|
111
|
+
unicode-emoji (~> 4.1)
|
|
112
|
+
unicode-emoji (4.2.0)
|
|
113
|
+
yard (0.9.38)
|
|
114
|
+
yardstick (0.9.9)
|
|
115
|
+
yard (~> 0.8, >= 0.8.7.2)
|
|
116
|
+
|
|
117
|
+
PLATFORMS
|
|
118
|
+
aarch64-linux-gnu
|
|
119
|
+
aarch64-linux-musl
|
|
120
|
+
arm-linux-gnu
|
|
121
|
+
arm-linux-musl
|
|
122
|
+
arm64-darwin
|
|
123
|
+
x86_64-darwin
|
|
124
|
+
x86_64-linux-gnu
|
|
125
|
+
x86_64-linux-musl
|
|
126
|
+
|
|
127
|
+
DEPENDENCIES
|
|
128
|
+
appraisal
|
|
129
|
+
bundler (>= 2.0)
|
|
130
|
+
importu!
|
|
131
|
+
irb
|
|
132
|
+
redcarpet
|
|
133
|
+
rspec (~> 3.13)
|
|
134
|
+
rubocop (~> 1.70)
|
|
135
|
+
rubocop-performance (~> 1.24)
|
|
136
|
+
simplecov (~> 0.22)
|
|
137
|
+
yard
|
|
138
|
+
yardstick (~> 0.9)
|
|
139
|
+
|
|
140
|
+
CHECKSUMS
|
|
141
|
+
appraisal (2.5.0) sha256=36989221be127913b0dba8d114da2001e6b2dceea7bd4951200eaba764eed3ce
|
|
142
|
+
ast (2.4.3) sha256=954615157c1d6a382bc27d690d973195e79db7f55e9765ac7c481c60bdb4d383
|
|
143
|
+
bigdecimal (4.0.1) sha256=8b07d3d065a9f921c80ceaea7c9d4ae596697295b584c296fe599dd0ad01c4a7
|
|
144
|
+
csv (3.3.5) sha256=6e5134ac3383ef728b7f02725d9872934f523cb40b961479f69cf3afa6c8e73f
|
|
145
|
+
date (3.5.1) sha256=750d06384d7b9c15d562c76291407d89e368dda4d4fff957eb94962d325a0dc0
|
|
146
|
+
diff-lcs (1.6.2) sha256=9ae0d2cba7d4df3075fe8cd8602a8604993efc0dfa934cff568969efb1909962
|
|
147
|
+
docile (1.4.1) sha256=96159be799bfa73cdb721b840e9802126e4e03dfc26863db73647204c727f21e
|
|
148
|
+
erb (6.0.1) sha256=28ecdd99c5472aebd5674d6061e3c6b0a45c049578b071e5a52c2a7f13c197e5
|
|
149
|
+
importu (0.1.0)
|
|
150
|
+
io-console (0.8.2) sha256=d6e3ae7a7cc7574f4b8893b4fca2162e57a825b223a177b7afa236c5ef9814cc
|
|
151
|
+
irb (1.16.0) sha256=2abe56c9ac947cdcb2f150572904ba798c1e93c890c256f8429981a7675b0806
|
|
152
|
+
json (2.18.0) sha256=b10506aee4183f5cf49e0efc48073d7b75843ce3782c68dbeb763351c08fd505
|
|
153
|
+
language_server-protocol (3.17.0.5) sha256=fd1e39a51a28bf3eec959379985a72e296e9f9acfce46f6a79d31ca8760803cc
|
|
154
|
+
lint_roller (1.1.0) sha256=2c0c845b632a7d172cb849cc90c1bce937a28c5c8ccccb50dfd46a485003cc87
|
|
155
|
+
nokogiri (1.19.0-aarch64-linux-gnu) sha256=11a97ecc3c0e7e5edcf395720b10860ef493b768f6aa80c539573530bc933767
|
|
156
|
+
nokogiri (1.19.0-aarch64-linux-musl) sha256=eb70507f5e01bc23dad9b8dbec2b36ad0e61d227b42d292835020ff754fb7ba9
|
|
157
|
+
nokogiri (1.19.0-arm-linux-gnu) sha256=572a259026b2c8b7c161fdb6469fa2d0edd2b61cd599db4bbda93289abefbfe5
|
|
158
|
+
nokogiri (1.19.0-arm-linux-musl) sha256=23ed90922f1a38aed555d3de4d058e90850c731c5b756d191b3dc8055948e73c
|
|
159
|
+
nokogiri (1.19.0-arm64-darwin) sha256=0811dfd936d5f6dd3f6d32ef790568bf29b2b7bead9ba68866847b33c9cf5810
|
|
160
|
+
nokogiri (1.19.0-x86_64-darwin) sha256=1dad56220b603a8edb9750cd95798bffa2b8dd9dd9aa47f664009ee5b43e3067
|
|
161
|
+
nokogiri (1.19.0-x86_64-linux-gnu) sha256=f482b95c713d60031d48c44ce14562f8d2ce31e3a9e8dd0ccb131e9e5a68b58c
|
|
162
|
+
nokogiri (1.19.0-x86_64-linux-musl) sha256=1c4ca6b381622420073ce6043443af1d321e8ed93cc18b08e2666e5bd02ffae4
|
|
163
|
+
parallel (1.27.0) sha256=4ac151e1806b755fb4e2dc2332cbf0e54f2e24ba821ff2d3dcf86bf6dc4ae130
|
|
164
|
+
parser (3.3.10.1) sha256=06f6a725d2cd91e5e7f2b7c32ba143631e1f7c8ae2fb918fc4cebec187e6a688
|
|
165
|
+
pp (0.6.3) sha256=2951d514450b93ccfeb1df7d021cae0da16e0a7f95ee1e2273719669d0ab9df6
|
|
166
|
+
prettyprint (0.2.0) sha256=2bc9e15581a94742064a3cc8b0fb9d45aae3d03a1baa6ef80922627a0766f193
|
|
167
|
+
prism (1.9.0) sha256=7b530c6a9f92c24300014919c9dcbc055bf4cdf51ec30aed099b06cd6674ef85
|
|
168
|
+
psych (5.3.1) sha256=eb7a57cef10c9d70173ff74e739d843ac3b2c019a003de48447b2963d81b1974
|
|
169
|
+
racc (1.8.1) sha256=4a7f6929691dbec8b5209a0b373bc2614882b55fc5d2e447a21aaa691303d62f
|
|
170
|
+
rainbow (3.1.1) sha256=039491aa3a89f42efa1d6dec2fc4e62ede96eb6acd95e52f1ad581182b79bc6a
|
|
171
|
+
rake (13.3.1) sha256=8c9e89d09f66a26a01264e7e3480ec0607f0c497a861ef16063604b1b08eb19c
|
|
172
|
+
rdoc (7.1.0) sha256=494899df0706c178596ca6e1d50f1b7eb285a9b2aae715be5abd742734f17363
|
|
173
|
+
redcarpet (3.6.1) sha256=d444910e6aa55480c6bcdc0cdb057626e8a32c054c29e793fa642ba2f155f445
|
|
174
|
+
regexp_parser (2.11.3) sha256=ca13f381a173b7a93450e53459075c9b76a10433caadcb2f1180f2c741fc55a4
|
|
175
|
+
reline (0.6.3) sha256=1198b04973565b36ec0f11542ab3f5cfeeec34823f4e54cebde90968092b1835
|
|
176
|
+
rspec (3.13.2) sha256=206284a08ad798e61f86d7ca3e376718d52c0bc944626b2349266f239f820587
|
|
177
|
+
rspec-core (3.13.6) sha256=a8823c6411667b60a8bca135364351dda34cd55e44ff94c4be4633b37d828b2d
|
|
178
|
+
rspec-expectations (3.13.5) sha256=33a4d3a1d95060aea4c94e9f237030a8f9eae5615e9bd85718fe3a09e4b58836
|
|
179
|
+
rspec-mocks (3.13.7) sha256=0979034e64b1d7a838aaaddf12bf065ea4dc40ef3d4c39f01f93ae2c66c62b1c
|
|
180
|
+
rspec-support (3.13.6) sha256=2e8de3702427eab064c9352fe74488cc12a1bfae887ad8b91cba480ec9f8afb2
|
|
181
|
+
rubocop (1.84.0) sha256=88dec310153bb685a879f5a7cdb601f6287b8f0ee675d9dc63a17c7204c4190a
|
|
182
|
+
rubocop-ast (1.49.0) sha256=49c3676d3123a0923d333e20c6c2dbaaae2d2287b475273fddee0c61da9f71fd
|
|
183
|
+
rubocop-performance (1.26.1) sha256=cd19b936ff196df85829d264b522fd4f98b6c89ad271fa52744a8c11b8f71834
|
|
184
|
+
ruby-progressbar (1.13.0) sha256=80fc9c47a9b640d6834e0dc7b3c94c9df37f08cb072b7761e4a71e22cff29b33
|
|
185
|
+
simplecov (0.22.0) sha256=fe2622c7834ff23b98066bb0a854284b2729a569ac659f82621fc22ef36213a5
|
|
186
|
+
simplecov-html (0.13.2) sha256=bd0b8e54e7c2d7685927e8d6286466359b6f16b18cb0df47b508e8d73c777246
|
|
187
|
+
simplecov_json_formatter (0.1.4) sha256=529418fbe8de1713ac2b2d612aa3daa56d316975d307244399fa4838c601b428
|
|
188
|
+
stringio (3.2.0) sha256=c37cb2e58b4ffbd33fe5cd948c05934af997b36e0b6ca6fdf43afa234cf222e1
|
|
189
|
+
thor (1.5.0) sha256=e3a9e55fe857e44859ce104a84675ab6e8cd59c650a49106a05f55f136425e73
|
|
190
|
+
tsort (0.2.0) sha256=9650a793f6859a43b6641671278f79cfead60ac714148aabe4e3f0060480089f
|
|
191
|
+
unicode-display_width (3.2.0) sha256=0cdd96b5681a5949cdbc2c55e7b420facae74c4aaf9a9815eee1087cb1853c42
|
|
192
|
+
unicode-emoji (4.2.0) sha256=519e69150f75652e40bf736106cfbc8f0f73aa3fb6a65afe62fefa7f80b0f80f
|
|
193
|
+
yard (0.9.38) sha256=721fb82afb10532aa49860655f6cc2eaa7130889df291b052e1e6b268283010f
|
|
194
|
+
yardstick (0.9.9) sha256=c39a166a9a37c0b164dc1969c8e20fe158061079e97c95454ceb58d0515ec06e
|
|
195
|
+
|
|
196
|
+
BUNDLED WITH
|
|
197
|
+
4.0.3
|
data/importu.gemspec
CHANGED
|
@@ -1,29 +1,48 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
$LOAD_PATH.unshift File.expand_path("lib", __dir__)
|
|
3
|
+
require "importu/version"
|
|
3
4
|
|
|
4
5
|
Gem::Specification.new do |s|
|
|
5
|
-
s.name =
|
|
6
|
+
s.name = "importu"
|
|
6
7
|
s.version = Importu::VERSION
|
|
7
8
|
s.platform = Gem::Platform::RUBY
|
|
8
|
-
s.
|
|
9
|
-
s.
|
|
10
|
-
s.
|
|
11
|
-
s.
|
|
12
|
-
s.
|
|
9
|
+
s.required_ruby_version = ">= 3.1"
|
|
10
|
+
s.authors = ["Daniel Hedlund"]
|
|
11
|
+
s.email = ["daniel@digitree.org"]
|
|
12
|
+
s.homepage = "https://github.com/dhedlund/importu"
|
|
13
|
+
s.summary = "Because an import should be defined by a contract, not " +
|
|
14
|
+
"a sequence of commands"
|
|
15
|
+
s.description = "Importu simplifies the process of defining and sharing " +
|
|
16
|
+
"contracts that structured data must conform to in order " +
|
|
17
|
+
"to be importable into your application."
|
|
13
18
|
|
|
14
19
|
s.files = `git ls-files`.split("\n")
|
|
15
|
-
s.
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
s.
|
|
27
|
-
s.
|
|
28
|
-
s.
|
|
20
|
+
s.require_paths = ["lib"]
|
|
21
|
+
|
|
22
|
+
s.licenses = ["MIT"]
|
|
23
|
+
|
|
24
|
+
s.metadata = {
|
|
25
|
+
"homepage_uri" => s.homepage,
|
|
26
|
+
"source_code_uri" => s.homepage,
|
|
27
|
+
"changelog_uri" => "#{s.homepage}/blob/master/CHANGELOG.md",
|
|
28
|
+
"bug_tracker_uri" => "#{s.homepage}/issues",
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
s.add_dependency "bigdecimal"
|
|
32
|
+
s.add_dependency "csv"
|
|
33
|
+
s.add_dependency "nokogiri", ">= 1.18.4"
|
|
34
|
+
|
|
35
|
+
s.add_development_dependency "appraisal"
|
|
36
|
+
s.add_development_dependency "bundler", ">= 2.0"
|
|
37
|
+
s.add_development_dependency "rspec", "~> 3.13"
|
|
38
|
+
s.add_development_dependency "simplecov", "~> 0.22"
|
|
39
|
+
|
|
40
|
+
# Code Quality
|
|
41
|
+
s.add_development_dependency "rubocop", "~> 1.70"
|
|
42
|
+
s.add_development_dependency "rubocop-performance", "~> 1.24"
|
|
43
|
+
|
|
44
|
+
# YARD Documentation
|
|
45
|
+
s.add_development_dependency "redcarpet" # For README.md
|
|
46
|
+
s.add_development_dependency "yard"
|
|
47
|
+
|
|
29
48
|
end
|
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "importu/backends"
|
|
3
|
+
require "importu/exceptions"
|
|
4
|
+
|
|
5
|
+
# Backend for persisting records to ActiveRecord models.
|
|
6
|
+
#
|
|
7
|
+
# This backend is automatically registered and detected when your model
|
|
8
|
+
# inherits from ActiveRecord::Base.
|
|
9
|
+
#
|
|
10
|
+
# @example Basic usage in an importer
|
|
11
|
+
# class BookImporter < Importu::Importer
|
|
12
|
+
# model "Book"
|
|
13
|
+
# allow_actions :create, :update
|
|
14
|
+
# find_by :isbn
|
|
15
|
+
# end
|
|
16
|
+
#
|
|
17
|
+
# @api semipublic
|
|
18
|
+
class Importu::Backends::ActiveRecord
|
|
19
|
+
# Checks if this backend supports the given model.
|
|
20
|
+
#
|
|
21
|
+
# @param model [Class] the model class
|
|
22
|
+
# @return [Boolean] true if model inherits from ActiveRecord::Base
|
|
23
|
+
def self.supported_by_model?(model)
|
|
24
|
+
model < ActiveRecord::Base # Inherits from
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# Creates a new ActiveRecord backend.
|
|
28
|
+
#
|
|
29
|
+
# @param model [Class, String] the ActiveRecord model class or name
|
|
30
|
+
# @param finder_fields [Array] field groups for finding existing records
|
|
31
|
+
# @param before_save [Proc, nil] optional callback before saving
|
|
32
|
+
# @api private
|
|
33
|
+
def initialize(model:, finder_fields:, before_save: nil, **)
|
|
34
|
+
@model = model.is_a?(String) ? self.class.const_get(model) : model
|
|
35
|
+
@finder_fields = finder_fields
|
|
36
|
+
@before_save = before_save
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# Finds an existing record matching the import data.
|
|
40
|
+
#
|
|
41
|
+
# @param record [Importu::Record] the import record
|
|
42
|
+
# @return [ActiveRecord::Base, nil] the matching record or nil
|
|
43
|
+
def find(record)
|
|
44
|
+
@finder_fields.each do |field_group|
|
|
45
|
+
if field_group.respond_to?(:call) # proc
|
|
46
|
+
object = @model.instance_exec(record, &field_group)
|
|
47
|
+
else
|
|
48
|
+
conditions = field_group.to_h {|f| [f, record[f]]}
|
|
49
|
+
object = @model.where(conditions).first
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
return object if object
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
nil
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# The unique id representing the object in the database, if one exists.
|
|
59
|
+
def unique_id(object)
|
|
60
|
+
object.respond_to?(:id) ? object.id : nil
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Creates a new record in the database.
|
|
64
|
+
#
|
|
65
|
+
# @param record [Importu::Record] the import record
|
|
66
|
+
# @return [Array<(Symbol, ActiveRecord::Base)>] status and object
|
|
67
|
+
# @raise [Importu::InvalidRecord] if validation fails
|
|
68
|
+
def create(record)
|
|
69
|
+
object = @model.new
|
|
70
|
+
perform_assignment(record, object, :create)
|
|
71
|
+
save(record, object)
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# Updates an existing record in the database.
|
|
75
|
+
#
|
|
76
|
+
# @param record [Importu::Record] the import record
|
|
77
|
+
# @param object [ActiveRecord::Base] the existing record
|
|
78
|
+
# @return [Array<(Symbol, ActiveRecord::Base)>] status and object
|
|
79
|
+
# @raise [Importu::InvalidRecord] if validation fails
|
|
80
|
+
def update(record, object)
|
|
81
|
+
perform_assignment(record, object, :update)
|
|
82
|
+
save(record, object)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private def perform_assignment(record, object, action)
|
|
86
|
+
AssignmentContext.new(record, object, action).tap do |context|
|
|
87
|
+
context.assign_values
|
|
88
|
+
context.apply(&@before_save)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
private def save(_record, object)
|
|
93
|
+
return [:unchanged, object] unless object.changed?
|
|
94
|
+
new_record = object.new_record?
|
|
95
|
+
|
|
96
|
+
begin
|
|
97
|
+
object.save!
|
|
98
|
+
rescue ActiveRecord::RecordInvalid
|
|
99
|
+
error_msgs = object.errors.map do |error|
|
|
100
|
+
error.attribute == :base ? error.message : "#{error.attribute} #{error.message}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Normalize messages by stripping trailing parenthesized data (e.g., "(value: foo)")
|
|
104
|
+
# This allows similar errors to be aggregated in the Summary.
|
|
105
|
+
normalized_msgs = error_msgs.map { |msg| msg.gsub(/ *\([^)]+\) *$/, "") }
|
|
106
|
+
|
|
107
|
+
raise Importu::InvalidRecord.new(
|
|
108
|
+
error_msgs.join(", "),
|
|
109
|
+
object.errors.full_messages,
|
|
110
|
+
normalized_message: normalized_msgs.join(", ")
|
|
111
|
+
)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
[(new_record ? :created : :updated), object]
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
# Context for assigning field values to a model instance.
|
|
118
|
+
#
|
|
119
|
+
# This is the execution context for before_save hooks, providing access
|
|
120
|
+
# to the record data, model object, and current action.
|
|
121
|
+
#
|
|
122
|
+
# @api semipublic
|
|
123
|
+
class AssignmentContext
|
|
124
|
+
# @return [Importu::Record] the import record
|
|
125
|
+
attr_reader :record
|
|
126
|
+
|
|
127
|
+
# @return [ActiveRecord::Base] the model instance
|
|
128
|
+
attr_reader :object
|
|
129
|
+
|
|
130
|
+
# @return [Symbol] the action (:create or :update)
|
|
131
|
+
attr_reader :action
|
|
132
|
+
|
|
133
|
+
# @api private
|
|
134
|
+
def initialize(record, object, action)
|
|
135
|
+
@record, @object, @action = record, object, action
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
# Executes a block in this context.
|
|
139
|
+
#
|
|
140
|
+
# @yield block to execute with access to record, object, and action
|
|
141
|
+
# @api private
|
|
142
|
+
def apply(&block)
|
|
143
|
+
instance_eval(&block) if block
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Assigns field values from the record to the object.
|
|
147
|
+
#
|
|
148
|
+
# @raise [Importu::UnassignableFields] if fields cannot be assigned
|
|
149
|
+
# @api private
|
|
150
|
+
def assign_values
|
|
151
|
+
field_names = record.assignable_fields_for(action)
|
|
152
|
+
|
|
153
|
+
begin
|
|
154
|
+
field_names.each do |name|
|
|
155
|
+
object.send("#{name}=", record[name])
|
|
156
|
+
end
|
|
157
|
+
rescue NoMethodError
|
|
158
|
+
raise_unassignable_fields!(field_names)
|
|
159
|
+
end
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
private def raise_unassignable_fields!(field_names)
|
|
163
|
+
unassignable = field_names.reject {|n| object.respond_to?("#{n}=") }
|
|
164
|
+
raise Importu::UnassignableFields, "model does not support assigning " +
|
|
165
|
+
"all fields: " + unassignable.join(", ")
|
|
166
|
+
end
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
end
|
|
170
|
+
|
|
171
|
+
Importu::Backends.registry.register(:active_record, Importu::Backends::ActiveRecord)
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "delegate"
|
|
3
|
+
|
|
4
|
+
require "importu/backends/middleware"
|
|
5
|
+
require "importu/duplicate_manager"
|
|
6
|
+
|
|
7
|
+
# Wraps any model-based backend adapter to provide duplicate detection.
|
|
8
|
+
class Importu::Backends::Middleware::DuplicateManagerProxy < SimpleDelegator
|
|
9
|
+
|
|
10
|
+
def initialize(backend, finder_fields:, **)
|
|
11
|
+
super(backend)
|
|
12
|
+
@manager = Importu::DuplicateManager.new(finder_fields: finder_fields)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
# Before passing to the backend for create, make sure the record is not
|
|
16
|
+
# a duplicate by using the finder field information that is available.
|
|
17
|
+
# When creating a new object, we will need to record that object as
|
|
18
|
+
# encountered as soon as it has been given a unique id. Any updates
|
|
19
|
+
# to that record within the same import will be treated as duplicate.
|
|
20
|
+
def create(record)
|
|
21
|
+
@manager.check_record!(record)
|
|
22
|
+
|
|
23
|
+
result, object = super
|
|
24
|
+
|
|
25
|
+
# Record the newly created object as encountered
|
|
26
|
+
@manager.check_object!(unique_id(object))
|
|
27
|
+
|
|
28
|
+
[result, object]
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Before passing to the backend for update, make sure the record is not
|
|
32
|
+
# a duplicate by using the finder field information that is available.
|
|
33
|
+
# Also check the object's unique identifier to ensure we have not already
|
|
34
|
+
# tried to change the object from a previous record import.
|
|
35
|
+
def update(record, object)
|
|
36
|
+
@manager.check_record!(record)
|
|
37
|
+
@manager.check_object!(unique_id(object))
|
|
38
|
+
super
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
end
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "delegate"
|
|
3
|
+
|
|
4
|
+
require "importu/backends/middleware"
|
|
5
|
+
require "importu/exceptions"
|
|
6
|
+
|
|
7
|
+
# Middleware that enforces allowed actions on a backend.
|
|
8
|
+
#
|
|
9
|
+
# Wraps a backend and raises InvalidRecord if a create or update action
|
|
10
|
+
# is attempted that isn't in the allowed_actions list.
|
|
11
|
+
#
|
|
12
|
+
# @api private
|
|
13
|
+
class Importu::Backends::Middleware::EnforceAllowedActions < SimpleDelegator
|
|
14
|
+
# Creates middleware wrapping the given backend.
|
|
15
|
+
#
|
|
16
|
+
# @param backend [Object] the backend to wrap
|
|
17
|
+
# @param allowed_actions [Array<Symbol>] actions that are permitted
|
|
18
|
+
def initialize(backend, allowed_actions:, **)
|
|
19
|
+
super(backend)
|
|
20
|
+
@allowed_actions = allowed_actions
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
# Creates a record, if create action is allowed.
|
|
24
|
+
#
|
|
25
|
+
# @param record [Importu::Record] the import record
|
|
26
|
+
# @return [Array] status and object from wrapped backend
|
|
27
|
+
# @raise [Importu::InvalidRecord] if create not allowed
|
|
28
|
+
def create(record)
|
|
29
|
+
if @allowed_actions.include?(:create)
|
|
30
|
+
super
|
|
31
|
+
else
|
|
32
|
+
raise Importu::InvalidRecord,
|
|
33
|
+
"create action not allowed; add `allow_actions :create` to your importer"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Updates a record, if update action is allowed.
|
|
38
|
+
#
|
|
39
|
+
# @param record [Importu::Record] the import record
|
|
40
|
+
# @param object [Object] the existing record
|
|
41
|
+
# @return [Array] status and object from wrapped backend
|
|
42
|
+
# @raise [Importu::InvalidRecord] if update not allowed
|
|
43
|
+
def update(record, object)
|
|
44
|
+
if @allowed_actions.include?(:update)
|
|
45
|
+
super
|
|
46
|
+
else
|
|
47
|
+
raise Importu::InvalidRecord,
|
|
48
|
+
"update action not allowed; add `allow_actions :create, :update` to your importer"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
end
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
class Importu::Backends
|
|
3
|
+
# Namespace for backend middleware.
|
|
4
|
+
#
|
|
5
|
+
# Middleware wraps backends to add cross-cutting behavior like enforcing
|
|
6
|
+
# allowed actions and detecting duplicates within an import.
|
|
7
|
+
#
|
|
8
|
+
# @api private
|
|
9
|
+
module Middleware
|
|
10
|
+
end
|
|
11
|
+
end
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "importu/backends/middleware/enforce_allowed_actions"
|
|
3
|
+
require "importu/backends/middleware/duplicate_manager_proxy"
|
|
4
|
+
require "importu/exceptions"
|
|
5
|
+
|
|
6
|
+
# Registry for persistence backends.
|
|
7
|
+
#
|
|
8
|
+
# Backends handle persisting imported records to a data store. The registry
|
|
9
|
+
# allows registering custom backends and auto-detecting which backend to use
|
|
10
|
+
# based on the model class.
|
|
11
|
+
#
|
|
12
|
+
# @example Registering a custom backend
|
|
13
|
+
# Importu::Backends.registry.register(:sequel, MySequelBackend)
|
|
14
|
+
#
|
|
15
|
+
# @example Looking up a backend
|
|
16
|
+
# backend_class = Importu::Backends.registry.lookup(:active_record)
|
|
17
|
+
#
|
|
18
|
+
# @api semipublic
|
|
19
|
+
class Importu::Backends
|
|
20
|
+
# Returns the middleware classes applied to all backends.
|
|
21
|
+
#
|
|
22
|
+
# @return [Array<Class>] middleware classes in application order
|
|
23
|
+
def self.middleware
|
|
24
|
+
[
|
|
25
|
+
Importu::Backends::Middleware::EnforceAllowedActions,
|
|
26
|
+
Importu::Backends::Middleware::DuplicateManagerProxy,
|
|
27
|
+
]
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# Returns the global backend registry instance.
|
|
31
|
+
#
|
|
32
|
+
# @return [Importu::Backends] the singleton registry
|
|
33
|
+
def self.registry
|
|
34
|
+
@registry ||= new
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# Creates a new backend registry.
|
|
38
|
+
#
|
|
39
|
+
# @api private
|
|
40
|
+
def initialize
|
|
41
|
+
@registered = Hash.new do |_hash, key|
|
|
42
|
+
raise Importu::BackendNotRegistered, key
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# Resolves a backend class from importer configuration.
|
|
47
|
+
#
|
|
48
|
+
# @param name [Symbol, nil] explicit backend name, :auto, or nil for auto-detect
|
|
49
|
+
# @param model [Class, String] the model class or class name
|
|
50
|
+
# @return [Class] the backend class
|
|
51
|
+
# @raise [Importu::BackendNotRegistered] if named backend not found
|
|
52
|
+
# @raise [Importu::BackendMatchError] if auto-detection fails
|
|
53
|
+
def from_config!(name:, model:, **)
|
|
54
|
+
model = model.is_a?(String) ? self.class.const_get(model) : model
|
|
55
|
+
# Support :auto as an explicit way to request auto-detection (same as nil)
|
|
56
|
+
name && name != :auto ? lookup(name) : detect_from_model(model)
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Looks up a backend by name.
|
|
60
|
+
#
|
|
61
|
+
# @param name [Symbol, String] the backend name
|
|
62
|
+
# @return [Class] the backend class
|
|
63
|
+
# @raise [Importu::BackendNotRegistered] if not found
|
|
64
|
+
def lookup(name)
|
|
65
|
+
@registered[name.to_sym]
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
# Returns all registered backend names.
|
|
69
|
+
#
|
|
70
|
+
# @return [Array<Symbol>] registered backend names
|
|
71
|
+
def names
|
|
72
|
+
@registered.keys
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
# Registers a backend class.
|
|
76
|
+
#
|
|
77
|
+
# @param name [Symbol, String] the backend name
|
|
78
|
+
# @param klass [Class] the backend class
|
|
79
|
+
# @return [Class] the registered class
|
|
80
|
+
def register(name, klass)
|
|
81
|
+
@registered[name.to_sym] = klass
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
private def detect_from_model(model)
|
|
85
|
+
matched = @registered.select do |_name, backend|
|
|
86
|
+
backend.supported_by_model?(model) rescue false
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
if matched.one?
|
|
90
|
+
matched.values.first
|
|
91
|
+
elsif matched.none?
|
|
92
|
+
raise Importu::BackendMatchError, "No backends detected from importer " +
|
|
93
|
+
"model. Try adding `backend: :mybackend` to your model definition. " +
|
|
94
|
+
"Known values are: " + names.map {|v| ":#{v}"}.join(", ")
|
|
95
|
+
else
|
|
96
|
+
raise Importu::BackendMatchError, "Backend auto-detection is " +
|
|
97
|
+
"ambiguous, multiple candidates match. Try adding `backend: " +
|
|
98
|
+
":mybackend` to your model definition. Matched backends are: " +
|
|
99
|
+
matched.keys.map {|v| ":#{v}"}.join(", ")
|
|
100
|
+
end
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
end
|