row_boat 0.1.0.alpha.3 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +14 -1
- data/API.md +355 -0
- data/Appraisals +6 -0
- data/README.md +58 -15
- data/bin/setup +2 -0
- data/devmynd-logo.png +0 -0
- data/gemfiles/csv_import_1.1_and_0.18.gemfile +8 -0
- data/gemfiles/csv_import_1.1_and_0.18.gemfile.lock +144 -0
- data/lib/row_boat/base.rb +131 -0
- data/lib/row_boat/helpers.rb +1 -0
- data/lib/row_boat/value_converter.rb +1 -0
- data/lib/row_boat/version.rb +1 -1
- data/row_boat.gemspec +2 -0
- metadata +37 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 42b9a926cbca073ac25906f8f3ca224d2d665b62
|
4
|
+
data.tar.gz: 28e138c7db5dfa6e8ef4139c82033551da7b6dfe
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cfc989f812ad764feae56ab730d64fe5f6f5e57306eef8cf46c3940f5ba22394916bd9bd7372d2468a4f06141d9348fa82eae5a7a49f8dc47f165f0e6bf1b5ab
|
7
|
+
data.tar.gz: 7007c0b60355e42523f5f6b4158c7ae5c82c7d80e012f557dda2093841db42be017f8c54f7e142ee761764ef5f2476d0073af1d35b58b636be393e9b8ea24afc
|
data/.travis.yml
CHANGED
@@ -1,5 +1,18 @@
|
|
1
|
+
dist: trusty
|
1
2
|
sudo: false
|
2
3
|
language: ruby
|
3
4
|
rvm:
|
4
5
|
- 2.3.3
|
5
|
-
|
6
|
+
- 2.4.1
|
7
|
+
addons:
|
8
|
+
postgresql: '9.6'
|
9
|
+
before_install:
|
10
|
+
- gem update --system
|
11
|
+
- gem --version
|
12
|
+
- gem install bundler -v 1.14.6
|
13
|
+
before_script:
|
14
|
+
- bundle exec rake db:create
|
15
|
+
- bundle exec rake db:migrate
|
16
|
+
cache: bundler
|
17
|
+
gemfile:
|
18
|
+
- gemfiles/csv_import_1.1_and_0.18.gemfile
|
data/API.md
ADDED
@@ -0,0 +1,355 @@
|
|
1
|
+
# RowBoat API
|
2
|
+
|
3
|
+
This is really more of a summary of what you can do with `RowBoat::Base` since you subclass it to do everything :)
|
4
|
+
|
5
|
+
## Contents
|
6
|
+
|
7
|
+
- [Basic Usage](#basic-usage)
|
8
|
+
- [`.import`](#import)
|
9
|
+
- [`initialize`](#initialize)
|
10
|
+
- [`import`](#import-1)
|
11
|
+
- [`import_into`](#import_into)
|
12
|
+
- [`csv_source`](#csv_source)
|
13
|
+
- [`column_mapping`](#column_mapping)
|
14
|
+
- [`preprocess_row`](#preprocess_row)
|
15
|
+
- [`preprocess_rows`](#preprocess_rows)
|
16
|
+
- [`options`](#options)
|
17
|
+
- [`handle_failed_row`](#handle_failed_row)
|
18
|
+
- [`handle_failed_rows`](#handle_failed_rows)
|
19
|
+
- [`value_converters`](#value_converters)
|
20
|
+
|
21
|
+
## Basic Usage
|
22
|
+
|
23
|
+
Just subclass `RowBoat::Base` and define the [`import_into`](#import_into) and [`column_mapping`](#column_mapping) methods to get started (They're the only methods that you're required to implement).
|
24
|
+
|
25
|
+
```ruby
|
26
|
+
class ImportProduct < RowBoat::Base
|
27
|
+
def import_into
|
28
|
+
Product
|
29
|
+
end
|
30
|
+
|
31
|
+
def column_mapping
|
32
|
+
{
|
33
|
+
downcased_csv_column_header: :model_attribute_name,
|
34
|
+
another_downcased_csv_column_header: :another_model_attribute_name
|
35
|
+
}
|
36
|
+
end
|
37
|
+
end
|
38
|
+
```
|
39
|
+
|
40
|
+
## `.import`
|
41
|
+
|
42
|
+
### Description
|
43
|
+
Imports database records form the given CSV-like object. The CSV-like object can be anything that can be passed to [`SmarterCSV.process`](https://github.com/tilo/smarter_csv#documentation) (string paths to files, files, tempfiles, instances of StringIO, etc).
|
44
|
+
|
45
|
+
It returns a hash containing
|
46
|
+
|
47
|
+
- `:invalid_records` - an array of all records that failed to import since they were invalid. If you've configured the `:validate` option to be `false` it will be an empty array.
|
48
|
+
- `:total_inserted` - the total number of records inserted into the database.
|
49
|
+
- `:inserted_ids` - an array of all of the ids of records inserted into the database.
|
50
|
+
|
51
|
+
If you want to pass additional information to help import CSVs, *don't override this method*. It just passes through to [`initialize`](#initialize) so override that :)
|
52
|
+
|
53
|
+
### Example
|
54
|
+
|
55
|
+
#### Basic Use
|
56
|
+
|
57
|
+
```ruby
|
58
|
+
class ImportProduct < RowBoat::Base
|
59
|
+
# required configuration omitted for brevity
|
60
|
+
end
|
61
|
+
|
62
|
+
ImportProduct.import("path/to/my.csv")
|
63
|
+
```
|
64
|
+
|
65
|
+
#### Advanced Use
|
66
|
+
|
67
|
+
```ruby
|
68
|
+
class ImportProduct < RowBoat::Base
|
69
|
+
# required configuration omitted for brevity
|
70
|
+
def intitialize(csv_source, my_options)
|
71
|
+
super(csv_source)
|
72
|
+
@my_options = my_options
|
73
|
+
end
|
74
|
+
end
|
75
|
+
|
76
|
+
ImportProduct.import("path/to/my.csv", foo: "bar")
|
77
|
+
```
|
78
|
+
|
79
|
+
## `initialize`
|
80
|
+
|
81
|
+
### Description
|
82
|
+
|
83
|
+
Makes a new instance with the given CSV-like object. See [`.import`](#import) for more details around when and how to override this method.
|
84
|
+
|
85
|
+
## `import`
|
86
|
+
|
87
|
+
### Description
|
88
|
+
|
89
|
+
The instance method that actually parses and imports the CSV. Generally, you wouldn't call this directly and would instead call [`.import`](#import).
|
90
|
+
|
91
|
+
## `import_into`
|
92
|
+
|
93
|
+
### Description
|
94
|
+
|
95
|
+
It is required that you override this method to return whatever ActiveRecord class you want your CSV imported into.
|
96
|
+
|
97
|
+
### Example
|
98
|
+
|
99
|
+
#### Basic Use
|
100
|
+
|
101
|
+
```ruby
|
102
|
+
class ImportProduct < RowBoat::Base
|
103
|
+
# other required configuration omitted for brevity
|
104
|
+
def import_into
|
105
|
+
Product
|
106
|
+
end
|
107
|
+
end
|
108
|
+
```
|
109
|
+
|
110
|
+
#### Advanced Use
|
111
|
+
|
112
|
+
```ruby
|
113
|
+
class ImportProduct < RowBoat::Base
|
114
|
+
# other required configuration omitted for brevity
|
115
|
+
def import_into
|
116
|
+
if csv_source.is_a?(String) && csv_source.match(/category/i)
|
117
|
+
ProductCategory
|
118
|
+
else
|
119
|
+
Product
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
ImportProduct.import("path/to/category.csv")
|
125
|
+
ImportProduct.import("path/to/product.csv")
|
126
|
+
```
|
127
|
+
|
128
|
+
|
129
|
+
## `csv_source`
|
130
|
+
|
131
|
+
### Description
|
132
|
+
|
133
|
+
Whatever you originally passed in as the CSV source.
|
134
|
+
|
135
|
+
### Example
|
136
|
+
|
137
|
+
```ruby
|
138
|
+
class ImportProduct < RowBoat::Base
|
139
|
+
# other required configuration omitted for brevity
|
140
|
+
def import_into
|
141
|
+
# `csv_source` is available in any of our instance methods
|
142
|
+
if csv_source.is_a?(String) && csv_source.match(/category/i)
|
143
|
+
ProductCategory
|
144
|
+
else
|
145
|
+
Product
|
146
|
+
end
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
ImportProduct.import("path/to/category.csv")
|
151
|
+
ImportProduct.import("path/to/product.csv")
|
152
|
+
```
|
153
|
+
|
154
|
+
## `column_mapping`
|
155
|
+
|
156
|
+
### Description
|
157
|
+
|
158
|
+
It is required that you override this method with a hash that maps columns in your CSV to their preferred names.
|
159
|
+
|
160
|
+
By default
|
161
|
+
- CSV column names are downcased symbols of what they look like in the CSV.
|
162
|
+
- CSV columns that are not mapped are ignored when processing the CSV.
|
163
|
+
|
164
|
+
If you're familiar with [SmarterCSV](https://github.com/tilo/smarter_csv#documentation), this method essentially defines your `:key_mapping` and with the `:remove_unmapped_keys` setting set to `true`.
|
165
|
+
|
166
|
+
You can change these defaults by overriding the [`options`](#options) method.
|
167
|
+
|
168
|
+
### Example
|
169
|
+
|
170
|
+
```ruby
|
171
|
+
class ImportProduct < RowBoat::Base
|
172
|
+
# other required configuration omitted for brevity
|
173
|
+
def column_mapping
|
174
|
+
{
|
175
|
+
prdct_nm: :name,
|
176
|
+
"price/cost_amnt": :price_in_cents
|
177
|
+
}
|
178
|
+
end
|
179
|
+
end
|
180
|
+
```
|
181
|
+
|
182
|
+
## `preprocess_row`
|
183
|
+
|
184
|
+
### Description
|
185
|
+
|
186
|
+
Implement this method if you need to do some work on the row before the record is inserted/updated.
|
187
|
+
|
188
|
+
If you return `nil` from this method, the row will be skipped in the import.
|
189
|
+
|
190
|
+
If the work you intend to do with the row only requires changing one attribute, it is recommended that you override [`value_converters`](#value_converters) instead of this.
|
191
|
+
|
192
|
+
### Example
|
193
|
+
|
194
|
+
```ruby
|
195
|
+
class ImportProduct < RowBoat::Base
|
196
|
+
# required configuration omitted for brevity
|
197
|
+
def preprocess_row(row)
|
198
|
+
{ default: :value }.merge(row)
|
199
|
+
end
|
200
|
+
# or...
|
201
|
+
def preprocess_row(row)
|
202
|
+
if row[:name] && row[:price]
|
203
|
+
row
|
204
|
+
else
|
205
|
+
nil
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
```
|
210
|
+
|
211
|
+
## `preprocess_rows`
|
212
|
+
|
213
|
+
### Description
|
214
|
+
|
215
|
+
Override this method if you need to do something with a chunk of rows (the chunk size is determined by the `:chunk_size` option in the [`options`](#options) method).
|
216
|
+
|
217
|
+
If you need to filter particular rows, it's better to just implement [`preprocess_row`](#preprocess_row) and return `nil` for the rows you want to ignore.
|
218
|
+
|
219
|
+
### Example
|
220
|
+
|
221
|
+
```ruby
|
222
|
+
class ImportProduct < RowBoat::Base
|
223
|
+
# required configuration omitted for brevity
|
224
|
+
def preprocess_rows(rows)
|
225
|
+
if skip_batch?(rows)
|
226
|
+
super([])
|
227
|
+
else
|
228
|
+
super
|
229
|
+
end
|
230
|
+
end
|
231
|
+
|
232
|
+
def skip_batch?(rows)
|
233
|
+
# decide whether or not to skip the batch
|
234
|
+
end
|
235
|
+
end
|
236
|
+
```
|
237
|
+
|
238
|
+
## `options`
|
239
|
+
|
240
|
+
### Description
|
241
|
+
|
242
|
+
Implement this to configure RowBoat, [SmarterCSV](https://github.com/tilo/smarter_csv), and [activerecord-import](https://github.com/zdennis/activerecord-import).
|
243
|
+
|
244
|
+
Except for `:wrap_in_transaction`, all options pass through to SmarterCSV and activerecord-import.
|
245
|
+
|
246
|
+
`:wrap_in_transaction` simply tells RowBoat whether or not you want your whole import wrapped in a database transaction.
|
247
|
+
|
248
|
+
Whatever you define in this method will be merged into the defaults:
|
249
|
+
|
250
|
+
- `:chunk_size` - `500`
|
251
|
+
- `:key_mapping` - `column_mapping`
|
252
|
+
- `:recursive` - `true`
|
253
|
+
- `:remove_unmapped_keys` - `true`
|
254
|
+
- `:validate` - `true`
|
255
|
+
- `:value_converters` - `csv_value_converters`
|
256
|
+
- `:wrap_in_transaction` - `true`
|
257
|
+
|
258
|
+
Don't provide `value_converters` or `key_mapping` options here. Implement the [`value_converters`](#value_converters) and [`column_mapping`](#column_mapping) respectively.
|
259
|
+
|
260
|
+
### Example
|
261
|
+
|
262
|
+
```ruby
|
263
|
+
class ImportProduct < RowBoat::Base
|
264
|
+
# required configuration omitted for brevity
|
265
|
+
def options
|
266
|
+
{
|
267
|
+
chunk_size: 1000,
|
268
|
+
validate: false,
|
269
|
+
wrap_in_transaction: false
|
270
|
+
}
|
271
|
+
end
|
272
|
+
end
|
273
|
+
```
|
274
|
+
|
275
|
+
## `handle_failed_row`
|
276
|
+
|
277
|
+
### Description
|
278
|
+
|
279
|
+
Implement this to do some work with a row that has failed to import.
|
280
|
+
|
281
|
+
It's important to note that
|
282
|
+
- This happens after the import has completed.
|
283
|
+
- The given row is an instance of whatever class was returned by [`import_into`](#import_into).
|
284
|
+
|
285
|
+
These records are also available in the return value of [`.import`](#import).
|
286
|
+
|
287
|
+
### Example
|
288
|
+
|
289
|
+
```ruby
|
290
|
+
class ImportProduct < RowBoat::Base
|
291
|
+
# required configuration omitted for brevity
|
292
|
+
def handle_failed_row(row)
|
293
|
+
puts row.errors.full_messages.join(", ")
|
294
|
+
end
|
295
|
+
end
|
296
|
+
```
|
297
|
+
|
298
|
+
## `handle_failed_rows`
|
299
|
+
|
300
|
+
### Description
|
301
|
+
|
302
|
+
Override this method to do some work will all of the rows that failed to import.
|
303
|
+
|
304
|
+
### Example
|
305
|
+
|
306
|
+
```ruby
|
307
|
+
class ImportProduct < RowBoat::Base
|
308
|
+
# required configuration omitted for brevity
|
309
|
+
def handle_failed_rows(rows)
|
310
|
+
puts "Failed to import #{rows.size} rows :("
|
311
|
+
super
|
312
|
+
end
|
313
|
+
end
|
314
|
+
```
|
315
|
+
|
316
|
+
## `value_converters`
|
317
|
+
|
318
|
+
### Description
|
319
|
+
|
320
|
+
Implement to specify how to translate values from the CSV into whatever sorts of objects you need.
|
321
|
+
|
322
|
+
Simply return a hash that has the mapped column name (ie, what you mapped it to in the [`column_mapping`](#column_mapping) method) as a key pointing to either
|
323
|
+
- a method name as a symbol
|
324
|
+
- a proc or lambda
|
325
|
+
- an object that implements `convert`
|
326
|
+
|
327
|
+
Regardless of which one you choose, it takes a value and returns a converted value.
|
328
|
+
|
329
|
+
This is essentially a sugared up version of `:value_converters` option in [SmarterCSV](https://github.com/tilo/smarter_csv#documentation).
|
330
|
+
|
331
|
+
### Example
|
332
|
+
|
333
|
+
```ruby
|
334
|
+
class ImportProduct < RowBoat::Base
|
335
|
+
# required configuration omitted for brevity
|
336
|
+
def value_converters
|
337
|
+
{
|
338
|
+
sell_by: :convert_date,
|
339
|
+
name: -> (value) { value.titlelize },
|
340
|
+
price: proc { |value| value.to_i },
|
341
|
+
description: DescriptionConverter
|
342
|
+
}
|
343
|
+
end
|
344
|
+
|
345
|
+
def convert_date(value)
|
346
|
+
Date.parse(value) rescue nil
|
347
|
+
end
|
348
|
+
end
|
349
|
+
|
350
|
+
module DescriptionConverter
|
351
|
+
def self.convert(value)
|
352
|
+
value.present? ? value : "default description :("
|
353
|
+
end
|
354
|
+
end
|
355
|
+
```
|
data/Appraisals
ADDED
data/README.md
CHANGED
@@ -1,13 +1,29 @@
|
|
1
1
|
# RowBoat
|
2
2
|
|
3
|
+
Created by [<img src="https://raw.githubusercontent.com/devmynd/row_boat/master/devmynd-logo.png" alt="DevMynd Logo" />](https://www.devmynd.com/)
|
4
|
+
|
5
|
+
[![Gem Version](https://badge.fury.io/rb/row_boat.svg)](http://badge.fury.io/rb/row_boat) [![Build Status](https://travis-ci.org/devmynd/row_boat.svg?branch=master)](https://travis-ci.org/devmynd/row_boat)
|
6
|
+
|
3
7
|
A simple gem to help you import CSVs into your ActiveRecord models.
|
4
8
|
|
9
|
+
[Check out the documentation!](/API.md#rowboat-api)
|
10
|
+
|
11
|
+
It uses [SmarterCSV](https://github.com/tilo/smarter_csv) and [`activerecord-import`](https://github.com/zdennis/activerecord-import) to import database records from your CSVs.
|
12
|
+
|
13
|
+
## Contents
|
14
|
+
|
15
|
+
- [Installation](#installation)
|
16
|
+
- [Basic Usage](#basic-usage)
|
17
|
+
- [Development](#development)
|
18
|
+
- [Contributing](#contributing)
|
19
|
+
- [License](#license)
|
20
|
+
|
5
21
|
## Installation
|
6
22
|
|
7
23
|
Add this line to your application's Gemfile:
|
8
24
|
|
9
25
|
```ruby
|
10
|
-
gem "row_boat"
|
26
|
+
gem "row_boat", "~> 0.1"
|
11
27
|
```
|
12
28
|
|
13
29
|
And then execute:
|
@@ -18,45 +34,72 @@ Or install it yourself as:
|
|
18
34
|
|
19
35
|
$ gem install row_boat
|
20
36
|
|
21
|
-
## Usage
|
37
|
+
## Basic Usage
|
38
|
+
|
39
|
+
#### [Full documentation can be found here.](/API.md#rowboat-api)
|
40
|
+
|
41
|
+
Below we're defining the required methods ([`import_into`](/API.md#import_into) and [`column_mapping`](/API.md#column_mapping)) and a few additional options as well (via [`value_converters`](/API.md#value_converters) and [`options`](/API.md#options)). Checkout [API.md](/API.md#rowboat-api) for the full documentation for more :)
|
22
42
|
|
23
43
|
```ruby
|
24
|
-
class
|
44
|
+
class ImportProduct
|
25
45
|
# required
|
26
46
|
def import_into
|
27
|
-
Product
|
47
|
+
Product # The ActiveRecord class we want to import records into.
|
28
48
|
end
|
29
49
|
|
30
50
|
# required
|
31
51
|
def column_mapping
|
32
52
|
{
|
33
|
-
|
34
|
-
|
53
|
+
# `:prdct_name` is the downcased and symbolized version
|
54
|
+
# of our column header, while `:name` is the attribute
|
55
|
+
# of our model we want to receive `:prdct_name`'s value
|
56
|
+
prdct_name: :name,
|
57
|
+
dllr_amnt: :price_in_cents,
|
58
|
+
desc: :description
|
35
59
|
}
|
36
60
|
end
|
37
61
|
|
38
62
|
# optional
|
39
|
-
def
|
40
|
-
|
63
|
+
def value_converters
|
64
|
+
{
|
65
|
+
# Allows us to change values we want to import
|
66
|
+
# before we import them
|
67
|
+
price_in_cents: -> (value) { value * 1000 }
|
68
|
+
}
|
41
69
|
end
|
42
|
-
end
|
43
70
|
|
44
|
-
#
|
71
|
+
# optional
|
72
|
+
def preprocess_row(row)
|
73
|
+
if row[:name] && row[:description] && row[:price]
|
74
|
+
row
|
75
|
+
else
|
76
|
+
nil # return nil to skip a row
|
77
|
+
end
|
78
|
+
# we could also remove some attributes or do any
|
79
|
+
# other kind of work we want with the given row.
|
80
|
+
end
|
45
81
|
|
46
|
-
|
82
|
+
#optional
|
83
|
+
def options
|
84
|
+
{
|
85
|
+
# These are additional configurations that
|
86
|
+
# are generally passed through to SmarterCSV
|
87
|
+
# and activerecord-import
|
88
|
+
validate: false, # this defaults to `true`
|
89
|
+
wrap_in_transaction: false # this defaults to `true`
|
90
|
+
}
|
91
|
+
end
|
92
|
+
end
|
47
93
|
```
|
48
94
|
|
49
95
|
## Development
|
50
96
|
|
51
|
-
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
52
|
-
|
53
|
-
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
97
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake` to run the tests (run `appraisal install` and `appraisal rake` to run the tests against different combinations of dependencies). You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
54
98
|
|
55
99
|
## Contributing
|
56
100
|
|
57
101
|
Bug reports and pull requests are welcome on GitHub at https://github.com/devmynd/row_boat. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
|
58
102
|
|
59
|
-
|
60
103
|
## License
|
61
104
|
|
62
105
|
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
data/bin/setup
CHANGED
data/devmynd-logo.png
ADDED
Binary file
|
@@ -0,0 +1,144 @@
|
|
1
|
+
PATH
|
2
|
+
remote: ..
|
3
|
+
specs:
|
4
|
+
row_boat (0.1.0)
|
5
|
+
activerecord (>= 5.0.0)
|
6
|
+
activerecord-import (~> 0.18.2)
|
7
|
+
smarter_csv (~> 1.1)
|
8
|
+
|
9
|
+
GEM
|
10
|
+
remote: https://rubygems.org/
|
11
|
+
specs:
|
12
|
+
actionpack (5.1.0)
|
13
|
+
actionview (= 5.1.0)
|
14
|
+
activesupport (= 5.1.0)
|
15
|
+
rack (~> 2.0)
|
16
|
+
rack-test (~> 0.6.3)
|
17
|
+
rails-dom-testing (~> 2.0)
|
18
|
+
rails-html-sanitizer (~> 1.0, >= 1.0.2)
|
19
|
+
actionview (5.1.0)
|
20
|
+
activesupport (= 5.1.0)
|
21
|
+
builder (~> 3.1)
|
22
|
+
erubi (~> 1.4)
|
23
|
+
rails-dom-testing (~> 2.0)
|
24
|
+
rails-html-sanitizer (~> 1.0, >= 1.0.3)
|
25
|
+
activemodel (5.1.0)
|
26
|
+
activesupport (= 5.1.0)
|
27
|
+
activerecord (5.1.0)
|
28
|
+
activemodel (= 5.1.0)
|
29
|
+
activesupport (= 5.1.0)
|
30
|
+
arel (~> 8.0)
|
31
|
+
activerecord-import (0.18.3)
|
32
|
+
activerecord (>= 3.2)
|
33
|
+
activesupport (5.1.0)
|
34
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
35
|
+
i18n (~> 0.7)
|
36
|
+
minitest (~> 5.1)
|
37
|
+
tzinfo (~> 1.1)
|
38
|
+
appraisal (2.2.0)
|
39
|
+
bundler
|
40
|
+
rake
|
41
|
+
thor (>= 0.14.0)
|
42
|
+
arel (8.0.0)
|
43
|
+
ast (2.3.0)
|
44
|
+
awesome_print (1.7.0)
|
45
|
+
builder (3.2.3)
|
46
|
+
coderay (1.1.1)
|
47
|
+
concurrent-ruby (1.0.5)
|
48
|
+
database_cleaner (1.6.1)
|
49
|
+
diff-lcs (1.3)
|
50
|
+
erubi (1.6.0)
|
51
|
+
i18n (0.8.1)
|
52
|
+
loofah (2.0.3)
|
53
|
+
nokogiri (>= 1.5.9)
|
54
|
+
method_source (0.8.2)
|
55
|
+
mini_portile2 (2.1.0)
|
56
|
+
minitest (5.10.2)
|
57
|
+
nokogiri (1.7.2)
|
58
|
+
mini_portile2 (~> 2.1.0)
|
59
|
+
parser (2.4.0.0)
|
60
|
+
ast (~> 2.2)
|
61
|
+
pg (0.20.0)
|
62
|
+
powerpack (0.1.1)
|
63
|
+
pry (0.10.4)
|
64
|
+
coderay (~> 1.1.0)
|
65
|
+
method_source (~> 0.8.1)
|
66
|
+
slop (~> 3.4)
|
67
|
+
pry-doc (0.10.0)
|
68
|
+
pry (~> 0.9)
|
69
|
+
yard (~> 0.9)
|
70
|
+
pry-nav (0.2.4)
|
71
|
+
pry (>= 0.9.10, < 0.11.0)
|
72
|
+
rack (2.0.2)
|
73
|
+
rack-test (0.6.3)
|
74
|
+
rack (>= 1.0)
|
75
|
+
rails-dom-testing (2.0.3)
|
76
|
+
activesupport (>= 4.2.0)
|
77
|
+
nokogiri (>= 1.6)
|
78
|
+
rails-html-sanitizer (1.0.3)
|
79
|
+
loofah (~> 2.0)
|
80
|
+
railties (5.1.0)
|
81
|
+
actionpack (= 5.1.0)
|
82
|
+
activesupport (= 5.1.0)
|
83
|
+
method_source
|
84
|
+
rake (>= 0.8.7)
|
85
|
+
thor (>= 0.18.1, < 2.0)
|
86
|
+
rainbow (2.2.2)
|
87
|
+
rake
|
88
|
+
rake (10.5.0)
|
89
|
+
rspec (3.6.0)
|
90
|
+
rspec-core (~> 3.6.0)
|
91
|
+
rspec-expectations (~> 3.6.0)
|
92
|
+
rspec-mocks (~> 3.6.0)
|
93
|
+
rspec-core (3.6.0)
|
94
|
+
rspec-support (~> 3.6.0)
|
95
|
+
rspec-expectations (3.6.0)
|
96
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
97
|
+
rspec-support (~> 3.6.0)
|
98
|
+
rspec-mocks (3.6.0)
|
99
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
100
|
+
rspec-support (~> 3.6.0)
|
101
|
+
rspec-support (3.6.0)
|
102
|
+
rubocop (0.48.1)
|
103
|
+
parser (>= 2.3.3.1, < 3.0)
|
104
|
+
powerpack (~> 0.1)
|
105
|
+
rainbow (>= 1.99.1, < 3.0)
|
106
|
+
ruby-progressbar (~> 1.7)
|
107
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
108
|
+
ruby-progressbar (1.8.1)
|
109
|
+
slop (3.6.0)
|
110
|
+
smarter_csv (1.1.4)
|
111
|
+
standalone_migrations (5.2.1)
|
112
|
+
activerecord (>= 4.2.7, < 5.2.0)
|
113
|
+
railties (>= 4.2.7, < 5.2.0)
|
114
|
+
rake (~> 10.0)
|
115
|
+
thor (0.19.4)
|
116
|
+
thread_safe (0.3.6)
|
117
|
+
tzinfo (1.2.3)
|
118
|
+
thread_safe (~> 0.1)
|
119
|
+
unicode-display_width (1.2.1)
|
120
|
+
yard (0.9.9)
|
121
|
+
|
122
|
+
PLATFORMS
|
123
|
+
ruby
|
124
|
+
|
125
|
+
DEPENDENCIES
|
126
|
+
activerecord-import (~> 0.18.2)
|
127
|
+
appraisal (~> 2.2.0)
|
128
|
+
awesome_print
|
129
|
+
bundler (~> 1.14)
|
130
|
+
database_cleaner (~> 1.6.0)
|
131
|
+
pg
|
132
|
+
pry
|
133
|
+
pry-doc
|
134
|
+
pry-nav
|
135
|
+
rake (~> 10.0)
|
136
|
+
row_boat!
|
137
|
+
rspec (~> 3.0)
|
138
|
+
rubocop (~> 0.48.1)
|
139
|
+
smarter_csv (~> 1.1.0)
|
140
|
+
standalone_migrations (~> 5.2.0)
|
141
|
+
yard (~> 0.9.9)
|
142
|
+
|
143
|
+
BUNDLED WITH
|
144
|
+
1.14.6
|
data/lib/row_boat/base.rb
CHANGED
@@ -9,15 +9,36 @@ module RowBoat
|
|
9
9
|
attr_reader :csv_source
|
10
10
|
|
11
11
|
class << self
|
12
|
+
# Imports database records from the given CSV-like object.
|
13
|
+
#
|
14
|
+
# @overload import(csv_source)
|
15
|
+
# @param csv_source [String, #read] a CSV-like object that SmarterCSV can read.
|
16
|
+
#
|
17
|
+
# @return [Hash] a hash with +:invalid_records+, +:total_inserted+ and +:inserted_ids+.
|
18
|
+
#
|
19
|
+
# @see https://github.com/tilo/smarter_csv#documentation SmarterCSV Docs
|
12
20
|
def import(*args, &block)
|
13
21
|
new(*args, &block).import
|
14
22
|
end
|
15
23
|
end
|
16
24
|
|
25
|
+
# Makes a new instance with the given +csv_source+.
|
26
|
+
#
|
27
|
+
# @abstract Override this method if you need additional arguments to process your CSV (like defaults).
|
28
|
+
#
|
29
|
+
# @example
|
30
|
+
# def initialize(csv_source, default_name)
|
31
|
+
# super(csv_source)
|
32
|
+
# @default_name = default_name
|
33
|
+
# end
|
17
34
|
def initialize(csv_source)
|
18
35
|
@csv_source = csv_source
|
19
36
|
end
|
20
37
|
|
38
|
+
# Parses the csv and inserts/updates the database. You probably won't call this method directly,
|
39
|
+
# instead you would call {RowBoat::Base.import}.
|
40
|
+
#
|
41
|
+
# @return [Hash] a hash with +:invalid_records+, +:total_inserted+ and +:inserted_ids+.
|
21
42
|
def import
|
22
43
|
import_results = []
|
23
44
|
|
@@ -32,24 +53,71 @@ module RowBoat
|
|
32
53
|
end
|
33
54
|
end
|
34
55
|
|
56
|
+
# Override with the ActiveRecord class that the CSV should be imported into.
|
57
|
+
#
|
58
|
+
# @abstract
|
59
|
+
#
|
60
|
+
# @note You must implement this method.
|
61
|
+
#
|
62
|
+
# @example
|
63
|
+
# def import_into
|
64
|
+
# Product
|
65
|
+
# end
|
35
66
|
def import_into
|
36
67
|
raise NotImplementedError, not_implemented_error_message(__method__)
|
37
68
|
end
|
38
69
|
|
70
|
+
# Override with a hash that maps CSV column names to their preferred names.
|
71
|
+
# Oftentimes these are the names of the attributes on the model class from {#import_into}.
|
72
|
+
#
|
73
|
+
# @abstract
|
74
|
+
#
|
75
|
+
# @note You must implement this method.
|
76
|
+
#
|
77
|
+
# @example
|
78
|
+
# def column_mapping
|
79
|
+
# {
|
80
|
+
# prdct_name: :name,
|
81
|
+
# price: :price,
|
82
|
+
# sl_exp: :sale_expires_at
|
83
|
+
# }
|
84
|
+
# end
|
85
|
+
#
|
86
|
+
# @see #import_into
|
39
87
|
def column_mapping
|
40
88
|
raise NotImplementedError, not_implemented_error_message(__method__)
|
41
89
|
end
|
42
90
|
|
91
|
+
# Override this method if you need to do some work on the row before the record is
|
92
|
+
# inserted/updated or want to skip the row in the import. Simply return +nil+ to skip the row.
|
93
|
+
#
|
94
|
+
# @abstract
|
95
|
+
#
|
96
|
+
# @note If you only need to manipulate one attribute (ie parse a date from a string, etc), then
|
97
|
+
# you should probably use {#value_converters}
|
98
|
+
#
|
99
|
+
# @return [Hash,NilClass] a hash of attributes, +nil+, or even and instance of the class returned
|
100
|
+
# in {#import_into}.
|
101
|
+
#
|
102
|
+
# @see #import_into
|
43
103
|
def preprocess_row(row)
|
44
104
|
row
|
45
105
|
end
|
46
106
|
|
107
|
+
# @api private
|
47
108
|
def import_rows(rows)
|
48
109
|
import_options = ::RowBoat::Helpers.extract_import_options(merged_options)
|
49
110
|
preprocessed_rows = preprocess_rows(rows)
|
50
111
|
import_into.import(preprocessed_rows, import_options)
|
51
112
|
end
|
52
113
|
|
114
|
+
# Override this method if you need to do something with a chunk of rows.
|
115
|
+
#
|
116
|
+
# @abstract
|
117
|
+
#
|
118
|
+
# @note If you want to filter out a row, you can just return +nil+ from {#preprocess_row}.
|
119
|
+
#
|
120
|
+
# @see #preprocess_row
|
53
121
|
def preprocess_rows(rows)
|
54
122
|
rows.each_with_object([]) do |row, preprocessed_rows|
|
55
123
|
preprocessed_row = preprocess_row(row)
|
@@ -57,10 +125,31 @@ module RowBoat
|
|
57
125
|
end
|
58
126
|
end
|
59
127
|
|
128
|
+
# Override this method to specify CSV parsing and importing options.
|
129
|
+
# All SmarterCSV and activerecord-import options can be listed here along with
|
130
|
+
# +:wrap_in_transaction+. The defaults provided by RowBoat can be found in {#default_options}
|
131
|
+
#
|
132
|
+
# @abstract
|
133
|
+
#
|
134
|
+
# @note If you want to use the +:value_converters+ option provided by SmarterCSV
|
135
|
+
# just override {#value_converters}.
|
136
|
+
#
|
137
|
+
# @return [Hash] a hash of configuration options.
|
138
|
+
#
|
139
|
+
# @see https://github.com/tilo/smarter_csv#documentation SmarterCSV docs
|
140
|
+
# @see https://github.com/zdennis/activerecord-import/wiki activerecord-import docs
|
141
|
+
# @see #value_converters
|
60
142
|
def options
|
61
143
|
{}
|
62
144
|
end
|
63
145
|
|
146
|
+
# Default options provided by RowBoat for CSV parsing and importing.
|
147
|
+
#
|
148
|
+
# @note Do not override.
|
149
|
+
#
|
150
|
+
# @return [Hash] a hash of configuration options.
|
151
|
+
#
|
152
|
+
# @api private
|
64
153
|
def default_options
|
65
154
|
{
|
66
155
|
chunk_size: 500,
|
@@ -73,22 +162,56 @@ module RowBoat
|
|
73
162
|
}
|
74
163
|
end
|
75
164
|
|
165
|
+
# @api private
|
76
166
|
def merged_options
|
77
167
|
default_options.merge(options)
|
78
168
|
end
|
79
169
|
|
170
|
+
# Override this method to do some work with a row that has failed to import.
|
171
|
+
#
|
172
|
+
# @abstract
|
173
|
+
#
|
174
|
+
# @note +row+ here is actually an instance of the class returned in {#import_into}
|
175
|
+
#
|
176
|
+
# @see #import_into
|
80
177
|
def handle_failed_row(row)
|
81
178
|
row
|
82
179
|
end
|
83
180
|
|
181
|
+
# Override this method to do some work will all of the rows that failed to import.
|
182
|
+
#
|
183
|
+
# @abstract
|
184
|
+
#
|
185
|
+
# @note If you override this method and {#handle_failed_row}, be sure to call +super+.
|
84
186
|
def handle_failed_rows(rows)
|
85
187
|
rows.each { |row| handle_failed_row(row) }
|
86
188
|
end
|
87
189
|
|
190
|
+
# Override this method to specify how to translate values from the CSV
|
191
|
+
# into ruby objects.
|
192
|
+
#
|
193
|
+
# You can provide an object that implements +convert+, a proc or lambda, or the
|
194
|
+
# the name of a method as a Symbol
|
195
|
+
#
|
196
|
+
# @abstract
|
197
|
+
#
|
198
|
+
# @example
|
199
|
+
# def value_converters
|
200
|
+
# {
|
201
|
+
# name: -> (value) { value.titleize }
|
202
|
+
# price: :convert_price,
|
203
|
+
# expires_at: CustomDateConverter
|
204
|
+
# }
|
205
|
+
# end
|
206
|
+
#
|
207
|
+
# def convert_price(value)
|
208
|
+
# value || 0
|
209
|
+
# end
|
88
210
|
def value_converters
|
89
211
|
{}
|
90
212
|
end
|
91
213
|
|
214
|
+
# @api private
|
92
215
|
def csv_value_converters
|
93
216
|
value_converters.each_with_object({}) do |(key, potential_converter), converters_hash|
|
94
217
|
case potential_converter
|
@@ -106,19 +229,27 @@ module RowBoat
|
|
106
229
|
|
107
230
|
private
|
108
231
|
|
232
|
+
# @api private
|
233
|
+
# @private
|
109
234
|
def not_implemented_error_message(method_name)
|
110
235
|
"Subclasses of #{self.class.name} must implement `#{method_name}`"
|
111
236
|
end
|
112
237
|
|
238
|
+
# @api private
|
239
|
+
# @private
|
113
240
|
def parse_rows(&block)
|
114
241
|
csv_options = ::RowBoat::Helpers.extract_csv_options(merged_options)
|
115
242
|
::SmarterCSV.process(csv_source, csv_options, &block)
|
116
243
|
end
|
117
244
|
|
245
|
+
# @api private
|
246
|
+
# @private
|
118
247
|
def transaction_if_needed(&block)
|
119
248
|
merged_options[:wrap_in_transaction] ? import_into.transaction(&block) : yield
|
120
249
|
end
|
121
250
|
|
251
|
+
# @api private
|
252
|
+
# @private
|
122
253
|
def process_import_results(import_results)
|
123
254
|
import_results.each_with_object(
|
124
255
|
invalid_records: [],
|
data/lib/row_boat/helpers.rb
CHANGED
data/lib/row_boat/version.rb
CHANGED
data/row_boat.gemspec
CHANGED
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
|
|
27
27
|
spec.add_dependency "activerecord-import", "~> 0.18.2"
|
28
28
|
spec.add_dependency "smarter_csv", "~> 1.1"
|
29
29
|
|
30
|
+
spec.add_development_dependency "appraisal", "~> 2.2.0"
|
30
31
|
spec.add_development_dependency "awesome_print"
|
31
32
|
spec.add_development_dependency "bundler", "~> 1.14"
|
32
33
|
spec.add_development_dependency "database_cleaner", "~> 1.6.0"
|
@@ -38,4 +39,5 @@ Gem::Specification.new do |spec|
|
|
38
39
|
spec.add_development_dependency "rspec", "~> 3.0"
|
39
40
|
spec.add_development_dependency "rubocop", "~> 0.48.1"
|
40
41
|
spec.add_development_dependency "standalone_migrations", "~> 5.2.0"
|
42
|
+
spec.add_development_dependency "yard", "~> 0.9.9"
|
41
43
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: row_boat
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Crismali
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-05-
|
11
|
+
date: 2017-05-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activerecord
|
@@ -52,6 +52,20 @@ dependencies:
|
|
52
52
|
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '1.1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: appraisal
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: 2.2.0
|
62
|
+
type: :development
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: 2.2.0
|
55
69
|
- !ruby/object:Gem::Dependency
|
56
70
|
name: awesome_print
|
57
71
|
requirement: !ruby/object:Gem::Requirement
|
@@ -206,6 +220,20 @@ dependencies:
|
|
206
220
|
- - "~>"
|
207
221
|
- !ruby/object:Gem::Version
|
208
222
|
version: 5.2.0
|
223
|
+
- !ruby/object:Gem::Dependency
|
224
|
+
name: yard
|
225
|
+
requirement: !ruby/object:Gem::Requirement
|
226
|
+
requirements:
|
227
|
+
- - "~>"
|
228
|
+
- !ruby/object:Gem::Version
|
229
|
+
version: 0.9.9
|
230
|
+
type: :development
|
231
|
+
prerelease: false
|
232
|
+
version_requirements: !ruby/object:Gem::Requirement
|
233
|
+
requirements:
|
234
|
+
- - "~>"
|
235
|
+
- !ruby/object:Gem::Version
|
236
|
+
version: 0.9.9
|
209
237
|
description: Turn the rows of your CSV into rows in your database
|
210
238
|
email:
|
211
239
|
- michael@crismali.com
|
@@ -218,6 +246,8 @@ files:
|
|
218
246
|
- ".rubocop.yml"
|
219
247
|
- ".ruby-version"
|
220
248
|
- ".travis.yml"
|
249
|
+
- API.md
|
250
|
+
- Appraisals
|
221
251
|
- CODE_OF_CONDUCT.md
|
222
252
|
- Gemfile
|
223
253
|
- LICENSE.txt
|
@@ -228,6 +258,9 @@ files:
|
|
228
258
|
- db/config.yml
|
229
259
|
- db/migrate/20170511160154_create_products.rb
|
230
260
|
- db/schema.rb
|
261
|
+
- devmynd-logo.png
|
262
|
+
- gemfiles/csv_import_1.1_and_0.18.gemfile
|
263
|
+
- gemfiles/csv_import_1.1_and_0.18.gemfile.lock
|
231
264
|
- lib/row_boat.rb
|
232
265
|
- lib/row_boat/base.rb
|
233
266
|
- lib/row_boat/helpers.rb
|
@@ -249,9 +282,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
249
282
|
version: '0'
|
250
283
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
251
284
|
requirements:
|
252
|
-
- - "
|
285
|
+
- - ">="
|
253
286
|
- !ruby/object:Gem::Version
|
254
|
-
version:
|
287
|
+
version: '0'
|
255
288
|
requirements: []
|
256
289
|
rubyforge_project:
|
257
290
|
rubygems_version: 2.6.11
|