speaky_csv 0.0.3 → 0.0.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Guardfile +1 -1
- data/README.md +258 -67
- data/lib/speaky_csv.rb +3 -0
- data/lib/speaky_csv/active_record_import.rb +8 -6
- data/lib/speaky_csv/attr_import.rb +29 -10
- data/lib/speaky_csv/base.rb +9 -73
- data/lib/speaky_csv/config.rb +30 -0
- data/lib/speaky_csv/config_builder.rb +74 -0
- data/lib/speaky_csv/export.rb +14 -1
- data/lib/speaky_csv/version.rb +1 -1
- data/spec/active_record_import_spec.rb +78 -4
- data/spec/attr_import_spec.rb +160 -2
- data/spec/base_spec.rb +33 -10
- data/spec/export_spec.rb +144 -0
- data/spec/readme_spec.rb +95 -0
- data/spec/support/active_record.rb +1 -0
- metadata +40 -36
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1d5c7405ab99d006069011e4008088bbd2ff062b
|
4
|
+
data.tar.gz: 4b6e533cd7619fb4a8bdb9c01bddbf28d4b59fc3
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4f1a2152d8b1ecb4cb05cb0f682ebba9cd65f2b8fdeb9e4e733899abfc55e4be9d87e5473aebd420a7e6276a4136d21743a93d149b7ef9376f2e8055714f033b
|
7
|
+
data.tar.gz: 07e0c6cc87f6f47454e8a28eb27d75e9176cf7ef7fd9e943f3428fb474c4defff85bfa88580f081c1b95ccefba171d483e80971bfc997075cb038f4e3c632ab8
|
data/Guardfile
CHANGED
data/README.md
CHANGED
@@ -2,104 +2,293 @@
|
|
2
2
|
|
3
3
|
CSV imports and exports for ActiveRecord.
|
4
4
|
|
5
|
-
|
5
|
+
Speaky CSV features:
|
6
6
|
|
7
|
-
|
7
|
+
* An easy to use API,
|
8
|
+
* Speedy stream processing using enumerators.
|
8
9
|
|
9
|
-
|
10
|
-
class User < ActiveRecord::Base
|
11
|
-
...
|
12
|
-
end
|
10
|
+
## Installation
|
13
11
|
|
14
|
-
|
15
|
-
the csv format could look like this:
|
12
|
+
Add this line to your application's Gemfile:
|
16
13
|
|
17
|
-
|
18
|
-
class UserCsv < SpeakyCsv::Base
|
19
|
-
define_csv_fields do |config|
|
20
|
-
config.field :id, :email, :roles
|
21
|
-
end
|
22
|
-
end
|
14
|
+
gem 'speaky_csv'
|
23
15
|
|
24
|
-
|
25
|
-
initial header row, with each following row representing a user record.
|
26
|
-
lets import the following csv file (whitespace for clarity):
|
16
|
+
And then execute:
|
27
17
|
|
28
|
-
|
29
|
-
id, email, roles
|
30
|
-
22, admin@example.test, admin
|
31
|
-
, newbie@user.test, user
|
18
|
+
$ bundle
|
32
19
|
|
33
|
-
|
20
|
+
Or install it yourself as:
|
34
21
|
|
35
|
-
|
36
|
-
importer = UserCsv.new.active_record_importer io, User
|
37
|
-
importer.each { |user| user.save }
|
38
|
-
end
|
22
|
+
$ gem install speaky_csv
|
39
23
|
|
40
|
-
##
|
24
|
+
## Usage
|
41
25
|
|
42
|
-
|
26
|
+
Let's say you build software for your local public library and there
|
27
|
+
exists a Book model:
|
43
28
|
|
29
|
+
```ruby
|
30
|
+
# in app/models/book.rb
|
31
|
+
class Book < ActiveRecord::Base
|
32
|
+
# ...
|
33
|
+
end
|
34
|
+
```
|
44
35
|
|
45
|
-
Speaky
|
46
|
-
|
36
|
+
Speaky can be used to import and export book records using csv files.
|
37
|
+
The definition of the csv format could look like this:
|
47
38
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
39
|
+
```ruby
|
40
|
+
# in app/csvs/book_csv.rb
|
41
|
+
class BookCsv < SpeakyCsv::Base
|
42
|
+
define_csv_fields do |config|
|
43
|
+
config.field :id, :author
|
44
|
+
end
|
45
|
+
end
|
46
|
+
```
|
53
47
|
|
54
|
-
|
48
|
+
This defines a CSV format that looks like this:
|
55
49
|
|
56
|
-
|
50
|
+
id,author
|
51
|
+
3,Stevenson
|
52
|
+
19,Melville
|
53
|
+
1,Macaulay
|
57
54
|
|
58
|
-
|
55
|
+
## Exporting
|
59
56
|
|
60
|
-
|
57
|
+
Creating a csv file from records in a database can be done with the
|
58
|
+
exporter:
|
61
59
|
|
62
|
-
|
60
|
+
```ruby
|
61
|
+
# in app/csvs/book_csv.rb
|
62
|
+
class BookCsv < SpeakyCsv::Base
|
63
|
+
define_csv_fields do |config|
|
64
|
+
config.field :id, :author, :_destroy
|
65
|
+
end
|
66
|
+
end
|
63
67
|
|
64
|
-
|
68
|
+
books = [
|
69
|
+
Book.create!(author: 'Stevenson'),
|
70
|
+
Book.create!(author: 'Melville'),
|
71
|
+
Book.create!(author: 'Macaulay'),
|
72
|
+
]
|
65
73
|
|
66
|
-
|
74
|
+
exporter = BookCsv.exporter books
|
67
75
|
|
68
|
-
|
76
|
+
io = StringIO.new
|
77
|
+
exporter.each { |row| io.write row }
|
78
|
+
```
|
79
|
+
|
80
|
+
`io` will have the following contents:
|
81
|
+
|
82
|
+
id,author,_destroy
|
83
|
+
2,Stevenson,false
|
84
|
+
3,Melville,false
|
85
|
+
4,Macaulay,false
|
86
|
+
|
87
|
+
##### With Associations
|
88
|
+
|
89
|
+
Associations can also be exported.
|
90
|
+
|
91
|
+
```ruby
|
92
|
+
# in app/csvs/book_csv.rb
|
93
|
+
class BookCsv < SpeakyCsv::Base
|
94
|
+
define_csv_fields do |config|
|
95
|
+
config.field :id, :author
|
96
|
+
|
97
|
+
config.belongs_to :publisher do |p|
|
98
|
+
p.field :id, :name
|
99
|
+
end
|
100
|
+
|
101
|
+
config.has_many :reviews do |r|
|
102
|
+
r.field :id, :tomatoes, :publication
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
```
|
107
|
+
|
108
|
+
This defines a CSV format that looks like this:
|
109
|
+
|
110
|
+
id,author,publisher_id,publisher_name
|
111
|
+
3,Stevenson,22,Blam Ltd
|
112
|
+
19,Melville,,
|
113
|
+
1,Macaulay,83,NY Tiempo,reviews_0_id,8,reviews_0_tomatoes,50,review_0_publication,Daily
|
114
|
+
|
115
|
+
Since a book only ever has one publisher, these can get dedicated
|
116
|
+
columns with headers (the `publisher_id` and `publisher_name` columns).
|
117
|
+
Reviews are more tricky because there can be several that need to be
|
118
|
+
serialized to a single csv row. Speaky CSV uses a convention similar to
|
119
|
+
how rails and rack deal with query parameters for things like multi
|
120
|
+
select form inputs.
|
121
|
+
|
122
|
+
## Importing
|
123
|
+
|
124
|
+
Now lets import some books. Speaky will expect an import to have
|
125
|
+
an initial header row, and each subsequent row to represent a user record.
|
126
|
+
Columns can be in any order.
|
127
|
+
|
128
|
+
Let's create a book by importing a csv.
|
129
|
+
|
130
|
+
```ruby
|
131
|
+
csv_io = StringIO.new <<-CSV
|
132
|
+
id,author
|
133
|
+
,Sneed
|
134
|
+
CSV
|
135
|
+
```
|
136
|
+
|
137
|
+
Notice the empty id column. This tells Speaky that the operation is
|
138
|
+
a create. The file can be imported like this:
|
139
|
+
|
140
|
+
```ruby
|
141
|
+
importer = BookCsv.active_record_importer csv_io, Book
|
142
|
+
importer.each { |book| book.save }
|
143
|
+
Book.last.author == 'Sneed' # => true
|
144
|
+
```
|
145
|
+
|
146
|
+
This importer is an `active record importer`, which means that `#each`
|
147
|
+
will return active record instances. There is also an `attribute importer`
|
148
|
+
that will return hashes of attribute name => values. See the rdoc for
|
149
|
+
more info on that.
|
150
|
+
|
151
|
+
##### Update
|
152
|
+
|
153
|
+
Let's change the author value:
|
154
|
+
|
155
|
+
```ruby
|
156
|
+
csv_io = StringIO.new <<-CSV
|
157
|
+
id,author
|
158
|
+
1,Simon Sneed
|
159
|
+
CSV
|
160
|
+
```
|
161
|
+
|
162
|
+
Now there is an id value in the csv. Having an id value will cause
|
163
|
+
Speaky to find the record with the given id and update it.
|
164
|
+
|
165
|
+
```ruby
|
166
|
+
importer = BookCsv.active_record_importer csv_io, Book
|
167
|
+
importer.each { |book| book.save }
|
168
|
+
expect(Book.last.author).to eq 'Simon Sneed'
|
169
|
+
```
|
170
|
+
|
171
|
+
If a record with the given id isn't found, the importer will return a
|
172
|
+
nil for that row instead of an active record and add a message a log
|
173
|
+
file:
|
174
|
+
|
175
|
+
```ruby
|
176
|
+
csv_io = StringIO.new <<-CSV
|
177
|
+
id,author
|
178
|
+
234,I dont exist
|
179
|
+
CSV
|
180
|
+
|
181
|
+
importer = BookCsv.active_record_importer csv_io, Book
|
182
|
+
importer.to_a # => [nil]
|
183
|
+
importer.log # => '...[row 1] record not found with primary key: "234"....'
|
184
|
+
```
|
185
|
+
|
186
|
+
For more info on the log file see below.
|
187
|
+
|
188
|
+
##### Destroy
|
189
|
+
|
190
|
+
To destroy the record, we'll need to change the csv format to add a
|
191
|
+
`_destroy` field. If this column contains a true value like: 'true' or
|
192
|
+
'1', the record will be marked for destruction.
|
193
|
+
|
194
|
+
Marking an active record for destruction is documented here:
|
195
|
+
http://api.rubyonrails.org/v4.2.0/classes/ActiveRecord/AutosaveAssociation.html#method-i-marked_for_destruction-3F
|
196
|
+
|
197
|
+
```ruby
|
198
|
+
# in app/csvs/book_csv.rb
|
199
|
+
class BookCsv < SpeakyCsv::Base
|
200
|
+
define_csv_fields do |config|
|
201
|
+
config.field :id, :author, :_destroy
|
202
|
+
end
|
203
|
+
end
|
204
|
+
|
205
|
+
csv_io = StringIO.new <<-CSV
|
206
|
+
id,_destroy
|
207
|
+
1,true
|
208
|
+
CSV
|
209
|
+
|
210
|
+
importer = BookCsv.active_record_importer csv_io, Book
|
211
|
+
book = importer.to_a.first
|
212
|
+
if book.marked_for_destruction?
|
213
|
+
book.destroy
|
214
|
+
end
|
215
|
+
```
|
216
|
+
|
217
|
+
##### With Associations
|
218
|
+
|
219
|
+
Speaky uses the active record `accepts_nested_attributes_for` feature to
|
220
|
+
deal with importing association data.
|
221
|
+
|
222
|
+
For example, if a belongs\_to association is configured:
|
223
|
+
|
224
|
+
```ruby
|
225
|
+
# in app/csvs/book_csv.rb
|
226
|
+
class BookCsv < SpeakyCsv::Base
|
227
|
+
define_csv_fields do |config|
|
228
|
+
config.field :id, :author
|
229
|
+
|
230
|
+
config.belongs_to :publisher do |p|
|
231
|
+
p.field :id, :name
|
232
|
+
end
|
233
|
+
end
|
234
|
+
end
|
235
|
+
```
|
236
|
+
|
237
|
+
And the csv file being imported is this:
|
238
|
+
|
239
|
+
id,author,publisher_id,publisher_name
|
240
|
+
3,Stevenson,22,Blam Ltd
|
69
241
|
|
70
|
-
|
71
|
-
record class. For example:
|
242
|
+
Then speaky will find a Booking record with id `3` and call:
|
72
243
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
config.field :id, :first_name, :last_name, :email
|
244
|
+
```ruby
|
245
|
+
booking.publisher_attributes = {id: '22', name: 'Blam Ltd'}
|
246
|
+
```
|
77
247
|
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
248
|
+
For a has\_many association, if the configuration looked like this:
|
249
|
+
|
250
|
+
```ruby
|
251
|
+
# in app/csvs/book_csv.rb
|
252
|
+
class BookCsv < SpeakyCsv::Base
|
253
|
+
define_csv_fields do |config|
|
254
|
+
config.field :id, :author
|
255
|
+
|
256
|
+
config.has_many :reviews do |r|
|
257
|
+
r.field :id, :tomatoes, :publication
|
82
258
|
end
|
259
|
+
end
|
260
|
+
end
|
261
|
+
```
|
262
|
+
|
263
|
+
And an import csv looked like this:
|
264
|
+
|
265
|
+
id,author,publisher_id,publisher_name
|
266
|
+
1,Macaulay,83,NY Tiempo,reviews_0_id,8,reviews_0_tomatoes,50,review_0_publication,Daily
|
83
267
|
|
84
|
-
|
268
|
+
The speaky will find a Booking record with id `1` and call:
|
85
269
|
|
86
|
-
|
270
|
+
```ruby
|
271
|
+
booking.reviews_attributes = [{id: '8', tomatoes: '50', publication: 'Daily'}]
|
272
|
+
```
|
87
273
|
|
88
|
-
|
89
|
-
$ File.open('users.csv', 'w') { |io| exporter.each { |row| io.write row } }
|
274
|
+
## Log Messages
|
90
275
|
|
91
|
-
|
276
|
+
Importers and exporters use a `Logger` instance to write messages during
|
277
|
+
processing. The default logger writes to a string that can be retrieved
|
278
|
+
by the `#log` method. A custom Logger can be set by the `#logger=`
|
279
|
+
method.
|
92
280
|
|
93
|
-
|
94
|
-
* describe importing to to active records
|
95
|
-
* describe how to transform with an enumerator
|
281
|
+
See `Logger` in the ruby stdlib for more details.
|
96
282
|
|
97
|
-
##
|
283
|
+
## Best Practices
|
98
284
|
|
99
|
-
*
|
285
|
+
* Configure speaky with `id` and `_destroy` fields for active record models
|
100
286
|
* For associations, use `nested_attributes_for` and add `id` and
|
101
287
|
`_destroy` fields
|
102
|
-
* Use optimistic locking and
|
288
|
+
* Use optimistic locking and configure a `lock_version` field
|
289
|
+
* Consider building a draft or preview feature for importing which
|
290
|
+
doesn't persist the record by calling `save` but instead reports what
|
291
|
+
the changes would be using `ActiveModel::Dirty`
|
103
292
|
|
104
293
|
## TODO
|
105
294
|
|
@@ -108,11 +297,13 @@ TODO:
|
|
108
297
|
* [x] export validations
|
109
298
|
* [x] attr import validations
|
110
299
|
* [x] active record import validations
|
111
|
-
* [
|
300
|
+
* [x] `has_one` associations
|
112
301
|
* [ ] required fields (make `lock_version` required for example)
|
113
302
|
* [ ] transformations for values via accessors on class
|
114
|
-
* [
|
115
|
-
* [
|
303
|
+
* [x] public stable api for csv format definition
|
304
|
+
* [x] assign attrs one at a time so they don't all fail together
|
305
|
+
* [x] decide what empty cells mean
|
306
|
+
* [ ] figure out why SpeakyCsv is a class and not a module
|
116
307
|
|
117
308
|
## Contributing
|
118
309
|
|
data/lib/speaky_csv.rb
CHANGED
@@ -73,7 +73,7 @@ module SpeakyCsv
|
|
73
73
|
end
|
74
74
|
|
75
75
|
unless record
|
76
|
-
logger.error "[row #{row_index}] record not found with primary key #{attrs[@config.primary_key]}"
|
76
|
+
logger.error "[row #{row_index}] record not found with primary key: #{attrs[@config.primary_key.to_s].inspect}"
|
77
77
|
yielder << nil
|
78
78
|
next
|
79
79
|
end
|
@@ -89,11 +89,13 @@ module SpeakyCsv
|
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
|
-
@config.has_manys.keys.each do |name|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
92
|
+
@config.has_manys.keys.map(&:to_s).select{|n| attrs.key? n}.each do |name|
|
93
|
+
# assume nested attributes feature is used
|
94
|
+
attrs["#{name}_attributes"] = attrs.delete name
|
95
|
+
end
|
96
|
+
|
97
|
+
@config.has_ones.keys.map(&:to_s).select{|n| attrs.key? n}.each do |name|
|
98
|
+
attrs["#{name}_attributes"] = attrs.delete name.to_s
|
97
99
|
end
|
98
100
|
|
99
101
|
attrs.each do |attr, value|
|
@@ -37,8 +37,10 @@ module SpeakyCsv
|
|
37
37
|
|
38
38
|
csv.each do |row|
|
39
39
|
attrs = {}
|
40
|
+
validate_headers row
|
40
41
|
add_fields row, attrs
|
41
42
|
add_has_manys row, attrs
|
43
|
+
add_has_ones row, attrs
|
42
44
|
yielder << attrs
|
43
45
|
end
|
44
46
|
|
@@ -48,19 +50,24 @@ module SpeakyCsv
|
|
48
50
|
end
|
49
51
|
end
|
50
52
|
|
51
|
-
#
|
52
|
-
def
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
if @config.export_only_fields.include?(h.to_sym)
|
53
|
+
# TODO: don't warn on has_one headers and clean up clunky loop
|
54
|
+
def validate_headers(row)
|
55
|
+
valid_headers = @config.fields - @config.export_only_fields
|
56
|
+
#valid_headers += @config.has_ones.map
|
57
|
+
|
58
|
+
row.headers.compact.map(&:to_sym).each do |h|
|
59
|
+
unless valid_headers.include?(h)
|
59
60
|
logger.warn "ignoring unknown column #{h}"
|
60
|
-
next
|
61
61
|
end
|
62
|
+
end
|
63
|
+
end
|
62
64
|
|
63
|
-
|
65
|
+
# Adds configured fields to attrs
|
66
|
+
def add_fields(row, attrs)
|
67
|
+
fields = (@config.fields - @config.export_only_fields).map(&:to_s)
|
68
|
+
fields.each do |name|
|
69
|
+
row.has_key? name or next
|
70
|
+
attrs[name] = row.field name
|
64
71
|
end
|
65
72
|
end
|
66
73
|
|
@@ -90,5 +97,17 @@ module SpeakyCsv
|
|
90
97
|
attrs[has_many_name][has_many_index][has_many_field] = has_many_value
|
91
98
|
end
|
92
99
|
end
|
100
|
+
|
101
|
+
# Adds configured has ones to attrs
|
102
|
+
def add_has_ones(row, attrs)
|
103
|
+
@config.has_ones.each do |name,assoc_config|
|
104
|
+
fields = (assoc_config.fields - assoc_config.export_only_fields).map(&:to_s)
|
105
|
+
fields.each do |f|
|
106
|
+
csv_name = "#{name}_#{f}"
|
107
|
+
row.has_key? csv_name or next
|
108
|
+
(attrs[name.to_s] ||= {})[f] = row.field "#{name}_#{f}"
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
93
112
|
end
|
94
113
|
end
|