datashift 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. data/.document +5 -5
  2. data/Gemfile +28 -25
  3. data/LICENSE.txt +26 -26
  4. data/README.markdown +302 -285
  5. data/README.rdoc +19 -19
  6. data/Rakefile +93 -95
  7. data/VERSION +5 -5
  8. data/datashift.gemspec +162 -178
  9. data/lib/applications/jruby/jexcel_file.rb +396 -396
  10. data/lib/applications/jruby/word.rb +79 -79
  11. data/lib/datashift.rb +152 -113
  12. data/lib/datashift/exceptions.rb +11 -11
  13. data/lib/datashift/file_definitions.rb +353 -353
  14. data/lib/datashift/mapping_file_definitions.rb +87 -87
  15. data/lib/datashift/method_detail.rb +236 -236
  16. data/lib/datashift/method_mapper.rb +256 -256
  17. data/lib/generators/csv_generator.rb +36 -36
  18. data/lib/generators/excel_generator.rb +121 -121
  19. data/lib/generators/generator_base.rb +13 -13
  20. data/lib/helpers/core_ext/to_b.rb +24 -24
  21. data/lib/helpers/spree_helper.rb +131 -131
  22. data/lib/java/poi-3.7/LICENSE +507 -507
  23. data/lib/java/poi-3.7/NOTICE +21 -21
  24. data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
  25. data/lib/loaders/csv_loader.rb +98 -98
  26. data/lib/loaders/excel_loader.rb +154 -149
  27. data/lib/loaders/loader_base.rb +403 -331
  28. data/lib/loaders/spreadsheet_loader.rb +136 -136
  29. data/lib/loaders/spree/image_loader.rb +45 -45
  30. data/lib/loaders/spree/product_loader.rb +224 -224
  31. data/spec/csv_loader_spec.rb +30 -30
  32. data/spec/datashift_spec.rb +26 -26
  33. data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
  34. data/spec/excel_generator_spec.rb +78 -78
  35. data/spec/excel_loader_spec.rb +204 -176
  36. data/spec/file_definitions.rb +141 -141
  37. data/spec/fixtures/.~lock.ProjectsSingleCategories.xls# +1 -0
  38. data/spec/fixtures/ProjectsDefaults.yml +29 -0
  39. data/spec/fixtures/config/database.yml +24 -24
  40. data/spec/fixtures/interact_models_db.sqlite +0 -0
  41. data/spec/fixtures/interact_spree_db.sqlite +0 -0
  42. data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
  43. data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
  44. data/spec/fixtures/spree/SpreeProducts.csv +4 -4
  45. data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
  46. data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
  47. data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
  48. data/spec/fixtures/test_model_defs.rb +57 -57
  49. data/spec/loader_spec.rb +120 -120
  50. data/spec/method_mapper_spec.rb +237 -237
  51. data/spec/spec_helper.rb +115 -115
  52. data/spec/spree_generator_spec.rb +64 -64
  53. data/spec/spree_loader_spec.rb +310 -310
  54. data/spec/spree_method_mapping_spec.rb +214 -214
  55. data/tasks/config/seed_fu_product_template.erb +15 -15
  56. data/tasks/config/tidy_config.txt +12 -12
  57. data/tasks/db_tasks.rake +65 -64
  58. data/tasks/excel_generator.rake +78 -78
  59. data/tasks/file_tasks.rake +36 -36
  60. data/tasks/import/csv.rake +49 -49
  61. data/tasks/import/excel.rake +71 -66
  62. data/tasks/spree/image_load.rake +108 -108
  63. data/tasks/spree/product_loader.rake +43 -43
  64. data/tasks/word_to_seedfu.rake +166 -166
  65. data/test/helper.rb +18 -18
  66. data/test/test_interact.rb +7 -7
  67. metadata +7 -38
  68. data/Gemfile.lock +0 -211
  69. data/bin/autospec +0 -16
  70. data/bin/convert_to_should_syntax +0 -16
  71. data/bin/erubis +0 -16
  72. data/bin/htmldiff +0 -16
  73. data/bin/jeweler +0 -16
  74. data/bin/ldiff +0 -16
  75. data/bin/nokogiri +0 -16
  76. data/bin/rackup +0 -16
  77. data/bin/rails +0 -16
  78. data/bin/rake +0 -16
  79. data/bin/rake2thor +0 -16
  80. data/bin/ri +0 -16
  81. data/bin/rspec +0 -16
  82. data/bin/spree +0 -16
  83. data/bin/thor +0 -16
  84. data/bin/tilt +0 -16
  85. data/bin/tt +0 -16
data/.document CHANGED
@@ -1,5 +1,5 @@
1
- lib/**/*.rb
2
- bin/*
3
- -
4
- features/**/*.feature
5
- LICENSE.txt
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/Gemfile CHANGED
@@ -1,25 +1,28 @@
1
- source "http://rubygems.org"
2
- # Add dependencies required to use your gem here.
3
- # Example:
4
- # gem "activesupport", ">= 2.3.5"
5
-
6
- # Add dependencies to develop your gem here.
7
- # Include everything needed to run rake, tests, features, etc.
8
- group :development do
9
- gem "rails", '3.1.3'
10
- gem "activerecord", '3.1.3'
11
- gem "activesupport", '3.1.3'
12
-
13
- platform :jruby do
14
- gem 'jruby-openssl'
15
- gem 'activerecord-jdbcsqlite3-adapter'
16
- end
17
-
18
- gem 'spree', '0.70.3'
19
- gem "rspec", ">= 0"
20
- gem "shoulda", ">= 0"
21
- gem "rdoc", "~> 3.12"
22
- gem "bundler", "~> 1.0.0"
23
- gem "jeweler", "~> 1.8.3"
24
- end
25
-
1
+ source "http://rubygems.org"
2
+ # Add dependencies required to use your gem here.
3
+ # Example:
4
+ # gem "activesupport", ">= 2.3.5"
5
+
6
+ # Add dependencies to develop your gem here.
7
+ # Include everything needed to run rake, tests, features, etc.
8
+
9
+
10
+ # We leave it up to the main Rails app to define the actual versions required
11
+ group :development do
12
+ gem "rails"
13
+ gem "activerecord"
14
+ gem "activesupport"
15
+
16
+ platform :jruby do
17
+ gem 'jruby-openssl'
18
+ gem 'activerecord-jdbcsqlite3-adapter'
19
+ end
20
+
21
+ gem 'spree', '0.70.3'
22
+ gem "rspec", ">= 0"
23
+ gem "shoulda", ">= 0"
24
+ gem "rdoc", "~> 3.12"
25
+ gem "bundler", "~> 1.0.0"
26
+ gem "jeweler", "~> 1.8.3"
27
+ end
28
+
data/LICENSE.txt CHANGED
@@ -1,27 +1,27 @@
1
- == DataShift
2
-
3
- ## License
4
-
5
- Copyright:: (c) Autotelik Media Ltd 2011
6
- Author :: Tom Statter
7
- Date :: Sept 2011
8
-
9
- The MIT License
10
-
11
- Permission is hereby granted, free of charge, to any person obtaining a copy
12
- of this software and associated documentation files (the "Software"), to deal
13
- in the Software without restriction, including without limitation the rights
14
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
- copies of the Software, and to permit persons to whom the Software is
16
- furnished to do so, subject to the following conditions:
17
-
18
- The above copyright notice and this permission notice shall be included in
19
- all copies or substantial portions of the Software.
20
-
21
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ == DataShift
2
+
3
+ ## License
4
+
5
+ Copyright:: (c) Autotelik Media Ltd 2011
6
+ Author :: Tom Statter
7
+ Date :: Sept 2011
8
+
9
+ The MIT License
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in
19
+ all copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
27
27
  THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,286 +1,303 @@
1
- ## DataShift
2
-
3
- Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
-
5
- ### Features
6
-
7
- Map Active Record models and associations to files.
8
-
9
- Import and Export ActiveRecord based data through .xls or CSV files.
10
-
11
- Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
12
-
13
- High level rake tasks provided.
14
-
15
- Spree E-Commerce specific loaders and rake tasks provided enabling import/export of all data including Products with
16
- complex associations such as Options/Variants and Images.
17
-
18
- Example spreadsheets, fully documented with comments for each column.
19
-
20
-
21
- ## Installation
22
-
23
- Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
24
-
25
- `gem install datashift`
26
-
27
- To use :
28
-
29
- gem 'datashift'
30
- require 'datashift'
31
-
32
- To pull the tasks in, add this call to your Rakefile :
33
-
34
- DataShift ::load_tasks
35
-
36
- N.B - To use the Excel loader, OLE and Excel are NOT required, however
37
- JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
38
-
39
- Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
40
-
41
- if(DataShift::Guards::jruby? )
42
- ..do something with Excel
43
- else
44
- ..do something with CSV
45
- end
46
-
47
- ## Active Record - Import/Export
48
-
49
- Provides wrappers and rake tasks for importing/exporting data to DBs, from various sources,
50
- including csv and .xls files (Excel/Open Office)
51
-
52
- Simplifies the exchange of data, between such files and any active record supported database.
53
-
54
- A core feature of DataShift is the MethodMapper, which provides features for collecting
55
- reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
56
-
57
- A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
58
-
59
- MethodMapper.find_operators( Blog )
60
-
61
- This then allows Import/Export to be achieved, by mapping the file's header and column data to MethodMapper's operators
62
- (i.e. the model's attributes and associations).
63
-
64
- Here we retrieve the method details for a column name from a file, "Blog Date"
65
-
66
- MethodMapper.find_method_detail( Blog, "Blog Date" )
67
-
68
- Loaders can use this method lookup functionality, to find the correct association for a column heading,
69
- and populate AR object with row data.
70
-
71
- This means data can be mapped to any model without any further coding. Generators are also supplied to export
72
- a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
73
-
74
- MethodMapper also stores column type information so the raw file data can be provided as is,
75
- and whenever possible, under the bonnet the data will be cast to correct DB type.
76
-
77
- Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
78
-
79
- MethodMapper.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
80
-
81
- Because it's based on reflection against the model, can build complex relationships between tables during import/export,
82
- and extending data files with new columns need not require any additional Ruby coding.
83
-
84
- New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
85
- attribute or association name in the header row.
86
-
87
-
88
- The Loader attempts to handle various human read-able forms of column names.
89
-
90
- For example, given an association on the model called, product_properties, will successfully load
91
- from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
92
-
93
- For has_many associations, either multiple columns can be used,
94
- or multiple values can be specified in a single column using suitable delimiters.
95
-
96
- Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
97
-
98
- Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
99
- for loading Spree Products, and associated data such as Product Variants, and Images.
100
-
101
- ## Template Generation and Export
102
-
103
- Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
104
- These can be provided to developers or business users, as a template for data collection and then loading.
105
-
106
- Export tasks can be used to export of a model's definition and any existing data stored in the database.
107
-
108
- This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
109
-
110
-
111
- ## Example Spreadsheets
112
-
113
- A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
114
-
115
- Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
116
- taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
117
-
118
- Column headings contain comments with full descriptions and instructions on syntax.
119
-
120
-
121
- ## Features
122
-
123
- - *High level wrappers around applications including Excel and Word
124
-
125
- Quickly and easily access common enterprise applications through Ruby
126
-
127
- Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
128
- and Excel does not need to be installed.
129
-
130
- The required POI jars are already included.
131
-
132
- - *Direct Excel export*
133
-
134
- Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
135
- easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
136
-
137
- No need to save to CSV or map to YAML.
138
-
139
- - *Semi-Smart Name Lookup*
140
-
141
- Includes helper classes that find and store details of all possible associations on an AR class.
142
- Given a user supplied name, attempts to find the requested association.
143
-
144
- Example usage, load from a file or spreadsheet where the column names are only
145
- an approximation of the actual associations, so given 'Product Properties' heading,
146
- finds real association 'product_properties' to send or call on the AR object
147
-
148
- - *Associations*
149
-
150
- Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
151
- via either multiple columns, or via specially delimited entry in a single (column). See Details section.
152
-
153
- Supports delegated attributes.
154
-
155
- - *Rake Tasks*
156
-
157
- High level Rake tasks are provided, only required to supply model class, and file location :
158
-
159
- jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
160
-
161
-
162
- - *Spree Rake Tasks*
163
-
164
- Specific Rake tasks are also provided for Spree loading - currently supports Product with associations,
165
- and Image loading.
166
-
167
- jruby -S rake datashift:spree:products input=C:\MyProducts.xls
168
-
169
-
170
- **Product loading from Excel files specifically requires JRuby (But not Excel or OLE)**.
171
-
172
-
173
- - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
174
-
175
- Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
176
-
177
- Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
178
- parameter klass=Xyz.
179
-
180
- In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
181
-
182
- Image loading **does not** specifically require JRuby
183
-
184
- A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
185
-
186
- rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
187
-
188
- rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
189
-
190
- ## Import to Active Record
191
-
192
- ### Associations
193
-
194
- To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
195
-
196
- A single association column can contain multiple name/value sets, in string form :
197
-
198
- column:lookup_key_1, lookup_key_2,...
199
-
200
- So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
201
-
202
- |Categories|
203
-
204
- reference:category_001,category_002
205
-
206
- During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
207
- and our Project model will contain those two i.e project.categories = [category_002,category_003]
208
-
209
- ## Spree Suppprt
210
-
211
- ### OptionTypes & Variants
212
-
213
- When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
214
- support for OptionTypes with values
215
-
216
- Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
217
- appropriate OptionValues to go with that Type.
218
-
219
- For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
220
- (such as small, medium etc)
221
-
222
- 'Option Types'
223
- size:small,medium,large|colour:red,white
224
-
225
- If no such OptionType exists, e.g size, then a new one is created with the supplied name.
226
-
227
- Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
228
-
229
- Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
230
- Currently a unique SKU is created by adding an index to the master's sku.
231
-
232
- TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
233
-
234
- ### Properties
235
-
236
- The properties to associate with this product.
237
- Properties are for small snippets of text, shared across many products,
238
- and are for display purposes only.
239
-
240
- An optional display value can be supplied to supplement the displayed text.
241
-
242
- As for all associations can contain multiple name/value sets in default form :
243
-
244
- Property:display_value|Property:display_value
245
-
246
- Example - No values :
247
- manufacturer|standard
248
-
249
- Example - Display values :
250
- manufacturer:somebody else plc|standard:ISOBlah21
251
-
252
- ## TODO
253
-
254
- - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
255
-
256
- - Smart sorting of column processing order ....
257
-
258
- - Does not currently ensure mandatory columns (for valid?) processed first.
259
-
260
- ## License
261
-
262
- Copyright:: (c) Autotelik Media Ltd 2011
263
-
264
- Author :: Tom Statter
265
-
266
- Date :: Dec 2011
267
-
268
- The MIT License
269
-
270
- Permission is hereby granted, free of charge, to any person obtaining a copy
271
- of this software and associated documentation files (the "Software"), to deal
272
- in the Software without restriction, including without limitation the rights
273
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
274
- copies of the Software, and to permit persons to whom the Software is
275
- furnished to do so, subject to the following conditions:
276
-
277
- The above copyright notice and this permission notice shall be included in
278
- all copies or substantial portions of the Software.
279
-
280
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
281
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
282
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
283
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
284
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
285
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ ## DataShift
2
+
3
+ Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
+
5
+ ### Features
6
+
7
+ Map Active Record models and associations to files, generate sample templates.
8
+
9
+ Import and Export ActiveRecord models through .xls or CSV files, including
10
+ all associations and with configurable defaults.
11
+
12
+ Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
13
+
14
+ High level rake tasks provided for key features.
15
+
16
+ Easily extendable Loader functionality to deal with non trivial import cases, such
17
+ as complex association lookups.
18
+
19
+ Specific loaders and rake tasks provided out the box for Spree E-Commerce,
20
+ enabling import/export of all data including Products with complex associations such as Options/Variants and Images.
21
+
22
+ Many example Spreadsheets provided, fully documented with comments for each column.
23
+
24
+
25
+ ## Installation
26
+
27
+ Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
28
+
29
+ `gem install datashift`
30
+
31
+ To use :
32
+
33
+ gem 'datashift'
34
+ require 'datashift'
35
+
36
+ To pull the tasks in, add this call to your Rakefile :
37
+
38
+ DataShift::load_tasks
39
+
40
+ N.B - To use the Excel loader, OLE and Excel are NOT required, however
41
+ JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
42
+
43
+ Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
44
+
45
+ if(DataShift::Guards::jruby? )
46
+ ..do something with Excel
47
+ else
48
+ ..do something with CSV
49
+ end
50
+
51
+ ## Active Record - Import/Export
52
+
53
+ Provides high level rake tasks for importing data via ActiveRecord models into a DB,
54
+ from various sources, currently csv or .xls files (Excel/Open Office)
55
+
56
+
57
+ bundle exec rake datashift:import:excel model=BlogPost input=BlogPostImport.xls verbose=true
58
+
59
+
60
+ The library can be easily extended with Loaders to deal with non trivial cases required to exchange data between
61
+ such files and any active record supported database.
62
+
63
+ Spree loaders are an example, these illustrate over riding processing for specific columns with
64
+ complicated lookup requirements.
65
+
66
+ A core feature of DataShift is the MethodMapper, which provides features for collecting
67
+ reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
68
+
69
+ A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
70
+
71
+ MethodMapper.find_operators( Blog )
72
+
73
+ This then allows Import/Export to be achieved, by mapping the file's header and column data to MethodMapper's operators
74
+ (i.e. the model's attributes and associations).
75
+
76
+ Here we retrieve the method details for a column name from a file, "Blog Date"
77
+
78
+ MethodMapper.find_method_detail( Blog, "Blog Date" )
79
+
80
+ Loaders can use this method lookup functionality, to find the correct association for a column heading,
81
+ and populate AR object with row data.
82
+
83
+ This means data can be mapped to any model without any further coding. Generators are also supplied to export
84
+ a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
85
+
86
+ MethodMapper also stores column type information so the raw file data can be provided as is,
87
+ and whenever possible, under the bonnet the data will be cast to correct DB type.
88
+
89
+ Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
90
+
91
+ MethodMapper.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
92
+
93
+ Because it's based on reflection against the model, can build complex relationships between tables during import/export,
94
+ and extending data files with new columns need not require any additional Ruby coding.
95
+
96
+ New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
97
+ attribute or association name in the header row.
98
+
99
+
100
+ The Loader attempts to handle various human read-able forms of column names.
101
+
102
+ For example, given an association on the model called, product_properties, will successfully load
103
+ from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
104
+
105
+ For has_many associations, either multiple columns can be used,
106
+ or multiple values can be specified in a single column using suitable delimiters.
107
+
108
+ Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
109
+
110
+ Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
111
+ for loading Spree Products, and associated data such as Product Variants, and Images.
112
+
113
+ ## Template Generation and Export
114
+
115
+ Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
116
+ These can be provided to developers or business users, as a template for data collection and then loading.
117
+
118
+ Export tasks can be used to export of a model's definition and any existing data stored in the database.
119
+
120
+ This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
121
+
122
+
123
+ ## Example Spreadsheets
124
+
125
+ A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
126
+
127
+ Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
128
+ taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
129
+
130
+ Column headings contain comments with full descriptions and instructions on syntax.
131
+
132
+
133
+ ## Features
134
+
135
+ - *High level wrappers around applications including Excel and Word
136
+
137
+ Quickly and easily access common enterprise applications through Ruby
138
+
139
+ Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
140
+ and Excel does not need to be installed.
141
+
142
+ The required POI jars are already included.
143
+
144
+ - *Direct Excel export*
145
+
146
+ Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
147
+ easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
148
+
149
+ No need to save to CSV or map to YAML.
150
+
151
+ - *Semi-Smart Name Lookup*
152
+
153
+ Includes helper classes that find and store details of all possible associations on an AR class.
154
+ Given a user supplied name, attempts to find the requested association.
155
+
156
+ Example usage, load from a file or spreadsheet where the column names are only
157
+ an approximation of the actual associations, so given 'Product Properties' heading,
158
+ finds real association 'product_properties' to send or call on the AR object
159
+
160
+ - *Associations*
161
+
162
+ Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
163
+ via either multiple columns, or via specially delimited entry in a single (column). See Details section.
164
+
165
+ Supports delegated attributes.
166
+
167
+ - *Rake Tasks*
168
+
169
+ High level Rake tasks are provided, only required to supply model class, and file location :
170
+
171
+ jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
172
+
173
+
174
+ - *Spree Rake Tasks*
175
+
176
+ Specific Rake tasks are also provided for Spree loading - currently supports Product with associations,
177
+ and Image loading.
178
+
179
+ jruby -S rake datashift:spree:products input=C:\MyProducts.xls
180
+
181
+
182
+ **Product loading from Excel files specifically requires JRuby (But not Excel or OLE)**.
183
+
184
+
185
+ - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
186
+
187
+ Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
188
+
189
+ Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
190
+ parameter klass=Xyz.
191
+
192
+ In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
193
+
194
+ Image loading **does not** specifically require JRuby
195
+
196
+ A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
197
+
198
+ rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
199
+
200
+ rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
201
+
202
+ ## Import to Active Record
203
+
204
+ ### Associations
205
+
206
+ To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
207
+
208
+ A single association column can contain multiple name/value sets, in string form :
209
+
210
+ column:lookup_key_1, lookup_key_2,...
211
+
212
+ So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
213
+
214
+ |Categories|
215
+
216
+ reference:category_001,category_002
217
+
218
+ During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
219
+ and our Project model will contain those two i.e project.categories = [category_002,category_003]
220
+
221
+ ## Spree Suppprt
222
+
223
+ ### OptionTypes & Variants
224
+
225
+ When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
226
+ support for OptionTypes with values
227
+
228
+ Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
229
+ appropriate OptionValues to go with that Type.
230
+
231
+ For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
232
+ (such as small, medium etc)
233
+
234
+ 'Option Types'
235
+ size:small,medium,large|colour:red,white
236
+
237
+ If no such OptionType exists, e.g size, then a new one is created with the supplied name.
238
+
239
+ Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
240
+
241
+ Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
242
+ Currently a unique SKU is created by adding an index to the master's sku.
243
+
244
+ TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
245
+
246
+ ### Properties
247
+
248
+ The properties to associate with this product.
249
+ Properties are for small snippets of text, shared across many products,
250
+ and are for display purposes only.
251
+
252
+ An optional display value can be supplied to supplement the displayed text.
253
+
254
+ As for all associations can contain multiple name/value sets in default form :
255
+
256
+ Property:display_value|Property:display_value
257
+
258
+ Example - No values :
259
+ manufacturer|standard
260
+
261
+ Example - Display values :
262
+ manufacturer:somebody else plc|standard:ISOBlah21
263
+
264
+ ## TODO
265
+
266
+ - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
267
+
268
+ - Smart sorting of column processing order ....
269
+
270
+ - Does not currently ensure mandatory columns (for valid?) processed first.
271
+
272
+ - Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
273
+ rather than active record, so we can support additional ORMs
274
+
275
+ - Create separate Spree extension to support import/export via the admin gui
276
+
277
+ ## License
278
+
279
+ Copyright:: (c) Autotelik Media Ltd 2011
280
+
281
+ Author :: Tom Statter
282
+
283
+ Date :: Dec 2011
284
+
285
+ The MIT License
286
+
287
+ Permission is hereby granted, free of charge, to any person obtaining a copy
288
+ of this software and associated documentation files (the "Software"), to deal
289
+ in the Software without restriction, including without limitation the rights
290
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
291
+ copies of the Software, and to permit persons to whom the Software is
292
+ furnished to do so, subject to the following conditions:
293
+
294
+ The above copyright notice and this permission notice shall be included in
295
+ all copies or substantial portions of the Software.
296
+
297
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
298
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
299
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
300
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
301
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
302
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
286
303
  THE SOFTWARE.