datashift 0.2.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (84) hide show
  1. data/.document +5 -5
  2. data/LICENSE.txt +26 -26
  3. data/README.markdown +326 -305
  4. data/README.rdoc +19 -19
  5. data/Rakefile +86 -93
  6. data/VERSION +1 -1
  7. data/datashift.gemspec +163 -152
  8. data/lib/applications/jruby/jexcel_file.rb +410 -408
  9. data/lib/applications/jruby/word.rb +79 -79
  10. data/lib/datashift.rb +183 -152
  11. data/lib/datashift/exceptions.rb +11 -11
  12. data/lib/datashift/file_definitions.rb +353 -353
  13. data/lib/datashift/mapping_file_definitions.rb +87 -87
  14. data/lib/datashift/method_detail.rb +293 -275
  15. data/lib/datashift/method_dictionary.rb +208 -209
  16. data/lib/datashift/method_mapper.rb +90 -90
  17. data/lib/datashift/model_mapper.rb +27 -0
  18. data/lib/exporters/csv_exporter.rb +36 -0
  19. data/lib/exporters/excel_exporter.rb +116 -0
  20. data/lib/exporters/exporter_base.rb +15 -0
  21. data/lib/generators/csv_generator.rb +36 -36
  22. data/lib/generators/excel_generator.rb +106 -122
  23. data/lib/generators/generator_base.rb +13 -13
  24. data/lib/helpers/core_ext/to_b.rb +24 -24
  25. data/lib/helpers/rake_utils.rb +42 -0
  26. data/lib/helpers/spree_helper.rb +194 -153
  27. data/lib/java/poi-3.7/LICENSE +507 -507
  28. data/lib/java/poi-3.7/NOTICE +21 -21
  29. data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
  30. data/lib/loaders/csv_loader.rb +98 -98
  31. data/lib/loaders/excel_loader.rb +155 -155
  32. data/lib/loaders/loader_base.rb +420 -420
  33. data/lib/loaders/spreadsheet_loader.rb +136 -136
  34. data/lib/loaders/spree/image_loader.rb +67 -63
  35. data/lib/loaders/spree/product_loader.rb +289 -248
  36. data/lib/thor/generate_excel.thor +54 -0
  37. data/sandbox/app/controllers/application_controller.rb +3 -0
  38. data/sandbox/config/application.rb +43 -0
  39. data/sandbox/config/database.yml +34 -0
  40. data/sandbox/config/environment.rb +7 -0
  41. data/sandbox/config/environments/development.rb +30 -0
  42. data/spec/csv_loader_spec.rb +30 -30
  43. data/spec/datashift_spec.rb +26 -26
  44. data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
  45. data/spec/excel_exporter_spec.rb +78 -78
  46. data/spec/excel_generator_spec.rb +78 -78
  47. data/spec/excel_loader_spec.rb +223 -223
  48. data/spec/file_definitions.rb +141 -141
  49. data/spec/fixtures/ProjectsDefaults.yml +29 -29
  50. data/spec/fixtures/config/database.yml +27 -27
  51. data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
  52. data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
  53. data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
  54. data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
  55. data/spec/fixtures/spree/SpreeProducts.csv +4 -4
  56. data/spec/fixtures/spree/SpreeProducts.xls +0 -0
  57. data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
  58. data/spec/fixtures/spree/SpreeProductsMultiColumn.xls +0 -0
  59. data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
  60. data/spec/fixtures/spree/SpreeProductsWithImages.csv +4 -4
  61. data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
  62. data/spec/fixtures/test_model_defs.rb +57 -57
  63. data/spec/loader_spec.rb +120 -120
  64. data/spec/method_dictionary_spec.rb +242 -242
  65. data/spec/method_mapper_spec.rb +41 -41
  66. data/spec/spec_helper.rb +154 -116
  67. data/spec/spree_exporter_spec.rb +67 -0
  68. data/spec/spree_generator_spec.rb +77 -64
  69. data/spec/spree_loader_spec.rb +363 -324
  70. data/spec/spree_method_mapping_spec.rb +218 -214
  71. data/tasks/config/seed_fu_product_template.erb +15 -15
  72. data/tasks/config/tidy_config.txt +12 -12
  73. data/tasks/{excel_generator.rake → export/excel_generator.rake} +101 -78
  74. data/tasks/file_tasks.rake +36 -36
  75. data/tasks/import/csv.rake +50 -49
  76. data/tasks/import/excel.rake +74 -71
  77. data/tasks/spree/image_load.rake +108 -108
  78. data/tasks/spree/product_loader.rake +43 -43
  79. data/tasks/word_to_seedfu.rake +166 -166
  80. data/test/helper.rb +18 -18
  81. data/test/test_interact.rb +7 -7
  82. metadata +16 -8
  83. data/datashift-0.1.0.gem +0 -0
  84. data/tasks/db_tasks.rake +0 -66
data/.document CHANGED
@@ -1,5 +1,5 @@
1
- lib/**/*.rb
2
- bin/*
3
- -
4
- features/**/*.feature
5
- LICENSE.txt
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/LICENSE.txt CHANGED
@@ -1,27 +1,27 @@
1
- == DataShift
2
-
3
- ## License
4
-
5
- Copyright:: (c) Autotelik Media Ltd 2011
6
- Author :: Tom Statter
7
- Date :: Sept 2011
8
-
9
- The MIT License
10
-
11
- Permission is hereby granted, free of charge, to any person obtaining a copy
12
- of this software and associated documentation files (the "Software"), to deal
13
- in the Software without restriction, including without limitation the rights
14
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
- copies of the Software, and to permit persons to whom the Software is
16
- furnished to do so, subject to the following conditions:
17
-
18
- The above copyright notice and this permission notice shall be included in
19
- all copies or substantial portions of the Software.
20
-
21
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ == DataShift
2
+
3
+ ## License
4
+
5
+ Copyright:: (c) Autotelik Media Ltd 2011
6
+ Author :: Tom Statter
7
+ Date :: Sept 2011
8
+
9
+ The MIT License
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in
19
+ all copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
27
27
  THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,306 +1,327 @@
1
- ## DataShift
2
-
3
- Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
-
5
- ### Features
6
-
7
- Map Active Record models and associations to files, generate sample templates.
8
-
9
- Import and Export ActiveRecord models through .xls or CSV files, including
10
- all associations and with configurable defaults.
11
-
12
- Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
13
-
14
- Easily extendable Loader functionality to deal with non trivial import cases, such
15
- as complex association lookups.
16
-
17
- High level rake tasks for import/export provided.
18
-
19
- Specific loaders and rake tasks for Spree E-Commerce, enabling import/export of all data including Products with
20
- complex associations such as Properties/Taxons/Options/Variants and Images.
21
-
22
- Specific loaders and rake tasks provided out the box for Spree E-Commerce,
23
- enabling import/export of all data including Products with complex associations such as Options/Variants and Images.
24
-
25
- Many example Spreadsheets in spec/fixtures, fully documented with comments for each column.
26
-
27
- ## Installation
28
-
29
- Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
30
-
31
- `gem install datashift`
32
-
33
- To use :
34
-
35
- gem 'datashift'
36
- require 'datashift'
37
-
38
- To pull the tasks in, add this call to your Rakefile :
39
-
40
- DataShift::load_tasks
41
-
42
- N.B - To use the Excel loader, OLE and Excel are NOT required, however
43
- JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
44
-
45
- Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
46
-
47
- if(DataShift::Guards::jruby? )
48
- ..do something with Excel
49
- else
50
- ..do something with CSV
51
- end
52
-
53
- ## Active Record - Import/Export
54
-
55
- Provides high level rake tasks for importing data via ActiveRecord models into a DB,
56
- from various sources, currently csv or .xls files (Excel/Open Office)
57
-
58
-
59
- jruby -S rake datashift:import:excel model=BlogPost input=BlogPostImport.xls verbose=true
60
-
61
- Provides high level rake tasks for exporting data to various sources, currently .xls files (Excel/Open Office)
62
-
63
- jruby -S rake datashift:export:excel model=BlogPost result=BlogExport.xls
64
-
65
-
66
- The library can be easily extended with Loaders to deal with non trivial cases,
67
- for example when multiple lookups required to find right association.
68
-
69
- Spree loaders are an example, these illustrate over riding processing for specific columns with
70
- complicated lookup requirements.
71
-
72
- A core feature of DataShift is the MethodDictionary and MethodMapper, which provides features for collecting
73
- reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
74
-
75
- A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
76
-
77
- MethodDictionary.find_operators( Blog )
78
-
79
- This then allows Import/Export to be achieved, by mapping the file's header and column data to the found operators
80
- i.e. the methods to set data on model's attributes and associations.
81
-
82
- Here we retrieve the method details for a column name from a file, "Blog Date"
83
-
84
- MethodDictionary.find_method_detail( Blog, "Blog Date" )
85
-
86
- Loaders can use this method lookup functionality, to find the correct association for a column heading,
87
- and populate AR object with row data.
88
-
89
- This means data can be mapped to any model without any further coding. Generators are also supplied to export
90
- a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
91
-
92
- MethodMapper also stores column type information so the raw file data can be provided as is,
93
- and whenever possible, under the bonnet the data will be cast to correct DB type.
94
-
95
- Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
96
-
97
- MethodDictionary.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
98
-
99
- Because it's based on reflection against the model, can build complex relationships between tables during import/export,
100
- and extending data files with new columns need not require any additional Ruby coding.
101
-
102
- New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
103
- attribute or association name in the header row.
104
-
105
-
106
- The Loader attempts to handle various human read-able forms of column names.
107
-
108
- For example, given an association on the model called, product_properties, will successfully load
109
- from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
110
-
111
- For has_many associations, either multiple columns can be used,
112
- or multiple values can be specified in a single column using suitable delimiters.
113
-
114
- Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
115
-
116
- Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
117
- for loading Spree Products, and associated data such as Product Variants, and Images.
118
-
119
- ## Template Generation and Export
120
-
121
- Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
122
- These can be provided to developers or business users, as a template for data collection and then loading.
123
-
124
- Export tasks can be used to export of a model's definition and any existing data stored in the database.
125
-
126
- This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
127
-
128
-
129
- ## Example Spreadsheets
130
-
131
- A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
132
-
133
- Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
134
- taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
135
-
136
- Column headings contain comments with full descriptions and instructions on syntax.
137
-
138
-
139
- ## Features
140
-
141
- - *High level wrappers around applications including Excel and Word
142
-
143
- Quickly and easily access common enterprise applications through Ruby
144
-
145
- Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
146
- and Excel does not need to be installed.
147
-
148
- The required POI jars are already included.
149
-
150
- - *Direct Excel export*
151
-
152
- Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
153
- easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
154
-
155
- No need to save to CSV or map to YAML.
156
-
157
- - *Semi-Smart Name Lookup*
158
-
159
- Includes helper classes that find and store details of all possible associations on an AR class.
160
- Given a user supplied name, attempts to find the requested association.
161
-
162
- Example usage, load from a file or spreadsheet where the column names are only
163
- an approximation of the actual associations, so given 'Product Properties' heading,
164
- finds real association 'product_properties' to send or call on the AR object
165
-
166
- - *Associations*
167
-
168
- Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
169
- via either multiple columns, or via specially delimited entry in a single (column). See Details section.
170
-
171
- Supports delegated attributes.
172
-
173
- - *Rake Tasks*
174
-
175
- High level Rake tasks are provided, only required to supply model class, and file location :
176
-
177
- jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
178
-
179
-
180
- - *Spree Rake Tasks*
181
-
182
- Spree's product associations are non trivial so specific Rake tasks are also provided for loading Spree Producta
183
- with all associations and Image loading.
184
-
185
- jruby -S rake datashift:spree:products input=C:\MyProducts.xls
186
-
187
-
188
- - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
189
-
190
- Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
191
-
192
- Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
193
- parameter klass=Xyz.
194
-
195
- In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
196
-
197
- Image loading **does not** specifically require JRuby
198
-
199
- A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
200
-
201
- rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
202
-
203
- rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
204
-
205
- ## Import to Active Record
206
-
207
- ### Associations
208
-
209
- To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
210
-
211
- A single association column can contain multiple name/value sets, in string form :
212
-
213
- column:lookup_key_1, lookup_key_2,...
214
-
215
- So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
216
-
217
- |Categories|
218
-
219
- reference:category_001,category_002
220
-
221
- During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
222
- and our Project model will contain those two i.e project.categories = [category_002,category_003]
223
-
224
- ## Spree Suppprt
225
-
226
- ### OptionTypes & Variants
227
-
228
- When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
229
- support for OptionTypes with values
230
-
231
- Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
232
- appropriate OptionValues to go with that Type.
233
-
234
- For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
235
- (such as small, medium etc)
236
-
237
- 'Option Types'
238
- size:small,medium,large|colour:red,white
239
-
240
- If no such OptionType exists, e.g size, then a new one is created with the supplied name.
241
-
242
- Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
243
-
244
- Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
245
- Currently a unique SKU is created by adding an index to the master's sku.
246
-
247
- TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
248
-
249
- ### Properties
250
-
251
- The properties to associate with this product.
252
- Properties are for small snippets of text, shared across many products,
253
- and are for display purposes only.
254
-
255
- An optional display value can be supplied to supplement the displayed text.
256
-
257
- As for all associations can contain multiple name/value sets in default form :
258
-
259
- Property:display_value|Property:display_value
260
-
261
- Example - No values :
262
- manufacturer|standard
263
-
264
- Example - Display values :
265
- manufacturer:somebody else plc|standard:ISOBlah21
266
-
267
- ## TODO
268
-
269
- - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
270
-
271
- - Smart sorting of column processing order ....
272
-
273
- - Does not currently ensure mandatory columns (for valid?) processed first.
274
-
275
- - Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
276
- rather than active record, so we can support additional ORMs
277
-
278
- - Create separate Spree extension to support import/export via the admin gui
279
-
280
- ## License
281
-
282
- Copyright:: (c) Autotelik Media Ltd 2011
283
-
284
- Author :: Tom Statter
285
-
286
- Date :: Dec 2011
287
-
288
- The MIT License
289
-
290
- Permission is hereby granted, free of charge, to any person obtaining a copy
291
- of this software and associated documentation files (the "Software"), to deal
292
- in the Software without restriction, including without limitation the rights
293
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
294
- copies of the Software, and to permit persons to whom the Software is
295
- furnished to do so, subject to the following conditions:
296
-
297
- The above copyright notice and this permission notice shall be included in
298
- all copies or substantial portions of the Software.
299
-
300
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
301
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
302
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
303
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
304
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
305
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ ## DataShift
2
+
3
+ Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
+
5
+ ### Features
6
+
7
+ Map Active Record models and associations to files, generate sample templates.
8
+
9
+ Import and Export ActiveRecord models through .xls or CSV files, including
10
+ all associations and with configurable defaults.
11
+
12
+ Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
13
+
14
+ Easily extendable Loader functionality to deal with non trivial import cases, such
15
+ as complex association lookups.
16
+
17
+ High level rake/thor command line tasks for import/export provided.
18
+
19
+ Specific loaders and rake tasks for Spree E-Commerce, enabling import/export of all data including Products with
20
+ complex associations such as Properties/Taxons/Options/Variants and Images.
21
+
22
+ Specific loaders and rake tasks provided out the box for Spree E-Commerce,
23
+ enabling import/export of all data including Products with complex associations such as Options/Variants and Images.
24
+
25
+ Many example Spreadsheets in spec/fixtures, fully documented with comments for each column.
26
+
27
+ ## Installation
28
+
29
+ Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
30
+
31
+ `gem install datashift`
32
+
33
+ To use :
34
+
35
+ gem 'datashift'
36
+ require 'datashift'
37
+
38
+ To pull the tasks in, add this call to your Rakefile :
39
+
40
+ DataShift::load_tasks
41
+
42
+ To use the Thor command line applications :
43
+
44
+ Create a high level .thor file - e.g mysite.thor - in your applications root directory
45
+
46
+ Pull in the thor commands :
47
+
48
+ require 'thor'
49
+ require 'datashift'
50
+
51
+ DataShift::load_commands
52
+
53
+ To check the available tasks run
54
+
55
+ bundle exec rake -T datashift
56
+
57
+ and/or
58
+
59
+ bundle exc thor list datashift
60
+
61
+ N.B - To use the Excel loader, OLE and Excel are NOT required, however
62
+ JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
63
+
64
+ Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
65
+
66
+ if(DataShift::Guards::jruby? )
67
+ ..do something with Excel
68
+ else
69
+ ..do something with CSV
70
+ end
71
+
72
+ ## Active Record - Import/Export
73
+
74
+ Provides high level rake tasks for importing data via ActiveRecord models into a DB,
75
+ from various sources, currently csv or .xls files (Excel/Open Office)
76
+
77
+
78
+ bundle exec rake datashift:import:csv model=BlogPost input=BlogPostImport.csv verbose=true
79
+
80
+ jruby -S rake datashift:import:excel model=BlogPost input=BlogPostImport.xls verbose=true
81
+
82
+ Provides high level rake tasks for exporting data to various sources, currently .xls files (Excel/Open Office)
83
+
84
+ jruby -S rake datashift:export:excel model=BlogPost result=BlogExport.xls
85
+
86
+
87
+ The library can be easily extended with Loaders to deal with non trivial cases,
88
+ for example when multiple lookups required to find right association.
89
+
90
+ Spree loaders are an example, these illustrate over riding processing for specific columns with
91
+ complicated lookup requirements.
92
+
93
+ A core feature of DataShift is the MethodDictionary and MethodMapper, which provides features for collecting
94
+ reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
95
+
96
+ A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
97
+
98
+ MethodDictionary.find_operators( Blog )
99
+
100
+ This then allows Import/Export to be achieved, by mapping the file's header and column data to the found operators
101
+ i.e. the methods to set data on model's attributes and associations.
102
+
103
+ Here we retrieve the method details for a column name from a file, "Blog Date"
104
+
105
+ MethodDictionary.find_method_detail( Blog, "Blog Date" )
106
+
107
+ Loaders can use this method lookup functionality, to find the correct association for a column heading,
108
+ and populate AR object with row data.
109
+
110
+ This means data can be mapped to any model without any further coding. Generators are also supplied to export
111
+ a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
112
+
113
+ MethodMapper also stores column type information so the raw file data can be provided as is,
114
+ and whenever possible, under the bonnet the data will be cast to correct DB type.
115
+
116
+ Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
117
+
118
+ MethodDictionary.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
119
+
120
+ Because it's based on reflection against the model, can build complex relationships between tables during import/export,
121
+ and extending data files with new columns need not require any additional Ruby coding.
122
+
123
+ New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
124
+ attribute or association name in the header row.
125
+
126
+
127
+ The Loader attempts to handle various human read-able forms of column names.
128
+
129
+ For example, given an association on the model called, product_properties, will successfully load
130
+ from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
131
+
132
+ For has_many associations, either multiple columns can be used,
133
+ or multiple values can be specified in a single column using suitable delimiters.
134
+
135
+ Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
136
+
137
+ Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
138
+ for loading Spree Products, and associated data such as Product Variants, and Images.
139
+
140
+ ## Template Generation and Export
141
+
142
+ Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
143
+ These can be provided to developers or business users, as a template for data collection and then loading.
144
+
145
+ Export tasks can be used to export of a model's definition and any existing data stored in the database.
146
+
147
+ This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
148
+
149
+
150
+ ## Example Spreadsheets
151
+
152
+ A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
153
+
154
+ Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
155
+ taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
156
+
157
+ Column headings contain comments with full descriptions and instructions on syntax.
158
+
159
+
160
+ ## Features
161
+
162
+ - *High level wrappers around applications including Excel and Word
163
+
164
+ Quickly and easily access common enterprise applications through Ruby
165
+
166
+ Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
167
+ and Excel does not need to be installed.
168
+
169
+ The required POI jars are already included.
170
+
171
+ - *Direct Excel export*
172
+
173
+ Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
174
+ easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
175
+
176
+ No need to save to CSV or map to YAML.
177
+
178
+ - *Semi-Smart Name Lookup*
179
+
180
+ Includes helper classes that find and store details of all possible associations on an AR class.
181
+ Given a user supplied name, attempts to find the requested association.
182
+
183
+ Example usage, load from a file or spreadsheet where the column names are only
184
+ an approximation of the actual associations, so given 'Product Properties' heading,
185
+ finds real association 'product_properties' to send or call on the AR object
186
+
187
+ - *Associations*
188
+
189
+ Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
190
+ via either multiple columns, or via specially delimited entry in a single (column). See Details section.
191
+
192
+ Supports delegated attributes.
193
+
194
+ - *Rake Tasks*
195
+
196
+ High level Rake tasks are provided, only required to supply model class, and file location :
197
+
198
+ jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
199
+
200
+
201
+ - *Spree Rake Tasks*
202
+
203
+ Spree's product associations are non trivial so specific Rake tasks are also provided for loading Spree Producta
204
+ with all associations and Image loading.
205
+
206
+ jruby -S rake datashift:spree:products input=C:\MyProducts.xls
207
+
208
+
209
+ - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
210
+
211
+ Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
212
+
213
+ Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
214
+ parameter klass=Xyz.
215
+
216
+ In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
217
+
218
+ Image loading **does not** specifically require JRuby
219
+
220
+ A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
221
+
222
+ rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
223
+
224
+ rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
225
+
226
+ ## Import to Active Record
227
+
228
+ ### Associations
229
+
230
+ To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
231
+
232
+ A single association column can contain multiple name/value sets, in string form :
233
+
234
+ column:lookup_key_1, lookup_key_2,...
235
+
236
+ So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
237
+
238
+ |Categories|
239
+
240
+ reference:category_001,category_002
241
+
242
+ During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
243
+ and our Project model will contain those two i.e project.categories = [category_002,category_003]
244
+
245
+ ## Spree Suppprt
246
+
247
+ ### OptionTypes & Variants
248
+
249
+ When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
250
+ support for OptionTypes with values
251
+
252
+ Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
253
+ appropriate OptionValues to go with that Type.
254
+
255
+ For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
256
+ (such as small, medium etc)
257
+
258
+ 'Option Types'
259
+ size:small,medium,large|colour:red,white
260
+
261
+ If no such OptionType exists, e.g size, then a new one is created with the supplied name.
262
+
263
+ Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
264
+
265
+ Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
266
+ Currently a unique SKU is created by adding an index to the master's sku.
267
+
268
+ TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
269
+
270
+ ### Properties
271
+
272
+ The properties to associate with this product.
273
+ Properties are for small snippets of text, shared across many products,
274
+ and are for display purposes only.
275
+
276
+ An optional display value can be supplied to supplement the displayed text.
277
+
278
+ As for all associations can contain multiple name/value sets in default form :
279
+
280
+ Property:display_value|Property:display_value
281
+
282
+ Example - No values :
283
+ manufacturer|standard
284
+
285
+ Example - Display values :
286
+ manufacturer:somebody else plc|standard:ISOBlah21
287
+
288
+ ## TODO
289
+
290
+ - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
291
+
292
+ - Smart sorting of column processing order ....
293
+
294
+ - Does not currently ensure mandatory columns (for valid?) processed first.
295
+
296
+ - Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
297
+ rather than active record, so we can support additional ORMs
298
+
299
+ - Create separate Spree extension to support import/export via the admin gui
300
+
301
+ ## License
302
+
303
+ Copyright:: (c) Autotelik Media Ltd 2011
304
+
305
+ Author :: Tom Statter
306
+
307
+ Date :: Dec 2011
308
+
309
+ The MIT License
310
+
311
+ Permission is hereby granted, free of charge, to any person obtaining a copy
312
+ of this software and associated documentation files (the "Software"), to deal
313
+ in the Software without restriction, including without limitation the rights
314
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
315
+ copies of the Software, and to permit persons to whom the Software is
316
+ furnished to do so, subject to the following conditions:
317
+
318
+ The above copyright notice and this permission notice shall be included in
319
+ all copies or substantial portions of the Software.
320
+
321
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
322
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
323
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
324
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
325
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
326
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
306
327
  THE SOFTWARE.