datashift 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/.document +5 -5
  2. data/Gemfile +28 -25
  3. data/LICENSE.txt +26 -26
  4. data/README.markdown +302 -285
  5. data/README.rdoc +19 -19
  6. data/Rakefile +93 -95
  7. data/VERSION +5 -5
  8. data/datashift.gemspec +162 -178
  9. data/lib/applications/jruby/jexcel_file.rb +396 -396
  10. data/lib/applications/jruby/word.rb +79 -79
  11. data/lib/datashift.rb +152 -113
  12. data/lib/datashift/exceptions.rb +11 -11
  13. data/lib/datashift/file_definitions.rb +353 -353
  14. data/lib/datashift/mapping_file_definitions.rb +87 -87
  15. data/lib/datashift/method_detail.rb +236 -236
  16. data/lib/datashift/method_mapper.rb +256 -256
  17. data/lib/generators/csv_generator.rb +36 -36
  18. data/lib/generators/excel_generator.rb +121 -121
  19. data/lib/generators/generator_base.rb +13 -13
  20. data/lib/helpers/core_ext/to_b.rb +24 -24
  21. data/lib/helpers/spree_helper.rb +131 -131
  22. data/lib/java/poi-3.7/LICENSE +507 -507
  23. data/lib/java/poi-3.7/NOTICE +21 -21
  24. data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
  25. data/lib/loaders/csv_loader.rb +98 -98
  26. data/lib/loaders/excel_loader.rb +154 -149
  27. data/lib/loaders/loader_base.rb +403 -331
  28. data/lib/loaders/spreadsheet_loader.rb +136 -136
  29. data/lib/loaders/spree/image_loader.rb +45 -45
  30. data/lib/loaders/spree/product_loader.rb +224 -224
  31. data/spec/csv_loader_spec.rb +30 -30
  32. data/spec/datashift_spec.rb +26 -26
  33. data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
  34. data/spec/excel_generator_spec.rb +78 -78
  35. data/spec/excel_loader_spec.rb +204 -176
  36. data/spec/file_definitions.rb +141 -141
  37. data/spec/fixtures/.~lock.ProjectsSingleCategories.xls# +1 -0
  38. data/spec/fixtures/ProjectsDefaults.yml +29 -0
  39. data/spec/fixtures/config/database.yml +24 -24
  40. data/spec/fixtures/interact_models_db.sqlite +0 -0
  41. data/spec/fixtures/interact_spree_db.sqlite +0 -0
  42. data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
  43. data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
  44. data/spec/fixtures/spree/SpreeProducts.csv +4 -4
  45. data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
  46. data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
  47. data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
  48. data/spec/fixtures/test_model_defs.rb +57 -57
  49. data/spec/loader_spec.rb +120 -120
  50. data/spec/method_mapper_spec.rb +237 -237
  51. data/spec/spec_helper.rb +115 -115
  52. data/spec/spree_generator_spec.rb +64 -64
  53. data/spec/spree_loader_spec.rb +310 -310
  54. data/spec/spree_method_mapping_spec.rb +214 -214
  55. data/tasks/config/seed_fu_product_template.erb +15 -15
  56. data/tasks/config/tidy_config.txt +12 -12
  57. data/tasks/db_tasks.rake +65 -64
  58. data/tasks/excel_generator.rake +78 -78
  59. data/tasks/file_tasks.rake +36 -36
  60. data/tasks/import/csv.rake +49 -49
  61. data/tasks/import/excel.rake +71 -66
  62. data/tasks/spree/image_load.rake +108 -108
  63. data/tasks/spree/product_loader.rake +43 -43
  64. data/tasks/word_to_seedfu.rake +166 -166
  65. data/test/helper.rb +18 -18
  66. data/test/test_interact.rb +7 -7
  67. metadata +7 -38
  68. data/Gemfile.lock +0 -211
  69. data/bin/autospec +0 -16
  70. data/bin/convert_to_should_syntax +0 -16
  71. data/bin/erubis +0 -16
  72. data/bin/htmldiff +0 -16
  73. data/bin/jeweler +0 -16
  74. data/bin/ldiff +0 -16
  75. data/bin/nokogiri +0 -16
  76. data/bin/rackup +0 -16
  77. data/bin/rails +0 -16
  78. data/bin/rake +0 -16
  79. data/bin/rake2thor +0 -16
  80. data/bin/ri +0 -16
  81. data/bin/rspec +0 -16
  82. data/bin/spree +0 -16
  83. data/bin/thor +0 -16
  84. data/bin/tilt +0 -16
  85. data/bin/tt +0 -16
data/.document CHANGED
@@ -1,5 +1,5 @@
1
- lib/**/*.rb
2
- bin/*
3
- -
4
- features/**/*.feature
5
- LICENSE.txt
1
+ lib/**/*.rb
2
+ bin/*
3
+ -
4
+ features/**/*.feature
5
+ LICENSE.txt
data/Gemfile CHANGED
@@ -1,25 +1,28 @@
1
- source "http://rubygems.org"
2
- # Add dependencies required to use your gem here.
3
- # Example:
4
- # gem "activesupport", ">= 2.3.5"
5
-
6
- # Add dependencies to develop your gem here.
7
- # Include everything needed to run rake, tests, features, etc.
8
- group :development do
9
- gem "rails", '3.1.3'
10
- gem "activerecord", '3.1.3'
11
- gem "activesupport", '3.1.3'
12
-
13
- platform :jruby do
14
- gem 'jruby-openssl'
15
- gem 'activerecord-jdbcsqlite3-adapter'
16
- end
17
-
18
- gem 'spree', '0.70.3'
19
- gem "rspec", ">= 0"
20
- gem "shoulda", ">= 0"
21
- gem "rdoc", "~> 3.12"
22
- gem "bundler", "~> 1.0.0"
23
- gem "jeweler", "~> 1.8.3"
24
- end
25
-
1
+ source "http://rubygems.org"
2
+ # Add dependencies required to use your gem here.
3
+ # Example:
4
+ # gem "activesupport", ">= 2.3.5"
5
+
6
+ # Add dependencies to develop your gem here.
7
+ # Include everything needed to run rake, tests, features, etc.
8
+
9
+
10
+ # We leave it up to the main Rails app to define the actual versions required
11
+ group :development do
12
+ gem "rails"
13
+ gem "activerecord"
14
+ gem "activesupport"
15
+
16
+ platform :jruby do
17
+ gem 'jruby-openssl'
18
+ gem 'activerecord-jdbcsqlite3-adapter'
19
+ end
20
+
21
+ gem 'spree', '0.70.3'
22
+ gem "rspec", ">= 0"
23
+ gem "shoulda", ">= 0"
24
+ gem "rdoc", "~> 3.12"
25
+ gem "bundler", "~> 1.0.0"
26
+ gem "jeweler", "~> 1.8.3"
27
+ end
28
+
data/LICENSE.txt CHANGED
@@ -1,27 +1,27 @@
1
- == DataShift
2
-
3
- ## License
4
-
5
- Copyright:: (c) Autotelik Media Ltd 2011
6
- Author :: Tom Statter
7
- Date :: Sept 2011
8
-
9
- The MIT License
10
-
11
- Permission is hereby granted, free of charge, to any person obtaining a copy
12
- of this software and associated documentation files (the "Software"), to deal
13
- in the Software without restriction, including without limitation the rights
14
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
- copies of the Software, and to permit persons to whom the Software is
16
- furnished to do so, subject to the following conditions:
17
-
18
- The above copyright notice and this permission notice shall be included in
19
- all copies or substantial portions of the Software.
20
-
21
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ == DataShift
2
+
3
+ ## License
4
+
5
+ Copyright:: (c) Autotelik Media Ltd 2011
6
+ Author :: Tom Statter
7
+ Date :: Sept 2011
8
+
9
+ The MIT License
10
+
11
+ Permission is hereby granted, free of charge, to any person obtaining a copy
12
+ of this software and associated documentation files (the "Software"), to deal
13
+ in the Software without restriction, including without limitation the rights
14
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
15
+ copies of the Software, and to permit persons to whom the Software is
16
+ furnished to do so, subject to the following conditions:
17
+
18
+ The above copyright notice and this permission notice shall be included in
19
+ all copies or substantial portions of the Software.
20
+
21
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
22
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
23
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
24
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
25
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
26
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
27
27
  THE SOFTWARE.
data/README.markdown CHANGED
@@ -1,286 +1,303 @@
1
- ## DataShift
2
-
3
- Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
-
5
- ### Features
6
-
7
- Map Active Record models and associations to files.
8
-
9
- Import and Export ActiveRecord based data through .xls or CSV files.
10
-
11
- Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
12
-
13
- High level rake tasks provided.
14
-
15
- Spree E-Commerce specific loaders and rake tasks provided enabling import/export of all data including Products with
16
- complex associations such as Options/Variants and Images.
17
-
18
- Example spreadsheets, fully documented with comments for each column.
19
-
20
-
21
- ## Installation
22
-
23
- Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
24
-
25
- `gem install datashift`
26
-
27
- To use :
28
-
29
- gem 'datashift'
30
- require 'datashift'
31
-
32
- To pull the tasks in, add this call to your Rakefile :
33
-
34
- DataShift ::load_tasks
35
-
36
- N.B - To use the Excel loader, OLE and Excel are NOT required, however
37
- JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
38
-
39
- Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
40
-
41
- if(DataShift::Guards::jruby? )
42
- ..do something with Excel
43
- else
44
- ..do something with CSV
45
- end
46
-
47
- ## Active Record - Import/Export
48
-
49
- Provides wrappers and rake tasks for importing/exporting data to DBs, from various sources,
50
- including csv and .xls files (Excel/Open Office)
51
-
52
- Simplifies the exchange of data, between such files and any active record supported database.
53
-
54
- A core feature of DataShift is the MethodMapper, which provides features for collecting
55
- reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
56
-
57
- A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
58
-
59
- MethodMapper.find_operators( Blog )
60
-
61
- This then allows Import/Export to be achieved, by mapping the file's header and column data to MethodMapper's operators
62
- (i.e. the model's attributes and associations).
63
-
64
- Here we retrieve the method details for a column name from a file, "Blog Date"
65
-
66
- MethodMapper.find_method_detail( Blog, "Blog Date" )
67
-
68
- Loaders can use this method lookup functionality, to find the correct association for a column heading,
69
- and populate AR object with row data.
70
-
71
- This means data can be mapped to any model without any further coding. Generators are also supplied to export
72
- a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
73
-
74
- MethodMapper also stores column type information so the raw file data can be provided as is,
75
- and whenever possible, under the bonnet the data will be cast to correct DB type.
76
-
77
- Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
78
-
79
- MethodMapper.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
80
-
81
- Because it's based on reflection against the model, can build complex relationships between tables during import/export,
82
- and extending data files with new columns need not require any additional Ruby coding.
83
-
84
- New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
85
- attribute or association name in the header row.
86
-
87
-
88
- The Loader attempts to handle various human read-able forms of column names.
89
-
90
- For example, given an association on the model called, product_properties, will successfully load
91
- from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
92
-
93
- For has_many associations, either multiple columns can be used,
94
- or multiple values can be specified in a single column using suitable delimiters.
95
-
96
- Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
97
-
98
- Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
99
- for loading Spree Products, and associated data such as Product Variants, and Images.
100
-
101
- ## Template Generation and Export
102
-
103
- Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
104
- These can be provided to developers or business users, as a template for data collection and then loading.
105
-
106
- Export tasks can be used to export of a model's definition and any existing data stored in the database.
107
-
108
- This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
109
-
110
-
111
- ## Example Spreadsheets
112
-
113
- A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
114
-
115
- Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
116
- taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
117
-
118
- Column headings contain comments with full descriptions and instructions on syntax.
119
-
120
-
121
- ## Features
122
-
123
- - *High level wrappers around applications including Excel and Word
124
-
125
- Quickly and easily access common enterprise applications through Ruby
126
-
127
- Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
128
- and Excel does not need to be installed.
129
-
130
- The required POI jars are already included.
131
-
132
- - *Direct Excel export*
133
-
134
- Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
135
- easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
136
-
137
- No need to save to CSV or map to YAML.
138
-
139
- - *Semi-Smart Name Lookup*
140
-
141
- Includes helper classes that find and store details of all possible associations on an AR class.
142
- Given a user supplied name, attempts to find the requested association.
143
-
144
- Example usage, load from a file or spreadsheet where the column names are only
145
- an approximation of the actual associations, so given 'Product Properties' heading,
146
- finds real association 'product_properties' to send or call on the AR object
147
-
148
- - *Associations*
149
-
150
- Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
151
- via either multiple columns, or via specially delimited entry in a single (column). See Details section.
152
-
153
- Supports delegated attributes.
154
-
155
- - *Rake Tasks*
156
-
157
- High level Rake tasks are provided, only required to supply model class, and file location :
158
-
159
- jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
160
-
161
-
162
- - *Spree Rake Tasks*
163
-
164
- Specific Rake tasks are also provided for Spree loading - currently supports Product with associations,
165
- and Image loading.
166
-
167
- jruby -S rake datashift:spree:products input=C:\MyProducts.xls
168
-
169
-
170
- **Product loading from Excel files specifically requires JRuby (But not Excel or OLE)**.
171
-
172
-
173
- - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
174
-
175
- Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
176
-
177
- Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
178
- parameter klass=Xyz.
179
-
180
- In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
181
-
182
- Image loading **does not** specifically require JRuby
183
-
184
- A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
185
-
186
- rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
187
-
188
- rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
189
-
190
- ## Import to Active Record
191
-
192
- ### Associations
193
-
194
- To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
195
-
196
- A single association column can contain multiple name/value sets, in string form :
197
-
198
- column:lookup_key_1, lookup_key_2,...
199
-
200
- So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
201
-
202
- |Categories|
203
-
204
- reference:category_001,category_002
205
-
206
- During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
207
- and our Project model will contain those two i.e project.categories = [category_002,category_003]
208
-
209
- ## Spree Suppprt
210
-
211
- ### OptionTypes & Variants
212
-
213
- When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
214
- support for OptionTypes with values
215
-
216
- Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
217
- appropriate OptionValues to go with that Type.
218
-
219
- For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
220
- (such as small, medium etc)
221
-
222
- 'Option Types'
223
- size:small,medium,large|colour:red,white
224
-
225
- If no such OptionType exists, e.g size, then a new one is created with the supplied name.
226
-
227
- Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
228
-
229
- Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
230
- Currently a unique SKU is created by adding an index to the master's sku.
231
-
232
- TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
233
-
234
- ### Properties
235
-
236
- The properties to associate with this product.
237
- Properties are for small snippets of text, shared across many products,
238
- and are for display purposes only.
239
-
240
- An optional display value can be supplied to supplement the displayed text.
241
-
242
- As for all associations can contain multiple name/value sets in default form :
243
-
244
- Property:display_value|Property:display_value
245
-
246
- Example - No values :
247
- manufacturer|standard
248
-
249
- Example - Display values :
250
- manufacturer:somebody else plc|standard:ISOBlah21
251
-
252
- ## TODO
253
-
254
- - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
255
-
256
- - Smart sorting of column processing order ....
257
-
258
- - Does not currently ensure mandatory columns (for valid?) processed first.
259
-
260
- ## License
261
-
262
- Copyright:: (c) Autotelik Media Ltd 2011
263
-
264
- Author :: Tom Statter
265
-
266
- Date :: Dec 2011
267
-
268
- The MIT License
269
-
270
- Permission is hereby granted, free of charge, to any person obtaining a copy
271
- of this software and associated documentation files (the "Software"), to deal
272
- in the Software without restriction, including without limitation the rights
273
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
274
- copies of the Software, and to permit persons to whom the Software is
275
- furnished to do so, subject to the following conditions:
276
-
277
- The above copyright notice and this permission notice shall be included in
278
- all copies or substantial portions of the Software.
279
-
280
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
281
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
282
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
283
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
284
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
285
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
1
+ ## DataShift
2
+
3
+ Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
4
+
5
+ ### Features
6
+
7
+ Map Active Record models and associations to files, generate sample templates.
8
+
9
+ Import and Export ActiveRecord models through .xls or CSV files, including
10
+ all associations and with configurable defaults.
11
+
12
+ Create, parse and use Excel/OpenOffice (.xls) documents dynamically from Ruby.
13
+
14
+ High level rake tasks provided for key features.
15
+
16
+ Easily extendable Loader functionality to deal with non trivial import cases, such
17
+ as complex association lookups.
18
+
19
+ Specific loaders and rake tasks provided out the box for Spree E-Commerce,
20
+ enabling import/export of all data including Products with complex associations such as Options/Variants and Images.
21
+
22
+ Many example Spreadsheets provided, fully documented with comments for each column.
23
+
24
+
25
+ ## Installation
26
+
27
+ Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
28
+
29
+ `gem install datashift`
30
+
31
+ To use :
32
+
33
+ gem 'datashift'
34
+ require 'datashift'
35
+
36
+ To pull the tasks in, add this call to your Rakefile :
37
+
38
+ DataShift::load_tasks
39
+
40
+ N.B - To use the Excel loader, OLE and Excel are NOT required, however
41
+ JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
42
+
43
+ Guards are provided, and used internally, for mixed Ruby setups. Can be used like :
44
+
45
+ if(DataShift::Guards::jruby? )
46
+ ..do something with Excel
47
+ else
48
+ ..do something with CSV
49
+ end
50
+
51
+ ## Active Record - Import/Export
52
+
53
+ Provides high level rake tasks for importing data via ActiveRecord models into a DB,
54
+ from various sources, currently csv or .xls files (Excel/Open Office)
55
+
56
+
57
+ bundle exec rake datashift:import:excel model=BlogPost input=BlogPostImport.xls verbose=true
58
+
59
+
60
+ The library can be easily extended with Loaders to deal with non trivial cases required to exchange data between
61
+ such files and any active record supported database.
62
+
63
+ Spree loaders are an example, these illustrate over riding processing for specific columns with
64
+ complicated lookup requirements.
65
+
66
+ A core feature of DataShift is the MethodMapper, which provides features for collecting
67
+ reflection information from ActiveRecord models (all different associations, including join tables with many-to-many relationships).
68
+
69
+ A full picture of all possible operations on a class can be created very easily, for example ona Blog model :
70
+
71
+ MethodMapper.find_operators( Blog )
72
+
73
+ This then allows Import/Export to be achieved, by mapping the file's header and column data to MethodMapper's operators
74
+ (i.e. the model's attributes and associations).
75
+
76
+ Here we retrieve the method details for a column name from a file, "Blog Date"
77
+
78
+ MethodMapper.find_method_detail( Blog, "Blog Date" )
79
+
80
+ Loaders can use this method lookup functionality, to find the correct association for a column heading,
81
+ and populate AR object with row data.
82
+
83
+ This means data can be mapped to any model without any further coding. Generators are also supplied to export
84
+ a models attributes and associations to files, thus providing template spreadsheets that any user can fill out.
85
+
86
+ MethodMapper also stores column type information so the raw file data can be provided as is,
87
+ and whenever possible, under the bonnet the data will be cast to correct DB type.
88
+
89
+ Here we show how a column name from a file, "Blog Date", can be mapped to Assign a stringified date, to the blog_date column, on a new Blog object :
90
+
91
+ MethodMapper.find_method_detail( Blog, "Blog Date" ).assign( Blog.new, "Sat Jul 23 2011" )
92
+
93
+ Because it's based on reflection against the model, can build complex relationships between tables during import/export,
94
+ and extending data files with new columns need not require any additional Ruby coding.
95
+
96
+ New columns can simply be added to the Excel/Open Office spreadsheet, or CSV file, setting the new
97
+ attribute or association name in the header row.
98
+
99
+
100
+ The Loader attempts to handle various human read-able forms of column names.
101
+
102
+ For example, given an association on the model called, product_properties, will successfully load
103
+ from columns with headings such as 'product_properties', 'Product Properties', 'ProductProperties' 'product properties' etc
104
+
105
+ For has_many associations, either multiple columns can be used,
106
+ or multiple values can be specified in a single column using suitable delimiters.
107
+
108
+ Modular - so complex associations/mappings that require non generic lookups, can be handled by extending the loader engine.
109
+
110
+ Original focus was on support for the Open Source Spree e-commerce project, so includes specific loaders and rake tasks
111
+ for loading Spree Products, and associated data such as Product Variants, and Images.
112
+
113
+ ## Template Generation and Export
114
+
115
+ Template generation tasks can be used to export a model's definition as column headings to CSV or .xls.
116
+ These can be provided to developers or business users, as a template for data collection and then loading.
117
+
118
+ Export tasks can be used to export of a model's definition and any existing data stored in the database.
119
+
120
+ This data can be exported directly to CSV or Excel/OpenOffice spreadsheets.
121
+
122
+
123
+ ## Example Spreadsheets
124
+
125
+ A number of example Spreadsheets with headers and comments, can be found in the spec/fixtures directory.
126
+
127
+ Extensive Spree samples - including .xls and csv versions for simple Products or complex Products with multiple
128
+ taxons, variants properties etc - can be found in the spec/fixtures/spree subdirectory.
129
+
130
+ Column headings contain comments with full descriptions and instructions on syntax.
131
+
132
+
133
+ ## Features
134
+
135
+ - *High level wrappers around applications including Excel and Word
136
+
137
+ Quickly and easily access common enterprise applications through Ruby
138
+
139
+ Wrapper around MS Excel File format, acheived via Apache POI under JRuby, so not restricted to Windows
140
+ and Excel does not need to be installed.
141
+
142
+ The required POI jars are already included.
143
+
144
+ - *Direct Excel export*
145
+
146
+ Excel/OpenOffice spreadsheets are heavily used in many sectors, so direct support makes it
147
+ easier and quicker to migrate your client's data into a Rails/ActiveRecord project.
148
+
149
+ No need to save to CSV or map to YAML.
150
+
151
+ - *Semi-Smart Name Lookup*
152
+
153
+ Includes helper classes that find and store details of all possible associations on an AR class.
154
+ Given a user supplied name, attempts to find the requested association.
155
+
156
+ Example usage, load from a file or spreadsheet where the column names are only
157
+ an approximation of the actual associations, so given 'Product Properties' heading,
158
+ finds real association 'product_properties' to send or call on the AR object
159
+
160
+ - *Associations*
161
+
162
+ Can handle 'belongs_to, 'has_many' and 'has_one' associations, including assignment of multiple objects
163
+ via either multiple columns, or via specially delimited entry in a single (column). See Details section.
164
+
165
+ Supports delegated attributes.
166
+
167
+ - *Rake Tasks*
168
+
169
+ High level Rake tasks are provided, only required to supply model class, and file location :
170
+
171
+ jruby -S rake datashift:import:excel model=MusicTrack input=MyTrackListing.xls
172
+
173
+
174
+ - *Spree Rake Tasks*
175
+
176
+ Specific Rake tasks are also provided for Spree loading - currently supports Product with associations,
177
+ and Image loading.
178
+
179
+ jruby -S rake datashift:spree:products input=C:\MyProducts.xls
180
+
181
+
182
+ **Product loading from Excel files specifically requires JRuby (But not Excel or OLE)**.
183
+
184
+
185
+ - *Seamless Spree Image loading can be achieved by ensuring SKU or class Name features in Image filename.
186
+
187
+ Lookup is performed either via the SKU being prepended to the image name, or by the image name being equal to the **name attribute** of the klass in question.
188
+
189
+ Images can be attached to any class defined with a suitable association. The class to use can be configured in rake task via
190
+ parameter klass=Xyz.
191
+
192
+ In the Spree tasks, this defaults to Product, so attempts to attach Image to a Product via Product SKU or Name.
193
+
194
+ Image loading **does not** specifically require JRuby
195
+
196
+ A report is generated in the current working directory detailing any Images in the paths that could not be matched with a Product.
197
+
198
+ rake datashift:spree:images input=C:\images\product_images skip_if_no_assoc=true
199
+
200
+ rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
201
+
202
+ ## Import to Active Record
203
+
204
+ ### Associations
205
+
206
+ To perform a lookup for an associated model, the primary column(s) must be supplied, along with required select values for those columns.
207
+
208
+ A single association column can contain multiple name/value sets, in string form :
209
+
210
+ column:lookup_key_1, lookup_key_2,...
211
+
212
+ So if our Project model has many Categories, we can supply a Category list, which is keyed on the column Category.reference with :
213
+
214
+ |Categories|
215
+
216
+ reference:category_001,category_002
217
+
218
+ During loading, a call to find_all_by_reference will be made, picking up the 2 categories with matching references,
219
+ and our Project model will contain those two i.e project.categories = [category_002,category_003]
220
+
221
+ ## Spree Suppprt
222
+
223
+ ### OptionTypes & Variants
224
+
225
+ When loaded with the Spree specific tasks, spree specific over rides are supported, such as direct s
226
+ support for OptionTypes with values
227
+
228
+ Any 'Option Types' columns can contain the OptionType to associate with the Product, plus a selection of
229
+ appropriate OptionValues to go with that Type.
230
+
231
+ For example, in a single column/row we could supply 2 OptionTypes (named, size & colour), with a selection values
232
+ (such as small, medium etc)
233
+
234
+ 'Option Types'
235
+ size:small,medium,large|colour:red,white
236
+
237
+ If no such OptionType exists, e.g size, then a new one is created with the supplied name.
238
+
239
+ Next the OptionValues are also parsed, again if no such OptionValue exists, e.g small, then a new one is created with the supplied name.
240
+
241
+ Lastly a Variant is created on each OptionValue, with price and availaable dates being copied from Master.
242
+ Currently a unique SKU is created by adding an index to the master's sku.
243
+
244
+ TODO - Enable a hash of attributes to be supplied in association columns to enable more control over creation of associated objects.
245
+
246
+ ### Properties
247
+
248
+ The properties to associate with this product.
249
+ Properties are for small snippets of text, shared across many products,
250
+ and are for display purposes only.
251
+
252
+ An optional display value can be supplied to supplement the displayed text.
253
+
254
+ As for all associations can contain multiple name/value sets in default form :
255
+
256
+ Property:display_value|Property:display_value
257
+
258
+ Example - No values :
259
+ manufacturer|standard
260
+
261
+ Example - Display values :
262
+ manufacturer:somebody else plc|standard:ISOBlah21
263
+
264
+ ## TODO
265
+
266
+ - Add direct Image loading to Spree i.e should be able to specify just path in a column .. "/images/red-tshirt.jpg"
267
+
268
+ - Smart sorting of column processing order ....
269
+
270
+ - Does not currently ensure mandatory columns (for valid?) processed first.
271
+
272
+ - Look at implementing import/export API using something like https://github.com/ianwhite/orm_adapter
273
+ rather than active record, so we can support additional ORMs
274
+
275
+ - Create separate Spree extension to support import/export via the admin gui
276
+
277
+ ## License
278
+
279
+ Copyright:: (c) Autotelik Media Ltd 2011
280
+
281
+ Author :: Tom Statter
282
+
283
+ Date :: Dec 2011
284
+
285
+ The MIT License
286
+
287
+ Permission is hereby granted, free of charge, to any person obtaining a copy
288
+ of this software and associated documentation files (the "Software"), to deal
289
+ in the Software without restriction, including without limitation the rights
290
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
291
+ copies of the Software, and to permit persons to whom the Software is
292
+ furnished to do so, subject to the following conditions:
293
+
294
+ The above copyright notice and this permission notice shall be included in
295
+ all copies or substantial portions of the Software.
296
+
297
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
298
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
299
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
300
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
301
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
302
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
286
303
  THE SOFTWARE.