datashift 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (85) hide show
  1. data/.document +5 -5
  2. data/Gemfile +28 -25
  3. data/LICENSE.txt +26 -26
  4. data/README.markdown +302 -285
  5. data/README.rdoc +19 -19
  6. data/Rakefile +93 -95
  7. data/VERSION +5 -5
  8. data/datashift.gemspec +162 -178
  9. data/lib/applications/jruby/jexcel_file.rb +396 -396
  10. data/lib/applications/jruby/word.rb +79 -79
  11. data/lib/datashift.rb +152 -113
  12. data/lib/datashift/exceptions.rb +11 -11
  13. data/lib/datashift/file_definitions.rb +353 -353
  14. data/lib/datashift/mapping_file_definitions.rb +87 -87
  15. data/lib/datashift/method_detail.rb +236 -236
  16. data/lib/datashift/method_mapper.rb +256 -256
  17. data/lib/generators/csv_generator.rb +36 -36
  18. data/lib/generators/excel_generator.rb +121 -121
  19. data/lib/generators/generator_base.rb +13 -13
  20. data/lib/helpers/core_ext/to_b.rb +24 -24
  21. data/lib/helpers/spree_helper.rb +131 -131
  22. data/lib/java/poi-3.7/LICENSE +507 -507
  23. data/lib/java/poi-3.7/NOTICE +21 -21
  24. data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
  25. data/lib/loaders/csv_loader.rb +98 -98
  26. data/lib/loaders/excel_loader.rb +154 -149
  27. data/lib/loaders/loader_base.rb +403 -331
  28. data/lib/loaders/spreadsheet_loader.rb +136 -136
  29. data/lib/loaders/spree/image_loader.rb +45 -45
  30. data/lib/loaders/spree/product_loader.rb +224 -224
  31. data/spec/csv_loader_spec.rb +30 -30
  32. data/spec/datashift_spec.rb +26 -26
  33. data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
  34. data/spec/excel_generator_spec.rb +78 -78
  35. data/spec/excel_loader_spec.rb +204 -176
  36. data/spec/file_definitions.rb +141 -141
  37. data/spec/fixtures/.~lock.ProjectsSingleCategories.xls# +1 -0
  38. data/spec/fixtures/ProjectsDefaults.yml +29 -0
  39. data/spec/fixtures/config/database.yml +24 -24
  40. data/spec/fixtures/interact_models_db.sqlite +0 -0
  41. data/spec/fixtures/interact_spree_db.sqlite +0 -0
  42. data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
  43. data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
  44. data/spec/fixtures/spree/SpreeProducts.csv +4 -4
  45. data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
  46. data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
  47. data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
  48. data/spec/fixtures/test_model_defs.rb +57 -57
  49. data/spec/loader_spec.rb +120 -120
  50. data/spec/method_mapper_spec.rb +237 -237
  51. data/spec/spec_helper.rb +115 -115
  52. data/spec/spree_generator_spec.rb +64 -64
  53. data/spec/spree_loader_spec.rb +310 -310
  54. data/spec/spree_method_mapping_spec.rb +214 -214
  55. data/tasks/config/seed_fu_product_template.erb +15 -15
  56. data/tasks/config/tidy_config.txt +12 -12
  57. data/tasks/db_tasks.rake +65 -64
  58. data/tasks/excel_generator.rake +78 -78
  59. data/tasks/file_tasks.rake +36 -36
  60. data/tasks/import/csv.rake +49 -49
  61. data/tasks/import/excel.rake +71 -66
  62. data/tasks/spree/image_load.rake +108 -108
  63. data/tasks/spree/product_loader.rake +43 -43
  64. data/tasks/word_to_seedfu.rake +166 -166
  65. data/test/helper.rb +18 -18
  66. data/test/test_interact.rb +7 -7
  67. metadata +7 -38
  68. data/Gemfile.lock +0 -211
  69. data/bin/autospec +0 -16
  70. data/bin/convert_to_should_syntax +0 -16
  71. data/bin/erubis +0 -16
  72. data/bin/htmldiff +0 -16
  73. data/bin/jeweler +0 -16
  74. data/bin/ldiff +0 -16
  75. data/bin/nokogiri +0 -16
  76. data/bin/rackup +0 -16
  77. data/bin/rails +0 -16
  78. data/bin/rake +0 -16
  79. data/bin/rake2thor +0 -16
  80. data/bin/ri +0 -16
  81. data/bin/rspec +0 -16
  82. data/bin/spree +0 -16
  83. data/bin/thor +0 -16
  84. data/bin/tilt +0 -16
  85. data/bin/tt +0 -16
@@ -1,257 +1,257 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2010
4
- # License:: MIT
5
- #
6
- # Details:: A base class that stores details of all possible associations on AR classes and,
7
- # given user supplied class and name, attempts to find correct attribute/association.
8
- #
9
- # Derived classes define where the user supplied list of names originates from.
10
- #
11
- # Example usage, load from a spreadsheet where the column names are only
12
- # an approximation of the actual associations. Given a column heading of
13
- # 'Product Properties' on class Product, find_method_detail() would search AR model,
14
- # and return details of real has_many association 'product_properties'.
15
- #
16
- # This real association can then be used to send spreadsheet row data to the AR object.
17
- #
18
- require 'method_detail'
19
-
20
- module DataShift
21
-
22
- class MethodMapper
23
-
24
- attr_accessor :header_row, :headers
25
- attr_accessor :method_details, :missing_methods
26
-
27
- @@has_many = Hash.new
28
- @@belongs_to = Hash.new
29
- @@assignments = Hash.new
30
- @@column_types = Hash.new
31
-
32
- def initialize
33
- @method_details = []
34
- @headers = []
35
- end
36
-
37
- # Build complete picture of the methods whose names listed in method_list
38
- # Handles method names as defined by a user or in file headers where names may
39
- # not be exactly as required e.g handles capitalisation, white space, _ etc
40
- # Returns: Array of matching method_details
41
- #
42
- def populate_methods( klass, method_list )
43
- @method_details, @missing_methods = [], []
44
-
45
- method_list.each do |x|
46
- md = MethodMapper::find_method_detail( klass, x )
47
- md ? @method_details << md : @missing_methods << x
48
- end
49
- #@method_details.compact! .. currently we may neeed to map via the index on @method_details so don't remove nils for now
50
- @method_details
51
- end
52
-
53
- # The raw client supplied names
54
- def method_names()
55
- @method_details.collect( &:name )
56
- end
57
-
58
- # The true operator names discovered from model
59
- def operator_names()
60
- @method_details.collect( &:operator )
61
- end
62
-
63
- # Returns true if discovered methods contain every operator in mandatory_list
64
- def contains_mandatory?( mandatory_list )
65
- [ [*mandatory_list] - operator_names].flatten.empty?
66
- end
67
-
68
- def missing_mandatory( mandatory_list )
69
- [ [*mandatory_list] - operator_names].flatten
70
- end
71
-
72
- # Create picture of the operators for assignment available on an AR model,
73
- # including via associations (which provide both << and = )
74
- # Options:
75
- # :reload => clear caches and reperform lookup
76
- # :instance_methods => if true include instance method type assignment operators as well as model's pure columns
77
- #
78
- def self.find_operators(klass, options = {} )
79
-
80
- # Find the has_many associations which can be populated via <<
81
- if( options[:reload] || @@has_many[klass].nil? )
82
- @@has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
83
- klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(@@has_many[klass]) { |x,i| x << i.name.to_s }
84
- end
85
- # puts "DEBUG: Has Many Associations:", @@has_many[klass].inspect
86
-
87
- # Find the belongs_to associations which can be populated via Model.belongs_to_name = OtherArModelObject
88
- if( options[:reload] || @@belongs_to[klass].nil? )
89
- @@belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
90
- end
91
-
92
- #puts "Belongs To Associations:", @@belongs_to[klass].inspect
93
-
94
- # Find the has_one associations which can be populated via Model.has_one_name = OtherArModelObject
95
- if( options[:reload] || self.has_one[klass].nil? )
96
- self.has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
97
- end
98
-
99
- #puts "has_one Associations:", self.has_one[klass].inspect
100
-
101
- # Find the model's column associations which can be populated via xxxxxx= value
102
- # Note, not all reflections return method names in same style so we convert all to
103
- # the raw form i.e without the '=' for consistency
104
- if( options[:reload] || @@assignments[klass].nil? )
105
-
106
- @@assignments[klass] = klass.column_names
107
-
108
- if(options[:instance_methods] == true)
109
- setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
110
-
111
- if(klass.respond_to? :defined_activerecord_methods)
112
- setters = setters - klass.defined_activerecord_methods.to_a
113
- end
114
-
115
- # get into same format as other names
116
- @@assignments[klass] += setters.map{|i| i.gsub(/=/, '')}
117
- end
118
-
119
- @@assignments[klass] -= @@has_many[klass] if(@@has_many[klass])
120
- @@assignments[klass] -= @@belongs_to[klass] if(@@belongs_to[klass])
121
- @@assignments[klass] -= self.has_one[klass] if(self.has_one[klass])
122
-
123
- @@assignments[klass].uniq!
124
-
125
- @@assignments[klass].each do |assign|
126
- @@column_types[klass] ||= {}
127
- column_def = klass.columns.find{ |col| col.name == assign }
128
- @@column_types[klass].merge!( assign => column_def) if column_def
129
- end
130
- end
131
- end
132
-
133
- def self.build_method_details( klass )
134
- @method_details ||= {}
135
-
136
- @method_details[klass] = []
137
-
138
- assignments_for(klass).each do |n|
139
- @method_details[klass] << MethodDetail.new(n, klass, n, :assignment)
140
- end
141
-
142
- has_one_for(klass).each do |n|
143
- @method_details[klass] << MethodDetail.new(n, klass, n, :has_one)
144
- end
145
-
146
- has_many_for(klass).each do |n|
147
- @method_details[klass] << MethodDetail.new(n, klass, n, :has_many)
148
- end
149
-
150
- belongs_to_for(klass).each do |n|
151
- @method_details[klass] << MethodDetail.new(n, klass, n, :belongs_to)
152
- end
153
- end
154
-
155
- def self.method_details
156
- @method_details ||= {}
157
- @method_details
158
- end
159
-
160
- # Find the proper format of name, appropriate call + column type for a given name.
161
- # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
162
- #
163
- # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
164
- #
165
- def self.find_method_detail( klass, external_name )
166
- operator = nil
167
-
168
- name = external_name.to_s
169
-
170
- # TODO - check out regexp to do this work better plus Inflections ??
171
- # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
172
- [
173
- name,
174
- name.tableize,
175
- name.gsub(' ', '_'),
176
- name.gsub(' ', '_').downcase,
177
- name.gsub(/(\s+)/, '_').downcase,
178
- name.gsub(' ', ''),
179
- name.gsub(' ', '').downcase,
180
- name.gsub(' ', '_').underscore].each do |n|
181
-
182
- operator = (assignments_for(klass).include?(n)) ? n : nil
183
-
184
- return MethodDetail.new(name, klass, operator, :assignment, @@column_types[klass]) if(operator)
185
-
186
- operator = (has_one_for(klass).include?(n)) ? n : nil
187
-
188
- return MethodDetail.new(name, klass, operator, :has_one, @@column_types[klass]) if(operator)
189
-
190
- operator = (has_many_for(klass).include?(n)) ? n : nil
191
-
192
- return MethodDetail.new(name, klass, operator, :has_many, @@column_types[klass]) if(operator)
193
-
194
- operator = (belongs_to_for(klass).include?(n)) ? n : nil
195
-
196
- return MethodDetail.new(name, klass, operator, :belongs_to, @@column_types[klass]) if(operator)
197
-
198
- end
199
-
200
- nil
201
- end
202
-
203
- def self.clear
204
- @@belongs_to.clear
205
- @@has_many.clear
206
- @@assignments.clear
207
- @@column_types.clear
208
- self.has_one.clear
209
- end
210
-
211
- def self.column_key(klass, column)
212
- "#{klass.name}:#{column}"
213
- end
214
-
215
- # TODO - remove use of class variables - not good Ruby design
216
- def self.belongs_to
217
- @@belongs_to
218
- end
219
-
220
- def self.has_many
221
- @@has_many
222
- end
223
-
224
- def self.has_one
225
- @has_one ||= {}
226
- @has_one
227
- end
228
-
229
- def self.assignments
230
- @@assignments
231
- end
232
- def self.column_types
233
- @@column_types
234
- end
235
-
236
-
237
- def self.belongs_to_for(klass)
238
- @@belongs_to[klass] || []
239
- end
240
- def self.has_many_for(klass)
241
- @@has_many[klass] || []
242
- end
243
-
244
- def self.has_one_for(klass)
245
- self.has_one[klass] || []
246
- end
247
-
248
- def self.assignments_for(klass)
249
- @@assignments[klass] || []
250
- end
251
- def self.column_type_for(klass, column)
252
- @@column_types[klass] ? @@column_types[klass][column] : []
253
- end
254
-
255
- end
256
-
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2010
4
+ # License:: MIT
5
+ #
6
+ # Details:: A base class that stores details of all possible associations on AR classes and,
7
+ # given user supplied class and name, attempts to find correct attribute/association.
8
+ #
9
+ # Derived classes define where the user supplied list of names originates from.
10
+ #
11
+ # Example usage, load from a spreadsheet where the column names are only
12
+ # an approximation of the actual associations. Given a column heading of
13
+ # 'Product Properties' on class Product, find_method_detail() would search AR model,
14
+ # and return details of real has_many association 'product_properties'.
15
+ #
16
+ # This real association can then be used to send spreadsheet row data to the AR object.
17
+ #
18
+ require 'method_detail'
19
+
20
+ module DataShift
21
+
22
+ class MethodMapper
23
+
24
+ attr_accessor :header_row, :headers
25
+ attr_accessor :method_details, :missing_methods
26
+
27
+ @@has_many = Hash.new
28
+ @@belongs_to = Hash.new
29
+ @@assignments = Hash.new
30
+ @@column_types = Hash.new
31
+
32
+ def initialize
33
+ @method_details = []
34
+ @headers = []
35
+ end
36
+
37
+ # Build complete picture of the methods whose names listed in method_list
38
+ # Handles method names as defined by a user or in file headers where names may
39
+ # not be exactly as required e.g handles capitalisation, white space, _ etc
40
+ # Returns: Array of matching method_details
41
+ #
42
+ def populate_methods( klass, method_list )
43
+ @method_details, @missing_methods = [], []
44
+
45
+ method_list.each do |x|
46
+ md = MethodMapper::find_method_detail( klass, x )
47
+ md ? @method_details << md : @missing_methods << x
48
+ end
49
+ #@method_details.compact! .. currently we may neeed to map via the index on @method_details so don't remove nils for now
50
+ @method_details
51
+ end
52
+
53
+ # The raw client supplied names
54
+ def method_names()
55
+ @method_details.collect( &:name )
56
+ end
57
+
58
+ # The true operator names discovered from model
59
+ def operator_names()
60
+ @method_details.collect( &:operator )
61
+ end
62
+
63
+ # Returns true if discovered methods contain every operator in mandatory_list
64
+ def contains_mandatory?( mandatory_list )
65
+ [ [*mandatory_list] - operator_names].flatten.empty?
66
+ end
67
+
68
+ def missing_mandatory( mandatory_list )
69
+ [ [*mandatory_list] - operator_names].flatten
70
+ end
71
+
72
+ # Create picture of the operators for assignment available on an AR model,
73
+ # including via associations (which provide both << and = )
74
+ # Options:
75
+ # :reload => clear caches and reperform lookup
76
+ # :instance_methods => if true include instance method type assignment operators as well as model's pure columns
77
+ #
78
+ def self.find_operators(klass, options = {} )
79
+
80
+ # Find the has_many associations which can be populated via <<
81
+ if( options[:reload] || @@has_many[klass].nil? )
82
+ @@has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
83
+ klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(@@has_many[klass]) { |x,i| x << i.name.to_s }
84
+ end
85
+ # puts "DEBUG: Has Many Associations:", @@has_many[klass].inspect
86
+
87
+ # Find the belongs_to associations which can be populated via Model.belongs_to_name = OtherArModelObject
88
+ if( options[:reload] || @@belongs_to[klass].nil? )
89
+ @@belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
90
+ end
91
+
92
+ #puts "Belongs To Associations:", @@belongs_to[klass].inspect
93
+
94
+ # Find the has_one associations which can be populated via Model.has_one_name = OtherArModelObject
95
+ if( options[:reload] || self.has_one[klass].nil? )
96
+ self.has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
97
+ end
98
+
99
+ #puts "has_one Associations:", self.has_one[klass].inspect
100
+
101
+ # Find the model's column associations which can be populated via xxxxxx= value
102
+ # Note, not all reflections return method names in same style so we convert all to
103
+ # the raw form i.e without the '=' for consistency
104
+ if( options[:reload] || @@assignments[klass].nil? )
105
+
106
+ @@assignments[klass] = klass.column_names
107
+
108
+ if(options[:instance_methods] == true)
109
+ setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
110
+
111
+ if(klass.respond_to? :defined_activerecord_methods)
112
+ setters = setters - klass.defined_activerecord_methods.to_a
113
+ end
114
+
115
+ # get into same format as other names
116
+ @@assignments[klass] += setters.map{|i| i.gsub(/=/, '')}
117
+ end
118
+
119
+ @@assignments[klass] -= @@has_many[klass] if(@@has_many[klass])
120
+ @@assignments[klass] -= @@belongs_to[klass] if(@@belongs_to[klass])
121
+ @@assignments[klass] -= self.has_one[klass] if(self.has_one[klass])
122
+
123
+ @@assignments[klass].uniq!
124
+
125
+ @@assignments[klass].each do |assign|
126
+ @@column_types[klass] ||= {}
127
+ column_def = klass.columns.find{ |col| col.name == assign }
128
+ @@column_types[klass].merge!( assign => column_def) if column_def
129
+ end
130
+ end
131
+ end
132
+
133
+ def self.build_method_details( klass )
134
+ @method_details ||= {}
135
+
136
+ @method_details[klass] = []
137
+
138
+ assignments_for(klass).each do |n|
139
+ @method_details[klass] << MethodDetail.new(n, klass, n, :assignment)
140
+ end
141
+
142
+ has_one_for(klass).each do |n|
143
+ @method_details[klass] << MethodDetail.new(n, klass, n, :has_one)
144
+ end
145
+
146
+ has_many_for(klass).each do |n|
147
+ @method_details[klass] << MethodDetail.new(n, klass, n, :has_many)
148
+ end
149
+
150
+ belongs_to_for(klass).each do |n|
151
+ @method_details[klass] << MethodDetail.new(n, klass, n, :belongs_to)
152
+ end
153
+ end
154
+
155
+ def self.method_details
156
+ @method_details ||= {}
157
+ @method_details
158
+ end
159
+
160
+ # Find the proper format of name, appropriate call + column type for a given name.
161
+ # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
162
+ #
163
+ # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
164
+ #
165
+ def self.find_method_detail( klass, external_name )
166
+ operator = nil
167
+
168
+ name = external_name.to_s
169
+
170
+ # TODO - check out regexp to do this work better plus Inflections ??
171
+ # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
172
+ [
173
+ name,
174
+ name.tableize,
175
+ name.gsub(' ', '_'),
176
+ name.gsub(' ', '_').downcase,
177
+ name.gsub(/(\s+)/, '_').downcase,
178
+ name.gsub(' ', ''),
179
+ name.gsub(' ', '').downcase,
180
+ name.gsub(' ', '_').underscore].each do |n|
181
+
182
+ operator = (assignments_for(klass).include?(n)) ? n : nil
183
+
184
+ return MethodDetail.new(name, klass, operator, :assignment, @@column_types[klass]) if(operator)
185
+
186
+ operator = (has_one_for(klass).include?(n)) ? n : nil
187
+
188
+ return MethodDetail.new(name, klass, operator, :has_one, @@column_types[klass]) if(operator)
189
+
190
+ operator = (has_many_for(klass).include?(n)) ? n : nil
191
+
192
+ return MethodDetail.new(name, klass, operator, :has_many, @@column_types[klass]) if(operator)
193
+
194
+ operator = (belongs_to_for(klass).include?(n)) ? n : nil
195
+
196
+ return MethodDetail.new(name, klass, operator, :belongs_to, @@column_types[klass]) if(operator)
197
+
198
+ end
199
+
200
+ nil
201
+ end
202
+
203
+ def self.clear
204
+ @@belongs_to.clear
205
+ @@has_many.clear
206
+ @@assignments.clear
207
+ @@column_types.clear
208
+ self.has_one.clear
209
+ end
210
+
211
+ def self.column_key(klass, column)
212
+ "#{klass.name}:#{column}"
213
+ end
214
+
215
+ # TODO - remove use of class variables - not good Ruby design
216
+ def self.belongs_to
217
+ @@belongs_to
218
+ end
219
+
220
+ def self.has_many
221
+ @@has_many
222
+ end
223
+
224
+ def self.has_one
225
+ @has_one ||= {}
226
+ @has_one
227
+ end
228
+
229
+ def self.assignments
230
+ @@assignments
231
+ end
232
+ def self.column_types
233
+ @@column_types
234
+ end
235
+
236
+
237
+ def self.belongs_to_for(klass)
238
+ @@belongs_to[klass] || []
239
+ end
240
+ def self.has_many_for(klass)
241
+ @@has_many[klass] || []
242
+ end
243
+
244
+ def self.has_one_for(klass)
245
+ self.has_one[klass] || []
246
+ end
247
+
248
+ def self.assignments_for(klass)
249
+ @@assignments[klass] || []
250
+ end
251
+ def self.column_type_for(klass, column)
252
+ @@column_types[klass] ? @@column_types[klass][column] : []
253
+ end
254
+
255
+ end
256
+
257
257
  end