datashift 0.0.2 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (35) hide show
  1. data/README.markdown +13 -12
  2. data/Rakefile +8 -8
  3. data/VERSION +1 -5
  4. data/datashift.gemspec +12 -38
  5. data/lib/applications/jruby/jexcel_file.rb +23 -11
  6. data/lib/datashift/method_detail.rb +44 -5
  7. data/lib/datashift/method_dictionary.rb +210 -0
  8. data/lib/datashift/method_mapper.rb +25 -191
  9. data/lib/generators/excel_generator.rb +12 -11
  10. data/lib/helpers/spree_helper.rb +36 -12
  11. data/lib/loaders/excel_loader.rb +2 -1
  12. data/lib/loaders/loader_base.rb +37 -20
  13. data/lib/loaders/spree/image_loader.rb +35 -16
  14. data/lib/loaders/spree/product_loader.rb +27 -1
  15. data/spec/csv_loader_spec.rb +3 -3
  16. data/spec/excel_exporter_spec.rb +79 -0
  17. data/spec/excel_generator_spec.rb +3 -3
  18. data/spec/excel_loader_spec.rb +35 -16
  19. data/spec/fixtures/ProjectsMultiCategoriesHeaderLookup.xls +0 -0
  20. data/spec/fixtures/images/DEMO_001_ror_bag.jpeg +0 -0
  21. data/spec/fixtures/images/DEMO_002_Powerstation.jpg +0 -0
  22. data/spec/fixtures/images/DEMO_003_ror_mug.jpeg +0 -0
  23. data/spec/fixtures/images/DEMO_004_ror_ringer.jpeg +0 -0
  24. data/spec/fixtures/interact_models_db.sqlite +0 -0
  25. data/spec/fixtures/interact_spree_db.sqlite +0 -0
  26. data/spec/fixtures/spree/SpreeProductsWithImages.xls +0 -0
  27. data/spec/loader_spec.rb +4 -4
  28. data/spec/method_dictionary_spec.rb +243 -0
  29. data/spec/method_mapper_spec.rb +17 -213
  30. data/spec/spec_helper.rb +1 -0
  31. data/spec/spree_loader_spec.rb +18 -1
  32. data/spec/spree_method_mapping_spec.rb +4 -4
  33. metadata +14 -130
  34. data/Gemfile +0 -28
  35. data/spec/fixtures/.~lock.ProjectsSingleCategories.xls# +0 -1
@@ -16,6 +16,7 @@
16
16
  # This real association can then be used to send spreadsheet row data to the AR object.
17
17
  #
18
18
  require 'method_detail'
19
+ require 'method_dictionary'
19
20
 
20
21
  module DataShift
21
22
 
@@ -24,11 +25,19 @@ module DataShift
24
25
  attr_accessor :header_row, :headers
25
26
  attr_accessor :method_details, :missing_methods
26
27
 
27
- @@has_many = Hash.new
28
- @@belongs_to = Hash.new
29
- @@assignments = Hash.new
30
- @@column_types = Hash.new
28
+
29
+ # As well as just the column name, support embedding find operators for that column
30
+ # in the heading .. i.e Column header => 'BlogPosts:user_id'
31
+ # ... association has many BlogPosts selected via find_by_user_id
32
+ #
33
+ def self.column_delim
34
+ @column_delim ||= ':'
35
+ @column_delim
36
+ end
31
37
 
38
+ def self.set_column_delim(x) @column_delim = x; end
39
+
40
+
32
41
  def initialize
33
42
  @method_details = []
34
43
  @headers = []
@@ -39,14 +48,22 @@ module DataShift
39
48
  # not be exactly as required e.g handles capitalisation, white space, _ etc
40
49
  # Returns: Array of matching method_details
41
50
  #
42
- def populate_methods( klass, method_list )
51
+ def map_inbound_to_methods( klass, method_list )
52
+
43
53
  @method_details, @missing_methods = [], []
44
54
 
45
- method_list.each do |x|
46
- md = MethodMapper::find_method_detail( klass, x )
55
+ method_list.each do |name|
56
+ x, lookup = name.split(MethodMapper::column_delim)
57
+ md = MethodDictionary::find_method_detail( klass, x )
58
+
59
+ # TODO be nice if we could cheeck that the assoc on klass responds to the specified
60
+ # lookup key now (nice n early)
61
+ # active_record_helper = "find_by_#{lookup}"
62
+
63
+ md.find_by_operator = lookup if(lookup) # TODO and klass.x.respond_to?(active_record_helper))
47
64
  md ? @method_details << md : @missing_methods << x
48
65
  end
49
- #@method_details.compact! .. currently we may neeed to map via the index on @method_details so don't remove nils for now
66
+ #@method_details.compact! .. currently we may need to map via the index on @method_details so don't remove nils for now
50
67
  @method_details
51
68
  end
52
69
 
@@ -69,189 +86,6 @@ module DataShift
69
86
  [ [*mandatory_list] - operator_names].flatten
70
87
  end
71
88
 
72
- # Create picture of the operators for assignment available on an AR model,
73
- # including via associations (which provide both << and = )
74
- # Options:
75
- # :reload => clear caches and reperform lookup
76
- # :instance_methods => if true include instance method type assignment operators as well as model's pure columns
77
- #
78
- def self.find_operators(klass, options = {} )
79
-
80
- # Find the has_many associations which can be populated via <<
81
- if( options[:reload] || @@has_many[klass].nil? )
82
- @@has_many[klass] = klass.reflect_on_all_associations(:has_many).map { |i| i.name.to_s }
83
- klass.reflect_on_all_associations(:has_and_belongs_to_many).inject(@@has_many[klass]) { |x,i| x << i.name.to_s }
84
- end
85
- # puts "DEBUG: Has Many Associations:", @@has_many[klass].inspect
86
-
87
- # Find the belongs_to associations which can be populated via Model.belongs_to_name = OtherArModelObject
88
- if( options[:reload] || @@belongs_to[klass].nil? )
89
- @@belongs_to[klass] = klass.reflect_on_all_associations(:belongs_to).map { |i| i.name.to_s }
90
- end
91
-
92
- #puts "Belongs To Associations:", @@belongs_to[klass].inspect
93
-
94
- # Find the has_one associations which can be populated via Model.has_one_name = OtherArModelObject
95
- if( options[:reload] || self.has_one[klass].nil? )
96
- self.has_one[klass] = klass.reflect_on_all_associations(:has_one).map { |i| i.name.to_s }
97
- end
98
-
99
- #puts "has_one Associations:", self.has_one[klass].inspect
100
-
101
- # Find the model's column associations which can be populated via xxxxxx= value
102
- # Note, not all reflections return method names in same style so we convert all to
103
- # the raw form i.e without the '=' for consistency
104
- if( options[:reload] || @@assignments[klass].nil? )
105
-
106
- @@assignments[klass] = klass.column_names
107
-
108
- if(options[:instance_methods] == true)
109
- setters = klass.instance_methods.grep(/\w+=/).collect {|x| x.to_s }
110
-
111
- if(klass.respond_to? :defined_activerecord_methods)
112
- setters = setters - klass.defined_activerecord_methods.to_a
113
- end
114
-
115
- # get into same format as other names
116
- @@assignments[klass] += setters.map{|i| i.gsub(/=/, '')}
117
- end
118
-
119
- @@assignments[klass] -= @@has_many[klass] if(@@has_many[klass])
120
- @@assignments[klass] -= @@belongs_to[klass] if(@@belongs_to[klass])
121
- @@assignments[klass] -= self.has_one[klass] if(self.has_one[klass])
122
-
123
- @@assignments[klass].uniq!
124
-
125
- @@assignments[klass].each do |assign|
126
- @@column_types[klass] ||= {}
127
- column_def = klass.columns.find{ |col| col.name == assign }
128
- @@column_types[klass].merge!( assign => column_def) if column_def
129
- end
130
- end
131
- end
132
-
133
- def self.build_method_details( klass )
134
- @method_details ||= {}
135
-
136
- @method_details[klass] = []
137
-
138
- assignments_for(klass).each do |n|
139
- @method_details[klass] << MethodDetail.new(n, klass, n, :assignment)
140
- end
141
-
142
- has_one_for(klass).each do |n|
143
- @method_details[klass] << MethodDetail.new(n, klass, n, :has_one)
144
- end
145
-
146
- has_many_for(klass).each do |n|
147
- @method_details[klass] << MethodDetail.new(n, klass, n, :has_many)
148
- end
149
-
150
- belongs_to_for(klass).each do |n|
151
- @method_details[klass] << MethodDetail.new(n, klass, n, :belongs_to)
152
- end
153
- end
154
-
155
- def self.method_details
156
- @method_details ||= {}
157
- @method_details
158
- end
159
-
160
- # Find the proper format of name, appropriate call + column type for a given name.
161
- # e.g Given users entry in spread sheet check for pluralization, missing underscores etc
162
- #
163
- # If not nil, returned method can be used directly in for example klass.new.send( call, .... )
164
- #
165
- def self.find_method_detail( klass, external_name )
166
- operator = nil
167
-
168
- name = external_name.to_s
169
-
170
- # TODO - check out regexp to do this work better plus Inflections ??
171
- # Want to be able to handle any of ["Count On hand", 'count_on_hand', "Count OnHand", "COUNT ONHand" etc]
172
- [
173
- name,
174
- name.tableize,
175
- name.gsub(' ', '_'),
176
- name.gsub(' ', '_').downcase,
177
- name.gsub(/(\s+)/, '_').downcase,
178
- name.gsub(' ', ''),
179
- name.gsub(' ', '').downcase,
180
- name.gsub(' ', '_').underscore].each do |n|
181
-
182
- operator = (assignments_for(klass).include?(n)) ? n : nil
183
-
184
- return MethodDetail.new(name, klass, operator, :assignment, @@column_types[klass]) if(operator)
185
-
186
- operator = (has_one_for(klass).include?(n)) ? n : nil
187
-
188
- return MethodDetail.new(name, klass, operator, :has_one, @@column_types[klass]) if(operator)
189
-
190
- operator = (has_many_for(klass).include?(n)) ? n : nil
191
-
192
- return MethodDetail.new(name, klass, operator, :has_many, @@column_types[klass]) if(operator)
193
-
194
- operator = (belongs_to_for(klass).include?(n)) ? n : nil
195
-
196
- return MethodDetail.new(name, klass, operator, :belongs_to, @@column_types[klass]) if(operator)
197
-
198
- end
199
-
200
- nil
201
- end
202
-
203
- def self.clear
204
- @@belongs_to.clear
205
- @@has_many.clear
206
- @@assignments.clear
207
- @@column_types.clear
208
- self.has_one.clear
209
- end
210
-
211
- def self.column_key(klass, column)
212
- "#{klass.name}:#{column}"
213
- end
214
-
215
- # TODO - remove use of class variables - not good Ruby design
216
- def self.belongs_to
217
- @@belongs_to
218
- end
219
-
220
- def self.has_many
221
- @@has_many
222
- end
223
-
224
- def self.has_one
225
- @has_one ||= {}
226
- @has_one
227
- end
228
-
229
- def self.assignments
230
- @@assignments
231
- end
232
- def self.column_types
233
- @@column_types
234
- end
235
-
236
-
237
- def self.belongs_to_for(klass)
238
- @@belongs_to[klass] || []
239
- end
240
- def self.has_many_for(klass)
241
- @@has_many[klass] || []
242
- end
243
-
244
- def self.has_one_for(klass)
245
- self.has_one[klass] || []
246
- end
247
-
248
- def self.assignments_for(klass)
249
- @@assignments[klass] || []
250
- end
251
- def self.column_type_for(klass, column)
252
- @@column_types[klass] ? @@column_types[klass][column] : []
253
- end
254
-
255
89
  end
256
90
 
257
91
  end
@@ -10,10 +10,11 @@
10
10
  #
11
11
  module DataShift
12
12
 
13
+ require 'generator_base'
14
+
13
15
  if(Guards::jruby?)
14
16
 
15
17
  require 'jruby/jexcel_file'
16
- require 'generator_base'
17
18
 
18
19
  class ExcelGenerator < GeneratorBase
19
20
 
@@ -26,7 +27,7 @@ module DataShift
26
27
  # Create an Excel file template (header row) representing supplied Model
27
28
 
28
29
  def generate(model, options = {})
29
- MethodMapper.find_operators( model )
30
+ MethodDictionary.find_operators( model )
30
31
 
31
32
  @filename = options[:filename] if options[:filename]
32
33
 
@@ -40,7 +41,7 @@ module DataShift
40
41
 
41
42
  raise "Failed to create Excel WorkSheet for #{model.name}" unless excel.sheet
42
43
 
43
- excel.set_headers(MethodMapper.assignments[model])
44
+ excel.set_headers(MethodDictionary.assignments[model])
44
45
 
45
46
  excel.save( @filename )
46
47
  end
@@ -78,20 +79,20 @@ module DataShift
78
79
  excel.create_sheet( items.first.class.name )
79
80
  end
80
81
 
81
- MethodMapper.find_operators( klass )
82
+ MethodDictionary.find_operators( klass )
82
83
 
83
- MethodMapper.build_method_details( klass )
84
+ MethodDictionary.build_method_details( klass )
84
85
 
85
- work_list = (options[:with]) ? options[:with] : [:assignment, :belongs_to, :has_one, :has_many]
86
+ work_list = (options[:with]) ? options[:with] : [:assignments, :belongs_to, :has_one, :has_many]
86
87
 
87
88
  headers = []
88
89
 
89
- MethodMapper.method_details[klass].each do |method_detail|
90
- if(method_detail.operator_type == :assignment)
91
- headers << "#{klass}:#{method_detail.operator}"
92
- end
90
+ work_list.each do |mdtype|
91
+ method_details = MethodDictionary.send("#{mdtype}_for", klass)
92
+
93
+ method_details.each {|md| headers << "#{md.operator}" }
93
94
  end
94
-
95
+
95
96
  excel.set_headers( headers )
96
97
 
97
98
  data = []
@@ -5,6 +5,8 @@
5
5
  #
6
6
  # Details:: Spree Helper mixing in Support for testing or loading Rails Spree e-commerce.
7
7
  #
8
+ # The Spree version you want to test should be picked up from the Gemfile
9
+ #
8
10
  # Since datashift gem is not a Rails app or a Spree App, provides utilities to internally
9
11
  # create a Spree Database, and to load Spree components, enabling standalone testing.
10
12
  #
@@ -41,20 +43,39 @@ module Spree
41
43
  end
42
44
 
43
45
  def self.boot
44
-
46
+
45
47
  require 'spree'
46
48
  require 'spree_core'
47
49
 
48
- $LOAD_PATH << root << lib_root << app_root << File.join(app_root, 'models')
50
+ #require 'rake'
51
+ #require 'rubygems/package_task'
52
+ #require 'thor/group'
53
+ require File.expand_path( lib_root + '/generators/spree/install/install_generator')
54
+ require 'spree/core/testing_support/common_rake'
49
55
 
50
- require 'spree_core/preferences/model_hooks'
51
56
 
52
- # Initialize preference system
53
- ActiveRecord::Base.class_eval do
54
- include Spree::Preferences
55
- include Spree::Preferences::ModelHooks
56
- end
57
+ Spree::SandboxGenerator.start ["--lib_name=spree", "--database=#{ENV['DB_NAME']}"]
58
+ Spree::InstallGenerator.start ["--auto-accept"]
57
59
 
60
+ cmd = "bundle exec rake assets:precompile:nondigest";
61
+ puts cmd; system cmd
62
+
63
+
64
+ return
65
+
66
+
67
+ # TODO how to check gem version actually loaded and do conditional
68
+ #
69
+ #if(PRE 1)
70
+ #require 'spree_core/preferences/model_hooks'
71
+ #
72
+ ## Initialize preference system
73
+ # ActiveRecord::Base.class_eval do
74
+ # include Spree::Preferences
75
+ # include Spree::Preferences::ModelHooks
76
+ # end
77
+ #end
78
+
58
79
  gem 'paperclip'
59
80
  gem 'nested_set'
60
81
 
@@ -71,7 +92,8 @@ module Spree
71
92
 
72
93
  ActiveRecord::Base.send(:include, ActiveMerchant::Billing)
73
94
 
74
- require 'scopes'
95
+
96
+ #require 'scopes'
75
97
 
76
98
  # Not sure how Rails manages this seems lots of circular dependencies so
77
99
  # keep trying stuff till no more errors
@@ -106,14 +128,16 @@ module Spree
106
128
  end
107
129
  end
108
130
 
109
- require 'product'
110
- require 'lib/product_filters'
131
+ #if(PRE 1)require 'product'
132
+ #require 'lib/product_filters'
133
+
134
+
111
135
  load_models( true )
112
136
 
113
137
  end
114
138
 
115
139
  def self.load_models( report_errors = nil )
116
- puts 'load from', root
140
+ puts 'Loading Spree models from', root
117
141
  Dir[root + '/app/models/**/*.rb'].each {|r|
118
142
  begin
119
143
  require r if File.file?(r)
@@ -51,7 +51,7 @@ module DataShift
51
51
  raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" unless(@header_row)
52
52
 
53
53
  @headers = []
54
-
54
+ category_003
55
55
  (0..JExcelFile::MAX_COLUMNS).each do |i|
56
56
  cell = @header_row.getCell(i)
57
57
  break unless cell
@@ -63,6 +63,7 @@ module DataShift
63
63
  raise MissingHeadersError, "No headers found - Check Sheet #{@sheet} is complete and Row #{header_row_index} contains headers" if(@headers.empty?)
64
64
 
65
65
  # Create a method_mapper which maps list of headers into suitable calls on the Active Record class
66
+ # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
66
67
  map_headers_to_operators( @headers, options[:strict] , @mandatory )
67
68
 
68
69
  logger.info "Excel Loader prcoessing #{@excel.num_rows} rows"
@@ -81,18 +81,23 @@ module DataShift
81
81
  @multi_assoc_delim
82
82
  end
83
83
 
84
-
84
+
85
85
  def self.set_multi_assoc_delim(x) @multi_assoc_delim = x; end
86
86
 
87
+
87
88
  # Options
88
89
  # :instance_methods => true
89
90
 
90
91
  def initialize(object_class, object = nil, options = {})
91
92
  @load_object_class = object_class
92
93
 
93
- # Gather list of all possible 'setter' methods on AR class (instance variables and associations)
94
- DataShift::MethodMapper.find_operators( @load_object_class, :reload => true, :instance_methods => options[:instance_methods] )
94
+ # Gather names of all possible 'setter' methods on AR class (instance variables and associations)
95
+ DataShift::MethodDictionary.find_operators( @load_object_class, :reload => true, :instance_methods => options[:instance_methods] )
95
96
 
97
+ # Create dictionary of data on all possible 'setter' methods which can be used to
98
+ # populate or integrate an object of type @load_object_class
99
+ DataShift::MethodDictionary.build_method_details(@load_object_class)
100
+
96
101
  @method_mapper = DataShift::MethodMapper.new
97
102
  @options = options.clone
98
103
  @headers = []
@@ -115,15 +120,18 @@ module DataShift
115
120
  end
116
121
 
117
122
 
118
- # Core API - Given a list of free text column names from a file, map all headers to
119
- # method mapper's operator list.
123
+ # Core API - Given a list of free text column names from a file,
124
+ # map all headers to a method detail containing operator details.
125
+ #
126
+ # This is then available through @method_mapper.method_details.each
127
+ #
120
128
  # Options:
121
129
  # strict : report any header values that can't be mapped as an error
122
130
  #
123
131
  def map_headers_to_operators( headers, strict, mandatory = [])
124
132
  @headers = headers
125
133
 
126
- @method_mapper.populate_methods( load_object_class, @headers )
134
+ method_details = @method_mapper.map_inbound_to_methods( load_object_class, @headers )
127
135
 
128
136
  unless(@method_mapper.missing_methods.empty?)
129
137
  puts "WARNING: Following column headings could not be mapped : #{@method_mapper.missing_methods.inspect}"
@@ -137,12 +145,12 @@ module DataShift
137
145
  end
138
146
 
139
147
 
140
- # Core API - Given a free text column name from a file, search method mapper for
148
+ # Core API - Given a single free text column name from a file, search method mapper for
141
149
  # associated operator on base object class.
142
150
  #
143
151
  # If suitable association found, process row data and then assign to current load_object
144
152
  def find_and_process(column_name, data)
145
- method_detail = MethodMapper.find_method_detail( load_object_class, column_name )
153
+ method_detail = MethodDictionary.find_method_detail( load_object_class, column_name )
146
154
 
147
155
  if(method_detail)
148
156
  prepare_data(method_detail, data)
@@ -242,33 +250,42 @@ module DataShift
242
250
  save_if_new
243
251
 
244
252
  # A single column can contain multiple associations delimited by special char
253
+ # Size:large|Colour:red,green,blue => ['Size:large', 'Colour:red,green,blue']
245
254
  columns = @current_value.to_s.split( LoaderBase::multi_assoc_delim)
246
255
 
247
256
  # Size:large|Colour:red,green,blue => generates find_by_size( 'large' ) and find_all_by_colour( ['red','green','blue'] )
248
257
 
249
- columns.each do |assoc|
250
- operator, values = assoc.split(LoaderBase::name_value_delim)
251
-
252
- lookups = values.split(LoaderBase::multi_value_delim)
258
+ columns.each do |col_str|
259
+
260
+ find_operator, col_values = "",""
261
+
262
+ if(@current_method_detail.find_by_operator)
263
+ find_operator, col_values = @current_method_detail.find_by_operator, col_str
264
+ else
265
+ find_operator, col_values = col_str.split(LoaderBase::name_value_delim)
266
+ raise "No key to find #{@current_method_detail.operator} in DB. Expected format key:value" unless(col_values)
267
+ end
268
+
269
+ find_by_values = col_values.split(LoaderBase::multi_value_delim)
253
270
 
254
- if(lookups.size > 1)
271
+ if(find_by_values.size > 1)
255
272
 
256
- @current_value = @current_method_detail.operator_class.send("find_all_by_#{operator}", lookups )
273
+ @current_value = @current_method_detail.operator_class.send("find_all_by_#{find_operator}", find_by_values )
257
274
 
258
- unless(lookups.size == @current_value.size)
259
- found = @current_value.collect {|f| f.send(operator) }
260
- @load_object.errors.add( method_detail.operator, "Association with key(s) #{(lookups - found).inspect} NOT found")
275
+ unless(find_by_values.size == @current_value.size)
276
+ found = @current_value.collect {|f| f.send(find_operator) }
277
+ @load_object.errors.add( @current_method_detail.operator, "Association with key(s) #{(find_by_values - found).inspect} NOT found")
261
278
  puts "WARNING: Association with key(s) #{(lookups - found).inspect} NOT found - Not added."
262
279
  next if(@current_value.empty?)
263
280
  end
264
281
 
265
282
  else
266
283
 
267
- @current_value = @current_method_detail.operator_class.send("find_by_#{operator}", lookups )
284
+ @current_value = @current_method_detail.operator_class.send("find_by_#{find_operator}", find_by_values )
268
285
 
269
286
  unless(@current_value)
270
- @load_object.errors.add( @current_method_detail.operator, "Association with key #{lookups} NOT found")
271
- puts "WARNING: Association with key #{lookups} NOT found - Not added."
287
+ @load_object.errors.add( @current_method_detail.operator, "Association with key #{find_by_values} NOT found")
288
+ puts "WARNING: Association with key #{find_by_values} NOT found - Not added."
272
289
  next
273
290
  end
274
291