datashift 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/README.markdown CHANGED
@@ -32,7 +32,7 @@ Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comm
32
32
 
33
33
  Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
34
34
 
35
- `gem install datashift`
35
+ gem install datashift
36
36
 
37
37
  To use :
38
38
 
@@ -53,8 +53,8 @@ To use the Thor command line applications :
53
53
 
54
54
  Create a high level .thor file - e.g mysite.thor - in your applications root directory
55
55
 
56
- Pull in the thor commands :
57
- ```
56
+ Edit the file and add the following to pull in the thor commands :
57
+ ```ruby
58
58
  require 'thor'
59
59
  require 'datashift'
60
60
 
@@ -68,7 +68,7 @@ and/or
68
68
 
69
69
  bundle exc thor list datashift
70
70
 
71
- To get usgae information use thor help <command>, for example
71
+ To get usage information use thor help <command>, for example
72
72
 
73
73
  bundle exec thor help datashift:generate:excel
74
74
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.1
1
+ 0.7.0
Binary file
Binary file
data/datashift.gemspec CHANGED
@@ -5,11 +5,11 @@
5
5
 
6
6
  Gem::Specification.new do |s|
7
7
  s.name = "datashift"
8
- s.version = "0.6.1"
8
+ s.version = "0.7.0"
9
9
 
10
10
  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
11
11
  s.authors = ["Thomas Statter"]
12
- s.date = "2012-05-24"
12
+ s.date = "2012-05-27"
13
13
  s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
14
14
  s.email = "rubygems@autotelik.co.uk"
15
15
  s.extra_rdoc_files = [
@@ -24,6 +24,8 @@ Gem::Specification.new do |s|
24
24
  "README.rdoc",
25
25
  "Rakefile",
26
26
  "VERSION",
27
+ "datashift-0.6.0.gem",
28
+ "datashift-0.6.1.gem",
27
29
  "datashift.gemspec",
28
30
  "lib/applications/jruby/jexcel_file.rb",
29
31
  "lib/applications/jruby/word.rb",
@@ -32,10 +32,8 @@ module DataShift
32
32
  # For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
33
33
  map_headers_to_operators( @parsed_file.shift, options)
34
34
 
35
- #if(options[:verbose])
36
35
  puts "\n\n\nLoading from CSV file: #{file_name}"
37
36
  puts "Processing #{@parsed_file.size} rows"
38
- # end
39
37
 
40
38
  load_object_class.transaction do
41
39
  @loaded_objects = []
@@ -66,7 +64,7 @@ module DataShift
66
64
  # TODO - handle when it's not valid ?
67
65
  # Process rest and dump out an exception list of Products ??
68
66
 
69
- logger.info "Saving csv row #{row} to table object : #{load_object.inspect}" #if options[:verbose]
67
+ logger.info "Saving csv row #{row} to table object : #{load_object.inspect}"
70
68
 
71
69
  save
72
70
 
@@ -28,7 +28,7 @@ module DataShift
28
28
 
29
29
  attr_accessor :loaded_objects, :failed_objects
30
30
 
31
- attr_accessor :options
31
+ attr_accessor :options, :verbose
32
32
 
33
33
  # Support multiple associations being added to a base object to be specified in a single column.
34
34
  #
@@ -100,7 +100,7 @@ module DataShift
100
100
 
101
101
  @method_mapper = DataShift::MethodMapper.new
102
102
  @options = options.clone
103
- @verbose_logging = @options[:verbose_logging]
103
+ @verbose = @options[:verbose]
104
104
  @headers = []
105
105
 
106
106
  @default_data_objects ||= {}
@@ -214,21 +214,50 @@ module DataShift
214
214
  end
215
215
  end
216
216
 
217
-
218
- # Default values can be provided in YAML config file
217
+ def get_record_by(klazz, field, value)
218
+ x = (@options[:sku_prefix]) ? "#{@options[:sku_prefix]}#{value}" : value
219
+
220
+ begin
221
+ if(@options[:case_sensitive])
222
+ puts "Search case sensitive for [#{x}] on #{field}" if(@verbose)
223
+ return klazz.send("find_by_#{field}", x)
224
+ elsif(@options[:use_like])
225
+ puts "Term : #{klazz}.where(\"#{field} LIKE '#{x}%'\")" if(@verbose)
226
+ return klazz.where("#{field} like ?", "#{x}%").first
227
+ else
228
+ puts "Term : #{klazz}.where(\"lower(#{field}) = '#{x.downcase}'\")" if(@verbose)
229
+ return klazz.where("lower(#{field}) = ?", x.downcase).first
230
+ end
231
+ rescue => e
232
+ logger.error("Exception attempting to find a record for [#{x}] on #{klazz}.#{field}")
233
+ logger.error e.backtrace
234
+ logger.error e.inspect
235
+ return nil
236
+ end
237
+ end
238
+
239
+ # Default values and over rides can be provided in YAML config file.
240
+ #
241
+ # Any Configuration under key 'LoaderBase' is merged into this classes
242
+ # existing options - taking precedence.
243
+ #
219
244
  # Format :
220
245
  #
221
246
  # LoaderClass:
222
247
  # option: value
223
248
  #
224
249
  # Load Class: (e.g Spree:Product)
225
- # atttribute: value
226
-
250
+ # datashift_defaults:
251
+ # value_as_string: "Default Project Value"
252
+ # category: reference:category_002
253
+ #
254
+ # datashift_overrides:
255
+ # value_as_double: 99.23546
256
+ #
227
257
  def configure_from( yaml_file )
228
258
 
229
259
  data = YAML::load( File.open(yaml_file) )
230
260
 
231
-
232
261
  # TODO - MOVE DEFAULTS TO OWN MODULE
233
262
  # decorate the loading class with the defaults/ove rides to manage itself
234
263
  # IDEAS .....
@@ -389,7 +418,7 @@ module DataShift
389
418
  end
390
419
 
391
420
  def save
392
- #puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" #if(options[:verbose])
421
+ puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" if(@verbose)
393
422
  begin
394
423
  result = @load_object.save
395
424
 
@@ -39,8 +39,7 @@ module DataShift
39
39
 
40
40
  @excel = Spreadsheet.open file_name
41
41
 
42
- #if(options[:verbose])
43
- puts "\n\n\nLoading from Excel file: #{file_name}"
42
+ puts "\nLoading from Excel file: #{file_name}"
44
43
 
45
44
  sheet_number = options[:sheet_number] || 0
46
45
 
@@ -60,46 +60,46 @@ module DataShift
60
60
  end
61
61
 
62
62
  module SpreeHelper
63
-
63
+
64
+ # TODO - extract this out of SpreeHelper to create a general paperclip loader
64
65
  class ImageLoader < LoaderBase
65
66
 
66
67
  include DataShift::ImageLoading
67
68
  include DataShift::CsvLoading
68
69
  include DataShift::ExcelLoading
69
70
 
70
- def initialize(image = nil)
71
- super( SpreeHelper::get_spree_class('Image'), image )
71
+ def initialize(image = nil, options = {})
72
+ super( SpreeHelper::get_spree_class('Image'), image, options )
73
+
74
+ if(SpreeHelper::version.to_f > 1.0 )
75
+ @attachment_klazz = DataShift::SpreeHelper::get_spree_class('Variant' )
76
+ else
77
+ @attachment_klazz = DataShift::SpreeHelper::get_spree_class('Product' )
78
+ end
72
79
 
80
+ puts "Attachment Class is #{@attachment_klazz}" if(@verbose)
81
+
73
82
  raise "Failed to create Image for loading" unless @load_object
74
83
  end
75
-
76
- def sku_klazz
77
- @sku_klazz ||= SpreeHelper::get_spree_class('Variant' )
78
- @sku_klazz
79
- end
80
84
 
81
85
  def process()
82
86
 
83
87
  if(current_value && @current_method_detail.operator?('attachment') )
88
+
89
+ # assign the image file data as an attachment
84
90
  @load_object.attachment = get_file(current_value)
85
91
 
86
- puts "Image attachment set : #{@load_object.inspect}"
92
+ puts "Image attachment created : #{@load_object.inspect}"
87
93
 
88
- elsif(current_value && @current_method_detail.operator?('sku') )
94
+ elsif(current_value && @current_method_detail.operator )
89
95
 
90
- return if(current_value.empty?)
91
- puts "Looking for record with SKU #{current_value}"
92
- add_record( sku_klazz.find_by_sku(current_value) )
93
-
94
- elsif(current_value && @current_method_detail.operator?('name') )
95
-
96
- puts "Looking for record with NAME [#{current_value}]"
97
- add_record attachment_klazz.find_by_name(current_value)
98
-
96
+ # find the db record to assign our Image to
97
+ add_record( get_record_by(@attachment_klazz, @current_method_detail.operator, current_value) )
98
+
99
99
  end
100
100
 
101
101
  end
102
-
102
+
103
103
  def add_record(record)
104
104
  if(record)
105
105
  if(SpreeHelper::version.to_f > 1 )
@@ -111,6 +111,7 @@ module DataShift
111
111
  puts "Image viewable set : #{record.inspect}"
112
112
 
113
113
  else
114
+ puts "WARNING - Cannot set viewable - No matching record supplied"
114
115
  logger.error"Failed to find a matching record"
115
116
  end
116
117
  end
@@ -55,7 +55,28 @@ module Datashift
55
55
 
56
56
  loader.perform_load(input, :mandatory => ['sku', 'name', 'price'] )
57
57
  end
58
+
59
+
60
+ desc "attach_images", "Populate Products with images from Excel/CSV\nProvide column SKU or Name\nProvide column 'attachment' containing full path to image"
61
+ # :dummy => dummy run without actual saving to DB
62
+ method_option :input, :aliases => '-i', :required => true, :desc => "The 2 column import file (.xls or .csv)"
58
63
 
64
+ def attach_images()
65
+
66
+ require File.expand_path('config/environment.rb')
67
+
68
+ require 'image_loader'
69
+
70
+ image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
71
+
72
+ # force inclusion means add to operator list even if not present
73
+ options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
74
+
75
+ loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
76
+
77
+ loader.perform_load( options[:input], options )
78
+ end
79
+
59
80
 
60
81
  #
61
82
  # => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
@@ -63,33 +84,37 @@ module Datashift
63
84
  #
64
85
  # => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
65
86
  #
66
- desc "images", "Populate the DB with images.\nDefault location db/image_seeds, or specify :input=<path> or dir under db/image_seeds with :folder"
87
+ desc "images", "Populate the DB with images from a directory where image names map to Product Sku/Name"
67
88
 
68
89
  # :dummy => dummy run without actual saving to DB
69
90
  method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
70
91
 
71
92
  method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
72
93
 
73
-
74
94
  method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
75
95
  method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
76
96
  method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
77
97
  method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
78
98
  method_option :config, :aliases => '-c', :type => :string, :desc => "Configuration file for Image Loader in YAML"
79
-
80
-
99
+ method_option :split_file_name_on, :type => :string, :desc => "delimiter to progressivley split filename for Prod lookup", :default => '_'
100
+ method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find Product"
101
+ method_option :use_like, :type => :boolean, :desc => "Use LIKE 'string%' instead of = 'string' in where clauses"
102
+
81
103
  def images()
82
104
 
83
105
  require File.expand_path('config/environment.rb')
84
106
 
85
107
  require 'image_loader'
108
+
109
+ @verbose = options[:verbose]
86
110
 
87
111
  puts "Using Product Name for lookup" unless(options[:sku])
88
112
  puts "Using SKU for lookup" if(options[:sku])
89
-
113
+
114
+ image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
115
+
90
116
  attachment_klazz = DataShift::SpreeHelper::get_spree_class('Product' )
91
117
  attachment_field = 'name'
92
- image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
93
118
 
94
119
  if(options[:sku])
95
120
  attachment_klazz = DataShift::SpreeHelper::get_spree_class('Variant' )
@@ -103,7 +128,7 @@ module Datashift
103
128
  # raise "Could not find contant for model #{args[:model]}"
104
129
  #end
105
130
 
106
- image_loader = DataShift::SpreeHelper::ImageLoader.new
131
+ image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options.dup)
107
132
 
108
133
  @loader_config = {}
109
134
 
@@ -121,84 +146,85 @@ module Datashift
121
146
 
122
147
  @image_cache = options[:input]
123
148
 
124
- if(File.directory? @image_cache )
125
- logger.info "Loading Spree images from #{@image_cache}"
149
+ unless(File.directory? @image_cache )
150
+ puts "ERROR: Supplied Path #{@image_cache} not accesible"
151
+ exit(-1)
152
+ end
153
+
154
+ logger.info "Loading Spree images from #{@image_cache}"
126
155
 
127
- missing_records = []
128
- Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
156
+ missing_records = []
157
+
158
+ # unless record # try splitting up filename in various ways looking for the SKU
159
+ split_on = @loader_config[:split_file_name_on] || options[:split_file_name_on]
160
+
161
+ Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
129
162
 
130
- base_name = File.basename(image_name, '.*')
131
- base_name.strip!
163
+ base_name = File.basename(image_name, '.*')
164
+ base_name.strip!
132
165
 
133
- logger.info "Processing image #{base_name} : #{File.exists?(image_name)}"
134
-
135
- record = nil
166
+ logger.info "Processing image file #{base_name} : #{File.exists?(image_name)}"
136
167
 
137
- put "Processing image [#{base_name}]"
168
+ record = nil
138
169
 
139
- # unless record # try splitting up filename in various ways looking for the SKU
140
- split_on = @loader_config[:split_file_name_on] || '_'
141
-
142
- base_name.split(split_on).each do |x|
143
- record = get_record_by(attachment_klazz, attachment_field, x)
144
- break if record
145
- end
170
+ puts "Search for product for image file [#{base_name}]" if(@verbose)
146
171
 
147
- record = record.product if(record) # SKU stored on Variant but we want it's master Product
172
+ record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
173
+
174
+ # try seperate portions of the filename, front -> back
175
+ base_name.split(split_on).each do |x|
176
+ record = image_loader.get_record_by(attachment_klazz, attachment_field, x)
177
+ break if record
178
+ end unless(record)
179
+
180
+ # this time try sequentially scanning
181
+ base_name.split(split_on).inject("") do |str, x|
182
+ record = image_loader.get_record_by(attachment_klazz, attachment_field, "#{str}#{x}")
183
+ break if record
184
+ x
185
+ end unless(record)
186
+
187
+ record = record.product if(record && record.respond_to?(:product)) # SKU stored on Variant but we want it's master Product
148
188
 
149
- if(record)
150
- logger.info "Found record for attachment : #{record.inspect}"
151
- exists = record.images.detect {|i| puts "COMPARE #{i.attachment_file_name} => #{image_name}"; i.attachment_file_name == image_name }
189
+ if(record)
190
+ logger.info "Found record for attachment : #{record.inspect}"
191
+
192
+ if(options[:skip_if_loaded])
193
+ exists = record.images.detect {|i| i.attachment_file_name == image_name }
152
194
 
153
- if(options[:skip_if_loaded] && !exists.nil?)
154
- logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
155
- next
156
- end
157
- else
158
- missing_records << image_name
195
+ logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
196
+ next if(exists)
159
197
  end
198
+ else
199
+ missing_records << image_name
200
+ end
160
201
 
161
- next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
162
-
163
- # Check if Image must have an associated record
164
- if(record || (record.nil? && options[:process_when_no_assoc]))
165
- image_loader.reset()
166
- puts "Processing Image #{image_name}"
167
- image_loader.create_image(image_klazz, image_name, record)
168
- end
169
-
202
+ next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
203
+
204
+ # Check if Image must have an associated record
205
+ if(record || (record.nil? && options[:process_when_no_assoc]))
206
+ image_loader.reset()
207
+ puts "Adding Image #{image_name} to Product #{record.name}" if(@verbose)
208
+ logger.info("Adding Image #{image_name} to Product #{record.name}")
209
+ image_loader.create_image(image_klazz, image_name, record)
170
210
  end
171
211
 
172
- unless missing_records.empty?
173
- FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
212
+ end
213
+
214
+ unless missing_records.empty?
215
+ FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
174
216
 
175
- puts '\nMISSING Records Report>>'
176
- missing_records.each do |i|
177
- puts "Copy #{i} to MissingRecords folder"
178
- FileUtils.cp( i, 'MissingRecords') unless(options[:dummy] == 'true')
179
- end
217
+ puts 'MISSING Records Report>>'
218
+ missing_records.each do |i|
219
+ puts "Copy #{i} to MissingRecords folder"
220
+ FileUtils.cp( i, 'MissingRecords') unless(options[:dummy] == 'true')
180
221
  end
181
-
182
- puts "Dummy Run - if happy run without -d" if(options[:dummy])
183
- else
184
- puts "ERROR: Supplied Path #{@image_cache} not accesible"
185
- exit(-1)
186
222
  end
187
- end
188
-
189
- private
190
-
191
- def get_record_by(klazz, field, value)
192
- x = (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{value}" : value
193
223
 
194
- if(@loader_config['case_sensitive'])
195
- puts "Search case sensitive for [#{x}] on #{field}"
196
- return klazz.find(:first, :conditions => [ "? = ?", field, x ])
197
- else
198
- puts "Search for [#{x}] on #{field}"
199
- return klazz.find(:first, :conditions => [ "lower(?) = ?", field, x.downcase ])
200
- end
224
+ puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
225
+
201
226
  end
227
+
202
228
  end
203
229
 
204
230
  end
@@ -101,9 +101,10 @@ describe 'SpreeImageLoading' do
101
101
 
102
102
  @Image_klass.all.size.should == 0
103
103
 
104
- options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
104
+ # force inclusion means add to operator list even if not present
105
+ options = { :verbose => true, :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
105
106
 
106
- loader = DataShift::SpreeHelper::ImageLoader.new
107
+ loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
107
108
 
108
109
  loader.perform_load( SpecHelper::spree_fixture('SpreeImages.xls'), options )
109
110
 
metadata CHANGED
@@ -2,7 +2,7 @@
2
2
  name: datashift
3
3
  version: !ruby/object:Gem::Version
4
4
  prerelease:
5
- version: 0.6.1
5
+ version: 0.7.0
6
6
  platform: ruby
7
7
  authors:
8
8
  - Thomas Statter
@@ -10,7 +10,7 @@ autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
12
 
13
- date: 2012-05-24 00:00:00 Z
13
+ date: 2012-05-27 00:00:00 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: spreadsheet
@@ -40,6 +40,8 @@ files:
40
40
  - README.rdoc
41
41
  - Rakefile
42
42
  - VERSION
43
+ - datashift-0.6.0.gem
44
+ - datashift-0.6.1.gem
43
45
  - datashift.gemspec
44
46
  - lib/applications/jruby/jexcel_file.rb
45
47
  - lib/applications/jruby/word.rb