datashift 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.markdown +4 -4
- data/VERSION +1 -1
- data/datashift-0.6.0.gem +0 -0
- data/datashift-0.6.1.gem +0 -0
- data/datashift.gemspec +4 -2
- data/lib/loaders/csv_loader.rb +1 -3
- data/lib/loaders/loader_base.rb +37 -8
- data/lib/loaders/spreadsheet_loader.rb +1 -2
- data/lib/loaders/spree/image_loader.rb +21 -20
- data/lib/thor/spree/products_images.thor +94 -68
- data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
- data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
- data/spec/spree_images_loader_spec.rb +3 -2
- metadata +4 -2
data/README.markdown
CHANGED
@@ -32,7 +32,7 @@ Many example Spreadsheets/CSV files in spec/fixtures, fully documented with comm
|
|
32
32
|
|
33
33
|
Add gem 'datashift' to your Gemfile/bundle, or install the latest gem as usual :
|
34
34
|
|
35
|
-
|
35
|
+
gem install datashift
|
36
36
|
|
37
37
|
To use :
|
38
38
|
|
@@ -53,8 +53,8 @@ To use the Thor command line applications :
|
|
53
53
|
|
54
54
|
Create a high level .thor file - e.g mysite.thor - in your applications root directory
|
55
55
|
|
56
|
-
|
57
|
-
```
|
56
|
+
Edit the file and add the following to pull in the thor commands :
|
57
|
+
```ruby
|
58
58
|
require 'thor'
|
59
59
|
require 'datashift'
|
60
60
|
|
@@ -68,7 +68,7 @@ and/or
|
|
68
68
|
|
69
69
|
bundle exc thor list datashift
|
70
70
|
|
71
|
-
To get
|
71
|
+
To get usage information use thor help <command>, for example
|
72
72
|
|
73
73
|
bundle exec thor help datashift:generate:excel
|
74
74
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.7.0
|
data/datashift-0.6.0.gem
ADDED
Binary file
|
data/datashift-0.6.1.gem
ADDED
Binary file
|
data/datashift.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "datashift"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.7.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Thomas Statter"]
|
12
|
-
s.date = "2012-05-
|
12
|
+
s.date = "2012-05-27"
|
13
13
|
s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
|
14
14
|
s.email = "rubygems@autotelik.co.uk"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -24,6 +24,8 @@ Gem::Specification.new do |s|
|
|
24
24
|
"README.rdoc",
|
25
25
|
"Rakefile",
|
26
26
|
"VERSION",
|
27
|
+
"datashift-0.6.0.gem",
|
28
|
+
"datashift-0.6.1.gem",
|
27
29
|
"datashift.gemspec",
|
28
30
|
"lib/applications/jruby/jexcel_file.rb",
|
29
31
|
"lib/applications/jruby/word.rb",
|
data/lib/loaders/csv_loader.rb
CHANGED
@@ -32,10 +32,8 @@ module DataShift
|
|
32
32
|
# For example if model has an attribute 'price' will map columns called Price, price, PRICE etc to this attribute
|
33
33
|
map_headers_to_operators( @parsed_file.shift, options)
|
34
34
|
|
35
|
-
#if(options[:verbose])
|
36
35
|
puts "\n\n\nLoading from CSV file: #{file_name}"
|
37
36
|
puts "Processing #{@parsed_file.size} rows"
|
38
|
-
# end
|
39
37
|
|
40
38
|
load_object_class.transaction do
|
41
39
|
@loaded_objects = []
|
@@ -66,7 +64,7 @@ module DataShift
|
|
66
64
|
# TODO - handle when it's not valid ?
|
67
65
|
# Process rest and dump out an exception list of Products ??
|
68
66
|
|
69
|
-
logger.info "Saving csv row #{row} to table object : #{load_object.inspect}"
|
67
|
+
logger.info "Saving csv row #{row} to table object : #{load_object.inspect}"
|
70
68
|
|
71
69
|
save
|
72
70
|
|
data/lib/loaders/loader_base.rb
CHANGED
@@ -28,7 +28,7 @@ module DataShift
|
|
28
28
|
|
29
29
|
attr_accessor :loaded_objects, :failed_objects
|
30
30
|
|
31
|
-
attr_accessor :options
|
31
|
+
attr_accessor :options, :verbose
|
32
32
|
|
33
33
|
# Support multiple associations being added to a base object to be specified in a single column.
|
34
34
|
#
|
@@ -100,7 +100,7 @@ module DataShift
|
|
100
100
|
|
101
101
|
@method_mapper = DataShift::MethodMapper.new
|
102
102
|
@options = options.clone
|
103
|
-
@
|
103
|
+
@verbose = @options[:verbose]
|
104
104
|
@headers = []
|
105
105
|
|
106
106
|
@default_data_objects ||= {}
|
@@ -214,21 +214,50 @@ module DataShift
|
|
214
214
|
end
|
215
215
|
end
|
216
216
|
|
217
|
-
|
218
|
-
|
217
|
+
def get_record_by(klazz, field, value)
|
218
|
+
x = (@options[:sku_prefix]) ? "#{@options[:sku_prefix]}#{value}" : value
|
219
|
+
|
220
|
+
begin
|
221
|
+
if(@options[:case_sensitive])
|
222
|
+
puts "Search case sensitive for [#{x}] on #{field}" if(@verbose)
|
223
|
+
return klazz.send("find_by_#{field}", x)
|
224
|
+
elsif(@options[:use_like])
|
225
|
+
puts "Term : #{klazz}.where(\"#{field} LIKE '#{x}%'\")" if(@verbose)
|
226
|
+
return klazz.where("#{field} like ?", "#{x}%").first
|
227
|
+
else
|
228
|
+
puts "Term : #{klazz}.where(\"lower(#{field}) = '#{x.downcase}'\")" if(@verbose)
|
229
|
+
return klazz.where("lower(#{field}) = ?", x.downcase).first
|
230
|
+
end
|
231
|
+
rescue => e
|
232
|
+
logger.error("Exception attempting to find a record for [#{x}] on #{klazz}.#{field}")
|
233
|
+
logger.error e.backtrace
|
234
|
+
logger.error e.inspect
|
235
|
+
return nil
|
236
|
+
end
|
237
|
+
end
|
238
|
+
|
239
|
+
# Default values and over rides can be provided in YAML config file.
|
240
|
+
#
|
241
|
+
# Any Configuration under key 'LoaderBase' is merged into this classes
|
242
|
+
# existing options - taking precedence.
|
243
|
+
#
|
219
244
|
# Format :
|
220
245
|
#
|
221
246
|
# LoaderClass:
|
222
247
|
# option: value
|
223
248
|
#
|
224
249
|
# Load Class: (e.g Spree:Product)
|
225
|
-
#
|
226
|
-
|
250
|
+
# datashift_defaults:
|
251
|
+
# value_as_string: "Default Project Value"
|
252
|
+
# category: reference:category_002
|
253
|
+
#
|
254
|
+
# datashift_overrides:
|
255
|
+
# value_as_double: 99.23546
|
256
|
+
#
|
227
257
|
def configure_from( yaml_file )
|
228
258
|
|
229
259
|
data = YAML::load( File.open(yaml_file) )
|
230
260
|
|
231
|
-
|
232
261
|
# TODO - MOVE DEFAULTS TO OWN MODULE
|
233
262
|
# decorate the loading class with the defaults/ove rides to manage itself
|
234
263
|
# IDEAS .....
|
@@ -389,7 +418,7 @@ module DataShift
|
|
389
418
|
end
|
390
419
|
|
391
420
|
def save
|
392
|
-
|
421
|
+
puts "DEBUG: SAVING #{load_object.class} : #{load_object.inspect}" if(@verbose)
|
393
422
|
begin
|
394
423
|
result = @load_object.save
|
395
424
|
|
@@ -60,46 +60,46 @@ module DataShift
|
|
60
60
|
end
|
61
61
|
|
62
62
|
module SpreeHelper
|
63
|
-
|
63
|
+
|
64
|
+
# TODO - extract this out of SpreeHelper to create a general paperclip loader
|
64
65
|
class ImageLoader < LoaderBase
|
65
66
|
|
66
67
|
include DataShift::ImageLoading
|
67
68
|
include DataShift::CsvLoading
|
68
69
|
include DataShift::ExcelLoading
|
69
70
|
|
70
|
-
def initialize(image = nil)
|
71
|
-
super( SpreeHelper::get_spree_class('Image'), image )
|
71
|
+
def initialize(image = nil, options = {})
|
72
|
+
super( SpreeHelper::get_spree_class('Image'), image, options )
|
73
|
+
|
74
|
+
if(SpreeHelper::version.to_f > 1.0 )
|
75
|
+
@attachment_klazz = DataShift::SpreeHelper::get_spree_class('Variant' )
|
76
|
+
else
|
77
|
+
@attachment_klazz = DataShift::SpreeHelper::get_spree_class('Product' )
|
78
|
+
end
|
72
79
|
|
80
|
+
puts "Attachment Class is #{@attachment_klazz}" if(@verbose)
|
81
|
+
|
73
82
|
raise "Failed to create Image for loading" unless @load_object
|
74
83
|
end
|
75
|
-
|
76
|
-
def sku_klazz
|
77
|
-
@sku_klazz ||= SpreeHelper::get_spree_class('Variant' )
|
78
|
-
@sku_klazz
|
79
|
-
end
|
80
84
|
|
81
85
|
def process()
|
82
86
|
|
83
87
|
if(current_value && @current_method_detail.operator?('attachment') )
|
88
|
+
|
89
|
+
# assign the image file data as an attachment
|
84
90
|
@load_object.attachment = get_file(current_value)
|
85
91
|
|
86
|
-
puts "Image attachment
|
92
|
+
puts "Image attachment created : #{@load_object.inspect}"
|
87
93
|
|
88
|
-
elsif(current_value && @current_method_detail.operator
|
94
|
+
elsif(current_value && @current_method_detail.operator )
|
89
95
|
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
elsif(current_value && @current_method_detail.operator?('name') )
|
95
|
-
|
96
|
-
puts "Looking for record with NAME [#{current_value}]"
|
97
|
-
add_record attachment_klazz.find_by_name(current_value)
|
98
|
-
|
96
|
+
# find the db record to assign our Image to
|
97
|
+
add_record( get_record_by(@attachment_klazz, @current_method_detail.operator, current_value) )
|
98
|
+
|
99
99
|
end
|
100
100
|
|
101
101
|
end
|
102
|
-
|
102
|
+
|
103
103
|
def add_record(record)
|
104
104
|
if(record)
|
105
105
|
if(SpreeHelper::version.to_f > 1 )
|
@@ -111,6 +111,7 @@ module DataShift
|
|
111
111
|
puts "Image viewable set : #{record.inspect}"
|
112
112
|
|
113
113
|
else
|
114
|
+
puts "WARNING - Cannot set viewable - No matching record supplied"
|
114
115
|
logger.error"Failed to find a matching record"
|
115
116
|
end
|
116
117
|
end
|
@@ -55,7 +55,28 @@ module Datashift
|
|
55
55
|
|
56
56
|
loader.perform_load(input, :mandatory => ['sku', 'name', 'price'] )
|
57
57
|
end
|
58
|
+
|
59
|
+
|
60
|
+
desc "attach_images", "Populate Products with images from Excel/CSV\nProvide column SKU or Name\nProvide column 'attachment' containing full path to image"
|
61
|
+
# :dummy => dummy run without actual saving to DB
|
62
|
+
method_option :input, :aliases => '-i', :required => true, :desc => "The 2 column import file (.xls or .csv)"
|
58
63
|
|
64
|
+
def attach_images()
|
65
|
+
|
66
|
+
require File.expand_path('config/environment.rb')
|
67
|
+
|
68
|
+
require 'image_loader'
|
69
|
+
|
70
|
+
image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
|
71
|
+
|
72
|
+
# force inclusion means add to operator list even if not present
|
73
|
+
options = { :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
|
74
|
+
|
75
|
+
loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
|
76
|
+
|
77
|
+
loader.perform_load( options[:input], options )
|
78
|
+
end
|
79
|
+
|
59
80
|
|
60
81
|
#
|
61
82
|
# => rake datashift:spree:images input=vendor/extensions/site/fixtures/images
|
@@ -63,33 +84,37 @@ module Datashift
|
|
63
84
|
#
|
64
85
|
# => rake datashift:spree:images input=C:\images\taxon_icons skip_if_no_assoc=true klass=Taxon
|
65
86
|
#
|
66
|
-
desc "images", "Populate the DB with images
|
87
|
+
desc "images", "Populate the DB with images from a directory where image names map to Product Sku/Name"
|
67
88
|
|
68
89
|
# :dummy => dummy run without actual saving to DB
|
69
90
|
method_option :input, :aliases => '-i', :required => true, :desc => "The import file (.xls or .csv)"
|
70
91
|
|
71
92
|
method_option :process_when_no_assoc, :aliases => '-f', :type => :boolean, :desc => "Process image even if no Product found - force loading"
|
72
93
|
|
73
|
-
|
74
94
|
method_option :sku, :aliases => '-s', :desc => "Lookup Product based on image name starting with sku"
|
75
95
|
method_option :sku_prefix, :aliases => '-p', :desc => "Prefix to add to each SKU in import file"
|
76
96
|
method_option :dummy, :aliases => '-d', :type => :boolean, :desc => "Dummy run, do not actually save Image or Product"
|
77
97
|
method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
|
78
98
|
method_option :config, :aliases => '-c', :type => :string, :desc => "Configuration file for Image Loader in YAML"
|
79
|
-
|
80
|
-
|
99
|
+
method_option :split_file_name_on, :type => :string, :desc => "delimiter to progressivley split filename for Prod lookup", :default => '_'
|
100
|
+
method_option :case_sensitive, :type => :boolean, :desc => "Use case sensitive where clause to find Product"
|
101
|
+
method_option :use_like, :type => :boolean, :desc => "Use LIKE 'string%' instead of = 'string' in where clauses"
|
102
|
+
|
81
103
|
def images()
|
82
104
|
|
83
105
|
require File.expand_path('config/environment.rb')
|
84
106
|
|
85
107
|
require 'image_loader'
|
108
|
+
|
109
|
+
@verbose = options[:verbose]
|
86
110
|
|
87
111
|
puts "Using Product Name for lookup" unless(options[:sku])
|
88
112
|
puts "Using SKU for lookup" if(options[:sku])
|
89
|
-
|
113
|
+
|
114
|
+
image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
|
115
|
+
|
90
116
|
attachment_klazz = DataShift::SpreeHelper::get_spree_class('Product' )
|
91
117
|
attachment_field = 'name'
|
92
|
-
image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
|
93
118
|
|
94
119
|
if(options[:sku])
|
95
120
|
attachment_klazz = DataShift::SpreeHelper::get_spree_class('Variant' )
|
@@ -103,7 +128,7 @@ module Datashift
|
|
103
128
|
# raise "Could not find contant for model #{args[:model]}"
|
104
129
|
#end
|
105
130
|
|
106
|
-
image_loader = DataShift::SpreeHelper::ImageLoader.new
|
131
|
+
image_loader = DataShift::SpreeHelper::ImageLoader.new(nil, options.dup)
|
107
132
|
|
108
133
|
@loader_config = {}
|
109
134
|
|
@@ -121,84 +146,85 @@ module Datashift
|
|
121
146
|
|
122
147
|
@image_cache = options[:input]
|
123
148
|
|
124
|
-
|
125
|
-
|
149
|
+
unless(File.directory? @image_cache )
|
150
|
+
puts "ERROR: Supplied Path #{@image_cache} not accesible"
|
151
|
+
exit(-1)
|
152
|
+
end
|
153
|
+
|
154
|
+
logger.info "Loading Spree images from #{@image_cache}"
|
126
155
|
|
127
|
-
|
128
|
-
|
156
|
+
missing_records = []
|
157
|
+
|
158
|
+
# unless record # try splitting up filename in various ways looking for the SKU
|
159
|
+
split_on = @loader_config[:split_file_name_on] || options[:split_file_name_on]
|
160
|
+
|
161
|
+
Dir.glob("#{@image_cache}/**/*.{jpg,png,gif}") do |image_name|
|
129
162
|
|
130
|
-
|
131
|
-
|
163
|
+
base_name = File.basename(image_name, '.*')
|
164
|
+
base_name.strip!
|
132
165
|
|
133
|
-
|
134
|
-
|
135
|
-
record = nil
|
166
|
+
logger.info "Processing image file #{base_name} : #{File.exists?(image_name)}"
|
136
167
|
|
137
|
-
|
168
|
+
record = nil
|
138
169
|
|
139
|
-
|
140
|
-
split_on = @loader_config[:split_file_name_on] || '_'
|
141
|
-
|
142
|
-
base_name.split(split_on).each do |x|
|
143
|
-
record = get_record_by(attachment_klazz, attachment_field, x)
|
144
|
-
break if record
|
145
|
-
end
|
170
|
+
puts "Search for product for image file [#{base_name}]" if(@verbose)
|
146
171
|
|
147
|
-
|
172
|
+
record = image_loader.get_record_by(attachment_klazz, attachment_field, base_name)
|
173
|
+
|
174
|
+
# try seperate portions of the filename, front -> back
|
175
|
+
base_name.split(split_on).each do |x|
|
176
|
+
record = image_loader.get_record_by(attachment_klazz, attachment_field, x)
|
177
|
+
break if record
|
178
|
+
end unless(record)
|
179
|
+
|
180
|
+
# this time try sequentially scanning
|
181
|
+
base_name.split(split_on).inject("") do |str, x|
|
182
|
+
record = image_loader.get_record_by(attachment_klazz, attachment_field, "#{str}#{x}")
|
183
|
+
break if record
|
184
|
+
x
|
185
|
+
end unless(record)
|
186
|
+
|
187
|
+
record = record.product if(record && record.respond_to?(:product)) # SKU stored on Variant but we want it's master Product
|
148
188
|
|
149
|
-
|
150
|
-
|
151
|
-
|
189
|
+
if(record)
|
190
|
+
logger.info "Found record for attachment : #{record.inspect}"
|
191
|
+
|
192
|
+
if(options[:skip_if_loaded])
|
193
|
+
exists = record.images.detect {|i| i.attachment_file_name == image_name }
|
152
194
|
|
153
|
-
|
154
|
-
|
155
|
-
next
|
156
|
-
end
|
157
|
-
else
|
158
|
-
missing_records << image_name
|
195
|
+
logger.info "Skipping - Image #{image_name} already loaded for #{attachment_klazz}"
|
196
|
+
next if(exists)
|
159
197
|
end
|
198
|
+
else
|
199
|
+
missing_records << image_name
|
200
|
+
end
|
160
201
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
202
|
+
next if(options[:dummy]) # Don't actually create/upload to DB if we are doing dummy run
|
203
|
+
|
204
|
+
# Check if Image must have an associated record
|
205
|
+
if(record || (record.nil? && options[:process_when_no_assoc]))
|
206
|
+
image_loader.reset()
|
207
|
+
puts "Adding Image #{image_name} to Product #{record.name}" if(@verbose)
|
208
|
+
logger.info("Adding Image #{image_name} to Product #{record.name}")
|
209
|
+
image_loader.create_image(image_klazz, image_name, record)
|
170
210
|
end
|
171
211
|
|
172
|
-
|
173
|
-
|
212
|
+
end
|
213
|
+
|
214
|
+
unless missing_records.empty?
|
215
|
+
FileUtils.mkdir_p('MissingRecords') unless File.directory?('MissingRecords')
|
174
216
|
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
end
|
217
|
+
puts 'MISSING Records Report>>'
|
218
|
+
missing_records.each do |i|
|
219
|
+
puts "Copy #{i} to MissingRecords folder"
|
220
|
+
FileUtils.cp( i, 'MissingRecords') unless(options[:dummy] == 'true')
|
180
221
|
end
|
181
|
-
|
182
|
-
puts "Dummy Run - if happy run without -d" if(options[:dummy])
|
183
|
-
else
|
184
|
-
puts "ERROR: Supplied Path #{@image_cache} not accesible"
|
185
|
-
exit(-1)
|
186
222
|
end
|
187
|
-
end
|
188
|
-
|
189
|
-
private
|
190
|
-
|
191
|
-
def get_record_by(klazz, field, value)
|
192
|
-
x = (options[:sku_prefix]) ? "#{options[:sku_prefix]}#{value}" : value
|
193
223
|
|
194
|
-
if(
|
195
|
-
|
196
|
-
return klazz.find(:first, :conditions => [ "? = ?", field, x ])
|
197
|
-
else
|
198
|
-
puts "Search for [#{x}] on #{field}"
|
199
|
-
return klazz.find(:first, :conditions => [ "lower(?) = ?", field, x.downcase ])
|
200
|
-
end
|
224
|
+
puts "Dummy Run Complete- if happy run without -d" if(options[:dummy])
|
225
|
+
|
201
226
|
end
|
227
|
+
|
202
228
|
end
|
203
229
|
|
204
230
|
end
|
Binary file
|
Binary file
|
@@ -101,9 +101,10 @@ describe 'SpreeImageLoading' do
|
|
101
101
|
|
102
102
|
@Image_klass.all.size.should == 0
|
103
103
|
|
104
|
-
|
104
|
+
# force inclusion means add to operator list even if not present
|
105
|
+
options = { :verbose => true, :force_inclusion => ['sku', 'attachment'] } if(SpreeHelper::version.to_f > 1 )
|
105
106
|
|
106
|
-
loader = DataShift::SpreeHelper::ImageLoader.new
|
107
|
+
loader = DataShift::SpreeHelper::ImageLoader.new(nil, options)
|
107
108
|
|
108
109
|
loader.perform_load( SpecHelper::spree_fixture('SpreeImages.xls'), options )
|
109
110
|
|
metadata
CHANGED
@@ -2,7 +2,7 @@
|
|
2
2
|
name: datashift
|
3
3
|
version: !ruby/object:Gem::Version
|
4
4
|
prerelease:
|
5
|
-
version: 0.
|
5
|
+
version: 0.7.0
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
8
8
|
- Thomas Statter
|
@@ -10,7 +10,7 @@ autorequire:
|
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
12
|
|
13
|
-
date: 2012-05-
|
13
|
+
date: 2012-05-27 00:00:00 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: spreadsheet
|
@@ -40,6 +40,8 @@ files:
|
|
40
40
|
- README.rdoc
|
41
41
|
- Rakefile
|
42
42
|
- VERSION
|
43
|
+
- datashift-0.6.0.gem
|
44
|
+
- datashift-0.6.1.gem
|
43
45
|
- datashift.gemspec
|
44
46
|
- lib/applications/jruby/jexcel_file.rb
|
45
47
|
- lib/applications/jruby/word.rb
|