datashift 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (37) hide show
  1. data/README.markdown +12 -0
  2. data/Rakefile +4 -2
  3. data/VERSION +1 -1
  4. data/datashift.gemspec +13 -12
  5. data/lib/datashift/delimiters.rb +87 -0
  6. data/lib/datashift/method_detail.rb +5 -0
  7. data/lib/datashift/method_details_manager.rb +5 -1
  8. data/lib/datashift/method_dictionary.rb +3 -1
  9. data/lib/exporters/csv_exporter.rb +158 -10
  10. data/lib/exporters/excel_exporter.rb +6 -2
  11. data/lib/exporters/exporter_base.rb +6 -0
  12. data/lib/helpers/spree_helper.rb +2 -1
  13. data/lib/loaders/paperclip/image_loader.rb +32 -2
  14. data/lib/loaders/spree/product_loader.rb +3 -2
  15. data/lib/thor/export.thor +111 -0
  16. data/lib/thor/{import_excel.thor → import.thor} +40 -2
  17. data/lib/thor/spree/bootstrap_cleanup.thor +7 -4
  18. data/lib/thor/spree/products_images.thor +26 -17
  19. data/lib/thor/spree/reports.thor +30 -40
  20. data/lib/thor/tools.thor +63 -0
  21. data/spec/Gemfile +3 -2
  22. data/spec/csv_exporter_spec.rb +101 -0
  23. data/spec/excel_exporter_spec.rb +1 -1
  24. data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
  25. data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
  26. data/spec/fixtures/test_model_defs.rb +5 -0
  27. data/spec/spree_loader_spec.rb +28 -131
  28. data/spec/spree_method_mapping_spec.rb +1 -1
  29. data/spec/spree_variants_loader_spec.rb +189 -0
  30. metadata +193 -167
  31. data/lib/thor/export_excel.thor +0 -74
  32. data/sandbox/app/controllers/application_controller.rb +0 -3
  33. data/sandbox/config/application.rb +0 -59
  34. data/sandbox/config/database.yml +0 -20
  35. data/sandbox/config/environment.rb +0 -5
  36. data/sandbox/config/environments/development.rb +0 -37
  37. data/tasks/import/csv.rake +0 -51
@@ -0,0 +1,63 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2012
2
+ # Author :: Tom Statter
3
+ # Date :: Sept 2012
4
+ # License:: MIT.
5
+ #
6
+ # Usage::
7
+ #
8
+ # To pull Datashift commands into your main application :
9
+ #
10
+ # require 'datashift'
11
+ #
12
+ # DataShift::load_commands
13
+ #
14
+ require 'datashift'
15
+
16
+ # Note, not DataShift, case sensitive, create namespace for command line : datashift
17
+ module Datashift
18
+ class Tools < Thor
19
+
20
+ include DataShift::Logging
21
+
22
+ desc "zip", "Create zip of matching digital files"
23
+
24
+ method_option :path, :aliases => '-p', :required => true, :desc => "The path to the digital files"
25
+ method_option :results, :aliases => '-r', :required => true, :desc => "The path to store resulting zip files"
26
+
27
+ def zip()
28
+
29
+ require 'zip/zip'
30
+ require 'zip/zipfilesystem'
31
+
32
+ ready_to_zip = {}
33
+ Dir[File.join(options[:path], '**', '*.*')].each do |p|
34
+ next if File.directory? p
35
+
36
+ basename = File.basename(p, '.*')
37
+ ready_to_zip[basename] ||= []
38
+ ready_to_zip[basename] << p
39
+ end
40
+
41
+ output = options[:results]
42
+
43
+ FileUtils::mkdir_p(output) unless File.exists?(output)
44
+
45
+ puts "Creating #{ready_to_zip.keys.size} new zips"
46
+ ready_to_zip.each do |basename, paths|
47
+
48
+ z= File.join(output, basename + '.zip')
49
+ puts "zipping to #{z}"
50
+
51
+ Zip::ZipOutputStream.open(z) do |zos|
52
+ paths.each do |file|
53
+ zos.put_next_entry(File.basename(file))
54
+ zos.print IO.read(file)
55
+ end
56
+ end
57
+ end
58
+
59
+ end
60
+ end
61
+
62
+ end
63
+
@@ -20,8 +20,9 @@ end
20
20
 
21
21
  gem 'spreadsheet'
22
22
 
23
- gem 'rails', '3.2.3'
24
- gem 'spree', '1.1.1'
23
+ gem 'rails', '3.2.7'
24
+ gem 'spree', '1.1.3'
25
+ #gem 'spree', '1.1.2'
25
26
 
26
27
  #gem 'rails', '3.1.3'
27
28
  #gem 'spree', '1.0.0'
@@ -0,0 +1,101 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2012
2
+ # Author :: Tom Statter
3
+ # Date :: Sept 2012
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for CSV aspect of export
7
+ #
8
+ require File.dirname(__FILE__) + '/spec_helper'
9
+
10
+ require 'erb'
11
+ require 'csv_exporter'
12
+
13
+ describe 'CSV Loader' do
14
+
15
+ before(:all) do
16
+
17
+ db_connect( 'test_file' ) # , test_memory, test_mysql
18
+
19
+ # load our test model definitions - Project etc
20
+ require File.join($DataShiftFixturePath, 'test_model_defs')
21
+
22
+ # handle migration changes or reset of test DB
23
+ migrate_up
24
+
25
+ results_clear()
26
+ end
27
+
28
+ before(:each) do
29
+ MethodDictionary.clear
30
+ MethodDictionary.find_operators( Project )
31
+
32
+ db_clear() # todo read up about proper transactional fixtures
33
+ end
34
+
35
+ it "should be able to create a new CSV exporter" do
36
+ generator = CsvExporter.new( 'rspec_csv_empty.csv' )
37
+
38
+ generator.should_not be_nil
39
+ end
40
+
41
+ it "should export a model to csv file" do
42
+
43
+ expect = result_file('project_export_spec.csv')
44
+
45
+ exporter = CsvExporter.new( expect )
46
+
47
+ Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
48
+
49
+ exporter.export(Project.all)
50
+
51
+ File.exists?(expect).should be_true
52
+
53
+ puts "Can manually check file @ #{expect}"
54
+
55
+ File.foreach(expect) {}
56
+ count = $.
57
+ count.should == Project.count + 1
58
+ end
59
+
60
+ it "should export a model and result of method calls on it to csv file" do
61
+
62
+ expect = result_file('project__with_methods_export_spec.csv')
63
+
64
+ exporter = CsvExporter.new( expect )
65
+
66
+ Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
67
+ Project.create( :value_as_string => 'Another Value as String', :value_as_boolean => false, :value_as_double => 12)
68
+
69
+ exporter.export(Project.all, {:methods => [:multiply]})
70
+
71
+ File.exists?(expect).should be_true
72
+
73
+ puts "Can manually check file @ #{expect}"
74
+
75
+ File.foreach(expect) {}
76
+ count = $.
77
+ count.should == Project.count + 1
78
+ end
79
+
80
+ it "should export a model and associations to .xls file" do
81
+
82
+ p = Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
83
+
84
+ p.milestones.create( :name => 'milestone_1', :cost => 23.45)
85
+
86
+ expect= result_file('project_plus_assoc_export_spec.csv')
87
+
88
+ gen = CsvExporter.new(expect)
89
+
90
+ gen.export_with_associations(Project, Project.all)
91
+
92
+ File.exists?(expect).should be_true
93
+
94
+ File.foreach(expect) {}
95
+ count = $.
96
+ count.should == Project.count + 1
97
+
98
+ end
99
+
100
+
101
+ end
@@ -49,7 +49,7 @@ if(Guards::jruby?)
49
49
 
50
50
  gen = ExcelExporter.new( expect )
51
51
 
52
- gen.export(Project)
52
+ gen.export(Project.all)
53
53
 
54
54
  File.exists?(expect).should be_true
55
55
 
@@ -18,6 +18,11 @@ class Project < ActiveRecord::Base
18
18
  has_and_belongs_to_many :categories
19
19
 
20
20
  attr_accessible :value_as_string, :value_as_boolean, :value_as_double
21
+
22
+ def multiply
23
+ 10 * value_as_double
24
+ end
25
+
21
26
  end
22
27
 
23
28
  class Owner < ActiveRecord::Base
@@ -92,11 +92,11 @@ describe 'SpreeLoader' do
92
92
 
93
93
  # Loader should perform identically regardless of source, whether csv, .xls etc
94
94
 
95
- it "should load basic Products .xls via Spree loader", :opts => true do
95
+ it "should load basic Products .xls via Spree loader", :fail => true do
96
96
  test_basic_product('SpreeProductsSimple.xls')
97
97
  end
98
98
 
99
- it "should load basic Products from .csv via Spree loader", :csv => true do
99
+ it "should load basic Products from .csv via Spree loader", :fail => true do
100
100
  test_basic_product('SpreeProductsSimple.csv')
101
101
  end
102
102
 
@@ -117,6 +117,9 @@ describe 'SpreeLoader' do
117
117
 
118
118
  p = @Product_klass.first
119
119
 
120
+ puts p.inspect
121
+ puts p.master.inspect
122
+
120
123
  p.sku.should == "SIMPLE_001"
121
124
  p.price.should == 345.78
122
125
  p.name.should == "Simple Product for AR Loader"
@@ -124,8 +127,17 @@ describe 'SpreeLoader' do
124
127
  p.cost_price.should == 320.00
125
128
  p.option_types.should have_exactly(1).items
126
129
  p.option_types.should have_exactly(1).items
127
- p.count_on_hand.should == 12
128
- @Product_klass.last.count_on_hand.should == 23
130
+
131
+ p.has_variants?.should be false
132
+ p.master.count_on_hand.should == 12
133
+
134
+ puts SpreeHelper::version
135
+ puts SpreeHelper::version.to_f
136
+ puts SpreeHelper::version > "1.1.2"
137
+
138
+ SpreeHelper::version < "1.1.3" ? p.count_on_hand.should == 12 : p.count_on_hand.should == 0
139
+
140
+ @Product_klass.last.master.count_on_hand.should == 23
129
141
  end
130
142
 
131
143
 
@@ -176,133 +188,6 @@ describe 'SpreeLoader' do
176
188
  }
177
189
  end
178
190
 
179
- # Operation and results should be identical when loading multiple associations
180
- # if using either single column embedded syntax, or one column per entry.
181
-
182
- it "should load Products and create Variants from single column" do
183
- test_variants_creation('SpreeProducts.xls')
184
- end
185
-
186
-
187
- it "should load Products and create Variants from multiple column #{SpecHelper::spree_fixture('SpreeProductsMultiColumn.xls')}", :fail => true do
188
- test_variants_creation('SpreeProductsMultiColumn.xls')
189
- end
190
-
191
- def test_variants_creation( source )
192
- @Product_klass.count.should == 0
193
- @Variant_klass.count.should == 0
194
-
195
- @product_loader.perform_load( SpecHelper::spree_fixture(source), :mandatory => ['sku', 'name', 'price'] )
196
-
197
- expected_multi_column_variants
198
- end
199
-
200
-
201
- def expected_multi_column_variants
202
-
203
- # 3 MASTER products, 11 VARIANTS
204
- @Product_klass.count.should == 3
205
- @Variant_klass.count.should == 14
206
-
207
- p = @Product_klass.first
208
-
209
- p.sku.should == "DEMO_001"
210
-
211
- p.sku.should == "DEMO_001"
212
- p.price.should == 399.99
213
- p.description.should == "blah blah"
214
- p.cost_price.should == 320.00
215
-
216
- @Product_klass.all.select {|m| m.is_master.should == true }
217
-
218
-
219
- # mime_type:jpeg mime_type:PDF mime_type:PNG
220
-
221
- p.variants.should have_exactly(3).items
222
-
223
- p.option_types.should have_exactly(1).items # mime_type
224
-
225
- p.option_types[0].name.should == "mime_type"
226
- p.option_types[0].presentation.should == "Mime type"
227
-
228
- @Variant_klass.all[1].sku.should == "DEMO_001_1"
229
- @Variant_klass.all[1].price.should == 399.99
230
-
231
- # V1
232
- v1 = p.variants[0]
233
-
234
- v1.sku.should == "DEMO_001_1"
235
- v1.price.should == 399.99
236
- v1.count_on_hand.should == 12
237
-
238
-
239
- v1.option_values.should have_exactly(1).items # mime_type: jpeg
240
- v1.option_values[0].name.should == "jpeg"
241
-
242
-
243
- v2 = p.variants[1]
244
- v2.count_on_hand.should == 6
245
- v2.option_values.should have_exactly(1).items # mime_type: jpeg
246
- v2.option_values[0].name.should == "PDF"
247
-
248
- v2.option_values[0].option_type.should_not be_nil
249
- v2.option_values[0].option_type.position.should == 0
250
-
251
-
252
- v3 = p.variants[2]
253
- v3.count_on_hand.should == 7
254
- v3.option_values.should have_exactly(1).items # mime_type: jpeg
255
- v3.option_values[0].name.should == "PNG"
256
-
257
- @Variant_klass.last.price.should == 50.34
258
- @Variant_klass.last.count_on_hand.should == 18
259
-
260
- @product_loader.failed_objects.size.should == 0
261
- end
262
-
263
- # Composite Variant Syntax is option_type_A_name:value;option_type_B_name:value
264
- # which creates a SINGLE Variant with 2 option types
265
-
266
- it "should create Variants with MULTIPLE option types from single column", :new => true do
267
- @product_loader.perform_load( SpecHelper::spree_fixture('SpreeMultiVariant.csv'), :mandatory => ['sku', 'name', 'price'] )
268
-
269
- # Product 1)
270
- # 1 + 2) mime_type:jpeg,PDF;print_type:colour equivalent to (mime_type:jpeg;print_type:colour|mime_type:PDF;print_type:colour)
271
- # 3) mime_type:PNG
272
- #
273
- # Product 2
274
- # 4) mime_type:jpeg;print_type:black_white
275
- # 5) mime_type:PNG;print_type:black_white
276
- #
277
- # Product 3
278
- # 6 +7) mime_type:jpeg;print_type:colour,sepia;size:large
279
- # 8) mime_type:jpeg;print_type:colour
280
- # 9) mime_type:PNG
281
- # 9 + 10) mime_type:PDF|print_type:black_white
282
-
283
- prod_count = 3
284
- var_count = 10
285
-
286
- # plus 3 MASTER VARIANTS
287
- @Product_klass.count.should == prod_count
288
- @Variant_klass.count.should == prod_count + var_count
289
-
290
- p = @Product_klass.first
291
-
292
- p.variants_including_master.should have_exactly(4).items
293
- p.variants.should have_exactly(3).items
294
-
295
- p.variants.each { |v| v.option_values.each {|o| puts o.inspect } }
296
-
297
- p.option_types.each { |ot| puts ot.inspect }
298
- p.option_types.should have_exactly(2).items # mime_type, print_type
299
-
300
- v1 = p.variants[0]
301
- v1.option_values.should have_exactly(2).items
302
- v1.option_values.collect(&:name).sort.should == ['colour','jpeg']
303
-
304
- end
305
-
306
191
  ##################
307
192
  ### PROPERTIES ###
308
193
  ##################
@@ -435,6 +320,18 @@ describe 'SpreeLoader' do
435
320
 
436
321
  end
437
322
 
323
+ it "should correctly identify and create nested Taxons ", :taxons => true do#
324
+ x=<<-EOS
325
+ lucky_product : A>mysubcat
326
+ bad_luck : B>mysubcat
327
+
328
+ The bad_luck product is created with in B and somehow pushed into A>mysubcat.
329
+ The category B>mysubcat is not created at all...
330
+ EOS
331
+ pending(x)
332
+
333
+ end
334
+
438
335
 
439
336
  # REPEAT THE WHOLE TEST SUITE VIA CSV
440
337
 
@@ -12,7 +12,7 @@
12
12
  require File.dirname(__FILE__) + '/spec_helper'
13
13
 
14
14
  require 'spree_helper'
15
- require 'product_loader'
15
+ #require 'product_loader'
16
16
 
17
17
  include DataShift
18
18
 
@@ -0,0 +1,189 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Summer 2011
4
+ #
5
+ # License:: MIT - Free, OpenSource
6
+ #
7
+ # Details:: Specification for Spree aspect of datashift gem.
8
+ #
9
+ # Provides Loaders and rake tasks specifically tailored for uploading or exporting
10
+ # Spree Products, associations and Images
11
+ #
12
+ require File.dirname(__FILE__) + '/spec_helper'
13
+
14
+ require 'spree_helper'
15
+ require 'product_loader'
16
+
17
+ include DataShift
18
+
19
+ describe 'Spree Variants Loader' do
20
+
21
+ include SpecHelper
22
+ extend SpecHelper
23
+
24
+ before(:all) do
25
+ before_all_spree
26
+ end
27
+
28
+ before(:each) do
29
+
30
+ begin
31
+
32
+ before_each_spree
33
+
34
+ @Product_klass.count.should == 0
35
+ @Taxon_klass.count.should == 0
36
+ @Variant_klass.count.should == 0
37
+
38
+ MethodDictionary.clear
39
+
40
+ # For Spree important to get instance methods too as Product delegates
41
+ # many important attributes to Variant (master)
42
+ MethodDictionary.find_operators( @Product_klass, :instance_methods => true )
43
+
44
+ # want to test both lookup and dynamic creation - this Taxonomy should be found, rest created
45
+ root = @Taxonomy_klass.create( :name => 'Paintings' )
46
+
47
+ t = @Taxon_klass.new( :name => 'Landscape' )
48
+ t.taxonomy = root
49
+ t.save
50
+
51
+ @Taxon_klass.count.should == 2
52
+
53
+ @product_loader = DataShift::SpreeHelper::ProductLoader.new
54
+ rescue => e
55
+ puts e.inspect
56
+ puts e.backtrace
57
+ end
58
+ end
59
+
60
+ # Operation and results should be identical when loading multiple associations
61
+ # if using either single column embedded syntax, or one column per entry.
62
+
63
+ it "should load Products and create Variants from single column" do
64
+ test_variants_creation('SpreeProducts.xls')
65
+ end
66
+
67
+
68
+ it "should load Products and create Variants from multiple column #{SpecHelper::spree_fixture('SpreeProductsMultiColumn.xls')}" do
69
+ test_variants_creation('SpreeProductsMultiColumn.xls')
70
+ end
71
+
72
+ def test_variants_creation( source )
73
+ @Product_klass.count.should == 0
74
+ @Variant_klass.count.should == 0
75
+
76
+ @product_loader.perform_load( SpecHelper::spree_fixture(source), :mandatory => ['sku', 'name', 'price'] )
77
+
78
+ expected_multi_column_variants
79
+ end
80
+
81
+
82
+ def expected_multi_column_variants
83
+
84
+ # 3 MASTER products, 11 VARIANTS
85
+ @Product_klass.count.should == 3
86
+ @Variant_klass.count.should == 14
87
+
88
+ p = @Product_klass.first
89
+
90
+ p.sku.should == "DEMO_001"
91
+
92
+ p.sku.should == "DEMO_001"
93
+ p.price.should == 399.99
94
+ p.description.should == "blah blah"
95
+ p.cost_price.should == 320.00
96
+
97
+ @Product_klass.all.select {|m| m.is_master.should == true }
98
+
99
+
100
+ # mime_type:jpeg mime_type:PDF mime_type:PNG
101
+
102
+ p.variants.should have_exactly(3).items
103
+
104
+ p.option_types.should have_exactly(1).items # mime_type
105
+
106
+ p.option_types[0].name.should == "mime_type"
107
+ p.option_types[0].presentation.should == "Mime type"
108
+
109
+ @Variant_klass.all[1].sku.should == "DEMO_001_1"
110
+ @Variant_klass.all[1].price.should == 399.99
111
+
112
+ # V1
113
+ v1 = p.variants[0]
114
+
115
+ v1.sku.should == "DEMO_001_1"
116
+ v1.price.should == 399.99
117
+ v1.count_on_hand.should == 12
118
+
119
+
120
+ v1.option_values.should have_exactly(1).items # mime_type: jpeg
121
+ v1.option_values[0].name.should == "jpeg"
122
+
123
+
124
+ v2 = p.variants[1]
125
+ v2.count_on_hand.should == 6
126
+ v2.option_values.should have_exactly(1).items # mime_type: jpeg
127
+ v2.option_values[0].name.should == "PDF"
128
+
129
+ v2.option_values[0].option_type.should_not be_nil
130
+ v2.option_values[0].option_type.position.should == 0
131
+
132
+
133
+ v3 = p.variants[2]
134
+ v3.count_on_hand.should == 7
135
+ v3.option_values.should have_exactly(1).items # mime_type: jpeg
136
+ v3.option_values[0].name.should == "PNG"
137
+
138
+ @Variant_klass.last.price.should == 50.34
139
+ @Variant_klass.last.count_on_hand.should == 18
140
+
141
+ @product_loader.failed_objects.size.should == 0
142
+ end
143
+
144
+ # Composite Variant Syntax is option_type_A_name:value;option_type_B_name:value
145
+ # which creates a SINGLE Variant with 2 option types
146
+
147
+ it "should create Variants with MULTIPLE option types from single column", :new => true do
148
+ @product_loader.perform_load( SpecHelper::spree_fixture('SpreeMultiVariant.csv'), :mandatory => ['sku', 'name', 'price'] )
149
+
150
+ # Product 1)
151
+ # 1 + 2) mime_type:jpeg,PDF;print_type:colour equivalent to (mime_type:jpeg;print_type:colour|mime_type:PDF;print_type:colour)
152
+ # 3) mime_type:PNG
153
+ #
154
+ # Product 2
155
+ # 4) mime_type:jpeg;print_type:black_white
156
+ # 5) mime_type:PNG;print_type:black_white
157
+ #
158
+ # Product 3
159
+ # 6 +7) mime_type:jpeg;print_type:colour,sepia;size:large
160
+ # 8) mime_type:jpeg;print_type:colour
161
+ # 9) mime_type:PNG
162
+ # 9 + 10) mime_type:PDF|print_type:black_white
163
+
164
+ prod_count = 3
165
+ var_count = 10
166
+
167
+ # plus 3 MASTER VARIANTS
168
+ @Product_klass.count.should == prod_count
169
+ @Variant_klass.count.should == prod_count + var_count
170
+
171
+ p = @Product_klass.first
172
+
173
+ p.variants_including_master.should have_exactly(4).items
174
+ p.variants.should have_exactly(3).items
175
+
176
+ p.variants.each { |v| v.option_values.each {|o| puts o.inspect } }
177
+
178
+ p.option_types.each { |ot| puts ot.inspect }
179
+ p.option_types.should have_exactly(2).items # mime_type, print_type
180
+
181
+ v1 = p.variants[0]
182
+ v1.option_values.should have_exactly(2).items
183
+ v1.option_values.collect(&:name).sort.should == ['colour','jpeg']
184
+
185
+ end
186
+
187
+
188
+
189
+ end