datashift 0.5.0 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (35) hide show
  1. data/Rakefile +3 -0
  2. data/VERSION +1 -1
  3. data/datashift.gemspec +6 -4
  4. data/lib/datashift/method_detail.rb +2 -2
  5. data/lib/datashift/method_dictionary.rb +15 -2
  6. data/lib/datashift/method_mapper.rb +18 -5
  7. data/lib/exporters/excel_exporter.rb +1 -2
  8. data/lib/helpers/spree_helper.rb +18 -12
  9. data/lib/loaders/csv_loader.rb +1 -8
  10. data/lib/loaders/excel_loader.rb +7 -7
  11. data/lib/loaders/loader_base.rb +34 -6
  12. data/lib/loaders/spreadsheet_loader.rb +86 -78
  13. data/lib/loaders/spree/image_loader.rb +72 -24
  14. data/lib/loaders/spree/product_loader.rb +3 -31
  15. data/lib/thor/import_excel.thor +15 -16
  16. data/lib/thor/spree/products_images.thor +5 -3
  17. data/sandbox/config/application.rb +21 -5
  18. data/sandbox/config/database.yml +18 -32
  19. data/sandbox/config/environment.rb +1 -3
  20. data/sandbox/config/environments/development.rb +9 -2
  21. data/spec/Gemfile +16 -5
  22. data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
  23. data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
  24. data/spec/fixtures/spree/{SpreeProductImages.xls → SpreeImages.xls} +0 -0
  25. data/spec/fixtures/spree/SpreeProductsWithImages.csv +3 -3
  26. data/spec/fixtures/spree/SpreeProductsWithImages.xls +0 -0
  27. data/spec/fixtures/test_model_defs.rb +1 -0
  28. data/spec/spec_helper.rb +13 -11
  29. data/spec/spree_exporter_spec.rb +6 -4
  30. data/spec/spree_generator_spec.rb +14 -10
  31. data/spec/spree_images_loader_spec.rb +59 -24
  32. data/spec/spree_loader_spec.rb +12 -10
  33. data/spec/spree_method_mapping_spec.rb +16 -9
  34. data/tasks/spree/image_load.rake +2 -8
  35. metadata +15 -6
@@ -12,35 +12,48 @@ module DataShift
12
12
 
13
13
  include DataShift::Logging
14
14
 
15
+ def get_file( attachment_path )
16
+
17
+ unless File.exists?(attachment_path) && File.readable?(attachment_path)
18
+ logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{attachment_path}")
19
+ raise "Cannot process Image : Invalid Path #{attachment_path}"
20
+ end
21
+
22
+ file = begin
23
+ File.new(attachment_path, "rb")
24
+ rescue => e
25
+ puts e.inspect
26
+ raise "ERROR : Failed to read image #{attachment_path}"
27
+ end
28
+
29
+ file
30
+ end
31
+
15
32
  # Note the Spree Image model sets default storage path to
16
33
  # => :path => ":rails_root/public/assets/products/:id/:style/:basename.:extension"
17
34
 
18
- def create_image(klass, image_path, viewable_record = nil, options = {})
35
+ def create_image(klass, attachment_path, viewable_record = nil, options = {})
19
36
 
20
- unless File.exists?(image_path) && File.readable?(image_path)
21
- logger.error("Cannot process Image from #{Dir.pwd}: Invalid Path #{image_path}")
22
- raise "Cannot process Image : Invalid Path #{image_path}"
23
- end
24
-
25
37
  alt = if(options[:alt])
26
38
  options[:alt]
27
39
  else
28
40
  (viewable_record and viewable_record.respond_to? :name) ? viewable_record.name : ""
29
41
  end
30
-
31
- file = begin
32
- File.new(image_path, "rb")
33
- rescue => e
34
- puts e.inspect
35
- raise "ERROR : Failed to read image #{image_path}"
36
- end
37
-
42
+
38
43
  position = (viewable_record and viewable_record.respond_to?(:images)) ? viewable_record.images.length : 0
44
+
45
+ file = get_file(attachment_path)
39
46
 
40
- image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)
47
+ if(SpreeHelper::version.to_f > 1 && viewable_record.is_a?(Spree::Product) )
48
+
49
+ image = klass.new( :attachment => file, :alt => alt, :position => position)
50
+
51
+ # mass assignment not allows for this field
52
+ image.viewable = viewable_record.master
53
+ else
54
+ image = klass.new( :attachment => file,:viewable => viewable_record, :alt => alt, :position => position)
55
+ end
41
56
  #image.attachment.reprocess!
42
-
43
- #image.viewable = viewable_record if viewable_record
44
57
 
45
58
  puts image.save ? "Success: Created Image: #{image.inspect}" : "ERROR : Problem saving to DB Image: #{image.inspect}"
46
59
  end
@@ -55,18 +68,53 @@ module DataShift
55
68
  include DataShift::ExcelLoading
56
69
 
57
70
  def initialize(image = nil)
58
- puts SpreeHelper::get_spree_class('Image')
71
+ super( SpreeHelper::get_spree_class('Image'), image )
59
72
 
60
- @@image_klass ||= SpreeHelper::get_spree_class('Image')
61
-
62
- super( @@image_klass, image )
63
73
  raise "Failed to create Image for loading" unless @load_object
64
74
  end
65
75
 
66
- # The path to the physical image on local disk
67
- def process(image_path, record = nil)
68
- @load_object = create_image(@@image_klass, image_path, record)
76
+ def sku_klazz
77
+ @sku_klazz ||= SpreeHelper::get_spree_class('Variant' )
78
+ @sku_klazz
79
+ end
80
+
81
+ def process()
82
+
83
+ if(current_value && @current_method_detail.operator?('attachment') )
84
+ @load_object.attachment = get_file(current_value)
85
+
86
+ puts "Image attachment set : #{@load_object.inspect}"
87
+
88
+ elsif(current_value && @current_method_detail.operator?('sku') )
89
+
90
+ return if(current_value.empty?)
91
+ puts "Looking for record with SKU #{current_value}"
92
+ add_record( sku_klazz.find_by_sku(current_value) )
93
+
94
+ elsif(current_value && @current_method_detail.operator?('name') )
95
+
96
+ puts "Looking for record with NAME [#{current_value}]"
97
+ add_record attachment_klazz.find_by_name(current_value)
98
+
99
+ end
100
+
101
+ end
102
+
103
+ def add_record(record)
104
+ if(record)
105
+ if(SpreeHelper::version.to_f > 1 )
106
+ @load_object.viewable = record
107
+ else
108
+ @load_object.viewable = record.product # SKU stored on Variant but we want it's master Product
109
+ end
110
+ @load_object.save
111
+ puts "Image viewable set : #{record.inspect}"
112
+
113
+ else
114
+ logger.error"Failed to find a matching record"
115
+ end
69
116
  end
70
117
  end
118
+
71
119
  end
72
120
  end
@@ -39,30 +39,7 @@ module DataShift
39
39
  logger.debug "PRODUCT #{@load_object.inspect} MASTER: #{@load_object.master.inspect}"
40
40
  end
41
41
 
42
- # Based on filename call appropriate loading function
43
- # Currently supports :
44
- # Excel/Open Office files saved as .xls
45
- # CSV files
46
- #
47
- # OPTIONS :
48
- # strict : Raise exception if any column cannot be mapped
49
-
50
- def perform_load( file_name, options = {} )
51
42
 
52
- raise DataShift::BadFile, "Cannot load #{file_name} file not found." unless(File.exists?(file_name))
53
-
54
- ext = File.extname(file_name)
55
-
56
- if(ext.casecmp('.xls') == 0)
57
- raise DataShift::BadRuby, "Please install and use JRuby for loading .xls files" unless(Guards::jruby?)
58
- perform_excel_load(file_name, options)
59
- elsif(ext.casecmp('.csv') == 0)
60
- perform_csv_load(file_name, options)
61
- else
62
- raise DataShift::UnsupportedFileType, "#{ext} files not supported - Try .csv or OpenOffice/Excel .xls"
63
- end
64
- end
65
-
66
43
  # Over ride base class process with some Spree::Product specifics
67
44
  #
68
45
  # What process a value string from a column, assigning value(s) to correct association on Product.
@@ -138,12 +115,7 @@ module DataShift
138
115
 
139
116
  private
140
117
 
141
- # Take current column data and split into each association
142
- # Supported Syntax :
143
- # assoc_find_name:value | assoc2_find_name:value | etc
144
- def get_each_assoc
145
- current_value.to_s.split( LoaderBase::multi_assoc_delim )
146
- end
118
+
147
119
 
148
120
  # Special case for OptionTypes as it's two stage process
149
121
  # First add the possible option_types to Product, then we are able
@@ -189,7 +161,7 @@ module DataShift
189
161
  begin
190
162
  # This one line seems to works for 1.1.0 - 3.2 but not 1.0.0 - 3.1 ??
191
163
  if(SpreeHelper::version.to_f >= 1.1)
192
- variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
164
+ variant = @load_object.variants.create( :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price)
193
165
  else
194
166
  variant = @@variant_klass.create( :product => @load_object, :sku => "#{@load_object.sku}_#{i}", :price => @load_object.price, :available_on => @load_object.available_on)
195
167
  #if(variant.valid?)
@@ -203,7 +175,7 @@ module DataShift
203
175
  rescue => e
204
176
  puts "Failed to create a Variant for Product #{@load_object.name}"
205
177
  puts e.inspect
206
- puts e.backtrace
178
+ #puts e.backtrace
207
179
  end
208
180
 
209
181
  logger.debug "Created New Variant: #{variant.inspect}"
@@ -11,15 +11,11 @@
11
11
  #
12
12
  # DataShift::load_commands
13
13
  #
14
- # Requires Jruby, cmd Line:
14
+ # N.B Requires JRuby
15
15
  #
16
- # => bundle exec thor datashift:import:excel -m <active record class> -r <output_template.xls> -a
17
- #
18
- # Cmd Line:
19
- #
20
- # => jruby -S rake datashift:import:excel model=<active record class> input=<file.xls>
21
- # => jruby -S rake datashift:import:excel model=<active record class> input=C:\MyProducts.xlsverbose=true
16
+ # => bundle exec thor datashift:import:excel -m <active record class> -i <output_template.xls> -a
22
17
  #
18
+
23
19
  require 'datashift'
24
20
 
25
21
 
@@ -34,6 +30,7 @@ module Datashift
34
30
  desc "excel", "import .xls file for specifiec active record model"
35
31
  method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
36
32
  method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
33
+ method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
37
34
  method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
38
35
  method_option :exclude, :aliases => '-e', :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
39
36
 
@@ -48,16 +45,17 @@ module Datashift
48
45
  model = options[:model]
49
46
  begin
50
47
  # support modules e.g "Spree::Property")
51
- klass = ModelMapper::class_from_string(model) #Kernel.const_get(model)
48
+ klass = ModelMapper::class_from_string(model)
52
49
  rescue NameError
53
- raise "ERROR: No such AR Model found - check valid model supplied via model=<Class>"
50
+ raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>"
54
51
  end
55
52
 
56
- if(ENV['loader'])
53
+ raise "ERROR: No such AR Model found - check valid model supplied with -m <Class>" if(klass.nil?)
54
+
55
+ if(options[:loader])
57
56
  begin
58
- #loader_klass = Kernel.const_get(ENV['loader'])
59
- # support modules e.g "Spree::Property")
60
- loader_klass = ModelMapper::class_from_string(ENV['loader']) #Kernel.const_get(model)
57
+
58
+ loader_klass = ModelMapper::class_from_string(options[:loader])
61
59
 
62
60
  loader = loader_klass.new(klass)
63
61
 
@@ -71,11 +69,12 @@ module Datashift
71
69
  loader = DataShift::ExcelLoader.new(klass)
72
70
  end
73
71
 
74
- logger.info("ARGS #{options.inspect} [#{options[:verbose]}]")
72
+ logger.info("ARGS #{options.inspect}")
75
73
  loader.logger.verbose if(ENV['verbose'])
76
74
 
77
- loader.configure_from( ENV['config'] ) if(ENV['config'])
78
-
75
+ loader.configure_from( options[:config] ) if(options[:config])
76
+
77
+
79
78
  loader.perform_load(options[:input])
80
79
  end
81
80
  end
@@ -77,7 +77,7 @@ module Datashift
77
77
  method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
78
78
  #method_option :config, :aliases => '-c', :type => :string, :desc => "Configuration file containg defaults or over rides in YAML"
79
79
 
80
- def images()#, [:input, :folder, :dummy, :sku, :skip_if_no_assoc, :skip_if_loaded, :model] => :environment do |t, args|
80
+ def images()
81
81
 
82
82
  require File.expand_path('config/environment.rb')
83
83
 
@@ -91,6 +91,7 @@ module Datashift
91
91
 
92
92
  attachment_klazz = DataShift::SpreeHelper::get_spree_class('Product' )
93
93
  sku_klazz = DataShift::SpreeHelper::get_spree_class('Variant' )
94
+ image_klazz = DataShift::SpreeHelper::get_spree_class('Image' )
94
95
 
95
96
  # TODO generalise for any paperclip project, for now just Spree
96
97
  #begin
@@ -152,8 +153,8 @@ module Datashift
152
153
  # Check if Image must have an associated record
153
154
  if(record || (record.nil? && options[:process_when_no_assoc]))
154
155
  image_loader.reset()
155
- puts "Process Image"
156
- image_loader.process( image_name, record )
156
+ puts "Processing Image #{image_name}"
157
+ image_loader.create_image(image_klazz, image_name, record)
157
158
  end
158
159
 
159
160
  end
@@ -168,6 +169,7 @@ module Datashift
168
169
  end
169
170
  end
170
171
 
172
+ puts "Dummy Run - if happy run without -d" if(options[:dummy])
171
173
  else
172
174
  puts "ERROR: Supplied Path #{@image_cache} not accesible"
173
175
  exit(-1)
@@ -1,11 +1,16 @@
1
+ require File.expand_path('../boot', __FILE__)
2
+
1
3
  require 'rails/all'
2
4
 
3
- module SpreeHelper
4
- class Application < Rails::Application
5
-
6
- config.to_prepare do
7
- end
5
+ if defined?(Bundler)
6
+ # If you precompile assets before deploying to production, use this line
7
+ Bundler.require(*Rails.groups(:assets => %w(development test)))
8
+ # If you want your assets lazily compiled in production, use this line
9
+ # Bundler.require(:default, :assets, Rails.env)
10
+ end
8
11
 
12
+ module Sandbox
13
+ class Application < Rails::Application
9
14
  # Settings in config/environments/* take precedence over those specified here.
10
15
  # Application configuration should go into files in config/initializers
11
16
  # -- all .rb files in that directory are automatically loaded.
@@ -34,6 +39,17 @@ module SpreeHelper
34
39
  # Configure sensitive parameters which will be filtered from the log file.
35
40
  config.filter_parameters += [:password]
36
41
 
42
+ # Use SQL instead of Active Record's schema dumper when creating the database.
43
+ # This is necessary if your schema can't be completely dumped by the schema dumper,
44
+ # like if you have constraints or database-specific column types
45
+ # config.active_record.schema_format = :sql
46
+
47
+ # Enforce whitelist mode for mass assignment.
48
+ # This will create an empty whitelist of attributes available for mass-assignment for all models
49
+ # in your app. As such, your models will need to explicitly whitelist or blacklist accessible
50
+ # parameters by using an attr_accessible or attr_protected declaration.
51
+ config.active_record.whitelist_attributes = true
52
+
37
53
  # Enable the asset pipeline
38
54
  config.assets.enabled = true
39
55
 
@@ -1,34 +1,20 @@
1
- # Used by spec helper to build test databases
2
- # These entires can be used in call to db_connect ( see spec_helper )
3
-
4
- <% adapter = 'sqlite3' %>
5
- <% adapter = 'jdbcsqlite3' if(Guards::jruby? ) %>
6
-
7
- test_mysql:
8
- adapter: jdbcmysql
9
- database: datashift_test_models_db
10
- username: test
11
- password: test
12
- host: localhost
13
-
14
- test_memory:
15
- adapter: <%= adapter %>
16
- database: :memory
1
+ # SQLite version 3.x
2
+ # gem 'activerecord-jdbcsqlite3-adapter'
3
+ #
4
+ # Configure Using Gemfile
5
+ # gem 'activerecord-jdbcsqlite3-adapter'
6
+ #
7
+ development:
8
+ adapter: sqlite3
9
+ database: db/development.sqlite3
17
10
 
18
- test_file:
19
- adapter: <%= adapter %>
20
- database: <%= File.join($DataShiftFixturePath, 'datashift_test_models_db.sqlite') %>
21
- encoding: utf8
22
- host: localhost
11
+ # Warning: The database defined as "test" will be erased and
12
+ # re-generated from your development database when you run "rake".
13
+ # Do not set this db to the same as development or production.
14
+ test:
15
+ adapter: sqlite3
16
+ database: db/test.sqlite3
23
17
 
24
- test_spree_standalone:
25
- adapter: <%= adapter %>
26
- database: <%= File.join($DataShiftFixturePath, 'datashift_Spree_db.sqlite') %>
27
- encoding: utf8
28
- host: localhost
29
-
30
- development:
31
- adapter: <%= adapter %>
32
- database: <%= File.join($DataShiftFixturePath, 'datashift_Spree_db.sqlite') %>
33
- encoding: utf8
34
- host: localhost
18
+ production:
19
+ adapter: sqlite3
20
+ database: db/production.sqlite3
@@ -2,6 +2,4 @@
2
2
  require File.expand_path('../application', __FILE__)
3
3
 
4
4
  # Initialize the rails application
5
- SpreeHelper::Application.initialize!
6
-
7
- ActiveRecord::Base.include_root_in_json = true
5
+ Sandbox::Application.initialize!
@@ -1,8 +1,8 @@
1
- SpreeHelper::Application.configure do
1
+ Sandbox::Application.configure do
2
2
  # Settings specified here will take precedence over those in config/application.rb
3
3
 
4
4
  # In the development environment your application's code is reloaded on
5
- # every request. This slows down response time but is perfect for development
5
+ # every request. This slows down response time but is perfect for development
6
6
  # since you don't have to restart the web server when you make code changes.
7
7
  config.cache_classes = false
8
8
 
@@ -22,6 +22,13 @@ SpreeHelper::Application.configure do
22
22
  # Only use best-standards-support built into browsers
23
23
  config.action_dispatch.best_standards_support = :builtin
24
24
 
25
+ # Raise exception on mass assignment protection for Active Record models
26
+ config.active_record.mass_assignment_sanitizer = :strict
27
+
28
+ # Log the query plan for queries taking more than this (works
29
+ # with SQLite, MySQL, and PostgreSQL)
30
+ config.active_record.auto_explain_threshold_in_seconds = 0.5
31
+
25
32
  # Do not compress assets
26
33
  config.assets.compress = false
27
34
 
data/spec/Gemfile CHANGED
@@ -5,14 +5,25 @@ gem 'rspec-core' # RSpec runner and example groups.
5
5
  gem 'rspec-expectations' # RSpec matchers for should and should_not.
6
6
  gem 'rspec-mocks' # RSpec test double framework with stubbing and mocking.
7
7
  gem 'rspec-rails' # RSpec version 2.x for Rails version 3.x.
8
- gem 'activerecord-jdbcsqlite3-adapter'
9
8
 
9
+ # we need both, for JRuby testing of Excel and non JRuby csv
10
+ platform :jruby do
11
+ gem 'jruby-openssl'
12
+ gem 'activerecord-jdbcsqlite3-adapter'
13
+ end
14
+
15
+ platform :ruby do
16
+ gem 'sqlite3'
17
+ end
18
+
10
19
  # DEFINE WHICH VERSIONS WE WANT TO TEST WITH
11
20
 
12
- #gem 'rails', '3.2.3'
13
- #gem 'spree', '1.1.0'
21
+ gem 'spreadsheet'
22
+
23
+ gem 'rails', '3.2.3'
24
+ gem 'spree', '1.1.1'
14
25
 
15
- gem 'rails', '3.1.3'
16
- gem 'spree', '1.0.0'
26
+ #gem 'rails', '3.1.3'
27
+ #gem 'spree', '1.0.0'
17
28
 
18
29