datashift 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (85) hide show
  1. data/.document +5 -5
  2. data/Gemfile +28 -25
  3. data/LICENSE.txt +26 -26
  4. data/README.markdown +302 -285
  5. data/README.rdoc +19 -19
  6. data/Rakefile +93 -95
  7. data/VERSION +5 -5
  8. data/datashift.gemspec +162 -178
  9. data/lib/applications/jruby/jexcel_file.rb +396 -396
  10. data/lib/applications/jruby/word.rb +79 -79
  11. data/lib/datashift.rb +152 -113
  12. data/lib/datashift/exceptions.rb +11 -11
  13. data/lib/datashift/file_definitions.rb +353 -353
  14. data/lib/datashift/mapping_file_definitions.rb +87 -87
  15. data/lib/datashift/method_detail.rb +236 -236
  16. data/lib/datashift/method_mapper.rb +256 -256
  17. data/lib/generators/csv_generator.rb +36 -36
  18. data/lib/generators/excel_generator.rb +121 -121
  19. data/lib/generators/generator_base.rb +13 -13
  20. data/lib/helpers/core_ext/to_b.rb +24 -24
  21. data/lib/helpers/spree_helper.rb +131 -131
  22. data/lib/java/poi-3.7/LICENSE +507 -507
  23. data/lib/java/poi-3.7/NOTICE +21 -21
  24. data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
  25. data/lib/loaders/csv_loader.rb +98 -98
  26. data/lib/loaders/excel_loader.rb +154 -149
  27. data/lib/loaders/loader_base.rb +403 -331
  28. data/lib/loaders/spreadsheet_loader.rb +136 -136
  29. data/lib/loaders/spree/image_loader.rb +45 -45
  30. data/lib/loaders/spree/product_loader.rb +224 -224
  31. data/spec/csv_loader_spec.rb +30 -30
  32. data/spec/datashift_spec.rb +26 -26
  33. data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
  34. data/spec/excel_generator_spec.rb +78 -78
  35. data/spec/excel_loader_spec.rb +204 -176
  36. data/spec/file_definitions.rb +141 -141
  37. data/spec/fixtures/.~lock.ProjectsSingleCategories.xls# +1 -0
  38. data/spec/fixtures/ProjectsDefaults.yml +29 -0
  39. data/spec/fixtures/config/database.yml +24 -24
  40. data/spec/fixtures/interact_models_db.sqlite +0 -0
  41. data/spec/fixtures/interact_spree_db.sqlite +0 -0
  42. data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
  43. data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
  44. data/spec/fixtures/spree/SpreeProducts.csv +4 -4
  45. data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
  46. data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
  47. data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
  48. data/spec/fixtures/test_model_defs.rb +57 -57
  49. data/spec/loader_spec.rb +120 -120
  50. data/spec/method_mapper_spec.rb +237 -237
  51. data/spec/spec_helper.rb +115 -115
  52. data/spec/spree_generator_spec.rb +64 -64
  53. data/spec/spree_loader_spec.rb +310 -310
  54. data/spec/spree_method_mapping_spec.rb +214 -214
  55. data/tasks/config/seed_fu_product_template.erb +15 -15
  56. data/tasks/config/tidy_config.txt +12 -12
  57. data/tasks/db_tasks.rake +65 -64
  58. data/tasks/excel_generator.rake +78 -78
  59. data/tasks/file_tasks.rake +36 -36
  60. data/tasks/import/csv.rake +49 -49
  61. data/tasks/import/excel.rake +71 -66
  62. data/tasks/spree/image_load.rake +108 -108
  63. data/tasks/spree/product_loader.rake +43 -43
  64. data/tasks/word_to_seedfu.rake +166 -166
  65. data/test/helper.rb +18 -18
  66. data/test/test_interact.rb +7 -7
  67. metadata +7 -38
  68. data/Gemfile.lock +0 -211
  69. data/bin/autospec +0 -16
  70. data/bin/convert_to_should_syntax +0 -16
  71. data/bin/erubis +0 -16
  72. data/bin/htmldiff +0 -16
  73. data/bin/jeweler +0 -16
  74. data/bin/ldiff +0 -16
  75. data/bin/nokogiri +0 -16
  76. data/bin/rackup +0 -16
  77. data/bin/rails +0 -16
  78. data/bin/rake +0 -16
  79. data/bin/rake2thor +0 -16
  80. data/bin/ri +0 -16
  81. data/bin/rspec +0 -16
  82. data/bin/spree +0 -16
  83. data/bin/thor +0 -16
  84. data/bin/tilt +0 -16
  85. data/bin/tt +0 -16
@@ -1,36 +1,36 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2011
4
- # License:: MIT
5
- #
6
- # Details:: Export a model to CSV
7
- #
8
- #
9
- require 'generator_base'
10
-
11
- module DataShift
12
-
13
- class CsvGenerator < GeneratorBase
14
-
15
- attr_accessor :excel, :filename
16
-
17
- def initialize(filename)
18
- @excel = nil
19
- @filename = filename
20
- end
21
-
22
- # Create CSV file representing supplied Model
23
-
24
- def generate(model, options = {})
25
-
26
- @filename = options[:filename] if options[:filename]
27
- end
28
-
29
-
30
- # Create an Csv file representing supplied Model
31
-
32
- def export(items, options = {})
33
- end
34
-
35
- end
36
- end
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2011
4
+ # License:: MIT
5
+ #
6
+ # Details:: Export a model to CSV
7
+ #
8
+ #
9
+ require 'generator_base'
10
+
11
+ module DataShift
12
+
13
+ class CsvGenerator < GeneratorBase
14
+
15
+ attr_accessor :excel, :filename
16
+
17
+ def initialize(filename)
18
+ @excel = nil
19
+ @filename = filename
20
+ end
21
+
22
+ # Create CSV file representing supplied Model
23
+
24
+ def generate(model, options = {})
25
+
26
+ @filename = options[:filename] if options[:filename]
27
+ end
28
+
29
+
30
+ # Create an Csv file representing supplied Model
31
+
32
+ def export(items, options = {})
33
+ end
34
+
35
+ end
36
+ end
@@ -1,122 +1,122 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2011
4
- # License:: MIT
5
- #
6
- # Details:: Export a model to Excel '97(-2007) file format.
7
- #
8
- # TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
9
- #
10
- #
11
- module DataShift
12
-
13
- if(Guards::jruby?)
14
-
15
- require 'jruby/jexcel_file'
16
- require 'generator_base'
17
-
18
- class ExcelGenerator < GeneratorBase
19
-
20
- attr_accessor :filename
21
-
22
- def initialize(filename)
23
- @filename = filename
24
- end
25
-
26
- # Create an Excel file template (header row) representing supplied Model
27
-
28
- def generate(model, options = {})
29
- MethodMapper.find_operators( model )
30
-
31
- @filename = options[:filename] if options[:filename]
32
-
33
- excel = JExcelFile.new()
34
-
35
- if(options[:sheet_name] )
36
- excel.create_sheet( options[:sheet_name] )
37
- else
38
- excel.create_sheet( model.name )
39
- end
40
-
41
- raise "Failed to create Excel WorkSheet for #{model.name}" unless excel.sheet
42
-
43
- excel.set_headers(MethodMapper.assignments[model])
44
-
45
- excel.save( @filename )
46
- end
47
-
48
-
49
- # Create an Excel file from list of ActiveRecord objects
50
- def export(records, options = {})
51
-
52
- excel = JExcelFile.new()
53
-
54
- if(options[:sheet_name] )
55
- excel.create_sheet( options[:sheet_name] )
56
- else
57
- excel.create_sheet( records.first.class.name )
58
- end
59
-
60
- excel.ar_to_headers(records)
61
-
62
- excel.ar_to_xls(records)
63
-
64
- excel.save( filename() )
65
- end
66
-
67
- # Create an Excel file from list of ActiveRecord objects
68
- # Specify which associations to export via :with or :exclude
69
- # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
70
- #
71
- def export_with_associations(klass, items, options = {})
72
-
73
- excel = JExcelFile.new()
74
-
75
- if(options[:sheet_name] )
76
- excel.create_sheet( options[:sheet_name] )
77
- else
78
- excel.create_sheet( items.first.class.name )
79
- end
80
-
81
- MethodMapper.find_operators( klass )
82
-
83
- MethodMapper.build_method_details( klass )
84
-
85
- work_list = (options[:with]) ? options[:with] : [:assignment, :belongs_to, :has_one, :has_many]
86
-
87
- headers = []
88
-
89
- MethodMapper.method_details[klass].each do |method_detail|
90
- if(method_detail.operator_type == :assignment)
91
- headers << "#{klass}:#{method_detail.operator}"
92
- end
93
- end
94
-
95
- excel.set_headers( headers )
96
-
97
- data = []
98
-
99
- items.each do |record|
100
-
101
- MethodMapper.method_details[klass].each do |method_detail|
102
- if(method_detail.operator_type == :assignment)
103
- data << record.send( method_detail.operator )
104
- end
105
- end
106
- end
107
-
108
- excel.set_row(2,1,items)
109
-
110
- excel.save( filename() )
111
- end
112
- end # ExcelGenerator
113
-
114
- else
115
- class ExcelGenerator < GeneratorBase
116
- def initialize
117
- raise DataShift::BadRuby, "Please install and use JRuby for working with .xls files"
118
- end
119
- end
120
- end # jruby
121
-
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2011
4
+ # License:: MIT
5
+ #
6
+ # Details:: Export a model to Excel '97(-2007) file format.
7
+ #
8
+ # TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
9
+ #
10
+ #
11
+ module DataShift
12
+
13
+ if(Guards::jruby?)
14
+
15
+ require 'jruby/jexcel_file'
16
+ require 'generator_base'
17
+
18
+ class ExcelGenerator < GeneratorBase
19
+
20
+ attr_accessor :filename
21
+
22
+ def initialize(filename)
23
+ @filename = filename
24
+ end
25
+
26
+ # Create an Excel file template (header row) representing supplied Model
27
+
28
+ def generate(model, options = {})
29
+ MethodMapper.find_operators( model )
30
+
31
+ @filename = options[:filename] if options[:filename]
32
+
33
+ excel = JExcelFile.new()
34
+
35
+ if(options[:sheet_name] )
36
+ excel.create_sheet( options[:sheet_name] )
37
+ else
38
+ excel.create_sheet( model.name )
39
+ end
40
+
41
+ raise "Failed to create Excel WorkSheet for #{model.name}" unless excel.sheet
42
+
43
+ excel.set_headers(MethodMapper.assignments[model])
44
+
45
+ excel.save( @filename )
46
+ end
47
+
48
+
49
+ # Create an Excel file from list of ActiveRecord objects
50
+ def export(records, options = {})
51
+
52
+ excel = JExcelFile.new()
53
+
54
+ if(options[:sheet_name] )
55
+ excel.create_sheet( options[:sheet_name] )
56
+ else
57
+ excel.create_sheet( records.first.class.name )
58
+ end
59
+
60
+ excel.ar_to_headers(records)
61
+
62
+ excel.ar_to_xls(records)
63
+
64
+ excel.save( filename() )
65
+ end
66
+
67
+ # Create an Excel file from list of ActiveRecord objects
68
+ # Specify which associations to export via :with or :exclude
69
+ # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
70
+ #
71
+ def export_with_associations(klass, items, options = {})
72
+
73
+ excel = JExcelFile.new()
74
+
75
+ if(options[:sheet_name] )
76
+ excel.create_sheet( options[:sheet_name] )
77
+ else
78
+ excel.create_sheet( items.first.class.name )
79
+ end
80
+
81
+ MethodMapper.find_operators( klass )
82
+
83
+ MethodMapper.build_method_details( klass )
84
+
85
+ work_list = (options[:with]) ? options[:with] : [:assignment, :belongs_to, :has_one, :has_many]
86
+
87
+ headers = []
88
+
89
+ MethodMapper.method_details[klass].each do |method_detail|
90
+ if(method_detail.operator_type == :assignment)
91
+ headers << "#{klass}:#{method_detail.operator}"
92
+ end
93
+ end
94
+
95
+ excel.set_headers( headers )
96
+
97
+ data = []
98
+
99
+ items.each do |record|
100
+
101
+ MethodMapper.method_details[klass].each do |method_detail|
102
+ if(method_detail.operator_type == :assignment)
103
+ data << record.send( method_detail.operator )
104
+ end
105
+ end
106
+ end
107
+
108
+ excel.set_row(2,1,items)
109
+
110
+ excel.save( filename() )
111
+ end
112
+ end # ExcelGenerator
113
+
114
+ else
115
+ class ExcelGenerator < GeneratorBase
116
+ def initialize
117
+ raise DataShift::BadRuby, "Please install and use JRuby for working with .xls files"
118
+ end
119
+ end
120
+ end # jruby
121
+
122
122
  end # DataShift
@@ -1,14 +1,14 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2010
4
- # License:: MIT
5
- #
6
- # Details:: Base class for generators, which provide serivrs to describe a Model in an external format
7
- #
8
- module DataShift
9
-
10
- class GeneratorBase
11
-
12
- end
13
-
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2010
4
+ # License:: MIT
5
+ #
6
+ # Details:: Base class for generators, which provide serivrs to describe a Model in an external format
7
+ #
8
+ module DataShift
9
+
10
+ class GeneratorBase
11
+
12
+ end
13
+
14
14
  end
@@ -1,24 +1,24 @@
1
- class Object
2
- def to_b
3
- case self
4
- when true, false then self
5
- when nil then false
6
- else
7
- to_i != 0
8
- end
9
- end
10
- end
11
-
12
- class String
13
- TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i.freeze
14
- FALSE_REGEXP = /^(no|false|off|f|0)$/i.freeze
15
-
16
- def to_b
17
- case self
18
- when TRUE_REGEXP then true
19
- when FALSE_REGEXP then false
20
- else
21
- to_i != 0
22
- end
23
- end
24
- end
1
+ class Object
2
+ def to_b
3
+ case self
4
+ when true, false then self
5
+ when nil then false
6
+ else
7
+ to_i != 0
8
+ end
9
+ end
10
+ end
11
+
12
+ class String
13
+ TRUE_REGEXP = /^(yes|true|on|t|1|\-1)$/i.freeze
14
+ FALSE_REGEXP = /^(no|false|off|f|0)$/i.freeze
15
+
16
+ def to_b
17
+ case self
18
+ when TRUE_REGEXP then true
19
+ when FALSE_REGEXP then false
20
+ else
21
+ to_i != 0
22
+ end
23
+ end
24
+ end
@@ -1,131 +1,131 @@
1
- # Copyright:: (c) Autotelik Media Ltd 2011
2
- # Author :: Tom Statter
3
- # Date :: Aug 2011
4
- # License:: MIT
5
- #
6
- # Details:: Spree Helper mixing in Support for testing or loading Rails Spree e-commerce.
7
- #
8
- # Since datashift gem is not a Rails app or a Spree App, provides utilities to internally
9
- # create a Spree Database, and to load Spree components, enabling standalone testing.
10
- #
11
- # => Has been tested with
12
- #
13
- # # => TODO - Can we move to a Gemfile/bunlder
14
- # require 'rubygems'
15
- # gemfile = File.expand_path("<%= gemfile_path %>", __FILE__)
16
- #
17
- # ENV['BUNDLE_GEMFILE'] = gemfile
18
- # require 'bundler'
19
- # Bundler.setup
20
- #
21
- # => TODO - See if we can improve DB creation/migration ....
22
- # N.B Some or all of Spree Tests may fail very first time run,
23
- # as the database is auto generated
24
- # =>
25
- module Spree
26
-
27
- def self.root
28
- Gem.loaded_specs['spree_core'] ? Gem.loaded_specs['spree_core'].full_gem_path : ""
29
- end
30
-
31
- def self.lib_root
32
- File.join(root, 'lib')
33
- end
34
-
35
- def self.app_root
36
- File.join(root, 'app')
37
- end
38
-
39
- def self.load()
40
- gem 'spree'
41
- end
42
-
43
- def self.boot
44
-
45
- require 'spree'
46
- require 'spree_core'
47
-
48
- $LOAD_PATH << root << lib_root << app_root << File.join(app_root, 'models')
49
-
50
- require 'spree_core/preferences/model_hooks'
51
-
52
- # Initialize preference system
53
- ActiveRecord::Base.class_eval do
54
- include Spree::Preferences
55
- include Spree::Preferences::ModelHooks
56
- end
57
-
58
- gem 'paperclip'
59
- gem 'nested_set'
60
-
61
- require 'nested_set'
62
- require 'paperclip'
63
- require 'acts_as_list'
64
-
65
- CollectiveIdea::Acts::NestedSet::Railtie.extend_active_record
66
- ActiveRecord::Base.send(:include, Paperclip::Glue)
67
-
68
- gem 'activemerchant'
69
- require 'active_merchant'
70
- require 'active_merchant/billing/gateway'
71
-
72
- ActiveRecord::Base.send(:include, ActiveMerchant::Billing)
73
-
74
- require 'scopes'
75
-
76
- # Not sure how Rails manages this seems lots of circular dependencies so
77
- # keep trying stuff till no more errors
78
-
79
- Dir[lib_root + '/*.rb'].each do |r|
80
- begin
81
- require r if File.file?(r)
82
- rescue => e
83
- end
84
- end
85
-
86
- Dir[lib_root + '/**/*.rb'].each do |r|
87
- begin
88
- require r if File.file?(r) && ! r.include?('testing') && ! r.include?('generators')
89
- rescue => e
90
- end
91
- end
92
-
93
- load_models( true )
94
-
95
- Dir[lib_root + '/*.rb'].each do |r|
96
- begin
97
- require r if File.file?(r)
98
- rescue => e
99
- end
100
- end
101
-
102
- Dir[lib_root + '/**/*.rb'].each do |r|
103
- begin
104
- require r if File.file?(r) && ! r.include?('testing') && ! r.include?('generators')
105
- rescue => e
106
- end
107
- end
108
-
109
- require 'product'
110
- require 'lib/product_filters'
111
- load_models( true )
112
-
113
- end
114
-
115
- def self.load_models( report_errors = nil )
116
- puts 'load from', root
117
- Dir[root + '/app/models/**/*.rb'].each {|r|
118
- begin
119
- require r if File.file?(r)
120
- rescue => e
121
- puts("WARNING failed to load #{r}", e.inspect) if(report_errors == true)
122
- end
123
- }
124
- end
125
-
126
- def self.migrate_up
127
- load
128
- ActiveRecord::Migrator.up( File.join(root, 'db/migrate') )
129
- end
130
-
131
- end
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2011
4
+ # License:: MIT
5
+ #
6
+ # Details:: Spree Helper mixing in Support for testing or loading Rails Spree e-commerce.
7
+ #
8
+ # Since datashift gem is not a Rails app or a Spree App, provides utilities to internally
9
+ # create a Spree Database, and to load Spree components, enabling standalone testing.
10
+ #
11
+ # => Has been tested with
12
+ #
13
+ # # => TODO - Can we move to a Gemfile/bunlder
14
+ # require 'rubygems'
15
+ # gemfile = File.expand_path("<%= gemfile_path %>", __FILE__)
16
+ #
17
+ # ENV['BUNDLE_GEMFILE'] = gemfile
18
+ # require 'bundler'
19
+ # Bundler.setup
20
+ #
21
+ # => TODO - See if we can improve DB creation/migration ....
22
+ # N.B Some or all of Spree Tests may fail very first time run,
23
+ # as the database is auto generated
24
+ # =>
25
+ module Spree
26
+
27
+ def self.root
28
+ Gem.loaded_specs['spree_core'] ? Gem.loaded_specs['spree_core'].full_gem_path : ""
29
+ end
30
+
31
+ def self.lib_root
32
+ File.join(root, 'lib')
33
+ end
34
+
35
+ def self.app_root
36
+ File.join(root, 'app')
37
+ end
38
+
39
+ def self.load()
40
+ gem 'spree'
41
+ end
42
+
43
+ def self.boot
44
+
45
+ require 'spree'
46
+ require 'spree_core'
47
+
48
+ $LOAD_PATH << root << lib_root << app_root << File.join(app_root, 'models')
49
+
50
+ require 'spree_core/preferences/model_hooks'
51
+
52
+ # Initialize preference system
53
+ ActiveRecord::Base.class_eval do
54
+ include Spree::Preferences
55
+ include Spree::Preferences::ModelHooks
56
+ end
57
+
58
+ gem 'paperclip'
59
+ gem 'nested_set'
60
+
61
+ require 'nested_set'
62
+ require 'paperclip'
63
+ require 'acts_as_list'
64
+
65
+ CollectiveIdea::Acts::NestedSet::Railtie.extend_active_record
66
+ ActiveRecord::Base.send(:include, Paperclip::Glue)
67
+
68
+ gem 'activemerchant'
69
+ require 'active_merchant'
70
+ require 'active_merchant/billing/gateway'
71
+
72
+ ActiveRecord::Base.send(:include, ActiveMerchant::Billing)
73
+
74
+ require 'scopes'
75
+
76
+ # Not sure how Rails manages this seems lots of circular dependencies so
77
+ # keep trying stuff till no more errors
78
+
79
+ Dir[lib_root + '/*.rb'].each do |r|
80
+ begin
81
+ require r if File.file?(r)
82
+ rescue => e
83
+ end
84
+ end
85
+
86
+ Dir[lib_root + '/**/*.rb'].each do |r|
87
+ begin
88
+ require r if File.file?(r) && ! r.include?('testing') && ! r.include?('generators')
89
+ rescue => e
90
+ end
91
+ end
92
+
93
+ load_models( true )
94
+
95
+ Dir[lib_root + '/*.rb'].each do |r|
96
+ begin
97
+ require r if File.file?(r)
98
+ rescue => e
99
+ end
100
+ end
101
+
102
+ Dir[lib_root + '/**/*.rb'].each do |r|
103
+ begin
104
+ require r if File.file?(r) && ! r.include?('testing') && ! r.include?('generators')
105
+ rescue => e
106
+ end
107
+ end
108
+
109
+ require 'product'
110
+ require 'lib/product_filters'
111
+ load_models( true )
112
+
113
+ end
114
+
115
+ def self.load_models( report_errors = nil )
116
+ puts 'load from', root
117
+ Dir[root + '/app/models/**/*.rb'].each {|r|
118
+ begin
119
+ require r if File.file?(r)
120
+ rescue => e
121
+ puts("WARNING failed to load #{r}", e.inspect) if(report_errors == true)
122
+ end
123
+ }
124
+ end
125
+
126
+ def self.migrate_up
127
+ load
128
+ ActiveRecord::Migrator.up( File.join(root, 'db/migrate') )
129
+ end
130
+
131
+ end