datashift 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.document +5 -5
- data/LICENSE.txt +26 -26
- data/README.markdown +326 -305
- data/README.rdoc +19 -19
- data/Rakefile +86 -93
- data/VERSION +1 -1
- data/datashift.gemspec +163 -152
- data/lib/applications/jruby/jexcel_file.rb +410 -408
- data/lib/applications/jruby/word.rb +79 -79
- data/lib/datashift.rb +183 -152
- data/lib/datashift/exceptions.rb +11 -11
- data/lib/datashift/file_definitions.rb +353 -353
- data/lib/datashift/mapping_file_definitions.rb +87 -87
- data/lib/datashift/method_detail.rb +293 -275
- data/lib/datashift/method_dictionary.rb +208 -209
- data/lib/datashift/method_mapper.rb +90 -90
- data/lib/datashift/model_mapper.rb +27 -0
- data/lib/exporters/csv_exporter.rb +36 -0
- data/lib/exporters/excel_exporter.rb +116 -0
- data/lib/exporters/exporter_base.rb +15 -0
- data/lib/generators/csv_generator.rb +36 -36
- data/lib/generators/excel_generator.rb +106 -122
- data/lib/generators/generator_base.rb +13 -13
- data/lib/helpers/core_ext/to_b.rb +24 -24
- data/lib/helpers/rake_utils.rb +42 -0
- data/lib/helpers/spree_helper.rb +194 -153
- data/lib/java/poi-3.7/LICENSE +507 -507
- data/lib/java/poi-3.7/NOTICE +21 -21
- data/lib/java/poi-3.7/RELEASE_NOTES.txt +115 -115
- data/lib/loaders/csv_loader.rb +98 -98
- data/lib/loaders/excel_loader.rb +155 -155
- data/lib/loaders/loader_base.rb +420 -420
- data/lib/loaders/spreadsheet_loader.rb +136 -136
- data/lib/loaders/spree/image_loader.rb +67 -63
- data/lib/loaders/spree/product_loader.rb +289 -248
- data/lib/thor/generate_excel.thor +54 -0
- data/sandbox/app/controllers/application_controller.rb +3 -0
- data/sandbox/config/application.rb +43 -0
- data/sandbox/config/database.yml +34 -0
- data/sandbox/config/environment.rb +7 -0
- data/sandbox/config/environments/development.rb +30 -0
- data/spec/csv_loader_spec.rb +30 -30
- data/spec/datashift_spec.rb +26 -26
- data/spec/db/migrate/20110803201325_create_test_bed.rb +85 -85
- data/spec/excel_exporter_spec.rb +78 -78
- data/spec/excel_generator_spec.rb +78 -78
- data/spec/excel_loader_spec.rb +223 -223
- data/spec/file_definitions.rb +141 -141
- data/spec/fixtures/ProjectsDefaults.yml +29 -29
- data/spec/fixtures/config/database.yml +27 -27
- data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
- data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
- data/spec/fixtures/negative/SpreeProdMiss1Mandatory.csv +4 -4
- data/spec/fixtures/negative/SpreeProdMissManyMandatory.csv +4 -4
- data/spec/fixtures/spree/SpreeProducts.csv +4 -4
- data/spec/fixtures/spree/SpreeProducts.xls +0 -0
- data/spec/fixtures/spree/SpreeProductsMultiColumn.csv +4 -4
- data/spec/fixtures/spree/SpreeProductsMultiColumn.xls +0 -0
- data/spec/fixtures/spree/SpreeProductsSimple.csv +4 -4
- data/spec/fixtures/spree/SpreeProductsWithImages.csv +4 -4
- data/spec/fixtures/spree/SpreeZoneExample.csv +5 -5
- data/spec/fixtures/test_model_defs.rb +57 -57
- data/spec/loader_spec.rb +120 -120
- data/spec/method_dictionary_spec.rb +242 -242
- data/spec/method_mapper_spec.rb +41 -41
- data/spec/spec_helper.rb +154 -116
- data/spec/spree_exporter_spec.rb +67 -0
- data/spec/spree_generator_spec.rb +77 -64
- data/spec/spree_loader_spec.rb +363 -324
- data/spec/spree_method_mapping_spec.rb +218 -214
- data/tasks/config/seed_fu_product_template.erb +15 -15
- data/tasks/config/tidy_config.txt +12 -12
- data/tasks/{excel_generator.rake → export/excel_generator.rake} +101 -78
- data/tasks/file_tasks.rake +36 -36
- data/tasks/import/csv.rake +50 -49
- data/tasks/import/excel.rake +74 -71
- data/tasks/spree/image_load.rake +108 -108
- data/tasks/spree/product_loader.rake +43 -43
- data/tasks/word_to_seedfu.rake +166 -166
- data/test/helper.rb +18 -18
- data/test/test_interact.rb +7 -7
- metadata +16 -8
- data/datashift-0.1.0.gem +0 -0
- data/tasks/db_tasks.rake +0 -66
@@ -0,0 +1,27 @@
|
|
1
|
+
class ModelMapper
|
2
|
+
|
3
|
+
# Helper to deal with string versions of modules/namespaced classes
|
4
|
+
# Find and return the base class from a string.
|
5
|
+
#
|
6
|
+
# e.g "Spree::Property" returns the Spree::Property class
|
7
|
+
# Raises exception if no such class found
|
8
|
+
def self.const_get_from_string(str)
|
9
|
+
str.split('::').inject(Object) do |mod, class_name|
|
10
|
+
mod.const_get(class_name)
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
|
15
|
+
# Similar to const_get_from_string except this version
|
16
|
+
# returns nil if no such class found
|
17
|
+
# Support modules e.g "Spree::Property"
|
18
|
+
#
|
19
|
+
def self.class_from_string( str )
|
20
|
+
begin
|
21
|
+
ModelMapper::const_get_from_string(str) #Kernel.const_get(model)
|
22
|
+
rescue NameError => e
|
23
|
+
return nil
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2011
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: Export a model to CSV
|
7
|
+
#
|
8
|
+
#
|
9
|
+
require 'exporter_base'
|
10
|
+
|
11
|
+
module DataShift
|
12
|
+
|
13
|
+
class CsvExporter < ExporterBase
|
14
|
+
|
15
|
+
attr_accessor :excel, :filename
|
16
|
+
|
17
|
+
def initialize(filename)
|
18
|
+
@excel = nil
|
19
|
+
@filename = filename
|
20
|
+
end
|
21
|
+
|
22
|
+
# Create CSV file representing supplied Model
|
23
|
+
|
24
|
+
def generate(model, options = {})
|
25
|
+
|
26
|
+
@filename = options[:filename] if options[:filename]
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# Create an Csv file representing supplied Model
|
31
|
+
|
32
|
+
def export(items, options = {})
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,116 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2011
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: Export a model to Excel '97(-2007) file format.
|
7
|
+
#
|
8
|
+
# TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
|
9
|
+
#
|
10
|
+
#
|
11
|
+
module DataShift
|
12
|
+
|
13
|
+
require 'exporter_base'
|
14
|
+
|
15
|
+
if(Guards::jruby?)
|
16
|
+
|
17
|
+
require 'jruby/jexcel_file'
|
18
|
+
|
19
|
+
class ExcelExporter < ExporterBase
|
20
|
+
|
21
|
+
attr_accessor :filename
|
22
|
+
|
23
|
+
def initialize(filename)
|
24
|
+
@filename = filename
|
25
|
+
end
|
26
|
+
|
27
|
+
|
28
|
+
# Create an Excel file from list of ActiveRecord objects
|
29
|
+
def export(records, options = {})
|
30
|
+
|
31
|
+
excel = JExcelFile.new()
|
32
|
+
|
33
|
+
if(options[:sheet_name] )
|
34
|
+
excel.create_sheet( options[:sheet_name] )
|
35
|
+
else
|
36
|
+
excel.create_sheet( records.first.class.name )
|
37
|
+
end
|
38
|
+
|
39
|
+
excel.ar_to_headers(records)
|
40
|
+
|
41
|
+
excel.ar_to_xls(records)
|
42
|
+
|
43
|
+
excel.save( filename() )
|
44
|
+
end
|
45
|
+
|
46
|
+
# Create an Excel file from list of ActiveRecord objects
|
47
|
+
# Specify which associations to export via :with or :exclude
|
48
|
+
# Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
|
49
|
+
#
|
50
|
+
def export_with_associations(klass, items, options = {})
|
51
|
+
|
52
|
+
excel = JExcelFile.new()
|
53
|
+
|
54
|
+
if(options[:sheet_name] )
|
55
|
+
excel.create_sheet( options[:sheet_name] )
|
56
|
+
else
|
57
|
+
excel.create_sheet( items.first.class.name )
|
58
|
+
end
|
59
|
+
|
60
|
+
MethodDictionary.find_operators( klass )
|
61
|
+
|
62
|
+
MethodDictionary.build_method_details( klass )
|
63
|
+
|
64
|
+
work_list = options[:with] || MethodDetail::supported_types_enum
|
65
|
+
|
66
|
+
headers = []
|
67
|
+
puts "work_list : [#{work_list.inspect}]"
|
68
|
+
|
69
|
+
details_mgr = MethodDictionary.method_details_mgrs[klass]
|
70
|
+
|
71
|
+
work_list.each do |op_type|
|
72
|
+
list_for_class_and_op = details_mgr.get_list(op_type)
|
73
|
+
|
74
|
+
next if(list_for_class_and_op.nil? || list_for_class_and_op.empty?)
|
75
|
+
#if(work_list.include?(md.operator_type))
|
76
|
+
#each do |mdtype|
|
77
|
+
#end
|
78
|
+
#if(MethodDictionary.respond_to?("#{mdtype}_for") )
|
79
|
+
# method_details = MethodDictionary.send("#{mdtype}_for", klass)
|
80
|
+
|
81
|
+
list_for_class_and_op.each {|md| headers << "#{md.operator}" }
|
82
|
+
#else
|
83
|
+
# puts "ERROR : Unknown option in :with [#{mdtype}]"
|
84
|
+
|
85
|
+
end
|
86
|
+
|
87
|
+
excel.set_headers( headers )
|
88
|
+
|
89
|
+
data = []
|
90
|
+
|
91
|
+
items.each do |record|
|
92
|
+
|
93
|
+
MethodMapper.method_details[klass].each do |method_detail|
|
94
|
+
if(method_detail.operator_type == :assignment)
|
95
|
+
data << record.send( method_detail.operator )
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
excel.set_row(2,1,items)
|
101
|
+
|
102
|
+
excel.save( filename() )
|
103
|
+
end
|
104
|
+
end # ExcelGenerator
|
105
|
+
|
106
|
+
else
|
107
|
+
class ExcelExporter < ExcelBase
|
108
|
+
|
109
|
+
def initialize(filename)
|
110
|
+
@filename = filename
|
111
|
+
raise DataShift::BadRuby, "Apologies but Datashift Excel facilities currently need JRuby. Please switch to, or install JRuby"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end # jruby
|
115
|
+
|
116
|
+
end # DataShift
|
@@ -0,0 +1,15 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2010
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: Base class for Exporters, which provide services to export a Model
|
7
|
+
# and it's data from database to an external format
|
8
|
+
#
|
9
|
+
module DataShift
|
10
|
+
|
11
|
+
class ExporterBase
|
12
|
+
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -1,36 +1,36 @@
|
|
1
|
-
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
-
# Author :: Tom Statter
|
3
|
-
# Date :: Aug 2011
|
4
|
-
# License:: MIT
|
5
|
-
#
|
6
|
-
# Details:: Export a model to CSV
|
7
|
-
#
|
8
|
-
#
|
9
|
-
require 'generator_base'
|
10
|
-
|
11
|
-
module DataShift
|
12
|
-
|
13
|
-
class CsvGenerator < GeneratorBase
|
14
|
-
|
15
|
-
attr_accessor :excel, :filename
|
16
|
-
|
17
|
-
def initialize(filename)
|
18
|
-
@excel = nil
|
19
|
-
@filename = filename
|
20
|
-
end
|
21
|
-
|
22
|
-
# Create CSV file representing supplied Model
|
23
|
-
|
24
|
-
def generate(model, options = {})
|
25
|
-
|
26
|
-
@filename = options[:filename] if options[:filename]
|
27
|
-
end
|
28
|
-
|
29
|
-
|
30
|
-
# Create an Csv file representing supplied Model
|
31
|
-
|
32
|
-
def export(items, options = {})
|
33
|
-
end
|
34
|
-
|
35
|
-
end
|
36
|
-
end
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2011
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: Export a model to CSV
|
7
|
+
#
|
8
|
+
#
|
9
|
+
require 'generator_base'
|
10
|
+
|
11
|
+
module DataShift
|
12
|
+
|
13
|
+
class CsvGenerator < GeneratorBase
|
14
|
+
|
15
|
+
attr_accessor :excel, :filename
|
16
|
+
|
17
|
+
def initialize(filename)
|
18
|
+
@excel = nil
|
19
|
+
@filename = filename
|
20
|
+
end
|
21
|
+
|
22
|
+
# Create CSV file representing supplied Model
|
23
|
+
|
24
|
+
def generate(model, options = {})
|
25
|
+
|
26
|
+
@filename = options[:filename] if options[:filename]
|
27
|
+
end
|
28
|
+
|
29
|
+
|
30
|
+
# Create an Csv file representing supplied Model
|
31
|
+
|
32
|
+
def export(items, options = {})
|
33
|
+
end
|
34
|
+
|
35
|
+
end
|
36
|
+
end
|
@@ -1,123 +1,107 @@
|
|
1
|
-
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
-
# Author :: Tom Statter
|
3
|
-
# Date :: Aug 2011
|
4
|
-
# License:: MIT
|
5
|
-
#
|
6
|
-
# Details:: Export a model to Excel '97(-2007) file format.
|
7
|
-
#
|
8
|
-
# TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
|
9
|
-
#
|
10
|
-
#
|
11
|
-
module DataShift
|
12
|
-
|
13
|
-
require 'generator_base'
|
14
|
-
|
15
|
-
if(Guards::jruby?)
|
16
|
-
|
17
|
-
require 'jruby/jexcel_file'
|
18
|
-
|
19
|
-
class ExcelGenerator < GeneratorBase
|
20
|
-
|
21
|
-
attr_accessor :filename
|
22
|
-
|
23
|
-
def initialize(filename)
|
24
|
-
@filename = filename
|
25
|
-
end
|
26
|
-
|
27
|
-
# Create an Excel file template (header row) representing supplied Model
|
28
|
-
|
29
|
-
def generate(
|
30
|
-
MethodDictionary.find_operators(
|
31
|
-
|
32
|
-
@filename = options[:filename] if options[:filename]
|
33
|
-
|
34
|
-
excel = JExcelFile.new()
|
35
|
-
|
36
|
-
if(options[:sheet_name] )
|
37
|
-
excel.create_sheet( options[:sheet_name] )
|
38
|
-
else
|
39
|
-
excel.create_sheet(
|
40
|
-
end
|
41
|
-
|
42
|
-
raise "Failed to create Excel WorkSheet for #{
|
43
|
-
|
44
|
-
excel.set_headers(MethodDictionary.assignments[
|
45
|
-
|
46
|
-
excel.save( @filename )
|
47
|
-
end
|
48
|
-
|
49
|
-
|
50
|
-
# Create an Excel file from list of ActiveRecord objects
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
end
|
108
|
-
|
109
|
-
excel.set_row(2,1,items)
|
110
|
-
|
111
|
-
excel.save( filename() )
|
112
|
-
end
|
113
|
-
end # ExcelGenerator
|
114
|
-
|
115
|
-
else
|
116
|
-
class ExcelGenerator < GeneratorBase
|
117
|
-
def initialize
|
118
|
-
raise DataShift::BadRuby, "Please install and use JRuby for working with .xls files"
|
119
|
-
end
|
120
|
-
end
|
121
|
-
end # jruby
|
122
|
-
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2011
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: Export a model to Excel '97(-2007) file format.
|
7
|
+
#
|
8
|
+
# TOD : Can we switch between .xls and XSSF (POI implementation of Excel 2007 OOXML (.xlsx) file format.)
|
9
|
+
#
|
10
|
+
#
|
11
|
+
module DataShift
|
12
|
+
|
13
|
+
require 'generator_base'
|
14
|
+
|
15
|
+
if(Guards::jruby?)
|
16
|
+
|
17
|
+
require 'jruby/jexcel_file'
|
18
|
+
|
19
|
+
class ExcelGenerator < GeneratorBase
|
20
|
+
|
21
|
+
attr_accessor :filename
|
22
|
+
|
23
|
+
def initialize(filename)
|
24
|
+
@filename = filename
|
25
|
+
end
|
26
|
+
|
27
|
+
# Create an Excel file template (header row) representing supplied Model
|
28
|
+
|
29
|
+
def generate(klass, options = {})
|
30
|
+
MethodDictionary.find_operators( klass )
|
31
|
+
|
32
|
+
@filename = options[:filename] if options[:filename]
|
33
|
+
|
34
|
+
excel = JExcelFile.new()
|
35
|
+
|
36
|
+
if(options[:sheet_name] )
|
37
|
+
excel.create_sheet( options[:sheet_name] )
|
38
|
+
else
|
39
|
+
excel.create_sheet( klass.name )
|
40
|
+
end
|
41
|
+
|
42
|
+
raise "Failed to create Excel WorkSheet for #{klass.name}" unless excel.sheet
|
43
|
+
|
44
|
+
excel.set_headers(MethodDictionary.assignments[klass])
|
45
|
+
|
46
|
+
excel.save( @filename )
|
47
|
+
end
|
48
|
+
|
49
|
+
|
50
|
+
# Create an Excel file from list of ActiveRecord objects
|
51
|
+
# Specify which associations to export via :with or :exclude
|
52
|
+
# Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
|
53
|
+
#
|
54
|
+
def generate_with_associations(klass, options = {})
|
55
|
+
|
56
|
+
excel = JExcelFile.new()
|
57
|
+
|
58
|
+
if(options[:sheet_name] )
|
59
|
+
excel.create_sheet( options[:sheet_name] )
|
60
|
+
else
|
61
|
+
excel.create_sheet( klass.name )
|
62
|
+
end
|
63
|
+
|
64
|
+
MethodDictionary.find_operators( klass )
|
65
|
+
|
66
|
+
MethodDictionary.build_method_details( klass )
|
67
|
+
|
68
|
+
work_list = options[:with] || MethodDetail::supported_types_enum
|
69
|
+
|
70
|
+
headers = []
|
71
|
+
puts "work_list : [#{work_list.inspect}]"
|
72
|
+
|
73
|
+
details_mgr = MethodDictionary.method_details_mgrs[klass]
|
74
|
+
|
75
|
+
work_list.each do |op_type|
|
76
|
+
list_for_class_and_op = details_mgr.get_list(op_type)
|
77
|
+
|
78
|
+
next if(list_for_class_and_op.nil? || list_for_class_and_op.empty?)
|
79
|
+
#if(work_list.include?(md.operator_type))
|
80
|
+
#each do |mdtype|
|
81
|
+
#end
|
82
|
+
#if(MethodDictionary.respond_to?("#{mdtype}_for") )
|
83
|
+
# method_details = MethodDictionary.send("#{mdtype}_for", klass)
|
84
|
+
|
85
|
+
list_for_class_and_op.each {|md| headers << "#{md.operator}" }
|
86
|
+
#else
|
87
|
+
# puts "ERROR : Unknown option in :with [#{mdtype}]"
|
88
|
+
|
89
|
+
end
|
90
|
+
|
91
|
+
excel.set_headers( headers )
|
92
|
+
|
93
|
+
excel.save( filename() )
|
94
|
+
end
|
95
|
+
end # ExcelGenerator
|
96
|
+
|
97
|
+
else
|
98
|
+
class ExcelGenerator < GeneratorBase
|
99
|
+
|
100
|
+
def initialize(filename)
|
101
|
+
@filename = filename
|
102
|
+
raise DataShift::BadRuby, "Apologies but Datashift Excel facilities currently need JRuby. Please switch to, or install JRuby"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end # jruby
|
106
|
+
|
123
107
|
end # DataShift
|