datashift 0.2.2 → 0.4.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.markdown +15 -3
- data/VERSION +1 -1
- data/datashift.gemspec +11 -3
- data/lib/applications/jruby/jexcel_file.rb +10 -3
- data/lib/datashift.rb +25 -62
- data/lib/datashift/exceptions.rb +1 -0
- data/lib/datashift/logging.rb +58 -0
- data/lib/datashift/method_detail.rb +6 -45
- data/lib/datashift/method_details_manager.rb +54 -0
- data/lib/datashift/method_dictionary.rb +6 -1
- data/lib/datashift/method_mapper.rb +12 -5
- data/lib/datashift/populator.rb +46 -0
- data/lib/exporters/excel_exporter.rb +1 -1
- data/lib/generators/excel_generator.rb +48 -44
- data/lib/helpers/spree_helper.rb +14 -3
- data/lib/loaders/csv_loader.rb +9 -6
- data/lib/loaders/excel_loader.rb +5 -1
- data/lib/loaders/loader_base.rb +28 -14
- data/lib/loaders/spree/image_loader.rb +36 -34
- data/lib/loaders/spree/product_loader.rb +17 -7
- data/lib/thor/generate_excel.thor +35 -15
- data/lib/thor/import_excel.thor +84 -0
- data/lib/thor/spree/bootstrap_cleanup.thor +33 -0
- data/lib/thor/spree/products_images.thor +171 -0
- data/spec/datashift_spec.rb +19 -0
- data/spec/excel_exporter_spec.rb +3 -3
- data/spec/fixtures/datashift_Spree_db.sqlite +0 -0
- data/spec/fixtures/datashift_test_models_db.sqlite +0 -0
- data/spec/fixtures/spree/SpreeProductsDefaults.yml +15 -0
- data/spec/fixtures/spree/SpreeProductsMandatoryOnly.xls +0 -0
- data/spec/fixtures/spree/SpreeProductsWithImages.xls +0 -0
- data/spec/spec_helper.rb +2 -2
- data/spec/spree_generator_spec.rb +14 -0
- data/spec/spree_images_loader_spec.rb +73 -0
- data/spec/spree_loader_spec.rb +53 -19
- data/tasks/spree/image_load.rake +18 -13
- metadata +11 -3
- data/tasks/spree/product_loader.rake +0 -44
data/README.markdown
CHANGED
@@ -2,6 +2,8 @@
|
|
2
2
|
|
3
3
|
Provides tools to shift data between Ruby, external applications, files and ActiveRecord.
|
4
4
|
|
5
|
+
Wiki taking shape with more info here : <b>https://github.com/autotelik/datashift/wiki</b>
|
6
|
+
|
5
7
|
### Features
|
6
8
|
|
7
9
|
Map Active Record models and associations to files, generate sample templates.
|
@@ -39,17 +41,23 @@ To pull the tasks in, add this call to your Rakefile :
|
|
39
41
|
|
40
42
|
DataShift::load_tasks
|
41
43
|
|
44
|
+
To keep the availability to only development mode use
|
45
|
+
|
46
|
+
if(Rails.env.development?)
|
47
|
+
DataShift::<b>load_tasks</b>
|
48
|
+
end
|
49
|
+
|
42
50
|
To use the Thor command line applications :
|
43
51
|
|
44
52
|
Create a high level .thor file - e.g mysite.thor - in your applications root directory
|
45
53
|
|
46
54
|
Pull in the thor commands :
|
47
|
-
|
55
|
+
```
|
48
56
|
require 'thor'
|
49
57
|
require 'datashift'
|
50
58
|
|
51
|
-
DataShift
|
52
|
-
|
59
|
+
DataShift::<b>load_commands</b>
|
60
|
+
```
|
53
61
|
To check the available tasks run
|
54
62
|
|
55
63
|
bundle exec rake -T datashift
|
@@ -58,6 +66,10 @@ and/or
|
|
58
66
|
|
59
67
|
bundle exc thor list datashift
|
60
68
|
|
69
|
+
To get usgae information use thor help <command>, for example
|
70
|
+
|
71
|
+
bundle exec thor help datashift:generate:excel
|
72
|
+
|
61
73
|
N.B - To use the Excel loader, OLE and Excel are NOT required, however
|
62
74
|
JRuby is required, since it uses Java's Apache POI under the hood to process .xls files.
|
63
75
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.4.0
|
data/datashift.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "datashift"
|
8
|
-
s.version = "0.
|
8
|
+
s.version = "0.4.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Thomas Statter"]
|
12
|
-
s.date = "2012-03-
|
12
|
+
s.date = "2012-03-12"
|
13
13
|
s.description = "A suite of tools to move data between ActiveRecord models,databases,applications like Excel/Open Office, files and projects including Spree"
|
14
14
|
s.email = "rubygems@autotelik.co.uk"
|
15
15
|
s.extra_rdoc_files = [
|
@@ -30,11 +30,14 @@ Gem::Specification.new do |s|
|
|
30
30
|
"lib/datashift.rb",
|
31
31
|
"lib/datashift/exceptions.rb",
|
32
32
|
"lib/datashift/file_definitions.rb",
|
33
|
+
"lib/datashift/logging.rb",
|
33
34
|
"lib/datashift/mapping_file_definitions.rb",
|
34
35
|
"lib/datashift/method_detail.rb",
|
36
|
+
"lib/datashift/method_details_manager.rb",
|
35
37
|
"lib/datashift/method_dictionary.rb",
|
36
38
|
"lib/datashift/method_mapper.rb",
|
37
39
|
"lib/datashift/model_mapper.rb",
|
40
|
+
"lib/datashift/populator.rb",
|
38
41
|
"lib/exporters/csv_exporter.rb",
|
39
42
|
"lib/exporters/excel_exporter.rb",
|
40
43
|
"lib/exporters/exporter_base.rb",
|
@@ -66,6 +69,9 @@ Gem::Specification.new do |s|
|
|
66
69
|
"lib/loaders/spree/image_loader.rb",
|
67
70
|
"lib/loaders/spree/product_loader.rb",
|
68
71
|
"lib/thor/generate_excel.thor",
|
72
|
+
"lib/thor/import_excel.thor",
|
73
|
+
"lib/thor/spree/bootstrap_cleanup.thor",
|
74
|
+
"lib/thor/spree/products_images.thor",
|
69
75
|
"public/spree/products/large/DEMO_001_ror_bag.jpeg",
|
70
76
|
"public/spree/products/large/DEMO_002_Powerstation.jpg",
|
71
77
|
"public/spree/products/large/DEMO_003_ror_mug.jpeg",
|
@@ -116,6 +122,8 @@ Gem::Specification.new do |s|
|
|
116
122
|
"spec/fixtures/simple_template_spec.xls",
|
117
123
|
"spec/fixtures/spree/SpreeProducts.csv",
|
118
124
|
"spec/fixtures/spree/SpreeProducts.xls",
|
125
|
+
"spec/fixtures/spree/SpreeProductsDefaults.yml",
|
126
|
+
"spec/fixtures/spree/SpreeProductsMandatoryOnly.xls",
|
119
127
|
"spec/fixtures/spree/SpreeProductsMultiColumn.csv",
|
120
128
|
"spec/fixtures/spree/SpreeProductsMultiColumn.xls",
|
121
129
|
"spec/fixtures/spree/SpreeProductsSimple.csv",
|
@@ -131,6 +139,7 @@ Gem::Specification.new do |s|
|
|
131
139
|
"spec/spec_helper.rb",
|
132
140
|
"spec/spree_exporter_spec.rb",
|
133
141
|
"spec/spree_generator_spec.rb",
|
142
|
+
"spec/spree_images_loader_spec.rb",
|
134
143
|
"spec/spree_loader_spec.rb",
|
135
144
|
"spec/spree_method_mapping_spec.rb",
|
136
145
|
"tasks/config/seed_fu_product_template.erb",
|
@@ -140,7 +149,6 @@ Gem::Specification.new do |s|
|
|
140
149
|
"tasks/import/csv.rake",
|
141
150
|
"tasks/import/excel.rake",
|
142
151
|
"tasks/spree/image_load.rake",
|
143
|
-
"tasks/spree/product_loader.rake",
|
144
152
|
"tasks/word_to_seedfu.rake",
|
145
153
|
"test/helper.rb",
|
146
154
|
"test/test_interact.rb"
|
@@ -272,10 +272,17 @@ if(DataShift::Guards::jruby?)
|
|
272
272
|
end
|
273
273
|
|
274
274
|
def save( filename = nil )
|
275
|
+
|
275
276
|
filename.nil? ? file = @filepath : file = filename
|
276
|
-
|
277
|
-
|
278
|
-
|
277
|
+
begin
|
278
|
+
out = FileOutputStream.new(file)
|
279
|
+
@book.write(out) unless @book.nil?
|
280
|
+
|
281
|
+
out.close
|
282
|
+
rescue => e
|
283
|
+
puts e
|
284
|
+
raise "Cannot write file - is file already open in Excel ?"
|
285
|
+
end
|
279
286
|
end
|
280
287
|
|
281
288
|
def save_to_text( filename )
|
data/lib/datashift.rb
CHANGED
@@ -1,11 +1,32 @@
|
|
1
|
-
# Copyright:: (c) Autotelik Media Ltd
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2010 - 2012 Tom Statter
|
2
2
|
# Author :: Tom Statter
|
3
3
|
# Date :: Aug 2010
|
4
|
-
# License::
|
4
|
+
# License:: Free, Open Source.
|
5
5
|
#
|
6
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
7
|
+
# a copy of this software and associated documentation files (the
|
8
|
+
# "Software"), to deal in the Software without restriction, including
|
9
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
10
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
11
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
12
|
+
# the following conditions:
|
13
|
+
#
|
14
|
+
# The above copyright notice and this permission notice shall be
|
15
|
+
# included in all copies or substantial portions of the Software.
|
16
|
+
#
|
17
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
18
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
19
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
20
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
21
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
22
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
23
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
24
|
+
#++
|
25
|
+
|
26
|
+
|
6
27
|
# Details:: Active Record Loader
|
7
28
|
#
|
8
|
-
#
|
29
|
+
# To pull DataShift commands into your main application :
|
9
30
|
#
|
10
31
|
# require 'datashift'
|
11
32
|
#
|
@@ -93,6 +114,7 @@ module DataShift
|
|
93
114
|
require_libs.each do |base|
|
94
115
|
Dir[File.join(library_path, base, '*.rb')].each do |rb|
|
95
116
|
unless File.directory? rb
|
117
|
+
#puts rb
|
96
118
|
require rb
|
97
119
|
end
|
98
120
|
end
|
@@ -112,72 +134,13 @@ module DataShift
|
|
112
134
|
# Load all the datashift Thor commands and make them available throughout app
|
113
135
|
|
114
136
|
def self.load_commands()
|
115
|
-
|
116
137
|
base = File.join(library_path, 'thor', '**')
|
117
138
|
|
118
139
|
Dir["#{base}/*.thor"].each do |f|
|
119
|
-
|
120
140
|
next unless File.file?(f)
|
121
141
|
Thor::Util.load_thorfile(f)
|
122
142
|
end
|
123
143
|
end
|
124
|
-
|
125
|
-
|
126
|
-
def self.load_thorfiles(dir)
|
127
|
-
Dir.chdir(dir) do
|
128
|
-
thor_files = Dir.glob('**/*.thor').delete_if { |x| not File.file?(x) }
|
129
|
-
thor_files.each do |f|
|
130
|
-
Thor::Util.load_thorfile(f)
|
131
|
-
end
|
132
|
-
end
|
133
|
-
end
|
134
|
-
|
135
|
-
|
136
|
-
module Logging
|
137
|
-
|
138
|
-
class MultiIO
|
139
|
-
|
140
|
-
def initialize(*targets)
|
141
|
-
@targets = []
|
142
|
-
targets.each {|t| @targets << Logger.new(t) }
|
143
|
-
end
|
144
|
-
|
145
|
-
def add(target)
|
146
|
-
@targets << Logger.new(target)
|
147
|
-
end
|
148
|
-
|
149
|
-
|
150
|
-
def method_missing(method, *args, &block)
|
151
|
-
@targets.each {|t| t.send(method, *args, &block) }
|
152
|
-
end
|
153
|
-
|
154
|
-
def verbose
|
155
|
-
add(STDOUT)
|
156
|
-
end
|
157
|
-
|
158
|
-
end
|
159
|
-
|
160
|
-
require 'logger'
|
161
|
-
|
162
|
-
def logdir
|
163
|
-
@logdir ||= 'log'
|
164
|
-
@logdir
|
165
|
-
end
|
166
|
-
|
167
|
-
def logger
|
168
|
-
@logger ||= open
|
169
|
-
@logger
|
170
|
-
end
|
171
|
-
|
172
|
-
private
|
173
|
-
|
174
|
-
def open( log = 'datashift.log')
|
175
|
-
FileUtils::mkdir(logdir) unless File.directory?(logdir)
|
176
|
-
log_file = File.open( File.join(logdir(), 'datashift.log'), "a")
|
177
|
-
@logger = MultiIO.new(log_file)
|
178
|
-
@logger
|
179
|
-
end
|
180
|
-
end
|
181
144
|
|
182
145
|
end
|
183
146
|
|
data/lib/datashift/exceptions.rb
CHANGED
@@ -0,0 +1,58 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2012
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2012
|
4
|
+
# License:: Free, Open Source. MIT.
|
5
|
+
#
|
6
|
+
# Details:: Logging facilities for datashift.
|
7
|
+
#
|
8
|
+
require 'fileutils'
|
9
|
+
|
10
|
+
module DataShift
|
11
|
+
|
12
|
+
module Logging
|
13
|
+
|
14
|
+
class MultiIO
|
15
|
+
|
16
|
+
def initialize(*targets)
|
17
|
+
@targets = []
|
18
|
+
targets.each {|t| @targets << Logger.new(t) }
|
19
|
+
end
|
20
|
+
|
21
|
+
def add(target)
|
22
|
+
@targets << Logger.new(target)
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
def method_missing(method, *args, &block)
|
27
|
+
@targets.each {|t| t.send(method, *args, &block) }
|
28
|
+
end
|
29
|
+
|
30
|
+
def verbose
|
31
|
+
add(STDOUT)
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
35
|
+
|
36
|
+
require 'logger'
|
37
|
+
|
38
|
+
def logdir
|
39
|
+
@logdir ||= 'log'
|
40
|
+
@logdir
|
41
|
+
end
|
42
|
+
|
43
|
+
def logger
|
44
|
+
@logger ||= open
|
45
|
+
@logger
|
46
|
+
end
|
47
|
+
|
48
|
+
private
|
49
|
+
|
50
|
+
def open( log = 'datashift.log')
|
51
|
+
return ActiveRecord::Base.logger if(defined?(ActiveRecord) && ActiveRecord::Base.logger)
|
52
|
+
FileUtils::mkdir(logdir) unless File.directory?(logdir)
|
53
|
+
log_file = File.open( File.join(logdir(), 'datashift.log'), "a")
|
54
|
+
MultiIO.new(log_file)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -14,49 +14,10 @@ require 'to_b'
|
|
14
14
|
|
15
15
|
module DataShift
|
16
16
|
|
17
|
-
# Stores MethodDetails for a class mapped by type
|
18
|
-
class MethodDetailsManager
|
19
|
-
|
20
|
-
attr_reader :method_details
|
21
|
-
|
22
|
-
def initialize( klass )
|
23
|
-
@parent_class = klass
|
24
|
-
@method_details = {}
|
25
|
-
@method_details_list = {}
|
26
|
-
end
|
27
|
-
|
28
|
-
def add(method_details)
|
29
|
-
@method_details[method_details.operator_type] ||= {}
|
30
|
-
@method_details_list[method_details.operator_type] ||= []
|
31
|
-
|
32
|
-
@method_details[method_details.operator_type][method_details.name] = method_details
|
33
|
-
@method_details_list[method_details.operator_type] << method_details
|
34
|
-
@method_details_list[method_details.operator_type].uniq!
|
35
|
-
end
|
36
|
-
|
37
|
-
def <<(method_details)
|
38
|
-
add(method_details)
|
39
|
-
end
|
40
|
-
|
41
|
-
def find(name, type)
|
42
|
-
method_details = get(type)
|
43
|
-
|
44
|
-
method_details ? method_details[name] : nil
|
45
|
-
end
|
46
|
-
|
47
|
-
# type is expected to be one of MethodDetail::supportedtype_enum
|
48
|
-
def get( type )
|
49
|
-
@method_details[type]
|
50
|
-
end
|
51
|
-
|
52
|
-
def get_list( type )
|
53
|
-
@method_details_list[type]
|
54
|
-
end
|
55
|
-
|
56
|
-
end
|
57
|
-
|
58
17
|
class MethodDetail
|
59
18
|
|
19
|
+
include DataShift::Logging
|
20
|
+
|
60
21
|
def self.supported_types_enum
|
61
22
|
@type_enum ||= Set[:assignment, :belongs_to, :has_one, :has_many]
|
62
23
|
@type_enum
|
@@ -149,7 +110,7 @@ module DataShift
|
|
149
110
|
|
150
111
|
elsif( operator_for(:has_many) )
|
151
112
|
|
152
|
-
puts "DEBUG : VALUE TYPE [#{value.class.name.include?(operator.classify)}] [#{ModelMapper.class_from_string(value.class.name)}]" unless(value.is_a?(Array))
|
113
|
+
#puts "DEBUG : VALUE TYPE [#{value.class.name.include?(operator.classify)}] [#{ModelMapper.class_from_string(value.class.name)}]" unless(value.is_a?(Array))
|
153
114
|
|
154
115
|
# The include? check is best I can come up with right now .. to handle module/namespaces
|
155
116
|
# TODO - can we determine the real class type of an association
|
@@ -168,20 +129,20 @@ module DataShift
|
|
168
129
|
if(value.is_a?(operator_class))
|
169
130
|
record.send(operator + '=', value)
|
170
131
|
else
|
171
|
-
|
132
|
+
logger.error("ERROR #{value.class} - Not expected type for has_one #{operator} - cannot assign")
|
172
133
|
# TODO - Not expected type - maybe try to look it up somehow ?"
|
173
134
|
#insistent_has_many(record, @current_value)
|
174
135
|
end
|
175
136
|
|
176
137
|
elsif( operator_for(:assignment) && @col_type )
|
177
138
|
#puts "DEBUG : COl TYPE defined for #{@name} : #{@assignment} => #{@current_value} #{@col_type.type}"
|
178
|
-
|
139
|
+
# puts "DEBUG : Column [#{@name}] : COl TYPE CAST: #{@current_value} => #{@col_type.type_cast( @current_value ).inspect}"
|
179
140
|
record.send( operator + '=' , @col_type.type_cast( @current_value ) )
|
180
141
|
|
181
142
|
#puts "DEBUG : MethodDetails Assignment RESULT: #{record.send(operator)}"
|
182
143
|
|
183
144
|
elsif( operator_for(:assignment) )
|
184
|
-
#puts "DEBUG : Brute force assignment of value #{@current_value}
|
145
|
+
#puts "DEBUG : Column [#{@name}] : Brute force assignment of value #{@current_value}"
|
185
146
|
# brute force case for assignments without a column type (which enables us to do correct type_cast)
|
186
147
|
# so in this case, attempt straightforward assignment then if that fails, basic ops such as to_s, to_i, to_f etc
|
187
148
|
insistent_assignment(record, @current_value)
|
@@ -0,0 +1,54 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2011
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Aug 2010
|
4
|
+
# License:: MIT
|
5
|
+
#
|
6
|
+
# Details:: This class provides info and access to groups of accessor methods,
|
7
|
+
# grouped by AR model.
|
8
|
+
#
|
9
|
+
require 'to_b'
|
10
|
+
|
11
|
+
module DataShift
|
12
|
+
|
13
|
+
# Stores MethodDetails for a class mapped by type
|
14
|
+
class MethodDetailsManager
|
15
|
+
|
16
|
+
attr_reader :method_details
|
17
|
+
|
18
|
+
def initialize( klass )
|
19
|
+
@parent_class = klass
|
20
|
+
@method_details = {}
|
21
|
+
@method_details_list = {}
|
22
|
+
end
|
23
|
+
|
24
|
+
def add(method_details)
|
25
|
+
@method_details[method_details.operator_type.to_sym] ||= {}
|
26
|
+
@method_details_list[method_details.operator_type.to_sym] ||= []
|
27
|
+
|
28
|
+
@method_details[method_details.operator_type.to_sym][method_details.name] = method_details
|
29
|
+
@method_details_list[method_details.operator_type.to_sym] << method_details
|
30
|
+
@method_details_list[method_details.operator_type.to_sym].uniq!
|
31
|
+
end
|
32
|
+
|
33
|
+
def <<(method_details)
|
34
|
+
add(method_details)
|
35
|
+
end
|
36
|
+
|
37
|
+
def find(name, type)
|
38
|
+
method_details = get(type)
|
39
|
+
|
40
|
+
method_details ? method_details[name] : nil
|
41
|
+
end
|
42
|
+
|
43
|
+
# type is expected to be one of MethodDetail::supportedtype_enum
|
44
|
+
def get( type )
|
45
|
+
@method_details[type.to_sym]
|
46
|
+
end
|
47
|
+
|
48
|
+
def get_list( type )
|
49
|
+
@method_details_list[type.to_sym]
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|