datashift 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.markdown +91 -55
- data/VERSION +1 -1
- data/datashift.gemspec +8 -23
- data/lib/applications/jexcel_file.rb +1 -2
- data/lib/datashift.rb +34 -15
- data/lib/datashift/column_packer.rb +98 -34
- data/lib/datashift/data_transforms.rb +83 -0
- data/lib/datashift/delimiters.rb +58 -10
- data/lib/datashift/excel_base.rb +123 -0
- data/lib/datashift/exceptions.rb +45 -7
- data/lib/datashift/load_object.rb +25 -0
- data/lib/datashift/mapping_service.rb +91 -0
- data/lib/datashift/method_detail.rb +40 -62
- data/lib/datashift/method_details_manager.rb +18 -2
- data/lib/datashift/method_dictionary.rb +27 -10
- data/lib/datashift/method_mapper.rb +49 -41
- data/lib/datashift/model_mapper.rb +42 -22
- data/lib/datashift/populator.rb +258 -143
- data/lib/datashift/thor_base.rb +38 -0
- data/lib/exporters/csv_exporter.rb +57 -145
- data/lib/exporters/excel_exporter.rb +73 -60
- data/lib/generators/csv_generator.rb +65 -5
- data/lib/generators/generator_base.rb +69 -3
- data/lib/generators/mapping_generator.rb +112 -0
- data/lib/helpers/core_ext/csv_file.rb +33 -0
- data/lib/loaders/csv_loader.rb +41 -39
- data/lib/loaders/excel_loader.rb +130 -116
- data/lib/loaders/loader_base.rb +190 -146
- data/lib/loaders/paperclip/attachment_loader.rb +4 -4
- data/lib/loaders/paperclip/datashift_paperclip.rb +5 -3
- data/lib/loaders/paperclip/image_loading.rb +9 -7
- data/lib/loaders/reporter.rb +17 -8
- data/lib/thor/export.thor +12 -13
- data/lib/thor/generate.thor +1 -9
- data/lib/thor/import.thor +13 -24
- data/lib/thor/mapping.thor +65 -0
- data/spec/Gemfile +13 -11
- data/spec/Gemfile.lock +98 -93
- data/spec/csv_exporter_spec.rb +104 -99
- data/spec/csv_generator_spec.rb +159 -0
- data/spec/csv_loader_spec.rb +197 -16
- data/spec/datashift_spec.rb +9 -0
- data/spec/excel_exporter_spec.rb +149 -58
- data/spec/excel_generator_spec.rb +35 -44
- data/spec/excel_loader_spec.rb +196 -178
- data/spec/excel_spec.rb +8 -5
- data/spec/loader_base_spec.rb +47 -7
- data/spec/mapping_spec.rb +117 -0
- data/spec/method_dictionary_spec.rb +24 -11
- data/spec/method_mapper_spec.rb +5 -7
- data/spec/model_mapper_spec.rb +41 -0
- data/spec/paperclip_loader_spec.rb +3 -6
- data/spec/populator_spec.rb +48 -14
- data/spec/spec_helper.rb +85 -73
- data/spec/thor_spec.rb +40 -5
- metadata +93 -86
- data/lib/applications/excel_base.rb +0 -63
@@ -0,0 +1,38 @@
|
|
1
|
+
# Copyright:: (c) Autotelik Media Ltd 2012
|
2
|
+
# Author :: Tom Statter
|
3
|
+
# Date :: Dec 2014
|
4
|
+
# License:: MIT.
|
5
|
+
#
|
6
|
+
# Note, not DataShift, case sensitive, create namespace for command line : datashift
|
7
|
+
require 'thor'
|
8
|
+
|
9
|
+
module DataShift
|
10
|
+
|
11
|
+
class DSThorBase < Thor
|
12
|
+
|
13
|
+
include DataShift::Logging
|
14
|
+
|
15
|
+
no_commands do
|
16
|
+
|
17
|
+
def start_connections()
|
18
|
+
|
19
|
+
# TODO - We're assuming run from a rails app/top level dir...
|
20
|
+
|
21
|
+
if(File.exists?(File.expand_path('config/environment.rb')))
|
22
|
+
begin
|
23
|
+
require File.expand_path('config/environment.rb')
|
24
|
+
rescue => e
|
25
|
+
logger.error("Failed to initialise ActiveRecord : #{e.message}")
|
26
|
+
raise ConnectionError.new("No config/environment.rb found - cannot initialise ActiveRecord")
|
27
|
+
end
|
28
|
+
|
29
|
+
else
|
30
|
+
raise PathError.new("No config/environment.rb found - cannot initialise ActiveRecord")
|
31
|
+
# TODO make this more robust ? e.g what about when using active record but not in Rails app, Sinatra etc
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
end
|
37
|
+
|
38
|
+
end
|
@@ -7,27 +7,19 @@
|
|
7
7
|
#
|
8
8
|
#
|
9
9
|
require 'exporter_base'
|
10
|
-
require '
|
10
|
+
require 'csv_file'
|
11
11
|
|
12
12
|
module DataShift
|
13
13
|
|
14
14
|
class CsvExporter < ExporterBase
|
15
|
-
|
15
|
+
|
16
16
|
include DataShift::Logging
|
17
|
-
|
18
|
-
|
19
|
-
|
17
|
+
include DataShift::ColumnPacker
|
18
|
+
|
20
19
|
def initialize(filename)
|
21
20
|
super(filename)
|
22
|
-
@text_delim = "\'"
|
23
21
|
end
|
24
22
|
|
25
|
-
# Return opposite of text delim - "hello, 'barry'" => '"hello, "barry""'
|
26
|
-
def escape_text_delim
|
27
|
-
return '"' if @text_delim == "\'"
|
28
|
-
"\'"
|
29
|
-
end
|
30
|
-
|
31
23
|
# Create CSV file from set of ActiveRecord objects
|
32
24
|
# Options :
|
33
25
|
# => :filename
|
@@ -35,161 +27,81 @@ module DataShift
|
|
35
27
|
# => ::methods => List of methods to additionally call on each record
|
36
28
|
#
|
37
29
|
def export(export_records, options = {})
|
38
|
-
|
30
|
+
|
39
31
|
records = [*export_records]
|
40
|
-
|
41
|
-
|
42
|
-
puts records, records.inspect
|
43
|
-
|
32
|
+
|
44
33
|
unless(records && records.size > 0)
|
45
|
-
logger.warn("No objects supplied for export")
|
46
|
-
return
|
34
|
+
logger.warn("No objects supplied for export")
|
35
|
+
return
|
47
36
|
end
|
48
|
-
|
37
|
+
|
49
38
|
first = records[0]
|
50
|
-
|
39
|
+
|
51
40
|
raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
headers = first.serializable_hash.keys
|
60
|
-
|
61
|
-
[*options[:methods]].each do |c| headers << c if(first.respond_to?(c)) end if(options[:methods])
|
62
|
-
|
63
|
-
csv << headers.join(Delimiters::csv_delim) << Delimiters::eol
|
64
|
-
|
41
|
+
|
42
|
+
Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
|
43
|
+
|
44
|
+
CSV.open( (options[:filename] || filename), "w" ) do |csv|
|
45
|
+
|
46
|
+
csv.ar_to_headers( records )
|
47
|
+
|
65
48
|
records.each do |r|
|
66
49
|
next unless(r.is_a?(ActiveRecord::Base))
|
67
|
-
|
68
|
-
csv << record_to_csv(r, options) << Delimiters::eol
|
50
|
+
csv.ar_to_csv(r, options)
|
69
51
|
end
|
70
52
|
end
|
71
53
|
end
|
72
|
-
|
54
|
+
|
73
55
|
# Create an Excel file from list of ActiveRecord objects
|
74
56
|
# Specify which associations to export via :with or :exclude
|
75
57
|
# Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
|
76
58
|
#
|
77
59
|
def export_with_associations(klass, records, options = {})
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
@text_delim = options[:text_delim] if(options[:text_delim])
|
82
|
-
|
60
|
+
|
61
|
+
Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
|
62
|
+
|
83
63
|
MethodDictionary.find_operators( klass )
|
84
|
-
|
85
|
-
# builds all possible operators
|
64
|
+
|
86
65
|
MethodDictionary.build_method_details( klass )
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
next unless(r.is_a?(ActiveRecord::Base))
|
120
|
-
|
121
|
-
csv_columns = []
|
122
|
-
# need to make sure order matches headers
|
123
|
-
# could look at collection headers via serializable hash.keys and then use
|
124
|
-
# csv << record_to_csv(r) ??
|
125
|
-
assignments.each {|x| csv << escape(r.send(x)) << Delimiters::csv_delim }
|
126
|
-
|
127
|
-
# done records basic attributes now deal with associations
|
128
|
-
|
129
|
-
#assoc_work_list.each do |op_type|
|
130
|
-
# details_mgr.get_operators(op_type).each do |operator|
|
131
|
-
assoc_operators.each do |operator|
|
132
|
-
assoc_object = r.send(operator)
|
133
|
-
|
134
|
-
if(assoc_object.is_a?ActiveRecord::Base)
|
135
|
-
column_text = record_to_column(assoc_object) # belongs_to or has_one
|
136
|
-
|
137
|
-
# TODO -ColumnPacker class shared between excel/csv
|
138
|
-
|
139
|
-
csv << "#{@text_delim}#{column_text}#{@text_delim}" << Delimiters::csv_delim
|
140
|
-
#csv << record_to_csv(r)
|
141
|
-
|
142
|
-
elsif(assoc_object.is_a? Array)
|
143
|
-
items_to_s = assoc_object.collect {|x| record_to_column(x) }
|
144
|
-
|
145
|
-
# create a single column
|
146
|
-
csv << "#{@text_delim}#{items_to_s.join(Delimiters::multi_assoc_delim)}#{@text_delim}" << Delimiters::csv_delim
|
147
|
-
|
148
|
-
else
|
149
|
-
csv << Delimiters::csv_delim
|
66
|
+
|
67
|
+
only = options[:only] ? [*options[:only]] : nil
|
68
|
+
|
69
|
+
# For each type belongs has_one, has_many etc find the operators
|
70
|
+
# and create headers, then for each record call those operators
|
71
|
+
operators = options[:with] || MethodDetail::supported_types_enum
|
72
|
+
|
73
|
+
CSV.open( (options[:filename] || filename), "w" ) do |csv|
|
74
|
+
|
75
|
+
csv.ar_to_headers( records, operators, options)
|
76
|
+
|
77
|
+
details_mgr = MethodDictionary.method_details_mgrs[klass]
|
78
|
+
|
79
|
+
records.each do |obj|
|
80
|
+
|
81
|
+
row = []
|
82
|
+
|
83
|
+
operators.each do |op_type|
|
84
|
+
|
85
|
+
operators_for_type = details_mgr.get_list(op_type)
|
86
|
+
|
87
|
+
next if(operators_for_type.nil? || operators_for_type.empty?)
|
88
|
+
|
89
|
+
operators_for_type.each do |md|
|
90
|
+
|
91
|
+
next if(only && !only.include?( md.name.to_sym ) )
|
92
|
+
|
93
|
+
if(MethodDetail.is_association_type?(op_type))
|
94
|
+
row << record_to_column( obj.send( md.operator )) # pack association into single column
|
95
|
+
else
|
96
|
+
row << escape_for_csv( obj.send( md.operator ) )
|
97
|
+
end
|
150
98
|
end
|
151
|
-
#end
|
152
99
|
end
|
153
|
-
|
154
|
-
csv <<
|
155
|
-
|
100
|
+
|
101
|
+
csv << row # next record
|
156
102
|
end
|
157
103
|
end
|
158
|
-
end
|
159
|
-
|
160
|
-
|
161
|
-
# Convert an AR instance to a single CSV column
|
162
|
-
|
163
|
-
def record_to_column(record)
|
164
|
-
|
165
|
-
csv_data = []
|
166
|
-
record.serializable_hash.each do |name, value|
|
167
|
-
value = 'nil' if value.nil?
|
168
|
-
text = value.to_s.gsub(@text_delim, escape_text_delim())
|
169
|
-
csv_data << "#{name.to_sym} => #{text}"
|
170
|
-
end
|
171
|
-
"#{csv_data.join(Delimiters::csv_delim)}"
|
172
|
-
end
|
173
|
-
|
174
|
-
|
175
|
-
# Convert an AR instance to a set of CSV columns
|
176
|
-
def record_to_csv(record, options = {})
|
177
|
-
csv_data = record.serializable_hash.values.collect { |value| escape(value) }
|
178
|
-
|
179
|
-
[*options[:methods]].each { |x| csv_data << escape(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
|
180
|
-
|
181
|
-
csv_data.join( Delimiters::csv_delim )
|
182
|
-
end
|
183
|
-
|
184
|
-
|
185
|
-
private
|
186
104
|
|
187
|
-
def escape(value)
|
188
|
-
text = value.to_s.gsub(@text_delim, escape_text_delim())
|
189
|
-
|
190
|
-
text = "#{@text_delim}#{text}#{@text_delim}" if(text.include?(Delimiters::csv_delim))
|
191
|
-
text
|
192
105
|
end
|
193
|
-
|
194
106
|
end
|
195
107
|
end
|
@@ -11,107 +11,120 @@
|
|
11
11
|
module DataShift
|
12
12
|
|
13
13
|
require 'exporter_base'
|
14
|
-
|
14
|
+
|
15
15
|
require 'excel'
|
16
16
|
|
17
17
|
class ExcelExporter < ExporterBase
|
18
|
-
|
18
|
+
|
19
19
|
include DataShift::Logging
|
20
|
-
|
20
|
+
include DataShift::ColumnPacker
|
21
|
+
|
21
22
|
def initialize(filename)
|
22
23
|
@filename = filename
|
23
24
|
end
|
24
25
|
|
25
26
|
# Create an Excel file from list of ActiveRecord objects
|
26
27
|
def export(export_records, options = {})
|
27
|
-
|
28
|
+
|
28
29
|
records = [*export_records]
|
29
|
-
|
30
|
-
puts records, records.inspect
|
31
|
-
|
30
|
+
|
32
31
|
unless(records && records.size > 0)
|
33
|
-
logger.warn("No objects supplied for export")
|
34
|
-
return
|
32
|
+
logger.warn("No objects supplied for export")
|
33
|
+
return
|
35
34
|
end
|
36
|
-
|
35
|
+
|
37
36
|
first = records[0]
|
38
|
-
|
37
|
+
|
38
|
+
logger.info("Exporting #{records.size} #{first.class} to Excel")
|
39
|
+
|
39
40
|
raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
|
40
|
-
|
41
|
-
|
41
|
+
|
42
|
+
|
42
43
|
raise ArgumentError.new('Please supply array of records to export') unless records.is_a? Array
|
43
44
|
|
44
45
|
excel = Excel.new
|
45
46
|
|
46
47
|
if(options[:sheet_name] )
|
47
|
-
excel.create_worksheet( :name => options[:sheet_name] )
|
48
|
+
excel.create_worksheet( :name => options[:sheet_name] )
|
48
49
|
else
|
49
50
|
excel.create_worksheet( :name => records.first.class.name )
|
50
51
|
end
|
51
|
-
|
52
|
+
|
52
53
|
excel.ar_to_headers(records)
|
53
|
-
|
54
|
+
|
54
55
|
excel.ar_to_xls(records)
|
55
|
-
|
56
|
+
|
56
57
|
excel.write( filename() )
|
57
58
|
end
|
58
|
-
|
59
|
-
# Create an Excel file from list of ActiveRecord objects
|
60
|
-
#
|
61
|
-
#
|
59
|
+
|
60
|
+
# Create an Excel file from list of ActiveRecord objects, includes relationships
|
61
|
+
#
|
62
|
+
# Options
|
63
|
+
#
|
64
|
+
# only Specify (as symbols) columns (assignments) to export from klass
|
65
|
+
# with: Specify which association types to export :with
|
66
|
+
# Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
|
67
|
+
# with_only Specify (as symbols) columns for association types to export
|
68
|
+
# sheet_name Else uses Class name
|
69
|
+
# json: Export association data in single column in JSON format
|
62
70
|
#
|
63
|
-
def export_with_associations(klass,
|
71
|
+
def export_with_associations(klass, records, options = {})
|
72
|
+
|
73
|
+
records = [*records]
|
74
|
+
|
75
|
+
only = options[:only] ? [*options[:only]] : nil
|
64
76
|
|
65
77
|
excel = Excel.new
|
66
78
|
|
67
79
|
if(options[:sheet_name] )
|
68
|
-
excel.create_worksheet( :name => options[:sheet_name] )
|
80
|
+
excel.create_worksheet( :name => options[:sheet_name] )
|
69
81
|
else
|
70
|
-
excel.create_worksheet( :name =>
|
82
|
+
excel.create_worksheet( :name => records.first.class.name )
|
71
83
|
end
|
72
|
-
|
84
|
+
|
73
85
|
MethodDictionary.find_operators( klass )
|
74
|
-
|
86
|
+
|
75
87
|
MethodDictionary.build_method_details( klass )
|
76
|
-
|
77
|
-
work_list = options[:with] || MethodDetail::supported_types_enum
|
78
|
-
|
79
|
-
headers = []
|
80
|
-
|
81
|
-
details_mgr = MethodDictionary.method_details_mgrs[klass]
|
82
|
-
|
83
|
-
data = []
|
88
|
+
|
84
89
|
# For each type belongs has_one, has_many etc find the operators
|
85
90
|
# and create headers, then for each record call those operators
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
items.each do |i|
|
97
|
-
data << i.send( md.operator )
|
98
|
-
end
|
99
|
-
|
100
|
-
end
|
101
|
-
|
102
|
-
excel.set_headers( headers )
|
103
|
-
|
104
|
-
row = 1
|
91
|
+
operators = options[:with] || MethodDetail::supported_types_enum
|
92
|
+
|
93
|
+
excel.ar_to_headers( records, operators, options)
|
94
|
+
|
95
|
+
details_mgr = MethodDictionary.method_details_mgrs[klass]
|
96
|
+
|
97
|
+
row = 1
|
98
|
+
|
99
|
+
records.each do |obj|
|
100
|
+
|
105
101
|
column = 0
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
102
|
+
|
103
|
+
[*operators].each do |op_type| # belongs_to, has_one, has_many etc
|
104
|
+
|
105
|
+
operators_for_type = details_mgr.get_list(op_type)
|
106
|
+
|
107
|
+
next if(operators_for_type.nil? || operators_for_type.empty?)
|
108
|
+
|
109
|
+
operators_for_type.each do |md| # actual associations on obj
|
110
|
+
|
111
|
+
next if(only && !only.include?( md.name.to_sym ) )
|
112
|
+
|
113
|
+
if(MethodDetail.is_association_type?(op_type))
|
114
|
+
excel[row, column] = record_to_column( obj.send( md.operator ), options ) # pack association into single column
|
115
|
+
else
|
116
|
+
excel[row, column] = obj.send( md.operator )
|
117
|
+
end
|
118
|
+
column += 1
|
119
|
+
end
|
110
120
|
end
|
111
|
-
|
112
|
-
|
121
|
+
|
122
|
+
row += 1
|
113
123
|
end
|
124
|
+
|
125
|
+
excel.write( filename() )
|
126
|
+
|
114
127
|
end
|
115
128
|
end # ExcelGenerator
|
116
|
-
|
129
|
+
|
117
130
|
end # DataShift
|