datashift 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/README.markdown +91 -55
  3. data/VERSION +1 -1
  4. data/datashift.gemspec +8 -23
  5. data/lib/applications/jexcel_file.rb +1 -2
  6. data/lib/datashift.rb +34 -15
  7. data/lib/datashift/column_packer.rb +98 -34
  8. data/lib/datashift/data_transforms.rb +83 -0
  9. data/lib/datashift/delimiters.rb +58 -10
  10. data/lib/datashift/excel_base.rb +123 -0
  11. data/lib/datashift/exceptions.rb +45 -7
  12. data/lib/datashift/load_object.rb +25 -0
  13. data/lib/datashift/mapping_service.rb +91 -0
  14. data/lib/datashift/method_detail.rb +40 -62
  15. data/lib/datashift/method_details_manager.rb +18 -2
  16. data/lib/datashift/method_dictionary.rb +27 -10
  17. data/lib/datashift/method_mapper.rb +49 -41
  18. data/lib/datashift/model_mapper.rb +42 -22
  19. data/lib/datashift/populator.rb +258 -143
  20. data/lib/datashift/thor_base.rb +38 -0
  21. data/lib/exporters/csv_exporter.rb +57 -145
  22. data/lib/exporters/excel_exporter.rb +73 -60
  23. data/lib/generators/csv_generator.rb +65 -5
  24. data/lib/generators/generator_base.rb +69 -3
  25. data/lib/generators/mapping_generator.rb +112 -0
  26. data/lib/helpers/core_ext/csv_file.rb +33 -0
  27. data/lib/loaders/csv_loader.rb +41 -39
  28. data/lib/loaders/excel_loader.rb +130 -116
  29. data/lib/loaders/loader_base.rb +190 -146
  30. data/lib/loaders/paperclip/attachment_loader.rb +4 -4
  31. data/lib/loaders/paperclip/datashift_paperclip.rb +5 -3
  32. data/lib/loaders/paperclip/image_loading.rb +9 -7
  33. data/lib/loaders/reporter.rb +17 -8
  34. data/lib/thor/export.thor +12 -13
  35. data/lib/thor/generate.thor +1 -9
  36. data/lib/thor/import.thor +13 -24
  37. data/lib/thor/mapping.thor +65 -0
  38. data/spec/Gemfile +13 -11
  39. data/spec/Gemfile.lock +98 -93
  40. data/spec/csv_exporter_spec.rb +104 -99
  41. data/spec/csv_generator_spec.rb +159 -0
  42. data/spec/csv_loader_spec.rb +197 -16
  43. data/spec/datashift_spec.rb +9 -0
  44. data/spec/excel_exporter_spec.rb +149 -58
  45. data/spec/excel_generator_spec.rb +35 -44
  46. data/spec/excel_loader_spec.rb +196 -178
  47. data/spec/excel_spec.rb +8 -5
  48. data/spec/loader_base_spec.rb +47 -7
  49. data/spec/mapping_spec.rb +117 -0
  50. data/spec/method_dictionary_spec.rb +24 -11
  51. data/spec/method_mapper_spec.rb +5 -7
  52. data/spec/model_mapper_spec.rb +41 -0
  53. data/spec/paperclip_loader_spec.rb +3 -6
  54. data/spec/populator_spec.rb +48 -14
  55. data/spec/spec_helper.rb +85 -73
  56. data/spec/thor_spec.rb +40 -5
  57. metadata +93 -86
  58. data/lib/applications/excel_base.rb +0 -63
@@ -0,0 +1,38 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2012
2
+ # Author :: Tom Statter
3
+ # Date :: Dec 2014
4
+ # License:: MIT.
5
+ #
6
+ # Note, not DataShift, case sensitive, create namespace for command line : datashift
7
+ require 'thor'
8
+
9
+ module DataShift
10
+
11
+ class DSThorBase < Thor
12
+
13
+ include DataShift::Logging
14
+
15
+ no_commands do
16
+
17
+ def start_connections()
18
+
19
+ # TODO - We're assuming run from a rails app/top level dir...
20
+
21
+ if(File.exists?(File.expand_path('config/environment.rb')))
22
+ begin
23
+ require File.expand_path('config/environment.rb')
24
+ rescue => e
25
+ logger.error("Failed to initialise ActiveRecord : #{e.message}")
26
+ raise ConnectionError.new("No config/environment.rb found - cannot initialise ActiveRecord")
27
+ end
28
+
29
+ else
30
+ raise PathError.new("No config/environment.rb found - cannot initialise ActiveRecord")
31
+ # TODO make this more robust ? e.g what about when using active record but not in Rails app, Sinatra etc
32
+ end
33
+ end
34
+ end
35
+
36
+ end
37
+
38
+ end
@@ -7,27 +7,19 @@
7
7
  #
8
8
  #
9
9
  require 'exporter_base'
10
- require 'csv'
10
+ require 'csv_file'
11
11
 
12
12
  module DataShift
13
13
 
14
14
  class CsvExporter < ExporterBase
15
-
15
+
16
16
  include DataShift::Logging
17
-
18
- attr_accessor :text_delim
19
-
17
+ include DataShift::ColumnPacker
18
+
20
19
  def initialize(filename)
21
20
  super(filename)
22
- @text_delim = "\'"
23
21
  end
24
22
 
25
- # Return opposite of text delim - "hello, 'barry'" => '"hello, "barry""'
26
- def escape_text_delim
27
- return '"' if @text_delim == "\'"
28
- "\'"
29
- end
30
-
31
23
  # Create CSV file from set of ActiveRecord objects
32
24
  # Options :
33
25
  # => :filename
@@ -35,161 +27,81 @@ module DataShift
35
27
  # => ::methods => List of methods to additionally call on each record
36
28
  #
37
29
  def export(export_records, options = {})
38
-
30
+
39
31
  records = [*export_records]
40
-
41
-
42
- puts records, records.inspect
43
-
32
+
44
33
  unless(records && records.size > 0)
45
- logger.warn("No objects supplied for export")
46
- return
34
+ logger.warn("No objects supplied for export")
35
+ return
47
36
  end
48
-
37
+
49
38
  first = records[0]
50
-
39
+
51
40
  raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
52
-
53
- f = options[:filename] || filename()
54
-
55
- @text_delim = options[:text_delim] if(options[:text_delim])
56
-
57
- File.open(f, "w") do |csv|
58
-
59
- headers = first.serializable_hash.keys
60
-
61
- [*options[:methods]].each do |c| headers << c if(first.respond_to?(c)) end if(options[:methods])
62
-
63
- csv << headers.join(Delimiters::csv_delim) << Delimiters::eol
64
-
41
+
42
+ Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
43
+
44
+ CSV.open( (options[:filename] || filename), "w" ) do |csv|
45
+
46
+ csv.ar_to_headers( records )
47
+
65
48
  records.each do |r|
66
49
  next unless(r.is_a?(ActiveRecord::Base))
67
-
68
- csv << record_to_csv(r, options) << Delimiters::eol
50
+ csv.ar_to_csv(r, options)
69
51
  end
70
52
  end
71
53
  end
72
-
54
+
73
55
  # Create an Excel file from list of ActiveRecord objects
74
56
  # Specify which associations to export via :with or :exclude
75
57
  # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
76
58
  #
77
59
  def export_with_associations(klass, records, options = {})
78
-
79
- f = options[:filename] || filename()
80
-
81
- @text_delim = options[:text_delim] if(options[:text_delim])
82
-
60
+
61
+ Delimiters.text_delim = options[:text_delim] if(options[:text_delim])
62
+
83
63
  MethodDictionary.find_operators( klass )
84
-
85
- # builds all possible operators
64
+
86
65
  MethodDictionary.build_method_details( klass )
87
-
88
- work_list = options[:with] ? Set(options[:with]) : MethodDetail::supported_types_enum
89
-
90
- assoc_work_list = work_list.dup
91
-
92
- details_mgr = MethodDictionary.method_details_mgrs[klass]
93
-
94
- headers, assoc_operators, assignments = [], [], []
95
-
96
- # headers
97
- if(work_list.include?(:assignment))
98
- assignments << details_mgr.get_list(:assignment).collect( &:operator)
99
- assoc_work_list.delete :assignment
100
- end
101
-
102
- headers << assignments.flatten!
103
- # based on users requested list ... belongs_to has_one, has_many etc ... select only those operators
104
- assoc_operators = assoc_work_list.collect do |op_type|
105
- details_mgr.get_list(op_type).collect(&:operator).flatten
106
- end
107
-
108
- assoc_operators.flatten!
109
-
110
- File.open(f, "w") do |csv|
111
-
112
- csv << headers.join(Delimiters::csv_delim)
113
-
114
- csv << Delimiters::csv_delim << assoc_operators.join(Delimiters::csv_delim) unless(assoc_operators.empty?)
115
-
116
- csv << Delimiters::eol
117
-
118
- records.each do |r|
119
- next unless(r.is_a?(ActiveRecord::Base))
120
-
121
- csv_columns = []
122
- # need to make sure order matches headers
123
- # could look at collection headers via serializable hash.keys and then use
124
- # csv << record_to_csv(r) ??
125
- assignments.each {|x| csv << escape(r.send(x)) << Delimiters::csv_delim }
126
-
127
- # done records basic attributes now deal with associations
128
-
129
- #assoc_work_list.each do |op_type|
130
- # details_mgr.get_operators(op_type).each do |operator|
131
- assoc_operators.each do |operator|
132
- assoc_object = r.send(operator)
133
-
134
- if(assoc_object.is_a?ActiveRecord::Base)
135
- column_text = record_to_column(assoc_object) # belongs_to or has_one
136
-
137
- # TODO -ColumnPacker class shared between excel/csv
138
-
139
- csv << "#{@text_delim}#{column_text}#{@text_delim}" << Delimiters::csv_delim
140
- #csv << record_to_csv(r)
141
-
142
- elsif(assoc_object.is_a? Array)
143
- items_to_s = assoc_object.collect {|x| record_to_column(x) }
144
-
145
- # create a single column
146
- csv << "#{@text_delim}#{items_to_s.join(Delimiters::multi_assoc_delim)}#{@text_delim}" << Delimiters::csv_delim
147
-
148
- else
149
- csv << Delimiters::csv_delim
66
+
67
+ only = options[:only] ? [*options[:only]] : nil
68
+
69
+ # For each type belongs has_one, has_many etc find the operators
70
+ # and create headers, then for each record call those operators
71
+ operators = options[:with] || MethodDetail::supported_types_enum
72
+
73
+ CSV.open( (options[:filename] || filename), "w" ) do |csv|
74
+
75
+ csv.ar_to_headers( records, operators, options)
76
+
77
+ details_mgr = MethodDictionary.method_details_mgrs[klass]
78
+
79
+ records.each do |obj|
80
+
81
+ row = []
82
+
83
+ operators.each do |op_type|
84
+
85
+ operators_for_type = details_mgr.get_list(op_type)
86
+
87
+ next if(operators_for_type.nil? || operators_for_type.empty?)
88
+
89
+ operators_for_type.each do |md|
90
+
91
+ next if(only && !only.include?( md.name.to_sym ) )
92
+
93
+ if(MethodDetail.is_association_type?(op_type))
94
+ row << record_to_column( obj.send( md.operator )) # pack association into single column
95
+ else
96
+ row << escape_for_csv( obj.send( md.operator ) )
97
+ end
150
98
  end
151
- #end
152
99
  end
153
-
154
- csv << Delimiters::eol # next record
155
-
100
+
101
+ csv << row # next record
156
102
  end
157
103
  end
158
- end
159
-
160
-
161
- # Convert an AR instance to a single CSV column
162
-
163
- def record_to_column(record)
164
-
165
- csv_data = []
166
- record.serializable_hash.each do |name, value|
167
- value = 'nil' if value.nil?
168
- text = value.to_s.gsub(@text_delim, escape_text_delim())
169
- csv_data << "#{name.to_sym} => #{text}"
170
- end
171
- "#{csv_data.join(Delimiters::csv_delim)}"
172
- end
173
-
174
-
175
- # Convert an AR instance to a set of CSV columns
176
- def record_to_csv(record, options = {})
177
- csv_data = record.serializable_hash.values.collect { |value| escape(value) }
178
-
179
- [*options[:methods]].each { |x| csv_data << escape(record.send(x)) if(record.respond_to?(x)) } if(options[:methods])
180
-
181
- csv_data.join( Delimiters::csv_delim )
182
- end
183
-
184
-
185
- private
186
104
 
187
- def escape(value)
188
- text = value.to_s.gsub(@text_delim, escape_text_delim())
189
-
190
- text = "#{@text_delim}#{text}#{@text_delim}" if(text.include?(Delimiters::csv_delim))
191
- text
192
105
  end
193
-
194
106
  end
195
107
  end
@@ -11,107 +11,120 @@
11
11
  module DataShift
12
12
 
13
13
  require 'exporter_base'
14
-
14
+
15
15
  require 'excel'
16
16
 
17
17
  class ExcelExporter < ExporterBase
18
-
18
+
19
19
  include DataShift::Logging
20
-
20
+ include DataShift::ColumnPacker
21
+
21
22
  def initialize(filename)
22
23
  @filename = filename
23
24
  end
24
25
 
25
26
  # Create an Excel file from list of ActiveRecord objects
26
27
  def export(export_records, options = {})
27
-
28
+
28
29
  records = [*export_records]
29
-
30
- puts records, records.inspect
31
-
30
+
32
31
  unless(records && records.size > 0)
33
- logger.warn("No objects supplied for export")
34
- return
32
+ logger.warn("No objects supplied for export")
33
+ return
35
34
  end
36
-
35
+
37
36
  first = records[0]
38
-
37
+
38
+ logger.info("Exporting #{records.size} #{first.class} to Excel")
39
+
39
40
  raise ArgumentError.new('Please supply set of ActiveRecord objects to export') unless(first.is_a?(ActiveRecord::Base))
40
-
41
-
41
+
42
+
42
43
  raise ArgumentError.new('Please supply array of records to export') unless records.is_a? Array
43
44
 
44
45
  excel = Excel.new
45
46
 
46
47
  if(options[:sheet_name] )
47
- excel.create_worksheet( :name => options[:sheet_name] )
48
+ excel.create_worksheet( :name => options[:sheet_name] )
48
49
  else
49
50
  excel.create_worksheet( :name => records.first.class.name )
50
51
  end
51
-
52
+
52
53
  excel.ar_to_headers(records)
53
-
54
+
54
55
  excel.ar_to_xls(records)
55
-
56
+
56
57
  excel.write( filename() )
57
58
  end
58
-
59
- # Create an Excel file from list of ActiveRecord objects
60
- # Specify which associations to export via :with or :exclude
61
- # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
59
+
60
+ # Create an Excel file from list of ActiveRecord objects, includes relationships
61
+ #
62
+ # Options
63
+ #
64
+ # only Specify (as symbols) columns (assignments) to export from klass
65
+ # with: Specify which association types to export :with
66
+ # Possible values are : [:assignment, :belongs_to, :has_one, :has_many]
67
+ # with_only Specify (as symbols) columns for association types to export
68
+ # sheet_name Else uses Class name
69
+ # json: Export association data in single column in JSON format
62
70
  #
63
- def export_with_associations(klass, items, options = {})
71
+ def export_with_associations(klass, records, options = {})
72
+
73
+ records = [*records]
74
+
75
+ only = options[:only] ? [*options[:only]] : nil
64
76
 
65
77
  excel = Excel.new
66
78
 
67
79
  if(options[:sheet_name] )
68
- excel.create_worksheet( :name => options[:sheet_name] )
80
+ excel.create_worksheet( :name => options[:sheet_name] )
69
81
  else
70
- excel.create_worksheet( :name => items.first.class.name )
82
+ excel.create_worksheet( :name => records.first.class.name )
71
83
  end
72
-
84
+
73
85
  MethodDictionary.find_operators( klass )
74
-
86
+
75
87
  MethodDictionary.build_method_details( klass )
76
-
77
- work_list = options[:with] || MethodDetail::supported_types_enum
78
-
79
- headers = []
80
-
81
- details_mgr = MethodDictionary.method_details_mgrs[klass]
82
-
83
- data = []
88
+
84
89
  # For each type belongs has_one, has_many etc find the operators
85
90
  # and create headers, then for each record call those operators
86
- work_list.each do |op_type|
87
-
88
- list_for_class_and_op = details_mgr.get_list(op_type)
89
-
90
- next if(list_for_class_and_op.nil? || list_for_class_and_op.empty?)
91
-
92
- # method_details = MethodDictionary.send("#{mdtype}_for", klass)
93
-
94
- list_for_class_and_op.each do |md|
95
- headers << "#{md.operator}"
96
- items.each do |i|
97
- data << i.send( md.operator )
98
- end
99
-
100
- end
101
-
102
- excel.set_headers( headers )
103
-
104
- row = 1
91
+ operators = options[:with] || MethodDetail::supported_types_enum
92
+
93
+ excel.ar_to_headers( records, operators, options)
94
+
95
+ details_mgr = MethodDictionary.method_details_mgrs[klass]
96
+
97
+ row = 1
98
+
99
+ records.each do |obj|
100
+
105
101
  column = 0
106
-
107
- items.each do |row_of_data|
108
- excel.ar_to_xls_row(row, column, row_of_data)
109
- row += 1
102
+
103
+ [*operators].each do |op_type| # belongs_to, has_one, has_many etc
104
+
105
+ operators_for_type = details_mgr.get_list(op_type)
106
+
107
+ next if(operators_for_type.nil? || operators_for_type.empty?)
108
+
109
+ operators_for_type.each do |md| # actual associations on obj
110
+
111
+ next if(only && !only.include?( md.name.to_sym ) )
112
+
113
+ if(MethodDetail.is_association_type?(op_type))
114
+ excel[row, column] = record_to_column( obj.send( md.operator ), options ) # pack association into single column
115
+ else
116
+ excel[row, column] = obj.send( md.operator )
117
+ end
118
+ column += 1
119
+ end
110
120
  end
111
-
112
- excel.write( filename() )
121
+
122
+ row += 1
113
123
  end
124
+
125
+ excel.write( filename() )
126
+
114
127
  end
115
128
  end # ExcelGenerator
116
-
129
+
117
130
  end # DataShift