datashift 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/README.markdown +91 -55
  3. data/VERSION +1 -1
  4. data/datashift.gemspec +8 -23
  5. data/lib/applications/jexcel_file.rb +1 -2
  6. data/lib/datashift.rb +34 -15
  7. data/lib/datashift/column_packer.rb +98 -34
  8. data/lib/datashift/data_transforms.rb +83 -0
  9. data/lib/datashift/delimiters.rb +58 -10
  10. data/lib/datashift/excel_base.rb +123 -0
  11. data/lib/datashift/exceptions.rb +45 -7
  12. data/lib/datashift/load_object.rb +25 -0
  13. data/lib/datashift/mapping_service.rb +91 -0
  14. data/lib/datashift/method_detail.rb +40 -62
  15. data/lib/datashift/method_details_manager.rb +18 -2
  16. data/lib/datashift/method_dictionary.rb +27 -10
  17. data/lib/datashift/method_mapper.rb +49 -41
  18. data/lib/datashift/model_mapper.rb +42 -22
  19. data/lib/datashift/populator.rb +258 -143
  20. data/lib/datashift/thor_base.rb +38 -0
  21. data/lib/exporters/csv_exporter.rb +57 -145
  22. data/lib/exporters/excel_exporter.rb +73 -60
  23. data/lib/generators/csv_generator.rb +65 -5
  24. data/lib/generators/generator_base.rb +69 -3
  25. data/lib/generators/mapping_generator.rb +112 -0
  26. data/lib/helpers/core_ext/csv_file.rb +33 -0
  27. data/lib/loaders/csv_loader.rb +41 -39
  28. data/lib/loaders/excel_loader.rb +130 -116
  29. data/lib/loaders/loader_base.rb +190 -146
  30. data/lib/loaders/paperclip/attachment_loader.rb +4 -4
  31. data/lib/loaders/paperclip/datashift_paperclip.rb +5 -3
  32. data/lib/loaders/paperclip/image_loading.rb +9 -7
  33. data/lib/loaders/reporter.rb +17 -8
  34. data/lib/thor/export.thor +12 -13
  35. data/lib/thor/generate.thor +1 -9
  36. data/lib/thor/import.thor +13 -24
  37. data/lib/thor/mapping.thor +65 -0
  38. data/spec/Gemfile +13 -11
  39. data/spec/Gemfile.lock +98 -93
  40. data/spec/csv_exporter_spec.rb +104 -99
  41. data/spec/csv_generator_spec.rb +159 -0
  42. data/spec/csv_loader_spec.rb +197 -16
  43. data/spec/datashift_spec.rb +9 -0
  44. data/spec/excel_exporter_spec.rb +149 -58
  45. data/spec/excel_generator_spec.rb +35 -44
  46. data/spec/excel_loader_spec.rb +196 -178
  47. data/spec/excel_spec.rb +8 -5
  48. data/spec/loader_base_spec.rb +47 -7
  49. data/spec/mapping_spec.rb +117 -0
  50. data/spec/method_dictionary_spec.rb +24 -11
  51. data/spec/method_mapper_spec.rb +5 -7
  52. data/spec/model_mapper_spec.rb +41 -0
  53. data/spec/paperclip_loader_spec.rb +3 -6
  54. data/spec/populator_spec.rb +48 -14
  55. data/spec/spec_helper.rb +85 -73
  56. data/spec/thor_spec.rb +40 -5
  57. metadata +93 -86
  58. data/lib/applications/excel_base.rb +0 -63
@@ -13,127 +13,132 @@ require 'csv_exporter'
13
13
  describe 'CSV Exporter' do
14
14
 
15
15
  before(:all) do
16
-
17
- # load our test model definitions - Project etc
18
- require ifixture_file('test_model_defs')
19
-
20
- db_connect( 'test_file' ) # , test_memory, test_mysql
21
-
22
- # handle migration changes or reset of test DB
23
- migrate_up
24
-
25
- results_clear()
16
+ results_clear( "*.csv" )
26
17
  end
27
-
18
+
28
19
  before(:each) do
29
20
  DataShift::MethodDictionary.clear
30
21
  DataShift::MethodDictionary.find_operators( Project )
31
-
22
+
32
23
  db_clear() # todo read up about proper transactional fixtures
33
-
34
- Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
35
- Project.create( :value_as_string => 'Another Value as String', :value_as_boolean => false, :value_as_double => 12)
36
-
37
- end
38
-
39
- it "should be able to create a new CSV exporter" do
40
- exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
41
-
42
- exporter.should_not be_nil
43
24
  end
44
25
 
45
- it "should throw if not active record objects" do
46
- exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
47
-
48
- expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
49
- end
50
-
51
-
52
- it "should export collection of model objects to .xls file" do
53
-
54
- expect = result_file('project_export_spec.csv')
55
-
56
- exporter = DataShift::CsvExporter.new( expect )
57
-
58
- count = Project.count
59
-
60
- Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
61
-
62
- Project.count.should == count + 1
63
-
64
- exporter.export(Project.all)
65
-
66
- File.exists?(expect).should be_true
67
-
68
- puts "Can manually check file @ #{expect}"
69
-
70
- File.foreach(expect) {}
71
- count = $.
72
- count.should == Project.count + 1
73
- end
26
+ context 'simple project' do
74
27
 
75
- it "should handle bad params to export" do
28
+ before(:each) do
29
+ create( :project )
30
+ end
76
31
 
77
- expect = result_file('project_first_export_spec.csv')
32
+ it "should be able to create a new CSV exporter" do
33
+ exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
78
34
 
79
- exporter = DataShift::CsvExporter.new( expect )
80
-
81
- expect{ exporter.export(nil) }.not_to raise_error
35
+ expect(exporter).not_to be_nil
36
+ end
82
37
 
83
- expect{ exporter.export([]) }.not_to raise_error
84
-
85
- puts "Can manually check file @ #{expect}"
86
- end
87
-
88
- it "should export a model object to csv file" do
89
-
90
- expect = result_file('project_first_export_spec.csv')
91
-
92
- exporter = DataShift::CsvExporter.new( expect )
93
-
94
- exporter.export(Project.all[0])
95
-
96
- File.exists?(expect).should be_true
97
-
98
- puts "Can manually check file @ #{expect}"
99
- end
38
+ it "should throw if not active record objects" do
39
+ exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
100
40
 
101
- it "should export a model and result of method calls on it to csv file" do
41
+ expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
42
+ end
102
43
 
103
- expect = result_file('project_with_methods_export_spec.csv')
104
44
 
105
- exporter = DataShift::CsvExporter.new( expect )
106
-
107
- exporter.export(Project.all, {:methods => [:multiply]})
108
-
109
- File.exists?(expect).should be_true
110
-
111
- puts "Can manually check file @ #{expect}"
112
-
113
- File.foreach(expect) {}
114
- count = $.
115
- count.should == Project.count + 1
45
+ it "should export collection of model objects to .xls file" do
46
+
47
+ expected = result_file('exp_project_collection_spec.csv')
48
+
49
+ exporter = DataShift::CsvExporter.new( expected )
50
+
51
+ count = Project.count
52
+
53
+ Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
54
+
55
+ Project.count.should == count + 1
56
+
57
+ exporter.export(Project.all)
58
+
59
+ expect(File.exists?(expected)).to eq true
60
+
61
+ puts "Can manually check file @ #{expected}"
62
+
63
+ File.foreach(expected) {}
64
+ count = $.
65
+ count.should == Project.count + 1
66
+ end
67
+
68
+ it "should handle bad params to export" do
69
+
70
+ expected = result_file('project_first_export_spec.csv')
71
+
72
+ exporter = DataShift::CsvExporter.new( expected )
73
+
74
+ expect{ exporter.export(nil) }.not_to raise_error
75
+
76
+ expect{ exporter.export([]) }.not_to raise_error
77
+
78
+ puts "Can manually check file @ #{expected}"
79
+ end
80
+
81
+ it "should export a model object to csv file" do
82
+
83
+ expected = result_file('project_first_export_spec.csv')
84
+
85
+ exporter = DataShift::CsvExporter.new( expected )
86
+
87
+ exporter.export(Project.all[0])
88
+
89
+ expect(File.exists?(expected)).to eq true
90
+
91
+ puts "Can manually check file @ #{expected}"
92
+ end
93
+
94
+ it "should export a model and result of method calls on it to csv file" do
95
+
96
+ expected = result_file('project_with_methods_export_spec.csv')
97
+
98
+ exporter = DataShift::CsvExporter.new( expected )
99
+
100
+ exporter.export(Project.all, {:methods => [:multiply]})
101
+
102
+ expect(File.exists?(expected)).to eq true
103
+
104
+ puts "Can manually check file @ #{expected}"
105
+
106
+ File.foreach(expected) {}
107
+ count = $.
108
+ count.should == Project.count + 1
109
+ end
110
+
116
111
  end
117
-
118
- it "should export a model and associations to .xls file" do
119
112
 
120
- p = Project.create( :value_as_string => 'Value as String', :value_as_boolean => true, :value_as_double => 75.672)
113
+ it "should export a model and associations to csv" do
121
114
 
122
- p.milestones.create( :name => 'milestone_1', :cost => 23.45)
123
-
124
- expect= result_file('project_plus_assoc_export_spec.csv')
115
+ create( :project_user )
116
+ create_list(:project, 7)
125
117
 
126
- gen = DataShift::CsvExporter.new(expect)
118
+ expected = result_file('exp_project_plus_assoc_export_spec.csv')
127
119
 
128
- gen.export_with_associations(Project, Project.all)
120
+ gen = DataShift::CsvExporter.new(expected)
129
121
 
130
- File.exists?(expect).should be_true
122
+ items = Project.all
131
123
 
132
- File.foreach(expect) {}
124
+ gen.export_with_associations(Project, items)
125
+
126
+ File.foreach(expected) {}
133
127
  count = $.
134
- count.should == Project.count + 1
135
-
136
- end
128
+ count.should == items.size + 1
129
+
130
+ expect(File.exists?(expected)).to eq true
137
131
 
132
+ csv = CSV.read(expected)
133
+
134
+ expect(csv[0]).to include 'owner'
135
+ expect(csv[0]).to include 'user'
136
+
137
+ user_inx = csv[0].index 'user'
138
+
139
+ expect(user_inx).to be > -1
140
+
141
+ expect( csv[1][user_inx] ).to include 'mr'
142
+ end
138
143
 
139
144
  end
@@ -0,0 +1,159 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2011
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2011
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for Excel aspect of Active Record Loader
7
+ #
8
+ require File.dirname(__FILE__) + '/spec_helper'
9
+
10
+ require 'erb'
11
+ require 'csv_generator'
12
+
13
+ include DataShift
14
+
15
+ describe 'CSV Generator' do
16
+
17
+ before(:all) do
18
+ results_clear("*.csv")
19
+
20
+ @klazz = Project
21
+ @assoc_klazz = Category
22
+ end
23
+
24
+ before(:each) do
25
+ MethodDictionary.clear
26
+ MethodDictionary.find_operators( @klazz )
27
+ MethodDictionary.find_operators( @assoc_klazz )
28
+ end
29
+
30
+ it "should be able to create a new csv generator" do
31
+ generator = CsvGenerator.new( 'dummy.csv' )
32
+
33
+ generator.should_not be_nil
34
+ end
35
+
36
+ it "should generate template .csv file from model" do
37
+
38
+ expected = result_file('project_template_spec.csv')
39
+
40
+ gen = CsvGenerator.new( expected )
41
+
42
+ gen.generate(Project)
43
+
44
+ expect(File.exists?(expected)).to eq true
45
+
46
+ puts "Can manually check file @ #{expected}"
47
+
48
+ csv = CSV.read(expected)
49
+
50
+ headers = csv[0]
51
+
52
+ [ "title", "value_as_string", "value_as_text", "value_as_boolean", "value_as_datetime", "value_as_integer", "value_as_double"].each do |check|
53
+ headers.include?(check).should == true
54
+ end
55
+ end
56
+
57
+ # has_one :owner
58
+ # has_many :milestones
59
+ # has_many :loader_releases
60
+ # has_many :versions, :through => :loader_releases
61
+ # has_and_belongs_to_many :categories
62
+
63
+ it "should include all associations in template .csv file from model" do
64
+
65
+ expected = result_file('project_plus_assoc_template_spec.csv')
66
+
67
+ gen = CsvGenerator.new(expected)
68
+
69
+ gen.generate_with_associations(Project)
70
+
71
+ expect( File.exists?(expected)).to eq true
72
+
73
+ csv = CSV.read(expected)
74
+
75
+ headers = csv[0]
76
+
77
+ ["owner", "milestones", "loader_releases", "versions", "categories"].each do |check|
78
+ headers.include?(check).should == true
79
+ end
80
+ end
81
+
82
+
83
+ it "should enable us to exclude associations by type in template .csv file" do
84
+
85
+ expected = result_file('project_plus_some_assoc_template_spec.csv')
86
+
87
+ gen = CsvGenerator.new(expected)
88
+
89
+ options = {:exclude => :has_many }
90
+
91
+ gen.generate_with_associations(Project, options)
92
+
93
+ expect(File.exists?(expected)).to eq true #, "Failed to find expected result file #{expected}"
94
+
95
+ csv = CSV.read(expected)
96
+
97
+ headers = csv[0]
98
+
99
+ headers.include?('title').should == true
100
+ headers.include?('owner').should == true
101
+
102
+ ["milestones", "loader_releases", "versions", "categories"].each do |check|
103
+ headers.should_not include check
104
+ end
105
+
106
+ end
107
+
108
+
109
+ it "should enable us to exclude certain associations in template .csv file " do
110
+
111
+ expected = result_file('project_plus_some_assoc_template_spec.csv')
112
+
113
+ gen = CsvGenerator.new(expected)
114
+
115
+ options = {:remove => [:milestones, :versions] }
116
+
117
+ gen.generate_with_associations(Project, options)
118
+
119
+ expect(File.exists?(expected)).to eq true#, "Failed to find expected result file #{expected}"
120
+
121
+ csv = CSV.read(expected)
122
+
123
+ headers = csv[0]
124
+
125
+ ["title", "loader_releases", "owner", "categories"].each do |check|
126
+ headers.should include check
127
+ end
128
+
129
+
130
+ ["milestones", "versions", ].each do |check|
131
+ headers.should_not include check
132
+ end
133
+
134
+ end
135
+
136
+
137
+ it "should enable us to remove standard rails feilds from template .csv file " do
138
+
139
+ expected = result_file('project_plus_some_assoc_template_spec.csv')
140
+
141
+ gen = CsvGenerator.new(expected)
142
+
143
+ options = {:remove_rails => true}
144
+
145
+ gen.generate_with_associations(Project, options)
146
+
147
+ expect(File.exists?(expected)).to eq true#, "Failed to find expected result file #{expected}"
148
+
149
+ csv = CSV.read(expected)
150
+
151
+ headers = csv[0]
152
+
153
+ ["id", "updated_at", "created_at"].each do |check|
154
+ headers.should_not include check
155
+ end
156
+
157
+
158
+ end
159
+ end
@@ -3,29 +3,210 @@
3
3
  # Date :: Aug 2011
4
4
  # License:: MIT
5
5
  #
6
- # Details:: Specs for Excel aspect of Active Record Loader
6
+ # Details:: Specs for CSV aspect of Active Record Loader
7
7
  #
8
8
  require File.dirname(__FILE__) + '/spec_helper'
9
9
 
10
10
  require 'erb'
11
- require 'excel_loader'
11
+ require 'csv_loader'
12
12
 
13
- describe 'CSV Loader' do
13
+ include DataShift
14
14
 
15
- before(:all) do
16
-
17
- # load our test model definitions - Project etc
18
- require ifixture_file('test_model_defs')
19
-
20
- db_connect( 'test_file' ) # , test_memory, test_mysql
21
- migrate_up
22
- @klazz = Project
15
+ describe 'Csv Loader' do
16
+
17
+ before(:each) do
18
+ DataShift::MethodDictionary.clear
19
+
20
+ @method_mapper = DataShift::MethodMapper.new
23
21
  end
24
-
22
+
23
+
25
24
  before(:each) do
26
- MethodDictionary.clear
27
- MethodDictionary.find_operators( @klazz )
28
- MethodDictionary.find_operators( @assoc_klazz )
25
+
26
+ %w{category_001 category_002 category_003 category_004 category_005}.each do |cat|
27
+ Category.find_or_create_by(reference: cat)
28
+ end
29
+
30
+
31
+ end
32
+
33
+ it "should be able to create a new csv loader and load object" do
34
+ loader = CsvLoader.new(Project)
35
+
36
+ loader.load_object.should_not be_nil
37
+ loader.load_object.should be_is_a(Project)
38
+ expect(loader.load_object.new_record?).to eq true
39
+ end
40
+
41
+ it "should process a simple .csv spreedsheet" do
42
+
43
+ loader = CsvLoader.new(Project)
44
+
45
+ count = Project.count
46
+ loader.perform_load ifixture_file('csv/SimpleProjects.csv')
47
+
48
+ loader.loaded_count.should == (Project.count - count)
49
+ end
50
+
51
+ it "should process multiple associationss from single column" do
52
+
53
+ DataShift::MethodDictionary.find_operators( Category )
54
+
55
+ DataShift::MethodDictionary.build_method_details( Category )
56
+
57
+ Project.find_by_title('001').should be_nil
58
+ count = Project.count
59
+
60
+ loader = CsvLoader.new(Project)
61
+
62
+ loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
63
+
64
+ loader.loaded_count.should be > 3
65
+ loader.loaded_count.should == (Project.count - count)
66
+
67
+ {'001' => 2, '002' => 1, '003' => 3, '099' => 0 }.each do|title, expected|
68
+ project = Project.find_by_title(title)
69
+
70
+ project.should_not be_nil
71
+ #puts "#{project.inspect} [#{project.categories.size}]"
72
+
73
+ expect(project.categories.size).to eq expected
74
+ end
75
+ end
76
+
77
+ it "should process multiple associations in csv file" do
78
+
79
+ loader = CsvLoader.new(Project)
80
+
81
+ count = Project.count
82
+ loader.perform_load( ifixture_file('csv/ProjectsMultiCategories.csv' ))
83
+
84
+ loader.loaded_count.should == (Project.count - count)
85
+
86
+ {'004' => 3, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
87
+ project = Project.find_by_title(title)
88
+
89
+ project.should_not be_nil
90
+
91
+ expect(project.categories.size).to eq expected
92
+ end
93
+
94
+ end
95
+
96
+ it "should process multiple associations with lookup specified in column from excel spreedsheet" do
97
+
98
+ loader = CsvLoader.new(Project)
99
+
100
+ count = Project.count
101
+ loader.perform_load( ifixture_file('csv/ProjectsMultiCategoriesHeaderLookup.csv'))
102
+
103
+ expect(loader.loaded_count).to eq (Project.count - count)
104
+ loader.loaded_count.should > 3
105
+
106
+ {'004' => 4, '005' => 1, '006' => 0, '007' => 1 }.each do|title, expected|
107
+ project = Project.find_by_title(title)
108
+
109
+ project.should_not be_nil
110
+
111
+ expect(project.categories.size).to eq expected
112
+ end
113
+
29
114
  end
30
115
 
31
- end
116
+ it "should process excel spreedsheet with extra undefined columns" do
117
+ loader = CsvLoader.new(Project)
118
+ lambda {loader.perform_load( ifixture_file('csv/BadAssociationName.csv') ) }.should_not raise_error
119
+ end
120
+
121
+ it "should NOT process excel spreedsheet with extra undefined columns when strict mode" do
122
+ loader = CsvLoader.new(Project)
123
+ expect {loader.perform_load( ifixture_file('csv/BadAssociationName.csv'), :strict => true)}.to raise_error(MappingDefinitionError)
124
+ end
125
+
126
+ it "should raise an error when mandatory columns missing" do
127
+ loader = CsvLoader.new(Project)
128
+ expect {loader.perform_load(ifixture_file('csv/ProjectsMultiCategories.csv'), :mandatory => ['not_an_option', 'must_be_there'])}.to raise_error(DataShift::MissingMandatoryError)
129
+ end
130
+
131
+ it "should provide facility to set default values", :focus => true do
132
+ loader = CsvLoader.new(Project)
133
+
134
+ populator = loader.populator
135
+
136
+ populator.set_default_value('value_as_string', 'some default text' )
137
+ populator.set_default_value('value_as_double', 45.467 )
138
+ populator.set_default_value('value_as_boolean', true )
139
+
140
+ texpected = Time.now.to_s(:db)
141
+
142
+ populator.set_default_value('value_as_datetime', texpected )
143
+
144
+ #value_as_string Value as Text value as datetime value_as_boolean value_as_double category
145
+
146
+ loader.perform_load(ifixture_file('csv/ProjectsSingleCategories.csv'))
147
+
148
+ p = Project.find_by_title( '099' )
149
+
150
+ p.should_not be_nil
151
+
152
+ p.value_as_string.should == 'some default text'
153
+ p.value_as_double.should == 45.467
154
+ p.value_as_boolean.should == true
155
+ p.value_as_datetime.to_s(:db).should == texpected
156
+
157
+ # expected: "2012-09-17 10:00:52"
158
+ # got: Mon Sep 17 10:00:52 +0100 2012 (using ==)
159
+
160
+ p_no_defs = Project.first
161
+
162
+ p_no_defs.value_as_string.should_not == 'some default text'
163
+ p_no_defs.value_as_double.should_not == 45.467
164
+ p_no_defs.value_as_datetime.should_not == texpected
165
+
166
+ end
167
+
168
+ it "should provide facility to set pre and post fix values" do
169
+ loader = CsvLoader.new(Project)
170
+
171
+ loader.populator.set_prefix('value_as_string', 'myprefix' )
172
+ loader.populator.set_postfix('value_as_string', 'my post fix' )
173
+
174
+ #value_as_string Value as Text value as datetime value_as_boolean value_as_double category
175
+
176
+ loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv'))
177
+
178
+ p = Project.find_by_title( '001' )
179
+
180
+ p.should_not be_nil
181
+
182
+ p.value_as_string.should == 'myprefixDemo stringmy post fix'
183
+ end
184
+
185
+ it "should provide facility to set default values via YAML configuration" do
186
+ loader = CsvLoader.new(Project)
187
+
188
+ loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
189
+
190
+ loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
191
+
192
+ p = Project.find_by_title( '099' )
193
+
194
+ p.should_not be_nil
195
+
196
+ p.value_as_string.should == "Default Project Value"
197
+ end
198
+
199
+
200
+ it "should provide facility to over ride values via YAML configuration", :fail => true do
201
+ loader = CsvLoader.new(Project)
202
+
203
+ loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
204
+
205
+
206
+ loader.perform_load( ifixture_file('csv/ProjectsSingleCategories.csv') )
207
+
208
+ Project.all.each {|p| p.value_as_double.should == 99.23546 }
209
+ end
210
+
211
+
212
+ end