datashift 0.15.0 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +7 -0
  2. data/README.markdown +91 -55
  3. data/VERSION +1 -1
  4. data/datashift.gemspec +8 -23
  5. data/lib/applications/jexcel_file.rb +1 -2
  6. data/lib/datashift.rb +34 -15
  7. data/lib/datashift/column_packer.rb +98 -34
  8. data/lib/datashift/data_transforms.rb +83 -0
  9. data/lib/datashift/delimiters.rb +58 -10
  10. data/lib/datashift/excel_base.rb +123 -0
  11. data/lib/datashift/exceptions.rb +45 -7
  12. data/lib/datashift/load_object.rb +25 -0
  13. data/lib/datashift/mapping_service.rb +91 -0
  14. data/lib/datashift/method_detail.rb +40 -62
  15. data/lib/datashift/method_details_manager.rb +18 -2
  16. data/lib/datashift/method_dictionary.rb +27 -10
  17. data/lib/datashift/method_mapper.rb +49 -41
  18. data/lib/datashift/model_mapper.rb +42 -22
  19. data/lib/datashift/populator.rb +258 -143
  20. data/lib/datashift/thor_base.rb +38 -0
  21. data/lib/exporters/csv_exporter.rb +57 -145
  22. data/lib/exporters/excel_exporter.rb +73 -60
  23. data/lib/generators/csv_generator.rb +65 -5
  24. data/lib/generators/generator_base.rb +69 -3
  25. data/lib/generators/mapping_generator.rb +112 -0
  26. data/lib/helpers/core_ext/csv_file.rb +33 -0
  27. data/lib/loaders/csv_loader.rb +41 -39
  28. data/lib/loaders/excel_loader.rb +130 -116
  29. data/lib/loaders/loader_base.rb +190 -146
  30. data/lib/loaders/paperclip/attachment_loader.rb +4 -4
  31. data/lib/loaders/paperclip/datashift_paperclip.rb +5 -3
  32. data/lib/loaders/paperclip/image_loading.rb +9 -7
  33. data/lib/loaders/reporter.rb +17 -8
  34. data/lib/thor/export.thor +12 -13
  35. data/lib/thor/generate.thor +1 -9
  36. data/lib/thor/import.thor +13 -24
  37. data/lib/thor/mapping.thor +65 -0
  38. data/spec/Gemfile +13 -11
  39. data/spec/Gemfile.lock +98 -93
  40. data/spec/csv_exporter_spec.rb +104 -99
  41. data/spec/csv_generator_spec.rb +159 -0
  42. data/spec/csv_loader_spec.rb +197 -16
  43. data/spec/datashift_spec.rb +9 -0
  44. data/spec/excel_exporter_spec.rb +149 -58
  45. data/spec/excel_generator_spec.rb +35 -44
  46. data/spec/excel_loader_spec.rb +196 -178
  47. data/spec/excel_spec.rb +8 -5
  48. data/spec/loader_base_spec.rb +47 -7
  49. data/spec/mapping_spec.rb +117 -0
  50. data/spec/method_dictionary_spec.rb +24 -11
  51. data/spec/method_mapper_spec.rb +5 -7
  52. data/spec/model_mapper_spec.rb +41 -0
  53. data/spec/paperclip_loader_spec.rb +3 -6
  54. data/spec/populator_spec.rb +48 -14
  55. data/spec/spec_helper.rb +85 -73
  56. data/spec/thor_spec.rb +40 -5
  57. metadata +93 -86
  58. data/lib/applications/excel_base.rb +0 -63
@@ -76,7 +76,7 @@ describe 'Excel Proxy' do
76
76
  @excel.create_worksheet( :name => "autechre")
77
77
  @excel.create_worksheet( :name => "swarms")
78
78
 
79
- @excel.worksheets.should have_exactly(4).items
79
+ expect(@excel.worksheets.size).to eq 4
80
80
  end
81
81
 
82
82
  it "can access a worksheet by index" do
@@ -175,9 +175,12 @@ describe 'Excel Proxy' do
175
175
  end
176
176
 
177
177
 
178
- it "can support bools" do
179
- pending("reading back value sometimes returns "" when cell was set to false")
180
- end
178
+ #it "can support bools" do
179
+ # pending "reading back value sometimes returns "" when cell was set to false"
180
+
181
+ # sheet = @excel.create_worksheet
182
+
183
+ #end
181
184
 
182
185
  it "can write an Excel file" do
183
186
  @excel = Excel.new
@@ -193,7 +196,7 @@ describe 'Excel Proxy' do
193
196
 
194
197
  @excel.write( expected )
195
198
 
196
- File.exists?(expected).should be_true
199
+ expect(File.exists?(expected)).to eq true
197
200
 
198
201
  end
199
202
  end
@@ -11,18 +11,16 @@ require 'erb'
11
11
 
12
12
  describe 'LoaderBase' do
13
13
 
14
- include_context "ActiveRecordTestModelsConnected"
15
-
16
- include_context "ClearAndPopulateProject"
17
-
18
14
  before(:each) do
19
15
  @loader = DataShift::LoaderBase.new(Project)
20
16
  end
17
+
18
+ let(:loader) {DataShift::LoaderBase.new(Project) }
21
19
 
22
20
  it "should be able to create a new loader and load object" do
23
21
  @loader.load_object.should_not be_nil
24
22
  @loader.load_object.should be_is_a(Project)
25
- @loader.load_object.new_record?.should be_true
23
+ expect(@loader.load_object.new_record?).to eq true
26
24
  end
27
25
 
28
26
 
@@ -33,7 +31,7 @@ describe 'LoaderBase' do
33
31
 
34
32
  @loader.find_and_process(column, row)
35
33
 
36
- @loader.load_object.errors.should have_exactly(0).items
34
+ expect(@loader.load_object.errors.size).to eq 0
37
35
  end
38
36
 
39
37
  it "should process a string field against an assigment method detail" do
@@ -114,7 +112,7 @@ describe 'LoaderBase' do
114
112
  it "should be able to mark a load attempt as a failure" do
115
113
 
116
114
  failed_count = @loader.failed_count
117
- @loader.load_object.new_record?.should be_true
115
+ expect(@loader.load_object.new_record?).to eq true
118
116
 
119
117
  @loader.load_object.save!
120
118
 
@@ -122,5 +120,47 @@ describe 'LoaderBase' do
122
120
 
123
121
  @loader.failed_count.should == failed_count + 1
124
122
  end
123
+
124
+
125
+ it "should be able to set a plain default value" do
126
+
127
+ loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
128
+
129
+ loader.process_defaults
130
+
131
+ expect(loader.load_object.value_as_string).to eq "Default Project Value"
132
+
133
+ end
134
+
135
+ it "should be able to set an association default value" do
136
+
137
+ skip "pending more work on Populator to make this advanced lookup style work"
138
+
139
+ loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
140
+
141
+ loader.process_defaults
142
+
143
+ expect(loader.load_object.categories.first.reference).to eq "category_002"
144
+
145
+ end
146
+
147
+
148
+ it "should be able to set a eval default value" do
149
+
150
+ loader.configure_from( ifixture_file('ProjectsDefaults.yml') )
151
+
152
+ puts loader.load_object.inspect
153
+ puts loader.load_object.value_as_datetime.inspect
154
+
155
+ loader.process_defaults
156
+
157
+ puts loader.load_object.inspect
158
+ puts loader.load_object.value_as_datetime.inspect
159
+
160
+ expect(loader.load_object.value_as_datetime).to be
161
+ expect(loader.load_object.value_as_datetime).to be <= Time.now.to_s(:db)
162
+ end
163
+
164
+
125
165
 
126
166
  end
@@ -0,0 +1,117 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2015
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for Mapping aspects
7
+ # Provides automatic mapping between different system's column headings
8
+ #
9
+ require File.join(File.dirname(__FILE__), 'spec_helper')
10
+
11
+ require 'mapping_generator'
12
+
13
+ describe 'Mapping Services' do
14
+
15
+ before(:all) do
16
+ DataShift::MethodDictionary.clear
17
+ DataShift::MethodDictionary.find_operators( Project )
18
+ end
19
+
20
+ before(:each) do
21
+ load File.join( rspec_datashift_root,'lib/thor/mapping.thor')
22
+ end
23
+
24
+
25
+ it "should generate an empty mapping doc" do
26
+
27
+ f = result_file("mapper.yaml")
28
+
29
+ mapper = DataShift::MappingGenerator.new(f)
30
+
31
+ mapper.generate(nil, {:file => f} )
32
+
33
+ expect(File.exists?(f)).to be true
34
+ end
35
+
36
+
37
+ it "should generate a mapping doc with pre supplied title" do
38
+
39
+ f = result_file("mapper.yaml")
40
+
41
+ mapper = DataShift::MappingGenerator.new(f)
42
+
43
+ mapper.generate(nil, {:file => f} )
44
+
45
+ expect(File.exists?(f)).to be true
46
+ end
47
+
48
+ it "should generate a populated mapping doc for a class" do
49
+
50
+ f = result_file("mapper_project.yaml")
51
+
52
+ mapper = DataShift::MappingGenerator.new(f)
53
+
54
+ mapper.generate( Project, {:file => f} )
55
+
56
+ expect(File.exists?(f)).to be true
57
+ end
58
+
59
+ it "should be able to create a mapping service for a class" do
60
+ mapping_services = DataShift::MappingService.new(Project)
61
+
62
+ expect(mapping_services).to be
63
+ end
64
+
65
+ it "should be able to read a mapping" do
66
+
67
+ f = result_file("mapping_service_project.yaml")
68
+
69
+ mapper = DataShift::MappingGenerator.new(f)
70
+
71
+ mapper.generate(Project, {:file => f} )
72
+
73
+ expect(File.exists?(f)).to be true
74
+
75
+ mapping_service = DataShift::MappingService.new(Project)
76
+
77
+ mapping_service.read(f)
78
+
79
+ expect(mapping_service.map_file_name).to eq f
80
+
81
+ expect(mapping_service.raw_data).to_not be_empty
82
+ expect(mapping_service.yaml_data).to_not be_empty
83
+
84
+ expect(mapping_service.mapping_entry).to be_a OpenStruct
85
+
86
+ # puts mapping_service.mapping_entry.inspect
87
+ expect(mapping_service.mapping_entry.mappings).to be_a Hash
88
+ expect(mapping_service.mapping_entry['mappings']).to be_a Hash
89
+
90
+ end
91
+
92
+ it "should be able to generate a mapping from_excel" do
93
+
94
+ f = result_file("mapping_service_excel.yaml")
95
+
96
+ mapper = DataShift::MappingGenerator.new(f)
97
+
98
+ mapper.generate_from_excel(ifixture_file('SimpleProjects.xls'), :file => f )
99
+
100
+ end
101
+
102
+ it "should provide tasks to generate a mapping doc" do
103
+
104
+ opts = {:model => "Project", :result => "#{results_path}"}
105
+
106
+ run_in(rails_sandbox()) do
107
+ output = capture(:stdout) { Datashift::Mapping.new.invoke(:template, [], opts) }
108
+
109
+ puts output
110
+
111
+ expect(output).to include("Output generated")
112
+ end
113
+ end
114
+
115
+
116
+
117
+ end
@@ -12,8 +12,6 @@ require File.join(File.dirname(__FILE__), 'spec_helper')
12
12
  require 'method_dictionary'
13
13
 
14
14
  describe 'Method Dictionary' do
15
-
16
- include_context "ActiveRecordTestModelsConnected"
17
15
 
18
16
  before(:each) do
19
17
  DataShift::MethodDictionary.clear
@@ -42,7 +40,7 @@ describe 'Method Dictionary' do
42
40
  DataShift::MethodDictionary.assignments[Project].should include('value_as_text')
43
41
 
44
42
  DataShift::MethodDictionary.belongs_to.should_not be_empty
45
- DataShift::MethodDictionary.belongs_to[Project].should be_empty
43
+ expect(DataShift::MethodDictionary.belongs_to[Project]).to include 'user'
46
44
 
47
45
 
48
46
  DataShift::MethodDictionary.column_types.should be_is_a(Hash)
@@ -80,11 +78,11 @@ describe 'Method Dictionary' do
80
78
  method_details.operator.should == 'value_as_string'
81
79
  method_details.operator_for(:assignment).should == 'value_as_string'
82
80
 
83
- method_details.operator?('value_as_string').should be_true
84
- method_details.operator?('blah_as_string').should be_false
81
+ expect(method_details.operator?('value_as_string')).to eq true
82
+ expect(method_details.operator?('blah_as_string')).to eq false
85
83
 
86
- method_details.operator_for(:belongs_to).should be_nil
87
- method_details.operator_for(:has_many).should be_nil
84
+ expect(method_details.operator_for(:belongs_to)).to eq nil
85
+ expect(method_details.operator_for(:has_many)).to eq nil
88
86
  end
89
87
  end
90
88
 
@@ -128,6 +126,25 @@ describe 'Method Dictionary' do
128
126
  end
129
127
 
130
128
  end
129
+
130
+
131
+ it "should populate method_details on Class for belongs_to" do
132
+
133
+ DataShift::MethodDictionary.find_operators( Owner )
134
+
135
+ DataShift::MethodDictionary.build_method_details( Owner )
136
+
137
+ [:project, 'PROJECT'].each do |format|
138
+
139
+ method_details = DataShift::MethodDictionary.find_method_detail( Owner, format )
140
+
141
+ expect(method_details.operator_class_name).to eq 'Project'
142
+ expect(method_details.operator_class).to be_a(Class)
143
+ expect(method_details.operator_class).to eq Project
144
+ end
145
+
146
+ end
147
+
131
148
 
132
149
  it "should populate belongs_to operator for method details for different forms of a column name" do
133
150
 
@@ -251,10 +268,6 @@ describe 'Method Dictionary' do
251
268
  end
252
269
 
253
270
 
254
- it "should find a set of methods based on a list of column names" do
255
- pending("key API - map column headers to set of methods")
256
- end
257
-
258
271
  it "should not by default map setter methods" do
259
272
  DataShift::MethodDictionary.find_operators( Milestone )
260
273
 
@@ -12,8 +12,6 @@ require File.join(File.dirname(__FILE__), 'spec_helper')
12
12
  require 'method_mapper'
13
13
 
14
14
  describe 'Method Mapper' do
15
-
16
- include_context "ActiveRecordTestModelsConnected"
17
15
 
18
16
  before(:each) do
19
17
  DataShift::MethodDictionary.clear
@@ -27,7 +25,7 @@ describe 'Method Mapper' do
27
25
 
28
26
  method_details = @method_mapper.map_inbound_headers_to_methods( Project, headers )
29
27
 
30
- method_details.should have_exactly(4).items
28
+ expect(method_details.size).to eq 4
31
29
  end
32
30
 
33
31
  it "should leave nil in set of methods when no such operator" do
@@ -36,7 +34,7 @@ describe 'Method Mapper' do
36
34
 
37
35
  method_details = @method_mapper.map_inbound_headers_to_methods( Project, headers )
38
36
 
39
- method_details.should have_exactly(6).items
37
+ expect(method_details.size).to eq 6
40
38
 
41
39
  method_details[2].should be_nil
42
40
  method_details[5].should be_nil
@@ -53,7 +51,7 @@ describe 'Method Mapper' do
53
51
 
54
52
  method_details = @method_mapper.map_inbound_headers_to_methods( Project, headers )
55
53
 
56
- method_details.should have_exactly(4).items
54
+ expect(method_details.size).to eq 4
57
55
 
58
56
  method_details[2].should be_nil
59
57
 
@@ -67,7 +65,7 @@ describe 'Method Mapper' do
67
65
 
68
66
  method_details = @method_mapper.map_inbound_headers_to_methods( Project, headers )
69
67
 
70
- method_details.should have_exactly(4).items
68
+ expect(method_details.size).to eq 4
71
69
 
72
70
  method_details[0].should be_a DataShift::MethodDetail
73
71
 
@@ -85,7 +83,7 @@ describe 'Method Mapper' do
85
83
 
86
84
  method_details = @method_mapper.map_inbound_headers_to_methods( Project, headers )
87
85
 
88
- method_details.should have_exactly(4).items
86
+ expect(method_details.size).to eq 4
89
87
 
90
88
  method_details.should_not include nil
91
89
 
@@ -0,0 +1,41 @@
1
+ # Copyright:: (c) Autotelik Media Ltd 2015
2
+ # Author :: Tom Statter
3
+ # Date :: Aug 2015
4
+ # License:: MIT
5
+ #
6
+ # Details:: Specs for Mapping aspects
7
+ # Provides automatic mapping between different system's column headings
8
+ #
9
+ require File.join(File.dirname(__FILE__), 'spec_helper')
10
+
11
+ require 'mapping_generator'
12
+
13
+ describe 'ModelMapper' do
14
+
15
+ before(:all) do
16
+ end
17
+
18
+ before(:each) do
19
+ end
20
+
21
+ it "should identify the class from a string" do
22
+ # Similar to const_get_from_string except this version
23
+ # returns nil if no such class found
24
+ # Support modules e.g "Spree::Property"
25
+ #
26
+ expect(DataShift::ModelMapper.class_from_string( Project)).to be_a Class
27
+
28
+ end
29
+
30
+ it "should identify the class from a string contianing modules" do
31
+ # Similar to const_get_from_string except this version
32
+ # returns nil if no such class found
33
+ # Support modules e.g "Spree::Property"
34
+ #
35
+ expect(DataShift::ModelMapper.class_from_string( DataShift::AClassInAModule )).to be_a Class
36
+
37
+ end
38
+
39
+
40
+
41
+ end
@@ -13,9 +13,6 @@ Paperclip.options[:command_path] = "/usr/local/bin/"
13
13
 
14
14
  describe 'PaperClip Bulk Loader' do
15
15
 
16
- include_context "ActiveRecordTestModelsConnected"
17
-
18
- include_context "ClearAndPopulateProject"
19
16
 
20
17
  include DataShift::Logging
21
18
 
@@ -38,7 +35,7 @@ describe 'PaperClip Bulk Loader' do
38
35
  end
39
36
 
40
37
  it "should create a new paperclip loader and define attachment class" do
41
- loader = DataShift::Paperclip::AttachmentLoader.new(@attachment_klass, true, nil, @common_options)
38
+ loader = DataShift::Paperclip::AttachmentLoader.new(@attachment_klass, nil, @common_options)
42
39
 
43
40
  loader.load_object_class.should == Digital
44
41
  loader.load_object.should be_a Digital
@@ -50,7 +47,7 @@ describe 'PaperClip Bulk Loader' do
50
47
 
51
48
  opts = { :attach_to_klass => Owner }.merge(@common_options)
52
49
 
53
- loader = DataShift::Paperclip::AttachmentLoader.new(@attachment_klass, true, nil, opts)
50
+ loader = DataShift::Paperclip::AttachmentLoader.new(@attachment_klass, nil, opts)
54
51
 
55
52
  loader.attach_to_klass.should == Owner
56
53
  end
@@ -89,7 +86,7 @@ describe 'PaperClip Bulk Loader' do
89
86
 
90
87
  loader.process_from_filesystem(@attachment_path, opts)
91
88
 
92
- Dir.glob("MissingAttachmentRecords/*.jpeg", File::FNM_CASEFOLD).should have_exactly(names.size).items
89
+ expect(Dir.glob("MissingAttachmentRecords/*.jpeg", File::FNM_CASEFOLD).size).to eq names.size
93
90
  end
94
91
 
95
92
  end
@@ -11,9 +11,6 @@ require 'erb'
11
11
 
12
12
  describe 'Populator' do
13
13
 
14
- include_context "ActiveRecordTestModelsConnected"
15
-
16
- include_context "ClearAndPopulateProject"
17
14
 
18
15
  before(:each) do
19
16
  @loader = DataShift::LoaderBase.new(Project)
@@ -35,7 +32,7 @@ describe 'Populator' do
35
32
 
36
33
  options = {:populator => 'AnotherPopulator' }
37
34
 
38
- local_loader = DataShift::LoaderBase.new(Project, true, nil, options)
35
+ local_loader = DataShift::LoaderBase.new(Project, nil, options)
39
36
 
40
37
  local_loader.populator.should_not be_nil
41
38
  local_loader.populator.should be_a AnotherPopulator
@@ -48,7 +45,7 @@ describe 'Populator' do
48
45
 
49
46
  options = {:populator => AnotherPopulator }
50
47
 
51
- local_loader = DataShift::LoaderBase.new(Project, true, nil, options)
48
+ local_loader = DataShift::LoaderBase.new(Project, nil, options)
52
49
 
53
50
  local_loader.populator.should_not be_nil
54
51
  local_loader.populator.should be_a AnotherPopulator
@@ -61,19 +58,54 @@ describe 'Populator' do
61
58
 
62
59
  method_detail = DataShift::MethodDictionary.find_method_detail( Project, column_heading )
63
60
 
64
- method_detail.should_not be_nil
61
+ expect(method_detail).to be_a DataShift::MethodDetail
65
62
 
66
- x, attributes = @populator.prepare_data(method_detail, value)
63
+ pop_value, attributes = @populator.prepare_data(method_detail, value)
67
64
 
68
- x.should == value
65
+ pop_value.should == value
69
66
  attributes.should be_a Hash
70
67
  attributes.should be_empty
71
68
 
69
+ # check for white space preservation
70
+ value = 'Another Lazy fox'
71
+
72
+ pop_value, attributes = @populator.prepare_data(method_detail, value)
73
+
74
+ pop_value.should == value
75
+
72
76
  end
73
77
 
74
- it "should process a string value against an assigment instance method" do
78
+ it "should process a DSL string into a real hash" do
79
+
80
+ str1 = "{:name => 'the_fox' }"
81
+
82
+ x = DataShift::Populator::string_to_hash( str1 )
83
+
84
+ expect(x).to be_a Hash
85
+ expect(x.size).to eq 1
86
+
87
+ str2 = "{:name => 'the_fox', 'occupation' => 'fantastic', :food => 'duck soup' }"
88
+
89
+ x = DataShift::Populator::string_to_hash( str2 )
90
+
91
+ expect(x.size).to eq 3
92
+ expect(x.keys).to include 'food'
93
+ expect(x['food']).to eq 'duck soup'
94
+
95
+ str3 = "{:cost_price => '13.45', :price => 23, :sale_price => 4.23 }"
96
+
97
+ x = DataShift::Populator::string_to_hash( str3 )
75
98
 
76
- value = 'Get up Lazy fox '
99
+ expect(x.keys).to include 'price'
100
+ expect(x['cost_price']).to eq '13.45'
101
+ expect(x['price']).to eq 23
102
+ expect(x['sale_price']).to eq 4.23
103
+
104
+ end
105
+
106
+ it "should process a string value against an assigment instance method" do
107
+
108
+ value = 'Get up Lazy fox {:name => \'the_fox\' }'
77
109
 
78
110
  DataShift::MethodDictionary.find_operators( Milestone, :instance_methods => true )
79
111
 
@@ -83,11 +115,13 @@ describe 'Populator' do
83
115
 
84
116
  method_detail.should_not be_nil
85
117
 
86
- x, attributes = @populator.prepare_data(method_detail, value)
118
+ pop_value, attrs = @populator.prepare_data(method_detail, value)
87
119
 
88
- x.should == value
89
- attributes.should be_a Hash
90
- attributes.should be_empty
120
+ expect(pop_value).to eq 'Get up Lazy fox '
121
+ expect(attrs).to be_a Hash
122
+ expect(attrs.size).to eq 1
123
+ expect(attrs.keys).to include 'name'
124
+ expect(attrs['name']).to eq 'the_fox'
91
125
 
92
126
  end
93
127