assimilate 0.3.1 → 0.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.travis.yml +5 -0
- data/assimilate.gemspec +2 -0
- data/lib/assimilate/extender.rb +9 -1
- data/lib/assimilate/version.rb +1 -1
- data/spec/data/master_records_conflicting.csv +8 -0
- data/spec/lib/extend_spec.rb +24 -0
- metadata +20 -17
data/.travis.yml
ADDED
data/assimilate.gemspec
CHANGED
@@ -16,6 +16,8 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
gem.version = Assimilate::VERSION
|
18
18
|
|
19
|
+
gem.required_ruby_version = '>= 1.9.2'
|
20
|
+
|
19
21
|
gem.add_dependency "mongo", "~> 1.6.0"
|
20
22
|
gem.add_dependency "bson_ext", "~> 1.6.0"
|
21
23
|
gem.add_dependency 'activesupport', "~> 3.2.0"
|
data/lib/assimilate/extender.rb
CHANGED
@@ -26,7 +26,15 @@ class Assimilate::Extender
|
|
26
26
|
if key
|
27
27
|
# ignore records that are missing a key value.
|
28
28
|
if h.include?(key)
|
29
|
-
|
29
|
+
# conflict with existing record for the same key
|
30
|
+
# HACK - HARD-CODED BEHAVIOR:
|
31
|
+
# * look for a boolean field called '_active'; if the previously-stored value says inactive and
|
32
|
+
# this one is active, then use the new one.
|
33
|
+
if h[key].include?('_active') && rec.include?('_active') && h[key]['_active'] != rec['_active']
|
34
|
+
rec = [h[key],rec].select {|r| r['_active']}.first
|
35
|
+
else
|
36
|
+
raise Assimilate::CorruptDataError, "Duplicate records for key [#{key}] in #{@domainkey} [#{@domain}]"
|
37
|
+
end
|
30
38
|
end
|
31
39
|
h[key] = rec
|
32
40
|
end
|
data/lib/assimilate/version.rb
CHANGED
@@ -0,0 +1,8 @@
|
|
1
|
+
ID,name,title,spouse,_active
|
2
|
+
1,George Washington,President,Martha,1
|
3
|
+
2,John Adams,Vice President,Abigail,1
|
4
|
+
3,Benjamin Franklin,Sage,Deborah,1
|
5
|
+
4,Aaron Burr,Duelist,Theodosia,1
|
6
|
+
5,Alexander Hamilton,Financier,Elizabeth,1
|
7
|
+
6,James Madison,Theorist,Dolly,1
|
8
|
+
5,Alexander Hamilton,Financier,PretendSecondWife,0
|
data/spec/lib/extend_spec.rb
CHANGED
@@ -210,6 +210,30 @@ describe "loading extended data" do
|
|
210
210
|
end
|
211
211
|
end
|
212
212
|
|
213
|
+
describe "with conflicting source records" do
|
214
|
+
before(:all) do
|
215
|
+
reset_catalog
|
216
|
+
import_base_data("123", "master_records_conflicting.csv")
|
217
|
+
end
|
218
|
+
|
219
|
+
before(:each) do
|
220
|
+
import_extended_data("1001", "dates.csv")
|
221
|
+
end
|
222
|
+
|
223
|
+
it "should capture changes" do
|
224
|
+
@extender.stats.should == {
|
225
|
+
:baseline_count => 6,
|
226
|
+
:final_count => 7,
|
227
|
+
:distinct_ids => 4,
|
228
|
+
:adds_count => 1,
|
229
|
+
:new_ids => ['16'],
|
230
|
+
:updates_count => 3,
|
231
|
+
:updated_fields => {'date' => 4},
|
232
|
+
:unchanged_count => 0
|
233
|
+
}
|
234
|
+
end
|
235
|
+
end
|
236
|
+
|
213
237
|
# test handling of multiple records for same ID in the extended-data file
|
214
238
|
# test importing data at top level (no keyfield for sub-attributes)
|
215
239
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: assimilate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.3.
|
4
|
+
version: 0.3.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,11 +9,11 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-06-
|
12
|
+
date: 2012-06-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: mongo
|
16
|
-
requirement: &
|
16
|
+
requirement: &2152542320 !ruby/object:Gem::Requirement
|
17
17
|
none: false
|
18
18
|
requirements:
|
19
19
|
- - ~>
|
@@ -21,10 +21,10 @@ dependencies:
|
|
21
21
|
version: 1.6.0
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
|
-
version_requirements: *
|
24
|
+
version_requirements: *2152542320
|
25
25
|
- !ruby/object:Gem::Dependency
|
26
26
|
name: bson_ext
|
27
|
-
requirement: &
|
27
|
+
requirement: &2152541840 !ruby/object:Gem::Requirement
|
28
28
|
none: false
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
@@ -32,10 +32,10 @@ dependencies:
|
|
32
32
|
version: 1.6.0
|
33
33
|
type: :runtime
|
34
34
|
prerelease: false
|
35
|
-
version_requirements: *
|
35
|
+
version_requirements: *2152541840
|
36
36
|
- !ruby/object:Gem::Dependency
|
37
37
|
name: activesupport
|
38
|
-
requirement: &
|
38
|
+
requirement: &2152541360 !ruby/object:Gem::Requirement
|
39
39
|
none: false
|
40
40
|
requirements:
|
41
41
|
- - ~>
|
@@ -43,10 +43,10 @@ dependencies:
|
|
43
43
|
version: 3.2.0
|
44
44
|
type: :runtime
|
45
45
|
prerelease: false
|
46
|
-
version_requirements: *
|
46
|
+
version_requirements: *2152541360
|
47
47
|
- !ruby/object:Gem::Dependency
|
48
48
|
name: rake
|
49
|
-
requirement: &
|
49
|
+
requirement: &2152540900 !ruby/object:Gem::Requirement
|
50
50
|
none: false
|
51
51
|
requirements:
|
52
52
|
- - ~>
|
@@ -54,10 +54,10 @@ dependencies:
|
|
54
54
|
version: 0.9.2
|
55
55
|
type: :development
|
56
56
|
prerelease: false
|
57
|
-
version_requirements: *
|
57
|
+
version_requirements: *2152540900
|
58
58
|
- !ruby/object:Gem::Dependency
|
59
59
|
name: rspec
|
60
|
-
requirement: &
|
60
|
+
requirement: &2152540420 !ruby/object:Gem::Requirement
|
61
61
|
none: false
|
62
62
|
requirements:
|
63
63
|
- - ~>
|
@@ -65,10 +65,10 @@ dependencies:
|
|
65
65
|
version: 2.9.0
|
66
66
|
type: :development
|
67
67
|
prerelease: false
|
68
|
-
version_requirements: *
|
68
|
+
version_requirements: *2152540420
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: guard-rspec
|
71
|
-
requirement: &
|
71
|
+
requirement: &2152539940 !ruby/object:Gem::Requirement
|
72
72
|
none: false
|
73
73
|
requirements:
|
74
74
|
- - ~>
|
@@ -76,10 +76,10 @@ dependencies:
|
|
76
76
|
version: 0.7.0
|
77
77
|
type: :development
|
78
78
|
prerelease: false
|
79
|
-
version_requirements: *
|
79
|
+
version_requirements: *2152539940
|
80
80
|
- !ruby/object:Gem::Dependency
|
81
81
|
name: ruby_gntp
|
82
|
-
requirement: &
|
82
|
+
requirement: &2152539480 !ruby/object:Gem::Requirement
|
83
83
|
none: false
|
84
84
|
requirements:
|
85
85
|
- - ~>
|
@@ -87,7 +87,7 @@ dependencies:
|
|
87
87
|
version: 0.3.4
|
88
88
|
type: :development
|
89
89
|
prerelease: false
|
90
|
-
version_requirements: *
|
90
|
+
version_requirements: *2152539480
|
91
91
|
description: Ingest updates from CSV and apply to set of hashes
|
92
92
|
email:
|
93
93
|
- jmay@pobox.com
|
@@ -97,6 +97,7 @@ extensions: []
|
|
97
97
|
extra_rdoc_files: []
|
98
98
|
files:
|
99
99
|
- .gitignore
|
100
|
+
- .travis.yml
|
100
101
|
- Gemfile
|
101
102
|
- Guardfile
|
102
103
|
- LICENSE
|
@@ -116,6 +117,7 @@ files:
|
|
116
117
|
- spec/data/duplicate_input.csv
|
117
118
|
- spec/data/logs1.csv
|
118
119
|
- spec/data/logs2.csv
|
120
|
+
- spec/data/master_records_conflicting.csv
|
119
121
|
- spec/data/test.yml
|
120
122
|
- spec/data/updates.csv
|
121
123
|
- spec/lib/batch_spec.rb
|
@@ -132,7 +134,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
132
134
|
requirements:
|
133
135
|
- - ! '>='
|
134
136
|
- !ruby/object:Gem::Version
|
135
|
-
version:
|
137
|
+
version: 1.9.2
|
136
138
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
137
139
|
none: false
|
138
140
|
requirements:
|
@@ -152,6 +154,7 @@ test_files:
|
|
152
154
|
- spec/data/duplicate_input.csv
|
153
155
|
- spec/data/logs1.csv
|
154
156
|
- spec/data/logs2.csv
|
157
|
+
- spec/data/master_records_conflicting.csv
|
155
158
|
- spec/data/test.yml
|
156
159
|
- spec/data/updates.csv
|
157
160
|
- spec/lib/batch_spec.rb
|