assimilate 0.4.2 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/assimilate.gemspec +7 -7
- data/lib/assimilate/batch.rb +4 -4
- data/lib/assimilate/catalog.rb +3 -3
- data/lib/assimilate/extender.rb +4 -4
- data/lib/assimilate/version.rb +1 -1
- data/spec/lib/batch_spec.rb +2 -2
- data/spec/lib/extend_spec.rb +2 -2
- data/spec/spec_helper.rb +2 -1
- metadata +24 -85
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 35f952558d63466990088b75567b8de91b113478
|
4
|
+
data.tar.gz: 2d28174a377bcde1acff585ddb66f0314b55dd68
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 18359a13a1ce54a56151e34f7ffb4588d316541b2b5c9b8e63c34333617e28429853f9ef717015478aa0bb8f265c35968de2dfbb8f0ee00bcd296ceb566cdadf
|
7
|
+
data.tar.gz: 8649d17882e3baf0f9229544e832d9dcfcba4fef6c4d5473bc64f3999291a45a5f992c0b8e22972fcd206370544e020a804958e19a1a09437d4809ee9b8b2e92
|
data/assimilate.gemspec
CHANGED
@@ -16,14 +16,14 @@ Gem::Specification.new do |gem|
|
|
16
16
|
gem.require_paths = ["lib"]
|
17
17
|
gem.version = Assimilate::VERSION
|
18
18
|
|
19
|
-
gem.required_ruby_version = '>=
|
19
|
+
gem.required_ruby_version = '>= 2.3.1'
|
20
20
|
|
21
|
-
gem.add_dependency "mongo", "~>
|
22
|
-
gem.add_dependency "bson_ext", "~> 1.
|
21
|
+
gem.add_dependency "mongo", "~> 2.3.0"
|
22
|
+
#gem.add_dependency "bson_ext", "~> 1.12.0"
|
23
23
|
gem.add_dependency 'activesupport', "~> 3.2.0"
|
24
24
|
|
25
|
-
gem.add_development_dependency "rake", "~>
|
26
|
-
gem.add_development_dependency "rspec", "~>
|
27
|
-
gem.add_development_dependency "guard-rspec", "~> 0.7.0"
|
28
|
-
gem.add_development_dependency "ruby_gntp", "~> 0.3.4"
|
25
|
+
gem.add_development_dependency "rake", "~> 11.2"
|
26
|
+
gem.add_development_dependency "rspec", "~> 3.5.0"
|
27
|
+
#gem.add_development_dependency "guard-rspec", "~> 0.7.0"
|
28
|
+
#gem.add_development_dependency "ruby_gntp", "~> 0.3.4"
|
29
29
|
end
|
data/lib/assimilate/batch.rb
CHANGED
@@ -120,7 +120,7 @@ class Assimilate::Batch
|
|
120
120
|
raise(Assimilate::DuplicateImportError, "duplicate batch for datestamp #{datestamp}") if @catalog.batches.find(dkey => @domain, 'datestamp' => @datestamp).to_a.any?
|
121
121
|
raise(Assimilate::DuplicateImportError, "duplicate batch for file #{@filename}") if @catalog.batches.find(dkey => @domain, 'filename' => @filename).to_a.any?
|
122
122
|
|
123
|
-
@catalog.batches.
|
123
|
+
@catalog.batches.insert_one({
|
124
124
|
dkey => @domain,
|
125
125
|
'datestamp' => @datestamp,
|
126
126
|
'filename' => @filename
|
@@ -130,7 +130,7 @@ class Assimilate::Batch
|
|
130
130
|
def apply_deletes
|
131
131
|
unless @suppress_deletes
|
132
132
|
@deleted_keys.each do |key|
|
133
|
-
@catalog.catalog.
|
133
|
+
@catalog.catalog.update_one(
|
134
134
|
{
|
135
135
|
@domainkey => domain,
|
136
136
|
idfield => key
|
@@ -148,14 +148,14 @@ class Assimilate::Batch
|
|
148
148
|
def apply_inserts
|
149
149
|
@adds.each_slice(INSERT_BATCH_SIZE) do |slice|
|
150
150
|
# mongo insert can't handle CSV::Row objects, must be converted to regular hashes
|
151
|
-
@catalog.catalog.
|
151
|
+
@catalog.catalog.insert_many(decorate(slice))
|
152
152
|
end
|
153
153
|
end
|
154
154
|
|
155
155
|
def apply_updates
|
156
156
|
marker = @catalog.config[:update_marker]
|
157
157
|
@changes.each do |key, diffs|
|
158
|
-
@catalog.catalog.
|
158
|
+
@catalog.catalog.update_one(
|
159
159
|
{
|
160
160
|
@domainkey => domain,
|
161
161
|
idfield => key
|
data/lib/assimilate/catalog.rb
CHANGED
@@ -26,9 +26,9 @@ class Assimilate::Catalog
|
|
26
26
|
@config = YAML.load(File.open(args[:config]))
|
27
27
|
check_config
|
28
28
|
|
29
|
-
@db = Mongo::
|
30
|
-
@catalog = @db
|
31
|
-
@batches = @db
|
29
|
+
@db = Mongo::Client.new("mongodb://127.0.0.1:27017/" + @config[:db])
|
30
|
+
@catalog = @db[@config[:catalog]]
|
31
|
+
@batches = @db[@config[:batch]]
|
32
32
|
end
|
33
33
|
|
34
34
|
def check_config
|
data/lib/assimilate/extender.rb
CHANGED
@@ -113,14 +113,14 @@ class Assimilate::Extender
|
|
113
113
|
@adds.each do |key|
|
114
114
|
data = @seen[key]
|
115
115
|
if keyfield
|
116
|
-
@catalog.catalog.
|
116
|
+
@catalog.catalog.insert_one(
|
117
117
|
@domainkey => domain,
|
118
118
|
idfield => key,
|
119
119
|
keyfield => data
|
120
120
|
)
|
121
121
|
else
|
122
122
|
# top-level extension
|
123
|
-
@catalog.catalog.
|
123
|
+
@catalog.catalog.insert_one(
|
124
124
|
data.merge(
|
125
125
|
@domainkey => domain,
|
126
126
|
idfield => key
|
@@ -135,7 +135,7 @@ class Assimilate::Extender
|
|
135
135
|
@changes.each do |key|
|
136
136
|
data = @seen[key]
|
137
137
|
if keyfield
|
138
|
-
@catalog.catalog.
|
138
|
+
@catalog.catalog.update_one(
|
139
139
|
{
|
140
140
|
@domainkey => domain,
|
141
141
|
idfield => key
|
@@ -147,7 +147,7 @@ class Assimilate::Extender
|
|
147
147
|
)
|
148
148
|
else
|
149
149
|
# top-level extension
|
150
|
-
@catalog.catalog.
|
150
|
+
@catalog.catalog.update_one(
|
151
151
|
{
|
152
152
|
@domainkey => domain,
|
153
153
|
idfield => key
|
data/lib/assimilate/version.rb
CHANGED
data/spec/lib/batch_spec.rb
CHANGED
data/spec/lib/extend_spec.rb
CHANGED
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,128 +1,71 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: assimilate
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
5
|
-
prerelease:
|
4
|
+
version: 0.5.0
|
6
5
|
platform: ruby
|
7
6
|
authors:
|
8
7
|
- Jason May
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2016-11-14 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: mongo
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
|
-
none: false
|
18
16
|
requirements:
|
19
|
-
- - ~>
|
17
|
+
- - "~>"
|
20
18
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
19
|
+
version: 2.3.0
|
22
20
|
type: :runtime
|
23
21
|
prerelease: false
|
24
22
|
version_requirements: !ruby/object:Gem::Requirement
|
25
|
-
none: false
|
26
23
|
requirements:
|
27
|
-
- - ~>
|
24
|
+
- - "~>"
|
28
25
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
30
|
-
- !ruby/object:Gem::Dependency
|
31
|
-
name: bson_ext
|
32
|
-
requirement: !ruby/object:Gem::Requirement
|
33
|
-
none: false
|
34
|
-
requirements:
|
35
|
-
- - ~>
|
36
|
-
- !ruby/object:Gem::Version
|
37
|
-
version: 1.6.0
|
38
|
-
type: :runtime
|
39
|
-
prerelease: false
|
40
|
-
version_requirements: !ruby/object:Gem::Requirement
|
41
|
-
none: false
|
42
|
-
requirements:
|
43
|
-
- - ~>
|
44
|
-
- !ruby/object:Gem::Version
|
45
|
-
version: 1.6.0
|
26
|
+
version: 2.3.0
|
46
27
|
- !ruby/object:Gem::Dependency
|
47
28
|
name: activesupport
|
48
29
|
requirement: !ruby/object:Gem::Requirement
|
49
|
-
none: false
|
50
30
|
requirements:
|
51
|
-
- - ~>
|
31
|
+
- - "~>"
|
52
32
|
- !ruby/object:Gem::Version
|
53
33
|
version: 3.2.0
|
54
34
|
type: :runtime
|
55
35
|
prerelease: false
|
56
36
|
version_requirements: !ruby/object:Gem::Requirement
|
57
|
-
none: false
|
58
37
|
requirements:
|
59
|
-
- - ~>
|
38
|
+
- - "~>"
|
60
39
|
- !ruby/object:Gem::Version
|
61
40
|
version: 3.2.0
|
62
41
|
- !ruby/object:Gem::Dependency
|
63
42
|
name: rake
|
64
43
|
requirement: !ruby/object:Gem::Requirement
|
65
|
-
none: false
|
66
44
|
requirements:
|
67
|
-
- - ~>
|
45
|
+
- - "~>"
|
68
46
|
- !ruby/object:Gem::Version
|
69
|
-
version:
|
47
|
+
version: '11.2'
|
70
48
|
type: :development
|
71
49
|
prerelease: false
|
72
50
|
version_requirements: !ruby/object:Gem::Requirement
|
73
|
-
none: false
|
74
51
|
requirements:
|
75
|
-
- - ~>
|
52
|
+
- - "~>"
|
76
53
|
- !ruby/object:Gem::Version
|
77
|
-
version:
|
54
|
+
version: '11.2'
|
78
55
|
- !ruby/object:Gem::Dependency
|
79
56
|
name: rspec
|
80
57
|
requirement: !ruby/object:Gem::Requirement
|
81
|
-
none: false
|
82
|
-
requirements:
|
83
|
-
- - ~>
|
84
|
-
- !ruby/object:Gem::Version
|
85
|
-
version: 2.9.0
|
86
|
-
type: :development
|
87
|
-
prerelease: false
|
88
|
-
version_requirements: !ruby/object:Gem::Requirement
|
89
|
-
none: false
|
90
|
-
requirements:
|
91
|
-
- - ~>
|
92
|
-
- !ruby/object:Gem::Version
|
93
|
-
version: 2.9.0
|
94
|
-
- !ruby/object:Gem::Dependency
|
95
|
-
name: guard-rspec
|
96
|
-
requirement: !ruby/object:Gem::Requirement
|
97
|
-
none: false
|
98
|
-
requirements:
|
99
|
-
- - ~>
|
100
|
-
- !ruby/object:Gem::Version
|
101
|
-
version: 0.7.0
|
102
|
-
type: :development
|
103
|
-
prerelease: false
|
104
|
-
version_requirements: !ruby/object:Gem::Requirement
|
105
|
-
none: false
|
106
|
-
requirements:
|
107
|
-
- - ~>
|
108
|
-
- !ruby/object:Gem::Version
|
109
|
-
version: 0.7.0
|
110
|
-
- !ruby/object:Gem::Dependency
|
111
|
-
name: ruby_gntp
|
112
|
-
requirement: !ruby/object:Gem::Requirement
|
113
|
-
none: false
|
114
58
|
requirements:
|
115
|
-
- - ~>
|
59
|
+
- - "~>"
|
116
60
|
- !ruby/object:Gem::Version
|
117
|
-
version:
|
61
|
+
version: 3.5.0
|
118
62
|
type: :development
|
119
63
|
prerelease: false
|
120
64
|
version_requirements: !ruby/object:Gem::Requirement
|
121
|
-
none: false
|
122
65
|
requirements:
|
123
|
-
- - ~>
|
66
|
+
- - "~>"
|
124
67
|
- !ruby/object:Gem::Version
|
125
|
-
version:
|
68
|
+
version: 3.5.0
|
126
69
|
description: Ingest updates from CSV and apply to set of hashes
|
127
70
|
email:
|
128
71
|
- jmay@pobox.com
|
@@ -131,8 +74,8 @@ executables:
|
|
131
74
|
extensions: []
|
132
75
|
extra_rdoc_files: []
|
133
76
|
files:
|
134
|
-
- .gitignore
|
135
|
-
- .travis.yml
|
77
|
+
- ".gitignore"
|
78
|
+
- ".travis.yml"
|
136
79
|
- Gemfile
|
137
80
|
- Guardfile
|
138
81
|
- LICENSE
|
@@ -160,30 +103,26 @@ files:
|
|
160
103
|
- spec/spec_helper.rb
|
161
104
|
homepage: ''
|
162
105
|
licenses: []
|
106
|
+
metadata: {}
|
163
107
|
post_install_message:
|
164
108
|
rdoc_options: []
|
165
109
|
require_paths:
|
166
110
|
- lib
|
167
111
|
required_ruby_version: !ruby/object:Gem::Requirement
|
168
|
-
none: false
|
169
112
|
requirements:
|
170
|
-
- -
|
113
|
+
- - ">="
|
171
114
|
- !ruby/object:Gem::Version
|
172
|
-
version:
|
115
|
+
version: 2.3.1
|
173
116
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
174
|
-
none: false
|
175
117
|
requirements:
|
176
|
-
- -
|
118
|
+
- - ">="
|
177
119
|
- !ruby/object:Gem::Version
|
178
120
|
version: '0'
|
179
|
-
segments:
|
180
|
-
- 0
|
181
|
-
hash: -258423526564247738
|
182
121
|
requirements: []
|
183
122
|
rubyforge_project: assimilate
|
184
|
-
rubygems_version:
|
123
|
+
rubygems_version: 2.5.1
|
185
124
|
signing_key:
|
186
|
-
specification_version:
|
125
|
+
specification_version: 4
|
187
126
|
summary: Review & incorporate changes to a repository of persistent hashes in mongodb.
|
188
127
|
test_files:
|
189
128
|
- spec/data/batch_input.csv
|