pupa 0.0.13 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +1 -0
  3. data/README.md +17 -0
  4. data/lib/pupa.rb +5 -8
  5. data/lib/pupa/errors.rb +4 -0
  6. data/lib/pupa/models/model.rb +5 -5
  7. data/lib/pupa/models/organization.rb +10 -6
  8. data/lib/pupa/models/person.rb +10 -6
  9. data/lib/pupa/processor.rb +14 -15
  10. data/lib/pupa/processor/connection.rb +26 -0
  11. data/lib/pupa/processor/connection_adapters/mongodb_adapter.rb +92 -0
  12. data/lib/pupa/processor/connection_adapters/postgresql_adapter.rb +116 -0
  13. data/lib/pupa/processor/document_store.rb +3 -0
  14. data/lib/pupa/processor/middleware/raise_error.rb +1 -0
  15. data/lib/pupa/refinements/faraday_middleware.rb +1 -1
  16. data/lib/pupa/runner.rb +14 -21
  17. data/lib/pupa/version.rb +1 -1
  18. data/pupa.gemspec +3 -2
  19. data/schemas/popolo/contact_detail.json +10 -0
  20. data/schemas/popolo/membership.json +29 -9
  21. data/schemas/popolo/organization.json +9 -2
  22. data/schemas/popolo/other_name.json +24 -0
  23. data/schemas/popolo/person.json +6 -3
  24. data/schemas/popolo/post.json +16 -2
  25. data/spec/models/model_spec.rb +1 -1
  26. data/spec/processor/connection_adapters/mongodb_adapter_spec.rb +61 -0
  27. data/spec/processor/connection_adapters/postgresql_adapter_spec.rb +70 -0
  28. data/spec/processor/connection_spec.rb +15 -0
  29. data/spec/processor/middleware/parse_json_spec.rb +90 -0
  30. data/spec/processor_spec.rb +9 -10
  31. data/spec/spec_helper.rb +0 -10
  32. metadata +83 -75
  33. data/lib/pupa/processor/persistence.rb +0 -85
  34. data/spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml +0 -56
  35. data/spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml +0 -48
  36. data/spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml +0 -54
  37. data/spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml +0 -26
  38. data/spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml +0 -46
  39. data/spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml +0 -26
  40. data/spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml +0 -62
  41. data/spec/processor/persistence_spec.rb +0 -51
@@ -0,0 +1,15 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
2
+
3
+ describe Pupa::Processor::Connection do
4
+ describe '.new' do
5
+ it 'should use MongoDB' do
6
+ Pupa::Processor::Connection::MongoDBAdapter.should_receive(:new).with('mongodb://localhost:27017/pupa').and_call_original
7
+ Pupa::Processor::Connection.new('mongodb://localhost:27017/pupa')
8
+ end
9
+
10
+ it 'should use PostgreSQL' do
11
+ Pupa::Processor::Connection::PostgreSQLAdapter.should_receive(:new).with('postgres://localhost:5432/pupa').and_call_original
12
+ Pupa::Processor::Connection.new('postgres://localhost:5432/pupa')
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,90 @@
1
+ require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
2
+
3
+ # @see spec/helper.rb and spec/parse_xml_spec.rb in faraday_middleware
4
+ describe Pupa::Processor::Middleware::ParseJson do
5
+ let(:options) { Hash.new }
6
+ let(:headers) { Hash.new }
7
+ let(:middleware) {
8
+ described_class.new(lambda {|env|
9
+ Faraday::Response.new(env)
10
+ }, options)
11
+ }
12
+
13
+ def process(body, content_type = nil, options = {})
14
+ env = {
15
+ :body => body, :request => options,
16
+ :response_headers => Faraday::Utils::Headers.new(headers)
17
+ }
18
+ env[:response_headers]['content-type'] = content_type if content_type
19
+ middleware.call(env)
20
+ end
21
+
22
+ let(:json) { '{"title":"foo","body":"bar"}' }
23
+ let(:title) { 'foo' }
24
+ let(:body) { 'bar' }
25
+
26
+ context "no type matching" do
27
+ it "doesn't change nil body" do
28
+ expect(process(nil).body).to be_nil
29
+ end
30
+
31
+ it "turns empty body into nil" do
32
+ expect(process('').body).to be_nil
33
+ end
34
+
35
+ it "parses json body" do
36
+ response = process(json)
37
+ expect(response.body['title']).to eq(title)
38
+ expect(response.body['body']).to eq(body)
39
+ expect(response.env[:raw_body]).to be_nil
40
+ end
41
+ end
42
+
43
+ context "with preserving raw" do
44
+ let(:options) { {:preserve_raw => true} }
45
+
46
+ it "parses json body" do
47
+ response = process(json)
48
+ expect(response.body['title']).to eq(title)
49
+ expect(response.body['body']).to eq(body)
50
+ expect(response.env[:raw_body]).to eq(json)
51
+ end
52
+
53
+ it "can opt out of preserving raw" do
54
+ response = process(json, nil, :preserve_raw => false)
55
+ expect(response.env[:raw_body]).to be_nil
56
+ end
57
+ end
58
+
59
+ context "with regexp type matching" do
60
+ let(:options) { {:content_type => /\bjson$/} }
61
+
62
+ it "parses json body of correct type" do
63
+ response = process(json, 'application/json')
64
+ expect(response.body['title']).to eq(title)
65
+ expect(response.body['body']).to eq(body)
66
+ end
67
+
68
+ it "ignores json body of incorrect type" do
69
+ response = process(json, 'application/xml')
70
+ expect(response.body).to eq(json)
71
+ end
72
+ end
73
+
74
+ context "with array type matching" do
75
+ let(:options) { {:content_type => %w[a/b c/d]} }
76
+
77
+ it "parses json body of correct type" do
78
+ expect(process(json, 'a/b').body).to be_a(Hash)
79
+ expect(process(json, 'c/d').body).to be_a(Hash)
80
+ end
81
+
82
+ it "ignores json body of incorrect type" do
83
+ expect(process(json, 'a/d').body).not_to be_a(Hash)
84
+ end
85
+ end
86
+
87
+ it "chokes on invalid json" do
88
+ expect{ process('{') }.to raise_error
89
+ end
90
+ end
@@ -11,7 +11,7 @@ describe Pupa::Processor do
11
11
  end
12
12
 
13
13
  def make_person_invalid
14
- @person = Pupa::Person.new
14
+ @person = Pupa::Person.new(name: ['foo'])
15
15
  end
16
16
 
17
17
  def scrape_people
@@ -28,11 +28,11 @@ describe Pupa::Processor do
28
28
  end
29
29
 
30
30
  let :processor do
31
- PersonProcessor.new('/tmp', level: 'WARN', logdev: io)
31
+ PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io)
32
32
  end
33
33
 
34
34
  let :novalidate do
35
- PersonProcessor.new('/tmp', level: 'WARN', logdev: io, validate: false)
35
+ PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io, validate: false)
36
36
  end
37
37
 
38
38
  describe '#get' do
@@ -98,8 +98,7 @@ describe Pupa::Processor do
98
98
 
99
99
  describe '#import' do
100
100
  before :each do
101
- Pupa.session = Moped::Session.new(['localhost:27017'], database: 'pupa_test')
102
- Pupa.session.collections.each(&:drop)
101
+ processor.connection.raw_connection[:organizations].drop
103
102
  end
104
103
 
105
104
  let :_type do
@@ -176,7 +175,7 @@ describe Pupa::Processor do
176
175
  processor.should_receive(:load_scraped_objects).and_return(graphable)
177
176
 
178
177
  processor.import
179
- documents = Pupa.session[:organizations].find.entries
178
+ documents = processor.connection.raw_connection[:organizations].find.entries
180
179
  documents.size.should == 2
181
180
  documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
182
181
  documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
@@ -193,7 +192,7 @@ describe Pupa::Processor do
193
192
  processor.should_receive(:load_scraped_objects).and_return(ungraphable)
194
193
 
195
194
  processor.import
196
- documents = Pupa.session[:organizations].find.entries
195
+ documents = processor.connection.raw_connection[:organizations].find.entries
197
196
  documents.size.should == 2
198
197
  documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '5', '_type' => _type, 'name' => 'Parent'}
199
198
  documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '4', '_type' => _type, 'name' => 'Child', 'parent_id' => '5'}
@@ -203,7 +202,7 @@ describe Pupa::Processor do
203
202
  processor.should_receive(:load_scraped_objects).and_return(foreign_keys_on_foreign_objects)
204
203
 
205
204
  processor.import
206
- documents = Pupa.session[:organizations].find.entries
205
+ documents = processor.connection.raw_connection[:organizations].find.entries
207
206
  documents.size.should == 3
208
207
  documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '9', '_type' => _type, 'name' => 'Parent'}
209
208
  documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '7', '_type' => _type, 'name' => 'Child', 'parent_id' => '9'}
@@ -315,7 +314,7 @@ describe Pupa::Processor do
315
314
  processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_key)
316
315
 
317
316
  processor.import
318
- documents = Pupa.session[:organizations].find.entries
317
+ documents = processor.connection.raw_connection[:organizations].find.entries
319
318
  documents.size.should == 2
320
319
  documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
321
320
  documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
@@ -340,7 +339,7 @@ describe Pupa::Processor do
340
339
  processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_keys_on_foreign_objects)
341
340
 
342
341
  processor.import
343
- documents = Pupa.session[:organizations].find.entries
342
+ documents = processor.connection.raw_connection[:organizations].find.entries
344
343
  documents.size.should == 3
345
344
  documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
346
345
  documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
data/spec/spec_helper.rb CHANGED
@@ -7,18 +7,8 @@ require 'multi_xml'
7
7
  require 'nokogiri'
8
8
  require 'redis-store'
9
9
  require 'rspec'
10
- require 'vcr'
11
10
  require File.dirname(__FILE__) + '/../lib/pupa'
12
11
 
13
- VCR.configure do |c|
14
- c.cassette_library_dir = 'spec/cassettes'
15
- c.hook_into :faraday
16
-
17
- c.around_http_request do |request|
18
- VCR.use_cassette(Digest::SHA1.hexdigest(request.uri + request.body + request.headers.to_s), &request)
19
- end
20
- end
21
-
22
12
  def testing_python_compatibility?
23
13
  ENV['MODE'] == 'compat'
24
14
  end
metadata CHANGED
@@ -1,267 +1,281 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: pupa
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.13
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Open North
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-02-11 00:00:00.000000000 Z
11
+ date: 2014-05-11 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
15
15
  requirement: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ~>
17
+ - - "~>"
18
18
  - !ruby/object:Gem::Version
19
19
  version: 4.0.0
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
- - - ~>
24
+ - - "~>"
25
25
  - !ruby/object:Gem::Version
26
26
  version: 4.0.0
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: colored
29
29
  requirement: !ruby/object:Gem::Requirement
30
30
  requirements:
31
- - - ~>
31
+ - - "~>"
32
32
  - !ruby/object:Gem::Version
33
33
  version: '1.2'
34
34
  type: :runtime
35
35
  prerelease: false
36
36
  version_requirements: !ruby/object:Gem::Requirement
37
37
  requirements:
38
- - - ~>
38
+ - - "~>"
39
39
  - !ruby/object:Gem::Version
40
40
  version: '1.2'
41
41
  - !ruby/object:Gem::Dependency
42
42
  name: faraday_middleware
43
43
  requirement: !ruby/object:Gem::Requirement
44
44
  requirements:
45
- - - ~>
45
+ - - "~>"
46
46
  - !ruby/object:Gem::Version
47
47
  version: 0.9.0
48
48
  type: :runtime
49
49
  prerelease: false
50
50
  version_requirements: !ruby/object:Gem::Requirement
51
51
  requirements:
52
- - - ~>
52
+ - - "~>"
53
53
  - !ruby/object:Gem::Version
54
54
  version: 0.9.0
55
55
  - !ruby/object:Gem::Dependency
56
56
  name: json-schema
57
57
  requirement: !ruby/object:Gem::Requirement
58
58
  requirements:
59
- - - ~>
59
+ - - "~>"
60
60
  - !ruby/object:Gem::Version
61
61
  version: 2.1.3
62
62
  type: :runtime
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
- - - ~>
66
+ - - "~>"
67
67
  - !ruby/object:Gem::Version
68
68
  version: 2.1.3
69
69
  - !ruby/object:Gem::Dependency
70
70
  name: mail
71
71
  requirement: !ruby/object:Gem::Requirement
72
72
  requirements:
73
- - - '>='
73
+ - - ">="
74
74
  - !ruby/object:Gem::Version
75
75
  version: '0'
76
76
  type: :runtime
77
77
  prerelease: false
78
78
  version_requirements: !ruby/object:Gem::Requirement
79
79
  requirements:
80
- - - '>='
80
+ - - ">="
81
81
  - !ruby/object:Gem::Version
82
82
  version: '0'
83
83
  - !ruby/object:Gem::Dependency
84
84
  name: moped
85
85
  requirement: !ruby/object:Gem::Requirement
86
86
  requirements:
87
- - - ~>
87
+ - - "~>"
88
88
  - !ruby/object:Gem::Version
89
- version: 1.5.1
89
+ version: 2.0.0.rc1
90
90
  type: :runtime
91
91
  prerelease: false
92
92
  version_requirements: !ruby/object:Gem::Requirement
93
93
  requirements:
94
- - - ~>
94
+ - - "~>"
95
95
  - !ruby/object:Gem::Version
96
- version: 1.5.1
96
+ version: 2.0.0.rc1
97
97
  - !ruby/object:Gem::Dependency
98
98
  name: oj
99
99
  requirement: !ruby/object:Gem::Requirement
100
100
  requirements:
101
- - - ~>
101
+ - - "~>"
102
102
  - !ruby/object:Gem::Version
103
103
  version: '2.1'
104
104
  type: :runtime
105
105
  prerelease: false
106
106
  version_requirements: !ruby/object:Gem::Requirement
107
107
  requirements:
108
- - - ~>
108
+ - - "~>"
109
109
  - !ruby/object:Gem::Version
110
110
  version: '2.1'
111
+ - !ruby/object:Gem::Dependency
112
+ name: sequel
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: 4.10.0
118
+ type: :runtime
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: 4.10.0
125
+ - !ruby/object:Gem::Dependency
126
+ name: pg
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: 0.17.0
132
+ type: :runtime
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: 0.17.0
111
139
  - !ruby/object:Gem::Dependency
112
140
  name: coveralls
113
141
  requirement: !ruby/object:Gem::Requirement
114
142
  requirements:
115
- - - '>='
143
+ - - ">="
116
144
  - !ruby/object:Gem::Version
117
145
  version: '0'
118
146
  type: :development
119
147
  prerelease: false
120
148
  version_requirements: !ruby/object:Gem::Requirement
121
149
  requirements:
122
- - - '>='
150
+ - - ">="
123
151
  - !ruby/object:Gem::Version
124
152
  version: '0'
125
153
  - !ruby/object:Gem::Dependency
126
154
  name: dalli
127
155
  requirement: !ruby/object:Gem::Requirement
128
156
  requirements:
129
- - - '>='
157
+ - - ">="
130
158
  - !ruby/object:Gem::Version
131
159
  version: '0'
132
160
  type: :development
133
161
  prerelease: false
134
162
  version_requirements: !ruby/object:Gem::Requirement
135
163
  requirements:
136
- - - '>='
164
+ - - ">="
137
165
  - !ruby/object:Gem::Version
138
166
  version: '0'
139
167
  - !ruby/object:Gem::Dependency
140
168
  name: json
141
169
  requirement: !ruby/object:Gem::Requirement
142
170
  requirements:
143
- - - ~>
171
+ - - "~>"
144
172
  - !ruby/object:Gem::Version
145
173
  version: 1.7.7
146
174
  type: :development
147
175
  prerelease: false
148
176
  version_requirements: !ruby/object:Gem::Requirement
149
177
  requirements:
150
- - - ~>
178
+ - - "~>"
151
179
  - !ruby/object:Gem::Version
152
180
  version: 1.7.7
153
181
  - !ruby/object:Gem::Dependency
154
182
  name: multi_xml
155
183
  requirement: !ruby/object:Gem::Requirement
156
184
  requirements:
157
- - - '>='
185
+ - - ">="
158
186
  - !ruby/object:Gem::Version
159
187
  version: '0'
160
188
  type: :development
161
189
  prerelease: false
162
190
  version_requirements: !ruby/object:Gem::Requirement
163
191
  requirements:
164
- - - '>='
192
+ - - ">="
165
193
  - !ruby/object:Gem::Version
166
194
  version: '0'
167
195
  - !ruby/object:Gem::Dependency
168
196
  name: nokogiri
169
197
  requirement: !ruby/object:Gem::Requirement
170
198
  requirements:
171
- - - ~>
199
+ - - "~>"
172
200
  - !ruby/object:Gem::Version
173
201
  version: 1.6.0
174
202
  type: :development
175
203
  prerelease: false
176
204
  version_requirements: !ruby/object:Gem::Requirement
177
205
  requirements:
178
- - - ~>
206
+ - - "~>"
179
207
  - !ruby/object:Gem::Version
180
208
  version: 1.6.0
181
209
  - !ruby/object:Gem::Dependency
182
210
  name: octokit
183
211
  requirement: !ruby/object:Gem::Requirement
184
212
  requirements:
185
- - - '>='
213
+ - - ">="
186
214
  - !ruby/object:Gem::Version
187
215
  version: '0'
188
216
  type: :development
189
217
  prerelease: false
190
218
  version_requirements: !ruby/object:Gem::Requirement
191
219
  requirements:
192
- - - '>='
220
+ - - ">="
193
221
  - !ruby/object:Gem::Version
194
222
  version: '0'
195
223
  - !ruby/object:Gem::Dependency
196
224
  name: rake
197
225
  requirement: !ruby/object:Gem::Requirement
198
226
  requirements:
199
- - - '>='
227
+ - - ">="
200
228
  - !ruby/object:Gem::Version
201
229
  version: '0'
202
230
  type: :development
203
231
  prerelease: false
204
232
  version_requirements: !ruby/object:Gem::Requirement
205
233
  requirements:
206
- - - '>='
234
+ - - ">="
207
235
  - !ruby/object:Gem::Version
208
236
  version: '0'
209
237
  - !ruby/object:Gem::Dependency
210
238
  name: redis-store
211
239
  requirement: !ruby/object:Gem::Requirement
212
240
  requirements:
213
- - - '>='
241
+ - - ">="
214
242
  - !ruby/object:Gem::Version
215
243
  version: '0'
216
244
  type: :development
217
245
  prerelease: false
218
246
  version_requirements: !ruby/object:Gem::Requirement
219
247
  requirements:
220
- - - '>='
248
+ - - ">="
221
249
  - !ruby/object:Gem::Version
222
250
  version: '0'
223
251
  - !ruby/object:Gem::Dependency
224
252
  name: rspec
225
253
  requirement: !ruby/object:Gem::Requirement
226
254
  requirements:
227
- - - ~>
255
+ - - "~>"
228
256
  - !ruby/object:Gem::Version
229
257
  version: '2.10'
230
258
  type: :development
231
259
  prerelease: false
232
260
  version_requirements: !ruby/object:Gem::Requirement
233
261
  requirements:
234
- - - ~>
262
+ - - "~>"
235
263
  - !ruby/object:Gem::Version
236
264
  version: '2.10'
237
265
  - !ruby/object:Gem::Dependency
238
266
  name: typhoeus
239
267
  requirement: !ruby/object:Gem::Requirement
240
268
  requirements:
241
- - - '>='
269
+ - - ">="
242
270
  - !ruby/object:Gem::Version
243
271
  version: '0'
244
272
  type: :development
245
273
  prerelease: false
246
274
  version_requirements: !ruby/object:Gem::Requirement
247
275
  requirements:
248
- - - '>='
276
+ - - ">="
249
277
  - !ruby/object:Gem::Version
250
278
  version: '0'
251
- - !ruby/object:Gem::Dependency
252
- name: vcr
253
- requirement: !ruby/object:Gem::Requirement
254
- requirements:
255
- - - ~>
256
- - !ruby/object:Gem::Version
257
- version: 2.5.0
258
- type: :development
259
- prerelease: false
260
- version_requirements: !ruby/object:Gem::Requirement
261
- requirements:
262
- - - ~>
263
- - !ruby/object:Gem::Version
264
- version: 2.5.0
265
279
  description:
266
280
  email:
267
281
  - info@opennorth.ca
@@ -269,9 +283,9 @@ executables: []
269
283
  extensions: []
270
284
  extra_rdoc_files: []
271
285
  files:
272
- - .gitignore
273
- - .travis.yml
274
- - .yardopts
286
+ - ".gitignore"
287
+ - ".travis.yml"
288
+ - ".yardopts"
275
289
  - Gemfile
276
290
  - LICENSE
277
291
  - README.md
@@ -297,6 +311,9 @@ files:
297
311
  - lib/pupa/models/post.rb
298
312
  - lib/pupa/processor.rb
299
313
  - lib/pupa/processor/client.rb
314
+ - lib/pupa/processor/connection.rb
315
+ - lib/pupa/processor/connection_adapters/mongodb_adapter.rb
316
+ - lib/pupa/processor/connection_adapters/postgresql_adapter.rb
300
317
  - lib/pupa/processor/dependency_graph.rb
301
318
  - lib/pupa/processor/document_store.rb
302
319
  - lib/pupa/processor/document_store/file_store.rb
@@ -307,7 +324,6 @@ files:
307
324
  - lib/pupa/processor/middleware/parse_html.rb
308
325
  - lib/pupa/processor/middleware/parse_json.rb
309
326
  - lib/pupa/processor/middleware/raise_error.rb
310
- - lib/pupa/processor/persistence.rb
311
327
  - lib/pupa/processor/yielder.rb
312
328
  - lib/pupa/refinements/faraday.rb
313
329
  - lib/pupa/refinements/faraday_middleware.rb
@@ -324,13 +340,6 @@ files:
324
340
  - schemas/popolo/other_name.json
325
341
  - schemas/popolo/person.json
326
342
  - schemas/popolo/post.json
327
- - spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
328
- - spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
329
- - spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
330
- - spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
331
- - spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
332
- - spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
333
- - spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
334
343
  - spec/fixtures/bar.json
335
344
  - spec/fixtures/baz.json
336
345
  - spec/fixtures/foo.json
@@ -349,6 +358,9 @@ files:
349
358
  - spec/models/person_spec.rb
350
359
  - spec/models/post_spec.rb
351
360
  - spec/processor/client_spec.rb
361
+ - spec/processor/connection_adapters/mongodb_adapter_spec.rb
362
+ - spec/processor/connection_adapters/postgresql_adapter_spec.rb
363
+ - spec/processor/connection_spec.rb
352
364
  - spec/processor/dependency_graph_spec.rb
353
365
  - spec/processor/document_store/file_store_spec.rb
354
366
  - spec/processor/document_store/redis_store_spec.rb
@@ -356,7 +368,7 @@ files:
356
368
  - spec/processor/helper_spec.rb
357
369
  - spec/processor/middleware/logger_spec.rb
358
370
  - spec/processor/middleware/parse_html_spec.rb
359
- - spec/processor/persistence_spec.rb
371
+ - spec/processor/middleware/parse_json_spec.rb
360
372
  - spec/processor/yielder_spec.rb
361
373
  - spec/processor_spec.rb
362
374
  - spec/refinements/opencivicdata_spec.rb
@@ -372,28 +384,21 @@ require_paths:
372
384
  - lib
373
385
  required_ruby_version: !ruby/object:Gem::Requirement
374
386
  requirements:
375
- - - '>='
387
+ - - ">="
376
388
  - !ruby/object:Gem::Version
377
389
  version: '0'
378
390
  required_rubygems_version: !ruby/object:Gem::Requirement
379
391
  requirements:
380
- - - '>='
392
+ - - ">="
381
393
  - !ruby/object:Gem::Version
382
394
  version: '0'
383
395
  requirements: []
384
396
  rubyforge_project:
385
- rubygems_version: 2.0.8
397
+ rubygems_version: 2.2.2
386
398
  signing_key:
387
399
  specification_version: 4
388
400
  summary: A data scraping framework
389
401
  test_files:
390
- - spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
391
- - spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
392
- - spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
393
- - spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
394
- - spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
395
- - spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
396
- - spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
397
402
  - spec/fixtures/bar.json
398
403
  - spec/fixtures/baz.json
399
404
  - spec/fixtures/foo.json
@@ -412,6 +417,9 @@ test_files:
412
417
  - spec/models/person_spec.rb
413
418
  - spec/models/post_spec.rb
414
419
  - spec/processor/client_spec.rb
420
+ - spec/processor/connection_adapters/mongodb_adapter_spec.rb
421
+ - spec/processor/connection_adapters/postgresql_adapter_spec.rb
422
+ - spec/processor/connection_spec.rb
415
423
  - spec/processor/dependency_graph_spec.rb
416
424
  - spec/processor/document_store/file_store_spec.rb
417
425
  - spec/processor/document_store/redis_store_spec.rb
@@ -419,7 +427,7 @@ test_files:
419
427
  - spec/processor/helper_spec.rb
420
428
  - spec/processor/middleware/logger_spec.rb
421
429
  - spec/processor/middleware/parse_html_spec.rb
422
- - spec/processor/persistence_spec.rb
430
+ - spec/processor/middleware/parse_json_spec.rb
423
431
  - spec/processor/yielder_spec.rb
424
432
  - spec/processor_spec.rb
425
433
  - spec/refinements/opencivicdata_spec.rb