pupa 0.0.13 → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +1 -0
- data/README.md +17 -0
- data/lib/pupa.rb +5 -8
- data/lib/pupa/errors.rb +4 -0
- data/lib/pupa/models/model.rb +5 -5
- data/lib/pupa/models/organization.rb +10 -6
- data/lib/pupa/models/person.rb +10 -6
- data/lib/pupa/processor.rb +14 -15
- data/lib/pupa/processor/connection.rb +26 -0
- data/lib/pupa/processor/connection_adapters/mongodb_adapter.rb +92 -0
- data/lib/pupa/processor/connection_adapters/postgresql_adapter.rb +116 -0
- data/lib/pupa/processor/document_store.rb +3 -0
- data/lib/pupa/processor/middleware/raise_error.rb +1 -0
- data/lib/pupa/refinements/faraday_middleware.rb +1 -1
- data/lib/pupa/runner.rb +14 -21
- data/lib/pupa/version.rb +1 -1
- data/pupa.gemspec +3 -2
- data/schemas/popolo/contact_detail.json +10 -0
- data/schemas/popolo/membership.json +29 -9
- data/schemas/popolo/organization.json +9 -2
- data/schemas/popolo/other_name.json +24 -0
- data/schemas/popolo/person.json +6 -3
- data/schemas/popolo/post.json +16 -2
- data/spec/models/model_spec.rb +1 -1
- data/spec/processor/connection_adapters/mongodb_adapter_spec.rb +61 -0
- data/spec/processor/connection_adapters/postgresql_adapter_spec.rb +70 -0
- data/spec/processor/connection_spec.rb +15 -0
- data/spec/processor/middleware/parse_json_spec.rb +90 -0
- data/spec/processor_spec.rb +9 -10
- data/spec/spec_helper.rb +0 -10
- metadata +83 -75
- data/lib/pupa/processor/persistence.rb +0 -85
- data/spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml +0 -56
- data/spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml +0 -48
- data/spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml +0 -54
- data/spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml +0 -26
- data/spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml +0 -46
- data/spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml +0 -26
- data/spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml +0 -62
- data/spec/processor/persistence_spec.rb +0 -51
@@ -0,0 +1,15 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
|
2
|
+
|
3
|
+
describe Pupa::Processor::Connection do
|
4
|
+
describe '.new' do
|
5
|
+
it 'should use MongoDB' do
|
6
|
+
Pupa::Processor::Connection::MongoDBAdapter.should_receive(:new).with('mongodb://localhost:27017/pupa').and_call_original
|
7
|
+
Pupa::Processor::Connection.new('mongodb://localhost:27017/pupa')
|
8
|
+
end
|
9
|
+
|
10
|
+
it 'should use PostgreSQL' do
|
11
|
+
Pupa::Processor::Connection::PostgreSQLAdapter.should_receive(:new).with('postgres://localhost:5432/pupa').and_call_original
|
12
|
+
Pupa::Processor::Connection.new('postgres://localhost:5432/pupa')
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,90 @@
|
|
1
|
+
require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
|
2
|
+
|
3
|
+
# @see spec/helper.rb and spec/parse_xml_spec.rb in faraday_middleware
|
4
|
+
describe Pupa::Processor::Middleware::ParseJson do
|
5
|
+
let(:options) { Hash.new }
|
6
|
+
let(:headers) { Hash.new }
|
7
|
+
let(:middleware) {
|
8
|
+
described_class.new(lambda {|env|
|
9
|
+
Faraday::Response.new(env)
|
10
|
+
}, options)
|
11
|
+
}
|
12
|
+
|
13
|
+
def process(body, content_type = nil, options = {})
|
14
|
+
env = {
|
15
|
+
:body => body, :request => options,
|
16
|
+
:response_headers => Faraday::Utils::Headers.new(headers)
|
17
|
+
}
|
18
|
+
env[:response_headers]['content-type'] = content_type if content_type
|
19
|
+
middleware.call(env)
|
20
|
+
end
|
21
|
+
|
22
|
+
let(:json) { '{"title":"foo","body":"bar"}' }
|
23
|
+
let(:title) { 'foo' }
|
24
|
+
let(:body) { 'bar' }
|
25
|
+
|
26
|
+
context "no type matching" do
|
27
|
+
it "doesn't change nil body" do
|
28
|
+
expect(process(nil).body).to be_nil
|
29
|
+
end
|
30
|
+
|
31
|
+
it "turns empty body into nil" do
|
32
|
+
expect(process('').body).to be_nil
|
33
|
+
end
|
34
|
+
|
35
|
+
it "parses json body" do
|
36
|
+
response = process(json)
|
37
|
+
expect(response.body['title']).to eq(title)
|
38
|
+
expect(response.body['body']).to eq(body)
|
39
|
+
expect(response.env[:raw_body]).to be_nil
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context "with preserving raw" do
|
44
|
+
let(:options) { {:preserve_raw => true} }
|
45
|
+
|
46
|
+
it "parses json body" do
|
47
|
+
response = process(json)
|
48
|
+
expect(response.body['title']).to eq(title)
|
49
|
+
expect(response.body['body']).to eq(body)
|
50
|
+
expect(response.env[:raw_body]).to eq(json)
|
51
|
+
end
|
52
|
+
|
53
|
+
it "can opt out of preserving raw" do
|
54
|
+
response = process(json, nil, :preserve_raw => false)
|
55
|
+
expect(response.env[:raw_body]).to be_nil
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
context "with regexp type matching" do
|
60
|
+
let(:options) { {:content_type => /\bjson$/} }
|
61
|
+
|
62
|
+
it "parses json body of correct type" do
|
63
|
+
response = process(json, 'application/json')
|
64
|
+
expect(response.body['title']).to eq(title)
|
65
|
+
expect(response.body['body']).to eq(body)
|
66
|
+
end
|
67
|
+
|
68
|
+
it "ignores json body of incorrect type" do
|
69
|
+
response = process(json, 'application/xml')
|
70
|
+
expect(response.body).to eq(json)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
context "with array type matching" do
|
75
|
+
let(:options) { {:content_type => %w[a/b c/d]} }
|
76
|
+
|
77
|
+
it "parses json body of correct type" do
|
78
|
+
expect(process(json, 'a/b').body).to be_a(Hash)
|
79
|
+
expect(process(json, 'c/d').body).to be_a(Hash)
|
80
|
+
end
|
81
|
+
|
82
|
+
it "ignores json body of incorrect type" do
|
83
|
+
expect(process(json, 'a/d').body).not_to be_a(Hash)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
|
87
|
+
it "chokes on invalid json" do
|
88
|
+
expect{ process('{') }.to raise_error
|
89
|
+
end
|
90
|
+
end
|
data/spec/processor_spec.rb
CHANGED
@@ -11,7 +11,7 @@ describe Pupa::Processor do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
def make_person_invalid
|
14
|
-
@person = Pupa::Person.new
|
14
|
+
@person = Pupa::Person.new(name: ['foo'])
|
15
15
|
end
|
16
16
|
|
17
17
|
def scrape_people
|
@@ -28,11 +28,11 @@ describe Pupa::Processor do
|
|
28
28
|
end
|
29
29
|
|
30
30
|
let :processor do
|
31
|
-
PersonProcessor.new('/tmp', level: 'WARN', logdev: io)
|
31
|
+
PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io)
|
32
32
|
end
|
33
33
|
|
34
34
|
let :novalidate do
|
35
|
-
PersonProcessor.new('/tmp', level: 'WARN', logdev: io, validate: false)
|
35
|
+
PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io, validate: false)
|
36
36
|
end
|
37
37
|
|
38
38
|
describe '#get' do
|
@@ -98,8 +98,7 @@ describe Pupa::Processor do
|
|
98
98
|
|
99
99
|
describe '#import' do
|
100
100
|
before :each do
|
101
|
-
|
102
|
-
Pupa.session.collections.each(&:drop)
|
101
|
+
processor.connection.raw_connection[:organizations].drop
|
103
102
|
end
|
104
103
|
|
105
104
|
let :_type do
|
@@ -176,7 +175,7 @@ describe Pupa::Processor do
|
|
176
175
|
processor.should_receive(:load_scraped_objects).and_return(graphable)
|
177
176
|
|
178
177
|
processor.import
|
179
|
-
documents =
|
178
|
+
documents = processor.connection.raw_connection[:organizations].find.entries
|
180
179
|
documents.size.should == 2
|
181
180
|
documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
|
182
181
|
documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
|
@@ -193,7 +192,7 @@ describe Pupa::Processor do
|
|
193
192
|
processor.should_receive(:load_scraped_objects).and_return(ungraphable)
|
194
193
|
|
195
194
|
processor.import
|
196
|
-
documents =
|
195
|
+
documents = processor.connection.raw_connection[:organizations].find.entries
|
197
196
|
documents.size.should == 2
|
198
197
|
documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '5', '_type' => _type, 'name' => 'Parent'}
|
199
198
|
documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '4', '_type' => _type, 'name' => 'Child', 'parent_id' => '5'}
|
@@ -203,7 +202,7 @@ describe Pupa::Processor do
|
|
203
202
|
processor.should_receive(:load_scraped_objects).and_return(foreign_keys_on_foreign_objects)
|
204
203
|
|
205
204
|
processor.import
|
206
|
-
documents =
|
205
|
+
documents = processor.connection.raw_connection[:organizations].find.entries
|
207
206
|
documents.size.should == 3
|
208
207
|
documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '9', '_type' => _type, 'name' => 'Parent'}
|
209
208
|
documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '7', '_type' => _type, 'name' => 'Child', 'parent_id' => '9'}
|
@@ -315,7 +314,7 @@ describe Pupa::Processor do
|
|
315
314
|
processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_key)
|
316
315
|
|
317
316
|
processor.import
|
318
|
-
documents =
|
317
|
+
documents = processor.connection.raw_connection[:organizations].find.entries
|
319
318
|
documents.size.should == 2
|
320
319
|
documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
|
321
320
|
documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
|
@@ -340,7 +339,7 @@ describe Pupa::Processor do
|
|
340
339
|
processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_keys_on_foreign_objects)
|
341
340
|
|
342
341
|
processor.import
|
343
|
-
documents =
|
342
|
+
documents = processor.connection.raw_connection[:organizations].find.entries
|
344
343
|
documents.size.should == 3
|
345
344
|
documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
|
346
345
|
documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
|
data/spec/spec_helper.rb
CHANGED
@@ -7,18 +7,8 @@ require 'multi_xml'
|
|
7
7
|
require 'nokogiri'
|
8
8
|
require 'redis-store'
|
9
9
|
require 'rspec'
|
10
|
-
require 'vcr'
|
11
10
|
require File.dirname(__FILE__) + '/../lib/pupa'
|
12
11
|
|
13
|
-
VCR.configure do |c|
|
14
|
-
c.cassette_library_dir = 'spec/cassettes'
|
15
|
-
c.hook_into :faraday
|
16
|
-
|
17
|
-
c.around_http_request do |request|
|
18
|
-
VCR.use_cassette(Digest::SHA1.hexdigest(request.uri + request.body + request.headers.to_s), &request)
|
19
|
-
end
|
20
|
-
end
|
21
|
-
|
22
12
|
def testing_python_compatibility?
|
23
13
|
ENV['MODE'] == 'compat'
|
24
14
|
end
|
metadata
CHANGED
@@ -1,267 +1,281 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: pupa
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Open North
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2014-
|
11
|
+
date: 2014-05-11 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- - ~>
|
17
|
+
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: 4.0.0
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- - ~>
|
24
|
+
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 4.0.0
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: colored
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- - ~>
|
31
|
+
- - "~>"
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '1.2'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- - ~>
|
38
|
+
- - "~>"
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '1.2'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: faraday_middleware
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- - ~>
|
45
|
+
- - "~>"
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: 0.9.0
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- - ~>
|
52
|
+
- - "~>"
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: 0.9.0
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: json-schema
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- - ~>
|
59
|
+
- - "~>"
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: 2.1.3
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- - ~>
|
66
|
+
- - "~>"
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: 2.1.3
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: mail
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - ">="
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - ">="
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: moped
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- - ~>
|
87
|
+
- - "~>"
|
88
88
|
- !ruby/object:Gem::Version
|
89
|
-
version:
|
89
|
+
version: 2.0.0.rc1
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- - ~>
|
94
|
+
- - "~>"
|
95
95
|
- !ruby/object:Gem::Version
|
96
|
-
version:
|
96
|
+
version: 2.0.0.rc1
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: oj
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- - ~>
|
101
|
+
- - "~>"
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '2.1'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- - ~>
|
108
|
+
- - "~>"
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '2.1'
|
111
|
+
- !ruby/object:Gem::Dependency
|
112
|
+
name: sequel
|
113
|
+
requirement: !ruby/object:Gem::Requirement
|
114
|
+
requirements:
|
115
|
+
- - "~>"
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: 4.10.0
|
118
|
+
type: :runtime
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
requirements:
|
122
|
+
- - "~>"
|
123
|
+
- !ruby/object:Gem::Version
|
124
|
+
version: 4.10.0
|
125
|
+
- !ruby/object:Gem::Dependency
|
126
|
+
name: pg
|
127
|
+
requirement: !ruby/object:Gem::Requirement
|
128
|
+
requirements:
|
129
|
+
- - "~>"
|
130
|
+
- !ruby/object:Gem::Version
|
131
|
+
version: 0.17.0
|
132
|
+
type: :runtime
|
133
|
+
prerelease: false
|
134
|
+
version_requirements: !ruby/object:Gem::Requirement
|
135
|
+
requirements:
|
136
|
+
- - "~>"
|
137
|
+
- !ruby/object:Gem::Version
|
138
|
+
version: 0.17.0
|
111
139
|
- !ruby/object:Gem::Dependency
|
112
140
|
name: coveralls
|
113
141
|
requirement: !ruby/object:Gem::Requirement
|
114
142
|
requirements:
|
115
|
-
- -
|
143
|
+
- - ">="
|
116
144
|
- !ruby/object:Gem::Version
|
117
145
|
version: '0'
|
118
146
|
type: :development
|
119
147
|
prerelease: false
|
120
148
|
version_requirements: !ruby/object:Gem::Requirement
|
121
149
|
requirements:
|
122
|
-
- -
|
150
|
+
- - ">="
|
123
151
|
- !ruby/object:Gem::Version
|
124
152
|
version: '0'
|
125
153
|
- !ruby/object:Gem::Dependency
|
126
154
|
name: dalli
|
127
155
|
requirement: !ruby/object:Gem::Requirement
|
128
156
|
requirements:
|
129
|
-
- -
|
157
|
+
- - ">="
|
130
158
|
- !ruby/object:Gem::Version
|
131
159
|
version: '0'
|
132
160
|
type: :development
|
133
161
|
prerelease: false
|
134
162
|
version_requirements: !ruby/object:Gem::Requirement
|
135
163
|
requirements:
|
136
|
-
- -
|
164
|
+
- - ">="
|
137
165
|
- !ruby/object:Gem::Version
|
138
166
|
version: '0'
|
139
167
|
- !ruby/object:Gem::Dependency
|
140
168
|
name: json
|
141
169
|
requirement: !ruby/object:Gem::Requirement
|
142
170
|
requirements:
|
143
|
-
- - ~>
|
171
|
+
- - "~>"
|
144
172
|
- !ruby/object:Gem::Version
|
145
173
|
version: 1.7.7
|
146
174
|
type: :development
|
147
175
|
prerelease: false
|
148
176
|
version_requirements: !ruby/object:Gem::Requirement
|
149
177
|
requirements:
|
150
|
-
- - ~>
|
178
|
+
- - "~>"
|
151
179
|
- !ruby/object:Gem::Version
|
152
180
|
version: 1.7.7
|
153
181
|
- !ruby/object:Gem::Dependency
|
154
182
|
name: multi_xml
|
155
183
|
requirement: !ruby/object:Gem::Requirement
|
156
184
|
requirements:
|
157
|
-
- -
|
185
|
+
- - ">="
|
158
186
|
- !ruby/object:Gem::Version
|
159
187
|
version: '0'
|
160
188
|
type: :development
|
161
189
|
prerelease: false
|
162
190
|
version_requirements: !ruby/object:Gem::Requirement
|
163
191
|
requirements:
|
164
|
-
- -
|
192
|
+
- - ">="
|
165
193
|
- !ruby/object:Gem::Version
|
166
194
|
version: '0'
|
167
195
|
- !ruby/object:Gem::Dependency
|
168
196
|
name: nokogiri
|
169
197
|
requirement: !ruby/object:Gem::Requirement
|
170
198
|
requirements:
|
171
|
-
- - ~>
|
199
|
+
- - "~>"
|
172
200
|
- !ruby/object:Gem::Version
|
173
201
|
version: 1.6.0
|
174
202
|
type: :development
|
175
203
|
prerelease: false
|
176
204
|
version_requirements: !ruby/object:Gem::Requirement
|
177
205
|
requirements:
|
178
|
-
- - ~>
|
206
|
+
- - "~>"
|
179
207
|
- !ruby/object:Gem::Version
|
180
208
|
version: 1.6.0
|
181
209
|
- !ruby/object:Gem::Dependency
|
182
210
|
name: octokit
|
183
211
|
requirement: !ruby/object:Gem::Requirement
|
184
212
|
requirements:
|
185
|
-
- -
|
213
|
+
- - ">="
|
186
214
|
- !ruby/object:Gem::Version
|
187
215
|
version: '0'
|
188
216
|
type: :development
|
189
217
|
prerelease: false
|
190
218
|
version_requirements: !ruby/object:Gem::Requirement
|
191
219
|
requirements:
|
192
|
-
- -
|
220
|
+
- - ">="
|
193
221
|
- !ruby/object:Gem::Version
|
194
222
|
version: '0'
|
195
223
|
- !ruby/object:Gem::Dependency
|
196
224
|
name: rake
|
197
225
|
requirement: !ruby/object:Gem::Requirement
|
198
226
|
requirements:
|
199
|
-
- -
|
227
|
+
- - ">="
|
200
228
|
- !ruby/object:Gem::Version
|
201
229
|
version: '0'
|
202
230
|
type: :development
|
203
231
|
prerelease: false
|
204
232
|
version_requirements: !ruby/object:Gem::Requirement
|
205
233
|
requirements:
|
206
|
-
- -
|
234
|
+
- - ">="
|
207
235
|
- !ruby/object:Gem::Version
|
208
236
|
version: '0'
|
209
237
|
- !ruby/object:Gem::Dependency
|
210
238
|
name: redis-store
|
211
239
|
requirement: !ruby/object:Gem::Requirement
|
212
240
|
requirements:
|
213
|
-
- -
|
241
|
+
- - ">="
|
214
242
|
- !ruby/object:Gem::Version
|
215
243
|
version: '0'
|
216
244
|
type: :development
|
217
245
|
prerelease: false
|
218
246
|
version_requirements: !ruby/object:Gem::Requirement
|
219
247
|
requirements:
|
220
|
-
- -
|
248
|
+
- - ">="
|
221
249
|
- !ruby/object:Gem::Version
|
222
250
|
version: '0'
|
223
251
|
- !ruby/object:Gem::Dependency
|
224
252
|
name: rspec
|
225
253
|
requirement: !ruby/object:Gem::Requirement
|
226
254
|
requirements:
|
227
|
-
- - ~>
|
255
|
+
- - "~>"
|
228
256
|
- !ruby/object:Gem::Version
|
229
257
|
version: '2.10'
|
230
258
|
type: :development
|
231
259
|
prerelease: false
|
232
260
|
version_requirements: !ruby/object:Gem::Requirement
|
233
261
|
requirements:
|
234
|
-
- - ~>
|
262
|
+
- - "~>"
|
235
263
|
- !ruby/object:Gem::Version
|
236
264
|
version: '2.10'
|
237
265
|
- !ruby/object:Gem::Dependency
|
238
266
|
name: typhoeus
|
239
267
|
requirement: !ruby/object:Gem::Requirement
|
240
268
|
requirements:
|
241
|
-
- -
|
269
|
+
- - ">="
|
242
270
|
- !ruby/object:Gem::Version
|
243
271
|
version: '0'
|
244
272
|
type: :development
|
245
273
|
prerelease: false
|
246
274
|
version_requirements: !ruby/object:Gem::Requirement
|
247
275
|
requirements:
|
248
|
-
- -
|
276
|
+
- - ">="
|
249
277
|
- !ruby/object:Gem::Version
|
250
278
|
version: '0'
|
251
|
-
- !ruby/object:Gem::Dependency
|
252
|
-
name: vcr
|
253
|
-
requirement: !ruby/object:Gem::Requirement
|
254
|
-
requirements:
|
255
|
-
- - ~>
|
256
|
-
- !ruby/object:Gem::Version
|
257
|
-
version: 2.5.0
|
258
|
-
type: :development
|
259
|
-
prerelease: false
|
260
|
-
version_requirements: !ruby/object:Gem::Requirement
|
261
|
-
requirements:
|
262
|
-
- - ~>
|
263
|
-
- !ruby/object:Gem::Version
|
264
|
-
version: 2.5.0
|
265
279
|
description:
|
266
280
|
email:
|
267
281
|
- info@opennorth.ca
|
@@ -269,9 +283,9 @@ executables: []
|
|
269
283
|
extensions: []
|
270
284
|
extra_rdoc_files: []
|
271
285
|
files:
|
272
|
-
- .gitignore
|
273
|
-
- .travis.yml
|
274
|
-
- .yardopts
|
286
|
+
- ".gitignore"
|
287
|
+
- ".travis.yml"
|
288
|
+
- ".yardopts"
|
275
289
|
- Gemfile
|
276
290
|
- LICENSE
|
277
291
|
- README.md
|
@@ -297,6 +311,9 @@ files:
|
|
297
311
|
- lib/pupa/models/post.rb
|
298
312
|
- lib/pupa/processor.rb
|
299
313
|
- lib/pupa/processor/client.rb
|
314
|
+
- lib/pupa/processor/connection.rb
|
315
|
+
- lib/pupa/processor/connection_adapters/mongodb_adapter.rb
|
316
|
+
- lib/pupa/processor/connection_adapters/postgresql_adapter.rb
|
300
317
|
- lib/pupa/processor/dependency_graph.rb
|
301
318
|
- lib/pupa/processor/document_store.rb
|
302
319
|
- lib/pupa/processor/document_store/file_store.rb
|
@@ -307,7 +324,6 @@ files:
|
|
307
324
|
- lib/pupa/processor/middleware/parse_html.rb
|
308
325
|
- lib/pupa/processor/middleware/parse_json.rb
|
309
326
|
- lib/pupa/processor/middleware/raise_error.rb
|
310
|
-
- lib/pupa/processor/persistence.rb
|
311
327
|
- lib/pupa/processor/yielder.rb
|
312
328
|
- lib/pupa/refinements/faraday.rb
|
313
329
|
- lib/pupa/refinements/faraday_middleware.rb
|
@@ -324,13 +340,6 @@ files:
|
|
324
340
|
- schemas/popolo/other_name.json
|
325
341
|
- schemas/popolo/person.json
|
326
342
|
- schemas/popolo/post.json
|
327
|
-
- spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
|
328
|
-
- spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
|
329
|
-
- spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
|
330
|
-
- spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
|
331
|
-
- spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
|
332
|
-
- spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
|
333
|
-
- spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
|
334
343
|
- spec/fixtures/bar.json
|
335
344
|
- spec/fixtures/baz.json
|
336
345
|
- spec/fixtures/foo.json
|
@@ -349,6 +358,9 @@ files:
|
|
349
358
|
- spec/models/person_spec.rb
|
350
359
|
- spec/models/post_spec.rb
|
351
360
|
- spec/processor/client_spec.rb
|
361
|
+
- spec/processor/connection_adapters/mongodb_adapter_spec.rb
|
362
|
+
- spec/processor/connection_adapters/postgresql_adapter_spec.rb
|
363
|
+
- spec/processor/connection_spec.rb
|
352
364
|
- spec/processor/dependency_graph_spec.rb
|
353
365
|
- spec/processor/document_store/file_store_spec.rb
|
354
366
|
- spec/processor/document_store/redis_store_spec.rb
|
@@ -356,7 +368,7 @@ files:
|
|
356
368
|
- spec/processor/helper_spec.rb
|
357
369
|
- spec/processor/middleware/logger_spec.rb
|
358
370
|
- spec/processor/middleware/parse_html_spec.rb
|
359
|
-
- spec/processor/
|
371
|
+
- spec/processor/middleware/parse_json_spec.rb
|
360
372
|
- spec/processor/yielder_spec.rb
|
361
373
|
- spec/processor_spec.rb
|
362
374
|
- spec/refinements/opencivicdata_spec.rb
|
@@ -372,28 +384,21 @@ require_paths:
|
|
372
384
|
- lib
|
373
385
|
required_ruby_version: !ruby/object:Gem::Requirement
|
374
386
|
requirements:
|
375
|
-
- -
|
387
|
+
- - ">="
|
376
388
|
- !ruby/object:Gem::Version
|
377
389
|
version: '0'
|
378
390
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
379
391
|
requirements:
|
380
|
-
- -
|
392
|
+
- - ">="
|
381
393
|
- !ruby/object:Gem::Version
|
382
394
|
version: '0'
|
383
395
|
requirements: []
|
384
396
|
rubyforge_project:
|
385
|
-
rubygems_version: 2.
|
397
|
+
rubygems_version: 2.2.2
|
386
398
|
signing_key:
|
387
399
|
specification_version: 4
|
388
400
|
summary: A data scraping framework
|
389
401
|
test_files:
|
390
|
-
- spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
|
391
|
-
- spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
|
392
|
-
- spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
|
393
|
-
- spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
|
394
|
-
- spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
|
395
|
-
- spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
|
396
|
-
- spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
|
397
402
|
- spec/fixtures/bar.json
|
398
403
|
- spec/fixtures/baz.json
|
399
404
|
- spec/fixtures/foo.json
|
@@ -412,6 +417,9 @@ test_files:
|
|
412
417
|
- spec/models/person_spec.rb
|
413
418
|
- spec/models/post_spec.rb
|
414
419
|
- spec/processor/client_spec.rb
|
420
|
+
- spec/processor/connection_adapters/mongodb_adapter_spec.rb
|
421
|
+
- spec/processor/connection_adapters/postgresql_adapter_spec.rb
|
422
|
+
- spec/processor/connection_spec.rb
|
415
423
|
- spec/processor/dependency_graph_spec.rb
|
416
424
|
- spec/processor/document_store/file_store_spec.rb
|
417
425
|
- spec/processor/document_store/redis_store_spec.rb
|
@@ -419,7 +427,7 @@ test_files:
|
|
419
427
|
- spec/processor/helper_spec.rb
|
420
428
|
- spec/processor/middleware/logger_spec.rb
|
421
429
|
- spec/processor/middleware/parse_html_spec.rb
|
422
|
-
- spec/processor/
|
430
|
+
- spec/processor/middleware/parse_json_spec.rb
|
423
431
|
- spec/processor/yielder_spec.rb
|
424
432
|
- spec/processor_spec.rb
|
425
433
|
- spec/refinements/opencivicdata_spec.rb
|