pupa 0.0.13 → 0.1.0

data/spec/processor/connection_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Pupa::Processor::Connection do
+  describe '.new' do
+    it 'should use MongoDB' do
+      Pupa::Processor::Connection::MongoDBAdapter.should_receive(:new).with('mongodb://localhost:27017/pupa').and_call_original
+      Pupa::Processor::Connection.new('mongodb://localhost:27017/pupa')
+    end
+
+    it 'should use PostgreSQL' do
+      Pupa::Processor::Connection::PostgreSQLAdapter.should_receive(:new).with('postgres://localhost:5432/pupa').and_call_original
+      Pupa::Processor::Connection.new('postgres://localhost:5432/pupa')
+    end
+  end
+end

data/spec/processor/middleware/parse_json_spec.rb

@@ -0,0 +1,90 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+# @see spec/helper.rb and spec/parse_xml_spec.rb in faraday_middleware
+describe Pupa::Processor::Middleware::ParseJson do
+  let(:options) { Hash.new }
+  let(:headers) { Hash.new }
+  let(:middleware) {
+    described_class.new(lambda {|env|
+      Faraday::Response.new(env)
+    }, options)
+  }
+
+  def process(body, content_type = nil, options = {})
+    env = {
+      :body => body, :request => options,
+      :response_headers => Faraday::Utils::Headers.new(headers)
+    }
+    env[:response_headers]['content-type'] = content_type if content_type
+    middleware.call(env)
+  end
+
+  let(:json)  { '{"title":"foo","body":"bar"}' }
+  let(:title) { 'foo' }
+  let(:body) { 'bar' }
+
+  context "no type matching" do
+    it "doesn't change nil body" do
+      expect(process(nil).body).to be_nil
+    end
+
+    it "turns empty body into nil" do
+      expect(process('').body).to be_nil
+    end
+
+    it "parses json body" do
+      response = process(json)
+      expect(response.body['title']).to eq(title)
+      expect(response.body['body']).to eq(body)
+      expect(response.env[:raw_body]).to be_nil
+    end
+  end
+
+  context "with preserving raw" do
+    let(:options) { {:preserve_raw => true} }
+
+    it "parses json body" do
+      response = process(json)
+      expect(response.body['title']).to eq(title)
+      expect(response.body['body']).to eq(body)
+      expect(response.env[:raw_body]).to eq(json)
+    end
+
+    it "can opt out of preserving raw" do
+      response = process(json, nil, :preserve_raw => false)
+      expect(response.env[:raw_body]).to be_nil
+    end
+  end
+
+  context "with regexp type matching" do
+    let(:options) { {:content_type => /\bjson$/} }
+
+    it "parses json body of correct type" do
+      response = process(json, 'application/json')
+      expect(response.body['title']).to eq(title)
+      expect(response.body['body']).to eq(body)
+    end
+
+    it "ignores json body of incorrect type" do
+      response = process(json, 'application/xml')
+      expect(response.body).to eq(json)
+    end
+  end
+
+  context "with array type matching" do
+    let(:options) { {:content_type => %w[a/b c/d]} }
+
+    it "parses json body of correct type" do
+      expect(process(json, 'a/b').body).to be_a(Hash)
+      expect(process(json, 'c/d').body).to be_a(Hash)
+    end
+
+    it "ignores json body of incorrect type" do
+      expect(process(json, 'a/d').body).not_to be_a(Hash)
+    end
+  end
+
+  it "chokes on invalid json" do
+    expect{ process('{') }.to raise_error
+  end
+end

data/spec/processor_spec.rb

@@ -11,7 +11,7 @@ describe Pupa::Processor do
     end
 
     def make_person_invalid
-      @person = Pupa::Person.new
+      @person = Pupa::Person.new(name: ['foo'])
     end
 
     def scrape_people
@@ -28,11 +28,11 @@ describe Pupa::Processor do
   end
 
   let :processor do
-    PersonProcessor.new('/tmp', level: 'WARN', logdev: io)
+    PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io)
   end
 
   let :novalidate do
-    PersonProcessor.new('/tmp', level: 'WARN', logdev: io, validate: false)
+    PersonProcessor.new('/tmp', database_url: 'mongodb://localhost:27017/pupa_test', level: 'WARN', logdev: io, validate: false)
   end
 
   describe '#get' do
@@ -98,8 +98,7 @@ describe Pupa::Processor do
 
   describe '#import' do
     before :each do
-      Pupa.session = Moped::Session.new(['localhost:27017'], database: 'pupa_test')
-      Pupa.session.collections.each(&:drop)
+      processor.connection.raw_connection[:organizations].drop
     end
 
     let :_type do
@@ -176,7 +175,7 @@ describe Pupa::Processor do
       processor.should_receive(:load_scraped_objects).and_return(graphable)
 
       processor.import
-      documents = Pupa.session[:organizations].find.entries
+      documents = processor.connection.raw_connection[:organizations].find.entries
       documents.size.should == 2
       documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
       documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
@@ -193,7 +192,7 @@ describe Pupa::Processor do
       processor.should_receive(:load_scraped_objects).and_return(ungraphable)
 
       processor.import
-      documents = Pupa.session[:organizations].find.entries
+      documents = processor.connection.raw_connection[:organizations].find.entries
       documents.size.should == 2
       documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '5', '_type' => _type, 'name' => 'Parent'}
       documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '4', '_type' => _type, 'name' => 'Child', 'parent_id' => '5'}
@@ -203,7 +202,7 @@ describe Pupa::Processor do
       processor.should_receive(:load_scraped_objects).and_return(foreign_keys_on_foreign_objects)
 
       processor.import
-      documents = Pupa.session[:organizations].find.entries
+      documents = processor.connection.raw_connection[:organizations].find.entries
       documents.size.should == 3
       documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '9', '_type' => _type, 'name' => 'Parent'}
       documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '7', '_type' => _type, 'name' => 'Child', 'parent_id' => '9'}
@@ -315,7 +314,7 @@ describe Pupa::Processor do
         processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_key)
 
         processor.import
-        documents = Pupa.session[:organizations].find.entries
+        documents = processor.connection.raw_connection[:organizations].find.entries
         documents.size.should == 2
         documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
         documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
@@ -340,7 +339,7 @@ describe Pupa::Processor do
         processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_keys_on_foreign_objects)
 
         processor.import
-        documents = Pupa.session[:organizations].find.entries
+        documents = processor.connection.raw_connection[:organizations].find.entries
         documents.size.should == 3
         documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
         documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}

data/spec/spec_helper.rb

@@ -7,18 +7,8 @@ require 'multi_xml'
 require 'nokogiri'
 require 'redis-store'
 require 'rspec'
-require 'vcr'
 require File.dirname(__FILE__) + '/../lib/pupa'
 
-VCR.configure do |c|
-  c.cassette_library_dir = 'spec/cassettes'
-  c.hook_into :faraday
-
-  c.around_http_request do |request|
-    VCR.use_cassette(Digest::SHA1.hexdigest(request.uri + request.body + request.headers.to_s), &request)
-  end
-end
-
 def testing_python_compatibility?
   ENV['MODE'] == 'compat'
 end

metadata

@@ -1,267 +1,281 @@
 --- !ruby/object:Gem::Specification
 name: pupa
 version: !ruby/object:Gem::Version
-  version: 0.0.13
+  version: 0.1.0
 platform: ruby
 authors:
 - Open North
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-02-11 00:00:00.000000000 Z
+date: 2014-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 4.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 4.0.0
 - !ruby/object:Gem::Dependency
   name: colored
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: faraday_middleware
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.1.3
 - !ruby/object:Gem::Dependency
   name: mail
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: moped
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.1
+        version: 2.0.0.rc1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.5.1
+        version: 2.0.0.rc1
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 4.10.0
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.17.0
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: dalli
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.7.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.7.7
 - !ruby/object:Gem::Dependency
   name: multi_xml
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.6.0
 - !ruby/object:Gem::Dependency
   name: octokit
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: redis-store
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.10'
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: vcr
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 2.5.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 2.5.0
 description: 
 email:
 - info@opennorth.ca
@@ -269,9 +283,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
-- .yardopts
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
 - Gemfile
 - LICENSE
 - README.md
@@ -297,6 +311,9 @@ files:
 - lib/pupa/models/post.rb
 - lib/pupa/processor.rb
 - lib/pupa/processor/client.rb
+- lib/pupa/processor/connection.rb
+- lib/pupa/processor/connection_adapters/mongodb_adapter.rb
+- lib/pupa/processor/connection_adapters/postgresql_adapter.rb
 - lib/pupa/processor/dependency_graph.rb
 - lib/pupa/processor/document_store.rb
 - lib/pupa/processor/document_store/file_store.rb
@@ -307,7 +324,6 @@ files:
 - lib/pupa/processor/middleware/parse_html.rb
 - lib/pupa/processor/middleware/parse_json.rb
 - lib/pupa/processor/middleware/raise_error.rb
-- lib/pupa/processor/persistence.rb
 - lib/pupa/processor/yielder.rb
 - lib/pupa/refinements/faraday.rb
 - lib/pupa/refinements/faraday_middleware.rb
@@ -324,13 +340,6 @@ files:
 - schemas/popolo/other_name.json
 - schemas/popolo/person.json
 - schemas/popolo/post.json
-- spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
-- spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
-- spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
-- spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
-- spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
-- spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
-- spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
 - spec/fixtures/bar.json
 - spec/fixtures/baz.json
 - spec/fixtures/foo.json
@@ -349,6 +358,9 @@ files:
 - spec/models/person_spec.rb
 - spec/models/post_spec.rb
 - spec/processor/client_spec.rb
+- spec/processor/connection_adapters/mongodb_adapter_spec.rb
+- spec/processor/connection_adapters/postgresql_adapter_spec.rb
+- spec/processor/connection_spec.rb
 - spec/processor/dependency_graph_spec.rb
 - spec/processor/document_store/file_store_spec.rb
 - spec/processor/document_store/redis_store_spec.rb
@@ -356,7 +368,7 @@ files:
 - spec/processor/helper_spec.rb
 - spec/processor/middleware/logger_spec.rb
 - spec/processor/middleware/parse_html_spec.rb
-- spec/processor/persistence_spec.rb
+- spec/processor/middleware/parse_json_spec.rb
 - spec/processor/yielder_spec.rb
 - spec/processor_spec.rb
 - spec/refinements/opencivicdata_spec.rb
@@ -372,28 +384,21 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.8
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: A data scraping framework
 test_files:
-- spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml
-- spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml
-- spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml
-- spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml
-- spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml
-- spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml
-- spec/cassettes/f861172f1df3bdb2052af5451f9922699d574b77.yml
 - spec/fixtures/bar.json
 - spec/fixtures/baz.json
 - spec/fixtures/foo.json
@@ -412,6 +417,9 @@ test_files:
 - spec/models/person_spec.rb
 - spec/models/post_spec.rb
 - spec/processor/client_spec.rb
+- spec/processor/connection_adapters/mongodb_adapter_spec.rb
+- spec/processor/connection_adapters/postgresql_adapter_spec.rb
+- spec/processor/connection_spec.rb
 - spec/processor/dependency_graph_spec.rb
 - spec/processor/document_store/file_store_spec.rb
 - spec/processor/document_store/redis_store_spec.rb
@@ -419,7 +427,7 @@ test_files:
 - spec/processor/helper_spec.rb
 - spec/processor/middleware/logger_spec.rb
 - spec/processor/middleware/parse_html_spec.rb
-- spec/processor/persistence_spec.rb
+- spec/processor/middleware/parse_json_spec.rb
 - spec/processor/yielder_spec.rb
 - spec/processor_spec.rb
 - spec/refinements/opencivicdata_spec.rb