pupa 0.1.11 → 0.2.0

data/spec/processor_spec.rb

@@ -37,28 +37,28 @@ describe Pupa::Processor do
 
   describe '#get' do
     it 'should send a GET request' do
-      processor.get('http://httpbin.org/get', 'foo=bar')['args'].should == {'foo' => 'bar'}
+      expect(processor.get('http://httpbin.org/get', 'foo=bar')['args']).to eq({'foo' => 'bar'})
     end
 
     it 'should automatically parse the response' do
-      processor.get('http://httpbin.org/get').should be_a(Hash)
+      expect(processor.get('http://httpbin.org/get')).to be_a(Hash)
     end
   end
 
   describe '#post' do
     it 'should send a POST request' do
-      processor.post('http://httpbin.org/post', 'foo=bar')['form'].should == {'foo' => 'bar'}
+      expect(processor.post('http://httpbin.org/post', 'foo=bar')['form']).to eq({'foo' => 'bar'})
     end
 
     it 'should automatically parse the response' do
-      processor.post('http://httpbin.org/post').should be_a(Hash)
+      expect(processor.post('http://httpbin.org/post')).to be_a(Hash)
     end
   end
 
   describe '.add_scraping_task' do
     it 'should add a scraping task and define a lazy method' do
-      PersonProcessor.tasks.should == [:people]
-      processor.should respond_to(:people)
+      expect(PersonProcessor.tasks).to eq([:people])
+      expect(processor).to respond_to(:people)
     end
   end
 
@@ -68,7 +68,7 @@ describe Pupa::Processor do
     end
 
     it 'should return the number of scraped objects by type' do
-      processor.dump_scraped_objects(:people).should == {'pupa/person' => 1}
+      expect(processor.dump_scraped_objects(:people)).to eq({'pupa/person' => 1})
     end
 
     it 'should not overwrite an existing file' do
@@ -79,20 +79,20 @@ describe Pupa::Processor do
 
     it 'should dump a JSON document' do
       processor.dump_scraped_objects(:people)
-      File.exist?(path).should == true
-      io.string.should_not match('http://www.popoloproject.com/schemas/person.json')
+      expect(File.exist?(path)).to eq(true)
+      expect(io.string).not_to match("The property '#/")
     end
 
     it 'should validate the object' do
       processor.make_person_invalid
       processor.dump_scraped_objects(:people)
-      io.string.should match('http://www.popoloproject.com/schemas/person.json')
+      expect(io.string).to match("The property '#/name' of type array did not match one or more of the following types: string, null")
     end
 
     it 'should not validate the object' do
       novalidate.make_person_invalid
       novalidate.dump_scraped_objects(:people)
-      io.string.should_not match('http://www.popoloproject.com/schemas/person.json')
+      expect(io.string).not_to match("The property '#/name' of type array did not match one or more of the following types: string, null")
     end
   end
 
@@ -161,53 +161,53 @@ describe Pupa::Processor do
     end
 
     it 'should use a dependency graph if possible' do
-      processor.should_receive(:load_scraped_objects).and_return(graphable)
+      expect(processor).to receive(:load_scraped_objects).and_return(graphable)
 
-      Pupa::Processor::DependencyGraph.any_instance.should_receive(:tsort).and_return(['2', '1'])
+      expect_any_instance_of(Pupa::Processor::DependencyGraph).to receive(:tsort).and_return(['2', '1'])
       processor.import
     end
 
     it 'should remove duplicate objects and re-assign foreign keys' do
-      processor.should_receive(:load_scraped_objects).and_return(graphable)
+      expect(processor).to receive(:load_scraped_objects).and_return(graphable)
 
       processor.import
       documents = processor.connection.raw_connection[:organizations].find.entries
-      documents.size.should == 2
-      documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
-      documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
+      expect(documents.size).to eq(2)
+      expect(documents[0].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '2', '_type' => _type, 'name' => 'Parent'})
+      expect(documents[1].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'})
     end
 
     it 'should not use a dependency graph if not possible' do
-      processor.should_receive(:load_scraped_objects).and_return(ungraphable)
+      expect(processor).to receive(:load_scraped_objects).and_return(ungraphable)
 
-      Pupa::Processor::DependencyGraph.any_instance.should_not_receive(:tsort)
+      expect_any_instance_of(Pupa::Processor::DependencyGraph).not_to receive(:tsort)
       processor.import
     end
 
     it 'should remove duplicate objects and resolve foreign objects' do
-      processor.should_receive(:load_scraped_objects).and_return(ungraphable)
+      expect(processor).to receive(:load_scraped_objects).and_return(ungraphable)
 
       processor.import
       documents = processor.connection.raw_connection[:organizations].find.entries
-      documents.size.should == 2
-      documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '5', '_type' => _type, 'name' => 'Parent'}
-      documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '4', '_type' => _type, 'name' => 'Child', 'parent_id' => '5'}
+      expect(documents.size).to eq(2)
+      expect(documents[0].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '5', '_type' => _type, 'name' => 'Parent'})
+      expect(documents[1].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '4', '_type' => _type, 'name' => 'Child', 'parent_id' => '5'})
     end
 
     it 'should resolve foreign keys on foreign objects' do
-      processor.should_receive(:load_scraped_objects).and_return(foreign_keys_on_foreign_objects)
+      expect(processor).to receive(:load_scraped_objects).and_return(foreign_keys_on_foreign_objects)
 
       processor.import
       documents = processor.connection.raw_connection[:organizations].find.entries
-      documents.size.should == 3
-      documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '9', '_type' => _type, 'name' => 'Parent'}
-      documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '7', '_type' => _type, 'name' => 'Child', 'parent_id' => '9'}
-      documents[2].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '8', '_type' => _type, 'name' => 'Grandchild', 'parent_id' => '7'}
+      expect(documents.size).to eq(3)
+      expect(documents[0].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '9', '_type' => _type, 'name' => 'Parent'})
+      expect(documents[1].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '7', '_type' => _type, 'name' => 'Child', 'parent_id' => '9'})
+      expect(documents[2].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '8', '_type' => _type, 'name' => 'Grandchild', 'parent_id' => '7'})
     end
 
     context 'with existing documents' do
       before :each do
-        processor.should_receive(:load_scraped_objects).and_return(graphable)
+        expect(processor).to receive(:load_scraped_objects).and_return(graphable)
         processor.import
       end
 
@@ -307,39 +307,39 @@ describe Pupa::Processor do
       end
 
       it 'should resolve foreign keys' do
-        processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_key)
+        expect(processor).to receive(:load_scraped_objects).and_return(resolvable_foreign_key)
 
         processor.import
         documents = processor.connection.raw_connection[:organizations].find.entries
-        documents.size.should == 2
-        documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
-        documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
+        expect(documents.size).to eq(2)
+        expect(documents[0].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '2', '_type' => _type, 'name' => 'Parent'})
+        expect(documents[1].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'})
       end
 
       it 'should raise an error if a foreign key cannot be resolved' do
-        processor.should_receive(:load_scraped_objects).and_return(unresolvable_foreign_key)
+        expect(processor).to receive(:load_scraped_objects).and_return(unresolvable_foreign_key)
         expect{processor.import}.to raise_error(Pupa::Errors::UnprocessableEntity)
       end
 
       it 'should raise an error if a foreign object cannot be resolved' do
-        processor.should_receive(:load_scraped_objects).and_return(unresolvable_foreign_object)
+        expect(processor).to receive(:load_scraped_objects).and_return(unresolvable_foreign_object)
         expect{processor.import}.to raise_error(Pupa::Errors::UnprocessableEntity)
       end
 
       it 'should raise an error if a duplicate was inadvertently saved' do
-        processor.should_receive(:load_scraped_objects).and_return(duplicate_documents)
+        expect(processor).to receive(:load_scraped_objects).and_return(duplicate_documents)
         expect{processor.import}.to raise_error(Pupa::Errors::DuplicateDocumentError)
       end
 
       it 'should resolve foreign keys on foreign objects' do
-        processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_keys_on_foreign_objects)
+        expect(processor).to receive(:load_scraped_objects).and_return(resolvable_foreign_keys_on_foreign_objects)
 
         processor.import
         documents = processor.connection.raw_connection[:organizations].find.entries
-        documents.size.should == 3
-        documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => _type, 'name' => 'Parent'}
-        documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'}
-        documents[2].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => 'b', '_type' => _type, 'name' => 'Grandchild', 'parent_id' => '1'}
+        expect(documents.size).to eq(3)
+        expect(documents[0].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '2', '_type' => _type, 'name' => 'Parent'})
+        expect(documents[1].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => '1', '_type' => _type, 'name' => 'Child', 'parent_id' => '2'})
+        expect(documents[2].slice('_id', '_type', 'name', 'parent_id')).to eq({'_id' => 'b', '_type' => _type, 'name' => 'Grandchild', 'parent_id' => '1'})
       end
     end
   end

data/spec/runner_spec.rb

@@ -26,13 +26,13 @@ describe Pupa::Runner do
 
   describe '#initialize' do
     it 'should accept default options' do
-      dry_runner.options.level.should_not == 'INFO'
+      expect(dry_runner.options.level).not_to eq('INFO')
     end
   end
 
   describe '#add_action' do
     it 'should add an action' do
-      dry_runner.actions.last.to_h.should == {name: 'example', description: 'An example action'}
+      expect(dry_runner.actions.last.to_h).to eq({name: 'example', description: 'An example action'})
     end
   end
 
@@ -47,17 +47,17 @@ describe Pupa::Runner do
 
     it 'should accept overridden options' do
       dry_run(['--quiet'], level: 'ERROR')
-      dry_runner.options.level.should == 'ERROR'
+      expect(dry_runner.options.level).to eq('ERROR')
     end
 
     it 'should use default actions if none set' do
       dry_run
-      dry_runner.options.actions.should == %w(scrape import)
+      expect(dry_runner.options.actions).to eq(%w(scrape import))
     end
 
     it 'should use default tasks if none set' do
       dry_run
-      dry_runner.options.tasks.should == %i(people organizations)
+      expect(dry_runner.options.tasks).to eq(%i(people organizations))
     end
 
     # Unlike an action, it's not possible for a task to be undefined, because
@@ -71,14 +71,14 @@ describe Pupa::Runner do
     end
 
     it 'should run actions' do
-      TestProcessor.any_instance.should_receive(:dump_scraped_objects).twice
-      TestProcessor.any_instance.should_receive(:import)
+      expect_any_instance_of(TestProcessor).to receive(:dump_scraped_objects).twice
+      expect_any_instance_of(TestProcessor).to receive(:import)
       runner.run([])
     end
 
     it 'should run tasks' do
-      TestProcessor.any_instance.should_receive(:people).and_return([])
-      TestProcessor.any_instance.should_receive(:organizations).and_return([])
+      expect_any_instance_of(TestProcessor).to receive(:people).and_return([])
+      expect_any_instance_of(TestProcessor).to receive(:organizations).and_return([])
       runner.run([])
     end
   end

data/spec/spec_helper.rb

@@ -1,10 +1,18 @@
 require 'rubygems'
 
+require 'simplecov'
 require 'coveralls'
-Coveralls.wear!
+SimpleCov.formatter = Coveralls::SimpleCov::Formatter
+SimpleCov.start do
+  add_filter 'spec'
+end
 
 require 'multi_xml'
 require 'nokogiri'
 require 'redis-store'
 require 'rspec'
+
+Dir["./spec/support/**/*.rb"].each {|f| require f}
 require File.dirname(__FILE__) + '/../lib/pupa'
+
+Mongo::Logger.logger.level = Logger::WARN

data/spec/support/shared_examples_for_connection_adapters.rb

@@ -0,0 +1,66 @@
+shared_examples 'SQL adapter' do |database_url|
+  before :all do
+    connection = Pupa::Processor::Connection::PostgreSQLAdapter.new(database_url)
+
+    connection.raw_connection.drop_table?(:people)
+    connection.raw_connection.create_table(:people) do
+      primary_key :id
+      String :_id
+      String :_type
+      String :name
+      String :email
+      Time :created_at
+      Time :updated_at
+    end
+
+    connection.save(Pupa::Person.new(_id: 'existing', name: 'existing', email: 'existing@example.com'))
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+  end
+
+  let :connection do
+    Pupa::Processor::Connection::PostgreSQLAdapter.new(database_url)
+  end
+
+  let :_type do
+    'pupa/person'
+  end
+
+  describe '.find' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.find(_type: _type)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should return nil if no matches' do
+      expect(connection.find(_type: _type, name: 'nonexistent')).to eq(nil)
+    end
+
+    it 'should return a document if one match' do
+      expect(connection.find(_type: _type, name: 'existing')).to be_a(Hash)
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.find(_type: 'pupa/person', name: 'non-unique')}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+
+  describe '.save' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.save(Pupa::Person.new)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should insert a document if no matches' do
+      expect(connection.save(Pupa::Person.new(_id: 'new', name: 'new', email: 'new@example.com'))).to eq([true, 'new'])
+      expect(connection.find(_type: _type, name: 'new')['email']).to eq('new@example.com')
+    end
+
+    it 'should update a document if one match' do
+      expect(connection.save(Pupa::Person.new(_id: 'changed', name: 'existing', email: 'changed@example.com'))).to eq([false, 'existing'])
+      expect(connection.find(_type: _type, name: 'existing')['email']).to eq('changed@example.com')
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.save(Pupa::Person.new(name: 'non-unique'))}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: pupa
 version: !ruby/object:Gem::Version
-  version: 0.1.11
+  version: 0.2.0
 platform: ruby
 authors:
-- Open North
+- James McKinney
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-07 00:00:00.000000000 Z
+date: 2015-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -44,28 +44,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: 0.9.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: 0.9.2
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.3
+        version: '2.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.3
+        version: '2.3'
 - !ruby/object:Gem::Dependency
   name: mail
   requirement: !ruby/object:Gem::Requirement
@@ -81,19 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: moped
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.rc1
+        version: '2.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.0.0.rc1
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: oj
   requirement: !ruby/object:Gem::Requirement
@@ -151,7 +151,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: dalli
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -165,19 +165,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: json
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: '2.10'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.7.7
+        version: '2.10'
+- !ruby/object:Gem::Dependency
+  name: dalli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: multi_xml
   requirement: !ruby/object:Gem::Requirement
@@ -221,7 +235,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
+  name: redis-store
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -235,7 +249,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: redis-store
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -248,20 +262,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.10'
 - !ruby/object:Gem::Dependency
   name: typhoeus
   requirement: !ruby/object:Gem::Requirement
@@ -277,13 +277,13 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 description: 
-email:
-- info@opennorth.ca
+email: 
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
+- ".rspec"
 - ".travis.yml"
 - ".yardopts"
 - Gemfile
@@ -291,7 +291,6 @@ files:
 - PERFORMANCE.md
 - README.md
 - Rakefile
-- USAGE
 - lib/pupa.rb
 - lib/pupa/errors.rb
 - lib/pupa/logger.rb
@@ -372,6 +371,7 @@ files:
 - spec/processor/client_spec.rb
 - spec/processor/connection_adapters/mongodb_adapter_spec.rb
 - spec/processor/connection_adapters/postgresql_adapter_spec.rb
+- spec/processor/connection_adapters/sqlite_adapter_spec.rb
 - spec/processor/connection_spec.rb
 - spec/processor/dependency_graph_spec.rb
 - spec/processor/document_store/file_store_spec.rb
@@ -386,7 +386,8 @@ files:
 - spec/refinements/json-schema_spec.rb
 - spec/runner_spec.rb
 - spec/spec_helper.rb
-homepage: http://github.com/opennorth/pupa-ruby
+- spec/support/shared_examples_for_connection_adapters.rb
+homepage: https://github.com/jpmckinney/pupa-ruby
 licenses:
 - MIT
 metadata: {}
@@ -406,7 +407,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.2
+rubygems_version: 2.4.5
 signing_key: 
 specification_version: 4
 summary: A data scraping framework
@@ -435,6 +436,7 @@ test_files:
 - spec/processor/client_spec.rb
 - spec/processor/connection_adapters/mongodb_adapter_spec.rb
 - spec/processor/connection_adapters/postgresql_adapter_spec.rb
+- spec/processor/connection_adapters/sqlite_adapter_spec.rb
 - spec/processor/connection_spec.rb
 - spec/processor/dependency_graph_spec.rb
 - spec/processor/document_store/file_store_spec.rb
@@ -449,3 +451,4 @@ test_files:
 - spec/refinements/json-schema_spec.rb
 - spec/runner_spec.rb
 - spec/spec_helper.rb
+- spec/support/shared_examples_for_connection_adapters.rb