RubyGems - pupa - Versions diffs - 0.0.1 - Mend

pupa 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (75) hide show

checksums.yaml +7 -0
data/.gitignore +6 -0
data/.travis.yml +5 -0
data/.yardopts +4 -0
data/Gemfile +4 -0
data/LICENSE +20 -0
data/README.md +52 -0
data/Rakefile +37 -0
data/USAGE +1 -0
data/lib/pupa/errors.rb +30 -0
data/lib/pupa/logger.rb +37 -0
data/lib/pupa/models/base.rb +190 -0
data/lib/pupa/models/concerns/contactable.rb +34 -0
data/lib/pupa/models/concerns/identifiable.rb +26 -0
data/lib/pupa/models/concerns/linkable.rb +26 -0
data/lib/pupa/models/concerns/nameable.rb +34 -0
data/lib/pupa/models/concerns/sourceable.rb +26 -0
data/lib/pupa/models/concerns/timestamps.rb +22 -0
data/lib/pupa/models/contact_detail_list.rb +28 -0
data/lib/pupa/models/membership.rb +37 -0
data/lib/pupa/models/organization.rb +40 -0
data/lib/pupa/models/person.rb +35 -0
data/lib/pupa/models/post.rb +28 -0
data/lib/pupa/processor/client.rb +42 -0
data/lib/pupa/processor/dependency_graph.rb +18 -0
data/lib/pupa/processor/helper.rb +15 -0
data/lib/pupa/processor/middleware/logger.rb +37 -0
data/lib/pupa/processor/middleware/parse_html.rb +16 -0
data/lib/pupa/processor/persistence.rb +80 -0
data/lib/pupa/processor/yielder.rb +50 -0
data/lib/pupa/processor.rb +351 -0
data/lib/pupa/refinements/faraday_middleware.rb +32 -0
data/lib/pupa/refinements/json-schema.rb +36 -0
data/lib/pupa/runner.rb +185 -0
data/lib/pupa/version.rb +3 -0
data/lib/pupa.rb +31 -0
data/pupa.gemspec +34 -0
data/schemas/popolo/contact_detail.json +44 -0
data/schemas/popolo/identifier.json +18 -0
data/schemas/popolo/link.json +19 -0
data/schemas/popolo/membership.json +86 -0
data/schemas/popolo/organization.json +104 -0
data/schemas/popolo/other_name.json +28 -0
data/schemas/popolo/person.json +130 -0
data/schemas/popolo/post.json +78 -0
data/spec/cassettes/31ac91ccad069eefc07d96cfbe66fa66c1b41fcf.yml +56 -0
data/spec/cassettes/4ff54d737afb5d693653752d7bf234a405a80172.yml +48 -0
data/spec/cassettes/898049a22e6ca51dfa2510d9e0e0207a5c396524.yml +54 -0
data/spec/cassettes/ce69ff734ce852d2bfaa482bbf55d7ffb4762e87.yml +26 -0
data/spec/cassettes/da629b01e0836deda8a5540a4e6a08783dd7aef9.yml +46 -0
data/spec/cassettes/e398f35bea86b3d4c87a6934bae1eb7fca8744f9.yml +26 -0
data/spec/logger_spec.rb +4 -0
data/spec/models/base_spec.rb +194 -0
data/spec/models/concerns/contactable_spec.rb +37 -0
data/spec/models/concerns/identifiable_spec.rb +25 -0
data/spec/models/concerns/linkable_spec.rb +25 -0
data/spec/models/concerns/nameable_spec.rb +25 -0
data/spec/models/concerns/sourceable_spec.rb +25 -0
data/spec/models/concerns/timestamps_spec.rb +32 -0
data/spec/models/contact_detail_list_spec.rb +44 -0
data/spec/models/membership_spec.rb +30 -0
data/spec/models/organization_spec.rb +24 -0
data/spec/models/person_spec.rb +24 -0
data/spec/models/post_spec.rb +19 -0
data/spec/processor/client_spec.rb +4 -0
data/spec/processor/dependency_graph_spec.rb +4 -0
data/spec/processor/helper_spec.rb +4 -0
data/spec/processor/middleware/logger_spec.rb +87 -0
data/spec/processor/middleware/parse_html_spec.rb +92 -0
data/spec/processor/persistence_spec.rb +41 -0
data/spec/processor/yielder_spec.rb +55 -0
data/spec/processor_spec.rb +268 -0
data/spec/runner_spec.rb +85 -0
data/spec/spec_helper.rb +17 -0
metadata +342 -0

data/spec/processor/middleware/logger_spec.rb ADDED Viewed

@@ -0,0 +1,87 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+# @see test/adapters/logger_test.rb in faraday
+describe Pupa::Processor::Middleware::Logger do
+  let :io do
+    StringIO.new
+  end
+  context 'with DEBUG log level' do
+    let :logger do
+      logger = Logger.new(io)
+      logger.level = Logger::DEBUG
+      logger
+    end
+    let :connection do
+      Faraday.new do |connection|
+        connection.use Pupa::Processor::Middleware::Logger, logger
+        connection.adapter :test do |stubs|
+          stubs.get('/hello') { [200, {'Content-Type' => 'text/html'}, 'hello'] }
+        end
+      end
+    end
+    before :each do
+      @response = connection.get('/hello', nil, :accept => 'text/html')
+    end
+    it 'should still return output' do
+      @response.body.should == 'hello'
+    end
+    it 'should log the method and URL' do
+      io.string.should match('get http:/hello')
+    end
+    it 'should log request headers' do
+      io.string.should match('Accept: "text/html')
+    end
+  end
+  context 'with INFO log level' do
+    let :logger do
+      logger = Logger.new(io)
+      logger.level = Logger::INFO
+      logger
+    end
+    let :connection do
+      Faraday.new do |connection|
+        connection.use Pupa::Processor::Middleware::Logger, logger
+        connection.adapter :test do |stubs|
+          stubs.get('/hello') { [200, {'Content-Type' => 'text/html'}, 'hello'] }
+          stubs.post('/hello') { [200, {'Content-Type' => 'text/html'}, 'hello'] }
+        end
+      end
+    end
+    context 'with GET request' do
+      before :each do
+        connection.get('/hello', nil, :accept => 'text/html')
+      end
+      it 'should log the method and URL' do
+        io.string.should match('get http:/hello')
+      end
+      it 'should not log request headers' do
+        io.string.should_not match('Accept: "text/html')
+      end
+    end
+    context 'with POST request' do
+      before :each do
+        connection.post('/hello', 'foo=bar', :accept => 'text/html')
+      end
+      it 'should log the method and URL' do
+        io.string.should match('post http:/hello foo=bar')
+      end
+      it 'should not log request headers' do
+        io.string.should_not match('Accept: "text/html')
+      end
+    end
+  end
+end

data/spec/processor/middleware/parse_html_spec.rb ADDED Viewed

@@ -0,0 +1,92 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+# @see spec/helper.rb and spec/parse_xml_spec.rb in faraday_middleware
+describe Pupa::Processor::Middleware::ParseHtml do
+  let(:options) { Hash.new }
+  let(:headers) { Hash.new }
+  let(:middleware) {
+    described_class.new(lambda {|env|
+      Faraday::Response.new(env)
+    }, options)
+  }
+  def process(body, content_type = nil, options = {})
+    env = {
+      :body => body, :request => options,
+      :response_headers => Faraday::Utils::Headers.new(headers)
+    }
+    env[:response_headers]['content-type'] = content_type if content_type
+    middleware.call(env)
+  end
+  let(:html)  { '<html><head><title>foo</title></head><body>bar</body></html>' }
+  let(:title) { 'foo' }
+  let(:body) { 'bar' }
+  context "no type matching" do
+    it "doesn't change nil body" do
+      expect(process(nil).body).to be_nil
+    end
+    it "turns empty body into nil" do
+      expect(process('').body).to be_nil
+    end
+    it "parses html body" do
+      response = process(html)
+      expect(response.body.at_css('title').text).to eq(title)
+      expect(response.body.at_css('body').text).to eq(body)
+      expect(response.env[:raw_body]).to be_nil
+    end
+  end
+  context "with preserving raw" do
+    let(:options) { {:preserve_raw => true} }
+    it "parses html body" do
+      response = process(html)
+      expect(response.body.at_css('title').text).to eq(title)
+      expect(response.body.at_css('body').text).to eq(body)
+      expect(response.env[:raw_body]).to eq(html)
+    end
+    it "can opt out of preserving raw" do
+      response = process(html, nil, :preserve_raw => false)
+      expect(response.env[:raw_body]).to be_nil
+    end
+  end
+  context "with regexp type matching" do
+    let(:options) { {:content_type => /\bhtml$/} }
+    it "parses html body of correct type" do
+      response = process(html, 'text/html')
+      expect(response.body.at_css('title').text).to eq(title)
+      expect(response.body.at_css('body').text).to eq(body)
+    end
+    it "ignores html body of incorrect type" do
+      response = process(html, 'application/xml')
+      expect(response.body).to eq(html)
+    end
+  end
+  context "with array type matching" do
+    let(:options) { {:content_type => %w[a/b c/d]} }
+    it "parses html body of correct type" do
+      expect(process(html, 'a/b').body).to be_a(Nokogiri::HTML::Document)
+      expect(process(html, 'c/d').body).to be_a(Nokogiri::HTML::Document)
+    end
+    it "ignores html body of incorrect type" do
+      expect(process(html, 'a/d').body).not_to be_a(Nokogiri::HTML::Document)
+    end
+  end
+  it "doesn't choke on invalid html" do
+    ['{!', '"a"', 'true', 'null', '1'].each do |data|
+      expect{ process(data) }.to_not raise_error
+    end
+  end
+end

data/spec/processor/persistence_spec.rb ADDED Viewed

@@ -0,0 +1,41 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+describe Pupa::Processor::Persistence do
+  before :all do
+    Pupa.session = Moped::Session.new(['localhost:27017'], database: 'pupa_test')
+    Pupa.session.collections.each(&:drop)
+    Pupa::Processor::Persistence.new(Pupa::Person.new(name: 'existing')).save
+    Pupa.session[:people].insert(_type: 'pupa/person', name: 'non-unique')
+    Pupa.session[:people].insert(_type: 'pupa/person', name: 'non-unique')
+  end
+  describe '#find' do
+    it 'should return nil if no matches' do
+      Pupa::Processor::Persistence.find(_type: 'pupa/person', name: 'nonexistent').should == nil
+    end
+    it 'should return a document if one match' do
+      Pupa::Processor::Persistence.find(_type: 'pupa/person', name: 'existing').should be_a(Hash)
+    end
+    it 'should raise an error if many matches' do
+      expect{Pupa::Processor::Persistence.find(_type: 'pupa/person', name: 'non-unique')}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+  describe '#save' do
+    it 'should insert a document if no matches' do
+      Pupa::Processor::Persistence.new(Pupa::Person.new(_id: 'new', name: 'new')).save.should == 'new'
+    end
+    it 'should update a document if one match' do
+      Pupa::Processor::Persistence.new(Pupa::Person.new(_id: 'existing', name: 'existing')).save.should_not == 'existing'
+    end
+    it 'should raise an error if many matches' do
+      expect{Pupa::Processor::Persistence.new(Pupa::Person.new(name: 'non-unique')).save}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+end

data/spec/processor/yielder_spec.rb ADDED Viewed

@@ -0,0 +1,55 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+describe Pupa::Processor::Yielder do
+  let :yielder do
+    Pupa::Processor::Yielder.new do
+      10.times do |n|
+        Fiber.yield(n)
+      end
+    end
+  end
+  let :raiser do
+    Pupa::Processor::Yielder.new do
+      raise
+    end
+  end
+  describe '#each' do
+    it 'should iterate over the items in the enumeration' do
+      array = []
+      yielder.each do |n|
+        array << n
+      end
+      array.should == (0..9).to_a
+    end
+    it 'should be composable with other iterators' do
+      yielder.each.map{|n| n}.should == (0..9).to_a
+    end
+  end
+  describe '#next' do
+    it 'should return the next item in the enumeration' do
+      array = []
+      10.times do |n|
+        array << yielder.next
+      end
+      array.should == (0..9).to_a
+    end
+    it 'should raise an error if the enumerator is at the end' do
+      expect{11.times{yielder.next}}.to raise_error(StopIteration)
+    end
+  end
+  describe '#to_enum' do
+    it 'should return an enumerator' do
+      yielder.to_enum.should be_a(Enumerator)
+    end
+    it 'should return a lazy enumerator' do
+      expect{raiser.to_enum}.to_not raise_error
+    end
+  end
+end

data/spec/processor_spec.rb ADDED Viewed

@@ -0,0 +1,268 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe Pupa::Processor do
+  class PersonProcessor < Pupa::Processor
+    def person
+      @person ||= make_person_valid
+    end
+    def make_person_valid
+      @person = Pupa::Person.new(name: 'foo')
+    end
+    def make_person_invalid
+      @person = Pupa::Person.new
+    end
+    def scrape_people
+      Fiber.yield(person)
+    end
+  end
+  before :all do
+    PersonProcessor.add_scraping_task(:people)
+  end
+  let :io do
+    StringIO.new
+  end
+  let :processor do
+    PersonProcessor.new('/tmp', level: 'WARN', logdev: io)
+  end
+  describe '#get' do
+    it 'should send a GET request' do
+      processor.get('http://httpbin.org/get', 'foo=bar')['args'].should == {'foo' => 'bar'}
+    end
+    it 'should automatically parse the response' do
+      processor.get('http://httpbin.org/get').should be_a(Hash)
+    end
+  end
+  describe '#post' do
+    it 'should send a POST request' do
+      processor.post('http://httpbin.org/post', 'foo=bar')['form'].should == {'foo' => 'bar'}
+    end
+    it 'should automatically parse the response' do
+      processor.post('http://httpbin.org/post').should be_a(Hash)
+    end
+  end
+  describe '#add_scraping_task' do
+    it 'should add a scraping task and define a lazy method' do
+      PersonProcessor.tasks.should == [:people]
+      processor.should respond_to(:people)
+    end
+  end
+  describe '#dump_scraped_objects' do
+    let :path do
+      path = "/tmp/person_#{processor.person._id}.json"
+    end
+    it 'should not overwrite an existing file' do
+      FileUtils.touch(path)
+      expect{processor.dump_scraped_objects(:people)}.to raise_error(Pupa::Errors::DuplicateObjectIdError)
+      FileUtils.rm(path)
+    end
+    it 'should dump a JSON document' do
+      processor.dump_scraped_objects(:people)
+      File.exist?(path).should == true
+      io.string.should_not match('http://popoloproject.com/schemas/person.json')
+    end
+    it 'should validate the object' do
+      processor.make_person_invalid
+      processor.dump_scraped_objects(:people)
+      io.string.should match('http://popoloproject.com/schemas/person.json')
+    end
+  end
+  describe '#import' do
+    before :each do
+      Pupa.session = Moped::Session.new(['localhost:27017'], database: 'pupa_test')
+      Pupa.session.collections.each(&:drop)
+    end
+    let :graphable do
+      {
+        '1' => Pupa::Organization.new({
+          _id: '1',
+          name: 'Child',
+          parent_id: '3',
+        }),
+        '2' => Pupa::Organization.new({
+          _id: '2',
+          name: 'Parent',
+        }),
+        '3' => Pupa::Organization.new({
+          _id: '3',
+          name: 'Parent',
+        }),
+      }
+    end
+    let :ungraphable do
+      {
+        '4' => Pupa::Organization.new({
+          _id: '4',
+          name: 'Child',
+          parent: {_type: 'pupa/organization', name: 'Parent'},
+        }),
+        '5' => Pupa::Organization.new({
+          _id: '5',
+          name: 'Parent',
+        }),
+        '6' => Pupa::Organization.new({
+          _id: '6',
+          name: 'Parent',
+        }),
+      }
+    end
+    it 'should use a dependency graph if possible' do
+      processor.should_receive(:load_scraped_objects).and_return(graphable)
+      Pupa::Processor::DependencyGraph.any_instance.should_receive(:tsort).and_return(['2', '1'])
+      processor.import
+    end
+    it 'should not use a dependency graph if not possible' do
+      processor.should_receive(:load_scraped_objects).and_return(ungraphable)
+      Pupa::Processor::DependencyGraph.any_instance.should_not_receive(:tsort)
+      processor.import
+    end
+    it 'should remove duplicate objects and re-assign foreign keys' do
+      processor.should_receive(:load_scraped_objects).and_return(graphable)
+      processor.import
+      documents = Pupa.session[:organizations].find.entries
+      documents.size.should == 2
+      documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => 'pupa/organization', 'name' => 'Parent'}
+      documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => 'pupa/organization', 'name' => 'Child', 'parent_id' => '2'}
+    end
+    it 'should resolve foreign objects' do
+      processor.should_receive(:load_scraped_objects).and_return(ungraphable)
+      processor.import
+      documents = Pupa.session[:organizations].find.entries
+      documents.size.should == 2
+      documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '5', '_type' => 'pupa/organization', 'name' => 'Parent'}
+      documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '4', '_type' => 'pupa/organization', 'name' => 'Child', 'parent_id' => '5'}
+    end
+    context 'with existing documents' do
+      before :each do
+        processor.should_receive(:load_scraped_objects).and_return(graphable)
+        processor.import
+      end
+      let :resolvable_foreign_key do
+        {
+          'a' => Pupa::Organization.new({
+            _id: 'a',
+            name: 'Child',
+            parent_id: 'c',
+          }),
+          'b' => Pupa::Organization.new({
+            _id: 'b',
+            name: 'Parent',
+          }),
+          'c' => Pupa::Organization.new({
+            _id: 'c',
+            name: 'Parent',
+          }),
+        }
+      end
+      let :unresolvable_foreign_key do
+        {
+          'a' => Pupa::Organization.new({
+            _id: 'a',
+            name: 'Child',
+            parent: {_type: 'pupa/organization', name: 'Parent'},
+          }),
+          'b' => Pupa::Organization.new({
+            _id: 'b',
+            name: 'Parent',
+          }),
+          'c' => Pupa::Organization.new({
+            _id: 'c',
+            name: 'Child',
+            parent_id: 'nonexistent',
+          }),
+        }
+      end
+      let :unresolvable_foreign_object do
+        {
+          'a' => Pupa::Organization.new({
+            _id: 'a',
+            name: 'Child',
+            parent: {_type: 'pupa/organization', name: 'Nonexistent'},
+          }),
+          'b' => Pupa::Organization.new({
+            _id: 'b',
+            name: 'Parent',
+          }),
+          'c' => Pupa::Organization.new({
+            _id: 'c',
+            name: 'Child',
+            parent_id: 'b',
+          }),
+        }
+      end
+      let :duplicate_documents do
+        {
+          'a' => Pupa::Organization.new({
+            _id: 'a',
+            name: 'Child',
+            parent: {_type: 'pupa/organization', name: 'Parent'},
+          }),
+          'b' => Pupa::Organization.new({
+            _id: 'b',
+            name: 'Parent',
+          }),
+          'c' => Pupa::Organization.new({
+            _id: 'c',
+            name: 'Child',
+            parent_id: 'b',
+          }),
+        }
+      end
+      it 'should resolve foreign keys' do
+        processor.should_receive(:load_scraped_objects).and_return(resolvable_foreign_key)
+        processor.import
+        documents = Pupa.session[:organizations].find.entries
+        documents.size.should == 2
+        documents[0].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '2', '_type' => 'pupa/organization', 'name' => 'Parent'}
+        documents[1].slice('_id', '_type', 'name', 'parent_id').should == {'_id' => '1', '_type' => 'pupa/organization', 'name' => 'Child', 'parent_id' => '2'}
+      end
+      it 'should raise an error if a foreign key cannot be resolved' do
+        processor.should_receive(:load_scraped_objects).and_return(unresolvable_foreign_key)
+        expect{processor.import}.to raise_error(Pupa::Errors::UnprocessableEntity)
+      end
+      it 'should raise an error if a foreign object cannot be resolved' do
+        processor.should_receive(:load_scraped_objects).and_return(unresolvable_foreign_object)
+        expect{processor.import}.to raise_error(Pupa::Errors::UnprocessableEntity)
+      end
+      it 'should raise an error if a duplicate was inadvertently saved' do
+        processor.should_receive(:load_scraped_objects).and_return(duplicate_documents)
+        expect{processor.import}.to raise_error(Pupa::Errors::DuplicateDocumentError)
+      end
+    end
+  end
+end

data/spec/runner_spec.rb ADDED Viewed

@@ -0,0 +1,85 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+describe Pupa::Runner do
+  class TestProcessor < Pupa::Processor
+    def scrape_people
+    end
+    def scrape_organizations
+    end
+  end
+  before :all do
+    TestProcessor.add_scraping_task(:people)
+    TestProcessor.add_scraping_task(:organizations)
+  end
+  let :dry_runner do
+    runner = Pupa::Runner.new(TestProcessor, level: 'UNKNOWN', dry_run: true)
+    runner.add_action(name: 'example', description: 'An example action')
+    runner
+  end
+  let :runner do
+    Pupa::Runner.new(TestProcessor, level: 'UNKNOWN')
+  end
+  describe '#initialize' do
+    it 'should accept default options' do
+      dry_runner.options.level.should_not == 'INFO'
+    end
+  end
+  describe '#add_action' do
+    it 'should add an action' do
+      dry_runner.actions.last.to_h.should == {name: 'example', description: 'An example action'}
+    end
+  end
+  describe '#run' do
+    def dry_run(argv = [], **kwargs)
+      begin
+        dry_runner.run(argv, kwargs)
+      rescue SystemExit
+        # pass
+      end
+    end
+    it 'should accept overridden options' do
+      dry_run(['--quiet'], level: 'ERROR')
+      dry_runner.options.level.should == 'ERROR'
+    end
+    it 'should use default actions if none set' do
+      dry_run
+      dry_runner.options.actions.should == %w(scrape import)
+    end
+    it 'should use default tasks if none set' do
+      dry_run
+      dry_runner.options.tasks.should == %i(people organizations)
+    end
+    # Unlike an action, it's not possible for a task to be undefined, because
+    # `add_scraping_task` would raise an error first.
+    it 'should abort if the action is not defined' do
+      expect{dry_runner.run(['--action', 'example'])}.to raise_error(SystemExit, "`example` is not a rspec action. See `rspec --help` for a list of available actions.")
+    end
+    it 'should not run any actions on a dry run' do
+      expect{dry_runner.run([])}.to raise_error(SystemExit, nil)
+    end
+    it 'should run actions' do
+      TestProcessor.any_instance.should_receive(:dump_scraped_objects).twice
+      TestProcessor.any_instance.should_receive(:import)
+      runner.run([])
+    end
+    it 'should run tasks' do
+      TestProcessor.any_instance.should_receive(:people).and_return([])
+      TestProcessor.any_instance.should_receive(:organizations).and_return([])
+      runner.run([])
+    end
+  end
+end

data/spec/spec_helper.rb ADDED Viewed

@@ -0,0 +1,17 @@
+require 'rubygems'
+require 'coveralls'
+Coveralls.wear!
+require 'rspec'
+require 'vcr'
+require File.dirname(__FILE__) + '/../lib/pupa'
+VCR.configure do |c|
+  c.cassette_library_dir = 'spec/cassettes'
+  c.hook_into :faraday
+  c.around_http_request do |request|
+    VCR.use_cassette(Digest::SHA1.hexdigest(request.uri + request.body + request.headers.to_s), &request)
+  end
+end