pupa 0.0.13 → 0.1.0

data/lib/pupa/processor/document_store.rb

@@ -1,3 +1,6 @@
+require 'pupa/processor/document_store/file_store'
+require 'pupa/processor/document_store/redis_store'
+
 module Pupa
   class Processor
     # An JSON document store factory.

data/lib/pupa/processor/middleware/raise_error.rb

@@ -4,6 +4,7 @@ module Pupa
       # A Faraday response middleware for raising an error if unsuccessful.
       #
       # @see Faraday::Response::RaiseError
+      # @note Faraday has no tests for this middleware.
       class RaiseError < Faraday::Response::Middleware
         def on_complete(env)
           case env[:status]

data/lib/pupa/refinements/faraday_middleware.rb

@@ -17,7 +17,7 @@ class FaradayMiddleware::Caching
     if url.query && params_to_ignore.any?
       params = parse_query url.query
       params.reject! {|k,| params_to_ignore.include? k }
-      url.query = build_query params
+      url.query = params.any? ? build_query(params) : nil
     end
     url.normalize!
     url.request_uri + env[:body].to_s # XXX add for POST requests

data/lib/pupa/runner.rb

@@ -1,8 +1,6 @@
 require 'optparse'
 require 'ostruct'
 
-require 'moped'
-
 module Pupa
   class Runner
     attr_reader :options, :actions
@@ -16,14 +14,13 @@ module Pupa
         actions:        [],
         tasks:          [],
         output_dir:     File.expand_path('scraped_data', Dir.pwd),
+        pipelined:      false,
         cache_dir:      File.expand_path('web_cache', Dir.pwd),
         expires_in:     86400, # 1 day
-        pipelined:      false,
+        database_url:   'mongodb://localhost:27017/pupa',
         validate:       true,
-        host_with_port: 'localhost:27017',
-        database:       'pupa',
-        dry_run:        false,
         level:          'INFO',
+        dry_run:        false,
       }.merge(defaults))
 
       @actions = {
@@ -76,27 +73,21 @@ module Pupa
         opts.on('-o', '--output_dir PATH', 'The directory or Redis address (e.g. redis://localhost:6379/0) in which to dump JSON documents') do |v|
           options.output_dir = v
         end
+        opts.on('--pipelined', 'Dump JSON documents all at once') do |v|
+          options.pipelined = v
+        end
         opts.on('-c', '--cache_dir PATH', 'The directory or Memcached address (e.g. memcached://localhost:11211) in which to cache HTTP requests') do |v|
           options.cache_dir = v
         end
         opts.on('-e', '--expires_in SECONDS', "The cache's expiration time in seconds") do |v|
           options.expires_in = v
         end
-        opts.on('--pipelined', 'Dump JSON documents all at once') do |v|
-          options.pipelined = v
+        opts.on('-d', '--database_url SCHEME://USERNAME:PASSWORD@HOST:PORT/DATABASE', 'The database URL') do |v|
+          options.database_url = v
         end
         opts.on('--[no-]validate', 'Validate JSON documents') do |v|
           options.validate = v
         end
-        opts.on('-H', '--host HOST:PORT', 'The host and port to MongoDB') do |v|
-          options.host_with_port = v
-        end
-        opts.on('-d', '--database NAME', 'The name of the MongoDB database') do |v|
-          options.database = v
-        end
-        opts.on('-n', '--dry-run', 'Show the plan without running any actions') do
-          options.dry_run = true
-        end
         opts.on('-v', '--verbose', 'Show all messages') do
           options.level = 'DEBUG'
         end
@@ -106,6 +97,9 @@ module Pupa
         opts.on('-s', '--silent', 'Show no messages') do
           options.level = 'UNKNOWN'
         end
+        opts.on('-n', '--dry-run', 'Show the plan without running any actions') do
+          options.dry_run = true
+        end
 
         opts.separator ''
         opts.separator 'Common options:'
@@ -145,9 +139,10 @@ module Pupa
       end
 
       processor = @processor_class.new(options.output_dir,
+        pipelined: options.pipelined,
         cache_dir: options.cache_dir,
         expires_in: options.expires_in,
-        pipelined: options.pipelined,
+        database_url: options.database_url,
         validate: options.validate,
         level: options.level,
         options: Hash[*rest])
@@ -165,7 +160,7 @@ module Pupa
       end
 
       if options.level == 'DEBUG'
-        %w(output_dir cache_dir expires_in host_with_port database level).each do |option|
+        %w(output_dir pipelined cache_dir expires_in database_url validate level).each do |option|
           puts "#{option}: #{options[option]}"
         end
         unless rest.empty?
@@ -184,8 +179,6 @@ module Pupa
         start: Time.now.utc,
       }
 
-      Pupa.session = Moped::Session.new([options.host_with_port], database: options.database)
-
       if options.actions.delete('scrape')
         processor.store.clear
         report[:scrape] = {}

data/lib/pupa/version.rb

@@ -1,3 +1,3 @@
 module Pupa
-  VERSION = "0.0.13"
+  VERSION = "0.1.0"
 end

data/pupa.gemspec

@@ -21,8 +21,10 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency('faraday_middleware', '~> 0.9.0')
   s.add_runtime_dependency('json-schema', '~> 2.1.3')
   s.add_runtime_dependency('mail')
-  s.add_runtime_dependency('moped', '~> 1.5.1')
+  s.add_runtime_dependency('moped', '~> 2.0.0.rc1')
   s.add_runtime_dependency('oj', '~> 2.1')
+  s.add_runtime_dependency('sequel', '~> 4.10.0')
+  s.add_runtime_dependency('pg', '~> 0.17.0')
 
   s.add_development_dependency('coveralls')
   s.add_development_dependency('dalli')
@@ -34,5 +36,4 @@ Gem::Specification.new do |s|
   s.add_development_dependency('redis-store')
   s.add_development_dependency('rspec', '~> 2.10')
   s.add_development_dependency('typhoeus')
-  s.add_development_dependency('vcr', '~> 2.5.0')
 end

data/schemas/popolo/contact_detail.json

@@ -23,6 +23,16 @@
       "description": "A note, e.g. for grouping contact details by physical location",
       "type": ["string", "null"]
     },
+    "valid_from": {
+      "description": "The date from which the contact detail is valid",
+      "type": ["string", "null"],
+      "pattern": "^[0-9]{4}(-[0-9]{2}){0,2}$"
+    },
+    "valid_through": {
+      "description": "The date from which the contact detail is no longer valid",
+      "type": ["string", "null"],
+      "pattern": "^[0-9]{4}(-[0-9]{2}){0,2}$"
+    },
     "created_at": {
       "description": "The time at which the resource was created",
       "type": ["string", "null"],

data/schemas/popolo/membership.json

@@ -2,7 +2,7 @@
   "$schema": "http://json-schema.org/draft-03/schema#",
   "id": "http://popoloproject.com/schemas/membership.json#",
   "title": "Membership",
-  "description": "A relationship between a person and an organization",
+  "description": "A relationship between a member and an organization",
   "type": "object",
   "properties": {
     "id": {
@@ -14,33 +14,53 @@
       "type": ["string", "null"]
     },
     "role": {
-      "description": "The role that the person fulfills in the organization",
+      "description": "The role that the member fulfills in the organization",
       "type": ["string", "null"]
     },
+    "member": {
+      "description": "The person or organization that is a member of the organization",
+      "type": ["object"]
+    },
     "person_id": {
-      "description": "The ID of the person who is a party to the relationship",
+      "description": "The ID of the person who is a member of the organization",
       "type": ["string", "null"]
     },
     "person": {
-      "description": "The person who is a party to the relationship",
+      "description": "The person who is a member of the organization",
       "$ref": "http://popoloproject.com/schemas/person.json#"
     },
     "organization_id": {
-      "description": "The ID of the organization that is a party to the relationship",
+      "description": "The ID of the organization in which the person or organization is a member",
       "type": ["string", "null"]
     },
     "organization": {
-      "description": "The organization that is a party to the relationship",
+      "description": "The organization in which the person or organization is a member",
       "$ref": "http://popoloproject.com/schemas/organization.json#"
     },
     "post_id": {
-      "description": "The ID of the post held by the person in the organization through this membership",
+      "description": "The ID of the post held by the member in the organization through this membership",
       "type": ["string", "null"]
     },
     "post": {
-      "description": "The post held by the person in the organization through this membership",
+      "description": "The post held by the member in the organization through this membership",
       "$ref": "http://popoloproject.com/schemas/post.json#"
     },
+    "on_behalf_of_id": {
+      "description": "The ID of the organization on whose behalf the person is a member of the organization",
+      "type": ["string", "null"]
+    },
+    "on_behalf_of": {
+      "description": "The organization on whose behalf the person is a member of the organization",
+      "$ref": "http://popoloproject.com/schemas/organization.json#"
+    },
+    "area_id": {
+      "description": "The ID of the geographic area to which this membership is related",
+      "type": ["string", "null"]
+    },
+    "area": {
+      "description": "The geographic area to which this membership is related",
+      "$ref": "http://popoloproject.com/schemas/area.json#"
+    },
     "start_date": {
       "description": "The date on which the relationship began",
       "type": ["string", "null"],
@@ -52,7 +72,7 @@
       "pattern": "^[0-9]{4}(-[0-9]{2}){0,2}$"
     },
     "contact_details": {
-      "description": "Means of contacting the person who is a party to the relationship",
+      "description": "Means of contacting the member of the organization",
       "type": "array",
       "items": {
         "$ref": "http://popoloproject.com/schemas/contact_detail.json#"

data/schemas/popolo/organization.json

@@ -11,8 +11,7 @@
     },
     "name": {
       "description": "A primary name, e.g. a legally recognized name",
-      "type": "string",
-      "required": true
+      "type": ["string", "null"]
     },
     "other_names": {
       "description": "Alternate or former names",
@@ -40,6 +39,14 @@
       "description": "The organization that contains this organization",
       "$ref": "http://popoloproject.com/schemas/organization.json#"
     },
+    "area_id": {
+      "description": "The ID of the geographic area to which this organization is related",
+      "type": ["string", "null"]
+    },
+    "area": {
+      "description": "The geographic area to which this organization is related",
+      "$ref": "http://popoloproject.com/schemas/area.json#"
+    },
     "founding_date": {
       "description": "A date of founding",
       "type": ["string", "null"],

data/schemas/popolo/other_name.json

@@ -10,6 +10,30 @@
       "type": "string",
       "required": true
     },
+    "family_name": {
+      "description": "One or more family names",
+      "type": ["string", "null"]
+    },
+    "given_name": {
+      "description": "One or more primary given names",
+      "type": ["string", "null"]
+    },
+    "additional_name": {
+      "description": "One or more secondary given names",
+      "type": ["string", "null"]
+    },
+    "honorific_prefix": {
+      "description": "One or more honorifics preceding a person's name",
+      "type": ["string", "null"]
+    },
+    "honorific_suffix": {
+      "description": "One or more honorifics following a person's name",
+      "type": ["string", "null"]
+    },
+    "patronymic_name": {
+      "description": "One or more patronymic names",
+      "type": ["string", "null"]
+    },
     "start_date": {
       "description": "The date on which the name was adopted",
       "type": ["string", "null"],

data/schemas/popolo/person.json

@@ -11,8 +11,7 @@
     },
     "name": {
       "description": "A person's preferred full name",
-      "type": "string",
-      "required": true
+      "type": ["string", "null"]
     },
     "other_names": {
       "description": "Alternate or former names",
@@ -53,7 +52,7 @@
       "type": ["string", "null"]
     },
     "sort_name": {
-      "description": "A name to use in an lexicographically ordered list",
+      "description": "A name to use in a lexicographically ordered list",
       "type": ["string", "null"]
     },
     "email": {
@@ -88,6 +87,10 @@
       "description": "An extended account of a person's life",
       "type": ["string", "null"]
     },
+    "national_identity": {
+      "description": "A national identity",
+      "type": ["string", "null"]
+    },
     "contact_details": {
       "description": "Means of contacting the person",
       "type": "array",

data/schemas/popolo/post.json

@@ -11,8 +11,14 @@
     },
     "label": {
       "description": "A label describing the post",
-      "type": "string",
-      "required": true
+      "type": ["string", "null"]
+    },
+    "other_label": {
+      "description": "An alternate label",
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
     },
     "role": {
       "description": "The function that the holder of the post fulfills",
@@ -26,6 +32,14 @@
       "description": "The organization in which the post is held",
       "$ref": "http://popoloproject.com/schemas/organization.json#"
     },
+    "area_id": {
+      "description": "The ID of the geographic area to which this post is related",
+      "type": ["string", "null"]
+    },
+    "area": {
+      "description": "The geographic area to which this post is related",
+      "$ref": "http://popoloproject.com/schemas/area.json#"
+    },
     "start_date": {
       "description": "The date on which the post was created",
       "type": ["string", "null"],

data/spec/models/model_spec.rb

@@ -146,7 +146,7 @@ describe Pupa::Model do
     end
 
     it 'should coerce the _id to a string' do
-      object._id = Moped::BSON::ObjectId.new
+      object._id = BSON::ObjectId.new
       object._id.should be_a(String)
     end
   end

data/spec/processor/connection_adapters/mongodb_adapter_spec.rb

@@ -0,0 +1,61 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+describe Pupa::Processor::Connection::MongoDBAdapter do
+  def _type
+    if testing_python_compatibility?
+      'person'
+    else
+      'pupa/person'
+    end
+  end
+
+  def connection
+    Pupa::Processor::Connection::MongoDBAdapter.new('mongodb://localhost:27017/pupa_test')
+  end
+
+  before :all do
+    connection.raw_connection[:people].drop
+
+    connection.save(Pupa::Person.new(_id: 'existing', name: 'existing', email: 'existing@example.com'))
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+  end
+
+  describe '.find' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.find(_type: _type)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should return nil if no matches' do
+      connection.find(_type: _type, name: 'nonexistent').should == nil
+    end
+
+    it 'should return a document if one match' do
+      connection.find(_type: _type, name: 'existing').should be_a(Hash)
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.find(_type: 'pupa/person', name: 'non-unique')}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+
+  describe '.save' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.save(Pupa::Person.new)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should insert a document if no matches' do
+      connection.save(Pupa::Person.new(_id: 'new', name: 'new', email: 'new@example.com')).should == [true, 'new']
+      connection.find(_type: _type, name: 'new')['email'].should == 'new@example.com'
+    end
+
+    it 'should update a document if one match' do
+      connection.save(Pupa::Person.new(_id: 'changed', name: 'existing', email: 'changed@example.com')).should == [false, 'existing']
+      connection.find(_type: _type, name: 'existing')['email'].should == 'changed@example.com'
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.save(Pupa::Person.new(name: 'non-unique'))}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+end

data/spec/processor/connection_adapters/postgresql_adapter_spec.rb

@@ -0,0 +1,70 @@
+require File.expand_path(File.dirname(__FILE__) + '/../../spec_helper')
+
+describe Pupa::Processor::Connection::PostgreSQLAdapter do
+  def _type
+    if testing_python_compatibility?
+      'person'
+    else
+      'pupa/person'
+    end
+  end
+
+  def connection
+    Pupa::Processor::Connection::PostgreSQLAdapter.new('postgres://localhost:5432/pupa_test')
+  end
+
+  before :all do
+    connection.raw_connection.drop_table(:people)
+    connection.raw_connection.create_table(:people) do
+      primary_key :id
+      String :_id
+      String :_type
+      String :name
+      String :email
+      Time :created_at
+      Time :updated_at
+    end
+
+    connection.save(Pupa::Person.new(_id: 'existing', name: 'existing', email: 'existing@example.com'))
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+    connection.raw_connection[:people].insert(_type: 'pupa/person', name: 'non-unique')
+  end
+
+  describe '.find' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.find(_type: _type)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should return nil if no matches' do
+      connection.find(_type: _type, name: 'nonexistent').should == nil
+    end
+
+    it 'should return a document if one match' do
+      connection.find(_type: _type, name: 'existing').should be_a(Hash)
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.find(_type: 'pupa/person', name: 'non-unique')}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+
+  describe '.save' do
+    it 'should raise an error if selector is empty' do
+      expect{connection.save(Pupa::Person.new)}.to raise_error(Pupa::Errors::EmptySelectorError)
+    end
+
+    it 'should insert a document if no matches' do
+      connection.save(Pupa::Person.new(_id: 'new', name: 'new', email: 'new@example.com')).should == [true, 'new']
+      connection.find(_type: _type, name: 'new')['email'].should == 'new@example.com'
+    end
+
+    it 'should update a document if one match' do
+      connection.save(Pupa::Person.new(_id: 'changed', name: 'existing', email: 'changed@example.com')).should == [false, 'existing']
+      connection.find(_type: _type, name: 'existing')['email'].should == 'changed@example.com'
+    end
+
+    it 'should raise an error if many matches' do
+      expect{connection.save(Pupa::Person.new(name: 'non-unique'))}.to raise_error(Pupa::Errors::TooManyMatches)
+    end
+  end
+end