pupa 0.0.13 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 37fa9e87e20d4fef24b1f028f046b695a3323b88
-  data.tar.gz: 0b2ae08ac3597955e7175812f05a700c76d91247
+  metadata.gz: e225ec9f62bb2c542da3f691b529cea5b6f99c15
+  data.tar.gz: 07726b314755421a2eac3d46756c46cd483ab913
 SHA512:
-  metadata.gz: de98e5ed8f0b145e0ba77401c3e2e983cd47dc2b7a327476a03b6e04eecaeb31f976d76e98a7d51114afd87a95e36c9da491e6a4f4d788f71b02060121cbb94e
-  data.tar.gz: 69fe22ea3079034b34aea5b244d233f66eebefd5010ea923ca9865d475bc81ccaebd6cab94e721ff2944cef3b4842dc297c932e8cbc69d949a593a74b2fc46a3
+  metadata.gz: 5732c1a25dcbcd7aaac28a049905c1fa2c6e4909f18e3b8c75b82e063264a871ec465b79aaa9c5371621b01f23ab4aa73135bb5757de6721e4521ec185d8e27a
+  data.tar.gz: d93fca74f1f8276c44de1ef228b85659875e839ce4a06a54a17f507423186485276c03c0f2be921b7f39e081fb622931f13659b116d76f9a920502f53427d5e6

data/.travis.yml

@@ -8,4 +8,5 @@ env:
 services:
   - memcached
   - mongodb
+  - postgresql
   - redis

data/README.md

@@ -7,6 +7,8 @@
 
 Pupa.rb is a Ruby 2.x fork of Sunlight Labs' [Pupa](https://github.com/opencivicdata/pupa). It implements an Extract, Transform and Load (ETL) process to scrape data from online sources, transform it, and write it to a database.
 
+    gem install pupa
+
 ## What it tries to solve
 
 Pupa.rb's goal is to make scraping less painful by solving common problems:
@@ -207,6 +209,21 @@ You may want to set the `CPUPROFILE_REALTIME=1` flag; however, it seems to inter
 
     pprof.rb --pdf /tmp/PROFILE_NAME > /tmp/PROFILE_NAME.pdf
 
+## Integration with ODMs
+
+### Mongoid
+
+`Pupa::Model` is incompatible with `Mongoid::Document`. Don't do this:
+
+```ruby
+class Cat
+  include Pupa::Model
+  include Mongoid::Document
+end
+```
+
+Instead, have a scraping model that includes `Pupa::Model` and an app model that includes `Mongoid::Document`.
+
 ## Testing
 
 **DO NOT** run this gem's specs if you are using Redis database number 15 on `localhost`!

data/lib/pupa.rb

@@ -9,11 +9,6 @@ require 'active_support/core_ext/hash/slice'
 require 'active_support/core_ext/object/blank'
 require 'active_support/inflector'
 
-require 'pupa/errors'
-require 'pupa/logger'
-require 'pupa/processor'
-require 'pupa/runner'
-
 require 'pupa/models/concerns/indifferent_access'
 require 'pupa/models/concerns/contactable'
 require 'pupa/models/concerns/identifiable'
@@ -22,6 +17,11 @@ require 'pupa/models/concerns/nameable'
 require 'pupa/models/concerns/sourceable'
 require 'pupa/models/concerns/timestamps'
 
+require 'pupa/errors'
+require 'pupa/logger'
+require 'pupa/processor'
+require 'pupa/runner'
+
 require 'pupa/models/foreign_object'
 require 'pupa/models/model'
 require 'pupa/models/contact_detail_list'
@@ -32,9 +32,6 @@ require 'pupa/models/person'
 require 'pupa/models/post'
 
 module Pupa
-  class << self
-    attr_accessor :session
-  end
 end
 
 # ActiveSupport's String methods become bottlenecks once:

data/lib/pupa/errors.rb

@@ -19,6 +19,10 @@ module Pupa
     # not exist in an object.
     class MissingAttributeError < Error; end
 
+    # This error is raised when saving an object to a database if the object
+    # produces an empty selector.
+    class EmptySelectorError < Error; end
+
     # This error is raised when saving an object to a database if the object
     # matches more than one document in the database.
     class TooManyMatches < Error; end

data/lib/pupa/models/model.rb

@@ -36,7 +36,7 @@ module Pupa
       attr_reader :extras
       # @return [String] The underscored, lowercase form of the object's class.
       attr_accessor :_type
-      # @return [Moped::BSON::Document,nil] The object's matching document in
+      # @return [BSON::Document,nil] The object's matching document in
       #   the database. Set before persisting the object to the database.
       attr_accessor :document
 
@@ -45,7 +45,7 @@ module Pupa
 
     module ClassMethods
       # Declare which properties should be dumped to JSON after a scraping task
-      # is complete. A subset of these properties will be imported to MongoDB.
+      # is complete. A subset of these will be imported to the database.
       #
       # @param [Array<Symbol>] the properties to dump to JSON
       def dump(*attributes)
@@ -129,9 +129,9 @@ module Pupa
 
     # Sets the object's ID.
     #
-    # @param [String,Moped::BSON::ObjectId] id an ID
+    # @param [String,BSON::ObjectId] id an ID
     def _id=(id)
-      @_id = id.to_s # in case of Moped::BSON::ObjectId
+      @_id = id.to_s # in case of BSON::ObjectId
     end
 
     # Sets the extras.
@@ -176,7 +176,7 @@ module Pupa
     # Returns the object as a hash.
     #
     # @param [Boolean] persist whether the object is being persisted, validated,
-    #   or used as a MongoDB selector, in which case foreign objects (i.e. hints)
+    #   or used as a database selector, in which case foreign objects (hints)
     #   are excluded
     # @return [Hash] the object as a hash
     def to_h(persist: false)

data/lib/pupa/models/organization.rb

@@ -33,12 +33,16 @@ module Pupa
     #   same name. Add a `jurisdiction` property?
     def fingerprint
       hash = super.slice(:classification, :parent_id)
-      {
-        '$or' => [
-          hash.merge('name' => name),
-          hash.merge('other_names.name' => name),
-        ],
-      }
+      if name
+        {
+          '$or' => [
+            hash.merge('name' => name),
+            hash.merge('other_names.name' => name),
+          ],
+        }
+      else
+        hash
+      end
     end
   end
 end

data/lib/pupa/models/person.rb

@@ -29,12 +29,16 @@ module Pupa
     # @todo This will obviously need to be scoped as in Python Pupa, to a
     #  jurisdiction, post, etc.
     def fingerprint
-      {
-        '$or' => [
-          {'name' => name},
-          {'other_names.name' => name},
-        ],
-      }
+      if name
+        {
+          '$or' => [
+            {'name' => name},
+            {'other_names.name' => name},
+          ],
+        }
+      else
+        {}
+      end
     end
   end
 end

data/lib/pupa/processor.rb

@@ -1,13 +1,10 @@
 require 'pupa/processor/client'
 require 'pupa/processor/dependency_graph'
 require 'pupa/processor/helper'
-require 'pupa/processor/persistence'
+require 'pupa/processor/connection'
 require 'pupa/processor/document_store'
 require 'pupa/processor/yielder'
 
-require 'pupa/processor/document_store/file_store'
-require 'pupa/processor/document_store/redis_store'
-
 module Pupa
   # An abstract processor class from which specific processors inherit.
   class Processor
@@ -17,27 +14,29 @@ module Pupa
     class_attribute :tasks
     self.tasks = []
 
-    attr_reader :report, :store, :client, :options
+    attr_reader :report, :store, :connection, :client, :options
 
     def_delegators :@logger, :debug, :info, :warn, :error, :fatal
 
     # @param [String] output_dir the directory or Redis address
     #   (e.g. `redis://localhost:6379`) in which to dump JSON documents
+    # @param [Boolean] pipelined whether to dump JSON documents all at once
     # @param [String] cache_dir the directory or Memcached address
     #   (e.g. `memcached://localhost:11211`) in which to cache HTTP responses
     # @param [Integer] expires_in the cache's expiration time in seconds
-    # @param [Boolean] pipelined whether to dump JSON documents all at once
+    # @param [String] database_url the database URL
     # @param [Boolean] validate whether to validate JSON documents
     # @param [String] level the log level
     # @param [String,IO] logdev the log device
     # @param [Hash] options criteria for selecting the methods to run
-    def initialize(output_dir, cache_dir: nil, expires_in: 86400, pipelined: false, validate: true, level: 'INFO', logdev: STDOUT, options: {})
-      @store    = DocumentStore.new(output_dir, pipelined: pipelined)
-      @client   = Client.new(cache_dir: cache_dir, expires_in: expires_in, level: level)
-      @logger   = Logger.new('pupa', level: level, logdev: logdev)
-      @validate = validate
-      @options  = options
-      @report   = {}
+    def initialize(output_dir, pipelined: false, cache_dir: nil, expires_in: 86400, database_url: 'mongodb://localhost:27017/pupa', validate: true, level: 'INFO', logdev: STDOUT, options: {})
+      @store      = DocumentStore.new(output_dir, pipelined: pipelined)
+      @client     = Client.new(cache_dir: cache_dir, expires_in: expires_in, level: level)
+      @connection = Connection.new(database_url)
+      @logger     = Logger.new('pupa', level: level, logdev: logdev)
+      @validate   = validate
+      @options    = options
+      @report     = {}
     end
 
     # Retrieves and parses a document with a GET request.
@@ -369,7 +368,7 @@ module Pupa
         if value.present?
           foreign_object = ForeignObject.new(value)
           resolve_foreign_keys(foreign_object, map)
-          document = Persistence.find(foreign_object.to_h)
+          document = connection.find(foreign_object.to_h)
 
           if document
             object["#{property}_id"] = document['_id']
@@ -382,7 +381,7 @@ module Pupa
 
     # @param [Object] object an object
     def import_object(object)
-      inserted, id = Persistence.new(object).save
+      inserted, id = connection.save(object)
       @report[:import][object._type] ||= Hash.new(0)
       if inserted
         @report[:import][object._type][:insert] += 1

data/lib/pupa/processor/connection.rb

@@ -0,0 +1,26 @@
+require 'pupa/processor/connection_adapters/mongodb_adapter'
+require 'pupa/processor/connection_adapters/postgresql_adapter'
+
+module Pupa
+  class Processor
+    # A database system connection factory.
+    class Connection
+      # Returns a configured connection to a database system.
+      #
+      # See each connection adapter for more information.
+      #
+      # @param [String] database_url the database URL
+      # @return a configured connection to a database system
+      def self.new(database_url)
+        case URI.parse(database_url).scheme
+        when 'postgres'
+          PostgreSQLAdapter.new(database_url)
+        when 'mongodb'
+          MongoDBAdapter.new(database_url)
+        else
+          raise NotImplementedError
+        end
+      end
+    end
+  end
+end

data/lib/pupa/processor/connection_adapters/mongodb_adapter.rb

@@ -0,0 +1,92 @@
+require 'moped'
+
+module Pupa
+  class Processor
+    class Connection
+      # A proxy class to save plain old Ruby objects to MongoDB.
+      class MongoDBAdapter
+        attr_reader :raw_connection
+
+        # @param [String] database_url the database URL
+        def initialize(database_url)
+          uri = URI.parse(database_url)
+          @raw_connection = Moped::Session.new(["#{uri.host}:#{uri.port}"], database: uri.path[1..-1])
+          @raw_connection.login(uri.user, uri.password) if uri.user && uri.password
+        end
+
+        # Finds a document matching the selection criteria.
+        #
+        # The selection criteria *must* set a `_type` key in order to determine
+        # the collection to query.
+        #
+        # @param [Hash] selector the selection criteria
+        # @return [Hash,nil] the matched document, or nil
+        # @raises [Pupa::Errors::TooManyMatches] if multiple documents are found
+        def find(selector)
+          collection_name = collection_name_from_class_name(selector[:_type].camelize)
+          if selector.except(:_type).empty?
+            raise Errors::EmptySelectorError, "selector is empty during find in collection #{collection_name}"
+          end
+          collection = raw_connection[collection_name]
+          query = collection.find(selector)
+
+          case query.count
+          when 0
+            nil
+          when 1
+            query.first
+          else
+            raise Errors::TooManyMatches, "selector matches multiple documents during find in collection #{collection_name}: #{JSON.dump(selector)}"
+          end
+        end
+
+        # Inserts or replaces a document in MongoDB.
+        #
+        # @param [Object] object an object
+        # @return [Array] whether the object was inserted and the object's database ID
+        # @raises [Pupa::Errors::TooManyMatches] if multiple documents would be updated
+        def save(object)
+          selector = object.fingerprint
+
+          collection_name = collection_name_from_class_name(object.class.to_s)
+          if selector.empty?
+            raise Errors::EmptySelectorError, "selector is empty during save in collection #{collection_name} for #{object._id}"
+          end
+          collection = raw_connection[collection_name]
+          query = collection.find(selector)
+
+          # Run query before callbacks to avoid e.g. timestamps in the selector.
+          case query.count
+          when 0
+            object.run_callbacks(:save) do
+              object.run_callbacks(:create) do
+                collection.insert(object.to_h(persist: true))
+                [true, object._id.to_s]
+              end
+            end
+          when 1
+            # Make the document available to the callbacks.
+            # @see https://github.com/opennorth/pupa-ruby/issues/17
+            object.document = query.first
+            object.run_callbacks(:save) do
+              query.update(object.to_h(persist: true).except(:_id))
+              [false, object.document['_id'].to_s]
+            end
+          else
+            raise Errors::TooManyMatches, "selector matches multiple documents during save in collection #{collection_name} for #{object._id}: #{JSON.dump(selector)}"
+          end
+        end
+
+      private
+
+        # Returns the name of the collection in which to save the object.
+        #
+        # @param [String] class_name the name of the object's class
+        # @return [String] the name of the collection in which to save the object
+        def collection_name_from_class_name(class_name)
+          class_name.demodulize.underscore.pluralize.to_sym
+        end
+      end
+    end
+  end
+end

data/lib/pupa/processor/connection_adapters/postgresql_adapter.rb

@@ -0,0 +1,116 @@
+require 'sequel'
+
+module Pupa
+  class Processor
+    class Connection
+      # A proxy class to save plain old Ruby objects to PostgreSQL.
+      class PostgreSQLAdapter
+        include Pupa::Concerns::IndifferentAccess
+
+        attr_reader :raw_connection
+
+        # @param [String] database_url the database URL
+        def initialize(database_url)
+          @raw_connection = Sequel.connect(database_url)
+        end
+
+        # Finds a document matching the selection criteria.
+        #
+        # The selection criteria *must* set a `_type` key in order to determine
+        # the collection to query.
+        #
+        # @param [Hash] selector the selection criteria
+        # @return [Hash,nil] the matched document, or nil
+        # @raises [Pupa::Errors::TooManyMatches] if multiple documents are found
+        def find(selector)
+          collection_name = collection_name_from_class_name(selector[:_type].camelize)
+          if selector.except(:_type).empty?
+            raise Errors::EmptySelectorError, "selector is empty during find in collection #{collection_name}"
+          end
+          collection = raw_connection[collection_name]
+          query = collection.filter(symbolize_keys(selector))
+
+          case query.count
+          when 0
+            nil
+          when 1
+            stringify_keys(query.first)
+          else
+            raise Errors::TooManyMatches, "selector matches multiple documents during find in collection #{collection_name}: #{JSON.dump(selector)}"
+          end
+        end
+
+        # Inserts or replaces a document in PostgreSQL.
+        #
+        # @param [Object] object an object
+        # @return [Array] whether the object was inserted and the object's database ID
+        # @raises [Pupa::Errors::TooManyMatches] if multiple documents would be updated
+        def save(object)
+          fingerprint = symbolize_keys(object.fingerprint) # Sequel needs symbols
+          selector = if fingerprint.key?(:$or) && fingerprint.size == 1
+            Sequel.or(reject_subdocument_criteria(fingerprint[:$or]))
+          else
+            reject_subdocument_criteria(fingerprint)
+          end
+
+          collection_name = collection_name_from_class_name(object.class.to_s)
+          if fingerprint.empty?
+            raise Errors::EmptySelectorError, "selector is empty during save in collection #{collection_name} for #{object._id}"
+          end
+          collection = raw_connection[collection_name]
+          query = collection.filter(selector)
+
+          # Run query before callbacks to avoid e.g. timestamps in the selector.
+          case query.count
+          when 0
+            object.run_callbacks(:save) do
+              object.run_callbacks(:create) do
+                collection.insert(object.to_h(persist: true))
+                [true, object._id.to_s]
+              end
+            end
+          when 1
+            # Make the document available to the callbacks.
+            # @see https://github.com/opennorth/pupa-ruby/issues/17
+            object.document = stringify_keys(query.first)
+            object.run_callbacks(:save) do
+              query.update(object.to_h(persist: true).except(:_id))
+              [false, object.document['_id'].to_s]
+            end
+          else
+            raise Errors::TooManyMatches, "selector matches multiple documents during save in collection #{collection_name} for #{object._id}: #{JSON.dump(selector)}"
+          end
+        end
+
+      private
+
+        # Returns the name of the collection in which to save the object.
+        #
+        # @param [String] class_name the name of the object's class
+        # @return [String] the name of the collection in which to save the object
+        def collection_name_from_class_name(class_name)
+          class_name.demodulize.underscore.pluralize.to_sym
+        end
+
+        def reject_subdocument_criteria(object)
+          case object
+          when Hash
+            array = []
+            object.each do |key,value|
+              unless key.to_s['.'] # @todo Support MongoDB subdocument criteria.
+                array += [key, reject_subdocument_criteria(value)]
+              end
+            end
+            array
+          when Array
+            object.map do |value|
+              reject_subdocument_criteria(value)
+            end.reject(&:empty?)
+          else
+            object
+          end
+        end
+      end
+    end
+  end
+end