pupa 0.0.1

data/lib/pupa/processor.rb

@@ -0,0 +1,351 @@
+require 'json'
+
+require 'nokogiri'
+
+require 'pupa/processor/client'
+require 'pupa/processor/dependency_graph'
+require 'pupa/processor/helper'
+require 'pupa/processor/persistence'
+require 'pupa/processor/yielder'
+
+module Pupa
+  # An abstract processor class from which specific processors inherit.
+  class Processor
+    extend Forwardable
+    include Helper
+
+    class_attribute :tasks
+    self.tasks = []
+
+    def_delegators :@logger, :debug, :info, :warn, :error, :fatal
+
+    # @param [String] output_dir the directory in which to dump JSON documents
+    # @param [String] cache_dir the directory in which to cache HTTP responses
+    # @param [Integer] expires_in the cache's expiration time in seconds
+    # @param [String] level the log level
+    # @param [String,IO] logdev the log device
+    # @param [Hash] options criteria for selecting the methods to run
+    def initialize(output_dir, cache_dir: nil, expires_in: 86400, level: 'INFO', logdev: STDOUT, options: {})
+      @output_dir = output_dir
+      @options    = options
+      @level      = level
+      @logger     = Logger.new('pupa', level: level, logdev: logdev)
+      @client     = Client.new(cache_dir: cache_dir, expires_in: expires_in, level: level)
+    end
+
+    # Retrieves and parses a document with a GET request.
+    #
+    # @param [String] url a URL to an HTML document
+    # @param [String,Hash] params query string parameters
+    # @return a parsed document
+    def get(url, params = {})
+      # Faraday requires `params` to be a hash.
+      if String === params
+        params = CGI.parse(params)
+
+        # Flatten the parameters for Faraday.
+        params.each do |key,value|
+          if Array === value && value.size == 1
+            params[key] = value.first
+          end
+        end
+      end
+
+      @client.get(url, params).body
+    end
+
+    # Retrieves and parses a document with a POST request.
+    #
+    # @param [String] url a URL to an HTML document
+    # @param [String,Hash] params query string parameters
+    # @return a parsed document
+    def post(url, params = {})
+      @client.post(url, params).body
+    end
+
+    # Adds a scraping task to Pupa.rb.
+    #
+    # Defines a method whose name is identical to `task_name`. This method
+    # selects a method to perform the scraping task using `scraping_task_method`
+    # and memoizes its return value. The return value is a lazy enumerator of
+    # objects scraped by the selected method. The selected method must yield
+    # objects to populate this lazy enumerator.
+    #
+    # For example, `MyProcessor.add_scraping_task(:people)` defines a `people`
+    # method on `MyProcessor`. This `people` method returns a lazy enumerator of
+    # objects (presumably Person objects in this case, but the enumerator can
+    # contain any object in the general case).
+    #
+    # In `MyProcessor`, you would define an `scrape_people` method, which must
+    # yield objects to populate the lazy enumerator. Alternatively, you may
+    # override `scraping_task_method` to change the method selected to perform
+    # the scraping task.
+    #
+    # The `people` method can then be called by transformation and import tasks.
+    #
+    # @param [Symbol] task_name a task name
+    # @see Pupa::Processor#scraping_task_method
+    def self.add_scraping_task(task_name)
+      self.tasks += [task_name]
+      define_method(task_name) do
+        ivar = "@#{task_name}"
+        if instance_variable_defined?(ivar)
+          instance_variable_get(ivar)
+        else
+          instance_variable_set(ivar, Yielder.new(&method(scraping_task_method(task_name))))
+        end
+      end
+    end
+
+    # Dumps scraped objects to disk.
+    #
+    # @param [Symbol] task_name the name of the scraping task to perform
+    def dump_scraped_objects(task_name)
+      send(task_name).each do |object|
+        dump_scraped_object(object)
+      end
+    end
+
+    # Saves scraped objects to a database.
+    #
+    # @raises [TSort::Cyclic] if the dependency graph is cyclic
+    # @raises [Pupa::Errors::UnprocessableEntity] if an object's foreign keys or
+    #   foreign objects cannot be resolved
+    # @raises [Pupa::Errors::DuplicateDocumentError] if duplicate objects were
+    #   inadvertently saved to the database
+    def import
+      objects = deduplicate(load_scraped_objects)
+
+      object_id_to_database_id = {}
+
+      if use_dependency_graph?(objects)
+        dependency_graph = build_dependency_graph(objects)
+
+        # Replace object IDs with database IDs in foreign keys and save objects.
+        dependency_graph.tsort.each do |id|
+          object = objects[id]
+          resolve_foreign_keys(object, object_id_to_database_id)
+          # The dependency graph strategy only works if there are no foreign objects.
+          object_id_to_database_id[id] = Persistence.new(object).save
+        end
+      else
+        size = objects.size
+
+        # Should be O(n²). If there are foreign objects, we do not know all the
+        # edges in the graph, and therefore cannot build a dependency graph or
+        # derive any evaluation order.
+        #
+        # An exception is raised if a foreign object matches multiple documents
+        # in the database. However, if a matching object is not yet saved, this
+        # exception may not be raised.
+        loop do
+          progress_made = false
+
+          objects.delete_if do |id,object|
+            resolvable = true
+
+            resolvable &= object.foreign_keys.all? do |property|
+              value = object[property]
+              value.nil? || object_id_to_database_id.key?(value)
+            end
+
+            resolvable &= object.foreign_objects.all? do |property|
+              selector = object[property]
+              selector.blank? || Persistence.find(selector)
+            end
+
+            if resolvable
+              progress_made = true
+              resolve_foreign_keys(object, object_id_to_database_id)
+              resolve_foreign_objects(object)
+              object_id_to_database_id[id] = Persistence.new(object).save
+            end
+          end
+
+          break if objects.empty? || !progress_made
+        end
+
+        unless objects.empty?
+          raise Errors::UnprocessableEntity, "couldn't resolve #{objects.size}/#{size} objects:\n  #{objects.values.map{|object| JSON.dump(object.foreign_properties)}.join("\n  ")}"
+        end
+      end
+
+      # Ensure that fingerprints uniquely identified objects.
+      counts = {}
+      object_id_to_database_id.each do |object_id,database_id|
+        (counts[database_id] ||= []) << object_id
+      end
+      duplicates = counts.select do |_,object_ids|
+        object_ids.size > 1
+      end
+      unless duplicates.empty?
+        raise Errors::DuplicateDocumentError, "multiple objects written to same document:\n" + duplicates.map{|database_id,object_ids| "  #{database_id} <- #{object_ids.join(' ')}"}.join("\n")
+      end
+    end
+
+  private
+
+    # Returns the name of the method - `scrape_<task_name>` by default - that
+    # would be used to perform the given scraping task.
+    #
+    # If you would like to change this default behavior, override this method in
+    # a subclass. For example, you may want to select a method according to the
+    # additional `options` passed from the command-line to the processor.
+    #
+    # @param [Symbol] task_name a task name
+    # @return [String] the name of the method to perform the scraping task
+    def scraping_task_method(task_name)
+      "scrape_#{task_name}"
+    end
+
+    # Dumps an scraped object to disk.
+    #
+    # @param [Object] object an scraped object
+    # @raises [Pupa::Errors::DuplicateObjectIdError]
+    def dump_scraped_object(object)
+      type = object.class.to_s.demodulize.underscore
+      basename = "#{type}_#{object._id}.json"
+      path = File.join(@output_dir, basename)
+
+      if File.exist?(path)
+        raise Errors::DuplicateObjectIdError, "duplicate object ID: #{object._id} (was the same objected yielded twice?)"
+      end
+
+      info {"save #{type} #{object.to_s} as #{basename}"}
+
+      File.open(path, 'w') do |f|
+        f.write(JSON.dump(object.to_h(include_foreign_objects: true)))
+      end
+
+      begin
+        object.validate!
+      rescue JSON::Schema::ValidationError => e
+        warn {e.message}
+      end
+    end
+
+    # Loads scraped objects from disk.
+    #
+    # @return [Hash] a hash of scraped objects keyed by ID
+    def load_scraped_objects
+      {}.tap do |objects|
+        Dir[File.join(@output_dir, '*.json')].each do |path|
+          data = JSON.load(File.read(path))
+          object = data['_type'].camelize.constantize.new(data)
+          objects[object._id] = object
+        end
+      end
+    end
+
+    # Removes all duplicate objects and re-assigns any foreign keys.
+    #
+    # @param [Hash] objects a hash of scraped objects keyed by ID
+    # @return [Hash] the objects without duplicates
+    def deduplicate(objects)
+      losers_to_winners = build_losers_to_winners_map(objects)
+
+      # Remove all losers.
+      losers_to_winners.each_key do |key|
+        objects.delete(key)
+      end
+
+      # Swap the IDs of losers for the IDs of winners.
+      objects.each do |id,object|
+        object.foreign_keys.each do |property|
+          value = object[property]
+          if value && losers_to_winners.key?(value)
+            object[property] = losers_to_winners[value]
+          end
+        end
+      end
+
+      objects
+    end
+
+    # For each object, map its ID to the ID of its duplicate, if any.
+    #
+    # @param [Hash] objects a hash of scraped objects keyed by ID
+    # @return [Hash] a mapping from an object ID to the ID of its duplicate
+    def build_losers_to_winners_map(objects)
+      {}.tap do |map|
+        objects.each_with_index do |(id1,object1),index|
+          unless map.key?(id1) # Don't search for duplicates of duplicates.
+            objects.drop(index + 1).each do |id2,object2|
+              if object1 == object2
+                map[id2] = id1
+              end
+            end
+          end
+        end
+      end
+    end
+
+    # If any objects have unresolved foreign objects, we cannot derive an
+    # evaluation order using a dependency graph.
+    #
+    # @param [Hash] objects a hash of scraped objects keyed by ID
+    # @return [Boolean] whether a dependency graph can be used to derive an
+    #   evaluation order
+    def use_dependency_graph?(objects)
+      objects.each do |id,object|
+        object.foreign_objects.each do |property|
+          if object[property].present?
+            return false
+          end
+        end
+      end
+      true
+    end
+
+    # Builds a dependency graph.
+    #
+    # @param [Hash] objects a hash of scraped objects keyed by ID
+    # @return [DependencyGraph] the dependency graph
+    def build_dependency_graph(objects)
+      DependencyGraph.new.tap do |graph|
+        objects.each do |id,object|
+          graph[id] = [] # no duplicate IDs
+          object.foreign_keys.each do |property|
+            value = object[property]
+            if value
+              graph[id] << value
+            end
+          end
+        end
+      end
+    end
+
+    # Resolves an object's foreign keys from object IDs to database IDs.
+    #
+    # @param [Object] an object
+    # @param [Hash] a map from object ID to database ID
+    # @raises [Pupa::Errors::MissingDatabaseIdError]
+    def resolve_foreign_keys(object, map)
+      object.foreign_keys.each do |property|
+        value = object[property]
+        if value
+          # If using a dependency graph, any foreign key that cannot be resolved
+          # will cause a key error while building the dependency graph.
+          #
+          # If not using a dependency graph, this method will not be called
+          # unless the foreign key is resolvable.
+          object[property] = map[value]
+        end
+      end
+    end
+
+    # Resolves an object's foreign objects to database IDs.
+    #
+    # @param [Object] an object
+    # @raises [Pupa::Errors::MissingDatabaseIdError]
+    def resolve_foreign_objects(object)
+      object.foreign_objects.each do |property|
+        selector = object[property]
+        if selector.present?
+          # This method will not be called unless the foreign key is resolvable.
+          object["#{property}_id"] = Persistence.find(selector)['_id']
+        end
+      end
+    end
+  end
+end

data/lib/pupa/refinements/faraday_middleware.rb

@@ -0,0 +1,32 @@
+module Pupa
+  class Refinements
+    # A refinement for the Faraday caching middleware to cache all requests, not
+    # only GET requests.
+    module FaradayMiddleware
+      refine ::FaradayMiddleware::Caching do
+        def call(env)
+          # Remove if-statement to cache any request, not only GET.
+          if env[:parallel_manager]
+            # callback mode
+            cache_on_complete(env)
+          else
+            # synchronous mode
+            response = cache.fetch(cache_key(env)) { @app.call(env) }
+            finalize_response(response, env)
+          end
+        end
+
+        def cache_key(env)
+          url = env[:url].dup
+          if url.query && params_to_ignore.any?
+            params = parse_query url.query
+            params.reject! {|k,| params_to_ignore.include? k }
+            url.query = build_query params
+          end
+          url.normalize!
+          url.request_uri + env[:body].to_s # Add for POST requests.
+        end
+      end
+    end
+  end
+end

data/lib/pupa/refinements/json-schema.rb

@@ -0,0 +1,36 @@
+module Pupa
+  class Refinements
+    # A refinement for JSON Schema to validate "email" and "uri" formats.
+    module Format
+      # @see http://my.rails-royce.org/2010/07/21/email-validation-in-ruby-on-rails-without-regexp/
+      def validate(current_schema, data, fragments, processor, validator, options = {})
+        case current_schema.schema['format']
+        when 'email'
+          error_message = "The property '#{build_fragment(fragments)}' must be a valid email address"
+          validation_error(processor, error_message, fragments, current_schema, self, options[:record_errors]) and return if !data.is_a?(String)
+          address = Mail::Address.new(data)
+          unless (address.address == data && address.domain && address.__send__(:tree).domain.dot_atom_text.elements.size > 1 rescue false)
+            validation_error(processor, error_message, fragments, current_schema, self, options[:record_errors])
+            return
+          end
+        when 'uri'
+          error_message = "The property '#{build_fragment(fragments)}' must be a valid URI"
+          validation_error(processor, error_message, fragments, current_schema, self, options[:record_errors]) and return if !data.is_a?(String)
+          r = URI::DEFAULT_PARSER.regexp[:ABS_URI]
+          unless r.match(data)
+            validation_error(processor, error_message, fragments, current_schema, self, options[:record_errors])
+            return
+          end
+        else
+          super
+        end
+      end
+    end
+
+    class ::JSON::Schema::FormatAttribute
+      class << self
+        prepend Pupa::Refinements::Format
+      end
+    end
+  end
+end

data/lib/pupa/runner.rb

@@ -0,0 +1,185 @@
+require 'fileutils'
+require 'optparse'
+require 'ostruct'
+
+require 'moped'
+
+module Pupa
+  class Runner
+    attr_reader :options, :actions
+
+    # @param [Pupa::Processor] a processor class
+    # @param [Hash] defaults change any default options
+    def initialize(processor_class, defaults = {})
+      @processor_class = processor_class
+
+      @options = OpenStruct.new({
+        actions:        [],
+        tasks:          [],
+        output_dir:     File.expand_path('scraped_data', Dir.pwd),
+        cache_dir:      File.expand_path('web_cache', Dir.pwd),
+        expires_in:     86400, # 1 day
+        host_with_port: 'localhost:27017',
+        database:       'pupa',
+        dry_run:        false,
+        level:          'INFO',
+      }.merge(defaults))
+
+      @actions = {
+        'scrape' => 'Scrapes data from online sources',
+        'import' => 'Imports scraped data into a database',
+      }.map do |name,description|
+        OpenStruct.new(name: name, description: description)
+      end
+    end
+
+    # @param [Hash] attributes the action's attributes
+    # @option attributes [String] :name the action's label
+    # @option attributes [String] :description a description of the action
+    def add_action(attributes)
+      @actions << OpenStruct.new(attributes)
+    end
+
+    # Returns the command-line option parser.
+    #
+    # @return [OptionParser] the command-line option parser
+    def opts
+      @opts ||= OptionParser.new do |opts|
+        opts.program_name = File.basename($PROGRAM_NAME)
+        opts.banner = "Usage: #{opts.program_name}"
+
+        opts.separator ''
+        opts.separator 'Actions:'
+
+        names = @actions.map(&:name)
+        padding = names.map(&:size).max
+        @actions.each do |action|
+          opts.separator "  #{action.name.ljust(padding)}  #{action.description}\n"
+        end
+
+        opts.separator ''
+        opts.separator 'Tasks:'
+
+        @processor_class.tasks.each do |task_name|
+          opts.separator "  #{task_name}"
+        end
+
+        opts.separator ''
+        opts.separator 'Specific options:'
+        opts.on('-a', '--action ACTION', names, 'Select an action to run (you may give this switch multiple times)', "  (#{names.join(', ')})") do |v|
+          options.actions << v
+        end
+        opts.on('-t', '--task TASK', @processor_class.tasks, 'Select a scraping task to run (you may give this switch multiple times)', "  (#{@processor_class.tasks.join(', ')})") do |v|
+          options.tasks << v
+        end
+        opts.on('-o', '--output_dir PATH', 'The directory in which to dump JSON documents') do |v|
+          options.output_dir = v
+        end
+        opts.on('-c', '--cache_dir PATH', 'The directory in which to cache HTTP requests') do |v|
+          options.cache_dir = v
+        end
+        opts.on('-e', '--expires_in SECONDS', "The cache's expiration time in seconds") do |v|
+          options.expires_in = v
+        end
+        opts.on('-H', '--host HOST:PORT', 'The host and port to MongoDB') do |v|
+          options.host_with_port = v
+        end
+        opts.on('-d', '--database NAME', 'The name of the MongoDB database') do |v|
+          options.database = v
+        end
+        opts.on('-n', '--dry-run', 'Show the plan without running any actions') do
+          options.dry_run = true
+        end
+        opts.on('-v', '--verbose', 'Show all messages') do
+          options.level = 'DEBUG'
+        end
+        opts.on('-q', '--quiet', 'Show only warning and error messages') do
+          options.level = 'WARN'
+        end
+        opts.on('-s', '--silent', 'Show no messages') do
+          options.level = 'UNKNOWN'
+        end
+
+        opts.separator ''
+        opts.separator 'Common options:'
+        opts.on_tail('-h', '--help', 'Show this message') do
+          puts opts
+          exit
+        end
+        opts.on_tail('-v', '--version', 'Show version') do
+          puts Pupa::VERSION
+          exit
+        end
+      end
+    end
+
+    # Runs the action.
+    #
+    # @example Run from a command-line script
+    #
+    #     runner.run(ARGV)
+    #
+    # @example Override the command-line options
+    #
+    #     runner.run(ARGV, expires_in: 3600) # 1 hour
+    #
+    # @param [Array] args command-line arguments
+    # @param [Hash] overrides any overridden options
+    def run(args, overrides = {})
+      rest = opts.parse!(args)
+
+      @options = OpenStruct.new(options.to_h.merge(overrides))
+
+      if options.actions.empty?
+        options.actions = %w(scrape import)
+      end
+      if options.tasks.empty?
+        options.tasks = @processor_class.tasks
+      end
+
+      processor = @processor_class.new(options.output_dir, cache_dir: options.cache_dir, expires_in: options.expires_in, level: options.level, options: Hash[*rest])
+
+      options.actions.each do |action|
+        unless action == 'scrape' || processor.respond_to?(action)
+          abort %(`#{action}` is not a #{opts.program_name} action. See `#{opts.program_name} --help` for a list of available actions.)
+        end
+      end
+
+      if %w(DEBUG INFO).include?(options.level)
+        puts "processor: #{@processor_class}"
+        puts "actions: #{options.actions.join(', ')}"
+        puts "tasks: #{options.tasks.join(', ')}"
+      end
+
+      if options.level == 'DEBUG'
+        %w(output_dir cache_dir expires_in host_with_port database level).each do |option|
+          puts "#{option}: #{options[option]}"
+        end
+        unless rest.empty?
+          puts "options: #{rest.join(' ')}"
+        end
+      end
+
+      exit if options.dry_run
+
+      Pupa.session = Moped::Session.new([options.host_with_port], database: options.database)
+
+      if options.actions.delete('scrape')
+        FileUtils.mkdir_p(options.output_dir)
+        FileUtils.mkdir_p(options.cache_dir)
+
+        Dir[File.join(options.output_dir, '*.json')].each do |path|
+          FileUtils.rm(path)
+        end
+
+        options.tasks.each do |task_name|
+          processor.dump_scraped_objects(task_name)
+        end
+      end
+
+      options.actions.each do |action|
+        processor.send(action)
+      end
+    end
+  end
+end

data/lib/pupa/version.rb

@@ -0,0 +1,3 @@
+module Pupa
+  VERSION = "0.0.1"
+end

data/lib/pupa.rb

@@ -0,0 +1,31 @@
+require 'forwardable'
+
+require 'active_support/concern'
+require 'active_support/core_ext/class/attribute'
+require 'active_support/core_ext/object/blank'
+require 'active_support/inflector'
+
+require 'pupa/errors'
+require 'pupa/logger'
+require 'pupa/processor'
+require 'pupa/runner'
+
+require 'pupa/models/concerns/contactable'
+require 'pupa/models/concerns/identifiable'
+require 'pupa/models/concerns/linkable'
+require 'pupa/models/concerns/nameable'
+require 'pupa/models/concerns/sourceable'
+require 'pupa/models/concerns/timestamps'
+
+require 'pupa/models/base'
+require 'pupa/models/contact_detail_list'
+require 'pupa/models/membership'
+require 'pupa/models/organization'
+require 'pupa/models/person'
+require 'pupa/models/post'
+
+module Pupa
+  class << self
+    attr_accessor :session
+  end
+end

data/pupa.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/pupa/version', __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "pupa"
+  s.version     = Pupa::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Open North"]
+  s.email       = ["info@opennorth.ca"]
+  s.homepage    = "http://github.com/opennorth/pupa-ruby"
+  s.summary     = %q{A data scraping framework}
+  s.license     = 'MIT'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency('activesupport', '~> 4.0.0')
+  s.add_runtime_dependency('colored', '~> 1.2')
+  s.add_runtime_dependency('faraday_middleware', '~> 0.9.0')
+  s.add_runtime_dependency('json-schema', '~> 2.1.3')
+  s.add_runtime_dependency('mail')
+  s.add_runtime_dependency('moped', '~> 1.5.1')
+  s.add_runtime_dependency('nokogiri', '~> 1.6.0')
+
+  s.add_development_dependency('coveralls')
+  s.add_development_dependency('json', '~> 1.7.7') # to silence coveralls warning
+  s.add_development_dependency('octokit') # to update Popolo schema
+  s.add_development_dependency('rake')
+  s.add_development_dependency('rspec', '~> 2.10')
+  s.add_development_dependency('vcr', '~> 2.5.0')
+  s.add_development_dependency('multi_xml')
+end