openc_bot 0.0.11

data/lib/openc_bot/templates/bin/fetch_data

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'trollop'
+require 'open3'
+
+opts = Trollop::options do
+  opt :test, "Run in test mode", :short => 't'
+
+end
+
+command = "bundle exec openc_bot rake bot:run[#{opts[:test]||''}]"
+options = { chdir: File.join(File.dirname(__FILE__), "..") }
+_, stdout, stderr, wait_thread = Open3::popen3(command, options)
+result = wait_thread.value
+
+puts "Running in test mode" if opts[:test]
+
+if result.success?
+  puts stdout.read
+  exit 0
+else
+  STDERR.puts stderr.read
+  exit 1
+end

data/lib/openc_bot/templates/bin/verify_data

@@ -0,0 +1 @@
+bundle exec openc_bot rake bot:test

data/lib/openc_bot/templates/config.yml

@@ -0,0 +1,21 @@
+# This is a short description of the kind of data the bot handles.
+description: ''
+
+# This is your confidence, between 0 and 100, in the accuracy of the data
+# provided by the data source.  It is suggested that you do not change this
+# without first discussing with OpenCorporates.
+default_confidence: 80
+
+# Don't change this.  OpenCorporates will bump this version number when the bot
+# is running in production.
+version: 0
+
+# the frequency that updates may happen at the source. One of daily,
+# weekly, monthly or yearly
+frequency: monthly
+
+# If this flag is true, our deployment script will set up the bot to
+# be run on OpenCorporates servers (specifically, deploy the bot and
+# allow it to be executed from the data pipeline). Please don't change
+# this; we'll change it when we're ready to test the bot.
+enabled: false

data/lib/openc_bot/templates/lib/bot.rb

@@ -0,0 +1,43 @@
+# encoding: UTF-8
+require 'openc_bot'
+
+# you may need to require other libraries here
+#
+# require 'nokogiri'
+
+module MyModule
+  extend OpencBot
+  extend self # make these methods as Module methods, rather than instance ones
+
+  def export_data
+    # This is the basic functionality for exporting the data from the database. By default the data
+    # table (what is created when you save_data) is called ocdata, but it can be called anything else,
+    # and the query can be more complex, returning, for example, only the most recent results.
+    sql_query = "ocdata.* from ocdata"
+    select(sql_query).collect do |raw_datum|
+      # raw_datum will be a Hash of field names (as symbols) for the keys and the values for each field.
+      # It should be converted to the format necessary for importing into OpenCorporates by using a
+      # prepare_for_export method.
+      prepare_for_export(raw_datum)
+    end
+  end
+
+  def prepare_for_export(raw_data)
+    # do something here to convert the raw data from the database (if you are using one) into
+    # the form required by the export.
+  end
+
+  def update_data
+    # write code here (using other methods if necessary) for
+    # updating your local database with data from the source
+    # that you are scraping or fetching from
+    #
+    # # See https://github.com/openc/openc_bot README for details
+    # save_data([:uid,:date], my_data, sometablename)
+    #
+    # After updating the data you should run save_run_report, which
+    # saves the status (and other data, if applicable)
+    save_run_report(:status => 'success')
+  end
+
+end

data/lib/openc_bot/templates/lib/company_fetcher_bot.rb

@@ -0,0 +1,95 @@
+# encoding: UTF-8
+require 'openc_bot'
+require 'openc_bot/company_fetcher_bot'
+
+# you may need to require other libraries here
+#
+# require 'nokogiri'
+
+# uncomment (and line further down) to get Date helper methods. (Also available csv and text helpers)
+# require 'openc_bot/helpers/dates'
+
+module MyModule
+  extend OpencBot
+  # This adds the CompanyFetcherBot functionality
+  extend OpencBot::CompanyFetcherBot
+  # uncomment to get Date helper methods
+  # extend OpencBot::Helpers::Dates
+  extend self # make these methods as Module methods, rather than instance ones
+
+
+  # Uncomment to use alpha search – default is incremental search
+  # USE_ALPHA_SEARCH = true
+
+  # Default number of characters used for search terms in alpha search. Default is 1 (i.e. 'A','B'...)
+  # NUMBER_OF_CHARS_IN_SEARCH = 3
+
+
+  # If the register has a GET'able URL based on the company_number define it here. This should mean that
+  # #fetch_datum 'just works'.
+  def computed_registry_url(company_number)
+    # e.g.
+    # "http://some,register.com/path/to/#{company_number}"
+  end
+
+  # #fetch_data is the primary method for getting companies from the register, and by default is
+  # called when the bot is 'run' (e.g. via bundle exec openc_bot rake bot:run, which calls
+  # #update_data, which in turn calls this)
+  # By default this uses an incremental search (which increments through :company_number identifiers),
+  # or if USE_ALPHA_SEARCH has been set, an alpha search (e.g. searching for entities using 'AA', 'AB')
+  # Define this locally if a different method for getting companies is going to done (e.g.
+  # parsing a CSV file)
+  # def fetch_data
+  # end
+
+  # This is called by #update_datum (defined in the IncrementalSearch helper module), which updates the
+  # information for a given company_number. This allows the individual records to be updated, for example,
+  # via the 'Update from Register' button on the company page on OpenCorporates. This method is also called
+  # by the #fetch_data method in the case of incremental_searches.
+  # By default it calls #fetch_registry_page with the company_number and returns the result in a hash,
+  # with :company_page as a key. This will then be processed or parsed by the #process_datum method,
+  # and the result will be saved by #update_datum, and also returned in a form that can be used by the
+  # main OpenCorporates system
+  #
+  # This hash can contain other data, such as a page of filings or shareholdings. The hash will be
+  # converted to json, and stored in the database in the row for that company number, under the
+  # :data key, so that it can be reused or referred it in the future.
+  # {:company_page => company_page_html, :filings_page => filings_page_html}
+  # def fetch_datum(company_number)
+  # end
+
+  # This method must be defined for all bots that can fetch and process individual records, e.g.
+  # incremental searchers, or individual company pages in an alpha search.
+  # Where the bot cannot do this (e.g. where the underlying data is
+  # only available as a CSV file, or there are no individual pages for each company, it can be
+  # left as a stub method)
+  # It should return a hash that conforms to the company-schema schema (and it will be checked)
+  # against this schema before saving
+  def process_datum(datum_hash)
+    # write your code to parse what is in the company pages/data
+  end
+
+  # This is the standard method for alpha searches e.g. where you are searching a series of terms,
+  # from A-Z0-9. You can increase the number of characters in the search term by setting the
+  # NUMBER_OF_CHARS_IN_SEARCH constant (see above). Define this method locally if you need different
+  # behavtiour o this
+  # def fetch_data_via_alpha_search(options={})
+  #   starting_term = options[:starting_term]||get_var('starting_term')
+  #   each_search_term(starting_term) do |term|
+  #     save_var('starting_term', term)
+  #     search_for_entities_for_term(term, options) do |entity_datum|
+  #       save_entity(entity_datum)
+  #     end
+  #   end
+  #   # reset pointer
+  #   save_var('starting_term',nil)
+  # end
+
+  # This method is called by #fetch_data_via_alpha_search (defined in AlphaSearch helper),
+  # and is passed a search term, typically search term of a number of characters (e.g. 'AB', 'AC'...).
+  # This method should yield a hash of company data which can be validated to the company-schema
+  def search_for_entities_for_term(term, options={})
+    # write your code to search all the pages for the given term, and yield a series of company hashes
+  end
+
+end

data/lib/openc_bot/templates/lib/simple_bot.rb

@@ -0,0 +1,67 @@
+# encoding: UTF-8
+require 'simple_openc_bot'
+
+# you may need to require other libraries here
+# require 'nokogiri'
+# require 'mechanize'
+
+class MyLicenceRecord < SimpleOpencBot::BaseLicenceRecord
+  # The JSON schema to use to validate records; correspond with files
+  # in `schema/*-schema.json`
+  schema :licence
+
+  # Fields you define here will be persisted to a local database when
+  # 'fetch_records' (see below) is run.
+  store_fields :name, :type, :reporting_date
+
+  # This is the field(s) which will uniquely define a record (think
+  # primary key in a database).
+  unique_fields :name
+
+  # This must be defined, and should return a timestamp in ISO8601
+  # format. Its value should change when something about the record
+  # has changed. It doesn't have to be a method - it can also be a
+  # member of `store_fields`, above.
+  def last_updated_at
+    reporting_date
+  end
+
+  # This method must be defined. You can test that you're outputting
+  # in the right format with `bin/verify_data`, which will validate
+  # any data you've fetched against the relevant schema. See
+  # `doc/SCHEMA.md` for documentation.
+  def to_pipeline
+    {
+      sample_date: last_updated_at,
+      company: {
+        name: name,
+        jurisdiction: "xx",
+      },
+      source_url: "xx",
+      data: [{
+        data_type: :licence,
+        properties: {
+          jurisdiction_code: "xx",
+          category: 'Financial',
+          jurisdiction_classification: [type],
+        }
+      }]
+    }
+  end
+
+end
+
+class MyLicence < SimpleOpencBot
+
+  # the class that `fetch_records` yields. Must be defined.
+  yields MyLicenceRecord
+
+  # This method should yield Records. It must be defined.
+  def fetch_all_records(opts={})
+    data = [{:name => "A", :type => "B"}]
+    data.each do |datum|
+      yield MyLicenceRecord.new(
+        datum.merge(:reporting_date => Time.now.iso8601(2)))
+    end
+  end
+end

data/lib/openc_bot/templates/spec/bot_spec.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+require_relative '../lib/my_module'
+
+describe MyModule do
+
+  it "should extend with OpencBot methods" do
+    MyModule.should respond_to :save_data
+  end
+
+end

data/lib/openc_bot/templates/spec/simple_bot_spec.rb

@@ -0,0 +1,11 @@
+# encoding: UTF-8
+require_relative 'spec_helper'
+require_relative '../lib/my_module'
+
+describe MyModule do
+
+  it "should extend with OpencBot methods" do
+    MyModule.should respond_to :save_data
+  end
+
+end

data/lib/openc_bot/templates/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+require 'rspec/autorun'
+require 'debugger'
+
+RSpec.configure do |config|
+
+end
+
+# Utility method to allow sample html pages, csv files, json or whatever.
+# Expects the files to be stored in a 'dummy_responses' folder in the spec directory
+#
+def dummy_response(response_name, options={})
+  IO.read(File.join(File.dirname(__FILE__),"dummy_responses",response_name.to_s), options)
+end

data/lib/openc_bot/version.rb

@@ -0,0 +1,3 @@
+module OpencBot
+  VERSION = "0.0.11"
+end

data/lib/simple_openc_bot.rb

@@ -0,0 +1,289 @@
+require 'active_support/core_ext'
+require 'openc_bot'
+require 'json-schema'
+require 'openc_bot/incrementers'
+
+class SimpleOpencBot
+  include OpencBot
+
+  class_attribute :_yields
+
+  def self.yields(*fields)
+    raise "We currently only support one Record type per bot" if fields.count > 1
+    self._yields = fields
+  end
+
+  def self.inherited(obj)
+    path, = caller[0].partition(":")
+    path = File.expand_path(File.join(File.dirname(path), ".."))
+    @@simple_app_directory = path
+  end
+
+  # Override default in ScraperWiki gem
+  def sqlite_magic_connection
+    db = @config ? @config[:db] : File.expand_path(File.join(@@simple_app_directory, 'db', db_name))
+    @sqlite_magic_connection ||= SqliteMagic::Connection.new(db)
+  end
+
+  def update_data(opts={})
+    if opts[:specific_ids].nil? || opts[:specific_ids].empty?
+      # fetch everything
+      record_enumerator = Enumerator.new do |yielder|
+        fetch_all_records(opts) do |result|
+          yielder.yield(result)
+        end
+      end
+    else
+      # fetch records with specified ids
+      record_enumerator = Enumerator.new do |yielder|
+        fetch_specific_records(opts) do |result|
+          yielder.yield(result)
+        end
+      end
+    end
+    saves_count = 0
+    batch_size = opts[:test_mode] ? 1 : 500
+    record_enumerator.each_slice(batch_size) do |records|
+      begin
+        sqlite_magic_connection.execute("BEGIN TRANSACTION")
+        records.each do |record|
+          insert_or_update(record.class.unique_fields,
+            record.to_hash)
+          saves_count += 1
+          if saves_count == 1
+            # TODO: move this validation to somewhere more explicit
+            raise "Bot must specify what record type it will yield" if _yields.nil?
+            check_unique_index(_yields[0])
+          end
+          STDOUT.print(".")
+          STDOUT.flush
+        end
+      ensure 
+        sqlite_magic_connection.execute("COMMIT") if sqlite_magic_connection.database.transaction_active?
+      end
+    end
+    # NB #save_run_report saves a timestamp called :run_at, so :completed_at is redundant.
+    # Probably should either remove or replace with another datapoint, e.g. started_at
+    save_run_report(:status => 'success', :completed_at => Time.now)
+    saves_count
+  end
+
+  # should this be a method in sqlite_magic gem?
+  def check_unique_index(record_class)
+    indexes = sqlite_magic_connection.execute("PRAGMA INDEX_LIST('ocdata')")
+    db_unique_fields = indexes.map do |i|
+      next if i["unique"] != 1
+      next unless i["name"] =~ /autoindex/
+      info = sqlite_magic_connection.execute("PRAGMA INDEX_INFO('#{i["name"]}')")
+      info.map{|x| x["name"]}
+    end.compact.flatten
+    record_unique_fields = record_class.unique_fields.map(&:to_s)
+    if db_unique_fields != record_unique_fields
+      sqlite_magic_connection.execute("ROLLBACK")
+      error = "Unique fields #{record_unique_fields} do not match the unique index(es) in `ocdata` table!"
+      error += "\nThis is usually because the value of unique_field has changed since the table was automatically created."
+      error += "\nUnique fields in `ocdata`: #{db_unique_fields.flatten}; in record #{record_class.name}: #{record_unique_fields}"
+      raise error
+    end
+  end
+
+  def count_stored_records
+    begin
+      all_stored_records(:count => true).first["count"]
+    rescue SqliteMagic::NoSuchTable
+      0
+    end
+  end
+
+  def all_stored_records(opts={})
+    if opts[:only_unexported]
+      opts[:limit] ||= opts[:batch] 
+    end
+
+    select = opts[:select] || "ocdata.*"
+    table = opts[:table] || "ocdata"
+    where = (opts[:where] ?  "\nWHERE #{opts[:where]}\n" : "\nWHERE 1 \n")
+    order = (opts[:order] ?  "\nORDER BY #{opts[:order]}\n" : "")
+    limit = (opts[:limit] ? "\nLIMIT #{opts[:limit]}\n" : "")
+
+    if opts[:only_unexported]
+      where += " AND (_last_exported_at IS NULL "\
+        "OR _last_exported_at < _last_updated_at)"
+
+      if !opts[:specific_ids].blank?
+        ids = opts[:specific_ids].map{|id| "'#{id}'"}.join(",")
+        where += " AND #{_yields[0].unique_field} IN (#{ids})"
+      end
+    end
+
+    if opts[:count]
+      sql = "COUNT(*) AS count from #{table} #{where}"
+      puts sql if opts[:debug]
+      select(sql)
+    else
+      sql = "#{select} from #{table} #{where} #{order} #{limit}"
+      puts sql if opts[:debug]
+      select_records(sql)
+    end
+  end
+
+  def unexported_stored_records(opts={})
+    all_stored_records(opts.merge!(:only_unexported => true))
+  end
+
+  def spotcheck_records(limit = 5)
+    all_stored_records(:order => "RANDOM()", :limit => limit)
+  end
+
+  def select_records(sql)
+    select(sql).map { |record| record['_type'].constantize.new(record) }
+  end
+
+  def export_data(opts={})
+    begin
+      sqlite_magic_connection.add_columns(
+        'ocdata', [:_last_exported_at, :_last_updated_at])
+    rescue SQLite3::SQLException
+    end
+    Enumerator.new do |yielder|
+      b = 1
+      loop do
+        if opts[:all]
+          break if b > 1
+          batch = all_stored_records(opts)
+        else
+          batch = unexported_stored_records(:batch => 100, :specific_ids => opts[:specific_ids])
+        end
+        break if batch.empty?
+        updates = {}
+        batch.map do |record|
+          pipeline_data = record.to_pipeline
+          next if pipeline_data.nil?
+          updates[record.class.name] ||= []
+          # opts[:all] is currently called in the bot:test rake task
+          # This has the unfortunate side effect of updating the _last_exported_at
+          # time when running the validation task, so I've added the following conditional
+          if !opts[:all]
+            updates[record.class.name] << record.to_hash.merge(
+              :_last_exported_at => Time.now.iso8601(2))
+          else
+            updates[record.class.name] << record.to_hash
+          end
+          yielder << pipeline_data
+        end
+        sqlite_magic_connection.execute("BEGIN TRANSACTION")
+        if b == 1
+          check_unique_index(_yields[0])
+        end
+        updates.each do |k, v|
+          save_data(k.constantize.unique_fields, v)
+        end
+        sqlite_magic_connection.execute("COMMIT")
+        b += 1
+      end
+    end
+  end
+
+  def spotcheck_data
+    batch = spotcheck_records
+    batch.collect do |record|
+      record.to_pipeline
+    end
+  end
+
+  def validate_data(opts={})
+    opts = {:limit => 1000}.merge(opts)
+    errors = all_stored_records(opts).map do |record|
+      record.errors
+    end.compact
+    total = count_stored_records
+    selected = [opts[:limit], total].min
+    puts "NOTICE: only validated first #{selected} of #{total} records"
+    errors
+  end
+
+
+  class BaseLicenceRecord
+    class_attribute :_store_fields, :_type, :_schema, :_unique_fields
+
+    def self.store_fields(*fields)
+      self._store_fields ||= []
+      self._store_fields.concat(fields)
+      fields << :_last_exported_at unless _store_fields.include?(:_last_exported_at)
+      fields << :_last_updated_at unless _store_fields.include?(:_last_updated_at)
+      fields.each do |field|
+        attr_accessor field
+      end
+    end
+
+    def self.unique_fields(*fields)
+      self._unique_fields = fields unless fields.empty?
+      self._unique_fields
+    end
+
+    def self.schema(schema)
+      hyphenated_name = schema.to_s.gsub("_", "-")
+      self._schema = File.expand_path("../../schemas/#{hyphenated_name}-schema.json", __FILE__)
+    end
+
+    def initialize(attrs={})
+      validate_instance!
+      attrs = attrs.with_indifferent_access
+      self._type = self.class.name
+      self._store_fields.each do |k|
+        send("#{k}=", attrs[k])
+      end
+    end
+
+    def validate_instance!
+      all_errors = []
+      required_functions = [:last_updated_at, :to_pipeline]
+      func_errors = []
+      required_functions.each do |func|
+        if !respond_to?(func)
+          func_errors << func
+        end
+      end
+      if !func_errors.empty?
+        all_errors << "You must define the following functions in your record class: #{func_errors.join(', ')}"
+      end
+      field_errors = []
+      required_fields = [:_store_fields, :_unique_fields, :_schema]
+      required_fields.each do |f|
+        if !send(f)
+          field_errors << f.to_s[1..-1]
+        end
+      end
+      if !field_errors.empty?
+        all_errors << "You must define the following fields on your record class: #{field_errors.join(', ')}"
+      end
+      raise all_errors.join('\n') unless all_errors.empty?
+    end
+
+    def to_hash
+      hsh = Hash[_store_fields.map{|field| [field, send(field)]}]
+      hsh[:_type] = self.class.name
+      hsh[:_last_updated_at] = last_updated_at
+      hsh
+    end
+
+    # return a structure including errors if invalid; otherwise return nil
+    def errors
+      data = self.to_pipeline
+      if data
+        if !self._schema
+          # backwards compatibility
+          self._schema = File.expand_path("../../schemas/licence-schema.json", __FILE__)
+        end
+        errors = JSON::Validator.fully_validate(
+          self._schema,
+          data.to_json,
+          {:errors_as_objects => true})
+        if !errors.empty?
+          data[:errors] = errors
+          data
+        end
+      end
+    end
+  end
+end