openc_bot 0.0.11

data/examples/basic/.gitignore

@@ -0,0 +1,3 @@
+/db
+/data
+/tmp

data/examples/basic/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gem 'openc_bot', :git => 'https://github.com/openc/openc_bot.git'
+gem 'mechanize'
+group :test do
+  gem 'rspec'
+  gem 'debugger'
+end

data/examples/basic/config.yml

@@ -0,0 +1,21 @@
+# This is a short description of the kind of data the bot handles.
+description: ''
+
+# This is your confidence, between 0 and 100, in the accuracy of the data
+# provided by the data source.  It is suggested that you do not change this
+# without first discussing with OpenCorporates.
+default_confidence: 80
+
+# Don't change this.  OpenCorporates will bump this version number when the bot
+# is running in production.
+version: 0
+
+# the frequency that updates may happen at the source. One of daily,
+# weekly, monthly or yearly
+frequency: monthly
+
+# If this flag is true, our deployment script will set up the bot to
+# be run on OpenCorporates servers (specifically, deploy the bot and
+# allow it to be executed from the data pipeline). Please don't change
+# this; we'll change it when we're ready to test the bot.
+enabled: false

data/examples/basic/lib/basic.rb

@@ -0,0 +1,88 @@
+# encoding: UTF-8
+require 'simple_openc_bot'
+require 'mechanize'
+
+# you may need to require other libraries here
+# require 'nokogiri'
+
+class BasicRecord < SimpleOpencBot::BaseLicenceRecord
+  # The JSON schema to use to validate records; correspond with files
+  # in `schema/*-schema.json`
+  schema :licence
+
+  # Fields you define here will be persisted to a local database when
+  # 'fetch_records' (see below) is run.
+  store_fields :name, :type, :reporting_date
+
+  # This is the field(s) which will uniquely define a record (think
+  # primary key in a database).
+  unique_fields :name
+
+  # These are just example methods and constants used by
+  # `to_pipeline`, below
+  JURISDICTION = "uk"
+  URL = "http://foo.com"
+
+  def jurisdiction_classification
+    type
+  end
+
+  # This must be defined, and should return a timestamp in ISO8601
+  # format. Its value should change when something about the record
+  # has changed. It doesn't have to be a method - it can also be a
+  # member of `store_fields`, above.
+  def last_updated_at
+    reporting_date
+  end
+
+  # This method must be defined. You can test that you're outputting
+  # in the right format with `bin/verify_data`, which will validate
+  # any data you've fetched against the relevant schema. See
+  # `doc/SCHEMA.md` for documentation.
+  def to_pipeline
+    {
+      sample_date: last_updated_at,
+      company: {
+        name: name,
+        jurisdiction: JURISDICTION,
+      },
+      source_url: URL,
+      data: [{
+        data_type: :licence,
+        properties: {
+          jurisdiction_code: JURISDICTION,
+          category: 'Financial',
+          jurisdiction_classification: [jurisdiction_classification],
+        }
+      }]
+    }
+  end
+
+end
+
+class Basic < SimpleOpencBot
+
+  # the class that `fetch_records` yields. Must be defined.
+  yields BasicRecord
+
+  # This method should yield Records. It must be defined.
+  def fetch_all_records(opts={})
+
+    # you can use any client here, e.g. HTTPClient, open-uri, etc.
+    agent = Mechanize.new
+
+    # This is a live page on our website - have a look to see what's going on.
+    page = agent.get("http://assets.opencorporates.com/test_bot_page.html")
+
+    # We tend to use Nokogiri to parse responses, but again this is up
+    # to you.
+    doc = Nokogiri::HTML(page.body)
+    doc.xpath("//li").map do |li|
+      name, type = li.content.split(":")
+      yield BasicRecord.new(
+        :name => name.strip,
+        :type => type.strip,
+        :reporting_date => Time.now.iso8601(2))
+    end
+  end
+end

data/examples/basic_with_proxy/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+gem 'openc_bot', :git => 'https://github.com/openc/openc_bot.git'
+gem 'mechanize'
+group :test do
+  gem 'rspec'
+  gem 'debugger'
+end

data/examples/basic_with_proxy/config.yml

@@ -0,0 +1,21 @@
+# This is a short description of the kind of data the bot handles.
+description: ''
+
+# This is your confidence, between 0 and 100, in the accuracy of the data
+# provided by the data source.  It is suggested that you do not change this
+# without first discussing with OpenCorporates.
+default_confidence: 80
+
+# Don't change this.  OpenCorporates will bump this version number when the bot
+# is running in production.
+version: 0
+
+# the frequency that updates may happen at the source. One of daily,
+# weekly, monthly or yearly
+frequency: monthly
+
+# If this flag is true, our deployment script will set up the bot to
+# be run on OpenCorporates servers (specifically, deploy the bot and
+# allow it to be executed from the data pipeline). Please don't change
+# this; we'll change it when we're ready to test the bot.
+enabled: false

data/examples/basic_with_proxy/lib/basic_with_proxy.rb

@@ -0,0 +1,103 @@
+# encoding: UTF-8
+require 'simple_openc_bot'
+require 'mechanize'
+
+# you may need to require other libraries here
+# require 'nokogiri'
+
+class BasicWithProxyRecord < SimpleOpencBot::BaseLicenceRecord
+  # The JSON schema to use to validate records; correspond with files
+  # in `schema/*-schema.json`
+  schema :licence
+
+  # Fields you define here will be persisted to a local database when
+  # 'fetch_records' (see below) is run.
+  store_fields :name, :type, :reporting_date
+
+  # This is the field(s) which will uniquely define a record (think
+  # primary key in a database).
+  unique_fields :name
+
+  # These are just example methods and constants used by
+  # `to_pipeline`, below
+  JURISDICTION = "uk"
+  URL = "http://foo.com"
+
+  def jurisdiction_classification
+    type
+  end
+
+  # This must be defined, and should return a timestamp in ISO8601
+  # format. Its value should change when something about the record
+  # has changed. It doesn't have to be a method - it can also be a
+  # member of `store_fields`, above.
+  def last_updated_at
+    reporting_date
+  end
+
+  # This method must be defined. You can test that you're outputting
+  # in the right format with `bin/verify_data`, which will validate
+  # any data you've fetched against the relevant schema. See
+  # `doc/SCHEMA.md` for documentation.
+  def to_pipeline
+    {
+      sample_date: last_updated_at,
+      company: {
+        name: name,
+        jurisdiction: JURISDICTION,
+      },
+      source_url: URL,
+      data: [{
+        data_type: :licence,
+        properties: {
+          jurisdiction_code: JURISDICTION,
+          category: 'Financial',
+          jurisdiction_classification: [jurisdiction_classification],
+        }
+      }]
+    }
+  end
+
+end
+
+class BasicWithProxy < SimpleOpencBot
+
+  # the class that `fetch_records` yields. Must be defined.
+  yields BasicWithProxyRecord
+
+  # This method should yield Records. It must be defined.
+  def fetch_all_records(opts={})
+
+    # you can use any client here, e.g. HTTPClient, open-uri, etc.
+    agent = Mechanize.new
+
+    # This option is set to true when the rake task is called with a
+    # --test switch
+    if opts[:test_mode]
+      # It is recommended to set up a proxy on your computer when
+      # developing and debugging bots. It can greatly speed things up
+      # by removing the network time from the equation (though things
+      # like POSTs won't be cached, anyway)
+
+      # Different agents have different ways of setting a proxy. This
+      # is how Mechanize does it:
+      agent.set_proxy 'localhost', 8123
+    end
+
+    # This is a live page on our website - have a look to see what's
+    # going on. If you have a proxy set up on your computer, the
+    # second time you run this bot, the website won't get hit.
+    page = agent.get("http://assets.opencorporates.com/test_bot_page.html")
+
+    # We tend to use Nokogiri to parse responses, but again this is up
+    # to you.
+    doc = Nokogiri::HTML(page.body)
+    doc.xpath("//li").map do |li|
+      name, type = li.content.split(":")
+      yield BasicWithProxyRecord.new(
+        :name => name.strip,
+        :type => type.strip,
+        :reporting_date => Time.now.iso8601(2))
+    end
+  end
+end

data/examples/bot_with_simple_iterator/Gemfile

@@ -0,0 +1,6 @@
+source 'https://rubygems.org'
+gem 'openc_bot', :git => 'https://github.com/openc/openc_bot.git'
+group :test do
+  gem 'rspec'
+  gem 'debugger'
+end

data/examples/bot_with_simple_iterator/config.yml

@@ -0,0 +1,21 @@
+# This is a short description of the kind of data the bot handles.
+description: ''
+
+# This is your confidence, between 0 and 100, in the accuracy of the data
+# provided by the data source.  It is suggested that you do not change this
+# without first discussing with OpenCorporates.
+default_confidence: 80
+
+# Don't change this.  OpenCorporates will bump this version number when the bot
+# is running in production.
+version: 0
+
+# the frequency that updates may happen at the source. One of daily,
+# weekly, monthly or yearly
+frequency: monthly
+
+# If this flag is true, our deployment script will set up the bot to
+# be run on OpenCorporates servers (specifically, deploy the bot and
+# allow it to be executed from the data pipeline). Please don't change
+# this; we'll change it when we're ready to test the bot.
+enabled: false

data/examples/bot_with_simple_iterator/lib/bot_with_simple_iterator.rb

@@ -0,0 +1,112 @@
+# encoding: UTF-8
+require 'simple_openc_bot'
+require 'mechanize'
+
+class BotWithSimpleIterator < SimpleOpencBot
+
+  yields Object
+
+  # This method should return an array of Records. It must be defined.
+  def fetch_all_records(opts={})
+
+    # The following methods illustrate four common incrementer
+    # patterns.
+
+    # If a run is interrupted, it will resume where it left off --
+    # unless you pass the reset flag (`bundle exec openc_bot rake
+    # bot:run -- --reset`), or a full iteration has previously
+    # completed (in which case it will start again)
+
+    # Try running `bundle exec openc_bot rake bot:run`, using CTRL-C
+    # to interrupt, and then try resuming.
+
+    increment_over_ascii(opts)
+    increment_over_number(opts)
+    increment_over_manual(opts)
+    combine_incrementers(opts)
+  end
+
+
+  def increment_over_ascii(opts)
+    # Create the incrementer
+    ascii_incrementer =  OpencBot::AsciiIncrementer.new(
+      :ascii_incrementer,
+      opts.merge(:size => 2))
+
+    ascii_incrementer.resumable.each do |letters|
+      # This will iterate over all two-digit combinations of 0-9 and
+      # a-z.
+      puts "http://assets.opencorporates.com/test_bot_page_#{letters}.html"
+    end
+  end
+
+  def increment_over_number(opts)
+    # Create the incrementer
+    numeric_incrementer = NumericIncrementer.new(
+      :numeric_incrementer,
+      opts.merge(
+        :start_val => 0,
+        :end_val => 20))
+
+    numeric_incrementer.resumable.each do |number|
+      # This will iterate over numbers 0 - 20
+      puts "http://assets.opencorporates.com/test_bot_page_#{number}.html"
+    end
+  end
+
+  def increment_over_manual(opts)
+    # Create the incrementer
+    manual_incrementer = OpencBot::ManualIncrementer.new(
+      :manual_incrementer,
+      opts.merge(:fields => [:name]))
+
+    if !manual_incrementer.populated
+      # Populate it, if it's not been done before
+      manual_incrementer.add_row({"name" => "Bob"})
+      manual_incrementer.add_row({"name" => "Sue"})
+    end
+
+    # Mark populating as complete.. the `populated` flag is not
+    # necessary, but it's useful when debugging to skip slow
+    # population steps.
+    manual_incrementer.populated
+
+    manual_incrementer.resumable.each do |row|
+      # This will iterate over all the rows added previously.
+      puts "http://assets.opencorporates.com/test_bot_page_#{row["name"]}.html"
+    end
+  end
+
+
+  # Often you will need to use an iterator to build a list of pages to
+  # get, using another iterator.
+  def combine_incrementers(opts)
+    ascii_incrementer =  OpencBot::AsciiIncrementer.new(
+      :ascii_incrementer_2,
+      opts.merge(:size => 1))
+
+    manual_incrementer = OpencBot::ManualIncrementer.new(
+      :manual_incrementer_2,
+      opts.merge(:fields => [:url]))
+
+    ascii_incrementer.resumable.each do |letters|
+      get_urls_for_letter_combination(letters).each do |url|
+        manual_incrementer.add_row({"url" => url})
+      end
+    end
+
+    manual_incrementer.resumable.each do |row|
+      puts row["url"]
+    end
+
+  end
+
+  def get_urls_for_letter_combination(letters)
+    # This method might do something like:
+    # page = http_client.get("http://somewhere.com/?q=#{letters}")
+    # urls = page.xpath("//a/@href")
+
+    # However, for demonstration purposes, we just return:
+    ["http://foo.com/#{letters}/1", "http://foo.com/#{letters}/2"]
+  end
+end

data/examples/company_fetchers/basic.rb

@@ -0,0 +1,49 @@
+require 'openc_bot'
+require 'openc_bot/company_fetcher_bot'
+# We tend to use Nokogiri to parse HTML//XML but this is optional
+require 'nokogiri'
+require 'open-uri'
+
+module XyCompaniesFetcher
+  extend OpencBot
+  # This adds the CompanyFetcherBot functionality
+  extend OpencBot::CompanyFetcherBot
+  extend self # make these methods as Module methods, rather than instance ones
+
+
+  # The update_data module method is called when the bot is run. This is the only required method a bot needs,
+  # and the only requirement that it needs to satisy is that it should save a company as a Hash that
+  # conforms to the company-schema (https://github.com/openc/openc_bot/blob/master/schemas/company-schema.json)
+  # using the #save_entity method. This method validates the hash, and saves in the database, adding the
+  # ISO-3166-2 jurisdiction_code inferred from the name of the module (in this case xy)
+  #
+  # There are various helpers that we've found useful (see https://github.com/openc/openc_bot/tree/master/lib/openc_bot/helpers)
+  # but not of them are required. For example, if you are doing an alpha search ('AA','AB',...) there are
+  # intelligent defaults for doing such a search, and in fact you don't even need to write the #update_data
+  # method. Similarly for incremental searches (where you are iterating through a series of increasing uids).
+  # There are also helpers for normalising dates and text.
+  def update_data
+    # This code is actually for the Bermuda company register
+    #
+    # Get all the pages containing companies...
+    a_z_links = Nokogiri.HTML(open( 'https://www.roc.gov.bm/roc/rocweb.nsf/ReviewPublicRegA-Z?OpenForm')).search('a')
+    # iterate through them...
+    a_z_links.each do |link|
+      page = Nokogiri.HTML(open('https://www.roc.gov.bm' + link[:href]))
+      # find all the companies in the table...
+      page.search('//table[2]//tr').each do |tr|
+        # extract the information
+        name = tr.at_xpath('.//td[2]//a').inner_text.strip rescue nil
+        company_number = tr.at('td a').inner_text.strip rescue nil
+        incorporation_date = tr.at('.//td[3]//a').inner_text.to_date.to_s rescue nil
+        next if !name&&!company_number&&!incorporation_date
+        # save the entity hash in the local database, using #save_entity helper method, which
+        # validating it against the company schema first
+        save_entity(:name => name, :company_number => company_number, :incorporation_date => incorporation_date, :retrieved_at => Time.now.to_s)
+      end
+    end
+
+  end
+
+
+end