openc_bot 0.0.11

data/spec/simple_openc_bot_spec.rb

@@ -0,0 +1,302 @@
+require 'rspec'
+require 'simple_openc_bot'
+
+class InvalidLicenceRecord < SimpleOpencBot::BaseLicenceRecord
+end
+
+class LicenceRecord < SimpleOpencBot::BaseLicenceRecord
+  JURISDICTION = "uk"
+  store_fields :name, :type, :sample_date
+  unique_fields :name
+  schema :licence
+
+  URL = "http://foo.com"
+
+  def jurisdiction_classification
+    type
+  end
+
+  def last_updated_at
+    sample_date
+  end
+
+  def to_pipeline
+    {
+      sample_date: "",
+      company: {
+        name: name,
+        jurisdiction: JURISDICTION,
+      },
+      source_url: URL,
+      data: [{
+        data_type: :licence,
+        properties: {
+          category: 'Financial',
+          jurisdiction_classification: [jurisdiction_classification],
+          jurisdiction_code: 'gb'
+        }
+      }]
+    }
+  end
+
+end
+
+class TestLicenceBot < SimpleOpencBot
+  yields LicenceRecord
+
+  def initialize(data={})
+    @data = data
+  end
+
+  def fetch_all_records(opts={})
+    @data.each do |datum|
+      yield LicenceRecord.new(datum)
+    end
+  end
+end
+
+describe InvalidLicenceRecord do
+  describe '#last_updated_at not defined' do
+    it "should raise an error mentioning to_pipeline" do
+      lambda do
+        InvalidLicenceRecord.new
+      end.should raise_error(/to_pipeline/)
+    end
+
+    it "should raise an error mentioning last_updated_at" do
+      lambda do
+        InvalidLicenceRecord.new
+      end.should raise_error(/last_updated_at/)
+    end
+
+    it "should raise an error mentioning schema" do
+      lambda do
+        InvalidLicenceRecord.new
+      end.should raise_error(/schema/)
+    end
+    it "should raise an error mentioning unique_fields" do
+      lambda do
+        InvalidLicenceRecord.new
+      end.should raise_error(/unique_fields/)
+    end
+    it "should raise an error mentioning store_fields" do
+      lambda do
+        InvalidLicenceRecord.new
+      end.should raise_error(/store_fields/)
+    end
+
+  end
+end
+
+describe LicenceRecord do
+  before do
+    @initial_values_hash = {:name => 'Foo',
+                            :type => 'Bar'}
+
+    @record = LicenceRecord.new(
+      @initial_values_hash
+    )
+  end
+
+  describe 'fields' do
+    it 'has _type attribute' do
+      @record._type.should == 'LicenceRecord'
+    end
+
+    it 'can get attribute' do
+      @record.name.should == 'Foo'
+    end
+
+    it 'can set attribute' do
+      @record.type = 'Baz'
+      @record.type.should == 'Baz'
+    end
+  end
+
+  describe "#to_hash" do
+    it "should include all the specified fields as a hash" do
+      @record.to_hash.should include(@initial_values_hash)
+    end
+  end
+end
+
+describe SimpleOpencBot do
+  before do
+  end
+
+  after do
+    table_names = %w(ocdata)
+    table_names.each do |table_name|
+      # flush table, but don't worry if it doesn't exist
+      begin
+        conn = TestLicenceBot.new.sqlite_magic_connection
+        conn.database.execute("DROP TABLE #{table_name}") if conn
+      rescue SQLite3::SQLException => e
+        raise unless e.message.match(/no such table/)
+      end
+    end
+  end
+
+
+  describe ".update_data" do
+    before do
+      @properties = [
+        {:name => 'Company 1', :type => 'Bank'},
+        {:name => 'Company 2', :type => 'Insurer'}
+      ]
+      @bot = TestLicenceBot.new(@properties)
+    end
+
+    it "should make sqlite database in same directory as bot" do
+      root = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+      path = File.join(root, "db", "testlicencebot.db")
+      SqliteMagic::Connection.should_receive(:new).with(path)
+    end
+
+    it "should call insert_or_update with correct unique fields" do
+      @bot.stub(:check_unique_index)
+      @bot.sqlite_magic_connection.stub(:add_columns)
+      @bot.should_receive(:insert_or_update).with(
+        LicenceRecord._unique_fields, anything).twice()
+      @bot.update_data
+    end
+
+    it "should update rather than insert rows the second time" do
+      @bot.update_data
+      @bot.update_data
+      @bot.count_stored_records.should == 2
+    end
+
+    it "should raise an error if the unique index has changed" do
+      @bot.update_data
+      LicenceRecord.stub(:unique_fields).and_return([:type])
+      lambda do
+        @bot.update_data
+      end.should raise_error
+    end
+
+    it "should call insert_or_update with all records in a hash" do
+      @bot.stub(:check_unique_index)
+      @bot.sqlite_magic_connection.stub(:add_columns)
+      @bot.should_receive(:insert_or_update).with(
+        anything,
+        hash_including(@properties.first))
+      @bot.should_receive(:insert_or_update).with(
+        anything,
+        hash_including(@properties.last))
+      @bot.update_data
+    end
+  end
+
+  describe "spotcheck_data" do
+    before do
+      @properties = [
+        {:name => 'Company 1', :type => 'Bank'},
+        {:name => 'Company 2', :type => 'Insurer'}
+      ]
+      @bot = TestLicenceBot.new(@properties)
+      @bot.update_data
+    end
+
+    it "should return all records the first time" do
+      @bot.spotcheck_data.count.should == 2
+    end
+
+    it "should return records converted to pipeline format" do
+      @bot.spotcheck_data.any? {|c| c[:company][:name] == @properties[0][:name]}.should be_true
+    end
+  end
+
+  describe "export_data" do
+    before do
+      @properties = [
+        {:name => 'Company 1', :type => 'Bank'},
+        {:name => 'Company 2', :type => 'Insurer'}
+      ]
+      @bot = TestLicenceBot.new(@properties)
+      @bot.update_data
+    end
+
+    it "should return all records the first time" do
+      [*@bot.export_data].count.should == 2
+    end
+
+    it "should return records converted to pipeline format" do
+      [*@bot.export_data][0][:company][:name].should == @properties[0][:name]
+    end
+
+    it "should set the last_exported_date on exported records" do
+      [*@bot.export_data]
+      @bot.all_stored_records.
+        map(&:_last_exported_at).compact.should_not be_nil
+    end
+
+    it "should not export data which has been exported before and for which the sample data has not changed" do
+      [*@bot.export_data]
+      [*@bot.export_data].count.should == 0
+    end
+
+    it "should export data which has been exported before and for which the last_updated_at has changed" do
+      results = [*@bot.export_data]
+      sleep 0.5 # give sqlite a chance
+      LicenceRecord.any_instance.stub(:last_updated_at).and_return(Time.now.iso8601(2))
+      bot = TestLicenceBot.new([
+        {:name => 'Company 2', :type => 'Insurer'}])
+      bot.update_data
+      [*@bot.export_data].count.should == 1
+    end
+  end
+
+  describe "stored data" do
+    before do
+      @properties = [
+        {:name => 'Company 1', :type => 'Bank'},
+        {:name => 'Company 2', :type => 'Insurer'}
+      ]
+      @bot = TestLicenceBot.new(@properties)
+      @bot.update_data
+    end
+
+    describe "#all_stored_records" do
+      it "should return an array of LicenceRecords" do
+        @bot.all_stored_records.count.should == 2
+        @bot.all_stored_records.map(&:class).uniq.should == [LicenceRecord]
+      end
+    end
+
+    describe "#export_data" do
+      it "should return an array of hashes" do
+        result = @bot.export_data
+        result.should be_a Enumerable
+        result.count.should == @properties.count
+      end
+
+      it "should not re-export data twice" do
+        @bot.export_data.count.should_not == 0
+        @bot.export_data.count.should == 0
+      end
+    end
+
+    describe "#validate_data" do
+      before :each do
+        SimpleOpencBot.any_instance.stub(:puts)
+      end
+
+      context "valid data" do
+        it "should return empty array" do
+          result = @bot.validate_data
+          result.should be_empty
+        end
+      end
+
+      context "invalid data" do
+        it "should return an array of hashes with errors" do
+          LicenceRecord.any_instance.stub(:to_pipeline).and_return({})
+          result = @bot.validate_data
+          result.count.should == 2
+          result[0][:errors].should_not be_empty
+        end
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'rspec/autorun'
+require 'debugger'
+
+RSpec.configure do |config|
+end
+
+def remove_test_database
+  File.delete(test_database_location) if File.exist?(test_database_location)
+end
+
+def test_database_location
+  db_dir = File.join(File.dirname(__FILE__),'db')
+  Dir.mkdir(db_dir) unless Dir.exist?(db_dir)
+  File.join(db_dir,'test_db.db')
+end
+
+def test_database_connection
+  @test_database_connection ||=SqliteMagic::Connection.new(test_database_location)
+end

metadata

@@ -0,0 +1,300 @@
+--- !ruby/object:Gem::Specification
+name: openc_bot
+version: !ruby/object:Gem::Version
+  version: 0.0.11
+platform: ruby
+authors:
+- Chris Taggart
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-06-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.17
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.2.17
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json-schema
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: backports
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: scraperwiki
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.0.2
+- !ruby/object:Gem::Dependency
+  name: perftools.rb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is to make the writing and running of bots for OpenCorporates
+  quick and easy
+email:
+- info@opencorporates.com
+executables:
+- openc_bot
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".travis.yml"
+- CHANGELOG.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/openc_bot
+- create_bot.sh
+- create_company_bot.sh
+- create_simple_licence_bot.sh
+- db/.gitkeep
+- doc/README-complex.md
+- doc/SCHEMAS.md
+- examples/basic/.gitignore
+- examples/basic/Gemfile
+- examples/basic/config.yml
+- examples/basic/lib/basic.rb
+- examples/basic_with_proxy/Gemfile
+- examples/basic_with_proxy/config.yml
+- examples/basic_with_proxy/lib/basic_with_proxy.rb
+- examples/bot_with_simple_iterator/Gemfile
+- examples/bot_with_simple_iterator/config.yml
+- examples/bot_with_simple_iterator/lib/bot_with_simple_iterator.rb
+- examples/company_fetchers/basic.rb
+- lib/monkey_patches/mechanize.rb
+- lib/openc_bot.rb
+- lib/openc_bot/bot_data_validator.rb
+- lib/openc_bot/company_fetcher_bot.rb
+- lib/openc_bot/exceptions.rb
+- lib/openc_bot/helpers/_csv.rb
+- lib/openc_bot/helpers/alpha_search.rb
+- lib/openc_bot/helpers/dates.rb
+- lib/openc_bot/helpers/html.rb
+- lib/openc_bot/helpers/incremental_search.rb
+- lib/openc_bot/helpers/register_methods.rb
+- lib/openc_bot/helpers/text.rb
+- lib/openc_bot/incrementers.rb
+- lib/openc_bot/incrementers/base.rb
+- lib/openc_bot/incrementers/common.rb
+- lib/openc_bot/tasks.rb
+- lib/openc_bot/templates/README.md
+- lib/openc_bot/templates/bin/export_data
+- lib/openc_bot/templates/bin/fetch_data
+- lib/openc_bot/templates/bin/verify_data
+- lib/openc_bot/templates/config.yml
+- lib/openc_bot/templates/lib/bot.rb
+- lib/openc_bot/templates/lib/company_fetcher_bot.rb
+- lib/openc_bot/templates/lib/simple_bot.rb
+- lib/openc_bot/templates/spec/bot_spec.rb
+- lib/openc_bot/templates/spec/simple_bot_spec.rb
+- lib/openc_bot/templates/spec/spec_helper.rb
+- lib/openc_bot/version.rb
+- lib/simple_openc_bot.rb
+- openc_bot.gemspec
+- schemas/company-schema.json
+- schemas/includes/address.json
+- schemas/includes/base-statement.json
+- schemas/includes/company.json
+- schemas/includes/filing.json
+- schemas/includes/license-data.json
+- schemas/includes/officer.json
+- schemas/includes/previous_name.json
+- schemas/includes/share-parcel-data.json
+- schemas/includes/share-parcel.json
+- schemas/includes/subsidiary-relationship-data.json
+- schemas/includes/total-shares.json
+- schemas/licence-schema.json
+- schemas/share-parcel-schema.json
+- schemas/subsidiary-relationship-schema.json
+- spec/dummy_classes/foo_bot.rb
+- spec/lib/bot_data_validator_spec.rb
+- spec/lib/company_fetcher_bot_spec.rb
+- spec/lib/exceptions_spec.rb
+- spec/lib/helpers/alpha_search_spec.rb
+- spec/lib/helpers/dates_spec.rb
+- spec/lib/helpers/incremental_search_spec.rb
+- spec/lib/helpers/register_methods_spec.rb
+- spec/lib/helpers/text_spec.rb
+- spec/lib/openc_bot/db/.gitkeep
+- spec/lib/openc_bot/incrementers/common_spec.rb
+- spec/lib/openc_bot_spec.rb
+- spec/schemas/company-schema_spec.rb
+- spec/simple_openc_bot_spec.rb
+- spec/spec_helper.rb
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+- lib/openc_bot/helpers
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Helper gem for writing external bots for OpenCorporates
+test_files:
+- spec/dummy_classes/foo_bot.rb
+- spec/lib/bot_data_validator_spec.rb
+- spec/lib/company_fetcher_bot_spec.rb
+- spec/lib/exceptions_spec.rb
+- spec/lib/helpers/alpha_search_spec.rb
+- spec/lib/helpers/dates_spec.rb
+- spec/lib/helpers/incremental_search_spec.rb
+- spec/lib/helpers/register_methods_spec.rb
+- spec/lib/helpers/text_spec.rb
+- spec/lib/openc_bot/db/.gitkeep
+- spec/lib/openc_bot/incrementers/common_spec.rb
+- spec/lib/openc_bot_spec.rb
+- spec/schemas/company-schema_spec.rb
+- spec/simple_openc_bot_spec.rb
+- spec/spec_helper.rb