openc_bot 0.0.11

data/schemas/includes/total-shares.json

@@ -0,0 +1,10 @@
+{
+  "name": "TotalShares",
+  "description": "The total number of shares a company has issued",
+  "type": "object",
+  "properties": { "number": { "type": "integer" },
+                  "share_class": { "type": "string", "minLength": 1 }
+  },
+  "required": [ "number" ]
+}
+

data/schemas/licence-schema.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Licence Schema",
+  "type": "object",
+  "allOf" : [
+    // The following is a basic statement with sample_date, etc
+    { "$ref": "includes/base-statement.json" },
+    // And this overrides it to provide data-type-specific information
+    {
+      "properties": {
+        "data": {
+          "items": {
+            "allOf": [
+              { "$ref": "includes/license-data.json" }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

data/schemas/share-parcel-schema.json

@@ -0,0 +1,21 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Share Parcel Schema",
+  "type": "object",
+  "allOf" : [
+    // The following is a basic statement with sample_date, etc
+    { "$ref": "includes/base-statement.json" },
+    // And this overrides it to provide data-type-specific information
+    {
+      "properties": {
+        "data": {
+          "items": {
+            "allOf": [
+              { "$ref": "includes/share-parcel-data.json" }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

data/schemas/subsidiary-relationship-schema.json

@@ -0,0 +1,19 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Subsidiary Relationship Schema",
+  "type": "object",
+  "allOf" : [
+    { "$ref": "includes/base-statement.json" },
+    {
+      "properties": {
+        "data": {
+          "items": {
+            "allOf": [
+              { "$ref": "includes/subsidiary-relationship-data.json" }
+            ]
+          }
+        }
+      }
+    }
+  ]
+}

data/spec/dummy_classes/foo_bot.rb

@@ -0,0 +1,4 @@
+module FooBot
+  extend OpencBot
+  # extend self
+end

data/spec/lib/bot_data_validator_spec.rb

@@ -0,0 +1,69 @@
+# encoding: UTF-8
+require_relative '../spec_helper'
+require 'openc_bot'
+
+describe OpencBot::BotDataValidator do
+
+  describe '#validate' do
+    before do
+      @valid_data =
+        { :company => {:name => "CENTRAL BANK", :identifier => "rssd/546544", :jurisdiction => "IA"},
+          :data => [{ :data_type => :subsidiary_relationship,
+                      :properties => {:foo => 'bar'}
+                      },
+                    { :data_type => :subsidiary_relationship,
+                      :properties => { :foo => 'baz' }
+                    }
+                      ],
+          :source_url => "http://www.ffiec.gov/nicpubweb/nicweb/OrgHierarchySearchForm.aspx?parID_RSSD=546544&parDT_END=99991231",
+          :reporting_date => "2013-01-18 12:52:20"
+        }
+
+
+    end
+    it 'should return true if data is valid' do
+      OpencBot::BotDataValidator.validate(@valid_data).should be_true
+    end
+
+    it 'should return false if data is not a hash' do
+      OpencBot::BotDataValidator.validate(nil).should be_false
+      OpencBot::BotDataValidator.validate('foo').should be_false
+      OpencBot::BotDataValidator.validate(['foo']).should be_false
+    end
+
+    it 'should return false if company_data is blank' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:company => nil)).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:company => '  ')).should be_false
+    end
+
+    it 'should return false if company_data is missing name' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:company => {:name => nil})).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:company => {:name => ' '})).should be_false
+    end
+
+    it 'should return false if source_url is blank' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:source_url => nil)).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:source_url => '  ')).should be_false
+    end
+
+    it 'should return false if data is empty' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => nil)).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => [])).should be_false
+    end
+
+    it 'should return false if data is missing data_type' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => [{:data_type => nil,
+                                                            :properties => {:foo => 'bar'}}])).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => [{:data_type => '  ',
+                                                            :properties => {:foo => 'bar'}}])).should be_false
+    end
+
+    it 'should return false if properties is blank' do
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => [{:data_type => :subsidiary_relationship,
+                                                            :properties => {}}])).should be_false
+      OpencBot::BotDataValidator.validate(@valid_data.merge(:data => [{:data_type => :subsidiary_relationship,
+                                                            :properties => nil}])).should be_false
+    end
+
+  end
+end

data/spec/lib/company_fetcher_bot_spec.rb

@@ -0,0 +1,93 @@
+# encoding: UTF-8
+require_relative '../spec_helper'
+require 'openc_bot'
+require 'openc_bot/company_fetcher_bot'
+
+module TestCompaniesFetcher
+  extend OpencBot::CompanyFetcherBot
+end
+
+module UsXxCompaniesFetcher
+  extend OpencBot::CompanyFetcherBot
+end
+
+describe "A module that extends CompanyFetcherBot" do
+
+  before do
+    @dummy_connection = double('database_connection', :save_data => nil)
+    TestCompaniesFetcher.stub(:sqlite_magic_connection).and_return(@dummy_connection)
+  end
+
+  it "should include OpencBot methods" do
+    TestCompaniesFetcher.should respond_to(:save_run_report)
+  end
+
+  it "should include IncrementalHelper methods" do
+    TestCompaniesFetcher.should respond_to(:incremental_search)
+  end
+
+  it "should include AlphaHelper methods" do
+    TestCompaniesFetcher.should respond_to(:letters_and_numbers)
+  end
+
+  it "should set primary_key_name to :company_number" do
+    TestCompaniesFetcher.primary_key_name.should == :company_number
+  end
+
+  describe "#fetch_datum for company_number" do
+    before do
+      TestCompaniesFetcher.stub(:fetch_registry_page)
+    end
+
+    it "should #fetch_registry_page for company_numbers" do
+      TestCompaniesFetcher.should_receive(:fetch_registry_page).with('76543')
+      TestCompaniesFetcher.fetch_datum('76543')
+    end
+
+    it "should stored result of #fetch_registry_page in hash keyed to :company_page" do
+      TestCompaniesFetcher.stub(:fetch_registry_page).and_return(:registry_page_html)
+      TestCompaniesFetcher.fetch_datum('76543').should == {:company_page => :registry_page_html}
+    end
+  end
+
+  describe "#schema_name" do
+    context 'and no SCHEMA_NAME constant' do
+      it "should return 'company-schema'" do
+        TestCompaniesFetcher.schema_name.should == 'company-schema'
+      end
+    end
+
+    context 'and SCHEMA_NAME constant set' do
+      it "should return SCHEMA_NAME" do
+        stub_const("TestCompaniesFetcher::SCHEMA_NAME", 'foo-schema')
+        TestCompaniesFetcher.schema_name.should == 'foo-schema'
+      end
+    end
+  end
+
+  describe "#inferred_jurisdiction_code" do
+    it "should return jurisdiction_code inferred from class_name" do
+      UsXxCompaniesFetcher.inferred_jurisdiction_code.should == 'us_xx'
+    end
+
+    it "should return nil if jurisdiction_code not correct format" do
+      TestCompaniesFetcher.inferred_jurisdiction_code.should be_nil
+    end
+  end
+
+  describe "#save_entity" do
+    before do
+      TestCompaniesFetcher.stub(:inferred_jurisdiction_code).and_return('ab_cd')
+    end
+
+    it "should save_entity with inferred_jurisdiction_code" do
+      TestCompaniesFetcher.should_receive(:prepare_and_save_data).with(:name => 'Foo Corp', :company_number => '12345', :jurisdiction_code => 'ab_cd')
+      TestCompaniesFetcher.save_entity(:name => 'Foo Corp', :company_number => '12345')
+    end
+
+    it "should save_entity with given jurisdiction_code" do
+      TestCompaniesFetcher.should_receive(:prepare_and_save_data).with(:name => 'Foo Corp', :company_number => '12345', :jurisdiction_code => 'xx')
+      TestCompaniesFetcher.save_entity(:name => 'Foo Corp', :company_number => '12345', :jurisdiction_code => 'xx')
+    end
+  end
+end

data/spec/lib/exceptions_spec.rb

@@ -0,0 +1,25 @@
+# encoding: UTF-8
+require_relative '../spec_helper'
+require 'openc_bot'
+
+describe 'OpencBot exceptions' do
+
+  describe OpencBot::OpencBotError do
+
+    it 'should have StandardError as superclass' do
+      OpencBot::OpencBotError.superclass.should == StandardError
+    end
+  end
+
+  describe OpencBot::RecordInvalid do
+
+    it 'should have OpencBotError as superclass' do
+      OpencBot::RecordInvalid.superclass.should == OpencBot::OpencBotError
+    end
+
+    it "should have set validation_errors accessor on instantiation" do
+      error = OpencBot::RecordInvalid.new(:some_validation_error)
+      error.validation_errors.should == :some_validation_error
+    end
+  end
+end

data/spec/lib/helpers/alpha_search_spec.rb

@@ -0,0 +1,173 @@
+# encoding: UTF-8
+require_relative '../../spec_helper'
+require 'openc_bot'
+require 'openc_bot/helpers/alpha_search'
+
+module ModuleThatIncludesAlphaSearch
+  extend OpencBot
+  extend OpencBot::Helpers::AlphaSearch
+end
+
+describe 'a module that includes AlphaSearch' do
+
+  before do
+    ModuleThatIncludesAlphaSearch.stub(:sqlite_magic_connection).
+                                  and_return(test_database_connection)
+  end
+
+  after do
+    remove_test_database
+  end
+
+  it "should include register_methods" do
+    ModuleThatIncludesAlphaSearch.should respond_to(:registry_url)
+  end
+
+  describe "#letters_and_numbers" do
+    it "should return an array of all letters and numbers" do
+      ModuleThatIncludesAlphaSearch.letters_and_numbers.should == ('A'..'Z').to_a + ('0'..'9').to_a
+    end
+  end
+
+  describe "numbers_of_chars_in_search" do
+    context 'and no NUMBER_OF_CHARS_IN_SEARCH constant' do
+      it "should return 1" do
+        ModuleThatIncludesAlphaSearch.numbers_of_chars_in_search.should == 1
+      end
+    end
+
+    context 'and has NUMBER_OF_CHARS_IN_SEARCH constant' do
+      it "should return NUMBER_OF_CHARS_IN_SEARCH" do
+        stub_const("ModuleThatIncludesAlphaSearch::NUMBER_OF_CHARS_IN_SEARCH", 4)
+        ModuleThatIncludesAlphaSearch.numbers_of_chars_in_search.should == ModuleThatIncludesAlphaSearch::NUMBER_OF_CHARS_IN_SEARCH
+      end
+    end
+  end
+
+  describe "#alpha_terms" do
+    before do
+      @letters_and_numbers = ['A','B','1','2']
+      ModuleThatIncludesAlphaSearch.stub(:letters_and_numbers).and_return(@letters_and_numbers)
+    end
+
+    it "should return array of letters_and_numbers based on numbers_of_chars_in_search" do
+      ModuleThatIncludesAlphaSearch.alpha_terms.should == @letters_and_numbers
+      ModuleThatIncludesAlphaSearch.should_receive(:numbers_of_chars_in_search).and_return(2)
+      ModuleThatIncludesAlphaSearch.alpha_terms.
+        should == @letters_and_numbers.repeated_permutation(2).collect(&:join)
+    end
+
+    context "and starting character given" do
+      it "should start array from given character" do
+        ModuleThatIncludesAlphaSearch.alpha_terms('B').should == @letters_and_numbers[1..-1]
+        ModuleThatIncludesAlphaSearch.stub(:numbers_of_chars_in_search).and_return(2)
+        ModuleThatIncludesAlphaSearch.alpha_terms('1B').
+          should == ["1B", "11", "12", "2A", "2B", "21", "22"]
+      end
+      it "should start array from beginning if no such character" do
+        ModuleThatIncludesAlphaSearch.alpha_terms('X').should == @letters_and_numbers
+        ModuleThatIncludesAlphaSearch.stub(:numbers_of_chars_in_search).and_return(2)
+        ModuleThatIncludesAlphaSearch.alpha_terms('C').
+          should == @letters_and_numbers.repeated_permutation(2).collect(&:join)
+      end
+    end
+  end
+
+  describe "each_search_term" do
+    before do
+      ModuleThatIncludesAlphaSearch.should_receive(:alpha_terms).with('B').and_return(['C','D'])
+    end
+
+    it "should iterate through alpha_terms and yield them" do
+      yielded_data = []
+      ModuleThatIncludesAlphaSearch.each_search_term('B') { |t| yielded_data << "#{t}#{t}" }
+      yielded_data.should == ['CC','DD']
+    end
+
+    context "and no block given" do
+      it "should return alpha_terms" do
+        ModuleThatIncludesAlphaSearch.each_search_term('B').should == ['C','D']
+      end
+    end
+  end
+
+  describe '#fetch_data_via_alpha_search' do
+    before do
+      @alpha_terms = ['A1','B2','XX','YY']
+      ModuleThatIncludesAlphaSearch.stub(:create_new_company)
+      ModuleThatIncludesAlphaSearch.stub(:save_entity)
+      ModuleThatIncludesAlphaSearch.stub(:alpha_terms).and_return(@alpha_terms)
+      ModuleThatIncludesAlphaSearch.stub(:search_for_entities_for_term).and_yield(nil)
+    end
+
+    it "should search_for_entities_for_term for each term in alpha_terms" do
+      @alpha_terms.each do |term|
+        ModuleThatIncludesAlphaSearch.should_receive(:search_for_entities_for_term).with(term, anything).and_yield(nil)
+      end
+      ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search
+    end
+
+    it "should process entity data yielded by search_for_entities_for_term" do
+      @alpha_terms.each do |term|
+        ModuleThatIncludesAlphaSearch.stub(:search_for_entities_for_term).with(term, anything).and_yield(:datum_1).and_yield(:datum_2)
+      end
+      ModuleThatIncludesAlphaSearch.should_receive(:save_entity).with(:datum_1)
+      ModuleThatIncludesAlphaSearch.should_receive(:save_entity).with(:datum_2)
+
+      ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search
+    end
+
+    it "should start from saved starting term" do
+      ModuleThatIncludesAlphaSearch.save_var('starting_term', 'B2')
+      ModuleThatIncludesAlphaSearch.should_receive(:alpha_terms).with('B2').and_return(@alpha_terms)
+
+      ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search
+    end
+
+    it "should pass options to search_for_entities_for_term" do
+      ModuleThatIncludesAlphaSearch.should_receive(:search_for_entities_for_term).with(anything, :foo => 'bar').and_yield(nil)
+
+      ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search(:foo => 'bar')
+    end
+
+    context "and explicit starting_term passed in as option" do
+      it "should start from given starting term" do
+        ModuleThatIncludesAlphaSearch.save_var('starting_term', 'B2')
+        ModuleThatIncludesAlphaSearch.should_receive(:alpha_terms).with('XX').and_return(@alpha_terms)
+
+        ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search(:starting_term => 'XX')
+      end
+    end
+
+    context "and search gets to end of alpha_terms" do
+      it "should flush starting_term record" do
+        ModuleThatIncludesAlphaSearch.save_var('starting_term', 'B2')
+
+        ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search
+        ModuleThatIncludesAlphaSearch.get_var('starting_term').should be_nil
+      end
+    end
+
+    context "and search finishes before getting to end of alpha_terms" do
+      it "should store term where it was working on where there was problem" do
+        ModuleThatIncludesAlphaSearch.should_receive(:search_for_entities_for_term).
+                                      with(@alpha_terms.first, anything).and_yield(nil)
+
+        ModuleThatIncludesAlphaSearch.should_receive(:search_for_entities_for_term).
+                                      with(@alpha_terms[1], anything).and_raise('Something has gone wrong')
+
+        lambda { ModuleThatIncludesAlphaSearch.fetch_data_via_alpha_search }.should raise_error
+        ModuleThatIncludesAlphaSearch.get_var('starting_term').should == @alpha_terms[1]
+      end
+    end
+  end
+
+  describe "#search_for_entities_for_term" do
+
+    it "should raise exception" do
+      lambda { ModuleThatIncludesAlphaSearch.search_for_entities_for_term('foo') }.should raise_error(/method has not been implemented/)
+    end
+
+  end
+
+end