openc_bot 0.0.18 → 0.0.26

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZGQyZmEzNWYyZmNiMWVmN2JkYmQwMzM3ZmU3YTQxMTRmZjBiNGU4MQ==
+  data.tar.gz: !binary |-
+    NWQ5ZDdmOTJjYzM1MDQxYzBhODZjMjNjN2YwY2U3MDViYmE1MzcxNw==
+SHA512:
+  metadata.gz: !binary |-
+    YzgwMjM1ZDFmN2UzMjUwZGVlNzVkYjNhODUxMDgzNjVjOTI2MDdmZmU3MTcx
+    YzUyOWQ1YWE3N2IxMmE4YjI4ZmI1MzA3OGIyOGVlNTBmOWQwZTllOWFlYWIx
+    MDE2ODgzYzk0YTQ3ZDBiMzA4OTkxOTQwNWZmN2ZkNmVhZjQyMzA=
+  data.tar.gz: !binary |-
+    ZTBjNTAxZDdkMzA3ZTJmNmYzOTU2ZmVhNGUyYmU2MjgxMDgxNTQ2NzE1ODlk
+    YTQ2MGU4NDU0NzkyZTBmM2I2NzlmNjM0OWQ4ZGU3YzJhZGY1YTNlNTNmMDdh
+    YTE5YTdhN2I1ODg3MDlkNGQzZjNiNmZiODliNGIyNmFlZWYxOGM=

data/lib/openc_bot/exceptions.rb

@@ -11,7 +11,10 @@ module OpencBot
 
     def initialize(validation_errors)
       @validation_errors = validation_errors
+      message = "Validation failed:" + validation_errors.collect{ |e| e[:message] }.join("\n")
+      super(message)
     end
+
   end
 
 end

data/lib/openc_bot/helpers/incremental_search.rb

@@ -21,7 +21,7 @@ module OpencBot
       def highest_entry_uids(force_get = false)
         bad_results = []
         results = get_var('highest_entry_uids')
-        if results.nil? || results.empty? || (results.is_a?(Array) && results.any?{ |r| r.nil? || r.empty? })
+        if force_get || results.nil? || results.empty? || (results.is_a?(Array) && results.any?{ |r| r.nil? || r.empty? })
           results = entity_uid_prefixes.collect do |prefix|
             hcn = highest_entry_uid_result(:prefix => prefix)
             bad_results << prefix if (hcn.nil? || hcn.empty?)

data/lib/openc_bot/helpers/register_methods.rb

@@ -6,6 +6,7 @@ require 'active_support/core_ext'
 module OpencBot
   module Helpers
     module RegisterMethods
+      MAX_BUSY_RETRIES = 3
 
       def use_alpha_search
         self.const_defined?('USE_ALPHA_SEARCH') && self.const_get('USE_ALPHA_SEARCH')
@@ -39,7 +40,21 @@ module OpencBot
 
       def prepare_and_save_data(all_data,options={})
         data_to_be_saved = prepare_for_saving(all_data)
-        insert_or_update([primary_key_name], data_to_be_saved)
+        fail_count, retry_interval = 0, 5
+        begin
+          insert_or_update([primary_key_name], data_to_be_saved)
+        rescue SQLite3::BusyException => e
+          fail_count += 1
+          if fail_count <= MAX_BUSY_RETRIES
+            puts "#{e.inspect} raised #{fail_count} times saving:\n#{all_data}\n\nNow retrying in #{retry_interval} seconds" if verbose?
+            sleep retry_interval
+            retry_interval = retry_interval * 2
+            retry
+          else
+            raise e
+          end
+        end
+
       end
 
       def primary_key_name
@@ -117,7 +132,8 @@ module OpencBot
       def update_datum(uid, output_as_json=false,replace_existing_data=false)
         return unless raw_data = fetch_datum(uid)
         default_options = {primary_key_name => uid, :retrieved_at => Time.now}
-        processed_data = default_options.merge(process_datum(raw_data))
+        return unless base_processed_data = process_datum(raw_data)
+        processed_data = default_options.merge(base_processed_data)
         # prepare the data for saving (converting Arrays, Hashes to json) and
         # save the original data too, as we may not extracting everything from it yet
         save_entity(processed_data.merge(:data => raw_data))
@@ -130,8 +146,9 @@ module OpencBot
         if output_as_json
           output_json_error_message(e)
         else
-          puts e.inspect if verbose?
-          raise e
+          rich_message = "#{e.message} updating entry with uid: #{uid}"
+          puts rich_message if verbose?
+          raise $!, rich_message, $!.backtrace
         end
       end
 
@@ -143,11 +160,13 @@ module OpencBot
       end
 
       def validate_datum(record)
-        schema = File.expand_path("../../../../schemas/schemas/#{schema_name}.json", __FILE__)
-        errors = JSON::Validator.fully_validate(
-          schema,
-          record.to_json,
-          {:errors_as_objects => true})
+        Dir.chdir('schemas/schemas') do
+          JSON::Validator.fully_validate(
+            "#{schema_name}.json",
+            record.to_json,
+            {:errors_as_objects => true}
+          )
+        end
       end
 
       def post_process(row_hash, skip_nulls=false)

data/lib/openc_bot/version.rb

@@ -1,3 +1,3 @@
 module OpencBot
-  VERSION = "0.0.18"
+  VERSION = "0.0.26"
 end

data/publish.sh

@@ -4,6 +4,8 @@ UNPUSHED=$(git log origin/master..HEAD)
 if [ "x$UNPUSHED" == "x" ]; then
     gem build openc_bot.gemspec
     gem push $(ls *gem|tail -1)
+else
+ echo "Unpushed commits!"
 fi
 
 function clean {

data/schemas/Gemfile

@@ -4,6 +4,6 @@ source "https://rubygems.org"
 group :test do
   gem "rspec"
   gem 'json-schema'
-  gem 'debugger'
+  # gem 'debugger'
   gem 'rake'
 end

data/schemas/schemas/company-schema.json

@@ -6,14 +6,17 @@
   "properties": {
     "company_number": {
       "type": "string",
+      "description": "unique identifier given by the corporate register with which it is incorporated",
       "minLength": 1
     },
     "name": {
       "type": "string",
+      "description": "Legal name of the company",
       "minLength": 1
     },
     "jurisdiction_code": {
       "type": "string",
+      "description": "Code representing the jurisdiction/company register which is the canonical record of the company’s existence. Uses underscored ISO 3166-2 to represent it, e.g. es for Spain, us_de for Delaware",
       "minLength": 2,
       "maxLength": 5
     },
@@ -35,9 +38,27 @@
     "registry_url": {
       "type": "string"
     },
+    "website": {
+      "type": "string",
+      "minLength": 5
+    },
+    "telephone_number": {
+      "type": "string",
+      "minLength": 4
+    },
+    "fax_number": {
+      "type": "string",
+      "minLength": 4
+    },
     "registered_address": {
       "$ref": "includes/address.json"
     },
+    "headquarters_address": {
+      "$ref": "includes/address.json"
+    },
+    "mailing_address": {
+      "$ref": "includes/address.json"
+    },
     "officers": {
       "type": "array",
       "items": {
@@ -112,9 +133,47 @@
           "minLength": 1
         },
         "number_of_employees": {
-          "type": "number",
-          "description": "The number of employees",
-          "minimum": 0
+          "anyOf": [
+            {"type":"string","minLength": 1},
+            {"type":"number","minimum": 0}
+          ],
+          "description": "The number of employees"
+        },
+        "merged_into": {
+          "type": "object",
+          "description": "Information on the merger of this company into a 'survivor' company",
+          "properties": {
+            "surviving_company": {
+              "type": ["object"],
+              "description": "the surviving_company that this has been merged into",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "description": "The legal name of the surviving company",
+                  "minLength": 1
+                },
+                "company_number": {
+                  "type": "string",
+                  "description": "The company_number of the surviving company, if known",
+                  "minLength": 1
+                },
+                "jurisdiction": {
+                  "type": "string",
+                  "description": "The jurisdiction of incorporation of the surviving company",
+                  "minLength": 2
+                }
+              },
+              "anyOf": [{"required": ["name"]},
+                        {"required": ["company_number"]}]
+            },
+            "effective_date": {
+              "type": "string",
+              "format": "date",
+              "description": "The date of the merger"
+            }
+          },
+          "additionalProperties": false,
+          "required": ["surviving_company"]
         }
       }
     }

data/schemas/schemas/includes/address.json

@@ -9,7 +9,7 @@
       "description": "An address object",
       "type": "object",
       "properties": { "street_address": { "type": "string" },
-                      "locality": { "type": "date" },
+                      "locality": {"type":"string"},
                       "region":   { "type": "string" },
                       "postal_code":   { "type": "string" },
                       "country":   { "type": "string" }

data/schemas/schemas/includes/alternative_name.json

@@ -3,8 +3,8 @@
   "description": "An alternative name of a company, e.g. abbreviation, trading (including dba or doing business as), legal. As well as trading names etc, it can be used for storing alternative language representations of the legal name, in which case the language should be represented as two-letter ISO-639 code",
   "type": "object",
   "properties": { "company_name": { "type": "string", "minLength": 1 },
-                  "start_date": { "type": "date" },
-                  "end_date": { "type": "date" },
+                  "start_date": {"type":"string","format": "date"},
+                  "end_date": {"type":"string","format": "date"},
                   "language": { "type": "string", "minLength": 2, "maxLength": 2 },
                   "type": { "type": "string", "enum": ["trading", "abbreviation", "legal"] }
   },

data/schemas/schemas/includes/filing.json

@@ -3,7 +3,7 @@
   "description": "A statutory filing",
   "type": "object",
   "properties": { "title": { "type": "string" },
-                  "date": { "type": "date" },
+                  "date": {"type":"string","format": "date"},
                   "description":   { "type": "string" },
                   "uid":   { "type": "string" },
                   "url":   { "type": "string" },

data/schemas/schemas/includes/industry_code.json

@@ -4,9 +4,12 @@
   "type": "object",
   "properties": { "name": { "type": "string" },
                   "code": { "type": "string" },
-                  "code_scheme_id": { "type": "string", "enum": ["eu_nace_2", "uk_sic_2003", "uk_sic_2007", "us_naics_2002", "us_naics_2007", "be_nace_2008"] },
-                  "start_date":  { "type": "date" },
-                  "end_date":  { "type": "date" }
+                  "code_scheme_id": {
+                    "type": "string",
+                    "description": "An identifier representing industry code scheme. At the moment these are eu_nace_2, uk_sic_2003, uk_sic_2007, us_naics_2002, us_naics_2007, be_nace_2008, dk_db_2007, nz_bic_2006, no_sic_2007, anz_sic_2006, nz_bic_2006, in_nic_2004_mca, ca_qc_cae, lu_nace_2. For other code schemes, or details of these, contact info@opencorporates.com"
+                   },
+                  "start_date":  {"type":"string","format": "date"},
+                  "end_date":  {"type":"string","format": "date"}
   },
   "required": [ "code", "code_scheme_id" ]
 }

data/schemas/schemas/includes/officer.json

@@ -3,11 +3,47 @@
   "description": "An officer (director, senior executive) of a company",
   "type": "object",
   "properties": { "name": { "type": "string", "minLength": 1 },
-                  "start_date": { "type": "date" },
-                  "end_date":   { "type": "date" },
+                  "start_date": {
+                    "anyOf": [
+                      {"type":"string","format": "date"},
+                      {"type":"null"}
+                    ]
+                  },
+                  "end_date":   {
+                    "anyOf": [
+                      {"type":"string","format": "date"},
+                      {"type":"null"}
+                    ]
+                  },
                   "position":   { "type": "string" },
-                  "other_attributes": { "type": "object" },
-                  "uid":   { "type": "string" }
+                  "other_attributes": {
+                    "type": "object",
+                    "properties": {
+                      "date_of_birth": {
+                        "type": "string",
+                        "format": "date"
+                      },
+                      "nationality": {
+                        "type": "string"
+                      },
+                      "person_uid": {
+                        "type": "string",
+                        "description": "a unique identifier given to the individual (as opposed to the officership)"
+                      },
+                      "address": {
+                        "type": "string"
+                      },
+                      "type": {
+                        "type": "string",
+                        "enum": ["Person", "Company"],
+                        "description": "The type of entity that is the officer (either 'Person' or 'Company')"
+                      }
+                    }
+                  },
+                  "uid":   {
+                    "type": "string",
+                    "description": "a unique identifier given to the officership"
+                   }
   },
   "required": [ "name" ]
 }

data/schemas/schemas/includes/previous_name.json

@@ -3,8 +3,11 @@
   "description": "A previous name of a company",
   "type": "object",
   "properties": { "company_name": { "type": "string", "minLength": 1 },
-                  "con_date": { "type": "date" },
-                  "start_date": { "type": "date" }
+                  "con_date": {
+                    "type": "date",
+                    "description": "The end (conversion) date of the name"
+                   },
+                  "start_date": {"type":"string","format": "date"}
   },
   "required": [ "company_name" ]
 }

data/schemas/schemas/includes/share-parcel.json

@@ -13,6 +13,9 @@
         "maximum": 100,
         "minimum": 0
       },
+      "start_date": {"type":"string","format": "date"},
+      "end_date": {"type":"string","format": "date"},
+      "sample_date": {"type":"string","format": "date"},
       "shareholders": {
         "description": "Legal persons who own this share parcel",
         "type": "array",

data/schemas/schemas/simple-licence-schema.json

@@ -5,11 +5,13 @@
   "properties": {
     "source_url": {
       "description": "Place where this fact can be verified",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "sample_date": {
       "description": "Date on which this fact was known to be true",
-      "type": "string"
+      "type": "date",
+      "minLength": 1
     },
     "confidence": {
       "description": "Confidence in accuracy of data",
@@ -18,11 +20,13 @@
     },
     "company_name": {
       "description": "Name of the company holding the licence",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "company_jurisdiction": {
       "description": "Jurisdiction where company is based",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "licence_number": {
       "description": "Licence number or code",
@@ -38,7 +42,8 @@
     },
     "licence_jurisdiction": {
       "description": "Jurisdiction for which licence is issued",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "status": {
       "description": "Status of the licence",

data/schemas/schemas/simple-subsidiary-schema.json

@@ -5,11 +5,13 @@
   "properties": {
     "source_url": {
       "description": "Place where this fact can be verified",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "sample_date": {
       "description": "Date on which this fact was known to be true",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "start_date": {
       "description": "Earliest known date this was known to be a subsidiary",
@@ -26,11 +28,13 @@
     },
     "parent_name": {
       "description": "Name of the controlling company",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "parent_jurisdiction": {
       "description": "Jurisdiction where controlling company is based",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "parent_identifier": {
       "description": "Any unique ID for the controlling company",
@@ -47,11 +51,13 @@
     },
     "subsidiary_name": {
       "description": "The name of the subsidiary",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "subsidiary_jurisdiction": {
       "description": "Jurisdiction of the subsidiary",
-      "type": "string"
+      "type": "string",
+      "minLength": 1
     },
     "subsidiary_identifier": {
       "description": "Any unique id for the subsidiary that might be provided in the source",