openc_bot 0.0.16 → 0.0.18

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "schemas"]
+	path = schemas
+	url = git://github.com/openc/openc-schema.git

data/.travis.yml

@@ -1,7 +1,8 @@
 language: ruby
 rvm:
-  - "1.9.2"
+  # - "1.9.2"
   - "1.9.3"
+  - "2.0.0"
   - "2.1.0"
   # - jruby-18mode # JRuby in 1.8 mode
   # - jruby-19mode # JRuby in 1.9 mode

data/examples/basic/lib/basic.rb

@@ -41,13 +41,14 @@ class BasicRecord < SimpleOpencBot::BaseLicenceRecord
   # `doc/SCHEMA.md` for documentation.
   def to_pipeline
     {
-      sample_date: last_updated_at,
       company: {
         name: name,
         jurisdiction: JURISDICTION,
       },
-      source_url: URL,
       data: [{
+        source_url: URL,
+        sample_date: last_updated_at,
+        confidence: "MEDIUM",
         data_type: :licence,
         properties: {
           jurisdiction_code: JURISDICTION,

data/lib/openc_bot/helpers/register_methods.rb

@@ -1,5 +1,6 @@
 # encoding: UTF-8
 require 'json-schema'
+require 'active_support'
 require 'active_support/core_ext'
 
 module OpencBot
@@ -142,7 +143,7 @@ module OpencBot
       end
 
       def validate_datum(record)
-        schema = File.expand_path("../../../../schemas/#{schema_name}.json", __FILE__)
+        schema = File.expand_path("../../../../schemas/schemas/#{schema_name}.json", __FILE__)
         errors = JSON::Validator.fully_validate(
           schema,
           record.to_json,

data/lib/openc_bot/templates/lib/simple_bot.rb

@@ -32,14 +32,15 @@ class MyLicenceRecord < SimpleOpencBot::BaseLicenceRecord
   # `doc/SCHEMA.md` for documentation.
   def to_pipeline
     {
-      sample_date: last_updated_at,
       company: {
         name: name,
         jurisdiction: "xx",
       },
-      source_url: "xx",
       data: [{
         data_type: :licence,
+        source_url: "xx",
+        sample_date: last_updated_at,
+        confidence: "MEDIUM",
         properties: {
           jurisdiction_code: "xx",
           category: 'Financial',

data/lib/openc_bot/version.rb

@@ -1,3 +1,3 @@
 module OpencBot
-  VERSION = "0.0.16"
+  VERSION = "0.0.18"
 end

data/lib/simple_openc_bot.rb

@@ -224,7 +224,7 @@ class SimpleOpencBot
 
     def self.schema(schema)
       hyphenated_name = schema.to_s.gsub("_", "-")
-      self._schema = File.expand_path("../../schemas/#{hyphenated_name}-schema.json", __FILE__)
+      self._schema = File.expand_path("../../schemas/schemas/#{hyphenated_name}-schema.json", __FILE__)
     end
 
     def initialize(attrs={})
@@ -274,7 +274,7 @@ class SimpleOpencBot
       if data
         if !self._schema
           # backwards compatibility
-          self._schema = File.expand_path("../../schemas/licence-schema.json", __FILE__)
+          self._schema = File.expand_path("../../schemas/schemas/licence-schema.json", __FILE__)
         end
         errors = JSON::Validator.fully_validate(
           self._schema,

data/openc_bot.gemspec

@@ -14,6 +14,19 @@ Gem::Specification.new do |gem|
 
   gem.files         = `git ls-files`.split($/)
 
+  # get an array of submodule dirs by executing 'pwd' inside each submodule
+  gem_dir = File.expand_path(File.dirname(__FILE__)) + "/"
+  `git submodule --quiet foreach pwd`.split($\).each do |submodule_path|
+    Dir.chdir(submodule_path) do
+      submodule_relative_path = submodule_path.sub gem_dir, ""
+      # issue git ls-files in submodule's directory and
+      # prepend the submodule path to create absolute file paths
+      `git ls-files`.split($\).each do |filename|
+        gem.files << "#{submodule_relative_path}/#{filename}"
+      end
+    end
+  end
+
   gem.executables   = ['openc_bot']
 
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})

data/publish.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+UNPUSHED=$(git log origin/master..HEAD)
+if [ "x$UNPUSHED" == "x" ]; then
+    gem build openc_bot.gemspec
+    gem push $(ls *gem|tail -1)
+fi
+
+function clean {
+ rm -f *gem
+}
+trap clean EXIT

data/schemas/.gitignore

@@ -0,0 +1,13 @@
+/.bundle
+/.rvmrc
+/coverage
+/pkg
+/rdoc
+/tags
+/vendor
+/.rbenv-version
+.rspec
+.DS_Store
+*~
+.#*
+#*#

data/schemas/.travis.yml

@@ -0,0 +1,5 @@
+language: ruby
+rvm:
+  - "1.9.3"
+  - "2.0.0"
+  - "2.1.0"

data/schemas/Gemfile

@@ -0,0 +1,9 @@
+# A sample Gemfile
+source "https://rubygems.org"
+
+group :test do
+  gem "rspec"
+  gem 'json-schema'
+  gem 'debugger'
+  gem 'rake'
+end

data/schemas/Gemfile.lock

@@ -0,0 +1,34 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    columnize (0.8.9)
+    debugger (1.6.8)
+      columnize (>= 0.3.1)
+      debugger-linecache (~> 1.2.0)
+      debugger-ruby_core_source (~> 1.3.5)
+    debugger-linecache (1.2.0)
+    debugger-ruby_core_source (1.3.5)
+    diff-lcs (1.2.5)
+    json-schema (2.2.2)
+    rake (10.3.2)
+    rspec (3.0.0)
+      rspec-core (~> 3.0.0)
+      rspec-expectations (~> 3.0.0)
+      rspec-mocks (~> 3.0.0)
+    rspec-core (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-expectations (3.0.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.0.0)
+    rspec-mocks (3.0.2)
+      rspec-support (~> 3.0.0)
+    rspec-support (3.0.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  debugger
+  json-schema
+  rake
+  rspec

data/schemas/README.md

@@ -0,0 +1,2 @@
+These are JSON schemas for validating data before it is sent to
+OpenCorporates.

data/schemas/Rakefile

@@ -0,0 +1,46 @@
+require "rubygems"
+
+PROJECT_ROOT = File.expand_path("..", __FILE__)
+
+require "rspec/core/rake_task"
+# require 'debugger'
+
+desc "Run all specs"
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.verbose = true
+end
+
+task :default => :spec
+
+## dist
+
+require "json"
+
+desc "Regenerate schema files from snippets and reference files"
+task :regenerate_data_object_schemas do
+  spacers = '='*20
+  puts "Regenerating OpenCorporates DataObject schemas"
+  puts spacers
+  base_object = JSON.parse(File.new(File.join(PROJECT_ROOT,'lib','base-statement.json')).read)
+  snippets = Dir.glob(File.join(PROJECT_ROOT,'lib','snippets','*base.json'))
+  snippets.each do |snippet|
+    object_name = snippet[/\/([^\/]+)-base.json/,1]
+    snippet_data = JSON.parse(File.new(snippet).read)
+    combined_data = {}.merge(snippet_data).merge(base_object)
+    combined_data['properties']['data']['items'] = {
+        "$ref" => "includes/#{object_name}-data-object.json"
+      }
+    File.open(File.join(PROJECT_ROOT,'schemas',"#{object_name}-schema.json"),'w') { |file| file.puts JSON.pretty_generate(combined_data) }
+    puts "Generated schema: '#{object_name}-schema.json'"
+  end
+  puts spacers
+  puts "Finished regenerating schemas.\nNow running validation specs"
+  puts spacers
+  Rake::Task['run_validation_specs'].invoke
+end
+
+task :run_validation_specs do
+  Rake::Task['default'].invoke
+end
+
+

data/schemas/lib/base-statement.json

@@ -0,0 +1,22 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "source_jurisdiction": {
+      "description": "Jurisdiction of the source of the data",
+      "type": "string"
+    },
+    "company": {
+      "$ref" : "includes/company.json"
+    },
+    "data": {
+      "type": "array",
+      "description": "This is an array of data objects, that is the objects which actually contain the data about the datum. Generally there will only be one element in this, although in for some types of data_types there may naturally be several releated to the same company, e.g. shareholders",
+      "items": {
+        "$ref": "includes/financial-payment-data-object.json"
+      },
+      "additionalItems": false
+    }
+  },
+  "required": ["company", "data"]
+}

data/schemas/lib/snippets/financial-payment-base.json

@@ -0,0 +1,4 @@
+{
+  "title": "Financial-Payment",
+  "description": "A Financial-Payment is a payment from government to a recipient"
+}

data/schemas/lib/snippets/licence-base.json

@@ -0,0 +1,4 @@
+{
+  "title": "Licence Schema",
+  "description": "A Licence is a permission for an entity to do something that would otherwise not be permitted. An example is a 'Financial' licence to operate as a bank, or an financial advisor, for example, or a 'Business' Licence to sell alcohol, or operate an amusement arcade. Future licence types (extractives, pollution) will be added later"
+}

data/schemas/schemas/financial-payment-schema.json

@@ -0,0 +1,27 @@
+{
+  "title": "Financial-Payment",
+  "description": "A Financial-Payment is a payment from government to a recipient",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "source_jurisdiction": {
+      "description": "Jurisdiction of the source of the data",
+      "type": "string"
+    },
+    "company": {
+      "$ref": "includes/company.json"
+    },
+    "data": {
+      "type": "array",
+      "description": "This is an array of data objects, that is the objects which actually contain the data about the datum. Generally there will only be one element in this, although in for some types of data_types there may naturally be several releated to the same company, e.g. shareholders",
+      "items": {
+        "$ref": "includes/financial-payment-data-object.json"
+      },
+      "additionalItems": false
+    }
+  },
+  "required": [
+    "company",
+    "data"
+  ]
+}

data/schemas/{includes → schemas/includes}/alternative_name.json

@@ -8,6 +8,6 @@
                   "language": { "type": "string", "minLength": 2, "maxLength": 2 },
                   "type": { "type": "string", "enum": ["trading", "abbreviation", "legal"] }
   },
-  "required": [ "company_name" ]
+  "required": [ "company_name", "type" ]
 }
 

data/schemas/{includes → schemas/includes}/company.json

@@ -5,8 +5,10 @@
   "properties": {
     "name": { "type": "string" },
     "jurisdiction": { "type": "string" },
+    "company_number": { "type": "string" },
     "identifier": {
       "type": "string",
+      "description": "An official identifier paired with a code for the issuer of the identifier, e.g sec/12345",
       "pattern": "^[^/]+/[^/]+$"
     }
   },

data/schemas/schemas/includes/financial-payment-data-object.json

@@ -0,0 +1,60 @@
+{
+  "name": "Financial Payment Datum",
+  "type": "object",
+  "properties": {
+    "data_type": { "enum": [ "financial_payment" ] },
+    "source_url": {
+      "type": "string"},
+    "confidence": {
+      "type": "string"},
+    "properties": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "description": "The amount transacted (e.g. 19.95)",
+          "type": "string"},
+        "payee_name": {
+          "description": "The name of the payee (the entity that received the money)",
+          "type": "string"},
+        "currency": {
+          "description": "The currency of the payment (three letter symbol, e.g. USD, GBP, EUR)",
+          "type": "string"},
+        "date": {
+          "description": "The date on which the payment was made",
+          "type": "date"},
+        "transaction_number": {
+          "description": "The unique identifier for the transaction, scoped to this bot",
+          "type": "string"},
+        "jurisdiction": {
+          "description": "The jurisdiction of the body that made the payment, eg. UK, France, Delaware, Manchester",
+          "type": "string"},
+        "more_details_url": {
+          "description": "A url from which more details can be seen (may be the same as the source_url)",
+          "type": ["string",null]},
+        "description": {
+          "description": "The description of the transaction as given in the raw data",
+          "type": ["string",null]},
+        "expense_type": {
+          "description": "The type of expense -- can be either capital, revenue (i.e. current expenditure) or null",
+          "enum": ["capital","revenue",null]},
+        "expense_area": {
+          "description": "category (in words) of the expenditure",
+          "type": ["string",null]},
+        "entity_name": {
+          "description": "The name of the government entity that made the payment, e.g. Environment Agency",
+          "type": "string"},
+        "entity_uri": {
+          "description": "A unique URL (ideally a dereferencable URI) for the government entity",
+          "type": ["string",null]},
+        "department_name": {
+          "description": "The name of the government department which the entity belongs to (if relevant), e.g. Department of Health",
+          "type": ["string",null]},
+        "csv_line_number": {
+          "description": "If the source for the data is a CSV file, you can optionally include the line number of the CSV from which this data was retrieved",
+          "type": ["string",null]}
+      },
+      "required": [ "value", "payee_name", "date","currency" ]
+    }
+  },
+  "required": [ "data_type", "properties","source_url","confidence" ]
+}

data/schemas/{includes/license-data.json → schemas/includes/licence-data-object.json}

@@ -1,27 +1,36 @@
 {
-  "name": "License Data",
+  "name": "Licence Data",
   "type": "object",
   "properties": {
     "data_type": { "enum": [ "licence" ] },
+    "source_url": {
+      "type": "string"},
+    "confidence": {
+      "type": "string"},
     "properties": {
       "type": "object",
       "properties": {
         "regulator": {
           "description": "The regulating body that issued the licence",
           "type": "string"},
+        "category": {
+          "description": "The category of licence. Current possible values are: 'Financial','Business'",
+          "enum": [ "Financial", "Business" ]},
         "jurisdiction_code": {
           "description": "The jurisdiction for which licence was issued",
           "type": "string"},
         "licence_number": {
-          "type": "string"},
+          "description": "Licence number or code (can be null)",
+          "type": ["string",null]},
         "jurisdiction_classification": {
           "type": "array",
+          "description": "The local classification given by the regulator",
           "minItems": 1},
         "oc_classification": {
           "type": "array"}
       },
-      "required": [ "jurisdiction_code", "jurisdiction_classification" ]
+      "required": [ "jurisdiction_code", "jurisdiction_classification", "category" ]
     }
   },
-  "required": [ "data_type", "properties" ]
+  "required": [ "data_type", "properties","source_url","confidence" ]
 }

data/schemas/{includes → schemas/includes}/subsidiary-relationship-data.json

@@ -11,6 +11,9 @@
         "direct": {
           "description": "If the control is direct (if via an intermediary, this value should be false; if unknown, left blank)",
           "type": "boolean"},
+        "percentage_controlled": {
+          "description": "Percentage controlled, either directly or indirectly",
+          "type": "string"},
         "significant": {
           "description": "Does the source define the control as somehow significant?",
           "type": "boolean"},
@@ -33,15 +36,7 @@
               "type": "string"
             },
             "identifier": {
-              "description": "Unique identifier of person",
-              "type": "string"
-            },
-            "address": {
-              "description": "Address given for owner of parcel",
-              "type": "string"
-            },
-            "address_country": {
-              "description": "Country part of owner's address",
+              "description": "Unique identifier of company",
               "type": "string"
             }
           }

data/schemas/schemas/licence-schema.json

@@ -0,0 +1,27 @@
+{
+  "title": "Licence Schema",
+  "description": "A Licence is a permission for an entity to do something that would otherwise not be permitted. An example is a 'Financial' licence to operate as a bank, or an financial advisor, for example, or a 'Business' Licence to sell alcohol, or operate an amusement arcade. Future licence types (extractives, pollution) will be added later",
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "properties": {
+    "source_jurisdiction": {
+      "description": "Jurisdiction of the source of the data",
+      "type": "string"
+    },
+    "company": {
+      "$ref": "includes/company.json"
+    },
+    "data": {
+      "type": "array",
+      "description": "This is an array of data objects, that is the objects which actually contain the data about the datum. Generally there will only be one element in this, although in for some types of data_types there may naturally be several releated to the same company, e.g. shareholders",
+      "items": {
+        "$ref": "includes/licence-data-object.json"
+      },
+      "additionalItems": false
+    }
+  },
+  "required": [
+    "company",
+    "data"
+  ]
+}

data/schemas/schemas/primary-data-schema.json

@@ -0,0 +1,17 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "title": "Primary Data Schema",
+    "type": "object",
+    "properties": {
+        "source_url": {
+            "description": "Place where this fact can be verified",
+            "type": "string"
+        },
+        "sample_date": {
+            "description": "Date on which this fact was known to be true",
+            "type": "string"
+        }
+    },
+    "required": ["source_url", "sample_date"],
+    "additionalProperties": true
+}

data/schemas/schemas/simple-financial-payment-schema.json

@@ -0,0 +1,85 @@
+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "title": "Simple Financial Payment",
+  "description": "A Financial Payment is a payment from government to a recipient",
+  "type": "object",
+  "properties": {
+    "source_url": {
+      "description": "Place where this fact can be verified",
+      "type": "string"
+    },
+    "sample_date": {
+      "description": "Date on which this fact was known to be true",
+      "type": "string"
+    },
+    "confidence": {
+      "description": "Confidence in accuracy of data",
+      "type": "string",
+      "enum": ["HIGH", "MEDIUM", "LOW"]
+    },
+    "company_name": {
+      "description": "Name of the company that received the money",
+      "type": "string"
+    },
+    "company_jurisdiction": {
+      "description": "Jurisdiction where company is based",
+      "type": "string"
+    },
+    "value": {
+      "description": "The amount transacted (e.g. 19.95)",
+      "type": "string"
+    },
+    "payee_name": {
+      "description": "The name of the payee (the entity that received the money)",
+      "type": "string"
+    },
+    "currency": {
+      "description": "The currency of the payment (three letter symbol, e.g. USD, GBP, EUR)",
+      "type": "string"
+    },
+    "date": {
+      "description": "The date on which the payment was made",
+      "type": "date"
+    },
+    "transaction_number": {
+      "description": "The unique identifier for the transaction, scoped to this bot",
+      "type": "string"
+    },
+    "jurisdiction": {
+      "description": "The jurisdiction of the body that made the payment, eg. UK, France, Delaware, Manchester",
+      "type": "string"
+    },
+    "more_details_url": {
+      "description": "A url from which more details can be seen (may be the same as the source_url)",
+      "type": ["string",null]
+    },
+    "description": {
+      "description": "The description of the transaction as given in the raw data",
+      "type": ["string",null]
+    },
+    "expense_type": {
+      "description": "The type of expense -- can be either capital, revenue (i.e. current expenditure) or null",
+      "enum": ["capital","revenue",null]
+    },
+    "expense_area": {
+      "description": "category (in words) of the expenditure",
+      "type": ["string",null]
+    },
+    "entity_name": {
+      "description": "The name of the government entity that made the payment, e.g. Environment Agency",
+      "type": "string"
+    },
+    "entity_uri": {
+      "description": "A unique URL (ideally a dereferencable URI) for the government entity",
+      "type": ["string",null]
+    },
+    "department_name": {
+      "description": "The name of the government department which the entity belongs to (if relevant), e.g. Department of Health",
+      "type": ["string",null]
+    },
+    "csv_line_number": {
+      "description": "If the source for the data is a CSV file, you can optionally include the line number of the CSV from which this data was retrieved",
+      "type": ["string",null]}
+  },
+  "required": ["source_url", "sample_date", "company_name", "company_jurisdiction", "value", "payee_name", "date", "currency"]
+}