whos_got_dirt 0.0.2

data/lib/whos_got_dirt/responses/entity/little_sis.rb

@@ -0,0 +1,66 @@
+module WhosGotDirt
+  module Responses
+    module Entity
+      # Converts entities from the LittleSis API to Popolo format.
+      #
+      # @see http://api.littlesis.org/documentation
+      class LittleSis < Helpers::LittleSisHelper
+        @count_field = 'Entities'
+
+        @template = {
+          '@type' => 'Entity',
+          'type' => '/primary_type',
+          'name' => '/name',
+          'description' => '/description',
+          'identifiers' => [{
+            'identifier' => '/id',
+            'scheme' => 'LittleSis',
+          }],
+          'links' => [{
+            'url' => '/website',
+            'note' => 'Website',
+          }, {
+            'url' => '/uri',
+            'note' => 'LittleSis page',
+          }, {
+            'url' => '/api_uri',
+            'note' => 'LittleSis API detail',
+          }],
+          'updated_at' => '/updated_at',
+
+          # Class-specific.
+          'start_date' => lambda{|data|
+            if JsonPointer.new(data, '/primary_type').value == 'Person'
+              k = 'birth_date'
+            else # 'Org'
+              k = 'founding_date'
+            end
+            [k, JsonPointer.new(data, '/start_date').value]
+          },
+          'end_date' => lambda{|data|
+            if JsonPointer.new(data, '/primary_type').value == 'Person'
+              k = 'death_date'
+            else # 'Org'
+              k = 'dissolution_date'
+            end
+            [k, JsonPointer.new(data, '/end_date').value]
+          },
+          'parent_id' => '/parent_id',
+
+          # API-specific.
+          'is_current' => '/is_current',
+          'summary' => '/summary',
+        }
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['Data']['Entities']['Entity'].map do |data|
+            Result.new('Entity', renderer.result(data), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/entity/open_corporates.rb

@@ -0,0 +1,61 @@
+module WhosGotDirt
+  module Responses
+    module Entity
+      # Converts companies from the OpenCorporates API to Popolo format.
+      #
+      # @see https://api.opencorporates.com/documentation/API-Reference
+      class OpenCorporates < Helpers::OpenCorporatesHelper
+        @template = {
+          '@type' => 'Entity',
+          'name' => '/name',
+          'classification' => '/company_type',
+          'founding_date' => '/incorporation_date',
+          'dissolution_date' => '/dissolution_date',
+          'created_at' => '/created_at',
+          'updated_at' => '/updated_at',
+          'other_names' => [{
+            'name' => '/previous_names/company_name',
+            'start_date' => '/previous_names/start_date',
+            'end_date' => '/previous_names/end_date',
+            'note' => '/previous_names/type',
+          }],
+          'identifiers' => [{
+            'identifier' => '/company_number',
+            'scheme' => 'Company Register',
+          }],
+          'contact_details' => [{
+            'type' => 'address',
+            'value' => '/registered_address_in_full',
+          }],
+          'links' => [{
+            'url' => '/opencorporates_url',
+            'note' => 'OpenCorporates page',
+          }, {
+            'url' => '/registry_url',
+            'note' => 'Register page',
+          }],
+          'sources' => [{
+            'url' => '/sources/url',
+            'note' => '/sources/publisher',
+            'retrieved_at' => '/sources/retrieved_at', # @todo check Dublin Core, etc.
+          }],
+          # API-specific.
+          'branch_status' => '/branch_status', # @todo if boolean, use "branch" as in request
+          'current_status' => '/current_status',
+          'inactive' => '/inactive',
+          'jurisdiction_code' => '/jurisdiction_code',
+          'retrieved_at' => '/retrieved_at', # @todo check Dublin Core, etc.
+        }
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['companies'].map do |data|
+            Result.new('Organization', renderer.result(data['company']), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/entity/open_duka.rb

@@ -0,0 +1,42 @@
+module WhosGotDirt
+  module Responses
+    module Entity
+      # Converts entities from the OpenDuka API to Popolo format.
+      #
+      # @see http://www.openduka.org/index.php/api/documentation
+      class OpenDuka < Response
+        @template = {
+          '@type' => 'Entity',
+          'name' => '/Name',
+          'identifiers' => [{
+            'identifier' => '/ID',
+            'scheme' => 'OpenDuka',
+          }],
+        }
+
+        # Parses the response body.
+        #
+        # @return [Array<Hash>] the parsed response body
+        def parse_body
+          JSON.load(body)
+        end
+
+        # Returns the total number of matching results.
+        #
+        # @return [Fixnum] the total number of matching results
+        def count
+          parsed_body.size
+        end
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body.map do |data|
+            Result.new('Entity', renderer.result(data), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/entity/poderopedia.rb

@@ -0,0 +1,44 @@
+module WhosGotDirt
+  module Responses
+    module Entity
+      # Converts entities from the Poderopedia API to Popolo format.
+      #
+      # @see http://api.poderopedia.org/search
+      class Poderopedia < Response
+        @template = {
+          '@type' => 'Entity',
+          'name' => '/alias',
+          'identifiers' => [{
+            'identifier' => '/id',
+            'scheme' => 'Poderopedia',
+          }],
+          'description' => '/shortBio'
+        }
+
+        # Parses the response body.
+        #
+        # @return [Array<Hash>] the parsed response body
+        def parse_body
+          parsed = JSON.load(body)
+          parsed['organization'] || parsed['person']
+        end
+
+        # Returns the total number of matching results.
+        #
+        # @return [Fixnum] the total number of matching results
+        def count
+          parsed_body.size
+        end
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body.map do |data|
+            Result.new('Entity', renderer.result(data), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/entity.rb

@@ -0,0 +1,7 @@
+module WhosGotDirt
+  module Responses
+    # Responses for entities.
+    module Entity
+    end
+  end
+end

data/lib/whos_got_dirt/responses/helpers/little_sis.rb

@@ -0,0 +1,34 @@
+module WhosGotDirt
+  module Responses
+    module Helpers
+      class LittleSisHelper < Response
+        class << self
+          # @!attribute [r] count_field
+          #   @return [Hash] the field storing the number of results
+          attr_reader :count_field
+        end
+
+        # Parses the response body.
+        #
+        # @return [Array<Hash>] the parsed response body
+        def parse_body
+          Nori.new.parse(body)['Response']
+        end
+
+        # Returns the total number of matching results.
+        #
+        # @return [Fixnum] the total number of matching results
+        def count
+          Integer(parsed_body['Meta']['TotalCount'] || parsed_body['Meta']['ResultCount'][self.class.count_field])
+        end
+
+        # Returns the current page number.
+        #
+        # @return [Fixnum] the current page number
+        def page
+          Integer(parsed_body['Meta']['Parameters']['page'] || 1)
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/helpers/open_corporates.rb

@@ -0,0 +1,28 @@
+module WhosGotDirt
+  module Responses
+    module Helpers
+      class OpenCorporatesHelper < Response
+        # Parses the response body.
+        #
+        # @return [Array<Hash>] the parsed response body
+        def parse_body
+          JSON.load(body)['results']
+        end
+
+        # Returns the total number of matching results.
+        #
+        # @return [Fixnum] the total number of matching results
+        def count
+          parsed_body['total_count']
+        end
+
+        # Returns the current page number.
+        #
+        # @return [Fixnum] the current page number
+        def page
+          parsed_body['page']
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/list/little_sis.rb

@@ -0,0 +1,42 @@
+module WhosGotDirt
+  module Responses
+    module List
+      # Converts lists from the OpenCorporates API to Popolo format.
+      #
+      # @see http://api.littlesis.org/documentation
+      class LittleSis < Helpers::LittleSisHelper
+        @count_field = 'Lists'
+
+        @template = {
+          '@type' => 'List',
+          'name' => '/name',
+          'description' => '/description',
+          'number_of_items' => '/num_entities',
+          'item_list_order' => lambda{|data|
+            v = JsonPointer.new(data, '/is_ranked').value
+            if v == '1'
+              v = 'ascending'
+            else
+              v = 'unordered'
+            end
+            ['item_list_order', v]
+          },
+          'updated_at' => '/updated_at',
+          'identifiers' => [{
+            'identifier' => '/id',
+            'scheme' => 'LittleSis',
+          }],
+        }
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['Data']['Lists']['List'].map do |data|
+            Result.new('List', renderer.result(data), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/list/open_corporates.rb

@@ -0,0 +1,34 @@
+module WhosGotDirt
+  module Responses
+    module List
+      # Converts corporate groupings from the OpenCorporates API to Popolo format.
+      #
+      # @see https://api.opencorporates.com/documentation/API-Reference
+      class OpenCorporates < Helpers::OpenCorporatesHelper
+        @template = {
+          '@type' => 'List',
+          'name' => '/name',
+          'created_at' => '/created_at',
+          'updated_at' => '/updated_at',
+          'identifiers' => [{
+            'identifier' => '/wikipedia_id',
+            'scheme' => 'Wikipedia',
+          }],
+          'links' => [{
+            'url' => '/opencorporates_url',
+            'note' => 'OpenCorporates page',
+          }],
+        }
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['corporate_groupings'].map do |data|
+            Result.new('List', renderer.result(data['corporate_grouping']), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/list.rb

@@ -0,0 +1,7 @@
+module WhosGotDirt
+  module Responses
+    # Responses for lists.
+    module List
+    end
+  end
+end

data/lib/whos_got_dirt/responses/relation/open_corporates.rb

@@ -0,0 +1,66 @@
+module WhosGotDirt
+  module Responses
+    module Relation
+      # Converts corporate officerships from the OpenCorporates API to Popolo format.
+      #
+      # @see https://api.opencorporates.com/documentation/API-Reference
+      class OpenCorporates < Helpers::OpenCorporatesHelper
+        @template = {
+          '@type' => 'Relation',
+          'subject' => [{
+            'name' => '/name',
+            'birth_date' => '/date_of_birth',
+            'contact_details' => [{
+              'type' => 'address',
+              'value' => '/address',
+            }],
+            # API-specific.
+            'nationality' => '/nationality',
+            'occupation' => '/occupation',
+          }],
+          'object' => {
+            'name' => '/company/name',
+            'identifiers' => [{
+              'identifier' => '/company/company_number',
+              'scheme' => 'Company Register',
+            }],
+            'links' => [{
+              'url' => '/company/opencorporates_url',
+              'note' => 'OpenCorporates page',
+            }],
+            # API-specific.
+            'jurisdiction_code' => '/company/jurisdiction_code',
+          },
+          'start_date' => '/start_date',
+          'end_date' => '/end_date',
+          'identifiers' => [{
+            'identifier' => '/id',
+            'scheme' => 'OpenCorporates',
+          }, {
+            'identifier' => '/uid',
+            'scheme' => 'Company Register',
+          }],
+          'links' => [{
+            'url' => '/opencorporates_url',
+            'note' => 'OpenCorporates page',
+          }],
+          'updated_at' => '/retrieved_at',
+          # API-specific.
+          'inactive' => '/inactive',
+          'current_status' => '/current_status',
+          'jurisdiction_code' => '/jurisdiction_code',
+          'role' => '/position',
+        }
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['officers'].map do |data|
+            Result.new('Relation', renderer.result(data['officer']), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/relation/open_oil.rb

@@ -0,0 +1,70 @@
+module WhosGotDirt
+  module Responses
+    module Relation
+      # Converts concessions from the OpenOil API to Popolo format.
+      #
+      # @see http://openoil.net/openoil-api/
+      class OpenOil < Response
+        @template = {
+          '@type' => 'Relation',
+          'subject' => lambda{|data|
+            v = JsonPointer.new(data, '/licensees').value
+            ['subject', v.map{|licensee| {'name' => licensee}}]
+          },
+          'identifiers' => [{
+            'identifier' => '/identifier',
+            'scheme' => 'OpenOil',
+          }],
+          'links' => [{
+            'url' => '/url_api',
+            'note' => 'OpenOil API detail',
+          }, {
+            'url' => '/url_wiki',
+            'note' => 'OpenOil wiki page',
+          }],
+          'name' => '/name',
+          'created_at' => '/source_date',
+          'updated_at' => '/retrieved_date',
+          'sources' => [{
+            'url' => '/source_document',
+          }],
+          # API-specific.
+          'additional_properties' => '/details',
+          'country_code' => '/country',
+          'status' => '/status',
+          'type' => '/type',
+        }
+
+        # Parses the response body.
+        #
+        # @return [Array<Hash>] the parsed response body
+        def parse_body
+          JSON.load(body)
+        end
+
+        # Returns the total number of matching results.
+        #
+        # @return [Fixnum] the total number of matching results
+        def count
+          parsed_body['result_count']
+        end
+
+        # Returns the current page number.
+        #
+        # @return [Fixnum] the current page number
+        def page
+          parsed_body['page']
+        end
+
+        # Transforms the parsed response body into results.
+        #
+        # @return [Array<Hash>] the results
+        def to_a
+          parsed_body['results'].map do |data|
+            Result.new('Relation', renderer.result(data), self).finalize!
+          end
+        end
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/responses/relation.rb

@@ -0,0 +1,7 @@
+module WhosGotDirt
+  module Responses
+    # Responses for relations.
+    module Relation
+    end
+  end
+end

data/lib/whos_got_dirt/result.rb

@@ -0,0 +1,106 @@
+module WhosGotDirt
+  # A result from a response.
+  class Result
+    # @!attribute [r] result
+    #   @return [Hash] the result
+    attr_reader :result
+
+    # @!attribute [r] response
+    #   @return [Response] the response from which the result was created
+    attr_reader :response
+
+    # @!attribute [r] type
+    #   @return [Response] the Popolo class to validate against
+    attr_reader :type
+
+    # Sets the result and response.
+    #
+    # @param [String] type the Popolo class to validate against
+    # @param [Hash] result the rendered result
+    # @param [Response] response the response from which the result was created
+    def initialize(type, result, response)
+      @type = type
+      @result = result
+      @response = response
+    end
+
+    # Adds the requested URL as a source.
+    #
+    # @return [Hash] the result
+    def add_source!
+      result['sources'] ||= []
+      result['sources'] << {
+        'url' => response.env.url.to_s,
+        'note' => response.class.name.rpartition('::')[2],
+      }
+      result
+    end
+
+    # Validates the result against its schema. If recognized errors occur, they
+    # are corrected, and the result is revalidated; if any error occurs during
+    # revalidation, an exception is raised.
+    #
+    # @param [Hash] opts options
+    # @option opts [Boolean] :strict (false) whether to raise an error if any
+    #   error occurs
+    # @return [Hash] the result
+    # @raise if an unrecognized error occurs, or if any error occurs during
+    #   revalidation
+    def validate!(opts = {})
+      # The code assumes that processing errors in reverse avoids re-indexing
+      # issues when deleting items from arrays. If this assumption is invalid,
+      # we can delete items on a time and re-validate using this code skeleton:
+      #
+      # begin
+      #   Validator.validate(result, type)
+      # rescue JSON::Schema::ValidationError => e
+      #   error = e.to_hash
+      #   case error[:failed_attribute]
+      #   when 'Properties'
+      #     ...
+      #     validate!
+      #   ...
+      #   end
+      # end
+
+      errors = Validator.validate(result, type)
+
+      if opts[:strict] && errors.any?
+        raise ValidationError.new(errors * '\n')
+      end
+
+      errors.reverse.each do |error|
+        pointer = JsonPointer.new(result, error[:fragment][1..-1])
+
+        case error.fetch(:failed_attribute)
+        when 'Properties'
+          # The property did not contain a required property. This should be due
+          # to the source having a null value.
+          pointer.delete
+        when 'Type'
+          # The property did not match one or more types. This should be due to
+          # the source having an integer instead of string value.
+          pointer.value = pointer.value.to_s
+        else
+          raise ValidationError.new(error.fetch(:message))
+        end
+      end
+
+      if errors.any?
+        validate!(strict: true)
+      end
+
+      result
+    end
+
+    # Adds the requested URL as a source, validates the result against its
+    # schema, and returns the updated result.
+    #
+    # @return [Hash] the result
+    def finalize!
+      add_source!
+      validate!
+      result
+    end
+  end
+end

data/lib/whos_got_dirt/validator.rb

@@ -0,0 +1,52 @@
+module WhosGotDirt
+  module Validator
+    class << self
+      # Validates the data against the named schema and returns any errors.
+      #
+      # @param [Hash] data the data to validate
+      # @param [String] name the name of the definition in the JSON Schema
+      # @return [Array<ValidationError>] a list of validation errors
+      def validate(data, name)
+        validator = validators[name.downcase]
+        # @see https://github.com/ruby-json-schema/json-schema/blob/fa316dc9d39b922935aed8ec9fa0e4139b724ef5/lib/json-schema/validator.rb#L40
+        validator.instance_variable_set('@errors', [])
+        # `JSON::Validator#initialize_data` does nothing, in our case.
+        # `@@original_data` doesn't need to be re-initialized.
+        # @see https://github.com/ruby-json-schema/json-schema/blob/fa316dc9d39b922935aed8ec9fa0e4139b724ef5/lib/json-schema/validator.rb#L53
+        validator.instance_variable_set('@data', data)
+        validator.validate
+      end
+
+    private
+
+      # The json-schema gem is very, very slow, so we implement optimizations.
+
+      # Memoizes and returns validators using a fragment of the schema.
+      def validators
+        @validators ||= Hash.new do |hash,name|
+          v = validator.dup
+          # @see https://github.com/ruby-json-schema/json-schema/blob/fa316dc9d39b922935aed8ec9fa0e4139b724ef5/lib/json-schema/validator.rb#L73
+          v.instance_variable_set('@base_schema', v.schema_from_fragment(v.instance_variable_get('@base_schema'), "#/definitions/#{name}"))
+          hash[name] = v
+        end
+      end
+
+      # Memoizes a validator using the schema.
+      def validator
+        # `JSON::Validator#initialize_schema` runs faster if given a `Hash`.
+        @@validator ||= JSON::Validator.new(JSON.load(File.read(File.expand_path(File.join('..', '..', '..', 'schemas', 'popolo.json'), __FILE__))), {}, {
+          # Keep the cache - whatever it is.
+          clear_cache: false,
+          # It's safe to skip data parsing if the data is a `Hash`.
+          parse_data: false,
+          # Push errors onto `@errors` instead of raising. Setting to false would
+          # result in a single error being reported.
+          record_errors: true,
+          # `ValidationError#to_hash` is probably slower than
+          # `ValidationError#to_string`, but it is not yet a bottleneck.
+          errors_as_objects: true,
+        })
+      end
+    end
+  end
+end

data/lib/whos_got_dirt/version.rb

@@ -0,0 +1,3 @@
+module WhosGotDirt
+  VERSION = "0.0.2"
+end