gimme_wikidata 0.4.2

data/lib/gimme_wikidata/entity.rb

@@ -0,0 +1,157 @@
+module GimmeWikidata
+
+  ##
+  # Models an Entity on Wikidata.
+  #
+  # An Entity is suclassed into Item and Property.
+  #
+  # An Entity has to have a valid Wikidata id, but the other class data is optional
+  #
+  # See: https://www.wikidata.org/wiki/Wikidata:Glossary#Entities.2C_items.2C_properties_and_queries
+  class Entity
+
+    ##
+    # The Wikidata id of the Entity in the form 'PN' or 'QN' where 'P' means Property and 'Q' means Item.  N can be any positive integer.
+    attr_reader :id
+    ##
+    # The Entity's label (in the language specified by WikidataAPI).  Can be nil in the case of an unresolved Entity.
+    attr_accessor :label
+    ##
+    # The Entity's description (in the language specified by WikidataAPI).  Can be nil in the case of an unresolved Entity.
+    attr_accessor :description
+    ##
+    # The Entity's aliases (in the language specified by WikidataAPI).  Can be an empty array in the case of an unresolved Entity.
+    attr_accessor :aliases
+    ##
+    # Each Entity has a number of claims, stored as an array of Claim objects.
+    attr_accessor :claims
+
+    def initialize(id, label = nil, description = nil, aliases = nil, claims = [])
+      throw ArgumentError.new "Invalid Wikidata Entity id: #{id}" unless GimmeWikidata.valid_id? id
+      @id = id
+      @label = label
+      @description = description
+      @aliases = aliases
+      @claims = claims
+      @claims = [] if @claims.nil?
+    end
+
+    ##
+    # Does this Entity have a fingerprint (an +id+, +title+ and +description+)
+    #
+    # Returns:
+    # - boolean
+    def has_fingerprint?
+      !(@id.nil? || @label.nil? || @description.nil?)
+    end
+
+    ##
+    # Does this Entity have any claims?
+    #
+    # Returns:
+    # - boolean
+    def has_claims?
+      !(@claims.empty?)
+    end
+
+    ##
+    # Get all the Claims which have Properties with the passed id
+    def claims_with_property_id(id)
+      raise ArgumentError.new 'Invaild Wikidata Id' unless GimmeWikidata.valid_id? id
+      claims = []
+      @claims.each do |c|
+        claims << c if c.property.id == id
+      end
+      claims
+    end
+
+    ##
+    # Get more data on an Entity's Properties
+    #
+    # Entities have Properties (through Claims), which are by default only returned as ids.  This function gets more details for each of these.
+    #
+    # * *Parameters:*
+    #   - +props+ -> The Props to fetch.  Defaults to +Props::LABELS+.
+    # * *Returns:*
+    #   - +nil+
+    # * *Raises:*
+    #   - +StandardError+ -> if there was an error in fetching the data
+    def resolve_properties(props = [Props::LABELS])
+      return nil unless has_claims?
+      unique_property_ids = (claims.map {|c| c.property.id}).uniq
+      # Get only the labels for the Entity's Properties from the Wikidata API
+      response = GimmeWikidata::fetch(unique_property_ids, props: [Props::LABELS])
+      raise StandardError.new "Could not resolve Entity (#{@id} - #{@label}) properties" unless response.was_successful?
+      response.entities.each do |property_details|
+        claims = claims_with_property_id(property_details.id)
+        claims.map {|c| c.property.resolve_with(property_details)}
+      end
+      return nil
+    end
+
+    ##
+    # Resolves the Claims that refer to Entities
+    #
+    # Claims often refer to another entity on Wikidata.  By default, we will only know the ids of these with one fetch Entity call.  This therefore gets more data regarding these.
+    #
+    # * *Parameters:*
+    #   - +props+ -> The Props to fetch.  Defaults to +Props::LABELS+
+    def resolve_claims(props = [Props::LABELS])
+      return unless has_claims?
+      item_claims = get_claims_by_value_type(:item)
+      item_ids = item_claims.map {|c| c.value.id }
+      entity_result = GimmeWikidata.fetch(item_ids, props: props)
+      entity_result.entities.each do |item_details|
+        item_index = item_ids.index(item_details.id)
+        item_to_resolve = item_claims[item_index].value
+        item_to_resolve.resolve_with(item_details)
+      end
+    end
+
+    ##
+    # Get all Claims that have a specific +value_type+ symbol
+    #
+    # * *Parameters:*
+    #   - +type+ -> Symbol value of the Claim's +value_type+ attribute
+    # * *Returns:*
+    #   - An Array of Claims
+    def get_claims_by_value_type(type)
+      return [] unless has_claims?
+      claims = []
+      @claims.each do |c|
+        claims << c if c.value_type == type.to_sym
+      end
+      return claims
+    end
+
+    ##
+    # Returns a simplified version of an Entity's Claims
+    # * *Returns:*
+    #   - Array of hashes representing simplified Claims.  See Claim.simplify()
+    def simple_claims
+      claims.map {|c| c.simplify }
+    end
+
+    ##
+    # Resolves an incomplete Entity with additional details
+    #
+    # * *Parameters:*
+    #   - +entity_details+ -> Either an Item or an Property
+    # * *Returns:*
+    # * *Raises:*
+    #   - +ArgumentError+ -> if attempting to resolve an Item with a Property, or a Property with an Item
+    #   - +StandardError+ -> if attempting to resolve an Entity with details that have unequal ids
+    def resolve_with(entity_details)
+      raise ArgumentError.new "Attempting to resolve an Item with a Property or vice versa" if entity_details.id[0] != @id[0]
+      raise StandardError.new "Attempting to resolve Entity with id #{@id} with entity_details with id #{entity_details.id}" unless @id == entity_details.id
+      #TODO: Wouldn't it be easier to simply overwrite the Entity? (self = entity_details?)
+      @label = entity_details.label
+      @description = entity_details.description
+      @aliases = entity_details.aliases
+      @claims = entity_details.claims
+      @claims = [] if @claims.nil?
+    end
+
+  end
+
+end

data/lib/gimme_wikidata/entity_result.rb

@@ -0,0 +1,37 @@
+module GimmeWikidata
+
+  ##
+  # Models a parsed response from a get entities query to the Wikidata API
+  class EntityResult
+
+    attr_reader :success, :error, :entities
+
+    def initialize(success, error = nil, entities = [])
+      @error = error
+      @success = @error.nil? ? (success == 1) : false
+      @entities = entities
+    end
+
+    def was_successful?
+      @success
+    end
+
+    def empty?
+      @entities.empty?
+    end
+
+    ##
+    # TODO: DOCUMENT!
+    def resolve_all_properties
+      entities.each {|e| e.resolve_properties }
+    end
+
+    ##
+    # TODO: DOCUMENT!
+    def resolve_all_claims
+      entities.each {|e| e.resolve_claims }
+    end
+
+  end
+
+end

data/lib/gimme_wikidata/enums.rb

@@ -0,0 +1,61 @@
+require 'ruby-enum'
+
+module GimmeWikidata
+
+  ##
+  # The languages possible when communicating with the Wikidata API
+  #
+  # Currently available values:
+  #  - Languages::ENGLISH = 'en'
+  #  - Languages::GERMAN = 'de'
+  #
+  # See: https://www.wikidata.org/w/api.php?action=help&modules=wbgetentities for a list of the supported languages
+  class Languages
+    include Ruby::Enum
+
+    define :ENGLISH, 'en'
+    define :GERMAN, 'de'
+    # etc...
+
+  end
+
+  ##
+  # Models an enum of the 'action' parameters in Wikidata API calls
+  #
+  # Available values:
+  # - +Actions::SEARCH+ = 'wbsearchentities
+  # - +Actions::GET_ENTITIES+ = 'wbgetentities
+  # See https://www.wikidata.org/w/api.php?action=help&modules=main
+  class Actions
+    include Ruby::Enum
+
+    define :SEARCH, 'wbsearchentities'
+    define :GET_ENTITIES, 'wbgetentities'
+  end
+
+  ##
+  # Models an enum of the 'props' arguments in wbgetentity calls.
+  #
+  # Available values
+  # - +Props::INFO+ = 'info'
+  # - +Props::SITELINKS+ = 'sitelinks'
+  # - +Props::ALIASES+ = 'aliases'
+  # - +Props::LABELS+ = 'labels'
+  # - +Props::DESCRIPTIONS+ = 'descriptions'
+  # - +Props::CLAIMS+ = 'claims'
+  # - +Props::DATATYPE+ = 'datatype'
+  #
+  # See https://www.wikidata.org/w/api.php?action=help&modules=wbgetentities
+  class Props
+    include Ruby::Enum
+
+    define :INFO, 'info'
+    define :SITELINKS, 'sitelinks'
+    define :ALIASES, 'aliases'
+    define :LABELS, 'labels'
+    define :DESCRIPTIONS, 'descriptions'
+    define :CLAIMS, 'claims'
+    define :DATATYPE, 'datatype'
+  end
+
+end

data/lib/gimme_wikidata/extensions.rb

@@ -0,0 +1,42 @@
+##
+# Class to deep symbolize keys in a Hash and Array of Hashes
+#
+# Code written by EdvardM at: http://apidock.com/rails/Hash/deep_symbolize_keys
+module SymbolizeHelper
+  extend self
+
+  def symbolize_recursive(hash)
+    {}.tap do |h|
+      hash.each { |key, value| h[key.to_sym] = transform(value) }
+    end
+  end
+
+  private
+
+  def transform(thing)
+    case thing
+    when Hash; symbolize_recursive(thing)
+    when Array; thing.map { |v| transform(v) }
+    else; thing
+    end
+  end
+
+  refine Hash do
+    def deep_symbolize_keys
+      SymbolizeHelper.symbolize_recursive(self)
+    end
+  end
+end
+
+class Array
+  ##
+  # Merges an Array of Hases into a single Hash, keeping duplicate values in an Array
+  #
+  # Raises StandardError if the Array is not an Array of Hashes, exclusively
+  def merge_hashes
+    raise StandardError.new "Array is not an Array of Hashes" unless self.all? {|e| e.is_a? Hash}
+    self.each_with_object({}) do |el, h|
+      el.each { |k, v| h[k] = h[k] ? [*h[k]] << v : v }
+    end
+  end
+end

data/lib/gimme_wikidata/item.rb

@@ -0,0 +1,18 @@
+require 'gimme_wikidata/entity'
+
+module GimmeWikidata
+
+  ##
+  # Models an Item on Wikidata, which is a "real-world object, concept, event"
+  #
+  # Please see: https://www.wikidata.org/wiki/Wikidata:Glossary#Entities.2C_items.2C_properties_and_queries for more details
+  class Item < Entity
+
+    def initialize(id, label = nil, description = nil, aliases = nil, claims = [])
+      throw ArgumentError.new "Invalid Wikidata Item id: #{id}" unless GimmeWikidata.valid_id?(id, [:item])
+      super(id, label, description, aliases, claims)
+    end
+
+  end
+
+end

data/lib/gimme_wikidata/parser.rb

@@ -0,0 +1,211 @@
+require 'gimme_wikidata/wikidata_api'
+require 'carbon_date'
+
+module GimmeWikidata
+
+  ##
+  # Responsible for parsing the JSON data returned from the Wikidata API.
+  #
+  #
+  # Parses responses to the following Wikidata API calls:
+  # - +wbsearchentities+
+  # - +wbgetentities+
+  class Parser
+
+    include CarbonDate
+
+    ##
+    # Parses the results from a search query (wbsearchentities)
+    # * *Args*    :
+    #   - +response+ -> hash form of the response from the Wikidata API
+    # * *Returns* :
+    #   - A +Search+ object representing and containing the +SearchResults+ found
+    # Returns a Search object representing a collection of the SearchResults found
+    def self.parse_search_response(response)
+      search = Search.new(
+        response.fetch(:success, false),
+        response.fetch(:error, nil),
+        response.fetch(:searchinfo, {}).fetch(:search, nil))
+      return search unless search.was_successful?
+      raise ArgumentError, 'response did not seem to be a response from a search query' if response[:searchinfo].nil?
+      response[:search].each do |r|
+        search.results << SearchResult.new(r[:id], r[:label], r[:description])
+      end
+      return search
+    end
+
+    ##
+    # Parses the results from a get entities query (wbgetentities)
+    #
+    # * *Args*    :
+    #   - +response+ -> hash form of the response from the Wikidata API
+    # * *Returns* :
+    #   - A +EntityResult+ object representing the +Entites+ fetched
+    def self.parse_entity_response(response)
+      entity_result = EntityResult.new(response.fetch(:success, false), response.fetch(:error, nil))
+      return entity_result unless entity_result.was_successful?
+      response[:entities].each do |key, value|
+        entity_result.entities << parse_entity(value)
+      end
+      return entity_result
+    end
+
+    ##
+    # Parses a single entity as part of a get entities query (wbgetentities)
+    #
+    # * *Args*    :
+    #   - +e+ -> hash form of entity from the Wikidata API
+    # * *Returns* :
+    #   - Either an +Item+ object or a +Property+ object representing the passed +Entity+
+    def self.parse_entity(e)
+      lang = WikidataAPI.get_language.to_sym
+      # Parse the fingerprint (id, label and description)
+      id = e.fetch(:id, nil)
+      label = e.fetch(:labels, {}).fetch(lang, {}).fetch(:value, nil)
+      description = e.fetch(:descriptions, {}).fetch(lang, {}).fetch(:value, nil)
+      # Parse aliases, if any:
+      aliases_hash = e.fetch(:aliases, {}).fetch(lang, nil)
+      aliases = aliases_hash.nil? ? [] :aliases_hash.map { |a| a.fetch(:value, nil) }
+      # Parse claims, if any
+      claims = e.fetch(:claims, nil).nil? ? [] : parse_claims(e.fetch(:claims, nil))
+      # Create an Item or a Property
+      case e.fetch(:type, nil)
+      when 'item'
+        return Item.new(id, label, description, aliases, claims)
+      when 'property'
+        return Property.new(id, label, description, aliases, claims)
+      else
+        return nil
+      end
+    end
+
+    ##
+    # TODO: DOCUMENT
+    def self.parse_claims(c)
+      claims = []
+      c.values.flatten.each do |snak|
+        claims << parse_snak(snak[:mainsnak])
+      end
+      return claims
+    end
+
+    ##
+    # TODO: DOCUMENT
+    #
+    # A List of all Wikidata datatypes: https://www.wikidata.org/wiki/Special:ListDatatypes
+    def self.parse_snak(s)
+      property = Property.new(s[:property])
+      raw_value = s.fetch(:datavalue, {}).fetch(:value, nil)
+
+      #TODO: Figure out why raw_value has some strange keys. Example => ':"key"'
+      #TODO: Correct for the very strange keys in raw_value
+
+      ##
+      # TODO: Use meta-programming and public_send() to DRY this code up:
+      value, value_type =
+      case s[:datatype]
+      when 'wikibase-item' then parse_snak_wikibase_item(raw_value)
+      when 'external-id' then parse_snak_external_id(raw_value)
+      when 'time' then parse_snak_time(raw_value)
+      when 'commonsMedia' then parse_snak_commons_media(raw_value)
+      when 'monolingualtext' then parse_snak_monolingual_text(raw_value)
+      when 'string' then parse_snak_string(raw_value)
+      when 'url' then parse_snak_url(raw_value)
+      when 'globe-coordinate' then parse_snak_gps_coordinate(raw_value)
+      when 'quantity' then parse_snak_quantity(raw_value)
+      when 'math' then parse_snak_math(raw_value)
+      else
+        raise StandardError.new "Unsupported Wikidata snak datatype: #{s[:datatype]}"
+      end
+
+      Claim.new(property, value, value_type)
+    end
+
+    # Individual Snak Parsing
+
+    ##
+    # TODO: DOCUMENT FULLY
+    #
+    # A List of all Wikidata datatypes: https://www.wikidata.org/wiki/Special:ListDatatypes
+    def self.parse_snak_wikibase_item(raw_value)
+      id = raw_value.fetch(:"numeric-id", nil)
+      type = raw_value.fetch(:"entity-type", nil)
+      case type
+      when 'item'
+        return Item.new("Q#{id}"), :item
+      when 'property'
+        return Property.new("P#{id}"), :property
+      else
+        raise StandardError.new "Unknown Wikibase item type #{raw_value[:entity_type]}"
+      end
+    end
+
+    def self.parse_snak_external_id(raw_value)
+      #TODO: Extract the authoritative source
+      return raw_value, :external_id
+    end
+
+    ##
+    # Parses a Wikidata Time value
+    #
+    # Times on Wikidata are stored as timestamp in the ISO8601 standard.  Use the CarbonDate gem (https://github.com/bradleymarques/carbon_date) to interpret these
+    #
+    # Params:
+    # - +raw_value+: a hash with the keys:
+    #   - +:time+: The time in the ISO8601 standard
+    #   - +:timezone+: currently unused
+    #   - +:before+: currently unused
+    #   - +:after+: currently unused
+    #   - +:precision+: an integer value (0..14)
+    #   - +:calendarmodel+: currently unused
+    #
+    # Example +raw_value+:
+    # {"time": "+1940-10-10T00:00:00Z", "timezone": 0, "before": 0, "after": 0, "precision": 11, "calendarmodel": "http://www.wikidata.org/entity/Q1985727"}
+    #
+    # Returns:
+    # - [CarbonDate::Date object, :carbon_date]
+    def self.parse_snak_time(raw_value)
+      time = raw_value.fetch(:time, nil)
+      precision = raw_value.fetch(:precision, nil)
+      return CarbonDate::Date.iso8601(time, precision), :carbon_date
+    end
+
+    def self.parse_snak_commons_media(raw_value)
+      file_name = raw_value.to_s.gsub(' ', '_')
+      full_url = "https://commons.wikimedia.org/wiki/File:" + file_name
+      return full_url, :media
+    end
+
+    def self.parse_snak_monolingual_text(raw_value)
+      return raw_value.fetch(:text, nil), :text
+    end
+
+    def self.parse_snak_string(raw_value)
+      return raw_value, :text
+    end
+
+    def self.parse_snak_url(raw_value)
+      return raw_value, :url
+    end
+
+    def self.parse_snak_gps_coordinate(raw_value)
+      return {latitude: raw_value[:latitude], longitude: raw_value[:longitude]}, :gps_coordinates
+    end
+
+    def self.parse_snak_quantity(raw_value)
+      quantity = {
+        amount: raw_value.fetch(:amount, 0).to_f,
+        upper_bound: raw_value.fetch(:upperBound, 0).to_f,
+        lower_bound: raw_value.fetch(:lowerBound, 0).to_f,
+        unit: raw_value.fetch(:unit, 0).to_f
+      }
+      return quantity, :quantity
+    end
+
+    def self.parse_snak_math(raw_value)
+      return raw_value, :math
+    end
+
+  end
+
+end