reality 0.0.2 → 0.0.3

data/reality.gemspec

@@ -1,7 +1,9 @@
+require_relative 'lib/reality/version'
+
 Gem::Specification.new do |s|
   s.name     = 'reality'
-  s.version  = '0.0.2'
-  s.authors  = ['Victor Shepelev']
+  s.version  = Reality::VERSION
+  s.authors  = ['Victor Shepelev and contributors']
   s.email    = 'zverok.offline@gmail.com'
   s.homepage = 'https://github.com/molybdenum-99/reality'
 
@@ -28,7 +30,14 @@ Gem::Specification.new do |s|
   s.bindir = 'bin'
   s.executables << 'reality'
 
-  s.add_dependency 'infoboxer', '>= 0.2.2'
+  s.add_dependency 'infoboxer', '>= 0.2.3'
+  s.add_dependency 'hashie'
+  s.add_dependency 'open-weather'
+  s.add_dependency 'timezone'
+  s.add_dependency 'geokit'
+  s.add_dependency 'tzinfo'
+  s.add_dependency 'ruby-sun-times'
+  s.add_dependency 'time_boots'
 
   s.add_development_dependency 'rspec', '~> 3'
   s.add_development_dependency 'rspec-its', '~> 1'

data/script/extract_wikidata_properties.rb

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+require 'bundler/setup'
+require 'open-uri'
+require 'progress_bar/core_ext/enumerable_with_progress'
+require 'nokogiri'
+require 'json'
+require_relative 'lib/nokogiri_more'
+
+start = Nokogiri::HTML(open('https://www.wikidata.org/wiki/Wikidata:List_of_properties'))
+res = start.
+      search('th:contains("By number")').first.parent.search('td > a').
+      with_progress.map{|a|
+        name = a.text
+        data = Nokogiri::HTML(open('https://www.wikidata.org' + a.href).read).
+                search('tr').
+                map{|tr| tr.search('td').map(&:text)}.
+                map{|tds| [tds[0], tds[1]]}.
+                map(&:reverse).
+                reject{|id, name| id.nil? || name.nil?}.
+                to_h
+      }.inject(&:merge)
+
+File.write 'data/wikidata-predicates.json', res.to_json      

data/script/lib/nokogiri_more.rb

@@ -0,0 +1,175 @@
+require 'naught'
+require 'addressable/uri'
+
+module Nokogiri
+    module More
+        module NodeOnlyFor
+            def only_for!(selector)
+                matches?(selector) or fail(ArgumentError, "Doesn't works for nodes other than '#{selector}'")
+            end
+        end
+        
+        module NodeHref
+            include NodeOnlyFor
+            
+            def href
+                #only_for!('a[href]')
+                document.absolute(self['href'])
+            end
+        end
+        
+        module DocumentURI
+            def uri
+                url ? Addressable::URI.parse(url) : nil
+            end
+            
+            def absolute(link)  
+                if uri
+                    (uri + link).to_s
+                else
+                    Addressable::URI.parse(link).to_s # double check it's really a link
+                end
+            end
+        end
+        
+        module NodeChildrenGroups
+            def children_groups(*selectors)
+                groups = []
+                flat = children.select{|node| selectors.any?{|s| node.matches?(s)}}
+                while !flat.empty?
+                    groups << make_group(flat, selectors)
+                end
+                groups
+            end
+            
+            include NodeOnlyFor
+            
+            def each_term
+                only_for!('dl')
+                children_groups('dt', 'dd')
+            end
+            
+            private
+            
+            def make_group(flat, selectors)
+                sel = selectors.dup
+                group = [[]]
+                while !flat.empty?
+                    if flat.first.matches?(sel.first)
+                        group.last << flat.shift
+                    elsif sel.size > 1 && flat.first.matches?(sel[1])
+                        sel.shift
+                        group << []
+                        group.last << flat.shift
+                    else
+                        break
+                    end
+                end
+                group
+            end
+        end
+        
+        module NodeText
+            def text_
+                text.strip 
+            end
+        end
+        
+        NodeNaught = Naught.build do |config|
+            config.black_hole
+            
+            def tap # so you can just at?(selector).tap{|node| - and never be here, if it's not found
+                self
+            end
+        end
+        
+        class NodeNotFound < RuntimeError
+        end
+        
+        module NodeBangMethods
+            def at!(selector)
+                bang!(at(selector), selector)
+            end
+            
+            def at_css!(selector)
+                bang!(at_css(selector), selector)
+            end
+
+            def at_xpath!(selector)
+                bang!(at_xpath(selector), selector)
+            end
+            
+            def find_child!(selector)
+                bang!(find_child(selector), selector)
+            end
+            
+            private
+            
+            def bang!(node, selector)
+                if node
+                    node
+                else
+                    no_node!(selector)
+                end
+            end
+            
+            def no_node!(selector)
+                #case Nokogiri::More::Config.bang_mode
+                #when :fail
+                    fail NodeNotFound, "#{name} have no node at #{selector}"
+                #when :naught
+                    #NodeNaught.new
+                #when :log
+                    #NodeNaught.new
+                #end
+            end
+        end
+        
+        module NodeQuestMethods
+            def at?(selector)
+                at(selector) || NodeNaught.new
+            end
+
+            def at_css?(selector)
+                at_css(selector) || NodeNaught.new
+            end
+
+            def at_xpath?(selector)
+                at_xpath(selector) || NodeNaught.new
+            end
+            
+            def find_child?(selector)
+                find_child(selector) || NodeNaught.new
+            end
+        end
+        
+        module NodeFindChildren
+            def find_child(selector)
+                children.filter(selector).first
+            end
+
+            def find_children(selector)
+                children.filter(selector)
+            end
+        end
+    end
+    
+    # now let's do evil
+    class ::Class
+        public :include
+    end
+    
+    Nokogiri::XML::Document.include More::DocumentURI
+    Nokogiri::XML::Node.include More::NodeText
+    Nokogiri::XML::Node.include More::NodeHref
+    Nokogiri::XML::Node.include More::NodeChildrenGroups
+    
+    Nokogiri::XML::Node.include More::NodeBangMethods
+    Nokogiri::XML::Node.include More::NodeQuestMethods
+    Nokogiri::XML::Node.include More::NodeFindChildren
+
+    Nokogiri::XML::NodeSet.include More::NodeBangMethods
+    Nokogiri::XML::NodeSet.include More::NodeQuestMethods
+    Nokogiri::XML::NodeSet.include More::NodeFindChildren
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: reality
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
-- Victor Shepelev
+- Victor Shepelev and contributors
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-03 00:00:00.000000000 Z
+date: 2016-03-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: infoboxer
@@ -16,14 +16,112 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.2.2
+        version: 0.2.3
+- !ruby/object:Gem::Dependency
+  name: hashie
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: open-weather
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timezone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: geokit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tzinfo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-sun-times
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: time_boots
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -90,26 +188,58 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".dokaz"
+- LICENSE.txt
 - README.md
 - bin/reality
+- config/demo.yml
 - data/countries.yaml
 - data/country_lists.txt
 - data/country_orgs.yaml
 - data/infoboxes.txt
 - data/infoboxes_freq.txt
 - data/infoboxes_freq_sorted.txt
-- examples/all_countries.rb
+- data/wikidata-predicates.json
+- data/wikidata-predicates.yaml
 - lib/reality.rb
-- lib/reality/country.rb
+- lib/reality/config.rb
+- lib/reality/definitions/dictionaries.rb
+- lib/reality/definitions/helpers.rb
+- lib/reality/definitions/wikidata.rb
+- lib/reality/definitions/wikipedia_character.rb
+- lib/reality/definitions/wikipedia_city.rb
+- lib/reality/definitions/wikipedia_continent.rb
+- lib/reality/definitions/wikipedia_country.rb
+- lib/reality/definitions/wikipedia_musical_artist.rb
+- lib/reality/definitions/wikipedia_person.rb
+- lib/reality/entity.rb
+- lib/reality/entity/coercion.rb
+- lib/reality/entity/wikidata_predicates.rb
+- lib/reality/entity/wikipedia_type.rb
+- lib/reality/extras/geonames.rb
+- lib/reality/extras/open_weather_map.rb
+- lib/reality/geo.rb
 - lib/reality/infoboxer_templates.rb
+- lib/reality/list.rb
 - lib/reality/measure.rb
 - lib/reality/measure/unit.rb
+- lib/reality/methods.rb
+- lib/reality/pretty_inspect.rb
+- lib/reality/refinements.rb
+- lib/reality/shortcuts.rb
+- lib/reality/tz_offset.rb
+- lib/reality/util/formatters.rb
+- lib/reality/util/parsers.rb
+- lib/reality/version.rb
+- lib/reality/wikidata.rb
 - reality.gemspec
 - script/extract_all_infoboxes.rb
 - script/extract_countries.rb
 - script/extract_country_categories.rb
 - script/extract_infobox_frequency.rb
+- script/extract_wikidata_properties.rb
 - script/lib/faraday_naive_cache.rb
+- script/lib/nokogiri_more.rb
 - script/out/categories.txt
 homepage: https://github.com/molybdenum-99/reality
 licenses:

data/examples/all_countries.rb

@@ -1,16 +0,0 @@
-#!/usr/bin/env ruby
-require 'rubygems'
-require 'bundler/setup'
-$:.unshift 'lib'
-require 'reality'
-require 'fileutils'
-require 'yaml'
-
-FileUtils.mkdir_p 'examples/output'
-
-start = Time.now
-
-File.write 'examples/output/countries.yaml',
-  Reality.countries.to_a.sort_by(&:name).map(&:to_h).to_yaml
-
-puts "Finished in %i seconds" % (Time.now - start)

data/lib/reality/country.rb

@@ -1,283 +0,0 @@
-# NB: all of this is early drafts, so may look naive and sub-optimal.
-# Just stay tuned!
-
-module Reality
-  class Country
-    class List
-      def initialize(*names)
-        @names = names
-      end
-
-      def count
-        @names.count
-      end
-
-      def first(n = nil)
-        res = get(*@names.first(n || 1))
-        n ? res : res.first
-      end
-
-      def last(n = nil)
-        res = get(*@names.last(n || 1))
-        n ? res : res.first
-      end
-
-      def sample(n = nil)
-        res = get(*@names.sample(n || 1))
-        n ? res : res.first
-      end
-
-      def each(&block)
-        @pages = get(*@names)
-        @pages.each(&block)
-      end
-
-      include Enumerable
-
-      def to_a
-        get(*@names)
-      end
-
-      def where(**filters)
-        names = @names & Reality::Country.
-          by_continents.
-          select{|k, v| v == filters[:continent]}.
-          map(&:first)
-          
-        self.class.new(*names)
-      end
-
-      private
-
-      def get(*names)
-        Reality.wp.get(*names).map{|page| Country.new(page)}
-      end
-    end
-    
-    def initialize(page)
-      @page = page
-    end
-
-    def name
-      page.title
-    end
-
-    def long_name
-      infobox.fetch('conventional_long_name').text.strip
-    end
-
-    def capital
-      infobox.fetch('capital').lookup(:Wikilink).first
-    end
-
-    def languages
-      [
-        ['Official', infobox_links('official_languages')],
-        [infobox.fetch('languages_type').text.sub(/ languages?$/, ''), infobox_links('languages')]
-      ].reject{|k, v| k.empty? || v.empty?}.to_h
-    end
-
-    def tld
-      tlds.first
-    end
-
-    def tlds
-      infobox_links('cctld').map(&:link)
-    end
-
-    def calling_code
-      infobox.fetch('calling_code').text.strip
-    end
-
-    def utc_offset
-      infobox.fetch('utc_offset').text.sub('−', '-').to_i
-    end
-
-    def currency
-      currencies.first
-    end
-
-    def currencies
-      infobox_links('currency').reject{|l| l.link == 'ISO 4217'}
-    end
-
-    def area
-      Reality::Measure(infobox.fetch('area_km2').text.gsub(',', '').to_i, 'km²')
-    end
-
-    def population
-      val = %w[population_estimate population_census].map{|var|
-        infobox.fetch(var).text.strip
-      }.reject(&:empty?).first
-      val && Reality::Measure(parse_maybe_scaled(val), 'person')
-    end
-
-    def gdp_ppp
-      val = infobox.fetch('GDP_PPP').text.strip.sub(/^((Int|US)?\$|USD)/, '')
-      val.empty? ? nil : Reality::Measure(parse_scaled(val), '$')
-    end
-
-    def gdp_nominal
-      val = infobox.fetch('GDP_nominal').text.strip.sub(/^((Int|US)?\$|USD)/, '')
-      val.empty? ? nil : Reality::Measure(parse_scaled(val), '$')
-    end
-
-    alias_method :gdp, :gdp_nominal
-
-    def leaders
-      titles = infobox.fetch(/^leader_title\d/).map(&:text_)
-      names = infobox.fetch(/^leader_name\d/).map{|v| v.lookup(:Wikilink).first}
-      titles.zip(names).to_h
-    end
-
-    def continent
-      self.class.by_continents[page.title]
-    end
-
-    def organizations
-      organizations_list.map{|o| o[:name]}
-    end
-
-    def member_of?(org)
-      organizations_list.any?{|o| o[:name] == org || o[:abbr] == org}
-    end
-
-    def to_s
-      name
-    end
-
-    def inspect
-      "#<#{self.class}(#{name})>"
-    end
-
-    PROPERTIES = %i[
-                    continent name long_name
-                    tld tlds calling_code utc_offset
-                    capital languages currency
-                    leaders area population
-                    gdp_ppp gdp_nominal
-                  ]
-
-    def to_h
-      #p self
-      PROPERTIES.
-        map{|prop| [prop, to_simple_type(send(prop))]  }.
-        #reject{|prop, val| !val || val.respond_to?(:empty?) && val.empty?}.
-        to_h
-    end
-
-    class << self
-      def by_continents
-        @by_continents ||= Reality.wp.
-          get('List of countries by continent').
-          sections.first.
-          sections.map{|s|
-            continent = s.heading.text_
-            s.tables.first.
-              lookup(:Wikilink, :bold?).map(&:link).
-              map{|country| [country, continent]}
-          }.flatten(1).
-          to_h
-      end
-
-      def organizations
-        @organizations ||= YAML.load(File.read(File.expand_path('../../../data/country_orgs.yaml', __FILE__)))
-      end
-    end
-
-    private
-
-    attr_reader :page
-
-    def infobox
-      page.infobox
-    end
-
-    def organizations_list
-      catnames = page.categories.map(&:name)
-      self.class.organizations.select{|o| catnames.include?(o[:category])}
-    end
-
-    def infobox_links(varname)
-      src = infobox.fetch(varname)
-      if tmpl = src.lookup(:Template, name: /list$/).first
-        # values could be both inside and outside list, see India's cctld value
-        src = Infoboxer::Tree::Nodes[src, tmpl.variables] 
-      end
-      src.lookup(:Wikilink).uniq
-    end
-
-    # See "Short scale": https://en.wikipedia.org/wiki/Long_and_short_scales#Comparison
-    SCALES = {
-      'million'     => 1_000_000,
-      'billion'     => 1_000_000_000,
-      'trillion'    => 1_000_000_000_000,
-      'quadrillion' => 1_000_000_000_000_000,
-      'quintillion' => 1_000_000_000_000_000_000,
-      'sextillion'  => 1_000_000_000_000_000_000_000,
-      'septillion'  => 1_000_000_000_000_000_000_000_000,
-    }
-    SCALES_REGEXP = Regexp.union(*SCALES.keys)
-
-    def parse_scaled(str)
-      match, amount, scale = */^([0-9.,]+)[[:space:]]*(#{SCALES_REGEXP})/.match(str)
-      match or
-        fail(ArgumentError, "Unparseable scaled value #{str} for #{self}")
-
-      (amount.gsub(/[,]/, '').to_f * fetch_scale(scale)).to_i
-    end
-
-    def parse_maybe_scaled(str)
-      match, amount, scale = */^([0-9.,]+)[[:space:]]*(#{SCALES_REGEXP})?/.match(str)
-      match or
-        fail(ArgumentError, "Unparseable scaled value #{str} for #{self}")
-
-      if scale
-        (amount.gsub(/[,]/, '').to_f * fetch_scale(scale)).to_i
-      else
-        amount.gsub(/[,]/, '').to_i
-      end
-    end
-
-    def fetch_scale(str)
-      _, res = SCALES.detect{|key, val| str.start_with?(key)}
-
-      res or fail("Scale not found: #{str} for #{self}")
-    end
-
-    def to_simple_type(val)
-      case val
-      when nil, Numeric, String, Symbol
-        val
-      when Array
-        val.map{|v| to_simple_type(v)}
-      when Hash
-        val.map{|k, v| [to_simple_type(k), to_simple_type(v)]}.to_h
-      when Infoboxer::Tree::Wikilink
-        val.link
-      when Infoboxer::Tree::Node
-        val.text_
-      when Reality::Measure
-        val.amount
-      else
-        fail ArgumentError, "Non-coercible value #{val.class}"
-      end
-    end
-  end
-
-  def Reality.country(name)
-    page = wp.get(name) or return nil
-    # FIXME: not very reliable, as some fictional countries, aliances
-    #   and country groups also have this infobox. Or maybe it is acceptable?..
-    page.templates(name: 'Infobox country').empty? ? nil : Country.new(page)
-  end
-
-  def Reality.countries(*names)
-    names = Country.by_continents.keys.sort if names.empty?
-    Country::List.new(*names)
-  end
-
-  def Reality.wp
-    @wp ||= Infoboxer.wp # while Infoboxer recreates wp for each request
-  end
-end