infoboxer 0.1.0

data/examples/to_text.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require_relative '../lib/infoboxer'
+
+document = Infoboxer::Parser.parse(File.read('examples/pages/argentina.wiki'))
+
+FileUtils.mkdir_p 'examples/output'
+
+File.write('examples/output/argentina-text.txt', document.to_text)

data/examples/tree.rb

@@ -0,0 +1,8 @@
+# encoding: utf-8
+require_relative '../lib/infoboxer'
+
+document = Infoboxer::Parser.document(File.read('examples/pages/argentina.wiki'))
+
+FileUtils.mkdir_p 'examples/output'
+
+File.write('examples/output/argentina-tree.txt', document.to_tree)

data/infoboxer.gemspec

@@ -0,0 +1,43 @@
+require './lib/infoboxer/version'
+
+Gem::Specification.new do |s|
+  s.name     = 'infoboxer'
+  s.version  = Infoboxer::VERSION
+  s.authors  = ['Victor Shepelev']
+  s.email    = 'zverok.offline@gmail.com'
+  s.homepage = 'https://github.com/zverok/infoboxer'
+
+  s.summary = 'MediaWiki client and parser, targeting information extraction.'
+  s.description = <<-EOF
+    Infoboxer is library targeting use of Wikipedia (or any other
+    MediaWiki-based wiki) as a rich powerful data source.
+  EOF
+  s.licenses = ['MIT']
+
+  s.files = `git ls-files`.split($RS).reject do |file|
+    file =~ /^(?:
+    spec\/.*
+    |Gemfile
+    |Rakefile
+    |\.rspec
+    |\.gitignore
+    |\.rubocop.yml
+    |\.travis.yml
+    )$/x
+  end
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'htmlentities'
+  s.add_dependency 'procme'
+  s.add_dependency 'rest-client'
+  s.add_dependency 'addressable'
+  s.add_dependency 'terminal-table'
+  s.add_dependency 'backports'
+
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec', '~> 3'
+  s.add_development_dependency 'rspec-its', '~> 1'
+  s.add_development_dependency 'ruby-prof'
+  s.add_development_dependency 'vcr'
+  s.add_development_dependency 'webmock'
+end

data/lib/infoboxer.rb

@@ -0,0 +1,196 @@
+# encoding: utf-8
+require 'procme'
+require 'backports/2.1.0/array/to_h'
+#require 'backports/2.2.0/object/itself' Y U NO???
+
+# Main client module for entire infoboxer functionality. If you're lucky,
+# there's no other classes/modules you need to instantiate or call
+# directly. You just do:
+#
+# ```ruby
+# Infoboxer.wp.get('List of radio telescopes')
+# # or 
+# Infoboxer.wikiquote.get('Vonnegut')
+# ```
+# ...and have fully navigable Wiki information.
+#
+# Please read [wiki](http://github.com/molybdenum-99/infoboxer/wiki)
+# for extensive [showcases](https://github.com/molybdenum-99/infoboxer/wiki/Showcase)
+# and usage recommendations.
+#
+# Here's main components list, which also can serve as a TOC for
+# Infoboxer's functionality (we suggest to read their docs in this order):
+#
+# * {Tree} -- nodes, of which Wikipedia AST is consisting; you'll be
+#   interested in basic {Tree::Node} functionality, as well as node
+#   classes list (which is useful for navigation);
+# * {Navigation} -- how to navigate the tree you have, basic way
+#   (children, parents, siblings) and hi-level shortcuts way (like
+#   all unnumbered list items in second level-3 section);
+# * {Templates} -- the most advanced data extraction from wikipedia definitely
+#   needs your undestanding of this (rather complicated) topic.
+#
+# You also may be interested in (though may be never need to use them directly):
+#
+# * {MediaWiki} client class;
+# * {Parser} -- which, you know, parses.
+#
+module Infoboxer
+  private # hiding constants from YARD
+
+  WIKIA_API_URL = 'http://%s.wikia.com/api.php'
+
+  WIKIMEDIA_PROJECTS = {
+    wikipedia: 'wikipedia.org',
+    wikivoyage: 'wikivoyage.org',
+    wikiquote: 'wikiquote.org',
+    wiktionary: 'wiktionary.org',
+    wikibooks: 'wikibooks.org',
+    wikinews: 'wikinews.org',
+    wikiversity: 'wikiversity.org',
+    wikisource: 'wikisource.org'
+  }
+
+  WIKIMEDIA_COMMONS = {
+    commons: 'commons.wikimedia.org',
+    species: 'species.wikimedia.org',
+  }
+
+  class << self
+
+    # Default method for creating MediaWiki API client.
+    #
+    # @param api_url should be URL of api.php for your MediaWiki
+    # @param options list of options.
+    #   The only recognized option for now, though, is
+    #   * `:user_agent` (also aliased as `:ua`) -- custom User-Agent header.
+    # @return [MediaWiki] an instance of API client, which you can
+    #   further use like this:
+    #
+    #   ```ruby
+    #   Infoboxer.wiki('some_url').get('Some page title')
+    #   ```
+    def wiki(api_url, options = {})
+      MediaWiki.new(api_url, options || {})
+    end
+
+    # @!method wikipedia(lang = 'en', options = {})
+    # Shortcut for creating Wikipedia client.
+    #
+    # @param lang two-character code for language version
+    # @param options (see #wiki for list of options)
+    # @return [MediaWiki]
+    
+    # @!method commons(options = {})
+    # Shortcut for creating [WikiMedia Commons](https://commons.wikimedia.org/) client.
+    #
+    # @param options (see #wiki for list of options)
+    # @return [MediaWiki]
+    
+    # @!method wikibooks(lang = 'en', options = {})
+    # Shortcut for creating [Wikibooks](https://en.wikibooks.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method wikiquote(lang = 'en', options = {})
+    # Shortcut for creating [Wikiquote](https://en.wikiquote.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method wikiversity(lang = 'en', options = {})
+    # Shortcut for creating [Wikiversity](https://en.wikiversity.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method wikisource(lang = 'en', options = {})
+    # Shortcut for creating [Wikisource](https://en.wikisource.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method wikivoyage(lang = 'en', options = {})
+    # Shortcut for creating [Wikivoyage](http://wikivoyage.org) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method wikinews(lang = 'en', options = {})
+    # Shortcut for creating [Wikinews](https://en.wikinews.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    # @!method species(options = {})
+    # Shortcut for creating [Wikispecies](https://species.wikimedia.org/) client.
+    #
+    # @param options (see #wiki for list of options)
+    # @return [MediaWiki]
+    
+    # @!method wiktionary(lang = 'en', options = {})
+    # Shortcut for creating [Wiktionary](https://en.wiktionary.org/) client.
+    # See {wikipedia} for params explanation.
+    # @return [MediaWiki]
+
+    WIKIMEDIA_PROJECTS.each do |name, domain|
+      define_method name do |lang = 'en', options = {}|
+        if lang.is_a?(Hash)
+          lang, options = 'en', lang
+        end
+
+        wiki("http://#{lang}.#{domain}/w/api.php", options)
+      end
+    end
+
+    alias_method :wp, :wikipedia
+
+    WIKIMEDIA_COMMONS.each do |name, domain|
+      define_method name do |options = {}|
+        wiki("http://#{domain}/w/api.php", options)
+      end
+    end
+
+    # Performs request to wikia.com wikis.
+    #
+    # @overload wikia(*domains)
+    #   @param *domains list of domains to merge, like this:
+    #   
+    #     ```ruby
+    #     Infoboxer.wikia('tardis') # looks at tardis.wikia.com
+    #     Infoboxer.wikia('tardis', 'ru') # looks in Russian version, ru.tardis.wikia.com
+    #     ```
+    #     If you are surprised by "reversing" list of subdomains, think of
+    #     it as of chain of refinements (looking in "tardis" wiki, its "ru" 
+    #     version, specifically).
+    #
+    # @overload wikia(*domains, options)
+    #   @param *domains same as above
+    #   @param options just last of params, if it is hash
+    #     (see {wiki} for list of options)
+    #
+    # @return [MediaWiki]
+    def wikia(*domains)
+      options = domains.last.is_a?(Hash) ? domains.pop : {}
+      wiki(WIKIA_API_URL % domains.reverse.join('.'), options)
+    end
+
+    # Sets user agent string globally. Default user agent is 
+    # {MediaWiki::UA}.
+    #
+    # User agent can also be rewriten as an option to {wiki} method (and
+    # its shortcuts like {wikipedia}), or by using {MediaWiki#initialize}
+    # explicitly.
+    #
+    def user_agent=(ua)
+      MediaWiki.user_agent = ua
+    end
+  end
+end
+
+require_relative 'infoboxer/version'
+require_relative 'infoboxer/core_ext'
+
+require_relative 'infoboxer/tree'
+require_relative 'infoboxer/parser'
+require_relative 'infoboxer/navigation'
+require_relative 'infoboxer/templates'
+
+require_relative 'infoboxer/media_wiki'
+
+require_relative 'infoboxer/definitions/en.wikipedia.org'

data/lib/infoboxer/core_ext.rb

@@ -0,0 +1,10 @@
+# encoding: utf-8
+# @private
+class Object
+  # Unfortunately, not in backports gem still :(
+  if RUBY_VERSION < '2.2.0'
+    def itself
+      self
+    end
+  end
+end

data/lib/infoboxer/definitions/en.wikipedia.org.rb

@@ -0,0 +1,355 @@
+# encoding: utf-8
+module Infoboxer
+  MediaWiki::Traits.for('en.wikipedia.org') do
+    templates do
+      # https://en.wikipedia.org/wiki/Category:Wikipedia_character-substitution_templates
+      # ---------------------------------------------------------------------------------
+      # Extracted semi-automatically
+      # TODO: fully automatical extraction
+      literal(
+        '&',
+        ';',
+        '=',
+        '?',
+        '—',
+        '1/2',
+        '1/3',
+        '1/4',
+        '2/3',
+        '3/4',
+      )
+      replace(
+        '!!' => '||',
+        '!(' => '[',
+        '!((' => '[[',
+        '!-' => '|-',
+        '!:' => ':',
+        '&' => '&',
+        "'" => " '",
+        "''" => '″',
+        "'s" => "'‍s",
+        '(' => '{',
+        '((' => '{{',
+        '(((' => '{{{',
+        ')' => '}',
+        ')!' => ']',
+        '))' => '}}',
+        '))!' => ']]',
+        ')))' => '}}}',
+        'Asterisk' => '*',
+        'Colon' => ':',
+        'Em dash' => '—',
+        'Gc' => "†",
+        'Ibeam' => 'I',
+        'Long dash' => ' ——— ',
+        'Nbhyph' => '‑',
+        'Number sign' => '#',
+        'Shy' => '­', # soft hyphen
+        'Single space' => "' ",
+        'Space single' => " '",
+        'Spaced ndash' => ' – ',
+        'Square bracket close' => ']',
+        'Square bracket open' => '[',
+        'Zwsp' => '',
+        '\\' => ' / ',
+        '`' => "'",
+        '·' => ' · ',
+        '‘' => '‘',
+        '•' => ' • ',
+      )
+
+      # https://en.wikipedia.org/wiki/Category:Line-handling_templates
+      # ------------------------------------------------------------------
+      replace(
+        '-' => "\n",
+        'Break' => "\n", # FIXME: in fact, break has optional parameter "how many breaks"
+        'Crlf' => "\n",  # FIXME: in fact, alias for break, should have DSL syntax for it!
+        'Crlf2' => "\n",
+        
+      )
+      show(
+        'Allow wrap',
+        'Nowrap',
+          'j', 'nobr', 'nobreak', # aliases for Nowrap
+        'nowraplinks',
+      )
+      # inflow_template('Normalwraplink') # TODO: tricky
+
+      # https://en.wikipedia.org/wiki/Category:List_formatting_and_function_templates
+      # -----------------------------------------------------------------------------
+      # NB: it's enough for most cases to have all list-representing templates
+      # just navigable inside and rendered as space-separated list of entries
+      show(
+        'Br separated entries',
+        'Bulleted list',
+        'Collapsible list',
+        'Comma separated entries',
+        'Hlist',
+        'Flatlist',
+        'Flowlist',
+        'Pagelist',
+        'Ordered list',
+        'Plainlist',
+        'Space separated entries',
+        'Toolbar',
+      )
+
+      # https://en.wikipedia.org/wiki/Category:Wikipedia_XHTML_tag-replacing_templates
+      # ------------------------------------------------------------------------------
+
+      show(
+        # Font size
+        'Small',
+        'Smaller',
+        'Midsize',
+        'Larger',
+        'Big',
+        'Large',
+        'Huge',
+
+        # Align
+        'left',
+        'Center',
+        'Right',
+
+        # Simple style
+        'Em',
+        'Kbd',
+        'Var',
+        'Varserif',
+        'Samp',
+        'Strikethrough',
+        'Strong',
+        'Sub',
+        'Sup',
+        'Underline',
+
+        # FIXME: should do something wiser
+        'Pre',
+        'Pre2',
+        'Code'
+      )
+
+      template 'Abbr' do
+        def children
+          fetch('1')
+        end
+      end
+      # TODO: has aliases: {{Define}}, {{Explain}}, {{Tooltip}}
+
+      template 'Align' do
+        def children
+          fetch('2')
+        end
+      end
+
+      template 'Dfn' do
+        def children
+          fetch('1')
+        end
+      end
+
+      template 'Resize' do
+        def children
+          unnamed_variables.count < 2 ? fetch('1') : fetch('2')
+        end
+      end
+
+      template 'Font' do
+        def children
+          res = fetch('text')
+          res.empty? ? fetch('1') : res
+        end
+      end
+
+      # https://en.wikipedia.org/wiki/Category:Text_color_templates
+      show(
+        'white', 'silver (color)', 'gray', 'black', 'pink', 'red', 'darkred',
+        'maroon', 'brown', 'orange (color)', 'gold (color)', 'yellow', 'olive',
+        'lime', 'green', 'aqua (color)', 'cyan', 'teal', 'blue', 'navy (color)',
+        'purple', 'fuchsia', 'magenta'
+      )
+      
+      # Some most popular templates, without categorical splitting
+      # https://en.wikipedia.org/wiki/Wikipedia:Database_reports/Templates_transcluded_on_the_most_pages
+      # ------------------------------------------------------------------------------------------------
+      # Currently scanned by eyes up to 250-th line, which is used in 130549 articles, according to the
+      # page - which, though, is dramatically outdated.
+
+      template 'Stub', match: /-stub$/ do
+        def stub?
+          true
+        end
+      end
+
+      template 'Infobox', match: /^Infobox/i do
+        def infobox?
+          true
+        end
+      end
+
+      template 'Coord' do
+        def model
+          @model ||= begin
+            npos = lookup_children(text: /^N|S$/).first.index rescue nil
+            case npos
+            when 1
+              :decimal
+            when 2
+              :min
+            when 3
+              :sec
+            else
+              :decimal_sign
+            end
+          end
+        end
+
+        def lat
+          case model
+          when :decimal
+            '%s°%s′%s' % fetch('1', '2').map(&:text)
+          when :decimal_sign
+            fetch('1').text
+          when :min
+            '%s°%s′%s' % fetch('1', '2', '3').map(&:text)
+          when :sec
+            '%s°%s′%s″%s' % fetch('1', '2', '3', '4').map(&:text)
+          end
+        end
+
+        def lng
+          case model
+          when :decimal, :decimal_sign
+            fetch('1').text
+          when :min
+            '%s°%s′%s' % fetch('1', '2', '3').map(&:text)
+          when :sec
+            '%s°%s′%s″%s' % fetch('1', '2', '3', '4').map(&:text)
+          end
+        end
+      end
+
+      template 'Convert' do
+        def value1
+          fetch('1').text
+        end
+
+        ALLOW_BETWEEN = ['-;', '–',
+          'and', '&', 'and(-)', ', and',
+          'or', ', or',
+          'to', 'to(-)', 'to about',
+          '+/-', '±', '+',
+          'by', 'x', '×', 'x',
+        ]
+
+        def between
+          ALLOW_BETWEEN.include?(fetch('2').text) ? fetch('2').text : nil
+        end
+
+        def value2
+          between ? fetch('3').text : nil
+        end
+
+        def measure_from
+          between ? fetch('4').text : fetch('2').text
+        end
+        
+        def measure_to
+          between ? fetch('5').text : fetch('3').text
+        end
+        
+        def text
+          [value1, between, value2, measure_from].compact.join(' ')
+        end
+      end
+
+      template 'Age' do
+        def from
+          fetch_date('1', '2', '3')
+        end
+
+        def to
+          fetch_date('4', '5', '6') || Date.today
+        end
+
+        def value
+          (to - from).to_i / 365 # FIXME: obviously
+        end
+
+        def text
+          "#{value} years"
+        end
+      end
+
+      template 'Birth date and age' do
+        def date
+          fetch_date('1', '2', '3')
+        end
+
+        def text
+          date.to_s
+        end
+      end
+      # TODO: aliased as bda
+
+      template 'Birth date' do
+        def date
+          fetch_date('1', '2', '3')
+        end
+
+        def text
+          date.to_s
+        end
+      end
+      # TODO: aliased as dob
+
+      template 'Time ago' do
+        def text
+          str = fetch('1').text
+          begin
+            date = Date.parse(str)
+            "#{(Date.today - date).to_i} days ago" # not trying complext time_distance_in_words formatting here
+          rescue ArgumentError
+            str
+          end
+        end
+      end
+
+      template 'Flagcountry' do # very popular instead of country name
+        def children
+          fetch('1')
+        end
+      end
+
+      template 'Flag' do # very popular instead of country name
+        def children
+          fetch('1')
+        end
+      end
+
+      show 'Plural'
+
+      template 'URL' do
+        def children
+          unnamed_variables.count > 1 ? fetch('2') : fetch('1')
+        end
+      end
+
+      # TODO: extremely popular:
+      # Str left - https://en.wikipedia.org/wiki/Category:String_manipulation_templates
+      # Rnd - https://en.wikipedia.org/wiki/Category:Mathematical_function_templates
+
+      # TODO: useful categories
+      # https://en.wikipedia.org/wiki/Category:Date_mathematics_templates
+      # https://en.wikipedia.org/wiki/Category:Mathematical_function_templates
+      # https://en.wikipedia.org/wiki/Category:Wikipedia_formatting_and_function_templates
+      # https://en.wikipedia.org/wiki/Category:Semantic_markup_templates
+      # https://en.wikipedia.org/wiki/Category:Quotation_templates
+      # https://en.wikipedia.org/wiki/Category:Typing-aid_templates
+      # https://en.wikipedia.org/wiki/Category:Inline_spacing_templates
+      # https://en.wikipedia.org/wiki/Category:Sorting_templates
+      # https://en.wikipedia.org/wiki/Wikipedia:Database_reports/Templates_transcluded_on_the_most_pages
+    end
+  end
+end