infoboxer 0.1.0

data/lib/infoboxer/media_wiki.rb

@@ -0,0 +1,162 @@
+# encoding: utf-8
+require 'rest-client'
+require 'json'
+require 'addressable/uri'
+
+require_relative 'media_wiki/traits'
+require_relative 'media_wiki/page'
+
+module Infoboxer
+  # MediaWiki client class.
+  #
+  # Usage:
+  #
+  # ```ruby
+  # client = Infoboxer::MediaWiki.new('http://en.wikipedia.org/w/api.php', user_agent: 'My Own Project')
+  # page = client.get('Argentina')
+  # ```
+  #
+  # Consider using shortcuts like {Infoboxer.wiki}, {Infoboxer.wikipedia},
+  # {Infoboxer.wp} and so on instead of direct instation of this class
+  # (although you can if you want to!)
+  #
+  class MediaWiki
+    # Default Infoboxer User-Agent header.
+    #
+    # You can set yours as an option to {Infoboxer.wiki} and its shortcuts,
+    # or to {#initialize}
+    UA = "Infoboxer/#{Infoboxer::VERSION} (https://github.com/molybdenum-99/infoboxer; zverok.offline@gmail.com)"
+
+    class << self
+      # User agent getter/setter.
+      #
+      # Default value is {UA}.
+      #
+      # You can also use per-instance option, see {#initialize}
+      attr_accessor :user_agent
+    end
+
+    attr_reader :api_base_url
+
+    # Creating new MediaWiki client. {Infoboxer.wiki} provides shortcut
+    # for it, as well as shortcuts for some well-known wikis, like
+    # {Infoboxer.wikipedia}.
+    #
+    # @param api_base_url URL of `api.php` file in your MediaWiki
+    #   installation. Typically, its `<domain>/w/api.php`, but can vary
+    #   in different wikis.
+    # @param options Only one option is currently supported:
+    #   * `:user_agent` (also aliased as `:ua`) -- custom User-Agent header.
+    def initialize(api_base_url, options = {})
+      @api_base_url = Addressable::URI.parse(api_base_url)
+      @resource = RestClient::Resource.new(api_base_url, headers: headers(options))
+    end
+
+    # Receive "raw" data from Wikipedia (without parsing or wrapping in
+    # classes).
+    #
+    # @return [Array<Hash>]
+    def raw(*titles)
+      postprocess @resource.get(
+        params: DEFAULT_PARAMS.merge(titles: titles.join('|'))
+      )
+    end
+
+    # Receive list of parsed wikipedia pages for list of titles provided.
+    # All pages are received with single query to MediaWiki API.
+    #
+    # **NB**: currently, if you are requesting more than 50 titles at
+    # once (MediaWiki limitation for single request), Infoboxer will
+    # **not** try to get other pages with subsequent queries. This will
+    # be fixed in future.
+    #
+    # @return [Tree::Nodes<Page>] array of parsed pages. Notes:
+    #   * if you call `get` with only one title, one page will be
+    #     returned instead of an array
+    #   * if some of pages are not in wiki, they will not be returned,
+    #     therefore resulting array can be shorter than titles array;
+    #     you can always check `pages.map(&:title)` to see what you've
+    #     really received; this approach allows you to write absent-minded
+    #     code like this:
+    #
+    #     ```ruby
+    #     Infoboxer.wp.get('Argentina', 'Chile', 'Something non-existing').
+    #        infobox.fetch('some value')
+    #     ```
+    #     and obtain meaningful results instead of NoMethodError or some
+    #     NotFound.
+    #
+    def get(*titles)
+      pages = raw(*titles).reject{|raw| raw[:content].nil?}.
+        map{|raw|
+          traits = Traits.get(@api_base_url.host, extract_traits(raw))
+          
+          Page.new(self,
+            Parser.paragraphs(raw[:content], traits),
+            raw.merge(traits: traits))
+        }
+      titles.count == 1 ? pages.first : Tree::Nodes[*pages]
+    end
+
+    private
+
+    # @private
+    PROP = [
+      'revisions',    # to extract content of the page
+      'info',         # to extract page canonical url
+      'categories',   # to extract default category prefix
+      'images'        # to extract default media prefix
+    ].join('|')
+
+    # @private
+    DEFAULT_PARAMS = {
+      action:    :query,
+      format:    :json,
+      redirects: true,
+
+      prop:      PROP,
+      rvprop:    :content,
+      inprop:    :url,
+    }
+
+    def headers(options)
+      {'User-Agent' => options[:user_agent] || options[:ua] || self.class.user_agent || UA}
+    end
+
+    def extract_traits(raw)
+      raw.select{|k, v| [:file_prefix, :category_prefix].include?(k)}
+    end
+
+    def guess_traits(pages)
+      categories = pages.map{|p| p['categories']}.compact.flatten
+      images = pages.map{|p| p['images']}.compact.flatten
+      {
+        file_prefix: images.map{|i| i['title'].scan(/^([^:]+):/)}.flatten.uniq,
+        category_prefix: categories.map{|i| i['title'].scan(/^([^:]+):/)}.flatten.uniq,
+      }
+    end
+
+    def postprocess(response)
+      pages = JSON.parse(response)['query']['pages']
+      traits = guess_traits(pages.values)
+      
+      pages.map{|id, data|
+        if id.to_i < 0
+          {
+            title: data['title'],
+            content: nil,
+            not_found: true
+          }
+        else
+          {
+            title: data['title'],
+            content: data['revisions'].first['*'],
+            url: data['fullurl'],
+          }.merge(traits)
+        end
+      }
+    rescue JSON::ParserError
+      fail RuntimeError, "Not a JSON response, seems there's not a MediaWiki API: #{@api_base_url}"
+    end
+  end
+end

data/lib/infoboxer/media_wiki/page.rb

@@ -0,0 +1,38 @@
+# encoding: utf-8
+module Infoboxer
+  class MediaWiki
+    # A descendant of {Tree::Document Document}, representing page,
+    # received from {MediaWiki} client.
+    #
+    # Alongside with document tree structure, knows document's title as
+    # represented by MediaWiki and human (non-API) URL.
+    class Page < Tree::Document
+      def initialize(client, children, raw)
+        @client = client
+        super(children, raw)
+      end
+
+      # Instance of {MediaWiki} which this page was received from
+      # @return {MediaWiki}
+      attr_reader :client
+
+      # @!attribute [r] title
+      #   Page title.
+      #   @return [String]
+
+      # @!attribute [r] url
+      #   Page friendly URL.
+      #   @return [String]
+
+      def_readers :title, :url, :traits
+
+      private
+
+      PARAMS_TO_INSPECT = [:url, :title, :domain]
+
+      def show_params
+        super(params.select{|k, v| PARAMS_TO_INSPECT.include?(k)})
+      end
+    end
+  end
+end

data/lib/infoboxer/media_wiki/traits.rb

@@ -0,0 +1,60 @@
+# encoding: utf-8
+module Infoboxer
+  class MediaWiki
+    class Traits
+      class << self
+        def templates(&definition)
+          @templates ||= Templates::Set.new
+
+          return @templates unless definition
+          
+          @templates.define(&definition)
+        end
+
+        # NB: explicitly store all domains in base Traits class
+        def domain(d)
+          Traits.domains.key?(d) and
+            fail(ArgumentError, "Domain binding redefinition: #{Traits.domains[d]}")
+
+          Traits.domains[d] = self
+        end
+
+        def get(domain, options = {})
+          cls = Traits.domains[domain]
+          cls ? cls.new(options) : Traits.new(options)
+        end
+
+        def domains
+          @domains ||= {}
+        end
+
+        def for(domain, &block)
+          Class.new(self, &block).domain(domain)
+        end
+
+        alias_method :default, :new
+      end
+
+      DEFAULTS = {
+        file_prefix: 'File',
+        category_prefix: 'Category'
+      }
+
+      def initialize(options = {})
+        @options = options
+        @file_prefix = [DEFAULTS[:file_prefix], options.delete(:file_prefix)].
+          flatten.compact.uniq
+        @category_prefix = [DEFAULTS[:category_prefix], options.delete(:category_prefix)].
+          flatten.compact.uniq
+      end
+
+      attr_reader :file_prefix, :category_prefix
+
+      #attr_accessor :re
+
+      def templates
+        self.class.templates
+      end
+    end
+  end
+end

data/lib/infoboxer/navigation.rb

@@ -0,0 +1,84 @@
+# encoding: utf-8
+module Infoboxer
+  # Navigation is one of the things Infoboxer is proud about. It tries
+  # to be logical, unobtrusive and compact.
+  #
+  # There's several levels of navigation:
+  # * simple tree navigation;
+  # * navigational shortcuts;
+  # * logical structure navigation (sections).
+  #
+  # ## Simple tree navigation
+  #
+  # It's somewhat similar to XPath/CSS selectors you'll use to navigate
+  # through HTML DOM. It is represented (and documented) in {Lookup::Node}
+  # module. To show you the taste of it:
+  #
+  # ```ruby
+  # document.
+  #  lookup(:Wikilink, text: /Chile/).
+  #  lookup_parents(:Table){|t| t.params[:class] == 'wikitable'}.
+  #  lookup_children(size: 3)
+  # ```
+  #
+  # ## Navigational shortcuts
+  #
+  # There is nothing too complicated, just pretty shortcuts over `lookup_*`
+  # methods, so, you can write just
+  #
+  # ```ruby
+  # document.paragraphs.last.wikilinks('Category')
+  # ```
+  # ...instead of
+  # ```ruby
+  # document.lookup(:Paragraph).last.lookup(:Wikilink, namespace: 'Category')
+  # ```
+  # ...and so on.
+  #
+  # Look into {Shortcuts::Node} documentation for list of shortcuts.
+  #
+  # ## Logical structure navigation
+  #
+  # MediaWiki page structure is flat, like HTML's (there's just sequence
+  # of headings and paragraphs). Though, for most tasks of information
+  # extraction it is usefult to think of page as a structure of nested
+  # sections. {Sections} module provides such ability. It treats document
+  # as an {Sections::Container#intro intro} and set of subsequent
+  # {Sections::Section section}s of same level, which, in turn, have inside
+  # they own intro and sections. Also, each node has
+  # {Sections::Node#in_sections #in_sections} method, returning all sections
+  # in which it is nested.
+  #
+  # The code with sections can feel like this:
+  #
+  # ```ruby
+  # page.sections('Culture' => 'Music').tables
+  # # or like this
+  # page.wikilinks.select{|link| link.in_sections.first.heading.text.include?('Culture')}
+  # ```
+  #
+  # See {Sections::Container} for downwards section navigation, and
+  # {Sections::Node} for upwards.
+  #
+  module Navigation
+    %w[lookup shortcuts sections].each do |nav|
+      require_relative "navigation/#{nav}"
+    end
+    
+    class Tree::Node
+      include Navigation::Lookup::Node
+      include Navigation::Shortcuts::Node
+      include Navigation::Sections::Node
+    end
+
+    class Tree::Nodes
+      include Navigation::Lookup::Nodes
+      include Navigation::Shortcuts::Nodes
+      include Navigation::Sections::Nodes
+    end
+
+    class Tree::Document
+      include Navigation::Sections::Container
+    end
+  end
+end

data/lib/infoboxer/navigation/lookup.rb

@@ -0,0 +1,216 @@
+# encoding: utf-8
+require_relative 'selector'
+
+module Infoboxer
+  module Navigation
+    # See {Lookup::Node Lookup::Node} for everything!
+    module Lookup
+      # `Lookup::Node` module provides methods for navigating through
+      # page tree in XPath-like manner.
+      #
+      # What you need to know about it:
+      #
+      # ## Selectors
+      #
+      # Each `lookup_*` method (and others similar) receive
+      # _list of selectors_. Examples of acceptable selectors:
+      #
+      # ```ruby
+      # # 1. Node class:
+      # document.lookup(Bold) # all Bolds
+      #
+      # # 2. Class symbol
+      # document.lookup(:Bold)
+      # # same as above, useful if you don't want to include Infoboxer::Tree
+      # # in all of your code or write things like lookup(Infoboxer::Tree::Bold)
+      #
+      # # 3. Getter/pattern:
+      # document.lookup(text: /something/)
+      # # finds all nodes where result of getter matches pattern
+      #
+      # # Checks against patterns are performed with `===`, so you can
+      # # use regexps to find by text, or ranges to find by number, like
+      # document.lookup(:Heading, level: (3..4))
+      #
+      # # Nodes where method is not defined are ignored, so you can
+      # # rewrite above example as just
+      # document.lookup(level: 3..4)
+      # # ...and receive meaningful result without any NoMethodError
+      #
+      # # 4. Check symbol
+      # document.lookup(:bold?)
+      # # finds all nodes for which `:bold?` is defined and returns
+      # # truthy value;
+      #
+      # # 5. Code block
+      # document.lookup{|node| node.params.has_key?(:class)}
+      # ```
+      #
+      # You also can use any of those method without **any** selector,
+      # thus receiving ALL parents, ALL children, ALL siblings and so on.
+      #
+      # ## Chainable navigation
+      #
+      # Each `lookup_*` method returns an instance of {Tree::Nodes} class,
+      # which behaves like an Array, but also defines similar set of
+      # `lookup_*` methods, so, you can brainlessly do the things like
+      #
+      # ```ruby
+      # document.
+      #   lookup(:Paragraph){|p| p.text.length > 100}.
+      #   lookup(:Wikilink, text: /^List of/).
+      #   select(&:bold?)
+      # ```
+      #
+      # ## Underscored methods
+      #
+      # For all methods of this module you can notice "underscored" version
+      # (`lookup_children` vs `_lookup_children` and so on). Basically,
+      # underscored versions accept instance of {Lookup::Selector}, which
+      # is already preprocessed version of all selectors. It is kinda
+      # internal thing, though can be useful if you store selectors in
+      # variables -- it is easier to have and use just one instance of
+      # Selector, than list of arguments and blocks.
+      #
+      module Node
+        # @!method matches?(*selectors, &block)
+        #   Checks if current node matches selectors.
+
+        # @!method lookup(*selectors, &block)
+        #   Selects matching nodes from entire subtree inside current node.
+
+        # @!method lookup_children(*selectors, &block)
+        #   Selects nodes only from this node's direct children.
+
+        # @!method lookup_parents(*selectors, &block)
+        #   Selects matching nodes of this node's parents chain, up to
+        #   entire {Tree::Document Document}.
+
+        # @!method lookup_siblings(*selectors, &block)
+        #   Selects matching nodes from current node's siblings.
+
+        # @!method lookup_next_siblings(*selectors, &block)
+        #   Selects matching nodes from current node's siblings, which
+        #   are below current node in parents children list.
+
+        # @!method lookup_prev_siblings(*selectors, &block)
+        #   Selects matching nodes from current node's siblings, which
+        #   are above current node in parents children list.
+
+        # Underscored version of {#matches?}
+        def _matches?(selector)
+          selector.matches?(self)
+        end
+
+        # Underscored version of {#lookup}
+        def _lookup(selector)
+          Tree::Nodes[_matches?(selector) ? self : nil, *children._lookup(selector)].
+            flatten.compact
+        end
+
+        # Underscored version of {#lookup_children}
+        def _lookup_children(selector)
+          @children._find(selector)
+        end
+
+        # Underscored version of {#lookup_parents}
+        def _lookup_parents(selector)
+          case
+          when !parent
+            Tree::Nodes[]
+          when parent._matches?(selector)
+            Tree::Nodes[parent, *parent._lookup_parents(selector)]
+          else
+            parent._lookup_parents(selector)
+          end
+        end
+
+        # Underscored version of {#lookup_siblings}
+        def _lookup_siblings(selector)
+          siblings._find(selector)
+        end
+
+        # Underscored version of {#lookup_prev_siblings}
+        def _lookup_prev_siblings(selector)
+          prev_siblings._find(selector)
+        end
+
+        # Underscored version of {#lookup_next_siblings}
+        def _lookup_next_siblings(selector)
+          next_siblings._find(selector)
+        end
+        
+        [:matches?,
+          :lookup, :lookup_children, :lookup_parents,
+          :lookup_siblings,
+          :lookup_next_siblings, :lookup_prev_siblings
+        ].map{|sym| [sym, :"_#{sym}"]}.each do |sym, underscored|
+
+          define_method(sym){|*args, &block|
+            send(underscored, Selector.new(*args, &block))
+          }
+        end
+
+        # Checks if node has any parent matching selectors.
+        def has_parent?(*selectors, &block)
+          !lookup_parents(*selectors, &block).empty?
+        end
+      end
+
+      # This module provides implementations for all `lookup_*` methods
+      # of {Lookup::Node} for be used on nodes list. Note, that all
+      # those methods return _flat_ list of results (so, if you have
+      # found several nodes, and then look for their siblings, you should
+      # not expect array of arrays -- just one array of nodes).
+      #
+      # See {Lookup::Node} for detailed lookups and selectors explanation.
+      module Nodes
+        # @!method lookup(*selectors, &block)
+        # @!method lookup_children(*selectors, &block)
+        # @!method lookup_parents(*selectors, &block)
+        # @!method lookup_siblings(*selectors, &block)
+        # @!method lookup_next_siblings(*selectors, &block)
+        # @!method lookup_prev_siblings(*selectors, &block)
+
+        # @!method _lookup(selector)
+        # @!method _lookup_children(selector)
+        # @!method _lookup_parents(selector)
+        # @!method _lookup_siblings(selector)
+        # @!method _lookup_next_siblings(selector)
+        # @!method _lookup_prev_siblings(selector)
+
+        # Underscored version of {#find}.
+        def _find(selector)
+          select{|n| n._matches?(selector)}
+        end
+
+        # Selects nodes of current list (and only it, no children checks),
+        # which are matching selectors.
+        def find(*selectors, &block)
+          _find(Selector.new(*selectors, &block))
+        end
+
+        [
+          :_lookup, :_lookup_children, :_lookup_parents,
+          :_lookup_siblings, :_lookup_prev_siblings, :_lookup_next_siblings
+        ].each do |sym|
+          define_method(sym){|*args|
+            make_nodes map{|n| n.send(sym, *args)}
+          }
+        end
+
+        # not delegate, but redefine: Selector should be constructed only once
+        [
+          :lookup, :lookup_children, :lookup_parents,
+          :lookup_siblings,
+          :lookup_next_siblings, :lookup_prev_siblings
+        ].map{|sym| [sym, :"_#{sym}"]}.each do |sym, underscored|
+
+          define_method(sym){|*args, &block|
+            send(underscored, Selector.new(*args, &block))
+          }
+        end
+      end
+    end
+  end
+end