infoboxer 0.1.0

data/lib/infoboxer/parser/table.rb

@@ -0,0 +1,132 @@
+# encoding: utf-8
+module Infoboxer
+  class Parser
+    # http://en.wikipedia.org/wiki/Help:Table
+    module Table
+      include Tree
+      
+      def table
+         @context.current =~ /^\s*{\|/ or
+          @context.fail!('Something went wrong: trying to parse not a table')
+
+        prms = table_params
+        table = Tree::Table.new(Nodes[], prms)
+
+        @context.next!
+
+        loop do
+          table_next_line(table) or break
+          @context.next!
+        end
+
+        # FIXME: not the most elegant way, huh?
+        table.children.reject!{|r| r.children.empty?}
+
+        table
+      end
+
+      def table_params
+        @context.skip(/\s*{\|/)
+        parse_params(@context.rest)
+      end
+
+      def table_next_line(table)
+        case @context.current
+        when /^\s*\|}(.*)$/                 # table end
+          @context.scan(/^\s*\|}/)
+          return false # should not continue
+
+        when /^\s*!/                        # heading (th) in a row
+          table_cells(table, TableHeading)
+
+        when /^\s*\|\+/                     # caption
+          table_caption(table)
+          
+        when /^\s*\|-(.*)$/                 # row start
+          table_row(table, $1)
+
+        when /^\s*\|/                       # cell in row
+          table_cells(table)
+
+        when /^\s*{{/                       # template can be at row level
+          table_template(table)
+
+        when nil
+          @context.fail!("End of input before table ended!")
+
+        else
+          table_cell_cont(table)
+        end
+        true # should continue parsing
+      end
+
+      def table_row(table, param_str)
+        table.push_children(TableRow.new(Nodes[], parse_params(param_str)))
+      end
+
+      def table_caption(table)
+        @context.skip(/^\s*\|\+\s*/)
+
+        children = inline(/^\s*([|!]|{\|)/)
+        @context.prev! # compensate next! which will be done in table()
+        table.push_children(TableCaption.new(children.strip))
+      end
+
+      def table_cells(table, cell_class = TableCell)
+        table.push_children(TableRow.new()) unless table.children.last.is_a?(TableRow)
+        row = table.children.last
+
+        @context.skip(/\s*[!|]\s*/)
+        guarded_loop do
+          if @context.check(/[^|{|\[]+\|([^\|]|$)/)
+            params = parse_params(@context.scan_until(/\|/))
+          else
+            params = {}
+          end
+          content = short_inline(/(\|\||!!)/)
+          row.push_children(cell_class.new(content, params))
+          break if @context.eol?
+        end
+      end
+
+      def table_template(table)
+        contents = paragraph(/^\s*([|!]|{\|)/).to_templates?
+        
+        if (row = table.children.last).is_a?(TableRow)
+          if (cell = row.children.last).is_a?(BaseCell)
+            cell.push_children(*contents)
+          else
+            row.push_children(*contents)
+          end
+        else
+          table.push_children(*contents)
+        end
+      end
+
+      # On-the-fly TableCaption creation handles (real life) case, when
+      # table has "HTML caption":
+      #   {|
+      #   <caption>....</caption>
+      #
+      # Solution is NOT elegant or semantically "right", yet it works.
+      # Somehow.
+      #
+      def table_cell_cont(table)
+        container = case (last = table.children.last)
+        when TableRow
+          cell = last.children.last
+          cell.is_a?(BaseCell) ? cell : TableCaption.new
+        when TableCaption
+          last
+        when nil
+          TableCaption.new
+        else
+          @context.fail!("Multiline cell inside #{last}")
+        end
+
+        container.push_children(paragraph(/^\s*([|!]|{\|)/))
+        table.push_children(container) unless container.parent
+      end
+    end
+  end
+end

data/lib/infoboxer/parser/template.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+module Infoboxer
+  class Parser
+    module Template
+      include Tree
+      
+      # NB: here we are not distingish templates like {{Infobox|variable}}
+      # and "magic words" like {{formatnum:123}}
+      # Just calling all of them "templates". This behaviour will change
+      # in future, I presume
+      # More about magic words: https://www.mediawiki.org/wiki/Help:Magic_words
+      def template
+        name = @context.scan_continued_until(/\||:|}}/) or
+          @context.fail!("Template name not found")
+          
+        name.strip!
+        vars = @context.eat_matched?('}}') ? Nodes[] : template_vars
+        @context.traits.templates.find(name).new(name, vars)
+      end
+
+      def template_vars
+        num = 1
+        res = Nodes[]
+        
+        guarded_loop do
+          if @context.check(/\s*([^ =}|]+)\s*=\s*/)
+            name = @context.scan(/\s*([^ =]+)/).strip
+            @context.skip(/\s*=\s*/)
+          else
+            name = num
+          end
+
+          value = long_inline(/\||}}/)
+          unless value.empty? && name.is_a?(Numeric) # it was just empty line otherwise
+            res << Var.new(name.to_s, value)
+          end
+
+          break if @context.eat_matched?('}}')
+          @context.eof? and @context.fail!("Unexpected break of template variables: #{res}")
+
+          num += 1
+        end
+        res
+      end
+    end
+  end
+end

data/lib/infoboxer/parser/util.rb

@@ -0,0 +1,73 @@
+# encoding: utf-8
+module Infoboxer
+  class Parser
+    module Util
+      attr_reader :re
+
+      FORMATTING = /(
+        '{2,5}        |     # bold, italic
+        \[\[          |     # link
+        {{            |     # template
+        \[[a-z]+:\/\/ |     # external link
+        <nowiki[^>]*> |     # reference
+        <ref[^>]*>    |     # nowiki
+        <                   # HTML tag
+      )/x
+
+      INLINE_EOL = %r[(?=   # if we have ahead... (not scanned, just checked
+        </ref>        |     # <ref> closed
+        }}                  # or template closed
+      )]x
+
+
+      def make_regexps
+        {
+          file_prefix: /(#{@context.traits.file_prefix.join('|')}):/,
+          formatting: FORMATTING,
+          inline_until_cache: Hash.new{|h, r|
+            h[r] = Regexp.union(*[r, FORMATTING, /$/].compact.uniq)
+          },
+          short_inline_until_cache: Hash.new{|h, r|
+            h[r] = Regexp.union(*[r, INLINE_EOL, FORMATTING, /$/].compact.uniq)
+          }
+        }
+      end
+
+      def parse_params(str)
+        return {} unless str
+        
+        scan = StringScanner.new(str)
+        params = {}
+        loop do
+          scan.skip(/\s*/)
+          name = scan.scan(/[^ \t=]+/) or break
+          scan.skip(/\s*/)
+          if scan.peek(1) == '='
+            scan.skip(/=\s*/)
+            q = scan.scan(/['"]/)
+            if q
+              value = scan.scan_until(/#{q}/).sub(q, '')
+            else
+              value = scan.scan_until(/\s|$/)
+            end
+            params[name.to_sym] = value
+          else
+            params[name.to_sym] = name
+          end
+        end
+        params
+      end
+
+      def guarded_loop
+        loop do
+          pos_before = @context.lineno, @context.colno
+          yield
+          pos_after = @context.lineno, @context.colno
+          pos_after == pos_before and
+            @context.fail!("Infinite loop on position #{pos_after.last}")
+        end
+      end
+
+    end
+  end
+end

data/lib/infoboxer/templates.rb

@@ -0,0 +1,10 @@
+module Infoboxer
+  # Templates are cool, powerful and undocumented. Sorry :(
+  #
+  # I do my best.
+  module Templates
+    %w[base set].each do |tmpl|
+      require_relative "templates/#{tmpl}"
+    end
+  end
+end

data/lib/infoboxer/templates/base.rb

@@ -0,0 +1,82 @@
+module Infoboxer
+  module Templates
+    class Base < Tree::Template
+      include Tree
+      
+      class << self
+        attr_accessor :template_name, :template_options
+
+        def inspect
+          template_name ? "Infoboxer::Templates::#{clean_name}" : super
+        end
+
+        def clean_name
+          template_name ? "Template[#{template_name}]" : 'Template'
+        end
+      end
+
+      def unnamed_variables
+        variables.select{|v| v.name =~ /^\d+$/}
+      end
+
+      def fetch(*patterns)
+        Nodes[*patterns.map{|p| variables.find(name: p)}.flatten]
+      end
+
+      def fetch_hash(*patterns)
+        fetch(*patterns).map{|v| [v.name, v]}.to_h
+      end
+
+      def fetch_date(*patterns)
+        components = fetch(*patterns)
+        components.pop while components.last.nil? && !components.empty?
+        
+        if components.empty?
+          nil
+        else
+          Date.new(*components.map{|v| v.to_s.to_i})
+        end
+      end
+
+      def ==(other)
+        other.kind_of?(Tree::Template) && _eq(other)
+      end
+
+      protected
+
+      def clean_class
+        if self.class.template_name == name
+          self.class.clean_name
+        else
+          super
+        end
+      end
+    end
+
+    # Renders all of its unnamed variables as space-separated text
+    # Also allows in-template navigation
+    class Show < Base
+      alias_method :children, :unnamed_variables
+
+      protected
+
+      def children_separator
+        ' '
+      end
+    end
+
+    class Replace < Base
+      def replace
+        fail(NotImplementedError, "Descendants should define :replace")
+      end
+
+      def text
+        replace
+      end
+    end
+
+    class Literal < Base
+      alias_method :text, :name
+    end
+  end
+end

data/lib/infoboxer/templates/set.rb

@@ -0,0 +1,72 @@
+# encoding: utf-8
+module Infoboxer
+  module Templates
+    class Set
+      def initialize(&definitions)
+        @templates = []
+        define(&definitions) if definitions
+      end
+      
+      def find(name)
+        _, template = @templates.detect{|m, t| m === name.downcase}
+        template || Base
+      end
+
+      def define(&definitions)
+        instance_eval(&definitions)
+      end
+
+      def clear
+        @templates.clear
+      end
+
+      private
+
+      def template(name, options = {}, &definition)
+        setup_class(name, Base, options, &definition)
+      end
+
+      def replace(*replacements)
+        case
+        when replacements.count == 2 && replacements.all?{|r| r.is_a?(String)}
+          name, what = *replacements
+          setup_class(name, Replace) do
+            define_method(:replace) do
+              what
+            end
+          end
+        when replacements.count == 1 && replacements.first.is_a?(Hash)
+          replacements.first.each do |name, what|
+            replace(name, what)
+          end
+        else
+          fail(ArgumentError, "Can't call :replace with #{replacements.join(', ')}")
+        end
+      end
+
+      def show(*names)
+        names.each do |name|
+          setup_class(name, Show)
+        end
+      end
+
+      def literal(*names)
+        names.each do |name|
+          setup_class(name, Literal)
+        end
+      end
+
+      def setup_class(name, base_class, options = {}, &definition)
+        match = options.fetch(:match, name.downcase)
+        base = options.fetch(:base, base_class)
+        base = self.find(base) if base.is_a?(String)
+
+        Class.new(base, &definition).tap{|cls|
+          cls.template_name = name
+          cls.template_options = options
+          @templates.unshift [match, cls]
+        }
+      end
+    end
+  end
+end

data/lib/infoboxer/tree.rb

@@ -0,0 +1,70 @@
+# encoding: utf-8
+module Infoboxer
+  # Infoboxer provides you with tree structure of the Wikipedia page,
+  # which you can introspect and navigate with ease. This tree structure
+  # tries to be simple, close to Wikipedia source and logical.
+  #
+  # You can always inspect entire page tree yourself:
+  #
+  # ```ruby
+  # page = Infoboxer.wp.get('Argentina')
+  # puts page.to_tree
+  # ```
+  #
+  # ## Inspecting and understanding single node
+  #
+  # Each tree node is descendant of {Tree::Node}, so you should look
+  # at this class to understand what you can do.
+  #
+  # Alongside with basic methods, defined in Node class, some useful
+  # utility methods are defined in subclasses.
+  #
+  # Here's full list of subclasses, representing real nodes, with their
+  # respective roles:
+  #
+  # * inline markup: {Text}, {Bold}, {Italic}, {BoldItalic}, {Wikilink},
+  #   {ExternalLink}, {Image};
+  # * embedded HTML: {HTMLTag}, {HTMLOpeningTag}, {HTMLClosingTag};
+  # * paragraph-level nodes: {Heading}, {Paragraph}, {Pre}, {HR};
+  # * lists: {OrderedList}, {UnorderedList}, {DefinitionList}, {ListItem},
+  #   {DTerm}, {DDefinition};
+  # * tables: {Table}, {TableCaption}, {TableRow}, {TableHeading}, {TableCell};
+  # * special elements: {Template}, {Ref}.
+  #
+  # ## Tree navigation
+  #
+  # {Tree::Node} class has a standard list of methods for traversing tree
+  # upwards, downwards and sideways: `children`, `parent`, `siblings`,
+  # `index`. Read through class documentation for their detailed
+  # descriptions.
+  #
+  # {Navigation} module contains more advanced navigational functionality,
+  # like XPath-like selectors, friendly shortcuts, breakup of document
+  # into logical "sections" and so on.
+  #
+  # Most of navigational and other Node's methods return {Nodes} type,
+  # which is an `Array` descendant with additional functionality.
+  # 
+  # ## Complex data extraction
+  #
+  # Most of uniform, machine-extractable data in Wikipedia is stored in
+  # templates and tables. There's entire {Templates} module, which is
+  # documented explaining what you can do about Wikipedia templates, how
+  # to understand them and use information. Also, you can look at {Table}
+  # class, which for now is not that powerful, yet allows you to extract
+  # some columns and rows.
+  #
+  # Also, consider that WIKIpedia is maid of WIKIlinks, and {Wikilink#follow}
+  # (as well as {Nodes#follow} for multiple links at once) is you good friend.
+  #
+  module Tree
+    require_relative 'tree/node'
+    require_relative 'tree/nodes'
+
+    %w[text compound inline
+      image html paragraphs list template table ref
+      document].each do |type|
+      require_relative "tree/#{type}"
+    end
+  end
+end