infoboxer 0.1.0

data/lib/infoboxer/tree/compound.rb

@@ -0,0 +1,81 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    # Base class for all nodes with children.
+    class Compound < Node
+      def initialize(children = Nodes.new, params = {})
+        super(params)
+        @children = Nodes[*children]
+        @children.each{|c| c.parent = self}
+      end
+
+      # List of children
+      #
+      # @return {Nodes}
+      attr_reader :children
+
+      # Index of provided node in children list
+      #
+      # @return [Fixnum] or `nil` if not a child
+      def index_of(child)
+        children.index(child)
+      end
+
+      # Internal, used by {Parser}
+      def push_children(*nodes)
+        nodes.each{|c| c.parent = self}.each do |n|
+          @children << n
+        end
+      end
+
+      # See {Node#text}
+      def text
+        children.map(&:text).join(children_separator)
+      end
+
+      # See {Node#to_tree}
+      def to_tree(level = 0)
+        if children.count == 1 && children.first.is_a?(Text)
+          "#{indent(level)}#{children.first.text} <#{descr}>\n"
+        else
+          "#{indent(level)}<#{descr}>\n" +
+            children.map(&call(to_tree: level+1)).join
+        end
+      end
+
+      # Kinda "private" methods, used by Parser only -------------------
+      
+      # Internal, used by {Parser}
+      def can_merge?(other)
+        false
+      end
+
+      # Internal, used by {Parser}
+      def closed!
+        @closed = true
+      end
+
+      # Internal, used by {Parser}
+      def closed?
+        @closed
+      end
+
+      # Internal, used by {Parser}
+      def empty?
+        children.empty?
+      end
+
+      protected
+
+      def children_separator
+        ''
+      end
+
+      private
+
+      def _eq(other)
+        children == other.children
+      end      
+    end
+  end
+end

data/lib/infoboxer/tree/document.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    # Represents entire document.
+    #
+    # Alongside with standard compound node functionality, is a
+    # {Navigation::Sections::Container}
+    class Document < Compound
+    end
+  end
+end

data/lib/infoboxer/tree/html.rb

@@ -0,0 +1,76 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    module HTMLTagCommons
+      BLOCK_TAGS = %w[div p br] # FIXME: are some other used in WP?
+
+      def text
+        super + (BLOCK_TAGS.include?(tag) ? "\n" : '')
+      end
+    end
+
+    # Represents HTML tag, surrounding some contents.
+    class HTMLTag < Compound
+      def initialize(tag, attrs, children = Nodes.new)
+        super(children, attrs)
+        @tag = tag
+      end
+
+      attr_reader :tag
+      alias_method :attrs, :params
+
+      include HTMLTagCommons
+
+      # Internal, used by {Parser}.
+      def empty?
+        # even empty tag, for ex., <br>, should not be dropped!
+        false
+      end
+      
+      private
+
+      def descr
+        "#{clean_class}:#{tag}(#{show_params})"
+      end
+    end
+
+    # Represents orphan opening HTML tag.
+    #
+    # NB: Infoboxer not tries to parse entire structure of HTML-heavy
+    # MediaWiki articles. So, if you have `<div>` at line 150 and closing
+    # `</div>` at line 875, there would be orphane `HTMLOpeningTag` and
+    # {HTMLClosingTag}. It is not always convenient, but reasonable enough.
+    #
+    class HTMLOpeningTag < Node
+      def initialize(tag, attrs)
+        super(attrs)
+        @tag = tag
+      end
+      
+      attr_reader :tag
+      alias_method :attrs, :params
+
+      include HTMLTagCommons
+      
+      private
+
+      def descr
+        "#{clean_class}:#{tag}(#{show_params})"
+      end
+    end
+
+    # Represents orphan closing HTML tag. See {HTMLOpeningTag} for
+    # explanation.
+    class HTMLClosingTag < Node
+      def initialize(tag)
+        @tag = tag
+      end
+
+      attr_reader :tag
+
+      def descr
+        "#{clean_class}:#{tag}"
+      end
+    end
+  end
+end

data/lib/infoboxer/tree/image.rb

@@ -0,0 +1,53 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    # Represents image (or other media file).
+    #
+    # See [Wikipedia Tutorial](https://en.wikipedia.org/wiki/Wikipedia:Extended_image_syntax)
+    # for explanation of attributes.
+    class Image < Node
+      def initialize(path, params = {})
+        @caption = params.delete(:caption)
+        super({path: path}.merge(params))
+      end
+
+      # Image caption. Can have (sometimes many) other nodes inside.
+      #
+      # @return [Nodes]
+      attr_reader :caption
+
+      # @!attribute [r] path 
+      # @!attribute [r] type
+      # @!attribute [r] location 
+      # @!attribute [r] alignment
+      # @!attribute [r] link
+      # @!attribute [r] alt 
+
+      def_readers :path, :type,
+        :location, :alignment, :link,
+        :alt
+
+      def border?
+        !params[:border].to_s.empty?
+      end
+
+      def width
+        params[:width].to_i
+      end
+
+      def height
+        params[:height].to_i
+      end
+
+      def to_tree(level = 0)
+        super(level) +
+          if caption && !caption.empty?
+            indent(level+1) + "caption:\n" +
+              caption.map(&call(to_tree: level+2)).join
+          else
+            ''
+          end
+      end
+    end
+  end
+end

data/lib/infoboxer/tree/inline.rb

@@ -0,0 +1,39 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    # Represents italic text.
+    class Italic < Compound
+    end
+
+    # Represents bold text.
+    class Bold < Compound
+    end
+
+    # Represents bold italic text (and no, it's not a comb of bold+italic,
+    # from Wikipedia's markup point of view).
+    class BoldItalic < Compound
+    end
+
+    # Base class for internal/external links,
+    class Link < Compound
+      def initialize(link, label = nil)
+        super(label || Nodes.new([Text.new(link)]), link: link)
+      end
+
+      #@!attribute [r] link
+
+      def_readers :link
+    end
+
+    # External link. Has other nodes as a contents, and, err, link (url).
+    class ExternalLink < Link
+    
+      #@!attribute [r] url
+      #  synonym for `#link`
+      
+      alias_method :url, :link
+    end
+  end
+end
+
+require_relative 'wikilink'

data/lib/infoboxer/tree/list.rb

@@ -0,0 +1,160 @@
+# encoding: utf-8
+module Infoboxer
+  module Tree
+    # Represents item of ordered or unordered list. 
+    class ListItem < BaseParagraph
+      # Internal, used by {Parser}
+      def can_merge?(other)
+        other.class == self.class &&
+          other.children.first.kind_of?(List)
+      end
+
+      # Internal, used by {Parser}
+      def merge!(other)
+        ochildren = other.children.dup
+        if children.last && children.last.can_merge?(ochildren.first)
+          children.last.merge!(ochildren.shift)
+        end
+        push_children(*ochildren)
+      end
+
+      def text
+        make_marker + if children.last.is_a?(List)
+          children[0..-2].map(&:text).join + "\n" + children.last.text
+        else
+          children.map(&:text).join + "\n"
+        end
+      end
+
+      private
+
+      def make_marker
+        parent ? parent.make_marker(self) : '* '
+      end
+    end
+
+    # "Imaginary" node, grouping {ListItem}s of same level and type.
+    #
+    # Base for concrete {OrderedList}, {UnorderedList} and {DefinitionList}.
+    #
+    # NB: Nested lists are represented by structures like:
+    #
+    # ```
+    # <OrderedList>
+    #  <ListItem>
+    #  <ListItem>
+    #    <Text>
+    #    <UnorderedList>
+    #      <ListItem>
+    #      <ListItem>
+    # ...and so on
+    # ```
+    class List < Compound
+      def list_level
+        lookup_parents(List).count
+      end
+
+      def list_text_indent
+        '  ' * list_level
+      end
+
+      def text
+        if list_level.zero?
+          super.sub(/\n+\Z/, "\n\n")
+        else
+          super.sub(/\n+\Z/, "\n")
+        end
+      end
+    end
+
+    # Represents unordered list (list with markers).
+    class UnorderedList < List
+      def make_marker(item)
+        list_text_indent + '* '
+      end
+    end
+
+    # Represents ordered list (list with numbers).
+    class OrderedList < List
+      def make_marker(item)
+        list_text_indent + "#{(item.index + 1)}. "
+      end
+    end
+
+    # Represents definitions list (`term: definition`  structure),
+    # consists of {DTerm}s and {DDefinition}s.
+    #
+    # NB: In fact, at least in English Wikipedia, orphan "definition terms"
+    # are used as a low-level headers, especially in lists of links/references. 
+    class DefinitionList < List
+      def make_marker(item)
+        case item
+        when DTerm
+          list_text_indent
+        when DDefinition
+          list_text_indent + '  '
+        end
+      end
+    end
+
+    # Term in {DefinitionList}
+    class DTerm < ListItem
+      def text
+        super.sub("\n", ":\n")
+      end
+    end
+
+    # Term definition in {DefinitionList}
+    class DDefinition < ListItem
+    end
+
+    class List < Compound
+      include Mergeable
+
+      # Internal, used by {Parser}
+      def merge!(other)
+        ochildren = other.children.dup
+        if children.last && ochildren.first &&
+          children.last.can_merge?(ochildren.first)
+
+          children.last.merge!(ochildren.shift)
+        end
+
+        push_children(*ochildren)
+      end
+
+      # Internal, used by {Parser}
+      def self.construct(marker, nodes)
+        m = marker.shift
+        klass = LISTS[m] or
+          fail("Something went wrong: undefined list marker type #{m}")
+        item_klass = ITEMS[m]
+        
+        if marker.empty?
+          klass.new(item_klass.new(nodes))
+        else
+          klass.new(item_klass.new(construct(marker, nodes)))
+        end
+      end
+
+      private
+
+      # @private
+      LISTS = {
+        ';' => DefinitionList,
+        ':' => DefinitionList,
+        '*' => UnorderedList,
+        '#' => OrderedList
+      }
+
+      # @private
+      ITEMS = {
+        ';' => DTerm,
+        ':' => DDefinition,
+        '*' => ListItem,
+        '#' => ListItem
+      }
+      
+    end
+  end
+end

data/lib/infoboxer/tree/node.rb

@@ -0,0 +1,181 @@
+# encoding: utf-8
+require 'htmlentities'
+
+module Infoboxer
+  module Tree
+    # This is the base class for all parse tree nodes.
+    #
+    # Basically, you'll
+    # never create instances of this class or its descendants by yourself,
+    # you will receive it from tree and use for navigations.
+    #
+    class Node
+      include ProcMe
+      
+      def initialize(params = {})
+        @params = params
+      end
+
+      # Hash of node "params".
+      #
+      # Params notin is roughly the same as tag attributes in HTML. This
+      # is actual for complex nodes like images, tables, raw HTML tags and
+      # so on.
+      #
+      # The most actual params are typically exposed by node as instance
+      # methods (like {Heading#level}).
+      #
+      # @return [Hash]
+      attr_reader :params
+
+      # Node's parent in tree
+      # @return {Node}
+      attr_accessor :parent
+
+      def ==(other)
+        self.class == other.class && _eq(other)
+      end
+
+      # Position in parent's children array (zero-based)
+      def index
+        parent ? parent.index_of(self) : 0
+      end
+
+      # List of all sibling nodes (children of same parent)
+      def siblings
+        parent ? parent.children - [self] : Nodes[]
+      end
+
+      # List of siblings before this one
+      def prev_siblings
+        siblings.select{|n| n.index < index}
+      end
+
+      # List of siblings after this one
+      def next_siblings
+        siblings.select{|n| n.index > index}
+      end
+
+      # Node children list
+      def children
+        Nodes[] # redefined in descendants
+      end
+
+      # Used only during tree construction in {Parser}.
+      def can_merge?(other)
+        false
+      end
+
+      # Whether node is empty (definition of "empty" varies for different
+      # kinds of nodes). Used mainly in {Parser}.
+      def empty?
+        false
+      end
+
+      # Textual representation of this node and its children, ready for
+      # pretty-printing. Use it like this:
+      #
+      # ```ruby
+      # puts page.lookup(:Paragraph).first.to_tree
+      # # Prints something like
+      # # <Paragraph>
+      # #   This <Italic>
+      # #   is <Text>
+      # #   <Wikilink(link: "Argentina")>
+      # #     pretty <Italic>
+      # #     complicated <Text>
+      # ```
+      # 
+      # Useful for understanding page structure, and Infoboxer's representation
+      # of this structure
+      def to_tree(level = 0)
+        indent(level) + "<#{descr}>\n"
+      end
+
+      def inspect
+        text.empty? ? "#<#{descr}>" : "#<#{descr}: #{shorten_text}>"
+      end
+
+      # Node text representation. It is defined for all nodes so, that
+      # entire `Document#text` produce readable text-only representation
+      # of Wiki page. Therefore, rules are those:
+      # * inline-formatting nodes (text, bold, italics) just return the
+      #   text;
+      # * paragraph-level nodes (headings, paragraphs, lists) add `"\n\n"`
+      #   after text;
+      # * list items add marker before text;
+      # * nodes, not belonging to "main" text flow (references, templates)
+      #   produce empty text.
+      #
+      # If you want just the text of some heading or list item (without
+      # "formatting" quircks), you can use {Node#text_} method.
+      #
+      def text
+        '' # redefined in descendants
+      end
+
+      # "Clean" version of node text: without trailing linefeeds, list
+      # markers and other things added for formatting.
+      #
+      def text_
+        text.strip
+      end
+
+      # See {Node#text_}
+      def to_s
+        # just aliases will not work when #text will be redefined in subclasses
+        text_
+      end
+
+      private
+
+      MAX_CHARS = 30
+
+      def shorten_text
+        text_.length > MAX_CHARS ? text_[0..MAX_CHARS] + '...' : text_
+      end
+
+      def clean_class
+        self.class.name.sub(/^.*::/, '')
+      end
+
+      def descr
+        if !params || params.empty?
+          "#{clean_class}"
+        else
+          "#{clean_class}(#{show_params})"
+        end
+      end
+
+      def show_params(prms = nil)
+        (prms || params).map{|k, v| "#{k}: #{v.inspect}"}.join(', ')
+      end
+
+      def indent(level)
+        '  ' * level
+      end
+
+      def _eq(other)
+        fail(NotImplementedError, "#_eq should be defined in subclasses")
+      end
+
+      def decode(str)
+        Node.coder.decode(str)
+      end
+      
+      class << self
+        # Internal: descendandts DSL
+        def def_readers(*keys)
+          keys.each do |k|
+            define_method(k){ params[k] }
+          end
+        end
+
+        # Internal: HTML entities decoder.
+        def coder
+          @coder ||= HTMLEntities.new
+        end
+      end
+    end
+  end
+end