ariel 0.0.1

data/lib/ariel/extracted_node.rb

@@ -0,0 +1,20 @@
+module Ariel
+  require 'ostruct'
+
+  # Each ExtractedNode has a name, a tokenstream and a structure which points to
+  # the relevant StructureNode.
+  class ExtractedNode
+    include NodeLike
+    attr_accessor :tokenstream
+
+    def initialize(name, tokenstream, structure)
+      @children={}
+      @meta = OpenStruct.new({:name=>name, :structure=>structure})
+      @tokenstream=tokenstream
+    end
+
+    def extracted_text
+      tokenstream.text
+    end
+  end
+end

data/lib/ariel/label_utils.rb

@@ -0,0 +1,71 @@
+module Ariel
+
+  # A set of methods for use when dealing with strings from labeled documents
+  module LabelUtils
+    S_LABEL="<"
+    E_LABEL=">"
+
+    # Returns an array containing a pair of regular expressions to match a start
+    # label tag and an end label tag. If the tag_contents is not modified the
+    # regular expressions will return any properly formatted label tag. The
+    # namespace to search for can also be modified. The returned regular
+    # expressions are case insensitive.
+    def self.label_regex(tag_contents='\w+', namespace='l')
+      [/#{S_LABEL}#{namespace}:#{tag_contents}#{E_LABEL}/i,
+      /#{S_LABEL}\/#{namespace}:#{tag_contents}#{E_LABEL}/i]
+    end
+
+    # Helper function that returns a regex that will return any open or closing
+    # label tags.
+    def self.any_label_regex()
+      Regexp.union(*self.label_regex)
+    end
+
+    # Removes all labels such as <l:title> from the given string and returns the
+    # result.
+    def self.clean_string(string)
+      string.gsub self.any_label_regex, ''
+    end
+
+    # Extracts the labeled region representing the given structure node from the
+    # parent_extracted_node. A new ExtractedNode is returned to be added as a
+    # child to the parent_extracted_node. Used when loading labeled documents.
+    def self.extract_labeled_region(structure, parent_extracted_node)
+      tokenstream=parent_extracted_node.tokenstream
+      start_idx=self.skip_to_label_tag(tokenstream, structure.meta.name, :open)
+      end_idx=self.skip_to_label_tag(tokenstream.reverse, structure.meta.name, :closed)
+      end_idx=tokenstream.reverse_pos end_idx
+      newstream=tokenstream.slice_by_token_index(start_idx, end_idx)
+      child_node=ExtractedNode.new(structure.meta.name, newstream, structure)
+      parent_extracted_node.add_child child_node
+      return child_node
+    end
+
+    private
+    # Locates a given label tag in a tokenstream
+    def self.skip_to_label_tag(tokenstream, name, type)
+      case type
+      when :open
+        re_index=0
+      when :closed
+        re_index=1
+      end
+      tokenstream.rewind
+      regex = self.label_regex(name.to_s)[re_index]
+      debug "Seeking #{name.to_s} of type #{type}"
+      nesting_level=0
+      tokenstream.each do |token|
+        if token.matches?(regex)
+          return tokenstream.cur_pos if nesting_level==0
+        end
+        if token.matches?(self.label_regex[0])
+          nesting_level+=1
+          debug "Encountered token \"#{token.text}\", nesting level=#{nesting_level}"
+        elsif token.matches?(self.label_regex[1])
+          nesting_level-=1
+          debug "Encountered token \"#{token.text}\", nesting level=#{nesting_level}"
+        end
+      end
+    end
+  end
+end

data/lib/ariel/learner.rb

@@ -0,0 +1,237 @@
+module Ariel
+
+  # Implements a fairly standard separate and conquer rule learning system.
+  # Using a list of labeled examples, candidate rules are generated. A rule is
+  # refined until it covers as many as possible of the labeled examples. This
+  # rule is recorded, the covered examples are removed and the process repeats
+  # on the remaining examples. Once all examples are covered, the disjunct of
+  # all generated rules is returned.
+
+  class Learner
+    attr_accessor :current_rule, :current_seed, :candidates, :direction
+    
+    # Takes a list of TokenStreams containing labels.
+    def initialize(*examples)
+      if examples.any? {|example| example.label_index.nil?}
+        raise ArgumentError, "Passed a TokenStream with no label"
+      end
+      debug "ATTENTION: New Learner instantiated with #{examples.size} labeled examples"
+      @examples=examples
+      @candidates=[]
+      set_seed
+    end
+
+    # Initiates and operates the whole rule induction process. Finds an example
+    # to use as its seed example, then finds a rule that matches the maximum
+    # number of examples correctly and fails on all overs. All matched examples
+    # are then removed and the process is repeated considering all examples that
+    # remain. Returns an array of the rules found (in order).
+    def learn_rule(direction)
+      debug "Searching for a #{direction} rule"
+      @direction=direction
+      @current_rule=Rule.new(direction)
+      combined_rules=[]
+      while not @examples.empty?
+        set_seed unless @examples.include? @current_seed
+        rule = find_best_rule() # Find the rule that matches the most examples and fails on the others
+        prev_size = @examples.size
+        @examples.delete_if {|example| rule.apply_to(example)} #separate and conquer!
+        debug "Removing #{prev_size - @examples.size} examples matched by the generated rule, #{@examples.size} remain"
+        combined_rules << rule
+      end
+#      rule = order_rule(rule) #STALKER paper suggests that the generated rules should be ordered. This doesn't make sense, seeing as they are all generated based only on examples not matched by previous rules
+      debug "Generated rules: #{combined_rules.inspect}"
+      return combined_rules
+    end
+
+    # The seed example is chosen from the array of remaining examples. The
+    # LabeledStream with the fewest tokens before the labeled token is chosen.
+    def set_seed
+      sorted = @examples.sort_by {|example| example.label_index}
+      self.current_seed=sorted.first
+      debug "current_seed=#{current_seed.text}"
+      return current_seed
+    end
+
+    # Using the seed example passed to it, generates a list of initial rule
+    # candidates for further refinement and evaluation. The Token prior to the
+    # labeled token is considered, and separate rules are generated that skip_to that
+    # token's text or any of it's matching wildcards.
+    def generate_initial_candidates
+      if current_seed.label_index==0
+        @candidates << Rule.new(@direction)
+      else
+        end_token=current_seed.tokens[current_seed.label_index-1]
+        debug "Creating initial candidates based on #{end_token.text}"
+        @candidates<< Rule.new(@direction, [[end_token.text]])
+        @candidates.concat(@candidates[0].generalise_feature(0))
+        debug "Initial candidates: #{@candidates.inspect} created"
+      end
+      return @candidates.size
+    end
+
+    # Equivalent of LearnDisjunct in STALKER algorithm. Generates initial
+    # candidate rules, refines, and then returns a single rule.
+    def find_best_rule
+      @candidates=[]
+      generate_initial_candidates
+      while true
+        best_refiner = get_best_refiner
+        best_solution = get_best_solution
+        @current_rule = best_refiner
+        break if perfect?(best_solution)
+        refine
+      end
+#     return post_process(best_solution)
+      debug "Rule found: #{best_solution.inspect}"
+      return best_solution
+    end
+
+    # A given rule is perfect if it successfully matches the label on at least
+    # one example and fails all others.
+    def perfect?(rule)
+      perfect_count=0
+      fail_count=0
+      @examples.each do |example|
+        if rule.matches(example, :perfect)
+          perfect_count+=1
+          debug "#{rule.inspect} matches #{example.text} perfectly"
+        elsif rule.matches(example, :fail) 
+          fail_count+=1
+          debug "#{rule.inspect} fails to match #{example.text}"
+        end
+      end
+      if (perfect_count >= 1) && (fail_count == (@examples.size - perfect_count))
+        return true
+      else
+        debug "Rule was not perfect, perfect_count=#{perfect_count}, fail_count=#{fail_count}"
+        return false
+      end
+    end
+
+    # Given a list of candidate rules, uses heuristics to determine a rule
+    # considered to be the best refiner. Prefers candidate rules that have:
+    # * Larger coverage = early + correct matches.
+    # * If equal, prefer more early matches - can be made in to fails or perfect matches.
+    #   Intuitively, if there are more equal matches the rule is finding features common to all documents.
+    # * If there is a tie, more failed matches wins - we want matches to fail rather than match incorrectly
+    # * Fewer wildcards - more specific, less likely to match by chance.
+    # * Shorter unconsumed prefixes - closer to matching correctly
+    # * fewer tokens in SkipUntil() - huh? Perhaps because skip_until relies on slot content rather than
+    #   document structure.
+    # * longer end landmarks - prefer "local context" landmarks.
+    def get_best_refiner
+      selector = CandidateSelector.new(@candidates, @examples)
+      selector.select_best_by_match_type :early, :perfect #Discriminate on coverage
+      selector.select_best_by_match_type :early
+      selector.select_best_by_match_type :fail
+      selector.select_with_fewer_wildcards
+      selector.select_closest_to_label
+      selector.select_with_longer_end_landmarks
+      best_refiner = selector.random_from_remaining #just pick a random one for now if still multiple
+      debug "best_refiner found => #{best_refiner.inspect}"
+      return best_refiner
+    end
+
+    # Given a list of candidate rules, use heuristics to determine the best solution. Prefers:
+    # * More correct matches
+    # * More failed matches if a tie - failed matches preferable to incorrect matchees.
+    # * Fewer tokens in SkipUntil()
+    # * fewer wildcards
+    # * longer end landmarks
+    # * shorter unconsumed prefixes
+    def get_best_solution
+      selector = CandidateSelector.new(@candidates, @examples)
+      selector.select_best_by_match_type :perfect
+      selector.select_best_by_match_type :fail
+      selector.select_with_fewer_wildcards
+      selector.select_closest_to_label
+      selector.select_with_longer_end_landmarks
+      best_solution = selector.random_from_remaining
+      debug "best_solution found => #{best_solution.inspect}"
+      return best_solution
+    end    
+
+    # Oversees both landmark (e.g. changing skip_to("<b>") in to
+    # skip_to("Price","<b>") and topology (skip_to(:html_tag) to a chain of
+    # skip_to() commands). Takes the current rule being generated and the
+    # example against which it is being created (the current seed_rule) as
+    # arguments. 
+    def refine
+      @candidates=[]
+      current_rule.landmarks.each_with_index do |landmark, index|
+        add_new_landmarks(landmark, index) #Topology refinements
+        lengthen_landmark(landmark, index) #Landmark refinements
+      end
+      return @candidates.size
+    end
+
+    # Implements landmark refinements. Landmarks are lengthened to make them
+    # more specific.
+    # * Takes a landmark and its index in the current rule.
+    # * Applies the rule consisting of all previous landmarks in the current
+    #   rule, so the landmark can be considered in the context of the point from
+    #   which it shall be applied.
+    # * Every point at which the landmark matches after the cur_loc is considered.
+    # * Two extended landmarks are generated - a landmark that includes the
+    #   token before the match, and a landmark that includes that token after the
+    #   match. 
+    # * Rules are generated incorporating these extended landmarks, including
+    #   alternative landmark extensions that use relevant wildcards.
+    def lengthen_landmark(landmark, index)
+      current_seed.rewind #In case apply_rule isn't called as index=0
+      result = @current_rule.partial(0..(index-1)).apply_to current_seed if index > 0 #Don't care about already matched tokens
+      return 0 unless result # Rule doesn't match, no point refining
+      refined_rules=[]
+      width = landmark.size
+      while current_seed.skip_to(*landmark) #Probably should stop when cur_pos > label_index
+        break if current_seed.cur_pos > current_seed.label_index
+        match_start = (current_seed.cur_pos - 1) - width #pos of first matched token
+        match_end = current_seed.cur_pos - 1 #pos of last matched token
+        preceding_token = current_seed.tokens[match_start-1]
+        trailing_token = current_seed.tokens[match_end+1]
+        front_extended_landmark = landmark.clone.insert(0, preceding_token.text) if preceding_token
+        back_extended_landmark = landmark.clone.insert(-1, trailing_token.text) if trailing_token
+        f = current_rule.deep_clone
+        b = current_rule.deep_clone
+        f.landmarks[index] = front_extended_landmark if front_extended_landmark
+        b.landmarks[index] = back_extended_landmark if back_extended_landmark
+        refined_rules << f
+        refined_rules.concat f.generalise_feature(index, 0)
+        refined_rules << b
+        refined_rules.concat b.generalise_feature(index, -1)
+      end
+      @candidates.concat refined_rules
+      debug "#{refined_rules.size} landmark refinements generated"
+      return refined_rules.size
+    end
+
+    # Implements topology refinements - new landmarks are added to the current rule.
+    # * Takes a landmark and its index in the current rule.
+    # * Applies the rule consisting of all landmarks up to and including the
+    #   current landmark to find where it matches.
+    # * Only tokens between the label_index and the position at which the partial rule matches are considered.
+    # * Tokens before the rule match location will have no effect, as adding new
+    #   landmarks before or after the current landmark will not make the rule
+    #   match any earlier.
+    # * For every token in this slice of the TokenStream, a new potential rule
+    #   is created by adding a new landmark consisting of that token. This
+    #   is also done for each of that token's matching wildcards.
+    def add_new_landmarks(landmark, index)
+      topology_refs=[]
+      start_pos = current_rule.partial(0..index).apply_to(current_seed)
+      end_pos = current_seed.label_index #No point adding tokens that occur after the label_index
+      current_seed.tokens[start_pos...end_pos].each do |token|
+          r=current_rule.deep_clone
+          r.landmarks.insert(index+1, [token.text])
+          topology_refs << r
+          topology_refs.concat r.generalise_feature(index+1)
+      end
+    debug "Topology refinements before uniq! #{topology_refs.size}"
+    topology_refs.uniq!
+    @candidates.concat topology_refs
+    debug "#{topology_refs.size} topology refinements generated"
+    return topology_refs.size
+    end
+  end
+end

data/lib/ariel/node_like.rb

@@ -0,0 +1,26 @@
+module Ariel
+  
+  module NodeLike
+    attr_accessor :parent, :children, :meta
+
+    # Given a Node object and a name, adds a child to the array of children,
+    # setting its parent as the current node, as well as creating an accessor
+    # method matching that name.
+    def add_child(node) 
+      @children[node.meta.name]=node
+      node.parent = self
+    end
+
+    def each_descendant(include_self=false)
+      if include_self
+        node_queue=[self]
+      else
+        node_queue=self.children.values
+      end
+      until node_queue.empty? do
+        node_queue.concat node_queue.first.children.values
+        yield node_queue.shift
+      end
+    end
+  end
+end

data/lib/ariel/rule.rb

@@ -0,0 +1,112 @@
+module Ariel
+
+  # A rule contains an array of landmarks (each of which is an array of
+  # individual landmark features. This landmark array is accessible through
+  # Rule#landmarks. A Rule also has a direction :forward or :back, which
+  # determines whether it is applied from the end or beginning of a tokenstream.
+  class Rule
+    attr_accessor :landmarks, :direction
+    @@RuleMatchData=Struct.new(:token_loc, :type)
+ 
+    # A rule's direction can be :back or :forward, which determines whether it
+    # is applied from the start of end of the TokenStream. The landmark array
+    # contains an array for each landmark, which consists of one or more
+    # features. e.g. Rule.new(:forward, [[:anything, "Example"], ["Test"]]).
+    def initialize(direction, landmarks=[])
+      @landmarks=landmarks
+      raise(ArgumentError, "Not a valid direction") unless [:forward, :back].include?(direction)
+      @direction=direction
+    end
+
+    # Two rules are equal if they have the same list of landmarks and the same
+    # direction
+    def ==(rule)
+      return ((self.landmarks == rule.landmarks) && self.direction==rule.direction)
+    end
+    alias :eql? :==
+
+    def hash
+      [@landmarks, @direction].hash
+    end
+
+    # Returns a rule that contains a given range of 
+    def partial(range)
+      return Rule.new(@direction, @landmarks[range])
+    end
+
+    def deep_clone
+      Marshal::load(Marshal.dump(self))
+    end
+
+    def generalise_feature(landmark_index, feature_index=0)
+      feature=self.landmarks[landmark_index][feature_index]
+      alternates=[]
+      Wildcards.matching(feature) do |wildcard|
+        r=self.deep_clone
+        r.landmarks[landmark_index][feature_index]=wildcard
+        alternates << r
+        yield r if block_given?
+      end
+      return alternates
+    end
+
+    # Returns the number of wildcards included as features in the list of rule
+    # landmarks.
+    def wildcard_count
+      @landmarks.flatten.select {|feature| feature.kind_of? Symbol}.size
+    end
+
+    # Given a TokenStream and a rule, applies the rule on the stream and
+    # returns nil if the match fails and the token_loc if the match succeeds.
+    # Yields a RuleMatchData Struct with accessors token_loc (the position of the match in the stream)
+    # and type if a block is given. type is nil if the TokenStream has no label,
+    # :perfect if all tokens up to the labeled token are consumed, :early if the rule's final position
+    # is before the labeled token, and :late if it is after. The returned
+    # token_loc is the position in the stream as it was passed in. That is, the
+    # token_loc is always from the left of the given stream whether it is in a
+    # reversed state or not.
+    def apply_to(tokenstream)
+      if tokenstream.reversed?
+        target=tokenstream if @direction==:back
+        target=tokenstream.reverse if @direction==:forward
+      elsif not tokenstream.reversed?
+        target=tokenstream if @direction==:forward
+        target=tokenstream.reverse if @direction==:back
+      end
+      target.rewind #rules are applied from the beginning of the stream
+      @landmarks.each do |landmark|
+        unless target.skip_to(*landmark)
+          return nil
+        end
+      end
+      token_loc=target.cur_pos
+      if @direction==:back && !tokenstream.reversed?
+        token_loc = tokenstream.reverse_pos(token_loc) #Return position from left of given stream
+      end
+      md = @@RuleMatchData.new(token_loc)
+      if target.label_index
+        idx = target.label_index
+        md.type = :perfect if token_loc == idx
+        md.type = :early if token_loc < idx
+        md.type = :late if token_loc > idx
+      end
+      yield md if block_given?
+      return token_loc
+    end
+
+    # Returns true or false depending on if the match of this rule on the given
+    # tokenstream is of any of the given types (could be a combination of
+    # :perfect, :early, :fail and :late). Only valid on streams with labels
+    def matches(tokenstream, *types)
+      raise ArgumentError, "No match types given" if types.empty?
+      match = nil
+      apply_to(tokenstream) {|md| match=md.type}
+      match = :fail if match.nil?
+      if types.include? match
+        return true
+      else
+        return false
+      end
+    end
+  end
+end

data/lib/ariel/rule_set.rb

@@ -0,0 +1,34 @@
+module Ariel
+
+  # A RuleSet acts as a container for a StructureNode's start and end rules.
+  # These are stored as an ordered array and are applied in turn until there is
+  # a successful match. A RuleSet takes responsibility for applying start and
+  # end rules to extract an ExtractedNode.
+  class RuleSet
+    def initialize(start_rules, end_rules)
+      @start_rules=start_rules
+      @end_rules=end_rules
+    end
+
+    def apply_to(tokenstream)
+      start_idx=nil
+      end_idx=nil
+      @start_rules.each do |rule|
+        start_idx=rule.apply_to tokenstream
+        break if start_idx
+      end
+      @end_rules.each do |rule|
+        end_idx=rule.apply_to tokenstream
+        break if end_idx
+      end
+      if start_idx && end_idx
+        debug "RuleSet matched with start_idx=#{start_idx} and end_idx=#{end_idx}"
+        return nil if end_idx < start_idx
+        return tokenstream.slice_by_token_index(start_idx, end_idx)
+      else
+        debug "No valid match was found"
+        return nil
+      end
+    end
+  end
+end

data/lib/ariel/structure_node.rb

@@ -0,0 +1,75 @@
+module Ariel
+  require 'ostruct'
+
+  # Implements a Node object used to represent the structure of the document
+  # tree. Each node stores start and end rules to extract the desired content
+  # from its parent node. Could be viewed as a rule-storing object.
+  class StructureNode
+    include NodeLike
+    attr_accessor :ruleset
+    def initialize(name=:root, type=:not_list, &block)
+      @children={}
+      @meta = OpenStruct.new({:name=>name, :node_type=>type})
+      yield self if block_given?
+    end
+
+    # Used to extend an already created Node. e.g.
+    #  node.extend_structure do |r|
+    #    r.new_field1
+    #    r.new_field2
+    #  end
+    def extend_structure(&block)
+      yield self if block_given?
+    end
+
+    # Given a Node to apply it's rules to, this function will create a new node
+    # and add it as a child of the given node. For StructureNodes of :list type,
+    # the list is extracted and so are each of the list items. In this case,
+    # only the list items are yielded.
+    def extract_from(node)
+      # Will be reimplemented to return an array of extracted items
+      newstream = @ruleset.apply_to(node.tokenstream)
+      extracted_node = ExtractedNode.new(meta.name, newstream, self)
+      node.add_child extracted_node if newstream
+      if self.meta.node_type == :list
+        #Do stuff
+      end
+      return extracted_node
+    end
+
+    # Applies the extraction rules stored in the current StructureNode and all its
+    # descendant children.
+    def apply_extraction_tree_on(root_node, extract_labels=false)
+      extraction_queue = [root_node]
+      until extraction_queue.empty? do
+        new_parent = extraction_queue.shift
+        new_parent.meta.structure.children.values.each do |child|
+          if extract_labels
+            extracted_node=LabelUtils.extract_labeled_region(child, new_parent)
+          else
+            extracted_node=child.extract_from(new_parent)
+          end
+          extraction_queue.push(extracted_node) if extracted_node
+        end
+      end
+      return root_node
+    end
+
+    def item(name, &block)
+      self.add_child(StructureNode.new(name, &block))
+    end
+
+    def list_item(name, &block)
+      self.add_child(StructureNode.new(name, :list, &block))
+    end
+
+    def method_missing(method, *args, &block)
+      if @children.has_key? method
+        @children[method]
+      else
+        super
+      end
+    end
+  end
+end
+

data/lib/ariel/token.rb

@@ -0,0 +1,68 @@
+module Ariel  
+
+  # Tokens populate a TokenStream. They know their position in the original
+  # document, can list the wildcards that match them and determine whether a
+  # given string or wildcard is a valid match. During the process of parsing a
+  # labeled document, some tokens may be marked as being a label_tag. These are
+  # filtered from the TokenStream before the rule learning phase.
+  class Token
+  attr_reader :text, :start_loc, :end_loc
+    
+    # Each new Token must have a string representing its content, its start position in the
+    # original document (start_loc) and the point at which it ends (end_loc).
+    # For instance, in str="This is an example", if "is" were to be made a
+    # Token it would be given a start_loc of 5 and and end_loc of 7, which is
+    # str[5...7]
+    def initialize(text, start_loc, end_loc, label_tag=false)
+      @text=text.to_s
+      @start_loc=start_loc
+      @end_loc=end_loc
+      @label_tag=label_tag
+    end
+
+    # Returns true or false depending on whether the token was marked as a label
+    # tag when it was initialized.
+    def is_label_tag?
+      @label_tag
+    end
+
+    # Tokens are only equal if they have an equal start_loc, end_loc and text.
+    def ==(t)
+      return (@start_loc==t.start_loc && @end_loc==t.end_loc && @text==t.text)
+    end
+
+    # Tokens are sorted based on their start_loc
+    def <=>(t)
+      @start_loc <=> t.start_loc
+    end
+      
+    # Accepts either a string or symbol representing a wildcard in
+    # Wildcards#list. Returns true if the whole Token is consumed by the wildcard or the
+    # string is equal to Token#text, and false if the match fails. Raises an
+    # error if the passed symbol is not a member of Wildcards#list.
+    def matches?(landmark)
+      if landmark.kind_of? Symbol or landmark.kind_of? Regexp
+        if landmark.kind_of? Symbol
+          raise ArgumentError, "#{landmark} is not a valid wildcard." unless Wildcards.list.has_key? landmark
+          regex = Wildcards.list[landmark]
+        else
+          regex = landmark
+        end
+        if self.text[regex] == self.text
+          return true
+        else
+          return false
+        end
+      else
+        return true if landmark==self.text
+      end
+      return false
+    end
+
+    # Returns an array of symbols corresponding to the Wildcards that match the
+    # Token.
+    def matching_wildcards
+      return Wildcards.matching(self.text)
+    end
+  end
+end