gullah 0.0.0

data/lib/gullah/parse.rb

@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+module Gullah
+  ##
+  # A parse is the collection of root nodes produced by parsing a text.
+  #
+  #   class Example
+  #     extend Gullah
+  #
+  #     rule :S, 'NP VP'
+  #     rule :NP, 'D N'
+  #     rule :VP, 'V'
+  #
+  #     leaf :D, /the/
+  #     leaf :N, /cat/
+  #     leaf :V, /sat/
+  #   end
+  #
+  #   parses = Example.parse 'the cat sat', n: 1
+  #
+  #   # this is a Parse
+  #   parse = parses.first
+  #   puts parse.length  # => 1
+  #   puts parse.size    # => 8
+  #   puts parse.summary # => S[NP[D,_ws,N],_ws,VP[V]]
+  #
+  class Parse
+    ##
+    # The root nodes of all subtrees found in this parse in sequence. This is an array.
+    attr_reader :roots
+
+    # The text parsed by this parse.
+    attr_reader :text
+
+    # A concise stringification of the syntactic structure of this parse.
+    # For a given string and grammar all the parses will have a unique
+    # summary.
+    attr_reader :summary
+
+    def initialize(text) # :nodoc:
+      @roots = []
+      @text = text
+    end
+
+    # produce a clone of this parse with a new node with the given offsets and rule
+    def add(s, e, rule, loop_check, trash = false, boundary = false) # :nodoc:
+      clone.tap do |b|
+        b._roots = roots.map(&:clone)
+        cz = if trash
+               Trash
+             elsif boundary
+               Boundary
+             else
+               Node
+             end
+        n = cz.new(b, s, e, rule)
+        return nil if loop_check && n._loop_check?
+
+        if n.leaf?
+          b.roots << n
+        else
+          b.roots[s...e] = [n]
+        end
+      end
+    end
+
+    ##
+    # The number of root nodes in this parse. This is *not* the same as size.
+    def length
+      roots.length
+    end
+
+    ##
+    # The total number of nodes in this parse. This is *not* the same as length.
+    def size
+      @size ||= roots.sum(&:size)
+    end
+
+    ##
+    # The count of nodes that failed some test. Structure tests mark both the child
+    # and the ancestor node where the test was run as erroneous,
+    # so they will increase the +incorrectness_count+ by 2.
+    def incorrectness_count
+      @incorrectness_count ||= roots.select(&:failed?).count
+    end
+
+    ##
+    # The count of nodes which have some structure test which was never
+    # successfully run.
+    def pending_count
+      @pending_count ||= roots.select(&:pending_tests?).count
+    end
+
+    ##
+    # Are there any nodes in the parse that are erroneous, either because
+    # some test failed or because they correspond to "trash" -- characters
+    # that matched no leaf rule?
+    def errors?
+      incorrectness_count.positive?
+    end
+
+    ##
+    # Are all leaves accounted for without errors and have all tests passed?
+    def success?
+      !errors? && roots.all? { |n| n.ignorable? || n.nonterminal? && !n.pending_tests? }
+    end
+
+    ##
+    # Not a +success?+
+    def failure?
+      !success?
+    end
+
+    # a simplified representation for debugging
+    # "so" = "significant only"
+    def dbg(so: false)
+      roots.map { |n| n.dbg so: so }
+    end
+
+    ##
+    # return an enumeration of all the nodes in the parse.
+    #
+    #   parses = Grammar.parse "this grammar uses the usual whitespace rule"
+    #
+    #   parses.first.nodes.select { |n| n.name == :_ws }.count  # => 6
+    def nodes
+      NodeIterator.new self
+    end
+
+    def clone # :nodoc:
+      super.tap do |c|
+        %i[@summary @size @incorrectness_count @pending_count].each do |v|
+          c.remove_instance_variable v if c.instance_variable_defined?(v)
+        end
+      end
+    end
+
+    ##
+    # The start offset of the first leaf in the parse.
+    def start
+      roots.first.start
+    end
+
+    ##
+    # The end offset of the last leaf in the parse.
+    def end
+      roots.last.end
+    end
+
+    ## ADVISORILY PRIVATE
+
+    # :stopdoc:
+
+    # for debugging
+    def own_text
+      text[start...self.end]
+    end
+
+    # make a new parse whose first part is this parse's nodes and whose
+    # second part is the later parse's nodes
+    def merge(later)
+      self.class.new(text).tap do |merged|
+        merged._roots = roots + later.roots
+      end
+    end
+
+    # split the parse into segments and boundaries
+    def split
+      last_index = 0
+      splits = []
+
+      # look for traversible sequences and boundaries
+      roots.each_with_index do |n, i|
+        next if n.traversible?
+
+        if i > last_index
+          # sometimes you can have two boundaries in a row,
+          # or you can begin with a boundary
+          segment = Parse.new text
+          segment._roots = roots[last_index...i]
+          splits << segment.initialize_summaries
+        end
+
+        # create boundary element
+        segment = Parse.new text
+        segment._roots = [n]
+        splits << segment.initialize_summaries
+        last_index = i + 1
+      end
+      return [initialize_summaries] if last_index.zero?
+
+      if last_index < roots.length
+        segment = Parse.new text
+        segment._roots = roots[last_index...roots.length]
+        splits << segment.initialize_summaries
+      end
+      splits
+    end
+
+    def _roots=(roots)
+      @roots = roots
+    end
+
+    # it would be conceptually simpler to lazily initialize the summary, but this
+    # gives us a speed boost
+    def initialize_summaries
+      @summary = roots.each { |n| n._summary = n.name unless n.summary }.map(&:summary).join(';')
+      self
+    end
+
+    def _summary=(str)
+      @summary = str
+    end
+
+    class NodeIterator # :nodoc:
+      include Enumerable
+
+      def initialize(parse)
+        @parse = parse
+      end
+
+      def each(&block)
+        @parse.roots.each do |root|
+          root.subtree.each(&block)
+        end
+      end
+
+      def last
+        @parse.roots.last.leaves.last
+      end
+    end
+  end
+end

data/lib/gullah/picker.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+module Gullah
+  # a Picker keeps a sorted set of iterators so we can always pick the iterator
+  # most likely to lead quickly to a satisfactory parse
+  class Picker # :nodoc:
+    def initialize
+      # a sorted list of the
+      @error_counts = []
+      @error_lists = []
+      @size_count_list = []
+    end
+
+    # add an iterator
+    def <<(iterator)
+      e_idx = iterator.errors
+      s_idx = iterator.length
+      e_list = @error_lists[e_idx] ||= []
+      list = e_list[s_idx] ||= []
+      sc_list = @size_count_list[e_idx] ||= []
+      if (i = @error_counts.bsearch_index { |c| c >= e_idx })
+        # *may* have to add this error count
+        @error_counts.insert i, e_idx if @error_counts[i] != e_idx
+      else
+        # this is a bigger error count than we currently have
+        @error_counts << e_idx
+      end
+      if (i = sc_list.bsearch_index { |c| c >= s_idx })
+        # *may* have to add this size
+        sc_list.insert i, s_idx if sc_list[i] != s_idx
+      else
+        # this size is bigger than we currently have for this error count
+        sc_list << s_idx
+      end
+      # finally, we stow the iterator
+      list << iterator
+    end
+
+    # remove the best iterator
+    def pop
+      error_idx = @error_counts.first
+      return nil unless error_idx
+
+      error_list = @error_lists[error_idx]
+      size_idx = @size_count_list[error_idx].first
+      size_list = error_list[size_idx]
+      iterator = size_list.pop
+      # remove indices if they're used up
+      if size_list.empty?
+        @size_count_list[error_idx].shift
+        @error_counts.shift if @size_count_list[error_idx].empty?
+      end
+      iterator
+    end
+  end
+end

data/lib/gullah/rule.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+module Gullah
+  # a non-terminal grammatical rule
+  class Rule # :nodoc:
+    # name -- a symbol identifying the rule
+    # body -- preserved for debugging
+    # tests -- tests that must be run after a match to determine whether the node is a keeper
+    # ancestor_tests -- tests that must be run after an ancestor match
+    # subrules/atoms -- if you have no subrules, you have a sequence of atoms
+    attr_reader :name, :body, :tests, :ancestor_tests, :subrules, :atoms, :preconditions
+
+    def initialize(name, body, tests: [], preconditions: [])
+      @name = name
+      @body = body
+      @tests = tests
+      @preconditions = preconditions
+      if body =~ /\|/
+        @subrules = body.split(/ ?\| ?/).map do |subrule|
+          Rule.new(name, subrule, tests: tests)
+        end
+      else
+        @atoms = body.split(/ /).map do |a|
+          Atom.new(a, self)
+        end
+        @atoms.each_with_index do |a, i|
+          a._next = @atoms[i + 1]
+        end
+      end
+    end
+
+    # the subrules that may start a match and their atoms
+    def starters
+      if subrules
+        subrules.flat_map(&:starters)
+      else
+        ar = []
+        atoms.each do |a|
+          ar << [a.seeking, a]
+          break if a.required?
+        end
+        ar
+      end
+    end
+
+    # could this rule participate in a loop?
+    def potentially_unary?
+      if subrules
+        subrules.any?(&:potentially_unary?)
+      else
+        atoms.sum(&:min_repeats) < 2
+      end
+    end
+
+    # collect all links from a sought symbol to the new name
+    # used in testing for potential infinite loops
+    def branches
+      if subrules
+        subrules.select(&:potentially_unary?).flat_map(&:branches)
+      else
+        atoms.map { |a| [a.seeking, name] }
+      end
+    end
+
+    # collect all the different rules some atom of this rule might match
+    def seeking
+      if subrules
+        subrules.flat_map(&:seeking).uniq
+      else
+        atoms.map(&:seeking).uniq
+      end
+    end
+
+    # obtain all the literals required by this rule
+    def literals
+      if subrules
+        subrules.flat_map(&:literals).uniq
+      else
+        atoms.select(&:literal).map(&:seeking).uniq
+      end
+    end
+
+    ## ADVISORILY PRIVATE
+
+    def _post_init(tests, preconditions)
+      @tests, @ancestor_tests = tests.partition { |m| m.arity == 1 }
+      @preconditions = preconditions
+    end
+  end
+end

data/lib/gullah/segment.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Gullah
+  # a segment handles the portion of a string between boundaries
+  # or a boundary itself
+  class Segment # :nodoc:
+    attr_reader :start, :end, :done
+    attr_accessor :continuations
+
+    def initialize(lexes, filters, starters, do_unary_branch_check, n)
+      # the lexes are all parses that begin and end at the same character offsets
+      # the one with the fewest root nodes is most likely to be correct
+      lexes = lexes.sort_by(&:length)
+      sample = lexes.first
+      @start = sample.start
+      @end = sample.end
+      @continuations = []
+      @mass = lexes.map(&:length).sum
+      @done = false
+      @hopper = Hopper.new(filters, n)
+      @starters = starters
+      @do_unary_branch_check = do_unary_branch_check
+      @bases = Picker.new
+      lexes.each do |p|
+        @bases << Iterator.new(p, @hopper, starters, do_unary_branch_check)
+      end
+    end
+
+    def total_parses
+      if @hopper.size.zero?
+        0
+      elsif continuations.any?
+        continuations.map { |c| c.total_parses * @hopper.size }.sum
+      else
+        @hopper.size
+      end
+    end
+
+    # used to pick the next segment to iterate
+    def weight
+      @mass * @hopper.size
+    end
+
+    # try to add one parse to the hopper
+    # returns whether or not this succeeded
+    def next
+      return false if @done
+
+      start_size = @hopper.size
+      catch :done do
+        while (iterator = @bases.pop)
+          unless @hopper.continuable?(iterator.parse)
+            @hopper << iterator.parse
+            throw :done if @hopper.satisfied?
+
+            next
+          end
+
+          if (p = iterator.next)
+            @bases << iterator
+            @bases << Iterator.new(p, @hopper, @starters, @do_unary_branch_check)
+          elsif iterator.never_returned_any?
+            # it looks this iterator was based on an unreducible parse
+            @hopper << iterator.parse
+            throw :done if @hopper.satisfied?
+          end
+        end
+      end
+      end_size = @hopper.size
+      if end_size == start_size
+        @done = true
+        false
+      else
+        true
+      end
+    end
+
+    def results
+      @results ||= if continuations.any?
+                     @hopper.dump.flat_map do |parse|
+                       continuations.flat_map do |c|
+                         c.results.flat_map do |p|
+                           parse.merge(p)
+                         end
+                       end
+                     end
+                   else
+                     @hopper.dump
+                   end
+    end
+  end
+end