kleene 0.6.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adae43aaa27339c7c8affc091b968c782c6494095fbb513c5f95ca423f854b56
-  data.tar.gz: 80b52c420273ba1f8c16a5b1e9aebbb18782477fe121ae71ec7603736f2b15ac
+  metadata.gz: 03ad3b5293809eb768a2bb47eb3f1a9ccee680d7d71873d75c7511eb9fc711be
+  data.tar.gz: cb0e8f6600e878153bf2454cea8e3dd4dc67d1872660bd1a77b0648a5f91efa5
 SHA512:
-  metadata.gz: 3572f4e64ce3941fe071bfefdc2e3e3c87c82a14ebf98049d323114397a367491ff76c25afbc7caad12b91cb9eddc17e598efcacc23eb59fe8d4996bb0602cd0
-  data.tar.gz: 0053e335ec9a68d34237ba83f281abf33c1b94f76a31437705a63e68583cc75af4f5b61b0e303ba149a0ce8062ecb4bd7bc4ad63f1293c4e1c905b390ba87df1
+  metadata.gz: f7043fb3024741baf02ed48de44d14c616d4bb83315ad23481d0908c962fbd41a56a9ccc8d0444577526baa94933621b43203c29b2cba14192d98d8f5a7246ef
+  data.tar.gz: a78f7aee47db290800efa79cb6096d604f30dc249907db8759bb319b29f01d378801395557a9a8760fc3b6d06e2e447335398de053e1594233da62128a3aafb9

data/.rubocop.yml

@@ -0,0 +1,14 @@
+AllCops:
+  StyleGuideBaseURL: https://rubystyle.guide
+
+Layout/SpaceInsideBlockBraces:
+  SpaceBeforeBlockParameters: false
+
+Layout/LineLength:
+  Max: 160
+
+Style/AccessorGrouping:
+  EnforcedStyle: separated
+
+Style/Encoding:
+  Enabled: true

data/Gemfile.lock

@@ -1,8 +1,9 @@
 PATH
   remote: .
   specs:
-    kleene (0.1.0)
+    kleene (0.7.0)
       activesupport (~> 7.1)
+      regexp_parser (~> 2.8)
 
 GEM
   remote: https://rubygems.org/
@@ -45,7 +46,7 @@ GEM
     parser (3.2.2.4)
       ast (~> 2.4.1)
       racc
-    racc (1.7.2)
+    racc (1.7.3)
     rainbow (3.1.1)
     rake (13.1.0)
     rbs (2.8.4)

data/kleene.gemspec

@@ -33,6 +33,7 @@ Gem::Specification.new do |spec|
 
   # Uncomment to register a new dependency of your gem
   spec.add_dependency "activesupport", "~> 7.1"
+  spec.add_dependency "regexp_parser", "~> 2.8"
 
   # For more information and examples about making a new gem, check out our
   # guide at: https://bundler.io/guides/creating_gem.html

data/lib/kleene/kleene.rb

@@ -1,8 +1,8 @@
 # this is a port and extension of https://github.com/davidkellis/kleene/
 
-require_relative "./dsl"
-require_relative "./nfa"
-require_relative "./dfa"
+require_relative './dsl'
+require_relative './nfa'
+require_relative './dfa'
 
 module Kleene
   # The default alphabet consists of the following:
@@ -28,7 +28,6 @@ module Kleene
       State.new(final, true)
     end
 
-
     attr_reader :id # : Int32
     attr_accessor :final # : Bool
     attr_accessor :error # : Bool
@@ -76,13 +75,11 @@ module Kleene
     end
 
     def ==(other)
-      @string == other.string &&
-      @range == other.range
+      @string == other.string && @range == other.range
     end
 
     def eql?(other)
       self == other
     end
   end
-
 end

data/lib/kleene/multi_match_dfa.rb

@@ -77,18 +77,10 @@ module Kleene
 
       nfa
     end
+  end
 
-    def match_tracker(input) # : MatchTracker
-      dfa = @composite_dfa.deep_clone
-      match_tracker = setup_callbacks(dfa)
-
-      input.each_char.with_index do |char, index|
-        dfa.handle_token!(char, index)
-      end
-
-      match_tracker
-    end
-
+  class BatchMultiMatchDFA < MultiMatchDFA
+    # #matches(input) is the batch-style matching interface
     def matches(input) # : Hash(NFA, Array(MatchRef))
       mt = match_tracker(input)
 
@@ -131,8 +123,19 @@ module Kleene
       mt.matches
     end
 
+    def match_tracker(input) # : BatchMatchTracker
+      dfa = @composite_dfa.deep_clone
+      match_tracker = setup_callbacks(dfa)
+
+      input.each_char.with_index do |char, index|
+        dfa.handle_token!(char, index)
+      end
+
+      match_tracker
+    end
+
     def setup_callbacks(dfa)
-      match_tracker = MatchTracker.new
+      match_tracker = BatchMatchTracker.new
 
       # 1. identify DFA states that correspond to successful match of first character of the NFAs
       epsilon_closure_of_nfa_start_state = composite_nfa.epsilon_closure(composite_nfa.start_state)
@@ -222,10 +225,9 @@ module Kleene
 
       match_tracker
     end
-
   end
 
-  class MatchTracker
+  class BatchMatchTracker
     # The NFA keys in the following two structures are not the original NFAs supplied to the MultiMatchDFA.
     # They are the original NFAs that have been augmented with a dead end error state, so the keys are objects that
     # are the internal state of a MultiMatchDFA
@@ -249,6 +251,10 @@ module Kleene
     attr_accessor :matches # : Hash(NFA, Array(MatchRef))  # NFA -> Array(MatchRef)
 
     def initialize
+      reset
+    end
+
+    def reset
       @candidate_match_start_positions = Hash.new
       @match_end_positions = Hash.new
       @empty_matches = Hash.new

data/lib/kleene/naive_online_regex.rb

@@ -0,0 +1,102 @@
+require 'set'
+require_relative './kleene'
+
+module Kleene
+  class NaiveOnlineRegex
+    def initialize(regexen, window_size = 100)
+      @regexen = regexen
+      @window_size = window_size
+
+      reset
+    end
+
+    def reset
+      @buffer = ''
+      @matches_per_regex = {} # Hash(Regexp, Set(OnlineMatch))
+    end
+
+    # #ingest(input) is the online-style matching interface
+    def ingest(input, _debug = false) # : Set(OnlineMatch)
+      @buffer << input
+      new_online_matches = Set.new
+      @regexen.each do |regex|
+        existing_matches_for_regex = (@matches_per_regex[regex] ||= Set.new)
+        scan_matches = @buffer.scan_matches(regex)
+        scan_online_matches = scan_matches.map {|match_data| OnlineMatch.new(regex, match_data) }.to_set
+        new_matches = scan_online_matches - existing_matches_for_regex # new_matches : Set(OnlineMatch)
+        existing_matches_for_regex.merge(new_matches)
+        new_online_matches.merge(new_matches)
+      end
+      resize_buffer!
+      new_online_matches
+    end
+
+    def matches # Hash(Regexp, Set(OnlineMatch))
+      @matches_per_regex
+    end
+
+    def matches_for(regex) # Set(OnlineMatch) | Nil
+      @matches_per_regex[regex]
+    end
+
+    def resize_buffer!
+      return unless @buffer.size > @window_size
+
+      number_of_chars_at_front_of_buffer_that_should_roll_off = @buffer.size - @window_size
+
+      @buffer = @buffer[-@window_size..-1]
+      drop_matches_that_have_rolled_off(number_of_chars_at_front_of_buffer_that_should_roll_off)
+    end
+
+    def drop_matches_that_have_rolled_off(number_of_chars_at_front_of_buffer_that_rolled_off)
+      @matches_per_regex.each do |regex, match_set|
+        match_set.reject! {|online_match| online_match.offsets.first < number_of_chars_at_front_of_buffer_that_rolled_off }
+      end
+    end
+  end
+
+  # A {Regexp, MatchData} pair
+  class OnlineMatch
+    # Regexp  # MatchData  # Array(Int) -> [start, end]    # excludes the end offset
+    attr_reader :regex
+    attr_reader :match
+    attr_reader :offsets # Regexp  # MatchData  # Array(Int) -> [start, end]    # excludes the end offset
+
+    def initialize(regex, match)
+      @regex, @match, @offsets = regex, match
+      @offsets = match.offset(0)
+    end
+
+    def identity
+      [@regex, @offsets, to_a]
+    end
+
+    def ==(other)
+      identity == other.identity
+    end
+
+    def eql?(other)
+      self == other
+    end
+
+    def hash
+      identity.hash
+    end
+
+    def to_a
+      @match.to_a
+    end
+
+    def to_h
+      { @regex => to_a, :offsets => @offsets }
+    end
+
+    def captures
+      @match.captures
+    end
+
+    def [](*args)
+      @match.method(:[]).call(*args)
+    end
+  end
+end

data/lib/kleene/online_dfa.rb

@@ -0,0 +1,323 @@
+require "stringio"
+require_relative "./kleene"
+
+module Kleene
+  class MachineTuple
+    attr_accessor :nfa # : NFA
+    attr_accessor :nfa_with_dead_err # : NFA
+    attr_accessor :dfa # : DFA
+
+    def initialize(nfa, nfa_with_dead_err, dfa)
+      @nfa, @nfa_with_dead_err, @dfa = nfa, nfa_with_dead_err, dfa
+    end
+  end
+
+  class OnlineDFA
+    include DSL
+
+    # @original_nfas : Array(NFA)
+    attr_reader :nfas_with_err_state # : Array(NFA)
+    attr_accessor :dead_end_nfa_state_to_dead_end_nfa # : Hash(State, NFA)
+    attr_accessor :composite_nfa # : NFA
+    attr_accessor :composite_dfa # : DFA
+
+    attr_accessor :machines_by_index # : Hash(Int32, MachineTuple)
+    attr_accessor :nfa_to_index # : Hash(NFA, Int32)
+    attr_accessor :nfa_with_dead_err_to_index # : Hash(NFA, Int32)
+    attr_accessor :dfa_to_index # : Hash(DFA, Int32)
+
+    def initialize(nfas)
+      composite_alphabet = nfas.reduce(Set.new) {|memo, nfa| memo | nfa.alphabet }
+
+      @original_nfas = nfas
+      @nfas_with_err_state = nfas.map {|nfa| with_err_dead_end(nfa, composite_alphabet) }     # copy NFAs and add dead-end error states to each of them
+      dfas = @original_nfas.map(&:to_dfa)
+
+      @nfa_to_index = @original_nfas.map.with_index {|nfa, index| [nfa, index] }.to_h
+      @nfa_with_dead_err_to_index = @nfas_with_err_state.map.with_index {|nfa, index| [nfa, index] }.to_h
+      @dfa_to_index = dfas.map.with_index {|dfa, index| [dfa, index] }.to_h
+      @machines_by_index = @original_nfas.zip(nfas_with_err_state, dfas).map.with_index {|tuple, index| nfa, nfa_with_dead_err, dfa = tuple; [index, MachineTuple.new(nfa, nfa_with_dead_err, dfa)] }.to_h
+
+      # build a mapping of (state -> nfa) pairs that capture which nfa owns each state
+      @dead_end_nfa_state_to_dead_end_nfa = Hash.new
+      @nfas_with_err_state.each do |nfa_with_dead_err|
+        nfa_with_dead_err.states.each do |state|
+          @dead_end_nfa_state_to_dead_end_nfa[state] = nfa_with_dead_err
+        end
+      end
+
+      # create a composite NFA as the union of all the NFAs with epsilon transitions from every NFA state back to the union NFA's start state
+      @composite_nfa = create_composite_nfa(@nfas_with_err_state)
+      @composite_dfa = @composite_nfa.to_dfa
+
+      reset
+    end
+
+    def machines_from_nfa(nfa) # : MachineTuple
+      machines_by_index[nfa_to_index[nfa]]
+    end
+
+    def machines_from_nfa_with_dead_err(nfa_with_dead_err) # : MachineTuple
+      machines_by_index[nfa_with_dead_err_to_index[nfa_with_dead_err]]
+    end
+
+    def machines_from_dfa(dfa) # : MachineTuple
+      machines_by_index[dfa_to_index[dfa]]
+    end
+
+    # create a composite NFA as the union of all the NFAs with epsilon transitions from every NFA state back to the union NFA's start state
+    def create_composite_nfa(nfas)
+      nfa = union!(nfas)
+
+      # add epsilon transitions from all the states except the start state back to the start state
+      nfa.states.each do |state|
+        if state != nfa.start_state
+          nfa.add_transition(NFATransition::Epsilon, state, nfa.start_state)
+        end
+      end
+
+      nfa.update_final_states
+
+      nfa
+    end
+
+    def reset # : OnlineMatchTracker
+      @active_composite_dfa = @composite_dfa.deep_clone
+      @active_candidate_dfas = []
+      @match_tracker = setup_callbacks(@active_composite_dfa)
+      @buffer = ""
+    end
+
+    # #ingest(input) is the online-style matching interface
+    def ingest(input, debug = false) # : Hash(NFA, Array(MatchRef))
+      mt = @match_tracker
+
+      start_index_of_input_fragment_in_buffer = @buffer.length
+
+      input.each_char.with_index do |char, index|
+        @active_composite_dfa.handle_token!(char, start_index_of_input_fragment_in_buffer + index)
+      end
+
+      @buffer << input
+
+      start_index_to_nfas_that_may_match = mt.invert_candidate_match_start_positions
+
+      mt.empty_matches.each do |nfa_with_dead_err, indices|
+        original_nfa = machines_from_nfa_with_dead_err(nfa_with_dead_err).nfa
+        indices.select {|index| index >= start_index_of_input_fragment_in_buffer }.each do |index|
+          mt.add_match(original_nfa, MatchRef.new(@buffer, index...index))
+        end
+      end
+
+      input.each_char.with_index do |char, index|
+        index_in_buffer = start_index_of_input_fragment_in_buffer + index
+
+        @active_candidate_dfas.reject! do |active_dfa_tuple|
+          dfa_clone, original_nfa, start_of_match_index = active_dfa_tuple
+
+          dfa_clone.handle_token!(char, index_in_buffer)
+          mt.add_match(original_nfa, MatchRef.new(@buffer, start_of_match_index..index_in_buffer)) if dfa_clone.accept?
+
+          dfa_clone.error?
+        end
+
+        if nfas_with_dead_err = start_index_to_nfas_that_may_match[index_in_buffer]
+          nfas_with_dead_err.each do |nfa_with_dead_err|
+            machines = machines_from_nfa_with_dead_err(nfa_with_dead_err)
+            original_nfa = machines.nfa
+            dfa = machines.dfa
+            dfa_clone = dfa.shallow_clone
+
+            dfa_clone.handle_token!(char, index_in_buffer)
+            mt.add_match(original_nfa, MatchRef.new(@buffer, index_in_buffer..index_in_buffer)) if dfa_clone.accept?
+
+            @active_candidate_dfas << [dfa_clone, original_nfa, index_in_buffer] unless dfa_clone.error?
+          end
+        end
+      end
+
+      matches
+    end
+
+    def matches
+      @match_tracker.matches
+    end
+
+    def setup_callbacks(dfa)
+      match_tracker = OnlineMatchTracker.new
+
+      # 1. identify DFA states that correspond to successful match of first character of the NFAs
+      epsilon_closure_of_nfa_start_state = composite_nfa.epsilon_closure(composite_nfa.start_state)
+      nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa = composite_nfa.transitions_from(epsilon_closure_of_nfa_start_state).
+                                                                                                         reject {|transition| transition.epsilon? || transition.to.error? }.
+                                                                                                         map(&:to).to_set
+      dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa = nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.
+                                                                                              compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                                              reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_state_to_dead_end_nfas_that_have_matched_their_first_character = Hash.new
+      dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.each do |dfa_state|
+        dfa_state_to_dead_end_nfas_that_have_matched_their_first_character[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                                            select {|nfa_state| nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.includes?(nfa_state) }.
+                                                                                            compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # 2. identify DFA states that correspond to final states in the NFAs
+      nfa_final_states = @nfas_with_err_state.map(&:final_states).reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_states_that_correspond_to_nfa_final_states = nfa_final_states.compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                        reduce(Set.new) {|memo, state_set| memo | state_set }
+      dead_end_nfas_that_have_transitioned_to_final_state = Hash.new
+      dfa_states_that_correspond_to_nfa_final_states.each do |dfa_state|
+        dead_end_nfas_that_have_transitioned_to_final_state[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                             select {|nfa_state| nfa_final_states.includes?(nfa_state) }.
+                                                                             compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # 3. Identify DFA states that correspond to successful match without even having seen any characters.
+      #    These are cases where the NFA's start state is a final state or can reach a final state by following only epsilon transitions.
+      nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state = epsilon_closure_of_nfa_start_state.select(&:final?).to_set
+      dfa_states_that_represent_both_start_states_and_final_states = nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state.
+                                                                        compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                        reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters = Hash.new
+      dfa_states_that_represent_both_start_states_and_final_states.each do |dfa_state|
+        dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                                                     select {|nfa_state| nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state.includes?(nfa_state) }.
+                                                                                                     compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # set up call transition call backs, since the callbacks may only be defined once per state and transition
+      # For (1):
+      #    Set up transition callbacks to push the index position of the start of a match of each NFA that has begun
+      #    to be matched on the transition to one of the states in (1)
+      # For (2):
+      #    set up transition callbacks to push the index position of the end of a successful match onto the list
+      #    of successful matches for the NFA that matched
+      # For (3):
+      #    set up transision callbacks to capture successful empty matches
+      destination_dfa_states_for_callbacks = dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa | dfa_states_that_correspond_to_nfa_final_states
+      destination_dfa_states_for_callbacks.each do |dfa_state|
+        dfa.on_transition_to(dfa_state) do |transition, token, token_index|
+          destination_dfa_state = transition.to
+
+          should_track_empty_match = dfa_states_that_represent_both_start_states_and_final_states.includes?(destination_dfa_state)
+          should_track_start_of_candidate_match = should_track_empty_match || dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.includes?(destination_dfa_state)
+          should_track_end_of_match = dfa_states_that_correspond_to_nfa_final_states.includes?(destination_dfa_state)
+
+          if should_track_empty_match
+            dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[destination_dfa_state].each do |nfa_with_dead_end|
+              match_tracker.add_empty_match(nfa_with_dead_end, token_index)
+            end
+          end
+
+          if should_track_start_of_candidate_match
+            nfas_that_matched_first_character = dfa_state_to_dead_end_nfas_that_have_matched_their_first_character[destination_dfa_state] || Set.new
+            nfas_that_matched_empty_match = dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[destination_dfa_state] || Set.new
+            dead_end_nfas_that_are_starting_to_match = nfas_that_matched_first_character | nfas_that_matched_empty_match
+            dead_end_nfas_that_are_starting_to_match.each do |nfa_with_dead_end|
+              match_tracker.add_start_of_candidate_match(nfa_with_dead_end, token_index)
+            end
+          end
+
+          if should_track_end_of_match
+            dead_end_nfas_that_have_transitioned_to_final_state[destination_dfa_state].each do |nfa_with_dead_end|
+              match_tracker.add_end_of_match(nfa_with_dead_end, token_index)
+            end
+          end
+        end
+      end
+
+      match_tracker
+    end
+  end
+
+  class OnlineMatchTracker
+    # The NFA keys in the following two structures are not the original NFAs supplied to the MultiMatchDFA.
+    # They are the original NFAs that have been augmented with a dead end error state, so the keys are objects that
+    # are the internal state of a MultiMatchDFA
+    attr_accessor :candidate_match_start_positions # : Hash(NFA, Array(Int32))     # NFA -> Array(IndexPositionOfStartOfMatch)
+    #  The end positions are indices at which, after handling the character, the DFA was observed to be in a match/accept state;
+    #  however, the interpretation is ambiguous, because the accepting state may be as a result of (1) transitioning to an error state that is also marked final/accepting,
+    #  OR it may be as a result of transitioning to (2) a non-error final state.
+    #  In the case of (1), the match may be an empty match, where after transitioning to an error state, the DFA is in a state that
+    #  is equivalent to the error state and start state and final state (e.g. as in an optional or kleene star DFA),
+    #  while in the case of (2), the match may be a "normal" match.
+    #  The ambiguity is problematic because it isn't clear whether the index position of the match is end inclusive end of a match
+    #  or the beginning of an empty match.
+    #  This ambiguity is all due to the construction of the composite DFA in the MultiMatchDFA - the dead end error states are epsilon-transitioned
+    #  to the composite DFA's start state.
+    attr_accessor :match_end_positions # : Hash(NFA, Array(Int32))                 # NFA -> Array(IndexPositionOfEndOfMatch)
+    attr_accessor :empty_matches # : Hash(NFA, Array(Int32))                       # NFA -> Array(IndexPositionOfEmptyMatch)
+
+    # The NFA keys in the following structure are the original NFAs supplied to the MultiMatchDFA.
+    # This is in contrast to the augmented NFAs that are used as keys in the candidate_match_start_positions and
+    # match_end_positions structures, documented above ^^^.
+    attr_accessor :matches # : Hash(NFA, Array(MatchRef))  # NFA -> Array(MatchRef)
+
+    def initialize
+      reset
+    end
+
+    def reset
+      @candidate_match_start_positions = Hash.new
+      @match_end_positions = Hash.new
+      @empty_matches = Hash.new
+      @matches = Hash.new
+    end
+
+    def start_positions(nfa)
+      candidate_match_start_positions[nfa] ||= Array.new
+    end
+
+    def end_positions(nfa)
+      match_end_positions[nfa] ||= Array.new
+    end
+
+    def empty_match_positions(nfa)
+      empty_matches[nfa] ||= Array.new
+    end
+
+    def matches_for(nfa)
+      matches[nfa] ||= Array.new
+    end
+
+    def add_start_of_candidate_match(nfa_with_dead_end, token_index)
+      # puts "add_start_of_candidate_match(#{nfa.object_id}, #{token_index})"
+      positions = start_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    #  the end positions are inclusive of the index of the last character matched, so empty matches are not accounted for in the match_end_positions array
+    def add_end_of_match(nfa_with_dead_end, token_index)
+      # puts "add_end_of_match(#{nfa.object_id}, #{token_index})"
+      positions = end_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    def add_empty_match(nfa_with_dead_end, token_index)
+      positions = empty_match_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    def invert_candidate_match_start_positions # : Hash(Int32, Array(NFA))
+      index_to_nfas = Hash.new
+      candidate_match_start_positions.each do |nfa_with_dead_end, indices|
+        indices.each do |index|
+          nfas = index_to_nfas[index] ||= Array.new
+          nfas << nfa_with_dead_end
+        end
+      end
+      index_to_nfas
+    end
+
+    def add_match(nfa, match)
+      matches = matches_for(nfa)
+      matches << match
+    end
+  end
+end

data/lib/kleene/parser.rb

@@ -0,0 +1,9 @@
+
+module Kleene
+  class Parser
+    def parse(pattern)
+      ast = Regexp::Parser.parse(pattern)
+      ast
+    end
+  end
+end

data/lib/kleene/patches.rb

@@ -12,12 +12,16 @@ module Enumerable
     ary = []
     each do |e|
       v = block.call(e)
-      unless v.nil?
-        ary << v
-      end
+      ary << v unless v.nil?
     end
     ary
   end
 
-  alias_method :includes?, :include?
+  alias includes? include?
+end
+
+class String
+  def scan_matches(pattern) # : Array(MatchData)
+    to_enum(:scan, pattern).map { Regexp.last_match }
+  end
 end

data/lib/kleene/version.rb

@@ -1,3 +1,3 @@
 module Kleene
-  VERSION = "0.6.0"
+  VERSION = "0.8.0"
 end

data/lib/kleene.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
-require "active_support"
-require "active_support/core_ext"
-require_relative "kleene/version"
-require_relative "kleene/patches"
-require_relative "kleene/kleene"
-require_relative "kleene/dsl"
-require_relative "kleene/nfa"
-require_relative "kleene/dfa"
-require_relative "kleene/multi_match_dfa"
-
+require 'active_support'
+require 'active_support/core_ext'
+require 'regexp_parser'
+require_relative 'kleene/version'
+require_relative 'kleene/patches'
+require_relative 'kleene/kleene'
+require_relative 'kleene/dsl'
+require_relative 'kleene/nfa'
+require_relative 'kleene/dfa'
+require_relative 'kleene/multi_match_dfa'
+require_relative 'kleene/online_dfa'
+require_relative 'kleene/naive_online_regex'
+require_relative 'kleene/parser'
 
 module Kleene
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kleene
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.8.0
 platform: ruby
 authors:
 - David Ellis
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2023-11-04 00:00:00.000000000 Z
+date: 2023-11-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '7.1'
+- !ruby/object:Gem::Dependency
+  name: regexp_parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.8'
 description: kleene is a library for building regular expression recognition automata
   - nfas, dfas, and some specialty structures.
 email:
@@ -33,6 +47,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".rspec"
+- ".rubocop.yml"
 - Gemfile
 - Gemfile.lock
 - LICENSE
@@ -45,7 +60,10 @@ files:
 - lib/kleene/dsl.rb
 - lib/kleene/kleene.rb
 - lib/kleene/multi_match_dfa.rb
+- lib/kleene/naive_online_regex.rb
 - lib/kleene/nfa.rb
+- lib/kleene/online_dfa.rb
+- lib/kleene/parser.rb
 - lib/kleene/patches.rb
 - lib/kleene/version.rb
 homepage: https://github.com/davidkellis/kleene-rb