kleene 0.4.0

data/lib/kleene/dsl.rb

@@ -0,0 +1,263 @@
+# Most of the machines constructed here are based on section 2.5 of the Ragel User Guide (http://www.colm.net/files/ragel/ragel-guide-6.10.pdf)
+
+module Kleene
+  module DSL
+    extend self
+
+    ############### The following methods create FSAs given a stream of input tokens #################
+
+    # given a string with N characters in it:
+    # N+1 states: start state and N other states
+    # structure: start state -> transition for first character in the string -> state for having observed first character in the string ->
+    #                           transition for second character in the string -> state for having observed second character in the string ->
+    #                           ...
+    #                           transition for last character in the string -> state for having observed last character in the string (marked final)
+    def literal(token_stream, alphabet = DEFAULT_ALPHABET)
+      start = current_state = State.new
+      nfa = NFA.new(start, alphabet)
+      token_stream.each_char do |token|
+        next_state = State.new
+        nfa.add_transition(token, current_state, next_state)
+        current_state = next_state
+      end
+      current_state.final = true
+      nfa.update_final_states
+      nfa.set_regex_pattern(token_stream)
+      nfa
+    end
+
+    # two states: start state and final state
+    # structure: start state -> transitions for each token in the token collection -> final state
+    def any(token_collection, alphabet = DEFAULT_ALPHABET)
+      start = State.new
+      nfa = NFA.new(start, alphabet)
+      final = State.new(true)
+      token_collection.each {|token| nfa.add_transition(token, start, final) }
+      nfa.update_final_states
+      regex_pattern = "[#{token_collection.join("")}]"
+      nfa.set_regex_pattern(regex_pattern)
+      nfa
+    end
+
+    # two states: start state and final state
+    # structure: start state -> transitions for every token in the alphabet -> final state
+    def dot(alphabet = DEFAULT_ALPHABET)
+      any(alphabet, alphabet).set_regex_pattern(".")
+    end
+
+    # This implements a character class, and is specifically for use with matching strings
+    def range(c_begin, c_end, alphabet = DEFAULT_ALPHABET)
+      any((c_begin..c_end).to_a, alphabet).set_regex_pattern("[#{c_begin}-#{c_end}]")
+    end
+
+    ############### The following methods create FSAs given other FSAs #################
+
+    # always clones the given nfa and returns a new nfa with a non-final error state
+    def with_err(nfa, alphabet = nfa.alphabet)
+      with_err!(nfa.deep_clone, alphabet)
+    end
+
+    # adds and error state to the NFA, create error transitions from all non-error states to the error state on any unhandled token.
+    # the error state transitions to itself on any token.
+    def with_err!(nfa, alphabet = nfa.alphabet)
+      error_state = nfa.states.find(&:error?)
+      return nfa if error_state
+
+      error_state = State.new_error_state
+      nfa.add_state(error_state)
+
+      nfa.states.each do |state|
+        tokens_on_outbound_transitions = nfa.transitions_from(state).map(&:token)
+        missing_tokens = alphabet - tokens_on_outbound_transitions
+        missing_tokens.each do |token|
+          nfa.add_transition(token, state, error_state)
+        end
+      end
+
+      nfa.remove_state(error_state) if nfa.all_transitions.none? {|transition| transition.from == error_state || transition.to == error_state }
+
+      nfa.set_regex_pattern("/#{nfa.regex_pattern}/E")
+    end
+
+    # always clones the given nfa and returns a new nfa with a non-final error state
+    def with_err_dead_end(nfa, alphabet = nfa.alphabet)
+      with_err_dead_end!(nfa.deep_clone, alphabet)
+    end
+
+    # adds and error state to the NFA, create error transitions from all non-error states to the error state on any unhandled token.
+    # the error state doesn't have any outbound transitions.
+    def with_err_dead_end!(nfa, alphabet = nfa.alphabet)
+      error_state = nfa.states.find(&:error?)
+      return nfa if error_state
+
+      error_state = State.new_error_state
+      nfa.add_state(error_state)
+
+      nfa.states.each do |state|
+        unless state.error?
+          tokens_on_outbound_transitions = nfa.transitions_from(state).map(&:token).to_set
+          only_outbound_transition_is_epsilon_transition = tokens_on_outbound_transitions.size == 1 && tokens_on_outbound_transitions.first == NFATransition::Epsilon
+          unless only_outbound_transition_is_epsilon_transition
+            missing_tokens = (alphabet - Set[NFATransition::Epsilon]) - tokens_on_outbound_transitions
+            missing_tokens.each do |token|
+              nfa.add_transition(token, state, error_state)
+            end
+          end
+        end
+      end
+
+      # remove the error state if it has no inbound or outbound transitions
+      nfa.remove_state(error_state) if nfa.all_transitions.none? {|transition| transition.from == error_state || transition.to == error_state }
+
+      nfa.set_regex_pattern("/#{nfa.regex_pattern}/DE")
+    end
+
+    # Append b onto a
+    # Appending produces a machine that matches all the strings in machine a followed by all the strings in machine b.
+    # This differs from `seq` in that the composite machine's final states are the union of machine a's final states and machine b's final states.
+    def append(a, b)
+      a = a.deep_clone
+      b = b.deep_clone
+      append!(a, b)
+    end
+
+    # Destructively append b onto a
+    # Appending produces a machine that matches all the strings in machine a followed by all the strings in machine b.
+    # This differs from `seq` in that the composite machine's final states are the union of machine a's final states and machine b's final states.
+    def append!(a, b)
+      a.alphabet = a.alphabet | b.alphabet
+
+      # add an epsilon transition from each final state of machine a to the start state of maachine b.
+      a.final_states.each do |final_state|
+        a.add_transition(NFATransition::Epsilon, final_state, b.start_state)
+      end
+
+      # add all of machine b's transitions to machine a
+      b.all_transitions.each {|transition| a.add_transition(transition.token, transition.from, transition.to) }
+      # a.final_states = a.final_states | b.final_states
+      a.update_final_states
+
+      a
+    end
+
+    def seq(*nfas)
+      nfas.flatten.reduce {|memo_nfa, nfa| seq2(memo_nfa, nfa) }
+    end
+
+    # Implements concatenation, as defined in the Ragel manual in section 2.5.5 of http://www.colm.net/files/ragel/ragel-guide-6.10.pdf:
+    # Seq produces a machine that matches all the strings in machine `a` followed by all the strings in machine `b`.
+    # Seq draws epsilon transitions from the final states of thefirst machine to the start state of the second machine.
+    # The final states of the first machine lose their final state status, unless the start state of the second machine is final as well.
+    def seq2(a, b)
+      a = a.deep_clone
+      b = b.deep_clone
+
+      a = append!(a, b)
+
+      # make sure that b's final states are the only final states in a after we have appended b onto a
+      a.states.each {|state| state.final = b.final_states.includes?(state) }
+      a.update_final_states
+
+      a.set_regex_pattern("#{a.regex_pattern}#{b.regex_pattern}")
+    end
+
+    # Build a new machine consisting of a new start state with epsilon transitions to the start state of all the given NFAs in `nfas`.
+    # The resulting machine's final states are the set of final states from all the NFAs in `nfas`.
+    #
+    # Implements Union, as defined in the Ragel manual in section 2.5.1 of http://www.colm.net/files/ragel/ragel-guide-6.10.pdf:
+    # The union operation produces a machine that matches any string in machine one or machine two.
+    # The operation first creates a new start state.
+    # Epsilon transitions are drawn from the new start state to the start states of both input machines.
+    # The resulting machine has a final state setequivalent to the union of the final state sets of both input machines.
+    def union(*nfas)
+      nfas.flatten!
+      nfas = nfas.map(&:deep_clone)
+      union!(nfas)
+    end
+
+    # same as union, but doesn't deep clone the constituent nfas
+    def union!(nfas)
+      start = State.new
+      composite_alphabet = nfas.map(&:alphabet).reduce {|memo, alphabet| memo | alphabet }
+      new_nfa = NFA.new(start, composite_alphabet)
+
+      # add epsilon transitions from the start state of the new machine to the start state of machines a and b
+      nfas.each do |nfa|
+        new_nfa.add_states(nfa.states)
+        new_nfa.add_transition(NFATransition::Epsilon, start, nfa.start_state)
+        nfa.all_transitions.each {|t| new_nfa.add_transition(t.token, t.from, t.to) }
+      end
+
+      new_nfa.update_final_states
+
+      new_nfa.set_regex_pattern("#{nfas.map(&:regex_pattern).join("|")}")
+    end
+
+    # Implements Kleene Star, as defined in the Ragel manual in section 2.5.6 of http://www.colm.net/files/ragel/ragel-guide-6.10.pdf:
+    # The machine resulting from the Kleene Star operator will match zero or more repetitions of the machine it is applied to.
+    # It creates a new start state and an additional final state.
+    # Epsilon transitions are drawn between the new start state and the old start state,
+    # between the new start state and the new final state, and between the final states of the machine and the new start state.
+    def kleene(machine)
+      machine = machine.deep_clone
+      start = State.new
+      final = State.new(true)
+
+      nfa = NFA.new(start, machine.alphabet)
+      nfa.add_states(machine.states)
+      nfa.add_transition(NFATransition::Epsilon, start, final)
+      nfa.add_transition(NFATransition::Epsilon, start, machine.start_state)
+      machine.final_states.each do |final_state|
+        nfa.add_transition(NFATransition::Epsilon, final_state, start)
+        final_state.final = false
+      end
+
+      # add all of machine's transitions to the new machine
+      (machine.all_transitions).each {|t| nfa.add_transition(t.token, t.from, t.to) }
+      nfa.update_final_states
+
+      nfa.set_regex_pattern("#{machine.regex_pattern}*")
+    end
+
+    def plus(machine)
+      seq(machine, kleene(machine)).set_regex_pattern("#{machine.regex_pattern}+")
+    end
+
+    def optional(machine)
+      empty = NFA.new(State.new(true), machine.alphabet).set_regex_pattern("")
+      union(machine, empty).set_regex_pattern("#{machine.regex_pattern}?")
+    end
+
+    # def repeat(machine, min, max = nil)
+    #   max ||= min
+    #   m = NFA.new(State.new(true), machine.alphabet)
+    #   min.times { m = seq(m, machine) }
+    #   (max - min).times { m = append(m, machine) }
+    #   if min != max
+    #     m.set_regex_pattern("#{machine.regex_pattern}{#{min},#{max}}")
+    #   else
+    #     m.set_regex_pattern("#{machine.regex_pattern}{#{min}}")
+    #   end
+    # end
+
+    # def negate(machine)
+    #   machine = machine.to_dfa
+
+    #   # invert the final flag of every state
+    #   machine.states.each {|state| state.final = !state.final? }
+    #   machine.update_final_states
+
+    #   machine.to_nfa.set_regex_pattern("(!#{machine.regex_pattern})")
+    # end
+
+    # # a - b == a && !b
+    # def difference(a, b)
+    #   intersection(a, negate(b))
+    # end
+
+    # # By De Morgan's Law: !(!a || !b) = a && b
+    # def intersection(a, b)
+    #   negate(union(negate(a), negate(b)))
+    # end
+  end
+end

data/lib/kleene/kleene.rb

@@ -0,0 +1,88 @@
+# this is a port and extension of https://github.com/davidkellis/kleene/
+
+require_relative "./dsl"
+require_relative "./nfa"
+require_relative "./dfa"
+
+module Kleene
+  # The default alphabet consists of the following:
+  # Set{' ', '!', '"', '#', '$', '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/',
+  #     '0', '1', '2', '3', '4', '5', '6', '7', '8', '9',
+  #     ':', ';', '<', '=', '>', '?', '@',
+  #     'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M',
+  #     'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z',
+  #     '[', '\\', ']', '^', '_', '`',
+  #     'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm',
+  #     'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z',
+  #     '{', '|', '}', '~', "\n", "\t"}
+  DEFAULT_ALPHABET = ((' '..'~').to_a + "\n\t".chars).to_set
+
+  class State
+    @@next_id = 0
+
+    def self.next_id
+      @@next_id += 1
+    end
+
+    def self.new_error_state(final = false)
+      State.new(final, true)
+    end
+
+
+    attr_reader :id # : Int32
+    attr_accessor :final # : Bool
+    attr_accessor :error # : Bool
+
+    def initialize(final = false, error = false, id = nil)
+      @final = final
+      @error = error
+      @id = id || State.next_id
+    end
+
+    # is this an error state?
+    def error?
+      @error
+    end
+
+    # is this a final state?
+    def final?
+      @final
+    end
+
+    def dup
+      State.new(@final, @error, nil)
+    end
+
+    def to_s
+      "State{id: #{id}, final: #{final}, error: #{error}}"
+    end
+  end
+
+  class MatchRef
+    attr_accessor :string # : String
+    attr_accessor :range # : Range(Int32, Int32)
+
+    def initialize(original_string, match_range)
+      @string = original_string
+      @range = match_range
+    end
+
+    def text
+      @string[@range]
+    end
+
+    def to_s
+      text
+    end
+
+    def ==(other)
+      @string == other.string &&
+      @range == other.range
+    end
+
+    def eql?(other)
+      self == other
+    end
+  end
+
+end

data/lib/kleene/multi_match_dfa.rb

@@ -0,0 +1,308 @@
+require_relative "./kleene"
+
+module Kleene
+  class MachineTuple
+    attr_accessor :nfa # : NFA
+    attr_accessor :nfa_with_dead_err # : NFA
+    attr_accessor :dfa # : DFA
+
+    def initialize(nfa, nfa_with_dead_err, dfa)
+      @nfa, @nfa_with_dead_err, @dfa = nfa, nfa_with_dead_err, dfa
+    end
+  end
+
+  class MultiMatchDFA
+    include DSL
+
+    # @original_nfas : Array(NFA)
+    attr_reader :nfas_with_err_state # : Array(NFA)
+    attr_accessor :dead_end_nfa_state_to_dead_end_nfa # : Hash(State, NFA)
+    attr_accessor :composite_nfa # : NFA
+    attr_accessor :composite_dfa # : DFA
+
+    attr_accessor :machines_by_index # : Hash(Int32, MachineTuple)
+    attr_accessor :nfa_to_index # : Hash(NFA, Int32)
+    attr_accessor :nfa_with_dead_err_to_index # : Hash(NFA, Int32)
+    attr_accessor :dfa_to_index # : Hash(DFA, Int32)
+
+    def initialize(nfas)
+      composite_alphabet = nfas.reduce(Set.new) {|memo, nfa| memo | nfa.alphabet }
+
+      @original_nfas = nfas
+      @nfas_with_err_state = nfas.map {|nfa| with_err_dead_end(nfa, composite_alphabet) }     # copy NFAs and add dead-end error states to each of them
+      dfas = @original_nfas.map(&:to_dfa)
+
+      @nfa_to_index = @original_nfas.map.with_index {|nfa, index| [nfa, index] }.to_h
+      @nfa_with_dead_err_to_index = @nfas_with_err_state.map.with_index {|nfa, index| [nfa, index] }.to_h
+      @dfa_to_index = dfas.map.with_index {|dfa, index| [dfa, index] }.to_h
+      @machines_by_index = @original_nfas.zip(nfas_with_err_state, dfas).map.with_index {|tuple, index| nfa, nfa_with_dead_err, dfa = tuple; [index, MachineTuple.new(nfa, nfa_with_dead_err, dfa)] }.to_h
+
+      # build a mapping of (state -> nfa) pairs that capture which nfa owns each state
+      @dead_end_nfa_state_to_dead_end_nfa = Hash.new
+      @nfas_with_err_state.each do |nfa_with_dead_err|
+        nfa_with_dead_err.states.each do |state|
+          @dead_end_nfa_state_to_dead_end_nfa[state] = nfa_with_dead_err
+        end
+      end
+
+      # create a composite NFA as the union of all the NFAs with epsilon transitions from every NFA state back to the union NFA's start state
+      @composite_nfa = create_composite_nfa(@nfas_with_err_state)
+      @composite_dfa = @composite_nfa.to_dfa
+    end
+
+    def machines_from_nfa(nfa) # : MachineTuple
+      machines_by_index[nfa_to_index[nfa]]
+    end
+
+    def machines_from_nfa_with_dead_err(nfa_with_dead_err) # : MachineTuple
+      machines_by_index[nfa_with_dead_err_to_index[nfa_with_dead_err]]
+    end
+
+    def machines_from_dfa(dfa) # : MachineTuple
+      machines_by_index[dfa_to_index[dfa]]
+    end
+
+    # create a composite NFA as the union of all the NFAs with epsilon transitions from every NFA state back to the union NFA's start state
+    def create_composite_nfa(nfas)
+      nfa = union!(nfas)
+
+      # add epsilon transitions from all the states except the start state back to the start state
+      nfa.states.each do |state|
+        if state != nfa.start_state
+          nfa.add_transition(NFATransition::Epsilon, state, nfa.start_state)
+        end
+      end
+
+      nfa.update_final_states
+
+      nfa
+    end
+
+    def match_tracker(input) # : MatchTracker
+      dfa = @composite_dfa.deep_clone
+      match_tracker = setup_callbacks(dfa)
+
+      input.each_char.with_index do |char, index|
+        dfa.handle_token!(char, index)
+      end
+
+      match_tracker
+    end
+
+    def matches(input) # : Hash(NFA, Array(MatchRef))
+      mt = match_tracker(input)
+
+      start_index_to_nfas_that_may_match = mt.invert_candidate_match_start_positions
+
+      mt.empty_matches.each do |nfa_with_dead_err, indices|
+        original_nfa = machines_from_nfa_with_dead_err(nfa_with_dead_err).nfa
+        indices.each do |index|
+          mt.add_match(original_nfa, MatchRef.new(input, index...index))
+        end
+      end
+
+      active_dfas = Array.new    # the Int32 represents the start of match
+
+      input.each_char.with_index do |char, index|
+        active_dfas.reject! do |active_dfa_tuple|
+          dfa_clone, original_nfa, start_of_match_index = active_dfa_tuple
+
+          dfa_clone.handle_token!(char, index)
+          mt.add_match(original_nfa, MatchRef.new(input, start_of_match_index..index)) if dfa_clone.accept?
+
+          dfa_clone.error?
+        end
+
+        if nfas_with_dead_err = start_index_to_nfas_that_may_match[index]
+          nfas_with_dead_err.each do |nfa_with_dead_err|
+            machines = machines_from_nfa_with_dead_err(nfa_with_dead_err)
+            original_nfa = machines.nfa
+            dfa = machines.dfa
+            dfa_clone = dfa.shallow_clone
+
+            dfa_clone.handle_token!(char, index)
+            mt.add_match(original_nfa, MatchRef.new(input, index..index)) if dfa_clone.accept?
+
+            active_dfas << [dfa_clone, original_nfa, index] unless dfa_clone.error?
+          end
+        end
+      end
+
+      mt.matches
+    end
+
+    def setup_callbacks(dfa)
+      match_tracker = MatchTracker.new
+
+      # 1. identify DFA states that correspond to successful match of first character of the NFAs
+      epsilon_closure_of_nfa_start_state = composite_nfa.epsilon_closure(composite_nfa.start_state)
+      nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa = composite_nfa.transitions_from(epsilon_closure_of_nfa_start_state).
+                                                                                                         reject {|transition| transition.epsilon? || transition.to.error? }.
+                                                                                                         map(&:to).to_set
+      dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa = nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.
+                                                                                              compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                                              reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_state_to_dead_end_nfas_that_have_matched_their_first_character = Hash.new
+      dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.each do |dfa_state|
+        dfa_state_to_dead_end_nfas_that_have_matched_their_first_character[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                                            select {|nfa_state| nfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.includes?(nfa_state) }.
+                                                                                            compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # 2. identify DFA states that correspond to final states in the NFAs
+      nfa_final_states = @nfas_with_err_state.map(&:final_states).reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_states_that_correspond_to_nfa_final_states = nfa_final_states.compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                        reduce(Set.new) {|memo, state_set| memo | state_set }
+      dead_end_nfas_that_have_transitioned_to_final_state = Hash.new
+      dfa_states_that_correspond_to_nfa_final_states.each do |dfa_state|
+        dead_end_nfas_that_have_transitioned_to_final_state[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                             select {|nfa_state| nfa_final_states.includes?(nfa_state) }.
+                                                                             compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # 3. Identify DFA states that correspond to successful match without even having seen any characters.
+      #    These are cases where the NFA's start state is a final state or can reach a final state by following only epsilon transitions.
+      nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state = epsilon_closure_of_nfa_start_state.select(&:final?).to_set
+      dfa_states_that_represent_both_start_states_and_final_states = nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state.
+                                                                        compact_map {|nfa_state| dfa.nfa_state_to_dfa_state_sets[nfa_state] }.
+                                                                        reduce(Set.new) {|memo, state_set| memo | state_set }
+      dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters = Hash.new
+      dfa_states_that_represent_both_start_states_and_final_states.each do |dfa_state|
+        dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[dfa_state] = dfa.dfa_state_to_nfa_state_sets[dfa_state].
+                                                                                                     select {|nfa_state| nfa_final_states_that_are_epsilon_reachable_from_nfa_start_state.includes?(nfa_state) }.
+                                                                                                     compact_map do |nfa_state|
+          dead_end_nfa_state_to_dead_end_nfa[nfa_state] unless nfa_state == composite_nfa.start_state    # composite_nfa.start_state is not referenced in the dead_end_nfa_state_to_dead_end_nfa map
+        end.to_set
+      end
+
+      # set up call transition call backs, since the callbacks may only be defined once per state and transition
+      # For (1):
+      #    Set up transition callbacks to push the index position of the start of a match of each NFA that has begun
+      #    to be matched on the transition to one of the states in (1)
+      # For (2):
+      #    set up transition callbacks to push the index position of the end of a successful match onto the list
+      #    of successful matches for the NFA that matched
+      # For (3):
+      #    set up transision callbacks to capture successful empty matches
+      destination_dfa_states_for_callbacks = dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa | dfa_states_that_correspond_to_nfa_final_states
+      destination_dfa_states_for_callbacks.each do |dfa_state|
+        dfa.on_transition_to(dfa_state) do |transition, token, token_index|
+          destination_dfa_state = transition.to
+
+          should_track_empty_match = dfa_states_that_represent_both_start_states_and_final_states.includes?(destination_dfa_state)
+          should_track_start_of_candidate_match = should_track_empty_match || dfa_states_that_correspond_to_successful_match_of_first_character_of_component_nfa.includes?(destination_dfa_state)
+          should_track_end_of_match = dfa_states_that_correspond_to_nfa_final_states.includes?(destination_dfa_state)
+
+          if should_track_empty_match
+            dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[destination_dfa_state].each do |nfa_with_dead_end|
+              match_tracker.add_empty_match(nfa_with_dead_end, token_index)
+            end
+          end
+
+          if should_track_start_of_candidate_match
+            nfas_that_matched_first_character = dfa_state_to_dead_end_nfas_that_have_matched_their_first_character[destination_dfa_state] || Set.new
+            nfas_that_matched_empty_match = dfa_state_to_dead_end_nfas_that_have_matched_before_handling_any_characters[destination_dfa_state] || Set.new
+            dead_end_nfas_that_are_starting_to_match = nfas_that_matched_first_character | nfas_that_matched_empty_match
+            dead_end_nfas_that_are_starting_to_match.each do |nfa_with_dead_end|
+              match_tracker.add_start_of_candidate_match(nfa_with_dead_end, token_index)
+            end
+          end
+
+          if should_track_end_of_match
+            dead_end_nfas_that_have_transitioned_to_final_state[destination_dfa_state].each do |nfa_with_dead_end|
+              match_tracker.add_end_of_match(nfa_with_dead_end, token_index)
+            end
+          end
+        end
+      end
+
+      match_tracker
+    end
+
+  end
+
+  class MatchTracker
+    # The NFA keys in the following two structures are not the original NFAs supplied to the MultiMatchDFA.
+    # They are the original NFAs that have been augmented with a dead end error state, so the keys are objects that
+    # are the internal state of a MultiMatchDFA
+    attr_accessor :candidate_match_start_positions # : Hash(NFA, Array(Int32))     # NFA -> Array(IndexPositionOfStartOfMatch)
+    #  The end positions are indices at which, after handling the character, the DFA was observed to be in a match/accept state;
+    #  however, the interpretation is ambiguous, because the accepting state may be as a result of (1) transitioning to an error state that is also marked final/accepting,
+    #  OR it may be as a result of transitioning to (2) a non-error final state.
+    #  In the case of (1), the match may be an empty match, where after transitioning to an error state, the DFA is in a state that
+    #  is equivalent to the error state and start state and final state (e.g. as in an optional or kleene star DFA),
+    #  while in the case of (2), the match may be a "normal" match.
+    #  The ambiguity is problematic because it isn't clear whether the index position of the match is end inclusive end of a match
+    #  or the beginning of an empty match.
+    #  This ambiguity is all due to the construction of the composite DFA in the MultiMatchDFA - the dead end error states are epsilon-transitioned
+    #  to the composite DFA's start state.
+    attr_accessor :match_end_positions # : Hash(NFA, Array(Int32))                 # NFA -> Array(IndexPositionOfEndOfMatch)
+    attr_accessor :empty_matches # : Hash(NFA, Array(Int32))                       # NFA -> Array(IndexPositionOfEmptyMatch)
+
+    # The NFA keys in the following structure are the original NFAs supplied to the MultiMatchDFA.
+    # This is in contrast to the augmented NFAs that are used as keys in the candidate_match_start_positions and
+    # match_end_positions structures, documented above ^^^.
+    attr_accessor :matches # : Hash(NFA, Array(MatchRef))  # NFA -> Array(MatchRef)
+
+    def initialize
+      @candidate_match_start_positions = Hash.new
+      @match_end_positions = Hash.new
+      @empty_matches = Hash.new
+      @matches = Hash.new
+    end
+
+    def start_positions(nfa)
+      candidate_match_start_positions[nfa] ||= Array.new
+    end
+
+    def end_positions(nfa)
+      match_end_positions[nfa] ||= Array.new
+    end
+
+    def empty_match_positions(nfa)
+      empty_matches[nfa] ||= Array.new
+    end
+
+    def matches_for(nfa)
+      matches[nfa] ||= Array.new
+    end
+
+    def add_start_of_candidate_match(nfa_with_dead_end, token_index)
+      # puts "add_start_of_candidate_match(#{nfa.object_id}, #{token_index})"
+      positions = start_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    #  the end positions are inclusive of the index of the last character matched, so empty matches are not accounted for in the match_end_positions array
+    def add_end_of_match(nfa_with_dead_end, token_index)
+      # puts "add_end_of_match(#{nfa.object_id}, #{token_index})"
+      positions = end_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    def add_empty_match(nfa_with_dead_end, token_index)
+      positions = empty_match_positions(nfa_with_dead_end)
+      positions << token_index
+    end
+
+    def invert_candidate_match_start_positions # : Hash(Int32, Array(NFA))
+      index_to_nfas = Hash.new
+      candidate_match_start_positions.each do |nfa_with_dead_end, indices|
+        indices.each do |index|
+          nfas = index_to_nfas[index] ||= Array.new
+          nfas << nfa_with_dead_end
+        end
+      end
+      index_to_nfas
+    end
+
+    def add_match(nfa, match)
+      matches = matches_for(nfa)
+      matches << match
+    end
+  end
+end