flexr 1.0.0 → 1.1.0

data/lib/flexr/automaton/dfa.rb

@@ -2,42 +2,81 @@
 
 module Flexr
   module Automaton
+    class TransitionRows
+      include Enumerable
+
+      attr_reader :length
+
+      def initialize(length, &loader)
+        @length = length
+        @loader = loader
+        @rows = Array.new(length)
+        freeze
+      end
+
+      alias size length
+
+      def [](state)
+        return unless state&.between?(0, length - 1)
+
+        @rows[state] ||= @loader.call(state).freeze
+      end
+
+      def fetch(state)
+        return self[state] if state.between?(0, length - 1)
+
+        raise IndexError, "transition row #{state} is out of bounds"
+      end
+
+      def each
+        return enum_for(__method__) unless block_given?
+
+        length.times { |state| yield self[state] }
+      end
+
+      def each_index(&)
+        return enum_for(__method__) unless block_given?
+
+        length.times(&)
+      end
+
+      def materialized_count
+        @rows.count { |row| !row.nil? }
+      end
+    end
+
     class DFA
       attr_reader :transitions, :accepts, :ec, :class_count, :start, :states, :rule_ids, :packed, :direct
 
-      def initialize(transitions:, accepts:, ec:, class_count:, start:, rule_ids:, packed: nil, direct: nil)
-        @transitions = transitions.freeze
+      def initialize(accepts:, ec:, class_count:, start:, rule_ids:, transitions: nil, packed: nil, direct: nil,
+                     state_count: nil)
+        @packed = freeze_representation(packed)
+        @direct = freeze_representation(direct)
+        @states = state_count || transitions&.length || direct_state_count
+        raise ArgumentError, "state_count is required without dense transitions" unless @states
+
+        @transitions = transition_rows(transitions)
         @accepts = accepts.map do |rules|
           rules.map do |acceptance|
-            next acceptance if acceptance.is_a?(Acceptance)
+            next acceptance.freeze if acceptance.is_a?(Acceptance)
 
             Acceptance.new(rule_index: acceptance[0], pattern_index: acceptance[1],
-                           bol_only: acceptance[2], end_anchor: acceptance[3])
+                           bol_only: acceptance[2], end_anchor: acceptance[3]).freeze
           end.freeze
         end.freeze
         @ec = ec.freeze
         @class_count = class_count
         @start = start
-        @states = transitions.length
         @rule_ids = rule_ids.map { |id| id.respond_to?(:rule_index) ? id.rule_index : id }.uniq.sort.freeze
-        @packed = packed
-        @direct = direct
+        freeze
       end
 
       def transition(state, byte)
         class_id = @ec[byte]
+        return direct_transition(state, class_id) if @direct
         return @transitions[state][class_id] unless @packed
 
-        cursor = state
-        loop do
-          index = @packed.fetch(:base).fetch(cursor) + class_id
-          return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
-
-          fallback = @packed[:fallback]&.fetch(cursor)
-          return @packed.fetch(:default).fetch(cursor) unless fallback
-
-          cursor = fallback
-        end
+        packed_transition(state, class_id)
       end
 
       # Generated direct lexers use a flattened dispatch representation. The
@@ -45,8 +84,7 @@ module Flexr
       def transition_direct(state, byte)
         if @direct
           class_id = @ec[byte]
-          value = @direct.fetch(:nxt).fetch((state * @direct.fetch(:classes)) + class_id)
-          return value >= 0 ? value : nil
+          return direct_transition(state, class_id)
         end
 
         @transitions[state][@ec[byte]]
@@ -65,6 +103,55 @@ module Flexr
       def stats
         { states: states, classes: class_count, accepting_states: accepts.count { |rules| !rules.empty? } }
       end
+
+      private
+
+      def transition_rows(transitions)
+        return transitions.map(&:freeze).freeze if transitions
+        if @packed
+          return TransitionRows.new(states) do |state|
+            Array.new(class_count) { |class_id| packed_transition(state, class_id) }
+          end
+        end
+        if @direct
+          return TransitionRows.new(states) do |state|
+            Array.new(class_count) { |class_id| direct_transition(state, class_id) }
+          end
+        end
+
+        raise ArgumentError, "a transition representation is required"
+      end
+
+      def packed_transition(state, class_id)
+        cursor = state
+        loop do
+          index = @packed.fetch(:base).fetch(cursor) + class_id
+          return @packed.fetch(:next).fetch(index) if @packed.fetch(:check)[index] == cursor
+
+          fallback = @packed[:fallback]&.fetch(cursor)
+          return @packed.fetch(:default).fetch(cursor) unless fallback
+
+          cursor = fallback
+        end
+      end
+
+      def direct_transition(state, class_id)
+        value = @direct.fetch(:nxt).fetch((state * @direct.fetch(:classes)) + class_id)
+        value >= 0 ? value : nil
+      end
+
+      def direct_state_count
+        return unless @direct
+
+        @direct.fetch(:nxt).length / @direct.fetch(:classes)
+      end
+
+      def freeze_representation(representation)
+        return unless representation
+
+        representation.each_value { |value| value.freeze if value.respond_to?(:freeze) }
+        representation.freeze
+      end
     end
 
     class ReferenceDFA

data/lib/flexr/automaton/minimizer.rb

@@ -6,65 +6,113 @@ module Flexr
       module_function
 
       def minimize(dfa)
-        partitions = initial_partitions(dfa)
-        loop do
-          groups = group_ids(partitions)
-          refined = partitions.flat_map do |partition|
-            partition.group_by do |state|
-              [dfa.accepts[state], dfa.transitions[state].map { |destination| destination.nil? ? -1 : groups[destination] }]
-            end.values
+        dead = dfa.states
+        rows = dfa.transitions.map { |row| row.map { |destination| destination.nil? ? dead : destination } }
+        rows << Array.new(dfa.class_count, dead)
+        partitions = initial_partitions(dfa, dead)
+        predecessors = predecessor_index(rows, dfa.class_count)
+        partitions = refine(partitions, predecessors, dfa.class_count)
+        rebuild(dfa, partitions, dead)
+      end
+
+      def initial_partitions(dfa, dead)
+        (0..dead).group_by { |state| state == dead ? [] : dfa.accepts[state] }.values
+      end
+
+      def predecessor_index(rows, class_count)
+        Array.new(class_count) { Array.new(rows.length) { [] } }.tap do |index|
+          rows.each_with_index do |row, state|
+            row.each_with_index { |destination, class_id| index[class_id][destination] << state }
           end
-          break if refined == partitions
+        end
+      end
 
-          partitions = refined
+      def refine(partitions, predecessors, class_count)
+        worklist = partitions.dup
+        pending = {}.compare_by_identity
+        partitions.each { |partition| pending[partition] = true }
+        cursor = 0
+        while cursor < worklist.length
+          splitter = worklist.fetch(cursor)
+          cursor += 1
+          next unless pending.delete(splitter)
+
+          class_count.times do |class_id|
+            marked = splitter.each_with_object({}) do |state, result|
+              predecessors[class_id][state].each { |predecessor| result[predecessor] = true }
+            end
+            next if marked.empty?
+
+            partitions = split_partitions(partitions, marked, worklist, pending)
+          end
         end
-        rebuild(dfa, partitions)
+        partitions
       end
 
-      def initial_partitions(dfa)
-        (0...dfa.states).group_by { |state| dfa.accepts[state] }.values
+      def split_partitions(partitions, marked, worklist, pending)
+        partitions.flat_map do |partition|
+          inside, outside = partition.partition { |state| marked[state] }
+          next [partition] if inside.empty? || outside.empty?
+
+          if pending.delete(partition)
+            enqueue(inside, worklist, pending)
+            enqueue(outside, worklist, pending)
+          else
+            enqueue(inside.length <= outside.length ? inside : outside, worklist, pending)
+          end
+          [inside, outside]
+        end
       end
 
-      def group_ids(partitions)
-        ids = {}
-        partitions.each_with_index { |partition, id| partition.each { |state| ids[state] = id } }
-        ids
+      def enqueue(partition, worklist, pending)
+        worklist << partition
+        pending[partition] = true
       end
 
-      def rebuild(dfa, partitions)
+      def rebuild(dfa, partitions, dead)
         groups = group_ids(partitions)
+        dead_group = groups.fetch(dead)
         start_group = groups.fetch(dfa.start)
-        order = bfs_groups(dfa, groups, start_group)
+        representatives = partitions.map { |partition| partition.find { |state| state != dead } }
+        order = reachable_groups(dfa, groups, representatives, start_group, dead_group)
         index = order.each_with_index.to_h
         transitions = order.map do |group|
-          representative = partitions[group].first
-          dfa.transitions[representative].map { |destination| destination.nil? ? nil : index[groups[destination]] }
+          representative = representatives.fetch(group)
+          dfa.transitions.fetch(representative).map do |destination|
+            next nil if destination.nil? || groups.fetch(destination) == dead_group
+
+            index.fetch(groups.fetch(destination))
+          end
         end
-        accepts = order.map { |group| dfa.accepts[partitions[group].first] }
+        accepts = order.map { |group| dfa.accepts.fetch(representatives.fetch(group)) }
         rule_ids = accepts.flatten.map(&:rule_index).uniq.sort
         DFA.new(transitions: transitions, accepts: accepts, ec: dfa.ec, class_count: dfa.class_count,
                 start: 0, rule_ids: rule_ids)
       end
 
-      def bfs_groups(dfa, groups, start_group)
+      def group_ids(partitions)
+        partitions.each_with_index.with_object({}) do |(partition, group), result|
+          partition.each { |state| result[state] = group }
+        end
+      end
+
+      def reachable_groups(dfa, groups, representatives, start_group, dead_group)
         result = []
         queue = [start_group]
         seen = {}
-        until queue.empty?
-          group = queue.shift
-          next if seen[group]
+        cursor = 0
+        while cursor < queue.length
+          group = queue.fetch(cursor)
+          cursor += 1
+          next if seen[group] || (group == dead_group && group != start_group)
 
           seen[group] = true
           result << group
-          representative = dfa.transitions[group_members(dfa, groups, group).first]
-          representative.compact.each { |destination| queue << groups[destination] }
+          representative = representatives.fetch(group)
+          dfa.transitions.fetch(representative).compact.each { |destination| queue << groups.fetch(destination) }
         end
         result
       end
-
-      def group_members(dfa, groups, group)
-        (0...dfa.states).select { |state| groups[state] == group }
-      end
     end
   end
 end

data/lib/flexr/automaton/nfa.rb

@@ -4,11 +4,6 @@ module Flexr
   module Automaton
     NFAState = Struct.new(:epsilon, :transitions, :accepts, keyword_init: true)
     NFATransition = Struct.new(:lo, :hi, :to, keyword_init: true)
-    Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
-      def inspect
-        [rule_index, pattern_index, bol_only, end_anchor].inspect
-      end
-    end
 
     class NFA
       attr_reader :states, :start, :byte_classes
@@ -56,6 +51,8 @@ module Flexr
         when Regexp::AST::Empty
           state = @nfa.new_state
           [state, state]
+        when Regexp::AST::Fail
+          [@nfa.new_state, @nfa.new_state]
         when Regexp::AST::ByteRange
           from = @nfa.new_state
           to = @nfa.new_state
@@ -75,18 +72,53 @@ module Flexr
           end
           [from, to]
         when Regexp::AST::Star
-          from = @nfa.new_state
-          to = @nfa.new_state
-          child_start, child_end = fragment(node.child)
-          @nfa.epsilon(from, to)
-          @nfa.epsilon(from, child_start)
-          @nfa.epsilon(child_end, child_start)
-          @nfa.epsilon(child_end, to)
-          [from, to]
+          repeat_fragment(node.child, 0, nil)
+        when Regexp::AST::Repeat
+          repeat_fragment(node.child, node.minimum, node.maximum)
         else
           raise CompileError, "cannot build NFA from #{node.class}"
         end
       end
+
+      def repeat_fragment(child, minimum, maximum)
+        return fragment(Regexp::AST::Empty.new(loc: nil)) if maximum&.zero?
+        return star_fragment(child) if minimum.zero? && maximum.nil?
+
+        from = @nfa.new_state
+        cursor = from
+        last_start = nil
+        minimum.times do
+          child_start, child_end = fragment(child)
+          @nfa.epsilon(cursor, child_start)
+          cursor = child_end
+          last_start = child_start
+        end
+        if maximum.nil?
+          @nfa.epsilon(cursor, last_start)
+          return [from, cursor]
+        end
+
+        (maximum - minimum).times do
+          child_start, child_end = fragment(child)
+          next_cursor = @nfa.new_state
+          @nfa.epsilon(cursor, next_cursor)
+          @nfa.epsilon(cursor, child_start)
+          @nfa.epsilon(child_end, next_cursor)
+          cursor = next_cursor
+        end
+        [from, cursor]
+      end
+
+      def star_fragment(child)
+        from = @nfa.new_state
+        to = @nfa.new_state
+        child_start, child_end = fragment(child)
+        @nfa.epsilon(from, to)
+        @nfa.epsilon(from, child_start)
+        @nfa.epsilon(child_end, child_start)
+        @nfa.epsilon(child_end, to)
+        [from, to]
+      end
     end
   end
 end

data/lib/flexr/automaton/types.rb

@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Automaton
+    Acceptance = Struct.new(:rule_index, :pattern_index, :bol_only, :end_anchor, keyword_init: true) do
+      def inspect
+        [rule_index, pattern_index, bol_only, end_anchor].inspect
+      end
+    end
+
+    CompiledSpec = Struct.new(:machines, :rules, :states, :stats, :diagnostics, keyword_init: true)
+    Machine = Struct.new(:dfa, :state_name, keyword_init: true)
+  end
+end

data/lib/flexr/cli.rb

@@ -1,5 +1,11 @@
 # frozen_string_literal: true
 
+require_relative "runtime" unless defined?(Flexr::Lexer)
+require_relative "artifact_writer"
+require_relative "generator"
+require_relative "importer"
+require_relative "options"
+
 module Flexr
   module CLI
     COMMANDS = %w[check stats tokens dot explain trace bench import].freeze
@@ -121,13 +127,13 @@ module Flexr
         return EXIT_FAILURE unless result.complete?
 
         if output
-          File.binwrite(output, result.source)
+          ArtifactWriter.write!(output, result.source, source_path: spec)
         else
           out.write(result.source)
         end
         EXIT_OK
       when :generate
-        target = output || spec.sub(/\.flexr\.rb\z/, ".rb")
+        target = output || ArtifactWriter.default_generated_path(spec)
         Generator.new(spec, output: target, eval_mode: options.eval_mode,
                       options: options.generator_options).generate
         EXIT_OK
@@ -142,7 +148,7 @@ module Flexr
                  else
                    compile(read_spec(spec), overrides: options.overrides)
                  end
-      diagnostics = compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)
+      diagnostics = (compiled.is_a?(Array) ? compiled : Array(compiled&.diagnostics)).dup
       diagnostics.select! do |diagnostic|
         options.warn_level == :all || (options.warn_level == :default && diagnostic.code != "FLEXR-W016")
       end
@@ -275,8 +281,8 @@ module Flexr
       klass.encoding(config.fetch(:encoding, Encoding::UTF_8))
       Array(config[:declared_tokens]).each { |token| klass.emits(token) }
       config_options = config.fetch(:options, {}).merge(overrides.slice(:experimental, :allow_empty_match))
-      config_options.each do |name, value|
-        value ? klass.option(name) : nil
+      config_options.slice(*Configuration::BOOLEAN_OPTIONS).each do |name, value|
+        klass.option(name) if value
       end
       klass.__flexr_config.options[:max_dfa_states] = overrides[:max_dfa_states] if overrides[:max_dfa_states]
       accel = overrides.fetch(:accel, config_options[:accel])

data/lib/flexr/codegen/direct.rb

@@ -4,49 +4,13 @@ module Flexr
   module Codegen
     class Direct < Table
       def generate(state: :initial)
-        table = super
-        table.merge(dispatch: :case).freeze
+        dfa = compiled.machines.fetch(state).dfa
+        {
+          nxt: dfa.transitions.flatten.map { |value| value || -1 }.freeze,
+          classes: dfa.class_count
+        }.freeze
       end
 
-      def source(indent: "  ")
-        lines = []
-        lines << "#{indent}def self.__flexr_generated_direct_transition(state_name, state, byte)"
-        lines << "#{indent}  case state_name"
-        compiled.machines.each do |state_name, machine|
-          lines << "#{indent}  when #{state_name.inspect}"
-          lines << "#{indent}    class_id = case byte"
-          byte_classes(machine.dfa.ec).each do |first, last, class_id|
-            selector = first == last ? first.to_s : "#{first}..#{last}"
-            lines << "#{indent}                 when #{selector} then #{class_id}"
-          end
-          lines << "#{indent}                 end"
-          lines << "#{indent}    case state"
-          machine.dfa.transitions.each_with_index do |row, state|
-            lines << "#{indent}    when #{state}"
-            lines << "#{indent}      case class_id"
-            row.each_with_index do |destination, class_id|
-              value = destination.nil? ? "nil" : destination
-              lines << "#{indent}      when #{class_id} then #{value}"
-            end
-            lines << "#{indent}      else nil"
-            lines << "#{indent}      end"
-          end
-          lines << "#{indent}    else nil"
-          lines << "#{indent}    end"
-        end
-        lines << "#{indent}  else nil"
-        lines << "#{indent}  end"
-        lines << "#{indent}end"
-        "#{lines.join("\n")}\n"
-      end
-
-      private
-
-      def byte_classes(ec)
-        ec.each_with_index.chunk_while { |left, right| right[0] == left[0] }.map do |run|
-          [run.first[1], run.last[1], run.first[0]]
-        end
-      end
     end
   end
 end