flexr 1.0.0

data/lib/flexr/codegen/table.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Codegen
+    class Table < Base
+      def generate(state: :initial)
+        machine = compiled.machines.fetch(state)
+        dfa = machine.dfa
+        {
+          ec: dfa.ec,
+          nxt: dfa.transitions.flatten.map { |value| value || -1 },
+          acc: dfa.accepts.map { |rules| rules.first&.rule_index || -1 },
+          start: dfa.start,
+          states: dfa.states,
+          classes: dfa.class_count
+        }.freeze
+      end
+
+      def header(source: nil, backend: :table)
+        stat = stats.fetch(:initial, {})
+        [
+          "# Generated by flexr. DO NOT EDIT.",
+          "# source: #{source}",
+          "# backend: #{backend} / states: #{stat[:states]} / byte-classes: #{stat[:classes]}"
+        ].join("\n")
+      end
+
+      def source(indent: "  ")
+        lines = []
+        lines << "#{indent}def scan_one"
+        lines << "#{indent}  return Flexr::Runtime::Interpreter.new(self).scan unless __flexr_generated_fast_path?"
+        lines << "#{indent}  machine = self.class.__flexr_compiled.machines.fetch(state)"
+        lines << "#{indent}  dfa = machine.dfa"
+        lines << "#{indent}  position = byte_pos"
+        lines << "#{indent}  return nil unless valid_utf8_at?(position)"
+        lines << "#{indent}  cursor = position"
+        lines << "#{indent}  current = dfa.start"
+        lines << "#{indent}  direct = dfa.direct"
+        lines << "#{indent}  source = buffer.source"
+        simple = if simple_fast_path?
+          lines << "#{indent}  best = nil"
+          true
+        else
+          lines << "#{indent}  best = __flexr_generated_acceptance(dfa, current, position, position, nil)"
+          false
+        end
+        lines << "#{indent}  accelerate = self.class.__flexr_config.options.fetch(:accel, :auto) != :none && !utf8_input?"
+        lines << "#{indent}  while cursor < source.bytesize || buffer.ensure_available?(cursor + 1)"
+        lines << "#{indent}    if accelerate"
+        lines << "#{indent}      region = __flexr_generated_acceleration_region(dfa, current)"
+        lines << "#{indent}      accelerated_end = __flexr_generated_accelerate(region, cursor) if region"
+        lines << "#{indent}      if accelerated_end && accelerated_end > cursor"
+        lines << "#{indent}        cursor = accelerated_end"
+        lines << "#{indent}        best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
+        lines << "#{indent}        next"
+        lines << "#{indent}      end"
+        lines << "#{indent}    end"
+        lines << "#{indent}    byte = source.getbyte(cursor)"
+        lines << "#{indent}    current = if direct"
+        lines << "#{indent}      class_id = dfa.ec[byte]"
+        lines << "#{indent}      value = direct[:nxt][(current * direct[:classes]) + class_id]"
+        lines << "#{indent}      value >= 0 ? value : nil"
+        lines << "#{indent}    else"
+        lines << "#{indent}      dfa.transition(current, byte)"
+        lines << "#{indent}    end"
+        lines << "#{indent}    break unless current"
+        lines << "#{indent}    cursor += 1"
+        if simple
+          lines << "#{indent}    acceptance = dfa.accepts[current].first"
+          lines << "#{indent}    if acceptance"
+          lines << "#{indent}      rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
+          lines << "#{indent}      best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
+          lines << "#{indent}      best.rule = rule"
+          lines << "#{indent}      best.start_pos = position"
+          lines << "#{indent}      best.end_pos = cursor"
+          lines << "#{indent}      best.total_end_pos = cursor"
+          lines << "#{indent}    end"
+        else
+          lines << "#{indent}    best = __flexr_generated_acceptance(dfa, current, position, cursor, best)"
+        end
+        lines << "#{indent}  end"
+        lines << "#{indent}  best"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_acceleration_region(dfa, state)"
+        lines << "#{indent}  @__flexr_generated_accel_regions ||= {}"
+        lines << "#{indent}  regions = (@__flexr_generated_accel_regions[dfa] ||= Flexr::Automaton::Accel.extract(dfa).to_h { |region| [region.state, region] })"
+        lines << "#{indent}  region = regions[state]"
+        lines << "#{indent}  return unless region"
+        lines << "#{indent}  return if dfa.accepts[state].any? do |acceptance|"
+        lines << "#{indent}    rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
+        lines << "#{indent}    acceptance.bol_only || acceptance.end_anchor || rule.trailing"
+        lines << "#{indent}  end"
+        lines << "#{indent}  region"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_accelerate(region, position)"
+        lines << "#{indent}  binary = buffer.source.b"
+        lines << "#{indent}  mode = self.class.__flexr_config.options.fetch(:accel, :auto)"
+        lines << "#{indent}  match_end = if %i[strscan auto].include?(mode) && defined?(StringScanner)"
+        lines << "#{indent}    scanner = StringScanner.new(binary)"
+        lines << "#{indent}    scanner.pos = position"
+        lines << "#{indent}    length = scanner.skip(region.regexp)"
+        lines << "#{indent}    length && scanner.pos"
+        lines << "#{indent}  else"
+        lines << "#{indent}    match = region.regexp.match(binary, position)"
+        lines << "#{indent}    match && match.begin(0) == position ? match.end(0) : nil"
+        lines << "#{indent}  end"
+        lines << "#{indent}  return match_end if match_end && match_end < buffer.bytesize"
+        lines << "#{indent}  return match_end if match_end && buffer.eof_loaded?"
+        lines << "#{indent}  return unless buffer.ensure_available?(buffer.bytesize + 1)"
+        lines << "#{indent}  __flexr_generated_accelerate(region, position)"
+        lines << "#{indent}rescue ArgumentError"
+        lines << "#{indent}  nil"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_fast_path?"
+        lines << "#{indent}  return @__flexr_generated_fast_path if defined?(@__flexr_generated_fast_path)"
+        lines << "#{indent}  @__flexr_generated_fast_path = if self.class.__flexr_config.backend == :firstmatch ||"
+        lines << "#{indent}                                  self.class.__flexr_config.options[:allow_empty_match] == true"
+        lines << "#{indent}    false"
+        lines << "#{indent}  else"
+        lines << "#{indent}    self.class.__flexr_rules.none? do |rule|"
+        lines << "#{indent}      !rule.trailing.nil? || rule.patterns.any? do |pattern|"
+        lines << "#{indent}        next false unless pattern.is_a?(Regexp)"
+        lines << "#{indent}      next true if pattern.source.match?(/\\\\[pP]\\\\{/) || pattern.source.match?(/\\[:(?:\\^)?[a-z]+:\\]/)"
+        lines << "#{indent}        self.class.__flexr_config.options[:unicode] == true && utf8_input? && pattern.source.match?(/\\\\[dDwWsS]/)"
+        lines << "#{indent}      end"
+        lines << "#{indent}    end"
+        lines << "#{indent}  end"
+        lines << "#{indent}end"
+        lines << "#{indent}def __flexr_generated_acceptance(dfa, state, start_position, cursor, best)"
+        lines << "#{indent}  dfa.accepts[state].each do |acceptance|"
+        lines << "#{indent}    next if acceptance.bol_only && !beginning_of_line?"
+        lines << "#{indent}    next if acceptance.end_anchor && !(buffer.eof?(cursor) || buffer.getbyte(cursor) == 0x0a)"
+        lines << "#{indent}    rule = self.class.__flexr_rules.fetch(acceptance.rule_index)"
+        lines << "#{indent}    defer_token_size_check!(cursor - start_position)"
+        lines << "#{indent}    next if best && cursor < best.total_end_pos"
+        lines << "#{indent}    next if best && cursor == best.total_end_pos &&"
+        lines << "#{indent}      rule.index > best.rule.index"
+        lines << "#{indent}    best ||= (@__flexr_generated_match ||= Flexr::Runtime::Match.new)"
+        lines << "#{indent}    best.rule = rule"
+        lines << "#{indent}    best.start_pos = start_position"
+        lines << "#{indent}    best.end_pos = cursor"
+        lines << "#{indent}    best.total_end_pos = cursor"
+        lines << "#{indent}  end"
+        lines << "#{indent}  best"
+        lines << "#{indent}end"
+        "#{lines.join("\n")}\n"
+      end
+
+      def simple_fast_path?
+        compiled.rules.all? do |rule|
+          rule.trailing.nil? && rule.pattern_conditions.all? do |condition|
+            condition && !condition.bol_only && !condition.end_anchor
+          end
+        end
+      end
+    end
+  end
+end

data/lib/flexr/codegen/table_packer.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Codegen
+    module TablePacker
+      module_function
+
+      def pack(rows, compression: :rows)
+        full = compression.to_sym == :full
+        base = []
+        default = rows.map { |row| row.tally.max_by { |_value, count| count }&.first }
+        fallback = full ? Array.new(rows.length) : nil
+        next_table = []
+        check = []
+        occupied = {}
+        rows.each_with_index do |row, state|
+          if full
+            candidate, matches = rows.each_index.take(state).map do |other_state|
+              [other_state, row.each_index.count { |class_id| row[class_id] == rows[other_state][class_id] }]
+            end.max_by(&:last)
+            fallback[state] = candidate if candidate && matches > row.count { |value| value == default[state] }
+          end
+          inherited = fallback && fallback[state] ? rows.fetch(fallback.fetch(state)) : nil
+          entries = row.each_index.reject do |class_id|
+            row[class_id] == (inherited ? inherited[class_id] : default[state])
+          end
+          offset = 0
+          while entries.any? { |class_id| occupied.key?(offset + class_id) }
+            offset += 1
+          end
+          base[state] = offset
+          entries.each do |class_id|
+            index = offset + class_id
+            next_table[index] = row[class_id]
+            check[index] = state
+            occupied[index] = true
+          end
+        end
+        result = { base: base.freeze, default: default.freeze, next: next_table.freeze, check: check.freeze }
+        result[:fallback] = fallback.freeze if fallback
+        result.freeze
+      end
+
+      def encode(packed)
+        result = {
+          encoding: :base64,
+          base: encode_array(packed.fetch(:base)),
+          default: encode_array(packed.fetch(:default), nil_value: -1),
+          next: encode_array(packed.fetch(:next), nil_value: -1),
+          check: encode_array(packed.fetch(:check), nil_value: -1)
+        }
+        result[:fallback] = encode_array(packed.fetch(:fallback), nil_value: -1) if packed[:fallback]
+        result.freeze
+      end
+
+      def encode_array(values, nil_value: 0)
+        [values.map { |value| value.nil? ? nil_value : value }.pack("l<*")].pack("m0")
+      end
+    end
+  end
+end

data/lib/flexr/diagnostics.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module Flexr
+  Diagnostic = Struct.new(:code, :severity, :message, :location, :help, :note, keyword_init: true) do
+    def error?
+      severity == :error
+    end
+
+    def warning?
+      severity == :warning
+    end
+
+    def to_h
+      {
+        code: code,
+        severity: severity,
+        message: message,
+        location: location,
+        help: help,
+        note: note
+      }.compact
+    end
+  end
+
+  class DiagnosticSet
+    include Enumerable
+
+    def initialize
+      @items = []
+    end
+
+    def each(&)
+      @items.each(&)
+    end
+
+    def <<(diagnostic)
+      @items << diagnostic
+      self
+    end
+
+    def any_error?
+      @items.any?(&:error?)
+    end
+
+    def empty?
+      @items.empty?
+    end
+
+    def to_a
+      @items.dup
+    end
+
+    def render(format: :human, color: :auto)
+      return JSON.generate(@items.map(&:to_h)) if format.to_sym == :json
+
+      @items.map { |item| render_one(item, color: color_enabled?(color)) }.join("\n")
+    end
+
+    private
+
+    def render_one(item, color:)
+      prefix = "#{item.severity}[#{item.code}]: #{item.message}"
+      prefix = "\e[31m#{prefix}\e[0m" if [true, :always].include?(color)
+      lines = [prefix]
+      lines << "  help: #{item.help}" if item.help
+      lines << "  note: #{item.note}" if item.note
+      lines.join("\n")
+    end
+
+    def color_enabled?(color)
+      return false if ENV.key?("NO_COLOR") || color == :never || color == false
+      return true if [:always, true].include?(color)
+
+      $stderr.tty?
+    end
+  end
+
+  module Diagnostics
+    module_function
+
+    def error(code, message, location: nil, help: nil, note: nil)
+      Diagnostic.new(code: code, severity: :error, message: message, location: location, help: help, note: note)
+    end
+
+    def warning(code, message, location: nil, help: nil, note: nil)
+      Diagnostic.new(code: code, severity: :warning, message: message, location: location, help: help, note: note)
+    end
+
+    def raise!(diagnostic)
+      klass = diagnostic.code == "FLEXR-E014" ? UnsupportedRegexpError : CompileError
+      raise klass.new(diagnostic.message, diagnostic: diagnostic)
+    end
+  end
+end

data/lib/flexr/dsl.rb

@@ -0,0 +1,182 @@
+# frozen_string_literal: true
+
+module Flexr
+  module DSL
+    DSL_METHODS = %i[rule state all_states on_eof emits backend token_kind encoding option accel].freeze
+    AUTO_DIRECT_CELL_THRESHOLD = 100_000
+
+    def inherited(child)
+      super
+      child.__flexr_reset!
+    end
+
+    def __flexr_reset!
+      @__flexr_rules = []
+      @__flexr_states = { initial: IR::State.new(name: :initial, inclusive: true, id: 0) }
+      @__flexr_state_stack = []
+      @__flexr_eof_rules = {}
+      @__flexr_config = IR::Config.new(
+        backend: :table, token_kind: :array, encoding: Encoding::UTF_8,
+        options: {}, declared_tokens: [], states: @__flexr_states
+      )
+      @__flexr_compiled = nil
+      @__flexr_generated = false
+      @__flexr_compile_mutex = Mutex.new
+    end
+
+    def rule(pattern, skip: false, emit: nil, followed_by: nil, &action)
+      patterns = normalize_patterns(pattern)
+      rule_action = if skip
+        :skip
+      elsif emit
+        [:emit, emit.to_sym]
+      else
+        action || proc { emit(nil, text) }
+      end
+      states = @__flexr_state_stack.empty? ? [:initial] : @__flexr_state_stack.dup
+      @__flexr_rules << IR::Rule.new(
+        index: @__flexr_rules.length, patterns: patterns, trailing: normalize_trailing(followed_by),
+        action: rule_action, states: states, bol_only: false, end_anchor: nil
+      )
+      nil
+    end
+
+    def state(*names, inclusive: false, &block)
+      raise ArgumentError, "state requires a block" unless block
+      raise ArgumentError, "state requires a name" if names.empty?
+
+      names.each do |name|
+        symbol = name.to_sym
+        @__flexr_states[symbol] ||= IR::State.new(name: symbol, inclusive: inclusive, id: @__flexr_states.length)
+      end
+      @__flexr_state_stack.concat(names.map(&:to_sym))
+      class_eval(&block)
+    ensure
+      names&.length&.times { @__flexr_state_stack.pop }
+    end
+
+    def all_states(&)
+      state(*@__flexr_states.keys, &)
+    end
+
+    def on_eof(&action)
+      state_name = @__flexr_state_stack.last || :initial
+      @__flexr_eof_rules[state_name] = action
+    end
+
+    def emits(*tokens)
+      @__flexr_config.declared_tokens.concat(tokens.flatten.map(&:to_sym)).uniq!
+    end
+
+    def backend(name)
+      @__flexr_config.backend = name.to_sym
+    end
+
+    def token_kind(name)
+      value = name.to_sym
+      raise ArgumentError, "unsupported token_kind: #{name}" unless %i[array struct yield].include?(value)
+
+      @__flexr_config.token_kind = value
+    end
+
+    def encoding(value)
+      encoding = value.is_a?(Encoding) ? value : Encoding.find(value.to_s)
+      unless [Encoding::UTF_8, Encoding::BINARY].include?(encoding)
+        diagnostic = Diagnostics.error("FLEXR-E011", "flexr supports UTF-8 and BINARY only")
+        raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+      end
+
+      @__flexr_config.encoding = encoding
+    end
+
+    def option(*values)
+      values.each { |value| @__flexr_config.options[value.to_sym] = true }
+    end
+
+    def accel(value)
+      @__flexr_config.options[:accel] = value.to_sym
+    end
+
+    def compile!
+      @__flexr_compile_mutex.synchronize do
+        # The ivar is part of the generated/runtime class contract.
+        # rubocop:disable Naming/MemoizedInstanceVariableName
+        @__flexr_compiled ||= begin
+          compiled = Automaton::Compiler.new(__flexr_spec).compile
+          @__flexr_config.backend = auto_direct?(compiled) ? :direct : :table if @__flexr_config.backend == :auto
+          compiled
+        end
+        # rubocop:enable Naming/MemoizedInstanceVariableName
+      end
+    end
+
+    def dfa
+      compile!
+      __flexr_compiled
+    end
+
+    def __flexr_spec
+      IR::Spec.new(
+        class_name: name,
+        superclass: superclass&.name,
+        backend: @__flexr_config.backend,
+        token_kind: @__flexr_config.token_kind,
+        encoding: @__flexr_config.encoding,
+        options: @__flexr_config.options,
+        declared_tokens: @__flexr_config.declared_tokens,
+        states: @__flexr_states,
+        rules: @__flexr_rules,
+        eof_rules: @__flexr_eof_rules,
+        verbatim: nil
+      )
+    end
+
+    attr_reader :__flexr_rules, :__flexr_states, :__flexr_config, :__flexr_compiled
+
+    def __flexr_add_generated_eof(state, action)
+      @__flexr_eof_rules[state.to_sym] = action
+    end
+
+    def __flexr_set_compiled!(compiled)
+      @__flexr_compiled = compiled
+    end
+
+    def __flexr_mark_generated!
+      @__flexr_generated = true
+    end
+
+    def __flexr_generated?
+      @__flexr_generated == true
+    end
+
+    private
+
+    def normalize_patterns(pattern)
+      values = pattern.is_a?(Array) ? pattern : [pattern]
+      values.each do |value|
+        unless value.is_a?(::Regexp) || value.is_a?(String)
+          diagnostic = Diagnostics.error("FLEXR-E018", "rule pattern must be a Regexp, String, or Array")
+          raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+        end
+      end
+      values
+    end
+
+    def normalize_trailing(value)
+      case value
+      when nil, ::Regexp
+        value
+      when String
+        ::Regexp.new(::Regexp.escape(value))
+      else
+        diagnostic = Diagnostics.error("FLEXR-E018", "followed_by must be a Regexp or String")
+        raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+      end
+    end
+
+    def auto_direct?(compiled)
+      cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
+      cells > AUTO_DIRECT_CELL_THRESHOLD
+    end
+  end
+end

data/lib/flexr/errors.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Flexr
+  class Error < StandardError
+    attr_reader :diagnostic
+
+    def initialize(message = nil, diagnostic: nil)
+      @diagnostic = diagnostic
+      super(message || diagnostic&.message)
+    end
+  end
+
+  class LexError < Error
+    attr_reader :filename, :byte_pos, :line, :text
+
+    def initialize(message, filename: nil, byte_pos: nil, line: nil, text: nil, diagnostic: nil)
+      @filename = filename
+      @byte_pos = byte_pos
+      @line = line
+      @text = text
+      super(message, diagnostic: diagnostic)
+    end
+  end
+
+  class CompileError < Error; end
+  class UnsupportedRegexpError < CompileError; end
+  class StaticResolutionError < CompileError; end
+end

data/lib/flexr/generated.rb

@@ -0,0 +1,125 @@
+# frozen_string_literal: true
+
+module Flexr
+  module Generated
+    module_function
+
+    def install!(klass, payload)
+      rules = payload.fetch(:rules)
+      config = payload
+      klass.__flexr_reset!
+      klass.backend(config.fetch(:backend, :table))
+      klass.token_kind(config.fetch(:token_kind, :array))
+      klass.encoding(config.fetch(:encoding, Encoding::UTF_8))
+      Array(config[:declared_tokens]).each { |token| klass.emits(token) }
+      config.fetch(:options, {}).each do |option, value|
+        if option == :accel
+          klass.accel(value)
+        elsif value == true
+          klass.option(option)
+        else
+          klass.__flexr_config.options[option] = value
+        end
+      end
+      Array(config[:states]).each do |state|
+        klass.state(state, inclusive: config.fetch(:inclusive_states, {}).fetch(state.to_sym, false)) { nil }
+      end
+      rules.each do |definition|
+        klass.__flexr_add_generated_rule(definition)
+      end
+      config.fetch(:eof_rules, {}).each do |state, action|
+        klass.__flexr_add_generated_eof(state, action)
+      end
+      klass
+    end
+
+    def install_compiled!(klass, payload)
+      install!(klass, payload)
+      machines = payload.fetch(:compiled).fetch(:machines).transform_values do |machine|
+        dfa_data = machine.fetch(:dfa)
+        packed = decode_packed(dfa_data[:packed])
+        dfa = Automaton::DFA.new(
+          transitions: dfa_data[:transitions] || inflate_packed(packed, dfa_data.fetch(:state_count),
+                                                                 dfa_data.fetch(:class_count)),
+          accepts: dfa_data.fetch(:accepts),
+          ec: dfa_data.fetch(:ec),
+          class_count: dfa_data.fetch(:class_count),
+          start: dfa_data.fetch(:start),
+          rule_ids: dfa_data.fetch(:rule_ids),
+          packed: packed,
+          direct: dfa_data[:direct]
+        )
+        Automaton::Machine.new(dfa: dfa, state_name: machine.fetch(:state_name).to_sym)
+      end
+      compiled = Automaton::CompiledSpec.new(
+        machines: machines,
+        rules: klass.__flexr_rules,
+        states: payload.fetch(:compiled).fetch(:states).map(&:to_sym),
+        stats: payload.fetch(:compiled).fetch(:stats),
+        diagnostics: payload.fetch(:compiled).fetch(:diagnostics, []).map do |diagnostic|
+          Diagnostic.new(**diagnostic.transform_keys(&:to_sym))
+        end
+      )
+      klass.__flexr_set_compiled!(compiled)
+      klass.__flexr_mark_generated!
+      klass
+    end
+
+    def decode_packed(packed)
+      return packed unless packed.is_a?(Hash) && packed[:encoding]&.to_sym == :base64
+
+      {
+        base: decode_array(packed.fetch(:base)),
+        default: decode_array(packed.fetch(:default), nil_value: -1),
+        next: decode_array(packed.fetch(:next), nil_value: -1),
+        check: decode_array(packed.fetch(:check), nil_value: -1),
+        fallback: packed.key?(:fallback) ? decode_array(packed.fetch(:fallback), nil_value: -1) : nil
+      }
+    end
+
+    def decode_array(encoded, nil_value: nil)
+      values = encoded.unpack1("m0").unpack("l<*")
+      return values unless nil_value
+
+      values.map { |value| value == nil_value ? nil : value }
+    end
+
+    def inflate_packed(packed, state_count, class_count)
+      return nil unless packed
+
+      Array.new(state_count) do |state|
+        Array.new(class_count) do |class_id|
+          cursor = state
+          loop do
+            index = packed.fetch(:base).fetch(cursor) + class_id
+            break packed.fetch(:next).fetch(index) if packed.fetch(:check)[index] == cursor
+
+            fallback = packed[:fallback]&.fetch(cursor)
+            break packed.fetch(:default).fetch(cursor) unless fallback
+
+            cursor = fallback
+          end
+        end
+      end
+    end
+  end
+
+  module DSL
+    def __flexr_add_generated_rule(definition)
+      action = definition.fetch(:action)
+      @__flexr_rules << IR::Rule.new(
+        index: definition.fetch(:index), patterns: Array(definition.fetch(:patterns)),
+        trailing: normalize_trailing(definition[:trailing]), action: action,
+        states: Array(definition.fetch(:states)).map(&:to_sym),
+        bol_only: definition.fetch(:bol_only, false), end_anchor: definition[:end_anchor],
+        location: definition[:span] || definition[:location],
+        pattern_conditions: Array(definition[:pattern_conditions]).map do |condition|
+          next unless condition
+
+          Automaton::Acceptance.new(rule_index: condition[0], pattern_index: condition[1],
+                                    bol_only: condition[2], end_anchor: condition[3])
+        end
+      )
+    end
+  end
+end