flexr 1.0.0

data/lib/flexr/generator.rb

@@ -0,0 +1,400 @@
+# frozen_string_literal: true
+
+module Flexr
+  class Generator
+    attr_reader :diagnostics
+
+    INLINE_EMIT = /\Aemit(?:\s+(.+?))?\z/
+    INLINE_EMIT_ARGUMENTS = /\A(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line)(?:\s*,\s*(?::[A-Za-z_]\w*|true|false|nil|-?\d+(?:\.\d+)?|"(?:\\.|[^"])*"|'(?:\\.|[^'])*'|text(?:\.(?:to_f|to_i|bytesize|byteslice\([^()\n]+\)))?|lineno|line))*\z/
+
+    RUNTIME_SOURCES = %w[
+      version.rb errors.rb diagnostics.rb ir.rb
+      regexp/ast.rb regexp/parser.rb regexp/normalizer.rb regexp/unsupported.rb regexp/char_class.rb
+      unicode/utf8_splitter.rb unicode/data/properties.rb unicode/data/case_folding.rb unicode/property.rb unicode/reference_regexp.rb unicode/case_fold.rb
+      automaton/byte_class_set.rb automaton/nfa.rb automaton/dfa.rb automaton/compiler.rb
+      automaton/analysis.rb automaton/minimizer.rb automaton/accel.rb
+      runtime/location.rb runtime/token.rb runtime/buffer.rb runtime/errors.rb
+      runtime/interpreter.rb runtime/core.rb dsl.rb lexer.rb generated.rb
+    ].freeze
+
+    def initialize(path, output: nil, eval_mode: false, options: {})
+      @path = path
+      @output = output
+      @eval_mode = eval_mode
+      @options = options
+      @diagnostics = []
+    end
+
+    def generate
+      # Keep generated source stable when a checkout presents the spec with
+      # Windows line endings. All offsets consumed by the parser and
+      # passthrough writer are relative to this normalized source.
+      source = File.binread(@path).gsub(/\r\n?/, "\n").force_encoding(Encoding::UTF_8)
+      result = if @eval_mode
+        generate_from_runtime(source)
+      else
+        parsed = Source::PrismReader.new(source, path: @path).read
+        generate_static(parsed)
+      end
+      File.binwrite(@output, result) if @output
+      result
+    end
+
+    private
+
+    def generate_static(parsed)
+      compiled = compile_parsed(parsed)
+      @diagnostics = Array(compiled.diagnostics)
+      validate_diagnostics!(compiled)
+      payload = generated_payload(parsed, compiled)
+      indent = Source::Passthrough.indentation(parsed.source, parsed.first_dsl_offset)
+      install = "Flexr::Generated.install_compiled!(self, #{payload})\n"
+      install = "#{install}#{Codegen::Table.new(compiled).source(indent: indent)}"
+      install = "#{install}#{generated_action_source(parsed, indent)}"
+      install = "#{install}#{Codegen::Direct.new(compiled).source(indent: indent)}#{indent}" if
+        effective_backend(parsed) == :direct
+      result = Source::Passthrough.remove_spans(parsed.source, parsed.dsl_spans, insertion: parsed.first_dsl_offset, payload: install)
+      result = result.gsub(/^[ \t]+(?=\n)/, "")
+      result = result.gsub(/^\s*require ["']flexr["']\s*\n/, "") if standalone?(parsed)
+      digest = Digest::SHA256.hexdigest(payload)
+      header = [
+        "# Generated by flexr. DO NOT EDIT.",
+        "# source: #{parsed.path}",
+        "# spec-digest: sha256:#{digest}",
+        "# unicode: #{Unicode::VERSION}",
+        "# backend: #{effective_backend(parsed)}",
+        "# compiled: true",
+        "# eval: #{@eval_mode}",
+        "# standalone: #{standalone?(parsed)}"
+      ].join("\n")
+      prefix = standalone?(parsed) ? "require \"json\"\n#{embedded_runtime}\n" : ""
+      result.sub(/\A(# frozen_string_literal: true\n)/) { "#{::Regexp.last_match(1)}#{header}\n#{prefix}" }
+    end
+
+    def generated_payload(parsed, compiled)
+      definitions = parsed.rules.map do |rule|
+        conditions = Array(rule.pattern_conditions).map do |condition|
+          [condition.rule_index, condition.pattern_index, condition.bol_only, condition.end_anchor]
+        end
+        "{ index: #{rule.index}, patterns: #{ruby_literal(rule.patterns)}, pattern_conditions: #{ruby_literal(conditions)}, trailing: #{ruby_literal(rule.trailing)}, " \
+          "action: #{action_expression(rule.action)}, states: #{rule.states.inspect}, " \
+          "bol_only: #{ruby_literal(rule.bol_only)}, end_anchor: #{ruby_literal(rule.end_anchor)} }"
+      end
+      "{ rules: [#{definitions.join(', ')}], " \
+        "backend: #{ruby_literal(effective_backend(parsed))}, " \
+        "token_kind: #{ruby_literal(@options.fetch(:token_kind, parsed.config[:token_kind]))}, " \
+        "encoding: #{encoding_expression(parsed.config[:encoding])}, " \
+        "declared_tokens: #{ruby_literal(parsed.config[:declared_tokens])}, " \
+        "options: #{ruby_literal(effective_options(parsed))}, " \
+        "eof_rules: #{eof_rules_expression(parsed.config[:eof_rules] || {})}, " \
+        "states: #{ruby_literal(parsed.states.keys.reject { |name| name == :initial })}, " \
+        "inclusive_states: #{ruby_literal(parsed.states.transform_values { |value| value[:inclusive] })}, " \
+        "compiled: #{compiled_expression(compiled, compression: table_compression(parsed),
+                                         backend: effective_backend(parsed))} }"
+    end
+
+    def compile_parsed(parsed)
+      states = parsed.states.each_with_index.to_h do |(name, value), index|
+        [name.to_sym, IR::State.new(name: name.to_sym, inclusive: value[:inclusive], id: index)]
+      end
+      rules = parsed.rules.map do |rule|
+        IR::Rule.new(index: rule.index, patterns: rule.patterns, trailing: normalize_trailing(rule.trailing),
+                     action: rule.action, states: rule.states, bol_only: rule.bol_only,
+                     end_anchor: rule.end_anchor, location: rule.span)
+      end
+      spec = IR::Spec.new(
+        class_name: parsed.class_name, superclass: "Flexr::Lexer",
+        backend: @options.fetch(:backend, parsed.config[:backend]),
+        token_kind: @options.fetch(:token_kind, parsed.config[:token_kind]),
+        encoding: parsed.config[:encoding], options: effective_options(parsed),
+        declared_tokens: parsed.config[:declared_tokens], states: states, rules: rules,
+        eof_rules: parsed.config[:eof_rules] || {}, verbatim: parsed.source
+      )
+      compiled = Automaton::Compiler.new(spec).compile
+      validate_firstmatch_equivalence!(spec, compiled) if spec.backend == :firstmatch
+      @resolved_backend = resolve_backend(spec.backend, compiled)
+      parsed.rules.each do |rule|
+        compiled_rule = spec.rules.fetch(rule.index)
+        rule.pattern_conditions = compiled_rule.pattern_conditions
+      end
+      compiled
+    end
+
+    def effective_backend(parsed)
+      @resolved_backend || resolve_backend(@options.fetch(:backend, parsed.config[:backend]).to_sym, nil)
+    end
+
+    def resolve_backend(requested, compiled)
+      return requested unless requested == :auto
+      return :table unless compiled
+
+      cells = compiled.stats.values.map { |stats| stats[:states] * stats[:classes] }.max.to_i
+      cells > DSL::AUTO_DIRECT_CELL_THRESHOLD ? :direct : :table
+    end
+
+    def normalize_trailing(value)
+      case value
+      when nil, ::Regexp
+        value
+      when String
+        ::Regexp.new(::Regexp.escape(value))
+      else
+        diagnostic = Diagnostics.error("FLEXR-E018", "followed_by must be a Regexp or String")
+        raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+      end
+    end
+
+    def validate_firstmatch_equivalence!(spec, compiled)
+      return if spec.rules.any? do |rule|
+        rule.trailing || rule.patterns.any? { |pattern| reference_pattern?(pattern, unicode: spec.options[:unicode] == true) }
+      end
+
+      machine = compiled.machines.fetch(:initial).dfa
+      random = Random.new(17)
+      inputs = ["", "a", "aa", "aaa", "ba", "aab"]
+      inputs.concat(Array.new(9_994) do
+        Array.new(random.rand(10)) { random.rand(32..126) }.pack("C*")
+      end)
+      inputs.each do |input|
+        table = table_match(machine, input)
+        firstmatch = firstmatch_match(spec.rules, input)
+        next if table == firstmatch
+
+        raise CompileError, "firstmatch differs from table for #{input.inspect}: #{firstmatch.inspect} vs #{table.inspect}"
+      end
+    end
+
+    def reference_pattern?(pattern, unicode: false)
+      return false unless pattern.is_a?(::Regexp)
+      return true if pattern.source.match?(/\\[pP]\{/) || pattern.source.match?(/\[:(?:\^)?[a-z]+:\]/)
+
+      unicode && pattern.encoding != Encoding::BINARY && pattern.source.match?(/\\[dDwWsS]/)
+    end
+
+    def table_match(dfa, input)
+      state = dfa.start
+      best = nil
+      input.each_byte.with_index do |byte, index|
+        state = dfa.transition(state, byte)
+        break unless state
+
+        acceptance = dfa.accepts[state].select do |candidate|
+          !candidate.end_anchor || index + 1 == input.bytesize || input.getbyte(index + 1) == 0x0a
+        end.min_by(&:rule_index)
+        next unless acceptance
+
+        candidate = [acceptance.rule_index, index + 1]
+        best = candidate if best.nil? || candidate[1] > best[1] ||
+          (candidate[1] == best[1] && candidate[0] < best[0])
+      end
+      best
+    end
+
+    def firstmatch_match(rules, input)
+      rules.sort_by(&:index).each do |rule|
+        lengths = rule.patterns.filter_map do |pattern|
+          regexp = pattern.is_a?(::Regexp) ? pattern : ::Regexp.new(::Regexp.escape(pattern.to_s))
+          match = regexp.match(input, 0)
+          match&.begin(0)&.zero? ? match[0].bytesize : nil
+        rescue ArgumentError, RegexpError
+          nil
+        end
+        return [rule.index, lengths.max] unless lengths.empty?
+      end
+      nil
+    end
+
+    def compiled_expression(compiled, compression: :none, backend: :table)
+      machines = compiled.machines.map do |name, machine|
+        dfa = machine.dfa
+        data = {
+          accepts: dfa.accepts,
+          ec: dfa.ec,
+          class_count: dfa.class_count,
+          state_count: dfa.states,
+          start: dfa.start,
+          rule_ids: dfa.rule_ids
+        }
+        pack_tables = %i[rows full].include?(compression) || @options.fetch(:table_format, :literal).to_sym == :packed
+        if pack_tables
+          packed = Codegen::TablePacker.pack(dfa.transitions, compression: compression)
+          data[:packed] = if @options.fetch(:table_format, :literal).to_sym == :packed
+            Codegen::TablePacker.encode(packed)
+          else
+            packed
+          end
+        else
+          data[:transitions] = dfa.transitions
+        end
+        data[:direct] = Codegen::Direct.new(compiled).generate(state: name) if backend == :direct
+        "#{ruby_literal(name)} => { state_name: #{ruby_literal(machine.state_name)}, dfa: #{ruby_literal(data)} }"
+      end
+      # W016 is derived from wall-clock compilation time and must not make
+      # generated source (or its golden digest) vary between runs.
+      diagnostics = compiled.diagnostics.reject { |diagnostic| diagnostic.code == "FLEXR-W016" }.map(&:to_h)
+      "{ machines: { #{machines.join(', ')} }, states: #{ruby_literal(compiled.states)}, " \
+        "stats: #{ruby_literal(compiled.stats)}, diagnostics: #{ruby_literal(diagnostics)} }"
+    end
+
+    def standalone?(parsed)
+      @options.fetch(:standalone, parsed.config[:options]&.key?(:standalone) || false)
+    end
+
+    def effective_options(parsed)
+      options = (parsed.config[:options] || {}).dup
+      %i[accel max_dfa_states table_compression table_format].each do |name|
+        options[name] = @options[name] if @options.key?(name)
+      end
+      options[:standalone] = true if standalone?(parsed)
+      options
+    end
+
+    def table_compression(parsed)
+      @options.fetch(:table_compression, parsed.config[:options]&.fetch(:table_compression, :rows) || :rows).to_sym
+    end
+
+    def embedded_runtime
+      RUNTIME_SOURCES.map do |relative|
+        File.binread(File.expand_path(relative, __dir__))
+      end.join("\n")
+    end
+
+    def encoding_expression(encoding)
+      encoding == Encoding::BINARY ? "Encoding::BINARY" : "Encoding::UTF_8"
+    end
+
+    def eof_rules_expression(rules)
+      pairs = rules.map { |state, action| "#{ruby_literal(state)} => #{action_expression(action)}" }
+      "{ #{pairs.join(', ')} }"
+    end
+
+    def action_expression(action)
+      return action if action.is_a?(String) && action.start_with?("proc")
+      return ruby_literal(action) unless action.is_a?(Proc)
+
+      "proc { emit(nil, text) }"
+    end
+
+    def generated_action_source(parsed, indent)
+      lines = ["#{indent}def __flexr_generated_execute(rule)", "#{indent}  case rule.index"]
+      parsed.rules.each do |rule|
+        lines << "#{indent}  when #{rule.index}"
+        body = inline_action_body(rule)
+        if body.nil?
+          lines << "#{indent}    instance_exec(&rule.action)"
+        else
+          body.lines.each { |line| lines << "#{indent}    #{line.rstrip}" }
+        end
+      end
+      lines << "#{indent}  else"
+      lines << "#{indent}    instance_exec(&rule.action)"
+      lines << "#{indent}  end"
+      lines << "#{indent}end\n"
+      lines.join("\n")
+    end
+
+    def inline_action_body(rule)
+      action = rule.action
+      case action
+      when :skip
+        ""
+      when Array
+        return "emit(#{ruby_literal(action.fetch(1))}, text)" if action.first == :emit
+
+        nil
+      when String
+        return inline_simple_action(rule.action_source) if rule.action_source
+        return unless action.start_with?("proc")
+
+        body = action.delete_prefix("proc").strip
+        if body.start_with?("{") && body.end_with?("}")
+          body = body[1...-1]
+          return nil if body.lstrip.start_with?("|")
+          return body.strip
+        end
+        return nil unless body.start_with?("do") && body.end_with?("end")
+
+        body.delete_prefix("do")[0...-3].strip
+      end
+    end
+
+    # Hash#inspect changed from hash rockets to keyword-style labels in Ruby
+    # 4. Generated source and its golden digests must be identical on every
+    # supported Ruby, so serialize the small set of Ruby literals used in the
+    # generated payload ourselves.
+    def ruby_literal(value)
+      case value
+      when Hash
+        return "{}" if value.empty?
+
+        entries = value.map do |key, nested|
+          if key.is_a?(Symbol) && key.to_s.match?(/\A[a-zA-Z_]\w*\z/)
+            "#{key}: #{ruby_literal(nested)}"
+          else
+            "#{ruby_literal(key)} => #{ruby_literal(nested)}"
+          end
+        end
+        "{#{entries.join(', ')}}"
+      when Array
+        "[#{value.map { |item| ruby_literal(item) }.join(', ')}]"
+      else
+        value.inspect
+      end
+    end
+
+    def inline_simple_action(source)
+      body = source.to_s.strip
+      body = body[1...-1].strip if body.start_with?("{") && body.end_with?("}")
+      match = INLINE_EMIT.match(body)
+      return unless match
+
+      arguments = match[1].to_s.strip
+      return unless arguments.empty? || INLINE_EMIT_ARGUMENTS.match?(arguments)
+
+      arguments.empty? ? "emit" : "emit #{arguments}"
+    end
+
+    def validate_diagnostics!(compiled)
+      return unless @options[:warn_as_error]
+
+      diagnostic = Array(compiled.diagnostics).find do |item|
+        item.warning? && (@options.fetch(:warn_level, :default) == :all || item.code != "FLEXR-W016")
+      end
+      return unless diagnostic
+
+      raise CompileError.new(diagnostic.message, diagnostic: diagnostic)
+    end
+
+    def generate_from_runtime(source)
+      loader = Object.new
+      loader.define_singleton_method(:load_spec) do
+        # --eval is opt-in and intentionally executes trusted specification code.
+        # rubocop:disable Security/Eval
+        eval(source, TOPLEVEL_BINDING, @path, 1)
+        # rubocop:enable Security/Eval
+      end
+      loader.instance_variable_set(:@path, @path)
+      loader.load_spec
+      passthrough = Source::PrismReader.new(source, path: @path).read(allow_dynamic: true)
+      classes = ObjectSpace.each_object(Class).select { |klass| klass.respond_to?(:__flexr_spec) }
+      classes.select! do |klass|
+        name = klass.name.to_s
+        (passthrough.class_name.nil? || name == passthrough.class_name || name.end_with?("::#{passthrough.class_name}")) &&
+          klass != Flexr::Lexer && klass.__flexr_rules.any?
+      end
+      klass = classes.last
+      raise CompileError, "--eval could not find a Flexr::Lexer class" unless klass
+
+      # --eval is intentionally explicit; the generated file still contains
+      # executable action source, but its metadata is deterministic for this run.
+      parsed = klass.__flexr_spec
+      runtime_rules = parsed.rules.to_h { |rule| [rule.index, rule] }
+      passthrough.rules.each do |rule|
+        runtime_rule = runtime_rules.fetch(rule.index)
+        rule.patterns = runtime_rule.patterns
+        rule.trailing = runtime_rule.trailing
+      end
+      generate_static(passthrough)
+    end
+  end
+end