descent 0.7.1

data/lib/descent/generator.rb

@@ -0,0 +1,489 @@
+# frozen_string_literal: true
+
+require 'liquid'
+
+module Descent
+  # Custom Liquid filters for code generation.
+  module LiquidFilters
+    # Escape sequences: DSL placeholder -> Rust byte literal
+    ESCAPE_SEQUENCES = {
+      '<P>' => "b'|'",
+      '<R>' => "b']'",
+      '<L>' => "b'['",
+      '<RB>' => "b'}'",
+      '<LB>' => "b'{'",
+      '<RP>' => "b')'",
+      '<LP>' => "b'('",
+      '<BS>' => "b'\\\\'",
+      '<SQ>' => "b'\\''",
+      '<DQ>' => "b'\"'",
+      '<NL>' => "b'\\n'",
+      '<WS>' => "b' '",
+      '<>' => 'b""' # Empty byte slice
+    }.freeze
+    # Convert a character to Rust byte literal format.
+    # Examples: "\n" -> "b'\\n'", "|" -> "b'|'", " " -> "b' '"
+    def escape_rust_char(char)
+      return 'b\'?\'' if char.nil?
+
+      escaped = case char
+                when "\n" then '\\n'
+                when "\t" then '\\t'
+                when "\r" then '\\r'
+                when '\\' then '\\\\'
+                when "'" then "\\'"
+                else char
+                end
+      "b'#{escaped}'"
+    end
+
+    # Convert snake_case, camelCase, or preserve PascalCase.
+    # Examples: "identity" -> "Identity", "after_name" -> "AfterName",
+    #           "UnclosedInterpolation" -> "UnclosedInterpolation"
+    def pascalcase(str)
+      return '' if str.nil?
+
+      # Split on delimiters AND on case transitions (lowercase followed by uppercase)
+      # This preserves existing PascalCase while converting snake_case and camelCase
+      str.to_s.split(/[_\s-]|(?<=[a-z])(?=[A-Z])/).map(&:capitalize).join
+    end
+
+    # Transform DSL expressions to Rust.
+    # - /function(args) -> self.parse_function(transformed_args, on_event)
+    # - /function -> self.parse_function(on_event)
+    # - COL -> self.col()
+    # - LINE -> self.line as i32
+    # - PREV -> self.prev()
+    # - Character literals: ' ' -> b' ', '\t' -> b'\t' (only if not already a byte literal)
+    # - Escape sequences: <R> -> b']', <RB> -> b'}', <P> -> b'|', etc.
+    # - Parameter references: :param -> param
+    def rust_expr(str)
+      return '' if str.nil?
+
+      result = str.to_s
+
+      # IMPORTANT: Process function calls FIRST, before COL/LINE/PREV expansion.
+      # Otherwise /element(COL) becomes /element(self.col()) and the regex
+      # [^)]* breaks on the ) inside self.col().
+      result = result.gsub(%r{/(\w+)\(([^)]*)\)}) do
+        func = ::Regexp.last_match(1)
+        # Transform args: :param, <R>, COL, etc.
+        args = transform_call_args(::Regexp.last_match(2))
+        args = expand_special_vars(args)
+        "self.parse_#{func}(#{args}, on_event)"
+      end
+      result = result.gsub(%r{/(\w+)}) { "self.parse_#{::Regexp.last_match(1)}(on_event)" }
+
+      # Now expand special variables in the rest of the expression
+      result = expand_special_vars(result)
+
+      # Transform standalone args (handles :param, <R>, etc. outside function calls)
+      result = transform_call_args(result)
+
+      result
+        .gsub(/(?<!b)'(\\.|.)'/, "b'\\1'") # Convert char literals to byte literals (only if not already b'...')
+        .gsub(/<[A-Z]+>/) { |m| ESCAPE_SEQUENCES[m] || m } # Escape sequences
+    end
+
+    # Expand special variables: COL, LINE, PREV, :param
+    def expand_special_vars(str)
+      str
+        .gsub(/\bCOL\b/, 'self.col()')
+        .gsub(/\bLINE\b/, 'self.line as i32')
+        .gsub(/\bPREV\b/, 'self.prev()')
+        .gsub(/:([a-z_]\w*)/i) { ::Regexp.last_match(1) } # :param -> param
+    end
+
+    # Transform function call arguments.
+    # - :param -> param (parameter references)
+    # - <R>, <RB>, etc. -> byte literals via ESCAPE_SEQUENCES
+    # - Bare quotes: " -> b'"', ' -> b'\''
+    def transform_call_args(args)
+      args.split(',').map do |arg|
+        arg = arg.strip
+        # Try escape sequence lookup first
+        if (escaped = ESCAPE_SEQUENCES[arg])
+          escaped
+        else
+          case arg
+          when /^:(\w+)$/           then ::Regexp.last_match(1) # :param -> param
+          when '"'                  then "b'\"'" # Bare double quote
+          when "'"                  then "b'\\''" # Bare single quote (escaped)
+          when /^\d+$/              then arg # numeric literals
+          when /^-?\d+$/            then arg # negative numbers
+          when /^'(.)'$/            then "b'#{::Regexp.last_match(1)}'" # char literal
+          when /^"(.)"$/            then "b'#{::Regexp.last_match(1)}'" # quoted char
+          when %r{^[!;:#*\-_<>/\\@$%^&+=?,.]$} then "b'#{arg}'" # Single punctuation → byte literal
+          else arg # pass through (variables, expressions)
+          end
+        end
+      end.join(', ')
+    end
+  end
+
+  # Custom file system for Liquid partials.
+  class TemplateFileSystem
+    def initialize(base_path) = @base_path = base_path
+
+    def read_template_file(template_path)
+      # Liquid looks for partials with underscore prefix
+      full_path = File.join(@base_path, "_#{template_path}.liquid")
+      raise Liquid::FileSystemError, "No such template: #{full_path}" unless File.exist?(full_path)
+
+      File.read(full_path)
+    end
+  end
+
+  # Renders IR to target language code using Liquid templates.
+  #
+  # All target-specific logic lives in templates, not here.
+  class Generator
+    TEMPLATE_DIR = File.expand_path('templates', __dir__)
+
+    def initialize(ir, target:, trace: false, streaming: true, **options)
+      @ir        = ir
+      @target    = target
+      @trace     = trace
+      @streaming = streaming
+      @options   = options
+    end
+
+    def generate
+      template_dir  = File.join(TEMPLATE_DIR, @target.to_s)
+      template_path = File.join(template_dir, 'parser.liquid')
+
+      raise Error, "No template for target: #{@target} (looked in #{template_path})" unless File.exist?(template_path)
+
+      # Build Liquid environment with filters and file system
+      env = Liquid::Environment.build do |e|
+        e.register_filter(LiquidFilters)
+        e.file_system = TemplateFileSystem.new(template_dir)
+      end
+
+      template = Liquid::Template.parse(File.read(template_path), environment: env)
+
+      result = template.render(
+        build_context,
+        strict_variables: false, # Partials may not have all variables
+        strict_filters: true
+      )
+
+      # Post-process: clean up whitespace from Liquid template
+      result
+        .gsub(/^[ \t]+$/, '')                                 # Remove whitespace-only lines
+        .gsub(/\n{2,}/, "\n")                                 # Collapse all blank lines
+        .gsub(%r{^(//.*)\n(use |pub |impl )}, "\\1\n\n\\2")   # Blank before use/pub/impl
+        .gsub(%r{(\})\n([ \t]*(?://|#\[|pub |fn ))}, "\\1\n\n\\2") # Blank after } before new item
+    end
+
+    private
+
+    # Unicode character classes that require the unicode-xid crate
+    UNICODE_CLASSES = %w[xid_start xid_cont xlbl_start xlbl_cont].freeze
+
+    def build_context
+      functions_data = @ir.functions.map { |f| function_to_hash(f) }
+      usage          = analyze_helper_usage(functions_data)
+      {
+        'parser' => @ir.name,
+        'entry_point' => @ir.entry_point,
+        'types' => @ir.types.map { |t| type_to_hash(t) },
+        'functions' => functions_data,
+        'keywords' => @ir.keywords.map { |k| keywords_to_hash(k) },
+        'custom_error_codes' => @ir.custom_error_codes,
+        'trace' => @trace,
+        'uses_unicode' => uses_unicode_classes?(functions_data),
+        # Helper usage flags - only emit helpers that are actually used
+        'uses_col' => usage[:col],
+        'uses_prev' => usage[:prev],
+        'uses_set_term' => usage[:set_term],
+        'uses_span' => usage[:span],
+        'uses_letter' => usage[:letter],
+        'uses_label_cont' => usage[:label_cont],
+        'uses_digit' => usage[:digit],
+        'uses_hex_digit' => usage[:hex_digit],
+        'uses_ws' => usage[:ws],
+        'uses_nl' => usage[:nl],
+        'max_scan_arity' => usage[:max_scan_arity],
+        'streaming' => @streaming
+      }
+    end
+
+    # Analyze which helper methods are actually used by the generated code.
+    # Returns a hash of usage flags that the template uses for conditional emission.
+    def analyze_helper_usage(functions_data)
+      usage = {
+        col: false, prev: false, set_term: false, span: false,
+        letter: false, label_cont: false, digit: false, hex_digit: false,
+        ws: false, nl: false, max_scan_arity: 0
+      }
+
+      functions_data.each do |func|
+        # Check conditions for COL/PREV usage
+        check_expressions_in_function(func, usage)
+
+        # Check special classes used in cases
+        func['states'].each do |state|
+          # Track max scan arity
+          if state['scannable'] && state['scan_chars']
+            usage[:max_scan_arity] =
+              [usage[:max_scan_arity], state['scan_chars'].size].max
+          end
+
+          state['cases'].each do |kase|
+            check_special_class(kase['special_class'], usage)
+          end
+        end
+      end
+
+      # span() is used for bracket types and errors (always needed if we have types)
+      usage[:span] = true
+
+      usage
+    end
+
+    # Check expressions in conditions/commands for COL/PREV usage
+    def check_expressions_in_function(func, usage)
+      all_commands = collect_all_commands(func)
+
+      all_commands.each do |cmd|
+        args = cmd['args'] || {}
+
+        # Check condition expressions (from if cases and conditionals)
+        check_expression(args['condition'], usage)
+
+        # Check call arguments for COL/PREV
+        check_expression(args['call_args'], usage) if cmd['type'] == 'call'
+
+        # Check assignment expressions
+        check_expression(args['expr'], usage) if %w[assign add_assign sub_assign].include?(cmd['type'])
+
+        # Check for set_term usage (any TERM command uses set_term)
+        usage[:set_term] = true if cmd['type'] == 'term'
+
+        # Check for advance_to - track scan arity for explicit ->[chars]
+        if cmd['type'] == 'advance_to' && args['value']
+          arity = args['value'].length
+          usage[:max_scan_arity] = [usage[:max_scan_arity], arity].max
+        end
+      end
+
+      # Check case conditions
+      func['states'].each do |state|
+        state['cases'].each do |kase|
+          check_expression(kase['condition'], usage)
+        end
+      end
+    end
+
+    # Collect all commands from a function (including nested in conditionals)
+    def collect_all_commands(func)
+      commands = []
+
+      # Entry actions
+      commands.concat(func['entry_actions'] || [])
+
+      # EOF handler
+      commands.concat(func['eof_handler'] || [])
+
+      # State commands
+      func['states'].each do |state|
+        commands.concat(state['eof_handler'] || [])
+        state['cases'].each do |kase|
+          kase['commands'].each do |cmd|
+            commands << cmd
+            # Recurse into conditional clauses
+            next unless cmd['type'] == 'conditional' && cmd.dig('args', 'clauses')
+
+            cmd['args']['clauses'].each do |clause|
+              commands.concat(clause['commands'] || [])
+            end
+          end
+        end
+      end
+
+      commands
+    end
+
+    def check_expression(expr, usage)
+      return unless expr.is_a?(String)
+
+      usage[:col]  = true if expr.match?(/\bCOL\b/)
+      usage[:prev] = true if expr.match?(/\bPREV\b/)
+    end
+
+    def check_special_class(special_class, usage)
+      return unless special_class
+
+      case special_class.to_s
+      when 'letter'     then usage[:letter] = true
+      when 'label_cont' then usage[:label_cont] = true
+      when 'digit'      then usage[:digit] = true
+      when 'hex_digit'  then usage[:hex_digit] = true
+      when 'ws'         then usage[:ws] = true
+      when 'nl'         then usage[:nl] = true
+      end
+    end
+
+    # Check if any function uses Unicode character classes
+    def uses_unicode_classes?(functions_data)
+      functions_data.any? do |func|
+        func['states'].any? do |state|
+          state['cases'].any? do |kase|
+            special_class = kase['special_class']
+            special_class && UNICODE_CLASSES.include?(special_class)
+          end
+        end
+      end
+    end
+
+    def keywords_to_hash(kw)
+      {
+        'name' => kw.name,
+        'const_name' => "#{kw.name.upcase}_KEYWORDS",
+        'fallback_func' => kw.fallback_func,
+        'fallback_args' => kw.fallback_args,
+        'mappings' => kw.mappings.map do |m|
+          {
+            'keyword' => m[:keyword],
+            'event_type' => m[:event_type]
+          }
+        end
+      }
+    end
+
+    def type_to_hash(type)
+      {
+        'name' => type.name,
+        'kind' => type.kind.to_s,
+        'emits_start' => type.emits_start,
+        'emits_end' => type.emits_end
+      }
+    end
+
+    def function_to_hash(func)
+      # Extract initial values from entry_actions for locals
+      # This allows template to initialize locals directly instead of "= 0" then assignment
+      local_init_values = extract_local_init_values(func.entry_actions || [])
+
+      # Determine which locals need to be mutable (reassigned in body or use +=/-=)
+      mutable_locals = find_mutable_locals(func)
+
+      # Filter out pure assignments from entry_actions (they become initializers)
+      # Keep conditionals and non-assignment commands
+      filtered_entry_actions = (func.entry_actions || []).reject do |cmd|
+        cmd.type == :assign && local_init_values.key?(cmd.args[:var])
+      end
+
+      {
+        'name' => func.name,
+        'return_type' => func.return_type,
+        'params' => func.params,
+        'param_types' => func.param_types.transform_keys(&:to_s).transform_values(&:to_s),
+        'locals' => func.locals.transform_keys(&:to_s),
+        'local_init_values' => local_init_values,
+        'mutable_locals' => mutable_locals,
+        'states' => func.states.map { |s| state_to_hash(s) },
+        'eof_handler' => func.eof_handler&.map { |c| command_to_hash(c) } || [],
+        'entry_actions' => filtered_entry_actions.map { |c| command_to_hash(c) },
+        'emits_events' => func.emits_events,
+        'expects_char' => func.expects_char,
+        'emits_content_on_close' => func.emits_content_on_close,
+        'prepend_values' => func.prepend_values.transform_keys(&:to_s),
+        'lineno' => func.lineno
+      }
+    end
+
+    # Extract initial values for locals from entry_actions assignments.
+    # We convert any expression to Rust syntax and use it as the initializer.
+    def extract_local_init_values(entry_actions)
+      init_values = {}
+      entry_actions.each do |cmd|
+        next unless cmd.type == :assign
+
+        var  = cmd.args[:var]
+        expr = cmd.args[:expr]
+        next unless var && expr
+
+        # Convert the expression to Rust (inline expansion for COL/LINE/PREV)
+        rust_expr = expr
+                    .gsub(/\bCOL\b/, 'self.col()')
+                    .gsub(/\bLINE\b/, 'self.line as i32')
+                    .gsub(/\bPREV\b/, 'self.prev()')
+                    .gsub(/:([a-z_]\w*)/i) { ::Regexp.last_match(1) }
+        init_values[var] = rust_expr
+      end
+      init_values
+    end
+
+    # Find locals that need to be mutable (reassigned in function body, not just entry)
+    def find_mutable_locals(func)
+      mutable = Set.new
+
+      func.states.each do |state|
+        state.cases.each do |kase|
+          collect_mutable_vars(kase.commands, mutable)
+        end
+        collect_mutable_vars(state.eof_handler || [], mutable)
+      end
+
+      mutable.to_a
+    end
+
+    def collect_mutable_vars(commands, mutable)
+      commands.each do |cmd|
+        case cmd.type
+        when :assign, :add_assign, :sub_assign then mutable << cmd.args[:var] if cmd.args[:var]
+        when :conditional                      then cmd.args[:clauses]&.each do |clause|
+          collect_mutable_vars(clause.commands, mutable)
+        end
+        end
+      end
+    end
+
+    def state_to_hash(state)
+      {
+        'name' => state.name,
+        'cases' => state.cases.map { |c| case_to_hash(c) },
+        'eof_handler' => state.eof_handler&.map { |c| command_to_hash(c) } || [],
+        'scan_chars' => state.scan_chars,
+        'scannable' => state.scannable?,
+        'is_self_looping' => state.is_self_looping,
+        'has_default' => state.has_default,
+        'is_unconditional' => state.is_unconditional,
+        'newline_injected' => state.newline_injected,
+        'lineno' => state.lineno
+      }
+    end
+
+    def case_to_hash(kase)
+      {
+        'chars' => kase.chars,
+        'special_class' => kase.special_class&.to_s,
+        'param_ref' => kase.param_ref,
+        'condition' => kase.condition,
+        'is_conditional' => kase.conditional?,
+        'substate' => kase.substate,
+        'commands' => kase.commands.map { |c| command_to_hash(c) },
+        'is_default' => kase.default?,
+        'lineno' => kase.lineno
+      }
+    end
+
+    def command_to_hash(cmd)
+      args = cmd.args.transform_keys(&:to_s)
+
+      # Recursively convert nested commands in conditionals
+      if cmd.type == :conditional && args['clauses']
+        args['clauses'] = args['clauses'].map do |clause|
+          {
+            'condition' => clause['condition'],
+            'commands' => clause['commands'].map { |c| command_to_hash(c) }
+          }
+        end
+      end
+
+      { 'type' => cmd.type.to_s, 'args' => args }
+    end
+  end
+end

data/lib/descent/ir.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Descent
+  # Intermediate Representation - semantic model after analysis.
+  #
+  # IR nodes are enriched with inferred information:
+  # - Resolved types
+  # - SCAN optimization characters (inferred from state structure)
+  # - EOF handling requirements
+  # - Local variable declarations with types
+  module IR
+    # Top-level parser definition
+    Parser = Data.define(:name, :entry_point, :types, :functions, :keywords, :custom_error_codes) do
+      def initialize(name:, entry_point:, types: [], functions: [], keywords: [], custom_error_codes: []) = super
+    end
+
+    # Keywords for perfect hash (phf) lookup
+    # Generates static phf::Map for O(1) keyword matching
+    Keywords = Data.define(:name, :fallback_func, :fallback_args, :mappings, :lineno) do
+      # name: identifier (e.g., "bare" generates BARE_KEYWORDS)
+      # fallback_func: function to call when no keyword matches
+      # fallback_args: arguments to pass to fallback
+      # mappings: Array of {keyword: "string", event_type: "TypeName"}
+      def initialize(name:, fallback_func: nil, fallback_args: nil, mappings: [], lineno: 0) = super
+    end
+
+    # Resolved type information
+    TypeInfo = Data.define(:name, :kind, :emits_start, :emits_end, :lineno) do
+      # kind: :bracket (emits Start/End), :content (emits on return), :internal (no emit)
+      def initialize(name:, kind:, emits_start: false, emits_end: false, lineno: 0) = super
+
+      def bracket?  = kind == :bracket
+      def content?  = kind == :content
+      def internal? = kind == :internal
+    end
+
+    # Function with resolved semantics
+    Function = Data.define(:name, :return_type, :params, :param_types, :locals, :states, :eof_handler, :entry_actions,
+                           :emits_events, :expects_char, :emits_content_on_close, :prepend_values, :lineno) do
+      # params: Array of parameter names
+      # param_types: Hash mapping param name -> :byte or :i32 (inferred from usage)
+      # entry_actions: Commands to execute on function entry (e.g., variable initialization)
+      # expects_char: Single char that must be seen to return (inferred from return cases)
+      # emits_content_on_close: Whether TERM appears before return (emit content on unclosed EOF)
+      # prepend_values: Hash mapping param name -> Array of byte values that could be passed (for PREPEND)
+      def initialize(name:, return_type: nil, params: [], param_types: {}, locals: {}, states: [], eof_handler: nil,
+                     entry_actions: [], emits_events: false, expects_char: nil, emits_content_on_close: false,
+                     prepend_values: {}, lineno: 0)
+        super
+      end
+
+      def expects_closer? = !expects_char.nil?
+    end
+
+    # State with inferred optimizations
+    State = Data.define(:name, :cases, :eof_handler, :scan_chars, :is_self_looping, :has_default, :is_unconditional, :newline_injected,
+                        :lineno) do
+      # scan_chars: Array of chars for SIMD memchr scan, or nil if not applicable
+      # is_self_looping: true if has default case that loops back to self
+      # has_default: true if state has a default case (no chars, no condition)
+      # is_unconditional: true if first case has no char match (bare action case)
+      # newline_injected: true if '\n' was added to scan_chars by the generator (not a user target).
+      #   When true, the template must add a match arm for '\n' that updates line/column and
+      #   continues scanning (no state transition). This enables correct line tracking during
+      #   SIMD scans without runtime checks for whether '\n' is a target.
+      def initialize(name:, cases: [], eof_handler: nil, scan_chars: nil, is_self_looping: false,
+                     has_default: false, is_unconditional: false, newline_injected: false, lineno: 0) = super
+
+      def scannable? = !scan_chars.nil? && !scan_chars.empty?
+    end
+
+    # Case with resolved actions
+    Case = Data.define(:chars, :special_class, :param_ref, :condition, :substate, :commands, :lineno) do
+      # chars: Array of literal chars to match, or nil for default
+      # special_class: Symbol like :letter, :label_cont for special matchers
+      # param_ref: Parameter name to match against (for |c[:param]|), or nil
+      # condition: String condition for if-cases, or nil
+      # lineno: Source line number from .desc file (for trace output)
+      def initialize(chars: nil, special_class: nil, param_ref: nil, condition: nil, substate: nil, commands: [],
+                     lineno: 0) = super
+
+      def default?     = chars.nil? && special_class.nil? && param_ref.nil? && condition.nil?
+      def conditional? = !condition.nil?
+    end
+
+    # Resolved command
+    Command = Data.define(:type, :args) do
+      # type: :mark, :term, :advance, :emit, :call, :assign, :return, :transition, etc.
+      # args: Hash of type-specific arguments
+      def initialize(type:, args: {}) = super
+    end
+
+    # Conditional with resolved conditions
+    Conditional = Data.define(:clauses)
+
+    Clause = Data.define(:condition, :commands)
+  end
+end