lexer_kit 0.5.0

data/lib/lexer_kit/builder/compiler.rb

@@ -0,0 +1,596 @@
+# frozen_string_literal: true
+
+module LexerKit
+  class Builder
+    # Compiler transforms Builder definitions into IR::CompiledProgram.
+    class Compiler
+      def initialize(builder)
+        @builder = builder
+        @instructions = []
+        @constant_pool = IR::ConstantPool.new
+        @jump_tables = []
+        @dfa_tables = []
+        @token_names = []
+        @token_ids = {}
+        @mode_names = []
+        @mode_ids = {}
+        @keyword_tables = []
+        @modes = {}
+        @labels = {}
+        @label_counter = 0
+        @pending_jumps = []
+        @pending_dfa_run_if_match = []
+        @pending_match_literal_or_jump = []
+        @pending_emit_and_jump = []
+        @pending_set_match = []
+        @pending_literal_trie_commit = []
+        @pending_trie_entries = []
+        @dfa_cache = {}      # pattern => dfa_id (cache for regex DFAs)
+        @keyword_cache = {}  # token_def => table_id (cache for keyword tables)
+      end
+
+      def compile
+        # Assign token IDs
+        assign_token_ids
+
+        # Compile each mode
+        @builder.mode_defs.each do |_name, mode_def|
+          compile_mode(mode_def)
+        end
+
+        # Add HALT at end
+        emit(IR::Opcode::HALT)
+
+        # Resolve pending jumps
+        resolve_jumps
+        finalize_literal_tries
+
+        IR::CompiledProgram.new(
+          instructions: @instructions,
+          dfa_tables: @dfa_tables,
+          jump_tables: @jump_tables,
+          constant_pool: @constant_pool,
+          modes: @modes,
+          token_names: @token_names,
+          mode_names: @mode_names,
+          keyword_tables: @keyword_tables,
+          token_meta: build_token_meta,
+          version: @builder.version
+        )
+      end
+
+      private
+
+      def assign_token_ids
+        # Reserved token IDs (0-7)
+        # 0: Internal sentinel (never emitted)
+        # 1: :INVALID
+        # 2-7: Reserved for future use
+        @token_names = [
+          :__RESERVED_0__,  # 0: reserved
+          :INVALID,         # 1: error token
+          :__RESERVED_2__,  # 2: reserved
+          :__RESERVED_3__,  # 3: reserved
+          :__RESERVED_4__,  # 4: reserved
+          :__RESERVED_5__,  # 5: reserved
+          :__RESERVED_6__,  # 6: reserved
+          :__RESERVED_7__ # 7: reserved
+        ]
+        @token_ids = {
+          INVALID: LexerKit::INVALID_TOKEN_ID
+        }
+
+        # User-defined tokens start from FIRST_USER_TOKEN_ID (8)
+        @builder.token_defs.each do |token_def|
+          next if @token_ids.key?(token_def.name)
+
+          id = @token_names.size
+          @token_names << token_def.name
+          @token_ids[token_def.name] = id
+          token_def.token_id = id
+        end
+
+        # Keywords as separate tokens
+        @builder.keywords.each_value do |name|
+          next if @token_ids.key?(name)
+
+          id = @token_names.size
+          @token_names << name
+          @token_ids[name] = id
+        end
+      end
+
+      # Build token metadata hash { token_id => meta_hash }
+      def build_token_meta
+        meta = {}
+        @builder.token_defs.each do |token_def|
+          next unless token_def.meta
+
+          token_id = @token_ids[token_def.name]
+          meta[token_id] = token_def.meta if token_id
+        end
+        meta
+      end
+
+      def compile_mode(mode_def)
+        ensure_mode_id(mode_def.name)
+        @modes[mode_def.name] = current_offset
+
+        if mode_def.delimited
+          compile_delimited_mode(mode_def)
+        else
+          compile_regular_mode(mode_def)
+        end
+      end
+
+      def compile_delimited_mode(mode_def)
+        delimited = mode_def.delimited
+        token_id = @token_ids[delimited.name]
+
+        # MARK current position
+        emit(IR::Opcode::MARK)
+
+        # SCAN delimiter
+        close_const_id = @constant_pool.add(delimited.delimiter)
+        if delimited.escape
+          config_id = add_escape_config(delimited)
+          emit(IR::Opcode::SCAN_UNTIL_ESCAPE, config_id)
+        else
+          emit(IR::Opcode::SCAN_UNTIL, close_const_id)
+        end
+
+        # EMIT TEXT token (if any content)
+        if delimited.skip
+          emit(IR::Opcode::EMIT_SKIP)
+        else
+          emit(IR::Opcode::EMIT, token_id)
+        end
+
+        # Check for EOF
+        eof_label = new_label
+        emit_jump(IR::Opcode::JUMP_IF_EOF, eof_label)
+
+        # Skip past delimiter
+        emit(IR::Opcode::MATCH_LITERAL, close_const_id)
+
+        # Switch to inner mode if defined
+        if delimited.inner_mode
+          inner_mode_id = ensure_mode_id(delimited.inner_mode)
+          emit(IR::Opcode::PUSH_MODE, inner_mode_id)
+          emit(IR::Opcode::JUMP, @modes[mode_def.name])
+        elsif delimited.pop
+          emit(IR::Opcode::POP_MODE)
+        else
+          # Jump back to start
+          emit(IR::Opcode::JUMP, @modes[mode_def.name])
+        end
+
+        # EOF label
+        mark_label(eof_label)
+        emit(IR::Opcode::HALT)
+      end
+
+      def compile_regular_mode(mode_def)
+        loop_start = current_offset
+
+        # Check EOF
+        eof_label = new_label
+        emit_jump(IR::Opcode::JUMP_IF_EOF, eof_label)
+
+        # MARK position
+        emit(IR::Opcode::MARK)
+
+        # Group tokens by first byte for SWITCH_BYTE
+        literals = mode_def.literal_tokens
+        regexes = mode_def.regex_tokens
+
+        if literals.any? || regexes.any?
+          # Build switch table for first-byte dispatch
+          compile_token_dispatch(mode_def, loop_start)
+        end
+
+        # EOF: HALT
+        mark_label(eof_label)
+        emit(IR::Opcode::HALT)
+      end
+
+      # Build escape config for delimited with escape sequence
+      # Format: [2bytes: close_len] [2bytes: escape_len] [close bytes] [escape bytes]
+      def add_escape_config(token_def)
+        close = token_def.delimiter
+        escape = token_def.escape
+        escape_bytes = escape.b
+        bytes = [
+          close.bytesize,
+          escape_bytes.bytesize
+        ].pack("S>S>") + close.b + escape_bytes
+
+        @constant_pool.add(bytes)
+      end
+
+      def compile_token_dispatch(mode_def, loop_start)
+        # Collect candidates by first byte, preserving definition order.
+        candidates_by_byte = {}
+        first_bytes_cache = {}
+        order_map = {}
+
+        mode_def.tokens.each_with_index do |token_def, idx|
+          next unless token_def.literal? || token_def.regex?
+
+          order_map[token_def] = idx
+
+          if token_def.literal?
+            byte = token_def.pattern.getbyte(0)
+            (candidates_by_byte[byte] ||= []) << token_def
+          else
+            regex = DFA::RegexAST::Regex.parse(token_def.pattern)
+            first_bytes = first_bytes_cache[token_def.pattern] ||= DFA.first_byte_set(regex)
+            first_bytes.each do |byte|
+              (candidates_by_byte[byte] ||= []) << token_def
+            end
+          end
+        end
+
+        # Create jump table
+        jump_table_id = @jump_tables.size
+        jump_table_entries = {}
+
+        # Group bytes by candidate sets - bytes with same candidates share one handler
+        groups = group_bytes_by_candidates(candidates_by_byte)
+        groups.each do |_candidates, bytes|
+          common_label = new_label
+          bytes.each { |byte| jump_table_entries[byte] = common_label }
+        end
+
+        # Default: error handling
+        default_label = new_label
+
+        @jump_tables << nil # Placeholder
+
+        # Emit SWITCH_BYTE
+        emit(IR::Opcode::SWITCH_BYTE, jump_table_id)
+
+        # Compile branches for each group (bytes with same candidates share handler)
+        groups.each do |candidates, bytes|
+          # All bytes in this group point to the same label
+          common_label = jump_table_entries[bytes.first]
+          mark_label(common_label)
+
+          if candidates.empty?
+            emit_jump(IR::Opcode::JUMP, default_label)
+            next
+          end
+
+          if candidates.size == 1
+            token_def = candidates.first
+            if token_def.literal?
+              const_id = @constant_pool.add(token_def.pattern)
+              # MATCH_LITERAL_OR_JUMP has embedded fail_target
+              emit_match_literal_or_jump(const_id, default_label)
+            elsif token_def.regex?
+              dfa_id = compile_regex_to_dfa(token_def.pattern)
+              # DFA_RUN_IF_MATCH has embedded fail_target
+              emit_dfa_run_if_match(dfa_id, default_label)
+            else
+              emit_jump(IR::Opcode::JUMP, default_label)
+              next
+            end
+
+            compile_token_action(token_def, loop_start)
+            next
+          end
+
+          action_labels = []
+          action_label_map = {}
+          candidates.each do |token_def|
+            action_label = new_label
+            action_labels << [action_label, token_def]
+            action_label_map[token_def] = action_label
+          end
+
+          literal_tokens = candidates.select(&:literal?)
+          regex_tokens = candidates.select(&:regex?)
+
+          if regex_tokens.empty?
+            # Literal-only: use optimized LITERAL_TRIE_COMMIT
+            trie_const_id = add_literal_trie(literal_tokens, order_map, action_label_map)
+            emit_literal_trie_commit(trie_const_id, default_label)
+          else
+            # Mixed literals and regex: use CLEAR_BEST + ... + COMMIT_BEST
+            emit(IR::Opcode::CLEAR_BEST)
+
+            if literal_tokens.any?
+              trie_const_id = add_literal_trie(literal_tokens, order_map, action_label_map)
+              # LITERAL_TRIE_RUN updates best match and restores pos to mark
+              emit(IR::Opcode::LITERAL_TRIE_RUN, trie_const_id)
+            end
+
+            regex_tokens.each do |token_def|
+              dfa_id = compile_regex_to_dfa(token_def.pattern)
+              emit(IR::Opcode::DFA_RUN, dfa_id)
+              # SET_MATCH sets candidate order/action and updates best match
+              emit_set_match(order_map[token_def], action_label_map[token_def])
+            end
+
+            emit_label_arg(IR::Opcode::COMMIT_BEST, default_label)
+          end
+
+          action_labels.each do |action_label, token_def|
+            mark_label(action_label)
+            compile_token_action(token_def, loop_start)
+          end
+        end
+
+        # Default branch (error handling)
+        mark_label(default_label)
+
+        # Error: advance one byte and emit :INVALID
+        emit(IR::Opcode::MATCH_RANGE, (0 << 8) | 255) # Match any byte
+        emit(IR::Opcode::EMIT_ERROR, LexerKit::INVALID_TOKEN_ID)
+        emit(IR::Opcode::JUMP, loop_start)
+
+        # Now resolve the jump table
+        resolved_entries = {}
+        jump_table_entries.each do |byte, label|
+          resolved_entries[byte] = @labels[label]
+        end
+        @jump_tables[jump_table_id] = IR::JumpTable.new(
+          entries: resolved_entries,
+          default_offset: @labels[default_label]
+        )
+      end
+
+      def compile_token_action(token_def, loop_start)
+        token_id = @token_ids[token_def.name]
+
+        if should_attach_keywords?(token_def)
+          table_id = build_keyword_table(token_def)
+          emit(IR::Opcode::KEYWORD_LOOKUP, table_id)
+        end
+
+        if token_def.skip
+          if token_def.push || token_def.pop
+            # When push/pop is involved, we need separate EMIT_SKIP
+            emit(IR::Opcode::EMIT_SKIP)
+          else
+            # Optimize EMIT_SKIP + JUMP into EMIT_SKIP_AND_JUMP
+            emit(IR::Opcode::EMIT_SKIP_AND_JUMP, loop_start)
+            return
+          end
+        elsif token_def.push || token_def.pop
+          # When push/pop is involved, we need separate EMIT
+          emit(IR::Opcode::EMIT, token_id)
+        else
+          # Optimize EMIT + JUMP into EMIT_AND_JUMP
+          emit_emit_and_jump(token_id, loop_start)
+          return
+        end
+
+        if token_def.push
+          mode_id = ensure_mode_id(token_def.push)
+          emit(IR::Opcode::PUSH_MODE, mode_id)
+        elsif token_def.pop
+          emit(IR::Opcode::POP_MODE)
+        end
+      end
+
+      def should_attach_keywords?(token_def)
+        return false if @builder.keywords.empty?
+        return false unless token_def.pattern.is_a?(Regexp)
+
+        @builder.keywords.keys.any? do |keyword_str|
+          token_def.pattern.match?(keyword_str)
+        end
+      end
+
+      def build_keyword_table(token_def)
+        # Cache keyword table by token_def to avoid duplicates
+        return @keyword_cache[token_def] if @keyword_cache.key?(token_def)
+
+        table_id = @keyword_tables.size
+        keywords_hash = {}
+        @builder.keywords.each do |value, name|
+          keywords_hash[value] = @token_ids[name]
+        end
+        @keyword_tables << IR::KeywordTable.new(
+          base_token_id: @token_ids[token_def.name],
+          keywords: keywords_hash
+        )
+        @keyword_cache[token_def] = table_id
+        table_id
+      end
+
+      def compile_regex_to_dfa(pattern)
+        # Cache DFA by pattern source to avoid duplicate compilation
+        cache_key = pattern.is_a?(Regexp) ? pattern.source : pattern
+        return @dfa_cache[cache_key] if @dfa_cache.key?(cache_key)
+
+        dfa_id = @dfa_tables.size
+
+        # Use proper regex→DFA compilation
+        regex = DFA::RegexAST::Regex.parse(pattern)
+        dfa_table = DFA.compile_regex(regex, 0)
+        @dfa_tables << dfa_table
+
+        @dfa_cache[cache_key] = dfa_id
+        dfa_id
+      end
+
+      def ensure_mode_id(mode_name)
+        return @mode_ids[mode_name] if @mode_ids.key?(mode_name)
+
+        id = @mode_names.size
+        @mode_names << mode_name
+        @mode_ids[mode_name] = id
+        id
+      end
+
+      def current_offset
+        @instructions.size
+      end
+
+      def emit(opcode, arg = 0)
+        @instructions << IR::Instruction.new(opcode, arg)
+      end
+
+      def emit_jump(opcode, label)
+        @pending_jumps << [@instructions.size, label]
+        emit(opcode, 0) # Placeholder
+      end
+
+      def emit_label_arg(opcode, label)
+        @pending_jumps << [@instructions.size, label]
+        emit(opcode, 0) # Placeholder
+      end
+
+      # Emit DFA_RUN_IF_MATCH with embedded fail_target
+      # Encoding: (dfa_id << 14) | fail_target (10+14 bits)
+      def emit_dfa_run_if_match(dfa_id, fail_label)
+        @pending_dfa_run_if_match << [@instructions.size, dfa_id, fail_label]
+        emit(IR::Opcode::DFA_RUN_IF_MATCH, dfa_id << 14) # Placeholder with dfa_id in upper bits
+      end
+
+      # Emit MATCH_LITERAL_OR_JUMP with embedded fail_target
+      # Encoding: (const_id << 14) | fail_target (10+14 bits)
+      def emit_match_literal_or_jump(const_id, fail_label)
+        @pending_match_literal_or_jump << [@instructions.size, const_id, fail_label]
+        emit(IR::Opcode::MATCH_LITERAL_OR_JUMP, const_id << 14) # Placeholder with const_id in upper bits
+      end
+
+      # Emit EMIT_AND_JUMP with embedded jump_target
+      # Encoding: (token_id << 14) | jump_target (10+14 bits)
+      def emit_emit_and_jump(token_id, jump_target)
+        @pending_emit_and_jump << [@instructions.size, token_id, jump_target]
+        emit(IR::Opcode::EMIT_AND_JUMP, token_id << 14) # Placeholder with token_id in upper bits
+      end
+
+      # Emit SET_MATCH with embedded order and action_label
+      # Encoding: (order << 14) | action_ip (10+14 bits)
+      def emit_set_match(order, action_label)
+        @pending_set_match << [@instructions.size, order, action_label]
+        emit(IR::Opcode::SET_MATCH, order << 14) # Placeholder with order in upper bits
+      end
+
+      # Emit LITERAL_TRIE_COMMIT with embedded const_id and fail_target
+      # Encoding: (const_id << 14) | fail_target (10+14 bits)
+      def emit_literal_trie_commit(const_id, fail_label)
+        @pending_literal_trie_commit << [@instructions.size, const_id, fail_label]
+        emit(IR::Opcode::LITERAL_TRIE_COMMIT, const_id << 14) # Placeholder with const_id in upper bits
+      end
+
+      def add_literal_trie(literals, order_map, action_label_map)
+        # Convert TokenDef objects to Trie entries
+        entries = literals.map do |token_def|
+          [
+            token_def.pattern,           # literal string
+            order_map[token_def],        # order
+            action_label_map[token_def]  # action_ref (label)
+          ]
+        end
+
+        trie = Trie.new(entries)
+        const_id = @constant_pool.add_uninterned("".b)
+        @pending_trie_entries << [const_id, trie]
+        const_id
+      end
+
+      def finalize_literal_tries
+        return if @pending_trie_entries.empty?
+
+        @pending_trie_entries.each do |const_id, trie|
+          data = trie.encode(labels: @labels)
+          @constant_pool.replace(const_id, data)
+        end
+      end
+
+      # Group bytes by their candidate sets.
+      # Returns array of [candidates, bytes] pairs where all bytes in a group
+      # share the exact same candidate set.
+      def group_bytes_by_candidates(candidates_by_byte)
+        grouped = Hash.new { |h, k| h[k] = [] }
+
+        candidates_by_byte.each do |byte, candidates|
+          # Use object_id array as key to identify same candidate set
+          key = candidates.map(&:object_id).sort
+          grouped[key] << byte
+        end
+
+        result = []
+        grouped.each do |_key, bytes|
+          candidates = candidates_by_byte[bytes.first]
+          result << [candidates, bytes.sort]
+        end
+
+        result
+      end
+
+      def new_label
+        label = :"label_#{@label_counter}"
+        @label_counter += 1
+        label
+      end
+
+      def mark_label(label)
+        @labels[label] = current_offset
+      end
+
+      def resolve_jumps
+        @pending_jumps.each do |instr_idx, label|
+          offset = @labels[label]
+          raise LexerKit::CompileError, "unresolved label #{label}" unless offset
+
+          instr = @instructions[instr_idx]
+          @instructions[instr_idx] = IR::Instruction.new(instr.opcode, offset)
+        end
+
+        # Resolve DFA_RUN_IF_MATCH fail targets
+        # Encoding: (dfa_id << 14) | fail_target (10+14 bits)
+        @pending_dfa_run_if_match.each do |instr_idx, dfa_id, fail_label|
+          fail_offset = @labels[fail_label]
+          raise LexerKit::CompileError, "unresolved label #{fail_label}" unless fail_offset
+
+          arg = (dfa_id << 14) | fail_offset
+          @instructions[instr_idx] = IR::Instruction.new(IR::Opcode::DFA_RUN_IF_MATCH, arg)
+        end
+
+        # Resolve MATCH_LITERAL_OR_JUMP fail targets
+        # Encoding: (const_id << 14) | fail_target (10+14 bits)
+        @pending_match_literal_or_jump.each do |instr_idx, const_id, fail_label|
+          fail_offset = @labels[fail_label]
+          raise LexerKit::CompileError, "unresolved label #{fail_label}" unless fail_offset
+
+          arg = (const_id << 14) | fail_offset
+          @instructions[instr_idx] = IR::Instruction.new(IR::Opcode::MATCH_LITERAL_OR_JUMP, arg)
+        end
+
+        # Resolve EMIT_AND_JUMP targets
+        # Encoding: (token_id << 14) | jump_target (10+14 bits)
+        # Note: jump_target is a direct offset (loop_start), not a label
+        @pending_emit_and_jump.each do |instr_idx, token_id, jump_target|
+          arg = (token_id << 14) | jump_target
+          @instructions[instr_idx] = IR::Instruction.new(IR::Opcode::EMIT_AND_JUMP, arg)
+        end
+
+        # Resolve SET_MATCH action targets
+        # Encoding: (order << 14) | action_ip (10+14 bits)
+        @pending_set_match.each do |instr_idx, order, action_label|
+          action_offset = @labels[action_label]
+          raise LexerKit::CompileError, "unresolved label #{action_label}" unless action_offset
+
+          arg = (order << 14) | action_offset
+          @instructions[instr_idx] = IR::Instruction.new(IR::Opcode::SET_MATCH, arg)
+        end
+
+        # Resolve LITERAL_TRIE_COMMIT fail targets
+        # Encoding: (const_id << 14) | fail_target (10+14 bits)
+        @pending_literal_trie_commit.each do |instr_idx, const_id, fail_label|
+          fail_offset = @labels[fail_label]
+          raise LexerKit::CompileError, "unresolved label #{fail_label}" unless fail_offset
+
+          arg = (const_id << 14) | fail_offset
+          @instructions[instr_idx] = IR::Instruction.new(IR::Opcode::LITERAL_TRIE_COMMIT, arg)
+        end
+      end
+    end
+  end
+end