lexer_kit 0.5.0

data/lib/lexer_kit/format/lkb1/decoder.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module LexerKit
+  module Format
+    class LKB1
+      # Decoder handles binary deserialization of LKB1 format
+      class Decoder
+        def initialize(bytes)
+          @bytes = bytes
+          @pos = 0
+        end
+
+        def decode_container
+          raise ArgumentError, "too short" if @bytes.bytesize < FIXED_HEADER_LEN
+
+          magic = read_bytes(4)
+          raise ArgumentError, "invalid magic: #{magic.inspect}" unless magic == MAGIC
+
+          header_version = read_u16
+          raise ArgumentError, "unsupported header version: #{header_version}" unless header_version == HEADER_VERSION
+
+          flags = read_u16
+
+          # Reject unsupported flags
+          if (flags & FLAG_PAYLOAD_COMPRESSED) != 0
+            raise ArgumentError, "compressed payload is not supported"
+          end
+
+          if (flags & FLAG_PAYLOAD_ENCRYPTED) != 0
+            raise ArgumentError, "encrypted payload is not supported"
+          end
+
+          header_len = read_u32
+          raise ArgumentError, "invalid header_len: #{header_len}" if header_len < FIXED_HEADER_LEN
+
+          payload_len = read_u32
+          format_version = read_u16
+
+          @pos += 2 # reserved
+
+          sha256 = read_bytes(32)
+
+          meta = {}
+          if header_len > @pos
+            meta_bytes = @bytes.byteslice(@pos, header_len - @pos)
+            meta = decode_tlv(meta_bytes)
+            @pos = header_len
+          end
+
+          total_len = header_len + payload_len
+          raise ArgumentError, "truncated payload" if @bytes.bytesize < total_len
+
+          payload = @bytes.byteslice(header_len, payload_len)
+
+          if (flags & FLAG_PAYLOAD_SHA256) != 0
+            actual = Digest::SHA256.digest(payload)
+            raise ArgumentError, "sha256 mismatch" unless actual == sha256
+          end
+
+          {
+            payload: payload,
+            meta: meta,
+            header: {
+              header_version: header_version,
+              flags: flags,
+              header_len: header_len,
+              payload_len: payload_len,
+              format_version: format_version,
+              sha256: sha256
+            }
+          }
+        end
+
+        private
+
+        def read_bytes(length)
+          result = @bytes.byteslice(@pos, length)
+          @pos += length
+          result
+        end
+
+        def read_u16
+          @bytes.byteslice(@pos, 2).unpack1("S>").tap { @pos += 2 }
+        end
+
+        def read_u32
+          @bytes.byteslice(@pos, 4).unpack1("L>").tap { @pos += 4 }
+        end
+
+        def decode_tlv(bytes)
+          pos = 0
+          meta = {}
+          while pos + 4 <= bytes.bytesize
+            type = bytes.byteslice(pos, 2).unpack1("S>")
+            pos += 2
+            len = bytes.byteslice(pos, 2).unpack1("S>")
+            pos += 2
+            raise ArgumentError, "invalid tlv length" if pos + len > bytes.bytesize
+
+            value = bytes.byteslice(pos, len)
+            pos += len
+            case type
+            when TLV_BUILD_ID
+              meta[:build_id] = value.force_encoding(Encoding::UTF_8)
+            when TLV_SOURCE_VERSION
+              meta[:source_version] = value.force_encoding(Encoding::UTF_8)
+            when TLV_TOKEN_COUNT
+              meta[:token_count] = value.unpack1("L>")
+            when TLV_MODE_COUNT
+              meta[:mode_count] = value.unpack1("S>")
+            when TLV_INSTRUCTION_COUNT
+              meta[:instruction_count] = value.unpack1("L>")
+            when TLV_CREATED_AT
+              meta[:created_at] = value.unpack1("Q>")
+            when TLV_GENERATOR_VERSION
+              meta[:generator_version] = value.force_encoding(Encoding::UTF_8)
+            end
+          end
+          meta
+        end
+      end
+    end
+  end
+end

data/lib/lexer_kit/format/lkb1.rb

@@ -0,0 +1,199 @@
+# frozen_string_literal: true
+
+require "digest"
+require_relative "lkb1/decoder"
+
+module LexerKit
+  module Format
+    # LKB1 (LexerKit Binary format version 1) is a binary container format
+    # for compiled lexer programs. It provides efficient loading with optional
+    # metadata and integrity checking via SHA256.
+    class LKB1
+      MAGIC = "LKB1"
+      HEADER_VERSION = 4
+      FIXED_HEADER_LEN = 52
+
+      FLAG_PAYLOAD_COMPRESSED = 1 << 0
+      FLAG_PAYLOAD_ENCRYPTED = 1 << 1
+      FLAG_PAYLOAD_SHA256 = 1 << 2
+      FLAG_META = 1 << 3
+
+      TLV_BUILD_ID = 0x0001
+      TLV_SOURCE_VERSION = 0x0002
+      TLV_TOKEN_COUNT = 0x0003
+      TLV_MODE_COUNT = 0x0004
+      TLV_INSTRUCTION_COUNT = 0x0005
+      TLV_CREATED_AT = 0x0006
+      TLV_GENERATOR_VERSION = 0x0007
+
+      # TLV length is u16, so string data cannot exceed this
+      TLV_MAX_STRING_LENGTH = (2**16) - 1 # 65535 bytes
+
+      attr_reader :program, :meta
+
+      # Create a new LKB1 instance from a compiled program
+      # @param program [IR::CompiledProgram] compiled lexer program
+      # @param meta [Hash] optional metadata
+      def initialize(program, meta: {})
+        @program = program
+        @meta = meta
+      end
+
+      # Load a compiled lexer program from a .lkb1 file
+      # @param path [String] path to .lkb1 file
+      # @return [LKB1] LKB1 instance
+      def self.load(path)
+        bytes = File.binread(path)
+        decode(bytes)
+      end
+
+      # Save a compiled lexer program to a .lkb1 file (shortcut)
+      # @param program [IR::CompiledProgram] compiled lexer program
+      # @param path [String] output file path
+      # @param meta [Hash] optional metadata for header
+      def self.save(program, path:, meta: {})
+        new(program, meta: meta).save(path)
+      end
+
+      # Decode a compiled lexer program from lkb1 binary data
+      # @param bytes [String] binary data
+      # @return [LKB1] LKB1 instance
+      def self.decode(bytes)
+        decoded = Decoder.new(bytes).decode_container
+        header = decoded[:header]
+
+        unless header[:format_version] == IR::Serializer::FORMAT_VERSION
+          raise LexerKit::IntegrityError, "Unsupported format version: #{header[:format_version]}"
+        end
+
+        program = IR::CompiledProgram.from_binary(decoded[:payload])
+        program.load_native! if LexerKit.native?
+        new(program, meta: decoded[:meta])
+      rescue ArgumentError => e
+        raise LexerKit::IntegrityError, e.message
+      end
+
+      # Encode the program to lkb1 binary data
+      # @return [String] binary data
+      def encode
+        payload = @program.to_binary
+
+        # Auto-generate metadata from program
+        default_meta = {
+          token_count: @program.token_names.size,
+          mode_count: @program.mode_names.size,
+          instruction_count: @program.instructions.size,
+          created_at: Time.now.to_i,
+          generator_version: LexerKit::VERSION
+        }
+
+        encode_container(
+          payload,
+          meta: default_meta.merge(@meta),
+          format_version: IR::Serializer::FORMAT_VERSION
+        )
+      end
+
+      # Save the program to a .lkb1 file
+      # @param path [String] output file path
+      def save(path)
+        File.binwrite(path, encode)
+      end
+
+      private
+
+      # Encode binary container with metadata
+      # @param payload [String] binary payload
+      # @param format_version [Integer] format version number
+      # @param meta [Hash] metadata
+      # @return [String] binary container
+      def encode_container(payload, format_version:, meta: {})
+        flags = FLAG_PAYLOAD_SHA256
+        tlv = encode_tlv(meta)
+        flags |= FLAG_META unless tlv.empty?
+
+        sha256 = Digest::SHA256.digest(payload)
+        header_len = FIXED_HEADER_LEN + tlv.bytesize
+
+        header = []
+        header << MAGIC
+        header << [HEADER_VERSION].pack("S>")
+        header << [flags].pack("S>")
+        header << [header_len].pack("L>")
+        header << [payload.bytesize].pack("L>")
+        header << [format_version].pack("S>")
+        header << [0].pack("S>")
+        header << sha256
+        header << tlv
+
+        header.join + payload
+      end
+
+      # Encode metadata as TLV (Type-Length-Value) format
+      # @param meta [Hash] metadata
+      # @return [String] TLV-encoded binary data
+      def encode_tlv(meta)
+        parts = []
+        add_tlv_string(parts, TLV_BUILD_ID, meta[:build_id])
+        add_tlv_string(parts, TLV_SOURCE_VERSION, meta[:source_version])
+        add_tlv_u32(parts, TLV_TOKEN_COUNT, meta[:token_count])
+        add_tlv_u16(parts, TLV_MODE_COUNT, meta[:mode_count])
+        add_tlv_u32(parts, TLV_INSTRUCTION_COUNT, meta[:instruction_count])
+        add_tlv_u64(parts, TLV_CREATED_AT, meta[:created_at])
+        add_tlv_string(parts, TLV_GENERATOR_VERSION, meta[:generator_version])
+        parts.join
+      end
+
+      def add_tlv_string(parts, type, value)
+        return if value.nil?
+
+        # Ensure UTF-8 encoding and validate
+        str = value.to_s
+        unless str.encoding == Encoding::UTF_8 || str.force_encoding(Encoding::UTF_8).valid_encoding?
+          raise ArgumentError, "TLV string must be valid UTF-8"
+        end
+
+        bytes = str.b
+        if bytes.bytesize > TLV_MAX_STRING_LENGTH
+          raise ArgumentError, "TLV string length exceeds maximum (#{TLV_MAX_STRING_LENGTH} bytes): #{bytes.bytesize}"
+        end
+
+        parts << [type, bytes.bytesize].pack("S>S>")
+        parts << bytes
+      end
+
+      def add_tlv_u16(parts, type, value)
+        return if value.nil?
+
+        unless value.is_a?(Integer) && value >= 0 && value <= (2**16) - 1
+          raise ArgumentError, "TLV u16 value out of range (0..#{(2**16) - 1}): #{value}"
+        end
+
+        parts << [type, 2].pack("S>S>")
+        parts << [value].pack("S>")
+      end
+
+      def add_tlv_u32(parts, type, value)
+        return if value.nil?
+
+        unless value.is_a?(Integer) && value >= 0 && value <= (2**32) - 1
+          raise ArgumentError, "TLV u32 value out of range (0..#{(2**32) - 1}): #{value}"
+        end
+
+        parts << [type, 4].pack("S>S>")
+        parts << [value].pack("L>")
+      end
+
+      def add_tlv_u64(parts, type, value)
+        return if value.nil?
+
+        unless value.is_a?(Integer) && value >= 0 && value <= (2**64) - 1
+          raise ArgumentError, "TLV u64 value out of range (0..#{(2**64) - 1}): #{value}"
+        end
+
+        parts << [type, 8].pack("S>S>")
+        parts << [value].pack("Q>")
+      end
+    end
+  end
+end

data/lib/lexer_kit/format/lkt1.rb

@@ -0,0 +1,111 @@
+# frozen_string_literal: true
+
+require "json"
+require "zlib"
+require "base64"
+require "digest"
+
+module LexerKit
+  module Format
+    # LKT1 (LexerKit Text format version 1) is a JSON-based container format
+    # for compiled lexer programs. It supports compression and includes
+    # integrity checking via SHA256.
+    class LKT1
+      attr_reader :program
+
+      # Create a new LKT1 instance from a compiled program
+      # @param program [IR::CompiledProgram] compiled lexer program
+      def initialize(program)
+        @program = program
+      end
+
+      # Load a compiled lexer program from a .lkt1 file
+      # @param path [String] path to .lkt1 file
+      # @return [LKT1] LKT1 instance
+      def self.load(path)
+        content = File.read(path)
+        decode(content)
+      end
+
+      # Save a compiled lexer program to a .lkt1 file (shortcut)
+      # @param program [IR::CompiledProgram] compiled lexer program
+      # @param path [String] output file path
+      def self.save(program, path:)
+        new(program).save(path)
+      end
+
+      # Decode a compiled lexer program from lkt1 JSON string
+      # @param json_string [String] lkt1 JSON string
+      # @return [LKT1] LKT1 instance
+      def self.decode(json_string)
+        container = JSON.parse(json_string, symbolize_names: true)
+
+        # Validate format
+        unless container[:format] == "lkt1"
+          raise LexerKit::IntegrityError, "Unknown format: #{container[:format]}"
+        end
+
+        unless container[:kind] == "program"
+          raise LexerKit::IntegrityError, "Unknown kind: #{container[:kind]}"
+        end
+
+        # Decode based on codec
+        unless container[:codec] == "deflate+base64"
+          raise LexerKit::IntegrityError, "Unknown codec: #{container[:codec]}"
+        end
+
+        compressed = Base64.strict_decode64(container[:data])
+        binary = Zlib::Inflate.inflate(compressed)
+
+        # Verify integrity (sha256 is required)
+        unless container[:sha256]
+          raise LexerKit::IntegrityError, "Missing required field: sha256"
+        end
+
+        actual_hash = Digest::SHA256.hexdigest(binary)
+        unless actual_hash == container[:sha256]
+          raise LexerKit::IntegrityError, "SHA256 mismatch: expected #{container[:sha256]}, got #{actual_hash}"
+        end
+
+        if container[:uncompressed_len] && binary.bytesize != container[:uncompressed_len]
+          raise LexerKit::IntegrityError, "Length mismatch: expected #{container[:uncompressed_len]}, got #{binary.bytesize}"
+        end
+
+        program = IR::CompiledProgram.from_binary(binary)
+        program.load_native! if LexerKit.native?
+        new(program)
+      rescue JSON::ParserError => e
+        raise LexerKit::IntegrityError, "Invalid JSON: #{e.message}"
+      rescue ArgumentError => e
+        # Base64 decode errors
+        raise LexerKit::IntegrityError, "Decoding error: #{e.message}"
+      rescue Zlib::Error => e
+        raise LexerKit::IntegrityError, "Decompression error: #{e.message}"
+      end
+
+      # Encode the program to lkt1 JSON string
+      # @return [String] JSON string
+      def encode
+        binary = @program.to_binary
+        compressed = Zlib::Deflate.deflate(binary)
+        encoded = Base64.strict_encode64(compressed)
+
+        JSON.generate({
+          format: "lkt1",
+          codec: "deflate+base64",
+          kind: "program",
+          table_version: 2,
+          uncompressed_len: binary.bytesize,
+          sha256: Digest::SHA256.hexdigest(binary),
+          data: encoded
+        })
+      end
+
+      # Save the program to a .lkt1 file
+      # @param path [String] output file path
+      def save(path)
+        File.write(path, encode)
+      end
+    end
+  end
+end

data/lib/lexer_kit/format.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative "format/lkt1"
+require_relative "format/lkb1"
+
+module LexerKit
+  # Format provides file format implementations for lexer programs.
+  # Supports LKT1 (JSON-based) and LKB1 (binary) formats.
+  #
+  # @example Loading a lexer
+  #   program = LexerKit::Format::LKT1.load("lexer.lkt1")
+  #   program = LexerKit::Format::LKB1.load("lexer.lkb1")
+  #
+  # @example Saving a lexer
+  #   LexerKit::Format::LKT1.save(program, "lexer.lkt1")
+  #   LexerKit::Format::LKB1.save(program, "lexer.lkb1")
+  module Format
+  end
+end

data/lib/lexer_kit/ir/compiled_program.rb

@@ -0,0 +1,228 @@
+# frozen_string_literal: true
+
+module LexerKit
+  module IR
+    # CompiledProgram is the complete compiled lexer ready for execution.
+    # It contains instructions, DFA tables, constants, and metadata.
+    # Binary serialization is handled by Serializer class.
+    #
+    # Note: Native methods are included by Rust extension when loaded.
+    class CompiledProgram
+      attr_reader :instructions, :dfa_tables, :jump_tables, :constant_pool, :token_names, :mode_names, :keyword_tables, :version, :mode_offsets, :token_meta
+
+      # @param instructions [Array<Instruction>] instruction list
+      # @param dfa_tables [Array<DFATable>] DFA tables
+      # @param jump_tables [Array<JumpTable>] jump tables
+      # @param constant_pool [ConstantPool] string constants
+      # @param modes [Hash<Symbol, Integer>] mode name → start instruction offset
+      # @param token_names [Array<Symbol>] token ID → name mapping
+      # @param mode_names [Array<Symbol>] mode ID → name mapping
+      # @param keyword_tables [Array<KeywordTable>] keyword tables
+      # @param token_meta [Hash<Integer, Hash>] token ID → metadata hash
+      # @param version [Integer] user-defined version number
+      def initialize(
+        instructions:,
+        dfa_tables: [],
+        jump_tables: [],
+        constant_pool: nil,
+        modes: {},
+        token_names: [],
+        mode_names: [],
+        keyword_tables: [],
+        token_meta: {},
+        version: 1
+      )
+        @instructions = instructions.freeze
+        @dfa_tables = dfa_tables.freeze
+        @jump_tables = jump_tables.freeze
+        @constant_pool = constant_pool || ConstantPool.new
+        @mode_offsets = modes.freeze
+        @token_names = token_names.freeze
+        @mode_names = mode_names.freeze
+        @keyword_tables = keyword_tables.freeze
+        @token_meta = token_meta.freeze
+        @version = version
+      end
+
+      # Kind of compiled program
+      # @return [Symbol]
+      def kind
+        :program
+      end
+
+      # Get token ID by name
+      # @param name [Symbol]
+      # @return [Integer, nil]
+      def token_id(name)
+        @token_names.index(name)
+      end
+
+      # Get token name by ID
+      # @param id [Integer]
+      # @return [Symbol, nil]
+      def token_name(id)
+        @token_names[id]
+      end
+
+      # Get all token names (excludes reserved placeholder tokens)
+      # @return [Array<Symbol>]
+      def tokens
+        @token_names.reject { |name| name.to_s.start_with?("__RESERVED_") }
+      end
+
+      # Get all mode names
+      # @return [Array<Symbol>]
+      def modes
+        @mode_names.dup
+      end
+
+      # Get mode ID by name
+      # @param name [Symbol]
+      # @return [Integer, nil]
+      def mode_id(name)
+        @mode_names.index(name)
+      end
+
+      # Get mode start offset
+      # @param name [Symbol]
+      # @return [Integer, nil]
+      def mode_offset(name)
+        @mode_offsets[name]
+      end
+
+      # Low-level lexing with callback (for performance-critical code)
+      # @param bytes [String] input bytes
+      # @yield [Integer, Integer, Integer] token_id, start, length
+      # @raise [NativeExtensionError] if Rust extension is not loaded
+      def lowlevel_each(bytes, &)
+        bytes = bytes.b
+        raise LexerKit::NativeExtensionError, "Rust extension not loaded" unless LexerKit.native?
+
+        ensure_rust_native!
+        lex_rust_native(bytes, &)
+      end
+
+      # Check if a token ID is an error token
+      # Use this for fast error detection in the lex loop
+      # @param tok [Integer] token ID
+      # @return [Boolean]
+      def error_token?(tok)
+        tok == LexerKit::INVALID_TOKEN_ID
+      end
+
+      # Get metadata for a token ID
+      # @param tok [Integer] token ID
+      # @return [Hash] metadata hash (empty hash if no metadata)
+      def token_meta_for(tok)
+        @token_meta[tok] || {}
+      end
+
+      # Create a Token object on demand
+      # Use this to get rich token info only when needed (e.g., for errors)
+      # Source is created internally, so there's zero overhead if not called
+      # @param tok [Integer] token ID
+      # @param start [Integer] start byte offset
+      # @param len [Integer] length in bytes
+      # @param input [String] original input string
+      # @param filename [String, nil] optional filename for diagnostics
+      # @return [Core::Token]
+      def make_token(tok, start, len, input:, filename: nil)
+        source = Core::Source.new(input, filename: filename)
+        Core::Token.new(
+          id: tok,
+          name: token_name(tok),
+          start: start,
+          len: len,
+          source: source,
+          meta: @token_meta[tok]
+        )
+      end
+
+      # Create a stream-based lexer with lookahead support
+      # Returns a Runner that wraps the underlying LexStream.
+      # @param input [String] input string
+      # @param filename [String, nil] optional filename for diagnostics
+      # @return [Runner]
+      # @raise [NativeExtensionError] if Rust extension is not loaded
+      def stream(input, filename: nil)
+        raise LexerKit::NativeExtensionError, "Rust extension not loaded" unless LexerKit.native?
+
+        ensure_rust_native!
+        lex_stream = create_rust_stream(input)
+        Runner.new(self, lex_stream, filename: filename)
+      end
+
+      # Tokenize input and return array of Token objects
+      # Source is shared across all tokens for efficient line/col lookup
+      # @param input [String] input string
+      # @param filename [String, nil] optional filename for diagnostics
+      # @return [Array<Core::Token>]
+      def tokenize(input, filename: nil)
+        bytes = input.b
+        source = Core::Source.new(bytes, filename: filename)
+
+        tokens = []
+        lowlevel_each(bytes) do |tok_id, start, len|
+          tokens << Core::Token.new(
+            id: tok_id,
+            name: token_name(tok_id),
+            start: start,
+            len: len,
+            source: source,
+            meta: @token_meta[tok_id]
+          )
+        end
+        tokens
+      end
+
+      # Load native representation for fast lexing
+      # @return [self]
+      def load_native!
+        return self unless LexerKit.native?
+
+        load_rust_native(to_native_data)
+        self
+      end
+
+      # Convert to data format for Rust native loading
+      # @return [Hash] data for Rust extension
+      def to_native_data
+        {
+          instructions: @instructions.map(&:to_binary).join,
+          dfa_tables: @dfa_tables.map(&:to_native_format),
+          jump_tables: @jump_tables.map(&:to_native_format),
+          keyword_tables: @keyword_tables.map(&:to_native_format),
+          constant_pool: @constant_pool.entries,
+          modes: @mode_offsets.map { |name, offset| [name.to_s, offset] }
+        }
+      end
+
+      # Encode to binary
+      # @return [String]
+      def to_binary
+        Serializer.to_binary(self)
+      end
+
+      # Decode from binary
+      # @param bytes [String]
+      # @return [CompiledProgram]
+      def self.from_binary(bytes)
+        Serializer.from_binary(bytes)
+      end
+
+      def inspect
+        "#<CompiledProgram v#{@version} instructions=#{@instructions.size} tokens=#{@token_names.size} native=#{LexerKit.native?}>"
+      end
+
+      private
+
+      # Ensure Rust native is loaded
+      # @api private
+      def ensure_rust_native!
+        return if respond_to?(:rust_native_loaded?) && rust_native_loaded?
+
+        load_rust_native(to_native_data)
+      end
+    end
+  end
+end