lex-llm 0.4.18 → 0.5.0

data/spec/legion/extensions/llm/conformance/client_translator_examples.rb

@@ -0,0 +1,269 @@
+# frozen_string_literal: true
+
+# Shared examples for canonical client translator conformance.
+#
+# Every client translator must implement:
+#   - parse_request(body, env) → Canonical::Request
+#   - format_response(canonical_response) → Hash
+#   - format_chunk(canonical_chunk) → Hash | nil
+#   - format_error(error, status) → [status, Hash]
+#
+# Usage:
+#   it_behaves_like 'a canonical client translator', MyClientTranslatorClass
+# rubocop:disable Lint/NonLocalExitFromIterator -- return guard is idiomatic in shared_example blocks
+RSpec.shared_examples 'a canonical client translator' do |translator_class|
+  let(:translator) { translator_class.new }
+  let(:canonical) { Legion::Extensions::Llm::Canonical }
+  let(:conformance) { Canonical::Conformance }
+
+  describe '#parse_request' do
+    context 'with a simple text request' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_simple_text_request'))
+      end
+
+      it 'returns a Canonical::Request' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        return unless formatted
+
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed).to be_a(canonical::Request)
+        expect(parsed.messages).to be_an(Array)
+        expect(parsed.messages.length).to be > 0
+      end
+    end
+
+    context 'with a system prompt' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_system_prompt_request'))
+      end
+
+      it 'preserves the system prompt' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        return unless formatted
+
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed.system).to be_a(String)
+        expect(parsed.system).to include('haiku')
+      end
+    end
+
+    context 'with tools defined' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_tools_request'))
+      end
+
+      it 'preserves tool definitions' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        return unless formatted
+
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed.tools).to be_a(Hash)
+        expect(parsed.tools.keys).to include(:get_weather)
+      end
+    end
+
+    context 'with thinking enabled' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_thinking_request'))
+      end
+
+      it 'preserves thinking configuration' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        return unless formatted
+
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed.thinking).to be_a(canonical::Thinking::Config)
+        expect(parsed.thinking.enabled?).to be true
+      end
+    end
+
+    context 'with parameter mapping' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_params_mapping_request'))
+      end
+
+      it 'preserves sampling parameters' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        return unless formatted
+
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed.params).to be_a(canonical::Params)
+        expect(parsed.params.max_tokens).to eq(2048)
+        expect(parsed.params.temperature).to eq(0.7)
+      end
+    end
+  end
+
+  describe '#format_response' do
+    context 'with a simple text response' do
+      let(:canonical_resp) do
+        canonical::Response.from_hash(conformance.fixture_symbolized('canonical_simple_text_response'))
+      end
+
+      it 'formats a valid client response' do
+        formatted = translator.format_response(canonical_resp)
+        expect(formatted).to be_a(Hash)
+        expect(formatted).not_to be_empty
+      end
+
+      it 'includes the text content' do
+        formatted = translator.format_response(canonical_resp)
+        formatted_str = formatted.to_s
+        expect(formatted_str).to include('doing well')
+      end
+    end
+
+    context 'with a tool use response' do
+      let(:canonical_resp) do
+        canonical::Response.from_hash(conformance.fixture_symbolized('canonical_tool_use_response'))
+      end
+
+      it 'formats tool calls in client-appropriate format' do
+        formatted = translator.format_response(canonical_resp)
+        formatted_str = formatted.to_s.downcase
+        expect(formatted_str).to include('get_weather')
+      end
+
+      it 'includes tool call arguments' do
+        formatted = translator.format_response(canonical_resp)
+        formatted_str = formatted.to_s
+        expect(formatted_str).to include('San Francisco')
+      end
+    end
+
+    context 'with a thinking response' do
+      let(:canonical_resp) do
+        canonical::Response.from_hash(conformance.fixture_symbolized('canonical_thinking_response'))
+      end
+
+      it 'includes thinking content in client format' do
+        formatted = translator.format_response(canonical_resp)
+        formatted_str = formatted.to_s.downcase
+        expect(formatted_str).to match(/think|reason|quantum/)
+      end
+    end
+
+    context 'with an error response' do
+      let(:canonical_resp) do
+        canonical::Response.from_hash(conformance.fixture_symbolized('canonical_error_response'))
+      end
+
+      it 'formats error responses without crashing' do
+        formatted = translator.format_response(canonical_resp)
+        expect(formatted).to be_a(Hash)
+      end
+    end
+  end
+
+  describe '#format_chunk' do
+    context 'with text delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_text_chunks') }
+      let(:chunks_data) { stream_fixture['chunks'] }
+
+      it 'formats text delta chunks' do
+        text_chunk_hash = chunks_data.find { |c| c['type'] == 'text_delta' }
+        chunk = canonical::Chunk.from_hash(text_chunk_hash)
+        formatted = translator.format_chunk(chunk)
+
+        return unless formatted
+
+        expect(formatted).to be_a(Hash)
+        formatted_str = formatted.to_s
+        expect(formatted_str).to include(chunk.delta)
+      end
+
+      it 'formats the done chunk' do
+        done_chunk_hash = chunks_data.find { |c| c['type'] == 'done' }
+        chunk = canonical::Chunk.from_hash(done_chunk_hash)
+        formatted = translator.format_chunk(chunk)
+
+        return unless formatted
+
+        expect(formatted).to be_a(Hash)
+      end
+    end
+
+    context 'with thinking delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_thinking_chunks') }
+      let(:chunks_data) { stream_fixture['chunks'] }
+
+      it 'formats thinking delta chunks' do
+        thinking_chunk_hash = chunks_data.find { |c| c['type'] == 'thinking_delta' }
+        chunk = canonical::Chunk.from_hash(thinking_chunk_hash)
+        formatted = translator.format_chunk(chunk)
+
+        return unless formatted
+
+        expect(formatted).to be_a(Hash)
+      end
+    end
+
+    context 'with tool call delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_tool_call_chunks') }
+      let(:chunks_data) { stream_fixture['chunks'] }
+
+      it 'formats tool call delta chunks' do
+        tool_chunk_hash = chunks_data.find { |c| c['type'] == 'tool_call_delta' }
+        chunk = canonical::Chunk.from_hash(tool_chunk_hash)
+        formatted = translator.format_chunk(chunk)
+
+        return unless formatted
+
+        expect(formatted).to be_a(Hash)
+        formatted_str = formatted.to_s.downcase
+        expect(formatted_str).to include('get_weather')
+      end
+    end
+  end
+
+  describe '#format_error' do
+    it 'formats an error with status code' do
+      error = StandardError.new('Test error')
+      result = translator.format_error(error, 500)
+      expect(result).to be_an(Array)
+      expect(result.length).to eq(2)
+      expect(result[0]).to eq(500)
+      expect(result[1]).to be_a(Hash)
+    end
+  end
+
+  describe 'round-trip consistency' do
+    context 'with request round-trip' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_simple_text_request'))
+      end
+
+      it 'preserves message content through format/parse cycle' do
+        return unless translator.respond_to?(:format_request)
+
+        formatted = translator.format_request(canonical_req)
+        parsed = translator.parse_request(formatted, {})
+        expect(parsed.messages.length).to eq(canonical_req.messages.length)
+      end
+    end
+
+    context 'with response round-trip' do
+      let(:canonical_resp) do
+        canonical::Response.from_hash(conformance.fixture_symbolized('canonical_simple_text_response'))
+      end
+
+      it 'preserves text through format cycle' do
+        formatted = translator.format_response(canonical_resp)
+        formatted_str = formatted.to_s
+        expect(formatted_str).to include(canonical_resp.text)
+      end
+    end
+  end
+end
+# rubocop:enable Lint/NonLocalExitFromIterator

data/spec/legion/extensions/llm/conformance/conformance.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+# Conformance kit: shared RSpec example groups for N×N canonical routing.
+#
+# Ship location: spec/legion/extensions/llm/conformance/
+# Module: Canonical::Conformance
+#
+# Consumer pattern (in provider gem spec_helper):
+#   kit = File.join(Gem.loaded_specs['lex-llm'].full_gem_path,
+#                   'spec/legion/extensions/llm/conformance')
+#   Dir[File.join(kit, '**', '*.rb')].sort.each { |f| require f }
+#
+# Then in specs:
+#   it_behaves_like 'a canonical provider translator', described_class
+#   it_behaves_like 'a canonical client translator', described_class
+
+module Canonical
+  module Conformance
+    class << self
+      def fixtures_path
+        @fixtures_path ||= File.expand_path('fixtures', __dir__)
+      end
+
+      def fixture(name)
+        path = File.join(fixtures_path, "#{name}.json")
+        raise ArgumentError, "Fixture not found: #{name}" unless File.exist?(path)
+
+        # Explicit encoding: fixtures contain UTF-8; a bare File.read obeys the
+        # ambient locale and breaks in shells without LANG set (CI, tool runners).
+        ::JSON.parse(File.read(path, encoding: 'UTF-8'))
+      end
+
+      def fixture_symbolized(name)
+        deep_symbolize(fixture(name))
+      end
+
+      private
+
+      def deep_symbolize(obj)
+        case obj
+        when Hash then obj.transform_keys(&:to_sym).transform_values { |v| deep_symbolize(v) }
+        when Array then obj.map { |v| deep_symbolize(v) }
+        else obj
+        end
+      end
+    end
+  end
+end
+
+require_relative 'provider_translator_examples'
+require_relative 'client_translator_examples'

data/spec/legion/extensions/llm/conformance/echo_translator.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+# Trivial echo translator for conformance kit self-testing.
+# Passes canonical types through unchanged, proving the shared example groups work.
+
+module Canonical
+  module Conformance
+    # Echo translator: identity transform for both provider and client sides.
+    # Used exclusively as a self-test to verify the conformance kit works.
+    class EchoTranslator
+      def capabilities
+        { provider: 'echo', thinking: true, streaming: true, tool_calls: true }
+      end
+
+      # Provider translator interface
+      def render_request(canonical_request)
+        canonical_request.to_h
+      end
+
+      def parse_response(wire_hash)
+        canonical::Response.from_hash(wire_hash)
+      end
+
+      def parse_chunk(raw_chunk)
+        canonical::Chunk.from_hash(raw_chunk)
+      end
+
+      # Client translator interface
+      def format_request(canonical_request)
+        canonical_request.to_h
+      end
+
+      def parse_request(body, _env = {})
+        canonical::Request.from_hash(body)
+      end
+
+      def format_response(canonical_response)
+        canonical_response.to_h
+      end
+
+      def format_chunk(canonical_chunk)
+        canonical_chunk.to_h
+      end
+
+      def format_error(error, status)
+        [status, { error: error.message, type: error.class.name }]
+      end
+
+      private
+
+      def canonical
+        Legion::Extensions::Llm::Canonical
+      end
+    end
+  end
+end

data/spec/legion/extensions/llm/conformance/echo_translator_spec.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_relative 'conformance'
+require_relative 'echo_translator'
+
+RSpec.describe Canonical::Conformance::EchoTranslator do
+  # Self-test: the echo translator passes both conformance groups,
+  # proving the shared example groups work correctly.
+
+  it_behaves_like 'a canonical provider translator', described_class
+  it_behaves_like 'a canonical client translator', described_class
+end

data/spec/legion/extensions/llm/conformance/fixtures/canonical_empty_response.json

@@ -0,0 +1,13 @@
+{
+  "text": "",
+  "thinking": null,
+  "tool_calls": [],
+  "usage": {
+    "input_tokens": 8,
+    "output_tokens": 0
+  },
+  "stop_reason": "end_turn",
+  "model": "test-model-1",
+  "routing": {},
+  "metadata": {}
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_error_response.json

@@ -0,0 +1,19 @@
+{
+  "text": "",
+  "thinking": null,
+  "tool_calls": [],
+  "usage": {
+    "input_tokens": 0,
+    "output_tokens": 0
+  },
+  "stop_reason": "error",
+  "model": "test-model-1",
+  "routing": {},
+  "metadata": {
+    "error": {
+      "type": "invalid_request_error",
+      "message": "Model nonexistent-model-xyz-12345 not found",
+      "code": 404
+    }
+  }
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_fleet_round_trip.json

@@ -0,0 +1,81 @@
+{
+  "description": "Fleet round-trip field mapping — R6",
+  "round_trip_request": {
+    "id": "req_fleet_rt_001",
+    "messages": [
+      {
+        "role": "user",
+        "content": [{ "type": "text", "text": "Fleet round-trip test message" }]
+      }
+    ],
+    "system": "You are a test assistant.",
+    "tools": {
+      "test_tool": {
+        "name": "test_tool",
+        "description": "A test tool for round-trip validation",
+        "parameters": {
+          "type": "object",
+          "properties": { "input": { "type": "string" } },
+          "required": ["input"]
+        }
+      }
+    },
+    "params": { "max_tokens": 1024, "temperature": 0.5 },
+    "thinking": { "effort": "low", "budget": 1024 },
+    "stream": false,
+    "conversation_id": "conv_fleet_rt_001",
+    "caller": { "type": "anthropic_messages", "client_id": "test_client" },
+    "routing": { "tier": "primary", "provider": "anthropic", "model": "claude-sonnet-4-6" },
+    "metadata": { "trace_id": "trace_abc123", "span_id": "span_def456" }
+  },
+  "round_trip_response": {
+    "text": "This is a fleet round-trip test response.",
+    "thinking": { "content": "Processing fleet round-trip test.", "signature": "sig_test_001" },
+    "tool_calls": [
+      {
+        "id": "call_fleet_rt_001",
+        "exchange_id": "exch_fleet_rt_001",
+        "name": "test_tool",
+        "arguments": { "input": "round_trip_test" },
+        "source": "client",
+        "status": "pending"
+      }
+    ],
+    "usage": {
+      "input_tokens": 50,
+      "output_tokens": 30,
+      "cache_read_tokens": 10,
+      "cache_write_tokens": 5,
+      "thinking_tokens": 20
+    },
+    "stop_reason": "tool_use",
+    "model": "claude-sonnet-4-6",
+    "routing": { "tier": "primary", "provider": "anthropic", "instance": "us-east-1" },
+    "metadata": { "trace_id": "trace_abc123", "span_id": "span_def456", "latency_ms": 1250 }
+  },
+  "field_mapping": {
+    "request": {
+      "id": "string — unique request identifier",
+      "messages": "array[Message] — conversation history",
+      "system": "string | nil — system prompt",
+      "tools": "hash[name -> ToolDefinition] — available tools",
+      "params": "Params — sampling parameters",
+      "thinking": "Thinking::Config | nil — thinking configuration",
+      "stream": "boolean — streaming mode",
+      "conversation_id": "string | nil — conversation grouping",
+      "caller": "hash — client translator identity",
+      "routing": "hash — routing hints",
+      "metadata": "hash — passthrough metadata"
+    },
+    "response": {
+      "text": "string — assistant text content",
+      "thinking": "Thinking | nil — thinking block with content and signature",
+      "tool_calls": "array[ToolCall] — tool call requests",
+      "usage": "Usage — token usage breakdown",
+      "stop_reason": "symbol — why generation stopped",
+      "model": "string — resolved model identifier",
+      "routing": "hash — actual routing used",
+      "metadata": "hash — provider passthrough quirks"
+    }
+  }
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_metering_audit_events.json

@@ -0,0 +1,101 @@
+{
+  "description": "Metering/audit event schemas — G15e",
+  "schemas": {
+    "metering_event": {
+      "event_type": "metering",
+      "required_fields": {
+        "exchange_id": "string",
+        "request_id": "string",
+        "conversation_id": "string | nil",
+        "model": "string",
+        "provider": "string",
+        "usage": {
+          "input_tokens": "integer",
+          "output_tokens": "integer",
+          "cache_read_tokens": "integer | nil",
+          "cache_write_tokens": "integer | nil",
+          "thinking_tokens": "integer | nil"
+        },
+        "cost": {
+          "input_cost_usd": "float | nil",
+          "output_cost_usd": "float | nil",
+          "total_cost_usd": "float | nil"
+        },
+        "latency_ms": "integer",
+        "timestamp": "ISO8601 string"
+      }
+    },
+    "audit_event": {
+      "event_type": "audit",
+      "required_fields": {
+        "exchange_id": "string",
+        "request_id": "string",
+        "conversation_id": "string | nil",
+        "model": "string",
+        "provider": "string",
+        "caller": "hash",
+        "status": "symbol — :success, :error, :partial",
+        "stop_reason": "symbol | nil",
+        "timestamp": "ISO8601 string"
+      }
+    },
+    "tool_call_audit_event": {
+      "event_type": "tool_call_audit",
+      "required_fields": {
+        "exchange_id": "string",
+        "tool_call_id": "string",
+        "name": "string",
+        "source": "symbol — :client, :registry, :special, :extension, :mcp",
+        "status": "symbol — :pending, :running, :success, :error",
+        "arguments": "hash",
+        "timestamp": "ISO8601 string"
+      }
+    }
+  },
+  "example_events": {
+    "metering_success": {
+      "event_type": "metering",
+      "exchange_id": "exch_meter_001",
+      "request_id": "req_meter_001",
+      "conversation_id": "conv_001",
+      "model": "claude-sonnet-4-6",
+      "provider": "anthropic",
+      "usage": { "input_tokens": 150, "output_tokens": 85, "cache_read_tokens": 20 },
+      "cost": { "input_cost_usd": 0.0015, "output_cost_usd": 0.00765, "total_cost_usd": 0.00915 },
+      "latency_ms": 1250,
+      "stop_reason": "end_turn",
+      "routing": { "tier": "primary", "provider": "anthropic" },
+      "tool_calls_count": 0,
+      "timestamp": "2026-06-10T12:00:00Z"
+    },
+    "audit_success": {
+      "event_type": "audit",
+      "exchange_id": "exch_audit_001",
+      "request_id": "req_audit_001",
+      "conversation_id": "conv_001",
+      "model": "claude-sonnet-4-6",
+      "provider": "anthropic",
+      "caller": { "type": "anthropic_messages", "client_id": "test_client" },
+      "status": "success",
+      "stop_reason": "end_turn",
+      "routing": { "tier": "primary", "provider": "anthropic" },
+      "usage": { "input_tokens": 150, "output_tokens": 85 },
+      "route_attempts": [
+        { "attempt": 1, "provider": "anthropic", "model": "claude-sonnet-4-6", "status": "success" }
+      ],
+      "timestamp": "2026-06-10T12:00:00Z"
+    },
+    "tool_call_audit": {
+      "event_type": "tool_call_audit",
+      "exchange_id": "exch_tc_audit_001",
+      "tool_call_id": "call_tc_audit_001",
+      "name": "get_weather",
+      "source": "client",
+      "status": "success",
+      "arguments": { "location": "San Francisco, CA", "unit": "fahrenheit" },
+      "result": { "temperature": 68, "unit": "fahrenheit", "conditions": "partly cloudy" },
+      "duration_ms": 250,
+      "timestamp": "2026-06-10T12:00:01Z"
+    }
+  }
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_params_mapping_request.json

@@ -0,0 +1,21 @@
+{
+  "id": "req_params_mapping_001",
+  "messages": [
+    {
+      "role": "user",
+      "content": [{ "type": "text", "text": "Generate a creative story." }]
+    }
+  ],
+  "params": {
+    "max_tokens": 2048,
+    "temperature": 0.7,
+    "top_p": 0.9,
+    "top_k": 50,
+    "stop_sequences": ["[END]", "\\n\\n"],
+    "seed": 42,
+    "frequency_penalty": 0.1,
+    "presence_penalty": 0.2,
+    "response_format": { "type": "json_object" }
+  },
+  "stream": false
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_simple_text_request.json

@@ -0,0 +1,13 @@
+{
+  "id": "req_simple_text_001",
+  "messages": [
+    {
+      "role": "user",
+      "content": [{ "type": "text", "text": "Hello, how are you?" }]
+    }
+  ],
+  "params": {
+    "max_tokens": 1024
+  },
+  "stream": false
+}

data/spec/legion/extensions/llm/conformance/fixtures/canonical_simple_text_response.json

@@ -0,0 +1,13 @@
+{
+  "text": "I'm doing well, thank you for asking!",
+  "thinking": null,
+  "tool_calls": [],
+  "usage": {
+    "input_tokens": 12,
+    "output_tokens": 10
+  },
+  "stop_reason": "end_turn",
+  "model": "test-model-1",
+  "routing": {},
+  "metadata": {}
+}