lex-llm 0.4.18 → 0.5.0

data/spec/legion/extensions/llm/conformance/provider_translator_examples.rb

@@ -0,0 +1,390 @@
+# frozen_string_literal: true
+
+# Shared examples for canonical provider translator conformance.
+#
+# Every provider translator must implement:
+#   - render_request(canonical_request) => wire Hash
+#   - parse_response(wire_hash) => Canonical::Response
+#   - parse_chunk(raw_chunk) => Canonical::Chunk | nil
+#   - capabilities => Hash
+#
+# Usage:
+#   it_behaves_like 'a canonical provider translator', MyTranslatorClass
+
+RSpec.shared_examples 'a canonical provider translator' do |translator_class|
+  let(:translator) { translator_class.new }
+  let(:canonical) { Legion::Extensions::Llm::Canonical }
+  let(:conformance) { Canonical::Conformance }
+
+  describe '#capabilities' do
+    it 'returns a Hash' do
+      expect(translator.capabilities).to be_a(Hash)
+    end
+
+    it 'includes a :provider key' do
+      expect(translator.capabilities).to have_key(:provider)
+      expect(translator.capabilities[:provider]).to be_a(String)
+    end
+  end
+
+  describe '#render_request' do
+    context 'with a simple text request' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_simple_text_request'))
+      end
+
+      it 'renders a non-empty wire payload' do
+        wire = translator.render_request(canonical_req)
+        expect(wire).to be_a(Hash)
+        expect(wire).not_to be_empty
+      end
+
+      it 'includes model or messages' do
+        wire = translator.render_request(canonical_req)
+        expect(wire.keys & %i[model messages]).not_to be_empty
+      end
+
+      it 'preserves message content' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s
+        expect(wire_str).to include('how are you')
+      end
+    end
+
+    context 'with a system prompt' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_system_prompt_request'))
+      end
+
+      it 'renders the system prompt in provider-appropriate format' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to match(/helpful|haiku/)
+      end
+    end
+
+    context 'with parameter mapping' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_params_mapping_request'))
+      end
+
+      it 'renders params in provider-appropriate format' do
+        wire = translator.render_request(canonical_req)
+        expect(wire).to be_a(Hash)
+        wire_str = wire.to_s
+        expect(wire_str).to match(/[0-9]+/)
+      end
+    end
+
+    context 'with tools defined' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_tools_request'))
+      end
+
+      it 'renders tools in provider format' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to include('get_weather')
+      end
+
+      it 'includes tool parameters' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to include('location')
+      end
+    end
+
+    context 'with tool results continuation (multi-turn)' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(
+          conformance.fixture_symbolized('canonical_tool_results_continuation_request')
+        )
+      end
+
+      it 'renders the full conversation history' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to include('weather')
+      end
+
+      it 'renders mixed client and registry tool calls' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to include('get_weather')
+        expect(wire_str).to include('summarize')
+      end
+    end
+
+    context 'with thinking enabled' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(conformance.fixture_symbolized('canonical_thinking_request'))
+      end
+
+      it 'renders thinking configuration' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to match(/think|reason|budget|effort/)
+      end
+    end
+
+    context 'with streaming request' do
+      let(:canonical_req) do
+        canonical::Request.from_hash(
+          conformance.fixture_symbolized('canonical_simple_text_request').merge({ 'stream' => true })
+        )
+      end
+
+      it 'renders with streaming indicator' do
+        wire = translator.render_request(canonical_req)
+        wire_str = wire.to_s.downcase
+        expect(wire_str).to include('stream')
+      end
+    end
+  end
+
+  # parse_response: tests translator.parse_response(wire_hash) => Canonical::Response
+  # For self-test (echo translator), wire == canonical-form (symbolized).
+  # Real provider translators convert provider-specific wire format to canonical.
+  describe '#parse_response' do
+    context 'with a simple text response' do
+      let(:wire_response) { conformance.fixture_symbolized('canonical_simple_text_response') }
+
+      it 'returns a Canonical::Response' do
+        response = translator.parse_response(wire_response)
+        expect(response).to be_a(canonical::Response)
+      end
+
+      it 'preserves text content' do
+        response = translator.parse_response(wire_response)
+        expect(response.text).to eq("I'm doing well, thank you for asking!")
+      end
+
+      it 'sets stop_reason' do
+        response = translator.parse_response(wire_response)
+        expect(response.stop_reason).to eq(:end_turn)
+      end
+
+      it 'includes usage data' do
+        response = translator.parse_response(wire_response)
+        expect(response.usage).to be_a(canonical::Usage)
+        expect(response.usage.input_tokens).to eq(12)
+        expect(response.usage.output_tokens).to eq(10)
+      end
+    end
+
+    context 'with a tool use response' do
+      let(:wire_response) { conformance.fixture_symbolized('canonical_tool_use_response') }
+
+      it 'parses tool calls correctly' do
+        response = translator.parse_response(wire_response)
+        expect(response).to be_a(canonical::Response)
+        expect(response.tool_call?).to be true
+        expect(response.tool_calls).to be_an(Array)
+        expect(response.tool_calls.first).to be_a(canonical::ToolCall)
+        expect(response.stop_reason).to eq(:tool_use)
+      end
+
+      it 'preserves tool call arguments as a Hash' do
+        response = translator.parse_response(wire_response)
+        args = response.tool_calls.first.arguments
+        expect(args).to be_a(Hash)
+        expect(args[:location]).to eq('San Francisco, CA')
+      end
+
+      it 'has no text when response is tool-only' do
+        response = translator.parse_response(wire_response)
+        expect(response.text).to eq('')
+      end
+    end
+
+    context 'with thinking response' do
+      let(:wire_response) { conformance.fixture_symbolized('canonical_thinking_response') }
+
+      it 'parses thinking content and signature' do
+        response = translator.parse_response(wire_response)
+        expect(response.thinking).to be_a(canonical::Thinking)
+        expect(response.thinking.content).to include('quantum')
+        expect(response.thinking.signature).to be_a(String)
+      end
+
+      it 'preserves thinking tokens in usage' do
+        response = translator.parse_response(wire_response)
+        expect(response.usage).to be_a(canonical::Usage)
+        expect(response.usage.thinking_tokens).to eq(120)
+      end
+    end
+
+    context 'with error response' do
+      let(:wire_response) { conformance.fixture_symbolized('canonical_error_response') }
+
+      it 'parses error responses without crashing' do
+        response = translator.parse_response(wire_response)
+        expect(response).to be_a(canonical::Response)
+        expect(response.error?).to be true
+        expect(response.stop_reason).to eq(:error)
+      end
+
+      it 'preserves error metadata' do
+        response = translator.parse_response(wire_response)
+        expect(response.metadata).to have_key(:error)
+      end
+    end
+
+    context 'with empty response' do
+      let(:wire_response) { conformance.fixture_symbolized('canonical_empty_response') }
+
+      it 'handles empty responses gracefully' do
+        response = translator.parse_response(wire_response)
+        expect(response).to be_a(canonical::Response)
+        expect(response.text).to eq('')
+        expect(response.tool_calls).to eq([])
+      end
+    end
+  end
+
+  describe '#parse_chunk' do
+    context 'with text delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_text_chunks') }
+      let(:chunks) { stream_fixture['chunks'] }
+
+      it 'parses text delta chunks' do
+        text_chunk = chunks.find { |c| c['type'] == 'text_delta' }
+        parsed = translator.parse_chunk(text_chunk)
+        expect(parsed).to be_a(canonical::Chunk)
+        expect(parsed.type).to eq(:text_delta)
+        expect(parsed.delta).to be_a(String)
+      end
+
+      it 'parses the done chunk' do
+        done_chunk = chunks.find { |c| c['type'] == 'done' }
+        parsed = translator.parse_chunk(done_chunk)
+        expect(parsed).to be_a(canonical::Chunk)
+        expect(parsed.type).to eq(:done)
+        expect(parsed.stop_reason).to eq(:end_turn)
+      end
+    end
+
+    context 'with thinking delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_thinking_chunks') }
+      let(:chunks) { stream_fixture['chunks'] }
+
+      it 'parses thinking delta chunks' do
+        thinking_chunk = chunks.find { |c| c['type'] == 'thinking_delta' }
+        parsed = translator.parse_chunk(thinking_chunk)
+        expect(parsed).to be_a(canonical::Chunk)
+        expect(parsed.type).to eq(:thinking_delta)
+      end
+
+      it 'preserves signature on thinking deltas' do
+        sig_chunk = chunks.find { |c| c['type'] == 'thinking_delta' && !c['signature'].nil? }
+        next if sig_chunk.nil?
+
+        parsed = translator.parse_chunk(sig_chunk)
+        expect(parsed.signature).to be_a(String)
+      end
+    end
+
+    context 'with tool call delta chunks' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_tool_call_chunks') }
+      let(:chunks) { stream_fixture['chunks'] }
+
+      it 'parses tool call delta chunks' do
+        tool_chunk = chunks.find { |c| c['type'] == 'tool_call_delta' }
+        parsed = translator.parse_chunk(tool_chunk)
+        expect(parsed).to be_a(canonical::Chunk)
+        expect(parsed.type).to eq(:tool_call_delta)
+      end
+
+      it 'preserves tool call identity across chunks' do
+        tool_chunks = chunks.select { |c| c['type'] == 'tool_call_delta' }
+        parsed_chunks = tool_chunks.map { |c| translator.parse_chunk(c) }
+        ids = parsed_chunks.map { |c| c.tool_call&.id }
+        expect(ids.uniq.length).to eq(1)
+      end
+    end
+
+    context 'with error chunk' do
+      let(:stream_fixture) { conformance.fixture('canonical_streaming_error_chunks') }
+      let(:chunks) { stream_fixture['chunks'] }
+
+      it 'parses error chunks' do
+        error_chunk = chunks.find { |c| c['type'] == 'error' }
+        parsed = translator.parse_chunk(error_chunk)
+        expect(parsed).to be_a(canonical::Chunk)
+        expect(parsed.type).to eq(:error)
+        expect(parsed.error?).to be true
+      end
+    end
+  end
+
+  describe 'stop_reason mapping' do
+    let(:matrix) { conformance.fixture_symbolized('canonical_stop_reason_matrix') }
+
+    it 'maps all canonical stop reasons' do
+      canonical::Response::STOP_REASONS.each do |reason|
+        resp = canonical::Response.build(stop_reason: reason, text: 'test')
+        expect(resp.stop_reason).to eq(reason)
+      end
+    end
+
+    it 'rejects invalid stop reasons' do
+      expect { canonical::Response.build(stop_reason: :invalid_reason, text: 'test') }
+        .to raise_error(ArgumentError, /Invalid stop_reason/)
+    end
+  end
+
+  describe 'round-trip consistency' do
+    it 'accumulated chunks equal non-streaming response for text' do
+      stream_fixture = conformance.fixture('canonical_streaming_text_chunks')
+      chunks = stream_fixture['chunks']
+
+      accumulated_text = ''
+      final_stop_reason = nil
+
+      chunks.each do |raw_chunk|
+        chunk = translator.parse_chunk(raw_chunk)
+        next unless chunk
+
+        case chunk.type
+        when :text_delta
+          accumulated_text += chunk.delta
+        when :done
+          final_stop_reason = chunk.stop_reason
+        end
+      end
+
+      expect(accumulated_text).to eq('Hello, world! How can I help you today?')
+      expect(final_stop_reason).to eq(:end_turn)
+    end
+
+    it 'accumulated chunks equal non-streaming response for thinking + text' do
+      stream_fixture = conformance.fixture('canonical_streaming_thinking_chunks')
+      chunks = stream_fixture['chunks']
+
+      accumulated_thinking = ''
+      accumulated_text = ''
+      final_stop_reason = nil
+      final_signature = nil
+
+      chunks.each do |raw_chunk|
+        chunk = translator.parse_chunk(raw_chunk)
+        next unless chunk
+
+        case chunk.type
+        when :thinking_delta
+          accumulated_thinking += chunk.delta
+          final_signature = chunk.signature if chunk.signature
+        when :text_delta
+          accumulated_text += chunk.delta
+        when :done
+          final_stop_reason = chunk.stop_reason
+        end
+      end
+
+      expect(accumulated_thinking).not_to be_empty
+      expect(accumulated_text).not_to be_empty
+      expect(final_signature).to be_a(String)
+      expect(final_stop_reason).to eq(:end_turn)
+    end
+  end
+end

data/spec/legion/extensions/llm/connection_logging_spec.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Legion::Extensions::Llm::Connection do
+  describe 'logging middleware configuration' do
+    let(:provider) do
+      instance_double(
+        Legion::Extensions::Llm::Provider,
+        api_base: 'https://example.com',
+        configured?: true,
+        headers: {}
+      )
+    end
+
+    let(:config) do
+      instance_double(
+        Legion::Extensions::Llm::Configuration,
+        request_timeout: 300,
+        max_retries: 3,
+        retry_interval: 0.1,
+        retry_interval_randomness: 0.5,
+        retry_backoff_factor: 2,
+        http_proxy: nil,
+        log_regexp_timeout: 1.0
+      )
+    end
+
+    it 'disables body logging when log level is above DEBUG' do
+      logger = Logger.new(File::NULL, level: Logger::INFO)
+      allow(config).to receive(:logger).and_return(logger)
+
+      connection = described_class.new(provider, config).connection
+      handler = connection.builder.handlers.find { |h| h.klass == Faraday::Response::Logger }
+      middleware = handler.build(->(_env) { Faraday::Response.new })
+      options = middleware.instance_variable_get(:@formatter).instance_variable_get(:@options)
+
+      expect(options[:bodies]).to be(false)
+    end
+
+    it 'enables body logging when log level is DEBUG' do
+      logger = Logger.new(File::NULL, level: Logger::DEBUG)
+      allow(config).to receive(:logger).and_return(logger)
+
+      connection = described_class.new(provider, config).connection
+      handler = connection.builder.handlers.find { |h| h.klass == Faraday::Response::Logger }
+      middleware = handler.build(->(_env) { Faraday::Response.new })
+      options = middleware.instance_variable_get(:@formatter).instance_variable_get(:@options)
+
+      expect(options[:bodies]).to be(true)
+    end
+  end
+end

data/spec/legion/extensions/llm/connection_retry_spec.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Legion::Extensions::Llm::Connection do
+  describe 'retry middleware configuration' do
+    let(:provider) do
+      instance_double(
+        Legion::Extensions::Llm::Provider,
+        api_base: 'https://example.com',
+        configured?: true,
+        headers: {}
+      )
+    end
+
+    let(:config) do
+      instance_double(
+        Legion::Extensions::Llm::Configuration,
+        request_timeout: 300,
+        max_retries: 3,
+        retry_interval: 0.1,
+        retry_interval_randomness: 0.5,
+        retry_backoff_factor: 2,
+        http_proxy: nil
+      )
+    end
+
+    it 'retries POST requests for transient failures' do
+      connection = described_class.new(provider, config).connection
+      retry_handler = connection.builder.handlers.find { |handler| handler.klass == Faraday::Retry::Middleware }
+      retry_options = retry_handler.instance_variable_get(:@args).first
+
+      expect(retry_options[:methods]).to include(:post)
+    end
+  end
+end

data/spec/legion/extensions/llm/context_spec.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Legion::Extensions::Llm::Context do
+  include_context 'with configured Legion::Extensions::Llm'
+  include_context 'with fake llm provider'
+
+  describe '#initialize' do
+    it 'creates a copy of the global configuration' do
+      # Get current config values
+      original_model = Legion::Extensions::Llm.config.default_model
+      original_log_regexp_timeout = Legion::Extensions::Llm.config.log_regexp_timeout
+
+      # Create context with modified config
+      context = Legion::Extensions::Llm.context do |config|
+        config.default_model = 'modified-model'
+        config.fake_llm_api_key = 'modified-key'
+        config.log_regexp_timeout = 5.0
+      end
+
+      # Verify global config is unchanged
+      expect(Legion::Extensions::Llm.config.default_model).to eq(original_model)
+      expect(Legion::Extensions::Llm.config.log_regexp_timeout).to eq(original_log_regexp_timeout)
+
+      # Verify context has modified config
+      expect(context.config.default_model).to eq('modified-model')
+      expect(context.config.fake_llm_api_key).to eq('modified-key')
+      expect(context.config.log_regexp_timeout).to eq(5.0)
+    end
+
+    it 'preserves log_regexp_timeout when Regexp timeout is unavailable' do
+      allow(Regexp).to receive(:respond_to?).and_call_original
+      allow(Regexp).to receive(:respond_to?).with(:timeout).and_return(false)
+      allow(Legion::Extensions::Llm.logger).to receive(:warn)
+
+      context = Legion::Extensions::Llm.context do |config|
+        config.log_regexp_timeout = 5.0
+      end
+
+      expect(context.config.log_regexp_timeout).to eq(5.0)
+    end
+  end
+
+  describe 'context chat operations' do
+    it 'creates a chat with context-specific configuration' do
+      context = Legion::Extensions::Llm.context do |config|
+        config.default_model = 'fake-chat-model'
+      end
+
+      chat = context.chat(provider: :fake_llm, assume_model_exists: true)
+      expect(chat.model.id).to eq('fake-chat-model')
+    end
+
+    it 'allows specifying a model when creating the chat' do
+      context = Legion::Extensions::Llm.context do |config|
+        config.default_model = 'fake-chat-model'
+      end
+
+      chat = context.chat(model: 'other-fake-chat-model', provider: :fake_llm, assume_model_exists: true)
+      expect(chat.model.id).to eq('other-fake-chat-model')
+    end
+  end
+
+  describe 'context embed operations' do
+    it 'respects context-specific embedding model' do
+      context = Legion::Extensions::Llm.context do |config|
+        config.default_embedding_model = 'fake-embed'
+      end
+
+      embedding = context.embed('Test embedding', provider: :fake_llm, assume_model_exists: true)
+      expect(embedding.model).to eq('fake-embed')
+    end
+
+    it 'allows specifying a model at embed time' do
+      context = Legion::Extensions::Llm.context do |config|
+        config.default_embedding_model = 'fake-embed'
+      end
+
+      embedding = context.embed('Test embedding', model: 'override-embed', provider: :fake_llm,
+                                                  assume_model_exists: true)
+      expect(embedding.model).to eq('override-embed')
+    end
+  end
+
+  describe 'multiple independent contexts' do
+    it 'allows multiple contexts with different configurations' do
+      context1 = Legion::Extensions::Llm.context do |config|
+        config.default_model = 'fake-chat-1'
+        config.log_regexp_timeout = 5.0
+      end
+
+      context2 = Legion::Extensions::Llm.context do |config|
+        config.default_model = 'fake-chat-2'
+      end
+
+      chat1 = context1.chat(provider: :fake_llm, assume_model_exists: true)
+      chat2 = context2.chat(provider: :fake_llm, assume_model_exists: true)
+
+      expect(chat1.model.id).to eq('fake-chat-1')
+      expect(context1.config.log_regexp_timeout).to eq(5.0)
+
+      expect(chat2.model.id).to eq('fake-chat-2')
+      expected_timeout = Regexp.respond_to?(:timeout) ? (Regexp.timeout || 1.0) : nil
+      expect(context2.config.log_regexp_timeout).to eq(expected_timeout)
+    end
+
+    it 'ensures changes in one context do not affect another' do
+      context1 = Legion::Extensions::Llm.context do |config|
+        config.fake_llm_api_key = 'key1'
+        config.default_model = 'model1'
+      end
+
+      context2 = Legion::Extensions::Llm.context do |config|
+        config.fake_llm_api_key = 'key2'
+        config.default_model = 'model2'
+      end
+
+      # Modify context1 after creation
+      context1.config.fake_llm_api_key = 'modified-key1'
+
+      # Context2 should be unaffected
+      expect(context2.config.fake_llm_api_key).to eq('key2')
+      expect(context2.config.default_model).to eq('model2')
+    end
+  end
+end