legion-llm 0.3.11 → 0.3.12

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 61c5173e4490643fbecb7977697159ffdeb082b63ec4140289128c69efd14806
-  data.tar.gz: c42e9f24e2ecc387c1076fe32f05b4636e17e9fe92b16bf7ed438198caaa3187
+  metadata.gz: 0551af82013a885240cd8d38ba1f991d470110d308925a8f4848b8650376d252
+  data.tar.gz: fc2da425ddafa426f89375dbffd9afccc2c5d318207ed2bff0513dc57cf7dc07
 SHA512:
-  metadata.gz: b05c1c69d88184ef4ceea383523c0b6538fd363a1cee9d12bf849ab2885bfdfe7fd59170fe059458c89378f66b93851ec69803ae03af6df1a3bbb55ab1aa432c
-  data.tar.gz: 57eab8217bdbc614a8602b45836f4b38bf099d9fc2e6b85cfbd92813856b837387e0901ac06af43085cfe0cd034146d160b05cd358afeb45c03ac628280ce9d6
+  metadata.gz: 3ebfd45a16cd899050c44c0e53b0ae9952c8c87f46381b0af4356cda6d03ebff05084c3d66923ffd9f3012674b41005e080fa0350cdb4a2f799cbaa83e1cd4dc
+  data.tar.gz: bfb977400e5c78caa90012af604ec47c7b4be94e1d268cfceb3b240b1e9731f678b4a0f6dc30de4df6e014e4714701b15f554e9a5858b3a94754aedaaa67da84

data/CHANGELOG.md

@@ -1,5 +1,16 @@
 # Legion LLM Changelog
 
+## [0.3.12] - 2026-03-19
+
+### Added
+- `Legion::LLM::Cache` module with deterministic SHA256 key generation, guarded `get`/`set`, and `enabled?` check
+- Application-level response caching in `chat_direct` via `legion-cache` (Legion::Cache guard required)
+- Cache skip conditions: `cache: false` option, `temperature > 0`, nil message, or cache disabled
+- Cache hits return `{ cached: true }` merged into response metadata
+- Anthropic prompt caching support: injects `cache_control: { type: "ephemeral" }` into system messages longer than `min_tokens` when provider is anthropic
+- `prompt_caching` settings section with `enabled`, `min_tokens`, `response_cache.enabled`, `response_cache.ttl_seconds` defaults
+- 25 new specs in `spec/legion/llm/cache_spec.rb` covering key determinism, hit/miss flows, skip conditions, and Legion::Cache unavailability guard
+
 ## [0.3.11] - 2026-03-20
 
 ### Added

data/lib/legion/llm/cache.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Legion
+  module LLM
+    module Cache
+      DEFAULT_TTL = 300
+
+      module_function
+
+      # Generates a deterministic SHA256 cache key from request parameters.
+      def key(model:, provider:, messages:, temperature: nil, tools: nil, schema: nil)
+        payload = ::JSON.dump({
+                                model:       model.to_s,
+                                provider:    provider.to_s,
+                                messages:    messages,
+                                temperature: temperature,
+                                tools:       tools,
+                                schema:      schema
+                              })
+        Digest::SHA256.hexdigest(payload)
+      end
+
+      # Returns the cached response hash, or nil on miss / cache unavailable.
+      def get(cache_key)
+        return nil unless available?
+
+        raw = Legion::Cache.get(cache_key)
+        return nil if raw.nil?
+
+        ::JSON.parse(raw, symbolize_names: true)
+      rescue StandardError
+        nil
+      end
+
+      # Stores a response in the cache with the given TTL.
+      def set(cache_key, response, ttl: DEFAULT_TTL)
+        return false unless available?
+
+        Legion::Cache.set(cache_key, ::JSON.dump(response), ttl)
+        true
+      rescue StandardError
+        false
+      end
+
+      # Returns true if response caching is enabled in settings and Legion::Cache is loaded.
+      def enabled?
+        return false unless available?
+
+        settings = llm_settings
+        settings.dig(:prompt_caching, :response_cache, :enabled) != false
+      end
+
+      private_class_method def self.available?
+        defined?(Legion::Cache) && Legion::Cache.respond_to?(:get)
+      end
+
+      private_class_method def self.llm_settings
+        if Legion.const_defined?('Settings')
+          Legion::Settings[:llm]
+        else
+          Legion::LLM::Settings.default
+        end
+      rescue StandardError
+        {}
+      end
+    end
+  end
+end

data/lib/legion/llm/settings.rb

@@ -14,7 +14,8 @@ module Legion
           routing:          routing_defaults,
           discovery:        discovery_defaults,
           gateway:          gateway_defaults,
-          daemon:           daemon_defaults
+          daemon:           daemon_defaults,
+          prompt_caching:   prompt_caching_defaults
         }
       end
 
@@ -25,6 +26,17 @@ module Legion
         }
       end
 
+      def self.prompt_caching_defaults
+        {
+          enabled:        true,
+          min_tokens:     1024,
+          response_cache: {
+            enabled:     true,
+            ttl_seconds: 300
+          }
+        }
+      end
+
       def self.discovery_defaults
         {
           enabled:         true,

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.3.11'
+    VERSION = '0.3.12'
   end
 end

data/lib/legion/llm.rb

@@ -9,6 +9,7 @@ require 'legion/llm/compressor'
 require 'legion/llm/quality_checker'
 require 'legion/llm/escalation_history'
 require 'legion/llm/hooks'
+require 'legion/llm/cache'
 require_relative 'llm/response_cache'
 require_relative 'llm/daemon_client'
 
@@ -95,18 +96,40 @@ module Legion
 
       # Direct chat bypassing gateway — used by gateway runners to avoid recursion
       def chat_direct(model: nil, provider: nil, intent: nil, tier: nil, escalate: nil,
-                      max_escalations: nil, quality_check: nil, message: nil, **)
+                      max_escalations: nil, quality_check: nil, message: nil, **kwargs)
+        cache_opt   = kwargs.delete(:cache) { true }
+        temperature = kwargs.delete(:temperature)
+
         escalate = escalation_enabled? if escalate.nil?
+        cache_key = build_cache_key(model, provider, message, temperature) if cacheable?(cache_opt, temperature, message)
+
+        if cache_key
+          cached = Cache.get(cache_key)
+          if cached
+            Legion::Logging.debug 'Legion::LLM cache hit'
+            cached_response = cached.dup
+            cached_response[:meta] = (cached_response[:meta] || {}).merge(cached: true)
+            return cached_response
+          end
+        end
 
-        if escalate && message
-          chat_with_escalation(
-            model: model, provider: provider, intent: intent, tier: tier,
-            max_escalations: max_escalations, quality_check: quality_check,
-            message: message, **
-          )
-        else
-          chat_single(model: model, provider: provider, intent: intent, tier: tier, **)
+        result = if escalate && message
+                   chat_with_escalation(
+                     model: model, provider: provider, intent: intent, tier: tier,
+                     max_escalations: max_escalations, quality_check: quality_check,
+                     message: message, temperature: temperature, **kwargs
+                   )
+                 else
+                   chat_single(model: model, provider: provider, intent: intent, tier: tier,
+                               temperature: temperature, **kwargs)
+                 end
+
+        if cache_key && result.is_a?(Hash)
+          ttl = settings.dig(:prompt_caching, :response_cache, :ttl_seconds) || Cache::DEFAULT_TTL
+          Cache.set(cache_key, result, ttl: ttl)
         end
+
+        result
       end
 
       # Generate embeddings — delegates to gateway when available
@@ -268,6 +291,9 @@ module Legion
         opts[:model]    = model    if model
         opts[:provider] = provider if provider
         opts.merge!(kwargs)
+        opts.delete(:temperature) if opts[:temperature].nil?
+
+        inject_anthropic_cache_control!(opts, provider)
 
         RubyLLM.chat(**opts)
       end
@@ -344,6 +370,37 @@ module Legion
         nil
       end
 
+      def cacheable?(cache_opt, temperature, message)
+        cache_opt != false && temperature.to_f.zero? && message && Cache.enabled?
+      end
+
+      def build_cache_key(model, provider, message, temperature)
+        messages_arr = message.is_a?(Array) ? message : [{ role: 'user', content: message.to_s }]
+        Cache.key(
+          model:       model || settings[:default_model],
+          provider:    provider || settings[:default_provider],
+          messages:    messages_arr,
+          temperature: temperature
+        )
+      end
+
+      def inject_anthropic_cache_control!(opts, provider)
+        resolved_provider = (provider || settings[:default_provider])&.to_sym
+        return unless resolved_provider == :anthropic
+
+        caching_settings = settings[:prompt_caching] || {}
+        return unless caching_settings[:enabled] != false
+
+        min_tokens = caching_settings[:min_tokens] || 1024
+        instructions = opts[:instructions]
+        return unless instructions.is_a?(String) && instructions.length > min_tokens
+
+        opts[:instructions] = {
+          content:       instructions,
+          cache_control: { type: 'ephemeral' }
+        }
+      end
+
       def escalation_enabled?
         routing = settings[:routing]
         return false unless routing.is_a?(Hash)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.3.11
+  version: 0.3.12
 platform: ruby
 authors:
 - Esity
@@ -131,6 +131,7 @@ files:
 - legion-llm.gemspec
 - lib/legion/llm.rb
 - lib/legion/llm/bedrock_bearer_auth.rb
+- lib/legion/llm/cache.rb
 - lib/legion/llm/claude_config_loader.rb
 - lib/legion/llm/compressor.rb
 - lib/legion/llm/daemon_client.rb