legion-llm 0.3.6 → 0.3.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0914899eb9eee81b947d95d617a16ddf152fb74fa7afb4c1c1cfca74c9c8445d'
-  data.tar.gz: d4146a95967ceffca175c531fd412089f3a13df4b4b60964598e123115d3c19f
+  metadata.gz: b06b6f10d21c6c4d03c73646fbecc2112e61e47e1dd82059076c61a721efb1aa
+  data.tar.gz: 541a1a1de0a108e95b5e2c204ec579a1b0a5f77e935bad64d7668aef9ae3322d
 SHA512:
-  metadata.gz: 8d9fb16e659a4f24d6c01bb3b7caa96d6814980e5b9866fe8ccc293bae57121f8d21acc95efef98832b015875abebfe1ca2cbba63f825a43d64cb9feac82f9b2
-  data.tar.gz: 4e6788a7b28889ed80ec1701e5a45a05bcfe71914610b538fae2f68d3b16ac4942e8edd0abbf2414d3dd124edc109817ceef3390d22108c1c9899a82b6d93c55
+  metadata.gz: 06ab55cec8a23d4be70ea3851fd4a7717686c4e02f7b4ca2f479e2353f79b14cacc770343a81ac54f492a05de3b5228aebb7c2e464203e434eba53e8b4144694
+  data.tar.gz: 93623de5b0baa0bb5390678daac043fc6c08111f965886b03b87f5e769aa6e5f267a713c1c20d93d061341a6ceca8d88fcbd0dfa431d4ce84c2caf5768b19609

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Legion LLM Changelog
 
+## [0.3.7] - 2026-03-19
+
+### Added
+- `ResponseCache` module for async response delivery via memcached with spool overflow at 8MB
+- `DaemonClient` module for HTTP routing to LegionIO daemon with health caching (30s TTL)
+- `Legion::LLM.ask` one-shot method: daemon-first routing with direct RubyLLM fallback
+- `DaemonDeniedError` and `DaemonRateLimitedError` error classes
+- Daemon settings: `daemon.url` and `daemon.enabled` in defaults
+- HTTP status code contract: 200 (cached), 201 (sync), 202 (async poll), 403, 429, 503
+
 ## [0.3.6] - 2026-03-18
 
 ### Added

data/CLAUDE.md

@@ -8,7 +8,7 @@
 Core LegionIO gem providing LLM capabilities to all extensions. Wraps ruby_llm to provide a consistent interface for chat, embeddings, tool use, and agents across multiple providers (Bedrock, Anthropic, OpenAI, Gemini, Ollama). Includes a dynamic weighted routing engine that dispatches requests across local, fleet, and cloud tiers based on caller intent, priority rules, time schedules, cost multipliers, and real-time provider health.
 
 **GitHub**: https://github.com/LegionIO/legion-llm
-**Version**: 0.3.5
+**Version**: 0.3.6
 **License**: Apache-2.0
 
 ## Architecture
@@ -303,7 +303,7 @@ In-memory signal consumer with pluggable handlers. Adjusts effective priorities
 | `lib/legion/llm/embeddings.rb` | Embeddings module: generate, generate_batch, default_model |
 | `lib/legion/llm/shadow_eval.rb` | Shadow evaluation: enabled?, should_sample?, evaluate, compare |
 | `lib/legion/llm/structured_output.rb` | JSON schema enforcement with native response_format and prompt fallback |
-| `lib/legion/llm/version.rb` | Version constant (0.3.5) |
+| `lib/legion/llm/version.rb` | Version constant (0.3.6) |
 | `lib/legion/llm/quality_checker.rb` | QualityChecker module with QualityResult struct |
 | `lib/legion/llm/escalation_history.rb` | EscalationHistory mixin: `escalation_history`, `escalated?`, `final_resolution`, `escalation_chain` |
 | `lib/legion/llm/router/escalation_chain.rb` | EscalationChain value object |

data/README.md

@@ -2,7 +2,7 @@
 
 LLM integration for the [LegionIO](https://github.com/LegionIO/LegionIO) framework. Wraps [ruby_llm](https://github.com/crmne/ruby_llm) to provide chat, embeddings, tool use, and agent capabilities to any Legion extension.
 
-**Version**: 0.3.5
+**Version**: 0.3.6
 
 ## Installation
 

data/lib/legion/llm/daemon_client.rb

@@ -0,0 +1,179 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'uri'
+require 'json'
+require 'securerandom'
+
+module Legion
+  module LLM
+    module DaemonClient
+      HEALTH_CACHE_TTL = 30
+      DEFAULT_TIMEOUT  = 60
+
+      module_function
+
+      # Returns true if the daemon is reachable and healthy.
+      # Returns false immediately if daemon_url is nil.
+      # Caches a positive health check for HEALTH_CACHE_TTL seconds.
+      # An unhealthy result is not cached — rechecks on every call.
+      def available?
+        return false if daemon_url.nil?
+
+        now = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+
+        return true if @healthy == true && @health_checked_at && (now - @health_checked_at) < HEALTH_CACHE_TTL
+
+        result = check_health
+        if result
+          @healthy           = true
+          @health_checked_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        end
+        result
+      end
+
+      # POSTs a chat request to the daemon REST API.
+      # Returns a status hash based on the HTTP response code.
+      def chat(message:, request_id: nil, context: {}, tier_preference: :auto, model: nil, provider: nil)
+        request_id ||= SecureRandom.uuid
+
+        body = {
+          message:         message,
+          request_id:      request_id,
+          context:         context,
+          tier_preference: tier_preference
+        }
+        body[:model]    = model    if model
+        body[:provider] = provider if provider
+
+        response = http_post('/api/llm/chat', body)
+        interpret_response(response)
+      rescue StandardError => e
+        mark_unhealthy
+        { status: :unavailable, error: e.message }
+      end
+
+      # Returns the daemon URL from settings, cached after first read.
+      # Returns nil if settings are unavailable or the key is missing.
+      def daemon_url
+        return @daemon_url if defined?(@daemon_url)
+
+        @daemon_url = fetch_daemon_url
+      end
+
+      # Clears all cached state. Returns self for chaining.
+      def reset!
+        remove_instance_variable(:@daemon_url) if defined?(@daemon_url)
+        @healthy           = nil
+        @health_checked_at = nil
+        self
+      end
+
+      # GETs /api/health. Returns true on 200, false otherwise.
+      # Updates @healthy and @health_checked_at.
+      def check_health
+        response = http_get('/api/health')
+        healthy = response.code == '200'
+        @healthy           = healthy
+        @health_checked_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+        healthy
+      rescue StandardError
+        mark_unhealthy
+        false
+      end
+
+      # Marks the daemon as unhealthy and records the timestamp.
+      def mark_unhealthy
+        @healthy           = false
+        @health_checked_at = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+
+      # Builds and sends a GET request. Returns Net::HTTPResponse.
+      def http_get(path)
+        uri     = URI.parse("#{daemon_url}#{path}")
+        http    = Net::HTTP.new(uri.host, uri.port)
+        http.open_timeout = 2
+        http.read_timeout = 2
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request['Content-Type'] = 'application/json'
+        http.request(request)
+      end
+
+      # Builds and sends a POST request with a JSON body.
+      # Returns Net::HTTPResponse.
+      def http_post(path, body)
+        uri     = URI.parse("#{daemon_url}#{path}")
+        http    = Net::HTTP.new(uri.host, uri.port)
+        http.open_timeout = 5
+        http.read_timeout = DEFAULT_TIMEOUT
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request['Content-Type'] = 'application/json'
+        request.body = ::JSON.dump(body)
+        http.request(request)
+      end
+
+      # Maps an HTTP response to a status hash.
+      # Follows the Legion API format: { data: {...} } for success,
+      # { error: {...} } for failure.
+      def interpret_response(response)
+        code   = response.code.to_i
+        parsed = safe_parse(response.body)
+
+        case code
+        when 200
+          { status: :immediate, body: parsed.fetch(:data, parsed) }
+        when 201
+          { status: :created,   body: parsed.fetch(:data, parsed) }
+        when 202
+          data = parsed.fetch(:data, {})
+          { status: :accepted, request_id: data[:request_id], poll_key: data[:poll_key] }
+        when 403
+          { status: :denied, error: parsed.fetch(:error, parsed) }
+        when 429
+          retry_after = extract_retry_after(response, parsed)
+          { status: :rate_limited, retry_after: retry_after }
+        when 503
+          { status: :unavailable }
+        else
+          { status: :error, code: code, body: parsed }
+        end
+      end
+
+      # ── private helpers ────────────────────────────────────────────────
+
+      def fetch_daemon_url
+        return nil unless defined?(Legion::LLM) && Legion::LLM.respond_to?(:settings)
+
+        settings = Legion::LLM.settings
+        return nil unless settings.is_a?(Hash)
+
+        daemon = settings[:daemon]
+        return nil unless daemon.is_a?(Hash)
+
+        daemon[:url]
+      rescue StandardError
+        nil
+      end
+
+      def safe_parse(body)
+        return {} if body.nil? || body.strip.empty?
+
+        ::JSON.parse(body, symbolize_names: true)
+      rescue ::JSON::ParserError
+        {}
+      end
+
+      def extract_retry_after(response, parsed)
+        from_body = parsed.dig(:error, :retry_after) || parsed[:retry_after]
+        return from_body.to_i if from_body
+
+        header = response['Retry-After']
+        return header.to_i if header
+
+        0
+      end
+
+      private_class_method :fetch_daemon_url, :safe_parse, :extract_retry_after
+    end
+  end
+end

data/lib/legion/llm/response_cache.rb

@@ -0,0 +1,133 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module Legion
+  module LLM
+    module ResponseCache
+      DEFAULT_TTL      = 300
+      SPOOL_THRESHOLD  = 8 * 1024 * 1024 # 8 MB
+      SPOOL_DIR        = File.expand_path('~/.legionio/data/spool/llm_responses').freeze
+
+      module_function
+
+      # Sets status to :pending for a new request.
+      def init_request(request_id, ttl: DEFAULT_TTL)
+        cache_set(status_key(request_id), 'pending', ttl)
+      end
+
+      # Writes response, meta, and marks status as :done.
+      def complete(request_id, response:, meta:, ttl: DEFAULT_TTL)
+        write_response(request_id, response, ttl)
+        cache_set(meta_key(request_id), ::JSON.dump(meta), ttl)
+        cache_set(status_key(request_id), 'done', ttl)
+      end
+
+      # Writes error details and marks status as :error.
+      def fail_request(request_id, code:, message:, ttl: DEFAULT_TTL)
+        payload = ::JSON.dump({ code: code, message: message })
+        cache_set(error_key(request_id), payload, ttl)
+        cache_set(status_key(request_id), 'error', ttl)
+      end
+
+      # Returns :pending, :done, :error, or nil.
+      def status(request_id)
+        raw = Legion::Cache.get(status_key(request_id))
+        raw&.to_sym
+      end
+
+      # Returns the response string (handles spool overflow transparently).
+      def response(request_id)
+        raw = Legion::Cache.get(response_key(request_id))
+        return nil if raw.nil?
+        return File.read(raw.delete_prefix('spool:')) if raw.start_with?('spool:')
+
+        raw
+      end
+
+      # Returns meta hash with symbolized keys, or nil.
+      def meta(request_id)
+        raw = Legion::Cache.get(meta_key(request_id))
+        return nil if raw.nil?
+
+        ::JSON.parse(raw, symbolize_names: true)
+      end
+
+      # Returns { code:, message: } hash, or nil.
+      def error(request_id)
+        raw = Legion::Cache.get(error_key(request_id))
+        return nil if raw.nil?
+
+        ::JSON.parse(raw, symbolize_names: true)
+      end
+
+      # Blocking poll. Returns { status: :done, response:, meta: },
+      # { status: :error, error: }, or { status: :timeout }.
+      def poll(request_id, timeout: DEFAULT_TTL, interval: 0.1)
+        deadline = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) + timeout
+
+        loop do
+          current = status(request_id)
+
+          case current
+          when :done
+            return { status: :done, response: response(request_id), meta: meta(request_id) }
+          when :error
+            return { status: :error, error: error(request_id) }
+          end
+
+          return { status: :timeout } if ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) >= deadline
+
+          sleep interval
+        end
+      end
+
+      # Removes all cache keys for a request (and any spool file).
+      def cleanup(request_id)
+        raw = Legion::Cache.get(response_key(request_id))
+        if raw&.start_with?('spool:')
+          path = raw.delete_prefix('spool:')
+          FileUtils.rm_f(path)
+        end
+
+        Legion::Cache.delete(status_key(request_id))
+        Legion::Cache.delete(response_key(request_id))
+        Legion::Cache.delete(meta_key(request_id))
+        Legion::Cache.delete(error_key(request_id))
+      end
+
+      # ── private helpers ────────────────────────────────────────────────
+      private_class_method def self.status_key(request_id)
+        "llm:#{request_id}:status"
+      end
+
+      private_class_method def self.response_key(request_id)
+        "llm:#{request_id}:response"
+      end
+
+      private_class_method def self.meta_key(request_id)
+        "llm:#{request_id}:meta"
+      end
+
+      private_class_method def self.error_key(request_id)
+        "llm:#{request_id}:error"
+      end
+
+      private_class_method def self.cache_set(key, value, ttl)
+        Legion::Cache.set(key, value, ttl)
+      end
+
+      private_class_method def self.write_response(request_id, response_text, ttl)
+        if response_text.bytesize > SPOOL_THRESHOLD
+          FileUtils.mkdir_p(SPOOL_DIR)
+          path = File.join(SPOOL_DIR, "#{request_id}.txt")
+          File.write(path, response_text)
+          cache_set(response_key(request_id), "spool:#{path}", ttl)
+        else
+          cache_set(response_key(request_id), response_text, ttl)
+        end
+      end
+    end
+  end
+end

data/lib/legion/llm/settings.rb

@@ -13,7 +13,15 @@ module Legion
           providers:        providers,
           routing:          routing_defaults,
           discovery:        discovery_defaults,
-          gateway:          gateway_defaults
+          gateway:          gateway_defaults,
+          daemon:           daemon_defaults
+        }
+      end
+
+      def self.daemon_defaults
+        {
+          url:     nil,
+          enabled: false
         }
       end
 

data/lib/legion/llm/version.rb

@@ -2,6 +2,6 @@
 
 module Legion
   module LLM
-    VERSION = '0.3.6'
+    VERSION = '0.3.7'
   end
 end

data/lib/legion/llm.rb

@@ -8,6 +8,8 @@ require 'legion/llm/router'
 require 'legion/llm/compressor'
 require 'legion/llm/quality_checker'
 require 'legion/llm/escalation_history'
+require_relative 'llm/response_cache'
+require_relative 'llm/daemon_client'
 
 begin
   require 'legion/extensions/llm/gateway'
@@ -18,6 +20,8 @@ end
 module Legion
   module LLM
     class EscalationExhausted < StandardError; end
+    class DaemonDeniedError < StandardError; end
+    class DaemonRateLimitedError < StandardError; end
 
     class << self
       include Legion::LLM::Providers
@@ -71,6 +75,19 @@ module Legion
                     quality_check: quality_check, message: message, **)
       end
 
+      # Send a single message — daemon-first, falls through to direct on unavailability.
+      def ask(message:, model: nil, provider: nil, intent: nil, tier: nil,
+              context: {}, identity: nil, &)
+        if DaemonClient.available?
+          result = daemon_ask(message: message, model: model, provider: provider,
+                              context: context, tier: tier, identity: identity)
+          return result if result
+        end
+
+        ask_direct(message: message, model: model, provider: provider,
+                   intent: intent, tier: tier, &)
+      end
+
       # Direct chat bypassing gateway — used by gateway runners to avoid recursion
       def chat_direct(model: nil, provider: nil, intent: nil, tier: nil, escalate: nil,
                       max_escalations: nil, quality_check: nil, message: nil, **)
@@ -135,6 +152,41 @@ module Legion
 
       private
 
+      def daemon_ask(message:, model: nil, provider: nil, context: {}, tier: nil, identity: nil) # rubocop:disable Lint/UnusedMethodArgument
+        result = DaemonClient.chat(
+          message: message, model: model, provider: provider,
+          context: context, tier_preference: tier || :auto
+        )
+
+        case result[:status]
+        when :immediate, :created
+          result[:body]
+        when :accepted
+          ResponseCache.poll(result[:request_id])
+        when :denied
+          raise DaemonDeniedError, result.dig(:error, :message) || 'Access denied'
+        when :rate_limited
+          raise DaemonRateLimitedError, "Rate limited. Retry after #{result[:retry_after]}s"
+        end
+        # Returns nil for :unavailable/:error — caller falls through to direct
+      end
+
+      def ask_direct(message:, model: nil, provider: nil, intent: nil, tier: nil, &block)
+        session = chat_direct(model: model, provider: provider, intent: intent, tier: tier)
+        response = block ? session.ask(message, &block) : session.ask(message)
+
+        {
+          status:   :done,
+          response: response.content,
+          meta:     {
+            tier:       :direct,
+            model:      session.model.to_s,
+            tokens_in:  response.respond_to?(:input_tokens) ? response.input_tokens : nil,
+            tokens_out: response.respond_to?(:output_tokens) ? response.output_tokens : nil
+          }
+        }
+      end
+
       def gateway_loaded?
         defined?(Legion::Extensions::LLM::Gateway::Runners::Inference)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: legion-llm
 version: !ruby/object:Gem::Version
-  version: 0.3.6
+  version: 0.3.7
 platform: ruby
 authors:
 - Esity
@@ -133,6 +133,7 @@ files:
 - lib/legion/llm/bedrock_bearer_auth.rb
 - lib/legion/llm/claude_config_loader.rb
 - lib/legion/llm/compressor.rb
+- lib/legion/llm/daemon_client.rb
 - lib/legion/llm/discovery/ollama.rb
 - lib/legion/llm/discovery/system.rb
 - lib/legion/llm/embeddings.rb
@@ -140,6 +141,7 @@ files:
 - lib/legion/llm/helpers/llm.rb
 - lib/legion/llm/providers.rb
 - lib/legion/llm/quality_checker.rb
+- lib/legion/llm/response_cache.rb
 - lib/legion/llm/router.rb
 - lib/legion/llm/router/escalation_chain.rb
 - lib/legion/llm/router/gateway_interceptor.rb