lex-llm-gateway 0.2.3 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 113ab7ef4818f7904351d1c4ebf08bdfc8beabc3310c219d909443399d051e41
-  data.tar.gz: '0285316fbedb4b01afed93c25d9897b7723de482d06dd38bb85b2c6e59bcf2d8'
+  metadata.gz: 44562169a54b0e789ff4656a30599b2c43314c59fd5b5bb3056bf790da5a7a01
+  data.tar.gz: 399266aaa074b1eeb0c69044987bf36d13a11668defcda7066133f5c25ed5a93
 SHA512:
-  metadata.gz: be41efa7a0e59477137225023730e271b04be3343cb178fd7110e863320f95d5ffeaafa4fc8d912056c78fd739e9ac0d6d2ec77b6d90c0d991b0df1622943e21
-  data.tar.gz: 985e1a3bc30d68657dacdc8f0df39bb5cd33d0cbc7192eb3336e99c7161d99e74af6909b99814b92ebd6084dc989ae5e0bb257c285482509be308cc43a5f9519
+  metadata.gz: 9d122cadcffddfa7fa848a2d5cd992682e40a2e9cf84e5868af845d90572a7965f9fd462e6f23bc1d275410f48332ce7111a4d823dbbf17de3d16583903f407c
+  data.tar.gz: a188786600832ed973a988da109d053a87e05b7cdea82ee08914f1cad7931d4db2f8f4f1f2ea37fe442f9382d731f4980e6a099c6e6db84af1bca10b49b95512

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Changelog
 
+## [0.2.5] - 2026-03-23
+
+### Added
+- FleetHandler multi-request-type dispatch: structured, embed, and multi-message chat
+- `call_chat` supports single and multi-message payloads
+- `call_structured` dispatches to `Legion::LLM.structured` with schema
+- `call_embed` dispatches to `Legion::LLM.embed` with text fallback from messages
+
+## [0.2.4] - 2026-03-23
+
+### Added
+- Implement fleet RPC `wait_for_response` with `Concurrent::Promises` future and correlation ID matching
+- Add `Helpers::ReplyDispatcher` process-singleton for managing reply queue consumer and pending futures
+- Add `FleetHandler.publish_reply` to send `InferenceResponse` back to requester via AMQP default exchange
+
+### Fixed
+- Fix `Actor::InferenceWorker` runner_class mismatch: now points to `FleetHandler` instead of `Inference`
+- Add `use_runner? false` to InferenceWorker so it dispatches directly to the runner module
+
 ## [0.2.3] - 2026-03-22
 
 ### Changed

data/lib/legion/extensions/llm/gateway/actors/inference_worker.rb

@@ -7,12 +7,16 @@ module Legion
         module Actor
           class InferenceWorker < Legion::Extensions::Actors::Subscription
             def runner_class
-              'Legion::Extensions::LLM::Gateway::Runners::Inference'
+              'Legion::Extensions::LLM::Gateway::Runners::FleetHandler'
             end
 
             def runner_function
               'handle_fleet_request'
             end
+
+            def use_runner?
+              false
+            end
           end
         end
       end

data/lib/legion/extensions/llm/gateway/helpers/reply_dispatcher.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'concurrent'
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Helpers
+          module ReplyDispatcher
+            @pending = Concurrent::Map.new
+            @mutex = Mutex.new
+            @consumer = nil
+
+            module_function
+
+            def register(correlation_id)
+              future = Concurrent::Promises.resolvable_future
+              @pending[correlation_id] = future
+              ensure_consumer
+              future
+            end
+
+            def deregister(correlation_id)
+              @pending.delete(correlation_id)
+            end
+
+            def handle_delivery(raw_payload, properties = {})
+              payload = parse_payload(raw_payload)
+              cid = properties[:correlation_id] || payload[:correlation_id]
+              return unless cid
+
+              future = @pending.delete(cid)
+              return unless future
+
+              future.fulfill(payload.merge(success: true))
+            rescue StandardError => e
+              log_warn("ReplyDispatcher: handle_delivery failed: #{e.message}")
+            end
+
+            def pending_count
+              @pending.size
+            end
+
+            def reset!
+              @mutex.synchronize do
+                cancel_consumer
+                @pending = Concurrent::Map.new
+              end
+            end
+
+            # private
+
+            def ensure_consumer # rubocop:disable Metrics/MethodLength
+              @mutex.synchronize do
+                return if @consumer
+                return unless transport_available?
+
+                queue_name = Rpc.agent_queue_name
+                return unless queue_name
+
+                channel = Legion::Transport.connection.create_channel
+                queue = channel.queue(queue_name, auto_delete: true, durable: false)
+                @consumer = queue.subscribe(manual_ack: false) do |_delivery, properties, body|
+                  props = { correlation_id: properties.correlation_id }
+                  handle_delivery(body, props)
+                end
+              end
+            rescue StandardError => e
+              log_warn("ReplyDispatcher: consumer setup failed: #{e.message}")
+            end
+
+            def cancel_consumer
+              @consumer&.cancel
+              @consumer = nil
+            rescue StandardError => e
+              log_warn("ReplyDispatcher: cancel failed: #{e.message}")
+            end
+
+            def transport_available?
+              defined?(Legion::Transport) &&
+                Legion::Transport.respond_to?(:connection) &&
+                Legion::Transport.connection
+            end
+
+            def parse_payload(raw)
+              return raw if raw.is_a?(Hash)
+
+              if defined?(Legion::JSON)
+                Legion::JSON.load(raw)
+              else
+                require 'json'
+                JSON.parse(raw, symbolize_names: true)
+              end
+            rescue StandardError
+              {}
+            end
+
+            def log_warn(msg)
+              Legion::Logging.warn(msg) if defined?(Legion::Logging)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/fleet.rb

@@ -78,6 +78,16 @@ module Legion
             end
 
             def wait_for_response(correlation_id, timeout:)
+              future = Helpers::ReplyDispatcher.register(correlation_id)
+              result = future.value!(timeout)
+              result || timeout_result(correlation_id, timeout)
+            rescue Concurrent::CancelledOperationError
+              timeout_result(correlation_id, timeout)
+            ensure
+              Helpers::ReplyDispatcher.deregister(correlation_id)
+            end
+
+            def timeout_result(correlation_id, timeout)
               { success: false, error: 'fleet_timeout', correlation_id: correlation_id, timeout: timeout }
             end
 

data/lib/legion/extensions/llm/gateway/runners/fleet_handler.rb

@@ -10,10 +10,16 @@ module Legion
 
             def handle_fleet_request(payload)
               token = payload[:signed_token]
-              return { success: false, error: 'invalid_token' } if require_auth? && !valid_token?(token)
+              if require_auth? && !valid_token?(token)
+                error_response = { success: false, error: 'invalid_token' }
+                publish_reply(payload[:reply_to], payload[:correlation_id], error_response) if payload[:reply_to]
+                return error_response
+              end
 
               response = call_local_llm(payload)
-              build_response(payload[:correlation_id], response)
+              response_hash = build_response(payload[:correlation_id], response)
+              publish_reply(payload[:reply_to], payload[:correlation_id], response_hash) if payload[:reply_to]
+              response_hash
             end
 
             def require_auth?
@@ -29,12 +35,38 @@ module Legion
             def call_local_llm(payload)
               return { error: 'llm_not_available' } unless defined?(Legion::LLM)
 
-              Legion::LLM.chat(
+              case payload[:request_type]&.to_s
+              when 'structured'
+                call_structured(payload)
+              when 'embed'
+                call_embed(payload)
+              else
+                call_chat(payload)
+              end
+            end
+
+            def call_chat(payload)
+              messages = payload[:messages]
+              if messages.is_a?(Array) && messages.size > 1
+                Legion::LLM.chat(model: payload[:model], messages: messages)
+              else
+                Legion::LLM.chat(model: payload[:model], message: messages&.dig(0, :content))
+              end
+            end
+
+            def call_structured(payload)
+              Legion::LLM.structured(
                 model: payload[:model],
-                message: payload.dig(:messages, 0, :content)
+                messages: payload[:messages],
+                schema: payload[:schema]
               )
             end
 
+            def call_embed(payload)
+              text = payload[:text] || payload.dig(:messages, 0, :content)
+              Legion::LLM.embed(model: payload[:model], text: text)
+            end
+
             def build_response(correlation_id, response)
               {
                 correlation_id: correlation_id,
@@ -47,6 +79,32 @@ module Legion
               }
             end
 
+            def publish_reply(reply_to, correlation_id, response_hash) # rubocop:disable Metrics/MethodLength
+              return unless defined?(Legion::Transport)
+
+              payload = if defined?(Legion::JSON)
+                          Legion::JSON.dump(response_hash)
+                        else
+                          require 'json'
+                          JSON.generate(response_hash)
+                        end
+
+              channel = Legion::Transport.connection.create_channel
+              channel.default_exchange.publish(
+                payload,
+                routing_key: reply_to,
+                correlation_id: correlation_id,
+                content_type: 'application/json'
+              )
+              channel.close
+            rescue StandardError => e
+              log_warn("FleetHandler: publish_reply failed: #{e.message}")
+            end
+
+            def log_warn(msg)
+              Legion::Logging.warn(msg) if defined?(Legion::Logging)
+            end
+
             def extract_token(response, field)
               return 0 unless response.respond_to?(field)
 

data/lib/legion/extensions/llm/gateway/version.rb

@@ -4,7 +4,7 @@ module Legion
   module Extensions
     module LLM
       module Gateway
-        VERSION = '0.2.3'
+        VERSION = '0.2.5'
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-llm-gateway
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.5
 platform: ruby
 authors:
 - Esity
@@ -182,6 +182,7 @@ files:
 - lib/legion/extensions/llm/gateway/actors/spool_flush.rb
 - lib/legion/extensions/llm/gateway/client.rb
 - lib/legion/extensions/llm/gateway/helpers/auth.rb
+- lib/legion/extensions/llm/gateway/helpers/reply_dispatcher.rb
 - lib/legion/extensions/llm/gateway/helpers/rpc.rb
 - lib/legion/extensions/llm/gateway/runners/fleet.rb
 - lib/legion/extensions/llm/gateway/runners/fleet_handler.rb