lex-ollama 0.3.4 → 0.3.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 978c53ff8a178c003a5bb593a934536c20b616500d80ea0624f97014f9a88213
-  data.tar.gz: d700e31e6f38fe2b9c6cac3da627d67ce1ccab9a75d3d6d741cdc04f5cc614bf
+  metadata.gz: c764634fdcad6f8a0d70a8221eb26979cd4bc0338e68b61b072d552574da5ced
+  data.tar.gz: cab333e0e78243cddfbc291fc9d243a652ecd4ff5fbcecf942a442e76ab171fc
 SHA512:
-  metadata.gz: 6f44dcfc98336bcd0d28e6985ed468f7676b156d5135ff642256120db59563e161d46615b9acab0e3cdac6b578144121d60a23efc17c31f6f6c686349519f076
-  data.tar.gz: b191eacce0844eb0be9b6b4b22f12969007b37f335da700f6ea6bd4936b22fd6aa2eec945dbdfbb419f5b1a9f9f1b0c9e15c004d772e18a1a98c059c133e83e8
+  metadata.gz: 6efd9372da01f35c6e23a81156e209937b594674c32255a3f15e3bde04196108d734478735bdbe0e081191354269be5df212e7635df0c5511aafa6f24e56c4fb
+  data.tar.gz: cc829781858f0786b3a29353b372e792b45363b3b294a604f6496b3f0779b698f229db99996c8f8ebeae9461f0cb150a725dfdf49209e87b8e7f89a06206d233

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Changelog
 
+## [0.3.5] - 2026-04-25
+
+### Added
+- Fleet model workers now bind transient classic queues to shared `llm.fleet` model lanes, with configurable consumer priority, queue expiration, and message TTL.
+- Subscription entries can provide a context window so inference workers bind lanes like `llm.fleet.inference.qwen3-5-27b.ctx32768`.
+
 ## [0.3.4] - 2026-04-24
 
 ### Fixed

data/lib/legion/extensions/ollama/actors/endpoint_puller.rb

@@ -0,0 +1,230 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Ollama
+      module Actor
+        # Polls configured fleet queues with basic_get so endpoint machines choose
+        # when they are ready for work instead of holding prefetched messages.
+        class EndpointPuller < Legion::Extensions::Actors::Every
+          def runner_class
+            self.class
+          end
+
+          def runner_function
+            'action'
+          end
+
+          def use_runner?
+            false
+          end
+
+          def check_subtask?
+            false
+          end
+
+          def generate_task?
+            false
+          end
+
+          def enabled?
+            fleet_scheduler == :basic_get && endpoint_enabled? && subscriptions.any?
+          rescue StandardError => e
+            handle_exception(e, level: :warn, handled: true)
+            false
+          end
+
+          def time
+            (setting_value(endpoint_settings, :idle_backoff_ms) || 1_000).to_f / 1000
+          end
+
+          def action
+            return unless enabled?
+
+            now = monotonic_time
+            ordered_subscriptions.each do |sub|
+              next if lane_backed_off?(sub, now)
+
+              pulled = drain_lane(sub)
+              mark_lane_empty(sub) if pulled.zero?
+            end
+          end
+
+          def ordered_subscriptions
+            subscriptions.sort_by do |sub|
+              type = sub[:type].to_s
+              [embed_type?(type) ? 0 : 1, context_limit(sub), sub[:model].to_s]
+            end
+          end
+
+          def drain_lane(subscription)
+            pulls = 0
+            queue = queue_for(subscription)
+
+            loop do
+              break if max_consecutive_pulls_per_lane.positive? && pulls >= max_consecutive_pulls_per_lane
+              break unless pull_one(queue, subscription)
+
+              pulls += 1
+            end
+            pulls
+          end
+
+          def pull_one(queue, subscription)
+            delivery_info, metadata, payload = queue.pop(manual_ack: true)
+            return false unless delivery_info
+
+            message = process_payload(payload, metadata, delivery_info, subscription)
+            Legion::Extensions::Ollama::Runners::Fleet.handle_request(**message)
+            queue.acknowledge(delivery_info.delivery_tag)
+            true
+          rescue StandardError => e
+            handle_exception(e, lex: lex_name, routing_key: delivery_info&.routing_key)
+            queue.reject(delivery_info.delivery_tag, requeue: false) if delivery_info
+            true
+          end
+
+          def queue_for(subscription)
+            @queues ||= {}
+            @queues[lane_key(subscription)] ||= ModelWorker.queue_class_for(
+              request_type:   subscription[:type],
+              model:          subscription[:model],
+              context_window: finite_context_limit(subscription),
+              queue_config:   queue_config
+            ).new
+          end
+
+          def process_payload(payload, metadata, delivery_info, subscription)
+            message = decode_payload(payload, metadata)
+            message = message.merge(metadata.headers.transform_keys(&:to_sym)) if metadata&.headers
+            message[:routing_key] = delivery_info.routing_key if delivery_info.respond_to?(:routing_key)
+            message[:request_type] ||= subscription[:type].to_s
+            message[:model] ||= subscription[:model].to_s
+            message[:message_context] ||= {}
+            message
+          end
+
+          def decode_payload(payload, metadata)
+            decoded = if metadata&.content_encoding == 'encrypted/cs'
+                        Legion::Crypt.decrypt(payload, metadata_header(metadata, :iv))
+                      elsif metadata&.content_encoding == 'encrypted/pk'
+                        Legion::Crypt.decrypt_from_keypair(metadata_header(metadata, :public_key), payload)
+                      else
+                        payload
+                      end
+
+            if metadata&.content_type == 'application/json'
+              Legion::JSON.load(decoded)
+            else
+              { value: decoded }
+            end
+          end
+
+          def subscriptions
+            configured = setting_value(settings, :subscriptions)
+            return [] unless configured.is_a?(Array)
+
+            configured.filter_map do |sub|
+              next unless sub.is_a?(Hash)
+
+              normalized = sub.transform_keys(&:to_sym)
+              next unless normalized[:type] && normalized[:model]
+
+              normalized
+            end
+          end
+
+          def queue_config
+            {
+              queue_expires_ms:        nested_setting(settings, :fleet, :queue_expires_ms),
+              message_ttl_ms:          nested_setting(settings, :fleet, :message_ttl_ms),
+              queue_max_length:        nested_setting(settings, :fleet, :queue_max_length),
+              delivery_limit:          nested_setting(settings, :fleet, :delivery_limit),
+              consumer_ack_timeout_ms: nested_setting(settings, :fleet, :consumer_ack_timeout_ms)
+            }.compact
+          end
+
+          def endpoint_settings
+            nested_setting(settings, :fleet, :endpoint) || {}
+          end
+
+          def endpoint_enabled?
+            setting_value(endpoint_settings, :enabled) == true
+          end
+
+          def max_consecutive_pulls_per_lane
+            setting_value(endpoint_settings, :max_consecutive_pulls_per_lane) || 0
+          end
+
+          def empty_lane_backoff_seconds
+            (setting_value(endpoint_settings, :empty_lane_backoff_ms) || 250).to_f / 1000
+          end
+
+          def lane_backed_off?(subscription, now)
+            (@empty_lanes ||= {}).fetch(lane_key(subscription), 0) > now
+          end
+
+          def mark_lane_empty(subscription)
+            (@empty_lanes ||= {})[lane_key(subscription)] = monotonic_time + empty_lane_backoff_seconds
+          end
+
+          def lane_key(subscription)
+            type = subscription[:type]
+            model = subscription[:model]
+            context = context_limit(subscription)
+            context.finite? ? "#{type}:#{model}:ctx#{context}" : "#{type}:#{model}"
+          end
+
+          def monotonic_time
+            Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          end
+
+          def fleet_scheduler
+            (nested_setting(settings, :fleet, :scheduler) || :basic_get).to_sym
+          end
+
+          def context_limit(subscription)
+            raw = setting_value(subscription, :max_context_size) ||
+                  setting_value(subscription, :context_window) ||
+                  setting_value(subscription, :max_input_tokens) ||
+                  setting_value(subscription, :context) ||
+                  setting_value(subscription, :ctx)
+            Integer(raw || Float::INFINITY)
+          rescue ArgumentError, TypeError, FloatDomainError
+            Float::INFINITY
+          end
+
+          def finite_context_limit(subscription)
+            context = context_limit(subscription)
+            context.finite? ? context : nil
+          end
+
+          def embed_type?(type)
+            %w[embed embedding embeddings].include?(type)
+          end
+
+          def metadata_header(metadata, key)
+            setting_value(metadata&.headers || {}, key)
+          end
+
+          def nested_setting(hash, *keys)
+            keys.reduce(hash) do |current, key|
+              return nil unless current.respond_to?(:key?)
+
+              setting_value(current, key)
+            end
+          end
+
+          def setting_value(hash, key)
+            return nil unless hash.respond_to?(:key?)
+
+            string_key = key.to_s
+            return hash[string_key] if hash.key?(string_key)
+
+            hash[key] if hash.key?(key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/ollama/actors/model_worker.rb

@@ -19,15 +19,17 @@ module Legion
         #         - type: chat
         #           model: "qwen3.5:27b"
         #
-        # The queue name and routing key both follow the schema:
-        #   llm.request.ollama.<type>.<model>
-        # where model colons are converted to dots (AMQP topic word separator).
+        # The queue name and routing key both follow shared fleet lane schemas:
+        #   llm.fleet.embed.<model>
+        #   llm.fleet.inference.<model>.ctx<context_window>
+        # when an inference context window is known.
         class ModelWorker < Legion::Extensions::Actors::Subscription
-          attr_reader :request_type, :model_name
+          attr_reader :request_type, :model_name, :context_window
 
-          def initialize(request_type:, model:, **)
-            @request_type = request_type.to_s
-            @model_name   = model.to_s
+          def initialize(request_type:, model:, context_window: nil, **)
+            @request_type    = request_type.to_s
+            @model_name      = model.to_s
+            @context_window  = context_window&.to_i
             super(**)
           end
 
@@ -59,7 +61,27 @@ module Legion
           # Standard scale: GPU server = 10, Mac Studio = 5, developer laptop = 1.
           # Defaults to 0 (equal priority) if not configured.
           def consumer_priority
-            settings.dig(:fleet, :consumer_priority) || 0
+            setting_value(fleet_settings, :consumer_priority) || 0
+          end
+
+          def queue_expires_ms
+            setting_value(fleet_settings, :queue_expires_ms) || 60_000
+          end
+
+          def message_ttl_ms
+            setting_value(fleet_settings, :message_ttl_ms) || 120_000
+          end
+
+          def queue_max_length
+            setting_value(fleet_settings, :queue_max_length) || 100
+          end
+
+          def delivery_limit
+            setting_value(fleet_settings, :delivery_limit) || 3
+          end
+
+          def consumer_ack_timeout_ms
+            setting_value(fleet_settings, :consumer_ack_timeout_ms) || 300_000
           end
 
           # Subscribe options include x-priority argument so RabbitMQ can honour
@@ -73,14 +95,45 @@ module Legion
             base.merge(arguments: { 'x-priority' => consumer_priority })
           end
 
-          # Returns a queue CLASS (not instance) bound to the llm.request exchange
-          # with the routing key for this worker's (type, model) pair.
+          # Returns a queue CLASS (not instance) bound to the llm.fleet exchange
+          # with the routing key for this worker's model offering lane.
           # The Subscription base class calls queue.new in initialize, so this must
           # return a class, not an instance.
           def queue
             @queue ||= build_queue_class
           end
 
+          def self.queue_class_for(request_type:, model:, context_window: nil, queue_config: {})
+            worker = allocate
+            worker.instance_variable_set(:@request_type, request_type.to_s)
+            worker.instance_variable_set(:@model_name, model.to_s)
+            worker.instance_variable_set(:@context_window, context_window&.to_i)
+            worker.send(:build_queue_class, queue_config)
+          end
+
+          def self.fallback_queue_options(settings)
+            {
+              durable:     true,
+              auto_delete: false,
+              arguments:   {
+                'x-queue-type'           => 'quorum',
+                'x-queue-leader-locator' => 'balanced',
+                'x-expires'              => settings.fetch(:queue_expires_ms),
+                'x-message-ttl'          => settings.fetch(:message_ttl_ms),
+                'x-overflow'             => 'reject-publish',
+                'x-max-length'           => settings.fetch(:queue_max_length),
+                'x-delivery-limit'       => settings.fetch(:delivery_limit),
+                'x-consumer-timeout'     => settings.fetch(:consumer_ack_timeout_ms)
+              }
+            }
+          end
+
+          def routing_key
+            parts = ['llm.fleet', lane_kind, sanitized_model]
+            parts << "ctx#{@context_window}" if lane_kind == 'inference' && @context_window
+            parts.join('.')
+          end
+
           # Enrich every inbound message with the worker's own request_type and model
           # so Runners::Fleet#handle_request always has them, even if the sender omitted
           # them. Also defaults message_context to {} if absent.
@@ -94,23 +147,62 @@ module Legion
 
           private
 
-          def build_queue_class
-            sanitised_model = @model_name.tr(':', '.')
-            routing_key     = "llm.request.ollama.#{@request_type}.#{sanitised_model}"
+          def build_queue_class(queue_config = {})
+            lane_key        = routing_key
             exchange_class  = Transport::Exchanges::LlmRequest
+            queue_settings  = {
+              queue_expires_ms:        queue_expires_ms,
+              message_ttl_ms:          message_ttl_ms,
+              queue_max_length:        queue_max_length,
+              delivery_limit:          delivery_limit,
+              consumer_ack_timeout_ms: consumer_ack_timeout_ms
+            }.merge((queue_config || {}).compact)
+
+            if defined?(::Legion::Extensions::Llm::Transport::FleetLane)
+              return ::Legion::Extensions::Llm::Transport::FleetLane.build_queue_class(
+                queue_name:       lane_key,
+                exchange_class:   exchange_class,
+                routing_key:      lane_key,
+                base_queue_class: Legion::Transport::Queue,
+                settings:         queue_settings
+              )
+            end
+
+            queue_options = self.class.fallback_queue_options(queue_settings)
 
             Class.new(Legion::Transport::Queue) do
-              define_method(:queue_name) { routing_key }
-              define_method(:queue_options) do
-                { durable: false, auto_delete: true, arguments: { 'x-max-priority' => 10 } }
-              end
+              define_method(:queue_name) { lane_key }
+              define_method(:queue_options) { queue_options }
               define_method(:dlx_enabled) { false }
               define_method(:initialize) do
                 super()
-                bind(exchange_class.new, routing_key: routing_key)
+                bind(exchange_class.new, routing_key: lane_key)
               end
             end
           end
+
+          def fleet_settings
+            setting_value(settings, :fleet) || {}
+          rescue NameError
+            {}
+          end
+
+          def setting_value(hash, key)
+            return nil unless hash.respond_to?(:key?)
+
+            string_key = key.to_s
+            return hash[string_key] if hash.key?(string_key)
+
+            hash[key] if hash.key?(key)
+          end
+
+          def lane_kind
+            %w[embed embedding embeddings].include?(@request_type) ? 'embed' : 'inference'
+          end
+
+          def sanitized_model
+            @model_name.downcase.gsub(/[^a-z0-9]+/, '-').gsub(/\A-+|-+\z/, '').squeeze('-')
+          end
         end
       end
     end

data/lib/legion/extensions/ollama/version.rb

@@ -3,7 +3,7 @@
 module Legion
   module Extensions
     module Ollama
-      VERSION = '0.3.4'
+      VERSION = '0.3.5'
     end
   end
 end

data/lib/legion/extensions/ollama.rb

@@ -14,6 +14,12 @@ require 'legion/extensions/ollama/runners/version'
 require 'legion/extensions/ollama/runners/fleet'
 require 'legion/extensions/ollama/client'
 
+begin
+  require 'legion/extensions/llm'
+rescue LoadError
+  nil
+end
+
 # Fleet transport and actor wiring — only loaded when Legion::Extensions::Core is present
 # so the gem still works as a standalone HTTP client without any AMQP runtime.
 if Legion::Extensions.const_defined?(:Core, false)
@@ -21,6 +27,7 @@ if Legion::Extensions.const_defined?(:Core, false)
   require 'legion/extensions/ollama/transport/messages/llm_response'
   require 'legion/extensions/ollama/transport'
   require 'legion/extensions/ollama/actors/model_worker'
+  require 'legion/extensions/ollama/actors/endpoint_puller'
   require 'legion/extensions/ollama/actors/model_sync'
 end
 
@@ -32,7 +39,22 @@ module Legion
       def self.default_settings
         {
           s3:    {},
-          fleet: {}
+          fleet: {
+            consumer_priority:       0,
+            scheduler:               :basic_get,
+            queue_expires_ms:        60_000,
+            message_ttl_ms:          120_000,
+            queue_max_length:        100,
+            delivery_limit:          3,
+            consumer_ack_timeout_ms: 300_000,
+            endpoint:                {
+              enabled:                        false,
+              empty_lane_backoff_ms:          250,
+              idle_backoff_ms:                1_000,
+              max_consecutive_pulls_per_lane: 0,
+              accept_when:                    []
+            }
+          }
         }
       end
 
@@ -43,17 +65,29 @@ module Legion
         super
         @actors.delete(:model_worker)
 
-        subs = settings[:subscriptions]
+        subs = setting_value(settings, :subscriptions)
+        valid_subscriptions = valid_fleet_subscriptions(subs)
+        endpoint_configured = fleet_scheduler == :basic_get &&
+                              nested_setting(settings, :fleet, :endpoint, :enabled) == true &&
+                              valid_subscriptions.any?
+        @actors.delete(:endpoint_puller) unless endpoint_configured
+
         return unless subs.is_a?(Array)
+        return if fleet_scheduler == :basic_get
 
         subs.each do |sub|
-          request_type = sub[:type]&.to_s
-          model        = sub[:model]&.to_s
+          request_type   = setting_value(sub, :type)&.to_s
+          model          = setting_value(sub, :model)&.to_s
+          limits         = setting_value(sub, :limits) || {}
+          context_window = setting_value(sub, :context_window) ||
+                           setting_value(limits, :context_window)
           next unless request_type && model
 
           actor_name   = :"model_worker_#{request_type}_#{model.tr(':.', '__')}"
           worker_class = Class.new(Legion::Extensions::Ollama::Actor::ModelWorker) do
-            define_method(:initialize) { super(request_type: request_type, model: model) }
+            define_method(:initialize) do
+              super(request_type: request_type, model: model, context_window: context_window)
+            end
           end
 
           @actors[actor_name] = {
@@ -65,6 +99,35 @@ module Legion
           }
         end
       end
+
+      def self.fleet_scheduler
+        (nested_setting(settings, :fleet, :scheduler) || :basic_get).to_sym
+      end
+
+      def self.valid_fleet_subscriptions(subscriptions)
+        return [] unless subscriptions.is_a?(Array)
+
+        subscriptions.select do |sub|
+          setting_value(sub, :type) && setting_value(sub, :model)
+        end
+      end
+
+      def self.setting_value(hash, key)
+        return nil unless hash.respond_to?(:key?)
+
+        string_key = key.to_s
+        return hash[string_key] if hash.key?(string_key)
+
+        hash[key] if hash.key?(key)
+      end
+
+      def self.nested_setting(hash, *keys)
+        keys.reduce(hash) do |current, key|
+          return nil unless current.respond_to?(:key?)
+
+          setting_value(current, key)
+        end
+      end
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: lex-ollama
 version: !ruby/object:Gem::Version
-  version: 0.3.4
+  version: 0.3.5
 platform: ruby
 authors:
 - Esity
@@ -56,6 +56,7 @@ files:
 - README.md
 - lex-ollama.gemspec
 - lib/legion/extensions/ollama.rb
+- lib/legion/extensions/ollama/actors/endpoint_puller.rb
 - lib/legion/extensions/ollama/actors/model_sync.rb
 - lib/legion/extensions/ollama/actors/model_worker.rb
 - lib/legion/extensions/ollama/client.rb