lex-llm-gateway 0.2.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: ae55215612840dc5c59339698ffd9b9ff3ab9cde5db07a38f0e537dd201e2f1d
+  data.tar.gz: ffa28291a888a470e761135d13ed1b289a5acca80d769e061b4ea589a94a3e48
+SHA512:
+  metadata.gz: 813cb8b76c470236fe69a494f03107233fbd92a58937f8b9c61dffa877fc6873d14848a1913051982f81d0c72882e288c9da0dd2596e47fc92f1cdf2be928284
+  data.tar.gz: 642c7ccb73faa53e80b7474cfa510a963d73521d96234fd434dfa292766dccc97c5246a607a437715f2791e78c4d28c2a0d9918bc33dd441eaefadfb3cf91192

data/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## [0.2.0] - 2026-03-18
+
+### Added
+- Transport topology: metering (topic) and inference (direct) exchanges, queues, messages
+- RPC correlation and JWT auth helpers
+- Metering event builder with publish-or-spool fallback
+- Inference runner: chat, embed, structured with auto-metering
+- Fleet RPC dispatch runner with JWT auth and timeout
+- Fleet handler for incoming inference requests
+- Metering writer: consumes RMQ events, writes to DB
+- Spool flush interval actor (every 60s)
+- Standalone Client class with all runners
+
+## [0.1.0] - 2026-03-18
+
+### Added
+- Initial gem scaffold
+- Extension entry point and version

data/Gemfile

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Esity
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,15 @@
+# lex-llm-gateway
+
+LLM inference gateway for LegionIO. Provides centralized metering over RabbitMQ, fleet RPC dispatch to GPU workers, and local disk spooling for offline resilience.
+
+## Installation
+
+Add to your Gemfile:
+
+```ruby
+gem 'lex-llm-gateway'
+```
+
+## License
+
+MIT

data/lex-llm-gateway.gemspec

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+lib = File.expand_path('lib', __dir__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'legion/extensions/llm/gateway/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'lex-llm-gateway'
+  spec.version       = Legion::Extensions::LLM::Gateway::VERSION
+  spec.authors       = ['Esity']
+  spec.email         = ['matthewdiverson@gmail.com']
+
+  spec.summary       = 'Legion::Extensions::LLM::Gateway'
+  spec.description   = 'LLM inference gateway: metering over RabbitMQ, fleet RPC dispatch, local disk spool'
+  spec.homepage      = 'https://github.com/LegionIO/lex-llm-gateway'
+  spec.license       = 'MIT'
+  spec.required_ruby_version = '>= 3.4'
+
+  spec.metadata['homepage_uri']          = spec.homepage
+  spec.metadata['source_code_uri']       = 'https://github.com/LegionIO/lex-llm-gateway'
+  spec.metadata['changelog_uri']         = 'https://github.com/LegionIO/lex-llm-gateway/blob/main/CHANGELOG.md'
+  spec.metadata['documentation_uri']     = 'https://github.com/LegionIO/lex-llm-gateway'
+  spec.metadata['bug_tracker_uri']       = 'https://github.com/LegionIO/lex-llm-gateway/issues'
+  spec.metadata['rubygems_mfa_required'] = 'true'
+
+  spec.files = Dir.chdir(File.expand_path(__dir__)) do
+    Dir.glob('{lib,exe}/**/*') + %w[lex-llm-gateway.gemspec Gemfile README.md CHANGELOG.md LICENSE]
+  end
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rubocop'
+  spec.add_development_dependency 'rubocop-rspec'
+end

data/lib/legion/extensions/llm/gateway/actors/inference_worker.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Actor
+          class InferenceWorker < Legion::Extensions::Actors::Subscription
+            def runner_class
+              'Legion::Extensions::LLM::Gateway::Runners::Inference'
+            end
+
+            def runner_function
+              'handle_fleet_request'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/actors/metering_writer.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Actor
+          class MeteringWriter < Legion::Extensions::Actors::Subscription
+            def runner_class
+              'Legion::Extensions::LLM::Gateway::Runners::MeteringWriter'
+            end
+
+            def runner_function
+              'write_metering_record'
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/actors/spool_flush.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Actor
+          class SpoolFlush < Legion::Extensions::Actors::Every
+            def runner_class
+              'Legion::Extensions::LLM::Gateway::Runners::Metering'
+            end
+
+            def runner_function
+              'flush_spool'
+            end
+
+            def time
+              60
+            end
+
+            def run_now?
+              false
+            end
+
+            def use_runner?
+              false
+            end
+
+            def check_subtask?
+              false
+            end
+
+            def generate_task?
+              false
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/client.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative 'runners/inference'
+require_relative 'runners/metering'
+require_relative 'runners/fleet'
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        class Client
+          def initialize(**opts)
+            @opts = opts
+          end
+
+          def settings
+            { options: @opts }
+          end
+
+          def chat(**)
+            Runners::Inference.chat(**)
+          end
+
+          def embed(**)
+            Runners::Inference.embed(**)
+          end
+
+          def structured(**)
+            Runners::Inference.structured(**)
+          end
+
+          def build_event(**)
+            Runners::Metering.build_event(**)
+          end
+
+          def publish_or_spool(event)
+            Runners::Metering.publish_or_spool(event)
+          end
+
+          def flush_spool
+            Runners::Metering.flush_spool
+          end
+
+          def dispatch(**)
+            Runners::Fleet.dispatch(**)
+          end
+
+          def fleet_available?
+            Runners::Fleet.fleet_available?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/helpers/auth.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Helpers
+          module Auth
+            module_function
+
+            def sign_request(payload)
+              return nil unless defined?(Legion::Crypt::JWT)
+
+              Legion::Crypt::JWT.encode(payload: payload, ttl: 60)
+            rescue StandardError
+              nil
+            end
+
+            def validate_token(token)
+              return nil unless defined?(Legion::Crypt::JWT)
+
+              Legion::Crypt::JWT.decode(token: token)
+            rescue StandardError
+              nil
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/helpers/rpc.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Helpers
+          module Rpc
+            module_function
+
+            def generate_correlation_id
+              SecureRandom.uuid
+            end
+
+            def agent_queue_name
+              if defined?(Legion::Transport) && Legion::Transport.respond_to?(:agent_queue_name)
+                return Legion::Transport.agent_queue_name
+              end
+
+              nil
+            end
+
+            def build_reply_headers(correlation_id:)
+              {
+                reply_to: agent_queue_name,
+                correlation_id: correlation_id
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/fleet.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Runners
+          module Fleet
+            DEFAULT_TIMEOUT = 30
+
+            module_function
+
+            def dispatch(model:, messages:, intent: nil, timeout: nil)
+              return error_result('fleet_unavailable') unless fleet_available?
+
+              token = Helpers::Auth.sign_request({ model: model, intent: intent })
+              return error_result('fleet_auth_failed') if token.nil? && require_auth?
+
+              correlation_id = Helpers::Rpc.generate_correlation_id
+              publish_request(model: model, messages: messages, intent: intent,
+                              correlation_id: correlation_id, signed_token: token)
+
+              wait_for_response(correlation_id, timeout: resolve_timeout(timeout))
+            end
+
+            def fleet_available?
+              transport_ready? && fleet_enabled?
+            end
+
+            def transport_ready?
+              !!(defined?(Legion::Transport) &&
+                 Legion::Transport.respond_to?(:connected?) &&
+                 Legion::Transport.connected?)
+            end
+
+            def fleet_enabled?
+              return true unless defined?(Legion::Settings)
+
+              settings = Legion::Settings[:llm] rescue nil # rubocop:disable Style/RescueModifier
+              return true unless settings.is_a?(Hash)
+
+              routing = settings[:routing]
+              return true unless routing.is_a?(Hash)
+
+              routing.fetch(:use_fleet, true)
+            end
+
+            def require_auth?
+              return false unless defined?(Legion::Settings)
+
+              settings = Legion::Settings[:llm] rescue nil # rubocop:disable Style/RescueModifier
+              return false unless settings.is_a?(Hash)
+
+              fleet = settings.dig(:routing, :fleet)
+              return false unless fleet.is_a?(Hash)
+
+              fleet.fetch(:require_auth, false)
+            end
+
+            def resolve_timeout(override)
+              return override if override
+
+              return DEFAULT_TIMEOUT unless defined?(Legion::Settings)
+
+              settings = Legion::Settings[:llm] rescue nil # rubocop:disable Style/RescueModifier
+              return DEFAULT_TIMEOUT unless settings.is_a?(Hash)
+
+              settings.dig(:routing, :fleet, :timeout_seconds) || DEFAULT_TIMEOUT
+            end
+
+            def publish_request(model:, messages:, intent:, correlation_id:, signed_token:)
+              reply_to = Helpers::Rpc.agent_queue_name
+              Transport::Messages::InferenceRequest.new(
+                model: model, messages: messages, intent: intent,
+                reply_to: reply_to, correlation_id: correlation_id,
+                signed_token: signed_token
+              ).publish
+            end
+
+            def wait_for_response(correlation_id, timeout:)
+              { success: false, error: 'fleet_timeout', correlation_id: correlation_id, timeout: timeout }
+            end
+
+            def error_result(reason)
+              { success: false, error: reason }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/fleet_handler.rb

@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Runners
+          module FleetHandler
+            module_function
+
+            def handle_fleet_request(payload)
+              token = payload[:signed_token]
+              return { success: false, error: 'invalid_token' } if require_auth? && !valid_token?(token)
+
+              response = call_local_llm(payload)
+              build_response(payload[:correlation_id], response)
+            end
+
+            def require_auth?
+              Fleet.require_auth?
+            end
+
+            def valid_token?(token)
+              return true if token.nil? && !require_auth?
+
+              !Helpers::Auth.validate_token(token).nil?
+            end
+
+            def call_local_llm(payload)
+              return { error: 'llm_not_available' } unless defined?(Legion::LLM)
+
+              Legion::LLM.chat(
+                model: payload[:model],
+                message: payload.dig(:messages, 0, :content)
+              )
+            end
+
+            def build_response(correlation_id, response)
+              {
+                correlation_id: correlation_id,
+                response: response,
+                input_tokens: extract_token(response, :input_tokens),
+                output_tokens: extract_token(response, :output_tokens),
+                thinking_tokens: extract_token(response, :thinking_tokens),
+                provider: extract_field(response, :provider),
+                model_id: extract_field(response, :model)
+              }
+            end
+
+            def extract_token(response, field)
+              return 0 unless response.respond_to?(field)
+
+              response.public_send(field).to_i
+            end
+
+            def extract_field(response, field)
+              return nil unless response.respond_to?(field)
+
+              response.public_send(field)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/inference.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Runners
+          module Inference
+            module_function
+
+            def chat(model: nil, provider: nil, **opts)
+              start_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+              response = dispatch_chat(model: model, provider: provider, **opts)
+              elapsed_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start_ms
+              meter_response(response, request_type: 'chat', provider: provider,
+                                       model_id: model, latency_ms: elapsed_ms, **opts.slice(:tier, :intent))
+              response
+            end
+
+            def embed(text: nil, model: nil, provider: nil, **)
+              start_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+              response = call_llm(:embed, text: text, model: model, provider: provider, **)
+              elapsed_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start_ms
+              meter_response(response, request_type: 'embed', provider: provider, model_id: model,
+                                       latency_ms: elapsed_ms)
+              response
+            end
+
+            def structured(messages: nil, schema: nil, model: nil, provider: nil, **)
+              start_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond)
+              response = call_llm(:structured, messages: messages, schema: schema, model: model,
+                                               provider: provider, **)
+              elapsed_ms = Process.clock_gettime(Process::CLOCK_MONOTONIC, :millisecond) - start_ms
+              meter_response(response, request_type: 'structured', provider: provider, model_id: model,
+                                       latency_ms: elapsed_ms)
+              response
+            end
+
+            def dispatch_chat(message: nil, model: nil, provider: nil, **opts)
+              tier = opts[:tier]
+              intent = opts[:intent]
+              if tier == 'fleet' && fleet_available?
+                Fleet.dispatch(model: model, messages: [{ role: 'user', content: message }],
+                               intent: intent)
+              else
+                call_llm(:chat, message: message, model: model, provider: provider, **opts)
+              end
+            end
+
+            def fleet_available?
+              defined?(Legion::Extensions::LLM::Gateway::Runners::Fleet) &&
+                Fleet.respond_to?(:fleet_available?) && Fleet.fleet_available?
+            end
+
+            def call_llm(method_name, **)
+              return { error: 'llm_not_available' } unless defined?(Legion::LLM)
+
+              direct = :"#{method_name}_direct"
+              if Legion::LLM.respond_to?(direct)
+                Legion::LLM.public_send(direct, **)
+              else
+                Legion::LLM.public_send(method_name, **)
+              end
+            end
+
+            def meter_response(response, **)
+              Metering.publish_or_spool(build_meter_event(response, **))
+            end
+
+            def build_meter_event(response, **opts)
+              Metering.build_event(**base_meter_fields(response, opts), **token_fields(response))
+            end
+
+            def base_meter_fields(response, opts)
+              {
+                request_type: opts[:request_type],
+                provider: extract_provider(response, opts[:provider]),
+                model_id: extract_model(response, opts[:model_id]),
+                latency_ms: opts[:latency_ms],
+                tier: opts[:tier],
+                routing_reason: opts[:intent]
+              }
+            end
+
+            def token_fields(response)
+              {
+                input_tokens: extract_tokens(response, :input_tokens),
+                output_tokens: extract_tokens(response, :output_tokens),
+                thinking_tokens: extract_tokens(response, :thinking_tokens)
+              }
+            end
+
+            def extract_tokens(response, field)
+              return 0 unless response.respond_to?(field)
+
+              response.public_send(field).to_i
+            end
+
+            def extract_provider(response, fallback)
+              return response.provider if response.respond_to?(:provider)
+
+              fallback
+            end
+
+            def extract_model(response, fallback)
+              return response.model if response.respond_to?(:model)
+
+              fallback
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/metering.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Runners
+          module Metering
+            module_function
+
+            def build_event(**opts)
+              identity_fields(opts).merge(token_fields(opts)).merge(timing_and_context(opts))
+            end
+
+            def identity_fields(opts)
+              {
+                node_id: opts[:node_id],
+                worker_id: opts[:worker_id],
+                agent_id: opts[:agent_id],
+                request_type: opts[:request_type],
+                tier: opts[:tier],
+                provider: opts[:provider],
+                model_id: opts[:model_id]
+              }
+            end
+
+            def token_fields(opts)
+              input    = opts.fetch(:input_tokens, 0)
+              output   = opts.fetch(:output_tokens, 0)
+              thinking = opts.fetch(:thinking_tokens, 0)
+              { input_tokens: input, output_tokens: output, thinking_tokens: thinking,
+                total_tokens: input + output + thinking }
+            end
+
+            def timing_and_context(opts)
+              {
+                latency_ms: opts.fetch(:latency_ms, 0),
+                wall_clock_ms: opts.fetch(:wall_clock_ms, 0),
+                routing_reason: opts[:routing_reason],
+                recorded_at: Time.now.utc.iso8601
+              }
+            end
+
+            def publish_or_spool(event)
+              if transport_connected?
+                publish_event(event)
+                :published
+              elsif spool_available?
+                spool_event(event)
+                :spooled
+              else
+                :dropped
+              end
+            end
+
+            def flush_spool
+              return 0 unless spool_available? && transport_connected?
+
+              spool = Legion::Data::Spool.for(Legion::Extensions::LLM::Gateway)
+              spool.flush(:metering) { |event| publish_event(event) }
+            end
+
+            def transport_connected?
+              !!(defined?(Legion::Transport) &&
+                Legion::Transport.respond_to?(:connected?) &&
+                Legion::Transport.connected?)
+            end
+
+            def spool_available?
+              !!defined?(Legion::Data::Spool)
+            end
+
+            def publish_event(event)
+              Legion::Extensions::LLM::Gateway::Transport::Messages::MeteringEvent.new(**event).publish
+            end
+
+            def spool_event(event)
+              spool = Legion::Data::Spool.for(Legion::Extensions::LLM::Gateway)
+              spool.write(:metering, event)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/runners/metering_writer.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Runners
+          module MeteringWriter
+            module_function
+
+            def write_metering_record(payload)
+              return { success: false, error: 'data_not_connected' } unless data_connected?
+
+              record = normalize_record(payload)
+              Legion::Data.connection[:metering_records].insert(record)
+              { success: true, recorded: record }
+            end
+
+            def data_connected?
+              !!(defined?(Legion::Data) &&
+                 Legion::Data.respond_to?(:connection) &&
+                 !Legion::Data.connection.nil?)
+            end
+
+            def normalize_record(payload)
+              identity_fields(payload).merge(metric_fields(payload))
+            end
+
+            def identity_fields(payload)
+              {
+                worker_id: payload[:worker_id],
+                task_id: payload[:task_id],
+                provider: payload[:provider],
+                model_id: payload[:model_id],
+                routing_reason: payload[:routing_reason],
+                recorded_at: payload[:recorded_at] || Time.now.utc
+              }
+            end
+
+            def metric_fields(payload)
+              {
+                input_tokens: payload[:input_tokens].to_i,
+                output_tokens: payload[:output_tokens].to_i,
+                thinking_tokens: payload[:thinking_tokens].to_i,
+                total_tokens: payload[:total_tokens].to_i,
+                latency_ms: payload[:latency_ms].to_i,
+                wall_clock_ms: payload[:wall_clock_ms].to_i
+              }
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/exchanges/inference.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Exchanges
+            class Inference < Legion::Transport::Exchange
+              def exchange_name
+                'llm.inference'
+              end
+
+              def exchange_type
+                :direct
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/exchanges/metering.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Exchanges
+            class Metering < Legion::Transport::Exchange
+              def exchange_name
+                'llm.metering'
+              end
+
+              def exchange_type
+                :topic
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/messages/inference_request.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Messages
+            class InferenceRequest < Legion::Transport::Message
+              def routing_key
+                'inference.request'
+              end
+
+              def type
+                'inference_request'
+              end
+
+              def encrypt?
+                false
+              end
+
+              def validate
+                raise 'model is required' unless @options[:model]
+                raise 'reply_to is required' unless @options[:reply_to]
+                raise 'correlation_id is required' unless @options[:correlation_id]
+
+                @valid = true
+              end
+
+              def message
+                {
+                  model: @options[:model],
+                  messages: @options[:messages] || [],
+                  intent: @options[:intent],
+                  reply_to: @options[:reply_to],
+                  correlation_id: @options[:correlation_id],
+                  signed_token: @options[:signed_token],
+                  provider: @options[:provider],
+                  tier: @options[:tier]
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/messages/inference_response.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Messages
+            class InferenceResponse < Legion::Transport::Message
+              def routing_key
+                'inference.response'
+              end
+
+              def type
+                'inference_response'
+              end
+
+              def encrypt?
+                false
+              end
+
+              def validate
+                raise 'correlation_id is required' unless @options[:correlation_id]
+
+                @valid = true
+              end
+
+              def message
+                token_fields.merge(
+                  correlation_id: @options[:correlation_id],
+                  response: @options[:response],
+                  provider: @options[:provider],
+                  model_id: @options[:model_id],
+                  error: @options[:error]
+                )
+              end
+
+              private
+
+              def token_fields
+                {
+                  input_tokens: @options[:input_tokens] || 0,
+                  output_tokens: @options[:output_tokens] || 0,
+                  thinking_tokens: @options[:thinking_tokens] || 0,
+                  latency_ms: @options[:latency_ms] || 0
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/messages/metering_event.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Messages
+            class MeteringEvent < Legion::Transport::Message
+              def routing_key
+                "metering.#{@options[:request_type] || 'unknown'}"
+              end
+
+              def type
+                'metering_event'
+              end
+
+              def encrypt?
+                false
+              end
+
+              def validate
+                raise 'request_type is required' unless @options[:request_type]
+                raise 'provider is required' unless @options[:provider]
+
+                @valid = true
+              end
+
+              def message
+                identity_fields.merge(token_fields).merge(timing_fields).merge(context_fields)
+              end
+
+              private
+
+              def identity_fields
+                {
+                  node_id: @options[:node_id],
+                  worker_id: @options[:worker_id],
+                  agent_id: @options[:agent_id],
+                  request_type: @options[:request_type],
+                  tier: @options[:tier],
+                  provider: @options[:provider],
+                  model_id: @options[:model_id]
+                }
+              end
+
+              def context_fields
+                {
+                  routing_reason: @options[:routing_reason],
+                  recorded_at: @options[:recorded_at] || Time.now.utc.iso8601
+                }
+              end
+
+              def token_fields
+                {
+                  input_tokens: @options[:input_tokens] || 0,
+                  output_tokens: @options[:output_tokens] || 0,
+                  thinking_tokens: @options[:thinking_tokens] || 0,
+                  total_tokens: @options[:total_tokens] || 0
+                }
+              end
+
+              def timing_fields
+                {
+                  latency_ms: @options[:latency_ms] || 0,
+                  wall_clock_ms: @options[:wall_clock_ms] || 0
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/queues/inference_process.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Queues
+            class InferenceProcess < Legion::Transport::Queue
+              def queue_name
+                'llm.inference.process'
+              end
+
+              def queue_options
+                {
+                  durable: true,
+                  auto_delete: false
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/transport/queues/metering_write.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        module Transport
+          module Queues
+            class MeteringWrite < Legion::Transport::Queue
+              def queue_name
+                'llm.metering.write'
+              end
+
+              def queue_options
+                {
+                  durable: true,
+                  auto_delete: false
+                }
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway/version.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        VERSION = '0.2.0'
+      end
+    end
+  end
+end

data/lib/legion/extensions/llm/gateway.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'legion/extensions/llm/gateway/version'
+
+module Legion
+  module Extensions
+    module LLM
+      module Gateway
+        extend Legion::Extensions::Core if Legion::Extensions.const_defined?(:Core)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,128 @@
+--- !ruby/object:Gem::Specification
+name: lex-llm-gateway
+version: !ruby/object:Gem::Version
+  version: 0.2.0
+platform: ruby
+authors:
+- Esity
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rubocop-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'LLM inference gateway: metering over RabbitMQ, fleet RPC dispatch, local
+  disk spool'
+email:
+- matthewdiverson@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- Gemfile
+- LICENSE
+- README.md
+- lex-llm-gateway.gemspec
+- lib/legion/extensions/llm/gateway.rb
+- lib/legion/extensions/llm/gateway/actors/inference_worker.rb
+- lib/legion/extensions/llm/gateway/actors/metering_writer.rb
+- lib/legion/extensions/llm/gateway/actors/spool_flush.rb
+- lib/legion/extensions/llm/gateway/client.rb
+- lib/legion/extensions/llm/gateway/helpers/auth.rb
+- lib/legion/extensions/llm/gateway/helpers/rpc.rb
+- lib/legion/extensions/llm/gateway/runners/fleet.rb
+- lib/legion/extensions/llm/gateway/runners/fleet_handler.rb
+- lib/legion/extensions/llm/gateway/runners/inference.rb
+- lib/legion/extensions/llm/gateway/runners/metering.rb
+- lib/legion/extensions/llm/gateway/runners/metering_writer.rb
+- lib/legion/extensions/llm/gateway/transport/exchanges/inference.rb
+- lib/legion/extensions/llm/gateway/transport/exchanges/metering.rb
+- lib/legion/extensions/llm/gateway/transport/messages/inference_request.rb
+- lib/legion/extensions/llm/gateway/transport/messages/inference_response.rb
+- lib/legion/extensions/llm/gateway/transport/messages/metering_event.rb
+- lib/legion/extensions/llm/gateway/transport/queues/inference_process.rb
+- lib/legion/extensions/llm/gateway/transport/queues/metering_write.rb
+- lib/legion/extensions/llm/gateway/version.rb
+homepage: https://github.com/LegionIO/lex-llm-gateway
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/LegionIO/lex-llm-gateway
+  source_code_uri: https://github.com/LegionIO/lex-llm-gateway
+  changelog_uri: https://github.com/LegionIO/lex-llm-gateway/blob/main/CHANGELOG.md
+  documentation_uri: https://github.com/LegionIO/lex-llm-gateway
+  bug_tracker_uri: https://github.com/LegionIO/lex-llm-gateway/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.6.9
+specification_version: 4
+summary: Legion::Extensions::LLM::Gateway
+test_files: []