@mcoda/mswarm 0.1.132 → 0.1.134

package/dist/runtime.js

@@ -202,6 +202,7 @@ function resolvePersistentNodePath(nodePath) {
 const DEFAULT_EXPOSE_ALL_MODELS = true;
 const LOCAL_OPENAI_COMPATIBLE_ADAPTERS = new Set([
     "openai-compatible-local",
+    "strata-local",
     "vllm-local",
     "llama-cpp-local",
     "llamacpp-local"
@@ -244,6 +245,7 @@ const DEFAULT_VIDEO_GENERATIVE_OUTPUT_BYTES = 256 * 1024 * 1024;
 const GENERATIVE_RESPONSE_JSON_OVERHEAD_BYTES = 64 * 1024;
 const LOCAL_RUNNER_KIND_ALIASES = {
     vllm: "vllm",
+    strata: "strata",
     "llama-cpp": "llama-cpp",
     "llama.cpp": "llama-cpp",
     llamacpp: "llama-cpp",
@@ -526,6 +528,8 @@ function defaultLocalRunnerKindForAdapter(adapter) {
     const lookupKey = normalizeLookupKey(normalized);
     if (lookupKey === "vllm-local")
         return "vllm";
+    if (lookupKey === "strata-local")
+        return "strata";
     if (lookupKey === "llama-cpp-local" || lookupKey === "llamacpp-local")
         return "llama-cpp";
     return undefined;
@@ -2082,6 +2086,9 @@ export function mapMcodaAgentToSelfHostedModel(agent, config) {
             : inferredInputModalities);
     const outputModalities = localRunner?.outputModalities ?? inferGenerativeModalities(operations, "output");
     const publicModelId = localRunner?.publicModelId;
+    const legacyConfig = isRecord(agent.config) ? agent.config : {};
+    const legacyNativeBase = normalizeLocalRunnerKind(legacyConfig.runnerKind ?? legacyConfig.runner_kind) === "llama-cpp"
+        ? mcodaAgentLocalRunnerBaseUrl(agent) : undefined;
     return {
         name: slug,
         provider: "mcoda",
@@ -2092,9 +2099,9 @@ export function mapMcodaAgentToSelfHostedModel(agent, config) {
         model_id: publicModelId || model,
         public_model_id: publicModelId,
         upstream_model: model,
-        base_url: localRunner?.baseUrl,
-        runner_kind: localRunner?.generativeRunnerKind ?? localRunner?.runnerKind,
-        auth_mode: localRunner?.authMode,
+        base_url: localRunner?.baseUrl ?? legacyNativeBase ?? undefined,
+        runner_kind: localRunner?.generativeRunnerKind ?? localRunner?.runnerKind ?? (legacyNativeBase ? "llama-cpp" : undefined),
+        auth_mode: localRunner?.authMode ?? (legacyNativeBase ? normalizeLocalRunnerAuthMode(legacyConfig.authMode) : undefined),
         response_format_strategy: localRunner?.responseFormatStrategy,
         health_path: localRunner?.healthPath,
         models_path: localRunner?.modelsPath,
@@ -4395,6 +4402,13 @@ export class MswarmSelfHostedNodeClient {
             body: JSON.stringify(payload)
         }, this.timeoutMs);
     }
+    async reportLoad(runtimeToken, payload) {
+        return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/load`, {
+            method: "POST",
+            headers: { "content-type": "application/json", authorization: `Bearer ${runtimeToken}` },
+            body: JSON.stringify(payload)
+        }, Math.min(this.timeoutMs, 5000));
+    }
     async uninstall(runtimeToken, payload) {
         return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/uninstall`, {
             method: "POST",
@@ -4459,6 +4473,15 @@ export class MswarmSelfHostedNodeClient {
 export class SelfHostedNodeRuntime {
     constructor(config, deps) {
         this.activeChatJobs = 0;
+        this.activeRequestIds = new Set();
+        this.lastModels = [];
+        this.loadRuntimeToken = null;
+        this.loadReportQueue = Promise.resolve();
+        this.fastLoadUnsupported = false;
+        this.nativeActiveJobs = 0;
+        this.nativeProbeKeys = new Map();
+        this.unavailableModelIds = [];
+        this.nativeModelCapacity = {};
         this.activeAgenticJobs = 0;
         this.activeGenericJobs = 0;
         this.queuedLlmJobs = 0;
@@ -4665,7 +4688,7 @@ export class SelfHostedNodeRuntime {
         const maxConcurrency = Math.max(1, Math.floor(this.config.maxConcurrentJobs || 1), llmMaxConcurrency, this.config.genericJobsEnabled ? genericMaxConcurrency : 1);
         const activeChatJobs = nonNegativeTelemetryInteger(this.activeChatJobs);
         const activeAgenticJobs = nonNegativeTelemetryInteger(this.activeAgenticJobs);
-        const activeLlmJobs = activeChatJobs + activeAgenticJobs;
+        const activeLlmJobs = Math.max(activeChatJobs + activeAgenticJobs, this.nativeActiveJobs);
         const activeGenericJobs = nonNegativeTelemetryInteger(this.activeGenericJobs);
         const queuedLlmJobs = nonNegativeTelemetryInteger(this.queuedLlmJobs);
         const queuedGenericJobs = nonNegativeTelemetryInteger(this.queuedGenericJobs);
@@ -5660,6 +5683,18 @@ export class SelfHostedNodeRuntime {
         }
     }
     async executeJob(job, options = {}) {
+        this.activeRequestIds.add(job.request_id);
+        try {
+            return await this.executeTrackedJob(job, options);
+        }
+        finally {
+            this.activeRequestIds.delete(job.request_id);
+            // A product often starts its next stage immediately after this result.
+            // Publish the newly free native slot before the gateway delivers it.
+            await this.reportLoad().catch(() => undefined);
+        }
+    }
+    async executeTrackedJob(job, options) {
         const startedAt = Date.now();
         // Resolved once, before any work: the slot is taken from the moment the job
         // is accepted, and a class settled later would leave the count attributed to
@@ -6027,6 +6062,111 @@ export class SelfHostedNodeRuntime {
             return result;
         }
     }
+    async refreshNativeLoad() {
+        const runners = new Map();
+        for (const model of this.lastModels) {
+            const isStrata = model.runner_kind === "strata" || model.adapter === "strata-local";
+            if (model.exposed === false || !model.base_url ||
+                (!isStrata && model.runner_kind !== "llama-cpp" && model.adapter !== "llama-cpp-local"))
+                continue;
+            try {
+                const url = new URL(model.base_url);
+                if (!["127.0.0.1", "localhost", "[::1]"].includes(url.hostname))
+                    continue;
+                const endpoint = new URL(isStrata ? "/v1/status" : "/slots", url).toString();
+                const models = runners.get(endpoint) || [];
+                models.push(model);
+                runners.set(endpoint, models);
+            }
+            catch { /* A malformed endpoint is handled by normal discovery health. */ }
+        }
+        let active = 0;
+        const modelCapacity = {};
+        const unavailable = [];
+        await Promise.all([...runners].map(async ([endpoint, models]) => {
+            try {
+                const authenticated = models.find((model) => model.auth_mode === "bearer" && model.source_agent_id);
+                let apiKey;
+                if (authenticated?.source_agent_id) {
+                    let pending = this.nativeProbeKeys.get(authenticated.source_agent_id);
+                    if (!pending) {
+                        pending = this.mcodaAgentAuthResolver({ id: authenticated.source_agent_id,
+                            slug: authenticated.source_agent_slug || authenticated.name }).catch(() => undefined);
+                        this.nativeProbeKeys.set(authenticated.source_agent_id, pending);
+                    }
+                    apiKey = await pending;
+                }
+                const response = await this.fetchImpl(endpoint, { signal: AbortSignal.timeout(1500),
+                    ...(apiKey ? { headers: { authorization: `Bearer ${apiKey}` } } : {}) });
+                const isStrata = new URL(endpoint).pathname === "/v1/status";
+                // llama.cpp slot introspection is optional. Strata status also proves the model is loaded.
+                if (!isStrata && [401, 403, 404].includes(response.status))
+                    return;
+                if (!response.ok)
+                    throw new Error("runner_load_unavailable");
+                const slots = await response.json();
+                if (isStrata) {
+                    const concurrency = isRecord(slots) && isRecord(slots.concurrency) ? slots.concurrency.serving : undefined;
+                    const inFlight = isRecord(slots) && isRecord(slots.activity) ? slots.activity.in_flight : undefined;
+                    if (!isRecord(slots) || slots.loaded !== true ||
+                        typeof concurrency !== "number" || !Number.isSafeInteger(concurrency) || concurrency < 1 ||
+                        typeof inFlight !== "number" || !Number.isSafeInteger(inFlight) || inFlight < 0) {
+                        throw new Error("runner_load_unavailable");
+                    }
+                    const processing = Math.min(concurrency, inFlight);
+                    active += processing;
+                    const capacity = { max_concurrency: concurrency, active_jobs: processing,
+                        queued_jobs: Math.max(0, inFlight - concurrency), free_slots: Math.max(0, concurrency - inFlight),
+                        pool: `runner:${createHash("sha256").update(endpoint).digest("hex").slice(0, 16)}` };
+                    for (const model of models)
+                        modelCapacity[model.model_id || model.model || model.name] = capacity;
+                    return;
+                }
+                if (!Array.isArray(slots))
+                    return;
+                const processing = slots.filter((slot) => isRecord(slot) && slot.is_processing === true).length;
+                active += processing;
+                if (slots.length > 0) {
+                    const capacity = { max_concurrency: slots.length, active_jobs: processing, queued_jobs: 0,
+                        free_slots: slots.length - processing, pool: `runner:${createHash("sha256").update(endpoint).digest("hex").slice(0, 16)}` };
+                    for (const model of models)
+                        modelCapacity[model.model_id || model.model || model.name] = capacity;
+                }
+            }
+            catch {
+                unavailable.push(...models.map((model) => model.model_id || model.model || model.name));
+            }
+        }));
+        this.nativeActiveJobs = active;
+        this.nativeModelCapacity = modelCapacity;
+        this.unavailableModelIds = [...new Set(unavailable)];
+    }
+    reportLoad() {
+        // Serialize probes and writes so an older busy snapshot cannot overwrite
+        // the completion snapshot when the periodic reporter is already running.
+        const pending = this.loadReportQueue.then(() => this.reportLoadSnapshot());
+        this.loadReportQueue = pending.catch(() => undefined);
+        return pending;
+    }
+    async reportLoadSnapshot() {
+        if (this.config.loadReportingEnabled === false || this.fastLoadUnsupported || !this.loadRuntimeToken)
+            return;
+        await this.refreshNativeLoad();
+        const telemetry = this.buildLoadTelemetry({ models: this.lastModels });
+        try {
+            await this.gateway.reportLoad(this.loadRuntimeToken, {
+                node_id: this.config.nodeId,
+                capacity: { ...telemetry, active_request_ids: [...this.activeRequestIds],
+                    unavailable_model_ids: this.unavailableModelIds, model_capacity: this.nativeModelCapacity }
+            });
+        }
+        catch (error) {
+            if (isHttpStatusError(error, 404))
+                this.fastLoadUnsupported = true;
+            else
+                throw error;
+        }
+    }
     async runOnce() {
         const enrollment = await this.ensureEnrolled();
         let status = "online";
@@ -6057,6 +6197,10 @@ export class SelfHostedNodeRuntime {
             this.lifecyclePollingDisabled = true;
         }
         const discoveryLatencyMs = Date.now() - startedAt;
+        if (models.length || status === "online")
+            this.lastModels = models;
+        this.nativeProbeKeys.clear();
+        await this.refreshNativeLoad();
         const capabilityPayload = await this.buildCapabilityHeartbeatPayload(enrollment.runtimeToken);
         const loadTelemetry = this.buildLoadTelemetry({
             models,
@@ -6077,6 +6221,9 @@ export class SelfHostedNodeRuntime {
                 max_concurrent_llm_jobs: loadTelemetry.max_concurrent_llm_jobs,
                 max_concurrent_generic_jobs: loadTelemetry.max_concurrent_generic_jobs,
                 active_jobs: loadTelemetry.active_jobs,
+                active_request_ids: [...this.activeRequestIds],
+                unavailable_model_ids: this.unavailableModelIds,
+                model_capacity: this.nativeModelCapacity,
                 queued_jobs: loadTelemetry.queued_jobs,
                 free_slots: loadTelemetry.free_slots,
                 drain_mode: loadTelemetry.drain_mode,
@@ -6148,6 +6295,7 @@ export class SelfHostedNodeRuntime {
             ...(loadTelemetry.hardware_pressure ? { hardware_pressure: loadTelemetry.hardware_pressure } : {})
         };
         const heartbeatResponse = await this.gateway.heartbeat(enrollment.runtimeToken, heartbeatPayload);
+        this.loadRuntimeToken = enrollment.runtimeToken;
         return {
             enrolled: enrollment.enrolled,
             status,
@@ -6402,6 +6550,8 @@ export class SelfHostedNodeRuntime {
     startDaemon() {
         let stopped = false;
         let timer = null;
+        let loadTimer = null;
+        let loadReporting = false;
         let pollTimer = null;
         let pollTimerDueAt = 0;
         let polling = false;
@@ -6425,6 +6575,10 @@ export class SelfHostedNodeRuntime {
         };
         const halt = () => {
             stopped = true;
+            if (loadTimer) {
+                clearInterval(loadTimer);
+                loadTimer = null;
+            }
             if (timer) {
                 clearTimeout(timer);
                 timer = null;
@@ -6574,6 +6728,13 @@ export class SelfHostedNodeRuntime {
                     .finally(schedule);
             }, revoked ? REVOKED_RECHECK_INTERVAL_MS : this.config.heartbeatIntervalSeconds * 1000);
         };
+        loadTimer = setInterval(() => {
+            if (stopped || revoked || loadReporting)
+                return;
+            loadReporting = true;
+            void this.reportLoad().catch((error) => { noteRevoked(error); }).finally(() => { loadReporting = false; });
+        }, 2000);
+        loadTimer.unref?.();
         void this.runOnce()
             .then(noteAlive)
             .catch((error) => {