@mcoda/mswarm 0.1.80 → 0.1.85

package/dist/runtime.js

@@ -3,8 +3,9 @@ import { dirname, isAbsolute, join, relative, resolve } from "node:path";
 import { cpus, freemem, hostname, homedir, loadavg, platform, totalmem, userInfo } from "node:os";
 import { spawn } from "node:child_process";
 import { createHash, createHmac, randomUUID } from "node:crypto";
+import { GlobalRepository } from "@mcoda/db";
 import { MswarmCodaliExecutor } from "./codali-executor.js";
-import { MSWARM_CAPABILITY_SCHEMA_VERSION, assertMswarmSafeRelativePath, validateMswarmArchiveEntry, buildMswarmCapabilityNames, buildMswarmPrivateCapabilityCatalogEntry, buildMswarmLocalArtifactUri, buildMswarmSandboxProfile, defaultMswarmArtifactAccessPolicy, defaultMswarmArtifactRetentionPolicy, projectMswarmPublicCapabilities, validateMswarmGenericJobRequest } from "@mcoda/shared";
+import { MSWARM_CAPABILITY_SCHEMA_VERSION, CryptoHelper, assertMswarmSafeRelativePath, validateMswarmArchiveEntry, buildMswarmCapabilityNames, buildMswarmPrivateCapabilityCatalogEntry, buildMswarmLocalArtifactUri, buildMswarmSandboxProfile, defaultMswarmArtifactAccessPolicy, defaultMswarmArtifactRetentionPolicy, projectMswarmPublicCapabilities, validateMswarmGenericJobRequest } from "@mcoda/shared";
 const DEFAULT_GATEWAY_BASE_URL = "http://127.0.0.1:8080";
 const DEFAULT_SETUP_GATEWAY_BASE_URL = "https://api.mswarm.org";
 const DEFAULT_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
@@ -26,7 +27,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
 const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
 const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
 const DEFAULT_JOB_POLL_WAIT_MS = 25000;
+const DEFAULT_SELF_HOSTED_JOBS_POLL_PATH = "/v1/swarm/self-hosted/node/jobs/poll";
+const DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/start";
+const DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/events";
+const DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/result";
 const DEFAULT_STREAM_EVENT_BATCH_SIZE = 8;
+const SELF_HOSTED_PROTOCOL_MISMATCH_CODE = "self_hosted_protocol_mismatch";
 const OWNER_LOCAL_TEST_ECHO_JOB_TYPE = "tenant.test-echo";
 const TEST_ECHO_RUNNER_ID = "test.echo";
 const RENDER_BLENDER_JOB_TYPE = "render.blender";
@@ -402,6 +408,37 @@ function parseCliOptions(argv) {
 function trimTrailingSlash(value) {
     return value.replace(/\/+$/g, "");
 }
+function lifecyclePath(value, fallback) {
+    return optionalText(value) || fallback;
+}
+function resolveLifecycleUrl(gatewayBaseUrl, pathOrUrl) {
+    try {
+        return new URL(pathOrUrl).toString();
+    }
+    catch {
+        return new URL(pathOrUrl.startsWith("/") ? pathOrUrl : `/${pathOrUrl}`, `${trimTrailingSlash(gatewayBaseUrl)}/`).toString();
+    }
+}
+function resolveLifecycleTemplate(template, jobId) {
+    const encoded = encodeURIComponent(jobId);
+    return template
+        .replace(/:jobId\b/g, encoded)
+        .replace(/:job_id\b/g, encoded)
+        .replace(/\{jobId\}/g, encoded)
+        .replace(/\{job_id\}/g, encoded);
+}
+function usageFromTokenCounts(promptTokens, completionTokens) {
+    const prompt = positiveInteger(promptTokens) ?? 0;
+    const completion = positiveInteger(completionTokens) ?? 0;
+    return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion };
+}
+function openAiUsage(response) {
+    const usage = isRecord(response?.usage) ? response.usage : undefined;
+    const prompt = positiveInteger(usage?.prompt_tokens) ?? 0;
+    const completion = positiveInteger(usage?.completion_tokens) ?? 0;
+    const total = positiveInteger(usage?.total_tokens) ?? prompt + completion;
+    return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: total };
+}
 function defaultStatePath() {
     return join(homedir(), ".mswarm", "self-hosted-node", "config.json");
 }
@@ -631,6 +668,18 @@ function isAgentExposed(agentSlug, defaultModel, bestUsage, config) {
     }
     return config.exposeAllModels;
 }
+class HttpStatusError extends Error {
+    constructor(url, status, responseText) {
+        super(`request_failed:${status}:${responseText.slice(0, 200)}`);
+        this.name = "HttpStatusError";
+        this.url = url;
+        this.status = status;
+        this.responseText = responseText;
+    }
+}
+function isHttpStatusError(error, status) {
+    return error instanceof HttpStatusError && (status === undefined || error.status === status);
+}
 async function fetchJson(fetchImpl, url, init, timeoutMs) {
     const controller = new AbortController();
     const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -638,7 +687,7 @@ async function fetchJson(fetchImpl, url, init, timeoutMs) {
         const response = await fetchImpl(url, { ...init, signal: controller.signal });
         if (!response.ok) {
             const text = await response.text().catch(() => "");
-            throw new Error(`request_failed:${response.status}:${text.slice(0, 200)}`);
+            throw new HttpStatusError(url, response.status, text);
         }
         return (await response.json());
     }
@@ -1168,6 +1217,18 @@ export async function readSelfHostedNodeConfig(env = process.env) {
     const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
     return {
         gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
+        jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
+            state.jobs_poll_path ||
+            DEFAULT_SELF_HOSTED_JOBS_POLL_PATH,
+        jobsStartPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_START_PATH_TEMPLATE) ||
+            state.jobs_start_path_template ||
+            DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE,
+        jobsEventsPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE) ||
+            state.jobs_events_path_template ||
+            DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE,
+        jobsResultPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE) ||
+            state.jobs_result_path_template ||
+            DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE,
         nodeId,
         serverName: optionalText(env.MSWARM_SELF_HOSTED_SERVER_NAME) || state.server_name || null,
         relayMode: parseRelayMode(env.MSWARM_SELF_HOSTED_RELAY_MODE || state.relay_mode),
@@ -1608,6 +1669,20 @@ function configText(config, ...keys) {
     }
     return undefined;
 }
+async function defaultMcodaAgentAuthResolver(agent) {
+    const agentId = optionalText(agent.id);
+    if (!agentId) {
+        return undefined;
+    }
+    const repo = await GlobalRepository.create();
+    try {
+        const secret = await repo.getAgentAuthSecret(agentId);
+        return secret?.encryptedSecret ? CryptoHelper.decryptSecret(secret.encryptedSecret) : undefined;
+    }
+    finally {
+        await repo.close();
+    }
+}
 function mcodaAgentDefaultModel(agent) {
     return (optionalText(agent.defaultModel) ||
         optionalText(agent.default_model) ||
@@ -1623,18 +1698,35 @@ function resolveCodaliProviderForAgent(agent) {
     if (["ollama-remote", "ollama-cli", "ollama", "local-model"].includes(adapter || "")) {
         return "ollama-remote";
     }
-    if (adapter === "openai" || adapter === "openai-compatible" || adapter === "openai-cli") {
+    if (adapter === "openai" ||
+        adapter === "openai-api" ||
+        adapter === "openai-compatible" ||
+        adapter === "openai-cli") {
         return "openai-compatible";
     }
     if (adapter === "codex-cli")
         return "codex-cli";
     return adapter || undefined;
 }
+function mcodaAgentRequiresApiKey(agent, mapped) {
+    if (mapped.provider !== "openai-compatible") {
+        return false;
+    }
+    if (mapped.localRunner) {
+        return mapped.authMode === "bearer";
+    }
+    const adapter = optionalText(agent.adapter) || "";
+    if (["openai", "openai-api", "openai-compatible"].includes(adapter)) {
+        return mapped.authMode !== "none";
+    }
+    return mapped.authMode === "bearer";
+}
 function mapMcodaAgentToCodaliAgent(agent, fallbackSlug) {
     const adapter = optionalText(agent.adapter) || "unknown";
     const model = mcodaAgentDefaultModel(agent) || fallbackSlug;
     const config = agent.config ?? null;
     const localRunner = normalizeMcodaLocalRunnerConfig(adapter, config);
+    const configAuthMode = normalizeLocalRunnerAuthMode(config?.authMode);
     const supportsTools = optionalBoolean(agent.supportsTools, agent.supports_tools, localRunner?.supportsTools) === true;
     return {
         slug: optionalText(agent.slug) || fallbackSlug,
@@ -1649,7 +1741,7 @@ function mapMcodaAgentToCodaliAgent(agent, fallbackSlug) {
             : configText(config, "apiKey", "api_key"),
         localRunner,
         runnerKind: localRunner?.runnerKind,
-        authMode: localRunner?.authMode,
+        authMode: localRunner?.authMode ?? configAuthMode,
         dummyBearerToken: localRunner?.dummyBearerToken,
         headers: localRunner?.headers,
         extraBody: localRunner?.extraBody,
@@ -1687,10 +1779,12 @@ const DOCDEX_JOB_ERROR_CODES = new Set([
 ]);
 const PRE_START_JOB_ERROR_CODES = new Set([
     "selected_agent_unavailable",
+    "selected_agent_auth_unavailable",
     "selected_agent_unhealthy",
     "validation_failed",
     "docdex_context_missing",
     "docdex_api_key_missing",
+    SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
 ]);
 class SelfHostedDocdexJobError extends Error {
     constructor(code, message) {
@@ -1706,6 +1800,18 @@ class SelfHostedPreStartJobError extends Error {
         this.code = code;
     }
 }
+class SelfHostedProtocolMismatchError extends Error {
+    constructor(input) {
+        super(`Gateway is missing lifecycle endpoint ${input.endpoint} at ${input.gatewayBaseUrl} ` +
+            `(runtime package version ${input.runtimePackageVersion}). ` +
+            "Run a direct local agent/model instead, for example qwen3.6-llama.cpp.");
+        this.code = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
+        this.name = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
+        this.endpoint = input.endpoint;
+        this.gatewayBaseUrl = input.gatewayBaseUrl;
+        this.runtimePackageVersion = input.runtimePackageVersion;
+    }
+}
 function normalizeDocdexCapabilityMap(value) {
     const record = objectRecord(value);
     if (!record)
@@ -3214,9 +3320,36 @@ export class McodaLocalAgentExecutor {
 export class MswarmSelfHostedNodeClient {
     constructor(input) {
         this.gatewayBaseUrl = trimTrailingSlash(input.gatewayBaseUrl);
+        this.jobsPollPath = lifecyclePath(input.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
+        this.jobsStartPathTemplate = lifecyclePath(input.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
+        this.jobsEventsPathTemplate = lifecyclePath(input.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
+        this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
         this.fetchImpl = input.fetchImpl || fetch;
         this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
     }
+    lifecycleEndpoint(kind) {
+        const path = kind === "poll"
+            ? this.jobsPollPath
+            : kind === "start"
+                ? this.jobsStartPathTemplate
+                : kind === "events"
+                    ? this.jobsEventsPathTemplate
+                    : this.jobsResultPathTemplate;
+        return `POST ${path}`;
+    }
+    lifecycleGatewayBaseUrl() {
+        return this.gatewayBaseUrl;
+    }
+    lifecycleUrl(kind, jobId) {
+        const path = kind === "poll"
+            ? this.jobsPollPath
+            : kind === "start"
+                ? this.jobsStartPathTemplate
+                : kind === "events"
+                    ? this.jobsEventsPathTemplate
+                    : this.jobsResultPathTemplate;
+        return resolveLifecycleUrl(this.gatewayBaseUrl, jobId ? resolveLifecycleTemplate(path, jobId) : path);
+    }
     async enroll(nodeId, enrollmentToken) {
         return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/enroll`, {
             method: "POST",
@@ -3268,7 +3401,7 @@ export class MswarmSelfHostedNodeClient {
         }, this.timeoutMs);
     }
     async pollJob(runtimeToken, payload) {
-        return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/poll`, {
+        return fetchJson(this.fetchImpl, this.lifecycleUrl("poll"), {
             method: "POST",
             headers: {
                 "content-type": "application/json",
@@ -3278,7 +3411,7 @@ export class MswarmSelfHostedNodeClient {
         }, Math.max(this.timeoutMs, (payload.wait_ms || 0) + 5000));
     }
     async postJobResult(runtimeToken, jobId, payload) {
-        return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/result`, {
+        return fetchJson(this.fetchImpl, this.lifecycleUrl("result", jobId), {
             method: "POST",
             headers: {
                 "content-type": "application/json",
@@ -3288,7 +3421,7 @@ export class MswarmSelfHostedNodeClient {
         }, this.timeoutMs);
     }
     async postJobStart(runtimeToken, jobId, payload) {
-        return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/start`, {
+        return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
             method: "POST",
             headers: {
                 "content-type": "application/json",
@@ -3298,7 +3431,7 @@ export class MswarmSelfHostedNodeClient {
         }, this.timeoutMs);
     }
     async postJobEvents(runtimeToken, jobId, payload) {
-        return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/events`, {
+        return fetchJson(this.fetchImpl, this.lifecycleUrl("events", jobId), {
             method: "POST",
             headers: {
                 "content-type": "application/json",
@@ -3316,11 +3449,16 @@ export class SelfHostedNodeRuntime {
         this.queuedGenericJobs = 0;
         this.latencySamplesMs = [];
         this.recentFailures = [];
+        this.lifecyclePollingDisabled = false;
         this.config = config;
         this.gateway =
             deps?.gateway ||
                 new MswarmSelfHostedNodeClient({
                     gatewayBaseUrl: config.gatewayBaseUrl,
+                    jobsPollPath: config.jobsPollPath,
+                    jobsStartPathTemplate: config.jobsStartPathTemplate,
+                    jobsEventsPathTemplate: config.jobsEventsPathTemplate,
+                    jobsResultPathTemplate: config.jobsResultPathTemplate,
                     fetchImpl: deps?.fetchImpl,
                     timeoutMs: config.requestTimeoutMs
                 });
@@ -3331,6 +3469,7 @@ export class SelfHostedNodeRuntime {
                     args: config.mcodaListArgs,
                     timeoutMs: config.requestTimeoutMs
                 });
+        this.mcodaAgentAuthResolver = deps?.mcodaAgentAuthResolver || defaultMcodaAgentAuthResolver;
         this.mcodaExecutor =
             deps?.mcodaExecutor ||
                 new McodaLocalAgentExecutor({
@@ -3395,6 +3534,53 @@ export class SelfHostedNodeRuntime {
             this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
         }
     }
+    lifecycleProtocolMismatch(kind, error) {
+        if (!isHttpStatusError(error, 404)) {
+            return undefined;
+        }
+        return new SelfHostedProtocolMismatchError({
+            endpoint: this.gateway.lifecycleEndpoint(kind),
+            gatewayBaseUrl: this.gateway.lifecycleGatewayBaseUrl(),
+            runtimePackageVersion: this.config.nodeVersion
+        });
+    }
+    isLifecycleProtocolDegradedState(state) {
+        return state?.lifecycle_health_reason === SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
+    }
+    async readLifecycleState() {
+        try {
+            return await readSelfHostedNodeState(this.config.statePath);
+        }
+        catch {
+            return {};
+        }
+    }
+    async markLifecycleProtocolDegraded(error) {
+        this.lifecyclePollingDisabled = true;
+        this.recentFailures.unshift({
+            execution_class: "agentic",
+            code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
+            at: new Date().toISOString()
+        });
+        this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
+        const state = await this.readLifecycleState();
+        await writeSelfHostedNodeState(this.config.statePath, {
+            ...state,
+            lifecycle_health_status: "degraded",
+            lifecycle_health_reason: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
+            lifecycle_health_message: error.message,
+            lifecycle_health_updated_at: new Date().toISOString()
+        });
+    }
+    jobResultPayload(result) {
+        return {
+            ...result,
+            node_id: this.config.nodeId,
+            ...(result.status === "success"
+                ? { usage: result.usage || openAiUsage(result.openai_response) }
+                : {})
+        };
+    }
     averageLatencyMs(fallback = null) {
         if (this.latencySamplesMs.length === 0) {
             return fallback;
@@ -3488,6 +3674,12 @@ export class SelfHostedNodeRuntime {
             throw new Error("Bootstrap response did not include node_id and runtime_token");
         }
         const heartbeatInterval = bootstrap.heartbeat_interval_seconds || setupConfig.heartbeatIntervalSeconds;
+        const relay = bootstrap.relay || {};
+        const relayGatewayBaseUrl = optionalText(relay.gateway_base_url) || setupConfig.gatewayBaseUrl;
+        const jobsPollPath = lifecyclePath(relay.jobs_poll_path, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
+        const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
+        const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
+        const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
         const state = {
             node_id: nodeId,
             server_name: optionalText(bootstrap.node?.server_name) || setupConfig.serverName,
@@ -3501,7 +3693,15 @@ export class SelfHostedNodeRuntime {
             heartbeat_timeout_seconds: bootstrap.heartbeat_timeout_seconds,
             enrolled_at: new Date().toISOString(),
             updated_at: new Date().toISOString(),
-            gateway_base_url: setupConfig.gatewayBaseUrl,
+            gateway_base_url: trimTrailingSlash(relayGatewayBaseUrl),
+            jobs_poll_path: jobsPollPath,
+            jobs_start_path_template: jobsStartPathTemplate,
+            jobs_events_path_template: jobsEventsPathTemplate,
+            jobs_result_path_template: jobsResultPathTemplate,
+            lifecycle_health_status: "healthy",
+            lifecycle_health_reason: undefined,
+            lifecycle_health_message: undefined,
+            lifecycle_health_updated_at: new Date().toISOString(),
             ollama_base_url: setupConfig.ollamaBaseUrl,
             discovery_mode: setupConfig.discoveryMode,
             mcoda_bin: setupConfig.mcodaBin,
@@ -3525,8 +3725,23 @@ export class SelfHostedNodeRuntime {
         };
         await writeSelfHostedNodeState(setupConfig.statePath, state);
         await writeSelfHostedRuntimeToken(setupConfig.runtimeTokenPath, runtimeToken);
+        const runtimeGateway = deps?.gateway && trimTrailingSlash(relayGatewayBaseUrl) === setupConfig.gatewayBaseUrl
+            ? deps.gateway
+            : new MswarmSelfHostedNodeClient({
+                gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
+                jobsPollPath,
+                jobsStartPathTemplate,
+                jobsEventsPathTemplate,
+                jobsResultPathTemplate,
+                fetchImpl: deps?.fetchImpl,
+                timeoutMs: setupConfig.requestTimeoutMs
+            });
         const runtime = new SelfHostedNodeRuntime({
-            gatewayBaseUrl: setupConfig.gatewayBaseUrl,
+            gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
+            jobsPollPath,
+            jobsStartPathTemplate,
+            jobsEventsPathTemplate,
+            jobsResultPathTemplate,
             nodeId,
             serverName: state.server_name,
             relayMode: state.relay_mode || setupConfig.relayMode,
@@ -3560,7 +3775,7 @@ export class SelfHostedNodeRuntime {
             exposeAllModels: setupConfig.exposeAllModels,
             modelAllowlist: setupConfig.modelAllowlist,
             modelBlocklist: setupConfig.modelBlocklist
-        }, { ...deps, gateway });
+        }, { ...deps, gateway: runtimeGateway });
         const once = await runtime.runOnce();
         return {
             created: bootstrap.created === true,
@@ -3646,6 +3861,12 @@ export class SelfHostedNodeRuntime {
         if (!runtimeToken) {
             throw new Error("Enrollment response did not include runtime_token");
         }
+        const relay = response.relay || {};
+        const gatewayBaseUrl = trimTrailingSlash(optionalText(relay.gateway_base_url) || this.config.gatewayBaseUrl);
+        const jobsPollPath = lifecyclePath(relay.jobs_poll_path || this.config.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
+        const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template || this.config.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
+        const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template || this.config.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
+        const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template || this.config.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
         const nextState = {
             ...currentState,
             node_id: this.config.nodeId,
@@ -3655,7 +3876,15 @@ export class SelfHostedNodeRuntime {
             heartbeat_timeout_seconds: response.heartbeat_timeout_seconds,
             enrolled_at: currentState.enrolled_at || new Date().toISOString(),
             updated_at: new Date().toISOString(),
-            gateway_base_url: this.config.gatewayBaseUrl,
+            gateway_base_url: gatewayBaseUrl,
+            jobs_poll_path: jobsPollPath,
+            jobs_start_path_template: jobsStartPathTemplate,
+            jobs_events_path_template: jobsEventsPathTemplate,
+            jobs_result_path_template: jobsResultPathTemplate,
+            lifecycle_health_status: "healthy",
+            lifecycle_health_reason: undefined,
+            lifecycle_health_message: undefined,
+            lifecycle_health_updated_at: new Date().toISOString(),
             ollama_base_url: this.config.ollamaBaseUrl,
             discovery_mode: this.config.discoveryMode,
             mcoda_bin: this.config.mcodaBin,
@@ -3708,7 +3937,21 @@ export class SelfHostedNodeRuntime {
         if (mapped.health_status && mapped.health_status !== "healthy" && mapped.health_status !== "unknown") {
             throw new SelfHostedPreStartJobError("selected_agent_unhealthy", `selected local mcoda agent ${selected} is ${mapped.health_status}`);
         }
-        return mapMcodaAgentToCodaliAgent(agent, selected);
+        const codaliAgent = mapMcodaAgentToCodaliAgent(agent, selected);
+        if (!codaliAgent.apiKey && mcodaAgentRequiresApiKey(agent, codaliAgent)) {
+            let apiKey;
+            try {
+                apiKey = optionalText(await this.mcodaAgentAuthResolver(agent)) || undefined;
+            }
+            catch {
+                throw new SelfHostedPreStartJobError("selected_agent_auth_unavailable", `selected local mcoda agent ${selected} auth could not be loaded`);
+            }
+            if (!apiKey) {
+                throw new SelfHostedPreStartJobError("selected_agent_auth_unavailable", `selected local mcoda agent ${selected} requires an API key; run "mcoda agent auth set ${selected}" on this node`);
+            }
+            codaliAgent.apiKey = apiKey;
+        }
+        return codaliAgent;
     }
     async executeGenericJob(envelope, options = {}) {
         const startedAt = Date.now();
@@ -3966,6 +4209,8 @@ export class SelfHostedNodeRuntime {
                         ]
                     });
                 }
+                const promptTokens = positiveInteger(ollamaResult.promptTokens);
+                const completionTokens = positiveInteger(ollamaResult.completionTokens);
                 const invocationResult = {
                     job_id: job.job_id,
                     request_id: job.request_id,
@@ -3974,10 +4219,11 @@ export class SelfHostedNodeRuntime {
                         requestId: job.request_id,
                         model: job.openai_request.model,
                         content: ollamaResult.content,
-                        promptTokens: ollamaResult.promptTokens,
-                        completionTokens: ollamaResult.completionTokens,
+                        promptTokens,
+                        completionTokens,
                         metadata: { provider: "ollama", raw: ollamaResult.raw }
                     }),
+                    usage: usageFromTokenCounts(promptTokens, completionTokens),
                     ...(streamEvents.length ? { stream_events: streamEvents } : {}),
                     ...(progressEvents.length ? { progress_events: progressEvents } : {}),
                     timing: { local_latency_ms: Date.now() - startedAt }
@@ -4058,6 +4304,7 @@ export class SelfHostedNodeRuntime {
                         mode: response.metadata.mode
                     }
                 }),
+                usage: usageFromTokenCounts(tokens.promptTokens, tokens.completionTokens),
                 ...(streamEvents.length ? { stream_events: streamEvents } : {}),
                 ...(progressEvents.length ? { progress_events: progressEvents } : {}),
                 timing: { local_latency_ms: Date.now() - startedAt }
@@ -4124,6 +4371,13 @@ export class SelfHostedNodeRuntime {
             models = [];
             version = null;
         }
+        const lifecycleState = await this.readLifecycleState();
+        const lifecycleProtocolDegraded = this.isLifecycleProtocolDegradedState(lifecycleState);
+        if (lifecycleProtocolDegraded) {
+            status = "degraded";
+            recentFailureCount = Math.max(recentFailureCount, 1);
+            this.lifecyclePollingDisabled = true;
+        }
         const discoveryLatencyMs = Date.now() - startedAt;
         const capabilityPayload = await this.buildCapabilityHeartbeatPayload(enrollment.runtimeToken);
         const loadTelemetry = this.buildLoadTelemetry({
@@ -4165,7 +4419,17 @@ export class SelfHostedNodeRuntime {
                 relay_mode: this.config.relayMode || "outbound",
                 load_reporting_enabled: loadReportingEnabled,
                 hardware_telemetry_enabled: this.config.hardwareTelemetryEnabled === true,
-                drain_mode: this.config.drainMode === true
+                drain_mode: this.config.drainMode === true,
+                ...(lifecycleProtocolDegraded
+                    ? {
+                        lifecycle: {
+                            status: "degraded",
+                            reason: lifecycleState.lifecycle_health_reason,
+                            message: lifecycleState.lifecycle_health_message,
+                            updated_at: lifecycleState.lifecycle_health_updated_at
+                        }
+                    }
+                    : {})
             },
             discovery: {
                 source: discoverySource,
@@ -4185,7 +4449,14 @@ export class SelfHostedNodeRuntime {
                 avg_latency_ms: loadTelemetry.avg_latency_ms ?? discoveryLatencyMs,
                 recent_failure_count: loadTelemetry.recent_failure_count,
                 recent_failures: loadTelemetry.recent_failures,
-                last_success_at: status === "online" ? new Date().toISOString() : null
+                last_success_at: status === "online" ? new Date().toISOString() : null,
+                ...(lifecycleProtocolDegraded
+                    ? {
+                        lifecycle_status: "degraded",
+                        lifecycle_reason: lifecycleState.lifecycle_health_reason,
+                        lifecycle_message: lifecycleState.lifecycle_health_message
+                    }
+                    : {})
             },
             local_agent_catalog: {
                 revision: loadTelemetry.catalog_fingerprint,
@@ -4239,20 +4510,36 @@ export class SelfHostedNodeRuntime {
         return { count: models.filter((model) => model.exposed !== false).length, response };
     }
     async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
+        const lifecycleState = await this.readLifecycleState();
+        if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
+            this.lifecyclePollingDisabled = true;
+            return { executed: false, status: "failed" };
+        }
         const enrollment = await this.ensureEnrolled();
         const pollCapacity = this.buildLoadTelemetry({ models: [] });
-        const response = await this.gateway.pollJob(enrollment.runtimeToken, {
-            node_id: this.config.nodeId,
-            capacity: {
-                active_jobs: pollCapacity.active_jobs,
-                queued_jobs: pollCapacity.queued_jobs,
-                max_jobs: pollCapacity.max_concurrency,
-                max_concurrency: pollCapacity.max_concurrency,
-                free_slots: pollCapacity.free_slots,
-                drain_mode: pollCapacity.drain_mode
-            },
-            wait_ms: waitMs
-        });
+        let response;
+        try {
+            response = await this.gateway.pollJob(enrollment.runtimeToken, {
+                node_id: this.config.nodeId,
+                capacity: {
+                    active_jobs: pollCapacity.active_jobs,
+                    queued_jobs: pollCapacity.queued_jobs,
+                    max_jobs: pollCapacity.max_concurrency,
+                    max_concurrency: pollCapacity.max_concurrency,
+                    free_slots: pollCapacity.free_slots,
+                    drain_mode: pollCapacity.drain_mode
+                },
+                wait_ms: waitMs
+            });
+        }
+        catch (error) {
+            const mismatch = this.lifecycleProtocolMismatch("poll", error);
+            if (mismatch) {
+                await this.markLifecycleProtocolDegraded(mismatch);
+                return { executed: false, status: "failed" };
+            }
+            throw error;
+        }
         const job = response.job || null;
         if (!job) {
             return { executed: false };
@@ -4272,19 +4559,35 @@ export class SelfHostedNodeRuntime {
                 });
                 forwardedStreamEventCount += stream_events.length;
             }
-            catch {
+            catch (error) {
+                const mismatch = this.lifecycleProtocolMismatch("events", error);
+                if (mismatch) {
+                    streamEventForwardingFailed = true;
+                    await this.markLifecycleProtocolDegraded(mismatch);
+                    throw mismatch;
+                }
                 streamEventForwardingFailed = true;
             }
         };
-        const result = await this.executeJob(job, {
+        let result = await this.executeJob(job, {
             attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined,
             onStarted: async (event) => {
-                await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
-                    node_id: this.config.nodeId,
-                    agent_slug: event.agent_slug || job.agent_slug,
-                    source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
-                    model: event.model || job.model || job.openai_request.model
-                });
+                try {
+                    await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
+                        node_id: this.config.nodeId,
+                        agent_slug: event.agent_slug || job.agent_slug,
+                        source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
+                        model: event.model || job.model || job.openai_request.model
+                    });
+                }
+                catch (error) {
+                    const mismatch = this.lifecycleProtocolMismatch("start", error);
+                    if (mismatch) {
+                        await this.markLifecycleProtocolDegraded(mismatch);
+                        throw mismatch;
+                    }
+                    throw error;
+                }
             },
             onOpenAIChunk: async (chunk) => {
                 if (job.openai_request.stream !== true || streamEventForwardingFailed) {
@@ -4296,7 +4599,23 @@ export class SelfHostedNodeRuntime {
                 }
             },
         });
-        await flushStreamEvents();
+        try {
+            await flushStreamEvents();
+        }
+        catch (error) {
+            if (!(error instanceof SelfHostedProtocolMismatchError)) {
+                throw error;
+            }
+            result = {
+                job_id: job.job_id,
+                request_id: job.request_id,
+                status: "failed",
+                error: { code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE, message: error.message },
+                ...(result.stream_events?.length ? { stream_events: result.stream_events } : {}),
+                ...(result.progress_events?.length ? { progress_events: result.progress_events } : {}),
+                timing: result.timing
+            };
+        }
         const postedResult = job.openai_request.stream === true
             ? streamEventForwardingFailed
                 ? {
@@ -4305,10 +4624,17 @@ export class SelfHostedNodeRuntime {
                 }
                 : (({ stream_events: _streamEvents, ...rest }) => rest)(result)
             : result;
-        await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, {
-            ...postedResult,
-            node_id: this.config.nodeId
-        });
+        try {
+            await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, this.jobResultPayload(postedResult));
+        }
+        catch (error) {
+            const mismatch = this.lifecycleProtocolMismatch("result", error);
+            if (mismatch) {
+                await this.markLifecycleProtocolDegraded(mismatch);
+                return { executed: true, job_id: job.job_id, status: "failed" };
+            }
+            throw error;
+        }
         return { executed: true, job_id: job.job_id, status: result.status };
     }
     async doctor() {
@@ -4350,7 +4676,7 @@ export class SelfHostedNodeRuntime {
         let timer = null;
         let polling = false;
         const poll = () => {
-            if (stopped || polling || this.config.relayMode === "direct")
+            if (stopped || polling || this.lifecyclePollingDisabled || this.config.relayMode === "direct")
                 return;
             polling = true;
             void this.pollAndExecuteJob()