@mcoda/mswarm 0.1.106 → 0.1.108

package/dist/runtime.js

@@ -4,6 +4,7 @@ import { cpus, freemem, hostname, homedir, loadavg, platform, totalmem, userInfo
 import { spawn } from "node:child_process";
 import { createHash, createHmac, randomBytes, randomUUID } from "node:crypto";
 import { isIP } from "node:net";
+import { Agent, fetch as undiciFetch } from "undici";
 import { GlobalRepository } from "@mcoda/db";
 import { MswarmCodaliExecutor } from "./codali-executor.js";
 import { GENERATIVE_OPERATION_DEFAULT_PATHS, GENERATIVE_REQUEST_PARAMETER_ALLOWLISTS, MSWARM_CAPABILITY_SCHEMA_VERSION, CryptoHelper, assertMswarmSafeRelativePath, validateMswarmArchiveEntry, buildMswarmCapabilityNames, buildMswarmPrivateCapabilityCatalogEntry, buildMswarmLocalArtifactUri, buildMswarmSandboxProfile, defaultMswarmArtifactAccessPolicy, defaultMswarmArtifactRetentionPolicy, normalizeGenerativeModalities, normalizeSelfHostedGenerativeOperation, projectMswarmPublicCapabilities, validateMswarmGenericJobRequest } from "@mcoda/shared";
@@ -61,6 +62,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30;
 const DEFAULT_SELF_HOSTED_NODE_VERSION = "0.1.70";
 const DEFAULT_REQUEST_TIMEOUT_MS = 10000;
 const DEFAULT_JOB_TIMEOUT_MS = 3600000;
+const DEFAULT_RESULT_POST_TIMEOUT_MS = 5 * 60000;
 const DEFAULT_SERVICE_COMMAND_TIMEOUT_MS = 60000;
 const DEFAULT_CAPABILITY_PROBE_TIMEOUT_MS = 2000;
 const SELF_HOSTED_RUNTIME_PROTOCOL_VERSION = 1;
@@ -73,6 +75,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
 const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
 const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
 const DEFAULT_JOB_POLL_WAIT_MS = 2000;
+const RESERVED_JOB_POLL_WAIT_MS = 250;
+const MAX_POLL_EXCLUDED_AGENT_SLUGS = 64;
+const MAX_RELAY_AGENT_SLUG_LENGTH = 255;
+const DEFAULT_RESERVED_PRIORITY_MAX = -1;
+const MIN_RELAY_PRIORITY = -100;
+const MAX_RELAY_PRIORITY = 100;
 const JOB_POLL_RETRY_BASE_MS = 1000;
 const JOB_POLL_RETRY_MAX_MS = 60000;
 /**
@@ -283,6 +291,33 @@ const LOCAL_RUNNER_RESPONSE_FORMAT_ALIASES = {
 function optionalText(value) {
     return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
 }
+function relayJobAgentAliases(job) {
+    const aliases = [job.agent_slug, job.source_agent_slug, job.remote_slug]
+        .map(optionalText)
+        .map((value) => value
+        ?.toLowerCase()
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "") || null)
+        .filter((value) => typeof value === "string" && value.length > 0 && value.length <= MAX_RELAY_AGENT_SLUG_LENGTH);
+    return [...new Set(aliases)];
+}
+function boundedRelayAgentAliases(activeAliases) {
+    const aliases = [];
+    const seen = new Set();
+    for (const group of activeAliases) {
+        for (const alias of group) {
+            if (seen.has(alias))
+                continue;
+            seen.add(alias);
+            aliases.push(alias);
+            if (aliases.length >= MAX_POLL_EXCLUDED_AGENT_SLUGS) {
+                return aliases;
+            }
+        }
+    }
+    return aliases;
+}
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
@@ -587,6 +622,39 @@ function parsePositiveInteger(value, fallback) {
     }
     return Math.floor(parsed);
 }
+function parseNonNegativeInteger(value, fallback) {
+    const parsed = value === undefined || value === null || value === ""
+        ? fallback
+        : typeof value === "number"
+            ? value
+            : typeof value === "string"
+                ? Number(value)
+                : Number.NaN;
+    if (!Number.isInteger(parsed) || parsed < 0) {
+        throw new Error("MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS must be a non-negative integer");
+    }
+    return parsed;
+}
+function parseRelayPriority(value, fallback) {
+    const parsed = value === undefined || value === null || value === ""
+        ? fallback
+        : typeof value === "number"
+            ? value
+            : typeof value === "string"
+                ? Number(value)
+                : Number.NaN;
+    if (!Number.isInteger(parsed) || parsed < MIN_RELAY_PRIORITY || parsed > MAX_RELAY_PRIORITY) {
+        throw new Error(`MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX must be an integer from ${MIN_RELAY_PRIORITY} to ${MAX_RELAY_PRIORITY}`);
+    }
+    return parsed;
+}
+function assertReservedLlmCapacity(input) {
+    const totalLlmJobs = input.maxConcurrentLlmJobs + input.reservedLlmJobs;
+    if (totalLlmJobs > input.maxConcurrentJobs) {
+        throw new Error("MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS must be greater than or equal to " +
+            "MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS plus MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS");
+    }
+}
 function parseBoolean(value, fallback) {
     if (typeof value === "boolean")
         return value;
@@ -1084,7 +1152,14 @@ function isHttpStatusError(error, status) {
 }
 const RELAY_RESULT_POST_RETRY_DELAYS_MS = [250, 1000, 2500];
 function runtimeErrorMessage(error) {
-    return error instanceof Error ? error.message : String(error);
+    const message = error instanceof Error ? error.message : String(error);
+    const cause = error instanceof Error
+        ? error.cause
+        : undefined;
+    const causeCode = isRecord(cause) ? optionalText(cause.code) : null;
+    return causeCode && !message.includes(causeCode)
+        ? `${message} (${causeCode})`
+        : message;
 }
 async function sleep(ms) {
     await new Promise((resolve) => setTimeout(resolve, ms));
@@ -1272,6 +1347,8 @@ function serviceEnvironment(config, env, homeDir) {
         MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS: String(config.jobTimeoutMs),
         MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS: String(config.maxConcurrentJobs || 1),
         MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS: String(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1),
+        MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS: String(config.reservedLlmJobs ?? 0),
+        MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX: String(config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX),
         MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED: config.genericJobsEnabled ? "true" : "false",
         MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS: String(config.genericJobTimeoutMs),
         MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY: String(config.genericJobMaxConcurrency),
@@ -1687,6 +1764,9 @@ export async function readSelfHostedNodeConfig(env = process.env) {
     const packageNodeVersion = await readPackageNodeVersion();
     const maxConcurrentJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, state.max_concurrent_jobs || 1);
     const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
+    const reservedLlmJobs = parseNonNegativeInteger(env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, state.reserved_llm_jobs ?? 0);
+    const reservedPriorityMax = parseRelayPriority(env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, state.reserved_priority_max ?? DEFAULT_RESERVED_PRIORITY_MAX);
+    assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
     return {
         gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
         jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
@@ -1730,6 +1810,8 @@ export async function readSelfHostedNodeConfig(env = process.env) {
         jobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
         maxConcurrentJobs,
         maxConcurrentLlmJobs,
+        reservedLlmJobs,
+        reservedPriorityMax,
         genericJobsEnabled: parseBoolean(env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED ?? env.MSWARM_SELF_HOSTED_GENERIC_JOBS, state.generic_jobs_enabled === true),
         genericJobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, state.generic_job_timeout_ms || state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
         genericJobMaxConcurrency: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, state.generic_job_max_concurrency || 1),
@@ -1779,6 +1861,9 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
     const packageNodeVersion = await readPackageNodeVersion();
     const maxConcurrentJobs = parsePositiveInteger(options["max-concurrent-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, 1);
     const maxConcurrentLlmJobs = parsePositiveInteger(options["max-concurrent-llm-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, maxConcurrentJobs);
+    const reservedLlmJobs = parseNonNegativeInteger(options["reserved-llm-jobs"] ?? env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, 0);
+    const reservedPriorityMax = parseRelayPriority(options["reserved-priority-max"] ?? env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, DEFAULT_RESERVED_PRIORITY_MAX);
+    assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
     return {
         apiKey,
         handle: handle || undefined,
@@ -1804,6 +1889,8 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
         jobTimeoutMs: parsePositiveInteger(options["job-timeout-ms"] || env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
         maxConcurrentJobs,
         maxConcurrentLlmJobs,
+        reservedLlmJobs,
+        reservedPriorityMax,
         genericJobsEnabled: parseBoolean(options["enable-generic-jobs"] || env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED || env.MSWARM_SELF_HOSTED_GENERIC_JOBS, false),
         genericJobTimeoutMs: parsePositiveInteger(options["generic-job-timeout-ms"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
         genericJobMaxConcurrency: parsePositiveInteger(options["generic-job-max-concurrency"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, 1),
@@ -4092,6 +4179,7 @@ export class MswarmSelfHostedNodeClient {
         this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
         this.fetchImpl = input.fetchImpl || fetch;
         this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
+        this.resultTimeoutMs = Math.max(this.timeoutMs, input.resultTimeoutMs || DEFAULT_RESULT_POST_TIMEOUT_MS);
     }
     lifecycleEndpoint(kind) {
         const path = kind === "poll"
@@ -4206,7 +4294,7 @@ export class MswarmSelfHostedNodeClient {
                 authorization: `Bearer ${runtimeToken}`
             },
             body: JSON.stringify(payload)
-        }, this.timeoutMs);
+        }, this.resultTimeoutMs);
     }
     async postJobStart(runtimeToken, jobId, payload) {
         return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
@@ -4238,8 +4326,15 @@ export class SelfHostedNodeRuntime {
         this.latencySamplesMs = [];
         this.recentFailures = [];
         this.lifecyclePollingDisabled = false;
+        assertReservedLlmCapacity({
+            maxConcurrentJobs: Math.max(1, Math.floor(config.maxConcurrentJobs || 1)),
+            maxConcurrentLlmJobs: Math.max(1, Math.floor(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1)),
+            reservedLlmJobs: parseNonNegativeInteger(config.reservedLlmJobs, 0)
+        });
+        parseRelayPriority(config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
         this.config = config;
         this.fetchImpl = deps?.fetchImpl || fetch;
+        this.useDedicatedGenerativeDispatcher = deps?.fetchImpl === undefined;
         this.gateway =
             deps?.gateway ||
                 new MswarmSelfHostedNodeClient({
@@ -4249,7 +4344,8 @@ export class SelfHostedNodeRuntime {
                     jobsEventsPathTemplate: config.jobsEventsPathTemplate,
                     jobsResultPathTemplate: config.jobsResultPathTemplate,
                     fetchImpl: this.fetchImpl,
-                    timeoutMs: config.requestTimeoutMs
+                    timeoutMs: config.requestTimeoutMs,
+                    resultTimeoutMs: Math.min(config.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
                 });
         this.mcoda =
             deps?.mcoda ||
@@ -4439,7 +4535,8 @@ export class SelfHostedNodeRuntime {
             new MswarmSelfHostedNodeClient({
                 gatewayBaseUrl: setupConfig.gatewayBaseUrl,
                 fetchImpl: deps?.fetchImpl,
-                timeoutMs: setupConfig.requestTimeoutMs
+                timeoutMs: setupConfig.requestTimeoutMs,
+                resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
             });
         const machineId = await readOrCreateSelfHostedMachineId(setupConfig.machineIdPath);
         const machineFingerprint = machineFingerprintFromId(machineId);
@@ -4508,6 +4605,8 @@ export class SelfHostedNodeRuntime {
             job_timeout_ms: setupConfig.jobTimeoutMs,
             max_concurrent_jobs: setupConfig.maxConcurrentJobs,
             max_concurrent_llm_jobs: setupConfig.maxConcurrentLlmJobs,
+            reserved_llm_jobs: setupConfig.reservedLlmJobs,
+            reserved_priority_max: setupConfig.reservedPriorityMax,
             generic_jobs_enabled: setupConfig.genericJobsEnabled,
             generic_job_timeout_ms: setupConfig.genericJobTimeoutMs,
             generic_job_max_concurrency: setupConfig.genericJobMaxConcurrency,
@@ -4532,7 +4631,8 @@ export class SelfHostedNodeRuntime {
                 jobsEventsPathTemplate,
                 jobsResultPathTemplate,
                 fetchImpl: deps?.fetchImpl,
-                timeoutMs: setupConfig.requestTimeoutMs
+                timeoutMs: setupConfig.requestTimeoutMs,
+                resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
             });
         const runtime = new SelfHostedNodeRuntime({
             gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
@@ -4563,6 +4663,8 @@ export class SelfHostedNodeRuntime {
             jobTimeoutMs: setupConfig.jobTimeoutMs,
             maxConcurrentJobs: setupConfig.maxConcurrentJobs,
             maxConcurrentLlmJobs: setupConfig.maxConcurrentLlmJobs,
+            reservedLlmJobs: setupConfig.reservedLlmJobs,
+            reservedPriorityMax: setupConfig.reservedPriorityMax,
             genericJobsEnabled: setupConfig.genericJobsEnabled,
             genericJobTimeoutMs: setupConfig.genericJobTimeoutMs,
             genericJobMaxConcurrency: setupConfig.genericJobMaxConcurrency,
@@ -4693,6 +4795,8 @@ export class SelfHostedNodeRuntime {
             job_timeout_ms: this.config.jobTimeoutMs,
             max_concurrent_jobs: this.config.maxConcurrentJobs,
             max_concurrent_llm_jobs: this.config.maxConcurrentLlmJobs,
+            reserved_llm_jobs: this.config.reservedLlmJobs ?? 0,
+            reserved_priority_max: this.config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX,
             generic_jobs_enabled: this.config.genericJobsEnabled,
             generic_job_timeout_ms: this.config.genericJobTimeoutMs,
             generic_job_max_concurrency: this.config.genericJobMaxConcurrency,
@@ -5016,14 +5120,20 @@ export class SelfHostedNodeRuntime {
         const jsonResponseByteLimit = generativeJsonResponseByteLimit(outputByteLimit);
         const controller = new AbortController();
         const timeout = setTimeout(() => controller.abort(), this.config.jobTimeoutMs);
+        const dispatcher = this.useDedicatedGenerativeDispatcher
+            ? new Agent({ headersTimeout: 0, bodyTimeout: 0 })
+            : null;
         let payload;
         try {
-            const response = await this.fetchImpl(prepared.url, {
+            const request = {
                 method: "POST",
                 headers: prepared.headers,
                 body: JSON.stringify(prepared.body),
                 signal: controller.signal
-            });
+            };
+            const response = dispatcher
+                ? (await undiciFetch(prepared.url, { ...request, dispatcher }))
+                : await this.fetchImpl(prepared.url, request);
             if (!response.ok) {
                 await response.body?.cancel().catch(() => undefined);
                 throw new Error(`local ${operation} runner request failed with status ${response.status}`);
@@ -5044,6 +5154,7 @@ export class SelfHostedNodeRuntime {
         }
         finally {
             clearTimeout(timeout);
+            await dispatcher?.close().catch(() => undefined);
         }
         if (!isRecord(payload) || !Array.isArray(payload.data) || payload.data.length === 0) {
             throw new Error(`local ${operation} runner returned an invalid base64 response`);
@@ -5643,7 +5754,7 @@ export class SelfHostedNodeRuntime {
             return result;
         }
         catch (error) {
-            const message = redactRuntimeSecretValues(error instanceof Error ? error.message : String(error), [selectedAgent?.apiKey, options.attachedMswarmApiKey]);
+            const message = redactRuntimeSecretValues(runtimeErrorMessage(error), [selectedAgent?.apiKey, options.attachedMswarmApiKey]);
             const explicitCode = selfHostedErrorCode(error);
             const code = explicitCode ??
                 (/timeout/i.test(message)
@@ -5837,11 +5948,11 @@ export class SelfHostedNodeRuntime {
         });
         return { count: models.filter((model) => model.exposed !== false).length, response };
     }
-    async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
+    async pollRelayJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS, options = {}) {
         const lifecycleState = await this.readLifecycleState();
         if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
             this.lifecyclePollingDisabled = true;
-            return { executed: false, status: "failed" };
+            return { claim: null, status: "failed" };
         }
         const enrollment = await this.ensureEnrolled();
         const pollCapacity = this.buildLoadTelemetry({ models: [] });
@@ -5855,7 +5966,17 @@ export class SelfHostedNodeRuntime {
                     max_jobs: pollCapacity.max_concurrency,
                     max_concurrency: pollCapacity.max_concurrency,
                     free_slots: pollCapacity.free_slots,
-                    drain_mode: pollCapacity.drain_mode
+                    drain_mode: pollCapacity.drain_mode,
+                    ...(options.excludedAgentSlugs !== undefined
+                        ? {
+                            excluded_agent_slugs: boundedRelayAgentAliases([
+                                options.excludedAgentSlugs
+                            ])
+                        }
+                        : {}),
+                    ...(options.maxRequestedPriority !== undefined
+                        ? { max_requested_priority: options.maxRequestedPriority }
+                        : {})
                 },
                 wait_ms: waitMs
             });
@@ -5864,14 +5985,24 @@ export class SelfHostedNodeRuntime {
             const mismatch = this.lifecycleProtocolMismatch("poll", error);
             if (mismatch) {
                 await this.markLifecycleProtocolDegraded(mismatch);
-                return { executed: false, status: "failed" };
+                return { claim: null, status: "failed" };
             }
             throw error;
         }
         const job = response.job || null;
         if (!job) {
-            return { executed: false };
+            return { claim: null };
         }
+        return {
+            claim: {
+                runtimeToken: enrollment.runtimeToken,
+                job,
+                attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined
+            }
+        };
+    }
+    async executeRelayJobClaim(claim) {
+        const { job, runtimeToken } = claim;
         const chatStreaming = (normalizeSelfHostedGenerativeOperationName(job.operation) ?? "chat.completions") ===
             "chat.completions" &&
             job.openai_request.stream === true;
@@ -5884,7 +6015,7 @@ export class SelfHostedNodeRuntime {
             }
             const stream_events = pendingStreamEvents.splice(0, pendingStreamEvents.length);
             try {
-                await this.gateway.postJobEvents(enrollment.runtimeToken, job.job_id, {
+                await this.gateway.postJobEvents(runtimeToken, job.job_id, {
                     node_id: this.config.nodeId,
                     stream_events
                 });
@@ -5901,10 +6032,10 @@ export class SelfHostedNodeRuntime {
             }
         };
         let result = await this.executeJob(job, {
-            attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined,
+            attachedMswarmApiKey: claim.attachedMswarmApiKey,
             onStarted: async (event) => {
                 try {
-                    await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
+                    await this.gateway.postJobStart(runtimeToken, job.job_id, {
                         node_id: this.config.nodeId,
                         agent_slug: event.agent_slug || job.agent_slug,
                         source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
@@ -5959,7 +6090,7 @@ export class SelfHostedNodeRuntime {
         let resultPostError = null;
         for (let attempt = 0; attempt <= RELAY_RESULT_POST_RETRY_DELAYS_MS.length; attempt += 1) {
             try {
-                await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, this.jobResultPayload(postedResult));
+                await this.gateway.postJobResult(runtimeToken, job.job_id, this.jobResultPayload(postedResult));
                 resultPostError = null;
                 break;
             }
@@ -5987,6 +6118,13 @@ export class SelfHostedNodeRuntime {
         }
         return { executed: true, job_id: job.job_id, status: result.status };
     }
+    async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
+        const polled = await this.pollRelayJob(waitMs);
+        if (!polled.claim) {
+            return { executed: false, ...(polled.status ? { status: polled.status } : {}) };
+        }
+        return this.executeRelayJobClaim(polled.claim);
+    }
     async doctor() {
         const checks = [];
         checks.push({ name: "config", ok: Boolean(this.config.nodeId), message: this.config.nodeId || "missing node id" });
@@ -6025,19 +6163,33 @@ export class SelfHostedNodeRuntime {
         let stopped = false;
         let timer = null;
         let pollTimer = null;
+        let pollTimerDueAt = 0;
         let polling = false;
-        let pollFailureStreak = 0;
+        let relayPollFailureStreak = 0;
+        let relayExecutionFailureStreak = 0;
+        let activeRelayExecutions = 0;
+        let activeRegularRelayExecutions = 0;
+        let relayExecutionSequence = 0;
+        const activeRelayAgentAliases = new Map();
         let revoked = false;
+        const regularRelayExecutionLimit = Math.max(1, Math.floor(this.config.maxConcurrentLlmJobs || this.config.maxConcurrentJobs || 1));
+        const reservedRelayExecutionLimit = parseNonNegativeInteger(this.config.reservedLlmJobs, 0);
+        const reservedPriorityMax = parseRelayPriority(this.config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
+        const relayExecutionLimit = regularRelayExecutionLimit + reservedRelayExecutionLimit;
+        const clearPollTimer = () => {
+            if (pollTimer) {
+                clearTimeout(pollTimer);
+                pollTimer = null;
+            }
+            pollTimerDueAt = 0;
+        };
         const halt = () => {
             stopped = true;
             if (timer) {
                 clearTimeout(timer);
                 timer = null;
             }
-            if (pollTimer) {
-                clearTimeout(pollTimer);
-                pollTimer = null;
-            }
+            clearPollTimer();
         };
         // 410 means the gateway has retired this node's credentials: it was removed there.
         // Retrying at full rate achieves nothing but a hot loop and a flooded journal, so
@@ -6051,10 +6203,7 @@ export class SelfHostedNodeRuntime {
             if (!isHttpStatusError(error, 410)) {
                 return false;
             }
-            if (pollTimer) {
-                clearTimeout(pollTimer);
-                pollTimer = null;
-            }
+            clearPollTimer();
             if (!revoked) {
                 revoked = true;
                 console.error(`[mswarm] this node was removed at the gateway (node ${this.config.nodeId}). ` +
@@ -6065,42 +6214,112 @@ export class SelfHostedNodeRuntime {
             return true;
         };
         const noteAlive = () => {
-            if (!revoked) {
+            const recoveredFromRevocation = revoked;
+            const relayPollingIsBackedOff = (relayPollFailureStreak > 0 || relayExecutionFailureStreak > 0) && !polling;
+            if (!recoveredFromRevocation && !relayPollingIsBackedOff) {
+                return;
+            }
+            if (recoveredFromRevocation) {
+                revoked = false;
+                console.error(`[mswarm] node ${this.config.nodeId} is accepted at the gateway again; resuming normal operation.`);
+            }
+            clearPollTimer();
+            relayPollFailureStreak = 0;
+            relayExecutionFailureStreak = 0;
+            schedulePoll(0);
+        };
+        const currentRelayRetryDelayMs = () => Math.max(pollRetryDelayMs(relayPollFailureStreak), pollRetryDelayMs(relayExecutionFailureStreak));
+        const schedulePoll = (delayMs) => {
+            if (stopped ||
+                polling ||
+                revoked ||
+                this.lifecyclePollingDisabled ||
+                this.config.relayMode === "direct" ||
+                activeRelayExecutions >= relayExecutionLimit) {
+                return;
+            }
+            const dueAt = Date.now() + Math.max(0, delayMs);
+            if (pollTimer && pollTimerDueAt >= dueAt) {
                 return;
             }
-            revoked = false;
-            console.error(`[mswarm] node ${this.config.nodeId} is accepted at the gateway again; resuming normal operation.`);
-            poll();
+            clearPollTimer();
+            pollTimerDueAt = dueAt;
+            pollTimer = setTimeout(() => {
+                pollTimer = null;
+                pollTimerDueAt = 0;
+                poll();
+            }, Math.max(0, dueAt - Date.now()));
         };
         const poll = () => {
             if (stopped ||
                 polling ||
                 revoked ||
                 this.lifecyclePollingDisabled ||
-                this.config.relayMode === "direct") {
+                this.config.relayMode === "direct" ||
+                activeRelayExecutions >= relayExecutionLimit) {
                 return;
             }
             polling = true;
-            void this.pollAndExecuteJob()
-                .then(() => {
-                pollFailureStreak = 0;
+            const reservedOnly = reservedRelayExecutionLimit > 0 &&
+                activeRegularRelayExecutions >= regularRelayExecutionLimit &&
+                activeRelayExecutions < relayExecutionLimit;
+            const excludedAgentSlugs = boundedRelayAgentAliases(activeRelayAgentAliases.values());
+            void this.pollRelayJob(reservedOnly ? RESERVED_JOB_POLL_WAIT_MS : DEFAULT_JOB_POLL_WAIT_MS, {
+                ...(reservedRelayExecutionLimit > 0 ? { excludedAgentSlugs } : {}),
+                ...(reservedOnly ? { maxRequestedPriority: reservedPriorityMax } : {})
+            })
+                .then((polled) => {
+                relayPollFailureStreak = 0;
+                const claim = polled.claim;
+                if (!claim) {
+                    return;
+                }
+                const regularLane = !reservedOnly;
+                const executionId = ++relayExecutionSequence;
+                activeRelayExecutions += 1;
+                if (regularLane) {
+                    activeRegularRelayExecutions += 1;
+                }
+                activeRelayAgentAliases.set(executionId, relayJobAgentAliases(claim.job));
+                void this.executeRelayJobClaim(claim)
+                    .then(() => {
+                    relayExecutionFailureStreak = 0;
+                })
+                    .catch((error) => {
+                    if (noteRevoked(error)) {
+                        return;
+                    }
+                    relayExecutionFailureStreak += 1;
+                    console.error(`[mswarm] self-hosted relay job ${claim.job.job_id} failed: ${runtimeErrorMessage(error)}`);
+                })
+                    .finally(() => {
+                    activeRelayExecutions = Math.max(0, activeRelayExecutions - 1);
+                    if (regularLane) {
+                        activeRegularRelayExecutions = Math.max(0, activeRegularRelayExecutions - 1);
+                    }
+                    activeRelayAgentAliases.delete(executionId);
+                    schedulePoll(currentRelayRetryDelayMs());
+                });
             })
                 .catch((error) => {
                 if (noteRevoked(error)) {
                     return;
                 }
-                pollFailureStreak += 1;
+                relayPollFailureStreak += 1;
                 console.error(`[mswarm] self-hosted relay poll failed: ${runtimeErrorMessage(error)}`);
             })
                 .finally(() => {
                 polling = false;
-                if (stopped || revoked) {
+                if (stopped ||
+                    revoked ||
+                    this.lifecyclePollingDisabled ||
+                    activeRelayExecutions >= relayExecutionLimit) {
                     return;
                 }
                 // A healthy poll long-polls at the gateway, so re-entering immediately is
                 // correct. A failing one returns at once, so without this the loop spins as
                 // fast as the network allows.
-                pollTimer = setTimeout(poll, pollRetryDelayMs(pollFailureStreak));
+                schedulePoll(currentRelayRetryDelayMs());
             });
         };
         const schedule = () => {
@@ -6122,7 +6341,7 @@ export class SelfHostedNodeRuntime {
         })
             .finally(() => {
             schedule();
-            poll();
+            schedulePoll(0);
         });
         return { stop: halt };
     }