@mcoda/mswarm 0.1.79 → 0.1.81

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/runtime.js CHANGED
@@ -26,7 +26,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
26
26
  const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
27
27
  const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
28
28
  const DEFAULT_JOB_POLL_WAIT_MS = 25000;
29
+ const DEFAULT_SELF_HOSTED_JOBS_POLL_PATH = "/v1/swarm/self-hosted/node/jobs/poll";
30
+ const DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/start";
31
+ const DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/events";
32
+ const DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/result";
29
33
  const DEFAULT_STREAM_EVENT_BATCH_SIZE = 8;
34
+ const SELF_HOSTED_PROTOCOL_MISMATCH_CODE = "self_hosted_protocol_mismatch";
30
35
  const OWNER_LOCAL_TEST_ECHO_JOB_TYPE = "tenant.test-echo";
31
36
  const TEST_ECHO_RUNNER_ID = "test.echo";
32
37
  const RENDER_BLENDER_JOB_TYPE = "render.blender";
@@ -402,6 +407,37 @@ function parseCliOptions(argv) {
402
407
  function trimTrailingSlash(value) {
403
408
  return value.replace(/\/+$/g, "");
404
409
  }
410
+ function lifecyclePath(value, fallback) {
411
+ return optionalText(value) || fallback;
412
+ }
413
+ function resolveLifecycleUrl(gatewayBaseUrl, pathOrUrl) {
414
+ try {
415
+ return new URL(pathOrUrl).toString();
416
+ }
417
+ catch {
418
+ return new URL(pathOrUrl.startsWith("/") ? pathOrUrl : `/${pathOrUrl}`, `${trimTrailingSlash(gatewayBaseUrl)}/`).toString();
419
+ }
420
+ }
421
+ function resolveLifecycleTemplate(template, jobId) {
422
+ const encoded = encodeURIComponent(jobId);
423
+ return template
424
+ .replace(/:jobId\b/g, encoded)
425
+ .replace(/:job_id\b/g, encoded)
426
+ .replace(/\{jobId\}/g, encoded)
427
+ .replace(/\{job_id\}/g, encoded);
428
+ }
429
+ function usageFromTokenCounts(promptTokens, completionTokens) {
430
+ const prompt = positiveInteger(promptTokens) ?? 0;
431
+ const completion = positiveInteger(completionTokens) ?? 0;
432
+ return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion };
433
+ }
434
+ function openAiUsage(response) {
435
+ const usage = isRecord(response?.usage) ? response.usage : undefined;
436
+ const prompt = positiveInteger(usage?.prompt_tokens) ?? 0;
437
+ const completion = positiveInteger(usage?.completion_tokens) ?? 0;
438
+ const total = positiveInteger(usage?.total_tokens) ?? prompt + completion;
439
+ return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: total };
440
+ }
405
441
  function defaultStatePath() {
406
442
  return join(homedir(), ".mswarm", "self-hosted-node", "config.json");
407
443
  }
@@ -592,12 +628,19 @@ function normalizeHealthStatus(value) {
592
628
  return "blocked";
593
629
  return "unknown";
594
630
  }
595
- function isMswarmManagedCloudAgent(agent) {
631
+ function isMswarmManagedAgent(agent) {
596
632
  const config = agent.config && typeof agent.config === "object" ? agent.config : {};
597
- const mswarmCloud = config.mswarmCloud;
598
- return Boolean(mswarmCloud &&
599
- typeof mswarmCloud === "object" &&
600
- mswarmCloud.managed === true);
633
+ const managedConfigs = [config.mswarmCloud, config.mswarmSelfHosted, config.mswarmWorker];
634
+ if (managedConfigs.some((managedConfig) => Boolean(managedConfig) &&
635
+ typeof managedConfig === "object" &&
636
+ managedConfig.managed === true)) {
637
+ return true;
638
+ }
639
+ const slug = optionalText(agent.slug)?.toLowerCase();
640
+ return Boolean(slug &&
641
+ (slug.startsWith("mswarm-cloud-") ||
642
+ slug.startsWith("mswarm-self-hosted-") ||
643
+ slug.startsWith("mswarm-worker-")));
601
644
  }
602
645
  function isModelExposed(modelName, family, config) {
603
646
  if (config.modelBlocklist.includes(modelName)) {
@@ -624,6 +667,18 @@ function isAgentExposed(agentSlug, defaultModel, bestUsage, config) {
624
667
  }
625
668
  return config.exposeAllModels;
626
669
  }
670
+ class HttpStatusError extends Error {
671
+ constructor(url, status, responseText) {
672
+ super(`request_failed:${status}:${responseText.slice(0, 200)}`);
673
+ this.name = "HttpStatusError";
674
+ this.url = url;
675
+ this.status = status;
676
+ this.responseText = responseText;
677
+ }
678
+ }
679
+ function isHttpStatusError(error, status) {
680
+ return error instanceof HttpStatusError && (status === undefined || error.status === status);
681
+ }
627
682
  async function fetchJson(fetchImpl, url, init, timeoutMs) {
628
683
  const controller = new AbortController();
629
684
  const timeout = setTimeout(() => controller.abort(), timeoutMs);
@@ -631,7 +686,7 @@ async function fetchJson(fetchImpl, url, init, timeoutMs) {
631
686
  const response = await fetchImpl(url, { ...init, signal: controller.signal });
632
687
  if (!response.ok) {
633
688
  const text = await response.text().catch(() => "");
634
- throw new Error(`request_failed:${response.status}:${text.slice(0, 200)}`);
689
+ throw new HttpStatusError(url, response.status, text);
635
690
  }
636
691
  return (await response.json());
637
692
  }
@@ -1161,6 +1216,18 @@ export async function readSelfHostedNodeConfig(env = process.env) {
1161
1216
  const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
1162
1217
  return {
1163
1218
  gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
1219
+ jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
1220
+ state.jobs_poll_path ||
1221
+ DEFAULT_SELF_HOSTED_JOBS_POLL_PATH,
1222
+ jobsStartPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_START_PATH_TEMPLATE) ||
1223
+ state.jobs_start_path_template ||
1224
+ DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE,
1225
+ jobsEventsPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE) ||
1226
+ state.jobs_events_path_template ||
1227
+ DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE,
1228
+ jobsResultPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE) ||
1229
+ state.jobs_result_path_template ||
1230
+ DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE,
1164
1231
  nodeId,
1165
1232
  serverName: optionalText(env.MSWARM_SELF_HOSTED_SERVER_NAME) || state.server_name || null,
1166
1233
  relayMode: parseRelayMode(env.MSWARM_SELF_HOSTED_RELAY_MODE || state.relay_mode),
@@ -1303,7 +1370,7 @@ export function mapOllamaModelToSelfHostedModel(model, config) {
1303
1370
  };
1304
1371
  }
1305
1372
  export function mapMcodaAgentToSelfHostedModel(agent, config) {
1306
- if (isMswarmManagedCloudAgent(agent)) {
1373
+ if (isMswarmManagedAgent(agent)) {
1307
1374
  return null;
1308
1375
  }
1309
1376
  const slug = optionalText(agent.slug);
@@ -1684,6 +1751,7 @@ const PRE_START_JOB_ERROR_CODES = new Set([
1684
1751
  "validation_failed",
1685
1752
  "docdex_context_missing",
1686
1753
  "docdex_api_key_missing",
1754
+ SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
1687
1755
  ]);
1688
1756
  class SelfHostedDocdexJobError extends Error {
1689
1757
  constructor(code, message) {
@@ -1699,6 +1767,18 @@ class SelfHostedPreStartJobError extends Error {
1699
1767
  this.code = code;
1700
1768
  }
1701
1769
  }
1770
+ class SelfHostedProtocolMismatchError extends Error {
1771
+ constructor(input) {
1772
+ super(`Gateway is missing lifecycle endpoint ${input.endpoint} at ${input.gatewayBaseUrl} ` +
1773
+ `(runtime package version ${input.runtimePackageVersion}). ` +
1774
+ "Run a direct local agent/model instead, for example qwen3.6-llama.cpp.");
1775
+ this.code = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
1776
+ this.name = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
1777
+ this.endpoint = input.endpoint;
1778
+ this.gatewayBaseUrl = input.gatewayBaseUrl;
1779
+ this.runtimePackageVersion = input.runtimePackageVersion;
1780
+ }
1781
+ }
1702
1782
  function normalizeDocdexCapabilityMap(value) {
1703
1783
  const record = objectRecord(value);
1704
1784
  if (!record)
@@ -3207,9 +3287,36 @@ export class McodaLocalAgentExecutor {
3207
3287
  export class MswarmSelfHostedNodeClient {
3208
3288
  constructor(input) {
3209
3289
  this.gatewayBaseUrl = trimTrailingSlash(input.gatewayBaseUrl);
3290
+ this.jobsPollPath = lifecyclePath(input.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
3291
+ this.jobsStartPathTemplate = lifecyclePath(input.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
3292
+ this.jobsEventsPathTemplate = lifecyclePath(input.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
3293
+ this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
3210
3294
  this.fetchImpl = input.fetchImpl || fetch;
3211
3295
  this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
3212
3296
  }
3297
+ lifecycleEndpoint(kind) {
3298
+ const path = kind === "poll"
3299
+ ? this.jobsPollPath
3300
+ : kind === "start"
3301
+ ? this.jobsStartPathTemplate
3302
+ : kind === "events"
3303
+ ? this.jobsEventsPathTemplate
3304
+ : this.jobsResultPathTemplate;
3305
+ return `POST ${path}`;
3306
+ }
3307
+ lifecycleGatewayBaseUrl() {
3308
+ return this.gatewayBaseUrl;
3309
+ }
3310
+ lifecycleUrl(kind, jobId) {
3311
+ const path = kind === "poll"
3312
+ ? this.jobsPollPath
3313
+ : kind === "start"
3314
+ ? this.jobsStartPathTemplate
3315
+ : kind === "events"
3316
+ ? this.jobsEventsPathTemplate
3317
+ : this.jobsResultPathTemplate;
3318
+ return resolveLifecycleUrl(this.gatewayBaseUrl, jobId ? resolveLifecycleTemplate(path, jobId) : path);
3319
+ }
3213
3320
  async enroll(nodeId, enrollmentToken) {
3214
3321
  return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/enroll`, {
3215
3322
  method: "POST",
@@ -3261,7 +3368,7 @@ export class MswarmSelfHostedNodeClient {
3261
3368
  }, this.timeoutMs);
3262
3369
  }
3263
3370
  async pollJob(runtimeToken, payload) {
3264
- return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/poll`, {
3371
+ return fetchJson(this.fetchImpl, this.lifecycleUrl("poll"), {
3265
3372
  method: "POST",
3266
3373
  headers: {
3267
3374
  "content-type": "application/json",
@@ -3271,7 +3378,7 @@ export class MswarmSelfHostedNodeClient {
3271
3378
  }, Math.max(this.timeoutMs, (payload.wait_ms || 0) + 5000));
3272
3379
  }
3273
3380
  async postJobResult(runtimeToken, jobId, payload) {
3274
- return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/result`, {
3381
+ return fetchJson(this.fetchImpl, this.lifecycleUrl("result", jobId), {
3275
3382
  method: "POST",
3276
3383
  headers: {
3277
3384
  "content-type": "application/json",
@@ -3281,7 +3388,7 @@ export class MswarmSelfHostedNodeClient {
3281
3388
  }, this.timeoutMs);
3282
3389
  }
3283
3390
  async postJobStart(runtimeToken, jobId, payload) {
3284
- return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/start`, {
3391
+ return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
3285
3392
  method: "POST",
3286
3393
  headers: {
3287
3394
  "content-type": "application/json",
@@ -3291,7 +3398,7 @@ export class MswarmSelfHostedNodeClient {
3291
3398
  }, this.timeoutMs);
3292
3399
  }
3293
3400
  async postJobEvents(runtimeToken, jobId, payload) {
3294
- return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/jobs/${encodeURIComponent(jobId)}/events`, {
3401
+ return fetchJson(this.fetchImpl, this.lifecycleUrl("events", jobId), {
3295
3402
  method: "POST",
3296
3403
  headers: {
3297
3404
  "content-type": "application/json",
@@ -3309,11 +3416,16 @@ export class SelfHostedNodeRuntime {
3309
3416
  this.queuedGenericJobs = 0;
3310
3417
  this.latencySamplesMs = [];
3311
3418
  this.recentFailures = [];
3419
+ this.lifecyclePollingDisabled = false;
3312
3420
  this.config = config;
3313
3421
  this.gateway =
3314
3422
  deps?.gateway ||
3315
3423
  new MswarmSelfHostedNodeClient({
3316
3424
  gatewayBaseUrl: config.gatewayBaseUrl,
3425
+ jobsPollPath: config.jobsPollPath,
3426
+ jobsStartPathTemplate: config.jobsStartPathTemplate,
3427
+ jobsEventsPathTemplate: config.jobsEventsPathTemplate,
3428
+ jobsResultPathTemplate: config.jobsResultPathTemplate,
3317
3429
  fetchImpl: deps?.fetchImpl,
3318
3430
  timeoutMs: config.requestTimeoutMs
3319
3431
  });
@@ -3388,6 +3500,53 @@ export class SelfHostedNodeRuntime {
3388
3500
  this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
3389
3501
  }
3390
3502
  }
3503
+ lifecycleProtocolMismatch(kind, error) {
3504
+ if (!isHttpStatusError(error, 404)) {
3505
+ return undefined;
3506
+ }
3507
+ return new SelfHostedProtocolMismatchError({
3508
+ endpoint: this.gateway.lifecycleEndpoint(kind),
3509
+ gatewayBaseUrl: this.gateway.lifecycleGatewayBaseUrl(),
3510
+ runtimePackageVersion: this.config.nodeVersion
3511
+ });
3512
+ }
3513
+ isLifecycleProtocolDegradedState(state) {
3514
+ return state?.lifecycle_health_reason === SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
3515
+ }
3516
+ async readLifecycleState() {
3517
+ try {
3518
+ return await readSelfHostedNodeState(this.config.statePath);
3519
+ }
3520
+ catch {
3521
+ return {};
3522
+ }
3523
+ }
3524
+ async markLifecycleProtocolDegraded(error) {
3525
+ this.lifecyclePollingDisabled = true;
3526
+ this.recentFailures.unshift({
3527
+ execution_class: "agentic",
3528
+ code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
3529
+ at: new Date().toISOString()
3530
+ });
3531
+ this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
3532
+ const state = await this.readLifecycleState();
3533
+ await writeSelfHostedNodeState(this.config.statePath, {
3534
+ ...state,
3535
+ lifecycle_health_status: "degraded",
3536
+ lifecycle_health_reason: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
3537
+ lifecycle_health_message: error.message,
3538
+ lifecycle_health_updated_at: new Date().toISOString()
3539
+ });
3540
+ }
3541
+ jobResultPayload(result) {
3542
+ return {
3543
+ ...result,
3544
+ node_id: this.config.nodeId,
3545
+ ...(result.status === "success"
3546
+ ? { usage: result.usage || openAiUsage(result.openai_response) }
3547
+ : {})
3548
+ };
3549
+ }
3391
3550
  averageLatencyMs(fallback = null) {
3392
3551
  if (this.latencySamplesMs.length === 0) {
3393
3552
  return fallback;
@@ -3481,6 +3640,12 @@ export class SelfHostedNodeRuntime {
3481
3640
  throw new Error("Bootstrap response did not include node_id and runtime_token");
3482
3641
  }
3483
3642
  const heartbeatInterval = bootstrap.heartbeat_interval_seconds || setupConfig.heartbeatIntervalSeconds;
3643
+ const relay = bootstrap.relay || {};
3644
+ const relayGatewayBaseUrl = optionalText(relay.gateway_base_url) || setupConfig.gatewayBaseUrl;
3645
+ const jobsPollPath = lifecyclePath(relay.jobs_poll_path, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
3646
+ const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
3647
+ const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
3648
+ const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
3484
3649
  const state = {
3485
3650
  node_id: nodeId,
3486
3651
  server_name: optionalText(bootstrap.node?.server_name) || setupConfig.serverName,
@@ -3494,7 +3659,15 @@ export class SelfHostedNodeRuntime {
3494
3659
  heartbeat_timeout_seconds: bootstrap.heartbeat_timeout_seconds,
3495
3660
  enrolled_at: new Date().toISOString(),
3496
3661
  updated_at: new Date().toISOString(),
3497
- gateway_base_url: setupConfig.gatewayBaseUrl,
3662
+ gateway_base_url: trimTrailingSlash(relayGatewayBaseUrl),
3663
+ jobs_poll_path: jobsPollPath,
3664
+ jobs_start_path_template: jobsStartPathTemplate,
3665
+ jobs_events_path_template: jobsEventsPathTemplate,
3666
+ jobs_result_path_template: jobsResultPathTemplate,
3667
+ lifecycle_health_status: "healthy",
3668
+ lifecycle_health_reason: undefined,
3669
+ lifecycle_health_message: undefined,
3670
+ lifecycle_health_updated_at: new Date().toISOString(),
3498
3671
  ollama_base_url: setupConfig.ollamaBaseUrl,
3499
3672
  discovery_mode: setupConfig.discoveryMode,
3500
3673
  mcoda_bin: setupConfig.mcodaBin,
@@ -3518,8 +3691,23 @@ export class SelfHostedNodeRuntime {
3518
3691
  };
3519
3692
  await writeSelfHostedNodeState(setupConfig.statePath, state);
3520
3693
  await writeSelfHostedRuntimeToken(setupConfig.runtimeTokenPath, runtimeToken);
3694
+ const runtimeGateway = deps?.gateway && trimTrailingSlash(relayGatewayBaseUrl) === setupConfig.gatewayBaseUrl
3695
+ ? deps.gateway
3696
+ : new MswarmSelfHostedNodeClient({
3697
+ gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
3698
+ jobsPollPath,
3699
+ jobsStartPathTemplate,
3700
+ jobsEventsPathTemplate,
3701
+ jobsResultPathTemplate,
3702
+ fetchImpl: deps?.fetchImpl,
3703
+ timeoutMs: setupConfig.requestTimeoutMs
3704
+ });
3521
3705
  const runtime = new SelfHostedNodeRuntime({
3522
- gatewayBaseUrl: setupConfig.gatewayBaseUrl,
3706
+ gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
3707
+ jobsPollPath,
3708
+ jobsStartPathTemplate,
3709
+ jobsEventsPathTemplate,
3710
+ jobsResultPathTemplate,
3523
3711
  nodeId,
3524
3712
  serverName: state.server_name,
3525
3713
  relayMode: state.relay_mode || setupConfig.relayMode,
@@ -3553,7 +3741,7 @@ export class SelfHostedNodeRuntime {
3553
3741
  exposeAllModels: setupConfig.exposeAllModels,
3554
3742
  modelAllowlist: setupConfig.modelAllowlist,
3555
3743
  modelBlocklist: setupConfig.modelBlocklist
3556
- }, { ...deps, gateway });
3744
+ }, { ...deps, gateway: runtimeGateway });
3557
3745
  const once = await runtime.runOnce();
3558
3746
  return {
3559
3747
  created: bootstrap.created === true,
@@ -3639,6 +3827,12 @@ export class SelfHostedNodeRuntime {
3639
3827
  if (!runtimeToken) {
3640
3828
  throw new Error("Enrollment response did not include runtime_token");
3641
3829
  }
3830
+ const relay = response.relay || {};
3831
+ const gatewayBaseUrl = trimTrailingSlash(optionalText(relay.gateway_base_url) || this.config.gatewayBaseUrl);
3832
+ const jobsPollPath = lifecyclePath(relay.jobs_poll_path || this.config.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
3833
+ const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template || this.config.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
3834
+ const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template || this.config.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
3835
+ const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template || this.config.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
3642
3836
  const nextState = {
3643
3837
  ...currentState,
3644
3838
  node_id: this.config.nodeId,
@@ -3648,7 +3842,15 @@ export class SelfHostedNodeRuntime {
3648
3842
  heartbeat_timeout_seconds: response.heartbeat_timeout_seconds,
3649
3843
  enrolled_at: currentState.enrolled_at || new Date().toISOString(),
3650
3844
  updated_at: new Date().toISOString(),
3651
- gateway_base_url: this.config.gatewayBaseUrl,
3845
+ gateway_base_url: gatewayBaseUrl,
3846
+ jobs_poll_path: jobsPollPath,
3847
+ jobs_start_path_template: jobsStartPathTemplate,
3848
+ jobs_events_path_template: jobsEventsPathTemplate,
3849
+ jobs_result_path_template: jobsResultPathTemplate,
3850
+ lifecycle_health_status: "healthy",
3851
+ lifecycle_health_reason: undefined,
3852
+ lifecycle_health_message: undefined,
3853
+ lifecycle_health_updated_at: new Date().toISOString(),
3652
3854
  ollama_base_url: this.config.ollamaBaseUrl,
3653
3855
  discovery_mode: this.config.discoveryMode,
3654
3856
  mcoda_bin: this.config.mcodaBin,
@@ -3959,6 +4161,8 @@ export class SelfHostedNodeRuntime {
3959
4161
  ]
3960
4162
  });
3961
4163
  }
4164
+ const promptTokens = positiveInteger(ollamaResult.promptTokens);
4165
+ const completionTokens = positiveInteger(ollamaResult.completionTokens);
3962
4166
  const invocationResult = {
3963
4167
  job_id: job.job_id,
3964
4168
  request_id: job.request_id,
@@ -3967,10 +4171,11 @@ export class SelfHostedNodeRuntime {
3967
4171
  requestId: job.request_id,
3968
4172
  model: job.openai_request.model,
3969
4173
  content: ollamaResult.content,
3970
- promptTokens: ollamaResult.promptTokens,
3971
- completionTokens: ollamaResult.completionTokens,
4174
+ promptTokens,
4175
+ completionTokens,
3972
4176
  metadata: { provider: "ollama", raw: ollamaResult.raw }
3973
4177
  }),
4178
+ usage: usageFromTokenCounts(promptTokens, completionTokens),
3974
4179
  ...(streamEvents.length ? { stream_events: streamEvents } : {}),
3975
4180
  ...(progressEvents.length ? { progress_events: progressEvents } : {}),
3976
4181
  timing: { local_latency_ms: Date.now() - startedAt }
@@ -4051,6 +4256,7 @@ export class SelfHostedNodeRuntime {
4051
4256
  mode: response.metadata.mode
4052
4257
  }
4053
4258
  }),
4259
+ usage: usageFromTokenCounts(tokens.promptTokens, tokens.completionTokens),
4054
4260
  ...(streamEvents.length ? { stream_events: streamEvents } : {}),
4055
4261
  ...(progressEvents.length ? { progress_events: progressEvents } : {}),
4056
4262
  timing: { local_latency_ms: Date.now() - startedAt }
@@ -4117,6 +4323,13 @@ export class SelfHostedNodeRuntime {
4117
4323
  models = [];
4118
4324
  version = null;
4119
4325
  }
4326
+ const lifecycleState = await this.readLifecycleState();
4327
+ const lifecycleProtocolDegraded = this.isLifecycleProtocolDegradedState(lifecycleState);
4328
+ if (lifecycleProtocolDegraded) {
4329
+ status = "degraded";
4330
+ recentFailureCount = Math.max(recentFailureCount, 1);
4331
+ this.lifecyclePollingDisabled = true;
4332
+ }
4120
4333
  const discoveryLatencyMs = Date.now() - startedAt;
4121
4334
  const capabilityPayload = await this.buildCapabilityHeartbeatPayload(enrollment.runtimeToken);
4122
4335
  const loadTelemetry = this.buildLoadTelemetry({
@@ -4158,7 +4371,17 @@ export class SelfHostedNodeRuntime {
4158
4371
  relay_mode: this.config.relayMode || "outbound",
4159
4372
  load_reporting_enabled: loadReportingEnabled,
4160
4373
  hardware_telemetry_enabled: this.config.hardwareTelemetryEnabled === true,
4161
- drain_mode: this.config.drainMode === true
4374
+ drain_mode: this.config.drainMode === true,
4375
+ ...(lifecycleProtocolDegraded
4376
+ ? {
4377
+ lifecycle: {
4378
+ status: "degraded",
4379
+ reason: lifecycleState.lifecycle_health_reason,
4380
+ message: lifecycleState.lifecycle_health_message,
4381
+ updated_at: lifecycleState.lifecycle_health_updated_at
4382
+ }
4383
+ }
4384
+ : {})
4162
4385
  },
4163
4386
  discovery: {
4164
4387
  source: discoverySource,
@@ -4178,7 +4401,14 @@ export class SelfHostedNodeRuntime {
4178
4401
  avg_latency_ms: loadTelemetry.avg_latency_ms ?? discoveryLatencyMs,
4179
4402
  recent_failure_count: loadTelemetry.recent_failure_count,
4180
4403
  recent_failures: loadTelemetry.recent_failures,
4181
- last_success_at: status === "online" ? new Date().toISOString() : null
4404
+ last_success_at: status === "online" ? new Date().toISOString() : null,
4405
+ ...(lifecycleProtocolDegraded
4406
+ ? {
4407
+ lifecycle_status: "degraded",
4408
+ lifecycle_reason: lifecycleState.lifecycle_health_reason,
4409
+ lifecycle_message: lifecycleState.lifecycle_health_message
4410
+ }
4411
+ : {})
4182
4412
  },
4183
4413
  local_agent_catalog: {
4184
4414
  revision: loadTelemetry.catalog_fingerprint,
@@ -4232,20 +4462,36 @@ export class SelfHostedNodeRuntime {
4232
4462
  return { count: models.filter((model) => model.exposed !== false).length, response };
4233
4463
  }
4234
4464
  async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
4465
+ const lifecycleState = await this.readLifecycleState();
4466
+ if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
4467
+ this.lifecyclePollingDisabled = true;
4468
+ return { executed: false, status: "failed" };
4469
+ }
4235
4470
  const enrollment = await this.ensureEnrolled();
4236
4471
  const pollCapacity = this.buildLoadTelemetry({ models: [] });
4237
- const response = await this.gateway.pollJob(enrollment.runtimeToken, {
4238
- node_id: this.config.nodeId,
4239
- capacity: {
4240
- active_jobs: pollCapacity.active_jobs,
4241
- queued_jobs: pollCapacity.queued_jobs,
4242
- max_jobs: pollCapacity.max_concurrency,
4243
- max_concurrency: pollCapacity.max_concurrency,
4244
- free_slots: pollCapacity.free_slots,
4245
- drain_mode: pollCapacity.drain_mode
4246
- },
4247
- wait_ms: waitMs
4248
- });
4472
+ let response;
4473
+ try {
4474
+ response = await this.gateway.pollJob(enrollment.runtimeToken, {
4475
+ node_id: this.config.nodeId,
4476
+ capacity: {
4477
+ active_jobs: pollCapacity.active_jobs,
4478
+ queued_jobs: pollCapacity.queued_jobs,
4479
+ max_jobs: pollCapacity.max_concurrency,
4480
+ max_concurrency: pollCapacity.max_concurrency,
4481
+ free_slots: pollCapacity.free_slots,
4482
+ drain_mode: pollCapacity.drain_mode
4483
+ },
4484
+ wait_ms: waitMs
4485
+ });
4486
+ }
4487
+ catch (error) {
4488
+ const mismatch = this.lifecycleProtocolMismatch("poll", error);
4489
+ if (mismatch) {
4490
+ await this.markLifecycleProtocolDegraded(mismatch);
4491
+ return { executed: false, status: "failed" };
4492
+ }
4493
+ throw error;
4494
+ }
4249
4495
  const job = response.job || null;
4250
4496
  if (!job) {
4251
4497
  return { executed: false };
@@ -4265,19 +4511,35 @@ export class SelfHostedNodeRuntime {
4265
4511
  });
4266
4512
  forwardedStreamEventCount += stream_events.length;
4267
4513
  }
4268
- catch {
4514
+ catch (error) {
4515
+ const mismatch = this.lifecycleProtocolMismatch("events", error);
4516
+ if (mismatch) {
4517
+ streamEventForwardingFailed = true;
4518
+ await this.markLifecycleProtocolDegraded(mismatch);
4519
+ throw mismatch;
4520
+ }
4269
4521
  streamEventForwardingFailed = true;
4270
4522
  }
4271
4523
  };
4272
- const result = await this.executeJob(job, {
4524
+ let result = await this.executeJob(job, {
4273
4525
  attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined,
4274
4526
  onStarted: async (event) => {
4275
- await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
4276
- node_id: this.config.nodeId,
4277
- agent_slug: event.agent_slug || job.agent_slug,
4278
- source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
4279
- model: event.model || job.model || job.openai_request.model
4280
- });
4527
+ try {
4528
+ await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
4529
+ node_id: this.config.nodeId,
4530
+ agent_slug: event.agent_slug || job.agent_slug,
4531
+ source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
4532
+ model: event.model || job.model || job.openai_request.model
4533
+ });
4534
+ }
4535
+ catch (error) {
4536
+ const mismatch = this.lifecycleProtocolMismatch("start", error);
4537
+ if (mismatch) {
4538
+ await this.markLifecycleProtocolDegraded(mismatch);
4539
+ throw mismatch;
4540
+ }
4541
+ throw error;
4542
+ }
4281
4543
  },
4282
4544
  onOpenAIChunk: async (chunk) => {
4283
4545
  if (job.openai_request.stream !== true || streamEventForwardingFailed) {
@@ -4289,7 +4551,23 @@ export class SelfHostedNodeRuntime {
4289
4551
  }
4290
4552
  },
4291
4553
  });
4292
- await flushStreamEvents();
4554
+ try {
4555
+ await flushStreamEvents();
4556
+ }
4557
+ catch (error) {
4558
+ if (!(error instanceof SelfHostedProtocolMismatchError)) {
4559
+ throw error;
4560
+ }
4561
+ result = {
4562
+ job_id: job.job_id,
4563
+ request_id: job.request_id,
4564
+ status: "failed",
4565
+ error: { code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE, message: error.message },
4566
+ ...(result.stream_events?.length ? { stream_events: result.stream_events } : {}),
4567
+ ...(result.progress_events?.length ? { progress_events: result.progress_events } : {}),
4568
+ timing: result.timing
4569
+ };
4570
+ }
4293
4571
  const postedResult = job.openai_request.stream === true
4294
4572
  ? streamEventForwardingFailed
4295
4573
  ? {
@@ -4298,10 +4576,17 @@ export class SelfHostedNodeRuntime {
4298
4576
  }
4299
4577
  : (({ stream_events: _streamEvents, ...rest }) => rest)(result)
4300
4578
  : result;
4301
- await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, {
4302
- ...postedResult,
4303
- node_id: this.config.nodeId
4304
- });
4579
+ try {
4580
+ await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, this.jobResultPayload(postedResult));
4581
+ }
4582
+ catch (error) {
4583
+ const mismatch = this.lifecycleProtocolMismatch("result", error);
4584
+ if (mismatch) {
4585
+ await this.markLifecycleProtocolDegraded(mismatch);
4586
+ return { executed: true, job_id: job.job_id, status: "failed" };
4587
+ }
4588
+ throw error;
4589
+ }
4305
4590
  return { executed: true, job_id: job.job_id, status: result.status };
4306
4591
  }
4307
4592
  async doctor() {
@@ -4343,7 +4628,7 @@ export class SelfHostedNodeRuntime {
4343
4628
  let timer = null;
4344
4629
  let polling = false;
4345
4630
  const poll = () => {
4346
- if (stopped || polling || this.config.relayMode === "direct")
4631
+ if (stopped || polling || this.lifecyclePollingDisabled || this.config.relayMode === "direct")
4347
4632
  return;
4348
4633
  polling = true;
4349
4634
  void this.pollAndExecuteJob()