@mcoda/mswarm 0.1.79 → 0.1.81
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/runtime.js
CHANGED
|
@@ -26,7 +26,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
|
|
|
26
26
|
const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
|
|
27
27
|
const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
|
|
28
28
|
const DEFAULT_JOB_POLL_WAIT_MS = 25000;
|
|
29
|
+
const DEFAULT_SELF_HOSTED_JOBS_POLL_PATH = "/v1/swarm/self-hosted/node/jobs/poll";
|
|
30
|
+
const DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/start";
|
|
31
|
+
const DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/events";
|
|
32
|
+
const DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE = "/v1/swarm/self-hosted/node/jobs/:jobId/result";
|
|
29
33
|
const DEFAULT_STREAM_EVENT_BATCH_SIZE = 8;
|
|
34
|
+
const SELF_HOSTED_PROTOCOL_MISMATCH_CODE = "self_hosted_protocol_mismatch";
|
|
30
35
|
const OWNER_LOCAL_TEST_ECHO_JOB_TYPE = "tenant.test-echo";
|
|
31
36
|
const TEST_ECHO_RUNNER_ID = "test.echo";
|
|
32
37
|
const RENDER_BLENDER_JOB_TYPE = "render.blender";
|
|
@@ -402,6 +407,37 @@ function parseCliOptions(argv) {
|
|
|
402
407
|
function trimTrailingSlash(value) {
|
|
403
408
|
return value.replace(/\/+$/g, "");
|
|
404
409
|
}
|
|
410
|
+
function lifecyclePath(value, fallback) {
|
|
411
|
+
return optionalText(value) || fallback;
|
|
412
|
+
}
|
|
413
|
+
function resolveLifecycleUrl(gatewayBaseUrl, pathOrUrl) {
|
|
414
|
+
try {
|
|
415
|
+
return new URL(pathOrUrl).toString();
|
|
416
|
+
}
|
|
417
|
+
catch {
|
|
418
|
+
return new URL(pathOrUrl.startsWith("/") ? pathOrUrl : `/${pathOrUrl}`, `${trimTrailingSlash(gatewayBaseUrl)}/`).toString();
|
|
419
|
+
}
|
|
420
|
+
}
|
|
421
|
+
function resolveLifecycleTemplate(template, jobId) {
|
|
422
|
+
const encoded = encodeURIComponent(jobId);
|
|
423
|
+
return template
|
|
424
|
+
.replace(/:jobId\b/g, encoded)
|
|
425
|
+
.replace(/:job_id\b/g, encoded)
|
|
426
|
+
.replace(/\{jobId\}/g, encoded)
|
|
427
|
+
.replace(/\{job_id\}/g, encoded);
|
|
428
|
+
}
|
|
429
|
+
function usageFromTokenCounts(promptTokens, completionTokens) {
|
|
430
|
+
const prompt = positiveInteger(promptTokens) ?? 0;
|
|
431
|
+
const completion = positiveInteger(completionTokens) ?? 0;
|
|
432
|
+
return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: prompt + completion };
|
|
433
|
+
}
|
|
434
|
+
function openAiUsage(response) {
|
|
435
|
+
const usage = isRecord(response?.usage) ? response.usage : undefined;
|
|
436
|
+
const prompt = positiveInteger(usage?.prompt_tokens) ?? 0;
|
|
437
|
+
const completion = positiveInteger(usage?.completion_tokens) ?? 0;
|
|
438
|
+
const total = positiveInteger(usage?.total_tokens) ?? prompt + completion;
|
|
439
|
+
return { prompt_tokens: prompt, completion_tokens: completion, total_tokens: total };
|
|
440
|
+
}
|
|
405
441
|
function defaultStatePath() {
|
|
406
442
|
return join(homedir(), ".mswarm", "self-hosted-node", "config.json");
|
|
407
443
|
}
|
|
@@ -592,12 +628,19 @@ function normalizeHealthStatus(value) {
|
|
|
592
628
|
return "blocked";
|
|
593
629
|
return "unknown";
|
|
594
630
|
}
|
|
595
|
-
function
|
|
631
|
+
function isMswarmManagedAgent(agent) {
|
|
596
632
|
const config = agent.config && typeof agent.config === "object" ? agent.config : {};
|
|
597
|
-
const
|
|
598
|
-
|
|
599
|
-
typeof
|
|
600
|
-
|
|
633
|
+
const managedConfigs = [config.mswarmCloud, config.mswarmSelfHosted, config.mswarmWorker];
|
|
634
|
+
if (managedConfigs.some((managedConfig) => Boolean(managedConfig) &&
|
|
635
|
+
typeof managedConfig === "object" &&
|
|
636
|
+
managedConfig.managed === true)) {
|
|
637
|
+
return true;
|
|
638
|
+
}
|
|
639
|
+
const slug = optionalText(agent.slug)?.toLowerCase();
|
|
640
|
+
return Boolean(slug &&
|
|
641
|
+
(slug.startsWith("mswarm-cloud-") ||
|
|
642
|
+
slug.startsWith("mswarm-self-hosted-") ||
|
|
643
|
+
slug.startsWith("mswarm-worker-")));
|
|
601
644
|
}
|
|
602
645
|
function isModelExposed(modelName, family, config) {
|
|
603
646
|
if (config.modelBlocklist.includes(modelName)) {
|
|
@@ -624,6 +667,18 @@ function isAgentExposed(agentSlug, defaultModel, bestUsage, config) {
|
|
|
624
667
|
}
|
|
625
668
|
return config.exposeAllModels;
|
|
626
669
|
}
|
|
670
|
+
class HttpStatusError extends Error {
|
|
671
|
+
constructor(url, status, responseText) {
|
|
672
|
+
super(`request_failed:${status}:${responseText.slice(0, 200)}`);
|
|
673
|
+
this.name = "HttpStatusError";
|
|
674
|
+
this.url = url;
|
|
675
|
+
this.status = status;
|
|
676
|
+
this.responseText = responseText;
|
|
677
|
+
}
|
|
678
|
+
}
|
|
679
|
+
function isHttpStatusError(error, status) {
|
|
680
|
+
return error instanceof HttpStatusError && (status === undefined || error.status === status);
|
|
681
|
+
}
|
|
627
682
|
async function fetchJson(fetchImpl, url, init, timeoutMs) {
|
|
628
683
|
const controller = new AbortController();
|
|
629
684
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
@@ -631,7 +686,7 @@ async function fetchJson(fetchImpl, url, init, timeoutMs) {
|
|
|
631
686
|
const response = await fetchImpl(url, { ...init, signal: controller.signal });
|
|
632
687
|
if (!response.ok) {
|
|
633
688
|
const text = await response.text().catch(() => "");
|
|
634
|
-
throw new
|
|
689
|
+
throw new HttpStatusError(url, response.status, text);
|
|
635
690
|
}
|
|
636
691
|
return (await response.json());
|
|
637
692
|
}
|
|
@@ -1161,6 +1216,18 @@ export async function readSelfHostedNodeConfig(env = process.env) {
|
|
|
1161
1216
|
const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
|
|
1162
1217
|
return {
|
|
1163
1218
|
gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
|
|
1219
|
+
jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
|
|
1220
|
+
state.jobs_poll_path ||
|
|
1221
|
+
DEFAULT_SELF_HOSTED_JOBS_POLL_PATH,
|
|
1222
|
+
jobsStartPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_START_PATH_TEMPLATE) ||
|
|
1223
|
+
state.jobs_start_path_template ||
|
|
1224
|
+
DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE,
|
|
1225
|
+
jobsEventsPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE) ||
|
|
1226
|
+
state.jobs_events_path_template ||
|
|
1227
|
+
DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE,
|
|
1228
|
+
jobsResultPathTemplate: optionalText(env.MSWARM_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE) ||
|
|
1229
|
+
state.jobs_result_path_template ||
|
|
1230
|
+
DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE,
|
|
1164
1231
|
nodeId,
|
|
1165
1232
|
serverName: optionalText(env.MSWARM_SELF_HOSTED_SERVER_NAME) || state.server_name || null,
|
|
1166
1233
|
relayMode: parseRelayMode(env.MSWARM_SELF_HOSTED_RELAY_MODE || state.relay_mode),
|
|
@@ -1303,7 +1370,7 @@ export function mapOllamaModelToSelfHostedModel(model, config) {
|
|
|
1303
1370
|
};
|
|
1304
1371
|
}
|
|
1305
1372
|
export function mapMcodaAgentToSelfHostedModel(agent, config) {
|
|
1306
|
-
if (
|
|
1373
|
+
if (isMswarmManagedAgent(agent)) {
|
|
1307
1374
|
return null;
|
|
1308
1375
|
}
|
|
1309
1376
|
const slug = optionalText(agent.slug);
|
|
@@ -1684,6 +1751,7 @@ const PRE_START_JOB_ERROR_CODES = new Set([
|
|
|
1684
1751
|
"validation_failed",
|
|
1685
1752
|
"docdex_context_missing",
|
|
1686
1753
|
"docdex_api_key_missing",
|
|
1754
|
+
SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
|
|
1687
1755
|
]);
|
|
1688
1756
|
class SelfHostedDocdexJobError extends Error {
|
|
1689
1757
|
constructor(code, message) {
|
|
@@ -1699,6 +1767,18 @@ class SelfHostedPreStartJobError extends Error {
|
|
|
1699
1767
|
this.code = code;
|
|
1700
1768
|
}
|
|
1701
1769
|
}
|
|
1770
|
+
class SelfHostedProtocolMismatchError extends Error {
|
|
1771
|
+
constructor(input) {
|
|
1772
|
+
super(`Gateway is missing lifecycle endpoint ${input.endpoint} at ${input.gatewayBaseUrl} ` +
|
|
1773
|
+
`(runtime package version ${input.runtimePackageVersion}). ` +
|
|
1774
|
+
"Run a direct local agent/model instead, for example qwen3.6-llama.cpp.");
|
|
1775
|
+
this.code = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
|
|
1776
|
+
this.name = SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
|
|
1777
|
+
this.endpoint = input.endpoint;
|
|
1778
|
+
this.gatewayBaseUrl = input.gatewayBaseUrl;
|
|
1779
|
+
this.runtimePackageVersion = input.runtimePackageVersion;
|
|
1780
|
+
}
|
|
1781
|
+
}
|
|
1702
1782
|
function normalizeDocdexCapabilityMap(value) {
|
|
1703
1783
|
const record = objectRecord(value);
|
|
1704
1784
|
if (!record)
|
|
@@ -3207,9 +3287,36 @@ export class McodaLocalAgentExecutor {
|
|
|
3207
3287
|
export class MswarmSelfHostedNodeClient {
|
|
3208
3288
|
constructor(input) {
|
|
3209
3289
|
this.gatewayBaseUrl = trimTrailingSlash(input.gatewayBaseUrl);
|
|
3290
|
+
this.jobsPollPath = lifecyclePath(input.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
|
|
3291
|
+
this.jobsStartPathTemplate = lifecyclePath(input.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
|
|
3292
|
+
this.jobsEventsPathTemplate = lifecyclePath(input.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
|
|
3293
|
+
this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
|
|
3210
3294
|
this.fetchImpl = input.fetchImpl || fetch;
|
|
3211
3295
|
this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
|
|
3212
3296
|
}
|
|
3297
|
+
lifecycleEndpoint(kind) {
|
|
3298
|
+
const path = kind === "poll"
|
|
3299
|
+
? this.jobsPollPath
|
|
3300
|
+
: kind === "start"
|
|
3301
|
+
? this.jobsStartPathTemplate
|
|
3302
|
+
: kind === "events"
|
|
3303
|
+
? this.jobsEventsPathTemplate
|
|
3304
|
+
: this.jobsResultPathTemplate;
|
|
3305
|
+
return `POST ${path}`;
|
|
3306
|
+
}
|
|
3307
|
+
lifecycleGatewayBaseUrl() {
|
|
3308
|
+
return this.gatewayBaseUrl;
|
|
3309
|
+
}
|
|
3310
|
+
lifecycleUrl(kind, jobId) {
|
|
3311
|
+
const path = kind === "poll"
|
|
3312
|
+
? this.jobsPollPath
|
|
3313
|
+
: kind === "start"
|
|
3314
|
+
? this.jobsStartPathTemplate
|
|
3315
|
+
: kind === "events"
|
|
3316
|
+
? this.jobsEventsPathTemplate
|
|
3317
|
+
: this.jobsResultPathTemplate;
|
|
3318
|
+
return resolveLifecycleUrl(this.gatewayBaseUrl, jobId ? resolveLifecycleTemplate(path, jobId) : path);
|
|
3319
|
+
}
|
|
3213
3320
|
async enroll(nodeId, enrollmentToken) {
|
|
3214
3321
|
return fetchJson(this.fetchImpl, `${this.gatewayBaseUrl}/v1/swarm/self-hosted/node/enroll`, {
|
|
3215
3322
|
method: "POST",
|
|
@@ -3261,7 +3368,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
3261
3368
|
}, this.timeoutMs);
|
|
3262
3369
|
}
|
|
3263
3370
|
async pollJob(runtimeToken, payload) {
|
|
3264
|
-
return fetchJson(this.fetchImpl,
|
|
3371
|
+
return fetchJson(this.fetchImpl, this.lifecycleUrl("poll"), {
|
|
3265
3372
|
method: "POST",
|
|
3266
3373
|
headers: {
|
|
3267
3374
|
"content-type": "application/json",
|
|
@@ -3271,7 +3378,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
3271
3378
|
}, Math.max(this.timeoutMs, (payload.wait_ms || 0) + 5000));
|
|
3272
3379
|
}
|
|
3273
3380
|
async postJobResult(runtimeToken, jobId, payload) {
|
|
3274
|
-
return fetchJson(this.fetchImpl,
|
|
3381
|
+
return fetchJson(this.fetchImpl, this.lifecycleUrl("result", jobId), {
|
|
3275
3382
|
method: "POST",
|
|
3276
3383
|
headers: {
|
|
3277
3384
|
"content-type": "application/json",
|
|
@@ -3281,7 +3388,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
3281
3388
|
}, this.timeoutMs);
|
|
3282
3389
|
}
|
|
3283
3390
|
async postJobStart(runtimeToken, jobId, payload) {
|
|
3284
|
-
return fetchJson(this.fetchImpl,
|
|
3391
|
+
return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
|
|
3285
3392
|
method: "POST",
|
|
3286
3393
|
headers: {
|
|
3287
3394
|
"content-type": "application/json",
|
|
@@ -3291,7 +3398,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
3291
3398
|
}, this.timeoutMs);
|
|
3292
3399
|
}
|
|
3293
3400
|
async postJobEvents(runtimeToken, jobId, payload) {
|
|
3294
|
-
return fetchJson(this.fetchImpl,
|
|
3401
|
+
return fetchJson(this.fetchImpl, this.lifecycleUrl("events", jobId), {
|
|
3295
3402
|
method: "POST",
|
|
3296
3403
|
headers: {
|
|
3297
3404
|
"content-type": "application/json",
|
|
@@ -3309,11 +3416,16 @@ export class SelfHostedNodeRuntime {
|
|
|
3309
3416
|
this.queuedGenericJobs = 0;
|
|
3310
3417
|
this.latencySamplesMs = [];
|
|
3311
3418
|
this.recentFailures = [];
|
|
3419
|
+
this.lifecyclePollingDisabled = false;
|
|
3312
3420
|
this.config = config;
|
|
3313
3421
|
this.gateway =
|
|
3314
3422
|
deps?.gateway ||
|
|
3315
3423
|
new MswarmSelfHostedNodeClient({
|
|
3316
3424
|
gatewayBaseUrl: config.gatewayBaseUrl,
|
|
3425
|
+
jobsPollPath: config.jobsPollPath,
|
|
3426
|
+
jobsStartPathTemplate: config.jobsStartPathTemplate,
|
|
3427
|
+
jobsEventsPathTemplate: config.jobsEventsPathTemplate,
|
|
3428
|
+
jobsResultPathTemplate: config.jobsResultPathTemplate,
|
|
3317
3429
|
fetchImpl: deps?.fetchImpl,
|
|
3318
3430
|
timeoutMs: config.requestTimeoutMs
|
|
3319
3431
|
});
|
|
@@ -3388,6 +3500,53 @@ export class SelfHostedNodeRuntime {
|
|
|
3388
3500
|
this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
|
|
3389
3501
|
}
|
|
3390
3502
|
}
|
|
3503
|
+
lifecycleProtocolMismatch(kind, error) {
|
|
3504
|
+
if (!isHttpStatusError(error, 404)) {
|
|
3505
|
+
return undefined;
|
|
3506
|
+
}
|
|
3507
|
+
return new SelfHostedProtocolMismatchError({
|
|
3508
|
+
endpoint: this.gateway.lifecycleEndpoint(kind),
|
|
3509
|
+
gatewayBaseUrl: this.gateway.lifecycleGatewayBaseUrl(),
|
|
3510
|
+
runtimePackageVersion: this.config.nodeVersion
|
|
3511
|
+
});
|
|
3512
|
+
}
|
|
3513
|
+
isLifecycleProtocolDegradedState(state) {
|
|
3514
|
+
return state?.lifecycle_health_reason === SELF_HOSTED_PROTOCOL_MISMATCH_CODE;
|
|
3515
|
+
}
|
|
3516
|
+
async readLifecycleState() {
|
|
3517
|
+
try {
|
|
3518
|
+
return await readSelfHostedNodeState(this.config.statePath);
|
|
3519
|
+
}
|
|
3520
|
+
catch {
|
|
3521
|
+
return {};
|
|
3522
|
+
}
|
|
3523
|
+
}
|
|
3524
|
+
async markLifecycleProtocolDegraded(error) {
|
|
3525
|
+
this.lifecyclePollingDisabled = true;
|
|
3526
|
+
this.recentFailures.unshift({
|
|
3527
|
+
execution_class: "agentic",
|
|
3528
|
+
code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
|
|
3529
|
+
at: new Date().toISOString()
|
|
3530
|
+
});
|
|
3531
|
+
this.recentFailures.splice(MAX_TELEMETRY_FAILURES);
|
|
3532
|
+
const state = await this.readLifecycleState();
|
|
3533
|
+
await writeSelfHostedNodeState(this.config.statePath, {
|
|
3534
|
+
...state,
|
|
3535
|
+
lifecycle_health_status: "degraded",
|
|
3536
|
+
lifecycle_health_reason: SELF_HOSTED_PROTOCOL_MISMATCH_CODE,
|
|
3537
|
+
lifecycle_health_message: error.message,
|
|
3538
|
+
lifecycle_health_updated_at: new Date().toISOString()
|
|
3539
|
+
});
|
|
3540
|
+
}
|
|
3541
|
+
jobResultPayload(result) {
|
|
3542
|
+
return {
|
|
3543
|
+
...result,
|
|
3544
|
+
node_id: this.config.nodeId,
|
|
3545
|
+
...(result.status === "success"
|
|
3546
|
+
? { usage: result.usage || openAiUsage(result.openai_response) }
|
|
3547
|
+
: {})
|
|
3548
|
+
};
|
|
3549
|
+
}
|
|
3391
3550
|
averageLatencyMs(fallback = null) {
|
|
3392
3551
|
if (this.latencySamplesMs.length === 0) {
|
|
3393
3552
|
return fallback;
|
|
@@ -3481,6 +3640,12 @@ export class SelfHostedNodeRuntime {
|
|
|
3481
3640
|
throw new Error("Bootstrap response did not include node_id and runtime_token");
|
|
3482
3641
|
}
|
|
3483
3642
|
const heartbeatInterval = bootstrap.heartbeat_interval_seconds || setupConfig.heartbeatIntervalSeconds;
|
|
3643
|
+
const relay = bootstrap.relay || {};
|
|
3644
|
+
const relayGatewayBaseUrl = optionalText(relay.gateway_base_url) || setupConfig.gatewayBaseUrl;
|
|
3645
|
+
const jobsPollPath = lifecyclePath(relay.jobs_poll_path, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
|
|
3646
|
+
const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
|
|
3647
|
+
const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
|
|
3648
|
+
const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
|
|
3484
3649
|
const state = {
|
|
3485
3650
|
node_id: nodeId,
|
|
3486
3651
|
server_name: optionalText(bootstrap.node?.server_name) || setupConfig.serverName,
|
|
@@ -3494,7 +3659,15 @@ export class SelfHostedNodeRuntime {
|
|
|
3494
3659
|
heartbeat_timeout_seconds: bootstrap.heartbeat_timeout_seconds,
|
|
3495
3660
|
enrolled_at: new Date().toISOString(),
|
|
3496
3661
|
updated_at: new Date().toISOString(),
|
|
3497
|
-
gateway_base_url:
|
|
3662
|
+
gateway_base_url: trimTrailingSlash(relayGatewayBaseUrl),
|
|
3663
|
+
jobs_poll_path: jobsPollPath,
|
|
3664
|
+
jobs_start_path_template: jobsStartPathTemplate,
|
|
3665
|
+
jobs_events_path_template: jobsEventsPathTemplate,
|
|
3666
|
+
jobs_result_path_template: jobsResultPathTemplate,
|
|
3667
|
+
lifecycle_health_status: "healthy",
|
|
3668
|
+
lifecycle_health_reason: undefined,
|
|
3669
|
+
lifecycle_health_message: undefined,
|
|
3670
|
+
lifecycle_health_updated_at: new Date().toISOString(),
|
|
3498
3671
|
ollama_base_url: setupConfig.ollamaBaseUrl,
|
|
3499
3672
|
discovery_mode: setupConfig.discoveryMode,
|
|
3500
3673
|
mcoda_bin: setupConfig.mcodaBin,
|
|
@@ -3518,8 +3691,23 @@ export class SelfHostedNodeRuntime {
|
|
|
3518
3691
|
};
|
|
3519
3692
|
await writeSelfHostedNodeState(setupConfig.statePath, state);
|
|
3520
3693
|
await writeSelfHostedRuntimeToken(setupConfig.runtimeTokenPath, runtimeToken);
|
|
3694
|
+
const runtimeGateway = deps?.gateway && trimTrailingSlash(relayGatewayBaseUrl) === setupConfig.gatewayBaseUrl
|
|
3695
|
+
? deps.gateway
|
|
3696
|
+
: new MswarmSelfHostedNodeClient({
|
|
3697
|
+
gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
|
|
3698
|
+
jobsPollPath,
|
|
3699
|
+
jobsStartPathTemplate,
|
|
3700
|
+
jobsEventsPathTemplate,
|
|
3701
|
+
jobsResultPathTemplate,
|
|
3702
|
+
fetchImpl: deps?.fetchImpl,
|
|
3703
|
+
timeoutMs: setupConfig.requestTimeoutMs
|
|
3704
|
+
});
|
|
3521
3705
|
const runtime = new SelfHostedNodeRuntime({
|
|
3522
|
-
gatewayBaseUrl:
|
|
3706
|
+
gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
|
|
3707
|
+
jobsPollPath,
|
|
3708
|
+
jobsStartPathTemplate,
|
|
3709
|
+
jobsEventsPathTemplate,
|
|
3710
|
+
jobsResultPathTemplate,
|
|
3523
3711
|
nodeId,
|
|
3524
3712
|
serverName: state.server_name,
|
|
3525
3713
|
relayMode: state.relay_mode || setupConfig.relayMode,
|
|
@@ -3553,7 +3741,7 @@ export class SelfHostedNodeRuntime {
|
|
|
3553
3741
|
exposeAllModels: setupConfig.exposeAllModels,
|
|
3554
3742
|
modelAllowlist: setupConfig.modelAllowlist,
|
|
3555
3743
|
modelBlocklist: setupConfig.modelBlocklist
|
|
3556
|
-
}, { ...deps, gateway });
|
|
3744
|
+
}, { ...deps, gateway: runtimeGateway });
|
|
3557
3745
|
const once = await runtime.runOnce();
|
|
3558
3746
|
return {
|
|
3559
3747
|
created: bootstrap.created === true,
|
|
@@ -3639,6 +3827,12 @@ export class SelfHostedNodeRuntime {
|
|
|
3639
3827
|
if (!runtimeToken) {
|
|
3640
3828
|
throw new Error("Enrollment response did not include runtime_token");
|
|
3641
3829
|
}
|
|
3830
|
+
const relay = response.relay || {};
|
|
3831
|
+
const gatewayBaseUrl = trimTrailingSlash(optionalText(relay.gateway_base_url) || this.config.gatewayBaseUrl);
|
|
3832
|
+
const jobsPollPath = lifecyclePath(relay.jobs_poll_path || this.config.jobsPollPath, DEFAULT_SELF_HOSTED_JOBS_POLL_PATH);
|
|
3833
|
+
const jobsStartPathTemplate = lifecyclePath(relay.jobs_start_path_template || this.config.jobsStartPathTemplate, DEFAULT_SELF_HOSTED_JOBS_START_PATH_TEMPLATE);
|
|
3834
|
+
const jobsEventsPathTemplate = lifecyclePath(relay.jobs_events_path_template || this.config.jobsEventsPathTemplate, DEFAULT_SELF_HOSTED_JOBS_EVENTS_PATH_TEMPLATE);
|
|
3835
|
+
const jobsResultPathTemplate = lifecyclePath(relay.jobs_result_path_template || this.config.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
|
|
3642
3836
|
const nextState = {
|
|
3643
3837
|
...currentState,
|
|
3644
3838
|
node_id: this.config.nodeId,
|
|
@@ -3648,7 +3842,15 @@ export class SelfHostedNodeRuntime {
|
|
|
3648
3842
|
heartbeat_timeout_seconds: response.heartbeat_timeout_seconds,
|
|
3649
3843
|
enrolled_at: currentState.enrolled_at || new Date().toISOString(),
|
|
3650
3844
|
updated_at: new Date().toISOString(),
|
|
3651
|
-
gateway_base_url:
|
|
3845
|
+
gateway_base_url: gatewayBaseUrl,
|
|
3846
|
+
jobs_poll_path: jobsPollPath,
|
|
3847
|
+
jobs_start_path_template: jobsStartPathTemplate,
|
|
3848
|
+
jobs_events_path_template: jobsEventsPathTemplate,
|
|
3849
|
+
jobs_result_path_template: jobsResultPathTemplate,
|
|
3850
|
+
lifecycle_health_status: "healthy",
|
|
3851
|
+
lifecycle_health_reason: undefined,
|
|
3852
|
+
lifecycle_health_message: undefined,
|
|
3853
|
+
lifecycle_health_updated_at: new Date().toISOString(),
|
|
3652
3854
|
ollama_base_url: this.config.ollamaBaseUrl,
|
|
3653
3855
|
discovery_mode: this.config.discoveryMode,
|
|
3654
3856
|
mcoda_bin: this.config.mcodaBin,
|
|
@@ -3959,6 +4161,8 @@ export class SelfHostedNodeRuntime {
|
|
|
3959
4161
|
]
|
|
3960
4162
|
});
|
|
3961
4163
|
}
|
|
4164
|
+
const promptTokens = positiveInteger(ollamaResult.promptTokens);
|
|
4165
|
+
const completionTokens = positiveInteger(ollamaResult.completionTokens);
|
|
3962
4166
|
const invocationResult = {
|
|
3963
4167
|
job_id: job.job_id,
|
|
3964
4168
|
request_id: job.request_id,
|
|
@@ -3967,10 +4171,11 @@ export class SelfHostedNodeRuntime {
|
|
|
3967
4171
|
requestId: job.request_id,
|
|
3968
4172
|
model: job.openai_request.model,
|
|
3969
4173
|
content: ollamaResult.content,
|
|
3970
|
-
promptTokens
|
|
3971
|
-
completionTokens
|
|
4174
|
+
promptTokens,
|
|
4175
|
+
completionTokens,
|
|
3972
4176
|
metadata: { provider: "ollama", raw: ollamaResult.raw }
|
|
3973
4177
|
}),
|
|
4178
|
+
usage: usageFromTokenCounts(promptTokens, completionTokens),
|
|
3974
4179
|
...(streamEvents.length ? { stream_events: streamEvents } : {}),
|
|
3975
4180
|
...(progressEvents.length ? { progress_events: progressEvents } : {}),
|
|
3976
4181
|
timing: { local_latency_ms: Date.now() - startedAt }
|
|
@@ -4051,6 +4256,7 @@ export class SelfHostedNodeRuntime {
|
|
|
4051
4256
|
mode: response.metadata.mode
|
|
4052
4257
|
}
|
|
4053
4258
|
}),
|
|
4259
|
+
usage: usageFromTokenCounts(tokens.promptTokens, tokens.completionTokens),
|
|
4054
4260
|
...(streamEvents.length ? { stream_events: streamEvents } : {}),
|
|
4055
4261
|
...(progressEvents.length ? { progress_events: progressEvents } : {}),
|
|
4056
4262
|
timing: { local_latency_ms: Date.now() - startedAt }
|
|
@@ -4117,6 +4323,13 @@ export class SelfHostedNodeRuntime {
|
|
|
4117
4323
|
models = [];
|
|
4118
4324
|
version = null;
|
|
4119
4325
|
}
|
|
4326
|
+
const lifecycleState = await this.readLifecycleState();
|
|
4327
|
+
const lifecycleProtocolDegraded = this.isLifecycleProtocolDegradedState(lifecycleState);
|
|
4328
|
+
if (lifecycleProtocolDegraded) {
|
|
4329
|
+
status = "degraded";
|
|
4330
|
+
recentFailureCount = Math.max(recentFailureCount, 1);
|
|
4331
|
+
this.lifecyclePollingDisabled = true;
|
|
4332
|
+
}
|
|
4120
4333
|
const discoveryLatencyMs = Date.now() - startedAt;
|
|
4121
4334
|
const capabilityPayload = await this.buildCapabilityHeartbeatPayload(enrollment.runtimeToken);
|
|
4122
4335
|
const loadTelemetry = this.buildLoadTelemetry({
|
|
@@ -4158,7 +4371,17 @@ export class SelfHostedNodeRuntime {
|
|
|
4158
4371
|
relay_mode: this.config.relayMode || "outbound",
|
|
4159
4372
|
load_reporting_enabled: loadReportingEnabled,
|
|
4160
4373
|
hardware_telemetry_enabled: this.config.hardwareTelemetryEnabled === true,
|
|
4161
|
-
drain_mode: this.config.drainMode === true
|
|
4374
|
+
drain_mode: this.config.drainMode === true,
|
|
4375
|
+
...(lifecycleProtocolDegraded
|
|
4376
|
+
? {
|
|
4377
|
+
lifecycle: {
|
|
4378
|
+
status: "degraded",
|
|
4379
|
+
reason: lifecycleState.lifecycle_health_reason,
|
|
4380
|
+
message: lifecycleState.lifecycle_health_message,
|
|
4381
|
+
updated_at: lifecycleState.lifecycle_health_updated_at
|
|
4382
|
+
}
|
|
4383
|
+
}
|
|
4384
|
+
: {})
|
|
4162
4385
|
},
|
|
4163
4386
|
discovery: {
|
|
4164
4387
|
source: discoverySource,
|
|
@@ -4178,7 +4401,14 @@ export class SelfHostedNodeRuntime {
|
|
|
4178
4401
|
avg_latency_ms: loadTelemetry.avg_latency_ms ?? discoveryLatencyMs,
|
|
4179
4402
|
recent_failure_count: loadTelemetry.recent_failure_count,
|
|
4180
4403
|
recent_failures: loadTelemetry.recent_failures,
|
|
4181
|
-
last_success_at: status === "online" ? new Date().toISOString() : null
|
|
4404
|
+
last_success_at: status === "online" ? new Date().toISOString() : null,
|
|
4405
|
+
...(lifecycleProtocolDegraded
|
|
4406
|
+
? {
|
|
4407
|
+
lifecycle_status: "degraded",
|
|
4408
|
+
lifecycle_reason: lifecycleState.lifecycle_health_reason,
|
|
4409
|
+
lifecycle_message: lifecycleState.lifecycle_health_message
|
|
4410
|
+
}
|
|
4411
|
+
: {})
|
|
4182
4412
|
},
|
|
4183
4413
|
local_agent_catalog: {
|
|
4184
4414
|
revision: loadTelemetry.catalog_fingerprint,
|
|
@@ -4232,20 +4462,36 @@ export class SelfHostedNodeRuntime {
|
|
|
4232
4462
|
return { count: models.filter((model) => model.exposed !== false).length, response };
|
|
4233
4463
|
}
|
|
4234
4464
|
async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
|
|
4465
|
+
const lifecycleState = await this.readLifecycleState();
|
|
4466
|
+
if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
|
|
4467
|
+
this.lifecyclePollingDisabled = true;
|
|
4468
|
+
return { executed: false, status: "failed" };
|
|
4469
|
+
}
|
|
4235
4470
|
const enrollment = await this.ensureEnrolled();
|
|
4236
4471
|
const pollCapacity = this.buildLoadTelemetry({ models: [] });
|
|
4237
|
-
|
|
4238
|
-
|
|
4239
|
-
|
|
4240
|
-
|
|
4241
|
-
|
|
4242
|
-
|
|
4243
|
-
|
|
4244
|
-
|
|
4245
|
-
|
|
4246
|
-
|
|
4247
|
-
|
|
4248
|
-
|
|
4472
|
+
let response;
|
|
4473
|
+
try {
|
|
4474
|
+
response = await this.gateway.pollJob(enrollment.runtimeToken, {
|
|
4475
|
+
node_id: this.config.nodeId,
|
|
4476
|
+
capacity: {
|
|
4477
|
+
active_jobs: pollCapacity.active_jobs,
|
|
4478
|
+
queued_jobs: pollCapacity.queued_jobs,
|
|
4479
|
+
max_jobs: pollCapacity.max_concurrency,
|
|
4480
|
+
max_concurrency: pollCapacity.max_concurrency,
|
|
4481
|
+
free_slots: pollCapacity.free_slots,
|
|
4482
|
+
drain_mode: pollCapacity.drain_mode
|
|
4483
|
+
},
|
|
4484
|
+
wait_ms: waitMs
|
|
4485
|
+
});
|
|
4486
|
+
}
|
|
4487
|
+
catch (error) {
|
|
4488
|
+
const mismatch = this.lifecycleProtocolMismatch("poll", error);
|
|
4489
|
+
if (mismatch) {
|
|
4490
|
+
await this.markLifecycleProtocolDegraded(mismatch);
|
|
4491
|
+
return { executed: false, status: "failed" };
|
|
4492
|
+
}
|
|
4493
|
+
throw error;
|
|
4494
|
+
}
|
|
4249
4495
|
const job = response.job || null;
|
|
4250
4496
|
if (!job) {
|
|
4251
4497
|
return { executed: false };
|
|
@@ -4265,19 +4511,35 @@ export class SelfHostedNodeRuntime {
|
|
|
4265
4511
|
});
|
|
4266
4512
|
forwardedStreamEventCount += stream_events.length;
|
|
4267
4513
|
}
|
|
4268
|
-
catch {
|
|
4514
|
+
catch (error) {
|
|
4515
|
+
const mismatch = this.lifecycleProtocolMismatch("events", error);
|
|
4516
|
+
if (mismatch) {
|
|
4517
|
+
streamEventForwardingFailed = true;
|
|
4518
|
+
await this.markLifecycleProtocolDegraded(mismatch);
|
|
4519
|
+
throw mismatch;
|
|
4520
|
+
}
|
|
4269
4521
|
streamEventForwardingFailed = true;
|
|
4270
4522
|
}
|
|
4271
4523
|
};
|
|
4272
|
-
|
|
4524
|
+
let result = await this.executeJob(job, {
|
|
4273
4525
|
attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined,
|
|
4274
4526
|
onStarted: async (event) => {
|
|
4275
|
-
|
|
4276
|
-
|
|
4277
|
-
|
|
4278
|
-
|
|
4279
|
-
|
|
4280
|
-
|
|
4527
|
+
try {
|
|
4528
|
+
await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
|
|
4529
|
+
node_id: this.config.nodeId,
|
|
4530
|
+
agent_slug: event.agent_slug || job.agent_slug,
|
|
4531
|
+
source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
|
|
4532
|
+
model: event.model || job.model || job.openai_request.model
|
|
4533
|
+
});
|
|
4534
|
+
}
|
|
4535
|
+
catch (error) {
|
|
4536
|
+
const mismatch = this.lifecycleProtocolMismatch("start", error);
|
|
4537
|
+
if (mismatch) {
|
|
4538
|
+
await this.markLifecycleProtocolDegraded(mismatch);
|
|
4539
|
+
throw mismatch;
|
|
4540
|
+
}
|
|
4541
|
+
throw error;
|
|
4542
|
+
}
|
|
4281
4543
|
},
|
|
4282
4544
|
onOpenAIChunk: async (chunk) => {
|
|
4283
4545
|
if (job.openai_request.stream !== true || streamEventForwardingFailed) {
|
|
@@ -4289,7 +4551,23 @@ export class SelfHostedNodeRuntime {
|
|
|
4289
4551
|
}
|
|
4290
4552
|
},
|
|
4291
4553
|
});
|
|
4292
|
-
|
|
4554
|
+
try {
|
|
4555
|
+
await flushStreamEvents();
|
|
4556
|
+
}
|
|
4557
|
+
catch (error) {
|
|
4558
|
+
if (!(error instanceof SelfHostedProtocolMismatchError)) {
|
|
4559
|
+
throw error;
|
|
4560
|
+
}
|
|
4561
|
+
result = {
|
|
4562
|
+
job_id: job.job_id,
|
|
4563
|
+
request_id: job.request_id,
|
|
4564
|
+
status: "failed",
|
|
4565
|
+
error: { code: SELF_HOSTED_PROTOCOL_MISMATCH_CODE, message: error.message },
|
|
4566
|
+
...(result.stream_events?.length ? { stream_events: result.stream_events } : {}),
|
|
4567
|
+
...(result.progress_events?.length ? { progress_events: result.progress_events } : {}),
|
|
4568
|
+
timing: result.timing
|
|
4569
|
+
};
|
|
4570
|
+
}
|
|
4293
4571
|
const postedResult = job.openai_request.stream === true
|
|
4294
4572
|
? streamEventForwardingFailed
|
|
4295
4573
|
? {
|
|
@@ -4298,10 +4576,17 @@ export class SelfHostedNodeRuntime {
|
|
|
4298
4576
|
}
|
|
4299
4577
|
: (({ stream_events: _streamEvents, ...rest }) => rest)(result)
|
|
4300
4578
|
: result;
|
|
4301
|
-
|
|
4302
|
-
|
|
4303
|
-
|
|
4304
|
-
|
|
4579
|
+
try {
|
|
4580
|
+
await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, this.jobResultPayload(postedResult));
|
|
4581
|
+
}
|
|
4582
|
+
catch (error) {
|
|
4583
|
+
const mismatch = this.lifecycleProtocolMismatch("result", error);
|
|
4584
|
+
if (mismatch) {
|
|
4585
|
+
await this.markLifecycleProtocolDegraded(mismatch);
|
|
4586
|
+
return { executed: true, job_id: job.job_id, status: "failed" };
|
|
4587
|
+
}
|
|
4588
|
+
throw error;
|
|
4589
|
+
}
|
|
4305
4590
|
return { executed: true, job_id: job.job_id, status: result.status };
|
|
4306
4591
|
}
|
|
4307
4592
|
async doctor() {
|
|
@@ -4343,7 +4628,7 @@ export class SelfHostedNodeRuntime {
|
|
|
4343
4628
|
let timer = null;
|
|
4344
4629
|
let polling = false;
|
|
4345
4630
|
const poll = () => {
|
|
4346
|
-
if (stopped || polling || this.config.relayMode === "direct")
|
|
4631
|
+
if (stopped || polling || this.lifecyclePollingDisabled || this.config.relayMode === "direct")
|
|
4347
4632
|
return;
|
|
4348
4633
|
polling = true;
|
|
4349
4634
|
void this.pollAndExecuteJob()
|