@mcoda/mswarm 0.1.107 → 0.1.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -0
- package/dist/runtime.d.ts +16 -5
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +237 -34
- package/dist/runtime.js.map +1 -1
- package/dist/server.d.ts.map +1 -1
- package/dist/server.js +4 -0
- package/dist/server.js.map +1 -1
- package/package.json +3 -3
package/dist/runtime.js
CHANGED
|
@@ -62,6 +62,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30;
|
|
|
62
62
|
const DEFAULT_SELF_HOSTED_NODE_VERSION = "0.1.70";
|
|
63
63
|
const DEFAULT_REQUEST_TIMEOUT_MS = 10000;
|
|
64
64
|
const DEFAULT_JOB_TIMEOUT_MS = 3600000;
|
|
65
|
+
const DEFAULT_RESULT_POST_TIMEOUT_MS = 5 * 60000;
|
|
65
66
|
const DEFAULT_SERVICE_COMMAND_TIMEOUT_MS = 60000;
|
|
66
67
|
const DEFAULT_CAPABILITY_PROBE_TIMEOUT_MS = 2000;
|
|
67
68
|
const SELF_HOSTED_RUNTIME_PROTOCOL_VERSION = 1;
|
|
@@ -74,6 +75,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
|
|
|
74
75
|
const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
|
|
75
76
|
const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
|
|
76
77
|
const DEFAULT_JOB_POLL_WAIT_MS = 2000;
|
|
78
|
+
const RESERVED_JOB_POLL_WAIT_MS = 250;
|
|
79
|
+
const MAX_POLL_EXCLUDED_AGENT_SLUGS = 64;
|
|
80
|
+
const MAX_RELAY_AGENT_SLUG_LENGTH = 255;
|
|
81
|
+
const DEFAULT_RESERVED_PRIORITY_MAX = -1;
|
|
82
|
+
const MIN_RELAY_PRIORITY = -100;
|
|
83
|
+
const MAX_RELAY_PRIORITY = 100;
|
|
77
84
|
const JOB_POLL_RETRY_BASE_MS = 1000;
|
|
78
85
|
const JOB_POLL_RETRY_MAX_MS = 60000;
|
|
79
86
|
/**
|
|
@@ -284,6 +291,33 @@ const LOCAL_RUNNER_RESPONSE_FORMAT_ALIASES = {
|
|
|
284
291
|
function optionalText(value) {
|
|
285
292
|
return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
|
|
286
293
|
}
|
|
294
|
+
function relayJobAgentAliases(job) {
|
|
295
|
+
const aliases = [job.agent_slug, job.source_agent_slug, job.remote_slug]
|
|
296
|
+
.map(optionalText)
|
|
297
|
+
.map((value) => value
|
|
298
|
+
?.toLowerCase()
|
|
299
|
+
.replace(/[^a-z0-9]+/g, "-")
|
|
300
|
+
.replace(/-+/g, "-")
|
|
301
|
+
.replace(/^-|-$/g, "") || null)
|
|
302
|
+
.filter((value) => typeof value === "string" && value.length > 0 && value.length <= MAX_RELAY_AGENT_SLUG_LENGTH);
|
|
303
|
+
return [...new Set(aliases)];
|
|
304
|
+
}
|
|
305
|
+
function boundedRelayAgentAliases(activeAliases) {
|
|
306
|
+
const aliases = [];
|
|
307
|
+
const seen = new Set();
|
|
308
|
+
for (const group of activeAliases) {
|
|
309
|
+
for (const alias of group) {
|
|
310
|
+
if (seen.has(alias))
|
|
311
|
+
continue;
|
|
312
|
+
seen.add(alias);
|
|
313
|
+
aliases.push(alias);
|
|
314
|
+
if (aliases.length >= MAX_POLL_EXCLUDED_AGENT_SLUGS) {
|
|
315
|
+
return aliases;
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
return aliases;
|
|
320
|
+
}
|
|
287
321
|
function isRecord(value) {
|
|
288
322
|
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
289
323
|
}
|
|
@@ -588,6 +622,39 @@ function parsePositiveInteger(value, fallback) {
|
|
|
588
622
|
}
|
|
589
623
|
return Math.floor(parsed);
|
|
590
624
|
}
|
|
625
|
+
function parseNonNegativeInteger(value, fallback) {
|
|
626
|
+
const parsed = value === undefined || value === null || value === ""
|
|
627
|
+
? fallback
|
|
628
|
+
: typeof value === "number"
|
|
629
|
+
? value
|
|
630
|
+
: typeof value === "string"
|
|
631
|
+
? Number(value)
|
|
632
|
+
: Number.NaN;
|
|
633
|
+
if (!Number.isInteger(parsed) || parsed < 0) {
|
|
634
|
+
throw new Error("MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS must be a non-negative integer");
|
|
635
|
+
}
|
|
636
|
+
return parsed;
|
|
637
|
+
}
|
|
638
|
+
function parseRelayPriority(value, fallback) {
|
|
639
|
+
const parsed = value === undefined || value === null || value === ""
|
|
640
|
+
? fallback
|
|
641
|
+
: typeof value === "number"
|
|
642
|
+
? value
|
|
643
|
+
: typeof value === "string"
|
|
644
|
+
? Number(value)
|
|
645
|
+
: Number.NaN;
|
|
646
|
+
if (!Number.isInteger(parsed) || parsed < MIN_RELAY_PRIORITY || parsed > MAX_RELAY_PRIORITY) {
|
|
647
|
+
throw new Error(`MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX must be an integer from ${MIN_RELAY_PRIORITY} to ${MAX_RELAY_PRIORITY}`);
|
|
648
|
+
}
|
|
649
|
+
return parsed;
|
|
650
|
+
}
|
|
651
|
+
function assertReservedLlmCapacity(input) {
|
|
652
|
+
const totalLlmJobs = input.maxConcurrentLlmJobs + input.reservedLlmJobs;
|
|
653
|
+
if (totalLlmJobs > input.maxConcurrentJobs) {
|
|
654
|
+
throw new Error("MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS must be greater than or equal to " +
|
|
655
|
+
"MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS plus MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS");
|
|
656
|
+
}
|
|
657
|
+
}
|
|
591
658
|
function parseBoolean(value, fallback) {
|
|
592
659
|
if (typeof value === "boolean")
|
|
593
660
|
return value;
|
|
@@ -1280,6 +1347,8 @@ function serviceEnvironment(config, env, homeDir) {
|
|
|
1280
1347
|
MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS: String(config.jobTimeoutMs),
|
|
1281
1348
|
MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS: String(config.maxConcurrentJobs || 1),
|
|
1282
1349
|
MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS: String(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1),
|
|
1350
|
+
MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS: String(config.reservedLlmJobs ?? 0),
|
|
1351
|
+
MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX: String(config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX),
|
|
1283
1352
|
MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED: config.genericJobsEnabled ? "true" : "false",
|
|
1284
1353
|
MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS: String(config.genericJobTimeoutMs),
|
|
1285
1354
|
MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY: String(config.genericJobMaxConcurrency),
|
|
@@ -1695,6 +1764,9 @@ export async function readSelfHostedNodeConfig(env = process.env) {
|
|
|
1695
1764
|
const packageNodeVersion = await readPackageNodeVersion();
|
|
1696
1765
|
const maxConcurrentJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, state.max_concurrent_jobs || 1);
|
|
1697
1766
|
const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
|
|
1767
|
+
const reservedLlmJobs = parseNonNegativeInteger(env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, state.reserved_llm_jobs ?? 0);
|
|
1768
|
+
const reservedPriorityMax = parseRelayPriority(env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, state.reserved_priority_max ?? DEFAULT_RESERVED_PRIORITY_MAX);
|
|
1769
|
+
assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
|
|
1698
1770
|
return {
|
|
1699
1771
|
gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
|
|
1700
1772
|
jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
|
|
@@ -1738,6 +1810,8 @@ export async function readSelfHostedNodeConfig(env = process.env) {
|
|
|
1738
1810
|
jobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
|
|
1739
1811
|
maxConcurrentJobs,
|
|
1740
1812
|
maxConcurrentLlmJobs,
|
|
1813
|
+
reservedLlmJobs,
|
|
1814
|
+
reservedPriorityMax,
|
|
1741
1815
|
genericJobsEnabled: parseBoolean(env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED ?? env.MSWARM_SELF_HOSTED_GENERIC_JOBS, state.generic_jobs_enabled === true),
|
|
1742
1816
|
genericJobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, state.generic_job_timeout_ms || state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
|
|
1743
1817
|
genericJobMaxConcurrency: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, state.generic_job_max_concurrency || 1),
|
|
@@ -1787,6 +1861,9 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
|
|
|
1787
1861
|
const packageNodeVersion = await readPackageNodeVersion();
|
|
1788
1862
|
const maxConcurrentJobs = parsePositiveInteger(options["max-concurrent-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, 1);
|
|
1789
1863
|
const maxConcurrentLlmJobs = parsePositiveInteger(options["max-concurrent-llm-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, maxConcurrentJobs);
|
|
1864
|
+
const reservedLlmJobs = parseNonNegativeInteger(options["reserved-llm-jobs"] ?? env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, 0);
|
|
1865
|
+
const reservedPriorityMax = parseRelayPriority(options["reserved-priority-max"] ?? env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, DEFAULT_RESERVED_PRIORITY_MAX);
|
|
1866
|
+
assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
|
|
1790
1867
|
return {
|
|
1791
1868
|
apiKey,
|
|
1792
1869
|
handle: handle || undefined,
|
|
@@ -1812,6 +1889,8 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
|
|
|
1812
1889
|
jobTimeoutMs: parsePositiveInteger(options["job-timeout-ms"] || env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
|
|
1813
1890
|
maxConcurrentJobs,
|
|
1814
1891
|
maxConcurrentLlmJobs,
|
|
1892
|
+
reservedLlmJobs,
|
|
1893
|
+
reservedPriorityMax,
|
|
1815
1894
|
genericJobsEnabled: parseBoolean(options["enable-generic-jobs"] || env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED || env.MSWARM_SELF_HOSTED_GENERIC_JOBS, false),
|
|
1816
1895
|
genericJobTimeoutMs: parsePositiveInteger(options["generic-job-timeout-ms"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
|
|
1817
1896
|
genericJobMaxConcurrency: parsePositiveInteger(options["generic-job-max-concurrency"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, 1),
|
|
@@ -4100,6 +4179,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
4100
4179
|
this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
|
|
4101
4180
|
this.fetchImpl = input.fetchImpl || fetch;
|
|
4102
4181
|
this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
|
|
4182
|
+
this.resultTimeoutMs = Math.max(this.timeoutMs, input.resultTimeoutMs || DEFAULT_RESULT_POST_TIMEOUT_MS);
|
|
4103
4183
|
}
|
|
4104
4184
|
lifecycleEndpoint(kind) {
|
|
4105
4185
|
const path = kind === "poll"
|
|
@@ -4214,7 +4294,7 @@ export class MswarmSelfHostedNodeClient {
|
|
|
4214
4294
|
authorization: `Bearer ${runtimeToken}`
|
|
4215
4295
|
},
|
|
4216
4296
|
body: JSON.stringify(payload)
|
|
4217
|
-
}, this.
|
|
4297
|
+
}, this.resultTimeoutMs);
|
|
4218
4298
|
}
|
|
4219
4299
|
async postJobStart(runtimeToken, jobId, payload) {
|
|
4220
4300
|
return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
|
|
@@ -4246,6 +4326,12 @@ export class SelfHostedNodeRuntime {
|
|
|
4246
4326
|
this.latencySamplesMs = [];
|
|
4247
4327
|
this.recentFailures = [];
|
|
4248
4328
|
this.lifecyclePollingDisabled = false;
|
|
4329
|
+
assertReservedLlmCapacity({
|
|
4330
|
+
maxConcurrentJobs: Math.max(1, Math.floor(config.maxConcurrentJobs || 1)),
|
|
4331
|
+
maxConcurrentLlmJobs: Math.max(1, Math.floor(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1)),
|
|
4332
|
+
reservedLlmJobs: parseNonNegativeInteger(config.reservedLlmJobs, 0)
|
|
4333
|
+
});
|
|
4334
|
+
parseRelayPriority(config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
|
|
4249
4335
|
this.config = config;
|
|
4250
4336
|
this.fetchImpl = deps?.fetchImpl || fetch;
|
|
4251
4337
|
this.useDedicatedGenerativeDispatcher = deps?.fetchImpl === undefined;
|
|
@@ -4258,7 +4344,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4258
4344
|
jobsEventsPathTemplate: config.jobsEventsPathTemplate,
|
|
4259
4345
|
jobsResultPathTemplate: config.jobsResultPathTemplate,
|
|
4260
4346
|
fetchImpl: this.fetchImpl,
|
|
4261
|
-
timeoutMs: config.requestTimeoutMs
|
|
4347
|
+
timeoutMs: config.requestTimeoutMs,
|
|
4348
|
+
resultTimeoutMs: Math.min(config.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
|
|
4262
4349
|
});
|
|
4263
4350
|
this.mcoda =
|
|
4264
4351
|
deps?.mcoda ||
|
|
@@ -4448,7 +4535,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4448
4535
|
new MswarmSelfHostedNodeClient({
|
|
4449
4536
|
gatewayBaseUrl: setupConfig.gatewayBaseUrl,
|
|
4450
4537
|
fetchImpl: deps?.fetchImpl,
|
|
4451
|
-
timeoutMs: setupConfig.requestTimeoutMs
|
|
4538
|
+
timeoutMs: setupConfig.requestTimeoutMs,
|
|
4539
|
+
resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
|
|
4452
4540
|
});
|
|
4453
4541
|
const machineId = await readOrCreateSelfHostedMachineId(setupConfig.machineIdPath);
|
|
4454
4542
|
const machineFingerprint = machineFingerprintFromId(machineId);
|
|
@@ -4517,6 +4605,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4517
4605
|
job_timeout_ms: setupConfig.jobTimeoutMs,
|
|
4518
4606
|
max_concurrent_jobs: setupConfig.maxConcurrentJobs,
|
|
4519
4607
|
max_concurrent_llm_jobs: setupConfig.maxConcurrentLlmJobs,
|
|
4608
|
+
reserved_llm_jobs: setupConfig.reservedLlmJobs,
|
|
4609
|
+
reserved_priority_max: setupConfig.reservedPriorityMax,
|
|
4520
4610
|
generic_jobs_enabled: setupConfig.genericJobsEnabled,
|
|
4521
4611
|
generic_job_timeout_ms: setupConfig.genericJobTimeoutMs,
|
|
4522
4612
|
generic_job_max_concurrency: setupConfig.genericJobMaxConcurrency,
|
|
@@ -4541,7 +4631,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4541
4631
|
jobsEventsPathTemplate,
|
|
4542
4632
|
jobsResultPathTemplate,
|
|
4543
4633
|
fetchImpl: deps?.fetchImpl,
|
|
4544
|
-
timeoutMs: setupConfig.requestTimeoutMs
|
|
4634
|
+
timeoutMs: setupConfig.requestTimeoutMs,
|
|
4635
|
+
resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
|
|
4545
4636
|
});
|
|
4546
4637
|
const runtime = new SelfHostedNodeRuntime({
|
|
4547
4638
|
gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
|
|
@@ -4572,6 +4663,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4572
4663
|
jobTimeoutMs: setupConfig.jobTimeoutMs,
|
|
4573
4664
|
maxConcurrentJobs: setupConfig.maxConcurrentJobs,
|
|
4574
4665
|
maxConcurrentLlmJobs: setupConfig.maxConcurrentLlmJobs,
|
|
4666
|
+
reservedLlmJobs: setupConfig.reservedLlmJobs,
|
|
4667
|
+
reservedPriorityMax: setupConfig.reservedPriorityMax,
|
|
4575
4668
|
genericJobsEnabled: setupConfig.genericJobsEnabled,
|
|
4576
4669
|
genericJobTimeoutMs: setupConfig.genericJobTimeoutMs,
|
|
4577
4670
|
genericJobMaxConcurrency: setupConfig.genericJobMaxConcurrency,
|
|
@@ -4702,6 +4795,8 @@ export class SelfHostedNodeRuntime {
|
|
|
4702
4795
|
job_timeout_ms: this.config.jobTimeoutMs,
|
|
4703
4796
|
max_concurrent_jobs: this.config.maxConcurrentJobs,
|
|
4704
4797
|
max_concurrent_llm_jobs: this.config.maxConcurrentLlmJobs,
|
|
4798
|
+
reserved_llm_jobs: this.config.reservedLlmJobs ?? 0,
|
|
4799
|
+
reserved_priority_max: this.config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX,
|
|
4705
4800
|
generic_jobs_enabled: this.config.genericJobsEnabled,
|
|
4706
4801
|
generic_job_timeout_ms: this.config.genericJobTimeoutMs,
|
|
4707
4802
|
generic_job_max_concurrency: this.config.genericJobMaxConcurrency,
|
|
@@ -5853,11 +5948,11 @@ export class SelfHostedNodeRuntime {
|
|
|
5853
5948
|
});
|
|
5854
5949
|
return { count: models.filter((model) => model.exposed !== false).length, response };
|
|
5855
5950
|
}
|
|
5856
|
-
async
|
|
5951
|
+
async pollRelayJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS, options = {}) {
|
|
5857
5952
|
const lifecycleState = await this.readLifecycleState();
|
|
5858
5953
|
if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
|
|
5859
5954
|
this.lifecyclePollingDisabled = true;
|
|
5860
|
-
return {
|
|
5955
|
+
return { claim: null, status: "failed" };
|
|
5861
5956
|
}
|
|
5862
5957
|
const enrollment = await this.ensureEnrolled();
|
|
5863
5958
|
const pollCapacity = this.buildLoadTelemetry({ models: [] });
|
|
@@ -5871,7 +5966,17 @@ export class SelfHostedNodeRuntime {
|
|
|
5871
5966
|
max_jobs: pollCapacity.max_concurrency,
|
|
5872
5967
|
max_concurrency: pollCapacity.max_concurrency,
|
|
5873
5968
|
free_slots: pollCapacity.free_slots,
|
|
5874
|
-
drain_mode: pollCapacity.drain_mode
|
|
5969
|
+
drain_mode: pollCapacity.drain_mode,
|
|
5970
|
+
...(options.excludedAgentSlugs !== undefined
|
|
5971
|
+
? {
|
|
5972
|
+
excluded_agent_slugs: boundedRelayAgentAliases([
|
|
5973
|
+
options.excludedAgentSlugs
|
|
5974
|
+
])
|
|
5975
|
+
}
|
|
5976
|
+
: {}),
|
|
5977
|
+
...(options.maxRequestedPriority !== undefined
|
|
5978
|
+
? { max_requested_priority: options.maxRequestedPriority }
|
|
5979
|
+
: {})
|
|
5875
5980
|
},
|
|
5876
5981
|
wait_ms: waitMs
|
|
5877
5982
|
});
|
|
@@ -5880,14 +5985,24 @@ export class SelfHostedNodeRuntime {
|
|
|
5880
5985
|
const mismatch = this.lifecycleProtocolMismatch("poll", error);
|
|
5881
5986
|
if (mismatch) {
|
|
5882
5987
|
await this.markLifecycleProtocolDegraded(mismatch);
|
|
5883
|
-
return {
|
|
5988
|
+
return { claim: null, status: "failed" };
|
|
5884
5989
|
}
|
|
5885
5990
|
throw error;
|
|
5886
5991
|
}
|
|
5887
5992
|
const job = response.job || null;
|
|
5888
5993
|
if (!job) {
|
|
5889
|
-
return {
|
|
5994
|
+
return { claim: null };
|
|
5890
5995
|
}
|
|
5996
|
+
return {
|
|
5997
|
+
claim: {
|
|
5998
|
+
runtimeToken: enrollment.runtimeToken,
|
|
5999
|
+
job,
|
|
6000
|
+
attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined
|
|
6001
|
+
}
|
|
6002
|
+
};
|
|
6003
|
+
}
|
|
6004
|
+
async executeRelayJobClaim(claim) {
|
|
6005
|
+
const { job, runtimeToken } = claim;
|
|
5891
6006
|
const chatStreaming = (normalizeSelfHostedGenerativeOperationName(job.operation) ?? "chat.completions") ===
|
|
5892
6007
|
"chat.completions" &&
|
|
5893
6008
|
job.openai_request.stream === true;
|
|
@@ -5900,7 +6015,7 @@ export class SelfHostedNodeRuntime {
|
|
|
5900
6015
|
}
|
|
5901
6016
|
const stream_events = pendingStreamEvents.splice(0, pendingStreamEvents.length);
|
|
5902
6017
|
try {
|
|
5903
|
-
await this.gateway.postJobEvents(
|
|
6018
|
+
await this.gateway.postJobEvents(runtimeToken, job.job_id, {
|
|
5904
6019
|
node_id: this.config.nodeId,
|
|
5905
6020
|
stream_events
|
|
5906
6021
|
});
|
|
@@ -5917,10 +6032,10 @@ export class SelfHostedNodeRuntime {
|
|
|
5917
6032
|
}
|
|
5918
6033
|
};
|
|
5919
6034
|
let result = await this.executeJob(job, {
|
|
5920
|
-
attachedMswarmApiKey:
|
|
6035
|
+
attachedMswarmApiKey: claim.attachedMswarmApiKey,
|
|
5921
6036
|
onStarted: async (event) => {
|
|
5922
6037
|
try {
|
|
5923
|
-
await this.gateway.postJobStart(
|
|
6038
|
+
await this.gateway.postJobStart(runtimeToken, job.job_id, {
|
|
5924
6039
|
node_id: this.config.nodeId,
|
|
5925
6040
|
agent_slug: event.agent_slug || job.agent_slug,
|
|
5926
6041
|
source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
|
|
@@ -5975,7 +6090,7 @@ export class SelfHostedNodeRuntime {
|
|
|
5975
6090
|
let resultPostError = null;
|
|
5976
6091
|
for (let attempt = 0; attempt <= RELAY_RESULT_POST_RETRY_DELAYS_MS.length; attempt += 1) {
|
|
5977
6092
|
try {
|
|
5978
|
-
await this.gateway.postJobResult(
|
|
6093
|
+
await this.gateway.postJobResult(runtimeToken, job.job_id, this.jobResultPayload(postedResult));
|
|
5979
6094
|
resultPostError = null;
|
|
5980
6095
|
break;
|
|
5981
6096
|
}
|
|
@@ -6003,6 +6118,13 @@ export class SelfHostedNodeRuntime {
|
|
|
6003
6118
|
}
|
|
6004
6119
|
return { executed: true, job_id: job.job_id, status: result.status };
|
|
6005
6120
|
}
|
|
6121
|
+
async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
|
|
6122
|
+
const polled = await this.pollRelayJob(waitMs);
|
|
6123
|
+
if (!polled.claim) {
|
|
6124
|
+
return { executed: false, ...(polled.status ? { status: polled.status } : {}) };
|
|
6125
|
+
}
|
|
6126
|
+
return this.executeRelayJobClaim(polled.claim);
|
|
6127
|
+
}
|
|
6006
6128
|
async doctor() {
|
|
6007
6129
|
const checks = [];
|
|
6008
6130
|
checks.push({ name: "config", ok: Boolean(this.config.nodeId), message: this.config.nodeId || "missing node id" });
|
|
@@ -6041,19 +6163,33 @@ export class SelfHostedNodeRuntime {
|
|
|
6041
6163
|
let stopped = false;
|
|
6042
6164
|
let timer = null;
|
|
6043
6165
|
let pollTimer = null;
|
|
6166
|
+
let pollTimerDueAt = 0;
|
|
6044
6167
|
let polling = false;
|
|
6045
|
-
let
|
|
6168
|
+
let relayPollFailureStreak = 0;
|
|
6169
|
+
let relayExecutionFailureStreak = 0;
|
|
6170
|
+
let activeRelayExecutions = 0;
|
|
6171
|
+
let activeRegularRelayExecutions = 0;
|
|
6172
|
+
let relayExecutionSequence = 0;
|
|
6173
|
+
const activeRelayAgentAliases = new Map();
|
|
6046
6174
|
let revoked = false;
|
|
6175
|
+
const regularRelayExecutionLimit = Math.max(1, Math.floor(this.config.maxConcurrentLlmJobs || this.config.maxConcurrentJobs || 1));
|
|
6176
|
+
const reservedRelayExecutionLimit = parseNonNegativeInteger(this.config.reservedLlmJobs, 0);
|
|
6177
|
+
const reservedPriorityMax = parseRelayPriority(this.config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
|
|
6178
|
+
const relayExecutionLimit = regularRelayExecutionLimit + reservedRelayExecutionLimit;
|
|
6179
|
+
const clearPollTimer = () => {
|
|
6180
|
+
if (pollTimer) {
|
|
6181
|
+
clearTimeout(pollTimer);
|
|
6182
|
+
pollTimer = null;
|
|
6183
|
+
}
|
|
6184
|
+
pollTimerDueAt = 0;
|
|
6185
|
+
};
|
|
6047
6186
|
const halt = () => {
|
|
6048
6187
|
stopped = true;
|
|
6049
6188
|
if (timer) {
|
|
6050
6189
|
clearTimeout(timer);
|
|
6051
6190
|
timer = null;
|
|
6052
6191
|
}
|
|
6053
|
-
|
|
6054
|
-
clearTimeout(pollTimer);
|
|
6055
|
-
pollTimer = null;
|
|
6056
|
-
}
|
|
6192
|
+
clearPollTimer();
|
|
6057
6193
|
};
|
|
6058
6194
|
// 410 means the gateway has retired this node's credentials: it was removed there.
|
|
6059
6195
|
// Retrying at full rate achieves nothing but a hot loop and a flooded journal, so
|
|
@@ -6067,10 +6203,7 @@ export class SelfHostedNodeRuntime {
|
|
|
6067
6203
|
if (!isHttpStatusError(error, 410)) {
|
|
6068
6204
|
return false;
|
|
6069
6205
|
}
|
|
6070
|
-
|
|
6071
|
-
clearTimeout(pollTimer);
|
|
6072
|
-
pollTimer = null;
|
|
6073
|
-
}
|
|
6206
|
+
clearPollTimer();
|
|
6074
6207
|
if (!revoked) {
|
|
6075
6208
|
revoked = true;
|
|
6076
6209
|
console.error(`[mswarm] this node was removed at the gateway (node ${this.config.nodeId}). ` +
|
|
@@ -6081,42 +6214,112 @@ export class SelfHostedNodeRuntime {
|
|
|
6081
6214
|
return true;
|
|
6082
6215
|
};
|
|
6083
6216
|
const noteAlive = () => {
|
|
6084
|
-
|
|
6217
|
+
const recoveredFromRevocation = revoked;
|
|
6218
|
+
const relayPollingIsBackedOff = (relayPollFailureStreak > 0 || relayExecutionFailureStreak > 0) && !polling;
|
|
6219
|
+
if (!recoveredFromRevocation && !relayPollingIsBackedOff) {
|
|
6220
|
+
return;
|
|
6221
|
+
}
|
|
6222
|
+
if (recoveredFromRevocation) {
|
|
6223
|
+
revoked = false;
|
|
6224
|
+
console.error(`[mswarm] node ${this.config.nodeId} is accepted at the gateway again; resuming normal operation.`);
|
|
6225
|
+
}
|
|
6226
|
+
clearPollTimer();
|
|
6227
|
+
relayPollFailureStreak = 0;
|
|
6228
|
+
relayExecutionFailureStreak = 0;
|
|
6229
|
+
schedulePoll(0);
|
|
6230
|
+
};
|
|
6231
|
+
const currentRelayRetryDelayMs = () => Math.max(pollRetryDelayMs(relayPollFailureStreak), pollRetryDelayMs(relayExecutionFailureStreak));
|
|
6232
|
+
const schedulePoll = (delayMs) => {
|
|
6233
|
+
if (stopped ||
|
|
6234
|
+
polling ||
|
|
6235
|
+
revoked ||
|
|
6236
|
+
this.lifecyclePollingDisabled ||
|
|
6237
|
+
this.config.relayMode === "direct" ||
|
|
6238
|
+
activeRelayExecutions >= relayExecutionLimit) {
|
|
6239
|
+
return;
|
|
6240
|
+
}
|
|
6241
|
+
const dueAt = Date.now() + Math.max(0, delayMs);
|
|
6242
|
+
if (pollTimer && pollTimerDueAt >= dueAt) {
|
|
6085
6243
|
return;
|
|
6086
6244
|
}
|
|
6087
|
-
|
|
6088
|
-
|
|
6089
|
-
|
|
6245
|
+
clearPollTimer();
|
|
6246
|
+
pollTimerDueAt = dueAt;
|
|
6247
|
+
pollTimer = setTimeout(() => {
|
|
6248
|
+
pollTimer = null;
|
|
6249
|
+
pollTimerDueAt = 0;
|
|
6250
|
+
poll();
|
|
6251
|
+
}, Math.max(0, dueAt - Date.now()));
|
|
6090
6252
|
};
|
|
6091
6253
|
const poll = () => {
|
|
6092
6254
|
if (stopped ||
|
|
6093
6255
|
polling ||
|
|
6094
6256
|
revoked ||
|
|
6095
6257
|
this.lifecyclePollingDisabled ||
|
|
6096
|
-
this.config.relayMode === "direct"
|
|
6258
|
+
this.config.relayMode === "direct" ||
|
|
6259
|
+
activeRelayExecutions >= relayExecutionLimit) {
|
|
6097
6260
|
return;
|
|
6098
6261
|
}
|
|
6099
6262
|
polling = true;
|
|
6100
|
-
|
|
6101
|
-
|
|
6102
|
-
|
|
6263
|
+
const reservedOnly = reservedRelayExecutionLimit > 0 &&
|
|
6264
|
+
activeRegularRelayExecutions >= regularRelayExecutionLimit &&
|
|
6265
|
+
activeRelayExecutions < relayExecutionLimit;
|
|
6266
|
+
const excludedAgentSlugs = boundedRelayAgentAliases(activeRelayAgentAliases.values());
|
|
6267
|
+
void this.pollRelayJob(reservedOnly ? RESERVED_JOB_POLL_WAIT_MS : DEFAULT_JOB_POLL_WAIT_MS, {
|
|
6268
|
+
...(reservedRelayExecutionLimit > 0 ? { excludedAgentSlugs } : {}),
|
|
6269
|
+
...(reservedOnly ? { maxRequestedPriority: reservedPriorityMax } : {})
|
|
6270
|
+
})
|
|
6271
|
+
.then((polled) => {
|
|
6272
|
+
relayPollFailureStreak = 0;
|
|
6273
|
+
const claim = polled.claim;
|
|
6274
|
+
if (!claim) {
|
|
6275
|
+
return;
|
|
6276
|
+
}
|
|
6277
|
+
const regularLane = !reservedOnly;
|
|
6278
|
+
const executionId = ++relayExecutionSequence;
|
|
6279
|
+
activeRelayExecutions += 1;
|
|
6280
|
+
if (regularLane) {
|
|
6281
|
+
activeRegularRelayExecutions += 1;
|
|
6282
|
+
}
|
|
6283
|
+
activeRelayAgentAliases.set(executionId, relayJobAgentAliases(claim.job));
|
|
6284
|
+
void this.executeRelayJobClaim(claim)
|
|
6285
|
+
.then(() => {
|
|
6286
|
+
relayExecutionFailureStreak = 0;
|
|
6287
|
+
})
|
|
6288
|
+
.catch((error) => {
|
|
6289
|
+
if (noteRevoked(error)) {
|
|
6290
|
+
return;
|
|
6291
|
+
}
|
|
6292
|
+
relayExecutionFailureStreak += 1;
|
|
6293
|
+
console.error(`[mswarm] self-hosted relay job ${claim.job.job_id} failed: ${runtimeErrorMessage(error)}`);
|
|
6294
|
+
})
|
|
6295
|
+
.finally(() => {
|
|
6296
|
+
activeRelayExecutions = Math.max(0, activeRelayExecutions - 1);
|
|
6297
|
+
if (regularLane) {
|
|
6298
|
+
activeRegularRelayExecutions = Math.max(0, activeRegularRelayExecutions - 1);
|
|
6299
|
+
}
|
|
6300
|
+
activeRelayAgentAliases.delete(executionId);
|
|
6301
|
+
schedulePoll(currentRelayRetryDelayMs());
|
|
6302
|
+
});
|
|
6103
6303
|
})
|
|
6104
6304
|
.catch((error) => {
|
|
6105
6305
|
if (noteRevoked(error)) {
|
|
6106
6306
|
return;
|
|
6107
6307
|
}
|
|
6108
|
-
|
|
6308
|
+
relayPollFailureStreak += 1;
|
|
6109
6309
|
console.error(`[mswarm] self-hosted relay poll failed: ${runtimeErrorMessage(error)}`);
|
|
6110
6310
|
})
|
|
6111
6311
|
.finally(() => {
|
|
6112
6312
|
polling = false;
|
|
6113
|
-
if (stopped ||
|
|
6313
|
+
if (stopped ||
|
|
6314
|
+
revoked ||
|
|
6315
|
+
this.lifecyclePollingDisabled ||
|
|
6316
|
+
activeRelayExecutions >= relayExecutionLimit) {
|
|
6114
6317
|
return;
|
|
6115
6318
|
}
|
|
6116
6319
|
// A healthy poll long-polls at the gateway, so re-entering immediately is
|
|
6117
6320
|
// correct. A failing one returns at once, so without this the loop spins as
|
|
6118
6321
|
// fast as the network allows.
|
|
6119
|
-
|
|
6322
|
+
schedulePoll(currentRelayRetryDelayMs());
|
|
6120
6323
|
});
|
|
6121
6324
|
};
|
|
6122
6325
|
const schedule = () => {
|
|
@@ -6138,7 +6341,7 @@ export class SelfHostedNodeRuntime {
|
|
|
6138
6341
|
})
|
|
6139
6342
|
.finally(() => {
|
|
6140
6343
|
schedule();
|
|
6141
|
-
|
|
6344
|
+
schedulePoll(0);
|
|
6142
6345
|
});
|
|
6143
6346
|
return { stop: halt };
|
|
6144
6347
|
}
|