@mcoda/mswarm 0.1.107 → 0.1.108

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/runtime.js CHANGED
@@ -62,6 +62,7 @@ const DEFAULT_HEARTBEAT_INTERVAL_SECONDS = 30;
62
62
  const DEFAULT_SELF_HOSTED_NODE_VERSION = "0.1.70";
63
63
  const DEFAULT_REQUEST_TIMEOUT_MS = 10000;
64
64
  const DEFAULT_JOB_TIMEOUT_MS = 3600000;
65
+ const DEFAULT_RESULT_POST_TIMEOUT_MS = 5 * 60000;
65
66
  const DEFAULT_SERVICE_COMMAND_TIMEOUT_MS = 60000;
66
67
  const DEFAULT_CAPABILITY_PROBE_TIMEOUT_MS = 2000;
67
68
  const SELF_HOSTED_RUNTIME_PROTOCOL_VERSION = 1;
@@ -74,6 +75,12 @@ const DEFAULT_MCODA_LIST_ARGS = ["agent", "list", "--json", "--refresh-health"];
74
75
  const DEFAULT_COMMAND_MAX_BUFFER = 16 * 1024 * 1024;
75
76
  const DEFAULT_LOCAL_ARTIFACT_MAX_BYTES = 512 * 1024 * 1024;
76
77
  const DEFAULT_JOB_POLL_WAIT_MS = 2000;
78
+ const RESERVED_JOB_POLL_WAIT_MS = 250;
79
+ const MAX_POLL_EXCLUDED_AGENT_SLUGS = 64;
80
+ const MAX_RELAY_AGENT_SLUG_LENGTH = 255;
81
+ const DEFAULT_RESERVED_PRIORITY_MAX = -1;
82
+ const MIN_RELAY_PRIORITY = -100;
83
+ const MAX_RELAY_PRIORITY = 100;
77
84
  const JOB_POLL_RETRY_BASE_MS = 1000;
78
85
  const JOB_POLL_RETRY_MAX_MS = 60000;
79
86
  /**
@@ -284,6 +291,33 @@ const LOCAL_RUNNER_RESPONSE_FORMAT_ALIASES = {
284
291
  function optionalText(value) {
285
292
  return typeof value === "string" && value.trim().length > 0 ? value.trim() : null;
286
293
  }
294
+ function relayJobAgentAliases(job) {
295
+ const aliases = [job.agent_slug, job.source_agent_slug, job.remote_slug]
296
+ .map(optionalText)
297
+ .map((value) => value
298
+ ?.toLowerCase()
299
+ .replace(/[^a-z0-9]+/g, "-")
300
+ .replace(/-+/g, "-")
301
+ .replace(/^-|-$/g, "") || null)
302
+ .filter((value) => typeof value === "string" && value.length > 0 && value.length <= MAX_RELAY_AGENT_SLUG_LENGTH);
303
+ return [...new Set(aliases)];
304
+ }
305
+ function boundedRelayAgentAliases(activeAliases) {
306
+ const aliases = [];
307
+ const seen = new Set();
308
+ for (const group of activeAliases) {
309
+ for (const alias of group) {
310
+ if (seen.has(alias))
311
+ continue;
312
+ seen.add(alias);
313
+ aliases.push(alias);
314
+ if (aliases.length >= MAX_POLL_EXCLUDED_AGENT_SLUGS) {
315
+ return aliases;
316
+ }
317
+ }
318
+ }
319
+ return aliases;
320
+ }
287
321
  function isRecord(value) {
288
322
  return typeof value === "object" && value !== null && !Array.isArray(value);
289
323
  }
@@ -588,6 +622,39 @@ function parsePositiveInteger(value, fallback) {
588
622
  }
589
623
  return Math.floor(parsed);
590
624
  }
625
+ function parseNonNegativeInteger(value, fallback) {
626
+ const parsed = value === undefined || value === null || value === ""
627
+ ? fallback
628
+ : typeof value === "number"
629
+ ? value
630
+ : typeof value === "string"
631
+ ? Number(value)
632
+ : Number.NaN;
633
+ if (!Number.isInteger(parsed) || parsed < 0) {
634
+ throw new Error("MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS must be a non-negative integer");
635
+ }
636
+ return parsed;
637
+ }
638
+ function parseRelayPriority(value, fallback) {
639
+ const parsed = value === undefined || value === null || value === ""
640
+ ? fallback
641
+ : typeof value === "number"
642
+ ? value
643
+ : typeof value === "string"
644
+ ? Number(value)
645
+ : Number.NaN;
646
+ if (!Number.isInteger(parsed) || parsed < MIN_RELAY_PRIORITY || parsed > MAX_RELAY_PRIORITY) {
647
+ throw new Error(`MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX must be an integer from ${MIN_RELAY_PRIORITY} to ${MAX_RELAY_PRIORITY}`);
648
+ }
649
+ return parsed;
650
+ }
651
+ function assertReservedLlmCapacity(input) {
652
+ const totalLlmJobs = input.maxConcurrentLlmJobs + input.reservedLlmJobs;
653
+ if (totalLlmJobs > input.maxConcurrentJobs) {
654
+ throw new Error("MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS must be greater than or equal to " +
655
+ "MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS plus MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS");
656
+ }
657
+ }
591
658
  function parseBoolean(value, fallback) {
592
659
  if (typeof value === "boolean")
593
660
  return value;
@@ -1280,6 +1347,8 @@ function serviceEnvironment(config, env, homeDir) {
1280
1347
  MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS: String(config.jobTimeoutMs),
1281
1348
  MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS: String(config.maxConcurrentJobs || 1),
1282
1349
  MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS: String(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1),
1350
+ MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS: String(config.reservedLlmJobs ?? 0),
1351
+ MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX: String(config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX),
1283
1352
  MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED: config.genericJobsEnabled ? "true" : "false",
1284
1353
  MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS: String(config.genericJobTimeoutMs),
1285
1354
  MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY: String(config.genericJobMaxConcurrency),
@@ -1695,6 +1764,9 @@ export async function readSelfHostedNodeConfig(env = process.env) {
1695
1764
  const packageNodeVersion = await readPackageNodeVersion();
1696
1765
  const maxConcurrentJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, state.max_concurrent_jobs || 1);
1697
1766
  const maxConcurrentLlmJobs = parsePositiveInteger(env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, state.max_concurrent_llm_jobs || maxConcurrentJobs);
1767
+ const reservedLlmJobs = parseNonNegativeInteger(env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, state.reserved_llm_jobs ?? 0);
1768
+ const reservedPriorityMax = parseRelayPriority(env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, state.reserved_priority_max ?? DEFAULT_RESERVED_PRIORITY_MAX);
1769
+ assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
1698
1770
  return {
1699
1771
  gatewayBaseUrl: trimTrailingSlash(gatewayBaseUrl),
1700
1772
  jobsPollPath: optionalText(env.MSWARM_SELF_HOSTED_JOBS_POLL_PATH) ||
@@ -1738,6 +1810,8 @@ export async function readSelfHostedNodeConfig(env = process.env) {
1738
1810
  jobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
1739
1811
  maxConcurrentJobs,
1740
1812
  maxConcurrentLlmJobs,
1813
+ reservedLlmJobs,
1814
+ reservedPriorityMax,
1741
1815
  genericJobsEnabled: parseBoolean(env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED ?? env.MSWARM_SELF_HOSTED_GENERIC_JOBS, state.generic_jobs_enabled === true),
1742
1816
  genericJobTimeoutMs: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, state.generic_job_timeout_ms || state.job_timeout_ms || DEFAULT_JOB_TIMEOUT_MS),
1743
1817
  genericJobMaxConcurrency: parsePositiveInteger(env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, state.generic_job_max_concurrency || 1),
@@ -1787,6 +1861,9 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
1787
1861
  const packageNodeVersion = await readPackageNodeVersion();
1788
1862
  const maxConcurrentJobs = parsePositiveInteger(options["max-concurrent-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_JOBS, 1);
1789
1863
  const maxConcurrentLlmJobs = parsePositiveInteger(options["max-concurrent-llm-jobs"] || env.MSWARM_SELF_HOSTED_MAX_CONCURRENT_LLM_JOBS, maxConcurrentJobs);
1864
+ const reservedLlmJobs = parseNonNegativeInteger(options["reserved-llm-jobs"] ?? env.MSWARM_SELF_HOSTED_RESERVED_LLM_JOBS, 0);
1865
+ const reservedPriorityMax = parseRelayPriority(options["reserved-priority-max"] ?? env.MSWARM_SELF_HOSTED_RESERVED_PRIORITY_MAX, DEFAULT_RESERVED_PRIORITY_MAX);
1866
+ assertReservedLlmCapacity({ maxConcurrentJobs, maxConcurrentLlmJobs, reservedLlmJobs });
1790
1867
  return {
1791
1868
  apiKey,
1792
1869
  handle: handle || undefined,
@@ -1812,6 +1889,8 @@ export async function readOwnerSetupConfig(argv = process.argv.slice(3), env = p
1812
1889
  jobTimeoutMs: parsePositiveInteger(options["job-timeout-ms"] || env.MSWARM_SELF_HOSTED_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
1813
1890
  maxConcurrentJobs,
1814
1891
  maxConcurrentLlmJobs,
1892
+ reservedLlmJobs,
1893
+ reservedPriorityMax,
1815
1894
  genericJobsEnabled: parseBoolean(options["enable-generic-jobs"] || env.MSWARM_SELF_HOSTED_GENERIC_JOBS_ENABLED || env.MSWARM_SELF_HOSTED_GENERIC_JOBS, false),
1816
1895
  genericJobTimeoutMs: parsePositiveInteger(options["generic-job-timeout-ms"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_TIMEOUT_MS, DEFAULT_JOB_TIMEOUT_MS),
1817
1896
  genericJobMaxConcurrency: parsePositiveInteger(options["generic-job-max-concurrency"] || env.MSWARM_SELF_HOSTED_GENERIC_JOB_MAX_CONCURRENCY, 1),
@@ -4100,6 +4179,7 @@ export class MswarmSelfHostedNodeClient {
4100
4179
  this.jobsResultPathTemplate = lifecyclePath(input.jobsResultPathTemplate, DEFAULT_SELF_HOSTED_JOBS_RESULT_PATH_TEMPLATE);
4101
4180
  this.fetchImpl = input.fetchImpl || fetch;
4102
4181
  this.timeoutMs = input.timeoutMs || DEFAULT_REQUEST_TIMEOUT_MS;
4182
+ this.resultTimeoutMs = Math.max(this.timeoutMs, input.resultTimeoutMs || DEFAULT_RESULT_POST_TIMEOUT_MS);
4103
4183
  }
4104
4184
  lifecycleEndpoint(kind) {
4105
4185
  const path = kind === "poll"
@@ -4214,7 +4294,7 @@ export class MswarmSelfHostedNodeClient {
4214
4294
  authorization: `Bearer ${runtimeToken}`
4215
4295
  },
4216
4296
  body: JSON.stringify(payload)
4217
- }, this.timeoutMs);
4297
+ }, this.resultTimeoutMs);
4218
4298
  }
4219
4299
  async postJobStart(runtimeToken, jobId, payload) {
4220
4300
  return fetchJson(this.fetchImpl, this.lifecycleUrl("start", jobId), {
@@ -4246,6 +4326,12 @@ export class SelfHostedNodeRuntime {
4246
4326
  this.latencySamplesMs = [];
4247
4327
  this.recentFailures = [];
4248
4328
  this.lifecyclePollingDisabled = false;
4329
+ assertReservedLlmCapacity({
4330
+ maxConcurrentJobs: Math.max(1, Math.floor(config.maxConcurrentJobs || 1)),
4331
+ maxConcurrentLlmJobs: Math.max(1, Math.floor(config.maxConcurrentLlmJobs || config.maxConcurrentJobs || 1)),
4332
+ reservedLlmJobs: parseNonNegativeInteger(config.reservedLlmJobs, 0)
4333
+ });
4334
+ parseRelayPriority(config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
4249
4335
  this.config = config;
4250
4336
  this.fetchImpl = deps?.fetchImpl || fetch;
4251
4337
  this.useDedicatedGenerativeDispatcher = deps?.fetchImpl === undefined;
@@ -4258,7 +4344,8 @@ export class SelfHostedNodeRuntime {
4258
4344
  jobsEventsPathTemplate: config.jobsEventsPathTemplate,
4259
4345
  jobsResultPathTemplate: config.jobsResultPathTemplate,
4260
4346
  fetchImpl: this.fetchImpl,
4261
- timeoutMs: config.requestTimeoutMs
4347
+ timeoutMs: config.requestTimeoutMs,
4348
+ resultTimeoutMs: Math.min(config.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
4262
4349
  });
4263
4350
  this.mcoda =
4264
4351
  deps?.mcoda ||
@@ -4448,7 +4535,8 @@ export class SelfHostedNodeRuntime {
4448
4535
  new MswarmSelfHostedNodeClient({
4449
4536
  gatewayBaseUrl: setupConfig.gatewayBaseUrl,
4450
4537
  fetchImpl: deps?.fetchImpl,
4451
- timeoutMs: setupConfig.requestTimeoutMs
4538
+ timeoutMs: setupConfig.requestTimeoutMs,
4539
+ resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
4452
4540
  });
4453
4541
  const machineId = await readOrCreateSelfHostedMachineId(setupConfig.machineIdPath);
4454
4542
  const machineFingerprint = machineFingerprintFromId(machineId);
@@ -4517,6 +4605,8 @@ export class SelfHostedNodeRuntime {
4517
4605
  job_timeout_ms: setupConfig.jobTimeoutMs,
4518
4606
  max_concurrent_jobs: setupConfig.maxConcurrentJobs,
4519
4607
  max_concurrent_llm_jobs: setupConfig.maxConcurrentLlmJobs,
4608
+ reserved_llm_jobs: setupConfig.reservedLlmJobs,
4609
+ reserved_priority_max: setupConfig.reservedPriorityMax,
4520
4610
  generic_jobs_enabled: setupConfig.genericJobsEnabled,
4521
4611
  generic_job_timeout_ms: setupConfig.genericJobTimeoutMs,
4522
4612
  generic_job_max_concurrency: setupConfig.genericJobMaxConcurrency,
@@ -4541,7 +4631,8 @@ export class SelfHostedNodeRuntime {
4541
4631
  jobsEventsPathTemplate,
4542
4632
  jobsResultPathTemplate,
4543
4633
  fetchImpl: deps?.fetchImpl,
4544
- timeoutMs: setupConfig.requestTimeoutMs
4634
+ timeoutMs: setupConfig.requestTimeoutMs,
4635
+ resultTimeoutMs: Math.min(setupConfig.jobTimeoutMs, DEFAULT_RESULT_POST_TIMEOUT_MS)
4545
4636
  });
4546
4637
  const runtime = new SelfHostedNodeRuntime({
4547
4638
  gatewayBaseUrl: trimTrailingSlash(relayGatewayBaseUrl),
@@ -4572,6 +4663,8 @@ export class SelfHostedNodeRuntime {
4572
4663
  jobTimeoutMs: setupConfig.jobTimeoutMs,
4573
4664
  maxConcurrentJobs: setupConfig.maxConcurrentJobs,
4574
4665
  maxConcurrentLlmJobs: setupConfig.maxConcurrentLlmJobs,
4666
+ reservedLlmJobs: setupConfig.reservedLlmJobs,
4667
+ reservedPriorityMax: setupConfig.reservedPriorityMax,
4575
4668
  genericJobsEnabled: setupConfig.genericJobsEnabled,
4576
4669
  genericJobTimeoutMs: setupConfig.genericJobTimeoutMs,
4577
4670
  genericJobMaxConcurrency: setupConfig.genericJobMaxConcurrency,
@@ -4702,6 +4795,8 @@ export class SelfHostedNodeRuntime {
4702
4795
  job_timeout_ms: this.config.jobTimeoutMs,
4703
4796
  max_concurrent_jobs: this.config.maxConcurrentJobs,
4704
4797
  max_concurrent_llm_jobs: this.config.maxConcurrentLlmJobs,
4798
+ reserved_llm_jobs: this.config.reservedLlmJobs ?? 0,
4799
+ reserved_priority_max: this.config.reservedPriorityMax ?? DEFAULT_RESERVED_PRIORITY_MAX,
4705
4800
  generic_jobs_enabled: this.config.genericJobsEnabled,
4706
4801
  generic_job_timeout_ms: this.config.genericJobTimeoutMs,
4707
4802
  generic_job_max_concurrency: this.config.genericJobMaxConcurrency,
@@ -5853,11 +5948,11 @@ export class SelfHostedNodeRuntime {
5853
5948
  });
5854
5949
  return { count: models.filter((model) => model.exposed !== false).length, response };
5855
5950
  }
5856
- async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
5951
+ async pollRelayJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS, options = {}) {
5857
5952
  const lifecycleState = await this.readLifecycleState();
5858
5953
  if (this.lifecyclePollingDisabled || this.isLifecycleProtocolDegradedState(lifecycleState)) {
5859
5954
  this.lifecyclePollingDisabled = true;
5860
- return { executed: false, status: "failed" };
5955
+ return { claim: null, status: "failed" };
5861
5956
  }
5862
5957
  const enrollment = await this.ensureEnrolled();
5863
5958
  const pollCapacity = this.buildLoadTelemetry({ models: [] });
@@ -5871,7 +5966,17 @@ export class SelfHostedNodeRuntime {
5871
5966
  max_jobs: pollCapacity.max_concurrency,
5872
5967
  max_concurrency: pollCapacity.max_concurrency,
5873
5968
  free_slots: pollCapacity.free_slots,
5874
- drain_mode: pollCapacity.drain_mode
5969
+ drain_mode: pollCapacity.drain_mode,
5970
+ ...(options.excludedAgentSlugs !== undefined
5971
+ ? {
5972
+ excluded_agent_slugs: boundedRelayAgentAliases([
5973
+ options.excludedAgentSlugs
5974
+ ])
5975
+ }
5976
+ : {}),
5977
+ ...(options.maxRequestedPriority !== undefined
5978
+ ? { max_requested_priority: options.maxRequestedPriority }
5979
+ : {})
5875
5980
  },
5876
5981
  wait_ms: waitMs
5877
5982
  });
@@ -5880,14 +5985,24 @@ export class SelfHostedNodeRuntime {
5880
5985
  const mismatch = this.lifecycleProtocolMismatch("poll", error);
5881
5986
  if (mismatch) {
5882
5987
  await this.markLifecycleProtocolDegraded(mismatch);
5883
- return { executed: false, status: "failed" };
5988
+ return { claim: null, status: "failed" };
5884
5989
  }
5885
5990
  throw error;
5886
5991
  }
5887
5992
  const job = response.job || null;
5888
5993
  if (!job) {
5889
- return { executed: false };
5994
+ return { claim: null };
5890
5995
  }
5996
+ return {
5997
+ claim: {
5998
+ runtimeToken: enrollment.runtimeToken,
5999
+ job,
6000
+ attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined
6001
+ }
6002
+ };
6003
+ }
6004
+ async executeRelayJobClaim(claim) {
6005
+ const { job, runtimeToken } = claim;
5891
6006
  const chatStreaming = (normalizeSelfHostedGenerativeOperationName(job.operation) ?? "chat.completions") ===
5892
6007
  "chat.completions" &&
5893
6008
  job.openai_request.stream === true;
@@ -5900,7 +6015,7 @@ export class SelfHostedNodeRuntime {
5900
6015
  }
5901
6016
  const stream_events = pendingStreamEvents.splice(0, pendingStreamEvents.length);
5902
6017
  try {
5903
- await this.gateway.postJobEvents(enrollment.runtimeToken, job.job_id, {
6018
+ await this.gateway.postJobEvents(runtimeToken, job.job_id, {
5904
6019
  node_id: this.config.nodeId,
5905
6020
  stream_events
5906
6021
  });
@@ -5917,10 +6032,10 @@ export class SelfHostedNodeRuntime {
5917
6032
  }
5918
6033
  };
5919
6034
  let result = await this.executeJob(job, {
5920
- attachedMswarmApiKey: optionalText(response.attached_mswarm_api_key) || undefined,
6035
+ attachedMswarmApiKey: claim.attachedMswarmApiKey,
5921
6036
  onStarted: async (event) => {
5922
6037
  try {
5923
- await this.gateway.postJobStart(enrollment.runtimeToken, job.job_id, {
6038
+ await this.gateway.postJobStart(runtimeToken, job.job_id, {
5924
6039
  node_id: this.config.nodeId,
5925
6040
  agent_slug: event.agent_slug || job.agent_slug,
5926
6041
  source_agent_slug: event.source_agent_slug || job.source_agent_slug || null,
@@ -5975,7 +6090,7 @@ export class SelfHostedNodeRuntime {
5975
6090
  let resultPostError = null;
5976
6091
  for (let attempt = 0; attempt <= RELAY_RESULT_POST_RETRY_DELAYS_MS.length; attempt += 1) {
5977
6092
  try {
5978
- await this.gateway.postJobResult(enrollment.runtimeToken, job.job_id, this.jobResultPayload(postedResult));
6093
+ await this.gateway.postJobResult(runtimeToken, job.job_id, this.jobResultPayload(postedResult));
5979
6094
  resultPostError = null;
5980
6095
  break;
5981
6096
  }
@@ -6003,6 +6118,13 @@ export class SelfHostedNodeRuntime {
6003
6118
  }
6004
6119
  return { executed: true, job_id: job.job_id, status: result.status };
6005
6120
  }
6121
+ async pollAndExecuteJob(waitMs = DEFAULT_JOB_POLL_WAIT_MS) {
6122
+ const polled = await this.pollRelayJob(waitMs);
6123
+ if (!polled.claim) {
6124
+ return { executed: false, ...(polled.status ? { status: polled.status } : {}) };
6125
+ }
6126
+ return this.executeRelayJobClaim(polled.claim);
6127
+ }
6006
6128
  async doctor() {
6007
6129
  const checks = [];
6008
6130
  checks.push({ name: "config", ok: Boolean(this.config.nodeId), message: this.config.nodeId || "missing node id" });
@@ -6041,19 +6163,33 @@ export class SelfHostedNodeRuntime {
6041
6163
  let stopped = false;
6042
6164
  let timer = null;
6043
6165
  let pollTimer = null;
6166
+ let pollTimerDueAt = 0;
6044
6167
  let polling = false;
6045
- let pollFailureStreak = 0;
6168
+ let relayPollFailureStreak = 0;
6169
+ let relayExecutionFailureStreak = 0;
6170
+ let activeRelayExecutions = 0;
6171
+ let activeRegularRelayExecutions = 0;
6172
+ let relayExecutionSequence = 0;
6173
+ const activeRelayAgentAliases = new Map();
6046
6174
  let revoked = false;
6175
+ const regularRelayExecutionLimit = Math.max(1, Math.floor(this.config.maxConcurrentLlmJobs || this.config.maxConcurrentJobs || 1));
6176
+ const reservedRelayExecutionLimit = parseNonNegativeInteger(this.config.reservedLlmJobs, 0);
6177
+ const reservedPriorityMax = parseRelayPriority(this.config.reservedPriorityMax, DEFAULT_RESERVED_PRIORITY_MAX);
6178
+ const relayExecutionLimit = regularRelayExecutionLimit + reservedRelayExecutionLimit;
6179
+ const clearPollTimer = () => {
6180
+ if (pollTimer) {
6181
+ clearTimeout(pollTimer);
6182
+ pollTimer = null;
6183
+ }
6184
+ pollTimerDueAt = 0;
6185
+ };
6047
6186
  const halt = () => {
6048
6187
  stopped = true;
6049
6188
  if (timer) {
6050
6189
  clearTimeout(timer);
6051
6190
  timer = null;
6052
6191
  }
6053
- if (pollTimer) {
6054
- clearTimeout(pollTimer);
6055
- pollTimer = null;
6056
- }
6192
+ clearPollTimer();
6057
6193
  };
6058
6194
  // 410 means the gateway has retired this node's credentials: it was removed there.
6059
6195
  // Retrying at full rate achieves nothing but a hot loop and a flooded journal, so
@@ -6067,10 +6203,7 @@ export class SelfHostedNodeRuntime {
6067
6203
  if (!isHttpStatusError(error, 410)) {
6068
6204
  return false;
6069
6205
  }
6070
- if (pollTimer) {
6071
- clearTimeout(pollTimer);
6072
- pollTimer = null;
6073
- }
6206
+ clearPollTimer();
6074
6207
  if (!revoked) {
6075
6208
  revoked = true;
6076
6209
  console.error(`[mswarm] this node was removed at the gateway (node ${this.config.nodeId}). ` +
@@ -6081,42 +6214,112 @@ export class SelfHostedNodeRuntime {
6081
6214
  return true;
6082
6215
  };
6083
6216
  const noteAlive = () => {
6084
- if (!revoked) {
6217
+ const recoveredFromRevocation = revoked;
6218
+ const relayPollingIsBackedOff = (relayPollFailureStreak > 0 || relayExecutionFailureStreak > 0) && !polling;
6219
+ if (!recoveredFromRevocation && !relayPollingIsBackedOff) {
6220
+ return;
6221
+ }
6222
+ if (recoveredFromRevocation) {
6223
+ revoked = false;
6224
+ console.error(`[mswarm] node ${this.config.nodeId} is accepted at the gateway again; resuming normal operation.`);
6225
+ }
6226
+ clearPollTimer();
6227
+ relayPollFailureStreak = 0;
6228
+ relayExecutionFailureStreak = 0;
6229
+ schedulePoll(0);
6230
+ };
6231
+ const currentRelayRetryDelayMs = () => Math.max(pollRetryDelayMs(relayPollFailureStreak), pollRetryDelayMs(relayExecutionFailureStreak));
6232
+ const schedulePoll = (delayMs) => {
6233
+ if (stopped ||
6234
+ polling ||
6235
+ revoked ||
6236
+ this.lifecyclePollingDisabled ||
6237
+ this.config.relayMode === "direct" ||
6238
+ activeRelayExecutions >= relayExecutionLimit) {
6239
+ return;
6240
+ }
6241
+ const dueAt = Date.now() + Math.max(0, delayMs);
6242
+ if (pollTimer && pollTimerDueAt >= dueAt) {
6085
6243
  return;
6086
6244
  }
6087
- revoked = false;
6088
- console.error(`[mswarm] node ${this.config.nodeId} is accepted at the gateway again; resuming normal operation.`);
6089
- poll();
6245
+ clearPollTimer();
6246
+ pollTimerDueAt = dueAt;
6247
+ pollTimer = setTimeout(() => {
6248
+ pollTimer = null;
6249
+ pollTimerDueAt = 0;
6250
+ poll();
6251
+ }, Math.max(0, dueAt - Date.now()));
6090
6252
  };
6091
6253
  const poll = () => {
6092
6254
  if (stopped ||
6093
6255
  polling ||
6094
6256
  revoked ||
6095
6257
  this.lifecyclePollingDisabled ||
6096
- this.config.relayMode === "direct") {
6258
+ this.config.relayMode === "direct" ||
6259
+ activeRelayExecutions >= relayExecutionLimit) {
6097
6260
  return;
6098
6261
  }
6099
6262
  polling = true;
6100
- void this.pollAndExecuteJob()
6101
- .then(() => {
6102
- pollFailureStreak = 0;
6263
+ const reservedOnly = reservedRelayExecutionLimit > 0 &&
6264
+ activeRegularRelayExecutions >= regularRelayExecutionLimit &&
6265
+ activeRelayExecutions < relayExecutionLimit;
6266
+ const excludedAgentSlugs = boundedRelayAgentAliases(activeRelayAgentAliases.values());
6267
+ void this.pollRelayJob(reservedOnly ? RESERVED_JOB_POLL_WAIT_MS : DEFAULT_JOB_POLL_WAIT_MS, {
6268
+ ...(reservedRelayExecutionLimit > 0 ? { excludedAgentSlugs } : {}),
6269
+ ...(reservedOnly ? { maxRequestedPriority: reservedPriorityMax } : {})
6270
+ })
6271
+ .then((polled) => {
6272
+ relayPollFailureStreak = 0;
6273
+ const claim = polled.claim;
6274
+ if (!claim) {
6275
+ return;
6276
+ }
6277
+ const regularLane = !reservedOnly;
6278
+ const executionId = ++relayExecutionSequence;
6279
+ activeRelayExecutions += 1;
6280
+ if (regularLane) {
6281
+ activeRegularRelayExecutions += 1;
6282
+ }
6283
+ activeRelayAgentAliases.set(executionId, relayJobAgentAliases(claim.job));
6284
+ void this.executeRelayJobClaim(claim)
6285
+ .then(() => {
6286
+ relayExecutionFailureStreak = 0;
6287
+ })
6288
+ .catch((error) => {
6289
+ if (noteRevoked(error)) {
6290
+ return;
6291
+ }
6292
+ relayExecutionFailureStreak += 1;
6293
+ console.error(`[mswarm] self-hosted relay job ${claim.job.job_id} failed: ${runtimeErrorMessage(error)}`);
6294
+ })
6295
+ .finally(() => {
6296
+ activeRelayExecutions = Math.max(0, activeRelayExecutions - 1);
6297
+ if (regularLane) {
6298
+ activeRegularRelayExecutions = Math.max(0, activeRegularRelayExecutions - 1);
6299
+ }
6300
+ activeRelayAgentAliases.delete(executionId);
6301
+ schedulePoll(currentRelayRetryDelayMs());
6302
+ });
6103
6303
  })
6104
6304
  .catch((error) => {
6105
6305
  if (noteRevoked(error)) {
6106
6306
  return;
6107
6307
  }
6108
- pollFailureStreak += 1;
6308
+ relayPollFailureStreak += 1;
6109
6309
  console.error(`[mswarm] self-hosted relay poll failed: ${runtimeErrorMessage(error)}`);
6110
6310
  })
6111
6311
  .finally(() => {
6112
6312
  polling = false;
6113
- if (stopped || revoked) {
6313
+ if (stopped ||
6314
+ revoked ||
6315
+ this.lifecyclePollingDisabled ||
6316
+ activeRelayExecutions >= relayExecutionLimit) {
6114
6317
  return;
6115
6318
  }
6116
6319
  // A healthy poll long-polls at the gateway, so re-entering immediately is
6117
6320
  // correct. A failing one returns at once, so without this the loop spins as
6118
6321
  // fast as the network allows.
6119
- pollTimer = setTimeout(poll, pollRetryDelayMs(pollFailureStreak));
6322
+ schedulePoll(currentRelayRetryDelayMs());
6120
6323
  });
6121
6324
  };
6122
6325
  const schedule = () => {
@@ -6138,7 +6341,7 @@ export class SelfHostedNodeRuntime {
6138
6341
  })
6139
6342
  .finally(() => {
6140
6343
  schedule();
6141
- poll();
6344
+ schedulePoll(0);
6142
6345
  });
6143
6346
  return { stop: halt };
6144
6347
  }