@agent-relay/factory 0.1.35 → 0.1.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/orchestrator/factory.ts"],"names":[],"mappings":"AAIA,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAI1E,OAAO,KAAK,EAMV,WAAW,EAOX,WAAW,EAKZ,MAAM,UAAU,CAAA;AA+BjB,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EAGnB,QAAQ,EACR,eAAe,EACf,WAAW,EAGX,cAAc,EAEf,MAAM,UAAU,CAAA;AAajB,KAAK,YAAY,GAAG,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,CAAA;AACjG,KAAK,QAAQ,GAAG,CAAC,OAAO,EAAE,mBAAmB,KAAK,IAAI,CAAA;
|
|
1
|
+
{"version":3,"file":"factory.d.ts","sourceRoot":"","sources":["../../src/orchestrator/factory.ts"],"names":[],"mappings":"AAIA,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,kBAAkB,CAAA;AAI1E,OAAO,KAAK,EAMV,WAAW,EAOX,WAAW,EAKZ,MAAM,UAAU,CAAA;AA+BjB,OAAO,KAAK,EACV,cAAc,EACd,OAAO,EACP,mBAAmB,EACnB,YAAY,EACZ,aAAa,EACb,mBAAmB,EAEnB,qBAAqB,EACrB,oBAAoB,EACpB,mBAAmB,EAGnB,QAAQ,EACR,eAAe,EACf,WAAW,EAGX,cAAc,EAEf,MAAM,UAAU,CAAA;AAajB,KAAK,YAAY,GAAG,cAAc,GAAG,YAAY,GAAG,YAAY,GAAG,oBAAoB,GAAG,OAAO,CAAA;AACjG,KAAK,QAAQ,GAAG,CAAC,OAAO,EAAE,mBAAmB,KAAK,IAAI,CAAA;AAwFtD,KAAK,iBAAiB,GAAG;IACvB,KAAK,EAAE,MAAM,CAAA;IACb,QAAQ,EAAE,MAAM,CAAA;IAChB,IAAI,EAAE,MAAM,CAAA;IACZ,MAAM,EAAE,MAAM,CAAA;IACd,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;IACb,MAAM,EAAE,MAAM,EAAE,CAAA;IAChB,MAAM,CAAC,EAAE,MAAM,CAAA;IACf,IAAI,EAAE,MAAM,CAAA;IACZ,GAAG,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;CAC7B,CAAA;AA0FD,eAAO,MAAM,mCAAmC,iDAAiD,CAAA;AACjG,eAAO,MAAM,kCAAkC,gDAAgD,CAAA;AAM/F,wBAAgB,aAAa,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,GAAG,OAAO,CAEjF;AAED,qBAAa,WAAY,YAAW,OAAO;;gBA4I7B,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY;IAkHhD,uBAAuB,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO,CAAC,IAAI,CAAC;IAiEvD,KAAK,CAAC,IAAI,GAAE,mBAAwB,GAAG,OAAO,CAAC,IAAI,CAAC;IAiGpD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAyHrB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAumBxB,OAAO,CAAC,IAAI,GAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAmnBlE,OAAO,CAAC,IAAI,GAAE,qBAA0B,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC;IAyDrE,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,cAAc,CAAC;IAOxD,QAAQ,CAAC,QAAQ,EAAE,cAAc,EAAE,IAAI,GAAE;QAAE,MAAM,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,cAAc,CAAC;IA+RlG,MAAM,IAAI,aAAa;IAWvB,EAAE,CAAC,KAAK,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,GAAG,MAAM,IAAI;CA2/NxD;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,WAAW,CAiC5E;AAyBD,wBAAsB,oCAAoC,CACxD,KAAK,EAAE,IAAI,CAAC,WAAW,EAAE,UAAU,CAAC,EACpC,IAAI,EAAE,MAAM,GACX,OAAO,CAAC,WAAW,CAAC,CAetB;AAED,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,iBAAiB,CA6BlF;AAED,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,GAAG,WAAW,CAEnF;AAoDD,wBAAsB,wBAAwB,CAC5C,IAAI,SAAsC,GACzC,OAAO,CAAC,oBAAoB,GAAG,SAAS,CAAC,CAM3C;AAED,wBAAgB,wBAAwB,CACtC,SAAS,EAAE,oBAAoB,GAAG,SAAS,EAC3C,IAAI,GAAE;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,MAAM,CAAA;CAAO,GAC9C,mBAAmB,CAgBrB;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAO7D;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAiB/D;AAogBD,eAAO,MAAM,oBAAoB,GAAI,MAAM,MAAM,KAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,IAAI,CAAC,EAAE,MAAM,CAAA;CAAE,GAAG,SAwBpH,CAAA;AAmYD,eAAO,MAAM,2BAA2B,GAAI,QAAQ,aAAa,KAAG,MAAM,EAItE,CAAA;AAsmBJ,eAAO,MAAM,WAAW,GAAI,MAAM,MAAM,KAAG,MAC4B,CAAA;AA6TvE,eAAO,MAAM,eAAe,GAAI,OAAO,WAAW,KAAG,MAAM,GAAG,SAG7D,CAAA"}
|
|
@@ -62,6 +62,17 @@ const INJECTION_RETRY_ATTEMPT_TIMEOUT_MS = 15_000;
|
|
|
62
62
|
const INJECTION_MAX_ATTEMPTS = 6;
|
|
63
63
|
const BABYSITTER_EVENT_COALESCE_MS = 750;
|
|
64
64
|
const BABYSITTER_EVENT_RETRY_MS = 1_000;
|
|
65
|
+
// A babysitter wake fails with a "registration lag" error (agent_not_found /
|
|
66
|
+
// recipient unavailable) both when a freshly spawned agent has not finished
|
|
67
|
+
// enrolling AND when an agent is up but its relay identity never becomes
|
|
68
|
+
// resolvable (e.g. a resumed agent whose relay enrollment silently dropped).
|
|
69
|
+
// The two are indistinguishable per-attempt, so treating every such failure as
|
|
70
|
+
// transient produced an unbounded 1s retry loop that never recovered. Once the
|
|
71
|
+
// same wake has been failing this long, stop the tight loop: back off to a slow
|
|
72
|
+
// cadence and flag the stuck babysitter once for human attention. Genuine
|
|
73
|
+
// startup lag clears well within this window.
|
|
74
|
+
const BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS = 120_000;
|
|
75
|
+
const BABYSITTER_WAKE_UNREACHABLE_RETRY_MS = 60_000;
|
|
65
76
|
const CLARIFICATION_WAKE_LEASE_MS = 60_000;
|
|
66
77
|
const CLARIFICATION_WAKE_RETRY_MS = 1_000;
|
|
67
78
|
const CLARIFICATION_PARK_RETRY_MS = 5_000;
|
|
@@ -116,6 +127,8 @@ export class FactoryLoop {
|
|
|
116
127
|
#kill;
|
|
117
128
|
#readChildPids;
|
|
118
129
|
#terminationGraceMs;
|
|
130
|
+
#babysitterWakeUnreachableEscalateMs;
|
|
131
|
+
#babysitterWakeUnreachableRetryMs;
|
|
119
132
|
#state;
|
|
120
133
|
#workspaceId;
|
|
121
134
|
#relayflows;
|
|
@@ -264,6 +277,8 @@ export class FactoryLoop {
|
|
|
264
277
|
this.#kill = ports.kill ?? process.kill;
|
|
265
278
|
this.#readChildPids = ports.readChildPids;
|
|
266
279
|
this.#terminationGraceMs = ports.terminationGraceMs;
|
|
280
|
+
this.#babysitterWakeUnreachableEscalateMs = ports.babysitterWakeUnreachableEscalateMs ?? BABYSITTER_WAKE_UNREACHABLE_ESCALATE_MS;
|
|
281
|
+
this.#babysitterWakeUnreachableRetryMs = ports.babysitterWakeUnreachableRetryMs ?? BABYSITTER_WAKE_UNREACHABLE_RETRY_MS;
|
|
267
282
|
this.#workspaceId = config.workspaceId ?? 'default';
|
|
268
283
|
this.#relayflows = ports.relayflows;
|
|
269
284
|
this.#worktrees = ports.worktrees;
|
|
@@ -6256,6 +6271,8 @@ export class FactoryLoop {
|
|
|
6256
6271
|
let targets;
|
|
6257
6272
|
if (!this.#fleet.waitForInjected) {
|
|
6258
6273
|
await this.#fleet.sendMessage(input);
|
|
6274
|
+
state.unreachableSinceMs = undefined;
|
|
6275
|
+
state.unreachableEscalated = false;
|
|
6259
6276
|
if (this.#stopping || state.cancelled) {
|
|
6260
6277
|
state.deliveringKinds = undefined;
|
|
6261
6278
|
return;
|
|
@@ -6267,6 +6284,10 @@ export class FactoryLoop {
|
|
|
6267
6284
|
}
|
|
6268
6285
|
else {
|
|
6269
6286
|
const ack = await this.#waitForInjectedWithRetry(input);
|
|
6287
|
+
// Delivery confirmed: the target is reachable again, so clear any
|
|
6288
|
+
// registration-lag backoff state accumulated by prior failures.
|
|
6289
|
+
state.unreachableSinceMs = undefined;
|
|
6290
|
+
state.unreachableEscalated = false;
|
|
6270
6291
|
targets = ack.targets.length > 0 ? [...new Set(ack.targets)] : [input.to];
|
|
6271
6292
|
}
|
|
6272
6293
|
if (this.#stopping || state.cancelled)
|
|
@@ -6313,14 +6334,50 @@ export class FactoryLoop {
|
|
|
6313
6334
|
});
|
|
6314
6335
|
}
|
|
6315
6336
|
this.#increment('babysitterEventWakeFailures');
|
|
6316
|
-
|
|
6317
|
-
|
|
6318
|
-
|
|
6319
|
-
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6323
|
-
|
|
6337
|
+
const registrationLag = isRegistrationLagInjectionError(error);
|
|
6338
|
+
if (registrationLag) {
|
|
6339
|
+
state.unreachableSinceMs ??= this.#clock.now();
|
|
6340
|
+
}
|
|
6341
|
+
else {
|
|
6342
|
+
// A different failure mode (not "target unreachable") resets the
|
|
6343
|
+
// unreachable window so a later genuine registration lag starts fresh.
|
|
6344
|
+
state.unreachableSinceMs = undefined;
|
|
6345
|
+
state.unreachableEscalated = false;
|
|
6346
|
+
}
|
|
6347
|
+
const unreachableMs = state.unreachableSinceMs !== undefined
|
|
6348
|
+
? this.#clock.now() - state.unreachableSinceMs
|
|
6349
|
+
: 0;
|
|
6350
|
+
if (registrationLag && unreachableMs >= this.#babysitterWakeUnreachableEscalateMs) {
|
|
6351
|
+
// The agent is up but its relay identity never became resolvable. Stop
|
|
6352
|
+
// the tight 1s loop: back off to a slow cadence (still eventually
|
|
6353
|
+
// recovering if the agent finally enrolls) and flag it once so an
|
|
6354
|
+
// operator can intervene (e.g. re-spawn / restart) instead of the
|
|
6355
|
+
// failure spinning silently forever.
|
|
6356
|
+
state.nextDelayMs = this.#babysitterWakeUnreachableRetryMs;
|
|
6357
|
+
if (!state.unreachableEscalated) {
|
|
6358
|
+
state.unreachableEscalated = true;
|
|
6359
|
+
this.#increment('babysitterEventWakeUnreachableEscalations');
|
|
6360
|
+
this.#logger.warn?.('[factory] babysitter unreachable past grace window; slowing wake retries and flagging for human attention', {
|
|
6361
|
+
issue: state.issue.key,
|
|
6362
|
+
repo: state.repo,
|
|
6363
|
+
prNumber: state.prNumber,
|
|
6364
|
+
babysitter: state.agentName,
|
|
6365
|
+
unreachableMs,
|
|
6366
|
+
retryDelayMs: this.#babysitterWakeUnreachableRetryMs,
|
|
6367
|
+
error: describeError(error).errorMessage,
|
|
6368
|
+
});
|
|
6369
|
+
}
|
|
6370
|
+
}
|
|
6371
|
+
else {
|
|
6372
|
+
state.nextDelayMs = BABYSITTER_EVENT_RETRY_MS;
|
|
6373
|
+
this.#logger.warn?.('[factory] babysitter event wake failed; preserving it for retry', {
|
|
6374
|
+
issue: state.issue.key,
|
|
6375
|
+
repo: state.repo,
|
|
6376
|
+
prNumber: state.prNumber,
|
|
6377
|
+
babysitter: state.agentName,
|
|
6378
|
+
error: describeError(error).errorMessage,
|
|
6379
|
+
});
|
|
6380
|
+
}
|
|
6324
6381
|
}
|
|
6325
6382
|
}
|
|
6326
6383
|
async #submitBabysitterWakeTargets(targets) {
|