@loucompanion/forge-bridge 0.1.3-dev.ba7d36726eda → 0.1.3-dev.bca7d0ccee53
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bridge/bridge.base.d.ts +2 -1
- package/dist/bridge/bridge.service.d.ts +8 -0
- package/dist/bridge/bridge.service.js +184 -12
- package/dist/bridge/bridge.types.d.ts +37 -0
- package/dist/cli/cli.help.js +1 -1
- package/dist/cli/cli.service.js +1 -1
- package/dist/cli/local-config.js +2 -1
- package/dist/runtime/attempt-node-runtime-launcher.d.ts +3 -1
- package/dist/runtime/attempt-node-runtime-launcher.js +62 -11
- package/dist/runtime/attempt-process-supervisor-registry.js +1 -1
- package/dist/runtime/attempt-process-supervisor.d.ts +12 -0
- package/dist/runtime/attempt-process-supervisor.js +75 -15
- package/dist/runtime/generated-output-reset.d.ts +10 -0
- package/dist/runtime/generated-output-reset.js +103 -0
- package/dist/runtime/host-listener-inspector.d.ts +16 -0
- package/dist/runtime/host-listener-inspector.js +143 -4
- package/dist/runtime/host-runtime-contract.d.ts +9 -1
- package/dist/runtime/host-runtime-contract.js +411 -6
- package/dist/runtime/host-runtime-lease.d.ts +12 -1
- package/dist/runtime/host-runtime-lease.js +271 -50
- package/dist/runtime/host-runtime-promotion.d.ts +2 -0
- package/dist/runtime/host-runtime-promotion.js +35 -4
- package/dist/runtime/node-host-runtime-adapter.d.ts +7 -0
- package/dist/runtime/node-host-runtime-adapter.js +71 -8
- package/dist/runtime/playwright-host-runtime-adapter.js +84 -0
- package/dist/sandbox/preflight.d.ts +9 -0
- package/dist/sandbox/preflight.js +39 -4
- package/dist/sandbox/workspace.service.js +7 -2
- package/dist/session/managed-attempt-tools.d.ts +17 -0
- package/dist/session/managed-attempt-tools.js +712 -0
- package/dist/session/phase-completion.artifacts.js +9 -2
- package/dist/session/phase-completion.d.ts +11 -2
- package/dist/session/phase-completion.helpers.d.ts +5 -0
- package/dist/session/phase-completion.helpers.js +45 -9
- package/dist/session/phase-completion.js +12 -3
- package/dist/session/phase-completion.prompts.d.ts +3 -3
- package/dist/session/phase-completion.prompts.js +21 -6
- package/dist/session/phase-completion.runtime.d.ts +13 -1
- package/dist/session/phase-completion.runtime.js +183 -8
- package/dist/session/phase-completion.schemas.js +108 -3
- package/dist/session/phase-completion.validation.d.ts +4 -1
- package/dist/session/phase-completion.validation.js +144 -18
- package/dist/session/provider-cli/ci-repair-readiness.d.ts +8 -0
- package/dist/session/provider-cli/ci-repair-readiness.js +95 -0
- package/dist/session/provider-cli/provider-cli.deterministic.js +47 -1
- package/dist/session/provider-cli/provider-cli.process.d.ts +14 -0
- package/dist/session/provider-cli/provider-cli.process.js +176 -18
- package/dist/session/provider-cli/provider-cli.prompts.js +30 -8
- package/dist/session/provider-cli/provider-cli.qa-sidecars.js +1 -0
- package/dist/session/provider-cli/provider-cli.service.d.ts +19 -2
- package/dist/session/provider-cli/provider-cli.service.js +390 -83
- package/dist/session/provider-cli/provider-cli.types.d.ts +3 -0
- package/dist/session/provider-cli/provider-output-state.d.ts +2 -0
- package/dist/session/provider-cli/provider-output-state.js +30 -2
- package/dist/session/session.base.d.ts +2 -1
- package/dist/session/session.service.d.ts +2 -1
- package/dist/session/session.service.js +303 -30
- package/dist/session/session.types.d.ts +38 -0
- package/dist/session/session.types.js +17 -0
- package/dist/shared/process/host-process-supervisor.d.ts +18 -1
- package/dist/shared/process/host-process-supervisor.js +94 -7
- package/dist/shared/process/process.d.ts +1 -0
- package/dist/shared/process/process.js +1 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/worktree/artifact-injector.d.ts +3 -2
- package/dist/worktree/artifact-injector.js +43 -4
- package/dist/worktree/dependency-warmer.d.ts +10 -1
- package/dist/worktree/dependency-warmer.js +94 -5
- package/dist/worktree/worktree.service.d.ts +1 -0
- package/dist/worktree/worktree.service.js +12 -1
- package/package.json +1 -1
- package/runtime/windows-job-supervisor.ps1 +89 -20
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type { BridgeCancelHandler, BridgeDispatchHandler, BridgeLeaseUpdateHandler, BridgeOfferHandler, BridgeReapHandler, BridgeServiceConfig, BridgeStartHandler } from "./bridge.types.js";
|
|
1
|
+
import type { BridgeCancelHandler, BridgeDispatchHandler, BridgeLeaseUpdateHandler, BridgeOfferHandler, BridgeReadinessProbeHandler, BridgeReapHandler, BridgeServiceConfig, BridgeStartHandler } from "./bridge.types.js";
|
|
2
2
|
export interface BridgeService {
|
|
3
3
|
connectOnce(): Promise<void>;
|
|
4
4
|
run(signal?: AbortSignal): Promise<void>;
|
|
@@ -14,6 +14,7 @@ export interface BridgeConnection {
|
|
|
14
14
|
onLeaseUpdate?(handler: BridgeLeaseUpdateHandler): void;
|
|
15
15
|
onCancel(handler: BridgeCancelHandler): void;
|
|
16
16
|
onReap?(handler: BridgeReapHandler): void;
|
|
17
|
+
onReadinessProbe?(handler: BridgeReadinessProbeHandler): void;
|
|
17
18
|
invoke<TResponse>(methodName: string, payload: unknown): Promise<TResponse>;
|
|
18
19
|
}
|
|
19
20
|
export interface BridgeConnectionFactory {
|
|
@@ -15,6 +15,9 @@ export declare class ForgeBridgeService implements BridgeService {
|
|
|
15
15
|
private readonly connections;
|
|
16
16
|
private connection;
|
|
17
17
|
private heartbeatTimer;
|
|
18
|
+
private heartbeatInFlight;
|
|
19
|
+
private consecutiveHeartbeatFailures;
|
|
20
|
+
private reconnectingFromHeartbeatFailure;
|
|
18
21
|
private leaseWatchdogTimer;
|
|
19
22
|
private sandboxMaintenanceTimer;
|
|
20
23
|
private sandboxMaintenanceRunning;
|
|
@@ -22,6 +25,7 @@ export declare class ForgeBridgeService implements BridgeService {
|
|
|
22
25
|
private stopped;
|
|
23
26
|
private readonly activeSessions;
|
|
24
27
|
private readonly reservations;
|
|
28
|
+
private readonly activeReadinessProbes;
|
|
25
29
|
private readonly preflight;
|
|
26
30
|
private readonly sandboxRuntime;
|
|
27
31
|
private readonly sandboxResourceResolver;
|
|
@@ -40,12 +44,15 @@ export declare class ForgeBridgeService implements BridgeService {
|
|
|
40
44
|
private parseDispatchOrReport;
|
|
41
45
|
private handleCancel;
|
|
42
46
|
private handleReap;
|
|
47
|
+
private handleReadinessProbe;
|
|
43
48
|
private abortActiveSessions;
|
|
49
|
+
private abortActiveReadinessProbes;
|
|
44
50
|
private normalizeDispatch;
|
|
45
51
|
private sendResult;
|
|
46
52
|
private sendLegacyResult;
|
|
47
53
|
private sendResultPayload;
|
|
48
54
|
private startHeartbeat;
|
|
55
|
+
private heartbeatTick;
|
|
49
56
|
private startLeaseWatchdog;
|
|
50
57
|
private stopLeaseWatchdog;
|
|
51
58
|
private expireLeases;
|
|
@@ -80,6 +87,7 @@ export declare class SignalRBridgeConnection implements BridgeConnection {
|
|
|
80
87
|
onLeaseUpdate(handler: (request: BridgeLeaseUpdateRequest) => Promise<void>): void;
|
|
81
88
|
onCancel(handler: BridgeCancelHandler): void;
|
|
82
89
|
onReap(handler: (request: BridgeReapRequest) => Promise<void>): void;
|
|
90
|
+
onReadinessProbe(handler: (request: unknown) => Promise<void>): void;
|
|
83
91
|
invoke<TResponse>(methodName: string, payload: unknown): Promise<TResponse>;
|
|
84
92
|
}
|
|
85
93
|
export declare function defaultBridgeServiceConfig(overrides?: Partial<BridgeServiceConfig>): BridgeServiceConfig;
|
|
@@ -25,6 +25,9 @@ export class ForgeBridgeService {
|
|
|
25
25
|
connections;
|
|
26
26
|
connection = null;
|
|
27
27
|
heartbeatTimer = null;
|
|
28
|
+
heartbeatInFlight = false;
|
|
29
|
+
consecutiveHeartbeatFailures = 0;
|
|
30
|
+
reconnectingFromHeartbeatFailure = false;
|
|
28
31
|
leaseWatchdogTimer = null;
|
|
29
32
|
sandboxMaintenanceTimer = null;
|
|
30
33
|
sandboxMaintenanceRunning = false;
|
|
@@ -32,6 +35,7 @@ export class ForgeBridgeService {
|
|
|
32
35
|
stopped = false;
|
|
33
36
|
activeSessions = new Map();
|
|
34
37
|
reservations = new Map();
|
|
38
|
+
activeReadinessProbes = new Map();
|
|
35
39
|
preflight;
|
|
36
40
|
sandboxRuntime;
|
|
37
41
|
sandboxResourceResolver;
|
|
@@ -82,6 +86,7 @@ export class ForgeBridgeService {
|
|
|
82
86
|
connection.onClose(() => {
|
|
83
87
|
this.stopHeartbeat();
|
|
84
88
|
this.abortActiveSessions("bridge connection closed");
|
|
89
|
+
this.abortActiveReadinessProbes("bridge connection closed");
|
|
85
90
|
resolve();
|
|
86
91
|
});
|
|
87
92
|
});
|
|
@@ -91,6 +96,7 @@ export class ForgeBridgeService {
|
|
|
91
96
|
connection.onLeaseUpdate?.((request) => this.handleLeaseUpdate(request));
|
|
92
97
|
connection.onCancel((request) => this.handleCancel(request));
|
|
93
98
|
connection.onReap?.((request) => this.handleReap(request));
|
|
99
|
+
connection.onReadinessProbe?.((request) => this.handleReadinessProbe(request));
|
|
94
100
|
await connection.start();
|
|
95
101
|
await this.invokeAck("Hello", {
|
|
96
102
|
version: this.config.version,
|
|
@@ -133,6 +139,7 @@ export class ForgeBridgeService {
|
|
|
133
139
|
finally {
|
|
134
140
|
if (signal?.aborted) {
|
|
135
141
|
this.abortActiveSessions(String(signal.reason ?? "bridge stopping"));
|
|
142
|
+
this.abortActiveReadinessProbes(String(signal.reason ?? "bridge stopping"));
|
|
136
143
|
}
|
|
137
144
|
await this.stopConnection();
|
|
138
145
|
}
|
|
@@ -140,6 +147,7 @@ export class ForgeBridgeService {
|
|
|
140
147
|
async stop() {
|
|
141
148
|
this.stopped = true;
|
|
142
149
|
this.abortActiveSessions("bridge stopping");
|
|
150
|
+
this.abortActiveReadinessProbes("bridge stopping");
|
|
143
151
|
await this.stopConnection();
|
|
144
152
|
await this.preflight.release();
|
|
145
153
|
}
|
|
@@ -221,7 +229,14 @@ export class ForgeBridgeService {
|
|
|
221
229
|
const summary = controller.signal.aborted
|
|
222
230
|
? `Dispatch cancelled: ${String(controller.signal.reason ?? "cancelled")}`
|
|
223
231
|
: redactText(error instanceof Error ? error.message : String(error), redactionSecrets);
|
|
224
|
-
|
|
232
|
+
const failureEventType = infrastructureFailureEventType(summary);
|
|
233
|
+
const failureEvents = failureEventType ? [{
|
|
234
|
+
role: "runner",
|
|
235
|
+
kind: "error",
|
|
236
|
+
content: summary,
|
|
237
|
+
eventType: failureEventType,
|
|
238
|
+
}] : [];
|
|
239
|
+
await this.sendResult(dispatch, failureResult(summary, failureEvents)).catch(() => undefined);
|
|
225
240
|
}
|
|
226
241
|
finally {
|
|
227
242
|
this.activeSessions.delete(leaseId);
|
|
@@ -270,6 +285,11 @@ export class ForgeBridgeService {
|
|
|
270
285
|
await this.sendOfferDecision(dispatch, false, "policy_mismatch", networkFailure);
|
|
271
286
|
return;
|
|
272
287
|
}
|
|
288
|
+
const providerNetworkFailure = sandboxProviderNetworkFailure(dispatch, this.config);
|
|
289
|
+
if (providerNetworkFailure) {
|
|
290
|
+
await this.sendOfferDecision(dispatch, false, "provider_network_unavailable", providerNetworkFailure);
|
|
291
|
+
return;
|
|
292
|
+
}
|
|
273
293
|
const authFailure = sandboxAuthFailure(dispatch, this.config);
|
|
274
294
|
if (authFailure) {
|
|
275
295
|
await this.sendOfferDecision(dispatch, false, "sandbox_auth_unavailable", authFailure);
|
|
@@ -452,19 +472,73 @@ export class ForgeBridgeService {
|
|
|
452
472
|
active.controller.abort(request.reason ?? "cancelled by server");
|
|
453
473
|
}
|
|
454
474
|
async handleReap(request) {
|
|
455
|
-
if (
|
|
456
|
-
|| request.runnerId !== this.config.runnerId
|
|
475
|
+
if (request.runnerId !== this.config.runnerId
|
|
457
476
|
|| request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
458
477
|
return;
|
|
459
478
|
await this.runSandboxMaintenance();
|
|
460
479
|
await this.sendHeartbeat();
|
|
461
480
|
}
|
|
481
|
+
async handleReadinessProbe(rawRequest) {
|
|
482
|
+
const request = parseReadinessProbe(rawRequest);
|
|
483
|
+
if (!request || request.runnerInstanceId !== this.config.runnerInstanceId)
|
|
484
|
+
return;
|
|
485
|
+
const existing = this.activeReadinessProbes.get(request.requestId);
|
|
486
|
+
if (existing)
|
|
487
|
+
return;
|
|
488
|
+
const controller = new AbortController();
|
|
489
|
+
this.activeReadinessProbes.set(request.requestId, controller);
|
|
490
|
+
const deadlineMs = Date.parse(request.expiresAt) - Date.now();
|
|
491
|
+
const timer = setTimeout(() => controller.abort("readiness probe expired"), Math.max(0, deadlineMs));
|
|
492
|
+
timer.unref?.();
|
|
493
|
+
let result;
|
|
494
|
+
try {
|
|
495
|
+
if (!this.sessions.probeHostRuntime) {
|
|
496
|
+
throw new Error("host_runtime_probe_unavailable: this bridge does not implement runtime readiness probes.");
|
|
497
|
+
}
|
|
498
|
+
const probe = await this.sessions.probeHostRuntime({
|
|
499
|
+
...request,
|
|
500
|
+
runnerId: this.config.runnerId,
|
|
501
|
+
repoRoot: this.config.defaultRepoRoot,
|
|
502
|
+
cleanupWorktrees: this.config.cleanupWorktrees,
|
|
503
|
+
}, controller.signal);
|
|
504
|
+
result = {
|
|
505
|
+
schemaVersion: 1,
|
|
506
|
+
requestId: request.requestId,
|
|
507
|
+
fencingToken: request.fencingToken,
|
|
508
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
509
|
+
...probe,
|
|
510
|
+
};
|
|
511
|
+
}
|
|
512
|
+
catch (error) {
|
|
513
|
+
result = {
|
|
514
|
+
schemaVersion: 1,
|
|
515
|
+
requestId: request.requestId,
|
|
516
|
+
fencingToken: request.fencingToken,
|
|
517
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
518
|
+
status: "failed",
|
|
519
|
+
reasonCode: "host_runtime_probe_failed",
|
|
520
|
+
reasonMessage: errorMessage(error).slice(0, 1000),
|
|
521
|
+
checkedAt: new Date().toISOString(),
|
|
522
|
+
};
|
|
523
|
+
}
|
|
524
|
+
finally {
|
|
525
|
+
clearTimeout(timer);
|
|
526
|
+
this.activeReadinessProbes.delete(request.requestId);
|
|
527
|
+
}
|
|
528
|
+
await this.invokeAck("ReadinessResult", result).catch(() => undefined);
|
|
529
|
+
await this.sendHeartbeat().catch(() => undefined);
|
|
530
|
+
}
|
|
462
531
|
abortActiveSessions(reason) {
|
|
463
532
|
for (const active of this.activeSessions.values()) {
|
|
464
533
|
active.controller.abort(reason);
|
|
465
534
|
}
|
|
466
535
|
this.activeSessions.clear();
|
|
467
536
|
}
|
|
537
|
+
abortActiveReadinessProbes(reason) {
|
|
538
|
+
for (const controller of this.activeReadinessProbes.values())
|
|
539
|
+
controller.abort(reason);
|
|
540
|
+
this.activeReadinessProbes.clear();
|
|
541
|
+
}
|
|
468
542
|
normalizeDispatch(dispatch) {
|
|
469
543
|
const mapping = findRepoMapping(dispatch, repoMappingsFor(this.config));
|
|
470
544
|
const localPath = mapping?.localPath?.trim();
|
|
@@ -515,12 +589,41 @@ export class ForgeBridgeService {
|
|
|
515
589
|
}
|
|
516
590
|
startHeartbeat() {
|
|
517
591
|
this.stopHeartbeat();
|
|
592
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
593
|
+
this.reconnectingFromHeartbeatFailure = false;
|
|
518
594
|
const intervalMs = positiveFiniteNumber(this.config.heartbeatIntervalMs, 30_000);
|
|
519
595
|
this.heartbeatTimer = setInterval(() => {
|
|
520
|
-
this.
|
|
596
|
+
void this.heartbeatTick();
|
|
521
597
|
}, intervalMs);
|
|
522
598
|
this.heartbeatTimer.unref?.();
|
|
523
599
|
}
|
|
600
|
+
async heartbeatTick() {
|
|
601
|
+
if (this.heartbeatInFlight || this.reconnectingFromHeartbeatFailure)
|
|
602
|
+
return;
|
|
603
|
+
const connection = this.connection;
|
|
604
|
+
if (!connection)
|
|
605
|
+
return;
|
|
606
|
+
this.heartbeatInFlight = true;
|
|
607
|
+
try {
|
|
608
|
+
await this.sendHeartbeat();
|
|
609
|
+
this.consecutiveHeartbeatFailures = 0;
|
|
610
|
+
}
|
|
611
|
+
catch (error) {
|
|
612
|
+
if (connection !== this.connection || this.stopped)
|
|
613
|
+
return;
|
|
614
|
+
this.consecutiveHeartbeatFailures += 1;
|
|
615
|
+
logBridgeStatus(`Heartbeat ${this.consecutiveHeartbeatFailures} failed: ${errorMessage(error)}.`);
|
|
616
|
+
if (this.consecutiveHeartbeatFailures < 2)
|
|
617
|
+
return;
|
|
618
|
+
this.reconnectingFromHeartbeatFailure = true;
|
|
619
|
+
this.stopHeartbeat();
|
|
620
|
+
logBridgeStatus("Heartbeat failed twice. Closing the stale connection so the Bridge can reconnect...");
|
|
621
|
+
await connection.stop().catch(() => undefined);
|
|
622
|
+
}
|
|
623
|
+
finally {
|
|
624
|
+
this.heartbeatInFlight = false;
|
|
625
|
+
}
|
|
626
|
+
}
|
|
524
627
|
startLeaseWatchdog() {
|
|
525
628
|
this.stopLeaseWatchdog();
|
|
526
629
|
this.leaseWatchdogTimer = setInterval(() => this.expireLeases(), 1000);
|
|
@@ -553,8 +656,6 @@ export class ForgeBridgeService {
|
|
|
553
656
|
}
|
|
554
657
|
startSandboxMaintenance() {
|
|
555
658
|
this.stopSandboxMaintenance();
|
|
556
|
-
if (!this.config.dockerEnabled)
|
|
557
|
-
return;
|
|
558
659
|
this.sandboxMaintenanceTimer = setInterval(() => {
|
|
559
660
|
this.runSandboxMaintenance().catch(() => undefined);
|
|
560
661
|
}, 60_000);
|
|
@@ -571,11 +672,20 @@ export class ForgeBridgeService {
|
|
|
571
672
|
return;
|
|
572
673
|
this.sandboxMaintenanceRunning = true;
|
|
573
674
|
try {
|
|
574
|
-
const
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
675
|
+
const hostReap = await this.sessions.reapHostRuntime?.({
|
|
676
|
+
runnerId: this.config.runnerId,
|
|
677
|
+
runnerInstanceId: this.config.runnerInstanceId,
|
|
678
|
+
});
|
|
679
|
+
if (hostReap && (hostReap.reaped > 0 || hostReap.degraded > 0)) {
|
|
680
|
+
logBridgeStatus(`Managed-host maintenance reaped ${hostReap.reaped} stale attempt(s) and left ${hostReap.degraded} degraded attempt(s) for reconciliation.`);
|
|
681
|
+
}
|
|
682
|
+
if (this.config.dockerEnabled) {
|
|
683
|
+
const preflight = await this.preflight.refresh();
|
|
684
|
+
if (canReapManagedSandboxes(preflight)) {
|
|
685
|
+
const attemptSnapshot = await this.readAttemptSnapshot();
|
|
686
|
+
await this.sandboxRuntime.reap({ diskLow: preflight.diskLow, attemptSnapshot });
|
|
687
|
+
await this.flushCleanupReports();
|
|
688
|
+
}
|
|
579
689
|
}
|
|
580
690
|
}
|
|
581
691
|
finally {
|
|
@@ -639,6 +749,7 @@ export class ForgeBridgeService {
|
|
|
639
749
|
sandboxRuntimes: preflight?.healthy ? ["docker"] : [],
|
|
640
750
|
sandboxRuntimeVersion: preflight?.runtimeVersion ?? null,
|
|
641
751
|
sandboxFeatures: preflight?.features ?? null,
|
|
752
|
+
allowsOpenPublicSandbox: this.config.allowOpenPublicSandbox,
|
|
642
753
|
sandboxPublicationModes: [this.config.sandboxPublicationMode ?? "shared_filesystem"],
|
|
643
754
|
sandboxAuthModes: sandboxAuthModesFor(this.config),
|
|
644
755
|
availableMemoryBytes: preflight ? Math.max(0, capacity.memoryBytes - used.memoryBytes) : null,
|
|
@@ -824,6 +935,11 @@ export class SignalRBridgeConnection {
|
|
|
824
935
|
void handler(request).catch(() => undefined);
|
|
825
936
|
});
|
|
826
937
|
}
|
|
938
|
+
onReadinessProbe(handler) {
|
|
939
|
+
this.connection.on("ReadinessProbe", (request) => {
|
|
940
|
+
void handler(request).catch(() => undefined);
|
|
941
|
+
});
|
|
942
|
+
}
|
|
827
943
|
invoke(methodName, payload) {
|
|
828
944
|
return this.connection.invoke(methodName, payload);
|
|
829
945
|
}
|
|
@@ -1030,10 +1146,11 @@ function normalizeQaAppLaunchTarget(target) {
|
|
|
1030
1146
|
command: cleanString(target.command),
|
|
1031
1147
|
working_directory: cleanString(target.working_directory),
|
|
1032
1148
|
ready_url: cleanString(target.ready_url),
|
|
1149
|
+
managed_runtime_port: cleanString(target.managed_runtime_port),
|
|
1033
1150
|
routes,
|
|
1034
1151
|
notes: cleanString(target.notes)
|
|
1035
1152
|
};
|
|
1036
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
1153
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
1037
1154
|
? normalized
|
|
1038
1155
|
: null;
|
|
1039
1156
|
}
|
|
@@ -1117,6 +1234,61 @@ function sandboxNetworkPolicyFailure(dispatch, config) {
|
|
|
1117
1234
|
}
|
|
1118
1235
|
return null;
|
|
1119
1236
|
}
|
|
1237
|
+
function infrastructureFailureEventType(summary) {
|
|
1238
|
+
if (summary.startsWith("worktree_prepare_failed:"))
|
|
1239
|
+
return "worktree_prepare_failed";
|
|
1240
|
+
if (summary.startsWith("provider_start_unavailable:"))
|
|
1241
|
+
return "provider_start_unavailable";
|
|
1242
|
+
if (summary.startsWith("host_port_collision:"))
|
|
1243
|
+
return "host_port_collision";
|
|
1244
|
+
if (summary.startsWith("host_runtime_process_exited:"))
|
|
1245
|
+
return "host_runtime_process_exited";
|
|
1246
|
+
if (summary.startsWith("host_runtime_listener_timeout:"))
|
|
1247
|
+
return "host_runtime_listener_timeout";
|
|
1248
|
+
if (summary.startsWith("host_runtime_http_readiness_timeout:"))
|
|
1249
|
+
return "host_runtime_http_readiness_timeout";
|
|
1250
|
+
if (summary.startsWith("attempt_process_supervisor_start_failed:"))
|
|
1251
|
+
return "attempt_process_supervisor_start_failed";
|
|
1252
|
+
if (summary.startsWith("attempt_process_supervisor_start_timeout:"))
|
|
1253
|
+
return "attempt_process_supervisor_start_timeout";
|
|
1254
|
+
return null;
|
|
1255
|
+
}
|
|
1256
|
+
function parseReadinessProbe(value) {
|
|
1257
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
1258
|
+
return null;
|
|
1259
|
+
const request = value;
|
|
1260
|
+
const uuid = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/iu;
|
|
1261
|
+
const bounded = (candidate, maximum) => typeof candidate === "string" && candidate.trim().length > 0 && candidate.length <= maximum;
|
|
1262
|
+
if (request.schemaVersion !== 1
|
|
1263
|
+
|| !uuid.test(request.requestId ?? "")
|
|
1264
|
+
|| !uuid.test(request.fencingToken ?? "")
|
|
1265
|
+
|| !uuid.test(request.userId ?? "")
|
|
1266
|
+
|| !uuid.test(request.projectId ?? "")
|
|
1267
|
+
|| !uuid.test(request.repoId ?? "")
|
|
1268
|
+
|| !uuid.test(request.runnerInstanceId ?? "")
|
|
1269
|
+
|| !Number.isInteger(request.attempt) || (request.attempt ?? 0) < 1
|
|
1270
|
+
|| !bounded(request.repoName, 160)
|
|
1271
|
+
|| typeof request.remoteUrl !== "string" || request.remoteUrl.length > 2048
|
|
1272
|
+
|| !bounded(request.defaultBranch, 512)
|
|
1273
|
+
|| typeof request.cloneOnDemand !== "boolean"
|
|
1274
|
+
|| !bounded(request.runnerRepoLocalPath, 4096)
|
|
1275
|
+
|| !bounded(request.revision, 512)
|
|
1276
|
+
|| !Number.isFinite(Date.parse(request.expiresAt ?? ""))
|
|
1277
|
+
|| Date.parse(request.expiresAt ?? "") <= Date.now()) {
|
|
1278
|
+
return null;
|
|
1279
|
+
}
|
|
1280
|
+
return request;
|
|
1281
|
+
}
|
|
1282
|
+
function sandboxProviderNetworkFailure(dispatch, config) {
|
|
1283
|
+
const sandbox = dispatch.sandbox;
|
|
1284
|
+
if (!sandbox || sandbox.networkPolicy.mode !== "offline")
|
|
1285
|
+
return null;
|
|
1286
|
+
const provider = dispatch.execution.provider.trim().toLowerCase();
|
|
1287
|
+
const isNetworkProvider = config.cliProviders.some(candidate => candidate.trim().toLowerCase() === provider);
|
|
1288
|
+
if (!isNetworkProvider)
|
|
1289
|
+
return null;
|
|
1290
|
+
return `Provider ${dispatch.execution.provider} requires outbound network access, but this Docker attempt uses the offline network policy.`;
|
|
1291
|
+
}
|
|
1120
1292
|
function sandboxAuthFailure(dispatch, config) {
|
|
1121
1293
|
const unsupported = sandboxProviderAuthUnavailableMessage(dispatch.execution.provider, dispatch.execution.authKind);
|
|
1122
1294
|
if (unsupported) {
|
|
@@ -63,6 +63,7 @@ export interface QaAppLaunchTarget {
|
|
|
63
63
|
readonly command?: string | null;
|
|
64
64
|
readonly working_directory?: string | null;
|
|
65
65
|
readonly ready_url?: string | null;
|
|
66
|
+
readonly managed_runtime_port?: string | null;
|
|
66
67
|
readonly routes?: readonly string[] | null;
|
|
67
68
|
readonly notes?: string | null;
|
|
68
69
|
}
|
|
@@ -85,6 +86,7 @@ export interface BridgeCapabilities {
|
|
|
85
86
|
readonly sandboxRuntimes?: readonly string[] | null;
|
|
86
87
|
readonly sandboxRuntimeVersion?: string | null;
|
|
87
88
|
readonly sandboxFeatures?: SandboxFeatureMatrix | null;
|
|
89
|
+
readonly allowsOpenPublicSandbox?: boolean | null;
|
|
88
90
|
readonly sandboxPublicationModes?: readonly string[] | null;
|
|
89
91
|
readonly sandboxAuthModes?: Readonly<Record<string, readonly string[]>> | null;
|
|
90
92
|
readonly maxConcurrentSandboxes?: number | null;
|
|
@@ -226,3 +228,38 @@ export type BridgeStartHandler = (dispatch: unknown) => Promise<void>;
|
|
|
226
228
|
export type BridgeLeaseUpdateHandler = (request: BridgeLeaseUpdateRequest) => Promise<void>;
|
|
227
229
|
export type BridgeCancelHandler = (request: BridgeCancelRequest) => Promise<void>;
|
|
228
230
|
export type BridgeReapHandler = (request: BridgeReapRequest) => Promise<void>;
|
|
231
|
+
export type BridgeReadinessProbeHandler = (request: unknown) => Promise<void>;
|
|
232
|
+
export interface BridgeRuntimeReadinessRequest {
|
|
233
|
+
readonly schemaVersion: 1;
|
|
234
|
+
readonly requestId: string;
|
|
235
|
+
readonly fencingToken: string;
|
|
236
|
+
readonly userId: string;
|
|
237
|
+
readonly projectId: string;
|
|
238
|
+
readonly repoId: string;
|
|
239
|
+
readonly runnerInstanceId: string;
|
|
240
|
+
readonly attempt: number;
|
|
241
|
+
readonly repoName: string;
|
|
242
|
+
readonly remoteUrl: string;
|
|
243
|
+
readonly defaultBranch: string;
|
|
244
|
+
readonly cloneOnDemand: boolean;
|
|
245
|
+
readonly runnerRepoLocalPath: string;
|
|
246
|
+
readonly revision: string;
|
|
247
|
+
readonly expiresAt: string;
|
|
248
|
+
}
|
|
249
|
+
export interface BridgeRuntimeReadinessResult {
|
|
250
|
+
readonly schemaVersion: 1;
|
|
251
|
+
readonly requestId: string;
|
|
252
|
+
readonly fencingToken: string;
|
|
253
|
+
readonly runnerInstanceId: string;
|
|
254
|
+
readonly status: "parallel_ready" | "serialized" | "failed";
|
|
255
|
+
readonly runtimeContractFingerprint?: string | null;
|
|
256
|
+
readonly runtimeContractSource?: string | null;
|
|
257
|
+
readonly reasonCode?: string | null;
|
|
258
|
+
readonly reasonMessage?: string | null;
|
|
259
|
+
readonly qaAppLaunchTarget?: BridgeRuntimeReadinessQaLaunchTarget | null;
|
|
260
|
+
readonly checkedAt: string;
|
|
261
|
+
}
|
|
262
|
+
export interface BridgeRuntimeReadinessQaLaunchTarget {
|
|
263
|
+
readonly managedRuntimePort: string;
|
|
264
|
+
readonly routes: readonly string[];
|
|
265
|
+
}
|
package/dist/cli/cli.help.js
CHANGED
|
@@ -194,7 +194,7 @@ export function commandHelpText(command) {
|
|
|
194
194
|
"Internal MCP server used by provider CLIs to report Forge phase artifacts.",
|
|
195
195
|
"",
|
|
196
196
|
"Usage:",
|
|
197
|
-
" forge-bridge mcp --worktree <path> --work-item-id <id> --phase <phase>",
|
|
197
|
+
" forge-bridge mcp --worktree <path> --work-item-id <id> --phase <phase> # includes ci-repair",
|
|
198
198
|
"",
|
|
199
199
|
].join("\n");
|
|
200
200
|
case "version":
|
package/dist/cli/cli.service.js
CHANGED
|
@@ -714,7 +714,7 @@ function parsePairArgs(argv) {
|
|
|
714
714
|
name: name?.trim() || defaultName,
|
|
715
715
|
fingerprint: fingerprint?.trim() || `local-bridge:${os.hostname()}:${user}`,
|
|
716
716
|
repoRoot: repoRoot?.trim() ? path.resolve(repoRoot) : undefined,
|
|
717
|
-
cleanupWorktrees: cleanupWorktrees ??
|
|
717
|
+
cleanupWorktrees: cleanupWorktrees ?? defaultConfig.cleanupWorktrees,
|
|
718
718
|
cliProviders: cliProviders.length > 0 ? cliProviders : localConfig.cliProviders && localConfig.cliProviders.length > 0 ? [...localConfig.cliProviders] : ["codex"],
|
|
719
719
|
projectLabels: projectLabels.length > 0 ? projectLabels : localConfig.projectLabels ? [...localConfig.projectLabels] : []
|
|
720
720
|
};
|
package/dist/cli/local-config.js
CHANGED
|
@@ -221,10 +221,11 @@ function normalizeQaAppLaunchTarget(value) {
|
|
|
221
221
|
command: cleanString(value.command),
|
|
222
222
|
working_directory: cleanString(value.working_directory),
|
|
223
223
|
ready_url: cleanString(value.ready_url),
|
|
224
|
+
managed_runtime_port: cleanString(value.managed_runtime_port),
|
|
224
225
|
routes,
|
|
225
226
|
notes: cleanString(value.notes)
|
|
226
227
|
};
|
|
227
|
-
return normalized.url || normalized.command || normalized.ready_url ||
|
|
228
|
+
return normalized.url || normalized.command || normalized.ready_url || normalized.managed_runtime_port
|
|
228
229
|
? normalized
|
|
229
230
|
: undefined;
|
|
230
231
|
}
|
|
@@ -1,15 +1,17 @@
|
|
|
1
1
|
import type { AttemptProcessChildHandle, AttemptProcessSupervisor } from "./attempt-process-supervisor.js";
|
|
2
2
|
import type { HostPortLeaseOwner } from "./host-port-broker.js";
|
|
3
3
|
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
4
|
-
import type { HostListenerOwnership, NodeRuntimeCommandLauncher, NodeRuntimeLaunchRequest } from "./node-host-runtime-adapter.js";
|
|
4
|
+
import type { HostListenerOwnership, NodeRuntimeCommandLauncher, NodeRuntimeLaunchRequest, NodeRuntimeReadinessDiagnostics } from "./node-host-runtime-adapter.js";
|
|
5
5
|
/** Launches one explicitly configured HTTP project beneath the attempt coordinator. */
|
|
6
6
|
export declare class AttemptNodeRuntimeLauncher implements NodeRuntimeCommandLauncher<AttemptProcessChildHandle> {
|
|
7
7
|
private readonly supervisor;
|
|
8
8
|
private readonly owner;
|
|
9
9
|
private readonly logDirectory;
|
|
10
10
|
private readonly listenerInspector;
|
|
11
|
+
private readonly diagnosticsByChild;
|
|
11
12
|
constructor(supervisor: AttemptProcessSupervisor, owner: HostPortLeaseOwner, logDirectory: string, listenerInspector?: HostListenerInspector);
|
|
12
13
|
launch(request: NodeRuntimeLaunchRequest, signal?: AbortSignal): Promise<AttemptProcessChildHandle>;
|
|
13
14
|
inspectListener(child: AttemptProcessChildHandle, request: NodeRuntimeLaunchRequest): Promise<HostListenerOwnership>;
|
|
14
15
|
stop(child: AttemptProcessChildHandle): Promise<void>;
|
|
16
|
+
readinessDiagnostics(child: AttemptProcessChildHandle): Promise<NodeRuntimeReadinessDiagnostics>;
|
|
15
17
|
}
|
|
@@ -2,12 +2,14 @@ import { createHash } from "node:crypto";
|
|
|
2
2
|
import fs from "node:fs/promises";
|
|
3
3
|
import path from "node:path";
|
|
4
4
|
import { HostListenerInspector } from "./host-listener-inspector.js";
|
|
5
|
+
const diagnosticTailLength = 4_000;
|
|
5
6
|
/** Launches one explicitly configured HTTP project beneath the attempt coordinator. */
|
|
6
7
|
export class AttemptNodeRuntimeLauncher {
|
|
7
8
|
supervisor;
|
|
8
9
|
owner;
|
|
9
10
|
logDirectory;
|
|
10
11
|
listenerInspector;
|
|
12
|
+
diagnosticsByChild = new WeakMap();
|
|
11
13
|
constructor(supervisor, owner, logDirectory, listenerInspector = new HostListenerInspector()) {
|
|
12
14
|
this.supervisor = supervisor;
|
|
13
15
|
this.owner = owner;
|
|
@@ -18,17 +20,39 @@ export class AttemptNodeRuntimeLauncher {
|
|
|
18
20
|
const childName = projectChildName(request.port.name);
|
|
19
21
|
const stdoutPath = path.join(this.logDirectory, `${childName}.stdout.log`);
|
|
20
22
|
const stderrPath = path.join(this.logDirectory, `${childName}.stderr.log`);
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
23
|
+
const diagnostics = { exited: false, stdoutTail: "", stderrTail: "" };
|
|
24
|
+
let child;
|
|
25
|
+
try {
|
|
26
|
+
child = await this.supervisor.startChildReady(childName, {
|
|
27
|
+
command: request.command,
|
|
28
|
+
args: request.args,
|
|
29
|
+
cwd: request.cwd,
|
|
30
|
+
env: request.environment,
|
|
31
|
+
captureOutput: false,
|
|
32
|
+
}, {
|
|
33
|
+
onStdoutLine: async (line) => {
|
|
34
|
+
diagnostics.stdoutTail = appendTail(diagnostics.stdoutTail, line);
|
|
35
|
+
await appendLine(stdoutPath, line);
|
|
36
|
+
},
|
|
37
|
+
onStderrLine: async (line) => {
|
|
38
|
+
diagnostics.stderrTail = appendTail(diagnostics.stderrTail, line);
|
|
39
|
+
await appendLine(stderrPath, line);
|
|
40
|
+
},
|
|
41
|
+
}, signal, { maxStartAttempts: 3, retryDelayMs: 250 });
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
throw enrichLaunchFailure(error, diagnostics);
|
|
45
|
+
}
|
|
46
|
+
this.diagnosticsByChild.set(child, diagnostics);
|
|
47
|
+
void child.result.then(result => {
|
|
48
|
+
diagnostics.exited = true;
|
|
49
|
+
diagnostics.exitCode = result.exitCode;
|
|
50
|
+
diagnostics.stdoutTail = appendTail(diagnostics.stdoutTail, result.stdout);
|
|
51
|
+
diagnostics.stderrTail = appendTail(diagnostics.stderrTail, result.stderr);
|
|
52
|
+
}, error => {
|
|
53
|
+
diagnostics.exited = true;
|
|
54
|
+
diagnostics.stderrTail = appendTail(diagnostics.stderrTail, error instanceof Error ? error.message : String(error));
|
|
55
|
+
});
|
|
32
56
|
return child;
|
|
33
57
|
}
|
|
34
58
|
async inspectListener(child, request) {
|
|
@@ -37,6 +61,12 @@ export class AttemptNodeRuntimeLauncher {
|
|
|
37
61
|
async stop(child) {
|
|
38
62
|
await child.stop(this.owner.fencingToken);
|
|
39
63
|
}
|
|
64
|
+
async readinessDiagnostics(child) {
|
|
65
|
+
// Yield once so an already-settled result can update the diagnostic record.
|
|
66
|
+
await Promise.resolve();
|
|
67
|
+
const diagnostics = this.diagnosticsByChild.get(child);
|
|
68
|
+
return diagnostics ?? { exited: false };
|
|
69
|
+
}
|
|
40
70
|
}
|
|
41
71
|
function projectChildName(portName) {
|
|
42
72
|
const candidate = `project-${portName}`;
|
|
@@ -48,3 +78,24 @@ function projectChildName(portName) {
|
|
|
48
78
|
async function appendLine(target, line) {
|
|
49
79
|
await fs.appendFile(target, `${line}\n`, { encoding: "utf8", mode: 0o600 });
|
|
50
80
|
}
|
|
81
|
+
function appendTail(current, value) {
|
|
82
|
+
if (!value)
|
|
83
|
+
return current;
|
|
84
|
+
const combined = current ? `${current}\n${value}` : value;
|
|
85
|
+
return combined.slice(-diagnosticTailLength);
|
|
86
|
+
}
|
|
87
|
+
function enrichLaunchFailure(error, diagnostics) {
|
|
88
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
89
|
+
const evidence = [
|
|
90
|
+
diagnostics.stderrTail ? `stderr tail: ${boundedDiagnostic(diagnostics.stderrTail)}` : "",
|
|
91
|
+
diagnostics.stdoutTail ? `stdout tail: ${boundedDiagnostic(diagnostics.stdoutTail)}` : "",
|
|
92
|
+
].filter(Boolean);
|
|
93
|
+
return evidence.length > 0
|
|
94
|
+
? new Error(`${message} ${evidence.join("; ")}`, { cause: error })
|
|
95
|
+
: error instanceof Error
|
|
96
|
+
? error
|
|
97
|
+
: new Error(message);
|
|
98
|
+
}
|
|
99
|
+
function boundedDiagnostic(value) {
|
|
100
|
+
return value.replaceAll(/[\r\n\t]+/gu, " ").trim().slice(-1_000);
|
|
101
|
+
}
|
|
@@ -52,7 +52,7 @@ export class AttemptCoordinatedProviderProcessRunner {
|
|
|
52
52
|
const dispatchAttemptId = requiredEnvironment(request, "FORGE_DISPATCH_ATTEMPT_ID");
|
|
53
53
|
const fencingToken = requiredEnvironment(request, "FORGE_FENCING_TOKEN");
|
|
54
54
|
const supervisor = this.registry.require(dispatchAttemptId, fencingToken);
|
|
55
|
-
const child = await supervisor.
|
|
55
|
+
const child = await supervisor.startChildReady("provider", request, handlers, signal, { maxStartAttempts: request.stdin === undefined ? 1 : 3, retryDelayMs: 100 });
|
|
56
56
|
return await child.result;
|
|
57
57
|
}
|
|
58
58
|
}
|
|
@@ -43,6 +43,10 @@ export interface AttemptProcessSupervisorOptions {
|
|
|
43
43
|
readonly metadataReadyTimeoutMs?: number;
|
|
44
44
|
readonly signal?: AbortSignal;
|
|
45
45
|
}
|
|
46
|
+
export interface AttemptProcessChildReadyOptions {
|
|
47
|
+
readonly maxStartAttempts?: number;
|
|
48
|
+
readonly retryDelayMs?: number;
|
|
49
|
+
}
|
|
46
50
|
/**
|
|
47
51
|
* Owns the independently supervised process trees that belong to one host
|
|
48
52
|
* dispatch attempt. Each named child gets a distinct Windows Job Object or
|
|
@@ -63,6 +67,14 @@ export declare class AttemptProcessSupervisor {
|
|
|
63
67
|
private stopOperation;
|
|
64
68
|
constructor(identity: AttemptProcessSupervisorIdentity, runner: ProviderProcessRunner, options?: AttemptProcessSupervisorOptions);
|
|
65
69
|
startChild(name: string, request: ProviderProcessRunRequest, handlers?: ProviderProcessHandlers, signal?: AbortSignal): Promise<AttemptProcessChildHandle>;
|
|
70
|
+
/**
|
|
71
|
+
* Starts a child and proves that the OS process tree is supervised before
|
|
72
|
+
* returning it to the caller. A launch that settles before producing its
|
|
73
|
+
* provider-ready supervision record is safe to retry: the raw runner only
|
|
74
|
+
* persists that record after the actual provider child has started, before
|
|
75
|
+
* Forge announces the provider as active.
|
|
76
|
+
*/
|
|
77
|
+
startChildReady(name: string, request: ProviderProcessRunRequest, handlers?: ProviderProcessHandlers, signal?: AbortSignal, options?: AttemptProcessChildReadyOptions): Promise<AttemptProcessChildHandle>;
|
|
66
78
|
stopChild(name: string, generation: number, fencingToken: string, reason?: unknown): Promise<ProviderProcessRunResult>;
|
|
67
79
|
stopAll(fencingToken: string, reason?: unknown): Promise<void>;
|
|
68
80
|
private performStopAll;
|