@bastani/atomic 0.9.57 → 0.9.58-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -0
- package/dist/builtin/intercom/package.json +1 -1
- package/dist/builtin/mcp/package.json +1 -1
- package/dist/builtin/subagents/CHANGELOG.md +12 -0
- package/dist/builtin/subagents/package.json +1 -1
- package/dist/builtin/subagents/src/extension/index.bundle.mjs +39 -5
- package/dist/builtin/web-access/CHANGELOG.md +12 -0
- package/dist/builtin/web-access/index.bundle.mjs +2 -2
- package/dist/builtin/web-access/package.json +1 -1
- package/dist/builtin/workflows/CHANGELOG.md +42 -0
- package/dist/builtin/workflows/package.json +1 -1
- package/dist/builtin/workflows/src/extension/index.bundle.mjs +885 -228
- package/dist/builtin/workflows/src/extension/local-worker-entry.mjs +933 -175
- package/dist/builtin/workflows/src/index.js +630 -162
- package/dist/builtin/workflows/src/local-worker-entry.mjs +933 -175
- package/dist/builtin/workflows/src/shared/authoring-contract-ui.d.ts +1 -1
- package/dist/builtin/workflows/src/shared/workflow-infrastructure-error.d.ts +13 -0
- package/dist/builtin/workflows/src/workers/public-options.d.ts +10 -0
- package/dist/core/agent-session-persistent-custom-messages.d.ts +5 -0
- package/dist/core/agent-session-persistent-custom-messages.d.ts.map +1 -1
- package/dist/core/agent-session-persistent-custom-messages.js +16 -0
- package/dist/core/agent-session-persistent-custom-messages.js.map +1 -1
- package/dist/core/agent-session-tasks.d.ts.map +1 -1
- package/dist/core/agent-session-tasks.js +20 -0
- package/dist/core/agent-session-tasks.js.map +1 -1
- package/dist/core/settings-defaults.d.ts +4 -0
- package/dist/core/settings-defaults.d.ts.map +1 -1
- package/dist/core/settings-defaults.js +4 -0
- package/dist/core/settings-defaults.js.map +1 -1
- package/dist/core/settings-manager-resource-accessors.d.ts +2 -0
- package/dist/core/settings-manager-resource-accessors.d.ts.map +1 -1
- package/dist/core/settings-manager-resource-accessors.js +6 -0
- package/dist/core/settings-manager-resource-accessors.js.map +1 -1
- package/dist/core/settings-schema.d.ts +9 -0
- package/dist/core/settings-schema.d.ts.map +1 -1
- package/dist/core/settings-schema.js +11 -0
- package/dist/core/settings-schema.js.map +1 -1
- package/dist/core/settings-types.d.ts +1 -1
- package/dist/core/settings-types.d.ts.map +1 -1
- package/dist/core/settings-types.js.map +1 -1
- package/dist/core/tasks/agent-adapter.d.ts +10 -1
- package/dist/core/tasks/agent-adapter.d.ts.map +1 -1
- package/dist/core/tasks/agent-adapter.js +20 -0
- package/dist/core/tasks/agent-adapter.js.map +1 -1
- package/dist/core/tasks/contracts.d.ts +6 -0
- package/dist/core/tasks/contracts.d.ts.map +1 -1
- package/dist/core/tasks/contracts.js.map +1 -1
- package/dist/core/tasks/heartbeat-notice.d.ts +64 -0
- package/dist/core/tasks/heartbeat-notice.d.ts.map +1 -0
- package/dist/core/tasks/heartbeat-notice.js +98 -0
- package/dist/core/tasks/heartbeat-notice.js.map +1 -0
- package/dist/core/tasks/heartbeat.d.ts +75 -0
- package/dist/core/tasks/heartbeat.d.ts.map +1 -0
- package/dist/core/tasks/heartbeat.js +440 -0
- package/dist/core/tasks/heartbeat.js.map +1 -0
- package/dist/core/tasks/supervisor.d.ts +2 -0
- package/dist/core/tasks/supervisor.d.ts.map +1 -1
- package/dist/core/tasks/supervisor.js +32 -4
- package/dist/core/tasks/supervisor.js.map +1 -1
- package/dist/core/tasks/task-view-source.d.ts +72 -0
- package/dist/core/tasks/task-view-source.d.ts.map +1 -0
- package/dist/core/tasks/task-view-source.js +155 -0
- package/dist/core/tasks/task-view-source.js.map +1 -0
- package/dist/core/tools/bash-pty-native.d.ts +2 -0
- package/dist/core/tools/bash-pty-native.d.ts.map +1 -1
- package/dist/core/tools/bash-pty-native.js +1 -0
- package/dist/core/tools/bash-pty-native.js.map +1 -1
- package/dist/core/tools/bash.d.ts +3 -0
- package/dist/core/tools/bash.d.ts.map +1 -1
- package/dist/core/tools/bash.js +20 -2
- package/dist/core/tools/bash.js.map +1 -1
- package/dist/core/tools/powershell.d.ts +16 -8
- package/dist/core/tools/powershell.d.ts.map +1 -1
- package/dist/index.d.ts +5 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -2
- package/dist/index.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-events.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-events.js +7 -0
- package/dist/modes/interactive/components/chat-session-host-events.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-state.d.ts +3 -0
- package/dist/modes/interactive/components/chat-session-host-state.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-state.js +4 -0
- package/dist/modes/interactive/components/chat-session-host-state.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-types.d.ts +11 -0
- package/dist/modes/interactive/components/chat-session-host-types.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host-types.js.map +1 -1
- package/dist/modes/interactive/components/chat-session-host.d.ts +2 -0
- package/dist/modes/interactive/components/chat-session-host.d.ts.map +1 -1
- package/dist/modes/interactive/components/chat-session-host.js +15 -8
- package/dist/modes/interactive/components/chat-session-host.js.map +1 -1
- package/dist/modes/interactive/components/custom-message.d.ts.map +1 -1
- package/dist/modes/interactive/components/custom-message.js +12 -0
- package/dist/modes/interactive/components/custom-message.js.map +1 -1
- package/dist/modes/interactive/components/task-detail.d.ts.map +1 -1
- package/dist/modes/interactive/components/task-detail.js +5 -0
- package/dist/modes/interactive/components/task-detail.js.map +1 -1
- package/dist/modes/interactive/components/task-heartbeat-message.d.ts +14 -0
- package/dist/modes/interactive/components/task-heartbeat-message.d.ts.map +1 -0
- package/dist/modes/interactive/components/task-heartbeat-message.js +76 -0
- package/dist/modes/interactive/components/task-heartbeat-message.js.map +1 -0
- package/dist/modes/interactive/components/task-inspector.d.ts +6 -3
- package/dist/modes/interactive/components/task-inspector.d.ts.map +1 -1
- package/dist/modes/interactive/components/task-inspector.js +53 -100
- package/dist/modes/interactive/components/task-inspector.js.map +1 -1
- package/dist/modes/interactive/skill-command-autocomplete.d.ts +22 -2
- package/dist/modes/interactive/skill-command-autocomplete.d.ts.map +1 -1
- package/dist/modes/interactive/skill-command-autocomplete.js +28 -9
- package/dist/modes/interactive/skill-command-autocomplete.js.map +1 -1
- package/docs/background-tasks.md +65 -1
- package/docs/settings.md +12 -0
- package/docs/subagents.md +2 -0
- package/docs/tools.md +2 -1
- package/docs/workflows/operations.md +5 -3
- package/npm-shrinkwrap.json +32 -32
- package/package.json +3 -3
- package/schemas/settings.schema.json +24 -0
|
@@ -8817,6 +8817,75 @@ function redactSensitiveText(value2) {
|
|
|
8817
8817
|
return value2.replace(/(sk-[A-Za-z0-9_-]{8})[A-Za-z0-9_-]+/g, redactedSecretReplacement("$1")).replace(/\b(authorization\s*:\s*bearer\s+)[^\s,;]+/gi, "$1[redacted]").replace(/\b(bearer\s+)[A-Za-z0-9._~+/-]{8,}=*/gi, "$1[redacted]").replace(/((?:api[_-]?key|token|credential|secret)\s*[:=]\s*)[^\s,;]+/gi, "$1[redacted]");
|
|
8818
8818
|
}
|
|
8819
8819
|
|
|
8820
|
+
// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
|
|
8821
|
+
function isInfrastructureError(value2) {
|
|
8822
|
+
if (value2 instanceof WorkflowInfrastructureError)
|
|
8823
|
+
return true;
|
|
8824
|
+
if (typeof value2 !== "object" || value2 === null)
|
|
8825
|
+
return false;
|
|
8826
|
+
const candidate = value2;
|
|
8827
|
+
return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
|
|
8828
|
+
}
|
|
8829
|
+
function isRetryableWorkflowError(error) {
|
|
8830
|
+
let current = error;
|
|
8831
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH && current !== undefined && current !== null; depth++) {
|
|
8832
|
+
if (isInfrastructureError(current))
|
|
8833
|
+
return true;
|
|
8834
|
+
current = current instanceof Error ? current.cause : undefined;
|
|
8835
|
+
}
|
|
8836
|
+
return false;
|
|
8837
|
+
}
|
|
8838
|
+
function workflowInfrastructureFault(message) {
|
|
8839
|
+
return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
|
|
8840
|
+
}
|
|
8841
|
+
function asWorkflowInfrastructureError(error) {
|
|
8842
|
+
if (isInfrastructureError(error))
|
|
8843
|
+
return error;
|
|
8844
|
+
if (!(error instanceof Error))
|
|
8845
|
+
return;
|
|
8846
|
+
const code = workflowInfrastructureFault(error.message);
|
|
8847
|
+
return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
|
|
8848
|
+
}
|
|
8849
|
+
function failedOnInfrastructureFault(result) {
|
|
8850
|
+
if (result.status !== "failed")
|
|
8851
|
+
return false;
|
|
8852
|
+
const names = (value2) => typeof value2 === "string" && (WORKFLOW_INFRASTRUCTURE_FAULTS.has(value2) || workflowInfrastructureFault(value2) !== undefined);
|
|
8853
|
+
const failure = result.failure;
|
|
8854
|
+
if (names(failure?.code) || names(result.error))
|
|
8855
|
+
return true;
|
|
8856
|
+
return Array.isArray(result.stages) && result.stages.some((stage) => stage?.status === "failed" && (names(stage.failureCode) || names(stage.error)));
|
|
8857
|
+
}
|
|
8858
|
+
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", WORKFLOW_INFRASTRUCTURE_FAULTS, MAX_CAUSE_DEPTH = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
|
|
8859
|
+
var init_workflow_infrastructure_error = __esm(() => {
|
|
8860
|
+
WORKFLOW_INFRASTRUCTURE_FAULTS = new Set([
|
|
8861
|
+
"worker_lost",
|
|
8862
|
+
"worker_cleanup_unconfirmed",
|
|
8863
|
+
"worker_retry_limit",
|
|
8864
|
+
"worker_startup_failed",
|
|
8865
|
+
"stale_work_lease",
|
|
8866
|
+
"intercom_unavailable",
|
|
8867
|
+
"storage_unavailable"
|
|
8868
|
+
]);
|
|
8869
|
+
FAULT_PATTERNS = [
|
|
8870
|
+
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
8871
|
+
[/Local worker retry limit/, "worker_retry_limit"],
|
|
8872
|
+
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
8873
|
+
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
8874
|
+
[/Stale work lease|Stale resource admission attempt/, "stale_work_lease"],
|
|
8875
|
+
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
8876
|
+
[/Storage unavailable/, "storage_unavailable"]
|
|
8877
|
+
];
|
|
8878
|
+
WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
|
|
8879
|
+
name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
|
|
8880
|
+
retryable = true;
|
|
8881
|
+
code;
|
|
8882
|
+
constructor(code, message, options) {
|
|
8883
|
+
super(message, options);
|
|
8884
|
+
this.code = code;
|
|
8885
|
+
}
|
|
8886
|
+
};
|
|
8887
|
+
});
|
|
8888
|
+
|
|
8820
8889
|
// dist/builtin/workflows/src/shared/workflow-failures-decisions.ts
|
|
8821
8890
|
function authDecision(code) {
|
|
8822
8891
|
if (code === "invalid_api_key") {
|
|
@@ -8905,6 +8974,16 @@ function cancelledDecision() {
|
|
|
8905
8974
|
disposition: "terminal_killed"
|
|
8906
8975
|
};
|
|
8907
8976
|
}
|
|
8977
|
+
function infrastructureFaultDecision(code) {
|
|
8978
|
+
return {
|
|
8979
|
+
kind: "unknown",
|
|
8980
|
+
code,
|
|
8981
|
+
retryable: true,
|
|
8982
|
+
resumable: true,
|
|
8983
|
+
recoverability: "recoverable",
|
|
8984
|
+
disposition: "terminal_failed"
|
|
8985
|
+
};
|
|
8986
|
+
}
|
|
8908
8987
|
function unknownDecision() {
|
|
8909
8988
|
return {
|
|
8910
8989
|
kind: "unknown",
|
|
@@ -8932,6 +9011,8 @@ function strongDecisionFromNormalizedCode(normalized, retryAfterMs) {
|
|
|
8932
9011
|
disposition: "active_blocked"
|
|
8933
9012
|
};
|
|
8934
9013
|
}
|
|
9014
|
+
if (WORKFLOW_INFRASTRUCTURE_FAULTS.has(normalized))
|
|
9015
|
+
return infrastructureFaultDecision(normalized);
|
|
8935
9016
|
if (CANCELLED_CODES.has(normalized))
|
|
8936
9017
|
return cancelledDecision();
|
|
8937
9018
|
if (INVALID_API_KEY_CODES.has(normalized))
|
|
@@ -9044,6 +9125,9 @@ function classificationFromNormalizedCode(normalized, retryAfterMs, source, mess
|
|
|
9044
9125
|
return weak !== undefined ? { weak: classificationForDecision(weak, source, message, "weak_signal") } : {};
|
|
9045
9126
|
}
|
|
9046
9127
|
function fallbackDecisionFromMessage(message, name) {
|
|
9128
|
+
const fault = workflowInfrastructureFault(message);
|
|
9129
|
+
if (fault !== undefined)
|
|
9130
|
+
return infrastructureFaultDecision(fault);
|
|
9047
9131
|
const tokens = tokenize(message);
|
|
9048
9132
|
const decision = decisionFromMessageTokens(tokens, name);
|
|
9049
9133
|
const providerAuthDecision = classifyFallbackProviderAuthMessage(message, tokens);
|
|
@@ -9058,6 +9142,7 @@ function fallbackDecisionFromMessage(message, name) {
|
|
|
9058
9142
|
var INVALID_API_KEY_CODES, LOGIN_REQUIRED_CODES, MISSING_API_KEY_CODES, RATE_LIMIT_CODES, QUOTA_LIMIT_CODES, CANCELLED_CODES, PROVIDER_UNAVAILABLE_CODES, UNKNOWN_MODEL_CODES, FORBIDDEN_CONFIG_CODES, REQUEST_AUTH_TIMEOUT_PHRASES, LOGIN_REQUIRED_PHRASES, LOCAL_LOGIN_REQUIRED_PHRASES, PROVIDER_AUTH_FALLBACK_PHRASES, MISSING_API_KEY_PHRASES, INVALID_API_KEY_PHRASES, INVALID_API_KEY_CONTEXT, HTTP_RATE_LIMIT_PHRASES, RATE_LIMIT_PHRASES, QUOTA_LIMIT_PHRASES, CANCELLED_PHRASES, UNKNOWN_MODEL_PHRASES, FORBIDDEN_CONFIG_PHRASES, PROVIDER_UNAVAILABLE_PHRASES, AUTH_CONTEXT, MODEL_PROVIDER_CONTEXT, PROVIDER_CONTEXT, STATUS_MESSAGE_REFINEMENT_CODES, BROAD_AUTH_MESSAGE_REFINEMENT_CODES, STATUS_RELATED_MESSAGE_REFINEMENT_CODES;
|
|
9059
9143
|
var init_workflow_failures_decisions = __esm(() => {
|
|
9060
9144
|
init_workflow_failures_contract();
|
|
9145
|
+
init_workflow_infrastructure_error();
|
|
9061
9146
|
INVALID_API_KEY_CODES = new Set([
|
|
9062
9147
|
"401",
|
|
9063
9148
|
"invalid_api_key",
|
|
@@ -9245,6 +9330,13 @@ function decisionOfRecordedCode(code, retryAfterMs) {
|
|
|
9245
9330
|
case "cancelled":
|
|
9246
9331
|
return cancelledDecision();
|
|
9247
9332
|
case "pending_stage_delivery_failed":
|
|
9333
|
+
case "worker_lost":
|
|
9334
|
+
case "worker_cleanup_unconfirmed":
|
|
9335
|
+
case "worker_retry_limit":
|
|
9336
|
+
case "worker_startup_failed":
|
|
9337
|
+
case "stale_work_lease":
|
|
9338
|
+
case "intercom_unavailable":
|
|
9339
|
+
case "storage_unavailable":
|
|
9248
9340
|
return strongDecisionFromNormalizedCode(code, retryAfterMs);
|
|
9249
9341
|
case "unknown":
|
|
9250
9342
|
return;
|
|
@@ -13865,7 +13957,7 @@ class JournalDurableBackend {
|
|
|
13865
13957
|
timer;
|
|
13866
13958
|
state = "open";
|
|
13867
13959
|
closing;
|
|
13868
|
-
renewing;
|
|
13960
|
+
renewing = new Map;
|
|
13869
13961
|
mutationContext = new AsyncLocalStorage3;
|
|
13870
13962
|
processExitCleanup;
|
|
13871
13963
|
unregisterExecutionStorages = registerExecutionStorages(this.executionContext, this.mutationContext);
|
|
@@ -14586,27 +14678,21 @@ class JournalDurableBackend {
|
|
|
14586
14678
|
yield this.enqueue(runId, () => this.deleteOwned(runId));
|
|
14587
14679
|
}
|
|
14588
14680
|
renewLeases() {
|
|
14589
|
-
if (this.renewing !== undefined)
|
|
14590
|
-
return this.renewing;
|
|
14591
14681
|
if (this.state !== "open")
|
|
14592
14682
|
return Promise.resolve();
|
|
14593
|
-
this.renewing = this.renewOwnedLeases().finally(() => {
|
|
14594
|
-
this.renewing = undefined;
|
|
14595
|
-
});
|
|
14596
|
-
return this.renewing;
|
|
14597
|
-
}
|
|
14598
|
-
async renewOwnedLeases() {
|
|
14599
14683
|
for (const [runId, lease] of this.leases) {
|
|
14600
|
-
if (this.
|
|
14601
|
-
|
|
14602
|
-
|
|
14603
|
-
|
|
14604
|
-
} catch (error) {
|
|
14605
|
-
if (error instanceof Error)
|
|
14606
|
-
this.lost.set(runId, error);
|
|
14684
|
+
if (this.renewing.has(runId))
|
|
14685
|
+
continue;
|
|
14686
|
+
const renewal = this.serialize(runId, () => this.renewLease(runId, lease), false).catch((error) => {
|
|
14687
|
+
this.lost.set(runId, error);
|
|
14607
14688
|
this.dropLease(runId);
|
|
14608
|
-
}
|
|
14689
|
+
}).finally(() => {
|
|
14690
|
+
if (this.renewing.get(runId) === renewal)
|
|
14691
|
+
this.renewing.delete(runId);
|
|
14692
|
+
});
|
|
14693
|
+
this.renewing.set(runId, renewal);
|
|
14609
14694
|
}
|
|
14695
|
+
return Promise.all(this.renewing.values()).then(() => {});
|
|
14610
14696
|
}
|
|
14611
14697
|
*renewLease(runId, lease) {
|
|
14612
14698
|
if (this.leases.get(runId) !== lease || this.lost.has(runId))
|
|
@@ -14614,9 +14700,10 @@ class JournalDurableBackend {
|
|
|
14614
14700
|
try {
|
|
14615
14701
|
yield* invoke(() => this.journal.renew(lease));
|
|
14616
14702
|
} catch (error) {
|
|
14703
|
+
if (!(error instanceof StaleLeaseError))
|
|
14704
|
+
return;
|
|
14617
14705
|
if (this.leases.get(runId) === lease) {
|
|
14618
|
-
|
|
14619
|
-
this.lost.set(runId, yield* this.mapOwnership(error, runId));
|
|
14706
|
+
this.lost.set(runId, yield* this.mapOwnership(error, runId));
|
|
14620
14707
|
this.dropLease(runId);
|
|
14621
14708
|
}
|
|
14622
14709
|
}
|
|
@@ -14710,7 +14797,7 @@ class JournalDurableBackend {
|
|
|
14710
14797
|
}
|
|
14711
14798
|
async drainAndClose() {
|
|
14712
14799
|
try {
|
|
14713
|
-
await this.renewing;
|
|
14800
|
+
await Promise.all(this.renewing.values());
|
|
14714
14801
|
while (this.pending.size > 0)
|
|
14715
14802
|
await Promise.all(this.pending.values());
|
|
14716
14803
|
for (const runId of this.leases.keys())
|
|
@@ -20179,7 +20266,7 @@ class SqlResourceAdmission {
|
|
|
20179
20266
|
for (const value2 of Object.values(limits ?? {}))
|
|
20180
20267
|
if (!Number.isSafeInteger(value2) || value2 < 1)
|
|
20181
20268
|
throw new Error("Invalid resource limit");
|
|
20182
|
-
for (const value2 of Object.values(options.host ?? {}))
|
|
20269
|
+
for (const value2 of [...Object.values(options.host ?? {}), options.stageStartMaxLoadPerCpu ?? 0])
|
|
20183
20270
|
if (!Number.isFinite(value2) || value2 < 0)
|
|
20184
20271
|
throw new Error("Invalid host admission threshold");
|
|
20185
20272
|
const admission = new SqlResourceAdmission(storage, {
|
|
@@ -20197,6 +20284,8 @@ class SqlResourceAdmission {
|
|
|
20197
20284
|
for (const column of ["owner", "attempt", "process_identity"])
|
|
20198
20285
|
if (!columns.some((entry) => entry.name === column))
|
|
20199
20286
|
await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN ${column} TEXT`);
|
|
20287
|
+
if (!columns.some((entry) => entry.name === "granted_at"))
|
|
20288
|
+
await tx.exec(`ALTER TABLE ${admission.permits} ADD COLUMN granted_at INTEGER`);
|
|
20200
20289
|
await tx.exec(`CREATE INDEX IF NOT EXISTS ${admission.permits}_owner ON ${admission.permits}(owner, state);
|
|
20201
20290
|
CREATE INDEX IF NOT EXISTS ${admission.permits}_state ON ${admission.permits}(state)`);
|
|
20202
20291
|
if (!(await tx.all(`PRAGMA table_info(${admission.owners})`)).some((entry) => entry.name === "boot"))
|
|
@@ -20226,14 +20315,24 @@ class SqlResourceAdmission {
|
|
|
20226
20315
|
await admission.reconcile();
|
|
20227
20316
|
return admission;
|
|
20228
20317
|
}
|
|
20318
|
+
sampleHost() {
|
|
20319
|
+
return this.options.sampleHost?.() ?? {
|
|
20320
|
+
freeMemoryBytes: freemem(),
|
|
20321
|
+
load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
|
|
20322
|
+
};
|
|
20323
|
+
}
|
|
20324
|
+
stageStartPressure(kind, delegated) {
|
|
20325
|
+
const threshold = this.options.stageStartMaxLoadPerCpu;
|
|
20326
|
+
if (kind !== "session" || delegated === true || threshold === undefined || threshold === 0)
|
|
20327
|
+
return false;
|
|
20328
|
+
const load = this.sampleHost().load;
|
|
20329
|
+
return load !== undefined && load > threshold;
|
|
20330
|
+
}
|
|
20229
20331
|
hostReason() {
|
|
20230
20332
|
const host = this.options.host;
|
|
20231
20333
|
if (!host)
|
|
20232
20334
|
return;
|
|
20233
|
-
const sample = this.
|
|
20234
|
-
freeMemoryBytes: freemem(),
|
|
20235
|
-
load: process.platform === "win32" ? undefined : loadavg()[0] / cpus().length
|
|
20236
|
-
};
|
|
20335
|
+
const sample = this.sampleHost();
|
|
20237
20336
|
if (host.minFreeMemoryBytes !== undefined) {
|
|
20238
20337
|
if (sample.freeMemoryBytes === undefined)
|
|
20239
20338
|
return "host memory unsupported";
|
|
@@ -20247,21 +20346,32 @@ class SqlResourceAdmission {
|
|
|
20247
20346
|
return "host load";
|
|
20248
20347
|
}
|
|
20249
20348
|
}
|
|
20250
|
-
async
|
|
20349
|
+
async attemptStanding(tx) {
|
|
20251
20350
|
const { root, attempt } = this.options;
|
|
20252
20351
|
if (!attempt)
|
|
20253
|
-
return;
|
|
20254
|
-
const work = await tx.get(`SELECT
|
|
20352
|
+
return "current";
|
|
20353
|
+
const work = await tx.get(`SELECT (w.expires_at < ${NOW_MS} OR l.renewed_at + p.timeout < ${NOW_MS}) AS late
|
|
20354
|
+
FROM ${this.storage.tables.runs}_work w JOIN ${this.storage.tables.leases} l ON l.run_id = w.root
|
|
20255
20355
|
JOIN ${this.storage.tables.runs}_pool p ON p.root = w.root JOIN ${this.storage.tables.runs} r ON r.run_id = w.root
|
|
20256
20356
|
WHERE w.id = ? AND w.root = ? AND w.generation = ? AND w.epoch = ? AND w.state = 'leased' AND r.status IN ('running', 'paused')
|
|
20257
|
-
AND
|
|
20258
|
-
AND l.epoch = w.epoch AND l.renewed_at + p.timeout >= CAST(unixepoch('subsec') * 1000 AS INTEGER)`, attempt.id, root, attempt.generation, attempt.epoch);
|
|
20357
|
+
AND l.epoch = w.epoch`, attempt.id, root, attempt.generation, attempt.epoch);
|
|
20259
20358
|
if (!work)
|
|
20359
|
+
return "stale";
|
|
20360
|
+
return Number(work.late) === 1 ? "late" : "current";
|
|
20361
|
+
}
|
|
20362
|
+
async assertCurrentAttempt(tx) {
|
|
20363
|
+
const standing = await this.attemptStanding(tx);
|
|
20364
|
+
if (standing === "stale")
|
|
20260
20365
|
throw new Error("Stale resource admission attempt");
|
|
20366
|
+
return standing === "current";
|
|
20261
20367
|
}
|
|
20262
20368
|
async validate(signal) {
|
|
20263
|
-
|
|
20264
|
-
|
|
20369
|
+
for (;; ) {
|
|
20370
|
+
signal?.throwIfAborted();
|
|
20371
|
+
if (await this.storage.transaction((tx) => this.assertCurrentAttempt(tx)))
|
|
20372
|
+
return;
|
|
20373
|
+
await delay(LATE_ATTEMPT_POLL_MS, undefined, { signal });
|
|
20374
|
+
}
|
|
20265
20375
|
}
|
|
20266
20376
|
async acquire(kind, signal, request) {
|
|
20267
20377
|
signal?.throwIfAborted();
|
|
@@ -20295,11 +20405,19 @@ class SqlResourceAdmission {
|
|
|
20295
20405
|
this.nextReconciliation = performance.now() + 1000;
|
|
20296
20406
|
}
|
|
20297
20407
|
const hostReason = this.hostReason();
|
|
20408
|
+
const pressure = this.stageStartPressure(kind, request?.delegated);
|
|
20298
20409
|
const granted = await this.storage.transaction(async (tx) => {
|
|
20299
20410
|
await tx.run(`UPDATE ${this.owners} SET expires_at = CAST(unixepoch('subsec') * 1000 AS INTEGER) + ? WHERE id = ?`, OWNER_LEASE_MS, this.owner);
|
|
20300
20411
|
const { root } = this.options;
|
|
20301
|
-
await this.assertCurrentAttempt(tx)
|
|
20412
|
+
if (!await this.assertCurrentAttempt(tx)) {
|
|
20413
|
+
const waiting = await tx.run(`UPDATE ${this.permits} SET reason = ? WHERE operation = ? AND state = 'waiting'`, LATE_ATTEMPT_REASON, operation);
|
|
20414
|
+
if (waiting.changes !== 1)
|
|
20415
|
+
throw new Error("Resource admission holder was stopped");
|
|
20416
|
+
return false;
|
|
20417
|
+
}
|
|
20302
20418
|
let reason = hostReason;
|
|
20419
|
+
if (pressure && (await tx.get(`SELECT 1 AS recent FROM ${this.permits} WHERE root = ? AND kind = 'session' AND granted_at >= ${NOW_MS} - ? LIMIT 1`, root, STAGE_START_SPACING_MS) || await tx.get(`SELECT 1 AS older FROM ${this.permits} WHERE root = ? AND kind = 'session' AND state = 'waiting' AND reason = ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, root, STAGE_START_PRESSURE_REASON, operation)))
|
|
20420
|
+
reason ??= STAGE_START_PRESSURE_REASON;
|
|
20303
20421
|
const controls = `${this.storage.tables.runs}_pool_control`;
|
|
20304
20422
|
if (await tx.get("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", controls)) {
|
|
20305
20423
|
const control = await tx.get(`SELECT state FROM ${controls} WHERE root = ?`, root);
|
|
@@ -20314,13 +20432,14 @@ class SqlResourceAdmission {
|
|
|
20314
20432
|
const rootClause = scope === "user" ? "" : "AND root = ?";
|
|
20315
20433
|
const args = scope === "user" ? [kind] : [kind, root];
|
|
20316
20434
|
const count = await tx.get(`SELECT COUNT(*) AS count FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'granted'`, ...args);
|
|
20317
|
-
const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, operation);
|
|
20435
|
+
const older = await tx.get(`SELECT 1 AS waiting FROM ${this.permits} WHERE kind = ? ${rootClause} AND state = 'waiting' AND reason IS NOT ? AND ticket < (SELECT ticket FROM ${this.permits} WHERE operation = ?) LIMIT 1`, ...args, STAGE_START_PRESSURE_REASON, operation);
|
|
20318
20436
|
if (Number(count?.count) >= limit || older)
|
|
20319
20437
|
reason ??= `${scope} ${kind}`;
|
|
20320
20438
|
}
|
|
20321
20439
|
if (reason && request?.delegated)
|
|
20322
20440
|
throw new Error(`Delegated session cannot be admitted without capacity headroom: ${reason}`);
|
|
20323
|
-
const
|
|
20441
|
+
const stageStart = kind === "session" && request?.delegated !== true;
|
|
20442
|
+
const updated = await tx.run(`UPDATE ${this.permits} SET state = ?, reason = ?, granted_at = CASE WHEN ? IS NULL AND ? THEN ${NOW_MS} ELSE NULL END WHERE operation = ? AND state = 'waiting'`, reason ? "waiting" : "granted", reason ?? null, reason ?? null, stageStart ? 1 : 0, operation);
|
|
20324
20443
|
if (updated.changes !== 1)
|
|
20325
20444
|
throw new Error("Resource admission holder was stopped");
|
|
20326
20445
|
return !reason;
|
|
@@ -20480,7 +20599,7 @@ async function reapProcessGroups(journal, root, scope) {
|
|
|
20480
20599
|
return { survivors: [], unprobeable: [] };
|
|
20481
20600
|
return settleProcessGroups(storage, root, scope, true);
|
|
20482
20601
|
}
|
|
20483
|
-
var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)";
|
|
20602
|
+
var OWNER_LEASE_MS = 30000, REAP_TERM_GRACE_MS = 5000, REAP_KILL_GRACE_MS = 1e4, REAP_POLL_MS = 50, RELEASED_PERMIT_RETENTION = 256, RELEASED_PERMIT_PRUNE_BATCH = 1000, EXPIRED_REASON = "owner expired; cleanup unconfirmed", UNCONFIRMED_REASON = "cleanup unconfirmed", ANONYMOUS_REASON = "owner identity unavailable; cleanup unconfirmed", NOW_MS = "CAST(unixepoch('subsec') * 1000 AS INTEGER)", LATE_ATTEMPT_POLL_MS = 100, LATE_ATTEMPT_REASON = "waiting for its attempt lease", STAGE_START_SPACING_MS = 5000, STAGE_START_PRESSURE_REASON = "paced for host load";
|
|
20484
20603
|
var init_resource_admission = __esm(() => {
|
|
20485
20604
|
init_internal();
|
|
20486
20605
|
init_sql_run_journal();
|
|
@@ -63094,14 +63213,23 @@ class SqlPoolQueue {
|
|
|
63094
63213
|
}
|
|
63095
63214
|
await tx.run(`UPDATE ${this.meta} SET version = ?`, POOL_SCHEMA_VERSION);
|
|
63096
63215
|
}
|
|
63216
|
+
leaseRow(tx, lease) {
|
|
63217
|
+
return tx.get(`SELECT l.*, r.status, p.timeout, ${STORE_NOW2} AS now FROM ${this.journal.leases} l JOIN ${this.journal.runs} r ON r.run_id = l.run_id LEFT JOIN ${this.roots} p ON p.root = l.run_id WHERE l.run_id = ?`, lease.runId);
|
|
63218
|
+
}
|
|
63097
63219
|
async authority(tx, lease, timeout) {
|
|
63098
|
-
const row = await
|
|
63220
|
+
const row = await this.leaseRow(tx, lease);
|
|
63099
63221
|
if (row?.timeout === null && timeout === undefined)
|
|
63100
63222
|
throw new Error("Pool not initialized");
|
|
63101
63223
|
if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId || isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout ?? 0, row.now))
|
|
63102
63224
|
throw new StaleLeaseError(lease.runId, lease.epoch, Number(row?.epoch ?? 0));
|
|
63103
63225
|
return row;
|
|
63104
63226
|
}
|
|
63227
|
+
async rootLease(lease, timeout) {
|
|
63228
|
+
const row = await this.storage.read((executor) => this.leaseRow(executor, lease));
|
|
63229
|
+
if (!row || row.epoch !== lease.epoch || row.holder_id !== lease.holderId)
|
|
63230
|
+
return "lost";
|
|
63231
|
+
return isWorkflowLeaseExpired(row.renewed_at, row.timeout ?? timeout, row.now) ? "late" : "current";
|
|
63232
|
+
}
|
|
63105
63233
|
async initialize(lease, coordinatorTimeoutMs) {
|
|
63106
63234
|
positive(coordinatorTimeoutMs);
|
|
63107
63235
|
lease = snapshotLease(lease);
|
|
@@ -63113,6 +63241,9 @@ class SqlPoolQueue {
|
|
|
63113
63241
|
throw new Error("Pool configuration is immutable");
|
|
63114
63242
|
});
|
|
63115
63243
|
}
|
|
63244
|
+
async initialized(root) {
|
|
63245
|
+
return await this.storage.read((executor) => executor.get(`SELECT 1 AS held FROM ${this.roots} WHERE root = ?`, root)) !== undefined;
|
|
63246
|
+
}
|
|
63116
63247
|
async control(lease) {
|
|
63117
63248
|
return this.transaction(async (tx) => {
|
|
63118
63249
|
await this.authority(tx, lease);
|
|
@@ -63507,6 +63638,21 @@ class SqlPoolQueue {
|
|
|
63507
63638
|
return row ? JSON.parse(String(row.result)) : undefined;
|
|
63508
63639
|
});
|
|
63509
63640
|
}
|
|
63641
|
+
async reopenCompleted(lease, id, rerun) {
|
|
63642
|
+
lease = snapshotLease(lease);
|
|
63643
|
+
return this.transaction(async (tx) => {
|
|
63644
|
+
await this.authority(tx, lease);
|
|
63645
|
+
const row = await tx.get(`SELECT * FROM ${this.work} WHERE id = ? AND root = ?`, id, lease.runId);
|
|
63646
|
+
if (row?.state !== "completed" || row.epoch >= lease.epoch)
|
|
63647
|
+
return;
|
|
63648
|
+
const stored = await tx.get(`SELECT result FROM ${this.work}_results WHERE work = ?`, id);
|
|
63649
|
+
if (!stored || !rerun(JSON.parse(String(stored.result))))
|
|
63650
|
+
return;
|
|
63651
|
+
await tx.run(`DELETE FROM ${this.work}_results WHERE work = ?`, id);
|
|
63652
|
+
await tx.run(`UPDATE ${this.work} SET state = 'ready', worker = NULL, expires_at = 0, epoch = ? WHERE id = ?`, lease.epoch, id);
|
|
63653
|
+
return fromRow({ ...row, state: "ready", worker: null, expires_at: 0, epoch: lease.epoch });
|
|
63654
|
+
});
|
|
63655
|
+
}
|
|
63510
63656
|
async expire(lease, token) {
|
|
63511
63657
|
lease = snapshotLease(lease);
|
|
63512
63658
|
token = { ...token };
|
|
@@ -63628,7 +63774,7 @@ function reportsClosedSession(error) {
|
|
|
63628
63774
|
}
|
|
63629
63775
|
function isSessionClosedError(error) {
|
|
63630
63776
|
let current = error;
|
|
63631
|
-
for (let depth = 0;depth <
|
|
63777
|
+
for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined; depth++) {
|
|
63632
63778
|
try {
|
|
63633
63779
|
if (reportsClosedSession(current))
|
|
63634
63780
|
return true;
|
|
@@ -63639,7 +63785,7 @@ function isSessionClosedError(error) {
|
|
|
63639
63785
|
}
|
|
63640
63786
|
return false;
|
|
63641
63787
|
}
|
|
63642
|
-
var
|
|
63788
|
+
var MAX_CAUSE_DEPTH2 = 8;
|
|
63643
63789
|
var init_session_closed_error = () => {};
|
|
63644
63790
|
|
|
63645
63791
|
// dist/builtin/workflows/src/shared/external-run-routes.ts
|
|
@@ -67741,57 +67887,6 @@ var init_sdk_intercom = __esm(() => {
|
|
|
67741
67887
|
owners2 = new WeakMap;
|
|
67742
67888
|
});
|
|
67743
67889
|
|
|
67744
|
-
// dist/builtin/workflows/src/shared/workflow-infrastructure-error.ts
|
|
67745
|
-
function isInfrastructureError(value2) {
|
|
67746
|
-
if (value2 instanceof WorkflowInfrastructureError)
|
|
67747
|
-
return true;
|
|
67748
|
-
if (typeof value2 !== "object" || value2 === null)
|
|
67749
|
-
return false;
|
|
67750
|
-
const candidate = value2;
|
|
67751
|
-
return candidate.name === WORKFLOW_INFRASTRUCTURE_ERROR_NAME && candidate.retryable === true && typeof candidate.code === "string";
|
|
67752
|
-
}
|
|
67753
|
-
function isRetryableWorkflowError(error) {
|
|
67754
|
-
let current = error;
|
|
67755
|
-
for (let depth = 0;depth < MAX_CAUSE_DEPTH2 && current !== undefined && current !== null; depth++) {
|
|
67756
|
-
if (isInfrastructureError(current))
|
|
67757
|
-
return true;
|
|
67758
|
-
current = current instanceof Error ? current.cause : undefined;
|
|
67759
|
-
}
|
|
67760
|
-
return false;
|
|
67761
|
-
}
|
|
67762
|
-
function workflowInfrastructureFault(message) {
|
|
67763
|
-
return FAULT_PATTERNS.find(([pattern]) => pattern.test(message))?.[1];
|
|
67764
|
-
}
|
|
67765
|
-
function asWorkflowInfrastructureError(error) {
|
|
67766
|
-
if (isInfrastructureError(error))
|
|
67767
|
-
return error;
|
|
67768
|
-
if (!(error instanceof Error))
|
|
67769
|
-
return;
|
|
67770
|
-
const code = workflowInfrastructureFault(error.message);
|
|
67771
|
-
return code === undefined ? undefined : new WorkflowInfrastructureError(code, error.message, { cause: error });
|
|
67772
|
-
}
|
|
67773
|
-
var WORKFLOW_INFRASTRUCTURE_ERROR_NAME = "WorkflowInfrastructureError", MAX_CAUSE_DEPTH2 = 8, FAULT_PATTERNS, WorkflowInfrastructureError;
|
|
67774
|
-
var init_workflow_infrastructure_error = __esm(() => {
|
|
67775
|
-
FAULT_PATTERNS = [
|
|
67776
|
-
[/Local worker (?:process )?cleanup (?:was not confirmed|unconfirmed)/, "worker_cleanup_unconfirmed"],
|
|
67777
|
-
[/Local worker retry limit/, "worker_retry_limit"],
|
|
67778
|
-
[/Local worker (?:startup timed out|exited before readiness)/, "worker_startup_failed"],
|
|
67779
|
-
[/Worker exited before delivery acknowledgement|Local worker pool coordinator exited/, "worker_lost"],
|
|
67780
|
-
[/Stale work lease/, "stale_work_lease"],
|
|
67781
|
-
[/Intercom runtime no longer active/, "intercom_unavailable"],
|
|
67782
|
-
[/Storage unavailable/, "storage_unavailable"]
|
|
67783
|
-
];
|
|
67784
|
-
WorkflowInfrastructureError = class WorkflowInfrastructureError extends Error {
|
|
67785
|
-
name = WORKFLOW_INFRASTRUCTURE_ERROR_NAME;
|
|
67786
|
-
retryable = true;
|
|
67787
|
-
code;
|
|
67788
|
-
constructor(code, message, options) {
|
|
67789
|
-
super(message, options);
|
|
67790
|
-
this.code = code;
|
|
67791
|
-
}
|
|
67792
|
-
};
|
|
67793
|
-
});
|
|
67794
|
-
|
|
67795
67890
|
// dist/builtin/workflows/src/workers/coordinator-protocol.ts
|
|
67796
67891
|
var POOL_COORDINATOR_ROLE = "pool-coordinator", POOL_SUMMARY_INTERVAL_MS = 250;
|
|
67797
67892
|
|
|
@@ -68075,6 +68170,16 @@ class RateLimitObservationOutbox {
|
|
|
68075
68170
|
}
|
|
68076
68171
|
var RATE_LIMIT_SHARE_INTERVAL_MS = 250;
|
|
68077
68172
|
|
|
68173
|
+
// dist/builtin/workflows/src/workers/stage-session-commands.ts
|
|
68174
|
+
function runsWhilePaused(command) {
|
|
68175
|
+
if (command.kind === "abort-bash" || command.kind === "abort-compaction")
|
|
68176
|
+
return true;
|
|
68177
|
+
if (command.kind !== "task")
|
|
68178
|
+
return false;
|
|
68179
|
+
const action = command.action.kind;
|
|
68180
|
+
return action === "cancel" || action === "output" || action === "detail" || action === "transcript";
|
|
68181
|
+
}
|
|
68182
|
+
|
|
68078
68183
|
// dist/builtin/workflows/src/workers/stage-watch-coalesce.ts
|
|
68079
68184
|
function streamedDelta(event) {
|
|
68080
68185
|
if (event.type !== "message_update")
|
|
@@ -68401,6 +68506,7 @@ class PoolCoordinator {
|
|
|
68401
68506
|
closing;
|
|
68402
68507
|
replacements = 0;
|
|
68403
68508
|
rootLeaseLost = false;
|
|
68509
|
+
lateRoot;
|
|
68404
68510
|
attemptBase = new Map;
|
|
68405
68511
|
constructor(port) {
|
|
68406
68512
|
this.port = port;
|
|
@@ -68414,6 +68520,47 @@ class PoolCoordinator {
|
|
|
68414
68520
|
await this.journal?.close().catch(() => {});
|
|
68415
68521
|
await this.durable?.close().catch(() => {});
|
|
68416
68522
|
}
|
|
68523
|
+
async whileRootLate(queue, action, resume) {
|
|
68524
|
+
for (let waited = false;; ) {
|
|
68525
|
+
try {
|
|
68526
|
+
if (waited)
|
|
68527
|
+
await resume?.();
|
|
68528
|
+
return await action();
|
|
68529
|
+
} catch (error) {
|
|
68530
|
+
if (!(error instanceof Error) || !isRootLeaseLost(error) || !await this.rootRenewal(queue))
|
|
68531
|
+
throw error;
|
|
68532
|
+
waited = true;
|
|
68533
|
+
}
|
|
68534
|
+
}
|
|
68535
|
+
}
|
|
68536
|
+
rootRenewal(queue) {
|
|
68537
|
+
this.lateRoot ??= this.awaitRootRenewal(queue).finally(() => {
|
|
68538
|
+
this.lateRoot = undefined;
|
|
68539
|
+
});
|
|
68540
|
+
return this.lateRoot;
|
|
68541
|
+
}
|
|
68542
|
+
async awaitRootRenewal(queue) {
|
|
68543
|
+
const lease = this.lease;
|
|
68544
|
+
const since = performance.now();
|
|
68545
|
+
let late = false;
|
|
68546
|
+
try {
|
|
68547
|
+
for (;; ) {
|
|
68548
|
+
if (this.closed)
|
|
68549
|
+
return false;
|
|
68550
|
+
const lost = await this.remote?.fresh(lease).then(() => false, () => true);
|
|
68551
|
+
if (lost)
|
|
68552
|
+
return false;
|
|
68553
|
+
const standing = await queue.rootLease(lease, COORDINATOR_LEASE_MS);
|
|
68554
|
+
if (standing !== "late")
|
|
68555
|
+
return standing === "current";
|
|
68556
|
+
late = true;
|
|
68557
|
+
await delay2(LATE_ROOT_POLL_MS);
|
|
68558
|
+
}
|
|
68559
|
+
} finally {
|
|
68560
|
+
if (late)
|
|
68561
|
+
this.stalledMs += performance.now() - since;
|
|
68562
|
+
}
|
|
68563
|
+
}
|
|
68417
68564
|
receive(message) {
|
|
68418
68565
|
if (message.kind === "request-reply") {
|
|
68419
68566
|
const pending = this.requests.get(message.id);
|
|
@@ -68482,6 +68629,8 @@ class PoolCoordinator {
|
|
|
68482
68629
|
return this.stageAnswer(command.runId, command.stageId, command.answer);
|
|
68483
68630
|
case "stage-pause":
|
|
68484
68631
|
return this.stageControl(command.watch, { op: "pause" });
|
|
68632
|
+
case "stage-session":
|
|
68633
|
+
return this.stageSession(command.watch, command.command);
|
|
68485
68634
|
case "stage-resume":
|
|
68486
68635
|
return this.stageControl(command.watch, {
|
|
68487
68636
|
op: "resume",
|
|
@@ -68511,8 +68660,8 @@ class PoolCoordinator {
|
|
|
68511
68660
|
this.usageTotal = await this.usageLedger.total(root);
|
|
68512
68661
|
this.lease = lease;
|
|
68513
68662
|
const queue = await openSqlPoolQueue(this.journal, this.remote);
|
|
68514
|
-
await queue.initialize(lease, COORDINATOR_LEASE_MS);
|
|
68515
|
-
await queue.drainOutbox(lease);
|
|
68663
|
+
await this.whileRootLate(queue, () => queue.initialize(lease, COORDINATOR_LEASE_MS));
|
|
68664
|
+
await this.whileRootLate(queue, () => queue.drainOutbox(lease));
|
|
68516
68665
|
if (this.remote !== undefined) {
|
|
68517
68666
|
const remote = this.remote;
|
|
68518
68667
|
this.mirrorTimer = setInterval(() => void remote.fresh(lease, 0).catch(() => {}), MIRROR_INTERVAL_MS).unref();
|
|
@@ -68521,10 +68670,10 @@ class PoolCoordinator {
|
|
|
68521
68670
|
for (const work of await queue.inspect(root))
|
|
68522
68671
|
if (work.state === "leased" && work.epoch < lease.epoch && (await unconfirmedProcessGroups(this.journal, root, { attempt: work })).length > 0)
|
|
68523
68672
|
retained.push(work);
|
|
68524
|
-
await queue.releasePreviousIncarnations(lease, retained.map((work) => work.id));
|
|
68673
|
+
await this.whileRootLate(queue, () => queue.releasePreviousIncarnations(lease, retained.map((work) => work.id)));
|
|
68525
68674
|
for (;; ) {
|
|
68526
68675
|
const intended = this.desiredControl ?? "running";
|
|
68527
|
-
await queue.setControl(lease, intended);
|
|
68676
|
+
await this.whileRootLate(queue, () => queue.setControl(lease, intended));
|
|
68528
68677
|
if (intended === (this.desiredControl ?? "running"))
|
|
68529
68678
|
break;
|
|
68530
68679
|
}
|
|
@@ -68560,7 +68709,7 @@ class PoolCoordinator {
|
|
|
68560
68709
|
return this.retainedTransition(async () => {
|
|
68561
68710
|
const record = this.retained.get(work.id);
|
|
68562
68711
|
this.retained.delete(work.id);
|
|
68563
|
-
await this.queue.releaseRetained(this.lease, work, record?.enqueued === true).catch((error) => {
|
|
68712
|
+
await this.whileRootLate(this.queue, () => this.queue.releaseRetained(this.lease, work, record?.enqueued === true)).catch((error) => {
|
|
68564
68713
|
if (error.message !== STALE_WORK_LEASE)
|
|
68565
68714
|
throw error;
|
|
68566
68715
|
});
|
|
@@ -68574,20 +68723,29 @@ class PoolCoordinator {
|
|
|
68574
68723
|
async control(state) {
|
|
68575
68724
|
this.desiredControl = state;
|
|
68576
68725
|
if (!this.queue || !this.lease)
|
|
68577
|
-
return;
|
|
68578
|
-
const
|
|
68579
|
-
const
|
|
68580
|
-
|
|
68726
|
+
return 0;
|
|
68727
|
+
const queue = this.queue;
|
|
68728
|
+
const lease = this.lease;
|
|
68729
|
+
const control = await this.whileRootLate(queue, () => queue.setControl(lease, state));
|
|
68730
|
+
const unacknowledged = () => [...this.readyWorkers].filter((worker) => (this.controlAcks.get(worker) ?? -1) < control.revision).length;
|
|
68731
|
+
let pending = unacknowledged();
|
|
68732
|
+
let progressAt = performance.now();
|
|
68733
|
+
let stalledAt = this.stalledMs;
|
|
68734
|
+
while (pending > 0) {
|
|
68581
68735
|
if (this.desiredControl !== state)
|
|
68582
|
-
return;
|
|
68583
|
-
if (Date.now() >= deadline) {
|
|
68584
|
-
if (state === "quit")
|
|
68585
|
-
break;
|
|
68586
|
-
throw new Error("Worker control acknowledgement unresolved");
|
|
68587
|
-
}
|
|
68736
|
+
return pending;
|
|
68588
68737
|
await delay2(25);
|
|
68738
|
+
this.observeResponsiveness();
|
|
68739
|
+
const now = unacknowledged();
|
|
68740
|
+
if (now < pending) {
|
|
68741
|
+
pending = now;
|
|
68742
|
+
progressAt = performance.now();
|
|
68743
|
+
stalledAt = this.stalledMs;
|
|
68744
|
+
} else if (performance.now() - progressAt - (this.stalledMs - stalledAt) >= CONTROL_ACKNOWLEDGEMENT_STALL_MS)
|
|
68745
|
+
break;
|
|
68589
68746
|
}
|
|
68590
68747
|
this.flush();
|
|
68748
|
+
return pending;
|
|
68591
68749
|
}
|
|
68592
68750
|
async start(sources, cwd, publish, rootRun) {
|
|
68593
68751
|
if (this.closed)
|
|
@@ -68615,7 +68773,11 @@ class PoolCoordinator {
|
|
|
68615
68773
|
return;
|
|
68616
68774
|
});
|
|
68617
68775
|
this.recoveryTimer = setInterval(() => {
|
|
68618
|
-
this.recoverExpired().catch((error) =>
|
|
68776
|
+
this.recoverExpired().catch(async (error) => {
|
|
68777
|
+
if (this.queue && isRootLeaseLost(error) && await this.rootRenewal(this.queue).catch(() => false))
|
|
68778
|
+
return;
|
|
68779
|
+
this.fail(error);
|
|
68780
|
+
});
|
|
68619
68781
|
}, RECOVERY_INTERVAL_MS).unref();
|
|
68620
68782
|
const workers = Array.from({ length: this.options.workers }, () => this.spawn());
|
|
68621
68783
|
try {
|
|
@@ -68632,7 +68794,9 @@ class PoolCoordinator {
|
|
|
68632
68794
|
const queue = this.queue;
|
|
68633
68795
|
const lease = this.lease;
|
|
68634
68796
|
const work = await this.retainedTransition(async () => {
|
|
68635
|
-
|
|
68797
|
+
let enqueued = await this.whileRootLate(queue, () => queue.enqueue(lease, command.descriptor, command.boundary));
|
|
68798
|
+
if (enqueued.state === "completed" && enqueued.epoch < lease.epoch)
|
|
68799
|
+
enqueued = await this.whileRootLate(queue, () => queue.reopenCompleted(lease, enqueued.id, failedOnInfrastructureFault)) ?? enqueued;
|
|
68636
68800
|
const retained = this.retained.get(enqueued.id);
|
|
68637
68801
|
if (retained)
|
|
68638
68802
|
retained.enqueued = true;
|
|
@@ -68645,7 +68809,9 @@ class PoolCoordinator {
|
|
|
68645
68809
|
}
|
|
68646
68810
|
async transport(request, payload) {
|
|
68647
68811
|
const target = this.deliveryRequests.get(request);
|
|
68648
|
-
if (target === undefined
|
|
68812
|
+
if (target === undefined)
|
|
68813
|
+
throw new Error("Pooled stage delivery request already answered");
|
|
68814
|
+
if (!this.incarnations.has(target.slot.worker))
|
|
68649
68815
|
throw new Error("Worker exited before delivery acknowledgement");
|
|
68650
68816
|
const { slot, workerRequest } = target;
|
|
68651
68817
|
await new Promise((resolve, reject) => {
|
|
@@ -68873,6 +69039,34 @@ class PoolCoordinator {
|
|
|
68873
69039
|
return { ok: false, refusal: "StaleAttempt" };
|
|
68874
69040
|
return { ok: true, ...result.action === undefined ? {} : { action: result.action } };
|
|
68875
69041
|
}
|
|
69042
|
+
async stageSession(id, command) {
|
|
69043
|
+
const watch = this.watches.get(id);
|
|
69044
|
+
if (watch === undefined || !watch.open)
|
|
69045
|
+
return {
|
|
69046
|
+
ok: false,
|
|
69047
|
+
refusal: this.endedWatches.get(id) === "stage_finished" ? "StageNotLive" : "StaleAttempt"
|
|
69048
|
+
};
|
|
69049
|
+
const { slot, attempt } = watch;
|
|
69050
|
+
if (slot.work?.id !== attempt.workId || slot.work.generation !== attempt.generation)
|
|
69051
|
+
return { ok: false, refusal: "StaleAttempt" };
|
|
69052
|
+
if (this.desiredControl !== undefined && this.desiredControl !== "running" && !runsWhilePaused(command))
|
|
69053
|
+
return { ok: false, refusal: "PoolPausedOrQuitting" };
|
|
69054
|
+
const result = await this.stageCommand(slot, { attempt, op: "session", watch: id, command });
|
|
69055
|
+
if (result === undefined)
|
|
69056
|
+
return { ok: false, refusal: "StaleAttempt" };
|
|
69057
|
+
if (result.error !== undefined)
|
|
69058
|
+
throw new Error(result.error);
|
|
69059
|
+
const refusal = result.refusal;
|
|
69060
|
+
if (refusal === "StageNotLive" || refusal === "PoolPausedOrQuitting")
|
|
69061
|
+
return { ok: false, refusal };
|
|
69062
|
+
if (refusal !== undefined)
|
|
69063
|
+
return { ok: false, refusal: "StaleAttempt" };
|
|
69064
|
+
return {
|
|
69065
|
+
ok: true,
|
|
69066
|
+
...result.bash === undefined ? {} : { bash: result.bash },
|
|
69067
|
+
...result.task === undefined ? {} : { task: result.task }
|
|
69068
|
+
};
|
|
69069
|
+
}
|
|
68876
69070
|
stageCommand(slot, command) {
|
|
68877
69071
|
if (!this.incarnations.has(slot.worker)) {
|
|
68878
69072
|
tracePooledWatch("coordinator.command.no-worker", { op: command.op, worker: slot.worker });
|
|
@@ -68988,7 +69182,8 @@ class PoolCoordinator {
|
|
|
68988
69182
|
root,
|
|
68989
69183
|
definitions: this.sources,
|
|
68990
69184
|
admission: options.admission,
|
|
68991
|
-
...this.promptNodes ? { promptNodes: true } : {}
|
|
69185
|
+
...this.promptNodes ? { promptNodes: true } : {},
|
|
69186
|
+
...options.stageStartMaxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: options.stageStartMaxLoadPerCpu }
|
|
68992
69187
|
})
|
|
68993
69188
|
], {
|
|
68994
69189
|
stdio: ["ignore", "ignore", "pipe", "ipc"],
|
|
@@ -69109,7 +69304,10 @@ class PoolCoordinator {
|
|
|
69109
69304
|
if (this.closed)
|
|
69110
69305
|
return;
|
|
69111
69306
|
const exhausted = work !== undefined && this.attempts(work) >= MAX_CHILD_ATTEMPTS;
|
|
69112
|
-
const
|
|
69307
|
+
const queue = this.queue;
|
|
69308
|
+
const lease = this.lease;
|
|
69309
|
+
const attempt = work;
|
|
69310
|
+
const settled = attempt !== undefined && await this.whileRootLate(queue, () => exhausted ? queue.failExited(lease, attempt) : queue.releaseExited(lease, attempt));
|
|
69113
69311
|
const superseded = work !== undefined && !settled && await this.superseded(work);
|
|
69114
69312
|
if (work !== undefined && exhausted && !superseded)
|
|
69115
69313
|
this.failWork(work, diagnosed(childRetryLimit(work, "lost its worker")));
|
|
@@ -69171,21 +69369,40 @@ class PoolCoordinator {
|
|
|
69171
69369
|
await this.starting;
|
|
69172
69370
|
const queue = this.queue;
|
|
69173
69371
|
const lease = this.lease;
|
|
69372
|
+
const renewAttempt = async () => {
|
|
69373
|
+
const attempt = slot.work;
|
|
69374
|
+
if (attempt === undefined)
|
|
69375
|
+
return;
|
|
69376
|
+
const renewed = await queue.extend(lease, attempt, options.leaseMs);
|
|
69377
|
+
if (slot.work === attempt)
|
|
69378
|
+
slot.work = renewed;
|
|
69379
|
+
};
|
|
69380
|
+
const current = async (action) => {
|
|
69381
|
+
try {
|
|
69382
|
+
return await action();
|
|
69383
|
+
} catch (error) {
|
|
69384
|
+
if (!(error instanceof Error) || error.message !== STALE_WORK_LEASE || slot.work === undefined)
|
|
69385
|
+
throw error;
|
|
69386
|
+
await renewAttempt();
|
|
69387
|
+
return action();
|
|
69388
|
+
}
|
|
69389
|
+
};
|
|
69390
|
+
const leased = (action) => this.whileRootLate(queue, () => current(action), renewAttempt);
|
|
69174
69391
|
if (message.kind === "control")
|
|
69175
|
-
return queue.control(lease);
|
|
69392
|
+
return leased(() => queue.control(lease));
|
|
69176
69393
|
if (message.kind === "fenced") {
|
|
69177
69394
|
const fenced = slot.work;
|
|
69178
69395
|
slot.work = undefined;
|
|
69179
69396
|
if (fenced && !await this.superseded(fenced)) {
|
|
69180
69397
|
if (this.attempts(fenced) < MAX_CHILD_ATTEMPTS)
|
|
69181
|
-
await queue.releaseExited(lease, fenced);
|
|
69182
|
-
else if (await queue.failExited(lease, fenced))
|
|
69398
|
+
await leased(() => queue.releaseExited(lease, fenced));
|
|
69399
|
+
else if (await leased(() => queue.failExited(lease, fenced)))
|
|
69183
69400
|
this.failWork(fenced, new Error(childRetryLimit(fenced, "was refused by the root as stale on every attempt")));
|
|
69184
69401
|
}
|
|
69185
69402
|
return;
|
|
69186
69403
|
}
|
|
69187
69404
|
if (message.kind === "controlled") {
|
|
69188
|
-
const control = await queue.control(lease);
|
|
69405
|
+
const control = await leased(() => queue.control(lease));
|
|
69189
69406
|
if (message.revision > control.revision)
|
|
69190
69407
|
throw new Error("Unknown control revision");
|
|
69191
69408
|
this.controlAcks.set(worker, Math.max(this.controlAcks.get(worker) ?? -1, message.revision));
|
|
@@ -69196,21 +69413,21 @@ class PoolCoordinator {
|
|
|
69196
69413
|
throw new Error("Worker already holds an attempt");
|
|
69197
69414
|
if (this.budgetBlocked)
|
|
69198
69415
|
return;
|
|
69199
|
-
await
|
|
69200
|
-
|
|
69201
|
-
|
|
69202
|
-
|
|
69203
|
-
|
|
69204
|
-
|
|
69416
|
+
slot.work = await leased(async () => {
|
|
69417
|
+
await this.recoverExpired();
|
|
69418
|
+
slot.runs.clear();
|
|
69419
|
+
return queue.claim(lease, worker, this.sources.map((source) => ({
|
|
69420
|
+
definition: `${source.path}#${source.exportName}`,
|
|
69421
|
+
digest: source.digest
|
|
69422
|
+
})), options.leaseMs);
|
|
69423
|
+
});
|
|
69205
69424
|
return slot.work;
|
|
69206
69425
|
}
|
|
69207
|
-
|
|
69208
|
-
if (!
|
|
69426
|
+
const held = slot.work;
|
|
69427
|
+
if (!held)
|
|
69209
69428
|
throw new Error("Worker has no active attempt");
|
|
69210
|
-
|
|
69211
|
-
|
|
69212
|
-
slot.work = work;
|
|
69213
|
-
}
|
|
69429
|
+
const work = message.kind === "renew" || held.expiresAt - Date.now() < options.leaseMs / 2 ? await leased(() => queue.extend(lease, held, options.leaseMs)) : held;
|
|
69430
|
+
slot.work = work;
|
|
69214
69431
|
if (message.kind === "activity") {
|
|
69215
69432
|
for (const run of message.runs)
|
|
69216
69433
|
slot.runs.add(run.id);
|
|
@@ -69270,24 +69487,24 @@ class PoolCoordinator {
|
|
|
69270
69487
|
if (message.kind === "renew")
|
|
69271
69488
|
return;
|
|
69272
69489
|
if (message.kind === "commit") {
|
|
69273
|
-
const receipt = await queue.commit(lease, work, message.operation, message.mutation);
|
|
69490
|
+
const receipt = await leased(() => queue.commit(lease, work, message.operation, message.mutation));
|
|
69274
69491
|
if (message.mutation.kind !== "session-append")
|
|
69275
69492
|
this.noteUsage(await this.usageLedger.total(root));
|
|
69276
69493
|
return receipt;
|
|
69277
69494
|
}
|
|
69278
69495
|
if (message.kind === "usage") {
|
|
69279
|
-
this.noteUsage(await this.usageLedger.report(lease, work, message.reports));
|
|
69496
|
+
this.noteUsage(await leased(() => this.usageLedger.report(lease, work, message.reports)));
|
|
69280
69497
|
return;
|
|
69281
69498
|
}
|
|
69282
69499
|
if (message.kind === "budget") {
|
|
69283
|
-
await queue.assertCurrent(lease, work);
|
|
69500
|
+
await leased(() => queue.assertCurrent(lease, work));
|
|
69284
69501
|
if (this.budgetBlocked)
|
|
69285
69502
|
throw new Error("Root workflow budget exhausted");
|
|
69286
|
-
return queue.control(lease);
|
|
69503
|
+
return leased(() => queue.control(lease));
|
|
69287
69504
|
}
|
|
69288
69505
|
if (message.kind !== "complete")
|
|
69289
69506
|
throw new Error("Invalid worker operation");
|
|
69290
|
-
await queue.complete(lease, work, message.result);
|
|
69507
|
+
await leased(() => queue.complete(lease, work, message.result));
|
|
69291
69508
|
this.endStageWatches((watch) => watch.slot === slot, "stage_finished");
|
|
69292
69509
|
this.replacements = 0;
|
|
69293
69510
|
slot.work = undefined;
|
|
@@ -69326,7 +69543,7 @@ class PoolCoordinator {
|
|
|
69326
69543
|
if (this.recovering)
|
|
69327
69544
|
return this.recovering;
|
|
69328
69545
|
const root = this.root;
|
|
69329
|
-
if (this.closed || root === undefined || !this.queue || performance.now() < this.nextRecoveryAt)
|
|
69546
|
+
if (this.closed || root === undefined || !this.queue || this.lateRoot !== undefined || performance.now() < this.nextRecoveryAt)
|
|
69330
69547
|
return Promise.resolve();
|
|
69331
69548
|
const queue = this.queue;
|
|
69332
69549
|
const lease = this.lease;
|
|
@@ -69335,9 +69552,12 @@ class PoolCoordinator {
|
|
|
69335
69552
|
await new Promise((resolve) => setImmediate(resolve));
|
|
69336
69553
|
await this.admission?.reconcile();
|
|
69337
69554
|
const expiredBefore = Date.now();
|
|
69338
|
-
|
|
69339
|
-
|
|
69340
|
-
|
|
69555
|
+
const expired = (await queue.inspect(root)).filter((work) => work.state === "leased" && work.expiresAt < expiredBefore);
|
|
69556
|
+
if (expired.length > 0 && await queue.rootLease(lease, COORDINATOR_LEASE_MS) === "late") {
|
|
69557
|
+
this.rootRenewal(queue).catch(() => {});
|
|
69558
|
+
return;
|
|
69559
|
+
}
|
|
69560
|
+
for (const work of expired) {
|
|
69341
69561
|
if (this.quarantined.has(work.id))
|
|
69342
69562
|
continue;
|
|
69343
69563
|
if (this.held.has(work.id)) {
|
|
@@ -69453,7 +69673,7 @@ function childRetryLimit(work, reason) {
|
|
|
69453
69673
|
const exportName = work.definition.slice(work.definition.lastIndexOf("#") + 1);
|
|
69454
69674
|
return `Local worker retry limit: child ${work.childRunId} (${exportName}) ${reason} ${MAX_CHILD_ATTEMPTS} times`;
|
|
69455
69675
|
}
|
|
69456
|
-
var
|
|
69676
|
+
var CONTROL_ACKNOWLEDGEMENT_STALL_MS = 5000, RECOVERY_INTERVAL_MS = 500, TICK_INTERVAL_MS = 100, STALL_THRESHOLD_MS = 500, COORDINATOR_LEASE_MS, LATE_ROOT_POLL_MS, MIRROR_INTERVAL_MS = 1e4, STDERR_TAIL_CHARACTERS = 8192, STDERR_DRAIN_GRACE_MS = 1000, MAX_CHILD_ATTEMPTS = 3, ROUTE_REGISTRATION_WAIT_MS = 120000, STAGE_WATCH_BACKLOG_EVENTS = 2000, STAGE_WATCH_BACKLOG_BYTES, PENDING_STAGE_MESSAGE_EVENT2 = "atomic:workflow-pending-stage-message", STICKY_LIVE_DELIVERED_EVENT2 = "atomic:workflow-sticky-live-delivered", PENDING_STAGE_UNDELIVERABLE_EVENT2 = "atomic:workflow-pending-stage-undeliverable", ENDED_WATCHES_REMEMBERED = 1024;
|
|
69457
69677
|
var init_pool_coordinator = __esm(() => {
|
|
69458
69678
|
init_internal();
|
|
69459
69679
|
init_pool_remote_journal();
|
|
@@ -69467,12 +69687,14 @@ var init_pool_coordinator = __esm(() => {
|
|
|
69467
69687
|
init_pending_stage_route_capability();
|
|
69468
69688
|
init_pending_stage_route_readiness();
|
|
69469
69689
|
init_store();
|
|
69690
|
+
init_workflow_infrastructure_error();
|
|
69470
69691
|
init_pool_journal();
|
|
69471
69692
|
init_pooled_watch_trace();
|
|
69472
69693
|
init_resource_admission();
|
|
69473
69694
|
init_stage_watch_coalesce();
|
|
69474
69695
|
init_startup_timeout();
|
|
69475
69696
|
COORDINATOR_LEASE_MS = 3 * STAGE_SESSION_HEARTBEAT_INTERVAL_MS;
|
|
69697
|
+
LATE_ROOT_POLL_MS = RECOVERY_INTERVAL_MS;
|
|
69476
69698
|
STAGE_WATCH_BACKLOG_BYTES = 8 * 1024 * 1024;
|
|
69477
69699
|
});
|
|
69478
69700
|
|
|
@@ -69546,7 +69768,8 @@ function createWorkflowPendingStageDelivery(activeStore, runId, stageId, stageNa
|
|
|
69546
69768
|
const boundedDrain = (drain, sessionId) => {
|
|
69547
69769
|
const waiting = new Promise((resolve, reject) => {
|
|
69548
69770
|
drain.then(resolve, reject);
|
|
69549
|
-
|
|
69771
|
+
if (options.holdUntilDrained !== true)
|
|
69772
|
+
watchStall(drain, sessionId, resolve);
|
|
69550
69773
|
});
|
|
69551
69774
|
waiting.catch(() => {});
|
|
69552
69775
|
return waiting;
|
|
@@ -69928,10 +70151,14 @@ class PoolStageHandoff {
|
|
|
69928
70151
|
if (run.endedAt !== undefined || this.owners.get(run.id) !== work.id)
|
|
69929
70152
|
continue;
|
|
69930
70153
|
if (run.id === work.childRunId) {
|
|
69931
|
-
const
|
|
70154
|
+
const executions = new Map(run.stages.map((stage) => [stage.id, stage.workerExecution]));
|
|
70155
|
+
const attempt = run.stages.flatMap((stage) => stage.workerExecution === undefined ? [] : [stage.workerExecution]).reduce((latest, execution) => latest === undefined || execution.renewedAt > latest.renewedAt ? execution : latest, undefined);
|
|
69932
70156
|
this.store.recordRunStart({
|
|
69933
70157
|
...run,
|
|
69934
|
-
stages: result.stages.map((stage) =>
|
|
70158
|
+
stages: result.stages.map((stage) => {
|
|
70159
|
+
const execution = executions.get(stage.id) ?? attempt;
|
|
70160
|
+
return execution === undefined ? stage : { ...stage, workerExecution: execution };
|
|
70161
|
+
}),
|
|
69935
70162
|
toolNodes: result.toolNodes
|
|
69936
70163
|
});
|
|
69937
70164
|
this.store.recordRunEnd(run.id, result.status, result.result, result.error);
|
|
@@ -69985,7 +70212,7 @@ class PoolStageHandoff {
|
|
|
69985
70212
|
pendingStageMessages: [
|
|
69986
70213
|
...known.get(run.id)?.pendingStageMessages ?? pendingStageMessagesForDurableRun(this.backend, run.id, root)
|
|
69987
70214
|
],
|
|
69988
|
-
stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: execution })), work.generation)
|
|
70215
|
+
stages: withReconnectingStages(known.get(run.id), run, run.stages.map((stage) => ({ ...stage, workerExecution: endedUnder(known.get(run.id), stage, execution) })), work.generation)
|
|
69989
70216
|
}));
|
|
69990
70217
|
const changed = published.some((run) => activity(known.get(run.id)) !== activity(run));
|
|
69991
70218
|
for (const run of published)
|
|
@@ -70116,7 +70343,7 @@ class PoolStageHandoff {
|
|
|
70116
70343
|
throw new Error(STALE);
|
|
70117
70344
|
if (attempt.sessions.get(key) !== sessionId)
|
|
70118
70345
|
throw new Error(`Stage session ${sessionId} was replaced before it acknowledged delivery`);
|
|
70119
|
-
}, action));
|
|
70346
|
+
}, action), { holdUntilDrained: true });
|
|
70120
70347
|
attempt.deliveries.set(key, delivery);
|
|
70121
70348
|
}
|
|
70122
70349
|
return delivery;
|
|
@@ -70128,6 +70355,12 @@ function outside2(runId) {
|
|
|
70128
70355
|
function isLive(stage) {
|
|
70129
70356
|
return (stage.sessionId !== undefined || stage.sessionFile !== undefined) && stage.status !== "completed" && stage.status !== "failed" && stage.status !== "skipped";
|
|
70130
70357
|
}
|
|
70358
|
+
function endedUnder(known, stage, execution) {
|
|
70359
|
+
if (!isTerminalStageStatus(stage.status))
|
|
70360
|
+
return execution;
|
|
70361
|
+
const previous = known?.stages.find((candidate) => candidate.id === stage.id);
|
|
70362
|
+
return previous !== undefined && isTerminalStageStatus(previous.status) && previous.workerExecution?.generation === execution.generation && previous.workerExecution.worker === execution.worker ? previous.workerExecution : execution;
|
|
70363
|
+
}
|
|
70131
70364
|
function withReconnectingStages(previous, run, stages, generation) {
|
|
70132
70365
|
if (previous === undefined || run.endedAt !== undefined || isTerminalRunStatus(run.status))
|
|
70133
70366
|
return stages;
|
|
@@ -70216,6 +70449,116 @@ class StageToolExecutionBuffer {
|
|
|
70216
70449
|
}
|
|
70217
70450
|
}
|
|
70218
70451
|
|
|
70452
|
+
// dist/builtin/workflows/src/workers/pooled-stage-task-source.ts
|
|
70453
|
+
function failure2(error) {
|
|
70454
|
+
return { ok: false, error: { message: typeof error === "string" ? error : error.message } };
|
|
70455
|
+
}
|
|
70456
|
+
|
|
70457
|
+
class PooledStageTaskSource {
|
|
70458
|
+
run;
|
|
70459
|
+
state = EMPTY;
|
|
70460
|
+
records = [];
|
|
70461
|
+
backgroundRecords = [];
|
|
70462
|
+
settlements = new Map;
|
|
70463
|
+
activity = new Map;
|
|
70464
|
+
statuses = new Map;
|
|
70465
|
+
stdin = new Set;
|
|
70466
|
+
listeners = new Set;
|
|
70467
|
+
constructor(run) {
|
|
70468
|
+
this.run = run;
|
|
70469
|
+
}
|
|
70470
|
+
apply(state) {
|
|
70471
|
+
this.state = state;
|
|
70472
|
+
this.records = [...state.tasks];
|
|
70473
|
+
const background = new Set(state.background);
|
|
70474
|
+
this.backgroundRecords = this.records.filter((task) => background.has(task.ref.taskId));
|
|
70475
|
+
this.settlements = new Map(state.settlement.map(([id, sequence]) => [id, BigInt(sequence)]));
|
|
70476
|
+
this.activity = new Map(state.activity.map((entry) => [entry.id, entry]));
|
|
70477
|
+
this.statuses = new Map(state.statuses.map((entry) => [entry.id, new Map(entry.entries)]));
|
|
70478
|
+
this.stdin = new Set(state.stdin);
|
|
70479
|
+
for (const listener of [...this.listeners])
|
|
70480
|
+
listener();
|
|
70481
|
+
}
|
|
70482
|
+
get empty() {
|
|
70483
|
+
return this.state.tasks.length === 0;
|
|
70484
|
+
}
|
|
70485
|
+
get tasks() {
|
|
70486
|
+
return this.records;
|
|
70487
|
+
}
|
|
70488
|
+
get backgroundTasks() {
|
|
70489
|
+
return this.backgroundRecords;
|
|
70490
|
+
}
|
|
70491
|
+
get settlementOrder() {
|
|
70492
|
+
return this.settlements;
|
|
70493
|
+
}
|
|
70494
|
+
recentActivity(id) {
|
|
70495
|
+
return [...this.activity.get(id)?.entries ?? []];
|
|
70496
|
+
}
|
|
70497
|
+
activityOmitted(id) {
|
|
70498
|
+
return this.activity.get(id)?.omitted === true;
|
|
70499
|
+
}
|
|
70500
|
+
subscribe(listener) {
|
|
70501
|
+
this.listeners.add(listener);
|
|
70502
|
+
return () => {
|
|
70503
|
+
this.listeners.delete(listener);
|
|
70504
|
+
};
|
|
70505
|
+
}
|
|
70506
|
+
async foregroundTask(id) {
|
|
70507
|
+
const reply = await this.request(id, { kind: "foreground" });
|
|
70508
|
+
return reply.ok ? reply.value.kind === "foreground" ? reply.value.result : unexpected() : reply;
|
|
70509
|
+
}
|
|
70510
|
+
async cancelTask(id) {
|
|
70511
|
+
const reply = await this.request(id, { kind: "cancel" });
|
|
70512
|
+
if (!reply.ok)
|
|
70513
|
+
return reply;
|
|
70514
|
+
return reply.value.kind === "cancel" ? reply.value.result ?? { ok: true, value: undefined } : unexpected();
|
|
70515
|
+
}
|
|
70516
|
+
stdinAvailable(id) {
|
|
70517
|
+
return this.stdin.has(id);
|
|
70518
|
+
}
|
|
70519
|
+
async writeTaskInput(id, text) {
|
|
70520
|
+
const reply = await this.request(id, { kind: "input", text });
|
|
70521
|
+
return reply.ok ? reply.value.kind === "input" ? reply.value.result : unexpected() : reply;
|
|
70522
|
+
}
|
|
70523
|
+
async readTaskOutput(id, range) {
|
|
70524
|
+
const reply = await this.request(id, { kind: "output", start: range.start, maximumBytes: range.maximumBytes });
|
|
70525
|
+
return reply.ok ? reply.value.kind === "output" ? reply.value.result : unexpected() : reply;
|
|
70526
|
+
}
|
|
70527
|
+
async taskDetailText(id) {
|
|
70528
|
+
const reply = await this.request(id, { kind: "detail" });
|
|
70529
|
+
return reply.ok && reply.value.kind === "detail" ? reply.value.detail : {};
|
|
70530
|
+
}
|
|
70531
|
+
async readTaskTranscript(id, cursor) {
|
|
70532
|
+
const reply = await this.request(id, { kind: "transcript", ...cursor === undefined ? {} : { cursor } });
|
|
70533
|
+
return reply.ok ? reply.value.kind === "transcript" ? reply.value.result : unexpected() : reply;
|
|
70534
|
+
}
|
|
70535
|
+
followTaskTranscript(id) {
|
|
70536
|
+
return this.records.some((task) => task.ref.taskId === id && task.kind === "agent") ? { historyOnly: true } : undefined;
|
|
70537
|
+
}
|
|
70538
|
+
extensionStatuses(id) {
|
|
70539
|
+
return this.statuses.get(id) ?? NO_STATUSES;
|
|
70540
|
+
}
|
|
70541
|
+
watchExtensionStatuses(_id, listener) {
|
|
70542
|
+
return this.subscribe(listener);
|
|
70543
|
+
}
|
|
70544
|
+
async request(id, action) {
|
|
70545
|
+
try {
|
|
70546
|
+
const reply = await this.run(id, action);
|
|
70547
|
+
return reply === undefined ? failure2("Task unavailable") : { ok: true, value: reply };
|
|
70548
|
+
} catch (error) {
|
|
70549
|
+
return failure2(error instanceof Error ? error : String(error));
|
|
70550
|
+
}
|
|
70551
|
+
}
|
|
70552
|
+
}
|
|
70553
|
+
function unexpected() {
|
|
70554
|
+
return failure2("Unexpected reply from the local worker");
|
|
70555
|
+
}
|
|
70556
|
+
var EMPTY, NO_STATUSES;
|
|
70557
|
+
var init_pooled_stage_task_source = __esm(() => {
|
|
70558
|
+
EMPTY = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
|
|
70559
|
+
NO_STATUSES = new Map;
|
|
70560
|
+
});
|
|
70561
|
+
|
|
70219
70562
|
// dist/builtin/workflows/src/workers/pooled-stage-handle.ts
|
|
70220
70563
|
import { randomUUID as randomUUID18 } from "node:crypto";
|
|
70221
70564
|
|
|
@@ -70246,6 +70589,11 @@ class PooledStageHandle {
|
|
|
70246
70589
|
undelivered;
|
|
70247
70590
|
disposed = false;
|
|
70248
70591
|
session;
|
|
70592
|
+
commands;
|
|
70593
|
+
sessionStarted = true;
|
|
70594
|
+
started = Promise.withResolvers();
|
|
70595
|
+
bashRuns = new Map;
|
|
70596
|
+
taskView = new PooledStageTaskSource(async (taskId, action) => (await this.sessionCommand({ kind: "task", taskId, action })).task);
|
|
70249
70597
|
pumping;
|
|
70250
70598
|
pumpAgain = false;
|
|
70251
70599
|
watchGone = Promise.withResolvers();
|
|
@@ -70308,6 +70656,43 @@ class PooledStageHandle {
|
|
|
70308
70656
|
streamingMessage() {
|
|
70309
70657
|
return this.streaming;
|
|
70310
70658
|
}
|
|
70659
|
+
commandCatalog() {
|
|
70660
|
+
return this.commands;
|
|
70661
|
+
}
|
|
70662
|
+
taskSource() {
|
|
70663
|
+
return this.taskView;
|
|
70664
|
+
}
|
|
70665
|
+
get isSessionStarting() {
|
|
70666
|
+
return !this.sessionStarted && this.ended === undefined;
|
|
70667
|
+
}
|
|
70668
|
+
get isBashRunning() {
|
|
70669
|
+
return this.bashRuns.size > 0;
|
|
70670
|
+
}
|
|
70671
|
+
async executeBash(command, onChunk, options) {
|
|
70672
|
+
const id = randomUUID18();
|
|
70673
|
+
this.bashRuns.set(id, onChunk);
|
|
70674
|
+
try {
|
|
70675
|
+
const result = await this.sessionCommand({
|
|
70676
|
+
kind: "bash",
|
|
70677
|
+
id,
|
|
70678
|
+
command,
|
|
70679
|
+
...options?.excludeFromContext === undefined ? {} : { excludeFromContext: options.excludeFromContext }
|
|
70680
|
+
});
|
|
70681
|
+
return result.bash ?? { output: "", exitCode: undefined, cancelled: true, truncated: false };
|
|
70682
|
+
} finally {
|
|
70683
|
+
this.bashRuns.delete(id);
|
|
70684
|
+
}
|
|
70685
|
+
}
|
|
70686
|
+
abortBash() {
|
|
70687
|
+
for (const id of this.bashRuns.keys())
|
|
70688
|
+
this.sessionCommand({ kind: "abort-bash", id }).catch(() => {});
|
|
70689
|
+
}
|
|
70690
|
+
async compact() {
|
|
70691
|
+
await this.sessionCommand({ kind: "compact" });
|
|
70692
|
+
}
|
|
70693
|
+
abortCompaction() {
|
|
70694
|
+
this.sessionCommand({ kind: "abort-compaction" }).catch(() => {});
|
|
70695
|
+
}
|
|
70311
70696
|
pendingToolExecutionEvents() {
|
|
70312
70697
|
return this.toolExecutions.replayEvents();
|
|
70313
70698
|
}
|
|
@@ -70455,6 +70840,17 @@ class PooledStageHandle {
|
|
|
70455
70840
|
if (this.disposed)
|
|
70456
70841
|
throw new Error(`Stage chat "${this.stageName}" is closed.`);
|
|
70457
70842
|
}
|
|
70843
|
+
async sessionCommand(command) {
|
|
70844
|
+
this.throwIfClosed();
|
|
70845
|
+
if (this.ended !== undefined)
|
|
70846
|
+
throw new PooledStageDeliveryError("StageNotLive", this.stageName);
|
|
70847
|
+
if (this.transport.session === undefined)
|
|
70848
|
+
throw new Error(`Stage "${this.stageName}" runs on a local worker that cannot run this command.`);
|
|
70849
|
+
const result = await this.transport.session(this.watchId, command);
|
|
70850
|
+
if (!result.ok)
|
|
70851
|
+
throw new PooledStageDeliveryError(result.refusal, this.stageName);
|
|
70852
|
+
return result;
|
|
70853
|
+
}
|
|
70458
70854
|
undeliveredCount() {
|
|
70459
70855
|
if (this.undelivered !== undefined)
|
|
70460
70856
|
return this.undelivered;
|
|
@@ -70520,6 +70916,16 @@ class PooledStageHandle {
|
|
|
70520
70916
|
entry.waiter = undefined;
|
|
70521
70917
|
}
|
|
70522
70918
|
}
|
|
70919
|
+
markSessionStarted(started) {
|
|
70920
|
+
if (started) {
|
|
70921
|
+
this.sessionStarted = true;
|
|
70922
|
+
this.started.resolve();
|
|
70923
|
+
return;
|
|
70924
|
+
}
|
|
70925
|
+
if (this.sessionStarted)
|
|
70926
|
+
this.started = Promise.withResolvers();
|
|
70927
|
+
this.sessionStarted = false;
|
|
70928
|
+
}
|
|
70523
70929
|
forget(op) {
|
|
70524
70930
|
const index = this.outbound.findIndex((entry) => entry.op === op);
|
|
70525
70931
|
if (index >= 0)
|
|
@@ -70576,10 +70982,17 @@ class PooledStageHandle {
|
|
|
70576
70982
|
const gone = this.watchGone.promise.then(() => ({ ok: false, refusal: "StaleAttempt" }));
|
|
70577
70983
|
const sending = this.sendEntry(entry, watch);
|
|
70578
70984
|
let bound;
|
|
70985
|
+
let stopped = false;
|
|
70579
70986
|
const unconfirmed = new Promise((resolve) => {
|
|
70580
|
-
|
|
70987
|
+
this.started.promise.then(() => {
|
|
70988
|
+
if (!stopped)
|
|
70989
|
+
bound = setTimeout(() => resolve("unconfirmed"), this.admissionWaitMs);
|
|
70990
|
+
});
|
|
70991
|
+
});
|
|
70992
|
+
const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => {
|
|
70993
|
+
stopped = true;
|
|
70994
|
+
clearTimeout(bound);
|
|
70581
70995
|
});
|
|
70582
|
-
const outcome = await Promise.race([sending, gone, unconfirmed]).finally(() => clearTimeout(bound));
|
|
70583
70996
|
if (outcome === "unconfirmed") {
|
|
70584
70997
|
tracePooledWatch("root.outbox.unconfirmed", { op: entry.op, watch, generation });
|
|
70585
70998
|
entry.unconfirmedOn = generation;
|
|
@@ -70669,6 +71082,10 @@ class PooledStageHandle {
|
|
|
70669
71082
|
}
|
|
70670
71083
|
adopt(snapshot) {
|
|
70671
71084
|
this.session = snapshot.sessionId;
|
|
71085
|
+
this.commands = snapshot.commands;
|
|
71086
|
+
if (!this.taskView.empty)
|
|
71087
|
+
this.taskView.apply(NO_TASKS);
|
|
71088
|
+
this.markSessionStarted(snapshot.sessionStarted !== false);
|
|
70672
71089
|
this.settled = [...snapshot.messages];
|
|
70673
71090
|
this.streaming = snapshot.streamingMessage;
|
|
70674
71091
|
this.currentStatus = snapshot.status;
|
|
@@ -70689,6 +71106,25 @@ class PooledStageHandle {
|
|
|
70689
71106
|
this.forget(event.op);
|
|
70690
71107
|
return false;
|
|
70691
71108
|
}
|
|
71109
|
+
if (event.type === STAGE_SESSION_STARTED_EVENT) {
|
|
71110
|
+
this.commands = event.commands;
|
|
71111
|
+
this.markSessionStarted(true);
|
|
71112
|
+
const queued = this.queuedUserMessages();
|
|
71113
|
+
const redraw = { type: "queue_update", steering: [...queued.steering], followUp: [...queued.followUp] };
|
|
71114
|
+
for (const listener of [...this.listeners])
|
|
71115
|
+
listener(redraw);
|
|
71116
|
+
return false;
|
|
71117
|
+
}
|
|
71118
|
+
if (event.type === STAGE_TASKS_EVENT) {
|
|
71119
|
+
this.taskView.apply(event.tasks);
|
|
71120
|
+
return false;
|
|
71121
|
+
}
|
|
71122
|
+
if (event.type === STAGE_MESSAGES_EVENT) {
|
|
71123
|
+
this.settled = [...event.messages];
|
|
71124
|
+
return false;
|
|
71125
|
+
}
|
|
71126
|
+
if (event.type === "bash_execution_update" && event.id !== undefined)
|
|
71127
|
+
this.bashRuns.get(event.id)?.(event.delta);
|
|
70692
71128
|
this.applyEvent(event);
|
|
70693
71129
|
const turnUnderway = TURN_EVENTS.has(event.type) || (event.type === "message_start" || event.type === "message_update") && event.message.role === "assistant";
|
|
70694
71130
|
if (turnUnderway && this.awaitingTurn) {
|
|
@@ -70771,10 +71207,12 @@ async function reconnectPooledStageHandle(handle, transport, wanted = () => true
|
|
|
70771
71207
|
await rebinding;
|
|
70772
71208
|
return !handle.isReconnecting;
|
|
70773
71209
|
}
|
|
70774
|
-
var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
|
|
71210
|
+
var DELIVERY_RECEIPT_EVENT = "atomic_pooled_delivery_receipt", STAGE_SESSION_STARTED_EVENT = "atomic_pooled_stage_session_started", STAGE_MESSAGES_EVENT = "atomic_pooled_stage_messages", STAGE_TASKS_EVENT = "atomic_pooled_stage_tasks", NO_TASKS, POOLED_ADMISSION_WAIT_MS = 15000, RECONNECTABLE, TURN_EVENTS, PooledStageDeliveryError;
|
|
70775
71211
|
var init_pooled_stage_handle = __esm(() => {
|
|
70776
71212
|
init_stage_queued_user_messages();
|
|
71213
|
+
init_pooled_stage_task_source();
|
|
70777
71214
|
init_pooled_watch_trace();
|
|
71215
|
+
NO_TASKS = { tasks: [], background: [], settlement: [], activity: [], stdin: [], statuses: [] };
|
|
70778
71216
|
RECONNECTABLE = new Set(["attempt_fenced", "worker_exited", "resync"]);
|
|
70779
71217
|
TURN_EVENTS = new Set(["agent_start", "turn_start", "tool_execution_start", "tool_execution_update"]);
|
|
70780
71218
|
PooledStageDeliveryError = class PooledStageDeliveryError extends Error {
|
|
@@ -70834,6 +71272,9 @@ class RegisteredPooledStageHandle {
|
|
|
70834
71272
|
pendingToolExecutionEvents() {
|
|
70835
71273
|
return this.current()?.pendingToolExecutionEvents?.() ?? [];
|
|
70836
71274
|
}
|
|
71275
|
+
commandCatalog() {
|
|
71276
|
+
return this.current()?.commandCatalog?.();
|
|
71277
|
+
}
|
|
70837
71278
|
update(stage) {
|
|
70838
71279
|
this.stage = stage;
|
|
70839
71280
|
}
|
|
@@ -71132,6 +71573,7 @@ class LocalWorkerPool {
|
|
|
71132
71573
|
rootSignature;
|
|
71133
71574
|
completing = 0;
|
|
71134
71575
|
rootFlush;
|
|
71576
|
+
connecting;
|
|
71135
71577
|
starting;
|
|
71136
71578
|
failure;
|
|
71137
71579
|
closed = false;
|
|
@@ -71161,6 +71603,8 @@ class LocalWorkerPool {
|
|
|
71161
71603
|
throw new Error("Invalid local worker startup timeout");
|
|
71162
71604
|
if (options.leaseMs !== undefined && (!Number.isSafeInteger(options.leaseMs) || options.leaseMs < MINIMUM_WORK_LEASE_MS || options.leaseMs > MAX_TIMER_DELAY_MS))
|
|
71163
71605
|
throw new Error(`Invalid local worker lease: must be an integer from ${MINIMUM_WORK_LEASE_MS} to ${MAX_TIMER_DELAY_MS} ms`);
|
|
71606
|
+
if (options.maxLoadPerCpu !== undefined && (!Number.isFinite(options.maxLoadPerCpu) || options.maxLoadPerCpu < 0))
|
|
71607
|
+
throw new Error("Invalid local worker load threshold: maxLoadPerCpu must be a number of at least 0");
|
|
71164
71608
|
this.leaseMs = options.leaseMs ?? DEFAULT_WORK_LEASE_MS;
|
|
71165
71609
|
this.releaseBinding = bindChildWorkerDispatcher(backend, this, options.rootId);
|
|
71166
71610
|
if (options.admission && options.rootId === undefined)
|
|
@@ -71188,23 +71632,35 @@ class LocalWorkerPool {
|
|
|
71188
71632
|
if (!state || state.holderId !== this.backend.holderId)
|
|
71189
71633
|
throw new Error("Local worker pool requires the root owner");
|
|
71190
71634
|
this.lease = { runId: root, holderId: state.holderId, epoch: state.epoch };
|
|
71191
|
-
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
71192
|
-
this.usageTotal = await this.command({
|
|
71193
|
-
kind: "prepare",
|
|
71194
|
-
root,
|
|
71195
|
-
lease: this.lease,
|
|
71196
|
-
options: {
|
|
71197
|
-
journalPath: this.options.journalPath,
|
|
71198
|
-
...this.options.durable === undefined ? {} : { durable: this.options.durable },
|
|
71199
|
-
workers: this.options.workers,
|
|
71200
|
-
leaseMs: this.leaseMs,
|
|
71201
|
-
startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
|
|
71202
|
-
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
71203
|
-
...workerEntry === undefined ? {} : { workerEntry }
|
|
71204
|
-
},
|
|
71205
|
-
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
71206
|
-
});
|
|
71207
71635
|
this.queue ??= await openSqlPoolQueue(poolJournal, this.options.durable === undefined ? undefined : new PoolRemoteJournal(this.backend.journal, sqlPoolStorage(poolJournal)));
|
|
71636
|
+
if (await this.queue.initialized(root))
|
|
71637
|
+
await this.connectPool();
|
|
71638
|
+
}
|
|
71639
|
+
connectPool() {
|
|
71640
|
+
this.connecting ??= (async () => {
|
|
71641
|
+
const root = this.root;
|
|
71642
|
+
const lease = this.lease;
|
|
71643
|
+
if (root === undefined || lease === undefined)
|
|
71644
|
+
throw new Error("Local worker pool requires root preparation");
|
|
71645
|
+
const workerEntry = this.options.workerEntry ?? (isBunBinary2 || this.options.coordinator === "in-process" ? defaultWorkerEntry() : undefined);
|
|
71646
|
+
this.usageTotal = await this.command({
|
|
71647
|
+
kind: "prepare",
|
|
71648
|
+
root,
|
|
71649
|
+
lease,
|
|
71650
|
+
options: {
|
|
71651
|
+
journalPath: this.options.journalPath,
|
|
71652
|
+
...this.options.durable === undefined ? {} : { durable: this.options.durable },
|
|
71653
|
+
workers: this.options.workers,
|
|
71654
|
+
leaseMs: this.leaseMs,
|
|
71655
|
+
startupTimeoutMs: this.options.startupTimeoutMs ?? defaultStartupTimeoutMs(this.options.workers),
|
|
71656
|
+
...this.options.admission === undefined ? {} : { admission: this.options.admission },
|
|
71657
|
+
...this.options.maxLoadPerCpu === undefined ? {} : { stageStartMaxLoadPerCpu: this.options.maxLoadPerCpu },
|
|
71658
|
+
...workerEntry === undefined ? {} : { workerEntry }
|
|
71659
|
+
},
|
|
71660
|
+
...this.desiredControl === undefined ? {} : { control: this.desiredControl }
|
|
71661
|
+
});
|
|
71662
|
+
})();
|
|
71663
|
+
return this.connecting;
|
|
71208
71664
|
}
|
|
71209
71665
|
async poolJournal() {
|
|
71210
71666
|
const journal = this.backend.journal;
|
|
@@ -71253,6 +71709,7 @@ class LocalWorkerPool {
|
|
|
71253
71709
|
async start(root, promptNodes) {
|
|
71254
71710
|
if (this.closed)
|
|
71255
71711
|
throw new Error("Local worker pool is closed");
|
|
71712
|
+
await this.connectPool();
|
|
71256
71713
|
if (!this.queue || !this.lease || this.lease.runId !== root)
|
|
71257
71714
|
throw new Error("Local worker pool requires root preparation");
|
|
71258
71715
|
this.sources = await Promise.all(this.options.definitions.map(async (binding) => {
|
|
@@ -71535,6 +71992,11 @@ class LocalWorkerPool {
|
|
|
71535
71992
|
...message === undefined ? {} : { message }
|
|
71536
71993
|
});
|
|
71537
71994
|
}
|
|
71995
|
+
async sessionCommandOnPooledStage(watch, command) {
|
|
71996
|
+
if (this.closed || this.coordinatorLost !== undefined)
|
|
71997
|
+
return { ok: false, refusal: "StaleAttempt" };
|
|
71998
|
+
return await this.command({ kind: "stage-session", watch, command });
|
|
71999
|
+
}
|
|
71538
72000
|
stageTransport(onEnded) {
|
|
71539
72001
|
return {
|
|
71540
72002
|
watch: (ref, listener) => this.watchPooledStage(ref, (message) => {
|
|
@@ -71545,7 +72007,8 @@ class LocalWorkerPool {
|
|
|
71545
72007
|
unwatch: (watch) => this.unwatchPooledStage(watch),
|
|
71546
72008
|
deliver: (delivery) => this.deliverToPooledStage(delivery),
|
|
71547
72009
|
pause: (watch) => this.pausePooledStage(watch),
|
|
71548
|
-
resume: (watch, message) => this.resumePooledStage(watch, message)
|
|
72010
|
+
resume: (watch, message) => this.resumePooledStage(watch, message),
|
|
72011
|
+
session: (watch, command) => this.sessionCommandOnPooledStage(watch, command)
|
|
71549
72012
|
};
|
|
71550
72013
|
}
|
|
71551
72014
|
async acquirePooledStage(runId, stageId) {
|
|
@@ -72208,6 +72671,9 @@ var init_local_pool = __esm(() => {
|
|
|
72208
72671
|
completeInTurns = createTurnSlicer(2, COMPLETION_TURN_BUDGET_MS);
|
|
72209
72672
|
});
|
|
72210
72673
|
|
|
72674
|
+
// dist/builtin/workflows/src/workers/public-options.ts
|
|
72675
|
+
var DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU = 3;
|
|
72676
|
+
|
|
72211
72677
|
// dist/builtin/workflows/src/workers/public-pool.ts
|
|
72212
72678
|
import { resolve as resolve16 } from "node:path";
|
|
72213
72679
|
import { isDeepStrictEqual as isDeepStrictEqual2 } from "node:util";
|
|
@@ -72268,6 +72734,7 @@ async function activatePublicPool(backend, options) {
|
|
|
72268
72734
|
...options.localWorkers,
|
|
72269
72735
|
startupTimeoutMs: undefined,
|
|
72270
72736
|
leaseMs: undefined,
|
|
72737
|
+
maxLoadPerCpu: undefined,
|
|
72271
72738
|
definitions: options.localWorkers.definitions.map((source) => ({ ...source, path: resolve16(cwd, source.path) }))
|
|
72272
72739
|
};
|
|
72273
72740
|
const admission = options.admission && {
|
|
@@ -72311,6 +72778,7 @@ async function activatePublicPool(backend, options) {
|
|
|
72311
72778
|
admission: resources.admission ?? {},
|
|
72312
72779
|
startupTimeoutMs: options.localWorkers?.startupTimeoutMs,
|
|
72313
72780
|
leaseMs: options.localWorkers?.leaseMs,
|
|
72781
|
+
maxLoadPerCpu: options.localWorkers?.maxLoadPerCpu ?? DEFAULT_POOLED_STAGE_START_MAX_LOAD_PER_CPU,
|
|
72314
72782
|
...options.stageControlRegistry === undefined || options[hostWorkerBindings] ? {} : { stageControlRegistry: options.stageControlRegistry }
|
|
72315
72783
|
});
|
|
72316
72784
|
try {
|
|
@@ -77554,7 +78022,7 @@ class StageSessionController {
|
|
|
77554
78022
|
return this.modelRoute ? this.checkedSession(await ready) : ready;
|
|
77555
78023
|
}
|
|
77556
78024
|
async ensureSessionFromId(sessionId, consumer = "prompt") {
|
|
77557
|
-
const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined);
|
|
78025
|
+
const joinsLiveGeneration = !this.sessionShutdownPromise && (this.sessionPromise !== undefined || this.session !== undefined || this.activeCreation !== undefined || this.ownedCreationPromise !== undefined);
|
|
77558
78026
|
if (!this.sessionShutdownPromise && !joinsLiveGeneration)
|
|
77559
78027
|
this.reattachSessionId = sessionId;
|
|
77560
78028
|
const session = await this.ensureSession(consumer);
|
|
@@ -84037,6 +84505,44 @@ function effectiveWidth(width) {
|
|
|
84037
84505
|
// dist/builtin/workflows/src/tui/run-detail.ts
|
|
84038
84506
|
init_pending_stage_status();
|
|
84039
84507
|
|
|
84508
|
+
// dist/builtin/workflows/src/shared/stage-display-status.ts
|
|
84509
|
+
var STARTUP_STEP_TEXT = {
|
|
84510
|
+
"model-resolution": "resolving model",
|
|
84511
|
+
"route-authority": "waiting for model route",
|
|
84512
|
+
"resource-preparation": "loading resources",
|
|
84513
|
+
"reload-queued": "waiting to reload resources",
|
|
84514
|
+
"reload-active": "reloading resources",
|
|
84515
|
+
"sdk-creation": "creating session",
|
|
84516
|
+
"extension-binding": "binding extensions",
|
|
84517
|
+
"session-attachment": "attaching session",
|
|
84518
|
+
"delivery-readiness": "preparing message delivery",
|
|
84519
|
+
ready: "session ready",
|
|
84520
|
+
"first-dispatch": "session ready"
|
|
84521
|
+
};
|
|
84522
|
+
function isPooledStageStarting(stage) {
|
|
84523
|
+
if (stage === undefined || stage.workerExecution === undefined || stage.status !== "running")
|
|
84524
|
+
return false;
|
|
84525
|
+
if (stage.nodeKind === "tool" || stage.workflowChild !== undefined || stage.workflowChildRun !== undefined)
|
|
84526
|
+
return false;
|
|
84527
|
+
const startup = stage.startup;
|
|
84528
|
+
return startup !== undefined && startup.state === "active" && startup.phase !== "ready";
|
|
84529
|
+
}
|
|
84530
|
+
function stageDisplayStatus(stage) {
|
|
84531
|
+
return isPooledStageStarting(stage) ? "queued" : stage.status;
|
|
84532
|
+
}
|
|
84533
|
+
function pooledStageStartingStep(stage) {
|
|
84534
|
+
const startup = stage?.startup;
|
|
84535
|
+
if (startup === undefined || !isPooledStageStarting(stage))
|
|
84536
|
+
return;
|
|
84537
|
+
return STARTUP_STEP_TEXT[startup.phase];
|
|
84538
|
+
}
|
|
84539
|
+
function pooledStageStartingText(stage, now = Date.now()) {
|
|
84540
|
+
const step = pooledStageStartingStep(stage);
|
|
84541
|
+
if (step === undefined || stage?.startup === undefined)
|
|
84542
|
+
return;
|
|
84543
|
+
const seconds = Math.max(0, Math.floor((now - stage.startup.startedAt) / 1000));
|
|
84544
|
+
return `${step} ${seconds}s`;
|
|
84545
|
+
}
|
|
84040
84546
|
// dist/builtin/workflows/src/tui/status-helpers.ts
|
|
84041
84547
|
function statusColor(status, theme) {
|
|
84042
84548
|
switch (status) {
|
|
@@ -84066,6 +84572,8 @@ function statusIcon(status) {
|
|
|
84066
84572
|
switch (status) {
|
|
84067
84573
|
case "pending":
|
|
84068
84574
|
return "○";
|
|
84575
|
+
case "queued":
|
|
84576
|
+
return "○";
|
|
84069
84577
|
case "awaiting_input":
|
|
84070
84578
|
return "?";
|
|
84071
84579
|
case "blocked":
|
|
@@ -84257,12 +84765,13 @@ function artifactRowsFor(detail) {
|
|
|
84257
84765
|
return rows;
|
|
84258
84766
|
}
|
|
84259
84767
|
function stageLinePlain(stage, now, width) {
|
|
84260
|
-
const
|
|
84768
|
+
const status = stageDisplayStatus(stage);
|
|
84769
|
+
const icon = statusIcon(status);
|
|
84261
84770
|
const dur = stageDurationString(stage, now);
|
|
84262
|
-
const activity = stageActivityString(stage, now);
|
|
84771
|
+
const activity = pooledStageStartingStep(stage) ?? stageActivityString(stage, now);
|
|
84263
84772
|
const name = truncateToWidth(`${icon} ${stage.name}`, STAGE_NAME_COL + 2, "…");
|
|
84264
84773
|
const activityText = activity ? truncateToWidth(activity, 16, "…") : undefined;
|
|
84265
|
-
const parts = [pad(name, STAGE_NAME_COL + 2), pad(
|
|
84774
|
+
const parts = [pad(name, STAGE_NAME_COL + 2), pad(status, 10)];
|
|
84266
84775
|
if (activityText)
|
|
84267
84776
|
parts.push(pad(activityText, 16));
|
|
84268
84777
|
if (dur)
|
|
@@ -84270,17 +84779,18 @@ function stageLinePlain(stage, now, width) {
|
|
|
84270
84779
|
return truncateToWidth(parts.join(""), width, "…");
|
|
84271
84780
|
}
|
|
84272
84781
|
function stageLineThemed(stage, now, theme, width) {
|
|
84273
|
-
const
|
|
84274
|
-
const
|
|
84782
|
+
const status = stageDisplayStatus(stage);
|
|
84783
|
+
const icon = statusIcon(status);
|
|
84784
|
+
const iconFg = hexToAnsi(statusColor(status, theme));
|
|
84275
84785
|
const text = hexToAnsi(theme.text);
|
|
84276
84786
|
const muted = hexToAnsi(theme.textMuted);
|
|
84277
84787
|
const dim = hexToAnsi(theme.dim);
|
|
84278
|
-
const stateFg = hexToAnsi(statusColor(
|
|
84279
|
-
const activity = stageActivityString(stage, now);
|
|
84788
|
+
const stateFg = hexToAnsi(statusColor(status, theme));
|
|
84789
|
+
const activity = pooledStageStartingStep(stage) ?? stageActivityString(stage, now);
|
|
84280
84790
|
const dur = stageDurationString(stage, now);
|
|
84281
84791
|
const nameText = truncateToWidth(stage.name, STAGE_NAME_COL, "…");
|
|
84282
84792
|
const namePad = pad(nameText, STAGE_NAME_COL);
|
|
84283
|
-
const statePad = pad(
|
|
84793
|
+
const statePad = pad(status, 10);
|
|
84284
84794
|
const activityText = activity ? truncateToWidth(activity, 16, "…") : undefined;
|
|
84285
84795
|
const activitySeg = activityText ? `${muted}${pad(activityText, 16)}${RESET}` : " ".repeat(16);
|
|
84286
84796
|
const durSeg = dur ? `${dim}${dur}${RESET}` : "";
|
|
@@ -85471,7 +85981,7 @@ function renderHeader(run, opts) {
|
|
|
85471
85981
|
skipped: 0
|
|
85472
85982
|
};
|
|
85473
85983
|
for (const s of run.stages)
|
|
85474
|
-
counts[s.status]++;
|
|
85984
|
+
counts[stageDisplayStatus(s) === "queued" ? "pending" : s.status]++;
|
|
85475
85985
|
const badges = [];
|
|
85476
85986
|
if (counts.completed > 0)
|
|
85477
85987
|
badges.push({ text: `✓ ${counts.completed}`, fg: theme.success });
|
|
@@ -87698,7 +88208,8 @@ function renderNodeCard(stage, opts) {
|
|
|
87698
88208
|
const focused = opts.focused ?? false;
|
|
87699
88209
|
const phase = opts.pulsePhase ?? 0;
|
|
87700
88210
|
const theme = opts.theme;
|
|
87701
|
-
const
|
|
88211
|
+
const status = stageDisplayStatus(stage);
|
|
88212
|
+
const borderHex = pickBorder(status, focused, phase, theme);
|
|
87702
88213
|
const bc = hexToAnsi(borderHex);
|
|
87703
88214
|
const bg = DEFAULT_BG;
|
|
87704
88215
|
const innerWidth = Math.max(2, width - 2);
|
|
@@ -87709,12 +88220,12 @@ function renderNodeCard(stage, opts) {
|
|
|
87709
88220
|
const topMiddle = `${bc}${"─".repeat(titleStart)}` + `${titleSlot}${bc}` + `${"─".repeat(Math.max(0, innerWidth - titleEnd))}`;
|
|
87710
88221
|
const top = `${bg}${bc}╭${topMiddle}╮${RESET}`;
|
|
87711
88222
|
const bottom = `${bg}${bc}╰${"─".repeat(innerWidth)}╯${RESET}`;
|
|
87712
|
-
const bodyText = stage.nodeKind === "tool" ? "durable tool" : stage.status === "blocked" ? blockedBadgeText(stage, opts.stages, innerWidth) : durationText(stage);
|
|
87713
|
-
const bodyHex = durationColor(
|
|
87714
|
-
const statusText = `${statusIcon(
|
|
88223
|
+
const bodyText = stage.nodeKind === "tool" ? "durable tool" : stage.status === "blocked" ? blockedBadgeText(stage, opts.stages, innerWidth) : pooledStageStartingText(stage) ?? durationText(stage);
|
|
88224
|
+
const bodyHex = durationColor(status, theme);
|
|
88225
|
+
const statusText = `${statusIcon(status)} ${stage.toolStatus ?? statusLabel(status)}`;
|
|
87715
88226
|
const queuedCount = queuedBadgeCount(opts.queuedMessageCount);
|
|
87716
88227
|
const statusLine = `${bg}${bc}│${RESET}` + centreColored(statusText, innerWidth, bodyHex, bg, {
|
|
87717
|
-
bold:
|
|
88228
|
+
bold: status === "running" || status === "awaiting_input"
|
|
87718
88229
|
}) + `${bg}${bc}│${RESET}`;
|
|
87719
88230
|
const durLine = `${bg}${bc}│${RESET}` + centreColored(bodyText, innerWidth, bodyHex, bg, {
|
|
87720
88231
|
bold: stage.status === "blocked"
|
|
@@ -88796,8 +89307,9 @@ function renderSwitcher(stages, state, opts) {
|
|
|
88796
89307
|
const stage = visible[i];
|
|
88797
89308
|
const idx = start + i;
|
|
88798
89309
|
const isSelected = idx === state.selectedIndex;
|
|
88799
|
-
const
|
|
88800
|
-
const
|
|
89310
|
+
const displayStatus = stageDisplayStatus(stage);
|
|
89311
|
+
const icon = statusIcon(displayStatus);
|
|
89312
|
+
const label = statusText(displayStatus);
|
|
88801
89313
|
const rowTextBudget = Math.max(8, innerWidth - 14);
|
|
88802
89314
|
const name = truncateToWidth(stage.name, rowTextBudget, "…");
|
|
88803
89315
|
if (isSelected) {
|
|
@@ -88810,7 +89322,7 @@ function renderSwitcher(stages, state, opts) {
|
|
|
88810
89322
|
const styled = `${accent}${accentFg}${BOLD}${padded}${RESET}`;
|
|
88811
89323
|
lines.push(`${border}│${RESET}${styled}${border}│${RESET}`);
|
|
88812
89324
|
} else {
|
|
88813
|
-
const iconColor = hexToAnsi(statusColor(
|
|
89325
|
+
const iconColor = hexToAnsi(statusColor(displayStatus, theme));
|
|
88814
89326
|
const status = `${dim}${label}${RESET}${panelBg}`;
|
|
88815
89327
|
const visibleRow = ` ${icon} ${name}`;
|
|
88816
89328
|
const gap = Math.max(1, innerWidth - visibleWidth(visibleRow) - visibleWidth(label) - 1);
|
|
@@ -90850,12 +91362,13 @@ function editorRuleColor(ctx, disabled, agentSession, state) {
|
|
|
90850
91362
|
}
|
|
90851
91363
|
|
|
90852
91364
|
// dist/builtin/workflows/src/tui/stage-chat-view-state.ts
|
|
90853
|
-
init_stage_ui_broker();
|
|
90854
91365
|
import {
|
|
90855
91366
|
ChatSessionHost,
|
|
90856
91367
|
classifyChatCommand
|
|
90857
91368
|
} from "@bastani/atomic";
|
|
90858
91369
|
import { Editor } from "@earendil-works/pi-tui";
|
|
91370
|
+
init_stage_ui_broker();
|
|
91371
|
+
init_store_internal();
|
|
90859
91372
|
|
|
90860
91373
|
// dist/builtin/workflows/src/tui/prompt-card-input.ts
|
|
90861
91374
|
function handlePromptCardInput(data, state, keybindings) {
|
|
@@ -91059,6 +91572,15 @@ function stageSkillAutocomplete(ctx) {
|
|
|
91059
91572
|
const handle = liveHandle(ctx);
|
|
91060
91573
|
if (!handle || isBlocked(ctx))
|
|
91061
91574
|
throw new Error("This stage chat is not editable.");
|
|
91575
|
+
if (handle.agentSession === undefined && handle.commandCatalog !== undefined) {
|
|
91576
|
+
assertComposerAdmission(ctx, handle);
|
|
91577
|
+
const catalog = handle.commandCatalog();
|
|
91578
|
+
if (catalog !== undefined)
|
|
91579
|
+
return catalog;
|
|
91580
|
+
if (handle.isSessionStarting === true)
|
|
91581
|
+
return { loading: `Loading skills: stage "${handle.stageName}" is still starting on its local worker.` };
|
|
91582
|
+
return;
|
|
91583
|
+
}
|
|
91062
91584
|
if (handle.agentSession === undefined) {
|
|
91063
91585
|
let attachment = pendingSkillAttachments.get(handle);
|
|
91064
91586
|
if (!attachment) {
|
|
@@ -91885,6 +92407,10 @@ function createChatHost(ctx, opts) {
|
|
|
91885
92407
|
throw new Error("no live handle on this stage");
|
|
91886
92408
|
await handle.ensureAttached();
|
|
91887
92409
|
const agentSession = handle.agentSession;
|
|
92410
|
+
if (!agentSession && handle.executeBash !== undefined)
|
|
92411
|
+
return handle.executeBash(request.command, request.onChunk, {
|
|
92412
|
+
excludeFromContext: request.excludeFromContext
|
|
92413
|
+
});
|
|
91888
92414
|
if (!agentSession)
|
|
91889
92415
|
throw new Error("no live agent session on this stage");
|
|
91890
92416
|
return agentSession.executeBash(request.command, request.onChunk, {
|
|
@@ -91892,16 +92418,35 @@ function createChatHost(ctx, opts) {
|
|
|
91892
92418
|
});
|
|
91893
92419
|
},
|
|
91894
92420
|
abortBash: async () => {
|
|
91895
|
-
liveHandle(ctx)
|
|
92421
|
+
const handle = liveHandle(ctx);
|
|
92422
|
+
if (handle?.agentSession)
|
|
92423
|
+
handle.agentSession.abortBash();
|
|
92424
|
+
else
|
|
92425
|
+
handle?.abortBash?.();
|
|
91896
92426
|
},
|
|
91897
92427
|
abortCompaction: async () => {
|
|
91898
|
-
liveHandle(ctx)
|
|
92428
|
+
const handle = liveHandle(ctx);
|
|
92429
|
+
if (handle?.agentSession)
|
|
92430
|
+
handle.agentSession.abortCompaction();
|
|
92431
|
+
else
|
|
92432
|
+
handle?.abortCompaction?.();
|
|
91899
92433
|
},
|
|
91900
92434
|
handleSlashCommand: async (text) => handleSlashCommand(ctx, text)
|
|
91901
92435
|
},
|
|
91902
|
-
isBashRunning: () =>
|
|
92436
|
+
isBashRunning: () => {
|
|
92437
|
+
const handle = liveHandle(ctx);
|
|
92438
|
+
return handle?.agentSession?.isBashRunning === true || handle?.isBashRunning === true;
|
|
92439
|
+
},
|
|
91903
92440
|
requestRender: opts.requestRender,
|
|
91904
92441
|
getAgentSession: () => liveHandle(ctx)?.agentSession,
|
|
92442
|
+
getSessionMessages: () => {
|
|
92443
|
+
const handle = liveHandle(ctx);
|
|
92444
|
+
return handle !== undefined && handle.agentSession === undefined ? handle.messages : undefined;
|
|
92445
|
+
},
|
|
92446
|
+
getTaskSource: () => {
|
|
92447
|
+
const handle = liveHandle(ctx);
|
|
92448
|
+
return handle?.agentSession === undefined ? handle?.taskSource?.() : undefined;
|
|
92449
|
+
},
|
|
91905
92450
|
isStreaming: () => isLiveHandleStreaming(ctx),
|
|
91906
92451
|
isPaused: () => isPaused(ctx),
|
|
91907
92452
|
isDisabled: () => isBlocked(ctx) || !liveHandle(ctx),
|
|
@@ -92106,6 +92651,28 @@ function viewLineCount(ctx) {
|
|
|
92106
92651
|
function liveHandle(ctx) {
|
|
92107
92652
|
return ctx.handle?.isDisposed === true ? undefined : ctx.handle;
|
|
92108
92653
|
}
|
|
92654
|
+
function pooledStageWaitingNotice(ctx, stage = currentStage(ctx)) {
|
|
92655
|
+
const handle = liveHandle(ctx);
|
|
92656
|
+
if (handle?.agentSession !== undefined)
|
|
92657
|
+
return;
|
|
92658
|
+
const starting = pooledStageStartingText(stage);
|
|
92659
|
+
if (handle === undefined) {
|
|
92660
|
+
if (stage?.workerExecution === undefined || isTerminalStageStatus(stage.status))
|
|
92661
|
+
return;
|
|
92662
|
+
return {
|
|
92663
|
+
label: starting === undefined ? "CONNECTING TO LOCAL WORKER" : "STARTING ON LOCAL WORKER",
|
|
92664
|
+
detail: starting ?? "opening a live view",
|
|
92665
|
+
message: "This stage runs on a local worker. Its chat opens here by itself as soon as its session is ready."
|
|
92666
|
+
};
|
|
92667
|
+
}
|
|
92668
|
+
if (!(handle.isSessionStarting ?? isPooledStageStarting(stage)))
|
|
92669
|
+
return;
|
|
92670
|
+
return {
|
|
92671
|
+
label: "STARTING ON LOCAL WORKER",
|
|
92672
|
+
detail: starting ?? "starting its session",
|
|
92673
|
+
message: "This stage's session is still starting on its local worker. Its transcript streams here once it starts, and messages you send now are delivered then."
|
|
92674
|
+
};
|
|
92675
|
+
}
|
|
92109
92676
|
function isLiveHandleStreaming(ctx) {
|
|
92110
92677
|
const handle = liveHandle(ctx);
|
|
92111
92678
|
if (!handle)
|
|
@@ -92149,11 +92716,15 @@ async function handleSlashCommand(ctx, text) {
|
|
|
92149
92716
|
if (!handle)
|
|
92150
92717
|
return false;
|
|
92151
92718
|
await handle.ensureAttached();
|
|
92152
|
-
|
|
92719
|
+
const compact = handle.agentSession !== undefined ? () => handle.agentSession.compact() : handle.compact === undefined ? undefined : () => handle.compact();
|
|
92720
|
+
if (compact === undefined)
|
|
92153
92721
|
return false;
|
|
92154
92722
|
try {
|
|
92155
|
-
await
|
|
92156
|
-
} catch {
|
|
92723
|
+
await compact();
|
|
92724
|
+
} catch (error) {
|
|
92725
|
+
if (handle.agentSession === undefined)
|
|
92726
|
+
ctx.chatHost.showWarning(error.message);
|
|
92727
|
+
}
|
|
92157
92728
|
return true;
|
|
92158
92729
|
}
|
|
92159
92730
|
case "/quit":
|
|
@@ -92325,6 +92896,22 @@ function renderPausedBody(ctx, width, budget, indicatorLines = [], renderIndicat
|
|
|
92325
92896
|
lines.length = budget;
|
|
92326
92897
|
return lines;
|
|
92327
92898
|
}
|
|
92899
|
+
function renderStartingBody(ctx, width, budget, notice, renderTranscript = (rows) => ctx.chatHost.renderBody(width, rows)) {
|
|
92900
|
+
const t = ctx.theme;
|
|
92901
|
+
const callout = [];
|
|
92902
|
+
callout.push(blankLine(width));
|
|
92903
|
+
callout.push(...bannerLines(ctx, width, "info", "○", notice.label, notice.detail));
|
|
92904
|
+
callout.push(...new Text3(paint2(notice.message, t.textMuted), 2, 0).render(width));
|
|
92905
|
+
const calloutRows = Math.min(callout.length, Math.max(0, budget));
|
|
92906
|
+
const transcriptRows = Math.max(0, budget - calloutRows);
|
|
92907
|
+
const lines = transcriptRows > 0 ? renderTranscript(transcriptRows) : [];
|
|
92908
|
+
lines.push(...callout.slice(0, calloutRows));
|
|
92909
|
+
while (lines.length < budget)
|
|
92910
|
+
lines.push(blankLine(width));
|
|
92911
|
+
if (lines.length > budget)
|
|
92912
|
+
lines.length = budget;
|
|
92913
|
+
return lines;
|
|
92914
|
+
}
|
|
92328
92915
|
function renderBlockedBody(ctx, width, budget, stage) {
|
|
92329
92916
|
const t = ctx.theme;
|
|
92330
92917
|
const upstream = stage?.blockedByStageId ?? "upstream stage";
|
|
@@ -92794,6 +93381,7 @@ class StageChatView {
|
|
|
92794
93381
|
keyLabel: () => keybindings?.getKeys?.(TUI_ACTION.altScreenBottom)?.join("/") ?? keyText2("tui.altScreen.bottom")
|
|
92795
93382
|
});
|
|
92796
93383
|
const renderTranscript = (rows) => this.chatHost.renderBody(w, rows);
|
|
93384
|
+
const waitingNotice = pooledStageWaitingNotice(ctx, stage);
|
|
92797
93385
|
if (bodyBudget <= 0) {
|
|
92798
93386
|
bodyLines = [];
|
|
92799
93387
|
} else if (promptActive) {
|
|
@@ -92803,6 +93391,8 @@ class StageChatView {
|
|
|
92803
93391
|
} else if (!readOnlyArchive && isPaused(ctx, stage)) {
|
|
92804
93392
|
reservedIndicatorLines = bodyBudget > 1 ? indicator.render(w) : [];
|
|
92805
93393
|
bodyLines = renderPausedBody(ctx, w, bodyBudget, reservedIndicatorLines, () => indicator.render(w), renderTranscript);
|
|
93394
|
+
} else if (waitingNotice !== undefined) {
|
|
93395
|
+
bodyLines = renderStartingBody(ctx, w, bodyBudget, waitingNotice, renderTranscript);
|
|
92806
93396
|
} else if (readOnlyArchive) {
|
|
92807
93397
|
reservedIndicatorLines = bodyBudget > 1 ? indicator.render(w) : [];
|
|
92808
93398
|
bodyLines = renderReadOnlyArchiveBody(ctx, w, bodyBudget, stage, reservedIndicatorLines, () => indicator.render(w), renderTranscript);
|
|
@@ -92932,6 +93522,7 @@ function attachedResolution(postMortem) {
|
|
|
92932
93522
|
// dist/builtin/workflows/src/tui/workflow-attach-pane.ts
|
|
92933
93523
|
var ENTER_TRANSITION_QUARANTINE_MS = 200;
|
|
92934
93524
|
var TERMINAL_STAGE_STATUSES = new Set(["completed", "failed", "skipped"]);
|
|
93525
|
+
var POOLED_VIEW_RETRY_MS = 1000;
|
|
92935
93526
|
|
|
92936
93527
|
class WorkflowAttachPane {
|
|
92937
93528
|
store;
|
|
@@ -92961,6 +93552,9 @@ class WorkflowAttachPane {
|
|
|
92961
93552
|
chatView = null;
|
|
92962
93553
|
attachedPromptPosition = null;
|
|
92963
93554
|
pooledAttachment;
|
|
93555
|
+
awaitingPooledView = false;
|
|
93556
|
+
pooledViewRetryAt = 0;
|
|
93557
|
+
pooledViewRetrying = false;
|
|
92964
93558
|
releaseChatHandle = null;
|
|
92965
93559
|
unsubscribeStore = null;
|
|
92966
93560
|
attachedRunId = null;
|
|
@@ -93054,7 +93648,7 @@ class WorkflowAttachPane {
|
|
|
93054
93648
|
const run = snap.runs.find((r) => r.id === runId);
|
|
93055
93649
|
return run?.name ?? "workflow";
|
|
93056
93650
|
}
|
|
93057
|
-
async _attachToStage(runId, stageId, options = { suppressInitialPromptSubmit: true }) {
|
|
93651
|
+
async _attachToStage(runId, stageId, options = { suppressInitialPromptSubmit: true }, pooledLease) {
|
|
93058
93652
|
this._flushStoreUpdate();
|
|
93059
93653
|
const generation = ++this.attachmentGeneration;
|
|
93060
93654
|
this.graphEnterQuarantineUntil = 0;
|
|
@@ -93066,7 +93660,8 @@ class WorkflowAttachPane {
|
|
|
93066
93660
|
this.attachedRunId = runId;
|
|
93067
93661
|
this.lastAttachedStageId = stageId;
|
|
93068
93662
|
const stage = this._stageSnapshot(runId, stageId);
|
|
93069
|
-
const
|
|
93663
|
+
const pooledLive = stage?.workerExecution !== undefined && !TERMINAL_STAGE_STATUSES.has(stage.status);
|
|
93664
|
+
const resolution = resolveAttachedStageHandle(this.registry, this.resolvePostMortemHandle, runId, stageId, pooledLive || pooledLease !== undefined ? () => pooledLease === undefined ? acquirePooledStage(this.store, runId, stageId) : Promise.resolve(pooledLease) : undefined);
|
|
93070
93665
|
const { handle, postMortemUnavailableReason, release, onReconnected } = resolution instanceof Promise ? await resolution : resolution;
|
|
93071
93666
|
if (generation !== this.attachmentGeneration || !this.visible) {
|
|
93072
93667
|
release?.();
|
|
@@ -93086,8 +93681,39 @@ class WorkflowAttachPane {
|
|
|
93086
93681
|
stopWatchingReconnects?.();
|
|
93087
93682
|
release();
|
|
93088
93683
|
};
|
|
93684
|
+
this.awaitingPooledView = pooledLive && release === undefined;
|
|
93685
|
+
this.pooledViewRetryAt = this.now();
|
|
93089
93686
|
this._mountChatView(runId, stageId, handle, postMortemUnavailableReason);
|
|
93090
93687
|
}
|
|
93688
|
+
_retryPooledStageView(stage) {
|
|
93689
|
+
const runId = this.attachedRunId;
|
|
93690
|
+
const stageId = this.lastAttachedStageId;
|
|
93691
|
+
if (!this.awaitingPooledView || this.pooledViewRetrying || this.chatView === null || !runId || !stageId)
|
|
93692
|
+
return;
|
|
93693
|
+
if (stage?.workerExecution === undefined || TERMINAL_STAGE_STATUSES.has(stage.status)) {
|
|
93694
|
+
this.awaitingPooledView = false;
|
|
93695
|
+
return;
|
|
93696
|
+
}
|
|
93697
|
+
const now = this.now();
|
|
93698
|
+
if (now - this.pooledViewRetryAt < POOLED_VIEW_RETRY_MS)
|
|
93699
|
+
return;
|
|
93700
|
+
this.pooledViewRetryAt = now;
|
|
93701
|
+
this.pooledViewRetrying = true;
|
|
93702
|
+
const generation = this.attachmentGeneration;
|
|
93703
|
+
acquirePooledStage(this.store, runId, stageId).then((lease) => {
|
|
93704
|
+
if (lease === undefined)
|
|
93705
|
+
return;
|
|
93706
|
+
const chat = this.chatView;
|
|
93707
|
+
if (generation !== this.attachmentGeneration || chat === null || !this.visible || this.attachedRunId !== runId || this.lastAttachedStageId !== stageId) {
|
|
93708
|
+
lease.release();
|
|
93709
|
+
return;
|
|
93710
|
+
}
|
|
93711
|
+
this.composerDrafts.capture(runId, stageId, chat._inputBuffer);
|
|
93712
|
+
this._attachToStage(runId, stageId, { suppressInitialPromptSubmit: true }, lease);
|
|
93713
|
+
}).catch(() => {}).finally(() => {
|
|
93714
|
+
this.pooledViewRetrying = false;
|
|
93715
|
+
});
|
|
93716
|
+
}
|
|
93091
93717
|
_mountChatView(runId, stageId, handle, postMortemUnavailableReason) {
|
|
93092
93718
|
let chatView;
|
|
93093
93719
|
chatView = new StageChatView({
|
|
@@ -93122,6 +93748,7 @@ class WorkflowAttachPane {
|
|
|
93122
93748
|
}
|
|
93123
93749
|
_disposeChatView() {
|
|
93124
93750
|
this.pooledAttachment = undefined;
|
|
93751
|
+
this.awaitingPooledView = false;
|
|
93125
93752
|
this.chatView?.dispose();
|
|
93126
93753
|
const release = this.releaseChatHandle;
|
|
93127
93754
|
this.releaseChatHandle = null;
|
|
@@ -93224,6 +93851,7 @@ class WorkflowAttachPane {
|
|
|
93224
93851
|
if (this.chatView !== null && stage !== undefined && stage.attached !== true)
|
|
93225
93852
|
this.store.recordStageAttached(this.attachedRunId, this.lastAttachedStageId, true);
|
|
93226
93853
|
this._followSupersedingPrompt(attachedRun, stage);
|
|
93854
|
+
this._retryPooledStageView(stage);
|
|
93227
93855
|
}
|
|
93228
93856
|
}
|
|
93229
93857
|
_followSupersedingPrompt(run, stage) {
|
|
@@ -96108,6 +96736,7 @@ async function resumeRun(runId, opts) {
|
|
|
96108
96736
|
await handle.resume();
|
|
96109
96737
|
opts?.onResumeAcknowledged?.();
|
|
96110
96738
|
}
|
|
96739
|
+
const stages = await settleResumeAcknowledgements(activeStore, expandedControlRunIds(activeStore, runId).flatMap((controlRunId) => registry.run(controlRunId).pausedStages().map((handle) => ({ controlRunId, handle }))), opts?.message);
|
|
96111
96740
|
activeStore.recordRunResumed(runId, undefined, {
|
|
96112
96741
|
source: "run_control",
|
|
96113
96742
|
...opts?.actor === undefined ? {} : { actor: opts.actor }
|
|
@@ -96116,9 +96745,9 @@ async function resumeRun(runId, opts) {
|
|
|
96116
96745
|
ok: true,
|
|
96117
96746
|
runId,
|
|
96118
96747
|
snapshot: structuredClone(activeStore.runs().find((candidate) => candidate.id === runId) ?? run),
|
|
96119
|
-
resumed: [],
|
|
96748
|
+
resumed: structuredClone([...stages.resumed]),
|
|
96120
96749
|
mode: "paused",
|
|
96121
|
-
message: `Resumed workflow runtime on ${runId}.`
|
|
96750
|
+
message: stages.failures.length === 0 ? `Resumed workflow runtime on ${runId}.` : `Resumed workflow runtime on ${runId}; failed to resume workflow stages: ${stages.failures.join("; ")}. Failed paused stages remain resumable.`
|
|
96122
96751
|
};
|
|
96123
96752
|
}
|
|
96124
96753
|
const resumed = [];
|
|
@@ -96242,7 +96871,11 @@ async function resumeRun(runId, opts) {
|
|
|
96242
96871
|
}
|
|
96243
96872
|
function isRunFullyPaused(activeStore, runId, controlRunIds) {
|
|
96244
96873
|
const runs = activeStore.runs();
|
|
96245
|
-
const
|
|
96874
|
+
const settledChild = (stage) => {
|
|
96875
|
+
const child = runs.find((candidate) => candidate.id === authoritativeWorkflowChildRunId(stage));
|
|
96876
|
+
return child !== undefined && controlRunIds.includes(child.id) && (child.status === "paused" || child.endedAt !== undefined);
|
|
96877
|
+
};
|
|
96878
|
+
const hasLiveStage = controlRunIds.some((controlRunId) => runs.find((candidate) => candidate.id === controlRunId)?.stages.some((stage) => (stage.status === "running" || stage.status === "pending") && !settledChild(stage)) === true);
|
|
96246
96879
|
return !hasLiveStage && runs.find((candidate) => candidate.id === runId)?.status === "paused";
|
|
96247
96880
|
}
|
|
96248
96881
|
async function pauseRunWithAction(runId, opts, action = "pause") {
|
|
@@ -96278,6 +96911,40 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
|
|
|
96278
96911
|
}
|
|
96279
96912
|
const controlRunIds = expandedControlRunIds(activeStore, runId);
|
|
96280
96913
|
const handles = controlRunIds.flatMap((controlRunId) => registry.run(controlRunId).stages().filter((handle) => handle.status === "running" || handle.status === "pending").map((handle) => ({ controlRunId, handle })));
|
|
96914
|
+
const pauseHandles = async () => {
|
|
96915
|
+
const paused = [];
|
|
96916
|
+
const pausedRunIds = new Set;
|
|
96917
|
+
for (const { controlRunId, handle } of handles) {
|
|
96918
|
+
await handle.pause();
|
|
96919
|
+
activeStore.recordStagePaused(controlRunId, handle.stageId);
|
|
96920
|
+
pausedRunIds.add(controlRunId);
|
|
96921
|
+
const controlRun = activeStore.runs().find((candidate) => candidate.id === controlRunId);
|
|
96922
|
+
const stage = controlRun?.stages.find((candidate) => candidate.id === handle.stageId);
|
|
96923
|
+
if (stage !== undefined)
|
|
96924
|
+
paused.push(structuredClone(stage));
|
|
96925
|
+
}
|
|
96926
|
+
for (const pausedRunId of pausedRunIds) {
|
|
96927
|
+
if (pausedRunId === runId)
|
|
96928
|
+
continue;
|
|
96929
|
+
activeStore.recordRunPaused(pausedRunId);
|
|
96930
|
+
}
|
|
96931
|
+
activeStore.recordRunPaused(runId);
|
|
96932
|
+
return paused;
|
|
96933
|
+
};
|
|
96934
|
+
const poolControl = (opts?.toolControlRegistry ?? toolControlRegistry).runControl(runId);
|
|
96935
|
+
if (poolControl !== undefined && liveWorkerStages(activeStore, controlRunIds).length > 0) {
|
|
96936
|
+
await poolControl.pause();
|
|
96937
|
+
const paused = await pauseHandles();
|
|
96938
|
+
const pausing = liveWorkerStages(activeStore, controlRunIds).filter((stage) => stage.status === "running").length;
|
|
96939
|
+
return {
|
|
96940
|
+
ok: true,
|
|
96941
|
+
runId,
|
|
96942
|
+
paused,
|
|
96943
|
+
...pausing === 0 ? {} : {
|
|
96944
|
+
message: `Run ${runId} paused and can be resumed. ${pausing} stage(s) on local workers have not confirmed the pause yet; their workers pause them on their next check.`
|
|
96945
|
+
}
|
|
96946
|
+
};
|
|
96947
|
+
}
|
|
96281
96948
|
if (handles.length === 0) {
|
|
96282
96949
|
const toolControls = opts?.toolControlRegistry ?? toolControlRegistry;
|
|
96283
96950
|
const runtimeControls = ownedRuntimeControls(activeStore, toolControls, runId);
|
|
@@ -96286,11 +96953,13 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
|
|
|
96286
96953
|
if (runtimeControls.some(({ controlRunId }) => controlRunId === runId) && [...activeRunIds].every((id) => toolControls.active(id).length === 0) && stageHandles.every((handle) => handle.status === "awaiting_input") && (stageHandles.length === 0 || !isRunFullyPaused(activeStore, runId, controlRunIds))) {
|
|
96287
96954
|
await Promise.all(runtimeControls.map(({ handle }) => handle.pause()));
|
|
96288
96955
|
activeStore.recordRunPaused(runId);
|
|
96956
|
+
const durable = toolControls.runControl(runId)?.durableBackend?.getWorkflow(runId);
|
|
96957
|
+
const resumableElsewhere = durable !== undefined && (durable.completedCheckpoints > 0 || durable.pendingPrompts > 0);
|
|
96289
96958
|
return {
|
|
96290
96959
|
ok: true,
|
|
96291
96960
|
runId,
|
|
96292
96961
|
paused: [],
|
|
96293
|
-
message: `Run ${runId} paused. Resume with /workflow resume on this live process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume. Cross-process resume requires durable checkpoint or pending prompt progress.`
|
|
96962
|
+
message: resumableElsewhere ? `Run ${runId} paused. Resume it with /workflow resume ${runId}, here or, once this process has exited, from a new Atomic process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume.` : `Run ${runId} paused. Resume with /workflow resume on this live process; untracked initialization or workflow code may still finish, but further workflow steps and completion wait for resume. Cross-process resume requires durable checkpoint or pending prompt progress.`
|
|
96294
96963
|
};
|
|
96295
96964
|
}
|
|
96296
96965
|
const controlRuns = activeStore.runs().filter((candidate) => controlRunIds.includes(candidate.id));
|
|
@@ -96307,24 +96976,11 @@ async function pauseRunWithAction(runId, opts, action = "pause") {
|
|
|
96307
96976
|
reason: isRunFullyPaused(activeStore, runId, controlRunIds) ? "already_paused" : "no_active_stages"
|
|
96308
96977
|
};
|
|
96309
96978
|
}
|
|
96310
|
-
|
|
96311
|
-
|
|
96312
|
-
|
|
96313
|
-
|
|
96314
|
-
|
|
96315
|
-
pausedRunIds.add(controlRunId);
|
|
96316
|
-
const controlRun = activeStore.runs().find((candidate) => candidate.id === controlRunId);
|
|
96317
|
-
const stage = controlRun?.stages.find((candidate) => candidate.id === handle.stageId);
|
|
96318
|
-
if (stage !== undefined)
|
|
96319
|
-
paused.push(structuredClone(stage));
|
|
96320
|
-
}
|
|
96321
|
-
for (const pausedRunId of pausedRunIds) {
|
|
96322
|
-
if (pausedRunId === runId)
|
|
96323
|
-
continue;
|
|
96324
|
-
activeStore.recordRunPaused(pausedRunId);
|
|
96325
|
-
}
|
|
96326
|
-
activeStore.recordRunPaused(runId);
|
|
96327
|
-
return { ok: true, runId, paused };
|
|
96979
|
+
return { ok: true, runId, paused: await pauseHandles() };
|
|
96980
|
+
}
|
|
96981
|
+
function liveWorkerStages(activeStore, runIds) {
|
|
96982
|
+
const ids = new Set(runIds);
|
|
96983
|
+
return activeStore.runs().filter((run) => ids.has(run.id) && run.endedAt === undefined).flatMap((run) => run.stages.filter((stage) => stage.workerExecution !== undefined && (stage.status === "running" || stage.status === "pending")));
|
|
96328
96984
|
}
|
|
96329
96985
|
async function pauseRun(runId, opts) {
|
|
96330
96986
|
if (opts?.stageId === undefined) {
|
|
@@ -98071,17 +98727,17 @@ async function openCompletedDurableWorkflow(workflowId, deps, catalog) {
|
|
|
98071
98727
|
const selectedStore = await stageSessionStore({ sessionStore: deps.sessionStore ?? deps.adapters?.sessionStore, cwd: deps.cwd }, deps.defaultSessionDir);
|
|
98072
98728
|
const resolved = await resolveCompletedWorkflow(workflowId, deps.durableBackend, catalog, selectedStore);
|
|
98073
98729
|
if (resolved.kind === "not_found") {
|
|
98074
|
-
return
|
|
98730
|
+
return failure3("not_found", `No completed durable workflow found for id: ${workflowId}`);
|
|
98075
98731
|
}
|
|
98076
98732
|
if (resolved.kind === "malformed") {
|
|
98077
|
-
return
|
|
98733
|
+
return failure3("malformed", resolved.message);
|
|
98078
98734
|
}
|
|
98079
98735
|
if (resolved.kind === "stale") {
|
|
98080
|
-
return
|
|
98736
|
+
return failure3("stale", `Completed workflow ${resolved.entry.workflowId} is stale or missing durable checkpoint/session data and cannot be opened.`);
|
|
98081
98737
|
}
|
|
98082
98738
|
const existing = deps.store.runs().find((run) => run.id === resolved.snapshot.id);
|
|
98083
98739
|
if (existing !== undefined && existing.status !== "completed") {
|
|
98084
|
-
return
|
|
98740
|
+
return failure3("active", `Workflow ${resolved.snapshot.id} is already active in this session; attach with /workflow connect ${resolved.snapshot.id} instead.`);
|
|
98085
98741
|
}
|
|
98086
98742
|
const snapshots = await completedWorkflowRunSnapshots(deps.durableBackend, resolved.entry, selectedStore);
|
|
98087
98743
|
const snapshot = snapshots.find((run) => run.id === resolved.snapshot.id) ?? resolved.snapshot;
|
|
@@ -98102,7 +98758,7 @@ async function openCompletedDurableWorkflow(workflowId, deps, catalog) {
|
|
|
98102
98758
|
message: `Opened completed durable workflow "${snapshot.name}" (${snapshot.id}) for read-only inspection${hasReopenableStage ? " and follow-up chat" : ""}.`
|
|
98103
98759
|
};
|
|
98104
98760
|
}
|
|
98105
|
-
function
|
|
98761
|
+
function failure3(reason, message) {
|
|
98106
98762
|
return { ok: false, reason, message };
|
|
98107
98763
|
}
|
|
98108
98764
|
function registerCompletedChatHandles(snapshot, deps, sessionStore) {
|
|
@@ -103669,7 +104325,7 @@ function errorOptions(runId, cause, failedRuns) {
|
|
|
103669
104325
|
...failedRuns === undefined ? {} : { failedRuns }
|
|
103670
104326
|
};
|
|
103671
104327
|
}
|
|
103672
|
-
function
|
|
104328
|
+
function failure4(code, message, runId, cause, failedRuns) {
|
|
103673
104329
|
const options = errorOptions(runId, cause, failedRuns);
|
|
103674
104330
|
switch (code) {
|
|
103675
104331
|
case "run_not_found":
|
|
@@ -103692,7 +104348,7 @@ function failure3(code, message, runId, cause, failedRuns) {
|
|
|
103692
104348
|
}
|
|
103693
104349
|
function translateThrown(error, runId) {
|
|
103694
104350
|
const code = classifyControlError(error);
|
|
103695
|
-
return code === "owned_elsewhere" || code === "storage_unavailable" ?
|
|
104351
|
+
return code === "owned_elsewhere" || code === "storage_unavailable" ? failure4(code, error.message, runId, error) : error;
|
|
103696
104352
|
}
|
|
103697
104353
|
function controlStatus(status, runId) {
|
|
103698
104354
|
switch (status) {
|
|
@@ -103704,15 +104360,15 @@ function controlStatus(status, runId) {
|
|
|
103704
104360
|
case "cancelled":
|
|
103705
104361
|
return status;
|
|
103706
104362
|
default:
|
|
103707
|
-
throw
|
|
104363
|
+
throw failure4("control_failed", `Unexpected workflow control status: ${status}`, runId);
|
|
103708
104364
|
}
|
|
103709
104365
|
}
|
|
103710
104366
|
function controlOutcome(result) {
|
|
103711
104367
|
if (result.action !== "pause" && result.action !== "quit" && result.action !== "resume") {
|
|
103712
|
-
throw
|
|
104368
|
+
throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
|
|
103713
104369
|
}
|
|
103714
104370
|
if ("code" in result && result.code !== undefined) {
|
|
103715
|
-
throw
|
|
104371
|
+
throw failure4(result.code, result.message, result.runId, undefined, "failedRuns" in result ? result.failedRuns : undefined);
|
|
103716
104372
|
}
|
|
103717
104373
|
return { ...result, status: controlStatus(result.status, result.runId) };
|
|
103718
104374
|
}
|
|
@@ -103722,7 +104378,7 @@ function rejectUnknownRunPrefix(store, runId) {
|
|
|
103722
104378
|
return;
|
|
103723
104379
|
const resolution = resolveRunIdTarget(target, store.runs().map((run) => run.id));
|
|
103724
104380
|
if (resolution.kind === "not_found") {
|
|
103725
|
-
throw
|
|
104381
|
+
throw failure4("run_not_found", `Run not found: ${target} (no run in this session has that prefix; pass the full run id for a run recorded elsewhere).`, runId);
|
|
103726
104382
|
}
|
|
103727
104383
|
}
|
|
103728
104384
|
function createSessionRunControl(host) {
|
|
@@ -103744,7 +104400,7 @@ function createSessionRunControl(host) {
|
|
|
103744
104400
|
...filter?.status === undefined ? {} : { statusFilter: filter.status }
|
|
103745
104401
|
});
|
|
103746
104402
|
if (result.action !== "status")
|
|
103747
|
-
throw
|
|
104403
|
+
throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, undefined);
|
|
103748
104404
|
const catalog = filter?.includeDurable ? await host.catalog?.() : undefined;
|
|
103749
104405
|
const cwd = host.context()?.cwd;
|
|
103750
104406
|
const canonicalPaths = new Map;
|
|
@@ -103798,9 +104454,9 @@ function createSessionRunControl(host) {
|
|
|
103798
104454
|
async getRun(runId) {
|
|
103799
104455
|
const result = await request({ action: "status", runId });
|
|
103800
104456
|
if (result.action !== "statusDetail")
|
|
103801
|
-
throw
|
|
104457
|
+
throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, runId);
|
|
103802
104458
|
if ("error" in result)
|
|
103803
|
-
throw
|
|
104459
|
+
throw failure4("run_not_found", result.error, runId);
|
|
103804
104460
|
return result.detail;
|
|
103805
104461
|
},
|
|
103806
104462
|
async getStages(runId, filter) {
|
|
@@ -103810,9 +104466,9 @@ function createSessionRunControl(host) {
|
|
|
103810
104466
|
...filter?.status === undefined ? {} : { statusFilter: filter.status }
|
|
103811
104467
|
});
|
|
103812
104468
|
if (result.action !== "stages")
|
|
103813
|
-
throw
|
|
104469
|
+
throw failure4("control_failed", `Unexpected workflow result: ${result.action}`, runId);
|
|
103814
104470
|
if (result.error !== undefined)
|
|
103815
|
-
throw
|
|
104471
|
+
throw failure4("run_not_found", result.error, runId);
|
|
103816
104472
|
return result.stages;
|
|
103817
104473
|
},
|
|
103818
104474
|
async pause(runId, options) {
|
|
@@ -103835,7 +104491,7 @@ function createSessionRunControl(host) {
|
|
|
103835
104491
|
} catch (error) {
|
|
103836
104492
|
const cause = error instanceof Error ? error : new Error(String(error));
|
|
103837
104493
|
const code = error instanceof JournalError ? "storage_unavailable" : classifyControlError(error);
|
|
103838
|
-
throw
|
|
104494
|
+
throw failure4(code, `Run ${runId} did not settle cleanly: ${describeErrorWithCauses(error)}`, runId, cause);
|
|
103839
104495
|
}
|
|
103840
104496
|
},
|
|
103841
104497
|
async resume(runId, options) {
|
|
@@ -104421,7 +105077,8 @@ function renderStagesToolContent(result) {
|
|
|
104421
105077
|
}
|
|
104422
105078
|
lines.push("stages:");
|
|
104423
105079
|
result.stages.forEach((stage, index) => {
|
|
104424
|
-
|
|
105080
|
+
const starting = pooledStageStartingText(stage);
|
|
105081
|
+
lines.push(`[${index + 1}] ${stage.name} (${stage.id}) ${stageDisplayStatus(stage)}${starting === undefined ? "" : ` (starting on its local worker: ${starting})`}`);
|
|
104425
105082
|
if (stage.startup)
|
|
104426
105083
|
lines.push(formatStageStartup(stage.startup));
|
|
104427
105084
|
if (stage.workerExecution)
|