@atolis-hq/wake 0.4.73 → 0.4.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/composition-root.js +2 -10
- package/dist/src/bootstrap/runner-tool-notices.d.ts +3 -0
- package/dist/src/bootstrap/runner-tool-notices.js +31 -0
- package/dist/src/bootstrap/surface-api-trigger-applications.js +38 -4
- package/dist/src/bootstrap/surface-api-work-applications.js +5 -3
- package/dist/src/bootstrap/surface-cli-applications.js +4 -1
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/activation-scheduler-ports.d.ts +2 -1
- package/dist/src/control-plane/application/activation-scheduler.js +14 -2
- package/dist/src/control-plane/application/advance-once-dispatch.js +1 -0
- package/dist/src/control-plane/application/trigger-inspection.d.ts +32 -0
- package/dist/src/control-plane/application/trigger-inspection.js +14 -0
- package/dist/src/control-plane/application/trigger-rules.js +3 -0
- package/dist/src/control-plane/application/trigger-service.d.ts +1 -0
- package/dist/src/control-plane/application/trigger-service.js +5 -5
- package/dist/src/execution/application/recovery-service.d.ts +2 -1
- package/dist/src/execution/application/recovery-service.js +20 -11
- package/dist/src/execution/contracts/config.d.ts +2 -1
- package/dist/src/execution/contracts/config.js +6 -5
- package/dist/src/execution/contracts/events.js +1 -1
- package/dist/src/execution/contracts/vocabulary.d.ts +12 -0
- package/dist/src/execution/contracts/vocabulary.js +18 -0
- package/dist/src/execution/index.d.ts +1 -0
- package/dist/src/execution/index.js +1 -0
- package/dist/src/execution/infrastructure/local-process-inspector.d.ts +4 -0
- package/dist/src/execution/infrastructure/local-process-inspector.js +85 -0
- package/dist/src/execution/infrastructure/process-execution.d.ts +1 -0
- package/dist/src/execution/infrastructure/process-execution.js +6 -1
- package/dist/src/execution/infrastructure/runners/claude.js +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.js +14 -7
- package/dist/src/orchestration/application/advance-workflow.d.ts +7 -8
- package/dist/src/orchestration/application/advance-workflow.js +21 -36
- package/dist/src/orchestration/application/operator-retry-attribution.d.ts +3 -0
- package/dist/src/orchestration/application/operator-retry-attribution.js +7 -0
- package/dist/src/orchestration/application/orchestration-service.d.ts +4 -1
- package/dist/src/orchestration/application/orchestration-service.js +4 -1
- package/dist/src/orchestration/application/resolve-execution-failure.d.ts +14 -0
- package/dist/src/orchestration/application/resolve-execution-failure.js +47 -0
- package/dist/src/surfaces/api/contracts/transport-values.d.ts +5 -1
- package/dist/src/surfaces/api/contracts/transport-values.js +13 -0
- package/dist/src/surfaces/api/contracts/triggers.d.ts +13 -1
- package/dist/src/surfaces/web-assets/assets/index-QnREfUBG.js +1192 -0
- package/dist/src/surfaces/web-assets/index.html +1 -1
- package/package.json +1 -1
- package/dist/src/surfaces/web-assets/assets/index-CxURzTmR.js +0 -1192
|
@@ -3,7 +3,7 @@ import { createPullRequestService } from '../activities/index.js';
|
|
|
3
3
|
import { ArtifactContextService, FileArtifactMcpSessionStore, FileArtifactStore, OperatorInstructionsService, createArtifactService, } from '../artifacts/index.js';
|
|
4
4
|
import { DispatchPolicy, createActivationScheduler, createActivationSchedulerSubscriber, createControlPlaneService, createDurableRunnerIneligibility, createRunnerControlService, } from '../control-plane/index.js';
|
|
5
5
|
import { createConversationService } from '../conversations/index.js';
|
|
6
|
-
import { ExecutionCancellationReason,
|
|
6
|
+
import { ExecutionCancellationReason, GitWorkspaceProvider, RecoveryService, TranscriptStore, createExecutionService, createLocalProcessInspector, createRunnerMemoryProfileDecorator, } from '../execution/index.js';
|
|
7
7
|
import { createGitHubAgentContextReader, gitHubProviderDefinition, resolveGitHubResourceUrl, } from '../integrations/github/index.js';
|
|
8
8
|
import { SystemClock, UlidIdGenerator } from '../kernel/index.js';
|
|
9
9
|
import { compileWorkflow, createOrchestrationService } from '../orchestration/index.js';
|
|
@@ -126,15 +126,7 @@ export async function createCompositionRoot(wakeRoot, options = {}) {
|
|
|
126
126
|
orchestration.setWatchChildCancellation({
|
|
127
127
|
cancelSupersededWatchChildren: (workflowInstanceIds) => execution.cancelActive(workflowInstanceIds, ExecutionCancellationReason.WorkflowSuperseded),
|
|
128
128
|
});
|
|
129
|
-
const recovery = new RecoveryService(journal, clock,
|
|
130
|
-
async inspect() {
|
|
131
|
-
// Unknown external work follows the safe ambiguity path until runners expose inspection.
|
|
132
|
-
return {
|
|
133
|
-
kind: ExternalExecutionState.Unknown,
|
|
134
|
-
reason: 'External execution inspection is not configured for this runtime',
|
|
135
|
-
};
|
|
136
|
-
},
|
|
137
|
-
}, activities, config.execution, orchestration);
|
|
129
|
+
const recovery = new RecoveryService(journal, clock, createLocalProcessInspector(), activities, config.execution, orchestration);
|
|
138
130
|
const controlPlane = createControlPlaneService({ journal, clock, ids });
|
|
139
131
|
const isRuntimePaused = async () => (await controlPlane.isDispatchPaused()) || (await maintenance.read()) !== null;
|
|
140
132
|
const runnerControls = createRunnerControlService({
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { BuiltInActivityName, agentActivityDefinition } from '../activities/index.js';
|
|
2
|
+
import { RunnerKind } from '../execution/index.js';
|
|
3
|
+
export function runnerToolNotices(config, templates = new Map()) {
|
|
4
|
+
const notices = [];
|
|
5
|
+
for (const [workflowName, workflow] of Object.entries(config.orchestration.workflows)) {
|
|
6
|
+
for (const [stageName, stage] of Object.entries(workflow.stages)) {
|
|
7
|
+
if (stage.activity !== BuiltInActivityName.Agent)
|
|
8
|
+
continue;
|
|
9
|
+
if (!hasToolList(stage.with, templates))
|
|
10
|
+
continue;
|
|
11
|
+
const pool = stage.execution?.runnerPool ?? config.execution.defaultRunnerPool;
|
|
12
|
+
const codexRunners = (config.execution.runnerPools[pool] ?? []).filter((runner) => config.execution.agentRunners?.[runner]?.kind === RunnerKind.CodexCli);
|
|
13
|
+
for (const runner of codexRunners) {
|
|
14
|
+
notices.push(`Workflow "${workflowName}" stage "${stageName}": allowedTools is advisory and is not enforced by Codex runner "${runner}" (pool "${pool}").`);
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
return notices;
|
|
19
|
+
}
|
|
20
|
+
function hasToolList(input, templates) {
|
|
21
|
+
const parsed = agentActivityDefinition.inputSchema.safeParse(input);
|
|
22
|
+
if (!parsed.success)
|
|
23
|
+
return false;
|
|
24
|
+
const stage = parsed.data;
|
|
25
|
+
if (stage.template !== undefined && !templates.has(stage.template))
|
|
26
|
+
return false;
|
|
27
|
+
const templateTools = stage.template === undefined
|
|
28
|
+
? undefined
|
|
29
|
+
: templates.get(stage.template)?.frontmatter.allowedTools;
|
|
30
|
+
return (stage.allowedTools ?? templateTools ?? []).length > 0;
|
|
31
|
+
}
|
|
@@ -2,7 +2,9 @@ import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { OperatorInstructionsConflict, operatorInstructionsProjection, } from '../artifacts/index.js';
|
|
4
4
|
import { CronTriggerPolicy } from '../control-plane/index.js';
|
|
5
|
+
import { selectOperatorRetryTarget } from '../orchestration/index.js';
|
|
5
6
|
import { ApiCommandStatus, toWorkItemKey, TriggerStatus, } from '../surfaces/index.js';
|
|
7
|
+
import { workItemId, WorkStatus } from '../work/index.js';
|
|
6
8
|
export function createTriggerApplications(root, now) {
|
|
7
9
|
async function inspect() {
|
|
8
10
|
const view = await root.triggers.get();
|
|
@@ -14,7 +16,10 @@ export function createTriggerApplications(root, now) {
|
|
|
14
16
|
...(view.diagnostics ?? []).map((entry) => entry.triggerId),
|
|
15
17
|
]);
|
|
16
18
|
const asOf = now();
|
|
17
|
-
return {
|
|
19
|
+
return {
|
|
20
|
+
data: await Promise.all([...ids].sort().map((id) => present(id, view, asOf))),
|
|
21
|
+
meta: { asOf },
|
|
22
|
+
};
|
|
18
23
|
}
|
|
19
24
|
async function list() {
|
|
20
25
|
const result = await inspect();
|
|
@@ -29,16 +34,19 @@ export function createTriggerApplications(root, now) {
|
|
|
29
34
|
}));
|
|
30
35
|
return { ...result, orphanedInstructions };
|
|
31
36
|
}
|
|
32
|
-
function present(id, view, asOf) {
|
|
37
|
+
async function present(id, view, asOf) {
|
|
33
38
|
const definition = root.config.controlPlane.triggers.find((config) => config.id === id) ?? null;
|
|
34
39
|
const invocations = view.invocations
|
|
35
40
|
.filter((entry) => entry.claim.triggerId === id)
|
|
36
41
|
.sort((a, b) => b.claimedAt.localeCompare(a.claimedAt))
|
|
37
42
|
.map((entry) => ({ ...entry, workItemKey: toWorkItemKey(entry.claim.workItemId) }));
|
|
38
43
|
const pauseReason = view.pauses[id] ?? null;
|
|
39
|
-
const
|
|
44
|
+
const coordinationKey = view.coordinationKeys[id] ?? id;
|
|
45
|
+
const activeInvocation = await inspectHolder(coordinationKey, view);
|
|
46
|
+
const running = activeInvocation !== null;
|
|
40
47
|
return {
|
|
41
48
|
triggerId: id,
|
|
49
|
+
activeInvocation,
|
|
42
50
|
definition,
|
|
43
51
|
paused: pauseReason !== null,
|
|
44
52
|
pauseReason,
|
|
@@ -52,11 +60,37 @@ export function createTriggerApplications(root, now) {
|
|
|
52
60
|
nextOccurrence: definition?.on.cron === undefined
|
|
53
61
|
? null
|
|
54
62
|
: new CronTriggerPolicy().nextOccurrence(definition.on.cron, asOf),
|
|
55
|
-
pending: Object.values(view.coalesced)
|
|
63
|
+
pending: Object.values(view.coalesced)
|
|
64
|
+
.filter((entry) => entry.coordinationKey === coordinationKey)
|
|
65
|
+
.map((entry) => ({
|
|
66
|
+
...entry,
|
|
67
|
+
activeWorkItemId: activeInvocation?.activeWorkItemId ?? null,
|
|
68
|
+
})),
|
|
56
69
|
latestSkip: view.diagnostics?.find((entry) => entry.triggerId === id) ?? null,
|
|
57
70
|
invocations,
|
|
58
71
|
};
|
|
59
72
|
}
|
|
73
|
+
async function inspectHolder(coordinationKey, view) {
|
|
74
|
+
const holder = view.invocations.find((entry) => entry.result === undefined && entry.claim.coordinationKey === coordinationKey);
|
|
75
|
+
if (holder === undefined)
|
|
76
|
+
return null;
|
|
77
|
+
const workflows = (await root.orchestration.listAll()).filter((entry) => entry.workItemId === holder.claim.workItemId);
|
|
78
|
+
const workflow = workflows.find((entry) => entry.workflowInstanceId === holder.claim.workflowInstanceId);
|
|
79
|
+
const work = await root.work.get(workItemId(holder.claim.workItemId));
|
|
80
|
+
return {
|
|
81
|
+
activeWorkItemId: holder.claim.workItemId,
|
|
82
|
+
workItemKey: toWorkItemKey(holder.claim.workItemId),
|
|
83
|
+
triggerId: holder.claim.triggerId,
|
|
84
|
+
workflowInstanceId: holder.claim.workflowInstanceId,
|
|
85
|
+
workflowStatus: workflow?.status ?? null,
|
|
86
|
+
blockReason: workflow?.blockReason ?? null,
|
|
87
|
+
retryEligible: work !== null &&
|
|
88
|
+
work.deleted !== true &&
|
|
89
|
+
!work.frozen &&
|
|
90
|
+
work.state === WorkStatus.Open &&
|
|
91
|
+
selectOperatorRetryTarget(workflows) !== undefined,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
60
94
|
async function command(id, request, pause) {
|
|
61
95
|
if (!(await inspect()).data.some((entry) => entry.triggerId === id))
|
|
62
96
|
return {
|
|
@@ -85,15 +85,17 @@ export function createSurfaceWorkApplications(root, now) {
|
|
|
85
85
|
const work = await root.work.get(id);
|
|
86
86
|
if (work === null)
|
|
87
87
|
return retryIneligible('Work item was not found');
|
|
88
|
+
const workflows = (await root.orchestration.listAll()).filter((workflow) => workflow.workItemId === id);
|
|
89
|
+
if (!workflows.some((workflow) => workflow.parentWorkflowInstanceId === undefined))
|
|
90
|
+
return retryIneligible('Work item has no primary workflow');
|
|
91
|
+
if (workflows.some((workflow) => workflow.operatorRetryCommandIds.includes(commandContext(command.idempotencyKey, now).commandId)))
|
|
92
|
+
return accepted(command.idempotencyKey, now());
|
|
88
93
|
if (work.deleted === true)
|
|
89
94
|
return retryIneligible('Work item is deleted');
|
|
90
95
|
if (work.frozen)
|
|
91
96
|
return retryIneligible('Work item is frozen');
|
|
92
97
|
if (work.state !== WorkStatus.Open)
|
|
93
98
|
return retryIneligible('Work item is not open');
|
|
94
|
-
const workflows = (await root.orchestration.listAll()).filter((workflow) => workflow.workItemId === id);
|
|
95
|
-
if (!workflows.some((workflow) => workflow.parentWorkflowInstanceId === undefined))
|
|
96
|
-
return retryIneligible('Work item has no primary workflow');
|
|
97
99
|
const target = selectOperatorRetryTarget(workflows);
|
|
98
100
|
if (target === undefined)
|
|
99
101
|
return retryIneligible('Workflow is not retry eligible');
|
|
@@ -19,6 +19,7 @@ import { createNpmUpdatePort } from './npm-update-port.js';
|
|
|
19
19
|
import { runtimeProjectionDefinitions } from './projection-runtime.js';
|
|
20
20
|
import { createRunnerRegistry } from './runner-registry.js';
|
|
21
21
|
import { createOneShotRunnerAdvance, createResidentRunnerAdvance, withFinalProjectionCatchUp, } from './runner-tick-adapter.js';
|
|
22
|
+
import { runnerToolNotices } from './runner-tool-notices.js';
|
|
22
23
|
import { createSelfUpdateApplication, } from './self-update-application.js';
|
|
23
24
|
import { createSelfUpdateFailureLog } from './self-update-failure-log.js';
|
|
24
25
|
import { createSourceUpdatePort } from './source-update-port.js';
|
|
@@ -660,6 +661,7 @@ function spawnDocker(arguments_, onChunk, cwd, options) {
|
|
|
660
661
|
async function doctorDiagnostics(root) {
|
|
661
662
|
const failures = [];
|
|
662
663
|
const notices = [];
|
|
664
|
+
const templates = new Map();
|
|
663
665
|
try {
|
|
664
666
|
createRunnerRegistry(root.config.execution, root.fakeScenarios).resolve(root.config.execution.defaultRunnerPool);
|
|
665
667
|
}
|
|
@@ -674,12 +676,13 @@ async function doctorDiagnostics(root) {
|
|
|
674
676
|
}
|
|
675
677
|
for (const name of referencedPromptTemplateNames(root.config.orchestration.workflows)) {
|
|
676
678
|
try {
|
|
677
|
-
await loadPromptTemplate(root.paths.wakeRoot, name);
|
|
679
|
+
templates.set(name, await loadPromptTemplate(root.paths.wakeRoot, name));
|
|
678
680
|
}
|
|
679
681
|
catch (error) {
|
|
680
682
|
failures.push(`prompt template "${name}" is not readable: ${error.message}`);
|
|
681
683
|
}
|
|
682
684
|
}
|
|
685
|
+
notices.push(...runnerToolNotices(root.config, templates));
|
|
683
686
|
await checkProviders(root, failures, notices);
|
|
684
687
|
await checkMaintenanceLease(root, failures);
|
|
685
688
|
notices.push(...(await dockerSandboxHealthNotices(root)));
|
|
@@ -124,6 +124,6 @@ export function resolveWakeVersion(options = {}) {
|
|
|
124
124
|
return `g${headHash.slice(0, 7)}`;
|
|
125
125
|
return '0.1.0-dev';
|
|
126
126
|
}
|
|
127
|
-
export const wakeVersion = "v0.4.
|
|
127
|
+
export const wakeVersion = "v0.4.75";
|
|
128
128
|
/** The registry-installable version used by packaged sandbox images. */
|
|
129
129
|
export const wakePackageVersion = packageVersion(defaultRepoRoot) ?? wakeVersion;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CommandContext } from '@atolis-hq/eventing';
|
|
2
|
-
import type { RunView, WorkspaceRecovery } from '../../execution/index.js';
|
|
2
|
+
import type { ExecutionFailureCode, RunView, WorkspaceRecovery } from '../../execution/index.js';
|
|
3
3
|
import type { IdGenerator } from '../../kernel/index.js';
|
|
4
4
|
import type { ActivityActivationView, WorkflowInstanceView } from '../../orchestration/index.js';
|
|
5
5
|
import type { ResourceService } from '../../resources/index.js';
|
|
@@ -25,6 +25,7 @@ export interface OrchestrationPort {
|
|
|
25
25
|
readonly activationId: string;
|
|
26
26
|
readonly runId: string;
|
|
27
27
|
readonly reason: string;
|
|
28
|
+
readonly failureKind?: ExecutionFailureCode;
|
|
28
29
|
}, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
29
30
|
retryRunnerQuotaFailure?(workflowInstanceId: string, input: {
|
|
30
31
|
readonly activationId: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* eslint-disable complexity, max-lines-per-function */
|
|
2
2
|
import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
3
|
-
import { RunStatus } from '../../execution/index.js';
|
|
3
|
+
import { ExecutionFailureCode, RunStatus } from '../../execution/index.js';
|
|
4
4
|
import { isAmbiguityResolutionBlock, WorkflowStatus, } from '../../orchestration/index.js';
|
|
5
5
|
import { WorkStatus } from '../../work/index.js';
|
|
6
6
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
@@ -134,11 +134,23 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
134
134
|
],
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
|
-
await orchestration.resolveExecutionFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
137
|
+
const resolved = await orchestration.resolveExecutionFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
138
138
|
activationId: recovery.item.activation.activationId,
|
|
139
139
|
runId: recovery.run.runId,
|
|
140
140
|
reason: recovery.run.failure?.message ?? 'execution failed',
|
|
141
|
+
...(recovery.run.failure === undefined
|
|
142
|
+
? {}
|
|
143
|
+
: { failureKind: recovery.run.failure.kind }),
|
|
141
144
|
}, context(recovery.run.runId));
|
|
145
|
+
if (recovery.run.failure?.kind === ExecutionFailureCode.Interrupted &&
|
|
146
|
+
resolved != null &&
|
|
147
|
+
resolved.status !== WorkflowStatus.Blocked)
|
|
148
|
+
return {
|
|
149
|
+
kind: 'progressed',
|
|
150
|
+
dispatched: [
|
|
151
|
+
{ activationId: recovery.item.activation.activationId, runId: recovery.run.runId },
|
|
152
|
+
],
|
|
153
|
+
};
|
|
142
154
|
return {
|
|
143
155
|
kind: WorkflowStatus.Blocked,
|
|
144
156
|
workflowInstanceId: recovery.item.workflow.workflowInstanceId,
|
|
@@ -150,6 +150,7 @@ export async function runDispatchLoop(pending, ctx) {
|
|
|
150
150
|
activationId: selected.activation.activationId,
|
|
151
151
|
runId: run.runId,
|
|
152
152
|
reason: run.failure?.message ?? 'execution failed',
|
|
153
|
+
...(run.failure === undefined ? {} : { failureKind: run.failure.kind }),
|
|
153
154
|
}, ctx.commandContext(run.runId));
|
|
154
155
|
if (run.status !== RunStatus.Succeeded && !isActiveRunStatus(run.status)) {
|
|
155
156
|
stopReason = {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { TriggerConfig } from '../contracts/config.js';
|
|
2
|
+
import type { TriggerSkipDiagnostic } from '../contracts/trigger-diagnostics.js';
|
|
3
|
+
import type { TriggerInvocationView, TriggerView } from './trigger-projection.js';
|
|
4
|
+
export declare function inspectTriggers(view: TriggerView, invocations: readonly TriggerInvocationView[], diagnostics: readonly TriggerSkipDiagnostic[], configurations: readonly TriggerConfig[], keyOf: (id: string) => string): {
|
|
5
|
+
invocations: readonly TriggerInvocationView[];
|
|
6
|
+
diagnostics: readonly {
|
|
7
|
+
triggerId: string;
|
|
8
|
+
source: {
|
|
9
|
+
kind: "manual";
|
|
10
|
+
commandId: string;
|
|
11
|
+
operator: string;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "event";
|
|
14
|
+
eventId: string;
|
|
15
|
+
eventType: string;
|
|
16
|
+
} | {
|
|
17
|
+
kind: "cron";
|
|
18
|
+
slot: string;
|
|
19
|
+
};
|
|
20
|
+
occurredAt: string;
|
|
21
|
+
reason: "paused" | "own-scope" | "removed" | "dispatch-paused" | "capacity-unavailable" | "pending-source-limit";
|
|
22
|
+
}[];
|
|
23
|
+
coordinationKeys: {
|
|
24
|
+
[k: string]: string;
|
|
25
|
+
};
|
|
26
|
+
lastStarts: Readonly<Record<string, string>>;
|
|
27
|
+
pauses: Readonly<Record<string, string>>;
|
|
28
|
+
requests: readonly import("../index.js").ControlEventPayloads["control-plane.trigger-request-recorded"][];
|
|
29
|
+
coalesced: Readonly<Record<string, import("../index.js").ControlEventPayloads["control-plane.trigger-request-recorded"] & {
|
|
30
|
+
readonly sources: readonly import("../index.js").TriggerSource[];
|
|
31
|
+
}>>;
|
|
32
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function inspectTriggers(view, invocations, diagnostics, configurations, keyOf) {
|
|
2
|
+
const ids = new Set([
|
|
3
|
+
...configurations.map((config) => config.id),
|
|
4
|
+
...view.requests.map((request) => request.triggerId),
|
|
5
|
+
...invocations.map((invocation) => invocation.claim.triggerId),
|
|
6
|
+
...Object.keys(view.pauses),
|
|
7
|
+
]);
|
|
8
|
+
return {
|
|
9
|
+
...view,
|
|
10
|
+
invocations,
|
|
11
|
+
diagnostics,
|
|
12
|
+
coordinationKeys: Object.fromEntries([...ids].map((id) => [id, keyOf(id)])),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ActivityOutcomeKind } from '../../activities/index.js';
|
|
2
|
+
import { isRecoveryBlockReason } from '../../execution/index.js';
|
|
2
3
|
import { WorkflowStatus } from '../../orchestration/index.js';
|
|
3
4
|
import { selectControlEvent, } from '../contracts/events.js';
|
|
4
5
|
export function hasActiveInvocation(view, coordinationKey) {
|
|
@@ -24,6 +25,8 @@ export async function isOwnScopeSource(view, scope, event, resolve) {
|
|
|
24
25
|
}
|
|
25
26
|
/** Blocking or a failed outcome with no retry wait left ends automatic workflow progress. */
|
|
26
27
|
export function exhaustedFailures(instance) {
|
|
28
|
+
if (isRecoveryBlockReason(instance.blockReason))
|
|
29
|
+
return false;
|
|
27
30
|
const failed = instance.lastOutcome?.kind === ActivityOutcomeKind.Failed;
|
|
28
31
|
return (instance.status === WorkflowStatus.Blocked ||
|
|
29
32
|
(failed && instance.status === WorkflowStatus.Completed) ||
|
|
@@ -8,6 +8,7 @@ export type { TriggerServiceDependencies } from './trigger-service-ports.js';
|
|
|
8
8
|
export type { TriggerCheckpointStore } from './trigger-source-cursor.js';
|
|
9
9
|
export type TriggerInspectionView = TriggerView & {
|
|
10
10
|
readonly diagnostics: readonly TriggerSkipDiagnostic[];
|
|
11
|
+
readonly coordinationKeys: Readonly<Record<string, string>>;
|
|
11
12
|
};
|
|
12
13
|
export declare const triggerCriticalSectionConsumer = "control-plane:trigger-coordination";
|
|
13
14
|
/** Requests and claims are journal facts; the cron cursor is processor recovery state. */
|
|
@@ -4,6 +4,7 @@ import { TriggerSkipReason } from '../contracts/trigger-diagnostics.js';
|
|
|
4
4
|
import { CronTriggerPolicy } from '../domain/cron-trigger-policy.js';
|
|
5
5
|
import { TriggerDiagnostics } from './trigger-diagnostics.js';
|
|
6
6
|
import { appendTriggerFact, consumedTriggerRequest, hasTriggerRequest, sourceIdentity, } from './trigger-facts.js';
|
|
7
|
+
import { inspectTriggers } from './trigger-inspection.js';
|
|
7
8
|
import { requestManualTrigger } from './trigger-manual-request.js';
|
|
8
9
|
import { maximumTriggerSources } from './trigger-projection.js';
|
|
9
10
|
import { recoverTriggerInvocation } from './trigger-recovery.js';
|
|
@@ -25,11 +26,10 @@ export class TriggerService {
|
|
|
25
26
|
this.sourceCursor = new TriggerSourceCursor(dependencies);
|
|
26
27
|
}
|
|
27
28
|
async get() {
|
|
28
|
-
return this.serialise(async () =>
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}));
|
|
29
|
+
return this.serialise(async () => {
|
|
30
|
+
const view = await this.reader.get();
|
|
31
|
+
return inspectTriggers(view, await this.dependencies.history(), await this.diagnostics.get(), this.dependencies.configurations(), (id) => this.commandKey(id));
|
|
32
|
+
});
|
|
33
33
|
}
|
|
34
34
|
async reconcile(context) {
|
|
35
35
|
await this.serialise(async () => {
|
|
@@ -30,7 +30,7 @@ export type AmbiguityResolution = {
|
|
|
30
30
|
};
|
|
31
31
|
};
|
|
32
32
|
export interface AmbiguityWorkflowBlocker {
|
|
33
|
-
|
|
33
|
+
blockAwaitingActivation(workflowInstanceId: string, activationId: string, reason: string, context: CommandContext): Promise<unknown>;
|
|
34
34
|
}
|
|
35
35
|
export declare class RecoveryService {
|
|
36
36
|
private readonly clock;
|
|
@@ -47,6 +47,7 @@ export declare class RecoveryService {
|
|
|
47
47
|
private appendRecovered;
|
|
48
48
|
private appendFailure;
|
|
49
49
|
private appendAmbiguity;
|
|
50
|
+
private blockEscalated;
|
|
50
51
|
private append;
|
|
51
52
|
}
|
|
52
53
|
export declare function createRecoveryCoordinator(service: RecoveryService): RecoveryCoordinator;
|
|
@@ -3,7 +3,7 @@ import { createExecutionEventData, } from '../contracts/event-factory.js';
|
|
|
3
3
|
import { ExecutionEventType, } from '../contracts/events.js';
|
|
4
4
|
import { runId } from '../contracts/identifiers.js';
|
|
5
5
|
import { runStream } from '../contracts/streams.js';
|
|
6
|
-
import { ExecutionFailureCode, ExternalExecutionState, isActiveRunStatus, RunStatus, } from '../contracts/vocabulary.js';
|
|
6
|
+
import { ExecutionFailureCode, ExternalExecutionState, isActiveRunStatus, RecoveryBlockReasonPrefix, RunStatus, } from '../contracts/vocabulary.js';
|
|
7
7
|
import { RunRepository } from './run-repository.js';
|
|
8
8
|
export class RecoveryService {
|
|
9
9
|
clock;
|
|
@@ -24,12 +24,14 @@ export class RecoveryService {
|
|
|
24
24
|
const currentRunId = runId(id);
|
|
25
25
|
const loaded = await this.repository.load(currentRunId);
|
|
26
26
|
const run = loaded.view;
|
|
27
|
+
if (run?.status === RunStatus.Ambiguous)
|
|
28
|
+
await this.blockEscalated(run);
|
|
27
29
|
if (run === null || !isActiveRunStatus(run.status))
|
|
28
30
|
return requireRun(run);
|
|
29
31
|
if (run.lease !== undefined && new Date(run.lease.expiresAt) > this.clock.now())
|
|
30
32
|
throw new Error(`Run ${id} does not have an expired lease`);
|
|
31
33
|
if (run.status === RunStatus.Starting)
|
|
32
|
-
return this.appendFailure(currentRunId, run, 'Preparation was interrupted before execution started');
|
|
34
|
+
return this.appendFailure(currentRunId, run, 'Preparation was interrupted before execution started', ExecutionFailureCode.Interrupted);
|
|
33
35
|
if (run.externalExecution === undefined)
|
|
34
36
|
return this.appendFailure(currentRunId, run, 'External execution was never reported');
|
|
35
37
|
const inspection = await this.inspector.inspect(run.externalExecution);
|
|
@@ -38,7 +40,7 @@ export class RecoveryService {
|
|
|
38
40
|
if (inspection.kind === ExternalExecutionState.Completed)
|
|
39
41
|
return this.appendRecovered(currentRunId, run, inspection.result);
|
|
40
42
|
if (inspection.kind === ExternalExecutionState.Absent)
|
|
41
|
-
return this.appendFailure(currentRunId, run, '
|
|
43
|
+
return this.appendFailure(currentRunId, run, 'Execution interrupted: original runner process is absent', ExecutionFailureCode.Interrupted);
|
|
42
44
|
return this.appendAmbiguity(currentRunId, run, inspection.reason);
|
|
43
45
|
}
|
|
44
46
|
async resolve(id, resolution, context) {
|
|
@@ -80,6 +82,10 @@ export class RecoveryService {
|
|
|
80
82
|
async recoverActive(owner, isLocallyActive = () => false) {
|
|
81
83
|
const recovered = [];
|
|
82
84
|
for (const run of await this.repository.list()) {
|
|
85
|
+
if (run.status === RunStatus.Ambiguous) {
|
|
86
|
+
await this.blockEscalated(run);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
83
89
|
if (!isActiveRunStatus(run.status))
|
|
84
90
|
continue;
|
|
85
91
|
if (isLocallyActive(run.runId))
|
|
@@ -103,9 +109,9 @@ export class RecoveryService {
|
|
|
103
109
|
const outcome = recoveredOutcome(run, result, this.activities);
|
|
104
110
|
return this.append(id, recoveredDraft(id, run, { result, outcome, finishedAt }, finishedAt));
|
|
105
111
|
}
|
|
106
|
-
async appendFailure(id, run, message) {
|
|
112
|
+
async appendFailure(id, run, message, kind = ExecutionFailureCode.Unexpected) {
|
|
107
113
|
const finishedAt = this.clock.now().toISOString();
|
|
108
|
-
return this.append(id, failedDraft(id, run, { failure: { kind
|
|
114
|
+
return this.append(id, failedDraft(id, run, { failure: { kind, message }, finishedAt }, finishedAt));
|
|
109
115
|
}
|
|
110
116
|
async appendAmbiguity(id, run, reason) {
|
|
111
117
|
const loaded = await this.repository.load(id);
|
|
@@ -123,14 +129,17 @@ export class RecoveryService {
|
|
|
123
129
|
await this.repository.append(id, loaded.sequence, drafts);
|
|
124
130
|
const updated = (await this.repository.load(id)).view;
|
|
125
131
|
if (updated.escalated)
|
|
126
|
-
await this.
|
|
127
|
-
commandId: `${id}:recovery-escalation:${attempt}`,
|
|
128
|
-
correlationId: correlationId(updated.orchestrationGroupId),
|
|
129
|
-
occurredAt,
|
|
130
|
-
actor: { kind: EventActorKind.System, id: 'execution-recovery' },
|
|
131
|
-
});
|
|
132
|
+
await this.blockEscalated(updated);
|
|
132
133
|
return updated;
|
|
133
134
|
}
|
|
135
|
+
async blockEscalated(run) {
|
|
136
|
+
await this.blocker?.blockAwaitingActivation(run.workflowInstanceId, run.activationId, `${RecoveryBlockReasonPrefix} run ${run.runId}: ${run.failure?.message ?? 'Verify the runner has stopped, then resolve the ambiguous run'}`, {
|
|
137
|
+
commandId: `${run.runId}:recovery-escalation:${run.ambiguityAttempts}`,
|
|
138
|
+
correlationId: correlationId(run.orchestrationGroupId),
|
|
139
|
+
occurredAt: run.finishedAt,
|
|
140
|
+
actor: { kind: EventActorKind.System, id: 'execution-recovery' },
|
|
141
|
+
});
|
|
142
|
+
}
|
|
134
143
|
async append(id, draft) {
|
|
135
144
|
const loaded = await this.repository.load(id);
|
|
136
145
|
if (loaded.view === null)
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { RunnerKind } from './vocabulary.js';
|
|
2
3
|
export declare const executionConfigSchema: z.ZodObject<{
|
|
3
4
|
agentRunners: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodDiscriminatedUnion<[z.ZodObject<{
|
|
4
5
|
model: z.ZodOptional<z.ZodString>;
|
|
@@ -69,7 +70,7 @@ export declare const executionConfigSchema: z.ZodObject<{
|
|
|
69
70
|
}, z.core.$strict>;
|
|
70
71
|
export interface ExecutionConfig {
|
|
71
72
|
readonly agentRunners?: Readonly<Record<string, {
|
|
72
|
-
readonly kind:
|
|
73
|
+
readonly kind: RunnerKind;
|
|
73
74
|
readonly command?: string;
|
|
74
75
|
readonly model?: string;
|
|
75
76
|
readonly effort?: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { z } from 'zod';
|
|
2
|
+
import { RunnerKind } from './vocabulary.js';
|
|
2
3
|
const isReservedFlag = (arg) => arg === '--output-format' || arg === '--resume';
|
|
3
4
|
const agentRunnerBase = {
|
|
4
5
|
model: z.string().trim().min(1).optional(),
|
|
@@ -21,11 +22,11 @@ export const executionConfigSchema = z
|
|
|
21
22
|
agentRunners: z
|
|
22
23
|
.record(z.string().trim().min(1), z
|
|
23
24
|
.discriminatedUnion('kind', [
|
|
24
|
-
commandRunner(
|
|
25
|
-
commandRunner(
|
|
26
|
-
commandRunner(
|
|
27
|
-
commandRunner(
|
|
28
|
-
z.object({ kind: z.literal(
|
|
25
|
+
commandRunner(RunnerKind.ClaudeCli),
|
|
26
|
+
commandRunner(RunnerKind.CodexCli),
|
|
27
|
+
commandRunner(RunnerKind.CursorCli),
|
|
28
|
+
commandRunner(RunnerKind.Command),
|
|
29
|
+
z.object({ kind: z.literal(RunnerKind.Fake), ...agentRunnerBase }).strict(),
|
|
29
30
|
])
|
|
30
31
|
.refine((value) => !value.args.some(isReservedFlag), {
|
|
31
32
|
message: 'args must not contain --output-format or --resume',
|
|
@@ -169,7 +169,7 @@ const runEventSchema = eventEnvelopeSchema.extend({
|
|
|
169
169
|
.object({
|
|
170
170
|
failure: z
|
|
171
171
|
.object({
|
|
172
|
-
kind: z.enum([ExecutionFailureCode.Unexpected]),
|
|
172
|
+
kind: z.enum([ExecutionFailureCode.Unexpected, ExecutionFailureCode.Interrupted]),
|
|
173
173
|
message: z.string(),
|
|
174
174
|
details: z
|
|
175
175
|
.object({
|
|
@@ -12,6 +12,7 @@ export type FinishedRunStatus = Exclude<RunStatus, typeof RunStatus.Starting | t
|
|
|
12
12
|
export declare function isActiveRunStatus(status: RunStatus): status is typeof RunStatus.Starting | typeof RunStatus.Started;
|
|
13
13
|
export declare const ExecutionFailureCode: {
|
|
14
14
|
readonly Unexpected: "unexpected-execution-failure";
|
|
15
|
+
readonly Interrupted: "interrupted-execution";
|
|
15
16
|
};
|
|
16
17
|
export type ExecutionFailureCode = ValueOf<typeof ExecutionFailureCode>;
|
|
17
18
|
export declare const ExecutionCancellationReason: {
|
|
@@ -48,3 +49,14 @@ export declare const TranscriptGroupKind: {
|
|
|
48
49
|
readonly Run: string;
|
|
49
50
|
};
|
|
50
51
|
export type TranscriptGroupKind = ValueOf<typeof TranscriptGroupKind>;
|
|
52
|
+
export declare const RunnerKind: {
|
|
53
|
+
readonly ClaudeCli: "claude-cli";
|
|
54
|
+
readonly CodexCli: "codex-cli";
|
|
55
|
+
readonly CursorCli: "cursor-cli";
|
|
56
|
+
readonly Command: "command";
|
|
57
|
+
readonly Fake: "fake";
|
|
58
|
+
};
|
|
59
|
+
export type RunnerKind = ValueOf<typeof RunnerKind>;
|
|
60
|
+
export declare const RecoveryBlockReasonPrefix = "execution-recovery-ambiguous:";
|
|
61
|
+
export declare function isRecoveryBlockReason(reason: string | undefined): boolean;
|
|
62
|
+
export declare function isAmbiguityResolutionBlockReason(reason: string | undefined): boolean;
|
|
@@ -12,6 +12,7 @@ export function isActiveRunStatus(status) {
|
|
|
12
12
|
}
|
|
13
13
|
export const ExecutionFailureCode = defineClosedVocabulary({
|
|
14
14
|
Unexpected: 'unexpected-execution-failure',
|
|
15
|
+
Interrupted: 'interrupted-execution',
|
|
15
16
|
});
|
|
16
17
|
export const ExecutionCancellationReason = defineClosedVocabulary({
|
|
17
18
|
Operator: 'operator',
|
|
@@ -46,3 +47,20 @@ export const TranscriptGroupKind = {
|
|
|
46
47
|
Session: transcriptGroupKinds[0],
|
|
47
48
|
Run: transcriptGroupKinds[1],
|
|
48
49
|
};
|
|
50
|
+
export const RunnerKind = {
|
|
51
|
+
ClaudeCli: 'claude-cli',
|
|
52
|
+
CodexCli: 'codex-cli',
|
|
53
|
+
CursorCli: 'cursor-cli',
|
|
54
|
+
Command: 'command',
|
|
55
|
+
Fake: 'fake',
|
|
56
|
+
};
|
|
57
|
+
export const RecoveryBlockReasonPrefix = 'execution-recovery-ambiguous:';
|
|
58
|
+
export function isRecoveryBlockReason(reason) {
|
|
59
|
+
return reason?.startsWith(RecoveryBlockReasonPrefix) === true;
|
|
60
|
+
}
|
|
61
|
+
// Journals written before the recovery prefix hold this reason permanently.
|
|
62
|
+
const legacyAmbiguityBlockReason = /^run-ambiguous-after-\d+-attempts$/;
|
|
63
|
+
export function isAmbiguityResolutionBlockReason(reason) {
|
|
64
|
+
return (isRecoveryBlockReason(reason) ||
|
|
65
|
+
(reason !== undefined && legacyAmbiguityBlockReason.test(reason)));
|
|
66
|
+
}
|
|
@@ -33,3 +33,4 @@ export * from './infrastructure/transcript-store.js';
|
|
|
33
33
|
export * from './infrastructure/workspace/fake-workspace.js';
|
|
34
34
|
export * from './infrastructure/workspace/git-workspace.js';
|
|
35
35
|
export * from './application/run-reports.js';
|
|
36
|
+
export { createLocalProcessInspector } from './infrastructure/local-process-inspector.js';
|
|
@@ -33,3 +33,4 @@ export * from './infrastructure/transcript-store.js';
|
|
|
33
33
|
export * from './infrastructure/workspace/fake-workspace.js';
|
|
34
34
|
export * from './infrastructure/workspace/git-workspace.js';
|
|
35
35
|
export * from './application/run-reports.js';
|
|
36
|
+
export { createLocalProcessInspector } from './infrastructure/local-process-inspector.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ExternalExecutionInspector } from '../application/recovery-service.js';
|
|
2
|
+
/** Opaque identity includes PID reuse protection and the inspection environment. */
|
|
3
|
+
export declare function localProcessIdentity(pid: number | undefined): string | undefined;
|
|
4
|
+
export declare function createLocalProcessInspector(): ExternalExecutionInspector;
|