@atolis-hq/wake 0.4.74 → 0.4.76
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/bootstrap/composition-root.js +2 -10
- package/dist/src/bootstrap/surface-api-run-context.js +2 -2
- package/dist/src/bootstrap/surface-api-trigger-applications.js +38 -4
- package/dist/src/bootstrap/surface-api-work-applications.js +5 -3
- package/dist/src/bootstrap/version.js +1 -1
- package/dist/src/control-plane/application/activation-scheduler-ports.d.ts +2 -1
- package/dist/src/control-plane/application/activation-scheduler.js +14 -2
- package/dist/src/control-plane/application/advance-once-dispatch.js +1 -0
- package/dist/src/control-plane/application/trigger-inspection.d.ts +32 -0
- package/dist/src/control-plane/application/trigger-inspection.js +14 -0
- package/dist/src/control-plane/application/trigger-rules.js +3 -0
- package/dist/src/control-plane/application/trigger-service.d.ts +1 -0
- package/dist/src/control-plane/application/trigger-service.js +5 -5
- package/dist/src/execution/application/recovery-service.d.ts +2 -1
- package/dist/src/execution/application/recovery-service.js +20 -11
- package/dist/src/execution/contracts/events.js +1 -1
- package/dist/src/execution/contracts/vocabulary.d.ts +4 -0
- package/dist/src/execution/contracts/vocabulary.js +11 -0
- package/dist/src/execution/index.d.ts +1 -0
- package/dist/src/execution/index.js +1 -0
- package/dist/src/execution/infrastructure/local-process-inspector.d.ts +4 -0
- package/dist/src/execution/infrastructure/local-process-inspector.js +85 -0
- package/dist/src/execution/infrastructure/process-execution.d.ts +1 -0
- package/dist/src/execution/infrastructure/process-execution.js +6 -1
- package/dist/src/execution/infrastructure/runners/claude.js +1 -1
- package/dist/src/integrations/delivery/application/delivery-projector.d.ts +5 -0
- package/dist/src/integrations/delivery/application/delivery-projector.js +9 -2
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.d.ts.map +1 -1
- package/dist/src/node_modules/@atolis-hq/eventing-filesystem/dist/file-lock.js +14 -7
- package/dist/src/orchestration/application/advance-workflow.d.ts +7 -8
- package/dist/src/orchestration/application/advance-workflow.js +21 -36
- package/dist/src/orchestration/application/operator-retry-attribution.d.ts +3 -0
- package/dist/src/orchestration/application/operator-retry-attribution.js +7 -0
- package/dist/src/orchestration/application/orchestration-service.d.ts +4 -1
- package/dist/src/orchestration/application/orchestration-service.js +4 -1
- package/dist/src/orchestration/application/resolve-execution-failure.d.ts +14 -0
- package/dist/src/orchestration/application/resolve-execution-failure.js +47 -0
- package/dist/src/surfaces/api/contracts/transport-values.d.ts +5 -1
- package/dist/src/surfaces/api/contracts/transport-values.js +13 -0
- package/dist/src/surfaces/api/contracts/triggers.d.ts +13 -1
- package/dist/src/surfaces/web-assets/assets/index-QnREfUBG.js +1192 -0
- package/dist/src/surfaces/web-assets/index.html +1 -1
- package/package.json +1 -1
- package/dist/src/surfaces/web-assets/assets/index-CxURzTmR.js +0 -1192
|
@@ -3,7 +3,7 @@ import { createPullRequestService } from '../activities/index.js';
|
|
|
3
3
|
import { ArtifactContextService, FileArtifactMcpSessionStore, FileArtifactStore, OperatorInstructionsService, createArtifactService, } from '../artifacts/index.js';
|
|
4
4
|
import { DispatchPolicy, createActivationScheduler, createActivationSchedulerSubscriber, createControlPlaneService, createDurableRunnerIneligibility, createRunnerControlService, } from '../control-plane/index.js';
|
|
5
5
|
import { createConversationService } from '../conversations/index.js';
|
|
6
|
-
import { ExecutionCancellationReason,
|
|
6
|
+
import { ExecutionCancellationReason, GitWorkspaceProvider, RecoveryService, TranscriptStore, createExecutionService, createLocalProcessInspector, createRunnerMemoryProfileDecorator, } from '../execution/index.js';
|
|
7
7
|
import { createGitHubAgentContextReader, gitHubProviderDefinition, resolveGitHubResourceUrl, } from '../integrations/github/index.js';
|
|
8
8
|
import { SystemClock, UlidIdGenerator } from '../kernel/index.js';
|
|
9
9
|
import { compileWorkflow, createOrchestrationService } from '../orchestration/index.js';
|
|
@@ -126,15 +126,7 @@ export async function createCompositionRoot(wakeRoot, options = {}) {
|
|
|
126
126
|
orchestration.setWatchChildCancellation({
|
|
127
127
|
cancelSupersededWatchChildren: (workflowInstanceIds) => execution.cancelActive(workflowInstanceIds, ExecutionCancellationReason.WorkflowSuperseded),
|
|
128
128
|
});
|
|
129
|
-
const recovery = new RecoveryService(journal, clock,
|
|
130
|
-
async inspect() {
|
|
131
|
-
// Unknown external work follows the safe ambiguity path until runners expose inspection.
|
|
132
|
-
return {
|
|
133
|
-
kind: ExternalExecutionState.Unknown,
|
|
134
|
-
reason: 'External execution inspection is not configured for this runtime',
|
|
135
|
-
};
|
|
136
|
-
},
|
|
137
|
-
}, activities, config.execution, orchestration);
|
|
129
|
+
const recovery = new RecoveryService(journal, clock, createLocalProcessInspector(), activities, config.execution, orchestration);
|
|
138
130
|
const controlPlane = createControlPlaneService({ journal, clock, ids });
|
|
139
131
|
const isRuntimePaused = async () => (await controlPlane.isDispatchPaused()) || (await maintenance.read()) !== null;
|
|
140
132
|
const runnerControls = createRunnerControlService({
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { ActivityOutcomeKind } from '../activities/index.js';
|
|
2
|
-
import { DeliveryState, IntegrationStreamKind, } from '../integrations/index.js';
|
|
2
|
+
import { deliveryProjectionKey, DeliveryState, IntegrationStreamKind, } from '../integrations/index.js';
|
|
3
3
|
import { workflowInstanceId } from '../orchestration/index.js';
|
|
4
4
|
export async function withWorkflowContext(root, run) {
|
|
5
5
|
const instance = await root.orchestration.get(workflowInstanceId(run.workflowInstanceId));
|
|
@@ -20,7 +20,7 @@ export async function withDeliveryResolution(root, run) {
|
|
|
20
20
|
const intentEventId = waitingDeliveryIntentEventId(run);
|
|
21
21
|
if (intentEventId === undefined)
|
|
22
22
|
return run;
|
|
23
|
-
const stored = await root.projections.read(IntegrationStreamKind.Delivery, intentEventId);
|
|
23
|
+
const stored = await root.projections.read(IntegrationStreamKind.Delivery, deliveryProjectionKey(intentEventId));
|
|
24
24
|
const delivery = stored?.value;
|
|
25
25
|
if (delivery === undefined || delivery.resolvedAt === undefined)
|
|
26
26
|
return run;
|
|
@@ -2,7 +2,9 @@ import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { OperatorInstructionsConflict, operatorInstructionsProjection, } from '../artifacts/index.js';
|
|
4
4
|
import { CronTriggerPolicy } from '../control-plane/index.js';
|
|
5
|
+
import { selectOperatorRetryTarget } from '../orchestration/index.js';
|
|
5
6
|
import { ApiCommandStatus, toWorkItemKey, TriggerStatus, } from '../surfaces/index.js';
|
|
7
|
+
import { workItemId, WorkStatus } from '../work/index.js';
|
|
6
8
|
export function createTriggerApplications(root, now) {
|
|
7
9
|
async function inspect() {
|
|
8
10
|
const view = await root.triggers.get();
|
|
@@ -14,7 +16,10 @@ export function createTriggerApplications(root, now) {
|
|
|
14
16
|
...(view.diagnostics ?? []).map((entry) => entry.triggerId),
|
|
15
17
|
]);
|
|
16
18
|
const asOf = now();
|
|
17
|
-
return {
|
|
19
|
+
return {
|
|
20
|
+
data: await Promise.all([...ids].sort().map((id) => present(id, view, asOf))),
|
|
21
|
+
meta: { asOf },
|
|
22
|
+
};
|
|
18
23
|
}
|
|
19
24
|
async function list() {
|
|
20
25
|
const result = await inspect();
|
|
@@ -29,16 +34,19 @@ export function createTriggerApplications(root, now) {
|
|
|
29
34
|
}));
|
|
30
35
|
return { ...result, orphanedInstructions };
|
|
31
36
|
}
|
|
32
|
-
function present(id, view, asOf) {
|
|
37
|
+
async function present(id, view, asOf) {
|
|
33
38
|
const definition = root.config.controlPlane.triggers.find((config) => config.id === id) ?? null;
|
|
34
39
|
const invocations = view.invocations
|
|
35
40
|
.filter((entry) => entry.claim.triggerId === id)
|
|
36
41
|
.sort((a, b) => b.claimedAt.localeCompare(a.claimedAt))
|
|
37
42
|
.map((entry) => ({ ...entry, workItemKey: toWorkItemKey(entry.claim.workItemId) }));
|
|
38
43
|
const pauseReason = view.pauses[id] ?? null;
|
|
39
|
-
const
|
|
44
|
+
const coordinationKey = view.coordinationKeys[id] ?? id;
|
|
45
|
+
const activeInvocation = await inspectHolder(coordinationKey, view);
|
|
46
|
+
const running = activeInvocation !== null;
|
|
40
47
|
return {
|
|
41
48
|
triggerId: id,
|
|
49
|
+
activeInvocation,
|
|
42
50
|
definition,
|
|
43
51
|
paused: pauseReason !== null,
|
|
44
52
|
pauseReason,
|
|
@@ -52,11 +60,37 @@ export function createTriggerApplications(root, now) {
|
|
|
52
60
|
nextOccurrence: definition?.on.cron === undefined
|
|
53
61
|
? null
|
|
54
62
|
: new CronTriggerPolicy().nextOccurrence(definition.on.cron, asOf),
|
|
55
|
-
pending: Object.values(view.coalesced)
|
|
63
|
+
pending: Object.values(view.coalesced)
|
|
64
|
+
.filter((entry) => entry.coordinationKey === coordinationKey)
|
|
65
|
+
.map((entry) => ({
|
|
66
|
+
...entry,
|
|
67
|
+
activeWorkItemId: activeInvocation?.activeWorkItemId ?? null,
|
|
68
|
+
})),
|
|
56
69
|
latestSkip: view.diagnostics?.find((entry) => entry.triggerId === id) ?? null,
|
|
57
70
|
invocations,
|
|
58
71
|
};
|
|
59
72
|
}
|
|
73
|
+
async function inspectHolder(coordinationKey, view) {
|
|
74
|
+
const holder = view.invocations.find((entry) => entry.result === undefined && entry.claim.coordinationKey === coordinationKey);
|
|
75
|
+
if (holder === undefined)
|
|
76
|
+
return null;
|
|
77
|
+
const workflows = (await root.orchestration.listAll()).filter((entry) => entry.workItemId === holder.claim.workItemId);
|
|
78
|
+
const workflow = workflows.find((entry) => entry.workflowInstanceId === holder.claim.workflowInstanceId);
|
|
79
|
+
const work = await root.work.get(workItemId(holder.claim.workItemId));
|
|
80
|
+
return {
|
|
81
|
+
activeWorkItemId: holder.claim.workItemId,
|
|
82
|
+
workItemKey: toWorkItemKey(holder.claim.workItemId),
|
|
83
|
+
triggerId: holder.claim.triggerId,
|
|
84
|
+
workflowInstanceId: holder.claim.workflowInstanceId,
|
|
85
|
+
workflowStatus: workflow?.status ?? null,
|
|
86
|
+
blockReason: workflow?.blockReason ?? null,
|
|
87
|
+
retryEligible: work !== null &&
|
|
88
|
+
work.deleted !== true &&
|
|
89
|
+
!work.frozen &&
|
|
90
|
+
work.state === WorkStatus.Open &&
|
|
91
|
+
selectOperatorRetryTarget(workflows) !== undefined,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
60
94
|
async function command(id, request, pause) {
|
|
61
95
|
if (!(await inspect()).data.some((entry) => entry.triggerId === id))
|
|
62
96
|
return {
|
|
@@ -85,15 +85,17 @@ export function createSurfaceWorkApplications(root, now) {
|
|
|
85
85
|
const work = await root.work.get(id);
|
|
86
86
|
if (work === null)
|
|
87
87
|
return retryIneligible('Work item was not found');
|
|
88
|
+
const workflows = (await root.orchestration.listAll()).filter((workflow) => workflow.workItemId === id);
|
|
89
|
+
if (!workflows.some((workflow) => workflow.parentWorkflowInstanceId === undefined))
|
|
90
|
+
return retryIneligible('Work item has no primary workflow');
|
|
91
|
+
if (workflows.some((workflow) => workflow.operatorRetryCommandIds.includes(commandContext(command.idempotencyKey, now).commandId)))
|
|
92
|
+
return accepted(command.idempotencyKey, now());
|
|
88
93
|
if (work.deleted === true)
|
|
89
94
|
return retryIneligible('Work item is deleted');
|
|
90
95
|
if (work.frozen)
|
|
91
96
|
return retryIneligible('Work item is frozen');
|
|
92
97
|
if (work.state !== WorkStatus.Open)
|
|
93
98
|
return retryIneligible('Work item is not open');
|
|
94
|
-
const workflows = (await root.orchestration.listAll()).filter((workflow) => workflow.workItemId === id);
|
|
95
|
-
if (!workflows.some((workflow) => workflow.parentWorkflowInstanceId === undefined))
|
|
96
|
-
return retryIneligible('Work item has no primary workflow');
|
|
97
99
|
const target = selectOperatorRetryTarget(workflows);
|
|
98
100
|
if (target === undefined)
|
|
99
101
|
return retryIneligible('Workflow is not retry eligible');
|
|
@@ -124,6 +124,6 @@ export function resolveWakeVersion(options = {}) {
|
|
|
124
124
|
return `g${headHash.slice(0, 7)}`;
|
|
125
125
|
return '0.1.0-dev';
|
|
126
126
|
}
|
|
127
|
-
export const wakeVersion = "v0.4.
|
|
127
|
+
export const wakeVersion = "v0.4.76";
|
|
128
128
|
/** The registry-installable version used by packaged sandbox images. */
|
|
129
129
|
export const wakePackageVersion = packageVersion(defaultRepoRoot) ?? wakeVersion;
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import type { CommandContext } from '@atolis-hq/eventing';
|
|
2
|
-
import type { RunView, WorkspaceRecovery } from '../../execution/index.js';
|
|
2
|
+
import type { ExecutionFailureCode, RunView, WorkspaceRecovery } from '../../execution/index.js';
|
|
3
3
|
import type { IdGenerator } from '../../kernel/index.js';
|
|
4
4
|
import type { ActivityActivationView, WorkflowInstanceView } from '../../orchestration/index.js';
|
|
5
5
|
import type { ResourceService } from '../../resources/index.js';
|
|
@@ -25,6 +25,7 @@ export interface OrchestrationPort {
|
|
|
25
25
|
readonly activationId: string;
|
|
26
26
|
readonly runId: string;
|
|
27
27
|
readonly reason: string;
|
|
28
|
+
readonly failureKind?: ExecutionFailureCode;
|
|
28
29
|
}, context: CommandContext): Promise<WorkflowInstanceView | null>;
|
|
29
30
|
retryRunnerQuotaFailure?(workflowInstanceId: string, input: {
|
|
30
31
|
readonly activationId: string;
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
/* eslint-disable complexity, max-lines-per-function */
|
|
2
2
|
import { correlationId, EventActorKind } from '@atolis-hq/eventing';
|
|
3
|
-
import { RunStatus } from '../../execution/index.js';
|
|
3
|
+
import { ExecutionFailureCode, RunStatus } from '../../execution/index.js';
|
|
4
4
|
import { isAmbiguityResolutionBlock, WorkflowStatus, } from '../../orchestration/index.js';
|
|
5
5
|
import { WorkStatus } from '../../work/index.js';
|
|
6
6
|
import { ControlStreamKind } from '../contracts/streams.js';
|
|
@@ -134,11 +134,23 @@ export function createActivationScheduler(orchestration, execution, resources, c
|
|
|
134
134
|
],
|
|
135
135
|
};
|
|
136
136
|
}
|
|
137
|
-
await orchestration.resolveExecutionFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
137
|
+
const resolved = await orchestration.resolveExecutionFailure?.(recovery.item.workflow.workflowInstanceId, {
|
|
138
138
|
activationId: recovery.item.activation.activationId,
|
|
139
139
|
runId: recovery.run.runId,
|
|
140
140
|
reason: recovery.run.failure?.message ?? 'execution failed',
|
|
141
|
+
...(recovery.run.failure === undefined
|
|
142
|
+
? {}
|
|
143
|
+
: { failureKind: recovery.run.failure.kind }),
|
|
141
144
|
}, context(recovery.run.runId));
|
|
145
|
+
if (recovery.run.failure?.kind === ExecutionFailureCode.Interrupted &&
|
|
146
|
+
resolved != null &&
|
|
147
|
+
resolved.status !== WorkflowStatus.Blocked)
|
|
148
|
+
return {
|
|
149
|
+
kind: 'progressed',
|
|
150
|
+
dispatched: [
|
|
151
|
+
{ activationId: recovery.item.activation.activationId, runId: recovery.run.runId },
|
|
152
|
+
],
|
|
153
|
+
};
|
|
142
154
|
return {
|
|
143
155
|
kind: WorkflowStatus.Blocked,
|
|
144
156
|
workflowInstanceId: recovery.item.workflow.workflowInstanceId,
|
|
@@ -150,6 +150,7 @@ export async function runDispatchLoop(pending, ctx) {
|
|
|
150
150
|
activationId: selected.activation.activationId,
|
|
151
151
|
runId: run.runId,
|
|
152
152
|
reason: run.failure?.message ?? 'execution failed',
|
|
153
|
+
...(run.failure === undefined ? {} : { failureKind: run.failure.kind }),
|
|
153
154
|
}, ctx.commandContext(run.runId));
|
|
154
155
|
if (run.status !== RunStatus.Succeeded && !isActiveRunStatus(run.status)) {
|
|
155
156
|
stopReason = {
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { TriggerConfig } from '../contracts/config.js';
|
|
2
|
+
import type { TriggerSkipDiagnostic } from '../contracts/trigger-diagnostics.js';
|
|
3
|
+
import type { TriggerInvocationView, TriggerView } from './trigger-projection.js';
|
|
4
|
+
export declare function inspectTriggers(view: TriggerView, invocations: readonly TriggerInvocationView[], diagnostics: readonly TriggerSkipDiagnostic[], configurations: readonly TriggerConfig[], keyOf: (id: string) => string): {
|
|
5
|
+
invocations: readonly TriggerInvocationView[];
|
|
6
|
+
diagnostics: readonly {
|
|
7
|
+
triggerId: string;
|
|
8
|
+
source: {
|
|
9
|
+
kind: "manual";
|
|
10
|
+
commandId: string;
|
|
11
|
+
operator: string;
|
|
12
|
+
} | {
|
|
13
|
+
kind: "event";
|
|
14
|
+
eventId: string;
|
|
15
|
+
eventType: string;
|
|
16
|
+
} | {
|
|
17
|
+
kind: "cron";
|
|
18
|
+
slot: string;
|
|
19
|
+
};
|
|
20
|
+
occurredAt: string;
|
|
21
|
+
reason: "paused" | "own-scope" | "removed" | "dispatch-paused" | "capacity-unavailable" | "pending-source-limit";
|
|
22
|
+
}[];
|
|
23
|
+
coordinationKeys: {
|
|
24
|
+
[k: string]: string;
|
|
25
|
+
};
|
|
26
|
+
lastStarts: Readonly<Record<string, string>>;
|
|
27
|
+
pauses: Readonly<Record<string, string>>;
|
|
28
|
+
requests: readonly import("../index.js").ControlEventPayloads["control-plane.trigger-request-recorded"][];
|
|
29
|
+
coalesced: Readonly<Record<string, import("../index.js").ControlEventPayloads["control-plane.trigger-request-recorded"] & {
|
|
30
|
+
readonly sources: readonly import("../index.js").TriggerSource[];
|
|
31
|
+
}>>;
|
|
32
|
+
};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export function inspectTriggers(view, invocations, diagnostics, configurations, keyOf) {
|
|
2
|
+
const ids = new Set([
|
|
3
|
+
...configurations.map((config) => config.id),
|
|
4
|
+
...view.requests.map((request) => request.triggerId),
|
|
5
|
+
...invocations.map((invocation) => invocation.claim.triggerId),
|
|
6
|
+
...Object.keys(view.pauses),
|
|
7
|
+
]);
|
|
8
|
+
return {
|
|
9
|
+
...view,
|
|
10
|
+
invocations,
|
|
11
|
+
diagnostics,
|
|
12
|
+
coordinationKeys: Object.fromEntries([...ids].map((id) => [id, keyOf(id)])),
|
|
13
|
+
};
|
|
14
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { ActivityOutcomeKind } from '../../activities/index.js';
|
|
2
|
+
import { isRecoveryBlockReason } from '../../execution/index.js';
|
|
2
3
|
import { WorkflowStatus } from '../../orchestration/index.js';
|
|
3
4
|
import { selectControlEvent, } from '../contracts/events.js';
|
|
4
5
|
export function hasActiveInvocation(view, coordinationKey) {
|
|
@@ -24,6 +25,8 @@ export async function isOwnScopeSource(view, scope, event, resolve) {
|
|
|
24
25
|
}
|
|
25
26
|
/** Blocking or a failed outcome with no retry wait left ends automatic workflow progress. */
|
|
26
27
|
export function exhaustedFailures(instance) {
|
|
28
|
+
if (isRecoveryBlockReason(instance.blockReason))
|
|
29
|
+
return false;
|
|
27
30
|
const failed = instance.lastOutcome?.kind === ActivityOutcomeKind.Failed;
|
|
28
31
|
return (instance.status === WorkflowStatus.Blocked ||
|
|
29
32
|
(failed && instance.status === WorkflowStatus.Completed) ||
|
|
@@ -8,6 +8,7 @@ export type { TriggerServiceDependencies } from './trigger-service-ports.js';
|
|
|
8
8
|
export type { TriggerCheckpointStore } from './trigger-source-cursor.js';
|
|
9
9
|
export type TriggerInspectionView = TriggerView & {
|
|
10
10
|
readonly diagnostics: readonly TriggerSkipDiagnostic[];
|
|
11
|
+
readonly coordinationKeys: Readonly<Record<string, string>>;
|
|
11
12
|
};
|
|
12
13
|
export declare const triggerCriticalSectionConsumer = "control-plane:trigger-coordination";
|
|
13
14
|
/** Requests and claims are journal facts; the cron cursor is processor recovery state. */
|
|
@@ -4,6 +4,7 @@ import { TriggerSkipReason } from '../contracts/trigger-diagnostics.js';
|
|
|
4
4
|
import { CronTriggerPolicy } from '../domain/cron-trigger-policy.js';
|
|
5
5
|
import { TriggerDiagnostics } from './trigger-diagnostics.js';
|
|
6
6
|
import { appendTriggerFact, consumedTriggerRequest, hasTriggerRequest, sourceIdentity, } from './trigger-facts.js';
|
|
7
|
+
import { inspectTriggers } from './trigger-inspection.js';
|
|
7
8
|
import { requestManualTrigger } from './trigger-manual-request.js';
|
|
8
9
|
import { maximumTriggerSources } from './trigger-projection.js';
|
|
9
10
|
import { recoverTriggerInvocation } from './trigger-recovery.js';
|
|
@@ -25,11 +26,10 @@ export class TriggerService {
|
|
|
25
26
|
this.sourceCursor = new TriggerSourceCursor(dependencies);
|
|
26
27
|
}
|
|
27
28
|
async get() {
|
|
28
|
-
return this.serialise(async () =>
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}));
|
|
29
|
+
return this.serialise(async () => {
|
|
30
|
+
const view = await this.reader.get();
|
|
31
|
+
return inspectTriggers(view, await this.dependencies.history(), await this.diagnostics.get(), this.dependencies.configurations(), (id) => this.commandKey(id));
|
|
32
|
+
});
|
|
33
33
|
}
|
|
34
34
|
async reconcile(context) {
|
|
35
35
|
await this.serialise(async () => {
|
|
@@ -30,7 +30,7 @@ export type AmbiguityResolution = {
|
|
|
30
30
|
};
|
|
31
31
|
};
|
|
32
32
|
export interface AmbiguityWorkflowBlocker {
|
|
33
|
-
|
|
33
|
+
blockAwaitingActivation(workflowInstanceId: string, activationId: string, reason: string, context: CommandContext): Promise<unknown>;
|
|
34
34
|
}
|
|
35
35
|
export declare class RecoveryService {
|
|
36
36
|
private readonly clock;
|
|
@@ -47,6 +47,7 @@ export declare class RecoveryService {
|
|
|
47
47
|
private appendRecovered;
|
|
48
48
|
private appendFailure;
|
|
49
49
|
private appendAmbiguity;
|
|
50
|
+
private blockEscalated;
|
|
50
51
|
private append;
|
|
51
52
|
}
|
|
52
53
|
export declare function createRecoveryCoordinator(service: RecoveryService): RecoveryCoordinator;
|
|
@@ -3,7 +3,7 @@ import { createExecutionEventData, } from '../contracts/event-factory.js';
|
|
|
3
3
|
import { ExecutionEventType, } from '../contracts/events.js';
|
|
4
4
|
import { runId } from '../contracts/identifiers.js';
|
|
5
5
|
import { runStream } from '../contracts/streams.js';
|
|
6
|
-
import { ExecutionFailureCode, ExternalExecutionState, isActiveRunStatus, RunStatus, } from '../contracts/vocabulary.js';
|
|
6
|
+
import { ExecutionFailureCode, ExternalExecutionState, isActiveRunStatus, RecoveryBlockReasonPrefix, RunStatus, } from '../contracts/vocabulary.js';
|
|
7
7
|
import { RunRepository } from './run-repository.js';
|
|
8
8
|
export class RecoveryService {
|
|
9
9
|
clock;
|
|
@@ -24,12 +24,14 @@ export class RecoveryService {
|
|
|
24
24
|
const currentRunId = runId(id);
|
|
25
25
|
const loaded = await this.repository.load(currentRunId);
|
|
26
26
|
const run = loaded.view;
|
|
27
|
+
if (run?.status === RunStatus.Ambiguous)
|
|
28
|
+
await this.blockEscalated(run);
|
|
27
29
|
if (run === null || !isActiveRunStatus(run.status))
|
|
28
30
|
return requireRun(run);
|
|
29
31
|
if (run.lease !== undefined && new Date(run.lease.expiresAt) > this.clock.now())
|
|
30
32
|
throw new Error(`Run ${id} does not have an expired lease`);
|
|
31
33
|
if (run.status === RunStatus.Starting)
|
|
32
|
-
return this.appendFailure(currentRunId, run, 'Preparation was interrupted before execution started');
|
|
34
|
+
return this.appendFailure(currentRunId, run, 'Preparation was interrupted before execution started', ExecutionFailureCode.Interrupted);
|
|
33
35
|
if (run.externalExecution === undefined)
|
|
34
36
|
return this.appendFailure(currentRunId, run, 'External execution was never reported');
|
|
35
37
|
const inspection = await this.inspector.inspect(run.externalExecution);
|
|
@@ -38,7 +40,7 @@ export class RecoveryService {
|
|
|
38
40
|
if (inspection.kind === ExternalExecutionState.Completed)
|
|
39
41
|
return this.appendRecovered(currentRunId, run, inspection.result);
|
|
40
42
|
if (inspection.kind === ExternalExecutionState.Absent)
|
|
41
|
-
return this.appendFailure(currentRunId, run, '
|
|
43
|
+
return this.appendFailure(currentRunId, run, 'Execution interrupted: original runner process is absent', ExecutionFailureCode.Interrupted);
|
|
42
44
|
return this.appendAmbiguity(currentRunId, run, inspection.reason);
|
|
43
45
|
}
|
|
44
46
|
async resolve(id, resolution, context) {
|
|
@@ -80,6 +82,10 @@ export class RecoveryService {
|
|
|
80
82
|
async recoverActive(owner, isLocallyActive = () => false) {
|
|
81
83
|
const recovered = [];
|
|
82
84
|
for (const run of await this.repository.list()) {
|
|
85
|
+
if (run.status === RunStatus.Ambiguous) {
|
|
86
|
+
await this.blockEscalated(run);
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
83
89
|
if (!isActiveRunStatus(run.status))
|
|
84
90
|
continue;
|
|
85
91
|
if (isLocallyActive(run.runId))
|
|
@@ -103,9 +109,9 @@ export class RecoveryService {
|
|
|
103
109
|
const outcome = recoveredOutcome(run, result, this.activities);
|
|
104
110
|
return this.append(id, recoveredDraft(id, run, { result, outcome, finishedAt }, finishedAt));
|
|
105
111
|
}
|
|
106
|
-
async appendFailure(id, run, message) {
|
|
112
|
+
async appendFailure(id, run, message, kind = ExecutionFailureCode.Unexpected) {
|
|
107
113
|
const finishedAt = this.clock.now().toISOString();
|
|
108
|
-
return this.append(id, failedDraft(id, run, { failure: { kind
|
|
114
|
+
return this.append(id, failedDraft(id, run, { failure: { kind, message }, finishedAt }, finishedAt));
|
|
109
115
|
}
|
|
110
116
|
async appendAmbiguity(id, run, reason) {
|
|
111
117
|
const loaded = await this.repository.load(id);
|
|
@@ -123,14 +129,17 @@ export class RecoveryService {
|
|
|
123
129
|
await this.repository.append(id, loaded.sequence, drafts);
|
|
124
130
|
const updated = (await this.repository.load(id)).view;
|
|
125
131
|
if (updated.escalated)
|
|
126
|
-
await this.
|
|
127
|
-
commandId: `${id}:recovery-escalation:${attempt}`,
|
|
128
|
-
correlationId: correlationId(updated.orchestrationGroupId),
|
|
129
|
-
occurredAt,
|
|
130
|
-
actor: { kind: EventActorKind.System, id: 'execution-recovery' },
|
|
131
|
-
});
|
|
132
|
+
await this.blockEscalated(updated);
|
|
132
133
|
return updated;
|
|
133
134
|
}
|
|
135
|
+
async blockEscalated(run) {
|
|
136
|
+
await this.blocker?.blockAwaitingActivation(run.workflowInstanceId, run.activationId, `${RecoveryBlockReasonPrefix} run ${run.runId}: ${run.failure?.message ?? 'Verify the runner has stopped, then resolve the ambiguous run'}`, {
|
|
137
|
+
commandId: `${run.runId}:recovery-escalation:${run.ambiguityAttempts}`,
|
|
138
|
+
correlationId: correlationId(run.orchestrationGroupId),
|
|
139
|
+
occurredAt: run.finishedAt,
|
|
140
|
+
actor: { kind: EventActorKind.System, id: 'execution-recovery' },
|
|
141
|
+
});
|
|
142
|
+
}
|
|
134
143
|
async append(id, draft) {
|
|
135
144
|
const loaded = await this.repository.load(id);
|
|
136
145
|
if (loaded.view === null)
|
|
@@ -169,7 +169,7 @@ const runEventSchema = eventEnvelopeSchema.extend({
|
|
|
169
169
|
.object({
|
|
170
170
|
failure: z
|
|
171
171
|
.object({
|
|
172
|
-
kind: z.enum([ExecutionFailureCode.Unexpected]),
|
|
172
|
+
kind: z.enum([ExecutionFailureCode.Unexpected, ExecutionFailureCode.Interrupted]),
|
|
173
173
|
message: z.string(),
|
|
174
174
|
details: z
|
|
175
175
|
.object({
|
|
@@ -12,6 +12,7 @@ export type FinishedRunStatus = Exclude<RunStatus, typeof RunStatus.Starting | t
|
|
|
12
12
|
export declare function isActiveRunStatus(status: RunStatus): status is typeof RunStatus.Starting | typeof RunStatus.Started;
|
|
13
13
|
export declare const ExecutionFailureCode: {
|
|
14
14
|
readonly Unexpected: "unexpected-execution-failure";
|
|
15
|
+
readonly Interrupted: "interrupted-execution";
|
|
15
16
|
};
|
|
16
17
|
export type ExecutionFailureCode = ValueOf<typeof ExecutionFailureCode>;
|
|
17
18
|
export declare const ExecutionCancellationReason: {
|
|
@@ -56,3 +57,6 @@ export declare const RunnerKind: {
|
|
|
56
57
|
readonly Fake: "fake";
|
|
57
58
|
};
|
|
58
59
|
export type RunnerKind = ValueOf<typeof RunnerKind>;
|
|
60
|
+
export declare const RecoveryBlockReasonPrefix = "execution-recovery-ambiguous:";
|
|
61
|
+
export declare function isRecoveryBlockReason(reason: string | undefined): boolean;
|
|
62
|
+
export declare function isAmbiguityResolutionBlockReason(reason: string | undefined): boolean;
|
|
@@ -12,6 +12,7 @@ export function isActiveRunStatus(status) {
|
|
|
12
12
|
}
|
|
13
13
|
export const ExecutionFailureCode = defineClosedVocabulary({
|
|
14
14
|
Unexpected: 'unexpected-execution-failure',
|
|
15
|
+
Interrupted: 'interrupted-execution',
|
|
15
16
|
});
|
|
16
17
|
export const ExecutionCancellationReason = defineClosedVocabulary({
|
|
17
18
|
Operator: 'operator',
|
|
@@ -53,3 +54,13 @@ export const RunnerKind = {
|
|
|
53
54
|
Command: 'command',
|
|
54
55
|
Fake: 'fake',
|
|
55
56
|
};
|
|
57
|
+
export const RecoveryBlockReasonPrefix = 'execution-recovery-ambiguous:';
|
|
58
|
+
export function isRecoveryBlockReason(reason) {
|
|
59
|
+
return reason?.startsWith(RecoveryBlockReasonPrefix) === true;
|
|
60
|
+
}
|
|
61
|
+
// Journals written before the recovery prefix hold this reason permanently.
|
|
62
|
+
const legacyAmbiguityBlockReason = /^run-ambiguous-after-\d+-attempts$/;
|
|
63
|
+
export function isAmbiguityResolutionBlockReason(reason) {
|
|
64
|
+
return (isRecoveryBlockReason(reason) ||
|
|
65
|
+
(reason !== undefined && legacyAmbiguityBlockReason.test(reason)));
|
|
66
|
+
}
|
|
@@ -33,3 +33,4 @@ export * from './infrastructure/transcript-store.js';
|
|
|
33
33
|
export * from './infrastructure/workspace/fake-workspace.js';
|
|
34
34
|
export * from './infrastructure/workspace/git-workspace.js';
|
|
35
35
|
export * from './application/run-reports.js';
|
|
36
|
+
export { createLocalProcessInspector } from './infrastructure/local-process-inspector.js';
|
|
@@ -33,3 +33,4 @@ export * from './infrastructure/transcript-store.js';
|
|
|
33
33
|
export * from './infrastructure/workspace/fake-workspace.js';
|
|
34
34
|
export * from './infrastructure/workspace/git-workspace.js';
|
|
35
35
|
export * from './application/run-reports.js';
|
|
36
|
+
export { createLocalProcessInspector } from './infrastructure/local-process-inspector.js';
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { ExternalExecutionInspector } from '../application/recovery-service.js';
|
|
2
|
+
/** Opaque identity includes PID reuse protection and the inspection environment. */
|
|
3
|
+
export declare function localProcessIdentity(pid: number | undefined): string | undefined;
|
|
4
|
+
export declare function createLocalProcessInspector(): ExternalExecutionInspector;
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
import { readFileSync, readlinkSync } from 'node:fs';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { ExternalExecutionKind } from '../../activities/index.js';
|
|
4
|
+
import { ExternalExecutionState } from '../contracts/vocabulary.js';
|
|
5
|
+
function environment() {
|
|
6
|
+
return {
|
|
7
|
+
boot: readFileSync('/proc/sys/kernel/random/boot_id', 'utf8').trim(),
|
|
8
|
+
namespace: readlinkSync('/proc/self/ns/pid'),
|
|
9
|
+
};
|
|
10
|
+
}
|
|
11
|
+
function processStart(pid) {
|
|
12
|
+
const stat = readFileSync(`/proc/${pid}/stat`, 'utf8');
|
|
13
|
+
const fields = stat.slice(stat.lastIndexOf(')') + 2).split(' ');
|
|
14
|
+
const start = fields[19];
|
|
15
|
+
if (fields[0] === 'Z' || fields[0] === 'X')
|
|
16
|
+
return 'terminated';
|
|
17
|
+
if (start === undefined || !/^\d+$/.test(start))
|
|
18
|
+
throw new Error('Invalid process stat');
|
|
19
|
+
return start;
|
|
20
|
+
}
|
|
21
|
+
/** Opaque identity includes PID reuse protection and the inspection environment. */
|
|
22
|
+
export function localProcessIdentity(pid) {
|
|
23
|
+
if (pid === undefined || process.platform !== 'linux')
|
|
24
|
+
return undefined;
|
|
25
|
+
try {
|
|
26
|
+
const start = processStart(pid);
|
|
27
|
+
if (start === 'terminated')
|
|
28
|
+
return undefined;
|
|
29
|
+
return JSON.stringify({ ...environment(), pid, start });
|
|
30
|
+
}
|
|
31
|
+
catch {
|
|
32
|
+
return undefined;
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
const processIdentitySchema = z
|
|
36
|
+
.object({
|
|
37
|
+
boot: z.string().min(1),
|
|
38
|
+
namespace: z.string().min(1),
|
|
39
|
+
pid: z.number().int().positive().safe(),
|
|
40
|
+
start: z.string().regex(/^\d+$/),
|
|
41
|
+
})
|
|
42
|
+
.strict();
|
|
43
|
+
function decodeIdentity(id) {
|
|
44
|
+
try {
|
|
45
|
+
const parsed = processIdentitySchema.safeParse(JSON.parse(id));
|
|
46
|
+
return parsed.success ? parsed.data : undefined;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
export function createLocalProcessInspector() {
|
|
53
|
+
return {
|
|
54
|
+
async inspect(reference) {
|
|
55
|
+
const unknown = (reason) => ({ kind: ExternalExecutionState.Unknown, reason });
|
|
56
|
+
const identity = decodeIdentity(reference.id);
|
|
57
|
+
if (reference.kind !== ExternalExecutionKind.Process ||
|
|
58
|
+
identity === undefined ||
|
|
59
|
+
process.platform !== 'linux')
|
|
60
|
+
return unknown('No inspectable local process identity; verify the runner has stopped, then resolve the ambiguous run');
|
|
61
|
+
try {
|
|
62
|
+
const current = environment();
|
|
63
|
+
if (current.boot !== identity.boot)
|
|
64
|
+
return { kind: ExternalExecutionState.Absent };
|
|
65
|
+
if (current.namespace !== identity.namespace)
|
|
66
|
+
return unknown('Runner PID namespace changed; verify the previous container has stopped, then resolve the ambiguous run');
|
|
67
|
+
try {
|
|
68
|
+
return {
|
|
69
|
+
kind: processStart(identity.pid) === identity.start
|
|
70
|
+
? ExternalExecutionState.Running
|
|
71
|
+
: ExternalExecutionState.Absent,
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
if (error instanceof Error && 'code' in error && error.code === 'ENOENT')
|
|
76
|
+
return { kind: ExternalExecutionState.Absent };
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
return unknown('Local process inspection failed; restore /proc access or verify the runner has stopped and resolve the ambiguous run');
|
|
82
|
+
}
|
|
83
|
+
},
|
|
84
|
+
};
|
|
85
|
+
}
|
|
@@ -27,6 +27,7 @@ export interface ProcessExecutionResult {
|
|
|
27
27
|
readonly failureMessage?: string;
|
|
28
28
|
}
|
|
29
29
|
export declare function runProcess(command: string, args: readonly string[], cwd: string | undefined, signal: AbortSignal, options?: ProcessExecutionOptions, shell?: boolean): {
|
|
30
|
+
readonly identity?: string;
|
|
30
31
|
readonly result: Promise<ProcessExecutionResult>;
|
|
31
32
|
cancel(): Promise<void>;
|
|
32
33
|
};
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process';
|
|
2
|
+
import { localProcessIdentity } from './local-process-inspector.js';
|
|
2
3
|
// Agent CLIs can emit arbitrarily large machine-readable transcripts. Capture
|
|
3
4
|
// raw bytes ourselves so overflow never enters a third-party string buffer.
|
|
4
5
|
export const maximumCapturedProcessOutputBytes = 1024 * 1024;
|
|
@@ -9,7 +10,11 @@ export function runProcess(command, args, cwd, signal, options, shell = false) {
|
|
|
9
10
|
...(shell && process.platform !== 'win32' ? { detached: true } : {}),
|
|
10
11
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
11
12
|
});
|
|
12
|
-
|
|
13
|
+
const identity = localProcessIdentity(child.pid);
|
|
14
|
+
return {
|
|
15
|
+
...captureProcessOutput(child, signal, options, shell),
|
|
16
|
+
...(identity === undefined ? {} : { identity }),
|
|
17
|
+
};
|
|
13
18
|
}
|
|
14
19
|
// Lifecycle coordination must share these closures so first-event-wins is atomic.
|
|
15
20
|
// eslint-disable-next-line max-lines-per-function
|
|
@@ -138,7 +138,7 @@ export function cliRunner(name, command, args, options = {}) {
|
|
|
138
138
|
return {
|
|
139
139
|
identity: {
|
|
140
140
|
kind: ExternalExecutionKind.Process,
|
|
141
|
-
id: request.runId,
|
|
141
|
+
id: process.identity ?? request.runId,
|
|
142
142
|
startedAt: new Date().toISOString(),
|
|
143
143
|
},
|
|
144
144
|
result: process.result.then((value) => value.exitCode === 0 && !value.timedOut
|
|
@@ -1,5 +1,10 @@
|
|
|
1
1
|
import { type EventEnvelope, type ProjectionDefinition } from '@atolis-hq/eventing';
|
|
2
2
|
import type { DeliveryIntentView } from '../contracts/views.js';
|
|
3
3
|
export declare function projectDeliveries(events: readonly EventEnvelope[]): readonly DeliveryIntentView[];
|
|
4
|
+
/**
|
|
5
|
+
* Intent event ids can contain path separators (GitHub comment ids embed owner/repo), which the
|
|
6
|
+
* filesystem projection store rejects. Encode only those, so existing stored keys stay valid.
|
|
7
|
+
*/
|
|
8
|
+
export declare function deliveryProjectionKey(eventId: string): string;
|
|
4
9
|
export declare const deliveryProjection: ProjectionDefinition<DeliveryIntentView | null>;
|
|
5
10
|
export declare const deliveryProjectionDefinitions: readonly ProjectionDefinition[];
|