@atolis-hq/wake 0.3.51 → 0.3.53

package/dist/src/bootstrap/composition-root.js

@@ -12,6 +12,7 @@ import { loadFakeScenarios } from './fake-scenarios.js';
 import { composeIntegrationRuntime } from './integration-runtime.js';
 import { resolveWakePaths } from './paths.js';
 import { composePersistence } from './persistence-composition.js';
+import { createFileProjectionRunSerialiser, } from './projection-runtime.js';
 import { createRunnerQuotaReporter } from './runner-quota-reporter.js';
 import { createRunnerRegistry } from './runner-registry.js';
 import { FileScheduleCheckpointStore } from './schedule-checkpoint-store.js';
@@ -130,6 +131,7 @@ export async function createCompositionRoot(wakeRoot, options = {}) {
         work,
         ids,
         wakeRoot,
+        projectionRunSerialiser: createFileProjectionRunSerialiser(paths.dataRoot),
         scheduleCheckpoints: options.scheduleCheckpoints ?? new FileScheduleCheckpointStore(paths.dataRoot),
         ...(options.decorateDeliveryAdapter === undefined
             ? {}
@@ -153,6 +155,7 @@ export async function createCompositionRoot(wakeRoot, options = {}) {
         lookup,
         orchestration,
         execution,
+        recovery,
         ...(transcriptStore === undefined ? {} : { transcriptStore }),
         runnerControls,
         controlPlane,

package/dist/src/bootstrap/integration-runtime.js

@@ -55,7 +55,7 @@ export async function composeIntegrationRuntime(input) {
     // pipeline's own catchUpProjections, the API's manual tick, and the
     // resident's standalone projection pump) shares this one instance, so
     // serializing it here in-process covers all of them without a file lock.
-    const projectionRunner = serializeRunRegisteredOnce(createRuntimeProjectionRunner(input.journal, input.projections, input.checkpoints));
+    const projectionRunner = serializeRunRegisteredOnce(createRuntimeProjectionRunner(input.journal, input.projections, input.checkpoints, input.projectionRunSerialiser));
     const delivery = new DeliveryService({
         journal: input.journal,
         intents: async () => (await input.projections.list(IntegrationStreamKind.Delivery)).map(({ value }) => value),

package/dist/src/bootstrap/projection-runtime.js

@@ -1,9 +1,10 @@
+import { join } from 'node:path';
 import { activityProjectionDefinitions } from '../activities/index.js';
 import { controlPlaneProjectionDefinitions } from '../control-plane/index.js';
 import { executionProjection, runsByWorkflowInstanceProjection } from '../execution/index.js';
 import { deliveryProjectionDefinitions } from '../integrations/index.js';
 import { orchestrationProjection, workflowDefinitionsProjection, workflowsByWorkItemProjection, } from '../orchestration/index.js';
-import { ProjectionRunner } from '../persistence/index.js';
+import { acquireFileLock, ProjectionRunner, } from '../persistence/index.js';
 import { resourceCorrelationProjection, resourceProjection, resourcesByExternalKeyProjection, workCorrelationsProjection, } from '../resources/index.js';
 import { workProjection } from '../work/index.js';
 import { analyticsProjection } from './analytics-projection.js';
@@ -25,6 +26,26 @@ export const runtimeProjectionDefinitions = [
     boardProjection,
     analyticsProjection,
 ];
-export function createRuntimeProjectionRunner(journal, projections, checkpoints) {
-    return new ProjectionRunner(journal, projections, checkpoints, runtimeProjectionDefinitions);
+export function createRuntimeProjectionRunner(journal, projections, checkpoints, serialiseRun) {
+    return new ProjectionRunner(journal, projections, checkpoints, runtimeProjectionDefinitions, serialiseRun);
+}
+export function createFileProjectionRunSerialiser(dataRoot) {
+    const path = join(dataRoot, 'locks', 'projection-runner.lock');
+    return async (operation) => {
+        while (true) {
+            const lock = await acquireFileLock(path, {
+                staleAfterMs: 60_000,
+                staleRequiresDeadProcess: true,
+            });
+            if (lock.acquired) {
+                try {
+                    return await operation();
+                }
+                finally {
+                    await lock.release();
+                }
+            }
+            await new Promise((resolve) => setTimeout(resolve, 10));
+        }
+    };
 }

package/dist/src/bootstrap/surface-api-execution-applications.js

@@ -1,4 +1,6 @@
 import { ControlStreamKind, ineligibleRunners, } from '../control-plane/index.js';
+import { ExecutionFailureCode, RunStatus } from '../execution/index.js';
+import { correlationId, EventActorKind } from '../kernel/index.js';
 import { ApiCommandStatus, presentRun } from '../surfaces/index.js';
 import { projectionMeta, sampledMeta } from './surface-api-metadata.js';
 import { projectionPage } from './surface-api-projection-pages.js';
@@ -6,6 +8,15 @@ import { withWorkflowContext } from './surface-api-run-context.js';
 import { readWorkTranscript } from './surface-api-transcripts.js';
 export function createExecutionApplications(root, now) {
     return {
+        async resolveAmbiguousRun(runId, command) {
+            const context = resolutionContext(runId, command.idempotencyKey, now);
+            const run = await root.recovery.resolve(runId, {
+                kind: RunStatus.Failed,
+                failure: { kind: ExecutionFailureCode.Unexpected, message: command.message },
+            }, context);
+            await root.orchestration.resolveExecutionFailure(run.workflowInstanceId, { activationId: run.activationId, runId: run.runId, reason: command.message }, context);
+            return commandAccepted(command, now());
+        },
         async pauseRunner(runnerId, command) {
             await root.runnerControls.pause(runnerId, command.idempotencyKey);
             return commandAccepted(command, now());
@@ -85,6 +96,15 @@ export function createExecutionApplications(root, now) {
         },
     };
 }
+function resolutionContext(runId, idempotencyKey, now) {
+    const commandId = `run:${runId}:resolve-failed:${idempotencyKey}`;
+    return {
+        commandId,
+        correlationId: correlationId(commandId),
+        occurredAt: now(),
+        actor: { kind: EventActorKind.Operator, id: 'web' },
+    };
+}
 function commandAccepted(command, acceptedAt) {
     return {
         commandId: `runner:${command.idempotencyKey}`,

package/dist/src/bootstrap/version.js

@@ -108,4 +108,4 @@ export function resolveWakeVersion(options = {}) {
         return `g${headHash.slice(0, 7)}`;
     return '0.1.0-dev';
 }
-export const wakeVersion = "gc813fba";
+export const wakeVersion = "gfcfd038";

package/dist/src/execution/application/recovery-service.js

@@ -46,8 +46,13 @@ export class RecoveryService {
         const loaded = await this.repository.load(currentRunId);
         if (loaded.view === null)
             throw new Error(`Run ${id} does not exist`);
-        if (!loaded.view.escalated || loaded.view.status !== RunStatus.Ambiguous)
+        if (!loaded.view.escalated || loaded.view.status !== RunStatus.Ambiguous) {
+            if (loaded.events?.some((event) => event.causationId === context.commandId &&
+                (event.eventType === ExecutionEventType.RunSucceeded ||
+                    event.eventType === ExecutionEventType.RunFailed)))
+                return loaded.view;
             throw new Error(`Run ${id} is not escalated`);
+        }
         const draft = resolution.kind === RunStatus.Succeeded
             ? createRunExecutionEventDraft({
                 ...resolutionMetadata(currentRunId, loaded.view, context),

package/dist/src/execution/application/run-repository.js

@@ -9,7 +9,8 @@ export class RunRepository {
     }
     async load(runId) {
         const events = await this.journal.readStream(runStream(runId));
-        return { sequence: events.length, view: foldRun(events.map(decodeRunExecutionEvent)) };
+        const decoded = events.map(decodeRunExecutionEvent);
+        return { sequence: events.length, events: decoded, view: foldRun(decoded) };
     }
     async append(runId, sequence, drafts) {
         const events = await this.journal.append(runStream(runId), sequence, drafts);

package/dist/src/orchestration/application/advance-workflow.js

@@ -80,7 +80,8 @@ export class AdvanceWorkflow {
     async resolveExecutionFailure(id, input, context) {
         const loaded = await this.repository.load(id);
         if (loaded.view === null ||
-            loaded.view.status === WorkflowStatus.Blocked ||
+            (loaded.view.status === WorkflowStatus.Blocked &&
+                !isAmbiguityResolutionBlock(loaded.view.blockReason)) ||
             loaded.view.pendingActivation?.activationId !== input.activationId ||
             loaded.view.acceptedOutcomes.includes(input.activationId))
             return loaded.view;
@@ -203,3 +204,6 @@ export class AdvanceWorkflow {
         return matchWatches(await this.listAllLoaded(), event, this.workflows, context);
     }
 }
+function isAmbiguityResolutionBlock(reason) {
+    return reason !== undefined && /^run-ambiguous-after-\d+-attempts$/.test(reason);
+}

package/dist/src/persistence/application/projection-runner.js

@@ -1,14 +1,19 @@
+async function runImmediately(operation) {
+    return operation();
+}
 export class ProjectionRunner {
     journal;
     projections;
     checkpoints;
     registered;
+    serialiseRun;
     caughtUpToGlobalPosition;
-    constructor(journal, projections, checkpoints, registered = []) {
+    constructor(journal, projections, checkpoints, registered = [], serialiseRun = runImmediately) {
         this.journal = journal;
         this.projections = projections;
         this.checkpoints = checkpoints;
         this.registered = registered;
+        this.serialiseRun = serialiseRun;
     }
     // One shared journal read for every registered definition, not one per
     // definition: readAll ultimately re-derives its content-fingerprint via
@@ -26,6 +31,9 @@ export class ProjectionRunner {
     // batch — otherwise a backlog bigger than `limit` would get marked caught
     // up after its first (partial) batch and never finish draining.
     async runRegisteredOnce(limit = 100) {
+        return this.serialiseRun(() => this.runRegisteredOnceUnlocked(limit));
+    }
+    async runRegisteredOnceUnlocked(limit) {
         const allEvents = await this.journal.readAll(0);
         const latestGlobalPosition = allEvents.at(-1)?.globalPosition ?? 0;
         if (this.caughtUpToGlobalPosition !== undefined &&
@@ -37,6 +45,9 @@ export class ProjectionRunner {
         return counts.reduce((total, count) => total + count, 0);
     }
     async runOnce(definition, limit = 100) {
+        return this.serialiseRun(() => this.runOnceUnlocked(definition, limit));
+    }
+    async runOnceUnlocked(definition, limit) {
         const consumer = `projection:${definition.name}`;
         const events = await this.journal.readAll(await this.checkpoints.load(consumer), limit);
         return this.apply(definition, consumer, events);
@@ -66,11 +77,14 @@ export class ProjectionRunner {
         return events.length;
     }
     async rebuild(definition) {
+        return this.serialiseRun(() => this.rebuildUnlocked(definition));
+    }
+    async rebuildUnlocked(definition) {
         await this.projections.clear(definition.name);
         await this.checkpoints.reset(`projection:${definition.name}`);
         let total = 0;
         while (true) {
-            const count = await this.runOnce(definition);
+            const count = await this.runOnceUnlocked(definition, 100);
             total += count;
             if (count < 100)
                 return total;

package/dist/src/surfaces/api/routes/commands.js

@@ -3,6 +3,9 @@ import { accepted, ApiPathError, decodePathSegment, invalidPath, invalidQuery, i
 export async function dispatchCommand(applications, url, body) {
     if (url.search !== '')
         return invalidQuery('Command routes do not accept query parameters');
+    const ambiguityResolution = await dispatchAmbiguousRunResolution(applications, url.pathname, body);
+    if (ambiguityResolution !== undefined)
+        return ambiguityResolution;
     const request = commandRequest(body);
     if ('status' in request)
         return request;
@@ -15,6 +18,21 @@ export async function dispatchCommand(applications, url, body) {
     const runner = await dispatchRunnerCommand(applications, url.pathname, request);
     return runner ?? problem(404, 'Not Found', `No command route for ${url.pathname}`);
 }
+async function dispatchAmbiguousRunResolution(applications, pathname, body) {
+    const match = /^\/api\/v1\/runs\/([^/]+)\/commands\/resolve-failed$/.exec(pathname);
+    if (match === null)
+        return undefined;
+    const runId = decodePathSegment(match[1]);
+    if (runId instanceof ApiPathError)
+        return invalidPath(runId.message);
+    const request = ambiguityFailureResolutionRequest(body);
+    if ('status' in request)
+        return request;
+    const operation = applications.execution.resolveAmbiguousRun;
+    if (operation === undefined)
+        return unavailable('resolve-ambiguous-run', await applications.execution.get?.(runId));
+    return accepted(await operation(runId, request), applications.now());
+}
 async function dispatchWorkCommand(applications, pathname, request) {
     const match = /^\/api\/v1\/work-items\/([^/]+)\/commands\/(freeze|unfreeze|delete|retry)$/.exec(pathname);
     if (match === null)
@@ -80,3 +98,20 @@ function commandRequest(body) {
         ? { idempotencyKey: value }
         : invalidRequest('idempotencyKey', 'Must be at most 200 characters');
 }
+function ambiguityFailureResolutionRequest(body) {
+    if (!isObject(body))
+        return invalidRequest('idempotencyKey', 'A JSON object with an idempotency key is required');
+    if (Object.keys(body).some((key) => key !== 'idempotencyKey' && key !== 'message'))
+        return invalidRequest('', 'The command body contains unknown fields');
+    const idempotencyKey = body.idempotencyKey;
+    if (typeof idempotencyKey !== 'string' || idempotencyKey.trim() === '')
+        return invalidRequest('idempotencyKey', 'Must be a non-empty string');
+    if (idempotencyKey.length > 200)
+        return invalidRequest('idempotencyKey', 'Must be at most 200 characters');
+    const message = body.message;
+    if (typeof message !== 'string' || message.trim() === '')
+        return invalidRequest('message', 'Must be a non-empty string');
+    if (message.length > 2_000)
+        return invalidRequest('message', 'Must be at most 2000 characters');
+    return { idempotencyKey, message };
+}

package/dist/src/surfaces/api/routes/execution.js

@@ -2,6 +2,7 @@ export const executionRoutes = [
     '/api/v1/runs',
     '/api/v1/runs/:runId',
     '/api/v1/runs/:runId/transcript',
+    '/api/v1/runs/:runId/commands/resolve-failed',
     '/api/v1/runners',
     '/api/v1/runners/:runnerId/commands/pause',
     '/api/v1/runners/:runnerId/commands/unpause',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.3.51",
+  "version": "0.3.53",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {