@atolis-hq/wake 0.2.95 → 0.2.96

package/dist/src/core/stale-run-reconciler.js

@@ -1,7 +1,10 @@
 import { createLabelsEvent } from './event-builders.js';
 import { RUN_COMPLETED_EVENT } from '../domain/event-types.js';
 import { labelsForWorkItem } from '../domain/work-item-labels.js';
+import { readJsonFile, writeJsonFile } from '../lib/json-file.js';
+import { isMissingPathError } from '../lib/state-health.js';
 import { createEventEnvelope } from '../lib/event-log.js';
+import { join } from 'node:path';
 // Reconciles run records still marked `running` past the runner timeout: a
 // record whose work item has already moved on is superseded, otherwise it is
 // failed and a completion event replayed so the projection stops waiting on it.
@@ -56,6 +59,97 @@ export function createStaleRunReconciler(deps) {
             retrySafety,
         };
     }
+    function watcherStateFile(key) {
+        return join(deps.stateStore.paths.dataRoot, 'watchers', `${key}.json`);
+    }
+    function isWatcherTrigger(value) {
+        if (value === null || typeof value !== 'object')
+            return false;
+        const record = value;
+        return ((record.kind === 'event' && typeof record.eventId === 'string') ||
+            (record.kind === 'schedule' && typeof record.slot === 'string'));
+    }
+    function watcherMetadata(record) {
+        const metadata = record.metadata;
+        if (metadata?.watcher !== true)
+            return {};
+        return {
+            ...(typeof metadata.watcherStateKey === 'string'
+                ? { stateKey: metadata.watcherStateKey }
+                : {}),
+            ...(isWatcherTrigger(metadata.watcherTrigger) ? { trigger: metadata.watcherTrigger } : {}),
+        };
+    }
+    async function readWatcherState(key) {
+        try {
+            const raw = await readJsonFile(watcherStateFile(key));
+            if (raw === null || typeof raw !== 'object')
+                return null;
+            const record = raw;
+            return record.schemaVersion === 1 &&
+                record.key === key &&
+                typeof record.updatedAt === 'string'
+                ? {
+                    schemaVersion: 1,
+                    key,
+                    ...(typeof record.lastDispatchedEventId === 'string'
+                        ? { lastDispatchedEventId: record.lastDispatchedEventId }
+                        : {}),
+                    ...(typeof record.lastDispatchedSlot === 'string'
+                        ? { lastDispatchedSlot: record.lastDispatchedSlot }
+                        : {}),
+                    ...(typeof record.failureCount === 'number' &&
+                        Number.isInteger(record.failureCount) &&
+                        record.failureCount >= 0
+                        ? { failureCount: record.failureCount }
+                        : {}),
+                    updatedAt: record.updatedAt,
+                }
+                : null;
+        }
+        catch (error) {
+            if (isMissingPathError(error))
+                return null;
+            throw error;
+        }
+    }
+    async function writeWatcherState(key, patch, updatedAt) {
+        const current = await readWatcherState(key);
+        await writeJsonFile(watcherStateFile(key), {
+            schemaVersion: 1,
+            key,
+            ...(current?.lastDispatchedEventId === undefined
+                ? {}
+                : { lastDispatchedEventId: current.lastDispatchedEventId }),
+            ...(current?.lastDispatchedSlot === undefined
+                ? {}
+                : { lastDispatchedSlot: current.lastDispatchedSlot }),
+            ...(current?.failureCount === undefined ? {} : { failureCount: current.failureCount }),
+            ...patch,
+            updatedAt,
+        });
+    }
+    function watcherCursorPatch(trigger) {
+        return {
+            ...(trigger.kind === 'event'
+                ? { lastDispatchedEventId: trigger.eventId }
+                : { lastDispatchedSlot: trigger.slot }),
+            failureCount: 0,
+        };
+    }
+    async function updateWatcherStateAfterCompletion(input) {
+        if (input.stateKey === undefined || input.trigger === undefined)
+            return;
+        if (input.retrySafety === 'SAFE_TO_RETRY' || input.retrySafety === 'SAFE_TO_RESUME') {
+            const current = await readWatcherState(input.stateKey);
+            const failureCount = (current?.failureCount ?? 0) + 1;
+            if (failureCount < deps.config.retry.maxFailureRetries) {
+                await writeWatcherState(input.stateKey, { failureCount }, input.updatedAt);
+                return;
+            }
+        }
+        await writeWatcherState(input.stateKey, watcherCursorPatch(input.trigger), input.updatedAt);
+    }
     async function staleReason(record, now) {
         if (record.status !== 'running') {
             return null;
@@ -166,11 +260,15 @@ export function createStaleRunReconciler(deps) {
                 candidate.runId !== record.runId &&
                 candidate.status !== 'running' &&
                 Date.parse(candidate.startedAt) > Date.parse(record.startedAt));
-            // Equivalent to the previous `projection?.wake.lastRunId !== record.runId`,
-            // spelled out so the non-null projection is available below for its
-            // workItemKey.
-            if (projection === null || projection.wake.lastRunId !== record.runId || newerCompletedRun) {
-                const fullRecord = (await deps.stateStore.readRunRecord(record.runId)) ?? record;
+            const fullRecord = (await deps.stateStore.readRunRecord(record.runId)) ?? record;
+            const watcher = watcherMetadata(fullRecord);
+            const watcherRun = watcher.trigger !== undefined;
+            // Parent-stage runs must still be the projection's latest run. Watcher
+            // runs are isolated child runs, so their claim events intentionally do
+            // not replace the parent's wake.lastRunId.
+            if (projection === null ||
+                (!watcherRun && projection.wake.lastRunId !== record.runId) ||
+                newerCompletedRun) {
                 await deps.stateStore.writeRunRecord({
                     ...fullRecord,
                     lifecycle: 'TERMINAL',
@@ -188,7 +286,6 @@ export function createStaleRunReconciler(deps) {
             }
             const staleExecutionOutcome = reason === 'timeout' ? 'TIMED_OUT' : recoveryOutcomeForLifecycle(record.lifecycle);
             const failureContext = classifyReconciledFailure(record);
-            const fullRecord = (await deps.stateStore.readRunRecord(record.runId)) ?? record;
             await deps.stateStore.writeRunRecord({
                 ...fullRecord,
                 lifecycle: 'TERMINAL',
@@ -234,12 +331,19 @@ export function createStaleRunReconciler(deps) {
                             ? `runner:recover-${record.lifecycle.toLowerCase()}`
                             : 'runner:orphaned-process',
                     ...(record.routing === undefined ? {} : { routing: record.routing }),
+                    ...(watcherRun ? { watcherRun: true, watcherTrigger: watcher.trigger } : {}),
                 },
             });
             await deps.stateStore.appendEventEnvelope(runCompletedEvent);
             await deps.projectionUpdater.rebuildFromEvents([runCompletedEvent]);
+            await updateWatcherStateAfterCompletion({
+                stateKey: watcher.stateKey,
+                trigger: watcher.trigger,
+                retrySafety: failureContext.retrySafety,
+                updatedAt: finishedAt,
+            });
             const updatedProjection = await deps.stateStore.readIssueState(projection.workItemKey);
-            if (updatedProjection !== null) {
+            if (!watcherRun && updatedProjection !== null) {
                 await deliverRecoveredLabels(updatedProjection, record.runId, finishedAt);
             }
         }

package/dist/src/core/tick-runner.js

@@ -978,6 +978,11 @@ export function createTickRunner(deps) {
                     ...(typeof record.lastDispatchedSlot === 'string'
                         ? { lastDispatchedSlot: record.lastDispatchedSlot }
                         : {}),
+                    ...(typeof record.failureCount === 'number' &&
+                        Number.isInteger(record.failureCount) &&
+                        record.failureCount >= 0
+                        ? { failureCount: record.failureCount }
+                        : {}),
                     updatedAt: record.updatedAt,
                 }
                 : null;
@@ -999,10 +1004,36 @@ export function createTickRunner(deps) {
             ...(current?.lastDispatchedSlot === undefined
                 ? {}
                 : { lastDispatchedSlot: current.lastDispatchedSlot }),
+            ...(current?.failureCount === undefined ? {} : { failureCount: current.failureCount }),
             ...patch,
             updatedAt: deps.clock.now().toISOString(),
         });
     }
+    function watcherCursorPatch(trigger) {
+        return {
+            ...(trigger.kind === 'event'
+                ? { lastDispatchedEventId: trigger.eventId }
+                : { lastDispatchedSlot: trigger.slot }),
+            failureCount: 0,
+        };
+    }
+    function isRetryEligibleWatcherFailure(input) {
+        return (input.sentinel === 'FAILED' &&
+            (input.retrySafety === 'SAFE_TO_RETRY' || input.retrySafety === 'SAFE_TO_RESUME'));
+    }
+    async function updateWatcherStateAfterCompletion(input) {
+        if (input.key === undefined || input.trigger === undefined)
+            return;
+        if (isRetryEligibleWatcherFailure(input)) {
+            const current = await readWatcherState(input.key);
+            const failureCount = (current?.failureCount ?? 0) + 1;
+            if (failureCount < deps.config.retry.maxFailureRetries) {
+                await writeWatcherState(input.key, { failureCount });
+                return;
+            }
+        }
+        await writeWatcherState(input.key, watcherCursorPatch(input.trigger));
+    }
     async function nextWatcherDispatch(projections, now) {
         for (const projection of projections) {
             // A closed issue can still carry a stale status label (e.g. squash-merged
@@ -1044,15 +1075,16 @@ export function createTickRunner(deps) {
                     const cursorIndex = state?.lastDispatchedEventId === undefined
                         ? -1
                         : matchingEvents.findIndex((event) => event.eventId === state.lastDispatchedEventId);
-                    const newest = matchingEvents.slice(cursorIndex + 1).at(-1);
-                    if (newest !== undefined) {
+                    const undispatched = matchingEvents.slice(cursorIndex + 1);
+                    const next = (state?.failureCount ?? 0) > 0 ? undispatched.at(0) : undispatched.at(-1);
+                    if (next !== undefined) {
                         return {
                             projection,
                             parentWorkflowName,
                             parentStage: projection.wake.stage,
                             watcherIndex,
                             targetWorkflowName: watcher.workflow,
-                            trigger: { kind: 'event', eventId: newest.eventId },
+                            trigger: { kind: 'event', eventId: next.eventId },
                         };
                     }
                 }
@@ -1492,6 +1524,7 @@ export function createTickRunner(deps) {
                         ? {
                             watcher: true,
                             watcherWorkflow: workflowName,
+                            watcherStateKey: watcherStateKeyForRun,
                             watcherTrigger: watcherTriggerForRun,
                         }
                         : {}),
@@ -1587,11 +1620,6 @@ export function createTickRunner(deps) {
             }
             await transitionRunLifecycle('CLAIMED');
             await projectionUpdater.rebuildFromEvents([claimedEvent]);
-            if (watcherStateKeyForRun !== undefined && watcherTriggerForRun !== undefined) {
-                await writeWatcherState(watcherStateKeyForRun, watcherTriggerForRun.kind === 'event'
-                    ? { lastDispatchedEventId: watcherTriggerForRun.eventId }
-                    : { lastDispatchedSlot: watcherTriggerForRun.slot });
-            }
             // Watcher runs execute a *different* workflow's stage (e.g. plan-review)
             // against the same parent projection, not a stage transition of the
             // parent's own workflow — writing labels here would stamp the child
@@ -2055,6 +2083,12 @@ export function createTickRunner(deps) {
                 });
                 await deps.stateStore.appendEventEnvelope(runCompletedEvent);
                 await projectionUpdater.rebuildFromEvents([runCompletedEvent]);
+                await updateWatcherStateAfterCompletion({
+                    key: watcherStateKeyForRun,
+                    trigger: watcherTriggerForRun,
+                    sentinel,
+                    retrySafety: failureContext?.retrySafety,
+                });
                 if (!watcherRun) {
                     await deliverOutboundEvent(createLabelsEvent({
                         projection: candidate,
@@ -2254,6 +2288,12 @@ export function createTickRunner(deps) {
                 });
                 await deps.stateStore.appendEventEnvelope(runCompletedEvent);
                 await projectionUpdater.rebuildFromEvents([runCompletedEvent]);
+                await updateWatcherStateAfterCompletion({
+                    key: watcherStateKeyForRun,
+                    trigger: watcherTriggerForRun,
+                    sentinel,
+                    retrySafety: failureContext.retrySafety,
+                });
                 // See the matching guard on the claim path above: a watcher run's
                 // failure is not the parent's own action failing, so it must not
                 // stamp the child workflow's stage/workflow onto the parent's labels.

package/dist/src/version.js

@@ -124,4 +124,4 @@ export function resolveWakeVersion(options = {}) {
     }
     return '0.1.0-dev';
 }
-export const wakeVersion = "g803f372";
+export const wakeVersion = "g49465cd";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.2.95",
+  "version": "0.2.96",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {