@atolis-hq/wake 0.2.28 → 0.2.29

package/dist/src/core/stale-run-reconciler.js

@@ -8,22 +8,111 @@ import { createEventEnvelope } from '../lib/event-log.js';
 // `deliverOutboundEvent` is injected rather than imported so this module does
 // not depend on the outbox module directly.
 export function createStaleRunReconciler(deps) {
+    function isTerminalLifecycle(lifecycle) {
+        return lifecycle === 'TERMINAL';
+    }
+    function recoveryOutcomeForLifecycle(lifecycle) {
+        switch (lifecycle) {
+            case 'CREATED':
+            case 'CLAIMED':
+            case 'PREPARING':
+                return 'CANCELED_BY_RECONCILIATION';
+            case 'PROCESS_STARTING':
+            case 'RUNNING':
+            case 'FINALISING':
+                return 'STALLED';
+            case 'TERMINAL':
+                return 'SUPERSEDED';
+        }
+    }
     async function staleReason(record, now) {
         if (record.status !== 'running') {
             return null;
         }
-        if (deps.isRunningRecordActive !== undefined && !(await deps.isRunningRecordActive(record))) {
+        if (isTerminalLifecycle(record.lifecycle)) {
+            return null;
+        }
+        if (deps.isRunningRecordActive === undefined) {
+            if (record.lifecycle !== 'CREATED' &&
+                record.lifecycle !== 'CLAIMED' &&
+                record.lifecycle !== 'PREPARING') {
+                const startedAtMs = Date.parse(record.startedAt);
+                if (!Number.isFinite(startedAtMs)) {
+                    return 'timeout';
+                }
+                return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
+            }
+            return 'startup-recovery';
+        }
+        const active = await deps.isRunningRecordActive(record);
+        if (active) {
+            const startedAtMs = Date.parse(record.startedAt);
+            if (!Number.isFinite(startedAtMs)) {
+                return 'timeout';
+            }
+            return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
+        }
+        if (record.lifecycle === 'RUNNING' || record.lifecycle === 'PROCESS_STARTING') {
             return 'runner-lock-not-active';
         }
-        const startedAtMs = Date.parse(record.startedAt);
-        if (!Number.isFinite(startedAtMs)) {
-            return 'timeout';
+        return 'startup-recovery';
+    }
+    async function deliverRecoveredLabels(projection, runId, finishedAt) {
+        await deps.deliverOutboundEvent(createLabelsEvent({
+            projection,
+            runId,
+            statusLabel: 'wake:status.failed',
+            stageLabel: stageLabelForStage(projection.wake.stage),
+            workflowLabel: workflowLabelForWorkflowName(workflowNameForProjection(projection, deps.config)),
+            occurredAt: finishedAt,
+        }));
+    }
+    async function recoverMissingRunRecordClaims(runRecords, finishedAt) {
+        const knownRunIds = new Set(runRecords.map((record) => record.runId));
+        const projections = await deps.stateStore.listIssueStates();
+        for (const projection of projections) {
+            const runId = projection.wake.lastRunId;
+            if (runId === undefined ||
+                knownRunIds.has(runId) ||
+                runRecords.some((record) => record.workItemKey === projection.workItemKey) ||
+                projection.context.lastRunSentinel !== undefined) {
+                continue;
+            }
+            const event = createEventEnvelope({
+                eventId: `${runId}-missing-run-record-recovered`,
+                workItemKey: projection.workItemKey,
+                streamScope: 'work-item',
+                direction: 'internal',
+                sourceSystem: 'wake',
+                sourceEventType: 'wake.run.completed',
+                sourceRefs: {
+                    repo: projection.issue.repo,
+                    issueNumber: projection.issue.number,
+                    runId,
+                },
+                occurredAt: finishedAt,
+                ingestedAt: finishedAt,
+                trigger: 'immediate',
+                payload: {
+                    action: projection.context.lastRunAction,
+                    sentinel: 'FAILED',
+                    executionOutcome: 'CANCELED_BY_RECONCILIATION',
+                    runId,
+                    reason: 'runner:missing-run-record',
+                },
+            });
+            await deps.stateStore.appendEventEnvelope(event);
+            await deps.projectionUpdater.rebuildFromEvents([event]);
+            const updatedProjection = await deps.stateStore.readIssueState(projection.workItemKey);
+            if (updatedProjection !== null) {
+                await deliverRecoveredLabels(updatedProjection, runId, finishedAt);
+            }
         }
-        return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
     }
     async function reconcileStaleRunningRecords(now) {
         const finishedAt = now.toISOString();
         const runRecords = await deps.stateStore.listRunRecords();
+        await recoverMissingRunRecordClaims(runRecords, finishedAt);
         const staleRecords = [];
         for (const record of runRecords) {
             const reason = await staleReason(record, now);
@@ -46,6 +135,7 @@ export function createStaleRunReconciler(deps) {
             if (projection === null || projection.wake.lastRunId !== record.runId || newerCompletedRun) {
                 await deps.stateStore.writeRunRecord({
                     ...record,
+                    lifecycle: 'TERMINAL',
                     status: 'superseded',
                     finishedAt,
                     executionOutcome: 'SUPERSEDED',
@@ -58,9 +148,10 @@ export function createStaleRunReconciler(deps) {
                 });
                 continue;
             }
-            const staleExecutionOutcome = reason === 'timeout' ? 'TIMED_OUT' : 'STALLED';
+            const staleExecutionOutcome = reason === 'timeout' ? 'TIMED_OUT' : recoveryOutcomeForLifecycle(record.lifecycle);
             await deps.stateStore.writeRunRecord({
                 ...record,
+                lifecycle: 'TERMINAL',
                 status: 'failed',
                 finishedAt,
                 sentinel: 'FAILED',
@@ -69,6 +160,7 @@ export function createStaleRunReconciler(deps) {
                 metadata: {
                     ...record.metadata,
                     reconciledBy: 'stale-running-record',
+                    recoveryLifecycle: record.lifecycle,
                     staleReason: reason,
                     timeoutMs: deps.runnerTimeoutMs(),
                 },
@@ -93,7 +185,11 @@ export function createStaleRunReconciler(deps) {
                     sentinel: 'FAILED',
                     executionOutcome: staleExecutionOutcome,
                     runId: record.runId,
-                    reason: reason === 'timeout' ? 'runner:stale-timeout' : 'runner:orphaned-process',
+                    reason: reason === 'timeout'
+                        ? 'runner:stale-timeout'
+                        : reason === 'startup-recovery'
+                            ? `runner:recover-${record.lifecycle.toLowerCase()}`
+                            : 'runner:orphaned-process',
                     ...(record.routing === undefined ? {} : { routing: record.routing }),
                 },
             });
@@ -101,14 +197,7 @@ export function createStaleRunReconciler(deps) {
             await deps.projectionUpdater.rebuildFromEvents([runCompletedEvent]);
             const updatedProjection = await deps.stateStore.readIssueState(projection.workItemKey);
             if (updatedProjection !== null) {
-                await deps.deliverOutboundEvent(createLabelsEvent({
-                    projection: updatedProjection,
-                    runId: record.runId,
-                    statusLabel: 'wake:status.failed',
-                    stageLabel: stageLabelForStage(updatedProjection.wake.stage),
-                    workflowLabel: workflowLabelForWorkflowName(workflowNameForProjection(updatedProjection, deps.config)),
-                    occurredAt: finishedAt,
-                }));
+                await deliverRecoveredLabels(updatedProjection, record.runId, finishedAt);
             }
         }
     }

package/dist/src/core/tick-runner.js

@@ -446,10 +446,18 @@ export function createTickRunner(deps) {
                 repo: candidate.issue.repo,
                 issueNumber: candidate.issue.number,
                 action,
+                lifecycle: 'CREATED',
                 status: 'running',
                 startedAt: nowIso,
+                routing,
             };
             await deps.stateStore.writeRunRecord(runningRecord);
+            async function transitionRunLifecycle(lifecycle) {
+                await deps.stateStore.writeRunRecord({
+                    ...(await deps.stateStore.readRunRecord(runId)),
+                    lifecycle,
+                });
+            }
             const claimedAt = eventStampNow();
             const claimedEvent = createEventEnvelope({
                 eventId: `${runId}-claimed`,
@@ -473,6 +481,7 @@ export function createTickRunner(deps) {
                 },
             });
             await deps.stateStore.appendEventEnvelope(claimedEvent);
+            await transitionRunLifecycle('CLAIMED');
             await projectionUpdater.rebuildFromEvents([claimedEvent]);
             await deliverOutboundEvent(createLabelsEvent({
                 projection: candidate,
@@ -483,6 +492,7 @@ export function createTickRunner(deps) {
                 occurredAt: eventStampNow(),
             }));
             try {
+                await transitionRunLifecycle('PREPARING');
                 const prepareResult = workspaceMode === 'branch'
                     ? await deps.workspaceManager.prepareWorkspace({
                         workId: candidate.workItemKey,
@@ -499,7 +509,18 @@ export function createTickRunner(deps) {
                 const upstreamChanges = 'upstreamChanges' in prepareResult && typeof prepareResult.upstreamChanges === 'string'
                     ? prepareResult.upstreamChanges
                     : undefined;
+                const preparedRecord = (await deps.stateStore.readRunRecord(runId));
+                await deps.stateStore.writeRunRecord({
+                    ...preparedRecord,
+                    lifecycle: 'PROCESS_STARTING',
+                    metadata: {
+                        ...preparedRecord.metadata,
+                        ...(workspacePath === undefined ? {} : { workspacePath }),
+                        workspaceMode,
+                    },
+                });
                 const recentEvents = await deps.stateStore.listEventEnvelopesForWorkItem(candidate.workItemKey, 6);
+                await transitionRunLifecycle('RUNNING');
                 const runnerResult = await deps.runner.run({
                     action,
                     projection: candidate,
@@ -583,8 +604,11 @@ export function createTickRunner(deps) {
                         : sentinel === 'AWAITING_APPROVAL'
                             ? 'AWAITING_APPROVAL'
                             : undefined;
+                await transitionRunLifecycle('FINALISING');
+                const finalisingRecord = (await deps.stateStore.readRunRecord(runId));
                 await deps.stateStore.writeRunRecord({
-                    ...runningRecord,
+                    ...finalisingRecord,
+                    lifecycle: 'TERMINAL',
                     status: sentinel === 'DONE'
                         ? 'completed'
                         : sentinel === 'BLOCKED'
@@ -600,7 +624,10 @@ export function createTickRunner(deps) {
                     summary: parsedRunnerResult.body,
                     ...(runnerResult.routing === undefined ? {} : { routing: runnerResult.routing }),
                     ...(runnerResult.tokenUsage === undefined ? {} : { tokenUsage: runnerResult.tokenUsage }),
-                    metadata: resultMetadata,
+                    metadata: {
+                        ...finalisingRecord.metadata,
+                        ...resultMetadata,
+                    },
                 });
                 const runCompletedEvent = createEventEnvelope({
                     eventId: `${runId}-completed`,
@@ -684,8 +711,10 @@ export function createTickRunner(deps) {
             catch (err) {
                 const finishedAt = deps.clock.now().toISOString();
                 const sentinel = 'FAILED';
+                const failedRecord = (await deps.stateStore.readRunRecord(runId)) ?? runningRecord;
                 await deps.stateStore.writeRunRecord({
-                    ...runningRecord,
+                    ...failedRecord,
+                    lifecycle: 'TERMINAL',
                     status: 'failed',
                     finishedAt,
                     sentinel,

package/dist/src/domain/schema.js

@@ -34,8 +34,18 @@ export const workflowOutcomeValues = [
     'AWAITING_INPUT',
     'CHANGES_REQUESTED',
 ];
+export const executionAttemptLifecycleValues = [
+    'CREATED',
+    'CLAIMED',
+    'PREPARING',
+    'PROCESS_STARTING',
+    'RUNNING',
+    'FINALISING',
+    'TERMINAL',
+];
 export const executionOutcomeSchema = z.enum(executionOutcomeValues);
 export const workflowOutcomeSchema = z.enum(workflowOutcomeValues);
+export const executionAttemptLifecycleSchema = z.enum(executionAttemptLifecycleValues);
 export const defaultAgentIdentity = 'Wake';
 export const defaultSmokePrompt = `This is ${defaultAgentIdentity}, reply with "hi ${defaultAgentIdentity} only"`;
 const modelOverridesSchema = z
@@ -291,7 +301,20 @@ const runTokenUsageSchema = z.object({
     costUsd: z.number().nonnegative().optional(),
     turns: z.number().nonnegative().optional(),
 });
-export const runRecordSchema = z.object({
+function legacyRunLifecycle(input) {
+    if (input.lifecycle !== undefined) {
+        return input;
+    }
+    return {
+        ...input,
+        lifecycle: input.status === 'running' ? 'RUNNING' : 'TERMINAL',
+    };
+}
+export const runRecordSchema = z.preprocess((input) => {
+    return input !== null && typeof input === 'object'
+        ? legacyRunLifecycle(input)
+        : input;
+}, z.object({
     schemaVersion: z.literal(1),
     runId: z.string(),
     // The work item this run belongs to. Required: run records are Wake-owned
@@ -306,7 +329,15 @@ export const runRecordSchema = z.object({
     repo: z.string(),
     issueNumber: z.number().int().positive(),
     action: identifierSchema,
-    status: z.enum(['running', 'completed', 'awaiting-approval', 'blocked', 'failed', 'superseded']),
+    lifecycle: executionAttemptLifecycleSchema,
+    status: z.enum([
+        'running',
+        'completed',
+        'awaiting-approval',
+        'blocked',
+        'failed',
+        'superseded',
+    ]),
     startedAt: isoTimestampSchema,
     finishedAt: isoTimestampSchema.optional(),
     sessionId: z.string().optional(),
@@ -317,7 +348,7 @@ export const runRecordSchema = z.object({
     routing: runnerRoutingSchema.optional(),
     tokenUsage: runTokenUsageSchema.optional(),
     metadata: z.record(z.string(), z.unknown()).optional(),
-});
+}));
 const runnerHealthEntrySchema = z.object({
     pausedUntil: isoTimestampSchema.optional(),
     // 'reported' = CLI told us the real reset time; 'estimated' = exponential

package/dist/src/version.js

@@ -124,4 +124,4 @@ export function resolveWakeVersion(options = {}) {
     }
     return '0.1.0-dev';
 }
-export const wakeVersion = "g73487cf";
+export const wakeVersion = "gf58d7a3";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.2.28",
+  "version": "0.2.29",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {