@atolis-hq/wake 0.2.12 → 0.2.14

package/dist/src/adapters/docker/docker-cli.js

@@ -57,6 +57,9 @@ function buildRunArgs(input) {
 }
 export function createDockerCli(deps) {
     return {
+        async inspectContainerImage(containerName) {
+            return (await deps.inspectContainerImage?.(containerName)) ?? null;
+        },
         async build(input) {
             const buildArgFlags = Object.entries(input.buildArgs ?? {}).flatMap(([key, value]) => [
                 '--build-arg',

package/dist/src/cli/sandbox-command.js

@@ -148,6 +148,9 @@ export async function runSandboxCommand(input) {
         await runStopCommand({
             args: input.args.slice(1),
             stateStore: input.stateStore,
+            ...(input.recoverActiveRuns === undefined
+                ? {}
+                : { recoverActiveRuns: input.recoverActiveRuns }),
             docker: input.docker,
             containerName: input.config.sandbox.containerName,
             sleep: input.sleep,
@@ -174,6 +177,9 @@ export async function runSandboxCommand(input) {
             imageRepository: input.config.sandbox.imageRepository,
             containerName: input.config.sandbox.containerName,
             stateStore: input.stateStore,
+            ...(input.recoverActiveRuns === undefined
+                ? {}
+                : { recoverActiveRuns: input.recoverActiveRuns }),
             docker: input.docker,
             git: input.selfUpdate.git,
             issueReporter: input.selfUpdate.issueReporter,

package/dist/src/cli/self-update-command.js

@@ -12,6 +12,13 @@ const START_PROCESS_CHECK = [
         'tr "\\0" " " < "/proc/$pid/cmdline" | grep -F "node /app/dist/src/main.js start --wake-root /wake" >/dev/null',
     ].join(' && '),
 ];
+function tagFromImage(imageRepository, image) {
+    if (image === null) {
+        return null;
+    }
+    const prefix = `${imageRepository}:`;
+    return image.startsWith(prefix) ? image.slice(prefix.length) : null;
+}
 function readFlag(name, args) {
     const index = args.indexOf(name);
     return index === -1 ? undefined : args[index + 1];
@@ -65,10 +72,20 @@ export async function runSelfUpdateCommand(input) {
     }
     await waitForActiveRuns({
         listRunRecords: input.stateStore.listRunRecords,
+        ...(input.recoverActiveRuns === undefined
+            ? {}
+            : { recoverActiveRuns: input.recoverActiveRuns }),
         sleep: input.sleep,
         logger: input.logger,
     });
     const newImage = `${input.imageRepository}:${tag}`;
+    const previousImage = (await input.docker.inspectContainerImage?.(input.containerName)) ?? null;
+    const previousImageTag = tagFromImage(input.imageRepository, previousImage);
+    const rollbackImage = previousImage ??
+        (ledger.lastKnownGoodTag !== null
+            ? `${input.imageRepository}:${ledger.lastKnownGoodTag}`
+            : null);
+    const rollbackTag = previousImageTag ?? ledger.lastKnownGoodTag;
     const updateInput = {
         containerName: input.containerName,
         wakeRoot: input.wakeRoot,
@@ -106,9 +123,10 @@ export async function runSelfUpdateCommand(input) {
     catch (error) {
         const reason = error instanceof Error ? error.message : String(error);
         input.logger.error(`[self-update] rollout of ${tag} failed: ${reason}`);
-        if (ledger.lastKnownGoodTag !== null) {
-            const rollbackImage = `${input.imageRepository}:${ledger.lastKnownGoodTag}`;
-            await input.git.checkoutTag(ledger.lastKnownGoodTag);
+        if (rollbackImage !== null) {
+            if (rollbackTag !== null) {
+                await input.git.checkoutTag(rollbackTag);
+            }
             await input.docker.update({ ...updateInput, image: rollbackImage });
             input.logger.info('[self-update] recreated rollback container; entrypoint will keep wake start running');
             await verifyResidentStart({
@@ -119,21 +137,21 @@ export async function runSelfUpdateCommand(input) {
                 sleep: input.sleep,
                 context: 'rollback',
             });
-            input.logger.info(`[self-update] rolled back to ${ledger.lastKnownGoodTag}`);
+            input.logger.info(`[self-update] rolled back to ${rollbackTag ?? rollbackImage}`);
         }
         else {
             input.logger.error('[self-update] no previous known-good tag to roll back to');
         }
         await input.writeLedger({
-            lastAppliedTag: ledger.lastKnownGoodTag,
-            lastKnownGoodTag: ledger.lastKnownGoodTag,
+            lastAppliedTag: rollbackTag,
+            lastKnownGoodTag: rollbackTag,
             badTags: [...ledger.badTags, { tag, reason, recordedAt: new Date().toISOString() }],
         });
         try {
             await input.issueReporter.createIssue({
                 title: `Self-update to ${tag} failed and was rolled back`,
                 body: [
-                    `Automated update to \`${tag}\` failed during rollout and was rolled back to \`${ledger.lastKnownGoodTag ?? 'unknown'}\`.`,
+                    `Automated update to \`${tag}\` failed during rollout and was rolled back to \`${rollbackTag ?? rollbackImage ?? 'unknown'}\`.`,
                     '',
                     '```',
                     reason,

package/dist/src/cli/stop-command.js

@@ -4,6 +4,7 @@ export async function waitForActiveRuns(input) {
     const pollIntervalMs = input.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
     const startedAt = Date.now();
     for (;;) {
+        await input.recoverActiveRuns?.();
         const records = await input.listRunRecords();
         const activeRuns = records.filter((record) => record.status === 'running');
         if (activeRuns.length === 0) {
@@ -32,6 +33,9 @@ function readNumberFlag(name, args) {
 export async function runStopCommand(input) {
     await waitForActiveRuns({
         listRunRecords: input.stateStore.listRunRecords,
+        ...(input.recoverActiveRuns === undefined
+            ? {}
+            : { recoverActiveRuns: input.recoverActiveRuns }),
         sleep: input.sleep,
         pollIntervalMs: readNumberFlag('--poll-interval-ms', input.args),
         timeoutMs: readNumberFlag('--timeout-ms', input.args),

package/dist/src/core/active-run-recovery.js

@@ -0,0 +1,39 @@
+import { readFileLockStatus } from '../lib/lock.js';
+import { maxConfiguredRunnerTimeoutMs } from '../domain/runner-routing.js';
+import { createOutbox } from './outbox.js';
+import { createProjectionUpdater } from './projection-updater.js';
+import { createStaleRunReconciler } from './stale-run-reconciler.js';
+export function createActiveRunRecovery(deps) {
+    const projectionUpdater = createProjectionUpdater({
+        stateStore: deps.stateStore,
+        resourceIndex: deps.resourceIndex,
+        config: deps.config,
+    });
+    const { deliverOutboundEvent } = createOutbox({
+        clock: deps.clock,
+        stateStore: deps.stateStore,
+        projectionUpdater,
+    });
+    async function isRunningRecordActive(record) {
+        const lock = await readFileLockStatus(deps.stateStore.paths.runnerLockFile, {
+            expectedCommandIncludes: process.argv[1] === undefined ? [] : [process.argv[1]],
+        });
+        if (!lock.active || lock.metadata === undefined) {
+            return false;
+        }
+        return Date.parse(lock.metadata.acquiredAt) <= Date.parse(record.startedAt);
+    }
+    const { reconcileStaleRunningRecords } = createStaleRunReconciler({
+        config: deps.config,
+        stateStore: deps.stateStore,
+        projectionUpdater,
+        runnerTimeoutMs: () => maxConfiguredRunnerTimeoutMs(deps.config),
+        isRunningRecordActive,
+        deliverOutboundEvent,
+    });
+    return {
+        recoverActiveRuns() {
+            return reconcileStaleRunningRecords(deps.clock.now());
+        },
+    };
+}

package/dist/src/core/stale-run-reconciler.js

@@ -8,21 +8,30 @@ import { createEventEnvelope } from '../lib/event-log.js';
 // `deliverOutboundEvent` is injected rather than imported so this module does
 // not depend on the outbox module directly.
 export function createStaleRunReconciler(deps) {
-    function isStaleRunningRecord(record, now) {
+    async function staleReason(record, now) {
         if (record.status !== 'running') {
-            return false;
+            return null;
+        }
+        if (deps.isRunningRecordActive !== undefined && !(await deps.isRunningRecordActive(record))) {
+            return 'runner-lock-not-active';
         }
         const startedAtMs = Date.parse(record.startedAt);
         if (!Number.isFinite(startedAtMs)) {
-            return true;
+            return 'timeout';
         }
-        return now.getTime() - startedAtMs >= deps.runnerTimeoutMs();
+        return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
     }
     async function reconcileStaleRunningRecords(now) {
         const finishedAt = now.toISOString();
         const runRecords = await deps.stateStore.listRunRecords();
-        const staleRecords = runRecords.filter((record) => isStaleRunningRecord(record, now));
-        for (const record of staleRecords) {
+        const staleRecords = [];
+        for (const record of runRecords) {
+            const reason = await staleReason(record, now);
+            if (reason !== null) {
+                staleRecords.push({ record, reason });
+            }
+        }
+        for (const { record, reason } of staleRecords) {
             // Run records carry the work item they belong to, so this is a direct
             // O(1) read — no scan, no index, no source ambiguity. The record's
             // repo/issueNumber are representation content and take no part in it.
@@ -57,6 +66,7 @@ export function createStaleRunReconciler(deps) {
                 metadata: {
                     ...record.metadata,
                     reconciledBy: 'stale-running-record',
+                    staleReason: reason,
                     timeoutMs: deps.runnerTimeoutMs(),
                 },
             });
@@ -79,7 +89,7 @@ export function createStaleRunReconciler(deps) {
                     action: record.action,
                     sentinel: 'FAILED',
                     runId: record.runId,
-                    reason: 'runner:stale-timeout',
+                    reason: reason === 'timeout' ? 'runner:stale-timeout' : 'runner:orphaned-process',
                     ...(record.routing === undefined ? {} : { routing: record.routing }),
                 },
             });

package/dist/src/core/tick-runner.js

@@ -1,7 +1,7 @@
 import { createLifecycleService } from './lifecycle-service.js';
 import { createPolicyEngine } from './policy-engine.js';
 import { createProjectionUpdater } from './projection-updater.js';
-import { acquireFileLock } from '../lib/lock.js';
+import { acquireFileLock, readFileLockStatus } from '../lib/lock.js';
 import { CORRELATION_REGISTERED_EVENT, parseRunnerArtifacts, parseRunnerResult, } from '../domain/schema.js';
 import { maxConfiguredRunnerTimeoutMs, resolveRunnerRouting } from '../domain/runner-routing.js';
 import { awaitingApprovalRunnerSentinel, stageLabelForStage } from '../domain/stages.js';
@@ -58,6 +58,7 @@ export function createTickRunner(deps) {
         stateStore: deps.stateStore,
         projectionUpdater,
         runnerTimeoutMs,
+        isRunningRecordActive,
         deliverOutboundEvent,
     });
     const { cleanupClosedIssueWorkspaces } = createWorkspaceCleanup({
@@ -203,6 +204,15 @@ export function createTickRunner(deps) {
     function runnerTimeoutMs() {
         return maxConfiguredRunnerTimeoutMs(deps.config);
     }
+    async function isRunningRecordActive(record) {
+        const lock = await readFileLockStatus(deps.stateStore.paths.runnerLockFile, {
+            expectedCommandIncludes: process.argv[1] === undefined ? [] : [process.argv[1]],
+        });
+        if (!lock.active || lock.metadata === undefined) {
+            return false;
+        }
+        return Date.parse(lock.metadata.acquiredAt) <= Date.parse(record.startedAt);
+    }
     async function parkConfigDriftedProjections(projections) {
         let parked = false;
         for (const projection of projections) {

package/dist/src/lib/lock.js

@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
 import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { randomUUID } from 'node:crypto';
@@ -21,6 +22,7 @@ function parseLockMetadata(raw) {
             pid: record.pid,
             acquiredAt: record.acquiredAt,
             ...(typeof record.lockId === 'string' ? { lockId: record.lockId } : {}),
+            ...(typeof record.commandLine === 'string' ? { commandLine: record.commandLine } : {}),
         };
     }
     catch {
@@ -43,6 +45,28 @@ function isPidAlive(pid) {
         return true;
     }
 }
+function readProcessCommandLine(pid) {
+    if (pid === process.pid) {
+        return process.argv.join(' ');
+    }
+    try {
+        return readFileSyncProcessCommandLine(pid);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readFileSyncProcessCommandLine(pid) {
+    const raw = readFileSync(`/proc/${pid}/cmdline`, 'utf8');
+    const commandLine = raw.replace(/\0/g, ' ').trim();
+    return commandLine.length === 0 ? undefined : commandLine;
+}
+function defaultProcessInspector() {
+    return {
+        isPidAlive,
+        readCommandLine: readProcessCommandLine,
+    };
+}
 async function readLockMetadata(path) {
     try {
         return parseLockMetadata(await readFile(path, 'utf8'));
@@ -51,13 +75,48 @@ async function readLockMetadata(path) {
         return null;
     }
 }
-async function lockIsStale(path, staleAfterMs, now) {
+export async function readFileLockStatus(path, options) {
     const metadata = await readLockMetadata(path);
     if (metadata === null) {
-        return true;
+        try {
+            await readFile(path, 'utf8');
+            return { present: true, active: false, staleReason: 'invalid-metadata' };
+        }
+        catch {
+            return { present: false, active: false, staleReason: 'missing' };
+        }
     }
+    const now = options?.now ?? new Date();
     const ageMs = now.getTime() - Date.parse(metadata.acquiredAt);
-    return ageMs >= staleAfterMs || !isPidAlive(metadata.pid);
+    if (options?.staleAfterMs !== undefined && ageMs >= options.staleAfterMs) {
+        return { present: true, active: false, metadata, ageMs, staleReason: 'expired' };
+    }
+    const inspector = options?.processInspector ?? defaultProcessInspector();
+    if (!inspector.isPidAlive(metadata.pid)) {
+        return { present: true, active: false, metadata, ageMs, staleReason: 'pid-not-alive' };
+    }
+    const commandLine = inspector.readCommandLine(metadata.pid);
+    if (commandLine !== undefined &&
+        ((metadata.commandLine !== undefined && commandLine !== metadata.commandLine) ||
+            (metadata.commandLine === undefined &&
+                options?.expectedCommandIncludes !== undefined &&
+                !options.expectedCommandIncludes.every((fragment) => commandLine.includes(fragment))))) {
+        return {
+            present: true,
+            active: false,
+            metadata,
+            ageMs,
+            commandLine,
+            staleReason: 'pid-command-mismatch',
+        };
+    }
+    return {
+        present: true,
+        active: true,
+        metadata,
+        ageMs,
+        ...(commandLine === undefined ? {} : { commandLine }),
+    };
 }
 export async function acquireFileLock(path, options) {
     await mkdir(dirname(path), { recursive: true });
@@ -65,6 +124,7 @@ export async function acquireFileLock(path, options) {
         pid: process.pid,
         acquiredAt: (options?.now ?? new Date()).toISOString(),
         lockId: randomUUID(),
+        commandLine: process.argv.join(' '),
     };
     async function tryAcquire() {
         const tempPath = join(dirname(path), `.tick-lock-${metadata.lockId}.tmp`);
@@ -97,7 +157,10 @@ export async function acquireFileLock(path, options) {
     catch (error) {
         if (error.code === 'EEXIST') {
             if (options?.staleAfterMs !== undefined &&
-                (await lockIsStale(path, options.staleAfterMs, options.now ?? new Date()))) {
+                !(await readFileLockStatus(path, {
+                    staleAfterMs: options.staleAfterMs,
+                    now: options.now ?? new Date(),
+                })).active) {
                 await rm(path, { force: true });
                 try {
                     return await tryAcquire();

package/dist/src/main.js

@@ -28,6 +28,7 @@ import { runSandboxSetupCommand } from './cli/sandbox-setup-command.js';
 import { collectStartupPreflightFailures, runStartupPreflight } from './cli/startup-preflight.js';
 import { runUiCommand } from './cli/ui-command.js';
 import { loadWakeConfig } from './config/load-config.js';
+import { createActiveRunRecovery } from './core/active-run-recovery.js';
 import { createControlPlane } from './core/control-plane.js';
 import { createOutboundSinkRouter, createWorkSourceFanIn } from './core/sink-router.js';
 import { createTickRunner } from './core/tick-runner.js';
@@ -346,6 +347,28 @@ async function inspectDockerContainer(containerName) {
         });
     });
 }
+async function inspectDockerContainerImage(containerName) {
+    return await new Promise((resolveInspect, reject) => {
+        const child = spawn('docker', ['container', 'inspect', '-f', '{{.Config.Image}}', containerName], {
+            cwd: process.cwd(),
+            env: process.env,
+            stdio: ['ignore', 'pipe', 'ignore'],
+        });
+        let stdout = '';
+        child.stdout.on('data', (chunk) => {
+            stdout += chunk.toString();
+        });
+        child.on('error', reject);
+        child.on('close', (exitCode) => {
+            if (exitCode !== 0) {
+                resolveInspect(null);
+                return;
+            }
+            const image = stdout.trim();
+            resolveInspect(image.length > 0 ? image : null);
+        });
+    });
+}
 async function dockerDaemonReachable() {
     return await new Promise((resolveReachable) => {
         const child = spawn('docker', ['info'], {
@@ -370,6 +393,7 @@ function createHostDockerCli() {
         run: (dockerArgs) => runCommand('docker', dockerArgs, { ...process.env, DOCKER_BUILDKIT: '1' }),
         inspectImage: inspectDockerImage,
         inspectContainer: inspectDockerContainer,
+        inspectContainerImage: inspectDockerContainerImage,
         spawnExec: (dockerArgs) => {
             const child = spawn('docker', dockerArgs, {
                 cwd: process.cwd(),
@@ -819,6 +843,12 @@ async function main() {
             wakeRoot,
         });
         const docker = createHostDockerCli();
+        const recoverActiveRuns = createActiveRunRecovery({
+            clock: systemClock,
+            config,
+            stateStore,
+            resourceIndex: createResourceIndex({ paths: stateStore.paths }),
+        }).recoverActiveRuns;
         const repoRoot = config.dev?.repoRoot;
         const selfUpdate = commandArgs[0] === 'self-update' &&
             config.dev?.mode === 'source' &&
@@ -889,6 +919,7 @@ async function main() {
             docker,
             packagedTemplatesRoot: resolve(resolvePackageRoot(), 'docker'),
             stateStore,
+            recoverActiveRuns,
             sleep: (ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)),
             logger: {
                 info(message) {

package/dist/src/version.js

@@ -1 +1 @@
-export const wakeVersion = "0.2.12+g6314122";
+export const wakeVersion = "0.2.14+gb810105";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.2.12",
+  "version": "0.2.14",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {