@atolis-hq/wake 0.2.13 → 0.2.15

package/dist/src/adapters/http/ui-server.js

@@ -96,6 +96,22 @@ async function handleRequest(req, res, options, now) {
         sendJson(res, 202, request);
         return;
     }
+    if (req.method === 'POST' &&
+        resource === 'runners' &&
+        segments.length === 3 &&
+        segments[2] === 'unpause') {
+        const runnerName = segments[1];
+        const ledger = await stateStore.readLedger();
+        const existingRunners = ledger?.runners ?? {};
+        if (existingRunners[runnerName] !== undefined) {
+            await stateStore.writeLedger({
+                schemaVersion: 1,
+                runners: { ...existingRunners, [runnerName]: { failureCount: 0 } },
+            });
+        }
+        sendJson(res, 200, { runnerName, unpaused: true });
+        return;
+    }
     if (req.method !== 'GET') {
         sendJson(res, 405, { error: `method not allowed for ${url.pathname}` });
         return;

package/dist/src/cli/sandbox-command.js

@@ -148,6 +148,9 @@ export async function runSandboxCommand(input) {
         await runStopCommand({
             args: input.args.slice(1),
             stateStore: input.stateStore,
+            ...(input.recoverActiveRuns === undefined
+                ? {}
+                : { recoverActiveRuns: input.recoverActiveRuns }),
             docker: input.docker,
             containerName: input.config.sandbox.containerName,
             sleep: input.sleep,
@@ -174,6 +177,9 @@ export async function runSandboxCommand(input) {
             imageRepository: input.config.sandbox.imageRepository,
             containerName: input.config.sandbox.containerName,
             stateStore: input.stateStore,
+            ...(input.recoverActiveRuns === undefined
+                ? {}
+                : { recoverActiveRuns: input.recoverActiveRuns }),
             docker: input.docker,
             git: input.selfUpdate.git,
             issueReporter: input.selfUpdate.issueReporter,

package/dist/src/cli/self-update-command.js

@@ -72,6 +72,9 @@ export async function runSelfUpdateCommand(input) {
     }
     await waitForActiveRuns({
         listRunRecords: input.stateStore.listRunRecords,
+        ...(input.recoverActiveRuns === undefined
+            ? {}
+            : { recoverActiveRuns: input.recoverActiveRuns }),
         sleep: input.sleep,
         logger: input.logger,
     });

package/dist/src/cli/stop-command.js

@@ -4,6 +4,7 @@ export async function waitForActiveRuns(input) {
     const pollIntervalMs = input.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
     const startedAt = Date.now();
     for (;;) {
+        await input.recoverActiveRuns?.();
         const records = await input.listRunRecords();
         const activeRuns = records.filter((record) => record.status === 'running');
         if (activeRuns.length === 0) {
@@ -32,6 +33,9 @@ function readNumberFlag(name, args) {
 export async function runStopCommand(input) {
     await waitForActiveRuns({
         listRunRecords: input.stateStore.listRunRecords,
+        ...(input.recoverActiveRuns === undefined
+            ? {}
+            : { recoverActiveRuns: input.recoverActiveRuns }),
         sleep: input.sleep,
         pollIntervalMs: readNumberFlag('--poll-interval-ms', input.args),
         timeoutMs: readNumberFlag('--timeout-ms', input.args),

package/dist/src/core/active-run-recovery.js

@@ -0,0 +1,39 @@
+import { readFileLockStatus } from '../lib/lock.js';
+import { maxConfiguredRunnerTimeoutMs } from '../domain/runner-routing.js';
+import { createOutbox } from './outbox.js';
+import { createProjectionUpdater } from './projection-updater.js';
+import { createStaleRunReconciler } from './stale-run-reconciler.js';
+export function createActiveRunRecovery(deps) {
+    const projectionUpdater = createProjectionUpdater({
+        stateStore: deps.stateStore,
+        resourceIndex: deps.resourceIndex,
+        config: deps.config,
+    });
+    const { deliverOutboundEvent } = createOutbox({
+        clock: deps.clock,
+        stateStore: deps.stateStore,
+        projectionUpdater,
+    });
+    async function isRunningRecordActive(record) {
+        const lock = await readFileLockStatus(deps.stateStore.paths.runnerLockFile, {
+            expectedCommandIncludes: process.argv[1] === undefined ? [] : [process.argv[1]],
+        });
+        if (!lock.active || lock.metadata === undefined) {
+            return false;
+        }
+        return Date.parse(lock.metadata.acquiredAt) <= Date.parse(record.startedAt);
+    }
+    const { reconcileStaleRunningRecords } = createStaleRunReconciler({
+        config: deps.config,
+        stateStore: deps.stateStore,
+        projectionUpdater,
+        runnerTimeoutMs: () => maxConfiguredRunnerTimeoutMs(deps.config),
+        isRunningRecordActive,
+        deliverOutboundEvent,
+    });
+    return {
+        recoverActiveRuns() {
+            return reconcileStaleRunningRecords(deps.clock.now());
+        },
+    };
+}

package/dist/src/core/stale-run-reconciler.js

@@ -8,21 +8,30 @@ import { createEventEnvelope } from '../lib/event-log.js';
 // `deliverOutboundEvent` is injected rather than imported so this module does
 // not depend on the outbox module directly.
 export function createStaleRunReconciler(deps) {
-    function isStaleRunningRecord(record, now) {
+    async function staleReason(record, now) {
         if (record.status !== 'running') {
-            return false;
+            return null;
+        }
+        if (deps.isRunningRecordActive !== undefined && !(await deps.isRunningRecordActive(record))) {
+            return 'runner-lock-not-active';
         }
         const startedAtMs = Date.parse(record.startedAt);
         if (!Number.isFinite(startedAtMs)) {
-            return true;
+            return 'timeout';
         }
-        return now.getTime() - startedAtMs >= deps.runnerTimeoutMs();
+        return now.getTime() - startedAtMs >= deps.runnerTimeoutMs() ? 'timeout' : null;
     }
     async function reconcileStaleRunningRecords(now) {
         const finishedAt = now.toISOString();
         const runRecords = await deps.stateStore.listRunRecords();
-        const staleRecords = runRecords.filter((record) => isStaleRunningRecord(record, now));
-        for (const record of staleRecords) {
+        const staleRecords = [];
+        for (const record of runRecords) {
+            const reason = await staleReason(record, now);
+            if (reason !== null) {
+                staleRecords.push({ record, reason });
+            }
+        }
+        for (const { record, reason } of staleRecords) {
             // Run records carry the work item they belong to, so this is a direct
             // O(1) read — no scan, no index, no source ambiguity. The record's
             // repo/issueNumber are representation content and take no part in it.
@@ -57,6 +66,7 @@ export function createStaleRunReconciler(deps) {
                 metadata: {
                     ...record.metadata,
                     reconciledBy: 'stale-running-record',
+                    staleReason: reason,
                     timeoutMs: deps.runnerTimeoutMs(),
                 },
             });
@@ -79,7 +89,7 @@ export function createStaleRunReconciler(deps) {
                     action: record.action,
                     sentinel: 'FAILED',
                     runId: record.runId,
-                    reason: 'runner:stale-timeout',
+                    reason: reason === 'timeout' ? 'runner:stale-timeout' : 'runner:orphaned-process',
                     ...(record.routing === undefined ? {} : { routing: record.routing }),
                 },
             });

package/dist/src/core/tick-runner.js

@@ -1,7 +1,7 @@
 import { createLifecycleService } from './lifecycle-service.js';
 import { createPolicyEngine } from './policy-engine.js';
 import { createProjectionUpdater } from './projection-updater.js';
-import { acquireFileLock } from '../lib/lock.js';
+import { acquireFileLock, readFileLockStatus } from '../lib/lock.js';
 import { CORRELATION_REGISTERED_EVENT, parseRunnerArtifacts, parseRunnerResult, } from '../domain/schema.js';
 import { maxConfiguredRunnerTimeoutMs, resolveRunnerRouting } from '../domain/runner-routing.js';
 import { awaitingApprovalRunnerSentinel, stageLabelForStage } from '../domain/stages.js';
@@ -58,6 +58,7 @@ export function createTickRunner(deps) {
         stateStore: deps.stateStore,
         projectionUpdater,
         runnerTimeoutMs,
+        isRunningRecordActive,
         deliverOutboundEvent,
     });
     const { cleanupClosedIssueWorkspaces } = createWorkspaceCleanup({
@@ -203,6 +204,15 @@ export function createTickRunner(deps) {
     function runnerTimeoutMs() {
         return maxConfiguredRunnerTimeoutMs(deps.config);
     }
+    async function isRunningRecordActive(record) {
+        const lock = await readFileLockStatus(deps.stateStore.paths.runnerLockFile, {
+            expectedCommandIncludes: process.argv[1] === undefined ? [] : [process.argv[1]],
+        });
+        if (!lock.active || lock.metadata === undefined) {
+            return false;
+        }
+        return Date.parse(lock.metadata.acquiredAt) <= Date.parse(record.startedAt);
+    }
     async function parkConfigDriftedProjections(projections) {
         let parked = false;
         for (const projection of projections) {

package/dist/src/lib/lock.js

@@ -1,3 +1,4 @@
+import { readFileSync } from 'node:fs';
 import { link, mkdir, readFile, rm, writeFile } from 'node:fs/promises';
 import { dirname, join } from 'node:path';
 import { randomUUID } from 'node:crypto';
@@ -21,6 +22,7 @@ function parseLockMetadata(raw) {
             pid: record.pid,
             acquiredAt: record.acquiredAt,
             ...(typeof record.lockId === 'string' ? { lockId: record.lockId } : {}),
+            ...(typeof record.commandLine === 'string' ? { commandLine: record.commandLine } : {}),
         };
     }
     catch {
@@ -43,6 +45,28 @@ function isPidAlive(pid) {
         return true;
     }
 }
+function readProcessCommandLine(pid) {
+    if (pid === process.pid) {
+        return process.argv.join(' ');
+    }
+    try {
+        return readFileSyncProcessCommandLine(pid);
+    }
+    catch {
+        return undefined;
+    }
+}
+function readFileSyncProcessCommandLine(pid) {
+    const raw = readFileSync(`/proc/${pid}/cmdline`, 'utf8');
+    const commandLine = raw.replace(/\0/g, ' ').trim();
+    return commandLine.length === 0 ? undefined : commandLine;
+}
+function defaultProcessInspector() {
+    return {
+        isPidAlive,
+        readCommandLine: readProcessCommandLine,
+    };
+}
 async function readLockMetadata(path) {
     try {
         return parseLockMetadata(await readFile(path, 'utf8'));
@@ -51,13 +75,48 @@ async function readLockMetadata(path) {
         return null;
     }
 }
-async function lockIsStale(path, staleAfterMs, now) {
+export async function readFileLockStatus(path, options) {
     const metadata = await readLockMetadata(path);
     if (metadata === null) {
-        return true;
+        try {
+            await readFile(path, 'utf8');
+            return { present: true, active: false, staleReason: 'invalid-metadata' };
+        }
+        catch {
+            return { present: false, active: false, staleReason: 'missing' };
+        }
     }
+    const now = options?.now ?? new Date();
     const ageMs = now.getTime() - Date.parse(metadata.acquiredAt);
-    return ageMs >= staleAfterMs || !isPidAlive(metadata.pid);
+    if (options?.staleAfterMs !== undefined && ageMs >= options.staleAfterMs) {
+        return { present: true, active: false, metadata, ageMs, staleReason: 'expired' };
+    }
+    const inspector = options?.processInspector ?? defaultProcessInspector();
+    if (!inspector.isPidAlive(metadata.pid)) {
+        return { present: true, active: false, metadata, ageMs, staleReason: 'pid-not-alive' };
+    }
+    const commandLine = inspector.readCommandLine(metadata.pid);
+    if (commandLine !== undefined &&
+        ((metadata.commandLine !== undefined && commandLine !== metadata.commandLine) ||
+            (metadata.commandLine === undefined &&
+                options?.expectedCommandIncludes !== undefined &&
+                !options.expectedCommandIncludes.every((fragment) => commandLine.includes(fragment))))) {
+        return {
+            present: true,
+            active: false,
+            metadata,
+            ageMs,
+            commandLine,
+            staleReason: 'pid-command-mismatch',
+        };
+    }
+    return {
+        present: true,
+        active: true,
+        metadata,
+        ageMs,
+        ...(commandLine === undefined ? {} : { commandLine }),
+    };
 }
 export async function acquireFileLock(path, options) {
     await mkdir(dirname(path), { recursive: true });
@@ -65,6 +124,7 @@ export async function acquireFileLock(path, options) {
         pid: process.pid,
         acquiredAt: (options?.now ?? new Date()).toISOString(),
         lockId: randomUUID(),
+        commandLine: process.argv.join(' '),
     };
     async function tryAcquire() {
         const tempPath = join(dirname(path), `.tick-lock-${metadata.lockId}.tmp`);
@@ -97,7 +157,10 @@ export async function acquireFileLock(path, options) {
     catch (error) {
         if (error.code === 'EEXIST') {
             if (options?.staleAfterMs !== undefined &&
-                (await lockIsStale(path, options.staleAfterMs, options.now ?? new Date()))) {
+                !(await readFileLockStatus(path, {
+                    staleAfterMs: options.staleAfterMs,
+                    now: options.now ?? new Date(),
+                })).active) {
                 await rm(path, { force: true });
                 try {
                     return await tryAcquire();

package/dist/src/main.js

@@ -28,6 +28,7 @@ import { runSandboxSetupCommand } from './cli/sandbox-setup-command.js';
 import { collectStartupPreflightFailures, runStartupPreflight } from './cli/startup-preflight.js';
 import { runUiCommand } from './cli/ui-command.js';
 import { loadWakeConfig } from './config/load-config.js';
+import { createActiveRunRecovery } from './core/active-run-recovery.js';
 import { createControlPlane } from './core/control-plane.js';
 import { createOutboundSinkRouter, createWorkSourceFanIn } from './core/sink-router.js';
 import { createTickRunner } from './core/tick-runner.js';
@@ -842,6 +843,12 @@ async function main() {
             wakeRoot,
         });
         const docker = createHostDockerCli();
+        const recoverActiveRuns = createActiveRunRecovery({
+            clock: systemClock,
+            config,
+            stateStore,
+            resourceIndex: createResourceIndex({ paths: stateStore.paths }),
+        }).recoverActiveRuns;
         const repoRoot = config.dev?.repoRoot;
         const selfUpdate = commandArgs[0] === 'self-update' &&
             config.dev?.mode === 'source' &&
@@ -912,6 +919,7 @@ async function main() {
             docker,
             packagedTemplatesRoot: resolve(resolvePackageRoot(), 'docker'),
             stateStore,
+            recoverActiveRuns,
             sleep: (ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)),
             logger: {
                 info(message) {

package/dist/src/version.js

@@ -1 +1 @@
-export const wakeVersion = "0.2.13+g3da36d3";
+export const wakeVersion = "0.2.15+g36c8953";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atolis-hq/wake",
-  "version": "0.2.13",
+  "version": "0.2.15",
   "description": "Local autonomous agent control plane for software development",
   "license": "Apache-2.0",
   "repository": {