@yycholla/pi-dynamic-workflows 3.3.0

package/dist/run-persistence.js

@@ -0,0 +1,294 @@
+/**
+ * Workflow run state persistence for pause/resume support.
+ */
+import { join } from "node:path";
+import { ensureDir as ensureDirFs, listJsonFilesSafe, readJsonWithBackupRecovery, resolvePersistenceFs, unlinkIfExistsSafe, writeJsonAtomicWithBackup, } from "./fs-persistence.js";
+import { workflowProjectPaths } from "./workflow-paths.js";
+/**
+ * Retention policy for terminal (completed/failed/aborted) runs kept on
+ * disk. Bounded so a long-lived project directory can't accumulate an
+ * unbounded number of run files (each polled/listed on every list() call).
+ * A run in "running" or "paused" status is NEVER counted against this cap
+ * or evicted by it — only genuinely finished runs age out, oldest (by
+ * updatedAt) first, once the terminal-run count exceeds the cap. 300 is
+ * generous enough to cover weeks of typical usage while keeping list()'s
+ * per-call directory scan bounded.
+ */
+export const DEFAULT_MAX_TERMINAL_RUNS_ON_DISK = 300;
+const TERMINAL_RUN_STATUSES = new Set(["completed", "failed", "aborted"]);
+/**
+ * `list()` does a full readdirSync + per-file readFileSync + JSON.parse of the
+ * entire lifetime run history. It is called on essentially every progress tick
+ * (task-panel re-render → WorkflowManager.listRuns()/listAllRuns()), so an
+ * unbounded number of ticks each re-walked and re-parsed every run file on
+ * disk. Cache the computed list for a short TTL — long enough to absorb a
+ * burst of same-tick reads, short enough that a read from a DIFFERENT process
+ * (or a mutation this instance doesn't own) still shows up quickly. Mirrors
+ * the ~1s settings-read TTL cache in task-panel.ts.
+ */
+const LIST_CACHE_TTL_MS = 300;
+export function createRunPersistence(cwd, fsOverride, options) {
+    const fs = resolvePersistenceFs(fsOverride);
+    const _existsSync = fs.existsSync;
+    const _readFileSync = fs.readFileSync;
+    const _statSync = fs.statSync;
+    const _unlinkSync = fs.unlinkSync;
+    const _writeFileSync = fs.writeFileSync;
+    const maxTerminalRunsOnDisk = options?.maxTerminalRunsOnDisk ?? DEFAULT_MAX_TERMINAL_RUNS_ON_DISK;
+    const paths = workflowProjectPaths(cwd);
+    const runsDir = paths.runsDir;
+    const legacyRunsDir = paths.legacyRunsDir;
+    const ensureDir = () => ensureDirFs(fs, runsDir);
+    const runPath = (dir, runId) => join(dir, `${runId}.json`);
+    const primaryRunPath = (runId) => runPath(runsDir, runId);
+    const legacyRunPath = (runId) => runPath(legacyRunsDir, runId);
+    const lockPath = (dir, runId) => join(dir, `${runId}.lock`);
+    const primaryLockPath = (runId) => lockPath(runsDir, runId);
+    const legacyLockPath = (runId) => lockPath(legacyRunsDir, runId);
+    const candidateRunPaths = (runId) => [primaryRunPath(runId), legacyRunPath(runId)];
+    const pidIsAlive = (pid) => {
+        if (!Number.isInteger(pid) || pid <= 0)
+            return false;
+        try {
+            process.kill(pid, 0);
+            return true;
+        }
+        catch (err) {
+            if (err.code === "EPERM")
+                return true;
+            return false;
+        }
+    };
+    const readLockAt = (path) => {
+        try {
+            return JSON.parse(_readFileSync(path, "utf-8"));
+        }
+        catch {
+            return null;
+        }
+    };
+    const readLock = (runId) => readLockAt(primaryLockPath(runId));
+    // list() cache: recomputed lazily, invalidated synchronously by every
+    // mutation this instance performs (save()/delete()) so a stale read can
+    // never outlive a mutation this process made. A read from another process
+    // (or a direct fs write bypassing this instance) is picked up once the TTL
+    // elapses, same as before this cache existed on the next un-cached call.
+    let listCache;
+    let listCacheAt = 0;
+    const invalidateListCache = () => {
+        listCache = undefined;
+    };
+    // Per-file mtime+size+ino cache, keyed by absolute path: even once the
+    // TTL-level listCache above expires (the active panel polls roughly every
+    // 300ms, i.e. faster than or comparable to the TTL), most run files on
+    // disk haven't changed since the last recompute. Re-stat is cheap; re-read
+    // + re-JSON.parse is not, and scales with total lifetime run history, not
+    // with what actually changed. A file whose (mtimeMs, size, ino) all match
+    // what we last parsed is reused as-is instead of being re-read; entries
+    // for files that vanished between recomputes are pruned so this cache
+    // can't grow unbounded independent of what's actually on disk.
+    //
+    // ino is load-bearing, not redundant with mtime+size: save() writes via
+    // tmp-write + rename (writeJsonAtomicWithBackup), and a rename onto an
+    // existing path allocates a NEW inode for the replacement file. Two
+    // consecutive saves landing in the same mtime tick (400ms-throttled
+    // progress persists vs. 1-2s mtime granularity on HFS+/many network
+    // mounts/some Docker volume drivers is entirely realistic) with
+    // coincidentally equal byte length (e.g. "paused" and "failed" are the
+    // same length) would otherwise be indistinguishable from "unchanged" by
+    // (mtimeMs, size) alone — serving stale, previously-cached content
+    // forever until something ELSE about the file changes. The inode always
+    // changes on such a rename, so adding it closes that hole for free.
+    const fileStateCache = new Map();
+    const removeStaleLegacyLock = (runId) => {
+        const lock = legacyLockPath(runId);
+        const existing = readLockAt(lock);
+        if (existing?.runId === runId && pidIsAlive(existing.pid))
+            return false;
+        try {
+            if (_existsSync(lock))
+                _unlinkSync(lock);
+        }
+        catch {
+            return false;
+        }
+        return true;
+    };
+    const computeList = () => {
+        const byRunId = new Map();
+        const seenPaths = new Set();
+        for (const dir of [runsDir, legacyRunsDir]) {
+            for (const file of listJsonFilesSafe(fs, dir)) {
+                const path = join(dir, file);
+                seenPaths.add(path);
+                try {
+                    const stat = _statSync(path);
+                    const cached = fileStateCache.get(path);
+                    // Reuse the last parse when the file is byte-identical (same
+                    // mtime + size + inode) to what produced it — the dominant case
+                    // on every poll tick once a run goes terminal and stops changing.
+                    // ino is what actually rules out a false "unchanged" match on a
+                    // coarse-mtime filesystem (see the field doc comment above).
+                    if (cached && cached.mtimeMs === stat.mtimeMs && cached.size === stat.size && cached.ino === stat.ino) {
+                        if (!byRunId.has(cached.state.runId))
+                            byRunId.set(cached.state.runId, cached.state);
+                        continue;
+                    }
+                    const state = JSON.parse(_readFileSync(path, "utf-8"));
+                    fileStateCache.set(path, { mtimeMs: stat.mtimeMs, size: stat.size, ino: stat.ino, state });
+                    if (!byRunId.has(state.runId))
+                        byRunId.set(state.runId, state);
+                }
+                catch {
+                    // Skip corrupted/unreadable files; don't let a stale cache entry
+                    // for a file that's now failing to read linger either.
+                    fileStateCache.delete(path);
+                }
+            }
+        }
+        // Prune cache entries for files that no longer exist (deleted runs) so
+        // this map's size tracks what's actually on disk, not lifetime history.
+        for (const path of fileStateCache.keys()) {
+            if (!seenPaths.has(path))
+                fileStateCache.delete(path);
+        }
+        return [...byRunId.values()].sort((a, b) => new Date(b.updatedAt).getTime() - new Date(a.updatedAt).getTime());
+    };
+    // Bound the number of terminal (completed/failed/aborted) runs kept on
+    // disk (see DEFAULT_MAX_TERMINAL_RUNS_ON_DISK) — called after every save()
+    // whose state is terminal, since that's the only time the terminal count
+    // can grow. Running/paused runs are never candidates: they're filtered out
+    // before the cap is even considered.
+    const enforceRetention = () => {
+        const terminal = computeList()
+            .filter((r) => TERMINAL_RUN_STATUSES.has(r.status))
+            .sort((a, b) => new Date(a.updatedAt).getTime() - new Date(b.updatedAt).getTime());
+        const excess = terminal.length - maxTerminalRunsOnDisk;
+        if (excess <= 0)
+            return;
+        for (const run of terminal.slice(0, excess)) {
+            deleteRunFiles(run.runId);
+        }
+        invalidateListCache();
+    };
+    const deleteRunFiles = (runId) => {
+        let deleted = false;
+        for (const path of candidateRunPaths(runId)) {
+            const dir = path === primaryRunPath(runId) ? runsDir : legacyRunsDir;
+            // Best-effort cleanup of the sidecar files alongside the primary.
+            for (const sidecar of [`${path}.bak`, `${path}.tmp`, lockPath(dir, runId)]) {
+                unlinkIfExistsSafe(fs, sidecar);
+                fileStateCache.delete(sidecar);
+            }
+            if (unlinkIfExistsSafe(fs, path))
+                deleted = true;
+            fileStateCache.delete(path);
+        }
+        return deleted;
+    };
+    return {
+        save(state) {
+            ensureDir();
+            state.updatedAt = new Date().toISOString();
+            const path = primaryRunPath(state.runId);
+            // Atomic write: a crash mid-write can't corrupt the live file (tmp+rename is
+            // atomic on the same filesystem). A .bak from the previous good save is the
+            // recovery fallback if the primary is somehow truncated.
+            writeJsonAtomicWithBackup(fs, path, state);
+            invalidateListCache();
+            // Only a terminal write can grow the terminal-run count, so only check
+            // the cap then — a "running"/"paused" save is on the hot path (every
+            // progress tick) and must not pay for a retention scan.
+            if (TERMINAL_RUN_STATUSES.has(state.status))
+                enforceRetention();
+        },
+        load(runId) {
+            // Try the primary, then the .bak — so a corrupt primary doesn't lose the run.
+            for (const path of candidateRunPaths(runId)) {
+                const state = readJsonWithBackupRecovery(fs, path);
+                if (state)
+                    return state;
+            }
+            return null;
+        },
+        list() {
+            const now = Date.now();
+            // Return a fresh array on every call (a cheap ref-copy) so a caller that
+            // sorts/reverses/mutates the result in place can't corrupt the cache — the
+            // pre-cache code re-parsed into a new array each call, preserve that.
+            if (listCache && now - listCacheAt < LIST_CACHE_TTL_MS) {
+                return [...listCache];
+            }
+            const result = computeList();
+            listCache = result;
+            listCacheAt = now;
+            return [...result];
+        },
+        delete(runId) {
+            try {
+                return deleteRunFiles(runId);
+            }
+            finally {
+                invalidateListCache();
+            }
+        },
+        acquireRunLease(runId) {
+            ensureDir();
+            const path = primaryRunPath(runId);
+            const lock = primaryLockPath(runId);
+            if (!removeStaleLegacyLock(runId))
+                return null;
+            for (let attempt = 0; attempt < 2; attempt++) {
+                const token = `${process.pid}-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`;
+                const payload = {
+                    runId,
+                    runPath: path,
+                    pid: process.pid,
+                    startedAt: new Date().toISOString(),
+                    token,
+                };
+                try {
+                    _writeFileSync(lock, JSON.stringify(payload, null, 2), { flag: "wx" });
+                    return { runId, token };
+                }
+                catch (err) {
+                    const code = err.code;
+                    if (code !== "EEXIST")
+                        throw err;
+                    const existing = readLock(runId);
+                    if (existing && existing.runPath === path && pidIsAlive(existing.pid)) {
+                        return null;
+                    }
+                    try {
+                        _unlinkSync(lock);
+                    }
+                    catch {
+                        return null;
+                    }
+                }
+            }
+            return null;
+        },
+        releaseRunLease(lease) {
+            try {
+                const existing = readLock(lease.runId);
+                if (existing?.token === lease.token)
+                    _unlinkSync(primaryLockPath(lease.runId));
+            }
+            catch {
+                // Best-effort cleanup only.
+            }
+        },
+        getRunsDir() {
+            return runsDir;
+        },
+    };
+}
+/**
+ * Generate a unique run ID.
+ */
+export function generateRunId() {
+    const timestamp = Date.now().toString(36);
+    const random = Math.random().toString(36).slice(2, 8);
+    return `${timestamp}-${random}`;
+}

package/dist/saved-commands.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Saved workflows as `/<name>` slash commands. Each saved workflow becomes a
+ * command that runs its script, passing parsed arguments through as `args`.
+ */
+import { type ExtensionAPI } from "@earendil-works/pi-coding-agent";
+import type { WorkflowManager } from "./workflow-manager.js";
+import type { SavedWorkflow, WorkflowStorage } from "./workflow-saved.js";
+/**
+ * Parse a command argument string into an `args` object for the script.
+ * Supports `key=value` tokens; everything else collects into `_` (and `_raw`).
+ * Declared parameter defaults fill in missing keys.
+ */
+export declare function parseCommandArgs(raw: string, parameters?: SavedWorkflow["parameters"]): Record<string, unknown>;
+/** Register one saved workflow as a `/<name>` command (idempotent).
+ * When a WorkflowManager is provided, the workflow runs through it (visible in
+ * /workflows TUI, background execution, task panel). Otherwise falls back to
+ * the inline runWorkflow() (foreground, no TUI tracking).
+ *
+ * Pi has no `unregisterCommand`, so a command cannot be removed mid-session
+ * after its workflow is deleted (it is correctly gone on next launch, since
+ * registerAllSavedWorkflows only registers what's in storage). The optional
+ * `exists` predicate lets the handler detect that case at invocation time and
+ * tell the user to reload rather than silently re-running a deleted workflow. */
+export declare function registerSavedWorkflow(pi: ExtensionAPI, cwd: string, wf: SavedWorkflow, manager?: WorkflowManager, exists?: () => boolean): void;
+/** Register every saved workflow found in storage.
+ * When a WorkflowManager is provided, workflows run through it (visible in
+ * /workflows TUI, background execution, task panel). */
+export declare function registerAllSavedWorkflows(pi: ExtensionAPI, cwd: string, storage: WorkflowStorage, manager?: WorkflowManager): void;

package/dist/saved-commands.js

@@ -0,0 +1,100 @@
+/**
+ * Saved workflows as `/<name>` slash commands. Each saved workflow becomes a
+ * command that runs its script, passing parsed arguments through as `args`.
+ */
+import { createCodingTools } from "@earendil-works/pi-coding-agent";
+import { runWorkflow } from "./workflow.js";
+function isRegistered(pi, name) {
+    try {
+        return (pi.getCommands?.() ?? []).some((c) => c.name === name);
+    }
+    catch {
+        return false;
+    }
+}
+function reportText(result) {
+    const r = result.result;
+    if (r && typeof r.report === "string" && r.report.trim())
+        return r.report;
+    return JSON.stringify(result.result, null, 2);
+}
+/**
+ * Parse a command argument string into an `args` object for the script.
+ * Supports `key=value` tokens; everything else collects into `_` (and `_raw`).
+ * Declared parameter defaults fill in missing keys.
+ */
+export function parseCommandArgs(raw, parameters) {
+    const out = {};
+    const positional = [];
+    for (const tok of raw.trim().split(/\s+/).filter(Boolean)) {
+        const eq = tok.indexOf("=");
+        if (eq > 0)
+            out[tok.slice(0, eq)] = tok.slice(eq + 1);
+        else
+            positional.push(tok);
+    }
+    out._ = positional.join(" ");
+    out._raw = raw.trim();
+    for (const [key, spec] of Object.entries(parameters ?? {})) {
+        if (out[key] === undefined && spec.default !== undefined)
+            out[key] = spec.default;
+    }
+    return out;
+}
+/** Register one saved workflow as a `/<name>` command (idempotent).
+ * When a WorkflowManager is provided, the workflow runs through it (visible in
+ * /workflows TUI, background execution, task panel). Otherwise falls back to
+ * the inline runWorkflow() (foreground, no TUI tracking).
+ *
+ * Pi has no `unregisterCommand`, so a command cannot be removed mid-session
+ * after its workflow is deleted (it is correctly gone on next launch, since
+ * registerAllSavedWorkflows only registers what's in storage). The optional
+ * `exists` predicate lets the handler detect that case at invocation time and
+ * tell the user to reload rather than silently re-running a deleted workflow. */
+export function registerSavedWorkflow(pi, cwd, wf, manager, exists) {
+    if (isRegistered(pi, wf.name))
+        return;
+    pi.registerCommand(wf.name, {
+        description: wf.description || `Saved workflow: ${wf.name}`,
+        async handler(args, ctx) {
+            if (exists && !exists()) {
+                ctx.ui.notify(`/${wf.name} was deleted — reload the session to remove this command.`, "warning");
+                return;
+            }
+            try {
+                if (manager) {
+                    // Run through the WorkflowManager's background path: the handler
+                    // returns immediately (awaiting the promise here would block the whole
+                    // session, #104), progress shows in the /workflows TUI and task panel,
+                    // and installResultDelivery posts the result back into the
+                    // conversation on completion — sending it here too would duplicate it.
+                    const { runId } = manager.startInBackground(wf.script, parseCommandArgs(args, wf.parameters));
+                    ctx.ui.notify(`/${wf.name} running in the background (${runId}) — watch the task panel or /workflows; the result is posted here when it finishes.`, "info");
+                    return;
+                }
+                // Fallback: inline runWorkflow (foreground, no TUI tracking, blocks).
+                ctx.ui.notify(`Starting /${wf.name}…`, "info");
+                const result = await runWorkflow(wf.script, {
+                    cwd,
+                    args: parseCommandArgs(args, wf.parameters),
+                    tools: createCodingTools(cwd),
+                    onPhase: (title) => ctx.ui.setStatus(`wf:${wf.name}`, `${wf.name}: ${title}`),
+                });
+                ctx.ui.setStatus(`wf:${wf.name}`, undefined);
+                await pi.sendMessage({ customType: `workflow:${wf.name}`, content: reportText(result), display: true });
+            }
+            catch (error) {
+                ctx.ui.setStatus(`wf:${wf.name}`, undefined);
+                ctx.ui.notify(`/${wf.name} failed: ${error instanceof Error ? error.message : error}`, "error");
+            }
+        },
+    });
+}
+/** Register every saved workflow found in storage.
+ * When a WorkflowManager is provided, workflows run through it (visible in
+ * /workflows TUI, background execution, task panel). */
+export function registerAllSavedWorkflows(pi, cwd, storage, manager) {
+    for (const wf of storage.list()) {
+        registerSavedWorkflow(pi, cwd, wf, manager, () => storage.list().some((w) => w.name === wf.name));
+    }
+}

package/dist/shared-store.d.ts

@@ -0,0 +1,98 @@
+/**
+ * In-memory key-value store scoped to a single workflow run.
+ *
+ * One `SharedStore` instance is created at run start and disposed when the run
+ * ends. Two MCP-compatible tool definitions (`store_put` / `store_get`) are
+ * injected into every agent's tool list so parallel agents can share
+ * intermediate state without coordinating through the script itself.
+ *
+ * Journal integration: callers capture `store.commitDelta(deltaKey)` alongside
+ * each agent result in the journal. On resume, `store.applyDelta(delta)` rebuilds
+ * the store state additively in callSeq order, so parallel-agent writes are
+ * replayed correctly without the last-complete-wins ordering bug that a
+ * whole-Map restore() would cause.
+ *
+ * `deltaKey` must be unique across every run that shares this store instance,
+ * not just within one run's callSeq. A nested `workflow()` call restarts its own
+ * callSeq at 0 while inheriting the parent's store (so parent and nested-run
+ * agents can share state), so a bare callIndex would collide between a parent
+ * agent and a concurrently-running nested-run agent that both got index 0 —
+ * whichever commits its delta last would clobber the other's entry in
+ * `agentDeltas`. Callers compose `deltaKey` as `${runId}:${callIndex}`, and
+ * since every run (including each nested run) gets its own distinct `runId`,
+ * the composite key is unique across the whole store's lifetime.
+ */
+import { type ToolDefinition } from "@earendil-works/pi-coding-agent";
+export declare class SharedStore {
+    private readonly map;
+    private readonly agentDeltas;
+    private readonly priorValues;
+    /** Store a value under `key`. Overwrites any existing value. */
+    put(key: string, value: unknown): void;
+    /**
+     * Store a value and record the write in the per-agent delta for `deltaKey`
+     * (a run-unique `${runId}:${callIndex}` string — see class doc). Used by
+     * per-agent tools created via `createAgentStoreTools` so that each agent's
+     * writes can be journaled and replayed independently.
+     */
+    trackPut(key: string, value: unknown, deltaKey: string): void;
+    /** Retrieve the value for `key`, or `undefined` when absent. */
+    get(key: string): unknown;
+    /** Whether `key` is present in the store. */
+    has(key: string): boolean;
+    /** Return a deep-copied plain-object snapshot of all entries. */
+    snapshot(): Record<string, unknown>;
+    /**
+     * Extract and clear the write delta accumulated for `deltaKey`.
+     * Called after an agent completes to get the set of keys it wrote.
+     */
+    commitDelta(deltaKey: string): Record<string, unknown>;
+    /**
+     * Undo the writes recorded for `deltaKey` and discard its bookkeeping,
+     * without touching any other key. Used when a retry attempt fails: that
+     * attempt's writes must not remain visible in the live store (e.g. to a
+     * concurrently-running sibling agent's store_get, or to script code reading
+     * `store.get` directly) and must not merge into the delta eventually
+     * recorded when a later attempt of the SAME call succeeds — otherwise a
+     * failed attempt's mutations would silently survive into the run's live
+     * state while being absent from the journaled delta that resume replay
+     * reconstructs from, leaving live execution and replay permanently
+     * inconsistent. Each key touched during this delta window is restored to
+     * whatever it held immediately before the window started (or deleted, if
+     * it did not exist yet) — never to some other attempt's or caller's value.
+     *
+     * Per-key guard: a key is only rolled back if the store STILL holds this
+     * attempt's own last write to it (checked with `Object.is` against the
+     * value recorded in `delta`). If a concurrently-running sibling (a
+     * different `deltaKey`, e.g. another agent in the same parallel() batch)
+     * legitimately overwrote the same key AFTER this attempt wrote it but
+     * BEFORE it failed, that sibling's write is left untouched — rolling back
+     * unconditionally would silently erase a live, unrelated write that this
+     * attempt never made and has no business undoing.
+     *
+     * A no-op if `deltaKey` never wrote anything (nothing to roll back).
+     */
+    discardDelta(deltaKey: string): void;
+    /**
+     * Apply a write delta additively — sets each key without clearing others.
+     * Used during resume replay so parallel-agent deltas applied in callSeq
+     * order accumulate correctly regardless of original completion order.
+     */
+    applyDelta(delta: Record<string, unknown>): void;
+    /**
+     * Replace all entries with a snapshot (for full resets).
+     * Prefer `applyDelta` for resume replay — see journal integration above.
+     */
+    restore(snap: Record<string, unknown>): void;
+    /** Clear all entries (called when the run ends). */
+    dispose(): void;
+}
+/**
+ * Create per-agent store tools that attribute writes to `deltaKey`, a
+ * run-unique `${runId}:${callIndex}` string (see the `SharedStore` class doc
+ * for why the bare callIndex alone is not enough once a nested `workflow()`
+ * call shares this store).
+ * Used internally by `runWorkflow` so each agent's puts are tracked in the
+ * store's delta journal and can be replayed additively on resume.
+ */
+export declare function createAgentStoreTools(store: SharedStore, deltaKey: string): ToolDefinition[];