@timqi/pier 0.0.1 → 0.0.3

package/dist/drain.js

@@ -0,0 +1,145 @@
+// A graceful restart: refuse new work, let running turns finish, and write
+// down what the deadline had to cut off so the next boot can tell the chats.
+//
+// The trigger is SIGUSR2 (main.ts); systemd's `Restart=always` is the "start
+// again" half. SIGTERM stays the fast path systemd expects — this file is only
+// the slow one. Nothing here is persisted for its own sake: everything durable
+// (transcripts, the chat → session map, task runs) already survives a restart,
+// so the ledger below holds only the one thing that would otherwise vanish
+// silently — turns and queued messages the deadline aborted (§5b).
+import { logger } from "./log.js";
+const log = logger("drain");
+/** How long running turns may take before they are aborted. Generous: a turn
+ *  can be a subagent fan-out, and an abort still persists the partial work. */
+export const DRAIN_DEADLINE_MS = 5 * 60_000;
+const POLL_MS = 1_000;
+/** Shared cleanup window after the deadline. All sessions use the same clock,
+ *  so N hung seams still cost at most this long rather than N times as long. */
+const CLEANUP_BOUND_MS = 10_000;
+/** What the dying process owes the chats, held for the next one to deliver. */
+export class RestartLedger {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    record(entry) {
+        this.db.prepare("INSERT INTO restart_ledger (channel_id, conversation_id, note, created_at) VALUES (?, ?, ?, ?)").run(entry.channelId, entry.conversationId, entry.note, Date.now());
+    }
+    list() {
+        const rows = this.db.prepare("SELECT id, channel_id, conversation_id, note FROM restart_ledger ORDER BY id").all();
+        return rows.map((row) => ({
+            id: row.id,
+            channelId: row.channel_id,
+            conversationId: row.conversation_id,
+            note: row.note,
+        }));
+    }
+    remove(id) {
+        this.db.prepare("DELETE FROM restart_ledger WHERE id = ?").run(id);
+    }
+}
+/**
+ * Resolve when the process may exit: every turn settled and every task run
+ * terminal, or the deadline reached and the stragglers aborted into the
+ * ledger. The caller (main.ts) owns what happens next — the ordinary shutdown,
+ * minus aborting task runs: the boot-time interrupted marking is the recovery
+ * path (tasks/service.ts start()), not a teardown race against dying channels.
+ */
+export async function drainForRestart(deps, deadlineMs = DRAIN_DEADLINE_MS, pollMs = POLL_MS, cleanupBoundMs = CLEANUP_BOUND_MS) {
+    const { router, tasks, ledger } = deps;
+    router.beginDrain();
+    tasks.pause();
+    const deadline = Date.now() + deadlineMs;
+    let lastReport = "";
+    for (;;) {
+        // Sleep first: a prompt accepted just before the gate closed may not have
+        // flipped its session to streaming yet, and exiting on that blink would
+        // cut off the very turn the drain exists to protect.
+        await new Promise((resolve) => setTimeout(resolve, pollMs));
+        const busy = router.busy();
+        const runs = tasks.activeRunCount();
+        if (busy.length === 0 && runs === 0) {
+            log.info("drained — nothing running");
+            return;
+        }
+        if (Date.now() >= deadline) {
+            log.warn(`drain deadline after ${String(Math.round(deadlineMs / 1000))}s — aborting ${String(busy.length)} turn(s); ` +
+                `${String(runs)} task run(s) will be marked interrupted at boot`);
+            const cleanupDeadline = Date.now() + cleanupBoundMs;
+            await Promise.all(busy.map(({ session, key }) => abortToLedger(session, key, ledger, cleanupDeadline)));
+            return;
+        }
+        const report = `draining: ${String(busy.length)} turn(s), ${String(runs)} active task run(s)`;
+        if (report !== lastReport)
+            log.info((lastReport = report));
+    }
+}
+/** A seam call the deadline cannot wait on forever: a hang or a rejection is
+ *  logged and answered with the fallback, and cleanup moves on. */
+async function bounded(work, ms, what, fallback) {
+    let timer;
+    const timeout = new Promise((resolve) => {
+        timer = setTimeout(() => {
+            log.error(`${what} did not answer within ${String(ms)}ms`);
+            resolve(fallback);
+        }, ms);
+        timer.unref();
+    });
+    try {
+        return await Promise.race([
+            work.catch((err) => {
+                log.error(`${what} failed`, err);
+                return fallback;
+            }),
+            timeout,
+        ]);
+    }
+    finally {
+        if (timer)
+            clearTimeout(timer);
+    }
+}
+/** Write the chat's entry, then abort the turn. The ledger comes first so a
+ *  hung abort cannot cost the note; the abort persists the partial transcript;
+ *  the pending queue would just vanish, so its texts ride along. */
+async function abortToLedger(session, key, ledger, cleanupDeadline) {
+    const remaining = () => Math.max(0, cleanupDeadline - Date.now());
+    const queued = await bounded(session.pendingQueue(), remaining(), `queue snapshot of session ${session.id}`, { steering: [], followUp: [] });
+    const pending = [...queued.steering, ...queued.followUp];
+    // A web or task key has no chat to write to: the transcript shows the
+    // aborted turn, and a task run's interruption is reported by its callback
+    // recovery. Only a dropped queue would be invisible there, so it is at
+    // least logged.
+    if (key.channelId === "web" || key.channelId === "task") {
+        if (pending.length) {
+            log.warn(`session ${session.id}: ${String(pending.length)} queued message(s) dropped by the restart`);
+        }
+    }
+    else {
+        const note = [
+            "Pier restarted before this turn finished — the last message may be unanswered.",
+            ...(pending.length ? ["Queued and not delivered:", ...pending.map((text) => `> ${text}`)] : []),
+        ].join("\n");
+        ledger.record({ channelId: key.channelId, conversationId: key.conversationId, note });
+    }
+    await bounded(session.abort(), remaining(), `abort of session ${session.id}`, undefined);
+}
+/**
+ * Deliver what a previous process wrote on its way out. Runs at boot once the
+ * adapters are up, and again on a Console unlock. Each entry is removed only
+ * after confirmed delivery. A missing adapter or a thrown notification keeps
+ * the debt for the next start: a duplicate apology is preferable to silence.
+ */
+export async function deliverLedger(ledger, notify) {
+    for (const entry of ledger.list()) {
+        const target = `${entry.channelId}:${entry.conversationId}`;
+        const delivered = await notify(entry).catch((err) => {
+            log.error(`restart note to ${target} failed`, err);
+            return null;
+        });
+        if (delivered === true)
+            ledger.remove(entry.id);
+        else if (delivered === false)
+            log.warn(`restart note waiting — ${target} is not running: ${entry.note}`);
+    }
+}

package/dist/main.js

@@ -18,6 +18,7 @@ import { handleSlackTool, slackToolSpec } from "./channels/slack-tool.js";
 import { parseConversation as parseSlackConversation } from "./channels/slack.js";
 import { EventHub } from "./core/hub.js";
 import { pierDb } from "./db.js";
+import { deliverLedger, drainForRestart, RestartLedger } from "./drain.js";
 import { surfacePrompt } from "./core/reply.js";
 import { Router } from "./core/router.js";
 import { logger } from "./log.js";
@@ -27,7 +28,9 @@ import { TaskStore } from "./tasks/store.js";
 import { taskToolSpec } from "./tasks/tool.js";
 import { PIER_HOME, pierPath } from "./paths.js";
 import { Secrets } from "./secrets.js";
+import { startUpdate, unitPath, updaterProblem } from "./service.js";
 import { SettingsStore } from "./settings.js";
+import { startAutoUpdate, UpdateCheck } from "./update.js";
 import { AuthStore, registerAuthRoutes, requireAuth } from "./web/auth.js";
 import { SessionStateStore } from "./web/session-state.js";
 import { createServer } from "./web/server.js";
@@ -64,6 +67,7 @@ let channelStore;
 // Shared by the adapter and the tool: a display name is looked up once per
 // process, not once per message and again per transcript.
 const slackDirectory = new SlackDirectory((m) => logger("slack").warn(m));
+const piConfig = new PiConfigStore();
 const factory = new PiAgentFactory([
     taskToolSpec((params, callerSessionId) => tasks.tool(params, callerSessionId)),
     slackToolSpec((params, callerSessionId) => handleSlackTool({
@@ -95,7 +99,9 @@ const factory = new PiAgentFactory([
 [fileURLToPath(new URL("../skills", import.meta.url))], 
 // Provider credentials live sealed in pier.db; a leftover auth.json is
 // imported on first use and renamed to auth.json.imported.
-new CredentialStore(db, secrets));
+new CredentialStore(db, secrets), piConfig, 
+// Operator pins ride ahead of the curated catalog in every model picker.
+() => settings.get().modelMenu);
 const hub = new EventHub();
 const router = new Router(hub, (key) => {
     // Web conversation ids ARE session ids; an IM conversation id is a chat or a
@@ -106,7 +112,12 @@ const router = new Router(hub, (key) => {
     }
     return resolveIm(key);
 });
-tasks = new TaskService(new TaskStore(db), factory, router, hub);
+// An attached session holds a live Pi runtime and its transcript, and nothing
+// else ever lets one go: without this, one per conversation ever answered.
+const stopEviction = router.startIdleEviction();
+tasks = new TaskService(new TaskStore(db), factory, router, hub, {
+    modelMenu: () => settings.get().modelMenu,
+});
 tasks.start();
 channelStore = new ChannelStore(db, secrets);
 const control = createControl({ router, factory, conversations, store: channelStore });
@@ -115,7 +126,110 @@ resolveIm = resolveConversation(conversations, factory, control.launchFor, (mess
 // Channels connect only once tokens are readable. A refused unlock (vt denial,
 // corrupt master.key) must not take the web surface down — it is where the
 // operator goes to repair — but it is named loudly, not served as silence.
-void secrets.unlock().then(() => channels.reload(), (err) => log.error("secrets locked — channels not started; unlock from Console → Settings → Security, or repair master.key", err));
+// Once they are up, the chats a previous restart cut off are told (drain.ts) —
+// on this path and on a later Console unlock alike, because a note held back
+// by locked secrets must not wait for yet another restart.
+const restartLedger = new RestartLedger(db);
+const startChannels = async () => {
+    await channels.reload();
+    await deliverLedger(restartLedger, (entry) => channels.notify(entry.channelId, entry.conversationId, entry.note))
+        .catch((err) => log.error("restart-note delivery failed", err));
+};
+void secrets.unlock().then(startChannels, (err) => log.error("secrets locked — channels not started; unlock from Console → Settings → Security, or repair master.key", err));
+// Replacing Pier is systemd's job, not this process's: the oneshot unit stops
+// the service, snapshots the database, installs and starts it again. Without
+// that unit there is nothing to hand the work to, and the Console says so
+// instead of offering a button that cannot work.
+const updates = new UpdateCheck();
+// Asked once at boot, not lazily on the first page load: a restart is exactly
+// when "am I current?" is worth knowing, and it puts the answer in the journal
+// of a Pier nobody has a browser open on.
+void updates.refresh();
+/**
+ * Hand over, but not onto a running turn. The updater's first act is
+ * `systemctl stop`, i.e. a SIGTERM, which is the *fast* teardown — so anything
+ * that started since the idle check would be killed with no note anywhere. The
+ * gate closes first and the drain waits, exactly as `pier restart` does,
+ * ledger included; only then is the install handed over. A handover that never
+ * starts reopens the gate, because a Pier that silently refuses every message
+ * forever is worse than the race it was avoiding.
+ */
+// Shared restart state. The updater's handover, the SIGUSR2 drain (below) and
+// the final teardown must see each other: without this, two paths drain the
+// same Pier at once, and a failure on one reopens the gate the other still
+// needs shut.
+let handingOver = false;
+let draining = false;
+let shuttingDown = false;
+const takeWorkAgain = (why) => {
+    handingOver = false;
+    log.error(`${why} — taking work again`);
+    // Not ours to reopen: a SIGUSR2 restart or the teardown owns the gate now,
+    // and reopening it would hand new work to a process that is exiting.
+    if (draining || shuttingDown)
+        return;
+    router.endDrain();
+    tasks.unpause();
+    // The drain may have deadline-aborted turns into the ledger. Without the
+    // restart that was supposed to follow, that debt would wait for one days
+    // away (§5b) — so the chats are told now, by the process that cut them off.
+    void deliverLedger(restartLedger, (entry) => channels.notify(entry.channelId, entry.conversationId, entry.note))
+        .catch((err) => log.error("restart-note delivery failed", err));
+};
+/** How long the handover has to actually stop us. `systemctl start --no-block`
+ *  returns when the job is *queued*, so "started" is not proof of anything;
+ *  the real outcome is a SIGTERM a second or two later. */
+const HANDOVER_GRACE_MS = 60_000;
+const handOverToUpdater = async () => {
+    // One handover at a time, and never on top of a restart: the Console button,
+    // the auto-update tick and SIGUSR2 would otherwise drain the same Pier
+    // twice, each believing the gate is its own to reopen on failure.
+    if (handingOver || draining || shuttingDown)
+        return "busy";
+    handingOver = true;
+    await drainForRestart({ router, tasks, ledger: restartLedger });
+    const started = startUpdate({ say: (message) => log.info(message) });
+    if (started !== "started") {
+        takeWorkAgain(`update not started (${started})`);
+        return started;
+    }
+    // The gate is closed and nothing in this process will open it again, so a
+    // handover that queues and then goes nowhere — npm failed, the unit was
+    // masked, the job sat behind another — would leave Pier alive and refusing
+    // every message with no way back. Unref'd: this must not be what keeps the
+    // process up while systemd is trying to stop it.
+    setTimeout(() => {
+        takeWorkAgain(`still running ${String(HANDOVER_GRACE_MS / 1000)}s after handing over — pier-update.service never stopped Pier` +
+            ` (check: journalctl --user -u pier-update.service -e)`);
+    }, HANDOVER_GRACE_MS).unref();
+    return started;
+};
+const updater = process.platform === "linux" && existsSync(unitPath())
+    ? { apply: handOverToUpdater, problem: () => updaterProblem() }
+    : null;
+// Unattended only when the operator asked for it *and* nothing is running.
+if (updater) {
+    const problem = updaterProblem();
+    // Loudly, at boot: this is the one moment the operator is looking, and the
+    // alternative is a restart that fails months from now.
+    if (problem)
+        log.warn(`the updater cannot run: ${problem}`);
+    startAutoUpdate(updates, {
+        enabled: () => settings.get().autoUpdate,
+        idle: () => router.busy().length === 0 && tasks.activeRunCount() === 0,
+        apply: async () => {
+            // Re-checked here, not only at boot: a version manager can remove the
+            // recorded Node months into an uptime, and draining for a handover that
+            // cannot happen would take the whole instance down with it.
+            const now = updaterProblem();
+            if (now) {
+                log.error(`auto-update skipped: ${now}`);
+                return "not-installed";
+            }
+            return handOverToUpdater();
+        },
+    });
+}
 // Composition happens here so web/ and tasks/ never import each other.
 const app = new Hono();
 // A route that threw would otherwise answer 500 and leave no trace anywhere:
@@ -129,8 +243,8 @@ app.onError((err, c) => {
 // so a surface added later is covered without knowing this exists. Built
 // before the listener: a first run generates and prints its password here.
 const auth = new AuthStore(db);
-registerAuthRoutes(app, auth);
 app.use("*", requireAuth(auth));
+registerAuthRoutes(app, auth);
 registerTaskRoutes(app, tasks, { factory, router });
 registerChannelRoutes(app, channelStore, channels);
 registerBoardRoutes(app);
@@ -139,11 +253,14 @@ app.route("/", createServer({
     router,
     hub,
     sessions: new SessionStateStore(db),
-    config: new PiConfigStore(),
+    config: piConfig,
+    providers: factory,
     settings,
     secrets,
+    updates,
+    updater,
     // Unlocked from the Console: start the channels boot held back.
-    onUnlocked: () => void channels.reload(),
+    onUnlocked: () => void startChannels(),
     backgroundRuns: (id) => tasks.backgroundRuns(id),
 }));
 const port = Number(process.env.PORT ?? 3141);
@@ -164,20 +281,65 @@ process.on("uncaughtException", (err) => {
 process.on("unhandledRejection", (reason) => {
     log.error("unhandled rejection", reason);
 });
+const shutdown = (stopTasks = true) => {
+    // Once: SIGTERM can land while a drain is finishing, and two teardowns
+    // racing each other close the same sockets twice.
+    if (shuttingDown)
+        return;
+    shuttingDown = true;
+    // Best-effort, and bounded: a socket an adapter cannot close must not turn
+    // `systemctl restart` into a 90-second wait for SIGKILL.
+    setTimeout(() => process.exit(0), 3000).unref();
+    stopEviction();
+    // The drain path leaves task runs alone: aborting them here would record
+    // them cancelled and race their callbacks against dying channels, when the
+    // boot-time interrupted marking is the recovery that was promised.
+    if (stopTasks)
+        tasks.stop();
+    void channels.stop().finally(() => {
+        server.close(() => process.exit(0));
+        // Every workbench tab holds an SSE stream open, so `close()` alone would
+        // always wait out the timer above. (`in` because the served type is a
+        // union with HTTP/2, which has no such method — and no such problem.)
+        if ("closeAllConnections" in server)
+            server.closeAllConnections();
+    });
+};
 for (const signal of ["SIGTERM", "SIGINT"]) {
     process.once(signal, () => {
         log.info(`${signal} received, shutting down`);
-        // Best-effort, and bounded: a socket an adapter cannot close must not turn
-        // `systemctl restart` into a 90-second wait for SIGKILL.
-        setTimeout(() => process.exit(0), 3000).unref();
-        tasks.stop();
-        void channels.stop().finally(() => {
-            server.close(() => process.exit(0));
-            // Every workbench tab holds an SSE stream open, so `close()` alone would
-            // always wait out the timer above. (`in` because the served type is a
-            // union with HTTP/2, which has no such method — and no such problem.)
-            if ("closeAllConnections" in server)
-                server.closeAllConnections();
-        });
+        shutdown();
+    });
+}
+// The slow restart (`pier restart`): refuse new work, let running turns finish
+// — bounded by the drain deadline — then exit for `Restart=always` to bring the
+// next process up. SIGTERM above stays the fast path systemd expects. `on`,
+// not `once`: a second SIGUSR2 with no handler would fall back to Node's
+// default and kill the drain it meant to hurry.
+process.on("SIGUSR2", () => {
+    if (draining) {
+        log.info("SIGUSR2 received again — already draining");
+        return;
+    }
+    draining = true;
+    log.info("SIGUSR2 received, draining for restart");
+    void drainForRestart({ router, tasks, ledger: restartLedger })
+        .catch((err) => log.error("drain failed — shutting down anyway", err))
+        .then(() => shutdown(false));
+});
+// Reload without a restart (`pier reload`): adapters re-read their config, and
+// idle sessions are let go so the next message re-opens them with the current
+// skills, extensions and prompts — all applied at attach, none stored in a
+// transcript. Streaming or watched sessions pick the change up at their next
+// natural eviction. Only under systemd (the CLI signals through systemctl):
+// a foreground `pier` keeps SIGHUP's default, dying with its terminal instead
+// of surviving as an orphan that holds the port.
+if (process.env.INVOCATION_ID) {
+    process.on("SIGHUP", () => {
+        log.info("SIGHUP received, reloading channels and recycling idle sessions");
+        void channels.reload();
+        void router.evictIdle(0)
+            .then((n) => log.info(`recycled ${String(n)} idle session(s)`))
+            .catch((err) => log.error("session recycle failed", err));
     });
 }

package/dist/secrets.js

@@ -26,7 +26,6 @@ export class Secrets {
     path;
     vt;
     #dek;
-    #kek;
     #file;
     /** Why decrypt is refused right now — "" once unlocked. */
     #lockedReason = "unlock() has not run";
@@ -58,7 +57,12 @@ export class Secrets {
             try {
                 raw = readFileSync(this.path, "utf8");
             }
-            catch {
+            catch (err) {
+                // Only a missing file means first boot. Any other read error (EACCES,
+                // EISDIR…) must not fall through to #create(), which would rename a
+                // fresh key over the existing one and destroy every sealed credential.
+                if (err.code !== "ENOENT")
+                    throw err;
                 this.#file = this.#create();
                 log.info(`created ${this.path} (file mode)`);
                 raw = readFileSync(this.path, "utf8");
@@ -66,12 +70,12 @@ export class Secrets {
             const file = JSON.parse(raw);
             if (!file.kek || !file.dek || !file.dekId)
                 throw new Error(`${this.path} is malformed`);
-            this.#kek = file.kek.startsWith("vt://")
+            const kek = file.kek.startsWith("vt://")
                 ? Buffer.from(await this.vt.read(file.kek), "base64")
                 : Buffer.from(file.kek, "base64");
-            if (this.#kek.length !== KEY_BYTES)
+            if (kek.length !== KEY_BYTES)
                 throw new Error(`${this.path} KEK is not ${KEY_BYTES} bytes`);
-            this.#dek = open(this.#kek, file.dek, `kek:${file.dekId}`);
+            this.#dek = open(kek, file.dek, `kek:${file.dekId}`);
             this.#file = file;
             this.#lockedReason = "";
             log.info(`secrets unlocked (${this.mode} mode, dek ${file.dekId})`);
@@ -112,7 +116,6 @@ export class Secrets {
             throw new Error("vt create did not return a vt:// record");
         }
         this.#write(next);
-        this.#kek = kek;
         this.#file = next;
         log.info(`KEK rotated (${mode} mode, dek ${file.dekId} unchanged)`);
     }
@@ -184,6 +187,7 @@ function run(cmd, args, stdin) {
             else
                 reject(new Error(`${cmd} ${args[0]} exited ${code}: ${err.trim() || out.trim()}`));
         });
+        child.stdin.on("error", reject); // EPIPE if vt exits before reading
         if (stdin !== undefined)
             child.stdin.write(stdin);
         child.stdin.end();