@hyperdrive.bot/paseo-server 0.2.8 → 0.3.1

package/dist/server/server/search/indexer.js

@@ -0,0 +1,362 @@
+// MARRA CUSTOMIZATION — incremental, provider-agnostic session indexer.
+//
+// Discovers transcripts from every installed provider, reads only the bytes
+// appended since the last pass, extracts searchable text, and commits to the
+// FTS5 store. Runs on a background timer (see session-search-service.ts), so
+// indexing never happens on the query path — that coupling is exactly what
+// made the recall-based search unusable on a loaded host.
+//
+// `runIndexPass` is a pure function of (store, adapters), so it can move to a
+// worker_thread unchanged if a future corpus makes passes expensive. Measured
+// today: a steady-state pass over 8,117 files reads ~0 bytes in 0.9 s, which
+// does not justify the thread.
+//
+// The append-only byte-offset scheme is the core of it. Claude transcripts
+// grow to 214 MB; re-reading them per pass is what killed the old design.
+// Here a file is read once in full, then each subsequent pass touches only the
+// tail. Safety valve: if a file SHRANK, the append-only assumption is broken
+// (rewritten/rotated), so we forget it and re-read from zero.
+//
+// Partial trailing lines matter. A transcript is often mid-append when we read
+// it, leaving a truncated final line that will not parse. We advance the
+// offset only to the last complete newline, so the fragment is re-read intact
+// on the next pass instead of being dropped.
+import { createReadStream } from "node:fs";
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { resolvePaseoHome } from "../paseo-home.js";
+import { extractClaudeRecord, extractCodexRecord, extractPaseoAgent, } from "./extract.js";
+// ---------------------------------------------------------------------------
+// Discovery helpers
+// ---------------------------------------------------------------------------
+/**
+ * Claude encodes the session cwd in the project dir name by replacing every
+ * non-alphanumeric character with "-". That is lossy (a real "-" and a "/"
+ * both become "-"), so this is a best-effort display value — matching what
+ * all-sessions-route.ts already does, deliberately, so both agree.
+ */
+function cwdFromClaudeSlug(slug) {
+    return slug.startsWith("-") ? `/${slug.slice(1).replace(/-/g, "/")}` : slug;
+}
+async function listDir(dir) {
+    try {
+        return await fs.readdir(dir);
+    }
+    catch {
+        return [];
+    }
+}
+async function walkFiles(root, suffix) {
+    const out = [];
+    const stack = [root];
+    while (stack.length > 0) {
+        const dir = stack.pop();
+        let entries;
+        try {
+            entries = await fs.readdir(dir, { withFileTypes: true });
+        }
+        catch {
+            continue;
+        }
+        for (const e of entries) {
+            const full = path.join(dir, e.name);
+            if (e.isDirectory())
+                stack.push(full);
+            else if (e.isFile() && e.name.endsWith(suffix))
+                out.push(full);
+        }
+    }
+    return out;
+}
+export function createClaudeAdapter(root) {
+    const projectsRoot = root ?? path.join(os.homedir(), ".claude", "projects");
+    return {
+        name: "claude",
+        extract: extractClaudeRecord,
+        async discover() {
+            const out = [];
+            for (const slug of await listDir(projectsRoot)) {
+                const dir = path.join(projectsRoot, slug);
+                const cwd = cwdFromClaudeSlug(slug);
+                // Two layouts live under a project dir, and a two-level walk finds
+                // only the first (measured: 5,449 of 6,957 files — 1,508 silently
+                // missing):
+                //   <slug>/<sid>.jsonl                            — main session
+                //   <slug>/<sid>/subagents/agent-<id>.jsonl       — subagent runs
+                // Subagent transcripts are real content, so they are indexed and
+                // attributed to the PARENT session id: a hit inside a subagent should
+                // open the conversation that spawned it, not a fragment nobody can
+                // navigate to.
+                for (const entry of await walkFiles(dir, ".jsonl")) {
+                    const rel = path.relative(dir, entry);
+                    const segments = rel.split(path.sep);
+                    const sid = segments.length === 1
+                        ? path.basename(entry, ".jsonl") // <sid>.jsonl
+                        : segments[0]; // <sid>/subagents/agent-*.jsonl
+                    out.push({ path: entry, source: "claude", sid, cwd });
+                }
+            }
+            return out;
+        },
+    };
+}
+export function createCodexAdapter(root) {
+    // UNVERIFIED against real data: ~/.codex/ on the dev box has no sessions,
+    // only prompts/. Adapter is fixture-tested; re-validate on first real run.
+    const sessionsRoot = root ?? path.join(os.homedir(), ".codex", "sessions");
+    return {
+        name: "codex",
+        extract: extractCodexRecord,
+        async discover() {
+            const files = await walkFiles(sessionsRoot, ".jsonl");
+            return files.map((p) => {
+                const base = path.basename(p, ".jsonl");
+                // rollout-2026-07-29T10-11-12-<uuid>.jsonl → trailing uuid is the sid.
+                const m = /([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})/i.exec(base);
+                return { path: p, source: "codex", sid: m?.[1] ?? base, cwd: "" };
+            });
+        },
+    };
+}
+export function createPaseoAdapter(root) {
+    // resolvePaseoHome(), not os.homedir(): a daemon running with PASEO_HOME set
+    // must index ITS OWN agents, not the ones belonging to the user's live daemon.
+    const agentsRoot = root ?? path.join(resolvePaseoHome(), "agents");
+    return {
+        name: "paseo",
+        // Whole-document JSON, pretty-printed and rewritten in place on every
+        // agent update — NOT append-only JSONL. Parsing it line-by-line yields
+        // nothing (each line is a JSON fragment).
+        format: "json",
+        // Paseo records are metadata-only (median 2 KB, no transcript text). The
+        // adapter indexes title + labels so agents are findable by name, and the
+        // record's runtimeInfo.sessionId is what links a hit to real transcript
+        // text held by the provider adapters.
+        extract(record) {
+            const { meta, blocks } = extractPaseoAgent(record);
+            // The agent record's own cwd is authoritative for a paseo hit.
+            return { blocks, timestamp: meta?.updatedAt ?? null, sid: null, cwd: meta?.cwd ?? null };
+        },
+        async discover() {
+            const files = await walkFiles(agentsRoot, ".json");
+            return files.map((p) => ({
+                path: p,
+                source: "paseo",
+                sid: path.basename(p, ".json"),
+                cwd: "",
+            }));
+        },
+    };
+}
+export function defaultAdapters() {
+    return [createClaudeAdapter(), createCodexAdapter(), createPaseoAdapter()];
+}
+// ---------------------------------------------------------------------------
+// Incremental pass
+// ---------------------------------------------------------------------------
+/** Read [start, end) of a file as UTF-8 without loading the whole thing. */
+async function readRange(filePath, start, end) {
+    if (end <= start)
+        return "";
+    return await new Promise((resolve, reject) => {
+        const chunks = [];
+        const stream = createReadStream(filePath, { start, end: end - 1 });
+        stream.on("data", (c) => chunks.push(c));
+        stream.on("error", reject);
+        stream.on("close", () => resolve(Buffer.concat(chunks).toString("utf-8")));
+    });
+}
+const DEFAULT_MAX_BYTES_PER_FILE = 64 * 1024 * 1024;
+/**
+ * One incremental pass over every discovered transcript. Safe to call
+ * repeatedly; work is proportional to bytes appended since the last call, not
+ * to corpus size.
+ */
+export async function runIndexPass(store, options = {}) {
+    const started = Date.now();
+    const adapters = options.adapters ?? defaultAdapters();
+    const log = options.logger;
+    const maxBytes = options.maxBytesPerFile ?? DEFAULT_MAX_BYTES_PER_FILE;
+    const result = {
+        filesPartial: 0,
+        filesScanned: 0,
+        filesUpdated: 0,
+        filesReset: 0,
+        filesPruned: 0,
+        bytesRead: 0,
+        chunksAdded: 0,
+        errors: 0,
+        durationMs: 0,
+    };
+    const known = store.getFileStates();
+    const seen = new Set();
+    for (const adapter of adapters) {
+        const discovered = await adapter.discover();
+        for (const file of discovered) {
+            if (options.signal?.aborted) {
+                result.durationMs = Date.now() - started;
+                return result;
+            }
+            seen.add(file.path);
+            result.filesScanned += 1;
+            await indexOneFile({ store, adapter, file, known, result, maxBytes, log });
+        }
+    }
+    result.filesPruned = store.pruneMissing(seen);
+    result.durationMs = Date.now() - started;
+    log?.info(result, "session-index: pass complete");
+    return result;
+}
+/**
+ * Decide how much of one file still needs reading, and where from.
+ * Returns null when there is nothing to do this pass.
+ */
+function planRead(args, isWholeDoc, stat) {
+    const { store, file, known, result } = args;
+    const prior = known.get(file.path);
+    // Unchanged since last pass — nothing appended, skip without reading.
+    if (prior && stat.size === prior.size && Math.floor(stat.mtimeMs) === prior.mtimeMs) {
+        return null;
+    }
+    let start = prior?.offset ?? 0;
+    if (isWholeDoc) {
+        // Whole-document files are rewritten in place, so any change invalidates
+        // every chunk we hold for them. Drop and re-read from zero.
+        if (prior)
+            store.forgetFile(file.path);
+        start = 0;
+    }
+    else if (prior && stat.size < prior.offset) {
+        // Shrank: append-only assumption broken (rewritten or rotated).
+        store.forgetFile(file.path);
+        start = 0;
+        result.filesReset += 1;
+    }
+    return stat.size <= start ? null : start;
+}
+/** Parse the consumable text into chunks, resolving the session's true cwd. */
+function buildChunks(args, consumable, isWholeDoc) {
+    const { adapter, file, result, log } = args;
+    // The cwd a hit is resumed into. Start from the slug-derived guess, but
+    // upgrade to the first authoritative cwd the transcript itself reports —
+    // the slug is a lossy encoding and decoding it yields directories that do
+    // not exist (`super-repo` → `super/repo`), which is precisely what makes a
+    // search hit unresumable.
+    let resolvedCwd = file.cwd;
+    let cwdIsAuthoritative = false;
+    const chunks = [];
+    for (const line of isWholeDoc ? [consumable] : consumable.split("\n")) {
+        if (line.trim().length === 0)
+            continue;
+        let record;
+        try {
+            record = JSON.parse(line);
+        }
+        catch {
+            continue; // Non-JSON noise, not an error worth counting.
+        }
+        let extracted;
+        try {
+            extracted = adapter.extract(record);
+        }
+        catch (err) {
+            result.errors += 1;
+            log?.debug({ err, path: file.path }, "session-index: extract failed");
+            continue;
+        }
+        if (!cwdIsAuthoritative && extracted.cwd) {
+            resolvedCwd = extracted.cwd;
+            cwdIsAuthoritative = true;
+        }
+        for (const block of extracted.blocks) {
+            chunks.push({
+                path: file.path,
+                sid: extracted.sid ?? file.sid,
+                source: file.source,
+                cwd: resolvedCwd,
+                timestamp: extracted.timestamp,
+                text: block.lane === "text" ? block.text : "",
+                tool: block.lane === "tool" ? block.text : "",
+                truncated: block.truncated,
+            });
+        }
+    }
+    return { chunks, resolvedCwd };
+}
+/** Read the pending tail of one file and commit its chunks. */
+async function indexOneFile(args) {
+    const { store, adapter, file, result, maxBytes, log } = args;
+    let stat;
+    try {
+        stat = await fs.stat(file.path);
+    }
+    catch {
+        return;
+    }
+    const isWholeDoc = (adapter.format ?? "jsonl") === "json";
+    const start = planRead(args, isWholeDoc, stat);
+    if (start === null)
+        return;
+    const end = Math.min(stat.size, start + maxBytes);
+    let raw;
+    try {
+        raw = await readRange(file.path, start, end);
+    }
+    catch (err) {
+        result.errors += 1;
+        log?.warn({ err, path: file.path }, "session-index: read failed");
+        return;
+    }
+    result.bytesRead += end - start;
+    // JSONL: advance only to the last complete newline, so a mid-append fragment
+    // is left unconsumed and re-read intact on the next pass. Whole-doc: the
+    // entire read IS the record; there is nothing to defer.
+    let consumable;
+    if (isWholeDoc) {
+        consumable = raw;
+    }
+    else {
+        const lastNewline = raw.lastIndexOf("\n");
+        consumable = lastNewline >= 0 ? raw.slice(0, lastNewline + 1) : "";
+    }
+    const consumedBytes = Buffer.byteLength(consumable, "utf-8");
+    if (consumedBytes === 0)
+        return;
+    const { chunks, resolvedCwd } = buildChunks(args, consumable, isWholeDoc);
+    const nextState = {
+        path: file.path,
+        size: stat.size,
+        // Only claim the mtime as fully-consumed when we reached EOF; a capped
+        // read must stay "dirty" so the next pass continues the same file.
+        mtimeMs: start + consumedBytes >= stat.size ? Math.floor(stat.mtimeMs) : 0,
+        offset: start + consumedBytes,
+    };
+    try {
+        store.commitFile(nextState, file.source, file.sid, resolvedCwd, chunks);
+        result.filesUpdated += 1;
+        result.chunksAdded += chunks.length;
+        // A file capped by maxBytesPerFile is NOT fully searchable yet. Count it so
+        // "indexing complete" can never be claimed while tails are missing —
+        // silent partial coverage is the failure mode that makes a search box look
+        // like it works while quietly losing your history.
+        if (nextState.offset < stat.size)
+            result.filesPartial += 1;
+    }
+    catch (err) {
+        result.errors += 1;
+        log?.warn({ err, path: file.path }, "session-index: commit failed");
+    }
+}
+/**
+ * Default on-disk location for the index (a derived cache, safe to delete).
+ *
+ * MUST resolve through `resolvePaseoHome()` and not `os.homedir()`: a daemon
+ * started with PASEO_HOME pointing elsewhere (tests, a second instance, an
+ * isolated E2E run) would otherwise write its index into the real ~/.paseo and
+ * silently share state with the user's live daemon.
+ */
+export function defaultIndexPath() {
+    return path.join(resolvePaseoHome(), "session-index.sqlite");
+}
+//# sourceMappingURL=indexer.js.map

package/dist/server/server/search/resolve-cwd.d.ts

@@ -0,0 +1,40 @@
+export interface ResolveCwdDeps {
+    /** Override for tests. Defaults to `fs.stat` + isDirectory. */
+    dirExists?: (candidate: string) => Promise<boolean>;
+    /** Override the current home. Defaults to `os.homedir()`. */
+    homeDir?: string;
+}
+/**
+ * Split `/Users/claudinho/super-repo` into `{ home: "/Users/claudinho",
+ * rest: "super-repo" }`. Returns null when the path is not under a recognised
+ * home layout (`/opt/...`, a bare `/tmp/x`, a Windows path, …), in which case
+ * no remap is attempted.
+ */
+export declare function splitHome(cwd: string): {
+    home: string;
+    rest: string;
+} | null;
+export interface ResolvedCwd {
+    /** The cwd to resume into. */
+    cwd: string;
+    /** How we got there — surfaced so the UI can be honest about a remap. */
+    resolution: "direct" | "remapped" | "unresolved";
+}
+/**
+ * Resolve a recorded cwd to one that exists on THIS machine.
+ *
+ * - `direct`     — the recorded cwd exists; use it unchanged.
+ * - `remapped`   — it did not exist, but the same relative path under the
+ *                  current home does. Resume there.
+ * - `unresolved` — neither exists. The original value is returned untouched;
+ *                  callers should surface it as not-resumable-here rather
+ *                  than silently resuming somewhere else.
+ */
+export declare function resolveResumableCwd(recordedCwd: string, deps?: ResolveCwdDeps): Promise<ResolvedCwd>;
+/**
+ * Batch resolver with a per-call cache. A search returns up to 100 hits that
+ * routinely share a handful of cwds, so this collapses ~100 stat storms into
+ * one stat per distinct directory.
+ */
+export declare function createCwdResolver(deps?: ResolveCwdDeps): (recordedCwd: string) => Promise<ResolvedCwd>;
+//# sourceMappingURL=resolve-cwd.d.ts.map

package/dist/server/server/search/resolve-cwd.js

@@ -0,0 +1,109 @@
+// MARRA CUSTOMIZATION — make a search hit's cwd actually resumable.
+//
+// A hit is only useful if clicking it reopens the session, and the palette
+// resumes a session INTO its cwd. Two things broke that before:
+//
+//  1. A FABRICATED cwd. Claude's project-dir slug replaces every
+//     non-alphanumeric character with "-", so it is lossy and irreversible:
+//     `super-repo` and `super/repo` produce the same slug. Decoding it back
+//     yielded `/home/claudiao/super/repo`, which does not exist. Fixed
+//     upstream of this file — the indexer now reads the authoritative `cwd`
+//     out of the transcript records themselves.
+//
+//  2. A CROSS-MACHINE cwd. History moves between machines while paths do not.
+//     Measured on the real corpus: 3,465 of 6,970 Claude sessions carry a
+//     `/Users/claudinho/...` macOS path that has no meaning on this Linux
+//     box. Correct value, wrong machine — 48.2% resumable in place.
+//
+// This module handles (2): if the recorded cwd is absent but the same
+// path under the CURRENT home exists, resume there. Measured effect on the
+// real corpus: 48.2% → 96.1% resumable (3,341 sessions recovered, 269 left
+// genuinely unresolvable — directories that exist on no machine here).
+//
+// Deliberately conservative: it only ever remaps when the target directory
+// EXISTS. It never invents a path, and an unresolvable hit keeps its original
+// cwd so the UI shows the truth rather than a plausible lie.
+import { promises as fsp } from "node:fs";
+import os from "node:os";
+import path from "node:path";
+/**
+ * Home-directory prefixes seen across this user's machines. Order matters
+ * only for determinism; the first candidate that EXISTS wins.
+ */
+const KNOWN_HOME_PREFIXES = ["/Users", "/home", "/root"];
+async function defaultDirExists(candidate) {
+    try {
+        const stats = await fsp.stat(candidate);
+        return stats.isDirectory();
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Split `/Users/claudinho/super-repo` into `{ home: "/Users/claudinho",
+ * rest: "super-repo" }`. Returns null when the path is not under a recognised
+ * home layout (`/opt/...`, a bare `/tmp/x`, a Windows path, …), in which case
+ * no remap is attempted.
+ */
+export function splitHome(cwd) {
+    for (const prefix of KNOWN_HOME_PREFIXES) {
+        if (!cwd.startsWith(`${prefix}/`))
+            continue;
+        const after = cwd.slice(prefix.length + 1);
+        const slash = after.indexOf("/");
+        // `/Users/claudinho` with no trailing path — a home dir itself.
+        if (slash === -1)
+            return { home: `${prefix}/${after}`, rest: "" };
+        return { home: `${prefix}/${after.slice(0, slash)}`, rest: after.slice(slash + 1) };
+    }
+    return null;
+}
+/**
+ * Resolve a recorded cwd to one that exists on THIS machine.
+ *
+ * - `direct`     — the recorded cwd exists; use it unchanged.
+ * - `remapped`   — it did not exist, but the same relative path under the
+ *                  current home does. Resume there.
+ * - `unresolved` — neither exists. The original value is returned untouched;
+ *                  callers should surface it as not-resumable-here rather
+ *                  than silently resuming somewhere else.
+ */
+export async function resolveResumableCwd(recordedCwd, deps = {}) {
+    const cwd = recordedCwd?.trim() ?? "";
+    if (cwd.length === 0)
+        return { cwd: recordedCwd ?? "", resolution: "unresolved" };
+    const dirExists = deps.dirExists ?? defaultDirExists;
+    if (await dirExists(cwd))
+        return { cwd, resolution: "direct" };
+    const home = deps.homeDir ?? os.homedir();
+    const split = splitHome(cwd);
+    if (!split)
+        return { cwd, resolution: "unresolved" };
+    // Already under the current home and still missing — the directory is gone,
+    // not relocated. Remapping would produce the same failing path.
+    if (split.home === home)
+        return { cwd, resolution: "unresolved" };
+    const candidate = split.rest.length === 0 ? home : path.join(home, split.rest);
+    if (await dirExists(candidate))
+        return { cwd: candidate, resolution: "remapped" };
+    return { cwd, resolution: "unresolved" };
+}
+/**
+ * Batch resolver with a per-call cache. A search returns up to 100 hits that
+ * routinely share a handful of cwds, so this collapses ~100 stat storms into
+ * one stat per distinct directory.
+ */
+export function createCwdResolver(deps = {}) {
+    const cache = new Map();
+    return (recordedCwd) => {
+        const key = recordedCwd ?? "";
+        const hit = cache.get(key);
+        if (hit)
+            return hit;
+        const pending = resolveResumableCwd(key, deps);
+        cache.set(key, pending);
+        return pending;
+    };
+}
+//# sourceMappingURL=resolve-cwd.js.map

package/dist/server/server/search/session-search-service.d.ts

@@ -0,0 +1,59 @@
+import type pino from "pino";
+import { type SourceAdapter } from "./indexer.js";
+/** Matches the pre-existing recall contract so no transport or client changes. */
+export interface SearchResultEntry {
+    sid: string;
+    source: string;
+    cwd: string;
+    timestamp: string;
+    snippet: string;
+    resumeCommand: string;
+}
+export interface SearchResultsResponse {
+    query: string;
+    results: SearchResultEntry[];
+}
+export interface SessionSearchServiceOptions {
+    indexPath?: string;
+    adapters?: SourceAdapter[];
+    logger?: pino.Logger;
+    /** Gap between background passes. Default 60s. */
+    intervalMs?: number;
+    /** Skip the timer (tests drive passes manually). */
+    manual?: boolean;
+}
+export declare class SessionSearchService {
+    private readonly store;
+    private readonly adapters;
+    private readonly logger;
+    private readonly intervalMs;
+    private timer;
+    private readonly abort;
+    private passInFlight;
+    /** False until the first pass finishes, so callers can be honest about warm-up. */
+    private warm;
+    constructor(options?: SessionSearchServiceOptions);
+    private start;
+    /** One indexing pass. Overlapping calls join the in-flight pass. */
+    runPass(): Promise<void>;
+    /** True once at least one indexing pass has completed. */
+    isWarm(): boolean;
+    stats(): import("./index-store.js").IndexStats;
+    /**
+     * Query the index and return hits in the transport's existing shape.
+     *
+     * Every hit's cwd is resolved to a directory that exists on THIS machine
+     * before it is returned, because the palette resumes a session into its cwd
+     * — a hit pointing at another machine's path is a result you cannot click.
+     */
+    search(params: {
+        query: string;
+        limit?: number;
+        sources?: string;
+    }): Promise<SearchResultsResponse>;
+    close(): void;
+}
+export declare function getSessionSearchService(options?: SessionSearchServiceOptions): SessionSearchService;
+/** Test seam: drop the singleton so a fixture instance can take its place. */
+export declare function resetSessionSearchServiceForTests(): void;
+//# sourceMappingURL=session-search-service.d.ts.map