@pugi/cli 0.1.0-beta.10 → 0.1.0-beta.101

package/dist/core/codegraph/watcher.js

@@ -0,0 +1,440 @@
+/**
+ * Pugi local symbol index - chokidar file watcher (PR L2).
+ *
+ * Live auto-sync layer на top of the parser (PR L1) и DB (PR L0). When
+ * the operator runs `pugi index --watch`, this module:
+ *
+ *   1. Spins up a chokidar watcher rooted at `workspaceRoot`.
+ *   2. Filters events to the v1 parseable extensions
+ *      (`PARSEABLE_EXTENSIONS` from parser.ts).
+ *   3. Coalesces bursts via a 2-second debounce (configurable; tests
+ *      pass `debounceMs: 50` to keep specs fast).
+ *   4. On flush: per file -> `deleteFile` cascade -> `parseFile` ->
+ *      `insertSymbols` -> resolve `pendingEdges` -> `insertEdges`.
+ *   5. On `unlink`: `deleteFile` only (cascade removes symbols + edges
+ *      via the schema's FK ON DELETE CASCADE).
+ *
+ * Architectural decisions:
+ *
+ *  - **No nested transactions.** `deleteFile` / `insertSymbols` /
+ *    `insertEdges` each open + commit their own BEGIN block. We call
+ *    them sequentially and rely on per-call atomicity. A higher-level
+ *    SAVEPOINT wrapper was considered и rejected: chokidar batches are
+ *    small (typically 1-5 files) and the per-file work is already
+ *    transactional, so the extra level adds complexity without a
+ *    customer-visible win.
+ *
+ *  - **Two-step edge resolution.** `parseFile` returns `pendingEdges`
+ *    keyed by name + scope. We resolve them after `insertSymbols`
+ *    populates ids, using the shared `resolvePendingEdges` helper.
+ *    Cross-file targets are looked up via a SQL `SELECT id FROM
+ *    symbols WHERE name = ?` closure - the live DB IS the cross-file
+ *    map for the watcher. Reindex builds an in-memory map because it
+ *    sees every file in one pass; the watcher sees files one delta at
+ *    a time.
+ *
+ *  - **Path normalization.** Chokidar emits absolute or cwd-relative
+ *    paths depending on options. We always work in workspace-relative
+ *    POSIX form (`path.relative(root, abs).split(sep).join('/')`)
+ *    because that is what `reindex.ts` writes to `Symbol.file` and
+ *    `files.path`. Mismatched separators on Windows would break the
+ *    FK + the `deleteFile` lookup.
+ *
+ *  - **Exclude list parity.** We use the EXACT same `PRUNE_DIRS` as
+ *    `reindex.ts` (extended slightly with the v1 extensions the spec
+ *    pinned). Divergence between the walker and the watcher would
+ *    silently produce different indexes depending on entry point.
+ *
+ *  - **In-flight queue at close.** SIGINT / SIGTERM trigger
+ *    `close()`. If a debounce timer is still pending, we flush it
+ *    synchronously before tearing down chokidar - dropping a pending
+ *    edit would leave the index out of sync until the next manual
+ *    `pugi index` run.
+ *
+ *  - **Initial scan is the operator's job.** `ignoreInitial: true`
+ *    means we do NOT re-index the workspace on startup. The CLI
+ *    command runs `reindexWorkspace` first (covered by PR L1) и THEN
+ *    starts the watcher; this avoids a double-pass over the
+ *    workspace.
+ */
+import { existsSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { relative, resolve, sep, extname } from 'node:path';
+import chokidar from 'chokidar';
+import { deleteFile, insertEdges, insertSymbols, upsertFile, } from './db.js';
+import { PARSEABLE_EXTENSIONS, parseFile } from './parser.js';
+import { buildPerFileNameMap, resolvePendingEdges, } from './resolve-edges.js';
+/**
+ * Directories pruned from the watch. EXACT mirror of
+ * `reindex.ts:PRUNE_DIRS` so the walker и watcher agree on what
+ * "workspace" means. Touching this list requires touching both
+ * sites.
+ */
+const PRUNE_DIRS = Object.freeze([
+    'node_modules',
+    '.git',
+    'dist',
+    'build',
+    '.next',
+    '.turbo',
+    '.cache',
+    'coverage',
+    '.pugi',
+]);
+/** Default debounce window between burst-of-events and flush. */
+const DEFAULT_DEBOUNCE_MS = 2000;
+/**
+ * Start a chokidar watcher rooted at `workspaceRoot` against the
+ * caller-owned IndexDB. Returns a handle the caller invokes к stop the
+ * watcher cleanly.
+ *
+ * The function is sync - the watcher is "started" the moment chokidar
+ * exists, even if it has не yet emitted `ready`. The first `add`/
+ * `change` events flow into the debounce queue immediately.
+ */
+export function startWatcher(opts) {
+    const root = resolve(opts.workspaceRoot);
+    if (!existsSync(root)) {
+        throw new Error(`startWatcher: workspaceRoot does not exist: ${root}`);
+    }
+    const debounceMs = Math.max(0, opts.debounceMs ?? DEFAULT_DEBOUNCE_MS);
+    const writeStderr = opts.writeStderr ?? ((text) => void process.stderr.write(text));
+    const watchFn = opts.chokidarWatchFn ?? chokidar.watch;
+    const quiet = opts.quiet === true;
+    const stats = {
+        added: 0,
+        changed: 0,
+        removed: 0,
+        reparsed: 0,
+        edgesInserted: 0,
+        edgesOrphaned: 0,
+        parseErrors: 0,
+        watcherErrors: 0,
+    };
+    /** Workspace-relative POSIX path. */
+    const toRelPosix = (abs) => {
+        const rel = relative(root, abs);
+        if (rel.length === 0 || rel.startsWith('..'))
+            return '';
+        return rel.split(sep).join('/');
+    };
+    /**
+     * Defense in depth: chokidar matches on directory globs, but a
+     * watched-parent-dir event still might fire для excluded subpaths
+     * on some platforms. Re-check the relative path against the prune
+     * list before doing any work.
+     */
+    const isExcluded = (relPosix) => {
+        if (relPosix.length === 0)
+            return true;
+        const parts = relPosix.split('/');
+        for (const p of parts) {
+            if (PRUNE_DIRS.includes(p))
+                return true;
+        }
+        return false;
+    };
+    const isParseable = (relPosix) => {
+        const ext = extname(relPosix).toLowerCase();
+        return PARSEABLE_EXTENSIONS.includes(ext);
+    };
+    // ---- queue + debounce -------------------------------------------------
+    /** Files queued для re-parse on the next flush. */
+    const dirty = new Set();
+    /** Files queued для unlink-cascade on the next flush. */
+    const removed = new Set();
+    let timer = null;
+    /** Tracks the in-progress flush so close() can await it. */
+    let flushing = null;
+    /** Once closed, ignore further events. */
+    let closed = false;
+    const armDebounce = () => {
+        if (closed)
+            return;
+        if (timer)
+            clearTimeout(timer);
+        timer = setTimeout(() => {
+            timer = null;
+            void flush();
+        }, debounceMs);
+    };
+    /**
+     * Drain the queue. Sequential per-file work - the parser is async,
+     * SQLite handles are sync but already wrapped in per-call BEGIN/
+     * COMMIT inside db.ts, so concurrency would only add lock contention
+     * without throughput gain on a single-process watcher.
+     */
+    const flush = async () => {
+        // If a flush is already in-flight, queue behind it. We serialize
+        // so the dirty Set drained by one flush never collides with one
+        // mutated by another.
+        if (flushing) {
+            await flushing;
+            // Nothing new since the previous flush picked it up? Bail.
+            if (dirty.size === 0 && removed.size === 0)
+                return;
+        }
+        flushing = doFlush().finally(() => {
+            flushing = null;
+        });
+        return flushing;
+    };
+    const doFlush = async () => {
+        const snapshotDirty = Array.from(dirty);
+        dirty.clear();
+        const snapshotRemoved = Array.from(removed);
+        removed.clear();
+        // Process removals first - if a file was deleted and re-added in
+        // the same window we want the add half to write fresh rows.
+        for (const relPath of snapshotRemoved) {
+            try {
+                deleteFile(opts.db, relPath);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                if (!quiet) {
+                    writeStderr(`pugi index --watch: deleteFile(${relPath}) failed: ${message}\n`);
+                }
+            }
+        }
+        for (const relPath of snapshotDirty) {
+            // Drop файлы that were also removed in this batch - the unlink
+            // above already wiped them.
+            if (snapshotRemoved.includes(relPath))
+                continue;
+            const abs = resolve(root, relPath);
+            let bytes;
+            try {
+                bytes = await readFile(abs, 'utf8');
+            }
+            catch {
+                // Disappeared between event и flush - treat as unlink.
+                try {
+                    deleteFile(opts.db, relPath);
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    if (!quiet) {
+                        writeStderr(`pugi index --watch: post-disappear deleteFile(${relPath}) failed: ${message}\n`);
+                    }
+                }
+                continue;
+            }
+            let result;
+            try {
+                result = await parseFile(abs, {
+                    relPath,
+                    quiet: true,
+                    sourceOverride: bytes,
+                });
+            }
+            catch (err) {
+                stats.parseErrors += 1;
+                if (!quiet) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    writeStderr(`pugi index --watch: parse(${relPath}) failed: ${message}\n`);
+                }
+                continue;
+            }
+            // Clear prior rows for this file before re-inserting. Cheap
+            // даже if the file is new - the lookup hits the `idx_symbols_file`
+            // index и returns zero rows.
+            try {
+                deleteFile(opts.db, relPath);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                if (!quiet) {
+                    writeStderr(`pugi index --watch: deleteFile(${relPath}) failed: ${message}\n`);
+                }
+                continue;
+            }
+            if (result.symbols.length === 0) {
+                // Record the file fingerprint so a future "did this change?"
+                // query can short-circuit. Only when we got далеко enough to
+                // recognise the language - skipping unsupported extensions
+                // earlier means we never see them here.
+                if (result.language !== null) {
+                    try {
+                        upsertFile(opts.db, {
+                            path: relPath,
+                            sha256: sha256(bytes),
+                            lastIndexedAt: new Date().toISOString(),
+                            symbolCount: 0,
+                        });
+                    }
+                    catch (err) {
+                        const message = err instanceof Error ? err.message : String(err);
+                        if (!quiet) {
+                            writeStderr(`pugi index --watch: upsertFile(${relPath}) failed: ${message}\n`);
+                        }
+                    }
+                }
+                stats.reparsed += 1;
+                continue;
+            }
+            let ids;
+            try {
+                ids = insertSymbols(opts.db, result.symbols);
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                if (!quiet) {
+                    writeStderr(`pugi index --watch: insertSymbols(${relPath}) failed: ${message}\n`);
+                }
+                continue;
+            }
+            const perFile = buildPerFileNameMap(result.symbols, ids);
+            const { resolved, orphaned } = resolvePendingEdges(result.pendingEdges, perFile, (name) => crossFileLookup(opts.db, name));
+            stats.edgesOrphaned += orphaned;
+            if (resolved.length > 0) {
+                try {
+                    insertEdges(opts.db, resolved);
+                    stats.edgesInserted += resolved.length;
+                }
+                catch (err) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    if (!quiet) {
+                        writeStderr(`pugi index --watch: insertEdges(${relPath}) failed: ${message}\n`);
+                    }
+                }
+            }
+            try {
+                upsertFile(opts.db, {
+                    path: relPath,
+                    sha256: sha256(bytes),
+                    lastIndexedAt: new Date().toISOString(),
+                    symbolCount: ids.length,
+                });
+            }
+            catch (err) {
+                const message = err instanceof Error ? err.message : String(err);
+                if (!quiet) {
+                    writeStderr(`pugi index --watch: upsertFile(${relPath}) failed: ${message}\n`);
+                }
+            }
+            stats.reparsed += 1;
+        }
+    };
+    // ---- chokidar wiring --------------------------------------------------
+    // We watch the workspace root recursively and let our own filter
+    // drop unparseable extensions + excluded directories. Chokidar's
+    // built-in `ignored` accepts glob/regex/function; we use a function
+    // form so we can re-use the prune list verbatim instead of
+    // duplicating it as a glob array.
+    const ignored = (p) => {
+        // chokidar may pass absolute or workspace-relative paths depending
+        // on platform и `cwd` option; normalize then reuse the predicate.
+        const abs = resolve(p);
+        const rel = toRelPosix(abs);
+        if (rel.length === 0)
+            return false; // root itself - keep watching
+        return isExcluded(rel);
+    };
+    const watcher = watchFn(root, {
+        ignored,
+        ignoreInitial: true,
+        persistent: true,
+        followSymlinks: false,
+        awaitWriteFinish: {
+            stabilityThreshold: Math.min(50, Math.max(10, Math.floor(debounceMs / 4))),
+            pollInterval: 10,
+        },
+    });
+    const onChange = (kind, abs) => {
+        if (closed)
+            return;
+        const rel = toRelPosix(abs);
+        if (isExcluded(rel) || !isParseable(rel))
+            return;
+        if (kind === 'add')
+            stats.added += 1;
+        else
+            stats.changed += 1;
+        dirty.add(rel);
+        armDebounce();
+    };
+    const onUnlink = (abs) => {
+        if (closed)
+            return;
+        const rel = toRelPosix(abs);
+        if (isExcluded(rel) || !isParseable(rel))
+            return;
+        stats.removed += 1;
+        // If the file is also pending как dirty, drop the dirty marker -
+        // the unlink wins.
+        dirty.delete(rel);
+        removed.add(rel);
+        armDebounce();
+    };
+    watcher.on('add', (p) => onChange('add', p));
+    watcher.on('change', (p) => onChange('change', p));
+    watcher.on('unlink', (p) => onUnlink(p));
+    watcher.on('error', (err) => {
+        stats.watcherErrors += 1;
+        if (!quiet) {
+            const message = err instanceof Error ? err.message : String(err);
+            writeStderr(`pugi index --watch: chokidar error: ${message}\n`);
+        }
+    });
+    const close = async () => {
+        if (closed)
+            return;
+        closed = true;
+        if (timer) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        // Drain any pending events before closing the watcher.
+        if (dirty.size > 0 || removed.size > 0) {
+            try {
+                await flush();
+            }
+            catch (err) {
+                if (!quiet) {
+                    const message = err instanceof Error ? err.message : String(err);
+                    writeStderr(`pugi index --watch: close-time flush failed: ${message}\n`);
+                }
+            }
+        }
+        // Wait for any in-progress flush even if the queue was empty.
+        if (flushing) {
+            try {
+                await flushing;
+            }
+            catch {
+                /* already surfaced via writeStderr above */
+            }
+        }
+        await watcher.close();
+    };
+    const flushNow = async () => {
+        if (timer) {
+            clearTimeout(timer);
+            timer = null;
+        }
+        await flush();
+    };
+    return {
+        close,
+        flushNow,
+        stats: () => ({ ...stats }),
+    };
+}
+/**
+ * Cross-file target resolver backed by the live `symbols` table.
+ * Returns the first matching row id - deterministic stability via the
+ * autoincrement order, mirrors `reindex.ts`'s "first-write-wins"
+ * convention. `null` when no symbol carries the requested name.
+ */
+function crossFileLookup(db, name) {
+    const row = db.conn
+        .prepare('SELECT id FROM symbols WHERE name = ? ORDER BY id ASC LIMIT 1')
+        .get(name);
+    return row?.id ?? null;
+}
+function sha256(s) {
+    return createHash('sha256').update(s).digest('hex');
+}
+//# sourceMappingURL=watcher.js.map

package/dist/core/compact/auto-trigger.js

@@ -0,0 +1,96 @@
+/**
+ * Auto-compact threshold gate.
+ *
+ * Decides whether the conversation buffer has crossed the threshold
+ * percent of the active model's context window. The check runs after
+ * every operator/persona turn before the NEXT operator input lands so
+ * the compaction completes BEFORE the model would have rejected the
+ * request with a context-overflow error.
+ *
+ * Design choices:
+ *
+ *  - Pure function. The caller passes (tokenCount, windowSize, env).
+ *    The gate returns a verdict; the session module owns the side
+ *    effect of invoking the summariser. Pure-function shape keeps the
+ *    spec exhaustive and the call site readable.
+ *
+ *  - Hysteresis: once a compaction lands, the marker resets the
+ *    baseline token count to "summary + tail" — the gate looks at the
+ *    POST-marker tokens only. This is enforced upstream by the caller
+ *    passing the post-marker count; the gate itself has no memory.
+ *
+ *  - Two env knobs:
+ *      PUGI_AUTOCOMPACT_DISABLED=1 — kill switch
+ *      PUGI_AUTOCOMPACT_THRESHOLD=N — float in (0, 1] (default 0.75)
+ *    Anything outside (0, 1] is rejected and the gate falls back to
+ *    the default. Bad input never crashes the REPL.
+ */
+/** Default trip point as a fraction of the context window. */
+export const DEFAULT_THRESHOLD = 0.75;
+/**
+ * Decide whether to fire `/compact` automatically. Pure; safe to call
+ * after every turn.
+ */
+export function evaluateAutoCompact(input) {
+    const env = input.env ?? process.env;
+    const threshold = resolveThreshold(env);
+    if (input.windowSize <= 0 || !Number.isFinite(input.windowSize)) {
+        return {
+            kind: 'skip',
+            reason: 'invalid-window',
+            tokenCount: input.tokenCount,
+            windowSize: input.windowSize,
+            threshold,
+            pressure: 0,
+        };
+    }
+    if (env['PUGI_AUTOCOMPACT_DISABLED'] === '1') {
+        return {
+            kind: 'skip',
+            reason: 'disabled',
+            tokenCount: input.tokenCount,
+            windowSize: input.windowSize,
+            threshold,
+            pressure: roundPressure(input.tokenCount / input.windowSize),
+        };
+    }
+    const pressure = roundPressure(input.tokenCount / input.windowSize);
+    if (pressure >= threshold) {
+        return {
+            kind: 'fire',
+            tokenCount: input.tokenCount,
+            windowSize: input.windowSize,
+            threshold,
+            pressure,
+        };
+    }
+    return {
+        kind: 'skip',
+        reason: 'below-threshold',
+        tokenCount: input.tokenCount,
+        windowSize: input.windowSize,
+        threshold,
+        pressure,
+    };
+}
+/**
+ * Resolve the threshold from env, clamping to the (0, 1] open-closed
+ * interval. Bad input silently falls back to DEFAULT_THRESHOLD so the
+ * REPL never crashes on a malformed environment variable.
+ */
+function resolveThreshold(env) {
+    const raw = env['PUGI_AUTOCOMPACT_THRESHOLD'];
+    if (!raw)
+        return DEFAULT_THRESHOLD;
+    const parsed = Number.parseFloat(raw);
+    if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) {
+        return DEFAULT_THRESHOLD;
+    }
+    return parsed;
+}
+function roundPressure(raw) {
+    if (!Number.isFinite(raw) || raw < 0)
+        return 0;
+    return Math.round(raw * 1000) / 1000;
+}
+//# sourceMappingURL=auto-trigger.js.map

package/dist/core/compact/buffer-rewriter.js

@@ -0,0 +1,115 @@
+/**
+ * Type guard: discriminate a SessionEvent against the `compaction`
+ * kind. Used by replay code so the boundary marker drives a different
+ * code path than `user`/`persona`/`system` transcript rows.
+ */
+export function isCompactBoundary(event) {
+    if (event.kind !== 'compaction')
+        return false;
+    const p = event.payload;
+    if (p === null || typeof p !== 'object')
+        return false;
+    if (p.version !== 1)
+        return false;
+    if (p.trigger !== 'manual' && p.trigger !== 'auto')
+        return false;
+    if (typeof p.summary !== 'string' || p.summary.length === 0)
+        return false;
+    if (typeof p.summaryTokenCount !== 'number')
+        return false;
+    if (typeof p.summaryTurnsBefore !== 'number')
+        return false;
+    if (typeof p.keptTailTurns !== 'number')
+        return false;
+    if (typeof p.coversUntilOffset !== 'number')
+        return false;
+    return true;
+}
+/**
+ * Append one `compaction` boundary marker to the SessionStore. Returns
+ * the SessionEvent we wrote so the caller can echo it into the in-
+ * memory transcript without a re-read. Throws on store error so the
+ * caller surfaces the failure inline.
+ */
+export async function appendCompactBoundary(input) {
+    const ts = (input.now ?? (() => Date.now()))();
+    const payload = {
+        version: 1,
+        trigger: input.trigger,
+        summary: input.summary,
+        summaryTokenCount: input.summaryTokenCount,
+        summaryTurnsBefore: input.summaryTurnsBefore,
+        keptTailTurns: input.keptTailTurns,
+        coversUntilOffset: input.coversUntilOffset,
+    };
+    const event = {
+        t: ts,
+        kind: 'compaction',
+        payload,
+    };
+    await input.store.appendEvent(event);
+    return event;
+}
+/**
+ * Apply replay masking to a chronological event list. Given the full
+ * ordered events.jsonl content, return only the events the caller
+ * should render: every event AFTER the latest `compaction` boundary,
+ * plus the boundary itself (so the renderer can show the banner +
+ * summary), and the K kept-tail events that landed BEFORE the boundary
+ * but were preserved per the marker's `keptTailTurns`.
+ *
+ * Mask logic:
+ *  1. Walk events. Find the LATEST boundary by offset.
+ *  2. Index 0 .. coversUntilOffset-1 are masked, EXCEPT the last
+ *     `keptTailTurns` of that range (which are the verbatim tail).
+ *  3. The boundary event itself + everything after it stays.
+ *
+ * Why we expose this here (and not in session.ts): keeping the mask
+ * logic next to the writer means the wire format is owned by one
+ * module. session.ts depends on this; this depends on nothing in
+ * session.ts. Unidirectional.
+ */
+export function applyCompactMask(events) {
+    // Find latest compaction event.
+    let latestIdx = -1;
+    let latestPayload = null;
+    for (let i = events.length - 1; i >= 0; i -= 1) {
+        const ev = events[i];
+        if (isCompactBoundary(ev)) {
+            latestIdx = i;
+            latestPayload = ev.payload;
+            break;
+        }
+    }
+    if (latestIdx === -1 || latestPayload === null) {
+        return events;
+    }
+    // `coversUntilOffset` is the count of events that existed in the
+    // store immediately before the marker append. Events 0 ..
+    // coversUntilOffset-1 are summarised; events keptTailTurns of them
+    // are surfaced anyway as the verbatim tail.
+    const cap = Math.max(0, Math.min(latestPayload.coversUntilOffset, latestIdx));
+    const tailKeepCount = Math.max(0, Math.min(latestPayload.keptTailTurns, cap));
+    // Take the LAST tailKeepCount events from the masked range, but only
+    // those that represent renderable turns (user/persona/system).
+    // Boundary markers and tool stream events are NOT counted as turns
+    // for the tail-keep window — using them would let the keptTailTurns
+    // budget be consumed by infra events and the operator would lose
+    // the last K real turns. The spec is about "last K human-visible
+    // turns", not "last K events".
+    const tailSlice = [];
+    for (let i = cap - 1; i >= 0 && tailSlice.length < tailKeepCount; i -= 1) {
+        const ev = events[i];
+        if (ev.kind === 'user' || ev.kind === 'persona' || ev.kind === 'system') {
+            tailSlice.push(ev);
+        }
+    }
+    tailSlice.reverse();
+    // After the marker: everything that landed AFTER the boundary
+    // append. These are post-compaction events the user has not yet
+    // seen folded into a summary; they pass through verbatim.
+    const afterMarker = events.slice(latestIdx + 1);
+    const markerEvent = events[latestIdx];
+    return [...tailSlice, markerEvent, ...afterMarker];
+}
+//# sourceMappingURL=buffer-rewriter.js.map