@pugi/cli 0.1.0-beta.8 → 0.1.0-beta.88

package/dist/core/memory/dual-write.js

@@ -0,0 +1,416 @@
+/**
+ * Pugi CLI dual-write client — Memory Phase 1 (2026-05-27, ).
+ *
+ * Promotes the per-session NDJSON event log (Memory Phase 0, PR)
+ * to a best-effort Prisma row store via the admin-api endpoint:
+ *
+ *  POST /api/pugi/sessions/:sessionId/events
+ *
+ * Design contract:
+ *
+ *  1. Local NDJSON is the SOURCE OF TRUTH. The dual-write client is
+ *     fire-and-forget — a network failure NEVER blocks the local
+ *     append path, and the operator never loses data because the
+ *     .pugi/sessions/<id>/events.<n>.jsonl file is always written
+ *     first by the SessionStore.
+ *
+ *  2. Async + debounced. Events are buffered in memory and flushed on
+ *     a timer (default 250 ms) or when the buffer crosses the batch
+ *     cap (default 50). The debounce + buffer keeps the network cost
+ *     proportional to operator activity, not per-event.
+ *
+ *  3. Retry with backoff. A failed flush retries up to 3 times with
+ *     exponential backoff (250 ms / 750 ms / 2250 ms). Failure beyond
+ *     that drops the batch; the next flush brings in the backlog from
+ *     the local NDJSON via the resume path.
+ *
+ *  4. Resume marker. The client tracks `lastSyncedSeq` per session in
+ *     `~/.pugi/memory-sync/<sessionId>.json`. On `flushBacklog()` it
+ *     reads any events from the local NDJSON whose seq > lastSyncedSeq
+ *     and posts them in order. The marker is updated only on a
+ *     successful POST so a crash mid-flush retries the unsent batch.
+ *
+ *  5. Kill switch. `PUGI_MEMORY_PHASE1_PRISMA_PERSIST_ENABLED=false`
+ *     shuts the dual-write off entirely (env var, no settings file
+ *     round-trip). The client is also a no-op when no credentials are
+ *     available — anon CLI sessions never POST.
+ *
+ *  6. Schema contract. The kinds mirror
+ *     `apps/admin-api/src/pugi-session-events/pugi-session-events.types.ts`
+ *     PUGI_SESSION_EVENT_KINDS. The CLI's broader NDJSON kind set
+ *     (`user`, `persona`, `rewind-marker`, etc.) is REMAPPED to the
+ *     Phase-1 closed set inside `eventToPhase1()` — adding a new
+ *     Phase-1 kind requires a coordinated CLI + server bump.
+ *
+ * The client carries no Nest / Prisma deps — pure node:fs + global
+ * fetch so it boots inside the CLI's REPL workflow without dragging in
+ * the admin-api graph.
+ */
+import { existsSync, mkdirSync, readFileSync, renameSync, writeFileSync, } from 'node:fs';
+import { homedir } from 'node:os';
+import { resolve } from 'node:path';
+import { PUGI_SESSION_EVENT_KINDS, } from './phase1-kinds.js';
+/* ------------------------------------------------------------------ */
+/* Constants                                                          */
+/* ------------------------------------------------------------------ */
+export const DEFAULT_DEBOUNCE_MS = 250;
+export const DEFAULT_MAX_BATCH_SIZE = 50;
+export const DEFAULT_MAX_RETRIES = 3;
+/** Server-side hard ceiling (mirror of admin-api MAX_BATCH_EVENTS). */
+const SERVER_MAX_BATCH_EVENTS = 200;
+/** Env kill-switch. Unset / 'true' / '1' = enabled. 'false' / '0' = off. */
+export function isDualWriteEnabledFromEnv(env = process.env) {
+    const raw = env.PUGI_MEMORY_PHASE1_PRISMA_PERSIST_ENABLED;
+    if (raw === undefined || raw === '')
+        return true;
+    const v = raw.toLowerCase().trim();
+    return v !== 'false' && v !== '0' && v !== 'off' && v !== 'no';
+}
+/** Env override for debounce (ms). Falls back to DEFAULT_DEBOUNCE_MS. */
+export function debounceMsFromEnv(env = process.env) {
+    const raw = env.PUGI_MEMORY_DUAL_WRITE_DEBOUNCE_MS;
+    if (!raw)
+        return DEFAULT_DEBOUNCE_MS;
+    const n = Number.parseInt(raw, 10);
+    if (!Number.isFinite(n) || n < 0 || n > 60_000)
+        return DEFAULT_DEBOUNCE_MS;
+    return n;
+}
+export function defaultStateDir(home = homedir()) {
+    return resolve(home, '.pugi', 'memory-sync');
+}
+export function syncStatePath(sessionId, stateDir) {
+    return resolve(stateDir, `${sessionId}.json`);
+}
+export function readSyncState(sessionId, stateDir = defaultStateDir()) {
+    const p = syncStatePath(sessionId, stateDir);
+    if (!existsSync(p))
+        return 0;
+    try {
+        const raw = readFileSync(p, 'utf-8');
+        const parsed = JSON.parse(raw);
+        if (parsed.schema === 1 &&
+            typeof parsed.lastSyncedSeq === 'number' &&
+            Number.isInteger(parsed.lastSyncedSeq) &&
+            parsed.lastSyncedSeq >= 0) {
+            return parsed.lastSyncedSeq;
+        }
+    }
+    catch {
+        // Best effort — a corrupt state file means "resync from zero".
+        return 0;
+    }
+    return 0;
+}
+export function writeSyncState(sessionId, lastSyncedSeq, stateDir = defaultStateDir()) {
+    if (!existsSync(stateDir)) {
+        mkdirSync(stateDir, { recursive: true, mode: 0o700 });
+    }
+    const p = syncStatePath(sessionId, stateDir);
+    const record = {
+        schema: 1,
+        sessionId,
+        lastSyncedSeq,
+        updatedAt: new Date().toISOString(),
+    };
+    // Atomic write: temp + rename so a crash mid-write leaves either the
+    // old contents or the new — never a partial JSON document.
+    const tmp = `${p}.tmp-${process.pid}-${Date.now()}`;
+    writeFileSync(tmp, JSON.stringify(record), { encoding: 'utf-8', mode: 0o600 });
+    renameSync(tmp, p);
+}
+/* ------------------------------------------------------------------ */
+/* DualWriteClient                                                   */
+/* ------------------------------------------------------------------ */
+/**
+ * Compute the next backoff delay (ms). Exponential growth from the
+ * debounce base — 1× / 3× / 9× — keeps the retry storm bounded.
+ */
+export function nextBackoffMs(attempt, base) {
+    // attempt 1 -> base, attempt 2 -> 3×base, attempt 3 -> 9×base.
+    return base * Math.pow(3, Math.max(0, attempt - 1));
+}
+/**
+ * Tiny await-able sleeper. Public for tests that want a deterministic
+ * scheduler — production callers go through the internal debounce
+ * timer, not this helper.
+ */
+function sleep(ms, scheduler = setTimeout) {
+    return new Promise((res) => scheduler(res, Math.max(0, ms)));
+}
+/**
+ * Buffered, debounced, retry-capable dual-write client.
+ *
+ * One instance per active session. The CLI's SessionStore creates one
+ * at `openSession()` and disposes it at `archive()` / `process.exit()`.
+ * The instance retains its own buffer + flushing promise so concurrent
+ * `enqueue()` calls from multiple producers (REPL turn + subagent
+ * dispatcher) coalesce into the same batch.
+ */
+export class DualWriteClient {
+    cfg;
+    buffer = [];
+    flushTimer = null;
+    /**
+     * Active flush promise (null when idle). Re-using the in-flight
+     * promise lets `enqueue` callers tail-chain a pending flush instead
+     * of stacking N flushes when the CLI is in a hot loop.
+     */
+    flushing = null;
+    /** Disposed clients reject further enqueues quietly. */
+    disposed = false;
+    /** Track the highest server-acknowledged seq. */
+    highestSyncedSeq;
+    constructor(config) {
+        const stateDir = config.stateDir ?? defaultStateDir();
+        this.cfg = {
+            apiUrl: config.apiUrl.replace(/\/+$/, ''),
+            apiKey: config.apiKey,
+            sessionId: config.sessionId,
+            debounceMs: config.debounceMs ?? debounceMsFromEnv(),
+            maxBatchSize: Math.min(config.maxBatchSize ?? DEFAULT_MAX_BATCH_SIZE, SERVER_MAX_BATCH_EVENTS),
+            maxRetries: config.maxRetries ?? DEFAULT_MAX_RETRIES,
+            fetchImpl: config.fetchImpl ?? globalThis.fetch,
+            stateDir,
+        };
+        this.highestSyncedSeq = readSyncState(config.sessionId, stateDir);
+    }
+    /** Currently-known synced seq (read-through for callers). */
+    get lastSyncedSeq() {
+        return this.highestSyncedSeq;
+    }
+    /**
+     * Buffer an event for the next flush. Returns immediately — the
+     * actual POST happens on the debounce timer.
+     *
+     * The producer (SessionStore) is expected to have ALREADY written
+     * the local NDJSON line before calling this. The dual-write client
+     * never touches the NDJSON itself.
+     */
+    enqueue(event) {
+        if (this.disposed)
+            return;
+        if (!isValidPhase1Kind(event.kind)) {
+            // Quietly ignore events the Phase-1 schema cannot represent. The
+            // local NDJSON keeps them; a future Phase-1.1 may widen the kind
+            // set.
+            return;
+        }
+        if (!Number.isInteger(event.seq) || event.seq <= 0)
+            return;
+        this.buffer.push(event);
+        // Crossed the batch cap — flush eagerly to bound buffer growth.
+        if (this.buffer.length >= this.cfg.maxBatchSize) {
+            this.scheduleFlush(0);
+            return;
+        }
+        this.scheduleFlush(this.cfg.debounceMs);
+    }
+    /**
+     * Force any pending events out. Used at session archive / shutdown
+     * so the trailing batch lands before the process exits.
+     */
+    async flush() {
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = null;
+        }
+        return this.doFlush();
+    }
+    /**
+     * Mark the client disposed. Subsequent enqueues are no-ops. Pending
+     * flush is awaited so the caller can `await client.dispose()` to
+     * fence completion.
+     */
+    async dispose() {
+        this.disposed = true;
+        if (this.flushTimer) {
+            clearTimeout(this.flushTimer);
+            this.flushTimer = null;
+        }
+        if (this.flushing) {
+            try {
+                await this.flushing;
+            }
+            catch {
+                // ignore
+            }
+        }
+    }
+    /**
+     * Backlog catch-up. The caller passes the local NDJSON's events
+     * (already filtered to the session) — the client posts any whose
+     * seq > lastSyncedSeq, in order, in batches of maxBatchSize.
+     *
+     * Returns the aggregate FlushResult across all batches (sums
+     * persisted + duplicate counts, lastSeq is the max).
+     */
+    async flushBacklog(allEvents) {
+        const pending = allEvents
+            .filter((e) => e.seq > this.highestSyncedSeq && isValidPhase1Kind(e.kind))
+            .sort((a, b) => a.seq - b.seq);
+        if (pending.length === 0) {
+            return {
+                persistedCount: 0,
+                duplicateCount: 0,
+                lastSeq: this.highestSyncedSeq,
+            };
+        }
+        let totalPersisted = 0;
+        let totalDuplicates = 0;
+        let maxLastSeq = this.highestSyncedSeq;
+        for (let i = 0; i < pending.length; i += this.cfg.maxBatchSize) {
+            const batch = pending.slice(i, i + this.cfg.maxBatchSize);
+            const result = await this.postBatchWithRetry(batch);
+            if (!result)
+                break; // surrender — backlog stays for next session
+            totalPersisted += result.persistedCount;
+            totalDuplicates += result.duplicateCount;
+            maxLastSeq = Math.max(maxLastSeq, result.lastSeq);
+        }
+        return {
+            persistedCount: totalPersisted,
+            duplicateCount: totalDuplicates,
+            lastSeq: maxLastSeq,
+        };
+    }
+    /* ---------------- internal ---------------- */
+    scheduleFlush(delay) {
+        if (this.flushTimer)
+            clearTimeout(this.flushTimer);
+        this.flushTimer = setTimeout(() => {
+            this.flushTimer = null;
+            // doFlush handles its own error swallowing; we drop the floating
+            // promise on the floor on purpose so the timer callback returns
+            // synchronously (Node requires that).
+            void this.doFlush();
+        }, delay);
+    }
+    async doFlush() {
+        if (this.flushing)
+            return this.flushing;
+        if (this.buffer.length === 0)
+            return null;
+        const drained = this.buffer.splice(0, this.buffer.length);
+        this.flushing = this.postBatchWithRetry(drained);
+        try {
+            return await this.flushing;
+        }
+        finally {
+            this.flushing = null;
+        }
+    }
+    async postBatchWithRetry(batch) {
+        if (batch.length === 0)
+            return null;
+        // Sort + within-batch dedup (defence in depth — the server also
+        // enforces monotonic seq + dedup, but enforcing here keeps the
+        // wire payload clean and avoids the 400 on a buggy producer).
+        const sorted = [...batch].sort((a, b) => a.seq - b.seq);
+        const deduped = [];
+        let prevSeq = -1;
+        for (const ev of sorted) {
+            if (ev.seq === prevSeq)
+                continue;
+            deduped.push(ev);
+            prevSeq = ev.seq;
+        }
+        for (let attempt = 1; attempt <= this.cfg.maxRetries; attempt++) {
+            try {
+                const result = await this.doPost(deduped);
+                // Persist the lastSyncedSeq marker so a crash before the next
+                // flush still rediscovers the high-water-mark on restart.
+                if (result.lastSeq > this.highestSyncedSeq) {
+                    this.highestSyncedSeq = result.lastSeq;
+                    try {
+                        writeSyncState(this.cfg.sessionId, this.highestSyncedSeq, this.cfg.stateDir);
+                    }
+                    catch {
+                        // Best effort — a marker write failure does not invalidate
+                        // the on-server data, only the resume hint.
+                    }
+                }
+                return result;
+            }
+            catch (err) {
+                if (attempt === this.cfg.maxRetries) {
+                    // Final surrender — drop the batch silently. Local NDJSON is
+                    // authoritative; the next session resume can replay the gap.
+                    return null;
+                }
+                await sleep(nextBackoffMs(attempt, this.cfg.debounceMs));
+            }
+        }
+        return null;
+    }
+    async doPost(events) {
+        const url = `${this.cfg.apiUrl}/api/pugi/sessions/${encodeURIComponent(this.cfg.sessionId)}/events`;
+        const res = await this.cfg.fetchImpl(url, {
+            method: 'POST',
+            headers: {
+                authorization: `Bearer ${this.cfg.apiKey}`,
+                'content-type': 'application/json',
+                accept: 'application/json',
+            },
+            body: JSON.stringify({ events }),
+        });
+        if (!res.ok) {
+            throw new Error(`dual-write POST ${url} returned ${res.status} ${res.statusText}`);
+        }
+        const body = (await res.json());
+        return {
+            persistedCount: body.persistedCount ?? 0,
+            duplicateCount: body.duplicateCount ?? 0,
+            lastSeq: body.lastSeq ?? 0,
+        };
+    }
+}
+/** Validate a kind string against the Phase-1 closed set. */
+export function isValidPhase1Kind(value) {
+    return PUGI_SESSION_EVENT_KINDS.includes(value);
+}
+/* ------------------------------------------------------------------ */
+/* Kind remap (CLI broad set -> Phase-1 closed set)                  */
+/* ------------------------------------------------------------------ */
+/**
+ * Map the CLI's broader NDJSON kind set to the Phase-1 server-side set.
+ * Returns `null` when the kind has no Phase-1 representation — the
+ * caller should drop that event from the dual-write (the local NDJSON
+ * still carries it for offline analysis).
+ *
+ * The CLI's kind set lives in
+ * `apps/pugi-cli/src/core/repl/store/types.ts`:
+ *
+ *  'user'          -> 'turn.user'
+ *  'persona'       -> 'turn.assistant'
+ *  'system'        -> 'system'
+ *  'tool.start'    -> 'tool.call'
+ *  'tool.result'   -> 'tool.result'
+ *  'agent.spawned' -> 'dispatch.start'
+ *  'agent.completed'-> 'dispatch.end'
+ *  'compaction'    -> 'compact.boundary'
+ *  'rewind-marker' -> null (Phase-1 has no analog yet)
+ */
+export function cliKindToPhase1(cliKind) {
+    switch (cliKind) {
+        case 'user':
+            return 'turn.user';
+        case 'persona':
+            return 'turn.assistant';
+        case 'system':
+            return 'system';
+        case 'tool.start':
+            return 'tool.call';
+        case 'tool.result':
+            return 'tool.result';
+        case 'agent.spawned':
+            return 'dispatch.start';
+        case 'agent.completed':
+            return 'dispatch.end';
+        case 'compaction':
+            return 'compact.boundary';
+        default:
+            return null;
+    }
+}
+//# sourceMappingURL=dual-write.js.map

package/dist/core/memory/passive-extract.js

@@ -0,0 +1,130 @@
+import { estimateTokens } from '../compact/token-counter.js';
+const VALID_KINDS = new Set([
+    'preference',
+    'decision',
+    'context',
+    'fact',
+]);
+const DEFAULT_MAX_ITEMS = 8;
+const DEFAULT_MIN_CONFIDENCE = 0.6;
+const DEFAULT_MAX_INPUT_TOKENS = 6000;
+function genId() {
+    const rand = Math.random().toString(36).slice(2, 10);
+    return `mem-${rand}`;
+}
+function truncateTurns(turns, maxInputTokens) {
+    if (turns.length === 0)
+        return [];
+    const kept = [];
+    let total = 0;
+    for (let i = turns.length - 1; i >= 0; i -= 1) {
+        const turn = turns[i];
+        if (!turn)
+            continue;
+        const cost = estimateTokens(`${turn.role}: ${turn.content}`);
+        if (total + cost > maxInputTokens && kept.length > 0) {
+            break;
+        }
+        kept.unshift(turn);
+        total += cost;
+    }
+    return kept;
+}
+function buildPrompt(turns) {
+    const dialog = turns
+        .map((t) => `[${t.id}] ${t.role}: ${t.content}`)
+        .join('\n');
+    return [
+        'You extract persistable memory items from a dialog between a user and an assistant.',
+        'Return strict JSON with shape:',
+        '{ "items": [ { "kind": "preference"|"decision"|"context"|"fact", "body": "1-2 sentences", "confidence": 0..1, "sourceTurn": "<turn id>" } ] }',
+        'Rules:',
+        '- Only include facts worth remembering across future sessions.',
+        '- Skip greetings, small talk, transient state, and tool noise.',
+        '- Confidence reflects how clearly the dialog supports the fact.',
+        '- Never invent facts that are not present in the dialog.',
+        '- If nothing is worth keeping, return { "items": [] }.',
+        'Dialog:',
+        dialog,
+        'Respond with JSON only.',
+    ].join('\n');
+}
+function extractJsonObject(raw) {
+    const trimmed = raw.trim();
+    const start = trimmed.indexOf('{');
+    const end = trimmed.lastIndexOf('}');
+    if (start === -1 || end === -1 || end <= start) {
+        throw new Error('no-json');
+    }
+    return JSON.parse(trimmed.slice(start, end + 1));
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function validateItem(value) {
+    if (!isRecord(value))
+        return null;
+    const { kind, body, confidence, sourceTurn } = value;
+    if (typeof kind !== 'string' || !VALID_KINDS.has(kind)) {
+        return null;
+    }
+    if (typeof body !== 'string' || body.trim().length === 0) {
+        return null;
+    }
+    if (typeof confidence !== 'number' || Number.isNaN(confidence)) {
+        return null;
+    }
+    if (confidence < 0 || confidence > 1) {
+        return null;
+    }
+    const out = {
+        kind: kind,
+        body: body.trim(),
+        confidence,
+    };
+    if (typeof sourceTurn === 'string' && sourceTurn.length > 0) {
+        out.sourceTurn = sourceTurn;
+    }
+    return out;
+}
+export async function extractPassiveMemory(options) {
+    const { transport, dialogTurns, maxItems = DEFAULT_MAX_ITEMS, minConfidence = DEFAULT_MIN_CONFIDENCE, maxInputTokens = DEFAULT_MAX_INPUT_TOKENS, } = options;
+    if (dialogTurns.length === 0) {
+        return { items: [], usedFallback: false };
+    }
+    const truncated = truncateTurns(dialogTurns, maxInputTokens);
+    const prompt = buildPrompt(truncated);
+    let raw;
+    try {
+        raw = await transport(prompt);
+    }
+    catch {
+        return { items: [], usedFallback: true, reason: 'transport-error' };
+    }
+    let parsed;
+    try {
+        parsed = extractJsonObject(raw);
+    }
+    catch {
+        return { items: [], usedFallback: true, reason: 'parse-failed' };
+    }
+    if (!isRecord(parsed) || !Array.isArray(parsed.items)) {
+        return { items: [], usedFallback: true, reason: 'parse-failed' };
+    }
+    const validated = [];
+    for (const candidate of parsed.items) {
+        const item = validateItem(candidate);
+        if (!item)
+            continue;
+        if (item.confidence < minConfidence)
+            continue;
+        validated.push({ id: genId(), ...item });
+    }
+    if (validated.length === 0) {
+        return { items: [], usedFallback: true, reason: 'no-signal' };
+    }
+    validated.sort((a, b) => b.confidence - a.confidence);
+    const capped = validated.slice(0, maxItems);
+    return { items: capped, usedFallback: false };
+}
+//# sourceMappingURL=passive-extract.js.map

package/dist/core/memory/phase1-kinds.js

@@ -0,0 +1,20 @@
+/**
+ * Phase-1 event kind closed set — mirror of
+ * `apps/admin-api/src/pugi-session-events/pugi-session-events.types.ts`
+ * PUGI_SESSION_EVENT_KINDS.
+ *
+ * Kept as a separate file so the CLI does not import from admin-api
+ * (would drag the whole Nest graph). The TWO lists must stay in
+ * lockstep — a new kind requires editing both files.
+ */
+export const PUGI_SESSION_EVENT_KINDS = [
+    'turn.user',
+    'turn.assistant',
+    'tool.call',
+    'tool.result',
+    'dispatch.start',
+    'dispatch.end',
+    'compact.boundary',
+    'system',
+];
+//# sourceMappingURL=phase1-kinds.js.map