@tpsdev-ai/flair 0.8.3 → 0.10.0

package/dist/rem/restore.js

@@ -0,0 +1,265 @@
+/**
+ * REM live replay — atomic state rewind from a snapshot tarball.
+ *
+ * Slice-2 PR-4. Implements the "every cycle is reversible" property:
+ *   `flair rem restore <date> --apply` actually rewinds Harper state to the
+ *   snapshot, not just extracts the tarball for inspection.
+ *
+ * Approach: client-side. The CLI sequentially calls existing `/Memory` and
+ * `/Soul` endpoints (DELETE current rows for the agent, PUT snapshot rows).
+ * No new server endpoint — keeps the auth surface unchanged and avoids the
+ * Harper body-size limit on uploading large memory exports inline.
+ *
+ * Reversibility-of-restore guarantee: before any destructive op, this
+ * module creates a pre-restore snapshot of the CURRENT state. If something
+ * fails mid-flight, the operator can restore from the pre-restore snapshot
+ * to roll back.
+ *
+ * Multi-agent restore (admin restoring another agent's state) is not in
+ * scope here; the operator's agent can only restore its own memories.
+ * Cross-agent restore is a 1.1+ feature.
+ */
+import { readFileSync, existsSync, rmSync, mkdtempSync } from "node:fs";
+import { resolve } from "node:path";
+import { tmpdir } from "node:os";
+import { extractSnapshot, createSnapshot } from "./snapshot.js";
+function asArray(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    if (raw && typeof raw === "object") {
+        const obj = raw;
+        if (Array.isArray(obj.results))
+            return obj.results;
+        if (Array.isArray(obj.items))
+            return obj.items;
+    }
+    return [];
+}
+function parseJsonlSafe(text) {
+    if (!text.trim())
+        return [];
+    return text
+        .split("\n")
+        .filter((line) => line.trim().length > 0)
+        .map((line) => JSON.parse(line));
+}
+/**
+ * Applies a snapshot tarball to live Harper state for the given agent.
+ *
+ * Steps:
+ *   1. Extract the snapshot to a tmpdir.
+ *   2. Parse memories.jsonl + soul.json + metadata.json.
+ *   3. Verify metadata.agentId matches opts.agentId (prevents accidental
+ *      cross-agent restore — the file might have been hand-copied).
+ *   4. Create a pre-restore snapshot of current state (skip in dry-run).
+ *   5. Fetch + delete current memories/souls for the agent (skip in dry-run).
+ *   6. PUT snapshot memories/souls back into Harper (skip in dry-run).
+ *   7. Return counts.
+ *
+ * On any error after step 4: the result reports `status: "failed"` with
+ * the pre-restore snapshot path included so the operator can roll back.
+ */
+export async function applySnapshot(opts) {
+    const errors = [];
+    const result = {
+        status: "completed",
+        agentId: opts.agentId,
+        snapshotPath: opts.snapshotPath,
+        deleted: { memories: 0, souls: 0 },
+        restored: { memories: 0, souls: 0 },
+        errors,
+    };
+    if (!existsSync(opts.snapshotPath)) {
+        errors.push(`snapshot does not exist: ${opts.snapshotPath}`);
+        result.status = "failed";
+        return result;
+    }
+    // 1+2. Extract and parse. mkdtempSync creates an empty dir; extractSnapshot
+    // refuses to extract into an existing dir, so we point it at a non-existing
+    // subdir of the tmp scratch space.
+    const tmpRoot = opts.tmpRootOverride ?? tmpdir();
+    const tmp = mkdtempSync(resolve(tmpRoot, "flair-rem-restore-"));
+    const extractTo = resolve(tmp, "snapshot");
+    let memories;
+    let souls;
+    let metadata;
+    try {
+        await extractSnapshot({ snapshotPath: opts.snapshotPath, targetDir: extractTo });
+        const memText = readFileSync(resolve(extractTo, "memories.jsonl"), "utf-8");
+        memories = parseJsonlSafe(memText);
+        const soulRaw = JSON.parse(readFileSync(resolve(extractTo, "soul.json"), "utf-8"));
+        souls = Array.isArray(soulRaw)
+            ? soulRaw
+            : soulRaw && typeof soulRaw === "object"
+                ? [soulRaw]
+                : [];
+        metadata = JSON.parse(readFileSync(resolve(extractTo, "metadata.json"), "utf-8"));
+    }
+    catch (err) {
+        errors.push(`extract: ${err?.message ?? String(err)}`);
+        result.status = "failed";
+        rmSync(tmp, { recursive: true, force: true });
+        return result;
+    }
+    // 3. Verify agent id matches. Hard-fail on missing OR mismatched —
+    //    the previous `metadata.agentId && ...` short-circuited on missing,
+    //    which means a snapshot crafted without metadata.agentId would bypass
+    //    the cross-agent guard entirely. v0.9.0+ snapshots always write
+    //    metadata.agentId (see src/rem/snapshot.ts), so the missing case can
+    //    only originate from pre-v0.9.0 snapshots or external/hand-edited
+    //    input — both of which we must reject.
+    if (!metadata.agentId) {
+        errors.push(`snapshot is missing metadata.agentId — refusing to restore (pre-v0.9.0 or untrusted snapshot)`);
+        result.status = "failed";
+        rmSync(tmp, { recursive: true, force: true });
+        return result;
+    }
+    if (metadata.agentId !== opts.agentId) {
+        errors.push(`snapshot agentId (${metadata.agentId}) does not match target (${opts.agentId}) — refusing to restore cross-agent`);
+        result.status = "failed";
+        rmSync(tmp, { recursive: true, force: true });
+        return result;
+    }
+    if (opts.dryRun) {
+        // In dry-run, report planned counts. Still fetch current state for
+        // accurate deleted-counts reporting.
+        try {
+            const currentMem = asArray(await opts.apiCall("GET", `/Memory?agentId=${encodeURIComponent(opts.agentId)}`));
+            const currentSouls = asArray(await opts.apiCall("GET", `/Soul?agentId=${encodeURIComponent(opts.agentId)}`));
+            result.deleted.memories = currentMem.length;
+            result.deleted.souls = currentSouls.length;
+            result.restored.memories = memories.length;
+            result.restored.souls = souls.length;
+            result.status = "dry-run";
+        }
+        catch (err) {
+            errors.push(`fetch-current: ${err?.message ?? String(err)}`);
+            result.status = "failed";
+        }
+        rmSync(tmp, { recursive: true, force: true });
+        return result;
+    }
+    // 4. Pre-restore snapshot of current state.
+    let currentMem;
+    let currentSouls;
+    try {
+        currentMem = asArray(await opts.apiCall("GET", `/Memory?agentId=${encodeURIComponent(opts.agentId)}`));
+        currentSouls = asArray(await opts.apiCall("GET", `/Soul?agentId=${encodeURIComponent(opts.agentId)}`));
+        const preRestore = await createSnapshot({
+            agentId: opts.agentId,
+            flairVersion: opts.flairVersion,
+            memories: currentMem,
+            soul: currentSouls.length === 1 ? currentSouls[0] : currentSouls.length > 1 ? currentSouls : null,
+            pendingCandidateCount: 0, // not load-bearing at restore time
+            rootOverride: opts.preRestoreSnapshotRoot,
+            runId: `rem-restore-pre-${(opts.nowOverride ?? new Date()).toISOString().replace(/[:.]/g, "-")}`,
+            nowOverride: opts.nowOverride,
+        });
+        result.preRestoreSnapshotPath = preRestore.path;
+    }
+    catch (err) {
+        errors.push(`pre-restore-snapshot: ${err?.message ?? String(err)}`);
+        result.status = "failed";
+        rmSync(tmp, { recursive: true, force: true });
+        return result;
+    }
+    // 5. Delete current memories + souls (sequential to keep error semantics).
+    for (const m of currentMem) {
+        if (!m?.id)
+            continue;
+        try {
+            await opts.apiCall("DELETE", `/Memory/${encodeURIComponent(String(m.id))}`);
+            result.deleted.memories++;
+        }
+        catch (err) {
+            errors.push(`delete-memory ${m.id}: ${err?.message ?? String(err)}`);
+        }
+    }
+    for (const s of currentSouls) {
+        if (!s?.id)
+            continue;
+        try {
+            await opts.apiCall("DELETE", `/Soul/${encodeURIComponent(String(s.id))}`);
+            result.deleted.souls++;
+        }
+        catch (err) {
+            errors.push(`delete-soul ${s.id}: ${err?.message ?? String(err)}`);
+        }
+    }
+    // 6. PUT snapshot rows.
+    for (const m of memories) {
+        if (!m?.id)
+            continue;
+        try {
+            await opts.apiCall("PUT", `/Memory/${encodeURIComponent(String(m.id))}`, m);
+            result.restored.memories++;
+        }
+        catch (err) {
+            errors.push(`put-memory ${m.id}: ${err?.message ?? String(err)}`);
+        }
+    }
+    for (const s of souls) {
+        if (!s?.id)
+            continue;
+        try {
+            await opts.apiCall("PUT", `/Soul/${encodeURIComponent(String(s.id))}`, s);
+            result.restored.souls++;
+        }
+        catch (err) {
+            errors.push(`put-soul ${s.id}: ${err?.message ?? String(err)}`);
+        }
+    }
+    // 7. Verify post-restore state (default on; opt-out via verifyPostRestore=false).
+    //    Catches silent failures: Harper schema coercion, 4xx responses the
+    //    apiCall layer masked as ok, partial-DELETE leftovers. Per-ID diff,
+    //    not just counts — count parity can hide simultaneous PUT+DELETE
+    //    failures that wash out numerically. See ops-90dq.
+    if (opts.verifyPostRestore !== false) {
+        try {
+            const verifyMem = asArray(await opts.apiCall("GET", `/Memory?agentId=${encodeURIComponent(opts.agentId)}`));
+            const verifySouls = asArray(await opts.apiCall("GET", `/Soul?agentId=${encodeURIComponent(opts.agentId)}`));
+            const expectedMemoryIds = memories.filter((m) => m?.id).map((m) => String(m.id));
+            const expectedSoulIds = souls.filter((s) => s?.id).map((s) => String(s.id));
+            const actualMemoryIds = verifyMem.filter((m) => m?.id).map((m) => String(m.id));
+            const actualSoulIds = verifySouls.filter((s) => s?.id).map((s) => String(s.id));
+            const expMem = new Set(expectedMemoryIds);
+            const expSoul = new Set(expectedSoulIds);
+            const actMem = new Set(actualMemoryIds);
+            const actSoul = new Set(actualSoulIds);
+            const missingMemoryIds = [...expMem].filter((id) => !actMem.has(id));
+            const missingSoulIds = [...expSoul].filter((id) => !actSoul.has(id));
+            const extraMemoryIds = [...actMem].filter((id) => !expMem.has(id));
+            const extraSoulIds = [...actSoul].filter((id) => !expSoul.has(id));
+            result.verified = {
+                expectedMemoryIds,
+                expectedSoulIds,
+                actualMemoryIds,
+                actualSoulIds,
+                missingMemoryIds,
+                missingSoulIds,
+                extraMemoryIds,
+                extraSoulIds,
+            };
+            if (missingMemoryIds.length > 0) {
+                errors.push(`post-restore-verify: ${missingMemoryIds.length} memory rows missing (e.g. ${missingMemoryIds.slice(0, 3).join(", ")})`);
+            }
+            if (missingSoulIds.length > 0) {
+                errors.push(`post-restore-verify: ${missingSoulIds.length} soul rows missing (e.g. ${missingSoulIds.slice(0, 3).join(", ")})`);
+            }
+            if (extraMemoryIds.length > 0) {
+                errors.push(`post-restore-verify: ${extraMemoryIds.length} unexpected memory rows present (e.g. ${extraMemoryIds.slice(0, 3).join(", ")})`);
+            }
+            if (extraSoulIds.length > 0) {
+                errors.push(`post-restore-verify: ${extraSoulIds.length} unexpected soul rows present (e.g. ${extraSoulIds.slice(0, 3).join(", ")})`);
+            }
+        }
+        catch (err) {
+            errors.push(`post-restore-verify: ${err?.message ?? String(err)}`);
+        }
+    }
+    rmSync(tmp, { recursive: true, force: true });
+    if (errors.length > 0) {
+        result.status = "failed";
+    }
+    return result;
+}

package/dist/rem/runner.js

@@ -0,0 +1,221 @@
+/**
+ * REM nightly runner — orchestrates the cycle.
+ *
+ * Per FLAIR-NIGHTLY-REM § 4, in order:
+ *   1. Pre-flight: check pause sentinel / FLAIR_REM_PAUSE env. Exit clean if paused.
+ *   2. Snapshot agent state (memory + soul) to ~/.flair/snapshots/<agent>/.
+ *   3. Maintenance — delegate to /MemoryMaintenance (same code path `flair rem light` uses).
+ *   4. (slice-2 next) trust-tier filter on input memories.
+ *   5. (slice-2 next) distillation — call /ReflectMemories, persist candidates.
+ *   6. Append a row to ~/.flair/logs/rem-nightly.jsonl.
+ *
+ * Status today:
+ *   - Steps 1, 2, 6 shipped in slice-1 PR-1 (#414).
+ *   - Step 3 (maintenance) ships in this PR — fills `archived`/`expired` in
+ *     the audit row.
+ *   - Steps 4, 5 are slice-2 follow-ups; require an in-process distillation
+ *     LLM path (today `/ReflectMemories` returns a prompt for human/agent
+ *     consumption, not server-side candidate generation).
+ *
+ * The audit row's `slice` field tells readers which steps populated which
+ * counts: `slice: "1"` rows have `archived`/`expired` undefined; `slice:
+ * "2-maintenance"` rows populate them; future slice-2 rows will populate
+ * `consolidated` and `candidates`.
+ *
+ * Pure dependency injection so the runner is unit-testable without Harper.
+ * The CLI wires the real `apiCall` + `pkgVersion`; tests pass stubs.
+ */
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { dirname, resolve } from "node:path";
+import { homedir } from "node:os";
+import { createSnapshot } from "./snapshot.js";
+export const REM_PAUSE_FLAG = resolve(homedir(), ".flair", "rem.paused");
+export const REM_NIGHTLY_LOG = resolve(homedir(), ".flair", "logs", "rem-nightly.jsonl");
+function readPauseSentinel(path) {
+    try {
+        const contents = readFileSync(path, "utf-8");
+        // Existence alone is enough; the body is just a touch-timestamp.
+        return contents.length >= 0;
+    }
+    catch {
+        return false;
+    }
+}
+function appendLogRow(logPath, row) {
+    const dir = dirname(logPath);
+    if (!existsSync(dir))
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+    appendFileSync(logPath, JSON.stringify(row) + "\n", { mode: 0o600 });
+}
+/**
+ * Coerces a Harper REST list response into a flat array. The /Memory and
+ * /Soul resources can return either an array directly or a paginated shape
+ * `{ results: [], items: [] }` depending on path/options.
+ */
+function asArray(raw) {
+    if (Array.isArray(raw))
+        return raw;
+    if (raw && typeof raw === "object") {
+        const obj = raw;
+        if (Array.isArray(obj.results))
+            return obj.results;
+        if (Array.isArray(obj.items))
+            return obj.items;
+    }
+    return [];
+}
+/**
+ * Counts pending memory candidates for the agent.
+ *
+ * Falls back to a `search_by_conditions` POST that mirrors the pattern
+ * `flair rem candidates` uses. Returns 0 on any error — the runner should
+ * not fail the cycle just because the candidate count couldn't be sampled.
+ */
+async function fetchPendingCandidateCount(api, agentId) {
+    try {
+        const result = await api("POST", "/MemoryCandidate/search_by_conditions", {
+            operator: "and",
+            conditions: [
+                { search_attribute: "agentId", search_type: "equals", search_value: agentId },
+                { search_attribute: "status", search_type: "equals", search_value: "pending" },
+            ],
+            get_attributes: ["id"],
+        });
+        const rows = asArray(result);
+        return rows.length;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Runs one nightly cycle for the given agent. See module header for steps.
+ * Pure orchestration; all I/O goes through injected dependencies.
+ */
+export async function runNightlyCycle(opts) {
+    const startedAt = opts.nowOverride ?? new Date();
+    const startedMs = startedAt.getTime();
+    const logPath = opts.logPath ?? REM_NIGHTLY_LOG;
+    const pauseFlagPath = opts.pauseFlagPath ?? REM_PAUSE_FLAG;
+    const envPaused = opts.envPaused ?? process.env.FLAIR_REM_PAUSE === "1";
+    const baseRow = {
+        agentId: opts.agentId,
+        runAt: startedAt.toISOString(),
+        slice: "1",
+        errors: [],
+    };
+    // Step 1: pre-flight (pause)
+    if (envPaused || (existsSync(pauseFlagPath) && readPauseSentinel(pauseFlagPath))) {
+        const row = {
+            ...baseRow,
+            status: "paused",
+            durationMs: Date.now() - startedMs,
+            errors: [],
+        };
+        appendLogRow(logPath, row);
+        return { status: "paused", logRow: row };
+    }
+    const errors = [];
+    // Step 2: snapshot
+    let snapshotPath;
+    let memoryCount = 0;
+    let soulCount = 0;
+    let pendingCandidates = 0;
+    try {
+        // Fetch agent data
+        const memoriesRaw = await opts.apiCall("GET", `/Memory?agentId=${encodeURIComponent(opts.agentId)}`);
+        const memories = asArray(memoriesRaw);
+        memoryCount = memories.length;
+        const soulRaw = await opts.apiCall("GET", `/Soul?agentId=${encodeURIComponent(opts.agentId)}`);
+        const souls = asArray(soulRaw);
+        soulCount = souls.length;
+        // For the snapshot's soul.json: keep the full multi-row shape if there
+        // are multiple souls (different keys), or unwrap a single-row response.
+        const soulForSnapshot = souls.length === 1 ? souls[0] : souls.length > 1 ? souls : null;
+        pendingCandidates = await fetchPendingCandidateCount(opts.apiCall, opts.agentId);
+        if (!opts.dryRun) {
+            const created = await createSnapshot({
+                agentId: opts.agentId,
+                flairVersion: opts.flairVersion,
+                memories,
+                soul: soulForSnapshot,
+                pendingCandidateCount: pendingCandidates,
+                rootOverride: opts.snapshotRoot,
+                nowOverride: opts.nowOverride,
+            });
+            snapshotPath = created.path;
+        }
+    }
+    catch (err) {
+        errors.push(`snapshot: ${err?.message ?? String(err)}`);
+        const row = {
+            ...baseRow,
+            status: "failed",
+            memoryCount,
+            soulCount,
+            pendingCandidates,
+            durationMs: Date.now() - startedMs,
+            errors,
+        };
+        appendLogRow(logPath, row);
+        return { status: "failed", logRow: row };
+    }
+    // Step 3: maintenance — soft-delete expired + soft-archive stale.
+    // Delegates to /MemoryMaintenance (same endpoint `flair rem light` uses).
+    // In dry-run mode the snapshot wasn't written, but we still want to know
+    // what maintenance WOULD do — POST with dryRun=true so the response counts
+    // are accurate without mutating state.
+    let archived = 0;
+    let expired = 0;
+    let sliceLabel = "2-maintenance";
+    try {
+        const maintRaw = await opts.apiCall("POST", "/MemoryMaintenance", {
+            agentId: opts.agentId,
+            dryRun: opts.dryRun ?? false,
+        });
+        if (maintRaw && typeof maintRaw === "object") {
+            const obj = maintRaw;
+            if (obj.error) {
+                // /MemoryMaintenance returned { error: "..." } — treat as failure.
+                throw new Error(String(obj.error));
+            }
+            expired = typeof obj.expired === "number" ? obj.expired : 0;
+            archived = typeof obj.archived === "number" ? obj.archived : 0;
+        }
+    }
+    catch (err) {
+        errors.push(`maintenance: ${err?.message ?? String(err)}`);
+        const row = {
+            ...baseRow,
+            slice: sliceLabel,
+            status: "failed",
+            snapshotPath,
+            memoryCount,
+            soulCount,
+            pendingCandidates,
+            durationMs: Date.now() - startedMs,
+            errors,
+        };
+        appendLogRow(logPath, row);
+        return { status: "failed", logRow: row, snapshotPath };
+    }
+    // Step 6: log
+    const row = {
+        ...baseRow,
+        slice: sliceLabel,
+        status: opts.dryRun ? "dry-run" : "completed",
+        dryRun: opts.dryRun || undefined,
+        snapshotPath,
+        memoryCount,
+        soulCount,
+        pendingCandidates,
+        archived,
+        expired,
+        durationMs: Date.now() - startedMs,
+        errors,
+        // `consolidated` and `candidates` populate when slice-2 PR-4 (distillation)
+        // lands; today they remain undefined so the row is honest about what ran.
+    };
+    appendLogRow(logPath, row);
+    return { status: row.status, logRow: row, snapshotPath };
+}