memwarden 0.0.1

package/dist/functions/observe.js

@@ -0,0 +1,326 @@
+//
+// The write path (mem::observe). Accepts a HookPayload, validates it, optionally
+// dedups, privacy-strips the raw payload, builds a RawObservation, then — inside
+// a per-session keyed lock — enforces the per-session cap, persists the raw
+// observation, updates/creates the session row (observationCount++, updatedAt,
+// firstPrompt), and runs the default zero-LLM synthetic compression: write the
+// synthetic over the same obsId and add it to the BM25 and vector indexes.
+// Referenced files are hashed into provenance for Verified Recall. Returns
+// { observationId }.
+//
+// Image detection + modality tagging are kept (pure, keeps the observation
+// shape stable); image-to-disk persistence and vision embedding are out of
+// scope. LLM-based compression (AUTO_COMPRESS) has no provider wired, so the
+// synthetic path is always taken.
+import { TriggerAction } from "../kernel/index.js";
+import { KV, STREAM, generateId } from "../state/schema.js";
+import { stripPrivateData } from "./privacy.js";
+import { withKeyedLock } from "./keyed-mutex.js";
+import { isAutoCompressEnabled, getAgentId } from "./config.js";
+import { buildSyntheticCompression } from "./compress-synthetic.js";
+import { extractProvenance } from "./provenance.js";
+import { hashFiles } from "./verify.js";
+import { recordFix, looksLikeResolvedFix } from "./dejafix.js";
+import { getSearchIndex, vectorIndexAddGuarded } from "./search.js";
+import { logger } from "./logger.js";
+import { metrics } from "../observability/metrics.js";
+export function extractImage(d) {
+    if (!d)
+        return undefined;
+    if (typeof d === "string") {
+        if (d.startsWith("data:image/") ||
+            d.startsWith("iVBORw0KGgo") ||
+            d.startsWith("/9j/")) {
+            return d;
+        }
+        return undefined;
+    }
+    if (typeof d === "object" && d !== null) {
+        const obj = d;
+        if (typeof obj["image_data"] === "string")
+            return obj["image_data"];
+        if (typeof obj["image_path"] === "string")
+            return obj["image_path"];
+        if (typeof obj["imageBase64"] === "string")
+            return obj["imageBase64"];
+        if (typeof obj["imagePath"] === "string")
+            return obj["imagePath"];
+        for (const key of Object.keys(obj)) {
+            const match = extractImage(obj[key]);
+            if (match)
+                return match;
+        }
+    }
+    return undefined;
+}
+export function registerObserveFunction(sdk, kv, dedupMap, maxObservationsPerSession) {
+    sdk.registerFunction("mem::observe", async (payload) => {
+        if (!payload?.sessionId ||
+            typeof payload.sessionId !== "string" ||
+            !payload.hookType ||
+            typeof payload.hookType !== "string" ||
+            !payload.timestamp ||
+            typeof payload.timestamp !== "string") {
+            return {
+                success: false,
+                error: "Invalid payload: sessionId, hookType, and timestamp are required",
+            };
+        }
+        const obsId = generateId("obs");
+        let dedupHash;
+        if (dedupMap) {
+            const d = typeof payload.data === "object" && payload.data !== null
+                ? payload.data
+                : {};
+            const toolName = d["tool_name"] || payload.hookType;
+            dedupHash = dedupMap.computeHash(payload.sessionId, toolName, d["tool_input"]);
+            if (dedupMap.isDuplicate(dedupHash)) {
+                return { deduplicated: true, sessionId: payload.sessionId };
+            }
+        }
+        let sanitizedRaw = payload.data;
+        try {
+            const jsonStr = JSON.stringify(payload.data);
+            const sanitized = stripPrivateData(jsonStr);
+            sanitizedRaw = JSON.parse(sanitized);
+        }
+        catch {
+            sanitizedRaw = stripPrivateData(String(payload.data));
+        }
+        const raw = {
+            id: obsId,
+            sessionId: payload.sessionId,
+            timestamp: payload.timestamp,
+            hookType: payload.hookType,
+            raw: sanitizedRaw,
+        };
+        let extractedImage;
+        if (typeof sanitizedRaw === "object" && sanitizedRaw !== null) {
+            const d = sanitizedRaw;
+            if (payload.hookType === "post_tool_use" ||
+                payload.hookType === "post_tool_failure") {
+                if (typeof d["tool_name"] === "string")
+                    raw.toolName = d["tool_name"];
+                raw.toolInput = d["tool_input"];
+                raw.toolOutput = d["tool_output"] || d["error"];
+            }
+            if (payload.hookType === "prompt_submit") {
+                if (typeof d["prompt"] === "string")
+                    raw.userPrompt = d["prompt"];
+            }
+            extractedImage = extractImage(sanitizedRaw);
+            if (extractedImage) {
+                raw.modality =
+                    raw.toolInput || raw.toolOutput || raw.userPrompt
+                        ? "mixed"
+                        : "image";
+            }
+        }
+        else if (typeof sanitizedRaw === "string") {
+            extractedImage = extractImage(sanitizedRaw);
+            if (extractedImage) {
+                raw.modality = "image";
+            }
+        }
+        return withKeyedLock(`obs:${payload.sessionId}`, async () => {
+            if (maxObservationsPerSession && maxObservationsPerSession > 0) {
+                const existing = await kv.list(KV.observations(payload.sessionId));
+                if (existing.length >= maxObservationsPerSession) {
+                    return {
+                        success: false,
+                        error: `Session observation limit reached (${maxObservationsPerSession})`,
+                    };
+                }
+            }
+            // Existing session is the source of truth for agentId (even
+            // undefined). Env AGENT_ID only fires when no session row exists yet —
+            // otherwise an unscoped session would get retroactively scoped by a
+            // later AGENT_ID export.
+            const existingSession = await kv.get(KV.sessions, payload.sessionId);
+            const inheritedAgentId = existingSession
+                ? existingSession.agentId
+                : getAgentId();
+            if (inheritedAgentId) {
+                raw.agentId = inheritedAgentId;
+            }
+            await kv.set(KV.observations(payload.sessionId), obsId, raw);
+            if (dedupMap && dedupHash) {
+                dedupMap.record(dedupHash);
+            }
+            // Live-viewer stream fan-out. The kernel routes stream::set /
+            // stream::send to its in-process pub/sub. Durably unused in-process,
+            // but kept so the viewer wiring stays identical.
+            await sdk.trigger({
+                function_id: "stream::set",
+                payload: {
+                    stream_name: STREAM.name,
+                    group_id: STREAM.group(payload.sessionId),
+                    item_id: obsId,
+                    data: { type: "raw", observation: raw },
+                },
+            });
+            sdk.trigger({
+                function_id: "stream::send",
+                payload: {
+                    stream_name: STREAM.name,
+                    group_id: STREAM.viewerGroup,
+                    id: `raw-${obsId}`,
+                    type: "raw_observation",
+                    data: {
+                        type: "raw",
+                        observation: raw,
+                        sessionId: payload.sessionId,
+                    },
+                },
+                action: TriggerAction.Void(),
+            });
+            const session = existingSession;
+            if (session) {
+                const updates = [
+                    { type: "set", path: "updatedAt", value: new Date().toISOString() },
+                    {
+                        type: "set",
+                        path: "observationCount",
+                        value: (session.observationCount || 0) + 1,
+                    },
+                ];
+                if (!session.firstPrompt && typeof raw.userPrompt === "string") {
+                    const trimmed = raw.userPrompt.replace(/\s+/g, " ").trim();
+                    if (trimmed.length > 0) {
+                        updates.push({
+                            type: "set",
+                            path: "firstPrompt",
+                            value: trimmed.slice(0, 200),
+                        });
+                    }
+                }
+                await kv.update(KV.sessions, payload.sessionId, updates);
+            }
+            else if (typeof payload.project === "string" &&
+                payload.project.trim().length > 0 &&
+                typeof payload.cwd === "string" &&
+                payload.cwd.trim().length > 0) {
+                // Connectors that skip POST /session/start can fire observations
+                // before the session record exists. Create it now from the
+                // observation payload — but only when project + cwd are present
+                // (HookPayload contract). Older payloads without those fields keep
+                // the original no-op behaviour.
+                const trimmedPrompt = typeof raw.userPrompt === "string"
+                    ? raw.userPrompt.replace(/\s+/g, " ").trim().slice(0, 200)
+                    : undefined;
+                const ts = new Date().toISOString();
+                await kv.set(KV.sessions, payload.sessionId, {
+                    id: payload.sessionId,
+                    project: payload.project,
+                    cwd: payload.cwd,
+                    startedAt: payload.timestamp ?? ts,
+                    updatedAt: ts,
+                    status: "active",
+                    observationCount: 1,
+                    ...(inheritedAgentId ? { agentId: inheritedAgentId } : {}),
+                    ...(trimmedPrompt && trimmedPrompt.length > 0
+                        ? { firstPrompt: trimmedPrompt }
+                        : {}),
+                });
+            }
+            // Per-observation LLM compression is opt-in .
+            // Default path: build a zero-LLM synthetic compression so recall and
+            // BM25 search work without an LLM. The memwarden has no LLM provider
+            // wired in the core, so the synthetic path is always taken.
+            if (isAutoCompressEnabled()) {
+                await sdk.trigger({
+                    function_id: "mem::compress",
+                    payload: {
+                        observationId: obsId,
+                        sessionId: payload.sessionId,
+                        raw,
+                    },
+                    action: TriggerAction.Void(),
+                });
+            }
+            else {
+                const synthetic = buildSyntheticCompression(raw);
+                // Attach the evidence trail so the doctor and Verified Recall can later
+                // judge whether this memory is sourced and still valid. Hash the
+                // referenced files now (under cwd) so content drift is detectable.
+                const prov = extractProvenance(payload);
+                if (prov.files && prov.files.length > 0 && payload.cwd) {
+                    const fileHashes = hashFiles(prov.files, payload.cwd);
+                    if (Object.keys(fileHashes).length > 0)
+                        prov.fileHashes = fileHashes;
+                }
+                synthetic.provenance = prov;
+                metrics.recordObserve(JSON.stringify(raw), JSON.stringify(synthetic));
+                // Déjà Fix opportunistic capture. When this observation already looks
+                // like a recorded fix (contains BOTH a recognizable error AND
+                // resolution language), extract its error signature and store a
+                // FixMemory so any agent that later hits the same error can recall the
+                // verified fix. Reuses the same provenance (with fileHashes) we just
+                // built, so Verified Recall can detect drift. Strictly best-effort and
+                // gated: it must never throw on or block the observe hot path, and
+                // non-fix observations are completely untouched.
+                try {
+                    const fixText = [
+                        synthetic.title,
+                        synthetic.narrative,
+                        ...(synthetic.facts ?? []),
+                    ]
+                        .filter((s) => typeof s === "string" && s.length > 0)
+                        .join("\n");
+                    if (payload.cwd && looksLikeResolvedFix(fixText)) {
+                        const tool = raw.agentId ?? payload.agent;
+                        await recordFix(kv, {
+                            errorText: fixText,
+                            observationId: obsId,
+                            fix: synthetic.narrative || synthetic.title,
+                            provenance: prov,
+                            cwd: payload.cwd,
+                            timestamp: payload.timestamp,
+                            sessionId: payload.sessionId,
+                            ...(tool ? { tool } : {}),
+                        });
+                    }
+                }
+                catch (err) {
+                    // The side path must never break observe.
+                    logger.warn("dejafix opportunistic capture failed", {
+                        obsId,
+                        error: err instanceof Error ? err.message : String(err),
+                    });
+                }
+                await kv.set(KV.observations(payload.sessionId), obsId, synthetic);
+                getSearchIndex().add(synthetic);
+                await vectorIndexAddGuarded(synthetic.id, synthetic.sessionId, synthetic.title + " " + (synthetic.narrative || ""), { kind: "synthetic", logId: synthetic.id });
+                await sdk.trigger({
+                    function_id: "stream::set",
+                    payload: {
+                        stream_name: STREAM.name,
+                        group_id: STREAM.group(payload.sessionId),
+                        item_id: obsId,
+                        data: { type: "compressed", observation: synthetic },
+                    },
+                });
+                await sdk.trigger({
+                    function_id: "stream::set",
+                    payload: {
+                        stream_name: STREAM.name,
+                        group_id: STREAM.viewerGroup,
+                        item_id: obsId,
+                        data: {
+                            type: "compressed",
+                            observation: synthetic,
+                            sessionId: payload.sessionId,
+                        },
+                    },
+                });
+            }
+            logger.info("Observation captured", {
+                obsId,
+                sessionId: payload.sessionId,
+                hook: payload.hookType,
+                compress: isAutoCompressEnabled() ? "llm" : "synthetic",
+            });
+            return { observationId: obsId };
+        });
+    });
+}

package/dist/functions/paths.d.ts

@@ -0,0 +1 @@
+export declare function canonicalizePath(p: string | undefined | null): string;

package/dist/functions/paths.js

@@ -0,0 +1,38 @@
+//
+// Path canonicalization for project/cwd scoping. The unified memory layer
+// scopes recall by working directory, and tools report that directory in
+// inconsistent forms: symlinked (/tmp -> /private/tmp on macOS), with or
+// without a trailing slash, with `..` segments. Exact string comparison then
+// silently misses — the worst failure mode for a memory layer, because it
+// looks like "no memory" rather than an error.
+//
+// canonicalizePath resolves an absolute path to its real, symlink-free form
+// so two spellings of the same directory compare equal. Non-absolute values
+// (labels like "mcp", or "/work/alpha"-style virtual projects in tests) pass
+// through unchanged, and a path that does not exist falls back to syntactic
+// normalization. Applied to BOTH the query filter and the stored value at
+// comparison time, so it is robust regardless of how the path was captured.
+import { realpathSync } from "node:fs";
+import { isAbsolute, normalize } from "node:path";
+// Cache only successfully-resolved (existing) paths: those are stable, while
+// a not-yet-existing path might come to exist later and must re-resolve.
+const cache = new Map();
+export function canonicalizePath(p) {
+    const s = (p ?? "").trim();
+    if (!s || !isAbsolute(s))
+        return s;
+    const hit = cache.get(s);
+    if (hit !== undefined)
+        return hit;
+    try {
+        const real = realpathSync(s);
+        cache.set(s, real);
+        return real;
+    }
+    catch {
+        let n = normalize(s);
+        if (n.length > 1 && n.endsWith("/"))
+            n = n.slice(0, -1);
+        return n;
+    }
+}

package/dist/functions/privacy.d.ts

@@ -0,0 +1 @@
+export declare function stripPrivateData(input: string): string;

package/dist/functions/privacy.js

@@ -0,0 +1,30 @@
+//
+// Redacts secrets from raw observation text before anything is persisted:
+// any span the user wraps in <private>...</private>, plus a set of well-known
+// credential formats (provider API keys, bearer tokens, JWTs, cloud keys).
+// The credential shapes are public, factual patterns.
+const PRIVATE_SPAN = /<private>[\s\S]*?<\/private>/gi;
+const SECRET_PATTERNS = [
+    /(?:api[_-]?key|secret|token|password|credential|auth)\s*[=:]\s*["']?[A-Za-z0-9_\-/.+]{20,}["']?/gi,
+    /Bearer\s+[A-Za-z0-9._\-+/=]{20,}/gi,
+    /sk-proj-[A-Za-z0-9\-_]{20,}/g,
+    /(?:sk|pk|rk|ak)-[A-Za-z0-9][A-Za-z0-9\-_]{19,}/g,
+    /sk-ant-[A-Za-z0-9\-_]{20,}/g,
+    /gh[pus]_[A-Za-z0-9]{36,}/g,
+    /github_pat_[A-Za-z0-9_]{22,}/g,
+    /xoxb-[A-Za-z0-9\-]+/g,
+    /AKIA[0-9A-Z]{16}/g,
+    /AIza[A-Za-z0-9\-_]{35}/g,
+    /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/g,
+    /npm_[A-Za-z0-9]{36}/g,
+    /glpat-[A-Za-z0-9\-_]{20,}/g,
+    /dop_v1_[A-Za-z0-9]{64}/g,
+];
+export function stripPrivateData(input) {
+    let out = input.replace(PRIVATE_SPAN, "[REDACTED]");
+    for (const pattern of SECRET_PATTERNS) {
+        // Fresh RegExp per pass to avoid any shared lastIndex state.
+        out = out.replace(new RegExp(pattern.source, pattern.flags), "[REDACTED_SECRET]");
+    }
+    return out;
+}

package/dist/functions/provenance.d.ts

@@ -0,0 +1,9 @@
+import type { Provenance } from "./types.js";
+export declare function extractProvenance(payload: {
+    cwd?: string;
+    timestamp?: string;
+    agent?: string;
+    data?: unknown;
+}): Provenance;
+/** True when a memory has no evidence backing it. */
+export declare function isUnsourced(p: Provenance | undefined): boolean;

package/dist/functions/provenance.js

@@ -0,0 +1,57 @@
+//
+// Extract provenance from an observe payload — the evidence trail that
+// lets the doctor judge a memory's trustworthiness later. Pure, no I/O.
+const FILE_KEYS = ["file_path", "filePath", "file", "path", "notebook_path"];
+// Looks like a path: has a slash or a dotted extension.
+const PATH_RE = /(^|\/)[\w.\-/]+\.\w{1,8}$|\//;
+function collectFiles(toolInput) {
+    if (!toolInput || typeof toolInput !== "object")
+        return [];
+    const obj = toolInput;
+    const files = new Set();
+    for (const k of FILE_KEYS) {
+        const v = obj[k];
+        if (typeof v === "string" && v.trim())
+            files.add(v.trim());
+    }
+    // Also catch path-shaped string values in any field (e.g. globs, targets).
+    for (const v of Object.values(obj)) {
+        if (typeof v === "string" && v.length < 400 && PATH_RE.test(v) && !v.includes(" ")) {
+            files.add(v.trim());
+        }
+    }
+    return Array.from(files);
+}
+export function extractProvenance(payload) {
+    const data = (payload.data ?? {});
+    const toolName = typeof data["tool_name"] === "string" ? data["tool_name"] : undefined;
+    const toolInput = data["tool_input"];
+    const files = collectFiles(toolInput);
+    let command = toolName;
+    // For shell tools, capture the actual command for a sharper source.
+    if (toolName && toolInput && typeof toolInput === "object") {
+        const cmd = toolInput["command"];
+        if (typeof cmd === "string" && cmd.trim()) {
+            command = `${toolName}: ${cmd.trim().slice(0, 200)}`;
+        }
+    }
+    const prov = { userConfirmed: false };
+    if (payload.cwd)
+        prov.cwd = payload.cwd;
+    if (files.length > 0)
+        prov.files = files;
+    if (command)
+        prov.command = command;
+    if (payload.agent)
+        prov.agent = payload.agent;
+    if (payload.timestamp)
+        prov.capturedAt = payload.timestamp;
+    return prov;
+}
+/** True when a memory has no evidence backing it. */
+export function isUnsourced(p) {
+    if (!p)
+        return true;
+    const hasFiles = Array.isArray(p.files) && p.files.length > 0;
+    return !hasFiles && !p.command && !p.userConfirmed;
+}

package/dist/functions/quantized-vector-index.d.ts

@@ -0,0 +1,60 @@
+import { type QuantBits } from "./turboquant.js";
+export interface QuantParams {
+    version: number;
+    bits: QuantBits;
+    dims: number;
+    paddedDims: number;
+    seed: string;
+    rounds: number;
+    levelHash: string;
+    rescoreDepth: number;
+}
+export declare class QuantizedVectorIndex {
+    readonly params: QuantParams;
+    private vectors;
+    private signFlips;
+    private scratch;
+    private queryScratch;
+    constructor(opts: {
+        dims: number;
+        bits: QuantBits;
+        seed: string;
+        rescoreDepth: number;
+    });
+    add(obsId: string, sessionId: string, embedding: Float32Array): void;
+    remove(obsId: string): void;
+    has(obsId: string): boolean;
+    ids(): string[];
+    /**
+     * Aligns the rescore setting with the current configuration after a
+     * restore: the persisted blob carries the rescoreDepth it was built
+     * with, which may no longer match the environment. Lowering to 0 drops
+     * the retained full vectors (reclaiming memory); raising it keeps
+     * working with whatever full vectors the blob had (entries without one
+     * simply keep their asymmetric score — the rescore pass guards on
+     * presence).
+     */
+    reconcileRescoreDepth(depth: number): void;
+    search(query: Float32Array, limit?: number): Array<{
+        obsId: string;
+        sessionId: string;
+        score: number;
+    }>;
+    get size(): number;
+    validateDimensions(expected: number): {
+        mismatches: Array<{
+            obsId: string;
+            dim: number;
+        }>;
+        seenDimensions: Set<number>;
+    };
+    clear(): void;
+    restoreFrom(other: QuantizedVectorIndex): void;
+    serialize(): string;
+    /**
+     * Returns null when the payload's params don't reproduce the current
+     * algorithm (version, bits, seed, dims, rounds or level-table hash
+     * mismatch) — the caller is expected to fall back to a full rebuild.
+     */
+    static deserialize(json: string): QuantizedVectorIndex | null;
+}