memwarden 0.0.1

package/dist/functions/vector-index.js

@@ -0,0 +1,125 @@
+//
+// Flat brute-force cosine vector index: the full-precision baseline behind the
+// VectorIndexLike contract (QuantizedVectorIndex is the compressed default).
+// Pure and engine-independent.
+//
+// The base64 helpers pass byteOffset + byteLength explicitly on purpose:
+// Buffer.from(b64, "base64") hands back a slice of Node's shared pool, and a
+// naive `new Float32Array(buf.buffer)` would mint a view over the whole pool
+// (phantom dimensions). The same care applies on encode if the source array is
+// itself a view. Keep these exact.
+function float32ToBase64(arr) {
+    return Buffer.from(arr.buffer, arr.byteOffset, arr.byteLength).toString("base64");
+}
+function base64ToFloat32(b64) {
+    const buf = Buffer.from(b64, "base64");
+    return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / Float32Array.BYTES_PER_ELEMENT);
+}
+function cosine(a, b) {
+    if (a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let na = 0;
+    let nb = 0;
+    for (let i = 0; i < a.length; i++) {
+        const x = a[i];
+        const y = b[i];
+        dot += x * y;
+        na += x * x;
+        nb += y * y;
+    }
+    const denom = Math.sqrt(na) * Math.sqrt(nb);
+    return denom === 0 ? 0 : dot / denom;
+}
+export class VectorIndex {
+    vectors = new Map();
+    add(obsId, sessionId, embedding) {
+        this.vectors.set(obsId, { embedding, sessionId });
+    }
+    remove(obsId) {
+        this.vectors.delete(obsId);
+    }
+    has(obsId) {
+        return this.vectors.has(obsId);
+    }
+    ids() {
+        return [...this.vectors.keys()];
+    }
+    get size() {
+        return this.vectors.size;
+    }
+    search(query, limit = 20) {
+        const scored = [];
+        for (const [obsId, entry] of this.vectors) {
+            scored.push({ obsId, sessionId: entry.sessionId, score: cosine(query, entry.embedding) });
+        }
+        scored.sort((a, b) => b.score - a.score);
+        return limit < scored.length ? scored.slice(0, limit) : scored;
+    }
+    // Reports any stored vectors whose dimension differs from `expected`, plus
+    // the distinct dimensions seen. The persistence guard refuses to load an
+    // index with mismatches; the only clean state is no mismatches and a single
+    // seen dimension equal to `expected`.
+    validateDimensions(expected) {
+        const mismatches = [];
+        const seenDimensions = new Set();
+        for (const [obsId, entry] of this.vectors) {
+            const dim = entry.embedding.length;
+            seenDimensions.add(dim);
+            if (dim !== expected)
+                mismatches.push({ obsId, dim });
+        }
+        return { mismatches, seenDimensions };
+    }
+    clear() {
+        this.vectors.clear();
+    }
+    restoreFrom(other) {
+        this.vectors = new Map();
+        for (const [obsId, entry] of other.vectors) {
+            this.vectors.set(obsId, {
+                embedding: new Float32Array(entry.embedding),
+                sessionId: entry.sessionId,
+            });
+        }
+    }
+    serialize() {
+        const rows = [];
+        for (const [obsId, entry] of this.vectors) {
+            rows.push([obsId, { embedding: float32ToBase64(entry.embedding), sessionId: entry.sessionId }]);
+        }
+        return JSON.stringify(rows);
+    }
+    static deserialize(json) {
+        const idx = new VectorIndex();
+        let rows;
+        try {
+            rows = JSON.parse(json);
+        }
+        catch {
+            return idx;
+        }
+        if (!Array.isArray(rows))
+            return idx;
+        for (const row of rows) {
+            if (!Array.isArray(row) || row.length < 2)
+                continue;
+            const [obsId, entry] = row;
+            if (typeof obsId !== "string" ||
+                typeof entry?.embedding !== "string" ||
+                typeof entry?.sessionId !== "string") {
+                continue;
+            }
+            try {
+                idx.vectors.set(obsId, {
+                    embedding: base64ToFloat32(entry.embedding),
+                    sessionId: entry.sessionId,
+                });
+            }
+            catch {
+                // skip a corrupt row rather than fail the whole restore
+            }
+        }
+        return idx;
+    }
+}

package/dist/functions/vector-persistence.d.ts

@@ -0,0 +1,14 @@
+import type { StateKV } from "../state/kv.js";
+/**
+ * Persists the current quantized index. No-op (false) when quantization is
+ * disabled or the active index is not a QuantizedVectorIndex.
+ */
+export declare function persistVectorIndex(kv: StateKV): Promise<boolean>;
+/**
+ * Loads a previously persisted quantized index and installs it as the
+ * active vector index. Returns true only when a valid blob was loaded AND
+ * its params match the current configuration (and the provider dimensions,
+ * when a provider is wired). Any mismatch leaves the current index in
+ * place and returns false so the caller can rebuild.
+ */
+export declare function loadVectorIndex(kv: StateKV): Promise<boolean>;

package/dist/functions/vector-persistence.js

@@ -0,0 +1,75 @@
+//
+// Quantized-vector-index persistence. Nothing else saves or loads vector
+// state, so this owns it. Best-effort soft-fail throughout, matching search.ts:
+// a persistence problem must never break observe/search.
+//
+// Layout: one blob under the `quantParams` scope. The blob embeds its own
+// params (seed, bits, dims, version, level-table hash); deserialize
+// validates them and returns null on any mismatch, which callers treat as
+// "rebuild from source of truth".
+import { KV } from "../state/schema.js";
+import { QuantizedVectorIndex } from "./quantized-vector-index.js";
+import { getVectorIndex, setVectorIndex, getEmbeddingProvider } from "./search.js";
+import { isQuantizedVectorEnabled, getQuantRescoreDepth } from "./config.js";
+import { logger } from "./logger.js";
+const BLOB_KEY = "index-blob";
+/**
+ * Persists the current quantized index. No-op (false) when quantization is
+ * disabled or the active index is not a QuantizedVectorIndex.
+ */
+export async function persistVectorIndex(kv) {
+    const idx = getVectorIndex();
+    if (!isQuantizedVectorEnabled() || !(idx instanceof QuantizedVectorIndex)) {
+        return false;
+    }
+    try {
+        await kv.set(KV.quantParams, BLOB_KEY, idx.serialize());
+        return true;
+    }
+    catch (err) {
+        logger.warn("vector-persistence: persist failed", {
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return false;
+    }
+}
+/**
+ * Loads a previously persisted quantized index and installs it as the
+ * active vector index. Returns true only when a valid blob was loaded AND
+ * its params match the current configuration (and the provider dimensions,
+ * when a provider is wired). Any mismatch leaves the current index in
+ * place and returns false so the caller can rebuild.
+ */
+export async function loadVectorIndex(kv) {
+    if (!isQuantizedVectorEnabled())
+        return false;
+    try {
+        const blob = await kv.get(KV.quantParams, BLOB_KEY);
+        if (typeof blob !== "string" || !blob)
+            return false;
+        const idx = QuantizedVectorIndex.deserialize(blob);
+        if (!idx) {
+            logger.warn("vector-persistence: stored index params no longer valid — rebuild required");
+            return false;
+        }
+        const provider = getEmbeddingProvider();
+        if (provider && provider.dimensions !== idx.params.dims) {
+            logger.warn("vector-persistence: stored dims mismatch provider — rebuild", {
+                stored: idx.params.dims,
+                provider: provider.dimensions,
+            });
+            return false;
+        }
+        // The blob carries the rescore setting it was built with; the current
+        // environment wins. Lowering to 0 also frees the retained full vectors.
+        idx.reconcileRescoreDepth(getQuantRescoreDepth());
+        setVectorIndex(idx);
+        return true;
+    }
+    catch (err) {
+        logger.warn("vector-persistence: load failed", {
+            error: err instanceof Error ? err.message : String(err),
+        });
+        return false;
+    }
+}

package/dist/functions/verify.d.ts

@@ -0,0 +1,13 @@
+import type { Provenance } from "./types.js";
+/**
+ * Hash the referenced files under `root` at capture time (best-effort). Files
+ * that don't exist or are too large are simply omitted; the result is stored
+ * in provenance so later recall can detect content drift.
+ */
+export declare function hashFiles(files: string[] | undefined, root: string): Record<string, string>;
+export type VerifyStatus = "verified" | "sourced_unverified" | "stale" | "unsourced";
+export interface Verdict {
+    status: VerifyStatus;
+    reason: string;
+}
+export declare function classifyProvenance(prov: Provenance | undefined, root: string): Verdict;

package/dist/functions/verify.js

@@ -0,0 +1,104 @@
+//
+// Verified Recall: classify a memory's trustworthiness against the live repo.
+// This is what makes "verified" literal — not just "does the file exist" but
+// "is the file still what it was when we learned this".
+//
+//   verified           a referenced file exists and still matches its
+//                      capture-time content hash (code-backed and current)
+//   sourced_unverified sourced (command/confirmation, or files present but
+//                      none hashable), so allowed, but NOT content-verified
+//   stale              a referenced file was deleted, or its content changed
+//   unsourced          no evidence at all (no files, no command, not confirmed)
+//
+// All checks read the repo, so this runs in the daemon (same machine). Hashing
+// is best-effort: files missing at capture, non-files, and files over the size
+// cap are not hashed, so such a memory verifies by existence only and reports
+// sourced_unverified rather than verified.
+import { createHash } from "node:crypto";
+import { existsSync, readFileSync, statSync } from "node:fs";
+import { isAbsolute, resolve } from "node:path";
+import { isUnsourced } from "./provenance.js";
+// Don't hash enormous files; treat them as unhashed (existence-only).
+const MAX_HASH_BYTES = 2_000_000;
+function hashFile(abs) {
+    try {
+        const st = statSync(abs);
+        if (!st.isFile() || st.size > MAX_HASH_BYTES)
+            return null;
+        return createHash("sha256").update(readFileSync(abs)).digest("hex");
+    }
+    catch {
+        return null;
+    }
+}
+function resolveUnder(root, file) {
+    return isAbsolute(file) ? file : resolve(root, file);
+}
+/**
+ * Hash the referenced files under `root` at capture time (best-effort). Files
+ * that don't exist or are too large are simply omitted; the result is stored
+ * in provenance so later recall can detect content drift.
+ */
+export function hashFiles(files, root) {
+    const out = {};
+    if (!files)
+        return out;
+    for (const f of files) {
+        const h = hashFile(resolveUnder(root, f));
+        if (h)
+            out[f] = h;
+    }
+    return out;
+}
+export function classifyProvenance(prov, root) {
+    if (isUnsourced(prov)) {
+        return { status: "unsourced", reason: "no file, command, or user-confirmation evidence" };
+    }
+    const files = prov?.files ?? [];
+    const hashes = prov?.fileHashes ?? {};
+    const deleted = [];
+    const changed = [];
+    let hashMatched = 0; // existing files whose captured hash still matches
+    let unchecked = 0; // existing files we could not content-check
+    for (const f of files) {
+        const abs = resolveUnder(root, f);
+        if (!existsSync(abs)) {
+            deleted.push(f);
+            continue;
+        }
+        const recorded = hashes[f];
+        if (!recorded) {
+            unchecked++; // no hash captured (e.g. too large at capture)
+            continue;
+        }
+        const current = hashFile(abs);
+        if (current && current !== recorded)
+            changed.push(f);
+        else if (current && current === recorded)
+            hashMatched++;
+        else
+            unchecked++; // can't hash now (e.g. grew past the cap) -> unverified
+    }
+    if (deleted.length > 0 || changed.length > 0) {
+        const parts = [];
+        if (deleted.length > 0)
+            parts.push(`deleted: ${deleted.slice(0, 2).join(", ")}`);
+        if (changed.length > 0)
+            parts.push(`changed: ${changed.slice(0, 2).join(", ")}`);
+        return { status: "stale", reason: `references files that no longer match (${parts.join("; ")})` };
+    }
+    // Verified only when EVERY existing referenced file was content-checked.
+    // A single unchecked file (unhashed, or too large) leaves the memory
+    // sourced-but-not-verified, so one matching hash can't vouch for the rest.
+    if (hashMatched > 0 && unchecked === 0) {
+        return { status: "verified", reason: "all referenced files exist and match their captured hashes" };
+    }
+    return {
+        status: "sourced_unverified",
+        reason: hashMatched > 0
+            ? "some referenced files verified, but others could not be content-checked"
+            : files.length > 0
+                ? "referenced files exist but were not hashed at capture (existence only)"
+                : "sourced by command or user, no file evidence to verify against",
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/index.js

@@ -0,0 +1,219 @@
+//
+// memwarden boot entrypoint.
+// 
+// - build the kernel via registerWorker,
+// - register app functions (./functions/*) if present,
+// - start the node:http REST server on restPort,
+// - keep the periodic sweeps as plain setInterval(...).unref() timers
+// that fire `trigger mem::*` (no scheduler in the SDK surface),
+// - graceful shutdown on SIGINT/SIGTERM.
+//
+// the core: the ./functions/* modules may not all exist yet. Function
+// registration is therefore best-effort: a missing module is logged and
+// skipped so the kernel still boots and serves whatever is wired.
+import { registerWorker, startHttpServer } from "./kernel/index.js";
+import { StoreLibsql } from "./state/store-libsql.js";
+import { StateKV } from "./state/kv.js";
+import { registerCoreFunctions, setEmbeddingProvider, setVectorIndex, makeVectorIndex, } from "./functions/index.js";
+import { isQuantizedVectorEnabled, isProxyEnabled, getUpstreamUrl, getUpstreamKey, getProxyPort, getSecret, } from "./functions/config.js";
+import { createEmbeddingProvider } from "./embedding/index.js";
+import { registerApiTriggers } from "./triggers/api.js";
+import { startProxyServer } from "./proxy/server.js";
+const REST_PORT = parseInt(process.env.MEMWARDEN_REST_PORT ?? "3111", 10);
+const STORE_URL = process.env.MEMWARDEN_STORE_URL ??
+    (process.env.MEMWARDEN_DATA_DIR
+        ? `file:${process.env.MEMWARDEN_DATA_DIR}/memwarden.db`
+        : "file:./data/memwarden.db");
+// Top-level safety net. Under sustained write load a single `state::*`
+// or fire-and-forget trigger rejection should never terminate the
+// long-lived memory service. The kernel surfaces rejections to the
+// relevant call site via .catch(); everything else is logged and
+// continued. Throttle to avoid spamming on bursts (matches the daemon
+// index.ts which reads reason.code / function_id / message).
+let lastUnhandledLogAt = 0;
+process.on("unhandledRejection", (reason) => {
+    const now = Date.now();
+    if (now - lastUnhandledLogAt < 60_000)
+        return;
+    lastUnhandledLogAt = now;
+    const r = reason;
+    console.warn(`[memwarden] unhandledRejection (suppressed):`, r?.code
+        ? `${r.code} ${r.function_id ?? ""} ${r.message ?? ""}`.trim()
+        : reason);
+});
+/**
+ * Optionally load a function-registration module by path and call its
+ * exported registrar. Missing modules are skipped.
+ */
+async function tryRegister(modulePath, exportName, ...args) {
+    try {
+        const mod = (await import(modulePath));
+        const fn = mod[exportName];
+        if (typeof fn === "function") {
+            fn(...args);
+            return true;
+        }
+        return false;
+    }
+    catch (err) {
+        const code = err.code;
+        // ERR_MODULE_NOT_FOUND is expected while functions are still being
+        // wired; anything else is a real registration failure worth a log.
+        if (code !== "ERR_MODULE_NOT_FOUND") {
+            console.warn(`[memwarden] failed to register ${exportName} from ${modulePath}:`, err instanceof Error ? err.message : err);
+        }
+        return false;
+    }
+}
+/**
+ * Register the application functions against the kernel.
+ *
+ * The core (mem::observe / mem::context / mem::search and their
+ * HTTP routes) is wired statically: the modules exist and share a single
+ * StateKV constructed over the kernel. Functions still being wired
+ * (smart-search, remember, enrich, events, health) remain best-effort
+ * dynamic imports so the kernel still boots while they land.
+ */
+async function registerFunctions(sdk) {
+    // Core path: one StateKV over the kernel, shared by all three functions.
+    const kv = new StateKV(sdk);
+    registerCoreFunctions(sdk, kv);
+    registerApiTriggers(sdk);
+    let registered = 3; // observe + context + search
+    // Functions still being wired; absent modules are no-ops.
+    const tasks = [
+        tryRegister("./functions/smart-search.js", "registerSmartSearchFunction", sdk),
+        tryRegister("./functions/remember.js", "registerRememberFunction", sdk),
+        tryRegister("./functions/enrich.js", "registerEnrichFunction", sdk),
+        tryRegister("./triggers/events.js", "registerEventTriggers", sdk),
+        tryRegister("./health/monitor.js", "registerHealthMonitor", sdk),
+    ];
+    const results = await Promise.all(tasks);
+    for (const ok of results)
+        if (ok)
+            registered++;
+    return registered;
+}
+/**
+ * Install the periodic maintenance sweeps. The SDK surface has no cron
+ * primitive: these are plain unref'd interval timers that fire a
+ * `trigger mem::*`. A trigger to an unregistered function rejects
+ * harmlessly (caught here), so this is safe to install before the
+ * corresponding functions are wired.
+ */
+function installSweeps(sdk) {
+    const timers = [];
+    const HOUR = 60 * 60 * 1000;
+    const DAY = 24 * HOUR;
+    const fire = (functionId, payload) => {
+        sdk
+            .trigger({ function_id: functionId, payload })
+            .catch(() => undefined);
+    };
+    const schedule = (enabled, intervalMs, functionId, payload) => {
+        if (!enabled)
+            return;
+        // Small jitter spreads sweep load so they don't all fire on the
+        // same tick after a restart.
+        const jitter = Math.floor(Math.random() * Math.min(intervalMs, 60_000));
+        const timer = setInterval(() => fire(functionId, payload), intervalMs);
+        timer.unref();
+        timers.push(timer);
+        const kickoff = setTimeout(() => fire(functionId, payload), jitter);
+        kickoff.unref();
+    };
+    const autoForgetInterval = parseInt(process.env.AUTO_FORGET_INTERVAL_MS ?? "3600000", 10);
+    const consolidationInterval = parseInt(process.env.CONSOLIDATION_INTERVAL_MS ?? "7200000", 10);
+    schedule(process.env.AUTO_FORGET_ENABLED !== "false", autoForgetInterval, "mem::auto-forget", { dryRun: false });
+    schedule(process.env.LESSON_DECAY_ENABLED !== "false", DAY, "mem::lesson-decay-sweep", {});
+    schedule(process.env.INSIGHT_DECAY_ENABLED !== "false", DAY, "mem::insight-decay-sweep", {});
+    schedule(true, HOUR, "mem::diagnostic::recent-searches-sweep", {});
+    schedule(process.env.CONSOLIDATION_ENABLED === "true", consolidationInterval, "mem::consolidate-pipeline", {});
+    return timers;
+}
+async function main() {
+    const store = new StoreLibsql({ url: STORE_URL });
+    const sdk = registerWorker("in-process", {
+        workerName: "memwarden",
+        invocationTimeoutMs: 180000,
+    }, { store });
+    const registered = await registerFunctions(sdk);
+    console.log(`[memwarden] kernel ready — ${registered} function module(s) registered, store=${STORE_URL}`);
+    // Semantic memory: wire the embedding provider and the (TurboQuant-
+    // compressed by default) vector index. With no provider, memwarden runs
+    // BM25-only — identical to the prior behavior. The model loads lazily on
+    // first observe/search; warm it in the background so the first request is
+    // fast without blocking boot.
+    const embProvider = createEmbeddingProvider();
+    if (embProvider) {
+        setEmbeddingProvider(embProvider);
+        setVectorIndex(makeVectorIndex(embProvider.dimensions));
+        const quantized = isQuantizedVectorEnabled();
+        console.log(`[memwarden] semantic memory: ${embProvider.name} (${embProvider.dimensions}d), ` +
+            `storage=${quantized ? "TurboQuant-compressed" : "full-precision"}`);
+        const warmable = embProvider;
+        if (typeof warmable.warmup === "function") {
+            warmable.warmup().catch((err) => {
+                console.warn(`[memwarden] embedding model warmup failed — vector stream stays off until it loads:`, err instanceof Error ? err.message : err);
+            });
+        }
+    }
+    else {
+        console.log(`[memwarden] semantic memory: disabled (BM25-only)`);
+    }
+    const http = startHttpServer(sdk, { port: REST_PORT });
+    // Race-safe self-heal: if another memwarden already holds the port, this
+    // spawn is redundant — exit cleanly (0) rather than crash, so concurrent
+    // ensureDaemon() callers never surface an error.
+    http.server.on("error", (err) => {
+        if (err.code === "EADDRINUSE") {
+            console.log(`[memwarden] port ${REST_PORT} already in use — another instance is running; exiting.`);
+            process.exit(0);
+        }
+        console.error(`[memwarden] HTTP server error:`, err);
+        process.exit(1);
+    });
+    console.log(`[memwarden] REST API: http://127.0.0.1:${REST_PORT}/memwarden/*`);
+    // The memory proxy — the universal cross-tool layer. Off until an upstream
+    // is configured (it has nothing to forward to otherwise). When on, point
+    // any OpenAI-compatible tool's base URL at it and every model call, local
+    // or paid, flows through memwarden's recall + capture.
+    let proxy;
+    const upstreamUrl = getUpstreamUrl();
+    if (isProxyEnabled() && upstreamUrl) {
+        const proxyPort = getProxyPort();
+        const cwd = process.cwd();
+        proxy = startProxyServer({
+            port: proxyPort,
+            upstreamUrl,
+            daemonUrl: `http://127.0.0.1:${REST_PORT}`,
+            project: cwd,
+            cwd,
+            ...(getUpstreamKey() ? { upstreamKey: getUpstreamKey() } : {}),
+            ...(getSecret() ? { secret: getSecret() } : {}),
+        });
+        console.log(`[memwarden] memory proxy: http://127.0.0.1:${proxyPort}/v1 -> ${upstreamUrl} ` +
+            `(point any OpenAI-compatible tool here for automatic memory)`);
+    }
+    const timers = installSweeps(sdk);
+    let shuttingDown = false;
+    const shutdown = async () => {
+        if (shuttingDown)
+            return;
+        shuttingDown = true;
+        console.log(`\n[memwarden] Shutting down...`);
+        for (const t of timers)
+            clearInterval(t);
+        await http.close().catch(() => undefined);
+        if (proxy)
+            await proxy.close().catch(() => undefined);
+        await sdk.shutdown();
+        process.exit(0);
+    };
+    process.on("SIGINT", () => void shutdown());
+    process.on("SIGTERM", () => void shutdown());
+}
+main().catch((err) => {
+    console.error(`[memwarden] Fatal:`, err);
+    process.exit(1);
+});

package/dist/kernel/http.d.ts

@@ -0,0 +1,24 @@
+import { type Server } from "node:http";
+import type { Kernel } from "./kernel.js";
+export interface HttpServerOptions {
+    port: number;
+    host?: string;
+    /** Allowed CORS origins. Defaults to the local viewer/REST quartet. */
+    allowedOrigins?: string[];
+    /** Max request body bytes before 413. Defaults to 16 MiB. */
+    maxBodyBytes?: number;
+}
+export interface RunningHttpServer {
+    server: Server;
+    port: number;
+    close(): Promise<void>;
+}
+export declare function startHttpServer(kernel: Kernel, opts: HttpServerOptions): RunningHttpServer;
+/**
+ * Accept only a loopback Host header bound to our port (or with no port). The
+ * Host header is case-insensitive and may carry `:port` or be a bracketed IPv6
+ * literal; we split host/port robustly and reject anything non-loopback. This
+ * is the DNS-rebinding guard: the value reflects the hostname the client
+ * actually targeted, which a rebinding attacker cannot forge to "localhost".
+ */
+export declare function isLoopbackHost(hostHeader: string | undefined, port: number | undefined): boolean;