memwarden 0.0.1

package/dist/functions/synonyms.js

@@ -0,0 +1,69 @@
+//
+// Query-time synonym expansion for BM25. Each group below is a set of common
+// developer terms that should match one another (abbreviation, full word,
+// inflections). Groups are stemmed once at load so lookups key off the same
+// stems the tokenizer emits; the BM25 scorer weights expanded hits below
+// exact ones.
+import { stem } from "./stemmer.js";
+// Factual term associations (dev abbreviations and their expansions).
+const GROUPS = [
+    ["auth", "authentication", "authn", "authenticating"],
+    ["authz", "authorization", "authorizing"],
+    ["db", "database", "datastore"],
+    ["perf", "performance", "latency", "throughput", "slow", "bottleneck"],
+    ["optim", "optimization", "optimizing", "optimise", "query-optimization"],
+    ["k8s", "kubernetes", "kube"],
+    ["config", "configuration", "configuring", "setup"],
+    ["deps", "dependencies", "dependency"],
+    ["env", "environment"],
+    ["fn", "function"],
+    ["impl", "implementation", "implementing"],
+    ["msg", "message", "messaging"],
+    ["repo", "repository"],
+    ["req", "request"],
+    ["res", "response"],
+    ["ts", "typescript"],
+    ["js", "javascript"],
+    ["pg", "postgres", "postgresql"],
+    ["err", "error", "errors"],
+    ["api", "endpoint", "endpoints"],
+    ["ci", "continuous-integration"],
+    ["cd", "continuous-deployment"],
+    ["test", "testing", "tests"],
+    ["doc", "documentation", "docs"],
+    ["infra", "infrastructure"],
+    ["deploy", "deployment", "deploying"],
+    ["cache", "caching", "cached"],
+    ["log", "logging", "logs"],
+    ["monitor", "monitoring"],
+    ["observe", "observability"],
+    ["sec", "security", "secure"],
+    ["validate", "validation", "validating"],
+    ["migrate", "migration", "migrations"],
+    ["debug", "debugging"],
+    ["container", "containerization", "docker"],
+    ["crash", "crashloop", "crashloopbackoff"],
+    ["webhook", "webhooks", "callback"],
+    ["middleware", "mw"],
+    ["paginate", "pagination"],
+    ["serialize", "serialization"],
+    ["encrypt", "encryption"],
+    ["hash", "hashing"],
+];
+// stem -> the other stems in its group
+const byStem = new Map();
+for (const group of GROUPS) {
+    const stems = [...new Set(group.map((term) => stem(term.toLowerCase())))];
+    for (const s of stems) {
+        let bucket = byStem.get(s);
+        if (!bucket)
+            byStem.set(s, (bucket = new Set()));
+        for (const other of stems)
+            if (other !== s)
+                bucket.add(other);
+    }
+}
+export function getSynonyms(stemmedTerm) {
+    const bucket = byStem.get(stemmedTerm);
+    return bucket ? [...bucket] : [];
+}

package/dist/functions/turboquant.d.ts

@@ -0,0 +1,53 @@
+export declare const TURBOQUANT_VERSION = 1;
+export declare const ROTATION_ROUNDS = 3;
+export type QuantBits = 2 | 4;
+export declare function nextPow2(n: number): number;
+export declare function seedFromString(seed: string): number;
+export declare function mulberry32(seed: number): () => number;
+/**
+ * One Int8Array of ±1 per rotation round, length D each. Generated by
+ * draining the PRNG in a fixed order, so (seed, D, rounds) fully determines
+ * the rotation.
+ */
+export declare function buildSignFlips(prng: () => number, D: number, rounds: number): Int8Array[];
+export declare function fwht(buf: Float32Array): void;
+/**
+ * Rotate `src` (length d <= D) into a length-D buffer: zero-pad, then
+ * `signFlips.length` rounds of (sign flip, FWHT, 1/sqrt(D) rescale).
+ * Norm-preserving up to f32 rounding. Non-finite inputs are sanitized to 0.
+ * Pass `scratch` (length D) to avoid per-call allocation.
+ */
+export declare function rotate(src: Float32Array, D: number, signFlips: Int8Array[], scratch?: Float32Array): Float32Array;
+export interface LevelTable {
+    levels: Float32Array;
+    boundaries: Float32Array;
+}
+/**
+ * Lloyd-Max iteration for N(0,1): boundaries are midpoints of adjacent
+ * levels; each level is the conditional mean of the cell. Converges in a
+ * few dozen iterations; deterministic, so the table never needs persisting
+ * (a short hash travels in index params to invalidate on algorithm change).
+ */
+export declare function lloydMaxLevels(bits: QuantBits): LevelTable;
+export declare function levelTableHash(bits: QuantBits): string;
+export declare function packCodes(codeIndices: Uint8Array, bits: QuantBits): Uint8Array;
+export declare function unpackCodes(packed: Uint8Array, D: number, bits: QuantBits): Uint8Array;
+/**
+ * Quantize an already-rotated length-D buffer. Returns the packed codes and
+ * the original vector norm (rotation is norm-preserving, so this is also
+ * the pre-rotation norm). A zero/degenerate vector encodes with norm 0 and
+ * the code of the level nearest zero.
+ */
+export declare function encodeRotated(rotated: Float32Array, D: number, bits: QuantBits): {
+    codes: Uint8Array;
+    norm: number;
+};
+/**
+ * Asymmetric partial inner product: Σ rotatedQuery[i] * LEVELS[code_i].
+ * The query stays full precision (rotated once per search); codes are read
+ * nibble/crumb-wise without unpacking allocations. f64 accumulation.
+ *
+ * Cosine estimate for a stored vector v: asymmetricDot / (sqrt(D) * |q|),
+ * because stored codes approximate sqrt(D) * v_i / ||v||.
+ */
+export declare function asymmetricDot(rotatedQuery: Float32Array, codes: Uint8Array, D: number, bits: QuantBits, table?: LevelTable): number;

package/dist/functions/turboquant.js

@@ -0,0 +1,278 @@
+//
+// TurboQuant-style data-oblivious vector quantization. Pure math, no I/O.
+//
+// Implements the core of "TurboQuant: Online Vector Quantization with
+// Near-optimal Distortion Rate" (Google Research, arXiv:2504.19874, ICLR
+// 2026): randomly rotate input vectors so coordinates become
+// near-independent with a known concentrated distribution, then apply a
+// FIXED precomputed optimal (Lloyd-Max) scalar quantizer per coordinate.
+// No codebook training, no data-dependent state: codes are computable on
+// the first add(). The MIT-licensed Rust reference is
+// https://github.com/RyanCodrai/turbovec.
+//
+// Approximation note: after `ROTATION_ROUNDS` randomized Hadamard rounds,
+// the coordinates of a unit vector are treated as i.i.d. N(0, 1/D). This is
+// exact in the D→infinity limit and close for D >= 128. Real embeddings are
+// anisotropic; the rotation Gaussianizes them and the optional rescore pass
+// (see quantized-vector-index.ts) absorbs the residual error.
+//
+// We quantize the scaled coordinate sqrt(D) * x_i / ||x||, whose reference
+// distribution is N(0, 1), so a single dimension-independent level table
+// serves every index.
+import { createHash } from "node:crypto";
+export const TURBOQUANT_VERSION = 1;
+export const ROTATION_ROUNDS = 3;
+export function nextPow2(n) {
+    if (n <= 1)
+        return 1;
+    return 2 ** Math.ceil(Math.log2(n));
+}
+// ---------------------------------------------------------------------------
+// Deterministic PRNG. The rotation must be reconstructible from params alone
+// across processes, so Math.random is banned. mulberry32 is a tiny 32-bit
+// generator with stable output for a given seed.
+export function seedFromString(seed) {
+    // FNV-1a over UTF-16 code units, folded to uint32.
+    let h = 0x811c9dc5;
+    for (let i = 0; i < seed.length; i++) {
+        h ^= seed.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) | 0;
+        let t = Math.imul(a ^ (a >>> 15), 1 | a);
+        t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/**
+ * One Int8Array of ±1 per rotation round, length D each. Generated by
+ * draining the PRNG in a fixed order, so (seed, D, rounds) fully determines
+ * the rotation.
+ */
+export function buildSignFlips(prng, D, rounds) {
+    const flips = [];
+    for (let r = 0; r < rounds; r++) {
+        const s = new Int8Array(D);
+        for (let i = 0; i < D; i++) {
+            s[i] = prng() < 0.5 ? -1 : 1;
+        }
+        flips.push(s);
+    }
+    return flips;
+}
+// ---------------------------------------------------------------------------
+// Fast Walsh-Hadamard Transform, in place, unnormalized. Callers rescale by
+// 1/sqrt(D) per application to keep the transform orthonormal.
+export function fwht(buf) {
+    const n = buf.length;
+    for (let len = 1; len < n; len <<= 1) {
+        for (let i = 0; i < n; i += len << 1) {
+            for (let j = i; j < i + len; j++) {
+                const a = buf[j];
+                const b = buf[j + len];
+                buf[j] = a + b;
+                buf[j + len] = a - b;
+            }
+        }
+    }
+}
+/**
+ * Rotate `src` (length d <= D) into a length-D buffer: zero-pad, then
+ * `signFlips.length` rounds of (sign flip, FWHT, 1/sqrt(D) rescale).
+ * Norm-preserving up to f32 rounding. Non-finite inputs are sanitized to 0.
+ * Pass `scratch` (length D) to avoid per-call allocation.
+ */
+export function rotate(src, D, signFlips, scratch) {
+    const out = scratch && scratch.length === D ? scratch : new Float32Array(D);
+    for (let i = 0; i < src.length; i++) {
+        const v = src[i];
+        out[i] = Number.isFinite(v) ? v : 0;
+    }
+    out.fill(0, src.length);
+    const invSqrtD = 1 / Math.sqrt(D);
+    for (const flips of signFlips) {
+        for (let i = 0; i < D; i++) {
+            out[i] = out[i] * flips[i];
+        }
+        fwht(out);
+        for (let i = 0; i < D; i++) {
+            out[i] = out[i] * invSqrtD;
+        }
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Lloyd-Max scalar quantizer for the standard normal, computed analytically
+// at init (no data involved) and cached per bit width.
+function normalPdf(x) {
+    return Math.exp(-0.5 * x * x) / Math.sqrt(2 * Math.PI);
+}
+// Abramowitz & Stegun 7.1.26 rational erf approximation (~1e-7 abs error).
+function erf(z) {
+    const sign = z < 0 ? -1 : 1;
+    const x = Math.abs(z);
+    const t = 1 / (1 + 0.3275911 * x);
+    const y = 1 -
+        (((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t -
+            0.284496736) *
+            t +
+            0.254829592) *
+            t) *
+            Math.exp(-x * x);
+    return sign * y;
+}
+function normalCdf(x) {
+    return 0.5 * (1 + erf(x / Math.SQRT2));
+}
+const levelTableCache = new Map();
+/**
+ * Lloyd-Max iteration for N(0,1): boundaries are midpoints of adjacent
+ * levels; each level is the conditional mean of the cell. Converges in a
+ * few dozen iterations; deterministic, so the table never needs persisting
+ * (a short hash travels in index params to invalidate on algorithm change).
+ */
+export function lloydMaxLevels(bits) {
+    const cached = levelTableCache.get(bits);
+    if (cached)
+        return cached;
+    const L = 2 ** bits;
+    const INF = 8; // φ(8) ≈ 5e-15: numerically infinite for our purposes
+    const levels = new Float64Array(L);
+    for (let k = 0; k < L; k++) {
+        levels[k] = -3 + (6 * (k + 0.5)) / L;
+    }
+    const bounds = new Float64Array(L - 1);
+    for (let iter = 0; iter < 200; iter++) {
+        for (let k = 0; k < L - 1; k++) {
+            bounds[k] = (levels[k] + levels[k + 1]) / 2;
+        }
+        let moved = 0;
+        for (let k = 0; k < L; k++) {
+            const a = k === 0 ? -INF : bounds[k - 1];
+            const b = k === L - 1 ? INF : bounds[k];
+            const mass = normalCdf(b) - normalCdf(a);
+            if (mass <= 0)
+                continue;
+            const centroid = (normalPdf(a) - normalPdf(b)) / mass;
+            moved = Math.max(moved, Math.abs(centroid - levels[k]));
+            levels[k] = centroid;
+        }
+        if (moved < 1e-10)
+            break;
+    }
+    const table = {
+        levels: Float32Array.from(levels),
+        boundaries: Float32Array.from(bounds),
+    };
+    levelTableCache.set(bits, table);
+    return table;
+}
+export function levelTableHash(bits) {
+    const { levels } = lloydMaxLevels(bits);
+    const repr = Array.from(levels, (v) => v.toFixed(6)).join(",");
+    return createHash("sha256").update(repr).digest("hex").slice(0, 16);
+}
+// ---------------------------------------------------------------------------
+// Code packing. 4-bit: two codes per byte (even index in the low nibble);
+// 2-bit: four codes per byte (index i in bits 2*(i % 4)).
+export function packCodes(codeIndices, bits) {
+    const D = codeIndices.length;
+    if (bits === 4) {
+        const out = new Uint8Array(Math.ceil(D / 2));
+        for (let i = 0; i < D; i++) {
+            const c = codeIndices[i] & 0xf;
+            if (i % 2 === 0)
+                out[i >> 1] = c;
+            else
+                out[i >> 1] = out[i >> 1] | (c << 4);
+        }
+        return out;
+    }
+    const out = new Uint8Array(Math.ceil(D / 4));
+    for (let i = 0; i < D; i++) {
+        const c = codeIndices[i] & 0x3;
+        out[i >> 2] = out[i >> 2] | (c << ((i % 4) * 2));
+    }
+    return out;
+}
+export function unpackCodes(packed, D, bits) {
+    const out = new Uint8Array(D);
+    if (bits === 4) {
+        for (let i = 0; i < D; i++) {
+            const byte = packed[i >> 1];
+            out[i] = i % 2 === 0 ? byte & 0xf : (byte >> 4) & 0xf;
+        }
+        return out;
+    }
+    for (let i = 0; i < D; i++) {
+        const byte = packed[i >> 2];
+        out[i] = (byte >> ((i % 4) * 2)) & 0x3;
+    }
+    return out;
+}
+// ---------------------------------------------------------------------------
+// Encode / score.
+/**
+ * Quantize an already-rotated length-D buffer. Returns the packed codes and
+ * the original vector norm (rotation is norm-preserving, so this is also
+ * the pre-rotation norm). A zero/degenerate vector encodes with norm 0 and
+ * the code of the level nearest zero.
+ */
+export function encodeRotated(rotated, D, bits) {
+    const { boundaries } = lloydMaxLevels(bits);
+    let normSq = 0;
+    for (let i = 0; i < D; i++) {
+        const v = rotated[i];
+        normSq += v * v;
+    }
+    const norm = Math.sqrt(normSq);
+    const scale = norm === 0 ? 0 : Math.sqrt(D) / norm;
+    const nB = boundaries.length;
+    const idx = new Uint8Array(D);
+    for (let i = 0; i < D; i++) {
+        const z = rotated[i] * scale;
+        // Binary search: count of boundaries < z is the code.
+        let lo = 0;
+        let hi = nB;
+        while (lo < hi) {
+            const mid = (lo + hi) >> 1;
+            if (boundaries[mid] < z)
+                lo = mid + 1;
+            else
+                hi = mid;
+        }
+        idx[i] = lo;
+    }
+    return { codes: packCodes(idx, bits), norm };
+}
+/**
+ * Asymmetric partial inner product: Σ rotatedQuery[i] * LEVELS[code_i].
+ * The query stays full precision (rotated once per search); codes are read
+ * nibble/crumb-wise without unpacking allocations. f64 accumulation.
+ *
+ * Cosine estimate for a stored vector v: asymmetricDot / (sqrt(D) * |q|),
+ * because stored codes approximate sqrt(D) * v_i / ||v||.
+ */
+export function asymmetricDot(rotatedQuery, codes, D, bits, table) {
+    const { levels } = table ?? lloydMaxLevels(bits);
+    let sum = 0;
+    if (bits === 4) {
+        for (let i = 0; i < D; i++) {
+            const byte = codes[i >> 1];
+            const code = i % 2 === 0 ? byte & 0xf : (byte >> 4) & 0xf;
+            sum += rotatedQuery[i] * levels[code];
+        }
+        return sum;
+    }
+    for (let i = 0; i < D; i++) {
+        const byte = codes[i >> 2];
+        const code = (byte >> ((i % 4) * 2)) & 0x3;
+        sum += rotatedQuery[i] * levels[code];
+    }
+    return sum;
+}

package/dist/functions/types.d.ts

@@ -0,0 +1,217 @@
+export type HookType = "session_start" | "prompt_submit" | "pre_tool_use" | "post_tool_use" | "post_tool_failure" | "pre_compact" | "subagent_start" | "subagent_stop" | "notification" | "task_completed" | "stop" | "session_end";
+export interface HookPayload {
+    hookType: HookType;
+    sessionId: string;
+    project: string;
+    cwd: string;
+    timestamp: string;
+    data: unknown;
+    /** Which agent captured this (claude, codex, cursor, …). Optional. */
+    agent?: string;
+}
+export interface Session {
+    id: string;
+    project: string;
+    cwd: string;
+    startedAt: string;
+    endedAt?: string;
+    status: "active" | "completed" | "abandoned";
+    observationCount: number;
+    model?: string;
+    tags?: string[];
+    firstPrompt?: string;
+    summary?: string;
+    commitShas?: string[];
+    agentId?: string;
+}
+export interface RawObservation {
+    id: string;
+    sessionId: string;
+    timestamp: string;
+    hookType: HookType;
+    toolName?: string;
+    toolInput?: unknown;
+    toolOutput?: unknown;
+    userPrompt?: string;
+    assistantResponse?: string;
+    raw: unknown;
+    modality?: "text" | "image" | "mixed";
+    imageData?: string;
+    agentId?: string;
+}
+export type ObservationType = "file_read" | "file_write" | "file_edit" | "command_run" | "search" | "web_fetch" | "conversation" | "error" | "decision" | "discovery" | "subagent" | "notification" | "task" | "image" | "other";
+/**
+ * Where a memory came from — the evidence the doctor audits. A memory with
+ * provenance can be checked for staleness (do its files still exist?) and
+ * sourcing (is there any evidence at all?); one without is "unsourced".
+ */
+export interface Provenance {
+    cwd?: string;
+    files?: string[];
+    fileHashes?: Record<string, string>;
+    command?: string;
+    agent?: string;
+    capturedAt?: string;
+    userConfirmed?: boolean;
+}
+export interface CompressedObservation {
+    id: string;
+    sessionId: string;
+    timestamp: string;
+    type: ObservationType;
+    title: string;
+    subtitle?: string;
+    facts: string[];
+    narrative: string;
+    concepts: string[];
+    files: string[];
+    importance: number;
+    confidence?: number;
+    imageRef?: string;
+    imageData?: string;
+    imageDescription?: string;
+    modality?: "text" | "image" | "mixed";
+    agentId?: string;
+    provenance?: Provenance;
+}
+export interface Memory {
+    id: string;
+    createdAt: string;
+    updatedAt: string;
+    type: "pattern" | "preference" | "architecture" | "bug" | "workflow" | "fact";
+    title: string;
+    content: string;
+    concepts: string[];
+    files: string[];
+    sessionIds: string[];
+    strength: number;
+    version: number;
+    parentId?: string;
+    supersedes?: string[];
+    relatedIds?: string[];
+    sourceObservationIds?: string[];
+    isLatest: boolean;
+    forgetAfter?: string;
+    imageRef?: string;
+    imageData?: string;
+    agentId?: string;
+    project?: string;
+    provenance?: Provenance;
+}
+export interface SessionSummary {
+    sessionId: string;
+    project: string;
+    createdAt: string;
+    title: string;
+    narrative: string;
+    keyDecisions: string[];
+    filesModified: string[];
+    concepts: string[];
+    observationCount: number;
+}
+export interface ProjectProfile {
+    project: string;
+    updatedAt: string;
+    topConcepts: Array<{
+        concept: string;
+        frequency: number;
+    }>;
+    topFiles: Array<{
+        file: string;
+        frequency: number;
+    }>;
+    conventions: string[];
+    commonErrors: string[];
+    recentActivity: string[];
+    sessionCount: number;
+    totalObservations: number;
+    summary?: string;
+}
+export interface ContextBlock {
+    type: "summary" | "observation" | "memory";
+    content: string;
+    tokens: number;
+    recency: number;
+    sourceIds?: string[];
+}
+export interface SearchResult {
+    observation: CompressedObservation;
+    score: number;
+    sessionId: string;
+}
+export interface CompactSearchResult {
+    obsId: string;
+    sessionId: string;
+    title: string;
+    type: ObservationType;
+    score: number;
+    timestamp: string;
+}
+export interface Lesson {
+    id: string;
+    content: string;
+    context: string;
+    confidence: number;
+    reinforcements: number;
+    source: "crystal" | "manual" | "consolidation";
+    sourceIds: string[];
+    project?: string;
+    tags: string[];
+    createdAt: string;
+    updatedAt: string;
+    lastReinforcedAt?: string;
+    lastDecayedAt?: string;
+    decayRate: number;
+    deleted?: boolean;
+}
+/**
+ * Embedding provider abstraction. The the core vector stream is stubbed to
+ * empty (no provider wired), so this only needs to exist for the
+ * VectorIndex / hybrid-fusion plumbing to typecheck; an actual provider
+ * lands in a later phase.
+ */
+export interface EmbeddingProvider {
+    name: string;
+    dimensions: number;
+    embed(text: string): Promise<Float32Array>;
+    embedBatch(texts: string[]): Promise<Float32Array[]>;
+    embedImage?(src: string): Promise<Float32Array>;
+}
+/** A single vector-stream hit, shared by every vector index implementation. */
+export interface VectorSearchHit {
+    obsId: string;
+    sessionId: string;
+    score: number;
+}
+/**
+ * The vector-index surface consumed by search.ts and hybrid-search.ts.
+ * Satisfied structurally by both VectorIndex (full-precision, the default)
+ * and QuantizedVectorIndex (TurboQuant codes, behind MEMWARDEN_QUANT_VECTOR).
+ */
+export interface VectorIndexLike {
+    add(obsId: string, sessionId: string, embedding: Float32Array): void;
+    remove(obsId: string): void;
+    has(obsId: string): boolean;
+    /** Snapshot of the stored obsIds; used for restore reconciliation. */
+    ids(): string[];
+    search(query: Float32Array, limit?: number): VectorSearchHit[];
+    readonly size: number;
+    clear(): void;
+    validateDimensions(expected: number): {
+        mismatches: Array<{
+            obsId: string;
+            dim: number;
+        }>;
+        seenDimensions: Set<number>;
+    };
+    serialize(): string;
+}
+export interface HybridSearchResult {
+    observation: CompressedObservation;
+    bm25Score: number;
+    vectorScore: number;
+    graphScore: number;
+    combinedScore: number;
+    sessionId: string;
+    graphContext?: string;
+}

package/dist/functions/types.js

@@ -0,0 +1,8 @@
+//
+// The subset of the shared data model that the wired core functions
+// (observe / context / search) and their supporting modules depend on.
+// Trimmed to only the shapes the core surface touches; the wire shapes
+// (HookPayload, RawObservation, CompressedObservation, Session, ...) are
+// kept the stable wire contract so existing connectors can
+// talk to memwarden unchanged.
+export {};

package/dist/functions/vector-index.d.ts

@@ -0,0 +1,25 @@
+export interface VectorHit {
+    obsId: string;
+    sessionId: string;
+    score: number;
+}
+export declare class VectorIndex {
+    private vectors;
+    add(obsId: string, sessionId: string, embedding: Float32Array): void;
+    remove(obsId: string): void;
+    has(obsId: string): boolean;
+    ids(): string[];
+    get size(): number;
+    search(query: Float32Array, limit?: number): VectorHit[];
+    validateDimensions(expected: number): {
+        mismatches: Array<{
+            obsId: string;
+            dim: number;
+        }>;
+        seenDimensions: Set<number>;
+    };
+    clear(): void;
+    restoreFrom(other: VectorIndex): void;
+    serialize(): string;
+    static deserialize(json: string): VectorIndex;
+}