@gamaze/hicortex 0.2.0

package/dist/retrieval.js

@@ -0,0 +1,320 @@
+"use strict";
+/**
+ * Retrieval layer with composite scoring, RRF fusion, and graph traversal.
+ * Ported from hicortex/retrieval.py — same scoring model and weights.
+ *
+ * Scoring model:
+ *   score = similarity * 0.4 + effective_strength * 0.3 + connection_score * 0.2 + recency * 0.1
+ *
+ * Decay model (B+E+D):
+ *   base_decay = 0.0005 (~60-day half-life at importance 0.5)
+ *   decay_rate = 1 - base_decay * (1 - importance)
+ *   decay_rate = 1 - (1 - decay_rate) * 0.7^access_count
+ *   decay_rate = 1 - (1 - decay_rate) * 0.7^link_count
+ *   floor = base_strength * importance * 0.1
+ *   effective = floor + (base - floor) * decay_rate^hours
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.effectiveStrength = effectiveStrength;
+exports.retrieve = retrieve;
+exports.searchContext = searchContext;
+const storage = __importStar(require("./storage.js"));
+const BASE_DECAY = 0.0005;
+const DEFAULT_GRAPH_DISTANCE = 0.5;
+const RRF_K = 60;
+// ---------------------------------------------------------------------------
+// Timestamp parsing
+// ---------------------------------------------------------------------------
+function parseTimestamp(ts) {
+    if (!ts)
+        return new Date();
+    try {
+        const dt = new Date(ts);
+        if (isNaN(dt.getTime()))
+            return new Date();
+        return dt;
+    }
+    catch {
+        return new Date();
+    }
+}
+// ---------------------------------------------------------------------------
+// Scoring helpers
+// ---------------------------------------------------------------------------
+/**
+ * Compute decayed strength with adaptive decay (B+E+D model).
+ * Exported for use by consolidation decay/prune stage.
+ */
+function effectiveStrength(baseStrength, lastAccessed, now, options) {
+    const importance = options?.importance ?? baseStrength;
+    const accessCount = options?.accessCount ?? 0;
+    const linkCount = options?.linkCount ?? 0;
+    const hours = Math.max((now.getTime() - parseTimestamp(lastAccessed).getTime()) / 3_600_000, 0);
+    // B: Importance slows decay
+    let decayRate = 1.0 - BASE_DECAY * (1.0 - importance);
+    // E: Access hardening
+    const hardening = 0.7;
+    decayRate = 1.0 - (1.0 - decayRate) * Math.pow(hardening, accessCount);
+    // E: Connectivity hardening
+    decayRate = 1.0 - (1.0 - decayRate) * Math.pow(hardening, linkCount);
+    // D: Asymptotic floor
+    const floor = baseStrength * importance * 0.1;
+    return floor + (baseStrength - floor) * Math.pow(decayRate, hours);
+}
+/**
+ * Return a composite relevance score in [0, 1] for a candidate memory.
+ */
+function computeScore(memory, distance, connectionCount, maxConnections, now) {
+    const similarity = Math.max(0, 1.0 - distance);
+    const effStrength = effectiveStrength(memory.base_strength ?? 0.5, memory.last_accessed, now, {
+        accessCount: memory.access_count ?? 0,
+        linkCount: connectionCount,
+    });
+    const connScore = maxConnections > 0 ? connectionCount / maxConnections : 0;
+    const hoursSinceCreated = Math.max((now.getTime() - parseTimestamp(memory.created_at).getTime()) / 3_600_000, 0);
+    const recency = Math.pow(0.9995, hoursSinceCreated);
+    return similarity * 0.4 + effStrength * 0.3 + connScore * 0.2 + recency * 0.1;
+}
+// ---------------------------------------------------------------------------
+// Graph traversal
+// ---------------------------------------------------------------------------
+function collectLinks(db, seedIds, maxHops = 2) {
+    const visited = new Set(seedIds);
+    const connectionCounts = new Map();
+    let frontier = new Set(seedIds);
+    for (let hop = 0; hop < maxHops; hop++) {
+        const nextFrontier = new Set();
+        for (const mid of frontier) {
+            const links = storage.getLinks(db, mid, "both");
+            const count = links.length;
+            connectionCounts.set(mid, (connectionCounts.get(mid) ?? 0) + count);
+            for (const link of links) {
+                const linkedId = link.source_id === mid ? link.target_id : link.source_id;
+                if (linkedId && !visited.has(linkedId)) {
+                    visited.add(linkedId);
+                    nextFrontier.add(linkedId);
+                }
+            }
+        }
+        frontier = nextFrontier;
+        if (frontier.size === 0)
+            break;
+    }
+    // Ensure newly discovered nodes also have a connection count
+    for (const mid of visited) {
+        if (!connectionCounts.has(mid)) {
+            const links = storage.getLinks(db, mid, "both");
+            connectionCounts.set(mid, links.length);
+        }
+    }
+    return connectionCounts;
+}
+// ---------------------------------------------------------------------------
+// Formatting
+// ---------------------------------------------------------------------------
+function formatResult(memory, score, effStr, connections) {
+    return {
+        id: memory.id,
+        content: memory.content ?? "",
+        score: Math.round(score * 1e6) / 1e6,
+        effective_strength: Math.round(effStr * 1e6) / 1e6,
+        access_count: memory.access_count ?? 0,
+        memory_type: memory.memory_type ?? "episode",
+        project: memory.project ?? null,
+        created_at: memory.created_at ?? "",
+        connections,
+    };
+}
+// ---------------------------------------------------------------------------
+// Strengthening
+// ---------------------------------------------------------------------------
+function strengthen(db, memories, now) {
+    const nowIso = now.toISOString();
+    for (const mem of memories) {
+        if (!mem.id)
+            continue;
+        try {
+            storage.strengthenMemory(db, mem.id, nowIso);
+        }
+        catch {
+            // Non-fatal — log would be ideal but we keep going
+        }
+    }
+}
+// ---------------------------------------------------------------------------
+// Reciprocal Rank Fusion
+// ---------------------------------------------------------------------------
+function reciprocalRankFusion(rankedLists, k = RRF_K) {
+    const scores = new Map();
+    for (const ranked of rankedLists) {
+        for (let rank = 0; rank < ranked.length; rank++) {
+            const mid = ranked[rank];
+            scores.set(mid, (scores.get(mid) ?? 0) + 1.0 / (k + rank + 1));
+        }
+    }
+    return scores;
+}
+/**
+ * Main retrieval: BM25 + vector search with RRF fusion, graph traversal,
+ * and composite scoring. Strengthens accessed memories.
+ */
+async function retrieve(db, embedFn, query, options) {
+    const limit = options?.limit ?? 5;
+    const project = options?.project;
+    const privacy = options?.privacy;
+    const sourceAgent = options?.sourceAgent;
+    const now = new Date();
+    // 1. Embed
+    const queryEmbedding = await embedFn(query);
+    // 2. Dual retrieval — vector + BM25
+    const fetchLimit = limit * 3;
+    let vecCandidates = storage.vectorSearch(db, queryEmbedding, fetchLimit, []);
+    let ftsCandidates = [];
+    try {
+        ftsCandidates = storage.searchFts(db, query, fetchLimit, privacy, sourceAgent);
+    }
+    catch {
+        // FTS5 search can fail on special characters; fall back to vector-only
+    }
+    if (vecCandidates.length === 0 && ftsCandidates.length === 0) {
+        return [];
+    }
+    // Post-filter vector candidates (sqlite-vec can't filter)
+    if (project) {
+        vecCandidates = vecCandidates.filter((c) => c.project === project);
+        ftsCandidates = ftsCandidates.filter((c) => c.project === project);
+    }
+    if (privacy) {
+        vecCandidates = vecCandidates.filter((c) => privacy.includes(c.privacy));
+    }
+    if (sourceAgent) {
+        vecCandidates = vecCandidates.filter((c) => c.source_agent === sourceAgent);
+    }
+    // 3. RRF fusion
+    const vecRanked = vecCandidates.map((c) => c.id);
+    const ftsRanked = ftsCandidates.map((c) => c.id);
+    const rrfScores = reciprocalRankFusion([vecRanked, ftsRanked]);
+    // Build unified candidate map
+    const candidateMap = new Map();
+    for (const c of vecCandidates) {
+        candidateMap.set(c.id, { mem: c, distance: c.distance });
+    }
+    for (const c of ftsCandidates) {
+        if (!candidateMap.has(c.id)) {
+            candidateMap.set(c.id, { mem: c, distance: DEFAULT_GRAPH_DISTANCE });
+        }
+    }
+    // 4. Graph traversal
+    const seedIds = [...candidateMap.keys()];
+    const connectionCounts = collectLinks(db, seedIds, 2);
+    // Pull in graph-discovered memories not in the candidate set
+    const graphIds = [...connectionCounts.keys()].filter((mid) => !candidateMap.has(mid));
+    for (const gid of graphIds) {
+        const mem = storage.getMemory(db, gid);
+        if (!mem)
+            continue;
+        if (project && mem.project !== project)
+            continue;
+        if (privacy && !privacy.includes(mem.privacy))
+            continue;
+        if (sourceAgent && mem.source_agent !== sourceAgent)
+            continue;
+        candidateMap.set(gid, { mem, distance: DEFAULT_GRAPH_DISTANCE });
+    }
+    // 5. Compute composite scores
+    const maxConnections = Math.max(...([...connectionCounts.values()].length > 0
+        ? [...connectionCounts.values()]
+        : [0]));
+    const scored = [];
+    const maxRrf = Math.max(...([...rrfScores.values()].length > 0 ? [...rrfScores.values()] : [1]));
+    for (const [mid, { mem, distance }] of candidateMap) {
+        const connCount = connectionCounts.get(mid) ?? 0;
+        const composite = computeScore(mem, distance, connCount, maxConnections, now);
+        const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
+            accessCount: mem.access_count ?? 0,
+            linkCount: connectionCounts.get(mem.id) ?? 0,
+        });
+        const rrf = rrfScores.get(mid) ?? 0;
+        const normalizedRrf = maxRrf > 0 ? rrf / maxRrf : 0;
+        const finalScore = composite * 0.8 + normalizedRrf * 0.2;
+        scored.push({ mem, finalScore, effStr, connCount });
+    }
+    // 6. Sort and take top N
+    scored.sort((a, b) => b.finalScore - a.finalScore);
+    const top = scored.slice(0, limit);
+    const results = top.map((t) => formatResult(t.mem, t.finalScore, t.effStr, t.connCount));
+    // 7. Strengthen
+    strengthen(db, top.map((t) => t.mem), now);
+    return results;
+}
+/**
+ * Get recent context, optionally filtered by project and privacy.
+ */
+function searchContext(db, options) {
+    const limit = options?.limit ?? 10;
+    const project = options?.project;
+    const privacy = options?.privacy;
+    const now = new Date();
+    let candidates = storage.getRecentMemories(db, 30, limit * 3);
+    if (project) {
+        candidates = candidates.filter((c) => c.project === project);
+    }
+    if (privacy) {
+        candidates = candidates.filter((c) => privacy.includes(c.privacy));
+    }
+    if (candidates.length === 0)
+        return [];
+    const allIds = candidates.map((c) => c.id);
+    const connectionCounts = collectLinks(db, allIds, 1);
+    const maxConnections = Math.max(...([...connectionCounts.values()].length > 0
+        ? [...connectionCounts.values()]
+        : [0]));
+    const scored = [];
+    for (const mem of candidates) {
+        const connCount = connectionCounts.get(mem.id) ?? 0;
+        const score = computeScore(mem, DEFAULT_GRAPH_DISTANCE, connCount, maxConnections, now);
+        const effStr = effectiveStrength(mem.base_strength ?? 0.5, mem.last_accessed, now, {
+            accessCount: mem.access_count ?? 0,
+            linkCount: connectionCounts.get(mem.id) ?? 0,
+        });
+        scored.push({ mem, score, effStr, connCount });
+    }
+    scored.sort((a, b) => b.score - a.score);
+    const top = scored.slice(0, limit);
+    const results = top.map((t) => formatResult(t.mem, t.score, t.effStr, t.connCount));
+    strengthen(db, top.map((t) => t.mem), now);
+    return results;
+}

package/dist/storage.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Storage layer — CRUD operations for the SQLite + sqlite-vec database.
+ * Ported from hicortex/storage.py. All functions are synchronous (better-sqlite3).
+ */
+import type Database from "better-sqlite3";
+import type { Memory, MemoryLink, InsertMemoryOptions } from "./types.js";
+/**
+ * Serialize a Float32Array embedding to a Buffer for sqlite-vec.
+ */
+export declare function embedToBlob(embedding: Float32Array): Buffer;
+/**
+ * Insert a memory and its vector embedding. Returns the generated UUID.
+ */
+export declare function insertMemory(db: Database.Database, content: string, embedding: Float32Array, opts?: InsertMemoryOptions): string;
+/**
+ * Get a single memory by ID. Returns null if not found.
+ */
+export declare function getMemory(db: Database.Database, memoryId: string): Memory | null;
+/**
+ * Update specific fields on a memory.
+ */
+export declare function updateMemory(db: Database.Database, memoryId: string, fields: Record<string, unknown>): void;
+/**
+ * Atomically increment access_count and reset last_accessed.
+ */
+export declare function strengthenMemory(db: Database.Database, memoryId: string, nowIsoStr: string): void;
+/**
+ * Delete a memory, its vector, and all its links.
+ */
+export declare function deleteMemory(db: Database.Database, memoryId: string): void;
+/**
+ * Find similar memories by vector distance. Returns memories with distance field.
+ */
+export declare function vectorSearch(db: Database.Database, queryEmbedding: Float32Array, limit?: number, excludeIds?: string[]): Array<Memory & {
+    distance: number;
+}>;
+/**
+ * Full-text search using FTS5 BM25 ranking.
+ * Returns memories with a rank field (lower is better).
+ */
+export declare function searchFts(db: Database.Database, query: string, limit?: number, privacy?: string[], sourceAgent?: string): Array<Memory & {
+    rank: number;
+}>;
+/**
+ * Create a link between two memories.
+ */
+export declare function addLink(db: Database.Database, sourceId: string, targetId: string, relationship: string, strength?: number): void;
+/**
+ * Get links for a memory. direction: 'outgoing', 'incoming', or 'both'.
+ */
+export declare function getLinks(db: Database.Database, memoryId: string, direction?: "outgoing" | "incoming" | "both"): MemoryLink[];
+/**
+ * Delete all links involving a memory.
+ */
+export declare function deleteLinks(db: Database.Database, memoryId: string): void;
+/**
+ * Batch insert memories. Returns count inserted.
+ */
+export declare function insertMemoriesBatch(db: Database.Database, memories: Array<{
+    content: string;
+    embedding: Float32Array;
+    sourceAgent?: string;
+    sourceSession?: string | null;
+    project?: string | null;
+    privacy?: string;
+    memoryType?: string;
+    baseStrength?: number;
+}>): number;
+/**
+ * Return total memory count.
+ */
+export declare function countMemories(db: Database.Database): number;
+/**
+ * Get memories created in the last N days, newest first.
+ */
+export declare function getRecentMemories(db: Database.Database, days?: number, limit?: number): Memory[];
+/**
+ * Get all memories ingested after a timestamp.
+ * Uses ingested_at (when the memory entered the DB) for consolidation correctness.
+ */
+export declare function getMemoriesSince(db: Database.Database, sinceIso: string): Memory[];
+/**
+ * Get lesson-type memories from the last N days.
+ */
+export declare function getLessons(db: Database.Database, days?: number, project?: string | null): Memory[];
+/**
+ * Get memories older than cutoff with zero access (prune candidates).
+ */
+export declare function getPruneCandidates(db: Database.Database, cutoffIso: string): Memory[];
+/**
+ * Get link counts for all memories in a single query.
+ * Returns a map of memory_id -> total link count (both directions).
+ */
+export declare function getAllLinkCounts(db: Database.Database): Map<string, number>;
+/**
+ * Get all memories with default base_strength (never scored).
+ */
+export declare function getUnscoredMemories(db: Database.Database): Memory[];