@gamaze/hicortex 0.2.0

package/dist/db.js

@@ -0,0 +1,140 @@
+"use strict";
+/**
+ * Database initialization with better-sqlite3 + sqlite-vec.
+ * Ported from hicortex/db.py — same schema for migration compatibility.
+ */
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.initDb = initDb;
+exports.getStats = getStats;
+const better_sqlite3_1 = __importDefault(require("better-sqlite3"));
+const node_fs_1 = require("node:fs");
+const EMBEDDING_DIMENSIONS = 384;
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS memories (
+    id TEXT PRIMARY KEY,
+    content TEXT NOT NULL,
+
+    -- Decay & Strengthening
+    base_strength REAL DEFAULT 0.5,
+    last_accessed TIMESTAMP,
+    access_count INTEGER DEFAULT 0,
+    created_at TIMESTAMP NOT NULL,
+    ingested_at TIMESTAMP NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%f+00:00', 'now')),
+
+    -- Classification
+    source_agent TEXT DEFAULT 'default',
+    source_session TEXT,
+    project TEXT,
+    privacy TEXT DEFAULT 'WORK',
+    memory_type TEXT DEFAULT 'episode'
+);
+
+CREATE TABLE IF NOT EXISTS memory_links (
+    source_id TEXT NOT NULL,
+    target_id TEXT NOT NULL,
+    relationship TEXT NOT NULL,
+    strength REAL DEFAULT 0.5,
+    created_at TIMESTAMP NOT NULL,
+    PRIMARY KEY (source_id, target_id),
+    FOREIGN KEY (source_id) REFERENCES memories(id),
+    FOREIGN KEY (target_id) REFERENCES memories(id)
+);
+
+CREATE INDEX IF NOT EXISTS idx_memories_project ON memories(project);
+CREATE INDEX IF NOT EXISTS idx_memories_type ON memories(memory_type);
+CREATE INDEX IF NOT EXISTS idx_memories_created ON memories(created_at);
+CREATE INDEX IF NOT EXISTS idx_links_source ON memory_links(source_id);
+CREATE INDEX IF NOT EXISTS idx_links_target ON memory_links(target_id);
+`;
+const FTS_SCHEMA = `
+CREATE VIRTUAL TABLE IF NOT EXISTS memories_fts USING fts5(
+    content,
+    content_rowid='rowid'
+);
+
+CREATE TRIGGER IF NOT EXISTS memories_fts_insert AFTER INSERT ON memories
+BEGIN
+    INSERT INTO memories_fts (rowid, content) VALUES (NEW.rowid, NEW.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_fts_update AFTER UPDATE OF content ON memories
+BEGIN
+    UPDATE memories_fts SET content = NEW.content WHERE rowid = NEW.rowid;
+END;
+
+CREATE TRIGGER IF NOT EXISTS memories_fts_delete AFTER DELETE ON memories
+BEGIN
+    DELETE FROM memories_fts WHERE rowid = OLD.rowid;
+END;
+`;
+const VEC_SCHEMA = `
+CREATE VIRTUAL TABLE IF NOT EXISTS memory_vectors USING vec0(
+    id TEXT PRIMARY KEY,
+    embedding float[${EMBEDDING_DIMENSIONS}]
+);
+`;
+/**
+ * Initialize the database: load sqlite-vec, enable WAL, create all tables.
+ * Returns the open Database instance (caller manages lifetime).
+ */
+function initDb(dbPath) {
+    const db = new better_sqlite3_1.default(dbPath);
+    // Load sqlite-vec extension
+    // eslint-disable-next-line @typescript-eslint/no-var-requires
+    const sqliteVec = require("sqlite-vec");
+    sqliteVec.load(db);
+    // Pragmas
+    db.pragma("journal_mode = WAL");
+    db.pragma("foreign_keys = ON");
+    // Create core tables and indexes
+    db.exec(SCHEMA);
+    // Create FTS5 virtual table and sync triggers
+    db.exec(FTS_SCHEMA);
+    // Create vec0 virtual table
+    db.exec(VEC_SCHEMA);
+    // Run migrations for existing databases
+    migrate(db);
+    return db;
+}
+/**
+ * Apply schema migrations for existing databases.
+ */
+function migrate(db) {
+    const cols = db.pragma("table_info(memories)");
+    const colNames = new Set(cols.map((c) => c.name));
+    if (!colNames.has("ingested_at")) {
+        db.exec("ALTER TABLE memories ADD COLUMN ingested_at TIMESTAMP");
+        db.exec("UPDATE memories SET ingested_at = created_at");
+        db.exec("CREATE INDEX IF NOT EXISTS idx_memories_ingested ON memories(ingested_at)");
+    }
+}
+/**
+ * Return database statistics.
+ */
+function getStats(db, dbPath) {
+    const memoryCount = db.prepare("SELECT count(*) as cnt FROM memories").get().cnt;
+    const linkCount = db.prepare("SELECT count(*) as cnt FROM memory_links").get().cnt;
+    let dbSize = 0;
+    try {
+        dbSize = (0, node_fs_1.statSync)(dbPath).size;
+    }
+    catch {
+        // File may not exist yet
+    }
+    const typeCounts = {};
+    const rows = db
+        .prepare("SELECT memory_type, count(*) as cnt FROM memories GROUP BY memory_type")
+        .all();
+    for (const row of rows) {
+        typeCounts[row.memory_type] = row.cnt;
+    }
+    return {
+        memories: memoryCount,
+        links: linkCount,
+        db_size_bytes: dbSize,
+        by_type: typeCounts,
+    };
+}

package/dist/distiller.d.ts

@@ -0,0 +1,15 @@
+/**
+ * Session knowledge extraction (distillation).
+ * Simplified from hicortex/distiller.py — messages come from agent_end hook,
+ * not from filesystem scanning.
+ */
+import type { LlmClient } from "./llm.js";
+/**
+ * Convert OpenClaw hook messages to a filtered transcript string.
+ */
+export declare function extractConversationText(messages: unknown[]): string;
+/**
+ * Send filtered conversation to LLM for knowledge extraction.
+ * Returns an array of memory entries to ingest, or empty array if nothing worth extracting.
+ */
+export declare function distillSession(llm: LlmClient, conversation: string, projectName: string, date: string): Promise<string[]>;

package/dist/distiller.js

@@ -0,0 +1,186 @@
+"use strict";
+/**
+ * Session knowledge extraction (distillation).
+ * Simplified from hicortex/distiller.py — messages come from agent_end hook,
+ * not from filesystem scanning.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.extractConversationText = extractConversationText;
+exports.distillSession = distillSession;
+const prompts_js_1 = require("./prompts.js");
+const MAX_TRANSCRIPT_CHARS = 80_000;
+const MIN_CONVERSATION_CHARS = 200;
+// Entry types to skip entirely (from the Python distiller)
+const SKIP_ENTRY_TYPES = new Set([
+    "progress",
+    "system",
+    "file-history-snapshot",
+    "queue-operation",
+    "summary",
+]);
+/**
+ * Extract readable text from a message content value (string or block list).
+ */
+function extractTextFromContent(content) {
+    if (typeof content === "string") {
+        return content.length > 20_000 ? content.slice(0, 20_000) : content;
+    }
+    if (!Array.isArray(content))
+        return "";
+    const texts = [];
+    let totalLen = 0;
+    for (const block of content) {
+        if (typeof block !== "object" || block === null)
+            continue;
+        const btype = block.type;
+        if (btype === "text") {
+            const t = String(block.text ?? "");
+            texts.push(t.length > 10_000 ? t.slice(0, 10_000) : t);
+            totalLen += t.length;
+        }
+        // Skip: tool_use, tool_result, thinking, image blocks
+        if (totalLen > 20_000)
+            break;
+    }
+    return texts.join("\n");
+}
+/**
+ * Strip noise from message text, keep the human conversation.
+ */
+function cleanMessageContent(text) {
+    // Hard cap
+    if (text.length > 50_000) {
+        text = text.slice(0, 50_000);
+    }
+    // Remove large code blocks (>10 lines)
+    const lines = text.split("\n");
+    const cleaned = [];
+    let inCodeBlock = false;
+    let codeBlockLines = 0;
+    let codeBlockStart = 0;
+    for (const line of lines) {
+        if (line.startsWith("```") && !inCodeBlock) {
+            inCodeBlock = true;
+            codeBlockLines = 0;
+            codeBlockStart = cleaned.length;
+            cleaned.push(line);
+        }
+        else if (line.startsWith("```") && inCodeBlock) {
+            inCodeBlock = false;
+            if (codeBlockLines > 10) {
+                cleaned.length = codeBlockStart;
+                cleaned.push("[code block removed]");
+            }
+            else {
+                cleaned.push(line);
+            }
+        }
+        else if (inCodeBlock) {
+            codeBlockLines++;
+            cleaned.push(line);
+        }
+        else {
+            cleaned.push(line);
+        }
+    }
+    if (inCodeBlock && codeBlockLines > 10) {
+        cleaned.length = codeBlockStart;
+        cleaned.push("[code block removed]");
+    }
+    text = cleaned.join("\n");
+    // Remove <system-reminder>...</system-reminder>
+    text = text.replace(/<system-reminder>[^<]{0,10000}<\/system-reminder>/g, "");
+    // Remove file path dumps (Read tool output: "  123->...")
+    text = text.replace(/^\s*\d+\u2192.*$/gm, "");
+    // Remove base64 content
+    text = text.replace(/[A-Za-z0-9+/]{100,}={0,2}/g, "[binary removed]");
+    // Collapse excessive whitespace
+    text = text.replace(/\n{3,}/g, "\n\n");
+    return text.trim();
+}
+/**
+ * Convert OpenClaw hook messages to a filtered transcript string.
+ */
+function extractConversationText(messages) {
+    const parts = [];
+    for (const msg of messages) {
+        if (typeof msg !== "object" || msg === null)
+            continue;
+        const m = msg;
+        // Entry-level filter
+        if (SKIP_ENTRY_TYPES.has(String(m.type ?? "")))
+            continue;
+        if (m.isSidechain)
+            continue;
+        // Extract content — OpenClaw messages have content at top level;
+        // Python distiller format has message.content
+        const content = m.content ?? m.message?.content;
+        if (content === undefined || content === null)
+            continue;
+        let text = extractTextFromContent(content);
+        text = cleanMessageContent(text);
+        if (text.length < 20)
+            continue;
+        const role = m.role === "user" || m.type === "user" ? "USER" : "ASSISTANT";
+        parts.push(`${role}: ${text}`);
+    }
+    return parts.join("\n\n");
+}
+/**
+ * Send filtered conversation to LLM for knowledge extraction.
+ * Returns an array of memory entries to ingest, or empty array if nothing worth extracting.
+ */
+async function distillSession(llm, conversation, projectName, date) {
+    if (conversation.length < MIN_CONVERSATION_CHARS) {
+        return [];
+    }
+    // Truncate if too long
+    let transcript = conversation;
+    if (transcript.length > MAX_TRANSCRIPT_CHARS) {
+        transcript = transcript.slice(0, MAX_TRANSCRIPT_CHARS) + "\n\n[...truncated...]";
+    }
+    const prompt = (0, prompts_js_1.distillation)(projectName, date, transcript);
+    try {
+        const result = await llm.completeDistill(prompt);
+        if (!result)
+            return [];
+        if (result === "NO_EXTRACT" || result.slice(0, 20).includes("NO_EXTRACT")) {
+            return [];
+        }
+        // Split distilled markdown into individual memory entries
+        return parseDistilledEntries(result);
+    }
+    catch (err) {
+        const msg = err instanceof Error ? err.message : String(err);
+        console.error(`[hicortex] Distillation LLM error: ${msg}`);
+        return [];
+    }
+}
+/**
+ * Parse distilled markdown into individual memory entry strings.
+ * Each section item becomes a separate memory.
+ */
+function parseDistilledEntries(markdown) {
+    const entries = [];
+    const lines = markdown.split("\n");
+    let currentSection = "";
+    for (const line of lines) {
+        const trimmed = line.trim();
+        // Section headers
+        if (trimmed.startsWith("### ")) {
+            currentSection = trimmed.slice(4).trim();
+            continue;
+        }
+        // Skip top-level headers and classification
+        if (trimmed.startsWith("# ") || trimmed.startsWith("## "))
+            continue;
+        // Bullet items are individual memories
+        if (trimmed.startsWith("- ") && trimmed.length > 5) {
+            const entry = currentSection
+                ? `[${currentSection}] ${trimmed.slice(2)}`
+                : trimmed.slice(2);
+            entries.push(entry);
+        }
+    }
+    return entries;
+}

package/dist/embedder.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Local embeddings using @huggingface/transformers.
+ * Ported from hicortex/embedder.py — same model (bge-small-en-v1.5, 384-dim).
+ *
+ * Uses dynamic import so the plugin compiles without @huggingface/transformers
+ * installed. The model is lazy-loaded on first call.
+ */
+export declare const EMBEDDING_DIMENSIONS = 384;
+/**
+ * Embed a single text string. Returns a Float32Array of 384 dimensions.
+ */
+export declare function embed(text: string): Promise<Float32Array>;
+/**
+ * Embed multiple texts. Returns an array of Float32Array embeddings.
+ */
+export declare function embedBatch(texts: string[]): Promise<Float32Array[]>;
+/**
+ * Return the embedding dimension count.
+ */
+export declare function dimensions(): number;

package/dist/embedder.js

@@ -0,0 +1,85 @@
+"use strict";
+/**
+ * Local embeddings using @huggingface/transformers.
+ * Ported from hicortex/embedder.py — same model (bge-small-en-v1.5, 384-dim).
+ *
+ * Uses dynamic import so the plugin compiles without @huggingface/transformers
+ * installed. The model is lazy-loaded on first call.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.EMBEDDING_DIMENSIONS = void 0;
+exports.embed = embed;
+exports.embedBatch = embedBatch;
+exports.dimensions = dimensions;
+exports.EMBEDDING_DIMENSIONS = 384;
+const MODEL_NAME = "Xenova/bge-small-en-v1.5";
+// Pipeline is lazy-loaded on first use
+let pipeline = null;
+let initPromise = null;
+/**
+ * Initialize the embedding pipeline (called lazily on first embed call).
+ * Throws with a clear error if @huggingface/transformers is not available.
+ */
+async function ensureInit() {
+    if (pipeline)
+        return;
+    if (initPromise) {
+        await initPromise;
+        return;
+    }
+    initPromise = (async () => {
+        try {
+            // Dynamic import — package may not be installed (it's optional)
+            // eslint-disable-next-line @typescript-eslint/no-require-imports
+            const transformers = await Function('return import("@huggingface/transformers")')();
+            const pipelineFn = transformers.pipeline ?? transformers.default?.pipeline;
+            if (!pipelineFn) {
+                throw new Error("Could not find pipeline function in @huggingface/transformers");
+            }
+            console.log("[hicortex] Loading embedding model (first run downloads ~130MB)...");
+            pipeline = await pipelineFn("feature-extraction", MODEL_NAME, {
+                dtype: "fp32",
+            });
+            console.log("[hicortex] Embedding model ready");
+        }
+        catch (err) {
+            initPromise = null;
+            const msg = err instanceof Error ? err.message : String(err);
+            if (msg.includes("Cannot find module") ||
+                msg.includes("MODULE_NOT_FOUND")) {
+                throw new Error(`@huggingface/transformers is not installed. ` +
+                    `Run: npm install @huggingface/transformers`);
+            }
+            throw err;
+        }
+    })();
+    await initPromise;
+}
+/**
+ * Embed a single text string. Returns a Float32Array of 384 dimensions.
+ */
+async function embed(text) {
+    await ensureInit();
+    const output = await pipeline(text, { pooling: "mean", normalize: true });
+    // output.data is a Float32Array from transformers.js
+    return new Float32Array(output.data);
+}
+/**
+ * Embed multiple texts. Returns an array of Float32Array embeddings.
+ */
+async function embedBatch(texts) {
+    if (texts.length === 0)
+        return [];
+    // Process sequentially to avoid OOM on large batches
+    const results = [];
+    for (const text of texts) {
+        results.push(await embed(text));
+    }
+    return results;
+}
+/**
+ * Return the embedding dimension count.
+ */
+function dimensions() {
+    return exports.EMBEDDING_DIMENSIONS;
+}

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+/**
+ * Hicortex OpenClaw Plugin — Long-term Memory That Learns.
+ *
+ * Pure in-process plugin: no sidecar, no HTTP. Uses better-sqlite3 + sqlite-vec
+ * for storage, @huggingface/transformers for embeddings, and multi-provider LLM
+ * for distillation and consolidation.
+ */
+declare const _default: {
+    id: string;
+    name: string;
+    kind: "lifecycle";
+    register(api: any): void;
+};
+export default _default;