@gamaze/hicortex 0.2.0

package/README.md

@@ -0,0 +1,119 @@
+# @gamaze/hicortex — Long-term Memory for OpenClaw Agents
+
+Your agents remember past decisions, avoid repeated mistakes, and get smarter every day.
+
+Hicortex captures session activity, distills knowledge nightly, scores importance, generates actionable lessons via reflection, and automatically injects them into agent context. No manual intervention needed after install.
+
+## Requirements
+
+- OpenClaw gateway (Node.js 18+)
+- LLM provider configured in OpenClaw (auto-detected, 20+ providers supported)
+- ~500MB disk for database + embedding model
+
+## Install
+
+```bash
+openclaw plugins install @gamaze/hicortex
+```
+
+## Configure
+
+No configuration needed for most users. The plugin auto-detects your LLM provider from OpenClaw settings on first startup.
+
+Optional config (add to plugin entry in `~/.openclaw/openclaw.json`):
+
+| Field | Default | Description |
+|-------|---------|-------------|
+| `licenseKey` | _(none)_ | License key. Free tier (250 memories) without key. |
+| `llmBaseUrl` | _(auto)_ | Override LLM base URL (auto-detected from OC config) |
+| `llmApiKey` | _(auto)_ | Override LLM API key (auto-detected from OC auth) |
+| `llmModel` | _(auto)_ | Override model for scoring and distillation |
+| `reflectModel` | _(auto)_ | Override model for nightly reflection |
+| `consolidateHour` | `2` | Hour (0-23, local time) for nightly consolidation |
+| `dbPath` | _(auto)_ | Custom database path |
+
+Restart the gateway after config changes:
+
+```bash
+openclaw gateway restart
+```
+
+## What Happens Automatically
+
+| When | What | How |
+|------|------|-----|
+| Agent start | Recent lessons injected into context | `before_agent_start` hook |
+| Agent end | Conversation captured and distilled | `agent_end` hook + LLM |
+| Nightly | Score importance, reflect, link, decay | In-process consolidation pipeline |
+
+## Agent Tools
+
+The plugin registers these tools for agents to use:
+
+- **hicortex_search** — Semantic search across all stored knowledge
+- **hicortex_context** — Get recent decisions and project state
+- **hicortex_ingest** — Store a piece of knowledge directly
+- **hicortex_lessons** — Get actionable lessons from reflection
+
+Skills: `/learn` to save explicit learnings.
+
+## Architecture
+
+```
+OpenClaw Gateway (Node.js)
+  └── @gamaze/hicortex plugin (TypeScript, in-process)
+      ├── before_agent_start → inject lessons into agent context
+      ├── agent_end → capture conversation, distill via LLM
+      ├── session_end → check if consolidation overdue
+      ├── registerTool → hicortex_search, hicortex_context, hicortex_ingest, hicortex_lessons
+      └── registerService → DB init, LLM auto-config, nightly consolidation
+          ├── SQLite + sqlite-vec + FTS5 (single file, in-process)
+          ├── bge-small-en-v1.5 embeddings (ONNX, local CPU)
+          └── BM25 + vector search with RRF fusion
+```
+
+No sidecar, no HTTP server, no separate process. Everything runs inside the gateway.
+
+## Pricing
+
+| Tier | Price | Memories | Features |
+|------|-------|----------|----------|
+| Free | $0 | 250 | Full features: search, reflection, lessons, linking |
+| Pro | $9/month | Unlimited | Everything in Free, unlimited |
+| Lifetime | $149 | Unlimited | Pro forever |
+| Team | $29/month | Unlimited | Multi-agent shared memory |
+
+Get a license key at [hicortex.gamaze.com](https://hicortex.gamaze.com).
+
+## Uninstall
+
+```bash
+openclaw plugins uninstall hicortex
+```
+
+Your memory database is preserved by default. To remove it: delete `~/.openclaw/data/hicortex.db`.
+
+## Development
+
+```bash
+# Local dev install
+openclaw plugins install -l ./packages/openclaw-plugin
+
+# Build
+cd packages/openclaw-plugin
+npm install
+npm run build
+
+# Test
+npm test
+```
+
+## Troubleshooting
+
+**Tools not visible to agent:** The plugin auto-adds tools to `tools.allow` on startup. If tools still don't appear, check that the gateway was restarted after install.
+
+**LLM auto-config failed:** Check the gateway log for `[hicortex] WARNING`. You may need to add `llmBaseUrl` to the plugin config manually for non-standard providers.
+
+**No lessons generated:** Reflection requires an LLM. Check that your LLM provider is accessible and has sufficient quota.
+
+**First startup slow:** The embedding model (~130MB) downloads on first run. Allow up to 2 minutes.

package/dist/consolidate.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Nightly consolidation pipeline — importance scoring, reflection,
+ * link discovery, decay/prune.
+ * Ported from hicortex/consolidate/ (stages.py, __init__.py, budget.py).
+ */
+import type Database from "better-sqlite3";
+import type { ConsolidationReport } from "./types.js";
+import type { LlmClient } from "./llm.js";
+import type { EmbedFn } from "./retrieval.js";
+export declare class BudgetTracker {
+    maxCalls: number;
+    callsUsed: number;
+    callsByStage: Record<string, number>;
+    constructor(maxCalls: number);
+    get exhausted(): boolean;
+    get remaining(): number;
+    use(stage: string, count?: number): boolean;
+    summary(): NonNullable<ConsolidationReport["budget"]>;
+}
+/**
+ * Parse JSON from LLM output, tolerating markdown fences and indexed formats.
+ */
+export declare function parseJsonLenient<T>(text: string, fallback: T): T;
+/**
+ * Run the full consolidation pipeline. Returns a structured report.
+ */
+export declare function runConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, dryRun?: boolean): Promise<ConsolidationReport>;
+/**
+ * Calculate milliseconds until the next occurrence of a given hour (local time).
+ */
+export declare function msUntilHour(hour: number): number;
+/**
+ * Schedule the consolidation pipeline to run nightly.
+ * Returns a cleanup function to cancel the timer.
+ */
+export declare function scheduleConsolidation(db: Database.Database, llm: LlmClient, embedFn: EmbedFn, hour?: number): () => void;

package/dist/consolidate.js

@@ -0,0 +1,482 @@
+"use strict";
+/**
+ * Nightly consolidation pipeline — importance scoring, reflection,
+ * link discovery, decay/prune.
+ * Ported from hicortex/consolidate/ (stages.py, __init__.py, budget.py).
+ */
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.BudgetTracker = void 0;
+exports.parseJsonLenient = parseJsonLenient;
+exports.runConsolidation = runConsolidation;
+exports.msUntilHour = msUntilHour;
+exports.scheduleConsolidation = scheduleConsolidation;
+const node_fs_1 = require("node:fs");
+const node_path_1 = require("node:path");
+const node_os_1 = require("node:os");
+const retrieval_js_1 = require("./retrieval.js");
+const storage = __importStar(require("./storage.js"));
+const prompts_js_1 = require("./prompts.js");
+const LAST_CONSOLIDATED_PATH = (0, node_path_1.join)((0, node_os_1.homedir)(), ".claude", "memory", "last-consolidated.txt");
+// Default config constants (matching Python config.py)
+const CONSOLIDATE_MAX_LLM_CALLS = 200;
+const CONSOLIDATE_PRUNE_MIN_AGE_DAYS = 90;
+const CONSOLIDATE_LINK_THRESHOLD = 0.55;
+// ---------------------------------------------------------------------------
+// BudgetTracker
+// ---------------------------------------------------------------------------
+class BudgetTracker {
+    maxCalls;
+    callsUsed = 0;
+    callsByStage = {};
+    constructor(maxCalls) {
+        this.maxCalls = maxCalls;
+    }
+    get exhausted() {
+        return this.callsUsed >= this.maxCalls;
+    }
+    get remaining() {
+        return Math.max(0, this.maxCalls - this.callsUsed);
+    }
+    use(stage, count = 1) {
+        if (this.callsUsed + count > this.maxCalls) {
+            console.warn(`[hicortex] Budget exhausted: ${this.callsUsed}/${this.maxCalls} used, ` +
+                `requested ${count} more (stage: ${stage})`);
+            return false;
+        }
+        this.callsUsed += count;
+        this.callsByStage[stage] = (this.callsByStage[stage] ?? 0) + count;
+        return true;
+    }
+    summary() {
+        return {
+            max_calls: this.maxCalls,
+            calls_used: this.callsUsed,
+            calls_remaining: this.remaining,
+            calls_by_stage: { ...this.callsByStage },
+        };
+    }
+}
+exports.BudgetTracker = BudgetTracker;
+// ---------------------------------------------------------------------------
+// JSON parsing helper
+// ---------------------------------------------------------------------------
+/**
+ * Parse JSON from LLM output, tolerating markdown fences and indexed formats.
+ */
+function parseJsonLenient(text, fallback) {
+    text = text.trim();
+    // Strip markdown code fences
+    if (text.startsWith("```")) {
+        const lines = text.split("\n");
+        const stripped = lines.slice(1);
+        if (stripped.length > 0 && stripped[stripped.length - 1].trim() === "```") {
+            stripped.pop();
+        }
+        text = stripped.join("\n").trim();
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch {
+        // Ignore
+    }
+    // Handle "[0] 0.7\n[1] 0.6\n..." format
+    const indexed = [...text.matchAll(/\[\d+\]\s*([\d.]+)/g)];
+    if (indexed.length > 0) {
+        try {
+            return indexed.map((m) => parseFloat(m[1]));
+        }
+        catch {
+            // Ignore
+        }
+    }
+    console.warn(`[hicortex] Failed to parse LLM output: ${text.slice(0, 200)}`);
+    return fallback;
+}
+// ---------------------------------------------------------------------------
+// Stage 1: Pre-check
+// ---------------------------------------------------------------------------
+function readLastConsolidated() {
+    try {
+        return (0, node_fs_1.readFileSync)(LAST_CONSOLIDATED_PATH, "utf-8").trim();
+    }
+    catch {
+        return "";
+    }
+}
+function stagePrecheck(db) {
+    const lastTs = readLastConsolidated();
+    const lastDt = lastTs || "1970-01-01T00:00:00.000Z";
+    const newMemories = storage.getMemoriesSince(db, lastDt);
+    if (newMemories.length === 0) {
+        return {
+            skip: true,
+            reason: "No new memories since last consolidation",
+            newMemories: [],
+            lastDt,
+        };
+    }
+    return {
+        skip: false,
+        reason: `${newMemories.length} new memories found`,
+        newMemories,
+        lastDt,
+    };
+}
+// ---------------------------------------------------------------------------
+// Stage 2: Importance Scoring
+// ---------------------------------------------------------------------------
+async function stageImportance(db, memories, llm, budget, dryRun) {
+    const batchSize = 10;
+    let scored = 0;
+    let failed = 0;
+    let skippedBudget = 0;
+    for (let i = 0; i < memories.length; i += batchSize) {
+        if (budget.exhausted) {
+            skippedBudget += memories.length - i;
+            break;
+        }
+        const batch = memories.slice(i, i + batchSize);
+        const lines = batch.map((mem, idx) => `[${idx}] ${mem.content.slice(0, 500)}`);
+        const memoriesBlock = lines.join("\n\n");
+        const prompt = (0, prompts_js_1.importanceScoring)(memoriesBlock);
+        if (dryRun)
+            continue;
+        if (!budget.use("importance")) {
+            skippedBudget += memories.length - i;
+            break;
+        }
+        try {
+            const raw = await llm.completeFast(prompt, 256);
+            let scores = parseJsonLenient(raw, null);
+            if (!Array.isArray(scores)) {
+                scores = new Array(batch.length).fill(0.5);
+            }
+            while (scores.length < batch.length)
+                scores.push(0.5);
+            scores = scores.slice(0, batch.length);
+            for (let j = 0; j < batch.length; j++) {
+                let scoreVal = 0.5;
+                try {
+                    scoreVal = Math.max(0, Math.min(1, Number(scores[j])));
+                    if (isNaN(scoreVal))
+                        scoreVal = 0.5;
+                }
+                catch {
+                    scoreVal = 0.5;
+                }
+                try {
+                    storage.updateMemory(db, batch[j].id, { base_strength: scoreVal });
+                    scored++;
+                }
+                catch {
+                    failed++;
+                }
+            }
+        }
+        catch {
+            failed += batch.length;
+        }
+    }
+    return { scored, failed, skipped_budget: skippedBudget };
+}
+// ---------------------------------------------------------------------------
+// Stage 2.5: Reflection
+// ---------------------------------------------------------------------------
+async function stageReflection(db, memories, llm, budget, embedFn, dryRun) {
+    if (memories.length === 0) {
+        return { lessons_generated: 0, skipped: true, reason: "no memories" };
+    }
+    // Build summary
+    const lines = memories.slice(0, 50).map((mem) => {
+        const project = mem.project ?? "unknown";
+        const agent = mem.source_agent ?? "unknown";
+        const content = mem.content.slice(0, 300);
+        return `[${project}] [${agent}] ${content}`;
+    });
+    const memoriesBlock = lines.join("\n\n");
+    const prompt = (0, prompts_js_1.reflection)(memoriesBlock);
+    if (dryRun) {
+        return { lessons_generated: 0, skipped: false };
+    }
+    if (!budget.use("reflection")) {
+        return { lessons_generated: 0, skipped: true, reason: "budget_exhausted" };
+    }
+    try {
+        const raw = await llm.completeReflect(prompt, 2048);
+        const lessons = parseJsonLenient(raw, []);
+        if (!Array.isArray(lessons)) {
+            return { lessons_generated: 0, failed: true };
+        }
+        let generated = 0;
+        for (const lessonObj of lessons) {
+            if (typeof lessonObj !== "object" || lessonObj === null)
+                continue;
+            const lo = lessonObj;
+            const lessonText = String(lo.lesson ?? "");
+            if (!lessonText)
+                continue;
+            const project = String(lo.project ?? "global");
+            const severity = String(lo.severity ?? "important");
+            const confidence = String(lo.confidence ?? "medium");
+            const sourcePattern = String(lo.source_pattern ?? "");
+            let content = `## Lesson: ${lessonText}\n\n`;
+            content += `**Severity:** ${severity}\n`;
+            content += `**Confidence:** ${confidence}\n`;
+            if (sourcePattern)
+                content += `**Pattern:** ${sourcePattern}\n`;
+            content += `**Generated:** ${new Date().toISOString().slice(0, 10)}`;
+            const baseStrength = {
+                critical: 0.95,
+                important: 0.8,
+                minor: 0.6,
+            };
+            try {
+                // Check memory cap before storing lesson
+                const { getFeatures } = await import("./license.js");
+                const features = getFeatures("");
+                if (features.maxMemories > 0 && storage.countMemories(db) >= features.maxMemories) {
+                    console.log(`[hicortex] Free tier limit (${features.maxMemories} memories). ` +
+                        `Existing memories and lessons still work. New lessons won't be saved. ` +
+                        `Upgrade for unlimited usage: https://hicortex.gamaze.com/`);
+                    break;
+                }
+                const embedding = await embedFn(content);
+                storage.insertMemory(db, content, embedding, {
+                    sourceAgent: "hicortex/reflection",
+                    project,
+                    memoryType: "lesson",
+                    baseStrength: baseStrength[severity] ?? 0.8,
+                    privacy: "WORK",
+                });
+                generated++;
+            }
+            catch {
+                // Failed to store lesson
+            }
+        }
+        return { lessons_generated: generated, failed: false };
+    }
+    catch {
+        return { lessons_generated: 0, failed: true };
+    }
+}
+// ---------------------------------------------------------------------------
+// Stage 3: Link Discovery (vector similarity auto-link)
+// ---------------------------------------------------------------------------
+async function stageLinks(db, memories, embedFn, dryRun) {
+    let autoLinked = 0;
+    let failed = 0;
+    for (const mem of memories) {
+        try {
+            const embedding = await embedFn(mem.content);
+            const neighbors = storage.vectorSearch(db, embedding, 10, [mem.id]);
+            for (const neighbor of neighbors) {
+                const similarity = 1.0 - neighbor.distance;
+                if (similarity > CONSOLIDATE_LINK_THRESHOLD) {
+                    if (!dryRun) {
+                        try {
+                            storage.addLink(db, mem.id, neighbor.id, "relates_to", similarity);
+                            autoLinked++;
+                        }
+                        catch {
+                            failed++;
+                        }
+                    }
+                    else {
+                        autoLinked++;
+                    }
+                }
+            }
+        }
+        catch {
+            failed++;
+        }
+    }
+    return { auto_linked: autoLinked, failed };
+}
+// ---------------------------------------------------------------------------
+// Stage 4: Decay & Prune
+// ---------------------------------------------------------------------------
+function stageDecayPrune(db, dryRun) {
+    const now = new Date();
+    const cutoff = new Date(now.getTime() - CONSOLIDATE_PRUNE_MIN_AGE_DAYS * 24 * 60 * 60 * 1000);
+    const oldUnaccessed = storage.getPruneCandidates(db, cutoff.toISOString());
+    const linkCounts = storage.getAllLinkCounts(db);
+    let candidates = 0;
+    let pruned = 0;
+    let failed = 0;
+    for (const mem of oldUnaccessed) {
+        const memLinkCount = linkCounts.get(mem.id) ?? 0;
+        const eff = (0, retrieval_js_1.effectiveStrength)(mem.base_strength ?? 0.5, mem.last_accessed, now, {
+            accessCount: 0,
+            linkCount: memLinkCount,
+        });
+        if (eff >= 0.01)
+            continue;
+        candidates++;
+        if (dryRun)
+            continue;
+        try {
+            storage.deleteMemory(db, mem.id);
+            pruned++;
+        }
+        catch {
+            failed++;
+        }
+    }
+    return { candidates, pruned, failed };
+}
+// ---------------------------------------------------------------------------
+// Full pipeline
+// ---------------------------------------------------------------------------
+/**
+ * Run the full consolidation pipeline. Returns a structured report.
+ */
+async function runConsolidation(db, llm, embedFn, dryRun = false) {
+    const start = new Date();
+    const report = {
+        started_at: start.toISOString(),
+        dry_run: dryRun,
+        status: "completed",
+        stages: {},
+    };
+    // Stage 1: Pre-check
+    const precheck = stagePrecheck(db);
+    // Also check for unscored memories
+    const unscored = storage.getUnscoredMemories(db);
+    const newIds = new Set(precheck.newMemories.map((m) => m.id));
+    const scoreMemories = [
+        ...precheck.newMemories,
+        ...unscored.filter((m) => !newIds.has(m.id)),
+    ];
+    const skip = scoreMemories.length === 0;
+    report.stages.precheck = {
+        skip,
+        reason: skip
+            ? precheck.reason
+            : `${precheck.newMemories.length} new + ${scoreMemories.length - precheck.newMemories.length} unscored memories`,
+        new_memory_count: precheck.newMemories.length,
+        unscored_count: scoreMemories.length - precheck.newMemories.length,
+    };
+    if (skip) {
+        report.status = "skipped";
+        report.completed_at = new Date().toISOString();
+        return report;
+    }
+    const budget = new BudgetTracker(CONSOLIDATE_MAX_LLM_CALLS);
+    try {
+        // Stage 2: Importance Scoring
+        report.stages.importance = await stageImportance(db, scoreMemories, llm, budget, dryRun);
+        // Stage 2.5: Reflection
+        report.stages.reflection = await stageReflection(db, precheck.newMemories, llm, budget, embedFn, dryRun);
+        // Stage 3: Link Discovery
+        report.stages.links = await stageLinks(db, precheck.newMemories, embedFn, dryRun);
+        // Stage 4: Decay & Prune
+        report.stages.decay_prune = stageDecayPrune(db, dryRun);
+    }
+    catch (err) {
+        report.status = "failed";
+        console.error("[hicortex] Consolidation pipeline error:", err);
+    }
+    // Update last-consolidated timestamp
+    if (!dryRun && report.status === "completed") {
+        try {
+            const dir = (0, node_path_1.join)((0, node_os_1.homedir)(), ".claude", "memory");
+            (0, node_fs_1.mkdirSync)(dir, { recursive: true });
+            (0, node_fs_1.writeFileSync)(LAST_CONSOLIDATED_PATH, new Date().toISOString());
+        }
+        catch {
+            // Non-fatal
+        }
+    }
+    report.budget = budget.summary();
+    report.completed_at = new Date().toISOString();
+    report.elapsed_seconds =
+        Math.round((Date.now() - start.getTime()) / 100) / 10;
+    return report;
+}
+// ---------------------------------------------------------------------------
+// Scheduling
+// ---------------------------------------------------------------------------
+/**
+ * Calculate milliseconds until the next occurrence of a given hour (local time).
+ */
+function msUntilHour(hour) {
+    const now = new Date();
+    const target = new Date(now);
+    target.setHours(hour, 30, 0, 0); // :30 past the hour
+    if (target.getTime() <= now.getTime()) {
+        // Already passed today, schedule for tomorrow
+        target.setDate(target.getDate() + 1);
+    }
+    return target.getTime() - now.getTime();
+}
+const ONE_DAY_MS = 24 * 60 * 60 * 1000;
+/**
+ * Schedule the consolidation pipeline to run nightly.
+ * Returns a cleanup function to cancel the timer.
+ */
+function scheduleConsolidation(db, llm, embedFn, hour = 2) {
+    let timeout = null;
+    let interval = null;
+    const runAndScheduleInterval = () => {
+        runConsolidation(db, llm, embedFn)
+            .then((report) => {
+            console.log(`[hicortex] Consolidation ${report.status} in ${report.elapsed_seconds}s`);
+        })
+            .catch((err) => {
+            console.error("[hicortex] Consolidation failed:", err);
+        });
+        // Schedule recurring daily runs
+        if (!interval) {
+            interval = setInterval(() => {
+                runConsolidation(db, llm, embedFn).catch((err) => {
+                    console.error("[hicortex] Consolidation failed:", err);
+                });
+            }, ONE_DAY_MS);
+        }
+    };
+    const delay = msUntilHour(hour);
+    console.log(`[hicortex] Consolidation scheduled in ${Math.round(delay / 60_000)} minutes`);
+    timeout = setTimeout(runAndScheduleInterval, delay);
+    return () => {
+        if (timeout)
+            clearTimeout(timeout);
+        if (interval)
+            clearInterval(interval);
+    };
+}

package/dist/db.d.ts

@@ -0,0 +1,19 @@
+/**
+ * Database initialization with better-sqlite3 + sqlite-vec.
+ * Ported from hicortex/db.py — same schema for migration compatibility.
+ */
+import Database from "better-sqlite3";
+/**
+ * Initialize the database: load sqlite-vec, enable WAL, create all tables.
+ * Returns the open Database instance (caller manages lifetime).
+ */
+export declare function initDb(dbPath: string): Database.Database;
+/**
+ * Return database statistics.
+ */
+export declare function getStats(db: Database.Database, dbPath: string): {
+    memories: number;
+    links: number;
+    db_size_bytes: number;
+    by_type: Record<string, number>;
+};