lancedb-opencode-pro 0.1.1

package/dist/store.js

@@ -0,0 +1,283 @@
+import { mkdir } from "node:fs/promises";
+import { dirname } from "node:path";
+import { tokenize } from "./utils.js";
+const TABLE_NAME = "memories";
+export class MemoryStore {
+    dbPath;
+    lancedb = null;
+    connection = null;
+    table = null;
+    indexState = {
+        vector: false,
+        fts: false,
+        ftsError: "",
+    };
+    scopeCache = new Map();
+    constructor(dbPath) {
+        this.dbPath = dbPath;
+    }
+    async init(vectorDim) {
+        await mkdir(this.dbPath, { recursive: true });
+        await mkdir(dirname(this.dbPath), { recursive: true });
+        this.lancedb = await import("@lancedb/lancedb");
+        this.connection = (await this.lancedb.connect(this.dbPath));
+        try {
+            this.table = await this.connection.openTable(TABLE_NAME);
+        }
+        catch {
+            const bootstrap = {
+                id: "__bootstrap__",
+                text: "",
+                vector: new Array(vectorDim).fill(0),
+                category: "other",
+                scope: "global",
+                importance: 0,
+                timestamp: 0,
+                schemaVersion: 1,
+                embeddingModel: "bootstrap",
+                vectorDim,
+                metadataJson: "{}",
+            };
+            this.table = await this.connection.createTable(TABLE_NAME, [bootstrap]);
+            await this.table.delete("id = '__bootstrap__'");
+        }
+        await this.ensureIndexes();
+    }
+    async put(record) {
+        const table = this.requireTable();
+        await table.add([record]);
+        this.invalidateScope(record.scope);
+    }
+    async search(params) {
+        const cached = await this.getCachedScopes(params.scopes);
+        if (cached.records.length === 0)
+            return [];
+        const queryTokens = tokenize(params.query);
+        const queryNorm = vecNorm(params.queryVector);
+        const scored = cached.records
+            .filter((record) => params.queryVector.length === 0 || record.vector.length === params.queryVector.length)
+            .map((record, index) => {
+            const recordNorm = cached.norms.get(record.id) ?? vecNorm(record.vector);
+            const vectorScore = fastCosine(params.queryVector, record.vector, queryNorm, recordNorm);
+            const bm25Score = bm25LikeScore(queryTokens, cached.tokenized[index], cached.idf);
+            const score = params.vectorWeight * vectorScore + params.bm25Weight * bm25Score;
+            return { record, score, vectorScore, bm25Score };
+        })
+            .filter((item) => item.score >= params.minScore)
+            .sort((a, b) => b.score - a.score)
+            .slice(0, params.limit);
+        return scored;
+    }
+    async deleteById(id, scopes) {
+        const rows = await this.readByScopes(scopes);
+        const match = rows.find((row) => row.id === id);
+        if (!match)
+            return false;
+        await this.requireTable().delete(`id = '${escapeSql(match.id)}'`);
+        this.invalidateScope(match.scope);
+        return true;
+    }
+    async clearScope(scope) {
+        const rows = await this.readByScopes([scope]);
+        if (rows.length === 0)
+            return 0;
+        await this.requireTable().delete(`scope = '${escapeSql(scope)}'`);
+        this.invalidateScope(scope);
+        return rows.length;
+    }
+    async list(scope, limit) {
+        const rows = await this.readByScopes([scope]);
+        return rows.sort((a, b) => b.timestamp - a.timestamp).slice(0, limit);
+    }
+    async pruneScope(scope, maxEntries) {
+        const rows = await this.list(scope, 100000);
+        if (rows.length <= maxEntries)
+            return 0;
+        const toDelete = rows.slice(maxEntries);
+        for (const row of toDelete) {
+            await this.requireTable().delete(`id = '${escapeSql(row.id)}'`);
+        }
+        this.invalidateScope(scope);
+        return toDelete.length;
+    }
+    async countIncompatibleVectors(scopes, expectedDim) {
+        const rows = await this.readByScopes(scopes);
+        return rows.filter((row) => row.vectorDim !== expectedDim).length;
+    }
+    getIndexHealth() {
+        return {
+            vector: this.indexState.vector,
+            fts: this.indexState.fts,
+            ftsError: this.indexState.ftsError || undefined,
+        };
+    }
+    invalidateScope(scope) {
+        this.scopeCache.delete(scope);
+    }
+    async getCachedScopes(scopes) {
+        const allRecords = [];
+        const allTokenized = [];
+        const allNorms = new Map();
+        for (const scope of scopes) {
+            let entry = this.scopeCache.get(scope);
+            if (!entry) {
+                const records = await this.readByScopes([scope]);
+                const tokenized = records.map((record) => tokenize(record.text));
+                const idf = computeIdf(tokenized);
+                const norms = new Map();
+                for (const record of records) {
+                    norms.set(record.id, vecNorm(record.vector));
+                }
+                entry = { records, tokenized, idf, norms };
+                this.scopeCache.set(scope, entry);
+            }
+            allRecords.push(...entry.records);
+            allTokenized.push(...entry.tokenized);
+            for (const [id, norm] of entry.norms) {
+                allNorms.set(id, norm);
+            }
+        }
+        const idf = scopes.length === 1 && this.scopeCache.has(scopes[0])
+            ? this.scopeCache.get(scopes[0]).idf
+            : computeIdf(allTokenized);
+        return { records: allRecords, tokenized: allTokenized, idf, norms: allNorms };
+    }
+    requireTable() {
+        if (!this.table) {
+            throw new Error("MemoryStore is not initialized");
+        }
+        return this.table;
+    }
+    async readByScopes(scopes) {
+        const table = this.requireTable();
+        if (scopes.length === 0)
+            return [];
+        const whereExpr = scopes.map((scope) => `scope = '${escapeSql(scope)}'`).join(" OR ");
+        const rows = await table
+            .query()
+            .where(`(${whereExpr})`)
+            .select([
+            "id",
+            "text",
+            "vector",
+            "category",
+            "scope",
+            "importance",
+            "timestamp",
+            "schemaVersion",
+            "embeddingModel",
+            "vectorDim",
+            "metadataJson",
+        ])
+            .limit(100000)
+            .toArray();
+        return rows
+            .map((row) => normalizeRow(row))
+            .filter((row) => row !== null);
+    }
+    async ensureIndexes() {
+        const table = this.requireTable();
+        try {
+            await table.createIndex("vector");
+            this.indexState.vector = true;
+        }
+        catch {
+            this.indexState.vector = false;
+        }
+        try {
+            if (this.lancedb && "Index" in this.lancedb) {
+                const anyLance = this.lancedb;
+                const cfg = anyLance.Index?.fts ? { config: anyLance.Index.fts() } : undefined;
+                await table.createIndex("text", cfg);
+            }
+            else {
+                await table.createIndex("text");
+            }
+            this.indexState.fts = true;
+            this.indexState.ftsError = "";
+        }
+        catch (error) {
+            this.indexState.fts = false;
+            this.indexState.ftsError = error instanceof Error ? error.message : String(error);
+        }
+    }
+}
+function normalizeRow(row) {
+    const vectorRaw = row.vector;
+    const vector = Array.isArray(vectorRaw) ? vectorRaw.map((item) => Number(item)) : Array.from((vectorRaw ?? []));
+    if (typeof row.id !== "string" || typeof row.text !== "string" || typeof row.scope !== "string") {
+        return null;
+    }
+    return {
+        id: row.id,
+        text: row.text,
+        vector,
+        category: row.category ?? "other",
+        scope: row.scope,
+        importance: Number(row.importance ?? 0.5),
+        timestamp: Number(row.timestamp ?? Date.now()),
+        schemaVersion: Number(row.schemaVersion ?? 1),
+        embeddingModel: String(row.embeddingModel ?? "unknown"),
+        vectorDim: Number(row.vectorDim ?? vector.length),
+        metadataJson: String(row.metadataJson ?? "{}"),
+    };
+}
+function escapeSql(value) {
+    return value.replace(/'/g, "''");
+}
+function computeIdf(docs) {
+    const df = new Map();
+    for (const doc of docs) {
+        const seen = new Set(doc);
+        for (const token of seen) {
+            df.set(token, (df.get(token) ?? 0) + 1);
+        }
+    }
+    const totalDocs = Math.max(1, docs.length);
+    const idf = new Map();
+    for (const [token, count] of df.entries()) {
+        idf.set(token, Math.log(1 + (totalDocs - count + 0.5) / (count + 0.5)));
+    }
+    return idf;
+}
+function vecNorm(v) {
+    let sum = 0;
+    for (let i = 0; i < v.length; i += 1) {
+        sum += v[i] * v[i];
+    }
+    return Math.sqrt(sum);
+}
+function fastCosine(a, b, normA, normB) {
+    if (a.length === 0 || b.length === 0 || a.length !== b.length)
+        return 0;
+    const denom = normA * normB;
+    if (denom === 0)
+        return 0;
+    let dot = 0;
+    for (let i = 0; i < a.length; i += 1) {
+        dot += a[i] * b[i];
+    }
+    return dot / denom;
+}
+function bm25LikeScore(query, doc, idf) {
+    if (query.length === 0 || doc.length === 0)
+        return 0;
+    const tf = new Map();
+    for (const token of doc) {
+        tf.set(token, (tf.get(token) ?? 0) + 1);
+    }
+    const avgDocLen = 120;
+    const k1 = 1.2;
+    const b = 0.75;
+    let score = 0;
+    for (const token of query) {
+        const freq = tf.get(token) ?? 0;
+        if (freq === 0)
+            continue;
+        const tokenIdf = idf.get(token) ?? 0.1;
+        const numerator = freq * (k1 + 1);
+        const denominator = freq + k1 * (1 - b + (b * doc.length) / avgDocLen);
+        score += tokenIdf * (numerator / denominator);
+    }
+    return 1 - Math.exp(-score);
+}

package/dist/types.d.ts

@@ -0,0 +1,48 @@
+export type EmbeddingProvider = "ollama";
+export type RetrievalMode = "hybrid" | "vector";
+export type MemoryCategory = "preference" | "fact" | "decision" | "entity" | "other";
+export interface EmbeddingConfig {
+    provider: EmbeddingProvider;
+    model: string;
+    baseUrl?: string;
+    timeoutMs?: number;
+}
+export interface RetrievalConfig {
+    mode: RetrievalMode;
+    vectorWeight: number;
+    bm25Weight: number;
+    minScore: number;
+}
+export interface MemoryRuntimeConfig {
+    provider: string;
+    dbPath: string;
+    embedding: EmbeddingConfig;
+    retrieval: RetrievalConfig;
+    includeGlobalScope: boolean;
+    minCaptureChars: number;
+    maxEntriesPerScope: number;
+}
+export interface MemoryRecord {
+    id: string;
+    text: string;
+    vector: number[];
+    category: MemoryCategory;
+    scope: string;
+    importance: number;
+    timestamp: number;
+    schemaVersion: number;
+    embeddingModel: string;
+    vectorDim: number;
+    metadataJson: string;
+}
+export interface SearchResult {
+    record: MemoryRecord;
+    score: number;
+    vectorScore: number;
+    bm25Score: number;
+}
+export interface CaptureCandidate {
+    text: string;
+    category: MemoryCategory;
+    importance: number;
+}

package/dist/types.js

@@ -0,0 +1 @@
+export {};

package/dist/utils.d.ts

@@ -0,0 +1,9 @@
+export declare function expandHomePath(input: string): string;
+export declare function toNumber(value: unknown, fallback: number): number;
+export declare function toBoolean(value: unknown, fallback: boolean): boolean;
+export declare function clamp(value: number, min: number, max: number): number;
+export declare function stableHash(input: string): string;
+export declare function tokenize(text: string): string[];
+export declare function cosineSimilarity(a: number[], b: number[]): number;
+export declare function generateId(): string;
+export declare function parseJsonObject<T>(value: string | undefined, fallback: T): T;

package/dist/utils.js

@@ -0,0 +1,73 @@
+import { createHash, randomUUID } from "node:crypto";
+import { homedir } from "node:os";
+import { join } from "node:path";
+export function expandHomePath(input) {
+    if (input === "~")
+        return homedir();
+    if (input.startsWith("~/"))
+        return join(homedir(), input.slice(2));
+    return input;
+}
+export function toNumber(value, fallback) {
+    if (typeof value === "number" && Number.isFinite(value))
+        return value;
+    if (typeof value === "string" && value.trim()) {
+        const parsed = Number(value);
+        if (Number.isFinite(parsed))
+            return parsed;
+    }
+    return fallback;
+}
+export function toBoolean(value, fallback) {
+    if (typeof value === "boolean")
+        return value;
+    if (typeof value === "string") {
+        const normalized = value.trim().toLowerCase();
+        if (["1", "true", "yes", "on"].includes(normalized))
+            return true;
+        if (["0", "false", "no", "off"].includes(normalized))
+            return false;
+    }
+    return fallback;
+}
+export function clamp(value, min, max) {
+    return Math.max(min, Math.min(max, value));
+}
+export function stableHash(input) {
+    return createHash("sha256").update(input, "utf8").digest("hex");
+}
+export function tokenize(text) {
+    return text
+        .toLowerCase()
+        .replace(/[^\p{L}\p{N}\s_-]+/gu, " ")
+        .split(/\s+/)
+        .filter((token) => token.length > 1);
+}
+export function cosineSimilarity(a, b) {
+    if (a.length === 0 || b.length === 0 || a.length !== b.length)
+        return 0;
+    let dot = 0;
+    let normA = 0;
+    let normB = 0;
+    for (let i = 0; i < a.length; i += 1) {
+        dot += a[i] * b[i];
+        normA += a[i] * a[i];
+        normB += b[i] * b[i];
+    }
+    const denom = Math.sqrt(normA) * Math.sqrt(normB);
+    return denom === 0 ? 0 : dot / denom;
+}
+export function generateId() {
+    return randomUUID();
+}
+export function parseJsonObject(value, fallback) {
+    if (!value)
+        return fallback;
+    try {
+        const parsed = JSON.parse(value);
+        return parsed;
+    }
+    catch {
+        return fallback;
+    }
+}

package/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "lancedb-opencode-pro",
+  "version": "0.1.1",
+  "description": "LanceDB-backed long-term memory provider for OpenCode",
+  "type": "module",
+  "main": "dist/index.js",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "types": "dist/index.d.ts",
+  "sideEffects": false,
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "opencode",
+    "plugin",
+    "memory",
+    "lancedb"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab-238.ichiayi.com/jonathan/lancedb-opencode-pro.git"
+  },
+  "homepage": "https://gitlab-238.ichiayi.com/jonathan/lancedb-opencode-pro",
+  "bugs": {
+    "url": "https://gitlab-238.ichiayi.com/jonathan/lancedb-opencode-pro/-/issues"
+  },
+  "license": "MIT",
+  "engines": {
+    "node": ">=22"
+  },
+  "publishConfig": {
+    "access": "public",
+    "registry": "https://registry.npmjs.org/"
+  },
+  "scripts": {
+    "build": "tsc -p tsconfig.json",
+    "build:test": "tsc -p tsconfig.test.json",
+    "typecheck": "tsc -p tsconfig.json --noEmit",
+    "test": "npm run typecheck",
+    "test:foundation": "npm run build:test && node --test dist-test/test/foundation/foundation.test.js",
+    "test:regression": "npm run build:test && node --test dist-test/test/regression/plugin.test.js",
+    "test:retrieval": "npm run build:test && node --test dist-test/test/retrieval/retrieval.test.js",
+    "benchmark:latency": "npm run build:test && node dist-test/test/benchmark/latency.js",
+    "test:e2e": "node scripts/e2e-opencode-memory.mjs",
+    "verify": "npm run typecheck && npm run build && npm run test:foundation && npm run test:regression && npm run test:retrieval",
+    "verify:full": "npm run verify && npm run benchmark:latency && npm pack",
+    "release:check": "npm run verify:full && npm publish --dry-run",
+    "prepublishOnly": "npm run verify:full"
+  },
+  "dependencies": {
+    "@lancedb/lancedb": "^0.26.2",
+    "@opencode-ai/plugin": "1.2.25",
+    "@opencode-ai/sdk": "1.2.25"
+  },
+  "devDependencies": {
+    "@types/node": "^22.13.9",
+    "typescript": "^5.8.2"
+  }
+}