om-memory-system 3.0.0

package/dist/services/turso/types.d.ts

@@ -0,0 +1,42 @@
+export interface ShardInfo {
+    id: number;
+    scope: "user" | "project";
+    scopeHash: string;
+    shardIndex: number;
+    dbPath: string;
+    vectorCount: number;
+    isActive: boolean;
+    createdAt: number;
+}
+export interface MemoryRecord {
+    id: string;
+    content: string;
+    vector: Float32Array;
+    tagsVector?: Float32Array;
+    containerTag: string;
+    tags?: string;
+    type?: string;
+    createdAt: number;
+    updatedAt: number;
+    metadata?: string;
+    displayName?: string;
+    userName?: string;
+    userEmail?: string;
+    projectPath?: string;
+    projectName?: string;
+    gitRepoUrl?: string;
+}
+export interface SearchResult {
+    id: string;
+    memory: string;
+    similarity: number;
+    createdAt?: number;
+    tags?: string[];
+    metadata?: Record<string, unknown>;
+    displayName?: string;
+    userName?: string;
+    userEmail?: string;
+    projectPath?: string;
+    projectName?: string;
+    gitRepoUrl?: string;
+}

package/dist/services/turso/types.js

@@ -0,0 +1 @@
+export {};

package/dist/services/turso/vector-search.d.ts

@@ -0,0 +1,41 @@
+import type { MemoryRecord, SearchResult, ShardInfo } from "./types.js";
+import type { TursoDb } from "./turso-db.js";
+import type { Transaction } from "@libsql/client";
+export declare class TursoVectorSearch {
+    insertVectorInTransaction(tx: Transaction, record: MemoryRecord): Promise<void>;
+    insertVector(db: TursoDb, record: MemoryRecord): Promise<void>;
+    searchInShard(shard: ShardInfo, queryVector: Float32Array, containerTag: string, limit: number, queryText?: string): Promise<SearchResult[]>;
+    private searchKind;
+    private exactScanKind;
+    searchAcrossShards(shards: ShardInfo[], queryVector: Float32Array, containerTag: string, limit: number, similarityThreshold: number, queryText?: string): Promise<{
+        results: SearchResult[];
+        warnings: string[];
+    }>;
+    deleteVector(db: TursoDb, memoryId: string): Promise<void>;
+    updateVector(db: TursoDb, memoryId: string, vector: Float32Array, tagsVector?: Float32Array): Promise<void>;
+    listMemories(db: TursoDb, containerTag: string, limit: number): Promise<Record<string, unknown>[]>;
+    getAllMemories(db: TursoDb): Promise<Record<string, unknown>[]>;
+    getAllMemoriesWithExtractedVectors(db: TursoDb): Promise<Array<Record<string, unknown> & {
+        vector_json: string | null;
+    }>>;
+    getMemoryById(db: TursoDb, memoryId: string): Promise<Record<string, unknown> | null>;
+    getMemoriesBySessionID(db: TursoDb, sessionID: string): Promise<Record<string, unknown>[]>;
+    countVectors(db: TursoDb, containerTag: string): Promise<number>;
+    countAllVectors(db: TursoDb): Promise<number>;
+    getDistinctTags(db: TursoDb): Promise<Record<string, unknown>[]>;
+    getProjectPathCounts(db: TursoDb): Promise<Array<{
+        projectPath: string;
+        count: number;
+        containerTag: string | null;
+    }>>;
+    updateProjectAssociation(db: TursoDb, oldContainerTag: string, update: {
+        containerTag: string;
+        projectPath?: string;
+        projectName?: string;
+        displayName?: string;
+        gitRepoUrl?: string | null;
+    }): Promise<number>;
+    pinMemory(db: TursoDb, memoryId: string): Promise<void>;
+    unpinMemory(db: TursoDb, memoryId: string): Promise<void>;
+}
+export declare const tursoVectorSearch: TursoVectorSearch;

package/dist/services/turso/vector-search.js

@@ -0,0 +1,384 @@
+import { tursoConnectionManager } from "./connection-manager.js";
+import { log } from "../logger.js";
+import { distanceToSimilarity, vectorToJson } from "./vector-utils.js";
+function parseMetadata(value) {
+    if (!value || typeof value !== "string")
+        return undefined;
+    try {
+        return JSON.parse(value);
+    }
+    catch {
+        return undefined;
+    }
+}
+export class TursoVectorSearch {
+    async insertVectorInTransaction(tx, record) {
+        const contentVector = vectorToJson(record.vector);
+        const commonArgs = [
+            record.containerTag,
+            record.tags || null,
+            record.type || null,
+            record.createdAt,
+            record.updatedAt,
+            record.metadata || null,
+            record.displayName || null,
+            record.userName || null,
+            record.userEmail || null,
+            record.projectPath || null,
+            record.projectName || null,
+            record.gitRepoUrl || null,
+        ];
+        if (record.tagsVector) {
+            await tx.execute({
+                sql: `
+        INSERT INTO memories (
+          id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
+          metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
+        ) VALUES (?, ?, vector32(?), vector32(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `,
+                args: [
+                    record.id,
+                    record.content,
+                    contentVector,
+                    vectorToJson(record.tagsVector),
+                    ...commonArgs,
+                ],
+            });
+            return;
+        }
+        await tx.execute({
+            sql: `
+      INSERT INTO memories (
+        id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
+        metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
+      ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `,
+            args: [record.id, record.content, contentVector, ...commonArgs],
+        });
+    }
+    async insertVector(db, record) {
+        const contentVector = vectorToJson(record.vector);
+        const commonArgs = [
+            record.containerTag,
+            record.tags || null,
+            record.type || null,
+            record.createdAt,
+            record.updatedAt,
+            record.metadata || null,
+            record.displayName || null,
+            record.userName || null,
+            record.userEmail || null,
+            record.projectPath || null,
+            record.projectName || null,
+            record.gitRepoUrl || null,
+        ];
+        if (record.tagsVector) {
+            await db.execute(`
+        INSERT INTO memories (
+          id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
+          metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
+        ) VALUES (?, ?, vector32(?), vector32(?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+      `, [record.id, record.content, contentVector, vectorToJson(record.tagsVector), ...commonArgs]);
+            return;
+        }
+        await db.execute(`
+      INSERT INTO memories (
+        id, content, vector, tags_vector, container_tag, tags, type, created_at, updated_at,
+        metadata, display_name, user_name, user_email, project_path, project_name, git_repo_url
+      ) VALUES (?, ?, vector32(?), NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `, [record.id, record.content, contentVector, ...commonArgs]);
+    }
+    async searchInShard(shard, queryVector, containerTag, limit, queryText) {
+        const db = await tursoConnectionManager.getConnection(shard.dbPath);
+        const queryJson = vectorToJson(queryVector);
+        // Over-fetch aggressively when filtering by container_tag after ANN,
+        // because post-filtering can discard many DiskANN neighbors.
+        const k = containerTag === "" ? Math.max(limit * 4, 32) : Math.max(limit * 16, 128);
+        const contentResults = await this.searchKind(db, queryJson, k, containerTag, "memories_vec_idx", "vector");
+        const tagsResults = await this.searchKind(db, queryJson, k, containerTag, "memories_tags_vec_idx", "tags_vector");
+        const candidateIds = new Set();
+        for (const result of contentResults)
+            candidateIds.add(result.id);
+        for (const result of tagsResults)
+            candidateIds.add(result.id);
+        const ids = Array.from(candidateIds);
+        if (ids.length === 0)
+            return [];
+        const placeholders = ids.map(() => "?").join(",");
+        // Recompute exact distances for hydrated rows so tag-only / content-only ANN
+        // hits still get a full hybrid score (missing ANN side is not forced to 0).
+        const rows = await db.all(containerTag === ""
+            ? `
+      SELECT id, content, tags, created_at, metadata, container_tag,
+             display_name, user_name, user_email, project_path, project_name,
+             git_repo_url, is_pinned,
+             vector_distance_cos(vector, vector32(?)) AS content_dist,
+             CASE WHEN tags_vector IS NOT NULL
+               THEN vector_distance_cos(tags_vector, vector32(?))
+               ELSE NULL END AS tags_dist
+      FROM memories
+      WHERE id IN (${placeholders})
+    `
+            : `
+      SELECT id, content, tags, created_at, metadata, container_tag,
+             display_name, user_name, user_email, project_path, project_name,
+             git_repo_url, is_pinned,
+             vector_distance_cos(vector, vector32(?)) AS content_dist,
+             CASE WHEN tags_vector IS NOT NULL
+               THEN vector_distance_cos(tags_vector, vector32(?))
+               ELSE NULL END AS tags_dist
+      FROM memories
+      WHERE id IN (${placeholders}) AND container_tag = ?
+    `, containerTag === ""
+            ? [queryJson, queryJson, ...ids]
+            : [queryJson, queryJson, ...ids, containerTag]);
+        const queryWords = queryText
+            ? queryText
+                .toLowerCase()
+                .split(/[\s,]+/)
+                .filter((word) => word.length > 1)
+            : [];
+        const hydratedResults = rows.map((row) => {
+            const contentSim = distanceToSimilarity(Number(row.content_dist));
+            const tagsSim = row.tags_dist == null || row.tags_dist === undefined
+                ? 0
+                : distanceToSimilarity(Number(row.tags_dist));
+            const memoryTagsStr = String(row.tags || "");
+            const memoryTags = memoryTagsStr.split(",").map((tag) => tag.trim().toLowerCase());
+            let exactMatchBoost = 0;
+            if (queryWords.length > 0 && memoryTags.length > 0) {
+                const matches = queryWords.filter((word) => memoryTags.some((tag) => tag.includes(word) || word.includes(tag))).length;
+                exactMatchBoost = matches / Math.max(queryWords.length, 1);
+            }
+            const finalTagsSim = Math.max(tagsSim, exactMatchBoost);
+            const similarity = contentSim * 0.6 + finalTagsSim * 0.4;
+            return {
+                id: String(row.id),
+                memory: String(row.content),
+                similarity,
+                createdAt: Number(row.created_at),
+                tags: memoryTagsStr ? memoryTagsStr.split(",") : [],
+                metadata: parseMetadata(row.metadata),
+                containerTag: String(row.container_tag),
+                displayName: row.display_name ? String(row.display_name) : undefined,
+                userName: row.user_name ? String(row.user_name) : undefined,
+                userEmail: row.user_email ? String(row.user_email) : undefined,
+                projectPath: row.project_path ? String(row.project_path) : undefined,
+                projectName: row.project_name ? String(row.project_name) : undefined,
+                gitRepoUrl: row.git_repo_url ? String(row.git_repo_url) : undefined,
+                isPinned: row.is_pinned,
+            };
+        });
+        hydratedResults.sort((a, b) => b.similarity - a.similarity);
+        return hydratedResults.slice(0, Math.max(0, limit));
+    }
+    async searchKind(db, queryJson, k, containerTag, indexName, columnName) {
+        try {
+            const rows = await db.all(containerTag === ""
+                ? `
+          SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
+          FROM vector_top_k('${indexName}', vector32(?), ?) AS v
+          CROSS JOIN memories m ON m.rowid = v.id
+          WHERE m.${columnName} IS NOT NULL
+        `
+                : `
+          SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
+          FROM vector_top_k('${indexName}', vector32(?), ?) AS v
+          CROSS JOIN memories m ON m.rowid = v.id
+          WHERE m.${columnName} IS NOT NULL AND m.container_tag = ?
+        `, containerTag === "" ? [queryJson, queryJson, k] : [queryJson, queryJson, k, containerTag]);
+            return rows.map((row) => ({
+                id: String(row.id),
+                similarity: distanceToSimilarity(Number(row.dist)),
+            }));
+        }
+        catch (error) {
+            log("Turso vector_top_k failed; falling back to exact scan", {
+                indexName,
+                error: String(error),
+            });
+            return this.exactScanKind(db, queryJson, k, containerTag, columnName);
+        }
+    }
+    async exactScanKind(db, queryJson, k, containerTag, columnName) {
+        const rows = await db.all(containerTag === ""
+            ? `
+        SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
+        FROM memories m
+        WHERE m.${columnName} IS NOT NULL
+        ORDER BY dist ASC
+        LIMIT ?
+      `
+            : `
+        SELECT m.id AS id, vector_distance_cos(m.${columnName}, vector32(?)) AS dist
+        FROM memories m
+        WHERE m.${columnName} IS NOT NULL AND m.container_tag = ?
+        ORDER BY dist ASC
+        LIMIT ?
+      `, containerTag === "" ? [queryJson, k] : [queryJson, containerTag, k]);
+        return rows.map((row) => ({
+            id: String(row.id),
+            similarity: distanceToSimilarity(Number(row.dist)),
+        }));
+    }
+    async searchAcrossShards(shards, queryVector, containerTag, limit, similarityThreshold, queryText) {
+        const shardErrors = [];
+        const shardPromises = shards.map(async (shard) => {
+            try {
+                return await this.searchInShard(shard, queryVector, containerTag, limit, queryText);
+            }
+            catch (error) {
+                const message = error instanceof Error ? error.message : String(error);
+                log("Shard search error", { shardId: shard.id, error: message });
+                shardErrors.push({ shardId: shard.id, error: message });
+                return [];
+            }
+        });
+        const resultsArray = await Promise.all(shardPromises);
+        if (shardErrors.length > 0 && shardErrors.length === shards.length) {
+            throw new Error(`Vector search failed on all shards: ${shardErrors.map((entry) => `${entry.shardId}: ${entry.error}`).join("; ")}`);
+        }
+        const warnings = shardErrors.length > 0
+            ? [
+                `Vector search completed with partial shard failures (${shardErrors.length}/${shards.length}): ${shardErrors
+                    .map((entry) => `${entry.shardId}: ${entry.error}`)
+                    .join("; ")}`,
+            ]
+            : [];
+        if (warnings.length > 0) {
+            log("Vector search completed with partial shard failures", {
+                failedShards: shardErrors,
+                totalShards: shards.length,
+            });
+        }
+        const allResults = resultsArray.flat();
+        allResults.sort((a, b) => b.similarity - a.similarity);
+        return {
+            results: allResults
+                .filter((result) => result.similarity >= similarityThreshold)
+                .slice(0, limit),
+            warnings,
+        };
+    }
+    async deleteVector(db, memoryId) {
+        await db.run(`DELETE FROM memories WHERE id = ?`, [memoryId]);
+    }
+    async updateVector(db, memoryId, vector, tagsVector) {
+        const contentVector = vectorToJson(vector);
+        if (tagsVector) {
+            await db.execute(`UPDATE memories SET vector = vector32(?), tags_vector = vector32(?) WHERE id = ?`, [contentVector, vectorToJson(tagsVector), memoryId]);
+        }
+        else {
+            await db.execute(`UPDATE memories SET vector = vector32(?), tags_vector = NULL WHERE id = ?`, [contentVector, memoryId]);
+        }
+    }
+    async listMemories(db, containerTag, limit) {
+        return containerTag === ""
+            ? db.all(`
+      SELECT * FROM memories
+      ORDER BY created_at DESC
+      LIMIT ?
+    `, [limit])
+            : db.all(`
+      SELECT * FROM memories
+      WHERE container_tag = ?
+      ORDER BY created_at DESC
+      LIMIT ?
+    `, [containerTag, limit]);
+    }
+    async getAllMemories(db) {
+        return db.all(`SELECT * FROM memories ORDER BY created_at DESC`);
+    }
+    async getAllMemoriesWithExtractedVectors(db) {
+        return db.all(`
+      SELECT
+        id,
+        content,
+        container_tag,
+        created_at,
+        vector_extract(vector) AS vector_json
+      FROM memories
+      ORDER BY created_at DESC
+    `);
+    }
+    async getMemoryById(db, memoryId) {
+        return db.get(`SELECT * FROM memories WHERE id = ?`, [memoryId]);
+    }
+    async getMemoriesBySessionID(db, sessionID) {
+        const rows = await db.all(`
+      SELECT * FROM memories
+      WHERE metadata LIKE ?
+      ORDER BY created_at DESC
+    `, [`%"sessionID":"${sessionID}"%`]);
+        return rows.map((row) => ({
+            ...row,
+            tags: row.tags ? String(row.tags).split(",") : [],
+            metadata: row.metadata ? (parseMetadata(String(row.metadata)) ?? {}) : {},
+        }));
+    }
+    async countVectors(db, containerTag) {
+        const row = await db.get(`SELECT COUNT(*) as count FROM memories WHERE container_tag = ?`, [
+            containerTag,
+        ]);
+        return Number(row?.count ?? 0);
+    }
+    async countAllVectors(db) {
+        const row = await db.get(`SELECT COUNT(*) as count FROM memories`);
+        return Number(row?.count ?? 0);
+    }
+    async getDistinctTags(db) {
+        return db.all(`
+      SELECT DISTINCT
+        container_tag,
+        display_name,
+        user_name,
+        user_email,
+        project_path,
+        project_name,
+        git_repo_url
+      FROM memories
+    `);
+    }
+    async getProjectPathCounts(db) {
+        const rows = await db.all(`
+      SELECT
+        project_path AS project_path,
+        container_tag AS container_tag,
+        COUNT(*) AS cnt
+      FROM memories
+      WHERE project_path IS NOT NULL AND project_path != ''
+      GROUP BY project_path, container_tag
+      ORDER BY cnt DESC, project_path ASC
+    `);
+        return rows.map((row) => ({
+            projectPath: String(row.project_path),
+            count: Number(row.cnt ?? 0),
+            containerTag: row.container_tag ? String(row.container_tag) : null,
+        }));
+    }
+    async updateProjectAssociation(db, oldContainerTag, update) {
+        return db.run(`
+      UPDATE memories SET
+        container_tag = ?,
+        project_path = ?,
+        project_name = ?,
+        display_name = ?,
+        git_repo_url = ?
+      WHERE container_tag = ?
+    `, [
+            update.containerTag,
+            update.projectPath ?? null,
+            update.projectName ?? null,
+            update.displayName ?? null,
+            update.gitRepoUrl ?? null,
+            oldContainerTag,
+        ]);
+    }
+    async pinMemory(db, memoryId) {
+        await db.run(`UPDATE memories SET is_pinned = 1 WHERE id = ?`, [memoryId]);
+    }
+    async unpinMemory(db, memoryId) {
+        await db.run(`UPDATE memories SET is_pinned = 0 WHERE id = ?`, [memoryId]);
+    }
+}
+export const tursoVectorSearch = new TursoVectorSearch();

package/dist/services/turso/vector-utils.d.ts

@@ -0,0 +1,7 @@
+export declare function vectorToJson(vector: Float32Array): string;
+export declare function blobToFloat32Array(value: unknown): Float32Array | null;
+/** Parse libSQL `vector_extract()` JSON output (preferred over raw F32_BLOB bytes). */
+export declare function parseExtractedVector(value: unknown): Float32Array | null;
+export declare function distanceToSimilarity(distance: number): number;
+/** Canonical text used for tag-vector embeddings (must stay consistent across write paths). */
+export declare function formatTagsForEmbedding(tags: string[]): string;

package/dist/services/turso/vector-utils.js

@@ -0,0 +1,67 @@
+export function vectorToJson(vector) {
+    return JSON.stringify(Array.from(vector));
+}
+export function blobToFloat32Array(value) {
+    if (value == null)
+        return null;
+    try {
+        if (value instanceof Float32Array) {
+            return value;
+        }
+        if (value instanceof ArrayBuffer) {
+            if (value.byteLength === 0 || value.byteLength % 4 !== 0) {
+                return null;
+            }
+            return new Float32Array(value);
+        }
+        if (ArrayBuffer.isView(value)) {
+            const view = value;
+            const byteLength = view.byteLength;
+            if (byteLength === 0 || byteLength % 4 !== 0) {
+                return null;
+            }
+            return new Float32Array(view.buffer.slice(view.byteOffset, view.byteOffset + view.byteLength));
+        }
+        if (typeof value === "string") {
+            try {
+                const parsed = JSON.parse(value);
+                return new Float32Array(parsed);
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    catch {
+        return null;
+    }
+    return null;
+}
+/** Parse libSQL `vector_extract()` JSON output (preferred over raw F32_BLOB bytes). */
+export function parseExtractedVector(value) {
+    if (value == null)
+        return null;
+    if (typeof value === "string") {
+        try {
+            const parsed = JSON.parse(value);
+            if (!Array.isArray(parsed) || parsed.length === 0)
+                return null;
+            return new Float32Array(parsed);
+        }
+        catch {
+            return null;
+        }
+    }
+    return blobToFloat32Array(value);
+}
+export function distanceToSimilarity(distance) {
+    const similarity = 1 - Number(distance);
+    if (!Number.isFinite(similarity))
+        return 0;
+    // Clamp float artifacts (docs: cos-distance can be slightly negative near exact matches).
+    return Math.max(0, Math.min(1, similarity));
+}
+/** Canonical text used for tag-vector embeddings (must stay consistent across write paths). */
+export function formatTagsForEmbedding(tags) {
+    return `Topics: ${tags.join(", ")}`;
+}

package/dist/services/user-memory-learning.d.ts

@@ -0,0 +1,97 @@
+import type { PluginInput } from "@opencode-ai/plugin";
+export declare function shouldRunAutomaticProfileCleanup(previousPromptCount: number, addedPromptCount: number, interval?: number): boolean;
+export declare function performUserProfileLearning(ctx: PluginInput, directory: string): Promise<void>;
+import { createUserProfileAnalysisSchema, USER_PROFILE_LLM_CONFIDENCE_MAX } from "../core/extraction.js";
+export { createUserProfileAnalysisSchema, USER_PROFILE_LLM_CONFIDENCE_MAX };
+export declare function createUserProfileToolSchema(existingProfile: boolean): {
+    type: "function";
+    function: {
+        name: string;
+        description: string;
+        parameters: {
+            type: string;
+            properties: {
+                preferences: {
+                    type: string;
+                    items: {
+                        type: string;
+                        properties: {
+                            category: {
+                                type: string;
+                            };
+                            description: {
+                                type: string;
+                            };
+                            confidence: {
+                                type: string;
+                                minimum: number;
+                                maximum: number;
+                            };
+                            evidence: {
+                                type: string;
+                                items: {
+                                    type: string;
+                                };
+                                maxItems: number;
+                            };
+                        };
+                        required: string[];
+                    };
+                };
+                patterns: {
+                    type: string;
+                    items: {
+                        type: string;
+                        properties: {
+                            category: {
+                                type: string;
+                            };
+                            description: {
+                                type: string;
+                            };
+                        };
+                        required: string[];
+                    };
+                };
+                workflows: {
+                    type: string;
+                    items: {
+                        type: string;
+                        properties: {
+                            description: {
+                                type: string;
+                            };
+                            steps: {
+                                type: string;
+                                items: {
+                                    type: string;
+                                };
+                            };
+                        };
+                        required: string[];
+                    };
+                };
+                validations: {
+                    type: string;
+                    items: {
+                        type: string;
+                        properties: {
+                            index: {
+                                type: string;
+                            };
+                            verdict: {
+                                type: string;
+                                enum: string[];
+                            };
+                            reason: {
+                                type: string;
+                            };
+                        };
+                        required: string[];
+                    };
+                };
+            };
+            required: string[];
+        };
+    };
+};