om-memory-system 4.24.0-next.147 → 4.25.0

package/dist/services/chat/chat-routes.js

@@ -0,0 +1,145 @@
+import { tryExtractScopeFromContainerTag } from "../memory-scope.js";
+import { log } from "../logger.js";
+import { chatServiceDeps, chatStore, existingMemoryIds, } from "./chat-runtime.js";
+import { askQuestion } from "./chat-service.js";
+import { chatStatus } from "./chat-status.js";
+import { ChatStoreError } from "./chat-types.js";
+const ok = (data) => ({ status: 200, body: { success: true, data } });
+const fail = (status, code, error) => ({
+    status,
+    body: { success: false, code, error },
+});
+const STATUS_BY_CODE = { not_found: 404, conflict: 409, invalid: 400, read_only: 409 };
+function scopeTag(value) {
+    if (value === null || value === undefined)
+        return null;
+    // A scope must be a real container tag; anything else could reach a shard path.
+    return typeof value === "string" && tryExtractScopeFromContainerTag(value) ? value : undefined;
+}
+function folderId(value) {
+    if (value === null)
+        return null;
+    return typeof value === "string" && value ? value : undefined;
+}
+const CONVERSATION = /^\/api\/chat\/conversations\/([^/]+)$/;
+const MESSAGES = /^\/api\/chat\/conversations\/([^/]+)\/messages$/;
+const FOLDER = /^\/api\/chat\/folders\/([^/]+)$/;
+/**
+ * The chat routes under `/api/chat/`. Returns undefined for any other path.
+ * Store errors map to 400, 404 or 409; anything else is logged by code only.
+ */
+export async function routeChat(path, method, readBody, config) {
+    if (!path.startsWith("/api/chat/"))
+        return undefined;
+    try {
+        return await dispatch(path, method, readBody, config);
+    }
+    catch (error) {
+        if (error instanceof ChatStoreError) {
+            return fail(STATUS_BY_CODE[error.code], error.code, error.message);
+        }
+        log("chat route: error", { path: path.replace(/\/[0-9a-f-]{36}/g, "/:id"), method });
+        return fail(500, "internal", "Chat request failed");
+    }
+}
+async function dispatch(path, method, readBody, config) {
+    if (path === "/api/chat/status" && method === "GET")
+        return ok(chatStatus(config));
+    const store = await chatStore(config);
+    if (path === "/api/chat/conversations") {
+        if (method === "GET") {
+            const [conversations, folders] = await Promise.all([
+                store.listConversations(),
+                store.listFolders(),
+            ]);
+            return ok({ conversations, folders, readOnly: store.isReadOnly() });
+        }
+        if (method === "POST") {
+            const body = await readBody();
+            const scope = scopeTag(body.scopeTag);
+            const folder = body.folderId === undefined ? null : folderId(body.folderId);
+            if (scope === undefined || folder === undefined) {
+                return fail(400, "invalid", "Invalid scope or folder");
+            }
+            return ok(await store.createConversation({ title: "", scopeTag: scope, folderId: folder }));
+        }
+    }
+    if (path === "/api/chat/conversations/bulk-delete" && method === "POST") {
+        const body = await readBody();
+        const ids = Array.isArray(body.ids) ? body.ids.filter((id) => typeof id === "string") : [];
+        if (!Array.isArray(body.ids) || ids.length !== body.ids.length) {
+            return fail(400, "invalid", "ids must be a list of conversation ids");
+        }
+        return ok({ deleted: await store.bulkDeleteConversations(ids) });
+    }
+    const messages = MESSAGES.exec(path);
+    if (messages && method === "POST") {
+        const body = await readBody();
+        const scope = scopeTag(body.scopeTag);
+        if (scope === undefined)
+            return fail(400, "invalid", "Invalid scope");
+        if (typeof body.question !== "string")
+            return fail(400, "empty", "A question is required");
+        const status = chatStatus(config);
+        if (!status.ready) {
+            return fail(409, "not_configured", `Missing settings: ${status.missing.join(", ")}`);
+        }
+        const outcome = await askQuestion(messages[1], { question: body.question, scopeTag: scope, retry: body.retry === true }, await chatServiceDeps(config));
+        return outcome.ok
+            ? ok({ messages: outcome.messages })
+            : fail(outcome.status, outcome.code, outcome.error);
+    }
+    const conversation = CONVERSATION.exec(path);
+    if (conversation) {
+        const id = conversation[1];
+        if (method === "GET") {
+            const found = await store.getConversation(id);
+            if (!found)
+                return fail(404, "not_found", "Conversation not found");
+            const cited = [
+                ...new Set(found.messages.flatMap((m) => m.sources.filter((s) => s.kind === "memory").map((s) => s.id))),
+            ];
+            const existing = await existingMemoryIds(cited).catch(() => new Set(cited));
+            return ok({
+                ...found,
+                missingMemoryIds: cited.filter((memoryId) => !existing.has(memoryId)),
+            });
+        }
+        if (method === "PATCH") {
+            const body = await readBody();
+            if (typeof body.title === "string")
+                await store.renameConversation(id, body.title);
+            if (body.folderId !== undefined) {
+                const folder = folderId(body.folderId);
+                if (folder === undefined)
+                    return fail(400, "invalid", "Invalid folder");
+                await store.moveConversation(id, folder);
+            }
+            return ok({ id });
+        }
+        if (method === "DELETE") {
+            await store.deleteConversation(id);
+            return ok({ id });
+        }
+    }
+    if (path === "/api/chat/folders") {
+        if (method === "GET")
+            return ok(await store.listFolders());
+        if (method === "POST") {
+            const body = await readBody();
+            return ok(await store.createFolder(typeof body.name === "string" ? body.name : ""));
+        }
+    }
+    const folder = FOLDER.exec(path);
+    if (folder) {
+        if (method === "PATCH") {
+            const body = await readBody();
+            await store.renameFolder(folder[1], typeof body.name === "string" ? body.name : "");
+            return ok({ id: folder[1] });
+        }
+        if (method === "DELETE") {
+            return ok({ id: folder[1], deletedConversations: await store.deleteFolder(folder[1]) });
+        }
+    }
+    return fail(404, "not_found", "Unknown chat route");
+}

package/dist/services/chat/chat-runtime.d.ts

@@ -0,0 +1,21 @@
+import type { ChatServiceDeps } from "./chat-service.js";
+import { ChatStore } from "./chat-store.js";
+/** Each model call gets this long before chat gives up on it. */
+export declare const CHAT_MODEL_TIMEOUT_MS = 60000;
+/** The settings chat reads. The caller passes `CONFIG`. */
+export interface ChatRuntimeConfig {
+    storagePath: string;
+    memoryProvider: string;
+    memoryModel?: string;
+    memoryApiUrl?: string;
+    memoryApiKey?: string;
+    memoryTemperature?: number | false;
+    memoryExtraParams?: Record<string, unknown>;
+    userEmailOverride?: string;
+}
+/** The chat store for this storage path, opened and migrated once. */
+export declare function chatStore(config: Pick<ChatRuntimeConfig, "storagePath">): Promise<ChatStore>;
+/** The memory ids from `ids` that still exist in any project or user shard. */
+export declare function existingMemoryIds(ids: string[]): Promise<Set<string>>;
+/** Real dependencies for `askQuestion`, built from the settings passed in. */
+export declare function chatServiceDeps(config: ChatRuntimeConfig): Promise<ChatServiceDeps>;

package/dist/services/chat/chat-runtime.js

@@ -0,0 +1,104 @@
+import { join } from "node:path";
+import { log } from "../logger.js";
+import { ChatStore } from "./chat-store.js";
+/** Each model call gets this long before chat gives up on it. */
+export const CHAT_MODEL_TIMEOUT_MS = 60_000;
+const stores = new Map();
+/** The chat store for this storage path, opened and migrated once. */
+export function chatStore(config) {
+    const path = join(config.storagePath, "chats.db");
+    let store = stores.get(path);
+    if (!store) {
+        const opened = new ChatStore(path);
+        store = opened.init().then(() => opened);
+        store.catch(() => stores.delete(path));
+        stores.set(path, store);
+    }
+    return store;
+}
+async function callTool(config, system, user, tool) {
+    const { AIProviderFactory } = await import("../ai/ai-provider-factory.js");
+    const { buildMemoryProviderConfig } = await import("../ai/provider-config.js");
+    const { aiSessionManager } = await import("../ai/session/ai-session-manager.js");
+    const provider = AIProviderFactory.createProvider(config.memoryProvider, buildMemoryProviderConfig(config, { maxIterations: 1, iterationTimeout: CHAT_MODEL_TIMEOUT_MS }));
+    const sessionId = `chat_${crypto.randomUUID()}`;
+    try {
+        return await provider.executeToolCall(system, user, tool, sessionId);
+    }
+    finally {
+        // Providers keep a session transcript; chat keeps its own history in chats.db.
+        await aiSessionManager
+            .deleteSession(sessionId, config.memoryProvider)
+            .catch(() => undefined);
+    }
+}
+async function activeProfileItems(config) {
+    const { resolveWebProfileUserId } = await import("../profile-identity.js");
+    const { userProfileManager } = await import("../user-profile/user-profile-manager.js");
+    const userId = await resolveWebProfileUserId(config);
+    const profile = userId ? await userProfileManager.getActiveProfile(userId) : null;
+    if (!profile)
+        return [];
+    const data = JSON.parse(profile.profileData);
+    const sections = ["preferences", "patterns", "workflows"];
+    return sections.flatMap((section) => (data[section] ?? []).map((item, index) => ({
+        section,
+        index,
+        category: item.category,
+        description: String(item.description ?? ""),
+        evidence: item.evidence,
+        centroid: Array.isArray(item.centroid) ? item.centroid : undefined,
+    })));
+}
+/** The memory ids from `ids` that still exist in any project or user shard. */
+export async function existingMemoryIds(ids) {
+    const found = new Set();
+    if (ids.length === 0)
+        return found;
+    const { tursoShardManager } = await import("../turso/shard-manager.js");
+    const { tursoConnectionManager } = await import("../turso/connection-manager.js");
+    const shards = [
+        ...(await tursoShardManager.getAllShards("user", "")),
+        ...(await tursoShardManager.getAllShards("project", "")),
+    ];
+    const marks = ids.map(() => "?").join(",");
+    for (const shard of shards) {
+        const db = await tursoConnectionManager.getConnection(shard.dbPath);
+        const rows = await db.all(`SELECT id FROM memories WHERE id IN (${marks})`, ids);
+        for (const row of rows)
+            found.add(String(row.id));
+    }
+    return found;
+}
+/** Real dependencies for `askQuestion`, built from the settings passed in. */
+export async function chatServiceDeps(config) {
+    const store = await chatStore(config);
+    return {
+        store,
+        model: config.memoryModel ?? config.memoryProvider,
+        callTool: (system, user, tool) => callTool(config, system, user, tool),
+        retrieve: {
+            searchMemories: async (query, scopeTag) => {
+                const { memoryClient } = await import("../client.js");
+                const result = await memoryClient.searchMemories(query, scopeTag ?? "", scopeTag ? "project" : "all-projects");
+                return result.success ? result.results : [];
+            },
+            embed: async (text) => {
+                const { embeddingService } = await import("../embedding.js");
+                return embeddingService.embedWithTimeout(text, { task: "query" });
+            },
+            profileItems: () => activeProfileItems(config),
+        },
+        listTags: async (scopeTag) => {
+            const { handleListKeywords } = await import("../api-handlers.js");
+            const result = await handleListKeywords(scopeTag ?? undefined);
+            return result.success && result.data ? result.data.keywords : [];
+        },
+        profileCategories: async () => [
+            ...new Set((await activeProfileItems(config))
+                .map((item) => item.category)
+                .filter((category) => !!category)),
+        ],
+        log: (message, meta) => log(message, meta),
+    };
+}

package/dist/services/chat/chat-service.d.ts

@@ -0,0 +1,42 @@
+import type { ToolCallResult } from "../ai/providers/base-provider.js";
+import { ANSWER_TOOL } from "./chat-answer.js";
+import { PLAN_TOOL, type PlanInput } from "./chat-plan.js";
+import { type RetrieveDeps } from "./chat-retrieve.js";
+import type { ChatStore } from "./chat-store.js";
+import type { ChatMessage } from "./chat-types.js";
+export declare const MAX_QUESTION_CHARS = 4000;
+type Tool = typeof PLAN_TOOL | typeof ANSWER_TOOL;
+export interface ChatServiceDeps {
+    store: ChatStore;
+    model: string;
+    /** One structured model call. Throws or returns `success: false` on failure. */
+    callTool: (system: string, user: string, tool: Tool) => Promise<ToolCallResult>;
+    retrieve: RetrieveDeps;
+    listTags: (scopeTag: string | null) => Promise<PlanInput["tags"]>;
+    profileCategories: () => Promise<string[]>;
+    /** Metadata only: sizes, counts, codes, timings. Never question or answer text. */
+    log: (message: string, meta: Record<string, unknown>) => void;
+}
+export interface AskInput {
+    question: string;
+    scopeTag: string | null;
+    /** Re-run a failed answer instead of adding the question again. */
+    retry?: boolean;
+}
+export type AskOutcome = {
+    ok: true;
+    messages: ChatMessage[];
+} | {
+    ok: false;
+    status: 400 | 404 | 409;
+    code: string;
+    error: string;
+};
+/**
+ * One question end to end: strip private text, save the question, plan the
+ * search, retrieve sources, answer with checked citations, save the answer.
+ * A model failure saves an assistant message with an error code so the page
+ * can offer a retry.
+ */
+export declare function askQuestion(conversationId: string, input: AskInput, deps: ChatServiceDeps): Promise<AskOutcome>;
+export {};

package/dist/services/chat/chat-service.js

@@ -0,0 +1,130 @@
+import { isFullyPrivate, stripPrivateContent } from "../privacy.js";
+import { ANSWER_TOOL, buildAnswerPrompt, finaliseAnswer, numberSources } from "./chat-answer.js";
+import { PLAN_TOOL, buildPlanPrompt, validatePlan } from "./chat-plan.js";
+import { retrieveSources } from "./chat-retrieve.js";
+export const MAX_QUESTION_CHARS = 4000;
+const TITLE_CHARS = 60;
+const DEFAULT_TITLE = "New chat";
+function fail(status, code, error) {
+    return { ok: false, status, code, error };
+}
+/** A short code for a failed model call. Provider error text can echo keys, so it is dropped. */
+function modelErrorCode(result) {
+    if (result?.transportError)
+        return "model_unreachable";
+    if (result?.httpStatus)
+        return `model_http_${result.httpStatus}`;
+    return "model_failed";
+}
+async function safeCall(deps, system, user, tool) {
+    try {
+        return await deps.callTool(system, user, tool);
+    }
+    catch {
+        return null;
+    }
+}
+async function plan(deps, input) {
+    const prompt = buildPlanPrompt(input);
+    const result = await safeCall(deps, prompt.system, prompt.user, PLAN_TOOL);
+    return validatePlan(result?.success ? result.data : null, input);
+}
+/**
+ * One question end to end: strip private text, save the question, plan the
+ * search, retrieve sources, answer with checked citations, save the answer.
+ * A model failure saves an assistant message with an error code so the page
+ * can offer a retry.
+ */
+export async function askQuestion(conversationId, input, deps) {
+    if (isFullyPrivate(input.question)) {
+        return fail(400, "private", "The question is fully private");
+    }
+    let question = stripPrivateContent(input.question).trim();
+    if (!question)
+        return fail(400, "empty", "A question is required");
+    if (question.length > MAX_QUESTION_CHARS) {
+        return fail(400, "too_long", `Questions have at most ${MAX_QUESTION_CHARS} characters`);
+    }
+    if (deps.store.isReadOnly()) {
+        return fail(409, "read_only", "chats.db is from a newer OMMS and is read-only");
+    }
+    const found = await deps.store.getConversation(conversationId);
+    if (!found)
+        return fail(404, "not_found", "Conversation not found");
+    let history = found.messages;
+    const last = history[history.length - 1];
+    const userTurn = history[history.length - 2];
+    if (input.retry && last?.role === "assistant" && last.errorCode && userTurn?.role === "user") {
+        await deps.store.deleteMessage(last.id);
+        history = history.slice(0, -2);
+        // Answer the question that was stored, whatever text the request carries.
+        question = userTurn.content;
+    }
+    else {
+        if (found.messages.length === 0 && found.conversation.title === DEFAULT_TITLE) {
+            await deps.store.renameConversation(conversationId, question.slice(0, TITLE_CHARS));
+        }
+        await deps.store.appendMessage(conversationId, { role: "user", content: question });
+    }
+    if (found.conversation.scopeTag !== input.scopeTag) {
+        await deps.store.setConversationScope(conversationId, input.scopeTag);
+    }
+    const started = Date.now();
+    const turns = history
+        .filter((message) => !message.errorCode && message.content)
+        .map((message) => ({ role: message.role, content: message.content }));
+    const [tags, categories] = await Promise.all([
+        deps.listTags(input.scopeTag).catch(() => []),
+        deps.profileCategories().catch(() => []),
+    ]);
+    const searchPlan = await plan(deps, { question, turns, tags, categories });
+    let sources;
+    try {
+        sources = numberSources(await retrieveSources(question, searchPlan, input.scopeTag, deps.retrieve));
+    }
+    catch {
+        return save(deps, conversationId, { errorCode: "search_failed", plan: searchPlan }, started);
+    }
+    if (sources.length === 0) {
+        return save(deps, conversationId, { notice: "no_sources", plan: searchPlan }, started);
+    }
+    const prompt = buildAnswerPrompt(question, turns, sources);
+    const result = await safeCall(deps, prompt.system, prompt.user, ANSWER_TOOL);
+    const answer = result?.success ? finaliseAnswer(result.data, sources) : null;
+    if (!answer) {
+        const errorCode = result?.success ? "model_invalid_reply" : modelErrorCode(result);
+        return save(deps, conversationId, { errorCode, plan: searchPlan, sources }, started);
+    }
+    return save(deps, conversationId, {
+        content: answer.text,
+        sources,
+        plan: searchPlan,
+        notice: answer.insufficient ? "insufficient" : answer.uncited ? "uncited" : null,
+        cited: answer.cited.length,
+    }, started);
+}
+async function save(deps, conversationId, out, started) {
+    const message = await deps.store.appendMessage(conversationId, {
+        role: "assistant",
+        content: out.content ?? "",
+        sources: out.sources ?? [],
+        plan: out.plan,
+        model: deps.model,
+        errorCode: out.errorCode ?? null,
+        notice: out.notice ?? null,
+    });
+    deps.log("chat: question answered", {
+        conversationId,
+        outcome: out.errorCode ?? out.notice ?? "answered",
+        planSkipped: out.plan.skipped,
+        queries: out.plan.queries.length,
+        tags: out.plan.tags.length,
+        categories: out.plan.profileCategories.length,
+        sources: out.sources?.length ?? 0,
+        cited: out.cited ?? 0,
+        answerChars: out.content?.length ?? 0,
+        ms: Date.now() - started,
+    });
+    const full = await deps.store.getConversation(conversationId);
+    return { ok: true, messages: full?.messages ?? [message] };
+}

package/dist/services/chat/chat-status.d.ts

@@ -0,0 +1,16 @@
+export interface ChatSettings {
+    memoryProvider?: string;
+    memoryModel?: string;
+    memoryApiUrl?: string;
+    memoryApiKey?: string;
+}
+export interface ChatStatus {
+    ready: boolean;
+    /** Names of the settings still needed. Never their values. */
+    missing: string[];
+    provider: string;
+    /** The host that receives questions and memory text, so the page can name it. */
+    apiHost: string | null;
+}
+/** Whether chat can call the external memory model. Mirrors `buildMemoryProviderConfig`. */
+export declare function chatStatus(config: ChatSettings): ChatStatus;

package/dist/services/chat/chat-status.js

@@ -0,0 +1,23 @@
+import { isPlaceholderApiKey } from "../ai/api-key-placeholder.js";
+/** Whether chat can call the external memory model. Mirrors `buildMemoryProviderConfig`. */
+export function chatStatus(config) {
+    const provider = config.memoryProvider ?? "openai-chat";
+    const presetEndpoint = provider === "orcarouter";
+    const missing = [];
+    if (!config.memoryModel && !presetEndpoint)
+        missing.push("memoryModel");
+    if (!config.memoryApiUrl && !presetEndpoint)
+        missing.push("memoryApiUrl");
+    if (!config.memoryApiKey || isPlaceholderApiKey(config.memoryApiKey)) {
+        missing.push("memoryApiKey");
+    }
+    return { ready: missing.length === 0, missing, provider, apiHost: hostOf(config.memoryApiUrl) };
+}
+function hostOf(url) {
+    try {
+        return url ? new URL(url).host : null;
+    }
+    catch {
+        return null;
+    }
+}

package/dist/services/chat/chat-store.d.ts

@@ -0,0 +1,60 @@
+import type { TursoDb } from "../turso/turso-db.js";
+import { type ChatConversation, type ChatFolder, type ChatMessage, type ChatNotice, type ChatPlanRecord, type ChatRole, type ChatSource } from "./chat-types.js";
+/** Bump when the tables change, and add the step to `migrate`. */
+export declare const CHAT_SCHEMA_VERSION = 1;
+export declare const FOLDER_NAME_MAX = 80;
+export declare const BULK_DELETE_MAX = 500;
+type GetConnection = (dbPath: string) => Promise<TursoDb>;
+/**
+ * Conversations, messages, and folders in `chats.db`. One instance per file;
+ * every write checks the schema version so a file from a newer OMMS stays
+ * untouched.
+ */
+export declare class ChatStore {
+    private readonly dbPath;
+    private readonly connect;
+    private readOnly;
+    private lastStamp;
+    constructor(dbPath: string, connect?: GetConnection);
+    private db;
+    private now;
+    init(): Promise<void>;
+    isReadOnly(): boolean;
+    private writable;
+    listConversations(): Promise<ChatConversation[]>;
+    getConversation(id: string): Promise<{
+        conversation: ChatConversation;
+        messages: ChatMessage[];
+    } | null>;
+    createConversation(input: {
+        title: string;
+        scopeTag: string | null;
+        folderId?: string | null;
+    }): Promise<ChatConversation>;
+    private updateConversation;
+    renameConversation(id: string, title: string): Promise<void>;
+    setConversationScope(id: string, scopeTag: string | null): Promise<void>;
+    moveConversation(id: string, folderId: string | null): Promise<void>;
+    deleteConversation(id: string): Promise<void>;
+    /** Deletes every id or none: an unknown id fails the whole request. */
+    bulkDeleteConversations(ids: string[]): Promise<number>;
+    appendMessage(conversationId: string, input: {
+        role: ChatRole;
+        content: string;
+        sources?: ChatSource[];
+        plan?: ChatPlanRecord | null;
+        model?: string | null;
+        errorCode?: string | null;
+        notice?: ChatNotice | null;
+    }): Promise<ChatMessage>;
+    deleteMessage(id: string): Promise<void>;
+    countMessages(): Promise<number>;
+    listFolders(): Promise<ChatFolder[]>;
+    private requireFolder;
+    private assertNameFree;
+    createFolder(name: string): Promise<ChatFolder>;
+    renameFolder(id: string, name: string): Promise<void>;
+    /** Deletes the folder and every conversation in it. Returns how many conversations went. */
+    deleteFolder(id: string): Promise<number>;
+}
+export {};