om-memory-system 4.30.1 → 4.31.2

package/dist/services/skill-learning/store.js

@@ -0,0 +1,245 @@
+import { existsSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { tursoConnectionManager } from "../turso/connection-manager.js";
+/** Bump when the tables change, and add the step to `init`. */
+export const SKILL_SCHEMA_VERSION = 1;
+export const SKILLS_DB_FILE = "skills.db";
+export class SkillStoreError extends Error {
+    code;
+    constructor(message, code) {
+        super(message);
+        this.code = code;
+        this.name = "SkillStoreError";
+    }
+}
+/** `skills.db` always sits in the global storage path, never in a project folder. */
+export function skillsDbPath(storagePath) {
+    return join(storagePath, SKILLS_DB_FILE);
+}
+// Like chats.db, the file keeps the default rollback journal, so committed
+// data lives in skills.db alone.
+const SCHEMA = [
+    `CREATE TABLE IF NOT EXISTS skill_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`,
+    `CREATE TABLE IF NOT EXISTS turns (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    turn_key TEXT NOT NULL UNIQUE,
+    host TEXT NOT NULL,
+    session_id TEXT NOT NULL,
+    project_tag TEXT,
+    project_name TEXT,
+    provider TEXT,
+    model TEXT,
+    created_at INTEGER NOT NULL,
+    prompt TEXT NOT NULL,
+    replies_json TEXT NOT NULL,
+    redacted INTEGER NOT NULL
+  )`,
+    `CREATE TABLE IF NOT EXISTS tool_calls (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    turn_id INTEGER NOT NULL REFERENCES turns(id) ON DELETE CASCADE,
+    position INTEGER NOT NULL,
+    name TEXT NOT NULL,
+    input TEXT NOT NULL,
+    outcome TEXT NOT NULL,
+    error TEXT
+  )`,
+    `CREATE TABLE IF NOT EXISTS skill_uses (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    turn_id INTEGER NOT NULL REFERENCES turns(id) ON DELETE CASCADE,
+    skill_name TEXT NOT NULL,
+    skill_path TEXT,
+    method TEXT NOT NULL,
+    position INTEGER
+  )`,
+    `CREATE TABLE IF NOT EXISTS skill_settings (
+    skill_path TEXT PRIMARY KEY,
+    mode TEXT NOT NULL,
+    last_review_at INTEGER,
+    uses_at_last_review INTEGER NOT NULL DEFAULT 0
+  )`,
+    // Evidence turn ids are a JSON list with no foreign key: deleting turns keeps proposals.
+    `CREATE TABLE IF NOT EXISTS skill_proposals (
+    id TEXT PRIMARY KEY,
+    kind TEXT NOT NULL,
+    skill_name TEXT NOT NULL,
+    skill_path TEXT,
+    base_hash TEXT,
+    proposed_content TEXT NOT NULL,
+    diff TEXT,
+    summary TEXT,
+    evidence_turn_ids TEXT NOT NULL DEFAULT '[]',
+    state TEXT NOT NULL,
+    applied_by TEXT,
+    drafted_by TEXT,
+    created_at INTEGER NOT NULL,
+    updated_at INTEGER NOT NULL
+  )`,
+    `CREATE TABLE IF NOT EXISTS skill_versions (
+    id INTEGER PRIMARY KEY AUTOINCREMENT,
+    proposal_id TEXT NOT NULL,
+    path TEXT NOT NULL,
+    previous_content TEXT,
+    written_hash TEXT NOT NULL,
+    saved_at INTEGER NOT NULL
+  )`,
+    `CREATE INDEX IF NOT EXISTS turns_created ON turns(created_at)`,
+    `CREATE INDEX IF NOT EXISTS tool_calls_turn ON tool_calls(turn_id, position)`,
+    `CREATE INDEX IF NOT EXISTS skill_uses_turn ON skill_uses(turn_id)`,
+    `CREATE INDEX IF NOT EXISTS skill_uses_name ON skill_uses(skill_name)`,
+    `CREATE INDEX IF NOT EXISTS skill_proposals_state ON skill_proposals(state, skill_name)`,
+];
+/** Adds a column to an existing table; a missing table or a present column is left as it is. */
+async function addMissingColumn(db, table, column) {
+    const exists = await db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = ?", [
+        table,
+    ]);
+    if (!exists)
+        return;
+    try {
+        await db.run(`ALTER TABLE ${table} ADD COLUMN ${column}`);
+    }
+    catch (error) {
+        if (!String(error?.message ?? error).includes("duplicate column"))
+            throw error;
+    }
+}
+/**
+ * Recorded turns, proposals and saved versions in `skills.db`. One instance
+ * per file. Writes check the schema version, so a file from a newer OMMS
+ * stays untouched. Reads on a missing file return empty values and do not
+ * create the file.
+ */
+export class SkillStore {
+    dbPath;
+    connect;
+    readOnly = false;
+    constructor(dbPath, connect = (path) => tursoConnectionManager.getConnection(path)) {
+        this.dbPath = dbPath;
+        this.connect = connect;
+    }
+    /** The open connection. `proposal-store.ts` and `ranking.ts` share it. */
+    db() {
+        return this.connect(this.dbPath);
+    }
+    /** Creates or upgrades the tables. Call it before any write. */
+    async init() {
+        const db = await this.db();
+        await db.execute(SCHEMA[0]);
+        const row = await db.get("SELECT value FROM skill_meta WHERE key = 'schema_version'");
+        const version = row ? Number(row.value) : 0;
+        if (version > SKILL_SCHEMA_VERSION) {
+            this.readOnly = true;
+            return;
+        }
+        this.readOnly = false;
+        // A dev skills.db made before a column has the table without it. The
+        // column must exist before the batch, which creates indexes on the table.
+        await addMissingColumn(db, "skill_uses", "position INTEGER");
+        await addMissingColumn(db, "skill_proposals", "drafted_by TEXT");
+        await db.batch(SCHEMA.slice(1).map((sql) => ({ sql })));
+        if (version < SKILL_SCHEMA_VERSION) {
+            await db.run("INSERT INTO skill_meta (key, value) VALUES ('schema_version', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value", [String(SKILL_SCHEMA_VERSION)]);
+        }
+    }
+    isReadOnly() {
+        return this.readOnly;
+    }
+    writable() {
+        if (this.readOnly) {
+            throw new SkillStoreError("skills.db is from a newer OMMS and is read-only", "read_only");
+        }
+    }
+    /** Opens the file only when it and the `turns` table exist, so a read never creates it. */
+    async existingDb() {
+        if (!existsSync(this.dbPath))
+            return null;
+        const db = await this.db();
+        const table = await db.get("SELECT name FROM sqlite_master WHERE type = 'table' AND name = 'turns'");
+        return table ? db : null;
+    }
+    /**
+     * Writes the turn, its tool calls and its skill uses in one transaction.
+     * Returns false when the `turn_key` is already stored; nothing is written then.
+     */
+    async insertTurn(turn) {
+        this.writable();
+        const db = await this.db();
+        return db.transaction("write", async (tx) => {
+            const inserted = await tx.execute({
+                sql: `INSERT OR IGNORE INTO turns
+          (turn_key, host, session_id, project_tag, project_name, provider, model, created_at, prompt, replies_json, redacted)
+          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+                args: [
+                    turn.turnKey,
+                    turn.host,
+                    turn.sessionId,
+                    turn.projectTag,
+                    turn.projectName,
+                    turn.provider,
+                    turn.model,
+                    turn.createdAt,
+                    turn.prompt,
+                    JSON.stringify(turn.replies),
+                    turn.redacted ? 1 : 0,
+                ],
+            });
+            if (Number(inserted.rowsAffected) === 0)
+                return false;
+            const turnId = inserted.lastInsertRowid;
+            for (const [position, call] of turn.toolCalls.entries()) {
+                await tx.execute({
+                    sql: "INSERT INTO tool_calls (turn_id, position, name, input, outcome, error) VALUES (?, ?, ?, ?, ?, ?)",
+                    args: [turnId, position, call.name, call.input, call.outcome, call.error],
+                });
+            }
+            for (const use of turn.skillUses) {
+                await tx.execute({
+                    sql: "INSERT INTO skill_uses (turn_id, skill_name, skill_path, method, position) VALUES (?, ?, ?, ?, ?)",
+                    args: [turnId, use.name, use.path, use.method, use.position ?? null],
+                });
+            }
+            return true;
+        });
+    }
+    /** Read-only lookup; a missing file gives false and stays missing. */
+    async hasTurn(turnKey) {
+        const db = await this.existingDb();
+        if (!db)
+            return false;
+        return (await db.get("SELECT 1 AS found FROM turns WHERE turn_key = ?", [turnKey])) !== null;
+    }
+    async countTurns() {
+        const db = await this.existingDb();
+        if (!db)
+            return 0;
+        const row = await db.get("SELECT COUNT(*) AS n FROM turns");
+        return Number(row?.n ?? 0);
+    }
+    /** Deletes every turn with its tool calls and skill uses. Proposals and versions stay. */
+    async deleteAllTurns() {
+        this.writable();
+        const db = await this.existingDb();
+        if (!db)
+            return 0;
+        const deleted = await db.transaction("write", async (tx) => {
+            const result = await tx.execute("DELETE FROM turns");
+            return Number(result.rowsAffected);
+        });
+        // A delete only marks pages free, so the turn text stays readable in the
+        // file. VACUUM rewrites it. `secure_delete` would not hold: after each
+        // transaction the libSQL client opens a fresh handle with default pragmas,
+        // which is also why busy_timeout is set again here.
+        await db.execute("PRAGMA busy_timeout = 5000");
+        await db.execute("VACUUM");
+        return deleted;
+    }
+    /** Size of `skills.db` in bytes, or 0 when the file does not exist. */
+    fileSize() {
+        try {
+            return statSync(this.dbPath).size;
+        }
+        catch {
+            return 0;
+        }
+    }
+}

package/dist/services/skill-learning/turn-output.d.ts

@@ -0,0 +1,34 @@
+import type { CaptureWorkUnit } from "../../core/capture.js";
+import { type CaptureDiagnosticsConfig } from "../capture-diagnostics.js";
+/**
+ * The resolved settings for one turn. The caller reads them from `CONFIG` for
+ * the turn's project (`settings-for.ts`), so this module never imports it.
+ */
+export interface SkillTurnSettings {
+    /** `skillLearning` after the project override. */
+    enabled: boolean;
+    /** `skillLearningRedact`. */
+    redact: boolean;
+    /** `skillFolders`, or the default folders when it is unset. */
+    folders: string[];
+    storagePath: string;
+    /** The configured secrets that `redactTraceText` removes. */
+    secrets: CaptureDiagnosticsConfig;
+}
+export type SkillTurnStatus = "disabled" | "private" | "new" | "duplicate";
+/**
+ * The turn's identity in `skills.db`: host, session and the host's id for the
+ * user message. Live capture and history import build the same key.
+ */
+export declare function skillTurnKey(unit: CaptureWorkUnit): string;
+/**
+ * Records one finished turn into `skills.db` (ADR-036). Off: returns at once
+ * and never opens the file. The log record holds counts and sizes only.
+ * Errors are thrown; `handleFinishedTurn` catches them.
+ */
+export declare function recordSkillTurn(unit: CaptureWorkUnit, settings: SkillTurnSettings): Promise<SkillTurnStatus>;
+/**
+ * What `recordSkillTurn` would return, for a dry run. Read-only: a missing
+ * `skills.db` stays missing.
+ */
+export declare function peekSkillTurn(unit: CaptureWorkUnit, settings: SkillTurnSettings): Promise<SkillTurnStatus>;

package/dist/services/skill-learning/turn-output.js

@@ -0,0 +1,98 @@
+import { redactTraceText } from "../capture-diagnostics.js";
+import { log } from "../logger.js";
+import { isFullyPrivate } from "../privacy.js";
+import { getTags } from "../tags.js";
+import { detectSkillUses } from "./skill-detect.js";
+import { getSkillIndex } from "./skill-index.js";
+import { SkillStore, skillsDbPath } from "./store.js";
+// One initialised store for each file, shared by every turn.
+const stores = new Map();
+function storeFor(storagePath) {
+    const path = skillsDbPath(storagePath);
+    let store = stores.get(path);
+    if (!store) {
+        const created = new SkillStore(path);
+        store = created.init().then(() => created);
+        // A failed open is tried again on the next turn.
+        store.catch(() => stores.delete(path));
+        stores.set(path, store);
+    }
+    return store;
+}
+/**
+ * The turn's identity in `skills.db`: host, session and the host's id for the
+ * user message. Live capture and history import build the same key.
+ */
+export function skillTurnKey(unit) {
+    return `${unit.host}:${unit.hostSessionId}:${unit.promptId}`;
+}
+function buildRecord(unit, settings) {
+    // redactTraceText removes `<private>` text first, then secrets.
+    const clean = (text) => settings.redact ? redactTraceText(text, settings.secrets) : text;
+    const project = getTags(unit.projectDirectory).project;
+    // Detection reads the unredacted calls, so a redacted path still matches.
+    const skillUses = detectSkillUses({ prompt: unit.userPrompt, toolCalls: unit.toolCalls }, getSkillIndex(settings.folders));
+    return {
+        turnKey: skillTurnKey(unit),
+        host: unit.host,
+        sessionId: unit.hostSessionId,
+        projectTag: project.tag,
+        projectName: project.projectName ?? null,
+        provider: unit.prompt?.providerId ?? null,
+        model: unit.prompt?.modelId ?? null,
+        createdAt: unit.sourceTimestamp ?? Date.now(),
+        prompt: clean(unit.userPrompt),
+        replies: unit.textResponses.map(clean),
+        redacted: settings.redact,
+        toolCalls: unit.toolCalls.map((call) => ({
+            name: call.name,
+            input: clean(call.input),
+            outcome: call.outcome ?? "unknown",
+            error: call.error === undefined ? null : clean(call.error),
+        })),
+        skillUses,
+    };
+}
+function storedChars(record) {
+    let chars = record.prompt.length;
+    for (const reply of record.replies)
+        chars += reply.length;
+    for (const call of record.toolCalls)
+        chars += call.input.length + (call.error?.length ?? 0);
+    return chars;
+}
+/**
+ * Records one finished turn into `skills.db` (ADR-036). Off: returns at once
+ * and never opens the file. The log record holds counts and sizes only.
+ * Errors are thrown; `handleFinishedTurn` catches them.
+ */
+export async function recordSkillTurn(unit, settings) {
+    if (!settings.enabled)
+        return "disabled";
+    if (settings.redact && isFullyPrivate(unit.userPrompt)) {
+        log("Skill turn skipped", { host: unit.host, code: "private" });
+        return "private";
+    }
+    const record = buildRecord(unit, settings);
+    const store = await storeFor(settings.storagePath);
+    const status = (await store.insertTurn(record)) ? "new" : "duplicate";
+    log("Skill turn recorded", {
+        host: unit.host,
+        toolCalls: record.toolCalls.length,
+        chars: storedChars(record),
+        result: status,
+    });
+    return status;
+}
+/**
+ * What `recordSkillTurn` would return, for a dry run. Read-only: a missing
+ * `skills.db` stays missing.
+ */
+export async function peekSkillTurn(unit, settings) {
+    if (!settings.enabled)
+        return "disabled";
+    if (settings.redact && isFullyPrivate(unit.userPrompt))
+        return "private";
+    const store = new SkillStore(skillsDbPath(settings.storagePath));
+    return (await store.hasTurn(skillTurnKey(unit))) ? "duplicate" : "new";
+}

package/dist/services/skill-learning/turn-summaries.d.ts

@@ -0,0 +1,24 @@
+import type { SkillIndex } from "./skill-index.js";
+import type { SkillStore } from "./store.js";
+/** The Skills page shows this much of a prompt. */
+export declare const PROMPT_PREVIEW_CHARS = 200;
+export declare const RECENT_TURN_LIMIT = 20;
+/** One recorded turn as the Skills page lists it: no replies, no tool input. */
+export interface SkillTurnSummary {
+    id: number;
+    createdAt: number;
+    host: string;
+    projectName: string | null;
+    promptPreview: string;
+    failedToolCalls: number;
+}
+/**
+ * The newest turns that used one skill. `path` is null for a skill that no
+ * watched folder holds. A missing `skills.db` gives no turns and stays missing.
+ */
+export declare function recentSkillTurns(store: SkillStore, skill: {
+    name: string;
+    path: string | null;
+}, index: SkillIndex, limit?: number): Promise<SkillTurnSummary[]>;
+/** The turns with these ids, newest first; ids of deleted turns are left out. */
+export declare function skillTurnsByIds(store: SkillStore, ids: readonly number[]): Promise<SkillTurnSummary[]>;

package/dist/services/skill-learning/turn-summaries.js

@@ -0,0 +1,64 @@
+/** The Skills page shows this much of a prompt. */
+export const PROMPT_PREVIEW_CHARS = 200;
+export const RECENT_TURN_LIMIT = 20;
+const COLUMNS = `t.id, t.created_at, t.host, t.project_name,
+  substr(t.prompt, 1, ${PROMPT_PREVIEW_CHARS}) AS preview,
+  (SELECT COUNT(*) FROM tool_calls tc WHERE tc.turn_id = t.id AND tc.outcome = 'error') AS failed`;
+function toSummary(row) {
+    return {
+        id: Number(row.id),
+        createdAt: Number(row.created_at),
+        host: String(row.host),
+        projectName: row.project_name == null ? null : String(row.project_name),
+        promptPreview: String(row.preview),
+        failedToolCalls: Number(row.failed),
+    };
+}
+// The same use-to-skill rule as the ranking: a recorded path counts while the
+// index lists it; a use with no path belongs to the one watched skill with its name.
+const BY_PATH = `
+SELECT ${COLUMNS} FROM turns t WHERE t.id IN (
+  SELECT su.turn_id FROM skill_uses su
+  WHERE su.skill_path = ? OR (su.skill_path IS NULL AND su.skill_name = ? AND ? = 1)
+) ORDER BY t.created_at DESC, t.id DESC LIMIT ?`;
+const BY_NAME = `
+SELECT ${COLUMNS} FROM turns t WHERE t.id IN (
+  SELECT su.turn_id FROM skill_uses su
+  WHERE su.skill_name = ? AND (
+    (su.skill_path IS NOT NULL AND su.skill_path NOT IN (SELECT value FROM json_each(?)))
+    OR (su.skill_path IS NULL AND ? = 0)
+  )
+) ORDER BY t.created_at DESC, t.id DESC LIMIT ?`;
+/**
+ * The newest turns that used one skill. `path` is null for a skill that no
+ * watched folder holds. A missing `skills.db` gives no turns and stays missing.
+ */
+export async function recentSkillTurns(store, skill, index, limit = RECENT_TURN_LIMIT) {
+    const db = await store.existingDb();
+    if (!db)
+        return [];
+    const named = index.entries.filter((e) => e.name === skill.name);
+    const rows = skill.path
+        ? await db.all(BY_PATH, [
+            skill.path,
+            skill.name,
+            named.length === 1 && named[0].path === skill.path ? 1 : 0,
+            limit,
+        ])
+        : await db.all(BY_NAME, [
+            skill.name,
+            JSON.stringify(index.entries.map((e) => e.path)),
+            named.length === 1 ? 1 : 0,
+            limit,
+        ]);
+    return rows.map(toSummary);
+}
+/** The turns with these ids, newest first; ids of deleted turns are left out. */
+export async function skillTurnsByIds(store, ids) {
+    const db = await store.existingDb();
+    if (!db || ids.length === 0)
+        return [];
+    const rows = await db.all(`SELECT ${COLUMNS} FROM turns t WHERE t.id IN (SELECT value FROM json_each(?))
+     ORDER BY t.created_at DESC, t.id DESC`, [JSON.stringify(ids)]);
+    return rows.map(toSummary);
+}

package/dist/services/skill-learning/web-review.d.ts

@@ -0,0 +1,18 @@
+import type { ChatRuntimeConfig } from "../chat/chat-runtime.js";
+import { type AutoReviewDeps } from "./auto-review.js";
+/** The settings the web app's review loop reads. The caller passes a getter for `CONFIG`. */
+export interface WebSkillReviewConfig extends ChatRuntimeConfig {
+    skillLearning?: boolean;
+    skillFolders?: string[];
+    claudeConfigDir?: string;
+    skillStoragePath?: string;
+}
+/**
+ * The deps for each tick. `getConfig` is called at the start of every tick,
+ * because a config reload replaces the `CONFIG` object: a saved Settings
+ * change then takes effect without a restart. Null while skill learning is
+ * off or before any turn is recorded; `skills.db` is never created here.
+ */
+export declare function webReviewDeps(getConfig: () => WebSkillReviewConfig): () => Promise<AutoReviewDeps | null>;
+/** Starts the automatic skill review in the web app that owns the port. Once per process. */
+export declare function startWebSkillReview(getConfig: () => WebSkillReviewConfig): void;

package/dist/services/skill-learning/web-review.js

@@ -0,0 +1,44 @@
+import { skillReviewLease, startAutoReviewLoop } from "./auto-review.js";
+import { skillStoragePath, watchedSkillFolders } from "./skill-folders.js";
+import { getSkillIndex } from "./skill-index.js";
+import { skillModel } from "./skill-model.js";
+import { SkillStore, skillsDbPath } from "./store.js";
+let stopLoop = null;
+const opened = new Map();
+/** The store and lease for this storage path, initialised once. */
+function openStore(storagePath) {
+    let entry = opened.get(storagePath);
+    if (!entry) {
+        const store = new SkillStore(skillsDbPath(storagePath));
+        entry = store.init().then(() => ({ store, lease: skillReviewLease(store) }));
+        entry.catch(() => opened.delete(storagePath));
+        opened.set(storagePath, entry);
+    }
+    return entry;
+}
+/**
+ * The deps for each tick. `getConfig` is called at the start of every tick,
+ * because a config reload replaces the `CONFIG` object: a saved Settings
+ * change then takes effect without a restart. Null while skill learning is
+ * off or before any turn is recorded; `skills.db` is never created here.
+ */
+export function webReviewDeps(getConfig) {
+    return async () => {
+        const config = getConfig();
+        if (!config.skillLearning)
+            return null;
+        const storagePath = skillStoragePath(config);
+        if (!opened.has(storagePath)) {
+            const probe = new SkillStore(skillsDbPath(storagePath));
+            if (!(await probe.existingDb()))
+                return null;
+        }
+        const folders = watchedSkillFolders(config);
+        const { store, lease } = await openStore(storagePath);
+        return { store, lease, folders, index: getSkillIndex(folders), model: skillModel(config) };
+    };
+}
+/** Starts the automatic skill review in the web app that owns the port. Once per process. */
+export function startWebSkillReview(getConfig) {
+    stopLoop ??= startAutoReviewLoop(webReviewDeps(getConfig));
+}

package/dist/services/skill-routes.d.ts

@@ -0,0 +1,20 @@
+import type { RouteResult } from "./chat/chat-routes.js";
+/**
+ * What the web server lends the skill routes: its folder, and a way to start
+ * Record past turns in its own import job slot. The importer stays behind
+ * web-server.ts, the one services file allowed to load it.
+ */
+export interface SkillRouteContext {
+    cwd: string;
+    startRecordPast: (cwd: string) => Promise<{
+        id: string;
+    }>;
+}
+type Body = Record<string, unknown>;
+/**
+ * The Skills page routes under `/api/skills`. Returns undefined for any other
+ * path. The web server applies the origin, token and JSON checks before this.
+ * Reads never create `skills.db`; a write opens it with `init` first.
+ */
+export declare function routeSkills(path: string, method: string, url: URL, readBody: () => Promise<Body>, ctx: SkillRouteContext): Promise<RouteResult | undefined>;
+export {};