@kisev/memomatic 1.0.0-dev.100.g21366c4ad740

package/dist/settings.js

@@ -0,0 +1,189 @@
+import { readFile } from "node:fs/promises";
+import { createHash } from "node:crypto";
+export const defaultSettings = () => ({
+    embedding: null,
+    sessions: {
+        model: null,
+        variant: null,
+        timeoutMs: 180000,
+        maxDurationMs: 0,
+        maxSessions: 0,
+        maxChars: 24000,
+        retries: 1,
+        idleMs: 600000,
+        opencodeUrl: null,
+    },
+    dream: {
+        model: null,
+        variant: null,
+        minScore: 0.45,
+        minUseful: 2,
+        minUniqueQueries: 2,
+        maxCandidates: 12,
+        memoryBudgetLines: 80,
+        maxPriorEntryLoss: 0.25,
+        timeoutMs: 180000,
+        maxDurationMs: 0,
+        retries: 1,
+    },
+    search: { maxResults: 6, minScore: 0.35, minSemanticScore: 0.45, halfLifeDays: 30 },
+    archive: { enabled: false, days: 60 },
+});
+function mergeSection(base, override) {
+    if (!override || typeof override !== "object")
+        return base;
+    const merged = { ...base };
+    for (const [key, value] of Object.entries(override)) {
+        if (key in base && value !== null && value !== undefined)
+            merged[key] = value;
+    }
+    return merged;
+}
+const SESSIONS_KEYS = [
+    "model",
+    "variant",
+    "timeoutMs",
+    "maxDurationMs",
+    "maxSessions",
+    "maxChars",
+    "retries",
+    "idleMs",
+    "opencodeUrl",
+];
+export function normalizeSettings(raw) {
+    const defaults = defaultSettings();
+    const source = (raw ?? {});
+    const embedding = source.embedding;
+    const search = mergeSection(defaults.search, source.search);
+    const dream = mergeSection(defaults.dream, source.dream);
+    // Legacy single-model configs put extraction knobs under `dream`; adopt them
+    // for `sessions` unless the new section overrides the key.
+    const migrated = {
+        ...(source.sessions ?? {}),
+    };
+    const legacyDream = source.dream ?? {};
+    for (const key of SESSIONS_KEYS)
+        if (migrated[key] === undefined && legacyDream[key] !== undefined)
+            migrated[key] = legacyDream[key];
+    const sessions = mergeSection(defaults.sessions, migrated);
+    for (const key of [
+        "timeoutMs",
+        "maxDurationMs",
+        "maxSessions",
+        "maxChars",
+        "retries",
+        "idleMs",
+    ])
+        if (!Number.isSafeInteger(sessions[key]) || sessions[key] < 0)
+            throw new Error(`invalid sessions.${key}`);
+    for (const key of ["timeoutMs", "maxDurationMs", "retries"])
+        if (!Number.isSafeInteger(dream[key]) || dream[key] < 0)
+            throw new Error(`invalid dream.${key}`);
+    if (sessions.timeoutMs < 1 ||
+        sessions.maxChars < 256 ||
+        sessions.maxChars > 200000 ||
+        sessions.retries > 5)
+        throw new Error("invalid sessions request limits");
+    if (dream.timeoutMs < 1 || dream.retries > 5)
+        throw new Error("invalid dream request limits");
+    for (const [section, value] of [
+        ["sessions", sessions],
+        ["dream", dream],
+    ])
+        if (value.model !== null &&
+            (typeof value.model !== "string" || !/^[^/]+\/.+/.test(value.model)))
+            throw new Error(`${section}.model must use provider/model format`);
+    if (sessions.variant !== null && typeof sessions.variant !== "string")
+        throw new Error("sessions.variant must be a string or null");
+    if (dream.variant !== null && typeof dream.variant !== "string")
+        throw new Error("dream.variant must be a string or null");
+    if (sessions.opencodeUrl !== null && typeof sessions.opencodeUrl !== "string")
+        throw new Error("sessions.opencodeUrl must be a string or null");
+    if (!Number.isInteger(search.maxResults) ||
+        search.maxResults < 1 ||
+        search.maxResults > 200 ||
+        !Number.isFinite(search.halfLifeDays) ||
+        search.halfLifeDays <= 0 ||
+        ![search.minScore, search.minSemanticScore].every((value) => Number.isFinite(value) && value >= 0 && value <= 1))
+        throw new Error("invalid search settings");
+    return {
+        embedding: embedding && typeof embedding.url === "string" && typeof embedding.model === "string"
+            ? {
+                url: embedding.url,
+                model: embedding.model,
+                ...(typeof embedding.queryInstruction === "string"
+                    ? { queryInstruction: embedding.queryInstruction }
+                    : {}),
+            }
+            : null,
+        sessions,
+        dream,
+        search,
+        archive: mergeSection(defaults.archive, source.archive),
+    };
+}
+export async function loadSettings(settingsFile) {
+    try {
+        return normalizeSettings(JSON.parse(await readFile(settingsFile, "utf8")));
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return defaultSettings();
+        throw error;
+    }
+}
+export async function embedTexts(settings, texts, purpose = "document", signal) {
+    if (!settings.embedding || !texts.length)
+        return null;
+    const response = await fetch(settings.embedding.url, {
+        body: JSON.stringify({
+            input: texts.map((text) => (purpose === "query" ? embeddingQuery(settings, text) : text)),
+            model: settings.embedding.model,
+        }),
+        headers: { "content-type": "application/json" },
+        method: "POST",
+        signal: signal
+            ? AbortSignal.any([signal, AbortSignal.timeout(30_000)])
+            : AbortSignal.timeout(30_000),
+    });
+    if (!response.ok)
+        throw new Error(`embedding request failed: ${response.status} ${await response.text()}`);
+    const payload = (await response.json());
+    const list = payload.data ?? [];
+    if (list.length !== texts.length)
+        throw new Error("embedding response is incomplete");
+    const byIndex = new Map();
+    for (const item of list) {
+        if (!Array.isArray(item.embedding) ||
+            !item.embedding.length ||
+            !item.embedding.every((value) => typeof value === "number" && Number.isFinite(value)) ||
+            !item.embedding.some((value) => value !== 0))
+            throw new Error("embedding response is malformed");
+        const index = item.index ?? byIndex.size;
+        if (!Number.isInteger(index) || index < 0 || index >= texts.length || byIndex.has(index))
+            throw new Error("embedding response has invalid indices");
+        byIndex.set(index, item.embedding);
+    }
+    const vectors = texts.map((_, index) => Float32Array.from(byIndex.get(index)));
+    if (!vectors.every((vector) => vector.length === vectors[0].length && vector.every(Number.isFinite)))
+        throw new Error("embedding response has incompatible vectors");
+    return vectors;
+}
+export const MEMORY_QUERY_INSTRUCTION = "Given a search query, retrieve relevant personal memory entries that answer the query.";
+export function embeddingQuery(settings, query) {
+    const embedding = settings.embedding;
+    const instruction = embedding?.queryInstruction ??
+        (/qwen3[-_]?embedding/i.test(embedding?.model ?? "") ? MEMORY_QUERY_INSTRUCTION : "");
+    return instruction ? `Instruct: ${instruction}\nQuery:${query}` : query;
+}
+export function embeddingFingerprint(settings) {
+    return createHash("sha256")
+        .update(JSON.stringify(settings.embedding
+        ? {
+            url: settings.embedding.url,
+            model: settings.embedding.model,
+            documentFormat: "raw-v1",
+        }
+        : null))
+        .digest("hex");
+}

package/dist/store.d.ts

@@ -0,0 +1,55 @@
+import { DatabaseSync } from "node:sqlite";
+export type EntryKind = "curated" | "user" | "episodic";
+export type IndexedEntry = {
+    stableId: string;
+    file: string;
+    line: number;
+    kind: EntryKind;
+    key: string | null;
+    text: string;
+    trigger: string[];
+    importance: number;
+    pinned: boolean;
+    project: string | null;
+    origin: "user" | "agent" | null;
+    observedAt: number | null;
+    status: "active" | null;
+    source: string | null;
+};
+export type VectorRow = {
+    stableId: string;
+    dim: number;
+    data: Float32Array;
+};
+export declare function stableIdFor(file: string, text: string, key: string | null): string;
+export declare class MemoryStore {
+    readonly db: DatabaseSync;
+    private readonly fts;
+    private constructor();
+    private migrate;
+    static open(indexFile: string): Promise<MemoryStore>;
+    private static copy;
+    fork(): MemoryStore;
+    static preview(indexFile: string): Promise<MemoryStore>;
+    private ensureFts;
+    hasFts(): boolean;
+    replaceEntries(entries: IndexedEntry[], manageTransaction?: boolean): void;
+    replaceIndex(entries: IndexedEntry[], vectors: Float32Array[] | null, fingerprint: string): void;
+    allEntries(): IndexedEntry[];
+    ftsSearch(query: string, limit: number): Map<string, number>;
+    markSurfaced(stableIds: string[], query: string, now?: number): void;
+    markUseful(stableIds: string[], now?: number): void;
+    usageFor(stableId: string): {
+        surfaced: number;
+        useful: number;
+        queries: string[];
+        days: string[];
+        lastUsefulAt: number | null;
+    };
+    setVector(stableId: string, vector: Float32Array): void;
+    vectors(): VectorRow[];
+    getMeta(key: string): string | null;
+    setMeta(key: string, value: string): void;
+    close(): void;
+}
+export declare function cosineSimilarity(left: Float32Array, right: Float32Array): number;

package/dist/store.js

@@ -0,0 +1,286 @@
+import { createHash } from "node:crypto";
+import { mkdir, stat } from "node:fs/promises";
+import { dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+export function stableIdFor(file, text, key) {
+    if (key)
+        return `key:${key}`;
+    return `sha:${createHash("sha256").update(`${file}\n${text}`).digest("hex").slice(0, 24)}`;
+}
+const SCHEMA = `
+CREATE TABLE IF NOT EXISTS entries (
+  stable_id TEXT PRIMARY KEY,
+  file TEXT NOT NULL,
+  line INTEGER NOT NULL,
+  kind TEXT NOT NULL,
+  key TEXT,
+  text TEXT NOT NULL,
+  trigger_phrases TEXT NOT NULL DEFAULT '[]',
+  importance INTEGER NOT NULL DEFAULT 5,
+  pinned INTEGER NOT NULL DEFAULT 0,
+  project TEXT,
+  origin TEXT,
+  observed_at INTEGER,
+  status TEXT
+);
+CREATE TABLE IF NOT EXISTS usage (
+  stable_id TEXT PRIMARY KEY,
+  surfaced INTEGER NOT NULL DEFAULT 0,
+  useful INTEGER NOT NULL DEFAULT 0,
+  queries TEXT NOT NULL DEFAULT '[]',
+  days TEXT NOT NULL DEFAULT '[]',
+  last_useful_at INTEGER
+);
+CREATE TABLE IF NOT EXISTS vectors (
+  stable_id TEXT PRIMARY KEY,
+  dim INTEGER NOT NULL,
+  data BLOB NOT NULL
+);
+CREATE TABLE IF NOT EXISTS meta (
+  key TEXT PRIMARY KEY,
+  value TEXT NOT NULL
+);
+`;
+export class MemoryStore {
+    db;
+    fts;
+    constructor(indexFile) {
+        this.db = new DatabaseSync(indexFile);
+        this.db.exec("PRAGMA journal_mode = WAL;");
+        this.db.exec(SCHEMA);
+        this.migrate();
+        this.fts = this.ensureFts();
+    }
+    migrate() {
+        const columns = this.db.prepare("PRAGMA table_info(entries);").all();
+        if (!columns.some((column) => column.name === "source"))
+            this.db.exec("ALTER TABLE entries ADD COLUMN source TEXT;");
+    }
+    static async open(indexFile) {
+        await mkdir(dirname(indexFile), { mode: 0o700, recursive: true });
+        return new MemoryStore(indexFile);
+    }
+    static copy(database) {
+        const copy = new MemoryStore(":memory:");
+        try {
+            for (const table of ["entries", "usage", "vectors", "meta"]) {
+                for (const row of database.prepare(`SELECT * FROM ${table}`).all()) {
+                    const columns = Object.keys(row);
+                    copy.db
+                        .prepare(`INSERT INTO ${table} (${columns.join(",")}) VALUES (${columns.map(() => "?").join(",")})`)
+                        .run(...Object.values(row));
+                }
+            }
+            if (copy.fts)
+                copy.db.exec("INSERT INTO entries_fts(stable_id,text) SELECT stable_id,text FROM entries");
+            return copy;
+        }
+        catch (error) {
+            copy.close();
+            throw error;
+        }
+    }
+    fork() {
+        return MemoryStore.copy(this.db);
+    }
+    static async preview(indexFile) {
+        try {
+            await stat(indexFile);
+        }
+        catch (error) {
+            if (error.code === "ENOENT")
+                return new MemoryStore(":memory:");
+            throw error;
+        }
+        const database = new DatabaseSync(indexFile, { readOnly: true });
+        try {
+            return MemoryStore.copy(database);
+        }
+        finally {
+            database.close();
+        }
+    }
+    ensureFts() {
+        try {
+            this.db.exec("CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5(stable_id UNINDEXED, text);");
+            return true;
+        }
+        catch {
+            return false;
+        }
+    }
+    hasFts() {
+        return this.fts;
+    }
+    replaceEntries(entries, manageTransaction = true) {
+        if (manageTransaction)
+            this.db.exec("BEGIN IMMEDIATE;");
+        try {
+            this.db.exec("DELETE FROM entries;");
+            if (this.fts)
+                this.db.exec("DELETE FROM entries_fts;");
+            const seen = new Set();
+            for (const entry of entries) {
+                if (seen.has(entry.stableId))
+                    continue;
+                seen.add(entry.stableId);
+                this.db
+                    .prepare(`INSERT INTO entries (stable_id, file, line, kind, key, text, trigger_phrases, importance, pinned, project, origin, observed_at, status, source)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
+                    .run(entry.stableId, entry.file, entry.line, entry.kind, entry.key, entry.text, JSON.stringify(entry.trigger), entry.importance, entry.pinned ? 1 : 0, entry.project, entry.origin, entry.observedAt, entry.status, entry.source ?? null);
+                if (this.fts)
+                    this.db
+                        .prepare("INSERT INTO entries_fts (stable_id, text) VALUES (?, ?);")
+                        .run(entry.stableId, entry.text);
+            }
+            this.db.exec("DELETE FROM usage WHERE stable_id NOT IN (SELECT stable_id FROM entries);");
+            this.db.exec("DELETE FROM vectors WHERE stable_id NOT IN (SELECT stable_id FROM entries);");
+            if (manageTransaction)
+                this.db.exec("COMMIT;");
+        }
+        catch (error) {
+            if (manageTransaction)
+                this.db.exec("ROLLBACK;");
+            throw error;
+        }
+    }
+    replaceIndex(entries, vectors, fingerprint) {
+        this.db.exec("BEGIN IMMEDIATE");
+        try {
+            this.replaceEntries(entries, false);
+            this.db.exec("DELETE FROM vectors");
+            if (vectors)
+                entries.forEach((entry, index) => this.setVector(entry.stableId, vectors[index]));
+            this.setMeta("embeddingFingerprint", fingerprint);
+            this.db.exec("COMMIT");
+        }
+        catch (error) {
+            this.db.exec("ROLLBACK");
+            throw error;
+        }
+    }
+    allEntries() {
+        return this.db
+            .prepare("SELECT stable_id, file, line, kind, key, text, trigger_phrases, importance, pinned, project, origin, observed_at, status, source FROM entries ORDER BY observed_at ASC")
+            .all()
+            .map((row) => {
+            const value = row;
+            return {
+                stableId: value.stable_id,
+                file: value.file,
+                line: value.line,
+                kind: value.kind,
+                key: value.key ?? null,
+                text: value.text,
+                trigger: JSON.parse(value.trigger_phrases),
+                importance: value.importance,
+                pinned: value.pinned === 1,
+                project: value.project ?? null,
+                origin: value.origin ?? null,
+                observedAt: value.observed_at ?? null,
+                status: value.status ?? null,
+                source: value.source ?? null,
+            };
+        });
+    }
+    ftsSearch(query, limit) {
+        if (!this.fts)
+            return new Map();
+        const tokens = query
+            .toLowerCase()
+            .split(/[^\p{L}\p{N}]+/u)
+            .filter((token) => token.length >= 2)
+            .slice(0, 12);
+        if (!tokens.length)
+            return new Map();
+        const pattern = tokens.map((token) => `"${token.replace(/"/g, "")}"*`).join(" OR ");
+        const rows = this.db
+            .prepare(`SELECT stable_id, rank FROM entries_fts WHERE entries_fts MATCH ? ORDER BY rank LIMIT ?;`)
+            .all(pattern, limit);
+        const result = new Map();
+        for (const row of rows) {
+            const quality = -row.rank;
+            const score = quality <= 0 ? 0.1 : Math.min(1, 0.5 + quality * 0.25);
+            result.set(row.stable_id, score);
+        }
+        return result;
+    }
+    markSurfaced(stableIds, query, now = Date.now()) {
+        const queryHash = createHash("sha256").update(query.toLowerCase()).digest("hex").slice(0, 16);
+        const day = new Date(now).toISOString().slice(0, 10);
+        const read = this.db.prepare("SELECT queries, days FROM usage WHERE stable_id = ?;");
+        const upsert = this.db.prepare(`INSERT INTO usage (stable_id, surfaced, useful, queries, days) VALUES (?, 1, 0, ?, ?)
+      ON CONFLICT(stable_id) DO UPDATE SET
+        surfaced = usage.surfaced + 1,
+        queries = excluded.queries,
+        days = excluded.days;`);
+        for (const stableId of stableIds) {
+            const row = read.get(stableId);
+            const queries = row ? JSON.parse(row.queries) : [];
+            const days = row ? JSON.parse(row.days) : [];
+            if (!queries.includes(queryHash) && queries.length < 32)
+                queries.push(queryHash);
+            if (!days.includes(day))
+                days.push(day);
+            upsert.run(stableId, JSON.stringify(queries), JSON.stringify(days));
+        }
+    }
+    markUseful(stableIds, now = Date.now()) {
+        const statement = this.db.prepare(`INSERT INTO usage (stable_id, useful, last_useful_at) VALUES (?, 1, ?)
+      ON CONFLICT(stable_id) DO UPDATE SET useful = useful + 1, last_useful_at = ?;`);
+        for (const stableId of stableIds)
+            statement.run(stableId, now, now);
+    }
+    usageFor(stableId) {
+        const row = this.db.prepare("SELECT * FROM usage WHERE stable_id = ?").get(stableId);
+        if (!row)
+            return { surfaced: 0, useful: 0, queries: [], days: [], lastUsefulAt: null };
+        return {
+            surfaced: row.surfaced,
+            useful: row.useful,
+            queries: JSON.parse(row.queries),
+            days: JSON.parse(row.days),
+            lastUsefulAt: row.last_useful_at ?? null,
+        };
+    }
+    setVector(stableId, vector) {
+        this.db
+            .prepare(`INSERT INTO vectors (stable_id, dim, data) VALUES (?, ?, ?)
+        ON CONFLICT(stable_id) DO UPDATE SET dim = excluded.dim, data = excluded.data;`)
+            .run(stableId, vector.length, Buffer.from(vector.buffer, vector.byteOffset, vector.byteLength));
+    }
+    vectors() {
+        return this.db.prepare("SELECT stable_id, dim, data FROM vectors").all().map((row) => ({
+            stableId: row.stable_id,
+            dim: row.dim,
+            data: new Float32Array(row.data.buffer.slice(row.data.byteOffset, row.data.byteOffset + row.data.byteLength)),
+        }));
+    }
+    getMeta(key) {
+        const row = this.db.prepare("SELECT value FROM meta WHERE key = ?").get(key);
+        return row?.value ?? null;
+    }
+    setMeta(key, value) {
+        this.db
+            .prepare(`INSERT INTO meta (key, value) VALUES (?, ?)
+        ON CONFLICT(key) DO UPDATE SET value = excluded.value;`)
+            .run(key, value);
+    }
+    close() {
+        this.db.close();
+    }
+}
+export function cosineSimilarity(left, right) {
+    const length = Math.min(left.length, right.length);
+    let dot = 0;
+    let leftNorm = 0;
+    let rightNorm = 0;
+    for (let index = 0; index < length; index += 1) {
+        dot += left[index] * right[index];
+        leftNorm += left[index] * left[index];
+        rightNorm += right[index] * right[index];
+    }
+    if (!leftNorm || !rightNorm)
+        return 0;
+    return dot / (Math.sqrt(leftNorm) * Math.sqrt(rightNorm));
+}

package/dist/visibility.d.ts

@@ -0,0 +1,7 @@
+export type EntryVisibility = "team" | "personal";
+/**
+ * Usage-level visibility derived from the entry source. Team-prefixed skills,
+ * the GitLab collector, and spec-manage produce entries citable in
+ * team-facing artifacts; every other source stays personal-only.
+ */
+export declare function visibilityForSource(source: string | null | undefined): EntryVisibility;

package/dist/visibility.js

@@ -0,0 +1,13 @@
+const TEAM_CITABLE_SOURCES = ["gitlab", "spec-manage"];
+/**
+ * Usage-level visibility derived from the entry source. Team-prefixed skills,
+ * the GitLab collector, and spec-manage produce entries citable in
+ * team-facing artifacts; every other source stays personal-only.
+ */
+export function visibilityForSource(source) {
+    if (!source)
+        return "personal";
+    if (TEAM_CITABLE_SOURCES.includes(source) || source.startsWith("team-"))
+        return "team";
+    return "personal";
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "@kisev/memomatic",
+  "version": "1.0.0-dev.100.g21366c4ad740",
+  "description": "Personal learning memory for agents: tiered Markdown corpus, rebuildable index, dream sweep.",
+  "homepage": "https://github.com/kisev/skills#readme",
+  "bugs": {
+    "url": "https://github.com/kisev/skills/issues"
+  },
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kisev/skills.git",
+    "directory": "apps/memomatic"
+  },
+  "bin": {
+    "memomatic": "./dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "assets",
+    "README.md",
+    "README.ru.md"
+  ],
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "scripts": {
+    "build": "node scripts/clean-dist.mjs && tsc --project tsconfig.json",
+    "test": "node --test test/*.test.mjs",
+    "pack:check": "node test/pack-allowlist.mjs"
+  },
+  "dependencies": {
+    "@kisev/safe-fs": "1.0.0-dev.100.g21366c4ad740",
+    "commander": "15.0.0",
+    "jsonc-parser": "3.3.1"
+  },
+  "devDependencies": {
+    "@types/node": "22.15.30",
+    "typescript": "7.0.2"
+  },
+  "engines": {
+    "node": ">=22.13"
+  }
+}