@wirecat/cli-core 0.499.0 → 0.501.0

package/dist/store/sqlite/memories.d.ts

@@ -1,5 +1,6 @@
 import { MEMORY_KINDS, MEMORY_SCOPES, type MEMORY_STATUSES } from "../../domain/knowledge.js";
 import { type Actor } from "./actors.js";
+import { type EvidenceSource } from "./evidence-sources.js";
 import { type NoteIndexState } from "./note-index.js";
 import type { StoreContext } from "./open.js";
 /** Who wrote it: the owner, or an agent by its bot handle and the model it ran on. */
@@ -14,6 +15,10 @@ export interface Memory {
     body: string;
     /** What it is about, as a reference; `null` for a general fact. */
     subject: string | null;
+    /** What the subject is called; `null` for the owner, who has no name in the store, and for a nameless row. */
+    subjectName: string | null;
+    /** The subject is the owner. */
+    subjectIsOwner: boolean;
     /** `name` is a bot's handle; `null` for a person. */
     author: {
         type: Actor["type"];
@@ -59,6 +64,17 @@ export interface MemoriesStore {
     confirm(reference: string): Promise<Memory>;
     /** Its evidence no longer holds. */
     markStale(reference: string): Promise<Memory>;
+    /**
+     * Erases the memory: its row, its text in the search and meaning indexes, and every link from or to it. A memory
+     * that replaced it no longer names it. A backup made before keeps it.
+     */
+    remove(reference: string): Promise<{
+        ref: string;
+        removed: true;
+        links: number;
+    }>;
+    /** Each reference it rests on, read back; one deleted since is `missing`. */
+    sources(reference: string): Promise<EvidenceSource[]>;
     /** Indexes what was written since, then finds memories holding every word or a longer one it starts, best first. */
     search(text: string, options?: {
         subject?: string;
@@ -82,5 +98,17 @@ export declare const evidenceOf: (context: StoreContext, from: {
     type: string;
     id: number;
 }) => string[];
+/** What a memory's or fact's subject is called, for a reader who should not have to look the reference up. */
+export declare const subjectOf: (database: StoreContext["database"], row: Row) => Pick<Memory, "subject" | "subjectName" | "subjectIsOwner">;
+/** Counts every link from or to a row and drops the ones its delete trigger does not: those pointing at it. */
+export declare const dropLinksOf: (database: StoreContext["database"], thing: {
+    type: string;
+    id: number;
+}) => number;
+/**
+ * Runs a delete so the text does not linger in the file: freed pages are zeroed, the words indexes merged so no
+ * segment still holds the erased words, and the write-ahead log emptied when no reader holds it.
+ */
+export declare const erase: <T>(database: StoreContext["database"], work: () => T) => T;
 export declare const memoriesStoreOver: (context: StoreContext) => MemoriesStore;
 export {};

package/dist/store/sqlite/memories.js

@@ -2,6 +2,7 @@ import { CliError } from "@wirecat/cli-base";
 import { MEMORY_KINDS, MEMORY_SCOPES } from "../../domain/knowledge.js";
 import { normalize } from "../normalize.js";
 import { botNamed, ownerPerson } from "./actors.js";
+import { evidenceSourcesOf } from "./evidence-sources.js";
 import { linkKind } from "./link-kinds.js";
 import { CORPORA, corpusIndexState, drainCorpus } from "./note-index.js";
 import { requiredThing } from "./notes.js";
@@ -42,6 +43,55 @@ export const evidenceOf = (context, from) => context.database
     .prepare(`SELECT to_type, to_id FROM links WHERE from_type = ? AND from_id = ? AND kind = ${linkKind("evidence")} AND to_id IS NOT NULL ORDER BY id`)
     .all(from.type, from.id)
     .flatMap((row) => referenceOfThing(context.database, storedThing(row.to_type, row.to_id)) ?? []);
+const SUBJECT_NAMES = {
+    person: "SELECT p.owner, coalesce(p.name, (SELECT a.name FROM aliases a WHERE a.aliasable_type = 'person' " +
+        "AND a.aliasable_id = p.id ORDER BY a.display DESC, a.id LIMIT 1), (SELECT i.name FROM identity_links l " +
+        "JOIN identities i ON i.id = l.identity_id WHERE l.person_id = p.id AND i.name IS NOT NULL ORDER BY i.id LIMIT 1)) " +
+        "AS name FROM persons p WHERE p.id = ?",
+    project: "SELECT 0 AS owner, name FROM projects WHERE id = ?",
+    organization: "SELECT 0 AS owner, name FROM organizations WHERE id = ?",
+};
+/** What a memory's or fact's subject is called, for a reader who should not have to look the reference up. */
+export const subjectOf = (database, row) => {
+    if (row.subject_type == null)
+        return { subject: null, subjectName: null, subjectIsOwner: false };
+    const thing = storedThing(row.subject_type, row.subject_id);
+    const read = SUBJECT_NAMES[thing.type];
+    const named = read === undefined ? undefined : database.prepare(read).get(thing.id);
+    return {
+        subject: referenceOfThing(database, thing) ?? null,
+        subjectName: named?.name == null ? null : String(named.name),
+        subjectIsOwner: Number(named?.owner) === 1,
+    };
+};
+/** Counts every link from or to a row and drops the ones its delete trigger does not: those pointing at it. */
+export const dropLinksOf = (database, thing) => {
+    const count = Number(database
+        .prepare("SELECT count(*) AS n FROM links WHERE (from_type = ? AND from_id = ?) OR (to_type = ? AND to_id = ?)")
+        .get(thing.type, thing.id, thing.type, thing.id)?.n);
+    database.prepare("DELETE FROM links WHERE to_type = ? AND to_id = ?").run(thing.type, thing.id);
+    return count;
+};
+/**
+ * Runs a delete so the text does not linger in the file: freed pages are zeroed, the words indexes merged so no
+ * segment still holds the erased words, and the write-ahead log emptied when no reader holds it.
+ */
+export const erase = (database, work) => {
+    database.exec("PRAGMA secure_delete = ON");
+    try {
+        const done = write(database, () => {
+            const result = work();
+            for (const index of ["memory_words", "memory_stems"])
+                database.prepare(`INSERT INTO ${index} (${index}) VALUES ('optimize')`).run();
+            return result;
+        });
+        database.exec("PRAGMA wal_checkpoint(TRUNCATE)");
+        return done;
+    }
+    finally {
+        database.exec("PRAGMA secure_delete = OFF");
+    }
+};
 export const memoriesStoreOver = (context) => {
     const { database, now } = context;
     const stemmerFor = stemmerCache();
@@ -50,9 +100,7 @@ export const memoriesStoreOver = (context) => {
         ref: `memory:${row.id}`,
         kind: row.kind,
         body: String(row.body),
-        subject: row.subject_type == null
-            ? null
-            : (referenceOfThing(database, storedThing(row.subject_type, row.subject_id)) ?? null),
+        ...subjectOf(database, row),
         author: authorOf(row),
         model: row.model == null ? null : String(row.model),
         confidence: row.confidence == null ? null : Number(row.confidence),
@@ -127,6 +175,17 @@ export const memoriesStoreOver = (context) => {
         },
         confirm: async (reference) => setStatus(reference, "confirmed", true),
         markStale: async (reference) => setStatus(reference, "stale", false),
+        remove: async (reference) => {
+            const id = Number(rowOf(reference).id);
+            const links = erase(database, () => {
+                database.prepare("UPDATE memories SET supersedes_id = NULL WHERE supersedes_id = ?").run(id);
+                const count = dropLinksOf(database, { type: "memory", id });
+                database.prepare("DELETE FROM memories WHERE id = ?").run(id);
+                return count;
+            });
+            return { ref: `memory:${id}`, removed: true, links };
+        },
+        sources: async (reference) => evidenceSourcesOf(database, { type: "memory", id: Number(rowOf(reference).id) }),
         search: async (text, { subject: subjectRef, scope, limit = 20 } = {}) => {
             const subject = subjectRef === undefined ? undefined : requiredThing(database, subjectRef);
             drainCorpus(database, CORPORA.memory, stemmerFor);

package/dist/store/sqlite/note-search.d.ts

@@ -79,6 +79,11 @@ export declare const searchNotes: (context: StoreContext, query: NoteQuery) => {
     items: NoteHit[];
     hasMore: boolean;
 };
+/** The refs of the newest `limit` notes matching a query, without reading the notes themselves. */
+export declare const matchingNoteRefs: (context: StoreContext, query: Omit<NoteQuery, "offset" | "newest">) => {
+    refs: string[];
+    hasMore: boolean;
+};
 export declare const searchCorpus: (context: StoreContext, query: CorpusQuery, corpus: Corpus) => CorpusRow[];
 export interface NoteChunkToEmbed {
     hash: string;
@@ -101,6 +106,11 @@ export interface NoteSearch extends DocumentPictureStore {
         items: NoteHit[];
         hasMore: boolean;
     }>;
+    /** The refs of the newest `limit` notes matching the query; `hasMore` when more match. */
+    matchingRefs(query: Omit<NoteQuery, "offset" | "newest">): Promise<{
+        refs: string[];
+        hasMore: boolean;
+    }>;
     indexState(): Promise<NoteIndexState | undefined>;
     /** Live notes' chunks with no vector of `model`, by hash from `after`, each with its text cut again. */
     chunksToEmbed(model: string, options: {
@@ -120,6 +130,14 @@ export interface NoteSearch extends DocumentPictureStore {
         withInternal?: boolean;
         source?: Note["source"];
     }): Promise<NearestNote[]>;
+    /**
+     * Each of `refs` scored by its best chunk, best first, every chunk compared: a filter's own notes ranked, not the
+     * nearest overall. A note with no vector of `model` is left out; the notes themselves are not read.
+     */
+    scoreNotes(model: string, query: Float32Array, refs: string[]): Promise<{
+        ref: string;
+        score: number;
+    }[]>;
 }
 export declare const noteSearchOver: (context: StoreContext) => NoteSearch;
 export {};

package/dist/store/sqlite/note-search.js

@@ -63,9 +63,7 @@ const CORPORA = [
         tagged: (tag) => bound("n.id IN (SELECT g.taggable_id FROM taggings g JOIN resolved_tags t ON t.id=g.tag_id WHERE t.name = ? AND g.taggable_type = 'note')", tag),
     },
 ];
-/** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
-export const searchNotes = (context, query) => {
-    const offset = query.offset ?? 0;
+const matchedRows = (context, query) => {
     const chosen = CORPORA.filter(({ type }) => (query.source === undefined || (query.source === "file") === (type === "document")) &&
         (!query.folderIds?.length || query.withInternal === true || type === "document"));
     const idsOf = (type) => query.refs?.flatMap((ref) => (ref.startsWith(`${type}:`) ? [Number(ref.slice(type.length + 1))] : []));
@@ -83,6 +81,12 @@ export const searchNotes = (context, query) => {
             (a.relevance ?? 0) - (b.relevance ?? 0) ||
             (b.at ?? 0) - (a.at ?? 0) ||
             a.id - b.id);
+    return rows;
+};
+/** Notes matching a parsed query, by the same language as messages: files and written notes, merged. */
+export const searchNotes = (context, query) => {
+    const offset = query.offset ?? 0;
+    const rows = matchedRows(context, query);
     const read = {
         document: context.database.prepare("SELECT * FROM documents WHERE id = ?"),
         note: context.database.prepare("SELECT * FROM notes WHERE id = ?"),
@@ -102,6 +106,11 @@ export const searchNotes = (context, query) => {
         hasMore: rows.length > offset + query.limit,
     };
 };
+/** The refs of the newest `limit` notes matching a query, without reading the notes themselves. */
+export const matchingNoteRefs = (context, query) => {
+    const rows = matchedRows(context, { ...query, newest: true });
+    return { refs: rows.slice(0, query.limit).map(({ type, id }) => `${type}:${id}`), hasMore: rows.length > query.limit };
+};
 export const searchCorpus = (context, query, corpus) => {
     const { database } = context;
     const wordMatch = (match) => ({
@@ -282,7 +291,7 @@ export const searchCorpus = (context, query, corpus) => {
             : "";
         const window = tests.size ? QUERY_LIMITS.candidates + 1 : query.limit + offset + 1;
         const rows = database
-            .prepare(`${ctes.length ? `WITH ${ctes.join(", ")} ` : ""}SELECT n.*, ${ranking ? "f.rank" : "NULL"} AS relevance${exactTier ? ", n.id IN (SELECT id FROM x) AS exact" : ""}${projection}
+            .prepare(`${ctes.length ? `WITH ${ctes.join(", ")} ` : ""}SELECT ${tests.size ? "n.*" : "n.id, n.at"}, ${ranking ? "f.rank" : "NULL"} AS relevance${exactTier ? ", n.id IN (SELECT id FROM x) AS exact" : ""}${projection}
            FROM ${corpus.view} n${ranking ? " LEFT JOIN f ON f.id = n.id" : ""} WHERE ${where.sql} ORDER BY ${order} LIMIT ?`)
             .all(...cteParams, ...leaves.flatMap(([, fragment]) => (tests.size ? fragment.params : [])), ...where.params, window);
         if (tests.size && rows.length > QUERY_LIMITS.candidates)
@@ -346,6 +355,10 @@ export const noteSearchOver = (context) => {
             drain();
             return searchNotes(context, query);
         },
+        matchingRefs: async (query) => {
+            drain();
+            return matchingNoteRefs(context, query);
+        },
         indexState: async () => noteIndexState(database),
         chunksToEmbed: async (model, { after, limit }) => {
             drain();
@@ -456,5 +469,28 @@ export const noteSearchOver = (context) => {
                 return note ? [{ ref: note.ref, note, score, range: { start, end } }] : [];
             });
         },
+        scoreNotes: async (model, query, refs) => {
+            drain();
+            const scope = live(undefined, undefined, true);
+            const owned = database.prepare(`SELECT k.chunkable_type AS type, k.chunkable_id AS owner, v.vector
+           FROM chunks k JOIN embeddings v ON v.model = ? AND v.content_hash = k.content_hash
+          WHERE k.chunkable_type = ? AND k.chunkable_id IN (SELECT value FROM json_each(?))${scope.sql}`);
+            const best = new Map();
+            for (const type of ["document", "note"]) {
+                const ids = refs.flatMap((ref) => (ref.startsWith(`${type}:`) ? [Number(ref.slice(type.length + 1))] : []));
+                if (!ids.length)
+                    continue;
+                for (const row of owned.all(model, type, JSON.stringify(ids), ...scope.params)) {
+                    const key = `${row.type}:${row.owner}`;
+                    const score = dot(query, row.vector);
+                    if (score > (best.get(key) ?? Number.NEGATIVE_INFINITY))
+                        best.set(key, score);
+                }
+            }
+            const order = new Map(refs.map((ref, index) => [ref, index]));
+            return [...best]
+                .map(([ref, score]) => ({ ref, score }))
+                .sort((a, b) => b.score - a.score || (order.get(a.ref) ?? 0) - (order.get(b.ref) ?? 0));
+        },
     };
 };

package/dist/store/sqlite/project-transfer/snapshot.d.ts

@@ -7,7 +7,7 @@ export declare const projectAnnotationSnapshot: (database: CacheDatabase, projec
             ref: string;
             type: "decision" | "fact" | "promise" | "question" | "statement";
             statement: string;
-            status: "accepted" | "proposed" | "reversed" | "superseded";
+            status: "accepted" | "proposed" | "rejected" | "reversed" | "superseded";
             subject: string | null;
             occurred_at: number | null;
             supersedes: string | null;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.499.0",
+  "version": "0.501.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -247,7 +247,7 @@
     "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
     "@vitest/coverage-v8": "^5.0.5",
-    "@wirecat/cli-base": "0.65.5",
+    "@wirecat/cli-base": "0.67.0",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
     "cspell": "10.4.0",
@@ -267,7 +267,7 @@
   },
   "peerDependencies": {
     "@napi-rs/canvas": ">=0.1.69 <2",
-    "@wirecat/cli-base": ">=0.65.0 <0.66.0",
+    "@wirecat/cli-base": ">=0.67.0 <0.68.0",
     "commander": ">=15"
   },
   "bin": {