@wirecat/cli-core 0.593.0 → 0.595.0

package/dist/store/sqlite/rollback-copy.js

@@ -0,0 +1,290 @@
+import { mkdirSync, rmSync, statfsSync, writeFileSync } from "node:fs";
+import { basename, dirname, join, parse } from "node:path";
+import { CliError } from "@wirecat/cli-base";
+/**
+ * A breaking migration (one that raises `min_compatible`) locks older tools out of the store. Before it runs, the
+ * base tables it names are copied into a small file in `backups/` beside the store, with the whole schema as it was;
+ * `store restore <copy>` puts them back and returns the file to the old version. Derived tables are never copied:
+ * search indexes, their queues, vectors, chunks, mentions and involvements are rebuilt from the base tables.
+ */
+const DERIVED = /(_fts|_words|_stems|_trigrams|_pending)$|^(embeddings|chunks|chunk_messages|mentions|involvements|search_terms)$|^vector_index/;
+export const isDerivedTable = (name) => DERIVED.test(name);
+/**
+ * Always copied, a few rows each: migrations seed rows into them, and the next upgrade after a rollback would insert
+ * those rows again and fail on its own unique keys.
+ */
+const BOOKKEEPING = ["store_settings", "search_index_state"];
+const KEY = "rollback_copy:";
+const SCHEMA = "rollback_copy";
+const ROWID = "__rollback_rowid";
+const DAY_MS = 24 * 60 * 60 * 1000;
+// The migration's own write-ahead log needs room too.
+const HEADROOM = 256 * 1024 * 1024;
+const quote = (name) => `"${name.replaceAll('"', '""')}"`;
+const megabytes = (bytes) => `${Math.ceil(bytes / 1024 / 1024)} MB`;
+const freeBytesOf = (dir) => {
+    const { bavail, bsize } = statfsSync(dir);
+    return Number(bavail) * Number(bsize);
+};
+const tableIn = (database, schema, name) => database.prepare(`SELECT 1 FROM ${schema}.sqlite_master WHERE type = 'table' AND name = ?`).get(name) !== undefined;
+const keepsRowid = (database, schema, table) => Number(database.prepare("SELECT wr FROM pragma_table_list WHERE schema = ? AND name = ?").get(schema, table)?.wr) ===
+    0;
+const columnsOf = (database, schema, table) => database
+    .prepare("SELECT name FROM pragma_table_info(?, ?)")
+    .all(table, schema)
+    .map((row) => String(row.name));
+const versionOf = (database) => Number(database.prepare("SELECT max(version) AS version FROM main.schema_migrations").get()?.version ?? 0);
+const tableBytes = (database, tables) => {
+    try {
+        return Number(database
+            .prepare(`SELECT coalesce(sum(pgsize), 0) AS bytes FROM dbstat
+           WHERE schema = 'main' AND aggregate = 1 AND name IN (${tables.map(() => "?").join(", ")})`)
+            .get(...tables)?.bytes ?? 0);
+    }
+    catch {
+        // Bun's SQLite is built without dbstat: the whole file is the bound.
+        const pragma = (name) => Number(database.prepare(`PRAGMA ${name}`).get()?.[name] ?? 0);
+        return pragma("page_count") * pragma("page_size");
+    }
+};
+const stampOf = (at) => new Date(at).toISOString().replace(/[:.]/g, "-");
+const attach = (database, path) => database.prepare(`ATTACH DATABASE ? AS ${SCHEMA}`).run(path);
+/**
+ * Outside the migration's transaction (SQLite cannot attach inside one), committed on its own, so a migration that
+ * fails leaves it whole. Never blocks the migration: no room, or any failure, is a warning and no copy.
+ */
+export const takeRollbackCopy = (database, { from, to, tables, at }, { store, freeBytes = freeBytesOf, warn = (line) => process.stderr.write(`${line}\n`) }) => {
+    const dir = join(dirname(store), "backups");
+    const present = [...new Set([...BOOKKEEPING, ...tables])].filter((table) => tableIn(database, "main", table));
+    const afterwards = `going back to schema ${from} then needs an empty store, and its data pulled again from the sources`;
+    let path;
+    try {
+        mkdirSync(dir, { recursive: true, mode: 0o700 });
+        const needed = tableBytes(database, present);
+        const free = freeBytes(dir);
+        if (free < needed + HEADROOM) {
+            warn(`not enough free disk space for a rollback copy before schema ${to} (about ${megabytes(needed + HEADROOM)} ` +
+                `needed, ${megabytes(free)} free in ${dir}) — migrating without one; ${afterwards}`);
+            return undefined;
+        }
+        path = join(dir, `${parse(store).name}-rollback-schema-${from}-${stampOf(at)}.db`);
+        writeFileSync(path, "", { flag: "wx", mode: 0o600 });
+        attach(database, path);
+        try {
+            database.exec("BEGIN IMMEDIATE");
+            try {
+                const copied = versionOf(database);
+                database.exec(`CREATE TABLE ${SCHEMA}.rollback_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)`);
+                database.exec(`CREATE TABLE ${SCHEMA}.rollback_schema AS
+           SELECT m.type, m.name, m.tbl_name, m.sql, coalesce(l.type, m.type) AS kind
+           FROM main.sqlite_master m LEFT JOIN pragma_table_list l ON l.schema = 'main' AND l.name = m.name
+           ORDER BY m.rowid`);
+                for (const table of present) {
+                    const rowid = keepsRowid(database, "main", table) ? `rowid AS ${ROWID}, ` : "";
+                    database.exec(`CREATE TABLE ${SCHEMA}.${quote(table)} AS SELECT ${rowid}* FROM main.${quote(table)}`);
+                }
+                const meta = { from: copied, to, tables: present, created_at: at };
+                const insert = database.prepare(`INSERT INTO ${SCHEMA}.rollback_meta (key, value) VALUES (?, ?)`);
+                for (const [key, value] of Object.entries(meta))
+                    insert.run(key, JSON.stringify(value));
+                database.exec("COMMIT");
+                return { path, from: copied, to };
+            }
+            catch (error) {
+                database.exec("ROLLBACK");
+                throw error;
+            }
+        }
+        finally {
+            database.exec(`DETACH DATABASE ${SCHEMA}`);
+        }
+    }
+    catch (error) {
+        if (path)
+            rmSync(path, { force: true });
+        warn(`the rollback copy before schema ${to} could not be written (${error instanceof Error ? error.message : String(error)}) — migrating without one; ${afterwards}`);
+        return undefined;
+    }
+};
+/** Inside the migration's transaction: a migration that fails never records its copy, so cleanup never takes it. */
+export const recordRollbackCopy = (database, copy, at) => {
+    database
+        .prepare("INSERT OR REPLACE INTO store_settings (key, value, at) VALUES (?, ?, ?)")
+        .run(`${KEY}${basename(copy.path)}`, JSON.stringify({ path: copy.path, from: copy.from, to: copy.to, at }), at);
+};
+export const discardRollbackCopy = (copy) => rmSync(copy.path, { force: true });
+/**
+ * The day after: a copy recorded by a migration that committed, once the store is still at that version. Only the
+ * recorded file is deleted, never anything else in `backups/`.
+ */
+export const pruneRollbackCopies = (database, now) => {
+    const recorded = database
+        .prepare("SELECT key, value FROM store_settings WHERE key LIKE 'rollback\\_copy:%' ESCAPE '\\'")
+        .all();
+    if (recorded.length === 0)
+        return;
+    const version = versionOf(database);
+    for (const { key, value } of recorded) {
+        let copy;
+        try {
+            copy = JSON.parse(String(value));
+        }
+        catch {
+            continue;
+        }
+        if (typeof copy.path !== "string" || now - Number(copy.at) < DAY_MS || version < Number(copy.to))
+            continue;
+        if (!database.prepare("SELECT 1 FROM schema_migrations WHERE version = ?").get(Number(copy.to)))
+            continue;
+        rmSync(copy.path, { force: true });
+        database.prepare("DELETE FROM store_settings WHERE key = ?").run(String(key));
+    }
+};
+const objectsOf = (database, sql) => database
+    .prepare(sql)
+    .all()
+    .map((row) => ({
+    type: String(row.type),
+    name: String(row.name),
+    sql: row.sql === null ? null : String(row.sql),
+    kind: String(row.kind),
+}));
+const currentObjects = (database) => objectsOf(database, `SELECT m.type, m.name, m.sql, coalesce(l.type, m.type) AS kind
+     FROM main.sqlite_master m LEFT JOIN pragma_table_list l ON l.schema = 'main' AND l.name = m.name`);
+const internal = ({ name, kind }) => name.startsWith("sqlite_") || kind === "shadow";
+const metaOf = (database) => {
+    const values = new Map(database
+        .prepare(`SELECT key, value FROM ${SCHEMA}.rollback_meta`)
+        .all()
+        .map((row) => [String(row.key), JSON.parse(String(row.value))]));
+    const tables = values.get("tables");
+    return {
+        from: Number(values.get("from")),
+        to: Number(values.get("to")),
+        tables: Array.isArray(tables) ? tables.map(String) : [],
+        createdAt: Number(values.get("created_at")),
+    };
+};
+/** FTS5 tables over a table's own rows; the other search indexes hold their own text and wait for `store rebuild`. */
+const contentOf = (sql) => /\bcontent\s*=\s*'([^']+)'/i.exec(sql ?? "")?.[1];
+const RANK = { table: 0, index: 1, view: 2, trigger: 3 };
+/**
+ * Returns the file to the schema the copy saved: every object made since is dropped, every table the migration
+ * changed is put back with its rows, a table only extended since is rebuilt to its old shape, and the history
+ * goes back to the copy's version. One transaction: on any failure the store is as it was.
+ */
+const rewind = (database, saved, named) => {
+    const before = new Map(saved.map((object) => [object.name, object]));
+    const same = (object) => before.get(object.name)?.type === object.type && before.get(object.name)?.sql === object.sql;
+    const touched = new Set(named);
+    const rebuilt = [];
+    // A trigger or view over a table dropped or rebuilt below would point nowhere; all of them come back from the copy.
+    for (const object of currentObjects(database)) {
+        if (object.type === "trigger")
+            database.exec(`DROP TRIGGER IF EXISTS ${quote(object.name)}`);
+        if (object.type === "view")
+            database.exec(`DROP VIEW IF EXISTS ${quote(object.name)}`);
+        if (object.type === "index" && object.sql !== null && !same(object))
+            database.exec(`DROP INDEX IF EXISTS ${quote(object.name)}`);
+    }
+    for (const object of currentObjects(database)) {
+        if (object.type !== "table" || internal(object))
+            continue;
+        const old = before.get(object.name);
+        if (named.has(object.name) || !old || (object.kind === "virtual" && !same(object))) {
+            database.exec(`DROP TABLE ${quote(object.name)}`);
+        }
+        else if (!same(object) && old.sql) {
+            rebuild(database, object.name, old.sql);
+            rebuilt.push(object.name);
+            touched.add(object.name);
+        }
+    }
+    const present = new Set(currentObjects(database).map(({ name }) => name));
+    const missing = saved
+        .filter((object) => !internal(object) && object.sql !== null && !present.has(object.name))
+        .sort((a, b) => (RANK[a.type] ?? 9) - (RANK[b.type] ?? 9));
+    for (const object of missing) {
+        database.exec(object.sql);
+        if (object.type === "table" && named.has(object.name))
+            restoreRows(database, object.name);
+        if (object.kind === "virtual")
+            touched.add(object.name);
+    }
+    for (const object of saved) {
+        const content = contentOf(object.sql);
+        if (object.kind !== "virtual" || !(content && (touched.has(content) || touched.has(object.name))))
+            continue;
+        database.exec(`INSERT INTO ${quote(object.name)} (${quote(object.name)}) VALUES ('rebuild')`);
+    }
+    return rebuilt;
+};
+const rebuild = (database, table, sql) => {
+    const aside = `__rollback_${table}`;
+    database.exec(`ALTER TABLE ${quote(table)} RENAME TO ${quote(aside)}`);
+    database.exec(sql);
+    const kept = new Set(columnsOf(database, "main", aside));
+    const columns = columnsOf(database, "main", table).filter((column) => kept.has(column));
+    const rowid = keepsRowid(database, "main", table) ? ["rowid"] : [];
+    const list = [...rowid, ...columns.map(quote)].join(", ");
+    database.exec(`INSERT INTO ${quote(table)} (${list}) SELECT ${list} FROM ${quote(aside)}`);
+    database.exec(`DROP TABLE ${quote(aside)}`);
+};
+const restoreRows = (database, table) => {
+    const copied = new Set(columnsOf(database, SCHEMA, table));
+    const columns = columnsOf(database, "main", table).filter((column) => copied.has(column));
+    const rowid = keepsRowid(database, "main", table) && copied.has(ROWID);
+    const into = [...(rowid ? ["rowid"] : []), ...columns.map(quote)].join(", ");
+    const from = [...(rowid ? [ROWID] : []), ...columns.map(quote)].join(", ");
+    database.exec(`INSERT INTO main.${quote(table)} (${into}) SELECT ${from} FROM ${SCHEMA}.${quote(table)}`);
+};
+/** Puts a rollback copy back into the store, in place. The caller makes sure nothing else has the store open. */
+export const rollBack = (database, path) => {
+    attach(database, path);
+    let result;
+    try {
+        if (!tableIn(database, SCHEMA, "rollback_meta"))
+            throw new CliError("validation_error", `${path} is not a rollback copy`);
+        const meta = metaOf(database);
+        const was = versionOf(database);
+        if (!database.prepare("SELECT 1 FROM main.schema_migrations WHERE version = ?").get(meta.to)) {
+            throw new CliError("validation_error", `the store is at schema ${was} and was never migrated to ${meta.to}, the schema this copy was taken before — nothing to roll back`);
+        }
+        const saved = objectsOf(database, `SELECT type, name, sql, kind FROM ${SCHEMA}.rollback_schema ORDER BY rowid`);
+        database.exec("PRAGMA foreign_keys = OFF");
+        // Renaming a table aside must not rewrite the references other tables hold to it.
+        database.exec("PRAGMA legacy_alter_table = ON");
+        try {
+            database.exec("BEGIN IMMEDIATE");
+            try {
+                const named = new Set(meta.tables);
+                const rebuilt = rewind(database, saved, named);
+                database.prepare("DELETE FROM main.schema_migrations WHERE version > ?").run(meta.from);
+                database.prepare("DELETE FROM main.store_settings WHERE key = ?").run(`${KEY}${basename(path)}`);
+                database.exec("COMMIT");
+                const count = (table) => Number(database.prepare(`SELECT count(*) AS n FROM main.${quote(table)}`).get()?.n ?? 0);
+                result = {
+                    ...meta,
+                    was,
+                    restored: meta.tables.map((table) => ({ table, rows: count(table) })),
+                    rebuilt,
+                };
+            }
+            catch (error) {
+                database.exec("ROLLBACK");
+                throw error;
+            }
+        }
+        finally {
+            database.exec("PRAGMA legacy_alter_table = OFF");
+            database.exec("PRAGMA foreign_keys = ON");
+        }
+    }
+    finally {
+        database.exec(`DETACH DATABASE ${SCHEMA}`);
+    }
+    // Only the tables put back: a check of the whole store reads every row of it.
+    const checked = [...result.restored.map(({ table }) => table), ...result.rebuilt];
+    const foreignKeyViolations = checked.reduce((total, table) => total + database.prepare(`PRAGMA foreign_key_check(${quote(table)})`).all().length, 0);
+    return { ...result, foreignKeyViolations };
+};

package/dist/store/sqlite/schema.d.ts

@@ -15527,6 +15527,22 @@ export declare const pipelines: import("drizzle-orm/sqlite-core").SQLiteTableWit
             identity: undefined;
             generated: undefined;
         }, {}>;
+        deletedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "pipelines";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
     };
     dialect: 'sqlite';
     isAlias: false;
@@ -15679,6 +15695,22 @@ export declare const pipelineStages: import("drizzle-orm/sqlite-core").SQLiteTab
             identity: undefined;
             generated: undefined;
         }, {}>;
+        deletedAt: import("drizzle-orm/sqlite-core").SQLiteColumn<{
+            name: string;
+            tableName: "pipeline_stages";
+            dataType: "number int53";
+            data: number;
+            driverParam: number;
+            notNull: false;
+            hasDefault: false;
+            isPrimaryKey: false;
+            isAutoincrement: false;
+            hasRuntimeDefault: false;
+            enumValues: undefined;
+            baseColumn: never;
+            identity: undefined;
+            generated: undefined;
+        }, {}>;
     };
     dialect: 'sqlite';
     isAlias: false;

package/dist/store/sqlite/schema.js

@@ -1545,9 +1545,10 @@ export const pipelines = sqliteTable("pipelines", {
     nameFolded: text("name_folded").notNull(),
     createdAt: integer("created_at").notNull(),
     updatedAt: integer("updated_at").notNull(),
+    deletedAt: integer("deleted_at"),
 }, (table) => [
-    uniqueIndex("pipelines_by_account_id_external_id")
-        .on(table.accountId, table.externalId)
+    uniqueIndex("pipelines_by_account_id_kind_external_id")
+        .on(table.accountId, table.kind, table.externalId)
         .where(sql `account_id IS NOT NULL AND external_id IS NOT NULL`),
     uniqueIndex("pipelines_by_name_folded_kind").on(table.nameFolded, table.kind).where(sql `account_id IS NULL`),
 ]);
@@ -1563,6 +1564,7 @@ export const pipelineStages = sqliteTable("pipeline_stages", {
     externalId: text("external_id"),
     createdAt: integer("created_at").notNull(),
     updatedAt: integer("updated_at").notNull(),
+    deletedAt: integer("deleted_at"),
 }, (table) => [
     unique().on(table.pipelineId, table.nameFolded),
     uniqueIndex("pipeline_stages_by_pipeline_id_external_id")

package/dist/store/store.d.ts

@@ -133,6 +133,7 @@ export interface MessageStore {
 export type { ChatHistoryShape, StoredHistorySkip } from "./sqlite/history-skips.js";
 export { BACKFILL_ON_OPEN, type StoreOptions } from "./sqlite/open-store.js";
 export type { ChatScope, NewestMessage, ReplyDelay, WaitingChat } from "./sqlite/reply-times.js";
+export { rollBack } from "./sqlite/rollback-copy.js";
 export { fileFacts, readStoreTables, type TableKind } from "./sqlite/store-tables.js";
 export declare const openStore: (options?: StoreOptions) => Promise<MessageStore>;
 export * from "./exports.js";

package/dist/store/store.js

@@ -50,6 +50,7 @@ import { syncStoreOver } from "./sync.js";
 import { tagsStoreOver } from "./tags.js";
 import { accountVectorsOver, vectorsStoreOver } from "./vectors.js";
 export { BACKFILL_ON_OPEN } from "./sqlite/open-store.js";
+export { rollBack } from "./sqlite/rollback-copy.js";
 export { fileFacts, readStoreTables } from "./sqlite/store-tables.js";
 export const openStore = async (options = {}) => {
     const context = await openStoreContext(options);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@wirecat/cli-core",
-  "version": "0.593.0",
+  "version": "0.595.0",
   "description": "The messenger-neutral half of a messaging CLI: domain model, message locators, rendering, name resolution, the SQLite seam, the command skeleton and shared read commands, the send guard, run records and the message store",
   "license": "Apache-2.0",
   "author": "Viacheslav Ptsarev",
@@ -270,7 +270,7 @@
     "@types/node": "^22.20.5",
     "@types/pngjs": "6.0.5",
     "@vitest/coverage-v8": "^5.0.5",
-    "@wirecat/cli-base": "0.80.0",
+    "@wirecat/cli-base": "0.80.2",
     "@wirecat/cli-core-legacy-tags": "npm:@wirecat/cli-core@0.414.0",
     "commander": "15.0.0",
     "cspell": "10.4.0",