@coffer-org/server 3.3.0 → 4.0.0

package/dist/migrations.js

@@ -1,6 +1,8 @@
 import { DatabaseSchema } from '@mikro-orm/sql';
+import { fieldEntries } from '@coffer-org/sdk/shelf';
 import { dialectOf } from "./dialect.js";
 import { getLogger } from '@coffer-org/sdk/logger';
+import { SYSTEM_SETTINGS } from "./system-settings.js";
 const log = getLogger('migrations');
 function sqlType(col) {
     if (col === 'integer' || col === 'boolean')
@@ -25,6 +27,13 @@ function pgType(col) {
     return 'text';
 }
 const q = (id) => `"${id.replace(/"/g, '""')}"`;
+const handedOverColumns = new Set();
+export function markColumnHandedOver(table, column) {
+    handedOverColumns.add(`${table}.${column}`);
+}
+export function isColumnHandedOver(table, column) {
+    return handedOverColumns.has(`${table}.${column}`);
+}
 export async function introspectTable(em, table) {
     const conn = em.getConnection();
     const platform = em.getPlatform();
@@ -38,31 +47,122 @@ export async function introspectTable(em, table) {
             : [],
     };
 }
-export function makeTable(em, table) {
+export async function introspectAllColumns(em, tables) {
+    const out = new Map();
+    if (!tables.length)
+        return out;
+    const conn = em.getConnection();
+    const platform = em.getPlatform();
+    const schema = await DatabaseSchema.create(conn, platform, em.config, undefined, undefined, tables);
+    for (const t of schema.getTables()) {
+        out.set(t.name, new Set(t.getColumns().map((c) => c.name)));
+    }
+    return out;
+}
+export const CONCERNING_CODES = new Set([
+    'no-source-column',
+    'target-column-exists',
+    'converted-nothing',
+]);
+export function describeMigrationConcerns(plugin, version, ops) {
+    const concerning = ops.filter((o) => !o.performed && o.code && CONCERNING_CODES.has(o.code));
+    const lopsided = ops.filter((o) => o.performed && o.emptyTargets?.length);
+    const stranded = ops.filter((o) => o.strandedSources?.length);
+    const performedAny = ops.some((o) => o.performed);
+    const describe = (o) => {
+        const bits = [];
+        if (!o.performed)
+            bits.push(`changed nothing — ${o.reason ?? 'unknown reason'}`);
+        if (o.emptyTargets?.length)
+            bits.push(`wrote nothing into ${o.emptyTargets.join(', ')} on any of ${o.scanned ?? 0} row(s)`);
+        if (o.strandedSources?.length)
+            bits.push(`left ${o.strandedSources.join(', ')} holding data that reached no target column`);
+        return `${o.op}(${o.table}) ${bits.join(', and ')}`;
+    };
+    const tail = `The journal is bumped to v${version}, so it will never run again on this database — ` +
+        `if data was supposed to move, it did not, and a later schema sync will drop the columns it still lives in.`;
+    if (!ops.length)
+        return (`${plugin} v${version}: the migration completed without performing a single table operation. ` +
+            `The journal is bumped to v${version} regardless, so if its body swallowed an error or was ` +
+            `written for a state this database is not in, it will never run again.`);
+    if (!concerning.length && !lopsided.length && !stranded.length)
+        return null;
+    if (!performedAny)
+        return `${plugin} v${version}: the migration completed WITHOUT changing anything (${ops
+            .map(describe)
+            .join('; ')}). ${tail}`;
+    const flagged = [...new Set([...concerning, ...lopsided, ...stranded])];
+    return (`${plugin} v${version}: the migration changed some data but ${flagged.length} of its operations did not ` +
+        `(${flagged.map(describe).join('; ')}). ${tail}`);
+}
+function sameCell(a, b) {
+    const na = a === undefined ? null : a;
+    const nb = b === undefined ? null : b;
+    if (na === null || nb === null)
+        return na === nb;
+    return String(na) === String(nb);
+}
+export function makeTable(em, table, record) {
     const conn = em.getConnection();
     const T = q(table);
+    const countRows = async (where = '') => {
+        const rows = (await conn.execute(`SELECT COUNT(*) AS n FROM ${T}${where ? ` WHERE ${where}` : ''}`));
+        return Number(rows[0]?.n ?? 0);
+    };
+    const done = (op, rows, scanned) => record?.({ op, table, performed: true, rows, ...(scanned === undefined ? {} : { scanned }) });
+    const nothing = (op, code, reason, extra) => record?.({ op, table, performed: false, code, reason, ...extra });
     return {
         async renameColumn(from, to) {
             const info = await introspectTable(em, table);
-            if (!info.exists || !info.columns.has(from))
-                return;
+            if (!info.exists)
+                return nothing('renameColumn', 'no-table', `table ${table} does not exist`);
+            if (!info.columns.has(from))
+                return nothing('renameColumn', 'no-source-column', `source column ${from} does not exist on a table that does — an earlier schema sync already dropped it`);
+            if (info.columns.has(to)) {
+                const stranded = await countRows(`${q(from)} IS NOT NULL`);
+                log.warn(`${table}: renameColumn(${from} → ${to}) skipped — target column already exists`);
+                return nothing('renameColumn', 'target-column-exists', `target column ${to} already exists, so the rename was skipped while ${stranded} row(s) still hold data in ${from}`, { rows: 0, scanned: stranded });
+            }
+            const moved = await countRows(`${q(from)} IS NOT NULL`);
             await conn.execute(`ALTER TABLE ${T} RENAME COLUMN ${q(from)} TO ${q(to)}`);
+            done('renameColumn', moved, await countRows());
         },
         async fill(column, value, opts) {
             const info = await introspectTable(em, table);
-            if (!info.exists || !info.columns.has(column))
-                return;
+            if (!info.exists)
+                return nothing('fill', 'no-table', `table ${table} does not exist`);
+            if (!info.columns.has(column))
+                return nothing('fill', 'no-source-column', `column ${column} does not exist on a table that does — an earlier schema sync already dropped it`);
             const where = opts?.where ?? {};
             const keys = Object.keys(where);
             const clauses = keys.map((k) => (where[k] === null ? `${q(k)} IS NULL` : `${q(k)} = ?`));
-            const params = [value, ...keys.filter((k) => where[k] !== null).map((k) => where[k])];
+            const whereParams = keys.filter((k) => where[k] !== null).map((k) => where[k]);
+            const params = [value, ...whereParams];
+            const matchSql = clauses.join(' AND ');
+            const differs = `(${q(column)} IS NULL OR ${q(column)} <> ?)`;
+            const countWhere = [matchSql, differs].filter(Boolean).join(' AND ');
+            const changing = (await conn.execute(`SELECT COUNT(*) AS n FROM ${T} WHERE ${countWhere}`, [
+                ...whereParams,
+                value,
+            ]));
+            const changed = Number(changing[0]?.n ?? 0);
             const sql = `UPDATE ${T} SET ${q(column)} = ?` + (clauses.length ? ` WHERE ${clauses.join(' AND ')}` : '');
             await conn.execute(sql, params);
+            const scanned = await countRows();
+            if (changed)
+                return done('fill', changed, scanned);
+            if (!scanned)
+                return nothing('fill', 'no-rows', `table ${table} has no rows`, { rows: 0, scanned });
+            nothing('fill', 'no-change', `every matching row in ${column} already held that value`, { rows: 0, scanned });
         },
         async changeType(column, newType, opts) {
             const info = await introspectTable(em, table);
-            if (!info.exists || !info.columns.has(column))
-                return;
+            if (!info.exists)
+                return nothing('changeType', 'no-table', `table ${table} does not exist`);
+            if (!info.columns.has(column))
+                return nothing('changeType', 'no-source-column', `column ${column} does not exist on a table that does — an earlier schema sync already dropped it`);
+            const carried = await countRows(`${q(column)} IS NOT NULL`);
+            const total = await countRows();
             const tmp = `__ct_${column}`;
             const def = opts?.default;
             if (dialectOf(em) === 'sqlite') {
@@ -108,6 +208,7 @@ export function makeTable(em, table) {
                     await conn.execute(`ROLLBACK`);
                     throw e;
                 }
+                done('changeType', carried, total);
                 return;
             }
             await conn.execute(`BEGIN`);
@@ -135,13 +236,14 @@ export function makeTable(em, table) {
                 await conn.execute(`ROLLBACK`);
                 throw e;
             }
+            done('changeType', carried, total);
         },
         async convert(c) {
             const info = await introspectTable(em, table);
             if (!info.exists)
-                return;
+                return nothing('convert', 'no-table', `table ${table} does not exist`);
             if (!c.from.some((f) => info.columns.has(f)))
-                return;
+                return nothing('convert', 'no-source-column', `none of the source columns (${c.from.join(', ')}) exist on a table that does — an earlier schema sync already dropped them`);
             const sources = new Set(c.from);
             for (const t of c.to) {
                 if (info.columns.has(t.name) && !sources.has(t.name)) {
@@ -152,24 +254,77 @@ export function makeTable(em, table) {
             const idCol = sqlite ? 'rowid' : 'id';
             const colType = sqlite ? sqlType : pgType;
             const present = c.from.filter((f) => info.columns.has(f));
+            const readable = [...new Set([...present, ...c.to.filter((t) => info.columns.has(t.name)).map((t) => t.name)])];
+            let scanned = 0;
+            let changed = 0;
+            let filled = 0;
+            const filledPerTarget = new Map(c.to.map((t) => [t.name, 0]));
+            const strandedPerSource = new Map(present.filter((f) => !c.to.some((t) => t.name === f)).map((f) => [f, 0]));
+            const inPlaceTargets = c.to.filter((t) => sources.has(t.name)).map((t) => t.name);
+            const blankTargets = Object.fromEntries(inPlaceTargets.map((n) => [n, null]));
+            const normalize = (dst) => c.to.map((t) => {
+                const v = dst[t.name];
+                return v === undefined || (typeof v === 'number' && Number.isNaN(v)) ? null : v;
+            });
             await conn.execute(`BEGIN`);
             try {
                 for (const t of c.to) {
                     if (!info.columns.has(t.name))
                         await conn.execute(`ALTER TABLE ${T} ADD COLUMN ${q(t.name)} ${colType(t.type)}`);
                 }
-                const select = present.map((f) => q(f)).join(', ');
+                const select = readable.map((f) => q(f)).join(', ');
                 const rows = (await conn.execute(`SELECT ${idCol} AS __rid, ${select} FROM ${T}`));
                 const setSql = c.to.map((t) => `${q(t.name)} = ?`).join(', ');
                 for (const r of rows) {
+                    scanned++;
                     const src = {};
                     for (const f of c.from)
                         src[f] = r[f] ?? null;
-                    const dst = c.map(src);
-                    const params = c.to.map((t) => {
-                        const v = dst[t.name];
-                        return v === undefined || (typeof v === 'number' && Number.isNaN(v)) ? null : v;
+                    const params = normalize(c.map(src));
+                    if (params.some((v) => v !== null))
+                        filled++;
+                    c.to.forEach((t, i) => {
+                        if (params[i] !== null)
+                            filledPerTarget.set(t.name, filledPerTarget.get(t.name) + 1);
                     });
+                    const differs = c.to.some((t, i) => !sameCell(params[i], r[t.name] ?? null));
+                    if (strandedPerSource.size) {
+                        const probe = (over) => {
+                            try {
+                                return normalize(c.map({ ...src, ...over }));
+                            }
+                            catch {
+                                return null;
+                            }
+                        };
+                        let fresh;
+                        const carriedAcross = (f, sv) => {
+                            if (params.some((v) => v !== null && sameCell(v, sv)))
+                                return true;
+                            const without = probe({ [f]: null });
+                            if (without && params.some((v, i) => v !== null && !sameCell(v, without[i])))
+                                return true;
+                            if (!inPlaceTargets.length)
+                                return false;
+                            if (fresh === undefined)
+                                fresh = probe(blankTargets);
+                            const freshWithout = fresh ? probe({ ...blankTargets, [f]: null }) : null;
+                            if (!fresh || !freshWithout)
+                                return false;
+                            return fresh.some((v, i) => v !== null && !sameCell(v, freshWithout[i]) && sameCell(params[i], v));
+                        };
+                        for (const [f, n] of strandedPerSource) {
+                            const sv = src[f];
+                            if (sv === null)
+                                continue;
+                            if (carriedAcross(f, sv))
+                                continue;
+                            strandedPerSource.set(f, n + 1);
+                        }
+                    }
+                    if (!differs)
+                        continue;
+                    changed++;
                     await conn.execute(`UPDATE ${T} SET ${setSql} WHERE ${idCol} = ?`, [...params, r['__rid']]);
                 }
                 await conn.execute(`COMMIT`);
@@ -178,11 +333,43 @@ export function makeTable(em, table) {
                 await conn.execute(`ROLLBACK`);
                 throw e;
             }
+            for (const [f, left] of strandedPerSource) {
+                if (left === 0) {
+                    markColumnHandedOver(table, f);
+                    continue;
+                }
+                log.warn(`${table}.${f}: convert left ${left} row(s) unconverted — their value in ${f} reached no target column ` +
+                    `(the map declined them, or a target-first map kept a value already stored). ` +
+                    `The column is NOT handed over to schema sync: it stays under the drop warning until the rows are repaired or removed.`);
+            }
+            const emptyTargets = [...filledPerTarget].filter(([, n]) => n === 0).map(([name]) => name);
+            const lopsided = emptyTargets.length && emptyTargets.length < c.to.length ? emptyTargets : [];
+            const strandedSources = [...strandedPerSource].filter(([, left]) => left > 0).map(([f]) => f);
+            const stranded = strandedSources.length ? { strandedSources } : {};
+            if (changed)
+                return record?.({
+                    op: 'convert',
+                    table,
+                    performed: true,
+                    rows: changed,
+                    scanned,
+                    ...(lopsided.length ? { emptyTargets: lopsided } : {}),
+                    ...stranded,
+                });
+            if (!scanned)
+                return nothing('convert', 'no-rows', `table ${table} has no rows`, { rows: 0, scanned });
+            if (!filled)
+                return nothing('convert', 'converted-nothing', `no row has a value in any target column (${c.to.map((t) => t.name).join(', ')}) across ${scanned} row(s). ` +
+                    `Column state cannot tell the two causes apart: the field may simply never have been used on this ` +
+                    `install, or its source data was dropped before this migration ran. Check one record before assuming either`, { rows: 0, scanned, ...stranded });
+            nothing('convert', 'no-change', `every one of ${scanned} row(s) already held the converted value in ${c.to.map((t) => t.name).join(', ')}`, { rows: 0, scanned, ...stranded });
         },
         async dropColumn(column) {
             const info = await introspectTable(em, table);
-            if (!info.exists || !info.columns.has(column))
-                return;
+            if (!info.exists)
+                return nothing('dropColumn', 'no-table', `table ${table} does not exist`);
+            if (!info.columns.has(column))
+                return nothing('dropColumn', 'no-change', `column ${column} is already absent`, { rows: 0 });
             if (dialectOf(em) === 'sqlite') {
                 const idxList = (await conn.execute(`PRAGMA index_list(${q(table)})`));
                 for (const idx of idxList) {
@@ -197,13 +384,136 @@ export function makeTable(em, table) {
                         await conn.execute(`DROP INDEX ${q(idx.name)}`);
                 }
             }
+            const held = await countRows(`${q(column)} IS NOT NULL`);
+            const total = await countRows();
             await conn.execute(`ALTER TABLE ${T} DROP COLUMN ${q(column)}`);
+            done('dropColumn', held, total);
         },
     };
 }
 export async function renameSystemShelfKey(em) {
     await makeTable(em, '_embeddings').renameColumn('type', 'shelf_key');
 }
+const LEGACY_ORCHESTRATOR_FIELDS = ['agent_id', 'access_password', 'trigger_prefix', 'reply_window'];
+export const SYSTEM_MIGRATIONS = [
+    {
+        name: 'embeddings-shelf-key',
+        up: renameSystemShelfKey,
+    },
+    {
+        name: 'core-settings-to-system',
+        up: async (em) => {
+            const conn = em.getConnection();
+            const src = await introspectTable(em, '_settings__core');
+            const dst = await introspectTable(em, '_settings__system');
+            if (!src.exists || !dst.exists)
+                return;
+            const already = (await conn.execute(`SELECT 1 FROM "_settings__system" WHERE "plugin_id" = 'system' LIMIT 1`));
+            if (already.length > 0)
+                return;
+            const rows = (await conn.execute(`SELECT * FROM "_settings__core" WHERE "plugin_id" = 'core' LIMIT 1`));
+            const row = rows[0];
+            if (!row)
+                return;
+            const carried = ['publicUrl', 'language'].filter((c) => src.columns.has(c) && dst.columns.has(c));
+            if (carried.length === 0)
+                return;
+            const cols = ['plugin_id', ...carried];
+            const values = ['system', ...carried.map((c) => row[c] ?? null)];
+            await conn.execute(`INSERT INTO "_settings__system" (${cols.map((c) => `"${c}"`).join(', ')}) ` +
+                `VALUES (${cols.map(() => '?').join(', ')})`, values);
+            log.info(`[system-migration] settings moved from _settings__core to _settings__system`);
+        },
+    },
+    {
+        name: 'core-state-to-system',
+        up: async (em) => {
+            const conn = em.getConnection();
+            if (!(await introspectTable(em, '_plugin_state')).exists)
+                return;
+            await conn.execute(`UPDATE "_plugin_state" SET "plugin" = 'system' ` +
+                `WHERE "plugin" = 'core' ` +
+                `AND "key" NOT IN (SELECT "key" FROM "_plugin_state" WHERE "plugin" = 'system')`);
+            await conn.execute(`DELETE FROM "_plugin_state" WHERE "plugin" = 'core'`);
+        },
+    },
+    {
+        name: 'drop-legacy-core-settings',
+        up: async (em) => {
+            const conn = em.getConnection();
+            if (!(await introspectTable(em, '_settings__core')).exists)
+                return;
+            if (!(await introspectTable(em, '_settings__system')).exists)
+                return;
+            const landed = (await conn.execute(`SELECT 1 FROM "_settings__system" WHERE "plugin_id" = 'system' LIMIT 1`));
+            if (landed.length === 0)
+                return;
+            await conn.execute(`DROP TABLE IF EXISTS "_settings__core"`);
+            log.info('[system-migration] dropped the legacy _settings__core table');
+        },
+    },
+    {
+        name: 'orchestrator-settings-to-system',
+        up: async (em) => {
+            const DEFAULTS = Object.fromEntries(fieldEntries(SYSTEM_SETTINGS.fields)
+                .filter(([k]) => LEGACY_ORCHESTRATOR_FIELDS.includes(k))
+                .map(([k, f]) => [k, f.default]));
+            const conn = em.getConnection();
+            const src = await introspectTable(em, '_settings__orchestrator');
+            const dst = await introspectTable(em, '_settings__system');
+            if (!src.exists || !dst.exists)
+                return;
+            const carried = LEGACY_ORCHESTRATOR_FIELDS.filter((c) => src.columns.has(c) && dst.columns.has(c));
+            if (carried.length === 0)
+                return;
+            const rows = (await conn.execute(`SELECT * FROM "_settings__orchestrator" WHERE "plugin_id" = 'orchestrator' LIMIT 1`));
+            const row = rows[0];
+            if (!row)
+                return;
+            let wrote = false;
+            for (const c of carried) {
+                const value = row[c];
+                if (value === undefined || value === null)
+                    continue;
+                const def = DEFAULTS[c];
+                const whereExtra = def === undefined ? `${q(c)} IS NULL` : `(${q(c)} IS NULL OR ${q(c)} = ?)`;
+                const params = def === undefined ? [value] : [value, def];
+                await conn.execute(`UPDATE "_settings__system" SET ${q(c)} = ? WHERE "plugin_id" = 'system' AND ${whereExtra}`, params);
+                wrote = true;
+            }
+            if (wrote)
+                log.info('[system-migration] orchestrator settings carried into _settings__system');
+        },
+    },
+    {
+        name: 'drop-legacy-orchestrator-settings',
+        up: async (em) => {
+            const conn = em.getConnection();
+            const src = await introspectTable(em, '_settings__orchestrator');
+            const dst = await introspectTable(em, '_settings__system');
+            if (!src.exists || !dst.exists)
+                return;
+            const fields = LEGACY_ORCHESTRATOR_FIELDS;
+            const srcRows = (await conn.execute(`SELECT * FROM "_settings__orchestrator" WHERE "plugin_id" = 'orchestrator' LIMIT 1`));
+            const srcRow = srcRows[0];
+            const needsLanding = fields.filter((c) => src.columns.has(c) && srcRow?.[c] !== undefined && srcRow?.[c] !== null);
+            if (needsLanding.length > 0) {
+                const dstRows = (await conn.execute(`SELECT * FROM "_settings__system" WHERE "plugin_id" = 'system' LIMIT 1`));
+                const dstRow = dstRows[0];
+                const landed = needsLanding.every((c) => dst.columns.has(c) && dstRow?.[c] !== undefined && dstRow?.[c] !== null);
+                if (!landed)
+                    return;
+            }
+            await conn.execute(`DROP TABLE IF EXISTS "_settings__orchestrator"`);
+            log.info('[system-migration] dropped the legacy _settings__orchestrator table');
+        },
+    },
+];
+export async function runSystemMigrations(em) {
+    for (const m of SYSTEM_MIGRATIONS) {
+        await m.up(em);
+    }
+}
 export function validateMigrations(pluginId, list) {
     const versions = list.map((m) => m.version).sort((a, b) => a - b);
     versions.forEach((v, i) => {
@@ -212,13 +522,46 @@ export function validateMigrations(pluginId, list) {
         }
     });
 }
+async function persistOps(em, plugin, version, ops) {
+    if (!ops.length)
+        return;
+    if (!(await introspectTable(em, '_migration_ops')).exists) {
+        log.debug(`${plugin} v${version}: _migration_ops is absent — the accounting is logged but not persisted`);
+        return;
+    }
+    const conn = em.getConnection();
+    const at = new Date().toISOString();
+    for (const [seq, o] of ops.entries()) {
+        await conn.execute(`INSERT INTO _migration_ops (plugin_id, version, seq, op, table_name, performed, rows, scanned, code, reason, empty_targets, stranded_sources, applied_at)
+       VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+       ON CONFLICT(plugin_id, version, seq) DO UPDATE SET
+         op = excluded.op, table_name = excluded.table_name, performed = excluded.performed,
+         rows = excluded.rows, scanned = excluded.scanned, code = excluded.code,
+         reason = excluded.reason, empty_targets = excluded.empty_targets,
+         stranded_sources = excluded.stranded_sources, applied_at = excluded.applied_at`, [
+            plugin,
+            version,
+            seq,
+            o.op,
+            o.table,
+            o.performed ? 1 : 0,
+            o.rows ?? null,
+            o.scanned ?? null,
+            o.code ?? null,
+            o.reason ?? null,
+            o.emptyTargets?.length ? o.emptyTargets.join(', ') : null,
+            o.strandedSources?.length ? o.strandedSources.join(', ') : null,
+            at,
+        ]);
+    }
+}
 async function anyTableExists(em, tables) {
     for (const t of tables)
         if ((await introspectTable(em, t)).exists)
             return true;
     return false;
 }
-export async function runMigrations({ em, plugins, hooks }) {
+export async function runMigrations({ em, plugins, hooks, onApplied }) {
     const conn = em.getConnection();
     for (const p of plugins) {
         const list = hooks[p.id]?.migrations ?? [];
@@ -237,9 +580,15 @@ export async function runMigrations({ em, plugins, hooks }) {
             continue;
         }
         const pending = [...list].filter((m) => m.version > stored).sort((a, b) => a.version - b.version);
-        const ctx = { table: (name) => makeTable(em, name) };
         for (const m of pending) {
+            const ops = [];
+            const ctx = { table: (name) => makeTable(em, name, (o) => ops.push(o)) };
             await m.up(ctx);
+            const concern = describeMigrationConcerns(p.id, m.version, ops);
+            if (concern)
+                log.warn(concern);
+            await persistOps(em, p.id, m.version, ops);
+            onApplied?.({ plugin: p.id, version: m.version, ops });
             await conn.execute(`INSERT INTO _migrations (plugin_id, version) VALUES (?, ?)
          ON CONFLICT(plugin_id) DO UPDATE SET version = excluded.version`, [p.id, m.version]);
         }