@ganju/utils 0.0.20 → 0.0.22

package/dist/dataEngine.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.exportRows = exports.dropColumn = exports.updateColumn = exports.copyTableStep = exports.clearFailedTableCopy = exports.writeReachesCopy = exports.readTableCopy = exports.dropColumnNeedsRebuild = exports.columnChangeNeedsRebuild = exports.addColumn = exports.dropTable = exports.updateTable = exports.createTable = exports.pruneHistory = exports.rowHistory = exports.deleteImpact = exports.write = exports.assertWritable = exports.assertActor = exports.query = exports.describe = exports.translateSqliteError = exports.normaliseColumn = exports.toTableInput = exports.loadSchema = exports.migrateInternal = exports.setMeta = exports.getMeta = exports.hasInternalTables = exports.INTERNAL_SCHEMA_VERSION = exports.INTERNAL_MIGRATIONS = void 0;
+exports.exportRows = exports.dropColumn = exports.updateColumn = exports.copyTableStep = exports.clearFailedTableCopy = exports.writeReachesCopy = exports.readTableCopy = exports.dropColumnNeedsRebuild = exports.columnChangeNeedsRebuild = exports.addColumn = exports.dropTable = exports.updateTable = exports.createTable = exports.pruneHistory = exports.rowHistory = exports.replaceFileLinks = exports.fileLinks = exports.deleteImpact = exports.recordSyncFailure = exports.syncRows = exports.write = exports.assertNotSynced = exports.assertWritable = exports.assertActor = exports.query = exports.describe = exports.translateSqliteError = exports.normaliseColumn = exports.toTableInput = exports.loadSchema = exports.migrateInternal = exports.setMeta = exports.getMeta = exports.hasInternalTables = exports.INTERNAL_SCHEMA_VERSION = exports.INTERNAL_MIGRATIONS = void 0;
 // What a project's database does, written against a tiny SQL runner so the
 // same code runs inside the Durable Object (apps/data) and on node:sqlite in
 // the tests and the tool-design eval. The runner executes; everything about
@@ -130,6 +130,26 @@ exports.INTERNAL_MIGRATIONS = [
       message TEXT NOT NULL,
       created_at TEXT NOT NULL DEFAULT (${dataDdl_1.DATA_NOW_SQL})
     )`
+    ],
+    // Time triggers: when each fires (`schedule`), how far its scans have got
+    // (`scan_state`), and the rows already fired for a due time, so none fires
+    // twice for the same one.
+    [
+        'ALTER TABLE _ganju_triggers ADD COLUMN schedule TEXT',
+        'ALTER TABLE _ganju_triggers ADD COLUMN scan_state TEXT',
+        `CREATE TABLE IF NOT EXISTS _ganju_trigger_fired (
+      trigger_id TEXT NOT NULL,
+      row_id INTEGER NOT NULL,
+      due_at TEXT NOT NULL,
+      PRIMARY KEY (trigger_id, row_id, due_at)
+    ) WITHOUT ROWID`,
+        'CREATE INDEX IF NOT EXISTS _ganju_trigger_fired_due ON _ganju_trigger_fired(due_at)'
+    ],
+    // Tables synced from a spreadsheet: where from (`source`), and how the
+    // last sync went (`sync_state`).
+    [
+        'ALTER TABLE _ganju_tables ADD COLUMN source TEXT',
+        'ALTER TABLE _ganju_tables ADD COLUMN sync_state TEXT'
     ]
 ];
 exports.INTERNAL_SCHEMA_VERSION = exports.INTERNAL_MIGRATIONS.length;
@@ -150,6 +170,17 @@ const setMeta = (runner, key, value) => {
     runner.all('INSERT INTO _ganju_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value', [key, value]);
 };
 exports.setMeta = setMeta;
+// SQLite has no ADD COLUMN IF NOT EXISTS: a column already there is
+// skipped, so every version can run again like the CREATE ... IF NOT EXISTS
+// ones.
+const ADD_COLUMN = /^ALTER TABLE (\w+) ADD COLUMN (\w+) /;
+const runMigration = (runner, statement) => {
+    const add = ADD_COLUMN.exec(statement);
+    if (add && runner.all(`PRAGMA table_info(${(0, dataIdentifiers_1.quoteIdentifier)(add[1])})`).some(c => c.name === add[2])) {
+        return;
+    }
+    runner.all(statement);
+};
 // Brings our own tables up to date. Returns the version now in force.
 const migrateInternal = (runner) => runner.transaction(() => {
     const current = (0, exports.hasInternalTables)(runner)
@@ -157,7 +188,7 @@ const migrateInternal = (runner) => runner.transaction(() => {
         : 0;
     for (let v = current; v < exports.INTERNAL_MIGRATIONS.length; v++) {
         for (const statement of exports.INTERNAL_MIGRATIONS[v])
-            runner.all(statement);
+            runMigration(runner, statement);
     }
     if (current < exports.INTERNAL_MIGRATIONS.length) {
         (0, exports.setMeta)(runner, 'schema_version', String(exports.INTERNAL_MIGRATIONS.length));
@@ -190,6 +221,8 @@ const loadSchema = (runner) => {
             customerScoped: row.customer_scoped === 1,
             history: row.history === 1,
             position: Number(row.position),
+            source: parseJson(row.source ?? null),
+            sync: parseJson(row.sync_state ?? null),
             columns: columns
                 .filter(c => c.table_id === row.id)
                 .map(c => {
@@ -335,8 +368,11 @@ const normaliseColumn = (input, schema, selfTable) => {
     if (column.unique && !dataDdl_1.UNIQUE_TYPES.has(column.type)) {
         throw new dataTypes_1.DataError(`${label}: a ${column.type} column can't be unique.`);
     }
-    if (column.searchable && !dataDdl_1.SEARCHABLE_TYPES.has(column.type)) {
-        throw new dataTypes_1.DataError(`${label}: only text columns can be searchable (text, long_text, email, url, phone, enum).`);
+    // On a file column, searchable means its uploads are indexed in knowledge.
+    if (column.searchable &&
+        !dataDdl_1.SEARCHABLE_TYPES.has(column.type) &&
+        column.type !== 'file') {
+        throw new dataTypes_1.DataError(`${label}: only text and file columns can be searchable (text, long_text, email, url, phone, enum, file).`);
     }
     // The search table has a column of its own called rank.
     if (column.searchable && column.name === 'rank') {
@@ -386,6 +422,47 @@ const normaliseAccess = (value, fallback) => {
     }
     return value;
 };
+// A table's sheet source, checked against the columns it will have. Null
+// stops syncing.
+const normaliseSource = (value, tableName, columns, customerScoped) => {
+    if (value === undefined || value === null)
+        return null;
+    // A sheet's rows have no customer, so a bot would never see them.
+    if (customerScoped)
+        throw syncedScopedError(tableName);
+    if (!isPlainObject(value)) {
+        throw new dataTypes_1.DataError('"source" is { "uri", "sheet"?, "key" }, or null to stop syncing.');
+    }
+    for (const key of Object.keys(value)) {
+        if (!['uri', 'sheet', 'key', 'title'].includes(key)) {
+            throw new dataTypes_1.DataError(`"source" takes uri, sheet and key, not "${key}".`);
+        }
+    }
+    const { uri, sheet, key, title } = value;
+    if (typeof uri !== 'string' || !/^[a-z][\w+.-]*:\/\/\S+$/i.test(uri) || uri.length > dataTypes_1.DATA_FILE_URI_MAX) {
+        throw new dataTypes_1.DataError('"source.uri" is the uri of a spreadsheet or CSV in the project\'s knowledge.');
+    }
+    if (sheet !== undefined && sheet !== null && (typeof sheet !== 'string' || !sheet.trim() || sheet.length > 100)) {
+        throw new dataTypes_1.DataError('"source.sheet" is the name of a sheet in the file, up to 100 characters.');
+    }
+    const column = columns.find(c => c.name === key);
+    if (typeof key !== 'string' || !column) {
+        throw new dataTypes_1.DataError(`"source.key" is the column that names a row in both the sheet and ${tableName}. Columns: ${columns.map(c => c.name).join(', ')}.`);
+    }
+    if (!column.unique) {
+        throw new dataTypes_1.DataError(`${tableName}.${key} must be unique to be the sync key.`);
+    }
+    return {
+        uri,
+        ...(typeof sheet === 'string' ? { sheet: sheet.trim() } : {}),
+        key,
+        ...(typeof title === 'string' && title.trim() ? { title: title.trim().slice(0, 200) } : {})
+    };
+};
+const syncedScopedError = (tableName) => new dataTypes_1.DataError(`${tableName} can't be both customer-scoped and synced from a sheet: synced rows have no customer, so the bots wouldn't see them.`);
+const sameSource = (a, b) => (a?.uri ?? null) === (b?.uri ?? null) &&
+    (a?.sheet ?? null) === (b?.sheet ?? null) &&
+    (a?.key ?? null) === (b?.key ?? null);
 // Keeping indexes, triggers and the search table in line with the metadata
 const tableObjects = (runner, type, table) => runner
     .all('SELECT name, sql FROM sqlite_master WHERE type = ? AND tbl_name = ?', [type, table])
@@ -548,6 +625,9 @@ const shapeRow = (schema, table, raw, caller) => {
                             : label
                     };
         }
+        else if (column.type === 'file') {
+            row[column.name] = value === null ? null : { uri: String(value) };
+        }
         else {
             row[column.name] = value;
         }
@@ -626,8 +706,19 @@ const query = (runner, schema, caller, rawArgs) => {
         caller,
         table,
         joins: new Map(),
-        params: []
+        params: [],
+        filtered: new Set()
     };
+    // What this query read, noted for index advice once it has its rows.
+    const readBefore = runner.usageSoFar?.().rowsRead ?? 0;
+    const sorted = [];
+    const noteShape = (returned) => runner.noteQuery?.({
+        table: table.name,
+        filtered: [...(ctx.filtered ?? [])],
+        sorted,
+        rowsRead: (runner.usageSoFar?.().rowsRead ?? 0) - readBefore,
+        returned
+    });
     const indexed = new Set();
     const where = (0, dataCompiler_1.compileWhere)(ctx, args.where, indexed);
     if ((0, dataCompiler_1.isScoped)(caller, table)) {
@@ -639,7 +730,10 @@ const query = (runner, schema, caller, rawArgs) => {
         throw new dataTypes_1.DataError(`"${table.name}" has no searchable columns; filter with "where" instead (e.g. { "name": { "contains": "..." } }).`);
     }
     if (args.aggregate !== undefined && args.aggregate !== null) {
-        return aggregate(runner, ctx, args, where, searching, indexed.size > 0);
+        const result = aggregate(runner, ctx, args, where, searching, indexed.size > 0);
+        const groups = result.groups;
+        noteShape(Array.isArray(groups) ? groups.length : 1);
+        return result;
     }
     const limit = args.limit === undefined || args.limit === null ? dataTypes_1.DATA_DEFAULT_PAGE : args.limit;
     if (typeof limit !== 'number' || !Number.isInteger(limit) || limit < 1 || limit > dataTypes_1.DATA_MAX_PAGE) {
@@ -672,6 +766,8 @@ const query = (runner, schema, caller, rawArgs) => {
             descending,
             ...(servedByIndex ? {} : { unindexed: resolved.column.name })
         });
+        if (resolved.own && resolved.column.name !== 'id')
+            sorted.push(resolved.column.name);
         if (resolved.own && resolved.column.name === 'id') {
             endsUnique = true;
             break;
@@ -763,14 +859,17 @@ const query = (runner, schema, caller, rawArgs) => {
         throw (0, exports.translateSqliteError)(schema, table, error);
     }
     const { rows: out, last, more } = result;
+    noteShape(out.length);
     const includeTruncated = args.include !== undefined && args.include !== null
         ? includeRelations(runner, schema, caller, table, out, args.include, dataTypes_1.DATA_MAX_RESULT_BYTES - result.bytes)
         : false;
+    const fileFields = fileFieldPaths(schema, table, args.include);
     return {
         table: table.name,
         ...((0, dataCompiler_1.isScoped)(caller, table) ? { scope: SCOPE_NOTE } : {}),
         ...(result.total !== undefined ? { total: result.total } : {}),
         ...(matched ? { searchMatched: matched } : {}),
+        ...(fileFields.length ? { fileFields } : {}),
         rows: out,
         ...(includeTruncated
             ? {
@@ -789,6 +888,25 @@ const query = (runner, schema, caller, rawArgs) => {
     };
 };
 exports.query = query;
+// Where a query's rows hold file values, as key paths within each row: the
+// table's file columns, and those of each included relation. The workers
+// fill those in from knowledge and drop this before answering; a json
+// column holding { "uri": … } is never mistaken for one.
+const fileFieldPaths = (schema, table, include) => {
+    const own = table.columns
+        .filter(column => column.type === 'file')
+        .map(column => [column.name]);
+    const included = (Array.isArray(include) ? include : []).flatMap(name => {
+        const column = typeof name === 'string' ? (0, dataCompiler_1.columnFor)(table, name) : undefined;
+        if (!column || (0, dataCompiler_1.isSystemColumn)(column) || column.type !== 'relation')
+            return [];
+        const target = schema.tables.find(t => t.name === column.table);
+        return (target?.columns ?? [])
+            .filter(c => c.type === 'file')
+            .map(c => [name, c.name]);
+    });
+    return [...own, ...included];
+};
 // Replaces each relation in `rows` named in `include` with the related row,
 // read as the caller may see it, within `budget` bytes. Returns whether any
 // was left as { id, label } to stay inside it.
@@ -990,6 +1108,15 @@ const assertWritable = (runner, caller, addsData) => {
     }
 };
 exports.assertWritable = assertWritable;
+// A table synced from a sheet takes its rows from the sheet alone, whoever
+// writes: the dashboard, the API, the CLI, functions and the assistant.
+const assertNotSynced = (table) => {
+    if (!table.source)
+        return;
+    const from = table.source.title || table.source.uri;
+    throw new dataTypes_1.DataError(`${table.name} is synced from "${from}"; change the sheet, or stop syncing it.`, 'read_only', { source: table.source });
+};
+exports.assertNotSynced = assertNotSynced;
 // Every write names its actor here first, so row history (kept by triggers)
 // records who made each change, cascades included.
 const setContextActor = (runner, actor) => {
@@ -1149,6 +1276,7 @@ const runOperation = (runner, schema, caller, rawOp, index, results) => {
         op = { ...rawOp, id };
     }
     const table = (0, dataCompiler_1.tableFor)(schema, caller, op.table, kind === 'insert' ? 'add' : 'full');
+    (0, exports.assertNotSynced)(table);
     const actor = caller.actor;
     const scoped = (0, dataCompiler_1.isScoped)(caller, table);
     if (kind === 'insert') {
@@ -1176,8 +1304,9 @@ const runOperation = (runner, schema, caller, rawOp, index, results) => {
         // One column list for the whole batch, so rows that leave a column out
         // get NULL there rather than its default only when another row sets it.
         const names = [...new Set(prepared.flatMap(values => [...values.keys()]))];
-        const columns = [...names, 'created_by', 'updated_by', ...(scoped ? ['participant_id'] : [])];
+        const columns = ['id', ...names, 'created_by', 'updated_by', ...(scoped ? ['participant_id'] : [])];
         const ids = [];
+        let nextId = nextRowId(runner, table);
         try {
             for (const chunk of (0, dataCompiler_1.chunkRows)(prepared, columns.length)) {
                 const usesDefaults = chunk.some(values => names.some(n => !values.has(n)));
@@ -1185,11 +1314,12 @@ const runOperation = (runner, schema, caller, rawOp, index, results) => {
                     // Rows with different columns go one by one, so each missing
                     // column takes its default.
                     for (const values of chunk)
-                        ids.push(insertOne(runner, table, values, actor, scoped, caller));
+                        ids.push(insertOne(runner, table, values, actor, scoped, caller, nextId++));
                     continue;
                 }
                 const params = [];
                 for (const values of chunk) {
+                    params.push(nextId++);
                     for (const name of names)
                         params.push(values.get(name) ?? null);
                     params.push(actor, actor);
@@ -1292,6 +1422,26 @@ const runOperation = (runner, schema, caller, rawOp, index, results) => {
                 .all(`SELECT t.id FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} t WHERE ${where}`, ctx.params)
                 .map(r => Number(r.id))
             : assertBoundedTarget();
+        // A delete here mustn't change a synced table's rows behind its sheet,
+        // through a cascade or a set null.
+        for (const other of schema.tables) {
+            if (!other.source || other.name === table.name || ids.length === 0)
+                continue;
+            for (const c of other.columns) {
+                if (c.type !== 'relation' || c.table !== table.name)
+                    continue;
+                if ((c.onDelete ?? 'restrict') === 'restrict')
+                    continue;
+                let n = 0;
+                for (let i = 0; i < ids.length; i += dataTypes_1.DATA_MAX_PARAMS) {
+                    const slice = ids.slice(i, i + dataTypes_1.DATA_MAX_PARAMS);
+                    n += Number(first(runner, `SELECT count(*) AS n FROM ${(0, dataIdentifiers_1.quoteIdentifier)(other.name)} WHERE ${(0, dataIdentifiers_1.quoteIdentifier)(c.name)} IN (${slice.map(() => '?').join(', ')})`, slice)?.n ?? 0);
+                }
+                if (n > 0) {
+                    throw new dataTypes_1.DataError(`Can't delete: it would change ${n === 1 ? 'a row' : `${n} rows`} of ${other.name} (${other.name}.${c.name}), which is synced from "${other.source.title || other.source.uri}"; change the sheet first, or stop syncing it.`, 'read_only', { table: other.name, column: c.name, count: n });
+                }
+            }
+        }
         // A readable "3 rows in order_lines still point at this order" before
         // SQLite's bare foreign key error. A row of the same table counts even
         // when this delete removes it too: SQLite checks a restrict as each row
@@ -1330,9 +1480,16 @@ const runOperation = (runner, schema, caller, rawOp, index, results) => {
     }
     return { op: kind, table: table.name, changed };
 };
-const insertOne = (runner, table, values, actor, scoped, caller) => {
-    const names = [...values.keys(), 'created_by', 'updated_by', ...(scoped ? ['participant_id'] : [])];
-    const params = [...values.values(), actor, actor, ...(scoped ? [(0, dataCompiler_1.participantOf)(caller)] : [])];
+// The id the table's next row gets. Tables have no AUTOINCREMENT, so SQLite
+// would hand out the highest id again once that row is deleted, and the
+// deleted row's history would read as the new one's. Above both, instead.
+const nextRowId = (runner, table) => Number(first(runner, `SELECT max(
+         coalesce((SELECT max(id) FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)}), 0),
+         coalesce((SELECT max(row_id) FROM _ganju_row_log WHERE table_id = ?), 0)
+       ) + 1 AS id`, [table.id])?.id ?? 1);
+const insertOne = (runner, table, values, actor, scoped, caller, id) => {
+    const names = ['id', ...values.keys(), 'created_by', 'updated_by', ...(scoped ? ['participant_id'] : [])];
+    const params = [id, ...values.values(), actor, actor, ...(scoped ? [(0, dataCompiler_1.participantOf)(caller)] : [])];
     return Number(runner.all(`INSERT INTO ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} (${names.map(dataIdentifiers_1.quoteIdentifier).join(', ')}) VALUES (${names.map(() => '?').join(', ')}) RETURNING id`, params)[0]?.id);
 };
 // A list of inserts, updates and deletes across tables, applied together or
@@ -1353,7 +1510,12 @@ const write = (runner, schema, caller, rawArgs) => {
         const results = [];
         operations.forEach((op, index) => {
             try {
+                const before = runner.usageSoFar?.().rowsWritten ?? 0;
                 results.push(runOperation(runner, schema, caller, op, index, results));
+                runner.noteWrite?.({
+                    table: results[results.length - 1].table,
+                    rowsWritten: (runner.usageSoFar?.().rowsWritten ?? 0) - before
+                });
             }
             catch (error) {
                 if (error instanceof dataTypes_1.DataError) {
@@ -1368,6 +1530,214 @@ const write = (runner, schema, caller, rawArgs) => {
     });
 };
 exports.write = write;
+// Whether `version` is older than what the table's rows were last made from.
+const staleVersion = (table, version) => typeof version === 'string' && !!table.sync?.version && version < table.sync.version;
+// The same JSON with its keys in order, so a sheet that writes it another
+// way round isn't a change.
+const canonicalJson = (value) => Array.isArray(value)
+    ? value.map(canonicalJson)
+    : isPlainObject(value)
+        ? Object.fromEntries(Object.keys(value).sort().map(k => [k, canonicalJson(value[k])]))
+        : value;
+const sameStored = (column, a, b) => {
+    if (a === b)
+        return true;
+    if (column.type !== 'json' || typeof a !== 'string' || typeof b !== 'string')
+        return false;
+    try {
+        return JSON.stringify(canonicalJson(JSON.parse(a))) === JSON.stringify(canonicalJson(JSON.parse(b)));
+    }
+    catch {
+        return false;
+    }
+};
+const nowStamp = () => new Date().toISOString().replace(/\.\d{3}Z$/, 'Z');
+// A refusal about one line of the sheet.
+const atLine = (line, error) => new dataTypes_1.DataError(`Line ${line}: ${error.message}`, error.code, error.details);
+const saveSyncState = (runner, table, state, title) => {
+    const source = table.source && typeof title === 'string' && title.trim()
+        ? { ...table.source, title: title.trim().slice(0, 200) }
+        : table.source;
+    runner.all('UPDATE _ganju_tables SET sync_state = ?, source = ? WHERE id = ?', [
+        JSON.stringify(state),
+        source ? JSON.stringify(source) : null,
+        table.id
+    ]);
+};
+// Makes a synced table's rows what the sheet says, in one transaction: new
+// keys are inserted, rows whose values differ updated, rows whose key is
+// gone deleted. Unchanged rows aren't written. Anything that doesn't fit
+// refuses the whole sync, and the table keeps its rows.
+const syncRows = (runner, schema, caller, args) => {
+    (0, exports.assertWritable)(runner, caller, true);
+    const table = tableNamed(schema, args?.table);
+    const source = table.source;
+    if (!source) {
+        throw new dataTypes_1.DataError(`${table.name} isn't synced from a sheet.`, 'conflict');
+    }
+    const keyColumn = columnNamed(table, source.key);
+    if (!Array.isArray(args.columns) || !Array.isArray(args.rows)) {
+        throw new dataTypes_1.DataError('A sync takes "columns" and "rows".');
+    }
+    const columns = [...new Set(args.columns)].map(name => columnNamed(table, name));
+    if (!columns.includes(keyColumn)) {
+        throw new dataTypes_1.DataError(`The sheet has no column for ${table.name}'s key, ${keyColumn.name}.`);
+    }
+    const unfilled = table.columns.filter(c => c.required && (c.default === undefined || c.default === null) && !columns.includes(c));
+    if (unfilled.length) {
+        throw new dataTypes_1.DataError(`The sheet has no column for ${unfilled.map(c => `${table.name}.${c.name}`).join(', ')}, which ${unfilled.length === 1 ? 'is' : 'are'} required.`);
+    }
+    if (args.rows.length > dataTypes_1.DATA_SHEET_MAX_ROWS) {
+        throw new dataTypes_1.DataError(`The sheet has ${args.rows.length.toLocaleString('en-US')} rows; a synced table takes up to ${dataTypes_1.DATA_SHEET_MAX_ROWS.toLocaleString('en-US')}.`, 'too_large');
+    }
+    // Every line as it will be stored, checked before anything is written.
+    const keyOf = (value) => JSON.stringify(value);
+    const wanted = new Map();
+    args.rows.forEach((row, i) => {
+        const line = args.lines?.[i] ?? i + 2;
+        if (!isPlainObject(row))
+            throw atLine(line, new dataTypes_1.DataError('A row is an object of column: value.'));
+        const values = new Map();
+        for (const column of columns) {
+            const raw = row[column.name];
+            let stored;
+            try {
+                stored = raw === undefined || raw === null ? null : (0, dataCompiler_1.toStored)(table, column, raw, caller.timeZone);
+            }
+            catch (error) {
+                if (error instanceof dataTypes_1.DataError)
+                    throw atLine(line, error);
+                throw error;
+            }
+            if (stored === null && column.required) {
+                throw atLine(line, new dataTypes_1.DataError(`${table.name}.${column.name} is required.`));
+            }
+            if (column.min !== null && typeof stored === 'number' && stored < column.min) {
+                throw atLine(line, new dataTypes_1.DataError(`${table.name}.${column.name} can't go below ${column.min}.`));
+            }
+            values.set(column.name, stored);
+        }
+        const key = values.get(keyColumn.name) ?? null;
+        if (key === null) {
+            throw atLine(line, new dataTypes_1.DataError(`The key, ${keyColumn.name}, is empty.`));
+        }
+        const seen = wanted.get(keyOf(key));
+        if (seen) {
+            throw atLine(line, new dataTypes_1.DataError(`${keyColumn.name} ${JSON.stringify(key)} is already on line ${seen.line}; each line of the sheet needs its own key.`, 'constraint', { key, lines: [seen.line, line] }));
+        }
+        wanted.set(keyOf(key), { line, values });
+    });
+    // Two syncs that overlap: the one that read the older file changes nothing.
+    if (staleVersion(table, args.version)) {
+        return { ...table.sync, inserted: 0, updated: 0, deleted: 0, unchanged: 0, stale: true };
+    }
+    return runner.transaction(() => {
+        setContextActor(runner, caller.actor);
+        runner.all('DELETE FROM _ganju_changes');
+        const before = runner.usageSoFar?.().rowsWritten ?? 0;
+        const names = columns.map(c => c.name);
+        const existing = new Map();
+        const stale = [];
+        let nextId = nextRowId(runner, table);
+        for (const row of runner.all(`SELECT id, ${names.map(n => (0, dataIdentifiers_1.quoteIdentifier)(n)).join(', ')} FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)}`)) {
+            const key = row[keyColumn.name] ?? null;
+            if (key !== null && wanted.has(keyOf(key)))
+                existing.set(keyOf(key), row);
+            else
+                stale.push(Number(row.id));
+        }
+        const counts = { inserted: 0, updated: 0, deleted: 0, unchanged: 0 };
+        // Rows the sheet no longer has go first, so a key that moved to a new
+        // line isn't taken twice. A row another table still points at with
+        // restrict stops the sync.
+        for (const other of schema.tables) {
+            for (const c of other.columns) {
+                if (c.type !== 'relation' || c.table !== table.name)
+                    continue;
+                if ((c.onDelete ?? 'restrict') !== 'restrict' || stale.length === 0)
+                    continue;
+                let n = 0;
+                for (let i = 0; i < stale.length; i += dataTypes_1.DATA_MAX_PARAMS) {
+                    const slice = stale.slice(i, i + dataTypes_1.DATA_MAX_PARAMS);
+                    n += Number(first(runner, `SELECT count(*) AS n FROM ${(0, dataIdentifiers_1.quoteIdentifier)(other.name)} WHERE ${(0, dataIdentifiers_1.quoteIdentifier)(c.name)} IN (${slice.map(() => '?').join(', ')})`, slice)?.n ?? 0);
+                }
+                if (n > 0) {
+                    throw new dataTypes_1.DataError(`Rows in ${other.name} still point at ${n === 1 ? 'a row' : `${n} rows`} the sheet no longer has (${other.name}.${c.name}); put them back in the sheet, or change those rows first.`, 'constraint', { table: other.name, column: c.name, count: n });
+                }
+            }
+        }
+        try {
+            for (let i = 0; i < stale.length; i += dataTypes_1.DATA_MAX_PARAMS) {
+                const slice = stale.slice(i, i + dataTypes_1.DATA_MAX_PARAMS);
+                counts.deleted += runner.all(`DELETE FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} WHERE id IN (${slice.map(() => '?').join(', ')}) RETURNING id`, slice).length;
+            }
+            for (const [key, { line, values }] of wanted) {
+                const current = existing.get(key);
+                try {
+                    if (!current) {
+                        insertOne(runner, table, values, caller.actor, false, caller, nextId++);
+                        counts.inserted++;
+                        continue;
+                    }
+                    const changed = columns
+                        .filter(c => !sameStored(c, current[c.name] ?? null, values.get(c.name) ?? null))
+                        .map(c => c.name);
+                    if (changed.length === 0) {
+                        counts.unchanged++;
+                        continue;
+                    }
+                    runner.all(`UPDATE ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} SET ${changed.map(n => `${(0, dataIdentifiers_1.quoteIdentifier)(n)} = ?`).join(', ')}, version = version + 1, updated_at = ${dataDdl_1.DATA_NOW_SQL}, updated_by = ? WHERE id = ?`, [...changed.map(n => values.get(n) ?? null), caller.actor, Number(current.id)]);
+                    counts.updated++;
+                }
+                catch (error) {
+                    throw atLine(line, (0, exports.translateSqliteError)(schema, table, error));
+                }
+            }
+        }
+        catch (error) {
+            throw (0, exports.translateSqliteError)(schema, table, error);
+        }
+        runner.noteWrite?.({
+            table: table.name,
+            rowsWritten: (runner.usageSoFar?.().rowsWritten ?? 0) - before
+        });
+        (0, dataTriggers_1.collectTriggerEvents)(runner, schema, caller);
+        const state = {
+            at: nowStamp(),
+            status: 'ok',
+            ...counts,
+            error: null,
+            version: typeof args.version === 'string' ? args.version : (table.sync?.version ?? null)
+        };
+        saveSyncState(runner, table, state, args.title);
+        return state;
+    });
+};
+exports.syncRows = syncRows;
+// Keeps why a sync was refused on the table, where the owner sees it. The
+// rows are as they were.
+const recordSyncFailure = (runner, schema, args) => {
+    const table = schema.tables.find(t => t.name === args?.table);
+    if (!table?.source)
+        return null;
+    // An older file's problem says nothing about the rows now.
+    if (staleVersion(table, args.version))
+        return table.sync ?? null;
+    const state = {
+        at: nowStamp(),
+        status: 'failed',
+        inserted: 0,
+        updated: 0,
+        deleted: 0,
+        unchanged: 0,
+        error: String(args.error ?? '').slice(0, 1000) || 'The sync failed.',
+        // The rows are still from the last file applied.
+        version: table.sync?.version ?? null
+    };
+    saveSyncState(runner, table, state, args.title);
+    return state;
+};
+exports.recordSyncFailure = recordSyncFailure;
 // Delete preview
 // What deleting `ids` of a table, or all its rows, would do to the rows that
 // point at them, counted the way SQLite applies it: cascades are followed to
@@ -1461,6 +1831,93 @@ const deleteImpact = (runner, schema, caller, rawArgs) => {
     };
 };
 exports.deleteImpact = deleteImpact;
+// File links
+const fileColumns = (schema) => schema.tables.flatMap(table => table.columns
+    .filter(column => column.type === 'file')
+    .map(column => ({ table, column })));
+const fileUriList = (value) => {
+    if (!Array.isArray(value) ||
+        value.length === 0 ||
+        value.length > dataTypes_1.DATA_FILE_LINKS_MAX_URIS) {
+        throw new dataTypes_1.DataError(`"uris" lists 1 to ${dataTypes_1.DATA_FILE_LINKS_MAX_URIS} knowledge item uris.`);
+    }
+    return [...new Set(value.map(uri => (0, dataCompiler_1.fileUriOf)(uri, 'uris')))];
+};
+// The rows whose file columns point at `uris`, per column: knowledge asks
+// before it deletes an item. Each file column is indexed.
+const fileLinks = (runner, schema, caller, rawArgs) => {
+    (0, exports.assertActor)(caller);
+    if (caller.restricted) {
+        throw new dataTypes_1.DataError('File links are for the owner.', 'forbidden');
+    }
+    const uris = JSON.stringify(fileUriList((0, dataCompiler_1.tidyArgs)(rawArgs).uris));
+    const links = [];
+    for (const { table, column } of fileColumns(schema)) {
+        const rows = Number(first(runner, `SELECT count(*) AS n FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} WHERE ${(0, dataIdentifiers_1.quoteIdentifier)(column.name)} IN (SELECT value FROM json_each(?))`, [uris])?.n ?? 0);
+        if (rows === 0)
+            continue;
+        links.push({
+            table: table.name,
+            tableLabel: table.label,
+            column: column.name,
+            columnLabel: column.label,
+            required: column.required,
+            rows
+        });
+    }
+    return { links };
+};
+exports.fileLinks = fileLinks;
+// Points the links at `from` to `to`, or clears them (`to: null`) once the
+// item is gone. A required column keeps its link, which then reads as
+// missing. Recorded in row history; triggers don't see it.
+const replaceFileLinks = (runner, schema, caller, rawArgs) => {
+    (0, exports.assertActor)(caller);
+    if (caller.restricted) {
+        throw new dataTypes_1.DataError('File links are for the owner.', 'forbidden');
+    }
+    const raw = (0, dataCompiler_1.tidyArgs)(rawArgs).changes;
+    if (!Array.isArray(raw) ||
+        raw.length === 0 ||
+        raw.length > dataTypes_1.DATA_FILE_LINKS_MAX_URIS) {
+        throw new dataTypes_1.DataError(`"changes" lists 1 to ${dataTypes_1.DATA_FILE_LINKS_MAX_URIS} { from, to } pairs.`);
+    }
+    const changes = raw.map(change => {
+        if (!isPlainObject(change)) {
+            throw new dataTypes_1.DataError('Each change is { from, to }; to is null to clear.');
+        }
+        return {
+            from: (0, dataCompiler_1.fileUriOf)(change.from, 'from'),
+            to: change.to === null ? null : (0, dataCompiler_1.fileUriOf)(change.to, 'to')
+        };
+    });
+    const columns = fileColumns(schema);
+    if (columns.length === 0)
+        return { updated: 0, kept: [] };
+    (0, exports.assertWritable)(runner, caller, changes.some(change => change.to !== null));
+    return runner.transaction(() => {
+        setContextActor(runner, caller.actor);
+        runner.all('DELETE FROM _ganju_changes');
+        let updated = 0;
+        const kept = [];
+        for (const { table, column } of columns) {
+            for (const change of changes) {
+                if (change.to === null && column.required) {
+                    const rows = Number(first(runner, `SELECT count(*) AS n FROM ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} WHERE ${(0, dataIdentifiers_1.quoteIdentifier)(column.name)} = ?`, [change.from])?.n ?? 0);
+                    if (rows > 0)
+                        kept.push({ table: table.name, column: column.name, rows });
+                    continue;
+                }
+                updated += runner.all(`UPDATE ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} SET ${(0, dataIdentifiers_1.quoteIdentifier)(column.name)} = ?, version = version + 1, updated_at = ${dataDdl_1.DATA_NOW_SQL}, updated_by = ? WHERE ${(0, dataIdentifiers_1.quoteIdentifier)(column.name)} = ? RETURNING id`, [change.to, caller.actor, change.from]).length;
+            }
+        }
+        // Tidying links after a knowledge change isn't the owner's edit: it
+        // stays in row history but fires no trigger.
+        runner.all('DELETE FROM _ganju_changes');
+        return { updated, kept };
+    });
+};
+exports.replaceFileLinks = replaceFileLinks;
 // Row history
 const rowHistory = (runner, schema, caller, args) => {
     const table = (0, dataCompiler_1.tableFor)(schema, caller, args.table);
@@ -1603,6 +2060,7 @@ const createTable = (runner, schema, caller, input) => {
     const access = normaliseAccess(input.access, 'read');
     const customerScoped = flag(input.customerScoped, 'customerScoped');
     const history = input.history === undefined ? true : flag(input.history, 'history');
+    const source = normaliseSource(input.source, name, columns, customerScoped);
     return runner.transaction(() => {
         const position = Number(first(runner, 'SELECT coalesce(max(position), -1) + 1 AS p FROM _ganju_tables')?.p ?? 0);
         // Never an id a dropped table had: its row history stays in the log
@@ -1612,7 +2070,7 @@ const createTable = (runner, schema, caller, input) => {
            coalesce((SELECT max(id) FROM _ganju_tables), 0),
            coalesce((SELECT max(table_id) FROM _ganju_row_log), 0)
          ) + 1 AS id`)?.id ?? 1);
-        const tableId = Number(runner.all('INSERT INTO _ganju_tables (id, name, label, description, access, customer_scoped, history, position) VALUES (?, ?, ?, ?, ?, ?, ?, ?) RETURNING id', [
+        const tableId = Number(runner.all('INSERT INTO _ganju_tables (id, name, label, description, access, customer_scoped, history, position, source) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) RETURNING id', [
             nextId,
             name,
             cleanText(input.label, dataTypes_1.DATA_MAX_LABEL, `${name} label`),
@@ -1620,7 +2078,8 @@ const createTable = (runner, schema, caller, input) => {
             access,
             customerScoped ? 1 : 0,
             history ? 1 : 0,
-            position
+            position,
+            source ? JSON.stringify(source) : null
         ])[0].id);
         const stored = columns.map((c, i) => ({
             ...c,
@@ -1641,7 +2100,7 @@ const updateTable = (runner, schema, caller, name, patch) => {
     const table = tableNamed(schema, name);
     if (!isPlainObject(patch))
         throw new dataTypes_1.DataError('The change is an object.');
-    const allowed = ['name', 'label', 'description', 'displayColumn', 'access', 'customerScoped', 'history'];
+    const allowed = ['name', 'label', 'description', 'displayColumn', 'access', 'customerScoped', 'history', 'source'];
     for (const key of Object.keys(patch)) {
         if (!allowed.includes(key)) {
             throw new dataTypes_1.DataError(`"${key}" can't be changed here; tables take ${allowed.join(', ')}.`);
@@ -1677,6 +2136,9 @@ const updateTable = (runner, schema, caller, name, patch) => {
             params.push(normaliseAccess(patch.access, table.access));
         }
         if ('customerScoped' in patch) {
+            const keepsSource = 'source' in patch ? patch.source !== null && patch.source !== undefined : !!table.source;
+            if (patch.customerScoped === true && keepsSource)
+                throw syncedScopedError(table.name);
             sets.push('customer_scoped = ?');
             params.push(flag(patch.customerScoped, 'customerScoped') ? 1 : 0);
         }
@@ -1684,6 +2146,15 @@ const updateTable = (runner, schema, caller, name, patch) => {
             sets.push('history = ?');
             params.push(flag(patch.history, 'history') ? 1 : 0);
         }
+        if ('source' in patch) {
+            const scoped = 'customerScoped' in patch ? patch.customerScoped === true : table.customerScoped;
+            const source = normaliseSource(patch.source, current, table.columns, scoped);
+            sets.push('source = ?');
+            params.push(source ? JSON.stringify(source) : null);
+            // A new file, sheet or key starts over: the last sync was of another.
+            if (!sameSource(table.source, source))
+                sets.push('sync_state = NULL');
+        }
         if (sets.length) {
             runner.all(`UPDATE _ganju_tables SET ${sets.join(', ')} WHERE id = ?`, [...params, table.id]);
         }
@@ -2041,7 +2512,21 @@ const updateColumn = (runner, schema, caller, tableName, columnName, patch) => {
     if (draft.name !== column.name && table.columns.some(c => c.name === draft.name)) {
         throw new dataTypes_1.DataError(`${table.name} already has a column "${draft.name}".`, 'conflict');
     }
+    // A time trigger counts from a date or datetime; anything else has no time.
+    if (draft.type !== 'date' && draft.type !== 'datetime') {
+        const timed = (0, dataTriggers_1.timeTriggersCountingFrom)(runner, table.id, column.name);
+        if (timed.length) {
+            throw new dataTypes_1.DataError(`${table.name}.${column.name} stays a date or datetime while time triggers count from it (${timed.join(', ')}); change or delete them first.`, 'constraint', { triggers: timed });
+        }
+    }
+    const isSyncKey = table.source?.key === column.name;
+    if (isSyncKey && !draft.unique) {
+        throw new dataTypes_1.DataError(`${table.name}.${column.name} is the key the sheet sync matches rows by, so it stays unique.`, 'constraint');
+    }
     if ((0, exports.columnChangeNeedsRebuild)(schema, table.name, column.name, patch)) {
+        if (isSyncKey && draft.name !== column.name) {
+            throw new dataTypes_1.DataError(`${table.name}.${column.name} is the sheet sync's key: rename it on its own, then make the other changes.`, 'constraint');
+        }
         // A rename renames the column in its triggers too; a copy can't, so the
         // two go one after the other.
         const triggers = draft.name !== column.name ? (0, dataTriggers_1.triggersUsingColumn)(runner, table.id, column.name) : [];
@@ -2056,6 +2541,12 @@ const updateColumn = (runner, schema, caller, tableName, columnName, patch) => {
             runner.all(`DROP TABLE IF EXISTS ${(0, dataIdentifiers_1.quoteIdentifier)((0, dataDdl_1.ftsTableName)(table))}`);
             runner.all(`ALTER TABLE ${(0, dataIdentifiers_1.quoteIdentifier)(table.name)} RENAME COLUMN ${(0, dataIdentifiers_1.quoteIdentifier)(column.name)} TO ${(0, dataIdentifiers_1.quoteIdentifier)(draft.name)}`);
             (0, dataTriggers_1.renameColumnInTriggers)(runner, table.id, column.name, draft.name);
+            if (isSyncKey && table.source) {
+                runner.all('UPDATE _ganju_tables SET source = ? WHERE id = ?', [
+                    JSON.stringify({ ...table.source, key: draft.name }),
+                    table.id
+                ]);
+            }
         }
         updateColumnMeta(runner, schema, table, column.id, draft);
         touchTable(runner, table.id);
@@ -2073,6 +2564,9 @@ const dropColumn = (runner, schema, caller, tableName, columnName) => {
         throw new dataTypes_1.DataError(`${table.name} needs at least one column; drop the table instead.`);
     }
     assertNoTriggerUses(runner, table, column.name);
+    if (table.source?.key === column.name) {
+        throw new dataTypes_1.DataError(`${table.name}.${column.name} is the key the sheet sync matches rows by; stop syncing the table, or pick another key, first.`, 'constraint');
+    }
     if ((0, exports.dropColumnNeedsRebuild)(schema, table.name, column.name)) {
         return rebuildTable(runner, schema, caller, table, table.columns.filter(c => c.id !== column.id).map(c => ({ draft: c, id: c.id })), { column: column.name, kind: 'drop' }, column.id);
     }