orez 0.4.73 → 0.5.2

package/dist/cf-do/worker.js

@@ -1,7 +1,9 @@
 // @ts-nocheck — cloudflare:workers types not available in orez
 import { DurableObject } from 'cloudflare:workers';
-import { isSqlMutation, RollingRowWriteBudget, trackSqlCursorRowsWritten, trackedChangeRow, WriteBudgetExceededError, } from '../do-sql-tracking.js';
-import { commitTxJournal, recoverTxJournal, rollbackTxJournal } from './tx-journal.js';
+import { isSqlMutation, isSqlRowMutation, RollingRowWriteBudget, trackSqlCursorRowsWritten, trackedChangeRow, WriteBudgetExceededError, } from '../do-sql-tracking.js';
+import { TransactionalCdc, } from './cdc.js';
+import { appendPendingChange, deletePendingChanges, ensurePendingChangesTable, rollbackPendingChanges, } from './row-undo.js';
+import { commitTxJournal, recoverTxJournal, rollbackTxJournal, snapshotSideEffectWriteTables, upgradeToTableSnapshot, } from './tx-journal.js';
 import { DurableWatermarkState } from './watermark.js';
 const SCHEMA_VERSION = 1;
 const SQL_ERROR_SNIPPET_RADIUS = 1600;
@@ -61,12 +63,14 @@ function sqlErrorSnippet(sql, message) {
 export class ZeroDO extends DurableObject {
     sql;
     watermarks;
+    cdc;
     schemaTables = new Set();
     tableSchemas = new Map();
     writeBudget;
     writeBudgetDisabled;
     writeBudgetAdminToken;
     activeWriteMeasurements = null;
+    pendingChangesSchemaReady = false;
     recordWriteBudgetRows(rows) {
         const wasTripped = this.writeBudget.status().tripped;
         try {
@@ -144,6 +148,7 @@ export class ZeroDO extends DurableObject {
                     this.writeBudget.restoreTrip(trippedAt);
             });
         }
+        this.cdc = new TransactionalCdc(this.sql);
         this.watermarks = new DurableWatermarkState(this.sql);
     }
     async fetch(request) {
@@ -421,15 +426,15 @@ export class ZeroDO extends DurableObject {
             const body = (await request.json());
             sql = body.sql;
             const params = Array.isArray(body.params) ? body.params : [];
-            // Only wrap in ctx.storage.transaction() when the call has change-tracking
-            // side effects (executeSQL writes BOTH the user table AND _zero_changes,
-            // which must commit together to keep source-tab sync flicker-free). A
-            // bare /exec is single-statement and ctx.storage.sql already serializes;
-            // the transaction wrap was adding ~2-5ms × every call, which on chat's
-            // 27k-stmt boot pushed orez backend startup past chat's 60s wait-for-port.
-            const result = await this.withSchemaProvisioningWait(() => body.track
-                ? this.ctx.storage.transaction(() => this.executeSQL(sql, params, body.track))
-                : this.executeSQL(sql, params));
+            const transactionID = String(body.transactionID || body.track?.transactionID || '');
+            // CDC rows are drained into the durable change log before the storage
+            // transaction returns. Keep unrelated DDL/read calls on the fast path:
+            // wrapping every /exec adds ~2-5ms and materially slows large schemas.
+            const needsAtomicCapture = !!body.track ||
+                (this.cdc.active && (isSqlRowMutation(sql) || this.cdc.capturesSchemaChange(sql)));
+            const result = await this.withSchemaProvisioningWait(() => needsAtomicCapture
+                ? this.atomically(() => this.executeSQL(sql, params, body.track, transactionID || undefined))
+                : this.executeSQL(sql, params, undefined, transactionID || undefined));
             return Response.json(measurements ? { ...result, writeMeasurements: measurements } : result);
         }
         catch (err) {
@@ -449,9 +454,11 @@ export class ZeroDO extends DurableObject {
     async handleBatch(request) {
         const measurements = this.startWriteMeasurement(request);
         try {
-            const { statements } = (await request.json());
-            const allRows = await this.withSchemaProvisioningWait(() => this.ctx.storage.transaction(() => {
+            const { statements, cdcTables } = (await request.json());
+            const allRows = await this.withSchemaProvisioningWait(() => this.atomically(() => {
                 const results = [];
+                if (Array.isArray(cdcTables))
+                    this.cdc.syncTables(cdcTables);
                 for (const statement of statements) {
                     const item = typeof statement === 'string' ? { sql: statement } : statement;
                     if (!item?.sql?.trim())
@@ -465,7 +472,7 @@ export class ZeroDO extends DurableObject {
                         continue;
                     }
                     try {
-                        results.push(this.executeSQL(item.sql, Array.isArray(item.params) ? item.params : [], item.track));
+                        results.push(this.executeSQL(item.sql, Array.isArray(item.params) ? item.params : [], item.track, item.transactionID || item.track?.transactionID));
                     }
                     catch (err) {
                         if (err instanceof WriteBudgetExceededError)
@@ -477,6 +484,7 @@ export class ZeroDO extends DurableObject {
             }));
             return Response.json({
                 results: allRows,
+                capturedChanges: allRows.reduce((total, result) => total + Number(result.capturedChanges ?? 0), 0),
                 ...(measurements ? { writeMeasurements: measurements } : null),
             });
         }
@@ -534,7 +542,7 @@ export class ZeroDO extends DurableObject {
             const transactionID = String(body.transactionID || '');
             if (!transactionID)
                 throw new Error('missing transactionID');
-            const count = await this.ctx.storage.transaction(() => {
+            const count = await this.atomically(() => {
                 const committed = this.commitPendingTrackedChanges(transactionID);
                 commitTxJournal(this.sql, transactionID);
                 return committed;
@@ -563,7 +571,8 @@ export class ZeroDO extends DurableObject {
             const transactionID = String(body.transactionID || '');
             if (!transactionID)
                 throw new Error('missing transactionID');
-            const count = await this.ctx.storage.transaction(() => {
+            const count = await this.atomically(() => {
+                this.rollbackPendingTrackedChanges(transactionID);
                 rollbackTxJournal(this.sql, transactionID);
                 return this.deletePendingTrackedChanges(transactionID);
             });
@@ -593,8 +602,10 @@ export class ZeroDO extends DurableObject {
         try {
             const body = (await request.json().catch(() => ({})));
             const owner = body.owner === undefined ? undefined : String(body.owner);
-            const transactionIDs = await this.ctx.storage.transaction(() => {
-                const recovered = recoverTxJournal(this.sql, owner);
+            const transactionIDs = await this.atomically(() => {
+                const recovered = recoverTxJournal(this.sql, owner, (txID) => {
+                    this.rollbackPendingTrackedChanges(txID);
+                });
                 for (const txID of recovered)
                     this.deletePendingTrackedChanges(txID);
                 return recovered;
@@ -643,7 +654,7 @@ export class ZeroDO extends DurableObject {
     }
     async handleSnapshot() {
         try {
-            return this.ctx.storage.transactionSync(() => {
+            return this.atomicallySync(() => {
                 this.ensureSchemaMetadataTable();
                 const names = this.sql
                     .exec('SELECT name FROM _zero_schema_tables ORDER BY name')
@@ -662,6 +673,42 @@ export class ZeroDO extends DurableObject {
             return Response.json({ error: err.message }, { status: 500 });
         }
     }
+    /**
+     * Run work in a storage transaction, re-deriving every in-memory schema cache
+     * from SQLite if it aborts.
+     *
+     * ctx.storage.transaction() rolls the SQLite side back on throw, but the
+     * caches are plain fields that keep asserting state SQLite no longer has: a
+     * CDC table stays "registered and verified" with no trigger left on disk, so
+     * ensureTable short-circuits and every later write to it goes silently
+     * uncaptured. The readiness flags are the same class: their CREATE TABLE is
+     * rolled back while the flag still says the table exists.
+     */
+    async atomically(work) {
+        try {
+            return await this.ctx.storage.transaction(work);
+        }
+        catch (error) {
+            this.invalidateSchemaCaches();
+            throw error;
+        }
+    }
+    atomicallySync(work) {
+        try {
+            return this.ctx.storage.transactionSync(work);
+        }
+        catch (error) {
+            this.invalidateSchemaCaches();
+            throw error;
+        }
+    }
+    invalidateSchemaCaches() {
+        this.watermarks.invalidateCache();
+        this.pendingChangesSchemaReady = false;
+        // Reload is intentionally last because corrupt persisted CDC metadata must
+        // throw (fail closed), without preventing the other caches from invalidating.
+        this.cdc.reload();
+    }
     // sync (storage.transaction-safe) column presence check for the /batch
     // skipIfColumnExists/skipIfColumnMissing conditions. a missing table reads
     // as "no columns", which makes ADD COLUMN skips behave like pg's
@@ -675,26 +722,97 @@ export class ZeroDO extends DurableObject {
             return false;
         }
     }
-    executeSQL(sql, params = [], track) {
-        const cursor = this.sql.exec(sql, ...params);
+    executeSQL(sql, params = [], track, transactionID) {
+        let capturesTrackedTable = false;
+        if (track?.physicalTableName) {
+            capturesTrackedTable = this.cdc.ensureTable({
+                physicalTableName: track.physicalTableName,
+                tableName: track.tableName,
+                ...(track.publish === false ? { publish: false } : null),
+                ...(track.rowColumns?.length ? { columns: track.rowColumns } : null),
+            });
+        }
+        else if (track) {
+            capturesTrackedTable = this.cdc.capturesTable(track.tableName);
+        }
+        // DoBackend already marked this table row-journaled, betting the DO could
+        // capture before/after images for it. When it cannot, that marker promises
+        // a rollback nothing can perform, so take the table snapshot the journal
+        // would otherwise have taken. It has to happen before the DML, while the
+        // table still holds its pre-transaction contents.
+        const trackedTransactionID = track ? transactionID || track.transactionID : undefined;
+        let snapshotsOwnStatement = false;
+        if (track && !capturesTrackedTable && trackedTransactionID) {
+            const physicalTableName = track.physicalTableName || track.tableName.replace(/^public\./, '');
+            if (this.tableExists(physicalTableName)) {
+                upgradeToTableSnapshot(this.sql, trackedTransactionID, physicalTableName);
+            }
+        }
+        if (track && trackedTransactionID) {
+            const physicalTableName = track.physicalTableName || track.tableName.replace(/^public\./, '');
+            snapshotsOwnStatement = snapshotSideEffectWriteTables(this.sql, trackedTransactionID, physicalTableName);
+        }
+        const suspendedCdc = this.cdc.beginSchemaChange(sql);
+        let cursor;
+        try {
+            cursor = this.sql.exec(sql, ...params);
+        }
+        catch (error) {
+            // Restore capture against the unchanged schema before propagating a DDL
+            // failure. In normal /exec and /batch paths this is also protected by the
+            // surrounding storage transaction.
+            this.cdc.finishSchemaChange(suspendedCdc);
+            throw error;
+        }
+        this.cdc.finishSchemaChange(suspendedCdc);
         const columns = Array.isArray(cursor.columnNames) ? cursor.columnNames : [];
         const rows = this.cursorRows(cursor, columns);
-        if (isSqlMutation(sql))
+        const mutation = isSqlMutation(sql);
+        if (mutation)
             this.writeBudget.recordLogical(rows.length);
-        if (!track)
-            return { rows, columns };
-        for (const row of rows) {
-            const trackedRow = trackedChangeRow(row, track);
-            if (track.operation === 'DELETE')
-                this.appendTrackedChange(track.tableName, 'DELETE', null, trackedRow, track.transactionID);
-            else
-                this.appendTrackedChange(track.tableName, track.operation, trackedRow, null, track.transactionID);
+        if (mutation && !isSqlRowMutation(sql))
+            this.cdc.invalidateSchema();
+        const captured = track || (this.cdc.active && isSqlRowMutation(sql)) ? this.cdc.drain() : [];
+        for (const change of captured) {
+            this.appendCapturedChange(change, transactionID || track?.transactionID, !snapshotsOwnStatement);
+        }
+        // Backward compatibility for callers that have not yet supplied the
+        // physical SQLite table identity needed to install a trigger. Once a
+        // table is registered, trigger capture is the sole source of truth and
+        // includes arbitrary business-trigger side effects.
+        //
+        // These rows carry no before-image, so they only feed the changefeed. Their
+        // rollback is owned by the table snapshot taken above, and `undoable: false`
+        // keeps the row-undo pass from trying to restore them from a wire image that
+        // cannot round-trip a blob or an int64.
+        if (track && !capturesTrackedTable) {
+            const physicalTableName = track.physicalTableName || track.tableName.replace(/^public\./, '');
+            for (const row of rows) {
+                const trackedRow = trackedChangeRow(row, track);
+                const isDelete = track.operation === 'DELETE';
+                this.appendTrackedChange({
+                    tableName: track.tableName,
+                    op: isDelete ? 'DELETE' : track.operation,
+                    rowData: isDelete ? null : trackedRow,
+                    oldData: isDelete ? trackedRow : null,
+                    transactionID: transactionID || track.transactionID,
+                    physicalTableName,
+                    publish: track.publish !== false,
+                    undoable: false,
+                });
+            }
+            if (track.publish !== false)
+                this.appendDerivedTrackedChanges(track, rows);
         }
-        this.appendDerivedTrackedChanges(track, rows);
+        const publishedCaptured = captured.filter((change) => change.publish !== false).length;
+        if (!track)
+            return { rows, columns, capturedChanges: publishedCaptured };
         return {
             rows: track.returnRows ? rows : [],
             columns: track.returnRows ? columns : [],
             affectedRows: rows.length,
+            capturedChanges: publishedCaptured ||
+                (capturesTrackedTable || track.publish === false ? 0 : rows.length),
         };
     }
     cursorRows(cursor, columns) {
@@ -745,8 +863,19 @@ export class ZeroDO extends DurableObject {
         const rows = this.sql
             .exec(`SELECT * FROM ${quoteIdent(sqliteTableName)} WHERE ${quoteIdent(keyColumn)} IN (${placeholders})`, ...values)
             .toArray();
+        // Derived notifications: the rows were written by business triggers and are
+        // captured for the changefeed only. They carry no physical table name, so
+        // the row-undo pass never sees them, and their real writes are rolled back
+        // by whichever journal entry owns the table.
         for (const row of rows) {
-            this.appendTrackedChange(publicTableName, 'UPDATE', row, null, transactionID);
+            this.appendTrackedChange({
+                tableName: publicTableName,
+                op: 'UPDATE',
+                rowData: row,
+                oldData: null,
+                transactionID,
+                undoable: false,
+            });
         }
     }
     messageRowUpdatesChannelLatestOrder(row) {
@@ -872,14 +1001,48 @@ export class ZeroDO extends DurableObject {
         this.appendChange(tn, 'DELETE', null, existing);
     }
     appendChange(tn, op, rowData, oldData) {
-        this.appendTrackedChange(tn, op, rowData, oldData);
+        this.appendTrackedChange({ tableName: tn, op, rowData, oldData });
+    }
+    /** Record a trigger-captured change, before-images and row identity included. */
+    appendCapturedChange(change, transactionID, undoable = true) {
+        this.appendTrackedChange({
+            tableName: change.tableName,
+            op: change.op,
+            rowData: change.rowData,
+            oldData: change.oldData,
+            transactionID,
+            physicalTableName: change.physicalTableName,
+            publish: change.publish !== false,
+            rowJournal: change.rowJournal,
+            oldJournal: change.oldJournal,
+            newRowid: change.newRowid,
+            oldRowid: change.oldRowid,
+            undoable,
+        });
     }
-    appendTrackedChange(tableName, op, rowData, oldData, transactionID) {
-        if (transactionID) {
-            this.appendPendingTrackedChange(tableName, op, rowData, oldData, transactionID);
+    appendTrackedChange(change) {
+        const publish = change.publish !== false;
+        if (!publish && !change.transactionID)
+            return;
+        if (change.transactionID) {
+            this.ensurePendingTrackedChangesTable();
+            appendPendingChange(this.sql, {
+                transactionID: change.transactionID,
+                physicalTableName: change.physicalTableName,
+                tableName: change.tableName,
+                publish,
+                op: change.op,
+                rowData: change.rowData,
+                oldData: change.oldData,
+                rowJournal: change.rowJournal ?? null,
+                oldJournal: change.oldJournal ?? null,
+                newRowid: change.newRowid ?? null,
+                oldRowid: change.oldRowid ?? null,
+                undoable: change.undoable === true,
+            });
             return;
         }
-        this.appendCommittedTrackedChange(tableName, op, rowData, oldData);
+        this.appendCommittedTrackedChange(change.tableName, change.op, change.rowData, change.oldData);
     }
     appendCommittedTrackedChange(tableName, op, rowData, oldData) {
         this.watermarks.ensureTables();
@@ -888,11 +1051,14 @@ export class ZeroDO extends DurableObject {
         this.watermarks.mark(watermark);
     }
     ensurePendingTrackedChangesTable() {
-        this.sql.exec("CREATE TABLE IF NOT EXISTS _zero_pending_changes (id INTEGER PRIMARY KEY AUTOINCREMENT, transaction_id TEXT NOT NULL, table_name TEXT NOT NULL, op TEXT NOT NULL CHECK (op IN ('INSERT', 'UPDATE', 'DELETE')), row_data TEXT, old_data TEXT, created_at INTEGER NOT NULL DEFAULT (unixepoch()))");
+        if (this.pendingChangesSchemaReady)
+            return;
+        ensurePendingChangesTable(this.sql);
+        this.pendingChangesSchemaReady = true;
     }
-    appendPendingTrackedChange(tableName, op, rowData, oldData, transactionID) {
+    rollbackPendingTrackedChanges(transactionID) {
         this.ensurePendingTrackedChangesTable();
-        this.sql.exec('INSERT INTO _zero_pending_changes (transaction_id, table_name, op, row_data, old_data) VALUES (?, ?, ?, ?, ?)', transactionID, tableName, op, rowData ? JSON.stringify(rowData) : null, oldData ? JSON.stringify(oldData) : null);
+        return rollbackPendingChanges(this.sql, transactionID);
     }
     commitPendingTrackedChanges(transactionID) {
         this.ensurePendingTrackedChangesTable();
@@ -901,7 +1067,7 @@ export class ZeroDO extends DurableObject {
             .exec(`INSERT INTO _zero_changes (table_name, op, row_data, old_data)
          SELECT table_name, op, row_data, old_data
          FROM _zero_pending_changes
-         WHERE transaction_id = ?
+         WHERE transaction_id = ? AND publish != 0
          ORDER BY id
          RETURNING watermark`, transactionID)
             .toArray();
@@ -916,10 +1082,7 @@ export class ZeroDO extends DurableObject {
     }
     deletePendingTrackedChanges(transactionID) {
         this.ensurePendingTrackedChangesTable();
-        const result = this.sql
-            .exec('DELETE FROM _zero_pending_changes WHERE transaction_id = ? RETURNING 1 AS deleted', transactionID)
-            .toArray();
-        return result.length;
+        return deletePendingChanges(this.sql, transactionID);
     }
     readChangesSince(watermark) {
         this.watermarks.ensureTables();
@@ -1078,7 +1241,16 @@ export class ZeroDO extends DurableObject {
                     value === true || value === 1 || value === '1' || value === 'true';
             }
             else if (type === 'number') {
-                normalized[key] = Number(value);
+                // timestamp/timestamptz columns are declared `number` in the zero
+                // schema but stored as postgres timestamp TEXT (pg-proxy-do-backend
+                // `postgresTimestampText`, e.g. "2026-07-11 13:34:46.000+00"). Coercing
+                // that text with Number() yields NaN, which JSON serializes as null and
+                // silently wipes every timestamp reaching the sync-cf-host snapshot
+                // feed. Forward a non-numeric value untouched so the engine's
+                // timestamp_text_to_epoch_ms decodes it, matching the /changes feed
+                // which forwards the raw text.
+                const numeric = Number(value);
+                normalized[key] = Number.isFinite(numeric) ? numeric : value;
             }
             else if (type === 'json' && typeof value === 'string') {
                 try {