@mason-data/cli 0.6.2 → 0.8.0

package/dist/src/duck/stream.js

@@ -1,6 +1,6 @@
 /**
  * THE DUCKDB COPY KEPT CURRENT BY LOGICAL REPLICATION (replication for the DuckDB case, only to sync the copied tables).
- * `mason start --replication` on Mason's own publication and
+ * The tables whose `sync:` is `replication`, on Mason's own publication (or the one the sources file names) and
  * slot on the developer's database (src/cli/replication-mode.ts), the slot's `pgoutput` stream (src/cdc/pgoutput.ts,
  * over src/cdc/pg-wire.ts) applied to the copied tables (src/duck/sync.ts `applyChanges`) instead of polling them.
  *
@@ -187,7 +187,7 @@ export function countsText(c) {
     return `${n(total)} change${total === 1 ? "" : "s"}${parts.length ? ` (${parts.join(", ")})` : ""}`;
 }
 /** a table's changes, none yet */
-export const noChanges = (columns = null) => ({ columns, truncated: false, keys: new Map(), counts: noCounts() });
+export const noChanges = (stay = null) => ({ stay, truncated: false, keys: new Map(), counts: noCounts() });
 /** The change folded into what the batch holds for its key: a later whole row or delete replaces it, a later partial
  *  row keeps the earlier row's values where it carries none. */
 export function fold(t, key, change) {
@@ -217,7 +217,7 @@ export function fold(t, key, change) {
  */
 export const PIECE_CHANGES = 10_000;
 /**
- * A BATCH THAT CHANGES A BIG TABLE (one over src/duck/sync.ts POLL_ROW_LIMIT, `big` in `StreamTarget.tables`) is
+ * A BATCH THAT CHANGES A BIG TABLE (one over src/lib/table-sync.ts BIG_TABLE_ROWS, `big` in `StreamTarget.tables`) is
  * applied at most every `BIG_APPLY_MS`, and a transaction's piece holds up to `BIG_PIECE_CHANGES`: each application
  * scans that table's key column (`PIECE_CHANGES` says what that costs), so fewer, larger ones — a 12 M-row migration is
  * 240 scans where it would be 1,200. The other tables in such a batch wait with it (seconds of staleness, said in
@@ -234,7 +234,7 @@ export function applySize(big, flushMs, flushRows) {
 /** A later batch's changes of a table folded into what is held of it (Sync `catchUp`): what they mean once applied in
  *  commit order after it, as `fold` does within a batch */
 export function foldInto(held, later) {
-    held.columns ??= later.columns;
+    held.stay ??= later.stay;
     if (later.truncated) {
         held.truncated = true;
         held.keys.clear();
@@ -367,6 +367,7 @@ export class ReplicationStream {
             firstAt = 0;
             pieces = 0;
             big = false;
+            this.target.current?.();
         };
         // a piece of the open transaction: applied and committed, the position left where it is
         const piece = async () => {
@@ -402,6 +403,7 @@ export class ReplicationStream {
                             this.acked = m.walEnd;
                             repl.sendStatus(this.acked);
                         }
+                        this.target.current?.();
                     }
                     else if (m.replyRequested)
                         repl.sendStatus(this.acked);
@@ -464,7 +466,10 @@ export class ReplicationStream {
                             }
                             if (table.big === true)
                                 big = true;
-                            const t = batch.get(rel.source) ?? noChanges(table.columns);
+                            // ONE STAY A BATCH: what an ended stay left in it would be dropped at the apply (`applyStreamed`), and held
+                            // beside these it is rows of another shape, or an old change replayed after the poll that read a newer
+                            const open = batch.get(rel.source);
+                            const t = open?.stay === table.stay ? open : noChanges(table.stay);
                             batch.set(rel.source, t);
                             txnChanges.set(rel.source, (txnChanges.get(rel.source) ?? 0) + 1);
                             t.counts[msg.type === "insert" ? "inserts" : msg.type === "update" ? "updates" : msg.type === "delete" ? "deletes" : "truncates"]++;