@mason-data/cli 0.7.0 → 0.9.0

package/dist/src/duck/stream.js

@@ -1,6 +1,6 @@
 /**
  * THE DUCKDB COPY KEPT CURRENT BY LOGICAL REPLICATION (replication for the DuckDB case, only to sync the copied tables).
- * `mason start --replication` on Mason's own publication and
+ * The tables whose `sync:` is `replication`, on Mason's own publication (or the one the sources file names) and
  * slot on the developer's database (src/cli/replication-mode.ts), the slot's `pgoutput` stream (src/cdc/pgoutput.ts,
  * over src/cdc/pg-wire.ts) applied to the copied tables (src/duck/sync.ts `applyChanges`) instead of polling them.
  *
@@ -45,6 +45,7 @@
  * unchanged-TOAST update) whose earlier row went in an earlier piece is applied as an UPDATE of the columns it carries,
  * as one whose row the copy holds always is.
  */
+import { errorText, messageOf } from "../lib/secret-mask.js";
 import { decode } from "../cdc/pgoutput.js";
 import { PgConnection, formatLsn, pgIdent, pgLiteral } from "../cdc/pg-wire.js";
 const lit = (s) => `'${s.replace(/'/g, "''")}'`;
@@ -187,7 +188,7 @@ export function countsText(c) {
     return `${n(total)} change${total === 1 ? "" : "s"}${parts.length ? ` (${parts.join(", ")})` : ""}`;
 }
 /** a table's changes, none yet */
-export const noChanges = (columns = null) => ({ columns, truncated: false, keys: new Map(), counts: noCounts() });
+export const noChanges = (stay = null) => ({ stay, truncated: false, keys: new Map(), counts: noCounts() });
 /** The change folded into what the batch holds for its key: a later whole row or delete replaces it, a later partial
  *  row keeps the earlier row's values where it carries none. */
 export function fold(t, key, change) {
@@ -217,7 +218,7 @@ export function fold(t, key, change) {
  */
 export const PIECE_CHANGES = 10_000;
 /**
- * A BATCH THAT CHANGES A BIG TABLE (one over src/duck/sync.ts POLL_ROW_LIMIT, `big` in `StreamTarget.tables`) is
+ * A BATCH THAT CHANGES A BIG TABLE (one over src/lib/table-sync.ts BIG_TABLE_ROWS, `big` in `StreamTarget.tables`) is
  * applied at most every `BIG_APPLY_MS`, and a transaction's piece holds up to `BIG_PIECE_CHANGES`: each application
  * scans that table's key column (`PIECE_CHANGES` says what that costs), so fewer, larger ones — a 12 M-row migration is
  * 240 scans where it would be 1,200. The other tables in such a batch wait with it (seconds of staleness, said in
@@ -234,7 +235,7 @@ export function applySize(big, flushMs, flushRows) {
 /** A later batch's changes of a table folded into what is held of it (Sync `catchUp`): what they mean once applied in
  *  commit order after it, as `fold` does within a batch */
 export function foldInto(held, later) {
-    held.columns ??= later.columns;
+    held.stay ??= later.stay;
     if (later.truncated) {
         held.truncated = true;
         held.keys.clear();
@@ -262,6 +263,7 @@ export class ReplicationStream {
     publication;
     target;
     log;
+    errorText;
     flushMs;
     flushRows;
     constructor(o) {
@@ -270,6 +272,7 @@ export class ReplicationStream {
         this.publication = o.publication;
         this.target = o.target;
         this.log = o.log;
+        this.errorText = o.errorText ?? ((e) => errorText(e));
         this.flushMs = o.flushMs ?? 500;
         this.flushRows = o.flushRows ?? PIECE_CHANGES;
         this.acked = this.target.applied();
@@ -289,7 +292,7 @@ export class ReplicationStream {
             this.repl?.sendStatus(this.acked, true);
         }
         catch (e) {
-            this.log(`replication: could not ask the source for its position (${e.message})`);
+            this.log(`replication: could not ask the source for its position (${this.errorText(e)})`);
         }
     }
     async run() {
@@ -302,7 +305,7 @@ export class ReplicationStream {
                 if (this.stopping)
                     break;
                 const wait = Math.min(30_000, 1_000 * 2 ** Math.min(attempts++, 5));
-                this.log(`replication: the source is away (${e.message}); trying again in ${wait / 1000} s`);
+                this.log(`replication: the source is away (${this.errorText(e)}); trying again in ${wait / 1000} s`);
                 await new Promise((r) => setTimeout(r, wait));
                 continue;
             }
@@ -329,7 +332,7 @@ export class ReplicationStream {
                     this.repl = null;
                     return e;
                 }
-                this.log(`replication: the source went away (${e.message}); reconnecting`);
+                this.log(`replication: the source went away (${this.errorText(e)}); reconnecting`);
             }
             finally {
                 this.repl?.close();
@@ -367,6 +370,7 @@ export class ReplicationStream {
             firstAt = 0;
             pieces = 0;
             big = false;
+            this.target.current?.();
         };
         // a piece of the open transaction: applied and committed, the position left where it is
         const piece = async () => {
@@ -386,7 +390,7 @@ export class ReplicationStream {
                 repl.sendStatus(this.acked, true);
             }
             catch (e) {
-                this.log(`replication: the status update could not be sent (${e.message})`);
+                this.log(`replication: the status update could not be sent (${this.errorText(e)})`);
             }
         }, 10_000);
         const command = `START_REPLICATION SLOT ${pgIdent(this.slot)} LOGICAL 0/0 (proto_version '1', publication_names ${pgLiteral(`"${this.publication.replace(/"/g, '""')}"`)})`;
@@ -402,6 +406,7 @@ export class ReplicationStream {
                             this.acked = m.walEnd;
                             repl.sendStatus(this.acked);
                         }
+                        this.target.current?.();
                     }
                     else if (m.replyRequested)
                         repl.sendStatus(this.acked);
@@ -464,7 +469,10 @@ export class ReplicationStream {
                             }
                             if (table.big === true)
                                 big = true;
-                            const t = batch.get(rel.source) ?? noChanges(table.columns);
+                            // ONE STAY A BATCH: what an ended stay left in it would be dropped at the apply (`applyStreamed`), and held
+                            // beside these it is rows of another shape, or an old change replayed after the poll that read a newer
+                            const open = batch.get(rel.source);
+                            const t = open?.stay === table.stay ? open : noChanges(table.stay);
                             batch.set(rel.source, t);
                             txnChanges.set(rel.source, (txnChanges.get(rel.source) ?? 0) + 1);
                             t.counts[msg.type === "insert" ? "inserts" : msg.type === "update" ? "updates" : msg.type === "delete" ? "deletes" : "truncates"]++;
@@ -513,6 +521,7 @@ export function pgTime(us) {
 }
 /** the source being away — closed, refused, reset, a shutdown — which a reconnect outlasts; anything else is the stream's */
 function away(e) {
-    const m = e instanceof Error ? e.message : String(e);
+    // a decision: the raw words (src/lib/secret-mask.ts messageOf), never the ones Mason repeats
+    const m = messageOf(e);
     return /connection closed|ECONNREFUSED|ECONNRESET|EPIPE|ETIMEDOUT|timed out|terminating connection|the database system is (shutting down|starting up)|server closed|is active for PID/i.test(m);
 }