@mason-data/cli 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (111) hide show
  1. package/README.md +90 -71
  2. package/dist/src/api/error.js +3 -0
  3. package/dist/src/api/postgres/catalog.js +222 -13
  4. package/dist/src/api/postgres/encode.js +3 -1
  5. package/dist/src/api/postgres/explain.js +3 -2
  6. package/dist/src/api/postgres/protocol.js +39 -23
  7. package/dist/src/api/postgres/server.js +1173 -207
  8. package/dist/src/api/postgres/translate.js +81 -9
  9. package/dist/src/api/request-log.js +18 -5
  10. package/dist/src/cdc/poll.js +5 -5
  11. package/dist/src/cli/background.js +29 -6
  12. package/dist/src/cli/cancel.js +75 -0
  13. package/dist/src/cli/command.js +1 -0
  14. package/dist/src/cli/commands.js +622 -183
  15. package/dist/src/cli/control.js +59 -0
  16. package/dist/src/cli/demo.js +19 -10
  17. package/dist/src/cli/dev.js +351 -111
  18. package/dist/src/cli/doctor.js +20 -10
  19. package/dist/src/cli/guides.js +79 -31
  20. package/dist/src/cli/init.js +119 -58
  21. package/dist/src/cli/machine.js +12 -11
  22. package/dist/src/cli/main.js +3 -3
  23. package/dist/src/cli/mcp.js +29 -20
  24. package/dist/src/cli/metrics.js +79 -0
  25. package/dist/src/cli/pg-tls.js +13 -3
  26. package/dist/src/cli/pg-url.js +4 -0
  27. package/dist/src/cli/pin.js +13 -12
  28. package/dist/src/cli/preflight.js +8 -7
  29. package/dist/src/cli/project.js +43 -4
  30. package/dist/src/cli/readiness.js +38 -4
  31. package/dist/src/cli/registry.js +53 -34
  32. package/dist/src/cli/replication-checks.js +7 -5
  33. package/dist/src/cli/replication-mode.js +105 -26
  34. package/dist/src/cli/request-filter.js +134 -0
  35. package/dist/src/cli/roles.js +584 -0
  36. package/dist/src/cli/run-server.js +483 -41
  37. package/dist/src/cli/runs.js +1 -1
  38. package/dist/src/cli/security.js +10 -8
  39. package/dist/src/cli/serve.js +144 -42
  40. package/dist/src/cli/share-run.js +286 -0
  41. package/dist/src/cli/share.js +286 -0
  42. package/dist/src/cli/source-edit.js +65 -13
  43. package/dist/src/cli/sources.js +287 -38
  44. package/dist/src/cli/supervise.js +12 -1
  45. package/dist/src/cli/sync.js +55 -25
  46. package/dist/src/cli/tunnel.js +200 -0
  47. package/dist/src/cli/ui.js +9 -6
  48. package/dist/src/cli/usage.js +128 -0
  49. package/dist/src/cli/user.js +496 -40
  50. package/dist/src/cli/version.js +9 -7
  51. package/dist/src/demo/seed.js +3 -0
  52. package/dist/src/duck/cancel.js +10 -1
  53. package/dist/src/duck/events.js +134 -0
  54. package/dist/src/duck/executor-worker.js +18 -9
  55. package/dist/src/duck/executor.js +43 -30
  56. package/dist/src/duck/explain.js +56 -46
  57. package/dist/src/duck/extensions.js +160 -0
  58. package/dist/src/duck/functions.js +13 -14
  59. package/dist/src/duck/gaps.js +17 -8
  60. package/dist/src/duck/grants.js +176 -0
  61. package/dist/src/duck/history.js +123 -0
  62. package/dist/src/duck/kept.js +6 -6
  63. package/dist/src/duck/plan.js +102 -25
  64. package/dist/src/duck/publication.js +6 -6
  65. package/dist/src/duck/requests.js +265 -93
  66. package/dist/src/duck/rewrite.js +25 -5
  67. package/dist/src/duck/row-filter.js +649 -0
  68. package/dist/src/duck/store.js +156 -120
  69. package/dist/src/duck/stream.js +10 -5
  70. package/dist/src/duck/sync.js +835 -268
  71. package/dist/src/duck/types.js +6 -5
  72. package/dist/src/duck/usage.js +324 -0
  73. package/dist/src/duck/users.js +64 -15
  74. package/dist/src/duck/verified.js +3 -0
  75. package/dist/src/duck/views.js +2 -3
  76. package/dist/src/lib/database-name.js +62 -0
  77. package/dist/src/lib/dotenv.js +17 -4
  78. package/dist/src/lib/log-buffer.js +68 -0
  79. package/dist/src/lib/pg-numeric.js +5 -2
  80. package/dist/src/lib/project-version.js +23 -23
  81. package/dist/src/lib/table-sync.js +267 -0
  82. package/dist/src/web/app.css +706 -66
  83. package/dist/src/web/app.js +327 -410
  84. package/dist/src/web/charts.js +229 -0
  85. package/dist/src/web/index.html +82 -18
  86. package/dist/src/web/pages/queries.js +192 -0
  87. package/dist/src/web/pages/status.js +446 -0
  88. package/dist/src/web/pages/syncs.js +155 -0
  89. package/dist/src/web/pages/tables.js +434 -0
  90. package/dist/src/web/pages/usage.js +397 -0
  91. package/dist/src/web/pages/users.js +218 -0
  92. package/dist/src/web/shared.js +568 -0
  93. package/dist/src/web/vendor/README.md +25 -0
  94. package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
  95. package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
  96. package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
  97. package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
  98. package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
  99. package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
  100. package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
  101. package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
  102. package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
  103. package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
  104. package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
  105. package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
  106. package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
  107. package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
  108. package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
  109. package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
  110. package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
  111. package/package.json +10 -3
@@ -11,6 +11,8 @@ import { createSlot, currentLsn, dropSlot, readSlot } from "../cdc/source.js";
11
11
  import { addCounts, countsText, duckValue, foldInto, noChanges, noCounts, ReplicationStream } from "./stream.js";
12
12
  import { didYouMean } from "../lib/did-you-mean.js";
13
13
  import { namedPublicationVerdict, readNamedPublication } from "./publication.js";
14
+ import { extensionFailure, loadPostgresSql } from "./extensions.js";
15
+ import { BIG_TABLE_ROWS, intervalOf, syncedText, syncText, syncWords } from "../lib/table-sync.js";
14
16
  export function syncModeOf(env) {
15
17
  const v = env.MASON_DUCK_SYNC || "poll";
16
18
  if (v !== "poll" && v !== "recopy")
@@ -18,17 +20,22 @@ export function syncModeOf(env) {
18
20
  return v;
19
21
  }
20
22
  /**
21
- * THE POLL'S ROW LIMIT: in poll mode a table the source estimates at more than this many rows
22
- * (`AddedTable.rows`, the catalog's estimate; unknown counts as small) is copied and served, but the passes do not poll
23
- * it. A poll scans its table whole (no index covers xmin) and counts it, and when the counts differ reads every key over
24
- * the wire: on a test server a 22 M-row table cost the production replica 12 s of scan and 2 s of count at every poll.
25
- * `mason sync <table>` polls one now (`Sync.pollNow`); `mason start --replication` keeps every table current, whatever
26
- * its size.
23
+ * WHEN A POLLED TABLE IS POLLED NEXT: its interval after its last poll ended, stretched to four times that poll's own
24
+ * time (`nextPollDelayMs`) — so one table's polls never take more than a fifth of the source's time, whatever its
25
+ * `sync: poll <interval>` says (a 20 s poll of a table set to `poll 5s` waits 80 s).
27
26
  */
28
- export const POLL_ROW_LIMIT = 1_000_000;
29
- /** why the passes do not keep a table over the limit current (`CopyState.unsynced`) */
30
- export const OVER_POLL_LIMIT = `over the ${POLL_ROW_LIMIT.toLocaleString("en-US")}-row sync limit`;
31
- const overPollLimit = (t) => t.rows !== null && t.rows > POLL_ROW_LIMIT;
27
+ export function nextPollAt(endedMs, intervalSeconds, pollMs) {
28
+ return endedMs + nextPollDelayMs(intervalSeconds * 1000, pollMs);
29
+ }
30
+ /**
31
+ * HOW LONG THE NEXT PASS WAITS: until the first polled table is due — and at least four times what the last pass took
32
+ * (its polls, one after another on the writer), so ALL the tables' polls together never take more than a fifth of the
33
+ * source's time, however many are due at once. Bounded by setTimeout's 2^31 - 1 ms: a later poll is waited for in
34
+ * steps, a pass that finds nothing due polling nothing.
35
+ */
36
+ export function passDelayMs(earliestDueMs, nowMs, lastPassMs) {
37
+ return Math.min(2 ** 31 - 1, Math.max(0, earliestDueMs - nowMs, 4 * lastPassMs));
38
+ }
32
39
  /** a poll asked for that cannot run (`Sync.pollNow`), its reason in the asker's words: `code` is the CLI's failure code */
33
40
  export class PollRefused extends Error {
34
41
  code;
@@ -38,6 +45,19 @@ export class PollRefused extends Error {
38
45
  export class Stopping extends Error {
39
46
  constructor() { super("Mason is stopping"); }
40
47
  }
48
+ /** How often the slot's WAL is read at most (`streamStatus`): a catalog view's row, on the connection the database's
49
+ * position is read on, so never a connection of its own — and never per poll or per batch. */
50
+ export const SLOT_READ_MS = 30_000;
51
+ /** The slot as `readSlot` read it, as `SlotWal` says it. */
52
+ export function slotWalOf(s, name, at) {
53
+ if (!s.exists)
54
+ return { error: `replication slot ${name} does not exist on the database (the next pass makes it again)` };
55
+ const status = s.walStatus === "reserved" || s.walStatus === "extended" || s.walStatus === "unreserved" || s.walStatus === "lost" ? s.walStatus : null;
56
+ // a lost slot's WAL was freed by Postgres (its restart_lsn is null): it holds none; any other null is not known
57
+ if (s.retainedBytes === null && status !== "lost")
58
+ return { error: `replication slot ${name} has no restart position yet: the WAL it holds is not known` };
59
+ return { walHeldBytes: s.retainedBytes === null ? 0 : Number(s.retainedBytes), walStatus: status, safeWalSizeBytes: s.safeBytes === null ? null : Number(s.safeBytes), at: new Date(at).toISOString() };
60
+ }
41
61
  export const ident = (name) => `"${name.replaceAll('"', '""')}"`;
42
62
  export const literal = (s) => `'${s.replaceAll("'", "''")}'`;
43
63
  export const SRC = "_mason_source";
@@ -90,7 +110,7 @@ export async function writeStaged(w, t, target, stage, where, kept) {
90
110
  * never converted in place, but copied again through the normal path (`needsCopy`), serving as it was meanwhile. Found
91
111
  * by its constraint (`duckdb_constraints()`), which every copy of that layout carries and no copy of this one does.
92
112
  */
93
- export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason: copied again in heap order";
113
+ export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason: read again whole, in heap order";
94
114
  /**
95
115
  * How long a failed copy waits before it is tried again: four times the attempt, at least 5 minutes, doubled for each
96
116
  * failure in a row, at most 6 hours (a copy whose sort failed for good read a production replica whole for
@@ -115,16 +135,16 @@ export function copyOrder(tables) {
115
135
  * of a copy that is not coming (the test server: every start said `added <all four tables>: they are copied at the
116
136
  * next pass` while all four were copied and streamed from the acknowledged position — read as a long re-copy that never
117
137
  * came). ONE line: the tables grouped by state, each group `<n> table(s) <state>: <tables>`, the current ones first,
118
- * those copied at the next pass last, each with its reason. `kept` is how a current copy is kept so (`keptWords`).
138
+ * those copied at the next pass last, each with its reason, a current copy with how it is kept so (`keptWords`).
119
139
  */
120
- export function addedText(tables, kept) {
140
+ export function addedText(tables) {
121
141
  const groups = new Map();
122
142
  for (const t of tables) {
123
- const [rank, state] = "copy" in t ? [3, `copied at the next pass (${t.copy})`]
124
- : "behind" in t ? [2, `already copied, caught up by one poll first (${t.behind})`]
125
- : "refused" in t ? [4, `not copied or streamed (${t.refused})`]
126
- : t.unsynced ? [1, `already copied, NOT kept current (${OVER_POLL_LIMIT}: mason sync polls one now, mason start --replication keeps it current)`]
127
- : [0, `already copied and ${kept}`];
143
+ const [rank, state] = "copy" in t ? [3, `synced at the next pass (${t.copy})`]
144
+ : "behind" in t ? [2, `already synced, caught up by one poll first (${t.behind})`]
145
+ : "refused" in t ? [4, `not synced or streamed (${t.refused})`]
146
+ : t.off ? [1, `already synced, ${t.kept}`]
147
+ : [0, `already synced and ${t.kept}`];
128
148
  const g = groups.get(state) ?? { rank, sources: [] };
129
149
  g.sources.push(t.source);
130
150
  groups.set(state, g);
@@ -163,8 +183,8 @@ export function copyChunkSecondsOf(env) {
163
183
  return COPY_CHUNK_SECONDS;
164
184
  const n = Number(v);
165
185
  if (!/^\d+(\.\d+)?$/.test(v.trim()) || n < COPY_CHUNK_SECONDS_MIN || n > COPY_CHUNK_SECONDS_MAX) {
166
- throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a copy's chunks is sized to take ` +
167
- `about that long to read, and a stop reads at most that much again, so set it lower for a copy that moves more often or higher for fewer, larger chunks, or unset it for ${COPY_CHUNK_SECONDS}`);
186
+ throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a first sync's chunks is sized to take ` +
187
+ `about that long to read, and a stop reads at most that much again, so set it lower for a first sync that moves more often or higher for fewer, larger chunks, or unset it for ${COPY_CHUNK_SECONDS}`);
168
188
  }
169
189
  return n;
170
190
  }
@@ -345,7 +365,7 @@ export function copyParallelOf(env) {
345
365
  return COPY_PARALLEL;
346
366
  const n = Number(v);
347
367
  if (!/^\d+$/.test(v.trim()) || n < 1 || n > COPY_PARALLEL_MAX) {
348
- throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a copy's chunks on a backend of the database's own, ` +
368
+ throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a first sync's chunks on a backend of the database's own, ` +
349
369
  `so set it to 1 (one at a time, the lightest on the database) up to ${COPY_PARALLEL_MAX}, or unset it for ${COPY_PARALLEL}`);
350
370
  }
351
371
  return n;
@@ -384,7 +404,7 @@ const pq = (sql) => `postgres_query(${literal(SRC)}, ${literal(sql)})`;
384
404
  * differ (`1` beside `1.50`); the copy answers them all with s decimals, also said.
385
405
  */
386
406
  const NUMERIC_TYPE = /^numeric(?:\((\d+)(?:,\s*(\d+))?\))?$/;
387
- const WIDER = "mason: a numeric value wider than its copy";
407
+ const WIDER = "mason: a numeric value wider than its synced column";
388
408
  /** a source numeric the extension reads as a DOUBLE: no precision, or more than DuckDB's 38 digits */
389
409
  function readsAsDouble(type) {
390
410
  const m = NUMERIC_TYPE.exec(type);
@@ -429,8 +449,20 @@ export class Sync {
429
449
  mode;
430
450
  /** each polled table's watermark: the snapshot of the last poll that stored it */
431
451
  snapshots = new Map();
432
- /** how long the last pass took: the next waits four times that, at least the interval */
452
+ /** how long the last pass took: the next waits at least four times that (its polls' share of the source's time) */
433
453
  lastPassMs = 0;
454
+ /** each polled table's next poll (ms since the epoch: its last poll's end plus its interval, stretched to four times
455
+ * that poll — `nextPollDelayMs`); absent: from its last poll's time in the ledger (`dueAt`). One pass polls the
456
+ * tables due and the timer waits for the next one due (`nextPassDelay`), never a timer per table: a pass also
457
+ * queues the copies and drops removed tables, and attaches the source once for all of it — timers of their own would
458
+ * each attach it, and race those for the writer's turn */
459
+ due = new Map();
460
+ /** each table's sync as its last poll or copy recorded it (`_mason.poll_state.sync`; null: recorded before syncs were) */
461
+ recordedSync = new Map();
462
+ /** the tables whose sync changed since they were last polled (`setTables`): kept, and caught up by ONE poll now */
463
+ switched = new Set();
464
+ /** how long each table's last poll took, this run (ms): its next waits four times that, and mason add estimates by it */
465
+ lastPollMs = new Map();
434
466
  /** each table whose copy failed: the failures in a row, and when it is tried again (`copyRetryDelayMs`) */
435
467
  retries = new Map();
436
468
  /** each copied table in a layout this Mason no longer keeps (`RETIRED_LAYOUT`), found at `open`: copied again */
@@ -455,10 +487,17 @@ export class Sync {
455
487
  attached = false;
456
488
  loaded = false;
457
489
  stopped = false;
458
- /** REPLICATION MODE: the publication and slot it reads (`owned`: the publication is Mason's own, made and kept equal
459
- * to the added tables; else the operator named it, and it is only read — src/duck/publication.ts), the stream while it
460
- * runs, the last commit DuckDB applied */
490
+ /** REPLICATION: the publication and slot the `replication` tables are read through (`owned`: the publication is
491
+ * Mason's own, made and kept equal to the replicated tables; else the operator named it, and it is only read —
492
+ * src/duck/publication.ts), null in a run that cannot stream (the demo's); the stream while it runs, the last commit
493
+ * DuckDB applied */
461
494
  replication;
495
+ /** told when the slot is made (true) and when it is dropped, verified gone (false): the run records it, so `mason
496
+ * destroy` drops what was made by its own name (src/cli/dev.ts) */
497
+ onReplication;
498
+ /** the last moment the stream had everything the database wrote (a keepalive with nothing pending, a batch applied):
499
+ * a streamed table's copy is current as of then (`keptOf`'s syncedAt) */
500
+ streamCurrentAt = null;
462
501
  stream = null;
463
502
  appliedLsn = 0n;
464
503
  /** the slot has a row in `_mason.stream_state`: what this copy applied of it is known */
@@ -472,8 +511,13 @@ export class Sync {
472
511
  held = new Map();
473
512
  /** REPLICATION: what the stream applies, by source — the tables joined, and those held — read at each change */
474
513
  streamed = new Map();
514
+ /** the last `StreamedTable.stay` given (`refreshStreamed`) */
515
+ lastStay = 0;
475
516
  /** a NAMED publication's gaps: each added table it does not carry, with the refusal (neither copied nor streamed) */
476
517
  unpublished = new Map();
518
+ /** the error each table a named publication did not carry was given (`settlePrepared`): cleared by the pass that finds
519
+ * it carried again, and by nothing else that pass */
520
+ gapErrors = new Map();
477
521
  /** a named publication that does not publish TRUNCATE has been said (once a run) */
478
522
  truncateSaid = false;
479
523
  /** the last pass failed: in replication mode it is tried again after the interval, the stream started by it */
@@ -486,12 +530,16 @@ export class Sync {
486
530
  * (a pass kicked sooner leaves it stopped) */
487
531
  streamFailures = 0;
488
532
  streamAt = 0;
489
- /** when each streamed table's rows were last counted for the ledger */
490
- countedAt = new Map();
491
- /** when each table was last copied or polled (`_mason.poll_state.polled_at`), as an unsynced table's state says it */
533
+ /** when each copy was last brought current, and how (`Synced`): seeded at `open` from the ledger, moved by every
534
+ * swap, poll commit and stream batch */
535
+ synced = new Map();
536
+ /** where the sync's failures and notable events go (src/duck/events.ts `SyncEvents.record`): never the writer's turn */
537
+ onEvent;
538
+ /** when each table was last copied or polled, its `syncedAt`: `_mason.poll_state.polled_at` and nothing else — the
539
+ * value `pollStateSql` wrote is the one set here once its transaction commits, and `readPolledAt` only moves a time
540
+ * forward, so nothing moves it back (a time read at the commit stood until that read replaced it: a `syncedAt` that
541
+ * went backwards). A copy's is its watermark's, taken before the swap; `copied_at`, after it, is `Synced.at`'s */
492
542
  polledAt = new Map();
493
- /** the tables over the poll's row limit already said in the log (said once, when first left out) */
494
- leftOut = new Set();
495
543
  /** the store's writer: the stream's batches, the polls, the source's attachment and every swap, one turn at a time */
496
544
  writerLane;
497
545
  /** the rebuilds' own connection (`open`): a copy or a catch-up, beside the writer's work */
@@ -505,7 +553,6 @@ export class Sync {
505
553
  /** the source attached for this many users (a pass, a rebuild): DETACHed when the last ends (`detach`) */
506
554
  sourceUsers = 0;
507
555
  currentSchema;
508
- database;
509
556
  /** what the catalog last built from: replaced (a new object) only when it changes */
510
557
  schemaKey = "";
511
558
  /** REPLICATION, WHERE THE STREAM IS (`streamStatus`, the stream's log): the changes it applied this run by table, and
@@ -540,15 +587,21 @@ export class Sync {
540
587
  catchingUp = null;
541
588
  /** the stream's last failure and when it starts again; cleared when it starts */
542
589
  streamFailure = null;
590
+ /** A WHOLE-SYNC FAILURE NOT YET OVER, as its event said it (`table: null`): the stream stopped, over when it next applies
591
+ * a batch; the publication and slot could not be made ready, over when a pass makes them so; the database's attach,
592
+ * over when an attach succeeds. Its end is a `recovered` event of no table, which lets the next such failure be said again
593
+ * (src/duck/events.ts: a failure said word for word within an hour is one event unless it recovered between) */
594
+ wholeFailure = { stream: null, prepare: null, attach: null };
543
595
  /** the database's position as last read (`databasePosition`), and the read in flight */
544
596
  dbPosition = null;
597
+ /** the slot's WAL as last read (`databasePosition`, at most every SLOT_READ_MS), and when */
598
+ slotWal = null;
545
599
  dbPositionRead = null;
546
600
  /** THE STORED JSON PATHS (src/duck/kept.ts): every write below computes them; the plans read them */
547
601
  kept;
548
602
  constructor(store, source, database, log, intervalMs, mode = "poll", replication = null, options = {}) {
549
603
  this.store = store;
550
604
  this.source = source;
551
- this.database = database;
552
605
  this.log = log;
553
606
  this.intervalMs = intervalMs;
554
607
  this.mode = mode;
@@ -556,16 +609,18 @@ export class Sync {
556
609
  this.copyChunkRows = options.copyChunkRows ?? COPY_CHUNK_ROWS;
557
610
  this.copyParallel = options.copyParallel ?? COPY_PARALLEL;
558
611
  if (!Number.isInteger(this.copyParallel) || this.copyParallel < 1 || this.copyParallel > COPY_PARALLEL_MAX) {
559
- throw new Error(`a copy reads with 1 to ${COPY_PARALLEL_MAX} readers, not ${this.copyParallel} (Mason bug: report it; copyParallelOf reads MASON_COPY_PARALLEL)`);
612
+ throw new Error(`a first sync reads with 1 to ${COPY_PARALLEL_MAX} readers, not ${this.copyParallel} (Mason bug: report it; copyParallelOf reads MASON_COPY_PARALLEL)`);
560
613
  }
561
614
  this.copyChunkSeconds = options.copyChunkSeconds ?? COPY_CHUNK_SECONDS;
562
615
  if (!(this.copyChunkSeconds > 0))
563
- throw new Error(`a copy's chunk takes a positive number of seconds, not ${this.copyChunkSeconds} (Mason bug: report it; copyChunkSecondsOf reads MASON_COPY_CHUNK_SECONDS)`);
616
+ throw new Error(`a first sync's chunk takes a positive number of seconds, not ${this.copyChunkSeconds} (Mason bug: report it; copyChunkSecondsOf reads MASON_COPY_CHUNK_SECONDS)`);
564
617
  this.onChunk = options.onChunk;
618
+ this.onEvent = options.onEvent ?? (() => undefined);
565
619
  this.diskFree = options.diskFree ?? ((dir) => { const st = fs.statfsSync(dir); return st.bavail * st.bsize; });
566
620
  if (mode === "replication" && replication === null)
567
621
  throw new Error("replication mode needs its slot and publication (Mason bug: report it)");
568
622
  this.replication = replication;
623
+ this.onReplication = options.onReplication;
569
624
  this.currentSchema = { database, tables: [] };
570
625
  this.writerLane = { conn: store.writer, inFlight: false };
571
626
  this.kept = new KeptJsonPaths(store, {
@@ -583,6 +638,24 @@ export class Sync {
583
638
  throw new Error("the sync's rebuild connection is not open (Mason bug: report it)");
584
639
  return this.rebuilderLane;
585
640
  }
641
+ /** an event of the sync, to its record (src/duck/events.ts), which never throws into the sync; a sink that does is
642
+ * said, and the sync goes on (its state and its log line hold what happened) */
643
+ event(e) {
644
+ try {
645
+ this.onEvent(e);
646
+ }
647
+ catch (x) {
648
+ this.log(`the sync's event (${e.kind}${e.table ? ` ${e.table}` : ""}) could not be recorded: ${x instanceof Error ? x.message : String(x)}`);
649
+ }
650
+ }
651
+ /** a whole-sync failure (`wholeFailure`) is over: said once, as a `recovered` event of no table */
652
+ recovered(what, words) {
653
+ const was = this.wholeFailure[what];
654
+ if (was === null)
655
+ return;
656
+ this.wholeFailure[what] = null;
657
+ this.event({ kind: "recovered", table: null, message: `${words} (it had failed: ${was.split("\n")[0]})` });
658
+ }
586
659
  /** `f` on the writer's next turn: once every job queued before it has ended, so no other job's statement lands inside
587
660
  * its transaction. A job never queues another and waits for it (it would wait for itself) */
588
661
  onWriter(f) {
@@ -597,7 +670,9 @@ export class Sync {
597
670
  // the re-sort's count of rows written out of key order (in the earlier layout): gone with the key order
598
671
  for (const gone of ["written", "counted"])
599
672
  await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.copy_state DROP COLUMN IF EXISTS ${gone}`);
600
- await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ)`);
673
+ await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ, sync VARCHAR)`);
674
+ // the sync each table was last polled or copied under (src/lib/table-sync.ts syncText): a change is caught up by one poll
675
+ await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.poll_state ADD COLUMN IF NOT EXISTS sync VARCHAR`);
601
676
  // replication mode's record: per slot, the end of the last commit applied, in the same transaction as its rows
602
677
  await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.stream_state (slot VARCHAR PRIMARY KEY, lsn VARCHAR, applied_at TIMESTAMPTZ)`);
603
678
  // the tables copied (or under a new slot) and not yet caught up by their one poll (`behind`), written
@@ -619,8 +694,9 @@ export class Sync {
619
694
  this.slotKnown = r.length > 0;
620
695
  this.appliedLsn = r.length > 0 ? parseLsn(String(r[0][0])) : 0n;
621
696
  }
622
- for (const [source, snapshot] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
697
+ for (const [source, snapshot, sync] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot, sync FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
623
698
  this.snapshots.set(source, snapshot);
699
+ this.recordedSync.set(source, sync);
624
700
  }
625
701
  await this.readPolledAt();
626
702
  const r = await this.store.writer.runAndReadAll(`SELECT source, phase, "rows", bytes, seconds, epoch_ms(copied_at), error, columns::VARCHAR FROM ${MASON_SCHEMA}.copy_state`);
@@ -635,6 +711,7 @@ export class Sync {
635
711
  }
636
712
  this.states.set(source, { source, phase: phase === "copying" ? "waiting" : phase, rows: rows === null ? null : Number(rows), bytes: bytes === null ? null : Number(bytes), seconds, copiedAt, error });
637
713
  }
714
+ await this.readSynced();
638
715
  await this.findRetired();
639
716
  this.removeStageLeftovers();
640
717
  // the stored JSON paths, before the catalog serves anything: each served copy's view, what the file holds
@@ -655,11 +732,51 @@ export class Sync {
655
732
  const added = tables.filter((t) => !before.has(t.source));
656
733
  const removed = [...before.keys()].filter((s) => !this.tables.has(s));
657
734
  if (added.length)
658
- this.log(addedText(added.map((t) => this.addedStatus(t)), this.keptWords()));
735
+ this.log(addedText(added.map((t) => this.addedStatus(t))));
736
+ // A TABLE WHOSE SYNC CHANGED — by mason add --sync now, or by hand since its last poll (the ledger's record) — keeps
737
+ // its copy and is caught up by ONE poll now, whatever it changed to, then kept as it says: never copied again
738
+ for (const t of tables) {
739
+ const now = syncText(this.syncOf(t)), prev = before.get(t.source);
740
+ const was = prev !== undefined ? syncText(this.syncOf(prev)) : this.recordedSync.get(t.source) ?? null;
741
+ if (was === null || was === now)
742
+ continue;
743
+ this.due.delete(t.source);
744
+ // recopy copies every table again whatever its sync, and a table whose columns changed too is copied again with
745
+ // them, its new sync recorded at that copy's swap
746
+ const copied = this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source) && this.copiedAs.get(t.source) === columnsKey(t.columns);
747
+ if (this.mode === "recopy" || !copied)
748
+ continue;
749
+ if (this.replicated(t)) {
750
+ // INTO THE STREAM (poll or off → replication): caught up by ONE poll with the stream's changes of it held from
751
+ // just before that poll (`catchUp`), as a slot new to the copy is — once the pass has the publication carry it
752
+ // (`prepareReplication` runs before the pass queues the catch-up). Marked in `_mason.catch_up`, on the writer's
753
+ // turn and so before the catch-up starts: the stream acknowledges what it holds, so a run that dies between
754
+ // the catch-up's poll and the held changes' apply must catch it up again (its recorded sync is already the new)
755
+ this.switched.delete(t.source);
756
+ const why = `after its sync changed from ${was}`;
757
+ if (!this.behind.has(t.source)) {
758
+ const mark = { why };
759
+ this.behind.set(t.source, mark);
760
+ void this.onWriter(() => this.store.writer.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(why)}, now())`)).catch((e) => {
761
+ // its next start catches it up again only by this row: said, and the mark kept for this run's own catch-up
762
+ this.log(`${t.source}: its catch-up could not be recorded (a stop before it ends catches it up again only if its sync changes): ${e instanceof Error ? e.message : String(e)}`);
763
+ });
764
+ }
765
+ }
766
+ else {
767
+ // OUT OF THE STREAM, or between off and poll: it leaves the stream at its next batch (`refreshStreamed`) and the
768
+ // next pass polls it from its watermark, which every change since is at or past
769
+ this.switched.add(t.source);
770
+ }
771
+ // said, and recorded as the sync's event (src/duck/events.ts): what a table's sync was is history the log alone loses
772
+ const switched = `sync: ${was} → ${now}: its synced data is kept, and caught up by one poll now${this.replicated(t) ? ", then streamed" : ""}`;
773
+ this.log(`${t.source}: ${switched}`);
774
+ this.event({ kind: "sync_changed", table: t.source, message: switched });
775
+ }
659
776
  // loud: the copy goes with the table's entry, and comes back only by reading the source again (the test server: a 53 GB
660
777
  // copy, 21 minutes of a production replica, dropped by a project file moved aside)
661
778
  if (removed.length)
662
- this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its copy is" : "their copies are"} DROPPED at the next pass — mason add copies ${removed.length === 1 ? "it" : "them"} again from the source, whole`);
779
+ this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its synced data is" : "their synced data is"} DROPPED at the next pass — mason add syncs ${removed.length === 1 ? "it" : "them"} again from the source, whole`);
663
780
  for (const t of tables)
664
781
  if (!this.states.has(t.source))
665
782
  this.states.set(t.source, { source: t.source, phase: "waiting", rows: null, bytes: t.bytes, seconds: null, copiedAt: null, error: null });
@@ -684,27 +801,107 @@ export class Sync {
684
801
  // a named publication's gap is said again by the pass, which refuses the table by name until its owner adds it
685
802
  if (st?.error?.startsWith("replication: "))
686
803
  return { source: t.source, refused: firstLine(st.error.slice("replication: ".length)) };
687
- const copy = st === undefined ? "never copied"
688
- : st.phase === "failed" ? `its last copy failed: ${firstLine(st.error)}`
689
- : st.phase !== "copied" ? "its copy has not finished"
690
- : this.retired.has(t.source) ? "its copy is in an older Mason's layout, key order with an index"
691
- : this.copiedAs.get(t.source) !== columnsKey(t.columns) ? "its columns changed"
692
- : !this.snapshots.has(t.source) ? "a numeric value outgrew its copy's DECIMAL"
693
- : null;
804
+ const copy = st === undefined ? "never synced"
805
+ : st.phase === "failed" ? `its first sync failed: ${firstLine(st.error)}`
806
+ : st.phase !== "copied" ? "its first sync has not finished"
807
+ : this.retired.has(t.source) ? "it is in an older Mason's layout, key order with an index"
808
+ : this.readAgainWhy(t);
694
809
  if (copy !== null)
695
810
  return { source: t.source, copy };
696
811
  const b = this.behind.get(t.source);
697
812
  if (b !== undefined)
698
813
  return { source: t.source, behind: b.why };
699
- if (this.replication !== null && !this.slotKnown)
700
- return { source: t.source, behind: `slot ${this.replication.slot} is new to this copy` };
701
- return { source: t.source, unsynced: this.mode === "poll" && overPollLimit(t) };
814
+ if (this.replicated(t) && this.replication !== null && !this.slotKnown)
815
+ return { source: t.source, behind: `slot ${this.replication.slot} is new to the synced data` };
816
+ const off = this.mode !== "recopy" && this.syncOf(t).mode === "off";
817
+ return { source: t.source, kept: this.keptWords(t), off };
702
818
  }
703
- /** how a copy that is current is kept so, in this run's mode: the start line's words (`addedText`) */
704
- keptWords() {
819
+ /** how a copy that is current is kept so, in this run's mode and by its sync: the start line's words (`addedText`) */
820
+ keptWords(t) {
705
821
  const every = `${Number((this.intervalMs / 1000).toFixed(1))} s`;
706
- return this.mode === "replication" ? `streamed from ${formatLsn(this.appliedLsn)}`
707
- : this.mode === "poll" ? `polled every ${every}` : `copied again whole every ${every}`;
822
+ const s = this.syncOf(t);
823
+ return this.mode === "recopy" ? `read again whole every ${every}`
824
+ : s.mode === "replication" ? `streamed from ${formatLsn(this.appliedLsn)}`
825
+ : s.mode === "off" ? `sync: off (read again only by mason sync ${t.source})` : syncWords(s);
826
+ }
827
+ /** how a table is kept current, as the sources file says (`AddedTable.sync`); a table with none — a test's — polled at
828
+ * the sync's own interval */
829
+ syncOf(t) {
830
+ if (t.sync !== undefined)
831
+ return t.sync;
832
+ return this.mode === "replication" ? { mode: "replication" } : { mode: "poll", every: intervalOf(this.intervalMs / 1000), seconds: this.intervalMs / 1000 };
833
+ }
834
+ /** a table the stream keeps current: its sync is `replication` (never in a run that copies every table again whole) */
835
+ replicated(t) {
836
+ return this.mode !== "recopy" && this.syncOf(t).mode === "replication";
837
+ }
838
+ /** the added tables the stream keeps current, in the project's order */
839
+ replicatedTables() {
840
+ return [...this.tables.values()].filter((t) => this.replicated(t));
841
+ }
842
+ /** THE RUN STREAMS NOW: a table's sync is `replication` (the slot is made, or kept, and the stream runs) */
843
+ streams() {
844
+ return this.replication !== null && this.replicatedTables().length > 0;
845
+ }
846
+ /**
847
+ * WHEN A TABLE'S NEXT POLL IS DUE (ms since the epoch), or null when none is: a table whose sync changed at once
848
+ * (`switched`), unless a failed attempt set its next; an `off` one never; a polled one at `due`, else its last poll's
849
+ * time in the ledger plus its interval (a restart does not poll a table before its interval), else now.
850
+ */
851
+ dueAt(t) {
852
+ const s = this.syncOf(t), d = this.due.get(t.source);
853
+ if (s.mode !== "poll" && !this.switched.has(t.source))
854
+ return null;
855
+ if (d !== undefined)
856
+ return d;
857
+ if (this.switched.has(t.source) || s.mode !== "poll")
858
+ return 0;
859
+ const at = this.polledAt.get(t.source);
860
+ return at === undefined ? 0 : Date.parse(at) + s.seconds * 1000;
861
+ }
862
+ /** a table polled (by a pass, `mason sync` or its catch-up) in `ms`: its next poll after its interval, stretched to four
863
+ * times that poll (`nextPollDelayMs`), and its sync now the one recorded (the poll wrote it, `poll`) */
864
+ polledNow(t, ms) {
865
+ const s = this.syncOf(t);
866
+ this.switched.delete(t.source);
867
+ this.lastPollMs.set(t.source, ms);
868
+ this.recordedSync.set(t.source, syncText(s));
869
+ if (s.mode === "poll")
870
+ this.due.set(t.source, nextPollAt(Date.now(), s.seconds, ms));
871
+ else
872
+ this.due.delete(t.source);
873
+ }
874
+ /**
875
+ * WHAT THIS RUN MEASURED OF A TABLE, for `mason add`'s estimate of what polling it costs the database
876
+ * (src/cli/source-edit.ts pollCost): its last poll's time, and ONE reader's rate at its copy — the copy reading now,
877
+ * else the last copy's rows over its time and its readers. Null when nothing of it is measured.
878
+ */
879
+ measured(source) {
880
+ const st = this.states.get(source);
881
+ const live = this.copyLive.get(source)?.rate?.rowsPerSecond ?? null;
882
+ const copied = st?.phase === "copied" && st.rows && st.seconds ? st.rows / st.seconds / this.copyParallel : null;
883
+ const last = this.lastPollMs.get(source) ?? null;
884
+ return last === null && live === null && copied === null ? null : { lastPollMs: last, copyRowsPerSecond: live ?? copied };
885
+ }
886
+ /**
887
+ * HOW LONG UNTIL THE NEXT PASS, in poll mode (null: none until something kicks one — every table `off`, or nothing
888
+ * copied yet): a pass that failed, or a table waiting for a copy nothing queued, after the sync's interval; else when
889
+ * the first polled table is due — and at least four times the last pass took, so the passes' polls together never
890
+ * take more than a fifth of the source's time. `recopy` copies every table again every interval, stretched the same.
891
+ */
892
+ nextPassDelay() {
893
+ if (this.passFailed || this.waiting())
894
+ return this.intervalMs;
895
+ if (this.mode === "recopy")
896
+ return nextPollDelayMs(this.intervalMs, this.lastPassMs);
897
+ // a table copying, queued, or owed its catch-up is not polled by a pass (its catch-up polls it, then `rearm`): counted
898
+ // as due it would run a pass every few ms that polls nothing and attaches the source each time
899
+ const dues = [...this.tables.values()].filter((t) => this.pollable(t)).flatMap((t) => this.dueAt(t) ?? []);
900
+ return dues.length ? passDelayMs(Math.min(...dues), Date.now(), this.lastPassMs) : null;
901
+ }
902
+ /** a table a pass would poll now if it were due: copied, not queued or rebuilt, and not owed its catch-up */
903
+ pollable(t) {
904
+ return !this.needsCopy(t) && !this.behind.has(t.source) && !this.queued.has(t.source) && this.rebuildingNow?.source !== t.source;
708
905
  }
709
906
  /** the source's views (src/cli/preflight.ts VIEWS_SQL), served over the copied tables: in the catalog from the next
710
907
  * pass. A view over a table that is not copied is left out of it (said in the catalog's log) */
@@ -731,24 +928,33 @@ export class Sync {
731
928
  if (t === undefined)
732
929
  return s;
733
930
  const kept = this.keptOf(t, now);
734
- return { ...s, kept, ...(kept.by === "poll" && kept.unsynced !== undefined ? { unsynced: { reason: kept.unsynced, asOf: kept.polledAt } } : {}) };
931
+ return { ...s, kept };
932
+ })
933
+ // when it was last brought current, whether it waits for its catch-up, and whether it is being rebuilt now
934
+ .map((s) => {
935
+ const synced = this.synced.get(s.source), behind = this.behind.get(s.source)?.why;
936
+ const rebuilding = this.rebuildingNow?.source === s.source ? this.rebuildingNow.kind : undefined;
937
+ return { ...s, ...(synced ? { synced } : {}), ...(behind !== undefined ? { behind } : {}), ...(rebuilding ? { rebuilding } : {}) };
735
938
  });
736
939
  }
737
940
  /**
738
941
  * HOW AN ADDED TABLE IS KEPT CURRENT, NOW (`KeptCurrent`): the run's mode, then in replication mode its part in the
739
- * stream (`streamTable`, what `streamStatus` says of it) and where the stream is; in poll mode its last poll and
740
- * whether the passes leave it as copied. The one place it is derived: `state` carries it to `mason status` and the
741
- * listener's EXPLAIN alike.
942
+ * stream (`streamTable`, what `streamStatus` says of it) and where the stream is; in poll mode its sync, its last copy
943
+ * or poll and its next. The one place it is derived: `state` carries it to `mason status` and the listener's EXPLAIN
944
+ * alike.
742
945
  */
743
946
  keptOf(t, now) {
744
947
  const st = this.states.get(t.source);
745
- if (this.mode === "poll") {
948
+ const sync = this.syncOf(t);
949
+ if (this.mode !== "recopy" && sync.mode !== "replication") {
746
950
  const polledAt = this.polledAt.get(t.source) ?? null;
747
- // a copy the passes do not keep current now, and why: not copied (waiting, being copied, its copy failed:
748
- // `needsCopy` holds for every phase but copied), to be copied again (its columns changed, a numeric outgrew it, a
749
- // retired layout), or caught up first (`pass`'s `polled`, and a table not yet caught up on the writer's turn)
750
- const pending = this.needsCopy(t) || this.behind.has(t.source) ? this.notStreamedWhy(t.source) : null;
751
- return { by: "poll", polledAt, ...(overPollLimit(t) ? { unsynced: OVER_POLL_LIMIT } : {}), ...(pending !== null ? { pending } : {}) };
951
+ // a copy the passes do not keep current now, and why, from what a poll of it asks (`pollable`): not copied (waiting,
952
+ // being copied, its copy failed: `needsCopy` holds for every phase but copied), to be copied again (its columns
953
+ // changed, a numeric outgrew it, a retired layout), caught up first, or its rebuild not ended yet: the moment this
954
+ // says nothing is pending, `mason sync` of it runs
955
+ const pending = this.pollable(t) ? null : this.notStreamedWhy(t.source);
956
+ const next = pending === null ? this.dueAt(t) : null;
957
+ return { by: "poll", sync, syncedAt: polledAt, nextAt: next === null ? null : new Date(Math.max(next, now)).toISOString(), ...(pending !== null ? { pending } : {}) };
752
958
  }
753
959
  // copied again whole every pass, whatever its watermark: only a table not copied now (waiting, being copied, its copy
754
960
  // failed) is not kept current
@@ -756,8 +962,12 @@ export class Sync {
756
962
  return { by: "recopy", copiedAt: st?.copiedAt ?? null, ...(st?.phase !== "copied" ? { pending: this.notStreamedWhy(t.source) } : {}) };
757
963
  const s = this.stream;
758
964
  const part = this.streamTable(t, now, s);
759
- return { by: "replication", slot: this.replication.slot, state: part.state, why: part.why, position: formatLsn(s?.position() ?? this.appliedLsn),
760
- lastCommit: this.lastBatch === null ? null : new Date(this.lastBatch.committedAt).toISOString(), notRunning: this.streamDownWhy(s, st) };
965
+ // current as of its last copy or poll, or — streamed — the last moment the stream had all the database wrote
966
+ const polled = this.polledAt.get(t.source) ?? null;
967
+ const streamed = part.state === "streamed" && this.streamCurrentAt !== null ? new Date(this.streamCurrentAt).toISOString() : null;
968
+ const syncedAt = streamed !== null && (polled === null || streamed > polled) ? streamed : polled;
969
+ return { by: "replication", slot: this.replication?.slot ?? "", state: part.state, why: part.why, position: formatLsn(s?.position() ?? this.appliedLsn),
970
+ lastCommit: this.lastBatch === null ? null : new Date(this.lastBatch.committedAt).toISOString(), notRunning: this.streamDownWhy(s, st), syncedAt };
761
971
  }
762
972
  /** why the stream is not running, in the reader's words (null: it runs): its last failure and when it starts again; the
763
973
  * replication check a pass failed (`prepareReplication`, recorded on each copy's state); a slot not made yet; a sync
@@ -770,6 +980,8 @@ export class Sync {
770
980
  return `${firstLine(this.streamFailure.error)} · it starts again after ${clock(this.streamFailure.retryAt)} UTC`;
771
981
  if (st?.error?.startsWith("replication: "))
772
982
  return firstLine(st.error);
983
+ if (this.replication === null)
984
+ return "this run reads no slot, so nothing streams it (the demo database has no change log): give its entry sync: poll <interval>";
773
985
  if (!this.slotKnown)
774
986
  return `slot ${this.replication.slot} is not made yet: it starts once it is`;
775
987
  if (this.stopped)
@@ -796,12 +1008,52 @@ export class Sync {
796
1008
  return { state: "streamed", why: null };
797
1009
  return { state: "waiting", why: this.notStreamedWhy(t.source) };
798
1010
  }
799
- /** each table's last copy or poll, from the ledger (a copy writes its watermark there too) */
1011
+ /**
1012
+ * Each table's last copy or poll, from the ledger (a copy writes its watermark there too), ONLY EVER FORWARD: this
1013
+ * SELECT's snapshot may be older than a commit on the rebuilds' connection whose time is set while it is answered
1014
+ * (`committed`, `swap`), so what it read never replaces a later time, and a table it does not hold is dropped (removed:
1015
+ * nothing else drops it) only when no such commit gave it a time meanwhile.
1016
+ */
800
1017
  async readPolledAt() {
1018
+ const asked = new Map(this.polledAt);
801
1019
  const r = await this.store.writer.runAndReadAll(`SELECT source, epoch_ms(polled_at) FROM ${MASON_SCHEMA}.poll_state WHERE polled_at IS NOT NULL`);
802
- this.polledAt.clear();
803
- for (const [source, ms] of r.getRowsJS())
804
- this.polledAt.set(source, new Date(Number(ms)).toISOString());
1020
+ const ledger = new Map(r.getRowsJS().map(([source, ms]) => [source, new Date(Number(ms)).toISOString()]));
1021
+ for (const [source, at] of this.polledAt)
1022
+ if (!ledger.has(source) && at === asked.get(source))
1023
+ this.polledAt.delete(source);
1024
+ for (const [source, at] of ledger) {
1025
+ const now = this.polledAt.get(source);
1026
+ if (now === undefined || at > now)
1027
+ this.polledAt.set(source, at);
1028
+ }
1029
+ }
1030
+ /**
1031
+ * EACH COPY'S LAST SYNC AS THE LEDGER KEEPS IT, at a start (`Synced`): the swap and every poll write
1032
+ * `poll_state.polled_at` in their own transaction, and a copy's `copied_at` is recorded right after its swap — so a
1033
+ * copy recorded at or after its table's `polled_at` was last brought current by that copy (its time the copy's
1034
+ * `seconds`), else by a poll (its time unknown: null). In replication mode the stream's last batch
1035
+ * (`stream_state.applied_at`) is newer for a table it streamed; a table not caught up is left at its copy, and one the
1036
+ * named publication does not carry (`unpublished`: never streamed, so the stream's time is not its own) at its copy
1037
+ * or poll. `only`: those tables alone, read again (the pass that found them out of the publication), the others'
1038
+ * last sync kept as it is.
1039
+ */
1040
+ async readSynced(only) {
1041
+ const r = await this.store.writer.runAndReadAll(`SELECT p.source, epoch_ms(p.polled_at), epoch_ms(c.copied_at), c.seconds FROM ${MASON_SCHEMA}.poll_state p ` +
1042
+ `LEFT JOIN ${MASON_SCHEMA}.copy_state c USING (source) WHERE p.polled_at IS NOT NULL`);
1043
+ const streamed = this.replication === null ? [] : (await this.store.writer.runAndReadAll(`SELECT epoch_ms(applied_at) FROM ${MASON_SCHEMA}.stream_state ` +
1044
+ `WHERE slot = $1 AND applied_at IS NOT NULL`, [this.replication.slot])).getRowsJS();
1045
+ const streamMs = streamed.length ? Number(streamed[0][0]) : null;
1046
+ if (only === undefined)
1047
+ this.synced.clear();
1048
+ for (const [source, polled, copied, seconds] of r.getRowsJS()) {
1049
+ if (only !== undefined && !only.has(source))
1050
+ continue;
1051
+ const at = Number(polled);
1052
+ const s = copied !== null && Number(copied) >= at ? { at: new Date(Number(copied)).toISOString(), seconds, by: "copy" }
1053
+ : { at: new Date(at).toISOString(), seconds: null, by: "poll" };
1054
+ const streamedIt = !this.behind.has(source) && !this.unpublished.has(source);
1055
+ this.synced.set(source, streamMs !== null && streamMs > Date.parse(s.at) && streamedIt ? { at: new Date(streamMs).toISOString(), seconds: null, by: "stream" } : s);
1056
+ }
805
1057
  }
806
1058
  schema() { return this.currentSchema; }
807
1059
  start() { this.kick(); }
@@ -824,13 +1076,12 @@ export class Sync {
824
1076
  this.running = null;
825
1077
  if (this.stopped)
826
1078
  return;
827
- // replication mode: the stream keeps the copies current between passes, and a pass runs when the tables change —
828
- // or after the interval, when it failed or a table waits for a copy or catch-up nothing has queued (a copy that
829
- // failed waits its turn, `copyRetryDelayMs`)
830
- if (this.mode !== "replication")
831
- this.timer = setTimeout(() => this.kick(), nextPollDelayMs(this.intervalMs, this.lastPassMs));
832
- else if (this.passFailed || this.waiting())
833
- this.timer = setTimeout(() => this.kick(), this.intervalMs);
1079
+ // when the next polled table is due (`nextPassDelay`) — the stream keeps a replicated table current between passes,
1080
+ // and a pass runs for it when the tables change — or after the interval, when it failed or a table waits for a copy
1081
+ // or catch-up nothing has queued (a copy that failed waits its turn, `copyRetryDelayMs`)
1082
+ const d = this.nextPassDelay();
1083
+ if (d !== null)
1084
+ this.timer = setTimeout(() => this.kick(), d);
834
1085
  });
835
1086
  }
836
1087
  /** an added table needs a copy or a catch-up that is neither queued nor running: a later pass must queue it */
@@ -873,19 +1124,42 @@ export class Sync {
873
1124
  */
874
1125
  followUp(source) {
875
1126
  const t = this.tables.get(source);
876
- if (t === undefined || this.stopped || this.unpublished.has(source) || (!this.needsCopy(t) && !this.behind.has(source)))
1127
+ if (t === undefined || this.stopped || this.unpublished.has(source))
1128
+ return;
1129
+ // copied and caught up: polled from now on, so the next pass is timed again (`rearm`)
1130
+ if (!this.needsCopy(t) && !this.behind.has(source)) {
1131
+ this.rearm();
877
1132
  return;
1133
+ }
878
1134
  const at = this.retries.get(source)?.at ?? 0;
879
1135
  if (at > Date.now())
880
1136
  this.wakeAt(at);
881
1137
  else
882
1138
  this.kick();
883
1139
  }
884
- /** one pass, now, and its end awaited (the one running first, if any), and the copies and catch-ups it queued: what
885
- * a test or a reread waits on */
1140
+ /**
1141
+ * THE NEXT PASS TIMED AGAIN, in poll mode, once a table is copied and caught up: a pass that ended while every table
1142
+ * still copied found none due (`nextPassDelay` skips a table that needs its copy) and set no timer, so without this a
1143
+ * table's polls never began. A pass running sets its own when it ends, from the state then.
1144
+ */
1145
+ rearm() {
1146
+ if (this.stopped || this.running !== null)
1147
+ return;
1148
+ const d = this.nextPassDelay();
1149
+ if (d === null)
1150
+ return;
1151
+ if (this.timer !== null)
1152
+ clearTimeout(this.timer);
1153
+ this.timer = setTimeout(() => this.kick(), d);
1154
+ }
1155
+ /** one pass, now, every polled table taken as due (an `off` one is not: it is polled only when asked), and its end
1156
+ * awaited (the one running first, if any), and the copies and catch-ups it queued: what a test waits on */
886
1157
  async once() {
887
1158
  while (this.running !== null)
888
1159
  await this.running;
1160
+ for (const t of this.tables.values())
1161
+ if (this.syncOf(t).mode === "poll")
1162
+ this.due.set(t.source, 0);
889
1163
  this.kick();
890
1164
  if (this.running !== null)
891
1165
  await this.running;
@@ -893,34 +1167,43 @@ export class Sync {
893
1167
  await this.rebuilding;
894
1168
  }
895
1169
  /**
896
- * A POLL OF EXACTLY THESE TABLES, NOW, whatever their size (`mason sync`; a table over POLL_ROW_LIMIT is polled only
897
- * so): a pass of its own, queued behind the one running, in one source snapshot as every poll is. Refused by name
898
- * (`PollRefused`) for a table not added or not copied yet, and in a run whose copies are kept current another way.
899
- * Its failure is its caller's to say: the copies serve as they were, and the passes go on.
1170
+ * A POLL OF EXACTLY THESE TABLES, NOW, whatever their sync says (`mason sync`: an `off` table is read again only so):
1171
+ * a pass of its own, queued behind the one running, in one source snapshot as every poll is; a polled table's next
1172
+ * poll waits its interval from this one. A replicated table is not polled: its answer is where the stream is
1173
+ * (`streamed`), beside the polled ones'. Refused by name (`PollRefused`) for a table not added or not copied yet, and in a run that copies
1174
+ * every table again whole. Its failure is its caller's to say: the copies serve as they were, and the passes go on.
900
1175
  */
901
1176
  async pollNow(sources) {
902
- if (this.mode !== "poll") {
903
- throw new PollRefused("refused", this.mode === "replication" ? "this run keeps every table current by logical replication, so there is nothing to poll"
904
- : "this run copies every table again whole at each pass (MASON_DUCK_SYNC=recopy), so there is nothing to poll");
905
- }
1177
+ if (this.mode === "recopy")
1178
+ throw new PollRefused("refused", "this run reads every table again whole at each pass (MASON_DUCK_SYNC=recopy), so there is nothing to poll");
906
1179
  const tables = sources.map((s) => {
907
1180
  const t = this.tables.get(s);
908
1181
  if (t === undefined)
909
1182
  throw new PollRefused("not_found", `${s} is not added (mason add adds it)${didYouMean(s, [...this.tables.keys()])}`);
910
1183
  return t;
911
1184
  });
1185
+ // a replicated table: the stream keeps it current, so nothing to poll — its answer is where the stream is, and its part
1186
+ const said = () => {
1187
+ const now = Date.now(), position = formatLsn(this.stream?.position() ?? this.appliedLsn);
1188
+ return tables.filter((t) => this.replicated(t)).map((t) => { const p = this.streamTable(t, now, this.stream); return { table: t.source, position, state: p.state, why: p.why }; });
1189
+ };
1190
+ const polled = tables.filter((t) => !this.replicated(t));
1191
+ if (polled.length === 0)
1192
+ return { at: new Date().toISOString(), tables: [], streamed: said() };
912
1193
  while (this.running !== null)
913
1194
  await this.running;
914
1195
  if (this.stopped)
915
1196
  throw new Stopping();
916
1197
  // asked after the pass ahead of it: that pass may have copied it, or failed its copy
917
- for (const t of tables) {
1198
+ for (const t of polled) {
918
1199
  const st = this.states.get(t.source);
919
- if (st?.phase !== "copied" || !this.snapshots.has(t.source) || this.copiedAs.get(t.source) !== columnsKey(t.columns)) {
920
- throw new PollRefused("nothing_yet", `${t.source} is not copied yet (${st?.phase === "failed" ? `its copy failed: ${st.error ?? "mason logs says why"}` : "its copy has not finished"}): mason wait waits for it`);
921
- }
1200
+ if (st?.phase !== "copied")
1201
+ throw new PollRefused("nothing_yet", `${t.source} is not synced yet (${st?.phase === "failed" ? `its first sync failed: ${st.error ?? "mason logs says why"}` : "its first sync has not finished"}): mason wait waits for it`);
1202
+ const again = this.readAgainWhy(t);
1203
+ if (again !== null)
1204
+ throw new PollRefused("nothing_yet", `${t.source} is being read again whole (${again}): mason wait waits for it`);
922
1205
  if (this.retired.has(t.source))
923
- throw new PollRefused("nothing_yet", `${t.source} is ${RETIRED_LAYOUT} (its copy serves as it was): it is current once copied — mason wait waits for it`);
1206
+ throw new PollRefused("nothing_yet", `${t.source} is ${RETIRED_LAYOUT} (its synced data serves as it was): it is current once read again — mason wait waits for it`);
924
1207
  this.refuseRebuilding(t);
925
1208
  }
926
1209
  const poll = (async () => {
@@ -928,8 +1211,12 @@ export class Sync {
928
1211
  try {
929
1212
  // a transaction per table, as the passes poll; a table whose rebuild started since it was asked is refused
930
1213
  const found = [];
931
- for (const t of tables)
932
- found.push(...await this.onWriter(async () => { this.refuseRebuilding(t); return this.poll(this.writerLane, [t]); }));
1214
+ for (const t of polled) {
1215
+ const f = await this.onWriter(async () => { this.refuseRebuilding(t); return this.poll(this.writerLane, [t]); });
1216
+ for (const x of f)
1217
+ this.polledNow(t, x.ms);
1218
+ found.push(...f);
1219
+ }
933
1220
  return found;
934
1221
  }
935
1222
  finally {
@@ -937,12 +1224,21 @@ export class Sync {
937
1224
  }
938
1225
  })();
939
1226
  // the pass slot is held until it ends (kick and once queue behind it); its failure reaches the caller through `poll`
940
- this.running = poll.then(() => undefined, () => undefined).finally(() => { this.running = null; });
1227
+ // and the next pass timed again as a pass's end times it (`rearm`): a copy that ended meanwhile found the slot held,
1228
+ // and a polled table's next poll moved past the timer set before
1229
+ this.running = poll.then(() => undefined, () => undefined).finally(() => { this.running = null; this.rearm(); });
941
1230
  const found = await poll;
942
- const missing = tables.filter((t) => !found.some((f) => f.table === t.source));
1231
+ // A SYNC BY HAND IS AN EVENT (src/duck/events.ts `synced`), in the words mason sync and the page's Sync now say:
1232
+ // a scheduled poll that succeeds records nothing (one every few seconds would bury the failures), so without it a
1233
+ // person who synced and then read the events found only an older copy, and read it as a sync lost. A replicated
1234
+ // table asked for is answered with the stream's position and records none: nothing was read for it
1235
+ for (const x of found)
1236
+ this.event({ kind: "synced", table: x.table, message: `synced by hand: ${syncedText(x)}` });
1237
+ const missing = polled.filter((t) => !found.some((f) => f.table === t.source));
943
1238
  if (missing.length)
944
- throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its copy's DECIMAL: it is copied again at the next pass, and current once copied`);
945
- return { at: new Date().toISOString(), tables: found };
1239
+ throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass, and current once read`);
1240
+ const streamed = said();
1241
+ return { at: new Date().toISOString(), tables: found, ...(streamed.length ? { streamed } : {}) };
946
1242
  }
947
1243
  /**
948
1244
  * The sync stopped: no pass starts, and the one running ends at once — its long statement (a copy's scan, a poll's
@@ -990,8 +1286,8 @@ export class Sync {
990
1286
  const now = this.rebuildingNow?.source === t.source ? this.rebuildingNow.kind : null;
991
1287
  if (now === null && !this.behind.has(t.source))
992
1288
  return;
993
- const doing = now === "copy" ? "being copied again" : `being caught up by one poll ${this.behind.get(t.source)?.why ?? ""}`.trimEnd();
994
- throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its copy serves as it was): it is caught up by one poll when that ends, so there is nothing to poll — mason wait waits for it`);
1289
+ const doing = now === "copy" ? "being read again whole" : `being caught up by one poll ${this.behind.get(t.source)?.why ?? ""}`.trimEnd();
1290
+ throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its synced data serves as it was): it is caught up by one poll when that ends, so there is nothing to poll — mason wait waits for it`);
995
1291
  }
996
1292
  /** the source's connection, as the sync's own reads and replication make it (never DuckDB's: that is `attach`) */
997
1293
  pgOptions() {
@@ -1015,7 +1311,7 @@ export class Sync {
1015
1311
  return columns === undefined ? t : { ...t, columns };
1016
1312
  }
1017
1313
  /**
1018
- * REPLICATION, BEFORE A PASS COPIES: the publication carries every added table, and the slot exists — so a table
1314
+ * REPLICATION, BEFORE A PASS COPIES: the publication carries every replicated table, and the slot exists — so a table
1019
1315
  * copied now has its changes in the stream from before it joins (src/duck/stream.ts "why replaying is safe"). A slot
1020
1316
  * made now, or one this copy has no record of, carries nothing from before it: the tables already copied are caught
1021
1317
  * up by one poll each before the stream applies their changes (`behind`, recorded with the slot in one transaction).
@@ -1024,7 +1320,7 @@ export class Sync {
1024
1320
  */
1025
1321
  async prepareReplication() {
1026
1322
  const r = this.replication;
1027
- const tables = [...this.tables.values()];
1323
+ const tables = this.replicatedTables();
1028
1324
  const pg = await PgConnection.connect(this.pgOptions());
1029
1325
  try {
1030
1326
  const identities = new Map((await pg.query(`SELECT n.nspname || '.' || c.relname, c.relreplident FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ` +
@@ -1034,7 +1330,8 @@ export class Sync {
1034
1330
  const nothing = tables.filter((t) => identities.get(t.source) === "n").map((t) => t.source);
1035
1331
  if (nothing.length) {
1036
1332
  throw new Error(`${nothing.join(", ")} ${nothing.length === 1 ? "has" : "have"} REPLICA IDENTITY NOTHING: a publication carrying ${nothing.length === 1 ? "it" : "them"} ` +
1037
- `would make the app's own UPDATEs and DELETEs fail, so Mason does not add ${nothing.length === 1 ? "it" : "them"}. ALTER TABLE … REPLICA IDENTITY DEFAULT, or mason start --poll`);
1333
+ `would make the app's own UPDATEs and DELETEs fail, so Mason does not replicate ${nothing.length === 1 ? "it" : "them"}. Give ${nothing.length === 1 ? "its entry" : "their entries"} ` +
1334
+ `sync: poll <interval> (poll 5m) in the sources file, or ALTER TABLE … REPLICA IDENTITY DEFAULT on the database`);
1038
1335
  }
1039
1336
  if (r.owned)
1040
1337
  await this.ensurePublication(pg, tables, identities);
@@ -1043,7 +1340,7 @@ export class Sync {
1043
1340
  const slot = await readSlot(pg, r.slot);
1044
1341
  let made = false;
1045
1342
  if (slot.exists && slot.walStatus === "lost") {
1046
- this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the copies caught up by one poll`);
1343
+ this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the synced tables caught up by one poll`);
1047
1344
  if (slot.active)
1048
1345
  throw new Error(`replication slot ${r.slot} is lost and still in use by pid ${slot.activePid}`);
1049
1346
  await dropSlot(pg, r.slot);
@@ -1054,14 +1351,16 @@ export class Sync {
1054
1351
  if (!slot.exists || slot.walStatus === "lost") {
1055
1352
  const repl = await PgConnection.connect({ ...this.pgOptions(), replication: true });
1056
1353
  try {
1057
- this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy, or mason start --poll, drops it)`);
1354
+ this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy drops it, and so does a sources file in which no table's sync is replication)`);
1058
1355
  }
1059
1356
  finally {
1060
1357
  repl.close();
1061
1358
  }
1062
1359
  made = true;
1360
+ this.onReplication?.(true);
1063
1361
  }
1064
1362
  if (made || !this.slotKnown) {
1363
+ // the replicated tables already copied: the slot carries nothing from before it (a polled one is the poll's)
1065
1364
  const copied = tables.filter((t) => this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source));
1066
1365
  await this.onWriter(async () => {
1067
1366
  const w = this.store.writer;
@@ -1087,24 +1386,44 @@ export class Sync {
1087
1386
  finally {
1088
1387
  pg.close();
1089
1388
  }
1090
- // what the last stream's failure said is the past once the publication and slot are as they should be; a table a
1091
- // named publication does not carry is refused on its own line (a copy it never had is failed), the others go on
1092
- await this.onWriter(async () => {
1093
- for (const t of tables) {
1094
- const st = this.states.get(t.source);
1095
- const gap = this.unpublished.get(t.source);
1096
- if (gap !== undefined) {
1097
- const error = `replication: ${gap}`;
1098
- if (st.error !== error)
1099
- this.log(`${t.source} is not copied or streamed: ${error}`);
1100
- await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error });
1389
+ await this.onWriter(() => this.settlePrepared(tables));
1390
+ }
1391
+ /**
1392
+ * WHAT A PASS'S PUBLICATION CHECK SETTLES, ON THE WRITER'S TURN (`prepareReplication`, once the publication and slot are
1393
+ * as they should be): a table a named publication does not carry is refused on its own line (a copy it never had is
1394
+ * failed), the others go on; a table it carries AGAIN has that gap's error cleared, and a table never copied the last
1395
+ * pass's failure to prepare. NOTHING ELSE: a copy whose stream failed, or that the failed preparation left behind, is
1396
+ * current again only once the stream applies a batch with a position (`applyChanges`) — the publication being right
1397
+ * says nothing of whether its changes are in the copy, and a pass runs while the stream waits out its back-off.
1398
+ */
1399
+ async settlePrepared(tables) {
1400
+ for (const t of tables) {
1401
+ const st = this.states.get(t.source);
1402
+ const gap = this.unpublished.get(t.source);
1403
+ if (gap !== undefined) {
1404
+ const error = `replication: ${gap}`;
1405
+ if (st.error !== error) {
1406
+ this.log(`${t.source} is not synced or streamed: ${error}`);
1407
+ this.event({ kind: "stream_failed", table: t.source, message: `not synced or streamed: ${error}` });
1101
1408
  }
1102
- else if (st.error?.startsWith("replication: "))
1103
- await this.record({ ...st, error: null });
1409
+ this.gapErrors.set(t.source, error);
1410
+ await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error });
1411
+ continue;
1104
1412
  }
1105
- // a table the publication stopped carrying leaves the stream; one it carries now may join
1106
- this.refreshStreamed();
1107
- });
1413
+ const gapClosed = this.gapErrors.get(t.source);
1414
+ this.gapErrors.delete(t.source);
1415
+ if (st.error === null)
1416
+ continue;
1417
+ if (st.error === gapClosed || (st.phase !== "copied" && st.error === this.wholeFailure.prepare))
1418
+ await this.record({ ...st, error: null });
1419
+ }
1420
+ // a table the publication does not carry was never streamed: its last sync is its copy's or its poll's, never
1421
+ // the stream's time a start credited it with (`readSynced` at `open`, before the publication was read)
1422
+ const credited = tables.filter((t) => this.unpublished.has(t.source) && this.synced.get(t.source)?.by === "stream").map((t) => t.source);
1423
+ if (credited.length)
1424
+ await this.readSynced(new Set(credited));
1425
+ // a table the publication stopped carrying leaves the stream; one it carries now may join
1426
+ this.refreshStreamed();
1108
1427
  }
1109
1428
  /**
1110
1429
  * A publication the operator NAMED: read as it is, verified, never created, altered or dropped (src/duck/publication.ts).
@@ -1113,14 +1432,14 @@ export class Sync {
1113
1432
  */
1114
1433
  async verifyPublication(pg, tables, identities) {
1115
1434
  const name = this.replication.publication;
1116
- const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables, this.database), tables, identities);
1435
+ const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables), tables, identities);
1117
1436
  if (v.refused !== null)
1118
1437
  throw new Error(v.refused);
1119
1438
  this.unpublished.clear();
1120
1439
  for (const [source, m] of v.tables)
1121
1440
  this.unpublished.set(source, m);
1122
1441
  if (v.noTruncate && !this.truncateSaid) {
1123
- this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the copy (the poll's count would see it; mason start --poll)`);
1442
+ this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the synced data (the poll's count would see it: sync: poll <interval> on its entry)`);
1124
1443
  this.truncateSaid = true;
1125
1444
  }
1126
1445
  }
@@ -1154,23 +1473,84 @@ export class Sync {
1154
1473
  throw new Error(`publication ${name} is FOR ALL TABLES: it is not the one Mason made (drop it, or name another project)`);
1155
1474
  await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET TABLE ${list}`);
1156
1475
  await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET (publish = 'insert, update, delete, truncate')`);
1157
- this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}`);
1476
+ this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}: the tables whose sync is replication`);
1477
+ }
1478
+ // VERIFIED, never assumed: what the publication carries is what the database says it does
1479
+ const carried = (await pg.query(`SELECT schemaname || '.' || tablename FROM pg_publication_tables WHERE pubname = ${pgLiteral(name)}`)).rows.map((x) => x[0]).sort();
1480
+ const wanted = tables.map((t) => t.source).sort();
1481
+ if (carried.join("\n") !== wanted.join("\n")) {
1482
+ throw new Error(`publication ${name} carries ${carried.join(", ") || "no table"} after it was set to ${wanted.join(", ")} (Mason bug: report it, with mason logs)`);
1158
1483
  }
1159
1484
  }
1485
+ /**
1486
+ * NO TABLE'S SYNC IS REPLICATION ANY MORE (at the start that finds none, or live when the last one leaves): the stream
1487
+ * stopped, the slot dropped — it holds the database's WAL while it exists, read or not — and Mason's own publication
1488
+ * with it (a named one is its owner's: left as it is), each VERIFIED gone; then this copy's record of the slot goes,
1489
+ * so a table replicated again later is caught up by one poll under the slot made anew. Idempotent: what is already
1490
+ * gone is not an error. A failure is the pass's, said, and tried again at the next.
1491
+ */
1492
+ async retireReplication() {
1493
+ const r = this.replication;
1494
+ const s = this.stream;
1495
+ if (s !== null) {
1496
+ await s.stop();
1497
+ if (this.stream === s)
1498
+ this.stream = null;
1499
+ }
1500
+ const pg = await PgConnection.connect(this.pgOptions());
1501
+ const dropped = [];
1502
+ try {
1503
+ // the walsender a stop ended lets the slot go a moment later: waited for, at most 10 s
1504
+ let slot = await readSlot(pg, r.slot);
1505
+ for (let i = 0; slot.exists && slot.active && i < 100; i++) {
1506
+ await new Promise((res) => setTimeout(res, 100));
1507
+ slot = await readSlot(pg, r.slot);
1508
+ }
1509
+ if (slot.exists) {
1510
+ if (slot.active)
1511
+ throw new Error(`replication slot ${r.slot} is in use by pid ${slot.activePid ?? "?"}, so it cannot be dropped (another Mason over the same project? stop it)`);
1512
+ await dropSlot(pg, r.slot);
1513
+ dropped.push(`replication slot ${r.slot}`);
1514
+ }
1515
+ const hasPub = async () => (await pg.query(`SELECT 1 FROM pg_publication WHERE pubname = ${pgLiteral(r.publication)}`)).rows.length > 0;
1516
+ if (r.owned && await hasPub()) {
1517
+ await pg.query(`DROP PUBLICATION IF EXISTS ${pgIdent(r.publication)}`);
1518
+ dropped.push(`publication ${r.publication}`);
1519
+ }
1520
+ if ((await readSlot(pg, r.slot)).exists)
1521
+ throw new Error(`replication slot ${r.slot} is still there after pg_drop_replication_slot`);
1522
+ if (r.owned && await hasPub())
1523
+ throw new Error(`publication ${r.publication} is still there after DROP PUBLICATION`);
1524
+ }
1525
+ finally {
1526
+ pg.close();
1527
+ }
1528
+ await this.onWriter(async () => {
1529
+ await this.store.writer.run(`DELETE FROM ${MASON_SCHEMA}.stream_state WHERE slot = $1`, [r.slot]);
1530
+ this.slotKnown = false;
1531
+ this.appliedLsn = 0n;
1532
+ this.streamCurrentAt = null;
1533
+ this.refreshStreamed();
1534
+ });
1535
+ this.log(`no table's sync is replication now: ${dropped.length ? `dropped ${dropped.join(" and ")} from ${this.source.database}, verified gone` : `replication slot ${r.slot} was already gone`}` +
1536
+ ` (the slot held the database's WAL)${r.owned ? "" : `; publication ${r.publication} is its owner's, left as it is`} — a table set to sync: replication makes the slot again`);
1537
+ this.onReplication?.(false);
1538
+ }
1160
1539
  /**
1161
1540
  * The stream started from the slot's acknowledged position, once the slot is known (a pass's `prepareReplication`),
1162
1541
  * and never stopped by a pass: a copy runs beside it. What it applies is `streamed`, read at each change, so a table
1163
1542
  * joins once caught up and leaves when copied again without a restart.
1164
1543
  */
1165
1544
  startStream() {
1166
- if (this.stream !== null || this.stopped || this.mode !== "replication" || !this.slotKnown || Date.now() < this.streamAt)
1545
+ if (this.stream !== null || this.stopped || !this.streams() || !this.slotKnown || Date.now() < this.streamAt)
1167
1546
  return;
1168
1547
  const r = this.replication;
1169
1548
  this.refreshStreamed();
1170
1549
  const s = new ReplicationStream({
1171
1550
  connect: () => PgConnection.connect({ ...this.pgOptions(), replication: true }), slot: r.slot, publication: r.publication, log: (l) => this.log(l),
1172
1551
  target: { tables: () => this.streamed, apply: (batch, lsn, committedAt) => this.applyStreamed(batch, lsn, committedAt), applied: () => this.appliedLsn,
1173
- refuse: (source, missing) => this.refuseStreamed(source, missing) },
1552
+ refuse: (source, missing) => this.refuseStreamed(source, missing), current: () => { if (this.stream === s)
1553
+ this.streamCurrentAt = Date.now(); } },
1174
1554
  });
1175
1555
  this.stream = s;
1176
1556
  this.streamFailure = null;
@@ -1182,7 +1562,7 @@ export class Sync {
1182
1562
  }
1183
1563
  const joined = [...this.streamed.keys()].filter((x) => !this.held.has(x));
1184
1564
  this.log(`streaming ${joined.length ? joined.join(", ") : "no table yet"} from slot ${r.slot} (publication ${r.publication}), applied through ${formatLsn(this.appliedLsn)}` +
1185
- `${joined.length < this.tables.size ? "; a table copied or caught up meanwhile joins it once current" : ""}`);
1565
+ `${joined.length < this.replicatedTables().length ? "; a table synced or caught up meanwhile joins it once current" : ""}`);
1186
1566
  void s.done.then(async (e) => {
1187
1567
  if (this.stream === s)
1188
1568
  this.stream = null;
@@ -1196,7 +1576,7 @@ export class Sync {
1196
1576
  if (wider !== undefined && n <= 3) {
1197
1577
  this.widened.set(wider.source, n);
1198
1578
  await this.onWriter(async () => { this.snapshots.delete(wider.source); this.refreshStreamed(); });
1199
- this.log(`replication: ${wider.source} has a numeric value wider than its copy's DECIMAL: it is copied again, then streamed`);
1579
+ this.log(`replication: ${wider.source} has a numeric value wider than its synced DECIMAL column: it is read again whole, then streamed`);
1200
1580
  this.startStream();
1201
1581
  this.kick();
1202
1582
  return;
@@ -1209,7 +1589,11 @@ export class Sync {
1209
1589
  // a timer may fire a little early: the pass it kicks starts the stream
1210
1590
  this.streamAt = Date.now() + wait - 50;
1211
1591
  this.streamFailure = { error: m, retryAt: this.streamAt };
1212
- this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the copies serve as they are)`);
1592
+ this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the synced tables serve as they are)`);
1593
+ // one event for the stream, never one per table it kept current (each table's error says it); its failures in a
1594
+ // row are left out of the words, so a stream failing for good is one event, not one per restart
1595
+ this.event({ kind: "stream_failed", table: null, message: `${m} (the stream starts again by itself; the synced tables serve as they are)` });
1596
+ this.wholeFailure.stream = m;
1213
1597
  // on the tables the stream kept current when it failed (one being caught up says its own)
1214
1598
  await this.onWriter(async () => { for (const source of this.streamed.keys())
1215
1599
  if (!this.held.has(source))
@@ -1238,10 +1622,11 @@ export class Sync {
1238
1622
  this.held.delete(source);
1239
1623
  const when = this.backOff(source, 0);
1240
1624
  const pub = this.replication.publication;
1241
- const m = `replication: the stream no longer carries ${source}'s ${cols}, which its copy holds (a column dropped or renamed on the database, or the publication's column list narrowed), ` +
1625
+ const m = `replication: the stream no longer carries ${source}'s ${cols}, which its synced table holds (a column dropped or renamed on the database, or the publication's column list narrowed), ` +
1242
1626
  `so its changes are not applied: it is caught up by one poll again after ${when}. The way out: remove ${cols} from its columns in the sources file (mason add ${source} --columns … ` +
1243
- `picks the columns it has; it is copied again at once)${this.replication.owned ? "" : `, or publish ${missing.length === 1 ? "it" : "them"} again (ALTER PUBLICATION ${pub} …)`}`;
1627
+ `picks the columns it has; it is read again whole at once)${this.replication.owned ? "" : `, or publish ${missing.length === 1 ? "it" : "them"} again (ALTER PUBLICATION ${pub} …)`}`;
1244
1628
  this.log(`${source}: ${m}`);
1629
+ this.event({ kind: "stream_failed", table: source, message: m });
1245
1630
  await this.record({ ...this.states.get(source), error: m });
1246
1631
  this.refreshStreamed();
1247
1632
  });
@@ -1250,18 +1635,22 @@ export class Sync {
1250
1635
  * What the stream applies, made again from the tables' state (on the writer's turn, or before the stream exists): a
1251
1636
  * table copied at its present columns and caught up, and a table being caught up (its changes held, `held`). A table
1252
1637
  * that is not — never copied, being copied again, or not caught up — is skipped: its copy's snapshot, or its catch-up,
1253
- * covers what the stream passes over (src/duck/stream.ts "why replaying is safe").
1638
+ * covers what the stream passes over (src/duck/stream.ts "why replaying is safe"). A table here since the last refresh
1639
+ * keeps its STAY, a table new here gets one, and `swap` ends it: changes decoded in a stay that ended committed before
1640
+ * what came next, so that copy and its catch-up hold them, and only that stay's are applied (`applyStreamed`).
1254
1641
  */
1255
1642
  refreshStreamed() {
1643
+ const before = new Map(this.streamed);
1256
1644
  this.streamed.clear();
1257
1645
  const now = new Set();
1258
1646
  for (const t of this.tables.values()) {
1259
- // a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot
1260
- const joined = !this.behind.has(t.source) && !this.needsCopy(t) && !this.unpublished.has(t.source);
1647
+ // a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot;
1648
+ // nor one whose sync is not replication (it is polled, or off)
1649
+ const joined = this.replicated(t) && !this.behind.has(t.source) && !this.needsCopy(t) && !this.unpublished.has(t.source);
1261
1650
  // in the COPY's columns: a held table's columns may have changed since its copy began (`asCopied`)
1262
1651
  const c = this.asCopied(t).columns;
1263
1652
  if (joined || this.held.has(t.source)) {
1264
- this.streamed.set(t.source, { columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows > POLL_ROW_LIMIT });
1653
+ this.streamed.set(t.source, { stay: before.get(t.source)?.stay ?? ++this.lastStay, columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows > BIG_TABLE_ROWS });
1265
1654
  }
1266
1655
  if (joined)
1267
1656
  now.add(t.source);
@@ -1270,11 +1659,18 @@ export class Sync {
1270
1659
  // while a stream runs: its start names what it streams. What it returns is the tables whose join was said
1271
1660
  const said = [];
1272
1661
  const s = this.stream;
1273
- if (s !== null && this.mode === "replication") {
1662
+ if (s !== null) {
1274
1663
  const at = formatLsn(s.position());
1275
- for (const source of this.joinedNow)
1276
- if (!now.has(source))
1277
- this.log(`${source} leaves the stream at ${at}: ${this.notStreamedWhy(source)}`);
1664
+ for (const source of this.joinedNow) {
1665
+ if (now.has(source))
1666
+ continue;
1667
+ const t = this.tables.get(source);
1668
+ // its sync changed: what keeps it current now, in the words of the switch
1669
+ const why = t !== undefined && !this.replicated(t)
1670
+ ? `its sync is ${syncText(this.syncOf(t))} now, so it is caught up by one poll from its watermark${this.syncOf(t).mode === "off" ? ", then read again only by mason sync" : ", then polled"}`
1671
+ : this.notStreamedWhy(source);
1672
+ this.log(`${source} leaves the stream at ${at}: ${why}`);
1673
+ }
1278
1674
  for (const source of now) {
1279
1675
  if (this.joinedNow.has(source))
1280
1676
  continue;
@@ -1296,52 +1692,71 @@ export class Sync {
1296
1692
  return `${why} — after a failure, tried again after ${clock(at)} UTC`;
1297
1693
  const ahead = this.rebuildingNow;
1298
1694
  if (this.queued.has(source) && ahead !== null && ahead.source !== source)
1299
- return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "copy" : "catch-up"} (one rebuild at a time)`;
1695
+ return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "whole read" : "catch-up"} (one rebuild at a time)`;
1300
1696
  return why;
1301
1697
  }
1698
+ /** why a table whose first sync finished is to be read again whole, or null: what the start's line and mason sync say */
1699
+ readAgainWhy(t) {
1700
+ if (this.copiedAs.get(t.source) !== columnsKey(t.columns))
1701
+ return "its columns changed";
1702
+ return this.snapshots.has(t.source) ? null : "a numeric value outgrew its synced DECIMAL column";
1703
+ }
1302
1704
  notStreamedReason(source) {
1303
1705
  const t = this.tables.get(source);
1304
1706
  if (t === undefined)
1305
- return "it is no longer added (its copy is dropped)";
1707
+ return "it is no longer added (its synced data is dropped)";
1306
1708
  const gap = this.unpublished.get(source);
1307
1709
  if (gap !== undefined)
1308
1710
  return gap.split("\n")[0];
1309
1711
  const st = this.states.get(source);
1310
1712
  if (this.copiedAs.has(source) && this.copiedAs.get(source) !== columnsKey(t.columns))
1311
- return "its columns changed: it is copied again, then caught up by one poll";
1713
+ return "its columns changed: it is read again whole, then caught up by one poll";
1312
1714
  if (st?.phase === "copied" && !this.snapshots.has(source))
1313
- return "a numeric value outgrew its copy's DECIMAL: it is copied again, wider, then caught up by one poll";
1715
+ return "a numeric value outgrew its synced DECIMAL column: it is read again whole, wider, then caught up by one poll";
1314
1716
  if (st?.phase === "copied" && this.retired.has(source))
1315
- return "its copy is in an older Mason's layout, key order with an index: it is copied again, then caught up by one poll";
1717
+ return "it is in an older Mason's layout, key order with an index: it is read again whole, then caught up by one poll";
1316
1718
  // what follows the copy: a catch-up poll, then the stream or the polls; copied again whole, nothing (`keptOf`'s recopy)
1317
- const joins = this.mode === "recopy" ? "it is kept current once copied" : "it joins once copied and caught up by one poll";
1719
+ const joins = this.mode === "recopy" ? "it is kept current once synced" : "it joins once synced and caught up by one poll";
1318
1720
  if (st?.phase === "copying" || this.rebuildingNow?.source === source && this.rebuildingNow.kind === "copy")
1319
- return `it is being copied: ${joins}`;
1721
+ return `it is being read whole: ${joins}`;
1320
1722
  if (st?.phase === "failed")
1321
- return `its copy failed (${(st.error ?? "").split("\n")[0]}): ${joins}`;
1723
+ return `its first sync failed (${(st.error ?? "").split("\n")[0]}): ${joins}`;
1322
1724
  const b = this.behind.get(source);
1323
1725
  if (!this.needsCopy(t) && b !== undefined)
1324
1726
  return `it is caught up by one poll ${b.why} first`;
1325
- return `it waits for its copy: ${joins}`;
1727
+ // synced, caught up, and not yet polled (`pollable`): its rebuild's last writer's turn, or a rebuild still queued
1728
+ if (st?.phase === "copied") {
1729
+ if (this.rebuildingNow?.source === source)
1730
+ return "its sync is ending: it is kept current from then on";
1731
+ const queued = this.queued.get(source)?.kind;
1732
+ if (queued !== undefined)
1733
+ return queued === "copy" ? `it is queued to be read whole: ${joins}` : "it is queued to be caught up by one poll";
1734
+ }
1735
+ return `it waits for its first sync: ${joins}`;
1326
1736
  }
1327
1737
  /**
1328
1738
  * WHERE THE STREAM IS (`StreamStatus`): its position, the database's, the lag, the last batch, and each added table's
1329
1739
  * part in it with what it applied this run. Null outside replication mode. The database's position costs a
1330
1740
  * connection at most every 5 s (`databasePosition`); everything else is the run's own.
1741
+ *
1742
+ * `refresh: false` NEVER REACHES THE DATABASE: the position and the slot as last read (`database.at`, `slotWal.at` say
1743
+ * when), or "not read yet". What an unauthenticated reader asks with (`GET /metrics`, src/cli/run-server.ts
1744
+ * `metricsFacts`): a scraper's rate, or anyone's loop on the health port, must never become connections and SELECTs
1745
+ * on the source; the usage sampler, `mason status` and `mason wait` keep the reads fresh.
1331
1746
  */
1332
- async streamStatus() {
1333
- if (this.mode !== "replication")
1747
+ async streamStatus({ refresh = true } = {}) {
1748
+ if (!this.streams())
1334
1749
  return null;
1335
1750
  const r = this.replication;
1336
1751
  const s = this.stream;
1337
1752
  const position = s?.position() ?? this.appliedLsn;
1338
- const db = await this.databasePosition();
1753
+ const db = refresh ? await this.databasePosition() : this.dbPosition ?? { error: "not read yet: nothing has asked where the database is since the run began", at: Date.now() };
1339
1754
  const now = Date.now();
1340
1755
  const iso = (ms) => new Date(ms).toISOString();
1341
1756
  // each table's part, as `keptOf` says it for EXPLAIN (`streamTable`), with what it applied this run
1342
- const tables = [...this.tables.values()].map((t) => ({ table: t.source, ...this.streamTable(t, now, s), applied: { ...(this.appliedCounts.get(t.source) ?? noCounts()) } }));
1757
+ const tables = this.replicatedTables().map((t) => ({ table: t.source, ...this.streamTable(t, now, s), applied: { ...(this.appliedCounts.get(t.source) ?? noCounts()) } }));
1343
1758
  return {
1344
- slot: r.slot, publication: r.publication, running: s !== null,
1759
+ slot: r.slot, slotWal: this.slotWal?.wal ?? { error: "not read yet: the database did not answer within 3 s" }, publication: r.publication, running: s !== null,
1345
1760
  failure: s === null && this.streamFailure !== null ? { error: this.streamFailure.error, retryAt: iso(this.streamFailure.retryAt) } : null,
1346
1761
  position: formatLsn(position), applied: formatLsn(this.appliedLsn),
1347
1762
  database: "lsn" in db ? { lsn: formatLsn(db.lsn), at: iso(db.at) } : { error: db.error },
@@ -1356,15 +1771,28 @@ export class Sync {
1356
1771
  * wait` asks every second), waited for at most 3 s (the last read is answered meanwhile, with its time). Not the
1357
1772
  * stream's keepalive: that carries the walsender's own read position, not the database's, so it cannot say how far
1358
1773
  * behind the stream is when that matters — while it decodes a backlog. A read that fails is answered as its error.
1774
+ * THE SLOT'S WAL (`SlotWal`) is read on the same connection, at most every SLOT_READ_MS (one row of
1775
+ * pg_replication_slots): what the slot holds on the database's disk, said by `mason status` and sampled by usage.
1359
1776
  */
1360
1777
  async databasePosition() {
1361
1778
  const fresh = this.dbPosition !== null && Date.now() - this.dbPosition.at < 5_000;
1362
1779
  if (!fresh && this.dbPositionRead === null) {
1780
+ const slotDue = this.slotWal === null || Date.now() - this.slotWal.at >= SLOT_READ_MS;
1781
+ const slot = this.replication.slot;
1363
1782
  this.dbPositionRead = (async () => {
1364
1783
  try {
1365
1784
  const pg = await PgConnection.connect(this.pgOptions());
1366
1785
  try {
1367
1786
  this.dbPosition = { lsn: await currentLsn(pg), at: Date.now() };
1787
+ if (slotDue) {
1788
+ // its own failure is the slot's line alone: the position read above stands
1789
+ try {
1790
+ this.slotWal = { wal: slotWalOf(await readSlot(pg, slot), slot, Date.now()), at: Date.now() };
1791
+ }
1792
+ catch (e) {
1793
+ this.slotWal = { wal: { error: `the slot could not be read: ${e instanceof Error ? e.message : String(e)}` }, at: Date.now() };
1794
+ }
1795
+ }
1368
1796
  }
1369
1797
  finally {
1370
1798
  pg.close();
@@ -1372,7 +1800,10 @@ export class Sync {
1372
1800
  }
1373
1801
  catch (e) {
1374
1802
  // answered as the status's (and the log line's) own words: the position could not be read, and why
1375
- this.dbPosition = { error: e instanceof Error ? e.message : String(e), at: Date.now() };
1803
+ const m = e instanceof Error ? e.message : String(e);
1804
+ this.dbPosition = { error: m, at: Date.now() };
1805
+ if (slotDue)
1806
+ this.slotWal = { wal: { error: `the database could not be reached: ${m}` }, at: Date.now() };
1376
1807
  }
1377
1808
  finally {
1378
1809
  this.dbPositionRead = null;
@@ -1413,10 +1844,10 @@ export class Sync {
1413
1844
  this.refreshStreamed();
1414
1845
  const applied = new Map();
1415
1846
  for (const [name, changes] of batch) {
1416
- const now = this.streamed.get(name);
1417
- // decoded in another shape than the copy's now (before it was copied again, a stream left it): what they changed
1418
- // committed before the new copy's snapshot, so it is in that copy — never written in a shape it does not have
1419
- if (now === undefined || changes.columns === null || changes.columns.join("\0") !== now.columns.join("\0"))
1847
+ // decoded in a stay that ended (before its table was copied again, or left the stream): what they changed
1848
+ // committed before the next copy's catch-up, so that copy or its catch-up holds it — never written over them, in
1849
+ // a shape the copy may not have, or as an old change after the poll that read a newer one
1850
+ if (this.streamed.get(name)?.stay !== changes.stay)
1420
1851
  continue;
1421
1852
  const held = this.held.get(name);
1422
1853
  if (held !== undefined) {
@@ -1425,7 +1856,7 @@ export class Sync {
1425
1856
  }
1426
1857
  applied.set(name, changes);
1427
1858
  }
1428
- await this.applyChanges(applied, lsn, committedAt);
1859
+ await this.applyChanges(applied, { lsn, committedAt });
1429
1860
  });
1430
1861
  }
1431
1862
  /**
@@ -1436,12 +1867,26 @@ export class Sync {
1436
1867
  * index each of those is a scan of the table's key column, so a batch costs a few scans per table, never one per row. A numeric wider than its copy's DECIMAL is the table's to copy
1437
1868
  * again (`WIDER`), never rounded. A null `lsn` is a piece of a large transaction (src/duck/stream.ts): committed, the
1438
1869
  * position left as it was, so a run that dies before the transaction's last piece applies it again whole. What it
1439
- * applied is counted by table (`streamStatus`), and the batch's newest commit time kept (null: changes held during a
1440
- * catch-up, whose time is the batches' that held them).
1870
+ * applied is counted by table (`streamStatus`), and the batch's newest commit time kept.
1871
+ *
1872
+ * `"held"` is THE CHANGES A CATCH-UP HELD, replayed after its poll (`catchUp`): written, and counted, as a batch's are,
1873
+ * but NOT A POSITION of the stream — no `stream_state` row (its LSN and `applied_at` are the batches' that held them),
1874
+ * no table's `synced`, no commit time, and not the stream working again (`streamFailures`): the replay ran on the
1875
+ * writer, whatever the stream is doing (it may be failing).
1441
1876
  */
1442
- async applyChanges(batch, lsn, committedAt = null) {
1877
+ async applyChanges(batch, from) {
1878
+ const lsn = from === "held" ? null : from.lsn;
1879
+ const committedAt = from === "held" ? null : from.committedAt;
1443
1880
  const w = this.store.writer;
1444
1881
  let source = null;
1882
+ const started = Date.now();
1883
+ // EACH TABLE'S ROWS AFTER THE BATCH, kept current at every batch (it was counted at most every 5 s, so a table's
1884
+ // count missed the batches after one until another came): what its writes added and removed, which DuckDB says
1885
+ // (`rowsChanged`) — never a count(*), which a write by key's deletes make a scan (12 ms on 20 M rows, measured) —
1886
+ // written to the ledger IN the batch's transaction, so a run that dies keeps a count equal to its rows
1887
+ const rowsAfter = new Map();
1888
+ // a batch with a position: its time in the ledger and in this run, ONE value (`Synced`)
1889
+ const position = lsn === null ? null : { lsn, at: new Date().toISOString() };
1445
1890
  await w.run("BEGIN TRANSACTION");
1446
1891
  try {
1447
1892
  for (const [name, changes] of batch) {
@@ -1461,8 +1906,7 @@ export class Sync {
1461
1906
  throw new Error(`${WIDER} [${name}]: ${c} = ${v}`);
1462
1907
  return duckValue(v, types.get(c));
1463
1908
  };
1464
- if (changes.truncated)
1465
- await w.run(`DELETE FROM ${target}`);
1909
+ let delta = changes.truncated ? -Number((await w.run(`DELETE FROM ${target}`)).rowsChanged) : 0;
1466
1910
  const whole = [];
1467
1911
  const fetch = [];
1468
1912
  const gone = [];
@@ -1492,10 +1936,16 @@ export class Sync {
1492
1936
  source ??= await PgConnection.connect(this.pgOptions());
1493
1937
  whole.push(...await this.sourceRows(source, t, fetch));
1494
1938
  }
1495
- await this.writeByKey(t, target, gone, whole, value);
1939
+ delta += await this.writeByKey(t, target, gone, whole, value);
1940
+ // a copy whose rows the ledger does not know (none should be: its copy counts them) is counted, once
1941
+ const known = this.states.get(name)?.rows ?? null;
1942
+ const rows = known === null ? Number((await w.runAndReadAll(`SELECT count(*) FROM ${target}`)).getRowsJS()[0][0]) : known + delta;
1943
+ if (rows !== known)
1944
+ await w.run(`UPDATE ${MASON_SCHEMA}.copy_state SET "rows" = $1 WHERE source = $2`, [rows, name]);
1945
+ rowsAfter.set(name, rows);
1496
1946
  }
1497
- if (lsn !== null)
1498
- await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2, now())`, [this.replication.slot, formatLsn(lsn)]);
1947
+ if (position !== null)
1948
+ await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2, $3::TIMESTAMPTZ)`, [this.replication.slot, formatLsn(position.lsn), position.at]);
1499
1949
  await w.run("COMMIT");
1500
1950
  }
1501
1951
  catch (e) {
@@ -1507,7 +1957,12 @@ export class Sync {
1507
1957
  }
1508
1958
  if (lsn !== null)
1509
1959
  this.appliedLsn = lsn;
1510
- this.streamFailures = 0;
1960
+ if (from !== "held")
1961
+ this.streamFailures = 0;
1962
+ // THE STREAM'S FAILURE IS OVER when it applies a batch WITH A POSITION: every table it keeps current is current as of
1963
+ // it. A piece of a large transaction (no position) is not that yet, nor the changes a catch-up held
1964
+ if (lsn !== null)
1965
+ this.recovered("stream", "the stream applies changes again");
1511
1966
  let moved = false;
1512
1967
  for (const [name, changes] of batch) {
1513
1968
  if (!this.tables.has(name))
@@ -1521,17 +1976,30 @@ export class Sync {
1521
1976
  }
1522
1977
  if (moved && committedAt !== null && committedAt.getTime() > 0)
1523
1978
  this.lastBatch = { appliedAt: Date.now(), committedAt: committedAt.getTime() };
1524
- // the ledger's row counts, for mason status: each changed table counted at most every 5 s
1525
- for (const name of batch.keys()) {
1979
+ // the rows, as the ledger has them now; with a position, a table the batch wrote is current again (its error, if
1980
+ // any, cleared), and so is every table the stream keeps current whose error was the replication's (the stream's
1981
+ // failure, the preparation's): the position covers each of them, changed in the batch or not
1982
+ for (const [name, rows] of rowsAfter) {
1526
1983
  const st = this.states.get(name);
1527
- if (st === undefined || Date.now() - (this.countedAt.get(name) ?? 0) < 5_000)
1984
+ if (st === undefined)
1528
1985
  continue;
1529
- const t = this.tables.get(name);
1530
- const rows = Number((await w.runAndReadAll(`SELECT count(*) FROM ${ident(t.schema)}.${ident(t.table)}`)).getRowsJS()[0][0]);
1531
- this.countedAt.set(name, Date.now());
1532
- if (rows !== st.rows || st.error !== null)
1986
+ if (st.error !== null && lsn !== null)
1533
1987
  await this.record({ ...st, rows, error: null });
1988
+ else
1989
+ this.states.set(name, { ...st, rows });
1534
1990
  }
1991
+ if (lsn !== null) {
1992
+ for (const source of this.joinedNow) {
1993
+ const st = this.states.get(source);
1994
+ if (st !== undefined && st.phase === "copied" && st.error?.startsWith("replication: "))
1995
+ await this.record({ ...st, error: null });
1996
+ }
1997
+ }
1998
+ // every table the stream keeps current is current as of this batch's position (`Synced`); a piece of a large
1999
+ // transaction (no position) and the changes a catch-up held are not a position
2000
+ if (position !== null)
2001
+ for (const source of this.joinedNow)
2002
+ this.synced.set(source, { at: position.at, seconds: (Date.now() - started) / 1000, by: "stream" });
1535
2003
  }
1536
2004
  /**
1537
2005
  * Rows an UPDATE left a large value unchanged in, every one carrying the same columns (`carried`, the key among them):
@@ -1561,11 +2029,13 @@ export class Sync {
1561
2029
  * join against the table's key column (`deleteKeys`), then the rows inserted. No copy has a primary key,
1562
2030
  * so there is no `INSERT OR REPLACE`; a key's row is the source's at the batch's end
1563
2031
  * whatever the table held, as replaying needs (src/duck/stream.ts). A table with no key (none can be added:
1564
- * src/cli/preflight.ts) is only appended to.
2032
+ * src/cli/preflight.ts) is only appended to. Answers what it did to the table's count: the rows inserted less the rows
2033
+ * its deletes removed (DuckDB's own count of each).
1565
2034
  */
1566
2035
  async writeByKey(t, target, gone, whole, value) {
1567
2036
  if (gone.length === 0 && whole.length === 0)
1568
- return;
2037
+ return 0;
2038
+ let delta = 0;
1569
2039
  const w = this.store.writer;
1570
2040
  const cols = t.columns.map((c) => c.name);
1571
2041
  const list = cols.map(ident).join(", ");
@@ -1583,15 +2053,16 @@ export class Sync {
1583
2053
  }
1584
2054
  if (whole.length)
1585
2055
  await w.run(`INSERT INTO ${WRITE_KEYS} SELECT ${keys} FROM ${WRITE_ROWS}`);
1586
- await deleteKeys(w, t, target, WRITE_KEYS, t.key);
2056
+ delta -= await deleteKeys(w, t, target, WRITE_KEYS, t.key);
1587
2057
  await w.run(`DROP TABLE ${WRITE_KEYS}`);
1588
2058
  }
1589
2059
  if (whole.length) {
1590
2060
  // and its stored JSON paths, computed from the rows' JSON (src/duck/kept.ts)
1591
2061
  const stored = this.kept.stored(t.source, ident);
1592
- await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`);
2062
+ delta += Number((await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`)).rowsChanged);
1593
2063
  await w.run(`DROP TABLE ${WRITE_ROWS}`);
1594
2064
  }
2065
+ return delta;
1595
2066
  }
1596
2067
  /** a copied table's columns' DuckDB types, as DESCRIBE spells them (what `duckValue` casts a value's text to) */
1597
2068
  async columnTypes(t, target) {
@@ -1667,15 +2138,18 @@ export class Sync {
1667
2138
  * whose attach throws is not one: it never detaches */
1668
2139
  async attach() {
1669
2140
  if (!this.attached) {
1670
- // INSTALL fetches the extension from DuckDB's repository the first time on a machine (then ~/.duckdb/extensions):
1671
- // offline, the first copy fails here saying so — bundling it is the plan's step 3
2141
+ // the extension ships with Mason (src/duck/extensions.ts): a LOAD from the platform's package, nothing downloaded;
2142
+ // without the package, INSTALL fetches it the first time on a machine (then ~/.duckdb/extensions), said once,
2143
+ // and offline the first copy fails here saying why
1672
2144
  if (!this.loaded) {
2145
+ const x = this.store.extensions;
2146
+ if (!x.shipped)
2147
+ this.log(`Mason's extensions are not installed with it here: ${x.why}. The first sync on this machine downloads its postgres extension`);
1673
2148
  try {
1674
- await this.store.writer.run("INSTALL postgres; LOAD postgres;");
2149
+ await this.store.writer.run(loadPostgresSql(x));
1675
2150
  }
1676
2151
  catch (e) {
1677
- throw new Error(`DuckDB could not load its postgres extension: ${e.message}. The first copy on a machine downloads it from ` +
1678
- "DuckDB's extension repository (then ~/.duckdb/extensions): connect to the internet once, and the copy is tried again at the next pass");
2152
+ throw new Error(extensionFailure(x, e.message));
1679
2153
  }
1680
2154
  this.loaded = true;
1681
2155
  }
@@ -1726,40 +2200,76 @@ export class Sync {
1726
2200
  });
1727
2201
  this.snapshots.delete(source);
1728
2202
  this.retries.delete(source);
2203
+ this.due.delete(source);
2204
+ this.lastPollMs.delete(source);
2205
+ this.recordedSync.delete(source);
2206
+ this.switched.delete(source);
1729
2207
  this.retired.delete(source);
1730
2208
  this.copiedAs.delete(source);
1731
2209
  this.copiedColumns.delete(source);
1732
2210
  this.states.delete(source);
1733
- this.log(`dropped ${source}'s copy: it is no longer added (mason add ${source} copies it again from the source, whole)`);
1734
- }
1735
- if (this.tables.size === 0) {
1736
- await this.rebuildSchema();
1737
- return;
2211
+ this.synced.delete(source);
2212
+ this.log(`dropped ${source}'s synced data: it is no longer added (mason add ${source} syncs it again from the source, whole)`);
2213
+ }
2214
+ // REPLICATION, as the tables' syncs say: the publication and slot made ready for the replicated tables and the stream
2215
+ // started — or, when none is replicated any more, the slot dropped (verified gone). Its failure is the replicated
2216
+ // tables' alone (not copied, not caught up: `work`), and the pass's, said once the polled tables are polled
2217
+ let replicationFailed = null;
2218
+ let replicationReady = false;
2219
+ for (const source of [...this.unpublished.keys()]) {
2220
+ const t = this.tables.get(source);
2221
+ if (t === undefined || !this.replicated(t))
2222
+ this.unpublished.delete(source);
1738
2223
  }
1739
- if (this.mode === "replication") {
2224
+ if (this.streams()) {
1740
2225
  try {
1741
2226
  await this.prepareReplication();
2227
+ replicationReady = true;
1742
2228
  }
1743
2229
  catch (e) {
1744
- const m = `replication: ${e instanceof Error ? e.message : String(e)}`;
2230
+ // the way out on the table's own line: a failure that names none (a slot the database will not make, a role that
2231
+ // may not replicate) says the line that keeps the tables polled meanwhile — never a bare SQLSTATE's text
2232
+ const said = e instanceof Error ? e.message : String(e);
2233
+ const names = this.replicatedTables().map((t) => t.source);
2234
+ const m = `replication: ${said}${/sync: poll/.test(said) ? "" : ` — ${names.join(", ")} ${names.length === 1 ? "is" : "are"} NOT kept current until it is fixed: ` +
2235
+ `give ${names.length === 1 ? "its entry" : "their entries"} sync: poll <interval> (poll 5m) in the sources file to poll ${names.length === 1 ? "it" : "them"} instead`}`;
2236
+ this.event({ kind: "stream_failed", table: null, message: m });
2237
+ this.wholeFailure.prepare = m;
1745
2238
  // a table copied before keeps serving its copy; one never copied is failed, so mason status says why
1746
2239
  await this.onWriter(async () => {
1747
- for (const t of this.tables.values()) {
2240
+ for (const t of this.replicatedTables()) {
1748
2241
  const st = this.states.get(t.source);
1749
2242
  await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m });
1750
2243
  }
1751
2244
  });
1752
- await this.rebuildSchema();
1753
- throw e;
2245
+ replicationFailed = e;
1754
2246
  }
2247
+ this.recovered("prepare", "the publication and the slot are ready again");
1755
2248
  // the stream starts once the slot is known, and runs from then on whatever the passes and rebuilds do
1756
- this.startStream();
2249
+ if (replicationReady)
2250
+ this.startStream();
2251
+ }
2252
+ else if (this.replication !== null && (this.slotKnown || this.stream !== null)) {
2253
+ try {
2254
+ await this.retireReplication();
2255
+ }
2256
+ catch (e) {
2257
+ replicationFailed = new Error(`no table's sync is replication now, but replication slot ${this.replication.slot} could not be dropped (it holds the database's WAL; tried again at the next pass): ${e instanceof Error ? e.message : String(e)}`);
2258
+ }
2259
+ }
2260
+ if (this.tables.size === 0) {
2261
+ await this.rebuildSchema();
2262
+ if (replicationFailed !== null)
2263
+ throw replicationFailed;
2264
+ return;
1757
2265
  }
1758
2266
  try {
1759
2267
  await this.onWriter(() => this.attach());
1760
2268
  }
1761
2269
  catch (e) {
1762
2270
  const m = e instanceof Error ? e.message : String(e);
2271
+ this.event({ kind: "source_unreachable", table: null, message: `could not attach the database: ${m}` });
2272
+ this.wholeFailure.attach = m;
1763
2273
  // a table copied before keeps serving its copy (and its place in the catalog): the source being away is its error,
1764
2274
  // not a reason to copy it whole again when the source is back; one never copied is failed
1765
2275
  await this.onWriter(async () => {
@@ -1770,18 +2280,22 @@ export class Sync {
1770
2280
  });
1771
2281
  // the copies serve while the source is away: the catalog lists them (on a fresh start, nothing else has)
1772
2282
  await this.rebuildSchema();
1773
- throw new Error(`DuckDB could not attach ${this.source.database} on ${this.source.host}:${this.source.port}: ${m}`);
2283
+ throw new Error(`Mason could not attach ${this.source.database} on ${this.source.host}:${this.source.port}: ${m}`);
1774
2284
  }
2285
+ this.recovered("attach", "the database is attached again");
1775
2286
  try {
1776
- await this.work();
2287
+ await this.work(replicationReady);
1777
2288
  }
1778
2289
  finally {
1779
2290
  await this.onWriter(() => this.detach());
1780
2291
  }
1781
2292
  await this.rebuildSchema();
2293
+ if (replicationFailed !== null)
2294
+ throw replicationFailed;
1782
2295
  }
1783
- /** a pass's queueing and polls, the source attached */
1784
- async work() {
2296
+ /** a pass's queueing and polls, the source attached; a replicated table is copied or caught up only once the publication
2297
+ * carries it and the slot exists (`replicationReady`): its changes are in the stream from before it joins */
2298
+ async work(replicationReady) {
1785
2299
  const started = Date.now();
1786
2300
  // a table never copied (or whose copy failed, or whose copy is out of date) is copied whole, beside the rest: the
1787
2301
  // smallest first (`copyOrder`)
@@ -1794,33 +2308,27 @@ export class Sync {
1794
2308
  // a table the named publication does not carry: copied, nothing would keep it current (prepareReplication)
1795
2309
  if (this.unpublished.has(t.source))
1796
2310
  continue;
2311
+ if (this.replicated(t) && !replicationReady)
2312
+ continue;
1797
2313
  if (this.mode !== "recopy" && !this.needsCopy(t))
1798
2314
  continue;
1799
2315
  if (this.queued.has(t.source) || this.rebuildingNow?.source === t.source)
1800
2316
  continue;
1801
2317
  if (this.states.get(t.source)?.phase === "copied" && this.copiedAs.get(t.source) !== columnsKey(t.columns))
1802
- this.log(`${t.source}: its columns are not the ones it was copied with: it is copied again`);
2318
+ this.log(`${t.source}: its columns are not the ones it was synced with: it is read again whole`);
1803
2319
  this.enqueue(t.source, { kind: "copy" });
1804
2320
  }
1805
2321
  // a table copied and not yet caught up (a slot new to the copy, or a run that stopped first): its one poll
1806
2322
  for (const source of this.behind.keys()) {
1807
2323
  const t = this.tables.get(source);
1808
2324
  // a failed catch-up waits its turn as a failed copy does (`backOff`); the error it recorded says when
1809
- if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now())
2325
+ if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now() && (!this.replicated(t) || replicationReady))
1810
2326
  this.enqueue(source, { kind: "catch-up" });
1811
2327
  }
1812
- // a table whose new columns could not be copied (its copy failed above) is not polled with them
1813
- const polled = [...this.tables.values()].filter((t) => !this.needsCopy(t));
1814
- // POLL MODE: a table over the row limit is served as copied and polled only when asked (POLL_ROW_LIMIT, pollNow)
1815
- const big = this.mode === "poll" ? polled.filter(overPollLimit) : [];
1816
- for (const t of big.filter((x) => !this.leftOut.has(x.source))) {
1817
- this.log(`${t.source} (about ${t.rows.toLocaleString("en-US")} rows) is ${OVER_POLL_LIMIT}: it is served as copied, NOT kept current by the poll ` +
1818
- `(each poll would scan it whole on the database) — mason sync ${t.source} syncs it now; mason start --replication keeps every table current`);
1819
- this.leftOut.add(t.source);
1820
- }
1821
- for (const s of [...this.leftOut].filter((x) => !big.some((t) => t.source === x)))
1822
- this.leftOut.delete(s);
1823
- const kept = this.mode === "poll" ? polled.filter((t) => !big.includes(t)) : [];
2328
+ // POLL MODE: the tables due (`dueAt`: a polled one past its interval, one whose sync changed), and not a table whose
2329
+ // new columns could not be copied (its copy failed above): it is not polled with them
2330
+ const now = Date.now();
2331
+ const kept = this.mode !== "recopy" ? [...this.tables.values()].filter((t) => !this.replicated(t) && this.pollable(t) && (this.dueAt(t) ?? Infinity) <= now) : [];
1824
2332
  for (const t of kept) {
1825
2333
  if (this.stopped)
1826
2334
  break;
@@ -1828,22 +2336,27 @@ export class Sync {
1828
2336
  // or a swap may run between two. A table rebuilt or not yet caught up is not polled (decided on the turn itself);
1829
2337
  // its catch-up polls it
1830
2338
  try {
1831
- await this.onWriter(async () => { if (this.rebuildingNow?.source !== t.source && !this.behind.has(t.source))
1832
- await this.poll(this.writerLane, [t]); });
2339
+ const found = await this.onWriter(async () => this.rebuildingNow?.source !== t.source && !this.behind.has(t.source) ? this.poll(this.writerLane, [t]) : []);
2340
+ for (const f of found)
2341
+ this.polledNow(t, f.ms);
1833
2342
  }
1834
2343
  catch (e) {
1835
2344
  if (e instanceof Stopping) {
1836
- this.log("poll stopped: Mason is stopping (rolled back; the copies serve as they were)");
2345
+ this.log("poll stopped: Mason is stopping (rolled back; the synced tables serve as they were)");
1837
2346
  return;
1838
2347
  }
2348
+ // tried again after its interval (an `off` table whose sync changed: after the sync's), never at every pass
2349
+ const s = this.syncOf(t);
2350
+ this.due.set(t.source, Date.now() + (s.mode === "poll" ? s.seconds * 1000 : this.intervalMs));
1839
2351
  // the table serves its copy as it was, and says why it is not current (mason status reads the ledger); the other
1840
2352
  // tables are polled as ever
1841
- const m = `the poll failed, and is tried again at the next one (the copy serves as it was): ${e instanceof Error ? e.message : String(e)}`;
2353
+ const m = `the poll failed, and is tried again at the next one (the synced data serves as it was): ${e instanceof Error ? e.message : String(e)}`;
1842
2354
  this.log(`${t.source}: ${m}`);
2355
+ this.event({ kind: "poll_failed", table: t.source, message: m });
1843
2356
  await this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
1844
2357
  }
1845
2358
  }
1846
- if (this.mode === "poll")
2359
+ if (this.mode !== "recopy")
1847
2360
  await this.onWriter(() => this.readPolledAt());
1848
2361
  this.lastPassMs = Date.now() - started;
1849
2362
  }
@@ -1906,12 +2419,16 @@ export class Sync {
1906
2419
  // the next pass says the same of every table, and queues it again
1907
2420
  const m = `could not attach the database: ${e instanceof Error ? e.message : String(e)}`;
1908
2421
  this.log(`${r.kind} of ${t.source} waits: ${m}`);
2422
+ this.event({ kind: "source_unreachable", table: t.source, message: `its ${r.kind} waits: ${m}` });
1909
2423
  const st = this.states.get(t.source);
1910
2424
  await this.onWriter(() => this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m }));
1911
2425
  return null;
1912
2426
  }
1913
2427
  if (r.kind === "copy")
1914
2428
  await this.copy(t);
2429
+ // the whole read is over, whatever comes next: what the table is said to be doing from here (`notStreamedReason`,
2430
+ // `refuseRebuilding`, `state`'s `rebuilding`)
2431
+ this.rebuildingNow = { source: t.source, kind: "catch-up" };
1915
2432
  // a table its copy left behind (`swap`), or one marked so before: one poll, unless a copy of it comes first — the
1916
2433
  // table as it is added NOW, which may have changed while it was copied (then copied again at once: `followUp`)
1917
2434
  const now = this.tables.get(t.source);
@@ -1936,11 +2453,11 @@ export class Sync {
1936
2453
  * copy (the copy's snapshot), or a slot new to the copy — on the rebuilds'
1937
2454
  * connection, so the stream and the other tables' polls go on meanwhile. In replication mode the stream holds the
1938
2455
  * table's changes from before the poll's snapshot and applies them after it (src/duck/stream.ts "why replaying is
1939
- * safe"). A table over POLL_ROW_LIMIT is caught up too: the one scan its rebuild costs, not the periodic poll.
2456
+ * safe"). A table whose sync is `off` is caught up too: the one scan its rebuild costs, then nothing until asked.
1940
2457
  */
1941
2458
  async catchUp(t) {
1942
2459
  const mark = this.behind.get(t.source);
1943
- const replication = this.mode === "replication";
2460
+ const replication = this.replicated(t);
1944
2461
  // J: from the writer's turn on, every batch's changes of this table are held (`applyStreamed`)
1945
2462
  if (replication)
1946
2463
  await this.onWriter(async () => { if (!this.held.has(t.source))
@@ -1953,8 +2470,9 @@ export class Sync {
1953
2470
  // mason status — and is caught up again then: what was held is read again by that poll (still behind, nothing lost)
1954
2471
  const failed = (what, e) => {
1955
2472
  const when = this.backOff(t.source, Date.now() - started);
1956
- const m = `${what} (the copy serves as it was; it is caught up by one poll again after ${when}): ${e instanceof Error ? e.message : String(e)}`;
2473
+ const m = `${what} (the synced data serves as it was; it is caught up by one poll again after ${when}): ${e instanceof Error ? e.message : String(e)}`;
1957
2474
  this.log(`${t.source}: ${m}`);
2475
+ this.event({ kind: "catch_up_failed", table: t.source, message: m });
1958
2476
  return this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
1959
2477
  };
1960
2478
  // the catch-up's poll reads the columns the copy holds (`asCopied`)
@@ -1977,6 +2495,7 @@ export class Sync {
1977
2495
  await release();
1978
2496
  return "copy again";
1979
2497
  }
2498
+ this.polledNow(t, found[0].ms);
1980
2499
  let saidJoin = false;
1981
2500
  if (this.catchingUp?.source === t.source)
1982
2501
  this.catchingUp.phase = "applying";
@@ -1987,7 +2506,7 @@ export class Sync {
1987
2506
  this.held.delete(t.source);
1988
2507
  if (held !== undefined && (held.truncated || held.keys.size > 0)) {
1989
2508
  try {
1990
- await this.applyChanges(new Map([[t.source, held]]), this.appliedLsn);
2509
+ await this.applyChanges(new Map([[t.source, held]]), "held");
1991
2510
  }
1992
2511
  catch (e) {
1993
2512
  const m = e instanceof Error ? e.message : String(e);
@@ -1995,7 +2514,7 @@ export class Sync {
1995
2514
  // a numeric wider than its copy: copied again (`needsCopy`), and caught up after that copy
1996
2515
  if (m.includes(`${WIDER} [${t.source}]`)) {
1997
2516
  this.snapshots.delete(t.source);
1998
- this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is copied again, wider (${m})`);
2517
+ this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is read again whole, wider (${m})`);
1999
2518
  return "copy again";
2000
2519
  }
2001
2520
  heldFailure = e;
@@ -2024,9 +2543,10 @@ export class Sync {
2024
2543
  return joined || null;
2025
2544
  if (saidJoin)
2026
2545
  return null;
2546
+ const s = this.syncOf(t);
2027
2547
  const tail = replication ? (this.behind.has(t.source) ? "it is caught up again first (the slot was made again meanwhile)" : "the stream carries what follows once it runs again")
2028
- : this.mode === "poll" && overPollLimit(t) ? `it is ${OVER_POLL_LIMIT}, so that was one scan for its rebuild, not the periodic poll: served as of now, mason sync ${t.source} syncs it again`
2029
- : "the poll keeps it current";
2548
+ : s.mode === "off" ? `its sync is off, so it is served as of now: mason sync ${t.source} syncs it again`
2549
+ : `the poll keeps it current (${syncWords(s)})`;
2030
2550
  this.log(`caught up ${t.source} by one poll ${mark.why}: ${tail}`);
2031
2551
  return null;
2032
2552
  }
@@ -2040,12 +2560,15 @@ export class Sync {
2040
2560
  async swap(t, side, o) {
2041
2561
  const w = this.store.writer;
2042
2562
  const target = `${ident(t.schema)}.${ident(t.table)}`;
2563
+ // the copy's time in the ledger and in this run, ONE value, and before `copied_at`, recorded after the swap
2564
+ // (`readSynced`: a copy recorded at or after its polled_at was last brought current by that copy)
2565
+ const at = new Date().toISOString();
2043
2566
  await w.run("BEGIN TRANSACTION");
2044
2567
  try {
2045
2568
  await w.run(`DROP TABLE IF EXISTS ${target}`);
2046
2569
  await w.run(`ALTER TABLE ${side} RENAME TO ${ident(t.table)}`);
2047
2570
  if (o.snapshot !== null)
2048
- await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state VALUES (${literal(t.source)}, ${literal(o.snapshot)}, now())`);
2571
+ await w.run(this.pollStateSql(t, o.snapshot, at));
2049
2572
  if (o.behind !== null)
2050
2573
  await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(o.behind)}, now())`);
2051
2574
  // the copy is whole: nothing of it is left to continue
@@ -2061,19 +2584,33 @@ export class Sync {
2061
2584
  }
2062
2585
  // the copy's view of its visible columns, made again (its own transaction: DuckDB writes one database a transaction)
2063
2586
  await this.kept.swapped(t.source);
2064
- if (o.snapshot !== null)
2587
+ if (o.snapshot !== null) {
2065
2588
  this.snapshots.set(t.source, o.snapshot);
2589
+ this.recordedSync.set(t.source, syncText(this.syncOf(t)));
2590
+ this.switched.delete(t.source);
2591
+ this.due.delete(t.source);
2592
+ // synced as of its copy, said at once (a review finding: until the next pass re-read the ledger, a table just
2593
+ // copied said "never synced"), and the value the ledger holds, so that read moves nothing
2594
+ this.polledAt.set(t.source, at);
2595
+ }
2066
2596
  // the columns this copy holds, from its swap on: what every write to it is shaped by (`asCopied`)
2067
2597
  this.copiedColumns.set(t.source, [...t.columns]);
2068
2598
  if (o.behind !== null) {
2069
2599
  this.behind.set(t.source, { why: o.behind });
2070
- if (this.mode === "replication")
2600
+ if (this.replicated(t))
2071
2601
  this.held.set(t.source, noChanges());
2072
2602
  }
2073
2603
  this.retired.delete(t.source);
2074
2604
  this.duckTypes.delete(t.source);
2605
+ // a new copy, a new stay, whatever the columns: what the stream decoded before this moment is not for it
2606
+ this.streamed.delete(t.source);
2075
2607
  this.refreshStreamed();
2076
2608
  }
2609
+ /** a table's watermark, the time and the sync it is polled or copied under (`recordedSync`): ONE row of `_mason.poll_state`.
2610
+ * `at` is the caller's, an ISO instant (`toISOString`, as `copied_at`), the very value it remembers (`polledAt`) */
2611
+ pollStateSql(t, snapshot, at) {
2612
+ return `INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state (source, snapshot, polled_at, sync) VALUES (${literal(t.source)}, ${literal(snapshot)}, ${literal(at)}::TIMESTAMPTZ, ${literal(syncText(this.syncOf(t)))})`;
2613
+ }
2077
2614
  /**
2078
2615
  * One poll of the tables given (the passes and `pollNow` give one at a time), in ONE DuckDB transaction over ONE
2079
2616
  * source snapshot (the module's header), on the lane's connection — the writer's, or the rebuilds' for a catch-up —
@@ -2111,12 +2648,12 @@ export class Sync {
2111
2648
  // (`adds`: the keys a stage still to be written adds to DuckDB's count)
2112
2649
  const reconcile = async (t, target, from, adds = 0n) => {
2113
2650
  const c0 = Date.now();
2114
- doing = `${t.source}: counting its rows on the source and in DuckDB`;
2651
+ doing = `${t.source}: counting its rows on the source and in the synced table`;
2115
2652
  const counts = (await w.runAndReadAll(`SELECT (SELECT count FROM ${pq(`SELECT count(*) AS count FROM ${from}`)}), (SELECT count(*) FROM ${target})`)).getRowsJS()[0];
2116
2653
  const countMs = Date.now() - c0;
2117
2654
  let deleted = 0;
2118
2655
  if (counts[0] !== counts[1] + adds) {
2119
- doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with DuckDB's)`;
2656
+ doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with the synced table's)`;
2120
2657
  const on = t.key.map((k) => `t.${ident(k)} = k.${ident(k)}`).join(" AND ");
2121
2658
  const gone = await w.run(`DELETE FROM ${target} WHERE rowid IN (SELECT t.rowid FROM ${target} t ANTI JOIN ${pq(`SELECT ${t.key.map(pgIdent).join(", ")} FROM ${from}`)} k ON ${on})`);
2122
2659
  deleted = Number(gone.rowsChanged);
@@ -2124,6 +2661,9 @@ export class Sync {
2124
2661
  return { rows: Number(counts[0]), deleted, countMs };
2125
2662
  };
2126
2663
  let snapshot = "";
2664
+ // THE POLL'S TIME, ONE VALUE: taken before its snapshot, so `syncedAt` never says more than the rows hold, written
2665
+ // with its watermark (`pollStateSql`) and remembered as written once that commits (`committed`, `polledAt`)
2666
+ const polledAt = new Date().toISOString();
2127
2667
  await w.run("BEGIN TRANSACTION");
2128
2668
  try {
2129
2669
  await this.interruptible(lane, async () => {
@@ -2149,7 +2689,7 @@ export class Sync {
2149
2689
  `${pq(`SELECT ${pgCols} FROM ${from} WHERE ${xmin8Sql(xmax)} >= ${watermark}`)}) AS polled(${t.columns.map((_, i) => `c${i}`).join(", ")})`)).rowsChanged);
2150
2690
  if (staged > POLL_PIECE_ROWS) {
2151
2691
  const c1 = Date.now();
2152
- doing = `${t.source}: counting the keys its ${staged} changed row(s) add to DuckDB (the stage anti-joined with DuckDB's keys)`;
2692
+ doing = `${t.source}: counting the keys its ${staged} changed row(s) add to the synced table (the stage anti-joined with its keys)`;
2153
2693
  const on = t.key.map((k) => `s.${ident(`c${t.columns.findIndex((c) => c.name === k)}`)} = t.${ident(k)}`).join(" AND ");
2154
2694
  const adds = (await w.runAndReadAll(`SELECT count(*) FROM ${stage} s ANTI JOIN ${target} t ON ${on}`)).getRowsJS()[0][0];
2155
2695
  const { rows, deleted, countMs } = await reconcile(t, target, from, adds);
@@ -2162,7 +2702,7 @@ export class Sync {
2162
2702
  await writeStaged(w, t, target, stage, "", this.kept);
2163
2703
  const c1 = Date.now();
2164
2704
  const { rows, deleted, countMs } = await reconcile(t, target, from);
2165
- await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state VALUES (${literal(t.source)}, ${literal(snapshot)}, now())`);
2705
+ await w.run(this.pollStateSql(t, snapshot, polledAt));
2166
2706
  await w.run(`DROP TABLE ${stage}`);
2167
2707
  timed.push(`${t.source} ${staged} changed in ${c1 - c0} ms, counted in ${countMs} ms${deleted ? `, ${deleted} deleted in ${Date.now() - c1 - countMs} ms` : ""}`);
2168
2708
  found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - c0 });
@@ -2185,22 +2725,25 @@ export class Sync {
2185
2725
  // matched against the tables' own names (a name may hold a space)
2186
2726
  const wider = e instanceof Error ? tables.find((t) => e.message.includes(`${WIDER} [${t.source}]`))?.source : undefined;
2187
2727
  if (wider !== undefined && this.snapshots.delete(wider)) {
2188
- this.log(`poll: ${wider} has a numeric value wider than its copy's DECIMAL: it is copied again at the next pass`);
2728
+ this.log(`poll: ${wider} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass`);
2189
2729
  return [];
2190
2730
  }
2191
2731
  throw named(e);
2192
2732
  }
2193
2733
  // committed: the watermarks and the row counts move together (a pieced table's with its last piece)
2194
- const committed = async (source, rows) => {
2734
+ // and the table's last sync (`Synced`): the poll's time as the ledger now holds it, and how long THIS table's poll took
2735
+ const committed = async (source, rows, ms) => {
2736
+ this.polledAt.set(source, polledAt);
2195
2737
  const st = this.states.get(source);
2196
2738
  if (st.rows !== rows || st.error !== null)
2197
2739
  await this.record({ ...st, rows, error: null }, undefined, w);
2198
2740
  this.snapshots.set(source, snapshot);
2741
+ this.synced.set(source, { at: polledAt, seconds: ms / 1000, by: lane === this.writerLane ? "poll" : "catch-up" });
2199
2742
  };
2200
2743
  try {
2201
2744
  for (const [source, rows] of pending)
2202
2745
  if (!pieced.some((p) => p.t.source === source))
2203
- await committed(source, rows);
2746
+ await committed(source, rows, found.find((f) => f.table === source)?.ms ?? 0);
2204
2747
  this.log(`poll: ${timed.join("; ")}`);
2205
2748
  for (const { t, target, stage, staged, deleted, started } of pieced) {
2206
2749
  const p0 = Date.now();
@@ -2211,7 +2754,7 @@ export class Sync {
2211
2754
  doing = `${t.source}: writing piece ${i + 1} of ${n} of its ${staged} changed row(s) by key`;
2212
2755
  await this.interruptible(lane, () => writeStaged(w, t, target, stage, ` WHERE rowid >= ${i * POLL_PIECE_ROWS} AND rowid < ${(i + 1) * POLL_PIECE_ROWS}`, this.kept));
2213
2756
  if (i === n - 1) {
2214
- await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state VALUES (${literal(t.source)}, ${literal(snapshot)}, now())`);
2757
+ await w.run(this.pollStateSql(t, snapshot, polledAt));
2215
2758
  }
2216
2759
  doing = `${t.source}: committing piece ${i + 1} of ${n}`;
2217
2760
  await w.run("COMMIT");
@@ -2222,7 +2765,7 @@ export class Sync {
2222
2765
  }
2223
2766
  }
2224
2767
  const rows = pending.get(t.source);
2225
- await committed(t.source, rows);
2768
+ await committed(t.source, rows, Date.now() - started);
2226
2769
  found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - started });
2227
2770
  this.log(`poll: ${t.source}: ${staged.toLocaleString("en-US")} changed row(s), written in ${n} pieces in ${Date.now() - p0} ms`);
2228
2771
  }
@@ -2295,10 +2838,10 @@ export class Sync {
2295
2838
  if (resumed === null) {
2296
2839
  // the stream (or the other tables' polls) goes on: the slot holds no WAL for this copy, and the table joins once
2297
2840
  // caught up from the copy's snapshot
2298
- const beside = this.mode === "replication" ? "beside the stream, which goes on keeping the other tables current: it is streamed once caught up by one poll from the copy's snapshot"
2299
- : this.mode === "poll" ? "beside the other tables' polls: it is caught up by one poll from the copy's snapshot once copied" : "beside the other tables";
2300
- const again = this.retired.has(t.source) ? `, again (its copy is ${RETIRED_LAYOUT})` : "";
2301
- this.log(`copying ${t.source}${again}: ${t.rows === null ? "rows unknown" : `about ${t.rows} row(s)`}, ${t.bytes} source bytes, ${beside}`);
2841
+ const beside = this.replicated(t) ? "beside the stream, which goes on keeping the other tables current: it is streamed once caught up by one poll from the snapshot it was read at"
2842
+ : this.mode !== "recopy" ? "beside the other tables' polls: once read, it is caught up by one poll from the snapshot it was read at" : "beside the other tables";
2843
+ const again = this.retired.has(t.source) ? `, again (it is ${RETIRED_LAYOUT})` : "";
2844
+ this.log(`syncing ${t.source}${again}: ${t.rows === null ? "rows unknown" : `about ${t.rows} row(s)`}, ${t.bytes} source bytes, ${beside}`);
2302
2845
  }
2303
2846
  let read;
2304
2847
  try {
@@ -2308,7 +2851,7 @@ export class Sync {
2308
2851
  if (!(e instanceof StartOver))
2309
2852
  throw e;
2310
2853
  // once: a second reason to start over is the copy's failure
2311
- this.log(`starting the copy of ${t.source} over, from its first page: ${e.message}`);
2854
+ this.log(`starting the sync of ${t.source} over, from its first page: ${e.message}`);
2312
2855
  kept = await this.forgetCopy(t, side);
2313
2856
  read = await this.readChunks(t, side, null, (p) => { kept = p; });
2314
2857
  }
@@ -2320,7 +2863,7 @@ export class Sync {
2320
2863
  const [rows, keys] = (await w.runAndReadAll(`SELECT count(*), count(DISTINCT (${t.key.map(ident).join(", ")})) FROM ${side}`)).getRowsJS()[0];
2321
2864
  if (rows !== keys) {
2322
2865
  kept = await this.forgetCopy(t, side);
2323
- throw new Error(`its copy holds ${rows} rows but ${keys} distinct keys (Mason bug: report it, with mason logs): it was not swapped in, and is copied again from its first page`);
2866
+ throw new Error(`its synced data holds ${rows} rows but ${keys} distinct keys (Mason bug: report it, with mason logs): it was not swapped in, and is read again from its first page`);
2324
2867
  }
2325
2868
  }
2326
2869
  // its stored JSON paths filled, so the table swapped in has every one whole (src/duck/kept.ts fillSide)
@@ -2328,14 +2871,17 @@ export class Sync {
2328
2871
  // THE SWAP, on the writer's turn: one transaction, between two of the stream's or the polls'. It carries the copy's
2329
2872
  // watermark (its first snapshot), removes its progress, and leaves the table behind until its catch-up (none in
2330
2873
  // recopy mode: no poll)
2331
- await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its copy" }));
2874
+ await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its whole read" }));
2332
2875
  this.copyLive.delete(t.source);
2333
2876
  kept = null;
2334
2877
  this.retries.delete(t.source);
2335
2878
  const seconds = (Date.now() - started) / 1000;
2336
- this.log(`copied ${t.source}: ${p.rows} row(s), ${t.bytes} source bytes, in ${seconds.toFixed(2)} s (read in ${read.seconds.toFixed(2)} s` +
2879
+ this.log(`synced ${t.source}: ${p.rows} row(s), ${t.bytes} source bytes, in ${seconds.toFixed(2)} s (read in ${read.seconds.toFixed(2)} s` +
2337
2880
  `${resumed === null ? "" : `, resumed at page ${readThrough(resumed.done) ?? p.pages} of ${resumed.pages}${pastPages(resumed.done)} with ${resumed.rows} row(s) in`}), stored as the source's scan delivered them, with no index`);
2338
- await this.record({ source: t.source, phase: "copied", rows: p.rows, bytes: t.bytes, seconds, copiedAt: new Date().toISOString(), error: null }, t.columns, w);
2881
+ const copiedAt = new Date().toISOString();
2882
+ await this.record({ source: t.source, phase: "copied", rows: p.rows, bytes: t.bytes, seconds, copiedAt, error: null }, t.columns, w);
2883
+ this.synced.set(t.source, { at: copiedAt, seconds, by: "copy" });
2884
+ this.event({ kind: "copied", table: t.source, message: `synced: ${p.rows.toLocaleString("en-US")} row(s) in ${seconds.toFixed(2)} s` });
2339
2885
  this.copiedAs.set(t.source, columnsKey(t.columns));
2340
2886
  // in the catalog at once: a pass's other copies may take hours
2341
2887
  await this.rebuildSchema();
@@ -2351,7 +2897,7 @@ export class Sync {
2351
2897
  k = await this.forgetCopy(t, side);
2352
2898
  }
2353
2899
  catch (f) {
2354
- this.log(`the copy of ${t.source} could not drop what it began (its next attempt continues it): ${f instanceof Error ? f.message : String(f)}`);
2900
+ this.log(`the sync of ${t.source} could not drop what it began (its next attempt continues it): ${f instanceof Error ? f.message : String(f)}`);
2355
2901
  }
2356
2902
  }
2357
2903
  const through = k === null ? null : readThrough(k.done);
@@ -2359,7 +2905,7 @@ export class Sync {
2359
2905
  if (e instanceof Stopping) {
2360
2906
  // the chunks in flight rolled back: a table never copied continues at the next start, one copied before serves its
2361
2907
  // copy as it was meanwhile
2362
- this.log(`copy of ${t.source} stopped: Mason is stopping (${where === null ? "rolled back; it is copied at the next start" : `the chunks in flight rolled back; it continues ${where} at the next start`})`);
2908
+ this.log(`sync of ${t.source} stopped: Mason is stopping (${where === null ? "rolled back; it is synced at the next start" : `the chunks in flight rolled back; it continues ${where} at the next start`})`);
2363
2909
  if (first)
2364
2910
  await this.record({ ...was, phase: "waiting", error: null }, undefined, w);
2365
2911
  return;
@@ -2373,11 +2919,24 @@ export class Sync {
2373
2919
  this.retries.set(t.source, { failures, at });
2374
2920
  const when = `tried again after ${new Date(at).toISOString().slice(11, 16)} UTC (failure ${failures} in a row), or at once when Mason starts again or its columns or key change` +
2375
2921
  (where === null ? "" : `; it continues ${where}`);
2376
- this.log(`copy of ${t.source} failed: ${m}; ${when}`);
2922
+ this.log(`sync of ${t.source} failed: ${m}; ${when}`);
2923
+ this.event({ kind: "copy_failed", table: t.source, message: `${m}; ${when}` });
2377
2924
  // a table that was copied keeps serving its last copy; one never copied is failed
2378
2925
  await this.record({ ...this.states.get(t.source), phase: first ? "failed" : "copied", error: `${m} (${when})` }, undefined, w);
2379
2926
  }
2380
2927
  }
2928
+ /** Where the copy's file is, the bytes free there, and what the copies running are projected to add (`DiskStatus`). */
2929
+ diskStatus() {
2930
+ const dir = path.dirname(this.store.file);
2931
+ const needed = [...this.copyLive.values()].map((l) => l.diskNeeded).filter((n) => n !== null);
2932
+ const projectedBytes = needed.length ? Math.round(needed.reduce((a, b) => a + b, 0)) : null;
2933
+ try {
2934
+ return { path: dir, freeBytes: this.diskFree(dir), projectedBytes };
2935
+ }
2936
+ catch (e) {
2937
+ return { path: dir, freeBytes: null, projectedBytes, error: `the free space where the synced data is could not be read: ${e instanceof Error ? e.message : String(e)}` };
2938
+ }
2939
+ }
2381
2940
  /** The DuckDB file's bytes with its write-ahead log's (what a copy adds lands in the log until a checkpoint). */
2382
2941
  fileBytes() {
2383
2942
  let n = 0;
@@ -2392,16 +2951,18 @@ export class Sync {
2392
2951
  }
2393
2952
  return n;
2394
2953
  }
2395
- /** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null. */
2396
- diskShort(t, p, disk) {
2954
+ /** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null. What
2955
+ * it projects is kept on the copy's `live` (`diskStatus` says it). */
2956
+ diskShort(t, p, disk, live) {
2397
2957
  const bytesNow = this.fileBytes();
2398
2958
  const projected = copyDiskProjection({ pages: p.pages, done: p.done, doneAtStart: disk.doneAtStart, bytesAtStart: disk.bytesAtStart, bytesNow });
2959
+ live.diskNeeded = projected?.needed ?? null;
2399
2960
  if (projected === null)
2400
2961
  return null;
2401
2962
  const free = this.diskFree(path.dirname(this.store.file));
2402
2963
  if (projected.needed <= free)
2403
2964
  return null;
2404
- return `the copy of ${t.source} would need about ${humanBytes(projected.needed)} more on disk than is free (${humanBytes(free)} free where ${this.store.file} is; ` +
2965
+ return `the sync of ${t.source} would need about ${humanBytes(projected.needed)} more on disk than is free (${humanBytes(free)} free where ${this.store.file} is; ` +
2405
2966
  `${pagesRead(p.done, p.pages)} of ${p.pages} pages read, ${humanBytes(bytesNow - disk.bytesAtStart)} added): free some disk, mason remove ${t.source}, or mason add ${t.source} --columns … with fewer columns`;
2406
2967
  }
2407
2968
  /**
@@ -2480,7 +3041,7 @@ export class Sync {
2480
3041
  const pagesDone = pagesRead(p.done, p.pages);
2481
3042
  const live = { pages: p.pages, chunksDone: p.done.length, pagesDone, rowsDone: p.rows, untaken: gapPages(gaps, p.pages),
2482
3043
  chunkPages: nextChunkPages(ceiling, null, this.copyChunkSeconds), readers, reading: [], rate: null,
2483
- rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null };
3044
+ rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null, diskNeeded: null };
2484
3045
  this.copyLive.set(t.source, live);
2485
3046
  for (let i = 1; i < readers; i++) {
2486
3047
  const lane = { conn: await this.store.instance.connect(), inFlight: false };
@@ -2535,7 +3096,7 @@ export class Sync {
2535
3096
  const node = Number((await c.runAndReadAll(`SELECT * FROM ${pq(`SELECT pg_relation_filenode(${pgLiteral(pgFrom)}::regclass)::bigint`)}`)).getRowsJS()[0][0]);
2536
3097
  filenode ??= node;
2537
3098
  if (node !== filenode) {
2538
- throw new StartOver(`the source rewrote ${t.source} since its copy began (its file changed: a VACUUM FULL, CLUSTER, TRUNCATE or column type change), so its rows moved between pages`);
3099
+ throw new StartOver(`the source rewrote ${t.source} since its sync began (its file changed: a VACUUM FULL, CLUSTER, TRUNCATE or column type change), so its rows moved between pages`);
2539
3100
  }
2540
3101
  const n = Number(r.rowsChanged);
2541
3102
  await this.onChunk?.({ ...told, rows: n, phase: "read" });
@@ -2577,7 +3138,7 @@ export class Sync {
2577
3138
  await c.run("ROLLBACK");
2578
3139
  }
2579
3140
  catch (r) {
2580
- this.log(`a chunk of ${t.source}'s copy could not be rolled back after it failed (its connection is closed with the copy): ${r instanceof Error ? r.message : String(r)}`);
3141
+ this.log(`a chunk of ${t.source}'s sync could not be rolled back after it failed (its connection is closed with the sync): ${r instanceof Error ? r.message : String(r)}`);
2581
3142
  }
2582
3143
  }
2583
3144
  fail(e);
@@ -2585,14 +3146,14 @@ export class Sync {
2585
3146
  }
2586
3147
  // STOPPED BEFORE THE DISK FILLS: the chunks committed are kept, the copy fails with the way out and is tried
2587
3148
  // again as a failed copy is (freed disk is used at the retry, or at once at the next start)
2588
- const short = recopy ? null : this.diskShort(t, p, disk);
3149
+ const short = recopy ? null : this.diskShort(t, p, disk, live);
2589
3150
  if (short !== null) {
2590
3151
  fail(new Error(short));
2591
3152
  return;
2592
3153
  }
2593
3154
  const at = readThrough(p.done);
2594
3155
  if (Date.now() - lineAt >= 60_000 && at !== null) {
2595
- this.log(`copying ${t.source}: page ${at} of ${p.pages}${pastPages(p.done)}, ${p.rows.toLocaleString("en-US")} row(s) in; ${chunks} chunk(s) in the last minute on ${readers} reader(s), ` +
3156
+ this.log(`syncing ${t.source}: page ${at} of ${p.pages}${pastPages(p.done)}, ${p.rows.toLocaleString("en-US")} row(s) in; ${chunks} chunk(s) in the last minute on ${readers} reader(s), ` +
2596
3157
  `${(chunkMs / chunks / 1000).toFixed(1)} s each` + (rate === null ? "" : `; a reader reads ~${Math.round(rate.rowsPerSecond).toLocaleString("en-US")} rows/s ` +
2597
3158
  `(${humanBytes(rate.bytesPerSecond)}/s of the table's pages), so the next chunk is ${live.chunkPages} page(s), ~${this.copyChunkSeconds} s (MASON_COPY_CHUNK_SECONDS; at most ${ceiling})`));
2598
3159
  lineAt = Date.now();
@@ -2650,7 +3211,7 @@ export class Sync {
2650
3211
  const ceiling = chunkPages(pages, t.rows, this.copyChunkRows);
2651
3212
  const firstSize = nextChunkPages(ceiling, null, this.copyChunkSeconds);
2652
3213
  const readers = this.mode === "recopy" ? 1 : Math.min(this.copyParallel, Math.max(1, Math.ceil(pages / firstSize)));
2653
- this.log(`copying ${t.source}${snapshot === null ? "" : ` as of the source's snapshot ${snapshot}: what commits after it is caught up by one poll once copied`}; ` +
3214
+ this.log(`syncing ${t.source}${snapshot === null ? "" : ` as of the source's snapshot ${snapshot}: what commits after it is caught up by one poll once read`}; ` +
2654
3215
  `${pages} page(s) in chunks sized to ~${this.copyChunkSeconds} s of reading each at the rate measured here (MASON_COPY_CHUNK_SECONDS; at most ${ceiling} page(s)), ` +
2655
3216
  `the first of ${firstSize} page(s) to measure it, ${readers === 1 ? "one at a time" : `${readers} at a time (MASON_COPY_PARALLEL), each a backend on the database`}`);
2656
3217
  return { snapshot, columns: columnsKey(t.columns), exact, filenode: null, pages, done: [], rows: 0 };
@@ -2676,7 +3237,7 @@ export class Sync {
2676
3237
  const row = r[0];
2677
3238
  if (row === undefined || row[0] === null || row[1] !== columnsKey(t.columns) || !there) {
2678
3239
  if (row !== undefined)
2679
- this.log(`starting the copy of ${t.source} over, from its first page: ${!there ? "the table it was filling is gone" : row[0] === null ? "it was begun as a recopy" : "its columns are not the ones its copy was begun with"}`);
3240
+ this.log(`starting the sync of ${t.source} over, from its first page: ${!there ? "the table it was filling is gone" : row[0] === null ? "it was begun as a recopy" : "its columns are not the ones its sync was begun with"}`);
2680
3241
  return this.forgetCopy(t, side);
2681
3242
  }
2682
3243
  const chunks = (await w.runAndReadAll(`SELECT first_page, end_page, "rows", filenode FROM ${MASON_SCHEMA}.copy_chunks WHERE source = ${literal(t.source)} ORDER BY first_page`)).getRowsJS();
@@ -2684,8 +3245,8 @@ export class Sync {
2684
3245
  const rows = chunks.reduce((n, c) => n + Number(c[2]), 0);
2685
3246
  const p = { snapshot: row[0], columns: row[1], exact: new Map(JSON.parse(row[2])), filenode: chunks.length ? Number(chunks[0][3]) : null, pages: Number(row[3]), done, rows };
2686
3247
  const at = readThrough(done);
2687
- this.log(at === null ? `resuming the copy of ${t.source} with every page read (${rows} row(s) in): its swap is next`
2688
- : `resuming the copy of ${t.source} at page ${at} of ${p.pages}${pastPages(done)}, ${rows} row(s) in, held to its first snapshot ${p.snapshot}: the pages read before are not read again`);
3248
+ this.log(at === null ? `resuming the sync of ${t.source} with every page read (${rows} row(s) in): its swap is next`
3249
+ : `resuming the sync of ${t.source} at page ${at} of ${p.pages}${pastPages(done)}, ${rows} row(s) in, held to its first snapshot ${p.snapshot}: the pages read before are not read again`);
2689
3250
  const now = this.states.get(t.source);
2690
3251
  if (now !== undefined && now.phase === "copying")
2691
3252
  this.states.set(t.source, { ...now, rowsSoFar: rows, rowsEstimate: t.rows });
@@ -2710,7 +3271,7 @@ export class Sync {
2710
3271
  await w.run(`ALTER TABLE ${MASON_SCHEMA}.copy_progress DROP COLUMN IF EXISTS ${ident(gone)}`);
2711
3272
  await w.run("COMMIT");
2712
3273
  if (moved > 0)
2713
- this.log(`${moved} copy(ies) in progress were begun by a Mason that read one chunk at a time: each continues where it was`);
3274
+ this.log(`${moved} sync(s) in progress were begun by a Mason that read one chunk at a time: each continues where it was`);
2714
3275
  }
2715
3276
  catch (e) {
2716
3277
  await w.run("ROLLBACK");
@@ -2738,7 +3299,7 @@ export class Sync {
2738
3299
  if (this.states.get(source)?.phase !== "copied")
2739
3300
  continue;
2740
3301
  this.retired.add(source);
2741
- this.log(`${source}: its copy is ${RETIRED_LAYOUT} — its rows are sorted on disk, and only the source has them in its own order, so it is read again whole; the copy serves as it was until then`);
3302
+ this.log(`${source}: it is ${RETIRED_LAYOUT} — its rows are sorted on disk, and only the source has them in its own order, so it is read again whole; its synced data serves as it was until then`);
2742
3303
  }
2743
3304
  }
2744
3305
  /** what a copy of the retired layout left beside the file (in that layout a copy was staged in a file of its own,
@@ -2747,7 +3308,7 @@ export class Sync {
2747
3308
  const dir = path.dirname(this.store.file), base = `${path.basename(this.store.file)}.stage-`;
2748
3309
  for (const f of fs.readdirSync(dir).filter((x) => x.startsWith(base))) {
2749
3310
  fs.rmSync(path.join(dir, f), { recursive: true, force: true });
2750
- this.log(`removed ${f}: a copy's stage, from an older Mason (a copy now reads the source once, straight into its table)`);
3311
+ this.log(`removed ${f}: a sync's stage, from an older Mason (a sync now reads the source once, straight into its table)`);
2751
3312
  }
2752
3313
  }
2753
3314
  /** The table's numerics the extension would read as doubles, each with the DECIMAL(38, s) scale that holds every value
@@ -2771,11 +3332,11 @@ export class Sync {
2771
3332
  const scale = declared[1] !== undefined ? used ?? 0 : maxScale ?? 0;
2772
3333
  const width = (digits ?? 1) + scale;
2773
3334
  if ((special ?? 0) > 0) {
2774
- this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which DuckDB's DECIMAL cannot hold: copied as a double`);
3335
+ this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which the synced table's DECIMAL cannot hold: synced as a double`);
2775
3336
  return;
2776
3337
  }
2777
3338
  if (scale > 38 || width > 38) {
2778
- this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit DuckDB's DECIMAL(38): copied as a double (15 significant digits)`);
3339
+ this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit the synced table's DECIMAL(38): synced as a double (15 significant digits)`);
2779
3340
  return;
2780
3341
  }
2781
3342
  if (declared[1] === undefined && minScale !== null && minScale !== maxScale)
@@ -2795,6 +3356,12 @@ export class Sync {
2795
3356
  * writer only on its turn: a statement on a connection lands inside the transaction open there). The table's row is
2796
3357
  * that flow's alone while it writes it (a rebuild's table is not polled or streamed until caught up) */
2797
3358
  async record(s, columns, c = this.store.writer) {
3359
+ // A COPY CURRENT AGAIN after its poll, catch-up or stream failed (its error cleared by the flow that keeps it current,
3360
+ // never by a copy of it: that is its own `copied` event) — the one place an error is cleared, so the one place said
3361
+ const was = this.states.get(s.source);
3362
+ if (was !== undefined && was.error !== null && s.error === null && was.phase === "copied" && s.phase === "copied" && was.copiedAt === s.copiedAt) {
3363
+ this.event({ kind: "recovered", table: s.source, message: `current again (it had failed: ${was.error.split("\n")[0]})` });
3364
+ }
2798
3365
  this.states.set(s.source, s);
2799
3366
  const v = (x) => x === null ? "NULL" : typeof x === "number" ? String(x) : literal(x);
2800
3367
  const sql = `INSERT OR REPLACE INTO ${MASON_SCHEMA}.copy_state (source, phase, "rows", bytes, seconds, copied_at, error, columns) VALUES (${literal(s.source)}, ${literal(s.phase)}, ${v(s.rows)}, ${v(s.bytes)}, ${v(s.seconds)}, ` +
@@ -2804,7 +3371,7 @@ export class Sync {
2804
3371
  await c.run(sql);
2805
3372
  }
2806
3373
  catch (e) {
2807
- this.log(`recording ${s.source}'s copy state failed: ${e instanceof Error ? e.message : String(e)}`);
3374
+ this.log(`recording ${s.source}'s sync state failed: ${e instanceof Error ? e.message : String(e)}`);
2808
3375
  }
2809
3376
  }
2810
3377
  /** the catalog rebuilt (`schemaBuilt`), after any rebuild already under way */
@@ -2846,7 +3413,7 @@ export class Sync {
2846
3413
  const key = JSON.stringify([tables, views]);
2847
3414
  if (key !== this.schemaKey) {
2848
3415
  this.schemaKey = key;
2849
- this.currentSchema = { database: this.database, tables, ...(views.length > 0 ? { views } : {}) };
3416
+ this.currentSchema = { database: this.currentSchema.database, tables, ...(views.length > 0 ? { views } : {}) };
2850
3417
  }
2851
3418
  }
2852
3419
  }