@mason-data/cli 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -69
- package/dist/src/api/error.js +3 -0
- package/dist/src/api/postgres/catalog.js +258 -14
- package/dist/src/api/postgres/encode.js +3 -1
- package/dist/src/api/postgres/explain.js +3 -2
- package/dist/src/api/postgres/protocol.js +39 -23
- package/dist/src/api/postgres/server.js +1290 -223
- package/dist/src/api/postgres/translate.js +81 -9
- package/dist/src/api/request-log.js +18 -5
- package/dist/src/cdc/pg-wire.js +144 -38
- package/dist/src/cdc/poll.js +5 -5
- package/dist/src/cli/background.js +55 -8
- package/dist/src/cli/cancel.js +75 -0
- package/dist/src/cli/command.js +5 -3
- package/dist/src/cli/commands.js +843 -272
- package/dist/src/cli/control.js +59 -0
- package/dist/src/cli/demo.js +158 -0
- package/dist/src/cli/dev.js +458 -125
- package/dist/src/cli/doctor.js +25 -12
- package/dist/src/cli/guides.js +103 -31
- package/dist/src/cli/init.js +159 -58
- package/dist/src/cli/machine.js +18 -12
- package/dist/src/cli/main.js +5 -5
- package/dist/src/cli/mcp.js +735 -0
- package/dist/src/cli/metrics.js +79 -0
- package/dist/src/cli/pg-tls.js +13 -3
- package/dist/src/cli/pg-url.js +4 -0
- package/dist/src/cli/pin.js +13 -12
- package/dist/src/cli/preflight.js +24 -14
- package/dist/src/cli/project.js +48 -7
- package/dist/src/cli/readiness.js +38 -4
- package/dist/src/cli/registry.js +57 -35
- package/dist/src/cli/replication-checks.js +7 -5
- package/dist/src/cli/replication-mode.js +105 -26
- package/dist/src/cli/request-filter.js +134 -0
- package/dist/src/cli/roles.js +584 -0
- package/dist/src/cli/run-server.js +498 -133
- package/dist/src/cli/runs.js +1 -1
- package/dist/src/cli/security.js +10 -8
- package/dist/src/cli/serve.js +147 -40
- package/dist/src/cli/share-run.js +286 -0
- package/dist/src/cli/share.js +286 -0
- package/dist/src/cli/source-edit.js +168 -0
- package/dist/src/cli/sources.js +290 -39
- package/dist/src/cli/supervise.js +12 -1
- package/dist/src/cli/sync.js +55 -25
- package/dist/src/cli/tunnel.js +200 -0
- package/dist/src/cli/ui.js +9 -6
- package/dist/src/cli/usage.js +128 -0
- package/dist/src/cli/user.js +496 -40
- package/dist/src/cli/version.js +9 -7
- package/dist/src/demo/seed.js +240 -0
- package/dist/src/demo/source.js +655 -0
- package/dist/src/duck/cancel.js +10 -1
- package/dist/src/duck/distinct-key.js +228 -0
- package/dist/src/duck/events.js +134 -0
- package/dist/src/duck/executor-worker.js +25 -11
- package/dist/src/duck/executor.js +107 -53
- package/dist/src/duck/explain.js +158 -29
- package/dist/src/duck/extensions.js +160 -0
- package/dist/src/duck/functions.js +73 -19
- package/dist/src/duck/gaps.js +25 -11
- package/dist/src/duck/grants.js +176 -0
- package/dist/src/duck/history.js +123 -0
- package/dist/src/duck/jsonb-order.js +175 -0
- package/dist/src/duck/jsonb.js +10 -8
- package/dist/src/duck/kept-route.js +82 -2
- package/dist/src/duck/kept.js +211 -44
- package/dist/src/duck/pgtree.js +40 -0
- package/dist/src/duck/plan.js +120 -30
- package/dist/src/duck/publication.js +6 -6
- package/dist/src/duck/requests.js +265 -93
- package/dist/src/duck/rewrite.js +1193 -73
- package/dist/src/duck/row-filter.js +649 -0
- package/dist/src/duck/store.js +271 -159
- package/dist/src/duck/stream.js +10 -5
- package/dist/src/duck/sync.js +907 -275
- package/dist/src/duck/types.js +6 -5
- package/dist/src/duck/usage.js +324 -0
- package/dist/src/duck/users.js +64 -15
- package/dist/src/duck/verified.js +9 -0
- package/dist/src/duck/views.js +2 -3
- package/dist/src/lib/database-name.js +62 -0
- package/dist/src/lib/dotenv.js +17 -4
- package/dist/src/lib/log-buffer.js +68 -0
- package/dist/src/lib/pg-numeric.js +5 -2
- package/dist/src/lib/project-version.js +23 -23
- package/dist/src/lib/table-sync.js +267 -0
- package/dist/src/lib/timezone.js +3 -2
- package/dist/src/web/app.css +706 -55
- package/dist/src/web/app.js +330 -258
- package/dist/src/web/charts.js +229 -0
- package/dist/src/web/index.html +84 -15
- package/dist/src/web/pages/queries.js +192 -0
- package/dist/src/web/pages/status.js +446 -0
- package/dist/src/web/pages/syncs.js +155 -0
- package/dist/src/web/pages/tables.js +434 -0
- package/dist/src/web/pages/usage.js +397 -0
- package/dist/src/web/pages/users.js +218 -0
- package/dist/src/web/shared.js +568 -0
- package/dist/src/web/vendor/README.md +25 -0
- package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
- package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
- package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
- package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
- package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
- package/package.json +10 -3
package/dist/src/duck/sync.js
CHANGED
|
@@ -11,6 +11,8 @@ import { createSlot, currentLsn, dropSlot, readSlot } from "../cdc/source.js";
|
|
|
11
11
|
import { addCounts, countsText, duckValue, foldInto, noChanges, noCounts, ReplicationStream } from "./stream.js";
|
|
12
12
|
import { didYouMean } from "../lib/did-you-mean.js";
|
|
13
13
|
import { namedPublicationVerdict, readNamedPublication } from "./publication.js";
|
|
14
|
+
import { extensionFailure, loadPostgresSql } from "./extensions.js";
|
|
15
|
+
import { BIG_TABLE_ROWS, intervalOf, syncedText, syncText, syncWords } from "../lib/table-sync.js";
|
|
14
16
|
export function syncModeOf(env) {
|
|
15
17
|
const v = env.MASON_DUCK_SYNC || "poll";
|
|
16
18
|
if (v !== "poll" && v !== "recopy")
|
|
@@ -18,17 +20,22 @@ export function syncModeOf(env) {
|
|
|
18
20
|
return v;
|
|
19
21
|
}
|
|
20
22
|
/**
|
|
21
|
-
*
|
|
22
|
-
* (`
|
|
23
|
-
*
|
|
24
|
-
* the wire: on a test server a 22 M-row table cost the production replica 12 s of scan and 2 s of count at every poll.
|
|
25
|
-
* `mason sync <table>` polls one now (`Sync.pollNow`); `mason start --replication` keeps every table current, whatever
|
|
26
|
-
* its size.
|
|
23
|
+
* WHEN A POLLED TABLE IS POLLED NEXT: its interval after its last poll ended, stretched to four times that poll's own
|
|
24
|
+
* time (`nextPollDelayMs`) — so one table's polls never take more than a fifth of the source's time, whatever its
|
|
25
|
+
* `sync: poll <interval>` says (a 20 s poll of a table set to `poll 5s` waits 80 s).
|
|
27
26
|
*/
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
27
|
+
export function nextPollAt(endedMs, intervalSeconds, pollMs) {
|
|
28
|
+
return endedMs + nextPollDelayMs(intervalSeconds * 1000, pollMs);
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* HOW LONG THE NEXT PASS WAITS: until the first polled table is due — and at least four times what the last pass took
|
|
32
|
+
* (its polls, one after another on the writer), so ALL the tables' polls together never take more than a fifth of the
|
|
33
|
+
* source's time, however many are due at once. Bounded by setTimeout's 2^31 - 1 ms: a later poll is waited for in
|
|
34
|
+
* steps, a pass that finds nothing due polling nothing.
|
|
35
|
+
*/
|
|
36
|
+
export function passDelayMs(earliestDueMs, nowMs, lastPassMs) {
|
|
37
|
+
return Math.min(2 ** 31 - 1, Math.max(0, earliestDueMs - nowMs, 4 * lastPassMs));
|
|
38
|
+
}
|
|
32
39
|
/** a poll asked for that cannot run (`Sync.pollNow`), its reason in the asker's words: `code` is the CLI's failure code */
|
|
33
40
|
export class PollRefused extends Error {
|
|
34
41
|
code;
|
|
@@ -38,6 +45,19 @@ export class PollRefused extends Error {
|
|
|
38
45
|
export class Stopping extends Error {
|
|
39
46
|
constructor() { super("Mason is stopping"); }
|
|
40
47
|
}
|
|
48
|
+
/** How often the slot's WAL is read at most (`streamStatus`): a catalog view's row, on the connection the database's
|
|
49
|
+
* position is read on, so never a connection of its own — and never per poll or per batch. */
|
|
50
|
+
export const SLOT_READ_MS = 30_000;
|
|
51
|
+
/** The slot as `readSlot` read it, as `SlotWal` says it. */
|
|
52
|
+
export function slotWalOf(s, name, at) {
|
|
53
|
+
if (!s.exists)
|
|
54
|
+
return { error: `replication slot ${name} does not exist on the database (the next pass makes it again)` };
|
|
55
|
+
const status = s.walStatus === "reserved" || s.walStatus === "extended" || s.walStatus === "unreserved" || s.walStatus === "lost" ? s.walStatus : null;
|
|
56
|
+
// a lost slot's WAL was freed by Postgres (its restart_lsn is null): it holds none; any other null is not known
|
|
57
|
+
if (s.retainedBytes === null && status !== "lost")
|
|
58
|
+
return { error: `replication slot ${name} has no restart position yet: the WAL it holds is not known` };
|
|
59
|
+
return { walHeldBytes: s.retainedBytes === null ? 0 : Number(s.retainedBytes), walStatus: status, safeWalSizeBytes: s.safeBytes === null ? null : Number(s.safeBytes), at: new Date(at).toISOString() };
|
|
60
|
+
}
|
|
41
61
|
export const ident = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
42
62
|
export const literal = (s) => `'${s.replaceAll("'", "''")}'`;
|
|
43
63
|
export const SRC = "_mason_source";
|
|
@@ -90,7 +110,7 @@ export async function writeStaged(w, t, target, stage, where, kept) {
|
|
|
90
110
|
* never converted in place, but copied again through the normal path (`needsCopy`), serving as it was meanwhile. Found
|
|
91
111
|
* by its constraint (`duckdb_constraints()`), which every copy of that layout carries and no copy of this one does.
|
|
92
112
|
*/
|
|
93
|
-
export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason:
|
|
113
|
+
export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason: read again whole, in heap order";
|
|
94
114
|
/**
|
|
95
115
|
* How long a failed copy waits before it is tried again: four times the attempt, at least 5 minutes, doubled for each
|
|
96
116
|
* failure in a row, at most 6 hours (a copy whose sort failed for good read a production replica whole for
|
|
@@ -115,16 +135,16 @@ export function copyOrder(tables) {
|
|
|
115
135
|
* of a copy that is not coming (the test server: every start said `added <all four tables>: they are copied at the
|
|
116
136
|
* next pass` while all four were copied and streamed from the acknowledged position — read as a long re-copy that never
|
|
117
137
|
* came). ONE line: the tables grouped by state, each group `<n> table(s) <state>: <tables>`, the current ones first,
|
|
118
|
-
* those copied at the next pass last, each with its reason
|
|
138
|
+
* those copied at the next pass last, each with its reason, a current copy with how it is kept so (`keptWords`).
|
|
119
139
|
*/
|
|
120
|
-
export function addedText(tables
|
|
140
|
+
export function addedText(tables) {
|
|
121
141
|
const groups = new Map();
|
|
122
142
|
for (const t of tables) {
|
|
123
|
-
const [rank, state] = "copy" in t ? [3, `
|
|
124
|
-
: "behind" in t ? [2, `already
|
|
125
|
-
: "refused" in t ? [4, `not
|
|
126
|
-
: t.
|
|
127
|
-
: [0, `already
|
|
143
|
+
const [rank, state] = "copy" in t ? [3, `synced at the next pass (${t.copy})`]
|
|
144
|
+
: "behind" in t ? [2, `already synced, caught up by one poll first (${t.behind})`]
|
|
145
|
+
: "refused" in t ? [4, `not synced or streamed (${t.refused})`]
|
|
146
|
+
: t.off ? [1, `already synced, ${t.kept}`]
|
|
147
|
+
: [0, `already synced and ${t.kept}`];
|
|
128
148
|
const g = groups.get(state) ?? { rank, sources: [] };
|
|
129
149
|
g.sources.push(t.source);
|
|
130
150
|
groups.set(state, g);
|
|
@@ -163,8 +183,8 @@ export function copyChunkSecondsOf(env) {
|
|
|
163
183
|
return COPY_CHUNK_SECONDS;
|
|
164
184
|
const n = Number(v);
|
|
165
185
|
if (!/^\d+(\.\d+)?$/.test(v.trim()) || n < COPY_CHUNK_SECONDS_MIN || n > COPY_CHUNK_SECONDS_MAX) {
|
|
166
|
-
throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a
|
|
167
|
-
`about that long to read, and a stop reads at most that much again, so set it lower for a
|
|
186
|
+
throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a first sync's chunks is sized to take ` +
|
|
187
|
+
`about that long to read, and a stop reads at most that much again, so set it lower for a first sync that moves more often or higher for fewer, larger chunks, or unset it for ${COPY_CHUNK_SECONDS}`);
|
|
168
188
|
}
|
|
169
189
|
return n;
|
|
170
190
|
}
|
|
@@ -345,7 +365,7 @@ export function copyParallelOf(env) {
|
|
|
345
365
|
return COPY_PARALLEL;
|
|
346
366
|
const n = Number(v);
|
|
347
367
|
if (!/^\d+$/.test(v.trim()) || n < 1 || n > COPY_PARALLEL_MAX) {
|
|
348
|
-
throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a
|
|
368
|
+
throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a first sync's chunks on a backend of the database's own, ` +
|
|
349
369
|
`so set it to 1 (one at a time, the lightest on the database) up to ${COPY_PARALLEL_MAX}, or unset it for ${COPY_PARALLEL}`);
|
|
350
370
|
}
|
|
351
371
|
return n;
|
|
@@ -384,7 +404,7 @@ const pq = (sql) => `postgres_query(${literal(SRC)}, ${literal(sql)})`;
|
|
|
384
404
|
* differ (`1` beside `1.50`); the copy answers them all with s decimals, also said.
|
|
385
405
|
*/
|
|
386
406
|
const NUMERIC_TYPE = /^numeric(?:\((\d+)(?:,\s*(\d+))?\))?$/;
|
|
387
|
-
const WIDER = "mason: a numeric value wider than its
|
|
407
|
+
const WIDER = "mason: a numeric value wider than its synced column";
|
|
388
408
|
/** a source numeric the extension reads as a DOUBLE: no precision, or more than DuckDB's 38 digits */
|
|
389
409
|
function readsAsDouble(type) {
|
|
390
410
|
const m = NUMERIC_TYPE.exec(type);
|
|
@@ -429,8 +449,20 @@ export class Sync {
|
|
|
429
449
|
mode;
|
|
430
450
|
/** each polled table's watermark: the snapshot of the last poll that stored it */
|
|
431
451
|
snapshots = new Map();
|
|
432
|
-
/** how long the last pass took: the next waits four times that
|
|
452
|
+
/** how long the last pass took: the next waits at least four times that (its polls' share of the source's time) */
|
|
433
453
|
lastPassMs = 0;
|
|
454
|
+
/** each polled table's next poll (ms since the epoch: its last poll's end plus its interval, stretched to four times
|
|
455
|
+
* that poll — `nextPollDelayMs`); absent: from its last poll's time in the ledger (`dueAt`). One pass polls the
|
|
456
|
+
* tables due and the timer waits for the next one due (`nextPassDelay`), never a timer per table: a pass also
|
|
457
|
+
* queues the copies and drops removed tables, and attaches the source once for all of it — timers of their own would
|
|
458
|
+
* each attach it, and race those for the writer's turn */
|
|
459
|
+
due = new Map();
|
|
460
|
+
/** each table's sync as its last poll or copy recorded it (`_mason.poll_state.sync`; null: recorded before syncs were) */
|
|
461
|
+
recordedSync = new Map();
|
|
462
|
+
/** the tables whose sync changed since they were last polled (`setTables`): kept, and caught up by ONE poll now */
|
|
463
|
+
switched = new Set();
|
|
464
|
+
/** how long each table's last poll took, this run (ms): its next waits four times that, and mason add estimates by it */
|
|
465
|
+
lastPollMs = new Map();
|
|
434
466
|
/** each table whose copy failed: the failures in a row, and when it is tried again (`copyRetryDelayMs`) */
|
|
435
467
|
retries = new Map();
|
|
436
468
|
/** each copied table in a layout this Mason no longer keeps (`RETIRED_LAYOUT`), found at `open`: copied again */
|
|
@@ -455,10 +487,17 @@ export class Sync {
|
|
|
455
487
|
attached = false;
|
|
456
488
|
loaded = false;
|
|
457
489
|
stopped = false;
|
|
458
|
-
/** REPLICATION
|
|
459
|
-
* to the
|
|
460
|
-
* runs, the last commit
|
|
490
|
+
/** REPLICATION: the publication and slot the `replication` tables are read through (`owned`: the publication is
|
|
491
|
+
* Mason's own, made and kept equal to the replicated tables; else the operator named it, and it is only read —
|
|
492
|
+
* src/duck/publication.ts), null in a run that cannot stream (the demo's); the stream while it runs, the last commit
|
|
493
|
+
* DuckDB applied */
|
|
461
494
|
replication;
|
|
495
|
+
/** told when the slot is made (true) and when it is dropped, verified gone (false): the run records it, so `mason
|
|
496
|
+
* destroy` drops what was made by its own name (src/cli/dev.ts) */
|
|
497
|
+
onReplication;
|
|
498
|
+
/** the last moment the stream had everything the database wrote (a keepalive with nothing pending, a batch applied):
|
|
499
|
+
* a streamed table's copy is current as of then (`keptOf`'s syncedAt) */
|
|
500
|
+
streamCurrentAt = null;
|
|
462
501
|
stream = null;
|
|
463
502
|
appliedLsn = 0n;
|
|
464
503
|
/** the slot has a row in `_mason.stream_state`: what this copy applied of it is known */
|
|
@@ -472,8 +511,13 @@ export class Sync {
|
|
|
472
511
|
held = new Map();
|
|
473
512
|
/** REPLICATION: what the stream applies, by source — the tables joined, and those held — read at each change */
|
|
474
513
|
streamed = new Map();
|
|
514
|
+
/** the last `StreamedTable.stay` given (`refreshStreamed`) */
|
|
515
|
+
lastStay = 0;
|
|
475
516
|
/** a NAMED publication's gaps: each added table it does not carry, with the refusal (neither copied nor streamed) */
|
|
476
517
|
unpublished = new Map();
|
|
518
|
+
/** the error each table a named publication did not carry was given (`settlePrepared`): cleared by the pass that finds
|
|
519
|
+
* it carried again, and by nothing else that pass */
|
|
520
|
+
gapErrors = new Map();
|
|
477
521
|
/** a named publication that does not publish TRUNCATE has been said (once a run) */
|
|
478
522
|
truncateSaid = false;
|
|
479
523
|
/** the last pass failed: in replication mode it is tried again after the interval, the stream started by it */
|
|
@@ -486,12 +530,16 @@ export class Sync {
|
|
|
486
530
|
* (a pass kicked sooner leaves it stopped) */
|
|
487
531
|
streamFailures = 0;
|
|
488
532
|
streamAt = 0;
|
|
489
|
-
/** when each
|
|
490
|
-
|
|
491
|
-
|
|
533
|
+
/** when each copy was last brought current, and how (`Synced`): seeded at `open` from the ledger, moved by every
|
|
534
|
+
* swap, poll commit and stream batch */
|
|
535
|
+
synced = new Map();
|
|
536
|
+
/** where the sync's failures and notable events go (src/duck/events.ts `SyncEvents.record`): never the writer's turn */
|
|
537
|
+
onEvent;
|
|
538
|
+
/** when each table was last copied or polled, its `syncedAt`: `_mason.poll_state.polled_at` and nothing else — the
|
|
539
|
+
* value `pollStateSql` wrote is the one set here once its transaction commits, and `readPolledAt` only moves a time
|
|
540
|
+
* forward, so nothing moves it back (a time read at the commit stood until that read replaced it: a `syncedAt` that
|
|
541
|
+
* went backwards). A copy's is its watermark's, taken before the swap; `copied_at`, after it, is `Synced.at`'s */
|
|
492
542
|
polledAt = new Map();
|
|
493
|
-
/** the tables over the poll's row limit already said in the log (said once, when first left out) */
|
|
494
|
-
leftOut = new Set();
|
|
495
543
|
/** the store's writer: the stream's batches, the polls, the source's attachment and every swap, one turn at a time */
|
|
496
544
|
writerLane;
|
|
497
545
|
/** the rebuilds' own connection (`open`): a copy or a catch-up, beside the writer's work */
|
|
@@ -505,7 +553,6 @@ export class Sync {
|
|
|
505
553
|
/** the source attached for this many users (a pass, a rebuild): DETACHed when the last ends (`detach`) */
|
|
506
554
|
sourceUsers = 0;
|
|
507
555
|
currentSchema;
|
|
508
|
-
database;
|
|
509
556
|
/** what the catalog last built from: replaced (a new object) only when it changes */
|
|
510
557
|
schemaKey = "";
|
|
511
558
|
/** REPLICATION, WHERE THE STREAM IS (`streamStatus`, the stream's log): the changes it applied this run by table, and
|
|
@@ -540,15 +587,21 @@ export class Sync {
|
|
|
540
587
|
catchingUp = null;
|
|
541
588
|
/** the stream's last failure and when it starts again; cleared when it starts */
|
|
542
589
|
streamFailure = null;
|
|
590
|
+
/** A WHOLE-SYNC FAILURE NOT YET OVER, as its event said it (`table: null`): the stream stopped, over when it next applies
|
|
591
|
+
* a batch; the publication and slot could not be made ready, over when a pass makes them so; the database's attach,
|
|
592
|
+
* over when an attach succeeds. Its end is a `recovered` event of no table, which lets the next such failure be said again
|
|
593
|
+
* (src/duck/events.ts: a failure said word for word within an hour is one event unless it recovered between) */
|
|
594
|
+
wholeFailure = { stream: null, prepare: null, attach: null };
|
|
543
595
|
/** the database's position as last read (`databasePosition`), and the read in flight */
|
|
544
596
|
dbPosition = null;
|
|
597
|
+
/** the slot's WAL as last read (`databasePosition`, at most every SLOT_READ_MS), and when */
|
|
598
|
+
slotWal = null;
|
|
545
599
|
dbPositionRead = null;
|
|
546
600
|
/** THE STORED JSON PATHS (src/duck/kept.ts): every write below computes them; the plans read them */
|
|
547
601
|
kept;
|
|
548
602
|
constructor(store, source, database, log, intervalMs, mode = "poll", replication = null, options = {}) {
|
|
549
603
|
this.store = store;
|
|
550
604
|
this.source = source;
|
|
551
|
-
this.database = database;
|
|
552
605
|
this.log = log;
|
|
553
606
|
this.intervalMs = intervalMs;
|
|
554
607
|
this.mode = mode;
|
|
@@ -556,16 +609,18 @@ export class Sync {
|
|
|
556
609
|
this.copyChunkRows = options.copyChunkRows ?? COPY_CHUNK_ROWS;
|
|
557
610
|
this.copyParallel = options.copyParallel ?? COPY_PARALLEL;
|
|
558
611
|
if (!Number.isInteger(this.copyParallel) || this.copyParallel < 1 || this.copyParallel > COPY_PARALLEL_MAX) {
|
|
559
|
-
throw new Error(`a
|
|
612
|
+
throw new Error(`a first sync reads with 1 to ${COPY_PARALLEL_MAX} readers, not ${this.copyParallel} (Mason bug: report it; copyParallelOf reads MASON_COPY_PARALLEL)`);
|
|
560
613
|
}
|
|
561
614
|
this.copyChunkSeconds = options.copyChunkSeconds ?? COPY_CHUNK_SECONDS;
|
|
562
615
|
if (!(this.copyChunkSeconds > 0))
|
|
563
|
-
throw new Error(`a
|
|
616
|
+
throw new Error(`a first sync's chunk takes a positive number of seconds, not ${this.copyChunkSeconds} (Mason bug: report it; copyChunkSecondsOf reads MASON_COPY_CHUNK_SECONDS)`);
|
|
564
617
|
this.onChunk = options.onChunk;
|
|
618
|
+
this.onEvent = options.onEvent ?? (() => undefined);
|
|
565
619
|
this.diskFree = options.diskFree ?? ((dir) => { const st = fs.statfsSync(dir); return st.bavail * st.bsize; });
|
|
566
620
|
if (mode === "replication" && replication === null)
|
|
567
621
|
throw new Error("replication mode needs its slot and publication (Mason bug: report it)");
|
|
568
622
|
this.replication = replication;
|
|
623
|
+
this.onReplication = options.onReplication;
|
|
569
624
|
this.currentSchema = { database, tables: [] };
|
|
570
625
|
this.writerLane = { conn: store.writer, inFlight: false };
|
|
571
626
|
this.kept = new KeptJsonPaths(store, {
|
|
@@ -583,6 +638,24 @@ export class Sync {
|
|
|
583
638
|
throw new Error("the sync's rebuild connection is not open (Mason bug: report it)");
|
|
584
639
|
return this.rebuilderLane;
|
|
585
640
|
}
|
|
641
|
+
/** an event of the sync, to its record (src/duck/events.ts), which never throws into the sync; a sink that does is
|
|
642
|
+
* said, and the sync goes on (its state and its log line hold what happened) */
|
|
643
|
+
event(e) {
|
|
644
|
+
try {
|
|
645
|
+
this.onEvent(e);
|
|
646
|
+
}
|
|
647
|
+
catch (x) {
|
|
648
|
+
this.log(`the sync's event (${e.kind}${e.table ? ` ${e.table}` : ""}) could not be recorded: ${x instanceof Error ? x.message : String(x)}`);
|
|
649
|
+
}
|
|
650
|
+
}
|
|
651
|
+
/** a whole-sync failure (`wholeFailure`) is over: said once, as a `recovered` event of no table */
|
|
652
|
+
recovered(what, words) {
|
|
653
|
+
const was = this.wholeFailure[what];
|
|
654
|
+
if (was === null)
|
|
655
|
+
return;
|
|
656
|
+
this.wholeFailure[what] = null;
|
|
657
|
+
this.event({ kind: "recovered", table: null, message: `${words} (it had failed: ${was.split("\n")[0]})` });
|
|
658
|
+
}
|
|
586
659
|
/** `f` on the writer's next turn: once every job queued before it has ended, so no other job's statement lands inside
|
|
587
660
|
* its transaction. A job never queues another and waits for it (it would wait for itself) */
|
|
588
661
|
onWriter(f) {
|
|
@@ -597,7 +670,9 @@ export class Sync {
|
|
|
597
670
|
// the re-sort's count of rows written out of key order (in the earlier layout): gone with the key order
|
|
598
671
|
for (const gone of ["written", "counted"])
|
|
599
672
|
await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.copy_state DROP COLUMN IF EXISTS ${gone}`);
|
|
600
|
-
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ)`);
|
|
673
|
+
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ, sync VARCHAR)`);
|
|
674
|
+
// the sync each table was last polled or copied under (src/lib/table-sync.ts syncText): a change is caught up by one poll
|
|
675
|
+
await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.poll_state ADD COLUMN IF NOT EXISTS sync VARCHAR`);
|
|
601
676
|
// replication mode's record: per slot, the end of the last commit applied, in the same transaction as its rows
|
|
602
677
|
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.stream_state (slot VARCHAR PRIMARY KEY, lsn VARCHAR, applied_at TIMESTAMPTZ)`);
|
|
603
678
|
// the tables copied (or under a new slot) and not yet caught up by their one poll (`behind`), written
|
|
@@ -619,13 +694,16 @@ export class Sync {
|
|
|
619
694
|
this.slotKnown = r.length > 0;
|
|
620
695
|
this.appliedLsn = r.length > 0 ? parseLsn(String(r[0][0])) : 0n;
|
|
621
696
|
}
|
|
622
|
-
for (const [source, snapshot] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
|
|
697
|
+
for (const [source, snapshot, sync] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot, sync FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
|
|
623
698
|
this.snapshots.set(source, snapshot);
|
|
699
|
+
this.recordedSync.set(source, sync);
|
|
624
700
|
}
|
|
625
701
|
await this.readPolledAt();
|
|
626
|
-
const r = await this.store.writer.runAndReadAll(`SELECT source, phase, "rows", bytes, seconds, copied_at
|
|
702
|
+
const r = await this.store.writer.runAndReadAll(`SELECT source, phase, "rows", bytes, seconds, epoch_ms(copied_at), error, columns::VARCHAR FROM ${MASON_SCHEMA}.copy_state`);
|
|
627
703
|
for (const row of r.getRowsJS()) {
|
|
628
|
-
const [source, phase, rows, bytes, seconds,
|
|
704
|
+
const [source, phase, rows, bytes, seconds, copiedMs, error, columns] = row;
|
|
705
|
+
// an instant as a run writes it (ISO, UTC), never DuckDB's text of it in the connection's zone
|
|
706
|
+
const copiedAt = copiedMs === null ? null : new Date(Number(copiedMs)).toISOString();
|
|
629
707
|
if (columns !== null) {
|
|
630
708
|
const c = JSON.parse(columns);
|
|
631
709
|
this.copiedAs.set(source, columnsKey(c));
|
|
@@ -633,6 +711,7 @@ export class Sync {
|
|
|
633
711
|
}
|
|
634
712
|
this.states.set(source, { source, phase: phase === "copying" ? "waiting" : phase, rows: rows === null ? null : Number(rows), bytes: bytes === null ? null : Number(bytes), seconds, copiedAt, error });
|
|
635
713
|
}
|
|
714
|
+
await this.readSynced();
|
|
636
715
|
await this.findRetired();
|
|
637
716
|
this.removeStageLeftovers();
|
|
638
717
|
// the stored JSON paths, before the catalog serves anything: each served copy's view, what the file holds
|
|
@@ -644,6 +723,8 @@ export class Sync {
|
|
|
644
723
|
* here and current (said as its records say it, `addedText`), a removed one dropped.
|
|
645
724
|
* What it returns is the catalog rebuilt over them (never rejected: a failure is said in the log) */
|
|
646
725
|
setTables(tables) {
|
|
726
|
+
// (called at mason start and at mason add / remove only: the catalog's keys and constraints are the source's as of
|
|
727
|
+
// then — docs/DESIGN.md "A count of distinct keys")
|
|
647
728
|
const before = this.tables;
|
|
648
729
|
this.tables = new Map(tables.map((t) => [t.source, t]));
|
|
649
730
|
// said as it happens: what is added, each table as its records say it is (`addedText`), and what is no longer
|
|
@@ -651,11 +732,51 @@ export class Sync {
|
|
|
651
732
|
const added = tables.filter((t) => !before.has(t.source));
|
|
652
733
|
const removed = [...before.keys()].filter((s) => !this.tables.has(s));
|
|
653
734
|
if (added.length)
|
|
654
|
-
this.log(addedText(added.map((t) => this.addedStatus(t))
|
|
735
|
+
this.log(addedText(added.map((t) => this.addedStatus(t))));
|
|
736
|
+
// A TABLE WHOSE SYNC CHANGED — by mason add --sync now, or by hand since its last poll (the ledger's record) — keeps
|
|
737
|
+
// its copy and is caught up by ONE poll now, whatever it changed to, then kept as it says: never copied again
|
|
738
|
+
for (const t of tables) {
|
|
739
|
+
const now = syncText(this.syncOf(t)), prev = before.get(t.source);
|
|
740
|
+
const was = prev !== undefined ? syncText(this.syncOf(prev)) : this.recordedSync.get(t.source) ?? null;
|
|
741
|
+
if (was === null || was === now)
|
|
742
|
+
continue;
|
|
743
|
+
this.due.delete(t.source);
|
|
744
|
+
// recopy copies every table again whatever its sync, and a table whose columns changed too is copied again with
|
|
745
|
+
// them, its new sync recorded at that copy's swap
|
|
746
|
+
const copied = this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source) && this.copiedAs.get(t.source) === columnsKey(t.columns);
|
|
747
|
+
if (this.mode === "recopy" || !copied)
|
|
748
|
+
continue;
|
|
749
|
+
if (this.replicated(t)) {
|
|
750
|
+
// INTO THE STREAM (poll or off → replication): caught up by ONE poll with the stream's changes of it held from
|
|
751
|
+
// just before that poll (`catchUp`), as a slot new to the copy is — once the pass has the publication carry it
|
|
752
|
+
// (`prepareReplication` runs before the pass queues the catch-up). Marked in `_mason.catch_up`, on the writer's
|
|
753
|
+
// turn and so before the catch-up starts: the stream acknowledges what it holds, so a run that dies between
|
|
754
|
+
// the catch-up's poll and the held changes' apply must catch it up again (its recorded sync is already the new)
|
|
755
|
+
this.switched.delete(t.source);
|
|
756
|
+
const why = `after its sync changed from ${was}`;
|
|
757
|
+
if (!this.behind.has(t.source)) {
|
|
758
|
+
const mark = { why };
|
|
759
|
+
this.behind.set(t.source, mark);
|
|
760
|
+
void this.onWriter(() => this.store.writer.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(why)}, now())`)).catch((e) => {
|
|
761
|
+
// its next start catches it up again only by this row: said, and the mark kept for this run's own catch-up
|
|
762
|
+
this.log(`${t.source}: its catch-up could not be recorded (a stop before it ends catches it up again only if its sync changes): ${e instanceof Error ? e.message : String(e)}`);
|
|
763
|
+
});
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
else {
|
|
767
|
+
// OUT OF THE STREAM, or between off and poll: it leaves the stream at its next batch (`refreshStreamed`) and the
|
|
768
|
+
// next pass polls it from its watermark, which every change since is at or past
|
|
769
|
+
this.switched.add(t.source);
|
|
770
|
+
}
|
|
771
|
+
// said, and recorded as the sync's event (src/duck/events.ts): what a table's sync was is history the log alone loses
|
|
772
|
+
const switched = `sync: ${was} → ${now}: its synced data is kept, and caught up by one poll now${this.replicated(t) ? ", then streamed" : ""}`;
|
|
773
|
+
this.log(`${t.source}: ${switched}`);
|
|
774
|
+
this.event({ kind: "sync_changed", table: t.source, message: switched });
|
|
775
|
+
}
|
|
655
776
|
// loud: the copy goes with the table's entry, and comes back only by reading the source again (the test server: a 53 GB
|
|
656
777
|
// copy, 21 minutes of a production replica, dropped by a project file moved aside)
|
|
657
778
|
if (removed.length)
|
|
658
|
-
this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its
|
|
779
|
+
this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its synced data is" : "their synced data is"} DROPPED at the next pass — mason add syncs ${removed.length === 1 ? "it" : "them"} again from the source, whole`);
|
|
659
780
|
for (const t of tables)
|
|
660
781
|
if (!this.states.has(t.source))
|
|
661
782
|
this.states.set(t.source, { source: t.source, phase: "waiting", rows: null, bytes: t.bytes, seconds: null, copiedAt: null, error: null });
|
|
@@ -680,27 +801,107 @@ export class Sync {
|
|
|
680
801
|
// a named publication's gap is said again by the pass, which refuses the table by name until its owner adds it
|
|
681
802
|
if (st?.error?.startsWith("replication: "))
|
|
682
803
|
return { source: t.source, refused: firstLine(st.error.slice("replication: ".length)) };
|
|
683
|
-
const copy = st === undefined ? "never
|
|
684
|
-
: st.phase === "failed" ? `its
|
|
685
|
-
: st.phase !== "copied" ? "its
|
|
686
|
-
: this.retired.has(t.source) ? "
|
|
687
|
-
: this.
|
|
688
|
-
: !this.snapshots.has(t.source) ? "a numeric value outgrew its copy's DECIMAL"
|
|
689
|
-
: null;
|
|
804
|
+
const copy = st === undefined ? "never synced"
|
|
805
|
+
: st.phase === "failed" ? `its first sync failed: ${firstLine(st.error)}`
|
|
806
|
+
: st.phase !== "copied" ? "its first sync has not finished"
|
|
807
|
+
: this.retired.has(t.source) ? "it is in an older Mason's layout, key order with an index"
|
|
808
|
+
: this.readAgainWhy(t);
|
|
690
809
|
if (copy !== null)
|
|
691
810
|
return { source: t.source, copy };
|
|
692
811
|
const b = this.behind.get(t.source);
|
|
693
812
|
if (b !== undefined)
|
|
694
813
|
return { source: t.source, behind: b.why };
|
|
695
|
-
if (this.replication !== null && !this.slotKnown)
|
|
696
|
-
return { source: t.source, behind: `slot ${this.replication.slot} is new to
|
|
697
|
-
|
|
814
|
+
if (this.replicated(t) && this.replication !== null && !this.slotKnown)
|
|
815
|
+
return { source: t.source, behind: `slot ${this.replication.slot} is new to the synced data` };
|
|
816
|
+
const off = this.mode !== "recopy" && this.syncOf(t).mode === "off";
|
|
817
|
+
return { source: t.source, kept: this.keptWords(t), off };
|
|
698
818
|
}
|
|
699
|
-
/** how a copy that is current is kept so, in this run's mode: the start line's words (`addedText`) */
|
|
700
|
-
keptWords() {
|
|
819
|
+
/** how a copy that is current is kept so, in this run's mode and by its sync: the start line's words (`addedText`) */
|
|
820
|
+
keptWords(t) {
|
|
701
821
|
const every = `${Number((this.intervalMs / 1000).toFixed(1))} s`;
|
|
702
|
-
|
|
703
|
-
|
|
822
|
+
const s = this.syncOf(t);
|
|
823
|
+
return this.mode === "recopy" ? `read again whole every ${every}`
|
|
824
|
+
: s.mode === "replication" ? `streamed from ${formatLsn(this.appliedLsn)}`
|
|
825
|
+
: s.mode === "off" ? `sync: off (read again only by mason sync ${t.source})` : syncWords(s);
|
|
826
|
+
}
|
|
827
|
+
/** how a table is kept current, as the sources file says (`AddedTable.sync`); a table with none — a test's — polled at
|
|
828
|
+
* the sync's own interval */
|
|
829
|
+
syncOf(t) {
|
|
830
|
+
if (t.sync !== undefined)
|
|
831
|
+
return t.sync;
|
|
832
|
+
return this.mode === "replication" ? { mode: "replication" } : { mode: "poll", every: intervalOf(this.intervalMs / 1000), seconds: this.intervalMs / 1000 };
|
|
833
|
+
}
|
|
834
|
+
/** a table the stream keeps current: its sync is `replication` (never in a run that copies every table again whole) */
|
|
835
|
+
replicated(t) {
|
|
836
|
+
return this.mode !== "recopy" && this.syncOf(t).mode === "replication";
|
|
837
|
+
}
|
|
838
|
+
/** the added tables the stream keeps current, in the project's order */
|
|
839
|
+
replicatedTables() {
|
|
840
|
+
return [...this.tables.values()].filter((t) => this.replicated(t));
|
|
841
|
+
}
|
|
842
|
+
/** THE RUN STREAMS NOW: a table's sync is `replication` (the slot is made, or kept, and the stream runs) */
|
|
843
|
+
streams() {
|
|
844
|
+
return this.replication !== null && this.replicatedTables().length > 0;
|
|
845
|
+
}
|
|
846
|
+
/**
|
|
847
|
+
* WHEN A TABLE'S NEXT POLL IS DUE (ms since the epoch), or null when none is: a table whose sync changed at once
|
|
848
|
+
* (`switched`), unless a failed attempt set its next; an `off` one never; a polled one at `due`, else its last poll's
|
|
849
|
+
* time in the ledger plus its interval (a restart does not poll a table before its interval), else now.
|
|
850
|
+
*/
|
|
851
|
+
dueAt(t) {
|
|
852
|
+
const s = this.syncOf(t), d = this.due.get(t.source);
|
|
853
|
+
if (s.mode !== "poll" && !this.switched.has(t.source))
|
|
854
|
+
return null;
|
|
855
|
+
if (d !== undefined)
|
|
856
|
+
return d;
|
|
857
|
+
if (this.switched.has(t.source) || s.mode !== "poll")
|
|
858
|
+
return 0;
|
|
859
|
+
const at = this.polledAt.get(t.source);
|
|
860
|
+
return at === undefined ? 0 : Date.parse(at) + s.seconds * 1000;
|
|
861
|
+
}
|
|
862
|
+
/** a table polled (by a pass, `mason sync` or its catch-up) in `ms`: its next poll after its interval, stretched to four
|
|
863
|
+
* times that poll (`nextPollDelayMs`), and its sync now the one recorded (the poll wrote it, `poll`) */
|
|
864
|
+
polledNow(t, ms) {
|
|
865
|
+
const s = this.syncOf(t);
|
|
866
|
+
this.switched.delete(t.source);
|
|
867
|
+
this.lastPollMs.set(t.source, ms);
|
|
868
|
+
this.recordedSync.set(t.source, syncText(s));
|
|
869
|
+
if (s.mode === "poll")
|
|
870
|
+
this.due.set(t.source, nextPollAt(Date.now(), s.seconds, ms));
|
|
871
|
+
else
|
|
872
|
+
this.due.delete(t.source);
|
|
873
|
+
}
|
|
874
|
+
/**
|
|
875
|
+
* WHAT THIS RUN MEASURED OF A TABLE, for `mason add`'s estimate of what polling it costs the database
|
|
876
|
+
* (src/cli/source-edit.ts pollCost): its last poll's time, and ONE reader's rate at its copy — the copy reading now,
|
|
877
|
+
* else the last copy's rows over its time and its readers. Null when nothing of it is measured.
|
|
878
|
+
*/
|
|
879
|
+
measured(source) {
|
|
880
|
+
const st = this.states.get(source);
|
|
881
|
+
const live = this.copyLive.get(source)?.rate?.rowsPerSecond ?? null;
|
|
882
|
+
const copied = st?.phase === "copied" && st.rows && st.seconds ? st.rows / st.seconds / this.copyParallel : null;
|
|
883
|
+
const last = this.lastPollMs.get(source) ?? null;
|
|
884
|
+
return last === null && live === null && copied === null ? null : { lastPollMs: last, copyRowsPerSecond: live ?? copied };
|
|
885
|
+
}
|
|
886
|
+
/**
|
|
887
|
+
* HOW LONG UNTIL THE NEXT PASS, in poll mode (null: none until something kicks one — every table `off`, or nothing
|
|
888
|
+
* copied yet): a pass that failed, or a table waiting for a copy nothing queued, after the sync's interval; else when
|
|
889
|
+
* the first polled table is due — and at least four times the last pass took, so the passes' polls together never
|
|
890
|
+
* take more than a fifth of the source's time. `recopy` copies every table again every interval, stretched the same.
|
|
891
|
+
*/
|
|
892
|
+
nextPassDelay() {
|
|
893
|
+
if (this.passFailed || this.waiting())
|
|
894
|
+
return this.intervalMs;
|
|
895
|
+
if (this.mode === "recopy")
|
|
896
|
+
return nextPollDelayMs(this.intervalMs, this.lastPassMs);
|
|
897
|
+
// a table copying, queued, or owed its catch-up is not polled by a pass (its catch-up polls it, then `rearm`): counted
|
|
898
|
+
// as due it would run a pass every few ms that polls nothing and attaches the source each time
|
|
899
|
+
const dues = [...this.tables.values()].filter((t) => this.pollable(t)).flatMap((t) => this.dueAt(t) ?? []);
|
|
900
|
+
return dues.length ? passDelayMs(Math.min(...dues), Date.now(), this.lastPassMs) : null;
|
|
901
|
+
}
|
|
902
|
+
/** a table a pass would poll now if it were due: copied, not queued or rebuilt, and not owed its catch-up */
|
|
903
|
+
pollable(t) {
|
|
904
|
+
return !this.needsCopy(t) && !this.behind.has(t.source) && !this.queued.has(t.source) && this.rebuildingNow?.source !== t.source;
|
|
704
905
|
}
|
|
705
906
|
/** the source's views (src/cli/preflight.ts VIEWS_SQL), served over the copied tables: in the catalog from the next
|
|
706
907
|
* pass. A view over a table that is not copied is left out of it (said in the catalog's log) */
|
|
@@ -715,19 +916,144 @@ export class Sync {
|
|
|
715
916
|
this.snapshots.set(s, "0:0:");
|
|
716
917
|
}
|
|
717
918
|
state() {
|
|
919
|
+
const now = Date.now();
|
|
718
920
|
// in the project's order, whatever order the copies ran and were recorded in (`copyOrder`)
|
|
719
921
|
return [...this.tables.keys()].flatMap((source) => this.states.get(source) ?? [])
|
|
720
922
|
.map((s) => this.retired.has(s.source) ? { ...s, retired: RETIRED_LAYOUT } : s)
|
|
721
923
|
// a copy's chunks as they are NOW: the rows the ones in flight have read by now at the measured rate
|
|
722
924
|
.map((s) => { const l = this.copyLive.get(s.source); return s.phase === "copying" && l !== undefined ? { ...s, progress: copyProgressNow(l, Date.now()) } : s; })
|
|
723
|
-
|
|
925
|
+
// how it is kept current, and a table the passes leave as copied, from ONE derivation (`keptOf`)
|
|
926
|
+
.map((s) => {
|
|
927
|
+
const t = this.tables.get(s.source);
|
|
928
|
+
if (t === undefined)
|
|
929
|
+
return s;
|
|
930
|
+
const kept = this.keptOf(t, now);
|
|
931
|
+
return { ...s, kept };
|
|
932
|
+
})
|
|
933
|
+
// when it was last brought current, whether it waits for its catch-up, and whether it is being rebuilt now
|
|
934
|
+
.map((s) => {
|
|
935
|
+
const synced = this.synced.get(s.source), behind = this.behind.get(s.source)?.why;
|
|
936
|
+
const rebuilding = this.rebuildingNow?.source === s.source ? this.rebuildingNow.kind : undefined;
|
|
937
|
+
return { ...s, ...(synced ? { synced } : {}), ...(behind !== undefined ? { behind } : {}), ...(rebuilding ? { rebuilding } : {}) };
|
|
938
|
+
});
|
|
939
|
+
}
|
|
940
|
+
/**
|
|
941
|
+
* HOW AN ADDED TABLE IS KEPT CURRENT, NOW (`KeptCurrent`): the run's mode, then in replication mode its part in the
|
|
942
|
+
* stream (`streamTable`, what `streamStatus` says of it) and where the stream is; in poll mode its sync, its last copy
|
|
943
|
+
* or poll and its next. The one place it is derived: `state` carries it to `mason status` and the listener's EXPLAIN
|
|
944
|
+
* alike.
|
|
945
|
+
*/
|
|
946
|
+
keptOf(t, now) {
|
|
947
|
+
const st = this.states.get(t.source);
|
|
948
|
+
const sync = this.syncOf(t);
|
|
949
|
+
if (this.mode !== "recopy" && sync.mode !== "replication") {
|
|
950
|
+
const polledAt = this.polledAt.get(t.source) ?? null;
|
|
951
|
+
// a copy the passes do not keep current now, and why, from what a poll of it asks (`pollable`): not copied (waiting,
|
|
952
|
+
// being copied, its copy failed: `needsCopy` holds for every phase but copied), to be copied again (its columns
|
|
953
|
+
// changed, a numeric outgrew it, a retired layout), caught up first, or its rebuild not ended yet: the moment this
|
|
954
|
+
// says nothing is pending, `mason sync` of it runs
|
|
955
|
+
const pending = this.pollable(t) ? null : this.notStreamedWhy(t.source);
|
|
956
|
+
const next = pending === null ? this.dueAt(t) : null;
|
|
957
|
+
return { by: "poll", sync, syncedAt: polledAt, nextAt: next === null ? null : new Date(Math.max(next, now)).toISOString(), ...(pending !== null ? { pending } : {}) };
|
|
958
|
+
}
|
|
959
|
+
// copied again whole every pass, whatever its watermark: only a table not copied now (waiting, being copied, its copy
|
|
960
|
+
// failed) is not kept current
|
|
961
|
+
if (this.mode === "recopy")
|
|
962
|
+
return { by: "recopy", copiedAt: st?.copiedAt ?? null, ...(st?.phase !== "copied" ? { pending: this.notStreamedWhy(t.source) } : {}) };
|
|
963
|
+
const s = this.stream;
|
|
964
|
+
const part = this.streamTable(t, now, s);
|
|
965
|
+
// current as of its last copy or poll, or — streamed — the last moment the stream had all the database wrote
|
|
966
|
+
const polled = this.polledAt.get(t.source) ?? null;
|
|
967
|
+
const streamed = part.state === "streamed" && this.streamCurrentAt !== null ? new Date(this.streamCurrentAt).toISOString() : null;
|
|
968
|
+
const syncedAt = streamed !== null && (polled === null || streamed > polled) ? streamed : polled;
|
|
969
|
+
return { by: "replication", slot: this.replication?.slot ?? "", state: part.state, why: part.why, position: formatLsn(s?.position() ?? this.appliedLsn),
|
|
970
|
+
lastCommit: this.lastBatch === null ? null : new Date(this.lastBatch.committedAt).toISOString(), notRunning: this.streamDownWhy(s, st), syncedAt };
|
|
971
|
+
}
|
|
972
|
+
/** why the stream is not running, in the reader's words (null: it runs): its last failure and when it starts again; the
|
|
973
|
+
* replication check a pass failed (`prepareReplication`, recorded on each copy's state); a slot not made yet; a sync
|
|
974
|
+
* stopping; else it starts at the next pass */
|
|
975
|
+
streamDownWhy(s, st) {
|
|
976
|
+
if (s !== null)
|
|
977
|
+
return null;
|
|
978
|
+
const firstLine = (e) => e.replace(/^replication: /, "").split("\n")[0];
|
|
979
|
+
if (this.streamFailure !== null)
|
|
980
|
+
return `${firstLine(this.streamFailure.error)} · it starts again after ${clock(this.streamFailure.retryAt)} UTC`;
|
|
981
|
+
if (st?.error?.startsWith("replication: "))
|
|
982
|
+
return firstLine(st.error);
|
|
983
|
+
if (this.replication === null)
|
|
984
|
+
return "this run reads no slot, so nothing streams it (the demo database has no change log): give its entry sync: poll <interval>";
|
|
985
|
+
if (!this.slotKnown)
|
|
986
|
+
return `slot ${this.replication.slot} is not made yet: it starts once it is`;
|
|
987
|
+
if (this.stopped)
|
|
988
|
+
return "Mason is stopping";
|
|
989
|
+
return "it has not started yet: it starts at the next pass";
|
|
990
|
+
}
|
|
991
|
+
/** an added table's part in the stream (`StreamTableStatus` but what it applied): streamed, being caught up, waiting
|
|
992
|
+
* for its copy, or left out of the publication, and why — what `streamStatus` and `keptOf` both say, each of the
|
|
993
|
+
* stream `s` it read once */
|
|
994
|
+
streamTable(t, now, s) {
|
|
995
|
+
const iso = (ms) => new Date(ms).toISOString();
|
|
996
|
+
if (this.unpublished.has(t.source))
|
|
997
|
+
return { state: "left-out", why: this.notStreamedWhy(t.source) };
|
|
998
|
+
const b = this.behind.get(t.source);
|
|
999
|
+
if (!this.needsCopy(t) && b !== undefined) {
|
|
1000
|
+
const c = this.catchingUp?.source === t.source ? this.catchingUp : null;
|
|
1001
|
+
const held = this.held.get(t.source)?.counts;
|
|
1002
|
+
const at = this.retries.get(t.source)?.at ?? 0;
|
|
1003
|
+
return { state: "catching-up", why: `caught up by one poll ${b.why}${at > now ? ` — its last catch-up failed, tried again after ${clock(at)} UTC` : ""}`,
|
|
1004
|
+
catchUp: { phase: c?.phase ?? "queued", from: this.polledAt.get(t.source) ?? null, startedAt: c ? iso(c.startedAt) : null, held: held ? held.inserts + held.updates + held.deletes + held.truncates : 0,
|
|
1005
|
+
retryAt: c === null && at > now ? iso(at) : null } };
|
|
1006
|
+
}
|
|
1007
|
+
if (this.joinedNow.has(t.source) || (s === null && !this.needsCopy(t)))
|
|
1008
|
+
return { state: "streamed", why: null };
|
|
1009
|
+
return { state: "waiting", why: this.notStreamedWhy(t.source) };
|
|
724
1010
|
}
|
|
725
|
-
/**
|
|
1011
|
+
/**
|
|
1012
|
+
* Each table's last copy or poll, from the ledger (a copy writes its watermark there too), ONLY EVER FORWARD: this
|
|
1013
|
+
* SELECT's snapshot may be older than a commit on the rebuilds' connection whose time is set while it is answered
|
|
1014
|
+
* (`committed`, `swap`), so what it read never replaces a later time, and a table it does not hold is dropped (removed:
|
|
1015
|
+
* nothing else drops it) only when no such commit gave it a time meanwhile.
|
|
1016
|
+
*/
|
|
726
1017
|
async readPolledAt() {
|
|
1018
|
+
const asked = new Map(this.polledAt);
|
|
727
1019
|
const r = await this.store.writer.runAndReadAll(`SELECT source, epoch_ms(polled_at) FROM ${MASON_SCHEMA}.poll_state WHERE polled_at IS NOT NULL`);
|
|
728
|
-
|
|
729
|
-
for (const [source,
|
|
730
|
-
|
|
1020
|
+
const ledger = new Map(r.getRowsJS().map(([source, ms]) => [source, new Date(Number(ms)).toISOString()]));
|
|
1021
|
+
for (const [source, at] of this.polledAt)
|
|
1022
|
+
if (!ledger.has(source) && at === asked.get(source))
|
|
1023
|
+
this.polledAt.delete(source);
|
|
1024
|
+
for (const [source, at] of ledger) {
|
|
1025
|
+
const now = this.polledAt.get(source);
|
|
1026
|
+
if (now === undefined || at > now)
|
|
1027
|
+
this.polledAt.set(source, at);
|
|
1028
|
+
}
|
|
1029
|
+
}
|
|
1030
|
+
/**
|
|
1031
|
+
* EACH COPY'S LAST SYNC AS THE LEDGER KEEPS IT, at a start (`Synced`): the swap and every poll write
|
|
1032
|
+
* `poll_state.polled_at` in their own transaction, and a copy's `copied_at` is recorded right after its swap — so a
|
|
1033
|
+
* copy recorded at or after its table's `polled_at` was last brought current by that copy (its time the copy's
|
|
1034
|
+
* `seconds`), else by a poll (its time unknown: null). In replication mode the stream's last batch
|
|
1035
|
+
* (`stream_state.applied_at`) is newer for a table it streamed; a table not caught up is left at its copy, and one the
|
|
1036
|
+
* named publication does not carry (`unpublished`: never streamed, so the stream's time is not its own) at its copy
|
|
1037
|
+
* or poll. `only`: those tables alone, read again (the pass that found them out of the publication), the others'
|
|
1038
|
+
* last sync kept as it is.
|
|
1039
|
+
*/
|
|
1040
|
+
async readSynced(only) {
|
|
1041
|
+
const r = await this.store.writer.runAndReadAll(`SELECT p.source, epoch_ms(p.polled_at), epoch_ms(c.copied_at), c.seconds FROM ${MASON_SCHEMA}.poll_state p ` +
|
|
1042
|
+
`LEFT JOIN ${MASON_SCHEMA}.copy_state c USING (source) WHERE p.polled_at IS NOT NULL`);
|
|
1043
|
+
const streamed = this.replication === null ? [] : (await this.store.writer.runAndReadAll(`SELECT epoch_ms(applied_at) FROM ${MASON_SCHEMA}.stream_state ` +
|
|
1044
|
+
`WHERE slot = $1 AND applied_at IS NOT NULL`, [this.replication.slot])).getRowsJS();
|
|
1045
|
+
const streamMs = streamed.length ? Number(streamed[0][0]) : null;
|
|
1046
|
+
if (only === undefined)
|
|
1047
|
+
this.synced.clear();
|
|
1048
|
+
for (const [source, polled, copied, seconds] of r.getRowsJS()) {
|
|
1049
|
+
if (only !== undefined && !only.has(source))
|
|
1050
|
+
continue;
|
|
1051
|
+
const at = Number(polled);
|
|
1052
|
+
const s = copied !== null && Number(copied) >= at ? { at: new Date(Number(copied)).toISOString(), seconds, by: "copy" }
|
|
1053
|
+
: { at: new Date(at).toISOString(), seconds: null, by: "poll" };
|
|
1054
|
+
const streamedIt = !this.behind.has(source) && !this.unpublished.has(source);
|
|
1055
|
+
this.synced.set(source, streamMs !== null && streamMs > Date.parse(s.at) && streamedIt ? { at: new Date(streamMs).toISOString(), seconds: null, by: "stream" } : s);
|
|
1056
|
+
}
|
|
731
1057
|
}
|
|
732
1058
|
schema() { return this.currentSchema; }
|
|
733
1059
|
start() { this.kick(); }
|
|
@@ -750,13 +1076,12 @@ export class Sync {
|
|
|
750
1076
|
this.running = null;
|
|
751
1077
|
if (this.stopped)
|
|
752
1078
|
return;
|
|
753
|
-
//
|
|
754
|
-
// or after the interval, when it failed or a table waits for a copy
|
|
755
|
-
// failed waits its turn, `copyRetryDelayMs`)
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
this.timer = setTimeout(() => this.kick(), this.intervalMs);
|
|
1079
|
+
// when the next polled table is due (`nextPassDelay`) — the stream keeps a replicated table current between passes,
|
|
1080
|
+
// and a pass runs for it when the tables change — or after the interval, when it failed or a table waits for a copy
|
|
1081
|
+
// or catch-up nothing has queued (a copy that failed waits its turn, `copyRetryDelayMs`)
|
|
1082
|
+
const d = this.nextPassDelay();
|
|
1083
|
+
if (d !== null)
|
|
1084
|
+
this.timer = setTimeout(() => this.kick(), d);
|
|
760
1085
|
});
|
|
761
1086
|
}
|
|
762
1087
|
/** an added table needs a copy or a catch-up that is neither queued nor running: a later pass must queue it */
|
|
@@ -799,19 +1124,42 @@ export class Sync {
|
|
|
799
1124
|
*/
|
|
800
1125
|
followUp(source) {
|
|
801
1126
|
const t = this.tables.get(source);
|
|
802
|
-
if (t === undefined || this.stopped || this.unpublished.has(source)
|
|
1127
|
+
if (t === undefined || this.stopped || this.unpublished.has(source))
|
|
1128
|
+
return;
|
|
1129
|
+
// copied and caught up: polled from now on, so the next pass is timed again (`rearm`)
|
|
1130
|
+
if (!this.needsCopy(t) && !this.behind.has(source)) {
|
|
1131
|
+
this.rearm();
|
|
803
1132
|
return;
|
|
1133
|
+
}
|
|
804
1134
|
const at = this.retries.get(source)?.at ?? 0;
|
|
805
1135
|
if (at > Date.now())
|
|
806
1136
|
this.wakeAt(at);
|
|
807
1137
|
else
|
|
808
1138
|
this.kick();
|
|
809
1139
|
}
|
|
810
|
-
/**
|
|
811
|
-
*
|
|
1140
|
+
/**
|
|
1141
|
+
* THE NEXT PASS TIMED AGAIN, in poll mode, once a table is copied and caught up: a pass that ended while every table
|
|
1142
|
+
* still copied found none due (`nextPassDelay` skips a table that needs its copy) and set no timer, so without this a
|
|
1143
|
+
* table's polls never began. A pass running sets its own when it ends, from the state then.
|
|
1144
|
+
*/
|
|
1145
|
+
rearm() {
|
|
1146
|
+
if (this.stopped || this.running !== null)
|
|
1147
|
+
return;
|
|
1148
|
+
const d = this.nextPassDelay();
|
|
1149
|
+
if (d === null)
|
|
1150
|
+
return;
|
|
1151
|
+
if (this.timer !== null)
|
|
1152
|
+
clearTimeout(this.timer);
|
|
1153
|
+
this.timer = setTimeout(() => this.kick(), d);
|
|
1154
|
+
}
|
|
1155
|
+
/** one pass, now, every polled table taken as due (an `off` one is not: it is polled only when asked), and its end
|
|
1156
|
+
* awaited (the one running first, if any), and the copies and catch-ups it queued: what a test waits on */
|
|
812
1157
|
async once() {
|
|
813
1158
|
while (this.running !== null)
|
|
814
1159
|
await this.running;
|
|
1160
|
+
for (const t of this.tables.values())
|
|
1161
|
+
if (this.syncOf(t).mode === "poll")
|
|
1162
|
+
this.due.set(t.source, 0);
|
|
815
1163
|
this.kick();
|
|
816
1164
|
if (this.running !== null)
|
|
817
1165
|
await this.running;
|
|
@@ -819,34 +1167,43 @@ export class Sync {
|
|
|
819
1167
|
await this.rebuilding;
|
|
820
1168
|
}
|
|
821
1169
|
/**
|
|
822
|
-
* A POLL OF EXACTLY THESE TABLES, NOW, whatever their
|
|
823
|
-
*
|
|
824
|
-
*
|
|
825
|
-
*
|
|
1170
|
+
* A POLL OF EXACTLY THESE TABLES, NOW, whatever their sync says (`mason sync`: an `off` table is read again only so):
|
|
1171
|
+
* a pass of its own, queued behind the one running, in one source snapshot as every poll is; a polled table's next
|
|
1172
|
+
* poll waits its interval from this one. A replicated table is not polled: its answer is where the stream is
|
|
1173
|
+
* (`streamed`), beside the polled ones'. Refused by name (`PollRefused`) for a table not added or not copied yet, and in a run that copies
|
|
1174
|
+
* every table again whole. Its failure is its caller's to say: the copies serve as they were, and the passes go on.
|
|
826
1175
|
*/
|
|
827
1176
|
async pollNow(sources) {
|
|
828
|
-
if (this.mode
|
|
829
|
-
throw new PollRefused("refused",
|
|
830
|
-
: "this run copies every table again whole at each pass (MASON_DUCK_SYNC=recopy), so there is nothing to poll");
|
|
831
|
-
}
|
|
1177
|
+
if (this.mode === "recopy")
|
|
1178
|
+
throw new PollRefused("refused", "this run reads every table again whole at each pass (MASON_DUCK_SYNC=recopy), so there is nothing to poll");
|
|
832
1179
|
const tables = sources.map((s) => {
|
|
833
1180
|
const t = this.tables.get(s);
|
|
834
1181
|
if (t === undefined)
|
|
835
1182
|
throw new PollRefused("not_found", `${s} is not added (mason add adds it)${didYouMean(s, [...this.tables.keys()])}`);
|
|
836
1183
|
return t;
|
|
837
1184
|
});
|
|
1185
|
+
// a replicated table: the stream keeps it current, so nothing to poll — its answer is where the stream is, and its part
|
|
1186
|
+
const said = () => {
|
|
1187
|
+
const now = Date.now(), position = formatLsn(this.stream?.position() ?? this.appliedLsn);
|
|
1188
|
+
return tables.filter((t) => this.replicated(t)).map((t) => { const p = this.streamTable(t, now, this.stream); return { table: t.source, position, state: p.state, why: p.why }; });
|
|
1189
|
+
};
|
|
1190
|
+
const polled = tables.filter((t) => !this.replicated(t));
|
|
1191
|
+
if (polled.length === 0)
|
|
1192
|
+
return { at: new Date().toISOString(), tables: [], streamed: said() };
|
|
838
1193
|
while (this.running !== null)
|
|
839
1194
|
await this.running;
|
|
840
1195
|
if (this.stopped)
|
|
841
1196
|
throw new Stopping();
|
|
842
1197
|
// asked after the pass ahead of it: that pass may have copied it, or failed its copy
|
|
843
|
-
for (const t of
|
|
1198
|
+
for (const t of polled) {
|
|
844
1199
|
const st = this.states.get(t.source);
|
|
845
|
-
if (st?.phase !== "copied"
|
|
846
|
-
throw new PollRefused("nothing_yet", `${t.source} is not
|
|
847
|
-
|
|
1200
|
+
if (st?.phase !== "copied")
|
|
1201
|
+
throw new PollRefused("nothing_yet", `${t.source} is not synced yet (${st?.phase === "failed" ? `its first sync failed: ${st.error ?? "mason logs says why"}` : "its first sync has not finished"}): mason wait waits for it`);
|
|
1202
|
+
const again = this.readAgainWhy(t);
|
|
1203
|
+
if (again !== null)
|
|
1204
|
+
throw new PollRefused("nothing_yet", `${t.source} is being read again whole (${again}): mason wait waits for it`);
|
|
848
1205
|
if (this.retired.has(t.source))
|
|
849
|
-
throw new PollRefused("nothing_yet", `${t.source} is ${RETIRED_LAYOUT} (its
|
|
1206
|
+
throw new PollRefused("nothing_yet", `${t.source} is ${RETIRED_LAYOUT} (its synced data serves as it was): it is current once read again — mason wait waits for it`);
|
|
850
1207
|
this.refuseRebuilding(t);
|
|
851
1208
|
}
|
|
852
1209
|
const poll = (async () => {
|
|
@@ -854,8 +1211,12 @@ export class Sync {
|
|
|
854
1211
|
try {
|
|
855
1212
|
// a transaction per table, as the passes poll; a table whose rebuild started since it was asked is refused
|
|
856
1213
|
const found = [];
|
|
857
|
-
for (const t of
|
|
858
|
-
|
|
1214
|
+
for (const t of polled) {
|
|
1215
|
+
const f = await this.onWriter(async () => { this.refuseRebuilding(t); return this.poll(this.writerLane, [t]); });
|
|
1216
|
+
for (const x of f)
|
|
1217
|
+
this.polledNow(t, x.ms);
|
|
1218
|
+
found.push(...f);
|
|
1219
|
+
}
|
|
859
1220
|
return found;
|
|
860
1221
|
}
|
|
861
1222
|
finally {
|
|
@@ -863,12 +1224,21 @@ export class Sync {
|
|
|
863
1224
|
}
|
|
864
1225
|
})();
|
|
865
1226
|
// the pass slot is held until it ends (kick and once queue behind it); its failure reaches the caller through `poll`
|
|
866
|
-
|
|
1227
|
+
// and the next pass timed again as a pass's end times it (`rearm`): a copy that ended meanwhile found the slot held,
|
|
1228
|
+
// and a polled table's next poll moved past the timer set before
|
|
1229
|
+
this.running = poll.then(() => undefined, () => undefined).finally(() => { this.running = null; this.rearm(); });
|
|
867
1230
|
const found = await poll;
|
|
868
|
-
|
|
1231
|
+
// A SYNC BY HAND IS AN EVENT (src/duck/events.ts `synced`), in the words mason sync and the page's Sync now say:
|
|
1232
|
+
// a scheduled poll that succeeds records nothing (one every few seconds would bury the failures), so without it a
|
|
1233
|
+
// person who synced and then read the events found only an older copy, and read it as a sync lost. A replicated
|
|
1234
|
+
// table asked for is answered with the stream's position and records none: nothing was read for it
|
|
1235
|
+
for (const x of found)
|
|
1236
|
+
this.event({ kind: "synced", table: x.table, message: `synced by hand: ${syncedText(x)}` });
|
|
1237
|
+
const missing = polled.filter((t) => !found.some((f) => f.table === t.source));
|
|
869
1238
|
if (missing.length)
|
|
870
|
-
throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its
|
|
871
|
-
|
|
1239
|
+
throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass, and current once read`);
|
|
1240
|
+
const streamed = said();
|
|
1241
|
+
return { at: new Date().toISOString(), tables: found, ...(streamed.length ? { streamed } : {}) };
|
|
872
1242
|
}
|
|
873
1243
|
/**
|
|
874
1244
|
* The sync stopped: no pass starts, and the one running ends at once — its long statement (a copy's scan, a poll's
|
|
@@ -916,8 +1286,8 @@ export class Sync {
|
|
|
916
1286
|
const now = this.rebuildingNow?.source === t.source ? this.rebuildingNow.kind : null;
|
|
917
1287
|
if (now === null && !this.behind.has(t.source))
|
|
918
1288
|
return;
|
|
919
|
-
const doing = now === "copy" ? "being
|
|
920
|
-
throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its
|
|
1289
|
+
const doing = now === "copy" ? "being read again whole" : `being caught up by one poll ${this.behind.get(t.source)?.why ?? ""}`.trimEnd();
|
|
1290
|
+
throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its synced data serves as it was): it is caught up by one poll when that ends, so there is nothing to poll — mason wait waits for it`);
|
|
921
1291
|
}
|
|
922
1292
|
/** the source's connection, as the sync's own reads and replication make it (never DuckDB's: that is `attach`) */
|
|
923
1293
|
pgOptions() {
|
|
@@ -941,7 +1311,7 @@ export class Sync {
|
|
|
941
1311
|
return columns === undefined ? t : { ...t, columns };
|
|
942
1312
|
}
|
|
943
1313
|
/**
|
|
944
|
-
* REPLICATION, BEFORE A PASS COPIES: the publication carries every
|
|
1314
|
+
* REPLICATION, BEFORE A PASS COPIES: the publication carries every replicated table, and the slot exists — so a table
|
|
945
1315
|
* copied now has its changes in the stream from before it joins (src/duck/stream.ts "why replaying is safe"). A slot
|
|
946
1316
|
* made now, or one this copy has no record of, carries nothing from before it: the tables already copied are caught
|
|
947
1317
|
* up by one poll each before the stream applies their changes (`behind`, recorded with the slot in one transaction).
|
|
@@ -950,7 +1320,7 @@ export class Sync {
|
|
|
950
1320
|
*/
|
|
951
1321
|
async prepareReplication() {
|
|
952
1322
|
const r = this.replication;
|
|
953
|
-
const tables =
|
|
1323
|
+
const tables = this.replicatedTables();
|
|
954
1324
|
const pg = await PgConnection.connect(this.pgOptions());
|
|
955
1325
|
try {
|
|
956
1326
|
const identities = new Map((await pg.query(`SELECT n.nspname || '.' || c.relname, c.relreplident FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ` +
|
|
@@ -960,7 +1330,8 @@ export class Sync {
|
|
|
960
1330
|
const nothing = tables.filter((t) => identities.get(t.source) === "n").map((t) => t.source);
|
|
961
1331
|
if (nothing.length) {
|
|
962
1332
|
throw new Error(`${nothing.join(", ")} ${nothing.length === 1 ? "has" : "have"} REPLICA IDENTITY NOTHING: a publication carrying ${nothing.length === 1 ? "it" : "them"} ` +
|
|
963
|
-
`would make the app's own UPDATEs and DELETEs fail, so Mason does not
|
|
1333
|
+
`would make the app's own UPDATEs and DELETEs fail, so Mason does not replicate ${nothing.length === 1 ? "it" : "them"}. Give ${nothing.length === 1 ? "its entry" : "their entries"} ` +
|
|
1334
|
+
`sync: poll <interval> (poll 5m) in the sources file, or ALTER TABLE … REPLICA IDENTITY DEFAULT on the database`);
|
|
964
1335
|
}
|
|
965
1336
|
if (r.owned)
|
|
966
1337
|
await this.ensurePublication(pg, tables, identities);
|
|
@@ -969,7 +1340,7 @@ export class Sync {
|
|
|
969
1340
|
const slot = await readSlot(pg, r.slot);
|
|
970
1341
|
let made = false;
|
|
971
1342
|
if (slot.exists && slot.walStatus === "lost") {
|
|
972
|
-
this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the
|
|
1343
|
+
this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the synced tables caught up by one poll`);
|
|
973
1344
|
if (slot.active)
|
|
974
1345
|
throw new Error(`replication slot ${r.slot} is lost and still in use by pid ${slot.activePid}`);
|
|
975
1346
|
await dropSlot(pg, r.slot);
|
|
@@ -980,14 +1351,16 @@ export class Sync {
|
|
|
980
1351
|
if (!slot.exists || slot.walStatus === "lost") {
|
|
981
1352
|
const repl = await PgConnection.connect({ ...this.pgOptions(), replication: true });
|
|
982
1353
|
try {
|
|
983
|
-
this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy,
|
|
1354
|
+
this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy drops it, and so does a sources file in which no table's sync is replication)`);
|
|
984
1355
|
}
|
|
985
1356
|
finally {
|
|
986
1357
|
repl.close();
|
|
987
1358
|
}
|
|
988
1359
|
made = true;
|
|
1360
|
+
this.onReplication?.(true);
|
|
989
1361
|
}
|
|
990
1362
|
if (made || !this.slotKnown) {
|
|
1363
|
+
// the replicated tables already copied: the slot carries nothing from before it (a polled one is the poll's)
|
|
991
1364
|
const copied = tables.filter((t) => this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source));
|
|
992
1365
|
await this.onWriter(async () => {
|
|
993
1366
|
const w = this.store.writer;
|
|
@@ -1013,24 +1386,44 @@ export class Sync {
|
|
|
1013
1386
|
finally {
|
|
1014
1387
|
pg.close();
|
|
1015
1388
|
}
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1389
|
+
await this.onWriter(() => this.settlePrepared(tables));
|
|
1390
|
+
}
|
|
1391
|
+
/**
|
|
1392
|
+
* WHAT A PASS'S PUBLICATION CHECK SETTLES, ON THE WRITER'S TURN (`prepareReplication`, once the publication and slot are
|
|
1393
|
+
* as they should be): a table a named publication does not carry is refused on its own line (a copy it never had is
|
|
1394
|
+
* failed), the others go on; a table it carries AGAIN has that gap's error cleared, and a table never copied the last
|
|
1395
|
+
* pass's failure to prepare. NOTHING ELSE: a copy whose stream failed, or that the failed preparation left behind, is
|
|
1396
|
+
* current again only once the stream applies a batch with a position (`applyChanges`) — the publication being right
|
|
1397
|
+
* says nothing of whether its changes are in the copy, and a pass runs while the stream waits out its back-off.
|
|
1398
|
+
*/
|
|
1399
|
+
async settlePrepared(tables) {
|
|
1400
|
+
for (const t of tables) {
|
|
1401
|
+
const st = this.states.get(t.source);
|
|
1402
|
+
const gap = this.unpublished.get(t.source);
|
|
1403
|
+
if (gap !== undefined) {
|
|
1404
|
+
const error = `replication: ${gap}`;
|
|
1405
|
+
if (st.error !== error) {
|
|
1406
|
+
this.log(`${t.source} is not synced or streamed: ${error}`);
|
|
1407
|
+
this.event({ kind: "stream_failed", table: t.source, message: `not synced or streamed: ${error}` });
|
|
1027
1408
|
}
|
|
1028
|
-
|
|
1029
|
-
|
|
1409
|
+
this.gapErrors.set(t.source, error);
|
|
1410
|
+
await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error });
|
|
1411
|
+
continue;
|
|
1030
1412
|
}
|
|
1031
|
-
|
|
1032
|
-
this.
|
|
1033
|
-
|
|
1413
|
+
const gapClosed = this.gapErrors.get(t.source);
|
|
1414
|
+
this.gapErrors.delete(t.source);
|
|
1415
|
+
if (st.error === null)
|
|
1416
|
+
continue;
|
|
1417
|
+
if (st.error === gapClosed || (st.phase !== "copied" && st.error === this.wholeFailure.prepare))
|
|
1418
|
+
await this.record({ ...st, error: null });
|
|
1419
|
+
}
|
|
1420
|
+
// a table the publication does not carry was never streamed: its last sync is its copy's or its poll's, never
|
|
1421
|
+
// the stream's time a start credited it with (`readSynced` at `open`, before the publication was read)
|
|
1422
|
+
const credited = tables.filter((t) => this.unpublished.has(t.source) && this.synced.get(t.source)?.by === "stream").map((t) => t.source);
|
|
1423
|
+
if (credited.length)
|
|
1424
|
+
await this.readSynced(new Set(credited));
|
|
1425
|
+
// a table the publication stopped carrying leaves the stream; one it carries now may join
|
|
1426
|
+
this.refreshStreamed();
|
|
1034
1427
|
}
|
|
1035
1428
|
/**
|
|
1036
1429
|
* A publication the operator NAMED: read as it is, verified, never created, altered or dropped (src/duck/publication.ts).
|
|
@@ -1039,14 +1432,14 @@ export class Sync {
|
|
|
1039
1432
|
*/
|
|
1040
1433
|
async verifyPublication(pg, tables, identities) {
|
|
1041
1434
|
const name = this.replication.publication;
|
|
1042
|
-
const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables
|
|
1435
|
+
const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables), tables, identities);
|
|
1043
1436
|
if (v.refused !== null)
|
|
1044
1437
|
throw new Error(v.refused);
|
|
1045
1438
|
this.unpublished.clear();
|
|
1046
1439
|
for (const [source, m] of v.tables)
|
|
1047
1440
|
this.unpublished.set(source, m);
|
|
1048
1441
|
if (v.noTruncate && !this.truncateSaid) {
|
|
1049
|
-
this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the
|
|
1442
|
+
this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the synced data (the poll's count would see it: sync: poll <interval> on its entry)`);
|
|
1050
1443
|
this.truncateSaid = true;
|
|
1051
1444
|
}
|
|
1052
1445
|
}
|
|
@@ -1080,8 +1473,68 @@ export class Sync {
|
|
|
1080
1473
|
throw new Error(`publication ${name} is FOR ALL TABLES: it is not the one Mason made (drop it, or name another project)`);
|
|
1081
1474
|
await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET TABLE ${list}`);
|
|
1082
1475
|
await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET (publish = 'insert, update, delete, truncate')`);
|
|
1083
|
-
this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}`);
|
|
1476
|
+
this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}: the tables whose sync is replication`);
|
|
1477
|
+
}
|
|
1478
|
+
// VERIFIED, never assumed: what the publication carries is what the database says it does
|
|
1479
|
+
const carried = (await pg.query(`SELECT schemaname || '.' || tablename FROM pg_publication_tables WHERE pubname = ${pgLiteral(name)}`)).rows.map((x) => x[0]).sort();
|
|
1480
|
+
const wanted = tables.map((t) => t.source).sort();
|
|
1481
|
+
if (carried.join("\n") !== wanted.join("\n")) {
|
|
1482
|
+
throw new Error(`publication ${name} carries ${carried.join(", ") || "no table"} after it was set to ${wanted.join(", ")} (Mason bug: report it, with mason logs)`);
|
|
1483
|
+
}
|
|
1484
|
+
}
|
|
1485
|
+
/**
|
|
1486
|
+
* NO TABLE'S SYNC IS REPLICATION ANY MORE (at the start that finds none, or live when the last one leaves): the stream
|
|
1487
|
+
* stopped, the slot dropped — it holds the database's WAL while it exists, read or not — and Mason's own publication
|
|
1488
|
+
* with it (a named one is its owner's: left as it is), each VERIFIED gone; then this copy's record of the slot goes,
|
|
1489
|
+
* so a table replicated again later is caught up by one poll under the slot made anew. Idempotent: what is already
|
|
1490
|
+
* gone is not an error. A failure is the pass's, said, and tried again at the next.
|
|
1491
|
+
*/
|
|
1492
|
+
async retireReplication() {
|
|
1493
|
+
const r = this.replication;
|
|
1494
|
+
const s = this.stream;
|
|
1495
|
+
if (s !== null) {
|
|
1496
|
+
await s.stop();
|
|
1497
|
+
if (this.stream === s)
|
|
1498
|
+
this.stream = null;
|
|
1499
|
+
}
|
|
1500
|
+
const pg = await PgConnection.connect(this.pgOptions());
|
|
1501
|
+
const dropped = [];
|
|
1502
|
+
try {
|
|
1503
|
+
// the walsender a stop ended lets the slot go a moment later: waited for, at most 10 s
|
|
1504
|
+
let slot = await readSlot(pg, r.slot);
|
|
1505
|
+
for (let i = 0; slot.exists && slot.active && i < 100; i++) {
|
|
1506
|
+
await new Promise((res) => setTimeout(res, 100));
|
|
1507
|
+
slot = await readSlot(pg, r.slot);
|
|
1508
|
+
}
|
|
1509
|
+
if (slot.exists) {
|
|
1510
|
+
if (slot.active)
|
|
1511
|
+
throw new Error(`replication slot ${r.slot} is in use by pid ${slot.activePid ?? "?"}, so it cannot be dropped (another Mason over the same project? stop it)`);
|
|
1512
|
+
await dropSlot(pg, r.slot);
|
|
1513
|
+
dropped.push(`replication slot ${r.slot}`);
|
|
1514
|
+
}
|
|
1515
|
+
const hasPub = async () => (await pg.query(`SELECT 1 FROM pg_publication WHERE pubname = ${pgLiteral(r.publication)}`)).rows.length > 0;
|
|
1516
|
+
if (r.owned && await hasPub()) {
|
|
1517
|
+
await pg.query(`DROP PUBLICATION IF EXISTS ${pgIdent(r.publication)}`);
|
|
1518
|
+
dropped.push(`publication ${r.publication}`);
|
|
1519
|
+
}
|
|
1520
|
+
if ((await readSlot(pg, r.slot)).exists)
|
|
1521
|
+
throw new Error(`replication slot ${r.slot} is still there after pg_drop_replication_slot`);
|
|
1522
|
+
if (r.owned && await hasPub())
|
|
1523
|
+
throw new Error(`publication ${r.publication} is still there after DROP PUBLICATION`);
|
|
1524
|
+
}
|
|
1525
|
+
finally {
|
|
1526
|
+
pg.close();
|
|
1084
1527
|
}
|
|
1528
|
+
await this.onWriter(async () => {
|
|
1529
|
+
await this.store.writer.run(`DELETE FROM ${MASON_SCHEMA}.stream_state WHERE slot = $1`, [r.slot]);
|
|
1530
|
+
this.slotKnown = false;
|
|
1531
|
+
this.appliedLsn = 0n;
|
|
1532
|
+
this.streamCurrentAt = null;
|
|
1533
|
+
this.refreshStreamed();
|
|
1534
|
+
});
|
|
1535
|
+
this.log(`no table's sync is replication now: ${dropped.length ? `dropped ${dropped.join(" and ")} from ${this.source.database}, verified gone` : `replication slot ${r.slot} was already gone`}` +
|
|
1536
|
+
` (the slot held the database's WAL)${r.owned ? "" : `; publication ${r.publication} is its owner's, left as it is`} — a table set to sync: replication makes the slot again`);
|
|
1537
|
+
this.onReplication?.(false);
|
|
1085
1538
|
}
|
|
1086
1539
|
/**
|
|
1087
1540
|
* The stream started from the slot's acknowledged position, once the slot is known (a pass's `prepareReplication`),
|
|
@@ -1089,14 +1542,15 @@ export class Sync {
|
|
|
1089
1542
|
* joins once caught up and leaves when copied again without a restart.
|
|
1090
1543
|
*/
|
|
1091
1544
|
startStream() {
|
|
1092
|
-
if (this.stream !== null || this.stopped || this.
|
|
1545
|
+
if (this.stream !== null || this.stopped || !this.streams() || !this.slotKnown || Date.now() < this.streamAt)
|
|
1093
1546
|
return;
|
|
1094
1547
|
const r = this.replication;
|
|
1095
1548
|
this.refreshStreamed();
|
|
1096
1549
|
const s = new ReplicationStream({
|
|
1097
1550
|
connect: () => PgConnection.connect({ ...this.pgOptions(), replication: true }), slot: r.slot, publication: r.publication, log: (l) => this.log(l),
|
|
1098
1551
|
target: { tables: () => this.streamed, apply: (batch, lsn, committedAt) => this.applyStreamed(batch, lsn, committedAt), applied: () => this.appliedLsn,
|
|
1099
|
-
refuse: (source, missing) => this.refuseStreamed(source, missing)
|
|
1552
|
+
refuse: (source, missing) => this.refuseStreamed(source, missing), current: () => { if (this.stream === s)
|
|
1553
|
+
this.streamCurrentAt = Date.now(); } },
|
|
1100
1554
|
});
|
|
1101
1555
|
this.stream = s;
|
|
1102
1556
|
this.streamFailure = null;
|
|
@@ -1108,7 +1562,7 @@ export class Sync {
|
|
|
1108
1562
|
}
|
|
1109
1563
|
const joined = [...this.streamed.keys()].filter((x) => !this.held.has(x));
|
|
1110
1564
|
this.log(`streaming ${joined.length ? joined.join(", ") : "no table yet"} from slot ${r.slot} (publication ${r.publication}), applied through ${formatLsn(this.appliedLsn)}` +
|
|
1111
|
-
`${joined.length < this.
|
|
1565
|
+
`${joined.length < this.replicatedTables().length ? "; a table synced or caught up meanwhile joins it once current" : ""}`);
|
|
1112
1566
|
void s.done.then(async (e) => {
|
|
1113
1567
|
if (this.stream === s)
|
|
1114
1568
|
this.stream = null;
|
|
@@ -1122,7 +1576,7 @@ export class Sync {
|
|
|
1122
1576
|
if (wider !== undefined && n <= 3) {
|
|
1123
1577
|
this.widened.set(wider.source, n);
|
|
1124
1578
|
await this.onWriter(async () => { this.snapshots.delete(wider.source); this.refreshStreamed(); });
|
|
1125
|
-
this.log(`replication: ${wider.source} has a numeric value wider than its
|
|
1579
|
+
this.log(`replication: ${wider.source} has a numeric value wider than its synced DECIMAL column: it is read again whole, then streamed`);
|
|
1126
1580
|
this.startStream();
|
|
1127
1581
|
this.kick();
|
|
1128
1582
|
return;
|
|
@@ -1135,7 +1589,11 @@ export class Sync {
|
|
|
1135
1589
|
// a timer may fire a little early: the pass it kicks starts the stream
|
|
1136
1590
|
this.streamAt = Date.now() + wait - 50;
|
|
1137
1591
|
this.streamFailure = { error: m, retryAt: this.streamAt };
|
|
1138
|
-
this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the
|
|
1592
|
+
this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the synced tables serve as they are)`);
|
|
1593
|
+
// one event for the stream, never one per table it kept current (each table's error says it); its failures in a
|
|
1594
|
+
// row are left out of the words, so a stream failing for good is one event, not one per restart
|
|
1595
|
+
this.event({ kind: "stream_failed", table: null, message: `${m} (the stream starts again by itself; the synced tables serve as they are)` });
|
|
1596
|
+
this.wholeFailure.stream = m;
|
|
1139
1597
|
// on the tables the stream kept current when it failed (one being caught up says its own)
|
|
1140
1598
|
await this.onWriter(async () => { for (const source of this.streamed.keys())
|
|
1141
1599
|
if (!this.held.has(source))
|
|
@@ -1164,10 +1622,11 @@ export class Sync {
|
|
|
1164
1622
|
this.held.delete(source);
|
|
1165
1623
|
const when = this.backOff(source, 0);
|
|
1166
1624
|
const pub = this.replication.publication;
|
|
1167
|
-
const m = `replication: the stream no longer carries ${source}'s ${cols}, which its
|
|
1625
|
+
const m = `replication: the stream no longer carries ${source}'s ${cols}, which its synced table holds (a column dropped or renamed on the database, or the publication's column list narrowed), ` +
|
|
1168
1626
|
`so its changes are not applied: it is caught up by one poll again after ${when}. The way out: remove ${cols} from its columns in the sources file (mason add ${source} --columns … ` +
|
|
1169
|
-
`picks the columns it has; it is
|
|
1627
|
+
`picks the columns it has; it is read again whole at once)${this.replication.owned ? "" : `, or publish ${missing.length === 1 ? "it" : "them"} again (ALTER PUBLICATION ${pub} …)`}`;
|
|
1170
1628
|
this.log(`${source}: ${m}`);
|
|
1629
|
+
this.event({ kind: "stream_failed", table: source, message: m });
|
|
1171
1630
|
await this.record({ ...this.states.get(source), error: m });
|
|
1172
1631
|
this.refreshStreamed();
|
|
1173
1632
|
});
|
|
@@ -1176,18 +1635,22 @@ export class Sync {
|
|
|
1176
1635
|
* What the stream applies, made again from the tables' state (on the writer's turn, or before the stream exists): a
|
|
1177
1636
|
* table copied at its present columns and caught up, and a table being caught up (its changes held, `held`). A table
|
|
1178
1637
|
* that is not — never copied, being copied again, or not caught up — is skipped: its copy's snapshot, or its catch-up,
|
|
1179
|
-
* covers what the stream passes over (src/duck/stream.ts "why replaying is safe").
|
|
1638
|
+
* covers what the stream passes over (src/duck/stream.ts "why replaying is safe"). A table here since the last refresh
|
|
1639
|
+
* keeps its STAY, a table new here gets one, and `swap` ends it: changes decoded in a stay that ended committed before
|
|
1640
|
+
* what came next, so that copy and its catch-up hold them, and only that stay's are applied (`applyStreamed`).
|
|
1180
1641
|
*/
|
|
1181
1642
|
refreshStreamed() {
|
|
1643
|
+
const before = new Map(this.streamed);
|
|
1182
1644
|
this.streamed.clear();
|
|
1183
1645
|
const now = new Set();
|
|
1184
1646
|
for (const t of this.tables.values()) {
|
|
1185
|
-
// a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot
|
|
1186
|
-
|
|
1647
|
+
// a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot;
|
|
1648
|
+
// nor one whose sync is not replication (it is polled, or off)
|
|
1649
|
+
const joined = this.replicated(t) && !this.behind.has(t.source) && !this.needsCopy(t) && !this.unpublished.has(t.source);
|
|
1187
1650
|
// in the COPY's columns: a held table's columns may have changed since its copy began (`asCopied`)
|
|
1188
1651
|
const c = this.asCopied(t).columns;
|
|
1189
1652
|
if (joined || this.held.has(t.source)) {
|
|
1190
|
-
this.streamed.set(t.source, { columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows >
|
|
1653
|
+
this.streamed.set(t.source, { stay: before.get(t.source)?.stay ?? ++this.lastStay, columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows > BIG_TABLE_ROWS });
|
|
1191
1654
|
}
|
|
1192
1655
|
if (joined)
|
|
1193
1656
|
now.add(t.source);
|
|
@@ -1196,11 +1659,18 @@ export class Sync {
|
|
|
1196
1659
|
// while a stream runs: its start names what it streams. What it returns is the tables whose join was said
|
|
1197
1660
|
const said = [];
|
|
1198
1661
|
const s = this.stream;
|
|
1199
|
-
if (s !== null
|
|
1662
|
+
if (s !== null) {
|
|
1200
1663
|
const at = formatLsn(s.position());
|
|
1201
|
-
for (const source of this.joinedNow)
|
|
1202
|
-
if (
|
|
1203
|
-
|
|
1664
|
+
for (const source of this.joinedNow) {
|
|
1665
|
+
if (now.has(source))
|
|
1666
|
+
continue;
|
|
1667
|
+
const t = this.tables.get(source);
|
|
1668
|
+
// its sync changed: what keeps it current now, in the words of the switch
|
|
1669
|
+
const why = t !== undefined && !this.replicated(t)
|
|
1670
|
+
? `its sync is ${syncText(this.syncOf(t))} now, so it is caught up by one poll from its watermark${this.syncOf(t).mode === "off" ? ", then read again only by mason sync" : ", then polled"}`
|
|
1671
|
+
: this.notStreamedWhy(source);
|
|
1672
|
+
this.log(`${source} leaves the stream at ${at}: ${why}`);
|
|
1673
|
+
}
|
|
1204
1674
|
for (const source of now) {
|
|
1205
1675
|
if (this.joinedNow.has(source))
|
|
1206
1676
|
continue;
|
|
@@ -1222,63 +1692,71 @@ export class Sync {
|
|
|
1222
1692
|
return `${why} — after a failure, tried again after ${clock(at)} UTC`;
|
|
1223
1693
|
const ahead = this.rebuildingNow;
|
|
1224
1694
|
if (this.queued.has(source) && ahead !== null && ahead.source !== source)
|
|
1225
|
-
return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "
|
|
1695
|
+
return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "whole read" : "catch-up"} (one rebuild at a time)`;
|
|
1226
1696
|
return why;
|
|
1227
1697
|
}
|
|
1698
|
+
/** why a table whose first sync finished is to be read again whole, or null: what the start's line and mason sync say */
|
|
1699
|
+
readAgainWhy(t) {
|
|
1700
|
+
if (this.copiedAs.get(t.source) !== columnsKey(t.columns))
|
|
1701
|
+
return "its columns changed";
|
|
1702
|
+
return this.snapshots.has(t.source) ? null : "a numeric value outgrew its synced DECIMAL column";
|
|
1703
|
+
}
|
|
1228
1704
|
notStreamedReason(source) {
|
|
1229
1705
|
const t = this.tables.get(source);
|
|
1230
1706
|
if (t === undefined)
|
|
1231
|
-
return "it is no longer added (its
|
|
1707
|
+
return "it is no longer added (its synced data is dropped)";
|
|
1232
1708
|
const gap = this.unpublished.get(source);
|
|
1233
1709
|
if (gap !== undefined)
|
|
1234
1710
|
return gap.split("\n")[0];
|
|
1235
1711
|
const st = this.states.get(source);
|
|
1236
1712
|
if (this.copiedAs.has(source) && this.copiedAs.get(source) !== columnsKey(t.columns))
|
|
1237
|
-
return "its columns changed: it is
|
|
1713
|
+
return "its columns changed: it is read again whole, then caught up by one poll";
|
|
1238
1714
|
if (st?.phase === "copied" && !this.snapshots.has(source))
|
|
1239
|
-
return "a numeric value outgrew its
|
|
1715
|
+
return "a numeric value outgrew its synced DECIMAL column: it is read again whole, wider, then caught up by one poll";
|
|
1716
|
+
if (st?.phase === "copied" && this.retired.has(source))
|
|
1717
|
+
return "it is in an older Mason's layout, key order with an index: it is read again whole, then caught up by one poll";
|
|
1718
|
+
// what follows the copy: a catch-up poll, then the stream or the polls; copied again whole, nothing (`keptOf`'s recopy)
|
|
1719
|
+
const joins = this.mode === "recopy" ? "it is kept current once synced" : "it joins once synced and caught up by one poll";
|
|
1240
1720
|
if (st?.phase === "copying" || this.rebuildingNow?.source === source && this.rebuildingNow.kind === "copy")
|
|
1241
|
-
return
|
|
1721
|
+
return `it is being read whole: ${joins}`;
|
|
1242
1722
|
if (st?.phase === "failed")
|
|
1243
|
-
return `its
|
|
1723
|
+
return `its first sync failed (${(st.error ?? "").split("\n")[0]}): ${joins}`;
|
|
1244
1724
|
const b = this.behind.get(source);
|
|
1245
1725
|
if (!this.needsCopy(t) && b !== undefined)
|
|
1246
1726
|
return `it is caught up by one poll ${b.why} first`;
|
|
1247
|
-
|
|
1727
|
+
// synced, caught up, and not yet polled (`pollable`): its rebuild's last writer's turn, or a rebuild still queued
|
|
1728
|
+
if (st?.phase === "copied") {
|
|
1729
|
+
if (this.rebuildingNow?.source === source)
|
|
1730
|
+
return "its sync is ending: it is kept current from then on";
|
|
1731
|
+
const queued = this.queued.get(source)?.kind;
|
|
1732
|
+
if (queued !== undefined)
|
|
1733
|
+
return queued === "copy" ? `it is queued to be read whole: ${joins}` : "it is queued to be caught up by one poll";
|
|
1734
|
+
}
|
|
1735
|
+
return `it waits for its first sync: ${joins}`;
|
|
1248
1736
|
}
|
|
1249
1737
|
/**
|
|
1250
1738
|
* WHERE THE STREAM IS (`StreamStatus`): its position, the database's, the lag, the last batch, and each added table's
|
|
1251
1739
|
* part in it with what it applied this run. Null outside replication mode. The database's position costs a
|
|
1252
1740
|
* connection at most every 5 s (`databasePosition`); everything else is the run's own.
|
|
1741
|
+
*
|
|
1742
|
+
* `refresh: false` NEVER REACHES THE DATABASE: the position and the slot as last read (`database.at`, `slotWal.at` say
|
|
1743
|
+
* when), or "not read yet". What an unauthenticated reader asks with (`GET /metrics`, src/cli/run-server.ts
|
|
1744
|
+
* `metricsFacts`): a scraper's rate, or anyone's loop on the health port, must never become connections and SELECTs
|
|
1745
|
+
* on the source; the usage sampler, `mason status` and `mason wait` keep the reads fresh.
|
|
1253
1746
|
*/
|
|
1254
|
-
async streamStatus() {
|
|
1255
|
-
if (this.
|
|
1747
|
+
async streamStatus({ refresh = true } = {}) {
|
|
1748
|
+
if (!this.streams())
|
|
1256
1749
|
return null;
|
|
1257
1750
|
const r = this.replication;
|
|
1258
1751
|
const s = this.stream;
|
|
1259
1752
|
const position = s?.position() ?? this.appliedLsn;
|
|
1260
|
-
const db = await this.databasePosition();
|
|
1753
|
+
const db = refresh ? await this.databasePosition() : this.dbPosition ?? { error: "not read yet: nothing has asked where the database is since the run began", at: Date.now() };
|
|
1261
1754
|
const now = Date.now();
|
|
1262
1755
|
const iso = (ms) => new Date(ms).toISOString();
|
|
1263
|
-
|
|
1264
|
-
|
|
1265
|
-
if (this.unpublished.has(t.source))
|
|
1266
|
-
return { table: t.source, state: "left-out", why: this.notStreamedWhy(t.source), applied };
|
|
1267
|
-
const b = this.behind.get(t.source);
|
|
1268
|
-
if (!this.needsCopy(t) && b !== undefined) {
|
|
1269
|
-
const c = this.catchingUp?.source === t.source ? this.catchingUp : null;
|
|
1270
|
-
const held = this.held.get(t.source)?.counts;
|
|
1271
|
-
const at = this.retries.get(t.source)?.at ?? 0;
|
|
1272
|
-
return { table: t.source, state: "catching-up", why: `caught up by one poll ${b.why}${at > now ? ` — its last catch-up failed, tried again after ${clock(at)} UTC` : ""}`,
|
|
1273
|
-
catchUp: { phase: c?.phase ?? "queued", from: this.polledAt.get(t.source) ?? null, startedAt: c ? iso(c.startedAt) : null, held: held ? held.inserts + held.updates + held.deletes + held.truncates : 0,
|
|
1274
|
-
retryAt: c === null && at > now ? iso(at) : null }, applied };
|
|
1275
|
-
}
|
|
1276
|
-
if (this.joinedNow.has(t.source) || (s === null && !this.needsCopy(t)))
|
|
1277
|
-
return { table: t.source, state: "streamed", why: null, applied };
|
|
1278
|
-
return { table: t.source, state: "waiting", why: this.notStreamedWhy(t.source), applied };
|
|
1279
|
-
});
|
|
1756
|
+
// each table's part, as `keptOf` says it for EXPLAIN (`streamTable`), with what it applied this run
|
|
1757
|
+
const tables = this.replicatedTables().map((t) => ({ table: t.source, ...this.streamTable(t, now, s), applied: { ...(this.appliedCounts.get(t.source) ?? noCounts()) } }));
|
|
1280
1758
|
return {
|
|
1281
|
-
slot: r.slot, publication: r.publication, running: s !== null,
|
|
1759
|
+
slot: r.slot, slotWal: this.slotWal?.wal ?? { error: "not read yet: the database did not answer within 3 s" }, publication: r.publication, running: s !== null,
|
|
1282
1760
|
failure: s === null && this.streamFailure !== null ? { error: this.streamFailure.error, retryAt: iso(this.streamFailure.retryAt) } : null,
|
|
1283
1761
|
position: formatLsn(position), applied: formatLsn(this.appliedLsn),
|
|
1284
1762
|
database: "lsn" in db ? { lsn: formatLsn(db.lsn), at: iso(db.at) } : { error: db.error },
|
|
@@ -1293,15 +1771,28 @@ export class Sync {
|
|
|
1293
1771
|
* wait` asks every second), waited for at most 3 s (the last read is answered meanwhile, with its time). Not the
|
|
1294
1772
|
* stream's keepalive: that carries the walsender's own read position, not the database's, so it cannot say how far
|
|
1295
1773
|
* behind the stream is when that matters — while it decodes a backlog. A read that fails is answered as its error.
|
|
1774
|
+
* THE SLOT'S WAL (`SlotWal`) is read on the same connection, at most every SLOT_READ_MS (one row of
|
|
1775
|
+
* pg_replication_slots): what the slot holds on the database's disk, said by `mason status` and sampled by usage.
|
|
1296
1776
|
*/
|
|
1297
1777
|
async databasePosition() {
|
|
1298
1778
|
const fresh = this.dbPosition !== null && Date.now() - this.dbPosition.at < 5_000;
|
|
1299
1779
|
if (!fresh && this.dbPositionRead === null) {
|
|
1780
|
+
const slotDue = this.slotWal === null || Date.now() - this.slotWal.at >= SLOT_READ_MS;
|
|
1781
|
+
const slot = this.replication.slot;
|
|
1300
1782
|
this.dbPositionRead = (async () => {
|
|
1301
1783
|
try {
|
|
1302
1784
|
const pg = await PgConnection.connect(this.pgOptions());
|
|
1303
1785
|
try {
|
|
1304
1786
|
this.dbPosition = { lsn: await currentLsn(pg), at: Date.now() };
|
|
1787
|
+
if (slotDue) {
|
|
1788
|
+
// its own failure is the slot's line alone: the position read above stands
|
|
1789
|
+
try {
|
|
1790
|
+
this.slotWal = { wal: slotWalOf(await readSlot(pg, slot), slot, Date.now()), at: Date.now() };
|
|
1791
|
+
}
|
|
1792
|
+
catch (e) {
|
|
1793
|
+
this.slotWal = { wal: { error: `the slot could not be read: ${e instanceof Error ? e.message : String(e)}` }, at: Date.now() };
|
|
1794
|
+
}
|
|
1795
|
+
}
|
|
1305
1796
|
}
|
|
1306
1797
|
finally {
|
|
1307
1798
|
pg.close();
|
|
@@ -1309,7 +1800,10 @@ export class Sync {
|
|
|
1309
1800
|
}
|
|
1310
1801
|
catch (e) {
|
|
1311
1802
|
// answered as the status's (and the log line's) own words: the position could not be read, and why
|
|
1312
|
-
|
|
1803
|
+
const m = e instanceof Error ? e.message : String(e);
|
|
1804
|
+
this.dbPosition = { error: m, at: Date.now() };
|
|
1805
|
+
if (slotDue)
|
|
1806
|
+
this.slotWal = { wal: { error: `the database could not be reached: ${m}` }, at: Date.now() };
|
|
1313
1807
|
}
|
|
1314
1808
|
finally {
|
|
1315
1809
|
this.dbPositionRead = null;
|
|
@@ -1350,10 +1844,10 @@ export class Sync {
|
|
|
1350
1844
|
this.refreshStreamed();
|
|
1351
1845
|
const applied = new Map();
|
|
1352
1846
|
for (const [name, changes] of batch) {
|
|
1353
|
-
|
|
1354
|
-
//
|
|
1355
|
-
//
|
|
1356
|
-
if (
|
|
1847
|
+
// decoded in a stay that ended (before its table was copied again, or left the stream): what they changed
|
|
1848
|
+
// committed before the next copy's catch-up, so that copy or its catch-up holds it — never written over them, in
|
|
1849
|
+
// a shape the copy may not have, or as an old change after the poll that read a newer one
|
|
1850
|
+
if (this.streamed.get(name)?.stay !== changes.stay)
|
|
1357
1851
|
continue;
|
|
1358
1852
|
const held = this.held.get(name);
|
|
1359
1853
|
if (held !== undefined) {
|
|
@@ -1362,7 +1856,7 @@ export class Sync {
|
|
|
1362
1856
|
}
|
|
1363
1857
|
applied.set(name, changes);
|
|
1364
1858
|
}
|
|
1365
|
-
await this.applyChanges(applied, lsn, committedAt);
|
|
1859
|
+
await this.applyChanges(applied, { lsn, committedAt });
|
|
1366
1860
|
});
|
|
1367
1861
|
}
|
|
1368
1862
|
/**
|
|
@@ -1373,12 +1867,26 @@ export class Sync {
|
|
|
1373
1867
|
* index each of those is a scan of the table's key column, so a batch costs a few scans per table, never one per row. A numeric wider than its copy's DECIMAL is the table's to copy
|
|
1374
1868
|
* again (`WIDER`), never rounded. A null `lsn` is a piece of a large transaction (src/duck/stream.ts): committed, the
|
|
1375
1869
|
* position left as it was, so a run that dies before the transaction's last piece applies it again whole. What it
|
|
1376
|
-
* applied is counted by table (`streamStatus`), and the batch's newest commit time kept
|
|
1377
|
-
*
|
|
1870
|
+
* applied is counted by table (`streamStatus`), and the batch's newest commit time kept.
|
|
1871
|
+
*
|
|
1872
|
+
* `"held"` is THE CHANGES A CATCH-UP HELD, replayed after its poll (`catchUp`): written, and counted, as a batch's are,
|
|
1873
|
+
* but NOT A POSITION of the stream — no `stream_state` row (its LSN and `applied_at` are the batches' that held them),
|
|
1874
|
+
* no table's `synced`, no commit time, and not the stream working again (`streamFailures`): the replay ran on the
|
|
1875
|
+
* writer, whatever the stream is doing (it may be failing).
|
|
1378
1876
|
*/
|
|
1379
|
-
async applyChanges(batch,
|
|
1877
|
+
async applyChanges(batch, from) {
|
|
1878
|
+
const lsn = from === "held" ? null : from.lsn;
|
|
1879
|
+
const committedAt = from === "held" ? null : from.committedAt;
|
|
1380
1880
|
const w = this.store.writer;
|
|
1381
1881
|
let source = null;
|
|
1882
|
+
const started = Date.now();
|
|
1883
|
+
// EACH TABLE'S ROWS AFTER THE BATCH, kept current at every batch (it was counted at most every 5 s, so a table's
|
|
1884
|
+
// count missed the batches after one until another came): what its writes added and removed, which DuckDB says
|
|
1885
|
+
// (`rowsChanged`) — never a count(*), which a write by key's deletes make a scan (12 ms on 20 M rows, measured) —
|
|
1886
|
+
// written to the ledger IN the batch's transaction, so a run that dies keeps a count equal to its rows
|
|
1887
|
+
const rowsAfter = new Map();
|
|
1888
|
+
// a batch with a position: its time in the ledger and in this run, ONE value (`Synced`)
|
|
1889
|
+
const position = lsn === null ? null : { lsn, at: new Date().toISOString() };
|
|
1382
1890
|
await w.run("BEGIN TRANSACTION");
|
|
1383
1891
|
try {
|
|
1384
1892
|
for (const [name, changes] of batch) {
|
|
@@ -1398,8 +1906,7 @@ export class Sync {
|
|
|
1398
1906
|
throw new Error(`${WIDER} [${name}]: ${c} = ${v}`);
|
|
1399
1907
|
return duckValue(v, types.get(c));
|
|
1400
1908
|
};
|
|
1401
|
-
|
|
1402
|
-
await w.run(`DELETE FROM ${target}`);
|
|
1909
|
+
let delta = changes.truncated ? -Number((await w.run(`DELETE FROM ${target}`)).rowsChanged) : 0;
|
|
1403
1910
|
const whole = [];
|
|
1404
1911
|
const fetch = [];
|
|
1405
1912
|
const gone = [];
|
|
@@ -1429,10 +1936,16 @@ export class Sync {
|
|
|
1429
1936
|
source ??= await PgConnection.connect(this.pgOptions());
|
|
1430
1937
|
whole.push(...await this.sourceRows(source, t, fetch));
|
|
1431
1938
|
}
|
|
1432
|
-
await this.writeByKey(t, target, gone, whole, value);
|
|
1939
|
+
delta += await this.writeByKey(t, target, gone, whole, value);
|
|
1940
|
+
// a copy whose rows the ledger does not know (none should be: its copy counts them) is counted, once
|
|
1941
|
+
const known = this.states.get(name)?.rows ?? null;
|
|
1942
|
+
const rows = known === null ? Number((await w.runAndReadAll(`SELECT count(*) FROM ${target}`)).getRowsJS()[0][0]) : known + delta;
|
|
1943
|
+
if (rows !== known)
|
|
1944
|
+
await w.run(`UPDATE ${MASON_SCHEMA}.copy_state SET "rows" = $1 WHERE source = $2`, [rows, name]);
|
|
1945
|
+
rowsAfter.set(name, rows);
|
|
1433
1946
|
}
|
|
1434
|
-
if (
|
|
1435
|
-
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2,
|
|
1947
|
+
if (position !== null)
|
|
1948
|
+
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2, $3::TIMESTAMPTZ)`, [this.replication.slot, formatLsn(position.lsn), position.at]);
|
|
1436
1949
|
await w.run("COMMIT");
|
|
1437
1950
|
}
|
|
1438
1951
|
catch (e) {
|
|
@@ -1444,7 +1957,12 @@ export class Sync {
|
|
|
1444
1957
|
}
|
|
1445
1958
|
if (lsn !== null)
|
|
1446
1959
|
this.appliedLsn = lsn;
|
|
1447
|
-
|
|
1960
|
+
if (from !== "held")
|
|
1961
|
+
this.streamFailures = 0;
|
|
1962
|
+
// THE STREAM'S FAILURE IS OVER when it applies a batch WITH A POSITION: every table it keeps current is current as of
|
|
1963
|
+
// it. A piece of a large transaction (no position) is not that yet, nor the changes a catch-up held
|
|
1964
|
+
if (lsn !== null)
|
|
1965
|
+
this.recovered("stream", "the stream applies changes again");
|
|
1448
1966
|
let moved = false;
|
|
1449
1967
|
for (const [name, changes] of batch) {
|
|
1450
1968
|
if (!this.tables.has(name))
|
|
@@ -1458,17 +1976,30 @@ export class Sync {
|
|
|
1458
1976
|
}
|
|
1459
1977
|
if (moved && committedAt !== null && committedAt.getTime() > 0)
|
|
1460
1978
|
this.lastBatch = { appliedAt: Date.now(), committedAt: committedAt.getTime() };
|
|
1461
|
-
// the ledger
|
|
1462
|
-
|
|
1979
|
+
// the rows, as the ledger has them now; with a position, a table the batch wrote is current again (its error, if
|
|
1980
|
+
// any, cleared), and so is every table the stream keeps current whose error was the replication's (the stream's
|
|
1981
|
+
// failure, the preparation's): the position covers each of them, changed in the batch or not
|
|
1982
|
+
for (const [name, rows] of rowsAfter) {
|
|
1463
1983
|
const st = this.states.get(name);
|
|
1464
|
-
if (st === undefined
|
|
1984
|
+
if (st === undefined)
|
|
1465
1985
|
continue;
|
|
1466
|
-
|
|
1467
|
-
const rows = Number((await w.runAndReadAll(`SELECT count(*) FROM ${ident(t.schema)}.${ident(t.table)}`)).getRowsJS()[0][0]);
|
|
1468
|
-
this.countedAt.set(name, Date.now());
|
|
1469
|
-
if (rows !== st.rows || st.error !== null)
|
|
1986
|
+
if (st.error !== null && lsn !== null)
|
|
1470
1987
|
await this.record({ ...st, rows, error: null });
|
|
1988
|
+
else
|
|
1989
|
+
this.states.set(name, { ...st, rows });
|
|
1471
1990
|
}
|
|
1991
|
+
if (lsn !== null) {
|
|
1992
|
+
for (const source of this.joinedNow) {
|
|
1993
|
+
const st = this.states.get(source);
|
|
1994
|
+
if (st !== undefined && st.phase === "copied" && st.error?.startsWith("replication: "))
|
|
1995
|
+
await this.record({ ...st, error: null });
|
|
1996
|
+
}
|
|
1997
|
+
}
|
|
1998
|
+
// every table the stream keeps current is current as of this batch's position (`Synced`); a piece of a large
|
|
1999
|
+
// transaction (no position) and the changes a catch-up held are not a position
|
|
2000
|
+
if (position !== null)
|
|
2001
|
+
for (const source of this.joinedNow)
|
|
2002
|
+
this.synced.set(source, { at: position.at, seconds: (Date.now() - started) / 1000, by: "stream" });
|
|
1472
2003
|
}
|
|
1473
2004
|
/**
|
|
1474
2005
|
* Rows an UPDATE left a large value unchanged in, every one carrying the same columns (`carried`, the key among them):
|
|
@@ -1498,11 +2029,13 @@ export class Sync {
|
|
|
1498
2029
|
* join against the table's key column (`deleteKeys`), then the rows inserted. No copy has a primary key,
|
|
1499
2030
|
* so there is no `INSERT OR REPLACE`; a key's row is the source's at the batch's end
|
|
1500
2031
|
* whatever the table held, as replaying needs (src/duck/stream.ts). A table with no key (none can be added:
|
|
1501
|
-
* src/cli/preflight.ts) is only appended to.
|
|
2032
|
+
* src/cli/preflight.ts) is only appended to. Answers what it did to the table's count: the rows inserted less the rows
|
|
2033
|
+
* its deletes removed (DuckDB's own count of each).
|
|
1502
2034
|
*/
|
|
1503
2035
|
async writeByKey(t, target, gone, whole, value) {
|
|
1504
2036
|
if (gone.length === 0 && whole.length === 0)
|
|
1505
|
-
return;
|
|
2037
|
+
return 0;
|
|
2038
|
+
let delta = 0;
|
|
1506
2039
|
const w = this.store.writer;
|
|
1507
2040
|
const cols = t.columns.map((c) => c.name);
|
|
1508
2041
|
const list = cols.map(ident).join(", ");
|
|
@@ -1520,15 +2053,16 @@ export class Sync {
|
|
|
1520
2053
|
}
|
|
1521
2054
|
if (whole.length)
|
|
1522
2055
|
await w.run(`INSERT INTO ${WRITE_KEYS} SELECT ${keys} FROM ${WRITE_ROWS}`);
|
|
1523
|
-
await deleteKeys(w, t, target, WRITE_KEYS, t.key);
|
|
2056
|
+
delta -= await deleteKeys(w, t, target, WRITE_KEYS, t.key);
|
|
1524
2057
|
await w.run(`DROP TABLE ${WRITE_KEYS}`);
|
|
1525
2058
|
}
|
|
1526
2059
|
if (whole.length) {
|
|
1527
2060
|
// and its stored JSON paths, computed from the rows' JSON (src/duck/kept.ts)
|
|
1528
2061
|
const stored = this.kept.stored(t.source, ident);
|
|
1529
|
-
await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`);
|
|
2062
|
+
delta += Number((await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`)).rowsChanged);
|
|
1530
2063
|
await w.run(`DROP TABLE ${WRITE_ROWS}`);
|
|
1531
2064
|
}
|
|
2065
|
+
return delta;
|
|
1532
2066
|
}
|
|
1533
2067
|
/** a copied table's columns' DuckDB types, as DESCRIBE spells them (what `duckValue` casts a value's text to) */
|
|
1534
2068
|
async columnTypes(t, target) {
|
|
@@ -1604,15 +2138,18 @@ export class Sync {
|
|
|
1604
2138
|
* whose attach throws is not one: it never detaches */
|
|
1605
2139
|
async attach() {
|
|
1606
2140
|
if (!this.attached) {
|
|
1607
|
-
//
|
|
1608
|
-
//
|
|
2141
|
+
// the extension ships with Mason (src/duck/extensions.ts): a LOAD from the platform's package, nothing downloaded;
|
|
2142
|
+
// without the package, INSTALL fetches it the first time on a machine (then ~/.duckdb/extensions), said once,
|
|
2143
|
+
// and offline the first copy fails here saying why
|
|
1609
2144
|
if (!this.loaded) {
|
|
2145
|
+
const x = this.store.extensions;
|
|
2146
|
+
if (!x.shipped)
|
|
2147
|
+
this.log(`Mason's extensions are not installed with it here: ${x.why}. The first sync on this machine downloads its postgres extension`);
|
|
1610
2148
|
try {
|
|
1611
|
-
await this.store.writer.run(
|
|
2149
|
+
await this.store.writer.run(loadPostgresSql(x));
|
|
1612
2150
|
}
|
|
1613
2151
|
catch (e) {
|
|
1614
|
-
throw new Error(
|
|
1615
|
-
"DuckDB's extension repository (then ~/.duckdb/extensions): connect to the internet once, and the copy is tried again at the next pass");
|
|
2152
|
+
throw new Error(extensionFailure(x, e.message));
|
|
1616
2153
|
}
|
|
1617
2154
|
this.loaded = true;
|
|
1618
2155
|
}
|
|
@@ -1663,40 +2200,76 @@ export class Sync {
|
|
|
1663
2200
|
});
|
|
1664
2201
|
this.snapshots.delete(source);
|
|
1665
2202
|
this.retries.delete(source);
|
|
2203
|
+
this.due.delete(source);
|
|
2204
|
+
this.lastPollMs.delete(source);
|
|
2205
|
+
this.recordedSync.delete(source);
|
|
2206
|
+
this.switched.delete(source);
|
|
1666
2207
|
this.retired.delete(source);
|
|
1667
2208
|
this.copiedAs.delete(source);
|
|
1668
2209
|
this.copiedColumns.delete(source);
|
|
1669
2210
|
this.states.delete(source);
|
|
1670
|
-
this.
|
|
1671
|
-
|
|
1672
|
-
|
|
1673
|
-
|
|
1674
|
-
|
|
2211
|
+
this.synced.delete(source);
|
|
2212
|
+
this.log(`dropped ${source}'s synced data: it is no longer added (mason add ${source} syncs it again from the source, whole)`);
|
|
2213
|
+
}
|
|
2214
|
+
// REPLICATION, as the tables' syncs say: the publication and slot made ready for the replicated tables and the stream
|
|
2215
|
+
// started — or, when none is replicated any more, the slot dropped (verified gone). Its failure is the replicated
|
|
2216
|
+
// tables' alone (not copied, not caught up: `work`), and the pass's, said once the polled tables are polled
|
|
2217
|
+
let replicationFailed = null;
|
|
2218
|
+
let replicationReady = false;
|
|
2219
|
+
for (const source of [...this.unpublished.keys()]) {
|
|
2220
|
+
const t = this.tables.get(source);
|
|
2221
|
+
if (t === undefined || !this.replicated(t))
|
|
2222
|
+
this.unpublished.delete(source);
|
|
1675
2223
|
}
|
|
1676
|
-
if (this.
|
|
2224
|
+
if (this.streams()) {
|
|
1677
2225
|
try {
|
|
1678
2226
|
await this.prepareReplication();
|
|
2227
|
+
replicationReady = true;
|
|
1679
2228
|
}
|
|
1680
2229
|
catch (e) {
|
|
1681
|
-
|
|
2230
|
+
// the way out on the table's own line: a failure that names none (a slot the database will not make, a role that
|
|
2231
|
+
// may not replicate) says the line that keeps the tables polled meanwhile — never a bare SQLSTATE's text
|
|
2232
|
+
const said = e instanceof Error ? e.message : String(e);
|
|
2233
|
+
const names = this.replicatedTables().map((t) => t.source);
|
|
2234
|
+
const m = `replication: ${said}${/sync: poll/.test(said) ? "" : ` — ${names.join(", ")} ${names.length === 1 ? "is" : "are"} NOT kept current until it is fixed: ` +
|
|
2235
|
+
`give ${names.length === 1 ? "its entry" : "their entries"} sync: poll <interval> (poll 5m) in the sources file to poll ${names.length === 1 ? "it" : "them"} instead`}`;
|
|
2236
|
+
this.event({ kind: "stream_failed", table: null, message: m });
|
|
2237
|
+
this.wholeFailure.prepare = m;
|
|
1682
2238
|
// a table copied before keeps serving its copy; one never copied is failed, so mason status says why
|
|
1683
2239
|
await this.onWriter(async () => {
|
|
1684
|
-
for (const t of this.
|
|
2240
|
+
for (const t of this.replicatedTables()) {
|
|
1685
2241
|
const st = this.states.get(t.source);
|
|
1686
2242
|
await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m });
|
|
1687
2243
|
}
|
|
1688
2244
|
});
|
|
1689
|
-
|
|
1690
|
-
throw e;
|
|
2245
|
+
replicationFailed = e;
|
|
1691
2246
|
}
|
|
2247
|
+
this.recovered("prepare", "the publication and the slot are ready again");
|
|
1692
2248
|
// the stream starts once the slot is known, and runs from then on whatever the passes and rebuilds do
|
|
1693
|
-
|
|
2249
|
+
if (replicationReady)
|
|
2250
|
+
this.startStream();
|
|
2251
|
+
}
|
|
2252
|
+
else if (this.replication !== null && (this.slotKnown || this.stream !== null)) {
|
|
2253
|
+
try {
|
|
2254
|
+
await this.retireReplication();
|
|
2255
|
+
}
|
|
2256
|
+
catch (e) {
|
|
2257
|
+
replicationFailed = new Error(`no table's sync is replication now, but replication slot ${this.replication.slot} could not be dropped (it holds the database's WAL; tried again at the next pass): ${e instanceof Error ? e.message : String(e)}`);
|
|
2258
|
+
}
|
|
2259
|
+
}
|
|
2260
|
+
if (this.tables.size === 0) {
|
|
2261
|
+
await this.rebuildSchema();
|
|
2262
|
+
if (replicationFailed !== null)
|
|
2263
|
+
throw replicationFailed;
|
|
2264
|
+
return;
|
|
1694
2265
|
}
|
|
1695
2266
|
try {
|
|
1696
2267
|
await this.onWriter(() => this.attach());
|
|
1697
2268
|
}
|
|
1698
2269
|
catch (e) {
|
|
1699
2270
|
const m = e instanceof Error ? e.message : String(e);
|
|
2271
|
+
this.event({ kind: "source_unreachable", table: null, message: `could not attach the database: ${m}` });
|
|
2272
|
+
this.wholeFailure.attach = m;
|
|
1700
2273
|
// a table copied before keeps serving its copy (and its place in the catalog): the source being away is its error,
|
|
1701
2274
|
// not a reason to copy it whole again when the source is back; one never copied is failed
|
|
1702
2275
|
await this.onWriter(async () => {
|
|
@@ -1707,18 +2280,22 @@ export class Sync {
|
|
|
1707
2280
|
});
|
|
1708
2281
|
// the copies serve while the source is away: the catalog lists them (on a fresh start, nothing else has)
|
|
1709
2282
|
await this.rebuildSchema();
|
|
1710
|
-
throw new Error(`
|
|
2283
|
+
throw new Error(`Mason could not attach ${this.source.database} on ${this.source.host}:${this.source.port}: ${m}`);
|
|
1711
2284
|
}
|
|
2285
|
+
this.recovered("attach", "the database is attached again");
|
|
1712
2286
|
try {
|
|
1713
|
-
await this.work();
|
|
2287
|
+
await this.work(replicationReady);
|
|
1714
2288
|
}
|
|
1715
2289
|
finally {
|
|
1716
2290
|
await this.onWriter(() => this.detach());
|
|
1717
2291
|
}
|
|
1718
2292
|
await this.rebuildSchema();
|
|
2293
|
+
if (replicationFailed !== null)
|
|
2294
|
+
throw replicationFailed;
|
|
1719
2295
|
}
|
|
1720
|
-
/** a pass's queueing and polls, the source attached
|
|
1721
|
-
|
|
2296
|
+
/** a pass's queueing and polls, the source attached; a replicated table is copied or caught up only once the publication
|
|
2297
|
+
* carries it and the slot exists (`replicationReady`): its changes are in the stream from before it joins */
|
|
2298
|
+
async work(replicationReady) {
|
|
1722
2299
|
const started = Date.now();
|
|
1723
2300
|
// a table never copied (or whose copy failed, or whose copy is out of date) is copied whole, beside the rest: the
|
|
1724
2301
|
// smallest first (`copyOrder`)
|
|
@@ -1731,33 +2308,27 @@ export class Sync {
|
|
|
1731
2308
|
// a table the named publication does not carry: copied, nothing would keep it current (prepareReplication)
|
|
1732
2309
|
if (this.unpublished.has(t.source))
|
|
1733
2310
|
continue;
|
|
2311
|
+
if (this.replicated(t) && !replicationReady)
|
|
2312
|
+
continue;
|
|
1734
2313
|
if (this.mode !== "recopy" && !this.needsCopy(t))
|
|
1735
2314
|
continue;
|
|
1736
2315
|
if (this.queued.has(t.source) || this.rebuildingNow?.source === t.source)
|
|
1737
2316
|
continue;
|
|
1738
2317
|
if (this.states.get(t.source)?.phase === "copied" && this.copiedAs.get(t.source) !== columnsKey(t.columns))
|
|
1739
|
-
this.log(`${t.source}: its columns are not the ones it was
|
|
2318
|
+
this.log(`${t.source}: its columns are not the ones it was synced with: it is read again whole`);
|
|
1740
2319
|
this.enqueue(t.source, { kind: "copy" });
|
|
1741
2320
|
}
|
|
1742
2321
|
// a table copied and not yet caught up (a slot new to the copy, or a run that stopped first): its one poll
|
|
1743
2322
|
for (const source of this.behind.keys()) {
|
|
1744
2323
|
const t = this.tables.get(source);
|
|
1745
2324
|
// a failed catch-up waits its turn as a failed copy does (`backOff`); the error it recorded says when
|
|
1746
|
-
if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now())
|
|
2325
|
+
if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now() && (!this.replicated(t) || replicationReady))
|
|
1747
2326
|
this.enqueue(source, { kind: "catch-up" });
|
|
1748
2327
|
}
|
|
1749
|
-
//
|
|
1750
|
-
|
|
1751
|
-
|
|
1752
|
-
const
|
|
1753
|
-
for (const t of big.filter((x) => !this.leftOut.has(x.source))) {
|
|
1754
|
-
this.log(`${t.source} (about ${t.rows.toLocaleString("en-US")} rows) is ${OVER_POLL_LIMIT}: it is served as copied, NOT kept current by the poll ` +
|
|
1755
|
-
`(each poll would scan it whole on the database) — mason sync ${t.source} syncs it now; mason start --replication keeps every table current`);
|
|
1756
|
-
this.leftOut.add(t.source);
|
|
1757
|
-
}
|
|
1758
|
-
for (const s of [...this.leftOut].filter((x) => !big.some((t) => t.source === x)))
|
|
1759
|
-
this.leftOut.delete(s);
|
|
1760
|
-
const kept = this.mode === "poll" ? polled.filter((t) => !big.includes(t)) : [];
|
|
2328
|
+
// POLL MODE: the tables due (`dueAt`: a polled one past its interval, one whose sync changed), and not a table whose
|
|
2329
|
+
// new columns could not be copied (its copy failed above): it is not polled with them
|
|
2330
|
+
const now = Date.now();
|
|
2331
|
+
const kept = this.mode !== "recopy" ? [...this.tables.values()].filter((t) => !this.replicated(t) && this.pollable(t) && (this.dueAt(t) ?? Infinity) <= now) : [];
|
|
1761
2332
|
for (const t of kept) {
|
|
1762
2333
|
if (this.stopped)
|
|
1763
2334
|
break;
|
|
@@ -1765,22 +2336,27 @@ export class Sync {
|
|
|
1765
2336
|
// or a swap may run between two. A table rebuilt or not yet caught up is not polled (decided on the turn itself);
|
|
1766
2337
|
// its catch-up polls it
|
|
1767
2338
|
try {
|
|
1768
|
-
await this.onWriter(async () =>
|
|
1769
|
-
|
|
2339
|
+
const found = await this.onWriter(async () => this.rebuildingNow?.source !== t.source && !this.behind.has(t.source) ? this.poll(this.writerLane, [t]) : []);
|
|
2340
|
+
for (const f of found)
|
|
2341
|
+
this.polledNow(t, f.ms);
|
|
1770
2342
|
}
|
|
1771
2343
|
catch (e) {
|
|
1772
2344
|
if (e instanceof Stopping) {
|
|
1773
|
-
this.log("poll stopped: Mason is stopping (rolled back; the
|
|
2345
|
+
this.log("poll stopped: Mason is stopping (rolled back; the synced tables serve as they were)");
|
|
1774
2346
|
return;
|
|
1775
2347
|
}
|
|
2348
|
+
// tried again after its interval (an `off` table whose sync changed: after the sync's), never at every pass
|
|
2349
|
+
const s = this.syncOf(t);
|
|
2350
|
+
this.due.set(t.source, Date.now() + (s.mode === "poll" ? s.seconds * 1000 : this.intervalMs));
|
|
1776
2351
|
// the table serves its copy as it was, and says why it is not current (mason status reads the ledger); the other
|
|
1777
2352
|
// tables are polled as ever
|
|
1778
|
-
const m = `the poll failed, and is tried again at the next one (the
|
|
2353
|
+
const m = `the poll failed, and is tried again at the next one (the synced data serves as it was): ${e instanceof Error ? e.message : String(e)}`;
|
|
1779
2354
|
this.log(`${t.source}: ${m}`);
|
|
2355
|
+
this.event({ kind: "poll_failed", table: t.source, message: m });
|
|
1780
2356
|
await this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
|
|
1781
2357
|
}
|
|
1782
2358
|
}
|
|
1783
|
-
if (this.mode
|
|
2359
|
+
if (this.mode !== "recopy")
|
|
1784
2360
|
await this.onWriter(() => this.readPolledAt());
|
|
1785
2361
|
this.lastPassMs = Date.now() - started;
|
|
1786
2362
|
}
|
|
@@ -1843,12 +2419,16 @@ export class Sync {
|
|
|
1843
2419
|
// the next pass says the same of every table, and queues it again
|
|
1844
2420
|
const m = `could not attach the database: ${e instanceof Error ? e.message : String(e)}`;
|
|
1845
2421
|
this.log(`${r.kind} of ${t.source} waits: ${m}`);
|
|
2422
|
+
this.event({ kind: "source_unreachable", table: t.source, message: `its ${r.kind} waits: ${m}` });
|
|
1846
2423
|
const st = this.states.get(t.source);
|
|
1847
2424
|
await this.onWriter(() => this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m }));
|
|
1848
2425
|
return null;
|
|
1849
2426
|
}
|
|
1850
2427
|
if (r.kind === "copy")
|
|
1851
2428
|
await this.copy(t);
|
|
2429
|
+
// the whole read is over, whatever comes next: what the table is said to be doing from here (`notStreamedReason`,
|
|
2430
|
+
// `refuseRebuilding`, `state`'s `rebuilding`)
|
|
2431
|
+
this.rebuildingNow = { source: t.source, kind: "catch-up" };
|
|
1852
2432
|
// a table its copy left behind (`swap`), or one marked so before: one poll, unless a copy of it comes first — the
|
|
1853
2433
|
// table as it is added NOW, which may have changed while it was copied (then copied again at once: `followUp`)
|
|
1854
2434
|
const now = this.tables.get(t.source);
|
|
@@ -1873,11 +2453,11 @@ export class Sync {
|
|
|
1873
2453
|
* copy (the copy's snapshot), or a slot new to the copy — on the rebuilds'
|
|
1874
2454
|
* connection, so the stream and the other tables' polls go on meanwhile. In replication mode the stream holds the
|
|
1875
2455
|
* table's changes from before the poll's snapshot and applies them after it (src/duck/stream.ts "why replaying is
|
|
1876
|
-
* safe"). A table
|
|
2456
|
+
* safe"). A table whose sync is `off` is caught up too: the one scan its rebuild costs, then nothing until asked.
|
|
1877
2457
|
*/
|
|
1878
2458
|
async catchUp(t) {
|
|
1879
2459
|
const mark = this.behind.get(t.source);
|
|
1880
|
-
const replication = this.
|
|
2460
|
+
const replication = this.replicated(t);
|
|
1881
2461
|
// J: from the writer's turn on, every batch's changes of this table are held (`applyStreamed`)
|
|
1882
2462
|
if (replication)
|
|
1883
2463
|
await this.onWriter(async () => { if (!this.held.has(t.source))
|
|
@@ -1890,8 +2470,9 @@ export class Sync {
|
|
|
1890
2470
|
// mason status — and is caught up again then: what was held is read again by that poll (still behind, nothing lost)
|
|
1891
2471
|
const failed = (what, e) => {
|
|
1892
2472
|
const when = this.backOff(t.source, Date.now() - started);
|
|
1893
|
-
const m = `${what} (the
|
|
2473
|
+
const m = `${what} (the synced data serves as it was; it is caught up by one poll again after ${when}): ${e instanceof Error ? e.message : String(e)}`;
|
|
1894
2474
|
this.log(`${t.source}: ${m}`);
|
|
2475
|
+
this.event({ kind: "catch_up_failed", table: t.source, message: m });
|
|
1895
2476
|
return this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
|
|
1896
2477
|
};
|
|
1897
2478
|
// the catch-up's poll reads the columns the copy holds (`asCopied`)
|
|
@@ -1914,6 +2495,7 @@ export class Sync {
|
|
|
1914
2495
|
await release();
|
|
1915
2496
|
return "copy again";
|
|
1916
2497
|
}
|
|
2498
|
+
this.polledNow(t, found[0].ms);
|
|
1917
2499
|
let saidJoin = false;
|
|
1918
2500
|
if (this.catchingUp?.source === t.source)
|
|
1919
2501
|
this.catchingUp.phase = "applying";
|
|
@@ -1924,7 +2506,7 @@ export class Sync {
|
|
|
1924
2506
|
this.held.delete(t.source);
|
|
1925
2507
|
if (held !== undefined && (held.truncated || held.keys.size > 0)) {
|
|
1926
2508
|
try {
|
|
1927
|
-
await this.applyChanges(new Map([[t.source, held]]),
|
|
2509
|
+
await this.applyChanges(new Map([[t.source, held]]), "held");
|
|
1928
2510
|
}
|
|
1929
2511
|
catch (e) {
|
|
1930
2512
|
const m = e instanceof Error ? e.message : String(e);
|
|
@@ -1932,7 +2514,7 @@ export class Sync {
|
|
|
1932
2514
|
// a numeric wider than its copy: copied again (`needsCopy`), and caught up after that copy
|
|
1933
2515
|
if (m.includes(`${WIDER} [${t.source}]`)) {
|
|
1934
2516
|
this.snapshots.delete(t.source);
|
|
1935
|
-
this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is
|
|
2517
|
+
this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is read again whole, wider (${m})`);
|
|
1936
2518
|
return "copy again";
|
|
1937
2519
|
}
|
|
1938
2520
|
heldFailure = e;
|
|
@@ -1961,9 +2543,10 @@ export class Sync {
|
|
|
1961
2543
|
return joined || null;
|
|
1962
2544
|
if (saidJoin)
|
|
1963
2545
|
return null;
|
|
2546
|
+
const s = this.syncOf(t);
|
|
1964
2547
|
const tail = replication ? (this.behind.has(t.source) ? "it is caught up again first (the slot was made again meanwhile)" : "the stream carries what follows once it runs again")
|
|
1965
|
-
:
|
|
1966
|
-
:
|
|
2548
|
+
: s.mode === "off" ? `its sync is off, so it is served as of now: mason sync ${t.source} syncs it again`
|
|
2549
|
+
: `the poll keeps it current (${syncWords(s)})`;
|
|
1967
2550
|
this.log(`caught up ${t.source} by one poll ${mark.why}: ${tail}`);
|
|
1968
2551
|
return null;
|
|
1969
2552
|
}
|
|
@@ -1977,12 +2560,15 @@ export class Sync {
|
|
|
1977
2560
|
async swap(t, side, o) {
|
|
1978
2561
|
const w = this.store.writer;
|
|
1979
2562
|
const target = `${ident(t.schema)}.${ident(t.table)}`;
|
|
2563
|
+
// the copy's time in the ledger and in this run, ONE value, and before `copied_at`, recorded after the swap
|
|
2564
|
+
// (`readSynced`: a copy recorded at or after its polled_at was last brought current by that copy)
|
|
2565
|
+
const at = new Date().toISOString();
|
|
1980
2566
|
await w.run("BEGIN TRANSACTION");
|
|
1981
2567
|
try {
|
|
1982
2568
|
await w.run(`DROP TABLE IF EXISTS ${target}`);
|
|
1983
2569
|
await w.run(`ALTER TABLE ${side} RENAME TO ${ident(t.table)}`);
|
|
1984
2570
|
if (o.snapshot !== null)
|
|
1985
|
-
await w.run(
|
|
2571
|
+
await w.run(this.pollStateSql(t, o.snapshot, at));
|
|
1986
2572
|
if (o.behind !== null)
|
|
1987
2573
|
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(o.behind)}, now())`);
|
|
1988
2574
|
// the copy is whole: nothing of it is left to continue
|
|
@@ -1998,19 +2584,33 @@ export class Sync {
|
|
|
1998
2584
|
}
|
|
1999
2585
|
// the copy's view of its visible columns, made again (its own transaction: DuckDB writes one database a transaction)
|
|
2000
2586
|
await this.kept.swapped(t.source);
|
|
2001
|
-
if (o.snapshot !== null)
|
|
2587
|
+
if (o.snapshot !== null) {
|
|
2002
2588
|
this.snapshots.set(t.source, o.snapshot);
|
|
2589
|
+
this.recordedSync.set(t.source, syncText(this.syncOf(t)));
|
|
2590
|
+
this.switched.delete(t.source);
|
|
2591
|
+
this.due.delete(t.source);
|
|
2592
|
+
// synced as of its copy, said at once (a review finding: until the next pass re-read the ledger, a table just
|
|
2593
|
+
// copied said "never synced"), and the value the ledger holds, so that read moves nothing
|
|
2594
|
+
this.polledAt.set(t.source, at);
|
|
2595
|
+
}
|
|
2003
2596
|
// the columns this copy holds, from its swap on: what every write to it is shaped by (`asCopied`)
|
|
2004
2597
|
this.copiedColumns.set(t.source, [...t.columns]);
|
|
2005
2598
|
if (o.behind !== null) {
|
|
2006
2599
|
this.behind.set(t.source, { why: o.behind });
|
|
2007
|
-
if (this.
|
|
2600
|
+
if (this.replicated(t))
|
|
2008
2601
|
this.held.set(t.source, noChanges());
|
|
2009
2602
|
}
|
|
2010
2603
|
this.retired.delete(t.source);
|
|
2011
2604
|
this.duckTypes.delete(t.source);
|
|
2605
|
+
// a new copy, a new stay, whatever the columns: what the stream decoded before this moment is not for it
|
|
2606
|
+
this.streamed.delete(t.source);
|
|
2012
2607
|
this.refreshStreamed();
|
|
2013
2608
|
}
|
|
2609
|
+
/** a table's watermark, the time and the sync it is polled or copied under (`recordedSync`): ONE row of `_mason.poll_state`.
|
|
2610
|
+
* `at` is the caller's, an ISO instant (`toISOString`, as `copied_at`), the very value it remembers (`polledAt`) */
|
|
2611
|
+
pollStateSql(t, snapshot, at) {
|
|
2612
|
+
return `INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state (source, snapshot, polled_at, sync) VALUES (${literal(t.source)}, ${literal(snapshot)}, ${literal(at)}::TIMESTAMPTZ, ${literal(syncText(this.syncOf(t)))})`;
|
|
2613
|
+
}
|
|
2014
2614
|
/**
|
|
2015
2615
|
* One poll of the tables given (the passes and `pollNow` give one at a time), in ONE DuckDB transaction over ONE
|
|
2016
2616
|
* source snapshot (the module's header), on the lane's connection — the writer's, or the rebuilds' for a catch-up —
|
|
@@ -2048,12 +2648,12 @@ export class Sync {
|
|
|
2048
2648
|
// (`adds`: the keys a stage still to be written adds to DuckDB's count)
|
|
2049
2649
|
const reconcile = async (t, target, from, adds = 0n) => {
|
|
2050
2650
|
const c0 = Date.now();
|
|
2051
|
-
doing = `${t.source}: counting its rows on the source and in
|
|
2651
|
+
doing = `${t.source}: counting its rows on the source and in the synced table`;
|
|
2052
2652
|
const counts = (await w.runAndReadAll(`SELECT (SELECT count FROM ${pq(`SELECT count(*) AS count FROM ${from}`)}), (SELECT count(*) FROM ${target})`)).getRowsJS()[0];
|
|
2053
2653
|
const countMs = Date.now() - c0;
|
|
2054
2654
|
let deleted = 0;
|
|
2055
2655
|
if (counts[0] !== counts[1] + adds) {
|
|
2056
|
-
doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with
|
|
2656
|
+
doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with the synced table's)`;
|
|
2057
2657
|
const on = t.key.map((k) => `t.${ident(k)} = k.${ident(k)}`).join(" AND ");
|
|
2058
2658
|
const gone = await w.run(`DELETE FROM ${target} WHERE rowid IN (SELECT t.rowid FROM ${target} t ANTI JOIN ${pq(`SELECT ${t.key.map(pgIdent).join(", ")} FROM ${from}`)} k ON ${on})`);
|
|
2059
2659
|
deleted = Number(gone.rowsChanged);
|
|
@@ -2061,6 +2661,9 @@ export class Sync {
|
|
|
2061
2661
|
return { rows: Number(counts[0]), deleted, countMs };
|
|
2062
2662
|
};
|
|
2063
2663
|
let snapshot = "";
|
|
2664
|
+
// THE POLL'S TIME, ONE VALUE: taken before its snapshot, so `syncedAt` never says more than the rows hold, written
|
|
2665
|
+
// with its watermark (`pollStateSql`) and remembered as written once that commits (`committed`, `polledAt`)
|
|
2666
|
+
const polledAt = new Date().toISOString();
|
|
2064
2667
|
await w.run("BEGIN TRANSACTION");
|
|
2065
2668
|
try {
|
|
2066
2669
|
await this.interruptible(lane, async () => {
|
|
@@ -2086,7 +2689,7 @@ export class Sync {
|
|
|
2086
2689
|
`${pq(`SELECT ${pgCols} FROM ${from} WHERE ${xmin8Sql(xmax)} >= ${watermark}`)}) AS polled(${t.columns.map((_, i) => `c${i}`).join(", ")})`)).rowsChanged);
|
|
2087
2690
|
if (staged > POLL_PIECE_ROWS) {
|
|
2088
2691
|
const c1 = Date.now();
|
|
2089
|
-
doing = `${t.source}: counting the keys its ${staged} changed row(s) add to
|
|
2692
|
+
doing = `${t.source}: counting the keys its ${staged} changed row(s) add to the synced table (the stage anti-joined with its keys)`;
|
|
2090
2693
|
const on = t.key.map((k) => `s.${ident(`c${t.columns.findIndex((c) => c.name === k)}`)} = t.${ident(k)}`).join(" AND ");
|
|
2091
2694
|
const adds = (await w.runAndReadAll(`SELECT count(*) FROM ${stage} s ANTI JOIN ${target} t ON ${on}`)).getRowsJS()[0][0];
|
|
2092
2695
|
const { rows, deleted, countMs } = await reconcile(t, target, from, adds);
|
|
@@ -2099,7 +2702,7 @@ export class Sync {
|
|
|
2099
2702
|
await writeStaged(w, t, target, stage, "", this.kept);
|
|
2100
2703
|
const c1 = Date.now();
|
|
2101
2704
|
const { rows, deleted, countMs } = await reconcile(t, target, from);
|
|
2102
|
-
await w.run(
|
|
2705
|
+
await w.run(this.pollStateSql(t, snapshot, polledAt));
|
|
2103
2706
|
await w.run(`DROP TABLE ${stage}`);
|
|
2104
2707
|
timed.push(`${t.source} ${staged} changed in ${c1 - c0} ms, counted in ${countMs} ms${deleted ? `, ${deleted} deleted in ${Date.now() - c1 - countMs} ms` : ""}`);
|
|
2105
2708
|
found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - c0 });
|
|
@@ -2122,22 +2725,25 @@ export class Sync {
|
|
|
2122
2725
|
// matched against the tables' own names (a name may hold a space)
|
|
2123
2726
|
const wider = e instanceof Error ? tables.find((t) => e.message.includes(`${WIDER} [${t.source}]`))?.source : undefined;
|
|
2124
2727
|
if (wider !== undefined && this.snapshots.delete(wider)) {
|
|
2125
|
-
this.log(`poll: ${wider} has a numeric value wider than its
|
|
2728
|
+
this.log(`poll: ${wider} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass`);
|
|
2126
2729
|
return [];
|
|
2127
2730
|
}
|
|
2128
2731
|
throw named(e);
|
|
2129
2732
|
}
|
|
2130
2733
|
// committed: the watermarks and the row counts move together (a pieced table's with its last piece)
|
|
2131
|
-
|
|
2734
|
+
// and the table's last sync (`Synced`): the poll's time as the ledger now holds it, and how long THIS table's poll took
|
|
2735
|
+
const committed = async (source, rows, ms) => {
|
|
2736
|
+
this.polledAt.set(source, polledAt);
|
|
2132
2737
|
const st = this.states.get(source);
|
|
2133
2738
|
if (st.rows !== rows || st.error !== null)
|
|
2134
2739
|
await this.record({ ...st, rows, error: null }, undefined, w);
|
|
2135
2740
|
this.snapshots.set(source, snapshot);
|
|
2741
|
+
this.synced.set(source, { at: polledAt, seconds: ms / 1000, by: lane === this.writerLane ? "poll" : "catch-up" });
|
|
2136
2742
|
};
|
|
2137
2743
|
try {
|
|
2138
2744
|
for (const [source, rows] of pending)
|
|
2139
2745
|
if (!pieced.some((p) => p.t.source === source))
|
|
2140
|
-
await committed(source, rows);
|
|
2746
|
+
await committed(source, rows, found.find((f) => f.table === source)?.ms ?? 0);
|
|
2141
2747
|
this.log(`poll: ${timed.join("; ")}`);
|
|
2142
2748
|
for (const { t, target, stage, staged, deleted, started } of pieced) {
|
|
2143
2749
|
const p0 = Date.now();
|
|
@@ -2148,7 +2754,7 @@ export class Sync {
|
|
|
2148
2754
|
doing = `${t.source}: writing piece ${i + 1} of ${n} of its ${staged} changed row(s) by key`;
|
|
2149
2755
|
await this.interruptible(lane, () => writeStaged(w, t, target, stage, ` WHERE rowid >= ${i * POLL_PIECE_ROWS} AND rowid < ${(i + 1) * POLL_PIECE_ROWS}`, this.kept));
|
|
2150
2756
|
if (i === n - 1) {
|
|
2151
|
-
await w.run(
|
|
2757
|
+
await w.run(this.pollStateSql(t, snapshot, polledAt));
|
|
2152
2758
|
}
|
|
2153
2759
|
doing = `${t.source}: committing piece ${i + 1} of ${n}`;
|
|
2154
2760
|
await w.run("COMMIT");
|
|
@@ -2159,7 +2765,7 @@ export class Sync {
|
|
|
2159
2765
|
}
|
|
2160
2766
|
}
|
|
2161
2767
|
const rows = pending.get(t.source);
|
|
2162
|
-
await committed(t.source, rows);
|
|
2768
|
+
await committed(t.source, rows, Date.now() - started);
|
|
2163
2769
|
found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - started });
|
|
2164
2770
|
this.log(`poll: ${t.source}: ${staged.toLocaleString("en-US")} changed row(s), written in ${n} pieces in ${Date.now() - p0} ms`);
|
|
2165
2771
|
}
|
|
@@ -2232,10 +2838,10 @@ export class Sync {
|
|
|
2232
2838
|
if (resumed === null) {
|
|
2233
2839
|
// the stream (or the other tables' polls) goes on: the slot holds no WAL for this copy, and the table joins once
|
|
2234
2840
|
// caught up from the copy's snapshot
|
|
2235
|
-
const beside = this.
|
|
2236
|
-
: this.mode
|
|
2237
|
-
const again = this.retired.has(t.source) ? `, again (
|
|
2238
|
-
this.log(`
|
|
2841
|
+
const beside = this.replicated(t) ? "beside the stream, which goes on keeping the other tables current: it is streamed once caught up by one poll from the snapshot it was read at"
|
|
2842
|
+
: this.mode !== "recopy" ? "beside the other tables' polls: once read, it is caught up by one poll from the snapshot it was read at" : "beside the other tables";
|
|
2843
|
+
const again = this.retired.has(t.source) ? `, again (it is ${RETIRED_LAYOUT})` : "";
|
|
2844
|
+
this.log(`syncing ${t.source}${again}: ${t.rows === null ? "rows unknown" : `about ${t.rows} row(s)`}, ${t.bytes} source bytes, ${beside}`);
|
|
2239
2845
|
}
|
|
2240
2846
|
let read;
|
|
2241
2847
|
try {
|
|
@@ -2245,7 +2851,7 @@ export class Sync {
|
|
|
2245
2851
|
if (!(e instanceof StartOver))
|
|
2246
2852
|
throw e;
|
|
2247
2853
|
// once: a second reason to start over is the copy's failure
|
|
2248
|
-
this.log(`starting the
|
|
2854
|
+
this.log(`starting the sync of ${t.source} over, from its first page: ${e.message}`);
|
|
2249
2855
|
kept = await this.forgetCopy(t, side);
|
|
2250
2856
|
read = await this.readChunks(t, side, null, (p) => { kept = p; });
|
|
2251
2857
|
}
|
|
@@ -2257,7 +2863,7 @@ export class Sync {
|
|
|
2257
2863
|
const [rows, keys] = (await w.runAndReadAll(`SELECT count(*), count(DISTINCT (${t.key.map(ident).join(", ")})) FROM ${side}`)).getRowsJS()[0];
|
|
2258
2864
|
if (rows !== keys) {
|
|
2259
2865
|
kept = await this.forgetCopy(t, side);
|
|
2260
|
-
throw new Error(`its
|
|
2866
|
+
throw new Error(`its synced data holds ${rows} rows but ${keys} distinct keys (Mason bug: report it, with mason logs): it was not swapped in, and is read again from its first page`);
|
|
2261
2867
|
}
|
|
2262
2868
|
}
|
|
2263
2869
|
// its stored JSON paths filled, so the table swapped in has every one whole (src/duck/kept.ts fillSide)
|
|
@@ -2265,14 +2871,17 @@ export class Sync {
|
|
|
2265
2871
|
// THE SWAP, on the writer's turn: one transaction, between two of the stream's or the polls'. It carries the copy's
|
|
2266
2872
|
// watermark (its first snapshot), removes its progress, and leaves the table behind until its catch-up (none in
|
|
2267
2873
|
// recopy mode: no poll)
|
|
2268
|
-
await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its
|
|
2874
|
+
await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its whole read" }));
|
|
2269
2875
|
this.copyLive.delete(t.source);
|
|
2270
2876
|
kept = null;
|
|
2271
2877
|
this.retries.delete(t.source);
|
|
2272
2878
|
const seconds = (Date.now() - started) / 1000;
|
|
2273
|
-
this.log(`
|
|
2879
|
+
this.log(`synced ${t.source}: ${p.rows} row(s), ${t.bytes} source bytes, in ${seconds.toFixed(2)} s (read in ${read.seconds.toFixed(2)} s` +
|
|
2274
2880
|
`${resumed === null ? "" : `, resumed at page ${readThrough(resumed.done) ?? p.pages} of ${resumed.pages}${pastPages(resumed.done)} with ${resumed.rows} row(s) in`}), stored as the source's scan delivered them, with no index`);
|
|
2275
|
-
|
|
2881
|
+
const copiedAt = new Date().toISOString();
|
|
2882
|
+
await this.record({ source: t.source, phase: "copied", rows: p.rows, bytes: t.bytes, seconds, copiedAt, error: null }, t.columns, w);
|
|
2883
|
+
this.synced.set(t.source, { at: copiedAt, seconds, by: "copy" });
|
|
2884
|
+
this.event({ kind: "copied", table: t.source, message: `synced: ${p.rows.toLocaleString("en-US")} row(s) in ${seconds.toFixed(2)} s` });
|
|
2276
2885
|
this.copiedAs.set(t.source, columnsKey(t.columns));
|
|
2277
2886
|
// in the catalog at once: a pass's other copies may take hours
|
|
2278
2887
|
await this.rebuildSchema();
|
|
@@ -2288,7 +2897,7 @@ export class Sync {
|
|
|
2288
2897
|
k = await this.forgetCopy(t, side);
|
|
2289
2898
|
}
|
|
2290
2899
|
catch (f) {
|
|
2291
|
-
this.log(`the
|
|
2900
|
+
this.log(`the sync of ${t.source} could not drop what it began (its next attempt continues it): ${f instanceof Error ? f.message : String(f)}`);
|
|
2292
2901
|
}
|
|
2293
2902
|
}
|
|
2294
2903
|
const through = k === null ? null : readThrough(k.done);
|
|
@@ -2296,7 +2905,7 @@ export class Sync {
|
|
|
2296
2905
|
if (e instanceof Stopping) {
|
|
2297
2906
|
// the chunks in flight rolled back: a table never copied continues at the next start, one copied before serves its
|
|
2298
2907
|
// copy as it was meanwhile
|
|
2299
|
-
this.log(`
|
|
2908
|
+
this.log(`sync of ${t.source} stopped: Mason is stopping (${where === null ? "rolled back; it is synced at the next start" : `the chunks in flight rolled back; it continues ${where} at the next start`})`);
|
|
2300
2909
|
if (first)
|
|
2301
2910
|
await this.record({ ...was, phase: "waiting", error: null }, undefined, w);
|
|
2302
2911
|
return;
|
|
@@ -2310,11 +2919,24 @@ export class Sync {
|
|
|
2310
2919
|
this.retries.set(t.source, { failures, at });
|
|
2311
2920
|
const when = `tried again after ${new Date(at).toISOString().slice(11, 16)} UTC (failure ${failures} in a row), or at once when Mason starts again or its columns or key change` +
|
|
2312
2921
|
(where === null ? "" : `; it continues ${where}`);
|
|
2313
|
-
this.log(`
|
|
2922
|
+
this.log(`sync of ${t.source} failed: ${m}; ${when}`);
|
|
2923
|
+
this.event({ kind: "copy_failed", table: t.source, message: `${m}; ${when}` });
|
|
2314
2924
|
// a table that was copied keeps serving its last copy; one never copied is failed
|
|
2315
2925
|
await this.record({ ...this.states.get(t.source), phase: first ? "failed" : "copied", error: `${m} (${when})` }, undefined, w);
|
|
2316
2926
|
}
|
|
2317
2927
|
}
|
|
2928
|
+
/** Where the copy's file is, the bytes free there, and what the copies running are projected to add (`DiskStatus`). */
|
|
2929
|
+
diskStatus() {
|
|
2930
|
+
const dir = path.dirname(this.store.file);
|
|
2931
|
+
const needed = [...this.copyLive.values()].map((l) => l.diskNeeded).filter((n) => n !== null);
|
|
2932
|
+
const projectedBytes = needed.length ? Math.round(needed.reduce((a, b) => a + b, 0)) : null;
|
|
2933
|
+
try {
|
|
2934
|
+
return { path: dir, freeBytes: this.diskFree(dir), projectedBytes };
|
|
2935
|
+
}
|
|
2936
|
+
catch (e) {
|
|
2937
|
+
return { path: dir, freeBytes: null, projectedBytes, error: `the free space where the synced data is could not be read: ${e instanceof Error ? e.message : String(e)}` };
|
|
2938
|
+
}
|
|
2939
|
+
}
|
|
2318
2940
|
/** The DuckDB file's bytes with its write-ahead log's (what a copy adds lands in the log until a checkpoint). */
|
|
2319
2941
|
fileBytes() {
|
|
2320
2942
|
let n = 0;
|
|
@@ -2329,16 +2951,18 @@ export class Sync {
|
|
|
2329
2951
|
}
|
|
2330
2952
|
return n;
|
|
2331
2953
|
}
|
|
2332
|
-
/** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null.
|
|
2333
|
-
|
|
2954
|
+
/** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null. What
|
|
2955
|
+
* it projects is kept on the copy's `live` (`diskStatus` says it). */
|
|
2956
|
+
diskShort(t, p, disk, live) {
|
|
2334
2957
|
const bytesNow = this.fileBytes();
|
|
2335
2958
|
const projected = copyDiskProjection({ pages: p.pages, done: p.done, doneAtStart: disk.doneAtStart, bytesAtStart: disk.bytesAtStart, bytesNow });
|
|
2959
|
+
live.diskNeeded = projected?.needed ?? null;
|
|
2336
2960
|
if (projected === null)
|
|
2337
2961
|
return null;
|
|
2338
2962
|
const free = this.diskFree(path.dirname(this.store.file));
|
|
2339
2963
|
if (projected.needed <= free)
|
|
2340
2964
|
return null;
|
|
2341
|
-
return `the
|
|
2965
|
+
return `the sync of ${t.source} would need about ${humanBytes(projected.needed)} more on disk than is free (${humanBytes(free)} free where ${this.store.file} is; ` +
|
|
2342
2966
|
`${pagesRead(p.done, p.pages)} of ${p.pages} pages read, ${humanBytes(bytesNow - disk.bytesAtStart)} added): free some disk, mason remove ${t.source}, or mason add ${t.source} --columns … with fewer columns`;
|
|
2343
2967
|
}
|
|
2344
2968
|
/**
|
|
@@ -2417,7 +3041,7 @@ export class Sync {
|
|
|
2417
3041
|
const pagesDone = pagesRead(p.done, p.pages);
|
|
2418
3042
|
const live = { pages: p.pages, chunksDone: p.done.length, pagesDone, rowsDone: p.rows, untaken: gapPages(gaps, p.pages),
|
|
2419
3043
|
chunkPages: nextChunkPages(ceiling, null, this.copyChunkSeconds), readers, reading: [], rate: null,
|
|
2420
|
-
rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null };
|
|
3044
|
+
rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null, diskNeeded: null };
|
|
2421
3045
|
this.copyLive.set(t.source, live);
|
|
2422
3046
|
for (let i = 1; i < readers; i++) {
|
|
2423
3047
|
const lane = { conn: await this.store.instance.connect(), inFlight: false };
|
|
@@ -2472,7 +3096,7 @@ export class Sync {
|
|
|
2472
3096
|
const node = Number((await c.runAndReadAll(`SELECT * FROM ${pq(`SELECT pg_relation_filenode(${pgLiteral(pgFrom)}::regclass)::bigint`)}`)).getRowsJS()[0][0]);
|
|
2473
3097
|
filenode ??= node;
|
|
2474
3098
|
if (node !== filenode) {
|
|
2475
|
-
throw new StartOver(`the source rewrote ${t.source} since its
|
|
3099
|
+
throw new StartOver(`the source rewrote ${t.source} since its sync began (its file changed: a VACUUM FULL, CLUSTER, TRUNCATE or column type change), so its rows moved between pages`);
|
|
2476
3100
|
}
|
|
2477
3101
|
const n = Number(r.rowsChanged);
|
|
2478
3102
|
await this.onChunk?.({ ...told, rows: n, phase: "read" });
|
|
@@ -2514,7 +3138,7 @@ export class Sync {
|
|
|
2514
3138
|
await c.run("ROLLBACK");
|
|
2515
3139
|
}
|
|
2516
3140
|
catch (r) {
|
|
2517
|
-
this.log(`a chunk of ${t.source}'s
|
|
3141
|
+
this.log(`a chunk of ${t.source}'s sync could not be rolled back after it failed (its connection is closed with the sync): ${r instanceof Error ? r.message : String(r)}`);
|
|
2518
3142
|
}
|
|
2519
3143
|
}
|
|
2520
3144
|
fail(e);
|
|
@@ -2522,14 +3146,14 @@ export class Sync {
|
|
|
2522
3146
|
}
|
|
2523
3147
|
// STOPPED BEFORE THE DISK FILLS: the chunks committed are kept, the copy fails with the way out and is tried
|
|
2524
3148
|
// again as a failed copy is (freed disk is used at the retry, or at once at the next start)
|
|
2525
|
-
const short = recopy ? null : this.diskShort(t, p, disk);
|
|
3149
|
+
const short = recopy ? null : this.diskShort(t, p, disk, live);
|
|
2526
3150
|
if (short !== null) {
|
|
2527
3151
|
fail(new Error(short));
|
|
2528
3152
|
return;
|
|
2529
3153
|
}
|
|
2530
3154
|
const at = readThrough(p.done);
|
|
2531
3155
|
if (Date.now() - lineAt >= 60_000 && at !== null) {
|
|
2532
|
-
this.log(`
|
|
3156
|
+
this.log(`syncing ${t.source}: page ${at} of ${p.pages}${pastPages(p.done)}, ${p.rows.toLocaleString("en-US")} row(s) in; ${chunks} chunk(s) in the last minute on ${readers} reader(s), ` +
|
|
2533
3157
|
`${(chunkMs / chunks / 1000).toFixed(1)} s each` + (rate === null ? "" : `; a reader reads ~${Math.round(rate.rowsPerSecond).toLocaleString("en-US")} rows/s ` +
|
|
2534
3158
|
`(${humanBytes(rate.bytesPerSecond)}/s of the table's pages), so the next chunk is ${live.chunkPages} page(s), ~${this.copyChunkSeconds} s (MASON_COPY_CHUNK_SECONDS; at most ${ceiling})`));
|
|
2535
3159
|
lineAt = Date.now();
|
|
@@ -2587,7 +3211,7 @@ export class Sync {
|
|
|
2587
3211
|
const ceiling = chunkPages(pages, t.rows, this.copyChunkRows);
|
|
2588
3212
|
const firstSize = nextChunkPages(ceiling, null, this.copyChunkSeconds);
|
|
2589
3213
|
const readers = this.mode === "recopy" ? 1 : Math.min(this.copyParallel, Math.max(1, Math.ceil(pages / firstSize)));
|
|
2590
|
-
this.log(`
|
|
3214
|
+
this.log(`syncing ${t.source}${snapshot === null ? "" : ` as of the source's snapshot ${snapshot}: what commits after it is caught up by one poll once read`}; ` +
|
|
2591
3215
|
`${pages} page(s) in chunks sized to ~${this.copyChunkSeconds} s of reading each at the rate measured here (MASON_COPY_CHUNK_SECONDS; at most ${ceiling} page(s)), ` +
|
|
2592
3216
|
`the first of ${firstSize} page(s) to measure it, ${readers === 1 ? "one at a time" : `${readers} at a time (MASON_COPY_PARALLEL), each a backend on the database`}`);
|
|
2593
3217
|
return { snapshot, columns: columnsKey(t.columns), exact, filenode: null, pages, done: [], rows: 0 };
|
|
@@ -2613,7 +3237,7 @@ export class Sync {
|
|
|
2613
3237
|
const row = r[0];
|
|
2614
3238
|
if (row === undefined || row[0] === null || row[1] !== columnsKey(t.columns) || !there) {
|
|
2615
3239
|
if (row !== undefined)
|
|
2616
|
-
this.log(`starting the
|
|
3240
|
+
this.log(`starting the sync of ${t.source} over, from its first page: ${!there ? "the table it was filling is gone" : row[0] === null ? "it was begun as a recopy" : "its columns are not the ones its sync was begun with"}`);
|
|
2617
3241
|
return this.forgetCopy(t, side);
|
|
2618
3242
|
}
|
|
2619
3243
|
const chunks = (await w.runAndReadAll(`SELECT first_page, end_page, "rows", filenode FROM ${MASON_SCHEMA}.copy_chunks WHERE source = ${literal(t.source)} ORDER BY first_page`)).getRowsJS();
|
|
@@ -2621,8 +3245,8 @@ export class Sync {
|
|
|
2621
3245
|
const rows = chunks.reduce((n, c) => n + Number(c[2]), 0);
|
|
2622
3246
|
const p = { snapshot: row[0], columns: row[1], exact: new Map(JSON.parse(row[2])), filenode: chunks.length ? Number(chunks[0][3]) : null, pages: Number(row[3]), done, rows };
|
|
2623
3247
|
const at = readThrough(done);
|
|
2624
|
-
this.log(at === null ? `resuming the
|
|
2625
|
-
: `resuming the
|
|
3248
|
+
this.log(at === null ? `resuming the sync of ${t.source} with every page read (${rows} row(s) in): its swap is next`
|
|
3249
|
+
: `resuming the sync of ${t.source} at page ${at} of ${p.pages}${pastPages(done)}, ${rows} row(s) in, held to its first snapshot ${p.snapshot}: the pages read before are not read again`);
|
|
2626
3250
|
const now = this.states.get(t.source);
|
|
2627
3251
|
if (now !== undefined && now.phase === "copying")
|
|
2628
3252
|
this.states.set(t.source, { ...now, rowsSoFar: rows, rowsEstimate: t.rows });
|
|
@@ -2647,7 +3271,7 @@ export class Sync {
|
|
|
2647
3271
|
await w.run(`ALTER TABLE ${MASON_SCHEMA}.copy_progress DROP COLUMN IF EXISTS ${ident(gone)}`);
|
|
2648
3272
|
await w.run("COMMIT");
|
|
2649
3273
|
if (moved > 0)
|
|
2650
|
-
this.log(`${moved}
|
|
3274
|
+
this.log(`${moved} sync(s) in progress were begun by a Mason that read one chunk at a time: each continues where it was`);
|
|
2651
3275
|
}
|
|
2652
3276
|
catch (e) {
|
|
2653
3277
|
await w.run("ROLLBACK");
|
|
@@ -2675,7 +3299,7 @@ export class Sync {
|
|
|
2675
3299
|
if (this.states.get(source)?.phase !== "copied")
|
|
2676
3300
|
continue;
|
|
2677
3301
|
this.retired.add(source);
|
|
2678
|
-
this.log(`${source}:
|
|
3302
|
+
this.log(`${source}: it is ${RETIRED_LAYOUT} — its rows are sorted on disk, and only the source has them in its own order, so it is read again whole; its synced data serves as it was until then`);
|
|
2679
3303
|
}
|
|
2680
3304
|
}
|
|
2681
3305
|
/** what a copy of the retired layout left beside the file (in that layout a copy was staged in a file of its own,
|
|
@@ -2684,7 +3308,7 @@ export class Sync {
|
|
|
2684
3308
|
const dir = path.dirname(this.store.file), base = `${path.basename(this.store.file)}.stage-`;
|
|
2685
3309
|
for (const f of fs.readdirSync(dir).filter((x) => x.startsWith(base))) {
|
|
2686
3310
|
fs.rmSync(path.join(dir, f), { recursive: true, force: true });
|
|
2687
|
-
this.log(`removed ${f}: a
|
|
3311
|
+
this.log(`removed ${f}: a sync's stage, from an older Mason (a sync now reads the source once, straight into its table)`);
|
|
2688
3312
|
}
|
|
2689
3313
|
}
|
|
2690
3314
|
/** The table's numerics the extension would read as doubles, each with the DECIMAL(38, s) scale that holds every value
|
|
@@ -2708,11 +3332,11 @@ export class Sync {
|
|
|
2708
3332
|
const scale = declared[1] !== undefined ? used ?? 0 : maxScale ?? 0;
|
|
2709
3333
|
const width = (digits ?? 1) + scale;
|
|
2710
3334
|
if ((special ?? 0) > 0) {
|
|
2711
|
-
this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which
|
|
3335
|
+
this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which the synced table's DECIMAL cannot hold: synced as a double`);
|
|
2712
3336
|
return;
|
|
2713
3337
|
}
|
|
2714
3338
|
if (scale > 38 || width > 38) {
|
|
2715
|
-
this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit
|
|
3339
|
+
this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit the synced table's DECIMAL(38): synced as a double (15 significant digits)`);
|
|
2716
3340
|
return;
|
|
2717
3341
|
}
|
|
2718
3342
|
if (declared[1] === undefined && minScale !== null && minScale !== maxScale)
|
|
@@ -2732,6 +3356,12 @@ export class Sync {
|
|
|
2732
3356
|
* writer only on its turn: a statement on a connection lands inside the transaction open there). The table's row is
|
|
2733
3357
|
* that flow's alone while it writes it (a rebuild's table is not polled or streamed until caught up) */
|
|
2734
3358
|
async record(s, columns, c = this.store.writer) {
|
|
3359
|
+
// A COPY CURRENT AGAIN after its poll, catch-up or stream failed (its error cleared by the flow that keeps it current,
|
|
3360
|
+
// never by a copy of it: that is its own `copied` event) — the one place an error is cleared, so the one place said
|
|
3361
|
+
const was = this.states.get(s.source);
|
|
3362
|
+
if (was !== undefined && was.error !== null && s.error === null && was.phase === "copied" && s.phase === "copied" && was.copiedAt === s.copiedAt) {
|
|
3363
|
+
this.event({ kind: "recovered", table: s.source, message: `current again (it had failed: ${was.error.split("\n")[0]})` });
|
|
3364
|
+
}
|
|
2735
3365
|
this.states.set(s.source, s);
|
|
2736
3366
|
const v = (x) => x === null ? "NULL" : typeof x === "number" ? String(x) : literal(x);
|
|
2737
3367
|
const sql = `INSERT OR REPLACE INTO ${MASON_SCHEMA}.copy_state (source, phase, "rows", bytes, seconds, copied_at, error, columns) VALUES (${literal(s.source)}, ${literal(s.phase)}, ${v(s.rows)}, ${v(s.bytes)}, ${v(s.seconds)}, ` +
|
|
@@ -2741,7 +3371,7 @@ export class Sync {
|
|
|
2741
3371
|
await c.run(sql);
|
|
2742
3372
|
}
|
|
2743
3373
|
catch (e) {
|
|
2744
|
-
this.log(`recording ${s.source}'s
|
|
3374
|
+
this.log(`recording ${s.source}'s sync state failed: ${e instanceof Error ? e.message : String(e)}`);
|
|
2745
3375
|
}
|
|
2746
3376
|
}
|
|
2747
3377
|
/** the catalog rebuilt (`schemaBuilt`), after any rebuild already under way */
|
|
@@ -2765,8 +3395,10 @@ export class Sync {
|
|
|
2765
3395
|
const types = await c.prepare(`SELECT * FROM ${ident(t.schema)}.${ident(t.table)} LIMIT 0`);
|
|
2766
3396
|
const duckTypes = new Map(Array.from({ length: types.columnCount }, (_, i) => [types.columnName(i), types.columnType(i)]));
|
|
2767
3397
|
types.destroySync();
|
|
2768
|
-
// the column's constraints and default
|
|
2769
|
-
//
|
|
3398
|
+
// the column's constraints and default — and below the table's keys and unique indexes — as the source had them
|
|
3399
|
+
// when the tables were last set (`setTables`: at mason start, and at mason add / remove; the run's 10-second
|
|
3400
|
+
// catalog read reaches the commands' facts, not this): what the copy recorded describes its types, which a
|
|
3401
|
+
// change of recopies
|
|
2770
3402
|
const now = new Map(t.columns.map((c) => [c.name, c]));
|
|
2771
3403
|
const columns = cols.filter((c) => duckTypes.has(c.name)).map((c) => pgColumn(c, duckTypes.get(c.name), now.get(c.name)));
|
|
2772
3404
|
const joins = t.foreignKeys.filter((f) => this.tables.has(f.target) && columns.some((c) => c.name === f.column))
|
|
@@ -2781,7 +3413,7 @@ export class Sync {
|
|
|
2781
3413
|
const key = JSON.stringify([tables, views]);
|
|
2782
3414
|
if (key !== this.schemaKey) {
|
|
2783
3415
|
this.schemaKey = key;
|
|
2784
|
-
this.currentSchema = { database: this.database, tables, ...(views.length > 0 ? { views } : {}) };
|
|
3416
|
+
this.currentSchema = { database: this.currentSchema.database, tables, ...(views.length > 0 ? { views } : {}) };
|
|
2785
3417
|
}
|
|
2786
3418
|
}
|
|
2787
3419
|
}
|