@mason-data/cli 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -73
- package/dist/src/api/error.js +3 -0
- package/dist/src/api/postgres/catalog-worker.js +49 -2
- package/dist/src/api/postgres/catalog.js +368 -25
- package/dist/src/api/postgres/encode.js +3 -1
- package/dist/src/api/postgres/explain.js +3 -2
- package/dist/src/api/postgres/listener.js +11 -3
- package/dist/src/api/postgres/protocol.js +39 -23
- package/dist/src/api/postgres/proxy.js +283 -0
- package/dist/src/api/postgres/schema.js +6 -0
- package/dist/src/api/postgres/server.js +1617 -245
- package/dist/src/api/postgres/translate.js +127 -10
- package/dist/src/api/request-log.js +18 -5
- package/dist/src/cdc/poll.js +5 -5
- package/dist/src/cli/background.js +47 -25
- package/dist/src/cli/bridge-stdin.js +101 -0
- package/dist/src/cli/cancel.js +75 -0
- package/dist/src/cli/command.js +170 -11
- package/dist/src/cli/commands.js +804 -209
- package/dist/src/cli/control.js +59 -0
- package/dist/src/cli/credentials.js +182 -0
- package/dist/src/cli/demo.js +20 -11
- package/dist/src/cli/deploy.js +574 -0
- package/dist/src/cli/dev.js +491 -130
- package/dist/src/cli/doctor.js +131 -38
- package/dist/src/cli/generated-files.js +143 -0
- package/dist/src/cli/guides.js +175 -36
- package/dist/src/cli/hosted.js +850 -0
- package/dist/src/cli/init.js +162 -65
- package/dist/src/cli/login.js +289 -0
- package/dist/src/cli/machine.js +13 -12
- package/dist/src/cli/main.js +11 -5
- package/dist/src/cli/mason.cjs +17 -0
- package/dist/src/cli/mcp.js +31 -22
- package/dist/src/cli/metrics.js +79 -0
- package/dist/src/cli/output-gate.js +176 -0
- package/dist/src/cli/page-share.js +96 -0
- package/dist/src/cli/pg-tls.js +14 -4
- package/dist/src/cli/pg-url.js +4 -0
- package/dist/src/cli/pin.js +19 -14
- package/dist/src/cli/pool-size.cjs +85 -0
- package/dist/src/cli/ports.js +2 -0
- package/dist/src/cli/preflight.js +138 -46
- package/dist/src/cli/project.js +142 -4
- package/dist/src/cli/readiness.js +76 -6
- package/dist/src/cli/registry.js +65 -38
- package/dist/src/cli/replica-hosted.js +665 -0
- package/dist/src/cli/replica.js +76 -0
- package/dist/src/cli/replication-checks.js +7 -5
- package/dist/src/cli/replication-mode.js +105 -26
- package/dist/src/cli/request-filter.js +134 -0
- package/dist/src/cli/roles.js +585 -0
- package/dist/src/cli/run-server.js +637 -48
- package/dist/src/cli/runs.js +1 -1
- package/dist/src/cli/security.js +67 -18
- package/dist/src/cli/serve.js +204 -46
- package/dist/src/cli/share-run.js +329 -0
- package/dist/src/cli/share.js +286 -0
- package/dist/src/cli/source-edit.js +75 -17
- package/dist/src/cli/source-role.js +344 -0
- package/dist/src/cli/sources.js +306 -47
- package/dist/src/cli/style.js +2 -1
- package/dist/src/cli/supervise.js +27 -24
- package/dist/src/cli/sync.js +55 -25
- package/dist/src/cli/token-hosted.js +337 -0
- package/dist/src/cli/tunnel.js +200 -0
- package/dist/src/cli/ui-ssh.js +460 -0
- package/dist/src/cli/ui.js +124 -17
- package/dist/src/cli/usage.js +128 -0
- package/dist/src/cli/user-hosted.js +299 -0
- package/dist/src/cli/user.js +573 -65
- package/dist/src/cli/version.js +9 -7
- package/dist/src/demo/seed.js +29 -0
- package/dist/src/duck/analyze.js +36 -20
- package/dist/src/duck/cancel.js +10 -1
- package/dist/src/duck/collation.js +266 -0
- package/dist/src/duck/constant-key.js +94 -0
- package/dist/src/duck/distinct-key.js +283 -26
- package/dist/src/duck/events.js +165 -0
- package/dist/src/duck/executor-worker.js +20 -11
- package/dist/src/duck/executor.js +47 -33
- package/dist/src/duck/explain.js +60 -46
- package/dist/src/duck/extensions.js +160 -0
- package/dist/src/duck/functions.js +461 -63
- package/dist/src/duck/gaps.js +48 -21
- package/dist/src/duck/generated-plan.js +1070 -0
- package/dist/src/duck/generated.js +906 -0
- package/dist/src/duck/grants.js +176 -0
- package/dist/src/duck/held-types.js +64 -0
- package/dist/src/duck/history.js +127 -0
- package/dist/src/duck/jsonb-order.js +9 -12
- package/dist/src/duck/kept-route.js +6 -5
- package/dist/src/duck/kept.js +15 -13
- package/dist/src/duck/late.js +1469 -40
- package/dist/src/duck/libc-bound.js +341 -0
- package/dist/src/duck/libc-order.js +892 -126
- package/dist/src/duck/pg-eval-worker.js +7 -1
- package/dist/src/duck/pg-eval.js +6 -4
- package/dist/src/duck/pgtree.js +774 -23
- package/dist/src/duck/plan.js +369 -47
- package/dist/src/duck/publication.js +20 -6
- package/dist/src/duck/requests.js +269 -93
- package/dist/src/duck/resolved.js +117 -2
- package/dist/src/duck/rewrite.js +4584 -580
- package/dist/src/duck/row-filter.js +655 -0
- package/dist/src/duck/statement.js +19 -0
- package/dist/src/duck/store.js +439 -177
- package/dist/src/duck/stream.js +19 -10
- package/dist/src/duck/sync.js +1411 -368
- package/dist/src/duck/token.js +4 -1
- package/dist/src/duck/types.js +69 -12
- package/dist/src/duck/usage.js +326 -0
- package/dist/src/duck/users.js +69 -15
- package/dist/src/duck/verified.js +161 -1
- package/dist/src/duck/views.js +5 -5
- package/dist/src/lib/database-name.js +62 -0
- package/dist/src/lib/database-url.js +55 -10
- package/dist/src/lib/dotenv.js +17 -4
- package/dist/src/lib/instant.js +50 -0
- package/dist/src/lib/log-buffer.js +77 -0
- package/dist/src/lib/log-read.js +196 -0
- package/dist/src/lib/pg-numeric.js +390 -49
- package/dist/src/lib/project-version.js +23 -23
- package/dist/src/lib/secret-mask.js +142 -0
- package/dist/src/lib/table-sync.js +282 -0
- package/dist/src/web/app.css +706 -66
- package/dist/src/web/app.js +331 -409
- package/dist/src/web/charts.js +229 -0
- package/dist/src/web/index.html +82 -18
- package/dist/src/web/pages/queries.js +192 -0
- package/dist/src/web/pages/status.js +611 -0
- package/dist/src/web/pages/syncs.js +155 -0
- package/dist/src/web/pages/tables.js +439 -0
- package/dist/src/web/pages/usage.js +397 -0
- package/dist/src/web/pages/users.js +219 -0
- package/dist/src/web/shared.js +584 -0
- package/dist/src/web/vendor/README.md +25 -0
- package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
- package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
- package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
- package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
- package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
- package/package.json +12 -4
package/dist/src/duck/sync.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import { MASON_SCHEMA } from "./store.js";
|
|
1
|
+
import { DEFAULT_MEMORY_RULE, freeBytesAt, MASON_SCHEMA, NO_ROLE_SESSION, pgErrorOf, SessionSetup } from "./store.js";
|
|
2
|
+
/** the session the generated tables' run passes the Mason functions: no slot of the cancel table (src/duck/cancel.ts
|
|
3
|
+
* `slotOf`), so never cancelled through it; a stop interrupts the run's connection instead */
|
|
4
|
+
const GENERATED_SESSION = "generated";
|
|
5
|
+
import { GeneratedTables } from "./generated.js";
|
|
2
6
|
import { KeptJsonPaths } from "./kept.js";
|
|
3
7
|
import { DuckDBEnumType, DuckDBListType } from "@duckdb/node-api";
|
|
4
8
|
import { wireColumn } from "./types.js";
|
|
@@ -8,9 +12,14 @@ import * as fs from "node:fs";
|
|
|
8
12
|
import * as path from "node:path";
|
|
9
13
|
import { PgConnection, formatLsn, parseLsn, pgLiteral } from "../cdc/pg-wire.js";
|
|
10
14
|
import { createSlot, currentLsn, dropSlot, readSlot } from "../cdc/source.js";
|
|
15
|
+
import { holdOf } from "./held-types.js";
|
|
11
16
|
import { addCounts, countsText, duckValue, foldInto, noChanges, noCounts, ReplicationStream } from "./stream.js";
|
|
12
17
|
import { didYouMean } from "../lib/did-you-mean.js";
|
|
13
|
-
import {
|
|
18
|
+
import { numericWiderThan } from "../lib/pg-numeric.js";
|
|
19
|
+
import { gapLine, namedPublicationVerdict, readNamedPublication } from "./publication.js";
|
|
20
|
+
import { extensionFailure, loadPostgresSql } from "./extensions.js";
|
|
21
|
+
import { errorCode, errorText, maskPasswordKeyword, messageOf } from "../lib/secret-mask.js";
|
|
22
|
+
import { BIG_TABLE_ROWS, intervalOf, msText, syncedText, syncText, syncWords } from "../lib/table-sync.js";
|
|
14
23
|
export function syncModeOf(env) {
|
|
15
24
|
const v = env.MASON_DUCK_SYNC || "poll";
|
|
16
25
|
if (v !== "poll" && v !== "recopy")
|
|
@@ -18,17 +27,70 @@ export function syncModeOf(env) {
|
|
|
18
27
|
return v;
|
|
19
28
|
}
|
|
20
29
|
/**
|
|
21
|
-
*
|
|
22
|
-
* (`
|
|
23
|
-
*
|
|
24
|
-
* the wire: on a test server a 22 M-row table cost the production replica 12 s of scan and 2 s of count at every poll.
|
|
25
|
-
* `mason sync <table>` polls one now (`Sync.pollNow`); `mason start --replication` keeps every table current, whatever
|
|
26
|
-
* its size.
|
|
30
|
+
* WHEN A POLLED TABLE IS POLLED NEXT: its interval after its last poll ended, stretched to four times that poll's own
|
|
31
|
+
* time (`nextPollDelayMs`) — so one table's polls never take more than a fifth of the source's time, whatever its
|
|
32
|
+
* `sync: poll <interval>` says (a 20 s poll of a table set to `poll 5s` waits 80 s).
|
|
27
33
|
*/
|
|
28
|
-
export
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
34
|
+
export function nextPollAt(endedMs, intervalSeconds, pollMs) {
|
|
35
|
+
return endedMs + nextPollDelayMs(intervalSeconds * 1000, pollMs);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* HOW LONG THE NEXT PASS WAITS: until the first polled table is due — and at least four times what the last pass took
|
|
39
|
+
* (its polls, one after another on the writer), so ALL the tables' polls together never take more than a fifth of the
|
|
40
|
+
* source's time, however many are due at once. Bounded by setTimeout's 2^31 - 1 ms: a later poll is waited for in
|
|
41
|
+
* steps, a pass that finds nothing due polling nothing.
|
|
42
|
+
*/
|
|
43
|
+
export function passDelayMs(earliestDueMs, nowMs, lastPassMs) {
|
|
44
|
+
return Math.min(2 ** 31 - 1, Math.max(0, earliestDueMs - nowMs, 4 * lastPassMs));
|
|
45
|
+
}
|
|
46
|
+
/** The least grace a polled or recopied table is given past its interval before it is behind (`keptPromise`): a pass
|
|
47
|
+
* attaches the source and queues on the writer, so a poll due now lands a little later. */
|
|
48
|
+
export const POLL_GRACE_FLOOR_SECONDS = 60;
|
|
49
|
+
/** How old a STREAMED table's synced data may be before it is behind (`keptPromise`): an idle healthy stream says it has
|
|
50
|
+
* everything the database wrote on the keepalive it asks for every 10 s (src/duck/stream.ts `sendStatus(…, true)`),
|
|
51
|
+
* so 120 s is twelve of them, and past `wal_sender_timeout`'s default 60 s: one reconnect never crosses it. */
|
|
52
|
+
export const REPLICATION_BEHIND_SECONDS = 120;
|
|
53
|
+
/**
|
|
54
|
+
* WHAT A TABLE'S `sync:` PROMISES: the age of its synced data past which it is BEHIND, and the facts that would make it
|
|
55
|
+
* so now — the one derivation (`Sync.keptOf` calls it; src/cli/readiness.ts `behindOf` compares it with the age at the
|
|
56
|
+
* moment asked). It reads what the schedule reads (`nextPollAt`'s table poll, `passDelayMs`'s last pass), never a second
|
|
57
|
+
* measure, and changes neither load rule: it says them.
|
|
58
|
+
* - `poll`: its interval plus max(60 s, twice its own last poll this run; 60 s when not measured, as after a start);
|
|
59
|
+
* - `recopy`: the run's interval plus max(60 s, twice the last pass);
|
|
60
|
+
* - `replication`: `REPLICATION_BEHIND_SECONDS`;
|
|
61
|
+
* - `off`: nothing (null): it is as of its last `mason sync`, and never behind.
|
|
62
|
+
*/
|
|
63
|
+
export function keptPromise(o) {
|
|
64
|
+
if (o.mode === "off")
|
|
65
|
+
return { promisedSeconds: null, lateWhy: null };
|
|
66
|
+
if (o.mode === "replication") {
|
|
67
|
+
return { promisedSeconds: REPLICATION_BEHIND_SECONDS, lateWhy: o.pending !== null ? o.pending : o.notRunning !== null ? `the stream is not running: ${o.notRunning}`
|
|
68
|
+
: "the stream has not had everything the database wrote since then (the Stream section says its lag)" };
|
|
69
|
+
}
|
|
70
|
+
const own = o.mode === "recopy" ? o.lastPassMs : o.lastPollMs;
|
|
71
|
+
const promisedSeconds = o.intervalSeconds + Math.max(POLL_GRACE_FLOOR_SECONDS, own === null ? 0 : (2 * own) / 1000);
|
|
72
|
+
const interval = o.intervalSeconds * 1000;
|
|
73
|
+
const fifth = "to keep the database's load to a fifth";
|
|
74
|
+
let lateWhy;
|
|
75
|
+
if (o.pending !== null)
|
|
76
|
+
lateWhy = o.pending;
|
|
77
|
+
else if (o.passFailed)
|
|
78
|
+
lateWhy = `the sync's last pass failed (mason logs --events says why): it is tried again every ${msText(interval)}`;
|
|
79
|
+
else if (o.mode === "recopy")
|
|
80
|
+
lateWhy = o.lastPassMs === 0 ? "it is read again whole at the next pass"
|
|
81
|
+
: `the last pass, which reads every table again whole, took ${msText(o.lastPassMs)}, and Mason waits four times a pass between passes ${fifth}`;
|
|
82
|
+
else if (4 * o.lastPassMs > interval && 4 * o.lastPassMs >= 4 * (o.lastPollMs ?? 0)) {
|
|
83
|
+
lateWhy = `every polled table's polls took ${msText(o.lastPassMs)} in the last pass, and Mason waits four times that between passes ${fifth}`;
|
|
84
|
+
}
|
|
85
|
+
else if (o.lastPollMs !== null && 4 * o.lastPollMs > interval) {
|
|
86
|
+
lateWhy = `its own poll took ${msText(o.lastPollMs)}, and Mason waits four times a table's poll before its next ${fifth}`;
|
|
87
|
+
}
|
|
88
|
+
else if (o.lastPollMs === null)
|
|
89
|
+
lateWhy = "it has not been polled since Mason started: it is polled at the next pass";
|
|
90
|
+
else
|
|
91
|
+
lateWhy = o.dueMs !== null && o.dueMs > o.now ? `its next poll is due ${clock(o.dueMs)} UTC` : "its poll is due, after the polls of the pass before it";
|
|
92
|
+
return { promisedSeconds, lateWhy };
|
|
93
|
+
}
|
|
32
94
|
/** a poll asked for that cannot run (`Sync.pollNow`), its reason in the asker's words: `code` is the CLI's failure code */
|
|
33
95
|
export class PollRefused extends Error {
|
|
34
96
|
code;
|
|
@@ -38,6 +100,19 @@ export class PollRefused extends Error {
|
|
|
38
100
|
export class Stopping extends Error {
|
|
39
101
|
constructor() { super("Mason is stopping"); }
|
|
40
102
|
}
|
|
103
|
+
/** How often the slot's WAL is read at most (`streamStatus`): a catalog view's row, on the connection the database's
|
|
104
|
+
* position is read on, so never a connection of its own — and never per poll or per batch. */
|
|
105
|
+
export const SLOT_READ_MS = 30_000;
|
|
106
|
+
/** The slot as `readSlot` read it, as `SlotWal` says it. */
|
|
107
|
+
export function slotWalOf(s, name, at) {
|
|
108
|
+
if (!s.exists)
|
|
109
|
+
return { error: `replication slot ${name} does not exist on the database (the next pass makes it again)` };
|
|
110
|
+
const status = s.walStatus === "reserved" || s.walStatus === "extended" || s.walStatus === "unreserved" || s.walStatus === "lost" ? s.walStatus : null;
|
|
111
|
+
// a lost slot's WAL was freed by Postgres (its restart_lsn is null): it holds none; any other null is not known
|
|
112
|
+
if (s.retainedBytes === null && status !== "lost")
|
|
113
|
+
return { error: `replication slot ${name} has no restart position yet: the WAL it holds is not known` };
|
|
114
|
+
return { walHeldBytes: s.retainedBytes === null ? 0 : Number(s.retainedBytes), walStatus: status, safeWalSizeBytes: s.safeBytes === null ? null : Number(s.safeBytes), at: new Date(at).toISOString() };
|
|
115
|
+
}
|
|
41
116
|
export const ident = (name) => `"${name.replaceAll('"', '""')}"`;
|
|
42
117
|
export const literal = (s) => `'${s.replaceAll("'", "''")}'`;
|
|
43
118
|
export const SRC = "_mason_source";
|
|
@@ -90,7 +165,15 @@ export async function writeStaged(w, t, target, stage, where, kept) {
|
|
|
90
165
|
* never converted in place, but copied again through the normal path (`needsCopy`), serving as it was meanwhile. Found
|
|
91
166
|
* by its constraint (`duckdb_constraints()`), which every copy of that layout carries and no copy of this one does.
|
|
92
167
|
*/
|
|
93
|
-
export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason:
|
|
168
|
+
export const RETIRED_LAYOUT = "stored in key order with an index, by an older Mason: read again whole, in heap order";
|
|
169
|
+
/**
|
|
170
|
+
* A COPY AN OLDER MASON MADE BEFORE A TYPE WAS HELD AS ITS TEXT (src/duck/held-types.ts): a column this Mason holds as the
|
|
171
|
+
* source's text (a `point`, a `money`, a range) stored as the extension's scan read it (a `point` as a STRUCT). Its
|
|
172
|
+
* record (`copy_state.columns`) was written by code that did not know the column's kind, so it cannot say so: found by
|
|
173
|
+
* what the copy physically is (`duckdb_columns()`), copied again through the normal path (`needsCopy`), serving as it
|
|
174
|
+
* was until its swap; never converted in place.
|
|
175
|
+
*/
|
|
176
|
+
export const HELD_LAYOUT = "stored by an older Mason with a column not held as the source's text: read again whole, each such column as its text";
|
|
94
177
|
/**
|
|
95
178
|
* How long a failed copy waits before it is tried again: four times the attempt, at least 5 minutes, doubled for each
|
|
96
179
|
* failure in a row, at most 6 hours (a copy whose sort failed for good read a production replica whole for
|
|
@@ -115,16 +198,16 @@ export function copyOrder(tables) {
|
|
|
115
198
|
* of a copy that is not coming (the test server: every start said `added <all four tables>: they are copied at the
|
|
116
199
|
* next pass` while all four were copied and streamed from the acknowledged position — read as a long re-copy that never
|
|
117
200
|
* came). ONE line: the tables grouped by state, each group `<n> table(s) <state>: <tables>`, the current ones first,
|
|
118
|
-
* those copied at the next pass last, each with its reason
|
|
201
|
+
* those copied at the next pass last, each with its reason, a current copy with how it is kept so (`keptWords`).
|
|
119
202
|
*/
|
|
120
|
-
export function addedText(tables
|
|
203
|
+
export function addedText(tables) {
|
|
121
204
|
const groups = new Map();
|
|
122
205
|
for (const t of tables) {
|
|
123
|
-
const [rank, state] = "copy" in t ? [3, `
|
|
124
|
-
: "behind" in t ? [2, `already
|
|
125
|
-
: "refused" in t ? [4, `not
|
|
126
|
-
: t.
|
|
127
|
-
: [0, `already
|
|
206
|
+
const [rank, state] = "copy" in t ? [3, `synced at the next pass (${t.copy})`]
|
|
207
|
+
: "behind" in t ? [2, `already synced, caught up by one poll first (${t.behind})`]
|
|
208
|
+
: "refused" in t ? [4, `not synced or streamed (${t.refused})`]
|
|
209
|
+
: t.off ? [1, `already synced, ${t.kept}`]
|
|
210
|
+
: [0, `already synced and ${t.kept}`];
|
|
128
211
|
const g = groups.get(state) ?? { rank, sources: [] };
|
|
129
212
|
g.sources.push(t.source);
|
|
130
213
|
groups.set(state, g);
|
|
@@ -148,6 +231,65 @@ export function chunkPages(pages, rows, targetRows = COPY_CHUNK_ROWS) {
|
|
|
148
231
|
return COPY_CHUNK_MAX_PAGES;
|
|
149
232
|
return Math.min(COPY_CHUNK_MAX_PAGES, Math.max(1, Math.round(targetRows / (rows / pages))));
|
|
150
233
|
}
|
|
234
|
+
/**
|
|
235
|
+
* A CHUNK'S OUT-OF-LINE BYTES ARE BOUNDED BY THE MEMORY (issue 96: a table of ~20,000 rows of ~68 KB of `jsonb` out of
|
|
236
|
+
* line has a heap of a few hundred pages, so its rows ceiling was the whole table, read as ONE chunk, which ran out of
|
|
237
|
+
* a 7.8 GiB limit). A heap page holds a TOAST pointer of ~20 bytes where the row holds its value, so the rows ceiling
|
|
238
|
+
* (`chunkPages`, measured on heap rows) cannot see what a chunk reads; the source's catalog can: its TOAST table's bytes
|
|
239
|
+
* (`pg_table_size` of `reltoastrelid`), a heap page's share of them, read each attempt (`Sync.outOfLineBytes`). The
|
|
240
|
+
* ceiling is then at most `budget / out-of-line bytes a page` pages, at least one: the budget is the memory limit
|
|
241
|
+
* (`store.memoryLimitBytes`) times `COPY_CHUNK_MEMORY_SHARE`, shared by the readers that may read at once
|
|
242
|
+
* (`MASON_COPY_PARALLEL`). A table with nothing out of line gets exactly the rows ceiling.
|
|
243
|
+
*
|
|
244
|
+
* MEASURED (DuckDB 2.0.0, `duckdb_memory()` sampled every 50 ms; docs/DESIGN.md "Large values out of line"): a chunk
|
|
245
|
+
* of `jsonb` held ~5.5 times its bytes on the source while it read (366 MiB for one 67 MB page), one of uncompressed
|
|
246
|
+
* text 1 to 2 times; so a chunk is given an eighth of the limit, the rest left to the readers beside it, the stream,
|
|
247
|
+
* the polls and the queries, and a chunk that still runs out goes on smaller (`readChunks`).
|
|
248
|
+
*/
|
|
249
|
+
export const COPY_CHUNK_MEMORY_SHARE = 1 / 8;
|
|
250
|
+
/** the bytes one reader's chunk may read out of line: the memory limit's share (`COPY_CHUNK_MEMORY_SHARE`), split
|
|
251
|
+
* between the readers that may read at once */
|
|
252
|
+
export function chunkBudgetBytes(memoryLimitBytes, readers) {
|
|
253
|
+
return (memoryLimitBytes * COPY_CHUNK_MEMORY_SHARE) / Math.max(1, readers);
|
|
254
|
+
}
|
|
255
|
+
/** THE CEILING of a chunk of a table of `pages` heap pages, about `rows` rows and `outOfLine` bytes out of line
|
|
256
|
+
* (its TOAST table's; null: unknown): the rows ceiling (`chunkPages`), held to `budget` bytes out of line, at least
|
|
257
|
+
* one page */
|
|
258
|
+
export function chunkCeiling(pages, rows, outOfLine, budget, targetRows = COPY_CHUNK_ROWS) {
|
|
259
|
+
const byRows = chunkPages(pages, rows, targetRows);
|
|
260
|
+
if (outOfLine === null || outOfLine <= 0 || pages <= 0)
|
|
261
|
+
return byRows;
|
|
262
|
+
return Math.min(byRows, Math.max(1, Math.floor(budget / (outOfLine / pages))));
|
|
263
|
+
}
|
|
264
|
+
/**
|
|
265
|
+
* NO TABLE IS REFUSED BY ARITHMETIC (fix round 1 of PR 380): whether the engine can write a table's values is the
|
|
266
|
+
* engine's to say, at the write (53200 at a chunk's read or commit, `readChunks`), never a formula over the source's
|
|
267
|
+
* catalog. A refusal decided up front from the TOAST bytes in a row group, measured on a few rows of ~100 KB, refused
|
|
268
|
+
* 140,000 rows of ~2.6 KB through a 128 MB limit that the base synced in one chunk. What stays open: a checkpoint
|
|
269
|
+
* merging a table's small row groups of LARGE values that do not fit invalidates the database (issue 371).
|
|
270
|
+
*/
|
|
271
|
+
/** WHY A TABLE IS NOT SYNCED IN THIS MEMORY (`readChunks`): its rows, the limit as set, and the ways out — in the
|
|
272
|
+
* operator's words, never the engine's own remedies (its threads, its settings) */
|
|
273
|
+
export function tooLargeText(source, what, memoryLimitBytes) {
|
|
274
|
+
return `the rows of ${source} are too large for the memory the synced data may use: ${what}, in ${humanBytes(memoryLimitBytes)} ` +
|
|
275
|
+
`(MASON_DUCKDB_MEMORY; unset, ${DEFAULT_MEMORY_RULE}). Raise MASON_DUCKDB_MEMORY and start mason again, or ` +
|
|
276
|
+
`mason add ${source} --columns … without its widest columns`;
|
|
277
|
+
}
|
|
278
|
+
/** A CHUNK'S RANGE GIVEN BACK to what is left to read (`readChunks`: its chunk ran out of memory and rolled back):
|
|
279
|
+
* put among `gaps` in page order and joined to the gaps it meets, so the gaps still tile what no chunk committed —
|
|
280
|
+
* no page twice, none skipped */
|
|
281
|
+
export function giveBack(gaps, r) {
|
|
282
|
+
gaps.push({ ...r });
|
|
283
|
+
gaps.sort((a, b) => a.first - b.first);
|
|
284
|
+
for (let i = 0; i + 1 < gaps.length;) {
|
|
285
|
+
const a = gaps[i], b = gaps[i + 1];
|
|
286
|
+
if (a.end !== null && a.end === b.first) {
|
|
287
|
+
gaps.splice(i, 2, { first: a.first, end: b.end });
|
|
288
|
+
continue;
|
|
289
|
+
}
|
|
290
|
+
i++;
|
|
291
|
+
}
|
|
292
|
+
}
|
|
151
293
|
/**
|
|
152
294
|
* HOW LONG A CHUNK SHOULD TAKE TO READ, in seconds (`MASON_COPY_CHUNK_SECONDS`, 1 to 300, default 12:
|
|
153
295
|
* a laptop copying a 20 M-row table from a database over the internet showed `0 of ~20,126,276 rows` for minutes, then
|
|
@@ -163,8 +305,8 @@ export function copyChunkSecondsOf(env) {
|
|
|
163
305
|
return COPY_CHUNK_SECONDS;
|
|
164
306
|
const n = Number(v);
|
|
165
307
|
if (!/^\d+(\.\d+)?$/.test(v.trim()) || n < COPY_CHUNK_SECONDS_MIN || n > COPY_CHUNK_SECONDS_MAX) {
|
|
166
|
-
throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a
|
|
167
|
-
`about that long to read, and a stop reads at most that much again, so set it lower for a
|
|
308
|
+
throw new Error(`MASON_COPY_CHUNK_SECONDS=${JSON.stringify(v)} is not a number of seconds from ${COPY_CHUNK_SECONDS_MIN} to ${COPY_CHUNK_SECONDS_MAX}: each of a first sync's chunks is sized to take ` +
|
|
309
|
+
`about that long to read, and a stop reads at most that much again, so set it lower for a first sync that moves more often or higher for fewer, larger chunks, or unset it for ${COPY_CHUNK_SECONDS}`);
|
|
168
310
|
}
|
|
169
311
|
return n;
|
|
170
312
|
}
|
|
@@ -345,7 +487,7 @@ export function copyParallelOf(env) {
|
|
|
345
487
|
return COPY_PARALLEL;
|
|
346
488
|
const n = Number(v);
|
|
347
489
|
if (!/^\d+$/.test(v.trim()) || n < 1 || n > COPY_PARALLEL_MAX) {
|
|
348
|
-
throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a
|
|
490
|
+
throw new Error(`MASON_COPY_PARALLEL=${JSON.stringify(v)} is not a number of readers from 1 to ${COPY_PARALLEL_MAX}: each reads a first sync's chunks on a backend of the database's own, ` +
|
|
349
491
|
`so set it to 1 (one at a time, the lightest on the database) up to ${COPY_PARALLEL_MAX}, or unset it for ${COPY_PARALLEL}`);
|
|
350
492
|
}
|
|
351
493
|
return n;
|
|
@@ -362,14 +504,46 @@ export function streamRetryDelayMs(intervalMs, failures) {
|
|
|
362
504
|
}
|
|
363
505
|
/** what Mason's backends on the source are called (pg_stat_activity.application_name): the copy's readers and the polls */
|
|
364
506
|
export const SOURCE_APPLICATION_NAME = "mason-data";
|
|
365
|
-
/**
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
507
|
+
/**
|
|
508
|
+
* THE ENGINE NEVER HOLDS THE SOURCE'S PASSWORD IN A STRING IT REPEATS (issue 599). DuckDB's `postgres` extension
|
|
509
|
+
* repeats an ATTACH's path whole in its error (`Unable to connect to Postgres at "<path>": …`), and every connection
|
|
510
|
+
* reads it in `duckdb_databases()`; a run whose source restarted under it wrote `password='…'` to its log, its ledger,
|
|
511
|
+
* `mason status` and the web page. So the path is `attachPath`'s, which takes nothing but the sslmode, and the
|
|
512
|
+
* credentials go through the extension's SECRET MANAGER (`sourceSecretSql`): a temporary, in-memory secret of TYPE
|
|
513
|
+
* postgres, made on the writer's turn just before the ATTACH that names it and dropped right after it, whatever the
|
|
514
|
+
* ATTACH did (`attach`). Measured on the extension pinned in cli/extensions.json: its error says the path alone,
|
|
515
|
+
* `duckdb_secrets()` says `password=redacted`, the attachment keeps the credentials it merged at the ATTACH (a copy's
|
|
516
|
+
* readers and a reconnect after a restart connect with the secret gone), and it quotes each value for libpq itself.
|
|
517
|
+
* Not PGPASSWORD: the process's environment is every child's, and setting it races the engine's threads reading it.
|
|
518
|
+
* Whatever a dependency repeats one day, the sync's exits mask it (src/lib/secret-mask.ts).
|
|
519
|
+
*/
|
|
520
|
+
export function attachPath(s) {
|
|
521
|
+
const v = (x) => `'${x.replace(/\\/g, "\\\\").replace(/'/g, "\\'")}'`;
|
|
522
|
+
return [`sslmode=${v(s.sslmode)}`, `application_name=${v(SOURCE_APPLICATION_NAME)}`].join(" ");
|
|
523
|
+
}
|
|
524
|
+
/** THE ATTACH THE SYNC RUNS: the secret path's, always. A test that shows the mask alone (tests/integration/duck-secret.itest.ts)
|
|
525
|
+
* replaces it through `attachSeamForTest`, which a run never imports: no option, setting or environment reaches it */
|
|
526
|
+
const secretAttachSql = (s) => `ATTACH ${literal(attachPath(s))} AS ${SRC} (TYPE postgres, SECRET ${SOURCE_SECRET}, READ_ONLY)`;
|
|
527
|
+
let attachSqlOf = secretAttachSql;
|
|
528
|
+
export const attachSeamForTest = {
|
|
529
|
+
/** replaces the ATTACH (null: the secret path's again), for a test's process alone */
|
|
530
|
+
set(f) { attachSqlOf = f ?? secretAttachSql; },
|
|
531
|
+
};
|
|
532
|
+
/** the secret the ATTACH names: an explicit name, so no other ATTACH takes it (only `__default_postgres` is taken
|
|
533
|
+
* unnamed), and a wire session's planned SQL cannot name it */
|
|
534
|
+
export const SOURCE_SECRET = "mason_source_credentials";
|
|
535
|
+
/** the source's credentials as the extension's secret: temporary (in memory, never written to disk) and replaced, its
|
|
536
|
+
* password left out when there is none (libpq then reads PGPASSWORD or a pgpass file, as it did before) */
|
|
537
|
+
export function sourceSecretSql(s) {
|
|
538
|
+
return `CREATE OR REPLACE TEMPORARY SECRET ${SOURCE_SECRET} (TYPE postgres, HOST ${literal(s.host)}, PORT ${literal(String(s.port))}, ` +
|
|
539
|
+
`DATABASE ${literal(s.database)}, USER ${literal(s.user)}${s.password ? `, PASSWORD ${literal(s.password)}` : ""})`;
|
|
370
540
|
}
|
|
371
541
|
/** a table's columns as a copy is made with them: their names and source types, in order */
|
|
372
|
-
const columnsKey = (columns) => JSON.stringify(columns.map((c) =>
|
|
542
|
+
const columnsKey = (columns) => JSON.stringify(columns.map((c) => {
|
|
543
|
+
// what the copy holds is part of the shape: a copy made before a column's type was held as its text is read again
|
|
544
|
+
const hold = holdOf(c.type, c.typeKind);
|
|
545
|
+
return hold === "same" && c.domain !== true ? [c.name, c.type] : [c.name, c.type, hold, ...(c.domain === true ? ["domain"] : [])];
|
|
546
|
+
}));
|
|
373
547
|
/** `postgres_query` over the attached source: a read of Postgres's own SQL in the DuckDB transaction's snapshot */
|
|
374
548
|
const pq = (sql) => `postgres_query(${literal(SRC)}, ${literal(sql)})`;
|
|
375
549
|
/**
|
|
@@ -384,7 +558,17 @@ const pq = (sql) => `postgres_query(${literal(SRC)}, ${literal(sql)})`;
|
|
|
384
558
|
* differ (`1` beside `1.50`); the copy answers them all with s decimals, also said.
|
|
385
559
|
*/
|
|
386
560
|
const NUMERIC_TYPE = /^numeric(?:\((\d+)(?:,\s*(\d+))?\))?$/;
|
|
387
|
-
const WIDER = "mason: a numeric value wider than its
|
|
561
|
+
const WIDER = "mason: a numeric value wider than its synced column";
|
|
562
|
+
/** Whether a ROLLBACK's `e` says the engine had ended the transaction itself ("cannot rollback - no transaction is
|
|
563
|
+
* active", after a COMMIT or a read out of memory): a decision, so it reads the raw words (`messageOf`) */
|
|
564
|
+
export function endedTransaction(e) {
|
|
565
|
+
return /no transaction is active/.test(messageOf(e));
|
|
566
|
+
}
|
|
567
|
+
/** Whether `e` is a numeric of `source` wider than its synced column (`WIDER`): a decision, so it reads the error's raw
|
|
568
|
+
* words (`messageOf`), never the words Mason repeats (`errorText`) — the stream's, the catch-up's and the poll's alike */
|
|
569
|
+
export function widerIn(e, source) {
|
|
570
|
+
return messageOf(e).includes(`${WIDER} [${source}]`);
|
|
571
|
+
}
|
|
388
572
|
/** a source numeric the extension reads as a DOUBLE: no precision, or more than DuckDB's 38 digits */
|
|
389
573
|
function readsAsDouble(type) {
|
|
390
574
|
const m = NUMERIC_TYPE.exec(type);
|
|
@@ -398,16 +582,9 @@ function fitsDecimal(column, scale, source, significant) {
|
|
|
398
582
|
return `CASE WHEN length(${decimals}) > ${scale} OR length(regexp_extract(${v}, '^-?(\\d+)', 1)) > ${38 - scale} ` +
|
|
399
583
|
`OR ${v} IN ('NaN', 'Infinity', '-Infinity') THEN error(${literal(`${WIDER} [${source}]`)}) ELSE ${v} END AS ${v}`;
|
|
400
584
|
}
|
|
401
|
-
/** a numeric's text that does not fit DECIMAL(38, scale): `fitsDecimal`'s rule, for a value the stream brings
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
return true;
|
|
405
|
-
const m = /^-?(\d*)(?:\.(\d+))?$/.exec(text);
|
|
406
|
-
if (m === null)
|
|
407
|
-
return true;
|
|
408
|
-
const decimals = significant ? (m[2] ?? "").replace(/0+$/, "") : m[2] ?? "";
|
|
409
|
-
return decimals.length > scale || m[1].length > 38 - scale;
|
|
410
|
-
}
|
|
585
|
+
/** a numeric's text that does not fit DECIMAL(38, scale): `fitsDecimal`'s rule, for a value the stream brings (the one
|
|
586
|
+
* rule, src/lib/pg-numeric.ts `numericWiderThan`) */
|
|
587
|
+
const widerThan = (text, scale, significant) => numericWiderThan(text, scale, significant);
|
|
411
588
|
/** The DuckDB type a copied column is stored as: the extension's own, but a JSON for a json / jsonb (it reads them as text). */
|
|
412
589
|
/** a char(n) column (format_type's `character(n)`): the extension reads its values with the padding trimmed */
|
|
413
590
|
const CHAR_TYPE = /^character\((\d+)\)$/;
|
|
@@ -417,10 +594,12 @@ function storedValue(c) {
|
|
|
417
594
|
return m ? `rpad(${ident(c.name)}, ${m[1]}, ' ')` : ident(c.name);
|
|
418
595
|
}
|
|
419
596
|
function storedType(c, scanned) {
|
|
420
|
-
|
|
597
|
+
const hold = holdOf(c.type, c.typeKind);
|
|
598
|
+
return hold === "text" ? "VARCHAR" : hold === "text[]" ? "VARCHAR[]" : /^jsonb?$/.test(c.type) ? "JSON" : scanned;
|
|
421
599
|
}
|
|
422
|
-
/** the order the queue takes them in: a catch-up is one poll that makes a copied table current
|
|
423
|
-
|
|
600
|
+
/** the order the queue takes them in: a catch-up is one poll that makes a copied table current; a generated table's
|
|
601
|
+
* re-run reads what the copies hold, so it goes behind them */
|
|
602
|
+
const REBUILD_ORDER = ["catch-up", "copy", "generated"];
|
|
424
603
|
export class Sync {
|
|
425
604
|
store;
|
|
426
605
|
source;
|
|
@@ -429,12 +608,24 @@ export class Sync {
|
|
|
429
608
|
mode;
|
|
430
609
|
/** each polled table's watermark: the snapshot of the last poll that stored it */
|
|
431
610
|
snapshots = new Map();
|
|
432
|
-
/** how long the last pass took: the next waits four times that
|
|
611
|
+
/** how long the last pass took: the next waits at least four times that (its polls' share of the source's time) */
|
|
433
612
|
lastPassMs = 0;
|
|
613
|
+
/** each polled table's next poll (ms since the epoch: its last poll's end plus its interval, stretched to four times
|
|
614
|
+
* that poll — `nextPollDelayMs`); absent: from its last poll's time in the ledger (`dueAt`). One pass polls the
|
|
615
|
+
* tables due and the timer waits for the next one due (`nextPassDelay`), never a timer per table: a pass also
|
|
616
|
+
* queues the copies and drops removed tables, and attaches the source once for all of it — timers of their own would
|
|
617
|
+
* each attach it, and race those for the writer's turn */
|
|
618
|
+
due = new Map();
|
|
619
|
+
/** each table's sync as its last poll or copy recorded it (`_mason.poll_state.sync`; null: recorded before syncs were) */
|
|
620
|
+
recordedSync = new Map();
|
|
621
|
+
/** the tables whose sync changed since they were last polled (`setTables`): kept, and caught up by ONE poll now */
|
|
622
|
+
switched = new Set();
|
|
623
|
+
/** how long each table's last poll took, this run (ms): its next waits four times that, and mason add estimates by it */
|
|
624
|
+
lastPollMs = new Map();
|
|
434
625
|
/** each table whose copy failed: the failures in a row, and when it is tried again (`copyRetryDelayMs`) */
|
|
435
626
|
retries = new Map();
|
|
436
627
|
/** each copied table in a layout this Mason no longer keeps (`RETIRED_LAYOUT`), found at `open`: copied again */
|
|
437
|
-
retired = new
|
|
628
|
+
retired = new Map();
|
|
438
629
|
/** the catalog's own connection (`rebuildSchema`): it sees only what a copy or poll has committed, and is never
|
|
439
630
|
* held by a copy's statement, so a table is served the moment its copy commits */
|
|
440
631
|
catalog = null;
|
|
@@ -442,6 +633,8 @@ export class Sync {
|
|
|
442
633
|
schemaBuild = Promise.resolve();
|
|
443
634
|
tables = new Map();
|
|
444
635
|
views = [];
|
|
636
|
+
/** the source's collations (src/cli/preflight.ts), which the catalog declares where it lacks one (`setCollations`) */
|
|
637
|
+
collations = [];
|
|
445
638
|
states = new Map();
|
|
446
639
|
/** each copied table's columns as its copy was made with them (`columnsKey`): a change copies it again */
|
|
447
640
|
copiedAs = new Map();
|
|
@@ -455,10 +648,17 @@ export class Sync {
|
|
|
455
648
|
attached = false;
|
|
456
649
|
loaded = false;
|
|
457
650
|
stopped = false;
|
|
458
|
-
/** REPLICATION
|
|
459
|
-
* to the
|
|
460
|
-
* runs, the last commit
|
|
651
|
+
/** REPLICATION: the publication and slot the `replication` tables are read through (`owned`: the publication is
|
|
652
|
+
* Mason's own, made and kept equal to the replicated tables; else the operator named it, and it is only read —
|
|
653
|
+
* src/duck/publication.ts), null in a run that cannot stream (the demo's); the stream while it runs, the last commit
|
|
654
|
+
* DuckDB applied */
|
|
461
655
|
replication;
|
|
656
|
+
/** told when the slot is made (true) and when it is dropped, verified gone (false): the run records it, so `mason
|
|
657
|
+
* destroy` drops what was made by its own name (src/cli/dev.ts) */
|
|
658
|
+
onReplication;
|
|
659
|
+
/** the last moment the stream had everything the database wrote (a keepalive with nothing pending, a batch applied):
|
|
660
|
+
* a streamed table's copy is current as of then (`keptOf`'s syncedAt) */
|
|
661
|
+
streamCurrentAt = null;
|
|
462
662
|
stream = null;
|
|
463
663
|
appliedLsn = 0n;
|
|
464
664
|
/** the slot has a row in `_mason.stream_state`: what this copy applied of it is known */
|
|
@@ -472,8 +672,13 @@ export class Sync {
|
|
|
472
672
|
held = new Map();
|
|
473
673
|
/** REPLICATION: what the stream applies, by source — the tables joined, and those held — read at each change */
|
|
474
674
|
streamed = new Map();
|
|
675
|
+
/** the last `StreamedTable.stay` given (`refreshStreamed`) */
|
|
676
|
+
lastStay = 0;
|
|
475
677
|
/** a NAMED publication's gaps: each added table it does not carry, with the refusal (neither copied nor streamed) */
|
|
476
678
|
unpublished = new Map();
|
|
679
|
+
/** the error each table a named publication did not carry was given (`settlePrepared`): cleared by the pass that finds
|
|
680
|
+
* it carried again, and by nothing else that pass */
|
|
681
|
+
gapErrors = new Map();
|
|
477
682
|
/** a named publication that does not publish TRUNCATE has been said (once a run) */
|
|
478
683
|
truncateSaid = false;
|
|
479
684
|
/** the last pass failed: in replication mode it is tried again after the interval, the stream started by it */
|
|
@@ -486,12 +691,16 @@ export class Sync {
|
|
|
486
691
|
* (a pass kicked sooner leaves it stopped) */
|
|
487
692
|
streamFailures = 0;
|
|
488
693
|
streamAt = 0;
|
|
489
|
-
/** when each
|
|
490
|
-
|
|
491
|
-
|
|
694
|
+
/** when each copy was last brought current, and how (`Synced`): seeded at `open` from the ledger, moved by every
|
|
695
|
+
* swap, poll commit and stream batch */
|
|
696
|
+
synced = new Map();
|
|
697
|
+
/** where the sync's failures and notable events go (src/duck/events.ts `SyncEvents.record`): never the writer's turn */
|
|
698
|
+
onEvent;
|
|
699
|
+
/** when each table was last copied or polled, its `syncedAt`: `_mason.poll_state.polled_at` and nothing else — the
|
|
700
|
+
* value `pollStateSql` wrote is the one set here once its transaction commits, and `readPolledAt` only moves a time
|
|
701
|
+
* forward, so nothing moves it back (a time read at the commit stood until that read replaced it: a `syncedAt` that
|
|
702
|
+
* went backwards). A copy's is its watermark's, taken before the swap; `copied_at`, after it, is `Synced.at`'s */
|
|
492
703
|
polledAt = new Map();
|
|
493
|
-
/** the tables over the poll's row limit already said in the log (said once, when first left out) */
|
|
494
|
-
leftOut = new Set();
|
|
495
704
|
/** the store's writer: the stream's batches, the polls, the source's attachment and every swap, one turn at a time */
|
|
496
705
|
writerLane;
|
|
497
706
|
/** the rebuilds' own connection (`open`): a copy or a catch-up, beside the writer's work */
|
|
@@ -505,7 +714,6 @@ export class Sync {
|
|
|
505
714
|
/** the source attached for this many users (a pass, a rebuild): DETACHed when the last ends (`detach`) */
|
|
506
715
|
sourceUsers = 0;
|
|
507
716
|
currentSchema;
|
|
508
|
-
database;
|
|
509
717
|
/** what the catalog last built from: replaced (a new object) only when it changes */
|
|
510
718
|
schemaKey = "";
|
|
511
719
|
/** REPLICATION, WHERE THE STREAM IS (`streamStatus`, the stream's log): the changes it applied this run by table, and
|
|
@@ -540,15 +748,29 @@ export class Sync {
|
|
|
540
748
|
catchingUp = null;
|
|
541
749
|
/** the stream's last failure and when it starts again; cleared when it starts */
|
|
542
750
|
streamFailure = null;
|
|
751
|
+
/** A WHOLE-SYNC FAILURE NOT YET OVER, as its event said it (`table: null`): the stream stopped, over when it next applies
|
|
752
|
+
* a batch; the publication and slot could not be made ready, over when a pass makes them so; the database's attach,
|
|
753
|
+
* over when an attach succeeds. Its end is a `recovered` event of no table, which lets the next such failure be said again
|
|
754
|
+
* (src/duck/events.ts: a failure said word for word within an hour is one event unless it recovered between) */
|
|
755
|
+
wholeFailure = { stream: null, prepare: null, attach: null };
|
|
543
756
|
/** the database's position as last read (`databasePosition`), and the read in flight */
|
|
544
757
|
dbPosition = null;
|
|
758
|
+
/** the slot's WAL as last read (`databasePosition`, at most every SLOT_READ_MS), and when */
|
|
759
|
+
slotWal = null;
|
|
545
760
|
dbPositionRead = null;
|
|
546
761
|
/** THE STORED JSON PATHS (src/duck/kept.ts): every write below computes them; the plans read them */
|
|
547
762
|
kept;
|
|
763
|
+
/** THE GENERATED TABLES (src/duck/generated.ts): every commit that changes a copy marks them, the rebuild queue runs them */
|
|
764
|
+
generated;
|
|
765
|
+
/** the connection their runs read and write on (`open`), set up as a session: what `stop` interrupts */
|
|
766
|
+
generatedLane = null;
|
|
767
|
+
/** the generated lane's session set-up: settled before each plan and run (`GeneratedHost.settle`) */
|
|
768
|
+
generatedSession = null;
|
|
769
|
+
/** the review's check's own connection and its set-up, never the run's (`GeneratedHost.checkConnection`) */
|
|
770
|
+
generatedCheck = null;
|
|
548
771
|
constructor(store, source, database, log, intervalMs, mode = "poll", replication = null, options = {}) {
|
|
549
772
|
this.store = store;
|
|
550
773
|
this.source = source;
|
|
551
|
-
this.database = database;
|
|
552
774
|
this.log = log;
|
|
553
775
|
this.intervalMs = intervalMs;
|
|
554
776
|
this.mode = mode;
|
|
@@ -556,16 +778,18 @@ export class Sync {
|
|
|
556
778
|
this.copyChunkRows = options.copyChunkRows ?? COPY_CHUNK_ROWS;
|
|
557
779
|
this.copyParallel = options.copyParallel ?? COPY_PARALLEL;
|
|
558
780
|
if (!Number.isInteger(this.copyParallel) || this.copyParallel < 1 || this.copyParallel > COPY_PARALLEL_MAX) {
|
|
559
|
-
throw new Error(`a
|
|
781
|
+
throw new Error(`a first sync reads with 1 to ${COPY_PARALLEL_MAX} readers, not ${this.copyParallel} (Mason bug: report it; copyParallelOf reads MASON_COPY_PARALLEL)`);
|
|
560
782
|
}
|
|
561
783
|
this.copyChunkSeconds = options.copyChunkSeconds ?? COPY_CHUNK_SECONDS;
|
|
562
784
|
if (!(this.copyChunkSeconds > 0))
|
|
563
|
-
throw new Error(`a
|
|
785
|
+
throw new Error(`a first sync's chunk takes a positive number of seconds, not ${this.copyChunkSeconds} (Mason bug: report it; copyChunkSecondsOf reads MASON_COPY_CHUNK_SECONDS)`);
|
|
564
786
|
this.onChunk = options.onChunk;
|
|
565
|
-
this.
|
|
787
|
+
this.onEvent = options.onEvent ?? (() => undefined);
|
|
788
|
+
this.diskFree = options.diskFree ?? freeBytesAt;
|
|
566
789
|
if (mode === "replication" && replication === null)
|
|
567
790
|
throw new Error("replication mode needs its slot and publication (Mason bug: report it)");
|
|
568
791
|
this.replication = replication;
|
|
792
|
+
this.onReplication = options.onReplication;
|
|
569
793
|
this.currentSchema = { database, tables: [] };
|
|
570
794
|
this.writerLane = { conn: store.writer, inFlight: false };
|
|
571
795
|
this.kept = new KeptJsonPaths(store, {
|
|
@@ -576,6 +800,55 @@ export class Sync {
|
|
|
576
800
|
rebuilding: (source) => this.rebuildingNow?.source === source,
|
|
577
801
|
log: (line) => this.log(line),
|
|
578
802
|
}, { enabled: options.json?.enabled ?? true, ...options.json });
|
|
803
|
+
const generatedLane = () => {
|
|
804
|
+
if (this.generatedLane === null)
|
|
805
|
+
throw new Error("the generated tables' connection is not open (Mason bug: report it)");
|
|
806
|
+
return this.generatedLane;
|
|
807
|
+
};
|
|
808
|
+
this.generated = new GeneratedTables(store, {
|
|
809
|
+
onWriter: (f) => this.onWriter(f),
|
|
810
|
+
connection: () => generatedLane().conn,
|
|
811
|
+
settle: async () => {
|
|
812
|
+
if (this.generatedSession === null)
|
|
813
|
+
throw new Error("the generated tables' connection is not open (Mason bug: report it)");
|
|
814
|
+
await this.generatedSession.settle(NO_ROLE_SESSION);
|
|
815
|
+
},
|
|
816
|
+
checkConnection: () => {
|
|
817
|
+
if (this.generatedCheck === null)
|
|
818
|
+
throw new Error("the generated tables' check connection is not open (Mason bug: report it)");
|
|
819
|
+
return this.generatedCheck.conn;
|
|
820
|
+
},
|
|
821
|
+
settleCheck: async () => {
|
|
822
|
+
if (this.generatedCheck === null)
|
|
823
|
+
throw new Error("the generated tables' check connection is not open (Mason bug: report it)");
|
|
824
|
+
await this.generatedCheck.setup.settle(NO_ROLE_SESSION);
|
|
825
|
+
},
|
|
826
|
+
enqueue: (name) => this.enqueue(name, { kind: "generated" }),
|
|
827
|
+
interruptible: (f) => this.interruptible(generatedLane(), f),
|
|
828
|
+
added: (name) => this.addedAs(name),
|
|
829
|
+
syncedAt: (base) => this.synced.get(base)?.at ?? null,
|
|
830
|
+
rebuildCatalog: () => this.rebuildSchema(),
|
|
831
|
+
schema: () => this.currentSchema,
|
|
832
|
+
retryDelayMs: copyRetryDelayMs,
|
|
833
|
+
stopping: (e) => e instanceof Stopping,
|
|
834
|
+
log: (line) => this.log(line),
|
|
835
|
+
event: (e) => this.event(e),
|
|
836
|
+
}, options.generated ?? {});
|
|
837
|
+
}
|
|
838
|
+
/**
|
|
839
|
+
* A BASE IS THERE WHEN THE CATALOG HOLDS IT (issue 387), the generated tables' `added` (src/duck/generated.ts
|
|
840
|
+
* `GeneratedHost.added`): `copied` only when the copy is recorded copied AND the catalog's schema (`currentSchema`, what
|
|
841
|
+
* the listener's catalog is replaced from and every definition is planned over) holds the table; a table the sources
|
|
842
|
+
* file adds that the catalog does not hold yet is `adding`, whatever its copy's phase. The copy records `copied` before
|
|
843
|
+
* its rebuild of the catalog (which reads that record), and the rebuild waits behind any other: said `copied` from the
|
|
844
|
+
* phase alone, a definition planned in between met 42P01 for a synced table and was refused, and a run could be planned
|
|
845
|
+
* over a catalog without its base. The one place this is decided: the check and `ready` read it.
|
|
846
|
+
*/
|
|
847
|
+
addedAs(name) {
|
|
848
|
+
const t = this.tables.get(name);
|
|
849
|
+
if (t !== undefined)
|
|
850
|
+
return this.states.get(name)?.phase === "copied" && this.currentSchema.tables.some((x) => x.schema === t.schema && x.name === t.table) ? "copied" : "adding";
|
|
851
|
+
return this.views.some((v) => `${v.schema ?? "public"}.${v.name}` === name) ? "view" : null;
|
|
579
852
|
}
|
|
580
853
|
/** the rebuilds' connection: open from `open` until `stop` */
|
|
581
854
|
get rebuilder() {
|
|
@@ -583,6 +856,30 @@ export class Sync {
|
|
|
583
856
|
throw new Error("the sync's rebuild connection is not open (Mason bug: report it)");
|
|
584
857
|
return this.rebuilderLane;
|
|
585
858
|
}
|
|
859
|
+
/** an event of the sync, to its record (src/duck/events.ts), which never throws into the sync; a sink that does is
|
|
860
|
+
* said, and the sync goes on (its state and its log line hold what happened) */
|
|
861
|
+
event(e) {
|
|
862
|
+
try {
|
|
863
|
+
this.onEvent(e);
|
|
864
|
+
}
|
|
865
|
+
catch (x) {
|
|
866
|
+
this.log(`the sync's event (${e.kind}${e.table ? ` ${e.table}` : ""}) could not be recorded: ${this.errorText(x)}`);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
/** A CAUGHT ERROR'S WORDS, AS THE SYNC REPEATS THEM (src/lib/secret-mask.ts `errorText`): every engine or driver
|
|
870
|
+
* error becomes text here, once, where it enters — a log line, an event, a table's `error`, an answer is built from
|
|
871
|
+
* it — a `password=` value masked, nothing else changed (no error the sync meets holds the password: it travels in
|
|
872
|
+
* the temporary secret alone, whose own error is never repeated). A decision never reads these words: it reads
|
|
873
|
+
* `messageOf`, the raw ones. tests/secret-mask.test.ts holds src/duck/sync.ts and stream.ts to both */
|
|
874
|
+
errorText(e, o = {}) { return errorText(e, o); }
|
|
875
|
+
/** a whole-sync failure (`wholeFailure`) is over: said once, as a `recovered` event of no table */
|
|
876
|
+
recovered(what, words) {
|
|
877
|
+
const was = this.wholeFailure[what];
|
|
878
|
+
if (was === null)
|
|
879
|
+
return;
|
|
880
|
+
this.wholeFailure[what] = null;
|
|
881
|
+
this.event({ kind: "recovered", table: null, message: `${words} (it had failed: ${was.split("\n")[0]})` });
|
|
882
|
+
}
|
|
586
883
|
/** `f` on the writer's next turn: once every job queued before it has ended, so no other job's statement lands inside
|
|
587
884
|
* its transaction. A job never queues another and waits for it (it would wait for itself) */
|
|
588
885
|
onWriter(f) {
|
|
@@ -597,7 +894,9 @@ export class Sync {
|
|
|
597
894
|
// the re-sort's count of rows written out of key order (in the earlier layout): gone with the key order
|
|
598
895
|
for (const gone of ["written", "counted"])
|
|
599
896
|
await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.copy_state DROP COLUMN IF EXISTS ${gone}`);
|
|
600
|
-
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ)`);
|
|
897
|
+
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.poll_state (source VARCHAR PRIMARY KEY, snapshot VARCHAR, polled_at TIMESTAMPTZ, sync VARCHAR)`);
|
|
898
|
+
// the sync each table was last polled or copied under (src/lib/table-sync.ts syncText): a change is caught up by one poll
|
|
899
|
+
await this.store.writer.run(`ALTER TABLE ${MASON_SCHEMA}.poll_state ADD COLUMN IF NOT EXISTS sync VARCHAR`);
|
|
601
900
|
// replication mode's record: per slot, the end of the last commit applied, in the same transaction as its rows
|
|
602
901
|
await this.store.writer.run(`CREATE TABLE IF NOT EXISTS ${MASON_SCHEMA}.stream_state (slot VARCHAR PRIMARY KEY, lsn VARCHAR, applied_at TIMESTAMPTZ)`);
|
|
603
902
|
// the tables copied (or under a new slot) and not yet caught up by their one poll (`behind`), written
|
|
@@ -619,10 +918,12 @@ export class Sync {
|
|
|
619
918
|
this.slotKnown = r.length > 0;
|
|
620
919
|
this.appliedLsn = r.length > 0 ? parseLsn(String(r[0][0])) : 0n;
|
|
621
920
|
}
|
|
622
|
-
for (const [source, snapshot] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
|
|
921
|
+
for (const [source, snapshot, sync] of (await this.store.writer.runAndReadAll(`SELECT source, snapshot, sync FROM ${MASON_SCHEMA}.poll_state`)).getRowsJS()) {
|
|
623
922
|
this.snapshots.set(source, snapshot);
|
|
923
|
+
this.recordedSync.set(source, sync);
|
|
624
924
|
}
|
|
625
925
|
await this.readPolledAt();
|
|
926
|
+
await this.scrubLedger();
|
|
626
927
|
const r = await this.store.writer.runAndReadAll(`SELECT source, phase, "rows", bytes, seconds, epoch_ms(copied_at), error, columns::VARCHAR FROM ${MASON_SCHEMA}.copy_state`);
|
|
627
928
|
for (const row of r.getRowsJS()) {
|
|
628
929
|
const [source, phase, rows, bytes, seconds, copiedMs, error, columns] = row;
|
|
@@ -635,13 +936,58 @@ export class Sync {
|
|
|
635
936
|
}
|
|
636
937
|
this.states.set(source, { source, phase: phase === "copying" ? "waiting" : phase, rows: rows === null ? null : Number(rows), bytes: bytes === null ? null : Number(bytes), seconds, copiedAt, error });
|
|
637
938
|
}
|
|
939
|
+
await this.readSynced();
|
|
638
940
|
await this.findRetired();
|
|
639
941
|
this.removeStageLeftovers();
|
|
640
942
|
// the stored JSON paths, before the catalog serves anything: each served copy's view, what the file holds
|
|
641
943
|
await this.kept.open();
|
|
944
|
+
// the run's connection is a session's, set up by the one set-up (src/duck/store.ts `SessionSetup`): a plan's SQL reads it
|
|
945
|
+
const conn = await this.store.instance.connect();
|
|
946
|
+
this.generatedSession = await SessionSetup.open(conn, GENERATED_SESSION);
|
|
947
|
+
this.generatedLane = { conn, inFlight: false };
|
|
948
|
+
const checkConn = await this.store.instance.connect();
|
|
949
|
+
this.generatedCheck = { conn: checkConn, setup: await SessionSetup.open(checkConn, GENERATED_SESSION) };
|
|
950
|
+
await this.generated.open();
|
|
642
951
|
this.catalog = await this.store.instance.connect();
|
|
643
952
|
await this.rebuildSchema();
|
|
644
953
|
}
|
|
954
|
+
/**
|
|
955
|
+
* THE LEDGER SCRUBBED OF A PASSWORD AN EARLIER MASON WROTE (issue 599): before the engine was given the credentials
|
|
956
|
+
* as a secret, a failed connection's error held `password='…'`, and every table's error (`copy_state.error`) and every
|
|
957
|
+
* event (`sync_events.message`) written then kept it, shown by mason status and the web page for as long as they stand.
|
|
958
|
+
* At every open, those two free-text fields alone are read where they hold a `password=` keyword, its value masked
|
|
959
|
+
* (`maskPasswordKeyword`: the exact leaked form, never a substring of the password now, which a table's name may hold)
|
|
960
|
+
* and written back by bound parameters where that changed them, said in the log with the way out: rotate the password.
|
|
961
|
+
*/
|
|
962
|
+
async scrubLedger() {
|
|
963
|
+
const w = this.store.writer;
|
|
964
|
+
const KEYWORD = "%password%=%";
|
|
965
|
+
let scrubbed = 0;
|
|
966
|
+
const errors = (await w.runAndReadAll(`SELECT source, error FROM ${MASON_SCHEMA}.copy_state WHERE error ILIKE $1`, [KEYWORD])).getRowsJS();
|
|
967
|
+
for (const [source, error] of errors) {
|
|
968
|
+
const masked = maskPasswordKeyword(error);
|
|
969
|
+
if (masked === error)
|
|
970
|
+
continue;
|
|
971
|
+
await w.run(`UPDATE ${MASON_SCHEMA}.copy_state SET error = $1 WHERE source = $2`, [masked, source]);
|
|
972
|
+
scrubbed++;
|
|
973
|
+
}
|
|
974
|
+
// the events' table is SyncEvents' (src/duck/events.ts), opened before the sync by the run; a test's store may have none
|
|
975
|
+
const events = (await w.runAndReadAll(`SELECT count(*) FROM duckdb_tables() WHERE schema_name = $1 AND table_name = 'sync_events'`, [MASON_SCHEMA])).getRowsJS();
|
|
976
|
+
if (Number(events[0][0]) > 0) {
|
|
977
|
+
const rows = (await w.runAndReadAll(`SELECT DISTINCT message FROM ${MASON_SCHEMA}.sync_events WHERE message ILIKE $1`, [KEYWORD])).getRowsJS();
|
|
978
|
+
for (const [message] of rows) {
|
|
979
|
+
const masked = maskPasswordKeyword(message);
|
|
980
|
+
if (masked === message)
|
|
981
|
+
continue;
|
|
982
|
+
const changed = await w.run(`UPDATE ${MASON_SCHEMA}.sync_events SET message = $1 WHERE message = $2`, [masked, message]);
|
|
983
|
+
scrubbed += changed.rowsChanged;
|
|
984
|
+
}
|
|
985
|
+
}
|
|
986
|
+
if (scrubbed > 0) {
|
|
987
|
+
this.log(`removed the database's password from ${scrubbed} error row(s) an earlier Mason wrote to its record of the sync (issue 599): ` +
|
|
988
|
+
`it was shown by mason status, mason logs --events and the web page, and may stand in .mason/*.log files — change the database user's password (ALTER ROLE … PASSWORD), then delete those log files`);
|
|
989
|
+
}
|
|
990
|
+
}
|
|
645
991
|
/** the tables the project adds now (its sources file): a new one is copied at the next pass unless its copy is already
|
|
646
992
|
* here and current (said as its records say it, `addedText`), a removed one dropped.
|
|
647
993
|
* What it returns is the catalog rebuilt over them (never rejected: a failure is said in the log) */
|
|
@@ -655,11 +1001,51 @@ export class Sync {
|
|
|
655
1001
|
const added = tables.filter((t) => !before.has(t.source));
|
|
656
1002
|
const removed = [...before.keys()].filter((s) => !this.tables.has(s));
|
|
657
1003
|
if (added.length)
|
|
658
|
-
this.log(addedText(added.map((t) => this.addedStatus(t))
|
|
1004
|
+
this.log(addedText(added.map((t) => this.addedStatus(t))));
|
|
1005
|
+
// A TABLE WHOSE SYNC CHANGED — by mason add --sync now, or by hand since its last poll (the ledger's record) — keeps
|
|
1006
|
+
// its copy and is caught up by ONE poll now, whatever it changed to, then kept as it says: never copied again
|
|
1007
|
+
for (const t of tables) {
|
|
1008
|
+
const now = syncText(this.syncOf(t)), prev = before.get(t.source);
|
|
1009
|
+
const was = prev !== undefined ? syncText(this.syncOf(prev)) : this.recordedSync.get(t.source) ?? null;
|
|
1010
|
+
if (was === null || was === now)
|
|
1011
|
+
continue;
|
|
1012
|
+
this.due.delete(t.source);
|
|
1013
|
+
// recopy copies every table again whatever its sync, and a table whose columns changed too is copied again with
|
|
1014
|
+
// them, its new sync recorded at that copy's swap
|
|
1015
|
+
const copied = this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source) && this.copiedAs.get(t.source) === columnsKey(t.columns);
|
|
1016
|
+
if (this.mode === "recopy" || !copied)
|
|
1017
|
+
continue;
|
|
1018
|
+
if (this.replicated(t)) {
|
|
1019
|
+
// INTO THE STREAM (poll or off → replication): caught up by ONE poll with the stream's changes of it held from
|
|
1020
|
+
// just before that poll (`catchUp`), as a slot new to the copy is — once the pass has the publication carry it
|
|
1021
|
+
// (`prepareReplication` runs before the pass queues the catch-up). Marked in `_mason.catch_up`, on the writer's
|
|
1022
|
+
// turn and so before the catch-up starts: the stream acknowledges what it holds, so a run that dies between
|
|
1023
|
+
// the catch-up's poll and the held changes' apply must catch it up again (its recorded sync is already the new)
|
|
1024
|
+
this.switched.delete(t.source);
|
|
1025
|
+
const why = `after its sync changed from ${was}`;
|
|
1026
|
+
if (!this.behind.has(t.source)) {
|
|
1027
|
+
const mark = { why };
|
|
1028
|
+
this.behind.set(t.source, mark);
|
|
1029
|
+
void this.onWriter(() => this.store.writer.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(why)}, now())`)).catch((e) => {
|
|
1030
|
+
// its next start catches it up again only by this row: said, and the mark kept for this run's own catch-up
|
|
1031
|
+
this.log(`${t.source}: its catch-up could not be recorded (a stop before it ends catches it up again only if its sync changes): ${this.errorText(e)}`);
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
}
|
|
1035
|
+
else {
|
|
1036
|
+
// OUT OF THE STREAM, or between off and poll: it leaves the stream at its next batch (`refreshStreamed`) and the
|
|
1037
|
+
// next pass polls it from its watermark, which every change since is at or past
|
|
1038
|
+
this.switched.add(t.source);
|
|
1039
|
+
}
|
|
1040
|
+
// said, and recorded as the sync's event (src/duck/events.ts): what a table's sync was is history the log alone loses
|
|
1041
|
+
const switched = `sync: ${was} → ${now}: its synced data is kept, and caught up by one poll now${this.replicated(t) ? ", then streamed" : ""}`;
|
|
1042
|
+
this.log(`${t.source}: ${switched}`);
|
|
1043
|
+
this.event({ kind: "sync_changed", table: t.source, message: switched });
|
|
1044
|
+
}
|
|
659
1045
|
// loud: the copy goes with the table's entry, and comes back only by reading the source again (the test server: a 53 GB
|
|
660
1046
|
// copy, 21 minutes of a production replica, dropped by a project file moved aside)
|
|
661
1047
|
if (removed.length)
|
|
662
|
-
this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its
|
|
1048
|
+
this.log(`removed ${removed.join(", ")}: no longer added, so ${removed.length === 1 ? "its synced data is" : "their synced data is"} DROPPED at the next pass — mason add syncs ${removed.length === 1 ? "it" : "them"} again from the source, whole`);
|
|
663
1049
|
for (const t of tables)
|
|
664
1050
|
if (!this.states.has(t.source))
|
|
665
1051
|
this.states.set(t.source, { source: t.source, phase: "waiting", rows: null, bytes: t.bytes, seconds: null, copiedAt: null, error: null });
|
|
@@ -671,7 +1057,7 @@ export class Sync {
|
|
|
671
1057
|
this.retries.delete(t.source);
|
|
672
1058
|
}
|
|
673
1059
|
// the tables copied before are served at once, not after this start's first pass (its copies may take hours)
|
|
674
|
-
const built = this.rebuildSchema().catch((e) => this.log(`the catalog could not be rebuilt (it is at the next pass): ${
|
|
1060
|
+
const built = this.rebuildSchema().catch((e) => this.log(`the catalog could not be rebuilt (it is at the next pass): ${this.errorText(e)}`));
|
|
675
1061
|
this.kick();
|
|
676
1062
|
return built;
|
|
677
1063
|
}
|
|
@@ -684,33 +1070,117 @@ export class Sync {
|
|
|
684
1070
|
// a named publication's gap is said again by the pass, which refuses the table by name until its owner adds it
|
|
685
1071
|
if (st?.error?.startsWith("replication: "))
|
|
686
1072
|
return { source: t.source, refused: firstLine(st.error.slice("replication: ".length)) };
|
|
687
|
-
const copy = st === undefined ? "never
|
|
688
|
-
: st.phase === "failed" ? `its
|
|
689
|
-
: st.phase !== "copied" ? "its
|
|
690
|
-
: this.retired.has(t.source) ?
|
|
691
|
-
: this.
|
|
692
|
-
: !this.snapshots.has(t.source) ? "a numeric value outgrew its copy's DECIMAL"
|
|
693
|
-
: null;
|
|
1073
|
+
const copy = st === undefined ? "never synced"
|
|
1074
|
+
: st.phase === "failed" ? `its first sync failed: ${firstLine(st.error)}`
|
|
1075
|
+
: st.phase !== "copied" ? "its first sync has not finished"
|
|
1076
|
+
: this.retired.has(t.source) ? `it is ${this.retired.get(t.source).short}`
|
|
1077
|
+
: this.readAgainWhy(t);
|
|
694
1078
|
if (copy !== null)
|
|
695
1079
|
return { source: t.source, copy };
|
|
696
1080
|
const b = this.behind.get(t.source);
|
|
697
1081
|
if (b !== undefined)
|
|
698
1082
|
return { source: t.source, behind: b.why };
|
|
699
|
-
if (this.replication !== null && !this.slotKnown)
|
|
700
|
-
return { source: t.source, behind: `slot ${this.replication.slot} is new to
|
|
701
|
-
|
|
1083
|
+
if (this.replicated(t) && this.replication !== null && !this.slotKnown)
|
|
1084
|
+
return { source: t.source, behind: `slot ${this.replication.slot} is new to the synced data` };
|
|
1085
|
+
const off = this.mode !== "recopy" && this.syncOf(t).mode === "off";
|
|
1086
|
+
return { source: t.source, kept: this.keptWords(t), off };
|
|
702
1087
|
}
|
|
703
|
-
/** how a copy that is current is kept so, in this run's mode: the start line's words (`addedText`) */
|
|
704
|
-
keptWords() {
|
|
1088
|
+
/** how a copy that is current is kept so, in this run's mode and by its sync: the start line's words (`addedText`) */
|
|
1089
|
+
keptWords(t) {
|
|
705
1090
|
const every = `${Number((this.intervalMs / 1000).toFixed(1))} s`;
|
|
706
|
-
|
|
707
|
-
|
|
1091
|
+
const s = this.syncOf(t);
|
|
1092
|
+
return this.mode === "recopy" ? `read again whole every ${every}`
|
|
1093
|
+
: s.mode === "replication" ? `streamed from ${formatLsn(this.appliedLsn)}`
|
|
1094
|
+
: s.mode === "off" ? `sync: off (read again only by mason sync ${t.source})` : syncWords(s);
|
|
1095
|
+
}
|
|
1096
|
+
/** how a table is kept current, as the sources file says (`AddedTable.sync`); a table with none — a test's — polled at
|
|
1097
|
+
* the sync's own interval */
|
|
1098
|
+
syncOf(t) {
|
|
1099
|
+
if (t.sync !== undefined)
|
|
1100
|
+
return t.sync;
|
|
1101
|
+
return this.mode === "replication" ? { mode: "replication" } : { mode: "poll", every: intervalOf(this.intervalMs / 1000), seconds: this.intervalMs / 1000 };
|
|
1102
|
+
}
|
|
1103
|
+
/** a table the stream keeps current: its sync is `replication` (never in a run that copies every table again whole) */
|
|
1104
|
+
replicated(t) {
|
|
1105
|
+
return this.mode !== "recopy" && this.syncOf(t).mode === "replication";
|
|
1106
|
+
}
|
|
1107
|
+
/** the added tables the stream keeps current, in the project's order */
|
|
1108
|
+
replicatedTables() {
|
|
1109
|
+
return [...this.tables.values()].filter((t) => this.replicated(t));
|
|
1110
|
+
}
|
|
1111
|
+
/** THE RUN STREAMS NOW: a table's sync is `replication` (the slot is made, or kept, and the stream runs) */
|
|
1112
|
+
streams() {
|
|
1113
|
+
return this.replication !== null && this.replicatedTables().length > 0;
|
|
1114
|
+
}
|
|
1115
|
+
/**
|
|
1116
|
+
* WHEN A TABLE'S NEXT POLL IS DUE (ms since the epoch), or null when none is: a table whose sync changed at once
|
|
1117
|
+
* (`switched`), unless a failed attempt set its next; an `off` one never; a polled one at `due`, else its last poll's
|
|
1118
|
+
* time in the ledger plus its interval (a restart does not poll a table before its interval), else now.
|
|
1119
|
+
*/
|
|
1120
|
+
dueAt(t) {
|
|
1121
|
+
const s = this.syncOf(t), d = this.due.get(t.source);
|
|
1122
|
+
if (s.mode !== "poll" && !this.switched.has(t.source))
|
|
1123
|
+
return null;
|
|
1124
|
+
if (d !== undefined)
|
|
1125
|
+
return d;
|
|
1126
|
+
if (this.switched.has(t.source) || s.mode !== "poll")
|
|
1127
|
+
return 0;
|
|
1128
|
+
const at = this.polledAt.get(t.source);
|
|
1129
|
+
return at === undefined ? 0 : Date.parse(at) + s.seconds * 1000;
|
|
1130
|
+
}
|
|
1131
|
+
/** a table polled (by a pass, `mason sync` or its catch-up) in `ms`: its next poll after its interval, stretched to four
|
|
1132
|
+
* times that poll (`nextPollDelayMs`), and its sync now the one recorded (the poll wrote it, `poll`) */
|
|
1133
|
+
polledNow(t, ms) {
|
|
1134
|
+
const s = this.syncOf(t);
|
|
1135
|
+
this.switched.delete(t.source);
|
|
1136
|
+
this.lastPollMs.set(t.source, ms);
|
|
1137
|
+
this.recordedSync.set(t.source, syncText(s));
|
|
1138
|
+
if (s.mode === "poll")
|
|
1139
|
+
this.due.set(t.source, nextPollAt(Date.now(), s.seconds, ms));
|
|
1140
|
+
else
|
|
1141
|
+
this.due.delete(t.source);
|
|
1142
|
+
}
|
|
1143
|
+
/**
|
|
1144
|
+
* WHAT THIS RUN MEASURED OF A TABLE, for `mason add`'s estimate of what polling it costs the database
|
|
1145
|
+
* (src/cli/source-edit.ts pollCost): its last poll's time, and ONE reader's rate at its copy — the copy reading now,
|
|
1146
|
+
* else the last copy's rows over its time and its readers. Null when nothing of it is measured.
|
|
1147
|
+
*/
|
|
1148
|
+
measured(source) {
|
|
1149
|
+
const st = this.states.get(source);
|
|
1150
|
+
const live = this.copyLive.get(source)?.rate?.rowsPerSecond ?? null;
|
|
1151
|
+
const copied = st?.phase === "copied" && st.rows && st.seconds ? st.rows / st.seconds / this.copyParallel : null;
|
|
1152
|
+
const last = this.lastPollMs.get(source) ?? null;
|
|
1153
|
+
return last === null && live === null && copied === null ? null : { lastPollMs: last, copyRowsPerSecond: live ?? copied };
|
|
1154
|
+
}
|
|
1155
|
+
/**
|
|
1156
|
+
* HOW LONG UNTIL THE NEXT PASS, in poll mode (null: none until something kicks one — every table `off`, or nothing
|
|
1157
|
+
* copied yet): a pass that failed, or a table waiting for a copy nothing queued, after the sync's interval; else when
|
|
1158
|
+
* the first polled table is due — and at least four times the last pass took, so the passes' polls together never
|
|
1159
|
+
* take more than a fifth of the source's time. `recopy` copies every table again every interval, stretched the same.
|
|
1160
|
+
*/
|
|
1161
|
+
nextPassDelay() {
|
|
1162
|
+
if (this.passFailed || this.waiting())
|
|
1163
|
+
return this.intervalMs;
|
|
1164
|
+
if (this.mode === "recopy")
|
|
1165
|
+
return nextPollDelayMs(this.intervalMs, this.lastPassMs);
|
|
1166
|
+
// a table copying, queued, or owed its catch-up is not polled by a pass (its catch-up polls it, then `rearm`): counted
|
|
1167
|
+
// as due it would run a pass every few ms that polls nothing and attaches the source each time
|
|
1168
|
+
const dues = [...this.tables.values()].filter((t) => this.pollable(t)).flatMap((t) => this.dueAt(t) ?? []);
|
|
1169
|
+
return dues.length ? passDelayMs(Math.min(...dues), Date.now(), this.lastPassMs) : null;
|
|
1170
|
+
}
|
|
1171
|
+
/** a table a pass would poll now if it were due: copied, not queued or rebuilt, and not owed its catch-up */
|
|
1172
|
+
pollable(t) {
|
|
1173
|
+
return !this.needsCopy(t) && !this.behind.has(t.source) && !this.queued.has(t.source) && this.rebuildingNow?.source !== t.source;
|
|
708
1174
|
}
|
|
709
1175
|
/** the source's views (src/cli/preflight.ts VIEWS_SQL), served over the copied tables: in the catalog from the next
|
|
710
1176
|
* pass. A view over a table that is not copied is left out of it (said in the catalog's log) */
|
|
711
1177
|
setViews(views) {
|
|
712
1178
|
this.views = views.map((v) => ({ schema: v.schema, name: v.name, definition: v.definition, ...(v.comment ? { description: v.comment } : {}) }));
|
|
713
1179
|
}
|
|
1180
|
+
/** the source's collations (src/duck/collation.ts): the catalog declares each it lacks, from the next pass */
|
|
1181
|
+
setCollations(collations) {
|
|
1182
|
+
this.collations = collations.map((c) => ({ schema: c.schema, name: c.name, provider: c.provider, collate: c.collate, locale: c.locale, deterministic: c.deterministic }));
|
|
1183
|
+
}
|
|
714
1184
|
/** every table read again from the start: the database was restored to another server (src/cli/dev.ts sourceMove) */
|
|
715
1185
|
async rereadAll() {
|
|
716
1186
|
// from watermark 0 every row is read and written again; the count check deletes what the new server lacks
|
|
@@ -722,7 +1192,7 @@ export class Sync {
|
|
|
722
1192
|
const now = Date.now();
|
|
723
1193
|
// in the project's order, whatever order the copies ran and were recorded in (`copyOrder`)
|
|
724
1194
|
return [...this.tables.keys()].flatMap((source) => this.states.get(source) ?? [])
|
|
725
|
-
.map((s) => this.retired.
|
|
1195
|
+
.map((s) => { const r = this.retired.get(s.source); return r !== undefined ? { ...s, retired: r.layout } : s; })
|
|
726
1196
|
// a copy's chunks as they are NOW: the rows the ones in flight have read by now at the measured rate
|
|
727
1197
|
.map((s) => { const l = this.copyLive.get(s.source); return s.phase === "copying" && l !== undefined ? { ...s, progress: copyProgressNow(l, Date.now()) } : s; })
|
|
728
1198
|
// how it is kept current, and a table the passes leave as copied, from ONE derivation (`keptOf`)
|
|
@@ -731,33 +1201,55 @@ export class Sync {
|
|
|
731
1201
|
if (t === undefined)
|
|
732
1202
|
return s;
|
|
733
1203
|
const kept = this.keptOf(t, now);
|
|
734
|
-
return { ...s, kept
|
|
1204
|
+
return { ...s, kept };
|
|
1205
|
+
})
|
|
1206
|
+
// when it was last brought current, whether it waits for its catch-up, and whether it is being rebuilt now
|
|
1207
|
+
.map((s) => {
|
|
1208
|
+
const synced = this.synced.get(s.source), behind = this.behind.get(s.source)?.why;
|
|
1209
|
+
const rebuilding = this.rebuildingNow?.source === s.source ? this.rebuildingNow.kind : undefined;
|
|
1210
|
+
return { ...s, ...(synced ? { synced } : {}), ...(behind !== undefined ? { behind } : {}), ...(rebuilding ? { rebuilding } : {}) };
|
|
735
1211
|
});
|
|
736
1212
|
}
|
|
737
1213
|
/**
|
|
738
1214
|
* HOW AN ADDED TABLE IS KEPT CURRENT, NOW (`KeptCurrent`): the run's mode, then in replication mode its part in the
|
|
739
|
-
* stream (`streamTable`, what `streamStatus` says of it) and where the stream is; in poll mode its last
|
|
740
|
-
*
|
|
741
|
-
*
|
|
1215
|
+
* stream (`streamTable`, what `streamStatus` says of it) and where the stream is; in poll mode its sync, its last copy
|
|
1216
|
+
* or poll and its next. The one place it is derived: `state` carries it to `mason status` and the listener's EXPLAIN
|
|
1217
|
+
* alike.
|
|
742
1218
|
*/
|
|
743
1219
|
keptOf(t, now) {
|
|
744
1220
|
const st = this.states.get(t.source);
|
|
745
|
-
|
|
1221
|
+
const sync = this.syncOf(t);
|
|
1222
|
+
if (this.mode !== "recopy" && sync.mode !== "replication") {
|
|
746
1223
|
const polledAt = this.polledAt.get(t.source) ?? null;
|
|
747
|
-
// a copy the passes do not keep current now, and why: not copied (waiting,
|
|
748
|
-
// `needsCopy` holds for every phase but copied), to be copied again (its columns
|
|
749
|
-
// retired layout),
|
|
750
|
-
|
|
751
|
-
|
|
1224
|
+
// a copy the passes do not keep current now, and why, from what a poll of it asks (`pollable`): not copied (waiting,
|
|
1225
|
+
// being copied, its copy failed: `needsCopy` holds for every phase but copied), to be copied again (its columns
|
|
1226
|
+
// changed, a numeric outgrew it, a retired layout), caught up first, or its rebuild not ended yet: the moment this
|
|
1227
|
+
// says nothing is pending, `mason sync` of it runs
|
|
1228
|
+
const pending = this.pollable(t) ? null : this.notStreamedWhy(t.source);
|
|
1229
|
+
const next = pending === null ? this.dueAt(t) : null;
|
|
1230
|
+
const promise = keptPromise({ mode: sync.mode === "off" ? "off" : "poll", intervalSeconds: sync.mode === "poll" ? sync.seconds : 0, lastPollMs: this.lastPollMs.get(t.source) ?? null,
|
|
1231
|
+
lastPassMs: this.lastPassMs, pending, notRunning: null, passFailed: this.passFailed, dueMs: next, now });
|
|
1232
|
+
return { by: "poll", sync, syncedAt: polledAt, nextAt: next === null ? null : new Date(Math.max(next, now)).toISOString(), ...(pending !== null ? { pending } : {}), ...promise };
|
|
752
1233
|
}
|
|
753
1234
|
// copied again whole every pass, whatever its watermark: only a table not copied now (waiting, being copied, its copy
|
|
754
|
-
// failed) is not kept current
|
|
755
|
-
if (this.mode === "recopy")
|
|
756
|
-
|
|
1235
|
+
// failed) is not kept current; its synced data is as of its copy's watermark (`polled_at`, taken before the swap)
|
|
1236
|
+
if (this.mode === "recopy") {
|
|
1237
|
+
const pending = st?.phase !== "copied" ? this.notStreamedWhy(t.source) : null;
|
|
1238
|
+
const promise = keptPromise({ mode: "recopy", intervalSeconds: this.intervalMs / 1000, lastPollMs: null, lastPassMs: this.lastPassMs, pending, notRunning: null,
|
|
1239
|
+
passFailed: this.passFailed, dueMs: null, now });
|
|
1240
|
+
return { by: "recopy", copiedAt: st?.copiedAt ?? null, syncedAt: this.polledAt.get(t.source) ?? null, ...(pending !== null ? { pending } : {}), ...promise };
|
|
1241
|
+
}
|
|
757
1242
|
const s = this.stream;
|
|
758
1243
|
const part = this.streamTable(t, now, s);
|
|
759
|
-
|
|
760
|
-
|
|
1244
|
+
// current as of its last copy or poll, or — streamed — the last moment the stream had all the database wrote
|
|
1245
|
+
const polled = this.polledAt.get(t.source) ?? null;
|
|
1246
|
+
const streamed = part.state === "streamed" && this.streamCurrentAt !== null ? new Date(this.streamCurrentAt).toISOString() : null;
|
|
1247
|
+
const syncedAt = streamed !== null && (polled === null || streamed > polled) ? streamed : polled;
|
|
1248
|
+
const notRunning = this.streamDownWhy(s, st);
|
|
1249
|
+
const promise = keptPromise({ mode: "replication", intervalSeconds: 0, lastPollMs: null, lastPassMs: this.lastPassMs, pending: part.state === "streamed" ? null : part.why,
|
|
1250
|
+
notRunning, passFailed: false, dueMs: null, now });
|
|
1251
|
+
return { by: "replication", slot: this.replication?.slot ?? "", state: part.state, why: part.why, position: formatLsn(s?.position() ?? this.appliedLsn),
|
|
1252
|
+
lastCommit: this.lastBatch === null ? null : new Date(this.lastBatch.committedAt).toISOString(), notRunning, syncedAt, ...promise };
|
|
761
1253
|
}
|
|
762
1254
|
/** why the stream is not running, in the reader's words (null: it runs): its last failure and when it starts again; the
|
|
763
1255
|
* replication check a pass failed (`prepareReplication`, recorded on each copy's state); a slot not made yet; a sync
|
|
@@ -770,6 +1262,8 @@ export class Sync {
|
|
|
770
1262
|
return `${firstLine(this.streamFailure.error)} · it starts again after ${clock(this.streamFailure.retryAt)} UTC`;
|
|
771
1263
|
if (st?.error?.startsWith("replication: "))
|
|
772
1264
|
return firstLine(st.error);
|
|
1265
|
+
if (this.replication === null)
|
|
1266
|
+
return "this run reads no slot, so nothing streams it (the demo database has no change log): give its entry sync: poll <interval>";
|
|
773
1267
|
if (!this.slotKnown)
|
|
774
1268
|
return `slot ${this.replication.slot} is not made yet: it starts once it is`;
|
|
775
1269
|
if (this.stopped)
|
|
@@ -796,12 +1290,52 @@ export class Sync {
|
|
|
796
1290
|
return { state: "streamed", why: null };
|
|
797
1291
|
return { state: "waiting", why: this.notStreamedWhy(t.source) };
|
|
798
1292
|
}
|
|
799
|
-
/**
|
|
1293
|
+
/**
|
|
1294
|
+
* Each table's last copy or poll, from the ledger (a copy writes its watermark there too), ONLY EVER FORWARD: this
|
|
1295
|
+
* SELECT's snapshot may be older than a commit on the rebuilds' connection whose time is set while it is answered
|
|
1296
|
+
* (`committed`, `swap`), so what it read never replaces a later time, and a table it does not hold is dropped (removed:
|
|
1297
|
+
* nothing else drops it) only when no such commit gave it a time meanwhile.
|
|
1298
|
+
*/
|
|
800
1299
|
async readPolledAt() {
|
|
1300
|
+
const asked = new Map(this.polledAt);
|
|
801
1301
|
const r = await this.store.writer.runAndReadAll(`SELECT source, epoch_ms(polled_at) FROM ${MASON_SCHEMA}.poll_state WHERE polled_at IS NOT NULL`);
|
|
802
|
-
|
|
803
|
-
for (const [source,
|
|
804
|
-
|
|
1302
|
+
const ledger = new Map(r.getRowsJS().map(([source, ms]) => [source, new Date(Number(ms)).toISOString()]));
|
|
1303
|
+
for (const [source, at] of this.polledAt)
|
|
1304
|
+
if (!ledger.has(source) && at === asked.get(source))
|
|
1305
|
+
this.polledAt.delete(source);
|
|
1306
|
+
for (const [source, at] of ledger) {
|
|
1307
|
+
const now = this.polledAt.get(source);
|
|
1308
|
+
if (now === undefined || at > now)
|
|
1309
|
+
this.polledAt.set(source, at);
|
|
1310
|
+
}
|
|
1311
|
+
}
|
|
1312
|
+
/**
|
|
1313
|
+
* EACH COPY'S LAST SYNC AS THE LEDGER KEEPS IT, at a start (`Synced`): the swap and every poll write
|
|
1314
|
+
* `poll_state.polled_at` in their own transaction, and a copy's `copied_at` is recorded right after its swap — so a
|
|
1315
|
+
* copy recorded at or after its table's `polled_at` was last brought current by that copy (its time the copy's
|
|
1316
|
+
* `seconds`), else by a poll (its time unknown: null). In replication mode the stream's last batch
|
|
1317
|
+
* (`stream_state.applied_at`) is newer for a table it streamed; a table not caught up is left at its copy, and one the
|
|
1318
|
+
* named publication does not carry (`unpublished`: never streamed, so the stream's time is not its own) at its copy
|
|
1319
|
+
* or poll. `only`: those tables alone, read again (the pass that found them out of the publication), the others'
|
|
1320
|
+
* last sync kept as it is.
|
|
1321
|
+
*/
|
|
1322
|
+
async readSynced(only) {
|
|
1323
|
+
const r = await this.store.writer.runAndReadAll(`SELECT p.source, epoch_ms(p.polled_at), epoch_ms(c.copied_at), c.seconds FROM ${MASON_SCHEMA}.poll_state p ` +
|
|
1324
|
+
`LEFT JOIN ${MASON_SCHEMA}.copy_state c USING (source) WHERE p.polled_at IS NOT NULL`);
|
|
1325
|
+
const streamed = this.replication === null ? [] : (await this.store.writer.runAndReadAll(`SELECT epoch_ms(applied_at) FROM ${MASON_SCHEMA}.stream_state ` +
|
|
1326
|
+
`WHERE slot = $1 AND applied_at IS NOT NULL`, [this.replication.slot])).getRowsJS();
|
|
1327
|
+
const streamMs = streamed.length ? Number(streamed[0][0]) : null;
|
|
1328
|
+
if (only === undefined)
|
|
1329
|
+
this.synced.clear();
|
|
1330
|
+
for (const [source, polled, copied, seconds] of r.getRowsJS()) {
|
|
1331
|
+
if (only !== undefined && !only.has(source))
|
|
1332
|
+
continue;
|
|
1333
|
+
const at = Number(polled);
|
|
1334
|
+
const s = copied !== null && Number(copied) >= at ? { at: new Date(Number(copied)).toISOString(), seconds, by: "copy" }
|
|
1335
|
+
: { at: new Date(at).toISOString(), seconds: null, by: "poll" };
|
|
1336
|
+
const streamedIt = !this.behind.has(source) && !this.unpublished.has(source);
|
|
1337
|
+
this.synced.set(source, streamMs !== null && streamMs > Date.parse(s.at) && streamedIt ? { at: new Date(streamMs).toISOString(), seconds: null, by: "stream" } : s);
|
|
1338
|
+
}
|
|
805
1339
|
}
|
|
806
1340
|
schema() { return this.currentSchema; }
|
|
807
1341
|
start() { this.kick(); }
|
|
@@ -819,18 +1353,17 @@ export class Sync {
|
|
|
819
1353
|
}
|
|
820
1354
|
this.running = this.pass().then(() => { this.passFailed = false; }, (e) => {
|
|
821
1355
|
this.passFailed = true;
|
|
822
|
-
this.log(`sync pass failed: ${
|
|
1356
|
+
this.log(`sync pass failed: ${this.errorText(e)}`);
|
|
823
1357
|
}).finally(() => {
|
|
824
1358
|
this.running = null;
|
|
825
1359
|
if (this.stopped)
|
|
826
1360
|
return;
|
|
827
|
-
//
|
|
828
|
-
// or after the interval, when it failed or a table waits for a copy
|
|
829
|
-
// failed waits its turn, `copyRetryDelayMs`)
|
|
830
|
-
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
this.timer = setTimeout(() => this.kick(), this.intervalMs);
|
|
1361
|
+
// when the next polled table is due (`nextPassDelay`) — the stream keeps a replicated table current between passes,
|
|
1362
|
+
// and a pass runs for it when the tables change — or after the interval, when it failed or a table waits for a copy
|
|
1363
|
+
// or catch-up nothing has queued (a copy that failed waits its turn, `copyRetryDelayMs`)
|
|
1364
|
+
const d = this.nextPassDelay();
|
|
1365
|
+
if (d !== null)
|
|
1366
|
+
this.timer = setTimeout(() => this.kick(), d);
|
|
834
1367
|
});
|
|
835
1368
|
}
|
|
836
1369
|
/** an added table needs a copy or a catch-up that is neither queued nor running: a later pass must queue it */
|
|
@@ -873,19 +1406,42 @@ export class Sync {
|
|
|
873
1406
|
*/
|
|
874
1407
|
followUp(source) {
|
|
875
1408
|
const t = this.tables.get(source);
|
|
876
|
-
if (t === undefined || this.stopped || this.unpublished.has(source)
|
|
1409
|
+
if (t === undefined || this.stopped || this.unpublished.has(source))
|
|
1410
|
+
return;
|
|
1411
|
+
// copied and caught up: polled from now on, so the next pass is timed again (`rearm`)
|
|
1412
|
+
if (!this.needsCopy(t) && !this.behind.has(source)) {
|
|
1413
|
+
this.rearm();
|
|
877
1414
|
return;
|
|
1415
|
+
}
|
|
878
1416
|
const at = this.retries.get(source)?.at ?? 0;
|
|
879
1417
|
if (at > Date.now())
|
|
880
1418
|
this.wakeAt(at);
|
|
881
1419
|
else
|
|
882
1420
|
this.kick();
|
|
883
1421
|
}
|
|
884
|
-
/**
|
|
885
|
-
*
|
|
1422
|
+
/**
|
|
1423
|
+
* THE NEXT PASS TIMED AGAIN, in poll mode, once a table is copied and caught up: a pass that ended while every table
|
|
1424
|
+
* still copied found none due (`nextPassDelay` skips a table that needs its copy) and set no timer, so without this a
|
|
1425
|
+
* table's polls never began. A pass running sets its own when it ends, from the state then.
|
|
1426
|
+
*/
|
|
1427
|
+
rearm() {
|
|
1428
|
+
if (this.stopped || this.running !== null)
|
|
1429
|
+
return;
|
|
1430
|
+
const d = this.nextPassDelay();
|
|
1431
|
+
if (d === null)
|
|
1432
|
+
return;
|
|
1433
|
+
if (this.timer !== null)
|
|
1434
|
+
clearTimeout(this.timer);
|
|
1435
|
+
this.timer = setTimeout(() => this.kick(), d);
|
|
1436
|
+
}
|
|
1437
|
+
/** one pass, now, every polled table taken as due (an `off` one is not: it is polled only when asked), and its end
|
|
1438
|
+
* awaited (the one running first, if any), and the copies and catch-ups it queued: what a test waits on */
|
|
886
1439
|
async once() {
|
|
887
1440
|
while (this.running !== null)
|
|
888
1441
|
await this.running;
|
|
1442
|
+
for (const t of this.tables.values())
|
|
1443
|
+
if (this.syncOf(t).mode === "poll")
|
|
1444
|
+
this.due.set(t.source, 0);
|
|
889
1445
|
this.kick();
|
|
890
1446
|
if (this.running !== null)
|
|
891
1447
|
await this.running;
|
|
@@ -893,34 +1449,54 @@ export class Sync {
|
|
|
893
1449
|
await this.rebuilding;
|
|
894
1450
|
}
|
|
895
1451
|
/**
|
|
896
|
-
* A POLL OF EXACTLY THESE TABLES, NOW, whatever their
|
|
897
|
-
*
|
|
898
|
-
*
|
|
899
|
-
*
|
|
1452
|
+
* A POLL OF EXACTLY THESE TABLES, NOW, whatever their sync says (`mason sync`: an `off` table is read again only so):
|
|
1453
|
+
* a pass of its own, queued behind the one running, in one source snapshot as every poll is; a polled table's next
|
|
1454
|
+
* poll waits its interval from this one. A replicated table is not polled: its answer is where the stream is
|
|
1455
|
+
* (`streamed`), beside the polled ones'. Refused by name (`PollRefused`) for a table not added or not copied yet, and in a run that copies
|
|
1456
|
+
* every table again whole. Its failure is its caller's to say: the copies serve as they were, and the passes go on.
|
|
900
1457
|
*/
|
|
901
1458
|
async pollNow(sources) {
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
1459
|
+
// its failure is the command server's answer, sent as it is: a refusal is Mason's own words, any other error's are
|
|
1460
|
+
// made Mason's here (`errorText`), where they leave the sync
|
|
1461
|
+
try {
|
|
1462
|
+
return await this.pollNowOf(sources);
|
|
905
1463
|
}
|
|
1464
|
+
catch (e) {
|
|
1465
|
+
throw e instanceof PollRefused || e instanceof Stopping ? e : new Error(this.errorText(e));
|
|
1466
|
+
}
|
|
1467
|
+
}
|
|
1468
|
+
async pollNowOf(sources) {
|
|
1469
|
+
if (this.mode === "recopy")
|
|
1470
|
+
throw new PollRefused("refused", "this run reads every table again whole at each pass (MASON_DUCK_SYNC=recopy), so there is nothing to poll");
|
|
906
1471
|
const tables = sources.map((s) => {
|
|
907
1472
|
const t = this.tables.get(s);
|
|
908
1473
|
if (t === undefined)
|
|
909
1474
|
throw new PollRefused("not_found", `${s} is not added (mason add adds it)${didYouMean(s, [...this.tables.keys()])}`);
|
|
910
1475
|
return t;
|
|
911
1476
|
});
|
|
1477
|
+
// a replicated table: the stream keeps it current, so nothing to poll — its answer is where the stream is, and its part
|
|
1478
|
+
const said = () => {
|
|
1479
|
+
const now = Date.now(), position = formatLsn(this.stream?.position() ?? this.appliedLsn);
|
|
1480
|
+
return tables.filter((t) => this.replicated(t)).map((t) => { const p = this.streamTable(t, now, this.stream); return { table: t.source, position, state: p.state, why: p.why }; });
|
|
1481
|
+
};
|
|
1482
|
+
const polled = tables.filter((t) => !this.replicated(t));
|
|
1483
|
+
if (polled.length === 0)
|
|
1484
|
+
return { at: new Date().toISOString(), tables: [], streamed: said() };
|
|
912
1485
|
while (this.running !== null)
|
|
913
1486
|
await this.running;
|
|
914
1487
|
if (this.stopped)
|
|
915
1488
|
throw new Stopping();
|
|
916
1489
|
// asked after the pass ahead of it: that pass may have copied it, or failed its copy
|
|
917
|
-
for (const t of
|
|
1490
|
+
for (const t of polled) {
|
|
918
1491
|
const st = this.states.get(t.source);
|
|
919
|
-
if (st?.phase !== "copied"
|
|
920
|
-
throw new PollRefused("nothing_yet", `${t.source} is not
|
|
921
|
-
|
|
922
|
-
if (
|
|
923
|
-
throw new PollRefused("nothing_yet", `${t.source} is
|
|
1492
|
+
if (st?.phase !== "copied")
|
|
1493
|
+
throw new PollRefused("nothing_yet", `${t.source} is not synced yet (${st?.phase === "failed" ? `its first sync failed: ${st.error ?? "mason logs says why"}` : "its first sync has not finished"}): mason wait waits for it`);
|
|
1494
|
+
const again = this.readAgainWhy(t);
|
|
1495
|
+
if (again !== null)
|
|
1496
|
+
throw new PollRefused("nothing_yet", `${t.source} is being read again whole (${again}): mason wait waits for it`);
|
|
1497
|
+
const r = this.retired.get(t.source);
|
|
1498
|
+
if (r !== undefined)
|
|
1499
|
+
throw new PollRefused("nothing_yet", `${t.source} is ${r.layout} (its synced data serves as it was): it is current once read again — mason wait waits for it`);
|
|
924
1500
|
this.refuseRebuilding(t);
|
|
925
1501
|
}
|
|
926
1502
|
const poll = (async () => {
|
|
@@ -928,8 +1504,12 @@ export class Sync {
|
|
|
928
1504
|
try {
|
|
929
1505
|
// a transaction per table, as the passes poll; a table whose rebuild started since it was asked is refused
|
|
930
1506
|
const found = [];
|
|
931
|
-
for (const t of
|
|
932
|
-
|
|
1507
|
+
for (const t of polled) {
|
|
1508
|
+
const f = await this.onWriter(async () => { this.refuseRebuilding(t); return this.poll(this.writerLane, [t]); });
|
|
1509
|
+
for (const x of f)
|
|
1510
|
+
this.polledNow(t, x.ms);
|
|
1511
|
+
found.push(...f);
|
|
1512
|
+
}
|
|
933
1513
|
return found;
|
|
934
1514
|
}
|
|
935
1515
|
finally {
|
|
@@ -937,12 +1517,21 @@ export class Sync {
|
|
|
937
1517
|
}
|
|
938
1518
|
})();
|
|
939
1519
|
// the pass slot is held until it ends (kick and once queue behind it); its failure reaches the caller through `poll`
|
|
940
|
-
|
|
1520
|
+
// and the next pass timed again as a pass's end times it (`rearm`): a copy that ended meanwhile found the slot held,
|
|
1521
|
+
// and a polled table's next poll moved past the timer set before
|
|
1522
|
+
this.running = poll.then(() => undefined, () => undefined).finally(() => { this.running = null; this.rearm(); });
|
|
941
1523
|
const found = await poll;
|
|
942
|
-
|
|
1524
|
+
// A SYNC BY HAND IS AN EVENT (src/duck/events.ts `synced`), in the words mason sync and the page's Sync now say:
|
|
1525
|
+
// a scheduled poll that succeeds records nothing (one every few seconds would bury the failures), so without it a
|
|
1526
|
+
// person who synced and then read the events found only an older copy, and read it as a sync lost. A replicated
|
|
1527
|
+
// table asked for is answered with the stream's position and records none: nothing was read for it
|
|
1528
|
+
for (const x of found)
|
|
1529
|
+
this.event({ kind: "synced", table: x.table, message: `synced by hand: ${syncedText(x)}` });
|
|
1530
|
+
const missing = polled.filter((t) => !found.some((f) => f.table === t.source));
|
|
943
1531
|
if (missing.length)
|
|
944
|
-
throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its
|
|
945
|
-
|
|
1532
|
+
throw new Error(`${missing.map((t) => t.source).join(", ")} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass, and current once read`);
|
|
1533
|
+
const streamed = said();
|
|
1534
|
+
return { at: new Date().toISOString(), tables: found, ...(streamed.length ? { streamed } : {}) };
|
|
946
1535
|
}
|
|
947
1536
|
/**
|
|
948
1537
|
* The sync stopped: no pass starts, and the one running ends at once — its long statement (a copy's scan, a poll's
|
|
@@ -964,14 +1553,14 @@ export class Sync {
|
|
|
964
1553
|
}
|
|
965
1554
|
// the stream, the pass and the rebuild end together: a batch waiting for the writer's turn behind a poll must not
|
|
966
1555
|
// wait for that poll, so each connection's long statement is interrupted from the start
|
|
967
|
-
const interrupt = () => { for (const lane of [this.writerLane, this.rebuilderLane, ...this.copyLanes])
|
|
1556
|
+
const interrupt = () => { for (const lane of [this.writerLane, this.rebuilderLane, this.generatedLane, ...this.copyLanes])
|
|
968
1557
|
if (lane?.inFlight)
|
|
969
1558
|
lane.conn.interrupt(); };
|
|
970
1559
|
const again = setInterval(interrupt, 50);
|
|
971
1560
|
interrupt();
|
|
972
1561
|
try {
|
|
973
1562
|
// none of them rejects: each says its own failure
|
|
974
|
-
await Promise.all([this.stream?.stop(), this.running, this.rebuilding, this.kept.stop()]);
|
|
1563
|
+
await Promise.all([this.stream?.stop(), this.running, this.rebuilding, this.kept.stop(), this.generated.stop()]);
|
|
975
1564
|
await this.writerTurn;
|
|
976
1565
|
}
|
|
977
1566
|
finally {
|
|
@@ -984,14 +1573,20 @@ export class Sync {
|
|
|
984
1573
|
this.catalog = null;
|
|
985
1574
|
this.rebuilderLane?.conn.closeSync();
|
|
986
1575
|
this.rebuilderLane = null;
|
|
1576
|
+
this.generatedLane?.conn.closeSync();
|
|
1577
|
+
this.generatedLane = null;
|
|
1578
|
+
this.generatedSession = null;
|
|
1579
|
+
// after `generated.stop`, which waited for the review and so for any check on it
|
|
1580
|
+
this.generatedCheck?.conn.closeSync();
|
|
1581
|
+
this.generatedCheck = null;
|
|
987
1582
|
}
|
|
988
1583
|
/** a poll asked of a table while it is rebuilt or not yet caught up: refused, by name (it is caught up when that ends) */
|
|
989
1584
|
refuseRebuilding(t) {
|
|
990
1585
|
const now = this.rebuildingNow?.source === t.source ? this.rebuildingNow.kind : null;
|
|
991
1586
|
if (now === null && !this.behind.has(t.source))
|
|
992
1587
|
return;
|
|
993
|
-
const doing = now === "copy" ? "being
|
|
994
|
-
throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its
|
|
1588
|
+
const doing = now === "copy" ? "being read again whole" : `being caught up by one poll ${this.behind.get(t.source)?.why ?? ""}`.trimEnd();
|
|
1589
|
+
throw new PollRefused("nothing_yet", `${t.source} is ${doing} (its synced data serves as it was): it is caught up by one poll when that ends, so there is nothing to poll — mason wait waits for it`);
|
|
995
1590
|
}
|
|
996
1591
|
/** the source's connection, as the sync's own reads and replication make it (never DuckDB's: that is `attach`) */
|
|
997
1592
|
pgOptions() {
|
|
@@ -1015,7 +1610,7 @@ export class Sync {
|
|
|
1015
1610
|
return columns === undefined ? t : { ...t, columns };
|
|
1016
1611
|
}
|
|
1017
1612
|
/**
|
|
1018
|
-
* REPLICATION, BEFORE A PASS COPIES: the publication carries every
|
|
1613
|
+
* REPLICATION, BEFORE A PASS COPIES: the publication carries every replicated table, and the slot exists — so a table
|
|
1019
1614
|
* copied now has its changes in the stream from before it joins (src/duck/stream.ts "why replaying is safe"). A slot
|
|
1020
1615
|
* made now, or one this copy has no record of, carries nothing from before it: the tables already copied are caught
|
|
1021
1616
|
* up by one poll each before the stream applies their changes (`behind`, recorded with the slot in one transaction).
|
|
@@ -1024,7 +1619,7 @@ export class Sync {
|
|
|
1024
1619
|
*/
|
|
1025
1620
|
async prepareReplication() {
|
|
1026
1621
|
const r = this.replication;
|
|
1027
|
-
const tables =
|
|
1622
|
+
const tables = this.replicatedTables();
|
|
1028
1623
|
const pg = await PgConnection.connect(this.pgOptions());
|
|
1029
1624
|
try {
|
|
1030
1625
|
const identities = new Map((await pg.query(`SELECT n.nspname || '.' || c.relname, c.relreplident FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace ` +
|
|
@@ -1034,7 +1629,8 @@ export class Sync {
|
|
|
1034
1629
|
const nothing = tables.filter((t) => identities.get(t.source) === "n").map((t) => t.source);
|
|
1035
1630
|
if (nothing.length) {
|
|
1036
1631
|
throw new Error(`${nothing.join(", ")} ${nothing.length === 1 ? "has" : "have"} REPLICA IDENTITY NOTHING: a publication carrying ${nothing.length === 1 ? "it" : "them"} ` +
|
|
1037
|
-
`would make the app's own UPDATEs and DELETEs fail, so Mason does not
|
|
1632
|
+
`would make the app's own UPDATEs and DELETEs fail, so Mason does not replicate ${nothing.length === 1 ? "it" : "them"}. Give ${nothing.length === 1 ? "its entry" : "their entries"} ` +
|
|
1633
|
+
`sync: poll <interval> (poll 5m) in the sources file, or ALTER TABLE … REPLICA IDENTITY DEFAULT on the database`);
|
|
1038
1634
|
}
|
|
1039
1635
|
if (r.owned)
|
|
1040
1636
|
await this.ensurePublication(pg, tables, identities);
|
|
@@ -1043,7 +1639,7 @@ export class Sync {
|
|
|
1043
1639
|
const slot = await readSlot(pg, r.slot);
|
|
1044
1640
|
let made = false;
|
|
1045
1641
|
if (slot.exists && slot.walStatus === "lost") {
|
|
1046
|
-
this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the
|
|
1642
|
+
this.log(`replication slot ${r.slot} was invalidated by the database (wal_status lost: it held more WAL than max_slot_wal_keep_size), so changes are missing from it: made again, and the synced tables caught up by one poll`);
|
|
1047
1643
|
if (slot.active)
|
|
1048
1644
|
throw new Error(`replication slot ${r.slot} is lost and still in use by pid ${slot.activePid}`);
|
|
1049
1645
|
await dropSlot(pg, r.slot);
|
|
@@ -1054,14 +1650,16 @@ export class Sync {
|
|
|
1054
1650
|
if (!slot.exists || slot.walStatus === "lost") {
|
|
1055
1651
|
const repl = await PgConnection.connect({ ...this.pgOptions(), replication: true });
|
|
1056
1652
|
try {
|
|
1057
|
-
this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy,
|
|
1653
|
+
this.log(`made replication slot ${r.slot} at ${await createSlot(repl, r.slot)} (it holds the database's WAL until Mason reads it: mason destroy drops it, and so does a sources file in which no table's sync is replication)`);
|
|
1058
1654
|
}
|
|
1059
1655
|
finally {
|
|
1060
1656
|
repl.close();
|
|
1061
1657
|
}
|
|
1062
1658
|
made = true;
|
|
1659
|
+
this.onReplication?.(true);
|
|
1063
1660
|
}
|
|
1064
1661
|
if (made || !this.slotKnown) {
|
|
1662
|
+
// the replicated tables already copied: the slot carries nothing from before it (a polled one is the poll's)
|
|
1065
1663
|
const copied = tables.filter((t) => this.states.get(t.source)?.phase === "copied" && this.snapshots.has(t.source));
|
|
1066
1664
|
await this.onWriter(async () => {
|
|
1067
1665
|
const w = this.store.writer;
|
|
@@ -1087,24 +1685,44 @@ export class Sync {
|
|
|
1087
1685
|
finally {
|
|
1088
1686
|
pg.close();
|
|
1089
1687
|
}
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1688
|
+
await this.onWriter(() => this.settlePrepared(tables));
|
|
1689
|
+
}
|
|
1690
|
+
/**
|
|
1691
|
+
* WHAT A PASS'S PUBLICATION CHECK SETTLES, ON THE WRITER'S TURN (`prepareReplication`, once the publication and slot are
|
|
1692
|
+
* as they should be): a table a named publication does not carry is refused on its own line (a copy it never had is
|
|
1693
|
+
* failed), the others go on; a table it carries AGAIN has that gap's error cleared, and a table never copied the last
|
|
1694
|
+
* pass's failure to prepare. NOTHING ELSE: a copy whose stream failed, or that the failed preparation left behind, is
|
|
1695
|
+
* current again only once the stream applies a batch with a position (`applyChanges`) — the publication being right
|
|
1696
|
+
* says nothing of whether its changes are in the copy, and a pass runs while the stream waits out its back-off.
|
|
1697
|
+
*/
|
|
1698
|
+
async settlePrepared(tables) {
|
|
1699
|
+
for (const t of tables) {
|
|
1700
|
+
const st = this.states.get(t.source);
|
|
1701
|
+
const gap = this.unpublished.get(t.source);
|
|
1702
|
+
if (gap !== undefined) {
|
|
1703
|
+
const error = `replication: ${gap}`;
|
|
1704
|
+
if (st.error !== error) {
|
|
1705
|
+
this.log(`${t.source} is not synced or streamed: ${error}`);
|
|
1706
|
+
this.event({ kind: "stream_failed", table: t.source, message: `not synced or streamed: ${error}` });
|
|
1101
1707
|
}
|
|
1102
|
-
|
|
1103
|
-
|
|
1708
|
+
this.gapErrors.set(t.source, error);
|
|
1709
|
+
await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error });
|
|
1710
|
+
continue;
|
|
1104
1711
|
}
|
|
1105
|
-
|
|
1106
|
-
this.
|
|
1107
|
-
|
|
1712
|
+
const gapClosed = this.gapErrors.get(t.source);
|
|
1713
|
+
this.gapErrors.delete(t.source);
|
|
1714
|
+
if (st.error === null)
|
|
1715
|
+
continue;
|
|
1716
|
+
if (st.error === gapClosed || (st.phase !== "copied" && st.error === this.wholeFailure.prepare))
|
|
1717
|
+
await this.record({ ...st, error: null });
|
|
1718
|
+
}
|
|
1719
|
+
// a table the publication does not carry was never streamed: its last sync is its copy's or its poll's, never
|
|
1720
|
+
// the stream's time a start credited it with (`readSynced` at `open`, before the publication was read)
|
|
1721
|
+
const credited = tables.filter((t) => this.unpublished.has(t.source) && this.synced.get(t.source)?.by === "stream").map((t) => t.source);
|
|
1722
|
+
if (credited.length)
|
|
1723
|
+
await this.readSynced(new Set(credited));
|
|
1724
|
+
// a table the publication stopped carrying leaves the stream; one it carries now may join
|
|
1725
|
+
this.refreshStreamed();
|
|
1108
1726
|
}
|
|
1109
1727
|
/**
|
|
1110
1728
|
* A publication the operator NAMED: read as it is, verified, never created, altered or dropped (src/duck/publication.ts).
|
|
@@ -1113,14 +1731,14 @@ export class Sync {
|
|
|
1113
1731
|
*/
|
|
1114
1732
|
async verifyPublication(pg, tables, identities) {
|
|
1115
1733
|
const name = this.replication.publication;
|
|
1116
|
-
const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables
|
|
1734
|
+
const v = namedPublicationVerdict(await readNamedPublication(pg, name, tables), tables, identities);
|
|
1117
1735
|
if (v.refused !== null)
|
|
1118
1736
|
throw new Error(v.refused);
|
|
1119
1737
|
this.unpublished.clear();
|
|
1120
1738
|
for (const [source, m] of v.tables)
|
|
1121
1739
|
this.unpublished.set(source, m);
|
|
1122
1740
|
if (v.noTruncate && !this.truncateSaid) {
|
|
1123
|
-
this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the
|
|
1741
|
+
this.log(`publication ${name} does not publish truncate: a TRUNCATE on the database does not reach the synced data (the poll's count would see it: sync: poll <interval> on its entry)`);
|
|
1124
1742
|
this.truncateSaid = true;
|
|
1125
1743
|
}
|
|
1126
1744
|
}
|
|
@@ -1154,23 +1772,111 @@ export class Sync {
|
|
|
1154
1772
|
throw new Error(`publication ${name} is FOR ALL TABLES: it is not the one Mason made (drop it, or name another project)`);
|
|
1155
1773
|
await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET TABLE ${list}`);
|
|
1156
1774
|
await pg.query(`ALTER PUBLICATION ${pgIdent(name)} SET (publish = 'insert, update, delete, truncate')`);
|
|
1157
|
-
this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}`);
|
|
1775
|
+
this.log(`re-pointed publication ${name} at ${tables.map((t) => t.source).join(", ")}: the tables whose sync is replication`);
|
|
1776
|
+
}
|
|
1777
|
+
// VERIFIED, never assumed: what the publication carries is what the database says it does
|
|
1778
|
+
const carried = (await pg.query(`SELECT schemaname || '.' || tablename FROM pg_publication_tables WHERE pubname = ${pgLiteral(name)}`)).rows.map((x) => x[0]).sort();
|
|
1779
|
+
const wanted = tables.map((t) => t.source).sort();
|
|
1780
|
+
if (carried.join("\n") !== wanted.join("\n")) {
|
|
1781
|
+
throw new Error(`publication ${name} carries ${carried.join(", ") || "no table"} after it was set to ${wanted.join(", ")} (Mason bug: report it, with mason logs)`);
|
|
1158
1782
|
}
|
|
1159
1783
|
}
|
|
1784
|
+
/**
|
|
1785
|
+
* NO TABLE'S SYNC IS REPLICATION ANY MORE (at the start that finds none, or live when the last one leaves): the stream
|
|
1786
|
+
* stopped, the slot dropped — it holds the database's WAL while it exists, read or not — and Mason's own publication
|
|
1787
|
+
* with it (a named one is its owner's: left as it is), each VERIFIED gone; then this copy's record of the slot goes,
|
|
1788
|
+
* so a table replicated again later is caught up by one poll under the slot made anew. Idempotent: what is already
|
|
1789
|
+
* gone is not an error. A failure is the pass's, said, and tried again at the next.
|
|
1790
|
+
*/
|
|
1791
|
+
async retireReplication() {
|
|
1792
|
+
const r = this.replication;
|
|
1793
|
+
const s = this.stream;
|
|
1794
|
+
if (s !== null) {
|
|
1795
|
+
await s.stop();
|
|
1796
|
+
if (this.stream === s)
|
|
1797
|
+
this.stream = null;
|
|
1798
|
+
}
|
|
1799
|
+
const pg = await PgConnection.connect(this.pgOptions());
|
|
1800
|
+
const dropped = [];
|
|
1801
|
+
try {
|
|
1802
|
+
// the walsender a stop ended lets the slot go a moment later: waited for, at most 10 s
|
|
1803
|
+
let slot = await readSlot(pg, r.slot);
|
|
1804
|
+
for (let i = 0; slot.exists && slot.active && i < 100; i++) {
|
|
1805
|
+
await new Promise((res) => setTimeout(res, 100));
|
|
1806
|
+
slot = await readSlot(pg, r.slot);
|
|
1807
|
+
}
|
|
1808
|
+
if (slot.exists) {
|
|
1809
|
+
if (slot.active)
|
|
1810
|
+
throw new Error(`replication slot ${r.slot} is in use by pid ${slot.activePid ?? "?"}, so it cannot be dropped (another Mason over the same project? stop it)`);
|
|
1811
|
+
await dropSlot(pg, r.slot);
|
|
1812
|
+
dropped.push(`replication slot ${r.slot}`);
|
|
1813
|
+
}
|
|
1814
|
+
const hasPub = async () => (await pg.query(`SELECT 1 FROM pg_publication WHERE pubname = ${pgLiteral(r.publication)}`)).rows.length > 0;
|
|
1815
|
+
if (r.owned && await hasPub()) {
|
|
1816
|
+
await pg.query(`DROP PUBLICATION IF EXISTS ${pgIdent(r.publication)}`);
|
|
1817
|
+
dropped.push(`publication ${r.publication}`);
|
|
1818
|
+
}
|
|
1819
|
+
if ((await readSlot(pg, r.slot)).exists)
|
|
1820
|
+
throw new Error(`replication slot ${r.slot} is still there after pg_drop_replication_slot`);
|
|
1821
|
+
if (r.owned && await hasPub())
|
|
1822
|
+
throw new Error(`publication ${r.publication} is still there after DROP PUBLICATION`);
|
|
1823
|
+
}
|
|
1824
|
+
finally {
|
|
1825
|
+
pg.close();
|
|
1826
|
+
}
|
|
1827
|
+
await this.onWriter(async () => {
|
|
1828
|
+
await this.store.writer.run(`DELETE FROM ${MASON_SCHEMA}.stream_state WHERE slot = $1`, [r.slot]);
|
|
1829
|
+
this.slotKnown = false;
|
|
1830
|
+
this.appliedLsn = 0n;
|
|
1831
|
+
this.streamCurrentAt = null;
|
|
1832
|
+
this.refreshStreamed();
|
|
1833
|
+
});
|
|
1834
|
+
this.log(`no table's sync is replication now: ${dropped.length ? `dropped ${dropped.join(" and ")} from ${this.source.database}, verified gone` : `replication slot ${r.slot} was already gone`}` +
|
|
1835
|
+
` (the slot held the database's WAL)${r.owned ? "" : `; publication ${r.publication} is its owner's, left as it is`} — a table set to sync: replication makes the slot again`);
|
|
1836
|
+
this.onReplication?.(false);
|
|
1837
|
+
}
|
|
1838
|
+
/**
|
|
1839
|
+
* THE STREAM HAS EVERYTHING THE DATABASE WROTE (a keepalive with nothing pending, or a batch flushed): every streamed
|
|
1840
|
+
* table is current as of now (`keptOf`'s syncedAt), and a copied one's replication error (the stream's failure, the
|
|
1841
|
+
* preparation's, from this run or one before) is over — the same proof a batch applied with a position gives
|
|
1842
|
+
* (`applyChanges`), which an idle database never sends: a healthy table kept a run-old stream error for ever, and
|
|
1843
|
+
* `mason wait` read it as failed. Cleared on the writer's turn, one turn at a time.
|
|
1844
|
+
*/
|
|
1845
|
+
streamCurrent() {
|
|
1846
|
+
this.streamCurrentAt = Date.now();
|
|
1847
|
+
if (this.clearingStreamErrors)
|
|
1848
|
+
return;
|
|
1849
|
+
const stale = (source) => {
|
|
1850
|
+
const st = this.states.get(source);
|
|
1851
|
+
return st !== undefined && st.phase === "copied" && st.error?.startsWith("replication: ") === true && !this.unpublished.has(source) && !this.held.has(source);
|
|
1852
|
+
};
|
|
1853
|
+
if (![...this.joinedNow].some(stale))
|
|
1854
|
+
return;
|
|
1855
|
+
this.clearingStreamErrors = true;
|
|
1856
|
+
void this.onWriter(async () => {
|
|
1857
|
+
for (const source of this.joinedNow)
|
|
1858
|
+
if (stale(source))
|
|
1859
|
+
await this.record({ ...this.states.get(source), error: null });
|
|
1860
|
+
this.recovered("stream", "the stream has everything the database wrote again");
|
|
1861
|
+
}).catch((e) => this.log(`replication: a table's stream error could not be cleared (it is cleared at the next batch): ${this.errorText(e)}`))
|
|
1862
|
+
.finally(() => { this.clearingStreamErrors = false; });
|
|
1863
|
+
}
|
|
1864
|
+
clearingStreamErrors = false;
|
|
1160
1865
|
/**
|
|
1161
1866
|
* The stream started from the slot's acknowledged position, once the slot is known (a pass's `prepareReplication`),
|
|
1162
1867
|
* and never stopped by a pass: a copy runs beside it. What it applies is `streamed`, read at each change, so a table
|
|
1163
1868
|
* joins once caught up and leaves when copied again without a restart.
|
|
1164
1869
|
*/
|
|
1165
1870
|
startStream() {
|
|
1166
|
-
if (this.stream !== null || this.stopped || this.
|
|
1871
|
+
if (this.stream !== null || this.stopped || !this.streams() || !this.slotKnown || Date.now() < this.streamAt)
|
|
1167
1872
|
return;
|
|
1168
1873
|
const r = this.replication;
|
|
1169
1874
|
this.refreshStreamed();
|
|
1170
1875
|
const s = new ReplicationStream({
|
|
1171
|
-
connect: () => PgConnection.connect({ ...this.pgOptions(), replication: true }), slot: r.slot, publication: r.publication, log: (l) => this.log(l),
|
|
1876
|
+
connect: () => PgConnection.connect({ ...this.pgOptions(), replication: true }), slot: r.slot, publication: r.publication, log: (l) => this.log(l), errorText: (e) => this.errorText(e),
|
|
1172
1877
|
target: { tables: () => this.streamed, apply: (batch, lsn, committedAt) => this.applyStreamed(batch, lsn, committedAt), applied: () => this.appliedLsn,
|
|
1173
|
-
refuse: (source, missing) => this.refuseStreamed(source, missing)
|
|
1878
|
+
refuse: (source, missing) => this.refuseStreamed(source, missing), current: () => { if (this.stream === s)
|
|
1879
|
+
this.streamCurrent(); } },
|
|
1174
1880
|
});
|
|
1175
1881
|
this.stream = s;
|
|
1176
1882
|
this.streamFailure = null;
|
|
@@ -1182,7 +1888,7 @@ export class Sync {
|
|
|
1182
1888
|
}
|
|
1183
1889
|
const joined = [...this.streamed.keys()].filter((x) => !this.held.has(x));
|
|
1184
1890
|
this.log(`streaming ${joined.length ? joined.join(", ") : "no table yet"} from slot ${r.slot} (publication ${r.publication}), applied through ${formatLsn(this.appliedLsn)}` +
|
|
1185
|
-
`${joined.length < this.
|
|
1891
|
+
`${joined.length < this.replicatedTables().length ? "; a table synced or caught up meanwhile joins it once current" : ""}`);
|
|
1186
1892
|
void s.done.then(async (e) => {
|
|
1187
1893
|
if (this.stream === s)
|
|
1188
1894
|
this.stream = null;
|
|
@@ -1191,17 +1897,17 @@ export class Sync {
|
|
|
1191
1897
|
// a numeric wider than its copy's DECIMAL: that table leaves the stream and is copied again, wider, and the stream
|
|
1192
1898
|
// goes on at once without it (it joins again caught up from its new copy) — at most three times a run, past which
|
|
1193
1899
|
// a value the stream keeps replaying is wider than any copy of now
|
|
1194
|
-
const wider = [...this.tables.values()].find((t) => e
|
|
1900
|
+
const wider = [...this.tables.values()].find((t) => widerIn(e, t.source));
|
|
1195
1901
|
const n = wider === undefined ? 0 : (this.widened.get(wider.source) ?? 0) + 1;
|
|
1196
1902
|
if (wider !== undefined && n <= 3) {
|
|
1197
1903
|
this.widened.set(wider.source, n);
|
|
1198
1904
|
await this.onWriter(async () => { this.snapshots.delete(wider.source); this.refreshStreamed(); });
|
|
1199
|
-
this.log(`replication: ${wider.source} has a numeric value wider than its
|
|
1905
|
+
this.log(`replication: ${wider.source} has a numeric value wider than its synced DECIMAL column: it is read again whole, then streamed`);
|
|
1200
1906
|
this.startStream();
|
|
1201
1907
|
this.kick();
|
|
1202
1908
|
return;
|
|
1203
1909
|
}
|
|
1204
|
-
const m = `replication: ${e
|
|
1910
|
+
const m = `replication: ${this.errorText(e)}`;
|
|
1205
1911
|
// a failure that recurs must not restart the stream every second: each restart decodes the slot's WAL again on
|
|
1206
1912
|
// the source
|
|
1207
1913
|
this.streamFailures++;
|
|
@@ -1209,7 +1915,11 @@ export class Sync {
|
|
|
1209
1915
|
// a timer may fire a little early: the pass it kicks starts the stream
|
|
1210
1916
|
this.streamAt = Date.now() + wait - 50;
|
|
1211
1917
|
this.streamFailure = { error: m, retryAt: this.streamAt };
|
|
1212
|
-
this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the
|
|
1918
|
+
this.log(`${m} (the stream starts again in ${wait / 1000} s, failure ${this.streamFailures} in a row; the synced tables serve as they are)`);
|
|
1919
|
+
// one event for the stream, never one per table it kept current (each table's error says it); its failures in a
|
|
1920
|
+
// row are left out of the words, so a stream failing for good is one event, not one per restart
|
|
1921
|
+
this.event({ kind: "stream_failed", table: null, message: `${m} (the stream starts again by itself; the synced tables serve as they are)` });
|
|
1922
|
+
this.wholeFailure.stream = m;
|
|
1213
1923
|
// on the tables the stream kept current when it failed (one being caught up says its own)
|
|
1214
1924
|
await this.onWriter(async () => { for (const source of this.streamed.keys())
|
|
1215
1925
|
if (!this.held.has(source))
|
|
@@ -1238,10 +1948,11 @@ export class Sync {
|
|
|
1238
1948
|
this.held.delete(source);
|
|
1239
1949
|
const when = this.backOff(source, 0);
|
|
1240
1950
|
const pub = this.replication.publication;
|
|
1241
|
-
const m = `replication: the stream no longer carries ${source}'s ${cols}, which its
|
|
1951
|
+
const m = `replication: the stream no longer carries ${source}'s ${cols}, which its synced table holds (a column dropped or renamed on the database, or the publication's column list narrowed), ` +
|
|
1242
1952
|
`so its changes are not applied: it is caught up by one poll again after ${when}. The way out: remove ${cols} from its columns in the sources file (mason add ${source} --columns … ` +
|
|
1243
|
-
`picks the columns it has; it is
|
|
1953
|
+
`picks the columns it has; it is read again whole at once)${this.replication.owned ? "" : `, or publish ${missing.length === 1 ? "it" : "them"} again (ALTER PUBLICATION ${pub} …)`}`;
|
|
1244
1954
|
this.log(`${source}: ${m}`);
|
|
1955
|
+
this.event({ kind: "stream_failed", table: source, message: m });
|
|
1245
1956
|
await this.record({ ...this.states.get(source), error: m });
|
|
1246
1957
|
this.refreshStreamed();
|
|
1247
1958
|
});
|
|
@@ -1250,18 +1961,22 @@ export class Sync {
|
|
|
1250
1961
|
* What the stream applies, made again from the tables' state (on the writer's turn, or before the stream exists): a
|
|
1251
1962
|
* table copied at its present columns and caught up, and a table being caught up (its changes held, `held`). A table
|
|
1252
1963
|
* that is not — never copied, being copied again, or not caught up — is skipped: its copy's snapshot, or its catch-up,
|
|
1253
|
-
* covers what the stream passes over (src/duck/stream.ts "why replaying is safe").
|
|
1964
|
+
* covers what the stream passes over (src/duck/stream.ts "why replaying is safe"). A table here since the last refresh
|
|
1965
|
+
* keeps its STAY, a table new here gets one, and `swap` ends it: changes decoded in a stay that ended committed before
|
|
1966
|
+
* what came next, so that copy and its catch-up hold them, and only that stay's are applied (`applyStreamed`).
|
|
1254
1967
|
*/
|
|
1255
1968
|
refreshStreamed() {
|
|
1969
|
+
const before = new Map(this.streamed);
|
|
1256
1970
|
this.streamed.clear();
|
|
1257
1971
|
const now = new Set();
|
|
1258
1972
|
for (const t of this.tables.values()) {
|
|
1259
|
-
// a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot
|
|
1260
|
-
|
|
1973
|
+
// a table a named publication does not carry is never streamed (`unpublished`): its changes are not in the slot;
|
|
1974
|
+
// nor one whose sync is not replication (it is polled, or off)
|
|
1975
|
+
const joined = this.replicated(t) && !this.behind.has(t.source) && !this.needsCopy(t) && !this.unpublished.has(t.source);
|
|
1261
1976
|
// in the COPY's columns: a held table's columns may have changed since its copy began (`asCopied`)
|
|
1262
1977
|
const c = this.asCopied(t).columns;
|
|
1263
1978
|
if (joined || this.held.has(t.source)) {
|
|
1264
|
-
this.streamed.set(t.source, { columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows >
|
|
1979
|
+
this.streamed.set(t.source, { stay: before.get(t.source)?.stay ?? ++this.lastStay, columns: c.map((x) => x.name), key: t.key, generated: c.filter((x) => x.generated).map((x) => x.name), big: t.rows !== null && t.rows > BIG_TABLE_ROWS });
|
|
1265
1980
|
}
|
|
1266
1981
|
if (joined)
|
|
1267
1982
|
now.add(t.source);
|
|
@@ -1270,11 +1985,18 @@ export class Sync {
|
|
|
1270
1985
|
// while a stream runs: its start names what it streams. What it returns is the tables whose join was said
|
|
1271
1986
|
const said = [];
|
|
1272
1987
|
const s = this.stream;
|
|
1273
|
-
if (s !== null
|
|
1988
|
+
if (s !== null) {
|
|
1274
1989
|
const at = formatLsn(s.position());
|
|
1275
|
-
for (const source of this.joinedNow)
|
|
1276
|
-
if (
|
|
1277
|
-
|
|
1990
|
+
for (const source of this.joinedNow) {
|
|
1991
|
+
if (now.has(source))
|
|
1992
|
+
continue;
|
|
1993
|
+
const t = this.tables.get(source);
|
|
1994
|
+
// its sync changed: what keeps it current now, in the words of the switch
|
|
1995
|
+
const why = t !== undefined && !this.replicated(t)
|
|
1996
|
+
? `its sync is ${syncText(this.syncOf(t))} now, so it is caught up by one poll from its watermark${this.syncOf(t).mode === "off" ? ", then read again only by mason sync" : ", then polled"}`
|
|
1997
|
+
: this.notStreamedWhy(source);
|
|
1998
|
+
this.log(`${source} leaves the stream at ${at}: ${why}`);
|
|
1999
|
+
}
|
|
1278
2000
|
for (const source of now) {
|
|
1279
2001
|
if (this.joinedNow.has(source))
|
|
1280
2002
|
continue;
|
|
@@ -1296,52 +2018,72 @@ export class Sync {
|
|
|
1296
2018
|
return `${why} — after a failure, tried again after ${clock(at)} UTC`;
|
|
1297
2019
|
const ahead = this.rebuildingNow;
|
|
1298
2020
|
if (this.queued.has(source) && ahead !== null && ahead.source !== source)
|
|
1299
|
-
return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "
|
|
2021
|
+
return `${why} — queued behind ${ahead.source}'s ${ahead.kind === "copy" ? "whole read" : "catch-up"} (one rebuild at a time)`;
|
|
1300
2022
|
return why;
|
|
1301
2023
|
}
|
|
2024
|
+
/** why a table whose first sync finished is to be read again whole, or null: what the start's line and mason sync say */
|
|
2025
|
+
readAgainWhy(t) {
|
|
2026
|
+
if (this.copiedAs.get(t.source) !== columnsKey(t.columns))
|
|
2027
|
+
return "its columns changed";
|
|
2028
|
+
return this.snapshots.has(t.source) ? null : "a numeric value outgrew its synced DECIMAL column";
|
|
2029
|
+
}
|
|
1302
2030
|
notStreamedReason(source) {
|
|
1303
2031
|
const t = this.tables.get(source);
|
|
1304
2032
|
if (t === undefined)
|
|
1305
|
-
return "it is no longer added (its
|
|
2033
|
+
return "it is no longer added (its synced data is dropped)";
|
|
1306
2034
|
const gap = this.unpublished.get(source);
|
|
1307
2035
|
if (gap !== undefined)
|
|
1308
|
-
return gap
|
|
2036
|
+
return gapLine(gap);
|
|
1309
2037
|
const st = this.states.get(source);
|
|
1310
2038
|
if (this.copiedAs.has(source) && this.copiedAs.get(source) !== columnsKey(t.columns))
|
|
1311
|
-
return "its columns changed: it is
|
|
2039
|
+
return "its columns changed: it is read again whole, then caught up by one poll";
|
|
1312
2040
|
if (st?.phase === "copied" && !this.snapshots.has(source))
|
|
1313
|
-
return "a numeric value outgrew its
|
|
1314
|
-
|
|
1315
|
-
|
|
2041
|
+
return "a numeric value outgrew its synced DECIMAL column: it is read again whole, wider, then caught up by one poll";
|
|
2042
|
+
const r = this.retired.get(source);
|
|
2043
|
+
if (st?.phase === "copied" && r !== undefined)
|
|
2044
|
+
return `it is ${r.short}: it is read again whole, then caught up by one poll`;
|
|
1316
2045
|
// what follows the copy: a catch-up poll, then the stream or the polls; copied again whole, nothing (`keptOf`'s recopy)
|
|
1317
|
-
const joins = this.mode === "recopy" ? "it is kept current once
|
|
2046
|
+
const joins = this.mode === "recopy" ? "it is kept current once synced" : "it joins once synced and caught up by one poll";
|
|
1318
2047
|
if (st?.phase === "copying" || this.rebuildingNow?.source === source && this.rebuildingNow.kind === "copy")
|
|
1319
|
-
return `it is being
|
|
2048
|
+
return `it is being read whole: ${joins}`;
|
|
1320
2049
|
if (st?.phase === "failed")
|
|
1321
|
-
return `its
|
|
2050
|
+
return `its first sync failed (${(st.error ?? "").split("\n")[0]}): ${joins}`;
|
|
1322
2051
|
const b = this.behind.get(source);
|
|
1323
2052
|
if (!this.needsCopy(t) && b !== undefined)
|
|
1324
2053
|
return `it is caught up by one poll ${b.why} first`;
|
|
1325
|
-
|
|
2054
|
+
// synced, caught up, and not yet polled (`pollable`): its rebuild's last writer's turn, or a rebuild still queued
|
|
2055
|
+
if (st?.phase === "copied") {
|
|
2056
|
+
if (this.rebuildingNow?.source === source)
|
|
2057
|
+
return "its sync is ending: it is kept current from then on";
|
|
2058
|
+
const queued = this.queued.get(source)?.kind;
|
|
2059
|
+
if (queued !== undefined)
|
|
2060
|
+
return queued === "copy" ? `it is queued to be read whole: ${joins}` : "it is queued to be caught up by one poll";
|
|
2061
|
+
}
|
|
2062
|
+
return `it waits for its first sync: ${joins}`;
|
|
1326
2063
|
}
|
|
1327
2064
|
/**
|
|
1328
2065
|
* WHERE THE STREAM IS (`StreamStatus`): its position, the database's, the lag, the last batch, and each added table's
|
|
1329
2066
|
* part in it with what it applied this run. Null outside replication mode. The database's position costs a
|
|
1330
2067
|
* connection at most every 5 s (`databasePosition`); everything else is the run's own.
|
|
2068
|
+
*
|
|
2069
|
+
* `refresh: false` NEVER REACHES THE DATABASE: the position and the slot as last read (`database.at`, `slotWal.at` say
|
|
2070
|
+
* when), or "not read yet". What an unauthenticated reader asks with (`GET /metrics`, src/cli/run-server.ts
|
|
2071
|
+
* `metricsFacts`): a scraper's rate, or anyone's loop on the health port, must never become connections and SELECTs
|
|
2072
|
+
* on the source; the usage sampler, `mason status` and `mason wait` keep the reads fresh.
|
|
1331
2073
|
*/
|
|
1332
|
-
async streamStatus() {
|
|
1333
|
-
if (this.
|
|
2074
|
+
async streamStatus({ refresh = true } = {}) {
|
|
2075
|
+
if (!this.streams())
|
|
1334
2076
|
return null;
|
|
1335
2077
|
const r = this.replication;
|
|
1336
2078
|
const s = this.stream;
|
|
1337
2079
|
const position = s?.position() ?? this.appliedLsn;
|
|
1338
|
-
const db = await this.databasePosition();
|
|
2080
|
+
const db = refresh ? await this.databasePosition() : this.dbPosition ?? { error: "not read yet: nothing has asked where the database is since the run began", at: Date.now() };
|
|
1339
2081
|
const now = Date.now();
|
|
1340
2082
|
const iso = (ms) => new Date(ms).toISOString();
|
|
1341
2083
|
// each table's part, as `keptOf` says it for EXPLAIN (`streamTable`), with what it applied this run
|
|
1342
|
-
const tables =
|
|
2084
|
+
const tables = this.replicatedTables().map((t) => ({ table: t.source, ...this.streamTable(t, now, s), applied: { ...(this.appliedCounts.get(t.source) ?? noCounts()) } }));
|
|
1343
2085
|
return {
|
|
1344
|
-
slot: r.slot, publication: r.publication, running: s !== null,
|
|
2086
|
+
slot: r.slot, slotWal: this.slotWal?.wal ?? { error: "not read yet: the database did not answer within 3 s" }, publication: r.publication, running: s !== null,
|
|
1345
2087
|
failure: s === null && this.streamFailure !== null ? { error: this.streamFailure.error, retryAt: iso(this.streamFailure.retryAt) } : null,
|
|
1346
2088
|
position: formatLsn(position), applied: formatLsn(this.appliedLsn),
|
|
1347
2089
|
database: "lsn" in db ? { lsn: formatLsn(db.lsn), at: iso(db.at) } : { error: db.error },
|
|
@@ -1356,15 +2098,28 @@ export class Sync {
|
|
|
1356
2098
|
* wait` asks every second), waited for at most 3 s (the last read is answered meanwhile, with its time). Not the
|
|
1357
2099
|
* stream's keepalive: that carries the walsender's own read position, not the database's, so it cannot say how far
|
|
1358
2100
|
* behind the stream is when that matters — while it decodes a backlog. A read that fails is answered as its error.
|
|
2101
|
+
* THE SLOT'S WAL (`SlotWal`) is read on the same connection, at most every SLOT_READ_MS (one row of
|
|
2102
|
+
* pg_replication_slots): what the slot holds on the database's disk, said by `mason status` and sampled by usage.
|
|
1359
2103
|
*/
|
|
1360
2104
|
async databasePosition() {
|
|
1361
2105
|
const fresh = this.dbPosition !== null && Date.now() - this.dbPosition.at < 5_000;
|
|
1362
2106
|
if (!fresh && this.dbPositionRead === null) {
|
|
2107
|
+
const slotDue = this.slotWal === null || Date.now() - this.slotWal.at >= SLOT_READ_MS;
|
|
2108
|
+
const slot = this.replication.slot;
|
|
1363
2109
|
this.dbPositionRead = (async () => {
|
|
1364
2110
|
try {
|
|
1365
2111
|
const pg = await PgConnection.connect(this.pgOptions());
|
|
1366
2112
|
try {
|
|
1367
2113
|
this.dbPosition = { lsn: await currentLsn(pg), at: Date.now() };
|
|
2114
|
+
if (slotDue) {
|
|
2115
|
+
// its own failure is the slot's line alone: the position read above stands
|
|
2116
|
+
try {
|
|
2117
|
+
this.slotWal = { wal: slotWalOf(await readSlot(pg, slot), slot, Date.now()), at: Date.now() };
|
|
2118
|
+
}
|
|
2119
|
+
catch (e) {
|
|
2120
|
+
this.slotWal = { wal: { error: `the slot could not be read: ${this.errorText(e)}` }, at: Date.now() };
|
|
2121
|
+
}
|
|
2122
|
+
}
|
|
1368
2123
|
}
|
|
1369
2124
|
finally {
|
|
1370
2125
|
pg.close();
|
|
@@ -1372,7 +2127,10 @@ export class Sync {
|
|
|
1372
2127
|
}
|
|
1373
2128
|
catch (e) {
|
|
1374
2129
|
// answered as the status's (and the log line's) own words: the position could not be read, and why
|
|
1375
|
-
|
|
2130
|
+
const m = this.errorText(e);
|
|
2131
|
+
this.dbPosition = { error: m, at: Date.now() };
|
|
2132
|
+
if (slotDue)
|
|
2133
|
+
this.slotWal = { wal: { error: `the database could not be reached: ${m}` }, at: Date.now() };
|
|
1376
2134
|
}
|
|
1377
2135
|
finally {
|
|
1378
2136
|
this.dbPositionRead = null;
|
|
@@ -1413,10 +2171,10 @@ export class Sync {
|
|
|
1413
2171
|
this.refreshStreamed();
|
|
1414
2172
|
const applied = new Map();
|
|
1415
2173
|
for (const [name, changes] of batch) {
|
|
1416
|
-
|
|
1417
|
-
//
|
|
1418
|
-
//
|
|
1419
|
-
if (
|
|
2174
|
+
// decoded in a stay that ended (before its table was copied again, or left the stream): what they changed
|
|
2175
|
+
// committed before the next copy's catch-up, so that copy or its catch-up holds it — never written over them, in
|
|
2176
|
+
// a shape the copy may not have, or as an old change after the poll that read a newer one
|
|
2177
|
+
if (this.streamed.get(name)?.stay !== changes.stay)
|
|
1420
2178
|
continue;
|
|
1421
2179
|
const held = this.held.get(name);
|
|
1422
2180
|
if (held !== undefined) {
|
|
@@ -1425,7 +2183,7 @@ export class Sync {
|
|
|
1425
2183
|
}
|
|
1426
2184
|
applied.set(name, changes);
|
|
1427
2185
|
}
|
|
1428
|
-
await this.applyChanges(applied, lsn, committedAt);
|
|
2186
|
+
await this.applyChanges(applied, { lsn, committedAt });
|
|
1429
2187
|
});
|
|
1430
2188
|
}
|
|
1431
2189
|
/**
|
|
@@ -1436,12 +2194,26 @@ export class Sync {
|
|
|
1436
2194
|
* index each of those is a scan of the table's key column, so a batch costs a few scans per table, never one per row. A numeric wider than its copy's DECIMAL is the table's to copy
|
|
1437
2195
|
* again (`WIDER`), never rounded. A null `lsn` is a piece of a large transaction (src/duck/stream.ts): committed, the
|
|
1438
2196
|
* position left as it was, so a run that dies before the transaction's last piece applies it again whole. What it
|
|
1439
|
-
* applied is counted by table (`streamStatus`), and the batch's newest commit time kept
|
|
1440
|
-
*
|
|
2197
|
+
* applied is counted by table (`streamStatus`), and the batch's newest commit time kept.
|
|
2198
|
+
*
|
|
2199
|
+
* `"held"` is THE CHANGES A CATCH-UP HELD, replayed after its poll (`catchUp`): written, and counted, as a batch's are,
|
|
2200
|
+
* but NOT A POSITION of the stream — no `stream_state` row (its LSN and `applied_at` are the batches' that held them),
|
|
2201
|
+
* no table's `synced`, no commit time, and not the stream working again (`streamFailures`): the replay ran on the
|
|
2202
|
+
* writer, whatever the stream is doing (it may be failing).
|
|
1441
2203
|
*/
|
|
1442
|
-
async applyChanges(batch,
|
|
2204
|
+
async applyChanges(batch, from) {
|
|
2205
|
+
const lsn = from === "held" ? null : from.lsn;
|
|
2206
|
+
const committedAt = from === "held" ? null : from.committedAt;
|
|
1443
2207
|
const w = this.store.writer;
|
|
1444
2208
|
let source = null;
|
|
2209
|
+
const started = Date.now();
|
|
2210
|
+
// EACH TABLE'S ROWS AFTER THE BATCH, kept current at every batch (it was counted at most every 5 s, so a table's
|
|
2211
|
+
// count missed the batches after one until another came): what its writes added and removed, which DuckDB says
|
|
2212
|
+
// (`rowsChanged`) — never a count(*), which a write by key's deletes make a scan (12 ms on 20 M rows, measured) —
|
|
2213
|
+
// written to the ledger IN the batch's transaction, so a run that dies keeps a count equal to its rows
|
|
2214
|
+
const rowsAfter = new Map();
|
|
2215
|
+
// a batch with a position: its time in the ledger and in this run, ONE value (`Synced`)
|
|
2216
|
+
const position = lsn === null ? null : { lsn, at: new Date().toISOString() };
|
|
1445
2217
|
await w.run("BEGIN TRANSACTION");
|
|
1446
2218
|
try {
|
|
1447
2219
|
for (const [name, changes] of batch) {
|
|
@@ -1461,8 +2233,7 @@ export class Sync {
|
|
|
1461
2233
|
throw new Error(`${WIDER} [${name}]: ${c} = ${v}`);
|
|
1462
2234
|
return duckValue(v, types.get(c));
|
|
1463
2235
|
};
|
|
1464
|
-
|
|
1465
|
-
await w.run(`DELETE FROM ${target}`);
|
|
2236
|
+
let delta = changes.truncated ? -Number((await w.run(`DELETE FROM ${target}`)).rowsChanged) : 0;
|
|
1466
2237
|
const whole = [];
|
|
1467
2238
|
const fetch = [];
|
|
1468
2239
|
const gone = [];
|
|
@@ -1492,10 +2263,16 @@ export class Sync {
|
|
|
1492
2263
|
source ??= await PgConnection.connect(this.pgOptions());
|
|
1493
2264
|
whole.push(...await this.sourceRows(source, t, fetch));
|
|
1494
2265
|
}
|
|
1495
|
-
await this.writeByKey(t, target, gone, whole, value);
|
|
2266
|
+
delta += await this.writeByKey(t, target, gone, whole, value);
|
|
2267
|
+
// a copy whose rows the ledger does not know (none should be: its copy counts them) is counted, once
|
|
2268
|
+
const known = this.states.get(name)?.rows ?? null;
|
|
2269
|
+
const rows = known === null ? Number((await w.runAndReadAll(`SELECT count(*) FROM ${target}`)).getRowsJS()[0][0]) : known + delta;
|
|
2270
|
+
if (rows !== known)
|
|
2271
|
+
await w.run(`UPDATE ${MASON_SCHEMA}.copy_state SET "rows" = $1 WHERE source = $2`, [rows, name]);
|
|
2272
|
+
rowsAfter.set(name, rows);
|
|
1496
2273
|
}
|
|
1497
|
-
if (
|
|
1498
|
-
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2,
|
|
2274
|
+
if (position !== null)
|
|
2275
|
+
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.stream_state VALUES ($1, $2, $3::TIMESTAMPTZ)`, [this.replication.slot, formatLsn(position.lsn), position.at]);
|
|
1499
2276
|
await w.run("COMMIT");
|
|
1500
2277
|
}
|
|
1501
2278
|
catch (e) {
|
|
@@ -1507,11 +2284,19 @@ export class Sync {
|
|
|
1507
2284
|
}
|
|
1508
2285
|
if (lsn !== null)
|
|
1509
2286
|
this.appliedLsn = lsn;
|
|
1510
|
-
|
|
2287
|
+
if (from !== "held")
|
|
2288
|
+
this.streamFailures = 0;
|
|
2289
|
+
// THE STREAM'S FAILURE IS OVER when it applies a batch WITH A POSITION: every table it keeps current is current as of
|
|
2290
|
+
// it. A piece of a large transaction (no position) is not that yet, nor the changes a catch-up held
|
|
2291
|
+
if (lsn !== null)
|
|
2292
|
+
this.recovered("stream", "the stream applies changes again");
|
|
1511
2293
|
let moved = false;
|
|
1512
2294
|
for (const [name, changes] of batch) {
|
|
1513
2295
|
if (!this.tables.has(name))
|
|
1514
2296
|
continue;
|
|
2297
|
+
// committed: a generated table over it is behind from here (nothing of this is in the transaction above)
|
|
2298
|
+
if (changes.truncated || changes.keys.size > 0)
|
|
2299
|
+
this.generated.changed(name);
|
|
1515
2300
|
for (const into of [this.appliedCounts, this.sinceLine]) {
|
|
1516
2301
|
const c = into.get(name) ?? noCounts();
|
|
1517
2302
|
addCounts(c, changes.counts);
|
|
@@ -1521,17 +2306,30 @@ export class Sync {
|
|
|
1521
2306
|
}
|
|
1522
2307
|
if (moved && committedAt !== null && committedAt.getTime() > 0)
|
|
1523
2308
|
this.lastBatch = { appliedAt: Date.now(), committedAt: committedAt.getTime() };
|
|
1524
|
-
// the ledger
|
|
1525
|
-
|
|
2309
|
+
// the rows, as the ledger has them now; with a position, a table the batch wrote is current again (its error, if
|
|
2310
|
+
// any, cleared), and so is every table the stream keeps current whose error was the replication's (the stream's
|
|
2311
|
+
// failure, the preparation's): the position covers each of them, changed in the batch or not
|
|
2312
|
+
for (const [name, rows] of rowsAfter) {
|
|
1526
2313
|
const st = this.states.get(name);
|
|
1527
|
-
if (st === undefined
|
|
2314
|
+
if (st === undefined)
|
|
1528
2315
|
continue;
|
|
1529
|
-
|
|
1530
|
-
const rows = Number((await w.runAndReadAll(`SELECT count(*) FROM ${ident(t.schema)}.${ident(t.table)}`)).getRowsJS()[0][0]);
|
|
1531
|
-
this.countedAt.set(name, Date.now());
|
|
1532
|
-
if (rows !== st.rows || st.error !== null)
|
|
2316
|
+
if (st.error !== null && lsn !== null)
|
|
1533
2317
|
await this.record({ ...st, rows, error: null });
|
|
2318
|
+
else
|
|
2319
|
+
this.states.set(name, { ...st, rows });
|
|
1534
2320
|
}
|
|
2321
|
+
if (lsn !== null) {
|
|
2322
|
+
for (const source of this.joinedNow) {
|
|
2323
|
+
const st = this.states.get(source);
|
|
2324
|
+
if (st !== undefined && st.phase === "copied" && st.error?.startsWith("replication: "))
|
|
2325
|
+
await this.record({ ...st, error: null });
|
|
2326
|
+
}
|
|
2327
|
+
}
|
|
2328
|
+
// every table the stream keeps current is current as of this batch's position (`Synced`); a piece of a large
|
|
2329
|
+
// transaction (no position) and the changes a catch-up held are not a position
|
|
2330
|
+
if (position !== null)
|
|
2331
|
+
for (const source of this.joinedNow)
|
|
2332
|
+
this.synced.set(source, { at: position.at, seconds: (Date.now() - started) / 1000, by: "stream" });
|
|
1535
2333
|
}
|
|
1536
2334
|
/**
|
|
1537
2335
|
* Rows an UPDATE left a large value unchanged in, every one carrying the same columns (`carried`, the key among them):
|
|
@@ -1561,11 +2359,13 @@ export class Sync {
|
|
|
1561
2359
|
* join against the table's key column (`deleteKeys`), then the rows inserted. No copy has a primary key,
|
|
1562
2360
|
* so there is no `INSERT OR REPLACE`; a key's row is the source's at the batch's end
|
|
1563
2361
|
* whatever the table held, as replaying needs (src/duck/stream.ts). A table with no key (none can be added:
|
|
1564
|
-
* src/cli/preflight.ts) is only appended to.
|
|
2362
|
+
* src/cli/preflight.ts) is only appended to. Answers what it did to the table's count: the rows inserted less the rows
|
|
2363
|
+
* its deletes removed (DuckDB's own count of each).
|
|
1565
2364
|
*/
|
|
1566
2365
|
async writeByKey(t, target, gone, whole, value) {
|
|
1567
2366
|
if (gone.length === 0 && whole.length === 0)
|
|
1568
|
-
return;
|
|
2367
|
+
return 0;
|
|
2368
|
+
let delta = 0;
|
|
1569
2369
|
const w = this.store.writer;
|
|
1570
2370
|
const cols = t.columns.map((c) => c.name);
|
|
1571
2371
|
const list = cols.map(ident).join(", ");
|
|
@@ -1583,15 +2383,16 @@ export class Sync {
|
|
|
1583
2383
|
}
|
|
1584
2384
|
if (whole.length)
|
|
1585
2385
|
await w.run(`INSERT INTO ${WRITE_KEYS} SELECT ${keys} FROM ${WRITE_ROWS}`);
|
|
1586
|
-
await deleteKeys(w, t, target, WRITE_KEYS, t.key);
|
|
2386
|
+
delta -= await deleteKeys(w, t, target, WRITE_KEYS, t.key);
|
|
1587
2387
|
await w.run(`DROP TABLE ${WRITE_KEYS}`);
|
|
1588
2388
|
}
|
|
1589
2389
|
if (whole.length) {
|
|
1590
2390
|
// and its stored JSON paths, computed from the rows' JSON (src/duck/kept.ts)
|
|
1591
2391
|
const stored = this.kept.stored(t.source, ident);
|
|
1592
|
-
await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`);
|
|
2392
|
+
delta += Number((await w.run(`INSERT INTO ${target} (${[list, ...stored.names.map(ident)].join(", ")}) SELECT ${[list, ...stored.values].join(", ")} FROM ${WRITE_ROWS}`)).rowsChanged);
|
|
1593
2393
|
await w.run(`DROP TABLE ${WRITE_ROWS}`);
|
|
1594
2394
|
}
|
|
2395
|
+
return delta;
|
|
1595
2396
|
}
|
|
1596
2397
|
/** a copied table's columns' DuckDB types, as DESCRIBE spells them (what `duckValue` casts a value's text to) */
|
|
1597
2398
|
async columnTypes(t, target) {
|
|
@@ -1667,26 +2468,55 @@ export class Sync {
|
|
|
1667
2468
|
* whose attach throws is not one: it never detaches */
|
|
1668
2469
|
async attach() {
|
|
1669
2470
|
if (!this.attached) {
|
|
1670
|
-
//
|
|
1671
|
-
//
|
|
2471
|
+
// the extension ships with Mason (src/duck/extensions.ts): a LOAD from the platform's package, nothing downloaded;
|
|
2472
|
+
// without the package, INSTALL fetches it the first time on a machine (then ~/.duckdb/extensions), said once,
|
|
2473
|
+
// and offline the first copy fails here saying why
|
|
1672
2474
|
if (!this.loaded) {
|
|
2475
|
+
const x = this.store.extensions;
|
|
2476
|
+
if (!x.shipped)
|
|
2477
|
+
this.log(`Mason's extensions are not installed with it here: ${x.why}. The first sync on this machine downloads its postgres extension`);
|
|
1673
2478
|
try {
|
|
1674
|
-
await this.store.writer.run(
|
|
2479
|
+
await this.store.writer.run(loadPostgresSql(x));
|
|
1675
2480
|
}
|
|
1676
2481
|
catch (e) {
|
|
1677
|
-
throw new Error(
|
|
1678
|
-
"DuckDB's extension repository (then ~/.duckdb/extensions): connect to the internet once, and the copy is tried again at the next pass");
|
|
2482
|
+
throw new Error(extensionFailure(x, this.errorText(e)));
|
|
1679
2483
|
}
|
|
1680
2484
|
this.loaded = true;
|
|
1681
2485
|
}
|
|
1682
|
-
|
|
2486
|
+
const w = this.store.writer;
|
|
2487
|
+
// the engine's own words are never repeated here: its statement held the password (`sourceSecretSql`)
|
|
2488
|
+
try {
|
|
2489
|
+
await w.run(sourceSecretSql(this.source));
|
|
2490
|
+
}
|
|
2491
|
+
// ITS OWN WORDS ARE NEVER REPEATED: the statement held the password, and an error may quote its statement — its
|
|
2492
|
+
// code alone is said (src/lib/secret-mask.ts errorCode)
|
|
2493
|
+
catch (e) {
|
|
2494
|
+
throw new Error(`the database's credentials could not be set up for the sync (Mason bug: report it, with this code; the error's own words are not shown, as its statement held the password): ${errorCode(e) ?? "no code"}`);
|
|
2495
|
+
}
|
|
2496
|
+
// the engine's failure to connect, in Mason's words (`errorText`) where it enters
|
|
2497
|
+
try {
|
|
2498
|
+
await w.run(attachSqlOf(this.source));
|
|
2499
|
+
}
|
|
2500
|
+
catch (e) {
|
|
2501
|
+
throw new Error(this.errorText(e));
|
|
2502
|
+
}
|
|
2503
|
+
finally {
|
|
2504
|
+
// the attachment keeps what it merged; the secret goes at once, whatever the ATTACH did. One left behind is
|
|
2505
|
+
// the instance's, in memory, unreadable (duckdb_secrets() redacts it) and replaced at the next attach: said
|
|
2506
|
+
try {
|
|
2507
|
+
await w.run(`DROP TEMPORARY SECRET IF EXISTS ${SOURCE_SECRET}`);
|
|
2508
|
+
}
|
|
2509
|
+
catch (e) {
|
|
2510
|
+
this.log(`the database's credentials could not be removed from memory once it was attached (they stay there, never on disk, until the next attach replaces them): ${this.errorText(e)}`);
|
|
2511
|
+
}
|
|
2512
|
+
}
|
|
1683
2513
|
this.attached = true;
|
|
1684
2514
|
}
|
|
1685
2515
|
this.sourceUsers++;
|
|
1686
2516
|
}
|
|
1687
2517
|
/**
|
|
1688
2518
|
* THE SOURCE IS ATTACHED ONLY WHILE A PASS OR A REBUILD READS IT. An attached database is every connection's — a wire
|
|
1689
|
-
* session's too, and the rebuilds' — and its path
|
|
2519
|
+
* session's too, and the rebuilds' — and its path (`duckdb_databases()`) is `attachPath`'s, which holds no credential.
|
|
1690
2520
|
* A session runs only planned SQL (src/duck/plan.ts PlannedSql), which cannot name it; detached when its last user
|
|
1691
2521
|
* ends (on the writer's turn, so never under a transaction reading it), it is not there to name either.
|
|
1692
2522
|
*/
|
|
@@ -1700,7 +2530,7 @@ export class Sync {
|
|
|
1700
2530
|
}
|
|
1701
2531
|
// the pass's work stands; still attached, the next pass detaches it again (and attach() sees it is attached)
|
|
1702
2532
|
catch (e) {
|
|
1703
|
-
this.log(`the source could not be detached after the pass (tried again after the next): ${
|
|
2533
|
+
this.log(`the source could not be detached after the pass (tried again after the next): ${this.errorText(e)}`);
|
|
1704
2534
|
}
|
|
1705
2535
|
}
|
|
1706
2536
|
/** one pass: every removed table dropped, every added table needing it queued for a copy (step 0: whole, again) or
|
|
@@ -1726,40 +2556,76 @@ export class Sync {
|
|
|
1726
2556
|
});
|
|
1727
2557
|
this.snapshots.delete(source);
|
|
1728
2558
|
this.retries.delete(source);
|
|
2559
|
+
this.due.delete(source);
|
|
2560
|
+
this.lastPollMs.delete(source);
|
|
2561
|
+
this.recordedSync.delete(source);
|
|
2562
|
+
this.switched.delete(source);
|
|
1729
2563
|
this.retired.delete(source);
|
|
1730
2564
|
this.copiedAs.delete(source);
|
|
1731
2565
|
this.copiedColumns.delete(source);
|
|
1732
2566
|
this.states.delete(source);
|
|
1733
|
-
this.
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
|
|
1737
|
-
|
|
2567
|
+
this.synced.delete(source);
|
|
2568
|
+
this.log(`dropped ${source}'s synced data: it is no longer added (mason add ${source} syncs it again from the source, whole)`);
|
|
2569
|
+
}
|
|
2570
|
+
// REPLICATION, as the tables' syncs say: the publication and slot made ready for the replicated tables and the stream
|
|
2571
|
+
// started — or, when none is replicated any more, the slot dropped (verified gone). Its failure is the replicated
|
|
2572
|
+
// tables' alone (not copied, not caught up: `work`), and the pass's, said once the polled tables are polled
|
|
2573
|
+
let replicationFailed = null;
|
|
2574
|
+
let replicationReady = false;
|
|
2575
|
+
for (const source of [...this.unpublished.keys()]) {
|
|
2576
|
+
const t = this.tables.get(source);
|
|
2577
|
+
if (t === undefined || !this.replicated(t))
|
|
2578
|
+
this.unpublished.delete(source);
|
|
1738
2579
|
}
|
|
1739
|
-
if (this.
|
|
2580
|
+
if (this.streams()) {
|
|
1740
2581
|
try {
|
|
1741
2582
|
await this.prepareReplication();
|
|
2583
|
+
replicationReady = true;
|
|
1742
2584
|
}
|
|
1743
2585
|
catch (e) {
|
|
1744
|
-
|
|
2586
|
+
// the way out on the table's own line: a failure that names none (a slot the database will not make, a role that
|
|
2587
|
+
// may not replicate) says the line that keeps the tables polled meanwhile — never a bare SQLSTATE's text
|
|
2588
|
+
const said = this.errorText(e);
|
|
2589
|
+
const names = this.replicatedTables().map((t) => t.source);
|
|
2590
|
+
const m = `replication: ${said}${/sync: poll/.test(messageOf(e)) ? "" : ` — ${names.join(", ")} ${names.length === 1 ? "is" : "are"} NOT kept current until it is fixed: ` +
|
|
2591
|
+
`give ${names.length === 1 ? "its entry" : "their entries"} sync: poll <interval> (poll 5m) in the sources file to poll ${names.length === 1 ? "it" : "them"} instead`}`;
|
|
2592
|
+
this.event({ kind: "stream_failed", table: null, message: m });
|
|
2593
|
+
this.wholeFailure.prepare = m;
|
|
1745
2594
|
// a table copied before keeps serving its copy; one never copied is failed, so mason status says why
|
|
1746
2595
|
await this.onWriter(async () => {
|
|
1747
|
-
for (const t of this.
|
|
2596
|
+
for (const t of this.replicatedTables()) {
|
|
1748
2597
|
const st = this.states.get(t.source);
|
|
1749
2598
|
await this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m });
|
|
1750
2599
|
}
|
|
1751
2600
|
});
|
|
1752
|
-
|
|
1753
|
-
throw e;
|
|
2601
|
+
replicationFailed = e;
|
|
1754
2602
|
}
|
|
2603
|
+
this.recovered("prepare", "the publication and the slot are ready again");
|
|
1755
2604
|
// the stream starts once the slot is known, and runs from then on whatever the passes and rebuilds do
|
|
1756
|
-
|
|
2605
|
+
if (replicationReady)
|
|
2606
|
+
this.startStream();
|
|
2607
|
+
}
|
|
2608
|
+
else if (this.replication !== null && (this.slotKnown || this.stream !== null)) {
|
|
2609
|
+
try {
|
|
2610
|
+
await this.retireReplication();
|
|
2611
|
+
}
|
|
2612
|
+
catch (e) {
|
|
2613
|
+
replicationFailed = new Error(`no table's sync is replication now, but replication slot ${this.replication.slot} could not be dropped (it holds the database's WAL; tried again at the next pass): ${this.errorText(e)}`);
|
|
2614
|
+
}
|
|
2615
|
+
}
|
|
2616
|
+
if (this.tables.size === 0) {
|
|
2617
|
+
await this.rebuildSchema();
|
|
2618
|
+
if (replicationFailed !== null)
|
|
2619
|
+
throw replicationFailed;
|
|
2620
|
+
return;
|
|
1757
2621
|
}
|
|
1758
2622
|
try {
|
|
1759
2623
|
await this.onWriter(() => this.attach());
|
|
1760
2624
|
}
|
|
1761
2625
|
catch (e) {
|
|
1762
|
-
const m =
|
|
2626
|
+
const m = this.errorText(e);
|
|
2627
|
+
this.event({ kind: "source_unreachable", table: null, message: `could not attach the database: ${m}` });
|
|
2628
|
+
this.wholeFailure.attach = m;
|
|
1763
2629
|
// a table copied before keeps serving its copy (and its place in the catalog): the source being away is its error,
|
|
1764
2630
|
// not a reason to copy it whole again when the source is back; one never copied is failed
|
|
1765
2631
|
await this.onWriter(async () => {
|
|
@@ -1770,18 +2636,22 @@ export class Sync {
|
|
|
1770
2636
|
});
|
|
1771
2637
|
// the copies serve while the source is away: the catalog lists them (on a fresh start, nothing else has)
|
|
1772
2638
|
await this.rebuildSchema();
|
|
1773
|
-
throw new Error(`
|
|
2639
|
+
throw new Error(`Mason could not attach ${this.source.database} on ${this.source.host}:${this.source.port}: ${m}`);
|
|
1774
2640
|
}
|
|
2641
|
+
this.recovered("attach", "the database is attached again");
|
|
1775
2642
|
try {
|
|
1776
|
-
await this.work();
|
|
2643
|
+
await this.work(replicationReady);
|
|
1777
2644
|
}
|
|
1778
2645
|
finally {
|
|
1779
2646
|
await this.onWriter(() => this.detach());
|
|
1780
2647
|
}
|
|
1781
2648
|
await this.rebuildSchema();
|
|
2649
|
+
if (replicationFailed !== null)
|
|
2650
|
+
throw replicationFailed;
|
|
1782
2651
|
}
|
|
1783
|
-
/** a pass's queueing and polls, the source attached
|
|
1784
|
-
|
|
2652
|
+
/** a pass's queueing and polls, the source attached; a replicated table is copied or caught up only once the publication
|
|
2653
|
+
* carries it and the slot exists (`replicationReady`): its changes are in the stream from before it joins */
|
|
2654
|
+
async work(replicationReady) {
|
|
1785
2655
|
const started = Date.now();
|
|
1786
2656
|
// a table never copied (or whose copy failed, or whose copy is out of date) is copied whole, beside the rest: the
|
|
1787
2657
|
// smallest first (`copyOrder`)
|
|
@@ -1794,33 +2664,27 @@ export class Sync {
|
|
|
1794
2664
|
// a table the named publication does not carry: copied, nothing would keep it current (prepareReplication)
|
|
1795
2665
|
if (this.unpublished.has(t.source))
|
|
1796
2666
|
continue;
|
|
2667
|
+
if (this.replicated(t) && !replicationReady)
|
|
2668
|
+
continue;
|
|
1797
2669
|
if (this.mode !== "recopy" && !this.needsCopy(t))
|
|
1798
2670
|
continue;
|
|
1799
2671
|
if (this.queued.has(t.source) || this.rebuildingNow?.source === t.source)
|
|
1800
2672
|
continue;
|
|
1801
2673
|
if (this.states.get(t.source)?.phase === "copied" && this.copiedAs.get(t.source) !== columnsKey(t.columns))
|
|
1802
|
-
this.log(`${t.source}: its columns are not the ones it was
|
|
2674
|
+
this.log(`${t.source}: its columns are not the ones it was synced with: it is read again whole`);
|
|
1803
2675
|
this.enqueue(t.source, { kind: "copy" });
|
|
1804
2676
|
}
|
|
1805
2677
|
// a table copied and not yet caught up (a slot new to the copy, or a run that stopped first): its one poll
|
|
1806
2678
|
for (const source of this.behind.keys()) {
|
|
1807
2679
|
const t = this.tables.get(source);
|
|
1808
2680
|
// a failed catch-up waits its turn as a failed copy does (`backOff`); the error it recorded says when
|
|
1809
|
-
if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now())
|
|
2681
|
+
if (t !== undefined && !this.needsCopy(t) && (this.retries.get(source)?.at ?? 0) <= Date.now() && (!this.replicated(t) || replicationReady))
|
|
1810
2682
|
this.enqueue(source, { kind: "catch-up" });
|
|
1811
2683
|
}
|
|
1812
|
-
//
|
|
1813
|
-
|
|
1814
|
-
|
|
1815
|
-
const
|
|
1816
|
-
for (const t of big.filter((x) => !this.leftOut.has(x.source))) {
|
|
1817
|
-
this.log(`${t.source} (about ${t.rows.toLocaleString("en-US")} rows) is ${OVER_POLL_LIMIT}: it is served as copied, NOT kept current by the poll ` +
|
|
1818
|
-
`(each poll would scan it whole on the database) — mason sync ${t.source} syncs it now; mason start --replication keeps every table current`);
|
|
1819
|
-
this.leftOut.add(t.source);
|
|
1820
|
-
}
|
|
1821
|
-
for (const s of [...this.leftOut].filter((x) => !big.some((t) => t.source === x)))
|
|
1822
|
-
this.leftOut.delete(s);
|
|
1823
|
-
const kept = this.mode === "poll" ? polled.filter((t) => !big.includes(t)) : [];
|
|
2684
|
+
// POLL MODE: the tables due (`dueAt`: a polled one past its interval, one whose sync changed), and not a table whose
|
|
2685
|
+
// new columns could not be copied (its copy failed above): it is not polled with them
|
|
2686
|
+
const now = Date.now();
|
|
2687
|
+
const kept = this.mode !== "recopy" ? [...this.tables.values()].filter((t) => !this.replicated(t) && this.pollable(t) && (this.dueAt(t) ?? Infinity) <= now) : [];
|
|
1824
2688
|
for (const t of kept) {
|
|
1825
2689
|
if (this.stopped)
|
|
1826
2690
|
break;
|
|
@@ -1828,22 +2692,27 @@ export class Sync {
|
|
|
1828
2692
|
// or a swap may run between two. A table rebuilt or not yet caught up is not polled (decided on the turn itself);
|
|
1829
2693
|
// its catch-up polls it
|
|
1830
2694
|
try {
|
|
1831
|
-
await this.onWriter(async () =>
|
|
1832
|
-
|
|
2695
|
+
const found = await this.onWriter(async () => this.rebuildingNow?.source !== t.source && !this.behind.has(t.source) ? this.poll(this.writerLane, [t]) : []);
|
|
2696
|
+
for (const f of found)
|
|
2697
|
+
this.polledNow(t, f.ms);
|
|
1833
2698
|
}
|
|
1834
2699
|
catch (e) {
|
|
1835
2700
|
if (e instanceof Stopping) {
|
|
1836
|
-
this.log("poll stopped: Mason is stopping (rolled back; the
|
|
2701
|
+
this.log("poll stopped: Mason is stopping (rolled back; the synced tables serve as they were)");
|
|
1837
2702
|
return;
|
|
1838
2703
|
}
|
|
2704
|
+
// tried again after its interval (an `off` table whose sync changed: after the sync's), never at every pass
|
|
2705
|
+
const s = this.syncOf(t);
|
|
2706
|
+
this.due.set(t.source, Date.now() + (s.mode === "poll" ? s.seconds * 1000 : this.intervalMs));
|
|
1839
2707
|
// the table serves its copy as it was, and says why it is not current (mason status reads the ledger); the other
|
|
1840
2708
|
// tables are polled as ever
|
|
1841
|
-
const m = `the poll failed, and is tried again at the next one (the
|
|
2709
|
+
const m = `the poll failed, and is tried again at the next one (the synced data serves as it was): ${this.errorText(e)}`;
|
|
1842
2710
|
this.log(`${t.source}: ${m}`);
|
|
2711
|
+
this.event({ kind: "poll_failed", table: t.source, message: m });
|
|
1843
2712
|
await this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
|
|
1844
2713
|
}
|
|
1845
2714
|
}
|
|
1846
|
-
if (this.mode
|
|
2715
|
+
if (this.mode !== "recopy")
|
|
1847
2716
|
await this.onWriter(() => this.readPolledAt());
|
|
1848
2717
|
this.lastPassMs = Date.now() - started;
|
|
1849
2718
|
}
|
|
@@ -1867,9 +2736,16 @@ export class Sync {
|
|
|
1867
2736
|
if (next === undefined)
|
|
1868
2737
|
return;
|
|
1869
2738
|
this.queued.delete(next[0]);
|
|
2739
|
+
// a generated table's run reads no source: nothing is attached and no table is marked as rebuilt
|
|
2740
|
+
const kind = next[1].kind;
|
|
2741
|
+
if (kind === "generated") {
|
|
2742
|
+
await this.generated.run(next[0]).catch((e) => { if (!(e instanceof Stopping))
|
|
2743
|
+
throw e; });
|
|
2744
|
+
continue;
|
|
2745
|
+
}
|
|
1870
2746
|
const t = this.tables.get(next[0]);
|
|
1871
2747
|
// a catch-up that found a numeric wider than the copy: copied again, next
|
|
1872
|
-
if (t !== undefined && await this.rebuild(t,
|
|
2748
|
+
if (t !== undefined && await this.rebuild(t, kind) === "copy again")
|
|
1873
2749
|
this.enqueue(t.source, { kind: "copy" });
|
|
1874
2750
|
// still needing a copy or a catch-up: a pass now, or when its back-off ends
|
|
1875
2751
|
else
|
|
@@ -1879,7 +2755,7 @@ export class Sync {
|
|
|
1879
2755
|
this.kick();
|
|
1880
2756
|
}
|
|
1881
2757
|
})().catch((e) => {
|
|
1882
|
-
this.log(`a rebuild failed outside its own handling (Mason bug: report it): ${e
|
|
2758
|
+
this.log(`a rebuild failed outside its own handling (Mason bug: report it): ${this.errorText(e, { stack: true })}`);
|
|
1883
2759
|
}).finally(() => {
|
|
1884
2760
|
this.rebuilding = null;
|
|
1885
2761
|
// what was queued behind a rebuild that threw is still queued
|
|
@@ -1891,9 +2767,9 @@ export class Sync {
|
|
|
1891
2767
|
* ONE REBUILD, beside the stream and the polls: the table marked as rebuilt on the writer's turn (from then on no poll
|
|
1892
2768
|
* starts on it), then its copy on the rebuilds' connection, then — a table left behind by it — its catch-up.
|
|
1893
2769
|
*/
|
|
1894
|
-
async rebuild(t,
|
|
2770
|
+
async rebuild(t, kind) {
|
|
1895
2771
|
await this.onWriter(async () => {
|
|
1896
|
-
this.rebuildingNow = { source: t.source, kind
|
|
2772
|
+
this.rebuildingNow = { source: t.source, kind };
|
|
1897
2773
|
this.refreshStreamed();
|
|
1898
2774
|
});
|
|
1899
2775
|
let attached = false;
|
|
@@ -1904,14 +2780,18 @@ export class Sync {
|
|
|
1904
2780
|
}
|
|
1905
2781
|
catch (e) {
|
|
1906
2782
|
// the next pass says the same of every table, and queues it again
|
|
1907
|
-
const m = `could not attach the database: ${
|
|
1908
|
-
this.log(`${
|
|
2783
|
+
const m = `could not attach the database: ${this.errorText(e)}`;
|
|
2784
|
+
this.log(`${kind} of ${t.source} waits: ${m}`);
|
|
2785
|
+
this.event({ kind: "source_unreachable", table: t.source, message: `its ${kind} waits: ${m}` });
|
|
1909
2786
|
const st = this.states.get(t.source);
|
|
1910
2787
|
await this.onWriter(() => this.record({ ...st, phase: st.phase === "copied" ? "copied" : "failed", error: m }));
|
|
1911
2788
|
return null;
|
|
1912
2789
|
}
|
|
1913
|
-
if (
|
|
2790
|
+
if (kind === "copy")
|
|
1914
2791
|
await this.copy(t);
|
|
2792
|
+
// the whole read is over, whatever comes next: what the table is said to be doing from here (`notStreamedReason`,
|
|
2793
|
+
// `refuseRebuilding`, `state`'s `rebuilding`)
|
|
2794
|
+
this.rebuildingNow = { source: t.source, kind: "catch-up" };
|
|
1915
2795
|
// a table its copy left behind (`swap`), or one marked so before: one poll, unless a copy of it comes first — the
|
|
1916
2796
|
// table as it is added NOW, which may have changed while it was copied (then copied again at once: `followUp`)
|
|
1917
2797
|
const now = this.tables.get(t.source);
|
|
@@ -1936,11 +2816,11 @@ export class Sync {
|
|
|
1936
2816
|
* copy (the copy's snapshot), or a slot new to the copy — on the rebuilds'
|
|
1937
2817
|
* connection, so the stream and the other tables' polls go on meanwhile. In replication mode the stream holds the
|
|
1938
2818
|
* table's changes from before the poll's snapshot and applies them after it (src/duck/stream.ts "why replaying is
|
|
1939
|
-
* safe"). A table
|
|
2819
|
+
* safe"). A table whose sync is `off` is caught up too: the one scan its rebuild costs, then nothing until asked.
|
|
1940
2820
|
*/
|
|
1941
2821
|
async catchUp(t) {
|
|
1942
2822
|
const mark = this.behind.get(t.source);
|
|
1943
|
-
const replication = this.
|
|
2823
|
+
const replication = this.replicated(t);
|
|
1944
2824
|
// J: from the writer's turn on, every batch's changes of this table are held (`applyStreamed`)
|
|
1945
2825
|
if (replication)
|
|
1946
2826
|
await this.onWriter(async () => { if (!this.held.has(t.source))
|
|
@@ -1953,8 +2833,9 @@ export class Sync {
|
|
|
1953
2833
|
// mason status — and is caught up again then: what was held is read again by that poll (still behind, nothing lost)
|
|
1954
2834
|
const failed = (what, e) => {
|
|
1955
2835
|
const when = this.backOff(t.source, Date.now() - started);
|
|
1956
|
-
const m = `${what} (the
|
|
2836
|
+
const m = `${what} (the synced data serves as it was; it is caught up by one poll again after ${when}): ${this.errorText(e)}`;
|
|
1957
2837
|
this.log(`${t.source}: ${m}`);
|
|
2838
|
+
this.event({ kind: "catch_up_failed", table: t.source, message: m });
|
|
1958
2839
|
return this.onWriter(() => this.record({ ...this.states.get(t.source), error: m }));
|
|
1959
2840
|
};
|
|
1960
2841
|
// the catch-up's poll reads the columns the copy holds (`asCopied`)
|
|
@@ -1977,6 +2858,7 @@ export class Sync {
|
|
|
1977
2858
|
await release();
|
|
1978
2859
|
return "copy again";
|
|
1979
2860
|
}
|
|
2861
|
+
this.polledNow(t, found[0].ms);
|
|
1980
2862
|
let saidJoin = false;
|
|
1981
2863
|
if (this.catchingUp?.source === t.source)
|
|
1982
2864
|
this.catchingUp.phase = "applying";
|
|
@@ -1987,15 +2869,15 @@ export class Sync {
|
|
|
1987
2869
|
this.held.delete(t.source);
|
|
1988
2870
|
if (held !== undefined && (held.truncated || held.keys.size > 0)) {
|
|
1989
2871
|
try {
|
|
1990
|
-
await this.applyChanges(new Map([[t.source, held]]),
|
|
2872
|
+
await this.applyChanges(new Map([[t.source, held]]), "held");
|
|
1991
2873
|
}
|
|
1992
2874
|
catch (e) {
|
|
1993
|
-
const m =
|
|
2875
|
+
const m = this.errorText(e);
|
|
1994
2876
|
this.refreshStreamed();
|
|
1995
2877
|
// a numeric wider than its copy: copied again (`needsCopy`), and caught up after that copy
|
|
1996
|
-
if (
|
|
2878
|
+
if (widerIn(e, t.source)) {
|
|
1997
2879
|
this.snapshots.delete(t.source);
|
|
1998
|
-
this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is
|
|
2880
|
+
this.log(`${t.source}: the changes the stream held during its catch-up could not be applied: it is read again whole, wider (${m})`);
|
|
1999
2881
|
return "copy again";
|
|
2000
2882
|
}
|
|
2001
2883
|
heldFailure = e;
|
|
@@ -2024,9 +2906,10 @@ export class Sync {
|
|
|
2024
2906
|
return joined || null;
|
|
2025
2907
|
if (saidJoin)
|
|
2026
2908
|
return null;
|
|
2909
|
+
const s = this.syncOf(t);
|
|
2027
2910
|
const tail = replication ? (this.behind.has(t.source) ? "it is caught up again first (the slot was made again meanwhile)" : "the stream carries what follows once it runs again")
|
|
2028
|
-
:
|
|
2029
|
-
:
|
|
2911
|
+
: s.mode === "off" ? `its sync is off, so it is served as of now: mason sync ${t.source} syncs it again`
|
|
2912
|
+
: `the poll keeps it current (${syncWords(s)})`;
|
|
2030
2913
|
this.log(`caught up ${t.source} by one poll ${mark.why}: ${tail}`);
|
|
2031
2914
|
return null;
|
|
2032
2915
|
}
|
|
@@ -2040,12 +2923,15 @@ export class Sync {
|
|
|
2040
2923
|
async swap(t, side, o) {
|
|
2041
2924
|
const w = this.store.writer;
|
|
2042
2925
|
const target = `${ident(t.schema)}.${ident(t.table)}`;
|
|
2926
|
+
// the copy's time in the ledger and in this run, ONE value, and before `copied_at`, recorded after the swap
|
|
2927
|
+
// (`readSynced`: a copy recorded at or after its polled_at was last brought current by that copy)
|
|
2928
|
+
const at = new Date().toISOString();
|
|
2043
2929
|
await w.run("BEGIN TRANSACTION");
|
|
2044
2930
|
try {
|
|
2045
2931
|
await w.run(`DROP TABLE IF EXISTS ${target}`);
|
|
2046
2932
|
await w.run(`ALTER TABLE ${side} RENAME TO ${ident(t.table)}`);
|
|
2047
2933
|
if (o.snapshot !== null)
|
|
2048
|
-
await w.run(
|
|
2934
|
+
await w.run(this.pollStateSql(t, o.snapshot, at));
|
|
2049
2935
|
if (o.behind !== null)
|
|
2050
2936
|
await w.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.catch_up VALUES (${literal(t.source)}, ${literal(o.behind)}, now())`);
|
|
2051
2937
|
// the copy is whole: nothing of it is left to continue
|
|
@@ -2059,21 +2945,37 @@ export class Sync {
|
|
|
2059
2945
|
await w.run("ROLLBACK");
|
|
2060
2946
|
throw e;
|
|
2061
2947
|
}
|
|
2948
|
+
// committed: a new copy, so a generated table over it is behind from here
|
|
2949
|
+
this.generated.changed(t.source);
|
|
2062
2950
|
// the copy's view of its visible columns, made again (its own transaction: DuckDB writes one database a transaction)
|
|
2063
2951
|
await this.kept.swapped(t.source);
|
|
2064
|
-
if (o.snapshot !== null)
|
|
2952
|
+
if (o.snapshot !== null) {
|
|
2065
2953
|
this.snapshots.set(t.source, o.snapshot);
|
|
2954
|
+
this.recordedSync.set(t.source, syncText(this.syncOf(t)));
|
|
2955
|
+
this.switched.delete(t.source);
|
|
2956
|
+
this.due.delete(t.source);
|
|
2957
|
+
// synced as of its copy, said at once (a review finding: until the next pass re-read the ledger, a table just
|
|
2958
|
+
// copied said "never synced"), and the value the ledger holds, so that read moves nothing
|
|
2959
|
+
this.polledAt.set(t.source, at);
|
|
2960
|
+
}
|
|
2066
2961
|
// the columns this copy holds, from its swap on: what every write to it is shaped by (`asCopied`)
|
|
2067
2962
|
this.copiedColumns.set(t.source, [...t.columns]);
|
|
2068
2963
|
if (o.behind !== null) {
|
|
2069
2964
|
this.behind.set(t.source, { why: o.behind });
|
|
2070
|
-
if (this.
|
|
2965
|
+
if (this.replicated(t))
|
|
2071
2966
|
this.held.set(t.source, noChanges());
|
|
2072
2967
|
}
|
|
2073
2968
|
this.retired.delete(t.source);
|
|
2074
2969
|
this.duckTypes.delete(t.source);
|
|
2970
|
+
// a new copy, a new stay, whatever the columns: what the stream decoded before this moment is not for it
|
|
2971
|
+
this.streamed.delete(t.source);
|
|
2075
2972
|
this.refreshStreamed();
|
|
2076
2973
|
}
|
|
2974
|
+
/** a table's watermark, the time and the sync it is polled or copied under (`recordedSync`): ONE row of `_mason.poll_state`.
|
|
2975
|
+
* `at` is the caller's, an ISO instant (`toISOString`, as `copied_at`), the very value it remembers (`polledAt`) */
|
|
2976
|
+
pollStateSql(t, snapshot, at) {
|
|
2977
|
+
return `INSERT OR REPLACE INTO ${MASON_SCHEMA}.poll_state (source, snapshot, polled_at, sync) VALUES (${literal(t.source)}, ${literal(snapshot)}, ${literal(at)}::TIMESTAMPTZ, ${literal(syncText(this.syncOf(t)))})`;
|
|
2978
|
+
}
|
|
2077
2979
|
/**
|
|
2078
2980
|
* One poll of the tables given (the passes and `pollNow` give one at a time), in ONE DuckDB transaction over ONE
|
|
2079
2981
|
* source snapshot (the module's header), on the lane's connection — the writer's, or the rebuilds' for a catch-up —
|
|
@@ -2111,12 +3013,12 @@ export class Sync {
|
|
|
2111
3013
|
// (`adds`: the keys a stage still to be written adds to DuckDB's count)
|
|
2112
3014
|
const reconcile = async (t, target, from, adds = 0n) => {
|
|
2113
3015
|
const c0 = Date.now();
|
|
2114
|
-
doing = `${t.source}: counting its rows on the source and in
|
|
3016
|
+
doing = `${t.source}: counting its rows on the source and in the synced table`;
|
|
2115
3017
|
const counts = (await w.runAndReadAll(`SELECT (SELECT count FROM ${pq(`SELECT count(*) AS count FROM ${from}`)}), (SELECT count(*) FROM ${target})`)).getRowsJS()[0];
|
|
2116
3018
|
const countMs = Date.now() - c0;
|
|
2117
3019
|
let deleted = 0;
|
|
2118
3020
|
if (counts[0] !== counts[1] + adds) {
|
|
2119
|
-
doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with
|
|
3021
|
+
doing = `${t.source}: deleting the keys the source no longer holds (its keys anti-joined with the synced table's)`;
|
|
2120
3022
|
const on = t.key.map((k) => `t.${ident(k)} = k.${ident(k)}`).join(" AND ");
|
|
2121
3023
|
const gone = await w.run(`DELETE FROM ${target} WHERE rowid IN (SELECT t.rowid FROM ${target} t ANTI JOIN ${pq(`SELECT ${t.key.map(pgIdent).join(", ")} FROM ${from}`)} k ON ${on})`);
|
|
2122
3024
|
deleted = Number(gone.rowsChanged);
|
|
@@ -2124,6 +3026,9 @@ export class Sync {
|
|
|
2124
3026
|
return { rows: Number(counts[0]), deleted, countMs };
|
|
2125
3027
|
};
|
|
2126
3028
|
let snapshot = "";
|
|
3029
|
+
// THE POLL'S TIME, ONE VALUE: taken before its snapshot, so `syncedAt` never says more than the rows hold, written
|
|
3030
|
+
// with its watermark (`pollStateSql`) and remembered as written once that commits (`committed`, `polledAt`)
|
|
3031
|
+
const polledAt = new Date().toISOString();
|
|
2127
3032
|
await w.run("BEGIN TRANSACTION");
|
|
2128
3033
|
try {
|
|
2129
3034
|
await this.interruptible(lane, async () => {
|
|
@@ -2149,7 +3054,7 @@ export class Sync {
|
|
|
2149
3054
|
`${pq(`SELECT ${pgCols} FROM ${from} WHERE ${xmin8Sql(xmax)} >= ${watermark}`)}) AS polled(${t.columns.map((_, i) => `c${i}`).join(", ")})`)).rowsChanged);
|
|
2150
3055
|
if (staged > POLL_PIECE_ROWS) {
|
|
2151
3056
|
const c1 = Date.now();
|
|
2152
|
-
doing = `${t.source}: counting the keys its ${staged} changed row(s) add to
|
|
3057
|
+
doing = `${t.source}: counting the keys its ${staged} changed row(s) add to the synced table (the stage anti-joined with its keys)`;
|
|
2153
3058
|
const on = t.key.map((k) => `s.${ident(`c${t.columns.findIndex((c) => c.name === k)}`)} = t.${ident(k)}`).join(" AND ");
|
|
2154
3059
|
const adds = (await w.runAndReadAll(`SELECT count(*) FROM ${stage} s ANTI JOIN ${target} t ON ${on}`)).getRowsJS()[0][0];
|
|
2155
3060
|
const { rows, deleted, countMs } = await reconcile(t, target, from, adds);
|
|
@@ -2162,7 +3067,7 @@ export class Sync {
|
|
|
2162
3067
|
await writeStaged(w, t, target, stage, "", this.kept);
|
|
2163
3068
|
const c1 = Date.now();
|
|
2164
3069
|
const { rows, deleted, countMs } = await reconcile(t, target, from);
|
|
2165
|
-
await w.run(
|
|
3070
|
+
await w.run(this.pollStateSql(t, snapshot, polledAt));
|
|
2166
3071
|
await w.run(`DROP TABLE ${stage}`);
|
|
2167
3072
|
timed.push(`${t.source} ${staged} changed in ${c1 - c0} ms, counted in ${countMs} ms${deleted ? `, ${deleted} deleted in ${Date.now() - c1 - countMs} ms` : ""}`);
|
|
2168
3073
|
found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - c0 });
|
|
@@ -2177,30 +3082,33 @@ export class Sync {
|
|
|
2177
3082
|
await w.run("ROLLBACK");
|
|
2178
3083
|
}
|
|
2179
3084
|
catch (r) {
|
|
2180
|
-
this.log(`the poll's ROLLBACK failed too: ${
|
|
3085
|
+
this.log(`the poll's ROLLBACK failed too: ${this.errorText(r)}`);
|
|
2181
3086
|
}
|
|
2182
3087
|
// a stage made in the rolled-back transaction went with it
|
|
2183
3088
|
pieced.length = 0;
|
|
2184
3089
|
// a numeric wider than its copy: that table is copied again (wider) at the next pass, the others polled
|
|
2185
3090
|
// matched against the tables' own names (a name may hold a space)
|
|
2186
|
-
const wider = e instanceof Error ? tables.find((t) => e
|
|
3091
|
+
const wider = e instanceof Error ? tables.find((t) => widerIn(e, t.source))?.source : undefined;
|
|
2187
3092
|
if (wider !== undefined && this.snapshots.delete(wider)) {
|
|
2188
|
-
this.log(`poll: ${wider} has a numeric value wider than its
|
|
3093
|
+
this.log(`poll: ${wider} has a numeric value wider than its synced DECIMAL column: it is read again whole at the next pass`);
|
|
2189
3094
|
return [];
|
|
2190
3095
|
}
|
|
2191
3096
|
throw named(e);
|
|
2192
3097
|
}
|
|
2193
3098
|
// committed: the watermarks and the row counts move together (a pieced table's with its last piece)
|
|
2194
|
-
|
|
3099
|
+
// and the table's last sync (`Synced`): the poll's time as the ledger now holds it, and how long THIS table's poll took
|
|
3100
|
+
const committed = async (source, rows, ms) => {
|
|
3101
|
+
this.polledAt.set(source, polledAt);
|
|
2195
3102
|
const st = this.states.get(source);
|
|
2196
3103
|
if (st.rows !== rows || st.error !== null)
|
|
2197
3104
|
await this.record({ ...st, rows, error: null }, undefined, w);
|
|
2198
3105
|
this.snapshots.set(source, snapshot);
|
|
3106
|
+
this.synced.set(source, { at: polledAt, seconds: ms / 1000, by: lane === this.writerLane ? "poll" : "catch-up" });
|
|
2199
3107
|
};
|
|
2200
3108
|
try {
|
|
2201
3109
|
for (const [source, rows] of pending)
|
|
2202
3110
|
if (!pieced.some((p) => p.t.source === source))
|
|
2203
|
-
await committed(source, rows);
|
|
3111
|
+
await committed(source, rows, found.find((f) => f.table === source)?.ms ?? 0);
|
|
2204
3112
|
this.log(`poll: ${timed.join("; ")}`);
|
|
2205
3113
|
for (const { t, target, stage, staged, deleted, started } of pieced) {
|
|
2206
3114
|
const p0 = Date.now();
|
|
@@ -2211,7 +3119,7 @@ export class Sync {
|
|
|
2211
3119
|
doing = `${t.source}: writing piece ${i + 1} of ${n} of its ${staged} changed row(s) by key`;
|
|
2212
3120
|
await this.interruptible(lane, () => writeStaged(w, t, target, stage, ` WHERE rowid >= ${i * POLL_PIECE_ROWS} AND rowid < ${(i + 1) * POLL_PIECE_ROWS}`, this.kept));
|
|
2213
3121
|
if (i === n - 1) {
|
|
2214
|
-
await w.run(
|
|
3122
|
+
await w.run(this.pollStateSql(t, snapshot, polledAt));
|
|
2215
3123
|
}
|
|
2216
3124
|
doing = `${t.source}: committing piece ${i + 1} of ${n}`;
|
|
2217
3125
|
await w.run("COMMIT");
|
|
@@ -2222,7 +3130,7 @@ export class Sync {
|
|
|
2222
3130
|
}
|
|
2223
3131
|
}
|
|
2224
3132
|
const rows = pending.get(t.source);
|
|
2225
|
-
await committed(t.source, rows);
|
|
3133
|
+
await committed(t.source, rows, Date.now() - started);
|
|
2226
3134
|
found.push({ table: t.source, changed: staged, deleted, rows, ms: Date.now() - started });
|
|
2227
3135
|
this.log(`poll: ${t.source}: ${staged.toLocaleString("en-US")} changed row(s), written in ${n} pieces in ${Date.now() - p0} ms`);
|
|
2228
3136
|
}
|
|
@@ -2231,6 +3139,10 @@ export class Sync {
|
|
|
2231
3139
|
for (const { stage } of pieced)
|
|
2232
3140
|
await w.run(`DROP TABLE IF EXISTS ${stage}`);
|
|
2233
3141
|
}
|
|
3142
|
+
// committed: a generated table over a table that changed is behind from here
|
|
3143
|
+
for (const f of found)
|
|
3144
|
+
if (f.changed > 0 || f.deleted > 0)
|
|
3145
|
+
this.generated.changed(f.table);
|
|
2234
3146
|
return found;
|
|
2235
3147
|
}
|
|
2236
3148
|
/**
|
|
@@ -2295,10 +3207,11 @@ export class Sync {
|
|
|
2295
3207
|
if (resumed === null) {
|
|
2296
3208
|
// the stream (or the other tables' polls) goes on: the slot holds no WAL for this copy, and the table joins once
|
|
2297
3209
|
// caught up from the copy's snapshot
|
|
2298
|
-
const beside = this.
|
|
2299
|
-
: this.mode
|
|
2300
|
-
const
|
|
2301
|
-
|
|
3210
|
+
const beside = this.replicated(t) ? "beside the stream, which goes on keeping the other tables current: it is streamed once caught up by one poll from the snapshot it was read at"
|
|
3211
|
+
: this.mode !== "recopy" ? "beside the other tables' polls: once read, it is caught up by one poll from the snapshot it was read at" : "beside the other tables";
|
|
3212
|
+
const r = this.retired.get(t.source);
|
|
3213
|
+
const again = r !== undefined ? `, again (it is ${r.layout})` : "";
|
|
3214
|
+
this.log(`syncing ${t.source}${again}: ${t.rows === null ? "rows unknown" : `about ${t.rows} row(s)`}, ${t.bytes} source bytes, ${beside}`);
|
|
2302
3215
|
}
|
|
2303
3216
|
let read;
|
|
2304
3217
|
try {
|
|
@@ -2308,7 +3221,7 @@ export class Sync {
|
|
|
2308
3221
|
if (!(e instanceof StartOver))
|
|
2309
3222
|
throw e;
|
|
2310
3223
|
// once: a second reason to start over is the copy's failure
|
|
2311
|
-
this.log(`starting the
|
|
3224
|
+
this.log(`starting the sync of ${t.source} over, from its first page: ${this.errorText(e)}`);
|
|
2312
3225
|
kept = await this.forgetCopy(t, side);
|
|
2313
3226
|
read = await this.readChunks(t, side, null, (p) => { kept = p; });
|
|
2314
3227
|
}
|
|
@@ -2320,7 +3233,7 @@ export class Sync {
|
|
|
2320
3233
|
const [rows, keys] = (await w.runAndReadAll(`SELECT count(*), count(DISTINCT (${t.key.map(ident).join(", ")})) FROM ${side}`)).getRowsJS()[0];
|
|
2321
3234
|
if (rows !== keys) {
|
|
2322
3235
|
kept = await this.forgetCopy(t, side);
|
|
2323
|
-
throw new Error(`its
|
|
3236
|
+
throw new Error(`its synced data holds ${rows} rows but ${keys} distinct keys (Mason bug: report it, with mason logs): it was not swapped in, and is read again from its first page`);
|
|
2324
3237
|
}
|
|
2325
3238
|
}
|
|
2326
3239
|
// its stored JSON paths filled, so the table swapped in has every one whole (src/duck/kept.ts fillSide)
|
|
@@ -2328,14 +3241,17 @@ export class Sync {
|
|
|
2328
3241
|
// THE SWAP, on the writer's turn: one transaction, between two of the stream's or the polls'. It carries the copy's
|
|
2329
3242
|
// watermark (its first snapshot), removes its progress, and leaves the table behind until its catch-up (none in
|
|
2330
3243
|
// recopy mode: no poll)
|
|
2331
|
-
await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its
|
|
3244
|
+
await this.onWriter(() => this.swap(t, side, { snapshot: p.snapshot, behind: p.snapshot === null ? null : "after its whole read" }));
|
|
2332
3245
|
this.copyLive.delete(t.source);
|
|
2333
3246
|
kept = null;
|
|
2334
3247
|
this.retries.delete(t.source);
|
|
2335
3248
|
const seconds = (Date.now() - started) / 1000;
|
|
2336
|
-
this.log(`
|
|
3249
|
+
this.log(`synced ${t.source}: ${p.rows} row(s), ${t.bytes} source bytes, in ${seconds.toFixed(2)} s (read in ${read.seconds.toFixed(2)} s` +
|
|
2337
3250
|
`${resumed === null ? "" : `, resumed at page ${readThrough(resumed.done) ?? p.pages} of ${resumed.pages}${pastPages(resumed.done)} with ${resumed.rows} row(s) in`}), stored as the source's scan delivered them, with no index`);
|
|
2338
|
-
|
|
3251
|
+
const copiedAt = new Date().toISOString();
|
|
3252
|
+
await this.record({ source: t.source, phase: "copied", rows: p.rows, bytes: t.bytes, seconds, copiedAt, error: null }, t.columns, w);
|
|
3253
|
+
this.synced.set(t.source, { at: copiedAt, seconds, by: "copy" });
|
|
3254
|
+
this.event({ kind: "copied", table: t.source, message: `synced: ${p.rows.toLocaleString("en-US")} row(s) in ${seconds.toFixed(2)} s` });
|
|
2339
3255
|
this.copiedAs.set(t.source, columnsKey(t.columns));
|
|
2340
3256
|
// in the catalog at once: a pass's other copies may take hours
|
|
2341
3257
|
await this.rebuildSchema();
|
|
@@ -2351,7 +3267,7 @@ export class Sync {
|
|
|
2351
3267
|
k = await this.forgetCopy(t, side);
|
|
2352
3268
|
}
|
|
2353
3269
|
catch (f) {
|
|
2354
|
-
this.log(`the
|
|
3270
|
+
this.log(`the sync of ${t.source} could not drop what it began (its next attempt continues it): ${this.errorText(f)}`);
|
|
2355
3271
|
}
|
|
2356
3272
|
}
|
|
2357
3273
|
const through = k === null ? null : readThrough(k.done);
|
|
@@ -2359,12 +3275,12 @@ export class Sync {
|
|
|
2359
3275
|
if (e instanceof Stopping) {
|
|
2360
3276
|
// the chunks in flight rolled back: a table never copied continues at the next start, one copied before serves its
|
|
2361
3277
|
// copy as it was meanwhile
|
|
2362
|
-
this.log(`
|
|
3278
|
+
this.log(`sync of ${t.source} stopped: Mason is stopping (${where === null ? "rolled back; it is synced at the next start" : `the chunks in flight rolled back; it continues ${where} at the next start`})`);
|
|
2363
3279
|
if (first)
|
|
2364
3280
|
await this.record({ ...was, phase: "waiting", error: null }, undefined, w);
|
|
2365
3281
|
return;
|
|
2366
3282
|
}
|
|
2367
|
-
const m =
|
|
3283
|
+
const m = this.errorText(e);
|
|
2368
3284
|
// tried again after a wait that grows with each failure in a row: a copy failing for good must not read the
|
|
2369
3285
|
// source at every pass (a 20-minute read of a production replica every 95 minutes) — and then
|
|
2370
3286
|
// continued where it was, never read again from its first page
|
|
@@ -2373,11 +3289,24 @@ export class Sync {
|
|
|
2373
3289
|
this.retries.set(t.source, { failures, at });
|
|
2374
3290
|
const when = `tried again after ${new Date(at).toISOString().slice(11, 16)} UTC (failure ${failures} in a row), or at once when Mason starts again or its columns or key change` +
|
|
2375
3291
|
(where === null ? "" : `; it continues ${where}`);
|
|
2376
|
-
this.log(`
|
|
3292
|
+
this.log(`sync of ${t.source} failed: ${m}; ${when}`);
|
|
3293
|
+
this.event({ kind: "copy_failed", table: t.source, message: `${m}; ${when}` });
|
|
2377
3294
|
// a table that was copied keeps serving its last copy; one never copied is failed
|
|
2378
3295
|
await this.record({ ...this.states.get(t.source), phase: first ? "failed" : "copied", error: `${m} (${when})` }, undefined, w);
|
|
2379
3296
|
}
|
|
2380
3297
|
}
|
|
3298
|
+
/** Where the copy's file is, the bytes free there, and what the copies running are projected to add (`DiskStatus`). */
|
|
3299
|
+
diskStatus() {
|
|
3300
|
+
const dir = path.dirname(this.store.file);
|
|
3301
|
+
const needed = [...this.copyLive.values()].map((l) => l.diskNeeded).filter((n) => n !== null);
|
|
3302
|
+
const projectedBytes = needed.length ? Math.round(needed.reduce((a, b) => a + b, 0)) : null;
|
|
3303
|
+
try {
|
|
3304
|
+
return { path: dir, freeBytes: this.diskFree(dir), projectedBytes };
|
|
3305
|
+
}
|
|
3306
|
+
catch (e) {
|
|
3307
|
+
return { path: dir, freeBytes: null, projectedBytes, error: `the free space where the synced data is could not be read: ${this.errorText(e)}` };
|
|
3308
|
+
}
|
|
3309
|
+
}
|
|
2381
3310
|
/** The DuckDB file's bytes with its write-ahead log's (what a copy adds lands in the log until a checkpoint). */
|
|
2382
3311
|
fileBytes() {
|
|
2383
3312
|
let n = 0;
|
|
@@ -2392,16 +3321,18 @@ export class Sync {
|
|
|
2392
3321
|
}
|
|
2393
3322
|
return n;
|
|
2394
3323
|
}
|
|
2395
|
-
/** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null.
|
|
2396
|
-
|
|
3324
|
+
/** Why the copy of `t` must stop now — what its pages left would add is more than the disk has free — or null. What
|
|
3325
|
+
* it projects is kept on the copy's `live` (`diskStatus` says it). */
|
|
3326
|
+
diskShort(t, p, disk, live) {
|
|
2397
3327
|
const bytesNow = this.fileBytes();
|
|
2398
3328
|
const projected = copyDiskProjection({ pages: p.pages, done: p.done, doneAtStart: disk.doneAtStart, bytesAtStart: disk.bytesAtStart, bytesNow });
|
|
3329
|
+
live.diskNeeded = projected?.needed ?? null;
|
|
2399
3330
|
if (projected === null)
|
|
2400
3331
|
return null;
|
|
2401
3332
|
const free = this.diskFree(path.dirname(this.store.file));
|
|
2402
3333
|
if (projected.needed <= free)
|
|
2403
3334
|
return null;
|
|
2404
|
-
return `the
|
|
3335
|
+
return `the sync of ${t.source} would need about ${humanBytes(projected.needed)} more on disk than is free (${humanBytes(free)} free where ${this.store.file} is; ` +
|
|
2405
3336
|
`${pagesRead(p.done, p.pages)} of ${p.pages} pages read, ${humanBytes(bytesNow - disk.bytesAtStart)} added): free some disk, mason remove ${t.source}, or mason add ${t.source} --columns … with fewer columns`;
|
|
2406
3337
|
}
|
|
2407
3338
|
/**
|
|
@@ -2433,6 +3364,11 @@ export class Sync {
|
|
|
2433
3364
|
const named = t.columns.map((c) => ident(c.name)).join(", ");
|
|
2434
3365
|
const started = Date.now();
|
|
2435
3366
|
const lanes = [this.rebuilder];
|
|
3367
|
+
// ITS VALUES OUT OF LINE, from the source's catalog (`chunkCeiling`): read once an attempt, before any transaction
|
|
3368
|
+
// of the copy (a begun copy's first read fixes its snapshot); sizing only, so a resumed copy reads it again
|
|
3369
|
+
const outOfLine = await this.interruptible(this.rebuilder, () => this.outOfLineBytes(pgFrom));
|
|
3370
|
+
const limit = this.store.memoryLimitBytes;
|
|
3371
|
+
const ceilingOf = (pages) => chunkCeiling(pages, t.rows, outOfLine, chunkBudgetBytes(limit, this.copyParallel), this.copyChunkRows);
|
|
2436
3372
|
if (recopy)
|
|
2437
3373
|
await w.run("BEGIN TRANSACTION");
|
|
2438
3374
|
try {
|
|
@@ -2441,7 +3377,7 @@ export class Sync {
|
|
|
2441
3377
|
if (!recopy)
|
|
2442
3378
|
await w.run("BEGIN TRANSACTION");
|
|
2443
3379
|
try {
|
|
2444
|
-
begun = await this.interruptible(this.rebuilder, () => this.beginCopy(t, side, pgFrom));
|
|
3380
|
+
begun = await this.interruptible(this.rebuilder, () => this.beginCopy(t, side, pgFrom, ceilingOf, outOfLine));
|
|
2445
3381
|
if (!recopy) {
|
|
2446
3382
|
await this.writeBegun(t, begun);
|
|
2447
3383
|
await w.run("COMMIT");
|
|
@@ -2462,7 +3398,7 @@ export class Sync {
|
|
|
2462
3398
|
// first chunks are a tenth of it, to measure, and every later one is what a reader reads in copyChunkSeconds at
|
|
2463
3399
|
// the rate measured since — cut from what no committed chunk covers (`copyGaps`, `takeChunk`), so a copy resumed
|
|
2464
3400
|
// reads exactly the ranges it did not commit, whatever size its chunks were
|
|
2465
|
-
const ceiling =
|
|
3401
|
+
const ceiling = ceilingOf(p.pages);
|
|
2466
3402
|
const gaps = copyGaps(p.done);
|
|
2467
3403
|
let rate = null;
|
|
2468
3404
|
// the source's page size, for the bytes a second its readers read (pg_relation_size is in bytes; pages are its unit)
|
|
@@ -2480,7 +3416,7 @@ export class Sync {
|
|
|
2480
3416
|
const pagesDone = pagesRead(p.done, p.pages);
|
|
2481
3417
|
const live = { pages: p.pages, chunksDone: p.done.length, pagesDone, rowsDone: p.rows, untaken: gapPages(gaps, p.pages),
|
|
2482
3418
|
chunkPages: nextChunkPages(ceiling, null, this.copyChunkSeconds), readers, reading: [], rate: null,
|
|
2483
|
-
rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null };
|
|
3419
|
+
rowsPerPage: pagesDone > 0 ? p.rows / pagesDone : t.rows !== null && t.rows > 0 && p.pages > 0 ? t.rows / p.pages : null, diskNeeded: null };
|
|
2484
3420
|
this.copyLive.set(t.source, live);
|
|
2485
3421
|
for (let i = 1; i < readers; i++) {
|
|
2486
3422
|
const lane = { conn: await this.store.instance.connect(), inFlight: false };
|
|
@@ -2495,6 +3431,10 @@ export class Sync {
|
|
|
2495
3431
|
if (l.inFlight)
|
|
2496
3432
|
l.conn.interrupt();
|
|
2497
3433
|
};
|
|
3434
|
+
// OUT OF MEMORY IN A CHUNK (53200, `pgErrorOf`; never 53400, the spill's ceiling, which is disk): its range is given
|
|
3435
|
+
// back (`giveBack`) and the copy reads on with ONE reader (`sole`), each chunk at most half the one that ran out
|
|
3436
|
+
// (`memoryPages`, which only shrinks in an attempt); a one-page chunk read alone that runs out fails the copy
|
|
3437
|
+
let sole = null, memoryPages = Infinity;
|
|
2498
3438
|
// the chunks since the last minute's line, and how long they took
|
|
2499
3439
|
let lineAt = Date.now(), chunks = 0, chunkMs = 0;
|
|
2500
3440
|
const reader = async (lane, k) => {
|
|
@@ -2502,8 +3442,10 @@ export class Sync {
|
|
|
2502
3442
|
for (;;) {
|
|
2503
3443
|
if (failure !== null)
|
|
2504
3444
|
return;
|
|
3445
|
+
if (sole !== null && sole !== k)
|
|
3446
|
+
return;
|
|
2505
3447
|
const sizedBy = rate;
|
|
2506
|
-
const range = takeChunk(gaps, p.pages, nextChunkPages(ceiling, sizedBy, this.copyChunkSeconds));
|
|
3448
|
+
const range = takeChunk(gaps, p.pages, Math.min(memoryPages, nextChunkPages(ceiling, sizedBy, this.copyChunkSeconds)));
|
|
2507
3449
|
if (range === undefined)
|
|
2508
3450
|
return;
|
|
2509
3451
|
const { first, end } = range;
|
|
@@ -2511,6 +3453,8 @@ export class Sync {
|
|
|
2511
3453
|
const chunkSize = Math.max(1, (end ?? p.pages) - first);
|
|
2512
3454
|
const slot = { since: Date.now(), pages: chunkSize };
|
|
2513
3455
|
live.reading.push(slot);
|
|
3456
|
+
// read with no other chunk in flight, by the one reader the copy reads with
|
|
3457
|
+
const alone = live.reading.length === 1 && (sole === k || readers === 1);
|
|
2514
3458
|
live.untaken = gapPages(gaps, p.pages);
|
|
2515
3459
|
live.chunkPages = nextChunkPages(ceiling, rate, this.copyChunkSeconds);
|
|
2516
3460
|
const told = { source: t.source, first, end, reader: k, pages: chunkSize, ceiling, sizedBy };
|
|
@@ -2535,7 +3479,7 @@ export class Sync {
|
|
|
2535
3479
|
const node = Number((await c.runAndReadAll(`SELECT * FROM ${pq(`SELECT pg_relation_filenode(${pgLiteral(pgFrom)}::regclass)::bigint`)}`)).getRowsJS()[0][0]);
|
|
2536
3480
|
filenode ??= node;
|
|
2537
3481
|
if (node !== filenode) {
|
|
2538
|
-
throw new StartOver(`the source rewrote ${t.source} since its
|
|
3482
|
+
throw new StartOver(`the source rewrote ${t.source} since its sync began (its file changed: a VACUUM FULL, CLUSTER, TRUNCATE or column type change), so its rows moved between pages`);
|
|
2539
3483
|
}
|
|
2540
3484
|
const n = Number(r.rowsChanged);
|
|
2541
3485
|
await this.onChunk?.({ ...told, rows: n, phase: "read" });
|
|
@@ -2572,27 +3516,61 @@ export class Sync {
|
|
|
2572
3516
|
const at = live.reading.indexOf(slot);
|
|
2573
3517
|
if (at >= 0)
|
|
2574
3518
|
live.reading.splice(at, 1);
|
|
2575
|
-
|
|
3519
|
+
// rolled back, so its connection reads on: a ROLLBACK out of memory itself (measured, with other readers'
|
|
3520
|
+
// chunks in flight) is tried again while the others' memory is given back, for about two seconds
|
|
3521
|
+
let rolledBack = !inTx;
|
|
3522
|
+
for (let attempt = 0; inTx && !rolledBack; attempt++) {
|
|
2576
3523
|
try {
|
|
2577
3524
|
await c.run("ROLLBACK");
|
|
3525
|
+
rolledBack = true;
|
|
2578
3526
|
}
|
|
2579
3527
|
catch (r) {
|
|
2580
|
-
|
|
3528
|
+
const said = this.errorText(r);
|
|
3529
|
+
// the engine ended the transaction itself (measured: "cannot rollback - no transaction is active" after a
|
|
3530
|
+
// COMMIT, or a read, out of memory): nothing of the chunk is left to roll back
|
|
3531
|
+
if (endedTransaction(r)) {
|
|
3532
|
+
rolledBack = true;
|
|
3533
|
+
break;
|
|
3534
|
+
}
|
|
3535
|
+
if (attempt < 20 && pgErrorOf(r, "").fields.code === "53200") {
|
|
3536
|
+
await new Promise((ok) => setTimeout(ok, 100));
|
|
3537
|
+
continue;
|
|
3538
|
+
}
|
|
3539
|
+
this.log(`a chunk of ${t.source}'s sync could not be rolled back after it failed (its connection is closed with the sync): ${said}`);
|
|
3540
|
+
break;
|
|
3541
|
+
}
|
|
3542
|
+
}
|
|
3543
|
+
if (failure === null && !recopy && rolledBack && pgErrorOf(e, "").fields.code === "53200") {
|
|
3544
|
+
giveBack(gaps, range);
|
|
3545
|
+
live.untaken = gapPages(gaps, p.pages);
|
|
3546
|
+
if (chunkSize <= 1 && alone) {
|
|
3547
|
+
const rpp = live.rowsPerPage;
|
|
3548
|
+
fail(new Error(tooLargeText(t.source, `one page of its table${rpp === null ? "" : ` (about ${Math.max(1, Math.round(rpp)).toLocaleString("en-US")} row(s))`}, read alone, ran out of memory`, limit)));
|
|
3549
|
+
return;
|
|
3550
|
+
}
|
|
3551
|
+
const was = { sole, memoryPages };
|
|
3552
|
+
sole ??= k;
|
|
3553
|
+
memoryPages = Math.max(1, Math.min(memoryPages, Math.floor(chunkSize / 2)));
|
|
3554
|
+
live.readers = 1;
|
|
3555
|
+
live.chunkPages = Math.min(memoryPages, live.chunkPages);
|
|
3556
|
+
if (was.sole === null || was.memoryPages !== memoryPages) {
|
|
3557
|
+
this.log(`a chunk of ${chunkSize} page(s) of ${t.source} ran out of memory: reading on one chunk at a time, ${memoryPages} page(s) each`);
|
|
2581
3558
|
}
|
|
3559
|
+
continue;
|
|
2582
3560
|
}
|
|
2583
3561
|
fail(e);
|
|
2584
3562
|
return;
|
|
2585
3563
|
}
|
|
2586
3564
|
// STOPPED BEFORE THE DISK FILLS: the chunks committed are kept, the copy fails with the way out and is tried
|
|
2587
3565
|
// again as a failed copy is (freed disk is used at the retry, or at once at the next start)
|
|
2588
|
-
const short = recopy ? null : this.diskShort(t, p, disk);
|
|
3566
|
+
const short = recopy ? null : this.diskShort(t, p, disk, live);
|
|
2589
3567
|
if (short !== null) {
|
|
2590
3568
|
fail(new Error(short));
|
|
2591
3569
|
return;
|
|
2592
3570
|
}
|
|
2593
3571
|
const at = readThrough(p.done);
|
|
2594
3572
|
if (Date.now() - lineAt >= 60_000 && at !== null) {
|
|
2595
|
-
this.log(`
|
|
3573
|
+
this.log(`syncing ${t.source}: page ${at} of ${p.pages}${pastPages(p.done)}, ${p.rows.toLocaleString("en-US")} row(s) in; ${chunks} chunk(s) in the last minute on ${readers} reader(s), ` +
|
|
2596
3574
|
`${(chunkMs / chunks / 1000).toFixed(1)} s each` + (rate === null ? "" : `; a reader reads ~${Math.round(rate.rowsPerSecond).toLocaleString("en-US")} rows/s ` +
|
|
2597
3575
|
`(${humanBytes(rate.bytesPerSecond)}/s of the table's pages), so the next chunk is ${live.chunkPages} page(s), ~${this.copyChunkSeconds} s (MASON_COPY_CHUNK_SECONDS; at most ${ceiling})`));
|
|
2598
3576
|
lineAt = Date.now();
|
|
@@ -2608,12 +3586,17 @@ export class Sync {
|
|
|
2608
3586
|
l.conn.interrupt(); }, 50);
|
|
2609
3587
|
try {
|
|
2610
3588
|
await Promise.all(lanes.map((lane, k) => reader(lane, k)));
|
|
3589
|
+
// a range given back once the other readers had finished: the one reader left reads it
|
|
3590
|
+
while (failure === null && sole !== null && gaps.length > 0)
|
|
3591
|
+
await reader(lanes[sole], sole);
|
|
2611
3592
|
}
|
|
2612
3593
|
finally {
|
|
2613
3594
|
clearInterval(again);
|
|
2614
3595
|
}
|
|
2615
3596
|
if (failure !== null)
|
|
2616
3597
|
throw failure.error;
|
|
3598
|
+
if (gaps.length > 0)
|
|
3599
|
+
throw new Error(`the sync of ${t.source} ended with pages ${JSON.stringify(gaps)} unread (Mason bug: report it, with mason logs)`);
|
|
2617
3600
|
if (recopy)
|
|
2618
3601
|
await w.run("COMMIT");
|
|
2619
3602
|
return { rows: p.rows, seconds: (Date.now() - started) / 1000, progress: p };
|
|
@@ -2636,25 +3619,40 @@ export class Sync {
|
|
|
2636
3619
|
* extension's REPEATABLE READ snapshot; its xmin is the copy's watermark, as the change reader records one before its
|
|
2637
3620
|
* copy), the exact numerics' widths, the side table (NO KEY: no PRIMARY KEY or UNIQUE, so no index), and the heap's
|
|
2638
3621
|
* pages now (`pg_relation_size`, exact where `relpages` is an estimate) */
|
|
2639
|
-
async beginCopy(t, side, pgFrom) {
|
|
3622
|
+
async beginCopy(t, side, pgFrom, ceilingOf, outOfLine) {
|
|
2640
3623
|
const w = this.rebuilder.conn;
|
|
2641
3624
|
const snapshot = this.mode !== "recopy" ? (await w.runAndReadAll(`SELECT * FROM ${pq(SNAPSHOT_SQL)}`)).getRowsJS()[0][0] : null;
|
|
2642
3625
|
const described = await w.runAndReadAll(`DESCRIBE SELECT ${t.columns.map((c) => ident(c.name)).join(", ")} FROM ${SRC}.${ident(t.schema)}.${ident(t.table)}`);
|
|
2643
3626
|
const scanned = new Map(described.getRowsJS().map((r) => [String(r[0]), String(r[1])]));
|
|
3627
|
+
// a domain's column as its base type reads: the extension's scan of the table does not know a domain
|
|
3628
|
+
const domains = t.columns.filter((c) => c.domain === true);
|
|
3629
|
+
if (domains.length > 0) {
|
|
3630
|
+
const read = await w.runAndReadAll(`DESCRIBE SELECT * FROM ${pq(`SELECT ${domains.map((c) => pgColumnSql(c, new Map(), new Set())).join(", ")} FROM ${pgFrom} LIMIT 0`)}`);
|
|
3631
|
+
for (const r of read.getRowsJS())
|
|
3632
|
+
scanned.set(String(r[0]), String(r[1]));
|
|
3633
|
+
}
|
|
2644
3634
|
// the numerics the extension would read as doubles: their widths, in the copy's snapshot (NUMERIC_TYPE)
|
|
2645
3635
|
const exact = await this.exactNumerics(t, scanned, pgFrom);
|
|
2646
3636
|
const defs = t.columns.map((c) => `${ident(c.name)} ${exact.has(c.name) ? `DECIMAL(38,${exact.get(c.name)})` : storedType(c, scanned.get(c.name) ?? "VARCHAR")}`);
|
|
2647
3637
|
await w.run(`CREATE SCHEMA IF NOT EXISTS ${ident(t.schema)}`);
|
|
2648
3638
|
await w.run(`CREATE OR REPLACE TABLE ${side} (${defs.join(", ")})`);
|
|
2649
3639
|
const pages = Number((await w.runAndReadAll(`SELECT * FROM ${pq(`SELECT (pg_relation_size(${pgLiteral(pgFrom)}::regclass) / current_setting('block_size')::int)::bigint`)}`)).getRowsJS()[0][0]);
|
|
2650
|
-
const ceiling =
|
|
3640
|
+
const ceiling = ceilingOf(pages);
|
|
3641
|
+
const byBytes = ceiling < chunkPages(pages, t.rows, this.copyChunkRows) && outOfLine !== null
|
|
3642
|
+
? ` for its ${humanBytes(outOfLine)} of values out of line in ${humanBytes(this.store.memoryLimitBytes)} of memory` : "";
|
|
2651
3643
|
const firstSize = nextChunkPages(ceiling, null, this.copyChunkSeconds);
|
|
2652
3644
|
const readers = this.mode === "recopy" ? 1 : Math.min(this.copyParallel, Math.max(1, Math.ceil(pages / firstSize)));
|
|
2653
|
-
this.log(`
|
|
2654
|
-
`${pages} page(s) in chunks sized to ~${this.copyChunkSeconds} s of reading each at the rate measured here (MASON_COPY_CHUNK_SECONDS; at most ${ceiling} page(s)), ` +
|
|
3645
|
+
this.log(`syncing ${t.source}${snapshot === null ? "" : ` as of the source's snapshot ${snapshot}: what commits after it is caught up by one poll once read`}; ` +
|
|
3646
|
+
`${pages} page(s) in chunks sized to ~${this.copyChunkSeconds} s of reading each at the rate measured here (MASON_COPY_CHUNK_SECONDS; at most ${ceiling} page(s)${byBytes}), ` +
|
|
2655
3647
|
`the first of ${firstSize} page(s) to measure it, ${readers === 1 ? "one at a time" : `${readers} at a time (MASON_COPY_PARALLEL), each a backend on the database`}`);
|
|
2656
3648
|
return { snapshot, columns: columnsKey(t.columns), exact, filenode: null, pages, done: [], rows: 0 };
|
|
2657
3649
|
}
|
|
3650
|
+
/** THE SOURCE TABLE'S VALUES OUT OF LINE, in bytes: its TOAST table's (`pg_table_size`, no index), 0 with none; the
|
|
3651
|
+
* one reading of it (`chunkCeiling`) */
|
|
3652
|
+
async outOfLineBytes(pgFrom) {
|
|
3653
|
+
const r = await this.rebuilder.conn.runAndReadAll(`SELECT * FROM ${pq(`SELECT COALESCE((SELECT pg_table_size(reltoastrelid) FROM pg_class WHERE oid = ${pgLiteral(pgFrom)}::regclass AND reltoastrelid <> 0), 0)::bigint`)}`);
|
|
3654
|
+
return Number(r.getRowsJS()[0][0]);
|
|
3655
|
+
}
|
|
2658
3656
|
/** what a copy was begun with, in its beginning's transaction (`CopyProgress`) */
|
|
2659
3657
|
async writeBegun(t, p) {
|
|
2660
3658
|
await this.rebuilder.conn.run(`INSERT OR REPLACE INTO ${MASON_SCHEMA}.copy_progress VALUES (${literal(t.source)}, ${p.snapshot === null ? "NULL" : literal(p.snapshot)}, ` +
|
|
@@ -2676,7 +3674,7 @@ export class Sync {
|
|
|
2676
3674
|
const row = r[0];
|
|
2677
3675
|
if (row === undefined || row[0] === null || row[1] !== columnsKey(t.columns) || !there) {
|
|
2678
3676
|
if (row !== undefined)
|
|
2679
|
-
this.log(`starting the
|
|
3677
|
+
this.log(`starting the sync of ${t.source} over, from its first page: ${!there ? "the table it was filling is gone" : row[0] === null ? "it was begun as a recopy" : "its columns are not the ones its sync was begun with"}`);
|
|
2680
3678
|
return this.forgetCopy(t, side);
|
|
2681
3679
|
}
|
|
2682
3680
|
const chunks = (await w.runAndReadAll(`SELECT first_page, end_page, "rows", filenode FROM ${MASON_SCHEMA}.copy_chunks WHERE source = ${literal(t.source)} ORDER BY first_page`)).getRowsJS();
|
|
@@ -2684,8 +3682,8 @@ export class Sync {
|
|
|
2684
3682
|
const rows = chunks.reduce((n, c) => n + Number(c[2]), 0);
|
|
2685
3683
|
const p = { snapshot: row[0], columns: row[1], exact: new Map(JSON.parse(row[2])), filenode: chunks.length ? Number(chunks[0][3]) : null, pages: Number(row[3]), done, rows };
|
|
2686
3684
|
const at = readThrough(done);
|
|
2687
|
-
this.log(at === null ? `resuming the
|
|
2688
|
-
: `resuming the
|
|
3685
|
+
this.log(at === null ? `resuming the sync of ${t.source} with every page read (${rows} row(s) in): its swap is next`
|
|
3686
|
+
: `resuming the sync of ${t.source} at page ${at} of ${p.pages}${pastPages(done)}, ${rows} row(s) in, held to its first snapshot ${p.snapshot}: the pages read before are not read again`);
|
|
2689
3687
|
const now = this.states.get(t.source);
|
|
2690
3688
|
if (now !== undefined && now.phase === "copying")
|
|
2691
3689
|
this.states.set(t.source, { ...now, rowsSoFar: rows, rowsEstimate: t.rows });
|
|
@@ -2710,7 +3708,7 @@ export class Sync {
|
|
|
2710
3708
|
await w.run(`ALTER TABLE ${MASON_SCHEMA}.copy_progress DROP COLUMN IF EXISTS ${ident(gone)}`);
|
|
2711
3709
|
await w.run("COMMIT");
|
|
2712
3710
|
if (moved > 0)
|
|
2713
|
-
this.log(`${moved}
|
|
3711
|
+
this.log(`${moved} sync(s) in progress were begun by a Mason that read one chunk at a time: each continues where it was`);
|
|
2714
3712
|
}
|
|
2715
3713
|
catch (e) {
|
|
2716
3714
|
await w.run("ROLLBACK");
|
|
@@ -2737,8 +3735,51 @@ export class Sync {
|
|
|
2737
3735
|
for (const [source] of r.getRowsJS()) {
|
|
2738
3736
|
if (this.states.get(source)?.phase !== "copied")
|
|
2739
3737
|
continue;
|
|
2740
|
-
this.retired.
|
|
2741
|
-
this.log(`${source}:
|
|
3738
|
+
this.retired.set(source, { layout: RETIRED_LAYOUT, short: "in an older Mason's layout, key order with an index" });
|
|
3739
|
+
this.log(`${source}: it is ${RETIRED_LAYOUT} — its rows are sorted on disk, and only the source has them in its own order, so it is read again whole; its synced data serves as it was until then`);
|
|
3740
|
+
}
|
|
3741
|
+
await this.findHeldLayout();
|
|
3742
|
+
}
|
|
3743
|
+
/**
|
|
3744
|
+
* THE COPIES AN OLDER MASON MADE BEFORE A TYPE WAS HELD AS ITS TEXT (`HELD_LAYOUT`), read at `open` from what each copy
|
|
3745
|
+
* physically is, never from its record alone: a record written before `typeKind` was read says neither that a column
|
|
3746
|
+
* is an enum nor how a held one was stored. Each copied column is held to what `storedType` makes of it now: an ENUM is
|
|
3747
|
+
* an enum (its record completed, so its key is the one a fresh read makes, and it is not read again for nothing); a
|
|
3748
|
+
* column held as its text that is not a VARCHAR (a `point` as the extension's STRUCT) marks the table, said by name and
|
|
3749
|
+
* copied again through the normal path (`needsCopy`), serving as it was until its swap.
|
|
3750
|
+
*/
|
|
3751
|
+
async findHeldLayout() {
|
|
3752
|
+
const r = await this.store.writer.runAndReadAll(`SELECT schema_name || '.' || table_name, column_name, data_type FROM duckdb_columns() ` +
|
|
3753
|
+
`WHERE database_name = current_database() AND schema_name <> ${literal(MASON_SCHEMA)}`);
|
|
3754
|
+
const physical = new Map();
|
|
3755
|
+
for (const [source, column, type] of r.getRowsJS()) {
|
|
3756
|
+
const m = physical.get(source) ?? new Map();
|
|
3757
|
+
m.set(column, type);
|
|
3758
|
+
physical.set(source, m);
|
|
3759
|
+
}
|
|
3760
|
+
for (const [source, columns] of this.copiedColumns) {
|
|
3761
|
+
const types = physical.get(source);
|
|
3762
|
+
if (this.states.get(source)?.phase !== "copied" || types === undefined || this.retired.has(source))
|
|
3763
|
+
continue;
|
|
3764
|
+
const mis = [];
|
|
3765
|
+
const completed = columns.map((c) => {
|
|
3766
|
+
const type = types.get(c.name);
|
|
3767
|
+
if (type === undefined)
|
|
3768
|
+
return c;
|
|
3769
|
+
if (c.typeKind === undefined && type.startsWith("ENUM("))
|
|
3770
|
+
return { ...c, typeKind: "e" };
|
|
3771
|
+
const hold = holdOf(c.type, c.typeKind);
|
|
3772
|
+
const want = hold === "text" ? "VARCHAR" : hold === "text[]" ? "VARCHAR[]" : null;
|
|
3773
|
+
if (want !== null && type !== want)
|
|
3774
|
+
mis.push(`${c.name} (${c.type})`);
|
|
3775
|
+
return c;
|
|
3776
|
+
});
|
|
3777
|
+
this.copiedColumns.set(source, completed);
|
|
3778
|
+
this.copiedAs.set(source, columnsKey(completed));
|
|
3779
|
+
if (mis.length === 0)
|
|
3780
|
+
continue;
|
|
3781
|
+
this.retired.set(source, { layout: HELD_LAYOUT, short: `in an older Mason's layout, ${mis.join(", ")} not held as the source's text` });
|
|
3782
|
+
this.log(`${source}: it is ${HELD_LAYOUT} (${mis.join(", ")}) — it is read again whole; its synced data serves as it was until then`);
|
|
2742
3783
|
}
|
|
2743
3784
|
}
|
|
2744
3785
|
/** what a copy of the retired layout left beside the file (in that layout a copy was staged in a file of its own,
|
|
@@ -2747,7 +3788,7 @@ export class Sync {
|
|
|
2747
3788
|
const dir = path.dirname(this.store.file), base = `${path.basename(this.store.file)}.stage-`;
|
|
2748
3789
|
for (const f of fs.readdirSync(dir).filter((x) => x.startsWith(base))) {
|
|
2749
3790
|
fs.rmSync(path.join(dir, f), { recursive: true, force: true });
|
|
2750
|
-
this.log(`removed ${f}: a
|
|
3791
|
+
this.log(`removed ${f}: a sync's stage, from an older Mason (a sync now reads the source once, straight into its table)`);
|
|
2751
3792
|
}
|
|
2752
3793
|
}
|
|
2753
3794
|
/** The table's numerics the extension would read as doubles, each with the DECIMAL(38, s) scale that holds every value
|
|
@@ -2771,11 +3812,11 @@ export class Sync {
|
|
|
2771
3812
|
const scale = declared[1] !== undefined ? used ?? 0 : maxScale ?? 0;
|
|
2772
3813
|
const width = (digits ?? 1) + scale;
|
|
2773
3814
|
if ((special ?? 0) > 0) {
|
|
2774
|
-
this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which
|
|
3815
|
+
this.log(`${t.source}.${c.name}: ${special} value(s) NaN or Infinity, which the synced table's DECIMAL cannot hold: synced as a double`);
|
|
2775
3816
|
return;
|
|
2776
3817
|
}
|
|
2777
3818
|
if (scale > 38 || width > 38) {
|
|
2778
|
-
this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit
|
|
3819
|
+
this.log(`${t.source}.${c.name}: values of ${digits ?? 1} digit(s) and ${scale} decimal(s) do not fit the synced table's DECIMAL(38): synced as a double (15 significant digits)`);
|
|
2779
3820
|
return;
|
|
2780
3821
|
}
|
|
2781
3822
|
if (declared[1] === undefined && minScale !== null && minScale !== maxScale)
|
|
@@ -2795,6 +3836,12 @@ export class Sync {
|
|
|
2795
3836
|
* writer only on its turn: a statement on a connection lands inside the transaction open there). The table's row is
|
|
2796
3837
|
* that flow's alone while it writes it (a rebuild's table is not polled or streamed until caught up) */
|
|
2797
3838
|
async record(s, columns, c = this.store.writer) {
|
|
3839
|
+
// A COPY CURRENT AGAIN after its poll, catch-up or stream failed (its error cleared by the flow that keeps it current,
|
|
3840
|
+
// never by a copy of it: that is its own `copied` event) — the one place an error is cleared, so the one place said
|
|
3841
|
+
const was = this.states.get(s.source);
|
|
3842
|
+
if (was !== undefined && was.error !== null && s.error === null && was.phase === "copied" && s.phase === "copied" && was.copiedAt === s.copiedAt) {
|
|
3843
|
+
this.event({ kind: "recovered", table: s.source, message: `current again (it had failed: ${was.error.split("\n")[0]})` });
|
|
3844
|
+
}
|
|
2798
3845
|
this.states.set(s.source, s);
|
|
2799
3846
|
const v = (x) => x === null ? "NULL" : typeof x === "number" ? String(x) : literal(x);
|
|
2800
3847
|
const sql = `INSERT OR REPLACE INTO ${MASON_SCHEMA}.copy_state (source, phase, "rows", bytes, seconds, copied_at, error, columns) VALUES (${literal(s.source)}, ${literal(s.phase)}, ${v(s.rows)}, ${v(s.bytes)}, ${v(s.seconds)}, ` +
|
|
@@ -2804,7 +3851,7 @@ export class Sync {
|
|
|
2804
3851
|
await c.run(sql);
|
|
2805
3852
|
}
|
|
2806
3853
|
catch (e) {
|
|
2807
|
-
this.log(`recording ${s.source}'s
|
|
3854
|
+
this.log(`recording ${s.source}'s sync state failed: ${this.errorText(e)}`);
|
|
2808
3855
|
}
|
|
2809
3856
|
}
|
|
2810
3857
|
/** the catalog rebuilt (`schemaBuilt`), after any rebuild already under way */
|
|
@@ -2842,11 +3889,24 @@ export class Sync {
|
|
|
2842
3889
|
...(uniqueKeys.length ? { uniqueKeys } : {}),
|
|
2843
3890
|
...(t.rowTypeOid !== undefined && columns.length === cols.length ? { rowTypeOid: t.rowTypeOid } : {}) });
|
|
2844
3891
|
}
|
|
2845
|
-
|
|
2846
|
-
|
|
3892
|
+
// ONE SOURCE OF AN ENUM'S LABELS, the synced table's ENUM: a generated table stores the ENUM it was built with, which
|
|
3893
|
+
// a value added since (ALTER TYPE … ADD VALUE, re-copied) is not in, so its column declares the type by the synced
|
|
3894
|
+
// labels; its own only where no synced column has the type
|
|
3895
|
+
const synced = new Map(tables.flatMap((t) => t.columns.flatMap((x) => x.enumLabels !== undefined && x.spelled !== undefined ? [[x.spelled.replace(/\[\]$/, ""), x.enumLabels]] : [])));
|
|
3896
|
+
// each generated table served, as a copy of its output's types: no key, no joins
|
|
3897
|
+
for (const g of this.generated.served()) {
|
|
3898
|
+
const types = await c.prepare(`SELECT * FROM ${ident(g.schema)}.${ident(g.table)} LIMIT 0`);
|
|
3899
|
+
const duckTypes = new Map(Array.from({ length: types.columnCount }, (_, i) => [types.columnName(i), types.columnType(i)]));
|
|
3900
|
+
types.destroySync();
|
|
3901
|
+
const columns = g.columns.filter((x) => duckTypes.has(x.name)).map((x) => pgColumn({ name: x.name, type: x.type, notNull: false, comment: null }, duckTypes.get(x.name), undefined))
|
|
3902
|
+
.map((x) => { const labels = x.enumLabels !== undefined && x.spelled !== undefined ? synced.get(x.spelled.replace(/\[\]$/, "")) : undefined; return labels === undefined ? x : { ...x, enumLabels: labels }; });
|
|
3903
|
+
tables.push({ name: g.table, schema: g.schema, primaryKey: [], columns, joins: [], ...(g.comment !== null ? { description: g.comment } : {}) });
|
|
3904
|
+
}
|
|
3905
|
+
const views = this.views, collations = this.collations;
|
|
3906
|
+
const key = JSON.stringify([tables, views, collations]);
|
|
2847
3907
|
if (key !== this.schemaKey) {
|
|
2848
3908
|
this.schemaKey = key;
|
|
2849
|
-
this.currentSchema = { database: this.database, tables, ...(views.length > 0 ? { views } : {}) };
|
|
3909
|
+
this.currentSchema = { database: this.currentSchema.database, tables, ...(views.length > 0 ? { views } : {}), ...(collations.length > 0 ? { collations } : {}) };
|
|
2850
3910
|
}
|
|
2851
3911
|
}
|
|
2852
3912
|
}
|
|
@@ -2865,33 +3925,14 @@ async function enumLists(c, table) {
|
|
|
2865
3925
|
function pgColumnSql(c, exact, lists) {
|
|
2866
3926
|
if (exact.has(c.name))
|
|
2867
3927
|
return `${pgIdent(c.name)}::text AS ${pgIdent(c.name)}`;
|
|
2868
|
-
|
|
3928
|
+
const hold = holdOf(c.type, c.typeKind);
|
|
3929
|
+
if (KEPT_ARRAYS.has(c.type) || lists.has(c.name) || hold === "text[]")
|
|
2869
3930
|
return `${pgIdent(c.name)}::text[] AS ${pgIdent(c.name)}`;
|
|
2870
|
-
|
|
2871
|
-
}
|
|
2872
|
-
|
|
2873
|
-
|
|
2874
|
-
const array = spelled.endsWith("[]");
|
|
2875
|
-
const base = spelled.replace(/\[\]$/, "").replace(/\(.*\)/, "").trim();
|
|
2876
|
-
const oid = {
|
|
2877
|
-
smallint: 21, integer: 23, bigint: 20, real: 700, "double precision": 701, numeric: 1700, text: 25, "character varying": 25, character: 25,
|
|
2878
|
-
boolean: 16, uuid: 2950, date: 1082, "time without time zone": 1083, "timestamp without time zone": 1114, "timestamp with time zone": 1184,
|
|
2879
|
-
interval: 1186, json: 3802, jsonb: 3802, bytea: 17,
|
|
2880
|
-
// held as DuckDB text, declared as themselves: the catalog Postgres resolves their operators (citext's
|
|
2881
|
-
// case-insensitive ones) and the wire sends their OID (src/duck/types.ts)
|
|
2882
|
-
inet: 25, citext: 25,
|
|
2883
|
-
};
|
|
2884
|
-
const o = oid[base];
|
|
2885
|
-
if (o === undefined)
|
|
2886
|
-
return null;
|
|
2887
|
-
return array ? -o : o;
|
|
3931
|
+
if (hold === "text")
|
|
3932
|
+
return `${pgIdent(c.name)}::text AS ${pgIdent(c.name)}`;
|
|
3933
|
+
// a domain's value as its base type, which the extension reads
|
|
3934
|
+
return c.domain === true ? `${pgIdent(c.name)}::${c.type} AS ${pgIdent(c.name)}` : pgIdent(c.name);
|
|
2888
3935
|
}
|
|
2889
|
-
/** the Postgres spelling of the wire's OID: what the catalog says when the source's spelling would disagree with the wire */
|
|
2890
|
-
const SPELLING_OF_OID = {
|
|
2891
|
-
21: "smallint", 23: "integer", 20: "bigint", 700: "real", 701: "double precision", 1700: "numeric", 25: "text", 16: "boolean", 2950: "uuid",
|
|
2892
|
-
1082: "date", 1083: "time without time zone", 1114: "timestamp without time zone", 1184: "timestamp with time zone", 1186: "interval", 3802: "jsonb", 17: "bytea",
|
|
2893
|
-
};
|
|
2894
|
-
const ELEMENT_OF_ARRAY = { 1000: 16, 1001: 17, 1016: 20, 1005: 21, 1007: 23, 1009: 25, 1021: 700, 1022: 701, 1182: 1082, 1183: 1083, 1115: 1114, 1185: 1184, 1187: 1186, 1231: 1700, 2951: 2950, 3807: 3802 };
|
|
2895
3936
|
const API_TYPE = { 21: "int2", 23: "int4", 20: "int8", 700: "float4", 701: "float8", 1700: "numeric", 25: "text", 16: "bool", 2950: "uuid", 1082: "date", 1184: "timestamptz", 3802: "jsonb" };
|
|
2896
3937
|
/** an ENUM's labels, of the column or of its list's element; null for any other type */
|
|
2897
3938
|
function enumLabelsOf(t) {
|
|
@@ -2899,28 +3940,30 @@ function enumLabelsOf(t) {
|
|
|
2899
3940
|
return own instanceof DuckDBEnumType ? own.values : null;
|
|
2900
3941
|
}
|
|
2901
3942
|
/**
|
|
2902
|
-
* A copied column in the catalog: the
|
|
2903
|
-
*
|
|
2904
|
-
*
|
|
3943
|
+
* A copied column in the catalog: ALWAYS the source's type, as the source spells it (`numeric(12,2)`, `character
|
|
3944
|
+
* varying(255)`, `time with time zone`, `point`), so a tool sees the source's own types whatever the copy holds
|
|
3945
|
+
* (src/duck/held-types.ts) — but for the one the copy computes as another: a `numeric` past DuckDB's DECIMAL, read as
|
|
3946
|
+
* DOUBLE, is `double precision` on the catalog as on the wire (the limits `numeric-over-38-digits`, `numeric-array`).
|
|
2905
3947
|
*/
|
|
2906
3948
|
function pgColumn(c, duck, now) {
|
|
2907
3949
|
const wire = wireColumn(c.name, duck).oid;
|
|
2908
|
-
|
|
2909
|
-
|
|
2910
|
-
// an array DuckDB holds as VARCHAR[] whose elements the wire writes as the source's type (src/duck/types.ts: a json's
|
|
2911
|
-
// text, a jsonb's canonical text, an inet's output, a citext's bytes): declared as the source spells it
|
|
2912
|
-
// an enum (or an array of one) DuckDB holds as its ENUM, in the declared order: declared as the source's type, with
|
|
2913
|
-
// its labels, so Postgres orders and compares it by them
|
|
3950
|
+
// an enum (or an array of one) DuckDB holds as its ENUM, in the declared order: declared with its labels, so Postgres
|
|
3951
|
+
// orders and compares it by them
|
|
2914
3952
|
const labels = enumLabelsOf(duck);
|
|
2915
|
-
const
|
|
2916
|
-
|
|
3953
|
+
const double = c.type.startsWith("numeric") ? wire === 701 ? "double precision" : wire === 1022 ? "double precision[]" : null : null;
|
|
3954
|
+
const spelled = double ?? c.type;
|
|
3955
|
+
const kind = (now ?? c).typeKind ?? c.typeKind;
|
|
3956
|
+
// a type a database made, held as its text: the catalog declares it as a type with nothing over it
|
|
3957
|
+
const made = c.typeOid !== undefined && labels === null && holdOf(c.type, kind) !== "same" ? kind === "c" ? "row" : true : undefined;
|
|
2917
3958
|
// `type` is the ClickHouse path's; the catalog reads `spelled`
|
|
2918
3959
|
const type = API_TYPE[wire] ?? "text";
|
|
2919
3960
|
const facts = now ?? c;
|
|
2920
|
-
return { name: c.name, type, spelled, ...(c.comment ? { description: c.comment } : {}), ...(labels !== null ? { enumLabels: labels } : {}),
|
|
3961
|
+
return { name: c.name, type, spelled, ...(c.comment ? { description: c.comment } : {}), ...(labels !== null ? { enumLabels: labels } : {}), ...(made !== undefined ? { made } : {}),
|
|
2921
3962
|
...(facts.notNull ? { notNull: true } : {}), ...(facts.default !== undefined ? { default: facts.default } : {}),
|
|
2922
3963
|
...(facts.identity !== undefined ? { identity: facts.identity } : {}), ...(facts.sequence !== undefined ? { sequence: facts.sequence } : {}),
|
|
2923
|
-
...(
|
|
3964
|
+
...(facts.collation !== undefined ? { collation: facts.collation } : {}),
|
|
3965
|
+
// a number, whatever spelling the column's record kept (an oid is a string in Postgres's JSON, and a copy's columns are JSON)
|
|
3966
|
+
...(c.typeOid !== undefined && double === null ? { typeOid: Number(c.typeOid), ...(c.typeArrayOid !== undefined ? { typeArrayOid: Number(c.typeArrayOid) } : {}) } : {}) };
|
|
2924
3967
|
}
|
|
2925
3968
|
/** a span as the stream's line says it: `2.1 s`, `3 min`, `2 h` */
|
|
2926
3969
|
function secondsText(ms) {
|