@mason-data/cli 0.0.0-stage → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +268 -2
  3. package/dist/src/api/error.js +18 -0
  4. package/dist/src/api/postgres/catalog-worker.js +92 -0
  5. package/dist/src/api/postgres/catalog.js +477 -0
  6. package/dist/src/api/postgres/encode.js +541 -0
  7. package/dist/src/api/postgres/explain.js +143 -0
  8. package/dist/src/api/postgres/listener.js +60 -0
  9. package/dist/src/api/postgres/parser-worker.js +58 -0
  10. package/dist/src/api/postgres/parser.js +121 -0
  11. package/dist/src/api/postgres/plain.js +33 -0
  12. package/dist/src/api/postgres/protocol.js +404 -0
  13. package/dist/src/api/postgres/schema.js +54 -0
  14. package/dist/src/api/postgres/server.js +1771 -0
  15. package/dist/src/api/postgres/translate.js +248 -0
  16. package/dist/src/api/request-log.js +50 -0
  17. package/dist/src/cdc/pg-wire.js +588 -0
  18. package/dist/src/cdc/pgoutput.js +107 -0
  19. package/dist/src/cdc/poll.js +67 -0
  20. package/dist/src/cdc/source.js +65 -0
  21. package/dist/src/cli/background.js +198 -0
  22. package/dist/src/cli/command.js +173 -0
  23. package/dist/src/cli/commands.js +1277 -0
  24. package/dist/src/cli/control.js +36 -0
  25. package/dist/src/cli/demo.js +149 -0
  26. package/dist/src/cli/dev.js +669 -0
  27. package/dist/src/cli/doctor.js +260 -0
  28. package/dist/src/cli/guides.js +158 -0
  29. package/dist/src/cli/init.js +366 -0
  30. package/dist/src/cli/machine.js +162 -0
  31. package/dist/src/cli/main.js +83 -0
  32. package/dist/src/cli/mcp.js +726 -0
  33. package/dist/src/cli/pg-tls.js +100 -0
  34. package/dist/src/cli/pg-url.js +52 -0
  35. package/dist/src/cli/pin.js +36 -0
  36. package/dist/src/cli/ports.js +118 -0
  37. package/dist/src/cli/preflight.js +347 -0
  38. package/dist/src/cli/project.js +129 -0
  39. package/dist/src/cli/readiness.js +76 -0
  40. package/dist/src/cli/registry.js +220 -0
  41. package/dist/src/cli/replication-checks.js +120 -0
  42. package/dist/src/cli/replication-mode.js +148 -0
  43. package/dist/src/cli/run-server.js +311 -0
  44. package/dist/src/cli/runs.js +120 -0
  45. package/dist/src/cli/security.js +86 -0
  46. package/dist/src/cli/self-signed-cert.js +64 -0
  47. package/dist/src/cli/serve.js +301 -0
  48. package/dist/src/cli/source-edit.js +116 -0
  49. package/dist/src/cli/sources.js +564 -0
  50. package/dist/src/cli/state.js +58 -0
  51. package/dist/src/cli/style.js +187 -0
  52. package/dist/src/cli/supervise.js +248 -0
  53. package/dist/src/cli/sync.js +94 -0
  54. package/dist/src/cli/ui.js +107 -0
  55. package/dist/src/cli/user.js +229 -0
  56. package/dist/src/cli/version.js +23 -0
  57. package/dist/src/demo/seed.js +237 -0
  58. package/dist/src/demo/source.js +655 -0
  59. package/dist/src/duck/analyze.js +113 -0
  60. package/dist/src/duck/cancel.js +49 -0
  61. package/dist/src/duck/distinct-key.js +228 -0
  62. package/dist/src/duck/executor-worker.js +268 -0
  63. package/dist/src/duck/executor.js +567 -0
  64. package/dist/src/duck/explain.js +238 -0
  65. package/dist/src/duck/functions.js +819 -0
  66. package/dist/src/duck/gaps.js +94 -0
  67. package/dist/src/duck/jsonb-order.js +175 -0
  68. package/dist/src/duck/jsonb.js +188 -0
  69. package/dist/src/duck/kept-route.js +299 -0
  70. package/dist/src/duck/kept.js +964 -0
  71. package/dist/src/duck/late.js +221 -0
  72. package/dist/src/duck/libc-order.js +190 -0
  73. package/dist/src/duck/pg-eval-worker.js +34 -0
  74. package/dist/src/duck/pg-eval.js +59 -0
  75. package/dist/src/duck/pgtree.js +378 -0
  76. package/dist/src/duck/plan.js +259 -0
  77. package/dist/src/duck/publication.js +207 -0
  78. package/dist/src/duck/requests.js +209 -0
  79. package/dist/src/duck/resolved.js +248 -0
  80. package/dist/src/duck/rewrite.js +5101 -0
  81. package/dist/src/duck/store.js +907 -0
  82. package/dist/src/duck/stream.js +518 -0
  83. package/dist/src/duck/sync.js +2929 -0
  84. package/dist/src/duck/to-char.js +482 -0
  85. package/dist/src/duck/token.js +31 -0
  86. package/dist/src/duck/types.js +427 -0
  87. package/dist/src/duck/users.js +231 -0
  88. package/dist/src/duck/verified.js +1001 -0
  89. package/dist/src/duck/views.js +150 -0
  90. package/dist/src/lib/database-url.js +74 -0
  91. package/dist/src/lib/did-you-mean.js +43 -0
  92. package/dist/src/lib/dotenv.js +72 -0
  93. package/dist/src/lib/exit-when-written.js +20 -0
  94. package/dist/src/lib/float-equal.js +69 -0
  95. package/dist/src/lib/http-json.js +22 -0
  96. package/dist/src/lib/log.js +161 -0
  97. package/dist/src/lib/pg-numeric.js +489 -0
  98. package/dist/src/lib/port.js +22 -0
  99. package/dist/src/lib/project-file.js +49 -0
  100. package/dist/src/lib/project-files.js +43 -0
  101. package/dist/src/lib/project-version.js +76 -0
  102. package/dist/src/lib/runnable.js +14 -0
  103. package/dist/src/lib/shell-word.js +4 -0
  104. package/dist/src/lib/timezone.js +237 -0
  105. package/dist/src/lib/zone-names.js +114 -0
  106. package/dist/src/web/app.css +76 -0
  107. package/dist/src/web/app.js +491 -0
  108. package/dist/src/web/index.html +44 -0
  109. package/package.json +22 -4
@@ -0,0 +1,220 @@
1
+ /**
2
+ * EVERY `mason` COMMAND, AND THE HELP MADE FROM THEM. `mason --help` is generated from this list — each
3
+ * command's summary, in this order — so a command cannot exist without being listed, and tests/cli-help.test.ts holds
4
+ * each one to src/cli/command.ts's rules (its own --help, an example, every option documented).
5
+ */
6
+ import { failed, parseCommand, reportText } from "./command.js";
7
+ import { dev } from "./dev.js";
8
+ import { devInBackground } from "./background.js";
9
+ import { ADD, LOGS, REMOVE, STATUS, STOP, TABLES, WAIT, add, logs, remove, status, stop, tables, wait } from "./commands.js";
10
+ import { USER, user } from "./user.js";
11
+ import { MCP, mcp } from "./mcp.js";
12
+ import { UI, ui } from "./ui.js";
13
+ import { GUIDES, guideText } from "./guides.js";
14
+ import { DOCTOR, NODE_MIN, doctor } from "./doctor.js";
15
+ import { DESTROY, destroy } from "./machine.js";
16
+ import { INIT, init } from "./init.js";
17
+ import { SYNC, sync } from "./sync.js";
18
+ import { SERVE, serve } from "./serve.js";
19
+ import { PROJECT_LINE } from "./dev.js";
20
+ import { version } from "./version.js";
21
+ import { didYouMean } from "../lib/did-you-mean.js";
22
+ import { out as styleOut, wrap } from "./style.js";
23
+ import { theProject } from "./project.js";
24
+ import { liveRun } from "./runs.js";
25
+ export { version } from "./version.js";
26
+ export const DEV = {
27
+ needs: "project",
28
+ name: "start",
29
+ summary: "start Mason for this folder's project, in the background (mason init makes the project)",
30
+ usage: `usage: mason start [--attach] [--replication [--publication <name>] | --poll] [--self-signed-tls] [--port <n>] [--json]
31
+
32
+ Starts Mason for the project of this folder (mason init made it): one process, which copies the tables you add
33
+ (mason add) from the project's database into its own file, keeps them current, and answers the Postgres protocol
34
+ for BI tools and your app. It only reads the database (SELECT): no wal_level, replication slot or publication is
35
+ needed or made. It returns once Mason is up; run it again any time (it says so when Mason is already running). The
36
+ copy and Mason's state stay between runs: the next start resumes.
37
+
38
+ With --replication it reads the database's change log instead: it needs wal_level = logical and a user that may
39
+ replicate, and makes its own publication and replication slot on the database (mason destroy, or mason start --poll,
40
+ drops them). The project keeps the mode until --poll; a switch keeps the copies and catches each table up by one
41
+ poll. --publication <name> (else PG_PUBLICATION) names a publication that is the database owner's: it is read as it
42
+ is, verified to carry every added table and column, and never created, altered or dropped — Mason makes only its
43
+ slot. A read replica needs one (it takes no CREATE PUBLICATION). The name is kept with the project, so the next
44
+ mason start --replication reads it again.
45
+
46
+ The Postgres API's port is the project's: the first start takes 5433, or the next port free on every address when a
47
+ second Postgres holds it, and records it; every later start listens on the same port, so a BI tool's saved connection
48
+ keeps working, and refuses, naming what holds it, rather than move. MASON_PG_PORT=<n> in your shell names the port,
49
+ or moves the project to it (said): that port is used, or start refuses, naming what holds it.
50
+
51
+ The Postgres API listens on 127.0.0.1 unless MASON_PG_HOST names another address (0.0.0.0: every address of this
52
+ machine). Off this machine a password travels on the connection, so it then needs TLS: MASON_PG_TLS_CERT and
53
+ MASON_PG_TLS_KEY (PEM files), or --self-signed-tls, which makes a certificate for the project in .mason/ and serves
54
+ it again at every later start with the flag. Every client then connects with sslmode=require. A team whose own proxy
55
+ or load balancer terminates TLS keeps it on 127.0.0.1, or sets MASON_PG_PLAIN=1 for a private network.
56
+
57
+ ${PROJECT_LINE}
58
+
59
+ mason start
60
+ mason start --attach in the foreground: Ctrl-C stops it
61
+ mason start --replication read the change log, as production does
62
+ mason start --replication --publication app_pub through the database owner's publication
63
+ MASON_PG_HOST=0.0.0.0 mason start --self-signed-tls the Postgres API for other machines, over TLS
64
+
65
+ --attach stay in the foreground (Ctrl-C stops Mason)
66
+ --replication read the database's changes from its change log (logical replication), as production does
67
+ --publication <name> with --replication: the publication to read, the owner's, never altered (PG_PUBLICATION;
68
+ default: Mason's own, mason_dev_<project>_<source>)
69
+ --poll read them with SELECTs (the default; takes back --replication)
70
+ --self-signed-tls the Postgres API requires TLS, from a certificate made for this project in .mason/ (kept and
71
+ served again by every later start with the flag); not with MASON_PG_TLS_CERT / _KEY
72
+ --port <n> the port the mason commands reach it on (default 4039, or the next free one)
73
+ --json as data, once it is up: { running, pid, postgres: { url, host, port, database, sslmode }, next }
74
+ `,
75
+ options: {
76
+ port: { type: "string" }, attach: { type: "boolean", default: false }, json: { type: "boolean", default: false },
77
+ replication: { type: "boolean", default: false }, poll: { type: "boolean", default: false }, publication: { type: "string" },
78
+ "self-signed-tls": { type: "boolean", default: false },
79
+ },
80
+ json: ["running", "pid", "postgres", "next"],
81
+ };
82
+ async function runDev(argv) {
83
+ const f = parseCommand(DEV, argv);
84
+ if (typeof f === "number")
85
+ return f;
86
+ const v = f.values;
87
+ if (v.replication && v.poll)
88
+ return failed("start", { message: "--replication or --poll, not both", code: "usage", usage: DEV.usage }, { json: v.json });
89
+ if (v.publication !== undefined && v.poll)
90
+ return failed("start", { message: "--publication goes with --replication, not --poll", code: "usage", usage: DEV.usage }, { json: v.json });
91
+ if (v.publication !== undefined && !/^\S+$/.test(v.publication))
92
+ return failed("start", { message: `--publication must name a publication, not ${JSON.stringify(v.publication)}`, code: "usage", usage: DEV.usage }, { json: v.json });
93
+ if (f.positionals.length) {
94
+ // the database is the project's now: a URL here is the old habit, answered with the new way
95
+ return failed("start", { message: "it takes no database URL: the project has one", code: "usage", why: "mason init <url> made it; .mason/config.json holds it", usage: DEV.usage }, { json: v.json });
96
+ }
97
+ const port = v.port === undefined ? 4039 : Number(v.port);
98
+ if (!Number.isInteger(port) || port < 1 || port > 65_535)
99
+ return failed("start", { message: `--port must be a port number (1-65535), not ${JSON.stringify(v.port)}`, code: "usage" }, { json: v.json });
100
+ if (!v.attach)
101
+ return devInBackground(argv, { json: v.json, selfSignedTls: v["self-signed-tls"], changes: v.replication ? "replication" : v.poll ? "poll" : null });
102
+ return dev({ port, version: version(), changes: v.replication ? "replication" : v.poll ? "poll" : null, publication: v.publication ?? null, selfSignedTls: v["self-signed-tls"] });
103
+ }
104
+ export const HELP = {
105
+ name: "help",
106
+ summary: "the commands by step, or one command's usage: mason help add",
107
+ usage: `usage: mason help [<command> | <guide>]
108
+
109
+ The commands by step (mason help), how to use one (mason help add — the same as mason add --help), or a guide,
110
+ offline, short enough to read whole and listing exactly what Mason accepts:
111
+
112
+ mason help
113
+ mason help add
114
+ ${Object.entries(GUIDES).map(([k, v]) => ` mason help ${k.padEnd(14)} ${v}`).join("\n")}
115
+ `,
116
+ options: {},
117
+ };
118
+ async function runHelp(argv) {
119
+ const f = parseCommand(HELP, argv);
120
+ if (typeof f === "number")
121
+ return f;
122
+ const [asked, ...rest] = f.positionals;
123
+ if (rest.length)
124
+ return failed("help", { message: "one command or guide at a time", code: "usage", usage: HELP.usage }, { json: false });
125
+ if (asked === undefined) {
126
+ process.stdout.write(topUsage());
127
+ return 0;
128
+ }
129
+ if (asked === "version") {
130
+ process.stdout.write("usage: mason version (or --version, -v): the CLI's version\n");
131
+ return 0;
132
+ }
133
+ if (Object.hasOwn(GUIDES, asked)) {
134
+ process.stdout.write(await guideText(asked));
135
+ return 0;
136
+ }
137
+ const cmd = COMMANDS.find((c) => c.name === asked);
138
+ if (cmd) {
139
+ process.stdout.write(cmd.usage);
140
+ return 0;
141
+ }
142
+ return failed("help", { message: `there is no command or guide ${JSON.stringify(asked)}${didYouMean(asked, [...COMMANDS.map((c) => c.name), ...Object.keys(GUIDES)])}`, code: "usage",
143
+ why: `the guides: ${Object.keys(GUIDES).join(", ")}`, next: [["mason help", "every command"]] }, { json: false });
144
+ }
145
+ /**
146
+ * EVERY COMMAND, IN THE ORDER A FIRST SESSION NEEDS THEM (listed in the order a developer will likely need them, the
147
+ * reference, `mason help`, first because it is read first), under the step each belongs to: `mason --help` prints the
148
+ * steps as headings, so the list reads as the runbook it is. A new command goes into its step here; COMMANDS is
149
+ * derived, so the order is written once.
150
+ */
151
+ export const STEPS = [
152
+ ["Learn", [{ ...HELP, run: runHelp }]],
153
+ ["Set up", [{ ...DOCTOR, run: (argv) => doctor(argv) }, { ...INIT, run: (argv) => init(argv) }, { ...DEV, run: runDev }]],
154
+ ["Add your tables", [{ ...TABLES, run: tables }, { ...ADD, run: add }, { ...REMOVE, run: remove }, { ...WAIT, run: wait }]],
155
+ ["Connect", [{ ...USER, run: user }, { ...MCP, run: mcp }]],
156
+ ["Run", [{ ...STATUS, run: status }, { ...UI, run: ui }, { ...SYNC, run: sync }, { ...LOGS, run: logs }, { ...STOP, run: stop }, { ...DESTROY, run: (argv) => destroy(argv) }]],
157
+ ["Go to production", [{ ...SERVE, run: serve }]],
158
+ ];
159
+ /** Every command, in the order `mason --help` lists them. */
160
+ export const COMMANDS = STEPS.flatMap(([, commands]) => commands);
161
+ /**
162
+ * `mason` ALONE: where you are and what to run next (`mason help` once showed the same as `mason` alone, and read as
163
+ * broken). One line on what Mason is, whether this folder is a project and whether Mason runs for
164
+ * it, then the next command for that moment — and mason help, every command. `mason help` and `mason --help` are the
165
+ * whole list (topUsage).
166
+ */
167
+ export async function orientation(cwd, s = styleOut()) {
168
+ const t = theProject(cwd);
169
+ const project = "project" in t ? t.project : null;
170
+ const run = project ? await liveRun(project) : null;
171
+ const rows = project
172
+ ? [["Project", `${project.config.name}, files in ${project.folder}`], ["Mason", run ? `running (pid ${run.pid})` : "not running"]]
173
+ : [["Project", "none here: mason init makes this folder one"]];
174
+ const next = !project
175
+ ? [["mason init", "make this folder a project (DATABASE_URL from .env, or mason init postgres://…)"],
176
+ ["mason init demo", "or try it on a demo database Mason makes itself: nothing to install"],
177
+ ["mason doctor", "what this machine needs for Mason, and whether it has it"]]
178
+ : !run ? [["mason start", "start Mason for this project, in the background"], ["mason status", "the project and its tables"]]
179
+ : [["mason status", "what Mason is doing: its tables, and the last day's slow and refused statements"], ["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"]];
180
+ return `${reportText({ headline: "Mason: your Postgres tables, copied and served to BI tools and your app.", details: rows,
181
+ next: [...next, ["mason help", "every command, and the guides"]] }, s)}\n`;
182
+ }
183
+ /** `mason --help`: what Mason is, every command with its summary, and where each one's help is. On a terminal the steps are
184
+ * bold, the commands cyan and every line wrapped at its width under its own column (src/cli/style.ts); anywhere else,
185
+ * plain and unwrapped. */
186
+ export function topUsage(s = styleOut()) {
187
+ // what each step lists: its commands, and under Learn the guides too (`mason help` shows the guides listed under
188
+ // Learn), so the pages a session reads first are a line each, not a paragraph after the list
189
+ const entries = (step, commands) => [
190
+ ...commands.map((c) => [c.name, c.summary]),
191
+ ...(step === "Learn" ? Object.entries(GUIDES).map(([g, what]) => [`help ${g}`, what]) : []),
192
+ ];
193
+ const listed = STEPS.map(([step, commands]) => [step, entries(step, commands)]);
194
+ // the commands' column is the longest COMMAND's, everywhere; a guide's longer name widens Learn alone, so it never
195
+ // pushes every other step's summaries right
196
+ const commandWidth = Math.max(...COMMANDS.map((c) => c.name.length));
197
+ const para = (text) => ` ${wrap(text, s.width, 2)}`;
198
+ const block = (es) => {
199
+ const w = Math.max(commandWidth, ...es.map(([name]) => name.length));
200
+ // the column a summary starts in: " mason " + the longest name + 3
201
+ const col = 4 + "mason ".length + w + 3;
202
+ return es.map(([name, summary]) => ` ${s.cyan(`mason ${name.padEnd(w)}`)} ${s.dim(wrap(summary, s.width, col))}`).join("\n");
203
+ };
204
+ return [
205
+ `${s.bold("usage:")} mason <command> [options]`,
206
+ "",
207
+ para("Mason: your Postgres tables, copied and served to BI tools and your app. Make your app's folder a project (mason init), " +
208
+ `add tables (mason add, before or after the start), start Mason (mason start), connect a tool (mason user add). It needs Node ${NODE_MIN} or later; mason doctor checks.`),
209
+ para(PROJECT_LINE),
210
+ "",
211
+ listed.map(([step, es]) => ` ${s.bold(step)}\n${block(es)}`).join("\n\n"),
212
+ "",
213
+ para("mason <command> --help (or mason help <command>) says how to use one, with examples; the guides are offline and short enough to read whole."),
214
+ para("A command means the project of the folder it runs in (or above it)."),
215
+ para("Every command but help and mcp (whose stdout is the MCP protocol's) takes --json: one JSON object for a program, always with next (what to run " +
216
+ "next); a failure is { error: { code, message, why, fix }, next }. The exit code says how it went: 0 done, 1 failed " +
217
+ "with the reason, 2 a wrong command line, 124 mason wait timed out. Driving Mason from a coding agent: mason help agent."),
218
+ "",
219
+ ].join("\n");
220
+ }
@@ -0,0 +1,120 @@
1
+ /** Everything the checks read of the server, in one statement: settings and roles only. */
2
+ export const SERVER_FACTS_SQL = `SELECT json_build_object(
3
+ 'version', current_setting('server_version_num')::int, 'version_text', current_setting('server_version'),
4
+ 'standby', pg_is_in_recovery(), 'user', current_user,
5
+ 'wal_level', current_setting('wal_level'),
6
+ 'slot_wal_keep_mb', (SELECT CASE WHEN setting = '-1' THEN -1 ELSE setting::bigint END FROM pg_settings WHERE name = 'max_slot_wal_keep_size'),
7
+ 'max_slots', current_setting('max_replication_slots')::int, 'slots_used', (SELECT count(*) FROM pg_replication_slots),
8
+ 'max_senders', current_setting('max_wal_senders')::int, 'senders_used', (SELECT count(*) FROM pg_stat_replication),
9
+ 'superuser', r.rolsuper, 'replication', r.rolreplication,
10
+ 'rds', CASE WHEN EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rds_replication')
11
+ THEN json_build_object('member', pg_has_role(current_user, 'rds_replication', 'member')) END,
12
+ 'cloud_sql', current_setting('cloudsql.logical_decoding', true) IS NOT NULL,
13
+ 'hot_standby_feedback', current_setting('hot_standby_feedback') = 'on',
14
+ 'idle_in_transaction_ms', (SELECT setting::bigint FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'),
15
+ 'statement_timeout_ms', (SELECT setting::bigint FROM pg_settings WHERE name = 'statement_timeout')
16
+ )::text FROM pg_roles r WHERE r.rolname = current_user`;
17
+ /** pg_stat_wal (Postgres 14+): the WAL written since the statistics were reset, and over how long. */
18
+ export const WAL_RATE_SQL = "SELECT wal_bytes::text, extract(epoch FROM now() - stats_reset)::text FROM pg_stat_wal";
19
+ export async function readServerFacts(pg) {
20
+ const j = JSON.parse((await pg.query(SERVER_FACTS_SQL)).rows[0]?.[0] ?? "null");
21
+ if (j === null)
22
+ throw new Error("the server's settings did not read (no pg_roles row for this user)");
23
+ let walBytesPerSecond = null;
24
+ if (Number(j.version) >= 140000 && j.standby !== true) {
25
+ const r = (await pg.query(WAL_RATE_SQL)).rows[0];
26
+ const bytes = Number(r?.[0]), seconds = Number(r?.[1]);
27
+ // a reset moments ago says nothing about the rate: an hour of statistics at least
28
+ if (Number.isFinite(bytes) && Number.isFinite(seconds) && seconds >= 3600)
29
+ walBytesPerSecond = bytes / seconds;
30
+ }
31
+ return {
32
+ version: Number(j.version), versionText: String(j.version_text), standby: j.standby === true, user: String(j.user),
33
+ walLevel: String(j.wal_level), slotWalKeepMb: Number(j.slot_wal_keep_mb),
34
+ maxSlots: Number(j.max_slots), slotsUsed: Number(j.slots_used), maxSenders: Number(j.max_senders), sendersUsed: Number(j.senders_used),
35
+ superuser: j.superuser === true, replication: j.replication === true,
36
+ rds: j.rds === null || j.rds === undefined ? null : { member: j.rds.member === true },
37
+ cloudSql: j.cloud_sql === true, hotStandbyFeedback: j.hot_standby_feedback === true,
38
+ idleInTransactionMs: Number(j.idle_in_transaction_ms), statementTimeoutMs: Number(j.statement_timeout_ms), walBytesPerSecond,
39
+ };
40
+ }
41
+ /** The slot Mason would read, if it exists: whether a reader holds it and whether Postgres invalidated it. */
42
+ export async function readSlotFacts(pg, slot) {
43
+ const r = (await pg.query(`SELECT active::text, active_pid::text, wal_status, plugin FROM pg_replication_slots WHERE slot_name = '${slot.replace(/'/g, "''")}'`)).rows[0];
44
+ return r ? { active: r[0] === "true", activePid: r[1] ? Number(r[1]) : null, walStatus: r[2] ?? null, plugin: r[3] ?? null } : null;
45
+ }
46
+ /** The way out of a slot another reader holds, in the words of the command that met it. */
47
+ /** Bytes as a person reads them: 1.4 GiB, 312 MiB, 18 KiB. */
48
+ export function bytesText(n) {
49
+ const units = ["bytes", "KiB", "MiB", "GiB", "TiB"];
50
+ let i = 0, v = n;
51
+ while (v >= 1024 && i < units.length - 1) {
52
+ v /= 1024;
53
+ i++;
54
+ }
55
+ return i === 0 ? `${n} bytes` : `${v < 10 ? v.toFixed(1) : Math.round(v)} ${units[i]}`;
56
+ }
57
+ const q = (s) => /^[a-z_][a-z0-9_]*$/.test(s) ? s : `"${s.replace(/"/g, '""')}"`;
58
+ /**
59
+ * A `max_slot_wal_keep_size` to suggest: a day of this server's WAL at its measured rate, rounded up to a whole GB, at
60
+ * least 4 GB; null when the rate is unknown (a standby, Postgres 13, statistics reset within the hour).
61
+ */
62
+ export function suggestedWalCapGb(bytesPerSecond) {
63
+ if (bytesPerSecond === null)
64
+ return null;
65
+ return Math.max(4, Math.ceil((bytesPerSecond * 86_400) / 1024 ** 3));
66
+ }
67
+ /** What wal_level = logical takes on this server: its provider's way, or postgresql.conf's. */
68
+ function walLevelFix(f) {
69
+ if (f.rds)
70
+ return "set rds.logical_replication = 1 in the DB parameter group (Aurora: the DB cluster parameter group), then reboot the instance";
71
+ if (f.cloudSql)
72
+ return "set the database flag cloudsql.logical_decoding = on (the instance restarts)";
73
+ return "set wal_level = logical (ALTER SYSTEM SET wal_level = logical, or postgresql.conf; a managed Postgres: its parameter or flag of that name), then restart Postgres";
74
+ }
75
+ /** The checks of the server for the stream (REPLICATION). */
76
+ export function replicationChecks(f, slot, slotState) {
77
+ const out = [];
78
+ out.push(f.walLevel === "logical"
79
+ ? { name: "wal_level", ok: true, detail: "logical" }
80
+ : { name: "wal_level", ok: false, detail: `${f.walLevel}: logical replication needs logical. To fix: ${walLevelFix(f)}. Or read changes by polling instead (--poll: SELECT alone, for a small database)` });
81
+ if (f.standby) {
82
+ out.push(f.version >= 160000
83
+ ? { name: "standby", ok: true, warn: !f.hotStandbyFeedback,
84
+ detail: `a read replica (Postgres 16+ decodes on a standby)${f.hotStandbyFeedback ? "" : ". hot_standby_feedback is off: the primary may remove catalog rows the slot still needs, which invalidates it (Mason then rebuilds from scratch) — turn it on"}` }
85
+ : { name: "standby", ok: false, detail: `a read replica on Postgres ${f.versionText}: logical decoding on a standby needs Postgres 16. Point Mason at the primary, upgrade the replica, or poll it (--poll)` });
86
+ }
87
+ const canReplicate = f.superuser || f.replication || f.rds?.member === true;
88
+ out.push(canReplicate
89
+ ? { name: "replication role", ok: true, detail: `${f.user} may replicate${f.superuser ? " (a superuser)" : f.rds?.member ? " (rds_replication)" : " (REPLICATION)"}` }
90
+ : { name: "replication role", ok: false, detail: `${f.user} may not open a replication connection`,
91
+ sql: f.rds ? `GRANT rds_replication TO ${q(f.user)};` : `ALTER ROLE ${q(f.user)} WITH REPLICATION;` });
92
+ const free = f.maxSlots - f.slotsUsed;
93
+ if (slotState) {
94
+ out.push(slotState.walStatus === "lost"
95
+ ? { name: "slot", ok: true, warn: true, detail: `${slot} exists and Postgres invalidated it (wal_status lost): Mason drops it and rebuilds from scratch at its start` }
96
+ : slotState.active
97
+ ? { name: "slot", ok: false, detail: `${slot} is in use by another reader${slotState.activePid !== null ? ` (pid ${slotState.activePid} on the database)` : ""}: one slot has one reader. ` +
98
+ `Stop the other Mason for this project (mason stop). A run that crashed holds it until its connection ends${slotState.activePid !== null ? `: SELECT pg_terminate_backend(${slotState.activePid}) ends it now` : ""}` }
99
+ : { name: "slot", ok: true, detail: `${slot} exists (${slotState.plugin ?? "?"}, ${slotState.walStatus ?? "?"}): Mason resumes from it` });
100
+ }
101
+ else {
102
+ out.push(free >= 1
103
+ ? { name: "slot", ok: true, detail: `${slot} is made at Mason's first start (${f.slotsUsed} of max_replication_slots ${f.maxSlots} in use)` }
104
+ : { name: "slot", ok: false, detail: `every replication slot is taken (${f.slotsUsed} of max_replication_slots ${f.maxSlots}): raise max_replication_slots (a restart), or drop a slot nobody reads` });
105
+ }
106
+ out.push(f.maxSenders - f.sendersUsed >= 1
107
+ ? { name: "wal senders", ok: true, detail: `${f.sendersUsed} of max_wal_senders ${f.maxSenders} in use` }
108
+ : { name: "wal senders", ok: false, detail: `every WAL sender is in use (${f.sendersUsed} of max_wal_senders ${f.maxSenders}): raise max_wal_senders (a restart)` });
109
+ const gb = suggestedWalCapGb(f.walBytesPerSecond);
110
+ const size = gb === null ? "a day of this database's WAL" : `${gb}GB`;
111
+ const basis = gb === null ? "" : ` (a day of the WAL this server writes: ${bytesText(f.walBytesPerSecond)}/s since its statistics were reset)`;
112
+ out.push(f.slotWalKeepMb >= 0
113
+ ? { name: "WAL cap", ok: true, detail: `max_slot_wal_keep_size = ${f.slotWalKeepMb >= 1024 ? `${(f.slotWalKeepMb / 1024).toFixed(f.slotWalKeepMb % 1024 ? 1 : 0)}GB` : `${f.slotWalKeepMb}MB`}: past it Postgres drops the slot rather than fill its disk, and Mason rebuilds from scratch${gb !== null && f.slotWalKeepMb < gb * 1024 ? `. A day of this server's WAL is about ${gb}GB${basis}: an outage longer than the cap's worth means a rebuild` : ""}` }
114
+ : { name: "WAL cap", ok: false,
115
+ detail: `max_slot_wal_keep_size is unlimited: while Mason is stopped its slot holds every byte of WAL, until the disk fills and Postgres stops taking writes. Cap it at the outage you want Mason to survive, e.g. ${size}${basis}` +
116
+ (f.rds || f.cloudSql ? ` — in the ${f.rds ? "DB parameter group" : "database flags"} (max_slot_wal_keep_size, in MB)` : ""),
117
+ ...(f.rds || f.cloudSql ? {} : { sql: `ALTER SYSTEM SET max_slot_wal_keep_size = '${gb ?? 4}GB';\nSELECT pg_reload_conf();` }) });
118
+ return out;
119
+ }
120
+ /** What polling costs this database: a scan of every replicated table at each poll, and the delete check's keys. */
@@ -0,0 +1,148 @@
1
+ /**
2
+ * `mason start --replication` (the journey to production, item 6): a project's run reading its database's
3
+ * change log as production does — a publication, a replication slot, `wal_level = logical` — instead of polling it with
4
+ * SELECTs (src/cdc/poll.ts). A developer then meets the setup production needs on their own database first. The mode is
5
+ * the project's (`.mason/config.json` `changes`), kept between starts until `--poll` takes it back.
6
+ *
7
+ * What it costs the developer's database, said where it happens: Mason creates its slot there (`mason_<deployment>`), and
8
+ * a slot holds WAL while nobody reads it — so `mason stop` says so, and `mason destroy`, or going back to polling, drops
9
+ * it (`dropOwnReplication`), verified gone, by the name the start that made it RECORDED (`madeReplication`), never one
10
+ * computed again. The PUBLICATION is Mason's own (`mason_<deployment>`, made and dropped with
11
+ * the slot) unless the operator names one (`--publication <name>`, else `PG_PUBLICATION`, else the one
12
+ * an earlier start recorded): that one is the database owner's, read as it is and never created, altered or dropped
13
+ * (src/duck/publication.ts) — on a read replica it is the only way, since CREATE PUBLICATION is refused on a standby.
14
+ *
15
+ * In the ClickHouse service this was cut from, the two modes kept row versions that could not continue each other (a
16
+ * counter per table against WAL positions): a switch was a copy from scratch, which that service did by itself. The DuckDB
17
+ * prototype keeps its copies through a switch: each table is caught up by one poll from its watermark (src/duck/sync.ts
18
+ * `prepareReplication`), and `mason start` says so.
19
+ */
20
+ import { createHash } from "node:crypto";
21
+ import { replicationNames } from "../cdc/source.js";
22
+ import { connectSource, systemIdentifier } from "./preflight.js";
23
+ import { readServerFacts, readSlotFacts, replicationChecks } from "./replication-checks.js";
24
+ /** Postgres's longest identifier (NAMEDATALEN - 1): a longer slot name is refused, a longer publication's cut with a notice. */
25
+ export const PG_NAME_BYTES = 63;
26
+ /**
27
+ * THE NAME A SOURCE'S SLOT AND PUBLICATION ARE MADE FROM, `dev_<project>_<source>` (src/cdc/source.ts replicationNames
28
+ * puts `mason_` before it): one per project AND source, the source the sources file's name, since a project will sync
29
+ * several databases, each with a slot of its own. Lower case, `_` for any other character, as a slot's name must be. A
30
+ * project (48 at most) and a source (48) can pass Postgres's 63 bytes: then a readable prefix and `_` and 8 hex
31
+ * characters of the whole name's sha256, so two names sharing a long prefix never meet and none is ever cut silently.
32
+ */
33
+ export function deploymentOf(project, source) {
34
+ const full = `dev_${project}_${source}`.toLowerCase().replace(/[^a-z0-9_]/g, "_");
35
+ const room = PG_NAME_BYTES - "mason_".length;
36
+ if (full.length <= room)
37
+ return full;
38
+ return `${full.slice(0, room - 9)}_${createHash("sha256").update(full).digest("hex").slice(0, 8)}`;
39
+ }
40
+ /**
41
+ * The publication and slot a project's run reads: the publication `named` (the flag, `PG_PUBLICATION` or the recorded
42
+ * one: `namedPublication`), else Mason's own — by the change reader's own rule (src/cdc/source.ts `replicationNames`),
43
+ * which also says whether it is Mason's to make. The slot is always the project's own.
44
+ */
45
+ export function projectReplication(name, source, named = null) {
46
+ const { publication, slot, owned } = replicationNames(named ? { PG_PUBLICATION: named } : {}, deploymentOf(name, source));
47
+ return { publication, slot, owned };
48
+ }
49
+ /**
50
+ * WHAT REPLICATION MODE MADE ON THE DATABASE, as the start that made it recorded it (`config.json` `replication`): what
51
+ * `--poll` and `mason destroy` drop. Recorded, never computed again, so a name that changes since — the sources file
52
+ * renamed, the naming rule — never leaves a slot that nobody drops, holding the database's WAL for ever. A project with
53
+ * no record (none started in replication mode since it was kept) has the names the rule gives now.
54
+ */
55
+ export function madeReplication(cfg, source) {
56
+ return cfg.replication ?? projectReplication(cfg.name, source, cfg.publication ?? null);
57
+ }
58
+ /** The publication an operator named, and where: `--publication`, else `PG_PUBLICATION`, else the one an earlier start
59
+ * of this copy recorded (`config.json` `publication`), else none (Mason's own). */
60
+ export function namedPublication(flag, env, recorded) {
61
+ if (flag)
62
+ return { name: flag, from: "flag" };
63
+ if (env.PG_PUBLICATION)
64
+ return { name: env.PG_PUBLICATION, from: "env" };
65
+ if (recorded)
66
+ return { name: recorded, from: "recorded" };
67
+ return null;
68
+ }
69
+ /** The mode a start runs: the flag, else the project's, else polling. */
70
+ export function changesOf(flag, recorded) {
71
+ return flag ?? recorded ?? "poll";
72
+ }
73
+ /**
74
+ * Whether the developer's database can be read by replication: what refuses it (wal_level, the role — REPLICATION, a
75
+ * superuser, or RDS's rds_replication membership, as `replicationChecks` reads them — a free slot and WAL sender, a
76
+ * standby before 16, and a standby with no publication named: Mason cannot make its own there), and what is only worth
77
+ * saying (the WAL cap), from the production checks' own rules.
78
+ */
79
+ export function readinessVerdict(facts, slot, slotFacts, repl) {
80
+ const checks = replicationChecks(facts, slot, slotFacts);
81
+ // a standby takes no CREATE PUBLICATION (it is read-only): the publication must be the primary's, named
82
+ if (facts.standby && repl.owned) {
83
+ checks.push({ name: "publication", ok: false, detail: `this database is a read replica, where Mason cannot make its publication ${repl.publication} (CREATE ` +
84
+ "PUBLICATION is refused on a standby). Create one on the primary for the tables you add, then name it: mason start --replication --publication <name>" });
85
+ }
86
+ // a laptop's database has no cap on its slots by default: worth saying, never a reason to refuse a dev run
87
+ return { refused: checks.filter((x) => x.ok === false && x.name !== "WAL cap"), warnings: checks.filter((x) => (x.name === "WAL cap" && x.ok === false) || x.warn === true) };
88
+ }
89
+ export async function replicationReadiness(source, repl) {
90
+ const c = await connectSource(source);
91
+ if ("failure" in c)
92
+ return { failure: c.failure.message };
93
+ try {
94
+ return readinessVerdict(await readServerFacts(c.pg), repl.slot, await readSlotFacts(c.pg, repl.slot), repl);
95
+ }
96
+ finally {
97
+ c.pg.close();
98
+ }
99
+ }
100
+ /** The statements that remove what replication mode made, for a person to run when Mason could not: the slot, and the
101
+ * publication only when it is Mason's own (a named one is its owner's). */
102
+ export function dropStatements(o) {
103
+ return `SELECT pg_drop_replication_slot('${o.slot}') FROM pg_replication_slots WHERE slot_name = '${o.slot}' AND NOT active;` +
104
+ (o.owned ? ` DROP PUBLICATION IF EXISTS ${o.publication};` : "");
105
+ }
106
+ /**
107
+ * Drop what replication mode made on the developer's database — the slot, and the publication when it is Mason's own,
108
+ * never one the operator named — and verify it is gone: what was dropped, in words. A slot a reader still holds is never
109
+ * dropped (the run must be stopped first): that is a failure.
110
+ */
111
+ export async function dropOwnReplication(source, o, from = null) {
112
+ const c = await connectSource(source);
113
+ if ("failure" in c)
114
+ throw new Error(`could not connect to drop them: ${c.failure.message}`);
115
+ const pg = c.pg;
116
+ const lit = (s) => `'${s.replace(/'/g, "''")}'`;
117
+ try {
118
+ // the server the copy came from (its system identifier, recorded at the copy): another one has none of its slot
119
+ if (from?.systemIdentifier) {
120
+ const now = await systemIdentifier(pg);
121
+ if (now !== null && now !== from.systemIdentifier) {
122
+ throw new Error(`${source.database} on ${source.host} is another server than the copy came from (system identifier ${now}, the copy's ${from.systemIdentifier}), where the slot is: put that server's URL back in the sources file and run mason destroy again, or drop them there by hand`);
123
+ }
124
+ }
125
+ const dropped = [];
126
+ const slot = await readSlotFacts(pg, o.slot);
127
+ if (slot) {
128
+ if (slot.active)
129
+ throw new Error(`replication slot ${o.slot} is in use by a reader: stop it first`);
130
+ await pg.query(`SELECT pg_drop_replication_slot(${lit(o.slot)})`);
131
+ dropped.push(`replication slot ${o.slot}`);
132
+ }
133
+ const hasPub = async () => (await pg.query(`SELECT 1 FROM pg_publication WHERE pubname = ${lit(o.publication)}`)).rows.length > 0;
134
+ if (o.owned && await hasPub()) {
135
+ await pg.query(`DROP PUBLICATION IF EXISTS "${o.publication.replace(/"/g, '""')}"`);
136
+ dropped.push(`publication ${o.publication}`);
137
+ }
138
+ // verified, never assumed: a slot left behind holds the database's WAL for ever
139
+ if (await readSlotFacts(pg, o.slot))
140
+ throw new Error(`replication slot ${o.slot} is still there after pg_drop_replication_slot`);
141
+ if (o.owned && await hasPub())
142
+ throw new Error(`publication ${o.publication} is still there after DROP PUBLICATION`);
143
+ return dropped;
144
+ }
145
+ finally {
146
+ pg.close();
147
+ }
148
+ }