@mason-data/cli 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +268 -2
- package/dist/src/api/error.js +18 -0
- package/dist/src/api/postgres/catalog-worker.js +92 -0
- package/dist/src/api/postgres/catalog.js +477 -0
- package/dist/src/api/postgres/encode.js +541 -0
- package/dist/src/api/postgres/explain.js +143 -0
- package/dist/src/api/postgres/listener.js +60 -0
- package/dist/src/api/postgres/parser-worker.js +58 -0
- package/dist/src/api/postgres/parser.js +121 -0
- package/dist/src/api/postgres/plain.js +33 -0
- package/dist/src/api/postgres/protocol.js +404 -0
- package/dist/src/api/postgres/schema.js +54 -0
- package/dist/src/api/postgres/server.js +1771 -0
- package/dist/src/api/postgres/translate.js +248 -0
- package/dist/src/api/request-log.js +50 -0
- package/dist/src/cdc/pg-wire.js +588 -0
- package/dist/src/cdc/pgoutput.js +107 -0
- package/dist/src/cdc/poll.js +67 -0
- package/dist/src/cdc/source.js +65 -0
- package/dist/src/cli/background.js +198 -0
- package/dist/src/cli/command.js +173 -0
- package/dist/src/cli/commands.js +1277 -0
- package/dist/src/cli/control.js +36 -0
- package/dist/src/cli/demo.js +149 -0
- package/dist/src/cli/dev.js +669 -0
- package/dist/src/cli/doctor.js +260 -0
- package/dist/src/cli/guides.js +158 -0
- package/dist/src/cli/init.js +366 -0
- package/dist/src/cli/machine.js +162 -0
- package/dist/src/cli/main.js +83 -0
- package/dist/src/cli/mcp.js +726 -0
- package/dist/src/cli/pg-tls.js +100 -0
- package/dist/src/cli/pg-url.js +52 -0
- package/dist/src/cli/pin.js +36 -0
- package/dist/src/cli/ports.js +118 -0
- package/dist/src/cli/preflight.js +347 -0
- package/dist/src/cli/project.js +129 -0
- package/dist/src/cli/readiness.js +76 -0
- package/dist/src/cli/registry.js +220 -0
- package/dist/src/cli/replication-checks.js +120 -0
- package/dist/src/cli/replication-mode.js +148 -0
- package/dist/src/cli/run-server.js +311 -0
- package/dist/src/cli/runs.js +120 -0
- package/dist/src/cli/security.js +86 -0
- package/dist/src/cli/self-signed-cert.js +64 -0
- package/dist/src/cli/serve.js +301 -0
- package/dist/src/cli/source-edit.js +116 -0
- package/dist/src/cli/sources.js +564 -0
- package/dist/src/cli/state.js +58 -0
- package/dist/src/cli/style.js +187 -0
- package/dist/src/cli/supervise.js +248 -0
- package/dist/src/cli/sync.js +94 -0
- package/dist/src/cli/ui.js +107 -0
- package/dist/src/cli/user.js +229 -0
- package/dist/src/cli/version.js +23 -0
- package/dist/src/demo/seed.js +237 -0
- package/dist/src/demo/source.js +655 -0
- package/dist/src/duck/analyze.js +113 -0
- package/dist/src/duck/cancel.js +49 -0
- package/dist/src/duck/distinct-key.js +228 -0
- package/dist/src/duck/executor-worker.js +268 -0
- package/dist/src/duck/executor.js +567 -0
- package/dist/src/duck/explain.js +238 -0
- package/dist/src/duck/functions.js +819 -0
- package/dist/src/duck/gaps.js +94 -0
- package/dist/src/duck/jsonb-order.js +175 -0
- package/dist/src/duck/jsonb.js +188 -0
- package/dist/src/duck/kept-route.js +299 -0
- package/dist/src/duck/kept.js +964 -0
- package/dist/src/duck/late.js +221 -0
- package/dist/src/duck/libc-order.js +190 -0
- package/dist/src/duck/pg-eval-worker.js +34 -0
- package/dist/src/duck/pg-eval.js +59 -0
- package/dist/src/duck/pgtree.js +378 -0
- package/dist/src/duck/plan.js +259 -0
- package/dist/src/duck/publication.js +207 -0
- package/dist/src/duck/requests.js +209 -0
- package/dist/src/duck/resolved.js +248 -0
- package/dist/src/duck/rewrite.js +5101 -0
- package/dist/src/duck/store.js +907 -0
- package/dist/src/duck/stream.js +518 -0
- package/dist/src/duck/sync.js +2929 -0
- package/dist/src/duck/to-char.js +482 -0
- package/dist/src/duck/token.js +31 -0
- package/dist/src/duck/types.js +427 -0
- package/dist/src/duck/users.js +231 -0
- package/dist/src/duck/verified.js +1001 -0
- package/dist/src/duck/views.js +150 -0
- package/dist/src/lib/database-url.js +74 -0
- package/dist/src/lib/did-you-mean.js +43 -0
- package/dist/src/lib/dotenv.js +72 -0
- package/dist/src/lib/exit-when-written.js +20 -0
- package/dist/src/lib/float-equal.js +69 -0
- package/dist/src/lib/http-json.js +22 -0
- package/dist/src/lib/log.js +161 -0
- package/dist/src/lib/pg-numeric.js +489 -0
- package/dist/src/lib/port.js +22 -0
- package/dist/src/lib/project-file.js +49 -0
- package/dist/src/lib/project-files.js +43 -0
- package/dist/src/lib/project-version.js +76 -0
- package/dist/src/lib/runnable.js +14 -0
- package/dist/src/lib/shell-word.js +4 -0
- package/dist/src/lib/timezone.js +237 -0
- package/dist/src/lib/zone-names.js +114 -0
- package/dist/src/web/app.css +76 -0
- package/dist/src/web/app.js +491 -0
- package/dist/src/web/index.html +44 -0
- package/package.json +22 -4
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* EVERY `mason` COMMAND, AND THE HELP MADE FROM THEM. `mason --help` is generated from this list — each
|
|
3
|
+
* command's summary, in this order — so a command cannot exist without being listed, and tests/cli-help.test.ts holds
|
|
4
|
+
* each one to src/cli/command.ts's rules (its own --help, an example, every option documented).
|
|
5
|
+
*/
|
|
6
|
+
import { failed, parseCommand, reportText } from "./command.js";
|
|
7
|
+
import { dev } from "./dev.js";
|
|
8
|
+
import { devInBackground } from "./background.js";
|
|
9
|
+
import { ADD, LOGS, REMOVE, STATUS, STOP, TABLES, WAIT, add, logs, remove, status, stop, tables, wait } from "./commands.js";
|
|
10
|
+
import { USER, user } from "./user.js";
|
|
11
|
+
import { MCP, mcp } from "./mcp.js";
|
|
12
|
+
import { UI, ui } from "./ui.js";
|
|
13
|
+
import { GUIDES, guideText } from "./guides.js";
|
|
14
|
+
import { DOCTOR, NODE_MIN, doctor } from "./doctor.js";
|
|
15
|
+
import { DESTROY, destroy } from "./machine.js";
|
|
16
|
+
import { INIT, init } from "./init.js";
|
|
17
|
+
import { SYNC, sync } from "./sync.js";
|
|
18
|
+
import { SERVE, serve } from "./serve.js";
|
|
19
|
+
import { PROJECT_LINE } from "./dev.js";
|
|
20
|
+
import { version } from "./version.js";
|
|
21
|
+
import { didYouMean } from "../lib/did-you-mean.js";
|
|
22
|
+
import { out as styleOut, wrap } from "./style.js";
|
|
23
|
+
import { theProject } from "./project.js";
|
|
24
|
+
import { liveRun } from "./runs.js";
|
|
25
|
+
export { version } from "./version.js";
|
|
26
|
+
export const DEV = {
|
|
27
|
+
needs: "project",
|
|
28
|
+
name: "start",
|
|
29
|
+
summary: "start Mason for this folder's project, in the background (mason init makes the project)",
|
|
30
|
+
usage: `usage: mason start [--attach] [--replication [--publication <name>] | --poll] [--self-signed-tls] [--port <n>] [--json]
|
|
31
|
+
|
|
32
|
+
Starts Mason for the project of this folder (mason init made it): one process, which copies the tables you add
|
|
33
|
+
(mason add) from the project's database into its own file, keeps them current, and answers the Postgres protocol
|
|
34
|
+
for BI tools and your app. It only reads the database (SELECT): no wal_level, replication slot or publication is
|
|
35
|
+
needed or made. It returns once Mason is up; run it again any time (it says so when Mason is already running). The
|
|
36
|
+
copy and Mason's state stay between runs: the next start resumes.
|
|
37
|
+
|
|
38
|
+
With --replication it reads the database's change log instead: it needs wal_level = logical and a user that may
|
|
39
|
+
replicate, and makes its own publication and replication slot on the database (mason destroy, or mason start --poll,
|
|
40
|
+
drops them). The project keeps the mode until --poll; a switch keeps the copies and catches each table up by one
|
|
41
|
+
poll. --publication <name> (else PG_PUBLICATION) names a publication that is the database owner's: it is read as it
|
|
42
|
+
is, verified to carry every added table and column, and never created, altered or dropped — Mason makes only its
|
|
43
|
+
slot. A read replica needs one (it takes no CREATE PUBLICATION). The name is kept with the project, so the next
|
|
44
|
+
mason start --replication reads it again.
|
|
45
|
+
|
|
46
|
+
The Postgres API's port is the project's: the first start takes 5433, or the next port free on every address when a
|
|
47
|
+
second Postgres holds it, and records it; every later start listens on the same port, so a BI tool's saved connection
|
|
48
|
+
keeps working, and refuses, naming what holds it, rather than move. MASON_PG_PORT=<n> in your shell names the port,
|
|
49
|
+
or moves the project to it (said): that port is used, or start refuses, naming what holds it.
|
|
50
|
+
|
|
51
|
+
The Postgres API listens on 127.0.0.1 unless MASON_PG_HOST names another address (0.0.0.0: every address of this
|
|
52
|
+
machine). Off this machine a password travels on the connection, so it then needs TLS: MASON_PG_TLS_CERT and
|
|
53
|
+
MASON_PG_TLS_KEY (PEM files), or --self-signed-tls, which makes a certificate for the project in .mason/ and serves
|
|
54
|
+
it again at every later start with the flag. Every client then connects with sslmode=require. A team whose own proxy
|
|
55
|
+
or load balancer terminates TLS keeps it on 127.0.0.1, or sets MASON_PG_PLAIN=1 for a private network.
|
|
56
|
+
|
|
57
|
+
${PROJECT_LINE}
|
|
58
|
+
|
|
59
|
+
mason start
|
|
60
|
+
mason start --attach in the foreground: Ctrl-C stops it
|
|
61
|
+
mason start --replication read the change log, as production does
|
|
62
|
+
mason start --replication --publication app_pub through the database owner's publication
|
|
63
|
+
MASON_PG_HOST=0.0.0.0 mason start --self-signed-tls the Postgres API for other machines, over TLS
|
|
64
|
+
|
|
65
|
+
--attach stay in the foreground (Ctrl-C stops Mason)
|
|
66
|
+
--replication read the database's changes from its change log (logical replication), as production does
|
|
67
|
+
--publication <name> with --replication: the publication to read, the owner's, never altered (PG_PUBLICATION;
|
|
68
|
+
default: Mason's own, mason_dev_<project>_<source>)
|
|
69
|
+
--poll read them with SELECTs (the default; takes back --replication)
|
|
70
|
+
--self-signed-tls the Postgres API requires TLS, from a certificate made for this project in .mason/ (kept and
|
|
71
|
+
served again by every later start with the flag); not with MASON_PG_TLS_CERT / _KEY
|
|
72
|
+
--port <n> the port the mason commands reach it on (default 4039, or the next free one)
|
|
73
|
+
--json as data, once it is up: { running, pid, postgres: { url, host, port, database, sslmode }, next }
|
|
74
|
+
`,
|
|
75
|
+
options: {
|
|
76
|
+
port: { type: "string" }, attach: { type: "boolean", default: false }, json: { type: "boolean", default: false },
|
|
77
|
+
replication: { type: "boolean", default: false }, poll: { type: "boolean", default: false }, publication: { type: "string" },
|
|
78
|
+
"self-signed-tls": { type: "boolean", default: false },
|
|
79
|
+
},
|
|
80
|
+
json: ["running", "pid", "postgres", "next"],
|
|
81
|
+
};
|
|
82
|
+
async function runDev(argv) {
|
|
83
|
+
const f = parseCommand(DEV, argv);
|
|
84
|
+
if (typeof f === "number")
|
|
85
|
+
return f;
|
|
86
|
+
const v = f.values;
|
|
87
|
+
if (v.replication && v.poll)
|
|
88
|
+
return failed("start", { message: "--replication or --poll, not both", code: "usage", usage: DEV.usage }, { json: v.json });
|
|
89
|
+
if (v.publication !== undefined && v.poll)
|
|
90
|
+
return failed("start", { message: "--publication goes with --replication, not --poll", code: "usage", usage: DEV.usage }, { json: v.json });
|
|
91
|
+
if (v.publication !== undefined && !/^\S+$/.test(v.publication))
|
|
92
|
+
return failed("start", { message: `--publication must name a publication, not ${JSON.stringify(v.publication)}`, code: "usage", usage: DEV.usage }, { json: v.json });
|
|
93
|
+
if (f.positionals.length) {
|
|
94
|
+
// the database is the project's now: a URL here is the old habit, answered with the new way
|
|
95
|
+
return failed("start", { message: "it takes no database URL: the project has one", code: "usage", why: "mason init <url> made it; .mason/config.json holds it", usage: DEV.usage }, { json: v.json });
|
|
96
|
+
}
|
|
97
|
+
const port = v.port === undefined ? 4039 : Number(v.port);
|
|
98
|
+
if (!Number.isInteger(port) || port < 1 || port > 65_535)
|
|
99
|
+
return failed("start", { message: `--port must be a port number (1-65535), not ${JSON.stringify(v.port)}`, code: "usage" }, { json: v.json });
|
|
100
|
+
if (!v.attach)
|
|
101
|
+
return devInBackground(argv, { json: v.json, selfSignedTls: v["self-signed-tls"], changes: v.replication ? "replication" : v.poll ? "poll" : null });
|
|
102
|
+
return dev({ port, version: version(), changes: v.replication ? "replication" : v.poll ? "poll" : null, publication: v.publication ?? null, selfSignedTls: v["self-signed-tls"] });
|
|
103
|
+
}
|
|
104
|
+
export const HELP = {
|
|
105
|
+
name: "help",
|
|
106
|
+
summary: "the commands by step, or one command's usage: mason help add",
|
|
107
|
+
usage: `usage: mason help [<command> | <guide>]
|
|
108
|
+
|
|
109
|
+
The commands by step (mason help), how to use one (mason help add — the same as mason add --help), or a guide,
|
|
110
|
+
offline, short enough to read whole and listing exactly what Mason accepts:
|
|
111
|
+
|
|
112
|
+
mason help
|
|
113
|
+
mason help add
|
|
114
|
+
${Object.entries(GUIDES).map(([k, v]) => ` mason help ${k.padEnd(14)} ${v}`).join("\n")}
|
|
115
|
+
`,
|
|
116
|
+
options: {},
|
|
117
|
+
};
|
|
118
|
+
async function runHelp(argv) {
|
|
119
|
+
const f = parseCommand(HELP, argv);
|
|
120
|
+
if (typeof f === "number")
|
|
121
|
+
return f;
|
|
122
|
+
const [asked, ...rest] = f.positionals;
|
|
123
|
+
if (rest.length)
|
|
124
|
+
return failed("help", { message: "one command or guide at a time", code: "usage", usage: HELP.usage }, { json: false });
|
|
125
|
+
if (asked === undefined) {
|
|
126
|
+
process.stdout.write(topUsage());
|
|
127
|
+
return 0;
|
|
128
|
+
}
|
|
129
|
+
if (asked === "version") {
|
|
130
|
+
process.stdout.write("usage: mason version (or --version, -v): the CLI's version\n");
|
|
131
|
+
return 0;
|
|
132
|
+
}
|
|
133
|
+
if (Object.hasOwn(GUIDES, asked)) {
|
|
134
|
+
process.stdout.write(await guideText(asked));
|
|
135
|
+
return 0;
|
|
136
|
+
}
|
|
137
|
+
const cmd = COMMANDS.find((c) => c.name === asked);
|
|
138
|
+
if (cmd) {
|
|
139
|
+
process.stdout.write(cmd.usage);
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
return failed("help", { message: `there is no command or guide ${JSON.stringify(asked)}${didYouMean(asked, [...COMMANDS.map((c) => c.name), ...Object.keys(GUIDES)])}`, code: "usage",
|
|
143
|
+
why: `the guides: ${Object.keys(GUIDES).join(", ")}`, next: [["mason help", "every command"]] }, { json: false });
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* EVERY COMMAND, IN THE ORDER A FIRST SESSION NEEDS THEM (listed in the order a developer will likely need them, the
|
|
147
|
+
* reference, `mason help`, first because it is read first), under the step each belongs to: `mason --help` prints the
|
|
148
|
+
* steps as headings, so the list reads as the runbook it is. A new command goes into its step here; COMMANDS is
|
|
149
|
+
* derived, so the order is written once.
|
|
150
|
+
*/
|
|
151
|
+
export const STEPS = [
|
|
152
|
+
["Learn", [{ ...HELP, run: runHelp }]],
|
|
153
|
+
["Set up", [{ ...DOCTOR, run: (argv) => doctor(argv) }, { ...INIT, run: (argv) => init(argv) }, { ...DEV, run: runDev }]],
|
|
154
|
+
["Add your tables", [{ ...TABLES, run: tables }, { ...ADD, run: add }, { ...REMOVE, run: remove }, { ...WAIT, run: wait }]],
|
|
155
|
+
["Connect", [{ ...USER, run: user }, { ...MCP, run: mcp }]],
|
|
156
|
+
["Run", [{ ...STATUS, run: status }, { ...UI, run: ui }, { ...SYNC, run: sync }, { ...LOGS, run: logs }, { ...STOP, run: stop }, { ...DESTROY, run: (argv) => destroy(argv) }]],
|
|
157
|
+
["Go to production", [{ ...SERVE, run: serve }]],
|
|
158
|
+
];
|
|
159
|
+
/** Every command, in the order `mason --help` lists them. */
|
|
160
|
+
export const COMMANDS = STEPS.flatMap(([, commands]) => commands);
|
|
161
|
+
/**
|
|
162
|
+
* `mason` ALONE: where you are and what to run next (`mason help` once showed the same as `mason` alone, and read as
|
|
163
|
+
* broken). One line on what Mason is, whether this folder is a project and whether Mason runs for
|
|
164
|
+
* it, then the next command for that moment — and mason help, every command. `mason help` and `mason --help` are the
|
|
165
|
+
* whole list (topUsage).
|
|
166
|
+
*/
|
|
167
|
+
export async function orientation(cwd, s = styleOut()) {
|
|
168
|
+
const t = theProject(cwd);
|
|
169
|
+
const project = "project" in t ? t.project : null;
|
|
170
|
+
const run = project ? await liveRun(project) : null;
|
|
171
|
+
const rows = project
|
|
172
|
+
? [["Project", `${project.config.name}, files in ${project.folder}`], ["Mason", run ? `running (pid ${run.pid})` : "not running"]]
|
|
173
|
+
: [["Project", "none here: mason init makes this folder one"]];
|
|
174
|
+
const next = !project
|
|
175
|
+
? [["mason init", "make this folder a project (DATABASE_URL from .env, or mason init postgres://…)"],
|
|
176
|
+
["mason init demo", "or try it on a demo database Mason makes itself: nothing to install"],
|
|
177
|
+
["mason doctor", "what this machine needs for Mason, and whether it has it"]]
|
|
178
|
+
: !run ? [["mason start", "start Mason for this project, in the background"], ["mason status", "the project and its tables"]]
|
|
179
|
+
: [["mason status", "what Mason is doing: its tables, and the last day's slow and refused statements"], ["mason user add <name>", "a user for a BI tool or your app: its password and URL, printed once"]];
|
|
180
|
+
return `${reportText({ headline: "Mason: your Postgres tables, copied and served to BI tools and your app.", details: rows,
|
|
181
|
+
next: [...next, ["mason help", "every command, and the guides"]] }, s)}\n`;
|
|
182
|
+
}
|
|
183
|
+
/** `mason --help`: what Mason is, every command with its summary, and where each one's help is. On a terminal the steps are
|
|
184
|
+
* bold, the commands cyan and every line wrapped at its width under its own column (src/cli/style.ts); anywhere else,
|
|
185
|
+
* plain and unwrapped. */
|
|
186
|
+
export function topUsage(s = styleOut()) {
|
|
187
|
+
// what each step lists: its commands, and under Learn the guides too (`mason help` shows the guides listed under
|
|
188
|
+
// Learn), so the pages a session reads first are a line each, not a paragraph after the list
|
|
189
|
+
const entries = (step, commands) => [
|
|
190
|
+
...commands.map((c) => [c.name, c.summary]),
|
|
191
|
+
...(step === "Learn" ? Object.entries(GUIDES).map(([g, what]) => [`help ${g}`, what]) : []),
|
|
192
|
+
];
|
|
193
|
+
const listed = STEPS.map(([step, commands]) => [step, entries(step, commands)]);
|
|
194
|
+
// the commands' column is the longest COMMAND's, everywhere; a guide's longer name widens Learn alone, so it never
|
|
195
|
+
// pushes every other step's summaries right
|
|
196
|
+
const commandWidth = Math.max(...COMMANDS.map((c) => c.name.length));
|
|
197
|
+
const para = (text) => ` ${wrap(text, s.width, 2)}`;
|
|
198
|
+
const block = (es) => {
|
|
199
|
+
const w = Math.max(commandWidth, ...es.map(([name]) => name.length));
|
|
200
|
+
// the column a summary starts in: " mason " + the longest name + 3
|
|
201
|
+
const col = 4 + "mason ".length + w + 3;
|
|
202
|
+
return es.map(([name, summary]) => ` ${s.cyan(`mason ${name.padEnd(w)}`)} ${s.dim(wrap(summary, s.width, col))}`).join("\n");
|
|
203
|
+
};
|
|
204
|
+
return [
|
|
205
|
+
`${s.bold("usage:")} mason <command> [options]`,
|
|
206
|
+
"",
|
|
207
|
+
para("Mason: your Postgres tables, copied and served to BI tools and your app. Make your app's folder a project (mason init), " +
|
|
208
|
+
`add tables (mason add, before or after the start), start Mason (mason start), connect a tool (mason user add). It needs Node ${NODE_MIN} or later; mason doctor checks.`),
|
|
209
|
+
para(PROJECT_LINE),
|
|
210
|
+
"",
|
|
211
|
+
listed.map(([step, es]) => ` ${s.bold(step)}\n${block(es)}`).join("\n\n"),
|
|
212
|
+
"",
|
|
213
|
+
para("mason <command> --help (or mason help <command>) says how to use one, with examples; the guides are offline and short enough to read whole."),
|
|
214
|
+
para("A command means the project of the folder it runs in (or above it)."),
|
|
215
|
+
para("Every command but help and mcp (whose stdout is the MCP protocol's) takes --json: one JSON object for a program, always with next (what to run " +
|
|
216
|
+
"next); a failure is { error: { code, message, why, fix }, next }. The exit code says how it went: 0 done, 1 failed " +
|
|
217
|
+
"with the reason, 2 a wrong command line, 124 mason wait timed out. Driving Mason from a coding agent: mason help agent."),
|
|
218
|
+
"",
|
|
219
|
+
].join("\n");
|
|
220
|
+
}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/** Everything the checks read of the server, in one statement: settings and roles only. */
|
|
2
|
+
export const SERVER_FACTS_SQL = `SELECT json_build_object(
|
|
3
|
+
'version', current_setting('server_version_num')::int, 'version_text', current_setting('server_version'),
|
|
4
|
+
'standby', pg_is_in_recovery(), 'user', current_user,
|
|
5
|
+
'wal_level', current_setting('wal_level'),
|
|
6
|
+
'slot_wal_keep_mb', (SELECT CASE WHEN setting = '-1' THEN -1 ELSE setting::bigint END FROM pg_settings WHERE name = 'max_slot_wal_keep_size'),
|
|
7
|
+
'max_slots', current_setting('max_replication_slots')::int, 'slots_used', (SELECT count(*) FROM pg_replication_slots),
|
|
8
|
+
'max_senders', current_setting('max_wal_senders')::int, 'senders_used', (SELECT count(*) FROM pg_stat_replication),
|
|
9
|
+
'superuser', r.rolsuper, 'replication', r.rolreplication,
|
|
10
|
+
'rds', CASE WHEN EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rds_replication')
|
|
11
|
+
THEN json_build_object('member', pg_has_role(current_user, 'rds_replication', 'member')) END,
|
|
12
|
+
'cloud_sql', current_setting('cloudsql.logical_decoding', true) IS NOT NULL,
|
|
13
|
+
'hot_standby_feedback', current_setting('hot_standby_feedback') = 'on',
|
|
14
|
+
'idle_in_transaction_ms', (SELECT setting::bigint FROM pg_settings WHERE name = 'idle_in_transaction_session_timeout'),
|
|
15
|
+
'statement_timeout_ms', (SELECT setting::bigint FROM pg_settings WHERE name = 'statement_timeout')
|
|
16
|
+
)::text FROM pg_roles r WHERE r.rolname = current_user`;
|
|
17
|
+
/** pg_stat_wal (Postgres 14+): the WAL written since the statistics were reset, and over how long. */
|
|
18
|
+
export const WAL_RATE_SQL = "SELECT wal_bytes::text, extract(epoch FROM now() - stats_reset)::text FROM pg_stat_wal";
|
|
19
|
+
export async function readServerFacts(pg) {
|
|
20
|
+
const j = JSON.parse((await pg.query(SERVER_FACTS_SQL)).rows[0]?.[0] ?? "null");
|
|
21
|
+
if (j === null)
|
|
22
|
+
throw new Error("the server's settings did not read (no pg_roles row for this user)");
|
|
23
|
+
let walBytesPerSecond = null;
|
|
24
|
+
if (Number(j.version) >= 140000 && j.standby !== true) {
|
|
25
|
+
const r = (await pg.query(WAL_RATE_SQL)).rows[0];
|
|
26
|
+
const bytes = Number(r?.[0]), seconds = Number(r?.[1]);
|
|
27
|
+
// a reset moments ago says nothing about the rate: an hour of statistics at least
|
|
28
|
+
if (Number.isFinite(bytes) && Number.isFinite(seconds) && seconds >= 3600)
|
|
29
|
+
walBytesPerSecond = bytes / seconds;
|
|
30
|
+
}
|
|
31
|
+
return {
|
|
32
|
+
version: Number(j.version), versionText: String(j.version_text), standby: j.standby === true, user: String(j.user),
|
|
33
|
+
walLevel: String(j.wal_level), slotWalKeepMb: Number(j.slot_wal_keep_mb),
|
|
34
|
+
maxSlots: Number(j.max_slots), slotsUsed: Number(j.slots_used), maxSenders: Number(j.max_senders), sendersUsed: Number(j.senders_used),
|
|
35
|
+
superuser: j.superuser === true, replication: j.replication === true,
|
|
36
|
+
rds: j.rds === null || j.rds === undefined ? null : { member: j.rds.member === true },
|
|
37
|
+
cloudSql: j.cloud_sql === true, hotStandbyFeedback: j.hot_standby_feedback === true,
|
|
38
|
+
idleInTransactionMs: Number(j.idle_in_transaction_ms), statementTimeoutMs: Number(j.statement_timeout_ms), walBytesPerSecond,
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
/** The slot Mason would read, if it exists: whether a reader holds it and whether Postgres invalidated it. */
|
|
42
|
+
export async function readSlotFacts(pg, slot) {
|
|
43
|
+
const r = (await pg.query(`SELECT active::text, active_pid::text, wal_status, plugin FROM pg_replication_slots WHERE slot_name = '${slot.replace(/'/g, "''")}'`)).rows[0];
|
|
44
|
+
return r ? { active: r[0] === "true", activePid: r[1] ? Number(r[1]) : null, walStatus: r[2] ?? null, plugin: r[3] ?? null } : null;
|
|
45
|
+
}
|
|
46
|
+
/** The way out of a slot another reader holds, in the words of the command that met it. */
|
|
47
|
+
/** Bytes as a person reads them: 1.4 GiB, 312 MiB, 18 KiB. */
|
|
48
|
+
export function bytesText(n) {
|
|
49
|
+
const units = ["bytes", "KiB", "MiB", "GiB", "TiB"];
|
|
50
|
+
let i = 0, v = n;
|
|
51
|
+
while (v >= 1024 && i < units.length - 1) {
|
|
52
|
+
v /= 1024;
|
|
53
|
+
i++;
|
|
54
|
+
}
|
|
55
|
+
return i === 0 ? `${n} bytes` : `${v < 10 ? v.toFixed(1) : Math.round(v)} ${units[i]}`;
|
|
56
|
+
}
|
|
57
|
+
const q = (s) => /^[a-z_][a-z0-9_]*$/.test(s) ? s : `"${s.replace(/"/g, '""')}"`;
|
|
58
|
+
/**
|
|
59
|
+
* A `max_slot_wal_keep_size` to suggest: a day of this server's WAL at its measured rate, rounded up to a whole GB, at
|
|
60
|
+
* least 4 GB; null when the rate is unknown (a standby, Postgres 13, statistics reset within the hour).
|
|
61
|
+
*/
|
|
62
|
+
export function suggestedWalCapGb(bytesPerSecond) {
|
|
63
|
+
if (bytesPerSecond === null)
|
|
64
|
+
return null;
|
|
65
|
+
return Math.max(4, Math.ceil((bytesPerSecond * 86_400) / 1024 ** 3));
|
|
66
|
+
}
|
|
67
|
+
/** What wal_level = logical takes on this server: its provider's way, or postgresql.conf's. */
|
|
68
|
+
function walLevelFix(f) {
|
|
69
|
+
if (f.rds)
|
|
70
|
+
return "set rds.logical_replication = 1 in the DB parameter group (Aurora: the DB cluster parameter group), then reboot the instance";
|
|
71
|
+
if (f.cloudSql)
|
|
72
|
+
return "set the database flag cloudsql.logical_decoding = on (the instance restarts)";
|
|
73
|
+
return "set wal_level = logical (ALTER SYSTEM SET wal_level = logical, or postgresql.conf; a managed Postgres: its parameter or flag of that name), then restart Postgres";
|
|
74
|
+
}
|
|
75
|
+
/** The checks of the server for the stream (REPLICATION). */
|
|
76
|
+
export function replicationChecks(f, slot, slotState) {
|
|
77
|
+
const out = [];
|
|
78
|
+
out.push(f.walLevel === "logical"
|
|
79
|
+
? { name: "wal_level", ok: true, detail: "logical" }
|
|
80
|
+
: { name: "wal_level", ok: false, detail: `${f.walLevel}: logical replication needs logical. To fix: ${walLevelFix(f)}. Or read changes by polling instead (--poll: SELECT alone, for a small database)` });
|
|
81
|
+
if (f.standby) {
|
|
82
|
+
out.push(f.version >= 160000
|
|
83
|
+
? { name: "standby", ok: true, warn: !f.hotStandbyFeedback,
|
|
84
|
+
detail: `a read replica (Postgres 16+ decodes on a standby)${f.hotStandbyFeedback ? "" : ". hot_standby_feedback is off: the primary may remove catalog rows the slot still needs, which invalidates it (Mason then rebuilds from scratch) — turn it on"}` }
|
|
85
|
+
: { name: "standby", ok: false, detail: `a read replica on Postgres ${f.versionText}: logical decoding on a standby needs Postgres 16. Point Mason at the primary, upgrade the replica, or poll it (--poll)` });
|
|
86
|
+
}
|
|
87
|
+
const canReplicate = f.superuser || f.replication || f.rds?.member === true;
|
|
88
|
+
out.push(canReplicate
|
|
89
|
+
? { name: "replication role", ok: true, detail: `${f.user} may replicate${f.superuser ? " (a superuser)" : f.rds?.member ? " (rds_replication)" : " (REPLICATION)"}` }
|
|
90
|
+
: { name: "replication role", ok: false, detail: `${f.user} may not open a replication connection`,
|
|
91
|
+
sql: f.rds ? `GRANT rds_replication TO ${q(f.user)};` : `ALTER ROLE ${q(f.user)} WITH REPLICATION;` });
|
|
92
|
+
const free = f.maxSlots - f.slotsUsed;
|
|
93
|
+
if (slotState) {
|
|
94
|
+
out.push(slotState.walStatus === "lost"
|
|
95
|
+
? { name: "slot", ok: true, warn: true, detail: `${slot} exists and Postgres invalidated it (wal_status lost): Mason drops it and rebuilds from scratch at its start` }
|
|
96
|
+
: slotState.active
|
|
97
|
+
? { name: "slot", ok: false, detail: `${slot} is in use by another reader${slotState.activePid !== null ? ` (pid ${slotState.activePid} on the database)` : ""}: one slot has one reader. ` +
|
|
98
|
+
`Stop the other Mason for this project (mason stop). A run that crashed holds it until its connection ends${slotState.activePid !== null ? `: SELECT pg_terminate_backend(${slotState.activePid}) ends it now` : ""}` }
|
|
99
|
+
: { name: "slot", ok: true, detail: `${slot} exists (${slotState.plugin ?? "?"}, ${slotState.walStatus ?? "?"}): Mason resumes from it` });
|
|
100
|
+
}
|
|
101
|
+
else {
|
|
102
|
+
out.push(free >= 1
|
|
103
|
+
? { name: "slot", ok: true, detail: `${slot} is made at Mason's first start (${f.slotsUsed} of max_replication_slots ${f.maxSlots} in use)` }
|
|
104
|
+
: { name: "slot", ok: false, detail: `every replication slot is taken (${f.slotsUsed} of max_replication_slots ${f.maxSlots}): raise max_replication_slots (a restart), or drop a slot nobody reads` });
|
|
105
|
+
}
|
|
106
|
+
out.push(f.maxSenders - f.sendersUsed >= 1
|
|
107
|
+
? { name: "wal senders", ok: true, detail: `${f.sendersUsed} of max_wal_senders ${f.maxSenders} in use` }
|
|
108
|
+
: { name: "wal senders", ok: false, detail: `every WAL sender is in use (${f.sendersUsed} of max_wal_senders ${f.maxSenders}): raise max_wal_senders (a restart)` });
|
|
109
|
+
const gb = suggestedWalCapGb(f.walBytesPerSecond);
|
|
110
|
+
const size = gb === null ? "a day of this database's WAL" : `${gb}GB`;
|
|
111
|
+
const basis = gb === null ? "" : ` (a day of the WAL this server writes: ${bytesText(f.walBytesPerSecond)}/s since its statistics were reset)`;
|
|
112
|
+
out.push(f.slotWalKeepMb >= 0
|
|
113
|
+
? { name: "WAL cap", ok: true, detail: `max_slot_wal_keep_size = ${f.slotWalKeepMb >= 1024 ? `${(f.slotWalKeepMb / 1024).toFixed(f.slotWalKeepMb % 1024 ? 1 : 0)}GB` : `${f.slotWalKeepMb}MB`}: past it Postgres drops the slot rather than fill its disk, and Mason rebuilds from scratch${gb !== null && f.slotWalKeepMb < gb * 1024 ? `. A day of this server's WAL is about ${gb}GB${basis}: an outage longer than the cap's worth means a rebuild` : ""}` }
|
|
114
|
+
: { name: "WAL cap", ok: false,
|
|
115
|
+
detail: `max_slot_wal_keep_size is unlimited: while Mason is stopped its slot holds every byte of WAL, until the disk fills and Postgres stops taking writes. Cap it at the outage you want Mason to survive, e.g. ${size}${basis}` +
|
|
116
|
+
(f.rds || f.cloudSql ? ` — in the ${f.rds ? "DB parameter group" : "database flags"} (max_slot_wal_keep_size, in MB)` : ""),
|
|
117
|
+
...(f.rds || f.cloudSql ? {} : { sql: `ALTER SYSTEM SET max_slot_wal_keep_size = '${gb ?? 4}GB';\nSELECT pg_reload_conf();` }) });
|
|
118
|
+
return out;
|
|
119
|
+
}
|
|
120
|
+
/** What polling costs this database: a scan of every replicated table at each poll, and the delete check's keys. */
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `mason start --replication` (the journey to production, item 6): a project's run reading its database's
|
|
3
|
+
* change log as production does — a publication, a replication slot, `wal_level = logical` — instead of polling it with
|
|
4
|
+
* SELECTs (src/cdc/poll.ts). A developer then meets the setup production needs on their own database first. The mode is
|
|
5
|
+
* the project's (`.mason/config.json` `changes`), kept between starts until `--poll` takes it back.
|
|
6
|
+
*
|
|
7
|
+
* What it costs the developer's database, said where it happens: Mason creates its slot there (`mason_<deployment>`), and
|
|
8
|
+
* a slot holds WAL while nobody reads it — so `mason stop` says so, and `mason destroy`, or going back to polling, drops
|
|
9
|
+
* it (`dropOwnReplication`), verified gone, by the name the start that made it RECORDED (`madeReplication`), never one
|
|
10
|
+
* computed again. The PUBLICATION is Mason's own (`mason_<deployment>`, made and dropped with
|
|
11
|
+
* the slot) unless the operator names one (`--publication <name>`, else `PG_PUBLICATION`, else the one
|
|
12
|
+
* an earlier start recorded): that one is the database owner's, read as it is and never created, altered or dropped
|
|
13
|
+
* (src/duck/publication.ts) — on a read replica it is the only way, since CREATE PUBLICATION is refused on a standby.
|
|
14
|
+
*
|
|
15
|
+
* In the ClickHouse service this was cut from, the two modes kept row versions that could not continue each other (a
|
|
16
|
+
* counter per table against WAL positions): a switch was a copy from scratch, which that service did by itself. The DuckDB
|
|
17
|
+
* prototype keeps its copies through a switch: each table is caught up by one poll from its watermark (src/duck/sync.ts
|
|
18
|
+
* `prepareReplication`), and `mason start` says so.
|
|
19
|
+
*/
|
|
20
|
+
import { createHash } from "node:crypto";
|
|
21
|
+
import { replicationNames } from "../cdc/source.js";
|
|
22
|
+
import { connectSource, systemIdentifier } from "./preflight.js";
|
|
23
|
+
import { readServerFacts, readSlotFacts, replicationChecks } from "./replication-checks.js";
|
|
24
|
+
/** Postgres's longest identifier (NAMEDATALEN - 1): a longer slot name is refused, a longer publication's cut with a notice. */
|
|
25
|
+
export const PG_NAME_BYTES = 63;
|
|
26
|
+
/**
|
|
27
|
+
* THE NAME A SOURCE'S SLOT AND PUBLICATION ARE MADE FROM, `dev_<project>_<source>` (src/cdc/source.ts replicationNames
|
|
28
|
+
* puts `mason_` before it): one per project AND source, the source the sources file's name, since a project will sync
|
|
29
|
+
* several databases, each with a slot of its own. Lower case, `_` for any other character, as a slot's name must be. A
|
|
30
|
+
* project (48 at most) and a source (48) can pass Postgres's 63 bytes: then a readable prefix and `_` and 8 hex
|
|
31
|
+
* characters of the whole name's sha256, so two names sharing a long prefix never meet and none is ever cut silently.
|
|
32
|
+
*/
|
|
33
|
+
export function deploymentOf(project, source) {
|
|
34
|
+
const full = `dev_${project}_${source}`.toLowerCase().replace(/[^a-z0-9_]/g, "_");
|
|
35
|
+
const room = PG_NAME_BYTES - "mason_".length;
|
|
36
|
+
if (full.length <= room)
|
|
37
|
+
return full;
|
|
38
|
+
return `${full.slice(0, room - 9)}_${createHash("sha256").update(full).digest("hex").slice(0, 8)}`;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* The publication and slot a project's run reads: the publication `named` (the flag, `PG_PUBLICATION` or the recorded
|
|
42
|
+
* one: `namedPublication`), else Mason's own — by the change reader's own rule (src/cdc/source.ts `replicationNames`),
|
|
43
|
+
* which also says whether it is Mason's to make. The slot is always the project's own.
|
|
44
|
+
*/
|
|
45
|
+
export function projectReplication(name, source, named = null) {
|
|
46
|
+
const { publication, slot, owned } = replicationNames(named ? { PG_PUBLICATION: named } : {}, deploymentOf(name, source));
|
|
47
|
+
return { publication, slot, owned };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* WHAT REPLICATION MODE MADE ON THE DATABASE, as the start that made it recorded it (`config.json` `replication`): what
|
|
51
|
+
* `--poll` and `mason destroy` drop. Recorded, never computed again, so a name that changes since — the sources file
|
|
52
|
+
* renamed, the naming rule — never leaves a slot that nobody drops, holding the database's WAL for ever. A project with
|
|
53
|
+
* no record (none started in replication mode since it was kept) has the names the rule gives now.
|
|
54
|
+
*/
|
|
55
|
+
export function madeReplication(cfg, source) {
|
|
56
|
+
return cfg.replication ?? projectReplication(cfg.name, source, cfg.publication ?? null);
|
|
57
|
+
}
|
|
58
|
+
/** The publication an operator named, and where: `--publication`, else `PG_PUBLICATION`, else the one an earlier start
|
|
59
|
+
* of this copy recorded (`config.json` `publication`), else none (Mason's own). */
|
|
60
|
+
export function namedPublication(flag, env, recorded) {
|
|
61
|
+
if (flag)
|
|
62
|
+
return { name: flag, from: "flag" };
|
|
63
|
+
if (env.PG_PUBLICATION)
|
|
64
|
+
return { name: env.PG_PUBLICATION, from: "env" };
|
|
65
|
+
if (recorded)
|
|
66
|
+
return { name: recorded, from: "recorded" };
|
|
67
|
+
return null;
|
|
68
|
+
}
|
|
69
|
+
/** The mode a start runs: the flag, else the project's, else polling. */
|
|
70
|
+
export function changesOf(flag, recorded) {
|
|
71
|
+
return flag ?? recorded ?? "poll";
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Whether the developer's database can be read by replication: what refuses it (wal_level, the role — REPLICATION, a
|
|
75
|
+
* superuser, or RDS's rds_replication membership, as `replicationChecks` reads them — a free slot and WAL sender, a
|
|
76
|
+
* standby before 16, and a standby with no publication named: Mason cannot make its own there), and what is only worth
|
|
77
|
+
* saying (the WAL cap), from the production checks' own rules.
|
|
78
|
+
*/
|
|
79
|
+
export function readinessVerdict(facts, slot, slotFacts, repl) {
|
|
80
|
+
const checks = replicationChecks(facts, slot, slotFacts);
|
|
81
|
+
// a standby takes no CREATE PUBLICATION (it is read-only): the publication must be the primary's, named
|
|
82
|
+
if (facts.standby && repl.owned) {
|
|
83
|
+
checks.push({ name: "publication", ok: false, detail: `this database is a read replica, where Mason cannot make its publication ${repl.publication} (CREATE ` +
|
|
84
|
+
"PUBLICATION is refused on a standby). Create one on the primary for the tables you add, then name it: mason start --replication --publication <name>" });
|
|
85
|
+
}
|
|
86
|
+
// a laptop's database has no cap on its slots by default: worth saying, never a reason to refuse a dev run
|
|
87
|
+
return { refused: checks.filter((x) => x.ok === false && x.name !== "WAL cap"), warnings: checks.filter((x) => (x.name === "WAL cap" && x.ok === false) || x.warn === true) };
|
|
88
|
+
}
|
|
89
|
+
export async function replicationReadiness(source, repl) {
|
|
90
|
+
const c = await connectSource(source);
|
|
91
|
+
if ("failure" in c)
|
|
92
|
+
return { failure: c.failure.message };
|
|
93
|
+
try {
|
|
94
|
+
return readinessVerdict(await readServerFacts(c.pg), repl.slot, await readSlotFacts(c.pg, repl.slot), repl);
|
|
95
|
+
}
|
|
96
|
+
finally {
|
|
97
|
+
c.pg.close();
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
/** The statements that remove what replication mode made, for a person to run when Mason could not: the slot, and the
|
|
101
|
+
* publication only when it is Mason's own (a named one is its owner's). */
|
|
102
|
+
export function dropStatements(o) {
|
|
103
|
+
return `SELECT pg_drop_replication_slot('${o.slot}') FROM pg_replication_slots WHERE slot_name = '${o.slot}' AND NOT active;` +
|
|
104
|
+
(o.owned ? ` DROP PUBLICATION IF EXISTS ${o.publication};` : "");
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Drop what replication mode made on the developer's database — the slot, and the publication when it is Mason's own,
|
|
108
|
+
* never one the operator named — and verify it is gone: what was dropped, in words. A slot a reader still holds is never
|
|
109
|
+
* dropped (the run must be stopped first): that is a failure.
|
|
110
|
+
*/
|
|
111
|
+
export async function dropOwnReplication(source, o, from = null) {
|
|
112
|
+
const c = await connectSource(source);
|
|
113
|
+
if ("failure" in c)
|
|
114
|
+
throw new Error(`could not connect to drop them: ${c.failure.message}`);
|
|
115
|
+
const pg = c.pg;
|
|
116
|
+
const lit = (s) => `'${s.replace(/'/g, "''")}'`;
|
|
117
|
+
try {
|
|
118
|
+
// the server the copy came from (its system identifier, recorded at the copy): another one has none of its slot
|
|
119
|
+
if (from?.systemIdentifier) {
|
|
120
|
+
const now = await systemIdentifier(pg);
|
|
121
|
+
if (now !== null && now !== from.systemIdentifier) {
|
|
122
|
+
throw new Error(`${source.database} on ${source.host} is another server than the copy came from (system identifier ${now}, the copy's ${from.systemIdentifier}), where the slot is: put that server's URL back in the sources file and run mason destroy again, or drop them there by hand`);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
const dropped = [];
|
|
126
|
+
const slot = await readSlotFacts(pg, o.slot);
|
|
127
|
+
if (slot) {
|
|
128
|
+
if (slot.active)
|
|
129
|
+
throw new Error(`replication slot ${o.slot} is in use by a reader: stop it first`);
|
|
130
|
+
await pg.query(`SELECT pg_drop_replication_slot(${lit(o.slot)})`);
|
|
131
|
+
dropped.push(`replication slot ${o.slot}`);
|
|
132
|
+
}
|
|
133
|
+
const hasPub = async () => (await pg.query(`SELECT 1 FROM pg_publication WHERE pubname = ${lit(o.publication)}`)).rows.length > 0;
|
|
134
|
+
if (o.owned && await hasPub()) {
|
|
135
|
+
await pg.query(`DROP PUBLICATION IF EXISTS "${o.publication.replace(/"/g, '""')}"`);
|
|
136
|
+
dropped.push(`publication ${o.publication}`);
|
|
137
|
+
}
|
|
138
|
+
// verified, never assumed: a slot left behind holds the database's WAL for ever
|
|
139
|
+
if (await readSlotFacts(pg, o.slot))
|
|
140
|
+
throw new Error(`replication slot ${o.slot} is still there after pg_drop_replication_slot`);
|
|
141
|
+
if (o.owned && await hasPub())
|
|
142
|
+
throw new Error(`publication ${o.publication} is still there after DROP PUBLICATION`);
|
|
143
|
+
return dropped;
|
|
144
|
+
}
|
|
145
|
+
finally {
|
|
146
|
+
pg.close();
|
|
147
|
+
}
|
|
148
|
+
}
|