@mason-data/cli 0.0.0-stage → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +268 -2
- package/dist/src/api/error.js +18 -0
- package/dist/src/api/postgres/catalog-worker.js +92 -0
- package/dist/src/api/postgres/catalog.js +477 -0
- package/dist/src/api/postgres/encode.js +541 -0
- package/dist/src/api/postgres/explain.js +143 -0
- package/dist/src/api/postgres/listener.js +60 -0
- package/dist/src/api/postgres/parser-worker.js +58 -0
- package/dist/src/api/postgres/parser.js +121 -0
- package/dist/src/api/postgres/plain.js +33 -0
- package/dist/src/api/postgres/protocol.js +404 -0
- package/dist/src/api/postgres/schema.js +54 -0
- package/dist/src/api/postgres/server.js +1771 -0
- package/dist/src/api/postgres/translate.js +248 -0
- package/dist/src/api/request-log.js +50 -0
- package/dist/src/cdc/pg-wire.js +588 -0
- package/dist/src/cdc/pgoutput.js +107 -0
- package/dist/src/cdc/poll.js +67 -0
- package/dist/src/cdc/source.js +65 -0
- package/dist/src/cli/background.js +198 -0
- package/dist/src/cli/command.js +173 -0
- package/dist/src/cli/commands.js +1277 -0
- package/dist/src/cli/control.js +36 -0
- package/dist/src/cli/demo.js +149 -0
- package/dist/src/cli/dev.js +669 -0
- package/dist/src/cli/doctor.js +260 -0
- package/dist/src/cli/guides.js +158 -0
- package/dist/src/cli/init.js +366 -0
- package/dist/src/cli/machine.js +162 -0
- package/dist/src/cli/main.js +83 -0
- package/dist/src/cli/mcp.js +726 -0
- package/dist/src/cli/pg-tls.js +100 -0
- package/dist/src/cli/pg-url.js +52 -0
- package/dist/src/cli/pin.js +36 -0
- package/dist/src/cli/ports.js +118 -0
- package/dist/src/cli/preflight.js +347 -0
- package/dist/src/cli/project.js +129 -0
- package/dist/src/cli/readiness.js +76 -0
- package/dist/src/cli/registry.js +220 -0
- package/dist/src/cli/replication-checks.js +120 -0
- package/dist/src/cli/replication-mode.js +148 -0
- package/dist/src/cli/run-server.js +311 -0
- package/dist/src/cli/runs.js +120 -0
- package/dist/src/cli/security.js +86 -0
- package/dist/src/cli/self-signed-cert.js +64 -0
- package/dist/src/cli/serve.js +301 -0
- package/dist/src/cli/source-edit.js +116 -0
- package/dist/src/cli/sources.js +564 -0
- package/dist/src/cli/state.js +58 -0
- package/dist/src/cli/style.js +187 -0
- package/dist/src/cli/supervise.js +248 -0
- package/dist/src/cli/sync.js +94 -0
- package/dist/src/cli/ui.js +107 -0
- package/dist/src/cli/user.js +229 -0
- package/dist/src/cli/version.js +23 -0
- package/dist/src/demo/seed.js +237 -0
- package/dist/src/demo/source.js +655 -0
- package/dist/src/duck/analyze.js +113 -0
- package/dist/src/duck/cancel.js +49 -0
- package/dist/src/duck/distinct-key.js +228 -0
- package/dist/src/duck/executor-worker.js +268 -0
- package/dist/src/duck/executor.js +567 -0
- package/dist/src/duck/explain.js +238 -0
- package/dist/src/duck/functions.js +819 -0
- package/dist/src/duck/gaps.js +94 -0
- package/dist/src/duck/jsonb-order.js +175 -0
- package/dist/src/duck/jsonb.js +188 -0
- package/dist/src/duck/kept-route.js +299 -0
- package/dist/src/duck/kept.js +964 -0
- package/dist/src/duck/late.js +221 -0
- package/dist/src/duck/libc-order.js +190 -0
- package/dist/src/duck/pg-eval-worker.js +34 -0
- package/dist/src/duck/pg-eval.js +59 -0
- package/dist/src/duck/pgtree.js +378 -0
- package/dist/src/duck/plan.js +259 -0
- package/dist/src/duck/publication.js +207 -0
- package/dist/src/duck/requests.js +209 -0
- package/dist/src/duck/resolved.js +248 -0
- package/dist/src/duck/rewrite.js +5101 -0
- package/dist/src/duck/store.js +907 -0
- package/dist/src/duck/stream.js +518 -0
- package/dist/src/duck/sync.js +2929 -0
- package/dist/src/duck/to-char.js +482 -0
- package/dist/src/duck/token.js +31 -0
- package/dist/src/duck/types.js +427 -0
- package/dist/src/duck/users.js +231 -0
- package/dist/src/duck/verified.js +1001 -0
- package/dist/src/duck/views.js +150 -0
- package/dist/src/lib/database-url.js +74 -0
- package/dist/src/lib/did-you-mean.js +43 -0
- package/dist/src/lib/dotenv.js +72 -0
- package/dist/src/lib/exit-when-written.js +20 -0
- package/dist/src/lib/float-equal.js +69 -0
- package/dist/src/lib/http-json.js +22 -0
- package/dist/src/lib/log.js +161 -0
- package/dist/src/lib/pg-numeric.js +489 -0
- package/dist/src/lib/port.js +22 -0
- package/dist/src/lib/project-file.js +49 -0
- package/dist/src/lib/project-files.js +43 -0
- package/dist/src/lib/project-version.js +76 -0
- package/dist/src/lib/runnable.js +14 -0
- package/dist/src/lib/shell-word.js +4 -0
- package/dist/src/lib/timezone.js +237 -0
- package/dist/src/lib/zone-names.js +114 -0
- package/dist/src/web/app.css +76 -0
- package/dist/src/web/app.js +491 -0
- package/dist/src/web/index.html +44 -0
- package/package.json +22 -4
|
@@ -0,0 +1,588 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* A POSTGRES CLIENT, DEPENDENCY-FREE, for the change reader: exactly what Mason needs from the
|
|
3
|
+
* wire protocol and nothing else — startup (TLS per libpq's sslmode), password / md5 / SCRAM-SHA-256 auth,
|
|
4
|
+
* the simple query protocol, and the streaming replication sub-protocol (CopyBoth, XLogData, keepalives,
|
|
5
|
+
* standby status updates). Every value travels as TEXT: the reader turns text into ClickHouse values itself,
|
|
6
|
+
* so no binary type decoding lives here.
|
|
7
|
+
*
|
|
8
|
+
* Why not `pg`: the scripts here run with no node_modules on the test server, the replication sub-protocol is a
|
|
9
|
+
* separate package on top of it, and the part Mason uses is a few hundred lines this file can hold whole.
|
|
10
|
+
*
|
|
11
|
+
* Protocol reference (Postgres's own documentation): https://www.postgresql.org/docs/current/protocol-message-formats.html and
|
|
12
|
+
* https://www.postgresql.org/docs/current/protocol-replication.html.
|
|
13
|
+
*/
|
|
14
|
+
import * as crypto from "node:crypto";
|
|
15
|
+
import * as net from "node:net";
|
|
16
|
+
import * as tls from "node:tls";
|
|
17
|
+
/** A server ErrorResponse, with its SQLSTATE: callers branch on `code` (42P01, 42710, 55006 …). */
|
|
18
|
+
export class PgError extends Error {
|
|
19
|
+
code;
|
|
20
|
+
detail;
|
|
21
|
+
/** the ErrorResponse's own fields, by their one-letter codes (M the message alone, H the hint, P the position) */
|
|
22
|
+
fields;
|
|
23
|
+
constructor(fields) {
|
|
24
|
+
super(`${fields.S ?? "ERROR"}: ${fields.M ?? "unknown error"}${fields.D ? ` (${fields.D})` : ""}`);
|
|
25
|
+
this.code = fields.C ?? "";
|
|
26
|
+
this.detail = fields.D;
|
|
27
|
+
this.fields = fields;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Microseconds between the Unix epoch and Postgres's (2000-01-01). */
|
|
31
|
+
const PG_EPOCH_OFFSET_US = 946684800000000n;
|
|
32
|
+
export class PgConnection {
|
|
33
|
+
socket;
|
|
34
|
+
buf = Buffer.alloc(0);
|
|
35
|
+
/** chunks received after `buf` while the message they complete is still short of `need` bytes (onData) */
|
|
36
|
+
chunks = [];
|
|
37
|
+
chunked = 0;
|
|
38
|
+
waiting = null;
|
|
39
|
+
queue = [];
|
|
40
|
+
/** the bytes of the messages in `queue`: what backpressure bounds (onData) */
|
|
41
|
+
queued = 0;
|
|
42
|
+
closedWith = null;
|
|
43
|
+
/** Server parameters (server_version, TimeZone, …) as reported at startup. */
|
|
44
|
+
parameters = {};
|
|
45
|
+
opts;
|
|
46
|
+
constructor(opts) { this.opts = opts; }
|
|
47
|
+
static async connect(opts) {
|
|
48
|
+
const c = new PgConnection(opts);
|
|
49
|
+
await c.open();
|
|
50
|
+
return c;
|
|
51
|
+
}
|
|
52
|
+
// ------------------------------------------------------------------ transport
|
|
53
|
+
/**
|
|
54
|
+
* THE CONNECT TIMEOUT BOUNDS THE WHOLE HANDSHAKE: it once covered the TCP connect alone, and the SSL
|
|
55
|
+
* answer and the startup exchange waited for ever. A socket something accepts and never answers — a port
|
|
56
|
+
* proxy in front of a container that is restarting, a half-open connection — hung the reader's reconnect loop
|
|
57
|
+
* past the test's 90 s with no "the source is away" line to say so (the ClickHouse stack's change-reader fill test,
|
|
58
|
+
* twice in CI). One deadline from the first SYN to ReadyForQuery; past it the socket is destroyed and the error reads as
|
|
59
|
+
* the source being away (reader.ts `sourceAway`), so the reconnect loop tries again.
|
|
60
|
+
*/
|
|
61
|
+
async open() {
|
|
62
|
+
const o = this.opts;
|
|
63
|
+
const deadline = o.connectTimeoutMs ?? 15_000;
|
|
64
|
+
let pending = null;
|
|
65
|
+
let timer;
|
|
66
|
+
const timedOut = new Promise((_, reject) => {
|
|
67
|
+
timer = setTimeout(() => { pending?.destroy(); reject(new Error(`connecting to ${o.host}:${o.port} timed out`)); }, deadline);
|
|
68
|
+
});
|
|
69
|
+
try {
|
|
70
|
+
await Promise.race([this.handshake((s) => { pending = s; }), timedOut]);
|
|
71
|
+
}
|
|
72
|
+
finally {
|
|
73
|
+
clearTimeout(timer);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
/** TCP connect, the SSL negotiation and the startup exchange; `hold` is told the live socket, for a timeout to destroy. */
|
|
77
|
+
async handshake(hold) {
|
|
78
|
+
const sock = await transport(this.opts, hold);
|
|
79
|
+
this.socket = sock;
|
|
80
|
+
sock.on("data", (d) => this.onData(d));
|
|
81
|
+
sock.on("error", (e) => this.fail(e));
|
|
82
|
+
sock.on("close", () => this.fail(new Error("connection closed by the server")));
|
|
83
|
+
await this.startup();
|
|
84
|
+
}
|
|
85
|
+
/** the server's key for this session (BackendKeyData), for a CancelRequest */
|
|
86
|
+
key = null;
|
|
87
|
+
/**
|
|
88
|
+
* ASK THE SERVER TO CANCEL THIS SESSION'S RUNNING STATEMENT (Postgres's CancelRequest, on a connection of its own,
|
|
89
|
+
* TLS as this one's sslmode): the statement ends with 57014 on this connection, which stays usable. Resolves once
|
|
90
|
+
* the request is written.
|
|
91
|
+
*/
|
|
92
|
+
async cancel() {
|
|
93
|
+
const key = this.key;
|
|
94
|
+
if (key === null)
|
|
95
|
+
throw new Error("the server sent no key for this session: it cannot be cancelled");
|
|
96
|
+
let held = null;
|
|
97
|
+
const timer = setTimeout(() => { held?.destroy(); }, this.opts.connectTimeoutMs ?? 15_000);
|
|
98
|
+
try {
|
|
99
|
+
const s = await transport(this.opts, (x) => { held = x; });
|
|
100
|
+
const b = Buffer.alloc(16);
|
|
101
|
+
b.writeInt32BE(16, 0);
|
|
102
|
+
b.writeInt32BE(80877102, 4);
|
|
103
|
+
b.writeInt32BE(key.pid, 8);
|
|
104
|
+
b.writeInt32BE(key.secret, 12);
|
|
105
|
+
await new Promise((resolve, reject) => { s.once("error", reject); s.end(b, () => resolve()); });
|
|
106
|
+
}
|
|
107
|
+
finally {
|
|
108
|
+
clearTimeout(timer);
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
fail(e) {
|
|
112
|
+
if (this.closedWith)
|
|
113
|
+
return;
|
|
114
|
+
this.closedWith = e;
|
|
115
|
+
const w = this.waiting;
|
|
116
|
+
this.waiting = null;
|
|
117
|
+
if (w)
|
|
118
|
+
w(e);
|
|
119
|
+
}
|
|
120
|
+
onData(d) {
|
|
121
|
+
// A LARGE MESSAGE ARRIVES IN MANY CHUNKS, and joining each chunk to what came before copies it over and over:
|
|
122
|
+
// quadratic in the message, all of it off the V8 heap, where no heap limit makes the collector hurry. A row with
|
|
123
|
+
// a multi-megabyte payload took the reader past its container's 512 MiB (a cgroup OOM kill, the test server's first
|
|
124
|
+
// copy). So chunks wait in a list until the message they belong to is complete, and join once.
|
|
125
|
+
this.chunks.push(d);
|
|
126
|
+
this.chunked += d.length;
|
|
127
|
+
const have = this.buf.length + this.chunked;
|
|
128
|
+
const need = this.buf.length >= 5 ? 1 + this.buf.readInt32BE(1) : 5;
|
|
129
|
+
if (have < need)
|
|
130
|
+
return;
|
|
131
|
+
this.buf = Buffer.concat([this.buf, ...this.chunks], have);
|
|
132
|
+
this.chunks = [];
|
|
133
|
+
this.chunked = 0;
|
|
134
|
+
while (this.buf.length >= 5) {
|
|
135
|
+
const len = this.buf.readInt32BE(1);
|
|
136
|
+
if (this.buf.length < 1 + len)
|
|
137
|
+
break;
|
|
138
|
+
const msg = { type: String.fromCharCode(this.buf[0]), body: this.buf.subarray(5, 1 + len) };
|
|
139
|
+
this.buf = this.buf.subarray(1 + len);
|
|
140
|
+
if (msg.type === "N")
|
|
141
|
+
continue; // NoticeResponse: informational
|
|
142
|
+
if (msg.type === "S") {
|
|
143
|
+
this.parameter(msg.body);
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
const w = this.waiting;
|
|
147
|
+
if (w) {
|
|
148
|
+
this.waiting = null;
|
|
149
|
+
w(msg);
|
|
150
|
+
}
|
|
151
|
+
else {
|
|
152
|
+
this.queue.push(msg);
|
|
153
|
+
this.queued += msg.body.length;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
// Backpressure: a consumer that is not reading (the reader flushing into ClickHouse) must not let the
|
|
157
|
+
// stream pile up in memory — counted in messages AND bytes, since one row may carry megabytes. The queue
|
|
158
|
+
// resumes the socket when it drains.
|
|
159
|
+
if (this.queue.length > 1000 || this.queued > QUEUE_PAUSE_BYTES)
|
|
160
|
+
this.socket.pause();
|
|
161
|
+
}
|
|
162
|
+
parameter(body) {
|
|
163
|
+
const [k, v] = cstrings(body, 2);
|
|
164
|
+
if (k !== undefined)
|
|
165
|
+
this.parameters[k] = v ?? "";
|
|
166
|
+
}
|
|
167
|
+
/** the next message of a statement's reply; when the server closed the connection after an ErrorResponse it sent
|
|
168
|
+
* (a FATAL: a user removed, a shutdown) and before ReadyForQuery, that error — never the bare "connection closed" */
|
|
169
|
+
async reply(error) {
|
|
170
|
+
try {
|
|
171
|
+
return await this.next();
|
|
172
|
+
}
|
|
173
|
+
catch (e) {
|
|
174
|
+
throw error ?? e;
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
next() {
|
|
178
|
+
const m = this.queue.shift();
|
|
179
|
+
if (m) {
|
|
180
|
+
this.queued -= m.body.length;
|
|
181
|
+
if (this.queue.length < 100 && this.queued < QUEUE_PAUSE_BYTES / 4 && this.socket.isPaused())
|
|
182
|
+
this.socket.resume();
|
|
183
|
+
return Promise.resolve(m);
|
|
184
|
+
}
|
|
185
|
+
if (this.closedWith)
|
|
186
|
+
return Promise.reject(this.closedWith);
|
|
187
|
+
return new Promise((resolve, reject) => {
|
|
188
|
+
this.waiting = (x) => x instanceof Error ? reject(x) : resolve(x);
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
send(type, body) {
|
|
192
|
+
if (this.closedWith)
|
|
193
|
+
return; // a status update after the server went away has nowhere to go
|
|
194
|
+
const head = Buffer.alloc(5);
|
|
195
|
+
head.write(type, 0, "latin1");
|
|
196
|
+
head.writeInt32BE(body.length + 4, 1);
|
|
197
|
+
this.socket.write(Buffer.concat([head, body]));
|
|
198
|
+
}
|
|
199
|
+
// ------------------------------------------------------------------ startup and auth
|
|
200
|
+
async startup() {
|
|
201
|
+
const o = this.opts;
|
|
202
|
+
const params = [
|
|
203
|
+
["user", o.user], ["database", o.database], ["client_encoding", "UTF8"],
|
|
204
|
+
// Text values in the forms the reader parses: ISO dates in UTC, floats round-trippable
|
|
205
|
+
...(o.textForms ?? [["DateStyle", "ISO, YMD"], ["TimeZone", "UTC"], ["extra_float_digits", "3"], ["IntervalStyle", "iso_8601"]]).map(([k, v]) => [k, v]),
|
|
206
|
+
["application_name", o.applicationName ?? "mason"],
|
|
207
|
+
];
|
|
208
|
+
if (o.replication)
|
|
209
|
+
params.push(["replication", "database"]);
|
|
210
|
+
const body = Buffer.concat([
|
|
211
|
+
int32(196608), // protocol 3.0
|
|
212
|
+
...params.map(([k, v]) => Buffer.concat([cstr(k), cstr(v)])),
|
|
213
|
+
Buffer.from([0]),
|
|
214
|
+
]);
|
|
215
|
+
const len = Buffer.alloc(4);
|
|
216
|
+
len.writeInt32BE(body.length + 4);
|
|
217
|
+
this.socket.write(Buffer.concat([len, body]));
|
|
218
|
+
for (;;) {
|
|
219
|
+
const m = await this.next();
|
|
220
|
+
if (m.type === "E")
|
|
221
|
+
throw new PgError(errorFields(m.body));
|
|
222
|
+
if (m.type === "R") {
|
|
223
|
+
await this.auth(m.body);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
if (m.type === "K") {
|
|
227
|
+
this.key = { pid: m.body.readInt32BE(0), secret: m.body.readInt32BE(4) };
|
|
228
|
+
continue;
|
|
229
|
+
} // BackendKeyData
|
|
230
|
+
if (m.type === "Z")
|
|
231
|
+
return;
|
|
232
|
+
throw new Error(`unexpected message ${m.type} during startup`);
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
scram = null;
|
|
236
|
+
async auth(body) {
|
|
237
|
+
const code = body.readInt32BE(0);
|
|
238
|
+
const pw = this.opts.password;
|
|
239
|
+
switch (code) {
|
|
240
|
+
case 0: return; // AuthenticationOk
|
|
241
|
+
case 3:
|
|
242
|
+
this.send("p", cstr(pw));
|
|
243
|
+
return;
|
|
244
|
+
case 5: {
|
|
245
|
+
const salt = body.subarray(4, 8);
|
|
246
|
+
const inner = md5hex(Buffer.concat([Buffer.from(pw), Buffer.from(this.opts.user)]));
|
|
247
|
+
this.send("p", cstr("md5" + md5hex(Buffer.concat([Buffer.from(inner), salt]))));
|
|
248
|
+
return;
|
|
249
|
+
}
|
|
250
|
+
case 10: {
|
|
251
|
+
const mechanisms = cstrings(body.subarray(4));
|
|
252
|
+
if (!mechanisms.includes("SCRAM-SHA-256"))
|
|
253
|
+
throw new Error(`the server offers ${mechanisms.join(", ")}; only SCRAM-SHA-256 is supported`);
|
|
254
|
+
const nonce = crypto.randomBytes(18).toString("base64");
|
|
255
|
+
const clientFirstBare = `n=,r=${nonce}`;
|
|
256
|
+
this.scram = { nonce, clientFirstBare };
|
|
257
|
+
const first = Buffer.from(`n,,${clientFirstBare}`);
|
|
258
|
+
this.send("p", Buffer.concat([cstr("SCRAM-SHA-256"), int32(first.length), first]));
|
|
259
|
+
return;
|
|
260
|
+
}
|
|
261
|
+
case 11: { // SASLContinue
|
|
262
|
+
const s = this.scram;
|
|
263
|
+
if (!s)
|
|
264
|
+
throw new Error("SASLContinue without a SASL exchange");
|
|
265
|
+
const serverFirst = body.subarray(4).toString();
|
|
266
|
+
const attrs = Object.fromEntries(serverFirst.split(",").map((kv) => [kv[0], kv.slice(2)]));
|
|
267
|
+
const r = attrs.r ?? "", salt = Buffer.from(attrs.s ?? "", "base64"), iterations = Number(attrs.i);
|
|
268
|
+
if (!r.startsWith(s.nonce))
|
|
269
|
+
throw new Error("SCRAM: the server's nonce does not extend ours");
|
|
270
|
+
const salted = crypto.pbkdf2Sync(pw.normalize("NFKC"), salt, iterations, 32, "sha256");
|
|
271
|
+
const clientKey = hmac(salted, "Client Key");
|
|
272
|
+
const storedKey = crypto.createHash("sha256").update(clientKey).digest();
|
|
273
|
+
const clientFinalNoProof = `c=biws,r=${r}`;
|
|
274
|
+
const authMessage = `${s.clientFirstBare},${serverFirst},${clientFinalNoProof}`;
|
|
275
|
+
const signature = hmac(storedKey, authMessage);
|
|
276
|
+
const proof = Buffer.alloc(clientKey.length);
|
|
277
|
+
for (let i = 0; i < proof.length; i++)
|
|
278
|
+
proof[i] = clientKey[i] ^ signature[i];
|
|
279
|
+
s.serverSignature = hmac(hmac(salted, "Server Key"), authMessage);
|
|
280
|
+
this.send("p", Buffer.from(`${clientFinalNoProof},p=${proof.toString("base64")}`));
|
|
281
|
+
return;
|
|
282
|
+
}
|
|
283
|
+
case 12: { // SASLFinal: the server proves it knew the password too
|
|
284
|
+
const v = /v=([^,]+)/.exec(body.subarray(4).toString())?.[1];
|
|
285
|
+
if (!this.scram?.serverSignature || v !== this.scram.serverSignature.toString("base64")) {
|
|
286
|
+
throw new Error("SCRAM: the server's signature does not verify");
|
|
287
|
+
}
|
|
288
|
+
return;
|
|
289
|
+
}
|
|
290
|
+
default: throw new Error(`authentication method ${code} is not supported (use password, md5 or scram-sha-256)`);
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
// ------------------------------------------------------------------ queries
|
|
294
|
+
/** One statement (or several `;`-separated) through the simple query protocol; the LAST result set. */
|
|
295
|
+
async query(sql) {
|
|
296
|
+
this.send("Q", cstr(sql));
|
|
297
|
+
let result = { fields: [], rows: [], command: "" };
|
|
298
|
+
let error = null;
|
|
299
|
+
for (;;) {
|
|
300
|
+
const m = await this.reply(error);
|
|
301
|
+
switch (m.type) {
|
|
302
|
+
case "T":
|
|
303
|
+
result = { fields: rowDescription(m.body), rows: [], command: "" };
|
|
304
|
+
break;
|
|
305
|
+
case "D":
|
|
306
|
+
result.rows.push(dataRow(m.body));
|
|
307
|
+
break;
|
|
308
|
+
case "C":
|
|
309
|
+
result.command = cstrings(m.body)[0] ?? "";
|
|
310
|
+
break;
|
|
311
|
+
case "E":
|
|
312
|
+
error = new PgError(errorFields(m.body));
|
|
313
|
+
break;
|
|
314
|
+
case "I": break; // EmptyQueryResponse
|
|
315
|
+
case "Z":
|
|
316
|
+
if (error)
|
|
317
|
+
throw error;
|
|
318
|
+
return result;
|
|
319
|
+
default: break;
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* One statement's rows AS THEY ARRIVE, never held together: a page of a copy may carry hundreds of megabytes of
|
|
325
|
+
* payloads (68 MB in 64 heap blocks of a table of wide JSON payloads), which `query` would hold whole. The socket
|
|
326
|
+
* pauses while the consumer is busy (onData's backpressure), so memory is bounded by the queue, not the result.
|
|
327
|
+
* A consumer that stops early must drain it: the connection is unusable until the server's ReadyForQuery.
|
|
328
|
+
*/
|
|
329
|
+
async *rows(sql) {
|
|
330
|
+
this.send("Q", cstr(sql));
|
|
331
|
+
let error = null;
|
|
332
|
+
for (;;) {
|
|
333
|
+
const m = await this.reply(error);
|
|
334
|
+
switch (m.type) {
|
|
335
|
+
case "D":
|
|
336
|
+
if (error === null)
|
|
337
|
+
yield dataRow(m.body);
|
|
338
|
+
break;
|
|
339
|
+
case "E":
|
|
340
|
+
error = new PgError(errorFields(m.body));
|
|
341
|
+
break;
|
|
342
|
+
case "Z":
|
|
343
|
+
if (error)
|
|
344
|
+
throw error;
|
|
345
|
+
return;
|
|
346
|
+
default: break;
|
|
347
|
+
}
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
/**
|
|
351
|
+
* ONE STATEMENT THROUGH THE EXTENDED PROTOCOL (`mason mcp`'s query): Parse, Bind, Describe and Execute of the unnamed
|
|
352
|
+
* portal, then Sync — so the server takes exactly one statement (a second is Postgres's 42601), and `maxRows` is the
|
|
353
|
+
* Execute's row count: the server stops at it and says so (PortalSuspended), so a query an agent leaves unbounded is
|
|
354
|
+
* never sent whole. 0 is every row. Every value is text.
|
|
355
|
+
*/
|
|
356
|
+
async extended(sql, maxRows) {
|
|
357
|
+
const int16 = (n) => { const b = Buffer.alloc(2); b.writeInt16BE(n); return b; };
|
|
358
|
+
this.send("P", Buffer.concat([cstr(""), cstr(sql), int16(0)]));
|
|
359
|
+
// no parameters, every result column in text
|
|
360
|
+
this.send("B", Buffer.concat([cstr(""), cstr(""), int16(0), int16(0), int16(0)]));
|
|
361
|
+
this.send("D", Buffer.concat([Buffer.from("P"), cstr("")]));
|
|
362
|
+
this.send("E", Buffer.concat([cstr(""), int32(maxRows)]));
|
|
363
|
+
this.send("S", Buffer.alloc(0));
|
|
364
|
+
const result = { fields: [], rows: [], suspended: false, command: "", empty: false };
|
|
365
|
+
let error = null;
|
|
366
|
+
for (;;) {
|
|
367
|
+
const m = await this.reply(error);
|
|
368
|
+
switch (m.type) {
|
|
369
|
+
case "T":
|
|
370
|
+
result.fields = rowFields(m.body);
|
|
371
|
+
break;
|
|
372
|
+
case "D":
|
|
373
|
+
if (error === null)
|
|
374
|
+
result.rows.push(dataRow(m.body));
|
|
375
|
+
break;
|
|
376
|
+
case "s":
|
|
377
|
+
result.suspended = true;
|
|
378
|
+
break;
|
|
379
|
+
case "C":
|
|
380
|
+
result.command = cstrings(m.body)[0] ?? "";
|
|
381
|
+
break;
|
|
382
|
+
case "E":
|
|
383
|
+
error = new PgError(errorFields(m.body));
|
|
384
|
+
break;
|
|
385
|
+
case "I":
|
|
386
|
+
result.empty = true;
|
|
387
|
+
break;
|
|
388
|
+
case "Z":
|
|
389
|
+
if (error)
|
|
390
|
+
throw error;
|
|
391
|
+
return result;
|
|
392
|
+
default: break; // ParseComplete, BindComplete, NoData
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
}
|
|
396
|
+
// ------------------------------------------------------------------ replication
|
|
397
|
+
/** START_REPLICATION, then an async stream of XLogData and keepalives until the connection ends. */
|
|
398
|
+
async *replicate(command) {
|
|
399
|
+
this.send("Q", cstr(command));
|
|
400
|
+
for (;;) {
|
|
401
|
+
const m = await this.next();
|
|
402
|
+
if (m.type === "E")
|
|
403
|
+
throw new PgError(errorFields(m.body));
|
|
404
|
+
if (m.type === "W")
|
|
405
|
+
break; // CopyBothResponse
|
|
406
|
+
}
|
|
407
|
+
for (;;) {
|
|
408
|
+
const m = await this.next();
|
|
409
|
+
if (m.type === "E")
|
|
410
|
+
throw new PgError(errorFields(m.body));
|
|
411
|
+
if (m.type === "c" || m.type === "C" || m.type === "Z")
|
|
412
|
+
return; // the server ended the stream
|
|
413
|
+
if (m.type !== "d")
|
|
414
|
+
continue;
|
|
415
|
+
const b = m.body;
|
|
416
|
+
if (b[0] === 0x77) { // 'w' XLogData
|
|
417
|
+
yield { kind: "data", start: b.readBigUInt64BE(1), walEnd: b.readBigUInt64BE(9), data: b.subarray(25) };
|
|
418
|
+
}
|
|
419
|
+
else if (b[0] === 0x6b) { // 'k' primary keepalive
|
|
420
|
+
yield { kind: "keepalive", walEnd: b.readBigUInt64BE(1), replyRequested: b[17] === 1 };
|
|
421
|
+
}
|
|
422
|
+
}
|
|
423
|
+
}
|
|
424
|
+
/**
|
|
425
|
+
* A standby status update: everything up to `flushed` is durable on our side, so the server may discard
|
|
426
|
+
* the WAL behind it. The ONLY place the reader tells Postgres a change is done.
|
|
427
|
+
*/
|
|
428
|
+
sendStatus(flushed, replyRequested = false) {
|
|
429
|
+
const b = Buffer.alloc(34);
|
|
430
|
+
b[0] = 0x72; // 'r'
|
|
431
|
+
b.writeBigUInt64BE(flushed, 1); // written
|
|
432
|
+
b.writeBigUInt64BE(flushed, 9); // flushed
|
|
433
|
+
b.writeBigUInt64BE(flushed, 17); // applied
|
|
434
|
+
b.writeBigUInt64BE(BigInt(Date.now()) * 1000n - PG_EPOCH_OFFSET_US, 25);
|
|
435
|
+
b[33] = replyRequested ? 1 : 0;
|
|
436
|
+
this.send("d", b);
|
|
437
|
+
}
|
|
438
|
+
close() {
|
|
439
|
+
if (this.closedWith)
|
|
440
|
+
return;
|
|
441
|
+
if (this.socket.writable)
|
|
442
|
+
this.send("X", Buffer.alloc(0)); // Terminate: the server ends the backend cleanly
|
|
443
|
+
// a read waiting for the server's next message (a replication stream between changes) ends with it, never waits on
|
|
444
|
+
// a socket nothing reads any more (stopping a DuckDB stream once hung on its idle START_REPLICATION)
|
|
445
|
+
this.fail(new Error("connection closed"));
|
|
446
|
+
this.socket.end();
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
// ------------------------------------------------------------------ helpers
|
|
450
|
+
/** TCP connect and the SSL negotiation (libpq's sslmode): the socket a startup, or a CancelRequest, is written to. */
|
|
451
|
+
async function transport(o, hold) {
|
|
452
|
+
const mode = o.sslmode ?? "prefer";
|
|
453
|
+
const raw = await new Promise((resolve, reject) => {
|
|
454
|
+
const s = net.connect({ host: o.host, port: o.port });
|
|
455
|
+
hold(s);
|
|
456
|
+
s.once("connect", () => resolve(s));
|
|
457
|
+
s.once("error", (e) => reject(new Error(`connecting to ${o.host}:${o.port}: ${socketErrorText(e)}`)));
|
|
458
|
+
});
|
|
459
|
+
let sock = raw;
|
|
460
|
+
if (mode !== "disable" && mode !== "allow") {
|
|
461
|
+
// SSLRequest: length 8, code 80877103; the server answers one byte, S or N
|
|
462
|
+
raw.write(Buffer.from([0, 0, 0, 8, 0x04, 0xd2, 0x16, 0x2f]));
|
|
463
|
+
const answer = await new Promise((resolve, reject) => {
|
|
464
|
+
raw.once("data", (d) => resolve(String.fromCharCode(d[0])));
|
|
465
|
+
raw.once("error", reject);
|
|
466
|
+
});
|
|
467
|
+
if (answer === "S") {
|
|
468
|
+
const verify = mode === "verify-ca" || mode === "verify-full";
|
|
469
|
+
sock = await new Promise((resolve, reject) => {
|
|
470
|
+
const t = tls.connect({
|
|
471
|
+
socket: raw, rejectUnauthorized: verify,
|
|
472
|
+
...(mode === "verify-full" ? { servername: o.host } : { checkServerIdentity: () => undefined }),
|
|
473
|
+
});
|
|
474
|
+
hold(t);
|
|
475
|
+
t.once("secureConnect", () => resolve(t));
|
|
476
|
+
t.once("error", reject);
|
|
477
|
+
});
|
|
478
|
+
}
|
|
479
|
+
else if (mode !== "prefer") {
|
|
480
|
+
raw.destroy();
|
|
481
|
+
throw new Error(`sslmode=${mode}, but ${o.host} does not accept TLS`);
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
return sock;
|
|
485
|
+
}
|
|
486
|
+
/** `16/B374D848` -> its 64-bit position; throws on anything else. */
|
|
487
|
+
export function parseLsn(text) {
|
|
488
|
+
const m = /^([0-9A-Fa-f]{1,8})\/([0-9A-Fa-f]{1,8})$/.exec(text.trim());
|
|
489
|
+
if (!m)
|
|
490
|
+
throw new Error(`not a WAL position: ${JSON.stringify(text)}`);
|
|
491
|
+
return (BigInt(`0x${m[1]}`) << 32n) | BigInt(`0x${m[2]}`);
|
|
492
|
+
}
|
|
493
|
+
export function formatLsn(lsn) {
|
|
494
|
+
return `${(lsn >> 32n).toString(16).toUpperCase()}/${(lsn & 0xffffffffn).toString(16).toUpperCase()}`;
|
|
495
|
+
}
|
|
496
|
+
/** A SQL string literal (standard_conforming_strings on, the default since 9.1). */
|
|
497
|
+
export function pgLiteral(value) {
|
|
498
|
+
if (value.includes("\0"))
|
|
499
|
+
throw new Error("a NUL byte cannot appear in a Postgres string");
|
|
500
|
+
return `'${value.replace(/'/g, "''")}'`;
|
|
501
|
+
}
|
|
502
|
+
/** A quoted identifier. */
|
|
503
|
+
export function pgIdent(name) {
|
|
504
|
+
return `"${name.replace(/"/g, '""')}"`;
|
|
505
|
+
}
|
|
506
|
+
function int32(n) { const b = Buffer.alloc(4); b.writeInt32BE(n); return b; }
|
|
507
|
+
function cstr(s) { return Buffer.concat([Buffer.from(s, "utf8"), Buffer.from([0])]); }
|
|
508
|
+
function md5hex(b) { return crypto.createHash("md5").update(b).digest("hex"); }
|
|
509
|
+
function hmac(key, data) { return crypto.createHmac("sha256", key).update(data).digest(); }
|
|
510
|
+
function cstrings(b, limit = Infinity) {
|
|
511
|
+
const out = [];
|
|
512
|
+
let i = 0;
|
|
513
|
+
while (i < b.length && out.length < limit) {
|
|
514
|
+
const end = b.indexOf(0, i);
|
|
515
|
+
if (end < 0)
|
|
516
|
+
break;
|
|
517
|
+
if (end === i && out.length > 0 && limit === Infinity)
|
|
518
|
+
break; // the list terminator
|
|
519
|
+
out.push(b.toString("utf8", i, end));
|
|
520
|
+
i = end + 1;
|
|
521
|
+
}
|
|
522
|
+
return out;
|
|
523
|
+
}
|
|
524
|
+
function errorFields(b) {
|
|
525
|
+
const f = {};
|
|
526
|
+
let i = 0;
|
|
527
|
+
while (i < b.length && b[i] !== 0) {
|
|
528
|
+
const code = String.fromCharCode(b[i]);
|
|
529
|
+
const end = b.indexOf(0, i + 1);
|
|
530
|
+
f[code] = b.toString("utf8", i + 1, end);
|
|
531
|
+
i = end + 1;
|
|
532
|
+
}
|
|
533
|
+
return f;
|
|
534
|
+
}
|
|
535
|
+
function rowDescription(b) {
|
|
536
|
+
const n = b.readInt16BE(0);
|
|
537
|
+
const names = [];
|
|
538
|
+
let i = 2;
|
|
539
|
+
for (let k = 0; k < n; k++) {
|
|
540
|
+
const end = b.indexOf(0, i);
|
|
541
|
+
names.push(b.toString("utf8", i, end));
|
|
542
|
+
i = end + 1 + 18; // table oid, column no, type oid, size, modifier, format
|
|
543
|
+
}
|
|
544
|
+
return names;
|
|
545
|
+
}
|
|
546
|
+
/** A RowDescription's columns with their types (rowDescription keeps the names alone). */
|
|
547
|
+
function rowFields(b) {
|
|
548
|
+
const n = b.readInt16BE(0);
|
|
549
|
+
const out = [];
|
|
550
|
+
let i = 2;
|
|
551
|
+
for (let k = 0; k < n; k++) {
|
|
552
|
+
const end = b.indexOf(0, i);
|
|
553
|
+
const name = b.toString("utf8", i, end);
|
|
554
|
+
i = end + 1;
|
|
555
|
+
// table oid (4), column number (2), type oid (4), type size (2), type modifier (4), format (2)
|
|
556
|
+
out.push({ name, typeOid: b.readUInt32BE(i + 6), typmod: b.readInt32BE(i + 12) });
|
|
557
|
+
i += 18;
|
|
558
|
+
}
|
|
559
|
+
return out;
|
|
560
|
+
}
|
|
561
|
+
/** onData pauses the socket above this many queued bytes, and next() resumes it below a quarter of it. */
|
|
562
|
+
const QUEUE_PAUSE_BYTES = 16 * 1024 * 1024;
|
|
563
|
+
function dataRow(b) {
|
|
564
|
+
const n = b.readInt16BE(0);
|
|
565
|
+
const out = [];
|
|
566
|
+
let i = 2;
|
|
567
|
+
for (let k = 0; k < n; k++) {
|
|
568
|
+
const len = b.readInt32BE(i);
|
|
569
|
+
i += 4;
|
|
570
|
+
if (len < 0) {
|
|
571
|
+
out.push(null);
|
|
572
|
+
continue;
|
|
573
|
+
}
|
|
574
|
+
out.push(b.toString("utf8", i, i + len));
|
|
575
|
+
i += len;
|
|
576
|
+
}
|
|
577
|
+
return out;
|
|
578
|
+
}
|
|
579
|
+
/**
|
|
580
|
+
* A socket's error as words: its message, or each address's where Node tried several (`localhost` is ::1 and 127.0.0.1:
|
|
581
|
+
* an AggregateError whose own message is EMPTY — `connecting to localhost:5432: ` said nothing, and `mason start` could
|
|
582
|
+
* not tell "nothing is listening" from a wrong URL), or its code.
|
|
583
|
+
*/
|
|
584
|
+
export function socketErrorText(e) {
|
|
585
|
+
if (e instanceof AggregateError && e.errors.length > 0)
|
|
586
|
+
return e.errors.map((x) => x instanceof Error ? socketErrorText(x) : String(x)).join("; ");
|
|
587
|
+
return e.message || e.code || e.name;
|
|
588
|
+
}
|