@mason-data/cli 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -8
- package/dist/src/api/error.js +4 -0
- package/dist/src/api/postgres/catalog-worker.js +49 -2
- package/dist/src/api/postgres/catalog.js +166 -32
- package/dist/src/api/postgres/listener.js +11 -3
- package/dist/src/api/postgres/proxy.js +129 -0
- package/dist/src/api/postgres/schema.js +6 -0
- package/dist/src/api/postgres/server.js +527 -103
- package/dist/src/api/postgres/translate.js +47 -2
- package/dist/src/cli/background.js +18 -19
- package/dist/src/cli/bridge-stdin.js +101 -0
- package/dist/src/cli/command.js +337 -15
- package/dist/src/cli/commands.js +458 -169
- package/dist/src/cli/credentials.js +182 -0
- package/dist/src/cli/demo.js +2 -2
- package/dist/src/cli/deploy.js +680 -0
- package/dist/src/cli/dev.js +148 -27
- package/dist/src/cli/doctor.js +122 -36
- package/dist/src/cli/generated-files.js +143 -0
- package/dist/src/cli/guides.js +116 -16
- package/dist/src/cli/hosted.js +850 -0
- package/dist/src/cli/init.js +154 -33
- package/dist/src/cli/login.js +292 -0
- package/dist/src/cli/machine.js +125 -40
- package/dist/src/cli/main.js +59 -10
- package/dist/src/cli/mason.cjs +17 -0
- package/dist/src/cli/mcp.js +17 -14
- package/dist/src/cli/output-gate.js +176 -0
- package/dist/src/cli/page-share.js +96 -0
- package/dist/src/cli/pg-tls.js +1 -1
- package/dist/src/cli/pin.js +6 -2
- package/dist/src/cli/pool-size.cjs +85 -0
- package/dist/src/cli/ports.js +2 -0
- package/dist/src/cli/preflight.js +130 -39
- package/dist/src/cli/project.js +100 -1
- package/dist/src/cli/readiness.js +42 -6
- package/dist/src/cli/registry.js +89 -36
- package/dist/src/cli/replica-hosted.js +1324 -0
- package/dist/src/cli/replica.js +114 -0
- package/dist/src/cli/replication-mode.js +5 -0
- package/dist/src/cli/request-filter.js +16 -6
- package/dist/src/cli/roles.js +2 -1
- package/dist/src/cli/run-server.js +179 -9
- package/dist/src/cli/secret-input.js +193 -0
- package/dist/src/cli/security.js +57 -10
- package/dist/src/cli/serve.js +65 -9
- package/dist/src/cli/share-run.js +51 -8
- package/dist/src/cli/share.js +23 -18
- package/dist/src/cli/source-edit.js +10 -4
- package/dist/src/cli/source-role.js +343 -0
- package/dist/src/cli/sources.js +50 -23
- package/dist/src/cli/style.js +16 -1
- package/dist/src/cli/supervise.js +15 -23
- package/dist/src/cli/sync.js +17 -8
- package/dist/src/cli/token-hosted.js +343 -0
- package/dist/src/cli/tunnel.js +1 -1
- package/dist/src/cli/ui-hosted.js +163 -0
- package/dist/src/cli/ui-ssh.js +460 -0
- package/dist/src/cli/ui.js +162 -17
- package/dist/src/cli/usage.js +19 -15
- package/dist/src/cli/user-hosted.js +364 -0
- package/dist/src/cli/user.js +275 -105
- package/dist/src/demo/seed.js +26 -0
- package/dist/src/duck/analyze.js +36 -20
- package/dist/src/duck/collation.js +266 -0
- package/dist/src/duck/constant-key.js +94 -0
- package/dist/src/duck/distinct-key.js +283 -26
- package/dist/src/duck/events.js +39 -8
- package/dist/src/duck/executor-worker.js +2 -2
- package/dist/src/duck/executor.js +6 -5
- package/dist/src/duck/explain.js +4 -0
- package/dist/src/duck/functions.js +448 -49
- package/dist/src/duck/gaps.js +34 -16
- package/dist/src/duck/generated-plan.js +1070 -0
- package/dist/src/duck/generated.js +906 -0
- package/dist/src/duck/grants.js +1 -1
- package/dist/src/duck/held-types.js +64 -0
- package/dist/src/duck/history.js +4 -0
- package/dist/src/duck/jsonb-order.js +9 -12
- package/dist/src/duck/kept-route.js +6 -5
- package/dist/src/duck/kept.js +9 -7
- package/dist/src/duck/late.js +1469 -40
- package/dist/src/duck/libc-bound.js +341 -0
- package/dist/src/duck/libc-order.js +892 -126
- package/dist/src/duck/pg-eval-worker.js +7 -1
- package/dist/src/duck/pg-eval.js +6 -4
- package/dist/src/duck/pgtree.js +774 -23
- package/dist/src/duck/plan.js +275 -30
- package/dist/src/duck/publication.js +14 -0
- package/dist/src/duck/requests.js +9 -5
- package/dist/src/duck/resolved.js +117 -2
- package/dist/src/duck/rewrite.js +4567 -583
- package/dist/src/duck/row-filter.js +12 -6
- package/dist/src/duck/statement.js +19 -0
- package/dist/src/duck/store.js +294 -68
- package/dist/src/duck/stream.js +9 -5
- package/dist/src/duck/sync.js +602 -126
- package/dist/src/duck/token.js +4 -1
- package/dist/src/duck/types.js +63 -7
- package/dist/src/duck/usage.js +6 -4
- package/dist/src/duck/users.js +110 -9
- package/dist/src/duck/verified.js +159 -2
- package/dist/src/duck/views.js +4 -3
- package/dist/src/lib/cidr.js +177 -0
- package/dist/src/lib/database-url.js +55 -10
- package/dist/src/lib/instant.js +50 -0
- package/dist/src/lib/log-buffer.js +13 -4
- package/dist/src/lib/log-read.js +196 -0
- package/dist/src/lib/pg-numeric.js +387 -49
- package/dist/src/lib/secret-mask.js +142 -0
- package/dist/src/lib/table-sync.js +84 -19
- package/dist/src/web/app.js +9 -4
- package/dist/src/web/pages/status.js +178 -13
- package/dist/src/web/pages/syncs.js +1 -1
- package/dist/src/web/pages/tables.js +8 -3
- package/dist/src/web/pages/users.js +4 -3
- package/dist/src/web/shared.js +19 -3
- package/package.json +8 -7
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ADDRESSES AND ADDRESS RANGES, READ BY ONE GRAMMAR: `MASON_PG_PROXY_FROM` (src/api/postgres/proxy.ts) and a `mason
|
|
3
|
+
* user`'s address ranges (src/duck/users.ts `from`) both read an entry here. An entry is an IPv4 or IPv6 address, alone
|
|
4
|
+
* (one address: /32 or /128) or with its prefix length, held to its family (`10.0.0.0/33` and `999.999.999.999/99` are
|
|
5
|
+
* refused), with no bits set past the prefix (`10.0.1.5/24` is refused with its network as the guess), and no host name
|
|
6
|
+
* (never looked up) nor zone index. An address is first held to `node:net`'s own reading (`isIPv4`, `isIPv6`), then
|
|
7
|
+
* read into its bytes here, so nothing either reading refuses is ever taken.
|
|
8
|
+
*
|
|
9
|
+
* EVERY ADDRESS IN ONE SPACE, as Mason means one: IPv6's 128 bits, an IPv4 address at its v4-mapped place
|
|
10
|
+
* (::ffff:a.b.c.d), since a v4-mapped peer or header source IS that IPv4 client (`shownAddress`). What an entry names and
|
|
11
|
+
* what a list covers are both read here, never from how an entry is spelt: `0.0.0.0/1,128.0.0.0/1` is every IPv4
|
|
12
|
+
* address, as `0.0.0.0/0` is, and `::/64` holds every v4-mapped address, so every IPv4 one too.
|
|
13
|
+
*/
|
|
14
|
+
import * as net from "node:net";
|
|
15
|
+
/** An address as Mason shows it: IPv4 dotted, IPv6 in RFC 5952's form, a v4-mapped IPv6 one as IPv4; null when it is no
|
|
16
|
+
* address (undefined, a zone index, a name). */
|
|
17
|
+
export function shownAddress(text) {
|
|
18
|
+
if (text === undefined)
|
|
19
|
+
return null;
|
|
20
|
+
const v4 = ipv4Bytes(text);
|
|
21
|
+
if (v4 !== null)
|
|
22
|
+
return Array.from(v4).join(".");
|
|
23
|
+
const v6 = ipv6Bytes(text);
|
|
24
|
+
return v6 === null ? null : ipv6Text(v6);
|
|
25
|
+
}
|
|
26
|
+
/** An IPv4 address's 4 bytes, from dotted decimal with no leading zero; null for anything else. */
|
|
27
|
+
function ipv4Bytes(text) {
|
|
28
|
+
if (!net.isIPv4(text))
|
|
29
|
+
return null;
|
|
30
|
+
const parts = text.split(".");
|
|
31
|
+
if (parts.length !== 4 || !parts.every((p) => /^(?:0|[1-9]\d{0,2})$/.test(p) && Number(p) <= 255))
|
|
32
|
+
return null;
|
|
33
|
+
return Uint8Array.from(parts.map(Number));
|
|
34
|
+
}
|
|
35
|
+
/** An IPv6 address's 16 bytes (`::` once at most, an IPv4 tail); null for anything else, a zone index included. */
|
|
36
|
+
export function ipv6Bytes(text) {
|
|
37
|
+
// node:net takes a zone index (fe80::1%eth0): an address with one names no place here
|
|
38
|
+
if (!/^[0-9A-Fa-f:.]+$/.test(text) || !net.isIPv6(text))
|
|
39
|
+
return null;
|
|
40
|
+
const halves = text.split("::");
|
|
41
|
+
if (halves.length > 2 || (halves.length === 2 && halves[0].includes(".")))
|
|
42
|
+
return null;
|
|
43
|
+
const groups = (s) => {
|
|
44
|
+
if (s === "")
|
|
45
|
+
return [];
|
|
46
|
+
const out = [];
|
|
47
|
+
const parts = s.split(":");
|
|
48
|
+
for (const [i, p] of parts.entries()) {
|
|
49
|
+
if (i === parts.length - 1 && p.includes(".")) {
|
|
50
|
+
const v4 = ipv4Bytes(p);
|
|
51
|
+
if (v4 === null)
|
|
52
|
+
return null;
|
|
53
|
+
out.push((v4[0] << 8) | v4[1], (v4[2] << 8) | v4[3]);
|
|
54
|
+
}
|
|
55
|
+
else if (/^[0-9A-Fa-f]{1,4}$/.test(p))
|
|
56
|
+
out.push(parseInt(p, 16));
|
|
57
|
+
else
|
|
58
|
+
return null;
|
|
59
|
+
}
|
|
60
|
+
return out;
|
|
61
|
+
};
|
|
62
|
+
const head = groups(halves[0]), tail = halves.length === 2 ? groups(halves[1]) : [];
|
|
63
|
+
if (head === null || tail === null)
|
|
64
|
+
return null;
|
|
65
|
+
const n = head.length + tail.length;
|
|
66
|
+
if (halves.length === 1 ? n !== 8 : n > 7)
|
|
67
|
+
return null;
|
|
68
|
+
const all = [...head, ...new Array(8 - n).fill(0), ...tail];
|
|
69
|
+
return Uint8Array.from(all.flatMap((g) => [g >> 8, g & 0xff]));
|
|
70
|
+
}
|
|
71
|
+
/** whether 16 bytes are an IPv4 address mapped into IPv6 (::ffff:a.b.c.d) */
|
|
72
|
+
const v4Mapped = (b) => b.subarray(0, 10).every((x) => x === 0) && b[10] === 0xff && b[11] === 0xff;
|
|
73
|
+
/** 16 bytes as Mason shows an IPv6 address: RFC 5952 (lower case, the longest run of two or more zero groups as `::`,
|
|
74
|
+
* the first of equal runs), and a v4-mapped one as its IPv4 address. */
|
|
75
|
+
export function ipv6Text(b) {
|
|
76
|
+
if (v4Mapped(b))
|
|
77
|
+
return Array.from(b.subarray(12, 16)).join(".");
|
|
78
|
+
const g = Array.from({ length: 8 }, (_, i) => (b[2 * i] << 8) | b[2 * i + 1]);
|
|
79
|
+
let at = -1, run = 0;
|
|
80
|
+
for (let i = 0; i < 8;) {
|
|
81
|
+
if (g[i] !== 0) {
|
|
82
|
+
i++;
|
|
83
|
+
continue;
|
|
84
|
+
}
|
|
85
|
+
let j = i;
|
|
86
|
+
while (j < 8 && g[j] === 0)
|
|
87
|
+
j++;
|
|
88
|
+
if (j - i > run) {
|
|
89
|
+
at = i;
|
|
90
|
+
run = j - i;
|
|
91
|
+
}
|
|
92
|
+
i = j;
|
|
93
|
+
}
|
|
94
|
+
const hex = g.map((x) => x.toString(16));
|
|
95
|
+
return run < 2 ? hex.join(":") : `${hex.slice(0, at).join(":")}::${hex.slice(at + run).join(":")}`;
|
|
96
|
+
}
|
|
97
|
+
export const V4_START = 0xffffn << 32n;
|
|
98
|
+
export const V4_END = V4_START + (1n << 32n) - 1n;
|
|
99
|
+
export const ALL_END = (1n << 128n) - 1n;
|
|
100
|
+
const bigOf = (b) => b.reduce((n, x) => (n << 8n) | BigInt(x), 0n);
|
|
101
|
+
/** an address's place (`V4_START`), or null when it is no address */
|
|
102
|
+
function placeOf(address) {
|
|
103
|
+
const v4 = ipv4Bytes(address);
|
|
104
|
+
if (v4 !== null)
|
|
105
|
+
return V4_START | bigOf(v4);
|
|
106
|
+
const v6 = ipv6Bytes(address);
|
|
107
|
+
return v6 === null ? null : bigOf(v6);
|
|
108
|
+
}
|
|
109
|
+
/** the places an entry names */
|
|
110
|
+
export function rangeOf(c) {
|
|
111
|
+
const start = c.family === 4 ? V4_START | bigOf(c.bytes) : bigOf(c.bytes);
|
|
112
|
+
return { start, end: start + (1n << BigInt((c.family === 4 ? 32 : 128) - c.prefix)) - 1n };
|
|
113
|
+
}
|
|
114
|
+
/** Whether `address` (as `shownAddress` shows one, or in any spelling of the same address) is in one of `ranges`; never
|
|
115
|
+
* for what is no address. */
|
|
116
|
+
export function inRanges(ranges, address) {
|
|
117
|
+
const at = placeOf(address);
|
|
118
|
+
return at !== null && ranges.some((r) => r.start <= at && at <= r.end);
|
|
119
|
+
}
|
|
120
|
+
/** whether `ranges` together hold every place from `lo` to `hi` */
|
|
121
|
+
export function covers(ranges, lo, hi) {
|
|
122
|
+
let next = lo;
|
|
123
|
+
for (const r of [...ranges].sort((a, b) => (a.start < b.start ? -1 : a.start > b.start ? 1 : 0))) {
|
|
124
|
+
if (r.start > next)
|
|
125
|
+
return false;
|
|
126
|
+
if (r.end >= next)
|
|
127
|
+
next = r.end + 1n;
|
|
128
|
+
if (next > hi)
|
|
129
|
+
return true;
|
|
130
|
+
}
|
|
131
|
+
return next > hi;
|
|
132
|
+
}
|
|
133
|
+
function prefixEqual(a, b, bits) {
|
|
134
|
+
for (let i = 0; i * 8 < bits; i++) {
|
|
135
|
+
const mask = bits - i * 8 >= 8 ? 0xff : (0xff << (8 - (bits - i * 8))) & 0xff;
|
|
136
|
+
if ((a[i] & mask) !== (b[i] & mask))
|
|
137
|
+
return false;
|
|
138
|
+
}
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
/** the network `bytes` / `prefix` names, its host bits cleared */
|
|
142
|
+
function network(bytes, prefix) {
|
|
143
|
+
return bytes.map((x, i) => i * 8 >= prefix ? 0 : prefix - i * 8 >= 8 ? x : x & ((0xff << (8 - (prefix - i * 8))) & 0xff));
|
|
144
|
+
}
|
|
145
|
+
const text = (family, bytes) => family === 4 ? Array.from(bytes).join(".") : ipv6Text(bytes);
|
|
146
|
+
/** An entry as Mason writes it back: the network in `shownAddress`'s form, its prefix only when it is not one address. */
|
|
147
|
+
export const cidrText = (c) => `${text(c.family, c.bytes)}${c.prefix === (c.family === 4 ? 32 : 128) ? "" : `/${c.prefix}`}`;
|
|
148
|
+
/**
|
|
149
|
+
* One entry read, or what is wrong with it, with a did-you-mean where one is a fair guess: a guess is offered only
|
|
150
|
+
* once it is read as an entry itself, so it is never one that would be refused in turn. `everyRefused`: a prefix of 0
|
|
151
|
+
* (every address of a family) is refused with these words (a PROXY header believed from anyone); else it is taken.
|
|
152
|
+
*/
|
|
153
|
+
export function cidrOf(entry, everyRefused = null) {
|
|
154
|
+
const quoted = JSON.stringify(entry);
|
|
155
|
+
const m = /^([^/]*)(?:\/(\d{1,3}))?$/.exec(entry);
|
|
156
|
+
const v4 = m === null ? null : ipv4Bytes(m[1]), v6 = m === null || v4 !== null ? null : ipv6Bytes(m[1]);
|
|
157
|
+
if (m === null || (v4 === null && v6 === null)) {
|
|
158
|
+
return { problem: `${quoted} is not an address or a CIDR (10.0.0.0/16, 2001:db8::/32, or one address such as 10.0.4.7; a host name is not looked up)` };
|
|
159
|
+
}
|
|
160
|
+
const family = v4 !== null ? 4 : 6, bytes = (v4 ?? v6), max = family === 4 ? 32 : 128;
|
|
161
|
+
const prefix = m[2] === undefined ? max : Number(m[2]);
|
|
162
|
+
if (prefix > max)
|
|
163
|
+
return { problem: `${quoted}: an IPv${family} prefix is at most /${max}` };
|
|
164
|
+
if (family === 6 && v4Mapped(bytes) && prefix >= 96) {
|
|
165
|
+
const p4 = prefix - 96, net4 = network(bytes.subarray(12, 16), p4);
|
|
166
|
+
const meant = `${text(4, net4)}${p4 === 32 ? "" : `/${p4}`}`, read = cidrOf(meant, everyRefused);
|
|
167
|
+
return { problem: "problem" in read
|
|
168
|
+
? `${quoted} is IPv4 written as IPv6, which Mason reads as ${meant}, and ${read.problem}`
|
|
169
|
+
: `${quoted} is IPv4 written as IPv6, which Mason reads as IPv4: did you mean ${meant}?` };
|
|
170
|
+
}
|
|
171
|
+
if (prefix === 0 && everyRefused !== null)
|
|
172
|
+
return { problem: `${quoted} names every address, ${everyRefused}` };
|
|
173
|
+
const net = network(bytes, prefix);
|
|
174
|
+
if (!prefixEqual(net, bytes, max))
|
|
175
|
+
return { problem: `${quoted} has bits set past its prefix: did you mean ${text(family, net)}/${prefix}?` };
|
|
176
|
+
return { family, bytes: net, prefix };
|
|
177
|
+
}
|
|
@@ -11,9 +11,57 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Pure; dependency-free.
|
|
13
13
|
*/
|
|
14
|
-
const SSLMODES = new Set(["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]);
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
export const SSLMODES = new Set(["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]);
|
|
15
|
+
/** The query parameters a URL may carry: the only ones read, and so the only ones a URL is shown with (src/cli/preflight.ts shownUrl). */
|
|
16
|
+
export const KNOWN_PARAMS = new Set(["sslmode", "application_name"]);
|
|
17
|
+
/** application_name's value as a URL may give it: what is shown of it, so it can hold nothing else (a `password=…`). */
|
|
18
|
+
const APPLICATION_NAME = /^[A-Za-z0-9._-]{1,64}$/;
|
|
19
|
+
/**
|
|
20
|
+
* The parameters a refusal may NAME: libpq's documented connection keywords and a few drivers' own (Prisma's, node-postgres's),
|
|
21
|
+
* a fixed list, so a key said is Mason's word, never an arbitrary part of the value (issue 541).
|
|
22
|
+
*/
|
|
23
|
+
const NAMED_PARAMS = new Set([
|
|
24
|
+
"host", "hostaddr", "port", "dbname", "user", "password", "passfile", "require_auth", "channel_binding", "connect_timeout",
|
|
25
|
+
"client_encoding", "options", "fallback_application_name", "keepalives", "keepalives_idle", "keepalives_interval",
|
|
26
|
+
"keepalives_count", "tcp_user_timeout", "replication", "gssencmode", "requiressl", "sslnegotiation", "sslcompression",
|
|
27
|
+
"sslcert", "sslkey", "sslpassword", "sslcertmode", "sslrootcert", "sslcrl", "sslcrldir", "sslsni", "requirepeer",
|
|
28
|
+
"ssl_min_protocol_version", "ssl_max_protocol_version", "min_protocol_version", "max_protocol_version", "krbsrvname",
|
|
29
|
+
"gsslib", "gssdelegation", "service", "target_session_attrs", "load_balance_hosts", "scram_client_key",
|
|
30
|
+
"scram_server_key", "oauth_issuer", "oauth_client_id", "oauth_client_secret", "oauth_scope",
|
|
31
|
+
"pgbouncer", "schema", "connection_limit", "pool_timeout", "socket_timeout", "statement_cache_size", "ssl", "sslaccept",
|
|
32
|
+
]);
|
|
33
|
+
/** The parameters that carry a password: where the URL puts it instead is said. */
|
|
34
|
+
const PASSWORD_PARAMS = new Set(["password", "sslpassword"]);
|
|
35
|
+
/**
|
|
36
|
+
* THE ONE READING OF A URL'S QUERY (issue 541): null when every parameter is one the URL is read with, given once, its
|
|
37
|
+
* value one the reading accepts — `sslmode` one of SSLMODES, `application_name` letters, digits, `.`, `_` and `-` (at
|
|
38
|
+
* most 64) — else what is wrong, in words that NAME a parameter only from Mason's own lists and never repeat a value.
|
|
39
|
+
* `parseDatabaseUrl` refuses by it, and `shownUrl` shows a URL only when it is null, so no part shown is one not read.
|
|
40
|
+
*/
|
|
41
|
+
export function queryProblem(params) {
|
|
42
|
+
const keys = [...params.keys()];
|
|
43
|
+
const unread = keys.filter((k) => !KNOWN_PARAMS.has(k));
|
|
44
|
+
if (unread.length) {
|
|
45
|
+
const named = [...new Set(unread.map((k) => k.toLowerCase()).filter((k) => NAMED_PARAMS.has(k)))];
|
|
46
|
+
const others = unread.some((k) => !NAMED_PARAMS.has(k.toLowerCase()));
|
|
47
|
+
const what = named.length === 0 ? "a parameter Mason does not read"
|
|
48
|
+
: `${named.length === 1 ? "the parameter" : "the parameters"} ${named.join(", ")}${others ? ", and another," : ","} which Mason does not read`;
|
|
49
|
+
const way = named.some((k) => PASSWORD_PARAMS.has(k)) ? "a password goes before the host, postgres://user:password@host:5432/your_app"
|
|
50
|
+
: "leave it out of the URL";
|
|
51
|
+
return `it has ${what} (it reads ${[...KNOWN_PARAMS].join(" and ")} alone): ${way}`;
|
|
52
|
+
}
|
|
53
|
+
for (const k of KNOWN_PARAMS)
|
|
54
|
+
if (params.getAll(k).length > 1)
|
|
55
|
+
return `it gives ${k} more than once: give it once`;
|
|
56
|
+
const sslmode = params.get("sslmode");
|
|
57
|
+
if (sslmode !== null && !SSLMODES.has(sslmode))
|
|
58
|
+
return `its sslmode is not one of ${[...SSLMODES].join(", ")}`;
|
|
59
|
+
const name = params.get("application_name");
|
|
60
|
+
if (name !== null && !APPLICATION_NAME.test(name))
|
|
61
|
+
return "its application_name is not letters, digits, . _ and - (at most 64): give it in those, or leave it out";
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/** Parse a `postgres://` / `postgresql://` URL. Throws on anything it cannot represent faithfully, never repeating a part of it. */
|
|
17
65
|
export function parseDatabaseUrl(url) {
|
|
18
66
|
let u;
|
|
19
67
|
try {
|
|
@@ -23,15 +71,12 @@ export function parseDatabaseUrl(url) {
|
|
|
23
71
|
throw new Error("DATABASE_URL is not a URL (expected postgres://user:password@host:port/database)");
|
|
24
72
|
}
|
|
25
73
|
if (u.protocol !== "postgres:" && u.protocol !== "postgresql:") {
|
|
26
|
-
throw new Error(
|
|
27
|
-
}
|
|
28
|
-
for (const key of u.searchParams.keys()) {
|
|
29
|
-
if (!KNOWN_PARAMS.has(key))
|
|
30
|
-
throw new Error(`DATABASE_URL: parameter ${key} is not supported (only ${[...KNOWN_PARAMS].join(", ")})`);
|
|
74
|
+
throw new Error("DATABASE_URL: its scheme is not postgres or postgresql");
|
|
31
75
|
}
|
|
76
|
+
const query = queryProblem(u.searchParams);
|
|
77
|
+
if (query !== null)
|
|
78
|
+
throw new Error(`DATABASE_URL: ${query}`);
|
|
32
79
|
const sslmode = u.searchParams.get("sslmode") ?? "prefer";
|
|
33
|
-
if (!SSLMODES.has(sslmode))
|
|
34
|
-
throw new Error(`DATABASE_URL: sslmode=${sslmode} is not one of ${[...SSLMODES].join(", ")}`);
|
|
35
80
|
const database = decodeURIComponent(u.pathname.replace(/^\//, ""));
|
|
36
81
|
if (!u.hostname)
|
|
37
82
|
throw new Error("DATABASE_URL has no host");
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE SESSION'S INSTANT IS MICROSECONDS (GitHub issues 372, 363): Postgres's transaction start is gettimeofday's
|
|
3
|
+
* microseconds, so `now()`, the clock keywords and a relative word (`'now'`) carry six fraction digits, and a prepared
|
|
4
|
+
* statement's `'now'` and its run's `now()` differ whenever their transactions do, even inside one millisecond. A
|
|
5
|
+
* JavaScript millisecond made them equal there (the flaky oracle of issue 372). So the instant is an integer count of
|
|
6
|
+
* microseconds since the Unix epoch (exact below 2^53, until the year 2255), read from ONE clock (`nowMicros`), written
|
|
7
|
+
* for DuckDB by ONE formatter (`instantText`), and handed to the catalog's PGlite as the wall clock that reads back as
|
|
8
|
+
* exactly that microsecond (`instantWallMs`).
|
|
9
|
+
*/
|
|
10
|
+
/** where the monotonic clock stood at the last re-anchoring, and the microsecond it was read as */
|
|
11
|
+
let anchor = null;
|
|
12
|
+
/**
|
|
13
|
+
* The wall clock in microseconds since the epoch. `wall` (Date.now) decides the millisecond, so the reading never
|
|
14
|
+
* drifts from the clock an operator sets; the monotonic clock (`performance.now()`) only fills the microseconds inside
|
|
15
|
+
* it. The reading is clamped into that millisecond and re-anchored there when the monotonic clock has parted from it
|
|
16
|
+
* (`performance.timeOrigin + performance.now()` alone drifts from the wall clock over a long uptime), so it is
|
|
17
|
+
* non-decreasing while the wall clock is. `wall`: the worker that replaces `Date.now` passes the real one.
|
|
18
|
+
*/
|
|
19
|
+
export function nowMicros(wall = Date.now) {
|
|
20
|
+
const ms = wall();
|
|
21
|
+
const mono = performance.now();
|
|
22
|
+
const lo = ms * 1000, hi = lo + 999;
|
|
23
|
+
anchor ??= { us: Math.floor((performance.timeOrigin + mono) * 1000), mono };
|
|
24
|
+
let us = anchor.us + Math.floor((mono - anchor.mono) * 1000);
|
|
25
|
+
if (us < lo || us > hi) {
|
|
26
|
+
us = us < lo ? lo : hi;
|
|
27
|
+
anchor = { us, mono };
|
|
28
|
+
}
|
|
29
|
+
return us;
|
|
30
|
+
}
|
|
31
|
+
const integral = (us) => {
|
|
32
|
+
if (!Number.isSafeInteger(us))
|
|
33
|
+
throw new Error(`an instant is a whole number of microseconds, not ${us}`);
|
|
34
|
+
return us;
|
|
35
|
+
};
|
|
36
|
+
/** An instant as the text DuckDB binds as a TIMESTAMPTZ: ISO 8601 in UTC with six fraction digits,
|
|
37
|
+
* `2026-01-02T20:00:00.123456Z` (Date's own text stops at the millisecond). */
|
|
38
|
+
export function instantText(us) {
|
|
39
|
+
const ms = Math.floor(integral(us) / 1000);
|
|
40
|
+
return `${new Date(ms).toISOString().slice(0, -1)}${String(us - ms * 1000).padStart(3, "0")}Z`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The `Date.now()` value PGlite reads back as exactly the microsecond `us`. Emscripten's `clock_time_get` takes
|
|
44
|
+
* `Math.round(Date.now() * 1e3 * 1e3)` nanoseconds, a product past 2^53 and so off by up to ~256 ns, and musl's
|
|
45
|
+
* gettimeofday truncates that to microseconds: `us / 1000` lands one microsecond low about half the time. The
|
|
46
|
+
* half-microsecond point lands inside `us` whatever that rounding does (tests/postgres-api/catalog.test.ts).
|
|
47
|
+
*/
|
|
48
|
+
export function instantWallMs(us) {
|
|
49
|
+
return (integral(us) + 0.5) / 1000;
|
|
50
|
+
}
|
|
@@ -9,8 +9,10 @@
|
|
|
9
9
|
*
|
|
10
10
|
* `LogBuffer` is fed by the one place the run writes its log (src/cli/dev.ts `logLine`: mason.log under `mason start`,
|
|
11
11
|
* stdout under `mason serve`), so both modes have it; the command server serves it (`GET /api/logs`). `mason logs` reads
|
|
12
|
-
* the FILE with `logEntries`, the same parser, so a line is the same `{ at, text }` from either.
|
|
12
|
+
* the FILE with `logEntries`, the same parser, so a line is the same `{ at, text }` from either. Both are masked by one
|
|
13
|
+
* mask (`PasswordLineMask`): the file as it is read (src/lib/log-read.ts), the buffer as an entry is kept.
|
|
13
14
|
*/
|
|
15
|
+
import { PasswordLineMask } from "./secret-mask.js";
|
|
14
16
|
/**
|
|
15
17
|
* How many entries the run keeps: a startup and a copy of a hundred tables log a few thousand lines, and an agent asking
|
|
16
18
|
* "what just happened" wants the last minutes, not the file (which `mason logs` reads whole). 2000 entries at a few hundred
|
|
@@ -25,10 +27,15 @@ const HEAD = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z) /;
|
|
|
25
27
|
* be an empty entry counted by `-n`.
|
|
26
28
|
*/
|
|
27
29
|
export function logEntries(text, o = {}) {
|
|
28
|
-
const out = [];
|
|
29
30
|
const lines = text.split("\n");
|
|
30
31
|
if (lines[lines.length - 1] === "")
|
|
31
32
|
lines.pop(); // the file's last newline ends its last line, it does not start another
|
|
33
|
+
return entriesOfLines(lines, o);
|
|
34
|
+
}
|
|
35
|
+
/** Lines (no newline), as `logEntries` reads a text's: what `mason logs` hands it from the reader (src/lib/log-read.ts
|
|
36
|
+
* `readLines`), so a long line is never joined to the next and split again. */
|
|
37
|
+
export function entriesOfLines(lines, o = {}) {
|
|
38
|
+
const out = [];
|
|
32
39
|
for (const line of lines) {
|
|
33
40
|
if (line.trim() === "") {
|
|
34
41
|
if (o.perLine)
|
|
@@ -52,9 +59,11 @@ export class LogBuffer {
|
|
|
52
59
|
capacity;
|
|
53
60
|
kept = [];
|
|
54
61
|
constructor(capacity = LOG_KEPT) { this.capacity = capacity; }
|
|
55
|
-
/** An entry as `textEntry` wrote it (one head line, the rest indented): kept as ONE entry
|
|
62
|
+
/** An entry as `textEntry` wrote it (one head line, the rest indented): kept as ONE entry, each of its lines masked
|
|
63
|
+
* as a log file's are read (src/lib/log-read.ts, issue 621), so `/api/logs` shows a line as `mason logs` does. */
|
|
56
64
|
push(entry) {
|
|
57
|
-
|
|
65
|
+
const mask = new PasswordLineMask();
|
|
66
|
+
for (const l of logEntries(entry.split("\n").map((line) => mask.line(line)).join("\n"))) {
|
|
58
67
|
this.kept.push(l);
|
|
59
68
|
if (this.kept.length > this.capacity)
|
|
60
69
|
this.kept.shift();
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE RUN'S LOG FILES, AND THEIR ONE READER (issue 621: `mason logs` showed lines an older Mason wrote with the source's
|
|
3
|
+
* password in a `password=` value, because it read the file as it was).
|
|
4
|
+
*
|
|
5
|
+
* `.mason/mason.log` (the run's log) and `.mason/dev.log` (the run's terminal, `mason start`'s) are named here alone, and
|
|
6
|
+
* every reader of either reads it through `LogReader`, which masks each line as it is read (`PasswordLineMask`,
|
|
7
|
+
* src/lib/secret-mask.ts): `mason logs` (whole, `--start`, `-f`), `mason start`'s own watch of the run's terminal and the
|
|
8
|
+
* supervisor's scan of it after a run exits (src/cli/background.ts, src/cli/supervise.ts). The run's memory of its log
|
|
9
|
+
* (src/lib/log-buffer.ts, `GET /api/logs`) is the same mask, applied as an entry is kept. So a line an older Mason wrote
|
|
10
|
+
* is shown masked wherever it is shown, and the file itself is never rewritten: a log a person or a collector already
|
|
11
|
+
* read cannot be unread (the release notes of 599 say to rotate the password and delete the old files).
|
|
12
|
+
*
|
|
13
|
+
* WHAT IS MASKED is #620's decision, kept: a libpq `password=` value, the one form the old string leaked, and never a
|
|
14
|
+
* substring of the password now, which a table's, a user's or a slot's name may hold (docs/DESIGN.md "How wide the mask
|
|
15
|
+
* is, and why").
|
|
16
|
+
*
|
|
17
|
+
* A LINE IS RELEASED ONLY WHOLE, AND READ ONCE MORE AT MOST. The reader reads the file in chunks (a log with every
|
|
18
|
+
* statement traced runs to hundreds of MB), and a chunk ends anywhere — inside a `password='…'`, inside a character's
|
|
19
|
+
* bytes. So it decodes and masks only the bytes up to a chunk's last newline, a line at a time, and keeps NOTHING of the
|
|
20
|
+
* line after it but where it starts in the file: when a later chunk ends that line, its bytes are read again from there,
|
|
21
|
+
* once, and decoded whole (the review of PR 641: a line held as text and joined to every chunk cost a line of L bytes
|
|
22
|
+
* L²/chunk, and twice the base's memory). A newline byte is never inside a UTF-8 character, so a line's bytes decode as
|
|
23
|
+
* the whole file's would. A value that runs on past its line is the mask's to carry (`PasswordLineMask`). What is read is
|
|
24
|
+
* the same however the file is cut; tests/log-read.test.ts holds it, and the reader's work to a count of the bytes it
|
|
25
|
+
* reads; tests/conventions.test.ts refuses a log file read in src/ any other way.
|
|
26
|
+
*
|
|
27
|
+
* A FOLLOW SHOWS THE LINE NOT ENDED YET (`follow`, `mason logs -f`): a run killed mid-write leaves its last words with
|
|
28
|
+
* no newline, so at the file's end the follow shows that line as it stands, masked, and then only what more of it adds.
|
|
29
|
+
*/
|
|
30
|
+
import * as fs from "node:fs";
|
|
31
|
+
import * as path from "node:path";
|
|
32
|
+
import { StringDecoder } from "node:string_decoder";
|
|
33
|
+
import { PasswordLineMask } from "./secret-mask.js";
|
|
34
|
+
/** The run's log, in the project's `.mason/`: `mason logs`. */
|
|
35
|
+
export const RUN_LOG = "mason.log";
|
|
36
|
+
/** The run's terminal, in the project's `.mason/`: `mason logs --start`. */
|
|
37
|
+
export const START_LOG = "dev.log";
|
|
38
|
+
/** A log file's path in a project's `.mason/` folder: the run's log, or its terminal (`start`). */
|
|
39
|
+
export function logPath(home, which) {
|
|
40
|
+
return path.join(home, which === "run" ? RUN_LOG : START_LOG);
|
|
41
|
+
}
|
|
42
|
+
/** How much of the file one read takes, at most. */
|
|
43
|
+
const CHUNK = 1 << 20;
|
|
44
|
+
/** A log file read from an offset on, its lines masked as they are read. */
|
|
45
|
+
export class LogReader {
|
|
46
|
+
file;
|
|
47
|
+
/** the next byte a read takes */
|
|
48
|
+
at;
|
|
49
|
+
/** where the line not ended yet starts: the bytes from here to `at` are read, and not yet released */
|
|
50
|
+
lineStart;
|
|
51
|
+
chunk;
|
|
52
|
+
mask = new PasswordLineMask();
|
|
53
|
+
/** what `follow` has shown of the line not ended yet, masked */
|
|
54
|
+
shown = "";
|
|
55
|
+
/** the file was begun again under a line `follow` had shown part of: the next it shows starts a line */
|
|
56
|
+
broken = false;
|
|
57
|
+
/** the offset `follow` last showed the unended line at: unchanged, it is not read again */
|
|
58
|
+
peeked = -1;
|
|
59
|
+
readBytes = 0;
|
|
60
|
+
searchedBytes = 0;
|
|
61
|
+
/** `from`: the byte to start at (default the file's start); `chunk`: the most one read takes (a test's to set). */
|
|
62
|
+
constructor(file, o = {}) {
|
|
63
|
+
this.file = file;
|
|
64
|
+
this.at = o.from ?? 0;
|
|
65
|
+
this.lineStart = this.at;
|
|
66
|
+
this.chunk = o.chunk ?? CHUNK;
|
|
67
|
+
}
|
|
68
|
+
/** The bytes read so far: the file's offset the next read starts at. */
|
|
69
|
+
get offset() { return this.at; }
|
|
70
|
+
/** The reader's own work, for a test to hold linear: the bytes it read from the file (a line's again once it ends) and
|
|
71
|
+
* those it searched for a newline, whatever the lines' lengths and the chunks. */
|
|
72
|
+
get work() { return { read: this.readBytes, searched: this.searchedBytes }; }
|
|
73
|
+
/**
|
|
74
|
+
* The lines the next chunk completes, each masked and ending in its newline ("" when the chunk completed none); null
|
|
75
|
+
* once the file has no byte more now. A file shorter than what was read was begun again (a new run's log): it is
|
|
76
|
+
* read from its start, nothing kept of the old.
|
|
77
|
+
*/
|
|
78
|
+
read() {
|
|
79
|
+
const lines = this.next();
|
|
80
|
+
if (lines === null)
|
|
81
|
+
return null;
|
|
82
|
+
let out = "";
|
|
83
|
+
for (const l of lines)
|
|
84
|
+
out += `${l}\n`;
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
/** Every line completed from where the reader is to the file's end now, masked, each without its newline: `mason
|
|
88
|
+
* logs`' (src/lib/log-buffer.ts `entriesOfLines`), so a line of hundreds of MB is never copied again to be split. */
|
|
89
|
+
readLines() {
|
|
90
|
+
const out = [];
|
|
91
|
+
for (let lines = this.next(); lines !== null; lines = this.next())
|
|
92
|
+
for (const l of lines)
|
|
93
|
+
out.push(l);
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Every line completed from where the reader is to the file's end now, masked. */
|
|
97
|
+
readAll() {
|
|
98
|
+
let out = "";
|
|
99
|
+
for (let t = this.read(); t !== null; t = this.read())
|
|
100
|
+
out += t;
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
/** The last line, which no newline ended yet, masked: read once the reader is done with the file ("" when none). */
|
|
104
|
+
rest() {
|
|
105
|
+
const last = this.bytes(this.lineStart, this.at).toString("utf8");
|
|
106
|
+
this.lineStart = this.at;
|
|
107
|
+
return last === "" ? "" : this.mask.line(last);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* WHAT A FOLLOW PRINTS NEXT (`mason logs -f`): every line written since, and then, the file's end reached, the line not
|
|
111
|
+
* ended yet as it stands, masked (the review of PR 641: a run killed mid-write leaves its last words unended, and a
|
|
112
|
+
* follow that waited for their newline never showed them). Printed one after another, the answers are the masked
|
|
113
|
+
* lines with the unended one's masked text so far; once more of it comes, only what its mask adds is printed, and
|
|
114
|
+
* should its mask read differently with more of it (never seen: a mask of a prefix is the whole mask's prefix), the
|
|
115
|
+
* line is shown again whole on a line of its own, never a byte unmasked. A character whose bytes are not all written
|
|
116
|
+
* yet waits for them.
|
|
117
|
+
*/
|
|
118
|
+
follow() {
|
|
119
|
+
let out = "";
|
|
120
|
+
const ended = this.readAll();
|
|
121
|
+
if (this.broken) {
|
|
122
|
+
out += "\n";
|
|
123
|
+
this.broken = false;
|
|
124
|
+
}
|
|
125
|
+
if (ended !== "") {
|
|
126
|
+
out += this.after(ended);
|
|
127
|
+
this.shown = "";
|
|
128
|
+
}
|
|
129
|
+
if (this.at !== this.peeked) {
|
|
130
|
+
this.peeked = this.at;
|
|
131
|
+
const unended = new StringDecoder("utf8").write(this.bytes(this.lineStart, this.at));
|
|
132
|
+
const now = unended === "" ? "" : this.mask.peek(unended);
|
|
133
|
+
if (now !== this.shown) {
|
|
134
|
+
out += this.after(now);
|
|
135
|
+
this.shown = now;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
/** `text` as it continues what `follow` has shown of the line: what it adds, or itself on a line of its own. */
|
|
141
|
+
after(text) {
|
|
142
|
+
if (this.shown === "")
|
|
143
|
+
return text;
|
|
144
|
+
return text.startsWith(this.shown) ? text.slice(this.shown.length) : `\n${text}`;
|
|
145
|
+
}
|
|
146
|
+
/** The lines the next chunk completes, masked, without their newlines; null once the file has no byte more now. */
|
|
147
|
+
next() {
|
|
148
|
+
const size = fs.statSync(this.file).size;
|
|
149
|
+
if (size < this.at) {
|
|
150
|
+
this.at = 0;
|
|
151
|
+
this.lineStart = 0;
|
|
152
|
+
this.mask = new PasswordLineMask();
|
|
153
|
+
if (this.shown !== "")
|
|
154
|
+
this.broken = true;
|
|
155
|
+
this.shown = "";
|
|
156
|
+
this.peeked = -1;
|
|
157
|
+
}
|
|
158
|
+
if (size === this.at)
|
|
159
|
+
return null;
|
|
160
|
+
const buf = this.bytes(this.at, Math.min(this.at + this.chunk, size));
|
|
161
|
+
if (buf.length === 0)
|
|
162
|
+
return null;
|
|
163
|
+
const from = this.at;
|
|
164
|
+
this.at += buf.length;
|
|
165
|
+
// the chunk alone is searched: of a line it does not end, nothing is kept but where it starts
|
|
166
|
+
this.searchedBytes += buf.length;
|
|
167
|
+
const last = buf.lastIndexOf(10);
|
|
168
|
+
if (last === -1)
|
|
169
|
+
return [];
|
|
170
|
+
const end = from + last + 1;
|
|
171
|
+
// the lines it ends, from the first one's start: read again from the file once when that start is in an earlier
|
|
172
|
+
// chunk, so a line is had whole without being held
|
|
173
|
+
const whole = this.lineStart === from ? buf.subarray(0, last + 1) : this.bytes(this.lineStart, end);
|
|
174
|
+
this.lineStart = end;
|
|
175
|
+
const lines = whole.toString("utf8").split("\n");
|
|
176
|
+
lines.pop(); // the last newline ends its line, it does not start another
|
|
177
|
+
return lines.map((l) => this.mask.line(l));
|
|
178
|
+
}
|
|
179
|
+
/** The file's bytes from `start` to `end` (fewer when it is shorter now), counted as read. */
|
|
180
|
+
bytes(start, end) {
|
|
181
|
+
if (end <= start)
|
|
182
|
+
return Buffer.alloc(0);
|
|
183
|
+
const buf = Buffer.alloc(end - start);
|
|
184
|
+
const fd = fs.openSync(this.file, "r");
|
|
185
|
+
let n = 0;
|
|
186
|
+
try {
|
|
187
|
+
for (let got = -1; n < buf.length && got !== 0; n += got)
|
|
188
|
+
got = fs.readSync(fd, buf, n, buf.length - n, start + n);
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
fs.closeSync(fd);
|
|
192
|
+
}
|
|
193
|
+
this.readBytes += n;
|
|
194
|
+
return n === buf.length ? buf : buf.subarray(0, n);
|
|
195
|
+
}
|
|
196
|
+
}
|