@mason-data/cli 0.7.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +101 -73
- package/dist/src/api/error.js +3 -0
- package/dist/src/api/postgres/catalog-worker.js +49 -2
- package/dist/src/api/postgres/catalog.js +368 -25
- package/dist/src/api/postgres/encode.js +3 -1
- package/dist/src/api/postgres/explain.js +3 -2
- package/dist/src/api/postgres/listener.js +11 -3
- package/dist/src/api/postgres/protocol.js +39 -23
- package/dist/src/api/postgres/proxy.js +283 -0
- package/dist/src/api/postgres/schema.js +6 -0
- package/dist/src/api/postgres/server.js +1617 -245
- package/dist/src/api/postgres/translate.js +127 -10
- package/dist/src/api/request-log.js +18 -5
- package/dist/src/cdc/poll.js +5 -5
- package/dist/src/cli/background.js +47 -25
- package/dist/src/cli/bridge-stdin.js +101 -0
- package/dist/src/cli/cancel.js +75 -0
- package/dist/src/cli/command.js +170 -11
- package/dist/src/cli/commands.js +804 -209
- package/dist/src/cli/control.js +59 -0
- package/dist/src/cli/credentials.js +182 -0
- package/dist/src/cli/demo.js +20 -11
- package/dist/src/cli/deploy.js +574 -0
- package/dist/src/cli/dev.js +491 -130
- package/dist/src/cli/doctor.js +131 -38
- package/dist/src/cli/generated-files.js +143 -0
- package/dist/src/cli/guides.js +175 -36
- package/dist/src/cli/hosted.js +850 -0
- package/dist/src/cli/init.js +162 -65
- package/dist/src/cli/login.js +289 -0
- package/dist/src/cli/machine.js +13 -12
- package/dist/src/cli/main.js +11 -5
- package/dist/src/cli/mason.cjs +17 -0
- package/dist/src/cli/mcp.js +31 -22
- package/dist/src/cli/metrics.js +79 -0
- package/dist/src/cli/output-gate.js +176 -0
- package/dist/src/cli/page-share.js +96 -0
- package/dist/src/cli/pg-tls.js +14 -4
- package/dist/src/cli/pg-url.js +4 -0
- package/dist/src/cli/pin.js +19 -14
- package/dist/src/cli/pool-size.cjs +85 -0
- package/dist/src/cli/ports.js +2 -0
- package/dist/src/cli/preflight.js +138 -46
- package/dist/src/cli/project.js +142 -4
- package/dist/src/cli/readiness.js +76 -6
- package/dist/src/cli/registry.js +65 -38
- package/dist/src/cli/replica-hosted.js +665 -0
- package/dist/src/cli/replica.js +76 -0
- package/dist/src/cli/replication-checks.js +7 -5
- package/dist/src/cli/replication-mode.js +105 -26
- package/dist/src/cli/request-filter.js +134 -0
- package/dist/src/cli/roles.js +585 -0
- package/dist/src/cli/run-server.js +637 -48
- package/dist/src/cli/runs.js +1 -1
- package/dist/src/cli/security.js +67 -18
- package/dist/src/cli/serve.js +204 -46
- package/dist/src/cli/share-run.js +329 -0
- package/dist/src/cli/share.js +286 -0
- package/dist/src/cli/source-edit.js +75 -17
- package/dist/src/cli/source-role.js +344 -0
- package/dist/src/cli/sources.js +306 -47
- package/dist/src/cli/style.js +2 -1
- package/dist/src/cli/supervise.js +27 -24
- package/dist/src/cli/sync.js +55 -25
- package/dist/src/cli/token-hosted.js +337 -0
- package/dist/src/cli/tunnel.js +200 -0
- package/dist/src/cli/ui-ssh.js +460 -0
- package/dist/src/cli/ui.js +124 -17
- package/dist/src/cli/usage.js +128 -0
- package/dist/src/cli/user-hosted.js +299 -0
- package/dist/src/cli/user.js +573 -65
- package/dist/src/cli/version.js +9 -7
- package/dist/src/demo/seed.js +29 -0
- package/dist/src/duck/analyze.js +36 -20
- package/dist/src/duck/cancel.js +10 -1
- package/dist/src/duck/collation.js +266 -0
- package/dist/src/duck/constant-key.js +94 -0
- package/dist/src/duck/distinct-key.js +283 -26
- package/dist/src/duck/events.js +165 -0
- package/dist/src/duck/executor-worker.js +20 -11
- package/dist/src/duck/executor.js +47 -33
- package/dist/src/duck/explain.js +60 -46
- package/dist/src/duck/extensions.js +160 -0
- package/dist/src/duck/functions.js +461 -63
- package/dist/src/duck/gaps.js +48 -21
- package/dist/src/duck/generated-plan.js +1070 -0
- package/dist/src/duck/generated.js +906 -0
- package/dist/src/duck/grants.js +176 -0
- package/dist/src/duck/held-types.js +64 -0
- package/dist/src/duck/history.js +127 -0
- package/dist/src/duck/jsonb-order.js +9 -12
- package/dist/src/duck/kept-route.js +6 -5
- package/dist/src/duck/kept.js +15 -13
- package/dist/src/duck/late.js +1469 -40
- package/dist/src/duck/libc-bound.js +341 -0
- package/dist/src/duck/libc-order.js +892 -126
- package/dist/src/duck/pg-eval-worker.js +7 -1
- package/dist/src/duck/pg-eval.js +6 -4
- package/dist/src/duck/pgtree.js +774 -23
- package/dist/src/duck/plan.js +369 -47
- package/dist/src/duck/publication.js +20 -6
- package/dist/src/duck/requests.js +269 -93
- package/dist/src/duck/resolved.js +117 -2
- package/dist/src/duck/rewrite.js +4584 -580
- package/dist/src/duck/row-filter.js +655 -0
- package/dist/src/duck/statement.js +19 -0
- package/dist/src/duck/store.js +439 -177
- package/dist/src/duck/stream.js +19 -10
- package/dist/src/duck/sync.js +1411 -368
- package/dist/src/duck/token.js +4 -1
- package/dist/src/duck/types.js +69 -12
- package/dist/src/duck/usage.js +326 -0
- package/dist/src/duck/users.js +69 -15
- package/dist/src/duck/verified.js +161 -1
- package/dist/src/duck/views.js +5 -5
- package/dist/src/lib/database-name.js +62 -0
- package/dist/src/lib/database-url.js +55 -10
- package/dist/src/lib/dotenv.js +17 -4
- package/dist/src/lib/instant.js +50 -0
- package/dist/src/lib/log-buffer.js +77 -0
- package/dist/src/lib/log-read.js +196 -0
- package/dist/src/lib/pg-numeric.js +390 -49
- package/dist/src/lib/project-version.js +23 -23
- package/dist/src/lib/secret-mask.js +142 -0
- package/dist/src/lib/table-sync.js +282 -0
- package/dist/src/web/app.css +706 -66
- package/dist/src/web/app.js +331 -409
- package/dist/src/web/charts.js +229 -0
- package/dist/src/web/index.html +82 -18
- package/dist/src/web/pages/queries.js +192 -0
- package/dist/src/web/pages/status.js +611 -0
- package/dist/src/web/pages/syncs.js +155 -0
- package/dist/src/web/pages/tables.js +439 -0
- package/dist/src/web/pages/usage.js +397 -0
- package/dist/src/web/pages/users.js +219 -0
- package/dist/src/web/shared.js +584 -0
- package/dist/src/web/vendor/README.md +25 -0
- package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
- package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
- package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
- package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
- package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
- package/package.json +12 -4
|
@@ -11,9 +11,57 @@
|
|
|
11
11
|
*
|
|
12
12
|
* Pure; dependency-free.
|
|
13
13
|
*/
|
|
14
|
-
const SSLMODES = new Set(["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]);
|
|
15
|
-
|
|
16
|
-
|
|
14
|
+
export const SSLMODES = new Set(["disable", "allow", "prefer", "require", "verify-ca", "verify-full"]);
|
|
15
|
+
/** The query parameters a URL may carry: the only ones read, and so the only ones a URL is shown with (src/cli/preflight.ts shownUrl). */
|
|
16
|
+
export const KNOWN_PARAMS = new Set(["sslmode", "application_name"]);
|
|
17
|
+
/** application_name's value as a URL may give it: what is shown of it, so it can hold nothing else (a `password=…`). */
|
|
18
|
+
const APPLICATION_NAME = /^[A-Za-z0-9._-]{1,64}$/;
|
|
19
|
+
/**
|
|
20
|
+
* The parameters a refusal may NAME: libpq's documented connection keywords and a few drivers' own (Prisma's, node-postgres's),
|
|
21
|
+
* a fixed list, so a key said is Mason's word, never an arbitrary part of the value (issue 541).
|
|
22
|
+
*/
|
|
23
|
+
const NAMED_PARAMS = new Set([
|
|
24
|
+
"host", "hostaddr", "port", "dbname", "user", "password", "passfile", "require_auth", "channel_binding", "connect_timeout",
|
|
25
|
+
"client_encoding", "options", "fallback_application_name", "keepalives", "keepalives_idle", "keepalives_interval",
|
|
26
|
+
"keepalives_count", "tcp_user_timeout", "replication", "gssencmode", "requiressl", "sslnegotiation", "sslcompression",
|
|
27
|
+
"sslcert", "sslkey", "sslpassword", "sslcertmode", "sslrootcert", "sslcrl", "sslcrldir", "sslsni", "requirepeer",
|
|
28
|
+
"ssl_min_protocol_version", "ssl_max_protocol_version", "min_protocol_version", "max_protocol_version", "krbsrvname",
|
|
29
|
+
"gsslib", "gssdelegation", "service", "target_session_attrs", "load_balance_hosts", "scram_client_key",
|
|
30
|
+
"scram_server_key", "oauth_issuer", "oauth_client_id", "oauth_client_secret", "oauth_scope",
|
|
31
|
+
"pgbouncer", "schema", "connection_limit", "pool_timeout", "socket_timeout", "statement_cache_size", "ssl", "sslaccept",
|
|
32
|
+
]);
|
|
33
|
+
/** The parameters that carry a password: where the URL puts it instead is said. */
|
|
34
|
+
const PASSWORD_PARAMS = new Set(["password", "sslpassword"]);
|
|
35
|
+
/**
|
|
36
|
+
* THE ONE READING OF A URL'S QUERY (issue 541): null when every parameter is one the URL is read with, given once, its
|
|
37
|
+
* value one the reading accepts — `sslmode` one of SSLMODES, `application_name` letters, digits, `.`, `_` and `-` (at
|
|
38
|
+
* most 64) — else what is wrong, in words that NAME a parameter only from Mason's own lists and never repeat a value.
|
|
39
|
+
* `parseDatabaseUrl` refuses by it, and `shownUrl` shows a URL only when it is null, so no part shown is one not read.
|
|
40
|
+
*/
|
|
41
|
+
export function queryProblem(params) {
|
|
42
|
+
const keys = [...params.keys()];
|
|
43
|
+
const unread = keys.filter((k) => !KNOWN_PARAMS.has(k));
|
|
44
|
+
if (unread.length) {
|
|
45
|
+
const named = [...new Set(unread.map((k) => k.toLowerCase()).filter((k) => NAMED_PARAMS.has(k)))];
|
|
46
|
+
const others = unread.some((k) => !NAMED_PARAMS.has(k.toLowerCase()));
|
|
47
|
+
const what = named.length === 0 ? "a parameter Mason does not read"
|
|
48
|
+
: `${named.length === 1 ? "the parameter" : "the parameters"} ${named.join(", ")}${others ? ", and another," : ","} which Mason does not read`;
|
|
49
|
+
const way = named.some((k) => PASSWORD_PARAMS.has(k)) ? "a password goes before the host, postgres://user:password@host:5432/your_app"
|
|
50
|
+
: "leave it out of the URL";
|
|
51
|
+
return `it has ${what} (it reads ${[...KNOWN_PARAMS].join(" and ")} alone): ${way}`;
|
|
52
|
+
}
|
|
53
|
+
for (const k of KNOWN_PARAMS)
|
|
54
|
+
if (params.getAll(k).length > 1)
|
|
55
|
+
return `it gives ${k} more than once: give it once`;
|
|
56
|
+
const sslmode = params.get("sslmode");
|
|
57
|
+
if (sslmode !== null && !SSLMODES.has(sslmode))
|
|
58
|
+
return `its sslmode is not one of ${[...SSLMODES].join(", ")}`;
|
|
59
|
+
const name = params.get("application_name");
|
|
60
|
+
if (name !== null && !APPLICATION_NAME.test(name))
|
|
61
|
+
return "its application_name is not letters, digits, . _ and - (at most 64): give it in those, or leave it out";
|
|
62
|
+
return null;
|
|
63
|
+
}
|
|
64
|
+
/** Parse a `postgres://` / `postgresql://` URL. Throws on anything it cannot represent faithfully, never repeating a part of it. */
|
|
17
65
|
export function parseDatabaseUrl(url) {
|
|
18
66
|
let u;
|
|
19
67
|
try {
|
|
@@ -23,15 +71,12 @@ export function parseDatabaseUrl(url) {
|
|
|
23
71
|
throw new Error("DATABASE_URL is not a URL (expected postgres://user:password@host:port/database)");
|
|
24
72
|
}
|
|
25
73
|
if (u.protocol !== "postgres:" && u.protocol !== "postgresql:") {
|
|
26
|
-
throw new Error(
|
|
27
|
-
}
|
|
28
|
-
for (const key of u.searchParams.keys()) {
|
|
29
|
-
if (!KNOWN_PARAMS.has(key))
|
|
30
|
-
throw new Error(`DATABASE_URL: parameter ${key} is not supported (only ${[...KNOWN_PARAMS].join(", ")})`);
|
|
74
|
+
throw new Error("DATABASE_URL: its scheme is not postgres or postgresql");
|
|
31
75
|
}
|
|
76
|
+
const query = queryProblem(u.searchParams);
|
|
77
|
+
if (query !== null)
|
|
78
|
+
throw new Error(`DATABASE_URL: ${query}`);
|
|
32
79
|
const sslmode = u.searchParams.get("sslmode") ?? "prefer";
|
|
33
|
-
if (!SSLMODES.has(sslmode))
|
|
34
|
-
throw new Error(`DATABASE_URL: sslmode=${sslmode} is not one of ${[...SSLMODES].join(", ")}`);
|
|
35
80
|
const database = decodeURIComponent(u.pathname.replace(/^\//, ""));
|
|
36
81
|
if (!u.hostname)
|
|
37
82
|
throw new Error("DATABASE_URL has no host");
|
package/dist/src/lib/dotenv.js
CHANGED
|
@@ -32,7 +32,7 @@ export function readEnv(root, processEnv = process.env) {
|
|
|
32
32
|
return env;
|
|
33
33
|
}
|
|
34
34
|
/**
|
|
35
|
-
* A VARIABLE OF AN APP'S FOLDER (`mason init`'s DATABASE_URL, and a source file's `url: ${NAME}`,
|
|
35
|
+
* A VARIABLE OF AN APP'S FOLDER (`mason init`'s MASON_DATABASE_URL or DATABASE_URL, and a source file's `url: ${NAME}`,
|
|
36
36
|
* src/cli/sources.ts): `name` from the folder's `.env.local`, then `.env` (an `export ` before a line is
|
|
37
37
|
* allowed, as shells write it), then the environment — the order web frameworks read them in, the app's own files
|
|
38
38
|
* first because they name the database this folder's app uses. A different question from readEnv's "this process's
|
|
@@ -56,10 +56,23 @@ export function appEnvValue(root, name, processEnv = process.env, o = {}) {
|
|
|
56
56
|
const value = processEnv[name];
|
|
57
57
|
return value ? { value, from: "the shell", file: null } : null;
|
|
58
58
|
}
|
|
59
|
-
/**
|
|
59
|
+
/**
|
|
60
|
+
* The variables `mason init` reads a database's URL from when none is given, in the order it reads them: Mason's own
|
|
61
|
+
* first, so a developer whose app's DATABASE_URL names another database can aim Mason elsewhere; then the app's.
|
|
62
|
+
*/
|
|
63
|
+
export const DATABASE_URL_VARIABLES = ["MASON_DATABASE_URL", "DATABASE_URL"];
|
|
64
|
+
/**
|
|
65
|
+
* What `mason init` reads when no URL is given: the first of `DATABASE_URL_VARIABLES` that is set, each read as
|
|
66
|
+
* `appEnvValue` reads it (`.env.local`, `.env`, then the shell). `variable` is the one read — the variable the sources
|
|
67
|
+
* file then names; `from` says it with where it was found (`MASON_DATABASE_URL in .env`, `DATABASE_URL in the shell`).
|
|
68
|
+
*/
|
|
60
69
|
export function appDatabaseUrl(root, processEnv = process.env) {
|
|
61
|
-
const
|
|
62
|
-
|
|
70
|
+
for (const variable of DATABASE_URL_VARIABLES) {
|
|
71
|
+
const v = appEnvValue(root, variable, processEnv);
|
|
72
|
+
if (v !== null)
|
|
73
|
+
return { url: v.value, variable, from: `${variable} in ${v.from}`, file: v.file };
|
|
74
|
+
}
|
|
75
|
+
return null;
|
|
63
76
|
}
|
|
64
77
|
/**
|
|
65
78
|
* `root/.env` with `line` (`NAME=value`) added under a `# comment`, the file made when there is none — owner-only
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE SESSION'S INSTANT IS MICROSECONDS (GitHub issues 372, 363): Postgres's transaction start is gettimeofday's
|
|
3
|
+
* microseconds, so `now()`, the clock keywords and a relative word (`'now'`) carry six fraction digits, and a prepared
|
|
4
|
+
* statement's `'now'` and its run's `now()` differ whenever their transactions do, even inside one millisecond. A
|
|
5
|
+
* JavaScript millisecond made them equal there (the flaky oracle of issue 372). So the instant is an integer count of
|
|
6
|
+
* microseconds since the Unix epoch (exact below 2^53, until the year 2255), read from ONE clock (`nowMicros`), written
|
|
7
|
+
* for DuckDB by ONE formatter (`instantText`), and handed to the catalog's PGlite as the wall clock that reads back as
|
|
8
|
+
* exactly that microsecond (`instantWallMs`).
|
|
9
|
+
*/
|
|
10
|
+
/** where the monotonic clock stood at the last re-anchoring, and the microsecond it was read as */
|
|
11
|
+
let anchor = null;
|
|
12
|
+
/**
|
|
13
|
+
* The wall clock in microseconds since the epoch. `wall` (Date.now) decides the millisecond, so the reading never
|
|
14
|
+
* drifts from the clock an operator sets; the monotonic clock (`performance.now()`) only fills the microseconds inside
|
|
15
|
+
* it. The reading is clamped into that millisecond and re-anchored there when the monotonic clock has parted from it
|
|
16
|
+
* (`performance.timeOrigin + performance.now()` alone drifts from the wall clock over a long uptime), so it is
|
|
17
|
+
* non-decreasing while the wall clock is. `wall`: the worker that replaces `Date.now` passes the real one.
|
|
18
|
+
*/
|
|
19
|
+
export function nowMicros(wall = Date.now) {
|
|
20
|
+
const ms = wall();
|
|
21
|
+
const mono = performance.now();
|
|
22
|
+
const lo = ms * 1000, hi = lo + 999;
|
|
23
|
+
anchor ??= { us: Math.floor((performance.timeOrigin + mono) * 1000), mono };
|
|
24
|
+
let us = anchor.us + Math.floor((mono - anchor.mono) * 1000);
|
|
25
|
+
if (us < lo || us > hi) {
|
|
26
|
+
us = us < lo ? lo : hi;
|
|
27
|
+
anchor = { us, mono };
|
|
28
|
+
}
|
|
29
|
+
return us;
|
|
30
|
+
}
|
|
31
|
+
const integral = (us) => {
|
|
32
|
+
if (!Number.isSafeInteger(us))
|
|
33
|
+
throw new Error(`an instant is a whole number of microseconds, not ${us}`);
|
|
34
|
+
return us;
|
|
35
|
+
};
|
|
36
|
+
/** An instant as the text DuckDB binds as a TIMESTAMPTZ: ISO 8601 in UTC with six fraction digits,
|
|
37
|
+
* `2026-01-02T20:00:00.123456Z` (Date's own text stops at the millisecond). */
|
|
38
|
+
export function instantText(us) {
|
|
39
|
+
const ms = Math.floor(integral(us) / 1000);
|
|
40
|
+
return `${new Date(ms).toISOString().slice(0, -1)}${String(us - ms * 1000).padStart(3, "0")}Z`;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* The `Date.now()` value PGlite reads back as exactly the microsecond `us`. Emscripten's `clock_time_get` takes
|
|
44
|
+
* `Math.round(Date.now() * 1e3 * 1e3)` nanoseconds, a product past 2^53 and so off by up to ~256 ns, and musl's
|
|
45
|
+
* gettimeofday truncates that to microseconds: `us / 1000` lands one microsecond low about half the time. The
|
|
46
|
+
* half-microsecond point lands inside `us` whatever that rounding does (tests/postgres-api/catalog.test.ts).
|
|
47
|
+
*/
|
|
48
|
+
export function instantWallMs(us) {
|
|
49
|
+
return (integral(us) + 0.5) / 1000;
|
|
50
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE RUN'S LOG AS ENTRIES, kept in memory and read back from the file: ONE shape, `{ at, text }`, for both.
|
|
3
|
+
*
|
|
4
|
+
* An entry is what `textEntry` (src/lib/log.ts) writes: a head line (the ISO time, then `[component] message`) and the
|
|
5
|
+
* message's further lines INDENTED under it. A line at the margin starts an entry, an indented one continues it, so a
|
|
6
|
+
* multi-line reason (a stack, a DuckDB error's candidates) is ONE entry here, never its first line and a pile of
|
|
7
|
+
* orphans. `at` is the head's time (null for a line that has none: `mason start`'s terminal, dev.log); `text` is the rest
|
|
8
|
+
* as written, continuation lines included.
|
|
9
|
+
*
|
|
10
|
+
* `LogBuffer` is fed by the one place the run writes its log (src/cli/dev.ts `logLine`: mason.log under `mason start`,
|
|
11
|
+
* stdout under `mason serve`), so both modes have it; the command server serves it (`GET /api/logs`). `mason logs` reads
|
|
12
|
+
* the FILE with `logEntries`, the same parser, so a line is the same `{ at, text }` from either. Both are masked by one
|
|
13
|
+
* mask (`PasswordLineMask`): the file as it is read (src/lib/log-read.ts), the buffer as an entry is kept.
|
|
14
|
+
*/
|
|
15
|
+
import { PasswordLineMask } from "./secret-mask.js";
|
|
16
|
+
/**
|
|
17
|
+
* How many entries the run keeps: a startup and a copy of a hundred tables log a few thousand lines, and an agent asking
|
|
18
|
+
* "what just happened" wants the last minutes, not the file (which `mason logs` reads whole). 2000 entries at a few hundred
|
|
19
|
+
* bytes is under a megabyte of memory, however long the run lives.
|
|
20
|
+
*/
|
|
21
|
+
export const LOG_KEPT = 2000;
|
|
22
|
+
const HEAD = /^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d+)?Z) /;
|
|
23
|
+
/**
|
|
24
|
+
* Text (a file's, or one entry as `textEntry` wrote it) as entries: a margin line starts one, an indented line continues it.
|
|
25
|
+
* `perLine` is a terminal's text (dev.log, `mason start`'s): a line each, a BLANK one kept as `{ at: null, text: "" }` so it
|
|
26
|
+
* prints as written. Without it (mason.log, the buffer) a blank line is no entry: `textEntry` writes none, and one would
|
|
27
|
+
* be an empty entry counted by `-n`.
|
|
28
|
+
*/
|
|
29
|
+
export function logEntries(text, o = {}) {
|
|
30
|
+
const lines = text.split("\n");
|
|
31
|
+
if (lines[lines.length - 1] === "")
|
|
32
|
+
lines.pop(); // the file's last newline ends its last line, it does not start another
|
|
33
|
+
return entriesOfLines(lines, o);
|
|
34
|
+
}
|
|
35
|
+
/** Lines (no newline), as `logEntries` reads a text's: what `mason logs` hands it from the reader (src/lib/log-read.ts
|
|
36
|
+
* `readLines`), so a long line is never joined to the next and split again. */
|
|
37
|
+
export function entriesOfLines(lines, o = {}) {
|
|
38
|
+
const out = [];
|
|
39
|
+
for (const line of lines) {
|
|
40
|
+
if (line.trim() === "") {
|
|
41
|
+
if (o.perLine)
|
|
42
|
+
out.push({ at: null, text: "" });
|
|
43
|
+
continue;
|
|
44
|
+
}
|
|
45
|
+
const last = out[out.length - 1];
|
|
46
|
+
if (!o.perLine && last !== undefined && /^\s/.test(line)) {
|
|
47
|
+
last.text += `\n${line.trimEnd()}`;
|
|
48
|
+
continue;
|
|
49
|
+
}
|
|
50
|
+
const m = HEAD.exec(line);
|
|
51
|
+
out.push(m ? { at: m[1], text: line.slice(m[0].length).trimEnd() } : { at: null, text: line.trimEnd() });
|
|
52
|
+
}
|
|
53
|
+
return out;
|
|
54
|
+
}
|
|
55
|
+
/** An entry as the file has it: the time, a space, the text. */
|
|
56
|
+
export const logLineText = (l) => (l.at === null ? l.text : `${l.at} ${l.text}`);
|
|
57
|
+
/** The last `capacity` entries, oldest dropped first. */
|
|
58
|
+
export class LogBuffer {
|
|
59
|
+
capacity;
|
|
60
|
+
kept = [];
|
|
61
|
+
constructor(capacity = LOG_KEPT) { this.capacity = capacity; }
|
|
62
|
+
/** An entry as `textEntry` wrote it (one head line, the rest indented): kept as ONE entry, each of its lines masked
|
|
63
|
+
* as a log file's are read (src/lib/log-read.ts, issue 621), so `/api/logs` shows a line as `mason logs` does. */
|
|
64
|
+
push(entry) {
|
|
65
|
+
const mask = new PasswordLineMask();
|
|
66
|
+
for (const l of logEntries(entry.split("\n").map((line) => mask.line(line)).join("\n"))) {
|
|
67
|
+
this.kept.push(l);
|
|
68
|
+
if (this.kept.length > this.capacity)
|
|
69
|
+
this.kept.shift();
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
/** The last `lines` entries, oldest first; with `since` (an ISO time) only those after it. */
|
|
73
|
+
recent(o) {
|
|
74
|
+
const from = o.since === undefined ? this.kept : this.kept.filter((l) => l.at !== null && l.at > o.since);
|
|
75
|
+
return o.lines <= 0 ? [] : from.slice(-o.lines);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* THE RUN'S LOG FILES, AND THEIR ONE READER (issue 621: `mason logs` showed lines an older Mason wrote with the source's
|
|
3
|
+
* password in a `password=` value, because it read the file as it was).
|
|
4
|
+
*
|
|
5
|
+
* `.mason/mason.log` (the run's log) and `.mason/dev.log` (the run's terminal, `mason start`'s) are named here alone, and
|
|
6
|
+
* every reader of either reads it through `LogReader`, which masks each line as it is read (`PasswordLineMask`,
|
|
7
|
+
* src/lib/secret-mask.ts): `mason logs` (whole, `--start`, `-f`), `mason start`'s own watch of the run's terminal and the
|
|
8
|
+
* supervisor's scan of it after a run exits (src/cli/background.ts, src/cli/supervise.ts). The run's memory of its log
|
|
9
|
+
* (src/lib/log-buffer.ts, `GET /api/logs`) is the same mask, applied as an entry is kept. So a line an older Mason wrote
|
|
10
|
+
* is shown masked wherever it is shown, and the file itself is never rewritten: a log a person or a collector already
|
|
11
|
+
* read cannot be unread (the release notes of 599 say to rotate the password and delete the old files).
|
|
12
|
+
*
|
|
13
|
+
* WHAT IS MASKED is #620's decision, kept: a libpq `password=` value, the one form the old string leaked, and never a
|
|
14
|
+
* substring of the password now, which a table's, a user's or a slot's name may hold (docs/DESIGN.md "How wide the mask
|
|
15
|
+
* is, and why").
|
|
16
|
+
*
|
|
17
|
+
* A LINE IS RELEASED ONLY WHOLE, AND READ ONCE MORE AT MOST. The reader reads the file in chunks (a log with every
|
|
18
|
+
* statement traced runs to hundreds of MB), and a chunk ends anywhere — inside a `password='…'`, inside a character's
|
|
19
|
+
* bytes. So it decodes and masks only the bytes up to a chunk's last newline, a line at a time, and keeps NOTHING of the
|
|
20
|
+
* line after it but where it starts in the file: when a later chunk ends that line, its bytes are read again from there,
|
|
21
|
+
* once, and decoded whole (the review of PR 641: a line held as text and joined to every chunk cost a line of L bytes
|
|
22
|
+
* L²/chunk, and twice the base's memory). A newline byte is never inside a UTF-8 character, so a line's bytes decode as
|
|
23
|
+
* the whole file's would. A value that runs on past its line is the mask's to carry (`PasswordLineMask`). What is read is
|
|
24
|
+
* the same however the file is cut; tests/log-read.test.ts holds it, and the reader's work to a count of the bytes it
|
|
25
|
+
* reads; tests/conventions.test.ts refuses a log file read in src/ any other way.
|
|
26
|
+
*
|
|
27
|
+
* A FOLLOW SHOWS THE LINE NOT ENDED YET (`follow`, `mason logs -f`): a run killed mid-write leaves its last words with
|
|
28
|
+
* no newline, so at the file's end the follow shows that line as it stands, masked, and then only what more of it adds.
|
|
29
|
+
*/
|
|
30
|
+
import * as fs from "node:fs";
|
|
31
|
+
import * as path from "node:path";
|
|
32
|
+
import { StringDecoder } from "node:string_decoder";
|
|
33
|
+
import { PasswordLineMask } from "./secret-mask.js";
|
|
34
|
+
/** The run's log, in the project's `.mason/`: `mason logs`. */
|
|
35
|
+
export const RUN_LOG = "mason.log";
|
|
36
|
+
/** The run's terminal, in the project's `.mason/`: `mason logs --start`. */
|
|
37
|
+
export const START_LOG = "dev.log";
|
|
38
|
+
/** A log file's path in a project's `.mason/` folder: the run's log, or its terminal (`start`). */
|
|
39
|
+
export function logPath(home, which) {
|
|
40
|
+
return path.join(home, which === "run" ? RUN_LOG : START_LOG);
|
|
41
|
+
}
|
|
42
|
+
/** How much of the file one read takes, at most. */
|
|
43
|
+
const CHUNK = 1 << 20;
|
|
44
|
+
/** A log file read from an offset on, its lines masked as they are read. */
|
|
45
|
+
export class LogReader {
|
|
46
|
+
file;
|
|
47
|
+
/** the next byte a read takes */
|
|
48
|
+
at;
|
|
49
|
+
/** where the line not ended yet starts: the bytes from here to `at` are read, and not yet released */
|
|
50
|
+
lineStart;
|
|
51
|
+
chunk;
|
|
52
|
+
mask = new PasswordLineMask();
|
|
53
|
+
/** what `follow` has shown of the line not ended yet, masked */
|
|
54
|
+
shown = "";
|
|
55
|
+
/** the file was begun again under a line `follow` had shown part of: the next it shows starts a line */
|
|
56
|
+
broken = false;
|
|
57
|
+
/** the offset `follow` last showed the unended line at: unchanged, it is not read again */
|
|
58
|
+
peeked = -1;
|
|
59
|
+
readBytes = 0;
|
|
60
|
+
searchedBytes = 0;
|
|
61
|
+
/** `from`: the byte to start at (default the file's start); `chunk`: the most one read takes (a test's to set). */
|
|
62
|
+
constructor(file, o = {}) {
|
|
63
|
+
this.file = file;
|
|
64
|
+
this.at = o.from ?? 0;
|
|
65
|
+
this.lineStart = this.at;
|
|
66
|
+
this.chunk = o.chunk ?? CHUNK;
|
|
67
|
+
}
|
|
68
|
+
/** The bytes read so far: the file's offset the next read starts at. */
|
|
69
|
+
get offset() { return this.at; }
|
|
70
|
+
/** The reader's own work, for a test to hold linear: the bytes it read from the file (a line's again once it ends) and
|
|
71
|
+
* those it searched for a newline, whatever the lines' lengths and the chunks. */
|
|
72
|
+
get work() { return { read: this.readBytes, searched: this.searchedBytes }; }
|
|
73
|
+
/**
|
|
74
|
+
* The lines the next chunk completes, each masked and ending in its newline ("" when the chunk completed none); null
|
|
75
|
+
* once the file has no byte more now. A file shorter than what was read was begun again (a new run's log): it is
|
|
76
|
+
* read from its start, nothing kept of the old.
|
|
77
|
+
*/
|
|
78
|
+
read() {
|
|
79
|
+
const lines = this.next();
|
|
80
|
+
if (lines === null)
|
|
81
|
+
return null;
|
|
82
|
+
let out = "";
|
|
83
|
+
for (const l of lines)
|
|
84
|
+
out += `${l}\n`;
|
|
85
|
+
return out;
|
|
86
|
+
}
|
|
87
|
+
/** Every line completed from where the reader is to the file's end now, masked, each without its newline: `mason
|
|
88
|
+
* logs`' (src/lib/log-buffer.ts `entriesOfLines`), so a line of hundreds of MB is never copied again to be split. */
|
|
89
|
+
readLines() {
|
|
90
|
+
const out = [];
|
|
91
|
+
for (let lines = this.next(); lines !== null; lines = this.next())
|
|
92
|
+
for (const l of lines)
|
|
93
|
+
out.push(l);
|
|
94
|
+
return out;
|
|
95
|
+
}
|
|
96
|
+
/** Every line completed from where the reader is to the file's end now, masked. */
|
|
97
|
+
readAll() {
|
|
98
|
+
let out = "";
|
|
99
|
+
for (let t = this.read(); t !== null; t = this.read())
|
|
100
|
+
out += t;
|
|
101
|
+
return out;
|
|
102
|
+
}
|
|
103
|
+
/** The last line, which no newline ended yet, masked: read once the reader is done with the file ("" when none). */
|
|
104
|
+
rest() {
|
|
105
|
+
const last = this.bytes(this.lineStart, this.at).toString("utf8");
|
|
106
|
+
this.lineStart = this.at;
|
|
107
|
+
return last === "" ? "" : this.mask.line(last);
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* WHAT A FOLLOW PRINTS NEXT (`mason logs -f`): every line written since, and then, the file's end reached, the line not
|
|
111
|
+
* ended yet as it stands, masked (the review of PR 641: a run killed mid-write leaves its last words unended, and a
|
|
112
|
+
* follow that waited for their newline never showed them). Printed one after another, the answers are the masked
|
|
113
|
+
* lines with the unended one's masked text so far; once more of it comes, only what its mask adds is printed, and
|
|
114
|
+
* should its mask read differently with more of it (never seen: a mask of a prefix is the whole mask's prefix), the
|
|
115
|
+
* line is shown again whole on a line of its own, never a byte unmasked. A character whose bytes are not all written
|
|
116
|
+
* yet waits for them.
|
|
117
|
+
*/
|
|
118
|
+
follow() {
|
|
119
|
+
let out = "";
|
|
120
|
+
const ended = this.readAll();
|
|
121
|
+
if (this.broken) {
|
|
122
|
+
out += "\n";
|
|
123
|
+
this.broken = false;
|
|
124
|
+
}
|
|
125
|
+
if (ended !== "") {
|
|
126
|
+
out += this.after(ended);
|
|
127
|
+
this.shown = "";
|
|
128
|
+
}
|
|
129
|
+
if (this.at !== this.peeked) {
|
|
130
|
+
this.peeked = this.at;
|
|
131
|
+
const unended = new StringDecoder("utf8").write(this.bytes(this.lineStart, this.at));
|
|
132
|
+
const now = unended === "" ? "" : this.mask.peek(unended);
|
|
133
|
+
if (now !== this.shown) {
|
|
134
|
+
out += this.after(now);
|
|
135
|
+
this.shown = now;
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return out;
|
|
139
|
+
}
|
|
140
|
+
/** `text` as it continues what `follow` has shown of the line: what it adds, or itself on a line of its own. */
|
|
141
|
+
after(text) {
|
|
142
|
+
if (this.shown === "")
|
|
143
|
+
return text;
|
|
144
|
+
return text.startsWith(this.shown) ? text.slice(this.shown.length) : `\n${text}`;
|
|
145
|
+
}
|
|
146
|
+
/** The lines the next chunk completes, masked, without their newlines; null once the file has no byte more now. */
|
|
147
|
+
next() {
|
|
148
|
+
const size = fs.statSync(this.file).size;
|
|
149
|
+
if (size < this.at) {
|
|
150
|
+
this.at = 0;
|
|
151
|
+
this.lineStart = 0;
|
|
152
|
+
this.mask = new PasswordLineMask();
|
|
153
|
+
if (this.shown !== "")
|
|
154
|
+
this.broken = true;
|
|
155
|
+
this.shown = "";
|
|
156
|
+
this.peeked = -1;
|
|
157
|
+
}
|
|
158
|
+
if (size === this.at)
|
|
159
|
+
return null;
|
|
160
|
+
const buf = this.bytes(this.at, Math.min(this.at + this.chunk, size));
|
|
161
|
+
if (buf.length === 0)
|
|
162
|
+
return null;
|
|
163
|
+
const from = this.at;
|
|
164
|
+
this.at += buf.length;
|
|
165
|
+
// the chunk alone is searched: of a line it does not end, nothing is kept but where it starts
|
|
166
|
+
this.searchedBytes += buf.length;
|
|
167
|
+
const last = buf.lastIndexOf(10);
|
|
168
|
+
if (last === -1)
|
|
169
|
+
return [];
|
|
170
|
+
const end = from + last + 1;
|
|
171
|
+
// the lines it ends, from the first one's start: read again from the file once when that start is in an earlier
|
|
172
|
+
// chunk, so a line is had whole without being held
|
|
173
|
+
const whole = this.lineStart === from ? buf.subarray(0, last + 1) : this.bytes(this.lineStart, end);
|
|
174
|
+
this.lineStart = end;
|
|
175
|
+
const lines = whole.toString("utf8").split("\n");
|
|
176
|
+
lines.pop(); // the last newline ends its line, it does not start another
|
|
177
|
+
return lines.map((l) => this.mask.line(l));
|
|
178
|
+
}
|
|
179
|
+
/** The file's bytes from `start` to `end` (fewer when it is shorter now), counted as read. */
|
|
180
|
+
bytes(start, end) {
|
|
181
|
+
if (end <= start)
|
|
182
|
+
return Buffer.alloc(0);
|
|
183
|
+
const buf = Buffer.alloc(end - start);
|
|
184
|
+
const fd = fs.openSync(this.file, "r");
|
|
185
|
+
let n = 0;
|
|
186
|
+
try {
|
|
187
|
+
for (let got = -1; n < buf.length && got !== 0; n += got)
|
|
188
|
+
got = fs.readSync(fd, buf, n, buf.length - n, start + n);
|
|
189
|
+
}
|
|
190
|
+
finally {
|
|
191
|
+
fs.closeSync(fd);
|
|
192
|
+
}
|
|
193
|
+
this.readBytes += n;
|
|
194
|
+
return n === buf.length ? buf : buf.subarray(0, n);
|
|
195
|
+
}
|
|
196
|
+
}
|