@mason-data/cli 0.6.2 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +147 -69
- package/dist/src/api/error.js +3 -0
- package/dist/src/api/postgres/catalog.js +258 -14
- package/dist/src/api/postgres/encode.js +3 -1
- package/dist/src/api/postgres/explain.js +3 -2
- package/dist/src/api/postgres/protocol.js +39 -23
- package/dist/src/api/postgres/server.js +1290 -223
- package/dist/src/api/postgres/translate.js +81 -9
- package/dist/src/api/request-log.js +18 -5
- package/dist/src/cdc/pg-wire.js +144 -38
- package/dist/src/cdc/poll.js +5 -5
- package/dist/src/cli/background.js +55 -8
- package/dist/src/cli/cancel.js +75 -0
- package/dist/src/cli/command.js +5 -3
- package/dist/src/cli/commands.js +843 -272
- package/dist/src/cli/control.js +59 -0
- package/dist/src/cli/demo.js +158 -0
- package/dist/src/cli/dev.js +458 -125
- package/dist/src/cli/doctor.js +25 -12
- package/dist/src/cli/guides.js +103 -31
- package/dist/src/cli/init.js +159 -58
- package/dist/src/cli/machine.js +18 -12
- package/dist/src/cli/main.js +5 -5
- package/dist/src/cli/mcp.js +735 -0
- package/dist/src/cli/metrics.js +79 -0
- package/dist/src/cli/pg-tls.js +13 -3
- package/dist/src/cli/pg-url.js +4 -0
- package/dist/src/cli/pin.js +13 -12
- package/dist/src/cli/preflight.js +24 -14
- package/dist/src/cli/project.js +48 -7
- package/dist/src/cli/readiness.js +38 -4
- package/dist/src/cli/registry.js +57 -35
- package/dist/src/cli/replication-checks.js +7 -5
- package/dist/src/cli/replication-mode.js +105 -26
- package/dist/src/cli/request-filter.js +134 -0
- package/dist/src/cli/roles.js +584 -0
- package/dist/src/cli/run-server.js +498 -133
- package/dist/src/cli/runs.js +1 -1
- package/dist/src/cli/security.js +10 -8
- package/dist/src/cli/serve.js +147 -40
- package/dist/src/cli/share-run.js +286 -0
- package/dist/src/cli/share.js +286 -0
- package/dist/src/cli/source-edit.js +168 -0
- package/dist/src/cli/sources.js +290 -39
- package/dist/src/cli/supervise.js +12 -1
- package/dist/src/cli/sync.js +55 -25
- package/dist/src/cli/tunnel.js +200 -0
- package/dist/src/cli/ui.js +9 -6
- package/dist/src/cli/usage.js +128 -0
- package/dist/src/cli/user.js +496 -40
- package/dist/src/cli/version.js +9 -7
- package/dist/src/demo/seed.js +240 -0
- package/dist/src/demo/source.js +655 -0
- package/dist/src/duck/cancel.js +10 -1
- package/dist/src/duck/distinct-key.js +228 -0
- package/dist/src/duck/events.js +134 -0
- package/dist/src/duck/executor-worker.js +25 -11
- package/dist/src/duck/executor.js +107 -53
- package/dist/src/duck/explain.js +158 -29
- package/dist/src/duck/extensions.js +160 -0
- package/dist/src/duck/functions.js +73 -19
- package/dist/src/duck/gaps.js +25 -11
- package/dist/src/duck/grants.js +176 -0
- package/dist/src/duck/history.js +123 -0
- package/dist/src/duck/jsonb-order.js +175 -0
- package/dist/src/duck/jsonb.js +10 -8
- package/dist/src/duck/kept-route.js +82 -2
- package/dist/src/duck/kept.js +211 -44
- package/dist/src/duck/pgtree.js +40 -0
- package/dist/src/duck/plan.js +120 -30
- package/dist/src/duck/publication.js +6 -6
- package/dist/src/duck/requests.js +265 -93
- package/dist/src/duck/rewrite.js +1193 -73
- package/dist/src/duck/row-filter.js +649 -0
- package/dist/src/duck/store.js +271 -159
- package/dist/src/duck/stream.js +10 -5
- package/dist/src/duck/sync.js +907 -275
- package/dist/src/duck/types.js +6 -5
- package/dist/src/duck/usage.js +324 -0
- package/dist/src/duck/users.js +64 -15
- package/dist/src/duck/verified.js +9 -0
- package/dist/src/duck/views.js +2 -3
- package/dist/src/lib/database-name.js +62 -0
- package/dist/src/lib/dotenv.js +17 -4
- package/dist/src/lib/log-buffer.js +68 -0
- package/dist/src/lib/pg-numeric.js +5 -2
- package/dist/src/lib/project-version.js +23 -23
- package/dist/src/lib/table-sync.js +267 -0
- package/dist/src/lib/timezone.js +3 -2
- package/dist/src/web/app.css +706 -55
- package/dist/src/web/app.js +330 -258
- package/dist/src/web/charts.js +229 -0
- package/dist/src/web/index.html +84 -15
- package/dist/src/web/pages/queries.js +192 -0
- package/dist/src/web/pages/status.js +446 -0
- package/dist/src/web/pages/syncs.js +155 -0
- package/dist/src/web/pages/tables.js +434 -0
- package/dist/src/web/pages/usage.js +397 -0
- package/dist/src/web/pages/users.js +218 -0
- package/dist/src/web/shared.js +568 -0
- package/dist/src/web/vendor/README.md +25 -0
- package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
- package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
- package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
- package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
- package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
- package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
- package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
- package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
- package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
- package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
- package/package.json +10 -3
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
/** The content type of the text exposition format, version 0.0.4. */
|
|
2
|
+
export const METRICS_CONTENT_TYPE = "text/plain; version=0.0.4; charset=utf-8";
|
|
3
|
+
/**
|
|
4
|
+
* HOW MANY SAMPLES OLD A NUMBER READ FROM THE DATABASE (the stream's position, the slot) or THE SAMPLER (p50 / p95) may
|
|
5
|
+
* be and still be said: the sampler refreshes each once a minute (src/duck/usage.ts SAMPLE_SECONDS), and a scrape never
|
|
6
|
+
* reads them itself (src/cli/run-server.ts `metricsFacts`), so one older than two samples means the sampler could not
|
|
7
|
+
* take them — said as unknown (absent), never as a stale number that looks current.
|
|
8
|
+
*/
|
|
9
|
+
export const FRESH_SAMPLES = 2;
|
|
10
|
+
/** a label value escaped as the format says: backslash, double quote and line feed */
|
|
11
|
+
const labelValue = (v) => v.replaceAll("\\", "\\\\").replaceAll('"', '\\"').replaceAll("\n", "\\n");
|
|
12
|
+
/** a sample's value: an integer as written, a fraction as JavaScript prints it (Go's ParseFloat reads both) */
|
|
13
|
+
const valueText = (n) => (Number.isInteger(n) ? n.toFixed(0) : String(n));
|
|
14
|
+
/** a sample that must be a count: a whole number, at least 0 */
|
|
15
|
+
const isCount = (s, kind) => kind === "counter" || s.suffix === "_count";
|
|
16
|
+
/** One metric's lines, its HELP and TYPE first — or none, the WHOLE family absent, when a sample it needs is unknown
|
|
17
|
+
* or any value is not finite (or a count not a whole number). */
|
|
18
|
+
function metric(name, kind, help, samples) {
|
|
19
|
+
const whole = samples.every((s) => s.value === null ? s.optional === true
|
|
20
|
+
: Number.isFinite(s.value) && (!isCount(s, kind) || (Number.isInteger(s.value) && s.value >= 0)));
|
|
21
|
+
const kept = samples.filter((s) => s.value !== null);
|
|
22
|
+
if (!whole || kept.length === 0)
|
|
23
|
+
return [];
|
|
24
|
+
const lines = [`# HELP ${name} ${help.replaceAll("\\", "\\\\").replaceAll("\n", "\\n")}`, `# TYPE ${name} ${kind}`];
|
|
25
|
+
for (const s of kept) {
|
|
26
|
+
const labels = s.labels ? Object.entries(s.labels).map(([k, v]) => `${k}="${labelValue(v ?? "")}"`).join(",") : "";
|
|
27
|
+
lines.push(`${name}${s.suffix ?? ""}${labels ? `{${labels}}` : ""} ${valueText(s.value)}`);
|
|
28
|
+
}
|
|
29
|
+
return lines;
|
|
30
|
+
}
|
|
31
|
+
/** The PHASES `mason_tables` says, in order: the sync's (src/duck/sync.ts `CopyState.phase`), as `/healthz` counts them. */
|
|
32
|
+
export const TABLE_PHASES = ["copied", "copying", "waiting", "failed"];
|
|
33
|
+
/** The OUTCOMES `mason_statements_total` says: the request log's (`outcomeOf`), `ok` said as `completed`. */
|
|
34
|
+
export const STATEMENT_OUTCOMES = ["completed", "failed", "refused", "cancelled"];
|
|
35
|
+
/**
|
|
36
|
+
* THE BODY OF `GET /metrics`. Starting or stopping: `mason_ready 0` and `mason_info` alone (a scrape must not fail
|
|
37
|
+
* while Mason starts; whether it is up is `/healthz`'s to say). Serving: `mason_ready 1` and the run's numbers.
|
|
38
|
+
*/
|
|
39
|
+
export function metricsText(f) {
|
|
40
|
+
const lines = [
|
|
41
|
+
...metric("mason_ready", "gauge", "1 once Mason serves the synced data; 0 while it starts or stops (/healthz answers 503 then).", [{ value: f.run === null ? 0 : 1 }]),
|
|
42
|
+
...metric("mason_info", "gauge", "Mason's version, as a label; the value is always 1.", [{ labels: { version: f.version }, value: 1 }]),
|
|
43
|
+
];
|
|
44
|
+
const r = f.run;
|
|
45
|
+
if (r !== null) {
|
|
46
|
+
const u = r.usage, st = r.statements;
|
|
47
|
+
// a number read from the database (the stream's position, the slot) as of `readAt`: said only while fresh
|
|
48
|
+
const fresh = (value, readAt) => readAt !== null && isFresh(r, readAt) ? value : null;
|
|
49
|
+
lines.push(...metric("mason_uptime_seconds", "gauge", "Seconds since mason serve started.", [{ value: r.uptimeSeconds }]), ...metric("mason_tables", "gauge", "The synced tables the run holds, by phase (a removed table's data is held until dropped).", TABLE_PHASES.map((phase) => ({ labels: { phase }, value: u.tables[phase] }))), ...metric("mason_copy_bytes", "gauge", "The size of the synced data's file on disk, bytes.", [{ value: u.copyBytes }]), ...metric("mason_copy_rows", "gauge", "The rows of the synced tables served.", [{ value: u.totals.rows }]),
|
|
50
|
+
// absent, not NaN, before any added table was synced: nothing has an age yet (see the module's comment)
|
|
51
|
+
...metric("mason_sync_oldest_age_seconds", "gauge", "Seconds since the stalest added table was last brought current; absent before any sync.", [{ value: u.oldestSyncSeconds }]),
|
|
52
|
+
// absent where no table is replicated (the run polls): a 0 would claim a stream that is not there. The database's
|
|
53
|
+
// position under the bytes, and the slot, are as the sampler last read them (a scrape never asks the database):
|
|
54
|
+
// absent when that is more than FRESH_SAMPLES samples old. The seconds are the run's own clock against the last
|
|
55
|
+
// batch's commit time, read from nothing: current at every scrape
|
|
56
|
+
...metric("mason_stream_lag_bytes", "gauge", "Replication: WAL bytes between the database's position (as last sampled, at most two samples old) and what Mason has applied; absent when nothing is replicated or not known.", [{ value: fresh(u.streamLag?.bytes ?? null, u.streamLagReadAt) }]), ...metric("mason_stream_lag_seconds", "gauge", "Replication: seconds since the newest change applied was committed on the database; absent when nothing is replicated.", [{ value: u.streamLag?.seconds ?? null }]), ...metric("mason_slot_wal_held_bytes", "gauge", "Replication: WAL bytes the replication slot holds on the database (as last sampled, at most two samples old); absent when nothing is replicated or not known.", [{ value: fresh(u.slotWalHeldBytes, u.slotWalReadAt) }]), ...metric("mason_disk_free_bytes", "gauge", "Bytes free on the disk the synced data is on.", [{ value: u.diskFreeBytes }]), ...metric("mason_sessions_open", "gauge", "Sessions logged in to Mason's Postgres API now.", [{ value: u.sessionsOpen }]), ...metric("mason_statements_total", "counter", "Statements the listener answered since the process started, whether or not the request log kept each, by outcome (refused: the statement's fault; failed: Mason's or the machine's).", STATEMENT_OUTCOMES.map((outcome) => ({ labels: { outcome }, value: outcome === "completed" ? st.completed : st[outcome] }))), ...durationSummary(r));
|
|
57
|
+
}
|
|
58
|
+
return `${lines.join("\n")}\n`;
|
|
59
|
+
}
|
|
60
|
+
/** a time `readAt` (ISO) at most `freshSeconds` before the facts were read */
|
|
61
|
+
const isFresh = (r, readAt) => (r.at - Date.parse(readAt)) / 1000 <= r.freshSeconds;
|
|
62
|
+
/**
|
|
63
|
+
* `mason_statement_duration_seconds`, a summary: the quantiles are the last usage sample's (its minute's completed
|
|
64
|
+
* statements, as `mason usage` says them: src/duck/usage.ts `lastDurations`; past DURATIONS_KEPT in a minute, of a
|
|
65
|
+
* uniform sample of them), absent before the first sample, for a minute none completed, and when the last sample is
|
|
66
|
+
* more than FRESH_SAMPLES samples old (a sample that could not be taken keeps the one before); `_sum` and `_count` are
|
|
67
|
+
* the completed statements since the process started (the counter), as a Prometheus client's summary is: quantiles
|
|
68
|
+
* over a recent window, sum and count cumulative.
|
|
69
|
+
*/
|
|
70
|
+
function durationSummary(r) {
|
|
71
|
+
const d = r.durations !== null && isFresh(r, r.durations.at) ? r.durations : null;
|
|
72
|
+
const seconds = (ms) => (ms === null || ms === undefined ? null : ms / 1000);
|
|
73
|
+
return metric("mason_statement_duration_seconds", "summary", "Completed statements' duration, seconds: p50 and p95 of the last minute's usage sample (sampled: a busy minute's of a uniform sample); sum and count since the process started.", [
|
|
74
|
+
{ labels: { quantile: "0.5" }, value: seconds(d?.p50Ms), optional: true },
|
|
75
|
+
{ labels: { quantile: "0.95" }, value: seconds(d?.p95Ms), optional: true },
|
|
76
|
+
{ suffix: "_sum", value: r.statements.completedMs / 1000 },
|
|
77
|
+
{ suffix: "_count", value: r.statements.completed },
|
|
78
|
+
]);
|
|
79
|
+
}
|
package/dist/src/cli/pg-tls.js
CHANGED
|
@@ -42,22 +42,30 @@ export function runListenerSettings(shell, o) {
|
|
|
42
42
|
...(s.fix ? { fix: s.kind === "plain" ? `${s.fix}. Or mason start --self-signed-tls: a certificate made for this project in .mason/` : s.fix } : {}),
|
|
43
43
|
...(s.kind === "plain" ? { next: [["mason start --self-signed-tls", "TLS from a certificate made in .mason/ (clients connect with sslmode=require)"]] } : {}) };
|
|
44
44
|
}
|
|
45
|
+
/** The name in every self-signed certificate: a certificate is handed to whoever connects, before any login, so it
|
|
46
|
+
* names no project. */
|
|
47
|
+
const SELF_SIGNED_NAME = "mason";
|
|
45
48
|
/**
|
|
46
49
|
* The project's self-signed certificate: the one in `.mason/` when it is still good (it parses, its key is its own, it
|
|
47
|
-
* has not expired), else a new one, both files 0600. `said` is why a new one was made, for the run to print.
|
|
50
|
+
* has not expired, it names no project), else a new one, both files 0600. `said` is why a new one was made, for the run to print.
|
|
48
51
|
*/
|
|
49
|
-
export function ensureSelfSigned(home,
|
|
52
|
+
export function ensureSelfSigned(home, now = new Date()) {
|
|
50
53
|
const certFile = path.join(home, SELF_SIGNED_FILES.cert), keyFile = path.join(home, SELF_SIGNED_FILES.key);
|
|
51
54
|
const kept = keptCertificate(certFile, keyFile, now);
|
|
52
55
|
if ("fingerprint" in kept)
|
|
53
56
|
return { fingerprint: kept.fingerprint, said: null };
|
|
54
|
-
const made = selfSignedCertificate(
|
|
57
|
+
const made = selfSignedCertificate(SELF_SIGNED_NAME, { now });
|
|
55
58
|
for (const [file, text] of [[keyFile, made.key], [certFile, made.cert]]) {
|
|
56
59
|
fs.writeFileSync(file, text, { mode: 0o600 });
|
|
57
60
|
fs.chmodSync(file, 0o600); // a file that was there keeps its mode through writeFileSync
|
|
58
61
|
}
|
|
59
62
|
return { fingerprint: new crypto.X509Certificate(made.cert).fingerprint256, said: kept.why };
|
|
60
63
|
}
|
|
64
|
+
/** The project's self-signed certificate as a listener reads it: the two files of `ensureSelfSigned`, and what it said. */
|
|
65
|
+
export function selfSignedPair(home) {
|
|
66
|
+
const { fingerprint, said } = ensureSelfSigned(home);
|
|
67
|
+
return { cert: fs.readFileSync(path.join(home, SELF_SIGNED_FILES.cert)), key: fs.readFileSync(path.join(home, SELF_SIGNED_FILES.key)), fingerprint, said };
|
|
68
|
+
}
|
|
61
69
|
function keptCertificate(certFile, keyFile, now) {
|
|
62
70
|
if (!fs.existsSync(certFile) || !fs.existsSync(keyFile))
|
|
63
71
|
return { why: `made a self-signed certificate: .mason/${SELF_SIGNED_FILES.cert}` };
|
|
@@ -65,6 +73,8 @@ function keptCertificate(certFile, keyFile, now) {
|
|
|
65
73
|
const x = new crypto.X509Certificate(fs.readFileSync(certFile));
|
|
66
74
|
if (!x.checkPrivateKey(crypto.createPrivateKey(fs.readFileSync(keyFile))))
|
|
67
75
|
return { why: `made a new self-signed certificate: .mason/${SELF_SIGNED_FILES.key} was not the key of .mason/${SELF_SIGNED_FILES.cert}` };
|
|
76
|
+
if (x.subject !== `CN=${SELF_SIGNED_NAME}` || x.issuer !== `CN=${SELF_SIGNED_NAME}`)
|
|
77
|
+
return { why: "made a new self-signed certificate: the one in .mason/ named this project to whoever connected" };
|
|
68
78
|
if (new Date(x.validTo).getTime() <= now.getTime())
|
|
69
79
|
return { why: `made a new self-signed certificate: the one in .mason/ expired ${x.validTo}` };
|
|
70
80
|
return { fingerprint: x.fingerprint256 };
|
package/dist/src/cli/pg-url.js
CHANGED
|
@@ -40,6 +40,10 @@ export const withPlaceholders = (recorded) => recorded.replace(/^postgresql:\/\/
|
|
|
40
40
|
export const withoutPlaceholders = (shown) => shown.replace(/^postgres:\/\/<user>:<password>@/, "postgresql://");
|
|
41
41
|
/** The psql line for a URL, quoted for a shell. */
|
|
42
42
|
export const psqlLine = (url) => `psql ${shellWord(url)}`;
|
|
43
|
+
/** WHERE A TOOL CONNECTS, as `mason status --json` and `/api/state` say it (`postgres`): the recorded URL's parts, the
|
|
44
|
+
* URL with the credentials' place shown (`shown`, what `mason start` and `mason status` print) and its psql line — one
|
|
45
|
+
* builder, so the web page's Connect panel and the commands print the same line. */
|
|
46
|
+
export const postgresFacts = (recorded) => ({ ...postgresLink(recorded), shown: withPlaceholders(recorded), psql: psqlLine(withPlaceholders(recorded)) });
|
|
43
47
|
/** What a tool connecting to the listener should know: it listens on every address; its certificate is self-signed. */
|
|
44
48
|
export function listenerTips(l) {
|
|
45
49
|
if (l === null)
|
package/dist/src/cli/pin.js
CHANGED
|
@@ -1,24 +1,25 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* THE PROJECT'S VERSION PIN, as the mason command holds it (src/lib/project-version.ts has the rule and the image's
|
|
3
3
|
* side): `mason init` writes `mason.yml` with this command's version, `mason start` writes it into a project made
|
|
4
|
-
* before there was one, and `mason init` and `mason
|
|
5
|
-
* developer runs is then what production
|
|
4
|
+
* before there was one, and `mason init`, `mason start` and a `mason serve` of the package refuse a command of another
|
|
5
|
+
* release line — what the developer runs is then what production is held to.
|
|
6
6
|
*/
|
|
7
7
|
import * as fs from "node:fs";
|
|
8
8
|
import * as path from "node:path";
|
|
9
|
-
import { parseVersion, pinFileText,
|
|
9
|
+
import { parseVersion, pinFileText, projectPinProblem, PIN_FILE } from "../lib/project-version.js";
|
|
10
10
|
/** Why this mason command may not run the project in `folder`, or null (a checkout's `dev` checks nothing). */
|
|
11
|
+
export function commandPinProblem(folder, cliVersion) {
|
|
12
|
+
return projectPinProblem(folder, cliVersion, { what: "this mason command",
|
|
13
|
+
run: (p) => `run the mason command it names: npm i -g @mason-data/cli@${p} (or npx @mason-data/cli@${p} …)` });
|
|
14
|
+
}
|
|
15
|
+
const sentence = (clause) => `${clause[0].toUpperCase()}${clause.slice(1)}.`;
|
|
16
|
+
/** The refusal as `mason init` and `mason start` say it: one line of sentences. */
|
|
11
17
|
export function cliPinProblem(folder, cliVersion) {
|
|
12
|
-
const
|
|
13
|
-
if (
|
|
14
|
-
return null;
|
|
15
|
-
const read = readPin(folder);
|
|
16
|
-
if ("problem" in read)
|
|
17
|
-
return read.problem;
|
|
18
|
-
if (read.pin === null)
|
|
18
|
+
const r = commandPinProblem(folder, cliVersion);
|
|
19
|
+
if (r === null)
|
|
19
20
|
return null;
|
|
20
|
-
const
|
|
21
|
-
return
|
|
21
|
+
const [run, ...others] = r.ways;
|
|
22
|
+
return run === undefined ? r.problem : [`${r.problem}.`, sentence(run), ...others.map((w) => sentence(`or ${w}`))].join(" ");
|
|
22
23
|
}
|
|
23
24
|
/**
|
|
24
25
|
* Write `folder`'s mason.yml with this command's version when it has none; the file written, or null when there was one
|
|
@@ -25,7 +25,9 @@ export function maskedUrl(url) {
|
|
|
25
25
|
return u.toString();
|
|
26
26
|
}
|
|
27
27
|
catch {
|
|
28
|
-
|
|
28
|
+
// not a URL: everything from the user's colon to the LAST @ is the password, whatever it holds (a / or an @ that
|
|
29
|
+
// was not encoded) — a host never holds an @
|
|
30
|
+
return url.replace(/:\/\/([^:@/]*):.*@/, "://$1:****@");
|
|
29
31
|
}
|
|
30
32
|
}
|
|
31
33
|
/**
|
|
@@ -38,7 +40,8 @@ export function parseSource(url) {
|
|
|
38
40
|
u = new URL(url);
|
|
39
41
|
}
|
|
40
42
|
catch {
|
|
41
|
-
|
|
43
|
+
// its password masked: the problem is printed (and, under mason mcp, kept in the MCP client's log)
|
|
44
|
+
return { problem: `${JSON.stringify(maskedUrl(url))} is not a URL. The form is postgres://user:password@localhost:5432/your_app — a password ` +
|
|
42
45
|
`containing @ : / # ? or % must be URL-encoded (encodeURIComponent; @ is %40).` };
|
|
43
46
|
}
|
|
44
47
|
if (u.protocol !== "postgres:" && u.protocol !== "postgresql:")
|
|
@@ -62,7 +65,7 @@ export function parseSource(url) {
|
|
|
62
65
|
}
|
|
63
66
|
}
|
|
64
67
|
/** What a failed connection means, and what to change. */
|
|
65
|
-
export function explainConnectError(e, s) {
|
|
68
|
+
export function explainConnectError(e, s, o = {}) {
|
|
66
69
|
const where = `${s.host}:${s.port}`;
|
|
67
70
|
const msg = e?.message ?? String(e);
|
|
68
71
|
if (e instanceof PgError) {
|
|
@@ -82,6 +85,9 @@ export function explainConnectError(e, s) {
|
|
|
82
85
|
return { message: `${where}: ${msg}`, hints: [] };
|
|
83
86
|
}
|
|
84
87
|
if (/ECONNREFUSED/.test(msg)) {
|
|
88
|
+
// the demo database is served by the project's own run: nothing else could be listening there
|
|
89
|
+
if (o.demo)
|
|
90
|
+
return { message: `the demo database is not running on ${where}`, hints: ["the demo database runs inside mason start: start it first"] };
|
|
85
91
|
return {
|
|
86
92
|
message: `nothing is listening on ${where}`,
|
|
87
93
|
hints: [
|
|
@@ -106,13 +112,13 @@ export function explainConnectError(e, s) {
|
|
|
106
112
|
return { message: `${where}: ${msg}`, hints: [] };
|
|
107
113
|
}
|
|
108
114
|
/** Connect as the service will. The failure is explained; for a missing database, the ones that exist are listed. */
|
|
109
|
-
export async function connectSource(s) {
|
|
115
|
+
export async function connectSource(s, o = {}) {
|
|
110
116
|
const opts = { host: s.host, port: s.port, database: s.database, user: s.user, password: s.password, sslmode: s.sslmode, connectTimeoutMs: 10_000, applicationName: "mason-dev" };
|
|
111
117
|
try {
|
|
112
118
|
return { pg: await PgConnection.connect(opts) };
|
|
113
119
|
}
|
|
114
120
|
catch (e) {
|
|
115
|
-
const failure = explainConnectError(e, s);
|
|
121
|
+
const failure = explainConnectError(e, s, o);
|
|
116
122
|
if (e instanceof PgError && e.code === "3D000") {
|
|
117
123
|
try {
|
|
118
124
|
const pg = await PgConnection.connect({ ...opts, database: "postgres" });
|
|
@@ -143,7 +149,7 @@ export const VIEWS_SQL = `SET LOCAL search_path = pg_catalog; SELECT coalesce(js
|
|
|
143
149
|
) v`;
|
|
144
150
|
/** One catalog read: every ordinary or partitioned table outside the system schemas, its key, size, comments and FKs. */
|
|
145
151
|
export const TABLES_SQL = `SELECT json_agg(t ORDER BY t.schema, t.table)::text FROM (
|
|
146
|
-
SELECT n.nspname AS schema, c.relname AS table, c.oid, c.reltype AS row_type_oid,
|
|
152
|
+
SELECT n.nspname AS schema, c.relname AS table, c.oid, c.reltype AS row_type_oid, c.relreplident AS replica_identity,
|
|
147
153
|
-- a PARTITIONED table's rows and bytes are its leaves' (its own are -1 / 0 and ~0 bytes); "never analysed" is
|
|
148
154
|
-- relpages = 0 AND reltuples <= 0 on any leaf, which Postgres 13 (reltuples 0) and 14+ (-1) both read
|
|
149
155
|
CASE WHEN c.relkind = 'p'
|
|
@@ -156,17 +162,20 @@ export const TABLES_SQL = `SELECT json_agg(t ORDER BY t.schema, t.table)::text F
|
|
|
156
162
|
THEN (SELECT coalesce(sum(pg_total_relation_size(pt.relid)), 0) FROM pg_partition_tree(c.oid) pt WHERE pt.isleaf)
|
|
157
163
|
ELSE pg_total_relation_size(c.oid) END AS bytes,
|
|
158
164
|
obj_description(c.oid, 'pg_class') AS comment,
|
|
165
|
+
-- the primary key: only a valid index enforces it
|
|
159
166
|
(SELECT coalesce(json_agg(a.attname ORDER BY k.n), '[]'::json)
|
|
160
167
|
FROM pg_index i CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n)
|
|
161
168
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum
|
|
162
|
-
WHERE i.indrelid = c.oid AND i.indisprimary) AS key,
|
|
169
|
+
WHERE i.indrelid = c.oid AND i.indisprimary AND i.indisvalid) AS key,
|
|
163
170
|
(SELECT coalesce(json_agg(json_build_object('name', ic.relname, 'constraint', EXISTS (SELECT FROM pg_constraint u WHERE u.conindid = i.indexrelid AND u.contype = 'u'),
|
|
164
171
|
-- indnullsnotdistinct is Postgres 15's: read through the row, which an older server's lacks
|
|
165
172
|
'nulls_not_distinct', coalesce((to_jsonb(i) ->> 'indnullsnotdistinct')::boolean, false),
|
|
166
173
|
'columns', (SELECT json_agg(a.attname ORDER BY k.n) FROM unnest(i.indkey) WITH ORDINALITY AS k(attnum, n)
|
|
167
174
|
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum)) ORDER BY ic.relname), '[]'::json)
|
|
168
175
|
FROM pg_index i JOIN pg_class ic ON ic.oid = i.indexrelid
|
|
169
|
-
|
|
176
|
+
-- an INVALID index (a CREATE UNIQUE INDEX CONCURRENTLY that failed, its duplicates still in the table) enforces
|
|
177
|
+
-- nothing: never declared to the catalog, never a key: Mason accepts
|
|
178
|
+
WHERE i.indrelid = c.oid AND i.indisunique AND i.indisvalid AND NOT i.indisprimary AND i.indpred IS NULL AND i.indexprs IS NULL
|
|
170
179
|
AND i.indnkeyatts = i.indnatts) AS unique_keys,
|
|
171
180
|
(SELECT coalesce(json_agg(json_build_object('name', a.attname, 'type', format_type(a.atttypid, a.atttypmod),
|
|
172
181
|
'not_null', a.attnotnull, 'comment', col_description(c.oid, a.attnum), 'type_oid', a.atttypid,
|
|
@@ -212,6 +221,7 @@ export async function readSourceFacts(pg, counted = COUNTED) {
|
|
|
212
221
|
foreignKeys: t.foreign_keys.map((f) => ({ column: f.column, target: f.target, targetColumn: f.target_column })),
|
|
213
222
|
uniqueKeys: t.unique_keys.map((u) => ({ name: u.name, columns: u.columns, constraint: u.constraint, nullsNotDistinct: u.nulls_not_distinct })),
|
|
214
223
|
...(t.row_type_oid > 0 ? { rowTypeOid: t.row_type_oid } : {}),
|
|
224
|
+
...(t.replica_identity === "d" || t.replica_identity === "n" || t.replica_identity === "f" || t.replica_identity === "i" ? { replicaIdentity: t.replica_identity } : {}),
|
|
215
225
|
});
|
|
216
226
|
}
|
|
217
227
|
// the columns differ by version (datlocprovider from 15, daticulocale 15–16, datlocale from 17): read the row whole
|
|
@@ -252,7 +262,7 @@ export function serverlessHost(host) {
|
|
|
252
262
|
* on a database elsewhere (the tables worth copying are the big ones). */
|
|
253
263
|
export function unpickableReason(t) {
|
|
254
264
|
if (t.key.length === 0)
|
|
255
|
-
return "no primary key: Mason keeps one row per key, so it can only
|
|
265
|
+
return "no primary key: Mason keeps one row per key, so it can only sync a table that has one";
|
|
256
266
|
return null;
|
|
257
267
|
}
|
|
258
268
|
/** The warnings `mason start` prints, and `mason status` shows, before anything is copied. */
|
|
@@ -271,8 +281,8 @@ export function sourceWarnings(s, f) {
|
|
|
271
281
|
/** The warning a database on another machine gets at every start: what the copy is and how it is kept current (true of
|
|
272
282
|
* both modes — the mode is the start's, not known here: a poll scans each table, replication reads the change log). */
|
|
273
283
|
export function remoteWarning(s) {
|
|
274
|
-
return `${s.host} is not this machine. mason start will
|
|
275
|
-
"with no access control beyond this machine's), and keep them current while it runs: a scan of each table at every poll, or its change log
|
|
284
|
+
return `${s.host} is not this machine. mason start will sync the tables you pick to this machine (into its synced data's file, ` +
|
|
285
|
+
"with no access control beyond this machine's), and keep them current while it runs: a scan of each table at every poll, or its change log for a table whose sync: is replication";
|
|
276
286
|
}
|
|
277
287
|
/**
|
|
278
288
|
* THE CONFIRMATION A DATABASE ON ANOTHER MACHINE NEEDS (the refusal read as a failed connection and then a fix: only a
|
|
@@ -282,9 +292,9 @@ export function remoteWarning(s) {
|
|
|
282
292
|
*/
|
|
283
293
|
export function remoteConfirmation(host, o) {
|
|
284
294
|
return { code: "failed", exit: 1,
|
|
285
|
-
message: `stopped before ${o.cmd === "mason init" ? "writing anything" : "starting"}: ${host} is on another machine, and
|
|
286
|
-
why: "mason start would
|
|
287
|
-
"and keep them current while it runs by reading that database: a scan of each table at every poll, or its change log
|
|
295
|
+
message: `stopped before ${o.cmd === "mason init" ? "writing anything" : "starting"}: ${host} is on another machine, and syncing from it needs your say-so`,
|
|
296
|
+
why: "mason start would sync the tables you pick to this machine: a file under .mason/, with no access control beyond this machine's\n" +
|
|
297
|
+
"and keep them current while it runs by reading that database: a scan of each table at every poll, or its change log for a table whose sync: is replication",
|
|
288
298
|
fix: o.wayOut, next: o.next };
|
|
289
299
|
}
|
|
290
300
|
/** `pg_current_snapshot()` and xid8, which polling reads, arrived in Postgres 13. */
|
package/dist/src/cli/project.js
CHANGED
|
@@ -18,14 +18,18 @@
|
|
|
18
18
|
import * as fs from "node:fs";
|
|
19
19
|
import * as path from "node:path";
|
|
20
20
|
import { randomBytes } from "node:crypto";
|
|
21
|
-
import {
|
|
21
|
+
import { connectDatabase, databaseSettingRefusal } from "../lib/database-name.js";
|
|
22
|
+
import { changesRetired, pollMsRetired } from "../lib/table-sync.js";
|
|
23
|
+
import { readSource, sourcesTop, writtenDatabase } from "./sources.js";
|
|
22
24
|
export const STATE_DIR = ".mason";
|
|
23
25
|
const CONFIG = "config.json";
|
|
24
|
-
/** A project's name:
|
|
25
|
-
*
|
|
26
|
+
/** A project's name: what its commands call it and (with the source's) what its replication slot is named for. Never
|
|
27
|
+
* the database name a tool connects with: src/lib/database-name.ts. */
|
|
26
28
|
export const NAME_RE = /^[a-z0-9][a-z0-9_-]{0,47}$/;
|
|
27
29
|
/** The copy of the data, a DuckDB file in the project's `.mason/`: kept by mason stop, deleted by mason destroy. */
|
|
28
30
|
export const DUCK_FILE = "duck.db";
|
|
31
|
+
/** The demo database's folder in the project's `.mason/` (`mason init demo`): hosted by mason start, deleted by mason destroy. */
|
|
32
|
+
export const DEMO_DIR = "demo";
|
|
29
33
|
/** A name from a folder's: lower case, the rest of it dashes. */
|
|
30
34
|
export function nameFromFolder(dir) {
|
|
31
35
|
const n = path.basename(path.resolve(dir)).toLowerCase().replace(/[^a-z0-9_-]+/g, "-").replace(/^[-_]+|[-_]+$/g, "").slice(0, 48);
|
|
@@ -97,8 +101,8 @@ export function writeConfig(dir, c) {
|
|
|
97
101
|
fs.chmodSync(file, 0o600);
|
|
98
102
|
}
|
|
99
103
|
/** A new project's config: the secrets made here, once. */
|
|
100
|
-
export function newConfig(name, folder, remote) {
|
|
101
|
-
return { name, folder, remote, apiKey: randomBytes(24).toString("hex"), sourceServer: null, running: null, lastRun: null };
|
|
104
|
+
export function newConfig(name, folder, remote, o = {}) {
|
|
105
|
+
return { name, folder, remote, ...(o.demo ? { demo: true } : {}), apiKey: randomBytes(24).toString("hex"), sourceServer: null, running: null, lastRun: null };
|
|
102
106
|
}
|
|
103
107
|
/**
|
|
104
108
|
* THE PROJECT A COMMAND MEANS: the nearest at or above `cwd`; or the sentence that says why there is none, with the
|
|
@@ -110,8 +114,8 @@ export function theProject(cwd) {
|
|
|
110
114
|
return brokenProblem(p);
|
|
111
115
|
if (p)
|
|
112
116
|
return { project: p };
|
|
113
|
-
return { problem: "no Mason project here or above", code: "no_project", fix: "cd into a project's folder, or make this folder one",
|
|
114
|
-
next: [["mason init", "make this folder a project (DATABASE_URL from .env, or mason init postgres://…)"]] };
|
|
117
|
+
return { problem: "no Mason project here or above", code: "no_project", fix: "cd into a project's folder, or make this folder one (mason init demo: a demo database, nothing to install)",
|
|
118
|
+
next: [["mason init", "make this folder a project (MASON_DATABASE_URL or DATABASE_URL from .env, or mason init postgres://…)"]] };
|
|
115
119
|
}
|
|
116
120
|
/** Whether `root`'s .gitignore already ignores `.mason/`. */
|
|
117
121
|
export function ignoresStateDir(gitignore) {
|
|
@@ -125,3 +129,40 @@ export function ignoresStateDir(gitignore) {
|
|
|
125
129
|
export function projectSource(p, env = process.env, o = {}) {
|
|
126
130
|
return readSource(p.folder, p.root, env, o);
|
|
127
131
|
}
|
|
132
|
+
/**
|
|
133
|
+
* THE DATABASE NAME THE PROJECT'S NEXT RUN WILL ANSWER AS, with Mason stopped (a run's is in its URL: `postgresLink`): by
|
|
134
|
+
* the one rule, from the sources file's one key, so it needs neither the database's URL nor the rest of the file right.
|
|
135
|
+
*/
|
|
136
|
+
export function projectDatabase(p, env = process.env) {
|
|
137
|
+
const written = writtenDatabase(p.folder, p.root);
|
|
138
|
+
return "message" in written ? written : connectDatabase(written, p.config, env);
|
|
139
|
+
}
|
|
140
|
+
/**
|
|
141
|
+
* THE NAME THE SOURCES FILE HAS MOVED TO SINCE A RUN STARTED answering as `answersAs` (the name is read at the start), or
|
|
142
|
+
* null. Only what the file says: a server's run may be named by its setting, in an environment the shell asking need
|
|
143
|
+
* not have, so a served project whose file writes no name has nothing to compare; a file that cannot be read is said
|
|
144
|
+
* as such by the commands that read it.
|
|
145
|
+
*/
|
|
146
|
+
export function databaseMoved(p, answersAs) {
|
|
147
|
+
const written = writtenDatabase(p.folder, p.root);
|
|
148
|
+
if ("message" in written || (written.database === null && p.config.served === true))
|
|
149
|
+
return null;
|
|
150
|
+
const says = connectDatabase(written, {}, {});
|
|
151
|
+
return says === answersAs ? null : says;
|
|
152
|
+
}
|
|
153
|
+
/**
|
|
154
|
+
* A SETTING A START REFUSES, for the start that is typed and the run it spawns alike: read by the parent where the person
|
|
155
|
+
* (or --json) reads the answer, and by the run, which `mason start --attach` and `mason serve` reach without the parent.
|
|
156
|
+
* One list, because a check in the run alone reaches a background start as "the run stopped", its way out in a log.
|
|
157
|
+
*/
|
|
158
|
+
export function startSettingRefusal(p, env, o) {
|
|
159
|
+
const top = () => sourcesTop(p.root, p.folder);
|
|
160
|
+
const pollMs = pollMsRetired(env, top);
|
|
161
|
+
if (pollMs !== null)
|
|
162
|
+
return { code: "refused", ...pollMs };
|
|
163
|
+
const changes = changesRetired(env, top);
|
|
164
|
+
if (changes !== null)
|
|
165
|
+
return { code: "refused", ...changes };
|
|
166
|
+
const database = o.serve ? null : databaseSettingRefusal(env, top().shown);
|
|
167
|
+
return database === null ? null : { code: "refused", ...database };
|
|
168
|
+
}
|
|
@@ -8,11 +8,13 @@ export function duration(seconds) {
|
|
|
8
8
|
const m = Math.round(s / 60);
|
|
9
9
|
return m < 90 ? `${m} min` : `${Math.floor(m / 60)} h ${m % 60} min`;
|
|
10
10
|
}
|
|
11
|
+
/** A duration in milliseconds as a person reads it (src/lib/table-sync.ts, where the sync reads it too). */
|
|
12
|
+
export { msText } from "../lib/table-sync.js";
|
|
11
13
|
/**
|
|
12
14
|
* WHAT A COPYING TABLE'S LINE SAYS, AND HOW FULL ITS BAR IS (a 20 M-row copy over the internet showed no progress: its
|
|
13
15
|
* first line waited on 0 for minutes, then jumped to 100%). A copy's rows
|
|
14
16
|
* land a chunk at a time, so the count is the committed chunks' rows PLUS what the chunks being read have read by now
|
|
15
|
-
* at the rate the copy measured (`progress.rowsReading`, held under each chunk's expected rows, so a commit moves it
|
|
17
|
+
* at the rate the copy measured (`progress.rowsReading`, held under each chunk's expected rows, so a commit that holds them moves it
|
|
16
18
|
* forward), and the text says what is happening: the chunks reading and for how long, the rate, and the time left at
|
|
17
19
|
* it — never from the row estimate alone. Before anything is measured it says the first chunk is being read, never
|
|
18
20
|
* `0 of`. The share is of the heap's PAGES read (`progress.share`: the ranges are exact), and only without the chunks'
|
|
@@ -35,7 +37,7 @@ export function copyingText(copied, rowsEstimate, p) {
|
|
|
35
37
|
if (p.rowsPerSecond === null) {
|
|
36
38
|
// nothing measured this attempt: its first chunks are being read (a copy resumed has rows from before)
|
|
37
39
|
const first = p.reading === 0 ? "about to read the first chunk" : `reading the first ${p.reading === 1 ? "chunk" : chunks(p.reading)}${since}`;
|
|
38
|
-
return { fraction, text: copied === 0 ? `${first}${estimate === null ? "" : ` · ~${n(estimate)} rows to
|
|
40
|
+
return { fraction, text: copied === 0 ? `${first}${estimate === null ? "" : ` · ~${n(estimate)} rows to sync`}` : `${rowsText(copied)} · resumed, ${first}` };
|
|
39
41
|
}
|
|
40
42
|
const parts = [rowsText(shown)];
|
|
41
43
|
const allIn = p.reading === 0 && p.chunksDone >= p.chunksTotal;
|
|
@@ -51,7 +53,39 @@ export function copyingText(copied, rowsEstimate, p) {
|
|
|
51
53
|
/** A copy's progress on one line, as `mason wait` lists what it waits for: `public.orders: copying 12% · …` (`copyingText`). */
|
|
52
54
|
export function copyProgress(source, copied, rowsEstimate, p) {
|
|
53
55
|
const c = copyingText(copied, rowsEstimate, p);
|
|
54
|
-
return `${source}: ${c.fraction !== null ? `
|
|
56
|
+
return `${source}: ${c.fraction !== null ? `syncing ${Math.floor(c.fraction * 100)}% · ` : /^\d/.test(c.text) ? "syncing " : ""}${c.text}`;
|
|
57
|
+
}
|
|
58
|
+
export function syncStateOf(c) {
|
|
59
|
+
if (c === undefined || c.phase === "waiting")
|
|
60
|
+
return "waiting";
|
|
61
|
+
if (c.phase === "failed" || c.error !== null)
|
|
62
|
+
return "failed";
|
|
63
|
+
if (c.phase === "copying" || c.rebuilding !== undefined || c.behind !== undefined)
|
|
64
|
+
return "syncing";
|
|
65
|
+
return "synced";
|
|
66
|
+
}
|
|
67
|
+
export function tableSync(c, now) {
|
|
68
|
+
const s = c?.synced;
|
|
69
|
+
return { state: syncStateOf(c), syncedAt: s?.at ?? null, syncSeconds: s?.seconds ?? null, syncedBy: s?.by ?? null,
|
|
70
|
+
ageSeconds: s === undefined ? null : Math.max(0, Math.round((now - Date.parse(s.at)) / 100) / 10) };
|
|
71
|
+
}
|
|
72
|
+
export function syncTotals(added, copies) {
|
|
73
|
+
const copied = copies.filter((c) => c.phase === "copied");
|
|
74
|
+
const latest = copies.flatMap((c) => c.synced === undefined ? [] : [c.synced.at]).sort().pop() ?? null;
|
|
75
|
+
return { added, copied: copied.length, failed: copies.filter((c) => syncStateOf(c) === "failed").length, rows: copied.reduce((n, c) => n + (c.rows ?? 0), 0), syncedAt: latest };
|
|
76
|
+
}
|
|
77
|
+
export function phaseCounts(copies) {
|
|
78
|
+
const counts = { total: 0, copied: 0, copying: 0, waiting: 0, failed: 0 };
|
|
79
|
+
for (const c of copies) {
|
|
80
|
+
counts.total++;
|
|
81
|
+
counts[c.phase]++;
|
|
82
|
+
}
|
|
83
|
+
return counts;
|
|
84
|
+
}
|
|
85
|
+
/** WHERE A RUN IS IN ONE WORD, as `mason status` heads its last line and the web page's Status says it: every added
|
|
86
|
+
* table copied and served, a failure, or still under way. */
|
|
87
|
+
export function readinessWord(r) {
|
|
88
|
+
return r.ready ? "Ready" : r.failed.length ? "Failed" : "Under way";
|
|
55
89
|
}
|
|
56
90
|
/**
|
|
57
91
|
* WHERE A RUN IS. Ready when every added table's copy has ended well: `mason add` followed by `mason wait` returns
|
|
@@ -70,7 +104,7 @@ export function readiness(s) {
|
|
|
70
104
|
continue;
|
|
71
105
|
}
|
|
72
106
|
if (!t.copy)
|
|
73
|
-
waiting.push(`${t.table}: waiting for its
|
|
107
|
+
waiting.push(`${t.table}: waiting for its first sync to start`);
|
|
74
108
|
}
|
|
75
109
|
return { ready: failed.length === 0 && waiting.length === 0, failed, waiting };
|
|
76
110
|
}
|