@mason-data/cli 0.7.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (155) hide show
  1. package/README.md +101 -73
  2. package/dist/src/api/error.js +3 -0
  3. package/dist/src/api/postgres/catalog-worker.js +49 -2
  4. package/dist/src/api/postgres/catalog.js +368 -25
  5. package/dist/src/api/postgres/encode.js +3 -1
  6. package/dist/src/api/postgres/explain.js +3 -2
  7. package/dist/src/api/postgres/listener.js +11 -3
  8. package/dist/src/api/postgres/protocol.js +39 -23
  9. package/dist/src/api/postgres/proxy.js +283 -0
  10. package/dist/src/api/postgres/schema.js +6 -0
  11. package/dist/src/api/postgres/server.js +1617 -245
  12. package/dist/src/api/postgres/translate.js +127 -10
  13. package/dist/src/api/request-log.js +18 -5
  14. package/dist/src/cdc/poll.js +5 -5
  15. package/dist/src/cli/background.js +47 -25
  16. package/dist/src/cli/bridge-stdin.js +101 -0
  17. package/dist/src/cli/cancel.js +75 -0
  18. package/dist/src/cli/command.js +170 -11
  19. package/dist/src/cli/commands.js +804 -209
  20. package/dist/src/cli/control.js +59 -0
  21. package/dist/src/cli/credentials.js +182 -0
  22. package/dist/src/cli/demo.js +20 -11
  23. package/dist/src/cli/deploy.js +574 -0
  24. package/dist/src/cli/dev.js +491 -130
  25. package/dist/src/cli/doctor.js +131 -38
  26. package/dist/src/cli/generated-files.js +143 -0
  27. package/dist/src/cli/guides.js +175 -36
  28. package/dist/src/cli/hosted.js +850 -0
  29. package/dist/src/cli/init.js +162 -65
  30. package/dist/src/cli/login.js +289 -0
  31. package/dist/src/cli/machine.js +13 -12
  32. package/dist/src/cli/main.js +11 -5
  33. package/dist/src/cli/mason.cjs +17 -0
  34. package/dist/src/cli/mcp.js +31 -22
  35. package/dist/src/cli/metrics.js +79 -0
  36. package/dist/src/cli/output-gate.js +176 -0
  37. package/dist/src/cli/page-share.js +96 -0
  38. package/dist/src/cli/pg-tls.js +14 -4
  39. package/dist/src/cli/pg-url.js +4 -0
  40. package/dist/src/cli/pin.js +19 -14
  41. package/dist/src/cli/pool-size.cjs +85 -0
  42. package/dist/src/cli/ports.js +2 -0
  43. package/dist/src/cli/preflight.js +138 -46
  44. package/dist/src/cli/project.js +142 -4
  45. package/dist/src/cli/readiness.js +76 -6
  46. package/dist/src/cli/registry.js +65 -38
  47. package/dist/src/cli/replica-hosted.js +665 -0
  48. package/dist/src/cli/replica.js +76 -0
  49. package/dist/src/cli/replication-checks.js +7 -5
  50. package/dist/src/cli/replication-mode.js +105 -26
  51. package/dist/src/cli/request-filter.js +134 -0
  52. package/dist/src/cli/roles.js +585 -0
  53. package/dist/src/cli/run-server.js +637 -48
  54. package/dist/src/cli/runs.js +1 -1
  55. package/dist/src/cli/security.js +67 -18
  56. package/dist/src/cli/serve.js +204 -46
  57. package/dist/src/cli/share-run.js +329 -0
  58. package/dist/src/cli/share.js +286 -0
  59. package/dist/src/cli/source-edit.js +75 -17
  60. package/dist/src/cli/source-role.js +344 -0
  61. package/dist/src/cli/sources.js +306 -47
  62. package/dist/src/cli/style.js +2 -1
  63. package/dist/src/cli/supervise.js +27 -24
  64. package/dist/src/cli/sync.js +55 -25
  65. package/dist/src/cli/token-hosted.js +337 -0
  66. package/dist/src/cli/tunnel.js +200 -0
  67. package/dist/src/cli/ui-ssh.js +460 -0
  68. package/dist/src/cli/ui.js +124 -17
  69. package/dist/src/cli/usage.js +128 -0
  70. package/dist/src/cli/user-hosted.js +299 -0
  71. package/dist/src/cli/user.js +573 -65
  72. package/dist/src/cli/version.js +9 -7
  73. package/dist/src/demo/seed.js +29 -0
  74. package/dist/src/duck/analyze.js +36 -20
  75. package/dist/src/duck/cancel.js +10 -1
  76. package/dist/src/duck/collation.js +266 -0
  77. package/dist/src/duck/constant-key.js +94 -0
  78. package/dist/src/duck/distinct-key.js +283 -26
  79. package/dist/src/duck/events.js +165 -0
  80. package/dist/src/duck/executor-worker.js +20 -11
  81. package/dist/src/duck/executor.js +47 -33
  82. package/dist/src/duck/explain.js +60 -46
  83. package/dist/src/duck/extensions.js +160 -0
  84. package/dist/src/duck/functions.js +461 -63
  85. package/dist/src/duck/gaps.js +48 -21
  86. package/dist/src/duck/generated-plan.js +1070 -0
  87. package/dist/src/duck/generated.js +906 -0
  88. package/dist/src/duck/grants.js +176 -0
  89. package/dist/src/duck/held-types.js +64 -0
  90. package/dist/src/duck/history.js +127 -0
  91. package/dist/src/duck/jsonb-order.js +9 -12
  92. package/dist/src/duck/kept-route.js +6 -5
  93. package/dist/src/duck/kept.js +15 -13
  94. package/dist/src/duck/late.js +1469 -40
  95. package/dist/src/duck/libc-bound.js +341 -0
  96. package/dist/src/duck/libc-order.js +892 -126
  97. package/dist/src/duck/pg-eval-worker.js +7 -1
  98. package/dist/src/duck/pg-eval.js +6 -4
  99. package/dist/src/duck/pgtree.js +774 -23
  100. package/dist/src/duck/plan.js +369 -47
  101. package/dist/src/duck/publication.js +20 -6
  102. package/dist/src/duck/requests.js +269 -93
  103. package/dist/src/duck/resolved.js +117 -2
  104. package/dist/src/duck/rewrite.js +4584 -580
  105. package/dist/src/duck/row-filter.js +655 -0
  106. package/dist/src/duck/statement.js +19 -0
  107. package/dist/src/duck/store.js +439 -177
  108. package/dist/src/duck/stream.js +19 -10
  109. package/dist/src/duck/sync.js +1411 -368
  110. package/dist/src/duck/token.js +4 -1
  111. package/dist/src/duck/types.js +69 -12
  112. package/dist/src/duck/usage.js +326 -0
  113. package/dist/src/duck/users.js +69 -15
  114. package/dist/src/duck/verified.js +161 -1
  115. package/dist/src/duck/views.js +5 -5
  116. package/dist/src/lib/database-name.js +62 -0
  117. package/dist/src/lib/database-url.js +55 -10
  118. package/dist/src/lib/dotenv.js +17 -4
  119. package/dist/src/lib/instant.js +50 -0
  120. package/dist/src/lib/log-buffer.js +77 -0
  121. package/dist/src/lib/log-read.js +196 -0
  122. package/dist/src/lib/pg-numeric.js +390 -49
  123. package/dist/src/lib/project-version.js +23 -23
  124. package/dist/src/lib/secret-mask.js +142 -0
  125. package/dist/src/lib/table-sync.js +282 -0
  126. package/dist/src/web/app.css +706 -66
  127. package/dist/src/web/app.js +331 -409
  128. package/dist/src/web/charts.js +229 -0
  129. package/dist/src/web/index.html +82 -18
  130. package/dist/src/web/pages/queries.js +192 -0
  131. package/dist/src/web/pages/status.js +611 -0
  132. package/dist/src/web/pages/syncs.js +155 -0
  133. package/dist/src/web/pages/tables.js +439 -0
  134. package/dist/src/web/pages/usage.js +397 -0
  135. package/dist/src/web/pages/users.js +219 -0
  136. package/dist/src/web/shared.js +584 -0
  137. package/dist/src/web/vendor/README.md +25 -0
  138. package/dist/src/web/vendor/geist-1.7.2/Geist-Variable.woff2 +0 -0
  139. package/dist/src/web/vendor/geist-1.7.2/GeistMono-Variable.woff2 +0 -0
  140. package/dist/src/web/vendor/geist-1.7.2/LICENSE.txt +92 -0
  141. package/dist/src/web/vendor/lucide-1.56.0/LICENSE.txt +43 -0
  142. package/dist/src/web/vendor/lucide-1.56.0/chart-column.mjs +29 -0
  143. package/dist/src/web/vendor/lucide-1.56.0/layout-dashboard.mjs +29 -0
  144. package/dist/src/web/vendor/lucide-1.56.0/logs.mjs +34 -0
  145. package/dist/src/web/vendor/lucide-1.56.0/menu.mjs +28 -0
  146. package/dist/src/web/vendor/lucide-1.56.0/square-terminal.mjs +28 -0
  147. package/dist/src/web/vendor/lucide-1.56.0/table.mjs +29 -0
  148. package/dist/src/web/vendor/lucide-1.56.0/users.mjs +29 -0
  149. package/dist/src/web/vendor/lucide-1.56.0/x.mjs +27 -0
  150. package/dist/src/web/vendor/uplot-1.6.32/LICENSE.txt +21 -0
  151. package/dist/src/web/vendor/uplot-1.6.32/uPlot.esm.js +6140 -0
  152. package/dist/src/web/vendor/uplot-1.6.32/uPlot.min.css +1 -0
  153. package/dist/src/web/vendor/zilla-slab-5.3.0/LICENSE.txt +93 -0
  154. package/dist/src/web/vendor/zilla-slab-5.3.0/zilla-slab-latin-700-normal.woff2 +0 -0
  155. package/package.json +12 -4
@@ -61,7 +61,7 @@ export const END_GRACE_MS = 15_000;
61
61
  export function notAnswering(p, run) {
62
62
  return { problem: `Mason for ${p.config.name} is running (pid ${run.pid}) but does not answer its commands (${run.control.url}, 3 s)`, code: "not_answering",
63
63
  why: "a statement may be holding its main thread",
64
- next: [["mason stop", `end it (SIGTERM, then SIGKILL after ${END_GRACE_MS / 1000} s); its copy stays and the next mason start resumes`], ["mason logs", "what it was doing"]] };
64
+ next: [["mason stop", `end it (SIGTERM, then SIGKILL after ${END_GRACE_MS / 1000} s); its synced data stays and the next mason start resumes`], ["mason logs", "what it was doing"]] };
65
65
  }
66
66
  /** Whether `pid` is a Mason run, by its command line (`mason start … --attach`, src/cli/background.ts). ps exits 1 for a pid that is gone. */
67
67
  export function masonProcess(pid) {
@@ -22,19 +22,31 @@
22
22
  * no Origin: a browser sends an Authorization header across sites only after a CORS preflight, which these servers
23
23
  * never approve, so a page on another site cannot send one at all.
24
24
  *
25
- * Cookies are per host, not per port, so one cookie serves every page of a run; a page of another run, or one
26
- * the token does not match, is refused. Pure over the request's headers, so the refusals are unit-tested
25
+ * A browser keeps cookies per host, never per port, so the cookie is NAMED for the run's port (`cookieName`,
26
+ * `mason_dev_<port>`): two runs on one machine each keep their own, and opening one's page never logs the other's out
27
+ * (one name for every run did). A page of another run, or one the token does not match, is refused. Pure over the request's headers, so the refusals are unit-tested
27
28
  * (tests/cli-security.test.ts).
29
+ *
30
+ * THROUGH A TUNNEL (`mason ui --ssh`, src/cli/ui-ssh.ts): the browser is on another machine, on a local port of its own,
31
+ * so its Host is 127.0.0.1:<that port>. The guard never takes any other Host: a tunnel is a `PageTunnel`, opened by a
32
+ * process holding the run's token (`POST /api/ui/tunnel`) and gone with that process's connection, and only the loopback
33
+ * names at ITS port are added, for as long as it lasts. A DNS-rebinding page still names itself, and is refused by that
34
+ * name. The run's token never crosses: the laptop gets a TICKET, one use and minutes old, because the URL a browser is
35
+ * opened at is on a command line; the first request trades it for a cookie holding the tunnel's own SESSION, named for
36
+ * the laptop's port, which dies with the tunnel. So nothing that reaches the laptop outlives it. The tunnel itself is raw
37
+ * TCP to this server, so a client chooses its path by the Host it sends: Bearer is refused by a tunnel's Host, and by the
38
+ * server's own Host it is the commands' path, which needs the token, which never crosses.
28
39
  */
29
40
  import { randomBytes, timingSafeEqual } from "node:crypto";
30
- export const COOKIE = "mason_dev";
41
+ /** The cookie of the run whose server is on `port`: one per run, as a browser shares a host's cookies across ports. */
42
+ export const cookieName = (port) => `mason_dev_${port}`;
31
43
  export function newToken() { return randomBytes(32).toString("hex"); }
32
44
  /** Constant-time string equality (a token compared byte by byte leaks its prefix through timing). */
33
45
  export function sameSecret(a, b) {
34
46
  const x = Buffer.from(a), y = Buffer.from(b);
35
47
  return x.length === y.length && timingSafeEqual(x, y);
36
48
  }
37
- export function cookieOf(header, name = COOKIE) {
49
+ export function cookieOf(header, name) {
38
50
  for (const part of (header ?? "").split(";")) {
39
51
  const eq = part.indexOf("=");
40
52
  if (eq > 0 && part.slice(0, eq).trim() === name)
@@ -42,12 +54,55 @@ export function cookieOf(header, name = COOKIE) {
42
54
  }
43
55
  return null;
44
56
  }
45
- export function setCookieHeader(token) {
46
- return `${COOKIE}=${token}; HttpOnly; SameSite=Strict; Path=/`;
57
+ /** How long a tunnel's ticket can be traded: a person with `--no-open` pastes it; nobody needs it longer. */
58
+ export const TICKET_MS = 5 * 60_000;
59
+ export function newTunnel(port, now = Date.now()) {
60
+ return { port, ticket: newToken(), ticketUntil: now + TICKET_MS, session: newToken() };
61
+ }
62
+ export function setCookieHeader(token, port) {
63
+ return `${cookieName(port)}=${token}; HttpOnly; SameSite=Strict; Path=/`;
64
+ }
65
+ /** The Host a browser may send for a loopback port: its two names, that port, and nothing else. */
66
+ const loopbackHosts = (port) => [`127.0.0.1:${port}`, `localhost:${port}`];
67
+ /** A change must come from the page at `hosts`: its Origin is one of them. */
68
+ function foreignOrigin(req, hosts) {
69
+ if (req.method === "GET" || req.method === "HEAD")
70
+ return null;
71
+ if (req.origin && hosts.map((h) => `http://${h}`).includes(req.origin.toLowerCase()))
72
+ return null;
73
+ return { ok: false, status: 403, reason: `a change must come from this page (Origin ${JSON.stringify(req.origin ?? "")})` };
74
+ }
75
+ /** A request by the Host of a tunnel's port (the header says why it is its own path): a ticket, or the session's cookie. */
76
+ function throughTunnel(req, tunnels, now) {
77
+ const port = tunnels[0].port;
78
+ if (req.authorization !== undefined)
79
+ return { ok: false, status: 403, reason: "a tunnel carries a browser, never a command: the run's token is not sent through one" };
80
+ const cookie = cookieOf(req.cookie, cookieName(port));
81
+ const held = cookie !== null && tunnels.some((t) => sameSecret(t.session, cookie));
82
+ const url = new URL(req.url, "http://tunnel");
83
+ const offered = url.searchParams.get("ticket");
84
+ if (offered !== null && (req.method === "GET" || req.method === "HEAD")) {
85
+ url.searchParams.delete("ticket");
86
+ const location = url.pathname + (url.search || "") + url.hash;
87
+ const t = tunnels.find((x) => x.ticket !== null && now < x.ticketUntil && sameSecret(x.ticket, offered));
88
+ if (t !== undefined) {
89
+ t.ticket = null;
90
+ return { ok: false, status: 303, location, setCookie: setCookieHeader(t.session, port) };
91
+ }
92
+ if (held)
93
+ return { ok: false, status: 303, location };
94
+ return { ok: false, status: 403, reason: "this ticket was used already, or is too old: mason ui --ssh gives a new one" };
95
+ }
96
+ if (!held)
97
+ return { ok: false, status: 403, reason: "this tunnel's page takes its ticket or its cookie: mason ui --ssh opens it" };
98
+ return foreignOrigin(req, loopbackHosts(port)) ?? { ok: true };
47
99
  }
48
- /** Whether `req` may be served by the server on `port` whose run token is `token`. */
49
- export function guard(req, port, token) {
50
- const hosts = [`127.0.0.1:${port}`, `localhost:${port}`];
100
+ /** Whether `req` may be served by the server on `port` whose run token is `token`, `tunnels` the ones open now. */
101
+ export function guard(req, port, token, tunnels = [], now = Date.now()) {
102
+ const hosts = loopbackHosts(port);
103
+ const through = tunnels.filter((t) => loopbackHosts(t.port).includes((req.host ?? "").toLowerCase()));
104
+ if (through.length > 0 && !hosts.includes((req.host ?? "").toLowerCase()))
105
+ return throughTunnel(req, through, now);
51
106
  if (!req.host || !hosts.includes(req.host.toLowerCase())) {
52
107
  return { ok: false, status: 421, reason: `this server answers only as ${hosts[0]} (the request named ${JSON.stringify(req.host ?? "")})` };
53
108
  }
@@ -62,19 +117,13 @@ export function guard(req, port, token) {
62
117
  if (!sameSecret(offered, token))
63
118
  return { ok: false, status: 403, reason: "the token in the URL is not this run's" };
64
119
  url.searchParams.delete("token");
65
- return { ok: false, status: 303, location: url.pathname + (url.search || "") + url.hash, setCookie: setCookieHeader(token) };
120
+ return { ok: false, status: 303, location: url.pathname + (url.search || "") + url.hash, setCookie: setCookieHeader(token, port) };
66
121
  }
67
- const cookie = cookieOf(req.cookie);
122
+ const cookie = cookieOf(req.cookie, cookieName(port));
68
123
  if (cookie === null || !sameSecret(cookie, token)) {
69
124
  return { ok: false, status: 403, reason: "this is mason start's command server for the mason command: a request carries the run's token (Authorization: Bearer, from .mason/config.json)" };
70
125
  }
71
- if (req.method !== "GET" && req.method !== "HEAD") {
72
- const own = hosts.map((h) => `http://${h}`);
73
- if (!req.origin || !own.includes(req.origin.toLowerCase())) {
74
- return { ok: false, status: 403, reason: `a change must come from this page (Origin ${JSON.stringify(req.origin ?? "")})` };
75
- }
76
- }
77
- return { ok: true };
126
+ return foreignOrigin(req, hosts) ?? { ok: true };
78
127
  }
79
128
  /** The headers every response of a guarded server carries. */
80
129
  export const SECURITY_HEADERS = {
@@ -13,20 +13,37 @@
13
13
  * platform restarts it; there is no supervisor of its own (src/cli/supervise.ts is `mason start`'s).
14
14
  * - A HEALTH ENDPOINT (`healthBody`): `GET /healthz` on MASON_HEALTH_PORT, 503 while it starts and stops, 200 once it
15
15
  * answers. It needs no credential, so it names nothing of the database: counts only.
16
+ * - METRICS BESIDE IT (src/cli/metrics.ts): `GET /metrics` on the same port, Prometheus's text format, numbers only
17
+ * (counts, lag, ages, sizes) for the same reason: never a table's, a user's or a host's name, never a statement.
16
18
  * - A CLEAN STOP ON SIGTERM: the run's own stop (a copy in flight interrupted and rolled back, DuckDB closed).
17
19
  *
20
+ * - ITS WEB PAGE ON LOOPBACK ALONE (src/cli/run-server.ts `listenOnLoopback`): the run's server, MASON_UI_PORT, reached
21
+ * from a developer's machine with `mason ui --ssh`, never published. No setting moves it to another address.
22
+ *
18
23
  * The commands beside it work as on a laptop, in the data folder: `docker compose exec mason mason add orders`.
19
24
  */
20
25
  import * as fs from "node:fs";
21
26
  import * as http from "node:http";
22
27
  import * as path from "node:path";
23
28
  import { failed, parseCommand } from "./command.js";
29
+ import { DEFAULT_UI_PORT, heldText, portHolder } from "./ports.js";
30
+ import { pgListenerAddress } from "../api/postgres/listener.js";
24
31
  import { dev } from "./dev.js";
25
32
  import { parseSource } from "./preflight.js";
26
- import { NAME_RE, newConfig, readProject, STATE_DIR, writeConfig } from "./project.js";
27
- import { newSourceText, OLD_TABLES_DIR, parseSourceText, readSource, sourceFileNames, sourceNameFor, sourcesDir } from "./sources.js";
33
+ import { NAME_RE, newConfig, projectReadOnlySetting, projectWriteRefusal, readProject, STATE_DIR, writeConfig } from "./project.js";
34
+ import { newSourceText, OLD_TABLES_DIR, parseSourceText, readSource, sourceFileNames, sourceNameFor, sourcesDir, sourcesTop, writtenDatabase } from "./sources.js";
35
+ import { connectDatabase, DATABASE_SETTING, databaseNameProblem } from "../lib/database-name.js";
28
36
  import { appEnvValue } from "../lib/dotenv.js";
29
- import { version } from "./version.js";
37
+ import { imagePinProblem, PIN_FILE } from "../lib/project-version.js";
38
+ import { commandPinProblem } from "./pin.js";
39
+ import { recordRefusal } from "./replication-mode.js";
40
+ import { changesRetiredText, pollMsRetiredText } from "../lib/table-sync.js";
41
+ import { historyDaysSetting, historyDaysWrongText } from "../duck/history.js";
42
+ import { DEFAULT_MEMORY_RULE, memoryProblems, memoryProblemText, memorySize, MEMORY_FLOOR_BYTES } from "../duck/store.js";
43
+ import { wrap } from "./style.js";
44
+ import { runningVersion, version } from "./version.js";
45
+ import { phaseCounts } from "./readiness.js";
46
+ import { METRICS_CONTENT_TYPE, metricsText } from "./metrics.js";
30
47
  export const SERVE = {
31
48
  name: "serve",
32
49
  summary: "run Mason on a server or in a container, in the foreground, configured by the environment",
@@ -34,33 +51,52 @@ export const SERVE = {
34
51
 
35
52
  Runs Mason as mason start --attach does, for a server or a container: nothing asked, its settings from the
36
53
  environment, its state in one folder (a volume), its log on stdout, GET /healthz for the platform's health check,
37
- and a clean stop on SIGTERM. The tables it copies are the ones the project folder's sources file names
38
- (sources/<name>.yml): make it with mason init and mason add on a laptop and commit it, or with mason add beside the
39
- running server. A project folder with no sources file gets one, url: \${DATABASE_URL}, with no table yet.
54
+ GET /metrics for Prometheus (numbers only: no name, no statement), and a clean stop on SIGTERM. The tables it
55
+ syncs are the ones the project folder's sources file names (sources/<name>.yml): make it with mason init and mason
56
+ add on a laptop and commit it, or with mason add beside the running server. A project folder with no sources file
57
+ gets one, url: \${DATABASE_URL}, with no table yet — unless MASON_PROJECT_READONLY=1, which writes none.
40
58
 
41
59
  DATABASE_URL=postgres://mason:…@db.internal:5432/app MASON_API_KEY=… mason serve
42
60
  docker compose exec mason mason add public.orders beside it, in its container
43
61
 
44
62
  Settings (the environment):
45
- DATABASE_URL the database to copy from, as the sources file's url: \${DATABASE_URL} names it (the variable
63
+ DATABASE_URL the database to sync from, as the sources file's url: \${DATABASE_URL} names it (the variable
46
64
  the file names is required); read-only: SELECTs, or a replication slot
65
+ PG_PUBLICATION the publication the tables under sync: replication are read through, the database owner's,
66
+ never altered: the sources file's publication: first (publication: \${PG_PUBLICATION} reads
67
+ this one), else this; default none: Mason makes its own
47
68
  MASON_API_KEY what the users' passwords are made with (mason user add), at least 32 characters (required)
48
69
  MASON_PROJECT the project folder: sources/, mason.yml (default ./mason)
49
- MASON_DATA the folder whose .mason/ holds the copy and the state: the volume (default .)
50
- MASON_NAME the project's name, the database name clients connect to (default mason)
51
- MASON_CHANGES poll (SELECTs, the default) or replication (the change log; PG_PUBLICATION names one)
52
- MASON_HEALTH_PORT /healthz's port (default 4140), on MASON_HEALTH_HOST (default 0.0.0.0)
70
+ MASON_PROJECT_READONLY 1 when the project folder is deployed as committed (CI, GitOps, a platform that deploys
71
+ the folder): every edit of its files is refused — mason add and mason remove, beside it or
72
+ on the web page — and says to edit the files and deploy them (default 0)
73
+ MASON_DATA the folder whose .mason/ holds the synced data and the state: the volume (default .)
74
+ MASON_NAME the project's name, which the replication slot is named for (default mason); renaming it
75
+ over the same synced data is refused
76
+ MASON_PG_DATABASE the database name a tool connects with, for a sources file with no database: (default mason;
77
+ the two must agree when both are set)
78
+ MASON_HEALTH_PORT /healthz's and /metrics' port (default 4140), on MASON_HEALTH_HOST (default 0.0.0.0)
79
+ MASON_UI_PORT the web page's and the commands' port, on 127.0.0.1 alone: no setting moves the address, and
80
+ mason ui --ssh <user@server> opens the page from your machine (default ${DEFAULT_UI_PORT}, or the next
81
+ free one; a port named here is that port, or the start is refused)
53
82
  MASON_PG_HOST, MASON_PG_PORT the Postgres API's address (default 127.0.0.1:5433)
54
83
  MASON_PG_TLS_CERT, MASON_PG_TLS_KEY its certificate, required off loopback; or MASON_PG_SELF_SIGNED_TLS=1
55
84
  for one made in .mason/, or MASON_PG_PLAIN=1 behind a TLS proxy of your own
56
- MASON_DUCKDB_MEMORY the copy's memory, 4GB or 512MiB (default half the container's memory limit, else half the
57
- machine's RAM; mason doctor says it); a statement past it fails alone with 53200
58
- MASON_POLL_MS how often the poll reads the tables' changes, in ms (default 5000)
85
+ MASON_PG_PROXY_FROM behind a TCP proxy or load balancer that sends the PROXY protocol v2: its own addresses,
86
+ CIDRs comma-separated (10.0.0.0/16); a connection from one must begin with a header, whose
87
+ client the request log names; a header from any other address is refused (default unset:
88
+ none is read)
89
+ MASON_DUCKDB_MEMORY ${wrap(`the engine's memory, 4GB or 512MiB (default ${DEFAULT_MEMORY_RULE}; mason doctor says it); a statement past it fails alone with 53200. Unset, under ${memorySize(MEMORY_FLOOR_BYTES)} Mason starts and warns: the system may stop it`, 120, 28)}
90
+ MASON_SPILL_LIMIT what statements may spill to disk, 40GB or 512MiB (default half of what is free, followed as
91
+ the disk fills; never past 90% of it); a statement past it fails alone with 53400
59
92
  MASON_STATEMENT_TIMEOUT each statement's limit, as Postgres's statement_timeout (default 5min; 0 for none)
60
- MASON_REQUEST_LOG_DAYS how long the request log keeps a statement, in days (default 7)
93
+ MASON_HISTORY_DAYS how many days Mason keeps its history: the request log, the sync's events, the usage
94
+ samples and the synced tables' sizes (default 7)
61
95
  MASON_STORE_JSON_PATHS on (the default) stores the JSON paths statements keep reading as columns; off stores none
62
- MASON_COPY_PARALLEL a copy's chunks read at once, each a backend on the database (1 to 8, default 3)
96
+ MASON_COPY_PARALLEL a first sync's chunks read at once, each a backend on the database (1 to 8, default 3)
63
97
  MASON_COPY_CHUNK_SECONDS how long each chunk takes to read, at the rate measured (1 to 300, default 12)
98
+ UV_THREADPOOL_SIZE threads for running statements, one each, 1 to 1024 (default 32, lowered to fit a limit
99
+ on threads; one that does not fit, or is no such number, refuses the start)
64
100
 
65
101
  --json a refusal to start as data: { error, next }; once it serves, its output is its log
66
102
 
@@ -70,6 +106,11 @@ export const SERVE = {
70
106
  json: ["next"],
71
107
  };
72
108
  const MIN_API_KEY = 32;
109
+ /** The Postgres API's port as the settings say it; null while MASON_PG_PORT is wrong, which the run's own check says. */
110
+ function pgPortOf(env) {
111
+ const a = pgListenerAddress(env, "127.0.0.1");
112
+ return "problem" in a ? null : a.port;
113
+ }
73
114
  /** THE SETTINGS, OR EVERY ONE MISSING OR WRONG AT ONCE: a platform's deploy is slow, and one variable a deploy wastes it. */
74
115
  export function serveSettings(env, cwd) {
75
116
  const wrong = [];
@@ -85,9 +126,28 @@ export function serveSettings(env, cwd) {
85
126
  const name = env.MASON_NAME || "mason";
86
127
  if (!NAME_RE.test(name))
87
128
  wrong.push(`MASON_NAME=${JSON.stringify(name)} is not a project name: lower case letters, digits, - and _, at most 48`);
88
- const changes = env.MASON_CHANGES || null;
89
- if (changes !== null && changes !== "poll" && changes !== "replication")
90
- wrong.push(`MASON_CHANGES=${JSON.stringify(changes)}: poll or replication`);
129
+ // the database name a tool connects with: the sources file's, else this, and never two names at once
130
+ wrong.push(...databaseSettingProblems(project, env));
131
+ // how each table is kept current, and how often a table is polled, are the sources file's sync: now, per table
132
+ // (src/lib/table-sync.ts): the old settings refused HERE with the others, before anything is written or connected
133
+ const top = () => sourcesTop(path.dirname(project), project);
134
+ const changes = changesRetiredText(env, top);
135
+ if (changes !== null)
136
+ wrong.push(changes);
137
+ const pollMs = pollMsRetiredText(env, top);
138
+ if (pollMs !== null)
139
+ wrong.push(pollMs);
140
+ // the history's days: MASON_HISTORY_DAYS, a number of days; the old name refused (src/duck/history.ts)
141
+ const days = historyDaysSetting(env);
142
+ if (typeof days !== "number")
143
+ wrong.push(historyDaysWrongText(days));
144
+ // MASON_DUCKDB_MEMORY not an amount (src/duck/store.ts memoryProblems), said here with the rest, before anything is
145
+ // written or connected; a container under the floor starts, and the run warns (memoryWarning)
146
+ wrong.push(...memoryProblems(env).map(memoryProblemText));
147
+ // the project's files deployed read-only (src/cli/project.ts projectWriteRefusal): 1 or 0, anything else refused here
148
+ const readOnly = projectReadOnlySetting(env);
149
+ if (typeof readOnly !== "boolean")
150
+ wrong.push(readOnly.problem);
91
151
  const selfSigned = env.MASON_PG_SELF_SIGNED_TLS || "0";
92
152
  if (selfSigned !== "0" && selfSigned !== "1")
93
153
  wrong.push(`MASON_PG_SELF_SIGNED_TLS=${JSON.stringify(selfSigned)}: 1, or 0 (the default)`);
@@ -95,6 +155,17 @@ export function serveSettings(env, cwd) {
95
155
  const port = Number(portText);
96
156
  if (!/^\d+$/.test(portText) || port < 1 || port > 65_535)
97
157
  wrong.push(`MASON_HEALTH_PORT=${JSON.stringify(portText)}: a port number, 1-65535`);
158
+ const uiText = env.MASON_UI_PORT || String(DEFAULT_UI_PORT);
159
+ const uiPort = Number(uiText);
160
+ if (!/^\d+$/.test(uiText) || uiPort < 1 || uiPort > 65_535)
161
+ wrong.push(`MASON_UI_PORT=${JSON.stringify(uiText)}: a port number, 1-65535`);
162
+ else if (env.MASON_UI_PORT && uiPort === port)
163
+ wrong.push(`MASON_UI_PORT=${uiText} is MASON_HEALTH_PORT too: two ports`);
164
+ else if (env.MASON_UI_PORT && uiPort === pgPortOf(env))
165
+ wrong.push(`MASON_UI_PORT=${uiText} is the Postgres API's port too (MASON_PG_PORT): two ports`);
166
+ // a host would only be believed: the page has no address but loopback, and running on while one is asked for would hide that
167
+ if (env.MASON_UI_HOST)
168
+ wrong.push("MASON_UI_HOST is set, and the web page has no address but 127.0.0.1: remove it (mason ui --ssh <user@server> opens the page from your machine)");
98
169
  if (wrong.length) {
99
170
  return { message: `${wrong.length === 1 ? "a setting is" : `${wrong.length} settings are`} missing or wrong: ${wrong.join("; ")}`, code: "bad_config",
100
171
  why: "mason serve reads its settings from the environment, and asks nothing",
@@ -102,11 +173,28 @@ export function serveSettings(env, cwd) {
102
173
  next: [["mason serve --help", "every setting, with its default"]] };
103
174
  }
104
175
  return {
105
- name, apiKey, changes: changes, selfSignedTls: selfSigned === "1",
176
+ name, apiKey, selfSignedTls: selfSigned === "1",
106
177
  project, data,
107
178
  health: { host: env.MASON_HEALTH_HOST || "0.0.0.0", port },
179
+ ui: { port: uiPort, named: !!env.MASON_UI_PORT },
108
180
  };
109
181
  }
182
+ /**
183
+ * THE DATABASE NAME AS A SETTING (src/lib/database-name.ts): a name, and one the sources file does not contradict, since
184
+ * two names for one thing would have to be chosen between. A file that cannot be read is serveSource's to say.
185
+ */
186
+ function databaseSettingProblems(project, env) {
187
+ const set = env[DATABASE_SETTING];
188
+ if (!set)
189
+ return [];
190
+ const bad = databaseNameProblem(set);
191
+ if (bad !== null)
192
+ return [`${DATABASE_SETTING}: ${bad}`];
193
+ const written = writtenDatabase(project, path.dirname(project));
194
+ if ("message" in written || written.database === null || written.database === set)
195
+ return [];
196
+ return [`${DATABASE_SETTING}=${JSON.stringify(set)}, and the sources file says database: ${written.database}: keep one`];
197
+ }
110
198
  /**
111
199
  * THE DATABASE'S URL AS A SETTING (the review of the sources file: an unset URL was found only after the other
112
200
  * settings passed, two deploys for two variables): the variable the project's sources file names — DATABASE_URL when it
@@ -141,25 +229,25 @@ function urlSettingProblems(project, data, env) {
141
229
  return [];
142
230
  const found = appEnvValue(data, variable, env, { shellFirst: true });
143
231
  if (found === null)
144
- return [`${variable} is not set: the database to copy from, as the sources file's url: \${${variable}} names it`];
145
- const p = parseSource(found.value);
146
- return "problem" in p ? [`${variable}: ${p.problem}`] : [];
232
+ return [`${variable} is not set: the database to sync from, as the sources file's url: \${${variable}} names it`];
233
+ const p = parseSource(found.value, variable);
234
+ return "problem" in p ? [p.problem] : [];
147
235
  }
148
236
  /**
149
237
  * THE CONFIG A SERVE RUNS ON, written from its settings at every start, over what the runs before recorded (the server
150
238
  * the copy was taken from, the publication, the certificate): the environment is the operator's, the rest the run's. A
151
- * project `mason init` made is a developer's and is never taken over; a name that moved would be another slot and
152
- * another database name over the same copy, so it is refused.
239
+ * project `mason init` made is a developer's and is never taken over; a name that moved would be another slot over the
240
+ * same copy, so it is refused.
153
241
  */
154
242
  export function serveConfig(s, existing) {
155
243
  const dir = path.join(s.data, STATE_DIR);
156
244
  if (existing === null) {
157
245
  const config = { ...newConfig(s.name, s.project, true), apiKey: s.apiKey, served: true };
158
- return { config, said: [`a new copy in ${dir}`] };
246
+ return { config, said: [`new synced data in ${dir}`] };
159
247
  }
160
248
  if ("broken" in existing) {
161
249
  return { message: `the state in ${dir} cannot be read: ${existing.file}: ${existing.broken}`, code: "bad_config",
162
- fix: `MASON_DATA names a folder of its own for mason serve; move ${dir} aside to copy again` };
250
+ fix: `MASON_DATA names a folder of its own for mason serve; move ${dir} aside to sync again` };
163
251
  }
164
252
  const was = existing.config;
165
253
  if (was.served !== true) {
@@ -168,17 +256,27 @@ export function serveConfig(s, existing) {
168
256
  fix: "point MASON_DATA at an empty folder (the volume), or run mason start here" };
169
257
  }
170
258
  if (was.name !== s.name) {
171
- return { message: `the copy in ${dir} is the project ${was.name}, and MASON_NAME says ${s.name}`, code: "refused",
172
- why: "the name is the database name clients connect to and names the replication slot: another name over the same copy is another project",
173
- fix: `set MASON_NAME=${was.name} again, or give the new name a new MASON_DATA (an empty volume), which copies again` };
259
+ return { message: `the synced data in ${dir} is the project ${was.name}, and MASON_NAME says ${s.name}`, code: "refused",
260
+ why: "the name names the replication slot: another name over the same synced data is another project (the database name a tool connects with is not this: MASON_PG_DATABASE, or database: in the sources file)",
261
+ fix: `set MASON_NAME=${was.name} again, or give the new name a new MASON_DATA (an empty volume), which syncs again` };
174
262
  }
175
- const said = [`the copy in ${dir}, resumed`];
263
+ const said = [`the synced data in ${dir}, resumed`];
176
264
  if (was.apiKey !== s.apiKey)
177
265
  said.push("MASON_API_KEY changed: every mason user is removed (mason user add each again)");
178
266
  // the source is the sources file's; a database restored elsewhere, or another one, is the run's to see (sourceMove)
179
267
  const { source: _old, ...kept } = was;
180
268
  return { config: { ...kept, apiKey: s.apiKey, folder: s.project, remote: true, served: true }, said };
181
269
  }
270
+ /**
271
+ * THE PROJECT'S PIN, read from the project folder (MASON_PROJECT), in the words of what is running (src/lib/project-version.ts):
272
+ * the image names the image to run, the package the npm line to run, each to the reader who can act on it. Before
273
+ * anything is written or connected, so a project of another release line leaves a volume as it found it. A build that
274
+ * is no release checks nothing.
275
+ */
276
+ export function servePin(s, running) {
277
+ const p = (running.from === "image" ? imagePinProblem : commandPinProblem)(s.project, running.text);
278
+ return p && { message: p.problem, code: "refused", fix: p.ways.join(", or ") || `correct ${PIN_FILE} in the project folder (MASON_PROJECT), then start it again` };
279
+ }
182
280
  /**
183
281
  * THE PROJECT FOLDER'S SOURCES FILE, for a serve: one there is checked as the run reads it (src/cli/sources.ts
184
282
  * readSource: its URL resolved from this environment, every mistake on one report); a folder with none gets the file
@@ -193,6 +291,14 @@ export function serveSource(s, env) {
193
291
  fix: "set DATABASE_URL=postgres://user:password@host:5432/app (the sources file mason init writes names it), or mount the project folder with its sources/ (MASON_PROJECT)",
194
292
  next: [["mason serve --help", "every setting, with its default"]] };
195
293
  }
294
+ // a deployed project's files are its commit's (src/cli/project.ts projectWriteRefusal: this serve's own environment
295
+ // decides, the config not written yet): a folder deployed without its sources file is said, never filled in
296
+ const readOnly = projectWriteRefusal(null, env);
297
+ if (readOnly !== null) {
298
+ return { ...readOnly, message: `the project folder (${s.project}) has no sources file, and ${readOnly.message}`,
299
+ fix: "edit the files and deploy them: mason init and mason add where the project is committed write its sources file; commit it, and deploy the folder with it",
300
+ next: [["mason serve --help", "every setting, with its default"]] };
301
+ }
196
302
  const p = parseSource(url);
197
303
  const file = path.join(sourcesDir(s.project), `${sourceNameFor("source" in p ? p.source.database : "")}.yml`);
198
304
  try {
@@ -206,36 +312,75 @@ export function serveSource(s, env) {
206
312
  wrote = file;
207
313
  }
208
314
  const r = readSource(s.project, s.data, env, { shellFirst: true });
209
- return "message" in r ? r : { wrote };
315
+ return "message" in r ? r : { wrote, database: connectDatabase(r, { served: true }, env) };
316
+ }
317
+ /**
318
+ * WHAT THE VOLUME'S CONFIG RECORDS AND THE SOURCES FILE DOES NOT SAY (src/cli/replication-mode.ts recordRefusal), asked
319
+ * BEFORE `serveSource` and the config are written: a file a serve writes itself says `sync: off`, no one's choice, and a
320
+ * refused serve leaves the volume and the project folder as it found them. A file that cannot be read, and a project a
321
+ * serve does not take over, are said by the steps after this.
322
+ */
323
+ export function serveRecordRefusal(s, existing, env) {
324
+ if (existing === null || "broken" in existing || existing.config.served !== true)
325
+ return null;
326
+ const written = sourceFileNames(s.project).length > 0;
327
+ if (!written && fs.existsSync(path.join(s.project, OLD_TABLES_DIR)))
328
+ return null;
329
+ const file = written ? readSource(s.project, s.data, env, { shellFirst: true }) : null;
330
+ if (file !== null && "message" in file)
331
+ return null;
332
+ return recordRefusal(existing.config, file, env, sourcesTop(path.dirname(s.project), s.project).shown, "mason serve");
210
333
  }
334
+ /** seconds since the serve started: /healthz's `uptimeSeconds` and /metrics' `mason_uptime_seconds`, one number */
335
+ const uptimeSeconds = (startedAt, now) => Math.floor((now - startedAt) / 1000);
211
336
  /**
212
337
  * /healthz's answer: 200 once the run answers, whatever its copies are doing (a table still copying, or one that failed,
213
338
  * leaves the rest served: the counts say it); 503 while it starts and stops. Counts only, never a table's name: the
214
339
  * endpoint takes no credential.
215
340
  */
216
341
  export function healthBody(state, startedAt, now = Date.now()) {
217
- const base = { status: state.phase, version: version(), uptimeSeconds: Math.floor((now - startedAt) / 1000) };
342
+ const base = { status: state.phase, version: version(), uptimeSeconds: uptimeSeconds(startedAt, now) };
218
343
  if (state.phase !== "ready")
219
344
  return { status: 503, body: base };
220
- const counts = { total: 0, copied: 0, copying: 0, waiting: 0, failed: 0 };
221
- for (const t of state.tables()) {
222
- counts.total++;
223
- counts[t.phase]++;
224
- }
225
- return { status: 200, body: { ...base, tables: counts } };
345
+ return { status: 200, body: { ...base, tables: phaseCounts(state.tables()) } };
226
346
  }
227
- /** The health endpoint: GET /healthz, nothing else. */
228
- export function healthServer(state, startedAt) {
347
+ /**
348
+ * /metrics' answer: ALWAYS 200 (a scrape that errors while Mason starts is an alert about nothing; whether it is up is
349
+ * /healthz's to say) — `mason_ready 0` and `mason_info` while it starts or stops, the run's numbers once it serves.
350
+ */
351
+ export async function metricsBody(state, startedAt, now = Date.now()) {
352
+ if (state.phase !== "ready")
353
+ return metricsText({ version: version(), run: null });
354
+ const run = await state.metrics();
355
+ return metricsText({ version: version(), run: { ...run, uptimeSeconds: uptimeSeconds(startedAt, now) } });
356
+ }
357
+ /** The health server: GET /healthz and GET /metrics, nothing else. `log` says a scrape whose numbers could not be read. */
358
+ export function healthServer(state, startedAt, log = say) {
229
359
  return http.createServer((req, res) => {
230
360
  const url = new URL(req.url ?? "/", "http://health");
231
- if (url.pathname !== "/healthz") {
232
- res.writeHead(404, { "content-type": "text/plain" }).end("GET /healthz\n");
361
+ if (url.pathname !== "/healthz" && url.pathname !== "/metrics") {
362
+ res.writeHead(404, { "content-type": "text/plain" }).end("GET /healthz or GET /metrics\n");
233
363
  return;
234
364
  }
235
365
  if (req.method !== "GET" && req.method !== "HEAD") {
236
366
  res.writeHead(405, { allow: "GET, HEAD" }).end();
237
367
  return;
238
368
  }
369
+ if (url.pathname === "/metrics") {
370
+ const asked = state();
371
+ metricsBody(asked, startedAt).then((body) => { res.writeHead(200, { "content-type": METRICS_CONTENT_TYPE, "cache-control": "no-store" }).end(req.method === "HEAD" ? undefined : body); }, (e) => {
372
+ // a run stopping under the scrape (its copy closing): said as stopping, as it now is
373
+ if (state().phase !== "ready") {
374
+ res.writeHead(200, { "content-type": METRICS_CONTENT_TYPE, "cache-control": "no-store" }).end(req.method === "HEAD" ? undefined : metricsText({ version: version(), run: null }));
375
+ return;
376
+ }
377
+ // anything else fails the scrape (Prometheus's `up` turns 0) and is said in the log; the error's text stays in
378
+ // the log, never in the answer, which takes no credential (it could name a table or a host)
379
+ log(` /metrics: the run's numbers could not be read: ${e.message}`);
380
+ res.writeHead(500, { "content-type": "text/plain", "cache-control": "no-store" }).end("the run's numbers could not be read: mason serve's log says why\n");
381
+ });
382
+ return;
383
+ }
239
384
  const h = healthBody(state(), startedAt);
240
385
  res.writeHead(h.status, { "content-type": "application/json", "cache-control": "no-store" }).end(`${JSON.stringify(h.body)}\n`);
241
386
  });
@@ -251,11 +396,18 @@ export async function serve(argv) {
251
396
  const s = serveSettings(process.env, process.cwd());
252
397
  if ("message" in s)
253
398
  return failed("serve", s, { json });
399
+ const pinned = servePin(s, runningVersion());
400
+ if (pinned)
401
+ return failed("serve", pinned, { json });
402
+ const existing = readProject(s.data);
403
+ const unsaid = serveRecordRefusal(s, existing, process.env);
404
+ if (unsaid !== null)
405
+ return failed("serve", unsaid, { json });
254
406
  // the project folder: its sources file is the list of what is copied, one with no table written when there is none
255
407
  const made = serveSource(s, process.env);
256
408
  if ("message" in made)
257
409
  return failed("serve", made, { json });
258
- const c = serveConfig(s, readProject(s.data));
410
+ const c = serveConfig(s, existing);
259
411
  if ("message" in c)
260
412
  return failed("serve", c, { json });
261
413
  try {
@@ -265,6 +417,10 @@ export async function serve(argv) {
265
417
  return failed("serve", { message: `the state cannot be written to ${path.join(s.data, STATE_DIR)}: ${e.message}`, code: "refused",
266
418
  fix: "MASON_DATA names a folder this user can write: the volume's mount point" }, { json });
267
419
  }
420
+ // a port named is that port: said here, before anything listens, as the next free one would hide the setting
421
+ const held = s.ui.named ? await portHolder(s.ui.port) : null;
422
+ if (held !== null)
423
+ return failed("serve", { message: `MASON_UI_PORT: ${heldText(s.ui.port, held)}`, code: "refused", fix: "name another port, or leave MASON_UI_PORT unset for the next free one" }, { json });
268
424
  const startedAt = Date.now();
269
425
  let state = { phase: "starting" };
270
426
  const health = healthServer(() => state, startedAt);
@@ -281,13 +437,15 @@ export async function serve(argv) {
281
437
  say(` wrote ${made.wrote}: the database (url: \${DATABASE_URL}), no table yet (mason add beside it adds one)`);
282
438
  for (const l of c.said)
283
439
  say(` ${l}`);
440
+ say(` database ${made.database}, what a tool connects with`);
284
441
  say(` health http://${s.health.host}:${s.health.port}/healthz`);
442
+ say(` metrics http://${s.health.host}:${s.health.port}/metrics`);
285
443
  // the run reads the project of the folder it runs in (src/cli/project.ts): the data folder's
286
444
  process.chdir(s.data);
287
445
  let code;
288
446
  try {
289
- code = await dev({ port: 4039, version: version(), changes: s.changes, publication: null, selfSignedTls: s.selfSignedTls,
290
- serve: { log: say, ready: (run) => { state = { phase: "ready", tables: run.tables }; }, stopping: () => { state = { phase: "stopping" }; } } });
447
+ code = await dev({ port: s.ui.port, exactPort: s.ui.named, version: version(), selfSignedTls: s.selfSignedTls,
448
+ serve: { log: say, ready: (run) => { state = { phase: "ready", tables: run.tables, metrics: run.metrics }; }, stopping: () => { state = { phase: "stopping" }; } } });
291
449
  }
292
450
  finally {
293
451
  // a platform's health check may hold a keep-alive connection open, which close() alone would wait for