@mason-data/cli 0.0.0-stage → 0.6.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +209 -2
  3. package/dist/src/api/error.js +18 -0
  4. package/dist/src/api/postgres/catalog-worker.js +92 -0
  5. package/dist/src/api/postgres/catalog.js +442 -0
  6. package/dist/src/api/postgres/encode.js +541 -0
  7. package/dist/src/api/postgres/explain.js +143 -0
  8. package/dist/src/api/postgres/listener.js +60 -0
  9. package/dist/src/api/postgres/parser-worker.js +58 -0
  10. package/dist/src/api/postgres/parser.js +121 -0
  11. package/dist/src/api/postgres/plain.js +33 -0
  12. package/dist/src/api/postgres/protocol.js +404 -0
  13. package/dist/src/api/postgres/schema.js +54 -0
  14. package/dist/src/api/postgres/server.js +1670 -0
  15. package/dist/src/api/postgres/translate.js +248 -0
  16. package/dist/src/api/request-log.js +50 -0
  17. package/dist/src/cdc/pg-wire.js +482 -0
  18. package/dist/src/cdc/pgoutput.js +107 -0
  19. package/dist/src/cdc/poll.js +67 -0
  20. package/dist/src/cdc/source.js +65 -0
  21. package/dist/src/cli/background.js +174 -0
  22. package/dist/src/cli/command.js +172 -0
  23. package/dist/src/cli/commands.js +1145 -0
  24. package/dist/src/cli/control.js +36 -0
  25. package/dist/src/cli/dev.js +576 -0
  26. package/dist/src/cli/doctor.js +257 -0
  27. package/dist/src/cli/guides.js +134 -0
  28. package/dist/src/cli/init.js +326 -0
  29. package/dist/src/cli/machine.js +157 -0
  30. package/dist/src/cli/main.js +83 -0
  31. package/dist/src/cli/pg-tls.js +100 -0
  32. package/dist/src/cli/pg-url.js +52 -0
  33. package/dist/src/cli/pin.js +36 -0
  34. package/dist/src/cli/ports.js +118 -0
  35. package/dist/src/cli/preflight.js +338 -0
  36. package/dist/src/cli/project.js +127 -0
  37. package/dist/src/cli/readiness.js +76 -0
  38. package/dist/src/cli/registry.js +217 -0
  39. package/dist/src/cli/replication-checks.js +120 -0
  40. package/dist/src/cli/replication-mode.js +148 -0
  41. package/dist/src/cli/run-server.js +388 -0
  42. package/dist/src/cli/runs.js +120 -0
  43. package/dist/src/cli/security.js +86 -0
  44. package/dist/src/cli/self-signed-cert.js +64 -0
  45. package/dist/src/cli/serve.js +296 -0
  46. package/dist/src/cli/sources.js +562 -0
  47. package/dist/src/cli/state.js +58 -0
  48. package/dist/src/cli/style.js +187 -0
  49. package/dist/src/cli/supervise.js +248 -0
  50. package/dist/src/cli/sync.js +94 -0
  51. package/dist/src/cli/ui.js +107 -0
  52. package/dist/src/cli/user.js +229 -0
  53. package/dist/src/cli/version.js +23 -0
  54. package/dist/src/duck/analyze.js +113 -0
  55. package/dist/src/duck/cancel.js +49 -0
  56. package/dist/src/duck/executor-worker.js +263 -0
  57. package/dist/src/duck/executor.js +526 -0
  58. package/dist/src/duck/explain.js +119 -0
  59. package/dist/src/duck/functions.js +764 -0
  60. package/dist/src/duck/gaps.js +89 -0
  61. package/dist/src/duck/jsonb.js +186 -0
  62. package/dist/src/duck/kept-route.js +219 -0
  63. package/dist/src/duck/kept.js +797 -0
  64. package/dist/src/duck/late.js +221 -0
  65. package/dist/src/duck/libc-order.js +190 -0
  66. package/dist/src/duck/pg-eval-worker.js +34 -0
  67. package/dist/src/duck/pg-eval.js +59 -0
  68. package/dist/src/duck/pgtree.js +338 -0
  69. package/dist/src/duck/plan.js +246 -0
  70. package/dist/src/duck/publication.js +207 -0
  71. package/dist/src/duck/requests.js +209 -0
  72. package/dist/src/duck/resolved.js +248 -0
  73. package/dist/src/duck/rewrite.js +4001 -0
  74. package/dist/src/duck/store.js +831 -0
  75. package/dist/src/duck/stream.js +518 -0
  76. package/dist/src/duck/sync.js +2864 -0
  77. package/dist/src/duck/to-char.js +482 -0
  78. package/dist/src/duck/token.js +31 -0
  79. package/dist/src/duck/types.js +427 -0
  80. package/dist/src/duck/users.js +231 -0
  81. package/dist/src/duck/verified.js +995 -0
  82. package/dist/src/duck/views.js +150 -0
  83. package/dist/src/lib/database-url.js +74 -0
  84. package/dist/src/lib/did-you-mean.js +43 -0
  85. package/dist/src/lib/dotenv.js +72 -0
  86. package/dist/src/lib/exit-when-written.js +20 -0
  87. package/dist/src/lib/float-equal.js +69 -0
  88. package/dist/src/lib/http-json.js +22 -0
  89. package/dist/src/lib/log.js +161 -0
  90. package/dist/src/lib/pg-numeric.js +489 -0
  91. package/dist/src/lib/port.js +22 -0
  92. package/dist/src/lib/project-file.js +49 -0
  93. package/dist/src/lib/project-files.js +43 -0
  94. package/dist/src/lib/project-version.js +76 -0
  95. package/dist/src/lib/runnable.js +14 -0
  96. package/dist/src/lib/shell-word.js +4 -0
  97. package/dist/src/lib/timezone.js +236 -0
  98. package/dist/src/lib/zone-names.js +114 -0
  99. package/dist/src/web/app.css +65 -0
  100. package/dist/src/web/app.js +336 -0
  101. package/dist/src/web/index.html +39 -0
  102. package/package.json +22 -4
@@ -0,0 +1,207 @@
1
+ const requiredOf = (t) => t.columns.filter((c) => !c.generated).map((c) => c.name);
2
+ /** what follows every refusal: Mason waits for the owner, and when it looks again */
3
+ const NEVER_ALTERED = "Mason reads a publication it is given as it is, never altering it: once its owner has run that, the next mason start " +
4
+ "(or mason add) checks it again. Or mason start --replication without --publication, where Mason makes its own (not on a read " +
5
+ "replica), or mason start --poll, which needs no publication.";
6
+ /**
7
+ * The verdict over a described publication (pure: `described` is what `readNamedPublication` read). `identities` are
8
+ * the tables' `relreplident`: a table whose identity is not its key is expected WHOLE (a column list missing an
9
+ * identity column would make the app's UPDATEs fail).
10
+ */
11
+ export function namedPublicationVerdict(described, tables, identities) {
12
+ const whole = new Set(tables.filter((t) => identities.get(t.source) !== "d").map((t) => t.source));
13
+ const pub = described.publication;
14
+ const noTruncate = pub.exists && !pub.publish.includes("truncate");
15
+ if (!pub.exists || !["insert", "update", "delete"].every((k) => pub.publish.includes(k))) {
16
+ const all = new Map(tables.map((t) => [t.source, requiredOf(t)]));
17
+ const m = checkPublication(all, described, whole) ?? `publication ${pub.name} cannot be read (Mason bug: report it)`;
18
+ return { refused: `${m}\n\n${NEVER_ALTERED}`, tables: new Map(), noTruncate };
19
+ }
20
+ const out = new Map();
21
+ for (const t of tables) {
22
+ const m = checkPublication(new Map([[t.source, requiredOf(t)]]), described, whole);
23
+ if (m !== null)
24
+ out.set(t.source, `${m}\n\n${NEVER_ALTERED}`);
25
+ }
26
+ return { refused: null, tables: out, noTruncate };
27
+ }
28
+ // ------------------------------------------------------------------ the check
29
+ const q = (s) => /^[a-z_][a-z0-9_]*$/.test(s) ? s : `"${s.replace(/"/g, '""')}"`;
30
+ const qualified = (t) => t.split(".").map(q).join(".");
31
+ /** `schema.table (a, b, c)`, one per line, as a publication's table list reads. */
32
+ function tableLines(tables, columnLists) {
33
+ return [...tables.entries()].sort(([a], [b]) => a.localeCompare(b))
34
+ .map(([t, cols]) => ` ${qualified(t)}${columnLists && cols !== null ? ` (${cols.map(q).join(", ")})` : ""}`);
35
+ }
36
+ /** A table's columns in the catalog's order. */
37
+ const inCatalogOrder = (d, cols) => {
38
+ const want = new Set(cols);
39
+ return d === undefined ? [...want] : d.columns.filter((c) => want.has(c));
40
+ };
41
+ /**
42
+ * What the publication lacks of the copy's tables and columns, as the message that refuses them, or null when it
43
+ * lacks nothing: each missing table or column, the publication EXPECTED (what it streams now PLUS what is missing,
44
+ * never less: the publication is the owner's "replicate this"), and the one FOUND. Refused besides a missing piece: a
45
+ * publication that does not publish every insert, update and delete, and a row filter on a copied table. A table the
46
+ * database does not have is no concern of the publication's (the copy refuses it).
47
+ */
48
+ function checkPublication(required, described, whole) {
49
+ const pub = described.publication;
50
+ const columnLists = described.serverVersion >= 150000;
51
+ const problems = [];
52
+ const missingLines = [];
53
+ for (const [table, cols] of [...required.entries()].sort(([a], [b]) => a.localeCompare(b))) {
54
+ const d = described.tables.get(table);
55
+ if (d === undefined || !pub.exists)
56
+ continue;
57
+ if (d.published === null) {
58
+ problems.push(`${table} is not published`);
59
+ missingLines.push(` missing: ${table} (read by Mason's copy)`);
60
+ continue;
61
+ }
62
+ const lacking = inCatalogOrder(d, cols).filter((c) => !d.published.includes(c));
63
+ if (lacking.length) {
64
+ problems.push(`${table} columns ${lacking.join(", ")} are not published`);
65
+ missingLines.push(` missing: ${table} columns ${lacking.join(", ")} (read by Mason's copy)`);
66
+ }
67
+ if (d.rowFilter !== null) {
68
+ problems.push(`${table} is published with a row filter`);
69
+ missingLines.push(` row filter: ${table} is published WHERE ${d.rowFilter}; a row the filter leaves out never ` +
70
+ `arrives, and neither does its delete. Publish the whole table.`);
71
+ }
72
+ }
73
+ if (!pub.exists)
74
+ problems.push(`publication ${pub.name} does not exist`);
75
+ const publishes = ["insert", "update", "delete"].filter((k) => !pub.publish.includes(k));
76
+ if (pub.exists && publishes.length)
77
+ problems.push(`publication ${pub.name} does not publish ${publishes.join(", ")}`);
78
+ if (problems.length === 0)
79
+ return null;
80
+ // EXPECTED: what it streams now, plus what the copy needs
81
+ const expected = new Map();
82
+ for (const [t, d] of described.tables)
83
+ if (d.published !== null)
84
+ expected.set(t, [...d.published]);
85
+ for (const [table, cols] of required) {
86
+ const d = described.tables.get(table);
87
+ if (d === undefined)
88
+ continue;
89
+ expected.set(table, inCatalogOrder(d, [...(expected.get(table) ?? []), ...cols]));
90
+ }
91
+ for (const t of whole)
92
+ if (expected.has(t))
93
+ expected.set(t, null);
94
+ const expectedSql = !pub.exists
95
+ ? [` CREATE PUBLICATION ${q(pub.name)} FOR TABLE`, tableLines(expected, columnLists).join(",\n"), ` WITH (publish = 'insert, update, delete');`]
96
+ : addStatements(pub, required, described, expected, columnLists, publishes);
97
+ const found = new Map();
98
+ for (const [t, d] of described.tables)
99
+ if (d.published !== null)
100
+ found.set(t, d.published);
101
+ const foundSql = pub.exists
102
+ ? [`Found: publication ${pub.name} (publish = '${pub.publish.join(", ")}'${pub.allTables ? ", FOR ALL TABLES" : ""})`,
103
+ (found.size ? tableLines(found, columnLists).join(",\n") : " (no tables)")]
104
+ : [`Found: no publication named ${pub.name}. Publications on this database: ${pub.others.length ? pub.others.join(", ") : "none"}.`];
105
+ return [
106
+ `publication ${pub.name} does not carry what Mason's copy of the table holds.`,
107
+ "",
108
+ ...(missingLines.length ? [...missingLines, ""] : []),
109
+ ...(publishes.length && pub.exists ? [` it publishes ${pub.publish.join(", ") || "nothing"}; the change reader needs insert, update and delete`, ""] : []),
110
+ `Expected (run as the tables' owner, or add it to your migrations):`,
111
+ ...expectedSql,
112
+ ...(columnLists ? [] : [` (Postgres ${described.serverVersion} publishes whole tables: column lists need Postgres 15)`]),
113
+ "",
114
+ ...foundSql,
115
+ ].join("\n");
116
+ }
117
+ /**
118
+ * The gap alone, as statements on the publication as it is: a table it lacks ADDED; one whose column list lacks a
119
+ * column, or that carries a row filter, dropped and added again with the found columns plus the missing ones, in one
120
+ * transaction (no statement widens one table's column list or drops its filter); the publish list widened by what is
121
+ * missing, what it published kept — never a restatement of the whole list, which could drop what the owner published.
122
+ */
123
+ function addStatements(pub, required, described, expected, columnLists, publishes) {
124
+ const missing = [], changed = [];
125
+ for (const [table, cols] of [...required.entries()].sort(([a], [b]) => a.localeCompare(b))) {
126
+ const d = described.tables.get(table);
127
+ if (d === undefined)
128
+ continue;
129
+ if (d.published === null)
130
+ missing.push(table);
131
+ else if (d.rowFilter !== null || cols.some((c) => !d.published.includes(c)))
132
+ changed.push(table);
133
+ }
134
+ const added = new Map([...changed, ...missing].map((t) => [t, expected.get(t) ?? null]));
135
+ const publish = ["insert", "update", "delete", "truncate"].filter((k) => pub.publish.includes(k) || publishes.includes(k));
136
+ return [
137
+ ...(changed.length ? [" BEGIN;", ` ALTER PUBLICATION ${q(pub.name)} DROP TABLE ${changed.map(qualified).join(", ")};`] : []),
138
+ ...(added.size ? [` ALTER PUBLICATION ${q(pub.name)} ADD TABLE`, tableLines(added, columnLists).join(",\n") + ";"] : []),
139
+ ...(publishes.length ? [` ALTER PUBLICATION ${q(pub.name)} SET (publish = '${publish.join(", ")}');`] : []),
140
+ ...(changed.length ? [" COMMIT;"] : []),
141
+ ];
142
+ }
143
+ // ------------------------------------------------------------------ the catalog
144
+ const lit = (s) => "'" + s.replace(/'/g, "''") + "'";
145
+ /**
146
+ * The one catalog query: the publication, and every table it streams or the copy holds (by `schema.table`), each with
147
+ * its columns in the catalog's order and what the publication carries of it, in one JSON value. Column lists and row
148
+ * filters exist from Postgres 15. Read-only: the catalog, never a row of data.
149
+ */
150
+ export function publicationQuery(publication, tables, version) {
151
+ const modern = version >= 150000;
152
+ const names = tables.length === 0 ? "NULL" : tables.map(lit).join(", ");
153
+ return `WITH pt AS (
154
+ SELECT schemaname || '.' || tablename AS t, ${modern ? "to_json(attnames::text[])" : "NULL::json"} AS cols, ${modern ? "rowfilter" : "NULL::text"} AS rf
155
+ FROM pg_publication_tables WHERE pubname = ${lit(publication)}
156
+ ), wanted AS (
157
+ SELECT c.oid, n.nspname || '.' || c.relname AS t
158
+ FROM pg_class c JOIN pg_namespace n ON n.oid = c.relnamespace
159
+ WHERE c.relkind IN ('r', 'p') AND (n.nspname || '.' || c.relname IN (SELECT t FROM pt) OR n.nspname || '.' || c.relname IN (${names}))
160
+ )
161
+ SELECT json_build_object(
162
+ 'version', current_setting('server_version_num')::int,
163
+ 'publication', (SELECT json_build_object('all_tables', puballtables, 'publish', to_json(array_remove(ARRAY[
164
+ CASE WHEN pubinsert THEN 'insert' END, CASE WHEN pubupdate THEN 'update' END,
165
+ CASE WHEN pubdelete THEN 'delete' END, CASE WHEN pubtruncate THEN 'truncate' END], NULL)))
166
+ FROM pg_publication WHERE pubname = ${lit(publication)}),
167
+ 'others', (SELECT coalesce(json_agg(pubname ORDER BY pubname), '[]'::json) FROM pg_publication WHERE pubname <> ${lit(publication)}),
168
+ 'tables', (SELECT coalesce(json_agg(json_build_object(
169
+ 't', w.t,
170
+ 'columns', (SELECT coalesce(json_agg(a.attname ORDER BY a.attnum), '[]'::json)
171
+ FROM pg_attribute a WHERE a.attrelid = w.oid AND a.attnum > 0 AND NOT a.attisdropped),
172
+ 'in_publication', EXISTS (SELECT 1 FROM pt WHERE pt.t = w.t),
173
+ 'published', (SELECT pt.cols FROM pt WHERE pt.t = w.t),
174
+ 'row_filter', (SELECT pt.rf FROM pt WHERE pt.t = w.t)
175
+ ) ORDER BY w.t), '[]'::json) FROM wanted w)
176
+ )::text`;
177
+ }
178
+ /** A catalog answer described: before Postgres 15 a publication carries whole rows, every column of a table it names. */
179
+ export function describedOf(answer, publication) {
180
+ return {
181
+ publication: { name: publication, exists: answer.publication !== null, publish: answer.publication?.publish ?? [], allTables: answer.publication?.all_tables ?? false, others: answer.others },
182
+ tables: new Map(answer.tables.map((t) => [t.t, { columns: t.columns, published: t.in_publication ? (t.published ?? t.columns) : null, rowFilter: t.row_filter }])),
183
+ serverVersion: answer.version,
184
+ };
185
+ }
186
+ /** The named publication and the copied tables as the live catalog describes them: read-only, two statements.
187
+ * (`_project`, the project's name, is unused: the tables are found by their own names.) */
188
+ export async function readNamedPublication(pg, publication, tables, _project) {
189
+ const one = async (sql) => {
190
+ const v = (await pg.query(sql)).rows[0]?.[0];
191
+ if (typeof v !== "string")
192
+ throw new Error("a catalog query answered no value");
193
+ return v;
194
+ };
195
+ const version = Number(await one("SELECT current_setting('server_version_num')"));
196
+ if (!Number.isInteger(version) || version <= 0)
197
+ throw new Error("the server's version did not read as a number");
198
+ const text = await one(publicationQuery(publication, tables.map((t) => t.source), version));
199
+ let answer;
200
+ try {
201
+ answer = JSON.parse(text);
202
+ }
203
+ catch (e) {
204
+ throw new Error(`the publication's catalog query did not answer JSON (${e.message}): ${text.slice(0, 200)}`);
205
+ }
206
+ return describedOf(answer, publication);
207
+ }
@@ -0,0 +1,209 @@
1
+ import { outcomeOf } from "../api/request-log.js";
2
+ import { MASON_SCHEMA } from "./store.js";
3
+ export const REQUESTS_TABLE = `${MASON_SCHEMA}.requests`;
4
+ /** How many days of requests are kept unless MASON_REQUEST_LOG_DAYS says. */
5
+ export const DEFAULT_REQUEST_LOG_DAYS = 7;
6
+ const FLUSH_MS = 1000;
7
+ const BATCH_ROWS = 1000;
8
+ /** how often the rows past the retention are deleted (and once at open) */
9
+ const TRIM_EVERY_MS = 60 * 60 * 1000;
10
+ /**
11
+ * The days MASON_REQUEST_LOG_DAYS keeps, or why the text is not a number of days. Unset or empty: the default.
12
+ */
13
+ export function requestLogDays(text) {
14
+ if (text === undefined || text.trim() === "")
15
+ return DEFAULT_REQUEST_LOG_DAYS;
16
+ const days = Number(text);
17
+ if (!Number.isInteger(days) || days < 1) {
18
+ return { problem: `MASON_REQUEST_LOG_DAYS=${JSON.stringify(text)} is not a number of days: write a whole number, 1 or more (unset, ${DEFAULT_REQUEST_LOG_DAYS})` };
19
+ }
20
+ return days;
21
+ }
22
+ /** A request's outcome in a word, from its SQLSTATE (src/api/request-log.ts: the listener's refusal line reads it too). */
23
+ export { outcomeOf };
24
+ const COLUMNS = ["request_id", "started_at", "session", "application_name", "token_name", "user_name", "method", "sqlstate", "message", "detail", "hint",
25
+ "rows", "duration_ms", "query", "query_truncated", "seq", "json_reads"];
26
+ /** the ledger's DDL: one row per request, named for a developer who knows pg_stat_activity */
27
+ /** a ledger made before `mason user` gains its column; a column added last is named in every INSERT */
28
+ export const REQUESTS_USER_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS user_name VARCHAR`;
29
+ /**
30
+ * The ledger's ORDER within a millisecond: `started_at` is the clock's millisecond and `request_id` a random
31
+ * UUID, so two requests recorded in one millisecond — a refused login and the statement before it — read back in either
32
+ * order. Every row is numbered as it is recorded (`seq`, continuing from the ledger's last number at open), and the
33
+ * ledger is read by start time THEN seq: the order the listener saw, never the UUID's. A ledger made before gains the
34
+ * column; its old rows (seq NULL) are older than any new one by more than a millisecond.
35
+ */
36
+ export const REQUESTS_SEQ_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS seq BIGINT`;
37
+ /**
38
+ * THE JSON PATHS EACH STATEMENT READ: a JSON array of the copied tables' JSON columns read by literal steps
39
+ * (src/duck/kept-route.ts readKey), NULL for none — what decides which paths Mason stores as columns (src/duck/kept.ts
40
+ * `review`: read by several statements, or by a slow one, within the log's retention). A ledger made before gains it.
41
+ */
42
+ export const REQUESTS_JSON_DDL = `ALTER TABLE ${REQUESTS_TABLE} ADD COLUMN IF NOT EXISTS json_reads VARCHAR`;
43
+ export const REQUESTS_DDL = `CREATE TABLE IF NOT EXISTS ${REQUESTS_TABLE} (request_id VARCHAR, started_at TIMESTAMPTZ, session INTEGER, ` +
44
+ `application_name VARCHAR, token_name VARCHAR, user_name VARCHAR, method VARCHAR, sqlstate VARCHAR, message VARCHAR, detail VARCHAR, hint VARCHAR, ` +
45
+ `"rows" BIGINT, duration_ms INTEGER, query VARCHAR, query_truncated BOOLEAN, seq BIGINT, json_reads VARCHAR)`;
46
+ /** an ApiLogRow's values in COLUMNS' order */
47
+ function valuesOf({ row: r, seq }) {
48
+ const app = (r.client ?? "").replace(/^postgres:/, "");
49
+ return [r.requestId, r.at.toISOString(), r.session ?? null, app === "unnamed" ? "" : app, r.tokenName ?? "", r.user ?? "", r.method,
50
+ r.sqlstate ?? null, r.error === undefined || r.error === "" ? null : r.error, r.detail ?? null, r.hint ?? null,
51
+ r.rows === undefined ? null : r.rows, Math.round(r.durationMs), r.payload ?? "", r.payloadTruncated === true, seq,
52
+ r.jsonReads === undefined || r.jsonReads.length === 0 ? null : JSON.stringify(r.jsonReads)];
53
+ }
54
+ /** A ledger row as the commands read it. */
55
+ function requestOf(o) {
56
+ const text = (v) => v === null || v === undefined ? null : String(v);
57
+ const sqlstate = text(o.sqlstate);
58
+ return {
59
+ requestId: String(o.request_id), startedAt: new Date(String(o.started_at)).toISOString(), session: o.session === null ? null : Number(o.session),
60
+ applicationName: String(o.application_name ?? ""), tokenName: String(o.token_name ?? ""), user: String(o.user_name ?? ""), method: String(o.method),
61
+ outcome: outcomeOf(sqlstate), sqlstate, message: text(o.message), detail: text(o.detail), hint: text(o.hint),
62
+ rows: o.rows === null ? null : Number(o.rows), durationMs: Number(o.duration_ms), query: String(o.query ?? ""), queryTruncated: o.query_truncated === true,
63
+ };
64
+ }
65
+ export class RequestLedger {
66
+ buffer = [];
67
+ /** the next row's number (REQUESTS_SEQ_DDL): one past the ledger's last at open, then one per request recorded */
68
+ next = 0;
69
+ timer = null;
70
+ trimTimer = null;
71
+ /** the flush in flight: one at a time, so batches commit in the order they were taken */
72
+ flushing = Promise.resolve();
73
+ /** rows lost since the last flush that succeeded: said once when the streak begins, and when it ends */
74
+ lost = 0;
75
+ closed = false;
76
+ writer;
77
+ reader;
78
+ log;
79
+ options;
80
+ /** days kept */
81
+ days;
82
+ constructor(writer, reader, log, options) {
83
+ this.writer = writer;
84
+ this.reader = reader;
85
+ this.log = log;
86
+ this.options = options;
87
+ this.days = options.days;
88
+ }
89
+ /** The ledger of a store: its table made, the rows past the retention deleted, and a trim every hour. */
90
+ static async open(store, log, options) {
91
+ const writer = await store.instance.connect();
92
+ await writer.run(`CREATE SCHEMA IF NOT EXISTS ${MASON_SCHEMA}`);
93
+ await writer.run(REQUESTS_DDL);
94
+ await writer.run(REQUESTS_USER_DDL);
95
+ await writer.run(REQUESTS_SEQ_DDL);
96
+ await writer.run(REQUESTS_JSON_DDL);
97
+ const reader = await store.instance.connect();
98
+ const ledger = new RequestLedger(writer, reader, log, options);
99
+ ledger.next = Number((await reader.runAndReadAll(`SELECT coalesce(max(seq), -1) + 1 FROM ${REQUESTS_TABLE}`)).getRowsJS()[0][0]);
100
+ await ledger.trim();
101
+ ledger.trimTimer = setInterval(() => {
102
+ ledger.trim().catch((e) => log(`request log: deleting the requests older than ${ledger.days} days failed: ${e.message}`));
103
+ }, TRIM_EVERY_MS);
104
+ ledger.trimTimer.unref();
105
+ return ledger;
106
+ }
107
+ /** One request, written with the next batch. Never throws: the statement it records was already answered. */
108
+ record(row) {
109
+ // the listener closes before the ledger (mason stop), so nothing should arrive after: said if it does, never written
110
+ if (this.closed) {
111
+ this.log(`request log: a request after the log closed, not recorded (${row.requestId})`);
112
+ return;
113
+ }
114
+ this.buffer.push({ row, seq: this.next++ });
115
+ if (this.buffer.length >= BATCH_ROWS) {
116
+ void this.flush();
117
+ return;
118
+ }
119
+ this.timer ??= setTimeout(() => { void this.flush(); }, this.options.flushMs ?? FLUSH_MS);
120
+ }
121
+ /** Writes what is waiting, after any flush in flight. Resolves whether it was written or not (a failure is said). */
122
+ flush() {
123
+ if (this.timer !== null) {
124
+ clearTimeout(this.timer);
125
+ this.timer = null;
126
+ }
127
+ const rows = this.buffer;
128
+ this.buffer = [];
129
+ if (rows.length === 0)
130
+ return this.flushing;
131
+ this.flushing = this.flushing.then(() => this.write(rows));
132
+ return this.flushing;
133
+ }
134
+ async write(rows) {
135
+ const width = COLUMNS.length;
136
+ const sql = `INSERT INTO ${REQUESTS_TABLE} (${COLUMNS.map((c) => c === "rows" ? `"rows"` : c).join(", ")}) VALUES ` +
137
+ rows.map((_, i) => `(${COLUMNS.map((c, j) => `$${i * width + j + 1}${c === "started_at" ? "::TIMESTAMPTZ" : ""}`).join(", ")})`).join(", ");
138
+ const values = rows.flatMap(valuesOf);
139
+ try {
140
+ if (this.options.insert !== undefined)
141
+ await this.options.insert(sql, values);
142
+ else
143
+ await this.writer.run(sql, values);
144
+ }
145
+ catch (e) {
146
+ // Tolerated, narrowly, as the ClickHouse writer tolerates it: the answers are sent, so there is no statement left
147
+ // to fail, and a ledger that cannot write must not take the listener down. SAID, once per streak with the rows it
148
+ // costs, so a ledger that stopped recording never reads as a quiet listener.
149
+ if (this.lost === 0)
150
+ this.log(`request log: could not record ${rows.length} request(s) in ${REQUESTS_TABLE} (${rows[0].row.requestId}` +
151
+ `${rows.length > 1 ? ` … ${rows[rows.length - 1].row.requestId}` : ""}): ${e.message}; said once until a write succeeds`);
152
+ this.lost += rows.length;
153
+ return;
154
+ }
155
+ if (this.lost > 0) {
156
+ this.log(`request log: recording again; ${this.lost} request(s) were not recorded`);
157
+ this.lost = 0;
158
+ }
159
+ }
160
+ /** Deletes the requests older than the retention; the count is said when there were any. */
161
+ async trim() {
162
+ const r = await this.writer.runAndReadAll(`DELETE FROM ${REQUESTS_TABLE} WHERE started_at < now() - INTERVAL (${this.days}) DAY`);
163
+ const n = Number(r.getRowsJS()[0]?.[0] ?? 0);
164
+ if (n > 0)
165
+ this.log(`request log: deleted ${n} request(s) older than ${this.days} days (MASON_REQUEST_LOG_DAYS)`);
166
+ return n;
167
+ }
168
+ /** The newest `limit` requests, newest LAST; `failed`: the refused and failed ones only. Committed rows only. */
169
+ async recent(o) {
170
+ const rows = (await this.reader.runAndReadAll(`SELECT * FROM (SELECT * FROM ${REQUESTS_TABLE}${o.failed ? " WHERE sqlstate IS NOT NULL" : ""} ` +
171
+ `ORDER BY started_at DESC, seq DESC LIMIT $1) ORDER BY started_at, seq`, [o.limit])).getRowObjectsJson();
172
+ return rows.map((r) => requestOf(r));
173
+ }
174
+ /**
175
+ * The last day's slow statements in `mason status`'s shape (src/cli/run-server.ts SlowQuery): each distinct statement
176
+ * whose LATEST answer still took `slowMs` or more, slowest first.
177
+ */
178
+ async slow(slowMs, limit = 5) {
179
+ const rows = (await this.reader.runAndReadAll(`SELECT query, count(*) AS runs, CAST(quantile_disc(duration_ms, 0.5) AS INTEGER) AS median_ms, ` +
180
+ `max(duration_ms) AS max_ms, arg_max(duration_ms, started_at) AS latest_ms, max(started_at) AS last FROM ${REQUESTS_TABLE} ` +
181
+ `WHERE sqlstate IS NULL AND started_at > now() - INTERVAL 1 DAY AND query <> '' AND NOT query_truncated ` +
182
+ `GROUP BY query HAVING arg_max(duration_ms, started_at) >= $1 ORDER BY latest_ms DESC LIMIT $2`, [slowMs, limit])).getRowObjectsJson();
183
+ return rows.map((r) => ({ route: "postgres", payload: String(r.query), runs: Number(r.runs), medianMs: Number(r.median_ms), maxMs: Number(r.max_ms),
184
+ latestMs: Number(r.latest_ms), last: new Date(String(r.last)).toISOString() }));
185
+ }
186
+ /**
187
+ * The last day's refused and failed statements in `mason status`'s shape (RefusedQuery): each distinct statement once,
188
+ * newest first, with its latest SQLSTATE and message; and how many there were in all.
189
+ */
190
+ async refused(limit = 8) {
191
+ const day = `sqlstate IS NOT NULL AND started_at > now() - INTERVAL 1 DAY`;
192
+ const rows = (await this.reader.runAndReadAll(`SELECT query, count(*) AS runs, max(started_at) AS last, arg_max(sqlstate, started_at) AS code, arg_max(message, started_at) AS message ` +
193
+ `FROM ${REQUESTS_TABLE} WHERE ${day} GROUP BY query ORDER BY last DESC LIMIT $1`, [limit])).getRowObjectsJson();
194
+ const total = Number((await this.reader.runAndReadAll(`SELECT count(*) FROM ${REQUESTS_TABLE} WHERE ${day}`)).getRowsJS()[0]?.[0] ?? 0);
195
+ return { total, refused: rows.map((r) => ({ route: "postgres", payload: String(r.query), runs: Number(r.runs), last: new Date(String(r.last)).toISOString(),
196
+ code: String(r.code), message: String(r.message ?? "") })) };
197
+ }
198
+ /** Writes what is waiting and closes the ledger's connections (mason stop). */
199
+ async close() {
200
+ if (this.trimTimer !== null) {
201
+ clearInterval(this.trimTimer);
202
+ this.trimTimer = null;
203
+ }
204
+ await this.flush();
205
+ this.closed = true;
206
+ this.writer.closeSync();
207
+ this.reader.closeSync();
208
+ }
209
+ }