@mason-data/cli 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +19 -8
- package/dist/src/api/error.js +4 -0
- package/dist/src/api/postgres/catalog-worker.js +49 -2
- package/dist/src/api/postgres/catalog.js +166 -32
- package/dist/src/api/postgres/listener.js +11 -3
- package/dist/src/api/postgres/proxy.js +129 -0
- package/dist/src/api/postgres/schema.js +6 -0
- package/dist/src/api/postgres/server.js +527 -103
- package/dist/src/api/postgres/translate.js +47 -2
- package/dist/src/cli/background.js +18 -19
- package/dist/src/cli/bridge-stdin.js +101 -0
- package/dist/src/cli/command.js +337 -15
- package/dist/src/cli/commands.js +458 -169
- package/dist/src/cli/credentials.js +182 -0
- package/dist/src/cli/demo.js +2 -2
- package/dist/src/cli/deploy.js +680 -0
- package/dist/src/cli/dev.js +148 -27
- package/dist/src/cli/doctor.js +122 -36
- package/dist/src/cli/generated-files.js +143 -0
- package/dist/src/cli/guides.js +116 -16
- package/dist/src/cli/hosted.js +850 -0
- package/dist/src/cli/init.js +154 -33
- package/dist/src/cli/login.js +292 -0
- package/dist/src/cli/machine.js +125 -40
- package/dist/src/cli/main.js +59 -10
- package/dist/src/cli/mason.cjs +17 -0
- package/dist/src/cli/mcp.js +17 -14
- package/dist/src/cli/output-gate.js +176 -0
- package/dist/src/cli/page-share.js +96 -0
- package/dist/src/cli/pg-tls.js +1 -1
- package/dist/src/cli/pin.js +6 -2
- package/dist/src/cli/pool-size.cjs +85 -0
- package/dist/src/cli/ports.js +2 -0
- package/dist/src/cli/preflight.js +130 -39
- package/dist/src/cli/project.js +100 -1
- package/dist/src/cli/readiness.js +42 -6
- package/dist/src/cli/registry.js +89 -36
- package/dist/src/cli/replica-hosted.js +1324 -0
- package/dist/src/cli/replica.js +114 -0
- package/dist/src/cli/replication-mode.js +5 -0
- package/dist/src/cli/request-filter.js +16 -6
- package/dist/src/cli/roles.js +2 -1
- package/dist/src/cli/run-server.js +179 -9
- package/dist/src/cli/secret-input.js +193 -0
- package/dist/src/cli/security.js +57 -10
- package/dist/src/cli/serve.js +65 -9
- package/dist/src/cli/share-run.js +51 -8
- package/dist/src/cli/share.js +23 -18
- package/dist/src/cli/source-edit.js +10 -4
- package/dist/src/cli/source-role.js +343 -0
- package/dist/src/cli/sources.js +50 -23
- package/dist/src/cli/style.js +16 -1
- package/dist/src/cli/supervise.js +15 -23
- package/dist/src/cli/sync.js +17 -8
- package/dist/src/cli/token-hosted.js +343 -0
- package/dist/src/cli/tunnel.js +1 -1
- package/dist/src/cli/ui-hosted.js +163 -0
- package/dist/src/cli/ui-ssh.js +460 -0
- package/dist/src/cli/ui.js +162 -17
- package/dist/src/cli/usage.js +19 -15
- package/dist/src/cli/user-hosted.js +364 -0
- package/dist/src/cli/user.js +275 -105
- package/dist/src/demo/seed.js +26 -0
- package/dist/src/duck/analyze.js +36 -20
- package/dist/src/duck/collation.js +266 -0
- package/dist/src/duck/constant-key.js +94 -0
- package/dist/src/duck/distinct-key.js +283 -26
- package/dist/src/duck/events.js +39 -8
- package/dist/src/duck/executor-worker.js +2 -2
- package/dist/src/duck/executor.js +6 -5
- package/dist/src/duck/explain.js +4 -0
- package/dist/src/duck/functions.js +448 -49
- package/dist/src/duck/gaps.js +34 -16
- package/dist/src/duck/generated-plan.js +1070 -0
- package/dist/src/duck/generated.js +906 -0
- package/dist/src/duck/grants.js +1 -1
- package/dist/src/duck/held-types.js +64 -0
- package/dist/src/duck/history.js +4 -0
- package/dist/src/duck/jsonb-order.js +9 -12
- package/dist/src/duck/kept-route.js +6 -5
- package/dist/src/duck/kept.js +9 -7
- package/dist/src/duck/late.js +1469 -40
- package/dist/src/duck/libc-bound.js +341 -0
- package/dist/src/duck/libc-order.js +892 -126
- package/dist/src/duck/pg-eval-worker.js +7 -1
- package/dist/src/duck/pg-eval.js +6 -4
- package/dist/src/duck/pgtree.js +774 -23
- package/dist/src/duck/plan.js +275 -30
- package/dist/src/duck/publication.js +14 -0
- package/dist/src/duck/requests.js +9 -5
- package/dist/src/duck/resolved.js +117 -2
- package/dist/src/duck/rewrite.js +4567 -583
- package/dist/src/duck/row-filter.js +12 -6
- package/dist/src/duck/statement.js +19 -0
- package/dist/src/duck/store.js +294 -68
- package/dist/src/duck/stream.js +9 -5
- package/dist/src/duck/sync.js +602 -126
- package/dist/src/duck/token.js +4 -1
- package/dist/src/duck/types.js +63 -7
- package/dist/src/duck/usage.js +6 -4
- package/dist/src/duck/users.js +110 -9
- package/dist/src/duck/verified.js +159 -2
- package/dist/src/duck/views.js +4 -3
- package/dist/src/lib/cidr.js +177 -0
- package/dist/src/lib/database-url.js +55 -10
- package/dist/src/lib/instant.js +50 -0
- package/dist/src/lib/log-buffer.js +13 -4
- package/dist/src/lib/log-read.js +196 -0
- package/dist/src/lib/pg-numeric.js +387 -49
- package/dist/src/lib/secret-mask.js +142 -0
- package/dist/src/lib/table-sync.js +84 -19
- package/dist/src/web/app.js +9 -4
- package/dist/src/web/pages/status.js +178 -13
- package/dist/src/web/pages/syncs.js +1 -1
- package/dist/src/web/pages/tables.js +8 -3
- package/dist/src/web/pages/users.js +4 -3
- package/dist/src/web/shared.js +19 -3
- package/package.json +8 -7
package/dist/src/cli/commands.js
CHANGED
|
@@ -8,31 +8,33 @@
|
|
|
8
8
|
* mason tables [<table>…] [--json] the database's tables, their columns, keys and foreign keys
|
|
9
9
|
* mason add <table>… [--columns …] copy tables into Mason: every column, or the ones --columns names
|
|
10
10
|
* mason remove <table>… take tables out of the sources file; their copies are dropped
|
|
11
|
-
* mason wait [--timeout <
|
|
11
|
+
* mason wait [--timeout <duration>] [--json] until every added table is synced: 0 ready, 1 a copy failed (said), 124 timed out
|
|
12
12
|
* mason logs [-n <lines>] [-f] [--start] [--requests] [--events] the run's log; --start, the start's terminal; --requests, its request log; --events, the sync's events
|
|
13
13
|
* mason stop stop Mason for this folder's project (the copy stays)
|
|
14
14
|
*/
|
|
15
15
|
import * as fs from "node:fs";
|
|
16
16
|
import * as path from "node:path";
|
|
17
|
-
import { asFailure, endText, errorJson, failed, jsonText, nextBlock, parseCommand, reportText, withEnd } from "./command.js";
|
|
17
|
+
import { asFailure, endText, errorJson, failed, failureCode, jsonText, listFlag, nextBlock, noArguments, parseCommand, reportText, withEnd } from "./command.js";
|
|
18
18
|
import { err as styleErr, out as styleOut, stepRenderer, wrap } from "./style.js";
|
|
19
19
|
import { alive, endRun, END_GRACE_MS, notAnswering, runState, theRun } from "./runs.js";
|
|
20
20
|
import { version } from "./version.js";
|
|
21
21
|
import { restartsFact, signalSupervisor, stopSupervisor, supervisorGone } from "./supervise.js";
|
|
22
|
-
import { DEMO_DIR, DUCK_FILE, STATE_DIR, databaseMoved, isProject, projectDatabase, projectSource, readProject, theProject, writeConfig } from "./project.js";
|
|
22
|
+
import { DEMO_DIR, DUCK_FILE, STATE_DIR, databaseMoved, isProject, projectDatabase, projectSource, projectWriteRefusal, readProject, theProject, writeConfig } from "./project.js";
|
|
23
23
|
import { callRun, RunUnreachable } from "./control.js";
|
|
24
24
|
import { withPlaceholders, postgresFacts, postgresLink } from "./pg-url.js";
|
|
25
25
|
import { listenerText } from "./pg-tls.js";
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
26
|
+
import { entriesOfLines, logLineText } from "../lib/log-buffer.js";
|
|
27
|
+
import { LogReader, logPath } from "../lib/log-read.js";
|
|
28
|
+
import { catalogTable, entryName, findEntry, looksSecret, tableNameProblem } from "./sources.js";
|
|
28
29
|
import { addEntry, Refusal, removeEntries, sourceToEdit } from "./source-edit.js";
|
|
29
30
|
import { connectSource, maskedUrl, parseSource, readSourceFacts } from "./preflight.js";
|
|
30
31
|
import { reach } from "./state.js";
|
|
31
|
-
import { copyingText, copyProgress, duration, msText, readiness, readinessWord } from "./readiness.js";
|
|
32
|
+
import { copyingText, copyProgress, duration, generatedReadiness, msText, readiness, readinessWord } from "./readiness.js";
|
|
32
33
|
import { pathText } from "../duck/kept-route.js";
|
|
33
34
|
import { DEFAULT_REQUEST_LIMIT, eventLimit, MAX_SEARCH, SLOW_MS, requestFilter, requestFilterParams } from "./request-filter.js";
|
|
34
35
|
import { countsText } from "../duck/stream.js";
|
|
35
|
-
import { agoText, BIG_TABLE_ROWS, parseSync, syncText } from "../lib/table-sync.js";
|
|
36
|
+
import { agoText, BIG_TABLE_ROWS, parseDuration, parseSync, syncText } from "../lib/table-sync.js";
|
|
37
|
+
import { didYouMean } from "../lib/did-you-mean.js";
|
|
36
38
|
const out = (m) => { process.stdout.write(`${m}\n`); };
|
|
37
39
|
const err = (m) => { process.stderr.write(`${m}\n`); };
|
|
38
40
|
// ------------------------------------------------------------------ shared
|
|
@@ -104,7 +106,8 @@ export { copyingText, copyProgress, duration, msText, readiness, readinessWord }
|
|
|
104
106
|
/** A table line's bar in its state's colour (on a terminal): green copied, red failed, yellow served but not kept current, cyan at work
|
|
105
107
|
* (a table whose sync is off is copied as it asked: green). */
|
|
106
108
|
export function paintBar(line, s) {
|
|
107
|
-
const colour = / its first sync failed:
|
|
109
|
+
const colour = / its first sync failed: | NOT kept current, its sync failed /.test(line) ? s.red : / NOT kept current | (—|·) behind by /.test(line) ? s.yellow
|
|
110
|
+
: / 100% [\d,]+ rows · synced( · |$)/.test(line) ? s.green : s.cyan;
|
|
108
111
|
return line.replace(/(█+)(░*)|(░+)/, (_m, full, empty, none) => full !== undefined ? `${colour(full)}${s.dim(empty ?? "")}` : s.dim(none ?? ""));
|
|
109
112
|
}
|
|
110
113
|
/** The width of a table's progress bar, in characters. */
|
|
@@ -127,12 +130,12 @@ export function tableLines(s, now = Date.now()) {
|
|
|
127
130
|
let fraction, what;
|
|
128
131
|
if (r === 0) {
|
|
129
132
|
fraction = 1;
|
|
130
|
-
const age =
|
|
133
|
+
const age = keptWords(t, now);
|
|
131
134
|
what = `${n(t.copy?.rowsCopied ?? 0)} rows · synced${t.copy?.retired ? ` · ${t.copy.retired}` : ""}${age ? ` · ${age}` : ""}`;
|
|
132
135
|
}
|
|
133
136
|
else if (r === 1) {
|
|
134
137
|
fraction = 1;
|
|
135
|
-
what = `its first sync failed: ${firstLine(t.error)}`;
|
|
138
|
+
what = t.copy?.phase === "done" ? copiedFailedText(t, now) : `its first sync failed: ${firstLine(t.error)}`;
|
|
136
139
|
}
|
|
137
140
|
else if (r === 2) {
|
|
138
141
|
const c = copyingText(t.copy.rowsCopied, t.rows, t.copy.progress);
|
|
@@ -163,7 +166,8 @@ export function keptAge(t, now = Date.now()) {
|
|
|
163
166
|
if (k.state !== "streamed")
|
|
164
167
|
return `NOT kept current (${k.syncedAt === null ? "never synced" : `last synced ${agoText(k.syncedAt, now)}`}): ${k.why ?? "it is not streamed yet"}`;
|
|
165
168
|
const at = `replicated, at ${k.position}, ${k.syncedAt === null ? "not synced yet" : agoText(k.syncedAt, now)}`;
|
|
166
|
-
|
|
169
|
+
// behind: `keptWords` says it, with the stream not running as its why
|
|
170
|
+
return k.notRunning === null || t.sync?.state === "behind" ? at : `${at} — NOT kept current now, the stream is not running: ${k.notRunning}`;
|
|
167
171
|
}
|
|
168
172
|
if (k?.by !== "poll")
|
|
169
173
|
return null;
|
|
@@ -175,6 +179,33 @@ export function keptAge(t, now = Date.now()) {
|
|
|
175
179
|
return `sync: off, ${synced} — mason sync ${tableArg(t.table)} syncs it now; sync: poll 5m on its entry polls it`;
|
|
176
180
|
return `polled every ${k.sync.every}, ${k.syncedAt === null ? "not polled yet" : `last ${agoText(k.syncedAt, now)}`}`;
|
|
177
181
|
}
|
|
182
|
+
/**
|
|
183
|
+
* WHAT A SERVED TABLE'S LINE SAYS AFTER `synced`: how it is kept and how old it is (`keptAge`, by the kind of its
|
|
184
|
+
* keeping), then — for EVERY kind, from the one word (`tables[].sync.state`, src/cli/readiness.ts behindOf), never a
|
|
185
|
+
* second reading of `kept` — behind by how much and why. A kind keptAge has no words for (recopy) still says behind;
|
|
186
|
+
* a why its words already say is not said twice. Null: nothing to say.
|
|
187
|
+
*/
|
|
188
|
+
export function keptWords(t, now = Date.now()) {
|
|
189
|
+
const age = keptAge(t, now), s = t.sync;
|
|
190
|
+
if (s?.state !== "behind" || s.behindBySeconds == null)
|
|
191
|
+
return age;
|
|
192
|
+
const why = s.behindWhy ?? "its next sync is late";
|
|
193
|
+
const by = `behind by ${duration(s.behindBySeconds)}${age !== null && age.includes(why) ? "" : `: ${why}`}`;
|
|
194
|
+
return age === null ? by : `${age} — ${by}`;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* A SERVED TABLE WHOSE SYNC FAILED (its first sync succeeded; its poll, catch-up, stream or publication check did not:
|
|
198
|
+
* issues 100, 82): served as of its age, NOT kept current, why — how it is kept, when it says it is not kept current
|
|
199
|
+
* now (`keptAge`: a catch-up waiting out its back-off, a table the publication does not carry, with the way out), else
|
|
200
|
+
* its error's first line — never "its first sync failed".
|
|
201
|
+
*/
|
|
202
|
+
function copiedFailedText(t, now) {
|
|
203
|
+
const k = t.kept;
|
|
204
|
+
const reason = k?.by === "replication" ? (k.state !== "streamed" ? k.why : k.notRunning === null ? null : `the stream is not running: ${k.notRunning}`)
|
|
205
|
+
: k?.by === "poll" || k?.by === "recopy" ? k.pending ?? null : null;
|
|
206
|
+
const asOf = t.sync?.asOf ?? (t.kept && t.kept.by !== "generated" && t.kept.by !== "recopy" ? t.kept.syncedAt : null) ?? t.sync?.syncedAt ?? null;
|
|
207
|
+
return `${n(t.copy?.rowsCopied ?? 0)} rows · synced · NOT kept current, its sync failed (${asOf === null ? "never synced" : `last synced ${agoText(asOf, now)}`}): ${reason ?? firstLine(t.error)}`;
|
|
208
|
+
}
|
|
178
209
|
/** A span of seconds as the stream's lines say it: `0.3 s` under ten seconds, then as a wait reads (`duration`). */
|
|
179
210
|
export const spanText = (seconds) => seconds < 10 ? `${Math.max(0, seconds).toFixed(1)} s` : duration(seconds);
|
|
180
211
|
/**
|
|
@@ -235,7 +266,14 @@ const STATUS_USAGE = `usage: mason status [--json]
|
|
|
235
266
|
|
|
236
267
|
Status: running, not running, or hung — then this project's facts, one a line: its folder, its database and how
|
|
237
268
|
its changes are read, the run and the synced data's file and the synced tables in all (synced, rows, the last sync), the Postgres API
|
|
238
|
-
and how it listens.
|
|
269
|
+
and how it listens.
|
|
270
|
+
|
|
271
|
+
mason status
|
|
272
|
+
mason status --json
|
|
273
|
+
|
|
274
|
+
--json as data: its shape is at the end
|
|
275
|
+
|
|
276
|
+
Then each added table as mason wait shows it, with how it is kept current and when it was last synced
|
|
239
277
|
(sync: off, last synced 3 days ago; polled every 30s, last 4 s ago); the sync's latest failures (mason logs --events
|
|
240
278
|
lists every event) and the latest statements refused, failed or cancelled (mason logs --requests --failed); the Postgres API's statements of the last day and the last week (each outcome, the completed
|
|
241
279
|
ones' average, p50 and p95 time, the rows and bytes sent); the ones it answered slowly in the last day; and the ones it refused or
|
|
@@ -243,62 +281,6 @@ const STATUS_USAGE = `usage: mason status [--json]
|
|
|
243
281
|
running", how the last run ended, and the tables are the ones the sources file names. Hung is a run that is alive and does
|
|
244
282
|
not answer its commands (a statement may be holding its main thread): the same rows, and mason stop ends it (exit 1).
|
|
245
283
|
|
|
246
|
-
mason status
|
|
247
|
-
mason status --json
|
|
248
|
-
|
|
249
|
-
--json as data: { running, pid, postgres: { url, host, port, database, sslmode (require when it is TLS), shown (the URL with
|
|
250
|
-
<user>:<password> in the credentials' place), psql (its psql line), listener:
|
|
251
|
-
{ host (what it is bound to), tls (certificate, self-signed or null), fingerprint } }, source: { name (the
|
|
252
|
-
source's: its sources file's), file (the sources file), shownFile (it from the project's root), url, database, remote, version, standby, changes,
|
|
253
|
-
replication, demo (the demo database this run hosts), sync: { default (every table's sync unless its entry
|
|
254
|
-
says its own: off, poll 30s), written (the file's top sync: says it; false: the default) } }, warnings, folder, project, tables: [{ table, rows,
|
|
255
|
-
bytes, key, added, addedColumns, copy, error, kept, sync: { state (synced, syncing, failed, waiting), syncedAt (when
|
|
256
|
-
it was last brought current), syncSeconds (how long that took), syncedBy (copy, poll, catch-up, stream),
|
|
257
|
-
ageSeconds (how long ago that was) }, … }], totals: { added, copied, failed, rows (the synced tables' rows),
|
|
258
|
-
syncedAt (the latest sync of any) }, engine: { file, bytes, keptDays (MASON_HISTORY_DAYS), version (the run's), startedAt,
|
|
259
|
-
uptimeSeconds, restarts (the times its supervisor started the run again: { count, last: { at, why, upSeconds }
|
|
260
|
-
or null }, or { error }) }, disk: { path (where the synced data's file is), freeBytes, copyBytes (the file),
|
|
261
|
-
projectedBytes (what the first syncs running are projected to add; null: none, or too early to say), error },
|
|
262
|
-
sessions: { open (the Postgres API's sessions logged in), byClient: [{ user ('' a token's), client (its
|
|
263
|
-
application_name), open, running }] }, share: (null, or what mason share has open) { port, fingerprint, sessions,
|
|
264
|
-
refusedLogins, droppedConnections, shares: [{ user, role, reads, openedAt, via, host, port, endsAt, state }] }, queries: { keptDays, day, week } (the last day's and the last 7
|
|
265
|
-
days' statements, each { days, covered (the days of it the request log keeps), statements, completed, refused,
|
|
266
|
-
failed, cancelled, rows, bytesSent, durationMs: { avg, p50, p95 } (the completed ones') or null }; { error }
|
|
267
|
-
when it could not be read), slowQueries, refusedQueries, refusedTotal (how many in all), unsupported: [{ gap
|
|
268
|
-
(its entry of docs/SQL-SUPPORT.md), title (what it is; null for an entry this Mason no longer has), runs
|
|
269
|
-
(the statements it refused in the last day), users, clients (application_name), last, example (the latest,
|
|
270
|
-
cut at 1,000 characters) }] (the most refused first: GET /api/unsupported) or { error },
|
|
271
|
-
recentErrors: { sync: [{ at, kind, failed, table, message }] (the latest failures of the sync and restarts,
|
|
272
|
-
newest last: mason logs --events), error (why they could not be read, or null), queries: [{ requestId,
|
|
273
|
-
startedAt, outcome, sqlstate, message, query, … }] (the latest statements refused, failed or cancelled, newest
|
|
274
|
-
last, as mason logs --requests --json shows each: mason logs --requests --failed), queriesError (or null) },
|
|
275
|
-
readiness (what mason wait waits on), ui (whether this run serves the web page: mason ui), next }.
|
|
276
|
-
Each table's kept is how it is kept current now (absent for a table the sync does not hold):
|
|
277
|
-
{ by: poll, sync: { mode: off } or { mode: poll, every (30s), seconds } (its entry's sync:, else the file's,
|
|
278
|
-
else off), syncedAt (its last whole read or poll), nextAt (when its next poll is due; null: off, or not polled
|
|
279
|
-
now), pending (only when it applies: why it is not kept current now, its phase included: it waits for
|
|
280
|
-
its first sync, it is being read whole, its first sync failed, it is read again whole, caught up first) }
|
|
281
|
-
or { by: recopy, copiedAt, pending (only when it applies, as the poll's) }
|
|
282
|
-
or { by: replication, slot, state (streamed, catching-up, waiting, left-out), why, position, lastCommit,
|
|
283
|
-
notRunning (why the stream is not running, null when it is) }; with Mason stopped: { running: false, project, source: { name,
|
|
284
|
-
file, url, error (the sources file's problem, or null) }, engine, tables: [{ table }], lastRun, lastStop:
|
|
285
|
-
{ at, how } or null, next }; hung: the same with hung: { pid, problem, why },
|
|
286
|
-
exit 1. In replication mode stream: { slot, slotWal: { walHeldBytes (the WAL the slot holds on the
|
|
287
|
-
database), walStatus (reserved, extended, unreserved, lost), safeWalSizeBytes (the WAL left before Postgres
|
|
288
|
-
invalidates it; null: max_slot_wal_keep_size unset, nothing caps it), at } or { error }, publication, running, failure, position,
|
|
289
|
-
applied, database: { lsn, at } or { error }, lag: { bytes, seconds }, lastBatch: { appliedAt, committedAt },
|
|
290
|
-
tables: [{ table, state (streamed, catching-up, waiting, left-out), why, catchUp: { phase (queued, polling,
|
|
291
|
-
applying), from, startedAt, held, retryAt }, applied: { inserts, updates, deletes, truncates } }] }.
|
|
292
|
-
jsonPaths: [{ table, column, steps, either, canonical, asText (false: a path's -> and ->> share it), name,
|
|
293
|
-
state (waiting, backfilling, ready, dropped), rowsTotal, rowsDone, marked, statements, slowestMs, totalMs,
|
|
294
|
-
fill: { rows, bytesPerRow, seconds, basis (measured: from the last fill of its column; disk or text: modelled from
|
|
295
|
-
the column's bytes a row on disk, or its text's before it is on disk), rowsPerSecond } (the fill's estimate it
|
|
296
|
-
was decided on), filled: { rows, seconds } (what its own fill measured), queued (a backfill waiting for the fill
|
|
297
|
-
under way), keptAt, readyAt, why }]. roles (a run with a roles file, mason/roles.yml: what a user added
|
|
298
|
-
with --role may read): { file, roles: [name], problem (what is wrong with the file now, every statement of a
|
|
299
|
-
user with a role refused while there is; null), default (the role of a user added without --role; null: the
|
|
300
|
-
file names none, and such a user reads nothing), all (the roles of tables: all) }
|
|
301
|
-
|
|
302
284
|
In replication mode a Stream section says where the stream is: its position against the database's (bytes behind),
|
|
303
285
|
when the newest change applied was committed and how long after it was applied, and each added table — streamed,
|
|
304
286
|
with the changes applied this run; being caught up by one poll; not streamed yet; or left out — and why.
|
|
@@ -311,6 +293,73 @@ const STATUS_USAGE = `usage: mason status [--json]
|
|
|
311
293
|
waiting (its fill's cost holds it back: the line says the estimate and what would store it), queued (its column
|
|
312
294
|
added, its fill after the one under way), backfilling (its existing rows being filled, N %) or ready (read from its
|
|
313
295
|
column). MASON_STORE_JSON_PATHS=off turns it off.
|
|
296
|
+
|
|
297
|
+
The --json answer: { running, pid, postgres: { url, host, port, database, sslmode (require when it is TLS), shown (the URL with
|
|
298
|
+
<user>:<password> in the credentials' place), psql (its psql line), listener:
|
|
299
|
+
{ host (what it is bound to), tls (certificate, self-signed or null), fingerprint } }, source: { name (the
|
|
300
|
+
source's: its sources file's), file (the sources file), shownFile (it from the project's root), url, database, remote, version, standby, changes,
|
|
301
|
+
replication, demo (the demo database this run hosts), textOrder (what reading the database's text order left to say), sync: { default (every table's sync unless its entry
|
|
302
|
+
says its own: off, poll 30s), written (the file's top sync: says it; false: the default) } }, warnings, folder, project, tables: [{ table, rows,
|
|
303
|
+
bytes, key, added, addedColumns, copy, error (why its sync failed: its first sync, or the poll, catch-up, stream or
|
|
304
|
+
publication check keeping a synced table current; null otherwise), kept, sync: { state (synced, syncing,
|
|
305
|
+
behind: served and kept but older than its sync: promises, failed, waiting), syncedAt (when
|
|
306
|
+
it was last brought current), syncSeconds (how long that took), syncedBy (copy, poll, catch-up, stream),
|
|
307
|
+
ageSeconds (how long ago that was), asOf (the time the synced data holds the database as of: kept's syncedAt),
|
|
308
|
+
promisedSeconds (the age its sync: promises not to pass; null: sync: off promises nothing), behindBySeconds
|
|
309
|
+
(how far past it, 0 when not; null when nothing is promised), behindWhy (why, while behind; else null) }, … }], totals: { added, copied, failed, rows (the synced tables' rows),
|
|
310
|
+
syncedAt (the latest sync of any) }, engine: { file, bytes, keptDays (MASON_HISTORY_DAYS), version (the run's), startedAt,
|
|
311
|
+
uptimeSeconds, restarts (the times its supervisor started the run again: { count, last: { at, why, upSeconds }
|
|
312
|
+
or null }, or { error }) }, disk: { path (where the synced data's file is), freeBytes, copyBytes (the file),
|
|
313
|
+
projectedBytes (what the first syncs running are projected to add; null: none, or too early to say), error },
|
|
314
|
+
sessions: { open (the Postgres API's sessions logged in), byClient: [{ user ('' a token's), client (its
|
|
315
|
+
application_name), open, running }] }, share: (null, or what mason share has open) { port, fingerprint, sessions,
|
|
316
|
+
refusedLogins, droppedConnections, shares: [{ user, from (command: mason share; page: the web page), role, reads, openedAt, via, host, port, endsAt, state }] }, shareEnded: (null, or the last share that ended) { user, from, why, at }, queries: { keptDays, day, week } (the last day's and the last 7
|
|
317
|
+
days' statements, each { days, covered (the days of it the request log keeps), statements, completed, refused,
|
|
318
|
+
failed, cancelled, rows, bytesSent, durationMs: { avg, p50, p95 } (the completed ones') or null }; { error }
|
|
319
|
+
when it could not be read), slowQueries, refusedQueries, refusedTotal (how many in all), unsupported: [{ gap
|
|
320
|
+
(its entry of docs/SQL-SUPPORT.md), title (what it is; null for an entry this Mason no longer has), runs
|
|
321
|
+
(the statements it refused in the last day), users, clients (application_name), last, example (the latest,
|
|
322
|
+
cut at 1,000 characters) }] (the most refused first: GET /api/unsupported) or { error },
|
|
323
|
+
recentErrors: { sync: [{ at, kind, failed, table, message }] (the latest failures of the sync and restarts,
|
|
324
|
+
newest last: mason logs --events), error (why they could not be read, or null), queries: [{ requestId,
|
|
325
|
+
startedAt, outcome, sqlstate, message, query, … }] (the latest statements refused, failed or cancelled, newest
|
|
326
|
+
last, as mason logs --requests --json shows each: mason logs --requests --failed), queriesError (or null) },
|
|
327
|
+
readiness (what mason wait waits on), ui (whether this run serves the web page: mason ui), served (this is
|
|
328
|
+
mason serve's run: mason ui --ssh opens its page), sourcesReadOnly (why the page offers no Add or Remove — the
|
|
329
|
+
project's files deployed read-only, MASON_PROJECT_READONLY, which mason add and mason remove refuse too, or a
|
|
330
|
+
server's sources file it cannot write — or null), next }.
|
|
331
|
+
Each table's kept is how it is kept current now (absent for a table the sync does not hold):
|
|
332
|
+
{ by: poll, sync: { mode: off } or { mode: poll, every (30s), seconds } (its entry's sync:, else the file's,
|
|
333
|
+
else off), syncedAt (its last whole read or poll), nextAt (when its next poll is due; null: off, or not polled
|
|
334
|
+
now), pending (only when it applies: why it is not kept current now, its phase included: it waits for
|
|
335
|
+
its first sync, it is being read whole, its first sync failed, it is read again whole, caught up first) }
|
|
336
|
+
or { by: recopy, copiedAt, syncedAt, pending (only when it applies, as the poll's) }
|
|
337
|
+
or { by: replication, slot, state (streamed, catching-up, waiting, left-out), why, position, lastCommit,
|
|
338
|
+
notRunning (why the stream is not running, null when it is) }; each with promisedSeconds (the age of
|
|
339
|
+
its synced data past which it is behind: poll, its interval plus the larger of 60 s and twice its last poll;
|
|
340
|
+
replication, 120 s; off, null) and lateWhy (what would make it late now: the pass's or its own poll's
|
|
341
|
+
time, which Mason waits four times, the stream not running); with Mason stopped: { running: false, project, source: { name,
|
|
342
|
+
file, url, error (the sources file's problem, or null) }, engine, tables: [{ table }], lastRun, lastStop:
|
|
343
|
+
{ at, how } or null, next }; hung: the same with hung: { pid, problem, why },
|
|
344
|
+
exit 1. In replication mode stream: { slot, slotWal: { walHeldBytes (the WAL the slot holds on the
|
|
345
|
+
database), walStatus (reserved, extended, unreserved, lost), safeWalSizeBytes (the WAL left before Postgres
|
|
346
|
+
invalidates it; null: max_slot_wal_keep_size unset, nothing caps it), at } or { error }, publication, running, failure, position,
|
|
347
|
+
applied, database: { lsn, at } or { error }, lag: { bytes, seconds }, lastBatch: { appliedAt, committedAt },
|
|
348
|
+
tables: [{ table, state (streamed, catching-up, waiting, left-out), why, catchUp: { phase (queued, polling,
|
|
349
|
+
applying), from, startedAt, held, retryAt }, applied: { inserts, updates, deletes, truncates } }] }.
|
|
350
|
+
jsonPaths: [{ table, column, steps, either, canonical, asText (false: a path's -> and ->> share it), name,
|
|
351
|
+
state (waiting, backfilling, ready, dropped), rowsTotal, rowsDone, marked, statements, slowestMs, totalMs,
|
|
352
|
+
fill: { rows, bytesPerRow, seconds, basis (measured: from the last fill of its column; disk or text: modelled from
|
|
353
|
+
the column's bytes a row on disk, or its text's before it is on disk), rowsPerSecond } (the fill's estimate it
|
|
354
|
+
was decided on), filled: { rows, seconds } (what its own fill measured), queued (a backfill waiting for the fill
|
|
355
|
+
under way), keptAt, readyAt, why }]. generatedTables: [{ table, file, kind, method (whole: its SQL run again),
|
|
356
|
+
state (refused, waiting, building, failed, current, behind), targetLag, behindSeconds (across its chain: the
|
|
357
|
+
oldest base change below it not in it), behindSince (when that change was committed; null: none), missingTarget, serving,
|
|
358
|
+
lastRun: { at, seconds, rows }, error (the last run's, no value of a row in it), retryAt,
|
|
359
|
+
refusals: [{ key, message, why, options, text }], waitingFor, warnings }]. roles (a run with a roles file, mason/roles.yml: what a user added
|
|
360
|
+
with --role may read): { file, roles: [name], problem (what is wrong with the file now, every statement of a
|
|
361
|
+
user with a role refused while there is; null), default (the role of a user added without --role; null: the
|
|
362
|
+
file names none, and such a user reads nothing), all (the roles of tables: all) }
|
|
314
363
|
`;
|
|
315
364
|
export const STATUS = {
|
|
316
365
|
needs: "project",
|
|
@@ -318,14 +367,17 @@ export const STATUS = {
|
|
|
318
367
|
summary: "what Mason is doing: the database, each synced table, the Postgres API, its statements and errors",
|
|
319
368
|
usage: STATUS_USAGE,
|
|
320
369
|
options: { json: { type: "boolean" } },
|
|
321
|
-
json: ["running", "pid", "postgres", "source", "warnings", "folder", "project", "tables", "totals", "engine", "disk", "sessions", "stream", "jsonPaths", "queries", "slowQueries", "refusedQueries",
|
|
322
|
-
"refusedTotal", "unsupported", "recentErrors", "readiness", "ui",
|
|
323
|
-
"roles", "share", "lastRun", "lastStop", "hung", "next"],
|
|
370
|
+
json: ["running", "pid", "postgres", "source", "warnings", "folder", "project", "tables", "totals", "engine", "disk", "sessions", "stream", "jsonPaths", "generatedTables", "queries", "slowQueries", "refusedQueries",
|
|
371
|
+
"refusedTotal", "unsupported", "recentErrors", "readiness", "ui", "served", "sourcesReadOnly",
|
|
372
|
+
"roles", "share", "shareEnded", "lastRun", "lastStop", "hung", "next"],
|
|
324
373
|
};
|
|
325
374
|
export async function status(argv) {
|
|
326
375
|
const f = parseCommand(STATUS, argv);
|
|
327
376
|
if (typeof f === "number")
|
|
328
377
|
return f;
|
|
378
|
+
const extra = noArguments(STATUS, f.positionals, f.values.json === true);
|
|
379
|
+
if (extra !== null)
|
|
380
|
+
return extra;
|
|
329
381
|
const r0 = await theRun(process.cwd());
|
|
330
382
|
// the first line goes straight to the point (Status: running); when the service is not running, the same rows and
|
|
331
383
|
// tables show, with "not running" for each service that is not
|
|
@@ -396,7 +448,7 @@ export async function status(argv) {
|
|
|
396
448
|
["Listener", listenerText(run.postgresListener, postgresLink(run.postgresUrl).port)],
|
|
397
449
|
...shareRows(s.share),
|
|
398
450
|
// the web page (mason ui), without its one-time token: mason ui prints the URL that carries it, and opens it
|
|
399
|
-
...(s.ui === true ? [["Web UI", `${run.control.url}/ (mason ui opens it)`]] : []),
|
|
451
|
+
...(s.ui === true ? [["Web UI", s.served === true ? "from your machine: mason ui --ssh <user@server>" : `${run.control.url}/ (mason ui opens it)`]] : []),
|
|
400
452
|
// the roles file as the listener reads it now (mason user add --role): its roles, or what is wrong with it
|
|
401
453
|
...(s.roles && s.roles.file !== null
|
|
402
454
|
? [["Roles", s.roles.problem !== null ? st.red(`${s.roles.problem} (every statement of a user with a role is refused until it is fixed)`)
|
|
@@ -424,6 +476,8 @@ export async function status(argv) {
|
|
|
424
476
|
out(l);
|
|
425
477
|
for (const l of jsonPathLines(s.jsonPaths ?? [], st))
|
|
426
478
|
out(l);
|
|
479
|
+
for (const l of generatedLines(s.generatedTables ?? [], st))
|
|
480
|
+
out(l);
|
|
427
481
|
for (const l of queryLines(queries, st))
|
|
428
482
|
out(l);
|
|
429
483
|
for (const l of slowLines(slowReply.status === 200 ? slow : { error: String(slowReply.body.error ?? slowReply.status) }, st))
|
|
@@ -459,6 +513,7 @@ export function sourceRows(s) {
|
|
|
459
513
|
`${s.source.replication.owned ? "Mason's own, carrying them alone" : "named: read as it is, never altered"}); the others by a SELECT of their changed rows`
|
|
460
514
|
: s.source.demo ? "the demo database is checked this way only"
|
|
461
515
|
: "sync: replication on an entry streams that table as it happens instead")],
|
|
516
|
+
...(s.source.textOrder ? [["Text order", s.source.textOrder]] : []),
|
|
462
517
|
];
|
|
463
518
|
}
|
|
464
519
|
/** An SQL statement on one line, cut at `max` characters (the whole of it is mason logs --requests --json's). */
|
|
@@ -487,6 +542,38 @@ export function jsonPathLines(paths, st) {
|
|
|
487
542
|
lines.push(` ${st.dim(`${dropped} dropped (mason status --json says why)`)}`);
|
|
488
543
|
return lines;
|
|
489
544
|
}
|
|
545
|
+
/** a generated table's state in the words mason status prints: how it is kept, how far behind, its last run */
|
|
546
|
+
function generatedWords(g) {
|
|
547
|
+
const how = `${g.kind ?? "?"} ${g.method}, target ${g.targetLag ?? "?"}`;
|
|
548
|
+
const last = g.lastRun === null ? "" : `; last re-run ${duration(g.lastRun.seconds)}, ${n(g.lastRun.rows)} row${g.lastRun.rows === 1 ? "" : "s"}`;
|
|
549
|
+
if (g.state === "waiting")
|
|
550
|
+
return `${how}, waiting for ${g.waitingFor ?? "a table"} to sync`;
|
|
551
|
+
if (g.state === "building")
|
|
552
|
+
return `${how}, being built`;
|
|
553
|
+
if (g.state === "refused")
|
|
554
|
+
return g.serving ? `${how}, its file is refused; it keeps serving what it was built from${last}` : "refused";
|
|
555
|
+
const lag = g.behindSeconds === 0 ? "nothing behind" : `${duration(g.behindSeconds)} behind`;
|
|
556
|
+
const missed = g.missingTarget ? ", past its target" : "";
|
|
557
|
+
const failed = g.error === null ? "" : `; its last run failed, tried again at ${g.retryAt ?? "?"}: ${g.error}`;
|
|
558
|
+
return `${how}, ${lag}${missed}${last}${failed}`;
|
|
559
|
+
}
|
|
560
|
+
/** mason status's generated tables: a line each, and a refused one's reasons and ways out under it, whole */
|
|
561
|
+
export function generatedLines(tables, st) {
|
|
562
|
+
if (tables.length === 0)
|
|
563
|
+
return [];
|
|
564
|
+
const width = Math.max(...tables.map((g) => entryName(g.table).length));
|
|
565
|
+
const lines = [`\n${st.bold("Generated tables")} ${st.dim("(tables declared in SQL; mason help generated)")}:`];
|
|
566
|
+
for (const g of tables) {
|
|
567
|
+
const words = generatedWords(g);
|
|
568
|
+
const paint = g.state === "refused" || g.state === "failed" || g.missingTarget ? st.red : g.state === "current" ? st.green : g.state === "behind" ? (x) => x : st.yellow;
|
|
569
|
+
lines.push(` ${entryName(g.table).padEnd(width)} ${paint(words)}`);
|
|
570
|
+
for (const r of g.refusals)
|
|
571
|
+
lines.push(...r.text.split("\n").map((l) => ` ${l}`));
|
|
572
|
+
for (const w of g.warnings)
|
|
573
|
+
lines.push(` ${st.yellow(w)}`);
|
|
574
|
+
}
|
|
575
|
+
return lines;
|
|
576
|
+
}
|
|
490
577
|
/** The copies in all, as mason status's `Copies` row says them: `3 of 3 added synced, 1,234 rows, the last synced 5 s ago`. */
|
|
491
578
|
export function totalsText(t, now = Date.now()) {
|
|
492
579
|
const last = t.syncedAt === null ? "" : `, the last synced ${duration((now - Date.parse(t.syncedAt)) / 1000)} ago`;
|
|
@@ -709,10 +796,12 @@ const TABLES_USAGE = `usage: mason tables [<table>…] [--all] [--json]
|
|
|
709
796
|
mason tables --all every table with its columns, at once (long)
|
|
710
797
|
|
|
711
798
|
--all every table with its columns, not only the named ones
|
|
712
|
-
--json as data:
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
799
|
+
--json as data: its shape is at the end
|
|
800
|
+
|
|
801
|
+
The --json answer: { database, source: { name (the source's: its sources file's), file (the sources file) },
|
|
802
|
+
tables: [{ table, rows, bytes, primaryKey, comment, addable,
|
|
803
|
+
reason, added, addedColumns (the columns it syncs: null when not added), copyKey (the key: the sources
|
|
804
|
+
file keys it by, null: its primary key), columns: [{ name, type, nullable, comment, references }] }], next }
|
|
716
805
|
`;
|
|
717
806
|
export const TABLES = {
|
|
718
807
|
// the project, not the run (the hands-on review of #451, item 15): what it lists is the database's catalog and the
|
|
@@ -815,9 +904,9 @@ async function readCatalog(p) {
|
|
|
815
904
|
return src;
|
|
816
905
|
const demo = p.config.demo === true;
|
|
817
906
|
const fix = `change the URL where it is (${src.from}) when the database's password or host changed`;
|
|
818
|
-
const parsed = parseSource(src.resolved);
|
|
907
|
+
const parsed = parseSource(src.resolved, src.from);
|
|
819
908
|
if ("problem" in parsed)
|
|
820
|
-
return { message:
|
|
909
|
+
return { message: parsed.problem, code: "bad_source", fix };
|
|
821
910
|
const connected = await connectSource(parsed.source, { demo });
|
|
822
911
|
if ("failure" in connected) {
|
|
823
912
|
// the demo database's way out is its run (src/cli/demo.ts): its hint is the fix, mason start the next command
|
|
@@ -899,12 +988,22 @@ async function viaRun(run, route, body) {
|
|
|
899
988
|
const ADD_USAGE = `usage: mason add <table>… [--columns <column>,…] [--include <column>,…] [--key <column>,…] [--sync <sync>] [--json]
|
|
900
989
|
|
|
901
990
|
Syncs tables into Mason: each one is added to the project's sources file (mason/sources/<name>.yml), under tables:
|
|
991
|
+
|
|
902
992
|
mason add orders every column: orders: {}
|
|
903
993
|
mason add orders --columns id,status,total those columns: orders: { columns: [id, status, total] }
|
|
904
994
|
mason add orders customers several tables, every column of each
|
|
905
995
|
mason add users --include password_hash every column, and one that looks like a secret
|
|
906
996
|
mason add event_log --key tenant_id,id a table with no primary key, keyed by those columns
|
|
907
997
|
mason add orders --sync "poll 30s" polled every 30 seconds: orders: { sync: poll 30s }
|
|
998
|
+
|
|
999
|
+
--columns <a,b,…> only these columns (one table at a time: the list is that table's); a list flag takes a comma
|
|
1000
|
+
list or the flag repeated, the same: --columns id,status is --columns id --columns status
|
|
1001
|
+
--include <a,b,…> these columns as well as every other one — a column that looks like a secret, named on purpose
|
|
1002
|
+
--key <a,b,…> for a table with no primary key: the columns that tell its rows apart, which it is keyed by (one
|
|
1003
|
+
table at a time; polled or off: logical replication cannot apply a change by them)
|
|
1004
|
+
--sync <sync> how it is kept current after its first sync, written on its entry: off, "poll 30s" or replication
|
|
1005
|
+
--json as data: its shape is at the end
|
|
1006
|
+
|
|
908
1007
|
A table is its name (orders) or schema.table (billing.invoices). Its primary key is always synced. The rest of the
|
|
909
1008
|
file — your comments, other tables — stays as it was.
|
|
910
1009
|
|
|
@@ -925,21 +1024,17 @@ const ADD_USAGE = `usage: mason add <table>… [--columns <column>,…] [--inclu
|
|
|
925
1024
|
With Mason running the first sync starts at once; mason wait returns when it is done and the table is served. Before
|
|
926
1025
|
mason start (mason init && mason add orders && mason start) the entry is written, checked against the database as
|
|
927
1026
|
the run checks it, and the first sync begins at mason start. Adding a table again with other columns changes its entry,
|
|
928
|
-
and the table is synced again whole with them.
|
|
1027
|
+
and the table is synced again whole with them. A project deployed as committed (MASON_PROJECT_READONLY=1 where Mason
|
|
1028
|
+
runs) is refused, its files left as they are: add the table where the project is committed, then deploy.
|
|
929
1029
|
|
|
930
|
-
--
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
bigOff (it is over ${BIG_TABLE_ROWS.toLocaleString("en-US")} rows and its entry was written sync: off), notReplicated (why
|
|
939
|
-
its entry was written sync: poll 5m under a file whose sync: is replication, or null), pollCost (an estimate of
|
|
940
|
-
what a frequent poll of a big table costs your database, or null) } or { table, error }],
|
|
941
|
-
file (the sources file), error (when a table was not added: { code, message, why }), next };
|
|
942
|
-
exit 1 when a table could not be added
|
|
1030
|
+
The --json answer: { added: [{ table, entry, columns (null: every one), exclude, key, addedKey (key columns
|
|
1031
|
+
added to --columns), skippedSecrets, already (the file said so already), sync: { value (off, poll
|
|
1032
|
+
30s), from (entry: its own sync:, file: the file's, default: off, as the file writes none) },
|
|
1033
|
+
bigOff (it is over ${BIG_TABLE_ROWS.toLocaleString("en-US")} rows and its entry was written sync: off), notReplicated (why
|
|
1034
|
+
its entry was written sync: poll 5m under a file whose sync: is replication, or null), pollCost (an estimate of
|
|
1035
|
+
what a frequent poll of a big table costs your database, or null) } or { table, error }],
|
|
1036
|
+
file (the sources file), error (when a table was not added: { code, message, why }), next };
|
|
1037
|
+
exit 1 when a table could not be added
|
|
943
1038
|
`;
|
|
944
1039
|
export const ADD = {
|
|
945
1040
|
// the project, not the run: the sources file is the list, so a table is added before mason start as well as after
|
|
@@ -947,7 +1042,7 @@ export const ADD = {
|
|
|
947
1042
|
name: "add",
|
|
948
1043
|
summary: "sync tables into Mason, adding them to the sources file: mason add orders (--columns id,status: those only)",
|
|
949
1044
|
usage: ADD_USAGE,
|
|
950
|
-
options: { json: { type: "boolean" }, columns: { type: "string" }, include: { type: "string" }, key: { type: "string" }, sync: { type: "string" } },
|
|
1045
|
+
options: { json: { type: "boolean" }, columns: { type: "string", multiple: true }, include: { type: "string", multiple: true }, key: { type: "string", multiple: true }, sync: { type: "string" } },
|
|
951
1046
|
json: ["added", "file", "error", "next"],
|
|
952
1047
|
};
|
|
953
1048
|
/** `--sync`, read by the sources file's own grammar (src/lib/table-sync.ts parseSync): a sync, or the usage's problem */
|
|
@@ -990,7 +1085,7 @@ export async function add(argv) {
|
|
|
990
1085
|
if (typeof f === "number")
|
|
991
1086
|
return f;
|
|
992
1087
|
const json = f.values.json === true;
|
|
993
|
-
const specs = addSpecs(f.positionals, f.values.columns, f.values.include, f.values.key);
|
|
1088
|
+
const specs = addSpecs(f.positionals, listFlag(f.values.columns), listFlag(f.values.include), listFlag(f.values.key));
|
|
994
1089
|
if (typeof specs === "string")
|
|
995
1090
|
return failed("add", { message: specs, code: "usage", usage: ADD_USAGE }, { json });
|
|
996
1091
|
const sync = addSync(f.values.sync);
|
|
@@ -1003,6 +1098,11 @@ export async function add(argv) {
|
|
|
1003
1098
|
// (a state with no project never meets the need: reach said it)
|
|
1004
1099
|
if (where === null || where.project === null)
|
|
1005
1100
|
return 1;
|
|
1101
|
+
// a project whose files are deployed read-only (src/cli/project.ts projectWriteRefusal): refused before the run or the
|
|
1102
|
+
// database is asked, whether Mason is up or down — the run's own record counts, whatever this shell's environment says
|
|
1103
|
+
const readOnly = projectWriteRefusal(where.project.root, process.env);
|
|
1104
|
+
if (readOnly !== null)
|
|
1105
|
+
return failed("add", readOnly, { json });
|
|
1006
1106
|
if (where.state === "up") {
|
|
1007
1107
|
// Mason up: the run adds it (the copy starts now, and the run's own re-read never races the file's write)
|
|
1008
1108
|
const run = where.run;
|
|
@@ -1128,16 +1228,22 @@ export function addedLines(b, file) {
|
|
|
1128
1228
|
] };
|
|
1129
1229
|
}
|
|
1130
1230
|
// ------------------------------------------------------------------ mason remove
|
|
1131
|
-
const REMOVE_USAGE = `usage: mason remove <table>… [--json]
|
|
1231
|
+
const REMOVE_USAGE = `usage: mason remove <table>… [--dry-run] [--json]
|
|
1132
1232
|
|
|
1133
1233
|
Takes tables out of the project: each one's entry in the sources file (mason/sources/<name>.yml) is removed, the
|
|
1134
1234
|
rest of the file as it was. With Mason running it stops syncing the table and drops its synced data at once; with Mason
|
|
1135
|
-
stopped, what an earlier run synced is dropped at the next mason start.
|
|
1235
|
+
stopped, what an earlier run synced is dropped at the next mason start. A project deployed as committed
|
|
1236
|
+
(MASON_PROJECT_READONLY=1 where Mason runs) is refused, its files left as they are: remove it where the project is
|
|
1237
|
+
committed, then deploy.
|
|
1136
1238
|
|
|
1137
1239
|
mason remove customers
|
|
1138
1240
|
mason remove customers billing.invoices several at once
|
|
1241
|
+
mason remove customers --dry-run what it would remove, changing nothing
|
|
1139
1242
|
|
|
1140
|
-
--
|
|
1243
|
+
--dry-run say which entries would go, and change nothing: the file is read, never written, and neither Mason nor
|
|
1244
|
+
the database is asked
|
|
1245
|
+
--json as data: { dryRun, removedTables (the tables, schema.table; on a dry run, the ones that would go), removed
|
|
1246
|
+
(the same, kept until 1.0; not on a dry run), file (the sources file), next }
|
|
1141
1247
|
`;
|
|
1142
1248
|
export const REMOVE = {
|
|
1143
1249
|
// the project, not the run, as mason add
|
|
@@ -1145,8 +1251,8 @@ export const REMOVE = {
|
|
|
1145
1251
|
name: "remove",
|
|
1146
1252
|
summary: "take tables out of the sources file: mason remove customers (its synced data is dropped)",
|
|
1147
1253
|
usage: REMOVE_USAGE,
|
|
1148
|
-
options: { json: { type: "boolean" } },
|
|
1149
|
-
json: ["removed", "file", "next"],
|
|
1254
|
+
options: { json: { type: "boolean" }, "dry-run": { type: "boolean" } },
|
|
1255
|
+
json: ["dryRun", "removedTables", "removed", "file", "next"],
|
|
1150
1256
|
};
|
|
1151
1257
|
export async function remove(argv) {
|
|
1152
1258
|
const f = parseCommand(REMOVE, argv);
|
|
@@ -1160,6 +1266,12 @@ export async function remove(argv) {
|
|
|
1160
1266
|
// (a state with no project never meets the need: reach said it)
|
|
1161
1267
|
if (where === null || where.project === null)
|
|
1162
1268
|
return 1;
|
|
1269
|
+
// read-only, as mason add (src/cli/project.ts projectWriteRefusal): refused before the run or the database is asked
|
|
1270
|
+
const readOnly = projectWriteRefusal(where.project.root, process.env);
|
|
1271
|
+
if (readOnly !== null)
|
|
1272
|
+
return failed("remove", readOnly, { json });
|
|
1273
|
+
if (f.values["dry-run"] === true)
|
|
1274
|
+
return removeDryRun(where.project, where.state === "up", names, json);
|
|
1163
1275
|
if (where.state === "up") {
|
|
1164
1276
|
// Mason up: the run removes it and drops the copy at once (src/cli/run-server.ts removeTables)
|
|
1165
1277
|
const run = where.run;
|
|
@@ -1181,13 +1293,53 @@ function removedAnswer(r, names, running, json) {
|
|
|
1181
1293
|
const next = running ? [["mason status", "the tables left"]]
|
|
1182
1294
|
: [["mason start", "Mason in the background, for this project: it syncs the tables left"], ["mason status", "the tables left"]];
|
|
1183
1295
|
if (json) {
|
|
1184
|
-
out(jsonText(REMOVE, { removed: b.removed, file: b.file }, next));
|
|
1296
|
+
out(jsonText(REMOVE, { dryRun: false, removed: b.removed, removedTables: b.removed, file: b.file }, next));
|
|
1185
1297
|
return 0;
|
|
1186
1298
|
}
|
|
1187
1299
|
const rel = path.relative(process.cwd(), b.file);
|
|
1188
1300
|
out(reportText({ ...removedReport(b.removed, rel.startsWith("..") ? b.file : rel, { running }), next }, styleOut()));
|
|
1189
1301
|
return 0;
|
|
1190
1302
|
}
|
|
1303
|
+
/**
|
|
1304
|
+
* `mason remove --dry-run`: the entries the sources file would lose, CHANGING NOTHING — read as the edit reads the file
|
|
1305
|
+
* (`sourceToEdit`: its refusals said the same), each name found as the edit finds it (`findEntry`); neither the run nor
|
|
1306
|
+
* the database is asked, so the file's other entries are not checked against the database here, as the edit does.
|
|
1307
|
+
*/
|
|
1308
|
+
async function removeDryRun(p, up, names, json) {
|
|
1309
|
+
if (!up) {
|
|
1310
|
+
const st = await runState(p);
|
|
1311
|
+
// a hung run refuses the edit itself (withoutRun): a dry run says what the edit would
|
|
1312
|
+
if (st.kind === "hung")
|
|
1313
|
+
return failed("remove", { ...asFailure(notAnswering(p, st.run)), code: "unreachable" }, { json });
|
|
1314
|
+
}
|
|
1315
|
+
const r = edited(() => {
|
|
1316
|
+
const at = sourceToEdit(p.folder, p.root, () => projectSource(p));
|
|
1317
|
+
const tables = names.map((n) => {
|
|
1318
|
+
const e = findEntry(at.parsed, n);
|
|
1319
|
+
if ("problem" in e)
|
|
1320
|
+
throw new Refusal(e.usage ? 400 : 404, e.problem);
|
|
1321
|
+
return e.table;
|
|
1322
|
+
});
|
|
1323
|
+
return { removed: [...new Set(tables)], file: at.file };
|
|
1324
|
+
});
|
|
1325
|
+
if (!("ok" in r)) {
|
|
1326
|
+
return failed("remove", { message: `${names.join(", ")} would not be removed`, code: r.status === 404 ? "not_found" : r.status === 400 ? "usage" : "refused", why: r.error,
|
|
1327
|
+
fix: "nothing was changed (a dry run)", next: [["mason status", "the tables added"]] }, { json });
|
|
1328
|
+
}
|
|
1329
|
+
const b = r.ok;
|
|
1330
|
+
const next = [[`mason remove ${names.join(" ")}`, "remove them"]];
|
|
1331
|
+
if (json) {
|
|
1332
|
+
out(jsonText(REMOVE, { dryRun: true, removedTables: b.removed, file: b.file }, next));
|
|
1333
|
+
return 0;
|
|
1334
|
+
}
|
|
1335
|
+
const rel = path.relative(process.cwd(), b.file);
|
|
1336
|
+
const one = b.removed.length === 1;
|
|
1337
|
+
out(reportText({ headline: `mason remove would remove ${b.removed.join(", ")}; nothing was changed`, details: [["from", rel.startsWith("..") ? b.file : rel]],
|
|
1338
|
+
tips: [up ? `Mason would stop syncing ${one ? "it" : "them"} and drop ${one ? "its" : "their"} synced data at once`
|
|
1339
|
+
: `what an earlier run synced of ${one ? "it" : "them"} would be dropped at the next mason start`,
|
|
1340
|
+
"mason remove also checks the file's other entries against the database, which a dry run does not ask"], next }, styleOut()));
|
|
1341
|
+
return 0;
|
|
1342
|
+
}
|
|
1191
1343
|
/** What `mason remove` prints for the tables removed, `file` the sources file as the reader is shown it (the web page
|
|
1192
1344
|
* says the same, src/web/app.js; tests/web-page.test.ts holds the two equal). */
|
|
1193
1345
|
export function removedReport(removed, file, o = {}) {
|
|
@@ -1201,48 +1353,86 @@ export function removedReport(removed, file, o = {}) {
|
|
|
1201
1353
|
/** An added table as mason wait --json says it: its copy, its error, and how it is kept current, as the run's state has
|
|
1202
1354
|
* them (mason status --json's own fields, never derived again here) */
|
|
1203
1355
|
export function waitTable(t) {
|
|
1204
|
-
return { table: t.table, copy: t.copy, error: t.error, ...(t.kept ? { kept: t.kept } : {}) };
|
|
1356
|
+
return { table: t.table, copy: t.copy, error: t.error, ...(t.kept ? { kept: t.kept } : {}), ...(t.sync ? { sync: t.sync } : {}) };
|
|
1205
1357
|
}
|
|
1206
|
-
const WAIT_USAGE = `usage: mason wait [--timeout <
|
|
1358
|
+
const WAIT_USAGE = `usage: mason wait [<table>…] [--timeout <duration>] [--json]
|
|
1207
1359
|
|
|
1208
1360
|
Waits until every added table is synced and served, showing every added table on a line of its own: 100% for the
|
|
1209
1361
|
finished (in the order they finished), a bar for the ones still syncing, at the bottom. Run it after mason add.
|
|
1362
|
+
Named synced tables: waits for those alone, until the first sync of each is done and it is served. Named, a
|
|
1363
|
+
generated table (alone) instead: waits until it holds every change Mason had synced when the wait began, across its
|
|
1364
|
+
chain (the tables it is generated from, and theirs), so what is read from it next includes them.
|
|
1210
1365
|
|
|
1211
1366
|
mason add orders && mason wait
|
|
1212
|
-
mason wait --timeout
|
|
1367
|
+
mason wait --timeout 1h a large table: wait up to an hour
|
|
1368
|
+
mason wait orders customers these synced tables only
|
|
1369
|
+
mason wait daily_requests a generated table: until it holds what is synced now
|
|
1370
|
+
|
|
1371
|
+
--timeout <duration> how long to wait (default 600, seconds; 10m, 1h); the sync goes on after, and mason wait
|
|
1372
|
+
again picks it up
|
|
1373
|
+
--json as data: its shape is at the end
|
|
1213
1374
|
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
it is not kept current now, its phase included),
|
|
1221
|
-
stream (replication mode: where the stream is, as mason status --json says it), error (on a
|
|
1222
|
-
failure or a timeout: { code, message, next }), next }
|
|
1375
|
+
Exit status: 0 ready (a table behind what its sync: promises is said on its line, and ready); 1 a first sync failed,
|
|
1376
|
+
or the sync keeping a synced table current failed (its poll, its catch-up, the stream, or a named publication that
|
|
1377
|
+
does not carry it), or the generated table's run failed, or it or a parent was refused
|
|
1378
|
+
before it was built (printed, with what to do; Mason tries a failed one again by itself); 124 timed out.
|
|
1379
|
+
A generated table that serves and holds everything synced is ready (0) though its last run failed: after editing its
|
|
1380
|
+
file, read mason status for whether the new definition was refused or failed.
|
|
1223
1381
|
|
|
1224
|
-
|
|
1382
|
+
The --json answer, once it ends: { ready, failed, waiting, timedOut, tables: [{ table, copy, error,
|
|
1383
|
+
kept }] (copy: { phase, rowsCopied, progress: the chunks done, reading and for how long, the
|
|
1384
|
+
rate, the seconds left, while its chunks are read }; kept as mason status --json says it —
|
|
1385
|
+
{ by: poll, sync: { mode: off } or { mode: poll, every, seconds }, syncedAt (its last whole read
|
|
1386
|
+
or poll), nextAt (its next poll; null: off) }; kept's pending, when it applies, says why
|
|
1387
|
+
it is not kept current now, its phase included; sync as mason status --json says it: its state —
|
|
1388
|
+
synced, syncing, behind, failed, waiting — asOf, promisedSeconds, behindBySeconds, behindWhy),
|
|
1389
|
+
stream (replication mode: where the stream is, as mason status --json says it), error (on a
|
|
1390
|
+
failure or a timeout: { code, message, next }), next }
|
|
1391
|
+
Named, a generated table: the same ready, failed, waiting, timedOut, error and next, tables
|
|
1392
|
+
empty, and generatedTable (as mason status --json's generatedTables says it: its state,
|
|
1393
|
+
behindSeconds, behindSince — when the oldest change not in it was committed, null: none)
|
|
1225
1394
|
`;
|
|
1395
|
+
/** how often a wait not drawn in place says its block again though nothing in it changed, so a long one never goes
|
|
1396
|
+
* quiet: the copies' wait and a generated table's */
|
|
1397
|
+
export const SAY_AGAIN_MS = 30_000;
|
|
1398
|
+
/** whether a wait says its block now: what it is changed (`what`, its numbers taken out), or `every` ms went by since it
|
|
1399
|
+
* last said it */
|
|
1400
|
+
export function sayAgain(what, said, saidAt, now, every = SAY_AGAIN_MS) {
|
|
1401
|
+
return what !== said || now - saidAt >= every;
|
|
1402
|
+
}
|
|
1226
1403
|
export const WAIT = {
|
|
1227
1404
|
needs: "run",
|
|
1228
1405
|
name: "wait",
|
|
1229
|
-
summary: "wait until every added table is synced
|
|
1406
|
+
summary: "wait until every added table (or those named) is synced, or a generated table holds what is synced now (1: a failure)",
|
|
1230
1407
|
usage: WAIT_USAGE,
|
|
1231
1408
|
options: { json: { type: "boolean" }, timeout: { type: "string" } },
|
|
1232
|
-
json: ["ready", "failed", "waiting", "timedOut", "tables", "stream", "error", "next"],
|
|
1409
|
+
json: ["ready", "failed", "waiting", "timedOut", "tables", "stream", "generatedTable", "error", "next"],
|
|
1233
1410
|
};
|
|
1234
1411
|
export async function wait(argv) {
|
|
1235
1412
|
const f = parseCommand(WAIT, argv);
|
|
1236
1413
|
if (typeof f === "number")
|
|
1237
1414
|
return f;
|
|
1238
1415
|
const json = f.values.json === true;
|
|
1239
|
-
const
|
|
1240
|
-
if (
|
|
1241
|
-
return failed("wait", { message: `--timeout
|
|
1416
|
+
const read = parseDuration(typeof f.values.timeout === "string" ? f.values.timeout : "600");
|
|
1417
|
+
if ("problem" in read)
|
|
1418
|
+
return failed("wait", { message: `--timeout ${read.problem}`, code: "usage", usage: WAIT_USAGE }, { json });
|
|
1419
|
+
const timeout = read.seconds;
|
|
1420
|
+
const named = [...new Set(f.positionals)];
|
|
1242
1421
|
const run = await runOr(WAIT, json);
|
|
1243
1422
|
if (!run)
|
|
1244
1423
|
return 1;
|
|
1245
1424
|
return guarded("wait", json, async () => {
|
|
1425
|
+
// tables named: a generated table alone waits for its chain; synced tables for their first syncs alone
|
|
1426
|
+
let only = null;
|
|
1427
|
+
if (named.length) {
|
|
1428
|
+
const pick = waitPick(await readState(run), named);
|
|
1429
|
+
if ("message" in pick)
|
|
1430
|
+
return failed("wait", pick, { json });
|
|
1431
|
+
if ("generated" in pick)
|
|
1432
|
+
return waitGenerated(run, pick.generated, timeout, json);
|
|
1433
|
+
only = new Set(pick.tables);
|
|
1434
|
+
}
|
|
1435
|
+
const again = named.length ? `mason wait ${named.join(" ")}` : "mason wait";
|
|
1246
1436
|
const until = Date.now() + timeout * 1000;
|
|
1247
1437
|
// a terminal gets the block — every added table on its line (tableLines), then the stream — redrawn in place;
|
|
1248
1438
|
// anything else (a coding agent's shell, a log) the block when what is under way changes, and every 30 s, so a
|
|
@@ -1266,7 +1456,7 @@ export async function wait(argv) {
|
|
|
1266
1456
|
}
|
|
1267
1457
|
// the same work, other numbers (a stream's positions included: they move with every change)
|
|
1268
1458
|
const what = lines.join("\n").replace(/[█░]+/g, "").replace(/\b[0-9A-F]+\/[0-9A-F]+\b/g, "#").replace(/[\d,.]+(%| rows\/s| s| min| h)?/g, "#");
|
|
1269
|
-
if (what
|
|
1459
|
+
if (sayAgain(what, said, saidAt, Date.now())) {
|
|
1270
1460
|
err(lines.map((l) => ` ${l}`).join("\n"));
|
|
1271
1461
|
said = what;
|
|
1272
1462
|
saidAt = Date.now();
|
|
@@ -1274,10 +1464,18 @@ export async function wait(argv) {
|
|
|
1274
1464
|
};
|
|
1275
1465
|
for (;;) {
|
|
1276
1466
|
// a copy's rate and progress are the run's own, measured per chunk (src/duck/sync.ts copyProgressNow)
|
|
1277
|
-
const
|
|
1467
|
+
const all = await readState(run);
|
|
1468
|
+
// named tables: the wait reads them alone, so its lines, its readiness and its answer are theirs
|
|
1469
|
+
const s = only === null ? all : { ...all, tables: all.tables.filter((t) => only.has(t.table)) };
|
|
1278
1470
|
const r = readiness(s);
|
|
1279
1471
|
const done = r.ready || r.failed.length > 0;
|
|
1280
1472
|
const picked = s.tables.filter((t) => t.added);
|
|
1473
|
+
// a table named and removed while it is waited for (mason remove, the page) has nothing left to wait for
|
|
1474
|
+
if (only !== null && picked.length < only.size) {
|
|
1475
|
+
const gone = [...only].filter((n) => !picked.some((t) => t.table === n));
|
|
1476
|
+
return failed("wait", { message: `${gone.join(", ")} ${gone.length === 1 ? "is" : "are"} not added any more`, code: "not_found",
|
|
1477
|
+
next: [["mason status", "the added tables"]] }, { json });
|
|
1478
|
+
}
|
|
1281
1479
|
// the outcome and what to run next, the same for a person and under --json
|
|
1282
1480
|
const readyNext = picked.length ? [...READY_STEPS] : addSteps(s.tables);
|
|
1283
1481
|
const failure = r.failed.length
|
|
@@ -1286,7 +1484,7 @@ export async function wait(argv) {
|
|
|
1286
1484
|
next: [["mason logs", "the detail of each failure"], ["mason status", "the rest of the project"]] }
|
|
1287
1485
|
: !done && Date.now() >= until
|
|
1288
1486
|
? { message: `not ready after ${timeout} s`, code: "timeout", why: r.waiting.join("\n"), fix: "nothing stopped: the syncs go on",
|
|
1289
|
-
next: [[
|
|
1487
|
+
next: [[again, "keep waiting (--timeout <duration> for longer)"], ["mason status", "what Mason is doing"]] }
|
|
1290
1488
|
: null;
|
|
1291
1489
|
if (json && (done || failure)) {
|
|
1292
1490
|
out(jsonText(WAIT, { ...r, timedOut: !done, tables: picked.map(waitTable),
|
|
@@ -1298,7 +1496,9 @@ export async function wait(argv) {
|
|
|
1298
1496
|
const meanwhile = done ? [] : [...(copied ? [["mason user add <name>", "a user for your BI tool or psql: a synced table can be queried now"]] : []), ["mason status", "what Mason is doing"]];
|
|
1299
1497
|
show([...tableLines(s), ...(s.stream ? ["Stream:", ...streamLines(s.stream).map((l) => ` ${l}`)] : [])], meanwhile); // done: the meanwhile Next is dropped
|
|
1300
1498
|
if (r.ready) {
|
|
1301
|
-
const head = !picked.length ? "Nothing to wait for: no table added yet."
|
|
1499
|
+
const head = !picked.length ? "Nothing to wait for: no table added yet."
|
|
1500
|
+
: only !== null ? `Ready: ${picked.map((t) => t.table).join(", ")} ${picked.length === 1 ? "is" : "are"} synced and served.`
|
|
1501
|
+
: "Ready: every added table is synced and served.";
|
|
1302
1502
|
if (!json)
|
|
1303
1503
|
out(withEnd(picked.length ? styleOut().green(head) : head, { next: readyNext }));
|
|
1304
1504
|
return 0;
|
|
@@ -1309,6 +1509,88 @@ export async function wait(argv) {
|
|
|
1309
1509
|
}
|
|
1310
1510
|
});
|
|
1311
1511
|
}
|
|
1512
|
+
/**
|
|
1513
|
+
* WHAT `mason wait <table>…` WAITS FOR: a generated table named alone (its chain: `waitGenerated`), or synced tables, each
|
|
1514
|
+
* found among the added ones as `mason sync` finds them (`findTable`) — a table of the database not added, or a name
|
|
1515
|
+
* nothing has, refused with the way out.
|
|
1516
|
+
*/
|
|
1517
|
+
export function waitPick(s, names) {
|
|
1518
|
+
const generated = s.generatedTables ?? [];
|
|
1519
|
+
const added = s.tables.filter((t) => t.added);
|
|
1520
|
+
const tables = [];
|
|
1521
|
+
for (const name of names) {
|
|
1522
|
+
const g = generated.find((t) => t.table === name || t.table === `public.${name}`);
|
|
1523
|
+
if (g !== undefined) {
|
|
1524
|
+
if (names.length > 1) {
|
|
1525
|
+
return { message: `${entryName(g.table)} is a generated table, which is waited for alone`, code: "usage",
|
|
1526
|
+
fix: `mason wait ${entryName(g.table)}, then mason wait ${names.filter((n) => n !== name).join(" ")}`, next: [["mason wait --help", "what a wait waits for"]] };
|
|
1527
|
+
}
|
|
1528
|
+
return { generated: name };
|
|
1529
|
+
}
|
|
1530
|
+
const t = findTable(added, name);
|
|
1531
|
+
if (!("message" in t)) {
|
|
1532
|
+
tables.push(t.table);
|
|
1533
|
+
continue;
|
|
1534
|
+
}
|
|
1535
|
+
// a table of the database that is not added is told so, with the command that adds it
|
|
1536
|
+
const db = findTable(s.tables, name);
|
|
1537
|
+
if ("message" in db && db.code === "usage")
|
|
1538
|
+
return db;
|
|
1539
|
+
if (!("message" in db)) {
|
|
1540
|
+
return { message: `${db.table} is not added, so Mason does not sync it`, code: "not_found", fix: `mason add ${tableArg(db.table)}`, next: [["mason status", "the added tables"]] };
|
|
1541
|
+
}
|
|
1542
|
+
const choices = [...added.flatMap((x) => [x.table, tableArg(x.table)]), ...generated.map((x) => entryName(x.table))];
|
|
1543
|
+
const named = [...added.map((x) => tableArg(x.table)), ...generated.map((x) => entryName(x.table))];
|
|
1544
|
+
return { message: `there is no added or generated table ${JSON.stringify(name)}${didYouMean(name, choices)}`, code: "not_found",
|
|
1545
|
+
...(named.length ? { fix: `name one of: ${named.join(", ")}` } : {}),
|
|
1546
|
+
next: [["mason status", "the added tables and the generated ones"], ["mason tables", "the database's tables"]] };
|
|
1547
|
+
}
|
|
1548
|
+
return { tables: [...new Set(tables)] };
|
|
1549
|
+
}
|
|
1550
|
+
/** `mason wait <generated table>`: until it holds every change synced when the wait began (src/cli/readiness.ts
|
|
1551
|
+
* generatedReadiness), a line when what it waits for changes; the same outcome and exit codes as the wait for copies */
|
|
1552
|
+
async function waitGenerated(run, named, timeout, json) {
|
|
1553
|
+
return guarded("wait", json, async () => {
|
|
1554
|
+
const since = Date.now(), until = since + timeout * 1000;
|
|
1555
|
+
let said = "", saidAt = 0;
|
|
1556
|
+
for (;;) {
|
|
1557
|
+
const s = await readState(run);
|
|
1558
|
+
const all = s.generatedTables ?? [];
|
|
1559
|
+
const g = all.find((t) => t.table === named) ?? all.find((t) => t.table === `public.${named}`);
|
|
1560
|
+
if (g === undefined) {
|
|
1561
|
+
const names = all.map((t) => entryName(t.table));
|
|
1562
|
+
return failed("wait", { message: `no generated table ${JSON.stringify(named)}${names.length ? "" : ": the project has none"}`, code: "not_found",
|
|
1563
|
+
...(names.length ? { fix: `name one of: ${names.join(", ")}` } : {}), next: [["mason status", "the generated tables and what each is doing"]] }, { json });
|
|
1564
|
+
}
|
|
1565
|
+
const r = generatedReadiness(g, since);
|
|
1566
|
+
const next = [["mason status", "the generated tables and what each is doing"]];
|
|
1567
|
+
const failure = r.failed.length
|
|
1568
|
+
? { message: r.failed[0], code: "failed", fix: "Mason tries a failed run again by itself; mason logs says why it failed", next: [["mason logs", "the detail of the failure"], ...next] }
|
|
1569
|
+
: !r.ready && Date.now() >= until
|
|
1570
|
+
? { message: `${g.table} does not hold what was synced ${timeout} s ago yet`, code: "timeout", why: r.waiting.join("\n"), fix: "nothing stopped: its runs go on",
|
|
1571
|
+
next: [[`mason wait ${entryName(g.table)}`, "keep waiting (--timeout <duration> for longer)"], ...next] }
|
|
1572
|
+
: null;
|
|
1573
|
+
if (json && (r.ready || failure)) {
|
|
1574
|
+
out(jsonText(WAIT, { ...r, timedOut: !r.ready && failure?.code === "timeout", tables: [], generatedTable: g, ...(failure ? { error: errorJson(failure) } : {}) }, failure ? failure.next ?? [] : next));
|
|
1575
|
+
}
|
|
1576
|
+
if (r.ready) {
|
|
1577
|
+
if (!json)
|
|
1578
|
+
out(withEnd(styleOut().green(`Ready: ${g.table} holds every change synced when the wait began.`), { next }));
|
|
1579
|
+
return 0;
|
|
1580
|
+
}
|
|
1581
|
+
if (failure)
|
|
1582
|
+
return json ? (failure.code === "timeout" ? 124 : 1) : failed("wait", failure, { json: false });
|
|
1583
|
+
const line = r.waiting.join("\n").replace(/\d+ s behind/, "# s behind");
|
|
1584
|
+
// when what it waits for changes, and every 30 s while it does not, as the copies' wait says its block
|
|
1585
|
+
if (!json && sayAgain(line, said, saidAt, Date.now())) {
|
|
1586
|
+
err(` ${r.waiting.join("\n ")}`);
|
|
1587
|
+
said = line;
|
|
1588
|
+
saidAt = Date.now();
|
|
1589
|
+
}
|
|
1590
|
+
await new Promise((res) => setTimeout(res, 1000));
|
|
1591
|
+
}
|
|
1592
|
+
});
|
|
1593
|
+
}
|
|
1312
1594
|
// ------------------------------------------------------------------ mason logs
|
|
1313
1595
|
/** What the sync's events leave out, said where they are listed (mason logs --help, the page's Sync events, held equal
|
|
1314
1596
|
* by tests/web-page.test.ts): a scheduled poll that succeeds is no event, so a person who synced reads where it is. */
|
|
@@ -1316,17 +1598,7 @@ export const EVENTS_LEAVE_OUT = "A scheduled poll that succeeds is not listed: t
|
|
|
1316
1598
|
const LOGS_USAGE = `usage: mason logs [-n <lines>] [-f] [--start] [--requests [<filters>]] [--events [--failed]] [--running] [--json]
|
|
1317
1599
|
|
|
1318
1600
|
The last lines of Mason's log: every sync, poll and statement it ran, warnings and errors with their reasons —
|
|
1319
|
-
running or not.
|
|
1320
|
-
Postgres API that Mason answered, and every one it refused, failed or cancelled, a line each, newest last — when, the
|
|
1321
|
-
outcome and SQLSTATE, how long, the rows sent, the client's application_name (and the mason user's name), the
|
|
1322
|
-
statement and the error's first line; the filters narrow it, and a page that has older rows ends with the line that
|
|
1323
|
-
reads them (--before). With --events, the sync's events: each first sync that failed or finished, each poll, catch-up or
|
|
1324
|
-
stream that failed, the database out of reach, a table current again, each sync asked by hand (mason sync, the
|
|
1325
|
-
page's Sync now) and what it did, and each restart of the run — a line each, newest last (a failure repeated word for
|
|
1326
|
-
word is said once an hour). A scheduled poll that succeeds is not listed: the last one's result is each table's last
|
|
1327
|
-
sync (mason status, the Tables page's Last sync). Both are kept MASON_HISTORY_DAYS days
|
|
1328
|
-
(default 7) in the run's synced data, and read through the running Mason. With --running, the statements running now
|
|
1329
|
-
instead, one per session, each with its number (mason cancel <session> --statement <n> stops one).
|
|
1601
|
+
running or not.
|
|
1330
1602
|
|
|
1331
1603
|
mason logs
|
|
1332
1604
|
mason logs -f keep printing new lines as Mason works, until Ctrl-C
|
|
@@ -1345,7 +1617,8 @@ const LOGS_USAGE = `usage: mason logs [-n <lines>] [-f] [--start] [--requests [<
|
|
|
1345
1617
|
--failed with --requests: only the ones not answered (refused, failed or cancelled); with --events: only
|
|
1346
1618
|
the failures
|
|
1347
1619
|
--outcome <o,…> with --requests: only these outcomes — ok, refused (the statement's own fault), failed (Mason's or
|
|
1348
|
-
the machine's), cancelled (a cancel or a statement timeout, SQLSTATE 57014)
|
|
1620
|
+
the machine's), cancelled (a cancel or a statement timeout, SQLSTATE 57014); a comma list or
|
|
1621
|
+
the flag repeated, the same
|
|
1349
1622
|
--since <time> with --requests: started at or after it — an age (30m, 2h, 7d) or an ISO 8601 date or time with
|
|
1350
1623
|
its zone (2026-10-28, 2026-10-28T09:00:00Z)
|
|
1351
1624
|
--until <time> with --requests: started before it (as --since)
|
|
@@ -1355,17 +1628,32 @@ const LOGS_USAGE = `usage: mason logs [-n <lines>] [-f] [--start] [--requests [<
|
|
|
1355
1628
|
--search <text> with --requests: only the statements whose text has it, in any case (at most ${MAX_SEARCH} characters)
|
|
1356
1629
|
--before <pos> with --requests: the page before: the position a page answered with (its last Next line)
|
|
1357
1630
|
--running the statements running now (Mason must be running)
|
|
1358
|
-
--json as data:
|
|
1359
|
-
|
|
1360
|
-
|
|
1361
|
-
|
|
1362
|
-
|
|
1363
|
-
|
|
1364
|
-
|
|
1365
|
-
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1631
|
+
--json as data: its shape is at the end
|
|
1632
|
+
|
|
1633
|
+
With --requests, the run's request log instead: every statement a tool sent the
|
|
1634
|
+
Postgres API that Mason answered, and every one it refused, failed or cancelled, a line each, newest last — when, the
|
|
1635
|
+
outcome and SQLSTATE, how long, the rows sent, the client's application_name (and the mason user's name), the
|
|
1636
|
+
statement and the error's first line; the filters narrow it, and a page that has older rows ends with the line that
|
|
1637
|
+
reads them (--before). With --events, the sync's events: each first sync that failed or finished, each poll, catch-up or
|
|
1638
|
+
stream that failed, the database out of reach, a table current again, each sync asked by hand (mason sync, the
|
|
1639
|
+
page's Sync now) and what it did, and each restart of the run — a line each, newest last (a failure repeated word for
|
|
1640
|
+
word is said once an hour). A scheduled poll that succeeds is not listed: the last one's result is each table's last
|
|
1641
|
+
sync (mason status, the Tables page's Last sync). Both are kept MASON_HISTORY_DAYS days
|
|
1642
|
+
(default 7) in the run's synced data, and read through the running Mason. With --running, the statements running now
|
|
1643
|
+
instead, one per session, each with its number (mason cancel <session> --statement <n> stops one).
|
|
1644
|
+
|
|
1645
|
+
The --json answer: { lines: [{ at, text }] } (the log's entries, oldest first; at is null for a line with no
|
|
1646
|
+
time); with --requests { requests: [{ requestId, startedAt, endedAt, session, applicationName,
|
|
1647
|
+
clientAddress (the client's address; behind a TCP proxy, the one its PROXY header carries),
|
|
1648
|
+
tokenName, user (the mason user), role, attributes (the user's values its role's row filters
|
|
1649
|
+
read, or null), method, outcome (ok, refused, failed, cancelled), status
|
|
1650
|
+
(completed, failed, cancelled: what a reader shows), sqlstate, message, detail, hint, rows,
|
|
1651
|
+
bytesSent, durationMs, query, queryTruncated }], keptDays, before (the position of the page before,
|
|
1652
|
+
or null: none older) }; with --events { events: [{ at, kind (copy_failed, poll_failed,
|
|
1653
|
+
catch_up_failed, stream_failed, source_unreachable, restart, copied, recovered, synced (a sync
|
|
1654
|
+
asked by hand), sync_changed (a table's sync: changed)), failed, table (null: the whole sync),
|
|
1655
|
+
message }], keptDays }; with --running { running: [{ session, user,
|
|
1656
|
+
applicationName, method, startedAt, runningMs, rows (sent so far), query, queryTruncated }] }; next
|
|
1369
1657
|
`;
|
|
1370
1658
|
export const LOGS = {
|
|
1371
1659
|
needs: "project",
|
|
@@ -1373,7 +1661,7 @@ export const LOGS = {
|
|
|
1373
1661
|
summary: "Mason's log (-f follows it); --requests: the Postgres API's statements; --events: the sync's; --running: running now",
|
|
1374
1662
|
usage: LOGS_USAGE,
|
|
1375
1663
|
options: { lines: { type: "string", short: "n" }, follow: { type: "boolean", short: "f" }, start: { type: "boolean" },
|
|
1376
|
-
requests: { type: "boolean" }, events: { type: "boolean" }, failed: { type: "boolean" }, outcome: { type: "string" }, since: { type: "string" },
|
|
1664
|
+
requests: { type: "boolean" }, events: { type: "boolean" }, failed: { type: "boolean" }, outcome: { type: "string", multiple: true }, since: { type: "string" },
|
|
1377
1665
|
until: { type: "string" }, user: { type: "string" }, client: { type: "string" }, slow: { type: "boolean" }, search: { type: "string" }, before: { type: "string" }, running: { type: "boolean" }, json: { type: "boolean" } },
|
|
1378
1666
|
json: ["lines", "requests", "events", "keptDays", "before", "running", "next"],
|
|
1379
1667
|
};
|
|
@@ -1513,7 +1801,7 @@ async function runningLogs(json) {
|
|
|
1513
1801
|
/** A refusal the run answered, in the Failure shape it carries (`failure`, src/cli/run-server.ts sendFailure). */
|
|
1514
1802
|
export function runFailure(reply, next) {
|
|
1515
1803
|
const f = reply.body.failure;
|
|
1516
|
-
const code =
|
|
1804
|
+
const code = failureCode(f?.code, reply.status === 400 ? "usage" : reply.status === 404 ? "not_found" : reply.status === 409 ? "refused" : "failed");
|
|
1517
1805
|
return { message: typeof f?.message === "string" ? f.message : String(reply.body.error ?? `the run answered ${reply.status}`), code,
|
|
1518
1806
|
...(typeof f?.why === "string" ? { why: f.why } : {}), ...(typeof f?.fix === "string" ? { fix: f.fix } : {}), next };
|
|
1519
1807
|
}
|
|
@@ -1521,6 +1809,9 @@ export async function logs(argv) {
|
|
|
1521
1809
|
const f = parseCommand(LOGS, argv);
|
|
1522
1810
|
if (typeof f === "number")
|
|
1523
1811
|
return f;
|
|
1812
|
+
const extra = noArguments(LOGS, f.positionals, f.values.json === true);
|
|
1813
|
+
if (extra !== null)
|
|
1814
|
+
return extra;
|
|
1524
1815
|
const json = f.values.json === true;
|
|
1525
1816
|
if (f.values.requests && f.values.events)
|
|
1526
1817
|
return failed("logs", { message: "--requests and --events are two lists: ask for one", code: "usage", fix: "mason logs --requests, or mason logs --events" }, { json });
|
|
@@ -1550,7 +1841,7 @@ export async function logs(argv) {
|
|
|
1550
1841
|
if (f.values.requests) {
|
|
1551
1842
|
// the filters, read as the run reads its query string (src/cli/request-filter.ts): a mistake is said here, exit 2
|
|
1552
1843
|
const text = (v) => typeof v === "string" ? v : undefined;
|
|
1553
|
-
const q = requestFilter({ limit: text(f.values.lines), failed: f.values.failed === true, outcome:
|
|
1844
|
+
const q = requestFilter({ limit: text(f.values.lines), failed: f.values.failed === true, outcome: listFlag(f.values.outcome), since: text(f.values.since),
|
|
1554
1845
|
until: text(f.values.until), user: text(f.values.user), client: text(f.values.client),
|
|
1555
1846
|
slow: f.values.slow === true, search: text(f.values.search), before: text(f.values.before) }, Date.now(), flagOf);
|
|
1556
1847
|
if ("message" in q)
|
|
@@ -1572,16 +1863,22 @@ export async function logs(argv) {
|
|
|
1572
1863
|
return failed("logs", t, { json });
|
|
1573
1864
|
// the run's log by default (mason logs once showed no actual logs: it showed
|
|
1574
1865
|
// mason start's terminal, which after the start carries only warnings and errors)
|
|
1575
|
-
const file =
|
|
1866
|
+
const file = logPath(t.project.dir, f.values.start ? "start" : "run");
|
|
1576
1867
|
if (!fs.existsSync(file)) {
|
|
1577
1868
|
return failed("logs", { message: `${file} does not exist yet`, code: "nothing_yet", why: `Mason has not run for ${t.project.config.name}`, next: [["mason start", "start it, in the background"]] }, { json });
|
|
1578
1869
|
}
|
|
1579
|
-
|
|
1870
|
+
// read through the one reader, every line masked as it is read (src/lib/log-read.ts, issue 621: an older Mason's lines
|
|
1871
|
+
// held the source's password in a `password=` value); a line not yet ended is -f's to show as it stands, else read whole
|
|
1872
|
+
const reader = new LogReader(file);
|
|
1873
|
+
const read = reader.readLines();
|
|
1874
|
+
const unended = f.values.follow ? "" : reader.rest();
|
|
1875
|
+
if (unended !== "")
|
|
1876
|
+
read.push(unended);
|
|
1580
1877
|
// WHY THE FILE, even with the run up: it is the whole record (the run's memory keeps only its last LOG_KEPT entries,
|
|
1581
1878
|
// src/lib/log-buffer.ts) and the only one when the run is down. The entries are parsed by the buffer's own parser, so
|
|
1582
1879
|
// `{ at, text }` is the shape of GET /api/logs too (the run's answer, which a `mason serve` run has no file for).
|
|
1583
1880
|
// -n counts entries, a multi-line one being ONE; dev.log's lines (the start's steps, indented under each other) are a line each.
|
|
1584
|
-
const all =
|
|
1881
|
+
const all = entriesOfLines(read, { perLine: f.values.start === true });
|
|
1585
1882
|
const last = all.slice(Math.max(0, all.length - lines));
|
|
1586
1883
|
if (json) {
|
|
1587
1884
|
out(jsonText(LOGS, { lines: last }, [["mason logs --requests", "every statement the Postgres API met"]]));
|
|
@@ -1591,24 +1888,13 @@ export async function logs(argv) {
|
|
|
1591
1888
|
out(logLineText(l));
|
|
1592
1889
|
if (!f.values.follow)
|
|
1593
1890
|
return 0;
|
|
1594
|
-
let offset = Buffer.byteLength(text);
|
|
1595
1891
|
for (;;) {
|
|
1892
|
+
// the lines written since, each masked once it is whole, then the line not ended yet as it stands, masked (a run
|
|
1893
|
+
// killed mid-write leaves its last words unended); a new run that started the file over is read from its start
|
|
1894
|
+
const more = reader.follow();
|
|
1895
|
+
if (more !== "")
|
|
1896
|
+
process.stdout.write(more);
|
|
1596
1897
|
await new Promise((res) => setTimeout(res, 500));
|
|
1597
|
-
const size = fs.statSync(file).size;
|
|
1598
|
-
if (size < offset)
|
|
1599
|
-
offset = 0; // a new run started the file over
|
|
1600
|
-
if (size === offset)
|
|
1601
|
-
continue;
|
|
1602
|
-
const buf = Buffer.alloc(size - offset);
|
|
1603
|
-
const fd = fs.openSync(file, "r");
|
|
1604
|
-
try {
|
|
1605
|
-
fs.readSync(fd, buf, 0, buf.length, offset);
|
|
1606
|
-
}
|
|
1607
|
-
finally {
|
|
1608
|
-
fs.closeSync(fd);
|
|
1609
|
-
}
|
|
1610
|
-
offset = size;
|
|
1611
|
-
process.stdout.write(buf.toString());
|
|
1612
1898
|
}
|
|
1613
1899
|
}
|
|
1614
1900
|
// ------------------------------------------------------------------ mason stop
|
|
@@ -1619,9 +1905,9 @@ const STOP_USAGE = `usage: mason stop [--json]
|
|
|
1619
1905
|
|
|
1620
1906
|
mason stop
|
|
1621
1907
|
|
|
1622
|
-
mason destroy removes the synced data and the state too (your files stay), to free the disk or begin again.
|
|
1623
|
-
|
|
1624
1908
|
--json as data: { stopped: [name], killed (a run that did not answer and ignored SIGTERM), supervisor (its pid, when one was stopped), next }
|
|
1909
|
+
|
|
1910
|
+
mason destroy removes the synced data and the state too (your files stay), to free the disk or begin again.
|
|
1625
1911
|
`;
|
|
1626
1912
|
export const STOP = {
|
|
1627
1913
|
needs: "project",
|
|
@@ -1635,6 +1921,9 @@ export async function stop(argv) {
|
|
|
1635
1921
|
const f = parseCommand(STOP, argv);
|
|
1636
1922
|
if (typeof f === "number")
|
|
1637
1923
|
return f;
|
|
1924
|
+
const extra = noArguments(STOP, f.positionals, f.values.json === true);
|
|
1925
|
+
if (extra !== null)
|
|
1926
|
+
return extra;
|
|
1638
1927
|
const json = f.values.json === true;
|
|
1639
1928
|
const next = [["mason start", "resume, in seconds (in the project's folder)"], ["mason destroy", "remove its synced data and its state; your files stay"]];
|
|
1640
1929
|
const r = await theRun(process.cwd());
|