@mnmal-ai/substrate 1.0.0-alpha.2 → 1.0.0-alpha.20
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +64 -14
- package/dist/boot-store-BaDqbBS-.js +4062 -0
- package/dist/{runtime-metrics-DwpDJ8vz.d.ts → bridge-BhiJKwfF.d.ts} +2 -87
- package/dist/cli-RRFphMtQ.js +3366 -0
- package/dist/client.browser.js +2485 -0
- package/dist/client.d.ts +273 -135
- package/dist/client.js +6 -6
- package/dist/cluster-worker.js +0 -1
- package/dist/{content-type-C7T28xNb.js → content-type-wLljot5M.js} +0 -1
- package/dist/{env-utils-CGNazUNt.js → env-utils-s25Z1YSA.js} +0 -1
- package/dist/{errors-D4vKuwz8.d.ts → errors-CYOaWZ8l.d.ts} +44 -4
- package/dist/{fake-DUVtKVSE.js → fake-DdXtb-m8.js} +0 -1
- package/dist/{index-D7mSOFlM.d.ts → index-ZKeYFHDD.d.ts} +7 -11
- package/dist/msgpackr-Dq18g6Ds.js +16 -0
- package/dist/node-version-BtK_Nq6z.js +32 -0
- package/dist/pack.d.ts +75 -11
- package/dist/pack.js +4 -4
- package/dist/pg-edges-store-C-IRv-O9.js +443 -0
- package/dist/{plugin-QuJN9pG_.js → plugin-MSbSZ6Al.js} +132 -8
- package/dist/proposals-bootstrap.js +6 -5
- package/dist/proposals.config.schema.json +5 -0
- package/dist/proposals.d.ts +17 -5
- package/dist/proposals.default-config.json +2 -1
- package/dist/proposals.js +2 -3
- package/dist/recall-bootstrap.js +5 -4
- package/dist/recall.config.schema.json +6 -0
- package/dist/recall.d.ts +35 -6
- package/dist/recall.js +2 -3
- package/dist/{repl-C52wyfMs.js → repl-Ob5OyQqj.js} +65 -33
- package/dist/{row-ByLFE2Wx.d.ts → row-CfsHSEfB.d.ts} +92 -12
- package/dist/run-B438GGJK.js +2435 -0
- package/dist/{plugin-t2g2wX0b.d.ts → schema-laGLMvMC.d.ts} +1287 -63
- package/dist/script-A9dxMRZA.js +143 -0
- package/dist/script-runner.js +77 -0
- package/dist/sdk.d.ts +227 -38
- package/dist/sdk.js +5 -110
- package/dist/semver-D_zdq0hi.js +17 -0
- package/dist/{src-DMzc3HfW.js → src-BNdhH_Hx.js} +745 -851
- package/dist/{src-CG7giuHr.js → src-BSL2yyVy.js} +779 -75
- package/dist/{src-BLgbLq2k.js → src-B_kHkbsr.js} +780 -330
- package/dist/{src-DWm0iA88.js → src-De9j33jv.js} +188 -87
- package/dist/src-DkxrpQOz.js +7709 -0
- package/dist/{row-DZ0Fnpc6.js → src-DmiUWOLp.js} +2 -1
- package/dist/{src-CHp3QyqB.js → src-S84ef4e3.js} +9177 -3377
- package/dist/substrate-admin.js +17 -782
- package/dist/substrate.js +4 -5487
- package/dist/testing.d.ts +730 -53
- package/dist/testing.js +20 -163
- package/dist/{ws-C72iVS5T.d.ts → ws-BT8kL1IF.d.ts} +4 -5
- package/package.json +26 -26
- package/dist/dataset-BQ8egGl5.js +0 -1535
- package/dist/schema-g4FQLr0V.d.ts +0 -433
- package/dist/src-CXKnqeTW.js +0 -2339
|
@@ -0,0 +1,2435 @@
|
|
|
1
|
+
import { _ as AUDIT_EVENT_TYPE, g as buildAuditRow } from "./src-S84ef4e3.js";
|
|
2
|
+
import { n as trimTrailingSlashes } from "./env-utils-s25Z1YSA.js";
|
|
3
|
+
import { I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./src-DmiUWOLp.js";
|
|
4
|
+
import "./src-B_kHkbsr.js";
|
|
5
|
+
import { A as UPDATE_EXIT_CODES, B as readKeyById, C as grantLines, D as parseBootstrapArgs, E as formatBootstrapOutput, F as CLI_ACTOR_KID, G as validateArgv, H as resolveAdminAuth, I as base64Url, J as argvError, K as RESTART_NOTE, L as forUpdate, M as runKeyPatchViaRoute, N as ACTOR_HINT, O as runBootstrap, P as BootstrapError, R as generateAdminKeypair, S as runRegisterViaRoute, T as formatDoctorReport, U as resolveAdminTarget, V as describeAdminTarget, W as ARGV_SPEC, X as pgNoticeOptions, Y as AdminCliError, _ as openPgDataset, b as parseRegisterArgs, c as refuseContractViolation, d as registeredTypes, g as version, h as loadPluginsFromDir, j as runKeyListViaRoute, k as REVOKE_EXIT_CODES, l as buildDeferredIndexes, n as resolvePluginDir, q as USAGE, r as BINARY_CONTRACT_VERSION, s as parseEnv, t as bootStore, v as onboardCommand, w as doctorCommand, x as runRegister, y as formatRegisterOutput, z as listActiveAdminKeys } from "./boot-store-BaDqbBS-.js";
|
|
6
|
+
import { T as substrateFetch } from "./src-BNdhH_Hx.js";
|
|
7
|
+
import { C as copyRows, S as detectLayout, a as ensureTypeTable, b as UUID_CHECK_MIGRATION, c as typeTableRef, d as PLAIN_SCHEMA_NAME, g as isDerivedMigration, i as seqName, p as assertStoreSchema, s as loadTypeTable, u as MIGRATE_STORE_MARKER_TABLE, w as parseSourceBody, x as UUID_REGEX } from "./src-DkxrpQOz.js";
|
|
8
|
+
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
9
|
+
import { dirname } from "node:path";
|
|
10
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
11
|
+
import { statfs } from "node:fs/promises";
|
|
12
|
+
import postgres from "postgres";
|
|
13
|
+
import { PassThrough, Transform } from "node:stream";
|
|
14
|
+
import { pipeline } from "node:stream/promises";
|
|
15
|
+
import { createInterface } from "node:readline";
|
|
16
|
+
|
|
17
|
+
//#region src/admin/commands/list.ts
|
|
18
|
+
/**
|
|
19
|
+
* `list` — print the active Key rows.
|
|
20
|
+
*
|
|
21
|
+
* Without `--url`: read-only, direct-to-postgres, and it DOES NOT
|
|
22
|
+
* AUTHENTICATE, so an admin row here is no evidence admin auth works — that is
|
|
23
|
+
* `doctor`'s question. With `--url`: an authenticated read through the server.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* Parse `list`'s arguments. Returns the `--url` base when one was given.
|
|
27
|
+
*/
|
|
28
|
+
function parseListArgs(argv) {
|
|
29
|
+
try {
|
|
30
|
+
return parseListArgsInner(argv);
|
|
31
|
+
} catch (err) {
|
|
32
|
+
if (err instanceof AdminCliError) throw argvError("list", err.message, err.exitCode);
|
|
33
|
+
throw err;
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
function parseListArgsInner(argv) {
|
|
37
|
+
validateArgv("list", argv);
|
|
38
|
+
const out = {};
|
|
39
|
+
for (let i = 0; i < argv.length; i++) {
|
|
40
|
+
const tok = argv[i] ?? "";
|
|
41
|
+
if (tok === "--url") {
|
|
42
|
+
const value = argv[i + 1];
|
|
43
|
+
if (value === void 0 || value.startsWith("-")) throw new AdminCliError("list: --url requires a base URL, e.g. --url https://host:3100", 1);
|
|
44
|
+
out.url = value;
|
|
45
|
+
i += 1;
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
if (tok.startsWith("--url=")) {
|
|
49
|
+
const value = tok.slice(6);
|
|
50
|
+
if (value === "") throw new AdminCliError("list: --url requires a base URL, e.g. --url https://host:3100", 1);
|
|
51
|
+
out.url = value;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return out;
|
|
55
|
+
}
|
|
56
|
+
async function runList(env, openDataset = openPgDataset) {
|
|
57
|
+
const { dataset, close } = await openDataset(env);
|
|
58
|
+
try {
|
|
59
|
+
const { rows } = await dataset.read("Key", { where: { revokedAt: { eq: null } } });
|
|
60
|
+
const out = [];
|
|
61
|
+
for (const row of rows) {
|
|
62
|
+
const kid = typeof row.kid === "string" ? row.kid : "";
|
|
63
|
+
if (kid.length === 0) continue;
|
|
64
|
+
if (row.revokedAt != null) continue;
|
|
65
|
+
const createdAt = row._metadata?.createdAt ?? "";
|
|
66
|
+
out.push({
|
|
67
|
+
kid,
|
|
68
|
+
name: typeof row.name === "string" ? row.name : "",
|
|
69
|
+
capabilities: typeof row.capabilities === "string" ? row.capabilities : "",
|
|
70
|
+
namespaces: typeof row.namespaces === "string" ? row.namespaces : "",
|
|
71
|
+
createdAt
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
return out;
|
|
75
|
+
} finally {
|
|
76
|
+
await close();
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Provenance line, printed with every result including the empty one, so an
|
|
81
|
+
* unauthenticated direct read is not mistaken for proof that admin auth works
|
|
82
|
+
* — or, when empty, for proof that the right store is empty.
|
|
83
|
+
*/
|
|
84
|
+
const PROVENANCE = "read directly from SUBSTRATE_DATABASE_URL — NOT authenticated, and not a check that any credential works.\nTo ask a running server whether admin auth actually works: substrate-admin doctor --url <base>\n";
|
|
85
|
+
function formatListOutput(rows) {
|
|
86
|
+
if (rows.length === 0) return `no active Key rows\n${PROVENANCE}`;
|
|
87
|
+
return `${formatTable(rows)}${PROVENANCE}`;
|
|
88
|
+
}
|
|
89
|
+
/** The aligned table, shared by both modes so they cannot drift apart. */
|
|
90
|
+
function formatTable(rows) {
|
|
91
|
+
const header = {
|
|
92
|
+
kid: "kid",
|
|
93
|
+
name: "name",
|
|
94
|
+
capabilities: "capabilities",
|
|
95
|
+
namespaces: "namespaces",
|
|
96
|
+
createdAt: "createdAt"
|
|
97
|
+
};
|
|
98
|
+
const all = [header, ...rows];
|
|
99
|
+
const widths = {
|
|
100
|
+
kid: Math.max(...all.map((r) => r.kid.length)),
|
|
101
|
+
name: Math.max(...all.map((r) => r.name.length)),
|
|
102
|
+
capabilities: Math.max(...all.map((r) => r.capabilities.length)),
|
|
103
|
+
namespaces: Math.max(...all.map((r) => r.namespaces.length)),
|
|
104
|
+
createdAt: Math.max(...all.map((r) => r.createdAt.length))
|
|
105
|
+
};
|
|
106
|
+
const pad = (s, w) => s + " ".repeat(Math.max(0, w - s.length));
|
|
107
|
+
const divider = [
|
|
108
|
+
"-".repeat(widths.kid),
|
|
109
|
+
"-".repeat(widths.name),
|
|
110
|
+
"-".repeat(widths.capabilities),
|
|
111
|
+
"-".repeat(widths.namespaces),
|
|
112
|
+
"-".repeat(widths.createdAt)
|
|
113
|
+
].join(" ");
|
|
114
|
+
const lines = [[
|
|
115
|
+
pad(header.kid, widths.kid),
|
|
116
|
+
pad(header.name, widths.name),
|
|
117
|
+
pad(header.capabilities, widths.capabilities),
|
|
118
|
+
pad(header.namespaces, widths.namespaces),
|
|
119
|
+
pad(header.createdAt, widths.createdAt)
|
|
120
|
+
].join(" "), divider];
|
|
121
|
+
for (const r of rows) lines.push([
|
|
122
|
+
pad(r.kid, widths.kid),
|
|
123
|
+
pad(r.name, widths.name),
|
|
124
|
+
pad(r.capabilities, widths.capabilities),
|
|
125
|
+
pad(r.namespaces, widths.namespaces),
|
|
126
|
+
pad(r.createdAt, widths.createdAt)
|
|
127
|
+
].join(" "));
|
|
128
|
+
return `${lines.join("\n")}\n`;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Render the AUTHENTICATED read, naming the target and stating that the
|
|
132
|
+
* server verified the credential before answering.
|
|
133
|
+
*/
|
|
134
|
+
function formatRemoteListOutput(rows, baseUrl) {
|
|
135
|
+
const mapped = rows.map((r) => ({
|
|
136
|
+
kid: typeof r.kid === "string" ? r.kid : "",
|
|
137
|
+
name: typeof r.name === "string" ? r.name : "",
|
|
138
|
+
capabilities: typeof r.capabilities === "string" ? r.capabilities : "",
|
|
139
|
+
namespaces: typeof r.namespaces === "string" ? r.namespaces : "",
|
|
140
|
+
createdAt: typeof r.createdAt === "string" ? r.createdAt : ""
|
|
141
|
+
}));
|
|
142
|
+
const provenance = `read from ${baseUrl} over the admin-gated route — AUTHENTICATED, and the server accepted this credential.\n`;
|
|
143
|
+
if (mapped.length === 0) return `no active Key rows\n${provenance}`;
|
|
144
|
+
return `${formatTable(mapped)}${provenance}`;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
//#endregion
|
|
148
|
+
//#region src/admin/migrate-store/progress.ts
|
|
149
|
+
/** Progress of a migrate-store copy, counted off the COPY text stream as it passes, or per converted batch. */
|
|
150
|
+
/**
|
|
151
|
+
* A pass-through that counts rows (one newline each: COPY text escapes the
|
|
152
|
+
* newlines inside values) and reports every `batchRows` rows. A throw from
|
|
153
|
+
* `onProgress` fails the stream, which aborts the copy.
|
|
154
|
+
*/
|
|
155
|
+
var RowCounter = class extends Transform {
|
|
156
|
+
table;
|
|
157
|
+
batchRows;
|
|
158
|
+
onProgress;
|
|
159
|
+
rows = 0;
|
|
160
|
+
bytes = 0;
|
|
161
|
+
#next;
|
|
162
|
+
constructor(table, batchRows, onProgress, from = {
|
|
163
|
+
rows: 0,
|
|
164
|
+
bytes: 0
|
|
165
|
+
}) {
|
|
166
|
+
super();
|
|
167
|
+
this.table = table;
|
|
168
|
+
this.batchRows = batchRows;
|
|
169
|
+
this.onProgress = onProgress;
|
|
170
|
+
if (!Number.isInteger(batchRows) || batchRows < 1) throw new Error(`batchRows must be a positive integer (got ${batchRows})`);
|
|
171
|
+
this.rows = from.rows;
|
|
172
|
+
this.bytes = from.bytes;
|
|
173
|
+
this.#next = from.rows - from.rows % batchRows + batchRows;
|
|
174
|
+
}
|
|
175
|
+
/** Counts `rows` converted rows of `bytes` body text, reporting as `_transform` does. */
|
|
176
|
+
add(rows, bytes) {
|
|
177
|
+
this.rows += rows;
|
|
178
|
+
this.bytes += bytes;
|
|
179
|
+
this.#report();
|
|
180
|
+
}
|
|
181
|
+
#report() {
|
|
182
|
+
if (this.rows < this.#next) return;
|
|
183
|
+
this.#next = this.rows - this.rows % this.batchRows + this.batchRows;
|
|
184
|
+
this.onProgress({
|
|
185
|
+
table: this.table,
|
|
186
|
+
rows: this.rows,
|
|
187
|
+
bytes: this.bytes,
|
|
188
|
+
done: false
|
|
189
|
+
});
|
|
190
|
+
}
|
|
191
|
+
_transform(chunk, _enc, cb) {
|
|
192
|
+
try {
|
|
193
|
+
this.bytes += chunk.length;
|
|
194
|
+
for (let i = chunk.indexOf(10); i !== -1; i = chunk.indexOf(10, i + 1)) this.rows++;
|
|
195
|
+
this.#report();
|
|
196
|
+
this.push(chunk);
|
|
197
|
+
cb(null);
|
|
198
|
+
} catch (err) {
|
|
199
|
+
cb(err);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
};
|
|
203
|
+
|
|
204
|
+
//#endregion
|
|
205
|
+
//#region src/admin/migrate-store/source.ts
|
|
206
|
+
/**
|
|
207
|
+
* Read-only inventory of a migrate-store source. Every read runs in one
|
|
208
|
+
* REPEATABLE READ, READ ONLY transaction, so the inventory is one snapshot and
|
|
209
|
+
* nothing here can write to the source. It never runs migrations and never
|
|
210
|
+
* goes through `assertServerVersion`: the source may be PG 17.
|
|
211
|
+
*/
|
|
212
|
+
async function inspectSource(sql) {
|
|
213
|
+
return await sql.begin("isolation level repeatable read read only", async (tx) => {
|
|
214
|
+
await assertStoreSchema(tx);
|
|
215
|
+
const layout = await detectLayout(tx);
|
|
216
|
+
if (layout === "absent") throw new Error("the source schema holds no rows table");
|
|
217
|
+
if (layout === "tables") throw new Error("the source is already in the tables layout; migrate-store copies a legacy or partitioned store");
|
|
218
|
+
const [version] = await tx`SELECT current_setting('server_version_num')::int AS v`;
|
|
219
|
+
const [ledger] = await tx`SELECT to_regclass('migrations') IS NOT NULL AS present`;
|
|
220
|
+
const appliedMigrations = ledger?.present ? (await tx`SELECT version FROM migrations ORDER BY version`).map((r) => r.version) : [];
|
|
221
|
+
const tableRows = await tx`
|
|
222
|
+
SELECT c.relname AS name,
|
|
223
|
+
-- pg_partition_tree is empty for an ordinary table.
|
|
224
|
+
(CASE WHEN c.relkind = 'p'
|
|
225
|
+
THEN (SELECT coalesce(sum(pg_table_size(t.relid)), 0) FROM pg_partition_tree(c.oid) t WHERE t.isleaf)
|
|
226
|
+
ELSE pg_table_size(c.oid) END)::text AS bytes
|
|
227
|
+
FROM pg_class c
|
|
228
|
+
WHERE c.relnamespace = current_schema()::regnamespace
|
|
229
|
+
AND c.relkind IN ('r', 'p') AND NOT c.relispartition
|
|
230
|
+
ORDER BY c.relname`;
|
|
231
|
+
const colRows = await tx`
|
|
232
|
+
SELECT c.relname AS "table", a.attname AS col
|
|
233
|
+
FROM pg_attribute a JOIN pg_class c ON c.oid = a.attrelid
|
|
234
|
+
WHERE c.relnamespace = current_schema()::regnamespace
|
|
235
|
+
AND c.relkind IN ('r', 'p') AND NOT c.relispartition
|
|
236
|
+
AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = ''
|
|
237
|
+
ORDER BY c.relname, a.attnum`;
|
|
238
|
+
const columns = {};
|
|
239
|
+
for (const r of colRows) {
|
|
240
|
+
const list = columns[r.table] ?? [];
|
|
241
|
+
list.push(r.col);
|
|
242
|
+
columns[r.table] = list;
|
|
243
|
+
}
|
|
244
|
+
const types = await tx`
|
|
245
|
+
SELECT namespace, type,
|
|
246
|
+
count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
|
|
247
|
+
count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
|
|
248
|
+
coalesce(sum(pg_column_size(r.*)), 0)::text AS bytes
|
|
249
|
+
FROM rows r GROUP BY namespace, type ORDER BY namespace, type`;
|
|
250
|
+
const nonUuid = (table, column, nsCol, typeCol) => tx`
|
|
251
|
+
SELECT ${tx(nsCol)} AS namespace, ${tx(typeCol)} AS type, ${tx(column)} AS id
|
|
252
|
+
FROM ${tx(table)} WHERE ${tx(column)} !~ ${UUID_REGEX}
|
|
253
|
+
ORDER BY 1, 2, 3 LIMIT 20`.then((rs) => rs.map((r) => ({
|
|
254
|
+
table,
|
|
255
|
+
column,
|
|
256
|
+
...r
|
|
257
|
+
})));
|
|
258
|
+
const nonUuidIds = appliedMigrations.includes("0002_uuid_format_check") ? [] : [
|
|
259
|
+
...await nonUuid("rows", "id", "namespace", "type"),
|
|
260
|
+
...columns.short_ids ? await nonUuid("short_ids", "row_id", "namespace", "type") : [],
|
|
261
|
+
...columns.edges ? await nonUuid("edges", "from_id", "from_ns", "from_type") : [],
|
|
262
|
+
...columns.edges ? await nonUuid("edges", "to_id", "to_ns", "to_type") : []
|
|
263
|
+
];
|
|
264
|
+
const [dims] = columns.embeddings ? await tx`
|
|
265
|
+
SELECT a.atttypmod AS dims FROM pg_attribute a
|
|
266
|
+
WHERE a.attrelid = to_regclass('embeddings') AND a.attname = 'embedding' AND NOT a.attisdropped` : [];
|
|
267
|
+
const extensions = await tx`SELECT extname, extversion FROM pg_extension ORDER BY extname`;
|
|
268
|
+
const sessions = await tx`
|
|
269
|
+
SELECT pid, extract(epoch FROM backend_start)::text AS start, usename,
|
|
270
|
+
application_name AS "applicationName", client_addr::text AS "clientAddr",
|
|
271
|
+
backend_type AS "backendType", state
|
|
272
|
+
FROM pg_stat_activity
|
|
273
|
+
WHERE datname = current_database() AND pid <> pg_backend_pid()
|
|
274
|
+
-- Another role's session shows a NULL backend_type to a non-monitoring role.
|
|
275
|
+
AND (backend_type = 'client backend' OR (backend_type IS NULL AND usesysid IS NOT NULL))
|
|
276
|
+
ORDER BY pid`;
|
|
277
|
+
return {
|
|
278
|
+
serverVersionNum: version?.v ?? 0,
|
|
279
|
+
layout,
|
|
280
|
+
appliedMigrations,
|
|
281
|
+
hasSeq: columns.rows?.includes("seq") ?? false,
|
|
282
|
+
hasVisibleSeqs: columns.visible_seqs !== void 0,
|
|
283
|
+
types: types.map((t) => ({
|
|
284
|
+
...t,
|
|
285
|
+
live: Number(t.live),
|
|
286
|
+
deleted: Number(t.deleted),
|
|
287
|
+
bytes: Number(t.bytes)
|
|
288
|
+
})),
|
|
289
|
+
tables: tableRows.map((t) => t.name),
|
|
290
|
+
columns,
|
|
291
|
+
tableBytes: Object.fromEntries(tableRows.map((t) => [t.name, Number(t.bytes)])),
|
|
292
|
+
nonUuidIds,
|
|
293
|
+
vectorDims: dims !== void 0 && dims.dims > 0 ? dims.dims : null,
|
|
294
|
+
extensions: Object.fromEntries(extensions.map((e) => [e.extname, e.extversion])),
|
|
295
|
+
otherSessions: sessions.length,
|
|
296
|
+
sessions
|
|
297
|
+
};
|
|
298
|
+
});
|
|
299
|
+
}
|
|
300
|
+
/** The `rows` columns the engine itself defined in the legacy and partitioned layouts. */
|
|
301
|
+
const ENGINE_ROWS_COLUMNS = /* @__PURE__ */ new Set([
|
|
302
|
+
"namespace",
|
|
303
|
+
"type",
|
|
304
|
+
"id",
|
|
305
|
+
"body",
|
|
306
|
+
"deleted_at",
|
|
307
|
+
"created_at",
|
|
308
|
+
"updated_at",
|
|
309
|
+
"created_by_kind",
|
|
310
|
+
"created_by_id",
|
|
311
|
+
"updated_by_kind",
|
|
312
|
+
"updated_by_id",
|
|
313
|
+
"seq"
|
|
314
|
+
]);
|
|
315
|
+
/** The fields a target type table declares, orphaned columns excluded. */
|
|
316
|
+
function declaredFields(table) {
|
|
317
|
+
if (table === void 0) return [];
|
|
318
|
+
return [...table.columns().filter((c) => c.state !== "orphaned").map((c) => c.field), ...table.pending];
|
|
319
|
+
}
|
|
320
|
+
/**
|
|
321
|
+
* SQL for a source row's body as the old engine read it. A plugin migration could add a `rows`
|
|
322
|
+
* column mirroring a body field (cortext's `archived boolean NOT NULL DEFAULT false`), and old
|
|
323
|
+
* filters read that column, so a body lacking the key still matched `archived eq false`. A
|
|
324
|
+
* non-null mirror column named after a declared field fills that field when the body lacks it;
|
|
325
|
+
* where the body has the key, the body wins.
|
|
326
|
+
* shortcut: matches a mirror column to its field by name only (every known plugin uses the field
|
|
327
|
+
* name); a plugin that renamed its column would need its promotedColumns map read here.
|
|
328
|
+
*/
|
|
329
|
+
function sourceBodySql(rowsColumns, declared) {
|
|
330
|
+
const mirrored = rowsColumns.filter((c) => !ENGINE_ROWS_COLUMNS.has(c) && declared.includes(c));
|
|
331
|
+
if (mirrored.length === 0) return "body";
|
|
332
|
+
return `(jsonb_strip_nulls(jsonb_build_object(${mirrored.map((c) => `'${c.replace(/'/g, "''")}', "${c.replace(/"/g, "\"\"")}"`).join(", ")})) || body)`;
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
//#endregion
|
|
336
|
+
//#region src/admin/migrate-store/copy.ts
|
|
337
|
+
/**
|
|
338
|
+
* The migrate-store copy: converts a legacy or partitioned source into a
|
|
339
|
+
* target `bootStore` prepared in the `tables` layout, all from one REPEATABLE
|
|
340
|
+
* READ, READ ONLY source snapshot. `rows` is converted: each source type is
|
|
341
|
+
* streamed with a cursor, in `batchRows` batches, into its type table by
|
|
342
|
+
* `copyRows` (columns plus `extra`, ids into `row_ids`). Every other table
|
|
343
|
+
* moves with `COPY … TO STDOUT` piped into `COPY … FROM STDIN`. Nothing here
|
|
344
|
+
* writes to the source.
|
|
345
|
+
*
|
|
346
|
+
* Two phases around a re-boot, on one source snapshot. Phase 1 copies the
|
|
347
|
+
* rows (and their shortIds) that declare storage: `_system`,
|
|
348
|
+
* `_system_proposals` and every TypeProposal and FieldProposal. The second
|
|
349
|
+
* boot replays those proposals as serve does, so every proposed type is
|
|
350
|
+
* registered, with its table and declared fields, before phase 2 copies
|
|
351
|
+
* everything else. A source type nothing declares gets a table with no
|
|
352
|
+
* columns, so its fields all go to `extra`, and is reported `defaulted`.
|
|
353
|
+
* Last, one promotion sweep moves the values of fields declared only by the
|
|
354
|
+
* replay (phase 1 converted them into `extra`) into their columns.
|
|
355
|
+
*
|
|
356
|
+
* Progress lives in the marker table on the target. A rerun after an
|
|
357
|
+
* interruption restarts cleanly: it empties the planned tables, keeps the
|
|
358
|
+
* seed rows the first run recorded, and copies again.
|
|
359
|
+
*/
|
|
360
|
+
/** The misfits listed per type, beside the full count. */
|
|
361
|
+
const MISFITS_LISTED = 100;
|
|
362
|
+
/** A source table or column with nowhere to go on the target. Thrown before any data moves. */
|
|
363
|
+
var MigrateStoreColumnError = class extends Error {
|
|
364
|
+
table;
|
|
365
|
+
column;
|
|
366
|
+
name = "MigrateStoreColumnError";
|
|
367
|
+
constructor(table, column) {
|
|
368
|
+
super(column === null ? `the target has no table "${table}" to copy into` : `the target's "${table}" has no column "${column}", which the source has`);
|
|
369
|
+
this.table = table;
|
|
370
|
+
this.column = column;
|
|
371
|
+
}
|
|
372
|
+
};
|
|
373
|
+
const STARTED = "copy:started";
|
|
374
|
+
const SEED = "seed:";
|
|
375
|
+
/** The `rows` / `short_ids` predicate phase 1 copies; phase 2 copies its complement. Both columns are NOT NULL. */
|
|
376
|
+
const PHASE1_ROWS = "namespace IN ('_system', '_system_proposals') OR type IN ('TypeProposal', 'FieldProposal')";
|
|
377
|
+
const PHASED = /* @__PURE__ */ new Set(["rows", "short_ids"]);
|
|
378
|
+
/** Converted rather than copied: the target has no `rows`. */
|
|
379
|
+
const CONVERTED = "rows";
|
|
380
|
+
async function copyStore(source, boot, plan, opts) {
|
|
381
|
+
let target = await boot();
|
|
382
|
+
let open = target;
|
|
383
|
+
try {
|
|
384
|
+
let tsql = sqlOf(target);
|
|
385
|
+
await tsql`CREATE TABLE IF NOT EXISTS ${tsql(MIGRATE_STORE_MARKER_TABLE)} (step text PRIMARY KEY, done_at timestamptz)`;
|
|
386
|
+
const identity = await checkColumns(tsql, plan);
|
|
387
|
+
if (await restartIfStarted(tsql, plan)) {
|
|
388
|
+
open = void 0;
|
|
389
|
+
await target.close();
|
|
390
|
+
target = await boot();
|
|
391
|
+
open = target;
|
|
392
|
+
tsql = sqlOf(target);
|
|
393
|
+
}
|
|
394
|
+
const src = await source.reserve();
|
|
395
|
+
let streaming = false;
|
|
396
|
+
let snapshotOpen = true;
|
|
397
|
+
try {
|
|
398
|
+
await src.unsafe("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY");
|
|
399
|
+
await src.unsafe("SET LOCAL DateStyle = 'ISO, YMD'");
|
|
400
|
+
await src.unsafe("SET LOCAL TimeZone = 'UTC'");
|
|
401
|
+
const keptSeeds = await reconcileSeeds(src, tsql, target);
|
|
402
|
+
const rowsPlan = plan.tables.find((t) => t.name === CONVERTED);
|
|
403
|
+
const misfits = {};
|
|
404
|
+
const totals = /* @__PURE__ */ new Map();
|
|
405
|
+
const copy = async (t, filter, tables) => {
|
|
406
|
+
const start = performance.now();
|
|
407
|
+
const prev = totals.get(t.name) ?? {
|
|
408
|
+
rows: 0,
|
|
409
|
+
bytes: 0,
|
|
410
|
+
ms: 0
|
|
411
|
+
};
|
|
412
|
+
const counter = new RowCounter(t.name, opts.batchRows, opts.onProgress, prev);
|
|
413
|
+
streaming = true;
|
|
414
|
+
if (t.name === CONVERTED) await convertRows(src, tsql, t, filter, tables, counter, opts.batchRows, misfits);
|
|
415
|
+
else await copyTable(src, tsql, t, identity.get(t.name) ?? [], counter, filter);
|
|
416
|
+
streaming = false;
|
|
417
|
+
totals.set(t.name, {
|
|
418
|
+
rows: counter.rows,
|
|
419
|
+
bytes: counter.bytes,
|
|
420
|
+
ms: prev.ms + Math.round(performance.now() - start)
|
|
421
|
+
});
|
|
422
|
+
};
|
|
423
|
+
const phase1 = await prepareTables(src, target, PHASE1_ROWS, false);
|
|
424
|
+
for (const t of plan.tables.filter((p) => PHASED.has(p.name))) await copy(t, PHASE1_ROWS, phase1.tables);
|
|
425
|
+
await markStep(tsql, "copy:phase1");
|
|
426
|
+
await snapshotRows(tsql);
|
|
427
|
+
open = void 0;
|
|
428
|
+
await target.close();
|
|
429
|
+
target = await boot();
|
|
430
|
+
open = target;
|
|
431
|
+
tsql = sqlOf(target);
|
|
432
|
+
await assertRebootWroteNothing(tsql);
|
|
433
|
+
const phase2 = await prepareTables(src, target, null, true);
|
|
434
|
+
const tables = [];
|
|
435
|
+
for (const t of plan.tables) {
|
|
436
|
+
await copy(t, PHASED.has(t.name) ? `NOT (${PHASE1_ROWS})` : null, phase2.tables);
|
|
437
|
+
await markStep(tsql, `copy:${t.name}`);
|
|
438
|
+
const done = totals.get(t.name);
|
|
439
|
+
tables.push({
|
|
440
|
+
name: t.name,
|
|
441
|
+
...done
|
|
442
|
+
});
|
|
443
|
+
opts.onProgress({
|
|
444
|
+
table: t.name,
|
|
445
|
+
rows: done.rows,
|
|
446
|
+
bytes: done.bytes,
|
|
447
|
+
done: true
|
|
448
|
+
});
|
|
449
|
+
}
|
|
450
|
+
await markStep(tsql, "copy:phase2");
|
|
451
|
+
await advanceSequences(tsql, identity);
|
|
452
|
+
await src.unsafe("COMMIT");
|
|
453
|
+
snapshotOpen = false;
|
|
454
|
+
src.release();
|
|
455
|
+
if (rowsPlan !== void 0) await settleFields(target, opts.batchRows);
|
|
456
|
+
await markStep(tsql, "copy:done");
|
|
457
|
+
return {
|
|
458
|
+
tables,
|
|
459
|
+
keptSeeds,
|
|
460
|
+
defaulted: phase2.defaulted,
|
|
461
|
+
misfits,
|
|
462
|
+
target
|
|
463
|
+
};
|
|
464
|
+
} catch (err) {
|
|
465
|
+
if (!snapshotOpen) throw err;
|
|
466
|
+
const released = src.unsafe("ROLLBACK").catch(() => void 0).then(() => src.release()).catch(() => void 0);
|
|
467
|
+
if (!streaming) await released;
|
|
468
|
+
throw err;
|
|
469
|
+
}
|
|
470
|
+
} catch (err) {
|
|
471
|
+
await open?.close().catch(() => void 0);
|
|
472
|
+
throw err;
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
function sqlOf(target) {
|
|
476
|
+
if (target.sql === null) throw new Error("migrate-store copies into a postgres target only");
|
|
477
|
+
return target.sql;
|
|
478
|
+
}
|
|
479
|
+
const quoteIdent = (name) => `"${name.replace(/"/g, "\"\"")}"`;
|
|
480
|
+
/** The target's type tables, each with its (quoted) relation. */
|
|
481
|
+
async function typeTablesOf(tsql) {
|
|
482
|
+
return (await tsql`
|
|
483
|
+
SELECT namespace, type, table_name FROM substrate_type_tables ORDER BY namespace, type`).map((r) => ({
|
|
484
|
+
namespace: r.namespace,
|
|
485
|
+
type: r.type,
|
|
486
|
+
rel: quoteIdent(r.table_name)
|
|
487
|
+
}));
|
|
488
|
+
}
|
|
489
|
+
/**
|
|
490
|
+
* SQL over every target row: `(namespace, type, id, h)`, `h` an md5 of the
|
|
491
|
+
* stored row with its null columns dropped, so a column added by a re-boot
|
|
492
|
+
* changes no row's hash. `null` when there are no type tables.
|
|
493
|
+
*/
|
|
494
|
+
async function targetRowsSql(tsql) {
|
|
495
|
+
const tables = await typeTablesOf(tsql);
|
|
496
|
+
if (tables.length === 0) return null;
|
|
497
|
+
const lit = (s) => `'${s.replace(/'/g, "''")}'`;
|
|
498
|
+
return tables.map((t) => `SELECT ${lit(t.namespace)}::text AS namespace, ${lit(t.type)}::text AS type, x.id, md5(jsonb_strip_nulls(to_jsonb(x))::text) AS h FROM ${t.rel} x`).join(" UNION ALL ");
|
|
499
|
+
}
|
|
500
|
+
const SNAPSHOT = "substrate_migrate_store_reboot_rows";
|
|
501
|
+
/** Records every target row's key and content hash, for `assertRebootWroteNothing`. */
|
|
502
|
+
async function snapshotRows(tsql) {
|
|
503
|
+
await tsql`DROP TABLE IF EXISTS ${tsql(SNAPSHOT)}`;
|
|
504
|
+
const all = await targetRowsSql(tsql);
|
|
505
|
+
await tsql.unsafe(`CREATE UNLOGGED TABLE ${quoteIdent(SNAPSHOT)} AS ${all ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false"}`);
|
|
506
|
+
}
|
|
507
|
+
/** Refuses a re-boot that added, removed or changed a row, naming up to ten of each. */
|
|
508
|
+
async function assertRebootWroteNothing(tsql) {
|
|
509
|
+
const all = await targetRowsSql(tsql) ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false";
|
|
510
|
+
const diff = await tsql.unsafe(`
|
|
511
|
+
SELECT CASE WHEN s.id IS NULL THEN 'added' WHEN r.id IS NULL THEN 'removed' ELSE 'changed' END AS change,
|
|
512
|
+
coalesce(r.namespace, s.namespace) AS namespace, coalesce(r.type, s.type) AS type,
|
|
513
|
+
coalesce(r.id, s.id) AS id
|
|
514
|
+
FROM (${all}) r
|
|
515
|
+
FULL JOIN ${quoteIdent(SNAPSHOT)} s USING (namespace, type, id)
|
|
516
|
+
WHERE s.id IS NULL OR r.id IS NULL OR r.h <> s.h
|
|
517
|
+
ORDER BY 1, 2, 3, 4 LIMIT 30`);
|
|
518
|
+
await tsql`DROP TABLE ${tsql(SNAPSHOT)}`;
|
|
519
|
+
if (diff.length === 0) return;
|
|
520
|
+
const named = [
|
|
521
|
+
"added",
|
|
522
|
+
"changed",
|
|
523
|
+
"removed"
|
|
524
|
+
].map((c) => {
|
|
525
|
+
const rows = diff.filter((d) => d.change === c).slice(0, 10);
|
|
526
|
+
return rows.length === 0 ? "" : `${c}: ${rows.map((d) => `${d.namespace}/${d.type}/${d.id}`).join(", ")}`;
|
|
527
|
+
}).filter((s) => s !== "").join("; ");
|
|
528
|
+
throw new Error(`the target's re-boot wrote rows the source does not have (${named}). The boot writes them because the source lacks them, usually the _system/AuditEvent of kind 'bootstrap' that serve records on a store's first boot with keys. Boot the source once with substrate serve so it records them itself, stop it, then rerun migrate-store, which restarts cleanly.`);
|
|
529
|
+
}
|
|
530
|
+
/** Refuses a plan column the target lacks; returns each table's identity-always columns among the planned ones. */
|
|
531
|
+
async function checkColumns(tsql, plan) {
|
|
532
|
+
const copied = plan.tables.filter((t) => t.name !== CONVERTED);
|
|
533
|
+
const rows = await tsql`
|
|
534
|
+
SELECT c.relname AS "table", a.attname AS col, a.attidentity AS ident
|
|
535
|
+
FROM pg_attribute a
|
|
536
|
+
JOIN pg_class c ON c.oid = a.attrelid
|
|
537
|
+
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
538
|
+
WHERE n.nspname = current_schema() AND c.relname = ANY(${copied.map((t) => t.name)}::text[])
|
|
539
|
+
AND c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = ''`;
|
|
540
|
+
const identity = /* @__PURE__ */ new Map();
|
|
541
|
+
for (const t of copied) {
|
|
542
|
+
const cols = rows.filter((r) => r.table === t.name);
|
|
543
|
+
if (cols.length === 0) throw new MigrateStoreColumnError(t.name, null);
|
|
544
|
+
const have = new Set(cols.map((c) => c.col));
|
|
545
|
+
const missing = t.columns.find((c) => !have.has(c));
|
|
546
|
+
if (missing !== void 0) throw new MigrateStoreColumnError(t.name, missing);
|
|
547
|
+
const always = cols.filter((c) => c.ident === "a" && t.columns.includes(c.col));
|
|
548
|
+
if (always.length > 0) identity.set(t.name, always.map((c) => c.col));
|
|
549
|
+
}
|
|
550
|
+
return identity;
|
|
551
|
+
}
|
|
552
|
+
const keyOf = (r) => JSON.stringify([
|
|
553
|
+
r.namespace,
|
|
554
|
+
r.type,
|
|
555
|
+
r.id
|
|
556
|
+
]);
|
|
557
|
+
/** `(namespace, type, id) IN (…)` over `keys`, as three bound arrays. */
|
|
558
|
+
function inKeys(sql, keys) {
|
|
559
|
+
return sql`(namespace, type, id) IN (SELECT * FROM unnest(
|
|
560
|
+
${keys.map((k) => k.namespace)}::text[], ${keys.map((k) => k.type)}::text[], ${keys.map((k) => k.id)}::text[]))`;
|
|
561
|
+
}
|
|
562
|
+
/**
|
|
563
|
+
* Deletes target rows: those in `keys`, or (`except`) all but those. From each
|
|
564
|
+
* type table and from `row_ids`, whose delete cascades to `short_ids`.
|
|
565
|
+
*/
|
|
566
|
+
async function deleteTargetRows(t, keys, except) {
|
|
567
|
+
for (const table of await typeTablesOf(t)) {
|
|
568
|
+
const ids = keys.filter((k) => k.namespace === table.namespace && k.type === table.type).map((k) => k.id);
|
|
569
|
+
if (!except && ids.length === 0) continue;
|
|
570
|
+
await t.unsafe(`DELETE FROM ${table.rel} WHERE ${except ? "NOT " : ""}(id = ANY($1::text[]))`, [ids]);
|
|
571
|
+
}
|
|
572
|
+
await t`DELETE FROM row_ids WHERE ${except ? t`NOT` : t``} ${inKeys(t, keys)}`;
|
|
573
|
+
}
|
|
574
|
+
/** After an interrupted run: empties every planned table but the seed rows that run recorded. True when it did. */
|
|
575
|
+
async function restartIfStarted(tsql, plan) {
|
|
576
|
+
const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
|
|
577
|
+
const [started] = await tsql`SELECT 1 FROM ${marker} WHERE step = ${STARTED}`;
|
|
578
|
+
if (started === void 0) return false;
|
|
579
|
+
const seeds = (await tsql`SELECT step FROM ${marker} WHERE starts_with(step, ${SEED})`).map((r) => {
|
|
580
|
+
const [namespace, type, id] = JSON.parse(r.step.slice(5));
|
|
581
|
+
return {
|
|
582
|
+
namespace,
|
|
583
|
+
type,
|
|
584
|
+
id
|
|
585
|
+
};
|
|
586
|
+
});
|
|
587
|
+
const others = plan.tables.map((p) => p.name).filter((n) => !PHASED.has(n));
|
|
588
|
+
await tsql.begin(async (tx) => {
|
|
589
|
+
const t = tx;
|
|
590
|
+
await deleteTargetRows(t, seeds, true);
|
|
591
|
+
if (others.length > 0) await t`TRUNCATE ${t(others)}`;
|
|
592
|
+
});
|
|
593
|
+
return true;
|
|
594
|
+
}
|
|
595
|
+
/**
|
|
596
|
+
* Deletes each target row the source also holds, so the source row replaces
|
|
597
|
+
* it, and records the rest as seeds. Every row still on the target must be of
|
|
598
|
+
* a type the target registers, as everything the prepare step writes is.
|
|
599
|
+
*/
|
|
600
|
+
async function reconcileSeeds(src, tsql, target) {
|
|
601
|
+
const present = await tsql`SELECT namespace, type, id FROM row_ids ORDER BY namespace, type, id`;
|
|
602
|
+
const inSource = present.length === 0 ? [] : await src`SELECT namespace, type, id FROM rows WHERE ${inKeys(src, [...present])}`;
|
|
603
|
+
const replaced = new Set(inSource.map(keyOf));
|
|
604
|
+
const kept = present.filter((r) => !replaced.has(keyOf(r))).map(({ namespace, type, id }) => ({
|
|
605
|
+
namespace,
|
|
606
|
+
type,
|
|
607
|
+
id
|
|
608
|
+
}));
|
|
609
|
+
for (const r of kept) if (registeredDataset(target, r.namespace, r.type) === void 0) throw new Error(`target row ${r.namespace}/${r.type}/${r.id} is not one the prepare step wrote; refusing to copy alongside it`);
|
|
610
|
+
const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
|
|
611
|
+
await tsql.begin(async (tx) => {
|
|
612
|
+
const t = tx;
|
|
613
|
+
if (inSource.length > 0) await deleteTargetRows(t, inSource.map(({ namespace, type, id }) => ({
|
|
614
|
+
namespace,
|
|
615
|
+
type,
|
|
616
|
+
id
|
|
617
|
+
})), false);
|
|
618
|
+
await t`DELETE FROM ${marker} WHERE starts_with(step, ${SEED}) OR starts_with(step, 'copy:')`;
|
|
619
|
+
const steps = [STARTED, ...kept.map((k) => SEED + keyOf(k))];
|
|
620
|
+
await t`INSERT INTO ${marker} ${t(steps.map((step) => ({
|
|
621
|
+
step,
|
|
622
|
+
done_at: /* @__PURE__ */ new Date()
|
|
623
|
+
})))}`;
|
|
624
|
+
});
|
|
625
|
+
return kept;
|
|
626
|
+
}
|
|
627
|
+
function registeredDataset(target, ns, type) {
|
|
628
|
+
if (!target.datasets.has(ns)) return void 0;
|
|
629
|
+
if (target.engine.registryFor(ns)?.get(type)?.kind !== "type") return void 0;
|
|
630
|
+
return target.datasets.get(ns);
|
|
631
|
+
}
|
|
632
|
+
/**
|
|
633
|
+
* Gives every source type `filter` picks its table and returns their
|
|
634
|
+
* descriptors: a registered type's through its dataset, with its declared
|
|
635
|
+
* fields and unique indexes; any other's with no field columns (its fields
|
|
636
|
+
* all go to `extra`), listed in `defaulted` when `report` is set.
|
|
637
|
+
*/
|
|
638
|
+
async function prepareTables(src, target, filter, report) {
|
|
639
|
+
const tsql = sqlOf(target);
|
|
640
|
+
const pairs = await src`SELECT DISTINCT namespace, type FROM rows ${filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`} ORDER BY namespace, type`;
|
|
641
|
+
const tables = /* @__PURE__ */ new Map();
|
|
642
|
+
const defaulted = [];
|
|
643
|
+
const [uuid] = await tsql`
|
|
644
|
+
SELECT EXISTS (SELECT 1 FROM migrations WHERE version = ${UUID_CHECK_MIGRATION}) AS on`;
|
|
645
|
+
for (const { namespace, type } of pairs) {
|
|
646
|
+
const key = `${namespace}/${type}`;
|
|
647
|
+
const dataset = registeredDataset(target, namespace, type);
|
|
648
|
+
if (dataset === void 0 && report) defaulted.push(key);
|
|
649
|
+
try {
|
|
650
|
+
if (dataset !== void 0) await dataset.ensureTypeStorage?.(type, { indexes: "unique" });
|
|
651
|
+
else await ensureTypeTable(tsql, {
|
|
652
|
+
namespace,
|
|
653
|
+
type,
|
|
654
|
+
fields: {},
|
|
655
|
+
newFields: "column",
|
|
656
|
+
enforceUuidCheck: uuid?.on === true,
|
|
657
|
+
createOnly: true
|
|
658
|
+
});
|
|
659
|
+
} catch (err) {
|
|
660
|
+
throw new Error(`preparing storage for ${key} failed: ${err.message}`, { cause: err });
|
|
661
|
+
}
|
|
662
|
+
const table = await loadTypeTable(tsql, namespace, type);
|
|
663
|
+
if (table === void 0) throw new Error(`preparing storage for ${key} made no table`);
|
|
664
|
+
tables.set(key, table);
|
|
665
|
+
}
|
|
666
|
+
return {
|
|
667
|
+
tables,
|
|
668
|
+
defaulted
|
|
669
|
+
};
|
|
670
|
+
}
|
|
671
|
+
/**
|
|
672
|
+
* Converts the source rows `filter` picks, type by type, each streamed with a
|
|
673
|
+
* cursor in `batchRows` batches into its type table. A row the table already
|
|
674
|
+
* holds, or a refusal (a declared unique index), fails the copy loudly.
|
|
675
|
+
*/
|
|
676
|
+
async function convertRows(src, tsql, t, filter, tables, counter, batchRows, misfits) {
|
|
677
|
+
const where = filter === null ? src`` : src`AND (${src.unsafe(filter)})`;
|
|
678
|
+
const seq = t.columns.includes("seq") ? src`seq::text` : src`NULL::text`;
|
|
679
|
+
const pairs = await src`
|
|
680
|
+
SELECT DISTINCT namespace, type FROM rows WHERE true ${where} ORDER BY namespace, type`;
|
|
681
|
+
for (const { namespace, type } of pairs) {
|
|
682
|
+
const key = `${namespace}/${type}`;
|
|
683
|
+
const table = tables.get(key);
|
|
684
|
+
if (table === void 0) throw new Error(`internal: no table prepared for ${key}`);
|
|
685
|
+
const cursor = src`
|
|
686
|
+
SELECT id, ${src.unsafe(sourceBodySql(t.columns, declaredFields(table)))}::text AS body, deleted_at::text, created_at::text, updated_at::text,
|
|
687
|
+
created_by_kind, created_by_id, updated_by_kind, updated_by_id,
|
|
688
|
+
${seq} AS seq
|
|
689
|
+
FROM rows WHERE namespace = ${namespace} AND type = ${type} ${where}
|
|
690
|
+
ORDER BY id`.cursor(batchRows);
|
|
691
|
+
for await (const batch of cursor) {
|
|
692
|
+
let result;
|
|
693
|
+
try {
|
|
694
|
+
result = await copyRows(tsql, table, batch.map((r) => ({
|
|
695
|
+
...r,
|
|
696
|
+
body: parseSourceBody(r.body)
|
|
697
|
+
})));
|
|
698
|
+
} catch (err) {
|
|
699
|
+
throw rowsError(key, err);
|
|
700
|
+
}
|
|
701
|
+
if (result.inserted !== batch.length) throw new Error(`copying rows failed: ${key}: ${batch.length - result.inserted} source row(s) were already on the target`);
|
|
702
|
+
if (result.misfits.length > 0) {
|
|
703
|
+
const listed = misfits[key] ?? {
|
|
704
|
+
count: 0,
|
|
705
|
+
first: []
|
|
706
|
+
};
|
|
707
|
+
listed.count += result.misfits.length;
|
|
708
|
+
listed.first.push(...result.misfits.slice(0, 100 - listed.first.length));
|
|
709
|
+
misfits[key] = listed;
|
|
710
|
+
}
|
|
711
|
+
counter.add(batch.length, batch.reduce((n, r) => n + Buffer.byteLength(r.body), 0));
|
|
712
|
+
}
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
/** A conversion failure as one line naming the type; a unique refusal names the declared index and the key. */
|
|
716
|
+
function rowsError(key, err) {
|
|
717
|
+
const pg = err;
|
|
718
|
+
if (pg?.code === "23505" && typeof pg.constraint_name === "string") return new Error(`copying rows failed: ${key}: unique index ${pg.constraint_name} refuses a source row whose key another row already holds${typeof pg.detail === "string" ? ` (${pg.detail})` : ""}. Resolve the duplicate in the source, then rerun migrate-store, which restarts cleanly.`, { cause: err });
|
|
719
|
+
const detail = typeof pg?.detail === "string" ? ` (${pg.detail})` : "";
|
|
720
|
+
return new Error(`copying rows failed: ${key}: ${err.message ?? String(err)}${detail}`, { cause: err });
|
|
721
|
+
}
|
|
722
|
+
/**
|
|
723
|
+
* One promotion sweep over the target: a field declared only once the re-boot
|
|
724
|
+
* replayed its proposal had its phase-1 values converted into `extra`, so it
|
|
725
|
+
* is `promoting` (or `pending`) until moved into its column. Fails loudly,
|
|
726
|
+
* naming each field, when one is not promoted.
|
|
727
|
+
*/
|
|
728
|
+
async function settleFields(target, batchRows) {
|
|
729
|
+
const stuck = (await target.dataset.promotePendingFields?.({
|
|
730
|
+
lockTimeoutMs: 6e4,
|
|
731
|
+
batchRows,
|
|
732
|
+
noWriters: true
|
|
733
|
+
}) ?? []).filter((e) => e.kind !== "promoted");
|
|
734
|
+
if (stuck.length > 0) throw new Error(`settling declared fields failed: ${stuck.map((e) => `${e.namespace}/${e.type}.${e.field} ${e.kind}${e.error ? ` (${e.error})` : ""}`).join("; ")}. Rerun migrate-store, which restarts cleanly.`);
|
|
735
|
+
}
|
|
736
|
+
async function copyTable(src, tsql, t, identityCols, counter, filter) {
|
|
737
|
+
const where = filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`;
|
|
738
|
+
const out = await src`COPY (SELECT ${src(t.columns)} FROM ${src(t.name)} ${where}) TO STDOUT`.readable();
|
|
739
|
+
const feed = new PassThrough();
|
|
740
|
+
out.on("error", (err) => feed.destroy(err));
|
|
741
|
+
out.pipe(feed);
|
|
742
|
+
try {
|
|
743
|
+
if (identityCols.length === 0) {
|
|
744
|
+
await copyIn(tsql`COPY ${tsql(t.name)} (${tsql(t.columns)}) FROM STDIN`, feed, counter);
|
|
745
|
+
return;
|
|
746
|
+
}
|
|
747
|
+
await tsql.begin(async (tx) => {
|
|
748
|
+
const x = tx;
|
|
749
|
+
const stage = x(`migrate_stage_${t.name}`);
|
|
750
|
+
await x`CREATE TEMP TABLE ${stage} (LIKE ${x(t.name)} INCLUDING DEFAULTS) ON COMMIT DROP`;
|
|
751
|
+
await copyIn(x`COPY ${stage} (${x(t.columns)}) FROM STDIN`, feed, counter);
|
|
752
|
+
await x`INSERT INTO ${x(t.name)} (${x(t.columns)}) OVERRIDING SYSTEM VALUE SELECT ${x(t.columns)} FROM ${stage}`;
|
|
753
|
+
});
|
|
754
|
+
} catch (err) {
|
|
755
|
+
out.unpipe(feed);
|
|
756
|
+
feed.destroy();
|
|
757
|
+
out.resume();
|
|
758
|
+
const detail = err.detail;
|
|
759
|
+
throw new Error(`copying ${t.name} failed: ${err.message}${detail ? ` (${detail})` : ""}`, { cause: err });
|
|
760
|
+
}
|
|
761
|
+
}
|
|
762
|
+
/**
|
|
763
|
+
* Pipes `feed` through `counter` into a `COPY … FROM STDIN`. postgres.js 3.4.9
|
|
764
|
+
* never finishes the writable when the server rejects the COPY, so the
|
|
765
|
+
* query's own reject is raced as the failure signal.
|
|
766
|
+
*/
|
|
767
|
+
async function copyIn(query, feed, counter) {
|
|
768
|
+
const q = query;
|
|
769
|
+
const reject = q.reject;
|
|
770
|
+
const failed = new Promise((_, fail) => {
|
|
771
|
+
q.reject = (err) => {
|
|
772
|
+
fail(err);
|
|
773
|
+
reject(err);
|
|
774
|
+
};
|
|
775
|
+
});
|
|
776
|
+
failed.catch(() => void 0);
|
|
777
|
+
const into = await query.writable();
|
|
778
|
+
await Promise.race([pipeline(feed, counter, into), failed]);
|
|
779
|
+
}
|
|
780
|
+
/**
|
|
781
|
+
* Moves every shortId sequence to the highest copied shortId of the
|
|
782
|
+
* (namespace, type) pairs sharing its name, creating it if missing, and every
|
|
783
|
+
* copied identity column's sequence to its column's maximum.
|
|
784
|
+
*/
|
|
785
|
+
async function advanceSequences(tsql, identity) {
|
|
786
|
+
const maxes = await tsql`SELECT namespace, type, max(short_id)::text AS max FROM short_ids GROUP BY namespace, type`;
|
|
787
|
+
const byName = /* @__PURE__ */ new Map();
|
|
788
|
+
for (const m of maxes) {
|
|
789
|
+
const name = seqName(m.namespace, m.type);
|
|
790
|
+
const max = BigInt(m.max);
|
|
791
|
+
if ((byName.get(name) ?? 0n) < max) byName.set(name, max);
|
|
792
|
+
}
|
|
793
|
+
for (const [name, max] of byName) {
|
|
794
|
+
await tsql`CREATE SEQUENCE IF NOT EXISTS ${tsql(name)}`;
|
|
795
|
+
await tsql`SELECT setval(${`"${name}"`}::regclass, ${max.toString()}::bigint)`;
|
|
796
|
+
}
|
|
797
|
+
for (const [table, cols] of identity) for (const col of cols) await tsql`
|
|
798
|
+
SELECT setval(pg_get_serial_sequence(quote_ident(${table}), ${col}), m)
|
|
799
|
+
FROM (SELECT max(${tsql(col)}) AS m FROM ${tsql(table)}) s WHERE m IS NOT NULL`;
|
|
800
|
+
}
|
|
801
|
+
async function markStep(tsql, step) {
|
|
802
|
+
await tsql`
|
|
803
|
+
INSERT INTO ${tsql(MIGRATE_STORE_MARKER_TABLE)} (step, done_at) VALUES (${step}, now())
|
|
804
|
+
ON CONFLICT (step) DO UPDATE SET done_at = excluded.done_at`;
|
|
805
|
+
}
|
|
806
|
+
|
|
807
|
+
//#endregion
|
|
808
|
+
//#region src/admin/migrate-store/preflight.ts
|
|
809
|
+
/** Pure preflight: turns a source inventory plus target facts into refusals, warnings and a copy plan. */
|
|
810
|
+
/** Each old layout's baseline ledger version, as the last release that booted it recorded it. */
|
|
811
|
+
const OLD_BASELINES = {
|
|
812
|
+
legacy: "0001_baseline",
|
|
813
|
+
partitioned: "0001p_partitioned_baseline"
|
|
814
|
+
};
|
|
815
|
+
/** Every static ledger version an old-layout store can hold. */
|
|
816
|
+
const OLD_MIGRATIONS = /* @__PURE__ */ new Set([
|
|
817
|
+
...Object.values(OLD_BASELINES),
|
|
818
|
+
UUID_CHECK_MIGRATION,
|
|
819
|
+
"0003_safe_casts",
|
|
820
|
+
"0004_visible_seq"
|
|
821
|
+
]);
|
|
822
|
+
/** `rows` is converted into type tables, which writes `row_ids` too; the rest are copied as they are. */
|
|
823
|
+
const ENGINE_COPIED = [
|
|
824
|
+
"rows",
|
|
825
|
+
"short_ids",
|
|
826
|
+
"edges",
|
|
827
|
+
"visible_seqs"
|
|
828
|
+
];
|
|
829
|
+
const NOT_COPIED = [
|
|
830
|
+
"idempotency",
|
|
831
|
+
"migrations",
|
|
832
|
+
"migrations_audit",
|
|
833
|
+
"kernel_migrations",
|
|
834
|
+
MIGRATE_STORE_MARKER_TABLE
|
|
835
|
+
];
|
|
836
|
+
/** The extension a copied table depends on. shortcut: recall's only, upgrade when another plugin table needs one. */
|
|
837
|
+
const TABLE_EXTENSION = { embeddings: "vector" };
|
|
838
|
+
const PG17 = 17e4;
|
|
839
|
+
const PG18 = 18e4;
|
|
840
|
+
function preflight(source, target, opts) {
|
|
841
|
+
const refusals = [];
|
|
842
|
+
const warnings = [];
|
|
843
|
+
if (source.serverVersionNum < PG17) refusals.push(`source is PostgreSQL ${source.serverVersionNum} (server_version_num); 170000 or later is required`);
|
|
844
|
+
if (target.serverVersionNum < PG18) refusals.push(`target is PostgreSQL ${target.serverVersionNum} (server_version_num); 180000 or later is required`);
|
|
845
|
+
const baselines = Object.values(OLD_BASELINES);
|
|
846
|
+
const unknownVersions = source.appliedMigrations.filter((v) => !OLD_MIGRATIONS.has(v) && !isDerivedMigration(v));
|
|
847
|
+
if (unknownVersions.length > 0) refusals.push(`source ledger holds migrations this binary does not know: ${unknownVersions.join(", ")}`);
|
|
848
|
+
if (!source.appliedMigrations.some((v) => baselines.includes(v))) refusals.push(`source ledger lacks ${baselines.join(" / ")}; found: ${source.appliedMigrations.join(", ") || "no migrations"}`);
|
|
849
|
+
if (target.layout === "legacy" || target.layout === "partitioned") refusals.push(`target is a ${target.layout} store; migrate-store copies into the tables layout, so point it at an empty database or schema`);
|
|
850
|
+
if (!target.empty && !target.createdByUs) refusals.push("target already holds data and was not created by migrate-store");
|
|
851
|
+
const otherBaseline = baselines.find((b) => b !== OLD_BASELINES[source.layout]);
|
|
852
|
+
if (otherBaseline !== void 0 && source.appliedMigrations.includes(otherBaseline)) refusals.push(`source ledger records ${otherBaseline} but rows is ${source.layout}`);
|
|
853
|
+
if (source.appliedMigrations.includes("0002_uuid_format_check") && !target.enforceUuidCheck) refusals.push(`source enforces the UUID check (${UUID_CHECK_MIGRATION}) but the target would not: run migrate-store with the service's environment, which sets SUBSTRATE_ENFORCE_UUID_CHECK=true`);
|
|
854
|
+
if (target.enforceUuidCheck && source.nonUuidIds.length > 0) {
|
|
855
|
+
const sample = source.nonUuidIds.slice(0, 5).map((r) => `${r.table}.${r.column}=${r.id}`).join(", ");
|
|
856
|
+
refusals.push(`source has ${source.nonUuidIds.length} non-UUID id(s) while the target enforces the UUID check (first 5 shown): ${sample}`);
|
|
857
|
+
}
|
|
858
|
+
const skip = new Set(opts.skipTables);
|
|
859
|
+
const knownTables = /* @__PURE__ */ new Set([
|
|
860
|
+
...ENGINE_COPIED,
|
|
861
|
+
...NOT_COPIED,
|
|
862
|
+
...opts.pluginTables
|
|
863
|
+
]);
|
|
864
|
+
const unknownTables = source.tables.filter((t) => !knownTables.has(t) && !skip.has(t));
|
|
865
|
+
for (const t of unknownTables) refusals.push(`source table "${t}" is not known to migrate-store; load the plugin that owns it, or pass --skip-table ${t}`);
|
|
866
|
+
const copied = /* @__PURE__ */ new Set([...ENGINE_COPIED, ...opts.pluginTables]);
|
|
867
|
+
for (const t of skip) if (copied.has(t)) refusals.push(`--skip-table ${t} names a table migrate-store copies; it cannot be skipped. --skip-table is for tables migrate-store does not know`);
|
|
868
|
+
else if (!source.tables.includes(t)) warnings.push(`--skip-table ${t} matches no source table`);
|
|
869
|
+
if (source.vectorDims !== null && target.recallDims !== null && source.vectorDims !== target.recallDims) refusals.push(`recall embedding dimensions differ: source ${source.vectorDims}, target ${target.recallDims}`);
|
|
870
|
+
const present = new Set(source.tables);
|
|
871
|
+
const plan = {
|
|
872
|
+
tables: [...ENGINE_COPIED, ...opts.pluginTables].filter((t, i, all) => present.has(t) && all.indexOf(t) === i).map((name) => ({
|
|
873
|
+
name,
|
|
874
|
+
columns: source.columns[name] ?? [],
|
|
875
|
+
bytes: source.tableBytes[name] ?? 0
|
|
876
|
+
})),
|
|
877
|
+
estimatedBytes: 0,
|
|
878
|
+
skipped: [...skip].filter((t) => present.has(t) && !copied.has(t))
|
|
879
|
+
};
|
|
880
|
+
plan.estimatedBytes = 2 * plan.tables.reduce((n, t) => n + t.bytes, 0);
|
|
881
|
+
for (const t of plan.tables) {
|
|
882
|
+
const ext = TABLE_EXTENSION[t.name];
|
|
883
|
+
if (ext === void 0) continue;
|
|
884
|
+
const [from, to] = [source.extensions[ext], target.extensions[ext]];
|
|
885
|
+
if (to === void 0) refusals.push(`table "${t.name}" needs extension "${ext}", which is not installed in a schema on the target's search_path; run CREATE EXTENSION ${ext}; in the target database as a superuser, in public or the target schema`);
|
|
886
|
+
else if (from !== void 0 && from !== to) warnings.push(`extension "${ext}" is version ${from} on the source and ${to} on the target, so verification of ${t.name} may fail; match the versions`);
|
|
887
|
+
}
|
|
888
|
+
if (target.freeBytes !== null && target.freeBytes < plan.estimatedBytes) {
|
|
889
|
+
const msg = `free disk ${target.freeBytes} bytes is below the estimated need of ${plan.estimatedBytes} bytes`;
|
|
890
|
+
(opts.dryRun ? warnings : refusals).push(msg);
|
|
891
|
+
}
|
|
892
|
+
if (source.otherSessions > 0) {
|
|
893
|
+
const named = source.sessions.slice(0, 10).map((s) => `usename=${s.usename ?? "hidden"} application_name=${s.applicationName ?? "hidden"} client_addr=${s.clientAddr ?? "local"} backend_type=${s.backendType ?? "hidden"} state=${s.state ?? "hidden"}`);
|
|
894
|
+
const more = source.sessions.length - named.length;
|
|
895
|
+
const list = named.length > 0 ? `: ${named.join("; ")}${more > 0 ? `; +${more} more` : ""}` : "";
|
|
896
|
+
(opts.dryRun ? warnings : refusals).push(`source has ${source.otherSessions} other active session(s) on its database; writes after the snapshot are not copied, so stop the service first${list}`);
|
|
897
|
+
}
|
|
898
|
+
return {
|
|
899
|
+
refusals,
|
|
900
|
+
warnings,
|
|
901
|
+
plan
|
|
902
|
+
};
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
//#endregion
|
|
906
|
+
//#region src/admin/migrate-store/verify.ts
|
|
907
|
+
/**
|
|
908
|
+
* The migrate-store verification: compares the source with its conversion
|
|
909
|
+
* per (namespace, type) and, only when everything agrees and the target holds
|
|
910
|
+
* every declared index, drops the in-progress marker so `serve` will boot the
|
|
911
|
+
* target. Both sides are read in their own REPEATABLE READ, READ ONLY
|
|
912
|
+
* transaction; nothing here writes to the source.
|
|
913
|
+
*
|
|
914
|
+
* Target rows the prepare step seeded and the source lacks (`keptSeeds`) are
|
|
915
|
+
* subtracted from the target side before every comparison.
|
|
916
|
+
*
|
|
917
|
+
* Every set is fingerprinted as `count(*)` plus the sum of a 64-bit prefix of
|
|
918
|
+
* each row's md5. That is independent of row order, of either cluster's
|
|
919
|
+
* collation, and of any join separator, and it builds no string. Row content
|
|
920
|
+
* is compared as the dataset reads it: the target's row as `decodeRow` builds
|
|
921
|
+
* it, the source's body under the same read rule (an explicit null on a
|
|
922
|
+
* declared column field reads absent), both as jsonb text, so a number's
|
|
923
|
+
* stored text counts, plus every metadata column.
|
|
924
|
+
*/
|
|
925
|
+
const quote = (cols) => cols.map((c) => `"${c.replace(/"/g, "\"\"")}"`).join(", ");
|
|
926
|
+
/** SQL for a set's fingerprint: its size and an order-independent sum of per-row hashes of `expr`. */
|
|
927
|
+
const fp = (expr) => `count(*)::text || ' ' || coalesce(sum(('x' || substr(md5(${expr}), 1, 16))::bit(64)::bigint::numeric), 0)::text`;
|
|
928
|
+
/**
|
|
929
|
+
* The declared indexes of every registered type that are not valid on the
|
|
930
|
+
* target: missing, invalid, building, failed, or built over a field that is
|
|
931
|
+
* not yet in its column. `uniqueOnly` leaves out the non-unique ones, which
|
|
932
|
+
* `--skip-index-build` defers to serve.
|
|
933
|
+
*/
|
|
934
|
+
async function missingIndexes(target, uniqueOnly) {
|
|
935
|
+
const out = [];
|
|
936
|
+
for (const { ns, type, dataset } of registeredTypes(target.engine, target.datasets)) {
|
|
937
|
+
const report = await dataset?.describeStorage?.(type);
|
|
938
|
+
for (const i of report?.indexes ?? []) if (i.state !== "valid" && (i.unique || !uniqueOnly)) out.push({
|
|
939
|
+
type: `${ns}/${type}`,
|
|
940
|
+
index: i.name
|
|
941
|
+
});
|
|
942
|
+
}
|
|
943
|
+
return out;
|
|
944
|
+
}
|
|
945
|
+
async function verifyStore(source, target, opts) {
|
|
946
|
+
const readOnly = "isolation level repeatable read read only";
|
|
947
|
+
const types = await target.begin(readOnly, (tx) => typeTables(tx));
|
|
948
|
+
const src = await source.begin(readOnly, (tx) => snapshot(tx, "source", [], null, opts, types));
|
|
949
|
+
const tgt = await target.begin(readOnly, (tx) => snapshot(tx, "target", opts.keptSeeds, src, opts, types));
|
|
950
|
+
const mismatches = [];
|
|
951
|
+
const keys = [.../* @__PURE__ */ new Set([...src.entries.keys(), ...tgt.entries.keys()])].sort();
|
|
952
|
+
for (const k of keys) {
|
|
953
|
+
const a = src.entries.get(k);
|
|
954
|
+
const b = tgt.entries.get(k);
|
|
955
|
+
if (a?.value === b?.value) continue;
|
|
956
|
+
const e = a ?? b;
|
|
957
|
+
mismatches.push({
|
|
958
|
+
check: e.check,
|
|
959
|
+
...e.namespace !== void 0 ? { namespace: e.namespace } : {},
|
|
960
|
+
...e.type !== void 0 ? { type: e.type } : {},
|
|
961
|
+
detail: `${e.what}: source ${a?.value ?? "absent"}, target ${b?.value ?? "absent"}`
|
|
962
|
+
});
|
|
963
|
+
}
|
|
964
|
+
for (const [key, table] of types) {
|
|
965
|
+
const [namespace, type] = [table.namespace, table.type];
|
|
966
|
+
const unsettled = [...table.pending.map((field) => ({
|
|
967
|
+
field,
|
|
968
|
+
state: "pending"
|
|
969
|
+
})), ...table.columns().filter((c) => c.state === "promoting")];
|
|
970
|
+
for (const f of unsettled) mismatches.push({
|
|
971
|
+
check: "fields",
|
|
972
|
+
namespace,
|
|
973
|
+
type,
|
|
974
|
+
detail: `field ${f.field} of ${key} is ${f.state}, so its values are not all in its column`
|
|
975
|
+
});
|
|
976
|
+
}
|
|
977
|
+
for (const w of (await opts.storageReport({ partitionBudget: opts.partitionBudget })).warnings) mismatches.push({
|
|
978
|
+
check: "storage",
|
|
979
|
+
detail: w
|
|
980
|
+
});
|
|
981
|
+
const missing = await opts.missingIndexes();
|
|
982
|
+
const ok = mismatches.length === 0 && missing.length === 0;
|
|
983
|
+
if (ok) await target`DROP TABLE IF EXISTS ${target(MIGRATE_STORE_MARKER_TABLE)}`;
|
|
984
|
+
return {
|
|
985
|
+
ok,
|
|
986
|
+
mismatches,
|
|
987
|
+
missingIndexes: missing
|
|
988
|
+
};
|
|
989
|
+
}
|
|
990
|
+
const COPY_MILESTONES = /* @__PURE__ */ new Set([
|
|
991
|
+
"copy:started",
|
|
992
|
+
"copy:phase1",
|
|
993
|
+
"copy:phase2",
|
|
994
|
+
"copy:done"
|
|
995
|
+
]);
|
|
996
|
+
/**
|
|
997
|
+
* What a finished copy left behind, for a verification that has no
|
|
998
|
+
* `CopyResult`: the kept seeds from the marker table, the types the target
|
|
999
|
+
* does not register (their fields all sit in `extra`), and the copied tables,
|
|
1000
|
+
* which are the marker's `copy:<table>` steps.
|
|
1001
|
+
*/
|
|
1002
|
+
async function readCopyState(target) {
|
|
1003
|
+
const sql = target.sql;
|
|
1004
|
+
if (sql === null) throw new Error("migrate-store verifies a postgres target only");
|
|
1005
|
+
const steps = await sql`
|
|
1006
|
+
SELECT step FROM ${sql(MIGRATE_STORE_MARKER_TABLE)}`.catch((err) => {
|
|
1007
|
+
if (err.code === "42P01") return [];
|
|
1008
|
+
throw err;
|
|
1009
|
+
});
|
|
1010
|
+
if (!steps.some((s) => s.step === "copy:done")) throw new Error("the target has no completed migrate-store copy to verify (its in-progress marker is missing or the copy did not finish)");
|
|
1011
|
+
return {
|
|
1012
|
+
keptSeeds: steps.filter((s) => s.step.startsWith(SEED)).map((s) => {
|
|
1013
|
+
const [namespace, type, id] = JSON.parse(s.step.slice(SEED.length));
|
|
1014
|
+
return {
|
|
1015
|
+
namespace,
|
|
1016
|
+
type,
|
|
1017
|
+
id
|
|
1018
|
+
};
|
|
1019
|
+
}).sort((a, b) => (a.namespace + a.type + a.id).localeCompare(b.namespace + b.type + b.id)),
|
|
1020
|
+
defaulted: (await sql`
|
|
1021
|
+
SELECT DISTINCT namespace, type FROM row_ids ORDER BY namespace, type`).filter((p) => registeredDataset(target, p.namespace, p.type) === void 0).map((p) => `${p.namespace}/${p.type}`),
|
|
1022
|
+
tables: steps.map((s) => s.step).filter((s) => s.startsWith("copy:") && !COPY_MILESTONES.has(s)).map((s) => s.slice(5)).sort()
|
|
1023
|
+
};
|
|
1024
|
+
}
|
|
1025
|
+
/** The target's type tables, by `<ns>/<Type>`. */
|
|
1026
|
+
async function typeTables(tx) {
|
|
1027
|
+
const types = /* @__PURE__ */ new Map();
|
|
1028
|
+
const listed = await tx`
|
|
1029
|
+
SELECT namespace, type FROM substrate_type_tables ORDER BY namespace, type`;
|
|
1030
|
+
for (const { namespace, type } of listed) {
|
|
1031
|
+
const table = await loadTypeTable(tx, namespace, type);
|
|
1032
|
+
if (table !== void 0) types.set(`${namespace}/${type}`, table);
|
|
1033
|
+
}
|
|
1034
|
+
return types;
|
|
1035
|
+
}
|
|
1036
|
+
/** Each row's id with its versions: a row updated or deleted after the copy's snapshot changes it, wherever the sample falls. */
|
|
1037
|
+
const VERSIONS = `id || E'\n' || coalesce(updated_at::text, '') || E'\n' || coalesce(deleted_at::text, '')`;
|
|
1038
|
+
const META = `deleted_at::text, created_at::text, updated_at::text, created_by_kind, created_by_id, updated_by_kind, updated_by_id`;
|
|
1039
|
+
/**
|
|
1040
|
+
* Every comparable fact about one side, keyed so the two sides diff by key.
|
|
1041
|
+
* `other` is the source's snapshot when this is the target; `types` are the
|
|
1042
|
+
* target's type tables, which give the source its read rule.
|
|
1043
|
+
*/
|
|
1044
|
+
async function snapshot(tx, side, seeds, other, opts, types) {
|
|
1045
|
+
await tx`SET LOCAL TimeZone = 'UTC'`;
|
|
1046
|
+
await tx`SET LOCAL DateStyle = 'ISO, YMD'`;
|
|
1047
|
+
await tx`SET LOCAL extra_float_digits = 3`;
|
|
1048
|
+
const entries = /* @__PURE__ */ new Map();
|
|
1049
|
+
const columns = /* @__PURE__ */ new Map();
|
|
1050
|
+
const add = (e) => entries.set(`${e.check}\0${e.namespace ?? ""}\0${e.type ?? ""}\0${e.what}`, e);
|
|
1051
|
+
const notSeed = (ns, ty, id) => tx`(${tx(ns)}, ${tx(ty)}, ${tx(id)}) NOT IN (SELECT * FROM unnest(
|
|
1052
|
+
${seeds.map((k) => k.namespace)}::text[], ${seeds.map((k) => k.type)}::text[], ${seeds.map((k) => k.id)}::text[]))`;
|
|
1053
|
+
const seedIds = (ns, ty) => seeds.filter((k) => k.namespace === ns && k.type === ty).map((k) => k.id);
|
|
1054
|
+
const colsOf = async (table) => {
|
|
1055
|
+
const have = await tx`
|
|
1056
|
+
SELECT a.attname AS col FROM pg_attribute a
|
|
1057
|
+
WHERE a.attrelid = to_regclass(${table}) AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = ''
|
|
1058
|
+
ORDER BY a.attname`;
|
|
1059
|
+
if (have.length === 0) return null;
|
|
1060
|
+
const theirs = other?.columns.get(table);
|
|
1061
|
+
const cols = have.map((c) => c.col).filter((c) => theirs === void 0 || theirs.includes(c));
|
|
1062
|
+
columns.set(table, cols);
|
|
1063
|
+
return cols;
|
|
1064
|
+
};
|
|
1065
|
+
const limit = opts.fullChecksum ? Number.MAX_SAFE_INTEGER : opts.sampleRows;
|
|
1066
|
+
const counted = [];
|
|
1067
|
+
if (side === "source") {
|
|
1068
|
+
const counts = await tx`
|
|
1069
|
+
SELECT namespace, type,
|
|
1070
|
+
count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
|
|
1071
|
+
count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
|
|
1072
|
+
${tx.unsafe(fp("id"))} AS ids,
|
|
1073
|
+
${tx.unsafe(fp(VERSIONS))} AS versions
|
|
1074
|
+
FROM rows GROUP BY namespace, type`;
|
|
1075
|
+
for (const r of counts) {
|
|
1076
|
+
const at = {
|
|
1077
|
+
namespace: r.namespace,
|
|
1078
|
+
type: r.type
|
|
1079
|
+
};
|
|
1080
|
+
add({
|
|
1081
|
+
check: "count",
|
|
1082
|
+
...at,
|
|
1083
|
+
what: "live rows",
|
|
1084
|
+
value: r.live
|
|
1085
|
+
});
|
|
1086
|
+
add({
|
|
1087
|
+
check: "count",
|
|
1088
|
+
...at,
|
|
1089
|
+
what: "deleted rows",
|
|
1090
|
+
value: r.deleted
|
|
1091
|
+
});
|
|
1092
|
+
add({
|
|
1093
|
+
check: "ids",
|
|
1094
|
+
...at,
|
|
1095
|
+
what: "id set",
|
|
1096
|
+
value: r.ids
|
|
1097
|
+
});
|
|
1098
|
+
add({
|
|
1099
|
+
check: "ids",
|
|
1100
|
+
...at,
|
|
1101
|
+
what: "registered ids (row_ids)",
|
|
1102
|
+
value: r.ids
|
|
1103
|
+
});
|
|
1104
|
+
add({
|
|
1105
|
+
check: "versions",
|
|
1106
|
+
...at,
|
|
1107
|
+
what: "row versions (updated_at, deleted_at)",
|
|
1108
|
+
value: r.versions
|
|
1109
|
+
});
|
|
1110
|
+
counted.push(at);
|
|
1111
|
+
}
|
|
1112
|
+
} else {
|
|
1113
|
+
for (const table of types.values()) {
|
|
1114
|
+
const { namespace, type } = table;
|
|
1115
|
+
const skip = seedIds(namespace, type);
|
|
1116
|
+
const [r] = await tx.unsafe(`SELECT count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
|
|
1117
|
+
count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
|
|
1118
|
+
${fp("id")} AS ids, ${fp(VERSIONS)} AS versions
|
|
1119
|
+
FROM ${table.from} WHERE NOT (id = ANY($1::text[]))`, [skip]);
|
|
1120
|
+
const at = {
|
|
1121
|
+
namespace,
|
|
1122
|
+
type
|
|
1123
|
+
};
|
|
1124
|
+
if (r !== void 0 && Number(r.live) + Number(r.deleted) > 0) {
|
|
1125
|
+
add({
|
|
1126
|
+
check: "count",
|
|
1127
|
+
...at,
|
|
1128
|
+
what: "live rows",
|
|
1129
|
+
value: r.live
|
|
1130
|
+
});
|
|
1131
|
+
add({
|
|
1132
|
+
check: "count",
|
|
1133
|
+
...at,
|
|
1134
|
+
what: "deleted rows",
|
|
1135
|
+
value: r.deleted
|
|
1136
|
+
});
|
|
1137
|
+
add({
|
|
1138
|
+
check: "ids",
|
|
1139
|
+
...at,
|
|
1140
|
+
what: "id set",
|
|
1141
|
+
value: r.ids
|
|
1142
|
+
});
|
|
1143
|
+
add({
|
|
1144
|
+
check: "versions",
|
|
1145
|
+
...at,
|
|
1146
|
+
what: "row versions (updated_at, deleted_at)",
|
|
1147
|
+
value: r.versions
|
|
1148
|
+
});
|
|
1149
|
+
counted.push(at);
|
|
1150
|
+
}
|
|
1151
|
+
}
|
|
1152
|
+
const registered = await tx`
|
|
1153
|
+
SELECT namespace, type, ${tx.unsafe(fp("id"))} AS ids
|
|
1154
|
+
FROM row_ids WHERE ${notSeed("namespace", "type", "id")} GROUP BY namespace, type`;
|
|
1155
|
+
for (const r of registered) add({
|
|
1156
|
+
check: "ids",
|
|
1157
|
+
namespace: r.namespace,
|
|
1158
|
+
type: r.type,
|
|
1159
|
+
what: "registered ids (row_ids)",
|
|
1160
|
+
value: r.ids
|
|
1161
|
+
});
|
|
1162
|
+
}
|
|
1163
|
+
const shorts = await tx`
|
|
1164
|
+
SELECT namespace, type, ${tx.unsafe(fp("row(short_id, row_id)::text"))} AS f
|
|
1165
|
+
FROM short_ids WHERE ${notSeed("namespace", "type", "row_id")} GROUP BY namespace, type`;
|
|
1166
|
+
for (const r of shorts) add({
|
|
1167
|
+
check: "short_ids",
|
|
1168
|
+
namespace: r.namespace,
|
|
1169
|
+
type: r.type,
|
|
1170
|
+
what: "shortIds",
|
|
1171
|
+
value: r.f
|
|
1172
|
+
});
|
|
1173
|
+
const edgeCols = await colsOf("edges");
|
|
1174
|
+
if (edgeCols !== null) {
|
|
1175
|
+
const edges = await tx`
|
|
1176
|
+
SELECT from_ns AS namespace, from_type AS type, field_name AS field,
|
|
1177
|
+
${tx.unsafe(fp(`row(${quote(edgeCols)})::text`))} AS f
|
|
1178
|
+
FROM edges WHERE ${notSeed("from_ns", "from_type", "from_id")} GROUP BY from_ns, from_type, field_name`;
|
|
1179
|
+
for (const r of edges) add({
|
|
1180
|
+
check: "edges",
|
|
1181
|
+
namespace: r.namespace,
|
|
1182
|
+
type: r.type,
|
|
1183
|
+
what: `edges of ${r.field}`,
|
|
1184
|
+
value: r.f
|
|
1185
|
+
});
|
|
1186
|
+
}
|
|
1187
|
+
const hasSeq = side === "source" ? (await colsOf("rows"))?.includes("seq") === true : other?.columns.get("rows")?.includes("seq") !== false;
|
|
1188
|
+
const meta = `row(${META}, ${hasSeq ? "seq::text" : "NULL::text"})::text`;
|
|
1189
|
+
const rowsColumns = side === "source" ? (await tx`
|
|
1190
|
+
SELECT attname AS col FROM pg_attribute
|
|
1191
|
+
WHERE attrelid = to_regclass('rows') AND attnum > 0 AND NOT attisdropped AND attgenerated = ''`).map((r) => r.col) : [];
|
|
1192
|
+
for (const { namespace, type } of counted) {
|
|
1193
|
+
let rows;
|
|
1194
|
+
let params;
|
|
1195
|
+
if (side === "source") {
|
|
1196
|
+
const declared = declaredFields(types.get(`${namespace}/${type}`));
|
|
1197
|
+
rows = `SELECT id || E'\n' || (SELECT coalesce(jsonb_object_agg(k, v), '{}'::jsonb) FROM jsonb_each(${sourceBodySql(rowsColumns, declared)}) e(k, v)
|
|
1198
|
+
WHERE NOT (v = 'null'::jsonb AND k = ANY($3::text[])))::text || E'\n' || ${meta} AS h
|
|
1199
|
+
FROM rows WHERE namespace = $1 AND type = $2 ORDER BY md5(id) COLLATE "C" LIMIT ${limit}`;
|
|
1200
|
+
params = [
|
|
1201
|
+
namespace,
|
|
1202
|
+
type,
|
|
1203
|
+
declared
|
|
1204
|
+
];
|
|
1205
|
+
} else {
|
|
1206
|
+
const ref = await typeTableRef(tx, namespace, type);
|
|
1207
|
+
if (ref === void 0) continue;
|
|
1208
|
+
rows = `SELECT x.id || E'\n' || ${ref.rowJson("x")}::text || E'\n' || ${meta} AS h FROM ${ref.from} x
|
|
1209
|
+
WHERE NOT (x.id = ANY($1::text[])) ORDER BY md5(x.id) COLLATE "C" LIMIT ${limit}`;
|
|
1210
|
+
params = [seedIds(namespace, type)];
|
|
1211
|
+
}
|
|
1212
|
+
const [r] = await tx.unsafe(`SELECT ${fp("h")} AS f FROM (${rows}) s`, params);
|
|
1213
|
+
add({
|
|
1214
|
+
check: "sample",
|
|
1215
|
+
namespace,
|
|
1216
|
+
type,
|
|
1217
|
+
what: "content",
|
|
1218
|
+
value: String(r?.f)
|
|
1219
|
+
});
|
|
1220
|
+
}
|
|
1221
|
+
const skip = /* @__PURE__ */ new Set([
|
|
1222
|
+
"rows",
|
|
1223
|
+
"short_ids",
|
|
1224
|
+
"edges"
|
|
1225
|
+
]);
|
|
1226
|
+
for (const name of opts.tables.filter((t) => !skip.has(t))) {
|
|
1227
|
+
if (other !== null && !other.columns.has(name)) continue;
|
|
1228
|
+
const cols = await colsOf(name);
|
|
1229
|
+
if (cols === null) continue;
|
|
1230
|
+
const where = [
|
|
1231
|
+
"row_namespace",
|
|
1232
|
+
"row_type",
|
|
1233
|
+
"row_id"
|
|
1234
|
+
].every((c) => cols.includes(c)) ? notSeed("row_namespace", "row_type", "row_id") : tx`true`;
|
|
1235
|
+
const [r] = await tx`
|
|
1236
|
+
SELECT ${tx.unsafe(fp(`row(${quote(cols)})::text`))} AS f FROM ${tx(name)} WHERE ${where}`;
|
|
1237
|
+
add({
|
|
1238
|
+
check: "table",
|
|
1239
|
+
what: name,
|
|
1240
|
+
value: String(r?.f)
|
|
1241
|
+
});
|
|
1242
|
+
}
|
|
1243
|
+
return {
|
|
1244
|
+
entries,
|
|
1245
|
+
columns
|
|
1246
|
+
};
|
|
1247
|
+
}
|
|
1248
|
+
|
|
1249
|
+
//#endregion
|
|
1250
|
+
//#region src/admin/commands/migrate-store.ts
|
|
1251
|
+
/**
|
|
1252
|
+
* `substrate-admin migrate-store`: converts a legacy or partitioned store into
|
|
1253
|
+
* a fresh, empty PostgreSQL 18 store in the `tables` layout. Preflight,
|
|
1254
|
+
* prepare and copy, index builds, verification, report. Connection strings come from named environment
|
|
1255
|
+
* variables only, and nothing this command prints carries one.
|
|
1256
|
+
*/
|
|
1257
|
+
const DEFAULT_FROM_ENV = "SUBSTRATE_MIGRATE_FROM_URL";
|
|
1258
|
+
const DEFAULT_TO_ENV = "SUBSTRATE_MIGRATE_TO_URL";
|
|
1259
|
+
const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
1260
|
+
const IDENT = PLAIN_SCHEMA_NAME;
|
|
1261
|
+
const SAFE_VALUE = (flag) => `migrate-store: ${flag} takes a plain name (a value that does not fit is not echoed)`;
|
|
1262
|
+
/** Parses the argv tail after `migrate-store`. Usage errors exit 2. */
|
|
1263
|
+
function parseMigrateStoreArgs(tail) {
|
|
1264
|
+
validateArgv("migrate-store", tail);
|
|
1265
|
+
const seen = /* @__PURE__ */ new Map();
|
|
1266
|
+
const args = {
|
|
1267
|
+
fromEnv: DEFAULT_FROM_ENV,
|
|
1268
|
+
toEnv: DEFAULT_TO_ENV,
|
|
1269
|
+
fromSchema: null,
|
|
1270
|
+
toSchema: null,
|
|
1271
|
+
batchRows: 1e4,
|
|
1272
|
+
skipTables: [],
|
|
1273
|
+
dryRun: false,
|
|
1274
|
+
verifyOnly: false,
|
|
1275
|
+
json: false,
|
|
1276
|
+
skipIndexBuild: false,
|
|
1277
|
+
fullChecksum: false
|
|
1278
|
+
};
|
|
1279
|
+
const fail = (msg) => argvError("migrate-store", msg, 2);
|
|
1280
|
+
for (let i = 0; i < tail.length; i++) {
|
|
1281
|
+
const tok = tail[i];
|
|
1282
|
+
switch (tok) {
|
|
1283
|
+
case "--dry-run":
|
|
1284
|
+
args.dryRun = true;
|
|
1285
|
+
continue;
|
|
1286
|
+
case "--verify-only":
|
|
1287
|
+
args.verifyOnly = true;
|
|
1288
|
+
continue;
|
|
1289
|
+
case "--json":
|
|
1290
|
+
args.json = true;
|
|
1291
|
+
continue;
|
|
1292
|
+
case "--skip-index-build":
|
|
1293
|
+
args.skipIndexBuild = true;
|
|
1294
|
+
continue;
|
|
1295
|
+
case "--full-checksum":
|
|
1296
|
+
args.fullChecksum = true;
|
|
1297
|
+
continue;
|
|
1298
|
+
}
|
|
1299
|
+
const value = tail[i + 1];
|
|
1300
|
+
if (value === void 0 || value.startsWith("--")) throw fail(`migrate-store: ${tok} needs a value`);
|
|
1301
|
+
i += 1;
|
|
1302
|
+
if (tok !== "--skip-table") {
|
|
1303
|
+
if (seen.has(tok)) throw fail(`migrate-store: ${tok} given more than once`);
|
|
1304
|
+
seen.set(tok, value);
|
|
1305
|
+
}
|
|
1306
|
+
switch (tok) {
|
|
1307
|
+
case "--from-env":
|
|
1308
|
+
case "--to-env":
|
|
1309
|
+
if (!ENV_NAME.test(value)) throw fail(`migrate-store: ${tok} takes the NAME of an environment variable that holds the connection string, never the string itself`);
|
|
1310
|
+
args[tok === "--from-env" ? "fromEnv" : "toEnv"] = value;
|
|
1311
|
+
break;
|
|
1312
|
+
case "--from-schema":
|
|
1313
|
+
case "--to-schema":
|
|
1314
|
+
if (!IDENT.test(value)) throw fail(SAFE_VALUE(tok));
|
|
1315
|
+
args[tok === "--from-schema" ? "fromSchema" : "toSchema"] = value;
|
|
1316
|
+
break;
|
|
1317
|
+
case "--skip-table":
|
|
1318
|
+
if (!IDENT.test(value)) throw fail(SAFE_VALUE(tok));
|
|
1319
|
+
args.skipTables.push(value);
|
|
1320
|
+
break;
|
|
1321
|
+
case "--batch-rows": {
|
|
1322
|
+
const n = Number(value);
|
|
1323
|
+
if (!/^\d+$/.test(value) || !Number.isSafeInteger(n) || n < 1) throw fail("migrate-store: --batch-rows takes a positive integer");
|
|
1324
|
+
args.batchRows = n;
|
|
1325
|
+
break;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1328
|
+
}
|
|
1329
|
+
if (args.dryRun && args.verifyOnly) throw fail("migrate-store: --dry-run and --verify-only are mutually exclusive");
|
|
1330
|
+
return args;
|
|
1331
|
+
}
|
|
1332
|
+
/** Replaces every URL, and each given secret, in `s`. */
|
|
1333
|
+
function redactor(secrets) {
|
|
1334
|
+
const literals = [...new Set(secrets.filter((x) => x.length >= 3))].sort((a, b) => b.length - a.length);
|
|
1335
|
+
return (s) => {
|
|
1336
|
+
let r = s.replace(/[a-z][a-z0-9+.-]*:\/\/[^\s'"`]+/gi, "<redacted url>");
|
|
1337
|
+
for (const l of literals) r = r.split(l).join("<redacted>");
|
|
1338
|
+
return r;
|
|
1339
|
+
};
|
|
1340
|
+
}
|
|
1341
|
+
/** The connection string, its password both as written and decoded, for `redactor`. */
|
|
1342
|
+
function secretsOf(url) {
|
|
1343
|
+
if (url === void 0 || url.length === 0) return [];
|
|
1344
|
+
const out = [url];
|
|
1345
|
+
let pw = "";
|
|
1346
|
+
try {
|
|
1347
|
+
pw = new URL(url).password;
|
|
1348
|
+
} catch {}
|
|
1349
|
+
if (pw.length === 0) return out;
|
|
1350
|
+
out.push(pw);
|
|
1351
|
+
try {
|
|
1352
|
+
out.push(decodeURIComponent(pw));
|
|
1353
|
+
} catch {}
|
|
1354
|
+
return out;
|
|
1355
|
+
}
|
|
1356
|
+
/** `url` with `search_path` pinned to `schema`, as a startup option so every pooled connection has it. */
|
|
1357
|
+
function withSchema(url, schema) {
|
|
1358
|
+
return schema === null ? url : withSettings(url, [`search_path=${schema},public`]);
|
|
1359
|
+
}
|
|
1360
|
+
/** `url` with each `name=value` in `settings` as a startup option (`-c`). */
|
|
1361
|
+
function withSettings(url, settings) {
|
|
1362
|
+
const u = new URL(url);
|
|
1363
|
+
const prior = u.searchParams.get("options");
|
|
1364
|
+
const params = [...u.searchParams.entries()].filter(([k]) => k !== "options");
|
|
1365
|
+
const added = settings.map((s) => `-c ${s}`).join(" ");
|
|
1366
|
+
params.push(["options", `${prior !== null ? `${prior} ` : ""}${added}`]);
|
|
1367
|
+
u.search = params.map(([k, v]) => `${encodeURIComponent(k)}=${encodeURIComponent(v)}`).join("&");
|
|
1368
|
+
return u.toString();
|
|
1369
|
+
}
|
|
1370
|
+
const DISK_PATH_UNSET = "SUBSTRATE_INDEX_DISK_PATH is unset, so the target cannot check free disk before index builds and its storage check fails verification: set it to the target database volume mount, or to off when the database has no local volume";
|
|
1371
|
+
const PENDING_INDEX = /^index \S+ on \S+ is pending( \(.*\))?: filters it would serve scan until it is valid$/;
|
|
1372
|
+
async function runMigrateStore(args, env, deps) {
|
|
1373
|
+
const started = performance.now();
|
|
1374
|
+
const vars = env;
|
|
1375
|
+
const urlOf = (name, side) => {
|
|
1376
|
+
const v = vars[name]?.trim();
|
|
1377
|
+
if (v === void 0 || v.length === 0) throw new AdminCliError(`migrate-store: ${name} (the ${side} connection string) is not set`, 2);
|
|
1378
|
+
if (!/^postgres(ql)?:\/\//.test(v)) throw new AdminCliError(`migrate-store: ${name} does not hold a postgres:// connection string`, 2);
|
|
1379
|
+
return v;
|
|
1380
|
+
};
|
|
1381
|
+
const fromUrl = withSchema(urlOf(args.fromEnv, "source"), args.fromSchema);
|
|
1382
|
+
const toStore = withSchema(urlOf(args.toEnv, "target"), args.toSchema);
|
|
1383
|
+
if (fromUrl === toStore) throw new AdminCliError("migrate-store: the source and the target are the same store", 2);
|
|
1384
|
+
const toUrl = withSettings(toStore, ["statement_timeout=0"]);
|
|
1385
|
+
const report = {
|
|
1386
|
+
mode: args.dryRun ? "dry-run" : args.verifyOnly ? "verify-only" : "migrate",
|
|
1387
|
+
ok: false,
|
|
1388
|
+
exitCode: 2,
|
|
1389
|
+
refusals: [],
|
|
1390
|
+
warnings: [],
|
|
1391
|
+
tables: [],
|
|
1392
|
+
phases: {},
|
|
1393
|
+
totalMs: 0,
|
|
1394
|
+
keptSeeds: [],
|
|
1395
|
+
defaulted: [],
|
|
1396
|
+
misfits: null,
|
|
1397
|
+
missingIndexes: [],
|
|
1398
|
+
pendingIndexes: [],
|
|
1399
|
+
mismatches: [],
|
|
1400
|
+
switchOver: null
|
|
1401
|
+
};
|
|
1402
|
+
const finish = (exitCode) => {
|
|
1403
|
+
report.exitCode = exitCode;
|
|
1404
|
+
report.ok = exitCode === 0;
|
|
1405
|
+
report.totalMs = Math.round(performance.now() - started);
|
|
1406
|
+
return report;
|
|
1407
|
+
};
|
|
1408
|
+
let running;
|
|
1409
|
+
const phase = async (name, fn) => {
|
|
1410
|
+
deps.progress(`migrate-store: ${name}`);
|
|
1411
|
+
const t = performance.now();
|
|
1412
|
+
running = name;
|
|
1413
|
+
try {
|
|
1414
|
+
const out = await fn();
|
|
1415
|
+
running = void 0;
|
|
1416
|
+
return out;
|
|
1417
|
+
} finally {
|
|
1418
|
+
report.phases[name] = Math.round(performance.now() - t);
|
|
1419
|
+
}
|
|
1420
|
+
};
|
|
1421
|
+
const diskPath = vars.SUBSTRATE_INDEX_DISK_PATH?.trim();
|
|
1422
|
+
if (diskPath === void 0 || diskPath.length === 0) (args.dryRun ? report.warnings : report.refusals).push(DISK_PATH_UNSET);
|
|
1423
|
+
if (report.refusals.length > 0) return finish(2);
|
|
1424
|
+
const targetEnv = {
|
|
1425
|
+
...vars,
|
|
1426
|
+
SUBSTRATE_DATABASE_URL: toUrl,
|
|
1427
|
+
HOST: "127.0.0.1",
|
|
1428
|
+
SUBSTRATE_SYSTEM_ACTOR: vars.SUBSTRATE_SYSTEM_ACTOR?.trim() || "migrate-store"
|
|
1429
|
+
};
|
|
1430
|
+
const pluginDir = resolvePluginDir(targetEnv);
|
|
1431
|
+
const boot = () => bootStore({
|
|
1432
|
+
config: parseEnv(targetEnv),
|
|
1433
|
+
pluginDir,
|
|
1434
|
+
allowMigrating: true,
|
|
1435
|
+
background: false
|
|
1436
|
+
});
|
|
1437
|
+
const notices = pgNoticeOptions(deps.notice ?? ((line) => deps.progress(line.trimEnd())));
|
|
1438
|
+
const openFacts = () => postgres(toUrl, {
|
|
1439
|
+
...notices,
|
|
1440
|
+
max: 1
|
|
1441
|
+
});
|
|
1442
|
+
const endFacts = (facts) => facts.end({ timeout: 5 }).catch(() => void 0);
|
|
1443
|
+
const src = postgres(fromUrl, {
|
|
1444
|
+
...notices,
|
|
1445
|
+
max: 1,
|
|
1446
|
+
connection: {
|
|
1447
|
+
default_transaction_read_only: true,
|
|
1448
|
+
statement_timeout: 0,
|
|
1449
|
+
idle_in_transaction_session_timeout: 0,
|
|
1450
|
+
transaction_timeout: 0
|
|
1451
|
+
}
|
|
1452
|
+
});
|
|
1453
|
+
let target;
|
|
1454
|
+
try {
|
|
1455
|
+
let keptSeeds;
|
|
1456
|
+
let defaulted;
|
|
1457
|
+
let tables;
|
|
1458
|
+
if (args.verifyOnly) {
|
|
1459
|
+
const facts = openFacts();
|
|
1460
|
+
try {
|
|
1461
|
+
await assertSchema(src, args.fromSchema, "source");
|
|
1462
|
+
await assertSchema(facts, args.toSchema, "target");
|
|
1463
|
+
await assertDistinct(src, facts);
|
|
1464
|
+
await assertCopyDone(facts);
|
|
1465
|
+
} finally {
|
|
1466
|
+
await endFacts(facts);
|
|
1467
|
+
}
|
|
1468
|
+
target = await phase("prepare", boot);
|
|
1469
|
+
const state = await readCopyState(target);
|
|
1470
|
+
({keptSeeds, defaulted, tables} = state);
|
|
1471
|
+
} else {
|
|
1472
|
+
const pre = await phase("preflight", async () => {
|
|
1473
|
+
const facts = openFacts();
|
|
1474
|
+
try {
|
|
1475
|
+
const own = await ownSession(facts);
|
|
1476
|
+
await assertSchema(src, args.fromSchema, "source");
|
|
1477
|
+
await assertSchema(facts, args.toSchema, "target");
|
|
1478
|
+
await assertDistinct(src, facts);
|
|
1479
|
+
const source = await inspectSource(src);
|
|
1480
|
+
source.sessions = source.sessions.filter((s) => !(s.pid === own.pid && s.start === own.start));
|
|
1481
|
+
source.otherSessions = deps.otherSessions ?? source.sessions.length;
|
|
1482
|
+
const target = await targetFacts(facts, env, diskPath);
|
|
1483
|
+
const pluginTables = await storeTables(pluginDir);
|
|
1484
|
+
return {
|
|
1485
|
+
source,
|
|
1486
|
+
...preflight(source, target, {
|
|
1487
|
+
skipTables: args.skipTables,
|
|
1488
|
+
pluginTables,
|
|
1489
|
+
dryRun: args.dryRun
|
|
1490
|
+
})
|
|
1491
|
+
};
|
|
1492
|
+
} finally {
|
|
1493
|
+
await endFacts(facts);
|
|
1494
|
+
}
|
|
1495
|
+
});
|
|
1496
|
+
report.refusals.push(...pre.refusals);
|
|
1497
|
+
report.warnings.push(...pre.warnings);
|
|
1498
|
+
report.plan = planOf(pre.plan);
|
|
1499
|
+
if (report.refusals.length > 0) return finish(2);
|
|
1500
|
+
if (args.dryRun) {
|
|
1501
|
+
report.types = pre.source.types;
|
|
1502
|
+
return finish(0);
|
|
1503
|
+
}
|
|
1504
|
+
const copied = await phase("prepare-and-copy", () => copyStore(src, boot, pre.plan, {
|
|
1505
|
+
batchRows: args.batchRows,
|
|
1506
|
+
onProgress: (e) => deps.progress(` ${e.table}: ${e.rows} rows, ${e.bytes} bytes${e.done ? ", done" : ""}`)
|
|
1507
|
+
}));
|
|
1508
|
+
target = copied.target;
|
|
1509
|
+
report.tables = copied.tables;
|
|
1510
|
+
report.misfits = copied.misfits;
|
|
1511
|
+
({keptSeeds, defaulted} = copied);
|
|
1512
|
+
tables = pre.plan.tables.map((t) => t.name);
|
|
1513
|
+
}
|
|
1514
|
+
const booted = target;
|
|
1515
|
+
const indexFailures = args.skipIndexBuild ? [] : await phase("indexes", () => (deps.buildIndexes ?? buildDeferredIndexes)(booted.engine, booted.datasets));
|
|
1516
|
+
report.keptSeeds = keptSeeds.map((k) => `${k.namespace}/${k.type}/${k.id}`);
|
|
1517
|
+
report.defaulted = defaulted;
|
|
1518
|
+
const tsql = booted.sql;
|
|
1519
|
+
await deps.beforeVerify?.(tsql);
|
|
1520
|
+
const verified = await phase("verify", () => verifyStore(src, tsql, {
|
|
1521
|
+
keptSeeds,
|
|
1522
|
+
tables,
|
|
1523
|
+
missingIndexes: () => missingIndexes(booted, args.skipIndexBuild),
|
|
1524
|
+
fullChecksum: args.fullChecksum,
|
|
1525
|
+
sampleRows: 1e3,
|
|
1526
|
+
partitionBudget: parseEnv(targetEnv).partitionBudget,
|
|
1527
|
+
storageReport: async (r) => {
|
|
1528
|
+
const { warnings } = await booted.dataset.storageReport(r);
|
|
1529
|
+
const failed = [...indexFailures, ...await invalidIndexes(tsql)];
|
|
1530
|
+
if (!args.skipIndexBuild) return { warnings: [...warnings, ...failed] };
|
|
1531
|
+
report.pendingIndexes = warnings.filter((w) => PENDING_INDEX.test(w));
|
|
1532
|
+
return { warnings: [...warnings.filter((w) => !PENDING_INDEX.test(w)), ...failed] };
|
|
1533
|
+
}
|
|
1534
|
+
}));
|
|
1535
|
+
report.mismatches = verified.mismatches;
|
|
1536
|
+
report.missingIndexes = verified.missingIndexes;
|
|
1537
|
+
if (!verified.ok) return finish(3);
|
|
1538
|
+
report.switchOver = `stop serve, set SUBSTRATE_DATABASE_URL to the value of ${args.toEnv}${args.toSchema !== null ? ` with search_path ${args.toSchema},public (options=-c search_path=${args.toSchema},public)` : ""}, then start serve. The source is untouched: pointing back at it is the rollback.`;
|
|
1539
|
+
return finish(0);
|
|
1540
|
+
} catch (e) {
|
|
1541
|
+
const message = e instanceof Error ? e.message : String(e);
|
|
1542
|
+
if (e instanceof AdminCliError && (e.exitCode === 2 || e.exitCode === 3)) {
|
|
1543
|
+
report.refusals.push(message);
|
|
1544
|
+
return finish(e.exitCode);
|
|
1545
|
+
}
|
|
1546
|
+
report.error = message;
|
|
1547
|
+
if (running !== void 0) report.failedPhase = running;
|
|
1548
|
+
return finish(1);
|
|
1549
|
+
} finally {
|
|
1550
|
+
await target?.close().catch(() => void 0);
|
|
1551
|
+
await src.end({ timeout: 5 }).catch(() => void 0);
|
|
1552
|
+
}
|
|
1553
|
+
}
|
|
1554
|
+
function planOf(plan) {
|
|
1555
|
+
return {
|
|
1556
|
+
tables: plan.tables.map((t) => ({
|
|
1557
|
+
name: t.name,
|
|
1558
|
+
bytes: t.bytes
|
|
1559
|
+
})),
|
|
1560
|
+
estimatedBytes: plan.estimatedBytes,
|
|
1561
|
+
skipped: plan.skipped
|
|
1562
|
+
};
|
|
1563
|
+
}
|
|
1564
|
+
/** The target schema's INVALID indexes, which a failed CREATE INDEX CONCURRENTLY leaves behind. */
|
|
1565
|
+
async function invalidIndexes(sql) {
|
|
1566
|
+
return (await sql`
|
|
1567
|
+
SELECT c.relname AS name, t.relname AS tbl
|
|
1568
|
+
FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid JOIN pg_class t ON t.oid = i.indrelid
|
|
1569
|
+
WHERE c.relnamespace = current_schema()::regnamespace AND NOT i.indisvalid
|
|
1570
|
+
ORDER BY c.relname`).map((r) => `index ${r.name} on ${r.tbl} is INVALID: rerun with --verify-only (without --skip-index-build), which rebuilds an engine index; drop or REINDEX any other INVALID index by hand`);
|
|
1571
|
+
}
|
|
1572
|
+
/** Refuses a named schema the connection did not land in: an unknown name on search_path falls through to public. */
|
|
1573
|
+
async function assertSchema(sql, schema, side) {
|
|
1574
|
+
const [r] = await sql`SELECT current_schema() AS s`;
|
|
1575
|
+
if (schema !== null && r?.s !== schema) throw new AdminCliError(`migrate-store: the ${side} has no schema ${schema}`, 2);
|
|
1576
|
+
}
|
|
1577
|
+
const storeId = async (sql) => {
|
|
1578
|
+
const [r] = await sql`
|
|
1579
|
+
SELECT concat_ws('/', (SELECT system_identifier FROM pg_control_system())::text,
|
|
1580
|
+
current_database(), current_schema()) AS id`;
|
|
1581
|
+
return r?.id;
|
|
1582
|
+
};
|
|
1583
|
+
/** Refuses a source and target that are one store: same cluster, database and schema. */
|
|
1584
|
+
async function assertDistinct(src, tgt) {
|
|
1585
|
+
if (await storeId(src) === await storeId(tgt)) throw new AdminCliError("migrate-store: the source and the target are the same store", 2);
|
|
1586
|
+
}
|
|
1587
|
+
/** Refuses `--verify-only` on a target with no finished migrate-store copy. */
|
|
1588
|
+
async function assertCopyDone(sql) {
|
|
1589
|
+
const [r] = await sql`
|
|
1590
|
+
SELECT to_regclass(quote_ident(current_schema()) || '.' || quote_ident(${MIGRATE_STORE_MARKER_TABLE}))::text AS marker`;
|
|
1591
|
+
if (r?.marker == null) throw new AdminCliError("migrate-store: the target is already verified, or not a migrate-store target (no marker)", 2);
|
|
1592
|
+
if ((await sql`SELECT 1 FROM ${sql(r.marker)} WHERE step = 'copy:done'`).length === 0) throw new AdminCliError("migrate-store: the target holds no finished migrate-store copy to verify (no in-progress marker with a completed copy)", 2);
|
|
1593
|
+
}
|
|
1594
|
+
async function ownSession(sql) {
|
|
1595
|
+
const [r] = await sql`
|
|
1596
|
+
SELECT pid, extract(epoch FROM backend_start)::text AS start
|
|
1597
|
+
FROM pg_stat_activity WHERE pid = pg_backend_pid()`;
|
|
1598
|
+
return r;
|
|
1599
|
+
}
|
|
1600
|
+
async function targetFacts(sql, env, diskPath) {
|
|
1601
|
+
const [r] = await sql`
|
|
1602
|
+
SELECT current_setting('server_version_num')::int AS v, current_schema() AS schema,
|
|
1603
|
+
to_regclass(quote_ident(current_schema()) || '.rows') IS NOT NULL AS rows,
|
|
1604
|
+
to_regclass(quote_ident(current_schema()) || '.row_ids') IS NOT NULL AS ids,
|
|
1605
|
+
to_regclass(quote_ident(current_schema()) || '.' || quote_ident(${MIGRATE_STORE_MARKER_TABLE})) IS NOT NULL AS marker,
|
|
1606
|
+
to_regclass(quote_ident(current_schema()) || '.embeddings') IS NOT NULL AS embeddings`;
|
|
1607
|
+
const facts = r;
|
|
1608
|
+
const holds = async (table) => (await sql`SELECT 1 FROM ${sql(facts.schema)}.${sql(table)} LIMIT 1`).length > 0;
|
|
1609
|
+
const empty = !(facts.rows && await holds("rows")) && !(facts.ids && await holds("row_ids"));
|
|
1610
|
+
const [dims] = facts.embeddings ? await sql`
|
|
1611
|
+
SELECT atttypmod AS d FROM pg_attribute
|
|
1612
|
+
WHERE attrelid = to_regclass(quote_ident(${facts.schema}) || '.embeddings')
|
|
1613
|
+
AND attname = 'embedding' AND NOT attisdropped` : [];
|
|
1614
|
+
const extensions = await sql`
|
|
1615
|
+
SELECT e.extname AS name, e.extversion AS version
|
|
1616
|
+
FROM pg_extension e JOIN pg_namespace n ON n.oid = e.extnamespace
|
|
1617
|
+
WHERE n.nspname = ANY (current_schemas(true))`;
|
|
1618
|
+
let freeBytes = null;
|
|
1619
|
+
if (diskPath !== void 0 && diskPath.length > 0 && diskPath !== "off") {
|
|
1620
|
+
const s = await statfs(diskPath).catch((err) => {
|
|
1621
|
+
if (err.code !== "ENOENT") throw err;
|
|
1622
|
+
throw new AdminCliError(`migrate-store: SUBSTRATE_INDEX_DISK_PATH names a path that does not exist; set it to the target database volume mount, or to off`, 2);
|
|
1623
|
+
});
|
|
1624
|
+
freeBytes = s.bavail * s.bsize;
|
|
1625
|
+
}
|
|
1626
|
+
return {
|
|
1627
|
+
serverVersionNum: facts.v,
|
|
1628
|
+
layout: await detectLayout(sql),
|
|
1629
|
+
empty,
|
|
1630
|
+
createdByUs: facts.marker,
|
|
1631
|
+
enforceUuidCheck: env.SUBSTRATE_ENFORCE_UUID_CHECK === "true",
|
|
1632
|
+
recallDims: dims !== void 0 && dims.d > 0 ? dims.d : null,
|
|
1633
|
+
extensions: Object.fromEntries(extensions.map((e) => [e.name, e.version])),
|
|
1634
|
+
freeBytes
|
|
1635
|
+
};
|
|
1636
|
+
}
|
|
1637
|
+
/** The loaded kernel plugins' `storeTables`, loaded as serve loads them but never composed. */
|
|
1638
|
+
async function storeTables(pluginDir) {
|
|
1639
|
+
if (pluginDir === null) return [];
|
|
1640
|
+
const { kernel } = await loadPluginsFromDir({
|
|
1641
|
+
pluginDir,
|
|
1642
|
+
kernelHostDeps: {},
|
|
1643
|
+
contractCheck: {
|
|
1644
|
+
hostVersion: BINARY_CONTRACT_VERSION,
|
|
1645
|
+
onViolation: refuseContractViolation
|
|
1646
|
+
}
|
|
1647
|
+
});
|
|
1648
|
+
return kernel.flatMap((p) => p.storeTables ?? []);
|
|
1649
|
+
}
|
|
1650
|
+
function formatMigrateStoreOutput(r, json) {
|
|
1651
|
+
if (json) return `${JSON.stringify(r)}\n`;
|
|
1652
|
+
const lines = [];
|
|
1653
|
+
const verdict = r.exitCode === 0 ? r.mode === "dry-run" ? "dry run: nothing written" : "copied and verified" : r.exitCode === 3 ? "VERIFICATION FAILED: the target keeps its in-progress marker, so serve will not boot it" : r.exitCode === 1 ? `FAILED${r.failedPhase !== void 0 ? ` in ${r.failedPhase}` : ""}: ${r.error ?? "unknown error"}` : "REFUSED: nothing was written to the target";
|
|
1654
|
+
lines.push(`migrate-store (${r.mode}): ${verdict}`);
|
|
1655
|
+
for (const x of r.refusals) lines.push(` refused: ${x}`);
|
|
1656
|
+
for (const x of r.warnings) lines.push(` warning: ${x}`);
|
|
1657
|
+
if (r.types !== void 0) {
|
|
1658
|
+
lines.push("source types (live / deleted / bytes):");
|
|
1659
|
+
for (const t of r.types) lines.push(` ${t.namespace}/${t.type}: ${t.live} / ${t.deleted} / ${t.bytes}`);
|
|
1660
|
+
}
|
|
1661
|
+
if (r.plan !== void 0) {
|
|
1662
|
+
lines.push(`plan (estimated target need ${r.plan.estimatedBytes} bytes):`);
|
|
1663
|
+
for (const t of r.plan.tables) lines.push(` ${t.name}: ${t.bytes} bytes`);
|
|
1664
|
+
if (r.plan.skipped.length > 0) lines.push(` skipped: ${r.plan.skipped.join(", ")}`);
|
|
1665
|
+
}
|
|
1666
|
+
if (r.tables.length > 0) {
|
|
1667
|
+
lines.push("copied (rows / bytes / ms):");
|
|
1668
|
+
for (const t of r.tables) lines.push(` ${t.name}: ${t.rows} / ${t.bytes} / ${t.ms}`);
|
|
1669
|
+
}
|
|
1670
|
+
if (r.keptSeeds.length > 0) lines.push(`kept seeds: ${r.keptSeeds.join(", ")}`);
|
|
1671
|
+
if (r.defaulted.length > 0) lines.push(`declared by no definition, every field in extra: ${r.defaulted.join(", ")}`);
|
|
1672
|
+
if (r.misfits === null) lines.push("misfits: not counted (only a copy run counts them)");
|
|
1673
|
+
for (const [type, m] of Object.entries(r.misfits ?? {})) {
|
|
1674
|
+
const more = m.count > 10 ? ` (first 10 shown; --json lists ${m.first.length})` : "";
|
|
1675
|
+
lines.push(`misfits in ${type}: ${m.count}, kept unchanged under $raw${more}: ${m.first.slice(0, 10).map((x) => `${x.id}.${x.field}`).join(", ")}`);
|
|
1676
|
+
}
|
|
1677
|
+
for (const m of r.missingIndexes) lines.push(` missing index [${m.type}]: ${m.index}`);
|
|
1678
|
+
if (r.pendingIndexes.length > 0) {
|
|
1679
|
+
lines.push(`${r.pendingIndexes.length} index build(s) left pending for serve (--skip-index-build; --json lists them all):`);
|
|
1680
|
+
for (const w of r.pendingIndexes.slice(0, 10)) lines.push(` ${w}`);
|
|
1681
|
+
}
|
|
1682
|
+
for (const m of r.mismatches) {
|
|
1683
|
+
const at = m.namespace !== void 0 ? ` ${m.namespace}/${m.type}` : "";
|
|
1684
|
+
lines.push(` mismatch [${m.check}]${at}: ${m.detail}`);
|
|
1685
|
+
}
|
|
1686
|
+
const phases = Object.entries(r.phases).map(([k, v]) => `${k} ${v}`);
|
|
1687
|
+
lines.push(`ms: ${[...phases, `total ${r.totalMs}`].join(", ")}`);
|
|
1688
|
+
if (r.switchOver !== null) lines.push(`switch over: ${r.switchOver}`);
|
|
1689
|
+
return `${lines.join("\n")}\n`;
|
|
1690
|
+
}
|
|
1691
|
+
/**
|
|
1692
|
+
* The CLI door: parse, run, print, exit code. Every string bound for stdout or
|
|
1693
|
+
* stderr passes one redactor, so no connection string reaches either.
|
|
1694
|
+
*/
|
|
1695
|
+
async function migrateStoreCommand(tail, env, out, errOut, hooks = {}) {
|
|
1696
|
+
const vars = env;
|
|
1697
|
+
const names = /* @__PURE__ */ new Set([
|
|
1698
|
+
DEFAULT_FROM_ENV,
|
|
1699
|
+
DEFAULT_TO_ENV,
|
|
1700
|
+
"SUBSTRATE_DATABASE_URL"
|
|
1701
|
+
]);
|
|
1702
|
+
for (let i = 0; i < tail.length - 1; i++) if (tail[i] === "--from-env" || tail[i] === "--to-env") names.add(tail[i + 1]);
|
|
1703
|
+
const redact = redactor([...names].flatMap((n) => secretsOf(vars[n])));
|
|
1704
|
+
const err = (s) => errOut(redact(s));
|
|
1705
|
+
const stdoutWrite = process.stdout.write;
|
|
1706
|
+
const stderrWrite = process.stderr.write;
|
|
1707
|
+
const realStderr = stderrWrite.bind(process.stderr);
|
|
1708
|
+
process.stdout.write = shim((s) => err(s));
|
|
1709
|
+
process.stderr.write = shim((s) => realStderr(redact(s)));
|
|
1710
|
+
let report;
|
|
1711
|
+
let json = false;
|
|
1712
|
+
try {
|
|
1713
|
+
const args = parseMigrateStoreArgs(tail);
|
|
1714
|
+
json = args.json;
|
|
1715
|
+
report = await runMigrateStore(args, env, {
|
|
1716
|
+
progress: (line) => err(`${line}\n`),
|
|
1717
|
+
...hooks
|
|
1718
|
+
});
|
|
1719
|
+
} catch (e) {
|
|
1720
|
+
err(`${e instanceof Error ? e.message : String(e)}\n`);
|
|
1721
|
+
return e instanceof AdminCliError ? e.exitCode : 1;
|
|
1722
|
+
} finally {
|
|
1723
|
+
process.stdout.write = stdoutWrite;
|
|
1724
|
+
process.stderr.write = stderrWrite;
|
|
1725
|
+
}
|
|
1726
|
+
out(redact(formatMigrateStoreOutput(report, json)));
|
|
1727
|
+
return report.exitCode;
|
|
1728
|
+
}
|
|
1729
|
+
/** A `stream.write` stand-in that hands decoded text to `sink`, honouring `encoding` and `callback`. */
|
|
1730
|
+
function shim(sink) {
|
|
1731
|
+
return ((chunk, encoding, callback) => {
|
|
1732
|
+
const cb = typeof encoding === "function" ? encoding : callback;
|
|
1733
|
+
const enc = typeof encoding === "string" ? encoding : void 0;
|
|
1734
|
+
sink(typeof chunk === "string" ? enc === void 0 || enc === "utf8" || enc === "utf-8" ? chunk : Buffer.from(chunk, enc).toString("utf8") : Buffer.from(chunk).toString("utf8"));
|
|
1735
|
+
if (cb !== void 0) process.nextTick(cb, null);
|
|
1736
|
+
return true;
|
|
1737
|
+
});
|
|
1738
|
+
}
|
|
1739
|
+
|
|
1740
|
+
//#endregion
|
|
1741
|
+
//#region src/admin/commands/promote-fields.ts
|
|
1742
|
+
/**
|
|
1743
|
+
* `substrate-admin promote-fields [--url <base>]` — run the promotion sweep
|
|
1744
|
+
* now, through the running server's admin-gated
|
|
1745
|
+
* `POST /substrate/v1/_system/promote-fields`, rather than waiting for
|
|
1746
|
+
* `SUBSTRATE_PROMOTE_INTERVAL_MS`. It goes through the server because the
|
|
1747
|
+
* sweep's audit events must reach its subscribers; there is no `--direct`.
|
|
1748
|
+
* Unlike the timed sweep, it retries a field a unique conflict holds back,
|
|
1749
|
+
* so an operator who has fixed the data can finish its promotion.
|
|
1750
|
+
*/
|
|
1751
|
+
/** Parses `argv` (after the subcommand), resolves the server, and runs one sweep there. */
|
|
1752
|
+
async function promoteFieldsCommand(argv, env, fetchImpl = substrateFetch) {
|
|
1753
|
+
validateArgv("promote-fields", argv);
|
|
1754
|
+
let url;
|
|
1755
|
+
for (let i = 0; i < argv.length; i++) {
|
|
1756
|
+
if (argv[i] !== "--url") continue;
|
|
1757
|
+
if (url !== void 0) throw argvError("promote-fields", "promote-fields: --url specified more than once");
|
|
1758
|
+
url = argv[i + 1] ?? "";
|
|
1759
|
+
i += 1;
|
|
1760
|
+
}
|
|
1761
|
+
if (url === "") throw argvError("promote-fields", "promote-fields: --url requires a base URL");
|
|
1762
|
+
const target = resolveAdminTarget(url ?? null, env);
|
|
1763
|
+
if (target.kind !== "route") throw new AdminCliError("promote-fields runs on a running server: pass --url <base> or set SUBSTRATE_URL", 1);
|
|
1764
|
+
const auth = resolveAdminAuth(env, target.baseUrl);
|
|
1765
|
+
if ("error" in auth) throw new AdminCliError(`promote-fields: ${auth.error}`, 1);
|
|
1766
|
+
const endpoint = `${trimTrailingSlashes(target.baseUrl)}/substrate/v1/_system/promote-fields`;
|
|
1767
|
+
let res;
|
|
1768
|
+
try {
|
|
1769
|
+
res = await fetchImpl(endpoint, {
|
|
1770
|
+
method: "POST",
|
|
1771
|
+
headers: {
|
|
1772
|
+
authorization: auth.ok,
|
|
1773
|
+
"content-type": "application/json"
|
|
1774
|
+
},
|
|
1775
|
+
body: "{}",
|
|
1776
|
+
redirect: "error"
|
|
1777
|
+
});
|
|
1778
|
+
} catch (err) {
|
|
1779
|
+
throw new AdminCliError(`could not reach ${endpoint}: ${err instanceof Error ? err.message : String(err)}`, 1);
|
|
1780
|
+
}
|
|
1781
|
+
const payload = await res.json().catch(() => ({}));
|
|
1782
|
+
if (!res.ok) {
|
|
1783
|
+
const reason = typeof payload.message === "string" ? payload.message : typeof payload.code === "string" ? payload.code : `HTTP ${res.status}`;
|
|
1784
|
+
throw new AdminCliError(`promote-fields: ${reason}`, res.status === 403 ? 5 : 1);
|
|
1785
|
+
}
|
|
1786
|
+
if (!Array.isArray(payload.events)) throw new AdminCliError("promote-fields: response carried no `events` array", 1);
|
|
1787
|
+
return {
|
|
1788
|
+
target: describeAdminTarget("promote-fields", target),
|
|
1789
|
+
events: payload.events
|
|
1790
|
+
};
|
|
1791
|
+
}
|
|
1792
|
+
/** One line per field, after the target line. */
|
|
1793
|
+
function formatPromoteFieldsOutput(result) {
|
|
1794
|
+
const lines = [result.target];
|
|
1795
|
+
if (result.events.length === 0) lines.push("no pending fields");
|
|
1796
|
+
for (const e of result.events) {
|
|
1797
|
+
const outcome = e.kind === "busy" && e.error !== void 0 ? `busy (${e.error})` : `${e.kind} (attempt ${e.attempts})${e.error !== void 0 ? ` — ${e.error}` : ""}`;
|
|
1798
|
+
lines.push(`${e.namespace}/${e.type}.${e.field}: ${outcome}`);
|
|
1799
|
+
}
|
|
1800
|
+
return `${lines.join("\n")}\n`;
|
|
1801
|
+
}
|
|
1802
|
+
|
|
1803
|
+
//#endregion
|
|
1804
|
+
//#region src/admin/commands/revoke.ts
|
|
1805
|
+
/**
|
|
1806
|
+
* `revoke` — soft-delete a Key by stamping `revokedAt`.
|
|
1807
|
+
*
|
|
1808
|
+
* Refuses the last active admin. `checkRevocable` is shared with the route so
|
|
1809
|
+
* both transports enforce the same floor.
|
|
1810
|
+
*/
|
|
1811
|
+
async function runRevoke(kid, env, openDataset = openPgDataset) {
|
|
1812
|
+
if (kid === void 0 || kid.length === 0) throw argvError("revoke", "revoke requires a <kid> positional arg");
|
|
1813
|
+
const adminCapability = env.SUBSTRATE_ADMIN_CAPABILITY !== void 0 && env.SUBSTRATE_ADMIN_CAPABILITY.length > 0 ? env.SUBSTRATE_ADMIN_CAPABILITY : "admin";
|
|
1814
|
+
const { dataset, close } = await openDataset(env);
|
|
1815
|
+
try {
|
|
1816
|
+
return await dataset.withTransaction(async (tx) => {
|
|
1817
|
+
const existing = await readKeyById(tx, kid);
|
|
1818
|
+
const refusal = checkRevocable({
|
|
1819
|
+
kid,
|
|
1820
|
+
exists: existing !== null,
|
|
1821
|
+
existingRevokedAt: existing?.revokedAt,
|
|
1822
|
+
activeAdminKids: (await listActiveAdminKeys(tx, adminCapability)).map((a) => a.kid)
|
|
1823
|
+
});
|
|
1824
|
+
if (refusal !== null) throw new AdminCliError(refusal.message, REVOKE_EXIT_CODES[refusal.code]);
|
|
1825
|
+
const now = /* @__PURE__ */ new Date();
|
|
1826
|
+
const mergedRow = forUpdate({
|
|
1827
|
+
...existing.row,
|
|
1828
|
+
revokedAt: now
|
|
1829
|
+
});
|
|
1830
|
+
await tx.write("Key", [mergedRow], ACTOR_HINT);
|
|
1831
|
+
await tx.write(AUDIT_EVENT_TYPE, [buildAuditRow({
|
|
1832
|
+
actorKid: CLI_ACTOR_KID,
|
|
1833
|
+
kind: "key_revoked",
|
|
1834
|
+
targetKid: kid,
|
|
1835
|
+
details: {
|
|
1836
|
+
revokedAt: now.toISOString(),
|
|
1837
|
+
cliSubcommand: "revoke"
|
|
1838
|
+
},
|
|
1839
|
+
now
|
|
1840
|
+
})], ACTOR_HINT);
|
|
1841
|
+
return {
|
|
1842
|
+
kid,
|
|
1843
|
+
revokedAt: now.toISOString()
|
|
1844
|
+
};
|
|
1845
|
+
});
|
|
1846
|
+
} finally {
|
|
1847
|
+
await close();
|
|
1848
|
+
}
|
|
1849
|
+
}
|
|
1850
|
+
function formatRevokeOutput(result) {
|
|
1851
|
+
return [
|
|
1852
|
+
`Key ${result.kid} revoked at ${result.revokedAt}`,
|
|
1853
|
+
"",
|
|
1854
|
+
RESTART_NOTE,
|
|
1855
|
+
""
|
|
1856
|
+
].join("\n");
|
|
1857
|
+
}
|
|
1858
|
+
|
|
1859
|
+
//#endregion
|
|
1860
|
+
//#region src/admin/commands/rotate.ts
|
|
1861
|
+
/**
|
|
1862
|
+
* `rotate` — mint a fresh keypair for an existing kid and retire the old row,
|
|
1863
|
+
* in one transaction.
|
|
1864
|
+
*
|
|
1865
|
+
* `--out` is required and must not already exist. An admin tool that silently
|
|
1866
|
+
* clobbers an operator's private key destroys the only copy of a credential.
|
|
1867
|
+
*/
|
|
1868
|
+
function parseRotateArgs(argv) {
|
|
1869
|
+
try {
|
|
1870
|
+
return parseRotateArgsInner(argv);
|
|
1871
|
+
} catch (err) {
|
|
1872
|
+
if (err instanceof AdminCliError) throw argvError("rotate", err.message, err.exitCode);
|
|
1873
|
+
throw err;
|
|
1874
|
+
}
|
|
1875
|
+
}
|
|
1876
|
+
function parseRotateArgsInner(argv) {
|
|
1877
|
+
const rest = argv.slice(1);
|
|
1878
|
+
validateArgv("rotate", rest);
|
|
1879
|
+
let kid;
|
|
1880
|
+
let outPath;
|
|
1881
|
+
let wantsEncrypt = false;
|
|
1882
|
+
for (let i = 0; i < rest.length; i++) {
|
|
1883
|
+
const tok = rest[i];
|
|
1884
|
+
if (tok === void 0) continue;
|
|
1885
|
+
if (tok === "--out") {
|
|
1886
|
+
if (outPath !== void 0) throw new AdminCliError("rotate: --out specified more than once", 1);
|
|
1887
|
+
const val = rest[i + 1];
|
|
1888
|
+
if (val === void 0 || val.length === 0 || val.startsWith("--")) throw new AdminCliError("rotate: --out requires a path argument", 1);
|
|
1889
|
+
outPath = val;
|
|
1890
|
+
i += 1;
|
|
1891
|
+
} else if (tok === "--encrypt") {
|
|
1892
|
+
if (wantsEncrypt) throw new AdminCliError("rotate: --encrypt specified more than once", 1);
|
|
1893
|
+
wantsEncrypt = true;
|
|
1894
|
+
} else kid = tok;
|
|
1895
|
+
}
|
|
1896
|
+
if (kid === void 0 || kid.length === 0) throw new AdminCliError("rotate requires a <kid> positional arg", 1);
|
|
1897
|
+
if (outPath === void 0 || outPath.length === 0) throw new AdminCliError("rotate requires --out <path>", 1);
|
|
1898
|
+
return {
|
|
1899
|
+
kid,
|
|
1900
|
+
outPath,
|
|
1901
|
+
wantsEncrypt
|
|
1902
|
+
};
|
|
1903
|
+
}
|
|
1904
|
+
async function runRotate(kid, outPath, env, wantsEncrypt = false, openDataset = openPgDataset) {
|
|
1905
|
+
if (kid === void 0 || kid.length === 0) throw new AdminCliError("rotate requires a <kid> positional arg");
|
|
1906
|
+
if (outPath === void 0 || outPath.length === 0) throw new AdminCliError("rotate requires --out <path>");
|
|
1907
|
+
if (existsSync(outPath)) throw new AdminCliError(`--out path already exists: ${outPath} (move it aside and retry)`, 2);
|
|
1908
|
+
const { publicKeyPem, privateKeyPem, encrypted } = generateAdminKeypair(env, wantsEncrypt);
|
|
1909
|
+
try {
|
|
1910
|
+
mkdirSync(dirname(outPath), {
|
|
1911
|
+
recursive: true,
|
|
1912
|
+
mode: 448
|
|
1913
|
+
});
|
|
1914
|
+
writeFileSync(outPath, privateKeyPem, {
|
|
1915
|
+
mode: 384,
|
|
1916
|
+
flag: "wx"
|
|
1917
|
+
});
|
|
1918
|
+
} catch (err) {
|
|
1919
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
1920
|
+
throw new AdminCliError(`failed to write private key to '${outPath}': ${msg}. Nothing was changed.`, 5);
|
|
1921
|
+
}
|
|
1922
|
+
let committed = false;
|
|
1923
|
+
const { dataset, close } = await openDataset(env).catch((err) => {
|
|
1924
|
+
rmSync(outPath, { force: true });
|
|
1925
|
+
throw err;
|
|
1926
|
+
});
|
|
1927
|
+
try {
|
|
1928
|
+
const newKid = randomUUID();
|
|
1929
|
+
const shaFull = createHash("sha256").update(publicKeyPem).digest();
|
|
1930
|
+
const fingerprint = `sha256:${base64Url(shaFull).slice(0, 8)}`;
|
|
1931
|
+
const { name, namespaces, capabilities } = await dataset.withTransaction(async (tx) => {
|
|
1932
|
+
const existing = await readKeyById(tx, kid);
|
|
1933
|
+
if (existing === null) throw new AdminCliError(`no Key with kid ${kid}`, 3);
|
|
1934
|
+
if (existing.revokedAt != null) throw new AdminCliError(`cannot rotate revoked Key ${kid}`, 4);
|
|
1935
|
+
const preservedName = typeof existing.row.name === "string" ? existing.row.name : "";
|
|
1936
|
+
const preservedAlgorithm = typeof existing.row.algorithm === "string" ? existing.row.algorithm : "EdDSA";
|
|
1937
|
+
if (preservedAlgorithm !== "EdDSA") throw new AdminCliError(`cannot rotate Key with algorithm '${preservedAlgorithm}' — only EdDSA is currently supported by the CLI`, 5);
|
|
1938
|
+
const preservedNamespaces = typeof existing.row.namespaces === "string" ? existing.row.namespaces : "";
|
|
1939
|
+
const preservedCapabilities = typeof existing.row.capabilities === "string" ? existing.row.capabilities : "";
|
|
1940
|
+
const preservedKid = existing.row.kid;
|
|
1941
|
+
const now = /* @__PURE__ */ new Date();
|
|
1942
|
+
await tx.write("Key", [forUpdate({
|
|
1943
|
+
...existing.row,
|
|
1944
|
+
revokedAt: now
|
|
1945
|
+
})], ACTOR_HINT);
|
|
1946
|
+
await tx.write("Key", [{
|
|
1947
|
+
id: newKid,
|
|
1948
|
+
kid: preservedKid,
|
|
1949
|
+
name: preservedName,
|
|
1950
|
+
algorithm: preservedAlgorithm,
|
|
1951
|
+
publicKey: publicKeyPem,
|
|
1952
|
+
namespaces: preservedNamespaces,
|
|
1953
|
+
capabilities: preservedCapabilities,
|
|
1954
|
+
revokedAt: null
|
|
1955
|
+
}], ACTOR_HINT);
|
|
1956
|
+
await tx.write(AUDIT_EVENT_TYPE, [buildAuditRow({
|
|
1957
|
+
actorKid: CLI_ACTOR_KID,
|
|
1958
|
+
kind: "key_rotated",
|
|
1959
|
+
targetKid: kid,
|
|
1960
|
+
details: {
|
|
1961
|
+
oldKid: kid,
|
|
1962
|
+
newKid,
|
|
1963
|
+
name: preservedName
|
|
1964
|
+
},
|
|
1965
|
+
now
|
|
1966
|
+
})], ACTOR_HINT);
|
|
1967
|
+
return {
|
|
1968
|
+
name: preservedName,
|
|
1969
|
+
namespaces: preservedNamespaces,
|
|
1970
|
+
capabilities: preservedCapabilities
|
|
1971
|
+
};
|
|
1972
|
+
});
|
|
1973
|
+
committed = true;
|
|
1974
|
+
return {
|
|
1975
|
+
oldKid: kid,
|
|
1976
|
+
newKid,
|
|
1977
|
+
fingerprint,
|
|
1978
|
+
privateKeyPath: outPath,
|
|
1979
|
+
name,
|
|
1980
|
+
namespaces,
|
|
1981
|
+
capabilities,
|
|
1982
|
+
encrypted
|
|
1983
|
+
};
|
|
1984
|
+
} finally {
|
|
1985
|
+
if (!committed) rmSync(outPath, { force: true });
|
|
1986
|
+
await close();
|
|
1987
|
+
}
|
|
1988
|
+
}
|
|
1989
|
+
function formatRotateOutput(result) {
|
|
1990
|
+
const encLabel = result.encrypted ? "encrypted: aes-256-cbc+pbes2" : "unencrypted";
|
|
1991
|
+
return [
|
|
1992
|
+
"substrate-admin rotate complete.",
|
|
1993
|
+
`old kid: ${result.oldKid}`,
|
|
1994
|
+
`new kid: ${result.newKid}`,
|
|
1995
|
+
`fingerprint: ${result.fingerprint}`,
|
|
1996
|
+
`private key: ${result.privateKeyPath} (mode 0600, ${encLabel})`,
|
|
1997
|
+
`namespaces: ${result.namespaces}`,
|
|
1998
|
+
`capabilities: ${result.capabilities}`,
|
|
1999
|
+
"",
|
|
2000
|
+
`Use the new private key to sign JWTs with kid=${result.newKid} to authenticate. The old key is revoked and no longer valid.`,
|
|
2001
|
+
"",
|
|
2002
|
+
RESTART_NOTE,
|
|
2003
|
+
""
|
|
2004
|
+
].join("\n");
|
|
2005
|
+
}
|
|
2006
|
+
|
|
2007
|
+
//#endregion
|
|
2008
|
+
//#region src/admin/commands/update.ts
|
|
2009
|
+
/**
|
|
2010
|
+
* `update` — change a live key's grant in place, leaving the keypair alone.
|
|
2011
|
+
*
|
|
2012
|
+
* An omitted flag leaves that field unchanged; `--ns` REPLACES rather than
|
|
2013
|
+
* merges. Widening authority needs `--widen`, narrowing does not, and the audit
|
|
2014
|
+
* event carries before AND after so a grant change is reconstructable from the
|
|
2015
|
+
* trail rather than from whoever remembers running it.
|
|
2016
|
+
*/
|
|
2017
|
+
/**
|
|
2018
|
+
* Parse `update <kid> [--ns <csv>] [--caps <csv>] [--name <n>] [--widen]
|
|
2019
|
+
* [--allow-admin]`.
|
|
2020
|
+
*
|
|
2021
|
+
* Unlike `parseRegisterArgs`, an omitted flag means LEAVE UNCHANGED: the
|
|
2022
|
+
* defaulting `register` does at parse time (caps → read,mutate, name → kid)
|
|
2023
|
+
* is deliberately not repeated, so `update <kid> --name foo` cannot change
|
|
2024
|
+
* the grant.
|
|
2025
|
+
*/
|
|
2026
|
+
function parseUpdateArgs(argv) {
|
|
2027
|
+
try {
|
|
2028
|
+
return parseUpdateArgsInner(argv);
|
|
2029
|
+
} catch (err) {
|
|
2030
|
+
if (err instanceof AdminCliError) throw argvError("update", err.message, err.exitCode);
|
|
2031
|
+
throw err;
|
|
2032
|
+
}
|
|
2033
|
+
}
|
|
2034
|
+
function parseUpdateArgsInner(argv) {
|
|
2035
|
+
const rest = argv.slice(1);
|
|
2036
|
+
validateArgv("update", rest);
|
|
2037
|
+
let kid;
|
|
2038
|
+
let name;
|
|
2039
|
+
let nsCsv;
|
|
2040
|
+
let capsCsv;
|
|
2041
|
+
let widen = false;
|
|
2042
|
+
let allowAdmin = false;
|
|
2043
|
+
let allowUnknownNs = false;
|
|
2044
|
+
let url = null;
|
|
2045
|
+
const takeValue = (flag, i) => {
|
|
2046
|
+
const val = rest[i + 1];
|
|
2047
|
+
if (val === void 0 || val.length === 0 || val.startsWith("--")) throw new AdminCliError(`update: ${flag} requires a value`, 1);
|
|
2048
|
+
return val;
|
|
2049
|
+
};
|
|
2050
|
+
for (let i = 0; i < rest.length; i++) {
|
|
2051
|
+
const tok = rest[i];
|
|
2052
|
+
if (tok === void 0) continue;
|
|
2053
|
+
if (tok === "--name") {
|
|
2054
|
+
if (name !== void 0) throw new AdminCliError("update: --name specified more than once", 1);
|
|
2055
|
+
name = takeValue("--name", i);
|
|
2056
|
+
i += 1;
|
|
2057
|
+
} else if (tok === "--ns") {
|
|
2058
|
+
if (nsCsv !== void 0) throw new AdminCliError("update: --ns specified more than once", 1);
|
|
2059
|
+
nsCsv = takeValue("--ns", i);
|
|
2060
|
+
i += 1;
|
|
2061
|
+
} else if (tok === "--caps") {
|
|
2062
|
+
if (capsCsv !== void 0) throw new AdminCliError("update: --caps specified more than once", 1);
|
|
2063
|
+
capsCsv = takeValue("--caps", i);
|
|
2064
|
+
i += 1;
|
|
2065
|
+
} else if (tok === "--url") {
|
|
2066
|
+
if (url === "") throw new AdminCliError("update: --direct and --url are mutually exclusive", 1);
|
|
2067
|
+
if (url !== null) throw new AdminCliError("update: --url specified more than once", 1);
|
|
2068
|
+
url = takeValue("--url", i);
|
|
2069
|
+
i += 1;
|
|
2070
|
+
} else if (tok === "--direct") {
|
|
2071
|
+
if (url !== null && url !== "") throw new AdminCliError("update: --direct and --url are mutually exclusive", 1);
|
|
2072
|
+
url = "";
|
|
2073
|
+
} else if (tok === "--allow-unknown-ns") allowUnknownNs = true;
|
|
2074
|
+
else if (tok === "--widen") {
|
|
2075
|
+
if (widen) throw new AdminCliError("update: --widen specified more than once", 1);
|
|
2076
|
+
widen = true;
|
|
2077
|
+
} else if (tok === "--allow-admin") {
|
|
2078
|
+
if (allowAdmin) throw new AdminCliError("update: --allow-admin specified more than once", 1);
|
|
2079
|
+
allowAdmin = true;
|
|
2080
|
+
} else kid = tok;
|
|
2081
|
+
}
|
|
2082
|
+
if (kid === void 0 || kid.length === 0) throw new AdminCliError("update requires a <kid> positional arg", 1);
|
|
2083
|
+
const splitCsv = (csv) => csv.split(",").map((s) => s.trim()).filter((s) => s.length > 0);
|
|
2084
|
+
const namespaces = nsCsv === void 0 ? void 0 : splitCsv(nsCsv);
|
|
2085
|
+
if (namespaces !== void 0) {
|
|
2086
|
+
const nsError = grantEntryError(namespaces, ["read", "mutate"]);
|
|
2087
|
+
if (nsError !== null) throw new AdminCliError(`update: ${nsError} — e.g. --ns inventory:read,notes:read+mutate`, 1);
|
|
2088
|
+
}
|
|
2089
|
+
return {
|
|
2090
|
+
kid,
|
|
2091
|
+
name,
|
|
2092
|
+
namespaces,
|
|
2093
|
+
capabilities: capsCsv === void 0 ? void 0 : splitCsv(capsCsv),
|
|
2094
|
+
widen,
|
|
2095
|
+
allowAdmin,
|
|
2096
|
+
allowUnknownNs,
|
|
2097
|
+
url
|
|
2098
|
+
};
|
|
2099
|
+
}
|
|
2100
|
+
/**
|
|
2101
|
+
* NOTE on `--allow-unknown-ns`: the unhosted-namespace check is ROUTE-SIDE
|
|
2102
|
+
* only, because it needs the engine's federated schema. A `--direct` update
|
|
2103
|
+
* can still write a grant naming a namespace nobody hosts.
|
|
2104
|
+
*/
|
|
2105
|
+
async function runUpdate(args, env, openDataset = openPgDataset) {
|
|
2106
|
+
if (args.name === void 0 && args.namespaces === void 0 && args.capabilities === void 0) throw new AdminCliError("update requires at least one of --ns, --caps or --name — there is nothing to change", 1);
|
|
2107
|
+
const adminCapability = env.SUBSTRATE_ADMIN_CAPABILITY !== void 0 && env.SUBSTRATE_ADMIN_CAPABILITY.length > 0 ? env.SUBSTRATE_ADMIN_CAPABILITY : "admin";
|
|
2108
|
+
const { dataset, close } = await openDataset(env);
|
|
2109
|
+
try {
|
|
2110
|
+
return await dataset.withTransaction(async (tx) => {
|
|
2111
|
+
const existing = await readKeyById(tx, args.kid);
|
|
2112
|
+
const prevNs = decodeList(existing?.row.namespaces);
|
|
2113
|
+
const prevCaps = decodeList(existing?.row.capabilities);
|
|
2114
|
+
const prevName = typeof existing?.row.name === "string" ? existing.row.name : "";
|
|
2115
|
+
const nextNs = args.namespaces ?? prevNs;
|
|
2116
|
+
const nextCaps = args.capabilities ?? prevCaps;
|
|
2117
|
+
const nextName = args.name ?? prevName;
|
|
2118
|
+
const refusal = checkUpdatable({
|
|
2119
|
+
kid: args.kid,
|
|
2120
|
+
exists: existing !== null,
|
|
2121
|
+
existingRevokedAt: existing?.revokedAt,
|
|
2122
|
+
prevNamespaces: prevNs,
|
|
2123
|
+
prevCapabilities: prevCaps,
|
|
2124
|
+
nextNamespaces: nextNs,
|
|
2125
|
+
nextCapabilities: nextCaps,
|
|
2126
|
+
activeAdminKids: (await listActiveAdminKeys(tx, adminCapability)).map((a) => a.kid),
|
|
2127
|
+
adminCapability,
|
|
2128
|
+
widen: args.widen === true,
|
|
2129
|
+
allowAdmin: args.allowAdmin === true
|
|
2130
|
+
});
|
|
2131
|
+
if (refusal !== null) {
|
|
2132
|
+
const suffix = refusal.code === "requires_widen" ? " Pass --widen to confirm it is deliberate. Narrowing a grant needs no flag." : refusal.code === "requires_allow_admin" ? "" : "";
|
|
2133
|
+
const message = refusal.code === "requires_allow_admin" ? "update: granting the 'admin' capability requires --allow-admin" : `${refusal.message}${suffix}`;
|
|
2134
|
+
throw new AdminCliError(message, UPDATE_EXIT_CODES[refusal.code]);
|
|
2135
|
+
}
|
|
2136
|
+
const nsGranted = newlyGranted(prevNs, nextNs);
|
|
2137
|
+
const capsGranted = newlyGranted(prevCaps, nextCaps);
|
|
2138
|
+
const before = {
|
|
2139
|
+
name: prevName,
|
|
2140
|
+
namespaces: encodeList(prevNs),
|
|
2141
|
+
capabilities: encodeList(prevCaps)
|
|
2142
|
+
};
|
|
2143
|
+
const after = {
|
|
2144
|
+
name: nextName,
|
|
2145
|
+
namespaces: encodeList(nextNs),
|
|
2146
|
+
capabilities: encodeList(nextCaps)
|
|
2147
|
+
};
|
|
2148
|
+
if (!(after.name !== before.name || after.namespaces !== before.namespaces || after.capabilities !== before.capabilities)) return {
|
|
2149
|
+
kid: args.kid,
|
|
2150
|
+
before,
|
|
2151
|
+
after,
|
|
2152
|
+
changed: false
|
|
2153
|
+
};
|
|
2154
|
+
const now = /* @__PURE__ */ new Date();
|
|
2155
|
+
await tx.write("Key", [forUpdate({
|
|
2156
|
+
...existing.row,
|
|
2157
|
+
name: after.name,
|
|
2158
|
+
namespaces: after.namespaces,
|
|
2159
|
+
capabilities: after.capabilities
|
|
2160
|
+
})], ACTOR_HINT);
|
|
2161
|
+
await tx.write(AUDIT_EVENT_TYPE, [buildAuditRow({
|
|
2162
|
+
actorKid: CLI_ACTOR_KID,
|
|
2163
|
+
kind: "key_authority_updated",
|
|
2164
|
+
targetKid: args.kid,
|
|
2165
|
+
details: {
|
|
2166
|
+
before,
|
|
2167
|
+
after,
|
|
2168
|
+
namespacesGranted: nsGranted,
|
|
2169
|
+
capabilitiesGranted: capsGranted,
|
|
2170
|
+
widened: nsGranted.length > 0 || capsGranted.length > 0,
|
|
2171
|
+
cliSubcommand: "update"
|
|
2172
|
+
},
|
|
2173
|
+
now
|
|
2174
|
+
})], ACTOR_HINT);
|
|
2175
|
+
return {
|
|
2176
|
+
kid: args.kid,
|
|
2177
|
+
before,
|
|
2178
|
+
after,
|
|
2179
|
+
changed: true
|
|
2180
|
+
};
|
|
2181
|
+
});
|
|
2182
|
+
} finally {
|
|
2183
|
+
await close();
|
|
2184
|
+
}
|
|
2185
|
+
}
|
|
2186
|
+
function formatUpdateOutput(result) {
|
|
2187
|
+
if (!result.changed) return [
|
|
2188
|
+
`substrate-admin update: no change — ${result.kid} already carries this authority.`,
|
|
2189
|
+
`name: ${result.before.name}`,
|
|
2190
|
+
...grantLines("grant: ", result.before.namespaces, result.before.capabilities),
|
|
2191
|
+
` (stored as: namespaces='${result.before.namespaces}' capabilities='${result.before.capabilities}')`,
|
|
2192
|
+
"",
|
|
2193
|
+
"Nothing was written and no audit row was emitted.",
|
|
2194
|
+
""
|
|
2195
|
+
].join("\n");
|
|
2196
|
+
const row = (label, b, a) => b === a ? `${label}${b === "" ? "(empty)" : b} (unchanged)` : `${label}${b} -> ${a}`;
|
|
2197
|
+
const beforeGrant = grantLines("grant before: ", result.before.namespaces, result.before.capabilities);
|
|
2198
|
+
const afterGrant = grantLines(" after: ", result.after.namespaces, result.after.capabilities);
|
|
2199
|
+
return [
|
|
2200
|
+
"substrate-admin update complete.",
|
|
2201
|
+
`kid: ${result.kid}`,
|
|
2202
|
+
row("name: ", result.before.name, result.after.name),
|
|
2203
|
+
...beforeGrant,
|
|
2204
|
+
...afterGrant,
|
|
2205
|
+
` (stored as: namespaces='${result.after.namespaces}' capabilities='${result.after.capabilities}')`,
|
|
2206
|
+
"",
|
|
2207
|
+
RESTART_NOTE,
|
|
2208
|
+
""
|
|
2209
|
+
].join("\n");
|
|
2210
|
+
}
|
|
2211
|
+
|
|
2212
|
+
//#endregion
|
|
2213
|
+
//#region src/admin/main.ts
|
|
2214
|
+
/**
|
|
2215
|
+
* Argv -> subcommand dispatch.
|
|
2216
|
+
*
|
|
2217
|
+
* Every case here is thin on purpose: parse, resolve the target, call the
|
|
2218
|
+
* command module, format. The logic lives in `./commands/`, so this file stays
|
|
2219
|
+
* readable as the map of what the CLI can do.
|
|
2220
|
+
*/
|
|
2221
|
+
/**
|
|
2222
|
+
* CLI entry point. Parses `process.argv`, dispatches to the requested
|
|
2223
|
+
* sub-command, and writes human-readable output. Returns the exit code so the
|
|
2224
|
+
* top-level runner can call `process.exit` once.
|
|
2225
|
+
*/
|
|
2226
|
+
async function main(argv, env, out, errOut, openDataset = openPgDataset, prompt = {
|
|
2227
|
+
ask: async () => null,
|
|
2228
|
+
interactive: false
|
|
2229
|
+
}) {
|
|
2230
|
+
const command = argv[0];
|
|
2231
|
+
if (command === void 0) {
|
|
2232
|
+
errOut(`${USAGE}\n`);
|
|
2233
|
+
return 1;
|
|
2234
|
+
}
|
|
2235
|
+
if (command === "--version" || command === "-v" || command === "version") {
|
|
2236
|
+
out(`${version}\n`);
|
|
2237
|
+
return 0;
|
|
2238
|
+
}
|
|
2239
|
+
if (command === "help" || argv.some((a) => a === "--help" || a === "-h")) {
|
|
2240
|
+
out(`${USAGE}\n`);
|
|
2241
|
+
return 0;
|
|
2242
|
+
}
|
|
2243
|
+
try {
|
|
2244
|
+
if (ARGV_SPEC[command] === void 0) {
|
|
2245
|
+
errOut(`unknown command: ${command}\n${USAGE}\n`);
|
|
2246
|
+
return 1;
|
|
2247
|
+
}
|
|
2248
|
+
if (command !== "migrate-store") validateArgv(command, argv.slice(1));
|
|
2249
|
+
switch (command) {
|
|
2250
|
+
case "bootstrap": {
|
|
2251
|
+
const parsed = parseBootstrapArgs(argv);
|
|
2252
|
+
const result = await runBootstrap(env, parsed.wantsEncrypt);
|
|
2253
|
+
out(formatBootstrapOutput(result));
|
|
2254
|
+
return 0;
|
|
2255
|
+
}
|
|
2256
|
+
case "doctor": {
|
|
2257
|
+
const report = await doctorCommand(argv.slice(1), env);
|
|
2258
|
+
out(formatDoctorReport(report));
|
|
2259
|
+
return report.exitCode;
|
|
2260
|
+
}
|
|
2261
|
+
case "onboard": return onboardCommand(argv.slice(1), env, {
|
|
2262
|
+
out,
|
|
2263
|
+
ask: prompt.ask
|
|
2264
|
+
}, prompt.interactive);
|
|
2265
|
+
case "list": {
|
|
2266
|
+
const args = parseListArgs(argv.slice(1));
|
|
2267
|
+
if (args.url !== void 0) {
|
|
2268
|
+
const remote = await runKeyListViaRoute(args.url, env);
|
|
2269
|
+
out(formatRemoteListOutput(remote, args.url));
|
|
2270
|
+
return 0;
|
|
2271
|
+
}
|
|
2272
|
+
const rows = await runList(env, openDataset);
|
|
2273
|
+
out(formatListOutput(rows));
|
|
2274
|
+
return 0;
|
|
2275
|
+
}
|
|
2276
|
+
case "revoke": {
|
|
2277
|
+
const rest = argv.slice(1);
|
|
2278
|
+
let kid;
|
|
2279
|
+
let url;
|
|
2280
|
+
let direct = false;
|
|
2281
|
+
for (let i = 0; i < rest.length; i++) {
|
|
2282
|
+
const tok = rest[i];
|
|
2283
|
+
if (tok === void 0) continue;
|
|
2284
|
+
if (tok === "--url") {
|
|
2285
|
+
if (url !== void 0) throw argvError("revoke", "revoke: --url specified more than once", 1);
|
|
2286
|
+
url = rest[i + 1] ?? "";
|
|
2287
|
+
i += 1;
|
|
2288
|
+
continue;
|
|
2289
|
+
}
|
|
2290
|
+
if (tok === "--direct") {
|
|
2291
|
+
direct = true;
|
|
2292
|
+
continue;
|
|
2293
|
+
}
|
|
2294
|
+
kid = tok;
|
|
2295
|
+
}
|
|
2296
|
+
if (kid === void 0 || kid.length === 0) {
|
|
2297
|
+
errOut(`revoke requires a <kid> positional arg\n${USAGE}\n`);
|
|
2298
|
+
return 1;
|
|
2299
|
+
}
|
|
2300
|
+
if (direct && url !== void 0) throw argvError("revoke", "revoke: --direct and --url are mutually exclusive", 1);
|
|
2301
|
+
const target = resolveAdminTarget(direct ? "" : url ?? null, env);
|
|
2302
|
+
errOut(`${describeAdminTarget("revoke", target)}\n`);
|
|
2303
|
+
if (target.kind === "route") {
|
|
2304
|
+
const payload = await runKeyPatchViaRoute({
|
|
2305
|
+
kid,
|
|
2306
|
+
revoke: true
|
|
2307
|
+
}, target.baseUrl, env, REVOKE_EXIT_CODES);
|
|
2308
|
+
out(`Key ${String(payload.kid)} revoked at ${String(payload.revokedAt)} — the running server saw it.
|
|
2309
|
+
`);
|
|
2310
|
+
return 0;
|
|
2311
|
+
}
|
|
2312
|
+
const result = await runRevoke(kid, env, openDataset);
|
|
2313
|
+
out(formatRevokeOutput(result));
|
|
2314
|
+
return 0;
|
|
2315
|
+
}
|
|
2316
|
+
case "rotate": {
|
|
2317
|
+
const parsed = parseRotateArgs(argv);
|
|
2318
|
+
const result = await runRotate(parsed.kid, parsed.outPath, env, parsed.wantsEncrypt, openDataset);
|
|
2319
|
+
out(formatRotateOutput(result));
|
|
2320
|
+
return 0;
|
|
2321
|
+
}
|
|
2322
|
+
case "register": {
|
|
2323
|
+
const parsed = parseRegisterArgs(argv);
|
|
2324
|
+
const target = resolveAdminTarget(parsed.url, env);
|
|
2325
|
+
errOut(`${describeAdminTarget("register", target)}\n`);
|
|
2326
|
+
const result = target.kind === "route" ? await runRegisterViaRoute(parsed, target.baseUrl, env) : await runRegister(parsed, env, openDataset);
|
|
2327
|
+
out(formatRegisterOutput(result));
|
|
2328
|
+
return 0;
|
|
2329
|
+
}
|
|
2330
|
+
case "update": {
|
|
2331
|
+
const parsed = parseUpdateArgs(argv);
|
|
2332
|
+
const target = resolveAdminTarget(parsed.url, env);
|
|
2333
|
+
errOut(`${describeAdminTarget("update", target)}\n`);
|
|
2334
|
+
if (target.kind === "route") {
|
|
2335
|
+
const payload = await runKeyPatchViaRoute({
|
|
2336
|
+
kid: parsed.kid,
|
|
2337
|
+
...parsed.name !== void 0 ? { name: parsed.name } : {},
|
|
2338
|
+
...parsed.namespaces !== void 0 ? { namespaces: parsed.namespaces } : {},
|
|
2339
|
+
...parsed.capabilities !== void 0 ? { capabilities: parsed.capabilities } : {},
|
|
2340
|
+
widen: parsed.widen === true,
|
|
2341
|
+
allowAdmin: parsed.allowAdmin === true,
|
|
2342
|
+
allowUnknownNs: parsed.allowUnknownNs
|
|
2343
|
+
}, target.baseUrl, env, UPDATE_EXIT_CODES);
|
|
2344
|
+
out(`Key ${String(payload.kid)} updated via the route — the running server saw it.\n`);
|
|
2345
|
+
return 0;
|
|
2346
|
+
}
|
|
2347
|
+
const result = await runUpdate(parsed, env, openDataset);
|
|
2348
|
+
out(formatUpdateOutput(result));
|
|
2349
|
+
return 0;
|
|
2350
|
+
}
|
|
2351
|
+
case "migrate-store": return migrateStoreCommand(argv.slice(1), env, out, errOut);
|
|
2352
|
+
case "promote-fields": {
|
|
2353
|
+
const result = await promoteFieldsCommand(argv.slice(1), env);
|
|
2354
|
+
out(formatPromoteFieldsOutput(result));
|
|
2355
|
+
return result.events.some((e) => e.kind === "failed") ? 1 : 0;
|
|
2356
|
+
}
|
|
2357
|
+
default:
|
|
2358
|
+
errOut(`unknown command: ${command}\n${USAGE}\n`);
|
|
2359
|
+
return 1;
|
|
2360
|
+
}
|
|
2361
|
+
} catch (err) {
|
|
2362
|
+
const exitCode = err instanceof BootstrapError || err instanceof AdminCliError ? err.exitCode : 1;
|
|
2363
|
+
errOut(`${err instanceof Error ? err.message : String(err)}\n`);
|
|
2364
|
+
return exitCode;
|
|
2365
|
+
}
|
|
2366
|
+
}
|
|
2367
|
+
|
|
2368
|
+
//#endregion
|
|
2369
|
+
//#region src/admin/stdin-ask.ts
|
|
2370
|
+
/**
|
|
2371
|
+
* The line reader behind an interactive `substrate-admin onboard`.
|
|
2372
|
+
*
|
|
2373
|
+
* - End of input (Ctrl-D, a closed pipe) resolves the open question, and every later
|
|
2374
|
+
* one, with null — the loop turns that into a quit with its summary.
|
|
2375
|
+
* - Ctrl-C at a prompt does the same. Ctrl-C with no prompt open (an action or a fetch in
|
|
2376
|
+
* flight) calls `onInterrupt`, because no prompt will come back to notice a close.
|
|
2377
|
+
* - `terminal` is set from the INPUT, so redirecting stderr cannot turn Ctrl-C back into
|
|
2378
|
+
* a process-killing SIGINT.
|
|
2379
|
+
* - One pending resolver and one permanent close listener, so nothing accumulates per
|
|
2380
|
+
* question.
|
|
2381
|
+
*/
|
|
2382
|
+
function createAsk(streams) {
|
|
2383
|
+
const rl = createInterface({
|
|
2384
|
+
input: streams.input,
|
|
2385
|
+
output: streams.output,
|
|
2386
|
+
terminal: streams.terminal
|
|
2387
|
+
});
|
|
2388
|
+
let closed = false;
|
|
2389
|
+
let pending = null;
|
|
2390
|
+
const settle = (line) => {
|
|
2391
|
+
const resolve = pending;
|
|
2392
|
+
pending = null;
|
|
2393
|
+
resolve?.(line);
|
|
2394
|
+
};
|
|
2395
|
+
rl.on("close", () => {
|
|
2396
|
+
closed = true;
|
|
2397
|
+
settle(null);
|
|
2398
|
+
});
|
|
2399
|
+
rl.on("SIGINT", () => {
|
|
2400
|
+
if (pending !== null) rl.close();
|
|
2401
|
+
else streams.onInterrupt();
|
|
2402
|
+
});
|
|
2403
|
+
return (prompt) => {
|
|
2404
|
+
if (closed) return Promise.resolve(null);
|
|
2405
|
+
return new Promise((resolve) => {
|
|
2406
|
+
pending = resolve;
|
|
2407
|
+
rl.question(prompt, (line) => settle(line));
|
|
2408
|
+
});
|
|
2409
|
+
};
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2412
|
+
//#endregion
|
|
2413
|
+
//#region src/admin/run.ts
|
|
2414
|
+
/** Runs `substrate-admin` on this process's argv, then exits with its code. */
|
|
2415
|
+
function runAdminCli() {
|
|
2416
|
+
const interactive = process.stdin.isTTY === true && process.argv[2] === "onboard";
|
|
2417
|
+
main(process.argv.slice(2), process.env, (s) => process.stdout.write(s), (s) => process.stderr.write(s), openPgDataset, interactive ? {
|
|
2418
|
+
ask: createAsk({
|
|
2419
|
+
input: process.stdin,
|
|
2420
|
+
output: process.stderr,
|
|
2421
|
+
terminal: true,
|
|
2422
|
+
onInterrupt: () => {
|
|
2423
|
+
process.stderr.write("\ninterrupted.\n");
|
|
2424
|
+
process.exit(130);
|
|
2425
|
+
}
|
|
2426
|
+
}),
|
|
2427
|
+
interactive: true
|
|
2428
|
+
} : {
|
|
2429
|
+
ask: async () => null,
|
|
2430
|
+
interactive: false
|
|
2431
|
+
}).then((exitCode) => process.exit(exitCode));
|
|
2432
|
+
}
|
|
2433
|
+
|
|
2434
|
+
//#endregion
|
|
2435
|
+
export { runAdminCli };
|