@mnmal-ai/substrate 1.0.0-alpha.11 → 1.0.0-alpha.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -2
- package/dist/{boot-store-C5vNZJ3i.js → boot-store-DHv9H1Gr.js} +165 -69
- package/dist/{bridge-COu0Tg_h.d.ts → bridge-DQyQ6Qhn.d.ts} +1 -1
- package/dist/{cli-BrwW4tjO.js → cli-_UNdneXr.js} +11 -9
- package/dist/client.d.ts +3 -3
- package/dist/client.js +3 -3
- package/dist/{errors-BOllGhVx.d.ts → errors-xP9Mo7Q1.d.ts} +25 -4
- package/dist/{index-Cp74OcmS.d.ts → index-baF5A5jU.d.ts} +1 -1
- package/dist/pack.d.ts +2 -2
- package/dist/pack.js +1 -1
- package/dist/{pg-edges-store-BLsbcXkk.js → pg-edges-store-B4FSN9yp.js} +72 -9
- package/dist/{plugin-DIYm2cZE.js → plugin-BLm5IRdX.js} +2 -2
- package/dist/proposals-bootstrap.js +2 -2
- package/dist/proposals.d.ts +2 -2
- package/dist/proposals.js +1 -1
- package/dist/recall-bootstrap.js +2 -2
- package/dist/recall.d.ts +10 -10
- package/dist/recall.js +2 -2
- package/dist/{repl-B8gy0YM7.js → repl-DUJGBwie.js} +25 -25
- package/dist/{row-DwaDU4b-.d.ts → row-Cu_iRhcR.d.ts} +2 -2
- package/dist/{run-B18qtGKc.js → run-BpsTNvdI.js} +520 -141
- package/dist/{schema-CijHRfuk.d.ts → schema-oXfQyo0s.d.ts} +272 -27
- package/dist/sdk.d.ts +93 -29
- package/dist/sdk.js +4 -4
- package/dist/{src-D041j6GS.js → src-Bew_JvTo.js} +32 -16
- package/dist/{src-DpPdw2oL.js → src-Bi3NhEsT.js} +1 -1
- package/dist/{src-DK7VEQT6.js → src-C4t1Gy-m.js} +2477 -830
- package/dist/{src-Br6aDUti.js → src-CHND7-Pm.js} +2 -2
- package/dist/src-CSe2tJDf.js +6940 -0
- package/dist/{src-DptNGNYQ.js → src-CzhKLsOV.js} +175 -91
- package/dist/{src-Dbvuel6w.js → src-DWqzrm3O.js} +18 -1
- package/dist/substrate-admin.js +1 -1
- package/dist/substrate.js +1 -1
- package/dist/testing.d.ts +233 -92
- package/dist/testing.js +6 -23
- package/dist/{ws-DC8OhMaV.d.ts → ws-Dx3A2B4U.d.ts} +2 -2
- package/package.json +19 -20
- package/dist/src-gz4I1F29.js +0 -5097
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
|
|
2
|
-
import {
|
|
3
|
-
import {
|
|
4
|
-
import {
|
|
5
|
-
import
|
|
2
|
+
import { _ as AUDIT_EVENT_TYPE, g as buildAuditRow } from "./src-C4t1Gy-m.js";
|
|
3
|
+
import { n as trimTrailingSlashes } from "./env-utils-CGNazUNt.js";
|
|
4
|
+
import { I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./src-Bi3NhEsT.js";
|
|
5
|
+
import "./src-DWqzrm3O.js";
|
|
6
|
+
import { A as UPDATE_EXIT_CODES, B as readKeyById, C as grantLines, D as parseBootstrapArgs, E as formatBootstrapOutput, F as CLI_ACTOR_KID, G as validateArgv, H as resolveAdminAuth, I as base64Url, J as argvError, K as RESTART_NOTE, L as forUpdate, M as runKeyPatchViaRoute, N as ACTOR_HINT, O as runBootstrap, P as BootstrapError, R as generateAdminKeypair, S as runRegisterViaRoute, T as formatDoctorReport, U as resolveAdminTarget, V as describeAdminTarget, W as ARGV_SPEC, X as pgNoticeOptions, Y as AdminCliError, _ as openPgDataset, b as parseRegisterArgs, c as refuseContractViolation, d as registeredTypes, g as version, h as loadPluginsFromDir, j as runKeyListViaRoute, k as REVOKE_EXIT_CODES, l as buildDeferredIndexes, n as resolvePluginDir, q as USAGE, r as BINARY_CONTRACT_VERSION, s as parseEnv, t as bootStore, v as onboardCommand, w as doctorCommand, x as runRegister, y as formatRegisterOutput, z as listActiveAdminKeys } from "./boot-store-DHv9H1Gr.js";
|
|
7
|
+
import { C as parseSourceBody, S as copyRows, b as UUID_REGEX, f as assertStoreSchema, h as isDerivedMigration, i as ensureTypeTable, l as MIGRATE_STORE_MARKER_TABLE, o as loadTypeTable, r as seqName, s as typeTableRef, u as PLAIN_SCHEMA_NAME, x as detectLayout, y as UUID_CHECK_MIGRATION } from "./src-CSe2tJDf.js";
|
|
6
8
|
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
|
|
7
9
|
import { dirname } from "node:path";
|
|
8
10
|
import { createHash, randomBytes, randomUUID } from "node:crypto";
|
|
@@ -144,7 +146,7 @@ function formatRemoteListOutput(rows, baseUrl) {
|
|
|
144
146
|
|
|
145
147
|
//#endregion
|
|
146
148
|
//#region src/admin/migrate-store/progress.ts
|
|
147
|
-
/** Progress of a migrate-store copy, counted off the COPY text stream as it passes. */
|
|
149
|
+
/** Progress of a migrate-store copy, counted off the COPY text stream as it passes, or per converted batch. */
|
|
148
150
|
/**
|
|
149
151
|
* A pass-through that counts rows (one newline each: COPY text escapes the
|
|
150
152
|
* newlines inside values) and reports every `batchRows` rows. A throw from
|
|
@@ -170,19 +172,27 @@ var RowCounter = class extends Transform {
|
|
|
170
172
|
this.bytes = from.bytes;
|
|
171
173
|
this.#next = from.rows - from.rows % batchRows + batchRows;
|
|
172
174
|
}
|
|
175
|
+
/** Counts `rows` converted rows of `bytes` body text, reporting as `_transform` does. */
|
|
176
|
+
add(rows, bytes) {
|
|
177
|
+
this.rows += rows;
|
|
178
|
+
this.bytes += bytes;
|
|
179
|
+
this.#report();
|
|
180
|
+
}
|
|
181
|
+
#report() {
|
|
182
|
+
if (this.rows < this.#next) return;
|
|
183
|
+
this.#next = this.rows - this.rows % this.batchRows + this.batchRows;
|
|
184
|
+
this.onProgress({
|
|
185
|
+
table: this.table,
|
|
186
|
+
rows: this.rows,
|
|
187
|
+
bytes: this.bytes,
|
|
188
|
+
done: false
|
|
189
|
+
});
|
|
190
|
+
}
|
|
173
191
|
_transform(chunk, _enc, cb) {
|
|
174
192
|
try {
|
|
175
193
|
this.bytes += chunk.length;
|
|
176
194
|
for (let i = chunk.indexOf(10); i !== -1; i = chunk.indexOf(10, i + 1)) this.rows++;
|
|
177
|
-
|
|
178
|
-
this.#next = this.rows - this.rows % this.batchRows + this.batchRows;
|
|
179
|
-
this.onProgress({
|
|
180
|
-
table: this.table,
|
|
181
|
-
rows: this.rows,
|
|
182
|
-
bytes: this.bytes,
|
|
183
|
-
done: false
|
|
184
|
-
});
|
|
185
|
-
}
|
|
195
|
+
this.#report();
|
|
186
196
|
this.push(chunk);
|
|
187
197
|
cb(null);
|
|
188
198
|
} catch (err) {
|
|
@@ -194,21 +204,30 @@ var RowCounter = class extends Transform {
|
|
|
194
204
|
//#endregion
|
|
195
205
|
//#region src/admin/migrate-store/copy.ts
|
|
196
206
|
/**
|
|
197
|
-
* The migrate-store copy:
|
|
198
|
-
*
|
|
199
|
-
*
|
|
200
|
-
*
|
|
207
|
+
* The migrate-store copy: converts a legacy or partitioned source into a
|
|
208
|
+
* target `bootStore` prepared in the `tables` layout, all from one REPEATABLE
|
|
209
|
+
* READ, READ ONLY source snapshot. `rows` is converted: each source type is
|
|
210
|
+
* streamed with a cursor, in `batchRows` batches, into its type table by
|
|
211
|
+
* `copyRows` (columns plus `extra`, ids into `row_ids`). Every other table
|
|
212
|
+
* moves with `COPY … TO STDOUT` piped into `COPY … FROM STDIN`. Nothing here
|
|
213
|
+
* writes to the source.
|
|
201
214
|
*
|
|
202
215
|
* Two phases around a re-boot, on one source snapshot. Phase 1 copies the
|
|
203
216
|
* rows (and their shortIds) that declare storage: `_system`,
|
|
204
217
|
* `_system_proposals` and every TypeProposal and FieldProposal. The second
|
|
205
|
-
* boot replays those proposals as serve does, so every proposed type
|
|
206
|
-
*
|
|
218
|
+
* boot replays those proposals as serve does, so every proposed type is
|
|
219
|
+
* registered, with its table and declared fields, before phase 2 copies
|
|
220
|
+
* everything else. A source type nothing declares gets a table with no
|
|
221
|
+
* columns, so its fields all go to `extra`, and is reported `defaulted`.
|
|
222
|
+
* Last, one promotion sweep moves the values of fields declared only by the
|
|
223
|
+
* replay (phase 1 converted them into `extra`) into their columns.
|
|
207
224
|
*
|
|
208
225
|
* Progress lives in the marker table on the target. A rerun after an
|
|
209
226
|
* interruption restarts cleanly: it empties the planned tables, keeps the
|
|
210
227
|
* seed rows the first run recorded, and copies again.
|
|
211
228
|
*/
|
|
229
|
+
/** The misfits listed per type, beside the full count. */
|
|
230
|
+
const MISFITS_LISTED = 100;
|
|
212
231
|
/** A source table or column with nowhere to go on the target. Thrown before any data moves. */
|
|
213
232
|
var MigrateStoreColumnError = class extends Error {
|
|
214
233
|
table;
|
|
@@ -225,6 +244,8 @@ const SEED = "seed:";
|
|
|
225
244
|
/** The `rows` / `short_ids` predicate phase 1 copies; phase 2 copies its complement. Both columns are NOT NULL. */
|
|
226
245
|
const PHASE1_ROWS = "namespace IN ('_system', '_system_proposals') OR type IN ('TypeProposal', 'FieldProposal')";
|
|
227
246
|
const PHASED = /* @__PURE__ */ new Set(["rows", "short_ids"]);
|
|
247
|
+
/** Converted rather than copied: the target has no `rows`. */
|
|
248
|
+
const CONVERTED = "rows";
|
|
228
249
|
async function copyStore(source, boot, plan, opts) {
|
|
229
250
|
let target = await boot();
|
|
230
251
|
let open = target;
|
|
@@ -241,11 +262,16 @@ async function copyStore(source, boot, plan, opts) {
|
|
|
241
262
|
}
|
|
242
263
|
const src = await source.reserve();
|
|
243
264
|
let streaming = false;
|
|
265
|
+
let snapshotOpen = true;
|
|
244
266
|
try {
|
|
245
267
|
await src.unsafe("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY");
|
|
268
|
+
await src.unsafe("SET LOCAL DateStyle = 'ISO, YMD'");
|
|
269
|
+
await src.unsafe("SET LOCAL TimeZone = 'UTC'");
|
|
246
270
|
const keptSeeds = await reconcileSeeds(src, tsql, target);
|
|
271
|
+
const rowsPlan = plan.tables.find((t) => t.name === CONVERTED);
|
|
272
|
+
const misfits = {};
|
|
247
273
|
const totals = /* @__PURE__ */ new Map();
|
|
248
|
-
const copy = async (t, filter) => {
|
|
274
|
+
const copy = async (t, filter, tables) => {
|
|
249
275
|
const start = performance.now();
|
|
250
276
|
const prev = totals.get(t.name) ?? {
|
|
251
277
|
rows: 0,
|
|
@@ -254,7 +280,8 @@ async function copyStore(source, boot, plan, opts) {
|
|
|
254
280
|
};
|
|
255
281
|
const counter = new RowCounter(t.name, opts.batchRows, opts.onProgress, prev);
|
|
256
282
|
streaming = true;
|
|
257
|
-
await
|
|
283
|
+
if (t.name === CONVERTED) await convertRows(src, tsql, t, filter, tables, counter, opts.batchRows, misfits);
|
|
284
|
+
else await copyTable(src, tsql, t, identity.get(t.name) ?? [], counter, filter);
|
|
258
285
|
streaming = false;
|
|
259
286
|
totals.set(t.name, {
|
|
260
287
|
rows: counter.rows,
|
|
@@ -262,7 +289,8 @@ async function copyStore(source, boot, plan, opts) {
|
|
|
262
289
|
ms: prev.ms + Math.round(performance.now() - start)
|
|
263
290
|
});
|
|
264
291
|
};
|
|
265
|
-
|
|
292
|
+
const phase1 = await prepareTables(src, target, PHASE1_ROWS, false);
|
|
293
|
+
for (const t of plan.tables.filter((p) => PHASED.has(p.name))) await copy(t, PHASE1_ROWS, phase1.tables);
|
|
266
294
|
await markStep(tsql, "copy:phase1");
|
|
267
295
|
await snapshotRows(tsql);
|
|
268
296
|
open = void 0;
|
|
@@ -271,10 +299,10 @@ async function copyStore(source, boot, plan, opts) {
|
|
|
271
299
|
open = target;
|
|
272
300
|
tsql = sqlOf(target);
|
|
273
301
|
await assertRebootWroteNothing(tsql);
|
|
274
|
-
const
|
|
302
|
+
const phase2 = await prepareTables(src, target, null, true);
|
|
275
303
|
const tables = [];
|
|
276
304
|
for (const t of plan.tables) {
|
|
277
|
-
await copy(t, PHASED.has(t.name) ? `NOT (${PHASE1_ROWS})` : null);
|
|
305
|
+
await copy(t, PHASED.has(t.name) ? `NOT (${PHASE1_ROWS})` : null, phase2.tables);
|
|
278
306
|
await markStep(tsql, `copy:${t.name}`);
|
|
279
307
|
const done = totals.get(t.name);
|
|
280
308
|
tables.push({
|
|
@@ -290,16 +318,20 @@ async function copyStore(source, boot, plan, opts) {
|
|
|
290
318
|
}
|
|
291
319
|
await markStep(tsql, "copy:phase2");
|
|
292
320
|
await advanceSequences(tsql, identity);
|
|
293
|
-
await markStep(tsql, "copy:done");
|
|
294
321
|
await src.unsafe("COMMIT");
|
|
322
|
+
snapshotOpen = false;
|
|
295
323
|
src.release();
|
|
324
|
+
if (rowsPlan !== void 0) await settleFields(target, opts.batchRows);
|
|
325
|
+
await markStep(tsql, "copy:done");
|
|
296
326
|
return {
|
|
297
327
|
tables,
|
|
298
328
|
keptSeeds,
|
|
299
|
-
defaulted,
|
|
329
|
+
defaulted: phase2.defaulted,
|
|
330
|
+
misfits,
|
|
300
331
|
target
|
|
301
332
|
};
|
|
302
333
|
} catch (err) {
|
|
334
|
+
if (!snapshotOpen) throw err;
|
|
303
335
|
const released = src.unsafe("ROLLBACK").catch(() => void 0).then(() => src.release()).catch(() => void 0);
|
|
304
336
|
if (!streaming) await released;
|
|
305
337
|
throw err;
|
|
@@ -313,25 +345,46 @@ function sqlOf(target) {
|
|
|
313
345
|
if (target.sql === null) throw new Error("migrate-store copies into a postgres target only");
|
|
314
346
|
return target.sql;
|
|
315
347
|
}
|
|
348
|
+
const quoteIdent = (name) => `"${name.replace(/"/g, "\"\"")}"`;
|
|
349
|
+
/** The target's type tables, each with its (quoted) relation. */
|
|
350
|
+
async function typeTablesOf(tsql) {
|
|
351
|
+
return (await tsql`
|
|
352
|
+
SELECT namespace, type, table_name FROM substrate_type_tables ORDER BY namespace, type`).map((r) => ({
|
|
353
|
+
namespace: r.namespace,
|
|
354
|
+
type: r.type,
|
|
355
|
+
rel: quoteIdent(r.table_name)
|
|
356
|
+
}));
|
|
357
|
+
}
|
|
358
|
+
/**
|
|
359
|
+
* SQL over every target row: `(namespace, type, id, h)`, `h` an md5 of the
|
|
360
|
+
* stored row with its null columns dropped, so a column added by a re-boot
|
|
361
|
+
* changes no row's hash. `null` when there are no type tables.
|
|
362
|
+
*/
|
|
363
|
+
async function targetRowsSql(tsql) {
|
|
364
|
+
const tables = await typeTablesOf(tsql);
|
|
365
|
+
if (tables.length === 0) return null;
|
|
366
|
+
const lit = (s) => `'${s.replace(/'/g, "''")}'`;
|
|
367
|
+
return tables.map((t) => `SELECT ${lit(t.namespace)}::text AS namespace, ${lit(t.type)}::text AS type, x.id, md5(jsonb_strip_nulls(to_jsonb(x))::text) AS h FROM ${t.rel} x`).join(" UNION ALL ");
|
|
368
|
+
}
|
|
316
369
|
const SNAPSHOT = "substrate_migrate_store_reboot_rows";
|
|
317
370
|
/** Records every target row's key and content hash, for `assertRebootWroteNothing`. */
|
|
318
371
|
async function snapshotRows(tsql) {
|
|
319
372
|
await tsql`DROP TABLE IF EXISTS ${tsql(SNAPSHOT)}`;
|
|
320
|
-
|
|
321
|
-
|
|
373
|
+
const all = await targetRowsSql(tsql);
|
|
374
|
+
await tsql.unsafe(`CREATE UNLOGGED TABLE ${quoteIdent(SNAPSHOT)} AS ${all ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false"}`);
|
|
322
375
|
}
|
|
323
376
|
/** Refuses a re-boot that added, removed or changed a row, naming up to ten of each. */
|
|
324
377
|
async function assertRebootWroteNothing(tsql) {
|
|
325
|
-
const
|
|
326
|
-
const diff = await tsql`
|
|
378
|
+
const all = await targetRowsSql(tsql) ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false";
|
|
379
|
+
const diff = await tsql.unsafe(`
|
|
327
380
|
SELECT CASE WHEN s.id IS NULL THEN 'added' WHEN r.id IS NULL THEN 'removed' ELSE 'changed' END AS change,
|
|
328
381
|
coalesce(r.namespace, s.namespace) AS namespace, coalesce(r.type, s.type) AS type,
|
|
329
382
|
coalesce(r.id, s.id) AS id
|
|
330
|
-
FROM (
|
|
331
|
-
FULL JOIN ${
|
|
383
|
+
FROM (${all}) r
|
|
384
|
+
FULL JOIN ${quoteIdent(SNAPSHOT)} s USING (namespace, type, id)
|
|
332
385
|
WHERE s.id IS NULL OR r.id IS NULL OR r.h <> s.h
|
|
333
|
-
ORDER BY 1, 2, 3, 4 LIMIT 30
|
|
334
|
-
await tsql`DROP TABLE ${
|
|
386
|
+
ORDER BY 1, 2, 3, 4 LIMIT 30`);
|
|
387
|
+
await tsql`DROP TABLE ${tsql(SNAPSHOT)}`;
|
|
335
388
|
if (diff.length === 0) return;
|
|
336
389
|
const named = [
|
|
337
390
|
"added",
|
|
@@ -345,15 +398,16 @@ async function assertRebootWroteNothing(tsql) {
|
|
|
345
398
|
}
|
|
346
399
|
/** Refuses a plan column the target lacks; returns each table's identity-always columns among the planned ones. */
|
|
347
400
|
async function checkColumns(tsql, plan) {
|
|
401
|
+
const copied = plan.tables.filter((t) => t.name !== CONVERTED);
|
|
348
402
|
const rows = await tsql`
|
|
349
403
|
SELECT c.relname AS "table", a.attname AS col, a.attidentity AS ident
|
|
350
404
|
FROM pg_attribute a
|
|
351
405
|
JOIN pg_class c ON c.oid = a.attrelid
|
|
352
406
|
JOIN pg_namespace n ON n.oid = c.relnamespace
|
|
353
|
-
WHERE n.nspname = current_schema() AND c.relname = ANY(${
|
|
407
|
+
WHERE n.nspname = current_schema() AND c.relname = ANY(${copied.map((t) => t.name)}::text[])
|
|
354
408
|
AND c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = ''`;
|
|
355
409
|
const identity = /* @__PURE__ */ new Map();
|
|
356
|
-
for (const t of
|
|
410
|
+
for (const t of copied) {
|
|
357
411
|
const cols = rows.filter((r) => r.table === t.name);
|
|
358
412
|
if (cols.length === 0) throw new MigrateStoreColumnError(t.name, null);
|
|
359
413
|
const have = new Set(cols.map((c) => c.col));
|
|
@@ -374,6 +428,18 @@ function inKeys(sql, keys) {
|
|
|
374
428
|
return sql`(namespace, type, id) IN (SELECT * FROM unnest(
|
|
375
429
|
${keys.map((k) => k.namespace)}::text[], ${keys.map((k) => k.type)}::text[], ${keys.map((k) => k.id)}::text[]))`;
|
|
376
430
|
}
|
|
431
|
+
/**
|
|
432
|
+
* Deletes target rows: those in `keys`, or (`except`) all but those. From each
|
|
433
|
+
* type table and from `row_ids`, whose delete cascades to `short_ids`.
|
|
434
|
+
*/
|
|
435
|
+
async function deleteTargetRows(t, keys, except) {
|
|
436
|
+
for (const table of await typeTablesOf(t)) {
|
|
437
|
+
const ids = keys.filter((k) => k.namespace === table.namespace && k.type === table.type).map((k) => k.id);
|
|
438
|
+
if (!except && ids.length === 0) continue;
|
|
439
|
+
await t.unsafe(`DELETE FROM ${table.rel} WHERE ${except ? "NOT " : ""}(id = ANY($1::text[]))`, [ids]);
|
|
440
|
+
}
|
|
441
|
+
await t`DELETE FROM row_ids WHERE ${except ? t`NOT` : t``} ${inKeys(t, keys)}`;
|
|
442
|
+
}
|
|
377
443
|
/** After an interrupted run: empties every planned table but the seed rows that run recorded. True when it did. */
|
|
378
444
|
async function restartIfStarted(tsql, plan) {
|
|
379
445
|
const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
|
|
@@ -387,10 +453,10 @@ async function restartIfStarted(tsql, plan) {
|
|
|
387
453
|
id
|
|
388
454
|
};
|
|
389
455
|
});
|
|
390
|
-
const others = plan.tables.map((p) => p.name).filter((n) => n
|
|
456
|
+
const others = plan.tables.map((p) => p.name).filter((n) => !PHASED.has(n));
|
|
391
457
|
await tsql.begin(async (tx) => {
|
|
392
458
|
const t = tx;
|
|
393
|
-
await
|
|
459
|
+
await deleteTargetRows(t, seeds, true);
|
|
394
460
|
if (others.length > 0) await t`TRUNCATE ${t(others)}`;
|
|
395
461
|
});
|
|
396
462
|
return true;
|
|
@@ -401,7 +467,7 @@ async function restartIfStarted(tsql, plan) {
|
|
|
401
467
|
* a type the target registers, as everything the prepare step writes is.
|
|
402
468
|
*/
|
|
403
469
|
async function reconcileSeeds(src, tsql, target) {
|
|
404
|
-
const present = await tsql`SELECT namespace, type, id FROM
|
|
470
|
+
const present = await tsql`SELECT namespace, type, id FROM row_ids ORDER BY namespace, type, id`;
|
|
405
471
|
const inSource = present.length === 0 ? [] : await src`SELECT namespace, type, id FROM rows WHERE ${inKeys(src, [...present])}`;
|
|
406
472
|
const replaced = new Set(inSource.map(keyOf));
|
|
407
473
|
const kept = present.filter((r) => !replaced.has(keyOf(r))).map(({ namespace, type, id }) => ({
|
|
@@ -413,7 +479,11 @@ async function reconcileSeeds(src, tsql, target) {
|
|
|
413
479
|
const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
|
|
414
480
|
await tsql.begin(async (tx) => {
|
|
415
481
|
const t = tx;
|
|
416
|
-
if (inSource.length > 0) await t
|
|
482
|
+
if (inSource.length > 0) await deleteTargetRows(t, inSource.map(({ namespace, type, id }) => ({
|
|
483
|
+
namespace,
|
|
484
|
+
type,
|
|
485
|
+
id
|
|
486
|
+
})), false);
|
|
417
487
|
await t`DELETE FROM ${marker} WHERE starts_with(step, ${SEED}) OR starts_with(step, 'copy:')`;
|
|
418
488
|
const steps = [STARTED, ...kept.map((k) => SEED + keyOf(k))];
|
|
419
489
|
await t`INSERT INTO ${marker} ${t(steps.map((step) => ({
|
|
@@ -428,26 +498,109 @@ function registeredDataset(target, ns, type) {
|
|
|
428
498
|
if (target.engine.registryFor(ns)?.get(type)?.kind !== "type") return void 0;
|
|
429
499
|
return target.datasets.get(ns);
|
|
430
500
|
}
|
|
431
|
-
/**
|
|
432
|
-
|
|
433
|
-
|
|
501
|
+
/**
|
|
502
|
+
* Gives every source type `filter` picks its table and returns their
|
|
503
|
+
* descriptors: a registered type's through its dataset, with its declared
|
|
504
|
+
* fields and unique indexes; any other's with no field columns (its fields
|
|
505
|
+
* all go to `extra`), listed in `defaulted` when `report` is set.
|
|
506
|
+
*/
|
|
507
|
+
async function prepareTables(src, target, filter, report) {
|
|
508
|
+
const tsql = sqlOf(target);
|
|
509
|
+
const pairs = await src`SELECT DISTINCT namespace, type FROM rows ${filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`} ORDER BY namespace, type`;
|
|
510
|
+
const tables = /* @__PURE__ */ new Map();
|
|
434
511
|
const defaulted = [];
|
|
512
|
+
const [uuid] = await tsql`
|
|
513
|
+
SELECT EXISTS (SELECT 1 FROM migrations WHERE version = ${UUID_CHECK_MIGRATION}) AS on`;
|
|
435
514
|
for (const { namespace, type } of pairs) {
|
|
515
|
+
const key = `${namespace}/${type}`;
|
|
436
516
|
const dataset = registeredDataset(target, namespace, type);
|
|
437
|
-
if (dataset
|
|
438
|
-
|
|
517
|
+
if (dataset === void 0 && report) defaulted.push(key);
|
|
518
|
+
try {
|
|
519
|
+
if (dataset !== void 0) await dataset.ensureTypeStorage?.(type, { indexes: "unique" });
|
|
520
|
+
else await ensureTypeTable(tsql, {
|
|
439
521
|
namespace,
|
|
440
|
-
type
|
|
522
|
+
type,
|
|
523
|
+
fields: {},
|
|
524
|
+
newFields: "column",
|
|
525
|
+
enforceUuidCheck: uuid?.on === true,
|
|
526
|
+
createOnly: true
|
|
441
527
|
});
|
|
442
|
-
continue;
|
|
443
|
-
}
|
|
444
|
-
try {
|
|
445
|
-
await dataset.ensureTypeStorage(type, { indexes: "unique" });
|
|
446
528
|
} catch (err) {
|
|
447
|
-
throw new Error(`preparing storage for ${
|
|
529
|
+
throw new Error(`preparing storage for ${key} failed: ${err.message}`, { cause: err });
|
|
448
530
|
}
|
|
531
|
+
const table = await loadTypeTable(tsql, namespace, type);
|
|
532
|
+
if (table === void 0) throw new Error(`preparing storage for ${key} made no table`);
|
|
533
|
+
tables.set(key, table);
|
|
449
534
|
}
|
|
450
|
-
return
|
|
535
|
+
return {
|
|
536
|
+
tables,
|
|
537
|
+
defaulted
|
|
538
|
+
};
|
|
539
|
+
}
|
|
540
|
+
/**
|
|
541
|
+
* Converts the source rows `filter` picks, type by type, each streamed with a
|
|
542
|
+
* cursor in `batchRows` batches into its type table. A row the table already
|
|
543
|
+
* holds, or a refusal (a declared unique index), fails the copy loudly.
|
|
544
|
+
*/
|
|
545
|
+
async function convertRows(src, tsql, t, filter, tables, counter, batchRows, misfits) {
|
|
546
|
+
const where = filter === null ? src`` : src`AND (${src.unsafe(filter)})`;
|
|
547
|
+
const seq = t.columns.includes("seq") ? src`seq::text` : src`NULL::text`;
|
|
548
|
+
const pairs = await src`
|
|
549
|
+
SELECT DISTINCT namespace, type FROM rows WHERE true ${where} ORDER BY namespace, type`;
|
|
550
|
+
for (const { namespace, type } of pairs) {
|
|
551
|
+
const key = `${namespace}/${type}`;
|
|
552
|
+
const table = tables.get(key);
|
|
553
|
+
if (table === void 0) throw new Error(`internal: no table prepared for ${key}`);
|
|
554
|
+
const cursor = src`
|
|
555
|
+
SELECT id, body::text AS body, deleted_at::text, created_at::text, updated_at::text,
|
|
556
|
+
created_by_kind, created_by_id, updated_by_kind, updated_by_id,
|
|
557
|
+
${seq} AS seq
|
|
558
|
+
FROM rows WHERE namespace = ${namespace} AND type = ${type} ${where}
|
|
559
|
+
ORDER BY id`.cursor(batchRows);
|
|
560
|
+
for await (const batch of cursor) {
|
|
561
|
+
let result;
|
|
562
|
+
try {
|
|
563
|
+
result = await copyRows(tsql, table, batch.map((r) => ({
|
|
564
|
+
...r,
|
|
565
|
+
body: parseSourceBody(r.body)
|
|
566
|
+
})));
|
|
567
|
+
} catch (err) {
|
|
568
|
+
throw rowsError(key, err);
|
|
569
|
+
}
|
|
570
|
+
if (result.inserted !== batch.length) throw new Error(`copying rows failed: ${key}: ${batch.length - result.inserted} source row(s) were already on the target`);
|
|
571
|
+
if (result.misfits.length > 0) {
|
|
572
|
+
const listed = misfits[key] ?? {
|
|
573
|
+
count: 0,
|
|
574
|
+
first: []
|
|
575
|
+
};
|
|
576
|
+
listed.count += result.misfits.length;
|
|
577
|
+
listed.first.push(...result.misfits.slice(0, 100 - listed.first.length));
|
|
578
|
+
misfits[key] = listed;
|
|
579
|
+
}
|
|
580
|
+
counter.add(batch.length, batch.reduce((n, r) => n + Buffer.byteLength(r.body), 0));
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
}
|
|
584
|
+
/** A conversion failure as one line naming the type; a unique refusal names the declared index and the key. */
|
|
585
|
+
function rowsError(key, err) {
|
|
586
|
+
const pg = err;
|
|
587
|
+
if (pg?.code === "23505" && typeof pg.constraint_name === "string") return new Error(`copying rows failed: ${key}: unique index ${pg.constraint_name} refuses a source row whose key another row already holds${typeof pg.detail === "string" ? ` (${pg.detail})` : ""}. Resolve the duplicate in the source, then rerun migrate-store, which restarts cleanly.`, { cause: err });
|
|
588
|
+
const detail = typeof pg?.detail === "string" ? ` (${pg.detail})` : "";
|
|
589
|
+
return new Error(`copying rows failed: ${key}: ${err.message ?? String(err)}${detail}`, { cause: err });
|
|
590
|
+
}
|
|
591
|
+
/**
|
|
592
|
+
* One promotion sweep over the target: a field declared only once the re-boot
|
|
593
|
+
* replayed its proposal had its phase-1 values converted into `extra`, so it
|
|
594
|
+
* is `promoting` (or `pending`) until moved into its column. Fails loudly,
|
|
595
|
+
* naming each field, when one is not promoted.
|
|
596
|
+
*/
|
|
597
|
+
async function settleFields(target, batchRows) {
|
|
598
|
+
const stuck = (await target.dataset.promotePendingFields?.({
|
|
599
|
+
lockTimeoutMs: 6e4,
|
|
600
|
+
batchRows,
|
|
601
|
+
noWriters: true
|
|
602
|
+
}) ?? []).filter((e) => e.kind !== "promoted");
|
|
603
|
+
if (stuck.length > 0) throw new Error(`settling declared fields failed: ${stuck.map((e) => `${e.namespace}/${e.type}.${e.field} ${e.kind}${e.error ? ` (${e.error})` : ""}`).join("; ")}. Rerun migrate-store, which restarts cleanly.`);
|
|
451
604
|
}
|
|
452
605
|
async function copyTable(src, tsql, t, identityCols, counter, filter) {
|
|
453
606
|
const where = filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`;
|
|
@@ -533,6 +686,7 @@ async function inspectSource(sql) {
|
|
|
533
686
|
await assertStoreSchema(tx);
|
|
534
687
|
const layout = await detectLayout(tx);
|
|
535
688
|
if (layout === "absent") throw new Error("the source schema holds no rows table");
|
|
689
|
+
if (layout === "tables") throw new Error("the source is already in the tables layout; migrate-store copies a legacy or partitioned store");
|
|
536
690
|
const [version] = await tx`SELECT current_setting('server_version_num')::int AS v`;
|
|
537
691
|
const [ledger] = await tx`SELECT to_regclass('migrations') IS NOT NULL AS present`;
|
|
538
692
|
const appliedMigrations = ledger?.present ? (await tx`SELECT version FROM migrations ORDER BY version`).map((r) => r.version) : [];
|
|
@@ -583,9 +737,15 @@ async function inspectSource(sql) {
|
|
|
583
737
|
SELECT a.atttypmod AS dims FROM pg_attribute a
|
|
584
738
|
WHERE a.attrelid = to_regclass('embeddings') AND a.attname = 'embedding' AND NOT a.attisdropped` : [];
|
|
585
739
|
const extensions = await tx`SELECT extname, extversion FROM pg_extension ORDER BY extname`;
|
|
586
|
-
const
|
|
587
|
-
SELECT
|
|
588
|
-
|
|
740
|
+
const sessions = await tx`
|
|
741
|
+
SELECT pid, extract(epoch FROM backend_start)::text AS start, usename,
|
|
742
|
+
application_name AS "applicationName", client_addr::text AS "clientAddr",
|
|
743
|
+
backend_type AS "backendType", state
|
|
744
|
+
FROM pg_stat_activity
|
|
745
|
+
WHERE datname = current_database() AND pid <> pg_backend_pid()
|
|
746
|
+
-- Another role's session shows a NULL backend_type to a non-monitoring role.
|
|
747
|
+
AND (backend_type = 'client backend' OR (backend_type IS NULL AND usesysid IS NOT NULL))
|
|
748
|
+
ORDER BY pid`;
|
|
589
749
|
return {
|
|
590
750
|
serverVersionNum: version?.v ?? 0,
|
|
591
751
|
layout,
|
|
@@ -604,7 +764,8 @@ async function inspectSource(sql) {
|
|
|
604
764
|
nonUuidIds,
|
|
605
765
|
vectorDims: dims !== void 0 && dims.dims > 0 ? dims.dims : null,
|
|
606
766
|
extensions: Object.fromEntries(extensions.map((e) => [e.extname, e.extversion])),
|
|
607
|
-
otherSessions:
|
|
767
|
+
otherSessions: sessions.length,
|
|
768
|
+
sessions
|
|
608
769
|
};
|
|
609
770
|
});
|
|
610
771
|
}
|
|
@@ -612,6 +773,19 @@ async function inspectSource(sql) {
|
|
|
612
773
|
//#endregion
|
|
613
774
|
//#region src/admin/migrate-store/preflight.ts
|
|
614
775
|
/** Pure preflight: turns a source inventory plus target facts into refusals, warnings and a copy plan. */
|
|
776
|
+
/** Each old layout's baseline ledger version, as the last release that booted it recorded it. */
|
|
777
|
+
const OLD_BASELINES = {
|
|
778
|
+
legacy: "0001_baseline",
|
|
779
|
+
partitioned: "0001p_partitioned_baseline"
|
|
780
|
+
};
|
|
781
|
+
/** Every static ledger version an old-layout store can hold. */
|
|
782
|
+
const OLD_MIGRATIONS = /* @__PURE__ */ new Set([
|
|
783
|
+
...Object.values(OLD_BASELINES),
|
|
784
|
+
UUID_CHECK_MIGRATION,
|
|
785
|
+
"0003_safe_casts",
|
|
786
|
+
"0004_visible_seq"
|
|
787
|
+
]);
|
|
788
|
+
/** `rows` is converted into type tables, which writes `row_ids` too; the rest are copied as they are. */
|
|
615
789
|
const ENGINE_COPIED = [
|
|
616
790
|
"rows",
|
|
617
791
|
"short_ids",
|
|
@@ -634,13 +808,13 @@ function preflight(source, target, opts) {
|
|
|
634
808
|
const warnings = [];
|
|
635
809
|
if (source.serverVersionNum < PG17) refusals.push(`source is PostgreSQL ${source.serverVersionNum} (server_version_num); 170000 or later is required`);
|
|
636
810
|
if (target.serverVersionNum < PG18) refusals.push(`target is PostgreSQL ${target.serverVersionNum} (server_version_num); 180000 or later is required`);
|
|
637
|
-
const baselines =
|
|
638
|
-
const
|
|
639
|
-
const unknownVersions = source.appliedMigrations.filter((v) => !known.has(v) && !isDerivedMigration(v));
|
|
811
|
+
const baselines = Object.values(OLD_BASELINES);
|
|
812
|
+
const unknownVersions = source.appliedMigrations.filter((v) => !OLD_MIGRATIONS.has(v) && !isDerivedMigration(v));
|
|
640
813
|
if (unknownVersions.length > 0) refusals.push(`source ledger holds migrations this binary does not know: ${unknownVersions.join(", ")}`);
|
|
641
814
|
if (!source.appliedMigrations.some((v) => baselines.includes(v))) refusals.push(`source ledger lacks ${baselines.join(" / ")}; found: ${source.appliedMigrations.join(", ") || "no migrations"}`);
|
|
815
|
+
if (target.layout === "legacy" || target.layout === "partitioned") refusals.push(`target is a ${target.layout} store; migrate-store copies into the tables layout, so point it at an empty database or schema`);
|
|
642
816
|
if (!target.empty && !target.createdByUs) refusals.push("target already holds data and was not created by migrate-store");
|
|
643
|
-
const otherBaseline = baselines.find((b) => b !==
|
|
817
|
+
const otherBaseline = baselines.find((b) => b !== OLD_BASELINES[source.layout]);
|
|
644
818
|
if (otherBaseline !== void 0 && source.appliedMigrations.includes(otherBaseline)) refusals.push(`source ledger records ${otherBaseline} but rows is ${source.layout}`);
|
|
645
819
|
if (source.appliedMigrations.includes("0002_uuid_format_check") && !target.enforceUuidCheck) refusals.push(`source enforces the UUID check (${UUID_CHECK_MIGRATION}) but the target would not: run migrate-store with the service's environment, which sets SUBSTRATE_ENFORCE_UUID_CHECK=true`);
|
|
646
820
|
if (target.enforceUuidCheck && source.nonUuidIds.length > 0) {
|
|
@@ -655,7 +829,9 @@ function preflight(source, target, opts) {
|
|
|
655
829
|
]);
|
|
656
830
|
const unknownTables = source.tables.filter((t) => !knownTables.has(t) && !skip.has(t));
|
|
657
831
|
for (const t of unknownTables) refusals.push(`source table "${t}" is not known to migrate-store; load the plugin that owns it, or pass --skip-table ${t}`);
|
|
658
|
-
|
|
832
|
+
const copied = /* @__PURE__ */ new Set([...ENGINE_COPIED, ...opts.pluginTables]);
|
|
833
|
+
for (const t of skip) if (copied.has(t)) refusals.push(`--skip-table ${t} names a table migrate-store copies; it cannot be skipped. --skip-table is for tables migrate-store does not know`);
|
|
834
|
+
else if (!source.tables.includes(t)) warnings.push(`--skip-table ${t} matches no source table`);
|
|
659
835
|
if (source.vectorDims !== null && target.recallDims !== null && source.vectorDims !== target.recallDims) refusals.push(`recall embedding dimensions differ: source ${source.vectorDims}, target ${target.recallDims}`);
|
|
660
836
|
const present = new Set(source.tables);
|
|
661
837
|
const plan = {
|
|
@@ -665,7 +841,7 @@ function preflight(source, target, opts) {
|
|
|
665
841
|
bytes: source.tableBytes[name] ?? 0
|
|
666
842
|
})),
|
|
667
843
|
estimatedBytes: 0,
|
|
668
|
-
skipped: [...skip].filter((t) => present.has(t))
|
|
844
|
+
skipped: [...skip].filter((t) => present.has(t) && !copied.has(t))
|
|
669
845
|
};
|
|
670
846
|
plan.estimatedBytes = 2 * plan.tables.reduce((n, t) => n + t.bytes, 0);
|
|
671
847
|
for (const t of plan.tables) {
|
|
@@ -679,7 +855,12 @@ function preflight(source, target, opts) {
|
|
|
679
855
|
const msg = `free disk ${target.freeBytes} bytes is below the estimated need of ${plan.estimatedBytes} bytes`;
|
|
680
856
|
(opts.dryRun ? warnings : refusals).push(msg);
|
|
681
857
|
}
|
|
682
|
-
if (source.otherSessions > 0)
|
|
858
|
+
if (source.otherSessions > 0) {
|
|
859
|
+
const named = source.sessions.slice(0, 10).map((s) => `usename=${s.usename ?? "hidden"} application_name=${s.applicationName ?? "hidden"} client_addr=${s.clientAddr ?? "local"} backend_type=${s.backendType ?? "hidden"} state=${s.state ?? "hidden"}`);
|
|
860
|
+
const more = source.sessions.length - named.length;
|
|
861
|
+
const list = named.length > 0 ? `: ${named.join("; ")}${more > 0 ? `; +${more} more` : ""}` : "";
|
|
862
|
+
(opts.dryRun ? warnings : refusals).push(`source has ${source.otherSessions} other active session(s) on its database; writes after the snapshot are not copied, so stop the service first${list}`);
|
|
863
|
+
}
|
|
683
864
|
return {
|
|
684
865
|
refusals,
|
|
685
866
|
warnings,
|
|
@@ -690,25 +871,48 @@ function preflight(source, target, opts) {
|
|
|
690
871
|
//#endregion
|
|
691
872
|
//#region src/admin/migrate-store/verify.ts
|
|
692
873
|
/**
|
|
693
|
-
* The migrate-store verification: compares the source
|
|
694
|
-
* (namespace, type) and, only when everything agrees
|
|
695
|
-
* marker so `serve` will boot the
|
|
696
|
-
*
|
|
874
|
+
* The migrate-store verification: compares the source with its conversion
|
|
875
|
+
* per (namespace, type) and, only when everything agrees and the target holds
|
|
876
|
+
* every declared index, drops the in-progress marker so `serve` will boot the
|
|
877
|
+
* target. Both sides are read in their own REPEATABLE READ, READ ONLY
|
|
878
|
+
* transaction; nothing here writes to the source.
|
|
697
879
|
*
|
|
698
880
|
* Target rows the prepare step seeded and the source lacks (`keptSeeds`) are
|
|
699
881
|
* subtracted from the target side before every comparison.
|
|
700
882
|
*
|
|
701
883
|
* Every set is fingerprinted as `count(*)` plus the sum of a 64-bit prefix of
|
|
702
884
|
* each row's md5. That is independent of row order, of either cluster's
|
|
703
|
-
* collation, and of any join separator, and it builds no string.
|
|
885
|
+
* collation, and of any join separator, and it builds no string. Row content
|
|
886
|
+
* is compared as the dataset reads it: the target's row as `decodeRow` builds
|
|
887
|
+
* it, the source's body under the same read rule (an explicit null on a
|
|
888
|
+
* declared column field reads absent), both as jsonb text, so a number's
|
|
889
|
+
* stored text counts, plus every metadata column.
|
|
704
890
|
*/
|
|
705
891
|
const quote = (cols) => cols.map((c) => `"${c.replace(/"/g, "\"\"")}"`).join(", ");
|
|
706
892
|
/** SQL for a set's fingerprint: its size and an order-independent sum of per-row hashes of `expr`. */
|
|
707
893
|
const fp = (expr) => `count(*)::text || ' ' || coalesce(sum(('x' || substr(md5(${expr}), 1, 16))::bit(64)::bigint::numeric), 0)::text`;
|
|
894
|
+
/**
|
|
895
|
+
* The declared indexes of every registered type that are not valid on the
|
|
896
|
+
* target: missing, invalid, building, failed, or built over a field that is
|
|
897
|
+
* not yet in its column. `uniqueOnly` leaves out the non-unique ones, which
|
|
898
|
+
* `--skip-index-build` defers to serve.
|
|
899
|
+
*/
|
|
900
|
+
async function missingIndexes(target, uniqueOnly) {
|
|
901
|
+
const out = [];
|
|
902
|
+
for (const { ns, type, dataset } of registeredTypes(target.engine, target.datasets)) {
|
|
903
|
+
const report = await dataset?.describeStorage?.(type);
|
|
904
|
+
for (const i of report?.indexes ?? []) if (i.state !== "valid" && (i.unique || !uniqueOnly)) out.push({
|
|
905
|
+
type: `${ns}/${type}`,
|
|
906
|
+
index: i.name
|
|
907
|
+
});
|
|
908
|
+
}
|
|
909
|
+
return out;
|
|
910
|
+
}
|
|
708
911
|
async function verifyStore(source, target, opts) {
|
|
709
912
|
const readOnly = "isolation level repeatable read read only";
|
|
710
|
-
const
|
|
711
|
-
const
|
|
913
|
+
const types = await target.begin(readOnly, (tx) => typeTables(tx));
|
|
914
|
+
const src = await source.begin(readOnly, (tx) => snapshot(tx, "source", [], null, opts, types));
|
|
915
|
+
const tgt = await target.begin(readOnly, (tx) => snapshot(tx, "target", opts.keptSeeds, src, opts, types));
|
|
712
916
|
const mismatches = [];
|
|
713
917
|
const keys = [.../* @__PURE__ */ new Set([...src.entries.keys(), ...tgt.entries.keys()])].sort();
|
|
714
918
|
for (const k of keys) {
|
|
@@ -723,24 +927,30 @@ async function verifyStore(source, target, opts) {
|
|
|
723
927
|
detail: `${e.what}: source ${a?.value ?? "absent"}, target ${b?.value ?? "absent"}`
|
|
724
928
|
});
|
|
725
929
|
}
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
const
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
detail:
|
|
930
|
+
for (const [key, table] of types) {
|
|
931
|
+
const [namespace, type] = [table.namespace, table.type];
|
|
932
|
+
const unsettled = [...table.pending.map((field) => ({
|
|
933
|
+
field,
|
|
934
|
+
state: "pending"
|
|
935
|
+
})), ...table.columns().filter((c) => c.state === "promoting")];
|
|
936
|
+
for (const f of unsettled) mismatches.push({
|
|
937
|
+
check: "fields",
|
|
938
|
+
namespace,
|
|
939
|
+
type,
|
|
940
|
+
detail: `field ${f.field} of ${key} is ${f.state}, so its values are not all in its column`
|
|
737
941
|
});
|
|
738
942
|
}
|
|
739
|
-
const
|
|
943
|
+
for (const w of (await opts.storageReport({ partitionBudget: opts.partitionBudget })).warnings) mismatches.push({
|
|
944
|
+
check: "storage",
|
|
945
|
+
detail: w
|
|
946
|
+
});
|
|
947
|
+
const missing = await opts.missingIndexes();
|
|
948
|
+
const ok = mismatches.length === 0 && missing.length === 0;
|
|
740
949
|
if (ok) await target`DROP TABLE IF EXISTS ${target(MIGRATE_STORE_MARKER_TABLE)}`;
|
|
741
950
|
return {
|
|
742
951
|
ok,
|
|
743
|
-
mismatches
|
|
952
|
+
mismatches,
|
|
953
|
+
missingIndexes: missing
|
|
744
954
|
};
|
|
745
955
|
}
|
|
746
956
|
const COPY_MILESTONES = /* @__PURE__ */ new Set([
|
|
@@ -751,9 +961,9 @@ const COPY_MILESTONES = /* @__PURE__ */ new Set([
|
|
|
751
961
|
]);
|
|
752
962
|
/**
|
|
753
963
|
* What a finished copy left behind, for a verification that has no
|
|
754
|
-
* `CopyResult`: the kept seeds from the marker table,
|
|
755
|
-
* does not register (their
|
|
756
|
-
*
|
|
964
|
+
* `CopyResult`: the kept seeds from the marker table, the types the target
|
|
965
|
+
* does not register (their fields all sit in `extra`), and the copied tables,
|
|
966
|
+
* which are the marker's `copy:<table>` steps.
|
|
757
967
|
*/
|
|
758
968
|
async function readCopyState(target) {
|
|
759
969
|
const sql = target.sql;
|
|
@@ -773,12 +983,31 @@ async function readCopyState(target) {
|
|
|
773
983
|
id
|
|
774
984
|
};
|
|
775
985
|
}).sort((a, b) => (a.namespace + a.type + a.id).localeCompare(b.namespace + b.type + b.id)),
|
|
776
|
-
defaulted: (await sql`
|
|
986
|
+
defaulted: (await sql`
|
|
987
|
+
SELECT DISTINCT namespace, type FROM row_ids ORDER BY namespace, type`).filter((p) => registeredDataset(target, p.namespace, p.type) === void 0).map((p) => `${p.namespace}/${p.type}`),
|
|
777
988
|
tables: steps.map((s) => s.step).filter((s) => s.startsWith("copy:") && !COPY_MILESTONES.has(s)).map((s) => s.slice(5)).sort()
|
|
778
989
|
};
|
|
779
990
|
}
|
|
780
|
-
/**
|
|
781
|
-
async function
|
|
991
|
+
/** The target's type tables, by `<ns>/<Type>`. */
|
|
992
|
+
async function typeTables(tx) {
|
|
993
|
+
const types = /* @__PURE__ */ new Map();
|
|
994
|
+
const listed = await tx`
|
|
995
|
+
SELECT namespace, type FROM substrate_type_tables ORDER BY namespace, type`;
|
|
996
|
+
for (const { namespace, type } of listed) {
|
|
997
|
+
const table = await loadTypeTable(tx, namespace, type);
|
|
998
|
+
if (table !== void 0) types.set(`${namespace}/${type}`, table);
|
|
999
|
+
}
|
|
1000
|
+
return types;
|
|
1001
|
+
}
|
|
1002
|
+
/** Each row's id with its versions: a row updated or deleted after the copy's snapshot changes it, wherever the sample falls. */
|
|
1003
|
+
const VERSIONS = `id || E'\n' || coalesce(updated_at::text, '') || E'\n' || coalesce(deleted_at::text, '')`;
|
|
1004
|
+
const META = `deleted_at::text, created_at::text, updated_at::text, created_by_kind, created_by_id, updated_by_kind, updated_by_id`;
|
|
1005
|
+
/**
|
|
1006
|
+
* Every comparable fact about one side, keyed so the two sides diff by key.
|
|
1007
|
+
* `other` is the source's snapshot when this is the target; `types` are the
|
|
1008
|
+
* target's type tables, which give the source its read rule.
|
|
1009
|
+
*/
|
|
1010
|
+
async function snapshot(tx, side, seeds, other, opts, types) {
|
|
782
1011
|
await tx`SET LOCAL TimeZone = 'UTC'`;
|
|
783
1012
|
await tx`SET LOCAL DateStyle = 'ISO, YMD'`;
|
|
784
1013
|
await tx`SET LOCAL extra_float_digits = 3`;
|
|
@@ -787,6 +1016,7 @@ async function snapshot(tx, seeds, other, opts) {
|
|
|
787
1016
|
const add = (e) => entries.set(`${e.check}\0${e.namespace ?? ""}\0${e.type ?? ""}\0${e.what}`, e);
|
|
788
1017
|
const notSeed = (ns, ty, id) => tx`(${tx(ns)}, ${tx(ty)}, ${tx(id)}) NOT IN (SELECT * FROM unnest(
|
|
789
1018
|
${seeds.map((k) => k.namespace)}::text[], ${seeds.map((k) => k.type)}::text[], ${seeds.map((k) => k.id)}::text[]))`;
|
|
1019
|
+
const seedIds = (ns, ty) => seeds.filter((k) => k.namespace === ns && k.type === ty).map((k) => k.id);
|
|
790
1020
|
const colsOf = async (table) => {
|
|
791
1021
|
const have = await tx`
|
|
792
1022
|
SELECT a.attname AS col FROM pg_attribute a
|
|
@@ -798,33 +1028,101 @@ async function snapshot(tx, seeds, other, opts) {
|
|
|
798
1028
|
columns.set(table, cols);
|
|
799
1029
|
return cols;
|
|
800
1030
|
};
|
|
801
|
-
const
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
1031
|
+
const limit = opts.fullChecksum ? Number.MAX_SAFE_INTEGER : opts.sampleRows;
|
|
1032
|
+
const counted = [];
|
|
1033
|
+
if (side === "source") {
|
|
1034
|
+
const counts = await tx`
|
|
1035
|
+
SELECT namespace, type,
|
|
1036
|
+
count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
|
|
1037
|
+
count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
|
|
1038
|
+
${tx.unsafe(fp("id"))} AS ids,
|
|
1039
|
+
${tx.unsafe(fp(VERSIONS))} AS versions
|
|
1040
|
+
FROM rows GROUP BY namespace, type`;
|
|
1041
|
+
for (const r of counts) {
|
|
1042
|
+
const at = {
|
|
1043
|
+
namespace: r.namespace,
|
|
1044
|
+
type: r.type
|
|
1045
|
+
};
|
|
1046
|
+
add({
|
|
1047
|
+
check: "count",
|
|
1048
|
+
...at,
|
|
1049
|
+
what: "live rows",
|
|
1050
|
+
value: r.live
|
|
1051
|
+
});
|
|
1052
|
+
add({
|
|
1053
|
+
check: "count",
|
|
1054
|
+
...at,
|
|
1055
|
+
what: "deleted rows",
|
|
1056
|
+
value: r.deleted
|
|
1057
|
+
});
|
|
1058
|
+
add({
|
|
1059
|
+
check: "ids",
|
|
1060
|
+
...at,
|
|
1061
|
+
what: "id set",
|
|
1062
|
+
value: r.ids
|
|
1063
|
+
});
|
|
1064
|
+
add({
|
|
1065
|
+
check: "ids",
|
|
1066
|
+
...at,
|
|
1067
|
+
what: "registered ids (row_ids)",
|
|
1068
|
+
value: r.ids
|
|
1069
|
+
});
|
|
1070
|
+
add({
|
|
1071
|
+
check: "versions",
|
|
1072
|
+
...at,
|
|
1073
|
+
what: "row versions (updated_at, deleted_at)",
|
|
1074
|
+
value: r.versions
|
|
1075
|
+
});
|
|
1076
|
+
counted.push(at);
|
|
1077
|
+
}
|
|
1078
|
+
} else {
|
|
1079
|
+
for (const table of types.values()) {
|
|
1080
|
+
const { namespace, type } = table;
|
|
1081
|
+
const skip = seedIds(namespace, type);
|
|
1082
|
+
const [r] = await tx.unsafe(`SELECT count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
|
|
1083
|
+
count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
|
|
1084
|
+
${fp("id")} AS ids, ${fp(VERSIONS)} AS versions
|
|
1085
|
+
FROM ${table.from} WHERE NOT (id = ANY($1::text[]))`, [skip]);
|
|
1086
|
+
const at = {
|
|
1087
|
+
namespace,
|
|
1088
|
+
type
|
|
1089
|
+
};
|
|
1090
|
+
if (r !== void 0 && Number(r.live) + Number(r.deleted) > 0) {
|
|
1091
|
+
add({
|
|
1092
|
+
check: "count",
|
|
1093
|
+
...at,
|
|
1094
|
+
what: "live rows",
|
|
1095
|
+
value: r.live
|
|
1096
|
+
});
|
|
1097
|
+
add({
|
|
1098
|
+
check: "count",
|
|
1099
|
+
...at,
|
|
1100
|
+
what: "deleted rows",
|
|
1101
|
+
value: r.deleted
|
|
1102
|
+
});
|
|
1103
|
+
add({
|
|
1104
|
+
check: "ids",
|
|
1105
|
+
...at,
|
|
1106
|
+
what: "id set",
|
|
1107
|
+
value: r.ids
|
|
1108
|
+
});
|
|
1109
|
+
add({
|
|
1110
|
+
check: "versions",
|
|
1111
|
+
...at,
|
|
1112
|
+
what: "row versions (updated_at, deleted_at)",
|
|
1113
|
+
value: r.versions
|
|
1114
|
+
});
|
|
1115
|
+
counted.push(at);
|
|
1116
|
+
}
|
|
1117
|
+
}
|
|
1118
|
+
const registered = await tx`
|
|
1119
|
+
SELECT namespace, type, ${tx.unsafe(fp("id"))} AS ids
|
|
1120
|
+
FROM row_ids WHERE ${notSeed("namespace", "type", "id")} GROUP BY namespace, type`;
|
|
1121
|
+
for (const r of registered) add({
|
|
825
1122
|
check: "ids",
|
|
826
|
-
|
|
827
|
-
|
|
1123
|
+
namespace: r.namespace,
|
|
1124
|
+
type: r.type,
|
|
1125
|
+
what: "registered ids (row_ids)",
|
|
828
1126
|
value: r.ids
|
|
829
1127
|
});
|
|
830
1128
|
}
|
|
@@ -852,14 +1150,30 @@ async function snapshot(tx, seeds, other, opts) {
|
|
|
852
1150
|
value: r.f
|
|
853
1151
|
});
|
|
854
1152
|
}
|
|
855
|
-
const
|
|
856
|
-
const
|
|
857
|
-
for (const { namespace, type } of
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
1153
|
+
const hasSeq = side === "source" ? (await colsOf("rows"))?.includes("seq") === true : other?.columns.get("rows")?.includes("seq") !== false;
|
|
1154
|
+
const meta = `row(${META}, ${hasSeq ? "seq::text" : "NULL::text"})::text`;
|
|
1155
|
+
for (const { namespace, type } of counted) {
|
|
1156
|
+
let rows;
|
|
1157
|
+
let params;
|
|
1158
|
+
if (side === "source") {
|
|
1159
|
+
const table = types.get(`${namespace}/${type}`);
|
|
1160
|
+
const declared = [...(table?.columns().filter((c) => c.state !== "orphaned") ?? []).map((c) => c.field), ...table?.pending ?? []];
|
|
1161
|
+
rows = `SELECT id || E'\n' || (SELECT coalesce(jsonb_object_agg(k, v), '{}'::jsonb) FROM jsonb_each(body) e(k, v)
|
|
1162
|
+
WHERE NOT (v = 'null'::jsonb AND k = ANY($3::text[])))::text || E'\n' || ${meta} AS h
|
|
1163
|
+
FROM rows WHERE namespace = $1 AND type = $2 ORDER BY md5(id) COLLATE "C" LIMIT ${limit}`;
|
|
1164
|
+
params = [
|
|
1165
|
+
namespace,
|
|
1166
|
+
type,
|
|
1167
|
+
declared
|
|
1168
|
+
];
|
|
1169
|
+
} else {
|
|
1170
|
+
const ref = await typeTableRef(tx, namespace, type);
|
|
1171
|
+
if (ref === void 0) continue;
|
|
1172
|
+
rows = `SELECT x.id || E'\n' || ${ref.rowJson("x")}::text || E'\n' || ${meta} AS h FROM ${ref.from} x
|
|
1173
|
+
WHERE NOT (x.id = ANY($1::text[])) ORDER BY md5(x.id) COLLATE "C" LIMIT ${limit}`;
|
|
1174
|
+
params = [seedIds(namespace, type)];
|
|
1175
|
+
}
|
|
1176
|
+
const [r] = await tx.unsafe(`SELECT ${fp("h")} AS f FROM (${rows}) s`, params);
|
|
863
1177
|
add({
|
|
864
1178
|
check: "sample",
|
|
865
1179
|
namespace,
|
|
@@ -899,9 +1213,9 @@ async function snapshot(tx, seeds, other, opts) {
|
|
|
899
1213
|
//#endregion
|
|
900
1214
|
//#region src/admin/commands/migrate-store.ts
|
|
901
1215
|
/**
|
|
902
|
-
* `substrate-admin migrate-store`:
|
|
903
|
-
*
|
|
904
|
-
* verification, report. Connection strings come from named environment
|
|
1216
|
+
* `substrate-admin migrate-store`: converts a legacy or partitioned store into
|
|
1217
|
+
* a fresh, empty PostgreSQL 18 store in the `tables` layout. Preflight,
|
|
1218
|
+
* prepare and copy, index builds, verification, report. Connection strings come from named environment
|
|
905
1219
|
* variables only, and nothing this command prints carries one.
|
|
906
1220
|
*/
|
|
907
1221
|
const DEFAULT_FROM_ENV = "SUBSTRATE_MIGRATE_FROM_URL";
|
|
@@ -1043,6 +1357,8 @@ async function runMigrateStore(args, env, deps) {
|
|
|
1043
1357
|
totalMs: 0,
|
|
1044
1358
|
keptSeeds: [],
|
|
1045
1359
|
defaulted: [],
|
|
1360
|
+
misfits: null,
|
|
1361
|
+
missingIndexes: [],
|
|
1046
1362
|
pendingIndexes: [],
|
|
1047
1363
|
mismatches: [],
|
|
1048
1364
|
switchOver: null
|
|
@@ -1073,13 +1389,13 @@ async function runMigrateStore(args, env, deps) {
|
|
|
1073
1389
|
...vars,
|
|
1074
1390
|
SUBSTRATE_DATABASE_URL: toUrl,
|
|
1075
1391
|
HOST: "127.0.0.1",
|
|
1392
|
+
SUBSTRATE_SYSTEM_ACTOR: vars.SUBSTRATE_SYSTEM_ACTOR?.trim() || "migrate-store",
|
|
1076
1393
|
SUBSTRATE_STREAM_TICKET_SECRET: vars.SUBSTRATE_STREAM_TICKET_SECRET?.trim() || randomBytes(32).toString("hex")
|
|
1077
1394
|
};
|
|
1078
1395
|
const pluginDir = resolvePluginDir(targetEnv);
|
|
1079
1396
|
const boot = () => bootStore({
|
|
1080
1397
|
config: parseEnv(targetEnv),
|
|
1081
1398
|
pluginDir,
|
|
1082
|
-
layout: "partitioned",
|
|
1083
1399
|
allowMigrating: true,
|
|
1084
1400
|
background: false
|
|
1085
1401
|
});
|
|
@@ -1126,7 +1442,8 @@ async function runMigrateStore(args, env, deps) {
|
|
|
1126
1442
|
await assertSchema(facts, args.toSchema, "target");
|
|
1127
1443
|
await assertDistinct(src, facts);
|
|
1128
1444
|
const source = await inspectSource(src);
|
|
1129
|
-
source.
|
|
1445
|
+
source.sessions = source.sessions.filter((s) => !(s.pid === own.pid && s.start === own.start));
|
|
1446
|
+
source.otherSessions = deps.otherSessions ?? source.sessions.length;
|
|
1130
1447
|
const target = await targetFacts(facts, env, diskPath);
|
|
1131
1448
|
const pluginTables = await storeTables(pluginDir);
|
|
1132
1449
|
return {
|
|
@@ -1155,19 +1472,20 @@ async function runMigrateStore(args, env, deps) {
|
|
|
1155
1472
|
}));
|
|
1156
1473
|
target = copied.target;
|
|
1157
1474
|
report.tables = copied.tables;
|
|
1475
|
+
report.misfits = copied.misfits;
|
|
1158
1476
|
({keptSeeds, defaulted} = copied);
|
|
1159
1477
|
tables = pre.plan.tables.map((t) => t.name);
|
|
1160
1478
|
}
|
|
1161
1479
|
const booted = target;
|
|
1162
1480
|
const indexFailures = args.skipIndexBuild ? [] : await phase("indexes", () => (deps.buildIndexes ?? buildDeferredIndexes)(booted.engine, booted.datasets));
|
|
1163
1481
|
report.keptSeeds = keptSeeds.map((k) => `${k.namespace}/${k.type}/${k.id}`);
|
|
1164
|
-
report.defaulted = defaulted
|
|
1482
|
+
report.defaulted = defaulted;
|
|
1165
1483
|
const tsql = booted.sql;
|
|
1166
1484
|
await deps.beforeVerify?.(tsql);
|
|
1167
1485
|
const verified = await phase("verify", () => verifyStore(src, tsql, {
|
|
1168
1486
|
keptSeeds,
|
|
1169
|
-
defaulted,
|
|
1170
1487
|
tables,
|
|
1488
|
+
missingIndexes: () => missingIndexes(booted, args.skipIndexBuild),
|
|
1171
1489
|
fullChecksum: args.fullChecksum,
|
|
1172
1490
|
sampleRows: 1e3,
|
|
1173
1491
|
partitionBudget: parseEnv(targetEnv).partitionBudget,
|
|
@@ -1180,6 +1498,7 @@ async function runMigrateStore(args, env, deps) {
|
|
|
1180
1498
|
}
|
|
1181
1499
|
}));
|
|
1182
1500
|
report.mismatches = verified.mismatches;
|
|
1501
|
+
report.missingIndexes = verified.missingIndexes;
|
|
1183
1502
|
if (!verified.ok) return finish(3);
|
|
1184
1503
|
report.switchOver = `stop serve, set SUBSTRATE_DATABASE_URL to the value of ${args.toEnv}${args.toSchema !== null ? ` with search_path ${args.toSchema},public (options=-c search_path=${args.toSchema},public)` : ""}, then start serve. The source is untouched: pointing back at it is the rollback.`;
|
|
1185
1504
|
return finish(0);
|
|
@@ -1243,22 +1562,16 @@ async function ownSession(sql) {
|
|
|
1243
1562
|
FROM pg_stat_activity WHERE pid = pg_backend_pid()`;
|
|
1244
1563
|
return r;
|
|
1245
1564
|
}
|
|
1246
|
-
/** 1 when `s` is a backend on `sql`'s database, else 0: our own target connection when both share one. */
|
|
1247
|
-
async function sessionsOn(sql, s) {
|
|
1248
|
-
const [r] = await sql`
|
|
1249
|
-
SELECT count(*)::int AS n FROM pg_stat_activity
|
|
1250
|
-
WHERE datname = current_database() AND pid = ${s.pid}
|
|
1251
|
-
AND extract(epoch FROM backend_start)::text = ${s.start}`;
|
|
1252
|
-
return r?.n ?? 0;
|
|
1253
|
-
}
|
|
1254
1565
|
async function targetFacts(sql, env, diskPath) {
|
|
1255
1566
|
const [r] = await sql`
|
|
1256
1567
|
SELECT current_setting('server_version_num')::int AS v, current_schema() AS schema,
|
|
1257
1568
|
to_regclass(quote_ident(current_schema()) || '.rows') IS NOT NULL AS rows,
|
|
1569
|
+
to_regclass(quote_ident(current_schema()) || '.row_ids') IS NOT NULL AS ids,
|
|
1258
1570
|
to_regclass(quote_ident(current_schema()) || '.' || quote_ident(${MIGRATE_STORE_MARKER_TABLE})) IS NOT NULL AS marker,
|
|
1259
1571
|
to_regclass(quote_ident(current_schema()) || '.embeddings') IS NOT NULL AS embeddings`;
|
|
1260
1572
|
const facts = r;
|
|
1261
|
-
const
|
|
1573
|
+
const holds = async (table) => (await sql`SELECT 1 FROM ${sql(facts.schema)}.${sql(table)} LIMIT 1`).length > 0;
|
|
1574
|
+
const empty = !(facts.rows && await holds("rows")) && !(facts.ids && await holds("row_ids"));
|
|
1262
1575
|
const [dims] = facts.embeddings ? await sql`
|
|
1263
1576
|
SELECT atttypmod AS d FROM pg_attribute
|
|
1264
1577
|
WHERE attrelid = to_regclass(quote_ident(${facts.schema}) || '.embeddings')
|
|
@@ -1277,6 +1590,7 @@ async function targetFacts(sql, env, diskPath) {
|
|
|
1277
1590
|
}
|
|
1278
1591
|
return {
|
|
1279
1592
|
serverVersionNum: facts.v,
|
|
1593
|
+
layout: await detectLayout(sql),
|
|
1280
1594
|
empty,
|
|
1281
1595
|
createdByUs: facts.marker,
|
|
1282
1596
|
enforceUuidCheck: env.SUBSTRATE_ENFORCE_UUID_CHECK === "true",
|
|
@@ -1319,7 +1633,13 @@ function formatMigrateStoreOutput(r, json) {
|
|
|
1319
1633
|
for (const t of r.tables) lines.push(` ${t.name}: ${t.rows} / ${t.bytes} / ${t.ms}`);
|
|
1320
1634
|
}
|
|
1321
1635
|
if (r.keptSeeds.length > 0) lines.push(`kept seeds: ${r.keptSeeds.join(", ")}`);
|
|
1322
|
-
if (r.defaulted.length > 0) lines.push(`
|
|
1636
|
+
if (r.defaulted.length > 0) lines.push(`declared by no definition, every field in extra: ${r.defaulted.join(", ")}`);
|
|
1637
|
+
if (r.misfits === null) lines.push("misfits: not counted (only a copy run counts them)");
|
|
1638
|
+
for (const [type, m] of Object.entries(r.misfits ?? {})) {
|
|
1639
|
+
const more = m.count > 10 ? ` (first 10 shown; --json lists ${m.first.length})` : "";
|
|
1640
|
+
lines.push(`misfits in ${type}: ${m.count}, kept unchanged under $raw${more}: ${m.first.slice(0, 10).map((x) => `${x.id}.${x.field}`).join(", ")}`);
|
|
1641
|
+
}
|
|
1642
|
+
for (const m of r.missingIndexes) lines.push(` missing index [${m.type}]: ${m.index}`);
|
|
1323
1643
|
if (r.pendingIndexes.length > 0) {
|
|
1324
1644
|
lines.push(`${r.pendingIndexes.length} index build(s) left pending for serve (--skip-index-build; --json lists them all):`);
|
|
1325
1645
|
for (const w of r.pendingIndexes.slice(0, 10)) lines.push(` ${w}`);
|
|
@@ -1382,6 +1702,60 @@ function shim(sink) {
|
|
|
1382
1702
|
});
|
|
1383
1703
|
}
|
|
1384
1704
|
|
|
1705
|
+
//#endregion
|
|
1706
|
+
//#region src/admin/commands/promote-fields.ts
|
|
1707
|
+
/** Parses `argv` (after the subcommand), resolves the server, and runs one sweep there. */
|
|
1708
|
+
async function promoteFieldsCommand(argv, env, fetchImpl = fetch) {
|
|
1709
|
+
validateArgv("promote-fields", argv);
|
|
1710
|
+
let url;
|
|
1711
|
+
for (let i = 0; i < argv.length; i++) {
|
|
1712
|
+
if (argv[i] !== "--url") continue;
|
|
1713
|
+
if (url !== void 0) throw argvError("promote-fields", "promote-fields: --url specified more than once");
|
|
1714
|
+
url = argv[i + 1] ?? "";
|
|
1715
|
+
i += 1;
|
|
1716
|
+
}
|
|
1717
|
+
if (url === "") throw argvError("promote-fields", "promote-fields: --url requires a base URL");
|
|
1718
|
+
const target = resolveAdminTarget(url ?? null, env);
|
|
1719
|
+
if (target.kind !== "route") throw new AdminCliError("promote-fields runs on a running server: pass --url <base> or set SUBSTRATE_URL", 1);
|
|
1720
|
+
const auth = resolveAdminAuth(env, target.baseUrl);
|
|
1721
|
+
if ("error" in auth) throw new AdminCliError(`promote-fields: ${auth.error}`, 1);
|
|
1722
|
+
const endpoint = `${trimTrailingSlashes(target.baseUrl)}/substrate/v1/_system/promote-fields`;
|
|
1723
|
+
let res;
|
|
1724
|
+
try {
|
|
1725
|
+
res = await fetchImpl(endpoint, {
|
|
1726
|
+
method: "POST",
|
|
1727
|
+
headers: {
|
|
1728
|
+
authorization: auth.ok,
|
|
1729
|
+
"content-type": "application/json"
|
|
1730
|
+
},
|
|
1731
|
+
body: "{}",
|
|
1732
|
+
redirect: "error"
|
|
1733
|
+
});
|
|
1734
|
+
} catch (err) {
|
|
1735
|
+
throw new AdminCliError(`could not reach ${endpoint}: ${err instanceof Error ? err.message : String(err)}`, 1);
|
|
1736
|
+
}
|
|
1737
|
+
const payload = await res.json().catch(() => ({}));
|
|
1738
|
+
if (!res.ok) {
|
|
1739
|
+
const reason = typeof payload.message === "string" ? payload.message : typeof payload.code === "string" ? payload.code : `HTTP ${res.status}`;
|
|
1740
|
+
throw new AdminCliError(`promote-fields: ${reason}`, res.status === 403 ? 5 : 1);
|
|
1741
|
+
}
|
|
1742
|
+
if (!Array.isArray(payload.events)) throw new AdminCliError("promote-fields: response carried no `events` array", 1);
|
|
1743
|
+
return {
|
|
1744
|
+
target: describeAdminTarget("promote-fields", target),
|
|
1745
|
+
events: payload.events
|
|
1746
|
+
};
|
|
1747
|
+
}
|
|
1748
|
+
/** One line per field, after the target line. */
|
|
1749
|
+
function formatPromoteFieldsOutput(result) {
|
|
1750
|
+
const lines = [result.target];
|
|
1751
|
+
if (result.events.length === 0) lines.push("no pending fields");
|
|
1752
|
+
for (const e of result.events) {
|
|
1753
|
+
const outcome = e.kind === "busy" && e.error !== void 0 ? `busy (${e.error})` : `${e.kind} (attempt ${e.attempts})${e.error !== void 0 ? ` — ${e.error}` : ""}`;
|
|
1754
|
+
lines.push(`${e.namespace}/${e.type}.${e.field}: ${outcome}`);
|
|
1755
|
+
}
|
|
1756
|
+
return `${lines.join("\n")}\n`;
|
|
1757
|
+
}
|
|
1758
|
+
|
|
1385
1759
|
//#endregion
|
|
1386
1760
|
//#region src/admin/commands/revoke.ts
|
|
1387
1761
|
/**
|
|
@@ -1931,6 +2305,11 @@ async function main(argv, env, out, errOut, openDataset = openPgDataset, prompt
|
|
|
1931
2305
|
return 0;
|
|
1932
2306
|
}
|
|
1933
2307
|
case "migrate-store": return migrateStoreCommand(argv.slice(1), env, out, errOut);
|
|
2308
|
+
case "promote-fields": {
|
|
2309
|
+
const result = await promoteFieldsCommand(argv.slice(1), env);
|
|
2310
|
+
out(formatPromoteFieldsOutput(result));
|
|
2311
|
+
return result.events.some((e) => e.kind === "failed") ? 1 : 0;
|
|
2312
|
+
}
|
|
1934
2313
|
default:
|
|
1935
2314
|
errOut(`unknown command: ${command}\n${USAGE}\n`);
|
|
1936
2315
|
return 1;
|