@mnmal-ai/substrate 1.0.0-alpha.10 → 1.0.0-alpha.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/README.md +4 -2
  2. package/dist/{boot-store-BVqfFxh1.js → boot-store-DHv9H1Gr.js} +170 -69
  3. package/dist/{bridge-Cp4PaIYO.d.ts → bridge-DQyQ6Qhn.d.ts} +1 -1
  4. package/dist/{cli-hSCBywhE.js → cli-_UNdneXr.js} +44 -24
  5. package/dist/client.d.ts +3 -4
  6. package/dist/client.js +3 -3
  7. package/dist/{errors-7mycvNGq.d.ts → errors-xP9Mo7Q1.d.ts} +38 -4
  8. package/dist/{index-DgSfwx7M.d.ts → index-baF5A5jU.d.ts} +1 -1
  9. package/dist/pack.d.ts +2 -2
  10. package/dist/pack.js +1 -1
  11. package/dist/{pg-edges-store-Lxsl5ZnK.js → pg-edges-store-B4FSN9yp.js} +72 -9
  12. package/dist/{plugin-CtwOINCM.js → plugin-BLm5IRdX.js} +2 -2
  13. package/dist/proposals-bootstrap.js +2 -2
  14. package/dist/proposals.d.ts +2 -3
  15. package/dist/proposals.js +1 -1
  16. package/dist/recall-bootstrap.js +2 -2
  17. package/dist/recall.d.ts +10 -11
  18. package/dist/recall.js +2 -2
  19. package/dist/{repl-DqfHelK7.js → repl-DUJGBwie.js} +25 -25
  20. package/dist/{row-BKKCXIG6.d.ts → row-Cu_iRhcR.d.ts} +57 -10
  21. package/dist/{run-B5dTbZ3g.js → run-BpsTNvdI.js} +539 -145
  22. package/dist/{dataset-CejF-Fr_.d.ts → schema-oXfQyo0s.d.ts} +2455 -1589
  23. package/dist/sdk.d.ts +108 -34
  24. package/dist/sdk.js +4 -4
  25. package/dist/{src-CqT-QWZ_.js → src-Bew_JvTo.js} +32 -16
  26. package/dist/{src-Da0t6si5.js → src-Bi3NhEsT.js} +1 -1
  27. package/dist/{src-5SJ5GEIM.js → src-C4t1Gy-m.js} +4844 -1367
  28. package/dist/{src-7CUSMK66.js → src-CHND7-Pm.js} +7 -3
  29. package/dist/src-CSe2tJDf.js +6940 -0
  30. package/dist/{src-Bi849yhn.js → src-CzhKLsOV.js} +175 -91
  31. package/dist/{src-hZb8wfvg.js → src-DWqzrm3O.js} +400 -284
  32. package/dist/substrate-admin.js +1 -1
  33. package/dist/substrate.js +1 -1
  34. package/dist/testing.d.ts +255 -90
  35. package/dist/testing.js +6 -23
  36. package/dist/{ws-ByZyHk7m.d.ts → ws-Dx3A2B4U.d.ts} +2 -3
  37. package/package.json +20 -20
  38. package/dist/schema-DoSc-4ig.d.ts +0 -470
  39. package/dist/src-CyvV7dCK.js +0 -4333
@@ -1,11 +1,13 @@
1
1
  import { createRequire as __binCreateRequire } from 'node:module'; const require = __binCreateRequire(import.meta.url);
2
- import { h as AUDIT_EVENT_TYPE, m as buildAuditRow } from "./src-5SJ5GEIM.js";
3
- import { I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./src-Da0t6si5.js";
4
- import { A as runKeyListViaRoute, B as describeAdminTarget, C as doctorCommand, D as runBootstrap, E as parseBootstrapArgs, F as base64Url, G as USAGE, H as ARGV_SPEC, I as forUpdate, J as pgNoticeOptions, K as argvError, L as generateAdminKeypair, M as ACTOR_HINT, N as BootstrapError, O as REVOKE_EXIT_CODES, P as CLI_ACTOR_KID, R as listActiveAdminKeys, S as grantLines, T as formatBootstrapOutput, U as validateArgv, V as resolveAdminTarget, W as RESTART_NOTE, _ as onboardCommand, b as runRegister, c as refuseContractViolation, g as openPgDataset, h as version, j as runKeyPatchViaRoute, k as UPDATE_EXIT_CODES, l as buildDeferredIndexes, m as loadPluginsFromDir, n as resolvePluginDir, q as AdminCliError, r as BINARY_CONTRACT_VERSION, s as parseEnv, t as bootStore, v as formatRegisterOutput, w as formatDoctorReport, x as runRegisterViaRoute, y as parseRegisterArgs, z as readKeyById } from "./boot-store-BVqfFxh1.js";
5
- import { a as PLAIN_SCHEMA_NAME, c as assertStoreSchema, d as isDerivedMigration, g as detectLayout, h as UUID_REGEX, i as MIGRATE_STORE_MARKER_TABLE, o as UUID_CHECK_MIGRATION, r as seqName, u as getMigrationList } from "./src-CyvV7dCK.js";
2
+ import { _ as AUDIT_EVENT_TYPE, g as buildAuditRow } from "./src-C4t1Gy-m.js";
3
+ import { n as trimTrailingSlashes } from "./env-utils-CGNazUNt.js";
4
+ import { I as decodeList, L as encodeList, M as grantEntryError, g as checkUpdatable, h as checkRevocable, y as newlyGranted } from "./src-Bi3NhEsT.js";
5
+ import "./src-DWqzrm3O.js";
6
+ import { A as UPDATE_EXIT_CODES, B as readKeyById, C as grantLines, D as parseBootstrapArgs, E as formatBootstrapOutput, F as CLI_ACTOR_KID, G as validateArgv, H as resolveAdminAuth, I as base64Url, J as argvError, K as RESTART_NOTE, L as forUpdate, M as runKeyPatchViaRoute, N as ACTOR_HINT, O as runBootstrap, P as BootstrapError, R as generateAdminKeypair, S as runRegisterViaRoute, T as formatDoctorReport, U as resolveAdminTarget, V as describeAdminTarget, W as ARGV_SPEC, X as pgNoticeOptions, Y as AdminCliError, _ as openPgDataset, b as parseRegisterArgs, c as refuseContractViolation, d as registeredTypes, g as version, h as loadPluginsFromDir, j as runKeyListViaRoute, k as REVOKE_EXIT_CODES, l as buildDeferredIndexes, n as resolvePluginDir, q as USAGE, r as BINARY_CONTRACT_VERSION, s as parseEnv, t as bootStore, v as onboardCommand, w as doctorCommand, x as runRegister, y as formatRegisterOutput, z as listActiveAdminKeys } from "./boot-store-DHv9H1Gr.js";
7
+ import { C as parseSourceBody, S as copyRows, b as UUID_REGEX, f as assertStoreSchema, h as isDerivedMigration, i as ensureTypeTable, l as MIGRATE_STORE_MARKER_TABLE, o as loadTypeTable, r as seqName, s as typeTableRef, u as PLAIN_SCHEMA_NAME, x as detectLayout, y as UUID_CHECK_MIGRATION } from "./src-CSe2tJDf.js";
6
8
  import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
7
9
  import { dirname } from "node:path";
8
- import { createHash, randomUUID } from "node:crypto";
10
+ import { createHash, randomBytes, randomUUID } from "node:crypto";
9
11
  import { statfs } from "node:fs/promises";
10
12
  import postgres from "postgres";
11
13
  import { PassThrough, Transform } from "node:stream";
@@ -144,7 +146,7 @@ function formatRemoteListOutput(rows, baseUrl) {
144
146
 
145
147
  //#endregion
146
148
  //#region src/admin/migrate-store/progress.ts
147
- /** Progress of a migrate-store copy, counted off the COPY text stream as it passes. */
149
+ /** Progress of a migrate-store copy, counted off the COPY text stream as it passes, or per converted batch. */
148
150
  /**
149
151
  * A pass-through that counts rows (one newline each: COPY text escapes the
150
152
  * newlines inside values) and reports every `batchRows` rows. A throw from
@@ -170,19 +172,27 @@ var RowCounter = class extends Transform {
170
172
  this.bytes = from.bytes;
171
173
  this.#next = from.rows - from.rows % batchRows + batchRows;
172
174
  }
175
+ /** Counts `rows` converted rows of `bytes` body text, reporting as `_transform` does. */
176
+ add(rows, bytes) {
177
+ this.rows += rows;
178
+ this.bytes += bytes;
179
+ this.#report();
180
+ }
181
+ #report() {
182
+ if (this.rows < this.#next) return;
183
+ this.#next = this.rows - this.rows % this.batchRows + this.batchRows;
184
+ this.onProgress({
185
+ table: this.table,
186
+ rows: this.rows,
187
+ bytes: this.bytes,
188
+ done: false
189
+ });
190
+ }
173
191
  _transform(chunk, _enc, cb) {
174
192
  try {
175
193
  this.bytes += chunk.length;
176
194
  for (let i = chunk.indexOf(10); i !== -1; i = chunk.indexOf(10, i + 1)) this.rows++;
177
- if (this.rows >= this.#next) {
178
- this.#next = this.rows - this.rows % this.batchRows + this.batchRows;
179
- this.onProgress({
180
- table: this.table,
181
- rows: this.rows,
182
- bytes: this.bytes,
183
- done: false
184
- });
185
- }
195
+ this.#report();
186
196
  this.push(chunk);
187
197
  cb(null);
188
198
  } catch (err) {
@@ -194,21 +204,30 @@ var RowCounter = class extends Transform {
194
204
  //#endregion
195
205
  //#region src/admin/migrate-store/copy.ts
196
206
  /**
197
- * The migrate-store copy: streams every planned table from one REPEATABLE
198
- * READ, READ ONLY source snapshot into a target `bootStore` prepared, with
199
- * `COPY … TO STDOUT` piped into `COPY … FROM STDIN`. Nothing here writes to
200
- * the source.
207
+ * The migrate-store copy: converts a legacy or partitioned source into a
208
+ * target `bootStore` prepared in the `tables` layout, all from one REPEATABLE
209
+ * READ, READ ONLY source snapshot. `rows` is converted: each source type is
210
+ * streamed with a cursor, in `batchRows` batches, into its type table by
211
+ * `copyRows` (columns plus `extra`, ids into `row_ids`). Every other table
212
+ * moves with `COPY … TO STDOUT` piped into `COPY … FROM STDIN`. Nothing here
213
+ * writes to the source.
201
214
  *
202
215
  * Two phases around a re-boot, on one source snapshot. Phase 1 copies the
203
216
  * rows (and their shortIds) that declare storage: `_system`,
204
217
  * `_system_proposals` and every TypeProposal and FieldProposal. The second
205
- * boot replays those proposals as serve does, so every proposed type has its
206
- * partition before phase 2 copies everything else.
218
+ * boot replays those proposals as serve does, so every proposed type is
219
+ * registered, with its table and declared fields, before phase 2 copies
220
+ * everything else. A source type nothing declares gets a table with no
221
+ * columns, so its fields all go to `extra`, and is reported `defaulted`.
222
+ * Last, one promotion sweep moves the values of fields declared only by the
223
+ * replay (phase 1 converted them into `extra`) into their columns.
207
224
  *
208
225
  * Progress lives in the marker table on the target. A rerun after an
209
226
  * interruption restarts cleanly: it empties the planned tables, keeps the
210
227
  * seed rows the first run recorded, and copies again.
211
228
  */
229
+ /** The misfits listed per type, beside the full count. */
230
+ const MISFITS_LISTED = 100;
212
231
  /** A source table or column with nowhere to go on the target. Thrown before any data moves. */
213
232
  var MigrateStoreColumnError = class extends Error {
214
233
  table;
@@ -225,6 +244,8 @@ const SEED = "seed:";
225
244
  /** The `rows` / `short_ids` predicate phase 1 copies; phase 2 copies its complement. Both columns are NOT NULL. */
226
245
  const PHASE1_ROWS = "namespace IN ('_system', '_system_proposals') OR type IN ('TypeProposal', 'FieldProposal')";
227
246
  const PHASED = /* @__PURE__ */ new Set(["rows", "short_ids"]);
247
+ /** Converted rather than copied: the target has no `rows`. */
248
+ const CONVERTED = "rows";
228
249
  async function copyStore(source, boot, plan, opts) {
229
250
  let target = await boot();
230
251
  let open = target;
@@ -241,11 +262,16 @@ async function copyStore(source, boot, plan, opts) {
241
262
  }
242
263
  const src = await source.reserve();
243
264
  let streaming = false;
265
+ let snapshotOpen = true;
244
266
  try {
245
267
  await src.unsafe("BEGIN ISOLATION LEVEL REPEATABLE READ, READ ONLY");
268
+ await src.unsafe("SET LOCAL DateStyle = 'ISO, YMD'");
269
+ await src.unsafe("SET LOCAL TimeZone = 'UTC'");
246
270
  const keptSeeds = await reconcileSeeds(src, tsql, target);
271
+ const rowsPlan = plan.tables.find((t) => t.name === CONVERTED);
272
+ const misfits = {};
247
273
  const totals = /* @__PURE__ */ new Map();
248
- const copy = async (t, filter) => {
274
+ const copy = async (t, filter, tables) => {
249
275
  const start = performance.now();
250
276
  const prev = totals.get(t.name) ?? {
251
277
  rows: 0,
@@ -254,7 +280,8 @@ async function copyStore(source, boot, plan, opts) {
254
280
  };
255
281
  const counter = new RowCounter(t.name, opts.batchRows, opts.onProgress, prev);
256
282
  streaming = true;
257
- await copyTable(src, tsql, t, identity.get(t.name) ?? [], counter, filter);
283
+ if (t.name === CONVERTED) await convertRows(src, tsql, t, filter, tables, counter, opts.batchRows, misfits);
284
+ else await copyTable(src, tsql, t, identity.get(t.name) ?? [], counter, filter);
258
285
  streaming = false;
259
286
  totals.set(t.name, {
260
287
  rows: counter.rows,
@@ -262,7 +289,8 @@ async function copyStore(source, boot, plan, opts) {
262
289
  ms: prev.ms + Math.round(performance.now() - start)
263
290
  });
264
291
  };
265
- for (const t of plan.tables.filter((p) => PHASED.has(p.name))) await copy(t, PHASE1_ROWS);
292
+ const phase1 = await prepareTables(src, target, PHASE1_ROWS, false);
293
+ for (const t of plan.tables.filter((p) => PHASED.has(p.name))) await copy(t, PHASE1_ROWS, phase1.tables);
266
294
  await markStep(tsql, "copy:phase1");
267
295
  await snapshotRows(tsql);
268
296
  open = void 0;
@@ -271,10 +299,10 @@ async function copyStore(source, boot, plan, opts) {
271
299
  open = target;
272
300
  tsql = sqlOf(target);
273
301
  await assertRebootWroteNothing(tsql);
274
- const defaulted = await prepareStorage(src, target);
302
+ const phase2 = await prepareTables(src, target, null, true);
275
303
  const tables = [];
276
304
  for (const t of plan.tables) {
277
- await copy(t, PHASED.has(t.name) ? `NOT (${PHASE1_ROWS})` : null);
305
+ await copy(t, PHASED.has(t.name) ? `NOT (${PHASE1_ROWS})` : null, phase2.tables);
278
306
  await markStep(tsql, `copy:${t.name}`);
279
307
  const done = totals.get(t.name);
280
308
  tables.push({
@@ -290,16 +318,20 @@ async function copyStore(source, boot, plan, opts) {
290
318
  }
291
319
  await markStep(tsql, "copy:phase2");
292
320
  await advanceSequences(tsql, identity);
293
- await markStep(tsql, "copy:done");
294
321
  await src.unsafe("COMMIT");
322
+ snapshotOpen = false;
295
323
  src.release();
324
+ if (rowsPlan !== void 0) await settleFields(target, opts.batchRows);
325
+ await markStep(tsql, "copy:done");
296
326
  return {
297
327
  tables,
298
328
  keptSeeds,
299
- defaulted,
329
+ defaulted: phase2.defaulted,
330
+ misfits,
300
331
  target
301
332
  };
302
333
  } catch (err) {
334
+ if (!snapshotOpen) throw err;
303
335
  const released = src.unsafe("ROLLBACK").catch(() => void 0).then(() => src.release()).catch(() => void 0);
304
336
  if (!streaming) await released;
305
337
  throw err;
@@ -313,25 +345,46 @@ function sqlOf(target) {
313
345
  if (target.sql === null) throw new Error("migrate-store copies into a postgres target only");
314
346
  return target.sql;
315
347
  }
348
+ const quoteIdent = (name) => `"${name.replace(/"/g, "\"\"")}"`;
349
+ /** The target's type tables, each with its (quoted) relation. */
350
+ async function typeTablesOf(tsql) {
351
+ return (await tsql`
352
+ SELECT namespace, type, table_name FROM substrate_type_tables ORDER BY namespace, type`).map((r) => ({
353
+ namespace: r.namespace,
354
+ type: r.type,
355
+ rel: quoteIdent(r.table_name)
356
+ }));
357
+ }
358
+ /**
359
+ * SQL over every target row: `(namespace, type, id, h)`, `h` an md5 of the
360
+ * stored row with its null columns dropped, so a column added by a re-boot
361
+ * changes no row's hash. `null` when there are no type tables.
362
+ */
363
+ async function targetRowsSql(tsql) {
364
+ const tables = await typeTablesOf(tsql);
365
+ if (tables.length === 0) return null;
366
+ const lit = (s) => `'${s.replace(/'/g, "''")}'`;
367
+ return tables.map((t) => `SELECT ${lit(t.namespace)}::text AS namespace, ${lit(t.type)}::text AS type, x.id, md5(jsonb_strip_nulls(to_jsonb(x))::text) AS h FROM ${t.rel} x`).join(" UNION ALL ");
368
+ }
316
369
  const SNAPSHOT = "substrate_migrate_store_reboot_rows";
317
370
  /** Records every target row's key and content hash, for `assertRebootWroteNothing`. */
318
371
  async function snapshotRows(tsql) {
319
372
  await tsql`DROP TABLE IF EXISTS ${tsql(SNAPSHOT)}`;
320
- await tsql`CREATE UNLOGGED TABLE ${tsql(SNAPSHOT)} AS
321
- SELECT namespace, type, id, md5(r::text) AS h FROM rows r`;
373
+ const all = await targetRowsSql(tsql);
374
+ await tsql.unsafe(`CREATE UNLOGGED TABLE ${quoteIdent(SNAPSHOT)} AS ${all ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false"}`);
322
375
  }
323
376
  /** Refuses a re-boot that added, removed or changed a row, naming up to ten of each. */
324
377
  async function assertRebootWroteNothing(tsql) {
325
- const snap = tsql(SNAPSHOT);
326
- const diff = await tsql`
378
+ const all = await targetRowsSql(tsql) ?? "SELECT NULL::text AS namespace, NULL::text AS type, NULL::text AS id, NULL::text AS h WHERE false";
379
+ const diff = await tsql.unsafe(`
327
380
  SELECT CASE WHEN s.id IS NULL THEN 'added' WHEN r.id IS NULL THEN 'removed' ELSE 'changed' END AS change,
328
381
  coalesce(r.namespace, s.namespace) AS namespace, coalesce(r.type, s.type) AS type,
329
382
  coalesce(r.id, s.id) AS id
330
- FROM (SELECT namespace, type, id, md5(x::text) AS h FROM rows x) r
331
- FULL JOIN ${snap} s USING (namespace, type, id)
383
+ FROM (${all}) r
384
+ FULL JOIN ${quoteIdent(SNAPSHOT)} s USING (namespace, type, id)
332
385
  WHERE s.id IS NULL OR r.id IS NULL OR r.h <> s.h
333
- ORDER BY 1, 2, 3, 4 LIMIT 30`;
334
- await tsql`DROP TABLE ${snap}`;
386
+ ORDER BY 1, 2, 3, 4 LIMIT 30`);
387
+ await tsql`DROP TABLE ${tsql(SNAPSHOT)}`;
335
388
  if (diff.length === 0) return;
336
389
  const named = [
337
390
  "added",
@@ -345,15 +398,16 @@ async function assertRebootWroteNothing(tsql) {
345
398
  }
346
399
  /** Refuses a plan column the target lacks; returns each table's identity-always columns among the planned ones. */
347
400
  async function checkColumns(tsql, plan) {
401
+ const copied = plan.tables.filter((t) => t.name !== CONVERTED);
348
402
  const rows = await tsql`
349
403
  SELECT c.relname AS "table", a.attname AS col, a.attidentity AS ident
350
404
  FROM pg_attribute a
351
405
  JOIN pg_class c ON c.oid = a.attrelid
352
406
  JOIN pg_namespace n ON n.oid = c.relnamespace
353
- WHERE n.nspname = current_schema() AND c.relname = ANY(${plan.tables.map((t) => t.name)}::text[])
407
+ WHERE n.nspname = current_schema() AND c.relname = ANY(${copied.map((t) => t.name)}::text[])
354
408
  AND c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped AND a.attgenerated = ''`;
355
409
  const identity = /* @__PURE__ */ new Map();
356
- for (const t of plan.tables) {
410
+ for (const t of copied) {
357
411
  const cols = rows.filter((r) => r.table === t.name);
358
412
  if (cols.length === 0) throw new MigrateStoreColumnError(t.name, null);
359
413
  const have = new Set(cols.map((c) => c.col));
@@ -374,6 +428,18 @@ function inKeys(sql, keys) {
374
428
  return sql`(namespace, type, id) IN (SELECT * FROM unnest(
375
429
  ${keys.map((k) => k.namespace)}::text[], ${keys.map((k) => k.type)}::text[], ${keys.map((k) => k.id)}::text[]))`;
376
430
  }
431
+ /**
432
+ * Deletes target rows: those in `keys`, or (`except`) all but those. From each
433
+ * type table and from `row_ids`, whose delete cascades to `short_ids`.
434
+ */
435
+ async function deleteTargetRows(t, keys, except) {
436
+ for (const table of await typeTablesOf(t)) {
437
+ const ids = keys.filter((k) => k.namespace === table.namespace && k.type === table.type).map((k) => k.id);
438
+ if (!except && ids.length === 0) continue;
439
+ await t.unsafe(`DELETE FROM ${table.rel} WHERE ${except ? "NOT " : ""}(id = ANY($1::text[]))`, [ids]);
440
+ }
441
+ await t`DELETE FROM row_ids WHERE ${except ? t`NOT` : t``} ${inKeys(t, keys)}`;
442
+ }
377
443
  /** After an interrupted run: empties every planned table but the seed rows that run recorded. True when it did. */
378
444
  async function restartIfStarted(tsql, plan) {
379
445
  const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
@@ -387,10 +453,10 @@ async function restartIfStarted(tsql, plan) {
387
453
  id
388
454
  };
389
455
  });
390
- const others = plan.tables.map((p) => p.name).filter((n) => n !== "rows" && n !== "short_ids");
456
+ const others = plan.tables.map((p) => p.name).filter((n) => !PHASED.has(n));
391
457
  await tsql.begin(async (tx) => {
392
458
  const t = tx;
393
- await t`DELETE FROM rows WHERE NOT ${inKeys(t, seeds)}`;
459
+ await deleteTargetRows(t, seeds, true);
394
460
  if (others.length > 0) await t`TRUNCATE ${t(others)}`;
395
461
  });
396
462
  return true;
@@ -401,7 +467,7 @@ async function restartIfStarted(tsql, plan) {
401
467
  * a type the target registers, as everything the prepare step writes is.
402
468
  */
403
469
  async function reconcileSeeds(src, tsql, target) {
404
- const present = await tsql`SELECT namespace, type, id FROM rows ORDER BY namespace, type, id`;
470
+ const present = await tsql`SELECT namespace, type, id FROM row_ids ORDER BY namespace, type, id`;
405
471
  const inSource = present.length === 0 ? [] : await src`SELECT namespace, type, id FROM rows WHERE ${inKeys(src, [...present])}`;
406
472
  const replaced = new Set(inSource.map(keyOf));
407
473
  const kept = present.filter((r) => !replaced.has(keyOf(r))).map(({ namespace, type, id }) => ({
@@ -413,7 +479,11 @@ async function reconcileSeeds(src, tsql, target) {
413
479
  const marker = tsql(MIGRATE_STORE_MARKER_TABLE);
414
480
  await tsql.begin(async (tx) => {
415
481
  const t = tx;
416
- if (inSource.length > 0) await t`DELETE FROM rows WHERE ${inKeys(t, [...inSource])}`;
482
+ if (inSource.length > 0) await deleteTargetRows(t, inSource.map(({ namespace, type, id }) => ({
483
+ namespace,
484
+ type,
485
+ id
486
+ })), false);
417
487
  await t`DELETE FROM ${marker} WHERE starts_with(step, ${SEED}) OR starts_with(step, 'copy:')`;
418
488
  const steps = [STARTED, ...kept.map((k) => SEED + keyOf(k))];
419
489
  await t`INSERT INTO ${marker} ${t(steps.map((step) => ({
@@ -428,26 +498,109 @@ function registeredDataset(target, ns, type) {
428
498
  if (target.engine.registryFor(ns)?.get(type)?.kind !== "type") return void 0;
429
499
  return target.datasets.get(ns);
430
500
  }
431
- /** Gives every source type the target registers its partition, so its rows never pass through DEFAULT. */
432
- async function prepareStorage(src, target) {
433
- const pairs = await src`SELECT DISTINCT namespace, type FROM rows ORDER BY namespace, type`;
501
+ /**
502
+ * Gives every source type `filter` picks its table and returns their
503
+ * descriptors: a registered type's through its dataset, with its declared
504
+ * fields and unique indexes; any other's with no field columns (its fields
505
+ * all go to `extra`), listed in `defaulted` when `report` is set.
506
+ */
507
+ async function prepareTables(src, target, filter, report) {
508
+ const tsql = sqlOf(target);
509
+ const pairs = await src`SELECT DISTINCT namespace, type FROM rows ${filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`} ORDER BY namespace, type`;
510
+ const tables = /* @__PURE__ */ new Map();
434
511
  const defaulted = [];
512
+ const [uuid] = await tsql`
513
+ SELECT EXISTS (SELECT 1 FROM migrations WHERE version = ${UUID_CHECK_MIGRATION}) AS on`;
435
514
  for (const { namespace, type } of pairs) {
515
+ const key = `${namespace}/${type}`;
436
516
  const dataset = registeredDataset(target, namespace, type);
437
- if (dataset?.ensureTypeStorage === void 0) {
438
- defaulted.push({
517
+ if (dataset === void 0 && report) defaulted.push(key);
518
+ try {
519
+ if (dataset !== void 0) await dataset.ensureTypeStorage?.(type, { indexes: "unique" });
520
+ else await ensureTypeTable(tsql, {
439
521
  namespace,
440
- type
522
+ type,
523
+ fields: {},
524
+ newFields: "column",
525
+ enforceUuidCheck: uuid?.on === true,
526
+ createOnly: true
441
527
  });
442
- continue;
443
- }
444
- try {
445
- await dataset.ensureTypeStorage(type, { indexes: "unique" });
446
528
  } catch (err) {
447
- throw new Error(`preparing storage for ${namespace}/${type} failed: ${err.message}`, { cause: err });
529
+ throw new Error(`preparing storage for ${key} failed: ${err.message}`, { cause: err });
448
530
  }
531
+ const table = await loadTypeTable(tsql, namespace, type);
532
+ if (table === void 0) throw new Error(`preparing storage for ${key} made no table`);
533
+ tables.set(key, table);
449
534
  }
450
- return defaulted;
535
+ return {
536
+ tables,
537
+ defaulted
538
+ };
539
+ }
540
+ /**
541
+ * Converts the source rows `filter` picks, type by type, each streamed with a
542
+ * cursor in `batchRows` batches into its type table. A row the table already
543
+ * holds, or a refusal (a declared unique index), fails the copy loudly.
544
+ */
545
+ async function convertRows(src, tsql, t, filter, tables, counter, batchRows, misfits) {
546
+ const where = filter === null ? src`` : src`AND (${src.unsafe(filter)})`;
547
+ const seq = t.columns.includes("seq") ? src`seq::text` : src`NULL::text`;
548
+ const pairs = await src`
549
+ SELECT DISTINCT namespace, type FROM rows WHERE true ${where} ORDER BY namespace, type`;
550
+ for (const { namespace, type } of pairs) {
551
+ const key = `${namespace}/${type}`;
552
+ const table = tables.get(key);
553
+ if (table === void 0) throw new Error(`internal: no table prepared for ${key}`);
554
+ const cursor = src`
555
+ SELECT id, body::text AS body, deleted_at::text, created_at::text, updated_at::text,
556
+ created_by_kind, created_by_id, updated_by_kind, updated_by_id,
557
+ ${seq} AS seq
558
+ FROM rows WHERE namespace = ${namespace} AND type = ${type} ${where}
559
+ ORDER BY id`.cursor(batchRows);
560
+ for await (const batch of cursor) {
561
+ let result;
562
+ try {
563
+ result = await copyRows(tsql, table, batch.map((r) => ({
564
+ ...r,
565
+ body: parseSourceBody(r.body)
566
+ })));
567
+ } catch (err) {
568
+ throw rowsError(key, err);
569
+ }
570
+ if (result.inserted !== batch.length) throw new Error(`copying rows failed: ${key}: ${batch.length - result.inserted} source row(s) were already on the target`);
571
+ if (result.misfits.length > 0) {
572
+ const listed = misfits[key] ?? {
573
+ count: 0,
574
+ first: []
575
+ };
576
+ listed.count += result.misfits.length;
577
+ listed.first.push(...result.misfits.slice(0, 100 - listed.first.length));
578
+ misfits[key] = listed;
579
+ }
580
+ counter.add(batch.length, batch.reduce((n, r) => n + Buffer.byteLength(r.body), 0));
581
+ }
582
+ }
583
+ }
584
+ /** A conversion failure as one line naming the type; a unique refusal names the declared index and the key. */
585
+ function rowsError(key, err) {
586
+ const pg = err;
587
+ if (pg?.code === "23505" && typeof pg.constraint_name === "string") return new Error(`copying rows failed: ${key}: unique index ${pg.constraint_name} refuses a source row whose key another row already holds${typeof pg.detail === "string" ? ` (${pg.detail})` : ""}. Resolve the duplicate in the source, then rerun migrate-store, which restarts cleanly.`, { cause: err });
588
+ const detail = typeof pg?.detail === "string" ? ` (${pg.detail})` : "";
589
+ return new Error(`copying rows failed: ${key}: ${err.message ?? String(err)}${detail}`, { cause: err });
590
+ }
591
+ /**
592
+ * One promotion sweep over the target: a field declared only once the re-boot
593
+ * replayed its proposal had its phase-1 values converted into `extra`, so it
594
+ * is `promoting` (or `pending`) until moved into its column. Fails loudly,
595
+ * naming each field, when one is not promoted.
596
+ */
597
+ async function settleFields(target, batchRows) {
598
+ const stuck = (await target.dataset.promotePendingFields?.({
599
+ lockTimeoutMs: 6e4,
600
+ batchRows,
601
+ noWriters: true
602
+ }) ?? []).filter((e) => e.kind !== "promoted");
603
+ if (stuck.length > 0) throw new Error(`settling declared fields failed: ${stuck.map((e) => `${e.namespace}/${e.type}.${e.field} ${e.kind}${e.error ? ` (${e.error})` : ""}`).join("; ")}. Rerun migrate-store, which restarts cleanly.`);
451
604
  }
452
605
  async function copyTable(src, tsql, t, identityCols, counter, filter) {
453
606
  const where = filter === null ? src`` : src`WHERE ${src.unsafe(filter)}`;
@@ -533,6 +686,7 @@ async function inspectSource(sql) {
533
686
  await assertStoreSchema(tx);
534
687
  const layout = await detectLayout(tx);
535
688
  if (layout === "absent") throw new Error("the source schema holds no rows table");
689
+ if (layout === "tables") throw new Error("the source is already in the tables layout; migrate-store copies a legacy or partitioned store");
536
690
  const [version] = await tx`SELECT current_setting('server_version_num')::int AS v`;
537
691
  const [ledger] = await tx`SELECT to_regclass('migrations') IS NOT NULL AS present`;
538
692
  const appliedMigrations = ledger?.present ? (await tx`SELECT version FROM migrations ORDER BY version`).map((r) => r.version) : [];
@@ -583,9 +737,15 @@ async function inspectSource(sql) {
583
737
  SELECT a.atttypmod AS dims FROM pg_attribute a
584
738
  WHERE a.attrelid = to_regclass('embeddings') AND a.attname = 'embedding' AND NOT a.attisdropped` : [];
585
739
  const extensions = await tx`SELECT extname, extversion FROM pg_extension ORDER BY extname`;
586
- const [others] = await tx`
587
- SELECT count(*)::text AS n FROM pg_stat_activity
588
- WHERE datname = current_database() AND pid <> pg_backend_pid() AND backend_type = 'client backend'`;
740
+ const sessions = await tx`
741
+ SELECT pid, extract(epoch FROM backend_start)::text AS start, usename,
742
+ application_name AS "applicationName", client_addr::text AS "clientAddr",
743
+ backend_type AS "backendType", state
744
+ FROM pg_stat_activity
745
+ WHERE datname = current_database() AND pid <> pg_backend_pid()
746
+ -- Another role's session shows a NULL backend_type to a non-monitoring role.
747
+ AND (backend_type = 'client backend' OR (backend_type IS NULL AND usesysid IS NOT NULL))
748
+ ORDER BY pid`;
589
749
  return {
590
750
  serverVersionNum: version?.v ?? 0,
591
751
  layout,
@@ -604,7 +764,8 @@ async function inspectSource(sql) {
604
764
  nonUuidIds,
605
765
  vectorDims: dims !== void 0 && dims.dims > 0 ? dims.dims : null,
606
766
  extensions: Object.fromEntries(extensions.map((e) => [e.extname, e.extversion])),
607
- otherSessions: Number(others?.n ?? 0)
767
+ otherSessions: sessions.length,
768
+ sessions
608
769
  };
609
770
  });
610
771
  }
@@ -612,6 +773,19 @@ async function inspectSource(sql) {
612
773
  //#endregion
613
774
  //#region src/admin/migrate-store/preflight.ts
614
775
  /** Pure preflight: turns a source inventory plus target facts into refusals, warnings and a copy plan. */
776
+ /** Each old layout's baseline ledger version, as the last release that booted it recorded it. */
777
+ const OLD_BASELINES = {
778
+ legacy: "0001_baseline",
779
+ partitioned: "0001p_partitioned_baseline"
780
+ };
781
+ /** Every static ledger version an old-layout store can hold. */
782
+ const OLD_MIGRATIONS = /* @__PURE__ */ new Set([
783
+ ...Object.values(OLD_BASELINES),
784
+ UUID_CHECK_MIGRATION,
785
+ "0003_safe_casts",
786
+ "0004_visible_seq"
787
+ ]);
788
+ /** `rows` is converted into type tables, which writes `row_ids` too; the rest are copied as they are. */
615
789
  const ENGINE_COPIED = [
616
790
  "rows",
617
791
  "short_ids",
@@ -634,13 +808,13 @@ function preflight(source, target, opts) {
634
808
  const warnings = [];
635
809
  if (source.serverVersionNum < PG17) refusals.push(`source is PostgreSQL ${source.serverVersionNum} (server_version_num); 170000 or later is required`);
636
810
  if (target.serverVersionNum < PG18) refusals.push(`target is PostgreSQL ${target.serverVersionNum} (server_version_num); 180000 or later is required`);
637
- const baselines = [getMigrationList(true, "legacy")[0], getMigrationList(true, "partitioned")[0]].map((m) => m?.version);
638
- const known = new Set([...getMigrationList(true, "legacy"), ...getMigrationList(true, "partitioned")].map((m) => m.version));
639
- const unknownVersions = source.appliedMigrations.filter((v) => !known.has(v) && !isDerivedMigration(v));
811
+ const baselines = Object.values(OLD_BASELINES);
812
+ const unknownVersions = source.appliedMigrations.filter((v) => !OLD_MIGRATIONS.has(v) && !isDerivedMigration(v));
640
813
  if (unknownVersions.length > 0) refusals.push(`source ledger holds migrations this binary does not know: ${unknownVersions.join(", ")}`);
641
814
  if (!source.appliedMigrations.some((v) => baselines.includes(v))) refusals.push(`source ledger lacks ${baselines.join(" / ")}; found: ${source.appliedMigrations.join(", ") || "no migrations"}`);
815
+ if (target.layout === "legacy" || target.layout === "partitioned") refusals.push(`target is a ${target.layout} store; migrate-store copies into the tables layout, so point it at an empty database or schema`);
642
816
  if (!target.empty && !target.createdByUs) refusals.push("target already holds data and was not created by migrate-store");
643
- const otherBaseline = baselines.find((b) => b !== getMigrationList(true, source.layout)[0]?.version);
817
+ const otherBaseline = baselines.find((b) => b !== OLD_BASELINES[source.layout]);
644
818
  if (otherBaseline !== void 0 && source.appliedMigrations.includes(otherBaseline)) refusals.push(`source ledger records ${otherBaseline} but rows is ${source.layout}`);
645
819
  if (source.appliedMigrations.includes("0002_uuid_format_check") && !target.enforceUuidCheck) refusals.push(`source enforces the UUID check (${UUID_CHECK_MIGRATION}) but the target would not: run migrate-store with the service's environment, which sets SUBSTRATE_ENFORCE_UUID_CHECK=true`);
646
820
  if (target.enforceUuidCheck && source.nonUuidIds.length > 0) {
@@ -655,7 +829,9 @@ function preflight(source, target, opts) {
655
829
  ]);
656
830
  const unknownTables = source.tables.filter((t) => !knownTables.has(t) && !skip.has(t));
657
831
  for (const t of unknownTables) refusals.push(`source table "${t}" is not known to migrate-store; load the plugin that owns it, or pass --skip-table ${t}`);
658
- for (const t of skip) if (!source.tables.includes(t)) warnings.push(`--skip-table ${t} matches no source table`);
832
+ const copied = /* @__PURE__ */ new Set([...ENGINE_COPIED, ...opts.pluginTables]);
833
+ for (const t of skip) if (copied.has(t)) refusals.push(`--skip-table ${t} names a table migrate-store copies; it cannot be skipped. --skip-table is for tables migrate-store does not know`);
834
+ else if (!source.tables.includes(t)) warnings.push(`--skip-table ${t} matches no source table`);
659
835
  if (source.vectorDims !== null && target.recallDims !== null && source.vectorDims !== target.recallDims) refusals.push(`recall embedding dimensions differ: source ${source.vectorDims}, target ${target.recallDims}`);
660
836
  const present = new Set(source.tables);
661
837
  const plan = {
@@ -665,7 +841,7 @@ function preflight(source, target, opts) {
665
841
  bytes: source.tableBytes[name] ?? 0
666
842
  })),
667
843
  estimatedBytes: 0,
668
- skipped: [...skip].filter((t) => present.has(t))
844
+ skipped: [...skip].filter((t) => present.has(t) && !copied.has(t))
669
845
  };
670
846
  plan.estimatedBytes = 2 * plan.tables.reduce((n, t) => n + t.bytes, 0);
671
847
  for (const t of plan.tables) {
@@ -679,7 +855,12 @@ function preflight(source, target, opts) {
679
855
  const msg = `free disk ${target.freeBytes} bytes is below the estimated need of ${plan.estimatedBytes} bytes`;
680
856
  (opts.dryRun ? warnings : refusals).push(msg);
681
857
  }
682
- if (source.otherSessions > 0) warnings.push(`source has ${source.otherSessions} other active session(s) on its database; writes after the snapshot are not copied, so stop the service first`);
858
+ if (source.otherSessions > 0) {
859
+ const named = source.sessions.slice(0, 10).map((s) => `usename=${s.usename ?? "hidden"} application_name=${s.applicationName ?? "hidden"} client_addr=${s.clientAddr ?? "local"} backend_type=${s.backendType ?? "hidden"} state=${s.state ?? "hidden"}`);
860
+ const more = source.sessions.length - named.length;
861
+ const list = named.length > 0 ? `: ${named.join("; ")}${more > 0 ? `; +${more} more` : ""}` : "";
862
+ (opts.dryRun ? warnings : refusals).push(`source has ${source.otherSessions} other active session(s) on its database; writes after the snapshot are not copied, so stop the service first${list}`);
863
+ }
683
864
  return {
684
865
  refusals,
685
866
  warnings,
@@ -690,25 +871,48 @@ function preflight(source, target, opts) {
690
871
  //#endregion
691
872
  //#region src/admin/migrate-store/verify.ts
692
873
  /**
693
- * The migrate-store verification: compares the source and the copy per
694
- * (namespace, type) and, only when everything agrees, drops the in-progress
695
- * marker so `serve` will boot the target. Both sides are read in their own
696
- * REPEATABLE READ, READ ONLY transaction; nothing here writes to the source.
874
+ * The migrate-store verification: compares the source with its conversion
875
+ * per (namespace, type) and, only when everything agrees and the target holds
876
+ * every declared index, drops the in-progress marker so `serve` will boot the
877
+ * target. Both sides are read in their own REPEATABLE READ, READ ONLY
878
+ * transaction; nothing here writes to the source.
697
879
  *
698
880
  * Target rows the prepare step seeded and the source lacks (`keptSeeds`) are
699
881
  * subtracted from the target side before every comparison.
700
882
  *
701
883
  * Every set is fingerprinted as `count(*)` plus the sum of a 64-bit prefix of
702
884
  * each row's md5. That is independent of row order, of either cluster's
703
- * collation, and of any join separator, and it builds no string.
885
+ * collation, and of any join separator, and it builds no string. Row content
886
+ * is compared as the dataset reads it: the target's row as `decodeRow` builds
887
+ * it, the source's body under the same read rule (an explicit null on a
888
+ * declared column field reads absent), both as jsonb text, so a number's
889
+ * stored text counts, plus every metadata column.
704
890
  */
705
891
  const quote = (cols) => cols.map((c) => `"${c.replace(/"/g, "\"\"")}"`).join(", ");
706
892
  /** SQL for a set's fingerprint: its size and an order-independent sum of per-row hashes of `expr`. */
707
893
  const fp = (expr) => `count(*)::text || ' ' || coalesce(sum(('x' || substr(md5(${expr}), 1, 16))::bit(64)::bigint::numeric), 0)::text`;
894
+ /**
895
+ * The declared indexes of every registered type that are not valid on the
896
+ * target: missing, invalid, building, failed, or built over a field that is
897
+ * not yet in its column. `uniqueOnly` leaves out the non-unique ones, which
898
+ * `--skip-index-build` defers to serve.
899
+ */
900
+ async function missingIndexes(target, uniqueOnly) {
901
+ const out = [];
902
+ for (const { ns, type, dataset } of registeredTypes(target.engine, target.datasets)) {
903
+ const report = await dataset?.describeStorage?.(type);
904
+ for (const i of report?.indexes ?? []) if (i.state !== "valid" && (i.unique || !uniqueOnly)) out.push({
905
+ type: `${ns}/${type}`,
906
+ index: i.name
907
+ });
908
+ }
909
+ return out;
910
+ }
708
911
  async function verifyStore(source, target, opts) {
709
912
  const readOnly = "isolation level repeatable read read only";
710
- const src = await source.begin(readOnly, (tx) => snapshot(tx, [], null, opts));
711
- const tgt = await target.begin(readOnly, (tx) => snapshot(tx, opts.keptSeeds, src, opts));
913
+ const types = await target.begin(readOnly, (tx) => typeTables(tx));
914
+ const src = await source.begin(readOnly, (tx) => snapshot(tx, "source", [], null, opts, types));
915
+ const tgt = await target.begin(readOnly, (tx) => snapshot(tx, "target", opts.keptSeeds, src, opts, types));
712
916
  const mismatches = [];
713
917
  const keys = [.../* @__PURE__ */ new Set([...src.entries.keys(), ...tgt.entries.keys()])].sort();
714
918
  for (const k of keys) {
@@ -723,24 +927,30 @@ async function verifyStore(source, target, opts) {
723
927
  detail: `${e.what}: source ${a?.value ?? "absent"}, target ${b?.value ?? "absent"}`
724
928
  });
725
929
  }
726
- const expected = new Set(opts.defaulted.map((d) => JSON.stringify([d.namespace, d.type])));
727
- for (const w of (await opts.storageReport({ partitionBudget: opts.partitionBudget })).warnings) {
728
- const m = /^\S+ holds \d+ rows of (.+)\/([^/]+)$/.exec(w);
729
- if (m !== null && expected.has(JSON.stringify([m[1], m[2]]))) continue;
730
- mismatches.push({
731
- check: "storage",
732
- ...m !== null ? {
733
- namespace: m[1],
734
- type: m[2]
735
- } : {},
736
- detail: w
930
+ for (const [key, table] of types) {
931
+ const [namespace, type] = [table.namespace, table.type];
932
+ const unsettled = [...table.pending.map((field) => ({
933
+ field,
934
+ state: "pending"
935
+ })), ...table.columns().filter((c) => c.state === "promoting")];
936
+ for (const f of unsettled) mismatches.push({
937
+ check: "fields",
938
+ namespace,
939
+ type,
940
+ detail: `field ${f.field} of ${key} is ${f.state}, so its values are not all in its column`
737
941
  });
738
942
  }
739
- const ok = mismatches.length === 0;
943
+ for (const w of (await opts.storageReport({ partitionBudget: opts.partitionBudget })).warnings) mismatches.push({
944
+ check: "storage",
945
+ detail: w
946
+ });
947
+ const missing = await opts.missingIndexes();
948
+ const ok = mismatches.length === 0 && missing.length === 0;
740
949
  if (ok) await target`DROP TABLE IF EXISTS ${target(MIGRATE_STORE_MARKER_TABLE)}`;
741
950
  return {
742
951
  ok,
743
- mismatches
952
+ mismatches,
953
+ missingIndexes: missing
744
954
  };
745
955
  }
746
956
  const COPY_MILESTONES = /* @__PURE__ */ new Set([
@@ -751,9 +961,9 @@ const COPY_MILESTONES = /* @__PURE__ */ new Set([
751
961
  ]);
752
962
  /**
753
963
  * What a finished copy left behind, for a verification that has no
754
- * `CopyResult`: the kept seeds from the marker table, and the types the target
755
- * does not register (their rows sit in a DEFAULT partition), and the copied
756
- * tables, which are the marker's `copy:<table>` steps.
964
+ * `CopyResult`: the kept seeds from the marker table, the types the target
965
+ * does not register (their fields all sit in `extra`), and the copied tables,
966
+ * which are the marker's `copy:<table>` steps.
757
967
  */
758
968
  async function readCopyState(target) {
759
969
  const sql = target.sql;
@@ -773,12 +983,31 @@ async function readCopyState(target) {
773
983
  id
774
984
  };
775
985
  }).sort((a, b) => (a.namespace + a.type + a.id).localeCompare(b.namespace + b.type + b.id)),
776
- defaulted: (await sql`SELECT DISTINCT namespace, type FROM rows ORDER BY namespace, type`).filter((p) => registeredDataset(target, p.namespace, p.type)?.ensureTypeStorage === void 0),
986
+ defaulted: (await sql`
987
+ SELECT DISTINCT namespace, type FROM row_ids ORDER BY namespace, type`).filter((p) => registeredDataset(target, p.namespace, p.type) === void 0).map((p) => `${p.namespace}/${p.type}`),
777
988
  tables: steps.map((s) => s.step).filter((s) => s.startsWith("copy:") && !COPY_MILESTONES.has(s)).map((s) => s.slice(5)).sort()
778
989
  };
779
990
  }
780
- /** Every comparable fact about one side, keyed so the two sides diff by key. `other` is the source's snapshot when this is the target. */
781
- async function snapshot(tx, seeds, other, opts) {
991
+ /** The target's type tables, by `<ns>/<Type>`. */
992
+ async function typeTables(tx) {
993
+ const types = /* @__PURE__ */ new Map();
994
+ const listed = await tx`
995
+ SELECT namespace, type FROM substrate_type_tables ORDER BY namespace, type`;
996
+ for (const { namespace, type } of listed) {
997
+ const table = await loadTypeTable(tx, namespace, type);
998
+ if (table !== void 0) types.set(`${namespace}/${type}`, table);
999
+ }
1000
+ return types;
1001
+ }
1002
+ /** Each row's id with its versions: a row updated or deleted after the copy's snapshot changes it, wherever the sample falls. */
1003
+ const VERSIONS = `id || E'\n' || coalesce(updated_at::text, '') || E'\n' || coalesce(deleted_at::text, '')`;
1004
+ const META = `deleted_at::text, created_at::text, updated_at::text, created_by_kind, created_by_id, updated_by_kind, updated_by_id`;
1005
+ /**
1006
+ * Every comparable fact about one side, keyed so the two sides diff by key.
1007
+ * `other` is the source's snapshot when this is the target; `types` are the
1008
+ * target's type tables, which give the source its read rule.
1009
+ */
1010
+ async function snapshot(tx, side, seeds, other, opts, types) {
782
1011
  await tx`SET LOCAL TimeZone = 'UTC'`;
783
1012
  await tx`SET LOCAL DateStyle = 'ISO, YMD'`;
784
1013
  await tx`SET LOCAL extra_float_digits = 3`;
@@ -787,6 +1016,7 @@ async function snapshot(tx, seeds, other, opts) {
787
1016
  const add = (e) => entries.set(`${e.check}\0${e.namespace ?? ""}\0${e.type ?? ""}\0${e.what}`, e);
788
1017
  const notSeed = (ns, ty, id) => tx`(${tx(ns)}, ${tx(ty)}, ${tx(id)}) NOT IN (SELECT * FROM unnest(
789
1018
  ${seeds.map((k) => k.namespace)}::text[], ${seeds.map((k) => k.type)}::text[], ${seeds.map((k) => k.id)}::text[]))`;
1019
+ const seedIds = (ns, ty) => seeds.filter((k) => k.namespace === ns && k.type === ty).map((k) => k.id);
790
1020
  const colsOf = async (table) => {
791
1021
  const have = await tx`
792
1022
  SELECT a.attname AS col FROM pg_attribute a
@@ -798,33 +1028,101 @@ async function snapshot(tx, seeds, other, opts) {
798
1028
  columns.set(table, cols);
799
1029
  return cols;
800
1030
  };
801
- const counts = await tx`
802
- SELECT namespace, type,
803
- count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
804
- count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
805
- ${tx.unsafe(fp("id"))} AS ids
806
- FROM rows WHERE ${notSeed("namespace", "type", "id")} GROUP BY namespace, type`;
807
- for (const r of counts) {
808
- const at = {
809
- namespace: r.namespace,
810
- type: r.type
811
- };
812
- add({
813
- check: "count",
814
- ...at,
815
- what: "live rows",
816
- value: r.live
817
- });
818
- add({
819
- check: "count",
820
- ...at,
821
- what: "deleted rows",
822
- value: r.deleted
823
- });
824
- add({
1031
+ const limit = opts.fullChecksum ? Number.MAX_SAFE_INTEGER : opts.sampleRows;
1032
+ const counted = [];
1033
+ if (side === "source") {
1034
+ const counts = await tx`
1035
+ SELECT namespace, type,
1036
+ count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
1037
+ count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
1038
+ ${tx.unsafe(fp("id"))} AS ids,
1039
+ ${tx.unsafe(fp(VERSIONS))} AS versions
1040
+ FROM rows GROUP BY namespace, type`;
1041
+ for (const r of counts) {
1042
+ const at = {
1043
+ namespace: r.namespace,
1044
+ type: r.type
1045
+ };
1046
+ add({
1047
+ check: "count",
1048
+ ...at,
1049
+ what: "live rows",
1050
+ value: r.live
1051
+ });
1052
+ add({
1053
+ check: "count",
1054
+ ...at,
1055
+ what: "deleted rows",
1056
+ value: r.deleted
1057
+ });
1058
+ add({
1059
+ check: "ids",
1060
+ ...at,
1061
+ what: "id set",
1062
+ value: r.ids
1063
+ });
1064
+ add({
1065
+ check: "ids",
1066
+ ...at,
1067
+ what: "registered ids (row_ids)",
1068
+ value: r.ids
1069
+ });
1070
+ add({
1071
+ check: "versions",
1072
+ ...at,
1073
+ what: "row versions (updated_at, deleted_at)",
1074
+ value: r.versions
1075
+ });
1076
+ counted.push(at);
1077
+ }
1078
+ } else {
1079
+ for (const table of types.values()) {
1080
+ const { namespace, type } = table;
1081
+ const skip = seedIds(namespace, type);
1082
+ const [r] = await tx.unsafe(`SELECT count(*) FILTER (WHERE deleted_at IS NULL)::text AS live,
1083
+ count(*) FILTER (WHERE deleted_at IS NOT NULL)::text AS deleted,
1084
+ ${fp("id")} AS ids, ${fp(VERSIONS)} AS versions
1085
+ FROM ${table.from} WHERE NOT (id = ANY($1::text[]))`, [skip]);
1086
+ const at = {
1087
+ namespace,
1088
+ type
1089
+ };
1090
+ if (r !== void 0 && Number(r.live) + Number(r.deleted) > 0) {
1091
+ add({
1092
+ check: "count",
1093
+ ...at,
1094
+ what: "live rows",
1095
+ value: r.live
1096
+ });
1097
+ add({
1098
+ check: "count",
1099
+ ...at,
1100
+ what: "deleted rows",
1101
+ value: r.deleted
1102
+ });
1103
+ add({
1104
+ check: "ids",
1105
+ ...at,
1106
+ what: "id set",
1107
+ value: r.ids
1108
+ });
1109
+ add({
1110
+ check: "versions",
1111
+ ...at,
1112
+ what: "row versions (updated_at, deleted_at)",
1113
+ value: r.versions
1114
+ });
1115
+ counted.push(at);
1116
+ }
1117
+ }
1118
+ const registered = await tx`
1119
+ SELECT namespace, type, ${tx.unsafe(fp("id"))} AS ids
1120
+ FROM row_ids WHERE ${notSeed("namespace", "type", "id")} GROUP BY namespace, type`;
1121
+ for (const r of registered) add({
825
1122
  check: "ids",
826
- ...at,
827
- what: "id set",
1123
+ namespace: r.namespace,
1124
+ type: r.type,
1125
+ what: "registered ids (row_ids)",
828
1126
  value: r.ids
829
1127
  });
830
1128
  }
@@ -852,14 +1150,30 @@ async function snapshot(tx, seeds, other, opts) {
852
1150
  value: r.f
853
1151
  });
854
1152
  }
855
- const rowCols = await colsOf("rows");
856
- const limit = opts.fullChecksum ? 2147483647 : opts.sampleRows;
857
- for (const { namespace, type } of counts) {
858
- const [r] = await tx`
859
- SELECT ${tx.unsafe(fp("h"))} AS f FROM (
860
- SELECT row(${tx.unsafe(quote(rowCols))})::text AS h FROM rows
861
- WHERE namespace = ${namespace} AND type = ${type} AND ${notSeed("namespace", "type", "id")}
862
- ORDER BY md5(id) COLLATE "C" LIMIT ${limit}) s`;
1153
+ const hasSeq = side === "source" ? (await colsOf("rows"))?.includes("seq") === true : other?.columns.get("rows")?.includes("seq") !== false;
1154
+ const meta = `row(${META}, ${hasSeq ? "seq::text" : "NULL::text"})::text`;
1155
+ for (const { namespace, type } of counted) {
1156
+ let rows;
1157
+ let params;
1158
+ if (side === "source") {
1159
+ const table = types.get(`${namespace}/${type}`);
1160
+ const declared = [...(table?.columns().filter((c) => c.state !== "orphaned") ?? []).map((c) => c.field), ...table?.pending ?? []];
1161
+ rows = `SELECT id || E'\n' || (SELECT coalesce(jsonb_object_agg(k, v), '{}'::jsonb) FROM jsonb_each(body) e(k, v)
1162
+ WHERE NOT (v = 'null'::jsonb AND k = ANY($3::text[])))::text || E'\n' || ${meta} AS h
1163
+ FROM rows WHERE namespace = $1 AND type = $2 ORDER BY md5(id) COLLATE "C" LIMIT ${limit}`;
1164
+ params = [
1165
+ namespace,
1166
+ type,
1167
+ declared
1168
+ ];
1169
+ } else {
1170
+ const ref = await typeTableRef(tx, namespace, type);
1171
+ if (ref === void 0) continue;
1172
+ rows = `SELECT x.id || E'\n' || ${ref.rowJson("x")}::text || E'\n' || ${meta} AS h FROM ${ref.from} x
1173
+ WHERE NOT (x.id = ANY($1::text[])) ORDER BY md5(x.id) COLLATE "C" LIMIT ${limit}`;
1174
+ params = [seedIds(namespace, type)];
1175
+ }
1176
+ const [r] = await tx.unsafe(`SELECT ${fp("h")} AS f FROM (${rows}) s`, params);
863
1177
  add({
864
1178
  check: "sample",
865
1179
  namespace,
@@ -899,9 +1213,9 @@ async function snapshot(tx, seeds, other, opts) {
899
1213
  //#endregion
900
1214
  //#region src/admin/commands/migrate-store.ts
901
1215
  /**
902
- * `substrate-admin migrate-store`: copies a store into a fresh, empty,
903
- * partitioned PostgreSQL 18 store. Preflight, prepare and copy, index builds,
904
- * verification, report. Connection strings come from named environment
1216
+ * `substrate-admin migrate-store`: converts a legacy or partitioned store into
1217
+ * a fresh, empty PostgreSQL 18 store in the `tables` layout. Preflight,
1218
+ * prepare and copy, index builds, verification, report. Connection strings come from named environment
905
1219
  * variables only, and nothing this command prints carries one.
906
1220
  */
907
1221
  const DEFAULT_FROM_ENV = "SUBSTRATE_MIGRATE_FROM_URL";
@@ -1043,6 +1357,8 @@ async function runMigrateStore(args, env, deps) {
1043
1357
  totalMs: 0,
1044
1358
  keptSeeds: [],
1045
1359
  defaulted: [],
1360
+ misfits: null,
1361
+ missingIndexes: [],
1046
1362
  pendingIndexes: [],
1047
1363
  mismatches: [],
1048
1364
  switchOver: null
@@ -1053,11 +1369,15 @@ async function runMigrateStore(args, env, deps) {
1053
1369
  report.totalMs = Math.round(performance.now() - started);
1054
1370
  return report;
1055
1371
  };
1372
+ let running;
1056
1373
  const phase = async (name, fn) => {
1057
1374
  deps.progress(`migrate-store: ${name}`);
1058
1375
  const t = performance.now();
1376
+ running = name;
1059
1377
  try {
1060
- return await fn();
1378
+ const out = await fn();
1379
+ running = void 0;
1380
+ return out;
1061
1381
  } finally {
1062
1382
  report.phases[name] = Math.round(performance.now() - t);
1063
1383
  }
@@ -1067,13 +1387,15 @@ async function runMigrateStore(args, env, deps) {
1067
1387
  if (report.refusals.length > 0) return finish(2);
1068
1388
  const targetEnv = {
1069
1389
  ...vars,
1070
- SUBSTRATE_DATABASE_URL: toUrl
1390
+ SUBSTRATE_DATABASE_URL: toUrl,
1391
+ HOST: "127.0.0.1",
1392
+ SUBSTRATE_SYSTEM_ACTOR: vars.SUBSTRATE_SYSTEM_ACTOR?.trim() || "migrate-store",
1393
+ SUBSTRATE_STREAM_TICKET_SECRET: vars.SUBSTRATE_STREAM_TICKET_SECRET?.trim() || randomBytes(32).toString("hex")
1071
1394
  };
1072
1395
  const pluginDir = resolvePluginDir(targetEnv);
1073
1396
  const boot = () => bootStore({
1074
1397
  config: parseEnv(targetEnv),
1075
1398
  pluginDir,
1076
- layout: "partitioned",
1077
1399
  allowMigrating: true,
1078
1400
  background: false
1079
1401
  });
@@ -1120,7 +1442,8 @@ async function runMigrateStore(args, env, deps) {
1120
1442
  await assertSchema(facts, args.toSchema, "target");
1121
1443
  await assertDistinct(src, facts);
1122
1444
  const source = await inspectSource(src);
1123
- source.otherSessions -= await sessionsOn(src, own);
1445
+ source.sessions = source.sessions.filter((s) => !(s.pid === own.pid && s.start === own.start));
1446
+ source.otherSessions = deps.otherSessions ?? source.sessions.length;
1124
1447
  const target = await targetFacts(facts, env, diskPath);
1125
1448
  const pluginTables = await storeTables(pluginDir);
1126
1449
  return {
@@ -1149,19 +1472,20 @@ async function runMigrateStore(args, env, deps) {
1149
1472
  }));
1150
1473
  target = copied.target;
1151
1474
  report.tables = copied.tables;
1475
+ report.misfits = copied.misfits;
1152
1476
  ({keptSeeds, defaulted} = copied);
1153
1477
  tables = pre.plan.tables.map((t) => t.name);
1154
1478
  }
1155
1479
  const booted = target;
1156
1480
  const indexFailures = args.skipIndexBuild ? [] : await phase("indexes", () => (deps.buildIndexes ?? buildDeferredIndexes)(booted.engine, booted.datasets));
1157
1481
  report.keptSeeds = keptSeeds.map((k) => `${k.namespace}/${k.type}/${k.id}`);
1158
- report.defaulted = defaulted.map((d) => `${d.namespace}/${d.type}`);
1482
+ report.defaulted = defaulted;
1159
1483
  const tsql = booted.sql;
1160
1484
  await deps.beforeVerify?.(tsql);
1161
1485
  const verified = await phase("verify", () => verifyStore(src, tsql, {
1162
1486
  keptSeeds,
1163
- defaulted,
1164
1487
  tables,
1488
+ missingIndexes: () => missingIndexes(booted, args.skipIndexBuild),
1165
1489
  fullChecksum: args.fullChecksum,
1166
1490
  sampleRows: 1e3,
1167
1491
  partitionBudget: parseEnv(targetEnv).partitionBudget,
@@ -1174,9 +1498,19 @@ async function runMigrateStore(args, env, deps) {
1174
1498
  }
1175
1499
  }));
1176
1500
  report.mismatches = verified.mismatches;
1501
+ report.missingIndexes = verified.missingIndexes;
1177
1502
  if (!verified.ok) return finish(3);
1178
1503
  report.switchOver = `stop serve, set SUBSTRATE_DATABASE_URL to the value of ${args.toEnv}${args.toSchema !== null ? ` with search_path ${args.toSchema},public (options=-c search_path=${args.toSchema},public)` : ""}, then start serve. The source is untouched: pointing back at it is the rollback.`;
1179
1504
  return finish(0);
1505
+ } catch (e) {
1506
+ const message = e instanceof Error ? e.message : String(e);
1507
+ if (e instanceof AdminCliError && (e.exitCode === 2 || e.exitCode === 3)) {
1508
+ report.refusals.push(message);
1509
+ return finish(e.exitCode);
1510
+ }
1511
+ report.error = message;
1512
+ if (running !== void 0) report.failedPhase = running;
1513
+ return finish(1);
1180
1514
  } finally {
1181
1515
  await target?.close().catch(() => void 0);
1182
1516
  await src.end({ timeout: 5 }).catch(() => void 0);
@@ -1228,22 +1562,16 @@ async function ownSession(sql) {
1228
1562
  FROM pg_stat_activity WHERE pid = pg_backend_pid()`;
1229
1563
  return r;
1230
1564
  }
1231
- /** 1 when `s` is a backend on `sql`'s database, else 0: our own target connection when both share one. */
1232
- async function sessionsOn(sql, s) {
1233
- const [r] = await sql`
1234
- SELECT count(*)::int AS n FROM pg_stat_activity
1235
- WHERE datname = current_database() AND pid = ${s.pid}
1236
- AND extract(epoch FROM backend_start)::text = ${s.start}`;
1237
- return r?.n ?? 0;
1238
- }
1239
1565
  async function targetFacts(sql, env, diskPath) {
1240
1566
  const [r] = await sql`
1241
1567
  SELECT current_setting('server_version_num')::int AS v, current_schema() AS schema,
1242
1568
  to_regclass(quote_ident(current_schema()) || '.rows') IS NOT NULL AS rows,
1569
+ to_regclass(quote_ident(current_schema()) || '.row_ids') IS NOT NULL AS ids,
1243
1570
  to_regclass(quote_ident(current_schema()) || '.' || quote_ident(${MIGRATE_STORE_MARKER_TABLE})) IS NOT NULL AS marker,
1244
1571
  to_regclass(quote_ident(current_schema()) || '.embeddings') IS NOT NULL AS embeddings`;
1245
1572
  const facts = r;
1246
- const empty = !facts.rows || (await sql`SELECT 1 FROM ${sql(facts.schema)}.rows LIMIT 1`).length === 0;
1573
+ const holds = async (table) => (await sql`SELECT 1 FROM ${sql(facts.schema)}.${sql(table)} LIMIT 1`).length > 0;
1574
+ const empty = !(facts.rows && await holds("rows")) && !(facts.ids && await holds("row_ids"));
1247
1575
  const [dims] = facts.embeddings ? await sql`
1248
1576
  SELECT atttypmod AS d FROM pg_attribute
1249
1577
  WHERE attrelid = to_regclass(quote_ident(${facts.schema}) || '.embeddings')
@@ -1262,6 +1590,7 @@ async function targetFacts(sql, env, diskPath) {
1262
1590
  }
1263
1591
  return {
1264
1592
  serverVersionNum: facts.v,
1593
+ layout: await detectLayout(sql),
1265
1594
  empty,
1266
1595
  createdByUs: facts.marker,
1267
1596
  enforceUuidCheck: env.SUBSTRATE_ENFORCE_UUID_CHECK === "true",
@@ -1286,7 +1615,7 @@ async function storeTables(pluginDir) {
1286
1615
  function formatMigrateStoreOutput(r, json) {
1287
1616
  if (json) return `${JSON.stringify(r)}\n`;
1288
1617
  const lines = [];
1289
- const verdict = r.exitCode === 0 ? r.mode === "dry-run" ? "dry run: nothing written" : "copied and verified" : r.exitCode === 3 ? "VERIFICATION FAILED: the target keeps its in-progress marker, so serve will not boot it" : "REFUSED: nothing was written to the target";
1618
+ const verdict = r.exitCode === 0 ? r.mode === "dry-run" ? "dry run: nothing written" : "copied and verified" : r.exitCode === 3 ? "VERIFICATION FAILED: the target keeps its in-progress marker, so serve will not boot it" : r.exitCode === 1 ? `FAILED${r.failedPhase !== void 0 ? ` in ${r.failedPhase}` : ""}: ${r.error ?? "unknown error"}` : "REFUSED: nothing was written to the target";
1290
1619
  lines.push(`migrate-store (${r.mode}): ${verdict}`);
1291
1620
  for (const x of r.refusals) lines.push(` refused: ${x}`);
1292
1621
  for (const x of r.warnings) lines.push(` warning: ${x}`);
@@ -1304,7 +1633,13 @@ function formatMigrateStoreOutput(r, json) {
1304
1633
  for (const t of r.tables) lines.push(` ${t.name}: ${t.rows} / ${t.bytes} / ${t.ms}`);
1305
1634
  }
1306
1635
  if (r.keptSeeds.length > 0) lines.push(`kept seeds: ${r.keptSeeds.join(", ")}`);
1307
- if (r.defaulted.length > 0) lines.push(`in the DEFAULT partition: ${r.defaulted.join(", ")}`);
1636
+ if (r.defaulted.length > 0) lines.push(`declared by no definition, every field in extra: ${r.defaulted.join(", ")}`);
1637
+ if (r.misfits === null) lines.push("misfits: not counted (only a copy run counts them)");
1638
+ for (const [type, m] of Object.entries(r.misfits ?? {})) {
1639
+ const more = m.count > 10 ? ` (first 10 shown; --json lists ${m.first.length})` : "";
1640
+ lines.push(`misfits in ${type}: ${m.count}, kept unchanged under $raw${more}: ${m.first.slice(0, 10).map((x) => `${x.id}.${x.field}`).join(", ")}`);
1641
+ }
1642
+ for (const m of r.missingIndexes) lines.push(` missing index [${m.type}]: ${m.index}`);
1308
1643
  if (r.pendingIndexes.length > 0) {
1309
1644
  lines.push(`${r.pendingIndexes.length} index build(s) left pending for serve (--skip-index-build; --json lists them all):`);
1310
1645
  for (const w of r.pendingIndexes.slice(0, 10)) lines.push(` ${w}`);
@@ -1367,6 +1702,60 @@ function shim(sink) {
1367
1702
  });
1368
1703
  }
1369
1704
 
1705
+ //#endregion
1706
+ //#region src/admin/commands/promote-fields.ts
1707
+ /** Parses `argv` (after the subcommand), resolves the server, and runs one sweep there. */
1708
+ async function promoteFieldsCommand(argv, env, fetchImpl = fetch) {
1709
+ validateArgv("promote-fields", argv);
1710
+ let url;
1711
+ for (let i = 0; i < argv.length; i++) {
1712
+ if (argv[i] !== "--url") continue;
1713
+ if (url !== void 0) throw argvError("promote-fields", "promote-fields: --url specified more than once");
1714
+ url = argv[i + 1] ?? "";
1715
+ i += 1;
1716
+ }
1717
+ if (url === "") throw argvError("promote-fields", "promote-fields: --url requires a base URL");
1718
+ const target = resolveAdminTarget(url ?? null, env);
1719
+ if (target.kind !== "route") throw new AdminCliError("promote-fields runs on a running server: pass --url <base> or set SUBSTRATE_URL", 1);
1720
+ const auth = resolveAdminAuth(env, target.baseUrl);
1721
+ if ("error" in auth) throw new AdminCliError(`promote-fields: ${auth.error}`, 1);
1722
+ const endpoint = `${trimTrailingSlashes(target.baseUrl)}/substrate/v1/_system/promote-fields`;
1723
+ let res;
1724
+ try {
1725
+ res = await fetchImpl(endpoint, {
1726
+ method: "POST",
1727
+ headers: {
1728
+ authorization: auth.ok,
1729
+ "content-type": "application/json"
1730
+ },
1731
+ body: "{}",
1732
+ redirect: "error"
1733
+ });
1734
+ } catch (err) {
1735
+ throw new AdminCliError(`could not reach ${endpoint}: ${err instanceof Error ? err.message : String(err)}`, 1);
1736
+ }
1737
+ const payload = await res.json().catch(() => ({}));
1738
+ if (!res.ok) {
1739
+ const reason = typeof payload.message === "string" ? payload.message : typeof payload.code === "string" ? payload.code : `HTTP ${res.status}`;
1740
+ throw new AdminCliError(`promote-fields: ${reason}`, res.status === 403 ? 5 : 1);
1741
+ }
1742
+ if (!Array.isArray(payload.events)) throw new AdminCliError("promote-fields: response carried no `events` array", 1);
1743
+ return {
1744
+ target: describeAdminTarget("promote-fields", target),
1745
+ events: payload.events
1746
+ };
1747
+ }
1748
+ /** One line per field, after the target line. */
1749
+ function formatPromoteFieldsOutput(result) {
1750
+ const lines = [result.target];
1751
+ if (result.events.length === 0) lines.push("no pending fields");
1752
+ for (const e of result.events) {
1753
+ const outcome = e.kind === "busy" && e.error !== void 0 ? `busy (${e.error})` : `${e.kind} (attempt ${e.attempts})${e.error !== void 0 ? ` — ${e.error}` : ""}`;
1754
+ lines.push(`${e.namespace}/${e.type}.${e.field}: ${outcome}`);
1755
+ }
1756
+ return `${lines.join("\n")}\n`;
1757
+ }
1758
+
1370
1759
  //#endregion
1371
1760
  //#region src/admin/commands/revoke.ts
1372
1761
  /**
@@ -1916,6 +2305,11 @@ async function main(argv, env, out, errOut, openDataset = openPgDataset, prompt
1916
2305
  return 0;
1917
2306
  }
1918
2307
  case "migrate-store": return migrateStoreCommand(argv.slice(1), env, out, errOut);
2308
+ case "promote-fields": {
2309
+ const result = await promoteFieldsCommand(argv.slice(1), env);
2310
+ out(formatPromoteFieldsOutput(result));
2311
+ return result.events.some((e) => e.kind === "failed") ? 1 : 0;
2312
+ }
1919
2313
  default:
1920
2314
  errOut(`unknown command: ${command}\n${USAGE}\n`);
1921
2315
  return 1;