@happyvertical/smrt-core 0.50.0 → 0.51.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +1 -0
- package/dist/index.js +2 -1
- package/dist/knowledge-graph.d.ts +95 -0
- package/dist/knowledge-graph.d.ts.map +1 -0
- package/dist/knowledge-graph.js +268 -0
- package/dist/knowledge-graph.js.map +1 -0
- package/dist/knowledge.d.ts +1 -0
- package/dist/knowledge.d.ts.map +1 -1
- package/dist/knowledge.js +57 -4
- package/dist/knowledge.js.map +1 -1
- package/dist/manifest/static-manifest.js +1 -1
- package/dist/manifest/static-manifest.js.map +1 -1
- package/dist/manifest/store.js +1 -1
- package/dist/manifest.json +1 -1
- package/dist/migrations/differ.d.ts +127 -21
- package/dist/migrations/differ.d.ts.map +1 -1
- package/dist/migrations/differ.js +379 -84
- package/dist/migrations/differ.js.map +1 -1
- package/dist/schema/column-data-probes.d.ts +38 -0
- package/dist/schema/column-data-probes.d.ts.map +1 -0
- package/dist/schema/column-data-probes.js +106 -0
- package/dist/schema/column-data-probes.js.map +1 -0
- package/dist/schema/live-parity.d.ts.map +1 -1
- package/dist/schema/live-parity.js +37 -46
- package/dist/schema/live-parity.js.map +1 -1
- package/dist/smrt-knowledge.json +12 -7
- package/package.json +4 -4
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
import { normalizeForeignKeyAction, requireForeignKeyAction } from "../schema/foreign-key-policy.js";
|
|
2
2
|
import { isJsonPathIndex, isStiSubtypeUniqueIndex, renderIndexTarget } from "../schema/index-utils.js";
|
|
3
|
-
import {
|
|
3
|
+
import { foreignKeyConstraintName, foreignKeyRelationshipKey, renderForeignKeyAddStatements, renderForeignKeyOrphanDetector, renderForeignKeyOrphanRepair, schemaForeignKeys, schemaForeignKeysForEngine } from "../schema/foreign-key-ddl.js";
|
|
4
4
|
import { renderNullEqualConflictIndex } from "../schema/ddl/null-equal-index.js";
|
|
5
5
|
import { detectEngine, getDDLStrategy } from "../schema/ddl/index.js";
|
|
6
|
+
import { columnsAllValuesUuidShapedBatch, columnsHaveNonEmptyValueBatch, nonEmptyValuePredicate, uuidInvalidShapePredicate } from "../schema/column-data-probes.js";
|
|
6
7
|
import { maskSampleValue, probeCastSafety, renderJsonbColumnConversion, renderTimestamptzColumnConversion } from "../schema/text-cast-probe.js";
|
|
7
8
|
import { describeForeignKeyTypeConflict, planUuidConvergence, renderUuidConvergenceRepairHint, renderUuidShapeProbe } from "../schema/uuid-convergence.js";
|
|
8
9
|
import "../schema/utils.js";
|
|
@@ -301,7 +302,7 @@ function canonicalizeDefault(fragment, columnType) {
|
|
|
301
302
|
/**
|
|
302
303
|
* SchemaComparer class for comparing manifest schemas to database
|
|
303
304
|
*/
|
|
304
|
-
var SchemaComparer = class {
|
|
305
|
+
var SchemaComparer = class SchemaComparer {
|
|
305
306
|
db;
|
|
306
307
|
options;
|
|
307
308
|
engine;
|
|
@@ -315,6 +316,40 @@ var SchemaComparer = class {
|
|
|
315
316
|
liveSchemas = /* @__PURE__ */ new Map();
|
|
316
317
|
/** Convergence plan for this run; `null` on non-PostgreSQL engines. */
|
|
317
318
|
uuidConvergence = null;
|
|
319
|
+
/**
|
|
320
|
+
* Rename-pending advisory findings for this `compare()` run, keyed by
|
|
321
|
+
* table name (#2878). `compareTable()` reads from here when it is being
|
|
322
|
+
* driven by `compare()` over the full manifest, which lets the probe be
|
|
323
|
+
* batched *across every table in one round trip* instead of paying at
|
|
324
|
+
* least one round trip per table (#2876 batched a table's own columns
|
|
325
|
+
* into one round trip but never batched across tables — with a
|
|
326
|
+
* realistic 71-table schema that per-table floor was itself the entire
|
|
327
|
+
* residual). `null` means "not precomputed for this run" (a standalone
|
|
328
|
+
* `compareTable()` call outside `compare()`, or a non-PostgreSQL/SQLite
|
|
329
|
+
* engine), in which case `detectRenameDataPending()` falls back to the
|
|
330
|
+
* original single-table probe so standalone callers keep working
|
|
331
|
+
* unchanged. Cleared every `compare()` call, same as `liveSchemas` —
|
|
332
|
+
* this is a per-run cache, never reused across runs, so a live schema
|
|
333
|
+
* change is always seen on the next comparison (no invalidation story
|
|
334
|
+
* needed because nothing survives past one `compare()` call).
|
|
335
|
+
*/
|
|
336
|
+
renameDataPendingCache = null;
|
|
337
|
+
/**
|
|
338
|
+
* Cross-table batch queries stay bounded regardless of schema shape: this
|
|
339
|
+
* caps how many scalar-subquery columns one probe statement packs into a
|
|
340
|
+
* single row. Conservative relative to PostgreSQL's ~1600 column limit
|
|
341
|
+
* per result row, and SQLite has no such ceiling but benefits from the
|
|
342
|
+
* same bound for query-text size. Applied twice (PR #2888 review): once
|
|
343
|
+
* per table, so a single pathologically wide table (more probed columns
|
|
344
|
+
* than this cap) is itself sliced across more than one statement rather
|
|
345
|
+
* than landing in one oversized chunk; and again across tables, packing
|
|
346
|
+
* every (possibly sliced) entry into as few chunks as this cap allows.
|
|
347
|
+
* A schema with no single table wider than this cap needs more than
|
|
348
|
+
* `MAX_CROSS_TABLE_PROBE_COLUMNS` probed columns *in total* before a
|
|
349
|
+
* second statement is needed at all — still O(1)-ish round trips for a
|
|
350
|
+
* realistic schema.
|
|
351
|
+
*/
|
|
352
|
+
static MAX_CROSS_TABLE_PROBE_COLUMNS = 400;
|
|
318
353
|
constructor(db, options = {}) {
|
|
319
354
|
this.db = db;
|
|
320
355
|
this.options = {
|
|
@@ -338,34 +373,40 @@ var SchemaComparer = class {
|
|
|
338
373
|
has_changes: false
|
|
339
374
|
};
|
|
340
375
|
this.liveSchemas.clear();
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
diff.has_changes = true;
|
|
350
|
-
} else {
|
|
351
|
-
const tableChanges = await this.compareTable(tableName, schema, manifestSchemas);
|
|
352
|
-
if (tableChanges.length > 0) {
|
|
353
|
-
diff.changes.push(...tableChanges);
|
|
354
|
-
if (tableChanges.some((change) => !isInfoOnlyChange(change))) diff.has_changes = true;
|
|
376
|
+
this.renameDataPendingCache = null;
|
|
377
|
+
try {
|
|
378
|
+
const existingTables = await this.getExistingTables();
|
|
379
|
+
await this.precomputeRenameDataPending(manifestSchemas, existingTables);
|
|
380
|
+
this.uuidConvergence = await this.buildUuidConvergencePlan(manifestSchemas, existingTables);
|
|
381
|
+
for (const change of this.uuidConvergenceChanges(manifestSchemas)) {
|
|
382
|
+
diff.changes.push(change);
|
|
383
|
+
if (!isInfoOnlyChange(change)) diff.has_changes = true;
|
|
355
384
|
}
|
|
385
|
+
for (const [tableName, schema] of Object.entries(manifestSchemas)) if (!existingTables.has(tableName)) {
|
|
386
|
+
diff.added_tables.push(schema);
|
|
387
|
+
diff.has_changes = true;
|
|
388
|
+
} else {
|
|
389
|
+
const tableChanges = await this.compareTable(tableName, schema, manifestSchemas);
|
|
390
|
+
if (tableChanges.length > 0) {
|
|
391
|
+
diff.changes.push(...tableChanges);
|
|
392
|
+
if (tableChanges.some((change) => !isInfoOnlyChange(change))) diff.has_changes = true;
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const orphanTables = [];
|
|
396
|
+
for (const tableName of existingTables) {
|
|
397
|
+
if (tableName.startsWith("_smrt_") || tableName.startsWith("sqlite_")) continue;
|
|
398
|
+
if (!manifestSchemas[tableName]) orphanTables.push(tableName);
|
|
399
|
+
}
|
|
400
|
+
orphanTables.sort();
|
|
401
|
+
diff.orphan_tables = orphanTables;
|
|
402
|
+
if (this.options.includeDroppedTables && orphanTables.length > 0) {
|
|
403
|
+
diff.dropped_tables.push(...orphanTables);
|
|
404
|
+
diff.has_changes = true;
|
|
405
|
+
}
|
|
406
|
+
return diff;
|
|
407
|
+
} finally {
|
|
408
|
+
this.renameDataPendingCache = null;
|
|
356
409
|
}
|
|
357
|
-
const orphanTables = [];
|
|
358
|
-
for (const tableName of existingTables) {
|
|
359
|
-
if (tableName.startsWith("_smrt_") || tableName.startsWith("sqlite_")) continue;
|
|
360
|
-
if (!manifestSchemas[tableName]) orphanTables.push(tableName);
|
|
361
|
-
}
|
|
362
|
-
orphanTables.sort();
|
|
363
|
-
diff.orphan_tables = orphanTables;
|
|
364
|
-
if (this.options.includeDroppedTables && orphanTables.length > 0) {
|
|
365
|
-
diff.dropped_tables.push(...orphanTables);
|
|
366
|
-
diff.has_changes = true;
|
|
367
|
-
}
|
|
368
|
-
return diff;
|
|
369
410
|
}
|
|
370
411
|
/**
|
|
371
412
|
* Compare a single table's schema to manifest
|
|
@@ -964,6 +1005,19 @@ var SchemaComparer = class {
|
|
|
964
1005
|
return changes;
|
|
965
1006
|
}
|
|
966
1007
|
/**
|
|
1008
|
+
* Detect a pending rename backfill (#2752) for one table. Reads
|
|
1009
|
+
* `renameDataPendingCache` when `compare()` has already batched this
|
|
1010
|
+
* table's probe across the whole manifest (#2878); otherwise falls back
|
|
1011
|
+
* to {@link detectRenameDataPendingSingleTable}, so a standalone
|
|
1012
|
+
* `compareTable()` call (outside `compare()`) still gets a correct
|
|
1013
|
+
* answer, just without the cross-table batching.
|
|
1014
|
+
*/
|
|
1015
|
+
async detectRenameDataPending(tableName, manifest, dbSchema) {
|
|
1016
|
+
const cached = this.renameDataPendingCache?.get(tableName);
|
|
1017
|
+
if (cached !== void 0) return cached;
|
|
1018
|
+
return this.detectRenameDataPendingSingleTable(tableName, manifest, dbSchema);
|
|
1019
|
+
}
|
|
1020
|
+
/**
|
|
967
1021
|
* Detect a pending rename backfill (#2752): a manifest-declared column
|
|
968
1022
|
* that exists live but holds no data, paired with an orphan (undeclared)
|
|
969
1023
|
* live column of a compatible type that does hold data — the shape a
|
|
@@ -975,89 +1029,330 @@ var SchemaComparer = class {
|
|
|
975
1029
|
* `DO $$ ... $$` block); SQLite has no conditional-DDL construct, so its
|
|
976
1030
|
* repair is operator-mediated instead — a guard query plus instructions,
|
|
977
1031
|
* not a blind-rerun-safe statement. See {@link describeRenameDataPending}.
|
|
1032
|
+
*
|
|
1033
|
+
* Single-table probe: at least one round trip for this table alone.
|
|
1034
|
+
* `compare()` does not call this directly — see
|
|
1035
|
+
* {@link precomputeRenameDataPending} for the cross-table batched path
|
|
1036
|
+
* that every ordinary `compare()` run takes instead (#2878). This stays
|
|
1037
|
+
* as the fallback for a standalone `compareTable()` call.
|
|
978
1038
|
*/
|
|
979
|
-
async
|
|
1039
|
+
async detectRenameDataPendingSingleTable(tableName, manifest, dbSchema) {
|
|
980
1040
|
if (this.engine !== "postgres" && this.engine !== "sqlite") return [];
|
|
981
1041
|
const dbColumnNames = new Set(Object.keys(dbSchema.columns));
|
|
982
1042
|
const manifestColumnNames = new Set(Object.keys(manifest.columns));
|
|
983
1043
|
const orphanColumnNames = [...dbColumnNames].filter((name) => !manifestColumnNames.has(name));
|
|
984
1044
|
if (orphanColumnNames.length === 0) return [];
|
|
1045
|
+
const declaredCandidateNames = Object.keys(manifest.columns).filter((colName) => dbSchema.columns[colName]);
|
|
1046
|
+
if (declaredCandidateNames.length === 0) return [];
|
|
1047
|
+
const declaredHasData = await columnsHaveNonEmptyValueBatch(this.db, tableName, declaredCandidateNames);
|
|
1048
|
+
const emptyDeclaredNames = declaredCandidateNames.filter((colName) => !(declaredHasData.get(colName) ?? true));
|
|
1049
|
+
if (emptyDeclaredNames.length === 0) return [];
|
|
1050
|
+
const compatibleOrphanNames = orphanColumnNames.filter((orphanName) => {
|
|
1051
|
+
const orphanNormalized = this.normalizeType(dbSchema.columns[orphanName].type);
|
|
1052
|
+
return emptyDeclaredNames.some((colName) => {
|
|
1053
|
+
const validatedType = isValidSQLDataType(manifest.columns[colName].type) ? manifest.columns[colName].type : "TEXT";
|
|
1054
|
+
const declaredNormalized = this.normalizeType(this.ddlStrategy.mapType(validatedType));
|
|
1055
|
+
return validatedType === "UUID" && orphanNormalized === "TEXT" || orphanNormalized === declaredNormalized;
|
|
1056
|
+
});
|
|
1057
|
+
});
|
|
1058
|
+
if (compatibleOrphanNames.length === 0) return [];
|
|
1059
|
+
const orphanHasData = await columnsHaveNonEmptyValueBatch(this.db, tableName, compatibleOrphanNames);
|
|
1060
|
+
const hasData = new Map([...declaredHasData, ...orphanHasData]);
|
|
985
1061
|
const changes = [];
|
|
1062
|
+
const pending = [];
|
|
1063
|
+
const shapeCheckOrphans = /* @__PURE__ */ new Set();
|
|
986
1064
|
for (const [colName, colDef] of Object.entries(manifest.columns)) {
|
|
987
1065
|
if (!dbSchema.columns[colName]) continue;
|
|
988
|
-
|
|
989
|
-
try {
|
|
990
|
-
declaredHasData = await this.columnHasNonEmptyValue(tableName, colName);
|
|
991
|
-
} catch {
|
|
992
|
-
continue;
|
|
993
|
-
}
|
|
994
|
-
if (declaredHasData) continue;
|
|
1066
|
+
if (hasData.get(colName) ?? true) continue;
|
|
995
1067
|
const validatedType = isValidSQLDataType(colDef.type) ? colDef.type : "TEXT";
|
|
996
1068
|
const isLogicalUuid = validatedType === "UUID";
|
|
997
1069
|
const declaredNormalized = this.normalizeType(this.ddlStrategy.mapType(validatedType));
|
|
998
|
-
const
|
|
999
|
-
for (const orphanName of orphanColumnNames) {
|
|
1070
|
+
for (const orphanName of compatibleOrphanNames) {
|
|
1000
1071
|
const orphanCol = dbSchema.columns[orphanName];
|
|
1001
1072
|
const orphanNormalized = this.normalizeType(orphanCol.type);
|
|
1002
1073
|
const requiresShapeCheck = isLogicalUuid && orphanNormalized === "TEXT";
|
|
1003
1074
|
if (!requiresShapeCheck && !(orphanNormalized === declaredNormalized)) continue;
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
continue;
|
|
1009
|
-
}
|
|
1010
|
-
if (!orphanHasData) continue;
|
|
1011
|
-
if (requiresShapeCheck) {
|
|
1012
|
-
let shaped;
|
|
1013
|
-
try {
|
|
1014
|
-
shaped = await this.allNonEmptyValuesUuidShaped(tableName, orphanName);
|
|
1015
|
-
} catch {
|
|
1016
|
-
continue;
|
|
1017
|
-
}
|
|
1018
|
-
if (!shaped) continue;
|
|
1019
|
-
}
|
|
1020
|
-
candidates.push({
|
|
1075
|
+
if (!(hasData.get(orphanName) ?? false)) continue;
|
|
1076
|
+
if (requiresShapeCheck) shapeCheckOrphans.add(orphanName);
|
|
1077
|
+
pending.push({
|
|
1078
|
+
colName,
|
|
1021
1079
|
orphanName,
|
|
1022
|
-
isUuidCast: declaredNormalized === "UUID"
|
|
1080
|
+
isUuidCast: declaredNormalized === "UUID",
|
|
1081
|
+
requiresShapeCheck
|
|
1023
1082
|
});
|
|
1024
1083
|
}
|
|
1025
|
-
if (candidates.length === 1) changes.push(this.describeRenameDataPending(tableName, colName, candidates[0].orphanName, candidates[0].isUuidCast));
|
|
1026
|
-
else if (candidates.length > 1) changes.push(this.describeRenameDataPendingAmbiguous(tableName, colName, candidates.map((c) => c.orphanName)));
|
|
1027
1084
|
}
|
|
1085
|
+
const shaped = shapeCheckOrphans.size > 0 ? await columnsAllValuesUuidShapedBatch(this.db, this.engine, tableName, [...shapeCheckOrphans]) : /* @__PURE__ */ new Map();
|
|
1086
|
+
const candidatesByColumn = /* @__PURE__ */ new Map();
|
|
1087
|
+
for (const candidate of pending) {
|
|
1088
|
+
if (candidate.requiresShapeCheck && !(shaped.get(candidate.orphanName) ?? false)) continue;
|
|
1089
|
+
const list = candidatesByColumn.get(candidate.colName) ?? [];
|
|
1090
|
+
list.push({
|
|
1091
|
+
orphanName: candidate.orphanName,
|
|
1092
|
+
isUuidCast: candidate.isUuidCast
|
|
1093
|
+
});
|
|
1094
|
+
candidatesByColumn.set(candidate.colName, list);
|
|
1095
|
+
}
|
|
1096
|
+
for (const [colName, candidates] of candidatesByColumn) if (candidates.length === 1) changes.push(this.describeRenameDataPending(tableName, colName, candidates[0].orphanName, candidates[0].isUuidCast));
|
|
1097
|
+
else if (candidates.length > 1) changes.push(this.describeRenameDataPendingAmbiguous(tableName, colName, candidates.map((c) => c.orphanName)));
|
|
1028
1098
|
return changes;
|
|
1029
1099
|
}
|
|
1030
1100
|
/**
|
|
1031
|
-
*
|
|
1032
|
-
*
|
|
1033
|
-
*
|
|
1101
|
+
* Batch every table's rename-pending advisory probe (#2752/#2878) into a
|
|
1102
|
+
* small, table-count-independent number of round trips, and populate
|
|
1103
|
+
* {@link renameDataPendingCache} so `compareTable()`'s per-table
|
|
1104
|
+
* `detectRenameDataPending()` call becomes a cache read for the rest of
|
|
1105
|
+
* this `compare()` run.
|
|
1106
|
+
*
|
|
1107
|
+
* Mirrors {@link detectRenameDataPendingSingleTable}'s phases exactly —
|
|
1108
|
+
* same gates, same type-compatibility rules, same per-column `LIMIT 1`
|
|
1109
|
+
* early exit — just resequenced so each phase's *query* covers every
|
|
1110
|
+
* table that needs it in one statement, instead of one statement per
|
|
1111
|
+
* table. With a 71-table schema this is the difference between a
|
|
1112
|
+
* 71-statement floor (#2878) and a handful of statements for the whole
|
|
1113
|
+
* schema, independent of table count.
|
|
1114
|
+
*
|
|
1115
|
+
* No caching beyond this: the cache this populates is cleared at the top
|
|
1116
|
+
* of every `compare()` call, so a live schema change is always visible
|
|
1117
|
+
* on the next comparison.
|
|
1034
1118
|
*/
|
|
1035
|
-
async
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
1119
|
+
async precomputeRenameDataPending(manifestSchemas, existingTables) {
|
|
1120
|
+
this.renameDataPendingCache = /* @__PURE__ */ new Map();
|
|
1121
|
+
if (this.engine !== "postgres" && this.engine !== "sqlite") return;
|
|
1122
|
+
const tables = [];
|
|
1123
|
+
for (const [tableName, manifest] of Object.entries(manifestSchemas)) {
|
|
1124
|
+
if (!existingTables.has(tableName)) continue;
|
|
1125
|
+
const dbSchema = await this.getLiveSchema(tableName);
|
|
1126
|
+
if (!dbSchema) {
|
|
1127
|
+
this.renameDataPendingCache.set(tableName, []);
|
|
1128
|
+
continue;
|
|
1129
|
+
}
|
|
1130
|
+
const dbColumnNames = new Set(Object.keys(dbSchema.columns));
|
|
1131
|
+
const manifestColumnNames = new Set(Object.keys(manifest.columns));
|
|
1132
|
+
const orphanColumnNames = [...dbColumnNames].filter((name) => !manifestColumnNames.has(name));
|
|
1133
|
+
if (orphanColumnNames.length === 0) {
|
|
1134
|
+
this.renameDataPendingCache.set(tableName, []);
|
|
1135
|
+
continue;
|
|
1136
|
+
}
|
|
1137
|
+
const declaredCandidateNames = Object.keys(manifest.columns).filter((colName) => dbSchema.columns[colName]);
|
|
1138
|
+
if (declaredCandidateNames.length === 0) {
|
|
1139
|
+
this.renameDataPendingCache.set(tableName, []);
|
|
1140
|
+
continue;
|
|
1141
|
+
}
|
|
1142
|
+
tables.push({
|
|
1143
|
+
tableName,
|
|
1144
|
+
manifest,
|
|
1145
|
+
dbSchema,
|
|
1146
|
+
declaredCandidateNames,
|
|
1147
|
+
orphanColumnNames
|
|
1148
|
+
});
|
|
1149
|
+
}
|
|
1150
|
+
if (tables.length === 0) return;
|
|
1151
|
+
const declaredHasDataByTable = await this.crossTableColumnsHaveNonEmptyValueBatch(tables.map((t) => ({
|
|
1152
|
+
tableName: t.tableName,
|
|
1153
|
+
colNames: t.declaredCandidateNames
|
|
1154
|
+
})));
|
|
1155
|
+
const phase2Tables = [];
|
|
1156
|
+
for (const t of tables) {
|
|
1157
|
+
const declaredHasData = declaredHasDataByTable.get(t.tableName) ?? /* @__PURE__ */ new Map();
|
|
1158
|
+
const emptyDeclaredNames = t.declaredCandidateNames.filter((colName) => !(declaredHasData.get(colName) ?? true));
|
|
1159
|
+
if (emptyDeclaredNames.length === 0) {
|
|
1160
|
+
this.renameDataPendingCache.set(t.tableName, []);
|
|
1161
|
+
continue;
|
|
1162
|
+
}
|
|
1163
|
+
const compatibleOrphanNames = t.orphanColumnNames.filter((orphanName) => {
|
|
1164
|
+
const orphanNormalized = this.normalizeType(t.dbSchema.columns[orphanName].type);
|
|
1165
|
+
return emptyDeclaredNames.some((colName) => {
|
|
1166
|
+
const validatedType = isValidSQLDataType(t.manifest.columns[colName].type) ? t.manifest.columns[colName].type : "TEXT";
|
|
1167
|
+
const declaredNormalized = this.normalizeType(this.ddlStrategy.mapType(validatedType));
|
|
1168
|
+
return validatedType === "UUID" && orphanNormalized === "TEXT" || orphanNormalized === declaredNormalized;
|
|
1169
|
+
});
|
|
1170
|
+
});
|
|
1171
|
+
if (compatibleOrphanNames.length === 0) {
|
|
1172
|
+
this.renameDataPendingCache.set(t.tableName, []);
|
|
1173
|
+
continue;
|
|
1174
|
+
}
|
|
1175
|
+
phase2Tables.push({
|
|
1176
|
+
tableName: t.tableName,
|
|
1177
|
+
compatibleOrphanNames
|
|
1178
|
+
});
|
|
1179
|
+
}
|
|
1180
|
+
if (phase2Tables.length === 0) return;
|
|
1181
|
+
const orphanHasDataByTable = await this.crossTableColumnsHaveNonEmptyValueBatch(phase2Tables.map((t) => ({
|
|
1182
|
+
tableName: t.tableName,
|
|
1183
|
+
colNames: t.compatibleOrphanNames
|
|
1184
|
+
})));
|
|
1185
|
+
const byName = new Map(tables.map((t) => [t.tableName, t]));
|
|
1186
|
+
const pendingByTable = /* @__PURE__ */ new Map();
|
|
1187
|
+
const shapeCheckByTable = /* @__PURE__ */ new Map();
|
|
1188
|
+
for (const { tableName, compatibleOrphanNames } of phase2Tables) {
|
|
1189
|
+
const t = byName.get(tableName);
|
|
1190
|
+
if (!t) continue;
|
|
1191
|
+
const declaredHasData = declaredHasDataByTable.get(tableName) ?? /* @__PURE__ */ new Map();
|
|
1192
|
+
const orphanHasData = orphanHasDataByTable.get(tableName) ?? /* @__PURE__ */ new Map();
|
|
1193
|
+
const hasData = new Map([...declaredHasData, ...orphanHasData]);
|
|
1194
|
+
const pending = [];
|
|
1195
|
+
const shapeCheckOrphans = /* @__PURE__ */ new Set();
|
|
1196
|
+
for (const [colName, colDef] of Object.entries(t.manifest.columns)) {
|
|
1197
|
+
if (!t.dbSchema.columns[colName]) continue;
|
|
1198
|
+
if (hasData.get(colName) ?? true) continue;
|
|
1199
|
+
const validatedType = isValidSQLDataType(colDef.type) ? colDef.type : "TEXT";
|
|
1200
|
+
const isLogicalUuid = validatedType === "UUID";
|
|
1201
|
+
const declaredNormalized = this.normalizeType(this.ddlStrategy.mapType(validatedType));
|
|
1202
|
+
for (const orphanName of compatibleOrphanNames) {
|
|
1203
|
+
const orphanCol = t.dbSchema.columns[orphanName];
|
|
1204
|
+
const orphanNormalized = this.normalizeType(orphanCol.type);
|
|
1205
|
+
const requiresShapeCheck = isLogicalUuid && orphanNormalized === "TEXT";
|
|
1206
|
+
if (!requiresShapeCheck && !(orphanNormalized === declaredNormalized)) continue;
|
|
1207
|
+
if (!(hasData.get(orphanName) ?? false)) continue;
|
|
1208
|
+
if (requiresShapeCheck) shapeCheckOrphans.add(orphanName);
|
|
1209
|
+
pending.push({
|
|
1210
|
+
colName,
|
|
1211
|
+
orphanName,
|
|
1212
|
+
isUuidCast: declaredNormalized === "UUID",
|
|
1213
|
+
requiresShapeCheck
|
|
1214
|
+
});
|
|
1215
|
+
}
|
|
1216
|
+
}
|
|
1217
|
+
pendingByTable.set(tableName, pending);
|
|
1218
|
+
if (shapeCheckOrphans.size > 0) shapeCheckByTable.set(tableName, shapeCheckOrphans);
|
|
1219
|
+
}
|
|
1220
|
+
let shapedByTable = /* @__PURE__ */ new Map();
|
|
1221
|
+
if (shapeCheckByTable.size > 0) shapedByTable = await this.crossTableColumnsAllValuesUuidShapedBatch([...shapeCheckByTable.entries()].map(([tableName, cols]) => ({
|
|
1222
|
+
tableName,
|
|
1223
|
+
colNames: [...cols]
|
|
1224
|
+
})));
|
|
1225
|
+
for (const [tableName, pending] of pendingByTable) {
|
|
1226
|
+
const shaped = shapedByTable.get(tableName) ?? /* @__PURE__ */ new Map();
|
|
1227
|
+
const changes = [];
|
|
1228
|
+
const candidatesByColumn = /* @__PURE__ */ new Map();
|
|
1229
|
+
for (const candidate of pending) {
|
|
1230
|
+
if (candidate.requiresShapeCheck && !(shaped.get(candidate.orphanName) ?? false)) continue;
|
|
1231
|
+
const list = candidatesByColumn.get(candidate.colName) ?? [];
|
|
1232
|
+
list.push({
|
|
1233
|
+
orphanName: candidate.orphanName,
|
|
1234
|
+
isUuidCast: candidate.isUuidCast
|
|
1235
|
+
});
|
|
1236
|
+
candidatesByColumn.set(candidate.colName, list);
|
|
1237
|
+
}
|
|
1238
|
+
for (const [colName, candidates] of candidatesByColumn) if (candidates.length === 1) changes.push(this.describeRenameDataPending(tableName, colName, candidates[0].orphanName, candidates[0].isUuidCast));
|
|
1239
|
+
else if (candidates.length > 1) changes.push(this.describeRenameDataPendingAmbiguous(tableName, colName, candidates.map((c) => c.orphanName)));
|
|
1240
|
+
this.renameDataPendingCache.set(tableName, changes);
|
|
1241
|
+
}
|
|
1039
1242
|
}
|
|
1040
1243
|
/**
|
|
1041
|
-
*
|
|
1042
|
-
*
|
|
1043
|
-
*
|
|
1044
|
-
*
|
|
1045
|
-
*
|
|
1046
|
-
*
|
|
1047
|
-
*
|
|
1048
|
-
*
|
|
1049
|
-
*
|
|
1244
|
+
* Cross-table variant of {@link columnsHaveNonEmptyValueBatch}: the same
|
|
1245
|
+
* uncorrelated-scalar-subquery, per-column `LIMIT 1` shape, but packed
|
|
1246
|
+
* into one row *per query* across every named table's columns instead of
|
|
1247
|
+
* one row per table — the #2878 lever. Chunks at
|
|
1248
|
+
* {@link MAX_CROSS_TABLE_PROBE_COLUMNS} columns per statement so a very
|
|
1249
|
+
* large schema still issues a small, bounded number of round trips
|
|
1250
|
+
* rather than one arbitrarily wide row.
|
|
1251
|
+
*
|
|
1252
|
+
* Falls back to the existing per-table batch (itself falling back
|
|
1253
|
+
* further to per-column) for any chunk whose combined statement fails,
|
|
1254
|
+
* so one bad table never discards another table's detection — the same
|
|
1255
|
+
* failure-isolation guarantee {@link columnsHaveNonEmptyValueBatch}
|
|
1256
|
+
* already gives per-column, extended one level up.
|
|
1050
1257
|
*/
|
|
1051
|
-
async
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1258
|
+
async crossTableColumnsHaveNonEmptyValueBatch(tables) {
|
|
1259
|
+
return this.crossTableProbeBatch(tables, (t) => columnsHaveNonEmptyValueBatch(this.db, t.tableName, t.colNames), (quotedTable, quotedCol, alias) => `(SELECT 1 FROM ${quotedTable} WHERE ${nonEmptyValuePredicate(quotedCol)} LIMIT 1) AS ${alias}`, (value) => value != null);
|
|
1260
|
+
}
|
|
1261
|
+
/**
|
|
1262
|
+
* Cross-table variant of {@link columnsAllValuesUuidShapedBatch} — same
|
|
1263
|
+
* shape as {@link crossTableColumnsHaveNonEmptyValueBatch}, engine-aware
|
|
1264
|
+
* invalid-shape predicate included.
|
|
1265
|
+
*/
|
|
1266
|
+
async crossTableColumnsAllValuesUuidShapedBatch(tables) {
|
|
1267
|
+
return this.crossTableProbeBatch(tables, (t) => columnsAllValuesUuidShapedBatch(this.db, this.engine, t.tableName, t.colNames), (quotedTable, quotedCol, alias) => `(SELECT 1 FROM ${quotedTable} WHERE ${nonEmptyValuePredicate(quotedCol)} AND ${uuidInvalidShapePredicate(this.engine, quotedCol)} LIMIT 1) AS ${alias}`, (value) => value == null);
|
|
1268
|
+
}
|
|
1269
|
+
/**
|
|
1270
|
+
* Shared cross-table batching engine for the two probes above. Builds one
|
|
1271
|
+
* `SELECT` per chunk with a scalar subquery per (table, column) pair,
|
|
1272
|
+
* globally aliased `t{tableIndex}_c{colIndex}` — PostgreSQL truncates a
|
|
1273
|
+
* `name` identifier to 63 bytes, so a positional alias is used instead of
|
|
1274
|
+
* the real column name for the same reason the single-table batch does
|
|
1275
|
+
* (#2874 review finding F2'). On a chunk's query failure, that chunk's
|
|
1276
|
+
* tables fall back to the single-table batch path individually (each of
|
|
1277
|
+
* which falls back further to per-column) rather than discarding every
|
|
1278
|
+
* table's result.
|
|
1279
|
+
*
|
|
1280
|
+
* Known lock-footprint change (#2878 final review F2, deferred to
|
|
1281
|
+
* #2887): one chunked statement takes `ACCESS SHARE` on every
|
|
1282
|
+
* table in that chunk *simultaneously*, where the pre-#2878 per-table
|
|
1283
|
+
* probe held at most one table's `ACCESS SHARE` at a time (each
|
|
1284
|
+
* autocommitted statement releases its lock before the next one runs).
|
|
1285
|
+
* That is a materially different PostgreSQL locking pattern: it is now
|
|
1286
|
+
* possible, in principle, for this session to be one side of a deadlock
|
|
1287
|
+
* with a concurrent multi-table DDL transaction (e.g. a `db:migrate` in
|
|
1288
|
+
* flight) in a way the old per-table probe structurally could not be.
|
|
1289
|
+
* PostgreSQL's deadlock detector resolves any such cycle by aborting one
|
|
1290
|
+
* side with a loud `deadlock detected` error — never silently, and never
|
|
1291
|
+
* with partial/corrupt state — so if this session is the victim, the
|
|
1292
|
+
* `catch` above already degrades it to the safe per-table fallback; if
|
|
1293
|
+
* the *other* side (the migration) is the victim, that transaction rolls
|
|
1294
|
+
* back cleanly and is safe to re-run. No test in this repository exercises
|
|
1295
|
+
* concurrent DDL against the probed tables, so this risk is unverified by
|
|
1296
|
+
* suite rather than disproven. Bounding it further — chunking by table
|
|
1297
|
+
* count as well as column count, and/or a short `lock_timeout` on these
|
|
1298
|
+
* statements so this session always yields first — is deferred to #2887
|
|
1299
|
+
* rather than fixed here.
|
|
1300
|
+
*/
|
|
1301
|
+
async crossTableProbeBatch(tables, singleTableFallback, renderSelect, isTrue) {
|
|
1302
|
+
const nonEmpty = tables.filter((t) => t.colNames.length > 0);
|
|
1303
|
+
if (nonEmpty.length === 0) return /* @__PURE__ */ new Map();
|
|
1304
|
+
const slices = [];
|
|
1305
|
+
for (const t of nonEmpty) for (let i = 0; i < t.colNames.length; i += SchemaComparer.MAX_CROSS_TABLE_PROBE_COLUMNS) slices.push({
|
|
1306
|
+
tableName: t.tableName,
|
|
1307
|
+
colNames: t.colNames.slice(i, i + SchemaComparer.MAX_CROSS_TABLE_PROBE_COLUMNS)
|
|
1308
|
+
});
|
|
1309
|
+
const chunks = [];
|
|
1310
|
+
let current = [];
|
|
1311
|
+
let currentWidth = 0;
|
|
1312
|
+
for (const slice of slices) {
|
|
1313
|
+
if (currentWidth + slice.colNames.length > SchemaComparer.MAX_CROSS_TABLE_PROBE_COLUMNS && current.length > 0) {
|
|
1314
|
+
chunks.push(current);
|
|
1315
|
+
current = [];
|
|
1316
|
+
currentWidth = 0;
|
|
1317
|
+
}
|
|
1318
|
+
current.push(slice);
|
|
1319
|
+
currentWidth += slice.colNames.length;
|
|
1320
|
+
}
|
|
1321
|
+
if (current.length > 0) chunks.push(current);
|
|
1322
|
+
const result = /* @__PURE__ */ new Map();
|
|
1323
|
+
for (const chunk of chunks) {
|
|
1324
|
+
const selects = [];
|
|
1325
|
+
const index = [];
|
|
1326
|
+
chunk.forEach((t, ti) => {
|
|
1327
|
+
const quotedTable = this.quoteIdentifier(t.tableName);
|
|
1328
|
+
t.colNames.forEach((colName, ci) => {
|
|
1329
|
+
const quotedCol = this.quoteIdentifier(colName);
|
|
1330
|
+
const alias = `t${ti}_c${ci}`;
|
|
1331
|
+
selects.push(renderSelect(quotedTable, quotedCol, alias));
|
|
1332
|
+
index.push({
|
|
1333
|
+
tableName: t.tableName,
|
|
1334
|
+
colName,
|
|
1335
|
+
alias
|
|
1336
|
+
});
|
|
1337
|
+
});
|
|
1338
|
+
});
|
|
1339
|
+
try {
|
|
1340
|
+
const row = (await this.db.query(`SELECT ${selects.join(", ")}`)).rows?.[0] ?? {};
|
|
1341
|
+
for (const { tableName, colName, alias } of index) {
|
|
1342
|
+
const m = result.get(tableName) ?? /* @__PURE__ */ new Map();
|
|
1343
|
+
m.set(colName, isTrue(row[alias]));
|
|
1344
|
+
result.set(tableName, m);
|
|
1345
|
+
}
|
|
1346
|
+
} catch {
|
|
1347
|
+
for (const t of chunk) {
|
|
1348
|
+
const m = result.get(t.tableName) ?? /* @__PURE__ */ new Map();
|
|
1349
|
+
const fallback = await singleTableFallback(t);
|
|
1350
|
+
for (const [colName, value] of fallback) m.set(colName, value);
|
|
1351
|
+
result.set(t.tableName, m);
|
|
1352
|
+
}
|
|
1353
|
+
}
|
|
1058
1354
|
}
|
|
1059
|
-
|
|
1060
|
-
return Number(row?.invalid_count ?? 0) === 0;
|
|
1355
|
+
return result;
|
|
1061
1356
|
}
|
|
1062
1357
|
/**
|
|
1063
1358
|
* Render the repair for one rename-data-pending pair (#2752): copy
|
|
@@ -1073,7 +1368,7 @@ var SchemaComparer = class {
|
|
|
1073
1368
|
* for `integer`/`boolean`/`timestamp`/etc. before a same-type rename
|
|
1074
1369
|
* pair of one of those types ever copies anything. Fixed by comparing
|
|
1075
1370
|
* `CAST(col AS TEXT) = ''` instead, mirroring the same portable
|
|
1076
|
-
* emptiness check {@link
|
|
1371
|
+
* emptiness check {@link columnsHaveNonEmptyValueBatch} already uses for
|
|
1077
1372
|
* detection, so the predicate is valid for every column type.
|
|
1078
1373
|
* - (P2) An `UPDATE` unconditionally naming `oldColumn` fails once that
|
|
1079
1374
|
* column is gone, on either engine, regardless of the DROP's own
|