@metaobjectsdev/migrate-ts 0.23.2 → 0.24.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/diff/index.d.ts.map +1 -1
- package/dist/diff/index.js +8 -7
- package/dist/diff/index.js.map +1 -1
- package/dist/drift/drift.d.ts +32 -2
- package/dist/drift/drift.d.ts.map +1 -1
- package/dist/drift/drift.js +15 -8
- package/dist/drift/drift.js.map +1 -1
- package/dist/emit/postgres.js +69 -12
- package/dist/emit/postgres.js.map +1 -1
- package/dist/emit/sqlite.d.ts.map +1 -1
- package/dist/emit/sqlite.js +9 -2
- package/dist/emit/sqlite.js.map +1 -1
- package/dist/expected-schema.d.ts +35 -0
- package/dist/expected-schema.d.ts.map +1 -1
- package/dist/expected-schema.js +29 -2
- package/dist/expected-schema.js.map +1 -1
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/qualified-name.d.ts +9 -0
- package/dist/qualified-name.d.ts.map +1 -0
- package/dist/qualified-name.js +21 -0
- package/dist/qualified-name.js.map +1 -0
- package/dist/scope.d.ts +121 -0
- package/dist/scope.d.ts.map +1 -0
- package/dist/scope.js +193 -0
- package/dist/scope.js.map +1 -0
- package/dist/snapshot/plan.d.ts +23 -1
- package/dist/snapshot/plan.d.ts.map +1 -1
- package/dist/snapshot/plan.js +17 -9
- package/dist/snapshot/plan.js.map +1 -1
- package/dist/types.d.ts +17 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/unmanaged.d.ts.map +1 -1
- package/dist/unmanaged.js +3 -3
- package/dist/unmanaged.js.map +1 -1
- package/dist/verify/replay-engine.d.ts +10 -0
- package/dist/verify/replay-engine.d.ts.map +1 -0
- package/dist/verify/replay-engine.js +161 -0
- package/dist/verify/replay-engine.js.map +1 -0
- package/dist/verify/replay.d.ts +15 -0
- package/dist/verify/replay.d.ts.map +1 -1
- package/dist/verify/replay.js +7 -1
- package/dist/verify/replay.js.map +1 -1
- package/package.json +14 -3
- package/src/diff/index.ts +8 -7
- package/src/drift/drift.ts +55 -15
- package/src/emit/postgres.ts +70 -12
- package/src/emit/sqlite.ts +9 -2
- package/src/expected-schema.ts +57 -2
- package/src/index.ts +18 -2
- package/src/qualified-name.ts +22 -0
- package/src/scope.ts +261 -0
- package/src/snapshot/plan.ts +45 -13
- package/src/types.ts +17 -0
- package/src/unmanaged.ts +2 -3
- package/src/verify/replay-engine.ts +184 -0
- package/src/verify/replay.ts +21 -1
package/src/scope.ts
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
// Per-command scope — narrowing a migrate/verify run to the objects it governs.
|
|
2
|
+
//
|
|
3
|
+
// A consumer sharing a database with another owner declares
|
|
4
|
+
// `"migrate": { "scope": ["acme::platform::**"] }`. Tables and views outside that
|
|
5
|
+
// scope are neither created nor dropped, which takes TWO suppressions:
|
|
6
|
+
//
|
|
7
|
+
// 1. drop them from the EXPECTED side, so nothing is created or altered;
|
|
8
|
+
// 2. suppress the same names on the ACTUAL side (via `diff`'s `unmanagedNames`,
|
|
9
|
+
// the seam `@unmanaged` already uses), so nothing is dropped.
|
|
10
|
+
//
|
|
11
|
+
// Doing only (1) is strictly worse than doing nothing: every out-of-scope table that
|
|
12
|
+
// EXISTS in the database becomes a proposed `DROP TABLE` — the precise hazard this
|
|
13
|
+
// feature exists to remove. `scopeExpectedSchema` therefore returns both halves and
|
|
14
|
+
// callers must thread `outOfScope` into the diff.
|
|
15
|
+
//
|
|
16
|
+
// There is a THIRD half, and it is the one that bites hardest when the scope is
|
|
17
|
+
// wrong. `diff` derives its SCHEMA scope from the schemas the expected side
|
|
18
|
+
// mentions, falling back to "no schema scoping at all" when expected is empty (the
|
|
19
|
+
// legacy whole-DB path for a project with no model). A scope matching NOTHING
|
|
20
|
+
// empties `expected`, reaches that fallback, and every actual table in every schema
|
|
21
|
+
// becomes a drop candidate — another owner's included, which was never in `expected`
|
|
22
|
+
// so it has no provenance and never lands in `outOfScope`. Narrowing must never
|
|
23
|
+
// WIDEN. `declaredSchemas` below reports the UNSCOPED model's schemas so callers can
|
|
24
|
+
// pin `diff`'s `scopeSchemas` to a property of the whole model, which `migrate.scope`
|
|
25
|
+
// then cannot move in either direction.
|
|
26
|
+
//
|
|
27
|
+
// THE RULE THAT FOLLOWS FROM THAT, stated once because it is easy to read the other
|
|
28
|
+
// way: **a scope narrows which OBJECTS the tool governs, never which SCHEMAS it is
|
|
29
|
+
// allowed to see.** Pinning `scopeSchemas` to the unscoped model means a scope that
|
|
30
|
+
// excludes every declared object in schema `X` leaves `X` in scope, so another
|
|
31
|
+
// owner's UNDECLARED table in `X` stays a drop candidate — exactly as it would be on
|
|
32
|
+
// an unscoped run of the same model. That is deliberate: a schema this model
|
|
33
|
+
// declares into is a schema this model manages, and deriving the schema set from the
|
|
34
|
+
// survivors instead is precisely the inversion above. Declaring a scope is not a way
|
|
35
|
+
// to hand a schema over; removing the objects from the model is.
|
|
36
|
+
//
|
|
37
|
+
// `scopedDiffInputs` exists so no caller has to remember any of this: it returns all
|
|
38
|
+
// three obligations as one object, and every scoped `diff` call goes through it.
|
|
39
|
+
|
|
40
|
+
import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata";
|
|
41
|
+
import type { DiffArgs } from "./diff/index.js";
|
|
42
|
+
import type { ExpectedSchemaWithProvenance } from "./expected-schema.js";
|
|
43
|
+
import { qualifiedDbName } from "./qualified-name.js";
|
|
44
|
+
import type { SchemaSnapshot } from "./types.js";
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Decides whether an object's fully-qualified name (`resolutionKey()`) is governed
|
|
48
|
+
* by this run. Supplied by the caller as a PREDICATE so migrate-ts never carries a
|
|
49
|
+
* second implementation of the scope-pattern grammar — `matchesScope` in
|
|
50
|
+
* `@metaobjectsdev/sdk` is the only one, and the CLI adapts a compiled scope to
|
|
51
|
+
* this seam.
|
|
52
|
+
*/
|
|
53
|
+
export type ObjectScopePredicate = (fqn: string) => boolean;
|
|
54
|
+
|
|
55
|
+
export interface ScopedExpectedSchema {
|
|
56
|
+
/** The expected schema narrowed to the governed objects. */
|
|
57
|
+
snapshot: SchemaSnapshot;
|
|
58
|
+
/**
|
|
59
|
+
* Qualified physical names (`<schema>.<name>`) of the tables and views removed
|
|
60
|
+
* above. Reaches `diff`'s `unmanagedNames` (MERGED with `collectUnmanagedNames`,
|
|
61
|
+
* never replacing it) so the actual side is suppressed too — `scopedDiffInputs`
|
|
62
|
+
* does that merge; see the module header for why omitting it inverts the feature.
|
|
63
|
+
*/
|
|
64
|
+
outOfScope: string[];
|
|
65
|
+
/**
|
|
66
|
+
* The database schemas the UNSCOPED model declares, for `diff`'s `scopeSchemas`.
|
|
67
|
+
* `scopedDiffInputs` threads it — see the module header: without it a scope
|
|
68
|
+
* matching nothing hands `diff` an empty expected side, which it reads as "no
|
|
69
|
+
* model, govern the whole database".
|
|
70
|
+
*
|
|
71
|
+
* `undefined` when no predicate was supplied (so `diff` derives its own set from
|
|
72
|
+
* an untouched `expected`, exactly as before — an unscoped project's arguments are
|
|
73
|
+
* unchanged) and also when the unscoped model declares no tables or views at all
|
|
74
|
+
* (nothing to derive from; `diff`'s legacy whole-DB fallback is preserved).
|
|
75
|
+
*/
|
|
76
|
+
declaredSchemas?: string[];
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Carry an out-of-scope object forward into the snapshot a run is about to commit.
|
|
81
|
+
*
|
|
82
|
+
* The committed snapshot is built from the metadata-expected schema, which a scoped
|
|
83
|
+
* run has already narrowed — so accepting a scoped run DELETES every out-of-scope
|
|
84
|
+
* entry the previous snapshot held. Widening or removing `migrate.scope` later then
|
|
85
|
+
* proposes `CREATE TABLE` for a table that exists, and the migration fails at apply.
|
|
86
|
+
*
|
|
87
|
+
* `prior` is the snapshot (or introspected schema) the run diffed against, and the
|
|
88
|
+
* entries taken from it are exactly the ones this run excluded — nothing else is
|
|
89
|
+
* carried, so a table the model never declared is unaffected either way. An empty
|
|
90
|
+
* `outOfScope` returns the SAME object, so an unscoped run commits a byte-identical
|
|
91
|
+
* snapshot.
|
|
92
|
+
*/
|
|
93
|
+
export function carryForwardOutOfScope(
|
|
94
|
+
next: SchemaSnapshot,
|
|
95
|
+
prior: SchemaSnapshot,
|
|
96
|
+
outOfScope: readonly string[],
|
|
97
|
+
): SchemaSnapshot {
|
|
98
|
+
if (outOfScope.length === 0) return next;
|
|
99
|
+
const excluded = new Set(outOfScope);
|
|
100
|
+
return {
|
|
101
|
+
...next,
|
|
102
|
+
tables: [...next.tables, ...splitOnName(prior.tables, excluded).named],
|
|
103
|
+
views: [...next.views, ...splitOnName(prior.views, excluded).named],
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Drop the out-of-scope entries from a COMMITTED SNAPSHOT, producing the same
|
|
109
|
+
* three-part shape `scopeExpectedSchema` produces so the result can go straight
|
|
110
|
+
* through {@link scopedDiffInputs}.
|
|
111
|
+
*
|
|
112
|
+
* `verify`'s committed-snapshot gate (#292) needs this: `unmanagedNames` suppresses
|
|
113
|
+
* only the ACTUAL side, which is right when the expected side is the metadata (it is
|
|
114
|
+
* already scoped) and wrong here, where the expected side IS the snapshot — a
|
|
115
|
+
* snapshot written before the scope was declared still carries the other owner's
|
|
116
|
+
* tables, and leaving them in reports a phantom disagreement about an object this
|
|
117
|
+
* consumer does not manage.
|
|
118
|
+
*
|
|
119
|
+
* `governed` is the scope decision the caller's drift comparison already made — pass
|
|
120
|
+
* the `DriftResult` itself, which satisfies this shape. Taking `declaredSchemas`
|
|
121
|
+
* from there rather than re-deriving it from the snapshot is what closes the last
|
|
122
|
+
* whole-database door: a snapshot that is present but EMPTY (a never-migrated
|
|
123
|
+
* project) declares no schemas at all, so deriving from it hands `diff` nothing and
|
|
124
|
+
* reaches its "no model, govern the whole database" fallback — the very inversion
|
|
125
|
+
* this module exists to prevent, at the one call site that was still re-deriving.
|
|
126
|
+
*
|
|
127
|
+
* An empty `outOfScope` returns the SAME snapshot object with no schema pin, so an
|
|
128
|
+
* unscoped project's `diff` arguments are byte-for-byte what they always were.
|
|
129
|
+
*/
|
|
130
|
+
export function excludeFromSnapshot(
|
|
131
|
+
snapshot: SchemaSnapshot,
|
|
132
|
+
governed: GovernedScope,
|
|
133
|
+
): ScopedExpectedSchema {
|
|
134
|
+
if (governed.outOfScope.length === 0) return { snapshot, outOfScope: [] };
|
|
135
|
+
const excluded = new Set(governed.outOfScope);
|
|
136
|
+
const declared = governed.declaredSchemas ?? declaredSchemasOf(snapshot);
|
|
137
|
+
return {
|
|
138
|
+
snapshot: {
|
|
139
|
+
...snapshot,
|
|
140
|
+
tables: splitOnName(snapshot.tables, excluded).rest,
|
|
141
|
+
views: splitOnName(snapshot.views, excluded).rest,
|
|
142
|
+
},
|
|
143
|
+
outOfScope: [...governed.outOfScope],
|
|
144
|
+
declaredSchemas: [...declared],
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/** The scope decision a run made, as `DriftResult` reports it. */
|
|
149
|
+
export interface GovernedScope {
|
|
150
|
+
/** Qualified physical names (`<schema>.<name>`) the run does not govern. */
|
|
151
|
+
readonly outOfScope: readonly string[];
|
|
152
|
+
/** The schemas the run governs — `ScopedExpectedSchema.declaredSchemas`. */
|
|
153
|
+
readonly declaredSchemas?: readonly string[] | undefined;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* Partition `objs` on whether `qualifiedDbName(o)` is in `names`.
|
|
158
|
+
*
|
|
159
|
+
* `carryForwardOutOfScope` wants the `named` half (carry the excluded entries
|
|
160
|
+
* forward) and `excludeFromSnapshot` wants the `rest` half (drop them). They are
|
|
161
|
+
* exact complements over the same key function, so they share one traversal rather
|
|
162
|
+
* than two filters that could come to key differently.
|
|
163
|
+
*/
|
|
164
|
+
function splitOnName<T extends { name: string; schema?: string }>(
|
|
165
|
+
objs: readonly T[],
|
|
166
|
+
names: ReadonlySet<string>,
|
|
167
|
+
): { named: T[]; rest: T[] } {
|
|
168
|
+
const named: T[] = [];
|
|
169
|
+
const rest: T[] = [];
|
|
170
|
+
for (const o of objs) (names.has(qualifiedDbName(o)) ? named : rest).push(o);
|
|
171
|
+
return { named, rest };
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* The three `diff` arguments a scoped run owes, as ONE value.
|
|
176
|
+
*
|
|
177
|
+
* The module header lists them as three separate obligations, and five call sites
|
|
178
|
+
* re-derived them by hand — one of which had already drifted into its own guard.
|
|
179
|
+
* Every scoped `diff` call is now
|
|
180
|
+
* `diff({ ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, ... })`,
|
|
181
|
+
* so the rule is enforced by the type rather than by the comment.
|
|
182
|
+
*
|
|
183
|
+
* `unmanaged` is the `@unmanaged`-declared set (`collectUnmanagedNames`); it is
|
|
184
|
+
* MERGED with `outOfScope`, never replaced by it — both must reach `diff`.
|
|
185
|
+
* `scopeSchemas` is omitted entirely when the run narrowed nothing, so an unscoped
|
|
186
|
+
* project's arguments are unchanged.
|
|
187
|
+
*/
|
|
188
|
+
export function scopedDiffInputs(
|
|
189
|
+
scoped: ScopedExpectedSchema,
|
|
190
|
+
unmanaged: readonly string[],
|
|
191
|
+
): Pick<DiffArgs, "expected" | "unmanagedNames" | "scopeSchemas"> {
|
|
192
|
+
return {
|
|
193
|
+
expected: scoped.snapshot,
|
|
194
|
+
unmanagedNames: [...unmanaged, ...scoped.outOfScope],
|
|
195
|
+
...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}),
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* The distinct database schemas a snapshot's tables and views sit in, absent
|
|
201
|
+
* normalized to the Postgres default — the value `diff` derives for itself when no
|
|
202
|
+
* `scopeSchemas` is supplied. The ONE definition: any caller narrowing an expected
|
|
203
|
+
* side must pin `diff`'s schema scope to the UNNARROWED snapshot's schemas, and a
|
|
204
|
+
* second encoding of "absent means public" here would silently disagree with the
|
|
205
|
+
* one inside `diff`.
|
|
206
|
+
*
|
|
207
|
+
* Empty in ⇒ empty out, which callers translate to "pass nothing", preserving
|
|
208
|
+
* `diff`'s legacy whole-database fallback for a genuinely empty model.
|
|
209
|
+
*/
|
|
210
|
+
export function declaredSchemasOf(snapshot: SchemaSnapshot): string[] {
|
|
211
|
+
return [
|
|
212
|
+
...new Set(
|
|
213
|
+
[...snapshot.tables, ...snapshot.views].map(
|
|
214
|
+
(o) => o.schema ?? DEFAULT_DB_SCHEMA_POSTGRES,
|
|
215
|
+
),
|
|
216
|
+
),
|
|
217
|
+
].sort();
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Narrow an expected schema to the objects inside `inScope`.
|
|
222
|
+
*
|
|
223
|
+
* An undefined predicate returns the input untouched — the SAME snapshot object,
|
|
224
|
+
* not an equal copy — so a project that declares no `migrate.scope` reaches the
|
|
225
|
+
* diff, the emitter and the committed snapshot through an unchanged value.
|
|
226
|
+
*
|
|
227
|
+
* A table or view with NO recorded provenance is KEPT. Scope decides on the
|
|
228
|
+
* declaring object's FQN, and an object whose FQN is unknown was never proven to be
|
|
229
|
+
* anyone else's; dropping it would silently un-manage it (and, worse, suppressing
|
|
230
|
+
* its name on the actual side would hide real drift).
|
|
231
|
+
*/
|
|
232
|
+
export function scopeExpectedSchema(
|
|
233
|
+
built: ExpectedSchemaWithProvenance,
|
|
234
|
+
inScope: ObjectScopePredicate | undefined,
|
|
235
|
+
): ScopedExpectedSchema {
|
|
236
|
+
if (inScope === undefined) return { snapshot: built.snapshot, outOfScope: [] };
|
|
237
|
+
|
|
238
|
+
// Computed from `built.snapshot` — the UNSCOPED side — deliberately, and before
|
|
239
|
+
// the filter below runs. Deriving it from the survivors would reproduce exactly
|
|
240
|
+
// the defect this exists to close.
|
|
241
|
+
const declared = declaredSchemasOf(built.snapshot);
|
|
242
|
+
|
|
243
|
+
const outOfScope: string[] = [];
|
|
244
|
+
const governed = <T extends { name: string; schema?: string }>(obj: T): boolean => {
|
|
245
|
+
const qualified = qualifiedDbName(obj);
|
|
246
|
+
const fqn = built.provenance.get(qualified);
|
|
247
|
+
if (fqn === undefined || inScope(fqn)) return true;
|
|
248
|
+
outOfScope.push(qualified);
|
|
249
|
+
return false;
|
|
250
|
+
};
|
|
251
|
+
|
|
252
|
+
return {
|
|
253
|
+
snapshot: {
|
|
254
|
+
...built.snapshot,
|
|
255
|
+
tables: built.snapshot.tables.filter(governed),
|
|
256
|
+
views: built.snapshot.views.filter(governed),
|
|
257
|
+
},
|
|
258
|
+
outOfScope,
|
|
259
|
+
...(declared.length > 0 ? { declaredSchemas: declared } : {}),
|
|
260
|
+
};
|
|
261
|
+
}
|
package/src/snapshot/plan.ts
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
1
|
// src/snapshot/plan.ts
|
|
2
2
|
import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata";
|
|
3
|
-
import { buildExpectedSchema } from "../expected-schema.js";
|
|
3
|
+
import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js";
|
|
4
4
|
import { diff, type DiffArgs } from "../diff/index.js";
|
|
5
5
|
import { collectUnmanagedNames } from "../unmanaged.js";
|
|
6
|
+
import { carryForwardOutOfScope, scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js";
|
|
6
7
|
import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js";
|
|
7
8
|
import type { ExpectedViewInput } from "../expected-schema.js";
|
|
8
9
|
|
|
@@ -14,13 +15,34 @@ export interface PlanOfflineArgs extends Pick<DiffArgs, "allow" | "onAmbiguous"
|
|
|
14
15
|
columnNamingStrategy?: ColumnNamingStrategy;
|
|
15
16
|
/** Expected views (via codegen-ts `buildProjectionViews`) — threaded into buildExpectedSchema. */
|
|
16
17
|
views?: readonly ExpectedViewInput[];
|
|
18
|
+
/**
|
|
19
|
+
* Per-command scope (`migrate.scope`): objects whose declaring FQN this predicate
|
|
20
|
+
* rejects are governed by somebody else. They leave the expected side (so no
|
|
21
|
+
* create/alter) AND the snapshot side (so no drop). Omit to govern everything
|
|
22
|
+
* loaded (unchanged behavior).
|
|
23
|
+
*/
|
|
24
|
+
inScope?: ObjectScopePredicate;
|
|
17
25
|
}
|
|
18
26
|
|
|
19
27
|
export interface PlanOfflineResult {
|
|
20
28
|
/** The change set to emit, from diffing metadata-expected against the snapshot. */
|
|
21
29
|
diff: DiffResult;
|
|
22
|
-
/**
|
|
30
|
+
/**
|
|
31
|
+
* The schema the migration brings us to — write this back as the new snapshot on
|
|
32
|
+
* accept. Under a scope this is the governed schema PLUS the out-of-scope entries
|
|
33
|
+
* the prior snapshot held: committing the narrowed schema would delete them, and a
|
|
34
|
+
* later widening would then propose CREATE TABLE for a table that exists.
|
|
35
|
+
*/
|
|
23
36
|
nextSnapshot: SchemaSnapshot;
|
|
37
|
+
/**
|
|
38
|
+
* The governed (narrowed) expected side — what the diff compared and what the
|
|
39
|
+
* emitter renders against. Identical to `nextSnapshot` for an unscoped run; under
|
|
40
|
+
* a scope it is deliberately the SMALLER of the two, since a run must emit DDL
|
|
41
|
+
* only for what it governs.
|
|
42
|
+
*/
|
|
43
|
+
expected: SchemaSnapshot;
|
|
44
|
+
/** Qualified physical names excluded by `inScope`; empty when no scope was given. */
|
|
45
|
+
outOfScope: readonly string[];
|
|
24
46
|
}
|
|
25
47
|
|
|
26
48
|
/**
|
|
@@ -29,28 +51,38 @@ export interface PlanOfflineResult {
|
|
|
29
51
|
* accept, persists `nextSnapshot` via writeSnapshot.
|
|
30
52
|
*/
|
|
31
53
|
export async function planOffline(args: PlanOfflineArgs): Promise<PlanOfflineResult> {
|
|
32
|
-
const
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
54
|
+
const scoped = scopeExpectedSchema(
|
|
55
|
+
buildExpectedSchemaWithProvenance(args.metadata, {
|
|
56
|
+
dialect: args.dialect,
|
|
57
|
+
...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}),
|
|
58
|
+
...(args.views !== undefined ? { views: args.views } : {}),
|
|
59
|
+
}),
|
|
60
|
+
args.inScope,
|
|
61
|
+
);
|
|
62
|
+
// The DIFF runs against the narrowed side; the SNAPSHOT keeps what this run
|
|
63
|
+
// excluded. Committing the narrowed schema would delete every out-of-scope entry
|
|
64
|
+
// the previous snapshot held, so removing or widening `migrate.scope` later would
|
|
65
|
+
// propose CREATE TABLE for a table that exists and fail at apply. Byte-identical
|
|
66
|
+
// for an unscoped run (`outOfScope` empty ⇒ the same object).
|
|
67
|
+
const nextSnapshot = carryForwardOutOfScope(scoped.snapshot, args.snapshot, scoped.outOfScope);
|
|
37
68
|
const result = await diff({
|
|
38
|
-
|
|
69
|
+
// The three scoped-diff obligations as one value (see scope.ts's header). The
|
|
70
|
+
// `unmanagedNames` merge matters as much on the OFFLINE path as anywhere: an
|
|
71
|
+
// out-of-scope table already recorded in the snapshot must not be dropped just
|
|
72
|
+
// because the scope excludes it, and neither must a declared-@unmanaged one a
|
|
73
|
+
// `baseline --from-db` captured (#208 §7).
|
|
74
|
+
...scopedDiffInputs(scoped, collectUnmanagedNames(args.metadata)),
|
|
39
75
|
actual: args.snapshot,
|
|
40
76
|
dialect: args.dialect,
|
|
41
77
|
// #258 — migration generation refuses a primary-key MOVE (there is no primary-key
|
|
42
78
|
// change kind to express it; it would otherwise silently drop the constraint). The
|
|
43
79
|
// read-only verify/drift path does NOT set this, so `meta verify` still reports drift.
|
|
44
80
|
refusePrimaryKeyChange: true,
|
|
45
|
-
// #208 §7 — exclude declared-@unmanaged objects from the actual (snapshot) side too,
|
|
46
|
-
// so the OFFLINE generate path never proposes DROP for an external table that a
|
|
47
|
-
// `baseline --from-db` captured into the snapshot (parity with the online/verify paths).
|
|
48
|
-
unmanagedNames: collectUnmanagedNames(args.metadata),
|
|
49
81
|
...(args.allow ? { allow: args.allow } : {}),
|
|
50
82
|
...(args.onAmbiguous ? { onAmbiguous: args.onAmbiguous } : {}),
|
|
51
83
|
...(args.ignoreTables ? { ignoreTables: args.ignoreTables } : {}),
|
|
52
84
|
});
|
|
53
|
-
return { diff: result, nextSnapshot };
|
|
85
|
+
return { diff: result, nextSnapshot, expected: scoped.snapshot, outOfScope: scoped.outOfScope };
|
|
54
86
|
}
|
|
55
87
|
|
|
56
88
|
/** Seed an initial reference snapshot from metadata (greenfield baseline). */
|
package/src/types.ts
CHANGED
|
@@ -347,6 +347,23 @@ export interface AllowOptions {
|
|
|
347
347
|
* skips the default-diff for a live auto-sequence default entirely.)
|
|
348
348
|
*/
|
|
349
349
|
dropIdentityDefault?: boolean;
|
|
350
|
+
/**
|
|
351
|
+
* Permits dropping an object the COMMITTED SNAPSHOT never contained — i.e. one
|
|
352
|
+
* this toolchain never managed. Without it, such a drop is refused at generation
|
|
353
|
+
* time, because the migration it would write cannot replay against a database
|
|
354
|
+
* where that object never existed (#313). `classify.ts` already states the
|
|
355
|
+
* doctrine: objects present in the DB but not the snapshot "must never be treated
|
|
356
|
+
* as actionable drift or auto-dropped".
|
|
357
|
+
*
|
|
358
|
+
* The ONE field here read by the CLI's generation-time provenance guard rather
|
|
359
|
+
* than by `diff()`'s status pass — `diff` compares metadata against introspection
|
|
360
|
+
* and never sees the snapshot, which is precisely why the doctrine was not
|
|
361
|
+
* enforced where it mattered. It lives in `AllowOptions` anyway so `--allow` keeps
|
|
362
|
+
* ONE token list and ONE grant map (`ALLOW_TOKENS` / `ALLOW_TOKEN_MAP`, pinned
|
|
363
|
+
* together by `cli/test/unit/allow-tokens-pinned.test.ts`): a second parallel
|
|
364
|
+
* validation path for a single token is the exact drift that pin exists to catch.
|
|
365
|
+
*/
|
|
366
|
+
dropUnmanaged?: boolean;
|
|
350
367
|
}
|
|
351
368
|
|
|
352
369
|
export type AmbiguousChange =
|
package/src/unmanaged.ts
CHANGED
|
@@ -10,10 +10,10 @@
|
|
|
10
10
|
import {
|
|
11
11
|
isMetaSource,
|
|
12
12
|
resolveTableSchema,
|
|
13
|
-
DEFAULT_DB_SCHEMA_POSTGRES,
|
|
14
13
|
TYPE_OBJECT,
|
|
15
14
|
type MetaData,
|
|
16
15
|
} from "@metaobjectsdev/metadata";
|
|
16
|
+
import { qualifiedDbName } from "./qualified-name.js";
|
|
17
17
|
|
|
18
18
|
/**
|
|
19
19
|
* The qualified physical names (`schema.name`, schema defaulting to Postgres `public`)
|
|
@@ -38,8 +38,7 @@ export function collectUnmanagedNames(root: MetaData): string[] {
|
|
|
38
38
|
// make the class check false and silently un-silence a declared-@unmanaged
|
|
39
39
|
// object, turning it back into a proposed drop.
|
|
40
40
|
if (!isMetaSource(src) || !src.isUnmanaged) continue;
|
|
41
|
-
|
|
42
|
-
out.push(`${schema}.${src.physicalName}`);
|
|
41
|
+
out.push(qualifiedDbName({ name: src.physicalName, schema: resolveTableSchema(obj) }));
|
|
43
42
|
}
|
|
44
43
|
}
|
|
45
44
|
return out;
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
// An empty, throwaway database that lives INSIDE this process.
|
|
2
|
+
//
|
|
3
|
+
// The replay gate has to apply a whole committed chain from nothing. Doing that
|
|
4
|
+
// against the user's server would mean CREATE DATABASE — which needs CREATEDB,
|
|
5
|
+
// breaks behind a connection pooler, is restricted on managed Postgres, collides
|
|
6
|
+
// between parallel CI jobs sharing one server, and puts a DROP DATABASE next to a
|
|
7
|
+
// name derived from a real one (Postgres truncates identifiers at 63 bytes, so a
|
|
8
|
+
// long enough target derives a scratch name that truncates back ONTO the target).
|
|
9
|
+
// None of that is worth it when the engine runs locally and disposably: PGlite is
|
|
10
|
+
// real Postgres compiled to WASM and lives in this process; sqlite is a throwaway
|
|
11
|
+
// file in a private temp directory (see `openMemorySqlite` for why not `:memory:`).
|
|
12
|
+
// Nothing to provision, nothing to name, nothing to drop by mistake.
|
|
13
|
+
//
|
|
14
|
+
// Both drivers are OPTIONAL peers imported lazily. PGlite is ~22 MB of WASM and must
|
|
15
|
+
// not land in the node_modules of every adopter who only ever runs `meta gen`; the
|
|
16
|
+
// install hints mirror `buildKyselyFromUrl`'s. `cli` already depends on
|
|
17
|
+
// `@libsql/kysely-libsql` outright, so only a direct embedder can miss that one.
|
|
18
|
+
import { Kysely } from "kysely";
|
|
19
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
20
|
+
import { tmpdir } from "node:os";
|
|
21
|
+
import { join } from "node:path";
|
|
22
|
+
|
|
23
|
+
export interface ReplayEngine {
|
|
24
|
+
/** An empty database. The caller owns applying migrations into it. */
|
|
25
|
+
db: Kysely<Record<string, unknown>>;
|
|
26
|
+
/** Release the engine. Safe to call more than once. */
|
|
27
|
+
dispose: () => Promise<void>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Open an empty in-process database of the given dialect. */
|
|
31
|
+
export async function openReplayEngine(
|
|
32
|
+
dialect: "postgres" | "sqlite",
|
|
33
|
+
): Promise<ReplayEngine> {
|
|
34
|
+
return dialect === "postgres" ? openPglite() : openMemorySqlite();
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* A throwaway sqlite database in a private temp directory, removed on dispose.
|
|
39
|
+
*
|
|
40
|
+
* NOT `:memory:`, and that is the whole point of this comment. Under
|
|
41
|
+
* `@libsql/kysely-libsql`, `:memory:` gives every CONNECTION its own database — so a
|
|
42
|
+
* table created inside a transaction is invisible the moment the transaction's
|
|
43
|
+
* connection is released. `applyPending` runs each migration file in a transaction,
|
|
44
|
+
* which means an in-memory engine would replay a whole chain into a series of
|
|
45
|
+
* throwaway databases, introspect an empty one, and never let migration 2 see
|
|
46
|
+
* migration 1's tables. The gate would pass having proved nothing.
|
|
47
|
+
*
|
|
48
|
+
* `file::memory:?cache=shared` fixes the visibility and breaks isolation instead —
|
|
49
|
+
* two engines in one process land in the SAME database — and libsql rejects the
|
|
50
|
+
* named `?mode=memory&cache=shared` form outright (`URL_PARAM_NOT_SUPPORTED`). A
|
|
51
|
+
* unique temp file is correct on both counts, and it is what the existing
|
|
52
|
+
* `test/integrity/replay.test.ts` has always used.
|
|
53
|
+
*/
|
|
54
|
+
async function openMemorySqlite(): Promise<ReplayEngine> {
|
|
55
|
+
type LibsqlDialectCtor = new (opts: { url: string }) =>
|
|
56
|
+
ConstructorParameters<typeof Kysely<Record<string, unknown>>>[0]["dialect"];
|
|
57
|
+
let LibsqlDialect: LibsqlDialectCtor;
|
|
58
|
+
try {
|
|
59
|
+
const mod = await import("@libsql/kysely-libsql");
|
|
60
|
+
LibsqlDialect = mod.LibsqlDialect as unknown as LibsqlDialectCtor;
|
|
61
|
+
} catch {
|
|
62
|
+
throw new Error(
|
|
63
|
+
`the sqlite replay engine requires '@libsql/kysely-libsql'; install it to run 'meta verify --replay'`,
|
|
64
|
+
);
|
|
65
|
+
}
|
|
66
|
+
const dir = mkdtempSync(join(tmpdir(), "meta-replay-"));
|
|
67
|
+
const db = new Kysely<Record<string, unknown>>({
|
|
68
|
+
dialect: new LibsqlDialect({ url: `file:${join(dir, "replay.db")}` }),
|
|
69
|
+
});
|
|
70
|
+
return disposable(db, async () => {
|
|
71
|
+
rmSync(dir, { recursive: true, force: true });
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
async function openPglite(): Promise<ReplayEngine> {
|
|
76
|
+
let PGliteCtor: new () => PgliteInstance;
|
|
77
|
+
try {
|
|
78
|
+
const mod = await import("@electric-sql/pglite");
|
|
79
|
+
PGliteCtor = mod.PGlite as unknown as new () => PgliteInstance;
|
|
80
|
+
} catch {
|
|
81
|
+
throw new Error(
|
|
82
|
+
`the postgres replay engine requires '@electric-sql/pglite' (in-process WASM Postgres); ` +
|
|
83
|
+
`install it to run 'meta verify --replay' against a postgres chain`,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
const { PostgresDialect } = await import("kysely");
|
|
87
|
+
|
|
88
|
+
// PGlite is Postgres compiled to WASM, and Emscripten propagates the WASM
|
|
89
|
+
// program's internal exit status into `process.exitCode` — it becomes 99 on the
|
|
90
|
+
// FIRST QUERY (not on teardown) and stays there for the life of the process.
|
|
91
|
+
// Opening an engine must not decide what the HOST process exits with, so the
|
|
92
|
+
// caller's value is captured here and restored on dispose.
|
|
93
|
+
//
|
|
94
|
+
// `bin/meta.ts` ends with `process.exit(code)`, which overrides this, so the
|
|
95
|
+
// shipped CLI never showed it. Anything that does NOT force its own exit did:
|
|
96
|
+
// this package's `bun test` exited 99 on 0 failures, turning two
|
|
97
|
+
// `ci-local.sh --only ts` gates red with no failing test to point at, and an
|
|
98
|
+
// embedder calling `openReplayEngine` directly would exit non-zero on success.
|
|
99
|
+
const hostExitCode = process.exitCode;
|
|
100
|
+
const pg = new PGliteCtor();
|
|
101
|
+
const db = new Kysely<Record<string, unknown>>({
|
|
102
|
+
dialect: new PostgresDialect({ pool: pgliteAsPool(pg) as never }),
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
// `?? 0` is load-bearing, not defensive: assigning `undefined` to
|
|
106
|
+
// `process.exitCode` is a NO-OP under Bun (measured — set 99, assign
|
|
107
|
+
// `undefined`, it stays 99; assign 0 and it clears). The pristine value IS
|
|
108
|
+
// `undefined`, so restoring it literally runs and changes nothing — which is
|
|
109
|
+
// the shape this bug already took once during the fix.
|
|
110
|
+
//
|
|
111
|
+
// The restore sits in a `finally` because this close is frequently the SECOND:
|
|
112
|
+
// `disposable` runs `db.destroy()` first, which drives the pool's `end()`,
|
|
113
|
+
// which already called `pg.close()`, so this call throws `PGlite is closed`.
|
|
114
|
+
return disposable(db, async () => {
|
|
115
|
+
try {
|
|
116
|
+
await pg.close();
|
|
117
|
+
} finally {
|
|
118
|
+
process.exitCode = hostExitCode ?? 0;
|
|
119
|
+
}
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** The slice of PGlite's surface this file uses. */
|
|
124
|
+
interface PgliteInstance {
|
|
125
|
+
query(
|
|
126
|
+
sql: string,
|
|
127
|
+
params?: unknown[],
|
|
128
|
+
): Promise<{ rows: unknown[]; affectedRows?: number; statement?: string }>;
|
|
129
|
+
close(): Promise<void>;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Adapt PGlite to the `pg.Pool` shape kysely's `PostgresDialect` expects: `connect()`
|
|
134
|
+
* returning a client with `query()`/`release()`, plus `end()`. PGlite offers only
|
|
135
|
+
* `query`/`close`, so without this the dialect cannot drive it at all.
|
|
136
|
+
*
|
|
137
|
+
* PGlite is a SINGLE session, so every `connect()` hands back the same underlying
|
|
138
|
+
* instance. That is correct here — a replay is strictly sequential — and it is what
|
|
139
|
+
* makes a session advisory lock taken on one kysely connection visible to the next.
|
|
140
|
+
*
|
|
141
|
+
* `command` is read by kysely only to decide whether to report numAffectedRows; the
|
|
142
|
+
* replay path never reads it, so PGlite's `statement` (or a SELECT default) suffices.
|
|
143
|
+
*/
|
|
144
|
+
function pgliteAsPool(pg: PgliteInstance): unknown {
|
|
145
|
+
return {
|
|
146
|
+
async connect() {
|
|
147
|
+
return {
|
|
148
|
+
async query(sqlText: unknown, params?: readonly unknown[]) {
|
|
149
|
+
if (typeof sqlText !== "string") {
|
|
150
|
+
throw new Error(`the PGlite replay engine does not support cursors`);
|
|
151
|
+
}
|
|
152
|
+
const r = await pg.query(sqlText, params ? [...params] : []);
|
|
153
|
+
return {
|
|
154
|
+
command: r.statement ?? "SELECT",
|
|
155
|
+
rowCount: r.affectedRows ?? r.rows.length,
|
|
156
|
+
rows: r.rows,
|
|
157
|
+
};
|
|
158
|
+
},
|
|
159
|
+
release() { /* single session — there is no pool to return to */ },
|
|
160
|
+
};
|
|
161
|
+
},
|
|
162
|
+
async end() {
|
|
163
|
+
await pg.close();
|
|
164
|
+
},
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function disposable(
|
|
169
|
+
db: Kysely<Record<string, unknown>>,
|
|
170
|
+
closeEngine: () => Promise<void>,
|
|
171
|
+
): ReplayEngine {
|
|
172
|
+
let disposed = false;
|
|
173
|
+
return {
|
|
174
|
+
db,
|
|
175
|
+
dispose: async () => {
|
|
176
|
+
if (disposed) return;
|
|
177
|
+
disposed = true;
|
|
178
|
+
// Both swallow: the engine is throwaway, and a teardown error must not mask
|
|
179
|
+
// the replay verdict the caller is about to report.
|
|
180
|
+
try { await db.destroy(); } catch { /* ignore */ }
|
|
181
|
+
try { await closeEngine(); } catch { /* ignore */ }
|
|
182
|
+
},
|
|
183
|
+
};
|
|
184
|
+
}
|
package/src/verify/replay.ts
CHANGED
|
@@ -4,6 +4,7 @@ import { applyPending } from "../apply/apply.js";
|
|
|
4
4
|
import { MIGRATIONS_TABLE } from "../apply/ledger.js";
|
|
5
5
|
import { introspect } from "../introspect/index.js";
|
|
6
6
|
import { driftAgainstSnapshot, type DriftClassification } from "../drift/classify.js";
|
|
7
|
+
import { excludeFromSnapshot, type GovernedScope } from "../scope.js";
|
|
7
8
|
import type { Dialect, SchemaSnapshot } from "../types.js";
|
|
8
9
|
|
|
9
10
|
export interface VerifyReplayArgs {
|
|
@@ -14,6 +15,20 @@ export interface VerifyReplayArgs {
|
|
|
14
15
|
migrationsDir: string;
|
|
15
16
|
/** The committed snapshot the migrations are expected to reproduce. */
|
|
16
17
|
snapshot: SchemaSnapshot;
|
|
18
|
+
/**
|
|
19
|
+
* The scope decision the run made, as `scopeExpectedSchema` reports it.
|
|
20
|
+
*
|
|
21
|
+
* A project declaring `migrate.scope` carries the OTHER owner's tables into its
|
|
22
|
+
* committed snapshot on purpose (`carryForwardOutOfScope`), and its chain — also on
|
|
23
|
+
* purpose — never creates them. Without this they read as missing on every replay,
|
|
24
|
+
* so a scoped project could never use this check at all.
|
|
25
|
+
*
|
|
26
|
+
* Excluded from the SNAPSHOT side only: the replayed database never had them
|
|
27
|
+
* either, so there is nothing to suppress on the actual side. Omitted ⇒ the
|
|
28
|
+
* comparison is byte-for-byte what it was, which is what every unscoped project
|
|
29
|
+
* gets.
|
|
30
|
+
*/
|
|
31
|
+
governed?: GovernedScope;
|
|
17
32
|
}
|
|
18
33
|
|
|
19
34
|
export interface VerifyReplayResult extends DriftClassification {
|
|
@@ -35,7 +50,12 @@ export async function verifyReplay(args: VerifyReplayArgs): Promise<VerifyReplay
|
|
|
35
50
|
...introspected,
|
|
36
51
|
tables: introspected.tables.filter((t) => t.name !== MIGRATIONS_TABLE),
|
|
37
52
|
};
|
|
38
|
-
|
|
53
|
+
// `excludeFromSnapshot` returns a ScopedExpectedSchema, so take `.snapshot`. With an
|
|
54
|
+
// empty `outOfScope` it returns the SAME object, not an equal copy.
|
|
55
|
+
const expected = args.governed !== undefined
|
|
56
|
+
? excludeFromSnapshot(args.snapshot, args.governed).snapshot
|
|
57
|
+
: args.snapshot;
|
|
58
|
+
const classification = await driftAgainstSnapshot(expected, actual, args.dialect);
|
|
39
59
|
return {
|
|
40
60
|
...classification,
|
|
41
61
|
ok: classification.drift.length === 0 && classification.unmanaged.length === 0,
|