@dudousxd/nestjs-catalog 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +123 -0
- package/dist/catalog.controller.d.ts +8 -0
- package/dist/catalog.controller.js +482 -0
- package/dist/catalog.decorators.d.ts +37 -0
- package/dist/catalog.decorators.js +50 -0
- package/dist/catalog.environment.d.ts +442 -0
- package/dist/catalog.environment.js +645 -0
- package/dist/catalog.events.d.ts +179 -0
- package/dist/catalog.events.js +110 -0
- package/dist/catalog.module.d.ts +5 -0
- package/dist/catalog.module.js +71 -0
- package/dist/catalog.options.d.ts +79 -0
- package/dist/catalog.options.js +4 -0
- package/dist/catalog.overlay-store.d.ts +25 -0
- package/dist/catalog.overlay-store.js +44 -0
- package/dist/catalog.overlay-store.token.d.ts +1 -0
- package/dist/catalog.overlay-store.token.js +4 -0
- package/dist/catalog.pipeline.d.ts +800 -0
- package/dist/catalog.pipeline.js +606 -0
- package/dist/catalog.principal.d.ts +209 -0
- package/dist/catalog.principal.js +245 -0
- package/dist/catalog.query-cache.d.ts +25 -0
- package/dist/catalog.query-cache.js +0 -0
- package/dist/catalog.query.d.ts +76 -0
- package/dist/catalog.query.js +64 -0
- package/dist/catalog.registry.base.d.ts +21 -0
- package/dist/catalog.registry.base.js +17 -0
- package/dist/catalog.registry.d.ts +44 -0
- package/dist/catalog.registry.js +359 -0
- package/dist/catalog.service.d.ts +115 -0
- package/dist/catalog.service.js +366 -0
- package/dist/catalog.store.d.ts +419 -0
- package/dist/catalog.store.js +175 -0
- package/dist/catalog.types.d.ts +165 -0
- package/dist/catalog.types.js +19 -0
- package/dist/catalog.workspace.d.ts +426 -0
- package/dist/catalog.workspace.js +87 -0
- package/dist/client.d.ts +86 -0
- package/dist/client.js +83 -0
- package/dist/index.d.ts +19 -0
- package/dist/index.js +109 -0
- package/dist/stores/mikro-orm-read.store.d.ts +20 -0
- package/dist/stores/mikro-orm-read.store.js +120 -0
- package/dist/transform-runner.d.ts +54 -0
- package/dist/transform-runner.js +280 -0
- package/package.json +54 -0
|
@@ -0,0 +1,419 @@
|
|
|
1
|
+
import { BadRequestException } from '@nestjs/common';
|
|
2
|
+
import type { CatalogObjectQuery, CatalogObjectTypeDef } from './catalog.types';
|
|
3
|
+
/**
|
|
4
|
+
* Where the objects actually live.
|
|
5
|
+
*
|
|
6
|
+
* The registry knows what the types *are*; a store knows how to read and write
|
|
7
|
+
* rows of them. Splitting the two is what lets the same catalog sit on top of
|
|
8
|
+
* the application's own tables, a separate warehouse schema, or a column store,
|
|
9
|
+
* without the screens above it changing.
|
|
10
|
+
*/
|
|
11
|
+
/** A point-in-time view of one object type. */
|
|
12
|
+
export interface SnapshotRef {
|
|
13
|
+
/**
|
|
14
|
+
* Opaque and caller-supplied. Callers are expected to pass something they
|
|
15
|
+
* already have that identifies the load — for a durable pipeline, its run id.
|
|
16
|
+
*/
|
|
17
|
+
id: string;
|
|
18
|
+
createdAt: string;
|
|
19
|
+
rowCount: number;
|
|
20
|
+
/**
|
|
21
|
+
* Which application loaded this. Required rather than optional: a shared
|
|
22
|
+
* write path where a snapshot cannot name its author is one where a bad load
|
|
23
|
+
* has no owner, and "who wrote this" is asked long after the logs have
|
|
24
|
+
* rotated.
|
|
25
|
+
*/
|
|
26
|
+
principalId: string;
|
|
27
|
+
/** Free-form provenance: which base, which file, which workflow run. */
|
|
28
|
+
labels?: Record<string, string>;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* How a store holds history. One list, so nothing narrows a stored or
|
|
32
|
+
* transmitted value against a second hand-maintained copy of these names.
|
|
33
|
+
*
|
|
34
|
+
* Ordered weakest to strongest, which is the order a composing store (a fan-out
|
|
35
|
+
* over a primary and its followers) has to intersect them in.
|
|
36
|
+
*/
|
|
37
|
+
export declare const CATALOG_SNAPSHOT_MODES: readonly ["none", "emulated", "native"];
|
|
38
|
+
export type CatalogSnapshotMode = (typeof CATALOG_SNAPSHOT_MODES)[number];
|
|
39
|
+
/**
|
|
40
|
+
* What a store can do. Declared rather than inferred from the engine's name,
|
|
41
|
+
* because the capabilities do not travel together.
|
|
42
|
+
*
|
|
43
|
+
* In particular `snapshots: "native"` is rarer than it looks. Neither DuckDB nor
|
|
44
|
+
* ClickHouse keeps history on its own — DuckDB has no time travel at all, and
|
|
45
|
+
* ClickHouse's ReplacingMergeTree collapses old versions rather than preserving
|
|
46
|
+
* them. What does give history for free is a *table format* (Iceberg, Delta),
|
|
47
|
+
* which those engines can read. So a column store is chosen for read speed over
|
|
48
|
+
* wide tables, not for versioning; it still emulates snapshots like MySQL does.
|
|
49
|
+
*
|
|
50
|
+
* **The three atomicity fields below are optional, and that is a decision.**
|
|
51
|
+
* They arrived after the first adapters shipped, and making them required would
|
|
52
|
+
* have done two bad things at once. The small one is that every existing
|
|
53
|
+
* adapter — including ones outside this repository — stops compiling on a minor
|
|
54
|
+
* version. The large one is that a required `boolean` has no way to say "this
|
|
55
|
+
* adapter has not measured that", so the day it is added every adapter author
|
|
56
|
+
* writes down a guess, and a guess about atomicity is indistinguishable from a
|
|
57
|
+
* measurement once it is a literal in a capability object. `undefined` is a
|
|
58
|
+
* third answer with a meaning: *not stated*. A caller must read it the
|
|
59
|
+
* pessimistic way — an absent `transactional` means "assume a crash can leave
|
|
60
|
+
* this half-done", an absent `atomicCutover` means "assume a reader can catch
|
|
61
|
+
* the swap in progress" — because the cost of assuming the optimistic reading
|
|
62
|
+
* and being wrong is a recovery routine that skips the repair it exists for.
|
|
63
|
+
*/
|
|
64
|
+
export interface CatalogStoreCapabilities {
|
|
65
|
+
/**
|
|
66
|
+
* - `native` — the engine keeps history; reading an old snapshot is a query.
|
|
67
|
+
* - `emulated` — the store tags rows with a snapshot id and filters on read.
|
|
68
|
+
* - `none` — only the current state exists.
|
|
69
|
+
*/
|
|
70
|
+
snapshots: CatalogSnapshotMode;
|
|
71
|
+
/** False for a read-through store over tables someone else owns. */
|
|
72
|
+
writable: boolean;
|
|
73
|
+
/** Whether `read` can be given a `snapshot` other than the latest. */
|
|
74
|
+
timeTravel: boolean;
|
|
75
|
+
/**
|
|
76
|
+
* Whether each step of a commit lands whole, so no reader ever observes a
|
|
77
|
+
* relation that is missing or half-swapped while the cutover happens.
|
|
78
|
+
*
|
|
79
|
+
* This is about each read path individually, not about all of them flipping
|
|
80
|
+
* at the same instant — a store typically repoints an internal pointer and
|
|
81
|
+
* replaces a SQL view in two statements, and whether *those* two can be torn
|
|
82
|
+
* apart by a crash is what {@link transactional} answers. What this field
|
|
83
|
+
* promises is narrower and more immediately useful: that a query issued
|
|
84
|
+
* during the swap gets the old snapshot or the new one, and never an error.
|
|
85
|
+
*
|
|
86
|
+
* It is not a property of the engine, it is a property of the statement the
|
|
87
|
+
* adapter chose. The ClickHouse adapter measured both: hammering a view with
|
|
88
|
+
* 400 concurrent reads while replacing it 400 times with `CREATE OR REPLACE
|
|
89
|
+
* VIEW` produced 18 `UNKNOWN_TABLE` errors, because that statement drops the
|
|
90
|
+
* name and recreates it; `EXCHANGE TABLES` on an Atomic database did the same
|
|
91
|
+
* 400 swaps with none. Same engine, same intent, different answer here.
|
|
92
|
+
*
|
|
93
|
+
* Absent means not stated; read it as false.
|
|
94
|
+
*/
|
|
95
|
+
atomicCutover?: boolean;
|
|
96
|
+
/**
|
|
97
|
+
* Whether re-sending a batch replaces it with no window in which neither the
|
|
98
|
+
* old rows nor the new ones are present.
|
|
99
|
+
*
|
|
100
|
+
* Every store in this ecosystem makes a re-sent batch *idempotent* — that is
|
|
101
|
+
* required by {@link CatalogWriteStore.write} and is not what this asks. This
|
|
102
|
+
* asks what a concurrent reader sees while the replacement happens. MySQL's
|
|
103
|
+
* `DELETE` then `INSERT` are two statements and briefly show neither copy;
|
|
104
|
+
* ClickHouse's `REPLACE PARTITION` from a staging table is one metadata
|
|
105
|
+
* commit and shows one or the other.
|
|
106
|
+
*
|
|
107
|
+
* That difference is invisible for the ordinary case, because a snapshot
|
|
108
|
+
* being written has not been committed and nobody is reading it. It matters
|
|
109
|
+
* for exactly one case, which is also the one nobody plans for: a durable run
|
|
110
|
+
* whose commit succeeded and which then retries from the top, re-sending
|
|
111
|
+
* every batch into a snapshot that is live and being served.
|
|
112
|
+
*
|
|
113
|
+
* Absent means not stated; read it as false.
|
|
114
|
+
*/
|
|
115
|
+
atomicBatchReplace?: boolean;
|
|
116
|
+
/**
|
|
117
|
+
* Whether a multi-statement operation of this store is all-or-nothing.
|
|
118
|
+
*
|
|
119
|
+
* The one a caller must consult before writing recovery logic. False means a
|
|
120
|
+
* crash part-way through `commit()` or `write()` can leave a state no single
|
|
121
|
+
* statement produced — a snapshot marked committed whose view still points at
|
|
122
|
+
* the previous one, say — and that the repair is to re-run the operation
|
|
123
|
+
* rather than to expect a rollback that never happened. Adapters that report
|
|
124
|
+
* false are expected to have ordered their statements so re-running *is* the
|
|
125
|
+
* repair, but a caller that assumes it without asking is assuming something
|
|
126
|
+
* only the adapter knows.
|
|
127
|
+
*
|
|
128
|
+
* ClickHouse reports false and cannot report anything else: it has no
|
|
129
|
+
* transactions across statements. A store on a transactional engine still
|
|
130
|
+
* only reports true if it actually wraps its steps in one, which is a
|
|
131
|
+
* different claim from the engine being capable of it.
|
|
132
|
+
*
|
|
133
|
+
* Absent means not stated; read it as false.
|
|
134
|
+
*/
|
|
135
|
+
transactional?: boolean;
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Narrow something that claims to be a capability object.
|
|
139
|
+
*
|
|
140
|
+
* Exported because a store arrives through an injection token, and a token can
|
|
141
|
+
* be bound to anything: a fan-out composing three stores and a dashboard
|
|
142
|
+
* rendering one both need to check the shape, and two hand-rolled checks that
|
|
143
|
+
* disagree about what counts as a capability object is a store being accepted
|
|
144
|
+
* by one and rejected by the other for reasons neither reports.
|
|
145
|
+
*
|
|
146
|
+
* The three optional fields are checked only when present. An adapter built
|
|
147
|
+
* against a newer version of this package may carry fields this copy has never
|
|
148
|
+
* heard of, and refusing it for that would turn a forward-compatible addition
|
|
149
|
+
* into a boot failure.
|
|
150
|
+
*/
|
|
151
|
+
export declare function isCatalogStoreCapabilities(value: unknown): value is CatalogStoreCapabilities;
|
|
152
|
+
export interface CatalogReadQuery extends CatalogObjectQuery {
|
|
153
|
+
/** Read as of a specific snapshot. Ignored when `timeTravel` is false. */
|
|
154
|
+
snapshot?: string;
|
|
155
|
+
}
|
|
156
|
+
export interface CatalogReadResult {
|
|
157
|
+
rows: Array<Record<string, unknown>>;
|
|
158
|
+
total: number;
|
|
159
|
+
}
|
|
160
|
+
/** The minimum a store must do: return rows of a catalogued type. */
|
|
161
|
+
export interface CatalogReadStore {
|
|
162
|
+
readonly capabilities: CatalogStoreCapabilities;
|
|
163
|
+
/**
|
|
164
|
+
* `type` carries the visibility and classification decisions already applied,
|
|
165
|
+
* and `fields` is the whitelist the caller vouched for — a store must never
|
|
166
|
+
* return a column outside it, whatever the underlying table holds.
|
|
167
|
+
*/
|
|
168
|
+
read(type: CatalogObjectTypeDef, fields: string[], query: CatalogReadQuery): Promise<CatalogReadResult>;
|
|
169
|
+
listSnapshots?(type: CatalogObjectTypeDef): Promise<SnapshotRef[]>;
|
|
170
|
+
}
|
|
171
|
+
/** A store that owns its copy of the data and can be loaded into. */
|
|
172
|
+
export interface CatalogWriteStore extends CatalogReadStore {
|
|
173
|
+
/**
|
|
174
|
+
* Make the physical target match the object type. For a row store this is
|
|
175
|
+
* DDL; for a file-backed format it may be a no-op.
|
|
176
|
+
*
|
|
177
|
+
* Separate from `write` on purpose: schema change and data load have very
|
|
178
|
+
* different blast radii, and a caller may want the first reviewed and the
|
|
179
|
+
* second automatic.
|
|
180
|
+
*/
|
|
181
|
+
ensureType(type: CatalogObjectTypeDef): Promise<void>;
|
|
182
|
+
/**
|
|
183
|
+
* Write a batch belonging to `snapshotId`.
|
|
184
|
+
*
|
|
185
|
+
* Idempotent per `(type, snapshotId, batch)` — a re-sent batch replaces
|
|
186
|
+
* itself rather than adding a second copy. This is not a nicety: a durable
|
|
187
|
+
* step that retries restarts from the top and re-sends every batch, so an
|
|
188
|
+
* append-only write silently doubles the load, and the only symptom is a row
|
|
189
|
+
* count that looks plausible.
|
|
190
|
+
*
|
|
191
|
+
* **`written` is rows-accepted-by-this-call, never rows-in-the-snapshot.** It
|
|
192
|
+
* counts how many of the `rows` handed in this call the store took under
|
|
193
|
+
* `(snapshotId, batch)`. It does not include the batches that came before it,
|
|
194
|
+
* the rows a `carryForward` copied in, or the rows a re-sent batch displaced.
|
|
195
|
+
*
|
|
196
|
+
* Written down because both readings look reasonable and they diverge exactly
|
|
197
|
+
* where it hurts. Two things depend on this one:
|
|
198
|
+
*
|
|
199
|
+
* - A caller that sums `written` across the batches of a load gets the rows
|
|
200
|
+
* that load produced. Under the other reading it would get a running total
|
|
201
|
+
* summed again per batch, which grows quadratically and looks merely large.
|
|
202
|
+
* - A fan-out that writes the same batch to a primary and a follower compares
|
|
203
|
+
* the two numbers and treats a difference as a follower that lost rows.
|
|
204
|
+
* That comparison is only meaningful because both sides are counting the
|
|
205
|
+
* same handful of rows they were just given; against snapshot totals it
|
|
206
|
+
* would report a false mismatch every time the two stores disagreed about
|
|
207
|
+
* anything earlier in the load, including a carry-forward.
|
|
208
|
+
*
|
|
209
|
+
* The snapshot's own total is a different question with its own answers:
|
|
210
|
+
* {@link commit} returns it on the `SnapshotRef`, and {@link
|
|
211
|
+
* CatalogMergeStore.carryForward} returns it as `total`.
|
|
212
|
+
*/
|
|
213
|
+
write(type: CatalogObjectTypeDef, rows: Array<Record<string, unknown>>, options: {
|
|
214
|
+
snapshotId: string;
|
|
215
|
+
principalId: string;
|
|
216
|
+
/** Batch index within this load. Defaults to 0, which is still stable. */
|
|
217
|
+
batch?: number;
|
|
218
|
+
labels?: Record<string, string>;
|
|
219
|
+
}): Promise<{
|
|
220
|
+
written: number;
|
|
221
|
+
}>;
|
|
222
|
+
/**
|
|
223
|
+
* Make a snapshot the one readers get by default. Until this is called, a
|
|
224
|
+
* half-written load must stay invisible — otherwise a crash mid-load is
|
|
225
|
+
* indistinguishable from a completed one that happened to lose rows.
|
|
226
|
+
*/
|
|
227
|
+
commit(type: CatalogObjectTypeDef, snapshotId: string): Promise<SnapshotRef>;
|
|
228
|
+
/** Drop a snapshot. Refuses the currently-committed one. */
|
|
229
|
+
dropSnapshot(type: CatalogObjectTypeDef, snapshotId: string): Promise<void>;
|
|
230
|
+
/**
|
|
231
|
+
* Which snapshot this store is serving for `type`, or `undefined` when it has
|
|
232
|
+
* never committed one.
|
|
233
|
+
*
|
|
234
|
+
* **Why this and not a `committed` flag on `SnapshotRef`.** The flag was the
|
|
235
|
+
* other candidate and it answers a different question. `committed` is true of
|
|
236
|
+
* every snapshot that was ever blessed, not of the one being served, and the
|
|
237
|
+
* two come apart precisely when somebody is relying on the difference: rolling
|
|
238
|
+
* a bad load back means committing an *older* snapshot, after which the newest
|
|
239
|
+
* committed snapshot and the current one are not the same row. A caller
|
|
240
|
+
* reconstructing "current" from a list of committed refs would order by
|
|
241
|
+
* `createdAt` and confidently name the load that was just rolled back. So the
|
|
242
|
+
* store is asked for the pointer it actually reads, because it is the only
|
|
243
|
+
* thing that has it.
|
|
244
|
+
*
|
|
245
|
+
* **Optional, and the fallback is bad enough to be worth stating.** Making it
|
|
246
|
+
* required would break every adapter compiled against an earlier version of
|
|
247
|
+
* this interface, including ones this repository does not own. A caller that
|
|
248
|
+
* finds it absent has only `listSnapshots`, which reports what has been
|
|
249
|
+
* written and does not distinguish committed from half-written — so the
|
|
250
|
+
* fallback is "newest snapshot in the list", and that is a guess. Guessing is
|
|
251
|
+
* survivable for a status screen and is not survivable for the thing this
|
|
252
|
+
* method was asked for, which is a replay deciding whether to commit a
|
|
253
|
+
* repaired snapshot on a follower: guess wrong and the follower is pointed at
|
|
254
|
+
* last week's data, silently, with a full set of plausible rows in it. A
|
|
255
|
+
* caller that cannot get a real answer should refuse and say so rather than
|
|
256
|
+
* commit on a guess.
|
|
257
|
+
*/
|
|
258
|
+
currentSnapshot?(type: CatalogObjectTypeDef): Promise<SnapshotRef | undefined>;
|
|
259
|
+
}
|
|
260
|
+
/** What a carry-forward did. */
|
|
261
|
+
export interface CarryForwardResult {
|
|
262
|
+
/**
|
|
263
|
+
* The snapshot the rows were copied out of. Absent when the type had nothing
|
|
264
|
+
* committed yet, which makes this load the whole of the state rather than an
|
|
265
|
+
* addition to it.
|
|
266
|
+
*/
|
|
267
|
+
from?: string;
|
|
268
|
+
/** Rows copied out of `from` because nothing in this load replaced them. */
|
|
269
|
+
carried: number;
|
|
270
|
+
/** What the snapshot holds now: the carried rows plus the load's own. */
|
|
271
|
+
total: number;
|
|
272
|
+
}
|
|
273
|
+
/**
|
|
274
|
+
* A store that can finish a partial load into a complete snapshot.
|
|
275
|
+
*
|
|
276
|
+
* **The decision this interface encodes.** An incremental load — one that reads
|
|
277
|
+
* only what changed since the last run — forks the snapshot model, and the fork
|
|
278
|
+
* has to be picked once and lived with:
|
|
279
|
+
*
|
|
280
|
+
* - **A snapshot stays the complete state.** The run writes only the rows that
|
|
281
|
+
* changed, then copies the previous snapshot's surviving rows in beside them.
|
|
282
|
+
* Reading is still `WHERE _snapshot_id = X`, time travel is still picking a
|
|
283
|
+
* different X, and the view a query selects from is still one predicate over
|
|
284
|
+
* one table. The cost is the copy.
|
|
285
|
+
* - **A snapshot becomes a delta.** The write is cheap, and everything else gets
|
|
286
|
+
* harder: every read has to union snapshots back to the last full one, the
|
|
287
|
+
* view stops being a filter and becomes a recursive assembly, and time travel
|
|
288
|
+
* turns into a replay whose answer depends on how far back the chain is
|
|
289
|
+
* intact.
|
|
290
|
+
*
|
|
291
|
+
* This library takes the first. Reads and time travel staying trivial is worth
|
|
292
|
+
* a copy that happens once per run, and the second choice would push the merge
|
|
293
|
+
* into every reader — including the ad-hoc SQL people type into the query
|
|
294
|
+
* screen, which is exactly the place a subtle wrong answer never gets caught.
|
|
295
|
+
*
|
|
296
|
+
* The merge key is the object type's primary key. A type without one cannot be
|
|
297
|
+
* merged at all, and implementations must say so rather than fall back to a
|
|
298
|
+
* full reload or append blindly: a load that appears to succeed while making
|
|
299
|
+
* the data meaningless is the worst outcome available here.
|
|
300
|
+
*/
|
|
301
|
+
export interface CatalogMergeStore extends CatalogWriteStore {
|
|
302
|
+
/**
|
|
303
|
+
* Copy the previously committed snapshot's rows into `snapshotId`, letting
|
|
304
|
+
* the rows already written there replace the ones they match on the primary
|
|
305
|
+
* key.
|
|
306
|
+
*
|
|
307
|
+
* **Order matters, and there is only one order.** For an incremental load:
|
|
308
|
+
*
|
|
309
|
+
* 1. `write()` every batch of the run, numbered as usual;
|
|
310
|
+
* 2. `carryForward()` exactly once, *after* the last batch;
|
|
311
|
+
* 3. `commit()`.
|
|
312
|
+
*
|
|
313
|
+
* Step 2 is last because the merge is decided against the batches that exist
|
|
314
|
+
* when it runs — a row written afterwards has nothing to displace, so the old
|
|
315
|
+
* version of it stays and the snapshot ends up holding both. Implementations
|
|
316
|
+
* are expected to notice that case and refuse the commit rather than serve
|
|
317
|
+
* it, because two versions of one object under one primary key is precisely
|
|
318
|
+
* the state the snapshot model promises cannot happen.
|
|
319
|
+
*
|
|
320
|
+
* Safe to call twice with the same arguments: a second call throws away what
|
|
321
|
+
* the first copied and recomputes it. That is not a nicety either — the same
|
|
322
|
+
* durable retry that re-sends every batch re-runs this too.
|
|
323
|
+
*/
|
|
324
|
+
carryForward(type: CatalogObjectTypeDef, snapshotId: string, options: {
|
|
325
|
+
/** Attribution for the snapshot, when this call is what creates it. */
|
|
326
|
+
principalId: string;
|
|
327
|
+
labels?: Record<string, string>;
|
|
328
|
+
}): Promise<CarryForwardResult>;
|
|
329
|
+
}
|
|
330
|
+
/**
|
|
331
|
+
* The columns a snapshot-emulating store adds to every object table.
|
|
332
|
+
*
|
|
333
|
+
* Every store in this ecosystem that keeps history in a column keeps it in
|
|
334
|
+
* these, and they are named here rather than in each adapter for one reason:
|
|
335
|
+
* they are part of what the catalog promises a *reader*. The SQL console
|
|
336
|
+
* documents `_snapshot_id` and `_batch` in its relation list, ad-hoc queries
|
|
337
|
+
* filter on them, and a type whose own properties landed in them would make
|
|
338
|
+
* every one of those queries wrong. A publisher deciding whether a property
|
|
339
|
+
* name is safe should be able to read the answer out of the contract instead of
|
|
340
|
+
* out of whichever adapter happens to be mounted.
|
|
341
|
+
*
|
|
342
|
+
* A store that lays its bookkeeping out differently is free to ignore this
|
|
343
|
+
* list; it is the ecosystem's convention, not a requirement on the interface.
|
|
344
|
+
*/
|
|
345
|
+
export declare const CATALOG_RESERVED_COLUMNS: readonly ["_snapshot_id", "_principal_id", "_loaded_at", "_batch", "_row"];
|
|
346
|
+
export type CatalogReservedColumn = (typeof CATALOG_RESERVED_COLUMNS)[number];
|
|
347
|
+
export declare function isReservedColumn(column: string): boolean;
|
|
348
|
+
/** One property, and the column it cannot have. */
|
|
349
|
+
export interface CatalogColumnCollision {
|
|
350
|
+
/** `reserved` — it lands on a store column. `shared` — two properties collide. */
|
|
351
|
+
kind: 'reserved' | 'shared';
|
|
352
|
+
/** The physical column both sides want. */
|
|
353
|
+
column: string;
|
|
354
|
+
/** The property names involved, in declaration order. */
|
|
355
|
+
properties: string[];
|
|
356
|
+
}
|
|
357
|
+
export interface ColumnCollisionOptions {
|
|
358
|
+
/**
|
|
359
|
+
* Whether the engine considers two column names differing only in case to be
|
|
360
|
+
* the same column. MySQL does; ClickHouse does not.
|
|
361
|
+
*
|
|
362
|
+
* Defaults to false, the narrower reading, so a store on a case-sensitive
|
|
363
|
+
* engine is never handed a refusal for a pair of columns it could genuinely
|
|
364
|
+
* have kept apart. A store that folds case passes true and gets the wider
|
|
365
|
+
* check it needs.
|
|
366
|
+
*/
|
|
367
|
+
foldsColumnCase?: boolean;
|
|
368
|
+
/** Named in the message, so a fan-out's refusal says which store refused. */
|
|
369
|
+
store?: string;
|
|
370
|
+
}
|
|
371
|
+
/**
|
|
372
|
+
* Every way a type's properties would fight over a physical column.
|
|
373
|
+
*
|
|
374
|
+
* Both kinds come from the same place: the property name a publisher chose is
|
|
375
|
+
* not a column name, so every store maps it — stripping the characters a column
|
|
376
|
+
* cannot have, and cutting it to whatever the engine's identifier limit is. The
|
|
377
|
+
* mapping is therefore lossy, and two things fall out of that.
|
|
378
|
+
*
|
|
379
|
+
* `Asset NSN` and `Asset/NSN` are different properties and one column. So are a
|
|
380
|
+
* pair of generated names that agree for their first sixty characters, which
|
|
381
|
+
* sounds unlikely until a publisher emits `metrics_by_installation_and_...`.
|
|
382
|
+
*
|
|
383
|
+
* And a property genuinely called `_batch`, or one called `_ batch` that cleans
|
|
384
|
+
* to the same thing, lands on a column the store keeps its own bookkeeping in.
|
|
385
|
+
*/
|
|
386
|
+
export declare function findColumnCollisions(type: CatalogObjectTypeDef, toColumn: (propertyName: string) => string, options?: ColumnCollisionOptions): CatalogColumnCollision[];
|
|
387
|
+
/** What {@link assertNoColumnCollisions} throws, so a caller can catch it by type. */
|
|
388
|
+
export declare class CatalogColumnCollisionError extends BadRequestException {
|
|
389
|
+
readonly typeName: string;
|
|
390
|
+
readonly collisions: CatalogColumnCollision[];
|
|
391
|
+
constructor(typeName: string, collisions: CatalogColumnCollision[], message: string);
|
|
392
|
+
}
|
|
393
|
+
/**
|
|
394
|
+
* Refuse a type whose properties cannot each have a column of their own.
|
|
395
|
+
*
|
|
396
|
+
* Called by a store before it emits DDL. Every engine this has been tried on
|
|
397
|
+
* *does* fail on its own — MySQL raises "Duplicate column name" on the CREATE
|
|
398
|
+
* TABLE and "Column specified twice" on an INSERT into an existing table — so
|
|
399
|
+
* this is not preventing silent corruption in those cases, with one exception:
|
|
400
|
+
* a property that lands on a reserved column of a store that adds that column
|
|
401
|
+
* separately from the declared ones would be written to happily, and every
|
|
402
|
+
* retry and every incremental merge from then on would replace the wrong rows.
|
|
403
|
+
*
|
|
404
|
+
* What it buys for the loud cases is the only thing they are missing, which is
|
|
405
|
+
* a name. A driver error names the column — and the column is the *derived*
|
|
406
|
+
* name, so the person reading it has to work backwards through a mapping they
|
|
407
|
+
* have never seen to find which two of their forty properties produced it. This
|
|
408
|
+
* says which properties, what they collided over, and what to do, at the moment
|
|
409
|
+
* the type is published rather than on the first load.
|
|
410
|
+
*/
|
|
411
|
+
export declare function assertNoColumnCollisions(type: CatalogObjectTypeDef, toColumn: (propertyName: string) => string, options?: ColumnCollisionOptions): void;
|
|
412
|
+
export declare function isWriteStore(store: CatalogReadStore): store is CatalogWriteStore;
|
|
413
|
+
/**
|
|
414
|
+
* Asked rather than assumed, so a store that cannot merge produces a refusal at
|
|
415
|
+
* the point of the incremental load instead of a snapshot that quietly contains
|
|
416
|
+
* only the rows that happened to change.
|
|
417
|
+
*/
|
|
418
|
+
export declare function supportsCarryForward(store: CatalogWriteStore): store is CatalogMergeStore;
|
|
419
|
+
export declare const CATALOG_STORE: unique symbol;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.CATALOG_STORE = exports.CatalogColumnCollisionError = exports.CATALOG_RESERVED_COLUMNS = exports.CATALOG_SNAPSHOT_MODES = void 0;
|
|
4
|
+
exports.isCatalogStoreCapabilities = isCatalogStoreCapabilities;
|
|
5
|
+
exports.isReservedColumn = isReservedColumn;
|
|
6
|
+
exports.findColumnCollisions = findColumnCollisions;
|
|
7
|
+
exports.assertNoColumnCollisions = assertNoColumnCollisions;
|
|
8
|
+
exports.isWriteStore = isWriteStore;
|
|
9
|
+
exports.supportsCarryForward = supportsCarryForward;
|
|
10
|
+
const common_1 = require("@nestjs/common");
|
|
11
|
+
/**
|
|
12
|
+
* How a store holds history. One list, so nothing narrows a stored or
|
|
13
|
+
* transmitted value against a second hand-maintained copy of these names.
|
|
14
|
+
*
|
|
15
|
+
* Ordered weakest to strongest, which is the order a composing store (a fan-out
|
|
16
|
+
* over a primary and its followers) has to intersect them in.
|
|
17
|
+
*/
|
|
18
|
+
exports.CATALOG_SNAPSHOT_MODES = ['none', 'emulated', 'native'];
|
|
19
|
+
/**
|
|
20
|
+
* Narrow something that claims to be a capability object.
|
|
21
|
+
*
|
|
22
|
+
* Exported because a store arrives through an injection token, and a token can
|
|
23
|
+
* be bound to anything: a fan-out composing three stores and a dashboard
|
|
24
|
+
* rendering one both need to check the shape, and two hand-rolled checks that
|
|
25
|
+
* disagree about what counts as a capability object is a store being accepted
|
|
26
|
+
* by one and rejected by the other for reasons neither reports.
|
|
27
|
+
*
|
|
28
|
+
* The three optional fields are checked only when present. An adapter built
|
|
29
|
+
* against a newer version of this package may carry fields this copy has never
|
|
30
|
+
* heard of, and refusing it for that would turn a forward-compatible addition
|
|
31
|
+
* into a boot failure.
|
|
32
|
+
*/
|
|
33
|
+
function isCatalogStoreCapabilities(value) {
|
|
34
|
+
if (typeof value !== 'object' || value === null)
|
|
35
|
+
return false;
|
|
36
|
+
const snapshots = Reflect.get(value, 'snapshots');
|
|
37
|
+
if (!exports.CATALOG_SNAPSHOT_MODES.some((mode) => mode === snapshots))
|
|
38
|
+
return false;
|
|
39
|
+
if (typeof Reflect.get(value, 'writable') !== 'boolean')
|
|
40
|
+
return false;
|
|
41
|
+
if (typeof Reflect.get(value, 'timeTravel') !== 'boolean')
|
|
42
|
+
return false;
|
|
43
|
+
for (const optional of ['atomicCutover', 'atomicBatchReplace', 'transactional']) {
|
|
44
|
+
const declared = Reflect.get(value, optional);
|
|
45
|
+
if (declared !== undefined && typeof declared !== 'boolean')
|
|
46
|
+
return false;
|
|
47
|
+
}
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* The columns a snapshot-emulating store adds to every object table.
|
|
52
|
+
*
|
|
53
|
+
* Every store in this ecosystem that keeps history in a column keeps it in
|
|
54
|
+
* these, and they are named here rather than in each adapter for one reason:
|
|
55
|
+
* they are part of what the catalog promises a *reader*. The SQL console
|
|
56
|
+
* documents `_snapshot_id` and `_batch` in its relation list, ad-hoc queries
|
|
57
|
+
* filter on them, and a type whose own properties landed in them would make
|
|
58
|
+
* every one of those queries wrong. A publisher deciding whether a property
|
|
59
|
+
* name is safe should be able to read the answer out of the contract instead of
|
|
60
|
+
* out of whichever adapter happens to be mounted.
|
|
61
|
+
*
|
|
62
|
+
* A store that lays its bookkeeping out differently is free to ignore this
|
|
63
|
+
* list; it is the ecosystem's convention, not a requirement on the interface.
|
|
64
|
+
*/
|
|
65
|
+
exports.CATALOG_RESERVED_COLUMNS = [
|
|
66
|
+
'_snapshot_id',
|
|
67
|
+
'_principal_id',
|
|
68
|
+
'_loaded_at',
|
|
69
|
+
'_batch',
|
|
70
|
+
'_row',
|
|
71
|
+
];
|
|
72
|
+
function isReservedColumn(column) {
|
|
73
|
+
return exports.CATALOG_RESERVED_COLUMNS.some((reserved) => reserved === column.toLowerCase());
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Every way a type's properties would fight over a physical column.
|
|
77
|
+
*
|
|
78
|
+
* Both kinds come from the same place: the property name a publisher chose is
|
|
79
|
+
* not a column name, so every store maps it — stripping the characters a column
|
|
80
|
+
* cannot have, and cutting it to whatever the engine's identifier limit is. The
|
|
81
|
+
* mapping is therefore lossy, and two things fall out of that.
|
|
82
|
+
*
|
|
83
|
+
* `Asset NSN` and `Asset/NSN` are different properties and one column. So are a
|
|
84
|
+
* pair of generated names that agree for their first sixty characters, which
|
|
85
|
+
* sounds unlikely until a publisher emits `metrics_by_installation_and_...`.
|
|
86
|
+
*
|
|
87
|
+
* And a property genuinely called `_batch`, or one called `_ batch` that cleans
|
|
88
|
+
* to the same thing, lands on a column the store keeps its own bookkeeping in.
|
|
89
|
+
*/
|
|
90
|
+
function findColumnCollisions(type, toColumn, options) {
|
|
91
|
+
const fold = (column) => options?.foldsColumnCase === true ? column.toLowerCase() : column;
|
|
92
|
+
const byColumn = new Map();
|
|
93
|
+
for (const property of type.properties) {
|
|
94
|
+
const column = toColumn(property.name);
|
|
95
|
+
const key = fold(column);
|
|
96
|
+
const entry = byColumn.get(key) ?? { column, properties: [] };
|
|
97
|
+
entry.properties.push(property.name);
|
|
98
|
+
byColumn.set(key, entry);
|
|
99
|
+
}
|
|
100
|
+
const collisions = [];
|
|
101
|
+
for (const [key, entry] of byColumn) {
|
|
102
|
+
// Reserved first, and reported instead of the shared case rather than
|
|
103
|
+
// beside it: two properties that both land on `_batch` have a worse problem
|
|
104
|
+
// than each other, and naming that one twice would bury it.
|
|
105
|
+
if (isReservedColumn(key)) {
|
|
106
|
+
collisions.push({
|
|
107
|
+
kind: 'reserved',
|
|
108
|
+
column: entry.column,
|
|
109
|
+
properties: entry.properties,
|
|
110
|
+
});
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
if (entry.properties.length > 1) {
|
|
114
|
+
collisions.push({
|
|
115
|
+
kind: 'shared',
|
|
116
|
+
column: entry.column,
|
|
117
|
+
properties: entry.properties,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
return collisions;
|
|
122
|
+
}
|
|
123
|
+
/** What {@link assertNoColumnCollisions} throws, so a caller can catch it by type. */
|
|
124
|
+
class CatalogColumnCollisionError extends common_1.BadRequestException {
|
|
125
|
+
typeName;
|
|
126
|
+
collisions;
|
|
127
|
+
constructor(typeName, collisions, message) {
|
|
128
|
+
super(message);
|
|
129
|
+
this.typeName = typeName;
|
|
130
|
+
this.collisions = collisions;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
exports.CatalogColumnCollisionError = CatalogColumnCollisionError;
|
|
134
|
+
/**
|
|
135
|
+
* Refuse a type whose properties cannot each have a column of their own.
|
|
136
|
+
*
|
|
137
|
+
* Called by a store before it emits DDL. Every engine this has been tried on
|
|
138
|
+
* *does* fail on its own — MySQL raises "Duplicate column name" on the CREATE
|
|
139
|
+
* TABLE and "Column specified twice" on an INSERT into an existing table — so
|
|
140
|
+
* this is not preventing silent corruption in those cases, with one exception:
|
|
141
|
+
* a property that lands on a reserved column of a store that adds that column
|
|
142
|
+
* separately from the declared ones would be written to happily, and every
|
|
143
|
+
* retry and every incremental merge from then on would replace the wrong rows.
|
|
144
|
+
*
|
|
145
|
+
* What it buys for the loud cases is the only thing they are missing, which is
|
|
146
|
+
* a name. A driver error names the column — and the column is the *derived*
|
|
147
|
+
* name, so the person reading it has to work backwards through a mapping they
|
|
148
|
+
* have never seen to find which two of their forty properties produced it. This
|
|
149
|
+
* says which properties, what they collided over, and what to do, at the moment
|
|
150
|
+
* the type is published rather than on the first load.
|
|
151
|
+
*/
|
|
152
|
+
function assertNoColumnCollisions(type, toColumn, options) {
|
|
153
|
+
const collisions = findColumnCollisions(type, toColumn, options);
|
|
154
|
+
if (collisions.length === 0)
|
|
155
|
+
return;
|
|
156
|
+
const where = options?.store ? ` in ${options.store}` : '';
|
|
157
|
+
const described = collisions.map((collision) => collision.kind === 'reserved'
|
|
158
|
+
? `${collision.properties.join(' and ')} would be stored in ${collision.column}, which the store keeps for itself`
|
|
159
|
+
: `${collision.properties.join(' and ')} would both be stored in ${collision.column}`);
|
|
160
|
+
throw new CatalogColumnCollisionError(type.name, collisions, `${type.name} cannot be stored${where} as declared: ${described.join('; ')}. Property names are mapped to columns by replacing anything that is not a letter, digit or underscore and cutting the result to the engine's identifier limit, so two different names can arrive at one column. Rename the ${collisions.some((collision) => collision.properties.length > 1)
|
|
161
|
+
? 'properties so they differ in more than punctuation or length'
|
|
162
|
+
: 'property'}. The reserved columns are ${exports.CATALOG_RESERVED_COLUMNS.join(', ')}.`);
|
|
163
|
+
}
|
|
164
|
+
function isWriteStore(store) {
|
|
165
|
+
return store.capabilities.writable && 'write' in store;
|
|
166
|
+
}
|
|
167
|
+
/**
|
|
168
|
+
* Asked rather than assumed, so a store that cannot merge produces a refusal at
|
|
169
|
+
* the point of the incremental load instead of a snapshot that quietly contains
|
|
170
|
+
* only the rows that happened to change.
|
|
171
|
+
*/
|
|
172
|
+
function supportsCarryForward(store) {
|
|
173
|
+
return 'carryForward' in store;
|
|
174
|
+
}
|
|
175
|
+
exports.CATALOG_STORE = Symbol('CATALOG_STORE');
|