@memberjunction/materialization 0.0.0 → 6.1.0-edge.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +183 -0
- package/README.md +25 -32
- package/dist/MaterializationFreshness.d.ts +55 -0
- package/dist/MaterializationFreshness.d.ts.map +1 -0
- package/dist/MaterializationFreshness.js +124 -0
- package/dist/MaterializationFreshness.js.map +1 -0
- package/dist/MaterializationRefresher.d.ts +621 -0
- package/dist/MaterializationRefresher.d.ts.map +1 -0
- package/dist/MaterializationRefresher.js +1558 -0
- package/dist/MaterializationRefresher.js.map +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/package.json +32 -8
|
@@ -0,0 +1,621 @@
|
|
|
1
|
+
import { IMetadataProvider, UserInfo, EntityInfo } from '@memberjunction/core';
|
|
2
|
+
import { MJMaterializedResultEntity } from '@memberjunction/core-entities';
|
|
3
|
+
/**
|
|
4
|
+
* Synthetic surrogate key column name for query-materialized tables. MUST match CodeGenLib's
|
|
5
|
+
* `MATERIALIZATION_SURROGATE_COLUMN` (materializationAnalysis.ts) — CodeGen creates the column with
|
|
6
|
+
* this name and the refresher must regenerate it on every rebuild. (A shared low-level home for this
|
|
7
|
+
* constant is a follow-up; duplicated deliberately to avoid a runtime dependency on the dev-time CodeGenLib.)
|
|
8
|
+
*/
|
|
9
|
+
export declare const MATERIALIZATION_SURROGATE_COLUMN = "__mj_MaterializedRowID";
|
|
10
|
+
/**
|
|
11
|
+
* Force a full rebuild after this many consecutive incremental (Incremental/DirtyGroupRecompute)
|
|
12
|
+
* refreshes. The incremental delete-detection guard only trips on a NET source row-count drop; a
|
|
13
|
+
* delete BALANCED by an insert in the same window (net-zero change) leaves the deleted row's group
|
|
14
|
+
* stale until another change touches it. This periodic full rebuild bounds that drift to at most
|
|
15
|
+
* this many refresh cycles without requiring the author to schedule a manual FullRebuild.
|
|
16
|
+
*/
|
|
17
|
+
export declare const FULL_REBUILD_EVERY_N_INCREMENTAL_REFRESHES = 10;
|
|
18
|
+
/**
|
|
19
|
+
* Safety lag subtracted from the probed `MAX(__mj_UpdatedAt)` before it is persisted as the incremental
|
|
20
|
+
* watermark. Closes a commit-ordering skew: a source row whose `__mj_UpdatedAt` was stamped at write-time T1
|
|
21
|
+
* but whose transaction COMMITS after the fingerprint probe (which already read a higher `MAX = T2 > T1`) would,
|
|
22
|
+
* without this lag, be permanently excluded by the strict `__mj_UpdatedAt > watermark` filter on the next pass.
|
|
23
|
+
* Storing `MAX - overlap` makes the next incremental RE-scan the last `overlap` window; the MERGE/`ON CONFLICT`
|
|
24
|
+
* upsert is idempotent so re-scanning already-applied rows is harmless. The overlap only needs to exceed the
|
|
25
|
+
* source's typical commit latency — longer skews are still backstopped by {@link FULL_REBUILD_EVERY_N_INCREMENTAL_REFRESHES}.
|
|
26
|
+
*/
|
|
27
|
+
export declare const WATERMARK_SAFETY_OVERLAP_MS = 10000;
|
|
28
|
+
/**
|
|
29
|
+
* Minimal structural type for a runtime SQL-executing provider. Both SQLServerDataProvider and
|
|
30
|
+
* PostgreSQLDataProvider expose `ExecuteSQL`; we depend on the shape, not the concrete class, to
|
|
31
|
+
* avoid coupling this engine to a specific provider package.
|
|
32
|
+
*/
|
|
33
|
+
export interface ISQLExecutor {
|
|
34
|
+
/**
|
|
35
|
+
* Execute SQL, optionally with positional bind parameters. `parameters` is a positional array bound
|
|
36
|
+
* as `@p0,@p1,…` on SQL Server (`request.input('p'+i)`) and `$1,$2,…` on PostgreSQL (node-pg values) —
|
|
37
|
+
* both MJ data providers accept this shape. Omit it for plain (DDL / no-value) statements.
|
|
38
|
+
*/
|
|
39
|
+
ExecuteSQL<T = unknown>(sql: string, parameters?: unknown): Promise<T[]>;
|
|
40
|
+
/** Database platform of the executing provider ('sqlserver' | 'postgresql'); absent => treated as SQL Server. */
|
|
41
|
+
PlatformKey?: string;
|
|
42
|
+
}
|
|
43
|
+
/** Outcome of refreshing a single materialized result. */
|
|
44
|
+
export interface MaterializationRefreshResult {
|
|
45
|
+
Success: boolean;
|
|
46
|
+
RowCount?: number;
|
|
47
|
+
ErrorMessage?: string;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Runtime engine that refreshes materialized query/entity results (materialization plan §11).
|
|
51
|
+
*
|
|
52
|
+
* v1: **full rebuild** with an **atomic wrapper-view swap** — build a shadow table from the source,
|
|
53
|
+
* repoint the stable wrapper view at it, then drop the stale table and rename the shadow into the
|
|
54
|
+
* canonical name. Readers (via the wrapper view) never see a half-populated or locked result.
|
|
55
|
+
* Cross-engine: SQL Server and PostgreSQL — the swap statements differ per engine (see the two
|
|
56
|
+
* `buildFullRebuild*` methods), selected at runtime from the provider's `PlatformKey`.
|
|
57
|
+
*
|
|
58
|
+
* Invoked by the scheduled-job refresh driver, and reusable by a manual "refresh now" path.
|
|
59
|
+
*/
|
|
60
|
+
export declare class MaterializationRefresher {
|
|
61
|
+
/**
|
|
62
|
+
* Forced-full-rebuild cadence decision: should this refresh be forced to a full rebuild? True once the
|
|
63
|
+
* count of consecutive incremental refreshes since the last full rebuild has reached
|
|
64
|
+
* {@link FULL_REBUILD_EVERY_N_INCREMENTAL_REFRESHES}. Pure (no IO) so the cadence boundary is
|
|
65
|
+
* unit-testable without a provider/DB. Null-safe: an unset counter is treated as 0.
|
|
66
|
+
*/
|
|
67
|
+
static shouldForceFullRebuild(refreshesSinceFullRebuild: number | null | undefined): boolean;
|
|
68
|
+
/**
|
|
69
|
+
* Forced-full-rebuild cadence counter transition. Increments on a genuine incremental refresh; resets to
|
|
70
|
+
* 0 on any full rebuild — so the counter measures how many refreshes we've gone WITHOUT a full reconcile.
|
|
71
|
+
* Pure (no IO) so the increment/reset semantics are unit-testable. Null-safe: an unset counter is 0.
|
|
72
|
+
*/
|
|
73
|
+
static nextRefreshesSinceFullRebuild(current: number | null | undefined, ranIncremental: boolean): number;
|
|
74
|
+
/** A plain, unquoted SQL identifier: leading letter/underscore, then letters/digits/underscores. */
|
|
75
|
+
private static readonly SAFE_SQL_IDENTIFIER;
|
|
76
|
+
/** Cached API-key row-filter targets for this refresher; `null` = not yet enumerated (see
|
|
77
|
+
* {@link loadAPIKeyRowFilterTargets}). `'unknown'` is a LOADED state meaning "assume restricted". */
|
|
78
|
+
private _apiKeyRowFilterTargets;
|
|
79
|
+
/**
|
|
80
|
+
* Guards the schema/table/view identifiers that get interpolated into materialization DDL/DML — names
|
|
81
|
+
* read from the *writable* `MJ: Materialized Results` metadata row. A materialization's names are always
|
|
82
|
+
* CodeName-derived (`materialized_<CodeName>`, schema `__mj`), so a legitimate row always passes. The
|
|
83
|
+
* assertion exists so a tampered metadata row can never drive the privileged refresh job's
|
|
84
|
+
* `EXEC(...)` / `sp_rename` / `CREATE VIEW` / `RENAME TO` statements to run arbitrary DDL: a value that
|
|
85
|
+
* matches {@link SAFE_SQL_IDENTIFIER} cannot contain `]`, `"`, or `'`, so this one check closes BOTH the
|
|
86
|
+
* identifier-quoting and the T-SQL string-literal injection surfaces the swap builders would otherwise
|
|
87
|
+
* expose. Fails closed — throws (→ the refresh is reported as failed) rather than emitting a suspect
|
|
88
|
+
* statement. This is the refresh-path complement to the mint-path dialect quoting.
|
|
89
|
+
*/
|
|
90
|
+
private static assertSafeObjectNames;
|
|
91
|
+
/**
|
|
92
|
+
* Non-throwing form of the identifier check. Needed by callers on the FAILURE path, which is precisely
|
|
93
|
+
* where {@link assertSafeObjectNames} may have just thrown — those callers must be able to re-check and
|
|
94
|
+
* decline quietly rather than re-enter (or bypass) the assertion that already rejected the value.
|
|
95
|
+
* @internal exposed for unit testing; not part of the supported surface.
|
|
96
|
+
*/
|
|
97
|
+
static isSafeObjectName(value: string): boolean;
|
|
98
|
+
/**
|
|
99
|
+
* Resolves the SQL that the READ path would execute for `queryId` on the engine we are refreshing against,
|
|
100
|
+
* so the snapshot is built from the same statement live serves. Mirrors the read path's
|
|
101
|
+
* `QueryInfo.GetPlatformSQL(PlatformKey)`, whose precedence is: `MJ: Query SQLs` child row for the platform
|
|
102
|
+
* → legacy PlatformVariants → base SQL.
|
|
103
|
+
*
|
|
104
|
+
* `GetPlatformSQL` lives on the metadata `QueryInfo`, not on the generated `MJQueryEntity`, so the variant
|
|
105
|
+
* is resolved through the provider's query metadata. Falls back to the entity's own SQL when the query
|
|
106
|
+
* isn't present in that metadata (e.g. a provider whose cache hasn't loaded it), which reproduces exactly
|
|
107
|
+
* the previous behavior rather than failing the refresh.
|
|
108
|
+
*
|
|
109
|
+
* @returns the platform-resolved SQL, or null when neither source yields a non-empty statement.
|
|
110
|
+
* @internal exposed for unit testing; not part of the supported surface.
|
|
111
|
+
*/
|
|
112
|
+
static resolvePlatformQuerySQL(provider: IMetadataProvider, queryId: string, entitySql: string | null, isPostgres: boolean): string | null;
|
|
113
|
+
/**
|
|
114
|
+
* Builds the ordered SQL statements for a SQL Server full rebuild with atomic swap (plan §11.2).
|
|
115
|
+
* Pure (no IO) so the swap sequence is unit-testable. Each returned string runs as its own batch.
|
|
116
|
+
*
|
|
117
|
+
* - query case (`surrogateColumn` set): the synthetic IDENTITY surrogate is (re)generated via
|
|
118
|
+
* `SELECT IDENTITY(int,1,1) AS <surrogate>, src.* INTO <shadow>`;
|
|
119
|
+
* - base-view case (no surrogate): `SELECT * INTO <shadow>` copies the source shape (incl. its PK column).
|
|
120
|
+
*/
|
|
121
|
+
static buildFullRebuildStatementsSQLServer(opts: {
|
|
122
|
+
schema: string;
|
|
123
|
+
tableName: string;
|
|
124
|
+
viewName: string;
|
|
125
|
+
sourceSelect: string;
|
|
126
|
+
surrogateColumn?: string;
|
|
127
|
+
hashKeyColumns?: {
|
|
128
|
+
name: string;
|
|
129
|
+
type: string;
|
|
130
|
+
}[];
|
|
131
|
+
/** Run-unique shadow table name (see {@link makeShadowTableName}) so two concurrent refreshes of the
|
|
132
|
+
* same materialization never share a shadow. Defaults to the legacy fixed name when omitted. */
|
|
133
|
+
shadowName?: string;
|
|
134
|
+
}): string[];
|
|
135
|
+
/**
|
|
136
|
+
* Builds the ordered SQL statements for a PostgreSQL full rebuild with atomic swap (plan §11.2) —
|
|
137
|
+
* the PG counterpart to {@link buildFullRebuildStatementsSQLServer}. Pure (no IO), unit-testable.
|
|
138
|
+
*
|
|
139
|
+
* Engine differences vs. SQL Server:
|
|
140
|
+
* - **Identifier quoting:** schema bare, object double-quoted (`__mj."materialized_x"`), matching the
|
|
141
|
+
* CodeGen provider's `QuoteSchema` convention so the view repoint references the same names.
|
|
142
|
+
* - **Surrogate (query case):** the synthetic surrogate is generated **as the first column** via
|
|
143
|
+
* `ROW_NUMBER() OVER ()` (a stable 1..N snapshot id; deterministic hashing is §5/Phase 3). It MUST
|
|
144
|
+
* be first because CodeGen prepends the surrogate, and PG's `CREATE OR REPLACE VIEW` is strict about
|
|
145
|
+
* column order (SQLSTATE 42P16) — an appended surrogate would break the repoint.
|
|
146
|
+
* - **Swap:** `CREATE OR REPLACE VIEW` (not `CREATE OR ALTER`), `ALTER TABLE ... RENAME TO` (not
|
|
147
|
+
* `sp_rename`), and `DROP TABLE IF EXISTS ... CASCADE` (PG blocks dropping a table a view depends on;
|
|
148
|
+
* CASCADE clears a transient wrapper-view dependency from a partially-failed prior run — the view is
|
|
149
|
+
* recreated within this sequence, so the stable contract is restored before the method returns).
|
|
150
|
+
*/
|
|
151
|
+
static buildFullRebuildStatementsPostgreSQL(opts: {
|
|
152
|
+
schema: string;
|
|
153
|
+
tableName: string;
|
|
154
|
+
viewName: string;
|
|
155
|
+
sourceSelect: string;
|
|
156
|
+
surrogateColumn?: string;
|
|
157
|
+
hashKeyColumns?: {
|
|
158
|
+
name: string;
|
|
159
|
+
type: string;
|
|
160
|
+
}[];
|
|
161
|
+
/** Run-unique shadow table name (see {@link makeShadowTableName}) so two concurrent refreshes of the
|
|
162
|
+
* same materialization never share a shadow. Defaults to the legacy fixed name when omitted. */
|
|
163
|
+
shadowName?: string;
|
|
164
|
+
}): string[];
|
|
165
|
+
/**
|
|
166
|
+
* Selects the materializations due for refresh: those with no `NextRefreshAt` (never run) or whose
|
|
167
|
+
* `NextRefreshAt` is at/before `now`. Pure (unit-testable); the caller supplies the candidate rows
|
|
168
|
+
* (e.g. all non-disabled, scheduled materializations).
|
|
169
|
+
*/
|
|
170
|
+
static filterDue<T extends {
|
|
171
|
+
NextRefreshAt?: Date | null;
|
|
172
|
+
}>(rows: T[], now: Date): T[];
|
|
173
|
+
/**
|
|
174
|
+
* Full-rebuild refresh of a single materialized result, then updates LastRefreshedAt / RowCount /
|
|
175
|
+
* Status='Active' (and `NextRefreshAt` when provided via options). Returns a structured result
|
|
176
|
+
* rather than throwing (errors are logged + reported).
|
|
177
|
+
*/
|
|
178
|
+
RefreshOne(matResult: MJMaterializedResultEntity, contextUser: UserInfo, provider: IMetadataProvider, options?: {
|
|
179
|
+
nextRefreshAt?: Date | null;
|
|
180
|
+
}): Promise<MaterializationRefreshResult>;
|
|
181
|
+
/**
|
|
182
|
+
* Drops a run's shadow table if it exists, swallowing any error (used only on the RefreshOne failure path so
|
|
183
|
+
* a crashed/failed rebuild leaves no orphan). Uses IF EXISTS so it's a no-op when the shadow was never
|
|
184
|
+
* created or was already renamed into the canonical table on success.
|
|
185
|
+
*/
|
|
186
|
+
/**
|
|
187
|
+
* Restores `XACT_ABORT` to the connection default after a failed refresh, swallowing any error. SQL Server
|
|
188
|
+
* only — PostgreSQL has no equivalent session setting, so this is a no-op there. Needed because a batch
|
|
189
|
+
* that aborts mid-swap never reaches the trailing `SET XACT_ABORT OFF` in its own statement list.
|
|
190
|
+
*/
|
|
191
|
+
private resetXactAbortBestEffort;
|
|
192
|
+
private dropShadowTableBestEffort;
|
|
193
|
+
/**
|
|
194
|
+
* Common failure exit for RefreshOne. Advances `NextRefreshAt` so a persistently-failing materialization
|
|
195
|
+
* backs off to its configured cadence instead of being retried on every sweep (the driver's filterDue
|
|
196
|
+
* treats an unchanged past/null NextRefreshAt as still-due — an unbounded rebuild/read storm against the
|
|
197
|
+
* source DB). Called only from PRE-SUCCESS failure paths: thrown errors caught in RefreshOne AND the
|
|
198
|
+
* returned {Success:false} paths for external-entity/query read failure and unresolvable source. (The
|
|
199
|
+
* post-success Save-failure path deliberately does NOT use this — see the comment there — because matResult
|
|
200
|
+
* would carry the full success state and re-Saving it would contradict the reported failure.)
|
|
201
|
+
*
|
|
202
|
+
* Only NextRefreshAt is written, via the SAME guarded conditional UPDATE the success path uses
|
|
203
|
+
* (`WHERE Status NOT IN ('DriftHold','Disabled')`) — so a concurrent hold/disable is genuinely never
|
|
204
|
+
* clobbered. (BaseEntity.Save could NOT guarantee this: its spUpdate binds every field with ISNULL + a
|
|
205
|
+
* PK-only WHERE, so the stale in-memory Status='Active' would have overwritten a concurrent hold even though
|
|
206
|
+
* this method never assigns Status — the previous "not dirtying Status keeps it safe" reasoning was wrong.)
|
|
207
|
+
* Best-effort and non-throwing: an update failure is logged, not thrown (the next sweep re-attempts). No-op
|
|
208
|
+
* when the caller supplied no schedule (a manual "refresh now" with no options).
|
|
209
|
+
*/
|
|
210
|
+
private failRefresh;
|
|
211
|
+
/** Core-schema reference for the `MJ: Materialized Results` metadata table. NOT `matResult.SchemaName` —
|
|
212
|
+
* that is the SNAPSHOT table's schema, which differs for a base-view materialization of a non-core entity.
|
|
213
|
+
* The metadata row lives where the entity is defined (`__mj`), read from the entity metadata. */
|
|
214
|
+
private materializedResultTableRef;
|
|
215
|
+
/** Persists the terminal SUCCESS state (Status='Active' + LastRefreshedAt/RowCount/Watermark/SourceRowCount/
|
|
216
|
+
* RefreshesSinceFullRebuild [+ NextRefreshAt]) via {@link execGuardedMaterializedResultUpdate}. Reads the
|
|
217
|
+
* values from `matResult` (RefreshOne has already assigned them in-memory). Returns false when the row was
|
|
218
|
+
* concurrently held/disabled (the UPDATE matched 0 rows). */
|
|
219
|
+
private persistTerminalStateGuarded;
|
|
220
|
+
/** Runs an UPDATE against the MaterializedResult metadata row guarded by `Status NOT IN ('DriftHold',
|
|
221
|
+
* 'Disabled')` — the atomic primitive that makes a status/state write unable to clobber a concurrently-set
|
|
222
|
+
* hold. Returns true iff exactly the target row was updated (i.e. it was still Active). The affected count is
|
|
223
|
+
* read back cross-engine: SQL Server via `@@ROWCOUNT`, PostgreSQL via a `RETURNING`-counting CTE. `matResult.ID`
|
|
224
|
+
* is a trusted entity PK (UUID), interpolated the same way the refresher interpolates its other metadata. */
|
|
225
|
+
private execGuardedMaterializedResultUpdate;
|
|
226
|
+
/**
|
|
227
|
+
* Phase 3/4: attempt an incremental in-place refresh of a keyed aggregation, recomputing only the
|
|
228
|
+
* groups whose source rows changed since the watermark. Two strategies share ALL eligibility/guard
|
|
229
|
+
* logic and differ only in how the recomputed groups are applied:
|
|
230
|
+
* - `DirtyGroupRecompute` (Phase 3) — DELETE the dirty groups then INSERT their fresh values;
|
|
231
|
+
* - `Incremental` (Phase 4) — UPSERT (MERGE / INSERT…ON CONFLICT) the fresh values onto the
|
|
232
|
+
* surrogate key, updating a surviving group's row in place (no churn). CodeGen assigns this to
|
|
233
|
+
* keyed single-source ADDITIVE aggregations.
|
|
234
|
+
* Returns `{handled:true}` when it ran; `{handled:false}` (caller then full-rebuilds) on any
|
|
235
|
+
* ineligibility OR a tripped guard. Guards (conservative — §10 refuse-under-uncertainty):
|
|
236
|
+
* - opt-in strategy for a keyed Query aggregation with a surrogate;
|
|
237
|
+
* - a watermark baseline must exist (first run full-rebuilds to establish it);
|
|
238
|
+
* - the source must be a SINGLE table exposing `__mj_UpdatedAt` and all key columns;
|
|
239
|
+
* - the current source row count must not be LOWER than the last (a net decrease = deletes → full
|
|
240
|
+
* rebuild self-heals).
|
|
241
|
+
* On success it advances the watermark + source-count on `matResult` (persisted by the caller's Save).
|
|
242
|
+
*/
|
|
243
|
+
private tryRefreshIncremental;
|
|
244
|
+
/**
|
|
245
|
+
* Phase 3: compute a Query materialization's source fingerprint (watermark = MAX(__mj_UpdatedAt) + source
|
|
246
|
+
* row count) so a subsequent DirtyGroupRecompute pass has a baseline. Returns null — meaning no baseline,
|
|
247
|
+
* so the materialization keeps full-rebuilding (correct by construction) — unless the source is a single
|
|
248
|
+
* table exposing `__mj_UpdatedAt`. PURE w.r.t. matResult: it does NOT mutate the entity; the caller applies
|
|
249
|
+
* the returned fingerprint ONLY on the success path, so a failed rebuild never advances the persisted
|
|
250
|
+
* watermark past data it didn't actually materialize.
|
|
251
|
+
*/
|
|
252
|
+
private computeSourceFingerprint;
|
|
253
|
+
/** Extracts the single source table of an aggregation SELECT, or null if it isn't exactly one table. */
|
|
254
|
+
private resolveSingleSourceTable;
|
|
255
|
+
/** Column names of a table (ordinal order) via INFORMATION_SCHEMA (identical query on both engines). */
|
|
256
|
+
private getTableColumns;
|
|
257
|
+
/**
|
|
258
|
+
* True only if (schema, table) is a BASE TABLE (not a view) per INFORMATION_SCHEMA.TABLES — used to gate
|
|
259
|
+
* incremental eligibility, since a watermark is only meaningful on a real table whose `__mj_UpdatedAt` tracks
|
|
260
|
+
* its own row changes. `TABLE_TYPE = 'BASE TABLE'` is standard on both SQL Server and PostgreSQL. Returns
|
|
261
|
+
* false when the object isn't found (fail-safe → no incremental).
|
|
262
|
+
*/
|
|
263
|
+
private sourceIsBaseTable;
|
|
264
|
+
/**
|
|
265
|
+
* Probes the source fingerprint used for incremental refresh: MAX(__mj_UpdatedAt) + COUNT(*).
|
|
266
|
+
*
|
|
267
|
+
* PRECISION SAFETY: the source column is datetimeoffset (100ns) but JS Date is millisecond-precision, so
|
|
268
|
+
* `new Date(w)` TRUNCATES the sub-millisecond part — and per the ECMAScript spec Date always truncates
|
|
269
|
+
* toward the past, never rounds up. That direction is the safe one: the strict incremental filter
|
|
270
|
+
* `__mj_UpdatedAt > watermark` then treats the exact boundary row (the row whose timestamp WAS the max) as
|
|
271
|
+
* still `>` the truncated watermark, so it is harmlessly RE-processed next pass (recompute is idempotent)
|
|
272
|
+
* rather than skipped. A round-UP would be the dangerous case (permanently excluding that row) — which
|
|
273
|
+
* cannot happen with Date truncation.
|
|
274
|
+
*/
|
|
275
|
+
private probeSourceFingerprint;
|
|
276
|
+
/** Counts the rows currently in the materialized wrapper view. */
|
|
277
|
+
private countMaterialized;
|
|
278
|
+
/** A SQL datetime literal (ISO-8601 UTC) parsed by both SQL Server and PostgreSQL. */
|
|
279
|
+
static sqlDateTimeLiteral(date: Date): string;
|
|
280
|
+
/**
|
|
281
|
+
* A globally-unique, length-safe shadow-table name for one refresh run. Deliberately NOT derived from the
|
|
282
|
+
* materialized table name: two refreshes of the SAME materialization (a manual "refresh now" racing the
|
|
283
|
+
* scheduled sweep, or overlapping sweeps under `ConcurrencyMode=Concurrent`) must not share a shadow, or one
|
|
284
|
+
* run's `DROP TABLE …__shadow` would yank the table the other is mid-build. A fixed short prefix + a random
|
|
285
|
+
* token keeps it well under both engines' identifier limits (PG 63 / SQL Server 128) regardless of how long
|
|
286
|
+
* the canonical table name is. The shadow is renamed INTO the canonical name on success (so it leaves no
|
|
287
|
+
* residue), and dropped by RefreshOne's failure cleanup on a caught error; only a hard process crash between
|
|
288
|
+
* shadow creation and swap can leak one — a harmless orphan table with no dependents.
|
|
289
|
+
*/
|
|
290
|
+
static makeShadowTableName(): string;
|
|
291
|
+
/**
|
|
292
|
+
* Applies the incremental-watermark safety overlap: returns `rawMax - WATERMARK_SAFETY_OVERLAP_MS` (null
|
|
293
|
+
* passes through). Persisting the reduced value makes the next incremental pass RE-scan the last `overlap`
|
|
294
|
+
* window, so a source row whose transaction commits after the fingerprint probe — but whose `__mj_UpdatedAt`
|
|
295
|
+
* predates the probed MAX — is re-processed (idempotent MERGE) instead of being skipped forever. Pure and
|
|
296
|
+
* unit-testable; extracted from probeSourceFingerprint so the skew-safety math is verifiable in isolation.
|
|
297
|
+
*/
|
|
298
|
+
static applyWatermarkSafetyOverlap(rawMax: Date | null): Date | null;
|
|
299
|
+
/**
|
|
300
|
+
* True if an entity is read-RLS-protected — any of its role permissions carries a non-empty `ReadRLSFilterID`.
|
|
301
|
+
* Matches CodeGenLib's `entityHasRowLevelSecurity` (and MJ's `GetUserRowLevelSecurityWhereClause`, which
|
|
302
|
+
* sources the read filter solely from `EntityPermission.ReadRLSFilterID`). Used by the runtime leak gate to
|
|
303
|
+
* refuse refreshing a local mirror of an EXTERNAL RLS-protected entity — a mirror can't reproduce remote RLS.
|
|
304
|
+
*/
|
|
305
|
+
static entityHasReadRLS(entity: EntityInfo): boolean;
|
|
306
|
+
/**
|
|
307
|
+
* The composite row-restriction test the Leak-1 gate uses: role RLS **or** an API-key row filter.
|
|
308
|
+
*
|
|
309
|
+
* {@link entityHasReadRLS} covers only the ROLE layer. `EntityInfo`'s equivalent role-only accessor is
|
|
310
|
+
* deprecated precisely because it omits API-key row filters, so a gate built on it alone judges an
|
|
311
|
+
* entity fenced only by a key filter to be unrestricted. CodeGen's mint and drift gates compose both
|
|
312
|
+
* layers; this is the runtime half, kept deliberately symmetric with them.
|
|
313
|
+
*
|
|
314
|
+
* @param apiKeyRowFilterTargets lowercased entity names carrying an API-key row filter, or `'unknown'`
|
|
315
|
+
* when that layer could not be enumerated — in which case every entity is treated as restricted,
|
|
316
|
+
* because refusing to refresh is recoverable and mirroring restricted rows is not.
|
|
317
|
+
*/
|
|
318
|
+
static entityHasRowLevelRestriction(entity: EntityInfo, apiKeyRowFilterTargets: ReadonlySet<string> | 'unknown'): boolean;
|
|
319
|
+
/**
|
|
320
|
+
* Enumerates the entities carrying an API-key row filter, cached for this refresher's lifetime (the gate
|
|
321
|
+
* runs per materialization; the fence changes far more slowly than a sweep).
|
|
322
|
+
*
|
|
323
|
+
* Mirrors CodeGen's enumeration contract exactly, because it shares the rule rather than restating it:
|
|
324
|
+
* entity NAMES normalized by {@link ResolveSingleEntityResourceTarget}, taken from the `ResourcePattern`
|
|
325
|
+
* of scope rows that carry a `RowFilterID`. A pattern that function cannot resolve to one exact entity
|
|
326
|
+
* collapses the WHOLE set to `'unknown'`, since a rule we cannot map may well name the entity we are
|
|
327
|
+
* about to mirror. Any read failure does the same.
|
|
328
|
+
*/
|
|
329
|
+
private loadAPIKeyRowFilterTargets;
|
|
330
|
+
/**
|
|
331
|
+
* Resolves the source SELECT a refresh rebuilds from: the source entity's base view (base-view case)
|
|
332
|
+
* or the stored Query's SQL (query case). Returns null when the source can't be resolved.
|
|
333
|
+
*/
|
|
334
|
+
private resolveSourceSelect;
|
|
335
|
+
/**
|
|
336
|
+
* Remove a TOP-LEVEL ORDER BY from a source SELECT so it can be wrapped in a derived table for the rebuild.
|
|
337
|
+
* See resolveSourceSelect for why (SQL Server error 1033) and why it's safe (a snapshot is unordered).
|
|
338
|
+
*
|
|
339
|
+
* Two guards keep this from corrupting results:
|
|
340
|
+
* - **PostgreSQL is a no-op** — PG permits ORDER BY inside a derived table, so there's nothing to fix and
|
|
341
|
+
* we skip the parser round-trip entirely.
|
|
342
|
+
* - **A query with a row-LIMITING clause (SQL Server TOP or OFFSET/FETCH) is left UNCHANGED** — there the
|
|
343
|
+
* ORDER BY is both (a) LEGAL in a derived table and (b) SEMANTICALLY REQUIRED: it decides WHICH rows
|
|
344
|
+
* TOP/FETCH keep, so stripping it would materialize an arbitrary subset (a silent wrong-data bug). Only a
|
|
345
|
+
* BARE top-level ORDER BY (pure presentation sort, no limiting) is both illegal-in-derived-table and safe
|
|
346
|
+
* to drop.
|
|
347
|
+
*
|
|
348
|
+
* Uses the SQL parser; on any parse/shape surprise, or no top-level ORDER BY, returns the SQL unchanged
|
|
349
|
+
* (an ORDER BY nested inside a subquery is legal and left intact).
|
|
350
|
+
*/
|
|
351
|
+
static stripTopLevelOrderBy(sql: string, isPostgres: boolean): string;
|
|
352
|
+
/**
|
|
353
|
+
* Phase 1.5 (EDS composition): if this materialization is backed by an EXTERNAL entity base view
|
|
354
|
+
* (the source entity carries an `ExternalDataSourceID`), returns that entity — the signal to rebuild
|
|
355
|
+
* by fetching remote rows through the EDS driver rather than by local SQL. Returns null otherwise
|
|
356
|
+
* (local sources, and — for now — external *queries*, which fall through to the local path).
|
|
357
|
+
*/
|
|
358
|
+
private resolveExternalEntity;
|
|
359
|
+
/**
|
|
360
|
+
* Phase 3: parse the materialization's `KeyColumns` metadata (JSON array of `{name, type}`) into the
|
|
361
|
+
* hash-key column list, or undefined when it isn't keyed. A null/empty/malformed value yields undefined
|
|
362
|
+
* — the caller then uses the synthetic IDENTITY/ROW_NUMBER surrogate (Phase 1/2 behavior).
|
|
363
|
+
*/
|
|
364
|
+
static parseKeyColumns(raw: string | null | undefined): {
|
|
365
|
+
name: string;
|
|
366
|
+
type: string;
|
|
367
|
+
}[] | undefined;
|
|
368
|
+
/**
|
|
369
|
+
* Phase 1.5: rebuild a materialized result from an EXTERNAL entity — "mirror external → join locally".
|
|
370
|
+
* Remote rows can't be read via local SQL, so we fetch them through the EDS read router (read-only)
|
|
371
|
+
* and persist into the MJ-managed shadow table (CREATE + batched INSERT), then reuse the Phase-1
|
|
372
|
+
* atomic wrapper-view swap. Once persisted the data is an ordinary local MJ table, joinable with
|
|
373
|
+
* internal entities.
|
|
374
|
+
*
|
|
375
|
+
* RLS is NOT downgraded here (§6.1): base-view materialization REUSES the source entity (no new entity,
|
|
376
|
+
* no changed permissions), so its `ReadRLSFilterID` is enforced at read time by the standard read
|
|
377
|
+
* pipeline against the materialized wrapper view exactly as it would be against the live base view
|
|
378
|
+
* (`DataSource:'Materialized'` only swaps the FROM). The physical mirror holding all rows is correct —
|
|
379
|
+
* RLS filters at READ, like any base table. (This is why only the QUERY case — a new, differently-shaped
|
|
380
|
+
* entity that loses source RLS — carries a mint-time refusal gate, not the base-view case.)
|
|
381
|
+
*/
|
|
382
|
+
private rebuildFromExternalEntity;
|
|
383
|
+
/**
|
|
384
|
+
* Runs a {@link buildExternalRebuildPlan}: DDL, then the parameterized insert batches (value binding —
|
|
385
|
+
* not literal inlining), then the atomic swap. Shared by the external-entity and external-query paths.
|
|
386
|
+
*/
|
|
387
|
+
private static executeExternalRebuildPlan;
|
|
388
|
+
/**
|
|
389
|
+
* Loads the materialization's source stored Query ONCE (Query source type only) and classifies it:
|
|
390
|
+
* returns the loaded `query` plus `externalSql` — the SQL to run remotely when the query is EXTERNAL
|
|
391
|
+
* (carries an ExternalDataSourceID: BroadSQL for RowFilterBroad, else the static query SQL), or null
|
|
392
|
+
* when it's a LOCAL query. The caller reuses this same loaded `query` to build the local source SELECT
|
|
393
|
+
* (no second Load). Returns null when the source isn't a stored Query.
|
|
394
|
+
*/
|
|
395
|
+
/**
|
|
396
|
+
* Resolve a materialization's source Query ID via the `MJ: Materialized Result Queries` join table.
|
|
397
|
+
* The MR<->Query link lives in that join table (there is no MaterializedResult.SourceQueryID column —
|
|
398
|
+
* the direct FK formed a circular dependency). Returns null when the materialization has no linked query.
|
|
399
|
+
*/
|
|
400
|
+
private resolveSourceQueryId;
|
|
401
|
+
private resolveSourceQuery;
|
|
402
|
+
/**
|
|
403
|
+
* Phase 1.5: rebuild a materialized result from an EXTERNAL stored query. Runs the (broad, for
|
|
404
|
+
* RowFilterBroad) query through the EDS native-query path, then persists the returned rows into the
|
|
405
|
+
* MJ-managed shadow. Query results have no natural PK, so — mirroring the local query case — a synthetic
|
|
406
|
+
* surrogate (MATERIALIZATION_SURROGATE_COLUMN) is prepended, populated by 1-based row index; column
|
|
407
|
+
* types are inferred from the returned values. Row-filter re-application at read is the caller's
|
|
408
|
+
* ExtraFilter (the Phase-2 convention), same as local RowFilterBroad materializations.
|
|
409
|
+
*/
|
|
410
|
+
private rebuildFromExternalQuery;
|
|
411
|
+
/**
|
|
412
|
+
* Infer a column's SQL type from its fetched values (external-query materialization, where no field
|
|
413
|
+
* metadata is available). All-null → nvarchar(max)/text; ALL-numbers → int/integer (bigint when any
|
|
414
|
+
* value exceeds signed-32-bit) else float/double precision; ALL-booleans → bit/boolean; ALL Date OBJECTS
|
|
415
|
+
* → datetime2/timestamptz; ANYTHING ELSE, including a column whose values are HETEROGENEOUS across rows or
|
|
416
|
+
* arrive as date STRINGS → nvarchar(max)/text.
|
|
417
|
+
*
|
|
418
|
+
* The type is decided from EVERY present value, not just the first: a loosely-typed source (REST/GraphQL)
|
|
419
|
+
* can return a field that is a number in one row and a string in another; typing the column from row 1
|
|
420
|
+
* would make later rows fail to bind. Falling back to text (which accepts any value) is the safe answer.
|
|
421
|
+
*
|
|
422
|
+
* Date-like STRINGS (ISO-8601 over JSON transport) are deliberately kept as text, NOT coerced to a
|
|
423
|
+
* temporal column: coerceExternalParamValue binds the raw string and relies on implicit conversion, which
|
|
424
|
+
* can reject offset-bearing / edge ISO forms and fail the entire rebuild. Text loses nothing that matters
|
|
425
|
+
* here — fixed-format ISO-8601 strings sort and range-compare CHRONOLOGICALLY under lexicographic text
|
|
426
|
+
* ordering, so ORDER BY / `> 'YYYY-MM-DD…'` filters stay correct. Only genuine Date objects, whose bind is
|
|
427
|
+
* well-defined, are typed as datetime2/timestamptz.
|
|
428
|
+
*/
|
|
429
|
+
static inferSqlType(values: unknown[], isPostgres: boolean): string;
|
|
430
|
+
/**
|
|
431
|
+
* External-source full rebuild PLAN (Phase 1.5), cross-engine and PARAMETERIZED. Pure (no IO) →
|
|
432
|
+
* fully unit-testable (asserts on the emitted SQL + the params arrays). Three parts, run in order:
|
|
433
|
+
*
|
|
434
|
+
* - `preStatements` — DROP + CREATE the shadow table (pure DDL, no params).
|
|
435
|
+
* - `insertBatches` — batched multi-row INSERTs as `{sql, params}`. NON-NULL values are bound as
|
|
436
|
+
* positional parameters (`@pN` on SQL Server, `$N` on PostgreSQL) instead of inlined as literals;
|
|
437
|
+
* NULLs are emitted as the literal `NULL` (no bind param — sidesteps driver null-typing quirks and
|
|
438
|
+
* carries no injection risk). This keeps the SQL TEXT small and constant regardless of row width or
|
|
439
|
+
* value size, so a large external mirror no longer builds enormous statements that pressure the Node
|
|
440
|
+
* heap or blow the database's parser/packet limits (the prior inline-VALUES limitation). Batches are
|
|
441
|
+
* sized by the engine's bind-parameter ceiling (SQL Server 2100 / PostgreSQL 65535, with headroom),
|
|
442
|
+
* capped at 1000 rows/statement.
|
|
443
|
+
* - `postStatements` — the atomic wrapper-view swap: transactional on SQL Server (the view only ever
|
|
444
|
+
* points at the canonical name; the transaction's Sch-M lock keeps readers on the old snapshot until
|
|
445
|
+
* commit — no "Invalid object name …__shadow" window; CREATE VIEW runs via EXEC() as its own batch),
|
|
446
|
+
* CASCADE-repoint sequence on PostgreSQL.
|
|
447
|
+
*
|
|
448
|
+
* NOTE: the source rows are already fully materialized in memory by the EDS read (RunViewExternal /
|
|
449
|
+
* RunQueryExternal return the complete result set), so this fixes the SQL-text/packet half of the scale
|
|
450
|
+
* problem; true end-to-end streaming would require a streaming read API on the EDS router (future work).
|
|
451
|
+
*/
|
|
452
|
+
static buildExternalRebuildPlan(opts: {
|
|
453
|
+
schema: string;
|
|
454
|
+
tableName: string;
|
|
455
|
+
viewName: string;
|
|
456
|
+
columns: {
|
|
457
|
+
name: string;
|
|
458
|
+
sqlType: string;
|
|
459
|
+
}[];
|
|
460
|
+
rows: Record<string, unknown>[];
|
|
461
|
+
isPostgres: boolean;
|
|
462
|
+
/** Query case: the synthetic surrogate column to restore a UNIQUE index on post-swap (the minted
|
|
463
|
+
* entity's PK). Omit for the base-view case (the source PK column carries its own identity). */
|
|
464
|
+
surrogateColumn?: string;
|
|
465
|
+
/** Run-unique shadow table name (see {@link makeShadowTableName}) so two concurrent refreshes of the
|
|
466
|
+
* same materialization never share a shadow. Defaults to the legacy fixed name when omitted. */
|
|
467
|
+
shadowName?: string;
|
|
468
|
+
}): {
|
|
469
|
+
preStatements: string[];
|
|
470
|
+
insertBatches: {
|
|
471
|
+
sql: string;
|
|
472
|
+
params: unknown[];
|
|
473
|
+
}[];
|
|
474
|
+
postStatements: string[];
|
|
475
|
+
};
|
|
476
|
+
/**
|
|
477
|
+
* Coerce a JS value fetched from an external source into a driver-bindable parameter value:
|
|
478
|
+
* null/undefined → null (the caller emits a literal `NULL` for these); non-finite numbers → null;
|
|
479
|
+
* plain objects → JSON text (matches the `inferSqlType` text mapping for object columns); Date and
|
|
480
|
+
* primitives (boolean/number/string) pass through — the driver binds them to the shadow column type.
|
|
481
|
+
*/
|
|
482
|
+
static coerceExternalParamValue(value: unknown): unknown;
|
|
483
|
+
/** Map a SQL-Server-style `SQLFullType` (e.g. `nvarchar(255)`, `int`, `bit`) to a PostgreSQL column type. */
|
|
484
|
+
static mapSqlTypeToPostgres(sqlFullType: string): string;
|
|
485
|
+
/**
|
|
486
|
+
* Quote a SQL identifier for the engine, ESCAPING the closing delimiter so a column name that contains
|
|
487
|
+
* it can't break out of the quotes: `]`→`]]` (SQL Server), `"`→`""` (PostgreSQL). Column names in the
|
|
488
|
+
* keyed/incremental builders are CodeGen-derived (entity field / KeyColumns names), so this is a
|
|
489
|
+
* consistency + robustness guard (matching buildExternalRebuildPlan's escId), not a live-injection fix.
|
|
490
|
+
*/
|
|
491
|
+
static quoteIdent(name: string, isPostgres: boolean): string;
|
|
492
|
+
/**
|
|
493
|
+
* Phase 3: SQL expression producing the CANONICAL TEXT of one key column for the combined-key
|
|
494
|
+
* surrogate hash (§17.1). Deterministic within an engine; a NULL is replaced by a control-char-wrapped
|
|
495
|
+
* sentinel (CHAR(30)) so it can't collide with a literal value. `type` is the column's SQL-Server-style
|
|
496
|
+
* type (EntityFieldInfo.SQLFullType); the base type drives the canonical cast.
|
|
497
|
+
*/
|
|
498
|
+
static canonicalKeyColumnSql(name: string, type: string, isPostgres: boolean): string;
|
|
499
|
+
/**
|
|
500
|
+
* Phase 3: SQL expression computing the combined-key surrogate — `SHA2_256` (lowercase hex) over the
|
|
501
|
+
* canonical key columns in declared key order (§17.1). Deterministic WITHIN an engine (the
|
|
502
|
+
* incremental-MERGE / dirty-group match key); cross-engine identity is best-effort.
|
|
503
|
+
*
|
|
504
|
+
* COLLISION SAFETY: each canonical part is hashed to a FIXED-WIDTH 64-char hex string FIRST, and the
|
|
505
|
+
* per-part hashes are what get delimited + hashed. A naive `part1 + CHAR(31) + part2` collides when a
|
|
506
|
+
* key value itself contains the CHAR(31) delimiter (or the CHAR(30) NULL sentinel) — e.g. ('x\x1f','y')
|
|
507
|
+
* and ('x','\x1fy') both flatten to `x\x1f\x1fy`. Hashing each part first makes every part pure hex
|
|
508
|
+
* (0-9a-f), which can NEVER contain a control char, so the delimiter is unambiguous and distinct tuples
|
|
509
|
+
* can no longer canonicalize to the same string. Each canonical part is NULL-free (COALESCE'd), so the
|
|
510
|
+
* inner hash inputs are never NULL. (This feature is unreleased, so no existing surrogates need
|
|
511
|
+
* migrating; a full rebuild regenerates them under the new scheme.)
|
|
512
|
+
* NOTE: the PostgreSQL `digest()` used here requires the `pgcrypto` extension. MJ's PostgreSQL baseline
|
|
513
|
+
* already runs `CREATE EXTENSION IF NOT EXISTS "pgcrypto"`, so keyed PG materializations get it for free;
|
|
514
|
+
* a deployment that dropped that baseline step would see refreshes fail with a clear `function digest(...)
|
|
515
|
+
* does not exist` — provision pgcrypto to resolve.
|
|
516
|
+
*/
|
|
517
|
+
static buildHashKeyExpression(keyColumns: {
|
|
518
|
+
name: string;
|
|
519
|
+
type: string;
|
|
520
|
+
}[], isPostgres: boolean): string;
|
|
521
|
+
/**
|
|
522
|
+
* Phase 3 (DirtyGroupRecompute): a NULL-safe equality predicate matching the key columns of two
|
|
523
|
+
* aliases (`(a.[k] = b.[k] OR (a.[k] IS NULL AND b.[k] IS NULL)) AND ...`). Two NULL keys are treated
|
|
524
|
+
* as equal (a materialized aggregation can legitimately have a NULL grouping value — it's one group).
|
|
525
|
+
* Portable across SQL Server and PostgreSQL (the `OR ... IS NULL` form works on both; we avoid
|
|
526
|
+
* `IS NOT DISTINCT FROM`, which SQL Server lacks pre-2022). Pure/unit-testable.
|
|
527
|
+
*/
|
|
528
|
+
static buildKeyMatchPredicate(aliasA: string, aliasB: string, keyColumns: {
|
|
529
|
+
name: string;
|
|
530
|
+
}[], isPostgres: boolean): string;
|
|
531
|
+
/**
|
|
532
|
+
* Phase 3 (DirtyGroupRecompute): the ordered statements that incrementally refresh a keyed aggregation
|
|
533
|
+
* IN PLACE (no shadow swap) by recomputing only the groups whose SOURCE rows changed since `watermarkSql`.
|
|
534
|
+
* Pure (no IO), so the sequence is unit-testable. Engine-agnostic core; {@link buildDirtyGroupRecomputeStatementsSQLServer}
|
|
535
|
+
* / {@link buildDirtyGroupRecomputeStatementsPostgreSQL} supply the per-engine quoting + DELETE syntax.
|
|
536
|
+
*
|
|
537
|
+
* Semantics (correct within the documented delete caveat — see RefreshOne):
|
|
538
|
+
* 1. DELETE every materialized row whose group has ANY source row updated since the watermark
|
|
539
|
+
* (removes stale values, AND removes a group that shrank/emptied among the changed groups);
|
|
540
|
+
* 2. INSERT the freshly-computed values for exactly those dirty groups (from the aggregation SELECT,
|
|
541
|
+
* filtered by the same "group has a changed source row" predicate), stamping the same hash surrogate
|
|
542
|
+
* the full rebuild uses so the key stays stable.
|
|
543
|
+
* A group whose rows were ALL deleted without any surviving-row update is NOT seen here (the deleted
|
|
544
|
+
* rows are gone); that case is caught by the source-count-drop → full-rebuild guard in RefreshOne.
|
|
545
|
+
*/
|
|
546
|
+
private static buildDirtyGroupRecomputeCore;
|
|
547
|
+
/** SQL Server dirty-group recompute (see {@link buildDirtyGroupRecomputeCore}). */
|
|
548
|
+
static buildDirtyGroupRecomputeStatementsSQLServer(opts: {
|
|
549
|
+
schema: string;
|
|
550
|
+
tableName: string;
|
|
551
|
+
sourceSchema: string;
|
|
552
|
+
sourceTable: string;
|
|
553
|
+
keyColumns: {
|
|
554
|
+
name: string;
|
|
555
|
+
type: string;
|
|
556
|
+
}[];
|
|
557
|
+
aggregationSelect: string;
|
|
558
|
+
surrogateColumn: string;
|
|
559
|
+
dataColumns: string[];
|
|
560
|
+
updatedAtColumn: string;
|
|
561
|
+
watermarkSql: string;
|
|
562
|
+
}): string[];
|
|
563
|
+
/** PostgreSQL dirty-group recompute (see {@link buildDirtyGroupRecomputeCore}). */
|
|
564
|
+
static buildDirtyGroupRecomputeStatementsPostgreSQL(opts: {
|
|
565
|
+
schema: string;
|
|
566
|
+
tableName: string;
|
|
567
|
+
sourceSchema: string;
|
|
568
|
+
sourceTable: string;
|
|
569
|
+
keyColumns: {
|
|
570
|
+
name: string;
|
|
571
|
+
type: string;
|
|
572
|
+
}[];
|
|
573
|
+
aggregationSelect: string;
|
|
574
|
+
surrogateColumn: string;
|
|
575
|
+
dataColumns: string[];
|
|
576
|
+
updatedAtColumn: string;
|
|
577
|
+
watermarkSql: string;
|
|
578
|
+
}): string[];
|
|
579
|
+
/**
|
|
580
|
+
* Phase 4 (RefreshStrategy = 'Incremental'): incrementally refresh a keyed ADDITIVE aggregation by
|
|
581
|
+
* recomputing only the changed groups and UPSERTING them onto the surrogate key — an in-place MERGE
|
|
582
|
+
* (SQL Server) / INSERT…ON CONFLICT (PostgreSQL) rather than the DirtyGroupRecompute delete-then-insert.
|
|
583
|
+
* The recomputed source is identical (the aggregation restricted to groups with a source row changed
|
|
584
|
+
* since the watermark); the difference is that a surviving group's row is UPDATED in place — no churn,
|
|
585
|
+
* no transient absence, one atomic statement. Correct for insert/update; a net source-count drop
|
|
586
|
+
* (deletes) still falls back to full rebuild via the RefreshOne guard. Requires the surrogate to be
|
|
587
|
+
* unique (it is the materialized table's PK). Pure (no IO) / unit-testable.
|
|
588
|
+
*/
|
|
589
|
+
static buildIncrementalMergeStatementsSQLServer(opts: {
|
|
590
|
+
schema: string;
|
|
591
|
+
tableName: string;
|
|
592
|
+
sourceSchema: string;
|
|
593
|
+
sourceTable: string;
|
|
594
|
+
keyColumns: {
|
|
595
|
+
name: string;
|
|
596
|
+
type: string;
|
|
597
|
+
}[];
|
|
598
|
+
aggregationSelect: string;
|
|
599
|
+
surrogateColumn: string;
|
|
600
|
+
dataColumns: string[];
|
|
601
|
+
updatedAtColumn: string;
|
|
602
|
+
watermarkSql: string;
|
|
603
|
+
}): string[];
|
|
604
|
+
/** PostgreSQL incremental upsert — the INSERT…ON CONFLICT counterpart of the SQL Server MERGE above. */
|
|
605
|
+
static buildIncrementalMergeStatementsPostgreSQL(opts: {
|
|
606
|
+
schema: string;
|
|
607
|
+
tableName: string;
|
|
608
|
+
sourceSchema: string;
|
|
609
|
+
sourceTable: string;
|
|
610
|
+
keyColumns: {
|
|
611
|
+
name: string;
|
|
612
|
+
type: string;
|
|
613
|
+
}[];
|
|
614
|
+
aggregationSelect: string;
|
|
615
|
+
surrogateColumn: string;
|
|
616
|
+
dataColumns: string[];
|
|
617
|
+
updatedAtColumn: string;
|
|
618
|
+
watermarkSql: string;
|
|
619
|
+
}): string[];
|
|
620
|
+
}
|
|
621
|
+
//# sourceMappingURL=MaterializationRefresher.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"MaterializationRefresher.d.ts","sourceRoot":"","sources":["../src/MaterializationRefresher.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,QAAQ,EAAuB,UAAU,EAAyC,MAAM,sBAAsB,CAAC;AAC3I,OAAO,EAAE,0BAA0B,EAAiB,MAAM,+BAA+B,CAAC;AAK1F;;;;;GAKG;AACH,eAAO,MAAM,gCAAgC,2BAA2B,CAAC;AAEzE;;;;;;GAMG;AACH,eAAO,MAAM,0CAA0C,KAAK,CAAC;AAE7D;;;;;;;;GAQG;AACH,eAAO,MAAM,2BAA2B,QAAS,CAAC;AAElD;;;;GAIG;AACH,MAAM,WAAW,YAAY;IACzB;;;;OAIG;IACH,UAAU,CAAC,CAAC,GAAG,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,OAAO,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACzE,iHAAiH;IACjH,WAAW,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,0DAA0D;AAC1D,MAAM,WAAW,4BAA4B;IACzC,OAAO,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,YAAY,CAAC,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;;;;;GAUG;AACH,qBAAa,wBAAwB;IACjC;;;;;OAKG;WACW,sBAAsB,CAAC,yBAAyB,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO;IAInG;;;;OAIG;WACW,6BAA6B,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,EAAE,cAAc,EAAE,OAAO,GAAG,MAAM;IAIhH,oGAAoG;IACpG,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,mBAAmB,CAA8B;IAEzE;0GACsG;IACtG,OAAO,CAAC,uBAAuB,CAAgD;IAE/E;;;;;;;;;;OAUG;IACH,OAAO,CAAC,MAAM,CAAC,qBAAqB;IAQpC;;;;;OAKG;WACW,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO;IAItD;;;;;;;;;;;;;OAaG;WACW,uBAAuB,CAAC,QAAQ,EAAE,iBAAiB,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI;IAQjJ;;;;;;;OAOG;WACW,mCAAmC,CAAC,IAAI,EAAE;QACpD,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAClD;yGACiG;QACjG,UAAU,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,MAAM,EAAE;IAqDZ;;;;;;;;;;;;;;;OAeG;WACW,oCAAoC,CAAC,IAAI,EAAE;QACrD,MAAM,EAAE,MAAM,CAAC;QACf,SAAS,EAAE,MAAM,CAAC;QAClB,QAAQ,EAAE,MAAM,CAAC;QACjB,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAClD;yGACiG;QACjG,UAAU,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG,MAAM,EAAE;IAgDZ;;;;OAIG;WACW,SAAS,CAAC,CAAC,SAAS;QAAE,aAAa,CAAC,EAAE,IAAI,GAAG,IAAI,CAAA;KAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,IAAI,GAAG,CAAC,EAAE;IAI7F;;;;OAIG;IACU,UAAU,CACnB,SAAS,EAAE,0BAA0B,EACrC,WAAW,EAAE,QAAQ,EACrB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,CAAC,EAAE;QAAE,aAAa,CAAC,EAAE,IAAI,GAAG,IAAI,CAAA;KAAE,GAC1C,OAAO,CAAC,4BAA4B,CAAC;IA0KxC;;;;OAIG;IACH;;;;OAIG;YACW,wBAAwB;YAUxB,yBAAyB;IAsBvC;;;;;;;;;;;;;;;;OAgBG;YACW,WAAW;IAoBzB;;sGAEkG;IAClG,OAAO,CAAC,0BAA0B;IAMlC;;;kEAG8D;YAChD,2BAA2B;IAgBzC;;;;kHAI8G;YAChG,mCAAmC;IAWjD;;;;;;;;;;;;;;;;OAgBG;YACW,qBAAqB;IA2FnC;;;;;;;OAOG;YACW,wBAAwB;IActC,wGAAwG;IACxG,OAAO,CAAC,wBAAwB;IAMhC,wGAAwG;YAC1F,eAAe;IAQ7B;;;;;OAKG;YACW,iBAAiB;IAQ/B;;;;;;;;;;OAUG;YACW,sBAAsB;IAapC,kEAAkE;YACpD,iBAAiB;IAQ/B,sFAAsF;WACxE,kBAAkB,CAAC,IAAI,EAAE,IAAI,GAAG,MAAM;IAIpD;;;;;;;;;OASG;WACW,mBAAmB,IAAI,MAAM;IAI3C;;;;;;OAMG;WACW,2BAA2B,CAAC,MAAM,EAAE,IAAI,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI;IAI3E;;;;;OAKG;WACW,gBAAgB,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO;IAI3D;;;;;;;;;;;OAWG;WACW,4BAA4B,CAAC,MAAM,EAAE,UAAU,EAAE,sBAAsB,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,SAAS,GAAG,OAAO;IAMhI;;;;;;;;;OASG;YACW,0BAA0B;IA0CxC;;;OAGG;YACW,mBAAmB;IA4CjC;;;;;;;;;;;;;;;OAeG;WACW,oBAAoB,CAAC,GAAG,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IAwB5E;;;;;OAKG;IACH,OAAO,CAAC,qBAAqB;IAM7B;;;;OAIG;WACW,eAAe,CAAC,GAAG,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,GAAG,SAAS;IAe3G;;;;;;;;;;;;;OAaG;YACW,yBAAyB;IAmCvC;;;OAGG;mBACkB,0BAA0B;IAkB/C;;;;;;OAMG;IACH;;;;OAIG;YACW,oBAAoB;YAepB,kBAAkB;IAehC;;;;;;;OAOG;YACW,wBAAwB;IAiEtC;;;;;;;;;;;;;;;;;OAiBG;WACW,YAAY,CAAC,MAAM,EAAE,OAAO,EAAE,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IAwB1E;;;;;;;;;;;;;;;;;;;;;OAqBG;WACW,wBAAwB,CAAC,IAAI,EAAE;QACzC,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QACpD,OAAO,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,OAAO,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAC;QAChC,UAAU,EAAE,OAAO,CAAC;QACpB;yGACiG;QACjG,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB;yGACiG;QACjG,UAAU,CAAC,EAAE,MAAM,CAAC;KACvB,GAAG;QAAE,aAAa,EAAE,MAAM,EAAE,CAAC;QAAC,aAAa,EAAE;YAAE,GAAG,EAAE,MAAM,CAAC;YAAC,MAAM,EAAE,OAAO,EAAE,CAAA;SAAE,EAAE,CAAC;QAAC,cAAc,EAAE,MAAM,EAAE,CAAA;KAAE;IA4E9G;;;;;OAKG;WACW,wBAAwB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO;IAO/D,6GAA6G;WAC/F,oBAAoB,CAAC,WAAW,EAAE,MAAM,GAAG,MAAM;IAgC/D;;;;;OAKG;WACW,UAAU,CAAC,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IAInE;;;;;OAKG;WACW,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IA4C5F;;;;;;;;;;;;;;;;;OAiBG;WACW,sBAAsB,CAAC,UAAU,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IAe/G;;;;;;OAMG;WACW,sBAAsB,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,UAAU,EAAE;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,EAAE,EAAE,UAAU,EAAE,OAAO,GAAG,MAAM;IAWjI;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,MAAM,CAAC,4BAA4B;IA6B3C,mFAAmF;WACrE,2CAA2C,CAAC,IAAI,EAAE;QAC5D,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAC1C,UAAU,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,iBAAiB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC1E,eAAe,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;KACjD,GAAG,MAAM,EAAE;IAYZ,mFAAmF;WACrE,4CAA4C,CAAC,IAAI,EAAE;QAC7D,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAC1C,UAAU,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,iBAAiB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC1E,eAAe,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;KACjD,GAAG,MAAM,EAAE;IAYZ;;;;;;;;;OASG;WACW,wCAAwC,CAAC,IAAI,EAAE;QACzD,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAC1C,UAAU,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,iBAAiB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC1E,eAAe,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;KACjD,GAAG,MAAM,EAAE;IAwBZ,wGAAwG;WAC1F,yCAAyC,CAAC,IAAI,EAAE;QAC1D,MAAM,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAClC,YAAY,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAC1C,UAAU,EAAE;YAAE,IAAI,EAAE,MAAM,CAAC;YAAC,IAAI,EAAE,MAAM,CAAA;SAAE,EAAE,CAAC;QAC7C,iBAAiB,EAAE,MAAM,CAAC;QAAC,eAAe,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAC;QAC1E,eAAe,EAAE,MAAM,CAAC;QAAC,YAAY,EAAE,MAAM,CAAC;KACjD,GAAG,MAAM,EAAE;CAiBf"}
|