@avytheone/efmesh 0.1.0-beta.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -92
- package/README.md +83 -9
- package/README.ru.md +80 -5
- package/SPEC.md +542 -459
- package/package.json +4 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +505 -85
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +28 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +44 -13
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +10 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +228 -4
- package/src/plan/executor.ts +223 -121
- package/src/plan/explain.ts +114 -0
- package/src/plan/fingerprint.ts +84 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +238 -52
- package/src/plan/run.ts +72 -26
- package/src/plan/schedule.ts +217 -0
- package/src/plan/status.ts +96 -0
- package/src/state/postgres.ts +68 -19
- package/src/state/sqlite.ts +74 -27
- package/src/state/store.ts +88 -45
- package/src/testing/index.ts +27 -26
package/src/core/model.ts
CHANGED
|
@@ -6,57 +6,57 @@ import { ModelDefinitionError } from "./errors.ts"
|
|
|
6
6
|
import type { BoundValue, IdentsValue, RefValue, SqlFragment } from "./sql.ts"
|
|
7
7
|
import { collectRefs, parseSqlText, sql, usesBounds } from "./sql.ts"
|
|
8
8
|
|
|
9
|
-
/**
|
|
9
|
+
/** Materialization kind (SPEC §3.1). */
|
|
10
10
|
export type ModelKind =
|
|
11
11
|
| { readonly _tag: "full" }
|
|
12
12
|
| { readonly _tag: "view" }
|
|
13
13
|
| {
|
|
14
|
-
/**
|
|
14
|
+
/** Inlined into consumers as a subquery, without materialization (SPEC §3.1). */
|
|
15
15
|
readonly _tag: "embedded"
|
|
16
16
|
}
|
|
17
17
|
| {
|
|
18
18
|
readonly _tag: "incrementalByTimeRange"
|
|
19
|
-
/**
|
|
19
|
+
/** Time column that intervals are sliced and reread by. */
|
|
20
20
|
readonly timeColumn: string
|
|
21
|
-
/**
|
|
21
|
+
/** Point in time to backfill from (ISO UTC). */
|
|
22
22
|
readonly start: string
|
|
23
|
-
/**
|
|
23
|
+
/** Interval grain. */
|
|
24
24
|
readonly interval: IntervalUnit
|
|
25
|
-
/**
|
|
25
|
+
/** How many grain intervals a single DELETE+INSERT executes. */
|
|
26
26
|
readonly batchSize: number
|
|
27
|
-
/**
|
|
27
|
+
/** How many of the most recent done intervals to recompute (late-arriving data). */
|
|
28
28
|
readonly lookback: number
|
|
29
29
|
}
|
|
30
30
|
| { readonly _tag: "external"; readonly source: ExternalSource }
|
|
31
31
|
| {
|
|
32
32
|
readonly _tag: "seed"
|
|
33
|
-
/** CSV/JSON
|
|
33
|
+
/** CSV/JSON file with the data; its contents feed into the fingerprint. */
|
|
34
34
|
readonly file: string
|
|
35
35
|
readonly format: "csv" | "json"
|
|
36
36
|
}
|
|
37
37
|
| {
|
|
38
38
|
readonly _tag: "incrementalByUniqueKey"
|
|
39
|
-
/**
|
|
39
|
+
/** Logical upsert key; every apply reruns the query and replaces rows by key. */
|
|
40
40
|
readonly key: ReadonlyArray<string>
|
|
41
41
|
}
|
|
42
42
|
| {
|
|
43
43
|
/**
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
*
|
|
44
|
+
* Slowly changing dimension, type 2 (SPEC §3.1): row version history.
|
|
45
|
+
* Every apply compares the query against the currently open rows: rows
|
|
46
|
+
* that changed or disappeared are closed (validTo = now), new versions
|
|
47
|
+
* are inserted open (validTo IS NULL).
|
|
48
48
|
*/
|
|
49
49
|
readonly _tag: "scdType2"
|
|
50
50
|
readonly key: ReadonlyArray<string>
|
|
51
|
-
/**
|
|
51
|
+
/** Versioning columns — managed by efmesh: declared in the schema, absent from the query. */
|
|
52
52
|
readonly validFrom: string
|
|
53
53
|
readonly validTo: string
|
|
54
54
|
}
|
|
55
55
|
|
|
56
56
|
/**
|
|
57
|
-
*
|
|
58
|
-
*
|
|
59
|
-
* HTTPS — REST-JSON
|
|
57
|
+
* External source definition (SPEC §9.3): an engine table/ATTACH database,
|
|
58
|
+
* or files by path/URL (`read_parquet`/`read_csv`/`read_json`, including
|
|
59
|
+
* HTTPS — REST-JSON lands here too).
|
|
60
60
|
*/
|
|
61
61
|
export type ExternalSource =
|
|
62
62
|
| { readonly _tag: "table"; readonly table: string }
|
|
@@ -110,7 +110,7 @@ export const external = {
|
|
|
110
110
|
}),
|
|
111
111
|
} as const
|
|
112
112
|
|
|
113
|
-
/**
|
|
113
|
+
/** Model name: `<schema>.<table>`. */
|
|
114
114
|
const MODEL_NAME = /^[a-zA-Z_][a-zA-Z0-9_]*\.[a-zA-Z_][a-zA-Z0-9_]*$/
|
|
115
115
|
|
|
116
116
|
export interface ModelName {
|
|
@@ -123,7 +123,7 @@ export const parseModelName = (raw: string): ModelName => {
|
|
|
123
123
|
if (!MODEL_NAME.test(raw)) {
|
|
124
124
|
throw new ModelDefinitionError({
|
|
125
125
|
model: raw,
|
|
126
|
-
reason: "
|
|
126
|
+
reason: "model name must be of the form <schema>.<table> (latin letters, digits, _)",
|
|
127
127
|
})
|
|
128
128
|
}
|
|
129
129
|
const [schema, table] = raw.split(".") as [string, string]
|
|
@@ -131,10 +131,11 @@ export const parseModelName = (raw: string): ModelName => {
|
|
|
131
131
|
}
|
|
132
132
|
|
|
133
133
|
/**
|
|
134
|
-
*
|
|
135
|
-
* parquet
|
|
136
|
-
* DuckLake
|
|
137
|
-
*
|
|
134
|
+
* Where to put the physical layer (SPEC §3.3): a native engine table,
|
|
135
|
+
* lake parquet files (interval = partition, a view over read_parquet), or
|
|
136
|
+
* a DuckLake catalog (SPEC §14.5: a table-per-fingerprint in an ATTACH
|
|
137
|
+
* catalog — DuckLake's own snapshots and time travel come along for free,
|
|
138
|
+
* but versioning stays ours).
|
|
138
139
|
*/
|
|
139
140
|
export type MaterializationTarget = "table" | "parquet" | "ducklake"
|
|
140
141
|
|
|
@@ -143,20 +144,20 @@ export interface ModelConfig<Fields extends Schema.Struct.Fields> {
|
|
|
143
144
|
readonly kind: ModelKind
|
|
144
145
|
readonly schema: Schema.Struct<Fields>
|
|
145
146
|
readonly description?: string
|
|
146
|
-
/**
|
|
147
|
+
/** Logical primary key; metadata only for now (unique audit — F2). */
|
|
147
148
|
readonly grain?: ReadonlyArray<Extract<keyof Fields, string>>
|
|
148
|
-
/**
|
|
149
|
+
/** Materialization target; defaults to a native engine table. */
|
|
149
150
|
readonly target?: MaterializationTarget
|
|
150
|
-
/**
|
|
151
|
+
/** Quality audits (SPEC §8); excluded from the fingerprint. */
|
|
151
152
|
readonly audits?: ReadonlyArray<Audit>
|
|
152
153
|
/**
|
|
153
|
-
*
|
|
154
|
-
*
|
|
154
|
+
* Export outward (SPEC §9.3): after audits and promotion, the finished
|
|
155
|
+
* result is shipped to an ATTACH database (`attach` — an alias from config).
|
|
155
156
|
*/
|
|
156
157
|
readonly export?: { readonly attach: string; readonly table: string }
|
|
157
158
|
}
|
|
158
159
|
|
|
159
|
-
/**
|
|
160
|
+
/** Rendering context for a model body. The body must be pure: all mutable state flows in from here. */
|
|
160
161
|
export interface ModelCtx {
|
|
161
162
|
readonly sql: typeof sql
|
|
162
163
|
readonly ref: (model: AnyModel) => RefValue
|
|
@@ -165,9 +166,9 @@ export interface ModelCtx {
|
|
|
165
166
|
...names: ReadonlyArray<Extract<keyof Fields, string>>
|
|
166
167
|
) => IdentsValue
|
|
167
168
|
/**
|
|
168
|
-
*
|
|
169
|
-
* incrementalByTimeRange.
|
|
170
|
-
*
|
|
169
|
+
* Bounds of the interval being processed `[start, end)` — only for
|
|
170
|
+
* incrementalByTimeRange. Substituted with literals at execution time;
|
|
171
|
+
* rendered as placeholders in the canonical text (SPEC §3).
|
|
171
172
|
*/
|
|
172
173
|
readonly start: BoundValue
|
|
173
174
|
readonly end: BoundValue
|
|
@@ -182,11 +183,11 @@ export interface Model<Fields extends Schema.Struct.Fields = Schema.Struct.Field
|
|
|
182
183
|
readonly grain: ReadonlyArray<string>
|
|
183
184
|
readonly target: MaterializationTarget
|
|
184
185
|
readonly audits: ReadonlyArray<Audit>
|
|
185
|
-
/**
|
|
186
|
+
/** Body, rendered into a fragment once at definition time. */
|
|
186
187
|
readonly fragment: SqlFragment
|
|
187
|
-
/**
|
|
188
|
+
/** Names of the models the body references via `ctx.ref`. */
|
|
188
189
|
readonly deps: ReadonlySet<string>
|
|
189
|
-
/**
|
|
190
|
+
/** The source models themselves, by name — schemas for fixture validation in testModel. */
|
|
190
191
|
readonly refs: ReadonlyMap<string, AnyModel>
|
|
191
192
|
readonly export?: { readonly attach: string; readonly table: string }
|
|
192
193
|
}
|
|
@@ -197,7 +198,7 @@ export type AnyModel = Model<any>
|
|
|
197
198
|
export const columnNames = (model: AnyModel): ReadonlyArray<string> =>
|
|
198
199
|
Object.keys(model.schema.fields)
|
|
199
200
|
|
|
200
|
-
/**
|
|
201
|
+
/** Shared model-kind config checks — for defineModel and defineSqlModel. */
|
|
201
202
|
const validateKindConfig = <Fields extends Schema.Struct.Fields>(
|
|
202
203
|
name: ModelName,
|
|
203
204
|
config: ModelConfig<Fields>,
|
|
@@ -205,13 +206,13 @@ const validateKindConfig = <Fields extends Schema.Struct.Fields>(
|
|
|
205
206
|
if (config.kind._tag === "external") {
|
|
206
207
|
throw new ModelDefinitionError({
|
|
207
208
|
model: name.full,
|
|
208
|
-
reason: "external
|
|
209
|
+
reason: "an external model has no body — use defineExternal",
|
|
209
210
|
})
|
|
210
211
|
}
|
|
211
212
|
if (config.kind._tag === "seed") {
|
|
212
213
|
throw new ModelDefinitionError({
|
|
213
214
|
model: name.full,
|
|
214
|
-
reason: "seed
|
|
215
|
+
reason: "a seed model has no body — use defineSeed",
|
|
215
216
|
})
|
|
216
217
|
}
|
|
217
218
|
if (config.kind._tag === "incrementalByUniqueKey" || config.kind._tag === "scdType2") {
|
|
@@ -219,12 +220,12 @@ const validateKindConfig = <Fields extends Schema.Struct.Fields>(
|
|
|
219
220
|
if (!(keyColumn in config.schema.fields)) {
|
|
220
221
|
throw new ModelDefinitionError({
|
|
221
222
|
model: name.full,
|
|
222
|
-
reason:
|
|
223
|
+
reason: `key column «${keyColumn}» is not in the model schema`,
|
|
223
224
|
})
|
|
224
225
|
}
|
|
225
226
|
}
|
|
226
227
|
if (config.kind.key.length === 0) {
|
|
227
|
-
throw new ModelDefinitionError({ model: name.full, reason: "key
|
|
228
|
+
throw new ModelDefinitionError({ model: name.full, reason: "key cannot be empty" })
|
|
228
229
|
}
|
|
229
230
|
}
|
|
230
231
|
if (config.kind._tag === "scdType2") {
|
|
@@ -232,27 +233,27 @@ const validateKindConfig = <Fields extends Schema.Struct.Fields>(
|
|
|
232
233
|
if (validFrom === validTo) {
|
|
233
234
|
throw new ModelDefinitionError({
|
|
234
235
|
model: name.full,
|
|
235
|
-
reason: "validFrom
|
|
236
|
+
reason: "validFrom and validTo cannot be the same",
|
|
236
237
|
})
|
|
237
238
|
}
|
|
238
239
|
for (const column of [validFrom, validTo]) {
|
|
239
240
|
if (!(column in config.schema.fields)) {
|
|
240
241
|
throw new ModelDefinitionError({
|
|
241
242
|
model: name.full,
|
|
242
|
-
reason:
|
|
243
|
+
reason: `versioning column «${column}» is not in the model schema — consumers must see it`,
|
|
243
244
|
})
|
|
244
245
|
}
|
|
245
246
|
if (config.kind.key.includes(column)) {
|
|
246
247
|
throw new ModelDefinitionError({
|
|
247
248
|
model: name.full,
|
|
248
|
-
reason:
|
|
249
|
+
reason: `versioning column «${column}» cannot be part of key`,
|
|
249
250
|
})
|
|
250
251
|
}
|
|
251
252
|
}
|
|
252
253
|
if (config.target === "parquet") {
|
|
253
254
|
throw new ModelDefinitionError({
|
|
254
255
|
model: name.full,
|
|
255
|
-
reason: "scdType2
|
|
256
|
+
reason: "scdType2 closes rows in place — a parquet target is not applicable",
|
|
256
257
|
})
|
|
257
258
|
}
|
|
258
259
|
}
|
|
@@ -263,32 +264,32 @@ const validateKindConfig = <Fields extends Schema.Struct.Fields>(
|
|
|
263
264
|
) {
|
|
264
265
|
throw new ModelDefinitionError({
|
|
265
266
|
model: name.full,
|
|
266
|
-
reason: `${config.kind._tag}
|
|
267
|
+
reason: `${config.kind._tag} is not materialized — target «${config.target}» is not applicable to it`,
|
|
267
268
|
})
|
|
268
269
|
}
|
|
269
270
|
if (config.kind._tag === "incrementalByUniqueKey" && config.target === "parquet") {
|
|
270
271
|
throw new ModelDefinitionError({
|
|
271
272
|
model: name.full,
|
|
272
|
-
reason: "upsert
|
|
273
|
+
reason: "key upsert into parquet files is impossible — use target: \"table\"",
|
|
273
274
|
})
|
|
274
275
|
}
|
|
275
276
|
if (config.kind._tag === "incrementalByTimeRange") {
|
|
276
277
|
if (!(config.kind.timeColumn in config.schema.fields)) {
|
|
277
278
|
throw new ModelDefinitionError({
|
|
278
279
|
model: name.full,
|
|
279
|
-
reason: `timeColumn «${config.kind.timeColumn}»
|
|
280
|
+
reason: `timeColumn «${config.kind.timeColumn}» is not in the model schema`,
|
|
280
281
|
})
|
|
281
282
|
}
|
|
282
283
|
if (Number.isNaN(Date.parse(config.kind.start))) {
|
|
283
284
|
throw new ModelDefinitionError({
|
|
284
285
|
model: name.full,
|
|
285
|
-
reason: `start «${config.kind.start}»
|
|
286
|
+
reason: `start «${config.kind.start}» is not an ISO time`,
|
|
286
287
|
})
|
|
287
288
|
}
|
|
288
289
|
}
|
|
289
290
|
}
|
|
290
291
|
|
|
291
|
-
/**
|
|
292
|
+
/** Final assembly of a model from a fragment — shared body invariants. */
|
|
292
293
|
const assembleModel = <Fields extends Schema.Struct.Fields>(
|
|
293
294
|
name: ModelName,
|
|
294
295
|
config: ModelConfig<Fields>,
|
|
@@ -298,12 +299,12 @@ const assembleModel = <Fields extends Schema.Struct.Fields>(
|
|
|
298
299
|
if (usesBounds(fragment) && config.kind._tag !== "incrementalByTimeRange") {
|
|
299
300
|
throw new ModelDefinitionError({
|
|
300
301
|
model: name.full,
|
|
301
|
-
reason: `ctx.start/ctx.end
|
|
302
|
+
reason: `ctx.start/ctx.end are available only in incrementalByTimeRange, model kind is ${config.kind._tag}`,
|
|
302
303
|
})
|
|
303
304
|
}
|
|
304
305
|
const deps = collectRefs(fragment)
|
|
305
306
|
if (deps.has(name.full)) {
|
|
306
|
-
throw new ModelDefinitionError({ model: name.full, reason: "
|
|
307
|
+
throw new ModelDefinitionError({ model: name.full, reason: "model references itself" })
|
|
307
308
|
}
|
|
308
309
|
return {
|
|
309
310
|
_tag: "Model",
|
|
@@ -322,9 +323,9 @@ const assembleModel = <Fields extends Schema.Struct.Fields>(
|
|
|
322
323
|
}
|
|
323
324
|
|
|
324
325
|
/**
|
|
325
|
-
*
|
|
326
|
-
*
|
|
327
|
-
*
|
|
326
|
+
* Defines a model. Called at module top level; the body runs exactly once,
|
|
327
|
+
* immediately, so references (`ctx.ref`) are known statically and the DAG
|
|
328
|
+
* is built without parsing SQL.
|
|
328
329
|
*/
|
|
329
330
|
export const defineModel = <const Fields extends Schema.Struct.Fields>(
|
|
330
331
|
config: ModelConfig<Fields>,
|
|
@@ -343,10 +344,10 @@ export const defineModel = <const Fields extends Schema.Struct.Fields>(
|
|
|
343
344
|
const known = new Set(Object.keys(model.schema.fields))
|
|
344
345
|
for (const column of names) {
|
|
345
346
|
if (!known.has(column)) {
|
|
346
|
-
//
|
|
347
|
+
// unreachable under honest typing; guards against `as any`
|
|
347
348
|
throw new ModelDefinitionError({
|
|
348
349
|
model: config.name,
|
|
349
|
-
reason:
|
|
350
|
+
reason: `column «${column}» is not in the schema of model ${model.name.full}`,
|
|
350
351
|
})
|
|
351
352
|
}
|
|
352
353
|
}
|
|
@@ -360,20 +361,20 @@ export const defineModel = <const Fields extends Schema.Struct.Fields>(
|
|
|
360
361
|
|
|
361
362
|
export interface SqlModelConfig<Fields extends Schema.Struct.Fields>
|
|
362
363
|
extends ModelConfig<Fields> {
|
|
363
|
-
/**
|
|
364
|
+
/** Path to the .sql body file: `@ref(schema.table)`, `@start`, `@end`. */
|
|
364
365
|
readonly file: string
|
|
365
366
|
/**
|
|
366
|
-
*
|
|
367
|
-
*
|
|
367
|
+
* Models the SQL text references via `@ref` — passed by value, so the
|
|
368
|
+
* DAG and testModel work the same way as for ordinary models.
|
|
368
369
|
*/
|
|
369
370
|
readonly refs?: ReadonlyArray<AnyModel>
|
|
370
371
|
}
|
|
371
372
|
|
|
372
373
|
/**
|
|
373
|
-
*
|
|
374
|
-
* dbt/sqlmesh
|
|
375
|
-
*
|
|
376
|
-
*
|
|
374
|
+
* Model from a raw .sql file (SPEC §14.1) — for migrating existing
|
|
375
|
+
* dbt/sqlmesh projects. Reference typing is lost (an honest price to pay):
|
|
376
|
+
* every `@ref` in the text must be declared in `refs`, and extra
|
|
377
|
+
* declarations are an error.
|
|
377
378
|
*/
|
|
378
379
|
export const defineSqlModel = <const Fields extends Schema.Struct.Fields>(
|
|
379
380
|
config: SqlModelConfig<Fields>,
|
|
@@ -386,7 +387,7 @@ export const defineSqlModel = <const Fields extends Schema.Struct.Fields>(
|
|
|
386
387
|
} catch (cause) {
|
|
387
388
|
throw new ModelDefinitionError({
|
|
388
389
|
model: name.full,
|
|
389
|
-
reason:
|
|
390
|
+
reason: `could not read ${config.file}: ${String(cause)}`,
|
|
390
391
|
})
|
|
391
392
|
}
|
|
392
393
|
const fragment = parseSqlText(text)
|
|
@@ -396,7 +397,7 @@ export const defineSqlModel = <const Fields extends Schema.Struct.Fields>(
|
|
|
396
397
|
if (!declared.has(ref)) {
|
|
397
398
|
throw new ModelDefinitionError({
|
|
398
399
|
model: name.full,
|
|
399
|
-
reason: `@ref(${ref})
|
|
400
|
+
reason: `@ref(${ref}) in ${config.file} is not declared in refs`,
|
|
400
401
|
})
|
|
401
402
|
}
|
|
402
403
|
}
|
|
@@ -404,7 +405,7 @@ export const defineSqlModel = <const Fields extends Schema.Struct.Fields>(
|
|
|
404
405
|
if (!used.has(declaredName)) {
|
|
405
406
|
throw new ModelDefinitionError({
|
|
406
407
|
model: name.full,
|
|
407
|
-
reason:
|
|
408
|
+
reason: `model ${declaredName} is declared in refs but @ref in ${config.file} does not use it`,
|
|
408
409
|
})
|
|
409
410
|
}
|
|
410
411
|
}
|
|
@@ -419,10 +420,10 @@ export interface ExternalConfig<Fields extends Schema.Struct.Fields> {
|
|
|
419
420
|
}
|
|
420
421
|
|
|
421
422
|
/**
|
|
422
|
-
*
|
|
423
|
-
*
|
|
424
|
-
*
|
|
425
|
-
*
|
|
423
|
+
* External source (SPEC §3.1, §9.3): not materialized, but participates
|
|
424
|
+
* in the DAG and lineage, and declares a schema. Only the source's
|
|
425
|
+
* *definition* feeds into the fingerprint — the contents change between
|
|
426
|
+
* runs, which is normal for raw data.
|
|
426
427
|
*/
|
|
427
428
|
export const defineExternal = <const Fields extends Schema.Struct.Fields>(
|
|
428
429
|
config: ExternalConfig<Fields>,
|
|
@@ -433,7 +434,7 @@ export const defineExternal = <const Fields extends Schema.Struct.Fields>(
|
|
|
433
434
|
schema: config.schema,
|
|
434
435
|
description: config.description,
|
|
435
436
|
grain: [],
|
|
436
|
-
target: "table", //
|
|
437
|
+
target: "table", // not materialized — field is unused
|
|
437
438
|
audits: [],
|
|
438
439
|
fragment: { _tag: "SqlFragment", nodes: [] },
|
|
439
440
|
deps: new Set(),
|
|
@@ -442,7 +443,7 @@ export const defineExternal = <const Fields extends Schema.Struct.Fields>(
|
|
|
442
443
|
|
|
443
444
|
export interface SeedConfig<Fields extends Schema.Struct.Fields> {
|
|
444
445
|
readonly name: string
|
|
445
|
-
/**
|
|
446
|
+
/** Path to the CSV/JSON file; format inferred from the extension or given explicitly. */
|
|
446
447
|
readonly file: string
|
|
447
448
|
readonly format?: "csv" | "json"
|
|
448
449
|
readonly schema: Schema.Struct<Fields>
|
|
@@ -451,9 +452,9 @@ export interface SeedConfig<Fields extends Schema.Struct.Fields> {
|
|
|
451
452
|
}
|
|
452
453
|
|
|
453
454
|
/**
|
|
454
|
-
* Seed (SPEC §3.1):
|
|
455
|
-
*
|
|
456
|
-
*
|
|
455
|
+
* Seed (SPEC §3.1): a reference table from a file. Unlike external, the
|
|
456
|
+
* file's contents feed into the fingerprint — editing the data means a new
|
|
457
|
+
* version and a rebuild; shape is checked by the schema contract at build time.
|
|
457
458
|
*/
|
|
458
459
|
export const defineSeed = <const Fields extends Schema.Struct.Fields>(
|
|
459
460
|
config: SeedConfig<Fields>,
|
package/src/core/sql.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* SQL
|
|
2
|
+
* SQL fragments: a tree of text, model references, and identifiers.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* fingerprint,
|
|
4
|
+
* A model body is rendered into a fragment once at definition time; the
|
|
5
|
+
* fragment turns into SQL text later — by the reference resolver
|
|
6
|
+
* (canonical rendering for the fingerprint, physical for execution).
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
export interface SqlFragment {
|
|
@@ -18,30 +18,30 @@ export type SqlNode =
|
|
|
18
18
|
| { readonly _tag: "Bound"; readonly which: "start" | "end" }
|
|
19
19
|
| { readonly _tag: "Self" }
|
|
20
20
|
|
|
21
|
-
/**
|
|
21
|
+
/** A model reference, obtained from `ctx.ref(model)`. */
|
|
22
22
|
export interface RefValue {
|
|
23
23
|
readonly _tag: "RefValue"
|
|
24
24
|
readonly modelName: string
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
/**
|
|
27
|
+
/** A list of columns, obtained from `ctx.cols(model, ...)`. */
|
|
28
28
|
export interface IdentsValue {
|
|
29
29
|
readonly _tag: "IdentsValue"
|
|
30
30
|
readonly names: ReadonlyArray<string>
|
|
31
31
|
}
|
|
32
32
|
|
|
33
|
-
/**
|
|
33
|
+
/** Bound of the interval being processed — `ctx.start` / `ctx.end` (SPEC §3). */
|
|
34
34
|
export interface BoundValue {
|
|
35
35
|
readonly _tag: "BoundValue"
|
|
36
36
|
readonly which: "start" | "end"
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
/**
|
|
39
|
+
/** The model's own result in an audit query — `ctx.self` (SPEC §8). */
|
|
40
40
|
export interface SelfValue {
|
|
41
41
|
readonly _tag: "SelfValue"
|
|
42
42
|
}
|
|
43
43
|
|
|
44
|
-
/**
|
|
44
|
+
/** Scalar values are inlined as SQL literals (deterministically). */
|
|
45
45
|
export type SqlLiteral = string | number | boolean | bigint | null
|
|
46
46
|
|
|
47
47
|
export type Interpolation =
|
|
@@ -124,16 +124,16 @@ export const sql = (
|
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
export interface RenderOptions {
|
|
127
|
-
/**
|
|
127
|
+
/** What a model reference turns into (a physical table, an environment view, a logical name…). */
|
|
128
128
|
readonly resolveRef: (modelName: string) => string
|
|
129
129
|
/**
|
|
130
|
-
*
|
|
131
|
-
* (`TIMESTAMP '…'`
|
|
132
|
-
*
|
|
133
|
-
*
|
|
130
|
+
* Bounds of the interval being processed — ready-made SQL expressions
|
|
131
|
+
* (`TIMESTAMP '…'` at execution time). Without them, `ctx.start`/`ctx.end`
|
|
132
|
+
* render as `$start`/`$end` placeholders — so the canonical text doesn't
|
|
133
|
+
* depend on specific dates and the fingerprint stays stable (SPEC §3, §4).
|
|
134
134
|
*/
|
|
135
135
|
readonly interval?: { readonly start: string; readonly end: string }
|
|
136
|
-
/**
|
|
136
|
+
/** What `ctx.self` renders to in an audit query (physical table or interval subquery). */
|
|
137
137
|
readonly self?: string
|
|
138
138
|
}
|
|
139
139
|
|
|
@@ -154,7 +154,7 @@ export const render = (fragment: SqlFragment, options: RenderOptions): string =>
|
|
|
154
154
|
out += options.interval === undefined ? `$${node.which}` : options.interval[node.which]
|
|
155
155
|
break
|
|
156
156
|
case "Self":
|
|
157
|
-
if (options.self === undefined) throw new Error("
|
|
157
|
+
if (options.self === undefined) throw new Error("rendering ctx.self without options.self")
|
|
158
158
|
out += options.self
|
|
159
159
|
break
|
|
160
160
|
}
|
|
@@ -163,10 +163,10 @@ export const render = (fragment: SqlFragment, options: RenderOptions): string =>
|
|
|
163
163
|
}
|
|
164
164
|
|
|
165
165
|
/**
|
|
166
|
-
*
|
|
167
|
-
*
|
|
168
|
-
*
|
|
169
|
-
*
|
|
166
|
+
* Parser for raw SQL text (SPEC §14.1): `@ref(schema.table)` is a model
|
|
167
|
+
* reference, `@start`/`@end` are interval bounds. Everything else is text
|
|
168
|
+
* as-is. For migrating existing dbt/sqlmesh projects: reference typing is
|
|
169
|
+
* lost, dependencies are declared alongside it (defineSqlModel.refs).
|
|
170
170
|
*/
|
|
171
171
|
export const parseSqlText = (text: string): SqlFragment => {
|
|
172
172
|
const nodes: Array<SqlNode> = []
|
|
@@ -195,6 +195,6 @@ export const collectRefs = (fragment: SqlFragment): ReadonlySet<string> => {
|
|
|
195
195
|
return refs
|
|
196
196
|
}
|
|
197
197
|
|
|
198
|
-
/**
|
|
198
|
+
/** Whether the fragment uses interval bounds (model-kind validation). */
|
|
199
199
|
export const usesBounds = (fragment: SqlFragment): boolean =>
|
|
200
200
|
fragment.nodes.some((node) => node._tag === "Bound")
|
package/src/discovery.ts
CHANGED
|
@@ -3,23 +3,31 @@ import { Data, Effect } from "effect"
|
|
|
3
3
|
import type { AnyModel } from "./core/model.ts"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
* defineSqlModel —
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
6
|
+
* Model discovery by glob (SPEC §12): files matching the config's masks are
|
|
7
|
+
* imported, and every model export (defineModel/defineExternal/defineSeed/
|
|
8
|
+
* defineSqlModel — all tagged _tag: "Model") joins the project. Order is
|
|
9
|
+
* deterministic (paths are sorted); two distinct definitions sharing a name
|
|
10
|
+
* are a load error; the same object re-exported from several files counts
|
|
11
|
+
* once.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
export class DiscoveryError extends Data.TaggedError("DiscoveryError")<{
|
|
15
15
|
readonly path: string
|
|
16
16
|
readonly reason: string
|
|
17
|
-
}> {
|
|
17
|
+
}> {
|
|
18
|
+
override get message(): string {
|
|
19
|
+
return `model discovery at ${this.path}: ${this.reason}`
|
|
20
|
+
}
|
|
21
|
+
}
|
|
18
22
|
|
|
19
23
|
export class DiscoveryConflictError extends Data.TaggedError("DiscoveryConflictError")<{
|
|
20
24
|
readonly name: string
|
|
21
25
|
readonly files: ReadonlyArray<string>
|
|
22
|
-
}> {
|
|
26
|
+
}> {
|
|
27
|
+
override get message(): string {
|
|
28
|
+
return `model «${this.name}» is defined in more than one place: ${this.files.join(", ")}`
|
|
29
|
+
}
|
|
30
|
+
}
|
|
23
31
|
|
|
24
32
|
const isModel = (value: unknown): value is AnyModel =>
|
|
25
33
|
typeof value === "object" &&
|