@avytheone/efmesh 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +22 -0
- package/README.ru.md +23 -0
- package/SPEC.md +1 -1
- package/package.json +1 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +204 -124
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +26 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +12 -11
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +7 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +33 -29
- package/src/plan/executor.ts +207 -125
- package/src/plan/explain.ts +29 -29
- package/src/plan/fingerprint.ts +35 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +107 -90
- package/src/plan/run.ts +27 -19
- package/src/plan/schedule.ts +46 -41
- package/src/plan/status.ts +15 -14
- package/src/state/postgres.ts +19 -19
- package/src/state/sqlite.ts +27 -27
- package/src/state/store.ts +67 -55
- package/src/testing/index.ts +27 -26
package/src/plan/naming.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import type { ExternalSource, ModelName } from "../core/model.ts"
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
* -
|
|
6
|
-
* -
|
|
7
|
-
*
|
|
4
|
+
* Object layout in the engine (SPEC §2):
|
|
5
|
+
* - physical: schema `_efmesh`, table `<schema>__<table>__<fp8>`;
|
|
6
|
+
* - virtual: prod lives in the models' native schemas (`med.stays`),
|
|
7
|
+
* other environments — in prefixed ones (`dev__med.stays`).
|
|
8
8
|
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
9
|
+
* The schema is `_efmesh`, not `efmesh`, precisely because: DuckDB names the
|
|
10
|
+
* catalog after the database file name, and for `efmesh.duckdb` the reference
|
|
11
|
+
* `efmesh.x` becomes ambiguous (catalog or schema) — a Binder Error.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
14
|
export const PROD_ENV = "prod"
|
|
@@ -31,28 +31,28 @@ export const envSchema = (env: string, modelSchema: string): string =>
|
|
|
31
31
|
env === PROD_ENV ? modelSchema : `${env}__${modelSchema}`
|
|
32
32
|
|
|
33
33
|
/**
|
|
34
|
-
*
|
|
35
|
-
*
|
|
36
|
-
*
|
|
37
|
-
*
|
|
34
|
+
* Parquet-lake layout (SPEC §3.3): `<lake>/<schema>/<table>/fp=<fp8>/…`,
|
|
35
|
+
* for incremental inside — partitions `interval=<key>/data.parquet`.
|
|
36
|
+
* Interval = partition: a recompute rewrites the partition's files, the
|
|
37
|
+
* source of truth is the interval bookkeeping, so a non-atomic rewrite is harmless.
|
|
38
38
|
*/
|
|
39
39
|
export const parquetPrefix = (lakePath: string, name: ModelName, fingerprint: string): string =>
|
|
40
40
|
`${lakePath.replace(/\/+$/, "")}/${name.schema}/${name.table}/fp=${fp8(fingerprint)}`
|
|
41
41
|
|
|
42
42
|
/**
|
|
43
|
-
* union_by_name:
|
|
44
|
-
* forward-only
|
|
45
|
-
*
|
|
43
|
+
* union_by_name: partitions of one prefix may differ in schema after
|
|
44
|
+
* forward-only evolution (new columns appear only in new files — history is
|
|
45
|
+
* read with NULL).
|
|
46
46
|
*/
|
|
47
47
|
export const parquetRef = (lakePath: string, name: ModelName, fingerprint: string): string =>
|
|
48
48
|
`read_parquet('${parquetPrefix(lakePath, name, fingerprint).replaceAll(`'`, `''`)}/**/*.parquet', union_by_name=true)`
|
|
49
49
|
|
|
50
50
|
/**
|
|
51
|
-
* DuckLake
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
51
|
+
* DuckLake target (SPEC §14.5): the catalog is attached under a fixed alias,
|
|
52
|
+
* the physical storage is a fingerprint table in it (the same name as native
|
|
53
|
+
* physical storage, only a different catalog). Versioning stays ours;
|
|
54
|
+
* DuckLake snapshots/time travel are a bonus. Consumers outside efmesh must
|
|
55
|
+
* ATTACH it themselves — the environments' views reference the catalog by alias.
|
|
56
56
|
*/
|
|
57
57
|
export const ducklakeAlias = "_efmesh_ducklake"
|
|
58
58
|
|
|
@@ -69,7 +69,7 @@ export const ducklakeAttachSql = (config: {
|
|
|
69
69
|
: ""
|
|
70
70
|
}`
|
|
71
71
|
|
|
72
|
-
/**
|
|
72
|
+
/** Interval partition key — filesystem-safe (no colons). */
|
|
73
73
|
export const intervalKey = (unit: "day" | "hour", startMs: number): string => {
|
|
74
74
|
const iso = new Date(startMs).toISOString()
|
|
75
75
|
return unit === "day" ? iso.slice(0, 10) : `${iso.slice(0, 10)}T${iso.slice(11, 13)}`
|
|
@@ -78,9 +78,9 @@ export const intervalKey = (unit: "day" | "hour", startMs: number): string => {
|
|
|
78
78
|
const READERS = { parquet: "read_parquet", csv: "read_csv", json: "read_json" } as const
|
|
79
79
|
|
|
80
80
|
/**
|
|
81
|
-
*
|
|
82
|
-
* (
|
|
83
|
-
* external
|
|
81
|
+
* What a reference to an external model renders to (SPEC §9.3): the table name
|
|
82
|
+
* (of the engine or an ATTACH database) as is, files/URLs — via read_*.
|
|
83
|
+
* external is not materialized, consumers read the source directly.
|
|
84
84
|
*/
|
|
85
85
|
export const externalSourceRef = (source: ExternalSource): string =>
|
|
86
86
|
source._tag === "table"
|
package/src/plan/planner.ts
CHANGED
|
@@ -15,52 +15,68 @@ import { validateEnvName } from "./naming.ts"
|
|
|
15
15
|
|
|
16
16
|
export class InvalidEnvironmentError extends Data.TaggedError("InvalidEnvironmentError")<{
|
|
17
17
|
readonly env: string
|
|
18
|
-
}> {
|
|
18
|
+
}> {
|
|
19
|
+
override get message(): string {
|
|
20
|
+
return `invalid environment name «${this.env}» — use latin letters, digits and _`
|
|
21
|
+
}
|
|
22
|
+
}
|
|
19
23
|
|
|
20
|
-
/**
|
|
24
|
+
/** The model cannot be applied forward-only (SPEC §5.2). */
|
|
21
25
|
export class ForwardOnlyError extends Data.TaggedError("ForwardOnlyError")<{
|
|
22
26
|
readonly model: string
|
|
23
27
|
readonly reason: string
|
|
24
|
-
}> {
|
|
28
|
+
}> {
|
|
29
|
+
override get message(): string {
|
|
30
|
+
return `model «${this.model}» cannot be applied forward-only: ${this.reason}`
|
|
31
|
+
}
|
|
32
|
+
}
|
|
25
33
|
|
|
26
34
|
/**
|
|
27
|
-
*
|
|
28
|
-
* AST
|
|
29
|
-
*
|
|
35
|
+
* A categorization override is rejected (#5): the model is not in the project
|
|
36
|
+
* or the AST diff obviously contradicts the claimed verdict (removed columns
|
|
37
|
+
* are never non-breaking — descendants read them by name).
|
|
30
38
|
*/
|
|
31
39
|
export class ReclassifyError extends Data.TaggedError("ReclassifyError")<{
|
|
32
40
|
readonly model: string
|
|
33
41
|
readonly reason: string
|
|
34
|
-
}> {
|
|
42
|
+
}> {
|
|
43
|
+
override get message(): string {
|
|
44
|
+
return `reclassify «${this.model}»: ${this.reason}`
|
|
45
|
+
}
|
|
46
|
+
}
|
|
35
47
|
|
|
36
48
|
/**
|
|
37
|
-
*
|
|
38
|
-
*
|
|
39
|
-
*
|
|
40
|
-
*
|
|
49
|
+
* The snapshot was computed by a different version of the fingerprint
|
|
50
|
+
* algorithm (SPEC §4): fingerprints of different versions are incomparable —
|
|
51
|
+
* the plan honestly stops instead of showing "everything breaking". Cured by
|
|
52
|
+
* migrating to the version of efmesh that changed the algorithm.
|
|
41
53
|
*/
|
|
42
54
|
export class FingerprintVersionError extends Data.TaggedError("FingerprintVersionError")<{
|
|
43
55
|
readonly model: string
|
|
44
56
|
readonly found: number
|
|
45
57
|
readonly wanted: number
|
|
46
|
-
}> {
|
|
58
|
+
}> {
|
|
59
|
+
override get message(): string {
|
|
60
|
+
return `model «${this.model}» was fingerprinted by algorithm v${this.found}, but this efmesh expects v${this.wanted}`
|
|
61
|
+
}
|
|
62
|
+
}
|
|
47
63
|
|
|
48
64
|
export type ChangeCategory =
|
|
49
|
-
/**
|
|
65
|
+
/** The model was not in the environment. */
|
|
50
66
|
| "added"
|
|
51
|
-
/**
|
|
67
|
+
/** A meaningful edit of the query or metadata: the model and its descendants rebuild. */
|
|
52
68
|
| "breaking"
|
|
53
|
-
/**
|
|
69
|
+
/** Columns appended to the end of the SELECT, the rest of the tree untouched (SPEC §5.2). */
|
|
54
70
|
| "non-breaking"
|
|
55
|
-
/**
|
|
71
|
+
/** The own AST did not change — the version shifted by cascade from a parent. */
|
|
56
72
|
| "indirect"
|
|
57
73
|
/**
|
|
58
|
-
*
|
|
59
|
-
*
|
|
60
|
-
*
|
|
74
|
+
* The physics and done-intervals of the old version are reused, history is
|
|
75
|
+
* not replayed — the new logic takes effect from the moment of application
|
|
76
|
+
* (SPEC §5.2). An explicit user flag or a cascade from forward-only parents.
|
|
61
77
|
*/
|
|
62
78
|
| "forward-only"
|
|
63
|
-
/**
|
|
79
|
+
/** Was in the environment, vanished from the project — the view is dropped at promotion. */
|
|
64
80
|
| "removed"
|
|
65
81
|
| "unchanged"
|
|
66
82
|
|
|
@@ -68,72 +84,73 @@ export interface PlanAction {
|
|
|
68
84
|
readonly name: string
|
|
69
85
|
readonly fingerprint: string
|
|
70
86
|
/**
|
|
71
|
-
*
|
|
72
|
-
*
|
|
87
|
+
* The fingerprint whose physics the snapshot uses: usually its own, and
|
|
88
|
+
* under forward-only — inherited from the previous version.
|
|
73
89
|
*/
|
|
74
90
|
readonly physicalFingerprint: string
|
|
75
|
-
/**
|
|
91
|
+
/** Where the physics and done-intervals are inherited from (fingerprint of the old version) under forward-only. */
|
|
76
92
|
readonly reusedFrom?: string
|
|
77
|
-
/**
|
|
93
|
+
/** Canonical AST of the body (null for external) — saved into the snapshot. */
|
|
78
94
|
readonly canonicalAst: string | null
|
|
79
95
|
readonly change: ChangeCategory
|
|
80
|
-
/**
|
|
96
|
+
/** The planner's verdict before the operator override (#5); journaled. */
|
|
81
97
|
readonly reclassifiedFrom?: ChangeCategory
|
|
82
98
|
/**
|
|
83
|
-
*
|
|
84
|
-
*
|
|
85
|
-
*
|
|
99
|
+
* Why the category is what it is (#4): the diverged nodes of the canonical
|
|
100
|
+
* AST and the reason in words. Present on all changed models except
|
|
101
|
+
* added/removed — there is nothing to compare there. The diverged paths are
|
|
102
|
+
* a debugging hint, not a contract.
|
|
86
103
|
*/
|
|
87
104
|
readonly explain?: ChangeExplanation
|
|
88
105
|
/**
|
|
89
|
-
*
|
|
90
|
-
*
|
|
91
|
-
*
|
|
106
|
+
* Whether physics needs building. false for unchanged/removed, external and
|
|
107
|
+
* for snapshots already built by another environment — then promotion is
|
|
108
|
+
* only a view-swap.
|
|
92
109
|
*/
|
|
93
110
|
readonly build: boolean
|
|
94
111
|
/**
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
* unchanged
|
|
112
|
+
* Ranges to recompute for incrementalByTimeRange (merged from missing
|
|
113
|
+
* intervals + lookback); empty for other kinds. Gaps happen even for an
|
|
114
|
+
* unchanged model — time passes, new intervals appear.
|
|
98
115
|
*/
|
|
99
116
|
readonly backfill: ReadonlyArray<Interval>
|
|
100
|
-
/** true
|
|
117
|
+
/** true for incrementalByUniqueKey: every apply re-runs the query (upsert by key). */
|
|
101
118
|
readonly refresh: boolean
|
|
102
119
|
}
|
|
103
120
|
|
|
104
121
|
export interface Plan {
|
|
105
122
|
readonly env: string
|
|
106
|
-
/**
|
|
123
|
+
/** In topological order; removed comes last. */
|
|
107
124
|
readonly actions: ReadonlyArray<PlanAction>
|
|
108
125
|
readonly hasChanges: boolean
|
|
109
126
|
}
|
|
110
127
|
|
|
111
128
|
export interface PlanOptions {
|
|
112
|
-
/**
|
|
129
|
+
/** "Now" for computing intervals; defaults to Clock. Injection point for tests. */
|
|
113
130
|
readonly now?: number
|
|
114
131
|
/**
|
|
115
|
-
*
|
|
116
|
-
*
|
|
117
|
-
*
|
|
118
|
-
*
|
|
132
|
+
* Models whose changes to apply forward-only (SPEC §5.2): the new version
|
|
133
|
+
* inherits the physical table and done-intervals of the old one, history is
|
|
134
|
+
* not replayed. Only incrementalByTimeRange — other kinds have no
|
|
135
|
+
* "retroactive" notion by construction.
|
|
119
136
|
*/
|
|
120
137
|
readonly forwardOnly?: ReadonlyArray<string>
|
|
121
138
|
/**
|
|
122
|
-
*
|
|
123
|
-
*
|
|
124
|
-
* (non-breaking
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
* added/removed/indirect
|
|
139
|
+
* Categorization override (#5, SPEC §5.2): the operator, looking at
|
|
140
|
+
* `--explain`, states the verdict instead of the planner. It governs the
|
|
141
|
+
* fate of DESCENDANTS (a non-breaking parent lets them reuse the physics);
|
|
142
|
+
* the model itself is freed from rebuilding not by this but by forwardOnly.
|
|
143
|
+
* Guardrail: an obvious AST contradiction (removed columns → non-breaking)
|
|
144
|
+
* is an error. Applies only to breaking/non-breaking verdicts; on unchanged/
|
|
145
|
+
* added/removed/indirect it silently has no effect.
|
|
129
146
|
*/
|
|
130
147
|
readonly reclassify?: Readonly<Record<string, "breaking" | "non-breaking">>
|
|
131
148
|
}
|
|
132
149
|
|
|
133
150
|
/**
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
151
|
+
* Kinds whose physics is worth inheriting on indirect reuse (#5):
|
|
152
|
+
* materialized tables. view/embedded have no materialization; seed rebuilds
|
|
153
|
+
* cheaply from a file and has no parents.
|
|
137
154
|
*/
|
|
138
155
|
const REUSABLE_KINDS: ReadonlySet<string> = new Set([
|
|
139
156
|
"full",
|
|
@@ -166,23 +183,23 @@ export const planChanges = (
|
|
|
166
183
|
for (const flagged of forwardOnly) {
|
|
167
184
|
const model = graph.models.get(flagged)
|
|
168
185
|
if (model === undefined) {
|
|
169
|
-
return yield* new ForwardOnlyError({ model: flagged, reason: "
|
|
186
|
+
return yield* new ForwardOnlyError({ model: flagged, reason: "model is not in the project" })
|
|
170
187
|
}
|
|
171
188
|
if (model.kind._tag !== "incrementalByTimeRange") {
|
|
172
189
|
return yield* new ForwardOnlyError({
|
|
173
190
|
model: flagged,
|
|
174
|
-
reason: `forward-only
|
|
191
|
+
reason: `forward-only reuses physics and interval accounting — applicable only to incrementalByTimeRange, model kind is ${model.kind._tag}`,
|
|
175
192
|
})
|
|
176
193
|
}
|
|
177
194
|
}
|
|
178
195
|
const reclassify = options?.reclassify ?? {}
|
|
179
196
|
for (const flagged of Object.keys(reclassify)) {
|
|
180
197
|
if (!graph.models.has(flagged)) {
|
|
181
|
-
return yield* new ReclassifyError({ model: flagged, reason: "
|
|
198
|
+
return yield* new ReclassifyError({ model: flagged, reason: "model is not in the project" })
|
|
182
199
|
}
|
|
183
200
|
}
|
|
184
|
-
//
|
|
185
|
-
//
|
|
201
|
+
// canonicalization cache (#8) — the store is at hand; its failures are
|
|
202
|
+
// swallowed: a cache miss is just an honest recompute, not a plan error
|
|
186
203
|
const versions = yield* fingerprintGraph(graph, {
|
|
187
204
|
get: (key) => store.getCanon(key).pipe(Effect.orElseSucceed(() => undefined)),
|
|
188
205
|
put: (key, canonical) => store.putCanon(key, canonical).pipe(Effect.ignore),
|
|
@@ -193,7 +210,7 @@ export const planChanges = (
|
|
|
193
210
|
|
|
194
211
|
const actions: Array<PlanAction> = []
|
|
195
212
|
const changeOf = new Map<string, ChangeCategory>()
|
|
196
|
-
//
|
|
213
|
+
// who in this plan inherits the physics of the old version (indirect reuse, #5)
|
|
197
214
|
const reusedPhysics = new Set<string>()
|
|
198
215
|
for (const name of graph.order) {
|
|
199
216
|
const model = graph.models.get(name)!
|
|
@@ -207,8 +224,8 @@ export const planChanges = (
|
|
|
207
224
|
if (known === undefined) change = "added"
|
|
208
225
|
else if (known === fingerprint) change = "unchanged"
|
|
209
226
|
else {
|
|
210
|
-
//
|
|
211
|
-
//
|
|
227
|
+
// categorization by AST against the last known snapshot (SPEC §5.2);
|
|
228
|
+
// old records without an AST and external — conservatively breaking
|
|
212
229
|
const previous = yield* store.getSnapshot(name, known)
|
|
213
230
|
if (previous !== undefined && previous.fingerprintVersion !== FINGERPRINT_VERSION) {
|
|
214
231
|
return yield* new FingerprintVersionError({
|
|
@@ -228,29 +245,29 @@ export const planChanges = (
|
|
|
228
245
|
diverged: [],
|
|
229
246
|
reason:
|
|
230
247
|
ast === null
|
|
231
|
-
? "
|
|
232
|
-
: "
|
|
248
|
+
? "model has no SQL body (external/seed) — the version was shifted by source/file, schema or parents; there is no AST to compare against"
|
|
249
|
+
: "the previous snapshot did not store a canonical AST — nothing to compare against, conservatively breaking",
|
|
233
250
|
}
|
|
234
251
|
} else if (oldAst === ast) {
|
|
235
|
-
change = "indirect" //
|
|
252
|
+
change = "indirect" // version shifted by a parent/metadata
|
|
236
253
|
explain =
|
|
237
254
|
changedParents.length > 0
|
|
238
255
|
? {
|
|
239
256
|
diverged: [],
|
|
240
|
-
reason:
|
|
257
|
+
reason: `own AST did not change — the version was shifted by a cascade from parents: ${changedParents.join(", ")}`,
|
|
241
258
|
cascadeFrom: changedParents,
|
|
242
259
|
}
|
|
243
260
|
: {
|
|
244
261
|
diverged: [],
|
|
245
262
|
reason:
|
|
246
|
-
"
|
|
263
|
+
"own AST did not change — metadata diverged (kind/grain/columns/target)",
|
|
247
264
|
}
|
|
248
265
|
} else {
|
|
249
266
|
change = categorizeAstChange(oldAst, ast)
|
|
250
267
|
explain = explainCategorized(oldAst, ast, change)
|
|
251
268
|
}
|
|
252
|
-
// override
|
|
253
|
-
//
|
|
269
|
+
// operator override (#5): the verdict is stated by a flag on top of
|
|
270
|
+
// --explain; the planner checks only an obvious AST contradiction
|
|
254
271
|
const override = reclassify[name]
|
|
255
272
|
if (
|
|
256
273
|
override !== undefined &&
|
|
@@ -261,26 +278,26 @@ export const planChanges = (
|
|
|
261
278
|
return yield* new ReclassifyError({
|
|
262
279
|
model: name,
|
|
263
280
|
reason:
|
|
264
|
-
"
|
|
281
|
+
"no canonical AST — nothing to check the verdict against, override is not accepted",
|
|
265
282
|
})
|
|
266
283
|
}
|
|
267
284
|
if (override === "non-breaking" && ast !== null && dropsColumns(oldAst, ast)) {
|
|
268
285
|
return yield* new ReclassifyError({
|
|
269
286
|
model: name,
|
|
270
287
|
reason:
|
|
271
|
-
"
|
|
288
|
+
"the new SELECT has fewer columns — descendants read the removed columns by name, non-breaking contradicts the AST",
|
|
272
289
|
})
|
|
273
290
|
}
|
|
274
291
|
reclassifiedFrom = change
|
|
275
292
|
explain = {
|
|
276
293
|
diverged: explain?.diverged ?? [],
|
|
277
|
-
reason: `override
|
|
294
|
+
reason: `operator override: ${change} → ${override}; planner verdict: ${explain?.reason ?? "—"}`,
|
|
278
295
|
}
|
|
279
296
|
change = override
|
|
280
297
|
}
|
|
281
|
-
//
|
|
282
|
-
//
|
|
283
|
-
//
|
|
298
|
+
// "version shifted by parents ONLY": a fingerprint with the old parent
|
|
299
|
+
// signatures must yield known — otherwise the metadata diverged along
|
|
300
|
+
// with the parents (kind/grain/columns/target) and physics cannot be inherited
|
|
284
301
|
const parentsOnly =
|
|
285
302
|
change === "indirect" &&
|
|
286
303
|
previous !== undefined &&
|
|
@@ -290,9 +307,9 @@ export const planChanges = (
|
|
|
290
307
|
ast,
|
|
291
308
|
[...model.deps].sort().map((dep) => `${dep}=${current.get(dep)!}`),
|
|
292
309
|
)) === known
|
|
293
|
-
// forward-only:
|
|
294
|
-
//
|
|
295
|
-
//
|
|
310
|
+
// forward-only: an explicit user flag — or a cascade: the own AST did
|
|
311
|
+
// not change and all changed parents are themselves forward-only (reuse
|
|
312
|
+
// of a parent's physics means the descendant has nothing to replay either)
|
|
296
313
|
const cascaded =
|
|
297
314
|
parentsOnly &&
|
|
298
315
|
model.kind._tag === "incrementalByTimeRange" &&
|
|
@@ -307,21 +324,21 @@ export const planChanges = (
|
|
|
307
324
|
explain = forwardOnly.has(name)
|
|
308
325
|
? {
|
|
309
326
|
diverged: explain?.diverged ?? [],
|
|
310
|
-
reason: `forward-only
|
|
327
|
+
reason: `forward-only by flag: physics and done intervals are inherited from @${known.slice(0, 8)}, history is not replayed`,
|
|
311
328
|
}
|
|
312
329
|
: {
|
|
313
330
|
diverged: [],
|
|
314
331
|
reason:
|
|
315
|
-
"
|
|
332
|
+
"own AST did not change, all changed parents are forward-only — physics is reused by cascade",
|
|
316
333
|
cascadeFrom: changedParents,
|
|
317
334
|
}
|
|
318
335
|
}
|
|
319
|
-
// indirect
|
|
320
|
-
//
|
|
321
|
-
//
|
|
322
|
-
//
|
|
323
|
-
//
|
|
324
|
-
//
|
|
336
|
+
// indirect reuse (#5, sqlmesh's "indirect non-breaking" class): the own
|
|
337
|
+
// body did not change, the version was shifted only by parents, and each
|
|
338
|
+
// changed parent guarantees the same data in the old columns (non-breaking
|
|
339
|
+
// — strictly a suffix; forward-only and reuse — literally the same
|
|
340
|
+
// physics) — the descendant has nothing to replay: physics and ledger are
|
|
341
|
+
// inherited, scdType2 does not lose its accumulated row history
|
|
325
342
|
if (
|
|
326
343
|
change === "indirect" &&
|
|
327
344
|
parentsOnly &&
|
|
@@ -341,13 +358,13 @@ export const planChanges = (
|
|
|
341
358
|
reusedPhysics.add(name)
|
|
342
359
|
explain = {
|
|
343
360
|
diverged: [],
|
|
344
|
-
reason:
|
|
361
|
+
reason: `changed parents do not touch existing data (non-breaking/forward-only) — physics and accounting are inherited from @${known.slice(0, 8)}, no rebuild`,
|
|
345
362
|
cascadeFrom: changedParents,
|
|
346
363
|
}
|
|
347
364
|
}
|
|
348
365
|
}
|
|
349
366
|
changeOf.set(name, change)
|
|
350
|
-
// external
|
|
367
|
+
// external is not materialized — the version participates in the diff, there is no physics
|
|
351
368
|
const existing = yield* store.getSnapshot(name, fingerprint)
|
|
352
369
|
if (existing !== undefined) physicalFingerprint = existing.physicalFp
|
|
353
370
|
const alreadyBuilt =
|
|
@@ -357,9 +374,9 @@ export const planChanges = (
|
|
|
357
374
|
if (model.kind._tag === "incrementalByTimeRange") {
|
|
358
375
|
const kind = model.kind
|
|
359
376
|
const wanted = enumerateIntervals(kind.interval, fromIso(kind.start), now)
|
|
360
|
-
//
|
|
361
|
-
//
|
|
362
|
-
//
|
|
377
|
+
// coverage is keyed to the fingerprint: a new version = empty ledger =
|
|
378
|
+
// full backfill; forward-only inherits the done-intervals of the old
|
|
379
|
+
// version — only what was missing is recomputed
|
|
363
380
|
const inherited =
|
|
364
381
|
reusedFrom === undefined ? [] : yield* store.listIntervals(reusedFrom)
|
|
365
382
|
const doneByStart = new Map<number, Interval>()
|
|
@@ -373,7 +390,7 @@ export const planChanges = (
|
|
|
373
390
|
const done = [...doneByStart.values()]
|
|
374
391
|
const missing = [...missingIntervals(wanted, done)]
|
|
375
392
|
if (kind.lookback > 0) {
|
|
376
|
-
//
|
|
393
|
+
// late-arriving data: the last done-intervals are re-read
|
|
377
394
|
const tail = done.sort((a, b) => a.start - b.start).slice(-kind.lookback)
|
|
378
395
|
missing.push(...tail)
|
|
379
396
|
}
|
|
@@ -391,7 +408,7 @@ export const planChanges = (
|
|
|
391
408
|
...(explain !== undefined ? { explain } : {}),
|
|
392
409
|
build: !alreadyBuilt,
|
|
393
410
|
backfill,
|
|
394
|
-
//
|
|
411
|
+
// every apply reconciles the query with the physics: upsert / SCD versioning
|
|
395
412
|
refresh:
|
|
396
413
|
model.kind._tag === "incrementalByUniqueKey" || model.kind._tag === "scdType2",
|
|
397
414
|
})
|
package/src/plan/run.ts
CHANGED
|
@@ -8,24 +8,28 @@ import { envLockName, withStateLock, type LockHeldError, type LockOptions } from
|
|
|
8
8
|
import { planChanges, type PlanOptions } from "./planner.ts"
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* `run` —
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
11
|
+
* `run` — a scheduler tick (SPEC §7): catches up intervals and upserts of
|
|
12
|
+
* EXISTING versions, never applies model changes — that is the job of
|
|
13
|
+
* plan/apply with a human at the helm. Idempotent and safe for cron/systemd:
|
|
14
|
+
* a parallel run is cut off by a lock in the state store — the same
|
|
15
|
+
* `env:<name>` as apply's (SPEC §14.6), so run will not wedge into someone
|
|
16
|
+
* else's apply and vice versa.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
export class RunBlockedByChangesError extends Data.TaggedError("RunBlockedByChangesError")<{
|
|
20
20
|
readonly env: string
|
|
21
21
|
readonly changes: ReadonlyArray<string>
|
|
22
|
-
}> {
|
|
22
|
+
}> {
|
|
23
|
+
override get message(): string {
|
|
24
|
+
return `environment «${this.env}» has unapplied structural changes: ${this.changes.join(", ")} — run \`efmesh apply ${this.env}\``
|
|
25
|
+
}
|
|
26
|
+
}
|
|
23
27
|
|
|
24
28
|
export type RunError = ApplyError | LockHeldError | RunBlockedByChangesError
|
|
25
29
|
|
|
26
30
|
export interface RunOptions extends PlanOptions, ApplyOptions, LockOptions {}
|
|
27
31
|
|
|
28
|
-
/**
|
|
32
|
+
/** Tick outcome for the journal: error → category + detail. */
|
|
29
33
|
const classify = (error: RunError): Pick<RunRecord, "outcome" | "detail"> => {
|
|
30
34
|
switch (error._tag) {
|
|
31
35
|
case "RunBlockedByChangesError":
|
|
@@ -45,9 +49,10 @@ export const run = (
|
|
|
45
49
|
Effect.gen(function* () {
|
|
46
50
|
const store = yield* StateStore
|
|
47
51
|
const startedAt = new Date(yield* Clock.currentTimeMillis).toISOString()
|
|
48
|
-
//
|
|
49
|
-
//
|
|
50
|
-
//
|
|
52
|
+
// tick journal (SPEC §7, #2): the outcome is written ALWAYS, including
|
|
53
|
+
// failure — a cron tick that fell over at three in the morning is debugged
|
|
54
|
+
// after the fact; a failure of the write itself does not mask the real
|
|
55
|
+
// outcome (log + ignore)
|
|
51
56
|
const journal = (entry: Pick<RunRecord, "outcome" | "detail">) =>
|
|
52
57
|
Clock.currentTimeMillis.pipe(
|
|
53
58
|
Effect.flatMap((now) =>
|
|
@@ -58,7 +63,7 @@ export const run = (
|
|
|
58
63
|
...entry,
|
|
59
64
|
}),
|
|
60
65
|
),
|
|
61
|
-
Effect.catchCause((cause) => Effect.logWarning("
|
|
66
|
+
Effect.catchCause((cause) => Effect.logWarning("tick journal is unavailable", cause)),
|
|
62
67
|
)
|
|
63
68
|
|
|
64
69
|
return yield* Effect.gen(function* () {
|
|
@@ -81,9 +86,9 @@ export const run = (
|
|
|
81
86
|
})
|
|
82
87
|
|
|
83
88
|
/**
|
|
84
|
-
*
|
|
85
|
-
*
|
|
86
|
-
* (
|
|
89
|
+
* Long-lived scheduler for embedding in an Effect application (SPEC §7):
|
|
90
|
+
* ticks on a Schedule; a tick error is logged and does not bring the daemon
|
|
91
|
+
* down (except a held lock — that is routine, logged more quietly).
|
|
87
92
|
*/
|
|
88
93
|
export const daemon = (
|
|
89
94
|
env: string,
|
|
@@ -94,13 +99,16 @@ export const daemon = (
|
|
|
94
99
|
run(env, models, options).pipe(
|
|
95
100
|
Effect.tap((applied) =>
|
|
96
101
|
applied.built.length > 0
|
|
97
|
-
? Effect.logInfo(`
|
|
98
|
-
: Effect.
|
|
102
|
+
? Effect.logInfo(`tick built ${applied.built.join(", ")}`)
|
|
103
|
+
: Effect.logInfo("tick: no new intervals"),
|
|
99
104
|
),
|
|
105
|
+
// a held lock is routine (another process ticks) — debug, not an alarm
|
|
100
106
|
Effect.catchTag("LockHeldError", () =>
|
|
101
|
-
Effect.logDebug(
|
|
107
|
+
Effect.logDebug("tick skipped: lock held by another process"),
|
|
102
108
|
),
|
|
103
|
-
Effect.catchCause((cause) => Effect.logError(
|
|
109
|
+
Effect.catchCause((cause) => Effect.logError("tick failed", cause)),
|
|
110
|
+
// env is a structured field, not baked into every message (#14)
|
|
111
|
+
Effect.annotateLogs("env", env),
|
|
104
112
|
Effect.schedule(schedule),
|
|
105
113
|
Effect.andThen(Effect.never),
|
|
106
114
|
)
|