@avytheone/efmesh 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +22 -0
- package/README.ru.md +23 -0
- package/SPEC.md +1 -1
- package/package.json +1 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +204 -124
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +26 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +12 -11
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +7 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +33 -29
- package/src/plan/executor.ts +207 -125
- package/src/plan/explain.ts +29 -29
- package/src/plan/fingerprint.ts +35 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +107 -90
- package/src/plan/run.ts +27 -19
- package/src/plan/schedule.ts +46 -41
- package/src/plan/status.ts +15 -14
- package/src/state/postgres.ts +19 -19
- package/src/state/sqlite.ts +27 -27
- package/src/state/store.ts +67 -55
- package/src/testing/index.ts +27 -26
package/src/plan/explain.ts
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
* libpg_query
|
|
6
|
-
*
|
|
7
|
-
*
|
|
2
|
+
* Categorization explanation (#4, SPEC §5.2): `plan --explain` shows WHICH
|
|
3
|
+
* nodes of the canonical AST diverged and why the category is what it is.
|
|
4
|
+
* Paths follow the engine's canon tree (json_serialize_sql on DuckDB,
|
|
5
|
+
* libpg_query on Postgres) — this is a debugging hint, not a contract:
|
|
6
|
+
* the shape of the paths moves with the canon and is not a semver event.
|
|
7
|
+
* The category itself is computed by categorize.ts; here — only its rationale.
|
|
8
8
|
*/
|
|
9
9
|
|
|
10
10
|
import { topSelect } from "./categorize.ts"
|
|
11
11
|
|
|
12
12
|
export interface ChangeExplanation {
|
|
13
|
-
/**
|
|
13
|
+
/** Paths of diverged canonical-AST nodes (no more than MAX_DIVERGED). */
|
|
14
14
|
readonly diverged: ReadonlyArray<string>
|
|
15
|
-
/**
|
|
15
|
+
/** Why the category is what it is. */
|
|
16
16
|
readonly reason: string
|
|
17
|
-
/**
|
|
17
|
+
/** Changed direct parents — the source of the indirect/forward-only cascade. */
|
|
18
18
|
readonly cascadeFrom?: ReadonlyArray<string>
|
|
19
19
|
}
|
|
20
20
|
|
|
21
|
-
/**
|
|
21
|
+
/** Beyond this, pointwise paths stop helping — better to open render/diff. */
|
|
22
22
|
const MAX_DIVERGED = 8
|
|
23
23
|
|
|
24
24
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
|
@@ -34,32 +34,32 @@ const walk = (before: unknown, after: unknown, path: string, out: Array<string>)
|
|
|
34
34
|
const shared = Math.min(before.length, after.length)
|
|
35
35
|
for (let i = 0; i < shared; i++) walk(before[i], after[i], `${path}[${i}]`, out)
|
|
36
36
|
for (let i = shared; i < after.length && out.length < MAX_DIVERGED; i++) {
|
|
37
|
-
out.push(`${path}[${i}] (
|
|
37
|
+
out.push(`${path}[${i}] (added)`)
|
|
38
38
|
}
|
|
39
39
|
for (let i = shared; i < before.length && out.length < MAX_DIVERGED; i++) {
|
|
40
|
-
out.push(`${path}[${i}] (
|
|
40
|
+
out.push(`${path}[${i}] (removed)`)
|
|
41
41
|
}
|
|
42
42
|
return
|
|
43
43
|
}
|
|
44
44
|
if (isRecord(before) && isRecord(after)) {
|
|
45
|
-
//
|
|
45
|
+
// node replaced by an expression of another type — pointwise paths inside are meaningless
|
|
46
46
|
if (before["type"] !== after["type"]) {
|
|
47
|
-
out.push(path === "" ? "(
|
|
47
|
+
out.push(path === "" ? "(root)" : path)
|
|
48
48
|
return
|
|
49
49
|
}
|
|
50
50
|
const keys = [...new Set([...Object.keys(before), ...Object.keys(after)])].sort()
|
|
51
51
|
for (const key of keys) walk(before[key], after[key], at(path, key), out)
|
|
52
52
|
return
|
|
53
53
|
}
|
|
54
|
-
out.push(path === "" ? "(
|
|
54
|
+
out.push(path === "" ? "(root)" : path)
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
/**
|
|
57
|
+
/** The statement's boilerplate wrapper in the path is uninteresting — trim to the query body. */
|
|
58
58
|
const trimStatement = (path: string): string =>
|
|
59
59
|
path.replace(/^statements\[0\]\.node\.?/, "").replace(/^stmts\[0\]\.stmt\.?/, "") ||
|
|
60
|
-
"(
|
|
60
|
+
"(root)"
|
|
61
61
|
|
|
62
|
-
/**
|
|
62
|
+
/** Divergence paths of two canonical ASTs (JSON strings); garbage input — empty. */
|
|
63
63
|
export const divergedPaths = (oldAst: string, newAst: string): ReadonlyArray<string> => {
|
|
64
64
|
try {
|
|
65
65
|
const out: Array<string> = []
|
|
@@ -71,10 +71,10 @@ export const divergedPaths = (oldAst: string, newAst: string): ReadonlyArray<str
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
76
|
-
* AST.
|
|
77
|
-
*
|
|
74
|
+
* Override guardrail (#5): the new top SELECT has fewer columns — they were
|
|
75
|
+
* removed, descendants read them by name, so «non-breaking» plainly contradicts
|
|
76
|
+
* the AST. An unparseable canon does not count as a contradiction: the decision
|
|
77
|
+
* is the operator's, the override is explicit and journaled.
|
|
78
78
|
*/
|
|
79
79
|
export const dropsColumns = (oldAst: string, newAst: string): boolean => {
|
|
80
80
|
const before = topSelect(oldAst)
|
|
@@ -84,8 +84,8 @@ export const dropsColumns = (oldAst: string, newAst: string): boolean => {
|
|
|
84
84
|
}
|
|
85
85
|
|
|
86
86
|
/**
|
|
87
|
-
*
|
|
88
|
-
*
|
|
87
|
+
* Justifies the categorizeAstChange verdict by the same rules that produced
|
|
88
|
+
* it (SPEC §5.2): non-breaking — only a suffix of the top SELECT.
|
|
89
89
|
*/
|
|
90
90
|
export const explainCategorized = (
|
|
91
91
|
oldAst: string,
|
|
@@ -97,18 +97,18 @@ export const explainCategorized = (
|
|
|
97
97
|
return {
|
|
98
98
|
diverged,
|
|
99
99
|
reason:
|
|
100
|
-
"
|
|
100
|
+
"columns were appended to the end of the top SELECT, the rest of the tree is untouched — consumers read by name, no rebuild needed",
|
|
101
101
|
}
|
|
102
102
|
}
|
|
103
103
|
const before = topSelect(oldAst)
|
|
104
104
|
const after = topSelect(newAst)
|
|
105
105
|
const reason =
|
|
106
106
|
before === null || after === null
|
|
107
|
-
? "
|
|
107
|
+
? "canonical AST of an unexpected shape — conservatively breaking"
|
|
108
108
|
: before.rest !== after.rest
|
|
109
|
-
? "
|
|
109
|
+
? "the tree diverged outside the SELECT list (FROM/WHERE/JOIN/GROUP BY/modifiers)"
|
|
110
110
|
: after.list.length < before.list.length
|
|
111
|
-
? "
|
|
112
|
-
: "
|
|
111
|
+
? "columns were removed from SELECT — descendants insert by position, physics must be rebuilt"
|
|
112
|
+
: "the SELECT list changed not only at its tail — editing or reordering columns breaks descendants' positions"
|
|
113
113
|
return { diverged, reason }
|
|
114
114
|
}
|
package/src/plan/fingerprint.ts
CHANGED
|
@@ -9,24 +9,26 @@ import type { EngineError, SqlParseError } from "../engine/adapter.ts"
|
|
|
9
9
|
import { EngineAdapter } from "../engine/adapter.ts"
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
12
|
+
* Snapshot fingerprint (SPEC §4): hash of the canonicalized AST (engine's
|
|
13
|
+
* native parser — reformatting a query does not change the fingerprint),
|
|
14
|
+
* of the metadata that affects data, and of the fingerprints of direct
|
|
15
|
+
* dependencies (transitivity).
|
|
15
16
|
*
|
|
16
|
-
* `batchSize`, `lookback`, `start`
|
|
17
|
-
*
|
|
18
|
-
*
|
|
17
|
+
* `batchSize`, `lookback`, `start` and `description` do not enter the
|
|
18
|
+
* fingerprint: they change execution or the amount of history, not the shape
|
|
19
|
+
* of the data — the interval ledger will notice missing intervals on its own.
|
|
19
20
|
*/
|
|
20
21
|
|
|
21
22
|
/**
|
|
22
|
-
*
|
|
23
|
-
*
|
|
24
|
-
*
|
|
25
|
-
*
|
|
26
|
-
*
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
*
|
|
23
|
+
* Fingerprint algorithm version — a CONTRACT (SPEC §4). The fingerprint
|
|
24
|
+
* depends on the engine canonicalization (DuckDB json_serialize_sql /
|
|
25
|
+
* libpg_query) and on the composition of the payload below: any change to
|
|
26
|
+
* them silently re-fingerprints all of a user's models and forces a full
|
|
27
|
+
* rebuild of the warehouse. Therefore: (1) canonicalization is frozen by
|
|
28
|
+
* golden tests (test/fingerprint-golden.test.ts) — a red test on a
|
|
29
|
+
* DuckDB/libpg_query upgrade means canon drift; (2) a deliberate change of
|
|
30
|
+
* algorithm = increment of this constant + a migration history; the plan does
|
|
31
|
+
* not compare snapshots of a different version, it honestly stops instead.
|
|
30
32
|
*/
|
|
31
33
|
export const FINGERPRINT_VERSION = 1
|
|
32
34
|
|
|
@@ -36,14 +38,16 @@ const sha256 = (input: string): string => {
|
|
|
36
38
|
return hasher.digest("hex")
|
|
37
39
|
}
|
|
38
40
|
|
|
39
|
-
/** Canonical
|
|
41
|
+
/** Canonical render: refs are logical names, bounds are $start/$end placeholders. */
|
|
40
42
|
export const canonicalSql = (graph: ModelGraph, name: string): string => {
|
|
41
43
|
const model = graph.models.get(name)
|
|
42
|
-
|
|
44
|
+
// invariant: callers pass a name already known to the graph (the facade
|
|
45
|
+
// guards user input with UnknownModelError before reaching here)
|
|
46
|
+
if (model === undefined) throw new Error(`invariant violated: model «${name}» is not in the graph`)
|
|
43
47
|
return render(model.fragment, { resolveRef: (ref) => ref })
|
|
44
48
|
}
|
|
45
49
|
|
|
46
|
-
/**
|
|
50
|
+
/** Part of kind that affects data. For seed — hash of the file contents: editing the data = a new version. */
|
|
47
51
|
const kindPayload = (
|
|
48
52
|
model: { readonly name: { readonly full: string } },
|
|
49
53
|
kind: ModelKind,
|
|
@@ -85,25 +89,25 @@ const kindPayload = (
|
|
|
85
89
|
|
|
86
90
|
export interface ModelVersion {
|
|
87
91
|
readonly fingerprint: string
|
|
88
|
-
/**
|
|
92
|
+
/** Canonical AST of the body; null for external. Stored in the snapshot for categorization (§5.2). */
|
|
89
93
|
readonly ast: string | null
|
|
90
94
|
}
|
|
91
95
|
|
|
92
96
|
/**
|
|
93
|
-
*
|
|
94
|
-
* json_serialize_sql
|
|
95
|
-
*
|
|
97
|
+
* Canonicalization cache (#8): a repeated plan consists almost entirely of
|
|
98
|
+
* json_serialize_sql round-trips over unchanged models. The cache is not
|
|
99
|
+
* data: get/put must be infallible (a miss/failure = recompute).
|
|
96
100
|
*/
|
|
97
101
|
export interface CanonCache {
|
|
98
102
|
readonly get: (key: string) => Effect.Effect<string | undefined>
|
|
99
103
|
readonly put: (key: string, canonical: string) => Effect.Effect<void>
|
|
100
104
|
}
|
|
101
105
|
|
|
102
|
-
/**
|
|
106
|
+
/** Cache key: algorithm version + dialect + source — a canon upgrade is not masked. */
|
|
103
107
|
export const canonCacheKey = (dialect: string, source: string): string =>
|
|
104
108
|
sha256(`${FINGERPRINT_VERSION}:${dialect}:${source}`)
|
|
105
109
|
|
|
106
|
-
/**
|
|
110
|
+
/** The model in the scope the fingerprint needs: kind, data-shape metadata, target. */
|
|
107
111
|
type FingerprintableModel = Parameters<typeof columnNames>[0] & {
|
|
108
112
|
readonly name: { readonly full: string }
|
|
109
113
|
readonly kind: ModelKind
|
|
@@ -112,11 +116,11 @@ type FingerprintableModel = Parameters<typeof columnNames>[0] & {
|
|
|
112
116
|
}
|
|
113
117
|
|
|
114
118
|
/**
|
|
115
|
-
* Fingerprint
|
|
116
|
-
* (
|
|
117
|
-
*
|
|
118
|
-
*
|
|
119
|
-
*
|
|
119
|
+
* Fingerprint of a single model from a ready AST and parent signatures
|
|
120
|
+
* (`name=fingerprint`, sorted). The planner needs it to check the
|
|
121
|
+
* "version shifted by parents ONLY" case (#5): a recompute with the old
|
|
122
|
+
* signatures must yield the old fingerprint — otherwise the metadata diverged
|
|
123
|
+
* too, and the physics cannot be reused.
|
|
120
124
|
*/
|
|
121
125
|
export const modelFingerprint = (
|
|
122
126
|
model: FingerprintableModel,
|
|
@@ -129,14 +133,14 @@ export const modelFingerprint = (
|
|
|
129
133
|
kind: yield* kindPayload(model, model.kind),
|
|
130
134
|
grain: model.grain,
|
|
131
135
|
columns: columnNames(model),
|
|
132
|
-
//
|
|
136
|
+
// a change of materialization target = new physics, consumers re-read it
|
|
133
137
|
target: model.target,
|
|
134
138
|
parents,
|
|
135
139
|
})
|
|
136
140
|
return sha256(payload)
|
|
137
141
|
})
|
|
138
142
|
|
|
139
|
-
/** Fingerprint
|
|
143
|
+
/** Fingerprint of all models in the graph; transitivity via parent hashes. */
|
|
140
144
|
export const fingerprintGraph = (
|
|
141
145
|
graph: ModelGraph,
|
|
142
146
|
cache?: CanonCache,
|
|
@@ -160,7 +164,7 @@ export const fingerprintGraph = (
|
|
|
160
164
|
const versions = new Map<string, ModelVersion>()
|
|
161
165
|
for (const name of graph.order) {
|
|
162
166
|
const model = graph.models.get(name)!
|
|
163
|
-
//
|
|
167
|
+
// external and seed have no SQL — the version is determined by source/file and schema
|
|
164
168
|
const ast =
|
|
165
169
|
model.kind._tag === "external" || model.kind._tag === "seed"
|
|
166
170
|
? null
|
package/src/plan/graph-html.ts
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
import type { ModelGraph } from "../core/graph.ts"
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
* `efmesh graph --html` (SPEC §11):
|
|
5
|
-
* SVG
|
|
6
|
-
*
|
|
4
|
+
* `efmesh graph --html` (SPEC §11): a self-contained page with the model DAG —
|
|
5
|
+
* SVG with no external dependencies, layers by the longest path from the
|
|
6
|
+
* roots, edges are Bézier curves, neighbors highlighted on hover.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
const KIND_COLOR: Record<string, string> = {
|
|
@@ -27,7 +27,7 @@ const ROW_GAP = 26
|
|
|
27
27
|
const PAD = 32
|
|
28
28
|
|
|
29
29
|
export const renderGraphHtml = (graph: ModelGraph): string => {
|
|
30
|
-
//
|
|
30
|
+
// layer = longest path from the roots: parents are always left of descendants
|
|
31
31
|
const depth = new Map<string, number>()
|
|
32
32
|
for (const name of graph.order) {
|
|
33
33
|
const model = graph.models.get(name)!
|
|
@@ -99,10 +99,10 @@ export const renderGraphHtml = (graph: ModelGraph): string => {
|
|
|
99
99
|
.join("")
|
|
100
100
|
|
|
101
101
|
return `<!doctype html>
|
|
102
|
-
<html lang="
|
|
102
|
+
<html lang="en">
|
|
103
103
|
<head>
|
|
104
104
|
<meta charset="utf-8">
|
|
105
|
-
<title>efmesh — DAG
|
|
105
|
+
<title>efmesh — model DAG</title>
|
|
106
106
|
<style>
|
|
107
107
|
:root { color-scheme: light dark; }
|
|
108
108
|
body { font: 14px/1.4 system-ui, sans-serif; margin: 0; padding: 16px;
|
|
@@ -124,7 +124,7 @@ export const renderGraphHtml = (graph: ModelGraph): string => {
|
|
|
124
124
|
</style>
|
|
125
125
|
</head>
|
|
126
126
|
<body>
|
|
127
|
-
<h1>efmesh — DAG
|
|
127
|
+
<h1>efmesh — model DAG</h1>
|
|
128
128
|
<div class="legend">${legend}</div>
|
|
129
129
|
<svg id="dag" width="${width}" height="${height}" viewBox="0 0 ${width} ${height}" xmlns="http://www.w3.org/2000/svg">
|
|
130
130
|
${edges.join("\n")}
|
package/src/plan/janitor.ts
CHANGED
|
@@ -9,32 +9,32 @@ import { janitorLockName, withStateLock, type LockHeldError, type LockOptions }
|
|
|
9
9
|
import { ducklakeAttachSql, ducklakeRef, parquetPrefix, physicalRef } from "./naming.ts"
|
|
10
10
|
|
|
11
11
|
/**
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
12
|
+
* Cleanup of orphaned physical storage (SPEC §5.4): snapshots referenced by
|
|
13
|
+
* no environment and orphaned longer than ttl ago are removed — the engine
|
|
14
|
+
* table/view, the lake parquet prefix, the snapshot record and the interval
|
|
15
|
+
* bookkeeping.
|
|
16
16
|
*
|
|
17
|
-
* ttl
|
|
18
|
-
*
|
|
19
|
-
*
|
|
20
|
-
*
|
|
21
|
-
* ttl
|
|
22
|
-
*
|
|
17
|
+
* ttl is measured from orphaned_at — the mark a promotion sets when the last
|
|
18
|
+
* reference is lost and clears on return (rolling back to an old version
|
|
19
|
+
* resets the counter). For records without the mark (never promoted — e.g.
|
|
20
|
+
* an apply that crashed before promotion) — from created_at.
|
|
21
|
+
* ttl defaults to 7 days — enough to roll back instantly by switching the
|
|
22
|
+
* view.
|
|
23
23
|
*/
|
|
24
24
|
|
|
25
25
|
export interface JanitorOptions extends LockOptions {
|
|
26
26
|
readonly ttlDays?: number
|
|
27
27
|
readonly lakePath?: string
|
|
28
|
-
/** DuckLake
|
|
28
|
+
/** DuckLake catalog (SPEC §14.5) — to also drop the fingerprint tables inside it. */
|
|
29
29
|
readonly ducklake?: { readonly catalog: string; readonly dataPath?: string }
|
|
30
|
-
/**
|
|
30
|
+
/** «Now» — injected for tests. */
|
|
31
31
|
readonly now?: number
|
|
32
32
|
}
|
|
33
33
|
|
|
34
34
|
export interface JanitorReport {
|
|
35
|
-
/**
|
|
35
|
+
/** Removed snapshots as `name@fp8`. */
|
|
36
36
|
readonly removed: ReadonlyArray<string>
|
|
37
|
-
/**
|
|
37
|
+
/** Orphaned but younger than ttl — kept until next time. */
|
|
38
38
|
readonly kept: ReadonlyArray<string>
|
|
39
39
|
}
|
|
40
40
|
|
|
@@ -53,8 +53,9 @@ export const janitor = (
|
|
|
53
53
|
const now = options?.now ?? (yield* Clock.currentTimeMillis)
|
|
54
54
|
const ttlMs = (options?.ttlDays ?? 7) * DAY_MS
|
|
55
55
|
|
|
56
|
-
//
|
|
57
|
-
//
|
|
56
|
+
// a snapshot does not store its materialization target — with a catalog
|
|
57
|
+
// configured, the table is dropped both there and in _efmesh (DROP IF EXISTS
|
|
58
|
+
// tolerates absence)
|
|
58
59
|
const ducklake = options?.ducklake
|
|
59
60
|
if (ducklake !== undefined && engine.dialect === "duckdb") {
|
|
60
61
|
yield* engine.execute(ducklakeAttachSql(ducklake))
|
|
@@ -70,10 +71,11 @@ export const janitor = (
|
|
|
70
71
|
!referenced.has(snapshot.fingerprint) &&
|
|
71
72
|
(snapshot.orphanedAt ?? snapshot.createdAt) <= deadline
|
|
72
73
|
|
|
73
|
-
//
|
|
74
|
-
//
|
|
75
|
-
//
|
|
76
|
-
//
|
|
74
|
+
// phase 1 — transactional claim of records: removal happens only if the
|
|
75
|
+
// snapshot is STILL not referenced and an orphan (the checks are atomic
|
|
76
|
+
// with the delete); against a parallel apply that resurrected the version
|
|
77
|
+
// (upsert clears orphaned_at) the claim loses — and its physical storage
|
|
78
|
+
// is left untouched (F6 race)
|
|
77
79
|
const claimed: Array<(typeof snapshots)[number]> = []
|
|
78
80
|
for (const snapshot of snapshots) {
|
|
79
81
|
if (referenced.has(snapshot.fingerprint)) continue
|
|
@@ -95,9 +97,9 @@ export const janitor = (
|
|
|
95
97
|
removed.push(label)
|
|
96
98
|
}
|
|
97
99
|
|
|
98
|
-
//
|
|
99
|
-
//
|
|
100
|
-
//
|
|
100
|
+
// phase 2 — drop physical storage by the FRESH store state: storage is
|
|
101
|
+
// shared between versions (forward-only) and dropped only if, after the
|
|
102
|
+
// claims, no surviving snapshot uses it
|
|
101
103
|
const survivors = yield* store.listSnapshots()
|
|
102
104
|
const physicalInUse = new Set(survivors.map((snapshot) => snapshot.physicalFp))
|
|
103
105
|
const dropped = new Set<string>()
|
|
@@ -106,6 +108,9 @@ export const janitor = (
|
|
|
106
108
|
dropped.add(snapshot.physicalFp)
|
|
107
109
|
const name = parseModelName(snapshot.name)
|
|
108
110
|
const target = physicalRef(name, snapshot.physicalFp)
|
|
111
|
+
yield* Effect.logDebug("dropping orphaned physics").pipe(
|
|
112
|
+
Effect.annotateLogs({ model: snapshot.name, physical: snapshot.physicalFp.slice(0, 8) }),
|
|
113
|
+
)
|
|
109
114
|
yield* engine.execute(
|
|
110
115
|
snapshot.kind === "view"
|
|
111
116
|
? `DROP VIEW IF EXISTS ${target}`
|
|
@@ -124,7 +129,7 @@ export const janitor = (
|
|
|
124
129
|
|
|
125
130
|
return { removed, kept }
|
|
126
131
|
}).pipe(
|
|
127
|
-
//
|
|
128
|
-
//
|
|
132
|
+
// two janitors from different processes must not race to remove the same
|
|
133
|
+
// thing; the janitor↔apply race is guarded by ttl (the window for an instant rollback)
|
|
129
134
|
withStateLock(janitorLockName, options?.lockTtlMs),
|
|
130
135
|
)
|
package/src/plan/lineage.ts
CHANGED
|
@@ -6,28 +6,32 @@ import { EngineAdapter } from "../engine/adapter.ts"
|
|
|
6
6
|
import { canonicalSql } from "./fingerprint.ts"
|
|
7
7
|
|
|
8
8
|
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
9
|
+
* Column-level lineage (SPEC §9.4): the chain from a model's column down to
|
|
10
|
+
* the raw columns of external/seed models. Precision is best-effort — a
|
|
11
|
+
* column's expression is taken from the canonical AST (the engine's native
|
|
12
|
+
* parser), column references are matched to parents' schemas by name,
|
|
13
|
+
* qualifiers and CTE aliases are not resolved. The model graph itself is
|
|
14
|
+
* always exact: dependencies are known from `ctx.ref`, not from text parsing.
|
|
15
15
|
*/
|
|
16
16
|
|
|
17
17
|
export class LineageError extends Data.TaggedError("LineageError")<{
|
|
18
18
|
readonly model: string
|
|
19
19
|
readonly reason: string
|
|
20
|
-
}> {
|
|
20
|
+
}> {
|
|
21
|
+
override get message(): string {
|
|
22
|
+
return `lineage «${this.model}»: ${this.reason}`
|
|
23
|
+
}
|
|
24
|
+
}
|
|
21
25
|
|
|
22
26
|
export interface LineageNode {
|
|
23
27
|
readonly model: string
|
|
24
28
|
readonly column: string
|
|
25
|
-
/**
|
|
29
|
+
/** Model kind — external/seed are leaves (raw sources). */
|
|
26
30
|
readonly kind: string
|
|
27
31
|
readonly sources: ReadonlyArray<LineageNode>
|
|
28
32
|
}
|
|
29
33
|
|
|
30
|
-
/**
|
|
34
|
+
/** All COLUMN_REFs in the expression subtree — column names without qualifiers. */
|
|
31
35
|
const collectColumnRefs = (node: unknown, out: Set<string>): void => {
|
|
32
36
|
if (Array.isArray(node)) {
|
|
33
37
|
for (const item of node) collectColumnRefs(item, out)
|
|
@@ -49,9 +53,9 @@ const selectItems = (ast: unknown): ReadonlyArray<Record<string, unknown>> => {
|
|
|
49
53
|
}
|
|
50
54
|
|
|
51
55
|
/**
|
|
52
|
-
*
|
|
53
|
-
*
|
|
54
|
-
* undefined —
|
|
56
|
+
* The columns `column` depends on in select_list: an aliased expression or a
|
|
57
|
+
* same-named COLUMN_REF; `SELECT *` — a pass-through of the name.
|
|
58
|
+
* undefined — the column's expression was not found (engine sugar) — best-effort.
|
|
55
59
|
*/
|
|
56
60
|
const sourceColumnsOf = (ast: unknown, column: string): ReadonlySet<string> | undefined => {
|
|
57
61
|
const items = selectItems(ast)
|
|
@@ -83,12 +87,12 @@ export const lineage = (
|
|
|
83
87
|
const engine = yield* EngineAdapter
|
|
84
88
|
const root = graph.models.get(modelName)
|
|
85
89
|
if (root === undefined) {
|
|
86
|
-
return yield* new LineageError({ model: modelName, reason: "
|
|
90
|
+
return yield* new LineageError({ model: modelName, reason: "model is not in the project" })
|
|
87
91
|
}
|
|
88
92
|
if (!(column in root.schema.fields)) {
|
|
89
93
|
return yield* new LineageError({
|
|
90
94
|
model: modelName,
|
|
91
|
-
reason:
|
|
95
|
+
reason: `column «${column}» is not in the schema`,
|
|
92
96
|
})
|
|
93
97
|
}
|
|
94
98
|
|
|
@@ -115,7 +119,7 @@ export const lineage = (
|
|
|
115
119
|
kind: model.kind._tag,
|
|
116
120
|
sources: [],
|
|
117
121
|
}
|
|
118
|
-
//
|
|
122
|
+
// raw source: beyond here the chain hits the outside world
|
|
119
123
|
if (model.kind._tag === "external" || model.kind._tag === "seed" || model.deps.size === 0) {
|
|
120
124
|
return leaf
|
|
121
125
|
}
|
|
@@ -134,7 +138,7 @@ export const lineage = (
|
|
|
134
138
|
return yield* trace(root, column)
|
|
135
139
|
})
|
|
136
140
|
|
|
137
|
-
/**
|
|
141
|
+
/** Flat printout of the lineage tree for the CLI. */
|
|
138
142
|
export const formatLineage = (node: LineageNode, indent = ""): ReadonlyArray<string> => {
|
|
139
143
|
const marker =
|
|
140
144
|
node.kind === "external" || node.kind === "seed" ? ` [${node.kind}]` : ""
|
package/src/plan/lock.ts
CHANGED
|
@@ -2,26 +2,30 @@ import { Data, Effect } from "effect"
|
|
|
2
2
|
import { StateStore, type StateError } from "../state/store.ts"
|
|
3
3
|
|
|
4
4
|
/**
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
5
|
+
* Cross-process lock via the state store (SPEC §7, §14.6): environment
|
|
6
|
+
* mutations — apply and run — go under ONE lock `env:<name>`, so parallel
|
|
7
|
+
* apply+apply and apply+run from different processes are cut off. A stale
|
|
8
|
+
* lock of a crashed process is reclaimed by ttl (the same-millisecond race is
|
|
9
|
+
* accounted for: expires_at <= now).
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
export class LockHeldError extends Data.TaggedError("LockHeldError")<{
|
|
13
13
|
readonly name: string
|
|
14
|
-
}> {
|
|
14
|
+
}> {
|
|
15
|
+
override get message(): string {
|
|
16
|
+
return `lock «${this.name}» is held by another apply/run process`
|
|
17
|
+
}
|
|
18
|
+
}
|
|
15
19
|
|
|
16
20
|
export interface LockOptions {
|
|
17
|
-
/**
|
|
21
|
+
/** How long the lock lives without being released (crashed process); by default 1 hour. */
|
|
18
22
|
readonly lockTtlMs?: number
|
|
19
23
|
}
|
|
20
24
|
|
|
21
|
-
/**
|
|
25
|
+
/** Name of the lock under which an environment is mutated (shared by apply and run). */
|
|
22
26
|
export const envLockName = (env: string): string => `env:${env}`
|
|
23
27
|
|
|
24
|
-
/**
|
|
28
|
+
/** The janitor lock is global: physical-storage cleanup is not tied to an environment. */
|
|
25
29
|
export const janitorLockName = "janitor"
|
|
26
30
|
|
|
27
31
|
export const withStateLock =
|
|
@@ -33,5 +37,18 @@ export const withStateLock =
|
|
|
33
37
|
const store = yield* StateStore
|
|
34
38
|
const acquired = yield* store.acquireLock(name, ttlMs ?? 3_600_000)
|
|
35
39
|
if (!acquired) return yield* new LockHeldError({ name })
|
|
36
|
-
|
|
40
|
+
// lock lifecycle is an internal detail — DEBUG only, keyed by lock name (#14)
|
|
41
|
+
yield* Effect.logDebug("lock acquired").pipe(Effect.annotateLogs("lock", name))
|
|
42
|
+
return yield* effect.pipe(
|
|
43
|
+
Effect.ensuring(
|
|
44
|
+
store
|
|
45
|
+
.releaseLock(name)
|
|
46
|
+
.pipe(
|
|
47
|
+
Effect.andThen(
|
|
48
|
+
Effect.logDebug("lock released").pipe(Effect.annotateLogs("lock", name)),
|
|
49
|
+
),
|
|
50
|
+
Effect.ignore,
|
|
51
|
+
),
|
|
52
|
+
),
|
|
53
|
+
)
|
|
37
54
|
})
|
package/src/plan/metrics.ts
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
import { Metric } from "effect"
|
|
2
2
|
|
|
3
|
-
/**
|
|
3
|
+
/** Observability out of the box (SPEC §10): pipeline counters. */
|
|
4
4
|
export const intervalsDone = Metric.counter("efmesh_intervals_done_total", {
|
|
5
|
-
description: "
|
|
5
|
+
description: "how many intervals were computed and marked done",
|
|
6
6
|
})
|
|
7
7
|
|
|
8
8
|
export const auditFailuresTotal = Metric.counter("efmesh_audit_failures_total", {
|
|
9
|
-
description: "
|
|
9
|
+
description: "how many blocking audits failed",
|
|
10
10
|
})
|
|
11
11
|
|
|
12
12
|
export const snapshotsBuilt = Metric.counter("efmesh_snapshots_built_total", {
|
|
13
|
-
description: "
|
|
13
|
+
description: "how many snapshots were built (physics or backfill)",
|
|
14
14
|
})
|