@avytheone/efmesh 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +22 -0
- package/README.ru.md +23 -0
- package/SPEC.md +1 -1
- package/package.json +1 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +204 -124
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +26 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +12 -11
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +7 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +33 -29
- package/src/plan/executor.ts +207 -125
- package/src/plan/explain.ts +29 -29
- package/src/plan/fingerprint.ts +35 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +107 -90
- package/src/plan/run.ts +27 -19
- package/src/plan/schedule.ts +46 -41
- package/src/plan/status.ts +15 -14
- package/src/state/postgres.ts +19 -19
- package/src/state/sqlite.ts +27 -27
- package/src/state/store.ts +67 -55
- package/src/testing/index.ts +27 -26
package/src/config.ts
CHANGED
|
@@ -1,43 +1,43 @@
|
|
|
1
1
|
import type { AnyModel } from "./core/model.ts"
|
|
2
2
|
|
|
3
3
|
/**
|
|
4
|
-
*
|
|
5
|
-
*
|
|
4
|
+
* Project config — `efmesh.config.ts` (SPEC §11): a typed TS module, no
|
|
5
|
+
* YAML. The CLI imports it and assembles the engine and state layers.
|
|
6
6
|
*/
|
|
7
7
|
export interface EfmeshConfig {
|
|
8
|
-
/**
|
|
8
|
+
/** Models by value; can be combined with discovery — a duplicate name is an error. */
|
|
9
9
|
readonly models?: ReadonlyArray<AnyModel>
|
|
10
10
|
/**
|
|
11
|
-
* Glob
|
|
12
|
-
*
|
|
11
|
+
* Glob masks for model files, relative to the config (SPEC §12):
|
|
12
|
+
* every model export found in matched files joins the project.
|
|
13
13
|
*/
|
|
14
14
|
readonly discovery?: string | ReadonlyArray<string>
|
|
15
15
|
readonly engine?: {
|
|
16
|
-
/**
|
|
16
|
+
/** Path to the DuckDB file; defaults to `efmesh.duckdb` next to the config. */
|
|
17
17
|
readonly path?: string
|
|
18
|
-
/** postgres://… —
|
|
18
|
+
/** postgres://… — switches the engine to Postgres (F3); path is ignored. */
|
|
19
19
|
readonly url?: string
|
|
20
|
-
/**
|
|
20
|
+
/** Postgres connection pool size — backfill parallelism (SPEC §5.3). */
|
|
21
21
|
readonly max?: number
|
|
22
22
|
}
|
|
23
23
|
readonly state?: {
|
|
24
|
-
/**
|
|
24
|
+
/** Path to the SQLite state file; defaults to `efmesh.state.sqlite`. */
|
|
25
25
|
readonly path?: string
|
|
26
|
-
/** postgres://… —
|
|
26
|
+
/** postgres://… — state lives in Postgres (schema efmesh_state, SPEC §6). */
|
|
27
27
|
readonly url?: string
|
|
28
28
|
}
|
|
29
29
|
readonly lake?: {
|
|
30
|
-
/**
|
|
30
|
+
/** Root of the parquet lake (SPEC §3.3) — a local directory or s3://…. */
|
|
31
31
|
readonly path: string
|
|
32
32
|
}
|
|
33
|
-
/** DuckLake
|
|
33
|
+
/** DuckLake catalog for target: "ducklake" (SPEC §14.5). DuckDB-only. */
|
|
34
34
|
readonly ducklake?: {
|
|
35
|
-
/**
|
|
35
|
+
/** Path to the DuckLake catalog's SQLite file. */
|
|
36
36
|
readonly catalog: string
|
|
37
|
-
/**
|
|
37
|
+
/** Where DuckLake writes parquet data; defaults to next to the catalog. */
|
|
38
38
|
readonly dataPath?: string
|
|
39
39
|
}
|
|
40
|
-
/** ATTACH
|
|
40
|
+
/** ATTACH databases by alias (SPEC §9.3): url + options (`TYPE postgres` etc.). */
|
|
41
41
|
readonly attach?: Readonly<Record<string, { readonly url: string; readonly options?: string }>>
|
|
42
42
|
}
|
|
43
43
|
|
package/src/core/audit.ts
CHANGED
|
@@ -2,26 +2,30 @@ import { Data, type Schema } from "effect"
|
|
|
2
2
|
import type { SelfValue, SqlFragment, SqlLiteral } from "./sql.ts"
|
|
3
3
|
import { quoteIdent, sql } from "./sql.ts"
|
|
4
4
|
|
|
5
|
-
/**
|
|
5
|
+
/** A blocking audit found violations: the snapshot/interval is considered unfit. */
|
|
6
6
|
export class AuditFailure extends Data.TaggedError("AuditFailure")<{
|
|
7
7
|
readonly model: string
|
|
8
8
|
readonly audit: string
|
|
9
9
|
readonly violations: number
|
|
10
|
-
}> {
|
|
10
|
+
}> {
|
|
11
|
+
override get message(): string {
|
|
12
|
+
return `audit ${this.audit} on model «${this.model}»: ${this.violations} violating row(s)`
|
|
13
|
+
}
|
|
14
|
+
}
|
|
11
15
|
|
|
12
16
|
/**
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
+
* Audit (SPEC §8) — a SQL predicate over a model's result: the query returns
|
|
18
|
+
* the VIOLATING rows, a non-empty result means failure. `ctx.self` renders
|
|
19
|
+
* to the snapshot's physical table (for incremental, to a subquery of the
|
|
20
|
+
* processed interval — the audit checks what was just loaded, not the whole history).
|
|
17
21
|
*
|
|
18
|
-
* blocking
|
|
19
|
-
* view
|
|
22
|
+
* A blocking audit (the default) fails apply: the interval is marked failed,
|
|
23
|
+
* the view is not promoted. warn — logs and lets the pipeline continue.
|
|
20
24
|
*/
|
|
21
25
|
export interface Audit {
|
|
22
26
|
readonly name: string
|
|
23
27
|
readonly blocking: boolean
|
|
24
|
-
/**
|
|
28
|
+
/** The violations query; contains a Self node. */
|
|
25
29
|
readonly fragment: SqlFragment
|
|
26
30
|
}
|
|
27
31
|
|
|
@@ -32,7 +36,7 @@ export interface AuditCtx {
|
|
|
32
36
|
|
|
33
37
|
const SELF: SelfValue = { _tag: "SelfValue" }
|
|
34
38
|
|
|
35
|
-
/**
|
|
39
|
+
/** Builders are typed by the model's columns via the Fields type parameter. */
|
|
36
40
|
export const audit = {
|
|
37
41
|
notNull: <Fields extends Schema.Struct.Fields>(
|
|
38
42
|
column: Extract<keyof Fields, string>,
|
|
@@ -72,7 +76,7 @@ export const audit = {
|
|
|
72
76
|
fragment: body({ sql, self: SELF }),
|
|
73
77
|
}),
|
|
74
78
|
|
|
75
|
-
/**
|
|
79
|
+
/** Downgrade an audit to a warning: log instead of failing. */
|
|
76
80
|
warn: (base: Audit): Audit => ({ ...base, blocking: false }),
|
|
77
81
|
} as const
|
|
78
82
|
|
package/src/core/errors.ts
CHANGED
|
@@ -1,27 +1,57 @@
|
|
|
1
1
|
import { Data } from "effect"
|
|
2
|
+
import { causeText } from "../error-text.ts"
|
|
2
3
|
|
|
3
|
-
/**
|
|
4
|
+
/** Invalid model configuration: a malformed name, a self-ref, etc. Thrown at module load time. */
|
|
4
5
|
export class ModelDefinitionError extends Data.TaggedError("ModelDefinitionError")<{
|
|
5
6
|
readonly model: string
|
|
6
7
|
readonly reason: string
|
|
7
|
-
}> {
|
|
8
|
+
}> {
|
|
9
|
+
override get message(): string {
|
|
10
|
+
return `model «${this.model}»: ${this.reason}`
|
|
11
|
+
}
|
|
12
|
+
}
|
|
8
13
|
|
|
9
14
|
export class DuplicateModelError extends Data.TaggedError("DuplicateModelError")<{
|
|
10
15
|
readonly name: string
|
|
11
|
-
}> {
|
|
16
|
+
}> {
|
|
17
|
+
override get message(): string {
|
|
18
|
+
return `model «${this.name}» is defined twice — names must be unique across the project`
|
|
19
|
+
}
|
|
20
|
+
}
|
|
12
21
|
|
|
13
22
|
export class UnknownDependencyError extends Data.TaggedError("UnknownDependencyError")<{
|
|
14
23
|
readonly model: string
|
|
15
24
|
readonly dependency: string
|
|
16
|
-
}> {
|
|
25
|
+
}> {
|
|
26
|
+
override get message(): string {
|
|
27
|
+
return `model «${this.model}» depends on «${this.dependency}», which is not in the project`
|
|
28
|
+
}
|
|
29
|
+
}
|
|
17
30
|
|
|
18
31
|
export class DagCycleError extends Data.TaggedError("DagCycleError")<{
|
|
19
32
|
readonly cycle: ReadonlyArray<string>
|
|
20
|
-
}> {
|
|
33
|
+
}> {
|
|
34
|
+
override get message(): string {
|
|
35
|
+
return `dependency cycle: ${this.cycle.join(" → ")}`
|
|
36
|
+
}
|
|
37
|
+
}
|
|
21
38
|
|
|
22
|
-
/**
|
|
39
|
+
/** A seed model's file can't be read — fingerprint and assembly are impossible. */
|
|
23
40
|
export class SeedReadError extends Data.TaggedError("SeedReadError")<{
|
|
24
41
|
readonly model: string
|
|
25
42
|
readonly file: string
|
|
26
43
|
readonly cause: unknown
|
|
27
|
-
}> {
|
|
44
|
+
}> {
|
|
45
|
+
override get message(): string {
|
|
46
|
+
return `seed «${this.model}»: cannot read ${this.file} — ${causeText(this.cause)}`
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
/** A model name handed to the facade/CLI (render, lineage) is not in the project. */
|
|
51
|
+
export class UnknownModelError extends Data.TaggedError("UnknownModelError")<{
|
|
52
|
+
readonly model: string
|
|
53
|
+
}> {
|
|
54
|
+
override get message(): string {
|
|
55
|
+
return `model «${this.model}» is not in the project`
|
|
56
|
+
}
|
|
57
|
+
}
|
package/src/core/graph.ts
CHANGED
|
@@ -4,15 +4,15 @@ import type { AnyModel } from "./model.ts"
|
|
|
4
4
|
|
|
5
5
|
export interface ModelGraph {
|
|
6
6
|
readonly models: ReadonlyMap<string, AnyModel>
|
|
7
|
-
/**
|
|
7
|
+
/** Topological order: parents before children. */
|
|
8
8
|
readonly order: ReadonlyArray<string>
|
|
9
|
-
/**
|
|
9
|
+
/** Direct dependents: who references this model. */
|
|
10
10
|
readonly dependents: ReadonlyMap<string, ReadonlySet<string>>
|
|
11
11
|
}
|
|
12
12
|
|
|
13
13
|
export type GraphError = DuplicateModelError | UnknownDependencyError | DagCycleError
|
|
14
14
|
|
|
15
|
-
/**
|
|
15
|
+
/** Builds a DAG from a set of models: duplicates, unknown dependencies, cycles — typed errors. */
|
|
16
16
|
export const buildGraph = (
|
|
17
17
|
input: Iterable<AnyModel>,
|
|
18
18
|
): Effect.Effect<ModelGraph, GraphError> =>
|
|
@@ -36,10 +36,10 @@ export const buildGraph = (
|
|
|
36
36
|
}
|
|
37
37
|
}
|
|
38
38
|
|
|
39
|
-
//
|
|
39
|
+
// Kahn's algorithm: count in-degrees (number of dependencies), drain the zeros.
|
|
40
40
|
const inDegree = new Map<string, number>()
|
|
41
41
|
for (const [name, model] of models) inDegree.set(name, model.deps.size)
|
|
42
|
-
//
|
|
42
|
+
// sort the queue so the order is deterministic across runs
|
|
43
43
|
const queue = [...inDegree.entries()].filter(([, d]) => d === 0).map(([n]) => n).sort()
|
|
44
44
|
const order: Array<string> = []
|
|
45
45
|
while (queue.length > 0) {
|
|
@@ -60,7 +60,7 @@ export const buildGraph = (
|
|
|
60
60
|
return { models, order, dependents }
|
|
61
61
|
})
|
|
62
62
|
|
|
63
|
-
/**
|
|
63
|
+
/** All transitive dependents of a model (for the breaking cascade in a plan). */
|
|
64
64
|
export const transitiveDependents = (graph: ModelGraph, name: string): ReadonlySet<string> => {
|
|
65
65
|
const out = new Set<string>()
|
|
66
66
|
const stack = [name]
|
package/src/core/interval.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* Interval arithmetic (SPEC §2, §5.3): half-open `[start, end)` UTC time
|
|
3
|
+
* intervals, aligned to the model's grain. Tracking which intervals are
|
|
4
|
+
* filled is the basis of incrementality and backfill; this module is pure
|
|
5
|
+
* math only, state lives in the state store.
|
|
6
6
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
7
|
+
* Internal representation is epoch millis: day/hour grains in UTC don't
|
|
8
|
+
* depend on the calendar, and floor division gives alignment.
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
export type IntervalUnit = "day" | "hour"
|
|
@@ -26,9 +26,9 @@ export const floorTo = (unit: IntervalUnit, ms: number): number => {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
/**
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* (`end <= floor(now)` —
|
|
29
|
+
* All completed grain intervals from `startMs` to `nowMs`:
|
|
30
|
+
* the start is floor-aligned, and the last interval is one that has
|
|
31
|
+
* already ended (`end <= floor(now)` — an unfinished "today" doesn't count).
|
|
32
32
|
*/
|
|
33
33
|
export const enumerateIntervals = (
|
|
34
34
|
unit: IntervalUnit,
|
|
@@ -45,7 +45,7 @@ export const enumerateIntervals = (
|
|
|
45
45
|
return intervals
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
-
/** `wanted`
|
|
48
|
+
/** `wanted` minus `covered` (compared by `start`; intervals of the same grain). */
|
|
49
49
|
export const missingIntervals = (
|
|
50
50
|
wanted: ReadonlyArray<Interval>,
|
|
51
51
|
covered: ReadonlyArray<Interval>,
|
|
@@ -54,7 +54,7 @@ export const missingIntervals = (
|
|
|
54
54
|
return wanted.filter((i) => !done.has(i.start))
|
|
55
55
|
}
|
|
56
56
|
|
|
57
|
-
/**
|
|
57
|
+
/** Adjacent intervals merge into contiguous ranges (input must be sorted). */
|
|
58
58
|
export const mergeIntervals = (
|
|
59
59
|
intervals: ReadonlyArray<Interval>,
|
|
60
60
|
): ReadonlyArray<Interval> => {
|
|
@@ -71,8 +71,8 @@ export const mergeIntervals = (
|
|
|
71
71
|
}
|
|
72
72
|
|
|
73
73
|
/**
|
|
74
|
-
*
|
|
75
|
-
*
|
|
74
|
+
* Slices a range into batches no longer than `batchSize` grain intervals.
|
|
75
|
+
* A batch is the unit of execution (one DELETE+INSERT); an interval is the unit of tracking.
|
|
76
76
|
*/
|
|
77
77
|
export const splitIntoBatches = (
|
|
78
78
|
range: Interval,
|
|
@@ -80,7 +80,7 @@ export const splitIntoBatches = (
|
|
|
80
80
|
batchSize: number,
|
|
81
81
|
): ReadonlyArray<Interval> => {
|
|
82
82
|
if (!Number.isInteger(batchSize) || batchSize < 1) {
|
|
83
|
-
throw new RangeError(`batchSize
|
|
83
|
+
throw new RangeError(`batchSize must be an integer ≥ 1, got ${batchSize}`)
|
|
84
84
|
}
|
|
85
85
|
const step = unitMillis[unit] * batchSize
|
|
86
86
|
const batches: Array<Interval> = []
|
|
@@ -90,7 +90,7 @@ export const splitIntoBatches = (
|
|
|
90
90
|
return batches
|
|
91
91
|
}
|
|
92
92
|
|
|
93
|
-
/**
|
|
93
|
+
/** Grain intervals within a range (for marking each one done after a batch). */
|
|
94
94
|
export const intervalsWithin = (
|
|
95
95
|
range: Interval,
|
|
96
96
|
unit: IntervalUnit,
|
|
@@ -103,16 +103,16 @@ export const intervalsWithin = (
|
|
|
103
103
|
return intervals
|
|
104
104
|
}
|
|
105
105
|
|
|
106
|
-
/** ISO-8601 UTC —
|
|
106
|
+
/** ISO-8601 UTC — the storage format for bounds in the state store (sorts lexicographically). */
|
|
107
107
|
export const toIso = (ms: number): string => new Date(ms).toISOString()
|
|
108
108
|
|
|
109
109
|
export const fromIso = (iso: string): number => {
|
|
110
110
|
const ms = Date.parse(iso)
|
|
111
|
-
if (Number.isNaN(ms)) throw new RangeError(
|
|
111
|
+
if (Number.isNaN(ms)) throw new RangeError(`not an ISO time: ${iso}`)
|
|
112
112
|
return ms
|
|
113
113
|
}
|
|
114
114
|
|
|
115
|
-
/**
|
|
115
|
+
/** Literal for substituting an interval bound into DuckDB SQL. */
|
|
116
116
|
export const sqlTimestamp = (ms: number): string => {
|
|
117
117
|
const iso = new Date(ms).toISOString()
|
|
118
118
|
return `TIMESTAMP '${iso.slice(0, 10)} ${iso.slice(11, 19)}'`
|