@avytheone/efmesh 0.1.0-beta.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -92
- package/README.md +83 -9
- package/README.ru.md +80 -5
- package/SPEC.md +542 -459
- package/package.json +4 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +505 -85
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +28 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +44 -13
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +10 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +228 -4
- package/src/plan/executor.ts +223 -121
- package/src/plan/explain.ts +114 -0
- package/src/plan/fingerprint.ts +84 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +238 -52
- package/src/plan/run.ts +72 -26
- package/src/plan/schedule.ts +217 -0
- package/src/plan/status.ts +96 -0
- package/src/state/postgres.ts +68 -19
- package/src/state/sqlite.ts +74 -27
- package/src/state/store.ts +88 -45
- package/src/testing/index.ts +27 -26
package/src/efmesh.ts
CHANGED
|
@@ -16,22 +16,24 @@ import {
|
|
|
16
16
|
planChanges,
|
|
17
17
|
type FingerprintVersionError,
|
|
18
18
|
type ForwardOnlyError,
|
|
19
|
+
type ReclassifyError,
|
|
19
20
|
type InvalidEnvironmentError,
|
|
20
21
|
type Plan,
|
|
21
22
|
type PlanOptions,
|
|
22
23
|
} from "./plan/planner.ts"
|
|
23
24
|
import type { SeedReadError } from "./core/errors.ts"
|
|
25
|
+
import { UnknownModelError } from "./core/errors.ts"
|
|
24
26
|
import type { GraphError } from "./core/graph.ts"
|
|
25
27
|
import type { StateError } from "./state/store.ts"
|
|
26
28
|
import { StateStore } from "./state/store.ts"
|
|
27
29
|
import { externalSourceRef, viewRef } from "./plan/naming.ts"
|
|
28
30
|
|
|
29
31
|
/**
|
|
30
|
-
*
|
|
31
|
-
*
|
|
32
|
+
* The efmesh facade (SPEC §10): plain Effects embeddable in any
|
|
33
|
+
* application; the CLI is a thin wrapper over them.
|
|
32
34
|
*/
|
|
33
35
|
export const Efmesh = {
|
|
34
|
-
/**
|
|
36
|
+
/** Compute a plan for an environment without changing anything. The engine is needed to canonicalize SQL. */
|
|
35
37
|
plan: (
|
|
36
38
|
env: string,
|
|
37
39
|
models: Iterable<AnyModel>,
|
|
@@ -42,6 +44,7 @@ export const Efmesh = {
|
|
|
42
44
|
| StateError
|
|
43
45
|
| InvalidEnvironmentError
|
|
44
46
|
| ForwardOnlyError
|
|
47
|
+
| ReclassifyError
|
|
45
48
|
| FingerprintVersionError
|
|
46
49
|
| EngineError
|
|
47
50
|
| SqlParseError
|
|
@@ -50,9 +53,10 @@ export const Efmesh = {
|
|
|
50
53
|
> => buildGraph(models).pipe(Effect.flatMap((graph) => planChanges(env, graph, options))),
|
|
51
54
|
|
|
52
55
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
* run:
|
|
56
|
+
* Plan + apply: physics, interval backfill, view layer, state.
|
|
57
|
+
* Runs under the cross-process lock `env:<name>` (SPEC §14.6) — the same
|
|
58
|
+
* one `run` uses: concurrent environment mutations from different
|
|
59
|
+
* processes are cut off.
|
|
56
60
|
*/
|
|
57
61
|
apply: (
|
|
58
62
|
env: string,
|
|
@@ -65,21 +69,30 @@ export const Efmesh = {
|
|
|
65
69
|
return yield* applyPlan(plan, graph, options)
|
|
66
70
|
}).pipe(withStateLock(envLockName(env), options?.lockTtlMs)),
|
|
67
71
|
|
|
68
|
-
/** Canonical
|
|
69
|
-
render: (
|
|
70
|
-
|
|
72
|
+
/** Canonical SQL render of a model (refs are logical names), for debugging. */
|
|
73
|
+
render: (
|
|
74
|
+
models: Iterable<AnyModel>,
|
|
75
|
+
name: string,
|
|
76
|
+
): Effect.Effect<string, GraphError | UnknownModelError> =>
|
|
77
|
+
buildGraph(models).pipe(
|
|
78
|
+
Effect.flatMap((graph) =>
|
|
79
|
+
graph.models.has(name)
|
|
80
|
+
? Effect.succeed(canonicalSql(graph, name))
|
|
81
|
+
: new UnknownModelError({ model: name }),
|
|
82
|
+
),
|
|
83
|
+
),
|
|
71
84
|
|
|
72
|
-
/**
|
|
85
|
+
/** SQL render of a model against an environment's view layer — "as the engine would run it". */
|
|
73
86
|
renderFor: (
|
|
74
87
|
models: Iterable<AnyModel>,
|
|
75
88
|
name: string,
|
|
76
89
|
env: string,
|
|
77
|
-
): Effect.Effect<string, GraphError> =>
|
|
90
|
+
): Effect.Effect<string, GraphError | UnknownModelError> =>
|
|
78
91
|
buildGraph(models).pipe(
|
|
79
|
-
Effect.
|
|
92
|
+
Effect.flatMap((graph) => {
|
|
80
93
|
const model = graph.models.get(name)
|
|
81
|
-
if (model === undefined)
|
|
82
|
-
//
|
|
94
|
+
if (model === undefined) return new UnknownModelError({ model: name })
|
|
95
|
+
// external and embedded have no view layer: source as-is / subquery
|
|
83
96
|
const resolve = (ref: string): string => {
|
|
84
97
|
const source = graph.models.get(ref)!
|
|
85
98
|
if (source.kind._tag === "external") return externalSourceRef(source.kind.source)
|
|
@@ -88,7 +101,7 @@ export const Efmesh = {
|
|
|
88
101
|
}
|
|
89
102
|
return viewRef(env, source.name)
|
|
90
103
|
}
|
|
91
|
-
return render(model.fragment, { resolveRef: resolve })
|
|
104
|
+
return Effect.succeed(render(model.fragment, { resolveRef: resolve }))
|
|
92
105
|
}),
|
|
93
106
|
),
|
|
94
107
|
} as const
|
package/src/engine/adapter.ts
CHANGED
|
@@ -1,13 +1,30 @@
|
|
|
1
1
|
import { Context, Data, Effect } from "effect"
|
|
2
|
+
import { causeText, sqlSnippet } from "../error-text.ts"
|
|
2
3
|
|
|
4
|
+
/**
|
|
5
|
+
* The engine rejected a statement (SPEC §9.1). `cause` is the engine's own
|
|
6
|
+
* error (DuckDB/Postgres) — never swallowed; `sql` is the statement that
|
|
7
|
+
* failed; `model` names the culprit when the failure happened while building a
|
|
8
|
+
* specific model (attached by the executor). The `message` is derived from all
|
|
9
|
+
* three so an operator or agent sees what the engine said and where, not an
|
|
10
|
+
* empty `EngineError:` (#13).
|
|
11
|
+
*/
|
|
3
12
|
export class EngineError extends Data.TaggedError("EngineError")<{
|
|
4
13
|
readonly sql: string
|
|
5
14
|
readonly cause: unknown
|
|
6
|
-
|
|
15
|
+
/** The model being built when the engine failed, when known. */
|
|
16
|
+
readonly model?: string
|
|
17
|
+
}> {
|
|
18
|
+
override get message(): string {
|
|
19
|
+
const where = this.model !== undefined ? `[model ${this.model}] ` : ""
|
|
20
|
+
return `${where}${causeText(this.cause)} — while running SQL: ${sqlSnippet(this.sql)}`
|
|
21
|
+
}
|
|
22
|
+
}
|
|
7
23
|
|
|
8
|
-
/**
|
|
24
|
+
/** The engine could not parse the model's SQL (SPEC §9.2). */
|
|
9
25
|
export class SqlParseError extends Data.TaggedError("SqlParseError")<{
|
|
10
26
|
readonly sql: string
|
|
27
|
+
/** The engine parser's own message; also this error's rendered `message`. */
|
|
11
28
|
readonly message: string
|
|
12
29
|
}> {}
|
|
13
30
|
|
|
@@ -17,9 +34,9 @@ export interface EngineColumn {
|
|
|
17
34
|
}
|
|
18
35
|
|
|
19
36
|
/**
|
|
20
|
-
*
|
|
21
|
-
*
|
|
22
|
-
*
|
|
37
|
+
* Engine adapter (SPEC §9.1). A minimal surface: DDL helpers live separately
|
|
38
|
+
* (executor) and go through execute — the adapter has no need to know about
|
|
39
|
+
* the physical/virtual layer.
|
|
23
40
|
*/
|
|
24
41
|
export type Dialect = "duckdb" | "postgres"
|
|
25
42
|
|
|
@@ -30,19 +47,19 @@ export interface Engine {
|
|
|
30
47
|
) => Effect.Effect<ReadonlyArray<Record<string, unknown>>, EngineError>
|
|
31
48
|
readonly execute: (sql: string) => Effect.Effect<void, EngineError>
|
|
32
49
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
* (Postgres)
|
|
50
|
+
* A set of statements in one engine transaction, rolled back on any error.
|
|
51
|
+
* An adapter primitive rather than BEGIN/COMMIT via execute: on a connection
|
|
52
|
+
* pool (Postgres) separate calls would scatter across different connections.
|
|
36
53
|
*/
|
|
37
54
|
readonly transaction: (
|
|
38
55
|
statements: ReadonlyArray<string>,
|
|
39
56
|
) => Effect.Effect<void, EngineError>
|
|
40
|
-
/**
|
|
57
|
+
/** Names and types of a query's columns without executing it (schema contract, SPEC §3.2). */
|
|
41
58
|
readonly describe: (sql: string) => Effect.Effect<ReadonlyArray<EngineColumn>, EngineError>
|
|
42
59
|
/**
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
60
|
+
* The canonical form of a SELECT query for the fingerprint (SPEC §4, §9.2):
|
|
61
|
+
* parsing by the engine's native parser, normalization, deterministic
|
|
62
|
+
* serialization. Reformatting the text does not change the result.
|
|
46
63
|
*/
|
|
47
64
|
readonly canonicalize: (sql: string) => Effect.Effect<string, EngineError | SqlParseError>
|
|
48
65
|
}
|
package/src/engine/duckdb.ts
CHANGED
|
@@ -4,9 +4,9 @@ import type { Engine, EngineColumn } from "./adapter.ts"
|
|
|
4
4
|
import { EngineAdapter, EngineError, SqlParseError } from "./adapter.ts"
|
|
5
5
|
|
|
6
6
|
/**
|
|
7
|
-
* AST
|
|
8
|
-
*
|
|
9
|
-
*
|
|
7
|
+
* The AST from json_serialize_sql contains token positions (query_location) —
|
|
8
|
+
* the only thing that differs between semantically identical but differently
|
|
9
|
+
* formatted queries. We strip it — leaving a format-invariant core.
|
|
10
10
|
*/
|
|
11
11
|
const stripLocations = (value: unknown): unknown => {
|
|
12
12
|
if (Array.isArray(value)) return value.map(stripLocations)
|
|
@@ -22,7 +22,7 @@ const stripLocations = (value: unknown): unknown => {
|
|
|
22
22
|
}
|
|
23
23
|
|
|
24
24
|
export interface DuckDBEngineOptions {
|
|
25
|
-
/**
|
|
25
|
+
/** Path to the database file; by default in-memory. */
|
|
26
26
|
readonly path?: string
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -65,8 +65,8 @@ export const DuckDBEngineLive = (
|
|
|
65
65
|
),
|
|
66
66
|
)
|
|
67
67
|
|
|
68
|
-
//
|
|
69
|
-
//
|
|
68
|
+
// single connection: parallel transactions are serialized by a semaphore
|
|
69
|
+
// so fibers do not interleave each other's BEGIN/COMMIT
|
|
70
70
|
const transactionLock = yield* Semaphore.make(1)
|
|
71
71
|
|
|
72
72
|
const service: Engine = {
|
|
@@ -101,7 +101,7 @@ export const DuckDBEngineLive = (
|
|
|
101
101
|
readonly error: boolean
|
|
102
102
|
readonly error_message?: string
|
|
103
103
|
}
|
|
104
|
-
//
|
|
104
|
+
// a parse error arrives as JSON content, not as an exception
|
|
105
105
|
if (ast.error) {
|
|
106
106
|
return new SqlParseError({
|
|
107
107
|
sql: sqlText,
|
package/src/engine/postgres.ts
CHANGED
|
@@ -5,17 +5,17 @@ import type { Engine, EngineColumn } from "./adapter.ts"
|
|
|
5
5
|
import { EngineAdapter, EngineError, SqlParseError } from "./adapter.ts"
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
|
-
* Postgres
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
8
|
+
* Postgres adapter (SPEC §9.1, F3) over Bun's built-in client (`Bun.SQL`):
|
|
9
|
+
* a connection pool out of the box — a transaction is pinned to a single
|
|
10
|
+
* connection via sql.begin, the rest of the queries are distributed freely
|
|
11
|
+
* across the pool, which is what gives honest backfill parallelism (SPEC §5.3).
|
|
12
12
|
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* Canonicalization — libpg_query (a WASM build of Postgres's own parser, SPEC
|
|
14
|
+
* §9.2): the parse tree with token positions stripped is format-invariant,
|
|
15
|
+
* just like DuckDB's json_serialize_sql.
|
|
16
16
|
*/
|
|
17
17
|
|
|
18
|
-
/**
|
|
18
|
+
/** Token positions (location) — the only format-dependency of the libpg_query tree. */
|
|
19
19
|
const stripLocations = (value: unknown): unknown => {
|
|
20
20
|
if (Array.isArray(value)) return value.map(stripLocations)
|
|
21
21
|
if (value !== null && typeof value === "object") {
|
|
@@ -30,15 +30,15 @@ const stripLocations = (value: unknown): unknown => {
|
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
|
-
*
|
|
34
|
-
*
|
|
35
|
-
*
|
|
33
|
+
* Postgres-SQL canonicalization is pure (libpg_query, no server needed);
|
|
34
|
+
* lifted out of the Layer so golden tests can freeze the canon independently
|
|
35
|
+
* of the pool (SPEC §4: canonicalization = the fingerprint contract).
|
|
36
36
|
*/
|
|
37
37
|
export const canonicalizePostgresSql = (
|
|
38
38
|
sqlText: string,
|
|
39
39
|
): Effect.Effect<string, SqlParseError> => {
|
|
40
|
-
// $start/$end
|
|
41
|
-
//
|
|
40
|
+
// $start/$end of the canonical render are not Postgres syntax;
|
|
41
|
+
// a deterministic replacement with $1/$2 keeps the fingerprint stable
|
|
42
42
|
const parameterized = sqlText.replace(/\$start\b/g, "$1").replace(/\$end\b/g, "$2")
|
|
43
43
|
return Effect.tryPromise({
|
|
44
44
|
try: () => parse(parameterized),
|
|
@@ -51,9 +51,9 @@ export const canonicalizePostgresSql = (
|
|
|
51
51
|
}
|
|
52
52
|
|
|
53
53
|
export interface PostgresEngineOptions {
|
|
54
|
-
/** postgres://…
|
|
54
|
+
/** postgres://… or a unix socket via ?host=/path. */
|
|
55
55
|
readonly url: string
|
|
56
|
-
/**
|
|
56
|
+
/** Connection pool size; by default 8. */
|
|
57
57
|
readonly max?: number
|
|
58
58
|
}
|
|
59
59
|
|
|
@@ -89,8 +89,8 @@ export const PostgresEngineLive = (
|
|
|
89
89
|
}),
|
|
90
90
|
catch: (cause) => new EngineError({ sql: statements.join(";\n"), cause }),
|
|
91
91
|
}).pipe(Effect.asVoid),
|
|
92
|
-
//
|
|
93
|
-
// (sql.begin)
|
|
92
|
+
// Postgres has no DESCRIBE: a temporary view on a single connection
|
|
93
|
+
// (sql.begin) gives names and types from the catalog without running the query
|
|
94
94
|
describe: (sqlText) =>
|
|
95
95
|
Effect.tryPromise({
|
|
96
96
|
try: () =>
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Message-building helpers shared by tagged errors (#13). Every error's
|
|
3
|
+
* `message` is DERIVED from its typed fields — no information lives only in a
|
|
4
|
+
* formatted string, and a vacuous message is constructively impossible: the
|
|
5
|
+
* culprit and the underlying cause are always carried in fields and rendered
|
|
6
|
+
* here. Kept out of the public API on purpose (internal plumbing).
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
/**
|
|
10
|
+
* Human text of an unknown caught value — the engine/syscall/parser's own
|
|
11
|
+
* message. This is the thing the code cannot invent: it must be surfaced from
|
|
12
|
+
* whatever the underlying layer threw, never swallowed.
|
|
13
|
+
*/
|
|
14
|
+
export const causeText = (cause: unknown): string => {
|
|
15
|
+
if (cause instanceof Error) {
|
|
16
|
+
return cause.message !== "" ? cause.message : cause.name
|
|
17
|
+
}
|
|
18
|
+
if (typeof cause === "string") return cause
|
|
19
|
+
if (cause === undefined) return "unknown cause"
|
|
20
|
+
if (typeof cause === "object" && cause !== null && "message" in cause) {
|
|
21
|
+
const message = (cause as { readonly message: unknown }).message
|
|
22
|
+
if (typeof message === "string" && message !== "") return message
|
|
23
|
+
}
|
|
24
|
+
return String(cause)
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* One-line SQL context for an error message: whitespace collapsed, truncated
|
|
29
|
+
* with an ellipsis. The full statement stays in the error's `sql` field for
|
|
30
|
+
* programmatic consumers; this is only the human-readable tail.
|
|
31
|
+
*/
|
|
32
|
+
export const sqlSnippet = (sql: string, max = 200): string => {
|
|
33
|
+
const collapsed = sql.replace(/\s+/g, " ").trim()
|
|
34
|
+
return collapsed.length > max ? `${collapsed.slice(0, max - 1)}…` : collapsed
|
|
35
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
2
|
+
* Public API of efmesh (F6): a deliberate whitelist instead of `export *` —
|
|
3
|
+
* everything exported here becomes a semver commitment of the package.
|
|
4
|
+
* Internals (naming, lock helpers, executor, planner, fingerprint plumbing)
|
|
5
|
+
* are deliberately not exported; unit-testing models lives in `efmesh/testing`.
|
|
6
6
|
*/
|
|
7
7
|
|
|
8
|
-
// —
|
|
8
|
+
// — project definition: models, kinds, audits, config —
|
|
9
9
|
export {
|
|
10
10
|
defineExternal,
|
|
11
11
|
defineModel,
|
|
@@ -30,7 +30,7 @@ export { audit, type Audit, type AuditCtx } from "./core/audit.ts"
|
|
|
30
30
|
export { defineConfig, type EfmeshConfig } from "./config.ts"
|
|
31
31
|
export { discoverModels, DiscoveryConflictError, DiscoveryError } from "./discovery.ts"
|
|
32
32
|
|
|
33
|
-
// —
|
|
33
|
+
// — facade and operations: plain Effects for embedding (SPEC §10) —
|
|
34
34
|
export { Efmesh } from "./efmesh.ts"
|
|
35
35
|
export { run, daemon, Runner, RunBlockedByChangesError, type RunError, type RunOptions } from "./plan/run.ts"
|
|
36
36
|
export { janitor, type JanitorOptions, type JanitorReport } from "./plan/janitor.ts"
|
|
@@ -41,17 +41,47 @@ export {
|
|
|
41
41
|
type AuditRunReport,
|
|
42
42
|
type AuditRunResult,
|
|
43
43
|
} from "./plan/audit-run.ts"
|
|
44
|
-
export {
|
|
44
|
+
export {
|
|
45
|
+
dataDiffEnvironments,
|
|
46
|
+
DataDiffError,
|
|
47
|
+
diffEnvironments,
|
|
48
|
+
type ColumnDrift,
|
|
49
|
+
type DataDiffOptions,
|
|
50
|
+
type DataDiffReport,
|
|
51
|
+
type EnvDiff,
|
|
52
|
+
type ModelDataDiff,
|
|
53
|
+
} from "./plan/diff.ts"
|
|
54
|
+
export {
|
|
55
|
+
environmentStatus,
|
|
56
|
+
type ModelLag,
|
|
57
|
+
type StatusOptions,
|
|
58
|
+
type StatusReport,
|
|
59
|
+
} from "./plan/status.ts"
|
|
45
60
|
export { formatLineage, lineage, LineageError, type LineageNode } from "./plan/lineage.ts"
|
|
61
|
+
export {
|
|
62
|
+
listSchedules,
|
|
63
|
+
registerSchedule,
|
|
64
|
+
removeSchedule,
|
|
65
|
+
ScheduleError,
|
|
66
|
+
scheduleTitle,
|
|
67
|
+
systemdUnits,
|
|
68
|
+
type ScheduleTarget,
|
|
69
|
+
} from "./plan/schedule.ts"
|
|
46
70
|
|
|
47
|
-
// —
|
|
71
|
+
// — plan and apply: result and options types —
|
|
48
72
|
export type {
|
|
49
73
|
ChangeCategory,
|
|
50
74
|
Plan,
|
|
51
75
|
PlanAction,
|
|
52
76
|
PlanOptions,
|
|
53
77
|
} from "./plan/planner.ts"
|
|
54
|
-
export {
|
|
78
|
+
export type { ChangeExplanation } from "./plan/explain.ts"
|
|
79
|
+
export {
|
|
80
|
+
FingerprintVersionError,
|
|
81
|
+
ForwardOnlyError,
|
|
82
|
+
InvalidEnvironmentError,
|
|
83
|
+
ReclassifyError,
|
|
84
|
+
} from "./plan/planner.ts"
|
|
55
85
|
export type { AppliedPlan, ApplyError, ApplyOptions } from "./plan/executor.ts"
|
|
56
86
|
export {
|
|
57
87
|
AttachNotConfiguredError,
|
|
@@ -64,17 +94,18 @@ export { SchemaMismatchError } from "./plan/contract.ts"
|
|
|
64
94
|
export { LockHeldError, type LockOptions } from "./plan/lock.ts"
|
|
65
95
|
export { FINGERPRINT_VERSION } from "./plan/fingerprint.ts"
|
|
66
96
|
|
|
67
|
-
// —
|
|
97
|
+
// — graph: compound project-loading errors —
|
|
68
98
|
export {
|
|
69
99
|
DagCycleError,
|
|
70
100
|
DuplicateModelError,
|
|
71
101
|
ModelDefinitionError,
|
|
72
102
|
SeedReadError,
|
|
73
103
|
UnknownDependencyError,
|
|
104
|
+
UnknownModelError,
|
|
74
105
|
} from "./core/errors.ts"
|
|
75
106
|
export type { GraphError } from "./core/graph.ts"
|
|
76
107
|
|
|
77
|
-
// —
|
|
108
|
+
// — engines: layers and service (custom wrappers — e.g. for tests) —
|
|
78
109
|
export {
|
|
79
110
|
EngineAdapter,
|
|
80
111
|
EngineError,
|
|
@@ -90,7 +121,7 @@ export {
|
|
|
90
121
|
type PostgresEngineOptions,
|
|
91
122
|
} from "./engine/postgres.ts"
|
|
92
123
|
|
|
93
|
-
// — state store:
|
|
124
|
+
// — state store: layers, migrations, records —
|
|
94
125
|
export {
|
|
95
126
|
StateStore,
|
|
96
127
|
StateError,
|
|
@@ -110,5 +141,5 @@ export {
|
|
|
110
141
|
type PostgresStateOptions,
|
|
111
142
|
} from "./state/postgres.ts"
|
|
112
143
|
|
|
113
|
-
// —
|
|
144
|
+
// — intervals: bounds for options.now and report parsing —
|
|
114
145
|
export { fromIso, toIso, type Interval, type IntervalUnit } from "./core/interval.ts"
|
package/src/init.ts
CHANGED
|
@@ -3,57 +3,129 @@ import * as NodePath from "node:path"
|
|
|
3
3
|
import { Data, Effect } from "effect"
|
|
4
4
|
|
|
5
5
|
/**
|
|
6
|
-
* `efmesh init` (SPEC §12):
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* `efmesh init` (SPEC §12): scaffold a minimal, runnable project — config, a
|
|
7
|
+
* seed, an incremental model with a backfill and a blocking audit, and a full
|
|
8
|
+
* rollup on top. `efmesh plan dev && efmesh apply dev` works immediately, with
|
|
9
|
+
* no data-generation step: the seed carries its own timestamped rows.
|
|
10
|
+
*
|
|
11
|
+
* The example is written to *teach* the plan/apply lifecycle to a first
|
|
12
|
+
* reader (often an evaluating agent): a seed whose content is versioned, an
|
|
13
|
+
* incrementalByTimeRange model that backfills day by day, and an audit that
|
|
14
|
+
* gates each interval. Overwrites nothing: an existing file is an honest error.
|
|
9
15
|
*/
|
|
10
16
|
|
|
11
17
|
export class InitError extends Data.TaggedError("InitError")<{
|
|
12
18
|
readonly path: string
|
|
13
19
|
readonly reason: string
|
|
14
|
-
}> {
|
|
20
|
+
}> {
|
|
21
|
+
override get message(): string {
|
|
22
|
+
return `${this.path}: ${this.reason}`
|
|
23
|
+
}
|
|
24
|
+
}
|
|
15
25
|
|
|
16
26
|
const MODELS_TS = `import { Schema } from "effect"
|
|
17
27
|
import { audit, defineModel, defineSeed, kind } from "@avytheone/efmesh"
|
|
18
28
|
|
|
19
|
-
// seed
|
|
20
|
-
//
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
29
|
+
// A seed is reference data loaded from a file. Its *content* feeds the
|
|
30
|
+
// fingerprint, so editing events.csv mints a new version and forces a rebuild
|
|
31
|
+
// of everything downstream — that is the guarantee the fingerprint buys you.
|
|
32
|
+
export const events = defineSeed({
|
|
33
|
+
name: "raw.events",
|
|
34
|
+
file: "seeds/events.csv",
|
|
35
|
+
schema: Schema.Struct({
|
|
36
|
+
event_at: Schema.DateTimeUtc,
|
|
37
|
+
region: Schema.String,
|
|
38
|
+
amount: Schema.Number,
|
|
39
|
+
}),
|
|
40
|
+
description: "Raw revenue events, one row per transaction",
|
|
25
41
|
})
|
|
26
42
|
|
|
27
|
-
//
|
|
28
|
-
|
|
43
|
+
// An incrementalByTimeRange model owns an interval ledger: apply backfills one
|
|
44
|
+
// day at a time from \`start\`, DELETE+INSERT per interval, and records which
|
|
45
|
+
// days are done. Re-running only fills the gaps — that is backfill. \`lookback: 1\`
|
|
46
|
+
// re-reads the most recent done day so late-arriving events still land.
|
|
47
|
+
//
|
|
48
|
+
// \`timeColumn\` names the OUTPUT column intervals are sliced by (here \`day\`); the
|
|
49
|
+
// body filters its source with ctx.start/ctx.end, the half-open bounds
|
|
50
|
+
// [start, end) of the interval being built. The blocking audit runs against
|
|
51
|
+
// each freshly loaded interval and, on a violation, fails apply for that day.
|
|
52
|
+
export const dailyRevenue = defineModel(
|
|
29
53
|
{
|
|
30
|
-
name: "mart.
|
|
54
|
+
name: "mart.daily_revenue",
|
|
55
|
+
kind: kind.incrementalByTimeRange({
|
|
56
|
+
timeColumn: "day",
|
|
57
|
+
start: "2026-01-01T00:00:00Z",
|
|
58
|
+
interval: "day",
|
|
59
|
+
lookback: 1,
|
|
60
|
+
}),
|
|
61
|
+
schema: Schema.Struct({
|
|
62
|
+
day: Schema.DateTimeUtc,
|
|
63
|
+
region: Schema.String,
|
|
64
|
+
revenue: Schema.Number,
|
|
65
|
+
}),
|
|
66
|
+
grain: ["day", "region"],
|
|
67
|
+
description: "Revenue per region per day, backfilled interval by interval",
|
|
68
|
+
audits: [audit.notNull("revenue")],
|
|
69
|
+
},
|
|
70
|
+
(ctx) => ctx.sql\`
|
|
71
|
+
SELECT
|
|
72
|
+
date_trunc('day', event_at) AS day,
|
|
73
|
+
region,
|
|
74
|
+
sum(amount)::BIGINT AS revenue
|
|
75
|
+
FROM \${ctx.ref(events)}
|
|
76
|
+
WHERE event_at >= \${ctx.start} AND event_at < \${ctx.end}
|
|
77
|
+
GROUP BY day, region
|
|
78
|
+
\`,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
// A full model rebuilds in one shot from the whole physical table of its
|
|
82
|
+
// parent. It depends on the incremental above via ctx.ref, so it sits
|
|
83
|
+
// downstream in the DAG and rebuilds when that parent's fingerprint changes.
|
|
84
|
+
export const regionRevenue = defineModel(
|
|
85
|
+
{
|
|
86
|
+
name: "mart.region_revenue",
|
|
31
87
|
kind: kind.full(),
|
|
32
|
-
schema: Schema.Struct({
|
|
33
|
-
|
|
88
|
+
schema: Schema.Struct({
|
|
89
|
+
region: Schema.String,
|
|
90
|
+
revenue: Schema.Number,
|
|
91
|
+
days: Schema.Number,
|
|
92
|
+
}),
|
|
93
|
+
description: "Lifetime revenue and active days per region",
|
|
34
94
|
},
|
|
35
95
|
(ctx) => ctx.sql\`
|
|
36
|
-
SELECT
|
|
37
|
-
FROM \${ctx.ref(
|
|
38
|
-
GROUP BY
|
|
96
|
+
SELECT region, sum(revenue)::BIGINT AS revenue, count(*)::BIGINT AS days
|
|
97
|
+
FROM \${ctx.ref(dailyRevenue)}
|
|
98
|
+
GROUP BY region
|
|
99
|
+
ORDER BY revenue DESC
|
|
39
100
|
\`,
|
|
40
101
|
)
|
|
41
102
|
`
|
|
42
103
|
|
|
43
104
|
const CONFIG_TS = `import { defineConfig } from "@avytheone/efmesh"
|
|
44
|
-
import {
|
|
105
|
+
import { dailyRevenue, events, regionRevenue } from "./models.ts"
|
|
45
106
|
|
|
107
|
+
// The project config is typed TypeScript, not YAML. The CLI imports it and
|
|
108
|
+
// assembles the engine and state layers. Defaults target a local DuckDB file
|
|
109
|
+
// (efmesh.duckdb) and a SQLite state store (efmesh.state.sqlite) next to it.
|
|
46
110
|
export default defineConfig({
|
|
47
|
-
models: [
|
|
48
|
-
// engine: { url: "postgres://…" },
|
|
49
|
-
// lake: { path: "lake" },
|
|
111
|
+
models: [events, dailyRevenue, regionRevenue],
|
|
112
|
+
// engine: { url: "postgres://…" }, // Postgres instead of the DuckDB file
|
|
113
|
+
// lake: { path: "lake" }, // enables target: "parquet" on a model
|
|
50
114
|
})
|
|
51
115
|
`
|
|
52
116
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
117
|
+
// Timestamps in plain \`YYYY-MM-DD HH:MM:SS\` so DuckDB's read_csv autodetects a
|
|
118
|
+
// TIMESTAMP column; two regions across three days give the backfill several
|
|
119
|
+
// non-empty intervals to fill.
|
|
120
|
+
const SEED_CSV = `event_at,region,amount
|
|
121
|
+
2026-01-01 08:30:00,north,100
|
|
122
|
+
2026-01-01 14:00:00,north,150
|
|
123
|
+
2026-01-01 16:45:00,south,90
|
|
124
|
+
2026-01-02 09:10:00,north,120
|
|
125
|
+
2026-01-02 11:20:00,south,200
|
|
126
|
+
2026-01-02 18:00:00,south,60
|
|
127
|
+
2026-01-03 07:50:00,north,80
|
|
128
|
+
2026-01-03 13:30:00,south,140
|
|
57
129
|
`
|
|
58
130
|
|
|
59
131
|
export const scaffold = (dir: string): Effect.Effect<ReadonlyArray<string>, InitError> =>
|
|
@@ -62,13 +134,13 @@ export const scaffold = (dir: string): Effect.Effect<ReadonlyArray<string>, Init
|
|
|
62
134
|
const files: ReadonlyArray<readonly [string, string]> = [
|
|
63
135
|
["efmesh.config.ts", CONFIG_TS],
|
|
64
136
|
["models.ts", MODELS_TS],
|
|
65
|
-
["seeds/
|
|
137
|
+
["seeds/events.csv", SEED_CSV],
|
|
66
138
|
]
|
|
67
139
|
for (const [relative] of files) {
|
|
68
140
|
if (existsSync(NodePath.join(root, relative))) {
|
|
69
141
|
return yield* new InitError({
|
|
70
142
|
path: NodePath.join(root, relative),
|
|
71
|
-
reason: "
|
|
143
|
+
reason: "file already exists — init overwrites nothing",
|
|
72
144
|
})
|
|
73
145
|
}
|
|
74
146
|
}
|