@avytheone/efmesh 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +22 -0
- package/README.ru.md +23 -0
- package/SPEC.md +1 -1
- package/package.json +1 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +204 -124
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +26 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +12 -11
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +7 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +33 -29
- package/src/plan/executor.ts +207 -125
- package/src/plan/explain.ts +29 -29
- package/src/plan/fingerprint.ts +35 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +107 -90
- package/src/plan/run.ts +27 -19
- package/src/plan/schedule.ts +46 -41
- package/src/plan/status.ts +15 -14
- package/src/state/postgres.ts +19 -19
- package/src/state/sqlite.ts +27 -27
- package/src/state/store.ts +67 -55
- package/src/testing/index.ts +27 -26
package/src/state/store.ts
CHANGED
|
@@ -1,65 +1,75 @@
|
|
|
1
1
|
import { Context, Data, Effect } from "effect"
|
|
2
|
+
import { causeText } from "../error-text.ts"
|
|
2
3
|
|
|
3
4
|
export class StateError extends Data.TaggedError("StateError")<{
|
|
4
5
|
readonly operation: string
|
|
5
6
|
readonly cause: unknown
|
|
6
|
-
}> {
|
|
7
|
+
}> {
|
|
8
|
+
override get message(): string {
|
|
9
|
+
return `state store: ${this.operation} failed — ${causeText(this.cause)}`
|
|
10
|
+
}
|
|
11
|
+
}
|
|
7
12
|
|
|
8
13
|
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
14
|
+
* Current schema version of the state store. A fresh store bootstraps
|
|
15
|
+
* directly at it; a store with an older schema (including one created
|
|
16
|
+
* before versioning existed) refuses to open — data catches up via an
|
|
17
|
+
* explicit `efmesh migrate`.
|
|
18
|
+
* 1 — base layout (F4), 2 — applied_by in the plan journal (F5),
|
|
19
|
+
* 3 — fingerprint_version in snapshots (F6), 4 — run tick journal (0.2.0),
|
|
20
|
+
* 5 — canonicalization cache canon_cache (0.2.0, #8).
|
|
15
21
|
*/
|
|
16
22
|
export const STATE_VERSION = 5
|
|
17
23
|
|
|
18
|
-
/**
|
|
24
|
+
/** Store schema doesn't match the binary's expectation — `efmesh migrate` is needed. */
|
|
19
25
|
export class StateSchemaError extends Data.TaggedError("StateSchemaError")<{
|
|
20
26
|
readonly found: number
|
|
21
27
|
readonly wanted: number
|
|
22
|
-
}> {
|
|
28
|
+
}> {
|
|
29
|
+
override get message(): string {
|
|
30
|
+
return `state store schema is v${this.found}, but this efmesh expects v${this.wanted} — run \`efmesh migrate\``
|
|
31
|
+
}
|
|
32
|
+
}
|
|
23
33
|
|
|
24
|
-
/**
|
|
34
|
+
/** Migration outcome for the CLI. */
|
|
25
35
|
export interface MigrationReport {
|
|
26
36
|
readonly from: number
|
|
27
37
|
readonly to: number
|
|
28
|
-
/**
|
|
38
|
+
/** Where the store's copy was stashed before the schema upgrade (SQLite; F6). */
|
|
29
39
|
readonly backup?: string
|
|
30
40
|
}
|
|
31
41
|
|
|
32
|
-
/**
|
|
42
|
+
/** A model version known to the state store (SPEC §6). */
|
|
33
43
|
export interface SnapshotRecord {
|
|
34
44
|
readonly name: string
|
|
35
45
|
readonly fingerprint: string
|
|
36
|
-
/** Canonical
|
|
46
|
+
/** Canonical SQL rendering — for diff display and debugging. */
|
|
37
47
|
readonly renderedSql: string
|
|
38
|
-
/**
|
|
48
|
+
/** Canonical AST of the body (JSON) — for categorizing changes (SPEC §5.2). */
|
|
39
49
|
readonly canonicalAst: string
|
|
40
50
|
/**
|
|
41
|
-
* Fingerprint
|
|
42
|
-
*
|
|
43
|
-
*
|
|
51
|
+
* Fingerprint whose physical table/prefix this snapshot uses.
|
|
52
|
+
* Usually its own; under forward-only (SPEC §5.2) it is inherited
|
|
53
|
+
* from the previous version: physics is reused, history is not replayed.
|
|
44
54
|
*/
|
|
45
55
|
readonly physicalFp: string
|
|
46
56
|
readonly kind: string
|
|
47
57
|
/**
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
58
|
+
* Version of the fingerprint algorithm used to compute this snapshot (SPEC §4).
|
|
59
|
+
* Plan only compares same-version fingerprints; a different version is
|
|
60
|
+
* a loud stop, not a silent "everything is breaking".
|
|
51
61
|
*/
|
|
52
62
|
readonly fingerprintVersion: number
|
|
53
63
|
readonly createdAt: string
|
|
54
64
|
/**
|
|
55
|
-
*
|
|
56
|
-
* null
|
|
57
|
-
*
|
|
65
|
+
* When the snapshot stopped being pointed to by any environment (ISO UTC);
|
|
66
|
+
* null means it is still referenced. Set and cleared on promotion,
|
|
67
|
+
* the janitor's ttl is counted from here (SPEC §5.4).
|
|
58
68
|
*/
|
|
59
69
|
readonly orphanedAt: string | null
|
|
60
70
|
}
|
|
61
71
|
|
|
62
|
-
/**
|
|
72
|
+
/** Environment row: logical name → snapshot the view points to. */
|
|
63
73
|
export interface EnvironmentRecord {
|
|
64
74
|
readonly env: string
|
|
65
75
|
readonly name: string
|
|
@@ -72,15 +82,15 @@ export interface PlanRecord {
|
|
|
72
82
|
readonly env: string
|
|
73
83
|
readonly summary: string
|
|
74
84
|
readonly appliedAt: string
|
|
75
|
-
/**
|
|
85
|
+
/** Who applied the plan (OS user or ApplyOptions.appliedBy); '' for records predating v2. */
|
|
76
86
|
readonly appliedBy: string
|
|
77
87
|
}
|
|
78
88
|
|
|
79
89
|
/**
|
|
80
|
-
*
|
|
81
|
-
*
|
|
82
|
-
* ok —
|
|
83
|
-
* lock-held —
|
|
90
|
+
* Run tick journal entry (SPEC §7, issue #2): a cron tick that fails at
|
|
91
|
+
* three in the morning must be debuggable after the fact. outcome:
|
|
92
|
+
* ok — tick succeeded; awaiting-human — there are unapplied changes (exit 2);
|
|
93
|
+
* lock-held — the environment is held by another process; error — an actual failure.
|
|
84
94
|
*/
|
|
85
95
|
export interface RunRecord {
|
|
86
96
|
readonly id: number
|
|
@@ -88,14 +98,14 @@ export interface RunRecord {
|
|
|
88
98
|
readonly startedAt: string
|
|
89
99
|
readonly finishedAt: string
|
|
90
100
|
readonly outcome: "ok" | "awaiting-human" | "lock-held" | "error"
|
|
91
|
-
/** ok: JSON
|
|
101
|
+
/** ok: JSON array of collected models; awaiting-human: list of changes; error: error tag. */
|
|
92
102
|
readonly detail: string
|
|
93
103
|
}
|
|
94
104
|
|
|
95
105
|
/**
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
106
|
+
* Bookkeeping of a snapshot's filled intervals (SPEC §6) — the single source
|
|
107
|
+
* of truth for what has been computed: a physical table with no records here
|
|
108
|
+
* is considered empty. Bounds are ISO UTC (sorted lexicographically).
|
|
99
109
|
*/
|
|
100
110
|
export interface IntervalRecord {
|
|
101
111
|
readonly snapshotFp: string
|
|
@@ -106,7 +116,7 @@ export interface IntervalRecord {
|
|
|
106
116
|
}
|
|
107
117
|
|
|
108
118
|
export interface StateStoreShape {
|
|
109
|
-
/**
|
|
119
|
+
/** Idempotent: (name, fingerprint) is unique, a repeated write is a no-op. */
|
|
110
120
|
readonly upsertSnapshot: (
|
|
111
121
|
snapshot: Omit<SnapshotRecord, "createdAt" | "orphanedAt">,
|
|
112
122
|
) => Effect.Effect<void, StateError>
|
|
@@ -114,17 +124,18 @@ export interface StateStoreShape {
|
|
|
114
124
|
name: string,
|
|
115
125
|
fingerprint: string,
|
|
116
126
|
) => Effect.Effect<SnapshotRecord | undefined, StateError>
|
|
117
|
-
/**
|
|
127
|
+
/** All snapshots referenced by at least one environment — for the janitor. */
|
|
118
128
|
readonly listReferencedFingerprints: () => Effect.Effect<ReadonlySet<string>, StateError>
|
|
119
129
|
readonly listSnapshots: () => Effect.Effect<ReadonlyArray<SnapshotRecord>, StateError>
|
|
120
|
-
/**
|
|
130
|
+
/** Deletes the snapshot record and its interval bookkeeping (the janitor removes the physics). */
|
|
121
131
|
readonly deleteSnapshot: (name: string, fingerprint: string) => Effect.Effect<void, StateError>
|
|
122
132
|
/**
|
|
123
|
-
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
133
|
+
* Transactional "claim" of an orphan by the janitor (SPEC §5.4, F6): the
|
|
134
|
+
* record and its interval bookkeeping are deleted only if the snapshot is
|
|
135
|
+
* STILL not referenced by any environment and orphaned no later than the
|
|
136
|
+
* deadline (ISO UTC) — both checks run in the same transaction as the
|
|
137
|
+
* delete, so a concurrent apply that revived the version can't lose it to
|
|
138
|
+
* removal. true — removed, false — state changed underneath.
|
|
128
139
|
*/
|
|
129
140
|
readonly deleteSnapshotIfDoomed: (
|
|
130
141
|
name: string,
|
|
@@ -135,10 +146,10 @@ export interface StateStoreShape {
|
|
|
135
146
|
env: string,
|
|
136
147
|
) => Effect.Effect<ReadonlyArray<EnvironmentRecord>, StateError>
|
|
137
148
|
/**
|
|
138
|
-
*
|
|
139
|
-
* `requireSnapshot: true`
|
|
140
|
-
*
|
|
141
|
-
* view
|
|
149
|
+
* Transactionally replaces the environment's whole set. Entries with
|
|
150
|
+
* `requireSnapshot: true` are checked for the snapshot's liveness in the
|
|
151
|
+
* same transaction: if the janitor already removed it, promotion fails
|
|
152
|
+
* loudly — the view never switches to demolished physics.
|
|
142
153
|
*/
|
|
143
154
|
readonly promote: (
|
|
144
155
|
env: string,
|
|
@@ -148,35 +159,36 @@ export interface StateStoreShape {
|
|
|
148
159
|
readonly requireSnapshot?: boolean
|
|
149
160
|
}>,
|
|
150
161
|
) => Effect.Effect<void, StateError>
|
|
151
|
-
/**
|
|
162
|
+
/** Journal of applied plans. */
|
|
152
163
|
readonly recordPlan: (
|
|
153
164
|
env: string,
|
|
154
165
|
summary: string,
|
|
155
166
|
appliedBy: string,
|
|
156
167
|
) => Effect.Effect<void, StateError>
|
|
157
168
|
/**
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
169
|
+
* Canonicalization cache (#8): the key already includes the dialect and
|
|
170
|
+
* FINGERPRINT_VERSION — a change in algorithm or engine can never hand back
|
|
171
|
+
* a stale canon. The cache is not data: a miss or failure is safe, the
|
|
172
|
+
* caller must swallow it.
|
|
161
173
|
*/
|
|
162
174
|
readonly getCanon: (key: string) => Effect.Effect<string | undefined, StateError>
|
|
163
175
|
readonly putCanon: (key: string, canonical: string) => Effect.Effect<void, StateError>
|
|
164
|
-
/**
|
|
176
|
+
/** Run tick journal (SPEC §7): the outcome of every tick, including unsuccessful ones. */
|
|
165
177
|
readonly recordRun: (record: Omit<RunRecord, "id">) => Effect.Effect<void, StateError>
|
|
166
|
-
/**
|
|
178
|
+
/** The environment's most recent ticks, newest first. */
|
|
167
179
|
readonly listRuns: (
|
|
168
180
|
env: string,
|
|
169
181
|
limit: number,
|
|
170
182
|
) => Effect.Effect<ReadonlyArray<RunRecord>, StateError>
|
|
171
183
|
readonly listPlans: (env: string) => Effect.Effect<ReadonlyArray<PlanRecord>, StateError>
|
|
172
184
|
/**
|
|
173
|
-
*
|
|
174
|
-
*
|
|
175
|
-
*
|
|
185
|
+
* Cross-process lock (SPEC §7): true — acquired, false — held by another
|
|
186
|
+
* process. Stale (expired) locks are reclaimed — a crashed process
|
|
187
|
+
* doesn't leave a lock held forever.
|
|
176
188
|
*/
|
|
177
189
|
readonly acquireLock: (name: string, ttlMs: number) => Effect.Effect<boolean, StateError>
|
|
178
190
|
readonly releaseLock: (name: string) => Effect.Effect<void, StateError>
|
|
179
|
-
/**
|
|
191
|
+
/** Transactional upsert of snapshot intervals (re-marking updates the status). */
|
|
180
192
|
readonly markIntervals: (
|
|
181
193
|
snapshotFp: string,
|
|
182
194
|
intervals: ReadonlyArray<{ readonly startTs: string; readonly endTs: string }>,
|
package/src/testing/index.ts
CHANGED
|
@@ -7,22 +7,22 @@ import { DuckDBEngineLive } from "../engine/duckdb.ts"
|
|
|
7
7
|
import { familyOfAst, type TypeFamily } from "../plan/contract.ts"
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (
|
|
13
|
-
*
|
|
14
|
-
*
|
|
10
|
+
* Unit-tests a model against fixtures (SPEC §8): `ctx.ref` is rendered into a
|
|
11
|
+
* CTE with VALUES from the fixtures, validated against the source model's
|
|
12
|
+
* Schema (the model remembers its sources by value — model.refs); the query
|
|
13
|
+
* runs on a throwaway in-memory DuckDB, and the result is compared to the
|
|
14
|
+
* expectation. Lives in bun test:
|
|
15
15
|
*
|
|
16
16
|
* test("stays", () => testModel(stays, { inputs: {...}, expect: [...] }))
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
export interface TestModelSpec {
|
|
20
|
-
/**
|
|
20
|
+
/** Fixtures keyed by source models' full names; required for every dep. */
|
|
21
21
|
readonly inputs?: Readonly<Record<string, ReadonlyArray<Record<string, unknown>>>>
|
|
22
|
-
/**
|
|
22
|
+
/** Interval bounds [start, end) — required for incremental models. */
|
|
23
23
|
readonly interval?: readonly [string, string]
|
|
24
24
|
readonly expect: ReadonlyArray<Record<string, unknown>>
|
|
25
|
-
/**
|
|
25
|
+
/** Whether row order matters; unordered by default. */
|
|
26
26
|
readonly strictOrder?: boolean
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -34,7 +34,7 @@ const DUCK_TYPE: Record<TypeFamily, string> = {
|
|
|
34
34
|
any: "VARCHAR",
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
/**
|
|
37
|
+
/** Fixture literal by column family (ISO time strings → TIMESTAMP). */
|
|
38
38
|
const fixtureLiteral = (value: unknown, family: TypeFamily): string => {
|
|
39
39
|
if (value === null || value === undefined) return "NULL"
|
|
40
40
|
if (family === "temporal") return sqlTimestamp(fromIso(String(value)))
|
|
@@ -46,14 +46,15 @@ const fixtureLiteral = (value: unknown, family: TypeFamily): string => {
|
|
|
46
46
|
) {
|
|
47
47
|
return escapeLiteral(value)
|
|
48
48
|
}
|
|
49
|
-
throw new Error(
|
|
49
|
+
throw new Error(`fixture contains a non-renderable value: ${String(value)}`)
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
53
|
+
* A fixture cannot lie about the shape of its input: values are checked
|
|
54
|
+
* against the source Schema's type families (time strings are ISO; in v4
|
|
55
|
+
* Schema.DateTimeUtc decodes to a ready DateTime.Utc, not a string, so the
|
|
56
|
+
* check is by family rather than via decode), and extra keys — typos — are
|
|
57
|
+
* rejected.
|
|
57
58
|
*/
|
|
58
59
|
const validateFixture = (
|
|
59
60
|
source: AnyModel,
|
|
@@ -63,7 +64,7 @@ const validateFixture = (
|
|
|
63
64
|
const known = new Set(fields.map(([name]) => name))
|
|
64
65
|
for (const key of Object.keys(row)) {
|
|
65
66
|
if (!known.has(key)) {
|
|
66
|
-
throw new Error(
|
|
67
|
+
throw new Error(`fixture ${source.name.full}: extra column «${key}»`)
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
for (const [name, field] of fields) {
|
|
@@ -82,13 +83,13 @@ const validateFixture = (
|
|
|
82
83
|
: true
|
|
83
84
|
if (!ok) {
|
|
84
85
|
throw new Error(
|
|
85
|
-
|
|
86
|
+
`fixture ${source.name.full}: column «${name}» expects ${family}, got ${JSON.stringify(value)}`,
|
|
86
87
|
)
|
|
87
88
|
}
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
91
|
|
|
91
|
-
/** CTE
|
|
92
|
+
/** Source CTE: VALUES from fixtures, or an empty SELECT typed by the schema. */
|
|
92
93
|
const fixtureCte = (source: AnyModel, rows: ReadonlyArray<Record<string, unknown>>): string => {
|
|
93
94
|
const fields = Object.entries(source.schema.fields) as ReadonlyArray<
|
|
94
95
|
[string, { readonly ast: unknown }]
|
|
@@ -109,7 +110,7 @@ const fixtureCte = (source: AnyModel, rows: ReadonlyArray<Record<string, unknown
|
|
|
109
110
|
return `SELECT * FROM (VALUES ${tuples.join(", ")}) AS t(${columns})`
|
|
110
111
|
}
|
|
111
112
|
|
|
112
|
-
/** DuckDB
|
|
113
|
+
/** DuckDB values → comparable primitives (bigint → number, time objects → string). */
|
|
113
114
|
const normalize = (row: Record<string, unknown>): Record<string, unknown> =>
|
|
114
115
|
Object.fromEntries(
|
|
115
116
|
Object.entries(row).map(([key, value]) => {
|
|
@@ -130,7 +131,7 @@ const canonical = (
|
|
|
130
131
|
return strictOrder ? normalized : [...normalized].sort()
|
|
131
132
|
}
|
|
132
133
|
|
|
133
|
-
/**
|
|
134
|
+
/** Runs the model against fixtures, returning rows — for non-standard checks. */
|
|
134
135
|
export const runModel = async (
|
|
135
136
|
model: AnyModel,
|
|
136
137
|
spec: Pick<TestModelSpec, "inputs" | "interval">,
|
|
@@ -138,16 +139,16 @@ export const runModel = async (
|
|
|
138
139
|
const inputs = spec.inputs ?? {}
|
|
139
140
|
for (const dep of model.deps) {
|
|
140
141
|
if (!(dep in inputs)) {
|
|
141
|
-
throw new Error(
|
|
142
|
+
throw new Error(`no fixture for source ${dep} (deps of model ${model.name.full})`)
|
|
142
143
|
}
|
|
143
144
|
}
|
|
144
145
|
for (const name of Object.keys(inputs)) {
|
|
145
146
|
if (!model.deps.has(name)) {
|
|
146
|
-
throw new Error(
|
|
147
|
+
throw new Error(`fixture «${name}» is not a source of model ${model.name.full}`)
|
|
147
148
|
}
|
|
148
149
|
}
|
|
149
150
|
if (usesBounds(model.fragment) && spec.interval === undefined) {
|
|
150
|
-
throw new Error(
|
|
151
|
+
throw new Error(`model ${model.name.full} uses ctx.start/ctx.end — provide interval`)
|
|
151
152
|
}
|
|
152
153
|
|
|
153
154
|
const ctes = [...model.deps]
|
|
@@ -174,7 +175,7 @@ export const runModel = async (
|
|
|
174
175
|
)
|
|
175
176
|
}
|
|
176
177
|
|
|
177
|
-
/**
|
|
178
|
+
/** Runs the model against fixtures and checks the result against the expectation. */
|
|
178
179
|
export const testModel = async (model: AnyModel, spec: TestModelSpec): Promise<void> => {
|
|
179
180
|
const rows = await runModel(model, spec)
|
|
180
181
|
const got = canonical(rows, spec.strictOrder ?? false)
|
|
@@ -182,10 +183,10 @@ export const testModel = async (model: AnyModel, spec: TestModelSpec): Promise<v
|
|
|
182
183
|
if (got.length !== want.length || got.some((row, index) => row !== want[index])) {
|
|
183
184
|
throw new Error(
|
|
184
185
|
[
|
|
185
|
-
|
|
186
|
-
`—
|
|
186
|
+
`result of model ${model.name.full} did not match the expectation:`,
|
|
187
|
+
`— got (${got.length}):`,
|
|
187
188
|
...got.map((row) => ` ${row}`),
|
|
188
|
-
`—
|
|
189
|
+
`— expected (${want.length}):`,
|
|
189
190
|
...want.map((row) => ` ${row}`),
|
|
190
191
|
].join("\n"),
|
|
191
192
|
)
|