@avytheone/efmesh 0.1.0-beta.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -92
- package/README.md +83 -9
- package/README.ru.md +80 -5
- package/SPEC.md +542 -459
- package/package.json +4 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +505 -85
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +28 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +44 -13
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +10 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +228 -4
- package/src/plan/executor.ts +223 -121
- package/src/plan/explain.ts +114 -0
- package/src/plan/fingerprint.ts +84 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +238 -52
- package/src/plan/run.ts +72 -26
- package/src/plan/schedule.ts +217 -0
- package/src/plan/status.ts +96 -0
- package/src/state/postgres.ts +68 -19
- package/src/state/sqlite.ts +74 -27
- package/src/state/store.ts +88 -45
- package/src/testing/index.ts +27 -26
package/src/state/sqlite.ts
CHANGED
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
IntervalRecord,
|
|
7
7
|
MigrationReport,
|
|
8
8
|
PlanRecord,
|
|
9
|
+
RunRecord,
|
|
9
10
|
SnapshotRecord,
|
|
10
11
|
StateStoreShape,
|
|
11
12
|
} from "./store.ts"
|
|
@@ -46,6 +47,18 @@ CREATE TABLE IF NOT EXISTS intervals (
|
|
|
46
47
|
updated_at TEXT NOT NULL,
|
|
47
48
|
PRIMARY KEY (snapshot_fp, start_ts)
|
|
48
49
|
);
|
|
50
|
+
CREATE TABLE IF NOT EXISTS runs (
|
|
51
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
52
|
+
env TEXT NOT NULL,
|
|
53
|
+
started_at TEXT NOT NULL,
|
|
54
|
+
finished_at TEXT NOT NULL,
|
|
55
|
+
outcome TEXT NOT NULL,
|
|
56
|
+
detail TEXT NOT NULL DEFAULT ''
|
|
57
|
+
);
|
|
58
|
+
CREATE TABLE IF NOT EXISTS canon_cache (
|
|
59
|
+
key TEXT PRIMARY KEY,
|
|
60
|
+
canonical TEXT NOT NULL
|
|
61
|
+
);
|
|
49
62
|
CREATE TABLE IF NOT EXISTS locks (
|
|
50
63
|
name TEXT PRIMARY KEY,
|
|
51
64
|
acquired_at TEXT NOT NULL,
|
|
@@ -54,7 +67,7 @@ CREATE TABLE IF NOT EXISTS locks (
|
|
|
54
67
|
`
|
|
55
68
|
|
|
56
69
|
export interface SqliteStateOptions {
|
|
57
|
-
/**
|
|
70
|
+
/** Path to the state file; defaults to in-memory (tests). */
|
|
58
71
|
readonly path?: string
|
|
59
72
|
}
|
|
60
73
|
|
|
@@ -65,7 +78,7 @@ const tableExists = (db: Database, name: string): boolean =>
|
|
|
65
78
|
.query(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?1`)
|
|
66
79
|
.get(name) !== null
|
|
67
80
|
|
|
68
|
-
/** 0 —
|
|
81
|
+
/** 0 — a store with no meta table (created before versioning existed, F0–F3). */
|
|
69
82
|
const readVersion = (db: Database): number => {
|
|
70
83
|
if (!tableExists(db, "meta")) return 0
|
|
71
84
|
const row = db.query(`SELECT version FROM meta`).get() as { version: number } | null
|
|
@@ -73,11 +86,11 @@ const readVersion = (db: Database): number => {
|
|
|
73
86
|
}
|
|
74
87
|
|
|
75
88
|
/**
|
|
76
|
-
*
|
|
77
|
-
*
|
|
78
|
-
*
|
|
79
|
-
* orphaned_at/physical_fp — F3, applied_by — F5)
|
|
80
|
-
*
|
|
89
|
+
* Catches the schema up to STATE_VERSION. Version 1 = base layout F4,
|
|
90
|
+
* version 2 = applied_by in the plan journal (F5); the ALTERs below pick up
|
|
91
|
+
* stores created before the corresponding columns existed (canonical_ast —
|
|
92
|
+
* F2, orphaned_at/physical_fp — F3, applied_by — F5) — on a fresh store
|
|
93
|
+
* they are a no-op via try/catch. Future versions get new entries right here.
|
|
81
94
|
*/
|
|
82
95
|
const applyMigrations = (db: Database): void => {
|
|
83
96
|
db.exec(SCHEMA)
|
|
@@ -91,7 +104,7 @@ const applyMigrations = (db: Database): void => {
|
|
|
91
104
|
try {
|
|
92
105
|
db.exec(alter)
|
|
93
106
|
} catch {
|
|
94
|
-
//
|
|
107
|
+
// column already exists
|
|
95
108
|
}
|
|
96
109
|
}
|
|
97
110
|
db.exec(`CREATE TABLE IF NOT EXISTS meta (version INTEGER NOT NULL)`)
|
|
@@ -100,9 +113,9 @@ const applyMigrations = (db: Database): void => {
|
|
|
100
113
|
}
|
|
101
114
|
|
|
102
115
|
/**
|
|
103
|
-
* `efmesh migrate`:
|
|
104
|
-
*
|
|
105
|
-
*
|
|
116
|
+
* `efmesh migrate`: an explicit schema upgrade of an existing store.
|
|
117
|
+
* Before the upgrade, the file is copied to `<path>.backup-v<from>` —
|
|
118
|
+
* otherwise rolling back to an older efmesh version would be a one-way trip (F6).
|
|
106
119
|
*/
|
|
107
120
|
export const migrateSqliteState = (
|
|
108
121
|
options?: SqliteStateOptions,
|
|
@@ -144,9 +157,9 @@ export const SqliteStateLive = (
|
|
|
144
157
|
}),
|
|
145
158
|
(db) => Effect.sync(() => db.close()),
|
|
146
159
|
)
|
|
147
|
-
//
|
|
148
|
-
//
|
|
149
|
-
//
|
|
160
|
+
// a fresh store bootstraps at the current version; an existing store
|
|
161
|
+
// with an older schema requires an explicit `efmesh migrate` — silently
|
|
162
|
+
// rewriting someone else's data on open is not allowed (SPEC §6)
|
|
150
163
|
const fresh = yield* Effect.try({
|
|
151
164
|
try: () => !tableExists(db, "snapshots"),
|
|
152
165
|
catch: (cause) => new StateError({ operation: "open", cause }),
|
|
@@ -174,10 +187,10 @@ export const SqliteStateLive = (
|
|
|
174
187
|
isoNow.pipe(
|
|
175
188
|
Effect.flatMap((now) =>
|
|
176
189
|
attempt("upsertSnapshot", () => {
|
|
177
|
-
//
|
|
178
|
-
//
|
|
179
|
-
//
|
|
180
|
-
// orphaned_at
|
|
190
|
+
// reviving a version (a repeated apply of an old fingerprint)
|
|
191
|
+
// clears orphan status AND refreshes created_at IMMEDIATELY: between
|
|
192
|
+
// the upsert and promotion, the janitor won't judge it doomed by
|
|
193
|
+
// either orphaned_at or a stale created_at (race, F6)
|
|
181
194
|
db.query(
|
|
182
195
|
`INSERT INTO snapshots (name, fingerprint, rendered_sql, canonical_ast, physical_fp, kind, fingerprint_version, created_at)
|
|
183
196
|
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
|
|
@@ -281,16 +294,16 @@ export const SqliteStateLive = (
|
|
|
281
294
|
Effect.flatMap((now) =>
|
|
282
295
|
attempt("promote", () => {
|
|
283
296
|
const replace = db.transaction(() => {
|
|
284
|
-
//
|
|
285
|
-
//
|
|
286
|
-
//
|
|
297
|
+
// snapshot liveness — in the same transaction: if the janitor
|
|
298
|
+
// already removed the version, promotion fails loudly, and the
|
|
299
|
+
// view never switches to demolished physics (race, F6)
|
|
287
300
|
const alive = db.query(
|
|
288
301
|
`SELECT 1 FROM snapshots WHERE name = ?1 AND fingerprint = ?2`,
|
|
289
302
|
)
|
|
290
303
|
for (const entry of entries) {
|
|
291
304
|
if (entry.requireSnapshot === true && alive.get(entry.name, entry.fingerprint) === null) {
|
|
292
305
|
throw new Error(
|
|
293
|
-
|
|
306
|
+
`promotion "${env}": snapshot ${entry.name}@${entry.fingerprint.slice(0, 8)} vanished from the store (removed by janitor?) — retry apply`,
|
|
294
307
|
)
|
|
295
308
|
}
|
|
296
309
|
}
|
|
@@ -302,9 +315,9 @@ export const SqliteStateLive = (
|
|
|
302
315
|
for (const entry of entries) {
|
|
303
316
|
insert.run(env, entry.name, entry.fingerprint, now)
|
|
304
317
|
}
|
|
305
|
-
//
|
|
306
|
-
//
|
|
307
|
-
// (
|
|
318
|
+
// referencing changes only here — so orphan bookkeeping happens
|
|
319
|
+
// right here too: those losing their last reference get marked,
|
|
320
|
+
// those returning (rollback to an old version) lose the mark (SPEC §5.4)
|
|
308
321
|
db.query(
|
|
309
322
|
`UPDATE snapshots SET orphaned_at = ?1
|
|
310
323
|
WHERE orphaned_at IS NULL
|
|
@@ -342,6 +355,40 @@ export const SqliteStateLive = (
|
|
|
342
355
|
.all(env) as ReadonlyArray<PlanRecord>
|
|
343
356
|
}),
|
|
344
357
|
|
|
358
|
+
getCanon: (key) =>
|
|
359
|
+
attempt("getCanon", () => {
|
|
360
|
+
const row = db
|
|
361
|
+
.query(`SELECT canonical FROM canon_cache WHERE key = ?1`)
|
|
362
|
+
.get(key) as { canonical: string } | null
|
|
363
|
+
return row?.canonical ?? undefined
|
|
364
|
+
}),
|
|
365
|
+
|
|
366
|
+
putCanon: (key, canonical) =>
|
|
367
|
+
attempt("putCanon", () => {
|
|
368
|
+
db.query(
|
|
369
|
+
`INSERT INTO canon_cache (key, canonical) VALUES (?1, ?2)
|
|
370
|
+
ON CONFLICT (key) DO NOTHING`,
|
|
371
|
+
).run(key, canonical)
|
|
372
|
+
}),
|
|
373
|
+
|
|
374
|
+
recordRun: (record) =>
|
|
375
|
+
attempt("recordRun", () => {
|
|
376
|
+
db.query(
|
|
377
|
+
`INSERT INTO runs (env, started_at, finished_at, outcome, detail)
|
|
378
|
+
VALUES (?1, ?2, ?3, ?4, ?5)`,
|
|
379
|
+
).run(record.env, record.startedAt, record.finishedAt, record.outcome, record.detail)
|
|
380
|
+
}),
|
|
381
|
+
|
|
382
|
+
listRuns: (env, limit) =>
|
|
383
|
+
attempt("listRuns", () => {
|
|
384
|
+
return db
|
|
385
|
+
.query(
|
|
386
|
+
`SELECT id, env, started_at AS startedAt, finished_at AS finishedAt, outcome, detail
|
|
387
|
+
FROM runs WHERE env = ?1 ORDER BY id DESC LIMIT ?2`,
|
|
388
|
+
)
|
|
389
|
+
.all(env, limit) as ReadonlyArray<RunRecord>
|
|
390
|
+
}),
|
|
391
|
+
|
|
345
392
|
acquireLock: (name, ttlMs) =>
|
|
346
393
|
Clock.currentTimeMillis.pipe(
|
|
347
394
|
Effect.flatMap((nowMs) =>
|
|
@@ -349,8 +396,8 @@ export const SqliteStateLive = (
|
|
|
349
396
|
const now = new Date(nowMs).toISOString()
|
|
350
397
|
const expires = new Date(nowMs + ttlMs).toISOString()
|
|
351
398
|
const acquire = db.transaction(() => {
|
|
352
|
-
//
|
|
353
|
-
// <= —
|
|
399
|
+
// a stale lock from a crashed process is reclaimed;
|
|
400
|
+
// <= — a lock that expires at instant T is free as of T (ttl=0 in the same ms)
|
|
354
401
|
db.query(`DELETE FROM locks WHERE name = ?1 AND expires_at <= ?2`).run(name, now)
|
|
355
402
|
const result = db
|
|
356
403
|
.query(
|
package/src/state/store.ts
CHANGED
|
@@ -1,64 +1,75 @@
|
|
|
1
1
|
import { Context, Data, Effect } from "effect"
|
|
2
|
+
import { causeText } from "../error-text.ts"
|
|
2
3
|
|
|
3
4
|
export class StateError extends Data.TaggedError("StateError")<{
|
|
4
5
|
readonly operation: string
|
|
5
6
|
readonly cause: unknown
|
|
6
|
-
}> {
|
|
7
|
+
}> {
|
|
8
|
+
override get message(): string {
|
|
9
|
+
return `state store: ${this.operation} failed — ${causeText(this.cause)}`
|
|
10
|
+
}
|
|
11
|
+
}
|
|
7
12
|
|
|
8
13
|
/**
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
+
* Current schema version of the state store. A fresh store bootstraps
|
|
15
|
+
* directly at it; a store with an older schema (including one created
|
|
16
|
+
* before versioning existed) refuses to open — data catches up via an
|
|
17
|
+
* explicit `efmesh migrate`.
|
|
18
|
+
* 1 — base layout (F4), 2 — applied_by in the plan journal (F5),
|
|
19
|
+
* 3 — fingerprint_version in snapshots (F6), 4 — run tick journal (0.2.0),
|
|
20
|
+
* 5 — canonicalization cache canon_cache (0.2.0, #8).
|
|
14
21
|
*/
|
|
15
|
-
export const STATE_VERSION =
|
|
22
|
+
export const STATE_VERSION = 5
|
|
16
23
|
|
|
17
|
-
/**
|
|
24
|
+
/** Store schema doesn't match the binary's expectation — `efmesh migrate` is needed. */
|
|
18
25
|
export class StateSchemaError extends Data.TaggedError("StateSchemaError")<{
|
|
19
26
|
readonly found: number
|
|
20
27
|
readonly wanted: number
|
|
21
|
-
}> {
|
|
28
|
+
}> {
|
|
29
|
+
override get message(): string {
|
|
30
|
+
return `state store schema is v${this.found}, but this efmesh expects v${this.wanted} — run \`efmesh migrate\``
|
|
31
|
+
}
|
|
32
|
+
}
|
|
22
33
|
|
|
23
|
-
/**
|
|
34
|
+
/** Migration outcome for the CLI. */
|
|
24
35
|
export interface MigrationReport {
|
|
25
36
|
readonly from: number
|
|
26
37
|
readonly to: number
|
|
27
|
-
/**
|
|
38
|
+
/** Where the store's copy was stashed before the schema upgrade (SQLite; F6). */
|
|
28
39
|
readonly backup?: string
|
|
29
40
|
}
|
|
30
41
|
|
|
31
|
-
/**
|
|
42
|
+
/** A model version known to the state store (SPEC §6). */
|
|
32
43
|
export interface SnapshotRecord {
|
|
33
44
|
readonly name: string
|
|
34
45
|
readonly fingerprint: string
|
|
35
|
-
/** Canonical
|
|
46
|
+
/** Canonical SQL rendering — for diff display and debugging. */
|
|
36
47
|
readonly renderedSql: string
|
|
37
|
-
/**
|
|
48
|
+
/** Canonical AST of the body (JSON) — for categorizing changes (SPEC §5.2). */
|
|
38
49
|
readonly canonicalAst: string
|
|
39
50
|
/**
|
|
40
|
-
* Fingerprint
|
|
41
|
-
*
|
|
42
|
-
*
|
|
51
|
+
* Fingerprint whose physical table/prefix this snapshot uses.
|
|
52
|
+
* Usually its own; under forward-only (SPEC §5.2) it is inherited
|
|
53
|
+
* from the previous version: physics is reused, history is not replayed.
|
|
43
54
|
*/
|
|
44
55
|
readonly physicalFp: string
|
|
45
56
|
readonly kind: string
|
|
46
57
|
/**
|
|
47
|
-
*
|
|
48
|
-
*
|
|
49
|
-
*
|
|
58
|
+
* Version of the fingerprint algorithm used to compute this snapshot (SPEC §4).
|
|
59
|
+
* Plan only compares same-version fingerprints; a different version is
|
|
60
|
+
* a loud stop, not a silent "everything is breaking".
|
|
50
61
|
*/
|
|
51
62
|
readonly fingerprintVersion: number
|
|
52
63
|
readonly createdAt: string
|
|
53
64
|
/**
|
|
54
|
-
*
|
|
55
|
-
* null
|
|
56
|
-
*
|
|
65
|
+
* When the snapshot stopped being pointed to by any environment (ISO UTC);
|
|
66
|
+
* null means it is still referenced. Set and cleared on promotion,
|
|
67
|
+
* the janitor's ttl is counted from here (SPEC §5.4).
|
|
57
68
|
*/
|
|
58
69
|
readonly orphanedAt: string | null
|
|
59
70
|
}
|
|
60
71
|
|
|
61
|
-
/**
|
|
72
|
+
/** Environment row: logical name → snapshot the view points to. */
|
|
62
73
|
export interface EnvironmentRecord {
|
|
63
74
|
readonly env: string
|
|
64
75
|
readonly name: string
|
|
@@ -71,14 +82,30 @@ export interface PlanRecord {
|
|
|
71
82
|
readonly env: string
|
|
72
83
|
readonly summary: string
|
|
73
84
|
readonly appliedAt: string
|
|
74
|
-
/**
|
|
85
|
+
/** Who applied the plan (OS user or ApplyOptions.appliedBy); '' for records predating v2. */
|
|
75
86
|
readonly appliedBy: string
|
|
76
87
|
}
|
|
77
88
|
|
|
78
89
|
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
*
|
|
90
|
+
* Run tick journal entry (SPEC §7, issue #2): a cron tick that fails at
|
|
91
|
+
* three in the morning must be debuggable after the fact. outcome:
|
|
92
|
+
* ok — tick succeeded; awaiting-human — there are unapplied changes (exit 2);
|
|
93
|
+
* lock-held — the environment is held by another process; error — an actual failure.
|
|
94
|
+
*/
|
|
95
|
+
export interface RunRecord {
|
|
96
|
+
readonly id: number
|
|
97
|
+
readonly env: string
|
|
98
|
+
readonly startedAt: string
|
|
99
|
+
readonly finishedAt: string
|
|
100
|
+
readonly outcome: "ok" | "awaiting-human" | "lock-held" | "error"
|
|
101
|
+
/** ok: JSON array of collected models; awaiting-human: list of changes; error: error tag. */
|
|
102
|
+
readonly detail: string
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
/**
|
|
106
|
+
* Bookkeeping of a snapshot's filled intervals (SPEC §6) — the single source
|
|
107
|
+
* of truth for what has been computed: a physical table with no records here
|
|
108
|
+
* is considered empty. Bounds are ISO UTC (sorted lexicographically).
|
|
82
109
|
*/
|
|
83
110
|
export interface IntervalRecord {
|
|
84
111
|
readonly snapshotFp: string
|
|
@@ -89,7 +116,7 @@ export interface IntervalRecord {
|
|
|
89
116
|
}
|
|
90
117
|
|
|
91
118
|
export interface StateStoreShape {
|
|
92
|
-
/**
|
|
119
|
+
/** Idempotent: (name, fingerprint) is unique, a repeated write is a no-op. */
|
|
93
120
|
readonly upsertSnapshot: (
|
|
94
121
|
snapshot: Omit<SnapshotRecord, "createdAt" | "orphanedAt">,
|
|
95
122
|
) => Effect.Effect<void, StateError>
|
|
@@ -97,17 +124,18 @@ export interface StateStoreShape {
|
|
|
97
124
|
name: string,
|
|
98
125
|
fingerprint: string,
|
|
99
126
|
) => Effect.Effect<SnapshotRecord | undefined, StateError>
|
|
100
|
-
/**
|
|
127
|
+
/** All snapshots referenced by at least one environment — for the janitor. */
|
|
101
128
|
readonly listReferencedFingerprints: () => Effect.Effect<ReadonlySet<string>, StateError>
|
|
102
129
|
readonly listSnapshots: () => Effect.Effect<ReadonlyArray<SnapshotRecord>, StateError>
|
|
103
|
-
/**
|
|
130
|
+
/** Deletes the snapshot record and its interval bookkeeping (the janitor removes the physics). */
|
|
104
131
|
readonly deleteSnapshot: (name: string, fingerprint: string) => Effect.Effect<void, StateError>
|
|
105
132
|
/**
|
|
106
|
-
*
|
|
107
|
-
*
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
*
|
|
133
|
+
* Transactional "claim" of an orphan by the janitor (SPEC §5.4, F6): the
|
|
134
|
+
* record and its interval bookkeeping are deleted only if the snapshot is
|
|
135
|
+
* STILL not referenced by any environment and orphaned no later than the
|
|
136
|
+
* deadline (ISO UTC) — both checks run in the same transaction as the
|
|
137
|
+
* delete, so a concurrent apply that revived the version can't lose it to
|
|
138
|
+
* removal. true — removed, false — state changed underneath.
|
|
111
139
|
*/
|
|
112
140
|
readonly deleteSnapshotIfDoomed: (
|
|
113
141
|
name: string,
|
|
@@ -118,10 +146,10 @@ export interface StateStoreShape {
|
|
|
118
146
|
env: string,
|
|
119
147
|
) => Effect.Effect<ReadonlyArray<EnvironmentRecord>, StateError>
|
|
120
148
|
/**
|
|
121
|
-
*
|
|
122
|
-
* `requireSnapshot: true`
|
|
123
|
-
*
|
|
124
|
-
* view
|
|
149
|
+
* Transactionally replaces the environment's whole set. Entries with
|
|
150
|
+
* `requireSnapshot: true` are checked for the snapshot's liveness in the
|
|
151
|
+
* same transaction: if the janitor already removed it, promotion fails
|
|
152
|
+
* loudly — the view never switches to demolished physics.
|
|
125
153
|
*/
|
|
126
154
|
readonly promote: (
|
|
127
155
|
env: string,
|
|
@@ -131,21 +159,36 @@ export interface StateStoreShape {
|
|
|
131
159
|
readonly requireSnapshot?: boolean
|
|
132
160
|
}>,
|
|
133
161
|
) => Effect.Effect<void, StateError>
|
|
134
|
-
/**
|
|
162
|
+
/** Journal of applied plans. */
|
|
135
163
|
readonly recordPlan: (
|
|
136
164
|
env: string,
|
|
137
165
|
summary: string,
|
|
138
166
|
appliedBy: string,
|
|
139
167
|
) => Effect.Effect<void, StateError>
|
|
168
|
+
/**
|
|
169
|
+
* Canonicalization cache (#8): the key already includes the dialect and
|
|
170
|
+
* FINGERPRINT_VERSION — a change in algorithm or engine can never hand back
|
|
171
|
+
* a stale canon. The cache is not data: a miss or failure is safe, the
|
|
172
|
+
* caller must swallow it.
|
|
173
|
+
*/
|
|
174
|
+
readonly getCanon: (key: string) => Effect.Effect<string | undefined, StateError>
|
|
175
|
+
readonly putCanon: (key: string, canonical: string) => Effect.Effect<void, StateError>
|
|
176
|
+
/** Run tick journal (SPEC §7): the outcome of every tick, including unsuccessful ones. */
|
|
177
|
+
readonly recordRun: (record: Omit<RunRecord, "id">) => Effect.Effect<void, StateError>
|
|
178
|
+
/** The environment's most recent ticks, newest first. */
|
|
179
|
+
readonly listRuns: (
|
|
180
|
+
env: string,
|
|
181
|
+
limit: number,
|
|
182
|
+
) => Effect.Effect<ReadonlyArray<RunRecord>, StateError>
|
|
140
183
|
readonly listPlans: (env: string) => Effect.Effect<ReadonlyArray<PlanRecord>, StateError>
|
|
141
184
|
/**
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
185
|
+
* Cross-process lock (SPEC §7): true — acquired, false — held by another
|
|
186
|
+
* process. Stale (expired) locks are reclaimed — a crashed process
|
|
187
|
+
* doesn't leave a lock held forever.
|
|
145
188
|
*/
|
|
146
189
|
readonly acquireLock: (name: string, ttlMs: number) => Effect.Effect<boolean, StateError>
|
|
147
190
|
readonly releaseLock: (name: string) => Effect.Effect<void, StateError>
|
|
148
|
-
/**
|
|
191
|
+
/** Transactional upsert of snapshot intervals (re-marking updates the status). */
|
|
149
192
|
readonly markIntervals: (
|
|
150
193
|
snapshotFp: string,
|
|
151
194
|
intervals: ReadonlyArray<{ readonly startTs: string; readonly endTs: string }>,
|
package/src/testing/index.ts
CHANGED
|
@@ -7,22 +7,22 @@ import { DuckDBEngineLive } from "../engine/duckdb.ts"
|
|
|
7
7
|
import { familyOfAst, type TypeFamily } from "../plan/contract.ts"
|
|
8
8
|
|
|
9
9
|
/**
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* (
|
|
13
|
-
*
|
|
14
|
-
*
|
|
10
|
+
* Unit-tests a model against fixtures (SPEC §8): `ctx.ref` is rendered into a
|
|
11
|
+
* CTE with VALUES from the fixtures, validated against the source model's
|
|
12
|
+
* Schema (the model remembers its sources by value — model.refs); the query
|
|
13
|
+
* runs on a throwaway in-memory DuckDB, and the result is compared to the
|
|
14
|
+
* expectation. Lives in bun test:
|
|
15
15
|
*
|
|
16
16
|
* test("stays", () => testModel(stays, { inputs: {...}, expect: [...] }))
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
export interface TestModelSpec {
|
|
20
|
-
/**
|
|
20
|
+
/** Fixtures keyed by source models' full names; required for every dep. */
|
|
21
21
|
readonly inputs?: Readonly<Record<string, ReadonlyArray<Record<string, unknown>>>>
|
|
22
|
-
/**
|
|
22
|
+
/** Interval bounds [start, end) — required for incremental models. */
|
|
23
23
|
readonly interval?: readonly [string, string]
|
|
24
24
|
readonly expect: ReadonlyArray<Record<string, unknown>>
|
|
25
|
-
/**
|
|
25
|
+
/** Whether row order matters; unordered by default. */
|
|
26
26
|
readonly strictOrder?: boolean
|
|
27
27
|
}
|
|
28
28
|
|
|
@@ -34,7 +34,7 @@ const DUCK_TYPE: Record<TypeFamily, string> = {
|
|
|
34
34
|
any: "VARCHAR",
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
-
/**
|
|
37
|
+
/** Fixture literal by column family (ISO time strings → TIMESTAMP). */
|
|
38
38
|
const fixtureLiteral = (value: unknown, family: TypeFamily): string => {
|
|
39
39
|
if (value === null || value === undefined) return "NULL"
|
|
40
40
|
if (family === "temporal") return sqlTimestamp(fromIso(String(value)))
|
|
@@ -46,14 +46,15 @@ const fixtureLiteral = (value: unknown, family: TypeFamily): string => {
|
|
|
46
46
|
) {
|
|
47
47
|
return escapeLiteral(value)
|
|
48
48
|
}
|
|
49
|
-
throw new Error(
|
|
49
|
+
throw new Error(`fixture contains a non-renderable value: ${String(value)}`)
|
|
50
50
|
}
|
|
51
51
|
|
|
52
52
|
/**
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
-
*
|
|
56
|
-
*
|
|
53
|
+
* A fixture cannot lie about the shape of its input: values are checked
|
|
54
|
+
* against the source Schema's type families (time strings are ISO; in v4
|
|
55
|
+
* Schema.DateTimeUtc decodes to a ready DateTime.Utc, not a string, so the
|
|
56
|
+
* check is by family rather than via decode), and extra keys — typos — are
|
|
57
|
+
* rejected.
|
|
57
58
|
*/
|
|
58
59
|
const validateFixture = (
|
|
59
60
|
source: AnyModel,
|
|
@@ -63,7 +64,7 @@ const validateFixture = (
|
|
|
63
64
|
const known = new Set(fields.map(([name]) => name))
|
|
64
65
|
for (const key of Object.keys(row)) {
|
|
65
66
|
if (!known.has(key)) {
|
|
66
|
-
throw new Error(
|
|
67
|
+
throw new Error(`fixture ${source.name.full}: extra column «${key}»`)
|
|
67
68
|
}
|
|
68
69
|
}
|
|
69
70
|
for (const [name, field] of fields) {
|
|
@@ -82,13 +83,13 @@ const validateFixture = (
|
|
|
82
83
|
: true
|
|
83
84
|
if (!ok) {
|
|
84
85
|
throw new Error(
|
|
85
|
-
|
|
86
|
+
`fixture ${source.name.full}: column «${name}» expects ${family}, got ${JSON.stringify(value)}`,
|
|
86
87
|
)
|
|
87
88
|
}
|
|
88
89
|
}
|
|
89
90
|
}
|
|
90
91
|
|
|
91
|
-
/** CTE
|
|
92
|
+
/** Source CTE: VALUES from fixtures, or an empty SELECT typed by the schema. */
|
|
92
93
|
const fixtureCte = (source: AnyModel, rows: ReadonlyArray<Record<string, unknown>>): string => {
|
|
93
94
|
const fields = Object.entries(source.schema.fields) as ReadonlyArray<
|
|
94
95
|
[string, { readonly ast: unknown }]
|
|
@@ -109,7 +110,7 @@ const fixtureCte = (source: AnyModel, rows: ReadonlyArray<Record<string, unknown
|
|
|
109
110
|
return `SELECT * FROM (VALUES ${tuples.join(", ")}) AS t(${columns})`
|
|
110
111
|
}
|
|
111
112
|
|
|
112
|
-
/** DuckDB
|
|
113
|
+
/** DuckDB values → comparable primitives (bigint → number, time objects → string). */
|
|
113
114
|
const normalize = (row: Record<string, unknown>): Record<string, unknown> =>
|
|
114
115
|
Object.fromEntries(
|
|
115
116
|
Object.entries(row).map(([key, value]) => {
|
|
@@ -130,7 +131,7 @@ const canonical = (
|
|
|
130
131
|
return strictOrder ? normalized : [...normalized].sort()
|
|
131
132
|
}
|
|
132
133
|
|
|
133
|
-
/**
|
|
134
|
+
/** Runs the model against fixtures, returning rows — for non-standard checks. */
|
|
134
135
|
export const runModel = async (
|
|
135
136
|
model: AnyModel,
|
|
136
137
|
spec: Pick<TestModelSpec, "inputs" | "interval">,
|
|
@@ -138,16 +139,16 @@ export const runModel = async (
|
|
|
138
139
|
const inputs = spec.inputs ?? {}
|
|
139
140
|
for (const dep of model.deps) {
|
|
140
141
|
if (!(dep in inputs)) {
|
|
141
|
-
throw new Error(
|
|
142
|
+
throw new Error(`no fixture for source ${dep} (deps of model ${model.name.full})`)
|
|
142
143
|
}
|
|
143
144
|
}
|
|
144
145
|
for (const name of Object.keys(inputs)) {
|
|
145
146
|
if (!model.deps.has(name)) {
|
|
146
|
-
throw new Error(
|
|
147
|
+
throw new Error(`fixture «${name}» is not a source of model ${model.name.full}`)
|
|
147
148
|
}
|
|
148
149
|
}
|
|
149
150
|
if (usesBounds(model.fragment) && spec.interval === undefined) {
|
|
150
|
-
throw new Error(
|
|
151
|
+
throw new Error(`model ${model.name.full} uses ctx.start/ctx.end — provide interval`)
|
|
151
152
|
}
|
|
152
153
|
|
|
153
154
|
const ctes = [...model.deps]
|
|
@@ -174,7 +175,7 @@ export const runModel = async (
|
|
|
174
175
|
)
|
|
175
176
|
}
|
|
176
177
|
|
|
177
|
-
/**
|
|
178
|
+
/** Runs the model against fixtures and checks the result against the expectation. */
|
|
178
179
|
export const testModel = async (model: AnyModel, spec: TestModelSpec): Promise<void> => {
|
|
179
180
|
const rows = await runModel(model, spec)
|
|
180
181
|
const got = canonical(rows, spec.strictOrder ?? false)
|
|
@@ -182,10 +183,10 @@ export const testModel = async (model: AnyModel, spec: TestModelSpec): Promise<v
|
|
|
182
183
|
if (got.length !== want.length || got.some((row, index) => row !== want[index])) {
|
|
183
184
|
throw new Error(
|
|
184
185
|
[
|
|
185
|
-
|
|
186
|
-
`—
|
|
186
|
+
`result of model ${model.name.full} did not match the expectation:`,
|
|
187
|
+
`— got (${got.length}):`,
|
|
187
188
|
...got.map((row) => ` ${row}`),
|
|
188
|
-
`—
|
|
189
|
+
`— expected (${want.length}):`,
|
|
189
190
|
...want.map((row) => ` ${row}`),
|
|
190
191
|
].join("\n"),
|
|
191
192
|
)
|