@avytheone/efmesh 0.1.0-beta.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -92
- package/README.md +83 -9
- package/README.ru.md +80 -5
- package/SPEC.md +542 -459
- package/package.json +4 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +505 -85
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +28 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +44 -13
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +10 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +228 -4
- package/src/plan/executor.ts +223 -121
- package/src/plan/explain.ts +114 -0
- package/src/plan/fingerprint.ts +84 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +238 -52
- package/src/plan/run.ts +72 -26
- package/src/plan/schedule.ts +217 -0
- package/src/plan/status.ts +96 -0
- package/src/state/postgres.ts +68 -19
- package/src/state/sqlite.ts +74 -27
- package/src/state/store.ts +88 -45
- package/src/testing/index.ts +27 -26
package/src/plan/run.ts
CHANGED
|
@@ -1,51 +1,94 @@
|
|
|
1
|
-
import { Data, Effect, Schedule } from "effect"
|
|
1
|
+
import { Clock, Data, Effect, Schedule } from "effect"
|
|
2
2
|
import { buildGraph } from "../core/graph.ts"
|
|
3
3
|
import type { AnyModel } from "../core/model.ts"
|
|
4
4
|
import { EngineAdapter } from "../engine/adapter.ts"
|
|
5
|
-
import { StateStore } from "../state/store.ts"
|
|
5
|
+
import { StateStore, type RunRecord } from "../state/store.ts"
|
|
6
6
|
import { applyPlan, type AppliedPlan, type ApplyError, type ApplyOptions } from "./executor.ts"
|
|
7
7
|
import { envLockName, withStateLock, type LockHeldError, type LockOptions } from "./lock.ts"
|
|
8
8
|
import { planChanges, type PlanOptions } from "./planner.ts"
|
|
9
9
|
|
|
10
10
|
/**
|
|
11
|
-
* `run` —
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
16
|
-
*
|
|
11
|
+
* `run` — a scheduler tick (SPEC §7): catches up intervals and upserts of
|
|
12
|
+
* EXISTING versions, never applies model changes — that is the job of
|
|
13
|
+
* plan/apply with a human at the helm. Idempotent and safe for cron/systemd:
|
|
14
|
+
* a parallel run is cut off by a lock in the state store — the same
|
|
15
|
+
* `env:<name>` as apply's (SPEC §14.6), so run will not wedge into someone
|
|
16
|
+
* else's apply and vice versa.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
19
|
export class RunBlockedByChangesError extends Data.TaggedError("RunBlockedByChangesError")<{
|
|
20
20
|
readonly env: string
|
|
21
21
|
readonly changes: ReadonlyArray<string>
|
|
22
|
-
}> {
|
|
22
|
+
}> {
|
|
23
|
+
override get message(): string {
|
|
24
|
+
return `environment «${this.env}» has unapplied structural changes: ${this.changes.join(", ")} — run \`efmesh apply ${this.env}\``
|
|
25
|
+
}
|
|
26
|
+
}
|
|
23
27
|
|
|
24
28
|
export type RunError = ApplyError | LockHeldError | RunBlockedByChangesError
|
|
25
29
|
|
|
26
30
|
export interface RunOptions extends PlanOptions, ApplyOptions, LockOptions {}
|
|
27
31
|
|
|
32
|
+
/** Tick outcome for the journal: error → category + detail. */
|
|
33
|
+
const classify = (error: RunError): Pick<RunRecord, "outcome" | "detail"> => {
|
|
34
|
+
switch (error._tag) {
|
|
35
|
+
case "RunBlockedByChangesError":
|
|
36
|
+
return { outcome: "awaiting-human", detail: error.changes.join("; ") }
|
|
37
|
+
case "LockHeldError":
|
|
38
|
+
return { outcome: "lock-held", detail: error.name }
|
|
39
|
+
default:
|
|
40
|
+
return { outcome: "error", detail: error._tag }
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
|
|
28
44
|
export const run = (
|
|
29
45
|
env: string,
|
|
30
46
|
models: Iterable<AnyModel>,
|
|
31
47
|
options?: RunOptions,
|
|
32
48
|
): Effect.Effect<AppliedPlan, RunError, EngineAdapter | StateStore> =>
|
|
33
49
|
Effect.gen(function* () {
|
|
34
|
-
const
|
|
35
|
-
const
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
50
|
+
const store = yield* StateStore
|
|
51
|
+
const startedAt = new Date(yield* Clock.currentTimeMillis).toISOString()
|
|
52
|
+
// tick journal (SPEC §7, #2): the outcome is written ALWAYS, including
|
|
53
|
+
// failure — a cron tick that fell over at three in the morning is debugged
|
|
54
|
+
// after the fact; a failure of the write itself does not mask the real
|
|
55
|
+
// outcome (log + ignore)
|
|
56
|
+
const journal = (entry: Pick<RunRecord, "outcome" | "detail">) =>
|
|
57
|
+
Clock.currentTimeMillis.pipe(
|
|
58
|
+
Effect.flatMap((now) =>
|
|
59
|
+
store.recordRun({
|
|
60
|
+
env,
|
|
61
|
+
startedAt,
|
|
62
|
+
finishedAt: new Date(now).toISOString(),
|
|
63
|
+
...entry,
|
|
64
|
+
}),
|
|
65
|
+
),
|
|
66
|
+
Effect.catchCause((cause) => Effect.logWarning("tick journal is unavailable", cause)),
|
|
67
|
+
)
|
|
68
|
+
|
|
69
|
+
return yield* Effect.gen(function* () {
|
|
70
|
+
const graph = yield* buildGraph(models)
|
|
71
|
+
const plan = yield* planChanges(env, graph, options)
|
|
72
|
+
const structural = plan.actions
|
|
73
|
+
.filter((a) => a.change !== "unchanged")
|
|
74
|
+
.map((a) => `${a.name}: ${a.change}`)
|
|
75
|
+
if (structural.length > 0) {
|
|
76
|
+
return yield* new RunBlockedByChangesError({ env, changes: structural })
|
|
77
|
+
}
|
|
78
|
+
return yield* applyPlan(plan, graph, options)
|
|
79
|
+
}).pipe(
|
|
80
|
+
withStateLock(envLockName(env), options?.lockTtlMs),
|
|
81
|
+
Effect.tap((applied) =>
|
|
82
|
+
journal({ outcome: "ok", detail: JSON.stringify(applied.built) }),
|
|
83
|
+
),
|
|
84
|
+
Effect.tapError((error) => journal(classify(error))),
|
|
85
|
+
)
|
|
86
|
+
})
|
|
44
87
|
|
|
45
88
|
/**
|
|
46
|
-
*
|
|
47
|
-
*
|
|
48
|
-
* (
|
|
89
|
+
* Long-lived scheduler for embedding in an Effect application (SPEC §7):
|
|
90
|
+
* ticks on a Schedule; a tick error is logged and does not bring the daemon
|
|
91
|
+
* down (except a held lock — that is routine, logged more quietly).
|
|
49
92
|
*/
|
|
50
93
|
export const daemon = (
|
|
51
94
|
env: string,
|
|
@@ -56,13 +99,16 @@ export const daemon = (
|
|
|
56
99
|
run(env, models, options).pipe(
|
|
57
100
|
Effect.tap((applied) =>
|
|
58
101
|
applied.built.length > 0
|
|
59
|
-
? Effect.logInfo(`
|
|
60
|
-
: Effect.
|
|
102
|
+
? Effect.logInfo(`tick built ${applied.built.join(", ")}`)
|
|
103
|
+
: Effect.logInfo("tick: no new intervals"),
|
|
61
104
|
),
|
|
105
|
+
// a held lock is routine (another process ticks) — debug, not an alarm
|
|
62
106
|
Effect.catchTag("LockHeldError", () =>
|
|
63
|
-
Effect.logDebug(
|
|
107
|
+
Effect.logDebug("tick skipped: lock held by another process"),
|
|
64
108
|
),
|
|
65
|
-
Effect.catchCause((cause) => Effect.logError(
|
|
109
|
+
Effect.catchCause((cause) => Effect.logError("tick failed", cause)),
|
|
110
|
+
// env is a structured field, not baked into every message (#14)
|
|
111
|
+
Effect.annotateLogs("env", env),
|
|
66
112
|
Effect.schedule(schedule),
|
|
67
113
|
Effect.andThen(Effect.never),
|
|
68
114
|
)
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs"
|
|
2
|
+
import { basename, dirname, join } from "node:path"
|
|
3
|
+
import { fileURLToPath } from "node:url"
|
|
4
|
+
import { Data, Effect } from "effect"
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* `efmesh schedule` (#10): registers a `run` tick in the OS scheduler via
|
|
8
|
+
* Bun.cron (>=1.3.11): crontab on Linux, launchd on macOS, Task Scheduler
|
|
9
|
+
* on Windows. One title = one entry; re-registration overwrites in place
|
|
10
|
+
* (idempotent).
|
|
11
|
+
*
|
|
12
|
+
* Honest caveats (documented in README/SPEC): cron does not catch up on
|
|
13
|
+
* missed runs (systemd Persistent=true is stricter — hence --print-systemd),
|
|
14
|
+
* the OS level lives in the LOCAL timezone (TZ=UTC on top), on Linux a live
|
|
15
|
+
* cron daemon is required — the Arch family without cronie has none at all.
|
|
16
|
+
* Overlapping ticks are safe by construction: run holds the env lock, and
|
|
17
|
+
* "awaiting a human" = exit 2, not a failure.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
export class ScheduleError extends Data.TaggedError("ScheduleError")<{
|
|
21
|
+
readonly reason: string
|
|
22
|
+
}> {
|
|
23
|
+
override get message(): string {
|
|
24
|
+
return this.reason
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** Everything the worker needs, as absolute paths (crontab has no cwd). */
|
|
29
|
+
export interface ScheduleTarget {
|
|
30
|
+
/** Project directory (where the config lives) — the tick's cwd. */
|
|
31
|
+
readonly project: string
|
|
32
|
+
/** Absolute path of the config. */
|
|
33
|
+
readonly config: string
|
|
34
|
+
readonly env: string
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** Bun.cron title: only [A-Za-z0-9_-]; includes the project and environment. */
|
|
38
|
+
export const scheduleTitle = (target: Pick<ScheduleTarget, "project" | "env">): string =>
|
|
39
|
+
`efmesh-${basename(target.project)}-${target.env}`.replaceAll(/[^A-Za-z0-9_-]/g, "-")
|
|
40
|
+
|
|
41
|
+
/** Where the worker is generated: next to the config, in the internal .efmesh/. */
|
|
42
|
+
export const workerPath = (target: ScheduleTarget): string =>
|
|
43
|
+
join(target.project, ".efmesh", `schedule-${target.env}.ts`)
|
|
44
|
+
|
|
45
|
+
/** CLI binary of this same package — the worker calls it without guessing npm names. */
|
|
46
|
+
const binPath = (): string => fileURLToPath(new URL("../bin.ts", import.meta.url))
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Worker source for Bun.cron: the OS scheduler runs scheduled(), which runs
|
|
50
|
+
* an ordinary `efmesh run` — the same exit-code semantics (2 = "awaiting a
|
|
51
|
+
* human") and the same tick journal in the store (`efmesh status`).
|
|
52
|
+
*/
|
|
53
|
+
export const workerSource = (target: ScheduleTarget): string => `// generated by \`efmesh schedule\` — do not edit: re-registration overwrites it
|
|
54
|
+
export default {
|
|
55
|
+
async scheduled() {
|
|
56
|
+
const proc = Bun.spawn(
|
|
57
|
+
[${JSON.stringify(process.execPath)}, ${JSON.stringify(binPath())}, "run", ${JSON.stringify(target.env)}, "--config", ${JSON.stringify(target.config)}],
|
|
58
|
+
{ cwd: ${JSON.stringify(target.project)}, stdout: "inherit", stderr: "inherit" },
|
|
59
|
+
)
|
|
60
|
+
process.exitCode = await proc.exited
|
|
61
|
+
},
|
|
62
|
+
}
|
|
63
|
+
`
|
|
64
|
+
|
|
65
|
+
/** cron nicknames → systemd OnCalendar; arbitrary expressions are not translated. */
|
|
66
|
+
export const cronToOnCalendar = (cron: string): string | undefined =>
|
|
67
|
+
(
|
|
68
|
+
{
|
|
69
|
+
"@hourly": "hourly",
|
|
70
|
+
"@daily": "daily",
|
|
71
|
+
"@midnight": "daily",
|
|
72
|
+
"@weekly": "weekly",
|
|
73
|
+
"@monthly": "monthly",
|
|
74
|
+
"@yearly": "yearly",
|
|
75
|
+
"@annually": "yearly",
|
|
76
|
+
} as Record<string, string>
|
|
77
|
+
)[cron.trim()]
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* systemd fallback (--print-systemd): cron has no catch-up for missed runs
|
|
81
|
+
* and the Arch family has no daemon — a user timer with Persistent=true is
|
|
82
|
+
* more honest.
|
|
83
|
+
*/
|
|
84
|
+
export const systemdUnits = (
|
|
85
|
+
target: ScheduleTarget,
|
|
86
|
+
cron: string,
|
|
87
|
+
): { readonly service: string; readonly timer: string; readonly name: string } => {
|
|
88
|
+
const name = scheduleTitle(target)
|
|
89
|
+
const calendar = cronToOnCalendar(cron)
|
|
90
|
+
return {
|
|
91
|
+
name,
|
|
92
|
+
service: `[Unit]
|
|
93
|
+
Description=efmesh run ${target.env} (${basename(target.project)})
|
|
94
|
+
|
|
95
|
+
[Service]
|
|
96
|
+
Type=oneshot
|
|
97
|
+
WorkingDirectory=${target.project}
|
|
98
|
+
ExecStart=${process.execPath} ${binPath()} run ${target.env} --config ${target.config}
|
|
99
|
+
# exit 2 = "awaiting a human" (apply needed) — deliberately a unit failure: visible in alerts
|
|
100
|
+
`,
|
|
101
|
+
timer: `[Unit]
|
|
102
|
+
Description=efmesh run ${target.env} — timer
|
|
103
|
+
|
|
104
|
+
[Timer]
|
|
105
|
+
OnCalendar=${calendar ?? `hourly # TODO: translate "${cron}" into OnCalendar by hand`}
|
|
106
|
+
Persistent=true
|
|
107
|
+
|
|
108
|
+
[Install]
|
|
109
|
+
WantedBy=timers.target
|
|
110
|
+
`,
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Validate the expression with the same parser that will execute it. */
|
|
115
|
+
export const validateCron = (cron: string): Effect.Effect<void, ScheduleError> =>
|
|
116
|
+
Effect.gen(function* () {
|
|
117
|
+
const next = yield* Effect.try({
|
|
118
|
+
try: () => Bun.cron.parse(cron),
|
|
119
|
+
catch: () => new ScheduleError({ reason: `cannot parse cron expression "${cron}"` }),
|
|
120
|
+
})
|
|
121
|
+
if (next === null) {
|
|
122
|
+
return yield* new ScheduleError({
|
|
123
|
+
reason: `cron expression "${cron}" will never fire`,
|
|
124
|
+
})
|
|
125
|
+
}
|
|
126
|
+
})
|
|
127
|
+
|
|
128
|
+
/** Linux without a crontab binary (Arch family) — an honest error with a recipe. */
|
|
129
|
+
const requireCrontab = (): Effect.Effect<void, ScheduleError> =>
|
|
130
|
+
Effect.gen(function* () {
|
|
131
|
+
if (process.platform !== "linux" || Bun.which("crontab") !== null) return
|
|
132
|
+
return yield* new ScheduleError({
|
|
133
|
+
reason:
|
|
134
|
+
"this machine has no crontab (the Arch family ships no cron daemon): " +
|
|
135
|
+
"install cronie OR use a systemd timer — efmesh schedule <env> --print-systemd",
|
|
136
|
+
})
|
|
137
|
+
})
|
|
138
|
+
|
|
139
|
+
export const registerSchedule = (
|
|
140
|
+
target: ScheduleTarget,
|
|
141
|
+
cron: string,
|
|
142
|
+
): Effect.Effect<{ readonly title: string; readonly worker: string }, ScheduleError> =>
|
|
143
|
+
Effect.gen(function* () {
|
|
144
|
+
yield* validateCron(cron)
|
|
145
|
+
yield* requireCrontab()
|
|
146
|
+
if (!existsSync(target.config)) {
|
|
147
|
+
return yield* new ScheduleError({ reason: `config not found: ${target.config}` })
|
|
148
|
+
}
|
|
149
|
+
const worker = workerPath(target)
|
|
150
|
+
yield* Effect.try({
|
|
151
|
+
try: () => {
|
|
152
|
+
mkdirSync(dirname(worker), { recursive: true })
|
|
153
|
+
writeFileSync(worker, workerSource(target))
|
|
154
|
+
},
|
|
155
|
+
catch: (cause) =>
|
|
156
|
+
new ScheduleError({ reason: `worker was not written: ${String(cause)}` }),
|
|
157
|
+
})
|
|
158
|
+
const title = scheduleTitle(target)
|
|
159
|
+
yield* Effect.tryPromise({
|
|
160
|
+
try: () => Bun.cron(worker, cron, title),
|
|
161
|
+
catch: (cause) =>
|
|
162
|
+
new ScheduleError({ reason: `Bun.cron did not register: ${String(cause)}` }),
|
|
163
|
+
})
|
|
164
|
+
return { title, worker }
|
|
165
|
+
})
|
|
166
|
+
|
|
167
|
+
export const removeSchedule = (
|
|
168
|
+
target: ScheduleTarget,
|
|
169
|
+
): Effect.Effect<{ readonly title: string }, ScheduleError> =>
|
|
170
|
+
Effect.gen(function* () {
|
|
171
|
+
const title = scheduleTitle(target)
|
|
172
|
+
yield* Effect.tryPromise({
|
|
173
|
+
try: () => Bun.cron.remove(title),
|
|
174
|
+
catch: (cause) => new ScheduleError({ reason: `Bun.cron.remove: ${String(cause)}` }),
|
|
175
|
+
})
|
|
176
|
+
// the worker is no longer executed by anyone — clean it up silently
|
|
177
|
+
yield* Effect.sync(() => rmSync(workerPath(target), { force: true }))
|
|
178
|
+
return { title }
|
|
179
|
+
})
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* List of efmesh registrations — by Bun.cron's platform traces: `# bun-cron:
|
|
183
|
+
* <title>` markers in crontab (Linux) / launchd plists (macOS).
|
|
184
|
+
*/
|
|
185
|
+
export const listSchedules = (): Effect.Effect<ReadonlyArray<string>, ScheduleError> =>
|
|
186
|
+
Effect.gen(function* () {
|
|
187
|
+
if (process.platform === "linux") {
|
|
188
|
+
if (Bun.which("crontab") === null) return []
|
|
189
|
+
const out = yield* Effect.tryPromise({
|
|
190
|
+
try: async () => {
|
|
191
|
+
const proc = Bun.spawn(["crontab", "-l"], { stdout: "pipe", stderr: "ignore" })
|
|
192
|
+
const text = await proc.stdout.text()
|
|
193
|
+
await proc.exited // an empty crontab exits with 1 — that is not an error
|
|
194
|
+
return text
|
|
195
|
+
},
|
|
196
|
+
catch: (cause) => new ScheduleError({ reason: `crontab -l: ${String(cause)}` }),
|
|
197
|
+
})
|
|
198
|
+
return out
|
|
199
|
+
.split("\n")
|
|
200
|
+
.filter((line) => line.includes("bun-cron") && line.includes("efmesh-"))
|
|
201
|
+
}
|
|
202
|
+
if (process.platform === "darwin") {
|
|
203
|
+
const out = yield* Effect.tryPromise({
|
|
204
|
+
try: async () => {
|
|
205
|
+
const proc = Bun.spawn(["launchctl", "list"], { stdout: "pipe", stderr: "ignore" })
|
|
206
|
+
const text = await proc.stdout.text()
|
|
207
|
+
await proc.exited
|
|
208
|
+
return text
|
|
209
|
+
},
|
|
210
|
+
catch: (cause) => new ScheduleError({ reason: `launchctl list: ${String(cause)}` }),
|
|
211
|
+
})
|
|
212
|
+
return out.split("\n").filter((line) => line.includes("bun.cron.efmesh-"))
|
|
213
|
+
}
|
|
214
|
+
return yield* new ScheduleError({
|
|
215
|
+
reason: `list on ${process.platform} via the OS tools (Task Scheduler: schtasks /query)`,
|
|
216
|
+
})
|
|
217
|
+
})
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import { Clock, Effect } from "effect"
|
|
2
|
+
import { buildGraph, type GraphError } from "../core/graph.ts"
|
|
3
|
+
import { enumerateIntervals, fromIso, missingIntervals, toIso } from "../core/interval.ts"
|
|
4
|
+
import type { AnyModel } from "../core/model.ts"
|
|
5
|
+
import { STATE_VERSION, StateStore } from "../state/store.ts"
|
|
6
|
+
import type { PlanRecord, RunRecord, StateError } from "../state/store.ts"
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* `efmesh status <env>` (issue #1): one command for the question «what is
|
|
10
|
+
* even going on» — for the nightly-cron operator and for the author who
|
|
11
|
+
* forgot the commands. Read-only: if the store opened → its schema version
|
|
12
|
+
* already matched.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
export interface ModelLag {
|
|
16
|
+
readonly model: string
|
|
17
|
+
/** End of the last done interval (ISO); null — nothing computed yet. */
|
|
18
|
+
readonly doneUpTo: string | null
|
|
19
|
+
/** How many intervals are missing up to «now». 0 — caught up. */
|
|
20
|
+
readonly missing: number
|
|
21
|
+
/** Intervals marked failed — a stuck backfill is visible at once. */
|
|
22
|
+
readonly failed: number
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface StatusReport {
|
|
26
|
+
readonly env: string
|
|
27
|
+
readonly storeVersion: number
|
|
28
|
+
/** Rows in the environment; 0 — the environment does not exist (never applied). */
|
|
29
|
+
readonly models: number
|
|
30
|
+
/** Last promotion (ISO); null — no environment. */
|
|
31
|
+
readonly promotedAt: string | null
|
|
32
|
+
readonly lastPlan: PlanRecord | null
|
|
33
|
+
/** Latest run ticks, freshest first. */
|
|
34
|
+
readonly ticks: ReadonlyArray<RunRecord>
|
|
35
|
+
/** Lag of the environment's incremental models. */
|
|
36
|
+
readonly lag: ReadonlyArray<ModelLag>
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
export interface StatusOptions {
|
|
40
|
+
/** «Now» for computing lag; by default — Clock. Injected for tests. */
|
|
41
|
+
readonly now?: number
|
|
42
|
+
/** How many latest ticks to show; by default 5. */
|
|
43
|
+
readonly ticks?: number
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export const environmentStatus = (
|
|
47
|
+
env: string,
|
|
48
|
+
models: Iterable<AnyModel>,
|
|
49
|
+
options?: StatusOptions,
|
|
50
|
+
): Effect.Effect<StatusReport, GraphError | StateError, StateStore> =>
|
|
51
|
+
Effect.gen(function* () {
|
|
52
|
+
const store = yield* StateStore
|
|
53
|
+
const graph = yield* buildGraph(models)
|
|
54
|
+
const now = options?.now ?? (yield* Clock.currentTimeMillis)
|
|
55
|
+
|
|
56
|
+
const rows = yield* store.getEnvironment(env)
|
|
57
|
+
const promotedAt =
|
|
58
|
+
rows.length === 0
|
|
59
|
+
? null
|
|
60
|
+
: rows.map((row) => row.promotedAt).reduce((a, b) => (a > b ? a : b))
|
|
61
|
+
|
|
62
|
+
const plans = yield* store.listPlans(env)
|
|
63
|
+
const lastPlan = plans.at(-1) ?? null
|
|
64
|
+
const ticks = yield* store.listRuns(env, options?.ticks ?? 5)
|
|
65
|
+
|
|
66
|
+
// lag — by the environment's POINTERS (what is actually served to
|
|
67
|
+
// consumers), not by the project's local fingerprints
|
|
68
|
+
const lag: Array<ModelLag> = []
|
|
69
|
+
for (const row of rows) {
|
|
70
|
+
const model = graph.models.get(row.name)
|
|
71
|
+
if (model === undefined || model.kind._tag !== "incrementalByTimeRange") continue
|
|
72
|
+
const kind = model.kind
|
|
73
|
+
const records = yield* store.listIntervals(row.fingerprint)
|
|
74
|
+
const done = records
|
|
75
|
+
.filter((record) => record.status === "done")
|
|
76
|
+
.map((record) => ({ start: fromIso(record.startTs), end: fromIso(record.endTs) }))
|
|
77
|
+
const wanted = enumerateIntervals(kind.interval, fromIso(kind.start), now)
|
|
78
|
+
const missing = missingIntervals(wanted, done)
|
|
79
|
+
lag.push({
|
|
80
|
+
model: row.name,
|
|
81
|
+
doneUpTo: done.length === 0 ? null : toIso(Math.max(...done.map((i) => i.end))),
|
|
82
|
+
missing: missing.length,
|
|
83
|
+
failed: records.filter((record) => record.status === "failed").length,
|
|
84
|
+
})
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
return {
|
|
88
|
+
env,
|
|
89
|
+
storeVersion: STATE_VERSION,
|
|
90
|
+
models: rows.length,
|
|
91
|
+
promotedAt,
|
|
92
|
+
lastPlan,
|
|
93
|
+
ticks,
|
|
94
|
+
lag,
|
|
95
|
+
}
|
|
96
|
+
})
|
package/src/state/postgres.ts
CHANGED
|
@@ -5,17 +5,18 @@ import type {
|
|
|
5
5
|
IntervalRecord,
|
|
6
6
|
MigrationReport,
|
|
7
7
|
PlanRecord,
|
|
8
|
+
RunRecord,
|
|
8
9
|
SnapshotRecord,
|
|
9
10
|
StateStoreShape,
|
|
10
11
|
} from "./store.ts"
|
|
11
12
|
import { STATE_VERSION, StateError, StateSchemaError, StateStore } from "./store.ts"
|
|
12
13
|
|
|
13
14
|
/**
|
|
14
|
-
* State store
|
|
15
|
-
*
|
|
16
|
-
*
|
|
17
|
-
*
|
|
18
|
-
*
|
|
15
|
+
* State store on Postgres (SPEC §6, F3) — for team/production work: state
|
|
16
|
+
* survives concurrent runs from different processes and machines.
|
|
17
|
+
* Schema `efmesh_state`, semantics identical to the bun:sqlite implementation;
|
|
18
|
+
* timestamps are ISO UTC text (lexicographically sortable), same as in
|
|
19
|
+
* SQLite: the store's contents are portable between backends.
|
|
19
20
|
*/
|
|
20
21
|
|
|
21
22
|
const SCHEMA = `
|
|
@@ -54,6 +55,18 @@ CREATE TABLE IF NOT EXISTS efmesh_state.intervals (
|
|
|
54
55
|
updated_at TEXT NOT NULL,
|
|
55
56
|
PRIMARY KEY (snapshot_fp, start_ts)
|
|
56
57
|
);
|
|
58
|
+
CREATE TABLE IF NOT EXISTS efmesh_state.runs (
|
|
59
|
+
id INTEGER GENERATED ALWAYS AS IDENTITY PRIMARY KEY,
|
|
60
|
+
env TEXT NOT NULL,
|
|
61
|
+
started_at TEXT NOT NULL,
|
|
62
|
+
finished_at TEXT NOT NULL,
|
|
63
|
+
outcome TEXT NOT NULL,
|
|
64
|
+
detail TEXT NOT NULL DEFAULT ''
|
|
65
|
+
);
|
|
66
|
+
CREATE TABLE IF NOT EXISTS efmesh_state.canon_cache (
|
|
67
|
+
key TEXT PRIMARY KEY,
|
|
68
|
+
canonical TEXT NOT NULL
|
|
69
|
+
);
|
|
57
70
|
CREATE TABLE IF NOT EXISTS efmesh_state.locks (
|
|
58
71
|
name TEXT PRIMARY KEY,
|
|
59
72
|
acquired_at TEXT NOT NULL,
|
|
@@ -62,9 +75,9 @@ CREATE TABLE IF NOT EXISTS efmesh_state.locks (
|
|
|
62
75
|
`
|
|
63
76
|
|
|
64
77
|
export interface PostgresStateOptions {
|
|
65
|
-
/** postgres://…
|
|
78
|
+
/** postgres://… or a unix socket via ?host=/path. */
|
|
66
79
|
readonly url: string
|
|
67
|
-
/**
|
|
80
|
+
/** Pool size; a couple of connections are enough for state. */
|
|
68
81
|
readonly max?: number
|
|
69
82
|
}
|
|
70
83
|
|
|
@@ -75,7 +88,7 @@ const regclass = async (pool: SQL, table: string): Promise<boolean> => {
|
|
|
75
88
|
return rows[0]?.r != null
|
|
76
89
|
}
|
|
77
90
|
|
|
78
|
-
/** 0 —
|
|
91
|
+
/** 0 — a store with no meta table (created before versioning existed, F0–F3). */
|
|
79
92
|
const readVersion = async (pool: SQL): Promise<number> => {
|
|
80
93
|
if (!(await regclass(pool, "meta"))) return 0
|
|
81
94
|
const rows = (await pool.unsafe(
|
|
@@ -84,7 +97,7 @@ const readVersion = async (pool: SQL): Promise<number> => {
|
|
|
84
97
|
return rows[0]?.version ?? 0
|
|
85
98
|
}
|
|
86
99
|
|
|
87
|
-
/**
|
|
100
|
+
/** Catches the schema up to STATE_VERSION (see the sqlite implementation — same semantics). */
|
|
88
101
|
const applyMigrations = async (pool: SQL): Promise<void> => {
|
|
89
102
|
await pool.unsafe(SCHEMA)
|
|
90
103
|
await pool.unsafe(`
|
|
@@ -101,7 +114,7 @@ const applyMigrations = async (pool: SQL): Promise<void> => {
|
|
|
101
114
|
})
|
|
102
115
|
}
|
|
103
116
|
|
|
104
|
-
/** `efmesh migrate`:
|
|
117
|
+
/** `efmesh migrate`: an explicit schema upgrade of an existing store. */
|
|
105
118
|
export const migratePostgresState = (
|
|
106
119
|
options: PostgresStateOptions,
|
|
107
120
|
): Effect.Effect<MigrationReport, StateError> =>
|
|
@@ -140,8 +153,8 @@ export const PostgresStateLive = (
|
|
|
140
153
|
}),
|
|
141
154
|
(pool) => Effect.promise(() => pool.end()).pipe(Effect.ignore),
|
|
142
155
|
)
|
|
143
|
-
//
|
|
144
|
-
//
|
|
156
|
+
// a fresh store bootstraps at the current version; an existing store
|
|
157
|
+
// with an older schema requires an explicit `efmesh migrate` (SPEC §6)
|
|
145
158
|
const fresh = yield* Effect.tryPromise({
|
|
146
159
|
try: async () => !(await regclass(sql, "snapshots")),
|
|
147
160
|
catch: (cause) => new StateError({ operation: "open", cause }),
|
|
@@ -177,7 +190,7 @@ export const PostgresStateLive = (
|
|
|
177
190
|
Effect.flatMap((now) =>
|
|
178
191
|
attempt("upsertSnapshot", async () => {
|
|
179
192
|
await sql.unsafe(
|
|
180
|
-
//
|
|
193
|
+
// reviving clears orphan status and refreshes created_at — same as sqlite (race, F6)
|
|
181
194
|
`INSERT INTO efmesh_state.snapshots
|
|
182
195
|
(name, fingerprint, rendered_sql, canonical_ast, physical_fp, kind, fingerprint_version, created_at)
|
|
183
196
|
VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
|
|
@@ -271,8 +284,8 @@ export const PostgresStateLive = (
|
|
|
271
284
|
Effect.flatMap((now) =>
|
|
272
285
|
attempt("promote", () =>
|
|
273
286
|
sql.begin(async (tx) => {
|
|
274
|
-
//
|
|
275
|
-
// janitor
|
|
287
|
+
// snapshot liveness — in the same transaction (race, F6):
|
|
288
|
+
// janitor removed the version → loud error, the view is left alone
|
|
276
289
|
for (const entry of entries) {
|
|
277
290
|
if (entry.requireSnapshot !== true) continue
|
|
278
291
|
const alive = (await tx.unsafe(
|
|
@@ -281,7 +294,7 @@ export const PostgresStateLive = (
|
|
|
281
294
|
)) as ReadonlyArray<unknown>
|
|
282
295
|
if (alive.length === 0) {
|
|
283
296
|
throw new Error(
|
|
284
|
-
|
|
297
|
+
`promotion "${env}": snapshot ${entry.name}@${entry.fingerprint.slice(0, 8)} vanished from the store (removed by janitor?) — retry apply`,
|
|
285
298
|
)
|
|
286
299
|
}
|
|
287
300
|
}
|
|
@@ -293,7 +306,7 @@ export const PostgresStateLive = (
|
|
|
293
306
|
[env, entry.name, entry.fingerprint, now],
|
|
294
307
|
)
|
|
295
308
|
}
|
|
296
|
-
//
|
|
309
|
+
// orphan bookkeeping — same as the sqlite implementation (SPEC §5.4)
|
|
297
310
|
await tx.unsafe(
|
|
298
311
|
`UPDATE efmesh_state.snapshots SET orphaned_at = $1
|
|
299
312
|
WHERE orphaned_at IS NULL
|
|
@@ -331,6 +344,42 @@ export const PostgresStateLive = (
|
|
|
331
344
|
)) as ReadonlyArray<PlanRecord>
|
|
332
345
|
}),
|
|
333
346
|
|
|
347
|
+
getCanon: (key) =>
|
|
348
|
+
attempt("getCanon", async () => {
|
|
349
|
+
const rows = (await sql.unsafe(
|
|
350
|
+
`SELECT canonical FROM efmesh_state.canon_cache WHERE key = $1`,
|
|
351
|
+
[key],
|
|
352
|
+
)) as ReadonlyArray<{ canonical: string }>
|
|
353
|
+
return rows[0]?.canonical
|
|
354
|
+
}),
|
|
355
|
+
|
|
356
|
+
putCanon: (key, canonical) =>
|
|
357
|
+
attempt("putCanon", async () => {
|
|
358
|
+
await sql.unsafe(
|
|
359
|
+
`INSERT INTO efmesh_state.canon_cache (key, canonical) VALUES ($1, $2)
|
|
360
|
+
ON CONFLICT (key) DO NOTHING`,
|
|
361
|
+
[key, canonical],
|
|
362
|
+
)
|
|
363
|
+
}),
|
|
364
|
+
|
|
365
|
+
recordRun: (record) =>
|
|
366
|
+
attempt("recordRun", async () => {
|
|
367
|
+
await sql.unsafe(
|
|
368
|
+
`INSERT INTO efmesh_state.runs (env, started_at, finished_at, outcome, detail)
|
|
369
|
+
VALUES ($1, $2, $3, $4, $5)`,
|
|
370
|
+
[record.env, record.startedAt, record.finishedAt, record.outcome, record.detail],
|
|
371
|
+
)
|
|
372
|
+
}),
|
|
373
|
+
|
|
374
|
+
listRuns: (env, limit) =>
|
|
375
|
+
attempt("listRuns", async () => {
|
|
376
|
+
return (await sql.unsafe(
|
|
377
|
+
`SELECT id, env, started_at AS "startedAt", finished_at AS "finishedAt", outcome, detail
|
|
378
|
+
FROM efmesh_state.runs WHERE env = $1 ORDER BY id DESC LIMIT $2`,
|
|
379
|
+
[env, limit],
|
|
380
|
+
)) as ReadonlyArray<RunRecord>
|
|
381
|
+
}),
|
|
382
|
+
|
|
334
383
|
acquireLock: (name, ttlMs) =>
|
|
335
384
|
Clock.currentTimeMillis.pipe(
|
|
336
385
|
Effect.flatMap((nowMs) =>
|
|
@@ -338,8 +387,8 @@ export const PostgresStateLive = (
|
|
|
338
387
|
sql.begin(async (tx) => {
|
|
339
388
|
const now = new Date(nowMs).toISOString()
|
|
340
389
|
const expires = new Date(nowMs + ttlMs).toISOString()
|
|
341
|
-
//
|
|
342
|
-
// <= —
|
|
390
|
+
// a stale lock from a crashed process is reclaimed;
|
|
391
|
+
// <= — a lock that expires at instant T is free as of T
|
|
343
392
|
await tx.unsafe(
|
|
344
393
|
`DELETE FROM efmesh_state.locks WHERE name = $1 AND expires_at <= $2`,
|
|
345
394
|
[name, now],
|