@avytheone/efmesh 0.1.0-beta.2 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +171 -92
- package/README.md +83 -9
- package/README.ru.md +80 -5
- package/SPEC.md +542 -459
- package/package.json +4 -1
- package/src/bin.ts +39 -2
- package/src/cli.ts +505 -85
- package/src/config.ts +15 -15
- package/src/core/audit.ts +15 -11
- package/src/core/errors.ts +37 -7
- package/src/core/graph.ts +6 -6
- package/src/core/interval.ts +18 -18
- package/src/core/model.ts +76 -75
- package/src/core/sql.ts +21 -21
- package/src/discovery.ts +16 -8
- package/src/efmesh.ts +28 -15
- package/src/engine/adapter.ts +29 -12
- package/src/engine/duckdb.ts +7 -7
- package/src/engine/postgres.ts +17 -17
- package/src/error-text.ts +35 -0
- package/src/index.ts +44 -13
- package/src/init.ts +100 -28
- package/src/plan/audit-run.ts +21 -13
- package/src/plan/categorize.ts +10 -7
- package/src/plan/contract.ts +23 -19
- package/src/plan/diff.ts +228 -4
- package/src/plan/executor.ts +223 -121
- package/src/plan/explain.ts +114 -0
- package/src/plan/fingerprint.ts +84 -31
- package/src/plan/graph-html.ts +7 -7
- package/src/plan/janitor.ts +30 -25
- package/src/plan/lineage.ts +20 -16
- package/src/plan/lock.ts +27 -10
- package/src/plan/metrics.ts +4 -4
- package/src/plan/naming.ts +23 -23
- package/src/plan/planner.ts +238 -52
- package/src/plan/run.ts +72 -26
- package/src/plan/schedule.ts +217 -0
- package/src/plan/status.ts +96 -0
- package/src/state/postgres.ts +68 -19
- package/src/state/sqlite.ts +74 -27
- package/src/state/store.ts +88 -45
- package/src/testing/index.ts +27 -26
package/src/cli.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { writeFileSync } from "node:fs"
|
|
2
2
|
import * as NodePath from "node:path"
|
|
3
|
-
import { Console, Data, Effect, Layer } from "effect"
|
|
3
|
+
import { Cause, Console, Data, Effect, Layer } from "effect"
|
|
4
4
|
import { Argument, Command, Flag } from "effect/unstable/cli"
|
|
5
5
|
import type { EfmeshConfig } from "./config.ts"
|
|
6
6
|
import { buildGraph } from "./core/graph.ts"
|
|
@@ -12,24 +12,41 @@ import { DuckDBEngineLive } from "./engine/duckdb.ts"
|
|
|
12
12
|
import { PostgresEngineLive } from "./engine/postgres.ts"
|
|
13
13
|
import { PostgresStateLive } from "./state/postgres.ts"
|
|
14
14
|
import { auditEnvironment, EnvironmentAuditError } from "./plan/audit-run.ts"
|
|
15
|
-
import {
|
|
15
|
+
import {
|
|
16
|
+
dataDiffEnvironments,
|
|
17
|
+
diffEnvironments,
|
|
18
|
+
type DataDiffReport,
|
|
19
|
+
} from "./plan/diff.ts"
|
|
20
|
+
import { EngineAdapter } from "./engine/adapter.ts"
|
|
21
|
+
import { ducklakeAttachSql } from "./plan/naming.ts"
|
|
22
|
+
import {
|
|
23
|
+
listSchedules,
|
|
24
|
+
registerSchedule,
|
|
25
|
+
removeSchedule,
|
|
26
|
+
systemdUnits,
|
|
27
|
+
} from "./plan/schedule.ts"
|
|
16
28
|
import { renderGraphHtml } from "./plan/graph-html.ts"
|
|
17
29
|
import { formatLineage, lineage, LineageError } from "./plan/lineage.ts"
|
|
30
|
+
import { environmentStatus } from "./plan/status.ts"
|
|
18
31
|
import { janitor } from "./plan/janitor.ts"
|
|
19
32
|
import { fp8 } from "./plan/naming.ts"
|
|
20
33
|
import { envLockName, withStateLock } from "./plan/lock.ts"
|
|
21
34
|
import { applyPlan } from "./plan/executor.ts"
|
|
22
35
|
import { run } from "./plan/run.ts"
|
|
23
|
-
import { planChanges, type Plan } from "./plan/planner.ts"
|
|
36
|
+
import { planChanges, ReclassifyError, type Plan } from "./plan/planner.ts"
|
|
24
37
|
import { migratePostgresState } from "./state/postgres.ts"
|
|
25
38
|
import { migrateSqliteState, SqliteStateLive } from "./state/sqlite.ts"
|
|
26
39
|
|
|
27
40
|
export class ConfigLoadError extends Data.TaggedError("ConfigLoadError")<{
|
|
28
41
|
readonly path: string
|
|
29
42
|
readonly reason: string
|
|
30
|
-
}> {
|
|
43
|
+
}> {
|
|
44
|
+
override get message(): string {
|
|
45
|
+
return `config ${this.path}: ${this.reason}`
|
|
46
|
+
}
|
|
47
|
+
}
|
|
31
48
|
|
|
32
|
-
/**
|
|
49
|
+
/** Config with an already-assembled model list: explicit ones + discovery finds. */
|
|
33
50
|
type LoadedConfig = EfmeshConfig & { readonly models: ReadonlyArray<AnyModel> }
|
|
34
51
|
|
|
35
52
|
const loadConfig = (
|
|
@@ -44,7 +61,7 @@ const loadConfig = (
|
|
|
44
61
|
module.default === undefined ||
|
|
45
62
|
(!Array.isArray(module.default.models) && module.default.discovery === undefined)
|
|
46
63
|
) {
|
|
47
|
-
throw new Error("
|
|
64
|
+
throw new Error("config must export a default with models and/or discovery")
|
|
48
65
|
}
|
|
49
66
|
return module.default
|
|
50
67
|
},
|
|
@@ -52,10 +69,10 @@ const loadConfig = (
|
|
|
52
69
|
})
|
|
53
70
|
const explicit = config.models ?? []
|
|
54
71
|
if (config.discovery === undefined) return { ...config, models: explicit }
|
|
55
|
-
//
|
|
72
|
+
// globs are relative to the config: the project is portable regardless of cwd
|
|
56
73
|
const discovered = yield* discoverModels(config.discovery, NodePath.dirname(absolute))
|
|
57
74
|
const seen = new Set(explicit)
|
|
58
|
-
const names = new Map(explicit.map((model) => [model.name.full, "models
|
|
75
|
+
const names = new Map(explicit.map((model) => [model.name.full, "config models"]))
|
|
59
76
|
const merged = [...explicit]
|
|
60
77
|
for (const model of discovered) {
|
|
61
78
|
if (seen.has(model)) continue
|
|
@@ -69,7 +86,7 @@ const loadConfig = (
|
|
|
69
86
|
return { ...config, models: merged }
|
|
70
87
|
})
|
|
71
88
|
|
|
72
|
-
/**
|
|
89
|
+
/** Engine and state layers from the config — shared by plan/apply. */
|
|
73
90
|
const configLayers = (config: EfmeshConfig) =>
|
|
74
91
|
Layer.mergeAll(
|
|
75
92
|
config.engine?.url !== undefined
|
|
@@ -85,7 +102,7 @@ const configLayers = (config: EfmeshConfig) =>
|
|
|
85
102
|
|
|
86
103
|
const configFlag = Flag.string("config").pipe(
|
|
87
104
|
Flag.withDefault("efmesh.config.ts"),
|
|
88
|
-
Flag.withDescription("
|
|
105
|
+
Flag.withDescription("path to efmesh.config.ts"),
|
|
89
106
|
)
|
|
90
107
|
|
|
91
108
|
const CHANGE_MARK: Record<string, string> = {
|
|
@@ -101,14 +118,21 @@ const CHANGE_MARK: Record<string, string> = {
|
|
|
101
118
|
const forwardOnlyFlag = Flag.string("forward-only").pipe(
|
|
102
119
|
Flag.withDefault(""),
|
|
103
120
|
Flag.withDescription(
|
|
104
|
-
"
|
|
121
|
+
"comma-separated models: changes apply forward-only — physics and history are reused",
|
|
122
|
+
),
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
const reclassifyFlag = Flag.string("reclassify").pipe(
|
|
126
|
+
Flag.withDefault(""),
|
|
127
|
+
Flag.withDescription(
|
|
128
|
+
'override categorization (#5): "model=breaking|non-breaking[,…]" on top of --explain; journaled with applied_by',
|
|
105
129
|
),
|
|
106
130
|
)
|
|
107
131
|
|
|
108
132
|
const jobsFlag = Flag.string("jobs").pipe(
|
|
109
133
|
Flag.withDefault(""),
|
|
110
134
|
Flag.withDescription(
|
|
111
|
-
"
|
|
135
|
+
"how many models to build at once (DAG concurrency; always 1 on DuckDB)",
|
|
112
136
|
),
|
|
113
137
|
)
|
|
114
138
|
|
|
@@ -120,7 +144,7 @@ const parseJobs = (value: string): number | undefined => {
|
|
|
120
144
|
const retriesFlag = Flag.string("retries").pipe(
|
|
121
145
|
Flag.withDefault(""),
|
|
122
146
|
Flag.withDescription(
|
|
123
|
-
"
|
|
147
|
+
"how many times to retry a failed backfill batch (exponential backoff; default 0)",
|
|
124
148
|
),
|
|
125
149
|
)
|
|
126
150
|
|
|
@@ -139,27 +163,56 @@ const parseForwardOnly = (value: string): ReadonlyArray<string> | undefined => {
|
|
|
139
163
|
return names.length > 0 ? names : undefined
|
|
140
164
|
}
|
|
141
165
|
|
|
166
|
+
/** `model=breaking|non-breaking[,…]` → record for PlanOptions.reclassify (#5). */
|
|
167
|
+
export const parseReclassify = (
|
|
168
|
+
value: string,
|
|
169
|
+
): Effect.Effect<Readonly<Record<string, "breaking" | "non-breaking">> | undefined, ReclassifyError> =>
|
|
170
|
+
Effect.gen(function* () {
|
|
171
|
+
const entries = value
|
|
172
|
+
.split(",")
|
|
173
|
+
.map((entry) => entry.trim())
|
|
174
|
+
.filter((entry) => entry !== "")
|
|
175
|
+
if (entries.length === 0) return undefined
|
|
176
|
+
const parsed: Record<string, "breaking" | "non-breaking"> = {}
|
|
177
|
+
for (const entry of entries) {
|
|
178
|
+
const [model, category, ...extra] = entry.split("=")
|
|
179
|
+
if (
|
|
180
|
+
model === undefined ||
|
|
181
|
+
model === "" ||
|
|
182
|
+
extra.length > 0 ||
|
|
183
|
+
(category !== "breaking" && category !== "non-breaking")
|
|
184
|
+
) {
|
|
185
|
+
return yield* new ReclassifyError({
|
|
186
|
+
model: entry,
|
|
187
|
+
reason: 'expected "model=breaking" or "model=non-breaking"',
|
|
188
|
+
})
|
|
189
|
+
}
|
|
190
|
+
parsed[model] = category
|
|
191
|
+
}
|
|
192
|
+
return parsed
|
|
193
|
+
})
|
|
194
|
+
|
|
142
195
|
const yesFlag = Flag.boolean("yes").pipe(
|
|
143
196
|
Flag.withAlias("y"),
|
|
144
|
-
Flag.withDescription("
|
|
197
|
+
Flag.withDescription("apply without confirmation (non-TTY never asks)"),
|
|
145
198
|
)
|
|
146
199
|
|
|
147
|
-
/** y/yes
|
|
200
|
+
/** Accepts y/yes, case-insensitive; anything else (including empty) is a refusal. */
|
|
148
201
|
export const isAffirmative = (answer: string | null): boolean =>
|
|
149
|
-
["y", "yes"
|
|
202
|
+
["y", "yes"].includes((answer ?? "").trim().toLowerCase())
|
|
150
203
|
|
|
151
204
|
/**
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
205
|
+
* The "work awaits a human" exit code (F6): the plan needs confirmation in a
|
|
206
|
+
* non-TTY, or run hit structural changes. Alerting must distinguish this
|
|
207
|
+
* normal state from real errors (code 1).
|
|
155
208
|
*/
|
|
156
209
|
export const EXIT_AWAITING_HUMAN = 2
|
|
157
210
|
|
|
158
211
|
/**
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
*
|
|
162
|
-
*
|
|
212
|
+
* The fate of a shown plan (SPEC §5.1, tightened in F6): no changes or
|
|
213
|
+
* --yes — apply; changes in a TTY — ask the human; changes in a
|
|
214
|
+
* non-TTY (CI, cron, pipe) — REFUSE: silently applying a plan nobody
|
|
215
|
+
* saw is forbidden, an explicit --yes is required.
|
|
163
216
|
*/
|
|
164
217
|
export const decideApply = (
|
|
165
218
|
hasChanges: boolean,
|
|
@@ -167,24 +220,76 @@ export const decideApply = (
|
|
|
167
220
|
tty: boolean,
|
|
168
221
|
): "apply" | "ask" | "refuse" => (!hasChanges || yes ? "apply" : tty ? "ask" : "refuse")
|
|
169
222
|
|
|
223
|
+
const jsonFlag = Flag.boolean("json").pipe(
|
|
224
|
+
Flag.withDescription("machine-readable output (stable shape — a contract for CI)"),
|
|
225
|
+
)
|
|
226
|
+
|
|
227
|
+
const explainFlag = Flag.boolean("explain").pipe(
|
|
228
|
+
Flag.withDescription(
|
|
229
|
+
"for each change — which canonical AST nodes diverged and why the category is what it is",
|
|
230
|
+
),
|
|
231
|
+
)
|
|
232
|
+
|
|
233
|
+
/**
|
|
234
|
+
* JSON shape of the plan (#3) — a CONTRACT for CI and bots: shape changes
|
|
235
|
+
* are package semver events. Intervals are ISO UTC, not epoch ms.
|
|
236
|
+
*/
|
|
237
|
+
export const planToJson = (plan: Plan): unknown => ({
|
|
238
|
+
env: plan.env,
|
|
239
|
+
hasChanges: plan.hasChanges,
|
|
240
|
+
actions: plan.actions.map((action) => ({
|
|
241
|
+
name: action.name,
|
|
242
|
+
change: action.change,
|
|
243
|
+
// operator override (#5) and physical reuse — additive contract fields
|
|
244
|
+
...(action.reclassifiedFrom !== undefined
|
|
245
|
+
? { reclassifiedFrom: action.reclassifiedFrom }
|
|
246
|
+
: {}),
|
|
247
|
+
...(action.reusedFrom !== undefined ? { reusedFrom: action.reusedFrom } : {}),
|
|
248
|
+
fingerprint: action.fingerprint,
|
|
249
|
+
build: action.build,
|
|
250
|
+
backfill: action.backfill.map((range) => ({
|
|
251
|
+
start: new Date(range.start).toISOString(),
|
|
252
|
+
end: new Date(range.end).toISOString(),
|
|
253
|
+
})),
|
|
254
|
+
// category reason (#4); diverged paths are a debug hint, not a contract
|
|
255
|
+
...(action.explain !== undefined ? { explain: action.explain } : {}),
|
|
256
|
+
})),
|
|
257
|
+
})
|
|
258
|
+
|
|
259
|
+
const printJson = (payload: unknown) => Console.log(JSON.stringify(payload, null, 2))
|
|
260
|
+
|
|
170
261
|
const formatRange = (range: { readonly start: number; readonly end: number }): string =>
|
|
171
|
-
`[${new Date(range.start).toISOString().slice(0, 10)}
|
|
262
|
+
`[${new Date(range.start).toISOString().slice(0, 10)}, ${new Date(range.end).toISOString().slice(0, 10)})`
|
|
172
263
|
|
|
173
|
-
const printPlan = (plan: Plan) =>
|
|
264
|
+
const printPlan = (plan: Plan, explain = false) =>
|
|
174
265
|
Effect.gen(function* () {
|
|
175
|
-
yield* Console.log(
|
|
266
|
+
yield* Console.log(`plan for environment "${plan.env}":`)
|
|
176
267
|
for (const action of plan.actions) {
|
|
177
268
|
const mark = CHANGE_MARK[action.change] ?? "?"
|
|
178
|
-
const
|
|
269
|
+
const overridden =
|
|
270
|
+
action.reclassifiedFrom !== undefined
|
|
271
|
+
? ` [override: was ${action.reclassifiedFrom}]`
|
|
272
|
+
: ""
|
|
273
|
+
const reused =
|
|
274
|
+
action.change === "indirect" && action.reusedFrom !== undefined
|
|
275
|
+
? " [physics reused]"
|
|
276
|
+
: ""
|
|
277
|
+
const build = action.build ? " [build]" : ""
|
|
179
278
|
const backfill =
|
|
180
279
|
action.backfill.length > 0
|
|
181
|
-
? `
|
|
280
|
+
? ` backfill ${action.backfill.map(formatRange).join(", ")}`
|
|
182
281
|
: ""
|
|
183
282
|
yield* Console.log(
|
|
184
|
-
` ${mark} ${action.name} ${action.change} @${fp8(action.fingerprint)}${build}${backfill}`,
|
|
283
|
+
` ${mark} ${action.name} ${action.change} @${fp8(action.fingerprint)}${overridden}${reused}${build}${backfill}`,
|
|
185
284
|
)
|
|
285
|
+
if (explain && action.explain !== undefined) {
|
|
286
|
+
yield* Console.log(` why: ${action.explain.reason}`)
|
|
287
|
+
if (action.explain.diverged.length > 0) {
|
|
288
|
+
yield* Console.log(` diverged: ${action.explain.diverged.join(", ")}`)
|
|
289
|
+
}
|
|
290
|
+
}
|
|
186
291
|
}
|
|
187
|
-
if (!plan.hasChanges) yield* Console.log("
|
|
292
|
+
if (!plan.hasChanges) yield* Console.log(" no changes")
|
|
188
293
|
})
|
|
189
294
|
|
|
190
295
|
const initCommand = Command.make(
|
|
@@ -193,24 +298,33 @@ const initCommand = Command.make(
|
|
|
193
298
|
({ dir }) =>
|
|
194
299
|
Effect.gen(function* () {
|
|
195
300
|
const created = yield* scaffold(dir)
|
|
196
|
-
for (const file of created) yield* Console.log(
|
|
197
|
-
yield* Console.log("
|
|
301
|
+
for (const file of created) yield* Console.log(`created ${file}`)
|
|
302
|
+
yield* Console.log("next: bunx efmesh plan dev && bunx efmesh apply dev")
|
|
198
303
|
}),
|
|
199
|
-
).pipe(Command.withDescription("
|
|
304
|
+
).pipe(Command.withDescription("scaffold a project: config, example models, seed"))
|
|
200
305
|
|
|
201
306
|
const planCommand = Command.make(
|
|
202
307
|
"plan",
|
|
203
|
-
{
|
|
204
|
-
|
|
308
|
+
{
|
|
309
|
+
env: Argument.string("env"),
|
|
310
|
+
config: configFlag,
|
|
311
|
+
forwardOnly: forwardOnlyFlag,
|
|
312
|
+
reclassify: reclassifyFlag,
|
|
313
|
+
json: jsonFlag,
|
|
314
|
+
explain: explainFlag,
|
|
315
|
+
},
|
|
316
|
+
({ config, env, explain, forwardOnly, json, reclassify }) =>
|
|
205
317
|
Effect.gen(function* () {
|
|
206
318
|
const loaded = yield* loadConfig(config)
|
|
207
319
|
const names = parseForwardOnly(forwardOnly)
|
|
320
|
+
const overrides = yield* parseReclassify(reclassify)
|
|
208
321
|
const plan = yield* Efmesh.plan(env, loaded.models, {
|
|
209
322
|
...(names !== undefined ? { forwardOnly: names } : {}),
|
|
323
|
+
...(overrides !== undefined ? { reclassify: overrides } : {}),
|
|
210
324
|
}).pipe(Effect.provide(configLayers(loaded)))
|
|
211
|
-
yield* printPlan(plan)
|
|
325
|
+
yield* json ? printJson(planToJson(plan)) : printPlan(plan, explain)
|
|
212
326
|
}),
|
|
213
|
-
).pipe(Command.withDescription("
|
|
327
|
+
).pipe(Command.withDescription("show the project diff against an environment, changing nothing"))
|
|
214
328
|
|
|
215
329
|
const applyCommand = Command.make(
|
|
216
330
|
"apply",
|
|
@@ -218,30 +332,33 @@ const applyCommand = Command.make(
|
|
|
218
332
|
env: Argument.string("env"),
|
|
219
333
|
config: configFlag,
|
|
220
334
|
forwardOnly: forwardOnlyFlag,
|
|
335
|
+
reclassify: reclassifyFlag,
|
|
221
336
|
jobs: jobsFlag,
|
|
222
337
|
retries: retriesFlag,
|
|
223
338
|
yes: yesFlag,
|
|
224
339
|
},
|
|
225
|
-
({ config, env, forwardOnly, jobs, retries, yes }) =>
|
|
340
|
+
({ config, env, forwardOnly, jobs, reclassify, retries, yes }) =>
|
|
226
341
|
Effect.gen(function* () {
|
|
227
342
|
const loaded = yield* loadConfig(config)
|
|
228
343
|
const names = parseForwardOnly(forwardOnly)
|
|
344
|
+
const overrides = yield* parseReclassify(reclassify)
|
|
229
345
|
const modelConcurrency = parseJobs(jobs)
|
|
230
346
|
const retry = parseRetries(retries)
|
|
231
|
-
//
|
|
232
|
-
//
|
|
233
|
-
// (
|
|
234
|
-
//
|
|
347
|
+
// plan and apply — under one layer and one cross-process lock:
|
|
348
|
+
// exactly the plan that was shown and confirmed gets applied, and no one
|
|
349
|
+
// (a second apply, cron with run) wedges in between them (SPEC §14.6);
|
|
350
|
+
// the cost — the lock is held even while the human ponders confirmation
|
|
235
351
|
yield* Effect.gen(function* () {
|
|
236
352
|
const graph = yield* buildGraph(loaded.models)
|
|
237
353
|
const plan = yield* planChanges(env, graph, {
|
|
238
354
|
...(names !== undefined ? { forwardOnly: names } : {}),
|
|
355
|
+
...(overrides !== undefined ? { reclassify: overrides } : {}),
|
|
239
356
|
})
|
|
240
357
|
yield* printPlan(plan)
|
|
241
358
|
const decision = decideApply(plan.hasChanges, yes, process.stdin.isTTY === true)
|
|
242
359
|
if (decision === "refuse") {
|
|
243
360
|
yield* Console.error(
|
|
244
|
-
"
|
|
361
|
+
"the plan changes models but there is no one to confirm (non-TTY): add --yes",
|
|
245
362
|
)
|
|
246
363
|
yield* Effect.sync(() => {
|
|
247
364
|
process.exitCode = EXIT_AWAITING_HUMAN
|
|
@@ -250,9 +367,9 @@ const applyCommand = Command.make(
|
|
|
250
367
|
}
|
|
251
368
|
if (
|
|
252
369
|
decision === "ask" &&
|
|
253
|
-
!isAffirmative(globalThis.prompt("
|
|
370
|
+
!isAffirmative(globalThis.prompt("apply the plan? [y/N]"))
|
|
254
371
|
) {
|
|
255
|
-
yield* Console.log("
|
|
372
|
+
yield* Console.log("apply cancelled")
|
|
256
373
|
return
|
|
257
374
|
}
|
|
258
375
|
const applied = yield* applyPlan(plan, graph, {
|
|
@@ -264,13 +381,13 @@ const applyCommand = Command.make(
|
|
|
264
381
|
})
|
|
265
382
|
yield* Console.log(
|
|
266
383
|
applied.built.length > 0
|
|
267
|
-
?
|
|
268
|
-
: "
|
|
384
|
+
? `built: ${applied.built.join(", ")}`
|
|
385
|
+
: "no build needed (view-swap only)",
|
|
269
386
|
)
|
|
270
|
-
yield* Console.log(
|
|
387
|
+
yield* Console.log(`environment "${applied.plan.env}" promoted`)
|
|
271
388
|
}).pipe(withStateLock(envLockName(env)), Effect.provide(configLayers(loaded)))
|
|
272
389
|
}),
|
|
273
|
-
).pipe(Command.withDescription("
|
|
390
|
+
).pipe(Command.withDescription("apply the plan: build physics and swap views"))
|
|
274
391
|
|
|
275
392
|
const renderCommand = Command.make(
|
|
276
393
|
"render",
|
|
@@ -279,7 +396,7 @@ const renderCommand = Command.make(
|
|
|
279
396
|
config: configFlag,
|
|
280
397
|
env: Flag.string("env").pipe(
|
|
281
398
|
Flag.withDefault(""),
|
|
282
|
-
Flag.withDescription("
|
|
399
|
+
Flag.withDescription("render against an environment's view layer instead of logical names"),
|
|
283
400
|
),
|
|
284
401
|
},
|
|
285
402
|
({ config, env, model }) =>
|
|
@@ -290,7 +407,7 @@ const renderCommand = Command.make(
|
|
|
290
407
|
: yield* Efmesh.renderFor(loaded.models, model, env)
|
|
291
408
|
yield* Console.log(sql.trim())
|
|
292
409
|
}),
|
|
293
|
-
).pipe(Command.withDescription("
|
|
410
|
+
).pipe(Command.withDescription("show a model's final SQL"))
|
|
294
411
|
|
|
295
412
|
const runCommand = Command.make(
|
|
296
413
|
"run",
|
|
@@ -308,12 +425,12 @@ const runCommand = Command.make(
|
|
|
308
425
|
...(retry !== undefined ? { retry } : {}),
|
|
309
426
|
}).pipe(
|
|
310
427
|
Effect.provide(configLayers(loaded)),
|
|
311
|
-
//
|
|
312
|
-
//
|
|
428
|
+
// structural changes are the normal "awaits a human with apply", not a failure:
|
|
429
|
+
// alerting tells it apart by exit code 2 (F6)
|
|
313
430
|
Effect.catchTag("RunBlockedByChangesError", (blocked) =>
|
|
314
431
|
Effect.gen(function* () {
|
|
315
432
|
yield* Console.error(
|
|
316
|
-
`run ${blocked.env}:
|
|
433
|
+
`run "${blocked.env}": unapplied changes present — apply needed:\n ${blocked.changes.join("\n ")}`,
|
|
317
434
|
)
|
|
318
435
|
yield* Effect.sync(() => {
|
|
319
436
|
process.exitCode = EXIT_AWAITING_HUMAN
|
|
@@ -325,35 +442,198 @@ const runCommand = Command.make(
|
|
|
325
442
|
if (applied === undefined) return
|
|
326
443
|
yield* Console.log(
|
|
327
444
|
applied.built.length > 0
|
|
328
|
-
?
|
|
329
|
-
: "
|
|
445
|
+
? `processed: ${applied.built.join(", ")}`
|
|
446
|
+
: "no new intervals",
|
|
330
447
|
)
|
|
331
448
|
}),
|
|
332
449
|
).pipe(
|
|
333
450
|
Command.withDescription(
|
|
334
|
-
"
|
|
451
|
+
"scheduler tick: catch up intervals of existing versions (changes go through apply)",
|
|
335
452
|
),
|
|
336
453
|
)
|
|
337
454
|
|
|
455
|
+
const printDataDiff = (report: DataDiffReport) =>
|
|
456
|
+
Effect.gen(function* () {
|
|
457
|
+
yield* Console.log(`data "${report.envA}" ↔ "${report.envB}":`)
|
|
458
|
+
if (report.models.length === 0) {
|
|
459
|
+
yield* Console.log(" no shared materializable models")
|
|
460
|
+
return
|
|
461
|
+
}
|
|
462
|
+
for (const entry of report.models) {
|
|
463
|
+
if (entry.key === undefined) {
|
|
464
|
+
yield* Console.log(
|
|
465
|
+
` · ${entry.model} A=${entry.rowsA} B=${entry.rowsB} no key (set a grain) — counts only`,
|
|
466
|
+
)
|
|
467
|
+
} else {
|
|
468
|
+
const clean =
|
|
469
|
+
entry.onlyInA === 0 &&
|
|
470
|
+
entry.onlyInB === 0 &&
|
|
471
|
+
(entry.columns?.length ?? 0) === 0 &&
|
|
472
|
+
entry.rowsA === entry.rowsB
|
|
473
|
+
const sampled =
|
|
474
|
+
entry.sampledPercent !== undefined ? ` (sample ${entry.sampledPercent}%)` : ""
|
|
475
|
+
yield* Console.log(
|
|
476
|
+
` ${clean ? "✓" : "≠"} ${entry.model} A=${entry.rowsA} B=${entry.rowsB} key (${entry.key.join(", ")}): only in A ${entry.onlyInA}, only in B ${entry.onlyInB}, matched ${entry.matched}${sampled}`,
|
|
477
|
+
)
|
|
478
|
+
for (const drift of entry.columns ?? []) {
|
|
479
|
+
yield* Console.log(
|
|
480
|
+
` ${drift.column}: ${drift.mismatches} of ${entry.matched} (${(drift.rate * 100).toFixed(2)}%)`,
|
|
481
|
+
)
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
if (entry.columnsOnlyInA !== undefined) {
|
|
485
|
+
yield* Console.log(` columns only in A: ${entry.columnsOnlyInA.join(", ")}`)
|
|
486
|
+
}
|
|
487
|
+
if (entry.columnsOnlyInB !== undefined) {
|
|
488
|
+
yield* Console.log(` columns only in B: ${entry.columnsOnlyInB.join(", ")}`)
|
|
489
|
+
}
|
|
490
|
+
}
|
|
491
|
+
})
|
|
492
|
+
|
|
338
493
|
const diffCommand = Command.make(
|
|
339
494
|
"diff",
|
|
340
|
-
{
|
|
341
|
-
|
|
495
|
+
{
|
|
496
|
+
envA: Argument.string("envA"),
|
|
497
|
+
envB: Argument.string("envB"),
|
|
498
|
+
config: configFlag,
|
|
499
|
+
data: Flag.boolean("data").pipe(
|
|
500
|
+
Flag.withDescription(
|
|
501
|
+
"compare view-layer DATA: row counts, key intersection, per-column divergences",
|
|
502
|
+
),
|
|
503
|
+
),
|
|
504
|
+
model: Flag.string("model").pipe(
|
|
505
|
+
Flag.withDefault(""),
|
|
506
|
+
Flag.withDescription("only these models, comma-separated (for --data)"),
|
|
507
|
+
),
|
|
508
|
+
sample: Flag.string("sample").pipe(
|
|
509
|
+
Flag.withDefault(""),
|
|
510
|
+
Flag.withDescription(
|
|
511
|
+
"percent 1–99: compare a deterministic fraction of keys (md5 buckets; for --data)",
|
|
512
|
+
),
|
|
513
|
+
),
|
|
514
|
+
json: jsonFlag,
|
|
515
|
+
},
|
|
516
|
+
({ config, data, envA, envB, json, model, sample }) =>
|
|
342
517
|
Effect.gen(function* () {
|
|
343
518
|
const loaded = yield* loadConfig(config)
|
|
519
|
+
if (data) {
|
|
520
|
+
const only = parseForwardOnly(model)
|
|
521
|
+
const percent = sample === "" ? undefined : Number(sample)
|
|
522
|
+
if (percent !== undefined && !(Number.isFinite(percent) && percent >= 1 && percent <= 99)) {
|
|
523
|
+
yield* Console.error("--sample expects a percent from 1 to 99")
|
|
524
|
+
return yield* Effect.sync(() => {
|
|
525
|
+
process.exitCode = 1
|
|
526
|
+
})
|
|
527
|
+
}
|
|
528
|
+
const report = yield* Effect.gen(function* () {
|
|
529
|
+
// ducklake marts are visible via ATTACH — the same way apply does it
|
|
530
|
+
if (loaded.ducklake !== undefined) {
|
|
531
|
+
const engine = yield* EngineAdapter
|
|
532
|
+
yield* engine.execute(ducklakeAttachSql(loaded.ducklake))
|
|
533
|
+
}
|
|
534
|
+
return yield* dataDiffEnvironments(envA, envB, loaded.models, {
|
|
535
|
+
...(only !== undefined ? { models: only } : {}),
|
|
536
|
+
...(percent !== undefined ? { samplePercent: percent } : {}),
|
|
537
|
+
})
|
|
538
|
+
}).pipe(Effect.provide(configLayers(loaded)))
|
|
539
|
+
yield* json ? printJson(report) : printDataDiff(report)
|
|
540
|
+
return
|
|
541
|
+
}
|
|
344
542
|
const diff = yield* diffEnvironments(envA, envB).pipe(
|
|
345
543
|
Effect.provide(configLayers(loaded)),
|
|
346
544
|
)
|
|
347
|
-
|
|
348
|
-
|
|
545
|
+
if (json) {
|
|
546
|
+
yield* printJson(diff)
|
|
547
|
+
return
|
|
548
|
+
}
|
|
549
|
+
for (const name of diff.onlyInA) yield* Console.log(`< ${name} only in ${envA}`)
|
|
550
|
+
for (const name of diff.onlyInB) yield* Console.log(`> ${name} only in ${envB}`)
|
|
349
551
|
for (const entry of diff.different) {
|
|
350
552
|
yield* Console.log(`≠ ${entry.name} ${envA}@${entry.a} vs ${envB}@${entry.b}`)
|
|
351
553
|
}
|
|
352
554
|
if (diff.onlyInA.length + diff.onlyInB.length + diff.different.length === 0) {
|
|
353
|
-
yield* Console.log("
|
|
555
|
+
yield* Console.log("environments are identical")
|
|
354
556
|
}
|
|
355
557
|
}),
|
|
356
|
-
).pipe(
|
|
558
|
+
).pipe(
|
|
559
|
+
Command.withDescription("how environments differ: versions (state store) or --data (row data)"),
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
const scheduleCommand = Command.make(
|
|
563
|
+
"schedule",
|
|
564
|
+
{
|
|
565
|
+
env: Argument.string("env").pipe(Argument.withDefault("")),
|
|
566
|
+
config: configFlag,
|
|
567
|
+
cron: Flag.string("cron").pipe(
|
|
568
|
+
Flag.withDefault("@hourly"),
|
|
569
|
+
Flag.withDescription("cron expression or nickname (@hourly, @daily, …)"),
|
|
570
|
+
),
|
|
571
|
+
remove: Flag.boolean("remove").pipe(
|
|
572
|
+
Flag.withDescription("unregister the environment from the OS scheduler"),
|
|
573
|
+
),
|
|
574
|
+
list: Flag.boolean("list").pipe(
|
|
575
|
+
Flag.withDescription("list efmesh entries in the OS scheduler"),
|
|
576
|
+
),
|
|
577
|
+
printSystemd: Flag.boolean("print-systemd").pipe(
|
|
578
|
+
Flag.withDescription(
|
|
579
|
+
"print systemd user units instead of cron (Persistent=true catches up misses; a lifeline without a cron daemon)",
|
|
580
|
+
),
|
|
581
|
+
),
|
|
582
|
+
},
|
|
583
|
+
({ config, cron, env, list, printSystemd, remove }) =>
|
|
584
|
+
Effect.gen(function* () {
|
|
585
|
+
if (list) {
|
|
586
|
+
const entries = yield* listSchedules()
|
|
587
|
+
if (entries.length === 0) yield* Console.log("no efmesh entries in the OS scheduler")
|
|
588
|
+
for (const entry of entries) yield* Console.log(` ${entry}`)
|
|
589
|
+
return
|
|
590
|
+
}
|
|
591
|
+
if (env === "") {
|
|
592
|
+
yield* Console.error("environment required: efmesh schedule <env> [--cron …]")
|
|
593
|
+
return yield* Effect.sync(() => {
|
|
594
|
+
process.exitCode = 1
|
|
595
|
+
})
|
|
596
|
+
}
|
|
597
|
+
const configAbs = NodePath.resolve(process.cwd(), config)
|
|
598
|
+
const target = { project: NodePath.dirname(configAbs), config: configAbs, env }
|
|
599
|
+
if (printSystemd) {
|
|
600
|
+
const units = systemdUnits(target, cron)
|
|
601
|
+
yield* Console.log(`# ~/.config/systemd/user/${units.name}.service`)
|
|
602
|
+
yield* Console.log(units.service)
|
|
603
|
+
yield* Console.log(`# ~/.config/systemd/user/${units.name}.timer`)
|
|
604
|
+
yield* Console.log(units.timer)
|
|
605
|
+
yield* Console.log(
|
|
606
|
+
`# enable: systemctl --user daemon-reload && systemctl --user enable --now ${units.name}.timer`,
|
|
607
|
+
)
|
|
608
|
+
return
|
|
609
|
+
}
|
|
610
|
+
if (remove) {
|
|
611
|
+
const removed = yield* removeSchedule(target)
|
|
612
|
+
yield* Console.log(`unregistered: ${removed.title}`)
|
|
613
|
+
return
|
|
614
|
+
}
|
|
615
|
+
const registered = yield* registerSchedule(target, cron)
|
|
616
|
+
yield* Console.log(`registered: ${registered.title} — "${cron}" (OS scheduler)`)
|
|
617
|
+
yield* Console.log(`worker: ${registered.worker}`)
|
|
618
|
+
yield* Console.log(
|
|
619
|
+
"tick journal: efmesh status " + env + "; NB: cron does not catch up missed runs — a systemd timer is stricter (--print-systemd)",
|
|
620
|
+
)
|
|
621
|
+
}).pipe(
|
|
622
|
+
// reason is the most valuable part (a recipe for the operator): surface it in words, not a stacktrace
|
|
623
|
+
Effect.catchTag("ScheduleError", (error) =>
|
|
624
|
+
Effect.gen(function* () {
|
|
625
|
+
yield* Console.error(`schedule: ${error.reason}`)
|
|
626
|
+
yield* Effect.sync(() => {
|
|
627
|
+
process.exitCode = 1
|
|
628
|
+
})
|
|
629
|
+
}),
|
|
630
|
+
),
|
|
631
|
+
),
|
|
632
|
+
).pipe(
|
|
633
|
+
Command.withDescription(
|
|
634
|
+
"register run <env> in the OS scheduler (Bun.cron: crontab/launchd/Task Scheduler)",
|
|
635
|
+
),
|
|
636
|
+
)
|
|
357
637
|
|
|
358
638
|
const janitorCommand = Command.make(
|
|
359
639
|
"janitor",
|
|
@@ -361,7 +641,7 @@ const janitorCommand = Command.make(
|
|
|
361
641
|
config: configFlag,
|
|
362
642
|
ttl: Flag.string("ttl").pipe(
|
|
363
643
|
Flag.withDefault("7"),
|
|
364
|
-
Flag.withDescription("
|
|
644
|
+
Flag.withDescription("how many days orphaned physics lives before removal"),
|
|
365
645
|
),
|
|
366
646
|
},
|
|
367
647
|
({ config, ttl }) =>
|
|
@@ -374,14 +654,64 @@ const janitorCommand = Command.make(
|
|
|
374
654
|
}).pipe(Effect.provide(configLayers(loaded)))
|
|
375
655
|
yield* Console.log(
|
|
376
656
|
report.removed.length > 0
|
|
377
|
-
?
|
|
378
|
-
: "
|
|
657
|
+
? `removed: ${report.removed.join(", ")}`
|
|
658
|
+
: "no orphaned physics older than ttl",
|
|
379
659
|
)
|
|
380
660
|
if (report.kept.length > 0) {
|
|
381
|
-
yield* Console.log(
|
|
661
|
+
yield* Console.log(`orphaned but younger than ttl: ${report.kept.join(", ")}`)
|
|
382
662
|
}
|
|
383
663
|
}),
|
|
384
|
-
).pipe(Command.withDescription("
|
|
664
|
+
).pipe(Command.withDescription("remove physics no environment references"))
|
|
665
|
+
|
|
666
|
+
const statusCommand = Command.make(
|
|
667
|
+
"status",
|
|
668
|
+
{ env: Argument.string("env"), config: configFlag, json: jsonFlag },
|
|
669
|
+
({ config, env, json }) =>
|
|
670
|
+
Effect.gen(function* () {
|
|
671
|
+
const loaded = yield* loadConfig(config)
|
|
672
|
+
const report = yield* environmentStatus(env, loaded.models).pipe(
|
|
673
|
+
Effect.provide(configLayers(loaded)),
|
|
674
|
+
)
|
|
675
|
+
if (json) {
|
|
676
|
+
yield* printJson(report)
|
|
677
|
+
return
|
|
678
|
+
}
|
|
679
|
+
if (report.models === 0) {
|
|
680
|
+
yield* Console.log(`environment "${env}" does not exist — the first apply creates it`)
|
|
681
|
+
return
|
|
682
|
+
}
|
|
683
|
+
yield* Console.log(
|
|
684
|
+
`environment "${env}": models ${report.models}, promoted ${report.promotedAt}, store schema v${report.storeVersion}`,
|
|
685
|
+
)
|
|
686
|
+
if (report.lastPlan !== null) {
|
|
687
|
+
yield* Console.log(
|
|
688
|
+
`last plan: ${report.lastPlan.appliedAt} (${report.lastPlan.appliedBy || "unknown"})`,
|
|
689
|
+
)
|
|
690
|
+
}
|
|
691
|
+
for (const lag of report.lag) {
|
|
692
|
+
const state =
|
|
693
|
+
lag.missing === 0
|
|
694
|
+
? `caught up to ${lag.doneUpTo}`
|
|
695
|
+
: `behind by ${lag.missing} interval(s), caught up to ${lag.doneUpTo ?? "—"}`
|
|
696
|
+
const failed = lag.failed > 0 ? ` ⚠ failed intervals: ${lag.failed}` : ""
|
|
697
|
+
yield* Console.log(` ${lag.missing === 0 ? "✓" : "…"} ${lag.model} ${state}${failed}`)
|
|
698
|
+
}
|
|
699
|
+
if (report.ticks.length === 0) {
|
|
700
|
+
yield* Console.log("no run ticks yet")
|
|
701
|
+
} else {
|
|
702
|
+
yield* Console.log("recent run ticks:")
|
|
703
|
+
for (const tick of report.ticks) {
|
|
704
|
+
const mark = tick.outcome === "ok" ? "✓" : tick.outcome === "error" ? "✗" : "…"
|
|
705
|
+
const ms = Date.parse(tick.finishedAt) - Date.parse(tick.startedAt)
|
|
706
|
+
yield* Console.log(
|
|
707
|
+
` ${mark} ${tick.startedAt} ${tick.outcome} (${ms} ms)${tick.detail !== "" ? ` ${tick.detail}` : ""}`,
|
|
708
|
+
)
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
}),
|
|
712
|
+
).pipe(
|
|
713
|
+
Command.withDescription("what is happening in an environment: promotion, lag, run ticks"),
|
|
714
|
+
)
|
|
385
715
|
|
|
386
716
|
const auditCommand = Command.make(
|
|
387
717
|
"audit",
|
|
@@ -390,25 +720,37 @@ const auditCommand = Command.make(
|
|
|
390
720
|
config: configFlag,
|
|
391
721
|
model: Flag.string("model").pipe(
|
|
392
722
|
Flag.withDefault(""),
|
|
393
|
-
Flag.withDescription("
|
|
723
|
+
Flag.withDescription("only these models, comma-separated (default — all with audits)"),
|
|
394
724
|
),
|
|
725
|
+
json: jsonFlag,
|
|
395
726
|
},
|
|
396
|
-
({ config, env, model }) =>
|
|
727
|
+
({ config, env, model, json }) =>
|
|
397
728
|
Effect.gen(function* () {
|
|
398
729
|
const loaded = yield* loadConfig(config)
|
|
399
730
|
const only = parseForwardOnly(model)
|
|
400
731
|
const report = yield* auditEnvironment(env, loaded.models, only).pipe(
|
|
401
732
|
Effect.provide(configLayers(loaded)),
|
|
402
733
|
)
|
|
734
|
+
if (json) {
|
|
735
|
+
// the whole report; the blocking-based exit code is preserved — stdout is pure JSON
|
|
736
|
+
yield* printJson(report)
|
|
737
|
+
if (report.blockingViolations > 0) {
|
|
738
|
+
return yield* new EnvironmentAuditError({
|
|
739
|
+
env,
|
|
740
|
+
blockingViolations: report.blockingViolations,
|
|
741
|
+
})
|
|
742
|
+
}
|
|
743
|
+
return
|
|
744
|
+
}
|
|
403
745
|
if (report.results.length === 0) {
|
|
404
|
-
yield* Console.log("
|
|
746
|
+
yield* Console.log("no audits — nothing to check")
|
|
405
747
|
return
|
|
406
748
|
}
|
|
407
749
|
for (const result of report.results) {
|
|
408
750
|
const mark = result.violations === 0 ? "✓" : result.blocking ? "✗" : "⚠"
|
|
409
751
|
const tail =
|
|
410
752
|
result.violations > 0
|
|
411
|
-
? ` ${result.violations}
|
|
753
|
+
? ` ${result.violations} violations${result.blocking ? "" : " (warn)"}`
|
|
412
754
|
: ""
|
|
413
755
|
yield* Console.log(` ${mark} ${result.model} ${result.audit}${tail}`)
|
|
414
756
|
}
|
|
@@ -418,10 +760,10 @@ const auditCommand = Command.make(
|
|
|
418
760
|
blockingViolations: report.blockingViolations,
|
|
419
761
|
})
|
|
420
762
|
}
|
|
421
|
-
yield* Console.log(`blocking
|
|
763
|
+
yield* Console.log(`blocking audits of environment "${env}" are clean`)
|
|
422
764
|
}),
|
|
423
765
|
).pipe(
|
|
424
|
-
Command.withDescription("
|
|
766
|
+
Command.withDescription("run audits over an environment's view layer, changing nothing"),
|
|
425
767
|
)
|
|
426
768
|
|
|
427
769
|
const migrateCommand = Command.make(
|
|
@@ -435,14 +777,14 @@ const migrateCommand = Command.make(
|
|
|
435
777
|
: migrateSqliteState({ path: loaded.state?.path ?? "efmesh.state.sqlite" }))
|
|
436
778
|
yield* Console.log(
|
|
437
779
|
report.from === report.to
|
|
438
|
-
? `state store
|
|
439
|
-
: `state store:
|
|
780
|
+
? `state store already at version ${report.to}`
|
|
781
|
+
: `state store: version ${report.from} → ${report.to}`,
|
|
440
782
|
)
|
|
441
783
|
if (report.backup !== undefined) {
|
|
442
|
-
yield* Console.log(
|
|
784
|
+
yield* Console.log(`backup of the old store: ${report.backup}`)
|
|
443
785
|
}
|
|
444
786
|
}),
|
|
445
|
-
).pipe(Command.withDescription("
|
|
787
|
+
).pipe(Command.withDescription("bring the state store schema up to the current version"))
|
|
446
788
|
|
|
447
789
|
const graphCommand = Command.make(
|
|
448
790
|
"graph",
|
|
@@ -450,7 +792,7 @@ const graphCommand = Command.make(
|
|
|
450
792
|
config: configFlag,
|
|
451
793
|
html: Flag.string("html").pipe(
|
|
452
794
|
Flag.withDefault(""),
|
|
453
|
-
Flag.withDescription("
|
|
795
|
+
Flag.withDescription("write the DAG as a self-contained HTML page at the given path"),
|
|
454
796
|
),
|
|
455
797
|
},
|
|
456
798
|
({ config, html }) =>
|
|
@@ -459,7 +801,7 @@ const graphCommand = Command.make(
|
|
|
459
801
|
const graph = yield* buildGraph(loaded.models)
|
|
460
802
|
if (html !== "") {
|
|
461
803
|
yield* Effect.sync(() => writeFileSync(html, renderGraphHtml(graph)))
|
|
462
|
-
yield* Console.log(`DAG
|
|
804
|
+
yield* Console.log(`DAG written: ${html}`)
|
|
463
805
|
return
|
|
464
806
|
}
|
|
465
807
|
for (const name of graph.order) {
|
|
@@ -468,7 +810,7 @@ const graphCommand = Command.make(
|
|
|
468
810
|
yield* Console.log(`${name} (${model.kind._tag})${deps}`)
|
|
469
811
|
}
|
|
470
812
|
}),
|
|
471
|
-
).pipe(Command.withDescription("DAG
|
|
813
|
+
).pipe(Command.withDescription("the model DAG in topological order (or an --html file)"))
|
|
472
814
|
|
|
473
815
|
const lineageCommand = Command.make(
|
|
474
816
|
"lineage",
|
|
@@ -480,14 +822,14 @@ const lineageCommand = Command.make(
|
|
|
480
822
|
if (segments.length < 2) {
|
|
481
823
|
return yield* new LineageError({
|
|
482
824
|
model: target,
|
|
483
|
-
reason: "
|
|
825
|
+
reason: "expected <schema>.<table>[.<column>]",
|
|
484
826
|
})
|
|
485
827
|
}
|
|
486
828
|
const modelName = `${segments[0]}.${segments[1]}`
|
|
487
829
|
const graph = yield* buildGraph(loaded.models)
|
|
488
830
|
const model = graph.models.get(modelName)
|
|
489
831
|
if (model === undefined) {
|
|
490
|
-
return yield* new LineageError({ model: modelName, reason: "
|
|
832
|
+
return yield* new LineageError({ model: modelName, reason: "model is not in the project" })
|
|
491
833
|
}
|
|
492
834
|
const columns =
|
|
493
835
|
segments.length >= 3 ? [segments.slice(2).join(".")] : Object.keys(model.schema.fields)
|
|
@@ -498,20 +840,98 @@ const lineageCommand = Command.make(
|
|
|
498
840
|
for (const line of formatLineage(tree)) yield* Console.log(line)
|
|
499
841
|
}
|
|
500
842
|
}),
|
|
501
|
-
).pipe(Command.withDescription("
|
|
843
|
+
).pipe(Command.withDescription("column lineage down to raw columns (best-effort)"))
|
|
844
|
+
|
|
845
|
+
/**
|
|
846
|
+
* Actionable next step for a failure, when one exists — the recipe the
|
|
847
|
+
* `schedule` command already prints for a missing cron daemon, generalized to
|
|
848
|
+
* every error that has one obvious cure (#13). No entry ⇒ the message already
|
|
849
|
+
* says everything; we do not invent filler advice.
|
|
850
|
+
*/
|
|
851
|
+
const FAILURE_HINTS: Readonly<Record<string, (error: Record<string, unknown>) => string>> = {
|
|
852
|
+
StateSchemaError: () => "run `efmesh migrate` to bring the state store up to date",
|
|
853
|
+
FingerprintVersionError: () => "run `efmesh migrate` or upgrade efmesh, then re-apply",
|
|
854
|
+
LockHeldError: (error) =>
|
|
855
|
+
`wait for the other apply/run to finish, or clear the stale lock «${String(error["name"])}»`,
|
|
856
|
+
LakeNotConfiguredError: () => "add `lake: { path: … }` to efmesh.config.ts",
|
|
857
|
+
DucklakeNotConfiguredError: () => "add `ducklake: { catalog: … }` to efmesh.config.ts",
|
|
858
|
+
AttachNotConfiguredError: (error) =>
|
|
859
|
+
`add «${String(error["attach"])}» to \`attach\` in efmesh.config.ts`,
|
|
860
|
+
ConfigLoadError: () =>
|
|
861
|
+
"check the --config path and that it default-exports defineConfig({ … })",
|
|
862
|
+
RunBlockedByChangesError: (error) => `run \`efmesh apply ${String(error["env"])}\``,
|
|
863
|
+
}
|
|
864
|
+
|
|
865
|
+
/** First line of a rendered failure: the tag names the class, the message the culprit + cause. */
|
|
866
|
+
const errorHeadline = (error: unknown): string => {
|
|
867
|
+
if (typeof error === "object" && error !== null && "_tag" in error) {
|
|
868
|
+
const record = error as Record<string, unknown>
|
|
869
|
+
const message = record["message"]
|
|
870
|
+
const detail = typeof message === "string" && message !== "" ? message : "(no detail)"
|
|
871
|
+
return `${String(record["_tag"])}: ${detail}`
|
|
872
|
+
}
|
|
873
|
+
if (error instanceof Error) {
|
|
874
|
+
return `${error.name}: ${error.message !== "" ? error.message : "(no detail)"}`
|
|
875
|
+
}
|
|
876
|
+
return String(error)
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
/**
|
|
880
|
+
* The single failure renderer (#13): cause first (the tagged error's derived
|
|
881
|
+
* message names the culprit), an actionable hint where one exists, and the
|
|
882
|
+
* Effect fiber trace ONLY under `--log-level debug` — an operator or agent
|
|
883
|
+
* sees one screen with the real cause, not a stack over an empty message. The
|
|
884
|
+
* exit code stays a frozen contract (0/1/2): the caller sets it, not this.
|
|
885
|
+
*/
|
|
886
|
+
export const renderFailure = (
|
|
887
|
+
cause: Cause.Cause<unknown>,
|
|
888
|
+
options?: { readonly debug?: boolean },
|
|
889
|
+
): string => {
|
|
890
|
+
const error = Cause.squash(cause)
|
|
891
|
+
const lines = [errorHeadline(error)]
|
|
892
|
+
const tag =
|
|
893
|
+
typeof error === "object" && error !== null && "_tag" in error
|
|
894
|
+
? String((error as Record<string, unknown>)["_tag"])
|
|
895
|
+
: ""
|
|
896
|
+
const hint = FAILURE_HINTS[tag]?.(error as Record<string, unknown>)
|
|
897
|
+
if (hint !== undefined) lines.push(` → ${hint}`)
|
|
898
|
+
if (options?.debug === true) {
|
|
899
|
+
lines.push("", "── trace (--log-level debug) ──", Cause.pretty(cause))
|
|
900
|
+
} else {
|
|
901
|
+
lines.push(" (re-run with --log-level debug for the full trace)")
|
|
902
|
+
}
|
|
903
|
+
return lines.join("\n")
|
|
904
|
+
}
|
|
905
|
+
|
|
906
|
+
/**
|
|
907
|
+
* Whether the run asked for the full trace. Reused from the already-parsed
|
|
908
|
+
* global `--log-level` flag (values trace/debug/all mean "show me everything")
|
|
909
|
+
* so there is one knob, not a second bespoke verbosity flag.
|
|
910
|
+
*/
|
|
911
|
+
export const wantsTrace = (argv: ReadonlyArray<string>): boolean => {
|
|
912
|
+
const verbose = new Set(["trace", "debug", "all"])
|
|
913
|
+
for (let index = 0; index < argv.length; index++) {
|
|
914
|
+
const arg = argv[index]!
|
|
915
|
+
if (arg.startsWith("--log-level=")) return verbose.has(arg.slice("--log-level=".length))
|
|
916
|
+
if (arg === "--log-level") return verbose.has((argv[index + 1] ?? "").toLowerCase())
|
|
917
|
+
}
|
|
918
|
+
return false
|
|
919
|
+
}
|
|
502
920
|
|
|
503
921
|
export const rootCommand = Command.make("efmesh").pipe(
|
|
504
|
-
Command.withDescription("sqlmesh
|
|
922
|
+
Command.withDescription("sqlmesh on bun, typescript and Effect"),
|
|
505
923
|
Command.withSubcommands([
|
|
506
924
|
initCommand,
|
|
507
925
|
planCommand,
|
|
508
926
|
applyCommand,
|
|
509
927
|
runCommand,
|
|
928
|
+
statusCommand,
|
|
510
929
|
auditCommand,
|
|
511
930
|
renderCommand,
|
|
512
931
|
graphCommand,
|
|
513
932
|
lineageCommand,
|
|
514
933
|
diffCommand,
|
|
934
|
+
scheduleCommand,
|
|
515
935
|
janitorCommand,
|
|
516
936
|
migrateCommand,
|
|
517
937
|
]),
|