@avytheone/efmesh 0.1.0-beta.2 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/cli.ts CHANGED
@@ -1,6 +1,6 @@
1
1
  import { writeFileSync } from "node:fs"
2
2
  import * as NodePath from "node:path"
3
- import { Console, Data, Effect, Layer } from "effect"
3
+ import { Cause, Console, Data, Effect, Layer } from "effect"
4
4
  import { Argument, Command, Flag } from "effect/unstable/cli"
5
5
  import type { EfmeshConfig } from "./config.ts"
6
6
  import { buildGraph } from "./core/graph.ts"
@@ -12,24 +12,41 @@ import { DuckDBEngineLive } from "./engine/duckdb.ts"
12
12
  import { PostgresEngineLive } from "./engine/postgres.ts"
13
13
  import { PostgresStateLive } from "./state/postgres.ts"
14
14
  import { auditEnvironment, EnvironmentAuditError } from "./plan/audit-run.ts"
15
- import { diffEnvironments } from "./plan/diff.ts"
15
+ import {
16
+ dataDiffEnvironments,
17
+ diffEnvironments,
18
+ type DataDiffReport,
19
+ } from "./plan/diff.ts"
20
+ import { EngineAdapter } from "./engine/adapter.ts"
21
+ import { ducklakeAttachSql } from "./plan/naming.ts"
22
+ import {
23
+ listSchedules,
24
+ registerSchedule,
25
+ removeSchedule,
26
+ systemdUnits,
27
+ } from "./plan/schedule.ts"
16
28
  import { renderGraphHtml } from "./plan/graph-html.ts"
17
29
  import { formatLineage, lineage, LineageError } from "./plan/lineage.ts"
30
+ import { environmentStatus } from "./plan/status.ts"
18
31
  import { janitor } from "./plan/janitor.ts"
19
32
  import { fp8 } from "./plan/naming.ts"
20
33
  import { envLockName, withStateLock } from "./plan/lock.ts"
21
34
  import { applyPlan } from "./plan/executor.ts"
22
35
  import { run } from "./plan/run.ts"
23
- import { planChanges, type Plan } from "./plan/planner.ts"
36
+ import { planChanges, ReclassifyError, type Plan } from "./plan/planner.ts"
24
37
  import { migratePostgresState } from "./state/postgres.ts"
25
38
  import { migrateSqliteState, SqliteStateLive } from "./state/sqlite.ts"
26
39
 
27
40
  export class ConfigLoadError extends Data.TaggedError("ConfigLoadError")<{
28
41
  readonly path: string
29
42
  readonly reason: string
30
- }> {}
43
+ }> {
44
+ override get message(): string {
45
+ return `config ${this.path}: ${this.reason}`
46
+ }
47
+ }
31
48
 
32
- /** Конфиг с уже собранным списком моделей: явные + найденные discovery. */
49
+ /** Config with an already-assembled model list: explicit ones + discovery finds. */
33
50
  type LoadedConfig = EfmeshConfig & { readonly models: ReadonlyArray<AnyModel> }
34
51
 
35
52
  const loadConfig = (
@@ -44,7 +61,7 @@ const loadConfig = (
44
61
  module.default === undefined ||
45
62
  (!Array.isArray(module.default.models) && module.default.discovery === undefined)
46
63
  ) {
47
- throw new Error("конфиг должен экспортировать default с models и/или discovery")
64
+ throw new Error("config must export a default with models and/or discovery")
48
65
  }
49
66
  return module.default
50
67
  },
@@ -52,10 +69,10 @@ const loadConfig = (
52
69
  })
53
70
  const explicit = config.models ?? []
54
71
  if (config.discovery === undefined) return { ...config, models: explicit }
55
- // маски относительно конфига: проект переносим независимо от cwd
72
+ // globs are relative to the config: the project is portable regardless of cwd
56
73
  const discovered = yield* discoverModels(config.discovery, NodePath.dirname(absolute))
57
74
  const seen = new Set(explicit)
58
- const names = new Map(explicit.map((model) => [model.name.full, "models в конфиге"]))
75
+ const names = new Map(explicit.map((model) => [model.name.full, "config models"]))
59
76
  const merged = [...explicit]
60
77
  for (const model of discovered) {
61
78
  if (seen.has(model)) continue
@@ -69,7 +86,7 @@ const loadConfig = (
69
86
  return { ...config, models: merged }
70
87
  })
71
88
 
72
- /** Слои движка и состояния из конфигаобщие для plan/apply. */
89
+ /** Engine and state layers from the config shared by plan/apply. */
73
90
  const configLayers = (config: EfmeshConfig) =>
74
91
  Layer.mergeAll(
75
92
  config.engine?.url !== undefined
@@ -85,7 +102,7 @@ const configLayers = (config: EfmeshConfig) =>
85
102
 
86
103
  const configFlag = Flag.string("config").pipe(
87
104
  Flag.withDefault("efmesh.config.ts"),
88
- Flag.withDescription("Путь к efmesh.config.ts"),
105
+ Flag.withDescription("path to efmesh.config.ts"),
89
106
  )
90
107
 
91
108
  const CHANGE_MARK: Record<string, string> = {
@@ -101,14 +118,21 @@ const CHANGE_MARK: Record<string, string> = {
101
118
  const forwardOnlyFlag = Flag.string("forward-only").pipe(
102
119
  Flag.withDefault(""),
103
120
  Flag.withDescription(
104
- "Модели через запятую: изменения применяются forward-only — физика и история переиспользуются",
121
+ "comma-separated models: changes apply forward-only — physics and history are reused",
122
+ ),
123
+ )
124
+
125
+ const reclassifyFlag = Flag.string("reclassify").pipe(
126
+ Flag.withDefault(""),
127
+ Flag.withDescription(
128
+ 'override categorization (#5): "model=breaking|non-breaking[,…]" on top of --explain; journaled with applied_by',
105
129
  ),
106
130
  )
107
131
 
108
132
  const jobsFlag = Flag.string("jobs").pipe(
109
133
  Flag.withDefault(""),
110
134
  Flag.withDescription(
111
- "Сколько моделей строить одновременно (DAG-конкурентность; на DuckDB всегда 1)",
135
+ "how many models to build at once (DAG concurrency; always 1 on DuckDB)",
112
136
  ),
113
137
  )
114
138
 
@@ -120,7 +144,7 @@ const parseJobs = (value: string): number | undefined => {
120
144
  const retriesFlag = Flag.string("retries").pipe(
121
145
  Flag.withDefault(""),
122
146
  Flag.withDescription(
123
- "Сколько раз ретраить упавший батч бэкфилла (экспоненциальная пауза; по умолчанию 0)",
147
+ "how many times to retry a failed backfill batch (exponential backoff; default 0)",
124
148
  ),
125
149
  )
126
150
 
@@ -139,27 +163,56 @@ const parseForwardOnly = (value: string): ReadonlyArray<string> | undefined => {
139
163
  return names.length > 0 ? names : undefined
140
164
  }
141
165
 
166
+ /** `model=breaking|non-breaking[,…]` → record for PlanOptions.reclassify (#5). */
167
+ export const parseReclassify = (
168
+ value: string,
169
+ ): Effect.Effect<Readonly<Record<string, "breaking" | "non-breaking">> | undefined, ReclassifyError> =>
170
+ Effect.gen(function* () {
171
+ const entries = value
172
+ .split(",")
173
+ .map((entry) => entry.trim())
174
+ .filter((entry) => entry !== "")
175
+ if (entries.length === 0) return undefined
176
+ const parsed: Record<string, "breaking" | "non-breaking"> = {}
177
+ for (const entry of entries) {
178
+ const [model, category, ...extra] = entry.split("=")
179
+ if (
180
+ model === undefined ||
181
+ model === "" ||
182
+ extra.length > 0 ||
183
+ (category !== "breaking" && category !== "non-breaking")
184
+ ) {
185
+ return yield* new ReclassifyError({
186
+ model: entry,
187
+ reason: 'expected "model=breaking" or "model=non-breaking"',
188
+ })
189
+ }
190
+ parsed[model] = category
191
+ }
192
+ return parsed
193
+ })
194
+
142
195
  const yesFlag = Flag.boolean("yes").pipe(
143
196
  Flag.withAlias("y"),
144
- Flag.withDescription("Применить без подтверждения (не-TTY подтверждения не спрашивает)"),
197
+ Flag.withDescription("apply without confirmation (non-TTY never asks)"),
145
198
  )
146
199
 
147
- /** y/yes/д/да регистронезависимо; всё остальное (включая пусто) отказ. */
200
+ /** Accepts y/yes, case-insensitive; anything else (including empty) is a refusal. */
148
201
  export const isAffirmative = (answer: string | null): boolean =>
149
- ["y", "yes", "д", "да"].includes((answer ?? "").trim().toLowerCase())
202
+ ["y", "yes"].includes((answer ?? "").trim().toLowerCase())
150
203
 
151
204
  /**
152
- * Exit-код «работа ждёт человека» (F6): план требует подтверждения в не-TTY
153
- * или run упёрся в структурные изменения. Алертинг обязан отличать это
154
- * штатное состояние от настоящих ошибок (код 1).
205
+ * The "work awaits a human" exit code (F6): the plan needs confirmation in a
206
+ * non-TTY, or run hit structural changes. Alerting must distinguish this
207
+ * normal state from real errors (code 1).
155
208
  */
156
209
  export const EXIT_AWAITING_HUMAN = 2
157
210
 
158
211
  /**
159
- * Судьба показанного плана (SPEC §5.1, ужесточено в F6): без изменений или
160
- * с --yes — применять; изменения в TTY — спросить человека; изменения в
161
- * не-TTY (CI, cron, пайп) — ОТКАЗ: молча применять план, который никто не
162
- * видел, нельзя, нужен явный --yes.
212
+ * The fate of a shown plan (SPEC §5.1, tightened in F6): no changes or
213
+ * --yes — apply; changes in a TTY — ask the human; changes in a
214
+ * non-TTY (CI, cron, pipe) — REFUSE: silently applying a plan nobody
215
+ * saw is forbidden, an explicit --yes is required.
163
216
  */
164
217
  export const decideApply = (
165
218
  hasChanges: boolean,
@@ -167,24 +220,76 @@ export const decideApply = (
167
220
  tty: boolean,
168
221
  ): "apply" | "ask" | "refuse" => (!hasChanges || yes ? "apply" : tty ? "ask" : "refuse")
169
222
 
223
+ const jsonFlag = Flag.boolean("json").pipe(
224
+ Flag.withDescription("machine-readable output (stable shape — a contract for CI)"),
225
+ )
226
+
227
+ const explainFlag = Flag.boolean("explain").pipe(
228
+ Flag.withDescription(
229
+ "for each change — which canonical AST nodes diverged and why the category is what it is",
230
+ ),
231
+ )
232
+
233
+ /**
234
+ * JSON shape of the plan (#3) — a CONTRACT for CI and bots: shape changes
235
+ * are package semver events. Intervals are ISO UTC, not epoch ms.
236
+ */
237
+ export const planToJson = (plan: Plan): unknown => ({
238
+ env: plan.env,
239
+ hasChanges: plan.hasChanges,
240
+ actions: plan.actions.map((action) => ({
241
+ name: action.name,
242
+ change: action.change,
243
+ // operator override (#5) and physical reuse — additive contract fields
244
+ ...(action.reclassifiedFrom !== undefined
245
+ ? { reclassifiedFrom: action.reclassifiedFrom }
246
+ : {}),
247
+ ...(action.reusedFrom !== undefined ? { reusedFrom: action.reusedFrom } : {}),
248
+ fingerprint: action.fingerprint,
249
+ build: action.build,
250
+ backfill: action.backfill.map((range) => ({
251
+ start: new Date(range.start).toISOString(),
252
+ end: new Date(range.end).toISOString(),
253
+ })),
254
+ // category reason (#4); diverged paths are a debug hint, not a contract
255
+ ...(action.explain !== undefined ? { explain: action.explain } : {}),
256
+ })),
257
+ })
258
+
259
+ const printJson = (payload: unknown) => Console.log(JSON.stringify(payload, null, 2))
260
+
170
261
  const formatRange = (range: { readonly start: number; readonly end: number }): string =>
171
- `[${new Date(range.start).toISOString().slice(0, 10)} ${new Date(range.end).toISOString().slice(0, 10)})`
262
+ `[${new Date(range.start).toISOString().slice(0, 10)}, ${new Date(range.end).toISOString().slice(0, 10)})`
172
263
 
173
- const printPlan = (plan: Plan) =>
264
+ const printPlan = (plan: Plan, explain = false) =>
174
265
  Effect.gen(function* () {
175
- yield* Console.log(`план для окружения «${plan.env}»:`)
266
+ yield* Console.log(`plan for environment "${plan.env}":`)
176
267
  for (const action of plan.actions) {
177
268
  const mark = CHANGE_MARK[action.change] ?? "?"
178
- const build = action.build ? " [сборка]" : ""
269
+ const overridden =
270
+ action.reclassifiedFrom !== undefined
271
+ ? ` [override: was ${action.reclassifiedFrom}]`
272
+ : ""
273
+ const reused =
274
+ action.change === "indirect" && action.reusedFrom !== undefined
275
+ ? " [physics reused]"
276
+ : ""
277
+ const build = action.build ? " [build]" : ""
179
278
  const backfill =
180
279
  action.backfill.length > 0
181
- ? ` бэкфилл ${action.backfill.map(formatRange).join(", ")}`
280
+ ? ` backfill ${action.backfill.map(formatRange).join(", ")}`
182
281
  : ""
183
282
  yield* Console.log(
184
- ` ${mark} ${action.name} ${action.change} @${fp8(action.fingerprint)}${build}${backfill}`,
283
+ ` ${mark} ${action.name} ${action.change} @${fp8(action.fingerprint)}${overridden}${reused}${build}${backfill}`,
185
284
  )
285
+ if (explain && action.explain !== undefined) {
286
+ yield* Console.log(` why: ${action.explain.reason}`)
287
+ if (action.explain.diverged.length > 0) {
288
+ yield* Console.log(` diverged: ${action.explain.diverged.join(", ")}`)
289
+ }
290
+ }
186
291
  }
187
- if (!plan.hasChanges) yield* Console.log(" изменений нет")
292
+ if (!plan.hasChanges) yield* Console.log(" no changes")
188
293
  })
189
294
 
190
295
  const initCommand = Command.make(
@@ -193,24 +298,33 @@ const initCommand = Command.make(
193
298
  ({ dir }) =>
194
299
  Effect.gen(function* () {
195
300
  const created = yield* scaffold(dir)
196
- for (const file of created) yield* Console.log(`создан ${file}`)
197
- yield* Console.log("дальше: bunx efmesh plan dev && bunx efmesh apply dev")
301
+ for (const file of created) yield* Console.log(`created ${file}`)
302
+ yield* Console.log("next: bunx efmesh plan dev && bunx efmesh apply dev")
198
303
  }),
199
- ).pipe(Command.withDescription("Скаффолд проекта: конфиг, модели-пример, seed"))
304
+ ).pipe(Command.withDescription("scaffold a project: config, example models, seed"))
200
305
 
201
306
  const planCommand = Command.make(
202
307
  "plan",
203
- { env: Argument.string("env"), config: configFlag, forwardOnly: forwardOnlyFlag },
204
- ({ config, env, forwardOnly }) =>
308
+ {
309
+ env: Argument.string("env"),
310
+ config: configFlag,
311
+ forwardOnly: forwardOnlyFlag,
312
+ reclassify: reclassifyFlag,
313
+ json: jsonFlag,
314
+ explain: explainFlag,
315
+ },
316
+ ({ config, env, explain, forwardOnly, json, reclassify }) =>
205
317
  Effect.gen(function* () {
206
318
  const loaded = yield* loadConfig(config)
207
319
  const names = parseForwardOnly(forwardOnly)
320
+ const overrides = yield* parseReclassify(reclassify)
208
321
  const plan = yield* Efmesh.plan(env, loaded.models, {
209
322
  ...(names !== undefined ? { forwardOnly: names } : {}),
323
+ ...(overrides !== undefined ? { reclassify: overrides } : {}),
210
324
  }).pipe(Effect.provide(configLayers(loaded)))
211
- yield* printPlan(plan)
325
+ yield* json ? printJson(planToJson(plan)) : printPlan(plan, explain)
212
326
  }),
213
- ).pipe(Command.withDescription("Показать diff проекта против окружения, ничего не меняя"))
327
+ ).pipe(Command.withDescription("show the project diff against an environment, changing nothing"))
214
328
 
215
329
  const applyCommand = Command.make(
216
330
  "apply",
@@ -218,30 +332,33 @@ const applyCommand = Command.make(
218
332
  env: Argument.string("env"),
219
333
  config: configFlag,
220
334
  forwardOnly: forwardOnlyFlag,
335
+ reclassify: reclassifyFlag,
221
336
  jobs: jobsFlag,
222
337
  retries: retriesFlag,
223
338
  yes: yesFlag,
224
339
  },
225
- ({ config, env, forwardOnly, jobs, retries, yes }) =>
340
+ ({ config, env, forwardOnly, jobs, reclassify, retries, yes }) =>
226
341
  Effect.gen(function* () {
227
342
  const loaded = yield* loadConfig(config)
228
343
  const names = parseForwardOnly(forwardOnly)
344
+ const overrides = yield* parseReclassify(reclassify)
229
345
  const modelConcurrency = parseJobs(jobs)
230
346
  const retry = parseRetries(retries)
231
- // план и применениепод одним слоем и одним межпроцессным локом:
232
- // применяется ровно тот план, который показан и подтверждён, и никто
233
- // (второй apply, cron с run) не вклинится между ними (SPEC §14.6);
234
- // ценалок держится и пока человек думает над подтверждением
347
+ // plan and applyunder one layer and one cross-process lock:
348
+ // exactly the plan that was shown and confirmed gets applied, and no one
349
+ // (a second apply, cron with run) wedges in between them (SPEC §14.6);
350
+ // the cost the lock is held even while the human ponders confirmation
235
351
  yield* Effect.gen(function* () {
236
352
  const graph = yield* buildGraph(loaded.models)
237
353
  const plan = yield* planChanges(env, graph, {
238
354
  ...(names !== undefined ? { forwardOnly: names } : {}),
355
+ ...(overrides !== undefined ? { reclassify: overrides } : {}),
239
356
  })
240
357
  yield* printPlan(plan)
241
358
  const decision = decideApply(plan.hasChanges, yes, process.stdin.isTTY === true)
242
359
  if (decision === "refuse") {
243
360
  yield* Console.error(
244
- "план меняет модели, а подтвердить некому (не-TTY): добавьте --yes",
361
+ "the plan changes models but there is no one to confirm (non-TTY): add --yes",
245
362
  )
246
363
  yield* Effect.sync(() => {
247
364
  process.exitCode = EXIT_AWAITING_HUMAN
@@ -250,9 +367,9 @@ const applyCommand = Command.make(
250
367
  }
251
368
  if (
252
369
  decision === "ask" &&
253
- !isAffirmative(globalThis.prompt("применить план? [y/N]"))
370
+ !isAffirmative(globalThis.prompt("apply the plan? [y/N]"))
254
371
  ) {
255
- yield* Console.log("применение отменено")
372
+ yield* Console.log("apply cancelled")
256
373
  return
257
374
  }
258
375
  const applied = yield* applyPlan(plan, graph, {
@@ -264,13 +381,13 @@ const applyCommand = Command.make(
264
381
  })
265
382
  yield* Console.log(
266
383
  applied.built.length > 0
267
- ? `собрано: ${applied.built.join(", ")}`
268
- : "сборка не потребовалась (только view-swap)",
384
+ ? `built: ${applied.built.join(", ")}`
385
+ : "no build needed (view-swap only)",
269
386
  )
270
- yield* Console.log(`окружение «${applied.plan.env}» промоутнуто`)
387
+ yield* Console.log(`environment "${applied.plan.env}" promoted`)
271
388
  }).pipe(withStateLock(envLockName(env)), Effect.provide(configLayers(loaded)))
272
389
  }),
273
- ).pipe(Command.withDescription("Применить план: собрать физику и переключить view"))
390
+ ).pipe(Command.withDescription("apply the plan: build physics and swap views"))
274
391
 
275
392
  const renderCommand = Command.make(
276
393
  "render",
@@ -279,7 +396,7 @@ const renderCommand = Command.make(
279
396
  config: configFlag,
280
397
  env: Flag.string("env").pipe(
281
398
  Flag.withDefault(""),
282
- Flag.withDescription("Рендер против view-слоя окружения вместо логических имён"),
399
+ Flag.withDescription("render against an environment's view layer instead of logical names"),
283
400
  ),
284
401
  },
285
402
  ({ config, env, model }) =>
@@ -290,7 +407,7 @@ const renderCommand = Command.make(
290
407
  : yield* Efmesh.renderFor(loaded.models, model, env)
291
408
  yield* Console.log(sql.trim())
292
409
  }),
293
- ).pipe(Command.withDescription("Показать итоговый SQL модели"))
410
+ ).pipe(Command.withDescription("show a model's final SQL"))
294
411
 
295
412
  const runCommand = Command.make(
296
413
  "run",
@@ -308,12 +425,12 @@ const runCommand = Command.make(
308
425
  ...(retry !== undefined ? { retry } : {}),
309
426
  }).pipe(
310
427
  Effect.provide(configLayers(loaded)),
311
- // структурные изменения штатное «ждёт человека с apply», не сбой:
312
- // алертинг различает по exit-коду 2 (F6)
428
+ // structural changes are the normal "awaits a human with apply", not a failure:
429
+ // alerting tells it apart by exit code 2 (F6)
313
430
  Effect.catchTag("RunBlockedByChangesError", (blocked) =>
314
431
  Effect.gen(function* () {
315
432
  yield* Console.error(
316
- `run ${blocked.env}: есть неприменённые изменениянужен apply:\n ${blocked.changes.join("\n ")}`,
433
+ `run "${blocked.env}": unapplied changes presentapply needed:\n ${blocked.changes.join("\n ")}`,
317
434
  )
318
435
  yield* Effect.sync(() => {
319
436
  process.exitCode = EXIT_AWAITING_HUMAN
@@ -325,35 +442,198 @@ const runCommand = Command.make(
325
442
  if (applied === undefined) return
326
443
  yield* Console.log(
327
444
  applied.built.length > 0
328
- ? `обработано: ${applied.built.join(", ")}`
329
- : "новых интервалов нет",
445
+ ? `processed: ${applied.built.join(", ")}`
446
+ : "no new intervals",
330
447
  )
331
448
  }),
332
449
  ).pipe(
333
450
  Command.withDescription(
334
- "Тик планировщика: догнать интервалы существующих версий (изменения через apply)",
451
+ "scheduler tick: catch up intervals of existing versions (changes go through apply)",
335
452
  ),
336
453
  )
337
454
 
455
+ const printDataDiff = (report: DataDiffReport) =>
456
+ Effect.gen(function* () {
457
+ yield* Console.log(`data "${report.envA}" ↔ "${report.envB}":`)
458
+ if (report.models.length === 0) {
459
+ yield* Console.log(" no shared materializable models")
460
+ return
461
+ }
462
+ for (const entry of report.models) {
463
+ if (entry.key === undefined) {
464
+ yield* Console.log(
465
+ ` · ${entry.model} A=${entry.rowsA} B=${entry.rowsB} no key (set a grain) — counts only`,
466
+ )
467
+ } else {
468
+ const clean =
469
+ entry.onlyInA === 0 &&
470
+ entry.onlyInB === 0 &&
471
+ (entry.columns?.length ?? 0) === 0 &&
472
+ entry.rowsA === entry.rowsB
473
+ const sampled =
474
+ entry.sampledPercent !== undefined ? ` (sample ${entry.sampledPercent}%)` : ""
475
+ yield* Console.log(
476
+ ` ${clean ? "✓" : "≠"} ${entry.model} A=${entry.rowsA} B=${entry.rowsB} key (${entry.key.join(", ")}): only in A ${entry.onlyInA}, only in B ${entry.onlyInB}, matched ${entry.matched}${sampled}`,
477
+ )
478
+ for (const drift of entry.columns ?? []) {
479
+ yield* Console.log(
480
+ ` ${drift.column}: ${drift.mismatches} of ${entry.matched} (${(drift.rate * 100).toFixed(2)}%)`,
481
+ )
482
+ }
483
+ }
484
+ if (entry.columnsOnlyInA !== undefined) {
485
+ yield* Console.log(` columns only in A: ${entry.columnsOnlyInA.join(", ")}`)
486
+ }
487
+ if (entry.columnsOnlyInB !== undefined) {
488
+ yield* Console.log(` columns only in B: ${entry.columnsOnlyInB.join(", ")}`)
489
+ }
490
+ }
491
+ })
492
+
338
493
  const diffCommand = Command.make(
339
494
  "diff",
340
- { envA: Argument.string("envA"), envB: Argument.string("envB"), config: configFlag },
341
- ({ config, envA, envB }) =>
495
+ {
496
+ envA: Argument.string("envA"),
497
+ envB: Argument.string("envB"),
498
+ config: configFlag,
499
+ data: Flag.boolean("data").pipe(
500
+ Flag.withDescription(
501
+ "compare view-layer DATA: row counts, key intersection, per-column divergences",
502
+ ),
503
+ ),
504
+ model: Flag.string("model").pipe(
505
+ Flag.withDefault(""),
506
+ Flag.withDescription("only these models, comma-separated (for --data)"),
507
+ ),
508
+ sample: Flag.string("sample").pipe(
509
+ Flag.withDefault(""),
510
+ Flag.withDescription(
511
+ "percent 1–99: compare a deterministic fraction of keys (md5 buckets; for --data)",
512
+ ),
513
+ ),
514
+ json: jsonFlag,
515
+ },
516
+ ({ config, data, envA, envB, json, model, sample }) =>
342
517
  Effect.gen(function* () {
343
518
  const loaded = yield* loadConfig(config)
519
+ if (data) {
520
+ const only = parseForwardOnly(model)
521
+ const percent = sample === "" ? undefined : Number(sample)
522
+ if (percent !== undefined && !(Number.isFinite(percent) && percent >= 1 && percent <= 99)) {
523
+ yield* Console.error("--sample expects a percent from 1 to 99")
524
+ return yield* Effect.sync(() => {
525
+ process.exitCode = 1
526
+ })
527
+ }
528
+ const report = yield* Effect.gen(function* () {
529
+ // ducklake marts are visible via ATTACH — the same way apply does it
530
+ if (loaded.ducklake !== undefined) {
531
+ const engine = yield* EngineAdapter
532
+ yield* engine.execute(ducklakeAttachSql(loaded.ducklake))
533
+ }
534
+ return yield* dataDiffEnvironments(envA, envB, loaded.models, {
535
+ ...(only !== undefined ? { models: only } : {}),
536
+ ...(percent !== undefined ? { samplePercent: percent } : {}),
537
+ })
538
+ }).pipe(Effect.provide(configLayers(loaded)))
539
+ yield* json ? printJson(report) : printDataDiff(report)
540
+ return
541
+ }
344
542
  const diff = yield* diffEnvironments(envA, envB).pipe(
345
543
  Effect.provide(configLayers(loaded)),
346
544
  )
347
- for (const name of diff.onlyInA) yield* Console.log(`< ${name} только в ${envA}`)
348
- for (const name of diff.onlyInB) yield* Console.log(`> ${name} только в ${envB}`)
545
+ if (json) {
546
+ yield* printJson(diff)
547
+ return
548
+ }
549
+ for (const name of diff.onlyInA) yield* Console.log(`< ${name} only in ${envA}`)
550
+ for (const name of diff.onlyInB) yield* Console.log(`> ${name} only in ${envB}`)
349
551
  for (const entry of diff.different) {
350
552
  yield* Console.log(`≠ ${entry.name} ${envA}@${entry.a} vs ${envB}@${entry.b}`)
351
553
  }
352
554
  if (diff.onlyInA.length + diff.onlyInB.length + diff.different.length === 0) {
353
- yield* Console.log("окружения идентичны")
555
+ yield* Console.log("environments are identical")
354
556
  }
355
557
  }),
356
- ).pipe(Command.withDescription("Чем окружения отличаются (по state store)"))
558
+ ).pipe(
559
+ Command.withDescription("how environments differ: versions (state store) or --data (row data)"),
560
+ )
561
+
562
+ const scheduleCommand = Command.make(
563
+ "schedule",
564
+ {
565
+ env: Argument.string("env").pipe(Argument.withDefault("")),
566
+ config: configFlag,
567
+ cron: Flag.string("cron").pipe(
568
+ Flag.withDefault("@hourly"),
569
+ Flag.withDescription("cron expression or nickname (@hourly, @daily, …)"),
570
+ ),
571
+ remove: Flag.boolean("remove").pipe(
572
+ Flag.withDescription("unregister the environment from the OS scheduler"),
573
+ ),
574
+ list: Flag.boolean("list").pipe(
575
+ Flag.withDescription("list efmesh entries in the OS scheduler"),
576
+ ),
577
+ printSystemd: Flag.boolean("print-systemd").pipe(
578
+ Flag.withDescription(
579
+ "print systemd user units instead of cron (Persistent=true catches up misses; a lifeline without a cron daemon)",
580
+ ),
581
+ ),
582
+ },
583
+ ({ config, cron, env, list, printSystemd, remove }) =>
584
+ Effect.gen(function* () {
585
+ if (list) {
586
+ const entries = yield* listSchedules()
587
+ if (entries.length === 0) yield* Console.log("no efmesh entries in the OS scheduler")
588
+ for (const entry of entries) yield* Console.log(` ${entry}`)
589
+ return
590
+ }
591
+ if (env === "") {
592
+ yield* Console.error("environment required: efmesh schedule <env> [--cron …]")
593
+ return yield* Effect.sync(() => {
594
+ process.exitCode = 1
595
+ })
596
+ }
597
+ const configAbs = NodePath.resolve(process.cwd(), config)
598
+ const target = { project: NodePath.dirname(configAbs), config: configAbs, env }
599
+ if (printSystemd) {
600
+ const units = systemdUnits(target, cron)
601
+ yield* Console.log(`# ~/.config/systemd/user/${units.name}.service`)
602
+ yield* Console.log(units.service)
603
+ yield* Console.log(`# ~/.config/systemd/user/${units.name}.timer`)
604
+ yield* Console.log(units.timer)
605
+ yield* Console.log(
606
+ `# enable: systemctl --user daemon-reload && systemctl --user enable --now ${units.name}.timer`,
607
+ )
608
+ return
609
+ }
610
+ if (remove) {
611
+ const removed = yield* removeSchedule(target)
612
+ yield* Console.log(`unregistered: ${removed.title}`)
613
+ return
614
+ }
615
+ const registered = yield* registerSchedule(target, cron)
616
+ yield* Console.log(`registered: ${registered.title} — "${cron}" (OS scheduler)`)
617
+ yield* Console.log(`worker: ${registered.worker}`)
618
+ yield* Console.log(
619
+ "tick journal: efmesh status " + env + "; NB: cron does not catch up missed runs — a systemd timer is stricter (--print-systemd)",
620
+ )
621
+ }).pipe(
622
+ // reason is the most valuable part (a recipe for the operator): surface it in words, not a stacktrace
623
+ Effect.catchTag("ScheduleError", (error) =>
624
+ Effect.gen(function* () {
625
+ yield* Console.error(`schedule: ${error.reason}`)
626
+ yield* Effect.sync(() => {
627
+ process.exitCode = 1
628
+ })
629
+ }),
630
+ ),
631
+ ),
632
+ ).pipe(
633
+ Command.withDescription(
634
+ "register run <env> in the OS scheduler (Bun.cron: crontab/launchd/Task Scheduler)",
635
+ ),
636
+ )
357
637
 
358
638
  const janitorCommand = Command.make(
359
639
  "janitor",
@@ -361,7 +641,7 @@ const janitorCommand = Command.make(
361
641
  config: configFlag,
362
642
  ttl: Flag.string("ttl").pipe(
363
643
  Flag.withDefault("7"),
364
- Flag.withDescription("Сколько дней осиротевшая физика живёт до сноса"),
644
+ Flag.withDescription("how many days orphaned physics lives before removal"),
365
645
  ),
366
646
  },
367
647
  ({ config, ttl }) =>
@@ -374,14 +654,64 @@ const janitorCommand = Command.make(
374
654
  }).pipe(Effect.provide(configLayers(loaded)))
375
655
  yield* Console.log(
376
656
  report.removed.length > 0
377
- ? `снесено: ${report.removed.join(", ")}`
378
- : "осиротевшей физики старше ttl нет",
657
+ ? `removed: ${report.removed.join(", ")}`
658
+ : "no orphaned physics older than ttl",
379
659
  )
380
660
  if (report.kept.length > 0) {
381
- yield* Console.log(`осиротело, но моложе ttl: ${report.kept.join(", ")}`)
661
+ yield* Console.log(`orphaned but younger than ttl: ${report.kept.join(", ")}`)
382
662
  }
383
663
  }),
384
- ).pipe(Command.withDescription("Убрать физику, на которую не ссылается ни одно окружение"))
664
+ ).pipe(Command.withDescription("remove physics no environment references"))
665
+
666
+ const statusCommand = Command.make(
667
+ "status",
668
+ { env: Argument.string("env"), config: configFlag, json: jsonFlag },
669
+ ({ config, env, json }) =>
670
+ Effect.gen(function* () {
671
+ const loaded = yield* loadConfig(config)
672
+ const report = yield* environmentStatus(env, loaded.models).pipe(
673
+ Effect.provide(configLayers(loaded)),
674
+ )
675
+ if (json) {
676
+ yield* printJson(report)
677
+ return
678
+ }
679
+ if (report.models === 0) {
680
+ yield* Console.log(`environment "${env}" does not exist — the first apply creates it`)
681
+ return
682
+ }
683
+ yield* Console.log(
684
+ `environment "${env}": models ${report.models}, promoted ${report.promotedAt}, store schema v${report.storeVersion}`,
685
+ )
686
+ if (report.lastPlan !== null) {
687
+ yield* Console.log(
688
+ `last plan: ${report.lastPlan.appliedAt} (${report.lastPlan.appliedBy || "unknown"})`,
689
+ )
690
+ }
691
+ for (const lag of report.lag) {
692
+ const state =
693
+ lag.missing === 0
694
+ ? `caught up to ${lag.doneUpTo}`
695
+ : `behind by ${lag.missing} interval(s), caught up to ${lag.doneUpTo ?? "—"}`
696
+ const failed = lag.failed > 0 ? ` ⚠ failed intervals: ${lag.failed}` : ""
697
+ yield* Console.log(` ${lag.missing === 0 ? "✓" : "…"} ${lag.model} ${state}${failed}`)
698
+ }
699
+ if (report.ticks.length === 0) {
700
+ yield* Console.log("no run ticks yet")
701
+ } else {
702
+ yield* Console.log("recent run ticks:")
703
+ for (const tick of report.ticks) {
704
+ const mark = tick.outcome === "ok" ? "✓" : tick.outcome === "error" ? "✗" : "…"
705
+ const ms = Date.parse(tick.finishedAt) - Date.parse(tick.startedAt)
706
+ yield* Console.log(
707
+ ` ${mark} ${tick.startedAt} ${tick.outcome} (${ms} ms)${tick.detail !== "" ? ` ${tick.detail}` : ""}`,
708
+ )
709
+ }
710
+ }
711
+ }),
712
+ ).pipe(
713
+ Command.withDescription("what is happening in an environment: promotion, lag, run ticks"),
714
+ )
385
715
 
386
716
  const auditCommand = Command.make(
387
717
  "audit",
@@ -390,25 +720,37 @@ const auditCommand = Command.make(
390
720
  config: configFlag,
391
721
  model: Flag.string("model").pipe(
392
722
  Flag.withDefault(""),
393
- Flag.withDescription("Только эти модели, через запятую (по умолчанию все с аудитами)"),
723
+ Flag.withDescription("only these models, comma-separated (defaultall with audits)"),
394
724
  ),
725
+ json: jsonFlag,
395
726
  },
396
- ({ config, env, model }) =>
727
+ ({ config, env, model, json }) =>
397
728
  Effect.gen(function* () {
398
729
  const loaded = yield* loadConfig(config)
399
730
  const only = parseForwardOnly(model)
400
731
  const report = yield* auditEnvironment(env, loaded.models, only).pipe(
401
732
  Effect.provide(configLayers(loaded)),
402
733
  )
734
+ if (json) {
735
+ // the whole report; the blocking-based exit code is preserved — stdout is pure JSON
736
+ yield* printJson(report)
737
+ if (report.blockingViolations > 0) {
738
+ return yield* new EnvironmentAuditError({
739
+ env,
740
+ blockingViolations: report.blockingViolations,
741
+ })
742
+ }
743
+ return
744
+ }
403
745
  if (report.results.length === 0) {
404
- yield* Console.log("аудитов нетнечего проверять")
746
+ yield* Console.log("no auditsnothing to check")
405
747
  return
406
748
  }
407
749
  for (const result of report.results) {
408
750
  const mark = result.violations === 0 ? "✓" : result.blocking ? "✗" : "⚠"
409
751
  const tail =
410
752
  result.violations > 0
411
- ? ` ${result.violations} нарушений${result.blocking ? "" : " (warn)"}`
753
+ ? ` ${result.violations} violations${result.blocking ? "" : " (warn)"}`
412
754
  : ""
413
755
  yield* Console.log(` ${mark} ${result.model} ${result.audit}${tail}`)
414
756
  }
@@ -418,10 +760,10 @@ const auditCommand = Command.make(
418
760
  blockingViolations: report.blockingViolations,
419
761
  })
420
762
  }
421
- yield* Console.log(`blocking-аудиты окружения «${env}» чисты`)
763
+ yield* Console.log(`blocking audits of environment "${env}" are clean`)
422
764
  }),
423
765
  ).pipe(
424
- Command.withDescription("Прогнать аудиты по view-слою окружения, ничего не меняя"),
766
+ Command.withDescription("run audits over an environment's view layer, changing nothing"),
425
767
  )
426
768
 
427
769
  const migrateCommand = Command.make(
@@ -435,14 +777,14 @@ const migrateCommand = Command.make(
435
777
  : migrateSqliteState({ path: loaded.state?.path ?? "efmesh.state.sqlite" }))
436
778
  yield* Console.log(
437
779
  report.from === report.to
438
- ? `state store уже на версии ${report.to}`
439
- : `state store: версия ${report.from} → ${report.to}`,
780
+ ? `state store already at version ${report.to}`
781
+ : `state store: version ${report.from} → ${report.to}`,
440
782
  )
441
783
  if (report.backup !== undefined) {
442
- yield* Console.log(`копия старого стора: ${report.backup}`)
784
+ yield* Console.log(`backup of the old store: ${report.backup}`)
443
785
  }
444
786
  }),
445
- ).pipe(Command.withDescription("Догнать схему state store до текущей версии"))
787
+ ).pipe(Command.withDescription("bring the state store schema up to the current version"))
446
788
 
447
789
  const graphCommand = Command.make(
448
790
  "graph",
@@ -450,7 +792,7 @@ const graphCommand = Command.make(
450
792
  config: configFlag,
451
793
  html: Flag.string("html").pipe(
452
794
  Flag.withDefault(""),
453
- Flag.withDescription("Записать DAG самодостаточной HTML-страницей по указанному пути"),
795
+ Flag.withDescription("write the DAG as a self-contained HTML page at the given path"),
454
796
  ),
455
797
  },
456
798
  ({ config, html }) =>
@@ -459,7 +801,7 @@ const graphCommand = Command.make(
459
801
  const graph = yield* buildGraph(loaded.models)
460
802
  if (html !== "") {
461
803
  yield* Effect.sync(() => writeFileSync(html, renderGraphHtml(graph)))
462
- yield* Console.log(`DAG записан: ${html}`)
804
+ yield* Console.log(`DAG written: ${html}`)
463
805
  return
464
806
  }
465
807
  for (const name of graph.order) {
@@ -468,7 +810,7 @@ const graphCommand = Command.make(
468
810
  yield* Console.log(`${name} (${model.kind._tag})${deps}`)
469
811
  }
470
812
  }),
471
- ).pipe(Command.withDescription("DAG моделей в топологическом порядке (или --html файл)"))
813
+ ).pipe(Command.withDescription("the model DAG in topological order (or an --html file)"))
472
814
 
473
815
  const lineageCommand = Command.make(
474
816
  "lineage",
@@ -480,14 +822,14 @@ const lineageCommand = Command.make(
480
822
  if (segments.length < 2) {
481
823
  return yield* new LineageError({
482
824
  model: target,
483
- reason: "ожидается <схема>.<таблица>[.<колонка>]",
825
+ reason: "expected <schema>.<table>[.<column>]",
484
826
  })
485
827
  }
486
828
  const modelName = `${segments[0]}.${segments[1]}`
487
829
  const graph = yield* buildGraph(loaded.models)
488
830
  const model = graph.models.get(modelName)
489
831
  if (model === undefined) {
490
- return yield* new LineageError({ model: modelName, reason: "модели нет в проекте" })
832
+ return yield* new LineageError({ model: modelName, reason: "model is not in the project" })
491
833
  }
492
834
  const columns =
493
835
  segments.length >= 3 ? [segments.slice(2).join(".")] : Object.keys(model.schema.fields)
@@ -498,20 +840,98 @@ const lineageCommand = Command.make(
498
840
  for (const line of formatLineage(tree)) yield* Console.log(line)
499
841
  }
500
842
  }),
501
- ).pipe(Command.withDescription("Колоночный lineage до сырьевых колонок (best-effort)"))
843
+ ).pipe(Command.withDescription("column lineage down to raw columns (best-effort)"))
844
+
845
+ /**
846
+ * Actionable next step for a failure, when one exists — the recipe the
847
+ * `schedule` command already prints for a missing cron daemon, generalized to
848
+ * every error that has one obvious cure (#13). No entry ⇒ the message already
849
+ * says everything; we do not invent filler advice.
850
+ */
851
+ const FAILURE_HINTS: Readonly<Record<string, (error: Record<string, unknown>) => string>> = {
852
+ StateSchemaError: () => "run `efmesh migrate` to bring the state store up to date",
853
+ FingerprintVersionError: () => "run `efmesh migrate` or upgrade efmesh, then re-apply",
854
+ LockHeldError: (error) =>
855
+ `wait for the other apply/run to finish, or clear the stale lock «${String(error["name"])}»`,
856
+ LakeNotConfiguredError: () => "add `lake: { path: … }` to efmesh.config.ts",
857
+ DucklakeNotConfiguredError: () => "add `ducklake: { catalog: … }` to efmesh.config.ts",
858
+ AttachNotConfiguredError: (error) =>
859
+ `add «${String(error["attach"])}» to \`attach\` in efmesh.config.ts`,
860
+ ConfigLoadError: () =>
861
+ "check the --config path and that it default-exports defineConfig({ … })",
862
+ RunBlockedByChangesError: (error) => `run \`efmesh apply ${String(error["env"])}\``,
863
+ }
864
+
865
+ /** First line of a rendered failure: the tag names the class, the message the culprit + cause. */
866
+ const errorHeadline = (error: unknown): string => {
867
+ if (typeof error === "object" && error !== null && "_tag" in error) {
868
+ const record = error as Record<string, unknown>
869
+ const message = record["message"]
870
+ const detail = typeof message === "string" && message !== "" ? message : "(no detail)"
871
+ return `${String(record["_tag"])}: ${detail}`
872
+ }
873
+ if (error instanceof Error) {
874
+ return `${error.name}: ${error.message !== "" ? error.message : "(no detail)"}`
875
+ }
876
+ return String(error)
877
+ }
878
+
879
+ /**
880
+ * The single failure renderer (#13): cause first (the tagged error's derived
881
+ * message names the culprit), an actionable hint where one exists, and the
882
+ * Effect fiber trace ONLY under `--log-level debug` — an operator or agent
883
+ * sees one screen with the real cause, not a stack over an empty message. The
884
+ * exit code stays a frozen contract (0/1/2): the caller sets it, not this.
885
+ */
886
+ export const renderFailure = (
887
+ cause: Cause.Cause<unknown>,
888
+ options?: { readonly debug?: boolean },
889
+ ): string => {
890
+ const error = Cause.squash(cause)
891
+ const lines = [errorHeadline(error)]
892
+ const tag =
893
+ typeof error === "object" && error !== null && "_tag" in error
894
+ ? String((error as Record<string, unknown>)["_tag"])
895
+ : ""
896
+ const hint = FAILURE_HINTS[tag]?.(error as Record<string, unknown>)
897
+ if (hint !== undefined) lines.push(` → ${hint}`)
898
+ if (options?.debug === true) {
899
+ lines.push("", "── trace (--log-level debug) ──", Cause.pretty(cause))
900
+ } else {
901
+ lines.push(" (re-run with --log-level debug for the full trace)")
902
+ }
903
+ return lines.join("\n")
904
+ }
905
+
906
+ /**
907
+ * Whether the run asked for the full trace. Reused from the already-parsed
908
+ * global `--log-level` flag (values trace/debug/all mean "show me everything")
909
+ * so there is one knob, not a second bespoke verbosity flag.
910
+ */
911
+ export const wantsTrace = (argv: ReadonlyArray<string>): boolean => {
912
+ const verbose = new Set(["trace", "debug", "all"])
913
+ for (let index = 0; index < argv.length; index++) {
914
+ const arg = argv[index]!
915
+ if (arg.startsWith("--log-level=")) return verbose.has(arg.slice("--log-level=".length))
916
+ if (arg === "--log-level") return verbose.has((argv[index + 1] ?? "").toLowerCase())
917
+ }
918
+ return false
919
+ }
502
920
 
503
921
  export const rootCommand = Command.make("efmesh").pipe(
504
- Command.withDescription("sqlmesh на bun, typescript и Effect"),
922
+ Command.withDescription("sqlmesh on bun, typescript and Effect"),
505
923
  Command.withSubcommands([
506
924
  initCommand,
507
925
  planCommand,
508
926
  applyCommand,
509
927
  runCommand,
928
+ statusCommand,
510
929
  auditCommand,
511
930
  renderCommand,
512
931
  graphCommand,
513
932
  lineageCommand,
514
933
  diffCommand,
934
+ scheduleCommand,
515
935
  janitorCommand,
516
936
  migrateCommand,
517
937
  ]),