@avytheone/efmesh 0.1.0-beta.1 → 0.2.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@avytheone/efmesh",
-  "version": "0.1.0-beta.1",
+  "version": "0.2.0",
   "description": "sqlmesh-like data transformation framework on TypeScript, Bun and Effect",
   "license": "MIT",
   "author": "Alexey Yakimanskiy",
@@ -30,6 +30,9 @@
   ],
   "type": "module",
   "module": "src/index.ts",
+  "engines": {
+    "bun": ">=1.3.11"
+  },
   "bin": {
     "efmesh": "src/bin.ts"
   },

package/src/cli.ts

@@ -12,15 +12,28 @@ import { DuckDBEngineLive } from "./engine/duckdb.ts"
 import { PostgresEngineLive } from "./engine/postgres.ts"
 import { PostgresStateLive } from "./state/postgres.ts"
 import { auditEnvironment, EnvironmentAuditError } from "./plan/audit-run.ts"
-import { diffEnvironments } from "./plan/diff.ts"
+import {
+  dataDiffEnvironments,
+  diffEnvironments,
+  type DataDiffReport,
+} from "./plan/diff.ts"
+import { EngineAdapter } from "./engine/adapter.ts"
+import { ducklakeAttachSql } from "./plan/naming.ts"
+import {
+  listSchedules,
+  registerSchedule,
+  removeSchedule,
+  systemdUnits,
+} from "./plan/schedule.ts"
 import { renderGraphHtml } from "./plan/graph-html.ts"
 import { formatLineage, lineage, LineageError } from "./plan/lineage.ts"
+import { environmentStatus } from "./plan/status.ts"
 import { janitor } from "./plan/janitor.ts"
 import { fp8 } from "./plan/naming.ts"
 import { envLockName, withStateLock } from "./plan/lock.ts"
 import { applyPlan } from "./plan/executor.ts"
 import { run } from "./plan/run.ts"
-import { planChanges, type Plan } from "./plan/planner.ts"
+import { planChanges, ReclassifyError, type Plan } from "./plan/planner.ts"
 import { migratePostgresState } from "./state/postgres.ts"
 import { migrateSqliteState, SqliteStateLive } from "./state/sqlite.ts"
 
@@ -105,6 +118,13 @@ const forwardOnlyFlag = Flag.string("forward-only").pipe(
   ),
 )
 
+const reclassifyFlag = Flag.string("reclassify").pipe(
+  Flag.withDefault(""),
+  Flag.withDescription(
+    "Override категоризации (#5): «модель=breaking|non-breaking[,…]» поверх --explain; журналируется с applied_by",
+  ),
+)
+
 const jobsFlag = Flag.string("jobs").pipe(
   Flag.withDefault(""),
   Flag.withDescription(
@@ -139,6 +159,35 @@ const parseForwardOnly = (value: string): ReadonlyArray<string> | undefined => {
   return names.length > 0 ? names : undefined
 }
 
+/** `модель=breaking|non-breaking[,…]` → запись для PlanOptions.reclassify (#5). */
+export const parseReclassify = (
+  value: string,
+): Effect.Effect<Readonly<Record<string, "breaking" | "non-breaking">> | undefined, ReclassifyError> =>
+  Effect.gen(function* () {
+    const entries = value
+      .split(",")
+      .map((entry) => entry.trim())
+      .filter((entry) => entry !== "")
+    if (entries.length === 0) return undefined
+    const parsed: Record<string, "breaking" | "non-breaking"> = {}
+    for (const entry of entries) {
+      const [model, category, ...extra] = entry.split("=")
+      if (
+        model === undefined ||
+        model === "" ||
+        extra.length > 0 ||
+        (category !== "breaking" && category !== "non-breaking")
+      ) {
+        return yield* new ReclassifyError({
+          model: entry,
+          reason: "ожидается «модель=breaking» или «модель=non-breaking»",
+        })
+      }
+      parsed[model] = category
+    }
+    return parsed
+  })
+
 const yesFlag = Flag.boolean("yes").pipe(
   Flag.withAlias("y"),
   Flag.withDescription("Применить без подтверждения (не-TTY подтверждения не спрашивает)"),
@@ -167,22 +216,74 @@ export const decideApply = (
   tty: boolean,
 ): "apply" | "ask" | "refuse" => (!hasChanges || yes ? "apply" : tty ? "ask" : "refuse")
 
+const jsonFlag = Flag.boolean("json").pipe(
+  Flag.withDescription("Машиночитаемый вывод (стабильная форма — контракт для CI)"),
+)
+
+const explainFlag = Flag.boolean("explain").pipe(
+  Flag.withDescription(
+    "К каждому изменению — какие узлы канонического AST разошлись и почему категория такая",
+  ),
+)
+
+/**
+ * JSON-форма плана (#3) — КОНТРАКТ для CI и ботов: изменения формы —
+ * semver-события пакета. Интервалы — ISO UTC, не epoch ms.
+ */
+export const planToJson = (plan: Plan): unknown => ({
+  env: plan.env,
+  hasChanges: plan.hasChanges,
+  actions: plan.actions.map((action) => ({
+    name: action.name,
+    change: action.change,
+    // override оператора (#5) и реюз физики — аддитивные поля контракта
+    ...(action.reclassifiedFrom !== undefined
+      ? { reclassifiedFrom: action.reclassifiedFrom }
+      : {}),
+    ...(action.reusedFrom !== undefined ? { reusedFrom: action.reusedFrom } : {}),
+    fingerprint: action.fingerprint,
+    build: action.build,
+    backfill: action.backfill.map((range) => ({
+      start: new Date(range.start).toISOString(),
+      end: new Date(range.end).toISOString(),
+    })),
+    // причина категории (#4); diverged-пути — отладочная подсказка, не контракт
+    ...(action.explain !== undefined ? { explain: action.explain } : {}),
+  })),
+})
+
+const printJson = (payload: unknown) => Console.log(JSON.stringify(payload, null, 2))
+
 const formatRange = (range: { readonly start: number; readonly end: number }): string =>
   `[${new Date(range.start).toISOString().slice(0, 10)} … ${new Date(range.end).toISOString().slice(0, 10)})`
 
-const printPlan = (plan: Plan) =>
+const printPlan = (plan: Plan, explain = false) =>
   Effect.gen(function* () {
     yield* Console.log(`план для окружения «${plan.env}»:`)
     for (const action of plan.actions) {
       const mark = CHANGE_MARK[action.change] ?? "?"
+      const overridden =
+        action.reclassifiedFrom !== undefined
+          ? `  [override: было ${action.reclassifiedFrom}]`
+          : ""
+      const reused =
+        action.change === "indirect" && action.reusedFrom !== undefined
+          ? "  [физика реюзается]"
+          : ""
       const build = action.build ? "  [сборка]" : ""
       const backfill =
         action.backfill.length > 0
           ? `  бэкфилл ${action.backfill.map(formatRange).join(", ")}`
           : ""
       yield* Console.log(
-        `  ${mark} ${action.name}  ${action.change} @${fp8(action.fingerprint)}${build}${backfill}`,
+        `  ${mark} ${action.name}  ${action.change} @${fp8(action.fingerprint)}${overridden}${reused}${build}${backfill}`,
       )
+      if (explain && action.explain !== undefined) {
+        yield* Console.log(`      почему: ${action.explain.reason}`)
+        if (action.explain.diverged.length > 0) {
+          yield* Console.log(`      разошлось: ${action.explain.diverged.join(", ")}`)
+        }
+      }
     }
     if (!plan.hasChanges) yield* Console.log("  изменений нет")
   })
@@ -200,15 +301,24 @@ const initCommand = Command.make(
 
 const planCommand = Command.make(
   "plan",
-  { env: Argument.string("env"), config: configFlag, forwardOnly: forwardOnlyFlag },
-  ({ config, env, forwardOnly }) =>
+  {
+    env: Argument.string("env"),
+    config: configFlag,
+    forwardOnly: forwardOnlyFlag,
+    reclassify: reclassifyFlag,
+    json: jsonFlag,
+    explain: explainFlag,
+  },
+  ({ config, env, explain, forwardOnly, json, reclassify }) =>
     Effect.gen(function* () {
       const loaded = yield* loadConfig(config)
       const names = parseForwardOnly(forwardOnly)
+      const overrides = yield* parseReclassify(reclassify)
       const plan = yield* Efmesh.plan(env, loaded.models, {
         ...(names !== undefined ? { forwardOnly: names } : {}),
+        ...(overrides !== undefined ? { reclassify: overrides } : {}),
       }).pipe(Effect.provide(configLayers(loaded)))
-      yield* printPlan(plan)
+      yield* json ? printJson(planToJson(plan)) : printPlan(plan, explain)
     }),
 ).pipe(Command.withDescription("Показать diff проекта против окружения, ничего не меняя"))
 
@@ -218,14 +328,16 @@ const applyCommand = Command.make(
     env: Argument.string("env"),
     config: configFlag,
     forwardOnly: forwardOnlyFlag,
+    reclassify: reclassifyFlag,
     jobs: jobsFlag,
     retries: retriesFlag,
     yes: yesFlag,
   },
-  ({ config, env, forwardOnly, jobs, retries, yes }) =>
+  ({ config, env, forwardOnly, jobs, reclassify, retries, yes }) =>
     Effect.gen(function* () {
       const loaded = yield* loadConfig(config)
       const names = parseForwardOnly(forwardOnly)
+      const overrides = yield* parseReclassify(reclassify)
       const modelConcurrency = parseJobs(jobs)
       const retry = parseRetries(retries)
       // план и применение — под одним слоем и одним межпроцессным локом:
@@ -236,6 +348,7 @@ const applyCommand = Command.make(
         const graph = yield* buildGraph(loaded.models)
         const plan = yield* planChanges(env, graph, {
           ...(names !== undefined ? { forwardOnly: names } : {}),
+          ...(overrides !== undefined ? { reclassify: overrides } : {}),
         })
         yield* printPlan(plan)
         const decision = decideApply(plan.hasChanges, yes, process.stdin.isTTY === true)
@@ -335,15 +448,100 @@ const runCommand = Command.make(
   ),
 )
 
+const printDataDiff = (report: DataDiffReport) =>
+  Effect.gen(function* () {
+    yield* Console.log(`данные ${report.envA} ↔ ${report.envB}:`)
+    if (report.models.length === 0) {
+      yield* Console.log("  общих материализуемых моделей нет")
+      return
+    }
+    for (const entry of report.models) {
+      if (entry.key === undefined) {
+        yield* Console.log(
+          `  · ${entry.model}  A=${entry.rowsA} B=${entry.rowsB}  без ключа (задайте grain) — только счётчики`,
+        )
+      } else {
+        const clean =
+          entry.onlyInA === 0 &&
+          entry.onlyInB === 0 &&
+          (entry.columns?.length ?? 0) === 0 &&
+          entry.rowsA === entry.rowsB
+        const sampled =
+          entry.sampledPercent !== undefined ? `  (выборка ${entry.sampledPercent}%)` : ""
+        yield* Console.log(
+          `  ${clean ? "✓" : "≠"} ${entry.model}  A=${entry.rowsA} B=${entry.rowsB}  ключ (${entry.key.join(", ")}): только в A ${entry.onlyInA}, только в B ${entry.onlyInB}, совпало ${entry.matched}${sampled}`,
+        )
+        for (const drift of entry.columns ?? []) {
+          yield* Console.log(
+            `      ${drift.column}: ${drift.mismatches} из ${entry.matched} (${(drift.rate * 100).toFixed(2)}%)`,
+          )
+        }
+      }
+      if (entry.columnsOnlyInA !== undefined) {
+        yield* Console.log(`      колонки только в A: ${entry.columnsOnlyInA.join(", ")}`)
+      }
+      if (entry.columnsOnlyInB !== undefined) {
+        yield* Console.log(`      колонки только в B: ${entry.columnsOnlyInB.join(", ")}`)
+      }
+    }
+  })
+
 const diffCommand = Command.make(
   "diff",
-  { envA: Argument.string("envA"), envB: Argument.string("envB"), config: configFlag },
-  ({ config, envA, envB }) =>
+  {
+    envA: Argument.string("envA"),
+    envB: Argument.string("envB"),
+    config: configFlag,
+    data: Flag.boolean("data").pipe(
+      Flag.withDescription(
+        "Сравнить ДАННЫЕ view-слоёв: счётчики строк, пересечение по ключу, расхождения по колонкам",
+      ),
+    ),
+    model: Flag.string("model").pipe(
+      Flag.withDefault(""),
+      Flag.withDescription("Только эти модели, через запятую (для --data)"),
+    ),
+    sample: Flag.string("sample").pipe(
+      Flag.withDefault(""),
+      Flag.withDescription(
+        "Процент 1–99: сравнивать детерминированную долю ключей (md5-бакеты; для --data)",
+      ),
+    ),
+    json: jsonFlag,
+  },
+  ({ config, data, envA, envB, json, model, sample }) =>
     Effect.gen(function* () {
       const loaded = yield* loadConfig(config)
+      if (data) {
+        const only = parseForwardOnly(model)
+        const percent = sample === "" ? undefined : Number(sample)
+        if (percent !== undefined && !(Number.isFinite(percent) && percent >= 1 && percent <= 99)) {
+          yield* Console.error("--sample ожидает процент от 1 до 99")
+          return yield* Effect.sync(() => {
+            process.exitCode = 1
+          })
+        }
+        const report = yield* Effect.gen(function* () {
+          // ducklake-витрины видны через ATTACH — как делает apply
+          if (loaded.ducklake !== undefined) {
+            const engine = yield* EngineAdapter
+            yield* engine.execute(ducklakeAttachSql(loaded.ducklake))
+          }
+          return yield* dataDiffEnvironments(envA, envB, loaded.models, {
+            ...(only !== undefined ? { models: only } : {}),
+            ...(percent !== undefined ? { samplePercent: percent } : {}),
+          })
+        }).pipe(Effect.provide(configLayers(loaded)))
+        yield* json ? printJson(report) : printDataDiff(report)
+        return
+      }
       const diff = yield* diffEnvironments(envA, envB).pipe(
         Effect.provide(configLayers(loaded)),
       )
+      if (json) {
+        yield* printJson(diff)
+        return
+      }
       for (const name of diff.onlyInA) yield* Console.log(`< ${name}  только в ${envA}`)
       for (const name of diff.onlyInB) yield* Console.log(`> ${name}  только в ${envB}`)
       for (const entry of diff.different) {
@@ -353,7 +551,85 @@ const diffCommand = Command.make(
         yield* Console.log("окружения идентичны")
       }
     }),
-).pipe(Command.withDescription("Чем окружения отличаются (по state store)"))
+).pipe(
+  Command.withDescription("Чем окружения отличаются: версии (state store) или --data (данные)"),
+)
+
+const scheduleCommand = Command.make(
+  "schedule",
+  {
+    env: Argument.string("env").pipe(Argument.withDefault("")),
+    config: configFlag,
+    cron: Flag.string("cron").pipe(
+      Flag.withDefault("@hourly"),
+      Flag.withDescription("Cron-выражение или никнейм (@hourly, @daily, …)"),
+    ),
+    remove: Flag.boolean("remove").pipe(
+      Flag.withDescription("Снять регистрацию окружения из OS-шедулера"),
+    ),
+    list: Flag.boolean("list").pipe(
+      Flag.withDescription("Показать efmesh-записи OS-шедулера"),
+    ),
+    printSystemd: Flag.boolean("print-systemd").pipe(
+      Flag.withDescription(
+        "Напечатать systemd user-юниты вместо cron (Persistent=true догоняет пропуски; спасение без cron-демона)",
+      ),
+    ),
+  },
+  ({ config, cron, env, list, printSystemd, remove }) =>
+    Effect.gen(function* () {
+      if (list) {
+        const entries = yield* listSchedules()
+        if (entries.length === 0) yield* Console.log("efmesh-записей в OS-шедулере нет")
+        for (const entry of entries) yield* Console.log(`  ${entry}`)
+        return
+      }
+      if (env === "") {
+        yield* Console.error("нужно окружение: efmesh schedule <env> [--cron …]")
+        return yield* Effect.sync(() => {
+          process.exitCode = 1
+        })
+      }
+      const configAbs = NodePath.resolve(process.cwd(), config)
+      const target = { project: NodePath.dirname(configAbs), config: configAbs, env }
+      if (printSystemd) {
+        const units = systemdUnits(target, cron)
+        yield* Console.log(`# ~/.config/systemd/user/${units.name}.service`)
+        yield* Console.log(units.service)
+        yield* Console.log(`# ~/.config/systemd/user/${units.name}.timer`)
+        yield* Console.log(units.timer)
+        yield* Console.log(
+          `# включить: systemctl --user daemon-reload && systemctl --user enable --now ${units.name}.timer`,
+        )
+        return
+      }
+      if (remove) {
+        const removed = yield* removeSchedule(target)
+        yield* Console.log(`снято: ${removed.title}`)
+        return
+      }
+      const registered = yield* registerSchedule(target, cron)
+      yield* Console.log(`зарегистрировано: ${registered.title} — «${cron}» (OS-шедулер)`)
+      yield* Console.log(`воркер: ${registered.worker}`)
+      yield* Console.log(
+        "журнал тиков: efmesh status " + env + "; NB: cron не догоняет пропущенные запуски — строже systemd-таймер (--print-systemd)",
+      )
+    }).pipe(
+      // reason — самое ценное (рецепт для оператора): наружу словами, не стектрейсом
+      Effect.catchTag("ScheduleError", (error) =>
+        Effect.gen(function* () {
+          yield* Console.error(`schedule: ${error.reason}`)
+          yield* Effect.sync(() => {
+            process.exitCode = 1
+          })
+        }),
+      ),
+    ),
+).pipe(
+  Command.withDescription(
+    "Зарегистрировать run <env> в OS-шедулере (Bun.cron: crontab/launchd/Task Scheduler)",
+  ),
+)
 
 const janitorCommand = Command.make(
   "janitor",
@@ -383,6 +659,56 @@ const janitorCommand = Command.make(
     }),
 ).pipe(Command.withDescription("Убрать физику, на которую не ссылается ни одно окружение"))
 
+const statusCommand = Command.make(
+  "status",
+  { env: Argument.string("env"), config: configFlag, json: jsonFlag },
+  ({ config, env, json }) =>
+    Effect.gen(function* () {
+      const loaded = yield* loadConfig(config)
+      const report = yield* environmentStatus(env, loaded.models).pipe(
+        Effect.provide(configLayers(loaded)),
+      )
+      if (json) {
+        yield* printJson(report)
+        return
+      }
+      if (report.models === 0) {
+        yield* Console.log(`окружение «${env}» не существует — его создаст первый apply`)
+        return
+      }
+      yield* Console.log(
+        `окружение «${env}»: моделей ${report.models}, промоушен ${report.promotedAt}, схема стора v${report.storeVersion}`,
+      )
+      if (report.lastPlan !== null) {
+        yield* Console.log(
+          `последний план: ${report.lastPlan.appliedAt} (${report.lastPlan.appliedBy || "неизвестно"})`,
+        )
+      }
+      for (const lag of report.lag) {
+        const state =
+          lag.missing === 0
+            ? `догнано до ${lag.doneUpTo}`
+            : `отстаёт на ${lag.missing} интервал(ов), догнано до ${lag.doneUpTo ?? "—"}`
+        const failed = lag.failed > 0 ? `  ⚠ failed-интервалов: ${lag.failed}` : ""
+        yield* Console.log(`  ${lag.missing === 0 ? "✓" : "…"} ${lag.model}  ${state}${failed}`)
+      }
+      if (report.ticks.length === 0) {
+        yield* Console.log("тиков run ещё не было")
+      } else {
+        yield* Console.log("последние тики run:")
+        for (const tick of report.ticks) {
+          const mark = tick.outcome === "ok" ? "✓" : tick.outcome === "error" ? "✗" : "…"
+          const ms = Date.parse(tick.finishedAt) - Date.parse(tick.startedAt)
+          yield* Console.log(
+            `  ${mark} ${tick.startedAt}  ${tick.outcome} (${ms} мс)${tick.detail !== "" ? `  ${tick.detail}` : ""}`,
+          )
+        }
+      }
+    }),
+).pipe(
+  Command.withDescription("Что происходит в окружении: промоушен, отставание, тики run"),
+)
+
 const auditCommand = Command.make(
   "audit",
   {
@@ -392,14 +718,26 @@ const auditCommand = Command.make(
       Flag.withDefault(""),
       Flag.withDescription("Только эти модели, через запятую (по умолчанию — все с аудитами)"),
     ),
+    json: jsonFlag,
   },
-  ({ config, env, model }) =>
+  ({ config, env, model, json }) =>
     Effect.gen(function* () {
       const loaded = yield* loadConfig(config)
       const only = parseForwardOnly(model)
       const report = yield* auditEnvironment(env, loaded.models, only).pipe(
         Effect.provide(configLayers(loaded)),
       )
+      if (json) {
+        // отчёт целиком; exit-код по blocking сохраняется — stdout чистый JSON
+        yield* printJson(report)
+        if (report.blockingViolations > 0) {
+          return yield* new EnvironmentAuditError({
+            env,
+            blockingViolations: report.blockingViolations,
+          })
+        }
+        return
+      }
       if (report.results.length === 0) {
         yield* Console.log("аудитов нет — нечего проверять")
         return
@@ -507,11 +845,13 @@ export const rootCommand = Command.make("efmesh").pipe(
     planCommand,
     applyCommand,
     runCommand,
+    statusCommand,
     auditCommand,
     renderCommand,
     graphCommand,
     lineageCommand,
     diffCommand,
+    scheduleCommand,
     janitorCommand,
     migrateCommand,
   ]),

package/src/efmesh.ts

@@ -16,6 +16,7 @@ import {
   planChanges,
   type FingerprintVersionError,
   type ForwardOnlyError,
+  type ReclassifyError,
   type InvalidEnvironmentError,
   type Plan,
   type PlanOptions,
@@ -42,6 +43,7 @@ export const Efmesh = {
     | StateError
     | InvalidEnvironmentError
     | ForwardOnlyError
+    | ReclassifyError
     | FingerprintVersionError
     | EngineError
     | SqlParseError

package/src/index.ts

@@ -41,8 +41,32 @@ export {
   type AuditRunReport,
   type AuditRunResult,
 } from "./plan/audit-run.ts"
-export { diffEnvironments, type EnvDiff } from "./plan/diff.ts"
+export {
+  dataDiffEnvironments,
+  DataDiffError,
+  diffEnvironments,
+  type ColumnDrift,
+  type DataDiffOptions,
+  type DataDiffReport,
+  type EnvDiff,
+  type ModelDataDiff,
+} from "./plan/diff.ts"
+export {
+  environmentStatus,
+  type ModelLag,
+  type StatusOptions,
+  type StatusReport,
+} from "./plan/status.ts"
 export { formatLineage, lineage, LineageError, type LineageNode } from "./plan/lineage.ts"
+export {
+  listSchedules,
+  registerSchedule,
+  removeSchedule,
+  ScheduleError,
+  scheduleTitle,
+  systemdUnits,
+  type ScheduleTarget,
+} from "./plan/schedule.ts"
 
 // — план и применение: типы результата и опций —
 export type {
@@ -51,7 +75,13 @@ export type {
   PlanAction,
   PlanOptions,
 } from "./plan/planner.ts"
-export { FingerprintVersionError, ForwardOnlyError, InvalidEnvironmentError } from "./plan/planner.ts"
+export type { ChangeExplanation } from "./plan/explain.ts"
+export {
+  FingerprintVersionError,
+  ForwardOnlyError,
+  InvalidEnvironmentError,
+  ReclassifyError,
+} from "./plan/planner.ts"
 export type { AppliedPlan, ApplyError, ApplyOptions } from "./plan/executor.ts"
 export {
   AttachNotConfiguredError,

package/src/plan/categorize.ts

@@ -8,7 +8,10 @@
 
 type AstNode = Record<string, unknown>
 
-const topSelect = (astJson: string): { list: ReadonlyArray<string>; rest: string } | null => {
+/** Верхний SELECT канона: список колонок + остальное дерево (для explain.ts). */
+export const topSelect = (
+  astJson: string,
+): { list: ReadonlyArray<string>; rest: string } | null => {
   try {
     const ast = JSON.parse(astJson) as {
       readonly statements?: ReadonlyArray<{ readonly node?: AstNode }>