@avytheone/efmesh 0.1.0-beta.2 → 0.2.1

package/src/plan/naming.ts

@@ -1,14 +1,14 @@
 import type { ExternalSource, ModelName } from "../core/model.ts"
 
 /**
- * Раскладка объектов в движке (SPEC §2):
- * - физика: схема `_efmesh`, таблица `<схема>__<таблица>__<fp8>`;
- * - виртуалка: prod живёт в родных схемах моделей (`med.stays`),
- *   остальные окружения — в префиксованных (`dev__med.stays`).
+ * Object layout in the engine (SPEC §2):
+ * - physical: schema `_efmesh`, table `<schema>__<table>__<fp8>`;
+ * - virtual: prod lives in the models' native schemas (`med.stays`),
+ *   other environments — in prefixed ones (`dev__med.stays`).
  *
- * Схема именно `_efmesh`, не `efmesh`: DuckDB называет каталог по имени
- * файла базы, и для `efmesh.duckdb` ссылка `efmesh.x` становится
- * неоднозначной (каталог или схема) — Binder Error.
+ * The schema is `_efmesh`, not `efmesh`, precisely because: DuckDB names the
+ * catalog after the database file name, and for `efmesh.duckdb` the reference
+ * `efmesh.x` becomes ambiguous (catalog or schema) — a Binder Error.
  */
 
 export const PROD_ENV = "prod"
@@ -31,28 +31,28 @@ export const envSchema = (env: string, modelSchema: string): string =>
   env === PROD_ENV ? modelSchema : `${env}__${modelSchema}`
 
 /**
- * Раскладка parquet-озера (SPEC §3.3): `<lake>/<схема>/<таблица>/fp=<fp8>/…`,
- * у incremental внутри — партиции `interval=<ключ>/data.parquet`.
- * Интервал = партиция: пересчёт — перезапись файлов партиции, источник
- * правды — учёт интервалов, поэтому неатомарность перезаписи не страшна.
+ * Parquet-lake layout (SPEC §3.3): `<lake>/<schema>/<table>/fp=<fp8>/…`,
+ * for incremental inside — partitions `interval=<key>/data.parquet`.
+ * Interval = partition: a recompute rewrites the partition's files, the
+ * source of truth is the interval bookkeeping, so a non-atomic rewrite is harmless.
  */
 export const parquetPrefix = (lakePath: string, name: ModelName, fingerprint: string): string =>
   `${lakePath.replace(/\/+$/, "")}/${name.schema}/${name.table}/fp=${fp8(fingerprint)}`
 
 /**
- * union_by_name: партиции одного префикса могут отличаться схемой после
- * forward-only-эволюции (новые колонки появляются только в новых файлах —
- * история читается с NULL).
+ * union_by_name: partitions of one prefix may differ in schema after
+ * forward-only evolution (new columns appear only in new files — history is
+ * read with NULL).
  */
 export const parquetRef = (lakePath: string, name: ModelName, fingerprint: string): string =>
   `read_parquet('${parquetPrefix(lakePath, name, fingerprint).replaceAll(`'`, `''`)}/**/*.parquet', union_by_name=true)`
 
 /**
- * DuckLake-цель (SPEC §14.5): каталог подключается под фиксированным
- * алиасом, физика — таблица-на-fingerprint в нём (то же имя, что у
- * нативной физики, только каталог другой). Версионность остаётся нашей;
- * снапшоты/time travel DuckLake — бонус. Потребители вне efmesh должны
- * сами сделать ATTACH — view окружений ссылаются на каталог по алиасу.
+ * DuckLake target (SPEC §14.5): the catalog is attached under a fixed alias,
+ * the physical storage is a fingerprint table in it (the same name as native
+ * physical storage, only a different catalog). Versioning stays ours;
+ * DuckLake snapshots/time travel are a bonus. Consumers outside efmesh must
+ * ATTACH it themselves — the environments' views reference the catalog by alias.
  */
 export const ducklakeAlias = "_efmesh_ducklake"
 
@@ -69,7 +69,7 @@ export const ducklakeAttachSql = (config: {
       : ""
   }`
 
-/** Ключ партиции интервала — безопасен для файловых систем (без двоеточий). */
+/** Interval partition key — filesystem-safe (no colons). */
 export const intervalKey = (unit: "day" | "hour", startMs: number): string => {
   const iso = new Date(startMs).toISOString()
   return unit === "day" ? iso.slice(0, 10) : `${iso.slice(0, 10)}T${iso.slice(11, 13)}`
@@ -78,9 +78,9 @@ export const intervalKey = (unit: "day" | "hour", startMs: number): string => {
 const READERS = { parquet: "read_parquet", csv: "read_csv", json: "read_json" } as const
 
 /**
- * Во что рендерится ссылка на external-модель (SPEC §9.3): имя таблицы
- * (движка или ATTACH-базы) как есть, файлы/URL — через read_*.
- * external не материализуется, потребители читают источник напрямую.
+ * What a reference to an external model renders to (SPEC §9.3): the table name
+ * (of the engine or an ATTACH database) as is, files/URLs — via read_*.
+ * external is not materialized, consumers read the source directly.
  */
 export const externalSourceRef = (source: ExternalSource): string =>
   source._tag === "table"

package/src/plan/planner.ts

@@ -8,47 +8,75 @@ import { EngineAdapter } from "../engine/adapter.ts"
 import { StateStore } from "../state/store.ts"
 import type { StateError } from "../state/store.ts"
 import { categorizeAstChange } from "./categorize.ts"
-import { FINGERPRINT_VERSION, fingerprintGraph } from "./fingerprint.ts"
+import type { ChangeExplanation } from "./explain.ts"
+import { dropsColumns, explainCategorized } from "./explain.ts"
+import { FINGERPRINT_VERSION, fingerprintGraph, modelFingerprint } from "./fingerprint.ts"
 import { validateEnvName } from "./naming.ts"
 
 export class InvalidEnvironmentError extends Data.TaggedError("InvalidEnvironmentError")<{
   readonly env: string
-}> {}
+}> {
+  override get message(): string {
+    return `invalid environment name «${this.env}» — use latin letters, digits and _`
+  }
+}
 
-/** Модель нельзя применить forward-only (SPEC §5.2). */
+/** The model cannot be applied forward-only (SPEC §5.2). */
 export class ForwardOnlyError extends Data.TaggedError("ForwardOnlyError")<{
   readonly model: string
   readonly reason: string
-}> {}
+}> {
+  override get message(): string {
+    return `model «${this.model}» cannot be applied forward-only: ${this.reason}`
+  }
+}
+
+/**
+ * A categorization override is rejected (#5): the model is not in the project
+ * or the AST diff obviously contradicts the claimed verdict (removed columns
+ * are never non-breaking — descendants read them by name).
+ */
+export class ReclassifyError extends Data.TaggedError("ReclassifyError")<{
+  readonly model: string
+  readonly reason: string
+}> {
+  override get message(): string {
+    return `reclassify «${this.model}»: ${this.reason}`
+  }
+}
 
 /**
- * Снапшот посчитан другой версией алгоритма fingerprint (SPEC §4):
- * отпечатки разных версий несравнимы — план честно останавливается,
- * а не показывает «всё breaking». Лечится миграцией той версии efmesh,
- * которая сменила алгоритм.
+ * The snapshot was computed by a different version of the fingerprint
+ * algorithm (SPEC §4): fingerprints of different versions are incomparable —
+ * the plan honestly stops instead of showing "everything breaking". Cured by
+ * migrating to the version of efmesh that changed the algorithm.
  */
 export class FingerprintVersionError extends Data.TaggedError("FingerprintVersionError")<{
   readonly model: string
   readonly found: number
   readonly wanted: number
-}> {}
+}> {
+  override get message(): string {
+    return `model «${this.model}» was fingerprinted by algorithm v${this.found}, but this efmesh expects v${this.wanted}`
+  }
+}
 
 export type ChangeCategory =
-  /** Модели не было в окружении. */
+  /** The model was not in the environment. */
   | "added"
-  /** Смысловая правка запроса или метаданных: модель и потомки пересобираются. */
+  /** A meaningful edit of the query or metadata: the model and its descendants rebuild. */
   | "breaking"
-  /** Добавлены колонки в конец SELECT, остальное дерево нетронуто (SPEC §5.2). */
+  /** Columns appended to the end of the SELECT, the rest of the tree untouched (SPEC §5.2). */
   | "non-breaking"
-  /** Собственный AST не менялся — версия сдвинулась каскадом от родителя. */
+  /** The own AST did not change — the version shifted by cascade from a parent. */
   | "indirect"
   /**
-   * Физика и done-интервалы старой версии переиспользуются, история не
-   * переигрывается — новая логика действует с момента применения (SPEC §5.2).
-   * Явный флаг пользователя или каскад от forward-only-родителей.
+   * The physics and done-intervals of the old version are reused, history is
+   * not replayed — the new logic takes effect from the moment of application
+   * (SPEC §5.2). An explicit user flag or a cascade from forward-only parents.
    */
   | "forward-only"
-  /** Была в окружении, из проекта исчезла — view будет снесён при промоушене. */
+  /** Was in the environment, vanished from the project — the view is dropped at promotion. */
   | "removed"
   | "unchanged"
 
@@ -56,50 +84,81 @@ export interface PlanAction {
   readonly name: string
   readonly fingerprint: string
   /**
-   * Fingerprint, чьей физикой пользуется снапшот: обычно собственный,
-   * при forward-only — унаследованный от предыдущей версии.
+   * The fingerprint whose physics the snapshot uses: usually its own, and
+   * under forward-only — inherited from the previous version.
    */
   readonly physicalFingerprint: string
-  /** Откуда наследуются физика и done-интервалы (fingerprint старой версии) при forward-only. */
+  /** Where the physics and done-intervals are inherited from (fingerprint of the old version) under forward-only. */
   readonly reusedFrom?: string
-  /** Канонический AST тела (null у external) — сохраняется в снапшот. */
+  /** Canonical AST of the body (null for external) — saved into the snapshot. */
   readonly canonicalAst: string | null
   readonly change: ChangeCategory
+  /** The planner's verdict before the operator override (#5); journaled. */
+  readonly reclassifiedFrom?: ChangeCategory
+  /**
+   * Why the category is what it is (#4): the diverged nodes of the canonical
+   * AST and the reason in words. Present on all changed models except
+   * added/removed — there is nothing to compare there. The diverged paths are
+   * a debugging hint, not a contract.
+   */
+  readonly explain?: ChangeExplanation
   /**
-   * Нужна ли сборка физики. false для unchanged/removed, external и для
-   * снапшотов, уже собранных другим окружением, — тогда промоушен это
-   * только view-swap.
+   * Whether physics needs building. false for unchanged/removed, external and
+   * for snapshots already built by another environment — then promotion is
+   * only a view-swap.
    */
   readonly build: boolean
   /**
-   * Диапазоны к пересчёту у incrementalByTimeRange (слитые из недостающих
-   * интервалов + lookback); у остальных видов пуст. Дыры бывают и у
-   * unchanged-модели — время идёт, появляются новые интервалы.
+   * Ranges to recompute for incrementalByTimeRange (merged from missing
+   * intervals + lookback); empty for other kinds. Gaps happen even for an
+   * unchanged model — time passes, new intervals appear.
    */
   readonly backfill: ReadonlyArray<Interval>
-  /** true у incrementalByUniqueKey: каждый apply перегоняет запрос (upsert по ключу). */
+  /** true for incrementalByUniqueKey: every apply re-runs the query (upsert by key). */
   readonly refresh: boolean
 }
 
 export interface Plan {
   readonly env: string
-  /** В топологическом порядке; removed — в конце. */
+  /** In topological order; removed comes last. */
   readonly actions: ReadonlyArray<PlanAction>
   readonly hasChanges: boolean
 }
 
 export interface PlanOptions {
-  /** «Сейчас» для расчёта интервалов; по умолчанию — Clock. Инъекция для тестов. */
+  /** "Now" for computing intervals; defaults to Clock. Injection point for tests. */
   readonly now?: number
   /**
-   * Модели, чьи изменения применить forward-only (SPEC §5.2): новая версия
-   * наследует физическую таблицу и done-интервалы старой, история не
-   * переигрывается. Только incrementalByTimeRange — у остальных видов
-   * «задним числом» не бывает по построению.
+   * Models whose changes to apply forward-only (SPEC §5.2): the new version
+   * inherits the physical table and done-intervals of the old one, history is
+   * not replayed. Only incrementalByTimeRange — other kinds have no
+   * "retroactive" notion by construction.
    */
   readonly forwardOnly?: ReadonlyArray<string>
+  /**
+   * Categorization override (#5, SPEC §5.2): the operator, looking at
+   * `--explain`, states the verdict instead of the planner. It governs the
+   * fate of DESCENDANTS (a non-breaking parent lets them reuse the physics);
+   * the model itself is freed from rebuilding not by this but by forwardOnly.
+   * Guardrail: an obvious AST contradiction (removed columns → non-breaking)
+   * is an error. Applies only to breaking/non-breaking verdicts; on unchanged/
+   * added/removed/indirect it silently has no effect.
+   */
+  readonly reclassify?: Readonly<Record<string, "breaking" | "non-breaking">>
 }
 
+/**
+ * Kinds whose physics is worth inheriting on indirect reuse (#5):
+ * materialized tables. view/embedded have no materialization; seed rebuilds
+ * cheaply from a file and has no parents.
+ */
+const REUSABLE_KINDS: ReadonlySet<string> = new Set([
+  "full",
+  "incrementalByTimeRange",
+  "incrementalByUniqueKey",
+  "scdType2",
+])
+
 export const planChanges = (
   env: string,
   graph: ModelGraph,
@@ -108,6 +167,7 @@ export const planChanges = (
   Plan,
   | InvalidEnvironmentError
   | ForwardOnlyError
+  | ReclassifyError
   | FingerprintVersionError
   | StateError
   | EngineError
@@ -123,34 +183,49 @@ export const planChanges = (
     for (const flagged of forwardOnly) {
       const model = graph.models.get(flagged)
       if (model === undefined) {
-        return yield* new ForwardOnlyError({ model: flagged, reason: "модели нет в проекте" })
+        return yield* new ForwardOnlyError({ model: flagged, reason: "model is not in the project" })
       }
       if (model.kind._tag !== "incrementalByTimeRange") {
         return yield* new ForwardOnlyError({
           model: flagged,
-          reason: `forward-only переиспользует физику и учёт интервалов — применим только к incrementalByTimeRange, вид модели — ${model.kind._tag}`,
+          reason: `forward-only reuses physics and interval accounting — applicable only to incrementalByTimeRange, model kind is ${model.kind._tag}`,
         })
       }
     }
-    const versions = yield* fingerprintGraph(graph)
+    const reclassify = options?.reclassify ?? {}
+    for (const flagged of Object.keys(reclassify)) {
+      if (!graph.models.has(flagged)) {
+        return yield* new ReclassifyError({ model: flagged, reason: "model is not in the project" })
+      }
+    }
+    // canonicalization cache (#8) — the store is at hand; its failures are
+    // swallowed: a cache miss is just an honest recompute, not a plan error
+    const versions = yield* fingerprintGraph(graph, {
+      get: (key) => store.getCanon(key).pipe(Effect.orElseSucceed(() => undefined)),
+      put: (key, canonical) => store.putCanon(key, canonical).pipe(Effect.ignore),
+    })
     const current = new Map(
       (yield* store.getEnvironment(env)).map((row) => [row.name, row.fingerprint]),
     )
 
     const actions: Array<PlanAction> = []
     const changeOf = new Map<string, ChangeCategory>()
+    // who in this plan inherits the physics of the old version (indirect reuse, #5)
+    const reusedPhysics = new Set<string>()
     for (const name of graph.order) {
       const model = graph.models.get(name)!
       const { fingerprint, ast } = versions.get(name)!
       const known = current.get(name)
       let change: ChangeCategory
+      let explain: ChangeExplanation | undefined
+      let reclassifiedFrom: ChangeCategory | undefined
       let reusedFrom: string | undefined
       let physicalFingerprint = fingerprint
       if (known === undefined) change = "added"
       else if (known === fingerprint) change = "unchanged"
       else {
-        // категоризация по AST против последнего известного снапшота (SPEC §5.2);
-        // старых записей без AST и external — консервативно breaking
+        // categorization by AST against the last known snapshot (SPEC §5.2);
+        // old records without an AST and external — conservatively breaking
         const previous = yield* store.getSnapshot(name, known)
         if (previous !== undefined && previous.fingerprintVersion !== FINGERPRINT_VERSION) {
           return yield* new FingerprintVersionError({
@@ -160,14 +235,83 @@ export const planChanges = (
           })
         }
         const oldAst = previous?.canonicalAst ?? ""
-        if (oldAst === "" || ast === null) change = "breaking"
-        else if (oldAst === ast) change = "indirect" // версия сдвинута родителем/метаданными
-        else change = categorizeAstChange(oldAst, ast)
-        // forward-only: явный флаг пользователя — либо каскад: собственный AST
-        // не менялся, а все изменившиеся родители сами forward-only (реюз
-        // физики родителя означает, что и потомку нечего переигрывать)
-        const cascaded =
+        const changedParents = [...model.deps].filter((dep) => {
+          const parent = changeOf.get(dep)
+          return parent !== undefined && parent !== "unchanged"
+        })
+        if (oldAst === "" || ast === null) {
+          change = "breaking"
+          explain = {
+            diverged: [],
+            reason:
+              ast === null
+                ? "model has no SQL body (external/seed) — the version was shifted by source/file, schema or parents; there is no AST to compare against"
+                : "the previous snapshot did not store a canonical AST — nothing to compare against, conservatively breaking",
+          }
+        } else if (oldAst === ast) {
+          change = "indirect" // version shifted by a parent/metadata
+          explain =
+            changedParents.length > 0
+              ? {
+                  diverged: [],
+                  reason: `own AST did not change — the version was shifted by a cascade from parents: ${changedParents.join(", ")}`,
+                  cascadeFrom: changedParents,
+                }
+              : {
+                  diverged: [],
+                  reason:
+                    "own AST did not change — metadata diverged (kind/grain/columns/target)",
+                }
+        } else {
+          change = categorizeAstChange(oldAst, ast)
+          explain = explainCategorized(oldAst, ast, change)
+        }
+        // operator override (#5): the verdict is stated by a flag on top of
+        // --explain; the planner checks only an obvious AST contradiction
+        const override = reclassify[name]
+        if (
+          override !== undefined &&
+          (change === "breaking" || change === "non-breaking") &&
+          change !== override
+        ) {
+          if (override === "non-breaking" && (oldAst === "" || ast === null)) {
+            return yield* new ReclassifyError({
+              model: name,
+              reason:
+                "no canonical AST — nothing to check the verdict against, override is not accepted",
+            })
+          }
+          if (override === "non-breaking" && ast !== null && dropsColumns(oldAst, ast)) {
+            return yield* new ReclassifyError({
+              model: name,
+              reason:
+                "the new SELECT has fewer columns — descendants read the removed columns by name, non-breaking contradicts the AST",
+            })
+          }
+          reclassifiedFrom = change
+          explain = {
+            diverged: explain?.diverged ?? [],
+            reason: `operator override: ${change} → ${override}; planner verdict: ${explain?.reason ?? "—"}`,
+          }
+          change = override
+        }
+        // "version shifted by parents ONLY": a fingerprint with the old parent
+        // signatures must yield known — otherwise the metadata diverged along
+        // with the parents (kind/grain/columns/target) and physics cannot be inherited
+        const parentsOnly =
           change === "indirect" &&
+          previous !== undefined &&
+          [...model.deps].every((dep) => current.has(dep)) &&
+          (yield* modelFingerprint(
+            model,
+            ast,
+            [...model.deps].sort().map((dep) => `${dep}=${current.get(dep)!}`),
+          )) === known
+        // forward-only: an explicit user flag — or a cascade: the own AST did
+        // not change and all changed parents are themselves forward-only (reuse
+        // of a parent's physics means the descendant has nothing to replay either)
+        const cascaded =
+          parentsOnly &&
           model.kind._tag === "incrementalByTimeRange" &&
           [...model.deps].every((dep) => {
             const parent = changeOf.get(dep)
@@ -177,10 +321,50 @@ export const planChanges = (
           change = "forward-only"
           reusedFrom = known
           physicalFingerprint = previous.physicalFp
+          explain = forwardOnly.has(name)
+            ? {
+                diverged: explain?.diverged ?? [],
+                reason: `forward-only by flag: physics and done intervals are inherited from @${known.slice(0, 8)}, history is not replayed`,
+              }
+            : {
+                diverged: [],
+                reason:
+                  "own AST did not change, all changed parents are forward-only — physics is reused by cascade",
+                cascadeFrom: changedParents,
+              }
+        }
+        // indirect reuse (#5, sqlmesh's "indirect non-breaking" class): the own
+        // body did not change, the version was shifted only by parents, and each
+        // changed parent guarantees the same data in the old columns (non-breaking
+        // — strictly a suffix; forward-only and reuse — literally the same
+        // physics) — the descendant has nothing to replay: physics and ledger are
+        // inherited, scdType2 does not lose its accumulated row history
+        if (
+          change === "indirect" &&
+          parentsOnly &&
+          REUSABLE_KINDS.has(model.kind._tag) &&
+          changedParents.length > 0 &&
+          changedParents.every((dep) => {
+            const parent = changeOf.get(dep)!
+            return (
+              parent === "non-breaking" ||
+              parent === "forward-only" ||
+              (parent === "indirect" && reusedPhysics.has(dep))
+            )
+          })
+        ) {
+          reusedFrom = known
+          physicalFingerprint = previous!.physicalFp
+          reusedPhysics.add(name)
+          explain = {
+            diverged: [],
+            reason: `changed parents do not touch existing data (non-breaking/forward-only) — physics and accounting are inherited from @${known.slice(0, 8)}, no rebuild`,
+            cascadeFrom: changedParents,
+          }
         }
       }
       changeOf.set(name, change)
-      // external не материализуется — версия участвует в diff, физики нет
+      // external is not materialized — the version participates in the diff, there is no physics
       const existing = yield* store.getSnapshot(name, fingerprint)
       if (existing !== undefined) physicalFingerprint = existing.physicalFp
       const alreadyBuilt =
@@ -190,9 +374,9 @@ export const planChanges = (
       if (model.kind._tag === "incrementalByTimeRange") {
         const kind = model.kind
         const wanted = enumerateIntervals(kind.interval, fromIso(kind.start), now)
-        // покрытие привязано к fingerprint: новая версия = пустой учёт = полный
-        // бэкфилл; forward-only наследует done-интервалы старой версии —
-        // пересчитывается только то, чего не было
+        // coverage is keyed to the fingerprint: a new version = empty ledger =
+        // full backfill; forward-only inherits the done-intervals of the old
+        // version — only what was missing is recomputed
         const inherited =
           reusedFrom === undefined ? [] : yield* store.listIntervals(reusedFrom)
         const doneByStart = new Map<number, Interval>()
@@ -206,7 +390,7 @@ export const planChanges = (
         const done = [...doneByStart.values()]
         const missing = [...missingIntervals(wanted, done)]
         if (kind.lookback > 0) {
-          // поздно приезжающие данные: последние done-интервалы перечитываются
+          // late-arriving data: the last done-intervals are re-read
           const tail = done.sort((a, b) => a.start - b.start).slice(-kind.lookback)
           missing.push(...tail)
         }
@@ -220,9 +404,11 @@ export const planChanges = (
         ...(reusedFrom !== undefined ? { reusedFrom } : {}),
         canonicalAst: ast,
         change,
+        ...(reclassifiedFrom !== undefined ? { reclassifiedFrom } : {}),
+        ...(explain !== undefined ? { explain } : {}),
         build: !alreadyBuilt,
         backfill,
-        // каждый apply сверяет запрос с физикой: upsert / SCD-версионирование
+        // every apply reconciles the query with the physics: upsert / SCD versioning
         refresh:
           model.kind._tag === "incrementalByUniqueKey" || model.kind._tag === "scdType2",
       })