@avytheone/efmesh 0.1.0-beta.2 → 0.2.1

package/src/plan/audit-run.ts

@@ -8,13 +8,13 @@ import type { EngineError } from "../engine/adapter.ts"
 import { externalSourceRef, viewRef } from "./naming.ts"
 
 /**
- * Автономный прогон аудитов (SPEC §8, F4): проверяет то, что окружение
- * отдаёт потребителям СЕЙЧАС — view-слой целиком, не свежезагруженный
- * интервал, как в apply. Ловит деградацию задним числом: поздние данные
- * через lookback, руками поправленную физику, дрейф external-источников.
+ * Standalone audit run (SPEC §8, F4): checks what the environment serves to
+ * consumers RIGHT NOW — the whole view-layer, not the freshly loaded interval
+ * as in apply. Catches degradation after the fact: late data via lookback,
+ * physical storage edited by hand, drift of external sources.
  *
- * Ничего не меняет и не помечает; отчёт целиком — упавший blocking-аудит
- * не прячет следующие за ним.
+ * Changes and marks nothing; the report is complete — a failed blocking audit
+ * does not hide the ones that follow it.
  */
 
 export interface AuditRunResult {
@@ -26,21 +26,29 @@ export interface AuditRunResult {
 
 export interface AuditRunReport {
   readonly results: ReadonlyArray<AuditRunResult>
-  /** Суммарные нарушения blocking-аудитов — не ноль ⇒ окружению нельзя верить. */
+  /** Total blocking-audit violations — nonzero ⇒ the environment cannot be trusted. */
   readonly blockingViolations: number
 }
 
-/** Запрошенной модели нет в проекте (или у неё нет аудитов — нечего гонять). */
+/** The requested model is not in the project (or has no audits — nothing to run). */
 export class AuditTargetError extends Data.TaggedError("AuditTargetError")<{
   readonly model: string
   readonly reason: string
-}> {}
+}> {
+  override get message(): string {
+    return `audit target «${this.model}»: ${this.reason}`
+  }
+}
 
-/** Итог для CLI: blocking-аудиты окружения нарушены. */
+/** Result for the CLI: the environment's blocking audits are violated. */
 export class EnvironmentAuditError extends Data.TaggedError("EnvironmentAuditError")<{
   readonly env: string
   readonly blockingViolations: number
-}> {}
+}> {
+  override get message(): string {
+    return `environment «${this.env}»: ${this.blockingViolations} blocking audit violation(s)`
+  }
+}
 
 const selfFor = (graph: ModelGraph, env: string, model: AnyModel): string => {
   const resolve = (ref: string): string => {
@@ -51,7 +59,7 @@ const selfFor = (graph: ModelGraph, env: string, model: AnyModel): string => {
     }
     return viewRef(env, source.name)
   }
-  // embedded не материализуется — аудируется подзапрос против view окружения
+  // embedded is not materialized — the subquery is audited against the environment's view
   if (model.kind._tag === "embedded") {
     return `(${render(model.fragment, { resolveRef: resolve })})`
   }
@@ -68,7 +76,7 @@ export const auditEnvironment = (
     const graph = yield* buildGraph(models)
     for (const name of only ?? []) {
       if (!graph.models.has(name)) {
-        return yield* new AuditTargetError({ model: name, reason: "модели нет в проекте" })
+        return yield* new AuditTargetError({ model: name, reason: "model is not in the project" })
       }
     }
     const wanted = only === undefined ? undefined : new Set(only)

package/src/plan/categorize.ts

@@ -1,14 +1,17 @@
 /**
- * Категоризация изменения по каноническим AST (SPEC §5.2): non-breaking —
- * это добавление колонок В КОНЕЦ верхнего SELECT при полностью неизменном
- * остальном дереве (потребители выбирают по именам, INSERT потомков — по
- * позициям, поэтому только суффикс безопасен). Всё прочее — breaking.
- * При любой неожиданной структуре — консервативно breaking.
+ * Change categorization by canonical ASTs (SPEC §5.2): non-breaking is adding
+ * columns TO THE END of the top SELECT with the rest of the tree entirely
+ * unchanged (consumers select by name, descendants' INSERT is by position, so
+ * only a suffix is safe). Everything else is breaking. On any unexpected
+ * structure — conservatively breaking.
  */
 
 type AstNode = Record<string, unknown>
 
-const topSelect = (astJson: string): { list: ReadonlyArray<string>; rest: string } | null => {
+/** The canon's top SELECT: the column list + the rest of the tree (for explain.ts). */
+export const topSelect = (
+  astJson: string,
+): { list: ReadonlyArray<string>; rest: string } | null => {
   try {
     const ast = JSON.parse(astJson) as {
       readonly statements?: ReadonlyArray<{ readonly node?: AstNode }>
@@ -34,7 +37,7 @@ export const categorizeAstChange = (
   if (before === null || after === null) return "breaking"
   if (before.rest !== after.rest) return "breaking"
   if (after.list.length <= before.list.length) return "breaking"
-  // старый select_list — точный префикс нового
+  // the old select_list is an exact prefix of the new one
   return before.list.every((item, index) => item === after.list[index])
     ? "non-breaking"
     : "breaking"

package/src/plan/contract.ts

@@ -3,24 +3,28 @@ import type { AnyModel } from "../core/model.ts"
 import type { Engine, EngineError } from "../engine/adapter.ts"
 
 /**
- * Контракт схемы (SPEC §3.2): объявленная Schema — не документация.
- * Перед сборкой снапшота efmesh делает DESCRIBE запроса (движок отдаёт
- * имена и типы, не выполняя его) и сверяет с объявлением. Дрейф типов
- * ловится до бэкфилла, а не после.
+ * Schema contract (SPEC §3.2): the declared Schema is not documentation.
+ * Before building a snapshot efmesh runs a DESCRIBE of the query (the engine
+ * returns names and types without executing it) and checks it against the
+ * declaration. Type drift is caught before the backfill, not after.
  *
- * Сверка по семействам типов: точное соответствие TS-типов типам движка
- * невозможно (Number — это и INTEGER, и DOUBLE), а имена проверяются
- * строго. Nullability DESCRIBE не отдаёт — это территория аудитов (§14.2).
+ * Checked by type families: an exact match of TS types to engine types is
+ * impossible (Number is both INTEGER and DOUBLE), while names are checked
+ * strictly. DESCRIBE does not report nullability — that is audit territory (§14.2).
  */
 
 export class SchemaMismatchError extends Data.TaggedError("SchemaMismatchError")<{
   readonly model: string
   readonly problems: ReadonlyArray<string>
-}> {}
+}> {
+  override get message(): string {
+    return `schema contract of model «${this.model}»: ${this.problems.join("; ")}`
+  }
+}
 
 export type TypeFamily = "text" | "numeric" | "boolean" | "temporal" | "any"
 
-/** Семейство, ожидаемое от поля Effect Schema (по AST). Реюз: testModel рендерит фикстуры по нему. */
+/** Family expected from an Effect Schema field (by AST). Reuse: testModel renders fixtures from it. */
 export const familyOfAst = (ast: unknown): TypeFamily => {
   const node = ast as {
     readonly _tag: string
@@ -40,7 +44,7 @@ export const familyOfAst = (ast: unknown): TypeFamily => {
       return constructorTag.startsWith("effect/DateTime") ? "temporal" : "any"
     }
     case "Union": {
-      // NullOr(X) и подобные: единственное распознанное семейство среди членов
+      // NullOr(X) and the like: the single recognized family among the members
       const families = new Set(
         (node.types ?? [])
           .map(familyOfAst)
@@ -53,7 +57,7 @@ export const familyOfAst = (ast: unknown): TypeFamily => {
   }
 }
 
-/** Семейство фактического типа DuckDB из DESCRIBE. */
+/** Family of an actual DuckDB type from DESCRIBE. */
 const familyOfEngineType = (engineType: string): TypeFamily => {
   const base = engineType.toUpperCase()
   if (base === "VARCHAR" || base.startsWith("VARCHAR(")) return "text"
@@ -69,11 +73,11 @@ const familyOfEngineType = (engineType: string): TypeFamily => {
 }
 
 /**
- * Сверяет объявленную схему модели с фактическим результатом запроса.
- * `renderedSql` — исполнимый рендер тела (ссылки уже в физике/источниках).
- * `managed` — колонки, которые efmesh ведёт сам (scdType2: valid_from/
- * valid_to): в схеме объявлены — потребители их видят, — но запрос
- * их не возвращает.
+ * Checks the model's declared schema against the query's actual result.
+ * `renderedSql` — the executable render of the body (references already point
+ * at physical storage/sources). `managed` — columns efmesh maintains itself
+ * (scdType2: valid_from/valid_to): declared in the schema — consumers see
+ * them — but the query does not return them.
  */
 export const checkContract = (
   engine: Engine,
@@ -92,19 +96,19 @@ export const checkContract = (
     for (const [name, field] of declared) {
       const engineType = actualByName.get(name)
       if (engineType === undefined) {
-        problems.push(`колонка «${name}» объявлена в схеме, но запрос её не возвращает`)
+        problems.push(`column «${name}» is declared in the schema but the query does not return it`)
         continue
       }
       const expected = familyOfAst(field.ast)
       const got = familyOfEngineType(engineType)
       if (expected !== "any" && got !== "any" && expected !== got) {
-        problems.push(`колонка «${name}»: схема ждёт ${expected}, запрос отдаёт ${engineType}`)
+        problems.push(`column «${name}»: schema expects ${expected}, query returns ${engineType}`)
       }
     }
     const declaredNames = new Set(declared.map(([name]) => name))
     for (const column of actual) {
       if (!declaredNames.has(column.name)) {
-        problems.push(`запрос возвращает колонку «${column.name}», которой нет в схеме`)
+        problems.push(`query returns column «${column.name}» which is not in the schema`)
       }
     }
 

package/src/plan/diff.ts

@@ -1,13 +1,20 @@
-import { Effect } from "effect"
+import { Data, Effect } from "effect"
+import type { GraphError } from "../core/graph.ts"
+import { buildGraph } from "../core/graph.ts"
+import type { AnyModel } from "../core/model.ts"
+import { quoteIdent } from "../core/sql.ts"
+import { EngineAdapter } from "../engine/adapter.ts"
+import type { EngineError } from "../engine/adapter.ts"
+import { viewRef } from "./naming.ts"
 import { StateStore } from "../state/store.ts"
 import type { StateError } from "../state/store.ts"
 
-/** Чем окружения отличаются (SPEC §11: `efmesh diff <envA> <envB>`). */
+/** How two environments differ (SPEC §11: `efmesh diff <envA> <envB>`). */
 export interface EnvDiff {
-  /** Модель есть только в A. */
+  /** Model present only in A. */
   readonly onlyInA: ReadonlyArray<string>
   readonly onlyInB: ReadonlyArray<string>
-  /** Разные версии: имя + fp8 обеих сторон. */
+  /** Differing versions: name + fp8 of both sides. */
   readonly different: ReadonlyArray<{ readonly name: string; readonly a: string; readonly b: string }>
   readonly same: ReadonlyArray<string>
 }
@@ -34,3 +41,220 @@ export const diffEnvironments = (
 
     return { onlyInA, onlyInB, different, same }
   })
+
+/**
+ * DATA comparison of two environments (#6, sqlmesh's table_diff class): row
+ * counts, intersection by key, per-model column-mismatch rates — between the
+ * A and B view-layers of one database. On DuckDB-class data this is cheap;
+ * for large tables — a deterministic sample by md5 of the key: both sides are
+ * filtered by the same buckets, so the sample is aligned and key pairs are
+ * not lost.
+ */
+
+export interface ColumnDrift {
+  readonly column: string
+  /** How many matched keys diverged in this column. */
+  readonly mismatches: number
+  /** Fraction of matched, 0..1. */
+  readonly rate: number
+}
+
+export interface ModelDataDiff {
+  readonly model: string
+  /** Full row counts (not affected by sampling). */
+  readonly rowsA: number
+  readonly rowsB: number
+  /** Matching key: grain or a kind's key (uniqueKey/scdType2). */
+  readonly key?: ReadonlyArray<string>
+  /** Below — only when there is a key; under sampling, counts over selected buckets. */
+  readonly onlyInA?: number
+  readonly onlyInB?: number
+  readonly matched?: number
+  readonly columns?: ReadonlyArray<ColumnDrift>
+  /** Columns present on only one side — schema drift between environments. */
+  readonly columnsOnlyInA?: ReadonlyArray<string>
+  readonly columnsOnlyInB?: ReadonlyArray<string>
+  /** Percentage of the key's md5 buckets compared; no field — full comparison. */
+  readonly sampledPercent?: number
+}
+
+export interface DataDiffReport {
+  readonly envA: string
+  readonly envB: string
+  readonly models: ReadonlyArray<ModelDataDiff>
+}
+
+export interface DataDiffOptions {
+  /** Only these models; by default — all materializable ones from both environments. */
+  readonly models?: ReadonlyArray<string>
+  /** 1–99: compare a fraction of keys (md5 buckets, aligned across sides). */
+  readonly samplePercent?: number
+}
+
+export class DataDiffError extends Data.TaggedError("DataDiffError")<{
+  readonly model: string
+  readonly reason: string
+}> {
+  override get message(): string {
+    return `data diff of model «${this.model}»: ${this.reason}`
+  }
+}
+
+/** Kinds with a view-layer in the environment — their data is comparable. */
+const COMPARABLE_KINDS: ReadonlySet<string> = new Set([
+  "full",
+  "view",
+  "incrementalByTimeRange",
+  "incrementalByUniqueKey",
+  "scdType2",
+  "seed",
+])
+
+const keyOf = (model: AnyModel): ReadonlyArray<string> | undefined => {
+  if (model.grain !== undefined && model.grain.length > 0) return model.grain
+  if (model.kind._tag === "incrementalByUniqueKey" || model.kind._tag === "scdType2") {
+    return model.kind.key
+  }
+  return undefined
+}
+
+/**
+ * Deterministic sampling filter: md5 of the concatenated key, first two
+ * hex chars = 256 buckets. Works on both DuckDB and Postgres, identically
+ * on both sides of the diff.
+ */
+const samplePredicate = (key: ReadonlyArray<string>, percent: number): string => {
+  const buckets = Math.max(1, Math.min(255, Math.floor((256 * percent) / 100)))
+  const threshold = buckets.toString(16).padStart(2, "0")
+  return `substr(md5(concat_ws('|', ${key.map(quoteIdent).join(", ")})), 1, 2) < '${threshold}'`
+}
+
+const asCount = (value: unknown): number => Number(value ?? 0)
+
+export const dataDiffEnvironments = (
+  envA: string,
+  envB: string,
+  models: Iterable<AnyModel>,
+  options?: DataDiffOptions,
+): Effect.Effect<
+  DataDiffReport,
+  GraphError | StateError | EngineError | DataDiffError,
+  StateStore | EngineAdapter
+> =>
+  Effect.gen(function* () {
+    const store = yield* StateStore
+    const engine = yield* EngineAdapter
+    const graph = yield* buildGraph(models)
+    const inA = new Set((yield* store.getEnvironment(envA)).map((row) => row.name))
+    const inB = new Set((yield* store.getEnvironment(envB)).map((row) => row.name))
+    for (const name of options?.models ?? []) {
+      if (!graph.models.has(name)) {
+        return yield* new DataDiffError({ model: name, reason: "model is not in the project" })
+      }
+      if (!inA.has(name) || !inB.has(name)) {
+        return yield* new DataDiffError({
+          model: name,
+          reason: `model is not in environment "${inA.has(name) ? envB : envA}"`,
+        })
+      }
+    }
+    const wanted = options?.models === undefined ? undefined : new Set(options.models)
+    const percent = options?.samplePercent
+
+    const reports: Array<ModelDataDiff> = []
+    for (const name of graph.order) {
+      if (wanted !== undefined && !wanted.has(name)) continue
+      if (!inA.has(name) || !inB.has(name)) continue
+      const model = graph.models.get(name)!
+      if (!COMPARABLE_KINDS.has(model.kind._tag)) continue
+      const refA = viewRef(envA, model.name)
+      const refB = viewRef(envB, model.name)
+
+      const rowsA = asCount(
+        (yield* engine.query(`SELECT count(*) AS n FROM ${refA}`))[0]?.["n"],
+      )
+      const rowsB = asCount(
+        (yield* engine.query(`SELECT count(*) AS n FROM ${refB}`))[0]?.["n"],
+      )
+
+      // actual columns of both sides: environments may point at different
+      // model versions — only common ones are compared, schema drift is recorded
+      const colsA = (yield* engine.describe(`SELECT * FROM ${refA}`)).map((c) => c.name)
+      const colsB = (yield* engine.describe(`SELECT * FROM ${refB}`)).map((c) => c.name)
+      const setB = new Set(colsB)
+      const setA = new Set(colsA)
+      const common = colsA.filter((column) => setB.has(column))
+      const columnsOnlyInA = colsA.filter((column) => !setB.has(column))
+      const columnsOnlyInB = colsB.filter((column) => !setA.has(column))
+
+      const key = keyOf(model)
+      const commonSet = new Set(common)
+      if (key === undefined || !key.every((column) => commonSet.has(column))) {
+        // nothing to match on (no grain/key, or the key is not on both sides) —
+        // honest row counts without pairing
+        reports.push({
+          model: name,
+          rowsA,
+          rowsB,
+          ...(columnsOnlyInA.length > 0 ? { columnsOnlyInA } : {}),
+          ...(columnsOnlyInB.length > 0 ? { columnsOnlyInB } : {}),
+        })
+        continue
+      }
+
+      const keySet = new Set(key)
+      const compared = common.filter((column) => !keySet.has(column))
+      const sample = percent === undefined ? "" : ` WHERE ${samplePredicate(key, percent)}`
+      const pairs = compared
+        .map(
+          (column, index) =>
+            `, a.${quoteIdent(column)} AS a_${index}, b.${quoteIdent(column)} AS b_${index}`,
+        )
+        .join("")
+      const mismatches = compared
+        .map(
+          (_, index) =>
+            `, count(*) FILTER (WHERE in_a AND in_b AND (a_${index} IS DISTINCT FROM b_${index})) AS mm_${index}`,
+        )
+        .join("")
+      const keyList = key.map(quoteIdent).join(", ")
+      const [row] = yield* engine.query(`
+        SELECT
+          count(*) FILTER (WHERE NOT in_b) AS only_a,
+          count(*) FILTER (WHERE NOT in_a) AS only_b,
+          count(*) FILTER (WHERE in_a AND in_b) AS matched
+          ${mismatches}
+        FROM (
+          SELECT coalesce(a.in_a, FALSE) AS in_a, coalesce(b.in_b, FALSE) AS in_b${pairs}
+          FROM (SELECT TRUE AS in_a, * FROM ${refA}${sample}) a
+          FULL OUTER JOIN (SELECT TRUE AS in_b, * FROM ${refB}${sample}) b
+          USING (${keyList})
+        ) j
+      `)
+      const matched = asCount(row?.["matched"])
+      const columns = compared
+        .map((column, index) => {
+          const drifted = asCount(row?.[`mm_${index}`])
+          return {
+            column,
+            mismatches: drifted,
+            rate: matched === 0 ? 0 : drifted / matched,
+          }
+        })
+        .filter((drift) => drift.mismatches > 0)
+      reports.push({
+        model: name,
+        rowsA,
+        rowsB,
+        key,
+        onlyInA: asCount(row?.["only_a"]),
+        onlyInB: asCount(row?.["only_b"]),
+        matched,
+        columns,
+        ...(columnsOnlyInA.length > 0 ? { columnsOnlyInA } : {}),
+        ...(columnsOnlyInB.length > 0 ? { columnsOnlyInB } : {}),
+        ...(percent !== undefined ? { sampledPercent: percent } : {}),
+      })
+    }
+    return { envA, envB, models: reports }
+  }).pipe(Effect.withSpan("efmesh.diff.data", { attributes: { envA, envB } }))