@intentius/chant-lexicon-sql 0.0.0-stage → 0.112.1

package/src/core/apply.ts

@@ -0,0 +1,153 @@
+/**
+ * The applier's plumbing, the same in every dialect (lexicon-authoring/
+ * applying.mdx): reading a build's output, the outcome's tri-state (applied,
+ * not attempted, and failed for a statement the server refused), its
+ * projection onto core's apply envelope, and the ownership marker an apply
+ * stamps and prunes by.
+ *
+ * What a dialect's applier decides for itself: the statements a change takes,
+ * which changes it refuses and why, how it waits for the server, and how a
+ * prune drops an object.
+ */
+
+import type { ChantConfig } from "@intentius/chant/config";
+import type { OwnershipMarker } from "@intentius/chant/ownership";
+import {
+  applyResult,
+  notAttemptedAll,
+  type AppliedResource,
+  type ApplyRef,
+  type ApplyResult,
+  type NotAttemptedResource,
+  type PrunedResource,
+} from "@intentius/chant/apply";
+
+type Json = Record<string, unknown>;
+
+const isObject = (v: unknown): v is Json => typeof v === "object" && v !== null && !Array.isArray(v);
+
+/** One object of a build's output, as the serializer wrote it. */
+export interface BuildObject {
+  export: string;
+  type: string;
+  ddl: string;
+  /** The export names it references (a column reference by its object's export). */
+  dependsOn: string[];
+}
+
+/**
+ * The objects a `chant build` output declares for `dialect`, in creation
+ * order, each passed through `map`. Takes the sql lexicon's primary output,
+ * or a multi-lexicon output holding it under `sql`.
+ */
+export function readBuildObjects<T>(json: string, dialect: string, map: (o: BuildObject, index: number) => T): T[] {
+  const raw = JSON.parse(json) as unknown;
+  const doc = isObject(raw) && isObject(raw.sql) ? raw.sql : raw;
+  if (!isObject(doc) || doc.dialect !== dialect || !Array.isArray(doc.objects)) {
+    throw new Error(`not a sql lexicon build output: expected { dialect: "${dialect}", objects: [...] }`);
+  }
+  return (doc.objects as unknown[]).map((o, i) => {
+    if (!isObject(o) || typeof o.export !== "string" || typeof o.ddl !== "string" || typeof o.type !== "string") {
+      throw new Error(`sql build output: objects[${i}] has no export, type or ddl`);
+    }
+    const dependsOn = [...new Set((Array.isArray(o.dependsOn) ? o.dependsOn : []).filter((d): d is string => typeof d === "string").map((d) => d.split(".")[0]!))];
+    return map({ export: o.export, type: o.type, ddl: o.ddl, dependsOn }, i);
+  });
+}
+
+/** An object the server refused a statement for: attempted, and not converged. */
+export interface FailedObject extends ApplyRef {
+  error: string;
+  /** The statements that did run before the one that failed. */
+  statements: string[];
+}
+
+/** What an apply did, per object. Every declared object is in exactly one of applied, not attempted or failed. */
+export interface SqlApplyOutcome {
+  /** The server's address. */
+  target: string;
+  /** Where the binding came from, e.g. `sql.profiles.<env>`. */
+  source: string;
+  applied: Array<AppliedResource & { statements: string[] }>;
+  pruned: Array<PrunedResource & { statement: string }>;
+  notAttempted: NotAttemptedResource[];
+  failed: FailedObject[];
+}
+
+/**
+ * An apply in which the server refused a statement. The envelope has no
+ * bucket for "attempted and not converged", so the applier finishes the rest
+ * and throws this, carrying the outcome.
+ */
+export class SqlApplyError<O extends SqlApplyOutcome = SqlApplyOutcome> extends Error {
+  constructor(
+    /** The dialect as a message names it: `ClickHouse`. */
+    dialect: string,
+    readonly outcome: O,
+  ) {
+    super(
+      `${dialect} apply to ${outcome.target}: ${outcome.failed.length} object(s) failed ` +
+        `(${outcome.failed.map((f) => `${f.kind}/${f.name}: ${f.error}`).join("; ")}); ` +
+        `${outcome.applied.length} applied, ${outcome.pruned.length} pruned, ${outcome.notAttempted.length} not attempted before and after`,
+    );
+    this.name = "SqlApplyError";
+  }
+}
+
+/** An outcome in which nothing was attempted: every object not attempted with one reason. */
+export function notAttemptedOutcome(refs: readonly ApplyRef[], reason: NotAttemptedResource["reason"], detail: string): SqlApplyOutcome {
+  return { target: "", source: "", applied: [], pruned: [], notAttempted: notAttemptedAll(refs, reason, detail), failed: [] };
+}
+
+/**
+ * Project an apply outcome onto core's apply envelope. The outcome keeps what
+ * the envelope has no room for: the statements each object took and the
+ * server they went to.
+ */
+export function toApplyResult(outcome: SqlApplyOutcome): ApplyResult {
+  return applyResult(
+    outcome.applied.map((a) => ({ kind: a.kind, name: a.name, action: a.action, ...(a.physicalId ? { physicalId: a.physicalId } : {}) })),
+    outcome.pruned.map((p) => ({ kind: p.kind, name: p.name, deleted: p.deleted })),
+    outcome.notAttempted.map((n): NotAttemptedResource => ({ kind: n.kind, name: n.name, reason: n.reason, ...(n.detail ? { detail: n.detail } : {}) })),
+  );
+}
+
+/**
+ * The marker to stamp and prune by: `stack` and `ownershipEnv` when passed,
+ * else the project's `ownership` config. Undefined without a stack. `who`
+ * names the caller in the error for an env an apply cannot resolve.
+ */
+export function resolveOwnershipMarker(
+  args: { stack?: string; ownershipEnv?: string },
+  config: Pick<ChantConfig, "ownership"> | undefined,
+  who: string,
+): OwnershipMarker | undefined {
+  const o = config?.ownership;
+  const configured = o && o.enabled !== false ? o : undefined;
+  const stack = args.stack ?? configured?.stack;
+  if (!stack) return undefined;
+  const env = args.ownershipEnv ?? (typeof configured?.env === "string" ? configured.env : undefined);
+  if (env === undefined && configured?.env !== undefined) {
+    throw new Error(`${who}: ownership.env is a build parameter reference, which an apply cannot resolve; pass ownershipEnv`);
+  }
+  return { stack, ...(env ? { env } : {}) };
+}
+
+/** Changes grouped by the object they change, in order. */
+export function changesByObject<C extends { object: string }>(changes: readonly C[]): Map<string, C[]> {
+  const byObject = new Map<string, C[]>();
+  for (const c of changes) byObject.set(c.object, [...(byObject.get(c.object) ?? []), c]);
+  return byObject;
+}
+
+/**
+ * The declared objects `dependsOn` names that are not on the server: declared
+ * in this build, and not created or found in this apply. An object with any
+ * is not attempted, `dependency-failed`, with {@link dependencyFailedDetail}.
+ */
+export function missingDependencies(self: string, dependsOn: readonly string[], declared: ReadonlySet<string>, onServer: ReadonlySet<string>): string[] {
+  return dependsOn.filter((d) => declared.has(d) && d !== self && !onServer.has(d));
+}
+
+export const dependencyFailedDetail = (missing: readonly string[]): string =>
+  `references ${missing.join(", ")}, which is not on the server: it failed or was not attempted in this apply`;

package/src/core/classifier.ts

@@ -0,0 +1,186 @@
+/**
+ * The change classifier's framework, the same in every dialect: the shape of
+ * a rule, a classified change, `classifyDisruption()` over a dialect's rules,
+ * and the text report.
+ *
+ * The classes are data, one set per dialect. ClickHouse classes a change by
+ * whether `ALTER` can make it (metadata, background rewrite, rebuild); another
+ * dialect classes by the lock a change takes and whether it rewrites the
+ * table. Each dialect declares its classes with {@link ChangeClasses} and its
+ * rules with {@link classifierRule}, every rule citing the database's own
+ * documentation for the restriction behind it.
+ */
+
+import type { DisruptionQuery, DisruptionVerdict, Disruption } from "@intentius/chant/lifecycle/disruption";
+
+/** One way a declared object can change, the class of change it is on the server, and why. */
+export interface ClassifierRule<C extends string = string> {
+  id: string;
+  class: C;
+  title: string;
+  /** The restriction, in a sentence. */
+  restriction: string;
+  /** The documentation page the restriction is stated on. */
+  cite: string;
+}
+
+export const classifierRule = <C extends string>(id: string, cls: C, title: string, restriction: string, cite: string): ClassifierRule<C> => ({
+  id,
+  class: cls,
+  title,
+  restriction,
+  cite,
+});
+
+/** A dialect's change classes as data: their order in a report, their labels, and what each costs to apply. */
+export interface ChangeClasses<C extends string> {
+  /** The order a report counts the classes in. */
+  order: readonly C[];
+  /** Each class as a report names it. */
+  label: Readonly<Record<C, string>>;
+  /** Each class as `classifyDisruption()` reports it. */
+  disruption: Readonly<Record<C, Disruption>>;
+}
+
+/** One classified change to one object. */
+export interface ClassifiedChange<R extends string = string, C extends string = string> {
+  /** The object's identity. */
+  object: string;
+  /** What changed, e.g. `columns.kind.type`, `orderBy`, `name`. */
+  field: string;
+  before?: string;
+  after?: string;
+  rule: R;
+  class: C;
+  /** Data is removed and not recoverable. */
+  destructive?: boolean;
+  /** What a reader should know besides the rule. */
+  note?: string;
+}
+
+/** A classified change from `before` to `after`, its class read from `rules`. */
+export function classifiedChange<R extends string, C extends string>(
+  rules: Readonly<Record<R, ClassifierRule<C>>>,
+  object: string,
+  field: string,
+  rule: R,
+  before?: unknown,
+  after?: unknown,
+  extra: Partial<ClassifiedChange<R, C>> = {},
+): ClassifiedChange<R, C> {
+  return {
+    object,
+    field,
+    ...(before !== undefined ? { before: String(before) } : {}),
+    ...(after !== undefined ? { after: String(after) } : {}),
+    rule,
+    class: rules[rule].class,
+    ...extra,
+  };
+}
+
+/** The changes between two schemas, and hints for the author. */
+export interface ChangeSet<Ch extends ClassifiedChange = ClassifiedChange> {
+  changes: Ch[];
+  /** Hints for the author: a drop and an add that look like a rename. */
+  hints: string[];
+}
+
+// ── classifyDisruption() ────────────────────────────────────────────────
+
+const RANK: Record<Disruption, number> = { "in-place": 0, rolling: 1, replace: 2, destroy: 3, unknown: 4 };
+
+export interface DisruptionClassifier<R extends string, C extends string> {
+  /** The entity type prefix the dialect's objects carry (`ClickHouse::`); other types are left out. */
+  typePrefix: string;
+  rules: Readonly<Record<R, ClassifierRule<C>>>;
+  classes: Pick<ChangeClasses<C>, "disruption">;
+  /** The rule a changed path falls under, `ambiguous` when the path alone cannot say, undefined when no rule covers it. */
+  ruleFor(path: string): R | "ambiguous" | undefined;
+  /** The detail of an `unknown` verdict: why the paths cannot decide, and what does. */
+  ambiguousDetail: string;
+}
+
+/**
+ * `classifyDisruption()` over a dialect's rules: each update's disruption is
+ * the worst class among its changed paths, and `unknown` when any path cannot
+ * be classified from the path alone.
+ */
+export function classifyDisruptionWith<R extends string, C extends string>(
+  classifier: DisruptionClassifier<R, C>,
+  options: { environment: string; changes: DisruptionQuery[] },
+): Record<string, DisruptionVerdict> {
+  const out: Record<string, DisruptionVerdict> = {};
+  for (const q of options.changes) {
+    if (q.type !== undefined && !q.type.startsWith(classifier.typePrefix)) continue;
+    let level: Disruption = "in-place";
+    const because: string[] = [];
+    const rules = new Set<string>();
+    let ambiguous = false;
+    for (const d of q.deltas) {
+      const r = classifier.ruleFor(d.path);
+      if (r === undefined || r === "ambiguous") {
+        ambiguous = true;
+        because.push(d.path);
+        continue;
+      }
+      const l = classifier.classes.disruption[classifier.rules[r].class]!;
+      if (RANK[l] > RANK[level]) {
+        level = l;
+        because.length = 0;
+      }
+      if (RANK[l] === RANK[level]) because.push(d.path);
+      rules.add(r);
+    }
+    if (ambiguous) {
+      out[q.name] = { disruption: "unknown", because, detail: classifier.ambiguousDetail };
+      continue;
+    }
+    out[q.name] = {
+      disruption: level,
+      because,
+      detail: [...rules].map((r) => `${r} ${classifier.rules[r as R].title}`).join("; "),
+    };
+  }
+  return out;
+}
+
+// ── The report ──────────────────────────────────────────────────────────
+
+/**
+ * A change set as text, for a terminal or a pull request: each object's
+ * changes with their class, rule, restriction and citation, then a count per
+ * class and the hints. `trailer` is the dialect's closing lines (what a plan
+ * refuses and what to run instead); it is left out when nothing changed.
+ */
+export function renderChangeSet<R extends string, C extends string>(
+  diff: ChangeSet<ClassifiedChange<R, C>>,
+  opts: { title?: string; rules: Readonly<Record<R, ClassifierRule<C>>>; classes: Pick<ChangeClasses<C>, "order" | "label">; trailer?: readonly string[] },
+): string {
+  const lines: string[] = [];
+  if (opts.title) lines.push(opts.title, "");
+  if (diff.changes.length === 0) {
+    lines.push("No changes.");
+    return lines.join("\n");
+  }
+  const byObject = new Map<string, typeof diff.changes>();
+  for (const c of diff.changes) byObject.set(c.object, [...(byObject.get(c.object) ?? []), c]);
+  for (const [object, changes] of byObject) {
+    lines.push(object);
+    for (const c of changes) {
+      const rule = opts.rules[c.rule];
+      const values = c.before !== undefined && c.after !== undefined ? `: ${c.before} -> ${c.after}` : c.after !== undefined ? `: ${c.after}` : c.before !== undefined ? `: ${c.before}` : "";
+      lines.push(`  [${opts.classes.label[c.class]}] ${c.field}${values}`);
+      lines.push(`      ${c.rule} ${rule.title}${c.destructive ? " (destroys data)" : ""}. ${rule.restriction} ${rule.cite}`);
+      if (c.note) lines.push(`      ${c.note}`);
+    }
+  }
+  const counts = opts.classes.order
+    .map((k) => [k, diff.changes.filter((c) => c.class === k).length] as const)
+    .filter(([, n]) => n > 0)
+    .map(([k, n]) => `${n} ${opts.classes.label[k]}`);
+  lines.push("", counts.join(", "));
+  for (const h of diff.hints) lines.push(`hint: ${h}`);
+  if (opts.trailer) lines.push(...opts.trailer);
+  return lines.join("\n");
+}

package/src/core/container.ts

@@ -0,0 +1,93 @@
+/**
+ * A throwaway database server in a container, for every dialect: started on a
+ * random loopback port under a unique name, waited on until it answers, and
+ * always removed.
+ *
+ * Generation uses it on a pin move to read the pinned server's catalog, and
+ * the live tests use it so every container chant starts is started and
+ * cleaned up one way. What "answers" means (an HTTP ping, a query) and the
+ * image's environment are the dialect's.
+ */
+
+import { execFile } from "node:child_process";
+import { promisify } from "node:util";
+import { randomBytes } from "node:crypto";
+
+const exec = promisify(execFile);
+
+/** True when a Docker daemon answers. Tests that need a server skip when this is false. */
+export async function dockerAvailable(): Promise<boolean> {
+  try {
+    await exec("docker", ["info", "--format", "{{.ServerVersion}}"], { timeout: 15_000 });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+/** A container name no other run picks: `<prefix>-<pid>-<random>`. */
+export function uniqueContainerName(prefix: string): string {
+  return `${prefix}-${process.pid}-${randomBytes(4).toString("hex")}`;
+}
+
+/** The loopback host port Docker published for `containerPort` on `name`. */
+export async function publishedPort(name: string, containerPort: number): Promise<string> {
+  const { stdout } = await exec("docker", ["port", name, `${containerPort}/tcp`]);
+  const port = stdout.split("\n")[0]?.trim().split(":").pop();
+  if (!port) throw new Error(`docker port reported no host port for ${name}`);
+  return port;
+}
+
+export interface ScratchContainer {
+  /** The container's name, for a log line. */
+  name: string;
+  /** The loopback host port `containerPort` is published on. */
+  port: string;
+  /** Remove the container. Safe to call twice. */
+  stop(): Promise<void>;
+}
+
+/**
+ * Run `image` detached, publish `containerPort` on a random loopback port,
+ * and wait until `ready(port)` is true, polling every half second. Pulls the
+ * image when it is not present, which is the one network step here. On any
+ * failure the container is removed before the error is thrown.
+ */
+export async function startScratchContainer(options: {
+  image: string;
+  namePrefix: string;
+  containerPort: number;
+  env?: Readonly<Record<string, string>>;
+  ready: (port: string) => Promise<boolean>;
+  readyTimeoutMs: number;
+  /** What the container did not do in time, for the error: `did not answer /ping`. */
+  notReady: string;
+  /** Called with the container's name once it is running, before the readiness wait. */
+  onStarted?: (name: string) => void;
+  /** Remove the container's anonymous volumes with it (an image that declares a `VOLUME`, as `postgres` does). */
+  removeVolumes?: boolean;
+}): Promise<ScratchContainer> {
+  const name = uniqueContainerName(options.namePrefix);
+  let removed = false;
+  const stop = async (): Promise<void> => {
+    if (removed) return;
+    removed = true;
+    await exec("docker", ["rm", "-f", ...(options.removeVolumes ? ["-v"] : []), name]).catch(() => undefined);
+  };
+
+  try {
+    const env = Object.entries(options.env ?? {}).flatMap(([k, v]) => ["-e", `${k}=${v}`]);
+    await exec("docker", ["run", "-d", "--name", name, "-p", `127.0.0.1::${options.containerPort}`, ...env, options.image], { timeout: 600_000 });
+    const port = await publishedPort(name, options.containerPort);
+    options.onStarted?.(name);
+    const deadline = Date.now() + options.readyTimeoutMs;
+    while (!(await options.ready(port))) {
+      if (Date.now() > deadline) throw new Error(`${name} ${options.notReady} within the timeout`);
+      await new Promise((r) => setTimeout(r, 500));
+    }
+    return { name, port, stop };
+  } catch (err) {
+    await stop();
+    throw err;
+  }
+}