@schemic/surrealdb 0.1.0-alpha.0

package/package.json

@@ -0,0 +1,68 @@
+{
+  "name": "@schemic/surrealdb",
+  "version": "0.1.0-alpha.0",
+  "description": "SurrealDB driver + authoring for Schemic — s.* schema definition, SurrealQL DDL, introspection, migrations.",
+  "license": "MIT",
+  "author": "Vertio Solutions",
+  "homepage": "https://surreal.schemic.dev",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/schemichq/schemic.git",
+    "directory": "packages/surrealdb"
+  },
+  "type": "module",
+  "sideEffects": [
+    "./src/index.ts",
+    "./src/driver/surreal.ts",
+    "./lib/index.js"
+  ],
+  "module": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "publishConfig": {
+    "access": "public"
+  },
+  "files": [
+    "lib",
+    "src",
+    "LICENSE"
+  ],
+  "exports": {
+    ".": {
+      "bun": "./src/index.ts",
+      "import": {
+        "types": "./lib/index.d.ts",
+        "default": "./lib/index.js"
+      }
+    },
+    "./package.json": "./package.json"
+  },
+  "scripts": {
+    "build": "tsup",
+    "prepack": "bun run build",
+    "test": "bun test",
+    "typecheck": "tsc --noEmit",
+    "lint": "biome check ."
+  },
+  "dependencies": {
+    "@schemic/core": "0.1.0-alpha.0",
+    "surrealdb": "^2.0.3"
+  },
+  "peerDependencies": {
+    "@surrealdb/node": "*",
+    "zod": "^4.3.5"
+  },
+  "peerDependenciesMeta": {
+    "@surrealdb/node": {
+      "optional": true
+    }
+  },
+  "devDependencies": {
+    "@biomejs/biome": "^2.3.11",
+    "@schemic/cli": "0.1.0-alpha.0",
+    "@types/bun": "latest",
+    "surrealdb": "^2.0.3",
+    "tsup": "^8",
+    "typescript": "^5",
+    "zod": "^4.3.5"
+  }
+}

package/src/cli/engine.ts

@@ -0,0 +1,189 @@
+import { type ChildProcess, execFileSync, spawn } from "node:child_process";
+import { createServer } from "node:net";
+import { escapeIdent, Surreal } from "surrealdb";
+import type { SurrealZodCheckEmbedded } from "../config";
+
+/** Pick a free localhost TCP port by binding to :0 and reading it back. */
+function freePort(): Promise<number> {
+  return new Promise((resolve, reject) => {
+    const srv = createServer();
+    srv.once("error", reject);
+    srv.listen(0, "127.0.0.1", () => {
+      const addr = srv.address();
+      const port = addr && typeof addr === "object" ? addr.port : 0;
+      srv.close(() => (port ? resolve(port) : reject(new Error("no port"))));
+    });
+  });
+}
+
+/** Whether the local `surreal` CLI is runnable (used to pick the `auto` check engine). */
+export function surrealBinaryAvailable(bin = "surreal"): boolean {
+  try {
+    execFileSync(bin, ["version"], { stdio: "ignore" });
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+export interface EphemeralServer {
+  url: string;
+  username: string;
+  password: string;
+  /** Stop the spawned process (idempotent). */
+  stop: () => Promise<void>;
+}
+
+const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
+
+/** Poll a Surreal connection until it succeeds, the process dies, or we time out. */
+async function waitUntilReady(
+  url: string,
+  username: string,
+  password: string,
+  child: ChildProcess,
+  stderr: () => string,
+): Promise<void> {
+  const deadline = Date.now() + 15_000;
+  while (Date.now() < deadline) {
+    if (child.exitCode !== null) {
+      throw new Error(
+        `surreal exited (${child.exitCode}): ${stderr().split("\n").filter(Boolean).pop() ?? "no output"}`,
+      );
+    }
+    const db = new Surreal();
+    try {
+      await db.connect(url, { reconnect: false });
+      await db.signin({ username, password });
+      await db.close();
+      return;
+    } catch {
+      await db.close().catch(() => {});
+      await sleep(100);
+    }
+  }
+  throw new Error("surreal did not become ready within 15s");
+}
+
+/**
+ * Spawn an ephemeral in-memory SurrealDB using the local `surreal` binary — for `schemic check`'s
+ * migration replay. It runs on the user's EXACT SurrealDB version, needs no external server, and
+ * never touches their data. Capabilities are fully allowed (`--allow-all`): it's a throwaway
+ * instance running the user's own schema, so asserts/defaults/scripted functions all work. The
+ * caller must `stop()` it (a `process.exit` hook is also registered as a backstop).
+ */
+export async function spawnEphemeralServer(
+  bin = "surreal",
+): Promise<EphemeralServer> {
+  const port = await freePort();
+  const username = "root";
+  const password = "root";
+  const child = spawn(
+    bin,
+    [
+      "start",
+      "--bind",
+      `127.0.0.1:${port}`,
+      "--username",
+      username,
+      "--password",
+      password,
+      "--allow-all",
+      "memory",
+    ],
+    { stdio: ["ignore", "ignore", "pipe"] },
+  );
+  let stderr = "";
+  child.stderr?.on("data", (d) => {
+    stderr += String(d);
+  });
+
+  let stopped = false;
+  const stop = (): Promise<void> =>
+    new Promise((resolve) => {
+      if (stopped || child.exitCode !== null) {
+        stopped = true;
+        return resolve();
+      }
+      stopped = true;
+      child.once("exit", () => resolve());
+      child.kill("SIGTERM");
+      // Hard-kill if it doesn't exit promptly.
+      setTimeout(() => child.kill("SIGKILL"), 2000).unref?.();
+    });
+  // Backstop: don't leak the process if the CLI exits abnormally.
+  const onExit = () => {
+    if (!stopped) child.kill("SIGKILL");
+  };
+  process.once("exit", onExit);
+
+  const url = `ws://127.0.0.1:${port}/rpc`;
+  try {
+    await waitUntilReady(url, username, password, child, () => stderr);
+  } catch (e) {
+    await stop();
+    throw e;
+  }
+  return { url, username, password, stop };
+}
+
+export interface EmbeddedConnection {
+  db: Surreal;
+  /** A display label for the engine, e.g. `embedded memory`. */
+  url: string;
+  stop: () => Promise<void>;
+}
+
+/**
+ * Connect to an EMBEDDED in-process SurrealDB via the optional `@surrealdb/node` package, selecting
+ * the `cfg.backend` storage and passing `cfg` (capabilities, timeouts, …) straight to
+ * `createNodeEngines`. Creates/selects the given namespace/database so the replay can run. The
+ * package is imported lazily (a non-literal specifier) so it's never required unless this engine is
+ * used; a clear error tells the user to install it otherwise.
+ */
+export async function connectEmbedded(
+  cfg: SurrealZodCheckEmbedded,
+  namespace: string,
+  database: string,
+): Promise<EmbeddedConnection> {
+  // Non-literal specifier so tsc/bundlers don't treat `@surrealdb/node` as a hard dependency.
+  const pkg: string = "@surrealdb/node";
+  let createNodeEngines: ((options?: unknown) => unknown) | undefined;
+  try {
+    const mod = (await import(pkg)) as {
+      createNodeEngines?: (options?: unknown) => unknown;
+    };
+    createNodeEngines = mod.createNodeEngines;
+  } catch {
+    createNodeEngines = undefined;
+  }
+  if (!createNodeEngines) {
+    throw new Error(
+      'check.engine embedded mode needs `@surrealdb/node` — install it (e.g. `npm i -D @surrealdb/node`), or use a string engine ("auto" / "binary" / "remote").',
+    );
+  }
+
+  const backend = cfg.backend ?? "memory";
+  const scheme = backend === "memory" ? "mem" : backend;
+  const url = `${scheme}://${cfg.path ?? ""}`;
+  const engines = createNodeEngines({
+    capabilities: cfg.capabilities ?? true,
+    strict: cfg.strict,
+    query_timeout: cfg.query_timeout,
+    transaction_timeout: cfg.transaction_timeout,
+  });
+  // biome-ignore lint/suspicious/noExplicitAny: the SDK's `engines` option is loosely typed here.
+  const db = new Surreal({ engines } as any);
+  await db.connect(url);
+  await db.query(`DEFINE NAMESPACE IF NOT EXISTS ${escapeIdent(namespace)}`);
+  await db.use({ namespace });
+  await db.query(`DEFINE DATABASE IF NOT EXISTS ${escapeIdent(database)}`);
+  await db.use({ namespace, database });
+  return {
+    db,
+    url: `embedded ${backend}`,
+    stop: async () => {
+      await db.close().catch(() => {});
+    },
+  };
+}

package/src/cli/introspect.ts

@@ -0,0 +1,275 @@
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import type { Diff, ResolvedConfig } from "@schemic/core";
+import {
+  type Filter,
+  listMigrations,
+  loadDefs,
+  parseFilter,
+} from "@schemic/core";
+import { escapeIdent, type Surreal } from "surrealdb";
+import type { SurrealParams } from "../config";
+import { type DefineStatement, overwriteStatement } from "../ddl";
+import { normalizeDb } from "./struct";
+import {
+  type DbStructured,
+  introspectStructured,
+  type Snapshot,
+  structuredSnapshot,
+} from "./structure";
+import { buildSnapshot, diffSnapshots } from "./surreal-diff";
+import { filterSnapshot, filterStructured } from "./surreal-filter";
+
+const SHADOW_DB = "__surreal_zod_shadow";
+const SHADOW_MIG_DB = "__surreal_zod_shadow_mig";
+
+// Apply order: tables, then fields, then indexes.
+const RANK: Record<DefineStatement["kind"], number> = {
+  analyzer: 0, // db-level; defined first (a FULLTEXT index references its analyzer)
+  function: 0, // db-level; defined first (tables/events may reference fn::…)
+  table: 1,
+  field: 2,
+  index: 3,
+  event: 4,
+  access: 5, // db-level; defined last (SIGNUP/SIGNIN reference tables)
+};
+const byCreate = (a: DefineStatement, b: DefineStatement) =>
+  RANK[a.kind] - RANK[b.kind];
+
+/**
+ * Read the live database into a snapshot of **canonical** DDL, skipping `exclude`d tables. Uses
+ * `INFO … STRUCTURE` and rebuilds each object's DDL deterministically (see `structuredSnapshot`),
+ * so equivalent schemas compare equal regardless of SurrealDB's formatting (e.g. union order).
+ */
+export async function introspect(
+  db: Surreal,
+  exclude: Set<string> = new Set(),
+): Promise<Snapshot> {
+  return structuredSnapshot(await introspectStructured(db, exclude));
+}
+
+/**
+ * Apply `ddl` to a fresh scratch database, introspect it back to a canonical snapshot, then drop it
+ * (restoring the original namespace/database). Normalizes a desired schema THROUGH SurrealDB so the
+ * comparison is free of formatting noise. Used by both `diff --live` and `verify`.
+ */
+async function applyToShadow(
+  db: Surreal,
+  config: ResolvedConfig,
+  shadowDb: string,
+  ddl: string,
+): Promise<Snapshot> {
+  const { namespace, database } = config.params as unknown as SurrealParams;
+  await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(shadowDb)};`);
+  await db.query(`DEFINE DATABASE ${escapeIdent(shadowDb)};`);
+  try {
+    await db.use({ namespace, database: shadowDb });
+    if (ddl) await db.query(`BEGIN;\n${ddl}\nCOMMIT;`);
+    return await introspect(db);
+  } finally {
+    await db.use({ namespace, database });
+    await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(shadowDb)};`);
+  }
+}
+
+/**
+ * Apply `ddl` to a fresh scratch database and return its STRUCTURED introspection (the Struct-IR
+ * form, not the canonical-DDL snapshot), then drop it. Backs `diff --ts`'s desired (schema) side —
+ * normalizing the schema THROUGH SurrealDB so it lands in the same form INFO returns for the live DB.
+ */
+export async function shadowStructured(
+  db: Surreal,
+  config: ResolvedConfig,
+  ddl: string,
+): Promise<DbStructured> {
+  const { namespace, database } = config.params as unknown as SurrealParams;
+  await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(SHADOW_DB)};`);
+  await db.query(`DEFINE DATABASE ${escapeIdent(SHADOW_DB)};`);
+  try {
+    await db.use({ namespace, database: SHADOW_DB });
+    if (ddl) await db.query(`BEGIN;\n${ddl}\nCOMMIT;`);
+    return await introspectStructured(db);
+  } finally {
+    await db.use({ namespace, database });
+    await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(SHADOW_DB)};`);
+  }
+}
+
+/**
+ * The two sides of `diff --ts --live` as normalized Struct-IR: the live database (`current`) and
+ * the declared schema (`desired`, normalized THROUGH SurrealDB via a shadow apply). Both go through
+ * the same normalize, so an unchanged schema yields deep-equal structs. The caller renders them.
+ */
+export async function tsStructsAgainstDb(
+  db: Surreal,
+  config: ResolvedConfig,
+  filter: Filter = parseFilter({}),
+): Promise<{ current: DbStructured; desired: DbStructured }> {
+  const exclude = new Set([
+    config.migrationsTable,
+    `${config.migrationsTable}_lock`,
+  ]);
+  const target = await introspectStructured(db, exclude);
+  const { tables, defs } = await loadDefs(config.schemaPath);
+  const ddl = Object.values(buildSnapshot(tables, defs).statements)
+    .sort(byCreate)
+    .map((s) => s.ddl)
+    .join("\n");
+  const desired = await shadowStructured(db, config, ddl);
+  const norm = (d: DbStructured) => normalizeDb(filterStructured(d, filter));
+  return { current: norm(target), desired: norm(desired) };
+}
+
+/**
+ * Replay every migration (direction `up`) from zero into a fresh scratch database, introspect the
+ * result, then drop it. Surfaces the offending migration if one fails to apply.
+ */
+async function replayMigrations(
+  db: Surreal,
+  config: ResolvedConfig,
+  shadowDb: string,
+  onApply?: (tag: string) => void,
+): Promise<Snapshot> {
+  const { namespace, database } = config.params as unknown as SurrealParams;
+  await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(shadowDb)};`);
+  await db.query(`DEFINE DATABASE ${escapeIdent(shadowDb)};`);
+  try {
+    await db.use({ namespace, database: shadowDb });
+    for (const m of listMigrations(config.migrationsDir)) {
+      onApply?.(m.tag);
+      const sql = readFileSync(join(config.migrationsDir, m.file), "utf8");
+      try {
+        await db.query(sql, { direction: "up" });
+      } catch (e) {
+        throw new Error(
+          `migration ${m.tag} failed to replay: ${(e as Error).message.split("\n")[0]}`,
+        );
+      }
+    }
+    return await introspect(db);
+  } finally {
+    await db.use({ namespace, database });
+    await db.query(`REMOVE DATABASE IF EXISTS ${escapeIdent(shadowDb)};`);
+  }
+}
+
+/**
+ * Verify that replaying every migration from zero reconstructs the declared schema — the only check
+ * that catches "the sum of the migrations no longer equals the schema" (a hand-edited migration, or
+ * a schema change someone forgot to `schemic gen`). Replays the migrations into one scratch DB and applies
+ * the current schema into another, then diffs the two introspected snapshots; since BOTH sides are
+ * normalized through SurrealDB (`INFO`), only genuine drift shows up. An empty diff means they agree.
+ * `up` is what the migrations are missing relative to the schema. Needs root/namespace auth.
+ */
+export async function verifyMigrations(
+  db: Surreal,
+  config: ResolvedConfig,
+  filter: Filter = parseFilter({}),
+  onApply?: (tag: string) => void,
+): Promise<Diff> {
+  const migrated = await replayMigrations(db, config, SHADOW_MIG_DB, onApply);
+  const { tables, defs } = await loadDefs(config.schemaPath);
+  const ddl = Object.values(buildSnapshot(tables, defs).statements)
+    .sort(byCreate)
+    .map((s) => s.ddl)
+    .join("\n");
+  const desired = await applyToShadow(db, config, SHADOW_DB, ddl);
+  return diffSnapshots(
+    filterSnapshot(migrated, filter),
+    filterSnapshot(desired, filter),
+  );
+}
+
+/**
+ * Diff the current schemas against the **live database**. Both sides are normalized through
+ * SurrealDB — the target via `INFO`, the desired by applying the schema to a temporary shadow
+ * database and reading IT back — so the comparison is free of formatting noise. The shadow
+ * database is created and dropped in the target's namespace (needs root/namespace auth).
+ */
+export async function diffAgainstDb(
+  db: Surreal,
+  config: ResolvedConfig,
+  filter: Filter = parseFilter({}),
+): Promise<Diff> {
+  // Exclude the CLI's own bookkeeping tables — they're not part of the schema (pull excludes
+  // them too); otherwise `diff --live`/`sync` always report dropping `_migrations_lock`.
+  const target = await introspect(
+    db,
+    new Set([config.migrationsTable, `${config.migrationsTable}_lock`]),
+  );
+
+  const { tables, defs } = await loadDefs(config.schemaPath);
+  const schema = buildSnapshot(tables, defs);
+  const ddl = Object.values(schema.statements)
+    .sort(byCreate)
+    .map((s) => s.ddl)
+    .join("\n");
+  const desired = await applyToShadow(db, config, SHADOW_DB, ddl);
+
+  // up = statements that would bring the live database in line with the schema. The filter drops
+  // both sides' excluded kinds (access is off by default) so they're neither applied nor pruned.
+  const diff = diffSnapshots(
+    filterSnapshot(target, filter),
+    filterSnapshot(desired, filter),
+  );
+
+  // Access is COMPARED via its canonical form (the signing key is redacted identically on both
+  // sides, so no false diff), but that form can't be APPLIED — the redacted KEY is gone. Swap in
+  // the schema's emit DDL (which carries the key) for any DEFINE ACCESS in the apply plan.
+  const accessEmit = new Map<string, string>();
+  for (const s of Object.values(schema.statements))
+    if (s.kind === "access") accessEmit.set(s.name, s.ddl);
+  if (accessEmit.size) {
+    const swap = (stmt: string): string => {
+      const m = /^DEFINE ACCESS (OVERWRITE )?(\S+)/.exec(stmt);
+      const emit = m && accessEmit.get(m[2]);
+      return emit ? (m[1] ? overwriteStatement(emit) : emit) : stmt;
+    };
+    diff.up = diff.up.map(swap);
+  }
+
+  // Implicit-wildcard fields (the `.*` element of an `array<object>`/`set<object>`, etc.) are
+  // auto-created when their parent field is defined, so the emitter marks them `DEFINE FIELD
+  // OVERWRITE`. The INFO-canonical diff form drops that OVERWRITE, so applying it to the live DB
+  // fails "already exists" and aborts the whole `push`/`sync` transaction. Re-mark exactly those
+  // fields OVERWRITE in the apply plan — the comparison form above is untouched.
+  const untick = (s: string) => s.replace(/`/g, "");
+  const overwriteFields = new Set<string>();
+  for (const s of Object.values(schema.statements))
+    if (
+      s.kind === "field" &&
+      s.table &&
+      /^DEFINE FIELD OVERWRITE\b/.test(s.ddl)
+    )
+      overwriteFields.add(`${s.table}${untick(s.name)}`);
+  if (overwriteFields.size) {
+    const fieldRef =
+      /^DEFINE FIELD (?:OVERWRITE |IF NOT EXISTS )?(`[^`]+`|\S+) ON TABLE (`[^`]+`|\S+)/;
+    diff.up = diff.up.map((stmt) => {
+      const m = fieldRef.exec(stmt);
+      return m && overwriteFields.has(`${untick(m[2])}${untick(m[1])}`)
+        ? overwriteStatement(stmt)
+        : stmt;
+    });
+  }
+  return diff;
+}
+
+/**
+ * The statements that would reconcile the live database with the schema (the `diff --live`
+ * forward changes). With `prune: false`, drops (`REMOVE`) are excluded so existing extras
+ * are kept. Run through `applyStatements` to apply.
+ */
+export function syncPlan(diff: Diff, prune?: boolean): string[] {
+  return prune === false
+    ? diff.up.filter((s) => !s.startsWith("REMOVE"))
+    : diff.up;
+}
+
+/** Apply a set of statements to the database in a single transaction. */
+export async function applyStatements(
+  db: Surreal,
+  stmts: string[],
+): Promise<void> {
+  if (stmts.length) await db.query(`BEGIN;\n${stmts.join("\n")}\nCOMMIT;`);
+}