dzql 0.7.0 → 0.9.0

package/src/postgres/schema.ts

@@ -0,0 +1,171 @@
+/**
+ * Delta Postgres — schema wrappers + Postgres-specific utilities.
+ *
+ * The shared schema vocabulary (ColumnDef, TableDef, Schema, ResolvedTable,
+ * DocDef, defineSchema, defineDoc) lives in `../../schema` and is identical
+ * across the Postgres and SQLite backends. This module re-exports those
+ * types and adds the Postgres helpers that hit the live database
+ * (time-travel, ops-log pruning, op validation).
+ */
+import type { Pool } from "pg";
+import { type DeltaOp, joinPath, splitPath } from "@blueshed/delta/core";
+import {
+  holdsField,
+  type Schema,
+  type DocDef,
+  type ValidationError,
+  isIsoTime,
+} from "@blueshed/delta/backend";
+import { pastSafeId, resolveScope } from "@blueshed/delta/backend";
+
+export type {
+  ColumnType,
+  ColumnDef,
+  ColumnShorthand,
+  TableDef,
+  Schema,
+  ResolvedTable,
+  DocDef,
+  ValidationError,
+} from "@blueshed/delta/backend";
+export { defineSchema, defineDoc } from "@blueshed/delta/backend";
+
+// ---------------------------------------------------------------------------
+// Postgres-specific doc helpers — all thin wrappers over stored functions.
+// ---------------------------------------------------------------------------
+
+export async function loadDocAt(pool: Pool, docName: string, at: string | Date): Promise<any | null> {
+  const ts = at instanceof Date ? at.toISOString() : at;
+  const { rows } = await pool.query("SELECT delta_open_at($1, $2) AS doc", [docName, ts]);
+  return rows[0]?.doc ?? null;
+}
+
+export async function pruneOpsLog(pool: Pool, keepInterval = "1 hour"): Promise<number> {
+  const { rows } = await pool.query("SELECT delta_prune_ops($1::interval) AS count", [keepInterval]);
+  return Number(rows[0]?.count ?? 0);
+}
+
+// ---------------------------------------------------------------------------
+// validateOps — what delta_apply refuses as a mistake (400), said ahead.
+// ---------------------------------------------------------------------------
+
+/**
+ * Validate delta ops against the schema, as `delta_apply` (001d) reads them,
+ * before they reach the database: it refuses each write this does (a 400), and
+ * takes each this takes -- but for what only the rows can say (a row not there,
+ * 404; already there, 409) and a value of its type its column cannot hold (a
+ * timestamp that is no date), which are the database's. Returns an array of errors (empty = valid).
+ *
+ * `doc`: the document's name, which says its mode and its scope's values, as
+ * `delta_apply` reads them (`_delta_resolve_scope`; `resolveScope` is its
+ * twin) -- the definition alone cannot: `all-courses:` is every course, and
+ * `all-courses:1` course 1 (todo #57). Or say them: `list`, the document is in
+ * list mode; `values`, its scope's equality bindings, which a list-mode add of
+ * a root row is given, so they count as given -- the options SQLite's
+ * validateOps takes, and they win over the name's. Given neither, the mode is
+ * not known, and each shape is taken as the mode that has it takes it:
+ * `replace /<root>` and `replace /<root>/<field>` as a single document's root
+ * row, `add /<root>/-` as a list's new row.
+ */
+export function validateOps(schema: Schema, def: DocDef, ops: DeltaOp[], opts: { doc?: string; list?: boolean; values?: Record<string, string> } = {}): ValidationError[] {
+  let rootId: string | undefined;   // a single document's own root, where its name says it
+  if (opts.doc !== undefined) {
+    if (!opts.doc.startsWith(def.prefix)) throw new Error(`validateOps: ${opts.doc} is not a document of ${def.prefix}`);
+    const scope = resolveScope(def, opts.doc.slice(def.prefix.length));
+    opts = { list: opts.list ?? scope.mode === "list", values: opts.values ?? scope.values };
+    rootId = scope.id;
+  }
+  const errors: ValidationError[] = [];
+  for (const op of ops) {
+    const fail = (message: string, path = op.path) => errors.push({ path: String(path), message });
+    let parts: string[];
+    try { parts = splitPath(op.path); }
+    catch (err: any) { fail(err.message); continue; }
+    const collKey = parts[0];
+    if (!collKey) { fail("Empty path"); continue; }
+    if (collKey !== def.root && !def.include.includes(collKey)) { fail(`Unknown collection: ${collKey}`); continue; }
+    const table = schema.tables[collKey];
+    if (!table) { fail(`No table for collection: ${collKey}`); continue; }
+
+    const fk = table.parent?.fkColumn;
+    const value = (op as any).value;
+    const isObject = value !== null && typeof value === "object" && !Array.isArray(value);
+    // A value's keys (_delta_assert_fields): its columns the document holds (todo #70), the row's id (the path's wins) and parent key, a temporal row's validity.
+    const held = (k: string) => holdsField(def, schema, collKey, k);
+    const known = (k: string) => (Object.hasOwn(table.columns, k) && held(k)) || k === "id" || k === fk || k === "valid_from" || k === "valid_to";
+    // What a written field may hold, beyond its column's cast: a column that is
+    // not nullable is never null (the table's NOT NULL); a parent key is an id.
+    const fieldErr = (field: string, v: unknown): string | null => {
+      if (field === fk) {
+        if (v === null || v === undefined) return `${field} cannot be null`;
+        return Number.isInteger(v) || (typeof v === "string" && /^\s*[+-]?\d+\s*$/.test(v)) ? null : `${field} must be an id: an integer`;
+      }
+      const col = Object.hasOwn(table.columns, field) ? table.columns[field]! : undefined;
+      if (!col || v === null || v === undefined) return col && !col.nullable && v === null ? `${field} cannot be null` : null;
+      // a value of its column's type, not cast (_delta_assert_fields)
+      const of: Record<string, [boolean, string]> = {
+        text: [typeof v === "string", "a string"], integer: [Number.isInteger(v), "an integer"], real: [typeof v === "number", "a number"],
+        boolean: [typeof v === "boolean", "a boolean"], timestamptz: [isIsoTime(v) || v instanceof Date, "an ISO-8601 time"],
+      };
+      const [ok, what] = of[col.type] ?? [true, ""];
+      return ok ? null : `${field} must be ${what}`;
+    };
+    // A whole-row value: an object of known keys, each field as it may be written.
+    const rowErrs = (row: Record<string, unknown>, verb: "add" | "replace") => {
+      for (const k of Object.keys(row)) if (!known(k)) fail(`Unknown field: ${k}`);
+      for (const [k, v] of Object.entries(row)) {
+        const err = verb === "add" && k === fk && v == null ? null : fieldErr(k, v);   // an add's parent key, left null, is its document's, or found in scope
+        if (err) fail(err);
+      }
+    };
+    // A row's value, added or replaced whole (RFC 6902: a replace of the row is the row): a column
+    // that is neither nullable nor has a default must be given -- by the value, or, for a list-mode
+    // root row's add, by the scope, as delta_apply checks after it stamps them (#39, #3).
+    const required = (row: Record<string, unknown>, given: Record<string, unknown> = {}, add = false) => {   // a replace sets the columns it holds
+      for (const [col, colDef] of Object.entries(table.columns)) {
+        if (!colDef.nullable && colDef.default === undefined && !Object.hasOwn(row, col) && !Object.hasOwn(given, col) && (add || held(col))) {
+          fail(`Required field missing: ${col} (give it a value, or declare a default or make it nullable in the schema)`);
+        }
+      }
+    };
+    // A row's id, as the path names it: the number its digits name, up to 2^53 - 1 (_delta_row_id).
+    const idErr = (seg: string): string | null =>
+      !/^[0-9]+$/.test(seg) ? `Row id "${seg}" in ${op.path} is not a number: Postgres mints row ids -- add to ${joinPath(collKey, "-")} and read the id from the echo`
+      : pastSafeId(seg) ? `Row id ${seg} in ${op.path} is past 2^53 - 1, which no number holds exactly -- add to ${joinPath(collKey, "-")} and read the id from the echo`
+      : null;
+
+    // A single document's root: `replace /<root>` is the root row, whole, and
+    // `replace /<root>/<field>` (a segment that is not an id) writes one field
+    // -- a column, or the parent key: the row moves (#37).
+    if (collKey === def.root && !opts.list && op.op === "replace" && (parts.length === 1 || (parts.length === 2 && !/^\d+$/.test(parts[1]!)))) {
+      if (parts[1] === "id") fail("Unknown field: id (a row keeps the id its path names)");
+      else if (parts.length === 2) rowErrs({ [parts[1]!]: value }, "replace");
+      else if (!isObject) fail("Replace value must be an object");
+      else { rowErrs(value, "replace"); required(value); }
+      continue;
+    }
+
+    // A row: add /<coll>/<id or ->, remove /<coll>/<id>, replace /<coll>/<id>[/<field>].
+    const rowOp = (op.op === "add" || op.op === "remove") ? parts.length === 2 : op.op === "replace" && (parts.length === 2 || parts.length === 3);
+    if (!rowOp) { fail(`Invalid op: ${op.op} ${op.path}`); continue; }
+    // a single document holds one root row, the one its name names: it adds no other (#52) --
+    // when the mode is known; given none, a root row's add is a list's, and is taken. Its own
+    // it may add back, by an undo of its removal (#43): the database says whether it is gone.
+    const other = parts[1] === "-" || (rootId !== undefined && /^[0-9]+$/.test(parts[1]!) && Number(parts[1]) !== Number(rootId));
+    if (op.op === "add" && collKey === def.root && opts.list === false && other) { fail(`Invalid op: add ${op.path} -- a single document holds one ${collKey}, its root`); continue; }
+    const idError = op.op === "add" && parts[1] === "-" ? null : idErr(parts[1]!);
+    if (idError) { fail(idError); continue; }
+    if (op.op === "remove") continue;
+    if (parts.length === 3) {
+      const field = parts[2]!;
+      if (!known(field) || field === "id") { fail(`Unknown field: ${field}`); continue; }   // a row keeps the id its path names
+      const err = fieldErr(field, value);
+      if (err) fail(err);
+      continue;
+    }
+    if (!isObject) { fail(`${op.op === "add" ? "Add" : "Replace"} value must be an object`); continue; }
+    rowErrs(value, op.op as "add" | "replace");
+    required(value, op.op === "add" && opts.list && collKey === def.root ? opts.values : {}, op.op === "add");
+  }
+  return errors;
+}

package/src/postgres/sql.ts

@@ -0,0 +1,42 @@
+/** SQL helpers for codegen.ts (the schema's DDL) and carry.ts (export and import). */
+import type { ColumnDef, Schema } from "./schema";
+
+/**
+ * Emit a single-quoted SQL string literal with embedded quotes escaped.
+ * Use for ANY value interpolated into a `'…'` literal position so an
+ * apostrophe in a name can't terminate the literal or inject SQL.
+ */
+export function lit(value: string): string {
+  return `'${value.replace(/'/g, "''")}'`;
+}
+
+export function columnSqlType(def: ColumnDef): string {
+  switch (def.type) {
+    case "text": return "TEXT";
+    case "json": return "JSONB";
+    case "integer": return "BIGINT";
+    case "boolean": return "BOOLEAN";
+    case "real": return "DOUBLE PRECISION";
+    case "timestamptz": return "TIMESTAMPTZ";
+  }
+}
+
+export function sqlDefault(value: unknown): string {
+  if (typeof value === "string") return lit(value);
+  if (typeof value === "number") return String(value);
+  if (typeof value === "boolean") return value ? "TRUE" : "FALSE";
+  return "NULL";
+}
+
+export function findCascadeOn(collKey: string, schema: Schema): { fk: string; collection: string }[] {
+  const result: { fk: string; collection: string }[] = [];
+  for (const [otherKey, otherTable] of Object.entries(schema.tables)) {
+    if (otherKey === collKey) continue;
+    for (const ref of otherTable.referencedBy) {
+      if (ref.collection === collKey) {
+        result.push({ fk: ref.fkColumn, collection: otherKey });
+      }
+    }
+  }
+  return result;
+}

package/src/sql/001a-delta-meta.sql

@@ -0,0 +1,204 @@
+-- =========================================================================
+-- Delta-doc: metadata tables
+-- =========================================================================
+-- Runtime schema registry populated by TypeScript (createTables / registerDocs).
+-- =========================================================================
+
+CREATE TABLE IF NOT EXISTS _delta_collections (
+  collection_key   TEXT PRIMARY KEY,
+  table_name       TEXT NOT NULL,
+  columns_def      JSONB NOT NULL DEFAULT '{}',
+  parent_collection TEXT,
+  parent_fk        TEXT,
+  temporal         BOOLEAN NOT NULL DEFAULT TRUE,
+  cascade_on       JSONB NOT NULL DEFAULT '[]'
+);
+
+CREATE TABLE IF NOT EXISTS _delta_docs (
+  prefix           TEXT PRIMARY KEY,
+  root_collection  TEXT NOT NULL,
+  include          TEXT[] NOT NULL DEFAULT '{}',
+  scope            JSONB NOT NULL DEFAULT '{}'
+);
+
+-- An implied document is there before its root row is (todo #34): it opens
+-- empty, and its first write makes the row -- as on SQLite and ./json.
+ALTER TABLE _delta_docs ADD COLUMN IF NOT EXISTS implied BOOLEAN NOT NULL DEFAULT FALSE;
+
+-- The columns a document holds of a collection it names (todo #70): collection
+-- key -> its column names. The row's id and parent key it holds always; a
+-- collection not named, whole. It is opened, told and written by those alone.
+ALTER TABLE _delta_docs ADD COLUMN IF NOT EXISTS held JSONB NOT NULL DEFAULT '{}';
+
+CREATE TABLE IF NOT EXISTS _delta_versions (
+  doc_name TEXT PRIMARY KEY,
+  version  BIGINT NOT NULL DEFAULT 0
+);
+
+-- The documents open now (todo #73): a row for each name a listener has
+-- subscribers for, and which listener. A write's fan-out asks these names, not
+-- every name ever opened. A listener is alive while its connection holds its
+-- lock (001c, _delta_holder_live); a dead one's rows are cleared.
+CREATE SEQUENCE IF NOT EXISTS _delta_holder_ids AS integer;
+CREATE TABLE IF NOT EXISTS _delta_held (
+  doc_name TEXT NOT NULL,
+  holder   INTEGER NOT NULL,
+  PRIMARY KEY (doc_name, holder)
+);
+
+CREATE TABLE IF NOT EXISTS _delta_ops_log (
+  id         BIGSERIAL PRIMARY KEY,
+  doc_name   TEXT NOT NULL,
+  version    BIGINT NOT NULL,
+  ops        JSONB NOT NULL,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+CREATE INDEX IF NOT EXISTS idx_delta_ops_log_fetch
+  ON _delta_ops_log (doc_name, version);
+
+-- The write itself, once, for the custom documents that watch its rows (todo
+-- #44): on the writer's entry, the ops as delta_apply applied them -- each row
+-- it changed as it now is, or its remove -- so a custom document hears a write
+-- once, as on SQLite, and not once for each document it was told to (where a
+-- row that left a document is a remove, which is not a delete). Kept only where
+-- they differ from what the writer was told: NULL is heard as told, so an
+-- entry whose ops are the write, and one logged some other way (a direct
+-- _delta_bump_and_notify), carry none. '[]' on the other documents' entries of
+-- the same write.
+ALTER TABLE _delta_ops_log ADD COLUMN IF NOT EXISTS applied JSONB;
+
+-- =========================================================================
+-- Pure helpers — composed by every read/write/ops function below
+-- =========================================================================
+
+-- Build an RFC 6901 JSON Pointer, each segment escaped (~ as ~0, / as ~1).
+--   _delta_build_path('sites', '42')  -> '/sites/42'
+--   _delta_build_path('venues')       -> '/venues'
+CREATE OR REPLACE FUNCTION _delta_build_path(p_collection TEXT, p_id TEXT DEFAULT NULL)
+RETURNS text LANGUAGE sql IMMUTABLE AS $$
+  SELECT '/' || replace(replace(p_collection, '~', '~0'), '/', '~1')
+      || CASE WHEN p_id IS NULL THEN '' ELSE '/' || replace(replace(p_id, '~', '~0'), '/', '~1') END;
+$$;
+
+-- Split an RFC 6901 JSON Pointer into its unescaped segments -- the grammar of
+-- splitPath in src/core.ts: '' is the whole document, every other path starts
+-- with '/', an empty segment is a key, and a '~' not followed by 0 or 1 is an
+-- error. SQLSTATE 22023 (invalid_parameter_value) answers 400 on the wire.
+--   _delta_split_path('/sites/42')  -> ARRAY['sites','42']
+--   _delta_split_path('/a~1b/c~0d') -> ARRAY['a/b','c~d']
+--   _delta_split_path('sites')      -> raises
+CREATE OR REPLACE FUNCTION _delta_split_path(p_path TEXT)
+RETURNS text[] LANGUAGE plpgsql IMMUTABLE AS $$
+BEGIN
+  IF p_path = '' THEN RETURN ARRAY[]::text[]; END IF;
+  IF p_path IS NULL OR left(p_path, 1) <> '/' THEN
+    RAISE EXCEPTION 'Invalid JSON Pointer "%": a path starts with "/" ("" is the whole document)', p_path
+      USING ERRCODE = '22023';
+  END IF;
+  IF p_path ~ '~([^01]|$)' THEN
+    RAISE EXCEPTION 'Invalid JSON Pointer "%": "~" is written "~0" and "/" is written "~1"', p_path
+      USING ERRCODE = '22023';
+  END IF;
+  RETURN ARRAY(
+    SELECT replace(replace(seg, '~1', '/'), '~0', '~')
+      FROM unnest(regexp_split_to_array(substr(p_path, 2), '/')) WITH ORDINALITY AS t(seg, n)
+     ORDER BY n
+  );
+END;
+$$;
+
+-- A row id from a path segment. This backend keeps BIGINT ids it mints from
+-- seq_<table>, so anything else is a client error (SQLSTATE 22P02 → 400) that
+-- names the fix. So is a number past 2^53 - 1: a row is read out as JSON, and
+-- no JavaScript number holds such an id exactly, so the row would be told under
+-- another (the SQLite backend refuses it too). Digits name their number, however
+-- many there are ("007" is 7).
+CREATE OR REPLACE FUNCTION _delta_row_id(p_collection TEXT, p_segment TEXT)
+RETURNS bigint LANGUAGE plpgsql IMMUTABLE AS $$
+BEGIN
+  IF p_segment IS NULL OR p_segment !~ '^[0-9]+$' THEN
+    RAISE EXCEPTION 'row id "%" in /%/% is not a number: Postgres mints row ids -- add to /%/- and read the id from the echo',
+      p_segment, p_collection, p_segment, p_collection
+      USING ERRCODE = '22P02';
+  END IF;
+  IF p_segment::numeric > 9007199254740991 THEN
+    RAISE EXCEPTION 'row id % in /%/% is past 2^53 - 1, which no number holds exactly -- add to /%/- and read the id from the echo',
+      p_segment, p_collection, p_segment, p_collection
+      USING ERRCODE = '22P02';
+  END IF;
+  RETURN p_segment::bigint;
+END;
+$$;
+
+-- Refuse a value that is not an object, names a field the collection does
+-- not have, or gives a column a value not of its type (SQLSTATE 22023 → 400).
+-- It used to be merged into the broadcast row, acked, and dropped (or cast) by
+-- jsonb_populate_record: stored otherwise than told. id, the parent key and
+-- the temporal columns are the row's own.
+CREATE OR REPLACE FUNCTION _delta_assert_fields(
+  p_collection TEXT, p_columns JSONB, p_parent_fk TEXT, p_value JSONB
+) RETURNS void LANGUAGE plpgsql IMMUTABLE AS $$
+DECLARE
+  v_unknown TEXT;
+BEGIN
+  IF p_value IS NULL OR jsonb_typeof(p_value) <> 'object' THEN
+    RAISE EXCEPTION 'a row value for % must be an object', p_collection USING ERRCODE = '22023';
+  END IF;
+  SELECT string_agg(k, ', ' ORDER BY k) INTO v_unknown
+    FROM jsonb_object_keys(p_value) AS k
+   WHERE NOT COALESCE(p_columns, '{}'::jsonb) ? k
+     AND k NOT IN ('id', 'valid_from', 'valid_to')
+     AND k IS DISTINCT FROM p_parent_fk;
+  IF v_unknown IS NOT NULL THEN
+    RAISE EXCEPTION 'Unknown field: % (not a column of %)', v_unknown, p_collection USING ERRCODE = '22023';
+  END IF;
+  -- the parent key, given, is a whole number or text; text that is no number's digits is
+  -- refused when the column casts it (22P02, 400), as SQLite refuses a key that names no row
+  IF p_parent_fk IS NOT NULL AND p_value ? p_parent_fk AND jsonb_typeof(p_value->p_parent_fk) <> 'null'
+     AND NOT (jsonb_typeof(p_value->p_parent_fk) = 'string'
+              OR (jsonb_typeof(p_value->p_parent_fk) = 'number' AND (p_value->>p_parent_fk)::numeric = trunc((p_value->>p_parent_fk)::numeric))) THEN
+    RAISE EXCEPTION '% must be an id: an integer', p_parent_fk USING ERRCODE = '22023';
+  END IF;
+  -- each column's value of its type, as SQLite's validateOps asks: not cast (5 into a text column is "5")
+  SELECT string_agg(format('%s must be %s', k, CASE c->>'type' WHEN 'text' THEN 'a string' WHEN 'integer' THEN 'an integer'
+           WHEN 'real' THEN 'a number' WHEN 'boolean' THEN 'a boolean' ELSE 'an ISO-8601 time' END), '; ' ORDER BY k) INTO v_unknown
+    FROM jsonb_each(p_value) AS v(k, val), LATERAL (SELECT p_columns->k AS c) col
+   WHERE c IS NOT NULL AND jsonb_typeof(val) <> 'null' AND NOT CASE c->>'type'
+     WHEN 'text' THEN jsonb_typeof(val) = 'string'
+     WHEN 'integer' THEN jsonb_typeof(val) = 'number' AND val::text::numeric = trunc(val::text::numeric)
+     WHEN 'real' THEN jsonb_typeof(val) = 'number'
+     WHEN 'boolean' THEN jsonb_typeof(val) = 'boolean'
+     -- ISO-8601, as isIsoTime (src/schema.ts) reads it; a day off the calendar is refused by the cast (22008, 400)
+     WHEN 'timestamptz' THEN jsonb_typeof(val) = 'string' AND (val #>> '{}') ~ '^\d{4}-\d{2}-\d{2}([T ]([01]\d|2[0-3]):[0-5]\d(:[0-5]\d(\.\d{1,6})?)?(Z|[+-]([01]\d|2[0-3])(:?[0-5]\d)?)?)?$'
+     ELSE TRUE END;
+  IF v_unknown IS NOT NULL THEN
+    RAISE EXCEPTION '%', v_unknown USING ERRCODE = '22023';
+  END IF;
+END;
+$$;
+
+-- Pick the source relation for a collection.
+--   temporal + no timestamp → current_<table> view
+--   temporal + timestamp    → base <table> (caller filters with _delta_temporal_where)
+--   non-temporal            → base <table>
+CREATE OR REPLACE FUNCTION _delta_source_view(
+  p_table_name TEXT, p_temporal BOOLEAN, p_at TIMESTAMPTZ DEFAULT NULL
+) RETURNS text LANGUAGE sql IMMUTABLE AS $$
+  SELECT CASE
+    WHEN p_temporal AND p_at IS NULL THEN 'current_' || p_table_name
+    ELSE p_table_name
+  END;
+$$;
+
+-- Temporal WHERE fragment for embedding into format()-built dynamic SQL.
+-- Inlines the timestamp as a %L literal so callers don't need extra param binding.
+-- Aliases the row with `p_alias` (default `t`).
+CREATE OR REPLACE FUNCTION _delta_temporal_where(
+  p_at TIMESTAMPTZ, p_alias TEXT DEFAULT 't'
+) RETURNS text LANGUAGE sql IMMUTABLE AS $$
+  SELECT format(
+    '%s.valid_from <= %L AND (%s.valid_to IS NULL OR %s.valid_to > %L)',
+    p_alias, p_at, p_alias, p_alias, p_at
+  );
+$$;