dzql 0.6.37 → 0.8.0

package/src/cli/compile.ts

@@ -0,0 +1,263 @@
+/**
+ * Domain -> SQL compilation pipeline.
+ *
+ * The CLI writes these strings to migration files; tests load the same strings
+ * into Postgres, so they exercise exactly the SQL the compiler produces.
+ */
+import type { DomainConfig } from "../shared/ir.js";
+import { analyzeDomain } from "./compiler/analyzer.js";
+import { generateIR } from "./compiler/ir.js";
+import { validateDomainPaths } from "./compiler/path-validator.js";
+import { generateCoreSQL, generateAuthSQL, generateEntitySQL, generateSchemaSQL } from "./codegen/sql.js";
+import { generateSubscribableSQL, generateComputeAffectedKeysFunction } from "./codegen/subscribable_sql.js";
+
+export interface CompiledDomain {
+  ir: any;
+  coreSQL: string;
+  /** Tables, CRUD functions, auth functions and custom functions */
+  schemaSQL: string;
+  /** Subscribable functions plus compute_affected_keys */
+  subscribableSQL: string;
+  customFunctionCount: number;
+}
+
+export class DomainValidationError extends Error {
+  constructor(public errors: string[]) {
+    super(`Domain validation failed:\n${errors.map(e => `  - ${e}`).join('\n')}`);
+    this.name = 'DomainValidationError';
+  }
+}
+
+export function compileDomain(domain: DomainConfig): CompiledDomain {
+  const errors = analyzeDomain(domain);
+  if (errors.length > 0) {
+    throw new DomainValidationError(errors);
+  }
+
+  const ir = generateIR(domain);
+
+  // Every permission, notification and canSubscribe path must compile against
+  // this domain. A path that can't is a build error, never FALSE or a fallback.
+  const pathErrors = validateDomainPaths(ir);
+  if (pathErrors.length > 0) {
+    throw new DomainValidationError(pathErrors);
+  }
+
+  const coreSQL = generateCoreSQL();
+
+  // Topologically sort entities by FK dependencies
+  // Entities must be created before entities that reference them
+  // Also detects circular FKs that need deferred constraint creation
+  const { sorted: sortedEntityNames, circularFKs } = topologicalSortEntities(ir.entities);
+
+  // Build set of circular FK columns for stripping during schema generation
+  const circularFKSet = new Set(circularFKs.map(fk => `${fk.table}.${fk.column}`));
+
+  const entitySQL: string[] = [];
+  for (const name of sortedEntityNames) {
+    const entityIR = ir.entities[name];
+    // Pass circular FK info so schema generation can strip those constraints
+    entitySQL.push(generateSchemaSQL(name, entityIR, circularFKSet));
+    // Skip CRUD generation for unmanaged entities (e.g., junction tables)
+    if (entityIR.managed !== false) {
+      entitySQL.push(generateEntitySQL(name, entityIR, ir.entities));
+    } else {
+      console.log(`[Compiler] Skipping CRUD for unmanaged entity: ${name}`);
+    }
+  }
+
+  // Add deferred FK constraints at the end (for circular dependencies)
+  if (circularFKs.length > 0) {
+    entitySQL.push(generateDeferredFKConstraints(circularFKs));
+  }
+
+  // Generate subscribable SQL functions
+  const subscribableSQL: string[] = [];
+  const subscribableNames = Object.keys(ir.subscribables);
+  for (const [name, subIR] of Object.entries(ir.subscribables)) {
+    console.log(`[Compiler] Generating SQL for subscribable: ${name}`);
+    subscribableSQL.push(generateSubscribableSQL(name, subIR as any, ir.entities));
+  }
+  // Generate central compute_affected_keys function that aggregates all subscribables
+  subscribableSQL.push(generateComputeAffectedKeysFunction(subscribableNames));
+
+  // Collect custom functions SQL
+  const customFunctionSQL: string[] = [];
+  for (const fn of ir.customFunctions) {
+    console.log(`[Compiler] Adding custom function: ${fn.name}`);
+    customFunctionSQL.push(fn.sql);
+  }
+
+  // Combine entity SQL with auth functions and custom functions
+  // Auth functions must come after schema (they depend on users table)
+  // Columns a client may set at registration: those declared in
+  // auth.registerParams (beyond the built-in email/password/options), else
+  // just the users label field. Everything else is rejected at sign-up.
+  const usersEntity = ir.entities.users;
+  const declaredRegisterable = Object.keys(ir.auth?.registerParams || {})
+    .filter((k) => !["email", "password", "options"].includes(k));
+  const registerableColumns = declaredRegisterable.length > 0
+    ? declaredRegisterable
+    : (usersEntity && usersEntity.labelField && usersEntity.labelField !== "id"
+        ? [usersEntity.labelField]
+        : []);
+  const authSQL = generateAuthSQL(registerableColumns);
+  let schemaSQL = entitySQL.join('\n');
+  schemaSQL += '\n\n-- Auth Functions\n' + authSQL;
+  if (customFunctionSQL.length > 0) {
+    schemaSQL += '\n\n-- Custom Functions\n' + customFunctionSQL.join('\n\n');
+  }
+
+  return {
+    ir,
+    coreSQL,
+    schemaSQL,
+    subscribableSQL: subscribableSQL.join('\n\n'),
+    customFunctionCount: customFunctionSQL.length,
+  };
+}
+
+/** FK info for dependency analysis */
+interface FKInfo {
+  table: string;
+  column: string;
+  referencedTable: string;
+  fullConstraint: string; // e.g., "int NOT NULL REFERENCES users(id)"
+}
+
+/** Result of topological sort with cycle detection */
+interface SortResult {
+  sorted: string[];
+  circularFKs: FKInfo[];
+}
+
+/**
+ * Topologically sort entities based on FK dependencies.
+ * Entities that are referenced by others come first.
+ * Uses Kahn's algorithm for topological sorting.
+ * Returns both sorted order and any circular FK constraints that need deferred creation.
+ */
+function topologicalSortEntities(entities: Record<string, any>): SortResult {
+  const entityNames = Object.keys(entities);
+
+  // Build dependency graph: entity -> entities it depends on (references)
+  const dependencies: Record<string, Set<string>> = {};
+  const dependents: Record<string, Set<string>> = {};
+
+  // Track all FK info for cycle detection
+  const allFKs: FKInfo[] = [];
+
+  for (const name of entityNames) {
+    dependencies[name] = new Set();
+    dependents[name] = new Set();
+  }
+
+  // Parse REFERENCES from column types
+  for (const name of entityNames) {
+    const entity = entities[name];
+    for (const col of entity.columns || []) {
+      const match = col.type?.match(/REFERENCES\s+(\w+)/i);
+      if (match) {
+        const referencedEntity = match[1];
+        // Only track dependencies to entities we're managing
+        if (entityNames.includes(referencedEntity)) {
+          dependencies[name].add(referencedEntity);
+          dependents[referencedEntity].add(name);
+          allFKs.push({
+            table: name,
+            column: col.name,
+            referencedTable: referencedEntity,
+            fullConstraint: col.type
+          });
+        }
+      }
+    }
+  }
+
+  // Kahn's algorithm
+  const result: string[] = [];
+  const noIncoming: string[] = [];
+  const remainingDeps: Record<string, Set<string>> = {};
+
+  // Copy dependencies for mutation
+  for (const name of entityNames) {
+    remainingDeps[name] = new Set(dependencies[name]);
+  }
+
+  // Find entities with no dependencies (no incoming edges)
+  for (const name of entityNames) {
+    if (remainingDeps[name].size === 0) {
+      noIncoming.push(name);
+    }
+  }
+
+  while (noIncoming.length > 0) {
+    const node = noIncoming.shift()!;
+    result.push(node);
+
+    // Remove this node from the graph
+    for (const dependent of dependents[node]) {
+      remainingDeps[dependent].delete(node);
+      if (remainingDeps[dependent].size === 0) {
+        noIncoming.push(dependent);
+      }
+    }
+  }
+
+  // Detect circular FKs
+  const circularFKs: FKInfo[] = [];
+  if (result.length !== entityNames.length) {
+    const cycleEntities = new Set(entityNames.filter(n => !result.includes(n)));
+    console.warn(`[Compiler] Warning: Circular FK dependencies detected among: ${[...cycleEntities].join(', ')}`);
+
+    // Find FKs that are part of the cycle - these need deferred creation
+    for (const fk of allFKs) {
+      if (cycleEntities.has(fk.table) && cycleEntities.has(fk.referencedTable)) {
+        circularFKs.push(fk);
+        console.log(`[Compiler] Deferring circular FK: ${fk.table}.${fk.column} -> ${fk.referencedTable}`);
+      }
+    }
+
+    // Add remaining entities (cycle members) to result
+    result.push(...cycleEntities);
+  }
+
+  return { sorted: result, circularFKs };
+}
+
+/**
+ * Generate ALTER TABLE statements for deferred circular FK constraints.
+ */
+function generateDeferredFKConstraints(circularFKs: FKInfo[]): string {
+  if (circularFKs.length === 0) return '';
+
+  const statements: string[] = [
+    '\n-- === DEFERRED FK CONSTRAINTS (for circular dependencies) ===\n'
+  ];
+
+  for (const fk of circularFKs) {
+    // Extract the REFERENCES part from the full constraint
+    const refMatch = fk.fullConstraint.match(/REFERENCES\s+(\w+)(\([^)]+\))?(\s+ON\s+.+)?/i);
+    if (refMatch) {
+      const targetTable = refMatch[1];
+      const targetCol = refMatch[2] || '(id)';
+      const cascadeClause = refMatch[3] || '';
+      const constraintName = `fk_${fk.table}_${fk.column}`;
+
+      statements.push(`ALTER TABLE ${fk.table} ADD CONSTRAINT ${constraintName} FOREIGN KEY (${fk.column}) REFERENCES ${targetTable}${targetCol}${cascadeClause};`);
+    }
+  }
+
+  return statements.join('\n');
+}
+
+/**
+ * Remove REFERENCES clause from a column type for deferred FK creation.
+ * Preserves NOT NULL and other modifiers.
+ */
+function stripReferences(columnType: string): string {
+  // Remove REFERENCES clause and any ON DELETE/UPDATE clauses
+  return columnType
+    .replace(/\s*REFERENCES\s+\w+(\([^)]+\))?(\s+ON\s+(DELETE|UPDATE)\s+(CASCADE|SET NULL|SET DEFAULT|RESTRICT|NO ACTION))*\s*/gi, ' ')
+    .trim();
+}

package/src/cli/compiler/path-validator.ts

@@ -1,118 +1,150 @@
 /**
  * Path Validator
  *
- * Validates all permission and notification paths in a domain configuration.
- * Used by the CLI `validate-paths` command.
+ * Checks every permission, notification and canSubscribe path in a domain
+ * against the domain itself: the syntax parses, every table is an entity,
+ * every column exists, and values compared with ids are integer keys.
+ * compileDomain refuses to build a domain with any error here, and the CLI
+ * `validate-paths` command reports them path by path.
  */
 
-import type { DomainConfig, EntityConfig } from "../../shared/ir.js";
-import { parsePath } from "./paths.js";
+import type { DomainConfig } from "../../shared/ir.js";
+import { generateIR } from "./ir.js";
+import { parsePath, isIntegerType } from "./paths.js";
 
 export interface PathValidationResult {
   entity: string;
-  pathType: 'permissions.view' | 'permissions.create' | 'permissions.update' | 'permissions.delete' | 'notifications';
+  pathType: 'permissions.view' | 'permissions.create' | 'permissions.update' | 'permissions.delete' | 'notifications' | 'canSubscribe';
   pathIndex: number;
   path: string;
   valid: boolean;
   error?: string;
 }
 
+interface EntityLike {
+  columns: Array<{ name: string; type: string }>;
+}
+
+type PathKind = 'permission' | 'notification' | 'subscribe-list';
+
 /**
- * Validate all paths in a domain configuration
+ * Check one path against the domain. Returns the problems found (empty if none).
+ *
+ * @param entity - the entity whose row the path starts from
+ * @param fieldMap - renames start fields (a subscribable's key param is the root's id)
  */
-export function validatePaths(domain: DomainConfig): PathValidationResult[] {
-  const results: PathValidationResult[] = [];
+export function checkPath(
+  rule: string,
+  entityName: string,
+  entities: Record<string, EntityLike>,
+  kind: PathKind = 'permission',
+  fieldMap: Record<string, string> = {}
+): string[] {
+  const parsed = parsePath(rule);
+  if (!parsed.isValid) return [parsed.error ?? 'invalid path'];
+
+  if (parsed.literal) {
+    return kind === 'notification' ? [`a notification path must lead to users, not ${parsed.literal}`] : [];
+  }
+  if (kind === 'subscribe-list') {
+    return ['a list subscribable has no root row to follow a path from; use TRUE, FALSE or AUTHENTICATED'];
+  }
 
-  for (const [entityName, entity] of Object.entries(domain.entities)) {
-    // Validate permission paths
-    const permissions = entity.permissions || {};
-
-    for (const permType of ['view', 'create', 'update', 'delete'] as const) {
-      const paths = permissions[permType] || [];
-      for (let i = 0; i < paths.length; i++) {
-        const path = paths[i];
-        const result = validateSinglePath(path, entityName, domain);
-        results.push({
-          entity: entityName,
-          pathType: `permissions.${permType}`,
-          pathIndex: i,
-          path,
-          valid: result.valid,
-          error: result.error
-        });
-      }
-    }
+  const problems: string[] = [];
+  const columnOf = (table: string, column: string) =>
+    entities[table]?.columns.find(c => c.name === column);
 
-    // Validate notification paths
-    const notifications = (entity as any).notifications || {};
-    for (const [notifyName, paths] of Object.entries(notifications)) {
-      if (!Array.isArray(paths)) continue;
-      for (let i = 0; i < paths.length; i++) {
-        const path = paths[i] as string;
-        const result = validateSinglePath(path, entityName, domain);
-        results.push({
-          entity: entityName,
-          pathType: 'notifications',
-          pathIndex: i,
-          path,
-          valid: result.valid,
-          error: result.error ? `${notifyName}: ${result.error}` : undefined
-        });
-      }
+  const requireColumn = (table: string, column: string, integer: boolean) => {
+    const col = columnOf(table, column);
+    if (!col) {
+      problems.push(`no column '${column}' on ${table}`);
+    } else if (integer && !isIntegerType(col.type)) {
+      problems.push(`'${column}' is ${col.type.split(/\s/)[0]}, not an integer key`);
     }
-  }
-
-  // Validate subscribable canSubscribe paths
-  for (const [subName, sub] of Object.entries(domain.subscribables || {})) {
-    const paths = sub.canSubscribe || [];
-    for (let i = 0; i < paths.length; i++) {
-      const path = paths[i];
-      const result = validateSinglePath(path, subName, domain);
-      results.push({
-        entity: subName,
-        pathType: 'permissions.view',
-        pathIndex: i,
-        path,
-        valid: result.valid,
-        error: result.error
-      });
+  };
+  // A field of the row being checked (its start field, or an @field in a filter)
+  const requireOwnField = (field: string) => requireColumn(entityName, fieldMap[field] ?? field, true);
+
+  if (!parsed.tableFirst) requireOwnField(parsed.startField!);
+
+  const steps = parsed.tableFirst ? parsed.hops : parsed.hops.slice(1);
+  for (const hop of steps) {
+    const table = hop.table!;
+    if (!entities[table]) {
+      problems.push(`unknown table '${table}'`);
+      continue;
     }
+    // A hop without [filters] joins the previous value on its id.
+    if (!hop.filters?.length) requireColumn(table, 'id', true);
+    for (const f of hop.filters ?? []) {
+      requireColumn(table, f.field, f.value === '$');
+      if (f.value.startsWith('@')) requireOwnField(f.value.slice(1));
+    }
+    for (const c of hop.conditions ?? []) requireColumn(table, c.field, false);
+    // The value passed on joins the next hop's key; the last one is compared with the user.
+    requireColumn(table, hop.outputField!, true);
   }
 
-  return results;
+  return problems;
 }
 
 /**
- * Validate a single path expression
+ * Validate every path in a compiled domain. Each error names where the path is
+ * written, what is wrong, and the path itself.
  */
-function validateSinglePath(
-  path: string,
-  entityName: string,
-  domain: DomainConfig
-): { valid: boolean; error?: string } {
-  // Parse the path
-  const parsed = parsePath(path);
-
-  if (!parsed.isValid) {
-    return { valid: false, error: parsed.error };
-  }
-
-  // Optionally validate that referenced tables exist
-  const errors: string[] = [];
+export function validateDomainPaths(ir: { entities: Record<string, any>; subscribables: Record<string, any> }): string[] {
+  return collectResults(ir).filter(r => !r.valid).map(r => r.error!);
+}
 
-  for (const hop of parsed.hops) {
-    if (hop.table && hop.type !== 'field') {
-      if (!domain.entities[hop.table]) {
-        errors.push(`Unknown table '${hop.table}'`);
-      }
+function collectResults(ir: { entities: Record<string, any>; subscribables: Record<string, any> }): PathValidationResult[] {
+  const results: PathValidationResult[] = [];
+  const record = (entity: string, pathType: PathValidationResult['pathType'], where: string, pathIndex: number, path: string, problems: string[]) => {
+    results.push({
+      entity,
+      pathType,
+      pathIndex,
+      path,
+      valid: problems.length === 0,
+      error: problems.length ? `${where}: ${problems.join('; ')} — in '${path}'` : undefined,
+    });
+  };
+
+  for (const [entityName, entity] of Object.entries<any>(ir.entities)) {
+    for (const action of ['view', 'create', 'update', 'delete'] as const) {
+      (entity.permissions?.[action] ?? []).forEach((path: string, i: number) => {
+        record(entityName, `permissions.${action}`, `${entityName}.permissions.${action}`, i, path,
+          checkPath(path, entityName, ir.entities));
+      });
+    }
+    for (const [notifyName, paths] of Object.entries<any>(entity.notifications ?? {})) {
+      if (!Array.isArray(paths)) continue;
+      paths.forEach((path: string, i: number) => {
+        record(entityName, 'notifications', `${entityName}.notifications.${notifyName}`, i, path,
+          checkPath(path, entityName, ir.entities, 'notification'));
+      });
     }
   }
 
-  if (errors.length > 0) {
-    return { valid: false, error: errors.join('; ') };
+  for (const [subName, sub] of Object.entries<any>(ir.subscribables)) {
+    const rootKey: string = sub.root?.key ?? '';
+    const isList = !rootKey;
+    const fieldMap = rootKey && !rootKey.startsWith('@') ? { [rootKey]: 'id' } : {};
+    (sub.canSubscribe ?? []).forEach((path: string, i: number) => {
+      const problems = !isList && !ir.entities[sub.root?.entity]
+        ? [`unknown root entity '${sub.root?.entity}'`]
+        : checkPath(path, sub.root.entity, ir.entities, isList ? 'subscribe-list' : 'permission', fieldMap);
+      record(subName, 'canSubscribe', `${subName}.canSubscribe`, i, path, problems);
+    });
   }
 
-  return { valid: true };
+  return results;
+}
+
+/**
+ * Validate all paths in a domain configuration (for the CLI `validate-paths` command)
+ */
+export function validatePaths(domain: DomainConfig): PathValidationResult[] {
+  return collectResults(generateIR(domain));
 }
 
 /**