dzql 0.5.33 → 0.6.1

package/src/cli/index.ts

@@ -0,0 +1,166 @@
+#!/usr/bin/env bun
+import { loadDomain } from "./compiler/loader.js";
+import { analyzeDomain } from "./compiler/analyzer.js";
+import { generateIR } from "./compiler/ir.js";
+import { generateCoreSQL, generateEntitySQL, generateSchemaSQL } from "./codegen/sql.js";
+import { generateSubscribableSQL } from "./codegen/subscribable_sql.js";
+import { generateManifest } from "./codegen/manifest.js";
+import { generateSubscribableStore } from "./codegen/subscribable_store.js";
+import { generateClientSDK } from "./codegen/client.js";
+import { writeFileSync, mkdirSync, copyFileSync, rmSync } from "fs";
+import { resolve, dirname } from "path";
+
+const args = process.argv.slice(2);
+let command = args[0];
+let input = args[1];
+let outputDir = "dist"; // Default output directory
+
+// If first arg looks like a file (ends with .ts or .js), treat it as compile target
+if (command && (command.endsWith('.ts') || command.endsWith('.js'))) {
+  input = command;
+  command = 'compile';
+}
+
+// Parse optional output directory flag
+const outputFlagIndex = args.indexOf('-o');
+const longOutputFlagIndex = args.indexOf('--output');
+
+if (outputFlagIndex > -1 && args[outputFlagIndex + 1]) {
+  outputDir = args[outputFlagIndex + 1];
+} else if (longOutputFlagIndex > -1 && args[longOutputFlagIndex + 1]) {
+  outputDir = args[longOutputFlagIndex + 1];
+}
+
+async function main() {
+  console.log("DZQL Compiler v0.6.0");
+
+  if (command === "compile") {
+    if (!input) {
+      console.error("Usage: dzql <file> or dzql compile <file>");
+      process.exit(1);
+    }
+
+    try {
+      // Phase 1: Load & Analyze
+      const fullInputPath = resolve(process.cwd(), input);
+
+      // Clean Output Directory
+      const absOutputDir = resolve(process.cwd(), outputDir);
+      console.log(`[Compiler] Cleaning ${absOutputDir}...`);
+      try {
+        rmSync(absOutputDir, { recursive: true, force: true });
+      } catch (e) { /* ignore */ }
+
+      const domain = await loadDomain(fullInputPath);
+      console.log("[Compiler] Domain loaded.");
+
+      const errors = analyzeDomain(domain);
+      if (errors.length > 0) {
+        console.error("[Compiler] Validation Failed:");
+        errors.forEach(err => console.error(`  - ${err}`));
+        process.exit(1);
+      }
+
+      // Phase 2: Generate IR
+      const ir = generateIR(domain);
+      console.log(`[Compiler] IR Generated. Subscribables: ${Object.keys(ir.subscribables).join(', ')}`);
+
+      // Phase 3: Generate SQL
+      const coreSQL = generateCoreSQL();
+      const entitySQL: string[] = [];
+      for (const [name, entityIR] of Object.entries(ir.entities)) {
+        entitySQL.push(generateSchemaSQL(name, entityIR));
+        // Skip CRUD generation for unmanaged entities (e.g., junction tables)
+        if (entityIR.managed !== false) {
+          entitySQL.push(generateEntitySQL(name, entityIR));
+        } else {
+          console.log(`[Compiler] Skipping CRUD for unmanaged entity: ${name}`);
+        }
+      }
+
+      // Generate subscribable SQL functions
+      const subscribableSQL: string[] = [];
+      for (const [name, subIR] of Object.entries(ir.subscribables)) {
+        console.log(`[Compiler] Generating SQL for subscribable: ${name}`);
+        subscribableSQL.push(generateSubscribableSQL(name, subIR as any, ir.entities));
+      }
+
+      // Collect custom functions SQL
+      const customFunctionSQL: string[] = [];
+      for (const fn of ir.customFunctions) {
+        console.log(`[Compiler] Adding custom function: ${fn.name}`);
+        customFunctionSQL.push(fn.sql);
+      }
+
+      // Phase 4: Generate Manifest
+      const manifest = generateManifest(ir);
+
+      // --- OUTPUT GENERATION ---
+
+      // 1. Database
+      const dbDir = resolve(outputDir, "db/migrations");
+      mkdirSync(dbDir, { recursive: true });
+
+      writeFileSync(resolve(dbDir, `000_core.sql`), coreSQL);
+      const timestamp = new Date().toISOString().replace(/[:.-]/g, '');
+
+      // Combine entity SQL with custom functions
+      const schemaContent = customFunctionSQL.length > 0
+        ? entitySQL.join('\n') + '\n\n-- Custom Functions\n' + customFunctionSQL.join('\n\n')
+        : entitySQL.join('\n');
+      writeFileSync(resolve(dbDir, `${timestamp}_schema.sql`), schemaContent);
+
+      if (customFunctionSQL.length > 0) {
+        console.log(`[Generated] ${customFunctionSQL.length} Custom Functions`);
+      }
+
+      // Write subscribable SQL
+      if (subscribableSQL.length > 0) {
+        writeFileSync(resolve(dbDir, `${timestamp}_subscribables.sql`), subscribableSQL.join('\n\n'));
+        console.log(`[Generated] ${subscribableSQL.length} Subscribable SQL functions`);
+      }
+
+      console.log(`[Generated] DB Migrations in ${dbDir}`);
+
+      // 2. Runtime
+      const runtimeDir = resolve(outputDir, "runtime");
+      mkdirSync(runtimeDir, { recursive: true });
+      writeFileSync(resolve(runtimeDir, `manifest.json`), JSON.stringify(manifest, null, 2));
+      console.log(`[Generated] Runtime Manifest in ${runtimeDir}`);
+
+      // 3. Client SDK (TypeScript)
+      const clientDir = resolve(outputDir, "client");
+      mkdirSync(clientDir, { recursive: true });
+
+      // Generate Core SDK as TypeScript
+      const clientCode = generateClientSDK(manifest);
+      writeFileSync(resolve(clientDir, `ws.ts`), clientCode);
+
+      // Generate Index
+      writeFileSync(resolve(clientDir, `index.ts`), `export * from './ws.js';`);
+
+      console.log(`[Generated] Client SDK in ${clientDir}`);
+
+      // 4. Stores (TypeScript)
+      const storeDir = resolve(clientDir, "stores");
+      mkdirSync(storeDir, { recursive: true });
+
+      for (const subName of Object.keys(ir.subscribables)) {
+          const storeCode = generateSubscribableStore(manifest, subName);
+          const fileName = `use${subName.replace(/(^|_)([a-z])/g, (g) => g.at(-1)!.toUpperCase())}Store.ts`;
+          writeFileSync(resolve(storeDir, fileName), storeCode);
+      }
+      console.log(`[Generated] ${Object.keys(ir.subscribables).length} Pinia Stores in ${storeDir}`);
+
+      console.log("[Compiler] Build Complete.");
+
+    } catch (e) {
+      console.error(e);
+      process.exit(1);
+    }
+  } else {
+    console.log("Unknown command. Try 'compile'.");
+  }
+}
+
+main();

package/src/client/index.ts

@@ -0,0 +1 @@
+export * from './ws.js';

package/src/client/ws.ts

@@ -0,0 +1,286 @@
+// Core WebSocket Manager for TZQL Client
+// Handles connection, auth, reconnects, and message dispatching.
+// This is a pure transport layer - it does not manage or cache data.
+
+export interface WebSocketOptions {
+  url?: string;
+  maxReconnectAttempts?: number;
+  tokenName?: string;
+}
+
+// Get default token name from environment (build-time injection)
+function getDefaultTokenName(): string {
+  // Vite: import.meta.env.VITE_TZQL_TOKEN_NAME
+  // @ts-ignore
+  if (typeof import.meta !== 'undefined' && import.meta.env?.VITE_TZQL_TOKEN_NAME) {
+    // @ts-ignore
+    return import.meta.env.VITE_TZQL_TOKEN_NAME;
+  }
+  // Node/bundlers: process.env.TZQL_TOKEN_NAME
+  if (typeof process !== 'undefined' && process.env?.TZQL_TOKEN_NAME) {
+    return process.env.TZQL_TOKEN_NAME;
+  }
+  return 'tzql_token';
+}
+
+export class WebSocketManager {
+  protected ws: WebSocket | null = null;
+  protected messageId = 0;
+  protected pendingRequests = new Map<number, { resolve: (val: any) => void; reject: (err: any) => void }>();
+  protected methodHandlers = new Map<string, Set<(params: any) => void>>();
+  protected subscriptionCallbacks = new Map<string, (event: any) => void>();
+  protected readyCallbacks = new Set<(user: any) => void>();
+  protected reconnectAttempts = 0;
+  protected maxReconnectAttempts = 5;
+  protected tokenName = 'tzql_token';
+  protected isShuttingDown = false;
+
+  // Connection state
+  public user: any = null;
+  public ready: boolean = false;
+
+  // To be populated by generated code
+  public api: any = {};
+
+  constructor(options: WebSocketOptions = {}) {
+    this.maxReconnectAttempts = options.maxReconnectAttempts ?? 5;
+    this.tokenName = options.tokenName ?? getDefaultTokenName();
+  }
+
+  async login(credentials: any) {
+    try {
+      const result = await this.call('login_user', credentials);
+      if (result && result.token) {
+        if (typeof localStorage !== 'undefined') {
+          localStorage.setItem(this.tokenName, result.token);
+        }
+        await this.authenticate(result.token);
+      }
+      return result;
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  async authenticate(token: string) {
+    return this.call('auth', { token });
+  }
+
+  async register(credentials: any, options: any = {}) {
+    try {
+      const params = { ...credentials, options };
+      const result = await this.call('register_user', params);
+      if (result && result.token) {
+        if (typeof localStorage !== 'undefined') {
+          localStorage.setItem(this.tokenName, result.token);
+        }
+      }
+      return result;
+    } catch (e) {
+      throw e;
+    }
+  }
+
+  async logout() {
+    if (typeof localStorage !== 'undefined') {
+      localStorage.removeItem(this.tokenName);
+    }
+    this.user = null;
+    this.ready = false;
+    try { await this.call('logout'); } catch(e) {}
+    this.ws?.close();
+  }
+
+  connect(url: string | null = null, timeout = 5000): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.ready = false;
+      this.user = null;
+
+      let wsUrl = url;
+      if (!wsUrl) {
+        if (typeof window !== "undefined") {
+          const protocol = window.location.protocol === "https:" ? "wss:" : "ws:";
+          wsUrl = protocol + "//" + window.location.host + "/ws";
+        } else {
+          wsUrl = "ws://localhost:3000/ws";
+        }
+      }
+
+      if (typeof localStorage !== 'undefined') {
+        const token = localStorage.getItem(this.tokenName);
+        if (token) {
+          if (wsUrl.includes('?')) wsUrl += '&token=' + encodeURIComponent(token);
+          else wsUrl += '?token=' + encodeURIComponent(token);
+        }
+      }
+
+      const connectionTimeout = setTimeout(() => {
+        if (this.ws) this.ws.close();
+        reject(new Error('WebSocket connection timed out after ' + timeout + 'ms'));
+      }, timeout);
+
+      this.ws = new WebSocket(wsUrl);
+
+      this.ws.onopen = () => {
+        clearTimeout(connectionTimeout);
+        console.log('[TZQL] Connected to ' + wsUrl);
+        this.reconnectAttempts = 0;
+        resolve();
+      };
+
+      this.ws.onmessage = (event) => {
+        try {
+          const message = JSON.parse(event.data);
+          this.handleMessage(message);
+        } catch (error) {
+          console.error("[TZQL] Failed to parse message:", error);
+        }
+      };
+
+      this.ws.onclose = () => {
+        console.log("[TZQL] Disconnected");
+        if (!this.isShuttingDown) {
+          this.attemptReconnect();
+        }
+      };
+
+      this.ws.onerror = (error) => {
+        clearTimeout(connectionTimeout);
+        console.error("[TZQL] Connection error:", error);
+        reject(error);
+      };
+    });
+  }
+
+  attemptReconnect() {
+    if (this.reconnectAttempts < this.maxReconnectAttempts) {
+      this.reconnectAttempts++;
+      const delay = 1000 * this.reconnectAttempts;
+      setTimeout(() => {
+        console.log('[TZQL] Reconnecting (' + this.reconnectAttempts + ')...');
+        this.connect();
+      }, delay);
+    }
+  }
+
+  handleMessage(message: any) {
+    // Handle connection:ready message
+    if (message.method === "connection:ready") {
+      this.user = message.params?.user || null;
+      this.ready = true;
+      this.readyCallbacks.forEach((cb) => cb(this.user));
+      return;
+    }
+
+    // Handle RPC responses (messages with id)
+    if (message.id && this.pendingRequests.has(message.id)) {
+      const resolver = this.pendingRequests.get(message.id);
+      if (resolver) {
+        this.pendingRequests.delete(message.id);
+        if (message.error) {
+          const err: any = new Error(message.error.message || 'Unknown error');
+          err.code = message.error.code;
+          resolver.reject(err);
+        } else {
+          resolver.resolve(message.result);
+        }
+      }
+      return;
+    }
+
+    // Handle subscription events - dispatch to registered subscription callbacks
+    if (message.method === "subscription:event") {
+      const event = message.params?.event;
+      if (event) {
+        // Dispatch to all subscription handlers - they filter by table/scope
+        for (const [subId, callback] of this.subscriptionCallbacks) {
+          callback(event);
+        }
+      }
+      return;
+    }
+
+    // Handle other server-initiated messages (broadcasts) - route to registered handlers
+    if (message.method) {
+      const handlers = this.methodHandlers.get(message.method);
+      if (handlers) {
+        handlers.forEach((cb) => cb(message.params));
+      }
+    }
+  }
+
+  call(method: string, params: any = {}) {
+    return new Promise((resolve, reject) => {
+      if (!this.ws || this.ws.readyState !== WebSocket.OPEN) {
+        reject(new Error("WebSocket not connected"));
+        return;
+      }
+      const id = ++this.messageId;
+      this.pendingRequests.set(id, { resolve, reject });
+      this.ws.send(JSON.stringify({ jsonrpc: "2.0", method, params, id }));
+    });
+  }
+
+  /**
+   * Register a callback for a server-initiated method
+   * @param method - The method name to listen for
+   * @param callback - Called with params when server sends this method
+   * @returns Unsubscribe function
+   */
+  on(method: string, callback: (params: any) => void) {
+    if (!this.methodHandlers.has(method)) {
+      this.methodHandlers.set(method, new Set());
+    }
+    this.methodHandlers.get(method)!.add(callback);
+    return () => {
+      const handlers = this.methodHandlers.get(method);
+      if (handlers) {
+        handlers.delete(callback);
+        if (handlers.size === 0) {
+          this.methodHandlers.delete(method);
+        }
+      }
+    };
+  }
+
+  /**
+   * Register a callback to be called when connection is ready
+   * @param callback - Called with user profile (or null if not authenticated)
+   * @returns Unsubscribe function
+   */
+  onReady(callback: (user: any) => void) {
+    if (this.ready) {
+      callback(this.user);
+    }
+    this.readyCallbacks.add(callback);
+    return () => this.readyCallbacks.delete(callback);
+  }
+
+  /**
+   * Subscribe to a subscribable document
+   * @param method - The subscribe method name (e.g., "subscribe_venue_detail")
+   * @param params - Subscription parameters
+   * @param callback - Called with initial data and on updates
+   * @returns Promise that resolves to an unsubscribe function
+   */
+  async subscribe(method: string, params: any, callback: (data: any) => void): Promise<() => void> {
+    // Call server to get initial snapshot and subscription_id
+    const result = await this.call(method, params) as {
+      subscription_id: string;
+      data: any;
+    };
+
+    // Register callback for subscription events
+    this.subscriptionCallbacks.set(result.subscription_id, callback);
+
+    // Call callback with initial data
+    callback(result.data);
+
+    // Return unsubscribe function
+    return () => {
+      this.subscriptionCallbacks.delete(result.subscription_id);
+      // Notify server
+      this.call(`unsubscribe_${method.replace('subscribe_', '')}`, { subscription_id: result.subscription_id }).catch(() => {});
+    };
+  }
+}

package/src/runtime/auth.ts

@@ -0,0 +1,39 @@
+import { jwtVerify, SignJWT } from "jose";
+
+const JWT_SECRET = process.env.JWT_SECRET || "default_dev_secret";
+const SECRET_KEY = new TextEncoder().encode(JWT_SECRET);
+
+export interface UserSession {
+  userId: number;
+  role?: string;
+  [key: string]: any;
+}
+
+export async function verifyToken(token: string): Promise<UserSession> {
+  try {
+    const { payload } = await jwtVerify(token, SECRET_KEY);
+    
+    // Normalize user_id vs sub
+    const uid = payload.user_id || payload.sub;
+    
+    if (!uid) {
+        throw new Error("Invalid token: missing user_id");
+    }
+    
+    return {
+      userId: Number(uid),
+      role: payload.role as string,
+      ...payload
+    };
+  } catch (err: any) {
+    throw new Error(`Authentication failed: ${err.message}`);
+  }
+}
+
+export async function signToken(payload: any): Promise<string> {
+    return await new SignJWT(payload)
+      .setProtectedHeader({ alg: 'HS256' })
+      .setIssuedAt()
+      .setExpirationTime('7d')
+      .sign(SECRET_KEY);
+}

package/src/runtime/db.ts

@@ -0,0 +1,33 @@
+import postgres from "postgres";
+
+export class Database {
+  sql: postgres.Sql;
+
+  constructor(connectionString: string, options: any = {}) {
+    this.sql = postgres(connectionString, {
+      max: options.max || 10,
+      idle_timeout: options.idleTimeout || 20,
+      connect_timeout: options.connectTimeout || 10,
+      onnotice: () => {}, // Suppress notices
+      ...options
+    });
+  }
+
+  async query(text: string, params: any[] = []) {
+    return this.sql.unsafe(text, params);
+  }
+
+  async listen(channel: string, callback: (payload: string) => void) {
+    console.log(`[DB] Setting up LISTEN on channel: ${channel}`);
+    const result = await this.sql.listen(channel, (payload) => {
+      console.log(`[DB] LISTEN callback triggered with payload:`, payload);
+      callback(payload);
+    });
+    console.log(`[DB] LISTEN setup complete, result:`, result);
+    return result;
+  }
+
+  async close() {
+    await this.sql.end();
+  }
+}

package/src/runtime/errors.ts

@@ -0,0 +1,51 @@
+export enum ErrorCode {
+  PERMISSION_DENIED = "PERMISSION_DENIED",
+  NOT_FOUND = "NOT_FOUND",
+  VALIDATION_ERROR = "VALIDATION_ERROR",
+  CONFLICT = "CONFLICT",
+  RATE_LIMITED = "RATE_LIMITED",
+  INTERNAL_ERROR = "INTERNAL_ERROR"
+}
+
+export class AppError extends Error {
+  code: ErrorCode;
+  constructor(code: ErrorCode, message?: string) {
+    super(message || code);
+    this.code = code;
+  }
+}
+
+export function mapDatabaseError(err: any): AppError {
+  // If it's already an AppError (e.g. thrown explicitly from PL/pgSQL RAISE), pass it through
+  // Postgres RAISE EXCEPTION '...' usually comes as a generic error with a message
+  
+  const code = err.code; // SQLSTATE
+  const message = err.message || "";
+
+  // 1. Explicit RAISE EXCEPTION from our PL/pgSQL functions
+  if (message.includes("permission_denied")) {
+    return new AppError(ErrorCode.PERMISSION_DENIED);
+  }
+  if (message.includes("not_found")) {
+    return new AppError(ErrorCode.NOT_FOUND);
+  }
+  if (message.includes("validation_error")) {
+    return new AppError(ErrorCode.VALIDATION_ERROR, message);
+  }
+
+  // 2. Standard SQLSTATE Mapping
+  switch (code) {
+    case "23505": // Unique violation
+      return new AppError(ErrorCode.CONFLICT, "Unique constraint violation");
+    case "23503": // Foreign key violation
+      return new AppError(ErrorCode.VALIDATION_ERROR, "Invalid reference");
+    case "23502": // Not null violation
+      return new AppError(ErrorCode.VALIDATION_ERROR, "Missing required field");
+    case "42P01": // Undefined table (shouldn't happen in prod if manifest is correct)
+      return new AppError(ErrorCode.INTERNAL_ERROR, "Table not found");
+    default:
+      // Log the real error internally, return generic to client
+      // console.error(err); // Done in server.ts
+      return new AppError(ErrorCode.INTERNAL_ERROR);
+  }
+}

package/src/runtime/index.ts

@@ -0,0 +1,98 @@
+import { serve } from "bun";
+import { config } from "dotenv";
+import { Database } from "./db.js";
+import { WebSocketServer } from "./ws.js";
+import { loadManifest } from "./manifest_loader.js";
+import { readFileSync } from "fs";
+import { resolve } from "path";
+
+// Re-export JS function registration API for custom functions
+export { registerJsFunction, type JsFunctionHandler, type JsFunctionContext } from "./js_functions.js";
+
+// Load .env file if present
+config();
+
+// Configuration
+const PORT = process.env.PORT || 3000;
+const DB_URL = process.env.DATABASE_URL || "postgres://dzql_test:dzql_test@localhost:5433/dzql_test";
+const MANIFEST_PATH = process.env.MANIFEST_PATH || "./dist/runtime/manifest.json";
+
+// 1. Initialize DB
+const db = new Database(DB_URL);
+
+// 2. Load Manifest
+try {
+  const manifestPath = resolve(process.cwd(), MANIFEST_PATH);
+  const manifestContent = readFileSync(manifestPath, "utf-8");
+  const manifest = JSON.parse(manifestContent);
+  loadManifest(manifest);
+  console.log(`[Runtime] Loaded Manifest v${manifest.version}`);
+} catch (e) {
+  console.warn(`[Runtime] Warning: Could not load manifest from ${MANIFEST_PATH}. Ensure you have compiled the project.`);
+}
+
+// 3. Initialize WebSocket Server
+const wsServer = new WebSocketServer(db);
+
+// 4. Start Commit Listener (Realtime)
+async function startListener() {
+  console.log("[Runtime] Setting up LISTEN on dzql_v2 channel...");
+  await db.listen("dzql_v2", async (payload) => {
+    console.log(`[Runtime] RAW NOTIFY received:`, payload);
+    try {
+      const { commit_id } = JSON.parse(payload);
+      console.log(`[Runtime] Received Commit: ${commit_id}`);
+
+      // Fetch events
+      const events = await db.query(`
+        SELECT * FROM dzql_v2.events
+        WHERE commit_id = $1
+        ORDER BY id ASC
+      `, [commit_id]);
+
+      for (const event of events) {
+        // Broadcast
+        const message = JSON.stringify({
+          jsonrpc: "2.0",
+          method: "subscription:event",
+          params: {
+            event: {
+              table: event.table_name,
+              op: event.op,
+              pk: event.pk,
+              data: event.data,
+              // old_data is filtered out
+              user_id: event.user_id
+            }
+          }
+        });
+        wsServer.broadcast(message);
+      }
+    } catch (e) {
+      console.error("[Runtime] Listener Error:", e);
+    }
+  });
+  console.log("[Runtime] Listening for DB Events...");
+}
+
+// Wait for listener to be ready before starting server
+await startListener();
+
+// 5. Start Web Server
+const server = serve({
+  port: PORT,
+  async fetch(req, server) {
+    const url = new URL(req.url);
+
+    // Extract token from query params for WebSocket connections
+    const token = url.searchParams.get("token");
+
+    if (server.upgrade(req, { data: { token } })) {
+        return;
+    }
+    return new Response("TZQL Runtime Active", { status: 200 });
+  },
+  websocket: wsServer.handlers
+});
+
+console.log(`[Runtime] Server listening on port ${PORT}`);