dzql 0.1.6 → 0.2.1

package/src/compiler/parser/subscribable-parser.js

@@ -0,0 +1,242 @@
+/**
+ * Subscribable Definition Parser
+ * Parses register_subscribable() calls and extracts configuration
+ */
+
+export class SubscribableParser {
+  /**
+   * Parse a dzql.register_subscribable() call from SQL
+   * @param {string} sql - SQL containing register_subscribable call
+   * @returns {Object} Parsed subscribable configuration
+   */
+  parseFromSQL(sql) {
+    // Extract the register_subscribable call
+    const registerMatch = sql.match(/dzql\.register_subscribable\s*\(([\s\S]*?)\);/i);
+    if (!registerMatch) {
+      throw new Error('No register_subscribable call found in SQL');
+    }
+
+    const params = this._parseParameters(registerMatch[1]);
+
+    return this._buildSubscribableConfig(params);
+  }
+
+  /**
+   * Parse parameters from register_subscribable call
+   * @private
+   */
+  _parseParameters(paramsString) {
+    // Split by commas that are not inside quotes, parentheses, or brackets
+    const params = [];
+    let currentParam = '';
+    let depth = 0;
+    let inString = false;
+    let stringChar = null;
+
+    for (let i = 0; i < paramsString.length; i++) {
+      const char = paramsString[i];
+      const prevChar = i > 0 ? paramsString[i - 1] : '';
+
+      if ((char === "'" || char === '"') && prevChar !== '\\') {
+        if (!inString) {
+          inString = true;
+          stringChar = char;
+        } else if (char === stringChar) {
+          inString = false;
+          stringChar = null;
+        }
+      }
+
+      if (!inString) {
+        if (char === '(' || char === '{' || char === '[') depth++;
+        if (char === ')' || char === '}' || char === ']') depth--;
+
+        if (char === ',' && depth === 0) {
+          params.push(currentParam.trim());
+          currentParam = '';
+          continue;
+        }
+      }
+
+      currentParam += char;
+    }
+
+    if (currentParam.trim()) {
+      params.push(currentParam.trim());
+    }
+
+    return params;
+  }
+
+  /**
+   * Build subscribable configuration from parsed parameters
+   * register_subscribable(name, permission_paths, param_schema, root_entity, relations)
+   * @private
+   */
+  _buildSubscribableConfig(params) {
+    const config = {
+      name: this._cleanString(params[0]),
+      permissionPaths: params[1] ? this._parseJSON(params[1]) : {},
+      paramSchema: params[2] ? this._parseJSON(params[2]) : {},
+      rootEntity: this._cleanString(params[3]),
+      relations: params[4] ? this._parseJSON(params[4]) : {}
+    };
+
+    return config;
+  }
+
+  /**
+   * Parse from JavaScript object (for programmatic usage)
+   * @param {Object} obj - Subscribable configuration object
+   * @returns {Object} Normalized configuration
+   */
+  parseFromObject(obj) {
+    return {
+      name: obj.name,
+      permissionPaths: obj.permissionPaths || obj.permissions || {},
+      paramSchema: obj.paramSchema || obj.params || {},
+      rootEntity: obj.rootEntity || obj.root,
+      relations: obj.relations || {}
+    };
+  }
+
+  /**
+   * Parse multiple subscribables from SQL file
+   * @param {string} sql - SQL file content
+   * @returns {Array} Array of subscribable configurations
+   */
+  parseAllFromSQL(sql) {
+    const subscribables = [];
+    const regex = /dzql\.register_subscribable\s*\(([\s\S]*?)\);/gi;
+    let match;
+
+    while ((match = regex.exec(sql)) !== null) {
+      try {
+        const params = this._parseParameters(match[1]);
+        const config = this._buildSubscribableConfig(params);
+        subscribables.push(config);
+      } catch (error) {
+        console.error('Failed to parse subscribable:', error.message);
+      }
+    }
+
+    return subscribables;
+  }
+
+  /**
+   * Clean a string parameter (remove quotes)
+   * @private
+   */
+  _cleanString(str) {
+    if (!str) return '';
+    // Remove outer quotes, SQL comments, then any remaining quotes and whitespace
+    let cleaned = str.replace(/^['"]|['"]$/g, '');  // Remove outer quotes
+    cleaned = cleaned.replace(/--[^\n]*/g, '');     // Remove SQL comments
+    cleaned = cleaned.replace(/['"\s]+$/g, '');     // Remove trailing quotes/whitespace
+    return cleaned.trim();
+  }
+
+  /**
+   * Parse JSONB object parameter
+   * @private
+   */
+  _parseJSON(str) {
+    if (!str || str === '{}' || str === "'{}'::jsonb") {
+      return {};
+    }
+
+    // Handle jsonb_build_object() syntax
+    if (str.includes('jsonb_build_object')) {
+      return this._parseJSONBuildObject(str);
+    }
+
+    // Handle plain JSON string
+    try {
+      // Remove ::jsonb cast
+      let cleaned = str.replace(/::jsonb$/i, '');
+      // Remove outer quotes if it's a string literal
+      cleaned = cleaned.replace(/^'(.*)'$/, '$1');
+      // Unescape internal quotes
+      cleaned = cleaned.replace(/''/g, "'");
+
+      return JSON.parse(cleaned);
+    } catch (error) {
+      console.error('Failed to parse JSON:', str, error);
+      return {};
+    }
+  }
+
+  /**
+   * Parse jsonb_build_object() calls
+   * @private
+   */
+  _parseJSONBuildObject(str) {
+    const result = {};
+
+    // Extract content between jsonb_build_object( and )
+    const match = str.match(/jsonb_build_object\s*\(([\s\S]*)\)/i);
+    if (!match) return result;
+
+    // Parse key-value pairs
+    const params = this._parseParameters(match[1]);
+
+    for (let i = 0; i < params.length; i += 2) {
+      if (i + 1 < params.length) {
+        const key = this._cleanString(params[i]);
+        let value = params[i + 1];
+
+        // Check if value is nested jsonb_build_object or array
+        if (value.includes('jsonb_build_object')) {
+          value = this._parseJSONBuildObject(value);
+        } else if (value.includes('jsonb_build_array')) {
+          value = this._parseJSONBArray(value);
+        } else if (value.includes('ARRAY[')) {
+          value = this._parseArray(value);
+        } else {
+          value = this._cleanString(value);
+        }
+
+        result[key] = value;
+      }
+    }
+
+    return result;
+  }
+
+  /**
+   * Parse jsonb_build_array() calls
+   * @private
+   */
+  _parseJSONBArray(str) {
+    const match = str.match(/jsonb_build_array\s*\(([\s\S]*)\)/i);
+    if (!match) return [];
+
+    const params = this._parseParameters(match[1]);
+    return params.map(p => {
+      if (p.includes('jsonb_build_object')) {
+        return this._parseJSONBuildObject(p);
+      }
+      return this._cleanString(p);
+    });
+  }
+
+  /**
+   * Parse ARRAY[...] syntax
+   * @private
+   */
+  _parseArray(str) {
+    if (!str || str === 'ARRAY[]::text[]') {
+      return [];
+    }
+
+    // Extract content between ARRAY[ and ]
+    const match = str.match(/ARRAY\[(.*?)\]/i);
+    if (!match) return [];
+
+    // Split by comma and clean each element
+    return match[1]
+      .split(',')
+      .map(s => this._cleanString(s))
+      .filter(s => s.length > 0);
+  }
+}

package/src/database/migrations/009_subscriptions.sql

@@ -0,0 +1,230 @@
+-- Migration 009: Live Query Subscriptions Infrastructure
+-- Adds support for subscribable documents (Pattern 1 from vision.md)
+
+-- ============================================================================
+-- Subscribables Registry (Metadata Only)
+-- ============================================================================
+
+-- Stores metadata for registered subscribables
+-- Note: Active subscriptions are held in-memory on the server
+CREATE TABLE IF NOT EXISTS dzql.subscribables (
+  name TEXT PRIMARY KEY,
+  permission_paths JSONB NOT NULL DEFAULT '{}'::jsonb,
+  param_schema JSONB NOT NULL DEFAULT '{}'::jsonb,
+  root_entity TEXT NOT NULL,
+  relations JSONB NOT NULL DEFAULT '{}'::jsonb,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+COMMENT ON TABLE dzql.subscribables IS
+  'Registry of subscribable documents. Subscribables define denormalized views that clients can subscribe to for real-time updates.';
+
+COMMENT ON COLUMN dzql.subscribables.name IS
+  'Subscribable identifier (used in subscribe_<name> RPC calls)';
+
+COMMENT ON COLUMN dzql.subscribables.permission_paths IS
+  'Access control paths (e.g., {"subscribe": ["@org_id->acts_for[org_id=$]{active}.user_id"]})';
+
+COMMENT ON COLUMN dzql.subscribables.param_schema IS
+  'Parameter schema defining subscription key (e.g., {"venue_id": "int"})';
+
+COMMENT ON COLUMN dzql.subscribables.root_entity IS
+  'Root table for the subscribable document';
+
+COMMENT ON COLUMN dzql.subscribables.relations IS
+  'Related entities to include (e.g., {"org": "organisations", "sites": {"entity": "sites", "filter": "venue_id=$venue_id"}})';
+
+-- Index for quick lookups
+CREATE INDEX IF NOT EXISTS idx_subscribables_root_entity
+  ON dzql.subscribables(root_entity);
+
+-- ============================================================================
+-- Register Subscribable Function
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION dzql.register_subscribable(
+  p_name TEXT,
+  p_permission_paths JSONB,
+  p_param_schema JSONB,
+  p_root_entity TEXT,
+  p_relations JSONB
+) RETURNS TEXT AS $$
+DECLARE
+  v_result TEXT;
+BEGIN
+  -- Validate inputs
+  IF p_name IS NULL OR p_name = '' THEN
+    RAISE EXCEPTION 'Subscribable name cannot be empty';
+  END IF;
+
+  IF p_root_entity IS NULL OR p_root_entity = '' THEN
+    RAISE EXCEPTION 'Root entity cannot be empty';
+  END IF;
+
+  -- Insert or update subscribable
+  INSERT INTO dzql.subscribables (
+    name,
+    permission_paths,
+    param_schema,
+    root_entity,
+    relations,
+    created_at,
+    updated_at
+  ) VALUES (
+    p_name,
+    COALESCE(p_permission_paths, '{}'::jsonb),
+    COALESCE(p_param_schema, '{}'::jsonb),
+    p_root_entity,
+    COALESCE(p_relations, '{}'::jsonb),
+    NOW(),
+    NOW()
+  )
+  ON CONFLICT (name) DO UPDATE SET
+    permission_paths = EXCLUDED.permission_paths,
+    param_schema = EXCLUDED.param_schema,
+    root_entity = EXCLUDED.root_entity,
+    relations = EXCLUDED.relations,
+    updated_at = NOW();
+
+  v_result := format('Subscribable "%s" registered successfully', p_name);
+
+  RAISE NOTICE '%', v_result;
+
+  RETURN v_result;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION dzql.register_subscribable IS
+  'Register a subscribable document definition. Used by the compiler to store subscribable metadata.';
+
+-- ============================================================================
+-- Helper Functions
+-- ============================================================================
+
+-- Get all registered subscribables
+CREATE OR REPLACE FUNCTION dzql.get_subscribables()
+RETURNS TABLE (
+  name TEXT,
+  permission_paths JSONB,
+  param_schema JSONB,
+  root_entity TEXT,
+  relations JSONB,
+  created_at TIMESTAMPTZ,
+  updated_at TIMESTAMPTZ
+) AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    s.name,
+    s.permission_paths,
+    s.param_schema,
+    s.root_entity,
+    s.relations,
+    s.created_at,
+    s.updated_at
+  FROM dzql.subscribables s
+  ORDER BY s.name;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION dzql.get_subscribables IS
+  'Retrieve all registered subscribables';
+
+-- Get subscribable by name
+CREATE OR REPLACE FUNCTION dzql.get_subscribable(p_name TEXT)
+RETURNS TABLE (
+  name TEXT,
+  permission_paths JSONB,
+  param_schema JSONB,
+  root_entity TEXT,
+  relations JSONB,
+  created_at TIMESTAMPTZ,
+  updated_at TIMESTAMPTZ
+) AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    s.name,
+    s.permission_paths,
+    s.param_schema,
+    s.root_entity,
+    s.relations,
+    s.created_at,
+    s.updated_at
+  FROM dzql.subscribables s
+  WHERE s.name = p_name;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION dzql.get_subscribable IS
+  'Retrieve a specific subscribable by name';
+
+-- Get subscribables by root entity
+CREATE OR REPLACE FUNCTION dzql.get_subscribables_by_entity(p_entity TEXT)
+RETURNS TABLE (
+  name TEXT,
+  permission_paths JSONB,
+  param_schema JSONB,
+  root_entity TEXT,
+  relations JSONB,
+  created_at TIMESTAMPTZ,
+  updated_at TIMESTAMPTZ
+) AS $$
+BEGIN
+  RETURN QUERY
+  SELECT
+    s.name,
+    s.permission_paths,
+    s.param_schema,
+    s.root_entity,
+    s.relations,
+    s.created_at,
+    s.updated_at
+  FROM dzql.subscribables s
+  WHERE s.root_entity = p_entity
+  ORDER BY s.name;
+END;
+$$ LANGUAGE plpgsql STABLE;
+
+COMMENT ON FUNCTION dzql.get_subscribables_by_entity IS
+  'Retrieve all subscribables for a given root entity';
+
+-- Delete subscribable
+CREATE OR REPLACE FUNCTION dzql.delete_subscribable(p_name TEXT)
+RETURNS TEXT AS $$
+DECLARE
+  v_result TEXT;
+BEGIN
+  DELETE FROM dzql.subscribables
+  WHERE name = p_name;
+
+  IF FOUND THEN
+    v_result := format('Subscribable "%s" deleted successfully', p_name);
+  ELSE
+    v_result := format('Subscribable "%s" not found', p_name);
+  END IF;
+
+  RAISE NOTICE '%', v_result;
+
+  RETURN v_result;
+END;
+$$ LANGUAGE plpgsql;
+
+COMMENT ON FUNCTION dzql.delete_subscribable IS
+  'Delete a subscribable by name';
+
+-- ============================================================================
+-- Verification
+-- ============================================================================
+
+DO $$
+BEGIN
+  RAISE NOTICE 'Migration 009: Live Query Subscriptions - Complete';
+  RAISE NOTICE 'Subscribables table created';
+  RAISE NOTICE 'Helper functions installed';
+  RAISE NOTICE '';
+  RAISE NOTICE 'Usage:';
+  RAISE NOTICE '  SELECT dzql.register_subscribable(''name'', permissions, params, root, relations);';
+  RAISE NOTICE '  SELECT * FROM dzql.get_subscribables();';
+END $$;

package/src/server/index.js

@@ -2,12 +2,96 @@ import { createWebSocketHandlers, verify_jwt_token } from "./ws.js";
 import { closeConnections, setupListeners, sql, db } from "./db.js";
 import * as defaultApi from "./api.js";
 import { serverLogger, notifyLogger } from "./logger.js";
+import { getSubscriptionsBySubscribable, paramsMatch } from "./subscriptions.js";
 
 // Re-export commonly used utilities
 export { sql, db } from "./db.js";
 export { metaRoute } from "./meta-route.js";
 export { createMCPRoute } from "./mcp.js";
 
+/**
+ * Process subscription updates when a database event occurs
+ * Checks if any active subscriptions are affected and sends updates
+ * @param {Object} event - Database event {table, op, pk, before, after}
+ * @param {Function} broadcast - Broadcast function from WebSocket handlers
+ */
+async function processSubscriptionUpdates(event, broadcast) {
+  const { table, op, before, after } = event;
+
+  // Get all active subscriptions grouped by subscribable
+  const subscriptionsByName = getSubscriptionsBySubscribable();
+
+  if (subscriptionsByName.size === 0) {
+    return; // No active subscriptions
+  }
+
+  notifyLogger.debug(`Checking ${subscriptionsByName.size} subscribable(s) for affected subscriptions`);
+
+  // For each unique subscribable, check if this event affects any subscriptions
+  for (const [subscribableName, subs] of subscriptionsByName.entries()) {
+    try {
+      // Ask PostgreSQL which subscription instances are affected
+      const result = await db.query(
+        `SELECT ${subscribableName}_affected_documents($1, $2, $3, $4) as affected`,
+        [table, op, before, after]
+      );
+
+      const affectedParamSets = result.rows[0]?.affected;
+
+      if (!affectedParamSets || affectedParamSets.length === 0) {
+        continue; // This subscribable not affected
+      }
+
+      notifyLogger.debug(`${subscribableName}: ${affectedParamSets.length} param set(s) affected`);
+
+      // Match affected params to active subscriptions
+      for (const affectedParams of affectedParamSets) {
+        for (const sub of subs) {
+          // Check if this subscription matches the affected params
+          if (paramsMatch(sub.params, affectedParams)) {
+            try {
+              // Re-execute query to get updated data
+              const updated = await db.query(
+                `SELECT get_${subscribableName}($1, $2) as data`,
+                [sub.params, sub.user_id]
+              );
+
+              const data = updated.rows[0]?.data;
+
+              // Send update to specific connection
+              const message = JSON.stringify({
+                jsonrpc: "2.0",
+                method: "subscription:update",
+                params: {
+                  subscription_id: sub.subscriptionId,
+                  subscribable: subscribableName,
+                  data
+                }
+              });
+
+              const sent = broadcast.toConnection(sub.connection_id, message);
+              if (sent) {
+                notifyLogger.debug(`Sent update to subscription ${sub.subscriptionId.slice(0, 8)}...`);
+              } else {
+                notifyLogger.warn(`Failed to send update to connection ${sub.connection_id.slice(0, 8)}...`);
+              }
+            } catch (error) {
+              notifyLogger.error(`Failed to update subscription ${sub.subscriptionId}:`, error.message);
+            }
+          }
+        }
+      }
+    } catch (error) {
+      // If the subscribable function doesn't exist, just skip
+      if (error.message && error.message.includes('does not exist')) {
+        notifyLogger.debug(`Subscribable ${subscribableName} functions not found, skipping`);
+      } else {
+        notifyLogger.error(`Error processing subscriptions for ${subscribableName}:`, error.message);
+      }
+    }
+  }
+}
+
 /**
  * Create a DZQL server with WebSocket support, real-time updates, and automatic CRUD operations
  *
@@ -97,10 +181,11 @@ export function createServer(options = {}) {
   });
 
   // Setup NOTIFY listeners for real-time events
-  setupListeners((event) => {
+  setupListeners(async (event) => {
     // Handle single dzql event with filtering
     const { notify_users, ...eventData } = event;
 
+    // PATTERN 2: Need to Know notifications (existing)
     // Create JSON-RPC notification
     const message = JSON.stringify({
       jsonrpc: "2.0",
@@ -118,6 +203,10 @@ export function createServer(options = {}) {
       notifyLogger.debug(`Broadcasting ${event.table}:${event.op} to all users`);
       broadcast(message);
     }
+
+    // PATTERN 1: Live Query subscriptions (new)
+    // Check if any subscriptions are affected by this event
+    await processSubscriptionUpdates(event, broadcast);
   });
 
   routes['/health'] = () => new Response("OK", { status: 200 });