dzql 0.5.5 → 0.5.7

package/src/database/migrations/009_subscriptions.sql

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS dzql.subscribables (
   param_schema JSONB NOT NULL DEFAULT '{}'::jsonb,
   root_entity TEXT NOT NULL,
   relations JSONB NOT NULL DEFAULT '{}'::jsonb,
+  scope_tables TEXT[] NOT NULL DEFAULT '{}',
   created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
   updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
 );
@@ -35,6 +36,9 @@ COMMENT ON COLUMN dzql.subscribables.root_entity IS
 COMMENT ON COLUMN dzql.subscribables.relations IS
   'Related entities to include (e.g., {"org": "organisations", "sites": {"entity": "sites", "filter": "venue_id=$venue_id"}})';
 
+COMMENT ON COLUMN dzql.subscribables.scope_tables IS
+  'Array of table names that are in scope for this subscribable (root + all relations). Used for efficient event filtering.';
+
 -- Index for quick lookups
 CREATE INDEX IF NOT EXISTS idx_subscribables_root_entity
   ON dzql.subscribables(root_entity);
@@ -110,6 +114,7 @@ RETURNS TABLE (
   param_schema JSONB,
   root_entity TEXT,
   relations JSONB,
+  scope_tables TEXT[],
   created_at TIMESTAMPTZ,
   updated_at TIMESTAMPTZ
 ) AS $$
@@ -121,6 +126,7 @@ BEGIN
     s.param_schema,
     s.root_entity,
     s.relations,
+    s.scope_tables,
     s.created_at,
     s.updated_at
   FROM dzql.subscribables s
@@ -139,6 +145,7 @@ RETURNS TABLE (
   param_schema JSONB,
   root_entity TEXT,
   relations JSONB,
+  scope_tables TEXT[],
   created_at TIMESTAMPTZ,
   updated_at TIMESTAMPTZ
 ) AS $$
@@ -150,6 +157,7 @@ BEGIN
     s.param_schema,
     s.root_entity,
     s.relations,
+    s.scope_tables,
     s.created_at,
     s.updated_at
   FROM dzql.subscribables s
@@ -168,6 +176,7 @@ RETURNS TABLE (
   param_schema JSONB,
   root_entity TEXT,
   relations JSONB,
+  scope_tables TEXT[],
   created_at TIMESTAMPTZ,
   updated_at TIMESTAMPTZ
 ) AS $$
@@ -179,6 +188,7 @@ BEGIN
     s.param_schema,
     s.root_entity,
     s.relations,
+    s.scope_tables,
     s.created_at,
     s.updated_at
   FROM dzql.subscribables s

package/src/database/migrations/010_atomic_updates.sql

@@ -0,0 +1,150 @@
+-- Migration 010: Atomic Updates for Subscribables
+-- Adds extract_scope_tables function and updates register_subscribable to auto-populate scope_tables
+
+-- ============================================================================
+-- Helper function to extract scope tables from relations
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION dzql.extract_scope_tables(
+  p_root_entity TEXT,
+  p_relations JSONB
+) RETURNS TEXT[] AS $$
+DECLARE
+  v_tables TEXT[];
+  v_key TEXT;
+  v_value JSONB;
+  v_entity TEXT;
+  v_nested JSONB;
+BEGIN
+  -- Start with root entity
+  v_tables := ARRAY[p_root_entity];
+
+  -- Return early if no relations
+  IF p_relations IS NULL OR p_relations = '{}'::jsonb THEN
+    RETURN v_tables;
+  END IF;
+
+  -- Iterate through relations
+  FOR v_key, v_value IN SELECT * FROM jsonb_each(p_relations)
+  LOOP
+    -- Handle string relation (simple FK expansion): "org": "organisations"
+    IF jsonb_typeof(v_value) = 'string' THEN
+      v_entity := v_value #>> '{}';
+      IF v_entity IS NOT NULL AND v_entity != '' THEN
+        v_tables := array_append(v_tables, v_entity);
+      END IF;
+    -- Handle object relation: {"entity": "sites", "filter": "..."}
+    ELSIF jsonb_typeof(v_value) = 'object' THEN
+      v_entity := v_value ->> 'entity';
+      IF v_entity IS NOT NULL AND v_entity != '' THEN
+        v_tables := array_append(v_tables, v_entity);
+      END IF;
+
+      -- Recursively handle nested relations (include or relations)
+      v_nested := v_value -> 'include';
+      IF v_nested IS NOT NULL AND jsonb_typeof(v_nested) = 'object' THEN
+        v_tables := v_tables || dzql.extract_scope_tables(NULL, v_nested);
+      END IF;
+
+      v_nested := v_value -> 'relations';
+      IF v_nested IS NOT NULL AND jsonb_typeof(v_nested) = 'object' THEN
+        v_tables := v_tables || dzql.extract_scope_tables(NULL, v_nested);
+      END IF;
+    END IF;
+  END LOOP;
+
+  -- Remove duplicates and nulls
+  SELECT array_agg(DISTINCT t) INTO v_tables
+  FROM unnest(v_tables) t
+  WHERE t IS NOT NULL;
+
+  RETURN COALESCE(v_tables, ARRAY[]::TEXT[]);
+END;
+$$ LANGUAGE plpgsql IMMUTABLE;
+
+COMMENT ON FUNCTION dzql.extract_scope_tables IS
+  'Extract all table names from a subscribable definition (root entity + all relations recursively)';
+
+-- ============================================================================
+-- Update register_subscribable to auto-populate scope_tables
+-- ============================================================================
+
+CREATE OR REPLACE FUNCTION dzql.register_subscribable(
+  p_name TEXT,
+  p_permission_paths JSONB,
+  p_param_schema JSONB,
+  p_root_entity TEXT,
+  p_relations JSONB
+) RETURNS TEXT AS $$
+DECLARE
+  v_result TEXT;
+  v_scope_tables TEXT[];
+BEGIN
+  -- Validate inputs
+  IF p_name IS NULL OR p_name = '' THEN
+    RAISE EXCEPTION 'Subscribable name cannot be empty';
+  END IF;
+
+  IF p_root_entity IS NULL OR p_root_entity = '' THEN
+    RAISE EXCEPTION 'Root entity cannot be empty';
+  END IF;
+
+  -- Extract scope tables from root entity and relations
+  v_scope_tables := dzql.extract_scope_tables(p_root_entity, p_relations);
+
+  -- Insert or update subscribable
+  INSERT INTO dzql.subscribables (
+    name,
+    permission_paths,
+    param_schema,
+    root_entity,
+    relations,
+    scope_tables,
+    created_at,
+    updated_at
+  ) VALUES (
+    p_name,
+    COALESCE(p_permission_paths, '{}'::jsonb),
+    COALESCE(p_param_schema, '{}'::jsonb),
+    p_root_entity,
+    COALESCE(p_relations, '{}'::jsonb),
+    v_scope_tables,
+    NOW(),
+    NOW()
+  )
+  ON CONFLICT (name) DO UPDATE SET
+    permission_paths = EXCLUDED.permission_paths,
+    param_schema = EXCLUDED.param_schema,
+    root_entity = EXCLUDED.root_entity,
+    relations = EXCLUDED.relations,
+    scope_tables = EXCLUDED.scope_tables,
+    updated_at = NOW();
+
+  v_result := format('Subscribable "%s" registered successfully with scope tables: %s',
+                     p_name, array_to_string(v_scope_tables, ', '));
+
+  RAISE NOTICE '%', v_result;
+
+  RETURN v_result;
+END;
+$$ LANGUAGE plpgsql;
+
+-- ============================================================================
+-- Backfill existing subscribables with scope_tables
+-- ============================================================================
+
+UPDATE dzql.subscribables s
+SET scope_tables = dzql.extract_scope_tables(s.root_entity, s.relations)
+WHERE scope_tables = '{}' OR scope_tables IS NULL;
+
+-- ============================================================================
+-- Verification
+-- ============================================================================
+
+DO $$
+BEGIN
+  RAISE NOTICE 'Migration 010: Atomic Updates - Complete';
+  RAISE NOTICE 'Created dzql.extract_scope_tables() function';
+  RAISE NOTICE 'Updated dzql.register_subscribable() to auto-populate scope_tables';
+  RAISE NOTICE 'Backfilled existing subscribables';
+END $$;

package/src/server/index.js

@@ -2,7 +2,7 @@ import { createWebSocketHandlers, verify_jwt_token } from "./ws.js";
 import { closeConnections, setupListeners, sql, db } from "./db.js";
 import * as defaultApi from "./api.js";
 import { serverLogger, notifyLogger } from "./logger.js";
-import { getSubscriptionsBySubscribable, paramsMatch } from "./subscriptions.js";
+import { getSubscriptionsBySubscribable, paramsMatch, getSubscribableScopeTables } from "./subscriptions.js";
 
 // Re-export commonly used utilities
 export { sql, db } from "./db.js";
@@ -11,12 +11,12 @@ export { createMCPRoute } from "./mcp.js";
 
 /**
  * Process subscription updates when a database event occurs
- * Checks if any active subscriptions are affected and sends updates
+ * Forwards atomic events to affected subscriptions for client-side patching
  * @param {Object} event - Database event {table, op, pk, before, after}
  * @param {Function} broadcast - Broadcast function from WebSocket handlers
  */
 async function processSubscriptionUpdates(event, broadcast) {
-  const { table, op, before, after } = event;
+  const { table, op, pk, before, after } = event;
 
   // Get all active subscriptions grouped by subscribable
   const subscriptionsByName = getSubscriptionsBySubscribable();
@@ -30,6 +30,14 @@ async function processSubscriptionUpdates(event, broadcast) {
   // For each unique subscribable, check if this event affects any subscriptions
   for (const [subscribableName, subs] of subscriptionsByName.entries()) {
     try {
+      // Check if this table is in scope for this subscribable
+      // This is an optimization to avoid calling _affected_documents for unrelated tables
+      const scopeTables = await getSubscribableScopeTables(subscribableName, sql);
+      if (scopeTables.length > 0 && !scopeTables.includes(table)) {
+        notifyLogger.debug(`Table ${table} not in scope for ${subscribableName}, skipping`);
+        continue;
+      }
+
       // Ask PostgreSQL which subscription instances are affected
       const result = await sql.unsafe(
         `SELECT ${subscribableName}_affected_documents($1, $2, $3, $4) as affected`,
@@ -42,7 +50,7 @@ async function processSubscriptionUpdates(event, broadcast) {
         continue; // This subscribable not affected
       }
 
-      notifyLogger.debug(`${subscribableName}: ${affectedParamSets.length} param set(s) affected`);
+      notifyLogger.debug(`${subscribableName}: ${affectedParamSets.length} param set(s) affected by ${table}:${op}`);
 
       // Match affected params to active subscriptions
       for (const affectedParams of affectedParamSets) {
@@ -50,33 +58,32 @@ async function processSubscriptionUpdates(event, broadcast) {
           // Check if this subscription matches the affected params
           if (paramsMatch(sub.params, affectedParams)) {
             try {
-              // Re-execute query to get updated data
-              const updated = await sql.unsafe(
-                `SELECT get_${subscribableName}($1, $2) as data`,
-                [sub.params, sub.user_id]
-              );
-
-              const data = updated[0]?.data;
-
-              // Send update to specific connection
+              // Forward atomic event instead of re-querying the full document
+              // Client will apply the patch to their local copy
               const message = JSON.stringify({
                 jsonrpc: "2.0",
-                method: "subscription:update",
+                method: "subscription:event",
                 params: {
                   subscription_id: sub.subscriptionId,
                   subscribable: subscribableName,
-                  data
+                  event: {
+                    table,
+                    op,
+                    pk,
+                    data: after,
+                    before
+                  }
                 }
               });
 
               const sent = broadcast.toConnection(sub.connection_id, message);
               if (sent) {
-                notifyLogger.debug(`Sent update to subscription ${sub.subscriptionId.slice(0, 8)}...`);
+                notifyLogger.debug(`Sent atomic event to subscription ${sub.subscriptionId.slice(0, 8)}... (${table}:${op})`);
               } else {
-                notifyLogger.warn(`Failed to send update to connection ${sub.connection_id.slice(0, 8)}...`);
+                notifyLogger.warn(`Failed to send event to connection ${sub.connection_id.slice(0, 8)}...`);
               }
             } catch (error) {
-              notifyLogger.error(`Failed to update subscription ${sub.subscriptionId}:`, error.message);
+              notifyLogger.error(`Failed to send event to subscription ${sub.subscriptionId}:`, error.message);
             }
           }
         }

package/src/server/subscriptions.js

@@ -18,6 +18,12 @@ const subscriptions = new Map();
  */
 const connectionSubscriptions = new Map();
 
+/**
+ * Cache for subscribable metadata (scope tables, path mappings)
+ * Structure: subscribable_name -> { scopeTables, pathMapping, rootEntity, relations }
+ */
+const subscribableMetadataCache = new Map();
+
 /**
  * Register a new subscription
  * @param {string} subscribableName - Name of the subscribable
@@ -207,3 +213,122 @@ export function getAllSubscriptions() {
     ...sub
   }));
 }
+
+/**
+ * Get subscribable metadata (scope tables, path mapping) with caching
+ * @param {string} subscribableName - Name of the subscribable
+ * @param {function} sql - Database query function
+ * @returns {Promise<{scopeTables: string[], pathMapping: object, rootEntity: string, relations: object}>}
+ */
+export async function getSubscribableMetadata(subscribableName, sql) {
+  // Check cache first
+  if (subscribableMetadataCache.has(subscribableName)) {
+    return subscribableMetadataCache.get(subscribableName);
+  }
+
+  // Fetch from database
+  const result = await sql`
+    SELECT scope_tables, root_entity, relations
+    FROM dzql.subscribables
+    WHERE name = ${subscribableName}
+  `;
+
+  if (!result || result.length === 0) {
+    // Return empty metadata if subscribable not found
+    const emptyMetadata = {
+      scopeTables: [],
+      pathMapping: {},
+      rootEntity: null,
+      relations: {}
+    };
+    return emptyMetadata;
+  }
+
+  const { scope_tables, root_entity, relations } = result[0];
+
+  // Build path mapping from relations
+  const pathMapping = buildPathMappingFromRelations(root_entity, relations);
+
+  const metadata = {
+    scopeTables: scope_tables || [],
+    pathMapping,
+    rootEntity: root_entity,
+    relations: relations || {}
+  };
+
+  // Cache for future use
+  subscribableMetadataCache.set(subscribableName, metadata);
+
+  wsLogger.debug(`Cached metadata for subscribable ${subscribableName}:`, {
+    scopeTables: metadata.scopeTables,
+    pathMapping: metadata.pathMapping
+  });
+
+  return metadata;
+}
+
+/**
+ * Build path mapping from relations configuration
+ * Maps table names to their document path for client-side patching
+ * @param {string} rootEntity - Root table name
+ * @param {object} relations - Relations configuration
+ * @returns {object} - Map of table name -> document path
+ */
+function buildPathMappingFromRelations(rootEntity, relations) {
+  const paths = {};
+
+  // Root entity maps to top level
+  if (rootEntity) {
+    paths[rootEntity] = '.';
+  }
+
+  const buildPaths = (rels, parentPath = '') => {
+    for (const [relName, relConfig] of Object.entries(rels || {})) {
+      const entity = typeof relConfig === 'string' ? relConfig : relConfig?.entity;
+      const currentPath = parentPath ? `${parentPath}.${relName}` : relName;
+
+      if (entity) {
+        paths[entity] = currentPath;
+      }
+
+      // Handle nested relations
+      if (typeof relConfig === 'object' && relConfig !== null) {
+        if (relConfig.include) {
+          buildPaths(relConfig.include, currentPath);
+        }
+        if (relConfig.relations) {
+          buildPaths(relConfig.relations, currentPath);
+        }
+      }
+    }
+  };
+
+  buildPaths(relations);
+  return paths;
+}
+
+/**
+ * Clear the subscribable metadata cache
+ * Called when subscribables are reregistered or updated
+ * @param {string} [subscribableName] - Optional: clear specific subscribable, or all if not provided
+ */
+export function clearSubscribableMetadataCache(subscribableName = null) {
+  if (subscribableName) {
+    subscribableMetadataCache.delete(subscribableName);
+    wsLogger.debug(`Cleared metadata cache for ${subscribableName}`);
+  } else {
+    subscribableMetadataCache.clear();
+    wsLogger.debug('Cleared all subscribable metadata cache');
+  }
+}
+
+/**
+ * Get scope tables for a subscribable (convenience function)
+ * @param {string} subscribableName - Name of the subscribable
+ * @param {function} sql - Database query function
+ * @returns {Promise<string[]>} - Array of table names in scope
+ */
+export async function getSubscribableScopeTables(subscribableName, sql) {
+  const metadata = await getSubscribableMetadata(subscribableName, sql);
+  return metadata.scopeTables;
+}

package/src/server/ws.js

@@ -11,7 +11,8 @@ import {
   registerSubscription,
   unregisterSubscription,
   unregisterSubscriptionByParams,
-  removeConnectionSubscriptions
+  removeConnectionSubscriptions,
+  getSubscribableMetadata
 } from "./subscriptions.js";
 
 // Environment configuration
@@ -327,6 +328,9 @@ export function createRPCHandler(customHandlers = {}) {
 
           const data = queryResult[0]?.data;
 
+          // Get subscribable metadata for schema (path mapping for atomic updates)
+          const metadata = await getSubscribableMetadata(subscribableName, sql);
+
           // Register subscription in memory
           const subscriptionId = registerSubscription(
             subscribableName,
@@ -335,9 +339,15 @@ export function createRPCHandler(customHandlers = {}) {
             params
           );
 
+          // Build result with schema for client-side patching
           const result = {
             subscription_id: subscriptionId,
-            data
+            data,
+            // Include schema for atomic update support
+            schema: {
+              root: metadata.rootEntity,
+              paths: metadata.pathMapping
+            }
           };
 
           wsLogger.response(method, result, Date.now() - startTime);