dzql 0.2.3 → 0.3.1

package/src/compiler/parser/entity-parser.js

@@ -7,18 +7,25 @@ export class EntityParser {
   /**
    * Parse a dzql.register_entity() call from SQL
    * @param {string} sql - SQL containing register_entity call
-   * @returns {Object} Parsed entity configuration
+   * @param {number} startOffset - Optional starting position in SQL
+   * @returns {Object} Parsed entity configuration with custom functions
    */
-  parseFromSQL(sql) {
+  parseFromSQL(sql, startOffset = 0) {
     // Extract the register_entity call
-    const registerMatch = sql.match(/dzql\.register_entity\s*\(([\s\S]*?)\);/i);
+    const searchSql = sql.substring(startOffset);
+    const registerMatch = searchSql.match(/dzql\.register_entity\s*\(([\s\S]*?)\);/i);
     if (!registerMatch) {
       throw new Error('No register_entity call found in SQL');
     }
 
     const params = this._parseParameters(registerMatch[1]);
+    const config = this._buildEntityConfig(params);
 
-    return this._buildEntityConfig(params);
+    // Extract custom functions after this register_entity call
+    const registerEndPos = startOffset + registerMatch.index + registerMatch[0].length;
+    config.customFunctions = this._extractCustomFunctions(sql, registerEndPos);
+
+    return config;
   }
 
   /**
@@ -73,6 +80,11 @@ export class EntityParser {
    * @private
    */
   _buildEntityConfig(params) {
+    const graphRules = params[8] ? this._parseJSON(params[8]) : {};
+
+    // Extract many_to_many from graph_rules if present
+    const manyToMany = graphRules.many_to_many || {};
+
     const config = {
       tableName: this._cleanString(params[0]),
       labelField: this._cleanString(params[1]),
@@ -82,7 +94,9 @@ export class EntityParser {
       temporalFields: params[5] ? this._parseJSON(params[5]) : {},
       notificationPaths: params[6] ? this._parseJSON(params[6]) : {},
       permissionPaths: params[7] ? this._parseJSON(params[7]) : {},
-      graphRules: params[8] ? this._parseJSON(params[8]) : {}
+      graphRules: graphRules,
+      fieldDefaults: params[9] ? this._parseJSON(params[9]) : {},
+      manyToMany: manyToMany
     };
 
     return config;
@@ -250,12 +264,56 @@ export class EntityParser {
     return cleaned === 'true' || cleaned === 't';
   }
 
+  /**
+   * Extract custom functions defined after register_entity() call
+   * @private
+   * @param {string} sql - Full SQL content
+   * @param {number} startPos - Position after register_entity call
+   * @returns {Array<string>} Array of custom function SQL statements
+   */
+  _extractCustomFunctions(sql, startPos) {
+    // Find the next register_entity call or end of file
+    const nextEntityMatch = sql.substring(startPos).match(/dzql\.register_entity\s*\(/i);
+    const endPos = nextEntityMatch ? startPos + nextEntityMatch.index : sql.length;
+
+    // Extract the SQL between this entity and the next
+    const customSql = sql.substring(startPos, endPos).trim();
+    if (!customSql) return [];
+
+    const functions = [];
+
+    // Extract CREATE [OR REPLACE] FUNCTION statements
+    // Match from CREATE to the final semicolon of the function (including $$ delimiters)
+    const functionPattern = /CREATE\s+(?:OR\s+REPLACE\s+)?FUNCTION\s+[\s\S]*?(?:\$\$|\$[A-Za-z_][A-Za-z0-9_]*\$)\s*(?:LANGUAGE|;)[\s\S]*?;/gi;
+    let match;
+    while ((match = functionPattern.exec(customSql)) !== null) {
+      functions.push(match[0].trim());
+    }
+
+    // Extract INSERT INTO dzql.registry statements
+    const registryPattern = /INSERT\s+INTO\s+dzql\.registry\s+[\s\S]*?;/gi;
+    while ((match = registryPattern.exec(customSql)) !== null) {
+      functions.push(match[0].trim());
+    }
+
+    // Extract SELECT dzql.register_function() calls
+    const registerFunctionPattern = /SELECT\s+dzql\.register_function\s*\([\s\S]*?\)\s*;/gi;
+    while ((match = registerFunctionPattern.exec(customSql)) !== null) {
+      functions.push(match[0].trim());
+    }
+
+    return functions;
+  }
+
   /**
    * Parse entity definition from JS object (for programmatic use)
    * @param {Object} entity - Entity definition object
    * @returns {Object} Normalized entity configuration
    */
   parseFromObject(entity) {
+    const graphRules = entity.graphRules || {};
+    const manyToMany = entity.manyToMany || graphRules.many_to_many || {};
+
     return {
       tableName: entity.tableName || entity.table,
       labelField: entity.labelField || 'name',
@@ -265,7 +323,10 @@ export class EntityParser {
       temporalFields: entity.temporalFields || {},
       notificationPaths: entity.notificationPaths || {},
       permissionPaths: entity.permissionPaths || {},
-      graphRules: entity.graphRules || {}
+      graphRules: graphRules,
+      fieldDefaults: entity.fieldDefaults || {},
+      manyToMany: manyToMany,
+      customFunctions: entity.customFunctions || []
     };
   }
 }
@@ -279,17 +340,16 @@ export function parseEntitiesFromSQL(sql) {
   const parser = new EntityParser();
   const entities = [];
 
-  // Find all register_entity calls
-  const registerCalls = sql.match(/dzql\.register_entity\s*\([\s\S]*?\);/gi);
-
-  if (!registerCalls) {
-    return entities;
-  }
+  // Find all register_entity calls with their positions
+  const registerPattern = /dzql\.register_entity\s*\(/gi;
+  let match;
+  let currentPos = 0;
 
-  for (const call of registerCalls) {
+  while ((match = registerPattern.exec(sql)) !== null) {
     try {
-      const entity = parser.parseFromSQL(call);
+      const entity = parser.parseFromSQL(sql, match.index);
       entities.push(entity);
+      currentPos = match.index + 1; // Move past this match
     } catch (error) {
       console.warn('Failed to parse entity:', error.message);
     }

package/src/database/migrations/001_schema.sql

@@ -23,7 +23,9 @@ CREATE TABLE IF NOT EXISTS dzql.entities (
   temporal_fields jsonb DEFAULT '{}',    -- valid_from/valid_to field names for temporal filtering
   notification_paths jsonb DEFAULT '{}', -- paths to determine who gets notified
   permission_paths jsonb DEFAULT '{}',   -- paths to determine who has permission for operations
-  graph_rules jsonb DEFAULT '{}'         -- graph evolution rules for automatic relationship management
+  graph_rules jsonb DEFAULT '{}',        -- graph evolution rules for automatic relationship management
+  field_defaults jsonb DEFAULT '{}',     -- default values to auto-populate on INSERT
+  many_to_many jsonb DEFAULT '{}'        -- many-to-many relationship configurations
 );
 
 -- === Registry (allowlist of callable functions) ===

package/src/database/migrations/002_functions.sql

@@ -744,6 +744,11 @@ BEGIN
   -- Validate top-level trigger types
   FOR l_trigger_key, l_trigger_rules IN SELECT * FROM jsonb_each(p_rules)
   LOOP
+    -- Skip validation for many_to_many (different structure)
+    IF l_trigger_key = 'many_to_many' THEN
+      CONTINUE;
+    END IF;
+
     -- Check valid trigger types
     IF l_trigger_key NOT IN ('on_create', 'on_update', 'on_delete', 'on_field_change') THEN
       RAISE WARNING 'Invalid trigger type: %', l_trigger_key;

package/src/database/migrations/003_operations.sql

@@ -223,6 +223,62 @@ BEGIN
     END LOOP;
   END IF;
 
+  -- Expand many-to-many relationships (if configured)
+  IF l_entity_config.many_to_many IS NOT NULL AND l_entity_config.many_to_many != '{}'::jsonb THEN
+    DECLARE
+      l_m2m_key text;
+      l_m2m_config jsonb;
+      l_id_field text;
+      l_junction_table text;
+      l_local_key text;
+      l_foreign_key text;
+      l_target_entity text;
+      l_expand boolean;
+      l_record_id text;
+      l_id_array jsonb;
+      l_expanded_objects jsonb;
+    BEGIN
+      -- Get the primary key value from the result
+      l_record_id := l_result->>l_pk_cols[1];  -- Assume single PK for now
+
+      FOR l_m2m_key IN SELECT jsonb_object_keys(l_entity_config.many_to_many)
+      LOOP
+        l_m2m_config := l_entity_config.many_to_many->l_m2m_key;
+        l_id_field := l_m2m_config->>'id_field';
+        l_junction_table := l_m2m_config->>'junction_table';
+        l_local_key := l_m2m_config->>'local_key';
+        l_foreign_key := l_m2m_config->>'foreign_key';
+        l_target_entity := l_m2m_config->>'target_entity';
+        l_expand := COALESCE((l_m2m_config->>'expand')::boolean, false);
+
+        -- Always include array of IDs
+        EXECUTE format('
+          SELECT COALESCE(jsonb_agg(%I), ''[]''::jsonb)
+          FROM %I
+          WHERE %I = $1::int
+        ', l_foreign_key, l_junction_table, l_local_key)
+        INTO l_id_array
+        USING l_record_id;
+
+        l_result := l_result || jsonb_build_object(l_id_field, l_id_array);
+
+        -- Conditionally include expanded objects if expand: true
+        IF l_expand THEN
+          EXECUTE format('
+            SELECT COALESCE(jsonb_agg(to_jsonb(t.*)), ''[]''::jsonb)
+            FROM %I jt
+            JOIN %I t ON t.id = jt.%I
+            WHERE jt.%I = $1::int
+          ', l_junction_table, l_target_entity, l_foreign_key, l_local_key)
+          INTO l_expanded_objects
+          USING l_record_id;
+
+          l_result := l_result || jsonb_build_object(l_m2m_key, l_expanded_objects);
+        END IF;
+      END LOOP;
+    END;
+  END IF;
+
   RETURN l_result;
 END $$;
 
@@ -333,7 +389,29 @@ BEGIN
     LOOP
       -- Don't update any primary key columns
       IF NOT (l_col_name = ANY(l_pk_cols)) THEN
-        l_set_clauses := l_set_clauses || format('%I = %L', l_col_name, l_merged_data ->> l_col_name);
+        -- Skip M2M ID fields (they're not real table columns)
+        IF l_entity_config.many_to_many IS NOT NULL THEN
+          DECLARE
+            l_m2m_id_field text;
+            l_skip boolean := false;
+          BEGIN
+            FOR l_m2m_id_field IN
+              SELECT value->>'id_field'
+              FROM jsonb_each(l_entity_config.many_to_many)
+            LOOP
+              IF l_col_name = l_m2m_id_field THEN
+                l_skip := true;
+                EXIT;
+              END IF;
+            END LOOP;
+
+            IF NOT l_skip THEN
+              l_set_clauses := l_set_clauses || format('%I = %L', l_col_name, l_merged_data ->> l_col_name);
+            END IF;
+          END;
+        ELSE
+          l_set_clauses := l_set_clauses || format('%I = %L', l_col_name, l_merged_data ->> l_col_name);
+        END IF;
       END IF;
     END LOOP;
 
@@ -359,6 +437,38 @@ BEGIN
       l_args_json := l_args_json || jsonb_build_object('user_id', p_user_id);
     END IF;
 
+    -- Apply field defaults for INSERT (if configured)
+    IF l_entity_config.field_defaults IS NOT NULL AND l_entity_config.field_defaults != '{}' THEN
+      FOR l_col_name IN SELECT jsonb_object_keys(l_entity_config.field_defaults)
+      LOOP
+        -- Only apply default if field is not already provided
+        IF NOT (l_args_json ? l_col_name) THEN
+          DECLARE
+            l_default_value text;
+            l_resolved_value text;
+          BEGIN
+            l_default_value := l_entity_config.field_defaults->>l_col_name;
+
+            -- Resolve variable if it starts with @
+            IF l_default_value LIKE '@%' THEN
+              l_resolved_value := dzql.resolve_graph_variable(
+                l_default_value,
+                NULL,  -- no before record for INSERT
+                l_args_json,  -- current data being inserted
+                p_user_id
+              );
+            ELSE
+              -- Use literal value
+              l_resolved_value := l_default_value;
+            END IF;
+
+            -- Add to l_args_json
+            l_args_json := l_args_json || jsonb_build_object(l_col_name, l_resolved_value);
+          END;
+        END IF;
+      END LOOP;
+    END IF;
+
     -- Check create permission on new values
     l_operation := 'create';
     l_permission_record := l_args_json;
@@ -371,6 +481,28 @@ BEGIN
 
     FOR l_col_name IN SELECT jsonb_object_keys(l_args_json)
     LOOP
+      -- Skip M2M ID fields (they're not real table columns)
+      IF l_entity_config.many_to_many IS NOT NULL THEN
+        DECLARE
+          l_m2m_id_field text;
+          l_skip boolean := false;
+        BEGIN
+          FOR l_m2m_id_field IN
+            SELECT value->>'id_field'
+            FROM jsonb_each(l_entity_config.many_to_many)
+          LOOP
+            IF l_col_name = l_m2m_id_field THEN
+              l_skip := true;
+              EXIT;
+            END IF;
+          END LOOP;
+
+          IF l_skip THEN
+            CONTINUE;
+          END IF;
+        END;
+      END IF;
+
       IF l_args_json ->> l_col_name IS NOT NULL AND l_args_json ->> l_col_name != '' THEN
         l_cols := l_cols || quote_ident(l_col_name);
         l_vals := l_vals || quote_literal(l_args_json ->> l_col_name);
@@ -391,6 +523,109 @@ BEGIN
 
   END IF;
 
+  -- Sync many-to-many relationships (if configured)
+  IF l_entity_config.many_to_many IS NOT NULL AND l_entity_config.many_to_many != '{}'::jsonb THEN
+    DECLARE
+      l_m2m_key text;
+      l_m2m_config jsonb;
+      l_id_field text;
+      l_junction_table text;
+      l_local_key text;
+      l_foreign_key text;
+      l_record_id text;
+    BEGIN
+      -- Get the primary key value from the result
+      l_record_id := l_result->>l_pk_cols[1];  -- Assume single PK for now
+
+      FOR l_m2m_key IN SELECT jsonb_object_keys(l_entity_config.many_to_many)
+      LOOP
+        l_m2m_config := l_entity_config.many_to_many->l_m2m_key;
+        l_id_field := l_m2m_config->>'id_field';
+
+        -- Only sync if the ID field is present in the data
+        IF l_args_json ? l_id_field THEN
+          l_junction_table := l_m2m_config->>'junction_table';
+          l_local_key := l_m2m_config->>'local_key';
+          l_foreign_key := l_m2m_config->>'foreign_key';
+
+          -- Delete relationships not in new list
+          EXECUTE format('
+            DELETE FROM %I
+            WHERE %I = $1::int
+              AND %I <> ALL($2::int[])
+          ', l_junction_table, l_local_key, l_foreign_key)
+          USING l_record_id,
+                ARRAY(SELECT jsonb_array_elements_text(l_args_json->l_id_field))::int[];
+
+          -- Insert new relationships (ignore conflicts)
+          EXECUTE format('
+            INSERT INTO %I (%I, %I)
+            SELECT $1::int, value::int
+            FROM jsonb_array_elements_text($2)
+            ON CONFLICT DO NOTHING
+          ', l_junction_table, l_local_key, l_foreign_key)
+          USING l_record_id, l_args_json->l_id_field;
+        END IF;
+      END LOOP;
+    END;
+  END IF;
+
+  -- Expand many-to-many relationships in result (after sync)
+  IF l_entity_config.many_to_many IS NOT NULL AND l_entity_config.many_to_many != '{}'::jsonb THEN
+    DECLARE
+      l_m2m_key text;
+      l_m2m_config jsonb;
+      l_id_field text;
+      l_junction_table text;
+      l_local_key text;
+      l_foreign_key text;
+      l_target_entity text;
+      l_expand boolean;
+      l_record_id text;
+      l_id_array jsonb;
+      l_expanded_objects jsonb;
+    BEGIN
+      -- Get the primary key value from the result
+      l_record_id := l_result->>l_pk_cols[1];  -- Assume single PK for now
+
+      FOR l_m2m_key IN SELECT jsonb_object_keys(l_entity_config.many_to_many)
+      LOOP
+        l_m2m_config := l_entity_config.many_to_many->l_m2m_key;
+        l_id_field := l_m2m_config->>'id_field';
+        l_junction_table := l_m2m_config->>'junction_table';
+        l_local_key := l_m2m_config->>'local_key';
+        l_foreign_key := l_m2m_config->>'foreign_key';
+        l_target_entity := l_m2m_config->>'target_entity';
+        l_expand := COALESCE((l_m2m_config->>'expand')::boolean, false);
+
+        -- Always include array of IDs
+        EXECUTE format('
+          SELECT COALESCE(jsonb_agg(%I), ''[]''::jsonb)
+          FROM %I
+          WHERE %I = $1::int
+        ', l_foreign_key, l_junction_table, l_local_key)
+        INTO l_id_array
+        USING l_record_id;
+
+        l_result := l_result || jsonb_build_object(l_id_field, l_id_array);
+
+        -- Conditionally include expanded objects if expand: true
+        IF l_expand THEN
+          EXECUTE format('
+            SELECT COALESCE(jsonb_agg(to_jsonb(t.*)), ''[]''::jsonb)
+            FROM %I jt
+            JOIN %I t ON t.id = jt.%I
+            WHERE jt.%I = $1::int
+          ', l_junction_table, l_target_entity, l_foreign_key, l_local_key)
+          INTO l_expanded_objects
+          USING l_record_id;
+
+          l_result := l_result || jsonb_build_object(l_m2m_key, l_expanded_objects);
+        END IF;
+      END LOOP;
+    END;
+  END IF;
+
   -- Execute graph rules for the appropriate operation
   l_graph_rules_result := dzql.execute_graph_rules(
     p_entity,

package/src/database/migrations/004_search.sql

@@ -398,6 +398,70 @@ BEGIN
         END IF;
       END LOOP;
 
+      -- Expand many-to-many relationships for this record (if configured)
+      IF l_entity_config.many_to_many IS NOT NULL AND l_entity_config.many_to_many != '{}'::jsonb THEN
+        DECLARE
+          l_m2m_key text;
+          l_m2m_config jsonb;
+          l_id_field text;
+          l_junction_table text;
+          l_local_key text;
+          l_foreign_key text;
+          l_target_entity text;
+          l_expand boolean;
+          l_record_id text;
+          l_id_array jsonb;
+          l_expanded_objects jsonb;
+          l_pk_cols text[];
+        BEGIN
+          -- Get primary key columns for this entity
+          SELECT array_agg(a.attname ORDER BY a.attnum)
+            INTO l_pk_cols
+          FROM pg_index idx
+          JOIN pg_attribute a ON a.attrelid = idx.indrelid AND a.attnum = ANY(idx.indkey)
+          WHERE idx.indrelid = p_entity::regclass AND idx.indisprimary;
+
+          -- Get the primary key value from the record
+          l_record_id := l_record->>l_pk_cols[1];  -- Assume single PK for now
+
+          FOR l_m2m_key IN SELECT jsonb_object_keys(l_entity_config.many_to_many)
+          LOOP
+            l_m2m_config := l_entity_config.many_to_many->l_m2m_key;
+            l_id_field := l_m2m_config->>'id_field';
+            l_junction_table := l_m2m_config->>'junction_table';
+            l_local_key := l_m2m_config->>'local_key';
+            l_foreign_key := l_m2m_config->>'foreign_key';
+            l_target_entity := l_m2m_config->>'target_entity';
+            l_expand := COALESCE((l_m2m_config->>'expand')::boolean, false);
+
+            -- Always include array of IDs
+            EXECUTE format('
+              SELECT COALESCE(jsonb_agg(%I), ''[]''::jsonb)
+              FROM %I
+              WHERE %I = $1::int
+            ', l_foreign_key, l_junction_table, l_local_key)
+            INTO l_id_array
+            USING l_record_id;
+
+            l_record := l_record || jsonb_build_object(l_id_field, l_id_array);
+
+            -- Conditionally include expanded objects if expand: true
+            IF l_expand THEN
+              EXECUTE format('
+                SELECT COALESCE(jsonb_agg(to_jsonb(t.*)), ''[]''::jsonb)
+                FROM %I jt
+                JOIN %I t ON t.id = jt.%I
+                WHERE jt.%I = $1::int
+              ', l_junction_table, l_target_entity, l_foreign_key, l_local_key)
+              INTO l_expanded_objects
+              USING l_record_id;
+
+              l_record := l_record || jsonb_build_object(l_m2m_key, l_expanded_objects);
+            END IF;
+          END LOOP;
+        END;
+      END IF;
+
       l_processed_data := l_processed_data || l_record;
     END LOOP;
 

package/src/database/migrations/005_entities.sql

@@ -579,7 +579,8 @@ CREATE OR REPLACE FUNCTION dzql.register_entity(
   p_temporal_fields jsonb DEFAULT '{}',
   p_notification_paths jsonb DEFAULT '{}',
   p_permission_paths jsonb DEFAULT '{}',
-  p_graph_rules jsonb DEFAULT '{}'
+  p_graph_rules jsonb DEFAULT '{}',
+  p_field_defaults jsonb DEFAULT '{}'
 ) RETURNS void
 LANGUAGE plpgsql AS $$
 DECLARE
@@ -587,6 +588,7 @@ DECLARE
   l_rule_name text;
   l_rule_config jsonb;
   l_action jsonb;
+  l_many_to_many jsonb;
 BEGIN
   -- Validate permission paths if provided
   IF p_permission_paths IS NOT NULL AND p_permission_paths != '{}' THEN
@@ -602,11 +604,14 @@ BEGIN
     END IF;
   END IF;
 
+  -- Extract many_to_many from graph_rules if present
+  l_many_to_many := COALESCE(p_graph_rules->'many_to_many', '{}'::jsonb);
+
   -- Insert or update entity configuration
   INSERT INTO dzql.entities
-    (table_name, label_field, searchable_fields, fk_includes, soft_delete, temporal_fields, notification_paths, permission_paths, graph_rules)
+    (table_name, label_field, searchable_fields, fk_includes, soft_delete, temporal_fields, notification_paths, permission_paths, graph_rules, field_defaults, many_to_many)
   VALUES
-    (p_table_name, p_label_field, p_searchable_fields, p_fk_includes, p_soft_delete, p_temporal_fields, p_notification_paths, p_permission_paths, p_graph_rules)
+    (p_table_name, p_label_field, p_searchable_fields, p_fk_includes, p_soft_delete, p_temporal_fields, p_notification_paths, p_permission_paths, p_graph_rules, p_field_defaults, l_many_to_many)
   ON CONFLICT (table_name) DO UPDATE SET
     label_field = EXCLUDED.label_field,
     searchable_fields = EXCLUDED.searchable_fields,
@@ -615,7 +620,9 @@ BEGIN
     temporal_fields = EXCLUDED.temporal_fields,
     notification_paths = EXCLUDED.notification_paths,
     permission_paths = EXCLUDED.permission_paths,
-    graph_rules = EXCLUDED.graph_rules;
+    graph_rules = EXCLUDED.graph_rules,
+    field_defaults = EXCLUDED.field_defaults,
+    many_to_many = EXCLUDED.many_to_many;
 
   -- Create API functions for this entity
   PERFORM dzql.create_entity_functions(p_table_name);