clearctx 3.0.0

package/src/pipeline-engine.js

@@ -0,0 +1,618 @@
+/**
+ * Pipeline Engine - Layer 3 Reactive Automation System
+ *
+ * This class manages reactive pipelines: "when X happens, automatically do Y."
+ * It listens for events (artifact published, contract completed, etc.) and
+ * fires rules that match the trigger conditions.
+ *
+ * Architecture:
+ * - Pipelines contain rules
+ * - Each rule has: trigger (when to fire), condition (filter), action (what to do)
+ * - Triggers match against events from the system
+ * - Actions can notify sessions, broadcast, or return actions for caller to execute
+ *
+ * @class PipelineEngine
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+
+// Import our atomic file operations and locking utilities
+const { atomicWriteJson, readJsonSafe, appendJsonl } = require('./atomic-io');
+const { acquireLock, releaseLock } = require('./file-lock');
+const TeamHub = require('./team-hub');
+
+/**
+ * PipelineEngine class - manages reactive automation pipelines
+ */
+class PipelineEngine {
+  /**
+   * Creates a new PipelineEngine instance
+   *
+   * @param {string} teamName - Name of the team (default: 'default')
+   */
+  constructor(teamName = 'default') {
+    // Set the team name
+    this.teamName = teamName;
+
+    // Create the base directory path: ~/.clearctx
+    const baseDir = path.join(os.homedir(), '.clearctx');
+
+    // Create the pipelines directory path: ~/.clearctx/team/{teamName}/pipelines
+    this.pipelinesDir = path.join(baseDir, 'team', teamName, 'pipelines');
+
+    // Set paths for pipeline data
+    this.indexPath = path.join(this.pipelinesDir, 'index.json');
+    this.logPath = path.join(this.pipelinesDir, 'log.jsonl');
+    this.locksDir = path.join(baseDir, 'locks');
+
+    // Make sure all directories exist
+    fs.mkdirSync(this.pipelinesDir, { recursive: true });
+    fs.mkdirSync(this.locksDir, { recursive: true });
+
+    // Initialize TeamHub for messaging capabilities
+    this.teamHub = new TeamHub(teamName);
+
+    // Safety limit: max number of rule evaluations per event
+    // This prevents infinite loops or runaway pipeline execution
+    this.maxEvaluationsPerCycle = 50;
+  }
+
+  /**
+   * Create a new pipeline
+   *
+   * @param {string} pipelineId - Unique identifier for the pipeline
+   * @param {Object} options - Pipeline configuration
+   * @param {Array} options.rules - Array of rule objects
+   * @param {string} options.owner - Who created this pipeline
+   * @param {boolean} [options.enabled=true] - Whether pipeline is active
+   * @returns {Object} The created pipeline object
+   * @throws {Error} If pipelineId already exists or validation fails
+   */
+  create(pipelineId, { rules, owner, enabled = true }) {
+    // Acquire lock to prevent concurrent modifications
+    acquireLock(this.locksDir, 'pipelines-index');
+
+    try {
+      // Read current pipeline index
+      const index = readJsonSafe(this.indexPath, {});
+
+      // Check if pipeline already exists
+      if (index[pipelineId]) {
+        throw new Error(`Pipeline ${pipelineId} already exists`);
+      }
+
+      // Validate rules - each must have required fields
+      for (const rule of rules) {
+        // Every rule must have a unique ruleId
+        if (!rule.ruleId) {
+          throw new Error('Rule missing ruleId');
+        }
+
+        // Every rule must have a trigger with a type
+        if (!rule.trigger || !rule.trigger.type) {
+          throw new Error(`Rule ${rule.ruleId} missing trigger or trigger.type`);
+        }
+
+        // Every rule must have an action with a type
+        if (!rule.action || !rule.action.type) {
+          throw new Error(`Rule ${rule.ruleId} missing action or action.type`);
+        }
+      }
+
+      // Create the new pipeline object
+      const pipeline = {
+        pipelineId,
+        owner,
+        enabled,
+        createdAt: new Date().toISOString(),
+        rules,
+        executionCount: 0,
+        lastExecutedAt: null
+      };
+
+      // Add to index
+      index[pipelineId] = pipeline;
+
+      // Save the updated index
+      atomicWriteJson(this.indexPath, index);
+
+      return pipeline;
+    } finally {
+      // Always release the lock, even if an error occurred
+      releaseLock(this.locksDir, 'pipelines-index');
+    }
+  }
+
+  /**
+   * Evaluate an event against all enabled pipelines
+   *
+   * @param {Object} event - The event to evaluate
+   * @param {string} event.type - Event type (e.g., "artifact_published")
+   * @param {string} [event.artifactId] - Artifact identifier (if applicable)
+   * @param {string} [event.artifactType] - Artifact type (if applicable)
+   * @param {string} [event.contractId] - Contract identifier (if applicable)
+   * @param {number} [event.version] - Version number (if applicable)
+   * @param {Object} [event.data] - Additional event data
+   * @returns {Object} Summary of evaluation results
+   */
+  evaluate(event) {
+    // Read pipeline index (no lock needed for reads - they're atomic)
+    const index = readJsonSafe(this.indexPath, {});
+
+    // Track evaluation statistics
+    let evaluationCount = 0;
+    const firedActions = [];
+
+    // Get all pipelines as an array
+    const pipelines = Object.values(index);
+
+    // Process each enabled pipeline
+    for (const pipeline of pipelines) {
+      // Skip disabled pipelines
+      if (!pipeline.enabled) {
+        continue;
+      }
+
+      // Evaluate each rule in the pipeline
+      for (const rule of pipeline.rules) {
+        // Check safety limit to prevent runaway execution
+        if (evaluationCount >= this.maxEvaluationsPerCycle) {
+          console.warn(
+            `Pipeline evaluation limit reached (${this.maxEvaluationsPerCycle}). ` +
+            `Stopping to prevent infinite loops.`
+          );
+          // Stop processing more rules
+          break;
+        }
+
+        // Increment evaluation counter
+        evaluationCount++;
+
+        // Check if trigger matches this event
+        const triggerMatches = this._matchTrigger(rule.trigger, event);
+        if (!triggerMatches) {
+          continue; // This rule doesn't apply to this event
+        }
+
+        // Check if condition passes (if there is one)
+        const conditionPasses = this._evaluateCondition(rule.condition, event);
+        if (!conditionPasses) {
+          continue; // Condition didn't pass, skip this rule
+        }
+
+        // Both trigger and condition passed - execute the action!
+        const actionResult = this._executeAction(rule.action, event);
+
+        // Log this execution to the append-only log
+        const logEntry = {
+          pipelineId: pipeline.pipelineId,
+          ruleId: rule.ruleId,
+          trigger: rule.trigger,
+          eventType: event.type,
+          eventSummary: {
+            artifactId: event.artifactId,
+            artifactType: event.artifactType,
+            contractId: event.contractId,
+            version: event.version
+          },
+          action: rule.action,
+          timestamp: new Date().toISOString()
+        };
+        appendJsonl(this.logPath, logEntry);
+
+        // Add to fired actions
+        firedActions.push({
+          pipelineId: pipeline.pipelineId,
+          ruleId: rule.ruleId,
+          result: actionResult
+        });
+
+        // Update pipeline execution stats
+        pipeline.executionCount++;
+        pipeline.lastExecutedAt = new Date().toISOString();
+      }
+    }
+
+    // Update pipeline stats in index (need lock for writes)
+    if (firedActions.length > 0) {
+      acquireLock(this.locksDir, 'pipelines-index');
+      try {
+        const currentIndex = readJsonSafe(this.indexPath, {});
+        // Update stats for pipelines that fired
+        for (const fired of firedActions) {
+          if (currentIndex[fired.pipelineId]) {
+            currentIndex[fired.pipelineId].executionCount =
+              (currentIndex[fired.pipelineId].executionCount || 0) + 1;
+            currentIndex[fired.pipelineId].lastExecutedAt = new Date().toISOString();
+          }
+        }
+        atomicWriteJson(this.indexPath, currentIndex);
+      } finally {
+        releaseLock(this.locksDir, 'pipelines-index');
+      }
+    }
+
+    // Return summary of what happened
+    return {
+      evaluated: evaluationCount,
+      fired: firedActions
+    };
+  }
+
+  /**
+   * List all pipelines, optionally filtered by owner
+   *
+   * @param {Object} [options={}] - Filter options
+   * @param {string} [options.owner] - Filter by owner name
+   * @returns {Array} Array of pipeline objects
+   */
+  list({ owner } = {}) {
+    // Read pipeline index
+    const index = readJsonSafe(this.indexPath, {});
+
+    // Get all pipelines as an array
+    let pipelines = Object.values(index);
+
+    // Filter by owner if specified
+    if (owner) {
+      pipelines = pipelines.filter(p => p.owner === owner);
+    }
+
+    return pipelines;
+  }
+
+  /**
+   * Pause (disable) a pipeline
+   *
+   * @param {string} pipelineId - Pipeline to pause
+   * @throws {Error} If pipeline not found
+   */
+  pause(pipelineId) {
+    // Acquire lock for modification
+    acquireLock(this.locksDir, 'pipelines-index');
+
+    try {
+      // Read current index
+      const index = readJsonSafe(this.indexPath, {});
+
+      // Check if pipeline exists
+      if (!index[pipelineId]) {
+        throw new Error(`Pipeline ${pipelineId} not found`);
+      }
+
+      // Set enabled to false
+      index[pipelineId].enabled = false;
+
+      // Save updated index
+      atomicWriteJson(this.indexPath, index);
+    } finally {
+      releaseLock(this.locksDir, 'pipelines-index');
+    }
+  }
+
+  /**
+   * Resume (enable) a pipeline
+   *
+   * @param {string} pipelineId - Pipeline to resume
+   * @throws {Error} If pipeline not found
+   */
+  resume(pipelineId) {
+    // Acquire lock for modification
+    acquireLock(this.locksDir, 'pipelines-index');
+
+    try {
+      // Read current index
+      const index = readJsonSafe(this.indexPath, {});
+
+      // Check if pipeline exists
+      if (!index[pipelineId]) {
+        throw new Error(`Pipeline ${pipelineId} not found`);
+      }
+
+      // Set enabled to true
+      index[pipelineId].enabled = true;
+
+      // Save updated index
+      atomicWriteJson(this.indexPath, index);
+    } finally {
+      releaseLock(this.locksDir, 'pipelines-index');
+    }
+  }
+
+  /**
+   * Get execution log entries
+   *
+   * @param {string|null} [pipelineId=null] - Filter by pipeline (null = all)
+   * @param {number} [limit=50] - Max number of entries to return
+   * @returns {Array} Array of log entries (most recent first)
+   */
+  getLog(pipelineId = null, limit = 50) {
+    // Check if log file exists
+    if (!fs.existsSync(this.logPath)) {
+      return []; // No log file yet
+    }
+
+    // Read the log file
+    const content = fs.readFileSync(this.logPath, 'utf-8');
+    const lines = content.trim().split('\n').filter(line => line.length > 0);
+
+    // Parse each line as JSON
+    let entries = lines.map(line => JSON.parse(line));
+
+    // Filter by pipelineId if specified
+    if (pipelineId) {
+      entries = entries.filter(e => e.pipelineId === pipelineId);
+    }
+
+    // Return last N entries in reverse order (most recent first)
+    return entries.slice(-limit).reverse();
+  }
+
+  // ========== HELPER METHODS ==========
+
+  /**
+   * Check if a trigger matches an event
+   *
+   * @private
+   * @param {Object} trigger - Rule trigger configuration
+   * @param {Object} event - Event to match against
+   * @returns {boolean} True if trigger matches event
+   */
+  _matchTrigger(trigger, event) {
+    // First check: trigger type must match event type
+    if (trigger.type !== event.type) {
+      return false;
+    }
+
+    // Check artifact type if specified in trigger
+    if (trigger.artifactType && event.artifactType) {
+      if (trigger.artifactType !== event.artifactType) {
+        return false;
+      }
+    }
+
+    // Check artifact ID if specified in trigger (supports wildcards)
+    if (trigger.artifactId && event.artifactId) {
+      // Convert wildcard pattern to regex
+      // Example: "api-contract-*" becomes /^api-contract-.*$/
+      const pattern = trigger.artifactId.replace(/\*/g, '.*');
+      const regex = new RegExp(`^${pattern}$`);
+
+      if (!regex.test(event.artifactId)) {
+        return false;
+      }
+    }
+
+    // Check contract ID if specified in trigger
+    if (trigger.contractId && event.contractId) {
+      if (trigger.contractId !== event.contractId) {
+        return false;
+      }
+    }
+
+    // All checks passed!
+    return true;
+  }
+
+  /**
+   * Evaluate a condition expression
+   *
+   * @private
+   * @param {string|null} condition - Condition expression or null
+   * @param {Object} event - Event data to evaluate against
+   * @returns {boolean} True if condition passes (or is null)
+   */
+  _evaluateCondition(condition, event) {
+    // If no condition, always pass
+    if (!condition || condition === null || condition === undefined) {
+      return true;
+    }
+
+    // Parse simple expressions like "data.failed > 0" or "version >= 3"
+    // Supported operators: >, <, >=, <=, ==, !=
+    const operators = ['>=', '<=', '==', '!=', '>', '<'];
+    let operator = null;
+    let parts = null;
+
+    // Find which operator is used
+    for (const op of operators) {
+      if (condition.includes(op)) {
+        operator = op;
+        parts = condition.split(op).map(s => s.trim());
+        break;
+      }
+    }
+
+    // If no operator found or invalid split, fail safely
+    if (!operator || !parts || parts.length !== 2) {
+      console.warn(`Invalid condition expression: ${condition}`);
+      return false;
+    }
+
+    const leftSide = parts[0];
+    const rightSide = parts[1];
+
+    // Evaluate left side (get value from event)
+    let leftValue;
+    if (leftSide.startsWith('data.')) {
+      // Extract from event.data
+      const fieldName = leftSide.substring(5); // Remove "data."
+      leftValue = event.data ? event.data[fieldName] : undefined;
+    } else if (leftSide === 'version') {
+      leftValue = event.version;
+    } else {
+      // Unknown left side
+      return false;
+    }
+
+    // If left value is undefined, condition fails
+    if (leftValue === undefined) {
+      return false;
+    }
+
+    // Parse right side as number
+    const rightValue = parseFloat(rightSide);
+    if (isNaN(rightValue)) {
+      console.warn(`Right side is not a number: ${rightSide}`);
+      return false;
+    }
+
+    // Convert left value to number
+    const leftNum = parseFloat(leftValue);
+    if (isNaN(leftNum)) {
+      console.warn(`Left side is not a number: ${leftValue}`);
+      return false;
+    }
+
+    // Evaluate based on operator
+    switch (operator) {
+      case '>':
+        return leftNum > rightValue;
+      case '<':
+        return leftNum < rightValue;
+      case '>=':
+        return leftNum >= rightValue;
+      case '<=':
+        return leftNum <= rightValue;
+      case '==':
+        return leftNum === rightValue;
+      case '!=':
+        return leftNum !== rightValue;
+      default:
+        return false;
+    }
+  }
+
+  /**
+   * Execute an action or return it for caller to handle
+   *
+   * @private
+   * @param {Object} action - Action configuration
+   * @param {Object} event - Event that triggered this action
+   * @returns {Object} Action result
+   */
+  _executeAction(action, event) {
+    // Get the action type
+    const actionType = action.type;
+
+    // Handle "notify_session" - send direct message via TeamHub
+    if (actionType === 'notify_session') {
+      // Interpolate message template with event data
+      const message = this._interpolate(action.message, event);
+
+      // Send direct message to target session
+      this.teamHub.sendDirect(
+        action.pipelineId || 'pipeline',  // From: pipeline system
+        action.target,                     // To: target session
+        message
+      );
+
+      return {
+        executed: true,
+        type: 'notify_session'
+      };
+    }
+
+    // Handle "broadcast" - send message to all team members
+    if (actionType === 'broadcast') {
+      // Interpolate message template with event data
+      const message = this._interpolate(action.message, event);
+
+      // Broadcast to all team members
+      this.teamHub.sendBroadcast(
+        'pipeline', // From: pipeline system
+        message
+      );
+
+      return {
+        executed: true,
+        type: 'broadcast'
+      };
+    }
+
+    // Handle "reopen_contract" - return for caller to execute
+    // We don't import ContractStore here to avoid circular dependencies
+    if (actionType === 'reopen_contract') {
+      return {
+        executed: false,
+        type: 'reopen_contract',
+        params: action
+      };
+    }
+
+    // Handle "create_contract" - return for caller to execute
+    if (actionType === 'create_contract') {
+      return {
+        executed: false,
+        type: 'create_contract',
+        params: action
+      };
+    }
+
+    // Handle "invalidate_downstream" - return for caller to execute
+    if (actionType === 'invalidate_downstream') {
+      return {
+        executed: false,
+        type: 'invalidate_downstream',
+        params: action
+      };
+    }
+
+    // Unknown action type
+    console.warn(`Unknown action type: ${actionType}`);
+    return {
+      executed: false,
+      type: 'unknown',
+      error: `Unknown action type: ${actionType}`
+    };
+  }
+
+  /**
+   * Interpolate template variables in a message
+   *
+   * @private
+   * @param {string} template - Message template with ${variable} placeholders
+   * @param {Object} event - Event data to interpolate from
+   * @returns {string} Interpolated message
+   */
+  _interpolate(template, event) {
+    // Replace ${data.fieldName} with values from event.data
+    let result = template.replace(/\$\{data\.(\w+)\}/g, (match, fieldName) => {
+      if (event.data && event.data[fieldName] !== undefined) {
+        return event.data[fieldName];
+      }
+      return match; // Keep placeholder if no value found
+    });
+
+    // Replace ${version} with event.version
+    result = result.replace(/\$\{version\}/g, () => {
+      if (event.version !== undefined) {
+        return event.version;
+      }
+      return '${version}'; // Keep placeholder if no value
+    });
+
+    // Replace ${artifactId} with event.artifactId
+    result = result.replace(/\$\{artifactId\}/g, () => {
+      if (event.artifactId !== undefined) {
+        return event.artifactId;
+      }
+      return '${artifactId}'; // Keep placeholder if no value
+    });
+
+    // Replace ${contractId} with event.contractId
+    result = result.replace(/\$\{contractId\}/g, () => {
+      if (event.contractId !== undefined) {
+        return event.contractId;
+      }
+      return '${contractId}'; // Keep placeholder if no value
+    });
+
+    return result;
+  }
+}
+
+// Export the PipelineEngine class
+module.exports = PipelineEngine;