@0xsequence/catapult 1.1.0

package/src/lib/core/graph.ts

@@ -0,0 +1,252 @@
+// src/lib/core/graph.ts
+import { Job, Template, isJobCompletedCondition } from '../types'
+import { isPrimitiveActionType } from '../types/actions'
+
+/**
+ * Represents the complete dependency graph of all jobs in a project.
+ * It is responsible for parsing job and template dependencies, detecting
+ * cycles, and providing a valid execution order.
+ */
+export class DependencyGraph {
+  private graph: Map<string, Set<string>> = new Map()
+  private executionOrder: string[] = []
+  private allJobNames: Set<string>
+
+  constructor(
+    private readonly jobs: Map<string, Job>,
+    private readonly templates: Map<string, Template>,
+  ) {
+    this.allJobNames = new Set(this.jobs.keys())
+    this.build()
+    this.checkForCycles()
+    this.executionOrder = this.topologicalSort()
+  }
+
+  /**
+   * Returns the list of job names in a valid execution order.
+   * Jobs with no dependencies come first.
+   */
+  public getExecutionOrder(): string[] {
+    return this.executionOrder
+  }
+
+  /**
+   * Returns the direct and transitive dependencies for a given job.
+   */
+  public getDependencies(jobName: string): Set<string> {
+    return this.graph.get(jobName) || new Set()
+  }
+
+  /**
+   * Populates the dependency graph by analyzing each job.
+   */
+  private build(): void {
+    for (const jobName of this.allJobNames) {
+      this.graph.set(jobName, this.findAllDependencies(jobName))
+    }
+  }
+
+  /**
+   * Recursively finds all dependencies for a given job, including those
+   * from its `depends_on` list and those hidden within its templates' setup blocks.
+   *
+   * @param jobName The name of the job to analyze.
+   * @param visited A set to track visited jobs in the current path to detect cycles.
+   * @returns A set of all job names that the given job depends on.
+   */
+  private findAllDependencies(jobName: string, visited: Set<string> = new Set()): Set<string> {
+    if (visited.has(jobName)) {
+      // This path is cyclic. The cycle will be properly reported by `checkForCycles`.
+      // Here, we just stop the recursion to prevent an infinite loop.
+      return new Set()
+    }
+    visited.add(jobName)
+
+    const job = this.jobs.get(jobName)
+    if (!job) {
+      throw new Error(`Integrity error: Job "${jobName}" not found during graph build.`)
+    }
+
+    const directDependencies = new Set<string>()
+
+    // 1. Add dependencies from the job's `depends_on` field.
+    job.depends_on?.forEach(dep => directDependencies.add(dep))
+
+    // 2. Find dependencies within the templates used by the job's actions.
+    for (const action of job.actions) {
+      // Get the template or type name
+      const templateName = action.template || action.type
+      if (!templateName) {
+        throw new Error(`Invalid configuration: Action in job "${jobName}" has no template or type field.`)
+      }
+      
+      // If the action is a primitive, it cannot have job dependencies. Skip it.
+      if (isPrimitiveActionType(templateName)) {
+        continue
+      }
+      
+      const template = this.templates.get(templateName)
+      if (!template) {
+        throw new Error(`Invalid configuration: Template "${templateName}" used by job "${jobName}" not found.`)
+      }
+
+      // Recursively find dependencies in the template's setup block.
+      this.findTemplateSetupDependencies(template).forEach(dep => directDependencies.add(dep))
+    }
+
+    // 3. Now, for each direct dependency, find its transitive dependencies.
+    const allDependencies = new Set<string>(directDependencies)
+    for (const dep of directDependencies) {
+      if (!this.allJobNames.has(dep)) {
+        throw new Error(`Invalid dependency: Job "${jobName}" depends on "${dep}", which does not exist.`)
+      }
+      // The `visited` set is passed down to detect cycles across calls.
+      const transitiveDeps = this.findAllDependencies(dep, new Set(visited))
+      transitiveDeps.forEach(transDep => allDependencies.add(transDep))
+    }
+
+    return allDependencies
+  }
+
+  /**
+   * Helper to extract job dependencies from a template's setup block.
+   */
+  private findTemplateSetupDependencies(template: Template): Set<string> {
+    const dependencies = new Set<string>()
+    const setup = template.setup
+    if (!setup) return dependencies
+
+    // Case 1: Dependencies from `skip_condition` (e.g., `job-completed`).
+    setup.skip_condition?.forEach(condition => {
+      if (isJobCompletedCondition(condition)) {
+        dependencies.add(condition.arguments.job)
+      }
+    })
+
+    // Case 2: Dependencies from setup actions that are themselves templates.
+    setup.actions?.forEach(action => {
+      // Ignore primitive actions within a setup block.
+      if (isPrimitiveActionType(action.type)) {
+        return
+      }
+
+      // In your YAML, setup blocks sometimes call other templates directly.
+      // e.g. `sequence-universal-deployer-2`'s setup calls `nano-universal-deployer`
+      // Here, `action.type` IS the template name.
+      const actionTemplate = this.templates.get(action.type)
+      if (actionTemplate) {
+        // This is a nested template call, so we must find its dependencies too.
+        this.findTemplateSetupDependencies(actionTemplate).forEach(dep => dependencies.add(dep))
+      }
+    })
+
+    return dependencies
+  }
+
+  /**
+   * Checks the entire graph for circular dependencies.
+   * @throws {Error} if a cycle is detected.
+   */
+  private checkForCycles(): void {
+    for (const [jobName, dependencies] of this.graph.entries()) {
+      if (dependencies.has(jobName)) {
+        // To provide a more helpful error, we need to find the actual path.
+        const path = this.findPath(jobName, jobName)
+        throw new Error(`Circular dependency detected: ${path.join(' -> ')}`)
+      }
+    }
+  }
+
+  /**
+   * Helper to find a dependency path from a start node to an end node.
+   * Used for creating helpful error messages for cycles.
+   */
+  private findPath(start: string, end: string, visited: Set<string> = new Set()): string[] {
+    visited.add(start)
+    const job = this.jobs.get(start)
+    if (!job) return []
+
+    const directDependencies = new Set(job.depends_on || [])
+    for (const action of job.actions) {
+      const templateName = action.template || action.type
+      if (!templateName || isPrimitiveActionType(templateName)) {
+        continue
+      }
+      const template = this.templates.get(templateName)!
+      this.findTemplateSetupDependencies(template).forEach(dep => directDependencies.add(dep))
+    }
+    
+    if (directDependencies.has(end)) {
+        return [start, end]
+    }
+
+    for (const dep of directDependencies) {
+        if (!visited.has(dep)) {
+            const path = this.findPath(dep, end, visited)
+            if (path.length > 0) {
+                return [start, ...path]
+            }
+        }
+    }
+
+    return []
+  }
+
+  /**
+   * Performs a topological sort on the graph to determine execution order.
+   * Uses Kahn's algorithm.
+   */
+  private topologicalSort(): string[] {
+    const inDegree = new Map<string, number>()
+    const sorted: string[] = []
+
+    // This map stores which jobs depend on a given key.
+    // e.g., `adjacency.get('A')` -> `['B', 'C']` means B and C depend on A.
+    const adjacency = new Map<string, string[]>()
+
+    // Initialize in-degrees and adjacency list for all jobs.
+    for (const jobName of this.allJobNames) {
+      inDegree.set(jobName, 0)
+      adjacency.set(jobName, [])
+    }
+    
+    // Build the inverted graph (adjacency list) and calculate initial in-degrees.
+    for (const [jobName, dependencies] of this.graph.entries()) {
+      inDegree.set(jobName, dependencies.size)
+      for (const dep of dependencies) {
+        // `jobName` depends on `dep`, so `dep` is a prerequisite for `jobName`.
+        adjacency.get(dep)?.push(jobName)
+      }
+    }
+
+    // Initialize queue with nodes having an in-degree of 0.
+    const queue: string[] = []
+    for (const [jobName, degree] of inDegree.entries()) {
+      if (degree === 0) {
+        queue.push(jobName)
+      }
+    }
+
+    // Process the queue.
+    while (queue.length > 0) {
+      const current = queue.shift()!
+      sorted.push(current)
+      
+      const dependents = adjacency.get(current) || []
+      for (const dependent of dependents) {
+        const newDegree = (inDegree.get(dependent) || 1) - 1
+        inDegree.set(dependent, newDegree)
+        if (newDegree === 0) {
+          queue.push(dependent)
+        }
+      }
+    }
+
+    if (sorted.length !== this.allJobNames.size) {
+      // This should theoretically be caught by `checkForCycles`, but it's good defense.
+      throw new Error('Topological sort failed. The graph likely has a cycle.')
+    }
+
+    return sorted
+  }
+}

package/src/lib/core/loader.ts

@@ -0,0 +1,263 @@
+import * as fs from 'fs/promises'
+import * as path from 'path'
+import { parseJob, parseTemplate } from '../parsers'
+import { Job, Template } from '../types'
+import { ContractRepository } from '../contracts/repository'
+import { parseConstants } from '../parsers/constants'
+
+export interface ProjectLoaderOptions {
+  loadStdTemplates?: boolean
+}
+
+export class ProjectLoader {
+  public jobs: Map<string, Job> = new Map()
+  public templates: Map<string, Template> = new Map()
+  public readonly contractRepository: ContractRepository
+
+  // Top-level constants registry
+  public constants: Map<string, any> = new Map()
+  // Track source files for constants for duplicate reporting
+  private constantSources: Map<string, string> = new Map()
+
+  constructor(
+    private readonly projectRoot: string,
+    private readonly options: ProjectLoaderOptions = {}
+  ) {
+    this.contractRepository = new ContractRepository()
+  }
+
+  async load() {
+    // Load all contracts from the project root first
+    await this.contractRepository.loadFrom(this.projectRoot)
+
+    // Load standard library templates (unless disabled)
+    if (this.options.loadStdTemplates !== false) {
+      const stdTemplatePath = path.resolve(__dirname, '..', 'std', 'templates')
+      if (await this.pathExists(stdTemplatePath)) {
+        await this.loadTemplatesFromDir(stdTemplatePath)
+      }
+    }
+    
+    // Load user-defined templates
+    const userTemplatePath = path.join(this.projectRoot, 'templates')
+    if (await this.pathExists(userTemplatePath)) {
+        await this.loadTemplatesFromDir(userTemplatePath)
+    }
+
+    // Load jobs
+    const jobsPath = path.join(this.projectRoot, 'jobs')
+    if (await this.pathExists(jobsPath)) {
+        await this.loadJobsFromDir(jobsPath)
+    }
+
+    // Load templates from within job directories
+    if (await this.pathExists(jobsPath)) {
+        await this.loadTemplatesFromJobDirs(jobsPath)
+    }
+
+    // Load top-level constants from anywhere in project root
+    await this.loadConstantsFromDir(this.projectRoot)
+  }
+
+  private async loadTemplatesFromDir(dir: string) {
+    const templateFiles = await this.findTemplateFiles(dir)
+    for (const filePath of templateFiles) {
+      try {
+        const content = await fs.readFile(filePath, 'utf-8')
+        const template = parseTemplate(content)
+        template._path = filePath
+        this.templates.set(template.name, template)
+      } catch (error) {
+        // Surface YAML/template parsing errors so malformed templates fail the load
+        if (error instanceof Error && (error.message.startsWith('Failed to parse template YAML:') || error.message.startsWith('Invalid template'))) {
+          throw new Error(`Template load error in ${filePath}: ${error.message}`)
+        }
+        // If it's a file system error (e.g., permission), rethrow to make it visible as well
+        if (error instanceof Error && (error as any).code) {
+          throw new Error(`Failed to read template file ${filePath}: ${(error as any).code} ${error.message}`)
+        }
+        // Otherwise rethrow to avoid silently ignoring real issues
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Recursively finds all template files (.yaml/.yml) in a directory.
+   */
+  private async findTemplateFiles(dir: string, ignoreDirs: Set<string> = new Set(['node_modules', 'dist', '.git', '.idea', '.vscode'])): Promise<string[]> {
+    let results: string[] = []
+    try {
+      const list = await fs.readdir(dir, { withFileTypes: true })
+
+      for (const dirent of list) {
+        const fullPath = path.resolve(dir, dirent.name)
+        if (dirent.isDirectory()) {
+          if (!ignoreDirs.has(dirent.name)) {
+            results = results.concat(await this.findTemplateFiles(fullPath, ignoreDirs))
+          }
+        } else if (dirent.isFile() && (dirent.name.endsWith('.yaml') || dirent.name.endsWith('.yml'))) {
+          results.push(fullPath)
+        }
+      }
+    } catch (err) {
+      // Ignore errors from trying to read directories we don't have access to, etc.
+    }
+    return results
+  }
+
+  private async loadJobsFromDir(dir: string) {
+    const jobFiles = await this.findJobFiles(dir)
+    for (const filePath of jobFiles) {
+      try {
+        const content = await fs.readFile(filePath, 'utf-8')
+        const raw: any = (() => { try { return require('yaml').parse(content) } catch { return {} } })()
+        if (raw && typeof raw === 'object' && raw.type === 'template') {
+          // This is actually a template file; load through the template path to avoid misclassification
+          const template = parseTemplate(content)
+          template._path = filePath
+          this.templates.set(template.name, template)
+          continue
+        }
+
+        const job = parseJob(content)
+        // Capture optional job-level constants by peeking the raw YAML for "constants"
+        try {
+          const raw = JSON.parse(JSON.stringify(require('yaml').parse(content)))
+          if (raw && typeof raw === 'object' && raw.constants !== undefined) {
+            if (typeof raw.constants !== 'object' || Array.isArray(raw.constants)) {
+              throw new Error(`Invalid job "${job.name}": "constants" field must be an object if provided.`)
+            }
+            (job as any).constants = raw.constants
+          }
+        } catch (peekErr) {
+          // parseJob already validated YAML; if this peek fails due to YAML parsing, surface it
+          if (peekErr instanceof Error && peekErr.message.includes('Failed to parse')) {
+            throw new Error(`Job constants peek failed for ${filePath}: ${peekErr.message}`)
+          }
+          // Otherwise ignore non-YAML related errors here
+        }
+        job._path = filePath
+        this.jobs.set(job.name, job)
+      } catch (error) {
+        // If job YAML is malformed or invalid, skip this job but continue loading others
+        if (error instanceof Error && (error.message.startsWith('Failed to parse job YAML:') || error.message.startsWith('Invalid job'))) {
+          console.warn(`Skipping malformed job at ${filePath}: ${error.message}`)
+          continue
+        }
+        // If it's a file system error (e.g., permission), rethrow to make it visible as well
+        if (error instanceof Error && (error as any).code) {
+          throw new Error(`Failed to read job file ${filePath}: ${(error as any).code} ${error.message}`)
+        }
+        // For other unexpected errors, rethrow
+        throw error
+      }
+    }
+  }
+
+  /**
+   * Recursively finds all job files (.yaml/.yml) in a directory.
+   */
+  private async findJobFiles(dir: string, ignoreDirs: Set<string> = new Set(['node_modules', 'dist', '.git', '.idea', '.vscode'])): Promise<string[]> {
+    let results: string[] = []
+    try {
+      const list = await fs.readdir(dir, { withFileTypes: true })
+
+      for (const dirent of list) {
+        const fullPath = path.resolve(dir, dirent.name)
+        if (dirent.isDirectory()) {
+          if (!ignoreDirs.has(dirent.name)) {
+            results = results.concat(await this.findJobFiles(fullPath, ignoreDirs))
+          }
+        } else if (dirent.isFile() && (dirent.name.endsWith('.yaml') || dirent.name.endsWith('.yml'))) {
+          results.push(fullPath)
+        }
+      }
+    } catch (err) {
+      // Ignore errors from trying to read directories we don't have access to, etc.
+    }
+    return results
+  }
+
+  /**
+   * Loads templates from within job directories by scanning for 'templates' subdirectories.
+   */
+  private async loadTemplatesFromJobDirs(jobsRootDir: string) {
+    await this.findAndLoadTemplatesInJobDirs(jobsRootDir)
+  }
+
+  /**
+   * Recursively searches for 'templates' directories within job directories and loads templates from them.
+   */
+  private async findAndLoadTemplatesInJobDirs(dir: string, ignoreDirs: Set<string> = new Set(['node_modules', 'dist', '.git', '.idea', '.vscode'])): Promise<void> {
+    try {
+      const list = await fs.readdir(dir, { withFileTypes: true })
+
+      for (const dirent of list) {
+        const fullPath = path.resolve(dir, dirent.name)
+        if (dirent.isDirectory()) {
+          if (!ignoreDirs.has(dirent.name)) {
+            // If this directory is named 'templates', load templates from it
+            if (dirent.name === 'templates') {
+              await this.loadTemplatesFromDir(fullPath)
+            }
+            // Continue recursively searching for more template directories
+            await this.findAndLoadTemplatesInJobDirs(fullPath, ignoreDirs)
+          }
+        }
+      }
+    } catch (err) {
+      // Ignore errors from trying to read directories we don't have access to, etc.
+    }
+  }
+
+  /**
+   * Load and merge all top-level constants from any YAML file with type: "constants"
+   * located anywhere under the given directory.
+   */
+  private async loadConstantsFromDir(dir: string, ignoreDirs: Set<string> = new Set(['node_modules', 'dist', '.git', '.idea', '.vscode'])): Promise<void> {
+    try {
+      const list = await fs.readdir(dir, { withFileTypes: true })
+      for (const dirent of list) {
+        const fullPath = path.resolve(dir, dirent.name)
+        if (dirent.isDirectory()) {
+          if (!ignoreDirs.has(dirent.name)) {
+            await this.loadConstantsFromDir(fullPath, ignoreDirs)
+          }
+        } else if (dirent.isFile() && (dirent.name.endsWith('.yaml') || dirent.name.endsWith('.yml'))) {
+          try {
+            const content = await fs.readFile(fullPath, 'utf-8')
+            const constantsDoc = parseConstants(content)
+            if (constantsDoc) {
+              for (const [key, value] of Object.entries(constantsDoc.constants)) {
+                if (this.constants.has(key)) {
+                  const prevSource = this.constantSources.get(key)
+                  throw new Error(`Duplicate constant "${key}" found in ${fullPath}${prevSource ? ` (previously defined in ${prevSource})` : ''}`)
+                }
+                this.constants.set(key, value)
+                this.constantSources.set(key, fullPath)
+              }
+            }
+          } catch (err) {
+            // For constants files, surface parsing errors to fail fast
+            if (err instanceof Error && (err.message.startsWith('Failed to parse constants YAML:') || err.message.startsWith('Invalid constants'))) {
+              throw new Error(`Constants load error in ${fullPath}: ${err.message}`)
+            }
+            // Otherwise, ignore if not a constants file
+          }
+        }
+      }
+    } catch (err) {
+      // Ignore directory read errors (like permission or non-existent)
+    }
+  }
+
+  private async pathExists(p: string): Promise<boolean> {
+      try {
+          await fs.access(p)
+          return true
+      } catch {
+          return false
+      }
+  }
+}