@ryanfw/prompt-orchestration-pipeline 1.3.2 → 1.3.4

package/src/core/config.ts

@@ -1,12 +1,13 @@
 import path from "node:path";
-import { readFile, stat } from "node:fs/promises";
-import { readFileSync, existsSync } from "node:fs";
+import { resolveAllPipelinesSync, resolvePipelineSync } from "../config/pipeline-registry.ts";
 
 export interface OrchestratorConfig {
   shutdownTimeout: number;
   processSpawnRetries: number;
   processSpawnRetryDelay: number;
   lockFileTimeout: number;
+  staleRunningGraceMs: number;
+  staleLeaseTimeoutMs: number;
   watchDebounce: number;
   watchStabilityThreshold: number;
   watchPollInterval: number;
@@ -23,6 +24,8 @@ export interface TaskRunnerConfig {
 export interface LLMConfig {
   defaultProvider: string;
   defaultModel: string;
+  internalProvider: string;
+  internalModel: string;
   maxConcurrency: number;
   retryMaxAttempts: number;
   retryBackoffMs: number;
@@ -35,14 +38,6 @@ export interface UIConfig {
   maxRecentChanges: number;
 }
 
-export interface PathsConfig {
-  root: string;
-  dataDir: string;
-  pendingDir: string;
-  currentDir: string;
-  completeDir: string;
-}
-
 export interface PipelineEntry {
   configDir: string;
   tasksDir: string;
@@ -67,17 +62,11 @@ export interface AppConfig {
   taskRunner: TaskRunnerConfig;
   llm: LLMConfig;
   ui: UIConfig;
-  paths: PathsConfig;
   pipelines: Record<string, PipelineEntry>;
   validation: ValidationConfig;
   logging: LoggingConfig;
 }
 
-export interface LoadConfigOptions {
-  configPath?: string;
-  validate?: boolean;
-}
-
 export interface PipelineConfigResult {
   pipelineJsonPath: string;
   tasksDir: string;
@@ -89,6 +78,8 @@ export const defaultConfig = {
     processSpawnRetries: 3,
     processSpawnRetryDelay: 1000,
     lockFileTimeout: 30000,
+    staleRunningGraceMs: 30000,
+    staleLeaseTimeoutMs: 30000,
     watchDebounce: 500,
     watchStabilityThreshold: 1000,
     watchPollInterval: 100,
@@ -103,6 +94,8 @@ export const defaultConfig = {
   llm: {
     defaultProvider: "openai",
     defaultModel: "gpt-4o",
+    internalProvider: "deepseek",
+    internalModel: "deepseek-v4-flash",
     maxConcurrency: 5,
     retryMaxAttempts: 3,
     retryBackoffMs: 1000,
@@ -113,13 +106,6 @@ export const defaultConfig = {
     heartbeatInterval: 30000,
     maxRecentChanges: 50,
   },
-  paths: {
-    root: process.cwd(),
-    dataDir: "data",
-    pendingDir: "data/pending",
-    currentDir: "data/current",
-    completeDir: "data/complete",
-  },
   pipelines: {},
   validation: {
     seedNameMinLength: 3,
@@ -133,6 +119,13 @@ export const defaultConfig = {
   },
 } satisfies AppConfig;
 
+type DeepReadonly<T> =
+  T extends (infer U)[]
+    ? ReadonlyArray<DeepReadonly<U>>
+    : T extends object
+      ? { readonly [K in keyof T]: DeepReadonly<T[K]> }
+      : T;
+
 // ─── Deep Merge ───────────────────────────────────────────────────────────────
 
 type PlainObject = Record<string, unknown>;
@@ -162,162 +155,161 @@ function deepMerge(target: PlainObject, source: PlainObject): PlainObject {
 
 const VALID_LOG_LEVELS = ["debug", "info", "warn", "error"] as const;
 
-function loadFromEnvironment(config: AppConfig): AppConfig {
-  const overrides: PlainObject = {};
+function envInt(
+  name: string,
+  raw: string | undefined,
+  opts?: { min?: number; max?: number }
+): number | undefined {
+  if (raw === undefined) return undefined;
+  if (raw === "") {
+    throw new Error(`${name} must not be empty`);
+  }
+  const n = Number(raw);
+  const min = opts?.min;
+  const max = opts?.max;
+  if (min !== undefined || max !== undefined) {
+    if (!Number.isInteger(n) || (min !== undefined && n < min) || (max !== undefined && n > max)) {
+      if (min !== undefined && max !== undefined) {
+        throw new Error(`${name} must be between ${min} and ${max}, got ${raw}`);
+      }
+      if (min !== undefined) {
+        throw new Error(`${name} must be an integer >= ${min}, got ${raw}`);
+      }
+      throw new Error(`${name} must be an integer <= ${max}, got ${raw}`);
+    }
+    return n;
+  }
+  if (!Number.isInteger(n)) {
+    throw new Error(`${name} must be an integer, got ${raw}`);
+  }
+  return n;
+}
 
-  const root = process.env["PO_ROOT"] ? path.resolve(process.env["PO_ROOT"]) : undefined;
-  const dataDir = process.env["PO_DATA_DIR"];
-  if (root || dataDir) {
-    overrides["paths"] = {
-      ...(root ? { root } : {}),
-      ...(dataDir ? { dataDir } : {}),
-    };
+function envString(name: string, raw: string | undefined): string | undefined {
+  if (raw === undefined) return undefined;
+  if (raw === "") {
+    throw new Error(`${name} must not be empty`);
   }
+  return raw;
+}
 
-  const portRaw = process.env["PORT"];
-  const poPortRaw = process.env["PO_UI_PORT"];
+function loadFromEnvironment(config: AppConfig): AppConfig {
+  const overrides: PlainObject = {};
+
+  const port = envInt("PORT", process.env["PORT"], { min: 1, max: 65535 });
+  const poPort = envInt("PO_UI_PORT", process.env["PO_UI_PORT"], { min: 1, max: 65535 });
   const host = process.env["PO_HOST"];
   const uiOverrides: PlainObject = {};
-  if (portRaw) uiOverrides["port"] = parseInt(portRaw, 10);
-  if (poPortRaw) uiOverrides["port"] = parseInt(poPortRaw, 10);
+  if (port !== undefined) uiOverrides["port"] = port;
+  if (poPort !== undefined) uiOverrides["port"] = poPort;
   if (host) uiOverrides["host"] = host;
   if (Object.keys(uiOverrides).length > 0) overrides["ui"] = uiOverrides;
 
-  const maxConcurrencyRaw = process.env["PO_MAX_CONCURRENCY"];
-  if (maxConcurrencyRaw) {
-    overrides["llm"] = { maxConcurrency: parseInt(maxConcurrencyRaw, 10) };
-  }
-
-  const maxAttemptsRaw = process.env["PO_TASK_MAX_ATTEMPTS"];
-  if (maxAttemptsRaw) {
-    const maxAttempts = Number(maxAttemptsRaw);
-    if (!Number.isInteger(maxAttempts) || maxAttempts < 1) {
-      throw new Error(`PO_TASK_MAX_ATTEMPTS must be an integer >= 1, got ${maxAttemptsRaw}`);
-    }
+  const maxConcurrency = envInt("PO_MAX_CONCURRENCY", process.env["PO_MAX_CONCURRENCY"], { min: 1 });
+  const internalProvider = envString("PO_INTERNAL_LLM_PROVIDER", process.env["PO_INTERNAL_LLM_PROVIDER"]);
+  const internalModel = envString("PO_INTERNAL_LLM_MODEL", process.env["PO_INTERNAL_LLM_MODEL"]);
+  const llmOverrides: PlainObject = {};
+  if (maxConcurrency !== undefined) llmOverrides["maxConcurrency"] = maxConcurrency;
+  if (internalProvider !== undefined) llmOverrides["internalProvider"] = internalProvider;
+  if (internalModel !== undefined) llmOverrides["internalModel"] = internalModel;
+  if (Object.keys(llmOverrides).length > 0) overrides["llm"] = llmOverrides;
+
+  const maxAttempts = envInt("PO_TASK_MAX_ATTEMPTS", process.env["PO_TASK_MAX_ATTEMPTS"], { min: 1 });
+  if (maxAttempts !== undefined) {
     overrides["taskRunner"] = {
       ...((overrides["taskRunner"] as PlainObject | undefined) ?? {}),
       maxAttempts,
     };
   }
 
-  const shutdownTimeoutRaw = process.env["PO_SHUTDOWN_TIMEOUT"];
-  if (shutdownTimeoutRaw) {
+  const shutdownTimeout = envInt("PO_SHUTDOWN_TIMEOUT", process.env["PO_SHUTDOWN_TIMEOUT"], { min: 0 });
+  if (shutdownTimeout !== undefined) {
     overrides["orchestrator"] = {
       ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
-      shutdownTimeout: parseInt(shutdownTimeoutRaw, 10),
+      shutdownTimeout,
     };
   }
 
-  const maxRunningJobsRaw = process.env["PO_MAX_RUNNING_JOBS"];
-  if (maxRunningJobsRaw !== undefined) {
-    const maxConcurrentJobs = Number(maxRunningJobsRaw);
-    if (!Number.isInteger(maxConcurrentJobs) || maxConcurrentJobs < 1) {
-      throw new Error(
-        `orchestrator.maxConcurrentJobs must be a positive integer (PO_MAX_RUNNING_JOBS), got ${maxRunningJobsRaw}`
-      );
-    }
+  const maxRunningJobs = envInt("PO_MAX_RUNNING_JOBS", process.env["PO_MAX_RUNNING_JOBS"], { min: 1 });
+  if (maxRunningJobs !== undefined) {
     overrides["orchestrator"] = {
       ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
-      maxConcurrentJobs,
+      maxConcurrentJobs: maxRunningJobs,
     };
   }
 
-  const logLevel = process.env["PO_LOG_LEVEL"];
-  if (logLevel && (VALID_LOG_LEVELS as readonly string[]).includes(logLevel)) {
-    overrides["logging"] = { level: logLevel };
+  const staleRunningGraceMs = envInt(
+    "PO_STALE_RUNNING_GRACE_MS",
+    process.env["PO_STALE_RUNNING_GRACE_MS"],
+    { min: 1 }
+  );
+  if (staleRunningGraceMs !== undefined) {
+    overrides["orchestrator"] = {
+      ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
+      staleRunningGraceMs,
+    };
   }
 
-  return deepMerge(config as unknown as PlainObject, overrides) as unknown as AppConfig;
-}
+  const staleLeaseTimeoutMs = envInt(
+    "PO_STALE_LEASE_TIMEOUT_MS",
+    process.env["PO_STALE_LEASE_TIMEOUT_MS"],
+    { min: 1 }
+  );
+  if (staleLeaseTimeoutMs !== undefined) {
+    overrides["orchestrator"] = {
+      ...((overrides["orchestrator"] as PlainObject | undefined) ?? {}),
+      staleLeaseTimeoutMs,
+    };
+  }
 
-async function loadFromFile(filePath: string): Promise<Record<string, unknown> | null> {
-  let text: string;
-  try {
-    text = await readFile(filePath, "utf8");
-  } catch (error) {
-    if ((error as NodeJS.ErrnoException).code === "ENOENT") {
-      return null;
+  const logLevel = process.env["PO_LOG_LEVEL"];
+  if (logLevel !== undefined) {
+    if (!(VALID_LOG_LEVELS as readonly string[]).includes(logLevel)) {
+      throw new Error(`PO_LOG_LEVEL must be one of ${VALID_LOG_LEVELS.join(", ")}, got ${logLevel}`);
     }
-    throw new Error(`Failed to load config file: ${filePath}`);
+    overrides["logging"] = { level: logLevel as "debug" | "info" | "warn" | "error" };
   }
 
-  try {
-    return JSON.parse(text) as Record<string, unknown>;
-  } catch (err) {
-    throw new Error(`Failed to load config file: ${filePath}: ${String(err)}`);
-  }
+  return deepMerge(config as unknown as PlainObject, overrides) as unknown as AppConfig;
 }
 
 function validateConfig(config: AppConfig): void {
-  const errors: string[] = [];
-
-  if (config.ui.port < 1 || config.ui.port > 65535) {
-    errors.push(`ui.port must be between 1 and 65535, got ${config.ui.port}`);
-  }
-  if (config.llm.maxConcurrency < 1) {
-    errors.push(`llm.maxConcurrency must be >= 1, got ${config.llm.maxConcurrency}`);
-  }
   if (config.taskRunner.maxRefinementAttempts < 1) {
-    errors.push(`taskRunner.maxRefinementAttempts must be >= 1, got ${config.taskRunner.maxRefinementAttempts}`);
-  }
-  if (!Number.isInteger(config.taskRunner.maxAttempts) || config.taskRunner.maxAttempts < 1) {
-    errors.push(`taskRunner.maxAttempts must be an integer >= 1, got ${config.taskRunner.maxAttempts}`);
-  }
-  if (!Number.isInteger(config.orchestrator.maxConcurrentJobs) || config.orchestrator.maxConcurrentJobs < 1) {
-    errors.push(`orchestrator.maxConcurrentJobs must be a positive integer, got ${config.orchestrator.maxConcurrentJobs}`);
-  }
-  if (!(VALID_LOG_LEVELS as readonly string[]).includes(config.logging.level)) {
-    errors.push(`logging.level must be one of ${VALID_LOG_LEVELS.join(", ")}, got ${config.logging.level}`);
+    throw new Error(
+      `taskRunner.maxRefinementAttempts must be >= 1, got ${config.taskRunner.maxRefinementAttempts}`
+    );
   }
+}
 
-  if (errors.length > 0) {
-    throw new Error(`Configuration validation failed: ${errors.join("; ")}`);
+function deepFreeze<T>(value: T): DeepReadonly<T> {
+  if (value === null || value === undefined) return value as DeepReadonly<T>;
+  if (typeof value !== "object") return value as DeepReadonly<T>;
+  Object.freeze(value);
+  if (Array.isArray(value)) {
+    for (const item of value) {
+      if (item !== null && typeof item === "object") {
+        deepFreeze(item);
+      }
+    }
+    return value as DeepReadonly<T>;
   }
+  for (const key of Object.keys(value as object)) {
+    const v = (value as Record<string, unknown>)[key];
+    if (v !== null && typeof v === "object") {
+      deepFreeze(v);
+    }
+  }
+  return value as DeepReadonly<T>;
 }
 
 // ─── Module State ─────────────────────────────────────────────────────────────
 
-let cachedConfig: AppConfig | null = null;
+let cachedConfig: DeepReadonly<AppConfig> | null = null;
 
 // ─── Registry Hydration ───────────────────────────────────────────────────────
 
-interface RegistryPipelines {
-  pipelines: Record<string, { configDir?: string; tasksDir?: string }>;
-}
-
-interface LegacyRegistry {
-  slugs: string[];
-}
-
-async function hydratePipelinesFromRegistry(config: AppConfig, registryPath: string): Promise<void> {
-  let raw: RegistryPipelines | LegacyRegistry;
-  try {
-    raw = JSON.parse(await readFile(registryPath, "utf8")) as RegistryPipelines | LegacyRegistry;
-  } catch (err) {
-    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
-      return;
-    }
-    throw new Error(`Failed to read pipeline registry: ${err instanceof Error ? err.message : String(err)}`);
-  }
-
-  if ("slugs" in raw) {
-    console.warn(`[config] Legacy registry format detected at ${registryPath} — skipping`);
-    return;
-  }
-
-  const root = config.paths.root;
-  for (const [slug, entry] of Object.entries(raw.pipelines)) {
-    const resolvedConfigDir = entry.configDir
-      ? (path.isAbsolute(entry.configDir) ? entry.configDir : path.join(root, entry.configDir))
-      : path.join(root, "pipeline-config", slug);
-    const resolvedTasksDir = entry.tasksDir
-      ? (path.isAbsolute(entry.tasksDir) ? entry.tasksDir : path.join(root, entry.tasksDir))
-      : path.join(resolvedConfigDir, "tasks");
-    config.pipelines[slug] = {
-      configDir: resolvedConfigDir,
-      tasksDir: resolvedTasksDir,
-    };
-  }
+function applyPipelineRegistryEntriesSync(config: AppConfig, poRoot: string): void {
+  Object.assign(config.pipelines, resolveAllPipelinesSync(poRoot));
 }
 
 // ─── Public API ───────────────────────────────────────────────────────────────
@@ -326,52 +318,7 @@ export function resetConfig(): void {
   cachedConfig = null;
 }
 
-export async function loadConfig(options?: LoadConfigOptions): Promise<AppConfig> {
-  let config = JSON.parse(JSON.stringify(defaultConfig)) as AppConfig;
-
-  if (options?.configPath) {
-    const fileData = await loadFromFile(options.configPath);
-    if (fileData) {
-      config = deepMerge(config as unknown as PlainObject, fileData) as unknown as AppConfig;
-    }
-  }
-
-  config = loadFromEnvironment(config);
-
-  if (!process.env["PO_ROOT"]) {
-    throw new Error("PO_ROOT is required");
-  }
-
-  await hydratePipelinesFromRegistry(config, path.join(config.paths.root, "pipeline-config", "registry.json"));
-
-  if (Object.keys(config.pipelines).length === 0) {
-    throw new Error("No pipelines are registered");
-  } else {
-    const errors: string[] = [];
-    for (const [slug, entry] of Object.entries(config.pipelines)) {
-      const configDirExists = existsSync(path.join(entry.configDir, "pipeline.json"));
-      const tasksDirExists = await stat(entry.tasksDir).then(() => true).catch(() => false);
-      if (!configDirExists) {
-        errors.push(`Pipeline '${slug}': pipeline.json not found`);
-      }
-      if (!tasksDirExists) {
-        errors.push(`Pipeline '${slug}': tasksDir not found`);
-      }
-    }
-    if (errors.length > 0) {
-      throw new Error(errors.join("; "));
-    }
-  }
-
-  if (options?.validate !== false) {
-    validateConfig(config);
-  }
-
-  cachedConfig = config;
-  return config;
-}
-
-export function getConfig(): AppConfig {
+export function getConfig(): DeepReadonly<AppConfig> {
   if (cachedConfig) return cachedConfig;
 
   let config = JSON.parse(JSON.stringify(defaultConfig)) as AppConfig;
@@ -380,37 +327,22 @@ export function getConfig(): AppConfig {
   const rawPoRoot = process.env["PO_ROOT"];
   if (!rawPoRoot) {
     if (process.env["NODE_ENV"] === "test") {
-      cachedConfig = config;
+      validateConfig(config);
+      cachedConfig = deepFreeze(config);
       return cachedConfig;
     }
     throw new Error("PO_ROOT is required");
   }
   const poRoot = path.resolve(rawPoRoot);
 
-  const registryPath = path.join(poRoot, "pipeline-config", "registry.json");
-  if (existsSync(registryPath)) {
-    const raw = JSON.parse(readFileSync(registryPath, "utf8")) as RegistryPipelines | LegacyRegistry;
-    if (!("slugs" in raw)) {
-      for (const [slug, entry] of Object.entries(raw.pipelines)) {
-        const resolvedConfigDir = entry.configDir
-          ? (path.isAbsolute(entry.configDir) ? entry.configDir : path.join(poRoot, entry.configDir))
-          : path.join(poRoot, "pipeline-config", slug);
-        const resolvedTasksDir = entry.tasksDir
-          ? (path.isAbsolute(entry.tasksDir) ? entry.tasksDir : path.join(poRoot, entry.tasksDir))
-          : path.join(resolvedConfigDir, "tasks");
-        config.pipelines[slug] = {
-          configDir: resolvedConfigDir,
-          tasksDir: resolvedTasksDir,
-        };
-      }
-    }
-  }
+  applyPipelineRegistryEntriesSync(config, poRoot);
 
   if (Object.keys(config.pipelines).length === 0 && process.env["NODE_ENV"] !== "test") {
     console.warn("[config] No pipelines found in registry");
   }
 
-  cachedConfig = config;
+  validateConfig(config);
+  cachedConfig = deepFreeze(config);
   return cachedConfig;
 }
 
@@ -435,27 +367,7 @@ export function getConfigValue(dotPath: string, defaultValue?: unknown): unknown
 
 export function getPipelineConfig(slug: string, root?: string): PipelineConfigResult {
   if (root) {
-    const registryPath = path.join(root, "pipeline-config", "registry.json");
-    if (existsSync(registryPath)) {
-      const raw = JSON.parse(readFileSync(registryPath, "utf8")) as RegistryPipelines | LegacyRegistry;
-      if (!("slugs" in raw)) {
-        const entry = raw.pipelines[slug];
-        if (!entry) {
-          throw new Error(`Pipeline '${slug}' not found in registry`);
-        }
-        const resolvedConfigDir = entry.configDir
-          ? (path.isAbsolute(entry.configDir) ? entry.configDir : path.join(root, entry.configDir))
-          : path.join(root, "pipeline-config", slug);
-        const resolvedTasksDir = entry.tasksDir
-          ? (path.isAbsolute(entry.tasksDir) ? entry.tasksDir : path.join(root, entry.tasksDir))
-          : path.join(resolvedConfigDir, "tasks");
-        return {
-          pipelineJsonPath: path.join(resolvedConfigDir, "pipeline.json"),
-          tasksDir: resolvedTasksDir,
-        };
-      }
-    }
-    throw new Error(`Pipeline '${slug}' not found in registry`);
+    return resolvePipelineSync(root, slug);
   }
 
   const config = getConfig();

package/src/core/control.ts

@@ -26,9 +26,12 @@ export const MAX_RUN_TASKS = 64;
 
 export class ControlValidationError extends Error {
   override name = "ControlValidationError";
+  readonly violations: string[];
 
   constructor(violations: string[]) {
+    const normalizedViolations = [...violations];
     super(violations.join("\n"));
+    this.violations = normalizedViolations;
   }
 }