@the-open-engine/zeroshot 5.4.0

package/src/template-validation/index.js

@@ -0,0 +1,338 @@
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { validateConfig } = require('../config-validator');
+const { getConfig } = require('../config-router');
+const TemplateResolver = require('../template-resolver');
+const { SHARED_TRIGGER_SCRIPT } = require('../agents/git-pusher-template');
+const { simulateConsensusGates } = require('./simulate-consensus-gates');
+const { simulateTwoStageValidation } = require('./simulate-two-stage-validation');
+const { simulateRandomTopology } = require('./simulate-random-topology');
+
+const CONDUCTOR_COMPLEXITIES = ['TRIVIAL', 'SIMPLE', 'STANDARD', 'CRITICAL'];
+const CONDUCTOR_TASK_TYPES = ['TASK', 'DEBUG', 'INQUIRY'];
+
+function findJsonFiles(dir) {
+  const files = [];
+  if (!fs.existsSync(dir)) return files;
+
+  for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+    const fullPath = path.join(dir, entry.name);
+    if (entry.isDirectory()) {
+      files.push(...findJsonFiles(fullPath));
+    } else if (entry.name.endsWith('.json')) {
+      files.push(fullPath);
+    }
+  }
+  return files;
+}
+
+function inferTemplateIdFromPath(filePath) {
+  const base = path.basename(filePath, '.json');
+  return base || 'unknown';
+}
+
+async function validateTemplateConfig({
+  config,
+  templateId,
+  deep,
+  randomSampling = false,
+  templatesDir,
+  randomOptions = {},
+}) {
+  const result = validateConfig(config);
+  const isParameterizedTemplate = !!(config?.params && Object.keys(config.params).length > 0);
+
+  if (result.valid) {
+    const simErrors = [];
+    simErrors.push(
+      ...simulateConsensusGates(config, {
+        allowExternalTopics: isParameterizedTemplate
+          ? ['IMPLEMENTATION_READY', 'QUICK_VALIDATION_PASSED']
+          : [],
+      })
+    );
+    if (deep) {
+      simErrors.push(...(await simulateTwoStageValidation({ templateId, config })));
+    }
+    if (randomSampling) {
+      simErrors.push(
+        ...(await simulateRandomTopology({
+          config,
+          templateId,
+          templatesDir,
+          ...randomOptions,
+        }))
+      );
+    }
+    if (simErrors.length > 0) {
+      result.valid = false;
+      result.errors.push(...simErrors);
+    }
+  }
+
+  return result;
+}
+
+function hasCompletionHandler(config) {
+  const agents = Array.isArray(config?.agents) ? config.agents : [];
+  return agents.some(
+    (agent) =>
+      agent.id === 'completion-detector' ||
+      agent.id === 'git-pusher' ||
+      agent.hooks?.onComplete?.config?.topic === 'CLUSTER_COMPLETE' ||
+      agent.triggers?.some((trigger) => trigger.action === 'stop_cluster')
+  );
+}
+
+function mergeAgentsById(primaryConfig, additionalConfig) {
+  const merged = {
+    ...primaryConfig,
+    agents: Array.isArray(primaryConfig?.agents) ? [...primaryConfig.agents] : [],
+  };
+  const seen = new Set(merged.agents.map((agent) => agent.id));
+
+  for (const agent of additionalConfig?.agents || []) {
+    if (seen.has(agent.id)) {
+      continue;
+    }
+    merged.agents.push(agent);
+    seen.add(agent.id);
+  }
+
+  return merged;
+}
+
+function augmentCriticalResolvedConfig({ resolver, config, params }) {
+  if (params?.complexity !== 'CRITICAL' || params?.validator_count !== 0) {
+    return config;
+  }
+
+  const quickValidationConfig = resolver.resolve('quick-validation', {
+    validator_level: params.validator_level,
+    max_tokens: params.max_tokens,
+    timeout: params.timeout || 0,
+  });
+  return mergeAgentsById(config, quickValidationConfig);
+}
+
+function buildSyntheticGitPusher() {
+  return {
+    id: 'git-pusher',
+    role: 'completion-detector',
+    modelLevel: 'level1',
+    triggers: [
+      {
+        topic: 'VALIDATION_RESULT',
+        logic: {
+          engine: 'javascript',
+          script: SHARED_TRIGGER_SCRIPT,
+        },
+        action: 'execute_task',
+      },
+    ],
+    hooks: {
+      onComplete: {
+        action: 'publish_message',
+        config: {
+          topic: 'CLUSTER_COMPLETE',
+          content: {
+            text: 'PR flow completed.',
+          },
+        },
+      },
+    },
+  };
+}
+
+function buildSyntheticCompletionDetector() {
+  return {
+    id: 'completion-detector',
+    role: 'orchestrator',
+    modelLevel: 'level1',
+    triggers: [
+      {
+        topic: 'VALIDATION_RESULT',
+        logic: {
+          engine: 'javascript',
+          script: SHARED_TRIGGER_SCRIPT,
+        },
+        action: 'stop_cluster',
+      },
+    ],
+  };
+}
+
+function buildResolvedRoute({ conductorTemplatePath, resolver, complexity, taskType, autoPr }) {
+  const { base, params } = getConfig(complexity, taskType, { autoPr });
+  const resolved = resolver.resolve(base, params);
+  const augmented = augmentCriticalResolvedConfig({
+    resolver,
+    config: resolved,
+    params,
+  });
+  const configWithCompletion = ensureCompletionHandler(augmented, { autoPr });
+  const modeSuffix = autoPr ? '-autopr' : '';
+
+  return {
+    filePath: `${conductorTemplatePath}#resolved${modeSuffix}:${complexity}-${taskType}`,
+    templateId: `resolved-${base}${modeSuffix}`,
+    config: configWithCompletion,
+  };
+}
+
+function getRouteModes(taskType) {
+  if (taskType === 'INQUIRY') {
+    return [false];
+  }
+
+  return [false, true];
+}
+
+function buildResolvedConductorRoutes(templatesDir) {
+  const conductorTemplatePath = path.join(templatesDir, 'conductor-bootstrap.json');
+  if (!fs.existsSync(conductorTemplatePath)) {
+    return [];
+  }
+
+  const resolver = new TemplateResolver(templatesDir);
+  const routeConfigs = [];
+
+  for (const complexity of CONDUCTOR_COMPLEXITIES) {
+    for (const taskType of CONDUCTOR_TASK_TYPES) {
+      for (const autoPr of getRouteModes(taskType)) {
+        routeConfigs.push(
+          buildResolvedRoute({
+            conductorTemplatePath,
+            resolver,
+            complexity,
+            taskType,
+            autoPr,
+          })
+        );
+      }
+    }
+  }
+
+  return routeConfigs;
+}
+
+function shouldSkipConfig(config) {
+  return !config.agents && !config.name;
+}
+
+function updateValidationSummary(summary, result) {
+  summary.results.push(result);
+  summary.validated++;
+  if (!result.result.valid) {
+    summary.hasErrors = true;
+  }
+}
+
+async function validateConfigEntry({
+  filePath,
+  config,
+  templateId,
+  deep,
+  randomSampling,
+  templatesDir,
+  randomOptions,
+}) {
+  try {
+    const result = await validateTemplateConfig({
+      config,
+      templateId,
+      deep,
+      randomSampling,
+      templatesDir,
+      randomOptions,
+    });
+
+    return { filePath, result };
+  } catch (err) {
+    return {
+      filePath,
+      result: { valid: false, errors: [err.message], warnings: [] },
+    };
+  }
+}
+
+async function validateTemplates({
+  templatesDir,
+  deep = false,
+  randomSampling = false,
+  randomScope = 'resolved',
+  randomOptions = {},
+}) {
+  const templateFiles = [...findJsonFiles(templatesDir)];
+  const resolvedConductorRoutes = buildResolvedConductorRoutes(templatesDir);
+
+  const summary = {
+    hasErrors: false,
+    validated: 0,
+    skipped: 0,
+    results: [],
+  };
+
+  for (const filePath of templateFiles) {
+    const content = fs.readFileSync(filePath, 'utf-8');
+    const config = JSON.parse(content);
+
+    if (shouldSkipConfig(config)) {
+      summary.skipped++;
+      continue;
+    }
+
+    const result = await validateConfigEntry({
+      filePath,
+      config,
+      templateId: inferTemplateIdFromPath(filePath),
+      deep,
+      randomSampling: randomSampling && randomScope === 'all',
+      templatesDir,
+      randomOptions,
+    });
+    updateValidationSummary(summary, result);
+  }
+
+  for (const resolvedRoute of resolvedConductorRoutes) {
+    const result = await validateConfigEntry({
+      filePath: resolvedRoute.filePath,
+      config: resolvedRoute.config,
+      templateId: resolvedRoute.templateId,
+      deep,
+      randomSampling,
+      templatesDir,
+      randomOptions,
+    });
+    updateValidationSummary(summary, result);
+  }
+
+  return {
+    valid: !summary.hasErrors,
+    validated: summary.validated,
+    skipped: summary.skipped,
+    results: summary.results,
+  };
+}
+
+function ensureCompletionHandler(config, options = {}) {
+  const { autoPr = false } = options;
+
+  if (hasCompletionHandler(config)) {
+    return config;
+  }
+
+  return {
+    ...config,
+    agents: [
+      ...(config.agents || []),
+      autoPr ? buildSyntheticGitPusher() : buildSyntheticCompletionDetector(),
+    ],
+  };
+}
+
+module.exports = {
+  validateTemplates,
+  validateTemplateConfig,
+};

package/src/template-validation/simulate-consensus-gates.js

@@ -0,0 +1,324 @@
+const Ledger = require('../ledger');
+const MessageBus = require('../message-bus');
+const LogicEngine = require('../logic-engine');
+
+const STAGE_START_TOPICS = ['IMPLEMENTATION_READY', 'QUICK_VALIDATION_PASSED'];
+const EXTERNAL_STAGE_SENDERS = {
+  IMPLEMENTATION_READY: 'worker',
+  QUICK_VALIDATION_PASSED: 'consensus-coordinator',
+};
+
+function scriptReferencesTopic(logicScript, topic) {
+  return logicScript.includes(`topic: '${topic}'`) || logicScript.includes(`topic: "${topic}"`);
+}
+
+function getRequiredStageTopics(logicScript) {
+  return STAGE_START_TOPICS.filter((topic) => scriptReferencesTopic(logicScript, topic));
+}
+
+function publishStageStartMessages({
+  messageBus,
+  clusterId,
+  producersByTopic,
+  requiredStageTopics,
+  allowExternalTopics,
+}) {
+  let timestamp = Date.now();
+
+  for (const topic of requiredStageTopics) {
+    const producers = Array.from(producersByTopic.get(topic) || []);
+    const sender =
+      producers[0] || (allowExternalTopics.includes(topic) ? EXTERNAL_STAGE_SENDERS[topic] : null);
+    if (!sender) {
+      continue;
+    }
+
+    messageBus.publish({
+      cluster_id: clusterId,
+      topic,
+      sender,
+      timestamp: timestamp++,
+    });
+  }
+}
+
+function collectTopicProducers(config) {
+  const producersByTopic = new Map();
+
+  for (const agent of config.agents || []) {
+    const topic = getPublishedTopic(agent);
+    if (!topic) continue;
+    if (!producersByTopic.has(topic)) {
+      producersByTopic.set(topic, new Set());
+    }
+    producersByTopic.get(topic).add(agent.id);
+  }
+
+  return producersByTopic;
+}
+
+function getPublishedTopic(agent) {
+  const onComplete = agent?.hooks?.onComplete;
+  if (!onComplete) return null;
+  if (onComplete.action !== 'publish_message') return null;
+  if (!onComplete.config?.topic) return null;
+  return String(onComplete.config.topic);
+}
+
+function hasConsensusLikeId(agentId) {
+  return agentId.includes('consensus') || agentId.includes('coordinator');
+}
+
+function hasStopClusterTrigger(agent) {
+  return agent?.triggers?.some((trigger) => trigger.action === 'stop_cluster');
+}
+
+function isConsensusLikeAgent(agent) {
+  const agentId = String(agent?.id || '');
+  const explicitIds = ['git-pusher', 'completion-detector'];
+
+  return (
+    agent?.role === 'coordinator' ||
+    explicitIds.includes(agentId) ||
+    hasConsensusLikeId(agentId) ||
+    hasStopClusterTrigger(agent)
+  );
+}
+
+function getMissingStageTopics(requiredStageTopics, producersByTopic, allowExternalTopics) {
+  return requiredStageTopics.filter((stageTopic) => {
+    const producers = producersByTopic.get(stageTopic);
+    return (!producers || producers.size === 0) && !allowExternalTopics.includes(stageTopic);
+  });
+}
+
+function createSimulationContext(config, options = {}) {
+  const agents = Array.isArray(config.agents) ? config.agents : [];
+
+  return {
+    agents,
+    producersByTopic: collectTopicProducers(config),
+    allowExternalTopics: Array.isArray(options.allowExternalTopics)
+      ? options.allowExternalTopics
+      : [],
+    cluster: {
+      id: 'template-sim',
+      agents: agents.map((a) => ({
+        id: a.id,
+        role: a.role,
+        requiredQualityGates: a.requiredQualityGates || [],
+      })),
+    },
+  };
+}
+
+function evaluateScenario({
+  agentId,
+  cluster,
+  topic,
+  script,
+  requiredQualityGates,
+  producers,
+  producersByTopic,
+  requiredStageTopics,
+  allowExternalTopics,
+  publishMessages,
+}) {
+  const ledger = new Ledger(':memory:');
+  const messageBus = new MessageBus(ledger);
+  const logicEngine = new LogicEngine(messageBus, cluster);
+
+  publishStageStartMessages({
+    messageBus,
+    clusterId: cluster.id,
+    producersByTopic,
+    requiredStageTopics,
+    allowExternalTopics,
+  });
+
+  publishMessages(messageBus, producers, cluster.id);
+
+  const result = logicEngine.evaluate(
+    script,
+    { id: agentId, cluster_id: cluster.id, requiredQualityGates },
+    { topic }
+  );
+  ledger.close();
+  return result;
+}
+
+function getPassingQualityGate(requiredGate) {
+  const scope = requiredGate.scope || 'template-sim';
+  return {
+    id: requiredGate.id,
+    status: 'PASS',
+    scope,
+    completedAt: Date.now(),
+    evidence: {
+      command: `quality-check --scope ${scope}`,
+      exitCode: 0,
+      output: 'template simulation quality pass',
+    },
+  };
+}
+
+function getApprovedResultData(context) {
+  const data = { approved: true };
+  const requiredQualityGates = Array.isArray(context.requiredQualityGates)
+    ? context.requiredQualityGates
+    : [];
+  if (
+    context.agentId === 'git-pusher' &&
+    context.topic === 'VALIDATION_RESULT' &&
+    requiredQualityGates.length > 0
+  ) {
+    data.qualityGates = requiredQualityGates.map(getPassingQualityGate);
+  }
+  return data;
+}
+
+function checkDuplicateProducerScenario(context) {
+  return evaluateScenario({
+    ...context,
+    publishMessages(messageBus, producers, clusterId) {
+      messageBus.publish({
+        cluster_id: clusterId,
+        topic: context.topic,
+        sender: producers[0],
+        content: { data: getApprovedResultData(context) },
+      });
+      messageBus.publish({
+        cluster_id: clusterId,
+        topic: context.topic,
+        sender: producers[0],
+        content: { data: getApprovedResultData(context) },
+      });
+    },
+  });
+}
+
+function checkDistinctProducerScenario(context) {
+  return evaluateScenario({
+    ...context,
+    publishMessages(messageBus, producers, clusterId) {
+      for (const producer of producers) {
+        messageBus.publish({
+          cluster_id: clusterId,
+          topic: context.topic,
+          sender: producer,
+          content: { data: getApprovedResultData(context) },
+        });
+      }
+    },
+  });
+}
+
+function getMissingStageFailure(agentId, topic, missingStageTopics) {
+  if (missingStageTopics.length === 0) {
+    return null;
+  }
+
+  return (
+    `Agent "${agentId}" trigger on "${topic}" depends on missing stage topic(s): ${missingStageTopics.join(', ')}. ` +
+    'Preflight must validate real producers, not synthesize stage-start messages.'
+  );
+}
+
+function getConsensusScenarioContext(agent, trigger, simulation) {
+  const topic = trigger?.topic;
+  const script = trigger?.logic?.script;
+  if (!topic || !script) {
+    return null;
+  }
+
+  const requiredStageTopics = getRequiredStageTopics(script);
+  const missingStageTopics = getMissingStageTopics(
+    requiredStageTopics,
+    simulation.producersByTopic,
+    simulation.allowExternalTopics
+  );
+  const missingStageFailure = getMissingStageFailure(agent.id, topic, missingStageTopics);
+  if (missingStageFailure) {
+    return { failure: missingStageFailure };
+  }
+
+  const producers = Array.from(simulation.producersByTopic.get(topic) || []);
+  if (producers.length < 2) {
+    return { failure: null };
+  }
+
+  return {
+    context: {
+      agentId: agent.id,
+      cluster: simulation.cluster,
+      topic,
+      script,
+      requiredQualityGates: agent.requiredQualityGates || [],
+      producers,
+      producersByTopic: simulation.producersByTopic,
+      requiredStageTopics,
+      allowExternalTopics: simulation.allowExternalTopics,
+    },
+    failure: null,
+  };
+}
+
+function validateConsensusTrigger(agent, trigger, simulation) {
+  const scenario = getConsensusScenarioContext(agent, trigger, simulation);
+  if (!scenario) {
+    return [];
+  }
+
+  if (scenario.failure) {
+    return [scenario.failure];
+  }
+
+  if (!scenario.context) {
+    return [];
+  }
+
+  const { context } = scenario;
+  const { producers, topic } = context;
+
+  const failures = [];
+  if (checkDuplicateProducerScenario(context)) {
+    failures.push(
+      `Agent "${agent.id}" trigger on "${topic}" fires early on duplicate sender (${producers[0]}). ` +
+        `Gate must require distinct producers: ${producers.join(', ')}`
+    );
+  }
+
+  if (!checkDistinctProducerScenario(context)) {
+    failures.push(
+      `Agent "${agent.id}" trigger on "${topic}" did not fire after all producers published. ` +
+        `Expected producers: ${producers.join(', ')}`
+    );
+  }
+
+  return failures;
+}
+
+/**
+ * Micro-sim: consensus-like trigger gates must not fire early due to duplicate publishes
+ * from the same producer (common in retries / double-publish bugs).
+ *
+ * Returns an array of error strings.
+ */
+function simulateConsensusGates(config, options = {}) {
+  const simulation = createSimulationContext(config, options);
+  const failures = [];
+
+  for (const agent of simulation.agents) {
+    if (!isConsensusLikeAgent(agent)) continue;
+    for (const trigger of agent.triggers || []) {
+      failures.push(...validateConsensusTrigger(agent, trigger, simulation));
+    }
+  }
+
+  return failures;
+}
+
+module.exports = {
+  collectTopicProducers,
+  simulateConsensusGates,
+};