@cassiomc1/forgeloop 1.5.0 → 1.6.1

package/src/core/trajectory-evaluation.js

@@ -0,0 +1,71 @@
+import { randomUUID } from "node:crypto";
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { canonicalFingerprint, writeJsonArtifact } from "./artifacts.js";
+import { buildTaskTrace } from "./trace.js";
+import { buildTrajectoryMetrics } from "./trajectory-metrics.js";
+import { assertSchema, readSchema } from "./schema-validation.js";
+import { taskEvaluationPath } from "./task-paths.js";
+import { appendProtocolEvent } from "./events.js";
+import { assertSafePath, ensureWithin } from "./filesystem.js";
+import { withTaskTransaction } from "./transaction.js";
+
+function evaluationError(code, message) { const error = new Error(message); error.code = code; return error; }
+
+export async function evaluateTrajectory({ target, packageRoot, taskId, scenarioPath, evaluationId = `eval-${randomUUID()}` }) {
+  if (typeof scenarioPath !== "string" || !scenarioPath || path.isAbsolute(scenarioPath) || scenarioPath.includes("..")) {
+    throw evaluationError("E_TRAJECTORY_SCENARIO_INVALID", "scenario path must be a relative project-local file");
+  }
+  let scenario;
+  try {
+    await assertSafePath(target, scenarioPath);
+    scenario = JSON.parse(await readFile(ensureWithin(target, scenarioPath), "utf8"));
+  } catch (error) {
+    throw evaluationError("E_TRAJECTORY_SCENARIO_INVALID", `unable to read scenario safely: ${error.message}`);
+  }
+  assertSchema(scenario, await readSchema("trajectory-scenario", packageRoot), "trajectory scenario");
+  if (!/^eval-[A-Za-z0-9_-]+$/.test(evaluationId)) throw evaluationError("E_TRAJECTORY_SCENARIO_INVALID", "evaluationId is invalid");
+  const trace = await buildTaskTrace({ target, packageRoot, taskId });
+  const metrics = await buildTrajectoryMetrics({ target, packageRoot, taskId });
+  const milestoneSet = new Set(trace.events.map((event) => event.type));
+  const missingMilestones = (scenario.requiredMilestones ?? []).filter((milestone) => !milestoneSet.has(milestone));
+  const completionEvent = trace.events.find((event) => event.type === "COMPLETION_VALIDATED");
+  const verificationEvent = trace.events.find((event) => event.type === "VERIFICATION_STARTED" || event.type === "VERIFICATION_RECORDED");
+  const completionBeforeVerification = Boolean(completionEvent && (!verificationEvent || completionEvent.sequence < verificationEvent.sequence));
+  // Canonical truth: ambiguous actions or unresolved trusted-readiness items
+  // block safety. A raw VERIFIED label alone never resolves a required action.
+  const unresolvedRequiredAction = (trace.actions?.ambiguous ?? 0) > 0
+    || ((metrics.actions.unresolvedRequired ?? null) !== null
+      ? metrics.actions.unresolvedRequired > 0
+      : (trace.actions?.required ?? 0) > (trace.actions?.verified ?? 0));
+  const limit = (name, actual, max) => ({ actual, ...(max === undefined ? {} : { max, pass: actual <= max }) });
+  const limits = {
+    verificationCycles: limit("verificationCycles", metrics.trajectory.verificationCycles, scenario.limits?.maxVerificationCycles),
+    nonInformativeInterventions: limit("nonInformativeInterventions", metrics.trajectory.interventions.nonInformative, scenario.limits?.maxNonInformativeInterventions),
+    ambiguousActions: limit("ambiguousActions", metrics.actions.ambiguous, scenario.limits?.maxAmbiguousActions),
+  };
+  const safetyValid = !(scenario.forbidden?.completionBeforeVerification && completionBeforeVerification)
+    && !(scenario.forbidden?.unresolvedRequiredAction && unresolvedRequiredAction);
+  const completionValid = metrics.completion.validated && missingMilestones.length === 0;
+  const limitsValid = Object.values(limits).every((item) => item.pass !== false);
+  // Comparable steps have exactly one owner: buildTrajectoryMetrics.
+  const efficiency = Number.isInteger(scenario.reference?.comparableSteps) && scenario.reference.comparableSteps > 0
+    ? { referenceComparableSteps: scenario.reference.comparableSteps, actualComparableSteps: metrics.comparableSteps,
+      ratio: scenario.reference.comparableSteps / Math.max(1, metrics.comparableSteps) }
+    : null;
+  const base = {
+    schemaVersion: 1, evaluationId, scenarioId: scenario.scenarioId,
+    scenarioFingerprint: canonicalFingerprint(scenario), taskId,
+    result: completionValid && safetyValid && limitsValid ? "PASS" : "FAIL",
+    completionValid, safetyValid, missingMilestones, limits, efficiency,
+    computedAt: new Date().toISOString(), source: "PROJECT_LOCAL_REFERENCE",
+  };
+  const evaluation = { ...base, evaluationFingerprint: canonicalFingerprint(base) };
+  assertSchema(evaluation, await readSchema("trajectory-evaluation", packageRoot), "trajectory evaluation");
+  await withTaskTransaction({ target, taskId, operation: "trajectory-evaluated" }, async () => {
+    await writeJsonArtifact(target, taskEvaluationPath(taskId, evaluationId), evaluation, "trajectory-evaluation", packageRoot);
+    await appendProtocolEvent(target, { taskId, event: "TRAJECTORY_EVALUATED", fingerprint: evaluation.evaluationFingerprint,
+      details: { evaluationId, scenarioId: scenario.scenarioId, evaluationFingerprint: evaluation.evaluationFingerprint } }, packageRoot, { taskId });
+  });
+  return evaluation;
+}

package/src/core/trajectory-metrics.js

@@ -0,0 +1,80 @@
+import { buildTaskTrace } from "./trace.js";
+import { buildTaskReflection } from "./reflection.js";
+
+export const COMPARABLE_WORK_EVENTS = new Set([
+  "EXECUTION_STARTED", "VERIFICATION_RECORDED", "DIAGNOSTIC_CASE_RECORDED",
+  "INTERVENTION_RECORDED", "ACTION_STARTED", "ACTION_RECONCILED", "REVIEW_STARTED",
+]);
+
+export async function buildTrajectoryMetrics({ target, packageRoot, taskId }) {
+  const trace = await buildTaskTrace({ target, packageRoot, taskId });
+  const reflection = await buildTaskReflection({ target, packageRoot, taskId });
+  const events = trace.events;
+  const verificationCycles = new Set(events.filter((event) => event.type === "VERIFICATION_STARTED")
+    .map((event) => event.data?.verificationCycle).filter(Number.isInteger));
+  const diagnosticCycles = new Set(trace.diagnostics.cases.map((item) => item.verificationCycle));
+  const firstEventAt = events.find((event) => event.timestampQuality === "authoritative")?.timestamp ?? null;
+  const lastEventAt = [...events].reverse().find((event) => event.timestampQuality === "authoritative")?.timestamp ?? null;
+  const firstMs = firstEventAt ? Date.parse(firstEventAt) : NaN;
+  const lastMs = lastEventAt ? Date.parse(lastEventAt) : NaN;
+  const interventions = reflection.interventions ?? { count: 0, informative: 0, nonInformative: 0 };
+  return {
+    schemaVersion: 1,
+    taskId,
+    completion: {
+      validated: Boolean(trace.completion.validatedAt),
+      phase: trace.task.phase,
+    },
+    trajectory: {
+      events: events.length,
+      verificationCycles: verificationCycles.size,
+      diagnosticCycles: diagnosticCycles.size,
+      strategyChanges: Math.max(0, (reflection.strategies?.length ?? 0) - 1),
+      oscillationDetected: Boolean(reflection.oscillation?.detected),
+      noEffectiveInformationGainCycles: reflection.informationGain?.cyclesWithoutEffectiveGain?.length ?? 0,
+      interventions: {
+        total: interventions.count ?? 0,
+        informative: interventions.informative ?? 0,
+        nonInformative: interventions.nonInformative ?? 0,
+      },
+    },
+    actions: {
+      total: trace.actions?.total ?? 0,
+      verified: trace.actions?.verified ?? 0,
+      // Canonical trust counts come from the action-readiness projection;
+      // raw VERIFIED labels are observability only.
+      trustedSatisfied: await (async () => {
+        try {
+          const { evaluateRequiredActionReadiness } = await import("./action-readiness.js");
+          const readiness = await evaluateRequiredActionReadiness({ target, packageRoot, taskId });
+          return readiness.satisfied;
+        } catch {
+          return null;
+        }
+      })(),
+      unresolvedRequired: await (async () => {
+        try {
+          const { evaluateRequiredActionReadiness } = await import("./action-readiness.js");
+          const readiness = await evaluateRequiredActionReadiness({ target, packageRoot, taskId });
+          return readiness.unresolved;
+        } catch {
+          return null;
+        }
+      })(),
+      failed: trace.actions?.failed ?? 0,
+      ambiguous: trace.actions?.ambiguous ?? 0,
+      reconciliations: trace.actions?.reconciliationCount ?? 0,
+    },
+    executions: {
+      observedCommands: trace.executions.length,
+      failedCommands: trace.executions.filter((execution) => execution.status === "failed").length,
+    },
+    timing: {
+      firstEventAt,
+      lastEventAt,
+      wallClockMs: Number.isFinite(firstMs) && Number.isFinite(lastMs) ? Math.max(0, lastMs - firstMs) : null,
+    },
+    usage: { tokens: null, costUsd: null, source: "UNKNOWN" },
+    comparableSteps: events.filter((event) => COMPARABLE_WORK_EVENTS.has(event.type)).length,
+  };
+}

package/src/core/transaction.js

@@ -103,6 +103,13 @@ export async function withTaskTransaction({
   recordCommitEvent = false,
 } = {}, callback) {
   if (!target || !taskId) throw new Error("target and taskId are required for a task transaction");
+  const activeTransaction = getActiveTaskTransaction();
+  if (activeTransaction) {
+    if (activeTransaction.taskId !== taskId) {
+      throw new Error(`cannot nest task transaction for ${taskId} inside ${activeTransaction.taskId}`);
+    }
+    return callback(activeTransaction);
+  }
   const started = Date.now();
   const runWithLock = async () => withTaskLock(target, lockTaskId, operation, async (lock) => {
     const transactionId = `txn-${randomUUID()}`;
@@ -115,6 +122,7 @@ export async function withTaskTransaction({
     await writeManifest(target, manifestPath, manifest);
     const tx = {
       transactionId,
+      taskId,
       lock,
       async readText(relativePath) {
         await assertSafePath(target, relativePath);

package/src/core/verification-execution.js

@@ -0,0 +1,257 @@
+export const VERIFICATION_ISOLATION_MODES = Object.freeze([
+  "NATIVE_PROJECT",
+  "PROJECT_ISOLATED",
+  "SYSTEM_ISOLATED",
+]);
+
+export const VERIFICATION_EXECUTION_POLICY_MODES = Object.freeze([
+  "NONE",
+  ...VERIFICATION_ISOLATION_MODES,
+]);
+
+export const E_VERIFICATION_ISOLATION_UNAVAILABLE = "E_VERIFICATION_ISOLATION_UNAVAILABLE";
+export const E_VERIFICATION_EXECUTION_INVALID = "E_VERIFICATION_EXECUTION_INVALID";
+
+const ISOLATION_RANK = Object.freeze({
+  NATIVE_PROJECT: 0,
+  PROJECT_ISOLATED: 1,
+  SYSTEM_ISOLATED: 2,
+});
+
+function isRecord(value) {
+  return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+
+function executionError(code, message) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function requiredString(value, field) {
+  if (typeof value !== "string" || value.trim() === "") {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, `Verification execution ${field} must be a non-empty string`);
+  }
+  return value;
+}
+
+function nullableString(value, field) {
+  if (value !== null && typeof value !== "string") {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, `Verification execution ${field} must be a string or null`);
+  }
+  return value;
+}
+
+function outputBytes(value, field) {
+  if (value === undefined || value === null) return Buffer.alloc(0);
+  if (typeof value === "string") return Buffer.from(value, "utf8");
+  if (Buffer.isBuffer(value) || value instanceof Uint8Array) return Buffer.from(value);
+  throw executionError(E_VERIFICATION_EXECUTION_INVALID, `Verification execution ${field} must be text or bytes`);
+}
+
+function nonNegativeInteger(value, field, fallback) {
+  const resolved = value === undefined ? fallback : value;
+  if (!Number.isInteger(resolved) || resolved < 0) {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, `Verification execution ${field} must be a non-negative integer`);
+  }
+  return resolved;
+}
+
+export function normalizeVerificationExecutionPolicy(policy = {}) {
+  if (policy === undefined || policy === null) return Object.freeze({ requiredIsolation: "NONE" });
+  if (!isRecord(policy)) {
+    throw executionError(E_VERIFICATION_ISOLATION_UNAVAILABLE, "Verification execution isolation policy is invalid");
+  }
+  const requiredIsolation = policy.requiredIsolation ?? "NONE";
+  if (!VERIFICATION_EXECUTION_POLICY_MODES.includes(requiredIsolation)) {
+    throw executionError(E_VERIFICATION_ISOLATION_UNAVAILABLE, "Verification execution isolation policy is unsupported");
+  }
+  return Object.freeze({ requiredIsolation });
+}
+
+export function isVerificationExecutionAdapter(value) {
+  return isRecord(value) && typeof value.execute === "function";
+}
+
+function readAdapter(runtimeContext) {
+  const value = isRecord(runtimeContext) ? runtimeContext.verificationExecutionAdapter : undefined;
+  if (value === undefined || value === null) return null;
+  if (!isVerificationExecutionAdapter(value)) {
+    throw executionError(E_VERIFICATION_ISOLATION_UNAVAILABLE, "Verification execution adapter is unavailable or invalid");
+  }
+  return value;
+}
+
+function assertIsolationMetadataConsistency(isolation) {
+  if (isolation.mode === "NATIVE_PROJECT") {
+    if (
+      isolation.isolated !== false
+      || isolation.liveProjectWritable !== true
+    ) {
+      throw executionError(
+        E_VERIFICATION_EXECUTION_INVALID,
+        "NATIVE_PROJECT must report isolated=false and liveProjectWritable=true",
+      );
+    }
+    return;
+  }
+
+  if (
+    isolation.isolated !== true
+    || isolation.liveProjectWritable !== false
+  ) {
+    throw executionError(
+      E_VERIFICATION_EXECUTION_INVALID,
+      `${isolation.mode} must report isolated=true and liveProjectWritable=false`,
+    );
+  }
+
+  if (
+    isolation.mode === "SYSTEM_ISOLATED"
+    && isolation.networkPolicy !== "DENIED"
+  ) {
+    throw executionError(
+      E_VERIFICATION_EXECUTION_INVALID,
+      "SYSTEM_ISOLATED must report networkPolicy=DENIED",
+    );
+  }
+}
+
+function normalizeIsolation(value) {
+  if (!isRecord(value)) {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution isolation metadata is missing");
+  }
+  const mode = value.mode;
+  if (!VERIFICATION_ISOLATION_MODES.includes(mode)) {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution isolation mode is unsupported");
+  }
+  if (typeof value.isolated !== "boolean" || typeof value.liveProjectWritable !== "boolean") {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution isolation booleans are invalid");
+  }
+  const isolation = {
+    mode,
+    isolated: value.isolated,
+    liveProjectWritable: value.liveProjectWritable,
+    networkPolicy: requiredString(value.networkPolicy, "networkPolicy"),
+    environmentPolicy: requiredString(value.environmentPolicy, "environmentPolicy"),
+  };
+  assertIsolationMetadataConsistency(isolation);
+  return isolation;
+}
+
+export function normalizeVerificationExecutionResult(value, { defaultCwd = null } = {}) {
+  if (!isRecord(value)) {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution adapter returned an invalid result");
+  }
+  const cwd = value.cwd ?? defaultCwd;
+  requiredString(cwd, "cwd");
+  if (value.exitCode !== null && !Number.isInteger(value.exitCode)) {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution exitCode must be an integer or null");
+  }
+  const signal = nullableString(value.signal ?? null, "signal");
+  if (typeof value.timedOut !== "boolean") {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution timedOut must be boolean");
+  }
+  if (typeof value.outputTruncated !== "boolean") {
+    throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution outputTruncated must be boolean");
+  }
+  const stdout = outputBytes(value.stdout, "stdout");
+  const stderr = outputBytes(value.stderr, "stderr");
+  return {
+    exitCode: value.exitCode,
+    signal,
+    timedOut: value.timedOut,
+    stdout,
+    stderr,
+    stdoutBytes: nonNegativeInteger(value.stdoutBytes, "stdoutBytes", stdout.length),
+    stderrBytes: nonNegativeInteger(value.stderrBytes, "stderrBytes", stderr.length),
+    outputTruncated: value.outputTruncated,
+    spawnError: Boolean(value.spawnError),
+    cwd,
+    isolation: normalizeIsolation(value.isolation),
+  };
+}
+
+function satisfiesIsolationPolicy(isolation, requiredIsolation) {
+  if (requiredIsolation === "NONE") return true;
+  if (ISOLATION_RANK[isolation.mode] < ISOLATION_RANK[requiredIsolation]) return false;
+  if (requiredIsolation === "NATIVE_PROJECT") return true;
+  return isolation.isolated === true
+    && isolation.liveProjectWritable === false
+    && (requiredIsolation !== "SYSTEM_ISOLATED" || isolation.networkPolicy === "DENIED");
+}
+
+export function assertVerificationIsolation(result, policy) {
+  const requiredIsolation = normalizeVerificationExecutionPolicy(policy).requiredIsolation;
+  if (!satisfiesIsolationPolicy(result.isolation, requiredIsolation)) {
+    throw executionError(
+      E_VERIFICATION_ISOLATION_UNAVAILABLE,
+      `Verification execution did not satisfy the required ${requiredIsolation} isolation boundary`,
+    );
+  }
+  return result;
+}
+
+export async function executeVerificationProcess({
+  argv,
+  protocolProjectRoot,
+  taskId,
+  checkId,
+  requirement,
+  resolution,
+  timeoutMs,
+  runtimeContext,
+  nativeExecute,
+}) {
+  const policy = normalizeVerificationExecutionPolicy(runtimeContext?.verificationExecutionPolicy);
+  const adapter = readAdapter(runtimeContext);
+  let rawResult;
+  if (adapter) {
+    try {
+      rawResult = await adapter.execute(Object.freeze({
+        argv: [...argv],
+        protocolProjectRoot,
+        taskId,
+        checkId,
+        requirement,
+        timeoutMs,
+        resolution,
+      }));
+    } catch (error) {
+      if (error?.code === E_VERIFICATION_ISOLATION_UNAVAILABLE) throw error;
+      throw executionError(E_VERIFICATION_EXECUTION_INVALID, "Verification execution adapter failed");
+    }
+  } else {
+    if (policy.requiredIsolation !== "NONE") {
+      throw executionError(
+        E_VERIFICATION_ISOLATION_UNAVAILABLE,
+        `Verification execution requires ${policy.requiredIsolation}, but no trusted adapter was supplied`,
+      );
+    }
+    const nativeResult = await nativeExecute(argv, protocolProjectRoot, { timeoutMs });
+    rawResult = {
+      ...nativeResult,
+      cwd: protocolProjectRoot,
+      isolation: {
+        mode: "NATIVE_PROJECT",
+        isolated: false,
+        liveProjectWritable: true,
+        networkPolicy: "INHERITED",
+        environmentPolicy: "INHERITED",
+      },
+    };
+  }
+
+  const normalized = normalizeVerificationExecutionResult(rawResult, { defaultCwd: protocolProjectRoot });
+  if (
+    policy.requiredIsolation !== "NONE"
+    && normalized.isolation.mode !== "NATIVE_PROJECT"
+    && normalized.cwd === protocolProjectRoot
+  ) {
+    throw executionError(
+      E_VERIFICATION_EXECUTION_INVALID,
+      "Isolated verification execution must use a cwd separate from the protocol project root",
+    );
+  }
+  return assertVerificationIsolation(normalized, policy);
+}

package/src/core/work-state.js

@@ -538,13 +538,18 @@ export async function classifyLoadedWorkState({ target, state, contractFile = nu
   };
 }
 
-export async function readAndClassifyWorkState({ target, packageRoot = getPackageRoot(), contractFile = null, maxAgeMs } = {}) {
+export async function readAndClassifyWorkState({ target, packageRoot = getPackageRoot(), contractFile = null, maxAgeMs, taskId = null, stateFile = null, statePath = null } = {}) {
+  const effectiveStateRel = stateFile ?? statePath ?? (taskId ? taskArtifactPath(taskId, "state") : WORK_STATE_PATH);
   let state = null;
   try {
-    state = await readWorkState(target, packageRoot);
+    state = await readWorkState(target, {
+      packageRoot,
+      ...(taskId ? { taskId } : {}),
+      ...(stateFile || statePath ? { statePath: effectiveStateRel } : {}),
+    });
   } catch (error) {
     return {
-      path: WORK_STATE_PATH,
+      path: effectiveStateRel,
       present: true,
       status: "INVALID",
       reasons: ["STATE_INVALID"],
@@ -565,7 +570,7 @@ export async function readAndClassifyWorkState({ target, packageRoot = getPackag
   if (!state) {
     const repository = await currentRepositoryFingerprint(target);
     return {
-      path: WORK_STATE_PATH,
+      path: effectiveStateRel,
       present: false,
       status: "ABSENT",
       reasons: ["NO_CHECKPOINT"],
@@ -589,7 +594,7 @@ export async function readAndClassifyWorkState({ target, packageRoot = getPackag
 
   const classification = await classifyLoadedWorkState({ target, state, contractFile, maxAgeMs });
   return {
-    path: WORK_STATE_PATH,
+    path: effectiveStateRel,
     present: true,
     ...classification,
     phase: state.phase,

package/src/integration.js

@@ -44,4 +44,12 @@ export {
   CLI_COMMAND_DEFINITIONS,
 };
 
+export { createForgeLoopContext } from "./core/runtime-context.js";
+export {
+  E_VERIFICATION_EXECUTION_INVALID,
+  E_VERIFICATION_ISOLATION_UNAVAILABLE,
+  VERIFICATION_EXECUTION_POLICY_MODES,
+  VERIFICATION_ISOLATION_MODES,
+} from "./core/verification-execution.js";
+
 export const FORGELOOP_INTEGRATION_API_VERSION = FORGELOOP_INTEGRATION_RUNTIME_VERSION;