@cassiomc1/forgeloop 1.6.5 → 1.8.0

package/src/core/command-executors.js

@@ -33,6 +33,8 @@ import { runActionAuthorize } from "../commands/action-authorize.js";
 import { runActionVerify } from "../commands/action-verify.js";
 import { runActionReconcile } from "../commands/action-reconcile.js";
 import { runMetrics } from "../commands/metrics.js";
+import { runUsageRecord } from "../commands/usage-record.js";
+import { runEfficiency } from "../commands/efficiency.js";
 import { runEval } from "../commands/eval.js";
 import { runApprovalRequest } from "../commands/approval-request.js";
 import { runApprovalResolve } from "../commands/approval-resolve.js";
@@ -117,6 +119,7 @@ export const COMMAND_EXECUTORS = {
       platforms: options.platforms,
       behaviorChange: options.behaviorChange,
       executableChange: options.executableChange,
+      executionProfile: options.executionProfile,
       taskId: options.taskId,
     }),
     exitCode: 0,
@@ -141,7 +144,7 @@ export const COMMAND_EXECUTORS = {
     exitCode: 0,
   }),
   next: async ({ target, packageRoot, options, authorityContext, runtimeContext }) => ({
-    result: await runNext({ target, packageRoot, taskId: options.taskId, authorityContext, runtimeContext }),
+    result: await runNext({ target, packageRoot, taskId: options.taskId, authorityContext, runtimeContext, compact: options.compact }),
     exitCode: 0,
   }),
   continuity: async ({ target, packageRoot, options }) => ({
@@ -305,9 +308,30 @@ export const COMMAND_EXECUTORS = {
     // executor parameter; actor input can never supply it.
     authorityContext,
   }), exitCode: 0 }),
-  metrics: async ({ target, packageRoot, options }) => ({ result: await runMetrics({ target, packageRoot, taskId: options.taskId }), exitCode: 0 }),
-  eval: async ({ target, packageRoot, options }) => {
-    const result = await runEval({ target, packageRoot, taskId: options.taskId, scenarioPath: options.scenarioPath });
+  metrics: async ({ target, packageRoot, options, runtimeContext }) => ({ result: await runMetrics({ target, packageRoot, taskId: options.taskId, runtimeContext }), exitCode: 0 }),
+  "usage-record": async ({ target, packageRoot, options }) => ({
+    result: await runUsageRecord({
+      target,
+      packageRoot,
+      taskId: options.taskId,
+      provider: options.usageProvider,
+      model: options.usageModel,
+      inputTokens: options.usageInputTokens,
+      outputTokens: options.usageOutputTokens,
+      cacheReadTokens: options.usageCacheReadTokens,
+      cacheWriteTokens: options.usageCacheWriteTokens,
+      totalTokens: options.usageTotalTokens,
+      costUsd: options.usageCostUsd,
+      source: options.usageSource ?? "ACTOR_REPORTED",
+    }),
+    exitCode: 0,
+  }),
+  efficiency: async ({ target, packageRoot, options, runtimeContext }) => ({
+    result: await runEfficiency({ target, packageRoot, taskId: options.taskId, baselinePath: options.baselinePath, runtimeContext }),
+    exitCode: 0,
+  }),
+  eval: async ({ target, packageRoot, options, runtimeContext }) => {
+    const result = await runEval({ target, packageRoot, taskId: options.taskId, scenarioPath: options.scenarioPath, runtimeContext });
     return { result, exitCode: result.result === "PASS" ? 0 : 1 };
   },
   "approval-request": async ({ target, packageRoot, options }) => ({ result: await runApprovalRequest({ target, packageRoot, taskId: options.taskId, approvalId: options.approvalId, actionId: options.actionId, reason: options.reason }), exitCode: 0 }),
@@ -547,7 +571,7 @@ export const COMMAND_EXECUTORS = {
     exitCode: 0,
   }),
   "task-show": async ({ target, packageRoot, options }) => ({
-    result: await runTaskShow({ target, packageRoot, taskId: options.taskId }),
+    result: await runTaskShow({ target, packageRoot, taskId: options.taskId, compact: options.compact }),
     exitCode: 0,
   }),
   "task-lock-status": async ({ target, packageRoot, options }) => ({

package/src/core/command-input.js

@@ -1,5 +1,6 @@
 import { continuityOptionDefaults, validateContinuityOptions } from "./continuity-cli-options.js";
 import { E_CLI_INVOCATION_INVALID } from "./error-codes.js";
+import { EXECUTION_PROFILE_REQUESTS } from "./execution-profile.js";
 
 function inputError(message) {
   const error = new Error(message);
@@ -17,6 +18,7 @@ export function defaultCommandInputValues() {
     path: ".",
     dryRun: false,
     json: false,
+    compact: false,
     strict: false,
     fix: false,
     adopt: [],
@@ -26,6 +28,17 @@ export function defaultCommandInputValues() {
     platforms: [],
     behaviorChange: false,
     executableChange: false,
+    executionProfile: null,
+    usageProvider: null,
+    usageModel: null,
+    usageInputTokens: null,
+    usageOutputTokens: null,
+    usageCacheReadTokens: null,
+    usageCacheWriteTokens: null,
+    usageTotalTokens: null,
+    usageCostUsd: null,
+    usageSource: "ACTOR_REPORTED",
+    baselinePath: null,
     to: null,
     file: null,
     contractFile: null,
@@ -102,6 +115,27 @@ export function validateForgeLoopCommandInput({ command, input, help = false } =
   if (command === "task-create" && !options.taskId) {
     throw inputError("task-create requires --task");
   }
+  if (options.executionProfile !== null && options.executionProfile !== undefined) {
+    if (command !== "route") throw inputError(`executionProfile is not valid for ${command}`);
+    if (!EXECUTION_PROFILE_REQUESTS.includes(options.executionProfile)) {
+      throw inputError(`route --execution-profile must be one of ${EXECUTION_PROFILE_REQUESTS.join(", ")}`);
+    }
+  }
+  if (command === "usage-record" && !help) {
+    if (!options.taskId) throw inputError("usage-record requires --task");
+    if ((options.usageSource ?? "ACTOR_REPORTED") !== "ACTOR_REPORTED") {
+      throw inputError("usage-record accepts only --source ACTOR_REPORTED");
+    }
+  }
+  if (command === "efficiency" && !help && !options.taskId) {
+    throw inputError("efficiency requires --task");
+  }
+  if (command !== "usage-record" && options.usageSource !== undefined && options.usageSource !== "ACTOR_REPORTED") {
+    throw inputError(`usageSource is not valid for ${command}`);
+  }
+  if (options.compact === true && !["next", "task-show"].includes(command)) {
+    throw inputError(`compact output is not valid for ${command}`);
+  }
   if (["workspace-bind", "workspace-status", "handoff-create", "handoff-list", "handoff-show", "responsibility-set", "responsibility-status", "verify-scope", "attestation-create", "attestation-status", "attestation-verify"].includes(command)
     && !options.taskId) {
     throw inputError(`${command} requires --task`);

package/src/core/config.js

@@ -2,6 +2,7 @@ import { PROTOCOL_VERSION } from "./protocol.js";
 import { ARTIFACT_PATHS, readJsonArtifact, writeJsonArtifact } from "./artifacts.js";
 import { E_ATTESTATION_CONFIGURATION_INVALID } from "./error-codes.js";
 import { normalizeVerificationConfiguration } from "./verification-scope-capability.js";
+import { EXECUTION_PROFILE_REQUESTS } from "./execution-profile.js";
 
 export const CONFIG_SCHEMA_VERSION = 1;
 export const COMPLIANCE_MODES = Object.freeze(["advisory", "standard", "strict"]);
@@ -72,6 +73,13 @@ export function createConfig(input = {}) {
       throw configurationError(error.message);
     }
   }
+  let executionProfile;
+  if (input.executionProfile !== undefined) {
+    if (!EXECUTION_PROFILE_REQUESTS.includes(input.executionProfile)) {
+      throw configurationError(`Unknown execution profile: ${input.executionProfile}`);
+    }
+    executionProfile = input.executionProfile;
+  }
   return {
     schemaVersion: CONFIG_SCHEMA_VERSION,
     protocolVersion: PROTOCOL_VERSION,
@@ -81,6 +89,7 @@ export function createConfig(input = {}) {
     ...(input.requiredEvidence !== undefined ? { requiredEvidence: stringArray(input.requiredEvidence, "requiredEvidence") } : {}),
     ...(verification ? { verification } : {}),
     ...(attestation ? { attestation } : {}),
+    ...(executionProfile ? { executionProfile } : {}),
   };
 }
 

package/src/core/efficiency.js

@@ -0,0 +1,197 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { assertSafePath, ensureWithin } from "./filesystem.js";
+import { readContract } from "./contract.js";
+import { currentRepositoryFingerprint } from "./repository.js";
+import { buildTrajectoryMetrics } from "./trajectory-metrics.js";
+
+function efficiencyError(code, message) {
+  const error = new Error(message);
+  error.code = code;
+  return error;
+}
+
+function environmentClass() {
+  return `${process.platform}-node${process.versions.node.split(".")[0]}`;
+}
+
+function validNumber(value) {
+  return typeof value === "number" && Number.isFinite(value) && value >= 0;
+}
+
+function validateBaseline(value) {
+  if (!value || typeof value !== "object" || Array.isArray(value)) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline must be a JSON object");
+  }
+  if (!Number.isInteger(value.comparableSteps) || value.comparableSteps < 0) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline comparableSteps must be a non-negative integer");
+  }
+  if (value.metadata !== undefined && (!value.metadata || typeof value.metadata !== "object" || Array.isArray(value.metadata))) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline metadata must be an object");
+  }
+  if (value.usage !== undefined && (!value.usage || typeof value.usage !== "object" || Array.isArray(value.usage))) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline usage must be an object");
+  }
+  if (value.timing !== undefined && (!value.timing || typeof value.timing !== "object" || Array.isArray(value.timing))) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline timing must be an object");
+  }
+  if (value.usage?.totalTokens !== undefined && value.usage.totalTokens !== null && !validNumber(value.usage.totalTokens)) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline usage.totalTokens must be a non-negative number or null");
+  }
+  if (value.timing?.wallClockMs !== undefined && value.timing.wallClockMs !== null && !validNumber(value.timing.wallClockMs)) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "efficiency baseline timing.wallClockMs must be a non-negative number or null");
+  }
+  return value;
+}
+
+function compareMetadata(actual, baseline) {
+  const expected = baseline.metadata ?? {};
+  const fields = [
+    "taskId",
+    "scenarioId",
+    "model",
+    "provider",
+    "promptSpecFingerprint",
+    "projectRevision",
+    "benchmarkVersion",
+    "environmentClass",
+    "usageSource",
+  ];
+  const mismatches = [];
+  for (const field of fields) {
+    if (!Object.prototype.hasOwnProperty.call(expected, field)) continue;
+    if (actual[field] === undefined || actual[field] === null || actual[field] !== expected[field]) {
+      mismatches.push({ field, expected: expected[field] ?? null, actual: actual[field] ?? null });
+    }
+  }
+  const required = ["taskId", "model", "provider", "projectRevision", "environmentClass"];
+  for (const field of required) {
+    if (!Object.prototype.hasOwnProperty.call(expected, field)) {
+      mismatches.push({ field, expected: "required", actual: actual[field] ?? null });
+    }
+  }
+  return { comparable: mismatches.length === 0, mismatches };
+}
+
+function ratio(actual, baseline) {
+  if (!validNumber(actual) || !validNumber(baseline) || baseline === 0) return null;
+  return Number(((actual - baseline) / baseline * 100).toFixed(4));
+}
+
+export async function readEfficiencyBaseline(target, baselinePath) {
+  if (typeof baselinePath !== "string" || !baselinePath.trim() || path.isAbsolute(baselinePath)) {
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", "baseline path must be a relative project-local file");
+  }
+  try {
+    await assertSafePath(target, baselinePath);
+    const parsed = JSON.parse(await readFile(ensureWithin(target, baselinePath), "utf8"));
+    return validateBaseline(parsed);
+  } catch (error) {
+    if (error.code === "E_EFFICIENCY_BASELINE_INVALID") throw error;
+    throw efficiencyError("E_EFFICIENCY_BASELINE_INVALID", `unable to read baseline safely: ${error.message}`);
+  }
+}
+
+export async function buildEfficiencyReport({ target, packageRoot, taskId, baselinePath = null, runtimeContext = null } = {}) {
+  const metrics = await buildTrajectoryMetrics({ target, packageRoot, taskId, runtimeContext });
+  const repository = await currentRepositoryFingerprint(target);
+  let promptSpecFingerprint = null;
+  try {
+    promptSpecFingerprint = (await readContract(target, packageRoot, { taskId })).fingerprint;
+  } catch {
+    // A task without a contract is not comparable to a contract-bound baseline.
+  }
+  const actualMetadata = {
+    taskId,
+    scenarioId: taskId,
+    model: metrics.usage.model,
+    provider: metrics.usage.provider,
+    promptSpecFingerprint,
+    projectRevision: repository.head ?? null,
+    benchmarkVersion: null,
+    environmentClass: environmentClass(),
+    usageSource: metrics.usage.source,
+  };
+  const base = {
+    taskId,
+    executionProfile: metrics.executionProfile,
+    usage: metrics.usage,
+    usageSource: metrics.usage.source,
+    timing: metrics.timing,
+    tokensPerComparableStep: validNumber(metrics.usage.totalTokens) && metrics.comparableSteps > 0
+      ? metrics.usage.totalTokens / metrics.comparableSteps
+      : null,
+  };
+  if (!baselinePath) {
+    return {
+      ...base,
+      comparison: {
+        status: "NOT_COMPARABLE",
+        comparable: false,
+        usageSource: metrics.usage.source,
+        reason: "No baseline was supplied.",
+        metadata: actualMetadata,
+        baseline: null,
+        ratios: { comparableStepsPercent: null, totalTokensPercent: null, wallClockMsPercent: null },
+        tokenOverheadRatio: null,
+        timeOverheadRatio: null,
+        tokenOverheadPercent: null,
+        timeOverheadPercent: null,
+      },
+    };
+  }
+
+  const baseline = await readEfficiencyBaseline(target, baselinePath);
+  const metadata = compareMetadata(actualMetadata, baseline);
+  if (!metadata.comparable) {
+    return {
+      ...base,
+      comparison: {
+        status: "NOT_COMPARABLE",
+        comparable: false,
+        usageSource: metrics.usage.source,
+        reason: "Baseline metadata does not match the current task execution.",
+        metadata: actualMetadata,
+        mismatches: metadata.mismatches,
+        baseline: { path: baselinePath, comparableSteps: baseline.comparableSteps },
+        ratios: { comparableStepsPercent: null, totalTokensPercent: null, wallClockMsPercent: null },
+        tokenOverheadRatio: null,
+        timeOverheadRatio: null,
+        tokenOverheadPercent: null,
+        timeOverheadPercent: null,
+      },
+    };
+  }
+
+  return {
+    ...base,
+    comparison: {
+      status: "COMPARABLE",
+      comparable: true,
+      usageSource: metrics.usage.source,
+      reason: "Baseline metadata matches the current task execution.",
+      metadata: actualMetadata,
+      baseline: {
+        path: baselinePath,
+        comparableSteps: baseline.comparableSteps,
+        usage: baseline.usage ?? null,
+        timing: baseline.timing ?? null,
+      },
+      ratios: {
+        comparableStepsPercent: ratio(metrics.comparableSteps, baseline.comparableSteps),
+        totalTokensPercent: ratio(metrics.usage.totalTokens, baseline.usage?.totalTokens),
+        wallClockMsPercent: ratio(metrics.timing.wallClockMs, baseline.timing?.wallClockMs),
+      },
+      tokenOverheadRatio: validNumber(metrics.usage.totalTokens) && validNumber(baseline.usage?.totalTokens)
+        && baseline.usage.totalTokens !== 0
+        ? Number((metrics.usage.totalTokens / baseline.usage.totalTokens).toFixed(4))
+        : null,
+      timeOverheadRatio: validNumber(metrics.timing.wallClockMs) && validNumber(baseline.timing?.wallClockMs)
+        && baseline.timing.wallClockMs !== 0
+        ? Number((metrics.timing.wallClockMs / baseline.timing.wallClockMs).toFixed(4))
+        : null,
+      tokenOverheadPercent: ratio(metrics.usage.totalTokens, baseline.usage?.totalTokens),
+      timeOverheadPercent: ratio(metrics.timing.wallClockMs, baseline.timing?.wallClockMs),
+    },
+  };
+}

package/src/core/error-codes.js

@@ -77,6 +77,12 @@ export const E_ATTESTATION_TARGET_REF_INVALID = "E_ATTESTATION_TARGET_REF_INVALI
 export const E_ATTESTATION_INVALID = "E_ATTESTATION_INVALID";
 export const E_ATTESTATION_CONFIGURATION_INVALID = "E_ATTESTATION_CONFIGURATION_INVALID";
 export const E_CLI_INVOCATION_INVALID = "E_CLI_INVOCATION_INVALID";
+export const E_EXECUTION_PROFILE_INVALID = "E_EXECUTION_PROFILE_INVALID";
+export const E_EXECUTION_PROFILE_INCONSISTENT = "E_EXECUTION_PROFILE_INCONSISTENT";
+export const E_EXECUTION_PROFILE_SAFETY_FLOOR_INVALID = "E_EXECUTION_PROFILE_SAFETY_FLOOR_INVALID";
+export const E_USAGE_INVALID = "E_USAGE_INVALID";
+export const E_USAGE_SOURCE_INVALID = "E_USAGE_SOURCE_INVALID";
+export const E_EFFICIENCY_BASELINE_INVALID = "E_EFFICIENCY_BASELINE_INVALID";
 
 const EXTENSION_PUBLIC_ERROR_CODES = Object.freeze(Object.fromEntries([
   E_WORKSPACE_IDENTITY_UNAVAILABLE,
@@ -124,6 +130,12 @@ const EXTENSION_PUBLIC_ERROR_CODES = Object.freeze(Object.fromEntries([
   E_ATTESTATION_INVALID,
   E_ATTESTATION_CONFIGURATION_INVALID,
   E_CLI_INVOCATION_INVALID,
+  E_EXECUTION_PROFILE_INVALID,
+  E_EXECUTION_PROFILE_INCONSISTENT,
+  E_EXECUTION_PROFILE_SAFETY_FLOOR_INVALID,
+  E_USAGE_INVALID,
+  E_USAGE_SOURCE_INVALID,
+  E_EFFICIENCY_BASELINE_INVALID,
 ].map((code) => [code, Object.freeze({
   code,
   category: code.slice(2).split("_")[0].toLowerCase(),
@@ -1026,6 +1038,12 @@ export const ALL_KNOWN_ERROR_CODES = Object.freeze(new Set([
   E_ATTESTATION_INVALID,
   E_ATTESTATION_CONFIGURATION_INVALID,
   E_CLI_INVOCATION_INVALID,
+  E_EXECUTION_PROFILE_INVALID,
+  E_EXECUTION_PROFILE_INCONSISTENT,
+  E_EXECUTION_PROFILE_SAFETY_FLOOR_INVALID,
+  E_USAGE_INVALID,
+  E_USAGE_SOURCE_INVALID,
+  E_EFFICIENCY_BASELINE_INVALID,
   E_RECONCILE_NOT_STALE,
   E_RECONCILE_PHASE_INVALID,
   E_RECONCILE_UNSUPPORTED_DRIFT,