chainlesschain 0.166.59 → 0.166.63

package/src/lib/evolution/pm-exploration-benchmark.js

@@ -0,0 +1,253 @@
+/** PM dataset tooling and launch preflight. No runtime or promotion authority. */
+import { createHash } from "node:crypto";
+import { isProxy } from "node:util/types";
+import {
+  buildEvolutionEvalSuite,
+  computeEvolutionEvalTrainingPartitionDigest,
+  verifyEvolutionEvalSuite,
+} from "./evolution-eval-gate.js";
+import { createPmExplorationPlan } from "./pm-exploration-rounds.js";
+import grader from "./pm-result-grader.cjs";
+
+const GROUPS = ["template", "project", "principal", "timeWindow"];
+const PREFIXES = ["template", "project", "principal", "time-window"];
+const GROUP_DIGEST = /^[a-f0-9]{64}$/u;
+const REQUIRED_ENV = Object.freeze({
+  NODE_ENV: "production",
+  MOCK_LLM: "false",
+  MOCK_HARDWARE: "false",
+  SKIP_SLOW_INIT: "false",
+  CHAINLESSCHAIN_DISABLE_NATIVE_DB: "0",
+  CHAINLESSCHAIN_DISABLE_DB_PERSISTENCE: "0",
+  CC_IPC_ACTOR_GUARD: "enforce",
+  CC_IPC_RBAC_GUARD: "enforce",
+});
+const PM_GRADER = "pm-objective-outcome-v1";
+
+function exact(value, keys, label) {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    isProxy(value) ||
+    Object.getPrototypeOf(value) !== Object.prototype
+  )
+    throw new TypeError(`${label} must be plain data`);
+  const actual = Reflect.ownKeys(value);
+  if (
+    actual.length !== keys.length ||
+    actual.some((key) => {
+      const descriptor = Object.getOwnPropertyDescriptor(value, key);
+      return (
+        !keys.includes(key) ||
+        !descriptor.enumerable ||
+        !("value" in descriptor)
+      );
+    })
+  )
+    throw new TypeError(`${label} has unexpected or missing fields`);
+}
+
+function groupKey(kind, value) {
+  if (
+    typeof value !== "string" ||
+    value.trim() !== value ||
+    !value ||
+    value.length > 256
+  )
+    throw new TypeError(`invalid ${kind} group`);
+  return `${kind}-${createHash("sha256").update(value).digest("hex")}`;
+}
+
+/** Trusted dataset-author input; store the result outside the Actor workspace. */
+export function buildPmExplorationSuite(input) {
+  exact(input, ["suiteId", "datasetVersion", "tasks"], "PM suite input");
+  if (
+    !Array.isArray(input.tasks) ||
+    input.tasks.length < 3 ||
+    input.tasks.length > 10_000
+  )
+    throw new TypeError(
+      "PM suite requires bounded training, validation and test tasks",
+    );
+  const tasks = input.tasks.map((task) => {
+    exact(task, ["id", "split", "groups", "prompt", "expected"], "PM task");
+    exact(task.groups, GROUPS, "PM task groups");
+    if (
+      typeof task.prompt !== "string" ||
+      !task.prompt.trim() ||
+      task.prompt.length > 16_384
+    )
+      throw new TypeError("PM task prompt is required and bounded");
+    const expected = grader.normalizePmExpectation(task.expected);
+    return {
+      id: task.id,
+      split: task.split,
+      groupKeys: GROUPS.map((key, i) =>
+        groupKey(PREFIXES[i], task.groups[key]),
+      ),
+      taskType: expected.kind === "file-export" ? "file" : "retrieval",
+      publicInput: { prompt: task.prompt },
+      graderId: PM_GRADER,
+      privateExpected: expected,
+    };
+  });
+  return verifyPmSuite(
+    buildEvolutionEvalSuite({
+      suiteId: input.suiteId,
+      datasetVersion: input.datasetVersion,
+      tasks,
+    }),
+  );
+}
+
+function verifyPmSuite(value) {
+  const suite = verifyEvolutionEvalSuite(value);
+  if (
+    !["training", "validation", "test"].every((split) =>
+      suite.tasks.some((task) => task.split === split),
+    )
+  )
+    throw new TypeError("PM suite must include all three partitions");
+  for (const task of suite.tasks) {
+    exact(task.publicInput, ["prompt"], "PM public input");
+    if (
+      typeof task.publicInput.prompt !== "string" ||
+      !task.publicInput.prompt.trim() ||
+      task.publicInput.prompt.length > 16_384
+    )
+      throw new TypeError("invalid PM public prompt");
+    const expected = grader.normalizePmExpectation(task.privateExpected);
+    if (
+      task.graderId !== PM_GRADER ||
+      task.taskType !==
+        (expected.kind === "file-export" ? "file" : "retrieval") ||
+      !task.groupKeys.every(
+        (key, index) =>
+          key.startsWith(`${PREFIXES[index]}-`) &&
+          GROUP_DIGEST.test(key.slice(PREFIXES[index].length + 1)),
+      )
+    )
+      throw new TypeError("suite is not a PM exploration dataset");
+  }
+  return suite;
+}
+
+/** Explicit projection: no validation/test tasks, expected answers or grader IDs. */
+export function projectPmExplorationTrainingView(value) {
+  const suite = verifyPmSuite(value);
+  return Object.freeze({
+    suiteDigest: suite.suiteDigest,
+    trainingPartitionDigest: computeEvolutionEvalTrainingPartitionDigest(suite),
+    tasks: Object.freeze(
+      suite.tasks
+        .filter((task) => task.split === "training")
+        .map((task) =>
+          Object.freeze({ id: task.id, publicInput: task.publicInput }),
+        ),
+    ),
+  });
+}
+
+/** Structural check only; signed provenance is still verified by EvolutionEvalGate. */
+export function assertPmExplorationTrainingSources(value, input) {
+  exact(
+    input,
+    ["trainingPartitionDigest", "sourceTaskIds"],
+    "PM memory sources",
+  );
+  const view = projectPmExplorationTrainingView(value);
+  if (input.trainingPartitionDigest !== view.trainingPartitionDigest)
+    throw new Error("PM training partition binding mismatch");
+  const ids = input.sourceTaskIds;
+  if (
+    !Array.isArray(ids) ||
+    ids.length === 0 ||
+    ids.length > view.tasks.length ||
+    new Set(ids).size !== ids.length
+  )
+    throw new TypeError("PM source task IDs must be nonempty and unique");
+  const allowed = new Set(view.tasks.map((task) => task.id));
+  if (ids.some((id) => !allowed.has(id)))
+    throw new Error("PM memory source is outside the training partition");
+  return Object.freeze({
+    trainingPartitionDigest: view.trainingPartitionDigest,
+    sourceTaskIds: Object.freeze([...ids]),
+  });
+}
+
+/** Builds the round protocol from the verified training-only projection. */
+export function buildPmExplorationRoundPlan(value, input) {
+  exact(
+    input,
+    [
+      "planId",
+      "environmentDigest",
+      "initialMemoryDigest",
+      "broadBranchIds",
+      "maxRounds",
+      "maxTokens",
+      "maxToolCalls",
+      "maxWallClockMs",
+      "maxConsecutiveNoGain",
+    ],
+    "PM round plan input",
+  );
+  const training = projectPmExplorationTrainingView(value);
+  return createPmExplorationPlan({
+    ...input,
+    suiteDigest: training.suiteDigest,
+    trainingPartitionDigest: training.trainingPartitionDigest,
+    trainingTaskIds: training.tasks.map((task) => task.id),
+  });
+}
+
+export function buildPmExplorationLaunchProfile(budget) {
+  exact(budget, ["maxTokens", "maxToolCalls", "maxWallClockMs"], "PM budget");
+  for (const key of Object.keys(budget)) {
+    if (!Number.isSafeInteger(budget[key]) || budget[key] <= 0)
+      throw new TypeError(`${key} must be a positive safe integer`);
+  }
+  return Object.freeze({
+    schema: "chainlesschain.pm-exploration-launch-profile/v1",
+    environment: REQUIRED_ENV,
+    budget: Object.freeze({ ...budget }),
+    budgetEnforced: false,
+    runtimeVerified: false,
+    productionQualified: false,
+  });
+}
+
+/** No secrets or raw environment values are copied to diagnostics. */
+export function inspectPmExplorationEnvironment(environment = process.env) {
+  if (!environment || typeof environment !== "object" || isProxy(environment))
+    throw new TypeError("environment must be a data object");
+  const issues = Object.entries(REQUIRED_ENV).flatMap(([key, required]) => {
+    const prop = Object.getOwnPropertyDescriptor(environment, key);
+    return prop && "value" in prop && prop.value === required
+      ? []
+      : [
+          Object.freeze({
+            code: "PM_RUNTIME_CONFIGURATION_MISMATCH",
+            variable: key,
+            required,
+          }),
+        ];
+  });
+  return Object.freeze({
+    schema: "chainlesschain.pm-exploration-preflight/v1",
+    status: issues.length ? "blocked" : "requires-host-verification",
+    configurationCompatible: issues.length === 0,
+    issues: Object.freeze(issues),
+    missingEvidence: Object.freeze([
+      "authenticated-model-ingress",
+      "unlocked-real-identity",
+      "enforced-tool-and-workspace-isolation",
+      "host-enforced-budget",
+      "independent-read-only-grader",
+      "reset-and-restart-persistence",
+    ]),
+    launchAllowed: false,
+    runtimeVerified: false,
+    productionQualified: false,
+  });
+}

package/src/lib/evolution/pm-exploration-budget-executor.js

@@ -0,0 +1,346 @@
+import { createHash } from "node:crypto";
+import { performance } from "node:perf_hooks";
+import { isProxy } from "node:util/types";
+
+export const PM_EXPLORATION_BUDGET_OUTCOME_SCHEMA =
+  "chainlesschain.pm-exploration-budget-outcome/v1";
+
+const TOOL_ID = /^[a-z][a-z0-9]*(?:[._:-][a-z0-9]+)*$/u;
+
+function exact(value, keys, label) {
+  if (
+    !value ||
+    typeof value !== "object" ||
+    Array.isArray(value) ||
+    isProxy(value) ||
+    Object.getPrototypeOf(value) !== Object.prototype
+  ) {
+    throw new TypeError(`${label} must be a plain object`);
+  }
+  const actual = Reflect.ownKeys(value);
+  if (
+    actual.length !== keys.length ||
+    actual.some((key) => {
+      const descriptor = Object.getOwnPropertyDescriptor(value, key);
+      return (
+        typeof key !== "string" ||
+        !keys.includes(key) ||
+        !descriptor ||
+        !descriptor.enumerable ||
+        !("value" in descriptor)
+      );
+    })
+  ) {
+    throw new TypeError(`${label} has unexpected or accessor fields`);
+  }
+}
+
+function integer(value, label) {
+  if (!Number.isSafeInteger(value) || value < 0)
+    throw new TypeError(`${label} must be a non-negative safe integer`);
+  return value;
+}
+
+function copyJson(
+  value,
+  label,
+  context = { bytes: 0, nodes: 0, seen: new WeakSet() },
+) {
+  if (
+    value === null ||
+    typeof value === "string" ||
+    typeof value === "boolean"
+  ) {
+    context.bytes += Buffer.byteLength(JSON.stringify(value));
+    if (context.bytes > 1024 * 1024)
+      throw new TypeError(`${label} exceeds its byte bound`);
+    return value;
+  }
+  if (typeof value === "number" && Number.isFinite(value)) return value;
+  if (
+    !value ||
+    typeof value !== "object" ||
+    isProxy(value) ||
+    context.seen.has(value) ||
+    context.nodes >= 20_000
+  ) {
+    throw new TypeError(`${label} must be bounded acyclic JSON data`);
+  }
+  context.seen.add(value);
+  context.nodes += 1;
+  if (Array.isArray(value)) {
+    if (
+      Object.getPrototypeOf(value) !== Array.prototype ||
+      Reflect.ownKeys(value).length !== value.length + 1
+    ) {
+      throw new TypeError(`${label} must be a dense plain array`);
+    }
+    const result = [];
+    for (let index = 0; index < value.length; index += 1) {
+      const descriptor = Object.getOwnPropertyDescriptor(value, String(index));
+      if (!descriptor || !descriptor.enumerable || !("value" in descriptor))
+        throw new TypeError(`${label} contains a hole or accessor`);
+      result.push(copyJson(descriptor.value, label, context));
+    }
+    context.seen.delete(value);
+    return result;
+  }
+  if (Object.getPrototypeOf(value) !== Object.prototype)
+    throw new TypeError(`${label} must use the default object prototype`);
+  const result = {};
+  for (const key of Reflect.ownKeys(value)) {
+    const descriptor = Object.getOwnPropertyDescriptor(value, key);
+    if (
+      typeof key !== "string" ||
+      !descriptor ||
+      !descriptor.enumerable ||
+      !("value" in descriptor)
+    ) {
+      throw new TypeError(`${label} contains a symbol or accessor`);
+    }
+    context.bytes += Buffer.byteLength(key);
+    if (context.bytes > 1024 * 1024)
+      throw new TypeError(`${label} exceeds its byte bound`);
+    Object.defineProperty(result, key, {
+      value: copyJson(descriptor.value, label, context),
+      enumerable: true,
+      configurable: true,
+      writable: true,
+    });
+  }
+  context.seen.delete(value);
+  return result;
+}
+
+function traceDigest(reason, failureClass) {
+  return `sha256:${createHash("sha256")
+    .update(PM_EXPLORATION_BUDGET_OUTCOME_SCHEMA)
+    .update("\0")
+    .update(reason)
+    .update("\0")
+    .update(failureClass)
+    .digest("hex")}`;
+}
+
+function abortReason(signal) {
+  const reason = signal?.reason;
+  if (reason instanceof PmExplorationBudgetError) return reason.reason;
+  return "parent-aborted";
+}
+
+export class PmExplorationBudgetError extends Error {
+  constructor(reason) {
+    super(`PM exploration execution stopped: ${reason}`);
+    this.name = "AbortError";
+    this.code = "CC_PM_EXPLORATION_BUDGET_EXHAUSTED";
+    this.reason = reason;
+  }
+}
+
+export async function executePmExplorationBudgetedOperation({
+  limits,
+  operation,
+  allowedToolIds = [],
+  invokeTool = null,
+  parentSignal = null,
+  now = performance.now.bind(performance),
+  setTimer = setTimeout,
+  clearTimer = clearTimeout,
+} = {}) {
+  exact(
+    limits,
+    ["maxTokens", "maxToolCalls", "maxWallClockMs"],
+    "PM exploration budget limits",
+  );
+  const maximum = Object.freeze({
+    tokens: integer(limits.maxTokens, "maxTokens"),
+    toolCalls: integer(limits.maxToolCalls, "maxToolCalls"),
+    wallClockMs: integer(limits.maxWallClockMs, "maxWallClockMs"),
+  });
+  if (typeof operation !== "function" || isProxy(operation))
+    throw new TypeError(
+      "PM exploration budget operation must be a direct function",
+    );
+  if (
+    invokeTool !== null &&
+    (typeof invokeTool !== "function" || isProxy(invokeTool))
+  )
+    throw new TypeError("PM exploration tool broker must be a direct function");
+  if (!Array.isArray(allowedToolIds) || isProxy(allowedToolIds))
+    throw new TypeError("allowedToolIds must be an array");
+  const toolIds = new Set();
+  for (const toolId of allowedToolIds) {
+    if (
+      typeof toolId !== "string" ||
+      toolId.length > 256 ||
+      !TOOL_ID.test(toolId) ||
+      toolIds.has(toolId)
+    ) {
+      throw new TypeError("allowedToolIds contains an invalid or duplicate ID");
+    }
+    toolIds.add(toolId);
+  }
+  if (
+    (parentSignal !== null && !(parentSignal instanceof AbortSignal)) ||
+    typeof now !== "function" ||
+    typeof setTimer !== "function" ||
+    typeof clearTimer !== "function"
+  ) {
+    throw new TypeError(
+      "PM exploration budget runtime dependencies are invalid",
+    );
+  }
+
+  const controller = new AbortController();
+  const totals = { tokens: 0, toolCalls: 0 };
+  let activeTools = 0;
+  let closed = false;
+  let stoppedReason = null;
+  const startedAt = Number(now());
+  if (!Number.isFinite(startedAt))
+    throw new TypeError("PM exploration monotonic clock is invalid");
+
+  const stop = (reason) => {
+    if (stoppedReason === null) stoppedReason = reason;
+    if (!controller.signal.aborted)
+      controller.abort(new PmExplorationBudgetError(stoppedReason));
+  };
+  const assertOpen = () => {
+    if (closed) throw new Error("PM exploration budget runtime is closed");
+    if (controller.signal.aborted)
+      throw (
+        controller.signal.reason ?? new PmExplorationBudgetError(stoppedReason)
+      );
+  };
+
+  const runtime = Object.freeze({
+    signal: controller.signal,
+    recordTokens(count) {
+      assertOpen();
+      const delta = integer(count, "token count");
+      if (!Number.isSafeInteger(totals.tokens + delta))
+        throw new TypeError("token total exceeds the safe integer range");
+      totals.tokens += delta;
+      if (totals.tokens > maximum.tokens) {
+        stop("max-tokens");
+        throw controller.signal.reason;
+      }
+      return totals.tokens;
+    },
+    async invokeTool(toolId, input) {
+      assertOpen();
+      if (!toolIds.has(toolId)) {
+        stop("tool-not-allowed");
+        throw controller.signal.reason;
+      }
+      totals.toolCalls += 1;
+      if (totals.toolCalls > maximum.toolCalls) {
+        stop("max-tool-calls");
+        throw controller.signal.reason;
+      }
+      if (invokeTool === null) {
+        stop("tool-broker-unavailable");
+        throw controller.signal.reason;
+      }
+      const safeInput = copyJson(input, "tool input");
+      activeTools += 1;
+      try {
+        const output = await invokeTool(
+          Object.freeze({
+            toolId,
+            input: safeInput,
+            signal: controller.signal,
+          }),
+        );
+        assertOpen();
+        return copyJson(output, "tool output");
+      } finally {
+        activeTools -= 1;
+      }
+    },
+    snapshot() {
+      const current = Number(now());
+      if (!Number.isFinite(current) || current < startedAt)
+        throw new TypeError("PM exploration monotonic clock moved backwards");
+      return Object.freeze({
+        tokens: totals.tokens,
+        toolCalls: totals.toolCalls,
+        wallClockMs: Math.ceil(current - startedAt),
+      });
+    },
+  });
+
+  const parentAbort = () => stop(abortReason(parentSignal));
+  if (parentSignal) {
+    if (parentSignal.aborted) parentAbort();
+    else parentSignal.addEventListener("abort", parentAbort, { once: true });
+  }
+  if (maximum.wallClockMs === 0) stop("max-wall-clock-ms");
+
+  let timer = null;
+  const timeout = new Promise((resolve) => {
+    timer = setTimer(() => {
+      stop("max-wall-clock-ms");
+      resolve(Object.freeze({ kind: "stopped" }));
+    }, maximum.wallClockMs);
+  });
+  const attempted = Promise.resolve()
+    .then(() => {
+      if (controller.signal.aborted)
+        throw (
+          controller.signal.reason ??
+          new PmExplorationBudgetError(stoppedReason)
+        );
+      return operation(runtime);
+    })
+    .then(
+      (value) => Object.freeze({ kind: "value", value }),
+      () => Object.freeze({ kind: "error" }),
+    );
+  const winner = controller.signal.aborted
+    ? Object.freeze({ kind: "stopped" })
+    : await Promise.race([attempted, timeout]);
+
+  if (timer !== null) clearTimer(timer);
+  if (parentSignal && !parentSignal.aborted)
+    parentSignal.removeEventListener("abort", parentAbort);
+  let current = Number(now());
+  if (!Number.isFinite(current) || current < startedAt) current = startedAt;
+  const wallClockMs = Math.ceil(current - startedAt);
+  if (wallClockMs > maximum.wallClockMs) stop("max-wall-clock-ms");
+  if (activeTools !== 0) stop("unsettled-tools");
+  closed = true;
+
+  const metrics = Object.freeze({
+    tokens: totals.tokens,
+    toolCalls: totals.toolCalls,
+    wallClockMs,
+  });
+  let status;
+  let failureClass;
+  let reason;
+  let value = null;
+  if (stoppedReason !== null || winner.kind === "stopped") {
+    status = "aborted";
+    reason = stoppedReason ?? "parent-aborted";
+    failureClass = reason.startsWith("max-") ? "budget" : "sandbox";
+  } else if (winner.kind === "error") {
+    status = "failed";
+    reason = "operation-failed";
+    failureClass = "infrastructure";
+  } else {
+    status = "succeeded";
+    reason = "completed";
+    failureClass = "none";
+    value = winner.value;
+  }
+  return Object.freeze({
+    schema: PM_EXPLORATION_BUDGET_OUTCOME_SCHEMA,
+    status,
+    failureClass,
+    reason,
+    metrics,
+    traceDigest: traceDigest(reason, failureClass),
+    value,
+  });
+}