@piagent/platform 1.6.1 → 1.7.1

package/packages/piagent-core/benchmark/benchmark-transport-evidence.js

@@ -0,0 +1,203 @@
+import { exactBenchmarkMeasuredUsage } from "./benchmark-usage.js";
+
+const benchmarkSurfaces = new Set(["raw-pi", "piagent", "codex-cli"]);
+const BENCHMARK_TRANSPORT_FAILURE_CLASSES = new Set([
+  "provider-transport",
+  "provider-infrastructure",
+  "transport-timeout"
+]);
+
+export const BENCHMARK_TRANSPORT_CIRCUIT_POLICY = Object.freeze({ schemaVersion: 1, maximumFailures: 2 });
+
+export function createBenchmarkTransportCircuit() {
+  return {
+    ...BENCHMARK_TRANSPORT_CIRCUIT_POLICY,
+    failures: 0,
+    state: "closed",
+    lastFailure: null
+  };
+}
+
+export function validBenchmarkTransportCircuit(value) {
+  const lastFailureValid = value?.failures === 0
+    ? value.lastFailure === null
+    : Boolean(value?.lastFailure
+      && Number.isSafeInteger(value.lastFailure.orderIndex) && value.lastFailure.orderIndex > 0
+      && typeof value.lastFailure.scenarioId === "string" && value.lastFailure.scenarioId.length > 0
+      && benchmarkSurfaces.has(value.lastFailure.surface)
+      && Number.isSafeInteger(value.lastFailure.repeat) && value.lastFailure.repeat > 0
+      && Number.isSafeInteger(value.lastFailure.infrastructureAttempt) && value.lastFailure.infrastructureAttempt > 0
+      && typeof value.lastFailure.failure === "string" && value.lastFailure.failure.length > 0);
+  return value?.schemaVersion === BENCHMARK_TRANSPORT_CIRCUIT_POLICY.schemaVersion
+    && value.maximumFailures === BENCHMARK_TRANSPORT_CIRCUIT_POLICY.maximumFailures
+    && Number.isSafeInteger(value.failures)
+    && value.failures >= 0
+    && value.failures <= value.maximumFailures
+    && ["closed", "open"].includes(value.state)
+    && value.state === (value.failures >= value.maximumFailures ? "open" : "closed")
+    && lastFailureValid;
+}
+
+export function observeBenchmarkTransportFailure(circuit, record) {
+  if (!validBenchmarkTransportCircuit(circuit)) throw new Error("Benchmark transport circuit-breaker state is malformed");
+  if (!BENCHMARK_TRANSPORT_FAILURE_CLASSES.has(record?.infrastructureClass)) return circuit;
+  const failures = Math.min(circuit.maximumFailures, circuit.failures + 1);
+  return {
+    ...circuit,
+    failures,
+    state: failures >= circuit.maximumFailures ? "open" : "closed",
+    lastFailure: {
+      orderIndex: record.orderIndex,
+      scenarioId: record.scenarioId,
+      surface: record.surface,
+      repeat: record.repeat,
+      infrastructureAttempt: record.infrastructureAttempt,
+      failure: record.infrastructureFailure ?? record.failure ?? "provider-transport-failure"
+    }
+  };
+}
+
+export function validBenchmarkCandidateOutcome(candidateOutcome) {
+  const validLifecycleOutcome = candidateOutcome?.schemaVersion === 2
+    && candidateOutcome.kind === "terminal-lifecycle-mismatch"
+    && ["completed", "blocked", "aborted", "error", "unknown"].includes(candidateOutcome.expectedOperationStatus)
+    && ["completed", "blocked", "aborted", "error", "unknown"].includes(candidateOutcome.observedOperationStatus)
+    && ["pending", "completed", "refused", "failed", "unknown"].includes(candidateOutcome.expectedTaskStatus)
+    && ["pending", "completed", "refused", "failed", "unknown"].includes(candidateOutcome.observedTaskStatus)
+    && (candidateOutcome.expectedOperationStatus !== candidateOutcome.observedOperationStatus
+      || candidateOutcome.expectedTaskStatus !== candidateOutcome.observedTaskStatus)
+    && Number.isSafeInteger(candidateOutcome.turnIndex) && candidateOutcome.turnIndex > 0;
+  const validLegacyOutcome = candidateOutcome?.schemaVersion === 1
+    && candidateOutcome.kind === "terminal-settlement-mismatch"
+    && ["completed", "blocked", "aborted", "error", "unknown", "refused"].includes(candidateOutcome.expectedSettlement)
+    && ["completed", "blocked", "aborted", "error", "unknown"].includes(candidateOutcome.observedSettlement)
+    && candidateOutcome.expectedSettlement !== candidateOutcome.observedSettlement
+    && Number.isSafeInteger(candidateOutcome.turnIndex) && candidateOutcome.turnIndex > 0;
+  return validLifecycleOutcome || validLegacyOutcome;
+}
+
+export function classifyPreUsageFailure(agent, usage, diagnosticInput,
+  { terminalProviderError = false, usageParsingError = false, candidateOutcome = null } = {}) {
+  const diagnostic = String(diagnosticInput ?? "").toLowerCase();
+  const measuredUsage = exactBenchmarkMeasuredUsage(usage);
+  const measuredCoreUsage = Number.isSafeInteger(usage?.sessions) && usage.sessions > 0
+    && ["input", "output", "cacheRead", "cacheWrite", "total", "fresh"]
+      .every((field) => Number.isSafeInteger(usage?.[field]) && usage[field] >= 0)
+    && usage.fresh === usage.input + usage.output
+    && usage.total === usage.input + usage.output + usage.cacheRead + usage.cacheWrite;
+  const measuredZeroUsage = measuredCoreUsage
+    && ["input", "output", "cacheRead", "cacheWrite", "reasoning", "total", "fresh"]
+      .every((field) => Number(usage[field]) === 0);
+  const measuredStatus = measuredUsage ? "measured-but-unaccepted" : "measured-lower-bound";
+  if (agent.timedOut) {
+    return measuredCoreUsage
+      ? {
+          failure: "agent-timeout-after-observed-usage",
+          class: "transport-timeout",
+          // The recorded tokens remain a useful floor, but a timed-out stream
+          // may have an unobserved suffix and therefore is never exact.
+          usageStatus: "measured-lower-bound",
+          retryable: false
+        }
+      : {
+          failure: "agent-timeout-with-terminal-usage-unknown",
+          class: "transport-timeout",
+          usageStatus: "unknown-after-provider-start",
+          retryable: false
+        };
+  }
+  const codexOutcome = usage?.codexEventOutcome;
+  const providerUnavailable = /\b(?:server(?:s)? (?:are )?(?:currently )?overloaded|temporarily unavailable|service unavailable|try again later)\b/.test(diagnostic);
+  const providerFetchFailed = /(?:\bfetch failed\b|\bnetwork(?: request)? (?:error|failed)\b|\bsocket hang up\b|\bconnection (?:reset|closed|terminated)\b|\beconnreset\b|\betimedout\b|\bund_err_[a-z_]+\b)/.test(diagnostic);
+  if ((terminalProviderError || agent.code !== 0) && measuredCoreUsage && providerFetchFailed) {
+    const afterUsage = Number(usage.fresh) > 0;
+    return {
+      failure: afterUsage
+        ? "provider-fetch-failed-after-measured-usage"
+        : "provider-fetch-failed-with-zero-measured-usage",
+      class: "provider-transport",
+      // A zero-shaped local usage record after a broken transport is not an
+      // authoritative provider receipt. The request may have been accepted
+      // before the stream failed, so zero cannot be claimed or replayed.
+      usageStatus: afterUsage ? measuredStatus : "unknown-after-provider-start",
+      retryable: false
+    };
+  }
+  if ((terminalProviderError || agent.code !== 0) && providerFetchFailed) {
+    return {
+      failure: "provider-fetch-failed-with-usage-unavailable",
+      class: "provider-transport",
+      usageStatus: "unknown-after-provider-start",
+      // A transport exception cannot prove that the provider did not accept
+      // the request. Never replay it blindly as a new paid attempt.
+      retryable: false
+    };
+  }
+  if ((terminalProviderError || agent.code !== 0) && measuredCoreUsage && providerUnavailable) {
+    const afterUsage = Number(usage.fresh) > 0;
+    return {
+      failure: afterUsage
+        ? "provider-temporarily-unavailable-after-measured-usage"
+        : "provider-temporarily-unavailable-with-zero-measured-usage",
+      class: "provider-infrastructure",
+      usageStatus: afterUsage ? measuredStatus : "unknown-after-provider-start",
+      retryable: false
+    };
+  }
+  if (agent.code === 0 && measuredZeroUsage && providerFetchFailed) {
+    return {
+      failure: "provider-fetch-failed-with-zero-measured-usage",
+      class: "provider-transport",
+      usageStatus: "unknown-after-provider-start",
+      retryable: false
+    };
+  }
+  if (agent.code === 0 && measuredZeroUsage && providerUnavailable) {
+    return {
+      failure: "provider-temporarily-unavailable-with-zero-measured-usage",
+      class: "provider-infrastructure",
+      usageStatus: "unknown-after-provider-start",
+      retryable: false
+    };
+  }
+  if (usageParsingError) {
+    return measuredCoreUsage
+      ? {
+          failure: "pi-session-usage-incomplete-after-provider-start",
+          class: "usage-accounting",
+          usageStatus: measuredStatus,
+          retryable: false
+        }
+      : {
+          failure: "pi-session-usage-unavailable-after-provider-start",
+          class: "unknown-cost",
+          usageStatus: "unknown-after-provider-start",
+          retryable: false
+        };
+  }
+  if (codexOutcome?.runValidity === "valid" && codexOutcome.failureClass && measuredUsage) return undefined;
+  const validCandidateOutcome = validBenchmarkCandidateOutcome(candidateOutcome);
+  if (agent.code !== 0 && validCandidateOutcome && measuredUsage && usage.fresh > 0) return undefined;
+  const explicitProviderPolicyRefusal = (
+    /\b(?:provider|safety|cyber safety)\b.{0,96}\b(?:refus(?:al|ed|e)|disallowed|not allowed|blocked)\b/.test(diagnostic)
+    || /\b(?:refus(?:al|ed|e)|disallowed|not allowed|blocked)\b.{0,96}\b(?:provider|safety|cyber safety)\b/.test(diagnostic)
+    || /\b(?:cannot assist|can't assist)\b/.test(diagnostic)
+    || (terminalProviderError && /\b(?:policy|refus(?:al|ed|e)|disallowed|not allowed|cyber safety)\b/.test(diagnostic))
+  );
+  if ((terminalProviderError || agent.code !== 0) && explicitProviderPolicyRefusal) {
+    return measuredCoreUsage
+      ? { failure: "provider-policy-refusal-after-measured-usage", class: "provider-policy", usageStatus: measuredStatus, retryable: false }
+      : { failure: "provider-policy-refusal-with-usage-unavailable", class: "provider-policy", usageStatus: "unknown-after-provider-start", retryable: false };
+  }
+  if (codexOutcome?.runValidity === "invalid_harness") {
+    return {
+      failure: `codex-event-contract-invalid:${(codexOutcome.reasonCodes ?? []).join(",") || "unspecified"}`,
+      class: "harness-contract",
+      usageStatus: measuredUsage ? "measured-but-unaccepted" : "unknown-after-provider-start",
+      retryable: false
+    };
+  }
+  if (agent.code === 0) return undefined;
+  if (measuredCoreUsage) return { failure: `agent-exit-${agent.code}-after-measured-usage`, class: "agent-process", usageStatus: measuredStatus, retryable: false };
+  return { failure: `agent-exit-${agent.code}-with-usage-unavailable`, class: "unknown-cost", usageStatus: "unknown-after-provider-start", retryable: true };
+}

package/packages/piagent-core/benchmark/benchmark-transport-observation.js

@@ -0,0 +1,147 @@
+import fs from "node:fs";
+import path from "node:path";
+import { createHash, createPublicKey, sign } from "node:crypto";
+
+const PROTOCOL = "provider-transport-observation-v1";
+const MAX_BYTES = 64 * 1024 * 1024;
+const digest = value => createHash("sha256").update(value).digest("hex");
+const terminalTypes = new Set(["response.done", "response.completed", "response.incomplete", "response.failed", "error"]);
+
+function requireValue(condition, reason) {
+  if (!condition) throw new Error(`transport-observation:${reason}`);
+}
+
+function privateJournal(file) {
+  requireValue(path.isAbsolute(file) && fs.realpathSync(file) === file, "journal-path");
+  const parent = fs.lstatSync(path.dirname(file)), stat = fs.lstatSync(file);
+  requireValue(parent.isDirectory() && parent.uid === process.getuid() && (parent.mode & 0o777) === 0o700, "journal-parent");
+  requireValue(stat.isFile() && stat.uid === process.getuid() && stat.nlink === 1 && (stat.mode & 0o777) === 0o600, "journal-file");
+  return { dev: stat.dev, ino: stat.ino, parentDev: parent.dev, parentIno: parent.ino };
+}
+
+function usageState(event) {
+  if (!event.usagePresent) return "unknown";
+  const usage = event.usage;
+  const count = value => Number.isSafeInteger(value) && value >= 0;
+  if (!usage || Array.isArray(usage) || !["input_tokens", "output_tokens", "total_tokens"].every(key => count(usage[key]))
+    || usage.input_tokens + usage.output_tokens !== usage.total_tokens) return "invalid";
+  for (const [key, maximum] of [["input_tokens_details", usage.input_tokens], ["output_tokens_details", usage.output_tokens]]) {
+    const details = usage[key];
+    if (details != null && (typeof details !== "object" || Array.isArray(details)
+      || !Object.values(details).every(value => count(value) && value <= maximum))) return "invalid";
+  }
+  const details = usage.input_tokens_details;
+  if ((details?.cached_tokens ?? 0) + (details?.cache_write_tokens ?? 0) > usage.input_tokens) return "invalid";
+  return "provider-reported";
+}
+
+function readJournal(file, identity) {
+  requireValue(JSON.stringify(privateJournal(file)) === JSON.stringify(identity), "journal-identity-changed");
+  const fd = fs.openSync(file, fs.constants.O_RDONLY | fs.constants.O_NOFOLLOW);
+  try {
+    const stat = fs.fstatSync(fd);
+    requireValue(stat.dev === identity.dev && stat.ino === identity.ino && stat.isFile()
+      && stat.uid === process.getuid() && stat.nlink === 1 && (stat.mode & 0o777) === 0o600, "journal-file");
+    requireValue(Number.isSafeInteger(stat.size) && stat.size >= 0 && stat.size <= MAX_BYTES, "journal-size");
+    const bytes = Buffer.alloc(stat.size);
+    let offset = 0;
+    while (offset < bytes.length) {
+      const read = fs.readSync(fd, bytes, offset, bytes.length - offset, offset);
+      requireValue(read > 0, "journal-truncated"); offset += read;
+    }
+    requireValue(fs.fstatSync(fd).size === stat.size
+      && JSON.stringify(privateJournal(file)) === JSON.stringify(identity), "journal-changed-during-read");
+    return bytes;
+  } finally { fs.closeSync(fd); }
+}
+
+/** Private feasibility journal. It deliberately refuses nonempty replay and grants no settlement authority. */
+export function createBenchmarkTransportObservation({ journalPath, identity, privateKey }) {
+  const binding = JSON.parse(JSON.stringify(identity));
+  requireValue(binding && ["runId", "attemptId", "armId", "logicalCallId", "sourceDigest"]
+    .every(key => typeof binding[key] === "string" && binding[key]), "identity");
+  const journalIdentity = privateJournal(journalPath);
+  let expected = readJournal(journalPath, journalIdentity), sequence = 0, previousHash = null, fault = null;
+  requireValue(expected.length === 0, "nonempty-resume-unqualified");
+  const attempts = new Map();
+
+  function assertCurrent() {
+    try {
+      if (fault) throw new Error(fault);
+      requireValue(readJournal(journalPath, journalIdentity).equals(expected), "journal-prefix-changed");
+    } catch (error) {
+      fault ??= error instanceof Error ? error.message : "transport-observation:failed";
+      throw new Error(fault);
+    }
+  }
+
+  function project(event) {
+    requireValue(event && typeof event === "object", "event");
+    if (event.kind === "connection-acquire-reserved") return { kind: event.kind, attemptId: null };
+    requireValue(typeof event.attemptId === "string" && /^[1-9][0-9]*$/.test(event.attemptId), "attempt-id");
+    if (event.kind === "transport-reserved") {
+      requireValue(!attempts.has(event.attemptId) && ["http-sse", "websocket-frame"].includes(event.transport), "duplicate-reservation");
+      requireValue(typeof event.json === "string", "serialized-payload");
+      const bytes = event.entity === null ? Buffer.from(event.json) : Buffer.from(event.entity);
+      const body = JSON.parse(event.json);
+      return { kind: event.kind, attemptId: event.attemptId, transport: event.transport,
+        jsonSha256: digest(event.json), jsonBytes: Buffer.byteLength(event.json),
+        entitySha256: digest(bytes), entityBytes: bytes.length,
+        previousResponseId: typeof body.previous_response_id === "string" ? body.previous_response_id : null };
+    }
+    requireValue(event.kind === "provider-event" && attempts.has(event.attemptId), "event-without-reservation");
+    requireValue(typeof event.eventType === "string" && typeof event.usagePresent === "boolean", "provider-event-shape");
+    const result = { kind: event.kind, attemptId: event.attemptId, eventType: event.eventType,
+      responseId: event.responseId, status: event.status, serviceTier: event.serviceTier,
+      usagePresent: event.usagePresent, usage: event.usage };
+    const serialized = JSON.stringify(result);
+    requireValue(Buffer.byteLength(serialized) <= 16 * 1024, "provider-event-bound");
+    const prior = attempts.get(event.attemptId);
+    if (terminalTypes.has(event.eventType) && prior.terminalHash) {
+      requireValue(prior.terminalHash === digest(serialized), "conflicting-terminal");
+    }
+    return JSON.parse(serialized);
+  }
+
+  function observe(event) {
+    try {
+      assertCurrent();
+      const projected = project(event);
+      const material = JSON.stringify({ protocol: PROTOCOL, identity: binding,
+        sequence: sequence + 1, previousHash, event: projected });
+      const envelope = { material, signature: sign(null, Buffer.from(material), privateKey).toString("base64") };
+      const line = Buffer.from(`${JSON.stringify(envelope)}\n`);
+      requireValue(expected.length + line.length <= MAX_BYTES, "journal-full");
+      const fd = fs.openSync(journalPath, fs.constants.O_APPEND | fs.constants.O_WRONLY | fs.constants.O_NOFOLLOW);
+      try {
+        const stat = fs.fstatSync(fd);
+        requireValue(stat.dev === journalIdentity.dev && stat.ino === journalIdentity.ino, "journal-replaced");
+        fs.writeFileSync(fd, line); fs.fsyncSync(fd);
+      } finally { fs.closeSync(fd); }
+      const next = Buffer.concat([expected, line]);
+      requireValue(JSON.stringify(privateJournal(journalPath)) === JSON.stringify(journalIdentity)
+        && readJournal(journalPath, journalIdentity).equals(next), "durable-readback");
+      expected = next; sequence++; previousHash = digest(line.subarray(0, -1));
+      if (projected.kind === "transport-reserved") {
+        attempts.set(projected.attemptId, { ...projected, usageStatus: "unknown", terminalHash: null });
+      } else if (projected.kind === "provider-event" && terminalTypes.has(projected.eventType)) {
+        const prior = attempts.get(projected.attemptId);
+        prior.terminalHash = digest(JSON.stringify(projected));
+        prior.usageStatus = usageState(projected); prior.usage = projected.usage;
+      }
+    } catch (error) {
+      fault = error instanceof Error ? error.message : "transport-observation:failed";
+      throw new Error(fault);
+    }
+  }
+
+  return Object.freeze({ observe, assertCurrent,
+    status() {
+      return { protocol: PROTOCOL, sequence, previousHash, fault,
+        attempts: structuredClone([...attempts.values()]),
+        unknownSpend: [...attempts.values()].some(attempt => attempt.usageStatus !== "provider-reported"),
+        settlementAuthorized: false, resumeQualified: false,
+        publicKey: createPublicKey(privateKey).export({ type: "spki", format: "der" }).toString("base64") };
+    }
+  });
+}

package/packages/piagent-core/benchmark/benchmark-tree-identity.js

@@ -136,7 +136,7 @@ export function assertBenchmarkTreeIdentity(expected, observed, label) {
   return observed;
 }
 
-export function benchmarkTreeStatIdentity(root, { rejectEscapingSymlinks = false } = {}) {
+export function benchmarkTreeStatIdentity(root, { rejectSymlinks = false, rejectEscapingSymlinks = false } = {}) {
   const canonical = fs.realpathSync(root);
   const hash = crypto.createHash("sha256");
   const pending = [canonical];
@@ -152,6 +152,7 @@ export function benchmarkTreeStatIdentity(root, { rejectEscapingSymlinks = false
       entryCount += 1;
       field(hash, relative);
       if (stat.isSymbolicLink()) {
+        if (rejectSymlinks) fail(`Benchmark tree rejects symbolic link: ${relative}`);
         const payload = fs.readlinkSync(target);
         if (rejectEscapingSymlinks) {
           if (path.isAbsolute(payload)) fail(`Benchmark tree symlink must be relative: ${relative}`);