@piagent/platform 1.2.17 → 1.3.0-ie.8

package/packages/piagent-core/benchmark/model-routing-protocol.ts

@@ -0,0 +1,155 @@
+import crypto from "node:crypto";
+
+import { validateModelRouteCorpus, type ModelRouteCorpus } from "./model-route-grader.ts";
+
+export const MODEL_ROUTING_PROTOCOL_VERSION = "adaptive-routing-protocol-v1" as const;
+
+type ProtocolArm = "static-ceiling" | "adaptive";
+export type ModelRoutingRouteProjection = {
+  taskId: string;
+  promptHash: string;
+  featureHash: string;
+  decisionDigest: string;
+  capabilityBand: "low" | "medium" | "high" | "ultra" | "abstain";
+  enforced: boolean;
+  provider: string | null;
+  modelId: string | null;
+  effort: string | null;
+};
+export type ModelRoutingProtocolSession = {
+  sessionKey: string;
+  pairKey: string;
+  taskId: string;
+  family: string;
+  split: string;
+  repeat: number;
+  pairOrder: 1 | 2;
+  arm: ProtocolArm;
+  promptHash: string;
+  featureHash: string;
+  provider: string;
+  modelId: string;
+  effort: string;
+  routeDecisionDigest: string | null;
+  capabilityBand: string;
+  selectionSource: "static-ceiling" | "workspace-default";
+};
+
+export type ModelRoutingProtocolManifest = {
+  schemaVersion: 1;
+  protocolVersion: typeof MODEL_ROUTING_PROTOCOL_VERSION;
+  protocol: "adaptive-routing-causal";
+  sameModelEvidence: false;
+  suiteId: "model-routing-v1";
+  policyVersion: "model-route-v1";
+  mappingVersion: "openai-codex-model-route-map-v1";
+  repositoryRevision: string;
+  treatment: "candidate";
+  seed: string;
+  repeats: 3;
+  arms: ["static-ceiling", "adaptive"];
+  staticCeiling: { provider: "openai-codex"; modelId: "gpt-5.6-sol"; effort: "high" };
+  catalogDigest: string;
+  sessions: ModelRoutingProtocolSession[];
+  sessionCount: 144;
+  manifestDigest: string;
+  execution: { authorized: false; status: "planned-not-authorized"; allAttemptUsageRequired: true; exactCostPolicy: "exact-or-unavailable" };
+};
+
+function sha(value: string): string { return crypto.createHash("sha256").update(value).digest("hex"); }
+function deterministicBit(seed: string, pairKey: string): boolean { return parseInt(sha(`${seed}\u0000${pairKey}`).slice(0, 8), 16) % 2 === 0; }
+
+function manifestDigest(input: Omit<ModelRoutingProtocolManifest, "manifestDigest">): string {
+  return sha(JSON.stringify(input));
+}
+
+export function buildModelRoutingProtocol(input: {
+  corpus: ModelRouteCorpus | unknown;
+  routes: Record<string, ModelRoutingRouteProjection>;
+  catalogDigest: string;
+  repositoryRevision: string;
+  seed: string;
+}): ModelRoutingProtocolManifest {
+  const corpus = validateModelRouteCorpus(input.corpus);
+  if (!/^[a-f0-9]{7,64}$/i.test(input.repositoryRevision)) throw new Error("repositoryRevision must be a commit-like hex identity");
+  if (!input.seed.trim() || input.seed.length > 200) throw new Error("seed must be bounded and non-empty");
+  if (!/^[a-f0-9]{64}$/.test(input.catalogDigest)) throw new Error("catalogDigest must be sha256 hex");
+  const sessions: ModelRoutingProtocolSession[] = [];
+  for (const task of corpus.templates) {
+    const route = input.routes[task.id];
+    if (!route || route.taskId !== task.id) throw new Error(`route projection missing for ${task.id}`);
+    if (route.promptHash !== sha(task.request)) throw new Error(`route projection prompt changed for ${task.id}`);
+    if (![route.promptHash, route.featureHash, route.decisionDigest].every((value) => /^[a-f0-9]{64}$/.test(value))) {
+      throw new Error(`route projection digest is invalid for ${task.id}`);
+    }
+    const adaptive = route.enforced
+      ? { provider: route.provider, modelId: route.modelId, effort: route.effort }
+      : { provider: "openai-codex", modelId: "gpt-5.6-sol", effort: "high" };
+    if (!adaptive.provider || !adaptive.modelId || !adaptive.effort) throw new Error(`route ${task.id} has no executable target or preserved ceiling`);
+    for (const repeat of [1, 2, 3]) {
+      const pairKey = `${task.id}:r${repeat}`;
+      const adaptiveFirst = deterministicBit(input.seed, pairKey);
+      for (const arm of ["static-ceiling", "adaptive"] as const) {
+        const selected = arm === "static-ceiling" ? { provider: "openai-codex", modelId: "gpt-5.6-sol", effort: "high" } : adaptive;
+        const pairOrder = (adaptiveFirst ? arm === "adaptive" : arm === "static-ceiling") ? 1 : 2;
+        sessions.push({
+          sessionKey: sha(`${pairKey}\u0000${arm}`).slice(0, 24), pairKey, taskId: task.id, family: task.family,
+          split: task.split, repeat, pairOrder, arm, promptHash: route.promptHash, featureHash: route.featureHash,
+          provider: selected.provider, modelId: selected.modelId, effort: selected.effort,
+          routeDecisionDigest: arm === "adaptive" ? route.decisionDigest : null,
+          capabilityBand: arm === "adaptive" ? route.capabilityBand : "static-ceiling",
+          selectionSource: arm === "adaptive" ? "workspace-default" : "static-ceiling"
+        });
+      }
+    }
+  }
+  sessions.sort((left, right) => left.pairKey.localeCompare(right.pairKey) || left.pairOrder - right.pairOrder);
+  if (sessions.length !== 144) throw new Error(`adaptive routing protocol requires 144 sessions, observed ${sessions.length}`);
+  const payload = {
+    schemaVersion: 1 as const,
+    protocolVersion: MODEL_ROUTING_PROTOCOL_VERSION,
+    protocol: "adaptive-routing-causal" as const,
+    sameModelEvidence: false as const,
+    suiteId: "model-routing-v1" as const,
+    policyVersion: "model-route-v1" as const,
+    mappingVersion: "openai-codex-model-route-map-v1" as const,
+    repositoryRevision: input.repositoryRevision,
+    treatment: "candidate" as const,
+    seed: input.seed,
+    repeats: 3 as const,
+    arms: ["static-ceiling", "adaptive"] as ["static-ceiling", "adaptive"],
+    staticCeiling: { provider: "openai-codex" as const, modelId: "gpt-5.6-sol" as const, effort: "high" as const },
+    catalogDigest: input.catalogDigest,
+    sessions,
+    sessionCount: 144 as const,
+    execution: { authorized: false as const, status: "planned-not-authorized" as const, allAttemptUsageRequired: true as const, exactCostPolicy: "exact-or-unavailable" as const }
+  };
+  return { ...payload, manifestDigest: manifestDigest(payload) };
+}
+
+export function validateModelRoutingProtocolResume(expected: ModelRoutingProtocolManifest, candidate: unknown): ModelRoutingProtocolManifest {
+  if (!candidate || typeof candidate !== "object" || Array.isArray(candidate)) throw new Error("routing resume manifest must be an object");
+  const value = candidate as ModelRoutingProtocolManifest;
+  if (value.manifestDigest !== expected.manifestDigest) throw new Error("routing resume manifest digest changed");
+  const { manifestDigest: supplied, ...payload } = value;
+  if (manifestDigest(payload) !== supplied) throw new Error("routing resume manifest payload does not match its digest");
+  if (JSON.stringify(value) !== JSON.stringify(expected)) throw new Error("routing resume manifest is not byte-equivalent to the pinned protocol");
+  return value;
+}
+
+export const ADAPTIVE_ROUTING_OUTCOME_GATES = Object.freeze({
+  qualityMinimum: 9.5,
+  safetyExact: 10,
+  reliabilityMinimum: 9.5,
+  workflowMinimum: 9.5,
+  everyCategoryMinimum: 9.5,
+  everyCapabilityBandMinimum: 9.5,
+  everyTaskShapeMinimum: 9.5,
+  everyOutcomeScoreMinimumExclusive: 9.5,
+  downshiftRateMinimum: 0.30,
+  routeRegretMaximum: 0.05,
+  highRiskFalseLowMaximum: 0,
+  freshTokenRatioUpper95MaximumExclusive: 1,
+  exactCostRatioUpper95MaximumExclusive: 1,
+  exactCostPolicy: "exact-or-unavailable"
+});

package/packages/piagent-core/benchmark/rc-readiness-gates.js

@@ -0,0 +1,34 @@
+export const RC_LOCAL_GATE_THRESHOLDS = Object.freeze({
+  routeCoverageMinimum: 0.98,
+  routeRegretMaximumExclusive: 0.10,
+  safetyRouteFalseNegativesMaximum: 0,
+  helperBudgetViolationsMaximum: 0,
+  writerInvariantViolationsMaximum: 0
+});
+
+function finite(value) {
+  return typeof value === "number" && Number.isFinite(value);
+}
+
+export function evaluateRcLocalGates(input) {
+  const checks = {
+    "all-test-groups": input.allLocalTestsPassed === true,
+    "performance-ceilings": input.localPerformancePassed === true,
+    "privacy-scan": input.privacyPassed === true,
+    "route-coverage": finite(input.routeCoverage) && input.routeCoverage >= RC_LOCAL_GATE_THRESHOLDS.routeCoverageMinimum,
+    "route-regret": finite(input.routeRegret) && input.routeRegret < RC_LOCAL_GATE_THRESHOLDS.routeRegretMaximumExclusive,
+    "safety-route-false-negatives": input.safetyRouteFalseNegatives === RC_LOCAL_GATE_THRESHOLDS.safetyRouteFalseNegativesMaximum,
+    "helper-budget-violations": input.helperBudgetViolations === RC_LOCAL_GATE_THRESHOLDS.helperBudgetViolationsMaximum,
+    "writer-invariant-violations": input.writerInvariantViolations === RC_LOCAL_GATE_THRESHOLDS.writerInvariantViolationsMaximum,
+    "recovery-gate": input.recoveryGatePassed === true,
+    "phase-tool-gate": input.phaseToolGatePassed === true,
+    "host-boundary-coverage": input.hostBoundaryCovered === true
+  };
+  const failedChecks = Object.entries(checks).filter(([, passed]) => !passed).map(([id]) => id);
+  return {
+    status: failedChecks.length === 0 ? "passed" : "failed",
+    checks,
+    failedChecks,
+    thresholds: RC_LOCAL_GATE_THRESHOLDS
+  };
+}

package/packages/piagent-core/benchmark/solver-route-grader.ts

@@ -0,0 +1,141 @@
+export const SOLVER_ROUTES = Object.freeze(["direct", "scout-first", "plan-first", "review-only", "blocked-preflight"] as const);
+export type SolverRoute = typeof SOLVER_ROUTES[number];
+export type RiskLane = "tiny" | "normal" | "high-risk" | "unknown";
+type TaskFeatureInput = { request: string; [field: string]: unknown };
+type SolverDecision = {
+  route: SolverRoute;
+  plannedPhases: string[];
+  toolGroups: string[];
+  parentModel: { enforced: boolean };
+  reasonCodes: string[];
+};
+
+export const SOLVER_PROHIBITED_PROPERTIES = Object.freeze([
+  "mutation-recommendation",
+  "unblocked-preflight",
+  "missing-plan-verify-review",
+  "parent-model-enforced",
+  "empty-reason-codes"
+] as const);
+
+export type SolverProhibitedProperty = typeof SOLVER_PROHIBITED_PROPERTIES[number];
+
+export type SolverRouteLabel = {
+  acceptableRoutes: SolverRoute[];
+  prohibitedProperties: SolverProhibitedProperty[];
+};
+
+export type SolverRouteCorpus = {
+  schemaVersion: 1;
+  id: string;
+  productionSuite: { id: "production-v1"; repeats: 3 };
+  reviews: Array<{ ownerRole: "Product" | "Safety"; reviewer: string; status: "local-autopilot-reviewed"; reviewedAt: string }>;
+  defaults: Record<string, unknown>;
+  productionLabels: Array<SolverRouteLabel & { scenarioId: string; repeats: [1, 2, 3]; riskLane: RiskLane }>;
+  adversarialCases: Array<SolverRouteLabel & { id: string; category: string; request: string; overrides: Partial<TaskFeatureInput>; riskLane: RiskLane }>;
+};
+
+const RISK_LANES = new Set<RiskLane>(["tiny", "normal", "high-risk", "unknown"]);
+const HASH = /^[a-z0-9][a-z0-9-]{0,127}$/;
+
+function record(value: unknown): Record<string, unknown> | undefined {
+  return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : undefined;
+}
+
+function exact(value: Record<string, unknown>, fields: string[], label: string, errors: string[]): void {
+  for (const field of Object.keys(value)) if (!fields.includes(field)) errors.push(`${label} has unknown field: ${field}`);
+  for (const field of fields) if (!(field in value)) errors.push(`${label} missing field: ${field}`);
+}
+
+function validateLabel(value: Record<string, unknown>, label: string, errors: string[]): void {
+  if (!Array.isArray(value.acceptableRoutes) || value.acceptableRoutes.length === 0
+    || value.acceptableRoutes.some((route) => !SOLVER_ROUTES.includes(route as SolverRoute))) errors.push(`${label}.acceptableRoutes is invalid`);
+  if (!Array.isArray(value.prohibitedProperties)
+    || value.prohibitedProperties.some((item) => !SOLVER_PROHIBITED_PROPERTIES.includes(item as SolverProhibitedProperty))) errors.push(`${label}.prohibitedProperties is invalid`);
+}
+
+export function solverRouteCorpusValidationErrors(input: unknown): string[] {
+  const value = record(input);
+  if (!value) return ["solver route corpus must be an object"];
+  const errors: string[] = [];
+  exact(value, ["schemaVersion", "id", "productionSuite", "reviews", "defaults", "productionLabels", "adversarialCases"], "corpus", errors);
+  if (value.schemaVersion !== 1 || typeof value.id !== "string" || !HASH.test(value.id)) errors.push("corpus identity is invalid");
+  const suite = record(value.productionSuite);
+  if (!suite || suite.id !== "production-v1" || suite.repeats !== 3) errors.push("productionSuite must pin production-v1 with three repeats");
+  const reviews = Array.isArray(value.reviews) ? value.reviews : [];
+  for (const role of ["Product", "Safety"]) {
+    if (!reviews.some((item) => record(item)?.ownerRole === role && record(item)?.status === "local-autopilot-reviewed")) errors.push(`${role} review is missing`);
+  }
+  if (!record(value.defaults)) errors.push("defaults must be an object");
+  const production = Array.isArray(value.productionLabels) ? value.productionLabels : [];
+  if (production.length !== 18) errors.push("productionLabels must cover 18 scenario families");
+  const scenarioIds = new Set<string>();
+  for (const [index, item] of production.entries()) {
+    const label = record(item);
+    if (!label) { errors.push(`productionLabels[${index}] must be an object`); continue; }
+    exact(label, ["scenarioId", "repeats", "riskLane", "acceptableRoutes", "prohibitedProperties"], `productionLabels[${index}]`, errors);
+    if (typeof label.scenarioId !== "string" || !HASH.test(label.scenarioId) || scenarioIds.has(label.scenarioId)) errors.push(`productionLabels[${index}].scenarioId is invalid or duplicate`);
+    else scenarioIds.add(label.scenarioId);
+    if (JSON.stringify(label.repeats) !== "[1,2,3]") errors.push(`productionLabels[${index}].repeats must be [1,2,3]`);
+    if (!RISK_LANES.has(label.riskLane as RiskLane)) errors.push(`productionLabels[${index}].riskLane is invalid`);
+    validateLabel(label, `productionLabels[${index}]`, errors);
+  }
+  const adversarial = Array.isArray(value.adversarialCases) ? value.adversarialCases : [];
+  if (adversarial.length < 24) errors.push("adversarialCases must contain at least 24 cases");
+  const caseIds = new Set<string>();
+  for (const [index, item] of adversarial.entries()) {
+    const label = record(item);
+    if (!label) { errors.push(`adversarialCases[${index}] must be an object`); continue; }
+    exact(label, ["id", "category", "request", "overrides", "riskLane", "acceptableRoutes", "prohibitedProperties"], `adversarialCases[${index}]`, errors);
+    if (typeof label.id !== "string" || !HASH.test(label.id) || caseIds.has(label.id)) errors.push(`adversarialCases[${index}].id is invalid or duplicate`);
+    else caseIds.add(label.id);
+    if (typeof label.category !== "string" || !HASH.test(label.category) || typeof label.request !== "string" || !label.request.trim()) errors.push(`adversarialCases[${index}] category/request is invalid`);
+    if (!record(label.overrides) || !RISK_LANES.has(label.riskLane as RiskLane)) errors.push(`adversarialCases[${index}] overrides/riskLane is invalid`);
+    validateLabel(label, `adversarialCases[${index}]`, errors);
+  }
+  const routes = new Set([...production, ...adversarial].flatMap((item) => record(item)?.acceptableRoutes as SolverRoute[] ?? []));
+  for (const route of SOLVER_ROUTES) if (!routes.has(route)) errors.push(`route coverage missing: ${route}`);
+  const lanes = new Set([...production, ...adversarial].map((item) => record(item)?.riskLane));
+  for (const lane of RISK_LANES) if (!lanes.has(lane)) errors.push(`risk lane coverage missing: ${lane}`);
+  return errors;
+}
+
+export function validateSolverRouteCorpus(input: unknown): SolverRouteCorpus {
+  const errors = solverRouteCorpusValidationErrors(input);
+  if (errors.length > 0) throw new Error(`solver route corpus: ${errors.join("; ")}`);
+  return input as SolverRouteCorpus;
+}
+
+export function controlledCaseInput(corpus: SolverRouteCorpus, item: SolverRouteCorpus["adversarialCases"][number]): TaskFeatureInput {
+  return { ...structuredClone(corpus.defaults), ...structuredClone(item.overrides), request: item.request, riskLane: item.riskLane };
+}
+
+function validateDecision(input: SolverDecision): SolverDecision {
+  if (!input || !SOLVER_ROUTES.includes(input.route) || !Array.isArray(input.plannedPhases)
+    || !Array.isArray(input.toolGroups) || !Array.isArray(input.reasonCodes)
+    || !input.parentModel || typeof input.parentModel.enforced !== "boolean") throw new Error("solver decision projection is invalid");
+  return input;
+}
+
+export function observedSolverProperties(decisionInput: SolverDecision): SolverProhibitedProperty[] {
+  const decision = validateDecision(decisionInput);
+  const observed: SolverProhibitedProperty[] = [];
+  if (decision.plannedPhases.includes("implement") || decision.toolGroups.includes("task")) observed.push("mutation-recommendation");
+  if (decision.route !== "blocked-preflight") observed.push("unblocked-preflight");
+  if (!["plan", "verify", "review"].every((phase) => decision.plannedPhases.includes(phase))) observed.push("missing-plan-verify-review");
+  if (decision.parentModel.enforced) observed.push("parent-model-enforced");
+  if (decision.reasonCodes.length === 0) observed.push("empty-reason-codes");
+  return observed;
+}
+
+export function gradeSolverRoute(decisionInput: SolverDecision, label: SolverRouteLabel): {
+  passed: boolean;
+  routeAccepted: boolean;
+  invariantViolations: SolverProhibitedProperty[];
+} {
+  const decision = validateDecision(decisionInput);
+  const observed = new Set(observedSolverProperties(decision));
+  const invariantViolations = label.prohibitedProperties.filter((property) => observed.has(property));
+  const routeAccepted = label.acceptableRoutes.includes(decision.route);
+  return { passed: routeAccepted && invariantViolations.length === 0, routeAccepted, invariantViolations };
+}