assertledger 1.0.0

package/schemas/agentic-benchmark-request.v1.json

@@ -0,0 +1,1034 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "$ref": "#/$defs/__schema0"
+    },
+    "sourceManifest": {
+      "$ref": "#/$defs/__schema1"
+    },
+    "referenceWorldId": {
+      "$ref": "#/$defs/__schema7"
+    },
+    "policy": {
+      "$ref": "#/$defs/__schema11"
+    },
+    "protocol": {
+      "$ref": "#/$defs/__schema12"
+    },
+    "fingerprint": {
+      "$ref": "#/$defs/__schema13"
+    },
+    "runs": {
+      "$ref": "#/$defs/__schema15"
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "sourceManifest",
+    "referenceWorldId",
+    "policy",
+    "protocol",
+    "fingerprint",
+    "runs"
+  ],
+  "additionalProperties": false,
+  "title": "TestForge Agentic Benchmark request",
+  "description": "A replay-valid source manifest and raw cold/warm benchmark observations.",
+  "$defs": {
+    "__schema0": {
+      "type": "string",
+      "const": "1.0.0"
+    },
+    "__schema1": {
+      "type": "object",
+      "properties": {
+        "schemaVersion": {
+          "type": "string",
+          "const": "1.0.0"
+        },
+        "repositoryDigest": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/__schema2"
+            },
+            {
+              "type": "string",
+              "const": "invalid"
+            }
+          ]
+        },
+        "evidenceContext": {
+          "type": "object",
+          "properties": {
+            "engine": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 128
+                },
+                "version": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 128
+                }
+              },
+              "required": ["name", "version"],
+              "additionalProperties": false
+            },
+            "adapter": {
+              "type": "object",
+              "properties": {
+                "name": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 128
+                },
+                "version": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 128
+                },
+                "configuration": {
+                  "$ref": "#/$defs/__schema3"
+                }
+              },
+              "required": ["name", "version", "configuration"],
+              "additionalProperties": false
+            },
+            "execution": {
+              "type": "object",
+              "properties": {
+                "isolation": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 128
+                },
+                "environmentAllowlist": {
+                  "maxItems": 100,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema4"
+                  }
+                },
+                "budgets": {
+                  "$ref": "#/$defs/__schema5"
+                },
+                "candidateRoots": {
+                  "maxItems": 100,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema6"
+                  }
+                }
+              },
+              "required": ["isolation", "environmentAllowlist", "budgets", "candidateRoots"],
+              "additionalProperties": false
+            },
+            "worlds": {
+              "type": "array",
+              "items": {
+                "type": "object",
+                "properties": {
+                  "id": {
+                    "$ref": "#/$defs/__schema7"
+                  },
+                  "provenance": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 1024
+                  },
+                  "digest": {
+                    "$ref": "#/$defs/__schema2"
+                  }
+                },
+                "required": ["id", "provenance", "digest"],
+                "additionalProperties": false
+              }
+            }
+          },
+          "required": ["engine", "adapter", "execution", "worlds"],
+          "additionalProperties": false
+        },
+        "policy": {
+          "anyOf": [
+            {
+              "type": "object",
+              "properties": {
+                "policyVersion": {
+                  "type": "string",
+                  "const": "1.0.0"
+                },
+                "requiredAttempts": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 1000
+                },
+                "minimumTargetWeightPermille": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 1000
+                },
+                "maximumSelectedCandidates": {
+                  "type": "integer",
+                  "minimum": 1,
+                  "maximum": 1000
+                },
+                "acceptedTargetOutcomes": {
+                  "minItems": 1,
+                  "maxItems": 1,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema8"
+                  }
+                }
+              },
+              "required": [
+                "policyVersion",
+                "requiredAttempts",
+                "minimumTargetWeightPermille",
+                "maximumSelectedCandidates",
+                "acceptedTargetOutcomes"
+              ],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "policyVersion": {
+                  "type": "string",
+                  "const": "invalid"
+                },
+                "requiredAttempts": {
+                  "type": "number",
+                  "const": 1
+                },
+                "minimumTargetWeightPermille": {
+                  "type": "number",
+                  "const": 1000
+                },
+                "maximumSelectedCandidates": {
+                  "type": "number",
+                  "const": 0
+                },
+                "acceptedTargetOutcomes": {
+                  "maxItems": 0,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema9"
+                  }
+                }
+              },
+              "required": [
+                "policyVersion",
+                "requiredAttempts",
+                "minimumTargetWeightPermille",
+                "maximumSelectedCandidates",
+                "acceptedTargetOutcomes"
+              ],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "worlds": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": {
+                "$ref": "#/$defs/__schema7"
+              },
+              "kind": {
+                "type": "string",
+                "enum": ["REFERENCE", "TARGET", "NEUTRAL"]
+              },
+              "required": {
+                "type": "boolean"
+              },
+              "weight": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 1000000
+              }
+            },
+            "required": ["id", "kind", "required", "weight"],
+            "additionalProperties": false
+          }
+        },
+        "candidates": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "id": {
+                "$ref": "#/$defs/__schema7"
+              },
+              "digest": {
+                "$ref": "#/$defs/__schema2"
+              },
+              "sizeBytes": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "status": {
+                "type": "string",
+                "enum": ["ELIGIBLE", "UNSTABLE", "INCONCLUSIVE", "INVALID", "WEAK_ORACLE"]
+              },
+              "killedTargetIds": {
+                "type": "array",
+                "items": {
+                  "$ref": "#/$defs/__schema7"
+                }
+              },
+              "targetWeightKilled": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "gates": {
+                "type": "array",
+                "items": {
+                  "type": "object",
+                  "properties": {
+                    "name": {
+                      "type": "string",
+                      "enum": [
+                        "COMPLETENESS",
+                        "STABILITY",
+                        "DISCOVERY",
+                        "REFERENCE",
+                        "NEUTRAL",
+                        "TARGET_STRENGTH"
+                      ]
+                    },
+                    "status": {
+                      "type": "string",
+                      "enum": ["PASSED", "FAILED", "NOT_RUN"]
+                    },
+                    "evidenceRunIds": {
+                      "type": "array",
+                      "items": {
+                        "type": "string",
+                        "minLength": 1,
+                        "maxLength": 512
+                      }
+                    },
+                    "reasonCodes": {
+                      "type": "array",
+                      "items": {
+                        "$ref": "#/$defs/__schema10"
+                      }
+                    }
+                  },
+                  "required": ["name", "status", "evidenceRunIds", "reasonCodes"],
+                  "additionalProperties": false
+                }
+              },
+              "reasonCodes": {
+                "type": "array",
+                "items": {
+                  "$ref": "#/$defs/__schema10"
+                }
+              }
+            },
+            "required": [
+              "id",
+              "digest",
+              "sizeBytes",
+              "status",
+              "killedTargetIds",
+              "targetWeightKilled",
+              "gates",
+              "reasonCodes"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "observations": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "properties": {
+              "runId": {
+                "type": "string",
+                "minLength": 1,
+                "maxLength": 512
+              },
+              "candidateId": {
+                "anyOf": [
+                  {
+                    "$ref": "#/$defs/__schema7"
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "worldId": {
+                "$ref": "#/$defs/__schema7"
+              },
+              "attempt": {
+                "type": "integer",
+                "minimum": 1,
+                "maximum": 9007199254740991
+              },
+              "outcome": {
+                "type": "string",
+                "enum": [
+                  "PASS",
+                  "ASSERTION_FAILURE",
+                  "COLLECTION_FAILURE",
+                  "COMPILE_FAILURE",
+                  "PROCESS_CRASH",
+                  "TIMEOUT",
+                  "INFRA_ERROR",
+                  "NO_TEST_DISCOVERED"
+                ]
+              },
+              "testsDiscovered": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "candidateTestsDiscovered": {
+                "type": "integer",
+                "minimum": 0,
+                "maximum": 9007199254740991
+              },
+              "attributed": {
+                "type": "boolean"
+              },
+              "durationMs": {
+                "type": "number",
+                "minimum": 0
+              },
+              "exitCode": {
+                "anyOf": [
+                  {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 9007199254740991
+                  },
+                  {
+                    "type": "null"
+                  }
+                ]
+              },
+              "stdoutDigest": {
+                "$ref": "#/$defs/__schema2"
+              },
+              "stderrDigest": {
+                "$ref": "#/$defs/__schema2"
+              }
+            },
+            "required": [
+              "runId",
+              "candidateId",
+              "worldId",
+              "attempt",
+              "outcome",
+              "testsDiscovered",
+              "candidateTestsDiscovered",
+              "attributed"
+            ],
+            "additionalProperties": false
+          }
+        },
+        "decision": {
+          "type": "object",
+          "properties": {
+            "status": {
+              "type": "string",
+              "enum": ["VERIFIED", "REJECTED", "INCONCLUSIVE", "ENGINE_ERROR"]
+            },
+            "selectedCandidateIds": {
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema7"
+              }
+            },
+            "reasonCodes": {
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema10"
+              }
+            }
+          },
+          "required": ["status", "selectedCandidateIds", "reasonCodes"],
+          "additionalProperties": false
+        },
+        "decisionDigest": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "artifactDigest": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "adapter": {
+          "oneOf": [
+            {
+              "type": "object",
+              "properties": {
+                "kind": {
+                  "type": "string",
+                  "const": "node-test"
+                }
+              },
+              "required": ["kind"],
+              "additionalProperties": false
+            },
+            {
+              "type": "object",
+              "properties": {
+                "kind": {
+                  "type": "string",
+                  "const": "testforge-command"
+                },
+                "protocolVersion": {
+                  "type": "string",
+                  "const": "1.0.0"
+                }
+              },
+              "required": ["kind", "protocolVersion"],
+              "additionalProperties": false
+            }
+          ]
+        },
+        "isolation": {
+          "type": "object",
+          "properties": {
+            "kind": {
+              "type": "string",
+              "const": "trusted-local"
+            },
+            "level": {
+              "type": "string",
+              "const": "UNSANDBOXED"
+            },
+            "acknowledgedUnsafeExecution": {
+              "type": "boolean",
+              "const": true
+            }
+          },
+          "required": ["kind", "level", "acknowledgedUnsafeExecution"],
+          "additionalProperties": false
+        },
+        "limitations": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "minLength": 1,
+            "maxLength": 2048
+          }
+        }
+      },
+      "required": [
+        "schemaVersion",
+        "repositoryDigest",
+        "evidenceContext",
+        "policy",
+        "worlds",
+        "candidates",
+        "observations",
+        "decision",
+        "decisionDigest",
+        "artifactDigest",
+        "adapter",
+        "isolation",
+        "limitations"
+      ],
+      "additionalProperties": false
+    },
+    "__schema2": {
+      "type": "string",
+      "pattern": "^sha256:[a-f0-9]{64}$"
+    },
+    "__schema3": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "number"
+        },
+        {
+          "type": "boolean"
+        },
+        {
+          "type": "null"
+        },
+        {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema3"
+          }
+        },
+        {
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {
+            "$ref": "#/$defs/__schema3"
+          }
+        }
+      ]
+    },
+    "__schema4": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128
+    },
+    "__schema5": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "number"
+        },
+        {
+          "type": "boolean"
+        },
+        {
+          "type": "null"
+        },
+        {
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema5"
+          }
+        },
+        {
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {
+            "$ref": "#/$defs/__schema5"
+          }
+        }
+      ]
+    },
+    "__schema6": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 512
+    },
+    "__schema7": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128,
+      "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$"
+    },
+    "__schema8": {
+      "type": "string",
+      "const": "ASSERTION_FAILURE"
+    },
+    "__schema9": {
+      "type": "string",
+      "const": "ASSERTION_FAILURE"
+    },
+    "__schema10": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 128,
+      "pattern": "^[A-Z][A-Z0-9_]*$"
+    },
+    "__schema11": {
+      "type": "object",
+      "properties": {
+        "benchmarkVersion": {
+          "type": "string",
+          "const": "1.0.0"
+        },
+        "minimumMeasuredSamplesPerRegime": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1000
+        },
+        "coldMeasuredSamples": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1000
+        },
+        "warmupSamples": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1000
+        },
+        "warmMeasuredSamples": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1000
+        },
+        "maximumExecutions": {
+          "type": "integer",
+          "minimum": 1,
+          "maximum": 1000000
+        }
+      },
+      "required": [
+        "benchmarkVersion",
+        "minimumMeasuredSamplesPerRegime",
+        "coldMeasuredSamples",
+        "warmupSamples",
+        "warmMeasuredSamples",
+        "maximumExecutions"
+      ],
+      "additionalProperties": false
+    },
+    "__schema12": {
+      "type": "object",
+      "properties": {
+        "protocolVersion": {
+          "type": "string",
+          "const": "1.0.0"
+        },
+        "clock": {
+          "type": "string",
+          "const": "MONOTONIC"
+        },
+        "unit": {
+          "type": "string",
+          "const": "MICROSECOND"
+        },
+        "quantile": {
+          "type": "string",
+          "const": "NEAREST_RANK"
+        },
+        "phases": {
+          "type": "array",
+          "prefixItems": [
+            {
+              "type": "string",
+              "const": "PREPARATION"
+            },
+            {
+              "type": "string",
+              "const": "STARTUP"
+            },
+            {
+              "type": "string",
+              "const": "COMPILE_OR_COLLECTION"
+            },
+            {
+              "type": "string",
+              "const": "EXECUTION"
+            }
+          ]
+        },
+        "coldDefinition": {
+          "type": "string",
+          "const": "FRESH_WORKSPACE_FRESH_PROCESS_RESET_DECLARED_CACHES"
+        },
+        "warmDefinition": {
+          "type": "string",
+          "const": "FRESH_WORKSPACE_FRESH_PROCESS_RETAIN_DECLARED_CACHES"
+        }
+      },
+      "required": [
+        "protocolVersion",
+        "clock",
+        "unit",
+        "quantile",
+        "phases",
+        "coldDefinition",
+        "warmDefinition"
+      ],
+      "additionalProperties": false
+    },
+    "__schema13": {
+      "type": "object",
+      "properties": {
+        "environmentId": {
+          "$ref": "#/$defs/__schema7"
+        },
+        "os": {
+          "type": "object",
+          "properties": {
+            "platform": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 128
+            },
+            "release": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 128
+            },
+            "arch": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 128
+            }
+          },
+          "required": ["platform", "release", "arch"],
+          "additionalProperties": false
+        },
+        "cpu": {
+          "type": "object",
+          "properties": {
+            "arch": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 128
+            },
+            "model": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 512
+            },
+            "logicalCores": {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 1000000
+            }
+          },
+          "required": ["arch", "model", "logicalCores"],
+          "additionalProperties": false
+        },
+        "logicalCpuLimit": {
+          "anyOf": [
+            {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 1000000
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "memoryLimitBytes": {
+          "anyOf": [
+            {
+              "type": "integer",
+              "minimum": 1,
+              "maximum": 9007199254740991
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "executionBoundary": {
+          "type": "string",
+          "const": "UNSANDBOXED"
+        },
+        "tools": {
+          "minItems": 1,
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema14"
+          }
+        },
+        "dependencyGraphDigest": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "phaseReporterDigest": {
+          "$ref": "#/$defs/__schema2"
+        }
+      },
+      "required": [
+        "environmentId",
+        "os",
+        "cpu",
+        "logicalCpuLimit",
+        "memoryLimitBytes",
+        "executionBoundary",
+        "tools",
+        "dependencyGraphDigest",
+        "phaseReporterDigest"
+      ],
+      "additionalProperties": false
+    },
+    "__schema14": {
+      "type": "object",
+      "properties": {
+        "role": {
+          "$ref": "#/$defs/__schema7"
+        },
+        "name": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 256
+        },
+        "version": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 256
+        },
+        "digest": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "configurationDigest": {
+          "$ref": "#/$defs/__schema2"
+        }
+      },
+      "required": ["role", "name", "version", "digest", "configurationDigest"],
+      "additionalProperties": false
+    },
+    "__schema15": {
+      "maxItems": 1000000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema16"
+      }
+    },
+    "__schema16": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "candidateId": {
+              "$ref": "#/$defs/__schema7"
+            },
+            "candidateDigest": {
+              "$ref": "#/$defs/__schema2"
+            },
+            "regime": {
+              "$ref": "#/$defs/__schema17"
+            },
+            "role": {
+              "$ref": "#/$defs/__schema18"
+            },
+            "ordinal": {
+              "$ref": "#/$defs/__schema19"
+            },
+            "status": {
+              "type": "string",
+              "const": "COMPLETE"
+            },
+            "outcome": {
+              "type": "string",
+              "const": "PASS"
+            },
+            "phases": {
+              "minItems": 4,
+              "maxItems": 4,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema20"
+              }
+            },
+            "totalUs": {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 9007199254740991
+            },
+            "cpuTimeUs": {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 9007199254740991
+            }
+          },
+          "required": [
+            "candidateId",
+            "candidateDigest",
+            "regime",
+            "role",
+            "ordinal",
+            "status",
+            "outcome",
+            "phases",
+            "totalUs"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "candidateId": {
+              "$ref": "#/$defs/__schema7"
+            },
+            "candidateDigest": {
+              "$ref": "#/$defs/__schema2"
+            },
+            "regime": {
+              "$ref": "#/$defs/__schema17"
+            },
+            "role": {
+              "$ref": "#/$defs/__schema18"
+            },
+            "ordinal": {
+              "$ref": "#/$defs/__schema19"
+            },
+            "status": {
+              "type": "string",
+              "const": "INCOMPLETE"
+            },
+            "outcome": {
+              "type": "string",
+              "enum": [
+                "ASSERTION_FAILURE",
+                "COLLECTION_FAILURE",
+                "COMPILE_FAILURE",
+                "PROCESS_CRASH",
+                "TIMEOUT",
+                "INFRA_ERROR",
+                "NO_TEST_DISCOVERED"
+              ]
+            },
+            "phases": {
+              "type": "null"
+            },
+            "totalUs": {
+              "type": "null"
+            },
+            "cpuTimeUs": {
+              "type": "null"
+            }
+          },
+          "required": [
+            "candidateId",
+            "candidateDigest",
+            "regime",
+            "role",
+            "ordinal",
+            "status",
+            "outcome",
+            "phases",
+            "totalUs",
+            "cpuTimeUs"
+          ],
+          "additionalProperties": false
+        }
+      ]
+    },
+    "__schema17": {
+      "type": "string",
+      "enum": ["COLD", "WARM"]
+    },
+    "__schema18": {
+      "type": "string",
+      "enum": ["WARMUP", "MEASUREMENT"]
+    },
+    "__schema19": {
+      "type": "integer",
+      "minimum": 1,
+      "maximum": 1000000
+    },
+    "__schema20": {
+      "type": "object",
+      "properties": {
+        "phase": {
+          "type": "string",
+          "enum": ["PREPARATION", "STARTUP", "COMPILE_OR_COLLECTION", "EXECUTION"]
+        },
+        "durationUs": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        }
+      },
+      "required": ["phase", "durationUs"],
+      "additionalProperties": false
+    }
+  },
+  "$id": "https://testforge.dev/schemas/agentic-benchmark-request.v1.json"
+}