@fitlab-ai/agent-infra 0.9.1 → 0.9.3

package/lib/benchmark/schemas/v1.0.0/case-manifest.schema.json

@@ -0,0 +1,120 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/fitlab-ai/agent-infra/lib/benchmark/schemas/v1.0.0/case-manifest.schema.json",
+  "title": "agent-infra benchmark case manifest",
+  "description": "A trusted-side Case description with an explicit Agent-visible projection.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "contractVersion",
+    "dataset",
+    "case",
+    "sourceRevision",
+    "sourceDigest",
+    "seed",
+    "agentVisible",
+    "graderOnly",
+    "qualification"
+  ],
+  "properties": {
+    "contractVersion": {
+      "$ref": "common.schema.json#/definitions/contractVersion"
+    },
+    "dataset": {
+      "$ref": "common.schema.json#/definitions/datasetIdentity"
+    },
+    "case": {
+      "$ref": "common.schema.json#/definitions/caseIdentity"
+    },
+    "sourceRevision": {
+      "type": "string",
+      "minLength": 1
+    },
+    "sourceDigest": {
+      "$ref": "common.schema.json#/definitions/digest"
+    },
+    "seed": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "agentVisible": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "task"
+      ],
+      "properties": {
+        "task": {
+          "type": "string",
+          "minLength": 1
+        },
+        "artifacts": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": [
+              "path",
+              "digest"
+            ],
+            "properties": {
+              "path": {
+                "type": "string",
+                "minLength": 1
+              },
+              "digest": {
+                "$ref": "common.schema.json#/definitions/digest"
+              }
+            }
+          }
+        },
+        "parameters": {
+          "type": "object",
+          "additionalProperties": true
+        }
+      }
+    },
+    "graderOnly": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "graderRef",
+        "inputDigest"
+      ],
+      "properties": {
+        "graderRef": {
+          "type": "string",
+          "minLength": 1
+        },
+        "inputDigest": {
+          "$ref": "common.schema.json#/definitions/digest"
+        }
+      }
+    },
+    "challengeDigest": {
+      "$ref": "common.schema.json#/definitions/digest"
+    },
+    "qualification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "status"
+      ],
+      "properties": {
+        "status": {
+          "enum": [
+            "qualified",
+            "rejected",
+            "pending"
+          ]
+        },
+        "evidenceSummary": {
+          "type": "string"
+        }
+      }
+    },
+    "extensions": {
+      "$ref": "common.schema.json#/definitions/extensions"
+    }
+  }
+}

package/lib/benchmark/schemas/v1.0.0/common.schema.json

@@ -0,0 +1,109 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/fitlab-ai/agent-infra/lib/benchmark/schemas/v1.0.0/common.schema.json",
+  "title": "agent-infra benchmark common definitions",
+  "description": "Shared definitions for the agent-infra Benchmark Contract v1.0.0.",
+  "definitions": {
+    "contractVersion": {
+      "const": "1.0.0"
+    },
+    "semanticVersion": {
+      "type": "string",
+      "pattern": "^v?[0-9]+\\.[0-9]+\\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\\+[0-9A-Za-z.-]+)?$"
+    },
+    "digest": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "algorithm",
+        "value"
+      ],
+      "properties": {
+        "algorithm": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "value": {
+          "type": "string",
+          "pattern": "^[0-9a-fA-F]+$"
+        }
+      }
+    },
+    "datasetIdentity": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "datasetId",
+        "datasetVersion",
+        "datasetDigest"
+      ],
+      "properties": {
+        "datasetId": {
+          "type": "string",
+          "minLength": 1
+        },
+        "datasetVersion": {
+          "type": "string",
+          "minLength": 1
+        },
+        "datasetDigest": {
+          "$ref": "#/definitions/digest"
+        }
+      }
+    },
+    "caseIdentity": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "caseId",
+        "caseVersion"
+      ],
+      "properties": {
+        "caseId": {
+          "type": "string",
+          "minLength": 1
+        },
+        "caseVersion": {
+          "type": "string",
+          "minLength": 1
+        }
+      }
+    },
+    "checkSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "checkId",
+        "status"
+      ],
+      "properties": {
+        "checkId": {
+          "type": "string",
+          "minLength": 1
+        },
+        "status": {
+          "enum": [
+            "passed",
+            "failed",
+            "blocked"
+          ]
+        },
+        "category": {
+          "type": "string",
+          "minLength": 1
+        },
+        "summary": {
+          "type": "string"
+        },
+        "durationMs": {
+          "type": "integer",
+          "minimum": 0
+        }
+      }
+    },
+    "extensions": {
+      "type": "object",
+      "additionalProperties": true
+    }
+  }
+}

package/lib/benchmark/schemas/v1.0.0/grader-result.schema.json

@@ -0,0 +1,160 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/fitlab-ai/agent-infra/lib/benchmark/schemas/v1.0.0/grader-result.schema.json",
+  "title": "agent-infra benchmark grader result",
+  "description": "The sanitized public projection of a trusted Grader result.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "contractVersion",
+    "runId",
+    "dataset",
+    "case",
+    "status",
+    "checks",
+    "durationMs",
+    "sanitization"
+  ],
+  "properties": {
+    "contractVersion": {
+      "$ref": "common.schema.json#/definitions/contractVersion"
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1
+    },
+    "dataset": {
+      "$ref": "common.schema.json#/definitions/datasetIdentity"
+    },
+    "case": {
+      "$ref": "common.schema.json#/definitions/caseIdentity"
+    },
+    "status": {
+      "enum": [
+        "passed",
+        "failed",
+        "blocked"
+      ]
+    },
+    "failureCategory": {
+      "type": "string",
+      "minLength": 1
+    },
+    "blockCategory": {
+      "type": "string",
+      "minLength": 1
+    },
+    "checks": {
+      "type": "array",
+      "items": {
+        "$ref": "common.schema.json#/definitions/checkSummary"
+      }
+    },
+    "durationMs": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "testCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "passedTestCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "extensions": {
+          "$ref": "common.schema.json#/definitions/extensions"
+        }
+      }
+    },
+    "sanitization": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "allowlistVersion",
+        "sanitized"
+      ],
+      "properties": {
+        "allowlistVersion": {
+          "type": "string",
+          "minLength": 1
+        },
+        "sanitized": {
+          "const": true
+        }
+      }
+    },
+    "extensions": {
+      "$ref": "common.schema.json#/definitions/extensions"
+    }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "status": {
+            "const": "passed"
+          }
+        }
+      },
+      "then": {
+        "not": {
+          "anyOf": [
+            {
+              "required": [
+                "failureCategory"
+              ]
+            },
+            {
+              "required": [
+                "blockCategory"
+              ]
+            }
+          ]
+        }
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "status": {
+            "const": "failed"
+          }
+        }
+      },
+      "then": {
+        "required": [
+          "failureCategory"
+        ],
+        "not": {
+          "required": [
+            "blockCategory"
+          ]
+        }
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "status": {
+            "const": "blocked"
+          }
+        }
+      },
+      "then": {
+        "required": [
+          "blockCategory"
+        ],
+        "not": {
+          "required": [
+            "failureCategory"
+          ]
+        }
+      }
+    }
+  ]
+}

package/lib/benchmark/schemas/v1.0.0/run-manifest.schema.json

@@ -0,0 +1,122 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://github.com/fitlab-ai/agent-infra/lib/benchmark/schemas/v1.0.0/run-manifest.schema.json",
+  "title": "agent-infra benchmark run manifest",
+  "description": "Reproducibility metadata for one Subject execution of one Case.",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "contractVersion",
+    "runId",
+    "frameworkVersion",
+    "dataset",
+    "case",
+    "subject",
+    "seed",
+    "repetitionIndex",
+    "repetitionCount",
+    "comparisonGroupId",
+    "conditionsDigest",
+    "startedAt",
+    "finishedAt"
+  ],
+  "properties": {
+    "contractVersion": {
+      "$ref": "common.schema.json#/definitions/contractVersion"
+    },
+    "runId": {
+      "type": "string",
+      "minLength": 1
+    },
+    "frameworkVersion": {
+      "$ref": "common.schema.json#/definitions/semanticVersion"
+    },
+    "dataset": {
+      "$ref": "common.schema.json#/definitions/datasetIdentity"
+    },
+    "case": {
+      "$ref": "common.schema.json#/definitions/caseIdentity"
+    },
+    "subject": {
+      "$ref": "subject.schema.json"
+    },
+    "seed": {
+      "type": "integer",
+      "minimum": 0
+    },
+    "repetitionIndex": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "repetitionCount": {
+      "type": "integer",
+      "minimum": 1
+    },
+    "comparisonGroupId": {
+      "type": "string",
+      "minLength": 1
+    },
+    "conditionsDigest": {
+      "$ref": "common.schema.json#/definitions/digest"
+    },
+    "startedAt": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "finishedAt": {
+      "type": "string",
+      "format": "date-time"
+    },
+    "resultRef": {
+      "type": "string",
+      "minLength": 1
+    },
+    "graderResult": {
+      "$ref": "grader-result.schema.json"
+    },
+    "metrics": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "durationMs": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "turnCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "tokenCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "cost": {
+          "type": "number",
+          "minimum": 0
+        },
+        "costCurrency": {
+          "type": "string",
+          "minLength": 1
+        },
+        "extensions": {
+          "$ref": "common.schema.json#/definitions/extensions"
+        }
+      }
+    },
+    "extensions": {
+      "$ref": "common.schema.json#/definitions/extensions"
+    }
+  },
+  "oneOf": [
+    {
+      "required": [
+        "resultRef"
+      ]
+    },
+    {
+      "required": [
+        "graderResult"
+      ]
+    }
+  ]
+}