@tangle-network/agent-eval 0.120.0 → 0.120.2

package/dist/openapi.json

@@ -1,992 +0,0 @@
-{
-  "openapi": "3.1.0",
-  "info": {
-    "title": "@tangle-network/agent-eval — wire protocol",
-    "version": "0.120.0",
-    "description": "HTTP and stdio RPC interface to agent-eval. The TypeScript runtime is the source of truth; this spec is the contract that cross-language clients (Python, Rust, Go) generate from.\n\nWire-protocol version: 1.0.0. Bumps on breaking changes to request/response schemas.",
-    "contact": {
-      "name": "Tangle Network",
-      "url": "https://github.com/tangle-network/agent-eval"
-    },
-    "license": {
-      "name": "MIT"
-    }
-  },
-  "servers": [
-    {
-      "url": "http://localhost:5005",
-      "description": "Local agent-eval serve"
-    }
-  ],
-  "components": {
-    "schemas": {
-      "JudgeRequest": {
-        "oneOf": [
-          {
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-              "rubricName",
-              "content"
-            ],
-            "properties": {
-              "rubricName": {
-                "type": "string",
-                "minLength": 1
-              },
-              "content": {
-                "type": "string",
-                "minLength": 1
-              },
-              "context": {
-                "type": "object",
-                "additionalProperties": true
-              },
-              "model": {
-                "type": "string"
-              }
-            }
-          },
-          {
-            "type": "object",
-            "additionalProperties": false,
-            "required": [
-              "rubric",
-              "content"
-            ],
-            "properties": {
-              "rubric": {
-                "$ref": "#/components/schemas/Rubric"
-              },
-              "content": {
-                "type": "string",
-                "minLength": 1
-              },
-              "context": {
-                "type": "object",
-                "additionalProperties": true
-              },
-              "model": {
-                "type": "string"
-              }
-            }
-          }
-        ],
-        "description": "Judge request. Provide exactly one of rubricName or rubric."
-      },
-      "Rubric": {
-        "type": "object",
-        "properties": {
-          "name": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Stable name like \"anti-slop\" — used by clients to invoke this rubric."
-          },
-          "description": {
-            "type": "string",
-            "minLength": 1,
-            "description": "What this rubric measures. Shown in /v1/rubrics listing."
-          },
-          "systemPrompt": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Instructs the judging LLM. Should explain the persona (e.g. \"senior engineer reviewing voice\"), what to score on, and what to return."
-          },
-          "dimensions": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/RubricDimension"
-            },
-            "minItems": 1,
-            "description": "Scoring axes. The composite score is a weighted sum of these."
-          },
-          "failureModes": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/FailureMode"
-            },
-            "default": [],
-            "description": "Patterns to detect; each detected mode appears in the result.failureModes list."
-          },
-          "wins": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/FailureMode"
-            },
-            "default": [],
-            "description": "Positive patterns; each detected one appears in the result.wins list."
-          }
-        },
-        "required": [
-          "name",
-          "description",
-          "systemPrompt",
-          "dimensions"
-        ],
-        "description": "Inline rubric definition. Mutually exclusive with `rubricName`."
-      },
-      "RubricDimension": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Short stable id like \"buyer_quality\" — used as the key in scoring output."
-          },
-          "description": {
-            "type": "string",
-            "minLength": 1,
-            "description": "One-line plain-English meaning. Read by humans reviewing low scores."
-          },
-          "weight": {
-            "type": "number",
-            "minimum": 0,
-            "default": 1,
-            "description": "Relative weight in the composite score. Default 1; 0 disables."
-          },
-          "min": {
-            "type": "number",
-            "default": 0,
-            "description": "Lower bound of valid score for this dimension."
-          },
-          "max": {
-            "type": "number",
-            "default": 1,
-            "description": "Upper bound of valid score for this dimension."
-          }
-        },
-        "required": [
-          "id",
-          "description"
-        ]
-      },
-      "FailureMode": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Short stable id like \"ai-cadence\" — used in detection lists."
-          },
-          "description": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Plain-English description of the failure pattern."
-          }
-        },
-        "required": [
-          "id",
-          "description"
-        ]
-      },
-      "JudgeResult": {
-        "type": "object",
-        "properties": {
-          "composite": {
-            "type": "number",
-            "minimum": 0,
-            "maximum": 1,
-            "description": "Weighted combination of dimension scores in 0..1. The single number to gate on."
-          },
-          "dimensions": {
-            "type": "object",
-            "additionalProperties": {
-              "type": "number"
-            },
-            "description": "Per-dimension score, keyed by RubricDimension.id."
-          },
-          "failureModes": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "default": [],
-            "description": "Failure-mode ids detected in the content (subset of rubric.failureModes ids)."
-          },
-          "wins": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "default": [],
-            "description": "Win ids detected in the content (subset of rubric.wins ids)."
-          },
-          "rationale": {
-            "type": "string",
-            "description": "Plain-English explanation of the score. Surfaced to the human reviewer."
-          },
-          "rubricVersion": {
-            "type": "string",
-            "description": "Stable hash of the rubric used. Scores are only comparable across runs when this matches."
-          },
-          "model": {
-            "type": "string",
-            "description": "Model that produced the judgement, for reproducibility."
-          },
-          "durationMs": {
-            "type": "integer",
-            "minimum": 0,
-            "description": "End-to-end wall time for this call."
-          }
-        },
-        "required": [
-          "composite",
-          "dimensions",
-          "rationale",
-          "rubricVersion",
-          "model",
-          "durationMs"
-        ]
-      },
-      "ListRubricsResponse": {
-        "type": "object",
-        "properties": {
-          "rubrics": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/RubricInfo"
-            }
-          }
-        },
-        "required": [
-          "rubrics"
-        ]
-      },
-      "RubricInfo": {
-        "type": "object",
-        "properties": {
-          "name": {
-            "type": "string",
-            "description": "Pass this to /v1/judge as `rubricName`."
-          },
-          "description": {
-            "type": "string",
-            "description": "What this rubric measures."
-          },
-          "dimensions": {
-            "type": "array",
-            "items": {
-              "type": "object",
-              "properties": {
-                "id": {
-                  "type": "string"
-                },
-                "description": {
-                  "type": "string"
-                },
-                "weight": {
-                  "type": "number"
-                }
-              },
-              "required": [
-                "id",
-                "description",
-                "weight"
-              ]
-            },
-            "description": "The scoring axes this rubric uses, with weights."
-          },
-          "failureModes": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "default": [],
-            "description": "Failure-mode ids this rubric detects."
-          },
-          "rubricVersion": {
-            "type": "string",
-            "description": "Stable hash — match this to compare scores across runs."
-          }
-        },
-        "required": [
-          "name",
-          "description",
-          "dimensions",
-          "rubricVersion"
-        ]
-      },
-      "VersionResponse": {
-        "type": "object",
-        "properties": {
-          "package": {
-            "type": "string",
-            "description": "Package name (always \"@tangle-network/agent-eval\")."
-          },
-          "version": {
-            "type": "string",
-            "description": "Semver of the running server. Match your client to this."
-          },
-          "wireVersion": {
-            "type": "string",
-            "description": "Wire-protocol semver. Bumps separately from package version when the schema changes."
-          },
-          "apiSurface": {
-            "type": "array",
-            "items": {
-              "type": "string"
-            },
-            "description": "List of supported method names."
-          }
-        },
-        "required": [
-          "package",
-          "version",
-          "wireVersion",
-          "apiSurface"
-        ]
-      },
-      "HealthResponse": {
-        "type": "object",
-        "properties": {
-          "status": {
-            "type": "string",
-            "enum": [
-              "ok"
-            ]
-          },
-          "uptimeSec": {
-            "type": "number"
-          }
-        },
-        "required": [
-          "status",
-          "uptimeSec"
-        ]
-      },
-      "ErrorResponse": {
-        "type": "object",
-        "properties": {
-          "error": {
-            "type": "object",
-            "properties": {
-              "code": {
-                "type": "string",
-                "description": "Machine-readable code: \"validation_error\", \"rubric_not_found\", \"judge_error\"."
-              },
-              "message": {
-                "type": "string",
-                "description": "Human-readable message."
-              },
-              "details": {
-                "description": "Optional structured detail."
-              }
-            },
-            "required": [
-              "code",
-              "message"
-            ],
-            "description": "Errors are always wrapped in this shape across all endpoints."
-          }
-        },
-        "required": [
-          "error"
-        ]
-      },
-      "TracesIngestRequest": {
-        "type": "object",
-        "properties": {
-          "events": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/TraceEvent"
-            },
-            "minItems": 1,
-            "maxItems": 10000,
-            "description": "Batch of events. Max 10k per call — bigger streams should be chunked."
-          }
-        },
-        "required": [
-          "events"
-        ]
-      },
-      "TraceEvent": {
-        "type": "object",
-        "properties": {
-          "eventId": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Stable id for the event. Use ULID or UUID."
-          },
-          "runId": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Run this event belongs to."
-          },
-          "spanId": {
-            "type": "string",
-            "description": "Span that emitted the event, if any."
-          },
-          "kind": {
-            "type": "string",
-            "enum": [
-              "log",
-              "error",
-              "budget_decrement",
-              "budget_breach",
-              "state_mutation",
-              "policy_violation",
-              "redaction_applied",
-              "custom"
-            ],
-            "description": "Coarse event category — matches the TraceSchema v1 EventKind enum."
-          },
-          "timestamp": {
-            "type": "integer",
-            "minimum": 0,
-            "description": "Unix millis. Must be monotonically non-decreasing within a span."
-          },
-          "payload": {
-            "type": "object",
-            "additionalProperties": {},
-            "description": "Free-form payload — the runtime owns the shape."
-          }
-        },
-        "required": [
-          "eventId",
-          "runId",
-          "kind",
-          "timestamp",
-          "payload"
-        ]
-      },
-      "TracesIngestResponse": {
-        "type": "object",
-        "properties": {
-          "accepted": {
-            "type": "integer",
-            "minimum": 0,
-            "description": "Number of events persisted."
-          },
-          "rejected": {
-            "type": "integer",
-            "minimum": 0,
-            "description": "Number of events the store refused — see `errors[]` for reasons."
-          },
-          "errors": {
-            "type": "array",
-            "items": {
-              "type": "object",
-              "properties": {
-                "eventId": {
-                  "type": "string",
-                  "description": "Event id this error applies to."
-                },
-                "message": {
-                  "type": "string",
-                  "description": "Why the event was rejected."
-                }
-              },
-              "required": [
-                "eventId",
-                "message"
-              ]
-            },
-            "default": []
-          }
-        },
-        "required": [
-          "accepted",
-          "rejected"
-        ]
-      },
-      "FeedbackTrajectory": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "minLength": 1,
-            "description": "Stable id; idempotency key for the trajectory."
-          },
-          "projectId": {
-            "type": "string"
-          },
-          "scenarioId": {
-            "type": "string"
-          },
-          "task": {
-            "type": "object",
-            "properties": {
-              "intent": {
-                "type": "string",
-                "minLength": 1
-              },
-              "context": {}
-            },
-            "required": [
-              "intent"
-            ]
-          },
-          "attempts": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/FeedbackAttempt"
-            },
-            "default": []
-          },
-          "labels": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/FeedbackLabel"
-            },
-            "default": []
-          },
-          "outcome": {
-            "type": "object",
-            "properties": {
-              "success": {
-                "type": "boolean"
-              },
-              "score": {
-                "type": "number"
-              },
-              "metrics": {
-                "type": "object",
-                "additionalProperties": {
-                  "type": "number"
-                }
-              },
-              "costUsd": {
-                "type": "number"
-              },
-              "detail": {
-                "type": "string"
-              },
-              "observedAt": {
-                "type": "string"
-              },
-              "metadata": {
-                "type": "object",
-                "additionalProperties": {}
-              }
-            }
-          },
-          "split": {
-            "type": "string",
-            "enum": [
-              "train",
-              "dev",
-              "test",
-              "holdout"
-            ]
-          },
-          "tags": {
-            "type": "object",
-            "additionalProperties": {
-              "type": "string"
-            }
-          },
-          "createdAt": {
-            "type": "string",
-            "description": "ISO-8601 UTC."
-          },
-          "updatedAt": {
-            "type": "string"
-          },
-          "metadata": {
-            "type": "object",
-            "additionalProperties": {}
-          }
-        },
-        "required": [
-          "id",
-          "task",
-          "createdAt"
-        ]
-      },
-      "FeedbackAttempt": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "minLength": 1
-          },
-          "stepIndex": {
-            "type": "integer",
-            "minimum": 0
-          },
-          "artifactType": {
-            "type": "string",
-            "enum": [
-              "text",
-              "code",
-              "plan",
-              "research",
-              "action",
-              "ui",
-              "decision",
-              "data",
-              "other"
-            ]
-          },
-          "artifact": {},
-          "options": {
-            "type": "array",
-            "items": {}
-          },
-          "proposedAction": {
-            "type": "object",
-            "properties": {
-              "type": {
-                "type": "string"
-              },
-              "risk": {
-                "type": "string",
-                "enum": [
-                  "low",
-                  "medium",
-                  "high"
-                ]
-              },
-              "costUsd": {
-                "type": "number"
-              },
-              "externalSideEffect": {
-                "type": "boolean"
-              },
-              "requiresApproval": {
-                "type": "boolean"
-              },
-              "metadata": {
-                "type": "object",
-                "additionalProperties": {}
-              }
-            },
-            "required": [
-              "type"
-            ]
-          },
-          "feedback": {
-            "type": "array",
-            "items": {
-              "$ref": "#/components/schemas/FeedbackLabel"
-            }
-          },
-          "createdAt": {
-            "type": "string"
-          },
-          "metadata": {
-            "type": "object",
-            "additionalProperties": {}
-          }
-        },
-        "required": [
-          "id",
-          "stepIndex",
-          "artifactType",
-          "createdAt"
-        ]
-      },
-      "FeedbackLabel": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string"
-          },
-          "source": {
-            "type": "string",
-            "enum": [
-              "user",
-              "judge",
-              "environment",
-              "metric",
-              "policy",
-              "system"
-            ]
-          },
-          "kind": {
-            "type": "string",
-            "enum": [
-              "approve",
-              "reject",
-              "select",
-              "edit",
-              "rank",
-              "rate",
-              "comment",
-              "metric_outcome",
-              "policy_block",
-              "revision_request"
-            ]
-          },
-          "value": {},
-          "reason": {
-            "type": "string"
-          },
-          "severity": {
-            "type": "string",
-            "enum": [
-              "info",
-              "warning",
-              "error",
-              "critical"
-            ]
-          },
-          "createdAt": {
-            "type": "string",
-            "description": "ISO-8601 UTC."
-          },
-          "metadata": {
-            "type": "object",
-            "additionalProperties": {}
-          }
-        },
-        "required": [
-          "source",
-          "kind",
-          "createdAt"
-        ]
-      },
-      "FeedbackIngestResponse": {
-        "type": "object",
-        "properties": {
-          "id": {
-            "type": "string",
-            "description": "Trajectory id that was persisted."
-          },
-          "persisted": {
-            "type": "boolean",
-            "description": "True when the trajectory was saved (idempotent on id)."
-          }
-        },
-        "required": [
-          "id",
-          "persisted"
-        ]
-      }
-    },
-    "parameters": {}
-  },
-  "paths": {
-    "/v1/judge": {
-      "post": {
-        "summary": "Score a piece of content against a rubric",
-        "description": "Runs the judging LLM with the named (or inline) rubric and returns dimension scores, detected failure modes, wins, and a composite score in 0..1.",
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/JudgeRequest"
-              }
-            }
-          }
-        },
-        "responses": {
-          "200": {
-            "description": "Successful judgement",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/JudgeResult"
-                }
-              }
-            }
-          },
-          "400": {
-            "description": "Validation error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "404": {
-            "description": "Rubric not found",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "500": {
-            "description": "Judge error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/v1/rubrics": {
-      "get": {
-        "summary": "List built-in rubrics",
-        "description": "Returns every rubric registered server-side, with their dimensions and stable rubricVersion hash.",
-        "responses": {
-          "200": {
-            "description": "Listing",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ListRubricsResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/v1/version": {
-      "get": {
-        "summary": "Server and wire-protocol version",
-        "description": "Match your client version to `version`; check `wireVersion` for compatibility.",
-        "responses": {
-          "200": {
-            "description": "Version info",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/VersionResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/healthz": {
-      "get": {
-        "summary": "Liveness check",
-        "responses": {
-          "200": {
-            "description": "OK",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/HealthResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/v1/traces/ingest": {
-      "post": {
-        "summary": "Ingest a batch of production TraceEvents",
-        "description": "Append a batch of TraceEvents to the configured TraceStore. Accepts application/json ({events:[...]}) or application/x-ndjson (one event per line). Returns counts of accepted + rejected events.",
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/TracesIngestRequest"
-              }
-            },
-            "application/x-ndjson": {
-              "schema": {
-                "$ref": "#/components/schemas/TracesIngestRequest"
-              }
-            }
-          }
-        },
-        "responses": {
-          "200": {
-            "description": "Ingestion summary",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/TracesIngestResponse"
-                }
-              }
-            }
-          },
-          "400": {
-            "description": "Validation error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "401": {
-            "description": "Unauthorized (when bearer auth is configured)",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "503": {
-            "description": "No trace store configured",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    },
-    "/v1/feedback": {
-      "post": {
-        "summary": "Ingest a FeedbackTrajectory from production",
-        "description": "Persist a single FeedbackTrajectory. Idempotent on trajectory.id — re-posting replaces the prior record. Used by production runtimes to forward user 👍/👎/edits into the eval substrate.",
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/FeedbackTrajectory"
-              }
-            }
-          }
-        },
-        "responses": {
-          "200": {
-            "description": "Persisted",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/FeedbackIngestResponse"
-                }
-              }
-            }
-          },
-          "400": {
-            "description": "Validation error",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "401": {
-            "description": "Unauthorized (when bearer auth is configured)",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          },
-          "503": {
-            "description": "No feedback store configured",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/ErrorResponse"
-                }
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "webhooks": {}
-}