@thinkai/tai-api-contract 2.175.0 → 2.176.0

package/openapi/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: ThinkAI API
-  version: 2.175.0
+  version: 2.176.0
   description: >
     Contract surface for the AI Driven SDLC backend used by ThinkAI.
     Workspace-scoped routes use `/workspaces/{workspaceId}/...`.
@@ -97,7 +97,7 @@ tags:
       Workspace prompt/model eval suites (`/evals/*`) — authored golden cases with
       deterministic expected-signal scoring, async runs (never scored on the request
       path), an Insights surface for regressions and token efficiency, and platform
-      starter packs (`/evals/packs`) that install as editable suites. Distinct
+      starter packs (`/evals/packs`) that install as editable suites; compare two succeeded runs and record promote/keep/reject decisions (`/compare`, `/decisions`). Distinct
       from AI Usage & Spend ingest and Agentic Foundation Token Efficiency (rubric I).
 
 paths:
@@ -12287,6 +12287,142 @@ paths:
               schema:
                 $ref: "#/components/schemas/ErrorMessageDto"
 
+  /workspaces/{workspaceId}/evals/suites/{suiteId}/compare:
+    get:
+      tags: [Evals]
+      summary: Compare two succeeded eval runs on a suite
+      description: >
+        Side-by-side compare of a baseline run vs a candidate run on the same suite.
+        Both runs must be `succeeded`. Aligns cases by `caseId` and reports suite- and
+        case-level score/token/cost deltas when present. Requires workspace membership.
+        Does not trigger scoring.
+      operationId: compareEvalRuns
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+        - name: baselineRunId
+          in: query
+          required: true
+          schema:
+            type: string
+            format: uuid
+        - name: candidateRunId
+          in: query
+          required: true
+          schema:
+            type: string
+            format: uuid
+      responses:
+        "200":
+          description: Compare result
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalCompareDto"
+        "400":
+          description: >
+            Runs not comparable (e.g. not succeeded, same id, or invalid query).
+            Codes: `eval_compare_runs_required`, `eval_compare_runs_not_succeeded`,
+            `eval_compare_same_run`.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          description: Workspace, suite, or run not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+
+  /workspaces/{workspaceId}/evals/suites/{suiteId}/decisions:
+    get:
+      tags: [Evals]
+      summary: List eval promote/keep/reject decisions for a suite
+      description: >
+        Returns the last 50 decisions for the suite (newest first). Empty →
+        `{ "decisions": [] }`. Requires workspace membership.
+      operationId: listEvalDecisions
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+      responses:
+        "200":
+          description: Decision history
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalDecisionListDto"
+              example:
+                decisions: []
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          description: Workspace or suite not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+    post:
+      tags: [Evals]
+      summary: Record a promote, keep, or reject decision
+      description: >
+        Persists an audit decision for a baseline vs candidate run pair. Requires
+        workspace editor or admin. Both runs must be `succeeded` on this suite.
+        `promote` also applies the candidate run's `suiteSnapshot.target` to the
+        live suite (version may bump). Promotion gate attach is deferred
+        (`gateApplied: false`, `gateDetail.reason: deferred`) until
+        thinkai-platform-workspace#492.
+      operationId: createEvalDecision
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/CreateEvalDecisionBodyDto"
+      responses:
+        "201":
+          description: Decision recorded
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalDecisionDto"
+        "400":
+          description: >
+            Validation failed. Codes: `eval_decision_invalid`,
+            `eval_compare_runs_not_succeeded`, `eval_compare_same_run`,
+            `eval_promote_snapshot_missing`.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          description: Authenticated but not a workspace editor/admin
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+              example:
+                error: Workspace editor role required
+                code: workspace_editor_required
+        "404":
+          description: Workspace, suite, or run not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+
   /workspaces/{workspaceId}/evals/suites/{suiteId}/runs:
     get:
       tags: [Evals]
@@ -26234,6 +26370,14 @@ components:
           maxItems: 20
           items:
             $ref: "#/components/schemas/EvalCaseDto"
+        latestDecision:
+          type: object
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalDecisionDto"
+          description: >
+            Most recent promote/keep/reject decision for this suite, when any exist.
+            Omitted or null when none recorded.
 
     EvalSuiteListDto:
       type: object
@@ -26653,3 +26797,244 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/EvalPackSummaryDto"
+
+    EvalCompareRunSummaryDto:
+      type: object
+      additionalProperties: false
+      required: [id, suiteVersion, score, finishedAt, target, passCount, failCount, errorCount]
+      properties:
+        id:
+          type: string
+          format: uuid
+        suiteVersion:
+          type: integer
+          minimum: 1
+        score:
+          type: number
+          nullable: true
+          minimum: 0
+          maximum: 1
+        finishedAt:
+          type: string
+          format: date-time
+          nullable: true
+        target:
+          $ref: "#/components/schemas/EvalSuiteTargetDto"
+        passCount:
+          type: integer
+          minimum: 0
+        failCount:
+          type: integer
+          minimum: 0
+        errorCount:
+          type: integer
+          minimum: 0
+
+    EvalCompareCaseDto:
+      type: object
+      additionalProperties: false
+      required:
+        - caseId
+        - name
+        - baselineStatus
+        - candidateStatus
+        - baselinePass
+        - candidatePass
+        - baselineFailedSignalIds
+        - candidateFailedSignalIds
+      properties:
+        caseId:
+          type: string
+        name:
+          type: string
+        baselineStatus:
+          type: string
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalCaseResultStatusDto"
+        candidateStatus:
+          type: string
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalCaseResultStatusDto"
+        baselinePass:
+          type: boolean
+          nullable: true
+        candidatePass:
+          type: boolean
+          nullable: true
+        baselineFailedSignalIds:
+          type: array
+          items:
+            type: string
+        candidateFailedSignalIds:
+          type: array
+          items:
+            type: string
+        baselineInputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        candidateInputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        baselineOutputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        candidateOutputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        baselineCostCents:
+          type: number
+          nullable: true
+          minimum: 0
+        candidateCostCents:
+          type: number
+          nullable: true
+          minimum: 0
+        scoreDelta:
+          type: number
+          nullable: true
+          description: Candidate pass (1/0) minus baseline pass when both scored.
+        inputTokensDelta:
+          type: integer
+          nullable: true
+        outputTokensDelta:
+          type: integer
+          nullable: true
+        costCentsDelta:
+          type: number
+          nullable: true
+
+    EvalCompareDto:
+      type: object
+      additionalProperties: false
+      required:
+        - suiteId
+        - baseline
+        - candidate
+        - scoreDelta
+        - cases
+        - unpairedCaseIds
+      properties:
+        suiteId:
+          type: string
+          format: uuid
+        baseline:
+          $ref: "#/components/schemas/EvalCompareRunSummaryDto"
+        candidate:
+          $ref: "#/components/schemas/EvalCompareRunSummaryDto"
+        scoreDelta:
+          type: number
+          nullable: true
+          description: candidate.score − baseline.score when both present.
+        avgInputTokensDelta:
+          type: number
+          nullable: true
+        avgOutputTokensDelta:
+          type: number
+          nullable: true
+        avgCostCentsDelta:
+          type: number
+          nullable: true
+        cases:
+          type: array
+          items:
+            $ref: "#/components/schemas/EvalCompareCaseDto"
+        unpairedCaseIds:
+          type: array
+          items:
+            type: string
+          description: Case ids present on only one run's results/snapshot.
+
+    EvalDecisionKindDto:
+      type: string
+      enum: [promote, keep, reject]
+
+    EvalGateDetailDto:
+      type: object
+      additionalProperties: false
+      required: [reason]
+      properties:
+        reason:
+          type: string
+          description: >
+            Machine-readable gate outcome. `deferred` until promotion-gate attach
+            (thinkai-platform-workspace#492) ships.
+
+    EvalDecisionDto:
+      type: object
+      additionalProperties: false
+      required:
+        - id
+        - suiteId
+        - decision
+        - baselineRunId
+        - candidateRunId
+        - decidedByUserId
+        - decidedAt
+        - gateApplied
+      properties:
+        id:
+          type: string
+          format: uuid
+        suiteId:
+          type: string
+          format: uuid
+        decision:
+          $ref: "#/components/schemas/EvalDecisionKindDto"
+        baselineRunId:
+          type: string
+          format: uuid
+        candidateRunId:
+          type: string
+          format: uuid
+        decidedByUserId:
+          type: string
+          format: uuid
+        decidedAt:
+          type: string
+          format: date-time
+        note:
+          type: string
+          maxLength: 2000
+          nullable: true
+        gateApplied:
+          type: boolean
+          description: True when a promotion gate attach point enforced this decision.
+        gateDetail:
+          type: object
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalGateDetailDto"
+
+    EvalDecisionListDto:
+      type: object
+      additionalProperties: false
+      required: [decisions]
+      properties:
+        decisions:
+          type: array
+          maxItems: 50
+          items:
+            $ref: "#/components/schemas/EvalDecisionDto"
+
+    CreateEvalDecisionBodyDto:
+      type: object
+      additionalProperties: false
+      required: [decision, baselineRunId, candidateRunId]
+      properties:
+        decision:
+          $ref: "#/components/schemas/EvalDecisionKindDto"
+        baselineRunId:
+          type: string
+          format: uuid
+        candidateRunId:
+          type: string
+          format: uuid
+        note:
+          type: string
+          maxLength: 2000

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thinkai/tai-api-contract",
-  "version": "2.175.0",
+  "version": "2.176.0",
   "private": false,
   "type": "module",
   "main": "dist/index.js",