@thinkai/tai-api-contract 2.174.0 → 2.176.0

package/openapi/openapi.yaml

@@ -1,7 +1,7 @@
 openapi: 3.0.3
 info:
   title: ThinkAI API
-  version: 2.173.0
+  version: 2.176.0
   description: >
     Contract surface for the AI Driven SDLC backend used by ThinkAI.
     Workspace-scoped routes use `/workspaces/{workspaceId}/...`.
@@ -96,7 +96,8 @@ tags:
     description: >
       Workspace prompt/model eval suites (`/evals/*`) — authored golden cases with
       deterministic expected-signal scoring, async runs (never scored on the request
-      path), and an Insights surface for regressions and token efficiency. Distinct
+      path), an Insights surface for regressions and token efficiency, and platform
+      starter packs (`/evals/packs`) that install as editable suites; compare two succeeded runs and record promote/keep/reject decisions (`/compare`, `/decisions`). Distinct
       from AI Usage & Spend ingest and Agentic Foundation Token Efficiency (rubric I).
 
 paths:
@@ -11998,6 +11999,95 @@ paths:
         "403":
           $ref: "#/components/responses/Forbidden"
 
+  /workspaces/{workspaceId}/evals/packs:
+    get:
+      tags: [Evals]
+      summary: List platform eval starter packs
+      description: >
+        Returns the platform starter-pack catalog for Prompt & model evals.
+        Packs are ThinkAI-authored templates (not workspace-owned and not a
+        public marketplace). Requires workspace membership. Empty catalog is
+        not expected in production builds; clients should still handle
+        `{ "packs": [] }`.
+      operationId: listEvalPacks
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+      responses:
+        "200":
+          description: Starter pack catalog
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalPackListDto"
+              example:
+                packs:
+                  - id: pr-review
+                    name: PR review comments
+                    description: Check review tone and clear blocking feedback
+                    workflow: pr-review
+                    caseCount: 3
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          description: Workspace not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+
+  /workspaces/{workspaceId}/evals/packs/{packId}/install:
+    post:
+      tags: [Evals]
+      summary: Install an eval starter pack as a suite
+      description: >
+        Creates a new editable eval suite from a platform starter pack
+        (`version: 1`). Sets `target.promptKey` to `pack:<packId>` for
+        traceability. Requires workspace editor or admin. Pack cases include
+        fixture completions so the first run can succeed without AI connectors.
+        Does not score on the request path.
+      operationId: installEvalPack
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalPackId"
+      responses:
+        "201":
+          description: Suite created from pack
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalSuiteDto"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          description: Viewer cannot install packs
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+              example:
+                error: Workspace editor role required
+                code: workspace_editor_required
+        "404":
+          description: Unknown pack id or workspace not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+              example:
+                error: Eval starter pack not found
+                code: eval_pack_not_found
+        "409":
+          description: Workspace already has the maximum number of suites (50)
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+              example:
+                error: Maximum of 50 eval suites per workspace
+                code: eval_suite_limit_reached
+
   /workspaces/{workspaceId}/evals/suites:
     get:
       tags: [Evals]
@@ -12197,6 +12287,142 @@ paths:
               schema:
                 $ref: "#/components/schemas/ErrorMessageDto"
 
+  /workspaces/{workspaceId}/evals/suites/{suiteId}/compare:
+    get:
+      tags: [Evals]
+      summary: Compare two succeeded eval runs on a suite
+      description: >
+        Side-by-side compare of a baseline run vs a candidate run on the same suite.
+        Both runs must be `succeeded`. Aligns cases by `caseId` and reports suite- and
+        case-level score/token/cost deltas when present. Requires workspace membership.
+        Does not trigger scoring.
+      operationId: compareEvalRuns
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+        - name: baselineRunId
+          in: query
+          required: true
+          schema:
+            type: string
+            format: uuid
+        - name: candidateRunId
+          in: query
+          required: true
+          schema:
+            type: string
+            format: uuid
+      responses:
+        "200":
+          description: Compare result
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalCompareDto"
+        "400":
+          description: >
+            Runs not comparable (e.g. not succeeded, same id, or invalid query).
+            Codes: `eval_compare_runs_required`, `eval_compare_runs_not_succeeded`,
+            `eval_compare_same_run`.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          description: Workspace, suite, or run not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+
+  /workspaces/{workspaceId}/evals/suites/{suiteId}/decisions:
+    get:
+      tags: [Evals]
+      summary: List eval promote/keep/reject decisions for a suite
+      description: >
+        Returns the last 50 decisions for the suite (newest first). Empty →
+        `{ "decisions": [] }`. Requires workspace membership.
+      operationId: listEvalDecisions
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+      responses:
+        "200":
+          description: Decision history
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalDecisionListDto"
+              example:
+                decisions: []
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          $ref: "#/components/responses/Forbidden"
+        "404":
+          description: Workspace or suite not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+    post:
+      tags: [Evals]
+      summary: Record a promote, keep, or reject decision
+      description: >
+        Persists an audit decision for a baseline vs candidate run pair. Requires
+        workspace editor or admin. Both runs must be `succeeded` on this suite.
+        `promote` also applies the candidate run's `suiteSnapshot.target` to the
+        live suite (version may bump). Promotion gate attach is deferred
+        (`gateApplied: false`, `gateDetail.reason: deferred`) until
+        thinkai-platform-workspace#492.
+      operationId: createEvalDecision
+      parameters:
+        - $ref: "#/components/parameters/WorkspaceId"
+        - $ref: "#/components/parameters/EvalSuiteId"
+      requestBody:
+        required: true
+        content:
+          application/json:
+            schema:
+              $ref: "#/components/schemas/CreateEvalDecisionBodyDto"
+      responses:
+        "201":
+          description: Decision recorded
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/EvalDecisionDto"
+        "400":
+          description: >
+            Validation failed. Codes: `eval_decision_invalid`,
+            `eval_compare_runs_not_succeeded`, `eval_compare_same_run`,
+            `eval_promote_snapshot_missing`.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+        "401":
+          $ref: "#/components/responses/Unauthorized"
+        "403":
+          description: Authenticated but not a workspace editor/admin
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+              example:
+                error: Workspace editor role required
+                code: workspace_editor_required
+        "404":
+          description: Workspace, suite, or run not found
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/ErrorMessageDto"
+
   /workspaces/{workspaceId}/evals/suites/{suiteId}/runs:
     get:
       tags: [Evals]
@@ -12365,6 +12591,17 @@ components:
         type: string
         format: uuid
       description: Eval suite id (`tai_workspace_eval_suites.id`).
+    EvalPackId:
+      name: packId
+      in: path
+      required: true
+      schema:
+        type: string
+        minLength: 1
+        maxLength: 64
+        pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$"
+      description: >
+        Platform starter pack id (kebab-case), e.g. `pr-review`. Not a UUID.
     EvalRunId:
       name: runId
       in: path
@@ -26133,6 +26370,14 @@ components:
           maxItems: 20
           items:
             $ref: "#/components/schemas/EvalCaseDto"
+        latestDecision:
+          type: object
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalDecisionDto"
+          description: >
+            Most recent promote/keep/reject decision for this suite, when any exist.
+            Omitted or null when none recorded.
 
     EvalSuiteListDto:
       type: object
@@ -26510,3 +26755,286 @@ components:
           type: array
           items:
             $ref: "#/components/schemas/EvalModelComparisonDto"
+
+    EvalPackSummaryDto:
+      type: object
+      additionalProperties: false
+      required: [id, name, description, workflow, caseCount]
+      description: >
+        Platform starter pack summary for the hub catalog. Install materializes a
+        full `EvalSuiteDto`; pack payloads are not returned on list.
+      properties:
+        id:
+          type: string
+          minLength: 1
+          maxLength: 64
+          pattern: "^[a-z0-9]+(?:-[a-z0-9]+)*$"
+          description: Stable pack id (kebab-case).
+        name:
+          type: string
+          minLength: 1
+          maxLength: 120
+        description:
+          type: string
+          maxLength: 2000
+        workflow:
+          type: string
+          minLength: 1
+          maxLength: 64
+          description: Workflow bucket for UI grouping (e.g. `pr-review`).
+        caseCount:
+          type: integer
+          minimum: 1
+          maximum: 20
+          description: Number of golden cases that will be created on install.
+
+    EvalPackListDto:
+      type: object
+      additionalProperties: false
+      required: [packs]
+      properties:
+        packs:
+          type: array
+          items:
+            $ref: "#/components/schemas/EvalPackSummaryDto"
+
+    EvalCompareRunSummaryDto:
+      type: object
+      additionalProperties: false
+      required: [id, suiteVersion, score, finishedAt, target, passCount, failCount, errorCount]
+      properties:
+        id:
+          type: string
+          format: uuid
+        suiteVersion:
+          type: integer
+          minimum: 1
+        score:
+          type: number
+          nullable: true
+          minimum: 0
+          maximum: 1
+        finishedAt:
+          type: string
+          format: date-time
+          nullable: true
+        target:
+          $ref: "#/components/schemas/EvalSuiteTargetDto"
+        passCount:
+          type: integer
+          minimum: 0
+        failCount:
+          type: integer
+          minimum: 0
+        errorCount:
+          type: integer
+          minimum: 0
+
+    EvalCompareCaseDto:
+      type: object
+      additionalProperties: false
+      required:
+        - caseId
+        - name
+        - baselineStatus
+        - candidateStatus
+        - baselinePass
+        - candidatePass
+        - baselineFailedSignalIds
+        - candidateFailedSignalIds
+      properties:
+        caseId:
+          type: string
+        name:
+          type: string
+        baselineStatus:
+          type: string
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalCaseResultStatusDto"
+        candidateStatus:
+          type: string
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalCaseResultStatusDto"
+        baselinePass:
+          type: boolean
+          nullable: true
+        candidatePass:
+          type: boolean
+          nullable: true
+        baselineFailedSignalIds:
+          type: array
+          items:
+            type: string
+        candidateFailedSignalIds:
+          type: array
+          items:
+            type: string
+        baselineInputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        candidateInputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        baselineOutputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        candidateOutputTokens:
+          type: integer
+          nullable: true
+          minimum: 0
+        baselineCostCents:
+          type: number
+          nullable: true
+          minimum: 0
+        candidateCostCents:
+          type: number
+          nullable: true
+          minimum: 0
+        scoreDelta:
+          type: number
+          nullable: true
+          description: Candidate pass (1/0) minus baseline pass when both scored.
+        inputTokensDelta:
+          type: integer
+          nullable: true
+        outputTokensDelta:
+          type: integer
+          nullable: true
+        costCentsDelta:
+          type: number
+          nullable: true
+
+    EvalCompareDto:
+      type: object
+      additionalProperties: false
+      required:
+        - suiteId
+        - baseline
+        - candidate
+        - scoreDelta
+        - cases
+        - unpairedCaseIds
+      properties:
+        suiteId:
+          type: string
+          format: uuid
+        baseline:
+          $ref: "#/components/schemas/EvalCompareRunSummaryDto"
+        candidate:
+          $ref: "#/components/schemas/EvalCompareRunSummaryDto"
+        scoreDelta:
+          type: number
+          nullable: true
+          description: candidate.score − baseline.score when both present.
+        avgInputTokensDelta:
+          type: number
+          nullable: true
+        avgOutputTokensDelta:
+          type: number
+          nullable: true
+        avgCostCentsDelta:
+          type: number
+          nullable: true
+        cases:
+          type: array
+          items:
+            $ref: "#/components/schemas/EvalCompareCaseDto"
+        unpairedCaseIds:
+          type: array
+          items:
+            type: string
+          description: Case ids present on only one run's results/snapshot.
+
+    EvalDecisionKindDto:
+      type: string
+      enum: [promote, keep, reject]
+
+    EvalGateDetailDto:
+      type: object
+      additionalProperties: false
+      required: [reason]
+      properties:
+        reason:
+          type: string
+          description: >
+            Machine-readable gate outcome. `deferred` until promotion-gate attach
+            (thinkai-platform-workspace#492) ships.
+
+    EvalDecisionDto:
+      type: object
+      additionalProperties: false
+      required:
+        - id
+        - suiteId
+        - decision
+        - baselineRunId
+        - candidateRunId
+        - decidedByUserId
+        - decidedAt
+        - gateApplied
+      properties:
+        id:
+          type: string
+          format: uuid
+        suiteId:
+          type: string
+          format: uuid
+        decision:
+          $ref: "#/components/schemas/EvalDecisionKindDto"
+        baselineRunId:
+          type: string
+          format: uuid
+        candidateRunId:
+          type: string
+          format: uuid
+        decidedByUserId:
+          type: string
+          format: uuid
+        decidedAt:
+          type: string
+          format: date-time
+        note:
+          type: string
+          maxLength: 2000
+          nullable: true
+        gateApplied:
+          type: boolean
+          description: True when a promotion gate attach point enforced this decision.
+        gateDetail:
+          type: object
+          nullable: true
+          allOf:
+            - $ref: "#/components/schemas/EvalGateDetailDto"
+
+    EvalDecisionListDto:
+      type: object
+      additionalProperties: false
+      required: [decisions]
+      properties:
+        decisions:
+          type: array
+          maxItems: 50
+          items:
+            $ref: "#/components/schemas/EvalDecisionDto"
+
+    CreateEvalDecisionBodyDto:
+      type: object
+      additionalProperties: false
+      required: [decision, baselineRunId, candidateRunId]
+      properties:
+        decision:
+          $ref: "#/components/schemas/EvalDecisionKindDto"
+        baselineRunId:
+          type: string
+          format: uuid
+        candidateRunId:
+          type: string
+          format: uuid
+        note:
+          type: string
+          maxLength: 2000

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thinkai/tai-api-contract",
-  "version": "2.174.0",
+  "version": "2.176.0",
   "private": false,
   "type": "module",
   "main": "dist/index.js",