@evolvingmachines/evolve 0.0.64 → 0.0.65

package/spec/openapi.yaml

@@ -483,11 +483,18 @@ paths:
       tags: [jobs]
       operationId: listJobs
       summary: List jobs
-      description: Newest first. Every filter is server-side.
+      description: >
+        Newest first. Every filter is server-side. A task quality check is a
+        job in Harbor (their `harbor check` runs its wrapper tasks as a single
+        Harbor job, analyze/checker.py:1-8, under the same `jobs/` directory
+        `harbor run` writes to), so `kind` lets one listing hold both:
+        `kind=all` merges jobs and checks newest first under the one cursor,
+        each row told apart by `JobListItem.kind`.
       parameters:
         - $ref: '#/components/parameters/Limit'
         - $ref: '#/components/parameters/Cursor'
         - $ref: '#/components/parameters/ListScope'
+        - $ref: '#/components/parameters/JobListKind'
         - name: status
           in: query
           description: Filter by job status (repeatable).
@@ -1809,9 +1816,10 @@ paths:
         stay stored. The body is optional — `{}` or absent means the
         defaults (openrouter/deepseek/deepseek-v4.1-flash at its per-model effort,
         the platform's default analyze rubric). CANCELLED trials are never analyzed. Which trials
-        and how wide are Harbor's own options on the body — `passing` /
-        `failing` (the reward filter), `n_trials` (the cap, after the
-        filter), `n_concurrent` (the width, beneath the organization's
+        and how wide are Harbor's own options on the body — `trial_ids` (the
+        named trials, before the filter and the cap), `passing` / `failing`
+        (the reward filter), `n_trials` (the cap, after the filter),
+        `n_concurrent` (the width, beneath the organization's
         `max_concurrent_analyses`).
 
         An analyzer run that completes without a valid `analysis.json` is
@@ -1830,9 +1838,12 @@ paths:
         the same way under `invalid_rubric`, PostgreSQL's jsonb ceiling
         named — an off-lineup `sandbox_provider`, an off-vocabulary
         `reasoning_effort`, an out-of-range `n_concurrent`, `passing` together
-        with `failing`, an invalid `n_trials` — in that order) → 409
+        with `failing`, an invalid `n_trials`, a malformed `trial_ids` — in
+        that order) → 400 `invalid_input` for a `trial_ids` entry that is not
+        a trial of this job (`details.unknown_trial_ids`) → 409
         `analysis_already_running` → 409 `no_analyzable_trials` (no analyzable
-        trial at all, or none on the side of the reward filter) → 202.
+        trial at all, none on the side of the reward filter, or none of the
+        named trials) → 202.
       parameters:
         - $ref: '#/components/parameters/JobId'
       requestBody:
@@ -1858,9 +1869,10 @@ paths:
           description: >
             `job_not_terminal`, `analysis_already_running` (one wave at a
             time; retry after the running wave settles), or
-            `no_analyzable_trials` (every trial CANCELLED, or no trial on
-            the side of the `passing` / `failing` filter — Harbor's "No
-            passing trial directories found", analyzer.py:192-200).
+            `no_analyzable_trials` (every trial CANCELLED, no trial on the
+            side of the `passing` / `failing` filter, or none of the named
+            `trial_ids` — Harbor's "No passing trial directories found",
+            analyzer.py:192-200).
           content:
             application/json:
               schema:
@@ -4974,6 +4986,44 @@ paths:
         '500':
           $ref: '#/components/responses/InternalError'
 
+  /api/analyses/defaults:
+    get:
+      tags: [analyses]
+      operationId: getAnalyzeDefaults
+      summary: The defaults an analysis runs under when its config names nothing
+      description: |
+        The resolved policy an empty `AnalyzeConfigInput` falls back to — the
+        five knobs every analysis that names none is stamped with:
+        `model_name`, `rubric` (the platform's analyze rubric, the seven
+        criteria `AnalyzeConfigInput` lists), `prompt` (the built-in analyze
+        body, as the TEMPLATE a caller's own `prompt` replaces — with its
+        `{trial_path}`, `{task_section}` and `{criteria_guidance}` slots
+        unrendered; the output section Harbor appends is not part of it),
+        `reasoning_effort` (the default model's per-model effort) and
+        `sandbox_provider` (the fleet default). Read them to see what a
+        bare `evolve analyze` runs under, or to start an edited rubric or
+        prompt from the platform's own. `model_name` and `rubric` sent
+        back resolve as if nothing was sent; `prompt`, `reasoning_effort`
+        and `sandbox_provider` sent back are stored as named, so the
+        analysis echoes the prompt text and keeps that effort and provider
+        even when the fleet default later changes. Harbor has no such verb:
+        its defaults are files on disk (analyze/prompts/analyze.txt and its
+        default analyze rubric), which a hosted caller cannot open — the
+        hosted reading of the same files.
+      responses:
+        '200':
+          description: The resolved defaults.
+          content:
+            application/json:
+              schema:
+                $ref: '#/components/schemas/AnalyzeDefaults'
+        '401':
+          $ref: '#/components/responses/Unauthorized'
+        '429':
+          $ref: '#/components/responses/TooManyRequests'
+        '500':
+          $ref: '#/components/responses/InternalError'
+
   /api/analyses/{analysisId}/filesystem:
     get:
       tags: [analyses]
@@ -10508,11 +10558,49 @@ components:
         address, and `org` the organization's — and its `liveCount` is
         informational under both: stopping stays creator-or-admin, so nothing
         it counts that is not yours is yours to
-        stop.
+        stop. That feed also takes `org` (slug or id) exactly as
+        `GET /api/jobs?org=` does — `GET /api/traces?org=` lists ONE
+        organization's runs of every kind; membership is required, a
+        stranger's org answers `org_not_found`, and `scope` with `org` is
+        refused (`invalid_input`), since each names a selection; so is
+        `admin=true` with `org`.
       schema:
         type: string
         enum: [my, shared, org]
         default: my
+    JobListKind:
+      name: kind
+      in: query
+      description: >
+        Which run kinds the page lists: `job` — jobs (regrades included),
+        the default and exactly what an absent parameter always listed;
+        `check` — task quality checks only; `all` — jobs and checks merged
+        newest first under the one cursor, each row a `JobListItem` told
+        apart by its `kind`. In Harbor a check IS a job: `harbor check` runs
+        its wrapper tasks as a single Harbor job (analyze/checker.py:1-8),
+        takes `--job-name` (cli/analyze.py:138-140) and writes under the
+        same `jobs/` directory `harbor run` uses (`--jobs-dir`, default
+        `jobs`, cli/analyze.py:141-143 and cli/jobs.py:442-450; the job dir
+        is `jobs_dir / job_name`, job.py:634-635). Recorded deviation:
+        Harbor lists runs and checks as the folders of that one directory
+        with no filter; the hosted list needs one so `job list` scripts keep
+        their meaning, so `kind` exists here and defaults to `job`. The same
+        `scope` words and `org` apply to both kinds, and a check is listed
+        under the rule that lets `GET /api/checks/{checkId}` read it. Every
+        other filter narrows a check by the same words: `status` by the
+        check's own ladder (`QUEUED` and `RUNNING` match `queued` and
+        `running`; `COMPLETED` matches a `completed` check none of whose task
+        checks was stopped; `CANCELLED` matches a `completed` check with a
+        stopped task check, the row the dashboard reads Stopped; `CANCELLING`
+        and `FAILED` match no check), `model` and `sandbox_provider` by the
+        checker's, `dataset` by the check's dataset source (every version of
+        that name), `search` by the check's name and dataset source; `agent`
+        names an arm, so it lists no check; `is_regrade=true` lists no check
+        and `is_regrade=false` keeps them (a check is not a regrade).
+      schema:
+        type: string
+        enum: [job, check, all]
+        default: job
     IdempotencyKey:
       name: Idempotency-Key
       in: header
@@ -12300,6 +12388,12 @@ components:
         (`-l/--n-trials`) — their cli/analyze.py:278-290. Every one is
         optional; all omitted is today's behaviour: every analyzable trial,
         as wide as the organization's `max_concurrent_analyses` allows.
+        `trial_ids` is Harbor's other way to choose: `harbor analyze <trial
+        directory>` — their PATH is "a trial directory or a job directory of
+        trials" (cli/analyze.py:242-245), and one trial directory is analyzed
+        as itself (analyzer.py:174-181 `_resolve_trial_dirs`) — hosted, the
+        trials of this job named by id. Only `POST /api/jobs/{jobId}/analyze`
+        takes it; on `JobCreate.analyze` the trials do not exist yet.
 
         Harbor's other analyze options, and why they are not here:
         `-a/--agent` — the analyzer harness is claude-code, fixed (the
@@ -12479,6 +12573,29 @@ components:
             spent as trials finish, so settle order decides. Not an
             integer of at least 1 is refused `invalid_input` naming
             `analyze.n_trials`.
+        trial_ids:
+          type: array
+          minItems: 1
+          uniqueItems: true
+          items:
+            type: string
+            minLength: 1
+          description: >
+            Analyze only these trials of the job — Harbor's `harbor analyze
+            <trial directory>` for one trial (their cli/analyze.py:242-245;
+            analyzer.py:174-181), given as ids here; combinable with
+            `passing`/`failing` (a listed trial on the other side of the
+            filter is skipped) and applied before `n_trials` (Harbor's
+            one-trial path skips its filter and cap, analyzer.py:180-181
+            before :189-191; hosted, both apply to the named trials too).
+            An id that is not a trial of this job is refused `invalid_input`
+            naming `analyze.trial_ids`, with the unknown ids named in the
+            message and in `details.unknown_trial_ids`; a list that leaves
+            nothing analyzable is the existing 409 `no_analyzable_trials`. An
+            empty list, a duplicate, an empty string or a non-string is
+            refused `invalid_input` naming `analyze.trial_ids`. On
+            `JobCreate.analyze` (the embedded trigger) it is refused
+            `invalid_input` — trials do not exist yet.
 
     AnalyzeConfig:
       type: object
@@ -12491,7 +12608,7 @@ components:
         (`Trial.analysis.model_name` / `.rubric` / `.prompt` /
         `.reasoning_effort`), which a later manual re-analysis may have
         changed.
-      required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials]
+      required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider, n_concurrent, passing, failing, n_trials, trial_ids]
       properties:
         model_name:
           type: string
@@ -12544,6 +12661,15 @@ components:
           description: >
             The trial cap as stored (`AnalyzeConfigInput.n_trials`); null =
             no cap.
+        trial_ids:
+          type: [array, 'null']
+          items:
+            type: string
+          description: >
+            Always null on the job body today: the stored policy is the
+            create-time one (`JobCreate.analyze`), which refuses `trial_ids`;
+            the trials a manual wave names are the analysis rows it enqueues,
+            never a field of the job.
 
     # -------------------------------------------------------------------------
     # Job — result side
@@ -12829,6 +12955,7 @@ components:
         not others. The create verb answers with JobCreated, which is this
         shape plus one aliased id and nothing else.
       required:
+        - kind
         - id
         - job_name
         - status
@@ -12857,12 +12984,19 @@ components:
         - failure
         - source_jobs
         - is_regrade
+        - viewer
         - upload
         - idempotent_replay
         - started_at
         - updated_at
         - finished_at
       properties:
+        kind:
+          type: string
+          enum: [job]
+          description: >
+            The row's kind on the jobs list (`JobListItem`): always `job` on
+            a Job body, a regrade included; a check row says `check`.
         id:
           type: string
           format: uuid
@@ -13010,6 +13144,19 @@ components:
         is_regrade:
           type: boolean
           description: 'Derived: any source_jobs entry with action "regrade".'
+        viewer:
+          type: [string, 'null']
+          enum: [creator, member, shared, link, null]
+          description: >
+            The caller's relation to the job on a read: `creator` made it,
+            `member` belongs to its organization, `shared` reads it through
+            an email share, `link` through its unlisted link. Null on
+            responses that echo a job the caller just acted on (create,
+            analyze, cancel, resume, retry, regrade), where the
+            caller is the creator or a member by construction, and on list
+            rows (`GET /api/jobs`), which are batched; read one job to learn
+            the relation. Acting verbs are open to `creator` and `member`
+            only.
         upload:
           oneOf:
             - $ref: '#/components/schemas/UploadProvenance'
@@ -13786,15 +13933,134 @@ components:
               format: uuid
               description: Alias of `id`, identical value. Create responses only.
 
+    JobListItem:
+      description: >
+        One row of the jobs list, told apart by `kind`: a `Job` (`kind:
+        job`), or a `CheckRow` (`kind: check`) when the listing asked for
+        `kind=check` or `kind=all`. Under the default `kind=job` every row
+        is a Job.
+      oneOf:
+        - $ref: '#/components/schemas/Job'
+        - $ref: '#/components/schemas/CheckRow'
+      discriminator:
+        propertyName: kind
+        mapping:
+          job: '#/components/schemas/Job'
+          check: '#/components/schemas/CheckRow'
+
+    CheckRow:
+      type: object
+      description: >
+        A task quality check as one row of the jobs list (`GET /api/jobs`
+        with `kind=check` or `kind=all`): the Check body's own facts without
+        its per-task results, rubric and prompt (`GET /api/checks/{checkId}`
+        serves those), plus the tally the list needs. In Harbor a check IS a
+        job — its wrapper tasks run as one Harbor job under
+        `jobs/<job-name>` (analyze/checker.py:1-8) — so it lists beside
+        jobs here. The row is not a Job: a check has no arms, attempts,
+        caps, retry policy, trials or upload provenance, so those fields are
+        absent rather than faked. Each field says what the Jobs page reads
+        it as.
+      required:
+        - kind
+        - id
+        - name
+        - status
+        - source
+        - model_name
+        - reasoning_effort
+        - sandbox_provider
+        - org
+        - visibility
+        - tasks
+        - cost_usd
+        - created_at
+        - finished_at
+      properties:
+        kind:
+          type: string
+          enum: [check]
+          description: Tells the row from a Job on `JobListItem`.
+        id:
+          type: string
+          format: uuid
+          description: The check's id — the row opens at `GET /api/checks/{checkId}`.
+        name:
+          type: string
+          description: Check.name, Harbor's `--job-name` (the caller's, or the accept stamp); the row's JOB cell.
+        status:
+          type: string
+          enum: [queued, running, completed]
+          description: Check.status, derived from its tasks; the row's STATUS cell. The list's `status` words map onto it as the `kind` parameter says (a `completed` check with a stopped task check answers `CANCELLED`, not `COMPLETED`).
+        source:
+          $ref: '#/components/schemas/CheckSource'
+          description: What was checked; the row's DATASETS cell (the dataset form's `name@version`; an archive shows as an archive with its sha256).
+        model_name:
+          type: string
+          description: The checker's model; the row's AGENTS cell, shown like an arm with its effort under it.
+        reasoning_effort:
+          type: string
+          description: The effort every task's checker ran at, as Check.reasoning_effort.
+        sandbox_provider:
+          allOf:
+            - $ref: '#/components/schemas/SandboxProvider'
+          description: Where the checker ran, as Check.sandbox_provider.
+        org:
+          type: string
+          description: The owning organization's slug (every check has one), like Job.org.
+        visibility:
+          $ref: '#/components/schemas/JobVisibility'
+          description: As Check.visibility — `PRIVATE`, or `LINK` while the check's unlisted link is on.
+        tasks:
+          $ref: '#/components/schemas/CheckTaskTally'
+          description: How many task checks, and how they break down; the row's TRIALS cell reads done (completed, failed and stopped) of total.
+        cost_usd:
+          type: [number, 'null']
+          description: Check.cost_usd, the sum of the measured task costs (null when none was measured); the row's COST cell.
+        created_at:
+          type: string
+          format: date-time
+          description: The accept instant; the list orders it with the jobs' `started_at` (newest first, `id` as the tiebreak); the row's CREATED cell.
+        finished_at:
+          type: [string, 'null']
+          format: date-time
+          description: When the last task settled; null until every task has.
+
+    CheckTaskTally:
+      type: object
+      description: >
+        The "how many" shape of a check's task checks — a total plus a
+        histogram over the task check statuses, zeros included
+        (TrialStatusTally's twin on the check's own ladder). A task check
+        the user stopped is stored `failed` with the stop phase
+        (`failure.phase` `stopped`, or `job_cancelled`); the tally counts
+        it under `stopped`, never `failed`, so a stop never reads as a
+        failure.
+      required: [total, byStatus]
+      properties:
+        total:
+          type: integer
+          minimum: 0
+        byStatus:
+          type: object
+          description: One integer per TaskCheck status plus `stopped`, zeros included.
+          required: [queued, running, completed, failed, stopped]
+          properties:
+            queued: { type: integer, minimum: 0 }
+            running: { type: integer, minimum: 0 }
+            completed: { type: integer, minimum: 0 }
+            failed: { type: integer, minimum: 0 }
+            stopped: { type: integer, minimum: 0 }
+
     JobPage:
       type: object
-      description: Cursor page of jobs (envelope keys frozen verbatim).
+      description: Cursor page of jobs (envelope keys frozen verbatim). Rows are `JobListItem`s — Jobs, and CheckRows when `kind` asked for them.
       required: [items, nextCursor, hasMore]
       properties:
         items:
           type: array
           items:
-            $ref: '#/components/schemas/Job'
+            $ref: '#/components/schemas/JobListItem'
         nextCursor:
           type: [string, 'null']
           description: Pass back as `?cursor=`; null means no next page.
@@ -15078,6 +15344,33 @@ components:
           format: date-time
           description: When the last task settled; null until every task has.
 
+    AnalyzeDefaults:
+      type: object
+      description: >
+        The policy an empty `AnalyzeConfigInput` resolves to (GET
+        /api/analyses/defaults) — each key the same value `AnalyzeConfig`
+        echoes for a job created with `analyze: {}`, except `prompt`, which
+        `AnalyzeConfig` serves as null (the built-in body) and this serves as
+        that body's template text.
+      required: [model_name, rubric, prompt, reasoning_effort, sandbox_provider]
+      properties:
+        model_name:
+          type: string
+        rubric:
+          $ref: '#/components/schemas/Rubric'
+        prompt:
+          type: string
+          description: >
+            The built-in analyze prompt template, unrendered — pass it as
+            `AnalyzeConfigInput.prompt` to run the default body explicitly,
+            or edit it from here.
+        reasoning_effort:
+          type: string
+          description: The effort the default model runs at when the config names none.
+        sandbox_provider:
+          allOf:
+            - $ref: '#/components/schemas/SandboxProvider'
+
     CheckDefaults:
       type: object
       description: >
@@ -18281,6 +18574,13 @@ components:
                   description: >
                     Most ids (trials and trace analyses) one stop request may
                     name (StopRequest mirrors it as maxItems).
+                max_run_ids:
+                  type: integer
+                  description: >
+                    Most ids the ids form of `POST /api/traces/stop` (the
+                    dashboard's stop verb over every live run kind, not a path
+                    of this contract) may name in one request; a client
+                    stopping more sends several.
             pagination:
               type: object
               properties: