@vzn/vx-history-server 0.0.651 → 0.0.652

package/PROTOCOL.md

@@ -1,15 +1,16 @@
 # vx-history protocol, v1
 
-An HTTP API for the task timings a scheduler asks for. It is a history,
+An HTTP API for the task timings a scheduler asks for, and the flaky
+record a retry asks for. It is a history,
 not a log: a server keeps one row per task and ref, and folds each
 report into it. `@vzn/vx-history-server` speaks it on Bun, on Cloudflare
 Workers and as a Docker image; any server that does can take its place.
 
-| Route                     | What it does                      |
-| ------------------------- | --------------------------------- |
-| `POST /v1/timings/upload` | a run's timings, once at its end  |
-| `POST /v1/timings/query`  | each task's numbers, ready to use |
-| `GET /v1/capabilities`    | the server's limits               |
+| Route                     | What it does                                  |
+| ------------------------- | --------------------------------------------- |
+| `POST /v1/timings/upload` | a run's timings and outcomes, once at its end |
+| `POST /v1/timings/query`  | each task's numbers and the flaky ones        |
+| `GET /v1/capabilities`    | the server's limits                           |
 
 ## Repositories and tokens
 
@@ -54,23 +55,43 @@ each write costs, and the fold would move it under 1.5%.
 The first report sets the row; one without `cores` or `rssBytes` keeps
 the row's. A row not reported for `TIMING_DAYS` (30) is deleted.
 
+### Outcomes
+
+An upload may also carry `"outcomes": [{ "hash", "key", "outcome" }]`,
+up to 10 000, each task and key once: every cached task the run ran.
+`hash` is the task's definition, `key` its cache key (both up to 512
+characters), `outcome` one of:
+
+- `pass`, `fail`: how it ended.
+- `flaky`: it passed on a retry, so the run saw it flake.
+
+The server keeps, per repository (not per ref: a key is the same inputs
+on every branch), only the task and keys that failed, each with a flaky
+flag. A pass of a failed key, or a run's `flaky`, sets the flag; any
+other pass is stored nowhere. A task is **flaky** when one of its keys
+is. A key with nothing for `TIMING_DAYS` is deleted.
+
 ## Query
 
 One request a run: `POST /v1/timings/query` with
 `{ "issuer", "repository", "refs": ["refs/heads/feature", "refs/heads/main"], "hashes": [ … ] }`,
 1 to 8 refs of up to 512 characters each and up to 10 000 hashes: each hash's row in the first ref
 named that holds one. A hash none holds is absent. The numbers are the
-scheduler's as they are:
+scheduler's as they are, and `flaky` lists the hashes asked that are
+flaky, by any ref:
 
 ```json
-{ "timings": { "3b0d…": { "durationMs": 4200, "cores": 1.8, "rssBytes": 310000000, "runs": 12 } } }
+{
+  "timings": { "3b0d…": { "durationMs": 4200, "cores": 1.8, "rssBytes": 310000000, "runs": 12 } },
+  "flaky": ["3b0d…"]
+}
 ```
 
 ## Limits and errors
 
 `GET /v1/capabilities`: `{ "maxUploadTimings": 10000, "maxQueryHashes": 10000, "maxQueryRefs": 8 }`.
 A server takes both at their limits, each hash 512 characters: an
-upload body up to 7 120 000 bytes, a query's up to 6 010 000. Past
+upload body up to 17 960 100 bytes, a query's up to 6 010 000. Past
 that, `413`.
 
 Every refusal is `{ code, message }`, the message saying what to fix. A

package/README.md

@@ -5,7 +5,9 @@ Website: <https://vznjs.github.io/vx/> · source: <https://github.com/vznjs/vx>
 A server for the task timings a vx scheduler asks for, that you host. It
 keeps one row per task, already the numbers the scheduler uses (duration,
 CPU cores, peak memory), and merges each CI run's report into it: a
-history, not a log. It speaks the [vx-history protocol](./PROTOCOL.md)
+history, not a log. It also keeps the flaky record every CI machine
+shares: only the cache keys that failed, each with a flaky flag,
+so a run retries a task that flaked on another machine. It speaks the [vx-history protocol](./PROTOCOL.md)
 (`/v1`) on three backends that answer alike:
 
 - **Bun**, with a SQLite file.
@@ -83,7 +85,7 @@ The same server as a `fetch` handler, over storage of your own
   `oidcTokenAuth` from `@vzn/vx-devtools`.
 - `sqliteStore(db: Db)` is the `Store` over a SQLite `Db`
   (`{ all(sql, params), run(sql, params) }`), as bun:sqlite and D1 are.
-  A `Store` merges `Report`s and reads `Timing`s, each of a `Repo`
+  A `Store` merges `Report`s, keeps `Outcome`s and reads `Timing`s, each of a `Repo`
   (`{ issuer, repository }`) and a ref.
 - `WEIGHT` (0.3) and `RSS_DECAY` (0.9) are the merge's constants;
   `MAX_UPLOAD_TIMINGS`, `MAX_QUERY_HASHES` and `MAX_QUERY_REFS` its

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-history-server",
-  "version": "0.0.651",
+  "version": "0.0.652",
   "license": "MIT",
   "description": "The task timings a vx scheduler asks for, one folded row per task, that you host: on Bun with SQLite, on Cloudflare Workers with D1, or as a Docker image.",
   "keywords": [
@@ -45,7 +45,7 @@
     "miniflare: tests/worker.test.ts runs src/worker.ts in workerd with D1, as Workers deploy it."
   ],
   "dependencies": {
-    "@vzn/vx-devtools": "0.0.651"
+    "@vzn/vx-devtools": "0.0.652"
   },
   "repository": {
     "type": "git",

package/src/index.ts

@@ -10,6 +10,7 @@ export {
 } from './server.js'
 export {
   type Db,
+  type Outcome,
   type Report,
   type Repo,
   RSS_DECAY,

package/src/server.ts

@@ -2,7 +2,7 @@
 // `Store`: Bun and Workers serve the same one.
 
 import { type Access, errorJson, notAllowed, Refusal } from '@vzn/vx-devtools'
-import type { Report, Repo, Store } from './store.js'
+import type { Outcome, Report, Repo, Store } from './store.js'
 
 /** Reports one upload folds in: a run's tasks. */
 export const MAX_UPLOAD_TIMINGS = 10_000
@@ -13,6 +13,8 @@ export const MAX_QUERY_REFS = 8
 const MAX_STRING = 512
 /** A report's bytes at most: its hash at `MAX_STRING`, its numbers at their longest. */
 const MAX_REPORT_BYTES = MAX_STRING + 200
+/** An outcome's bytes at most: its hash and key at `MAX_STRING`. */
+const MAX_OUTCOME_BYTES = 2 * MAX_STRING + 60
 const DAY_MS = 86_400_000
 
 export interface ServerOptions {
@@ -47,9 +49,16 @@ export function createHistoryServer(opts: ServerOptions): (req: Request) => Prom
   async function upload(req: Request): Promise<Response> {
     const access = await authorize(req)
     if (access?.write === undefined) throw new Refusal(403, 'this request reads only')
-    const reports = parseReports(await readJson(req, MAX_UPLOAD_TIMINGS * MAX_REPORT_BYTES))
+    const body = await readJson(
+      req,
+      MAX_UPLOAD_TIMINGS * (MAX_REPORT_BYTES + MAX_OUTCOME_BYTES) + 100,
+    )
+    const reports = parseReports(body)
+    const outcomes = parseOutcomes(body)
     await schema()
-    await store.fold(access, access.write, now(), reports)
+    const t = now()
+    await store.fold(access, access.write, t, reports)
+    if (outcomes.length > 0) await store.record(access, t, outcomes)
     return new Response(null, { status: 204 })
   }
 
@@ -94,8 +103,11 @@ export function createHistoryServer(opts: ServerOptions): (req: Request) => Prom
     }
     const repo = (await authorize(req, { issuer, repository }))!
     await schema()
-    const rows = await store.read(repo, refs as string[], hashes as string[])
-    return Response.json({ timings: Object.fromEntries(rows) })
+    const [rows, flaky] = await Promise.all([
+      store.read(repo, refs as string[], hashes as string[]),
+      store.flaky(repo, hashes as string[]),
+    ])
+    return Response.json({ timings: Object.fromEntries(rows), flaky })
   }
 
   let swept = 0
@@ -218,6 +230,32 @@ function parseReports(body: unknown): Report[] {
   })
 }
 
+/** A run's outcomes of cached tasks (optional), as the flaky record keeps them. */
+function parseOutcomes(body: unknown): Outcome[] {
+  const list = field(body, 'outcomes')
+  if (list === undefined) return []
+  if (!Array.isArray(list) || list.length > MAX_UPLOAD_TIMINGS) {
+    throw new Refusal(400, `outcomes: a list of up to ${MAX_UPLOAD_TIMINGS}`)
+  }
+  const seen = new Set<string>()
+  return list.map((x: unknown, i): Outcome => {
+    const bad = (what: string): never => {
+      throw new Refusal(400, `outcomes[${i}].${what}`)
+    }
+    const hash = field(x, 'hash')
+    const key = field(x, 'key')
+    const outcome = field(x, 'outcome')
+    if (!isOpaque(hash)) bad(`hash: a string of up to ${MAX_STRING} characters`)
+    if (!isOpaque(key)) bad(`key: a string of up to ${MAX_STRING} characters`)
+    const id = `${hash as string}\0${key as string}`
+    if (seen.has(id)) bad('key: each task and key once')
+    seen.add(id)
+    if (outcome !== 'pass' && outcome !== 'fail' && outcome !== 'flaky')
+      bad("outcome: 'pass', 'fail' or 'flaky'")
+    return { hash: hash as string, key: key as string, outcome: outcome as Outcome['outcome'] }
+  })
+}
+
 /**
  * The body as text, refused past `max` bytes as they arrive: a chunked body
  * names no length, and was read whole before it was measured.

package/src/store.ts

@@ -21,6 +21,16 @@ export interface Report {
   readonly rssBytes?: number
 }
 
+/** One run of a cached task, pass or fail, as a client reports it: its flaky record. */
+export interface Outcome {
+  /** The task's definition hash, as its timing names it. */
+  readonly hash: string
+  /** The run's cache key: one key that failed, then passed, is a flake. */
+  readonly key: string
+  /** `flaky`: the run saw the task flake itself (a retry that passed). */
+  readonly outcome: 'pass' | 'fail' | 'flaky'
+}
+
 /** What a scheduler reads for a task: the folded numbers and how many runs made them. */
 export interface Timing {
   readonly durationMs: number
@@ -59,6 +69,12 @@ export interface Repo {
 const DROP_OLD = 'DROP TABLE IF EXISTS task_timings'
 const SCHEMA = `CREATE TABLE IF NOT EXISTS timings (issuer TEXT NOT NULL, repository TEXT NOT NULL, ref TEXT NOT NULL, hash TEXT NOT NULL, duration_ms INTEGER NOT NULL, cores REAL, rss_bytes INTEGER, runs INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (issuer, repository, ref, hash)) WITHOUT ROWID`
 
+// The flaky record (2026-11-06): a row only for a task definition and
+// cache key that failed, and whether it flaked since. A pass with no row
+// writes nothing: passes are most outcomes. Kept by repository, not by
+// ref: a key is the same inputs on every branch.
+const OUTCOMES = `CREATE TABLE IF NOT EXISTS outcomes (issuer TEXT NOT NULL, repository TEXT NOT NULL, hash TEXT NOT NULL, key TEXT NOT NULL, flaky INTEGER NOT NULL, at INTEGER NOT NULL, PRIMARY KEY (issuer, repository, hash, key)) WITHOUT ROWID`
+
 // When the database was last swept, by whichever node: a Worker runs many
 // isolates and starts new ones all day, and each swept on its first upload.
 const SWEEPS =
@@ -83,6 +99,23 @@ WHERE abs(excluded.duration_ms - duration_ms) > duration_ms * ${NOISE}
   OR (excluded.rss_bytes IS NOT NULL AND (rss_bytes IS NULL OR excluded.rss_bytes > rss_bytes))
   OR excluded.updated_at - updated_at >= ${DAY_MS}`
 
+// A failure or a run's flake writes its row; a row already as it would
+// be is rewritten only when a day old, since each write is billed.
+const RECORD = `INSERT INTO outcomes (issuer, repository, hash, key, flaky, at)
+SELECT ?1, ?2, j.value ->> '$.hash', j.value ->> '$.key', j.value ->> '$.outcome' = 'flaky', ?3
+FROM json_each(?4) AS j WHERE j.value ->> '$.outcome' <> 'pass'
+ON CONFLICT (issuer, repository, hash, key) DO UPDATE SET flaky = max(flaky, excluded.flaky), at = excluded.at
+WHERE excluded.flaky > flaky OR excluded.at - at >= ${DAY_MS}`
+
+/** A pass of a key that failed: the key flaked. */
+const PASSED = `UPDATE outcomes SET flaky = 1, at = ?3
+WHERE issuer = ?1 AND repository = ?2 AND flaky = 0 AND (hash, key) IN
+  (SELECT j.value ->> '$.hash', j.value ->> '$.key' FROM json_each(?4) AS j WHERE j.value ->> '$.outcome' = 'pass')`
+
+/** The hashes asked that flaked: a key that failed and then passed, or a run that saw it. */
+const FLAKY = `SELECT DISTINCT hash FROM outcomes
+WHERE issuer = ?1 AND repository = ?2 AND hash IN (SELECT value FROM json_each(?3)) AND flaky = 1`
+
 /** The rows of `refs` for the hashes asked; a hash the first ref lacks reads the next. */
 const READ = `SELECT hash, ref, duration_ms, cores, rss_bytes, runs FROM timings
 WHERE issuer = ?1 AND repository = ?2 AND ref IN (SELECT value FROM json_each(?3)) AND hash IN (SELECT value FROM json_each(?4))`
@@ -93,6 +126,10 @@ export interface Store {
   fold(repo: Repo, ref: string, now: number, reports: readonly Report[]): Promise<void>
   /** Each hash's row in the first of `refs` that holds one. */
   read(repo: Repo, refs: readonly string[], hashes: readonly string[]): Promise<Map<string, Timing>>
+  /** Records each outcome in the repository's flaky record. */
+  record(repo: Repo, now: number, outcomes: readonly Outcome[]): Promise<void>
+  /** The hashes asked that the flaky record names flaky. */
+  flaky(repo: Repo, hashes: readonly string[]): Promise<string[]>
   /** When a node last swept the database; 0 before the first sweep. */
   sweptAt(): Promise<number>
   /** Deletes the rows not reported since `before`, and records the sweep at `now`; how many. */
@@ -131,6 +168,21 @@ export function sqliteStore(db: Db): Store {
       await db.run(DROP_OLD, [])
       await db.run(SCHEMA, [])
       await db.run(SWEEPS, [])
+      await db.run(OUTCOMES, [])
+    },
+    async record(repo, now, outcomes) {
+      for (const part of jsonParams(outcomes)) {
+        await db.run(PASSED, [repo.issuer, repo.repository, now, part])
+        await db.run(RECORD, [repo.issuer, repo.repository, now, part])
+      }
+    },
+    async flaky(repo, hashes) {
+      const rows = await Promise.all(
+        jsonParams(hashes).map((part) =>
+          db.all<{ hash: string }>(FLAKY, [repo.issuer, repo.repository, part]),
+        ),
+      )
+      return rows.flat().map((r) => r.hash)
     },
     async fold(repo, ref, now, reports) {
       for (const part of jsonParams(reports))
@@ -168,6 +220,7 @@ export function sqliteStore(db: Db): Store {
       // A report that would not move its row is not written for up to a
       // day (FOLD): a row's write time lags its last report by up to that.
       const deleted = await db.run('DELETE FROM timings WHERE updated_at < ?1', [before - DAY_MS])
+      await db.run('DELETE FROM outcomes WHERE at < ?1', [before - DAY_MS])
       await db.run(
         'INSERT INTO sweeps (id, at) VALUES (1, ?1) ON CONFLICT (id) DO UPDATE SET at = excluded.at',
         [now],