@vzn/vx-history 0.0.650 → 0.0.652

package/README.md

@@ -36,7 +36,8 @@ A recorded p50 always wins over an assumption, and assumptions never feed the wo
 `url` (or `VX_HISTORY_URL`), its base URL without `/v1`, names a
 [`@vzn/vx-history-server`](https://github.com/vznjs/vx/tree/main/packages/vx-history-server),
 over `https://` (`http://` only to `127.0.0.1`, `::1` or `localhost`: every
-request carries the CI token).
+request carries the CI token), with no `?query` or `#fragment` (each
+route is appended to it).
 On a CI job with a token (GitHub's OIDC token, which needs
 `permissions: id-token: write` to set `ACTIONS_ID_TOKEN_REQUEST_URL`
 and `ACTIONS_ID_TOKEN_REQUEST_TOKEN`, or a GitLab `id_tokens` entry
@@ -47,6 +48,8 @@ for its tasks' numbers, by task definition, in the refs the run reads,
 as a remote cache does: its branch's, the branch it merges into, then
 the default branch's (`baseBranch`, else git's or the CI's). A run
 that reads no ref (a detached HEAD with no default branch) asks nothing.
+A ref over the 512 characters a server takes is said and not read;
+the run reads the others.
 The run starts on its local history and re-ranks the tasks not yet
 started when they land; it never waits for them. At its end it sends the
 timings of the tasks it ran, once: under DTE that is the orchestrator,
@@ -75,6 +78,15 @@ unset value is left out, and a name the request sets itself
 redirect is not followed (it would carry them on): it fails the request,
 naming where it pointed.
 
+The same upload carries the flaky record: each cached task the run
+ran, by definition and cache key, with how it ended (`pass`, `fail`,
+or `flaky` when it passed on a retry). The server names a task flaky
+when one key failed and later passed, or a run saw it flake, on any
+branch of the repository. The run's query answers which of its tasks
+are flaky, and core gives such a task one retry when it fails, as it
+does a task its local history knows as flaky. So a CI machine that
+starts empty still retries a known flake.
+
 ```ts
 export default {
   plugins: [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vzn/vx-history",
-  "version": "0.0.650",
+  "version": "0.0.652",
   "license": "MIT",
   "description": "History-based scheduling for vx — a `schedule` plugin that orders ready tasks by their expected remaining critical path, learned from the local run history.",
   "keywords": [
@@ -37,7 +37,7 @@
     "access": "public"
   },
   "peerDependencies": {
-    "@vzn/vx": "^0.0.650"
+    "@vzn/vx": "^0.0.652"
   },
   "repository": {
     "type": "git",

package/src/history-server.ts

@@ -15,6 +15,35 @@ export interface ServerTiming {
   readonly rssBytes?: number
 }
 
+/** One cached task's run, as the server's flaky record keeps it (PROTOCOL.md "Outcomes"). */
+export interface ServerOutcome {
+  /** The task's definition hash. */
+  readonly hash: string
+  /** The run's cache key. */
+  readonly key: string
+  readonly outcome: 'pass' | 'fail' | 'flaky'
+}
+
+/** What a query answers: each held hash's numbers, and the hashes asked that are flaky. */
+export interface Answer {
+  readonly timings: Map<string, ServerTiming>
+  readonly flaky: Set<string>
+}
+
+/**
+ * A cached task the run ran, as the flaky record takes it; else undefined.
+ * A hit ran nothing, and a task behind a failure ran on bytes its key never
+ * named: neither says anything of the key.
+ */
+export function outcomeOf(o: TaskOutcome, def: string): ServerOutcome | undefined {
+  if (o.node.config.cache === undefined || o.hash === undefined || o.hash === '') return undefined
+  if (o.blockedBy !== undefined) return undefined
+  if (o.status === 'failed') return { hash: def, key: o.hash, outcome: 'fail' }
+  if (o.status !== 'success') return undefined
+  const flaky = (o.attempts ?? 1) > 1 || o.flaky !== undefined
+  return { hash: def, key: o.hash, outcome: flaky ? 'flaky' : 'pass' }
+}
+
 /** Every request's deadline: a run never waits on history longer. */
 const DEADLINE_MS = 10_000
 /** Tries of a request, to the server or GitHub's OIDC endpoint, before it fails (`tried`). */
@@ -181,7 +210,8 @@ export class HistoryServer {
       u.searchParams.set('audience', new URL(this.url).origin)
       const res = await this.askGithub(u, signal)
       if (!res.ok) throw new Error(`GitHub's OIDC endpoint: ${(await refusal(res)).message}`)
-      const jwt = ((await res.json()) as { value?: unknown }).value
+      // quiet: a body that is not JSON holds no token, said below
+      const jwt = ((await res.json().catch(() => null)) as { value?: unknown } | null)?.value
       if (typeof jwt !== 'string' || jwt === '') throw new Error("GitHub's OIDC endpoint: no token")
       return { jwt, exp: expiryOf(jwt) }
     })()
@@ -304,12 +334,9 @@ export class HistoryServer {
     return asked
   }
 
-  /** Each held hash's numbers under the first of `refs` that holds them: one request a run. */
-  async query(
-    refs: readonly string[],
-    hashes: readonly string[],
-  ): Promise<Map<string, ServerTiming>> {
-    if (hashes.length === 0 || this.refused) return new Map()
+  /** Each held hash's numbers under the first of `refs` that holds them, and the flaky ones: one request a run. */
+  async query(refs: readonly string[], hashes: readonly string[]): Promise<Answer> {
+    if (hashes.length === 0 || this.refused) return { timings: new Map(), flaky: new Set() }
     const per = (await this.limitsNow()).maxQueryHashes
     const parts = await Promise.all(
       Array.from({ length: Math.ceil(hashes.length / per) }, async (_, i) => {
@@ -318,18 +345,28 @@ export class HistoryServer {
           refs,
           hashes: hashes.slice(i * per, (i + 1) * per),
         })
-        const body = (await jsonOf(res, '/v1/timings/query')) as { timings?: unknown } | null
+        const body = (await jsonOf(res, '/v1/timings/query')) as {
+          timings?: unknown
+          flaky?: unknown
+        } | null
         if (typeof body?.timings !== 'object' || body.timings === null) {
           throw new Error('the query answered no `timings` object')
         }
-        return Object.entries(body.timings as Record<string, ServerTiming | null>)
+        // A server before the flaky record answers none: nothing is flaky.
+        const flaky = Array.isArray(body.flaky)
+          ? body.flaky.filter((h): h is string => typeof h === 'string')
+          : []
+        return {
+          timings: Object.entries(body.timings as Record<string, ServerTiming | null>),
+          flaky,
+        }
       }),
     )
     // Only numbers a server folds (PROTOCOL.md): `1e309` parses to Infinity,
     // and kept as a forecast it made a NaN weight that failed every later
     // run of the cache in `schedule`.
     const out = new Map<string, ServerTiming>()
-    for (const [hash, t] of parts.flat()) {
+    for (const [hash, t] of parts.flatMap((p) => p.timings)) {
       if (!isCount(t?.durationMs)) continue
       out.set(hash, {
         durationMs: t.durationMs,
@@ -339,16 +376,28 @@ export class HistoryServer {
         ...(isCount(t.rssBytes) ? { rssBytes: t.rssBytes } : {}),
       })
     }
-    return out
+    return { timings: out, flaky: new Set(parts.flatMap((p) => p.flaky)) }
   }
 
-  /** The run's timings, by task definition hash, in uploads of at most `maxUploadTimings`; `stop` abandons them. */
-  async upload(timings: ReadonlyMap<string, ServerTiming>, stop?: AbortSignal): Promise<void> {
-    if (timings.size === 0 || this.refused) return
+  /**
+   * The run's timings, by task definition hash, and its outcomes, in
+   * uploads of at most `maxUploadTimings` of each; `stop` abandons them.
+   */
+  async upload(
+    timings: ReadonlyMap<string, ServerTiming>,
+    stop?: AbortSignal,
+    outcomes: readonly ServerOutcome[] = [],
+  ): Promise<void> {
+    if ((timings.size === 0 && outcomes.length === 0) || this.refused) return
     const per = (await this.limitsNow()).maxUploadTimings
     const all = [...timings].map(([hash, t]) => ({ hash, ...t }))
-    for (let i = 0; i < all.length; i += per) {
-      await this.call('/v1/timings/upload', { timings: all.slice(i, i + per) }, stop)
+    for (let i = 0; i < Math.max(all.length, outcomes.length); i += per) {
+      const part = outcomes.slice(i, i + per)
+      await this.call(
+        '/v1/timings/upload',
+        { timings: all.slice(i, i + per), ...(part.length > 0 ? { outcomes: part } : {}) },
+        stop,
+      )
     }
   }
 }

package/src/index.ts

@@ -33,7 +33,7 @@ import {
 import type { CommandContext } from '@vzn/vx'
 import path from 'node:path'
 import type { HistoryRow } from './history-view.js'
-import type { HistoryServer, Named, ServerTiming } from './history-server.js'
+import type { HistoryServer, Named, ServerOutcome, ServerTiming } from './history-server.js'
 
 export interface VxHistoryOptions {
   /** How many recent invocations to learn from. Default 20. */
@@ -109,6 +109,9 @@ export interface VxHistoryOptions {
 // hint needs less and reads a 2.5× smaller slice of the run history.
 const DEFAULT_WINDOW = 20
 
+/** The longest ref a history server takes in a query (its PROTOCOL.md). */
+const REF_MAX = 512
+
 // Each option is read in ONE place that a run's hooks and `vx history`
 // share: two copies of each let either drift unseen (item 805).
 const windowOf = (options: VxHistoryOptions): number =>
@@ -277,15 +280,22 @@ function namedOf(issuer: unknown, repository: unknown): Named {
   return { issuer: issuer.replace(/\/$/, ''), repository: repository.toLowerCase() }
 }
 
+/** `url` as a refusal may print it: a `user:pass@` would reach the CI log. */
+function shown(url: string): string {
+  return url.replace(/^([a-z][a-z0-9+.-]*:\/\/)[^/?#]*@/i, '$1***@')
+}
+
 /**
  * https, or http to this machine: every request carries the CI token, and
  * http:// to another host sent it in clear.
  */
 function secure(url: string): boolean {
-  if (/^https:\/\/\S/.test(url)) return true
-  if (!/^http:\/\/\S/.test(url)) return false
-  const host = URL.parse(url)?.hostname
-  return host === '127.0.0.1' || host === '[::1]' || host === 'localhost'
+  // Parsed as the request will be: a scheme in any case, no url it cannot carry.
+  const u = URL.parse(url)
+  if (u === null || u.host === '') return false
+  if (u.protocol === 'https:') return true
+  if (u.protocol !== 'http:') return false
+  return u.hostname === '127.0.0.1' || u.hostname === '[::1]' || u.hostname === 'localhost'
 }
 
 /** `VX_HISTORY_URL` under the option, trailing slashes dropped; undefined when neither names one. */
@@ -299,12 +309,18 @@ function serverUrlOf(
   const from = option === undefined ? 'VX_HISTORY_URL' : 'url'
   if (!secure(url))
     throw new UserError(
-      `vxHistory(): ${from} must be https (http only to 127.0.0.1, ::1 or localhost), got ${url}`,
+      `vxHistory(): ${from} must be https (http only to 127.0.0.1, ::1 or localhost), got ${shown(url)}`,
+    )
+  // Each route is appended to the url: past a `?` or `#` it was query or
+  // fragment, and every request asked the server's root.
+  if (/[?#]/.test(url))
+    throw new UserError(
+      `vxHistory(): ${from} takes no ?query or #fragment: each route is appended to it, got ${shown(url)}`,
     )
   // Each route is the server's /v1/…: one that names /v1 asked /v1/v1/…, and every answer was a 404.
   if (url.endsWith('/v1'))
     throw new UserError(
-      `vxHistory(): ${from} is the server's base, without /v1: ${url.slice(0, -3)}`,
+      `vxHistory(): ${from} is the server's base, without /v1: ${shown(url.slice(0, -3))}`,
     )
   return url
 }
@@ -440,14 +456,23 @@ export function vxHistory(options: VxHistoryOptions = {}): VxPlugin {
       // branch it merges into, the default branch (`baseBranch`). `main`
       // was assumed, so a `master` or `baseBranch: 'trunk'` repository
       // read no trunk's numbers; a run with none asks nothing.
-      const refs = ctx.refs
+      // A ref the server cannot name was a 400 for the whole query, and
+      // the run read no ref's numbers, its base branch's too.
+      const refs = [...new Set(ctx.refs)].filter((ref) => {
+        if (ref.length <= REF_MAX) return true
+        ctx.warn(
+          `[vx] vx-history: ${ref} is ${ref.length} characters, over the ${REF_MAX} a history server takes: the run does not read it`,
+        )
+        return false
+      })
       if (remote !== undefined && refs.length > 0) {
         const server = remote
         let late = false
         stopImport = () => (late = true)
         const weights = (async () => {
           const hashes = [...new Set(ids.flatMap((id) => defs.get(id) ?? []))]
-          const got = await server.query([...new Set(refs)], hashes)
+          const { timings: got, flaky } = await server.query(refs, hashes)
+          if (flaky.size > 0 && !late) ctx.flaky(ids.filter((id) => flaky.has(defs.get(id) ?? '')))
           const timings = new Map<string, number>()
           const needs = new Map<string, TaskResources>()
           for (const id of ids) {
@@ -520,17 +545,22 @@ export function vxHistory(options: VxHistoryOptions = {}): VxPlugin {
   hooks.setup = async (ctx) => {
     remote ??= await serverOf()
     if (remote?.writes === true) {
-      const { timingOf: timing } = await import('./history-server.js')
+      const { timingOf: timing, outcomeOf } = await import('./history-server.js')
       // Only the run that ran the tasks hears them: under DTE, the
       // orchestrator, with each worker's measurements.
       const timings = new Map<string, ServerTiming>()
-      const sent = (reported = { timings, warn: ctx.warn })
+      // Each task and key once, as the server takes them.
+      const outcomes = new Map<string, ServerOutcome>()
+      const sent = (reported = { timings, outcomes, warn: ctx.warn })
       const server = remote
       ctx.on('onTaskComplete', (node, outcome) => {
         if (node.config.exec?.command === undefined) return
         const def = taskDefinitions([node]).get(node.id)
+        if (def === undefined) return
         const t = timing(outcome)
-        if (def !== undefined && t !== undefined) timings.set(def, t)
+        if (t !== undefined) timings.set(def, t)
+        const o = outcomeOf(outcome, def)
+        if (o !== undefined) outcomes.set(`${o.hash}\0${o.key}`, o)
       })
       // Every outcome is known here, and core's own end of run (history
       // rows, cache uploads, telemetry) is still ahead: sent from teardown,
@@ -643,6 +673,7 @@ export function vxHistory(options: VxHistoryOptions = {}): VxPlugin {
 
 interface Reported {
   readonly timings: Map<string, ServerTiming>
+  readonly outcomes: ReadonlyMap<string, ServerOutcome>
   readonly warn: (m: string) => void
 }
 
@@ -661,13 +692,15 @@ const UPLOAD_WAIT_MS = 2_000
 
 function sendTimings(server: HistoryServer, sent: Reported): Upload {
   const stop = new AbortController()
-  const done = server.upload(sent.timings, stop.signal).catch((err: unknown) => {
-    // Abandoned by teardown, which says so itself.
-    if (stop.signal.aborted) return
-    sent.warn(
-      `[vx] vx-history: the timings were not sent to the history server at ${server.url}: ${err instanceof Error ? err.message : String(err)}`,
-    )
-  })
+  const done = server
+    .upload(sent.timings, stop.signal, [...sent.outcomes.values()])
+    .catch((err: unknown) => {
+      // Abandoned by teardown, which says so itself.
+      if (stop.signal.aborted) return
+      sent.warn(
+        `[vx] vx-history: the timings were not sent to the history server at ${server.url}: ${err instanceof Error ? err.message : String(err)}`,
+      )
+    })
   return { done, stop }
 }
 
@@ -800,7 +833,7 @@ async function historyCmd(
     // A plugin verb owns its help; completions offer `--help` for every verb.
     if (a === '--help' || a === '-h') {
       ui.say('info', 'Usage: vx history [--format pretty|json]', 'out')
-      process.stdout.write('\n')
+      ui.print('out', ui.gap())
       ui.say(
         'info',
         'What vx-history learned per task (p50, peak RSS, CPU parallelism) and the reservation it packs.',
@@ -876,7 +909,7 @@ async function historyCmd(
     }
   })
   if (format === 'json') {
-    process.stdout.write(`${JSON.stringify({ window, budgets, tasks: rows })}\n`)
+    ui.data('out', `${JSON.stringify({ window, budgets, tasks: rows })}\n`)
     return 0
   }
   // The table's module loads with the verb, off every run's path.