@vzn/vx-history-server 0.0.0 → 0.0.649
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Dockerfile +22 -0
- package/LICENSE +21 -0
- package/PROTOCOL.md +78 -0
- package/README.md +88 -1
- package/index.ts +4 -0
- package/package.json +53 -4
- package/src/bun.ts +111 -0
- package/src/days.ts +24 -0
- package/src/index.ts +20 -0
- package/src/server.ts +236 -0
- package/src/store.ts +178 -0
- package/src/worker.ts +104 -0
- package/wrangler.jsonc +17 -0
package/Dockerfile
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# vx-history-server as a container, built from the repo root:
|
|
2
|
+
# docker build -f packages/vx-history-server/Dockerfile -t vx-history-server .
|
|
3
|
+
# docker run -p 8080:8080 -v vx-history:/data -e ALLOW='https://token.actions.githubusercontent.com:acme/app:ci.yml' vx-history-server
|
|
4
|
+
# One bundled file: the server and the CI token check it shares with
|
|
5
|
+
# vx's other servers (@vzn/vx-devtools), with jose.
|
|
6
|
+
FROM oven/bun:1.4.4-slim@sha256:1846adf22496bd711036aae8be147e57bdebbaea2e5264e3e0b5af1db187d9a4 AS build
|
|
7
|
+
WORKDIR /repo
|
|
8
|
+
COPY packages/vx-devtools/package.json packages/vx-devtools/
|
|
9
|
+
# jose as @vzn/vx-devtools pins it: the image names no version of its own.
|
|
10
|
+
RUN cd packages/vx-devtools && bun install --production --ignore-scripts
|
|
11
|
+
COPY packages/vx-devtools/src packages/vx-devtools/src/
|
|
12
|
+
COPY packages/vx-history-server/src packages/vx-history-server/src/
|
|
13
|
+
RUN mkdir -p node_modules/@vzn && ln -s ../../packages/vx-devtools node_modules/@vzn/vx-devtools \
|
|
14
|
+
&& bun build packages/vx-history-server/src/bun.ts --target=bun --outfile=/out/vx-history-server.js
|
|
15
|
+
|
|
16
|
+
FROM oven/bun:1.4.4-slim@sha256:1846adf22496bd711036aae8be147e57bdebbaea2e5264e3e0b5af1db187d9a4
|
|
17
|
+
COPY --from=build /out/vx-history-server.js /app/
|
|
18
|
+
RUN mkdir /data && chown bun /data
|
|
19
|
+
USER bun
|
|
20
|
+
VOLUME /data
|
|
21
|
+
EXPOSE 8080
|
|
22
|
+
ENTRYPOINT ["bun", "--no-env-file", "--no-install", "/app/vx-history-server.js", "--db", "/data/vx-history.sqlite"]
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 VZN
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/PROTOCOL.md
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
# vx-history protocol, v1
|
|
2
|
+
|
|
3
|
+
An HTTP API for the task timings a scheduler asks for. It is a history,
|
|
4
|
+
not a log: a server keeps one row per task and ref, and folds each
|
|
5
|
+
report into it. `@vzn/vx-history-server` speaks it on Bun, on Cloudflare
|
|
6
|
+
Workers and as a Docker image; any server that does can take its place.
|
|
7
|
+
|
|
8
|
+
| Route | What it does |
|
|
9
|
+
| ------------------------- | --------------------------------- |
|
|
10
|
+
| `POST /v1/timings/upload` | a run's timings, once at its end |
|
|
11
|
+
| `POST /v1/timings/query` | each task's numbers, ready to use |
|
|
12
|
+
| `GET /v1/capabilities` | the server's limits |
|
|
13
|
+
|
|
14
|
+
## Repositories and tokens
|
|
15
|
+
|
|
16
|
+
A row is keyed by the issuer (the CI issuer's URL, no trailing slash),
|
|
17
|
+
the repository (`org/name` there, lowercased), the ref
|
|
18
|
+
(`refs/heads/main`) and the task's definition hash, each stored plainly.
|
|
19
|
+
|
|
20
|
+
A request's `Authorization: Bearer <token>` is a CI job's OIDC token,
|
|
21
|
+
checked as `@vzn/vx-devtools` checks it (the same `ALLOW` as
|
|
22
|
+
vx-cache-server's): a repository, or a workflow, `ALLOW` does not name
|
|
23
|
+
is `403`. A token writes its own ref (`refs/heads/<branch>`; a
|
|
24
|
+
`pull_request` run its merge ref, `refs/pull/<n>/merge`, never its
|
|
25
|
+
`head_ref`; a `merge_group` run,
|
|
26
|
+
`refs/heads/gh-readonly-queue/<base>/pr-<n>-<sha>`, its base branch); a
|
|
27
|
+
`pull_request_target`, `issue_comment` or `workflow_run` run reads only,
|
|
28
|
+
and so does a GitLab merge request pipeline whose `source_project_path` (GitLab 19.5+)
|
|
29
|
+
does not name its own project. A query names its
|
|
30
|
+
repository; a token of another is `403`. A server whose `READ_ACCESS`
|
|
31
|
+
is `public` takes a query with no token for a repository `ALLOW` names;
|
|
32
|
+
an upload with none is `403`.
|
|
33
|
+
|
|
34
|
+
## Upload
|
|
35
|
+
|
|
36
|
+
`POST /v1/timings/upload` (a token that writes) with
|
|
37
|
+
`{ "timings": [{ "hash", "durationMs", "cores"?, "rssBytes"? }] }`, up to
|
|
38
|
+
10 000, each task once: `204`. `hash` names the task's definition, a
|
|
39
|
+
string of up to 512 characters (no lone surrogate) the server never
|
|
40
|
+
reads. Each folds into its row of the token's repository and ref:
|
|
41
|
+
|
|
42
|
+
- `durationMs` and `cores`: `old × 0.7 + new × 0.3`. A median needs the
|
|
43
|
+
samples and a mean never forgets; this follows a task that got slower
|
|
44
|
+
within about three runs, and one outlier moves it 30%.
|
|
45
|
+
- `rssBytes`: `max(new, old × 0.9)`. A reservation needs the peak; the
|
|
46
|
+
decay lets a task that shrank come down.
|
|
47
|
+
- `runs`: how many reports moved the row.
|
|
48
|
+
|
|
49
|
+
A report that would not move the row (`durationMs` and `cores` within
|
|
50
|
+
5%, no new `rssBytes` peak) is not written, unless the row is a day old:
|
|
51
|
+
each write costs, and the fold would move it under 1.5%.
|
|
52
|
+
|
|
53
|
+
The first report sets the row; one without `cores` or `rssBytes` keeps
|
|
54
|
+
the row's. A row not reported for `TIMING_DAYS` (30) is deleted.
|
|
55
|
+
|
|
56
|
+
## Query
|
|
57
|
+
|
|
58
|
+
One request a run: `POST /v1/timings/query` with
|
|
59
|
+
`{ "issuer", "repository", "refs": ["refs/heads/feature", "refs/heads/main"], "hashes": [ … ] }`,
|
|
60
|
+
1 to 8 refs and up to 10 000 hashes: each hash's row in the first ref
|
|
61
|
+
named that holds one. A hash none holds is absent. The numbers are the
|
|
62
|
+
scheduler's as they are:
|
|
63
|
+
|
|
64
|
+
```json
|
|
65
|
+
{ "timings": { "3b0d…": { "durationMs": 4200, "cores": 1.8, "rssBytes": 310000000, "runs": 12 } } }
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
## Limits and errors
|
|
69
|
+
|
|
70
|
+
`GET /v1/capabilities`: `{ "maxUploadTimings": 10000, "maxQueryHashes": 10000, "maxQueryRefs": 8 }`.
|
|
71
|
+
A server takes both at their limits, each hash 512 characters: an
|
|
72
|
+
upload body up to 7 120 000 bytes, a query's up to 6 010 000. Past
|
|
73
|
+
that, `413`.
|
|
74
|
+
|
|
75
|
+
Every refusal is `{ code, message }`, the message saying what to fix. A
|
|
76
|
+
JSON body without `Content-Type: application/json` is `400`, an unknown
|
|
77
|
+
route `404`, a route's wrong method `405` with `Allow`, a failure the
|
|
78
|
+
server did not expect `500`, its `message` what failed.
|
package/README.md
CHANGED
|
@@ -1,3 +1,90 @@
|
|
|
1
1
|
# @vzn/vx-history-server
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Website: <https://vznjs.github.io/vx/> · source: <https://github.com/vznjs/vx>
|
|
4
|
+
|
|
5
|
+
A server for the task timings a vx scheduler asks for, that you host. It
|
|
6
|
+
keeps one row per task, already the numbers the scheduler uses (duration,
|
|
7
|
+
CPU cores, peak memory), and merges each CI run's report into it: a
|
|
8
|
+
history, not a log. It speaks the [vx-history protocol](./PROTOCOL.md)
|
|
9
|
+
(`/v1`) on three backends that answer alike:
|
|
10
|
+
|
|
11
|
+
- **Bun**, with a SQLite file.
|
|
12
|
+
- **Cloudflare Workers**, with D1. It fits the free plan: an upload is one
|
|
13
|
+
statement per 2 MB of reports, a row write per task; a query one read
|
|
14
|
+
per 2 MB of hashes.
|
|
15
|
+
- **Docker**, the Bun server in an image.
|
|
16
|
+
|
|
17
|
+
`vxHistory()` ([`@vzn/vx-history`](https://github.com/vznjs/vx/tree/main/packages/vx-history))
|
|
18
|
+
points vx at it.
|
|
19
|
+
|
|
20
|
+
## CI tokens
|
|
21
|
+
|
|
22
|
+
Every request carries a CI job's own OIDC token, checked by `@vzn/vx-devtools`,
|
|
23
|
+
under the same `ALLOW` as [`@vzn/vx-cache-server`'s](https://github.com/vznjs/vx/tree/main/packages/vx-cache-server#ci-tokens)
|
|
24
|
+
(`issuer:owner/repo:workflow` entries): a job of a workflow it names
|
|
25
|
+
writes its own branch, and any other job is refused. A row is the issuer,
|
|
26
|
+
repository, ref and task hash, stored plainly, and its numbers. A run
|
|
27
|
+
sends one query: its repository, the refs it reads (its branch, then the
|
|
28
|
+
branch it merges into, then the default branch) and every task hash;
|
|
29
|
+
the server answers each from the first ref that holds it. A token of
|
|
30
|
+
another repository than the query names is refused. `AUDIENCE` is the
|
|
31
|
+
`aud` a token must name: the origin clients reach the server at. On
|
|
32
|
+
Workers, unset, it is that origin (Cloudflare routes only the Worker's
|
|
33
|
+
own hostnames to it); on Bun and Docker it is required, since there the
|
|
34
|
+
origin is the request's `Host` header, which a client picks.
|
|
35
|
+
`READ_ACCESS=public` lets a query with no token read a repository
|
|
36
|
+
`ALLOW` names; it uploads nothing.
|
|
37
|
+
|
|
38
|
+
## On Bun
|
|
39
|
+
|
|
40
|
+
```sh
|
|
41
|
+
ALLOW='https://token.actions.githubusercontent.com:acme/app:ci.yml' AUDIENCE=https://history.example.com bun packages/vx-history-server/src/bun.ts --db /var/lib/vx-history.sqlite --port 8080
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
`TIMING_DAYS` (default 30, whole days in decimal digits; on Workers a JSON number too)
|
|
45
|
+
is how long a task's row outlives its last report; the first upload of each day sweeps, whichever node or Workers
|
|
46
|
+
isolate takes it.
|
|
47
|
+
|
|
48
|
+
## With Docker
|
|
49
|
+
|
|
50
|
+
From the repo root:
|
|
51
|
+
|
|
52
|
+
```sh
|
|
53
|
+
docker build -f packages/vx-history-server/Dockerfile -t vx-history-server .
|
|
54
|
+
docker run -p 8080:8080 -v vx-history:/data -e ALLOW='…' -e AUDIENCE=https://history.example.com vx-history-server
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
The database is `/data/vx-history.sqlite`.
|
|
58
|
+
|
|
59
|
+
With vx's other servers: [`compose.yaml`](https://github.com/vznjs/vx/blob/main/private/vx-servers/compose.yaml)
|
|
60
|
+
builds and runs all three (`docker compose up -d`).
|
|
61
|
+
|
|
62
|
+
## On Cloudflare Workers
|
|
63
|
+
|
|
64
|
+
From this directory, with `ALLOW` filled in `wrangler.jsonc`:
|
|
65
|
+
|
|
66
|
+
```sh
|
|
67
|
+
bun install
|
|
68
|
+
bunx wrangler login
|
|
69
|
+
bunx wrangler deploy
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
The Worker reads its D1 database as the binding `DB`, as `wrangler.jsonc`
|
|
73
|
+
names it.
|
|
74
|
+
|
|
75
|
+
## API
|
|
76
|
+
|
|
77
|
+
The same server as a `fetch` handler, over storage of your own
|
|
78
|
+
(`src/index.ts`):
|
|
79
|
+
|
|
80
|
+
- `createHistoryServer(options: ServerOptions)` returns
|
|
81
|
+
`(req: Request) => Promise<Response>`. `ServerOptions` is
|
|
82
|
+
`{ store, authorize, timingDays?, now? }`: `authorize` is
|
|
83
|
+
`oidcTokenAuth` from `@vzn/vx-devtools`.
|
|
84
|
+
- `sqliteStore(db: Db)` is the `Store` over a SQLite `Db`
|
|
85
|
+
(`{ all(sql, params), run(sql, params) }`), as bun:sqlite and D1 are.
|
|
86
|
+
A `Store` merges `Report`s and reads `Timing`s, each of a `Repo`
|
|
87
|
+
(`{ issuer, repository }`) and a ref.
|
|
88
|
+
- `WEIGHT` (0.3) and `RSS_DECAY` (0.9) are the merge's constants;
|
|
89
|
+
`MAX_UPLOAD_TIMINGS`, `MAX_QUERY_HASHES` and `MAX_QUERY_REFS` its
|
|
90
|
+
limits. A refusal is `errorJson` from `@vzn/vx-devtools`.
|
package/index.ts
ADDED
package/package.json
CHANGED
|
@@ -1,8 +1,57 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vzn/vx-history-server",
|
|
3
|
-
"version": "0.0.
|
|
4
|
-
"description": "Placeholder; real releases are published by CI.",
|
|
3
|
+
"version": "0.0.649",
|
|
5
4
|
"license": "MIT",
|
|
6
|
-
"
|
|
7
|
-
"
|
|
5
|
+
"description": "The task timings a vx scheduler asks for, one folded row per task, that you host: on Bun with SQLite, on Cloudflare Workers with D1, or as a Docker image.",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"vx",
|
|
8
|
+
"monorepo",
|
|
9
|
+
"scheduler",
|
|
10
|
+
"history-server"
|
|
11
|
+
],
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "./src/index.ts",
|
|
14
|
+
"types": "./src/index.ts",
|
|
15
|
+
"bin": {
|
|
16
|
+
"vx-history-server": "./src/bun.ts"
|
|
17
|
+
},
|
|
18
|
+
"exports": {
|
|
19
|
+
".": {
|
|
20
|
+
"types": "./src/index.ts",
|
|
21
|
+
"import": "./src/index.ts"
|
|
22
|
+
},
|
|
23
|
+
"./worker": {
|
|
24
|
+
"types": "./src/worker.ts",
|
|
25
|
+
"import": "./src/worker.ts"
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"files": [
|
|
29
|
+
"index.ts",
|
|
30
|
+
"src",
|
|
31
|
+
"PROTOCOL.md",
|
|
32
|
+
"wrangler.jsonc",
|
|
33
|
+
"Dockerfile",
|
|
34
|
+
"README.md",
|
|
35
|
+
"LICENSE"
|
|
36
|
+
],
|
|
37
|
+
"engines": {
|
|
38
|
+
"bun": ">=1.4.4"
|
|
39
|
+
},
|
|
40
|
+
"//": [
|
|
41
|
+
"@vzn/vx: vx.config.ts's core; `*`, not workspace:, which npm refuses when a deploy installs this directory alone.",
|
|
42
|
+
"@vzn/vx-devtools: checks the CI tokens, the check vx's servers share (it imports only jose); `*` for the same reason.",
|
|
43
|
+
"wrangler: deploys src/worker.ts by wrangler.jsonc.",
|
|
44
|
+
"jose: the tests sign the CI tokens the server checks.",
|
|
45
|
+
"miniflare: tests/worker.test.ts runs src/worker.ts in workerd with D1, as Workers deploy it."
|
|
46
|
+
],
|
|
47
|
+
"dependencies": {
|
|
48
|
+
"@vzn/vx-devtools": "0.0.649"
|
|
49
|
+
},
|
|
50
|
+
"repository": {
|
|
51
|
+
"type": "git",
|
|
52
|
+
"url": "git+https://github.com/vznjs/vx.git",
|
|
53
|
+
"directory": "packages/vx-history-server"
|
|
54
|
+
},
|
|
55
|
+
"homepage": "https://github.com/vznjs/vx/tree/main/packages/vx-history-server#readme",
|
|
56
|
+
"bugs": "https://github.com/vznjs/vx/issues"
|
|
8
57
|
}
|
package/src/bun.ts
ADDED
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
#!/usr/bin/env -S bun --no-env-file --no-install
|
|
2
|
+
// `vx-history-server` on Bun: the timings in one SQLite file (PROTOCOL.md,
|
|
3
|
+
// README.md).
|
|
4
|
+
|
|
5
|
+
import { Database } from 'bun:sqlite'
|
|
6
|
+
import { parseArgs } from 'node:util'
|
|
7
|
+
import { oidcTokenAuth, parseAllow, statusLine } from '@vzn/vx-devtools'
|
|
8
|
+
import { timingDaysOf } from './days.js'
|
|
9
|
+
import { createHistoryServer } from './server.js'
|
|
10
|
+
import { type Db, sqliteStore } from './store.js'
|
|
11
|
+
|
|
12
|
+
const USAGE = `usage: vx-history-server [--db vx-history.sqlite] [--port 8080] [--host 0.0.0.0]
|
|
13
|
+
ALLOW who may take part, space separated, each issuer:owner/repo:workflow
|
|
14
|
+
(README.md § CI tokens); only their jobs' OIDC tokens are taken
|
|
15
|
+
AUDIENCE the aud a CI token names: the URL clients reach this server at
|
|
16
|
+
READ_ACCESS public: a request with no token reads a repository ALLOW names
|
|
17
|
+
TIMING_DAYS days a task's timings outlive its last report (default 30)`
|
|
18
|
+
|
|
19
|
+
/** A SQLite file through bun:sqlite. */
|
|
20
|
+
export function sqliteDb(path: string): Db {
|
|
21
|
+
const db = new Database(path, { create: true, strict: true })
|
|
22
|
+
db.run('PRAGMA journal_mode = WAL')
|
|
23
|
+
db.run('PRAGMA busy_timeout = 5000')
|
|
24
|
+
return {
|
|
25
|
+
all: async <T>(sql: string, params: readonly unknown[]) =>
|
|
26
|
+
db.query(sql).all(...(params as never[])) as T[],
|
|
27
|
+
run: async (sql, params) => db.query(sql).run(...(params as never[])).changes,
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function main(argv: string[]): void {
|
|
32
|
+
const { values } = parseArgs({
|
|
33
|
+
args: argv,
|
|
34
|
+
options: {
|
|
35
|
+
db: { type: 'string', default: 'vx-history.sqlite' },
|
|
36
|
+
port: { type: 'string', default: '8080' },
|
|
37
|
+
host: { type: 'string', default: '0.0.0.0' },
|
|
38
|
+
help: { type: 'boolean', short: 'h', default: false },
|
|
39
|
+
},
|
|
40
|
+
})
|
|
41
|
+
if (values.help) {
|
|
42
|
+
console.log(USAGE)
|
|
43
|
+
return
|
|
44
|
+
}
|
|
45
|
+
const env = Bun.env
|
|
46
|
+
// Digits only: Number('') is 0, a random port for `--port "$PORT"` with
|
|
47
|
+
// PORT unset, and Number read 0x50 as 80 and 1e1 as 10.
|
|
48
|
+
const port = /^\d+$/.test(values.port) ? Number(values.port) : Number.NaN
|
|
49
|
+
if (!Number.isInteger(port) || port < 0 || port > 65_535)
|
|
50
|
+
fail(`--port: not a port: ${values.port}`)
|
|
51
|
+
// Bun.serve reads '' as localhost: unreachable from another host or container.
|
|
52
|
+
if (values.host.trim() === '') fail(`--host: needs an address, got '${values.host}'`)
|
|
53
|
+
let allow
|
|
54
|
+
let timingDays
|
|
55
|
+
try {
|
|
56
|
+
allow = parseAllow(env.ALLOW ?? '')
|
|
57
|
+
} catch (err) {
|
|
58
|
+
fail(`ALLOW: ${(err as Error).message}`)
|
|
59
|
+
}
|
|
60
|
+
// An empty one started a server that refused every request.
|
|
61
|
+
if (allow.length === 0) fail(`set ALLOW\n${USAGE}`)
|
|
62
|
+
try {
|
|
63
|
+
timingDays = timingDaysOf(env.TIMING_DAYS)
|
|
64
|
+
} catch (err) {
|
|
65
|
+
fail((err as Error).message)
|
|
66
|
+
}
|
|
67
|
+
// bun:sqlite opens '' as a temporary database: every timing was gone at
|
|
68
|
+
// the next restart, and nothing said so.
|
|
69
|
+
if (values.db === '') fail(`--db: needs a file path, got ''\n${USAGE}`)
|
|
70
|
+
let db
|
|
71
|
+
try {
|
|
72
|
+
db = sqliteDb(values.db)
|
|
73
|
+
} catch (err) {
|
|
74
|
+
// SQLite's own message ("unable to open database file") named no path.
|
|
75
|
+
fail(`--db ${values.db}: ${(err as Error).message}`)
|
|
76
|
+
}
|
|
77
|
+
// Unset, the audience would be the origin a request names: its Host
|
|
78
|
+
// header, which the client picks. Workers know their own hostnames.
|
|
79
|
+
const audience = env.AUDIENCE?.trim() ?? ''
|
|
80
|
+
if (audience === '')
|
|
81
|
+
fail(`set AUDIENCE: the aud a CI token names, the URL clients reach this server at\n${USAGE}`)
|
|
82
|
+
const handle = createHistoryServer({
|
|
83
|
+
store: sqliteStore(db),
|
|
84
|
+
authorize: oidcTokenAuth({
|
|
85
|
+
allow,
|
|
86
|
+
readAccess: env.READ_ACCESS,
|
|
87
|
+
audience,
|
|
88
|
+
}),
|
|
89
|
+
...(timingDays === undefined ? {} : { timingDays }),
|
|
90
|
+
})
|
|
91
|
+
const server = Bun.serve({ hostname: values.host, port, fetch: handle })
|
|
92
|
+
process.stdout.write(statusLine('ok', 'vx-history-server', `listening on ${server.url.href}`))
|
|
93
|
+
const stop = (): void => void server.stop().then(() => process.exit(0))
|
|
94
|
+
process.on('SIGINT', stop)
|
|
95
|
+
process.on('SIGTERM', stop)
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
function fail(message: string): never {
|
|
99
|
+
process.stderr.write(statusLine('err', 'vx-history-server', message))
|
|
100
|
+
process.exit(2)
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
if (import.meta.main) {
|
|
104
|
+
try {
|
|
105
|
+
main(process.argv.slice(2))
|
|
106
|
+
} catch (err) {
|
|
107
|
+
// parseArgs names the flag it refused, not the ones there are.
|
|
108
|
+
const usage = (err as { code?: string }).code?.startsWith('ERR_PARSE_ARGS') ? `\n${USAGE}` : ''
|
|
109
|
+
fail(`${err instanceof Error ? err.message : String(err)}${usage}`)
|
|
110
|
+
}
|
|
111
|
+
}
|
package/src/days.ts
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
// `TIMING_DAYS`, as Bun's environment and Workers' vars both name it.
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* `TIMING_DAYS`: a whole number of days, or undefined for the default. A
|
|
5
|
+
* Worker var may be JSON: a number is read as Bun reads its decimal digits, and
|
|
6
|
+
* anything else is refused, as `TIMING_DAYS=true` is on Bun (`true` was
|
|
7
|
+
* read as 1 day).
|
|
8
|
+
*/
|
|
9
|
+
export function timingDaysOf(value: unknown): number | undefined {
|
|
10
|
+
if (value === undefined || value === '') return undefined
|
|
11
|
+
// A string is digits only, as --port is: Number read 0x1e as 30.
|
|
12
|
+
const days =
|
|
13
|
+
typeof value === 'number'
|
|
14
|
+
? value
|
|
15
|
+
: typeof value === 'string' && /^\d+$/.test(value)
|
|
16
|
+
? Number(value)
|
|
17
|
+
: Number.NaN
|
|
18
|
+
if (!Number.isSafeInteger(days) || days < 1) {
|
|
19
|
+
throw new Error(
|
|
20
|
+
`TIMING_DAYS: a whole number of days, got ${typeof value === 'string' || typeof value === 'number' ? value : JSON.stringify(value)}`,
|
|
21
|
+
)
|
|
22
|
+
}
|
|
23
|
+
return days
|
|
24
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
// @vzn/vx-history-server: the task timings a vx scheduler asks for, one
|
|
2
|
+
// folded row per task, on Bun with SQLite or on Cloudflare Workers with D1.
|
|
3
|
+
|
|
4
|
+
export {
|
|
5
|
+
createHistoryServer,
|
|
6
|
+
MAX_QUERY_HASHES,
|
|
7
|
+
MAX_QUERY_REFS,
|
|
8
|
+
MAX_UPLOAD_TIMINGS,
|
|
9
|
+
type ServerOptions,
|
|
10
|
+
} from './server.js'
|
|
11
|
+
export {
|
|
12
|
+
type Db,
|
|
13
|
+
type Report,
|
|
14
|
+
type Repo,
|
|
15
|
+
RSS_DECAY,
|
|
16
|
+
sqliteStore,
|
|
17
|
+
type Store,
|
|
18
|
+
type Timing,
|
|
19
|
+
WEIGHT,
|
|
20
|
+
} from './store.js'
|
package/src/server.ts
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// The vx-history protocol, v1 (PROTOCOL.md), as one fetch handler over a
|
|
2
|
+
// `Store`: Bun and Workers serve the same one.
|
|
3
|
+
|
|
4
|
+
import { type Access, errorJson, notAllowed, Refusal } from '@vzn/vx-devtools'
|
|
5
|
+
import type { Report, Repo, Store } from './store.js'
|
|
6
|
+
|
|
7
|
+
/** Reports one upload folds in: a run's tasks. */
|
|
8
|
+
export const MAX_UPLOAD_TIMINGS = 10_000
|
|
9
|
+
/** Hashes one query names. */
|
|
10
|
+
export const MAX_QUERY_HASHES = 10_000
|
|
11
|
+
/** Refs one query names, read in order. */
|
|
12
|
+
export const MAX_QUERY_REFS = 8
|
|
13
|
+
const MAX_STRING = 512
|
|
14
|
+
/** A report's bytes at most: its hash at `MAX_STRING`, its numbers at their longest. */
|
|
15
|
+
const MAX_REPORT_BYTES = MAX_STRING + 200
|
|
16
|
+
const DAY_MS = 86_400_000
|
|
17
|
+
|
|
18
|
+
export interface ServerOptions {
|
|
19
|
+
readonly store: Store
|
|
20
|
+
/** A request's verified token, checked against the repository it names (`oidcTokenAuth`). */
|
|
21
|
+
readonly authorize: (req: Request, named?: Repo) => Promise<Access | undefined>
|
|
22
|
+
/** Days a task's row outlives its last report (default 30). */
|
|
23
|
+
readonly timingDays?: number
|
|
24
|
+
/** Milliseconds since the epoch; tests move the clock. */
|
|
25
|
+
readonly now?: () => number
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A full ref, as a token names one. */
|
|
29
|
+
const REF = /^refs\/\S+$/
|
|
30
|
+
|
|
31
|
+
export function createHistoryServer(opts: ServerOptions): (req: Request) => Promise<Response> {
|
|
32
|
+
const { store, authorize } = opts
|
|
33
|
+
const now = opts.now ?? Date.now
|
|
34
|
+
let ready: Promise<void> | undefined
|
|
35
|
+
const schema = (): Promise<void> =>
|
|
36
|
+
(ready ??= store.init().catch((err: unknown) => {
|
|
37
|
+
ready = undefined
|
|
38
|
+
throw err
|
|
39
|
+
}))
|
|
40
|
+
const capabilities = {
|
|
41
|
+
maxUploadTimings: MAX_UPLOAD_TIMINGS,
|
|
42
|
+
maxQueryHashes: MAX_QUERY_HASHES,
|
|
43
|
+
maxQueryRefs: MAX_QUERY_REFS,
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
// The fold is one upsert, a few ms for a thousand tasks: done inline (2026-10-25).
|
|
47
|
+
async function upload(req: Request): Promise<Response> {
|
|
48
|
+
const access = await authorize(req)
|
|
49
|
+
if (access?.write === undefined) throw new Refusal(403, 'this request reads only')
|
|
50
|
+
const reports = parseReports(await readJson(req, MAX_UPLOAD_TIMINGS * MAX_REPORT_BYTES))
|
|
51
|
+
await schema()
|
|
52
|
+
await store.fold(access, access.write, now(), reports)
|
|
53
|
+
return new Response(null, { status: 204 })
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// One request a run: every hash, each answered from the first ref that holds it.
|
|
57
|
+
async function query(req: Request): Promise<Response> {
|
|
58
|
+
// No token, or a bad one, is refused before the body is read.
|
|
59
|
+
await authorize(req)
|
|
60
|
+
const body = await readJson(req, MAX_QUERY_HASHES * 600 + 10_000)
|
|
61
|
+
const issuer = field(body, 'issuer')
|
|
62
|
+
const repository = field(body, 'repository')
|
|
63
|
+
const refs = field(body, 'refs')
|
|
64
|
+
const hashes = field(body, 'hashes')
|
|
65
|
+
if (
|
|
66
|
+
typeof issuer !== 'string' ||
|
|
67
|
+
!/^https?:\/\/\S+$/.test(issuer) ||
|
|
68
|
+
issuer.length > MAX_STRING
|
|
69
|
+
)
|
|
70
|
+
throw new Refusal(400, "issuer: the CI issuer's URL, as its tokens name it")
|
|
71
|
+
if (
|
|
72
|
+
typeof repository !== 'string' ||
|
|
73
|
+
!/^[^/\s]+(\/[^/\s]+)+$/.test(repository) ||
|
|
74
|
+
repository.length > MAX_STRING
|
|
75
|
+
)
|
|
76
|
+
throw new Refusal(400, 'repository: org/name at the issuer')
|
|
77
|
+
if (
|
|
78
|
+
!Array.isArray(refs) ||
|
|
79
|
+
refs.length === 0 ||
|
|
80
|
+
refs.length > MAX_QUERY_REFS ||
|
|
81
|
+
!refs.every((r) => typeof r === 'string' && REF.test(r) && r.length <= MAX_STRING)
|
|
82
|
+
) {
|
|
83
|
+
throw new Refusal(
|
|
84
|
+
400,
|
|
85
|
+
`refs: 1 to ${MAX_QUERY_REFS} full refs (refs/heads/main), read in order`,
|
|
86
|
+
)
|
|
87
|
+
}
|
|
88
|
+
if (
|
|
89
|
+
!Array.isArray(hashes) ||
|
|
90
|
+
hashes.length > MAX_QUERY_HASHES ||
|
|
91
|
+
!hashes.every((h) => isOpaque(h))
|
|
92
|
+
) {
|
|
93
|
+
throw new Refusal(400, `hashes: a list of up to ${MAX_QUERY_HASHES} strings`)
|
|
94
|
+
}
|
|
95
|
+
const repo = (await authorize(req, { issuer, repository }))!
|
|
96
|
+
await schema()
|
|
97
|
+
const rows = await store.read(repo, refs as string[], hashes as string[])
|
|
98
|
+
return Response.json({ timings: Object.fromEntries(rows) })
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
let swept = 0
|
|
102
|
+
// At most one sweep a day, on an upload: no timer, no cron. Each node
|
|
103
|
+
// asks the database when any node last swept, once a day: a Worker's
|
|
104
|
+
// isolates each swept the whole table on their first upload. A failed
|
|
105
|
+
// sweep fails no upload, which is folded by then; the next tries again.
|
|
106
|
+
const sweep = async (): Promise<void> => {
|
|
107
|
+
const t = now()
|
|
108
|
+
if (t - swept < DAY_MS) return
|
|
109
|
+
swept = t
|
|
110
|
+
try {
|
|
111
|
+
const last = await store.sweptAt()
|
|
112
|
+
if (t - last < DAY_MS) {
|
|
113
|
+
swept = last
|
|
114
|
+
return
|
|
115
|
+
}
|
|
116
|
+
await store.sweep(t - (opts.timingDays ?? 30) * DAY_MS, t)
|
|
117
|
+
} catch (err) {
|
|
118
|
+
swept = 0
|
|
119
|
+
console.error(`vx-history-server: the sweep: ${String(err)}`)
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
return async (req) => {
|
|
124
|
+
const path = new URL(req.url).pathname
|
|
125
|
+
try {
|
|
126
|
+
const search = new URL(req.url).search
|
|
127
|
+
if (search !== '')
|
|
128
|
+
throw new Refusal(
|
|
129
|
+
400,
|
|
130
|
+
`this server takes no query parameters: drop ${search.slice(0, 200)} from the url`,
|
|
131
|
+
)
|
|
132
|
+
if (path === '/v1/timings/upload') {
|
|
133
|
+
if (req.method !== 'POST') return notAllowed('POST')
|
|
134
|
+
const res = await upload(req)
|
|
135
|
+
await sweep()
|
|
136
|
+
return res
|
|
137
|
+
}
|
|
138
|
+
if (path === '/v1/timings/query') {
|
|
139
|
+
return req.method === 'POST' ? await query(req) : notAllowed('POST')
|
|
140
|
+
}
|
|
141
|
+
if (path === '/v1/capabilities') {
|
|
142
|
+
if (req.method !== 'GET') return notAllowed('GET')
|
|
143
|
+
await authorize(req)
|
|
144
|
+
return Response.json(capabilities)
|
|
145
|
+
}
|
|
146
|
+
return errorJson(
|
|
147
|
+
404,
|
|
148
|
+
`no route ${req.method} ${path.slice(0, 200)}: name the server's base URL; it serves /v1/ (PROTOCOL.md)`,
|
|
149
|
+
)
|
|
150
|
+
} catch (err) {
|
|
151
|
+
if (err instanceof Refusal) return errorJson(err.status, err.message)
|
|
152
|
+
// Never swallowed: the client warns with what the server says.
|
|
153
|
+
console.error(`vx-history-server: ${req.method} ${path}: ${String(err)}`)
|
|
154
|
+
return errorJson(500, err instanceof Error ? err.message : String(err))
|
|
155
|
+
}
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
const field = (body: unknown, key: string): unknown =>
|
|
160
|
+
typeof body === 'object' && body !== null ? (body as Record<string, unknown>)[key] : undefined
|
|
161
|
+
|
|
162
|
+
// A lone surrogate is stored as bytes that read back as U+FFFD: the row
|
|
163
|
+
// answers to a hash no client asked for.
|
|
164
|
+
const isOpaque = (s: unknown): s is string =>
|
|
165
|
+
typeof s === 'string' && s.length > 0 && s.length <= MAX_STRING && !/\p{Cs}/u.test(s)
|
|
166
|
+
|
|
167
|
+
const isCount = (n: unknown): n is number =>
|
|
168
|
+
typeof n === 'number' && Number.isSafeInteger(n) && n >= 0
|
|
169
|
+
|
|
170
|
+
async function readJson(req: Request, max: number): Promise<unknown> {
|
|
171
|
+
if (!/^application\/json\b/i.test(req.headers.get('content-type') ?? '')) {
|
|
172
|
+
throw new Refusal(400, 'send the body as Content-Type: application/json')
|
|
173
|
+
}
|
|
174
|
+
if (Number(req.headers.get('content-length') ?? 0) > max) {
|
|
175
|
+
throw new Refusal(413, `the body is at most ${max} bytes`)
|
|
176
|
+
}
|
|
177
|
+
const text = await textUpTo(req, max)
|
|
178
|
+
try {
|
|
179
|
+
return JSON.parse(text)
|
|
180
|
+
} catch {
|
|
181
|
+
throw new Refusal(400, 'the body is not JSON')
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
/** A run's reports, as the store folds them: unknown fields are dropped. */
|
|
186
|
+
function parseReports(body: unknown): Report[] {
|
|
187
|
+
const list = field(body, 'timings')
|
|
188
|
+
if (!Array.isArray(list) || list.length > MAX_UPLOAD_TIMINGS) {
|
|
189
|
+
throw new Refusal(400, `timings: a list of up to ${MAX_UPLOAD_TIMINGS}`)
|
|
190
|
+
}
|
|
191
|
+
const seen = new Set<string>()
|
|
192
|
+
return list.map((x: unknown, i): Report => {
|
|
193
|
+
const bad = (what: string): never => {
|
|
194
|
+
throw new Refusal(400, `timings[${i}].${what}`)
|
|
195
|
+
}
|
|
196
|
+
const hash = field(x, 'hash')
|
|
197
|
+
const durationMs = field(x, 'durationMs')
|
|
198
|
+
const cores = field(x, 'cores')
|
|
199
|
+
const rssBytes = field(x, 'rssBytes')
|
|
200
|
+
if (!isOpaque(hash)) bad(`hash: a string of up to ${MAX_STRING} characters`)
|
|
201
|
+
// One row per task: a second report of it in one upload would fold twice.
|
|
202
|
+
if (seen.has(hash as string)) bad('hash: each task once')
|
|
203
|
+
seen.add(hash as string)
|
|
204
|
+
if (!isCount(durationMs)) bad('durationMs: whole milliseconds')
|
|
205
|
+
if (
|
|
206
|
+
cores !== undefined &&
|
|
207
|
+
!(typeof cores === 'number' && Number.isFinite(cores) && cores >= 0)
|
|
208
|
+
) {
|
|
209
|
+
bad('cores: a number of cores')
|
|
210
|
+
}
|
|
211
|
+
if (rssBytes !== undefined && !isCount(rssBytes)) bad('rssBytes: a byte count')
|
|
212
|
+
return {
|
|
213
|
+
hash: hash as string,
|
|
214
|
+
durationMs: durationMs as number,
|
|
215
|
+
...(cores === undefined ? {} : { cores: cores as number }),
|
|
216
|
+
...(rssBytes === undefined ? {} : { rssBytes: rssBytes as number }),
|
|
217
|
+
}
|
|
218
|
+
})
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* The body as text, refused past `max` bytes as they arrive: a chunked body
|
|
223
|
+
* names no length, and was read whole before it was measured.
|
|
224
|
+
*/
|
|
225
|
+
async function textUpTo(req: Request, max: number): Promise<string> {
|
|
226
|
+
const decoder = new TextDecoder()
|
|
227
|
+
let text = ''
|
|
228
|
+
let seen = 0
|
|
229
|
+
// Leaving the loop by a throw cancels the stream.
|
|
230
|
+
for await (const chunk of req.body ?? []) {
|
|
231
|
+
seen += chunk.byteLength
|
|
232
|
+
if (seen > max) throw new Refusal(413, `the body is at most ${max} bytes`)
|
|
233
|
+
text += decoder.decode(chunk, { stream: true })
|
|
234
|
+
}
|
|
235
|
+
return text + decoder.decode()
|
|
236
|
+
}
|
package/src/store.ts
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
// What the server keeps: one row per issuer, repository, ref and task
|
|
2
|
+
// definition, already the numbers a scheduler asks for (2026-10-25: a
|
|
3
|
+
// history, not a log). Each report folds into its row in one statement, so
|
|
4
|
+
// racing jobs lose nothing. Every column is plain (2026-10-27): the
|
|
5
|
+
// server is the user's own, and a hash only hid its rows from them.
|
|
6
|
+
|
|
7
|
+
/** A SQLite database, as bun:sqlite and D1 both serve one. */
|
|
8
|
+
export interface Db {
|
|
9
|
+
all<T>(sql: string, params: readonly unknown[]): Promise<T[]>
|
|
10
|
+
run(sql: string, params: readonly unknown[]): Promise<number>
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
/** One run of a task that succeeded, as a client reports it. */
|
|
14
|
+
export interface Report {
|
|
15
|
+
/** The task's definition hash, opaque to the server. */
|
|
16
|
+
readonly hash: string
|
|
17
|
+
readonly durationMs: number
|
|
18
|
+
/** CPU cores it kept busy on average. */
|
|
19
|
+
readonly cores?: number
|
|
20
|
+
/** Its peak memory. */
|
|
21
|
+
readonly rssBytes?: number
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/** What a scheduler reads for a task: the folded numbers and how many runs made them. */
|
|
25
|
+
export interface Timing {
|
|
26
|
+
readonly durationMs: number
|
|
27
|
+
readonly cores: number | null
|
|
28
|
+
readonly rssBytes: number | null
|
|
29
|
+
readonly runs: number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* The newest run's weight in a task's duration and cores: a median needs
|
|
34
|
+
* the samples, a plain mean never forgets; at 0.3 a task that truly got
|
|
35
|
+
* slower is followed within about three runs, and one outlier moves it 30%.
|
|
36
|
+
*/
|
|
37
|
+
export const WEIGHT = 0.3
|
|
38
|
+
/** Peak memory keeps the larger of the new peak and the old one decayed by this: a reservation needs the peak. */
|
|
39
|
+
export const RSS_DECAY = 0.9
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* A report within this share of the row (and no new memory peak) leaves
|
|
43
|
+
* it as it is, unless the row is a day old: the fold would move it under
|
|
44
|
+
* 1.5%, and each write is billed (D1's free plan: 100k rows a day). The
|
|
45
|
+
* day keeps a reported task's row past the sweep; `runs` counts the
|
|
46
|
+
* reports that moved it.
|
|
47
|
+
*/
|
|
48
|
+
const NOISE = 0.05
|
|
49
|
+
const DAY_MS = 86_400_000
|
|
50
|
+
|
|
51
|
+
/** A repository: its issuer's URL, no trailing slash, and its `org/name` there, lowercased. */
|
|
52
|
+
export interface Repo {
|
|
53
|
+
readonly issuer: string
|
|
54
|
+
readonly repository: string
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
// 2026-10-27: rows were keyed by hashes of these; nothing that table
|
|
58
|
+
// holds can be read by plain values.
|
|
59
|
+
const DROP_OLD = 'DROP TABLE IF EXISTS task_timings'
|
|
60
|
+
const SCHEMA = `CREATE TABLE IF NOT EXISTS timings (issuer TEXT NOT NULL, repository TEXT NOT NULL, ref TEXT NOT NULL, hash TEXT NOT NULL, duration_ms INTEGER NOT NULL, cores REAL, rss_bytes INTEGER, runs INTEGER NOT NULL, updated_at INTEGER NOT NULL, PRIMARY KEY (issuer, repository, ref, hash)) WITHOUT ROWID`
|
|
61
|
+
|
|
62
|
+
// When the database was last swept, by whichever node: a Worker runs many
|
|
63
|
+
// isolates and starts new ones all day, and each swept on its first upload.
|
|
64
|
+
const SWEEPS =
|
|
65
|
+
'CREATE TABLE IF NOT EXISTS sweeps (id INTEGER PRIMARY KEY CHECK (id = 1), at INTEGER NOT NULL)'
|
|
66
|
+
|
|
67
|
+
// One statement per 2 MB of a report's rows, passed as one JSON parameter:
|
|
68
|
+
// D1 binds at most 100 parameters, and a free plan's Worker runs at most
|
|
69
|
+
// 50 queries a request.
|
|
70
|
+
const FOLD = `INSERT INTO timings (issuer, repository, ref, hash, duration_ms, cores, rss_bytes, runs, updated_at)
|
|
71
|
+
SELECT ?1, ?2, ?3, j.value ->> '$.hash', j.value ->> '$.durationMs', j.value ->> '$.cores', j.value ->> '$.rssBytes', 1, ?4
|
|
72
|
+
FROM json_each(?5) AS j WHERE true
|
|
73
|
+
ON CONFLICT (issuer, repository, ref, hash) DO UPDATE SET
|
|
74
|
+
duration_ms = CAST(round(duration_ms * ${1 - WEIGHT} + excluded.duration_ms * ${WEIGHT}) AS INTEGER),
|
|
75
|
+
cores = CASE WHEN excluded.cores IS NULL THEN cores WHEN cores IS NULL THEN excluded.cores
|
|
76
|
+
ELSE cores * ${1 - WEIGHT} + excluded.cores * ${WEIGHT} END,
|
|
77
|
+
rss_bytes = CASE WHEN excluded.rss_bytes IS NULL THEN rss_bytes WHEN rss_bytes IS NULL THEN excluded.rss_bytes
|
|
78
|
+
ELSE max(excluded.rss_bytes, CAST(rss_bytes * ${RSS_DECAY} AS INTEGER)) END,
|
|
79
|
+
runs = runs + 1,
|
|
80
|
+
updated_at = excluded.updated_at
|
|
81
|
+
WHERE abs(excluded.duration_ms - duration_ms) > duration_ms * ${NOISE}
|
|
82
|
+
OR (excluded.cores IS NOT NULL AND (cores IS NULL OR abs(excluded.cores - cores) > cores * ${NOISE}))
|
|
83
|
+
OR (excluded.rss_bytes IS NOT NULL AND (rss_bytes IS NULL OR excluded.rss_bytes > rss_bytes))
|
|
84
|
+
OR excluded.updated_at - updated_at >= ${DAY_MS}`
|
|
85
|
+
|
|
86
|
+
/** The rows of `refs` for the hashes asked; a hash the first ref lacks reads the next. */
|
|
87
|
+
const READ = `SELECT hash, ref, duration_ms, cores, rss_bytes, runs FROM timings
|
|
88
|
+
WHERE issuer = ?1 AND repository = ?2 AND ref IN (SELECT value FROM json_each(?3)) AND hash IN (SELECT value FROM json_each(?4))`
|
|
89
|
+
|
|
90
|
+
export interface Store {
|
|
91
|
+
init(): Promise<void>
|
|
92
|
+
/** Folds each report into its row of `ref`. */
|
|
93
|
+
fold(repo: Repo, ref: string, now: number, reports: readonly Report[]): Promise<void>
|
|
94
|
+
/** Each hash's row in the first of `refs` that holds one. */
|
|
95
|
+
read(repo: Repo, refs: readonly string[], hashes: readonly string[]): Promise<Map<string, Timing>>
|
|
96
|
+
/** When a node last swept the database; 0 before the first sweep. */
|
|
97
|
+
sweptAt(): Promise<number>
|
|
98
|
+
/** Deletes the rows not reported since `before`, and records the sweep at `now`; how many. */
|
|
99
|
+
sweep(before: number, now: number): Promise<number>
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/**
|
|
103
|
+
* D1's limit on a bound string (developers.cloudflare.com/d1/platform/limits):
|
|
104
|
+
* a report's rows, or a query's hashes, past it failed the statement, so a
|
|
105
|
+
* server took less than its PROTOCOL.md limits name.
|
|
106
|
+
*/
|
|
107
|
+
const MAX_PARAM_BYTES = 2_000_000
|
|
108
|
+
|
|
109
|
+
/** `items` as JSON arrays each within `MAX_PARAM_BYTES`: a UTF-16 unit is at most 3 UTF-8 bytes. */
|
|
110
|
+
function jsonParams(items: readonly unknown[]): string[] {
|
|
111
|
+
const out: string[] = []
|
|
112
|
+
let part: string[] = []
|
|
113
|
+
let units = 2
|
|
114
|
+
for (const item of items) {
|
|
115
|
+
const json = JSON.stringify(item)
|
|
116
|
+
if (part.length > 0 && (units + json.length + 1) * 3 > MAX_PARAM_BYTES) {
|
|
117
|
+
out.push(`[${part.join(',')}]`)
|
|
118
|
+
part = []
|
|
119
|
+
units = 2
|
|
120
|
+
}
|
|
121
|
+
part.push(json)
|
|
122
|
+
units += json.length + 1
|
|
123
|
+
}
|
|
124
|
+
if (part.length > 0) out.push(`[${part.join(',')}]`)
|
|
125
|
+
return out
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function sqliteStore(db: Db): Store {
|
|
129
|
+
return {
|
|
130
|
+
async init() {
|
|
131
|
+
await db.run(DROP_OLD, [])
|
|
132
|
+
await db.run(SCHEMA, [])
|
|
133
|
+
await db.run(SWEEPS, [])
|
|
134
|
+
},
|
|
135
|
+
async fold(repo, ref, now, reports) {
|
|
136
|
+
for (const part of jsonParams(reports))
|
|
137
|
+
await db.run(FOLD, [repo.issuer, repo.repository, ref, now, part])
|
|
138
|
+
},
|
|
139
|
+
async read(repo, refs, hashes) {
|
|
140
|
+
const rows = (
|
|
141
|
+
await Promise.all(
|
|
142
|
+
jsonParams(hashes).map((part) =>
|
|
143
|
+
db.all<{
|
|
144
|
+
hash: string
|
|
145
|
+
ref: string
|
|
146
|
+
duration_ms: number
|
|
147
|
+
cores: number | null
|
|
148
|
+
rss_bytes: number | null
|
|
149
|
+
runs: number
|
|
150
|
+
}>(READ, [repo.issuer, repo.repository, JSON.stringify(refs), part]),
|
|
151
|
+
),
|
|
152
|
+
)
|
|
153
|
+
).flat()
|
|
154
|
+
// A hash's row in the earliest ref asked wins.
|
|
155
|
+
rows.sort((a, b) => refs.indexOf(b.ref) - refs.indexOf(a.ref))
|
|
156
|
+
return new Map(
|
|
157
|
+
rows.map((r) => [
|
|
158
|
+
r.hash,
|
|
159
|
+
{ durationMs: r.duration_ms, cores: r.cores, rssBytes: r.rss_bytes, runs: r.runs },
|
|
160
|
+
]),
|
|
161
|
+
)
|
|
162
|
+
},
|
|
163
|
+
async sweptAt() {
|
|
164
|
+
const [row] = await db.all<{ at: number }>('SELECT at FROM sweeps WHERE id = 1', [])
|
|
165
|
+
return row?.at ?? 0
|
|
166
|
+
},
|
|
167
|
+
async sweep(before, now) {
|
|
168
|
+
// A report that would not move its row is not written for up to a
|
|
169
|
+
// day (FOLD): a row's write time lags its last report by up to that.
|
|
170
|
+
const deleted = await db.run('DELETE FROM timings WHERE updated_at < ?1', [before - DAY_MS])
|
|
171
|
+
await db.run(
|
|
172
|
+
'INSERT INTO sweeps (id, at) VALUES (1, ?1) ON CONFLICT (id) DO UPDATE SET at = excluded.at',
|
|
173
|
+
[now],
|
|
174
|
+
)
|
|
175
|
+
return deleted
|
|
176
|
+
},
|
|
177
|
+
}
|
|
178
|
+
}
|
package/src/worker.ts
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// `vx-history-server` on Cloudflare Workers, the timings in D1.
|
|
2
|
+
// `wrangler.jsonc` deploys it; ALLOW, AUDIENCE, READ_ACCESS and TIMING_DAYS
|
|
3
|
+
// are vars in it.
|
|
4
|
+
|
|
5
|
+
import { errorJson, oidcTokenAuth, parseAllow } from '@vzn/vx-devtools'
|
|
6
|
+
import { timingDaysOf } from './days.js'
|
|
7
|
+
import { createHistoryServer } from './server.js'
|
|
8
|
+
import { type Db, sqliteStore } from './store.js'
|
|
9
|
+
|
|
10
|
+
/** The parts of the Workers runtime this file uses, so it needs no types package. */
|
|
11
|
+
interface D1Statement {
|
|
12
|
+
bind(...values: unknown[]): D1Statement
|
|
13
|
+
all<T>(): Promise<{ results: T[] }>
|
|
14
|
+
run(): Promise<{ meta: { changes: number } }>
|
|
15
|
+
}
|
|
16
|
+
export interface D1Database {
|
|
17
|
+
prepare(sql: string): D1Statement
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export interface Env {
|
|
21
|
+
readonly DB?: D1Database
|
|
22
|
+
/** Who may take part (`parseAllow`): `issuer:owner/repo:workflow` entries, space separated. */
|
|
23
|
+
readonly ALLOW?: string
|
|
24
|
+
/** The `aud` a CI token names; unset, the origin the Worker is reached at. */
|
|
25
|
+
readonly AUDIENCE?: string
|
|
26
|
+
/** `public`: a request with no token reads a repository ALLOW names. */
|
|
27
|
+
readonly READ_ACCESS?: string
|
|
28
|
+
/** Days a task's timings outlive its last report (default 30). */
|
|
29
|
+
readonly TIMING_DAYS?: string | number
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function d1Db(d1: D1Database): Db {
|
|
33
|
+
return {
|
|
34
|
+
all: async <T>(sql: string, params: readonly unknown[]) =>
|
|
35
|
+
(
|
|
36
|
+
await d1
|
|
37
|
+
.prepare(sql)
|
|
38
|
+
.bind(...params)
|
|
39
|
+
.all<T>()
|
|
40
|
+
).results,
|
|
41
|
+
run: async (sql, params) =>
|
|
42
|
+
(
|
|
43
|
+
await d1
|
|
44
|
+
.prepare(sql)
|
|
45
|
+
.bind(...params)
|
|
46
|
+
.run()
|
|
47
|
+
).meta.changes,
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/** One handler per isolate and settings: the schema is made once per isolate. */
|
|
52
|
+
let made: { key: string; handle: (req: Request) => Promise<Response> } | undefined
|
|
53
|
+
|
|
54
|
+
export default {
|
|
55
|
+
async fetch(req: Request, env: Env): Promise<Response> {
|
|
56
|
+
// A deploy's mistake: unbound, capabilities answered and every upload failed.
|
|
57
|
+
if (env.DB === undefined) {
|
|
58
|
+
return errorJson(500, 'DB: bind a D1 database as DB (wrangler.jsonc d1_databases)')
|
|
59
|
+
}
|
|
60
|
+
const key = [env.ALLOW, env.AUDIENCE, env.READ_ACCESS, env.TIMING_DAYS].join('\0')
|
|
61
|
+
if (made?.key !== key) {
|
|
62
|
+
let allow
|
|
63
|
+
// wrangler.jsonc may write a var as JSON: a list reached parseAllow.
|
|
64
|
+
if (env.ALLOW !== undefined && typeof env.ALLOW !== 'string') {
|
|
65
|
+
return errorJson(
|
|
66
|
+
500,
|
|
67
|
+
'ALLOW: one string of issuer:owner/repo:workflow entries, space separated',
|
|
68
|
+
)
|
|
69
|
+
}
|
|
70
|
+
// Likewise AUDIENCE: "env.AUDIENCE.trim is not a function".
|
|
71
|
+
if (env.AUDIENCE !== undefined && typeof (env.AUDIENCE as unknown) !== 'string') {
|
|
72
|
+
return errorJson(500, 'AUDIENCE: one string, the aud CI tokens name')
|
|
73
|
+
}
|
|
74
|
+
try {
|
|
75
|
+
allow = parseAllow(env.ALLOW ?? '')
|
|
76
|
+
} catch (err) {
|
|
77
|
+
return errorJson(500, `ALLOW: ${(err as Error).message}`)
|
|
78
|
+
}
|
|
79
|
+
let authorize
|
|
80
|
+
let timingDays
|
|
81
|
+
try {
|
|
82
|
+
timingDays = timingDaysOf(env.TIMING_DAYS)
|
|
83
|
+
authorize = oidcTokenAuth({
|
|
84
|
+
allow,
|
|
85
|
+
readAccess: env.READ_ACCESS,
|
|
86
|
+
// jose matches aud exactly: a pasted newline refused every
|
|
87
|
+
// token as the wrong audience.
|
|
88
|
+
...(env.AUDIENCE ? { audience: env.AUDIENCE.trim() } : {}),
|
|
89
|
+
})
|
|
90
|
+
} catch (err) {
|
|
91
|
+
return errorJson(500, (err as Error).message)
|
|
92
|
+
}
|
|
93
|
+
made = {
|
|
94
|
+
key,
|
|
95
|
+
handle: createHistoryServer({
|
|
96
|
+
store: sqliteStore(d1Db(env.DB)),
|
|
97
|
+
authorize,
|
|
98
|
+
...(timingDays === undefined ? {} : { timingDays }),
|
|
99
|
+
}),
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
return made.handle(req)
|
|
103
|
+
},
|
|
104
|
+
}
|
package/wrangler.jsonc
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
// vx-history-server on Cloudflare Workers (src/worker.ts): `bunx wrangler
|
|
2
|
+
// deploy` from this directory. The timings live in D1.
|
|
3
|
+
{
|
|
4
|
+
"name": "vx-history-server",
|
|
5
|
+
"main": "src/worker.ts",
|
|
6
|
+
"compatibility_date": "2026-10-01",
|
|
7
|
+
"workers_dev": true,
|
|
8
|
+
// ALLOW: who may take part, issuer:owner/repo:workflow entries
|
|
9
|
+
// separated by spaces (README.md § CI tokens); empty, every request is
|
|
10
|
+
// refused with "set ALLOW". AUDIENCE: the aud a
|
|
11
|
+
// CI token names, unset the Worker's own origin. READ_ACCESS: public
|
|
12
|
+
// lets a query with no token read a repository ALLOW names. TIMING_DAYS:
|
|
13
|
+
// days a task's timings outlive its last report (default 30), a string
|
|
14
|
+
// or a number.
|
|
15
|
+
"vars": { "ALLOW": "" },
|
|
16
|
+
"d1_databases": [{ "binding": "DB", "database_name": "vx-history" }],
|
|
17
|
+
}
|