@pithy-sh/leaderboard 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/docs/costs.md +102 -0
- package/package.json +52 -0
- package/pithy.manifest.json +36 -0
- package/src/board/registry.ts +94 -0
- package/src/capability.ts +93 -0
- package/src/cloudflare-test.d.ts +14 -0
- package/src/config/boardKey.ts +16 -0
- package/src/config/config.ts +261 -0
- package/src/data/boardRecord.ts +39 -0
- package/src/data/entry.ts +59 -0
- package/src/data/lock.ts +26 -0
- package/src/data/tables.ts +43 -0
- package/src/entry/store.ts +184 -0
- package/src/error/errors.ts +125 -0
- package/src/http/guard.ts +61 -0
- package/src/http/handlers.ts +221 -0
- package/src/http/routes.ts +202 -0
- package/src/http/schemas.ts +126 -0
- package/src/index.ts +37 -0
- package/src/migrations/0001_entries.ts +86 -0
- package/src/rank/lock.ts +75 -0
- package/src/rank/materialize.ts +206 -0
- package/src/rank/query.ts +208 -0
- package/src/rank/retryPolicy.ts +39 -0
- package/src/rank/segment.ts +37 -0
- package/src/rank/tiers.ts +29 -0
- package/src/rank/worker.entry.ts +133 -0
- package/src/rank/worker.ts +94 -0
- package/src/rank/wrangler.jsonc +53 -0
- package/src/retention/prune.ts +104 -0
- package/src/seeds/example.ts +72 -0
- package/src/session/bookmark.ts +68 -0
- package/src/version.generated.ts +16 -0
- package/src/window/schedule.ts +120 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { boundParameterBudget, MAX_BOUND_PARAMETERS } from "@pithy-sh/core/src/data/boundParameters";
|
|
5
|
+
import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
|
|
6
|
+
import { type Expression, type SqlBool, sql } from "kysely";
|
|
7
|
+
import type { LeaderboardBoard } from "../config/config";
|
|
8
|
+
import { LeaderboardEntry } from "../data/entry";
|
|
9
|
+
import { LEADERBOARD_ENTRIES_TABLE, type LeaderboardDatabase } from "../data/tables";
|
|
10
|
+
import { MAX_SEGMENT_SIZE, SEGMENT_FIXED_PARAMETERS } from "./segment";
|
|
11
|
+
import { classifyTier } from "./tiers";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Ranked reads: top-N, my-rank, and around-me.
|
|
15
|
+
*
|
|
16
|
+
* Two things shape every query here.
|
|
17
|
+
*
|
|
18
|
+
* **No window functions.** `RANK() OVER` and `ROW_NUMBER() OVER` are undocumented on D1 — Cloudflare's
|
|
19
|
+
* SQL reference neither supports nor denies them. They do execute under Miniflare, but Miniflare also
|
|
20
|
+
* rejects `sqlite_version()` with `not authorized to use function`, which proves D1 runs a function
|
|
21
|
+
* authorizer whose production allowlist is not visible from local. A local pass is therefore not evidence
|
|
22
|
+
* about production, so ranking is built on plain `COUNT(*)` and `ORDER BY` instead. See docs/costs.md.
|
|
23
|
+
*
|
|
24
|
+
* **The ordering is total.** Score, then earliest `achievedAt`, then `userId`. Because no two entries can
|
|
25
|
+
* tie, a rank is exactly "how many entries beat you, plus one" — so dense-vs-competition ranking never
|
|
26
|
+
* arises and neither is implemented. It also means a top-N page can number its own rows from the offset
|
|
27
|
+
* rather than asking the database to rank them.
|
|
28
|
+
*/
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The members a segment query may bind, or a refusal naming the cap.
|
|
32
|
+
*
|
|
33
|
+
* The HTTP boundary refuses an oversized segment already, which is why this was never a live defect —
|
|
34
|
+
* and is exactly why it is worth fixing anyway. `topEntries` and `rankOf` are exported: an adopter
|
|
35
|
+
* calling them directly got no such refusal, and a 120-friend segment reached D1 with 124 parameters.
|
|
36
|
+
* A rule enforced at one of several entrances is not enforced (#250).
|
|
37
|
+
*
|
|
38
|
+
* A refusal rather than a silent truncation: a segment is a *set of people*, and quietly ranking 80 of
|
|
39
|
+
* somebody's 120 friends would be a wrong answer presented as a right one.
|
|
40
|
+
*/
|
|
41
|
+
function segmentMembers(segment: readonly string[]): string[] {
|
|
42
|
+
if (segment.length > Math.min(MAX_SEGMENT_SIZE, boundParameterBudget(SEGMENT_FIXED_PARAMETERS))) {
|
|
43
|
+
throw new ValidationError({
|
|
44
|
+
message: `A segment is capped at ${MAX_SEGMENT_SIZE} players.`,
|
|
45
|
+
action: `Rank at most ${MAX_SEGMENT_SIZE} players at a time.`,
|
|
46
|
+
detail: `A segment of ${segment.length} exceeds the cap of ${MAX_SEGMENT_SIZE}; D1 accepts ${MAX_BOUND_PARAMETERS} bound parameters and a segment query spends ${SEGMENT_FIXED_PARAMETERS} of them before a single member.`,
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
return [...segment];
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
export interface RankedEntry {
|
|
53
|
+
userId: string;
|
|
54
|
+
score: number;
|
|
55
|
+
achievedAt: Date;
|
|
56
|
+
rank: number;
|
|
57
|
+
tier: string | null;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export interface ReadOptions {
|
|
61
|
+
/** Restrict the board to these players — a friends or cohort view over the same store, not a second board. */
|
|
62
|
+
segment?: readonly string[];
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/**
|
|
66
|
+
* "Strictly better than this entry", in the board's direction.
|
|
67
|
+
*
|
|
68
|
+
* Spelled out as an OR-of-ANDs rather than a row-value comparison (`(score, achieved_at) > (?, ?)`)
|
|
69
|
+
* because the sort directions are mixed — score descends while achievedAt ascends — and SQLite's
|
|
70
|
+
* row-value shortcut only applies when every key sorts the same way.
|
|
71
|
+
*/
|
|
72
|
+
function betterThan(board: LeaderboardBoard, score: number, achievedAt: number, userId: string): Expression<SqlBool> {
|
|
73
|
+
const scoreBeats = board.direction === "desc" ? sql<SqlBool>`score > ${score}` : sql<SqlBool>`score < ${score}`;
|
|
74
|
+
return sql<SqlBool>`(
|
|
75
|
+
${scoreBeats}
|
|
76
|
+
OR (score = ${score} AND achieved_at < ${achievedAt})
|
|
77
|
+
OR (score = ${score} AND achieved_at = ${achievedAt} AND user_id < ${userId})
|
|
78
|
+
)`;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function decorate(board: LeaderboardBoard, entry: LeaderboardEntry, rank: number): RankedEntry {
|
|
82
|
+
return {
|
|
83
|
+
userId: entry.userId,
|
|
84
|
+
score: entry.score,
|
|
85
|
+
achievedAt: entry.achievedAt,
|
|
86
|
+
rank,
|
|
87
|
+
tier: classifyTier(board.tiers, board.direction, entry.score),
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* A page of the board, best first. Ranks are numbered from the offset: the ordering is total, so row
|
|
93
|
+
* `n` of an offset page is rank `offset + n + 1` by construction — no count, no window function.
|
|
94
|
+
*/
|
|
95
|
+
export async function topEntries(
|
|
96
|
+
db: LeaderboardDatabase,
|
|
97
|
+
board: LeaderboardBoard,
|
|
98
|
+
windowKey: string,
|
|
99
|
+
limit: number,
|
|
100
|
+
offset = 0,
|
|
101
|
+
options: ReadOptions = {},
|
|
102
|
+
): Promise<RankedEntry[]> {
|
|
103
|
+
let query = db
|
|
104
|
+
.selectFrom(LEADERBOARD_ENTRIES_TABLE)
|
|
105
|
+
.selectAll()
|
|
106
|
+
.where("boardId", "=", board.key)
|
|
107
|
+
.where("windowKey", "=", windowKey)
|
|
108
|
+
// Only entries the player consented to show and a moderator has not hidden are ever ranked.
|
|
109
|
+
.where("visible", "=", 1)
|
|
110
|
+
.where("hidden", "=", 0);
|
|
111
|
+
if (options.segment) {
|
|
112
|
+
if (options.segment.length === 0) return [];
|
|
113
|
+
query = query.where("userId", "in", segmentMembers(options.segment));
|
|
114
|
+
}
|
|
115
|
+
const rows = await query
|
|
116
|
+
.orderBy("score", board.direction === "desc" ? "desc" : "asc")
|
|
117
|
+
.orderBy("achievedAt", "asc")
|
|
118
|
+
.orderBy("userId", "asc")
|
|
119
|
+
.limit(limit)
|
|
120
|
+
.offset(offset)
|
|
121
|
+
.execute();
|
|
122
|
+
return rows.map((row, index) => decorate(board, LeaderboardEntry.parse(row), offset + index + 1));
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
/**
|
|
126
|
+
* The player's own rank, or null if they have no visible entry.
|
|
127
|
+
*
|
|
128
|
+
* When `materialized` is set the stored rank column is read — one indexed point read, and the reason
|
|
129
|
+
* `rank: { materialize }` exists. Otherwise the rank is counted live: correct always, free under ~10k
|
|
130
|
+
* players, and O(rank position) in billed rows because D1 bills rows *scanned*, not returned. A stale
|
|
131
|
+
* materialized rank is deliberate; the player's own score beside it is always live.
|
|
132
|
+
*/
|
|
133
|
+
export async function rankOf(
|
|
134
|
+
db: LeaderboardDatabase,
|
|
135
|
+
board: LeaderboardBoard,
|
|
136
|
+
windowKey: string,
|
|
137
|
+
userId: string,
|
|
138
|
+
materialized: boolean,
|
|
139
|
+
options: ReadOptions = {},
|
|
140
|
+
): Promise<{ entry: LeaderboardEntry; rank: number | null; tier: string | null } | null> {
|
|
141
|
+
const row = await db
|
|
142
|
+
.selectFrom(LEADERBOARD_ENTRIES_TABLE)
|
|
143
|
+
.selectAll()
|
|
144
|
+
.where("boardId", "=", board.key)
|
|
145
|
+
.where("windowKey", "=", windowKey)
|
|
146
|
+
.where("userId", "=", userId)
|
|
147
|
+
.executeTakeFirst();
|
|
148
|
+
if (!row) return null;
|
|
149
|
+
const entry = LeaderboardEntry.parse(row);
|
|
150
|
+
const tier = classifyTier(board.tiers, board.direction, entry.score);
|
|
151
|
+
// A hidden or unconsented entry is not on the board, so it has no rank — but the player still gets
|
|
152
|
+
// their own score back, which is why this is a null rank rather than a 404.
|
|
153
|
+
if (!entry.visible || entry.hidden) return { entry, rank: null, tier };
|
|
154
|
+
// A segment view has no meaningful materialized rank: the stored column ranks the whole board.
|
|
155
|
+
if (materialized && !options.segment) return { entry, rank: entry.rank ?? null, tier };
|
|
156
|
+
if (options.segment?.length === 0) return { entry, rank: null, tier };
|
|
157
|
+
|
|
158
|
+
const counted = await rankCountQuery(db, board, windowKey, entry, options).executeTakeFirst();
|
|
159
|
+
return { entry, rank: Number(counted?.better ?? 0) + 1, tier };
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* The live-rank count: how many visible entries beat this one. Exported so `plan.workers.test.ts` can
|
|
164
|
+
* compile the real query and read D1's own `EXPLAIN QUERY PLAN` for it — a plan test that hand-wrote the
|
|
165
|
+
* SQL would only prove that the hand-written SQL is indexed.
|
|
166
|
+
*/
|
|
167
|
+
export function rankCountQuery(
|
|
168
|
+
db: LeaderboardDatabase,
|
|
169
|
+
board: LeaderboardBoard,
|
|
170
|
+
windowKey: string,
|
|
171
|
+
entry: Pick<LeaderboardEntry, "score" | "achievedAt" | "userId">,
|
|
172
|
+
options: ReadOptions = {},
|
|
173
|
+
) {
|
|
174
|
+
let query = db
|
|
175
|
+
.selectFrom(LEADERBOARD_ENTRIES_TABLE)
|
|
176
|
+
.select(({ fn }) => fn.countAll<number>().as("better"))
|
|
177
|
+
.where("boardId", "=", board.key)
|
|
178
|
+
.where("windowKey", "=", windowKey)
|
|
179
|
+
// Only entries the player consented to show and a moderator has not hidden are ever ranked.
|
|
180
|
+
.where("visible", "=", 1)
|
|
181
|
+
.where("hidden", "=", 0);
|
|
182
|
+
if (options.segment && options.segment.length > 0) {
|
|
183
|
+
query = query.where("userId", "in", segmentMembers(options.segment));
|
|
184
|
+
}
|
|
185
|
+
return query.where(betterThan(board, entry.score, entry.achievedAt.getTime(), entry.userId));
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
/**
|
|
189
|
+
* The slice of the board centered on a player: `radius` entries either side of them.
|
|
190
|
+
*
|
|
191
|
+
* Built on the total ordering — find the player's rank, then page the board around it — so it needs no
|
|
192
|
+
* second index and no window function. Rank is always counted live here even when the board is
|
|
193
|
+
* materialized: an "around me" page whose neighbors came from a stale rank column would show players
|
|
194
|
+
* who are no longer next to you.
|
|
195
|
+
*/
|
|
196
|
+
export async function entriesAround(
|
|
197
|
+
db: LeaderboardDatabase,
|
|
198
|
+
board: LeaderboardBoard,
|
|
199
|
+
windowKey: string,
|
|
200
|
+
userId: string,
|
|
201
|
+
radius: number,
|
|
202
|
+
options: ReadOptions = {},
|
|
203
|
+
): Promise<RankedEntry[]> {
|
|
204
|
+
const own = await rankOf(db, board, windowKey, userId, false, options);
|
|
205
|
+
if (!own || own.rank === null) return [];
|
|
206
|
+
const offset = Math.max(0, own.rank - radius - 1);
|
|
207
|
+
return topEntries(db, board, windowKey, radius * 2 + 1, offset, options);
|
|
208
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { WorkflowRetryPolicy } from "@pithy-sh/core/src/workflow/faults";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* **What the rank refresh retries, and what it refuses to.**
|
|
8
|
+
*
|
|
9
|
+
* The answer is short because the refresh's world is: every step it runs — the journalled context, the
|
|
10
|
+
* prune, and one keyset page of ranking per step — talks to D1 and to nothing else. There is no
|
|
11
|
+
* provider, no bucket, no model, no second account. So there is no `leaderboard/*` code this pass can
|
|
12
|
+
* usefully re-drive, and the record is empty on purpose (pithy-sh/pithy#348).
|
|
13
|
+
*
|
|
14
|
+
* **An empty record is a statement, not an omission.** Core still answers for D1 through `withD1Retry`'s
|
|
15
|
+
* vocabulary — busy, timed out, connection lost, storage reset, internal — so a database under
|
|
16
|
+
* contention is re-driven with the step's much longer backoff, and nothing about that is restated here:
|
|
17
|
+
* one D1 vocabulary, in core, or the two drift. What the empty record adds is the other half — that
|
|
18
|
+
* leaderboard has looked at its own codes and retries none of them.
|
|
19
|
+
*
|
|
20
|
+
* ## Terminal, and why
|
|
21
|
+
*
|
|
22
|
+
* - **`leaderboard/invalid_schedule`** — a board's window CRON will not parse. It is config, it is
|
|
23
|
+
* identical on the next attempt, and it wants the adopter to edit `pithy.config.ts`.
|
|
24
|
+
* - **`validation/invalid_input`** — a board whose keyset cursor or rank shape the materializer refuses.
|
|
25
|
+
* Deterministic in the row it read.
|
|
26
|
+
* - **`leaderboard/board_not_found`, `leaderboard/board_immutable`** and the rest of the submit-path
|
|
27
|
+
* codes. They belong to a request; a refresh that somehow raised one has found a bug, and a bug
|
|
28
|
+
* surfaces faster than it backs off.
|
|
29
|
+
*
|
|
30
|
+
* **The cron is the outer retry, and that is why terminal is cheap here.** A refresh fires on a
|
|
31
|
+
* schedule, takes an advisory lock, and re-ranks from the top; a run that fails releases its lock in a
|
|
32
|
+
* `finally` and the next fire does the whole job again. So a fault that stops one run costs one
|
|
33
|
+
* interval, where five platform attempts against an answer that cannot change cost the interval *and*
|
|
34
|
+
* hold the lock through it — which is the one thing that makes the next fire skip too.
|
|
35
|
+
*/
|
|
36
|
+
export const leaderboardWorkflowRetry: WorkflowRetryPolicy = {
|
|
37
|
+
capability: "leaderboard",
|
|
38
|
+
retryable: {},
|
|
39
|
+
};
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The segment cap, in the one module that has no reason to import anything (#430).
|
|
6
|
+
*
|
|
7
|
+
* `http/schemas.ts` states the same bound a caller is refused by, so it needs this number — and it used
|
|
8
|
+
* to reach it through `rank/query.ts`, which builds the SQL and therefore pulls Kysely, `kysely-d1` and
|
|
9
|
+
* `@cloudflare/workers-types` behind it. A request schema is a client's business: a management client
|
|
10
|
+
* building a call must be able to compile the shape it may send, in a browser, with no Worker types in
|
|
11
|
+
* reach. So the number moved and the query kept the query.
|
|
12
|
+
*
|
|
13
|
+
* **The relationship this number only means something against lives elsewhere, on purpose.**
|
|
14
|
+
* `boundParameterBudget` is in `@pithy-sh/core/src/data/boundParameters`, which imports `D1Database` for
|
|
15
|
+
* the guard beside it, so importing it here would put the data layer back under a browser program by a
|
|
16
|
+
* shorter route. `rank/query.workers.test.ts` asserts `MAX_SEGMENT_SIZE <=
|
|
17
|
+
* boundParameterBudget(SEGMENT_FIXED_PARAMETERS)` where the budget is already in scope, and that
|
|
18
|
+
* assertion is what ties these two constants to D1's ceiling. Moving them without it detaches the cap
|
|
19
|
+
* from the limit it exists for, which is #250 with no symptom until real data arrives.
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* What a segment query binds besides the members: 4 filters (boardId, windowKey, visible, hidden) plus
|
|
24
|
+
* the 6 of `betterThan` (score twice, achieved_at twice, score and userId again). `rankOf` within a
|
|
25
|
+
* segment is the tightest path, so its overhead is the one that sets the cap.
|
|
26
|
+
*/
|
|
27
|
+
export const SEGMENT_FIXED_PARAMETERS = 10;
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Cap on a segment's member count.
|
|
31
|
+
*
|
|
32
|
+
* `boundParameterBudget(SEGMENT_FIXED_PARAMETERS)` is 90 — the most D1 would take. 80 is deliberately
|
|
33
|
+
* inside it, so that adding one more filter to a segment query is a change to `rank/query.ts` rather
|
|
34
|
+
* than a change to what every caller may pass. `segmentMembers` asserts the relationship rather than
|
|
35
|
+
* trusting it, and `boundParameters.test.ts` in core owns the arithmetic itself.
|
|
36
|
+
*/
|
|
37
|
+
export const MAX_SEGMENT_SIZE = 80;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { LeaderboardTier, ScoreDirection } from "../config/config";
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Classify a score into one of the board's tiers, or null if it reaches none.
|
|
8
|
+
*
|
|
9
|
+
* Tiers are a read-side classification over the score already stored — no tier column, no write-side
|
|
10
|
+
* cost, and no second board type. Config validates that tiers are listed worst to best in the board's
|
|
11
|
+
* direction, so the last one the score qualifies for is the best one it qualifies for.
|
|
12
|
+
*
|
|
13
|
+
* This is not the bucketed-cohort model (Duolingo leagues, Clash Royale arenas), which needs durable
|
|
14
|
+
* per-window cohort assignment and provably cannot be a view over a global board. That is deferred.
|
|
15
|
+
*/
|
|
16
|
+
export function classifyTier(
|
|
17
|
+
tiers: readonly LeaderboardTier[] | undefined,
|
|
18
|
+
direction: ScoreDirection,
|
|
19
|
+
score: number,
|
|
20
|
+
): string | null {
|
|
21
|
+
if (!tiers || tiers.length === 0) return null;
|
|
22
|
+
let reached: string | null = null;
|
|
23
|
+
for (const tier of tiers) {
|
|
24
|
+
const qualifies = direction === "desc" ? score >= tier.from : score <= tier.from;
|
|
25
|
+
if (!qualifies) break;
|
|
26
|
+
reached = tier.key;
|
|
27
|
+
}
|
|
28
|
+
return reached;
|
|
29
|
+
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { WorkflowEntrypoint, type WorkflowEvent, type WorkflowStep } from "cloudflare:workers";
|
|
5
|
+
import { NonRetryableError } from "cloudflare:workflows";
|
|
6
|
+
import type { D1Database } from "@cloudflare/workers-types";
|
|
7
|
+
import { classifiedSteps } from "@pithy-sh/core/src/workflow/faults";
|
|
8
|
+
import { LeaderboardConfig } from "../config/config";
|
|
9
|
+
import { leaderboardDatabase } from "../data/tables";
|
|
10
|
+
import { pruneBoards } from "../retention/prune";
|
|
11
|
+
import { windowKeyAt } from "../window/schedule";
|
|
12
|
+
import { acquireRefreshLock, releaseRefreshLock } from "./lock";
|
|
13
|
+
import { type Keyset, REFRESH_BATCH_CHUNKS, type RefreshResult, refreshWindowRanks } from "./materialize";
|
|
14
|
+
import { leaderboardWorkflowRetry } from "./retryPolicy";
|
|
15
|
+
import { materializedBoards } from "./worker";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* The leaderboard rank refresh, as a cron-triggered Cloudflare Workflow.
|
|
19
|
+
*
|
|
20
|
+
* Why a Workflow rather than a plain `scheduled()` pass: a Worker invocation is wall-clock bounded, so
|
|
21
|
+
* an earlier design capped each run at ~64k entries and re-ranked from the top every tick — a board
|
|
22
|
+
* bigger than that never fully ranked. A Workflow step has unlimited wall-clock and does not burn CPU
|
|
23
|
+
* while awaiting D1 (all our work is D1 I/O), so a board of any size ranks across a series of bounded,
|
|
24
|
+
* individually-durable steps. Each step checkpoints the keyset cursor; a crash resumes from the last
|
|
25
|
+
* completed step instead of restarting. The 64k ceiling is gone.
|
|
26
|
+
*
|
|
27
|
+
* At-most-one at a time: a refresh takes a D1 advisory lock before it starts and releases it when done.
|
|
28
|
+
* If a cron fires again while a refresh is still running (a cron faster than the refresh takes), the
|
|
29
|
+
* second instance cannot take the lock and skips — so two passes never interleave their chunked writes
|
|
30
|
+
* into an incoherent rank set. A crashed instance's lock ages out (see `lock.ts`) so the next fire
|
|
31
|
+
* reclaims it.
|
|
32
|
+
*
|
|
33
|
+
* Why it does NOT require a dedicated worker: this is a Workflow class plus a `scheduled()` handler plus
|
|
34
|
+
* one cron trigger. `pithy add leaderboard` deploys it as its own small worker by default (the template
|
|
35
|
+
* in `wrangler.jsonc`), but the same three pieces can be merged into the adopter's app worker — the
|
|
36
|
+
* capability contributes the Workflow through its manifest. A cron trigger is the one hard requirement;
|
|
37
|
+
* a separate worker is not.
|
|
38
|
+
*
|
|
39
|
+
* Cost: on Workers Paid this stays inside the free allowances at any realistic cadence — a run is a
|
|
40
|
+
* handful of steps (one prune + a few per board), storage is zero (all state is D1), and idle/awaiting
|
|
41
|
+
* steps incur no CPU. Even firing every minute is well under the 500k-steps/month and 10M-requests/month
|
|
42
|
+
* included tiers.
|
|
43
|
+
*/
|
|
44
|
+
|
|
45
|
+
interface RankWorkerEnv {
|
|
46
|
+
DB: D1Database;
|
|
47
|
+
/** The resolved leaderboard config, as JSON. The board set is config; this worker needs the same one. */
|
|
48
|
+
LEADERBOARD_CONFIG: string;
|
|
49
|
+
/** Optional override for how long a held refresh lock stays valid before it is reclaimed (see lock.ts). */
|
|
50
|
+
LEADERBOARD_LOCK_STALE_MS?: string;
|
|
51
|
+
/** The Workflow binding — this worker's own class, used by `scheduled()` to start an instance. */
|
|
52
|
+
RANK_REFRESH: { create(options?: { id?: string }): Promise<unknown> };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export class RankRefreshWorkflow extends WorkflowEntrypoint<RankWorkerEnv, unknown> {
|
|
56
|
+
override async run(_event: WorkflowEvent<unknown>, step: WorkflowStep): Promise<void> {
|
|
57
|
+
// Every step below runs under `leaderboardWorkflowRetry`. Its record is empty and that is the whole
|
|
58
|
+
// statement: this refresh talks to D1 and to nothing else, core answers for D1's transient
|
|
59
|
+
// vocabulary, and leaderboard retries none of its own codes. See `retryPolicy.ts` — and note that
|
|
60
|
+
// terminal is cheap here precisely because the lock is released in the `finally` below, so the next
|
|
61
|
+
// cron fire does the whole job rather than skipping on a lock a doomed run was still holding.
|
|
62
|
+
const steps = classifiedSteps(step, leaderboardWorkflowRetry, NonRetryableError);
|
|
63
|
+
const config = LeaderboardConfig.parse(JSON.parse(this.env.LEADERBOARD_CONFIG));
|
|
64
|
+
const db = leaderboardDatabase(this.env.DB);
|
|
65
|
+
const staleMs = this.env.LEADERBOARD_LOCK_STALE_MS ? Number(this.env.LEADERBOARD_LOCK_STALE_MS) : undefined;
|
|
66
|
+
|
|
67
|
+
// Mint this instance's identity and its clock ONCE, in a memoized step, and read them from the step's
|
|
68
|
+
// return value. A replay does not re-run the step body, so it reuses the same holder token and the
|
|
69
|
+
// same `now` — recomputing `now` could cross a window boundary mid-refresh, and a fresh token would
|
|
70
|
+
// orphan the lock.
|
|
71
|
+
const ctx: { holder: string; nowMs: number } = await steps.do("refresh-context", async () => ({
|
|
72
|
+
holder: crypto.randomUUID(),
|
|
73
|
+
nowMs: Date.now(),
|
|
74
|
+
}));
|
|
75
|
+
const now = new Date(ctx.nowMs);
|
|
76
|
+
|
|
77
|
+
// Acquire and release run OUTSIDE steps, deliberately — the one place this file does side effects
|
|
78
|
+
// outside a step. Both are idempotent D1 ops, and re-running them on every replay is exactly what
|
|
79
|
+
// keeps the lock state honest: acquire re-validates against live D1 (an instance whose stale lock was
|
|
80
|
+
// reclaimed while it was interrupted re-checks, sees the lock is no longer its own, and stands down,
|
|
81
|
+
// instead of resuming on a memoized "acquired" and writing concurrently); release re-converges the
|
|
82
|
+
// same way. A memoized acquire/release step would instead freeze a stale decision across the replay.
|
|
83
|
+
if (!(await acquireRefreshLock(db, ctx.holder, now, staleMs))) return;
|
|
84
|
+
|
|
85
|
+
try {
|
|
86
|
+
// Prune first, in its own durable step, so the refresh never ranks rows about to be deleted.
|
|
87
|
+
await steps.do("prune", () => pruneBoards(db, config.boards, now));
|
|
88
|
+
|
|
89
|
+
for (const board of materializedBoards(config)) {
|
|
90
|
+
// Only the open window is refreshed; a closed window's ranks were finalized while it was open.
|
|
91
|
+
const window = windowKeyAt(board.window, now);
|
|
92
|
+
let cursor: Keyset | null = null;
|
|
93
|
+
let startRank = 0;
|
|
94
|
+
let batch = 0;
|
|
95
|
+
// One step per batch of ~64k entries, threading the cursor. A board of any size completes across
|
|
96
|
+
// however many steps it needs; each is independently retried by the Workflow runtime.
|
|
97
|
+
for (;;) {
|
|
98
|
+
const from: Keyset | undefined = cursor ?? undefined;
|
|
99
|
+
const result: RefreshResult = await steps.do(`refresh:${board.key}:${batch}`, () =>
|
|
100
|
+
refreshWindowRanks(db, board, window, {
|
|
101
|
+
maxChunks: REFRESH_BATCH_CHUNKS,
|
|
102
|
+
resumeAfter: from,
|
|
103
|
+
startRank,
|
|
104
|
+
}),
|
|
105
|
+
);
|
|
106
|
+
if (result.complete) break;
|
|
107
|
+
cursor = result.cursor;
|
|
108
|
+
startRank = result.ranked;
|
|
109
|
+
batch += 1;
|
|
110
|
+
// A batch that stopped without a cursor cannot resume — bail rather than loop forever.
|
|
111
|
+
if (!cursor) break;
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
} finally {
|
|
115
|
+
// Release even if a step threw, so a failure does not wedge the lock until it ages out. Outside a
|
|
116
|
+
// step (see the acquire note) so a replay re-releases rather than skipping a memoized release.
|
|
117
|
+
await releaseRefreshLock(db, ctx.holder);
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
export default {
|
|
123
|
+
/**
|
|
124
|
+
* Cron entry: start one rank-refresh Workflow instance per fire. Each instance re-ranks from the top;
|
|
125
|
+
* the checkpointing is within an instance (resume on crash), not across fires (a fire is a fresh full
|
|
126
|
+
* refresh). If a fire lands while a previous refresh is still running, the new instance takes no lock
|
|
127
|
+
* and exits immediately, so overlapping instances never write concurrently — see the lock acquisition
|
|
128
|
+
* in `run()` above.
|
|
129
|
+
*/
|
|
130
|
+
async scheduled(_controller: unknown, env: RankWorkerEnv): Promise<void> {
|
|
131
|
+
await env.RANK_REFRESH.create();
|
|
132
|
+
},
|
|
133
|
+
};
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { D1Database } from "@cloudflare/workers-types";
|
|
5
|
+
import type { LeaderboardBoard, LeaderboardConfig } from "../config/config";
|
|
6
|
+
import { materializeSchedule } from "../config/config";
|
|
7
|
+
import { leaderboardDatabase } from "../data/tables";
|
|
8
|
+
import { type PruneOutcome, pruneBoards } from "../retention/prune";
|
|
9
|
+
import { windowKeyAt } from "../window/schedule";
|
|
10
|
+
import { type RefreshResult, refreshWindowRanks } from "./materialize";
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The rank pass, in process: the retention sweep, then the rank refresh, each board ranked to completion.
|
|
14
|
+
*
|
|
15
|
+
* This is the reusable core the {@link RankRefreshWorkflow} drives step by step, and a standalone entry
|
|
16
|
+
* point for a caller who wants the whole pass in one call (tests, a simple scheduled handler). It runs
|
|
17
|
+
* every board to completion — there is no per-invocation chunk cap, so no board-size ceiling. The
|
|
18
|
+
* Workflow adds durability and per-board checkpointing on top of these same primitives; it does not need
|
|
19
|
+
* a different pass.
|
|
20
|
+
*
|
|
21
|
+
* Order matters. Pruning first means the refresh never spends a chunk ranking rows about to be deleted,
|
|
22
|
+
* and a board whose retention just dropped a window does not briefly publish ranks for it.
|
|
23
|
+
*/
|
|
24
|
+
/**
|
|
25
|
+
* One board's contribution to a rank pass: what the refresh did, or that it could not be done (#371).
|
|
26
|
+
*
|
|
27
|
+
* The state rides on the value. A board that threw is not a board that ranked nobody — `ranked: 0` is a
|
|
28
|
+
* real answer about an empty window — so the numbers live behind `refreshed` and a consumer reaches them
|
|
29
|
+
* only by narrowing. Forgetting the sick board is a type error rather than a zero on a dashboard.
|
|
30
|
+
*/
|
|
31
|
+
export type BoardRefresh =
|
|
32
|
+
| ({
|
|
33
|
+
/** This board's open window was ranked. */
|
|
34
|
+
state: "refreshed";
|
|
35
|
+
/** The board's key, as its config declares it. */
|
|
36
|
+
board: string;
|
|
37
|
+
/** The window key that was ranked. */
|
|
38
|
+
window: string;
|
|
39
|
+
} & RefreshResult)
|
|
40
|
+
| {
|
|
41
|
+
/** This board's refresh threw. Its ranks are whatever the previous pass left. */
|
|
42
|
+
state: "unavailable";
|
|
43
|
+
/** The board's key, as its config declares it. */
|
|
44
|
+
board: string;
|
|
45
|
+
/** The window key the pass was attempting. */
|
|
46
|
+
window: string;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
export interface RankPassResult {
|
|
50
|
+
/** What the retention sweep deleted, and whether it reached every board. */
|
|
51
|
+
pruned: PruneOutcome;
|
|
52
|
+
/** One entry per materialized board the refresh pass touched. Empty when rank is live. */
|
|
53
|
+
refreshed: BoardRefresh[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** The boards a refresh pass must rank: only materialized ones. Live boards compute rank per request. */
|
|
57
|
+
export function materializedBoards(config: LeaderboardConfig): LeaderboardBoard[] {
|
|
58
|
+
return materializeSchedule(config) === undefined ? [] : config.boards;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/**
|
|
62
|
+
* **Every contributor here is degraded, and none of them is load-bearing (#371).**
|
|
63
|
+
*
|
|
64
|
+
* The sweep runs first so the refresh never spends a chunk ranking rows about to be deleted — an
|
|
65
|
+
* efficiency, not a precondition. A board whose prune throws keeps its old rows a while longer and ranks
|
|
66
|
+
* correctly regardless, so the sweep failing is no reason to leave every board's ranks stale. And boards
|
|
67
|
+
* are independent of each other by construction: one board's entries, windows and ranks are its own, so a
|
|
68
|
+
* board that will not rank has no claim on any other board's pass.
|
|
69
|
+
*
|
|
70
|
+
* So one sick board costs its own line. What it must never do is read as a board that ranked nobody.
|
|
71
|
+
*/
|
|
72
|
+
export async function runRankPass(d1: D1Database, config: LeaderboardConfig, now: Date): Promise<RankPassResult> {
|
|
73
|
+
const db = leaderboardDatabase(d1);
|
|
74
|
+
const pruned = await pruneBoards(db, config.boards, now);
|
|
75
|
+
|
|
76
|
+
const refreshed: BoardRefresh[] = [];
|
|
77
|
+
for (const board of materializedBoards(config)) {
|
|
78
|
+
// Only the open window is refreshed. A closed window's ranks stopped moving when it closed, so the
|
|
79
|
+
// pass that ran while it was open already left them final.
|
|
80
|
+
const window = windowKeyAt(board.window, now);
|
|
81
|
+
let result: RefreshResult;
|
|
82
|
+
try {
|
|
83
|
+
result = await refreshWindowRanks(db, board, window);
|
|
84
|
+
} catch {
|
|
85
|
+
// No binding. A refresh throws out of D1 with a query and an entry's identifiers in it, and this
|
|
86
|
+
// result is rendered wherever a caller renders it — so what survives is the board key and the
|
|
87
|
+
// window, both of which are the adopter's own configuration.
|
|
88
|
+
refreshed.push({ state: "unavailable", board: board.key, window });
|
|
89
|
+
continue;
|
|
90
|
+
}
|
|
91
|
+
refreshed.push({ state: "refreshed", board: board.key, window, ...result });
|
|
92
|
+
}
|
|
93
|
+
return { pruned, refreshed };
|
|
94
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
{
|
|
2
|
+
// The leaderboard rank-refresh worker. This is a TEMPLATE, not a wrangler env-stanza file: `pithy add
|
|
3
|
+
// leaderboard` resolves it per environment — filling the `<...>` placeholders and the cron from the
|
|
4
|
+
// adopter's `rank: { materialize }` setting — and deploys each with its own config
|
|
5
|
+
// (`wrangler deploy --config <resolved>`). The adopter authors none of it.
|
|
6
|
+
//
|
|
7
|
+
// It does NOT have to be its own worker. What the refresh actually needs is three things: the
|
|
8
|
+
// `RankRefreshWorkflow` class, a `scheduled()` handler that starts an instance, and a cron trigger.
|
|
9
|
+
// This template deploys them as a small dedicated worker by default — simplest, and isolated from the
|
|
10
|
+
// request path — but the same three pieces can be folded into the adopter's app worker instead. A cron
|
|
11
|
+
// trigger is the one hard requirement; a separate worker is not.
|
|
12
|
+
//
|
|
13
|
+
// Why a Workflow rather than a plain scheduled pass: a Worker invocation is wall-clock bounded, which
|
|
14
|
+
// once capped the refresh at ~64k entries per run. A Workflow step has unlimited wall-clock and burns
|
|
15
|
+
// no CPU while awaiting D1, so a board of any size ranks across bounded, individually-durable steps
|
|
16
|
+
// that checkpoint a keyset cursor. See `worker.entry.ts`.
|
|
17
|
+
"name": "pithy-leaderboard-rank", // resolved per env → pithy-leaderboard-rank-staging / -production
|
|
18
|
+
"main": "./worker.entry.ts",
|
|
19
|
+
// The compatibility date every Worker in this repository runs on. Stated once in the repository
|
|
20
|
+
// root's `compatibility.ts` and copied here because JSONC cannot import it —
|
|
21
|
+
// `cli/src/ci/compatibilityDates.test.ts` fails on any Worker older than it.
|
|
22
|
+
"compatibility_date": "2026-06-01",
|
|
23
|
+
"compatibility_flags": ["nodejs_compat"],
|
|
24
|
+
|
|
25
|
+
// No public URL. This worker has no HTTP routes — it is reached only by its cron. `workers_dev: false`
|
|
26
|
+
// keeps it off workers.dev so nothing accidentally exposes a ranking rewrite to the internet.
|
|
27
|
+
"workers_dev": false,
|
|
28
|
+
|
|
29
|
+
// The app database. The same `DB` the routes write to — the rank column lives on the entries table, so
|
|
30
|
+
// there is no second store to keep in step.
|
|
31
|
+
"d1_databases": [
|
|
32
|
+
{ "binding": "DB", "database_name": "<filled-at-provision>", "database_id": "<filled-at-provision>" }
|
|
33
|
+
],
|
|
34
|
+
|
|
35
|
+
// The refresh Workflow this worker hosts and its own scheduled() handler starts.
|
|
36
|
+
"workflows": [{ "binding": "RANK_REFRESH", "name": "pithy-leaderboard-rank", "class_name": "RankRefreshWorkflow" }],
|
|
37
|
+
|
|
38
|
+
// The refresh cron. Resolved to the adopter's `rank: { materialize: <cron> }` schedule when set;
|
|
39
|
+
// otherwise hourly, which is only ever the retention sweep (one bounded DELETE per board that
|
|
40
|
+
// configures a limit — most keep everything, the default, and delete nothing).
|
|
41
|
+
"triggers": { "crons": ["<filled-at-provision>"] },
|
|
42
|
+
|
|
43
|
+
"vars": {
|
|
44
|
+
// The resolved leaderboard config, as JSON. The Workflow parses it back through `LeaderboardConfig`
|
|
45
|
+
// on each run — the board set is config, and this worker needs the same board set the routes have.
|
|
46
|
+
"LEADERBOARD_CONFIG": "<filled-at-provision>",
|
|
47
|
+
// Optional. How long a held refresh lock stays valid before a crashed instance's lock is reclaimed,
|
|
48
|
+
// in ms. Defaults to one hour — raise it only if a deployment refreshes boards that take longer than
|
|
49
|
+
// that (well past the ~1M-player shard boundary). See `rank/lock.ts`.
|
|
50
|
+
// "LEADERBOARD_LOCK_STALE_MS": "3600000",
|
|
51
|
+
"ENVIRONMENT": "<filled-at-provision>"
|
|
52
|
+
}
|
|
53
|
+
}
|