@pithy-sh/leaderboard 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +17 -0
- package/docs/costs.md +102 -0
- package/package.json +52 -0
- package/pithy.manifest.json +36 -0
- package/src/board/registry.ts +94 -0
- package/src/capability.ts +93 -0
- package/src/cloudflare-test.d.ts +14 -0
- package/src/config/boardKey.ts +16 -0
- package/src/config/config.ts +261 -0
- package/src/data/boardRecord.ts +39 -0
- package/src/data/entry.ts +59 -0
- package/src/data/lock.ts +26 -0
- package/src/data/tables.ts +43 -0
- package/src/entry/store.ts +184 -0
- package/src/error/errors.ts +125 -0
- package/src/http/guard.ts +61 -0
- package/src/http/handlers.ts +221 -0
- package/src/http/routes.ts +202 -0
- package/src/http/schemas.ts +126 -0
- package/src/index.ts +37 -0
- package/src/migrations/0001_entries.ts +86 -0
- package/src/rank/lock.ts +75 -0
- package/src/rank/materialize.ts +206 -0
- package/src/rank/query.ts +208 -0
- package/src/rank/retryPolicy.ts +39 -0
- package/src/rank/segment.ts +37 -0
- package/src/rank/tiers.ts +29 -0
- package/src/rank/worker.entry.ts +133 -0
- package/src/rank/worker.ts +94 -0
- package/src/rank/wrangler.jsonc +53 -0
- package/src/retention/prune.ts +104 -0
- package/src/seeds/example.ts +72 -0
- package/src/session/bookmark.ts +68 -0
- package/src/version.generated.ts +16 -0
- package/src/window/schedule.ts +120 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { z } from "zod";
|
|
5
|
+
import { BOARD_KEY_PATTERN } from "../config/boardKey";
|
|
6
|
+
import { MAX_SEGMENT_SIZE } from "../rank/segment";
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* The HTTP boundary shapes. Everything a client can send is parsed through one of these before it
|
|
10
|
+
* reaches a handler — declared on the route line with `zValidator(target, Schema, validationHook)`, so
|
|
11
|
+
* reading a route tells you what it takes.
|
|
12
|
+
*
|
|
13
|
+
* Note what is absent from {@link SubmitScoreBody}: `userId`, `achievedAt`, and `rank`. The player comes
|
|
14
|
+
* from the AuthContext seam, the clock comes from the server, and the rank is derived — a client that
|
|
15
|
+
* could set any of them could score as someone else, backdate its way past the tiebreak, or simply
|
|
16
|
+
* declare itself first. Server-authoritative is not only about who may call submit; it is about which
|
|
17
|
+
* fields a caller may name at all.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** A player id is opaque to us — it comes from the adopter's auth provider — so it is bounded, not shaped. */
|
|
21
|
+
const MAX_USER_ID_LENGTH = 256;
|
|
22
|
+
|
|
23
|
+
export const BoardParam = z
|
|
24
|
+
.object({
|
|
25
|
+
board: z
|
|
26
|
+
.string()
|
|
27
|
+
.min(1)
|
|
28
|
+
.max(64)
|
|
29
|
+
.regex(BOARD_KEY_PATTERN, "A board key is lowercase, digits, and dashes — it is a URL path segment.")
|
|
30
|
+
.describe(
|
|
31
|
+
"Which board the route addresses. A shape check only: the same pattern config already enforces on every board key, so nothing that resolves today is rejected. Whether the key is *configured* stays the handler's 404.",
|
|
32
|
+
),
|
|
33
|
+
})
|
|
34
|
+
.describe("The board a `/leaderboard/:board` route addresses.");
|
|
35
|
+
export type BoardParam = z.infer<typeof BoardParam>;
|
|
36
|
+
|
|
37
|
+
export const EntryParam = BoardParam.extend({
|
|
38
|
+
userId: z
|
|
39
|
+
.string()
|
|
40
|
+
.min(1)
|
|
41
|
+
.max(MAX_USER_ID_LENGTH)
|
|
42
|
+
.describe(
|
|
43
|
+
"The player whose entry a moderation route targets. Bounded in length, not in charset — player ids are minted by your auth provider, so we cap what reaches the store rather than guess its format.",
|
|
44
|
+
),
|
|
45
|
+
}).describe("The board and player a moderation route addresses.");
|
|
46
|
+
export type EntryParam = z.infer<typeof EntryParam>;
|
|
47
|
+
|
|
48
|
+
export const SubmitScoreBody = z
|
|
49
|
+
.object({
|
|
50
|
+
score: z
|
|
51
|
+
.number()
|
|
52
|
+
.finite()
|
|
53
|
+
.describe("The score to submit. Folded into the player's entry by the board's aggregation."),
|
|
54
|
+
})
|
|
55
|
+
.describe("A score submission. The player, the window, and the timestamp are all server-derived.");
|
|
56
|
+
export type SubmitScoreBody = z.infer<typeof SubmitScoreBody>;
|
|
57
|
+
|
|
58
|
+
export const WindowQuery = z
|
|
59
|
+
.object({
|
|
60
|
+
window: z
|
|
61
|
+
.string()
|
|
62
|
+
.optional()
|
|
63
|
+
.describe(
|
|
64
|
+
"Which window to read — the ISO key of a closed window, or omit for the one open now. Reading your own history is the point: these windows live in your D1 for as long as `retain` says, which no platform SDK offers.",
|
|
65
|
+
),
|
|
66
|
+
})
|
|
67
|
+
.describe("Selects the window a read applies to.");
|
|
68
|
+
export type WindowQuery = z.infer<typeof WindowQuery>;
|
|
69
|
+
|
|
70
|
+
export const TopQuery = WindowQuery.extend({
|
|
71
|
+
limit: z.coerce
|
|
72
|
+
.number()
|
|
73
|
+
.int()
|
|
74
|
+
.min(1)
|
|
75
|
+
.max(100)
|
|
76
|
+
.default(20)
|
|
77
|
+
.describe("How many entries to return, capped at 100 — a page, not the whole board."),
|
|
78
|
+
offset: z.coerce.number().int().min(0).default(0).describe("How many entries to skip. Ranks number from here."),
|
|
79
|
+
}).describe("A page of a board, best first.");
|
|
80
|
+
export type TopQuery = z.infer<typeof TopQuery>;
|
|
81
|
+
|
|
82
|
+
export const AroundQuery = WindowQuery.extend({
|
|
83
|
+
radius: z.coerce
|
|
84
|
+
.number()
|
|
85
|
+
.int()
|
|
86
|
+
.min(1)
|
|
87
|
+
.max(25)
|
|
88
|
+
.default(5)
|
|
89
|
+
.describe("How many entries to return either side of the player."),
|
|
90
|
+
}).describe("The slice of a board centered on the calling player.");
|
|
91
|
+
export type AroundQuery = z.infer<typeof AroundQuery>;
|
|
92
|
+
|
|
93
|
+
export const SegmentBody = z
|
|
94
|
+
.object({
|
|
95
|
+
userIds: z
|
|
96
|
+
.array(z.string().min(1))
|
|
97
|
+
.min(1)
|
|
98
|
+
.max(
|
|
99
|
+
MAX_SEGMENT_SIZE,
|
|
100
|
+
`A segment is capped at ${MAX_SEGMENT_SIZE} players: D1 allows 100 bound parameters per query and each member costs one.`,
|
|
101
|
+
)
|
|
102
|
+
.describe(
|
|
103
|
+
"The players to rank among — a friends list or any cohort. A collection dimension over the same store, not a second board.",
|
|
104
|
+
),
|
|
105
|
+
limit: z.number().int().min(1).max(100).default(20).describe("How many entries to return."),
|
|
106
|
+
offset: z.number().int().min(0).default(0).describe("How many entries to skip."),
|
|
107
|
+
window: z.string().optional().describe("Which window to read; omit for the one open now."),
|
|
108
|
+
})
|
|
109
|
+
.describe("A friends or cohort view of a board.");
|
|
110
|
+
export type SegmentBody = z.infer<typeof SegmentBody>;
|
|
111
|
+
|
|
112
|
+
export const VisibilityBody = z
|
|
113
|
+
.object({
|
|
114
|
+
visible: z
|
|
115
|
+
.boolean()
|
|
116
|
+
.describe("Whether this player consents to appear on the board. Gates every read, including segments."),
|
|
117
|
+
})
|
|
118
|
+
.describe("A player's own consent to be shown. Theirs to set — a submission never resets it.");
|
|
119
|
+
export type VisibilityBody = z.infer<typeof VisibilityBody>;
|
|
120
|
+
|
|
121
|
+
export const HideBody = z
|
|
122
|
+
.object({
|
|
123
|
+
hidden: z.boolean().describe("Whether to hide this entry from every read. The score is kept, not deleted."),
|
|
124
|
+
})
|
|
125
|
+
.describe("A moderator's hide toggle. Separate from player consent so a player cannot undo it.");
|
|
126
|
+
export type HideBody = z.infer<typeof HideBody>;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* The package entrypoint — the surface `pithy add leaderboard` wires into `pithy.config.ts`. Deliberately
|
|
6
|
+
* narrow: the capability factory, its config and options types, and the read shapes an app renders. Every
|
|
7
|
+
* other module is imported by deep path (`@pithy-sh/leaderboard/src/...`); this is the documented
|
|
8
|
+
* contract, not a barrel over the package.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
export {
|
|
12
|
+
isLeaderboardCapability,
|
|
13
|
+
LEADERBOARD_MIGRATION_ORDER,
|
|
14
|
+
type LeaderboardCapability,
|
|
15
|
+
type LeaderboardOptions,
|
|
16
|
+
leaderboard,
|
|
17
|
+
needsRankWorker,
|
|
18
|
+
} from "./capability";
|
|
19
|
+
export {
|
|
20
|
+
LeaderboardBoard,
|
|
21
|
+
LeaderboardConfig,
|
|
22
|
+
type LeaderboardConfigInput,
|
|
23
|
+
LeaderboardRank,
|
|
24
|
+
LeaderboardStore,
|
|
25
|
+
LeaderboardTier,
|
|
26
|
+
materializeSchedule,
|
|
27
|
+
resolveBoard,
|
|
28
|
+
ScoreAggregation,
|
|
29
|
+
ScoreDirection,
|
|
30
|
+
} from "./config/config";
|
|
31
|
+
export { LeaderboardEntry } from "./data/entry";
|
|
32
|
+
export type { OwnRank } from "./http/handlers";
|
|
33
|
+
export type { RankedEntry } from "./rank/query";
|
|
34
|
+
export { MAX_SEGMENT_SIZE } from "./rank/segment";
|
|
35
|
+
export { classifyTier } from "./rank/tiers";
|
|
36
|
+
export { BOOKMARK_HEADER } from "./session/bookmark";
|
|
37
|
+
export { ALL_TIME_WINDOW, previousWindowKeys, windowKeyAt } from "./window/schedule";
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import type { Kysely } from "kysely";
|
|
5
|
+
import type { Migration } from "kysely/migration";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The leaderboard tables: player entries, and the board drift guard.
|
|
9
|
+
*
|
|
10
|
+
* camelCase identifiers; `CamelCasePlugin` snake-cases them in the DDL. `down` is the tested inverse.
|
|
11
|
+
*/
|
|
12
|
+
export const leaderboard_0001_entries: Migration = {
|
|
13
|
+
up: async (db: Kysely<unknown>): Promise<void> => {
|
|
14
|
+
await db.schema
|
|
15
|
+
.createTable("pithyLeaderboardEntries")
|
|
16
|
+
// Plain INTEGER PRIMARY KEY (a rowid alias), deliberately NOT autoincrement. It still auto-assigns
|
|
17
|
+
// ascending ids; autoincrement would only add a never-reuse-a-deleted-id guarantee we do not need
|
|
18
|
+
// (the id is internal, never exposed) and would cost an extra `sqlite_sequence` row write on every
|
|
19
|
+
// insert — and even on a guarded no-op upsert, since SQLite reserves the sequence value before it
|
|
20
|
+
// detects the conflict. Dropping it makes a non-improving submission truly free.
|
|
21
|
+
.addColumn("id", "integer", (c) => c.primaryKey())
|
|
22
|
+
.addColumn("boardId", "text", (c) => c.notNull())
|
|
23
|
+
.addColumn("windowKey", "text", (c) => c.notNull())
|
|
24
|
+
.addColumn("userId", "text", (c) => c.notNull())
|
|
25
|
+
.addColumn("score", "real", (c) => c.notNull())
|
|
26
|
+
.addColumn("achievedAt", "integer", (c) => c.notNull())
|
|
27
|
+
.addColumn("submittedAt", "integer", (c) => c.notNull())
|
|
28
|
+
.addColumn("visible", "integer", (c) => c.notNull().defaultTo(1))
|
|
29
|
+
.addColumn("hidden", "integer", (c) => c.notNull().defaultTo(0))
|
|
30
|
+
.addColumn("rank", "integer")
|
|
31
|
+
.execute();
|
|
32
|
+
|
|
33
|
+
// One entry per player per window: the key the upsert-on-improve conflict target needs, and the
|
|
34
|
+
// reason a window carries aggregation state rather than being a filter over an append log.
|
|
35
|
+
await db.schema
|
|
36
|
+
.createIndex("pithyLeaderboardEntriesPlayerIdx")
|
|
37
|
+
.on("pithyLeaderboardEntries")
|
|
38
|
+
.columns(["boardId", "windowKey", "userId"])
|
|
39
|
+
.unique()
|
|
40
|
+
.execute();
|
|
41
|
+
|
|
42
|
+
// The ranking index. The ranking keys come LAST, after the equality-matched board and window:
|
|
43
|
+
// SQLite uses a multi-column index only on a left-prefix subset, so a score-first index would sit
|
|
44
|
+
// unused behind `board_id = ? AND window_key = ?`. The sort directions are deliberately mixed
|
|
45
|
+
// (score DESC, achievedAt ASC on a `desc` board) because the tiebreak is earliest-achieved-wins;
|
|
46
|
+
// that defeats SQLite's row-value comparison shortcut, so the rank predicate is spelled out as
|
|
47
|
+
// `score > ? OR (score = ? AND achieved_at < ?) OR (...)` instead. `rank/plan.workers.test.ts`
|
|
48
|
+
// reads EXPLAIN QUERY PLAN to prove this index is actually chosen rather than assuming it.
|
|
49
|
+
await db.schema
|
|
50
|
+
.createIndex("pithyLeaderboardEntriesRankIdx")
|
|
51
|
+
.on("pithyLeaderboardEntries")
|
|
52
|
+
.columns(["boardId", "windowKey", "score", "achievedAt"])
|
|
53
|
+
.execute();
|
|
54
|
+
|
|
55
|
+
await db.schema
|
|
56
|
+
.createTable("pithyLeaderboardBoards")
|
|
57
|
+
// Plain INTEGER PRIMARY KEY, not autoincrement — same reasoning as the entries table.
|
|
58
|
+
.addColumn("id", "integer", (c) => c.primaryKey())
|
|
59
|
+
.addColumn("boardKey", "text", (c) => c.notNull().unique())
|
|
60
|
+
.addColumn("store", "text", (c) => c.notNull().defaultTo("d1"))
|
|
61
|
+
.addColumn("direction", "text", (c) => c.notNull())
|
|
62
|
+
.addColumn("aggregation", "text", (c) => c.notNull())
|
|
63
|
+
.addColumn("window", "text")
|
|
64
|
+
.addColumn("createdAt", "integer", (c) => c.notNull())
|
|
65
|
+
.execute();
|
|
66
|
+
|
|
67
|
+
// A single-row advisory lock so at most one rank-refresh Workflow instance runs at a time. Without
|
|
68
|
+
// it, a cron that fires again before a refresh finishes would run two passes concurrently, and their
|
|
69
|
+
// chunked writes would interleave into an incoherent rank set (duplicate/gapped numbers) until the
|
|
70
|
+
// next fire. `name` is the lock's identity ("rank-refresh"); `holder` is the acquiring instance's
|
|
71
|
+
// token; `acquiredAt` lets a stale lock from a crashed instance be reclaimed.
|
|
72
|
+
await db.schema
|
|
73
|
+
.createTable("pithyLeaderboardLocks")
|
|
74
|
+
.addColumn("name", "text", (c) => c.primaryKey())
|
|
75
|
+
.addColumn("holder", "text", (c) => c.notNull())
|
|
76
|
+
.addColumn("acquiredAt", "integer", (c) => c.notNull())
|
|
77
|
+
.execute();
|
|
78
|
+
},
|
|
79
|
+
down: async (db: Kysely<unknown>): Promise<void> => {
|
|
80
|
+
await db.schema.dropTable("pithyLeaderboardLocks").execute();
|
|
81
|
+
await db.schema.dropTable("pithyLeaderboardBoards").execute();
|
|
82
|
+
await db.schema.dropIndex("pithyLeaderboardEntriesRankIdx").execute();
|
|
83
|
+
await db.schema.dropIndex("pithyLeaderboardEntriesPlayerIdx").execute();
|
|
84
|
+
await db.schema.dropTable("pithyLeaderboardEntries").execute();
|
|
85
|
+
},
|
|
86
|
+
};
|
package/src/rank/lock.ts
ADDED
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { LeaderboardLock } from "../data/lock";
|
|
5
|
+
import { LEADERBOARD_LOCKS_TABLE, type LeaderboardDatabase } from "../data/tables";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The rank-refresh advisory lock: at most one refresh runs at a time.
|
|
9
|
+
*
|
|
10
|
+
* Why it exists: the materialize refresh writes ranks in chunks, not atomically. If a cron fired again
|
|
11
|
+
* before a refresh finished, two passes would run concurrently and their chunked writes would interleave
|
|
12
|
+
* into an incoherent rank set — duplicate or gapped rank numbers — until the next fire corrected it. The
|
|
13
|
+
* lock serializes them: a second instance that cannot acquire it simply skips, and the pass it would have
|
|
14
|
+
* done is redundant anyway (the holder is already producing fresh ranks).
|
|
15
|
+
*
|
|
16
|
+
* The lock lives in D1, not in DO or KV, because the refresh already has the `DB` binding and D1's
|
|
17
|
+
* single-threaded execution makes the acquire genuinely atomic — the whole point.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
/** The one lock name the refresh uses. A single-row lock; there is no per-board locking. */
|
|
21
|
+
export const REFRESH_LOCK = "rank-refresh";
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* How long a held lock stays valid before it is treated as abandoned by a crashed instance.
|
|
25
|
+
*
|
|
26
|
+
* A refresh that finishes releases the lock immediately, so this only matters when an instance dies
|
|
27
|
+
* mid-pass. It must be comfortably longer than any real refresh so a slow-but-alive instance is never
|
|
28
|
+
* stolen from; one hour is far past the ~5-minute worst case at the ~1M-player shard boundary. Override
|
|
29
|
+
* with `LEADERBOARD_LOCK_STALE_MS` if a deployment refreshes boards larger than that.
|
|
30
|
+
*/
|
|
31
|
+
export const DEFAULT_LOCK_STALE_MS = 60 * 60 * 1000;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Try to take the refresh lock for `holder`, returning whether it was acquired.
|
|
35
|
+
*
|
|
36
|
+
* Atomic on D1's single thread: the upsert either inserts the row (no holder yet) or, on conflict, takes
|
|
37
|
+
* it over only if the current holder's lock is older than `staleMs` — a fresh lock is left untouched.
|
|
38
|
+
* A concurrent second caller therefore either finds no row and loses the insert race, or finds a fresh
|
|
39
|
+
* lock and is refused by the `WHERE`. Either way it reads back a `holder` that is not its own and knows
|
|
40
|
+
* to stand down.
|
|
41
|
+
*/
|
|
42
|
+
export async function acquireRefreshLock(
|
|
43
|
+
db: LeaderboardDatabase,
|
|
44
|
+
holder: string,
|
|
45
|
+
now: Date,
|
|
46
|
+
staleMs: number = DEFAULT_LOCK_STALE_MS,
|
|
47
|
+
): Promise<boolean> {
|
|
48
|
+
const staleBefore = LeaderboardLock.shape.acquiredAt.encode(new Date(now.getTime() - staleMs));
|
|
49
|
+
const row = LeaderboardLock.encode({ name: REFRESH_LOCK, holder, acquiredAt: now });
|
|
50
|
+
|
|
51
|
+
await db
|
|
52
|
+
.insertInto(LEADERBOARD_LOCKS_TABLE)
|
|
53
|
+
// biome-ignore lint/suspicious/noExplicitAny: the encoded row is the schema's `z.input` side.
|
|
54
|
+
.values(row as any)
|
|
55
|
+
.onConflict((oc) =>
|
|
56
|
+
oc
|
|
57
|
+
.column("name")
|
|
58
|
+
.doUpdateSet({ holder, acquiredAt: LeaderboardLock.shape.acquiredAt.encode(now) })
|
|
59
|
+
// Take over only an abandoned lock; a live holder's row is left as-is.
|
|
60
|
+
.where("pithyLeaderboardLocks.acquiredAt", "<", staleBefore),
|
|
61
|
+
)
|
|
62
|
+
.execute();
|
|
63
|
+
|
|
64
|
+
const current = await db
|
|
65
|
+
.selectFrom(LEADERBOARD_LOCKS_TABLE)
|
|
66
|
+
.select("holder")
|
|
67
|
+
.where("name", "=", REFRESH_LOCK)
|
|
68
|
+
.executeTakeFirst();
|
|
69
|
+
return current?.holder === holder;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
/** Release the lock if `holder` still holds it. A no-op if it was already reclaimed or released. */
|
|
73
|
+
export async function releaseRefreshLock(db: LeaderboardDatabase, holder: string): Promise<void> {
|
|
74
|
+
await db.deleteFrom(LEADERBOARD_LOCKS_TABLE).where("name", "=", REFRESH_LOCK).where("holder", "=", holder).execute();
|
|
75
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Pithy
|
|
2
|
+
// SPDX-License-Identifier: MIT
|
|
3
|
+
|
|
4
|
+
import { boundParameterBudget, chunkRowsByBoundParameters } from "@pithy-sh/core/src/data/boundParameters";
|
|
5
|
+
import { ValidationError } from "@pithy-sh/core/src/error/pithyError";
|
|
6
|
+
import { sql } from "kysely";
|
|
7
|
+
import type { LeaderboardBoard } from "../config/config";
|
|
8
|
+
import { LEADERBOARD_ENTRIES_TABLE, type LeaderboardDatabase } from "../data/tables";
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* The rank refresh pass — what `rank: { materialize }` buys and what it costs.
|
|
12
|
+
*
|
|
13
|
+
* A full-table rank rewrite is the obvious implementation and the wrong one. D1 executes one query at a
|
|
14
|
+
* time per database and caps a query at 30 seconds; a single `UPDATE` over a large board would hold the
|
|
15
|
+
* only thread for its whole duration and risk the documented `overloaded` error for every live
|
|
16
|
+
* submission behind it. So the pass is chunked: many small, bounded statements the runtime can
|
|
17
|
+
* interleave with real traffic.
|
|
18
|
+
*
|
|
19
|
+
* A chunk is a **pacing** unit: how much of the board one keyset step walks, kept well inside D1's
|
|
20
|
+
* 30-second per-query limit. It is no longer also the width of a statement — the bulk update sizes
|
|
21
|
+
* itself against D1's bound-parameter cap through core's arithmetic, so a chunk of any size is written
|
|
22
|
+
* in as many statements as it takes. That separation is the fix for #250: `chunkSize` was unvalidated,
|
|
23
|
+
* and `chunkSize: 40` bound 120 and broke the pass with no warning that a limit was even involved.
|
|
24
|
+
*
|
|
25
|
+
* Chunks walk the board by keyset, not by `OFFSET`: an offset page makes SQLite count past every row it
|
|
26
|
+
* skips, so an offset walk is quadratic in billed rows — the very cost `materialize` exists to avoid.
|
|
27
|
+
*
|
|
28
|
+
* Cloudflare documents no rank-materialization pattern. All of this is adopter-built, which is exactly
|
|
29
|
+
* why it lives in the package instead of in every adopter's repo.
|
|
30
|
+
*/
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* What one ranked row costs the bulk update: `WHEN id`, `THEN rank`, and the id again in the `IN` list.
|
|
34
|
+
*
|
|
35
|
+
* The cap itself is not restated here. It was, and a second copy of a platform limit is how a limit goes
|
|
36
|
+
* stale in one place and not the other — `MAX_BOUND_PARAMETERS` lives in `@pithy-sh/core`, once.
|
|
37
|
+
*/
|
|
38
|
+
export const RANK_PARAMETERS_PER_ROW = 3;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Rows per chunk by default: as many as one update statement can carry, from core's budget.
|
|
42
|
+
*
|
|
43
|
+
* Derived rather than written out, so it moves if the platform does. Any other size works — the update
|
|
44
|
+
* chunks itself — and this is simply the size at which a chunk is exactly one statement.
|
|
45
|
+
*/
|
|
46
|
+
export const RANK_CHUNK_SIZE = Math.floor(boundParameterBudget(0) / RANK_PARAMETERS_PER_ROW);
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Chunks a refresh ranks before it checkpoints its cursor. `2000 * 33` is ~66k entries per Workflow step.
|
|
50
|
+
*
|
|
51
|
+
* The batch cap is stated here rather than in `worker.entry.ts`, which is the module that passes it as
|
|
52
|
+
* `maxChunks`. That module imports `cloudflare:workers`, so anything it exports is unreachable from a
|
|
53
|
+
* plain Node process, and a constant that reads as ordinary is exactly how #172 and #180 happened twice:
|
|
54
|
+
* a Node-side caller imports the number, gets workerd behind it, and the failure surfaces as
|
|
55
|
+
* `Could not load pithy.config.ts` — naming the config rather than the import. A pure value belongs in a
|
|
56
|
+
* pure module. `configEntrypoints.test.ts` is what holds that.
|
|
57
|
+
*/
|
|
58
|
+
export const REFRESH_BATCH_CHUNKS = 2000;
|
|
59
|
+
|
|
60
|
+
/** The last entry a chunk ranked — where the next chunk (or the next batch's step) resumes. */
|
|
61
|
+
export interface Keyset {
|
|
62
|
+
score: number;
|
|
63
|
+
achievedAt: number;
|
|
64
|
+
userId: string;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Narrow a selected `achievedAt` to its stored epoch.
|
|
69
|
+
*
|
|
70
|
+
* The column's decode-side input is a union (`number | string | Date`) so the codec stays
|
|
71
|
+
* encode-compatible, but a chunk selects raw columns rather than parsing whole rows — D1 always hands
|
|
72
|
+
* back the stored integer. This keeps the keyset arithmetic honest without paying to parse every row.
|
|
73
|
+
*/
|
|
74
|
+
function toEpoch(value: number | string | Date): number {
|
|
75
|
+
return value instanceof Date ? value.getTime() : Number(value);
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export interface RefreshResult {
|
|
79
|
+
/** The cumulative number of entries ranked, including any `startRank` carried in from a prior batch. */
|
|
80
|
+
ranked: number;
|
|
81
|
+
/** How many chunks ran in this call. */
|
|
82
|
+
chunks: number;
|
|
83
|
+
/** True when the board is fully ranked; false when `maxChunks` stopped this batch mid-board. */
|
|
84
|
+
complete: boolean;
|
|
85
|
+
/**
|
|
86
|
+
* Where the next batch resumes, or null when the board is complete. Passing this back as `resumeAfter`
|
|
87
|
+
* (with `startRank` set to `ranked`) continues the pass exactly where it left off — the seam that makes
|
|
88
|
+
* a board rankable across many bounded steps of a Workflow, rather than one unbounded invocation.
|
|
89
|
+
*/
|
|
90
|
+
cursor: Keyset | null;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export interface RefreshOptions {
|
|
94
|
+
/**
|
|
95
|
+
* Rows per keyset step. Defaults to {@link RANK_CHUNK_SIZE}; lower it to be gentler on a hot database.
|
|
96
|
+
*
|
|
97
|
+
* It carries no bound-parameter ceiling — the update chunks itself — so the only thing refused is a
|
|
98
|
+
* value that is not a count. A `chunkSize` of 0 used to report a board complete having ranked nobody.
|
|
99
|
+
*/
|
|
100
|
+
chunkSize?: number;
|
|
101
|
+
/** Stop cleanly once this many chunks have run, whether or not the board is finished. Default: no cap. */
|
|
102
|
+
maxChunks?: number;
|
|
103
|
+
/** Resume the keyset walk after this entry — the `cursor` a prior batch returned. */
|
|
104
|
+
resumeAfter?: Keyset;
|
|
105
|
+
/** The rank already assigned before this batch, so numbering continues rather than restarting at 1. */
|
|
106
|
+
startRank?: number;
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
/**
|
|
110
|
+
* Recompute the stored rank for one board and window, best first.
|
|
111
|
+
*
|
|
112
|
+
* Ranks are positions in the total ordering, so a chunk needs no counting — the first chunk's first row
|
|
113
|
+
* is rank 1 and every chunk continues the count. The pass is **resumable**: with no `maxChunks` it ranks
|
|
114
|
+
* the whole board; with a `maxChunks` batch cap it ranks that many chunks, returns a `cursor`, and the
|
|
115
|
+
* caller feeds the cursor (and `startRank: ranked`) back to continue. That is what lets a board of any
|
|
116
|
+
* size be ranked across a series of bounded, individually-durable Workflow steps — there is no longer a
|
|
117
|
+
* per-invocation ceiling on how many entries a board can have.
|
|
118
|
+
*/
|
|
119
|
+
export async function refreshWindowRanks(
|
|
120
|
+
db: LeaderboardDatabase,
|
|
121
|
+
board: LeaderboardBoard,
|
|
122
|
+
windowKey: string,
|
|
123
|
+
options: RefreshOptions = {},
|
|
124
|
+
): Promise<RefreshResult> {
|
|
125
|
+
const chunkSize = options.chunkSize ?? RANK_CHUNK_SIZE;
|
|
126
|
+
if (!Number.isInteger(chunkSize) || chunkSize < 1) {
|
|
127
|
+
throw new ValidationError({
|
|
128
|
+
message: "The rank refresh was asked for an impossible chunk size.",
|
|
129
|
+
action: "Pass a whole number of one or more, or omit chunkSize for the default.",
|
|
130
|
+
detail: `refreshWindowRanks received chunkSize=${chunkSize}; it must be a positive integer.`,
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
const maxChunks = options.maxChunks ?? Number.POSITIVE_INFINITY;
|
|
134
|
+
const descending = board.direction === "desc";
|
|
135
|
+
|
|
136
|
+
let after: Keyset | undefined = options.resumeAfter;
|
|
137
|
+
let ranked = options.startRank ?? 0;
|
|
138
|
+
let chunks = 0;
|
|
139
|
+
|
|
140
|
+
while (chunks < maxChunks) {
|
|
141
|
+
let query = db
|
|
142
|
+
.selectFrom(LEADERBOARD_ENTRIES_TABLE)
|
|
143
|
+
.select(["id", "score", "achievedAt", "userId"])
|
|
144
|
+
.where("boardId", "=", board.key)
|
|
145
|
+
.where("windowKey", "=", windowKey)
|
|
146
|
+
.where("visible", "=", 1)
|
|
147
|
+
.where("hidden", "=", 0);
|
|
148
|
+
|
|
149
|
+
if (after) {
|
|
150
|
+
// Keyset resume: strictly worse than the last row ranked, in the board's total ordering. Spelled
|
|
151
|
+
// out as an OR-of-ANDs for the same reason as the rank predicate — the sort directions are mixed.
|
|
152
|
+
const scoreWorse = descending ? sql`score < ${after.score}` : sql`score > ${after.score}`;
|
|
153
|
+
query = query.where(
|
|
154
|
+
sql<boolean>`(
|
|
155
|
+
${scoreWorse}
|
|
156
|
+
OR (score = ${after.score} AND achieved_at > ${after.achievedAt})
|
|
157
|
+
OR (score = ${after.score} AND achieved_at = ${after.achievedAt} AND user_id > ${after.userId})
|
|
158
|
+
)`,
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
const rows = await query
|
|
163
|
+
.orderBy("score", descending ? "desc" : "asc")
|
|
164
|
+
.orderBy("achievedAt", "asc")
|
|
165
|
+
.orderBy("userId", "asc")
|
|
166
|
+
.limit(chunkSize)
|
|
167
|
+
.execute();
|
|
168
|
+
|
|
169
|
+
if (rows.length === 0) return { ranked, chunks, complete: true, cursor: null };
|
|
170
|
+
|
|
171
|
+
// One bulk UPDATE per statement's worth of rows: `SET rank = CASE id WHEN ? THEN ? … END WHERE id
|
|
172
|
+
// IN (…)`. Row-at-a-time updates would be correct too, and would cost one round trip each on a
|
|
173
|
+
// single-threaded database. How many rows fit is core's arithmetic, not a number written out here,
|
|
174
|
+
// so a chunk larger than one statement is simply written as several.
|
|
175
|
+
let written = 0;
|
|
176
|
+
for (const group of chunkRowsByBoundParameters(rows, RANK_PARAMETERS_PER_ROW)) {
|
|
177
|
+
const base = ranked + written;
|
|
178
|
+
let cases = sql``;
|
|
179
|
+
group.forEach((row, index) => {
|
|
180
|
+
cases = sql`${cases} WHEN ${row.id} THEN ${base + index + 1}`;
|
|
181
|
+
});
|
|
182
|
+
await db
|
|
183
|
+
.updateTable(LEADERBOARD_ENTRIES_TABLE)
|
|
184
|
+
.set({ rank: sql<number>`CASE id ${cases} END` })
|
|
185
|
+
.where(
|
|
186
|
+
"id",
|
|
187
|
+
"in",
|
|
188
|
+
group.map((row) => row.id),
|
|
189
|
+
)
|
|
190
|
+
.execute();
|
|
191
|
+
written += group.length;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
ranked += rows.length;
|
|
195
|
+
chunks += 1;
|
|
196
|
+
const last = rows[rows.length - 1];
|
|
197
|
+
if (!last) break;
|
|
198
|
+
after = { score: last.score, achievedAt: toEpoch(last.achievedAt), userId: last.userId };
|
|
199
|
+
|
|
200
|
+
// A short chunk means the board ran out, so there is nothing left to resume into.
|
|
201
|
+
if (rows.length < chunkSize) return { ranked, chunks, complete: true, cursor: null };
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
// The batch cap stopped us mid-board: hand back the cursor so the next step resumes here.
|
|
205
|
+
return { ranked, chunks, complete: false, cursor: after ?? null };
|
|
206
|
+
}
|