cursedbelt-server 4.37.0 → 4.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/server/ctgrChunkDurable.d.ts +62 -0
- package/dist/server/ctgrChunkDurable.js +155 -0
- package/dist/server/ctgrTunnel.d.ts +16 -4
- package/dist/server/ctgrTunnel.js +9 -2
- package/package.json +7 -1
- package/src/server/ctgrChunkDurable.spec.ts +144 -0
- package/src/server/ctgrChunkDurable.ts +188 -0
- package/src/server/ctgrTunnel.ts +23 -5
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ctgr chunk store on a Cloudflare Worker: ONE Durable Object per multi-chunk payload, so
|
|
3
|
+
* every chunk of it meets every other chunk no matter which isolate answered it.
|
|
4
|
+
*
|
|
5
|
+
* ── Why this exists (2026-09-26, desk task 2185) ─────────────────────────────
|
|
6
|
+
* A GET-only device splits any body over 1800 bytes into several `/ctgr<xx>` GETs, and
|
|
7
|
+
* {@link mountCtgrTunnel} reassembles them in a {@link ChunkStore} — memory. On a Worker, memory
|
|
8
|
+
* is one isolate: chunk 2 answered by another isolate (another colo, a recycled isolate, load
|
|
9
|
+
* spread across several) finds no chunk 1, answers 202, and the payload is silently never
|
|
10
|
+
* delivered. desk's large saves were exposed; auth's login body is one chunk.
|
|
11
|
+
*
|
|
12
|
+
* ── The object is a rendezvous, not a decoder ────────────────────────────────
|
|
13
|
+
* It keeps each chunk's raw query params by index and, when the last one arrives, hands the
|
|
14
|
+
* whole ordered set back. The Worker then runs them through a fresh {@link ChunkStore} — the
|
|
15
|
+
* SAME crc/sha/size/decode path a Mac process uses — so nothing about verification is
|
|
16
|
+
* re-implemented here, and the object never parses a body. A single-chunk twin (the norm)
|
|
17
|
+
* never reaches the object at all.
|
|
18
|
+
*
|
|
19
|
+
* ── One object per payload ───────────────────────────────────────────────────
|
|
20
|
+
* Addressed by `ctgr:<_cid>` (a `crypto.randomUUID()` the client mints per request), so the
|
|
21
|
+
* chunks of one request are serialised through one object and different requests never
|
|
22
|
+
* contend. The partial lives in the object's memory: chunks arrive milliseconds apart, the idle
|
|
23
|
+
* TTL is 30 s, and an object is only evicted after minutes idle — a partial it loses is one a
|
|
24
|
+
* deploy would have lost too. An object holding nothing costs nothing.
|
|
25
|
+
*
|
|
26
|
+
* ── Types are structural ────────────────────────────────────────────────────
|
|
27
|
+
* No `@cloudflare/workers-types`, and the classic `fetch` interface — the same reasons as
|
|
28
|
+
* `loginThrottleDurable.ts`.
|
|
29
|
+
*/
|
|
30
|
+
import { type ChunkStoreOpts } from 'cursedbelt-core/ctgr/chunk-store';
|
|
31
|
+
import type { CtgrChunkSink } from './ctgrTunnel.js';
|
|
32
|
+
/** The members of a `DurableObjectNamespace` binding the store calls. */
|
|
33
|
+
export interface CtgrChunkNamespaceLike {
|
|
34
|
+
idFromName(name: string): unknown;
|
|
35
|
+
get(id: unknown): {
|
|
36
|
+
fetch(request: Request): Promise<Response>;
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The Durable Object. Re-export it from the Worker's entry under the `class_name` your
|
|
41
|
+
* `wrangler.jsonc` binds, and add it in a migration (`new_sqlite_classes`):
|
|
42
|
+
*
|
|
43
|
+
* export { CtgrChunkObject as DeskCtgrChunks } from "cursedbelt-server/ctgr-tunnel/durable";
|
|
44
|
+
*/
|
|
45
|
+
export declare class CtgrChunkObject {
|
|
46
|
+
private partial;
|
|
47
|
+
private readonly maxPayloadBytes;
|
|
48
|
+
private readonly ttlMs;
|
|
49
|
+
private readonly now;
|
|
50
|
+
/** `options` is for tests; the runtime passes `(state, env)` only. */
|
|
51
|
+
constructor(_state?: unknown, _env?: unknown, options?: ChunkStoreOpts);
|
|
52
|
+
fetch(request: Request): Promise<Response>;
|
|
53
|
+
private add;
|
|
54
|
+
}
|
|
55
|
+
/**
|
|
56
|
+
* The Worker half: a {@link CtgrChunkSink} for {@link mountCtgrTunnel} whose multi-chunk
|
|
57
|
+
* partials live in the object. Cheap to build — build it per request over `env.<BINDING>`.
|
|
58
|
+
*
|
|
59
|
+
* An object that cannot be reached THROWS, and the tunnel answers 503: a payload half-held in
|
|
60
|
+
* one isolate's memory is exactly the silent loss this exists to end, so there is no fallback.
|
|
61
|
+
*/
|
|
62
|
+
export declare function createDurableChunkStore(namespace: CtgrChunkNamespaceLike, options?: ChunkStoreOpts): CtgrChunkSink;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ctgr chunk store on a Cloudflare Worker: ONE Durable Object per multi-chunk payload, so
|
|
3
|
+
* every chunk of it meets every other chunk no matter which isolate answered it.
|
|
4
|
+
*
|
|
5
|
+
* ── Why this exists (2026-09-26, desk task 2185) ─────────────────────────────
|
|
6
|
+
* A GET-only device splits any body over 1800 bytes into several `/ctgr<xx>` GETs, and
|
|
7
|
+
* {@link mountCtgrTunnel} reassembles them in a {@link ChunkStore} — memory. On a Worker, memory
|
|
8
|
+
* is one isolate: chunk 2 answered by another isolate (another colo, a recycled isolate, load
|
|
9
|
+
* spread across several) finds no chunk 1, answers 202, and the payload is silently never
|
|
10
|
+
* delivered. desk's large saves were exposed; auth's login body is one chunk.
|
|
11
|
+
*
|
|
12
|
+
* ── The object is a rendezvous, not a decoder ────────────────────────────────
|
|
13
|
+
* It keeps each chunk's raw query params by index and, when the last one arrives, hands the
|
|
14
|
+
* whole ordered set back. The Worker then runs them through a fresh {@link ChunkStore} — the
|
|
15
|
+
* SAME crc/sha/size/decode path a Mac process uses — so nothing about verification is
|
|
16
|
+
* re-implemented here, and the object never parses a body. A single-chunk twin (the norm)
|
|
17
|
+
* never reaches the object at all.
|
|
18
|
+
*
|
|
19
|
+
* ── One object per payload ───────────────────────────────────────────────────
|
|
20
|
+
* Addressed by `ctgr:<_cid>` (a `crypto.randomUUID()` the client mints per request), so the
|
|
21
|
+
* chunks of one request are serialised through one object and different requests never
|
|
22
|
+
* contend. The partial lives in the object's memory: chunks arrive milliseconds apart, the idle
|
|
23
|
+
* TTL is 30 s, and an object is only evicted after minutes idle — a partial it loses is one a
|
|
24
|
+
* deploy would have lost too. An object holding nothing costs nothing.
|
|
25
|
+
*
|
|
26
|
+
* ── Types are structural ────────────────────────────────────────────────────
|
|
27
|
+
* No `@cloudflare/workers-types`, and the classic `fetch` interface — the same reasons as
|
|
28
|
+
* `loginThrottleDurable.ts`.
|
|
29
|
+
*/
|
|
30
|
+
import { ChunkStore } from 'cursedbelt-core/ctgr/chunk-store';
|
|
31
|
+
import { parseChunk } from 'cursedbelt-core/ctgr';
|
|
32
|
+
const DEFAULT_MAX_PAYLOAD_BYTES = 10 * 1024 * 1024;
|
|
33
|
+
const DEFAULT_TTL_MS = 30_000;
|
|
34
|
+
const json = (value, status = 200) => new Response(JSON.stringify(value), { status, headers: { 'content-type': 'application/json' } });
|
|
35
|
+
/**
|
|
36
|
+
* The Durable Object. Re-export it from the Worker's entry under the `class_name` your
|
|
37
|
+
* `wrangler.jsonc` binds, and add it in a migration (`new_sqlite_classes`):
|
|
38
|
+
*
|
|
39
|
+
* export { CtgrChunkObject as DeskCtgrChunks } from "cursedbelt-server/ctgr-tunnel/durable";
|
|
40
|
+
*/
|
|
41
|
+
export class CtgrChunkObject {
|
|
42
|
+
partial = null;
|
|
43
|
+
maxPayloadBytes;
|
|
44
|
+
ttlMs;
|
|
45
|
+
now;
|
|
46
|
+
/** `options` is for tests; the runtime passes `(state, env)` only. */
|
|
47
|
+
constructor(_state, _env, options = {}) {
|
|
48
|
+
this.maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
|
|
49
|
+
this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
50
|
+
this.now = options.now ?? Date.now;
|
|
51
|
+
}
|
|
52
|
+
async fetch(request) {
|
|
53
|
+
const path = new URL(request.url).pathname;
|
|
54
|
+
if (request.method !== 'POST' || path !== '/ctgr/chunk') {
|
|
55
|
+
return json({ error: `${request.method} ${path}: only POST /ctgr/chunk` }, 404);
|
|
56
|
+
}
|
|
57
|
+
let params;
|
|
58
|
+
let chunk;
|
|
59
|
+
try {
|
|
60
|
+
params = (await request.json());
|
|
61
|
+
chunk = parseChunk(params);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
return json({ error: error instanceof Error ? error.message : String(error) }, 400);
|
|
65
|
+
}
|
|
66
|
+
return json(this.add(params, chunk));
|
|
67
|
+
}
|
|
68
|
+
add(params, chunk) {
|
|
69
|
+
if (this.partial && this.partial.updatedAt < this.now() - this.ttlMs)
|
|
70
|
+
this.partial = null;
|
|
71
|
+
if (chunk.payloadLength > this.maxPayloadBytes)
|
|
72
|
+
return { error: 'ctgr: payload exceeds size cap' };
|
|
73
|
+
let entry = this.partial;
|
|
74
|
+
if (!entry) {
|
|
75
|
+
entry = {
|
|
76
|
+
cid: chunk.cid,
|
|
77
|
+
total: chunk.total,
|
|
78
|
+
payloadLength: chunk.payloadLength,
|
|
79
|
+
gzip: chunk.gzip,
|
|
80
|
+
method: chunk.method,
|
|
81
|
+
parts: new Array(chunk.total),
|
|
82
|
+
received: 0,
|
|
83
|
+
bytes: 0,
|
|
84
|
+
updatedAt: this.now(),
|
|
85
|
+
};
|
|
86
|
+
this.partial = entry;
|
|
87
|
+
}
|
|
88
|
+
else if (entry.cid !== chunk.cid ||
|
|
89
|
+
entry.total !== chunk.total ||
|
|
90
|
+
entry.payloadLength !== chunk.payloadLength ||
|
|
91
|
+
entry.gzip !== chunk.gzip ||
|
|
92
|
+
entry.method !== chunk.method) {
|
|
93
|
+
// A payload's framing must not change mid-stream — the same rule `ChunkStore` keeps.
|
|
94
|
+
this.partial = null;
|
|
95
|
+
return { error: 'ctgr: inconsistent chunk metadata' };
|
|
96
|
+
}
|
|
97
|
+
entry.updatedAt = this.now();
|
|
98
|
+
if (entry.parts[chunk.index] === undefined) {
|
|
99
|
+
entry.parts[chunk.index] = params;
|
|
100
|
+
entry.received += 1;
|
|
101
|
+
entry.bytes += chunk.bytes.length;
|
|
102
|
+
if (entry.bytes > this.maxPayloadBytes) {
|
|
103
|
+
this.partial = null;
|
|
104
|
+
return { error: 'ctgr: payload exceeds size cap' };
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
if (entry.received < entry.total)
|
|
108
|
+
return { pending: true };
|
|
109
|
+
this.partial = null;
|
|
110
|
+
return { chunks: entry.parts };
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* The Worker half: a {@link CtgrChunkSink} for {@link mountCtgrTunnel} whose multi-chunk
|
|
115
|
+
* partials live in the object. Cheap to build — build it per request over `env.<BINDING>`.
|
|
116
|
+
*
|
|
117
|
+
* An object that cannot be reached THROWS, and the tunnel answers 503: a payload half-held in
|
|
118
|
+
* one isolate's memory is exactly the silent loss this exists to end, so there is no fallback.
|
|
119
|
+
*/
|
|
120
|
+
export function createDurableChunkStore(namespace, options = {}) {
|
|
121
|
+
return {
|
|
122
|
+
async add(input) {
|
|
123
|
+
const params = input instanceof URLSearchParams ? Object.fromEntries(input) : { ...input };
|
|
124
|
+
let chunk;
|
|
125
|
+
try {
|
|
126
|
+
chunk = parseChunk(params);
|
|
127
|
+
}
|
|
128
|
+
catch (error) {
|
|
129
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
130
|
+
}
|
|
131
|
+
// One chunk needs no rendezvous — decode it here, in this isolate.
|
|
132
|
+
if (chunk.total === 1)
|
|
133
|
+
return new ChunkStore(options).add(params);
|
|
134
|
+
const stub = namespace.get(namespace.idFromName(`ctgr:${chunk.cid}`));
|
|
135
|
+
const response = await stub.fetch(new Request('https://ctgr-chunks.invalid/ctgr/chunk', {
|
|
136
|
+
method: 'POST',
|
|
137
|
+
headers: { 'content-type': 'application/json' },
|
|
138
|
+
body: JSON.stringify(params),
|
|
139
|
+
}));
|
|
140
|
+
const reply = (await response.json().catch(() => null));
|
|
141
|
+
if (!reply)
|
|
142
|
+
throw new Error(`ctgr chunk object answered ${response.status} with no JSON`);
|
|
143
|
+
if ('error' in reply)
|
|
144
|
+
return new Error(reply.error);
|
|
145
|
+
if ('pending' in reply)
|
|
146
|
+
return false;
|
|
147
|
+
// Every chunk is here: the one verify/decode path, in a store that lives for this call.
|
|
148
|
+
const assemble = new ChunkStore(options);
|
|
149
|
+
let result = new Error('ctgr: empty chunk set');
|
|
150
|
+
for (const part of reply.chunks)
|
|
151
|
+
result = await assemble.add(part);
|
|
152
|
+
return result;
|
|
153
|
+
},
|
|
154
|
+
};
|
|
155
|
+
}
|
|
@@ -1,15 +1,27 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { type CtgrDecoded } from 'cursedbelt-core/ctgr/types';
|
|
2
2
|
import type { Env, Hono } from 'hono';
|
|
3
|
+
/**
|
|
4
|
+
* What the tunnel needs of an accumulator: a {@link ChunkStore}, or the Worker's
|
|
5
|
+
* `createDurableChunkStore` (`cursedbelt-server/ctgr-tunnel/durable`), whose partials every
|
|
6
|
+
* isolate shares. `false` is an intermediate chunk (→ 202), an `Error` a bad payload (→ 400);
|
|
7
|
+
* a THROW means the store itself could not answer (→ 503).
|
|
8
|
+
*/
|
|
9
|
+
export interface CtgrChunkSink {
|
|
10
|
+
add(params: Record<string, string> | URLSearchParams): Promise<CtgrDecoded | false | Error>;
|
|
11
|
+
}
|
|
3
12
|
export interface CtgrTunnelOptions {
|
|
4
13
|
/**
|
|
5
14
|
* The accumulator for multi-chunk payloads. Share ONE across an app (the default is
|
|
6
15
|
* a fresh singleton); pass your own to tune the DoS caps ({@link ChunkStore}).
|
|
16
|
+
* 🔴 On a Cloudflare Worker a module-scope `ChunkStore` is one ISOLATE's memory — pass
|
|
17
|
+
* `createDurableChunkStore(env.<BINDING>)` there, or a multi-chunk payload whose chunks
|
|
18
|
+
* land on different isolates is silently never delivered.
|
|
7
19
|
*/
|
|
8
|
-
store?:
|
|
20
|
+
store?: CtgrChunkSink;
|
|
9
21
|
}
|
|
10
22
|
/**
|
|
11
23
|
* Mount the GET-only tunnel on `app`. Call once, on the Hono app whose real routes sit
|
|
12
24
|
* at the paths the twins mirror (a twin path is `/ctgr<xx>` + the real path). Returns
|
|
13
|
-
* the
|
|
25
|
+
* the store in use, for tests and metrics.
|
|
14
26
|
*/
|
|
15
|
-
export declare function mountCtgrTunnel<E extends Env = Env>(app: Hono<E>, options?: CtgrTunnelOptions):
|
|
27
|
+
export declare function mountCtgrTunnel<E extends Env = Env>(app: Hono<E>, options?: CtgrTunnelOptions): CtgrChunkSink;
|
|
@@ -23,7 +23,7 @@ const isCtgrParam = (key) => key === '_d' || key === '_sha' || key.startsWith('_
|
|
|
23
23
|
/**
|
|
24
24
|
* Mount the GET-only tunnel on `app`. Call once, on the Hono app whose real routes sit
|
|
25
25
|
* at the paths the twins mirror (a twin path is `/ctgr<xx>` + the real path). Returns
|
|
26
|
-
* the
|
|
26
|
+
* the store in use, for tests and metrics.
|
|
27
27
|
*/
|
|
28
28
|
export function mountCtgrTunnel(app, options = {}) {
|
|
29
29
|
const store = options.store ?? new ChunkStore();
|
|
@@ -36,7 +36,14 @@ export function mountCtgrTunnel(app, options = {}) {
|
|
|
36
36
|
if (!pathMethod) {
|
|
37
37
|
return c.json({ error: 'not found', code: 'NOT_FOUND' }, 404);
|
|
38
38
|
}
|
|
39
|
-
|
|
39
|
+
let result;
|
|
40
|
+
try {
|
|
41
|
+
result = params._cv === undefined ? legacy.add(params, pathMethod) : await store.add(params);
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
console.error(`[ctgr] the chunk store could not answer: ${error instanceof Error ? error.message : String(error)}`);
|
|
45
|
+
return c.json({ error: 'ctgr: the chunk store could not answer — send it again', code: 'CTGR_STORE_UNAVAILABLE' }, 503);
|
|
46
|
+
}
|
|
40
47
|
if (result === false)
|
|
41
48
|
return c.json({ acknowledged: true }, 202);
|
|
42
49
|
if (result instanceof Error) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cursedbelt-server",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.38.0",
|
|
4
4
|
"license": "ISC",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"description": "The app-facing Bun/Hono server tier of the cursedbelt split — storage, sharing, activity, guard, sync. React-free; cursedbelt-core below it.",
|
|
@@ -65,6 +65,12 @@
|
|
|
65
65
|
"source": "./src/server/ctgrTunnel.ts",
|
|
66
66
|
"import": "./dist/server/ctgrTunnel.js"
|
|
67
67
|
},
|
|
68
|
+
"./ctgr-tunnel/durable": {
|
|
69
|
+
"types": "./dist/server/ctgrChunkDurable.d.ts",
|
|
70
|
+
"bun": "./src/server/ctgrChunkDurable.ts",
|
|
71
|
+
"source": "./src/server/ctgrChunkDurable.ts",
|
|
72
|
+
"import": "./dist/server/ctgrChunkDurable.js"
|
|
73
|
+
},
|
|
68
74
|
"./analytics": {
|
|
69
75
|
"types": "./dist/server/analytics/index.d.ts",
|
|
70
76
|
"bun": "./src/server/analytics/index.ts",
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { describe, expect, test } from 'bun:test';
|
|
2
|
+
import { encodeRequest } from 'cursedbelt-core/ctgr';
|
|
3
|
+
import { ChunkStore } from 'cursedbelt-core/ctgr/chunk-store';
|
|
4
|
+
import { Hono } from 'hono';
|
|
5
|
+
import { type CtgrChunkNamespaceLike, CtgrChunkObject, createDurableChunkStore } from './ctgrChunkDurable.js';
|
|
6
|
+
import { mountCtgrTunnel } from './ctgrTunnel.js';
|
|
7
|
+
|
|
8
|
+
/** A namespace like the runtime's: one object per name, however many isolates ask. */
|
|
9
|
+
function fakeNamespace(clock?: () => number) {
|
|
10
|
+
const objects = new Map<string, CtgrChunkObject>();
|
|
11
|
+
const calls: string[] = [];
|
|
12
|
+
const namespace: CtgrChunkNamespaceLike = {
|
|
13
|
+
idFromName: (name) => name,
|
|
14
|
+
get: (id) => ({
|
|
15
|
+
fetch: async (request: Request) => {
|
|
16
|
+
const name = id as string;
|
|
17
|
+
calls.push(name);
|
|
18
|
+
let object = objects.get(name);
|
|
19
|
+
if (!object) {
|
|
20
|
+
object = new CtgrChunkObject(undefined, undefined, clock ? { now: clock } : {});
|
|
21
|
+
objects.set(name, object);
|
|
22
|
+
}
|
|
23
|
+
return object.fetch(request);
|
|
24
|
+
},
|
|
25
|
+
}),
|
|
26
|
+
};
|
|
27
|
+
return { namespace, objects, calls };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
const bigBody = { note: 'x'.repeat(9000), n: 7 };
|
|
31
|
+
|
|
32
|
+
async function multiChunk() {
|
|
33
|
+
// Incompressible enough to stay multi-chunk after gzip.
|
|
34
|
+
const random = Array.from({ length: 6000 }, (_, i) => ((i * 2654435761) >>> 0).toString(36)).join('');
|
|
35
|
+
const { chunks } = await encodeRequest('POST', '/api/v1/notes', { ...bigBody, random });
|
|
36
|
+
expect(chunks.length).toBeGreaterThan(2);
|
|
37
|
+
return { chunks, body: { ...bigBody, random } };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
describe('createDurableChunkStore — two isolates, one object', () => {
|
|
41
|
+
test('chunks alternating between two isolates reassemble, in any order', async () => {
|
|
42
|
+
const { namespace } = fakeNamespace();
|
|
43
|
+
const isolates = [createDurableChunkStore(namespace), createDurableChunkStore(namespace)];
|
|
44
|
+
const { chunks, body } = await multiChunk();
|
|
45
|
+
const order = chunks.map((_, i) => i).reverse();
|
|
46
|
+
const results = [];
|
|
47
|
+
for (const [n, i] of order.entries()) results.push(await isolates[n % 2].add(chunks[i].params));
|
|
48
|
+
for (const intermediate of results.slice(0, -1)) expect(intermediate).toBe(false);
|
|
49
|
+
const last = results.at(-1);
|
|
50
|
+
expect(last).not.toBeInstanceOf(Error);
|
|
51
|
+
expect(last && !(last instanceof Error) ? last.body : null).toEqual(body);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test('🔴 the failure it replaces: two per-isolate ChunkStores never finish the payload', async () => {
|
|
55
|
+
const isolates = [new ChunkStore(), new ChunkStore()];
|
|
56
|
+
const { chunks } = await multiChunk();
|
|
57
|
+
const results = [];
|
|
58
|
+
for (const [n, chunk] of chunks.entries()) results.push(await isolates[n % 2].add(chunk.params));
|
|
59
|
+
expect(results.every((r) => r === false)).toBe(true);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
test('a single-chunk twin never reaches an object', async () => {
|
|
63
|
+
const { namespace, calls } = fakeNamespace();
|
|
64
|
+
const { chunks } = await encodeRequest('POST', '/api/v1/notes', { small: true });
|
|
65
|
+
expect(chunks).toHaveLength(1);
|
|
66
|
+
const result = await createDurableChunkStore(namespace).add(chunks[0].params);
|
|
67
|
+
expect(result && !(result instanceof Error) ? result.body : null).toEqual({ small: true });
|
|
68
|
+
expect(calls).toHaveLength(0);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
test('each payload has its own object, and a finished one holds nothing', async () => {
|
|
72
|
+
const { namespace, calls } = fakeNamespace();
|
|
73
|
+
const store = createDurableChunkStore(namespace);
|
|
74
|
+
const a = await multiChunk();
|
|
75
|
+
for (const chunk of a.chunks) await store.add(chunk.params);
|
|
76
|
+
expect(new Set(calls)).toEqual(new Set([`ctgr:${a.chunks[0].params._cid}`]));
|
|
77
|
+
// Replaying the first chunk starts a fresh partial rather than re-finishing the old one.
|
|
78
|
+
expect(await store.add(a.chunks[0].params)).toBe(false);
|
|
79
|
+
});
|
|
80
|
+
|
|
81
|
+
test('a corrupted chunk is a 4xx Error in the isolate, before any object is asked', async () => {
|
|
82
|
+
const { namespace, calls } = fakeNamespace();
|
|
83
|
+
const { chunks } = await multiChunk();
|
|
84
|
+
const bad = { ...chunks[1].params, _d: `${chunks[1].params._d.slice(0, -4)}AAAA` };
|
|
85
|
+
expect(await createDurableChunkStore(namespace).add(bad)).toBeInstanceOf(Error);
|
|
86
|
+
expect(calls).toHaveLength(0);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test('framing that changes mid-payload is refused, as ChunkStore refuses it', async () => {
|
|
90
|
+
const { namespace } = fakeNamespace();
|
|
91
|
+
const store = createDurableChunkStore(namespace);
|
|
92
|
+
const { chunks } = await multiChunk();
|
|
93
|
+
const other = await encodeRequest('PATCH', '/api/v1/notes/1', (await multiChunk()).body);
|
|
94
|
+
expect(await store.add(chunks[0].params)).toBe(false);
|
|
95
|
+
const forged = { ...other.chunks[1].params, _cid: chunks[0].params._cid };
|
|
96
|
+
expect(await store.add(forged)).toBeInstanceOf(Error);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test('a partial idle past the TTL is dropped, so a late chunk cannot complete it', async () => {
|
|
100
|
+
let now = 0;
|
|
101
|
+
const { namespace } = fakeNamespace(() => now);
|
|
102
|
+
const store = createDurableChunkStore(namespace);
|
|
103
|
+
const { chunks } = await multiChunk();
|
|
104
|
+
for (const chunk of chunks.slice(0, -1)) expect(await store.add(chunk.params)).toBe(false);
|
|
105
|
+
now = 31_000;
|
|
106
|
+
expect(await store.add(chunks.at(-1)!.params)).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test('an object that cannot answer is a THROW — the tunnel turns it into a 503', async () => {
|
|
110
|
+
const down: CtgrChunkNamespaceLike = {
|
|
111
|
+
idFromName: (name) => name,
|
|
112
|
+
get: () => ({ fetch: async () => new Response('Internal error', { status: 500 }) }),
|
|
113
|
+
};
|
|
114
|
+
const app = new Hono();
|
|
115
|
+
mountCtgrTunnel(app, { store: createDurableChunkStore(down) });
|
|
116
|
+
app.post('/api/v1/notes', (c) => c.json({ reached: true }));
|
|
117
|
+
const { chunks } = await multiChunk();
|
|
118
|
+
const res = await app.fetch(new Request(`http://localhost${chunks[0].url}`));
|
|
119
|
+
expect(res.status).toBe(503);
|
|
120
|
+
expect(((await res.json()) as { code: string }).code).toBe('CTGR_STORE_UNAVAILABLE');
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test('the whole tunnel over two isolates: every handler reached once, with the body', async () => {
|
|
124
|
+
const { namespace } = fakeNamespace();
|
|
125
|
+
const reached: unknown[] = [];
|
|
126
|
+
const isolate = () => {
|
|
127
|
+
const app = new Hono();
|
|
128
|
+
mountCtgrTunnel(app, { store: createDurableChunkStore(namespace) });
|
|
129
|
+
app.post('/api/v1/notes', async (c) => {
|
|
130
|
+
reached.push(await c.req.json());
|
|
131
|
+
return c.json({ ok: true }, 201);
|
|
132
|
+
});
|
|
133
|
+
return app;
|
|
134
|
+
};
|
|
135
|
+
const apps = [isolate(), isolate()];
|
|
136
|
+
const { chunks, body } = await multiChunk();
|
|
137
|
+
const statuses: number[] = [];
|
|
138
|
+
for (const [n, chunk] of chunks.entries()) {
|
|
139
|
+
statuses.push((await apps[n % 2].fetch(new Request(`http://localhost${chunk.url}`))).status);
|
|
140
|
+
}
|
|
141
|
+
expect(statuses).toEqual([...chunks.slice(1).map(() => 202), 201]);
|
|
142
|
+
expect(reached).toEqual([body]);
|
|
143
|
+
});
|
|
144
|
+
});
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The ctgr chunk store on a Cloudflare Worker: ONE Durable Object per multi-chunk payload, so
|
|
3
|
+
* every chunk of it meets every other chunk no matter which isolate answered it.
|
|
4
|
+
*
|
|
5
|
+
* ── Why this exists (2026-09-26, desk task 2185) ─────────────────────────────
|
|
6
|
+
* A GET-only device splits any body over 1800 bytes into several `/ctgr<xx>` GETs, and
|
|
7
|
+
* {@link mountCtgrTunnel} reassembles them in a {@link ChunkStore} — memory. On a Worker, memory
|
|
8
|
+
* is one isolate: chunk 2 answered by another isolate (another colo, a recycled isolate, load
|
|
9
|
+
* spread across several) finds no chunk 1, answers 202, and the payload is silently never
|
|
10
|
+
* delivered. desk's large saves were exposed; auth's login body is one chunk.
|
|
11
|
+
*
|
|
12
|
+
* ── The object is a rendezvous, not a decoder ────────────────────────────────
|
|
13
|
+
* It keeps each chunk's raw query params by index and, when the last one arrives, hands the
|
|
14
|
+
* whole ordered set back. The Worker then runs them through a fresh {@link ChunkStore} — the
|
|
15
|
+
* SAME crc/sha/size/decode path a Mac process uses — so nothing about verification is
|
|
16
|
+
* re-implemented here, and the object never parses a body. A single-chunk twin (the norm)
|
|
17
|
+
* never reaches the object at all.
|
|
18
|
+
*
|
|
19
|
+
* ── One object per payload ───────────────────────────────────────────────────
|
|
20
|
+
* Addressed by `ctgr:<_cid>` (a `crypto.randomUUID()` the client mints per request), so the
|
|
21
|
+
* chunks of one request are serialised through one object and different requests never
|
|
22
|
+
* contend. The partial lives in the object's memory: chunks arrive milliseconds apart, the idle
|
|
23
|
+
* TTL is 30 s, and an object is only evicted after minutes idle — a partial it loses is one a
|
|
24
|
+
* deploy would have lost too. An object holding nothing costs nothing.
|
|
25
|
+
*
|
|
26
|
+
* ── Types are structural ────────────────────────────────────────────────────
|
|
27
|
+
* No `@cloudflare/workers-types`, and the classic `fetch` interface — the same reasons as
|
|
28
|
+
* `loginThrottleDurable.ts`.
|
|
29
|
+
*/
|
|
30
|
+
import { ChunkStore, type ChunkStoreOpts } from 'cursedbelt-core/ctgr/chunk-store';
|
|
31
|
+
import { parseChunk, type ParsedChunk } from 'cursedbelt-core/ctgr';
|
|
32
|
+
import type { CtgrDecoded } from 'cursedbelt-core/ctgr/types';
|
|
33
|
+
import type { CtgrChunkSink } from './ctgrTunnel.js';
|
|
34
|
+
|
|
35
|
+
/** The members of a `DurableObjectNamespace` binding the store calls. */
|
|
36
|
+
export interface CtgrChunkNamespaceLike {
|
|
37
|
+
idFromName(name: string): unknown;
|
|
38
|
+
get(id: unknown): { fetch(request: Request): Promise<Response> };
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
/** What the object answers `POST /ctgr/chunk` with. */
|
|
42
|
+
type ChunkReply = { pending: true } | { chunks: Record<string, string>[] } | { error: string };
|
|
43
|
+
|
|
44
|
+
const DEFAULT_MAX_PAYLOAD_BYTES = 10 * 1024 * 1024;
|
|
45
|
+
const DEFAULT_TTL_MS = 30_000;
|
|
46
|
+
|
|
47
|
+
const json = (value: ChunkReply, status = 200): Response =>
|
|
48
|
+
new Response(JSON.stringify(value), { status, headers: { 'content-type': 'application/json' } });
|
|
49
|
+
|
|
50
|
+
interface Partial {
|
|
51
|
+
cid: string;
|
|
52
|
+
total: number;
|
|
53
|
+
payloadLength: number;
|
|
54
|
+
gzip: boolean;
|
|
55
|
+
method: string;
|
|
56
|
+
parts: (Record<string, string> | undefined)[];
|
|
57
|
+
received: number;
|
|
58
|
+
bytes: number;
|
|
59
|
+
updatedAt: number;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* The Durable Object. Re-export it from the Worker's entry under the `class_name` your
|
|
64
|
+
* `wrangler.jsonc` binds, and add it in a migration (`new_sqlite_classes`):
|
|
65
|
+
*
|
|
66
|
+
* export { CtgrChunkObject as DeskCtgrChunks } from "cursedbelt-server/ctgr-tunnel/durable";
|
|
67
|
+
*/
|
|
68
|
+
export class CtgrChunkObject {
|
|
69
|
+
private partial: Partial | null = null;
|
|
70
|
+
private readonly maxPayloadBytes: number;
|
|
71
|
+
private readonly ttlMs: number;
|
|
72
|
+
private readonly now: () => number;
|
|
73
|
+
|
|
74
|
+
/** `options` is for tests; the runtime passes `(state, env)` only. */
|
|
75
|
+
constructor(_state?: unknown, _env?: unknown, options: ChunkStoreOpts = {}) {
|
|
76
|
+
this.maxPayloadBytes = options.maxPayloadBytes ?? DEFAULT_MAX_PAYLOAD_BYTES;
|
|
77
|
+
this.ttlMs = options.ttlMs ?? DEFAULT_TTL_MS;
|
|
78
|
+
this.now = options.now ?? Date.now;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
async fetch(request: Request): Promise<Response> {
|
|
82
|
+
const path = new URL(request.url).pathname;
|
|
83
|
+
if (request.method !== 'POST' || path !== '/ctgr/chunk') {
|
|
84
|
+
return json({ error: `${request.method} ${path}: only POST /ctgr/chunk` }, 404);
|
|
85
|
+
}
|
|
86
|
+
let params: Record<string, string>;
|
|
87
|
+
let chunk: ParsedChunk;
|
|
88
|
+
try {
|
|
89
|
+
params = (await request.json()) as Record<string, string>;
|
|
90
|
+
chunk = parseChunk(params);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
return json({ error: error instanceof Error ? error.message : String(error) }, 400);
|
|
93
|
+
}
|
|
94
|
+
return json(this.add(params, chunk));
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
private add(params: Record<string, string>, chunk: ParsedChunk): ChunkReply {
|
|
98
|
+
if (this.partial && this.partial.updatedAt < this.now() - this.ttlMs) this.partial = null;
|
|
99
|
+
if (chunk.payloadLength > this.maxPayloadBytes) return { error: 'ctgr: payload exceeds size cap' };
|
|
100
|
+
|
|
101
|
+
let entry = this.partial;
|
|
102
|
+
if (!entry) {
|
|
103
|
+
entry = {
|
|
104
|
+
cid: chunk.cid,
|
|
105
|
+
total: chunk.total,
|
|
106
|
+
payloadLength: chunk.payloadLength,
|
|
107
|
+
gzip: chunk.gzip,
|
|
108
|
+
method: chunk.method,
|
|
109
|
+
parts: new Array(chunk.total),
|
|
110
|
+
received: 0,
|
|
111
|
+
bytes: 0,
|
|
112
|
+
updatedAt: this.now(),
|
|
113
|
+
};
|
|
114
|
+
this.partial = entry;
|
|
115
|
+
} else if (
|
|
116
|
+
entry.cid !== chunk.cid ||
|
|
117
|
+
entry.total !== chunk.total ||
|
|
118
|
+
entry.payloadLength !== chunk.payloadLength ||
|
|
119
|
+
entry.gzip !== chunk.gzip ||
|
|
120
|
+
entry.method !== chunk.method
|
|
121
|
+
) {
|
|
122
|
+
// A payload's framing must not change mid-stream — the same rule `ChunkStore` keeps.
|
|
123
|
+
this.partial = null;
|
|
124
|
+
return { error: 'ctgr: inconsistent chunk metadata' };
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
entry.updatedAt = this.now();
|
|
128
|
+
if (entry.parts[chunk.index] === undefined) {
|
|
129
|
+
entry.parts[chunk.index] = params;
|
|
130
|
+
entry.received += 1;
|
|
131
|
+
entry.bytes += chunk.bytes.length;
|
|
132
|
+
if (entry.bytes > this.maxPayloadBytes) {
|
|
133
|
+
this.partial = null;
|
|
134
|
+
return { error: 'ctgr: payload exceeds size cap' };
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if (entry.received < entry.total) return { pending: true };
|
|
138
|
+
|
|
139
|
+
this.partial = null;
|
|
140
|
+
return { chunks: entry.parts as Record<string, string>[] };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/**
|
|
145
|
+
* The Worker half: a {@link CtgrChunkSink} for {@link mountCtgrTunnel} whose multi-chunk
|
|
146
|
+
* partials live in the object. Cheap to build — build it per request over `env.<BINDING>`.
|
|
147
|
+
*
|
|
148
|
+
* An object that cannot be reached THROWS, and the tunnel answers 503: a payload half-held in
|
|
149
|
+
* one isolate's memory is exactly the silent loss this exists to end, so there is no fallback.
|
|
150
|
+
*/
|
|
151
|
+
export function createDurableChunkStore(
|
|
152
|
+
namespace: CtgrChunkNamespaceLike,
|
|
153
|
+
options: ChunkStoreOpts = {},
|
|
154
|
+
): CtgrChunkSink {
|
|
155
|
+
return {
|
|
156
|
+
async add(input) {
|
|
157
|
+
const params =
|
|
158
|
+
input instanceof URLSearchParams ? Object.fromEntries(input) : { ...input };
|
|
159
|
+
let chunk: ParsedChunk;
|
|
160
|
+
try {
|
|
161
|
+
chunk = parseChunk(params);
|
|
162
|
+
} catch (error) {
|
|
163
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
164
|
+
}
|
|
165
|
+
// One chunk needs no rendezvous — decode it here, in this isolate.
|
|
166
|
+
if (chunk.total === 1) return new ChunkStore(options).add(params);
|
|
167
|
+
|
|
168
|
+
const stub = namespace.get(namespace.idFromName(`ctgr:${chunk.cid}`));
|
|
169
|
+
const response = await stub.fetch(
|
|
170
|
+
new Request('https://ctgr-chunks.invalid/ctgr/chunk', {
|
|
171
|
+
method: 'POST',
|
|
172
|
+
headers: { 'content-type': 'application/json' },
|
|
173
|
+
body: JSON.stringify(params),
|
|
174
|
+
}),
|
|
175
|
+
);
|
|
176
|
+
const reply = (await response.json().catch(() => null)) as ChunkReply | null;
|
|
177
|
+
if (!reply) throw new Error(`ctgr chunk object answered ${response.status} with no JSON`);
|
|
178
|
+
if ('error' in reply) return new Error(reply.error);
|
|
179
|
+
if ('pending' in reply) return false;
|
|
180
|
+
|
|
181
|
+
// Every chunk is here: the one verify/decode path, in a store that lives for this call.
|
|
182
|
+
const assemble = new ChunkStore(options);
|
|
183
|
+
let result: CtgrDecoded | false | Error = new Error('ctgr: empty chunk set');
|
|
184
|
+
for (const part of reply.chunks) result = await assemble.add(part);
|
|
185
|
+
return result;
|
|
186
|
+
},
|
|
187
|
+
};
|
|
188
|
+
}
|
package/src/server/ctgrTunnel.ts
CHANGED
|
@@ -28,23 +28,36 @@ import { CTGR_ROUTE_PATTERN } from './ctgr.js';
|
|
|
28
28
|
const isCtgrParam = (key: string): boolean =>
|
|
29
29
|
key === '_d' || key === '_sha' || key.startsWith('_c');
|
|
30
30
|
|
|
31
|
+
/**
|
|
32
|
+
* What the tunnel needs of an accumulator: a {@link ChunkStore}, or the Worker's
|
|
33
|
+
* `createDurableChunkStore` (`cursedbelt-server/ctgr-tunnel/durable`), whose partials every
|
|
34
|
+
* isolate shares. `false` is an intermediate chunk (→ 202), an `Error` a bad payload (→ 400);
|
|
35
|
+
* a THROW means the store itself could not answer (→ 503).
|
|
36
|
+
*/
|
|
37
|
+
export interface CtgrChunkSink {
|
|
38
|
+
add(params: Record<string, string> | URLSearchParams): Promise<CtgrDecoded | false | Error>;
|
|
39
|
+
}
|
|
40
|
+
|
|
31
41
|
export interface CtgrTunnelOptions {
|
|
32
42
|
/**
|
|
33
43
|
* The accumulator for multi-chunk payloads. Share ONE across an app (the default is
|
|
34
44
|
* a fresh singleton); pass your own to tune the DoS caps ({@link ChunkStore}).
|
|
45
|
+
* 🔴 On a Cloudflare Worker a module-scope `ChunkStore` is one ISOLATE's memory — pass
|
|
46
|
+
* `createDurableChunkStore(env.<BINDING>)` there, or a multi-chunk payload whose chunks
|
|
47
|
+
* land on different isolates is silently never delivered.
|
|
35
48
|
*/
|
|
36
|
-
store?:
|
|
49
|
+
store?: CtgrChunkSink;
|
|
37
50
|
}
|
|
38
51
|
|
|
39
52
|
/**
|
|
40
53
|
* Mount the GET-only tunnel on `app`. Call once, on the Hono app whose real routes sit
|
|
41
54
|
* at the paths the twins mirror (a twin path is `/ctgr<xx>` + the real path). Returns
|
|
42
|
-
* the
|
|
55
|
+
* the store in use, for tests and metrics.
|
|
43
56
|
*/
|
|
44
57
|
export function mountCtgrTunnel<E extends Env = Env>(
|
|
45
58
|
app: Hono<E>,
|
|
46
59
|
options: CtgrTunnelOptions = {},
|
|
47
|
-
):
|
|
60
|
+
): CtgrChunkSink {
|
|
48
61
|
const store = options.store ?? new ChunkStore();
|
|
49
62
|
const legacy = new V0ChunkStore();
|
|
50
63
|
|
|
@@ -57,8 +70,13 @@ export function mountCtgrTunnel<E extends Env = Env>(
|
|
|
57
70
|
return c.json({ error: 'not found', code: 'NOT_FOUND' }, 404);
|
|
58
71
|
}
|
|
59
72
|
|
|
60
|
-
|
|
61
|
-
|
|
73
|
+
let result: CtgrDecoded | false | Error;
|
|
74
|
+
try {
|
|
75
|
+
result = params._cv === undefined ? legacy.add(params, pathMethod) : await store.add(params);
|
|
76
|
+
} catch (error) {
|
|
77
|
+
console.error(`[ctgr] the chunk store could not answer: ${error instanceof Error ? error.message : String(error)}`);
|
|
78
|
+
return c.json({ error: 'ctgr: the chunk store could not answer — send it again', code: 'CTGR_STORE_UNAVAILABLE' }, 503);
|
|
79
|
+
}
|
|
62
80
|
|
|
63
81
|
if (result === false) return c.json({ acknowledged: true }, 202);
|
|
64
82
|
if (result instanceof Error) {
|