@podosoft/podokit 1.3.0 → 1.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/add.d.ts +10 -1
- package/dist/add.js +20 -7
- package/dist/api-paths.d.ts +34 -0
- package/dist/api-paths.js +103 -0
- package/dist/deploy-compose-profile.d.ts +4 -0
- package/dist/deploy-compose-profile.js +13 -2
- package/dist/deploy-compose-render.d.ts +1 -1
- package/dist/deploy-compose-render.js +23 -7
- package/dist/deploy-compose.js +2 -1
- package/dist/deploy-profile.d.ts +3 -0
- package/dist/deploy-profile.js +12 -4
- package/dist/deploy-render.js +54 -8
- package/dist/deploy.js +21 -1
- package/dist/dev.d.ts +14 -0
- package/dist/dev.js +152 -10
- package/dist/index.js +15 -2
- package/dist/lib.d.ts +2 -0
- package/dist/lib.js +4 -2
- package/dist/lockfile.d.ts +3 -0
- package/dist/lockfile.js +11 -0
- package/dist/module-api-paths.d.ts +52 -0
- package/dist/module-api-paths.js +189 -0
- package/dist/remove.d.ts +3 -0
- package/dist/remove.js +12 -3
- package/dist/templates/fullstack/AGENTS.md +27 -5
- package/dist/templates/fullstack/README.md +28 -11
- package/dist/templates/fullstack/apps/api/package.json +2 -2
- package/dist/templates/fullstack/apps/api/scripts/run-migrations.mjs +25 -0
- package/dist/templates/fullstack/apps/api/src/app.extensions.ts +7 -0
- package/dist/templates/fullstack/apps/api/src/app.spec.ts +20 -1
- package/dist/templates/fullstack/apps/api/src/app.ts +11 -0
- package/dist/templates/fullstack/apps/api/src/config/env.validation.ts +3 -0
- package/dist/templates/fullstack/apps/api/src/config/redis.connection.spec.ts +48 -2
- package/dist/templates/fullstack/apps/api/src/config/redis.connection.ts +45 -0
- package/dist/templates/fullstack/apps/api/src/config/server-options.spec.ts +46 -0
- package/dist/templates/fullstack/apps/api/src/config/server-options.ts +48 -0
- package/dist/templates/fullstack/apps/api/src/core/api-contract.spec.ts +1 -0
- package/dist/templates/fullstack/apps/api/src/core/api-contract.ts +9 -1
- package/dist/templates/fullstack/apps/api/src/core/services.spec.ts +196 -0
- package/dist/templates/fullstack/apps/api/src/core/services.ts +181 -2
- package/dist/templates/fullstack/apps/api/src/database/database.spec.ts +26 -1
- package/dist/templates/fullstack/apps/api/src/database/database.ts +52 -11
- package/dist/templates/fullstack/apps/api/src/database/pool-guard.spec.ts +188 -0
- package/dist/templates/fullstack/apps/api/src/database/pool-guard.ts +151 -0
- package/dist/templates/fullstack/apps/api/src/database/run-migrations.spec.ts +70 -0
- package/dist/templates/fullstack/apps/api/src/health/health.plugin.spec.ts +28 -4
- package/dist/templates/fullstack/apps/api/src/health/health.plugin.ts +10 -2
- package/dist/templates/fullstack/apps/api/src/main.ts +2 -1
- package/dist/templates/fullstack/apps/web/components.json +3 -1
- package/dist/templates/fullstack/apps/web/package.json +1 -1
- package/dist/templates/fullstack/apps/web/src/error.html +73 -0
- package/dist/templates/fullstack/apps/web/src/lib/components/ui/textarea/index.ts +7 -0
- package/dist/templates/fullstack/apps/web/src/lib/components/ui/textarea/textarea.svelte +22 -0
- package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.test.ts +276 -1
- package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.ts +194 -9
- package/dist/templates/fullstack/apps/web/src/lib/server/body-limits.ts +15 -0
- package/dist/templates/fullstack/apps/web/src/lib/server/proxy-headers.ts +12 -0
- package/dist/templates/fullstack/dot-agents/skills/podokit-deploy/references/profile.md +10 -3
- package/dist/templates/fullstack/dot-claude/skills/podokit-elysia-endpoint/SKILL.md +5 -0
- package/dist/templates/fullstack/dot-claude/skills/podokit-sveltekit-route/SKILL.md +7 -1
- package/dist/templates/fullstack/dot-devcontainer/devcontainer.json +1 -0
- package/dist/templates/fullstack/{dot-env.docker → dot-env.docker.example} +7 -4
- package/dist/templates/fullstack/dot-env.example +10 -0
- package/dist/templates/fullstack/dot-gitignore +1 -0
- package/dist/templates/fullstack/dot-podokit/dev.json +2 -1
- package/dist/templates/fullstack/infra/k3s/README.md +3 -1
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/admin-sessions/admin-sessions.migration.spec.ts +32 -3
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/bootstrap-admin-runtime.ts +1 -1
- package/dist/templates/modules/admin-dashboard/files/apps/api/src/migrations/1792060000000-IndexSessionCreatedAt.ts +19 -1
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/hooks.server.ts +27 -9
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/asset-paths.ts +11 -0
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.test.ts +125 -0
- package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.ts +107 -0
- package/dist/templates/modules/admin-dashboard/files/tests/api/profile-image.api.spec.ts +2 -1
- package/dist/templates/modules/admin-dashboard/files/tests/api/site-settings.api.spec.ts +2 -1
- package/dist/templates/modules/admin-dashboard/files/tests/helpers/disposable-users.ts +10 -10
- package/dist/templates/modules/admin-dashboard/files/tests/helpers/object-response.ts +12 -0
- package/dist/templates/modules/admin-dashboard/module.manifest.json +3 -2
- package/dist/templates/modules/audit-log/module.manifest.json +1 -1
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth-provider.spec.ts +26 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth-provider.ts +17 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.service.ts +8 -4
- package/dist/templates/modules/auth/files/apps/api/src/auth/auth.ts +2 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.spec.ts +114 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.ts +50 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/db.spec.ts +4 -1
- package/dist/templates/modules/auth/files/apps/api/src/auth/db.ts +11 -2
- package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance-command.ts +5 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance.ts +14 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.spec.ts +136 -0
- package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.ts +40 -0
- package/dist/templates/modules/auth/files/apps/api/src/migrate.ts +3 -2
- package/dist/templates/modules/auth/module.manifest.json +0 -1
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts +121 -1
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.ts +108 -10
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/jobs.module.ts +3 -1
- package/dist/templates/modules/bullmq/files/apps/api/src/jobs/queue.ts +31 -3
- package/dist/templates/modules/bullmq/module.manifest.json +5 -1
- package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/events-redis.module.ts +2 -4
- package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.spec.ts +224 -0
- package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.ts +124 -35
- package/dist/templates/modules/events-redis/module.manifest.json +4 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.routes.ts +136 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.spec.ts +636 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.ts +673 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.spec.ts +10 -4
- package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.ts +37 -8
- package/dist/templates/modules/file-upload/files/apps/api/src/files/upload-config.ts +99 -0
- package/dist/templates/modules/file-upload/files/apps/api/src/migrations/1792100000000-InitFileUploadSessions.ts +49 -0
- package/dist/templates/modules/file-upload/files/apps/web/src/lib/uploads.test.ts +456 -0
- package/dist/templates/modules/file-upload/files/apps/web/src/lib/uploads.ts +356 -0
- package/dist/templates/modules/file-upload/files/tests/api/chunked-uploads.api.spec.ts +153 -0
- package/dist/templates/modules/file-upload/module.manifest.json +34 -6
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/jobs-local.module.ts +1 -1
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts +23 -0
- package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.ts +30 -20
- package/dist/templates/modules/jobs-local/module.manifest.json +3 -0
- package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/download-url.ts +20 -0
- package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.spec.ts +79 -12
- package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.ts +16 -1
- package/dist/templates/modules/object-storage-local/module.manifest.json +11 -2
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-multipart.spec.ts +115 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-multipart.ts +171 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.spec.ts +222 -9
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.ts +55 -19
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-signer.spec.ts +71 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-signer.ts +98 -0
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage-demo.spec.ts +6 -3
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.config.ts +7 -2
- package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.service.ts +187 -44
- package/dist/templates/modules/object-storage-s3/module.manifest.json +10 -2
- package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.module.ts +11 -22
- package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.routes.spec.ts +113 -0
- package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.routes.ts +122 -0
- package/dist/templates/modules/rate-limit/files/tests/api/rate-limit.api.spec.ts +24 -0
- package/dist/templates/modules/rate-limit/module.manifest.json +7 -0
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis-cache.store.spec.ts +14 -1
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis-cache.store.ts +9 -12
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis.module.ts +16 -4
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis.service.spec.ts +233 -5
- package/dist/templates/modules/redis/files/apps/api/src/redis/redis.service.ts +134 -32
- package/dist/templates/modules/redis/module.manifest.json +4 -0
- package/dist/templates/modules/sse/files/apps/api/src/events/event-stream.ts +11 -4
- package/dist/templates/modules/sse/files/apps/api/src/events/events.service.spec.ts +33 -0
- package/dist/templates/modules/sse/files/apps/api/src/events/events.service.ts +7 -0
- package/dist/templates/todo/AGENTS.md +19 -4
- package/dist/templates/todo/README.md +27 -12
- package/dist/update.d.ts +5 -0
- package/dist/update.js +35 -0
- package/package.json +2 -2
- package/dist/templates/modules/auth/files/apps/api/scripts/run-migrations.mjs +0 -14
- package/dist/templates/modules/auth/files/apps/api/src/auth/schema-validation.ts +0 -13
- package/dist/templates/modules/auth/files/apps/api/src/auth/skip-schema-validation.ts +0 -5
package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.service.ts
CHANGED
|
@@ -1,13 +1,24 @@
|
|
|
1
1
|
import { S3Client, type S3Stats } from "bun";
|
|
2
2
|
import {
|
|
3
|
+
IN_MEMORY_READ_BYTES,
|
|
4
|
+
objectBodyChunks,
|
|
3
5
|
objectListLimit,
|
|
6
|
+
type CompletedPart,
|
|
4
7
|
type ListObjectsOptions,
|
|
8
|
+
type ObjectBody,
|
|
5
9
|
type ObjectData,
|
|
6
10
|
type ObjectInfo,
|
|
7
11
|
type ObjectPage,
|
|
12
|
+
type UploadedPart,
|
|
8
13
|
} from "@podosoft/podokit-runtime";
|
|
9
14
|
import { ReadinessService } from "../health/readiness.service";
|
|
10
|
-
import {
|
|
15
|
+
import { S3Multipart } from "./s3-multipart";
|
|
16
|
+
import { storageSettings, type StorageSettings } from "./storage.config";
|
|
17
|
+
|
|
18
|
+
/** Part size for streamed writes. */
|
|
19
|
+
export const STREAM_PART_BYTES = 8 * 1024 * 1024;
|
|
20
|
+
/** Parts of one streamed write that may be in flight at once. */
|
|
21
|
+
const STREAM_QUEUE_SIZE = 2;
|
|
11
22
|
|
|
12
23
|
function isMissingObject(error: unknown): boolean {
|
|
13
24
|
return error instanceof Error && "code" in error && error.code === "NoSuchKey";
|
|
@@ -27,46 +38,56 @@ function objectInfo(key: string, stats: S3Stats): ObjectInfo {
|
|
|
27
38
|
};
|
|
28
39
|
}
|
|
29
40
|
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
41
|
+
function downloadedObjectInfo(key: string, headers: Headers): ObjectInfo {
|
|
42
|
+
const length = headers.get("content-length");
|
|
43
|
+
const etag = headers.get("etag");
|
|
44
|
+
const lastModified = new Date(headers.get("last-modified") ?? "");
|
|
45
|
+
const contentType = headers.get("content-type");
|
|
46
|
+
if (length === null || !/^\d+$/.test(length) || !etag || Number.isNaN(lastModified.getTime())) {
|
|
47
|
+
throw new Error(`Object ${key} download lacks its length, entity tag, or modification time`);
|
|
48
|
+
}
|
|
49
|
+
return {
|
|
50
|
+
key,
|
|
51
|
+
size: Number(length),
|
|
52
|
+
...(contentType ? { contentType } : {}),
|
|
53
|
+
etag: entityTag(etag),
|
|
54
|
+
lastModified,
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The S3 error code of a failed request, such as NoSuchKey or NoSuchBucket. */
|
|
59
|
+
async function errorCode(response: Response): Promise<string | undefined> {
|
|
60
|
+
const text = await response.text().catch(() => "");
|
|
61
|
+
return /<Code>([^<]+)<\/Code>/.exec(text)?.[1];
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function multipartClient(settings: StorageSettings): S3Multipart | undefined {
|
|
65
|
+
if (!settings.accessKeyId || !settings.secretAccessKey) return undefined;
|
|
66
|
+
return new S3Multipart({
|
|
67
|
+
bucket: settings.bucket,
|
|
68
|
+
region: settings.region,
|
|
69
|
+
virtualHostedStyle: settings.virtualHostedStyle,
|
|
70
|
+
...(settings.endpoint ? { endpoint: settings.endpoint } : {}),
|
|
71
|
+
credentials: {
|
|
72
|
+
accessKeyId: settings.accessKeyId,
|
|
73
|
+
secretAccessKey: settings.secretAccessKey,
|
|
74
|
+
...(settings.sessionToken ? { sessionToken: settings.sessionToken } : {}),
|
|
64
75
|
},
|
|
65
|
-
}
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/** S3 needs each part's length before it is sent, so a streamed part is collected first. */
|
|
80
|
+
async function partBytes(body: ObjectBody): Promise<Uint8Array | Blob> {
|
|
81
|
+
if (body instanceof Uint8Array || body instanceof Blob) return body;
|
|
82
|
+
if (typeof body === "string") return new TextEncoder().encode(body);
|
|
83
|
+
const chunks: Uint8Array[] = [];
|
|
84
|
+
for await (const chunk of objectBodyChunks(body)) chunks.push(chunk);
|
|
85
|
+
return Buffer.concat(chunks);
|
|
66
86
|
}
|
|
67
87
|
|
|
68
88
|
export class StorageService {
|
|
69
89
|
private readonly client: S3Client;
|
|
90
|
+
private readonly multipartRequests: S3Multipart | undefined;
|
|
70
91
|
private unregisterReadiness?: () => void;
|
|
71
92
|
|
|
72
93
|
constructor(readiness: ReadinessService) {
|
|
@@ -78,14 +99,108 @@ export class StorageService {
|
|
|
78
99
|
...(settings.endpoint ? { endpoint: settings.endpoint } : {}),
|
|
79
100
|
...(settings.accessKeyId ? { accessKeyId: settings.accessKeyId } : {}),
|
|
80
101
|
...(settings.secretAccessKey ? { secretAccessKey: settings.secretAccessKey } : {}),
|
|
102
|
+
...(settings.sessionToken ? { sessionToken: settings.sessionToken } : {}),
|
|
81
103
|
});
|
|
104
|
+
this.multipartRequests = multipartClient(settings);
|
|
82
105
|
this.unregisterReadiness = readiness.register("object-storage", async () => {
|
|
83
106
|
await this.client.exists(".podokit-readiness");
|
|
84
107
|
});
|
|
85
108
|
}
|
|
86
109
|
|
|
87
|
-
|
|
88
|
-
|
|
110
|
+
/**
|
|
111
|
+
* Writes an object and resolves with its size. Bytes, strings, and small
|
|
112
|
+
* blobs are sent in one request. Streams, async iterables, and larger blobs
|
|
113
|
+
* (including files) are sent in parts, holding about three parts in memory
|
|
114
|
+
* however large the object is; a source that fails aborts the upload, so no
|
|
115
|
+
* partial object is left behind.
|
|
116
|
+
*/
|
|
117
|
+
async put(key: string, body: ObjectBody, contentType?: string): Promise<number> {
|
|
118
|
+
const options = contentType ? { type: contentType } : undefined;
|
|
119
|
+
if (
|
|
120
|
+
typeof body === "string" ||
|
|
121
|
+
body instanceof Uint8Array ||
|
|
122
|
+
(body instanceof Blob && body.size <= STREAM_PART_BYTES)
|
|
123
|
+
) {
|
|
124
|
+
await this.client.file(key).write(body, options);
|
|
125
|
+
return typeof body === "string" ? Buffer.byteLength(body) : body instanceof Blob ? body.size : body.byteLength;
|
|
126
|
+
}
|
|
127
|
+
// Bun's S3 write reads a whole Blob, file-backed ones included, into memory,
|
|
128
|
+
// and its streaming writer completes rather than aborts when ended with an
|
|
129
|
+
// error, so streamed bodies use explicit multipart requests.
|
|
130
|
+
const parts: CompletedPart[] = [];
|
|
131
|
+
const pending = new Set<Promise<void>>();
|
|
132
|
+
let failure: { error: unknown } | undefined;
|
|
133
|
+
let uploadId: string | undefined;
|
|
134
|
+
let nextPartNumber = 1;
|
|
135
|
+
// Part buffers are reused once their upload finishes, so a long stream
|
|
136
|
+
// allocates at most one buffer per queue slot plus the one being filled.
|
|
137
|
+
const spare: Uint8Array[] = [];
|
|
138
|
+
let part: Uint8Array = new Uint8Array(STREAM_PART_BYTES);
|
|
139
|
+
let filled = 0;
|
|
140
|
+
let size = 0;
|
|
141
|
+
const send = async (buffer: Uint8Array, length: number): Promise<void> => {
|
|
142
|
+
uploadId ??= await this.multipart().create(key, contentType);
|
|
143
|
+
const id = uploadId;
|
|
144
|
+
const partNumber = nextPartNumber++;
|
|
145
|
+
const upload: Promise<void> = this.multipart()
|
|
146
|
+
.uploadPart(key, id, partNumber, buffer.subarray(0, length))
|
|
147
|
+
.then((etag) => { parts.push({ partNumber, etag }); })
|
|
148
|
+
.catch((error: unknown) => { failure ??= { error }; })
|
|
149
|
+
.finally(() => {
|
|
150
|
+
pending.delete(upload);
|
|
151
|
+
spare.push(buffer);
|
|
152
|
+
});
|
|
153
|
+
pending.add(upload);
|
|
154
|
+
while (pending.size >= STREAM_QUEUE_SIZE) await Promise.race(pending);
|
|
155
|
+
if (failure) throw failure.error;
|
|
156
|
+
};
|
|
157
|
+
try {
|
|
158
|
+
for await (const chunk of objectBodyChunks(body)) {
|
|
159
|
+
size += chunk.byteLength;
|
|
160
|
+
for (let offset = 0; offset < chunk.byteLength;) {
|
|
161
|
+
const length = Math.min(chunk.byteLength - offset, STREAM_PART_BYTES - filled);
|
|
162
|
+
part.set(chunk.subarray(offset, offset + length), filled);
|
|
163
|
+
filled += length;
|
|
164
|
+
offset += length;
|
|
165
|
+
if (filled === STREAM_PART_BYTES) {
|
|
166
|
+
await send(part, filled);
|
|
167
|
+
part = spare.pop() ?? new Uint8Array(STREAM_PART_BYTES);
|
|
168
|
+
filled = 0;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
if (uploadId === undefined) {
|
|
173
|
+
await this.client.file(key).write(part.subarray(0, filled), options);
|
|
174
|
+
return size;
|
|
175
|
+
}
|
|
176
|
+
if (filled > 0) await send(part, filled);
|
|
177
|
+
await Promise.all(pending);
|
|
178
|
+
if (failure) throw failure.error;
|
|
179
|
+
await this.multipart().complete(key, uploadId, parts.sort((left, right) => left.partNumber - right.partNumber));
|
|
180
|
+
return size;
|
|
181
|
+
} catch (error) {
|
|
182
|
+
await Promise.allSettled(pending);
|
|
183
|
+
if (uploadId !== undefined) await this.multipart().abort(key, uploadId).catch(() => undefined);
|
|
184
|
+
throw error;
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
async createMultipartUpload(key: string, contentType?: string): Promise<string> {
|
|
189
|
+
return this.multipart().create(key, contentType);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async uploadPart(key: string, uploadId: string, partNumber: number, body: ObjectBody): Promise<UploadedPart> {
|
|
193
|
+
const bytes = await partBytes(body);
|
|
194
|
+
const etag = await this.multipart().uploadPart(key, uploadId, partNumber, bytes);
|
|
195
|
+
return { etag, size: bytes instanceof Blob ? bytes.size : bytes.byteLength };
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
async completeMultipartUpload(key: string, uploadId: string, parts: readonly CompletedPart[]): Promise<void> {
|
|
199
|
+
await this.multipart().complete(key, uploadId, parts);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
async abortMultipartUpload(key: string, uploadId: string): Promise<void> {
|
|
203
|
+
await this.multipart().abort(key, uploadId);
|
|
89
204
|
}
|
|
90
205
|
|
|
91
206
|
/** Reads a whole object into memory. Use `open()` for anything that can be large. */
|
|
@@ -105,14 +220,35 @@ export class StorageService {
|
|
|
105
220
|
}
|
|
106
221
|
|
|
107
222
|
/**
|
|
108
|
-
* Metadata plus a streamed body, or null when the key does not exist.
|
|
109
|
-
*
|
|
110
|
-
*
|
|
223
|
+
* Metadata plus a streamed body, or null when the key does not exist. One
|
|
224
|
+
* GET describes and streams the same version, so the metadata always matches
|
|
225
|
+
* the content. Objects up to `IN_MEMORY_READ_BYTES` are read before this
|
|
226
|
+
* resolves, so a response built from them keeps its Content-Length. Larger
|
|
227
|
+
* ones stream from the GET, paced by the socket; Bun's `S3File.stream()` reads
|
|
228
|
+
* ahead without backpressure and would hold the rest of the object for a slow
|
|
229
|
+
* client. That download is already open, so read the body to the end or
|
|
230
|
+
* cancel it.
|
|
111
231
|
*/
|
|
112
232
|
async open(key: string): Promise<ObjectData | null> {
|
|
113
|
-
const
|
|
114
|
-
if (!
|
|
115
|
-
|
|
233
|
+
const response = await fetch(this.client.presign(key, { expiresIn: 60 }), { decompress: false });
|
|
234
|
+
if (!response.ok || !response.body) {
|
|
235
|
+
const code = await errorCode(response);
|
|
236
|
+
if (response.status === 404 && code === "NoSuchKey") return null;
|
|
237
|
+
throw new Error(`Object ${key} download failed with status ${response.status}${code ? ` (${code})` : ""}`);
|
|
238
|
+
}
|
|
239
|
+
let info: ObjectInfo;
|
|
240
|
+
try {
|
|
241
|
+
info = downloadedObjectInfo(key, response.headers);
|
|
242
|
+
} catch (error) {
|
|
243
|
+
await response.body.cancel().catch(() => undefined);
|
|
244
|
+
throw error;
|
|
245
|
+
}
|
|
246
|
+
if (info.size > IN_MEMORY_READ_BYTES) return { ...info, body: response.body };
|
|
247
|
+
const bytes = await response.bytes();
|
|
248
|
+
if (bytes.byteLength !== info.size) {
|
|
249
|
+
throw new Error(`Object ${key} download ended after ${bytes.byteLength} of ${info.size} bytes`);
|
|
250
|
+
}
|
|
251
|
+
return { ...info, body: new Blob([bytes]).stream() };
|
|
116
252
|
}
|
|
117
253
|
|
|
118
254
|
/** One page of the keys under `prefix`; the bucket is never listed in full. */
|
|
@@ -143,4 +279,11 @@ export class StorageService {
|
|
|
143
279
|
close(): void {
|
|
144
280
|
this.unregisterReadiness?.();
|
|
145
281
|
}
|
|
282
|
+
|
|
283
|
+
private multipart(): S3Multipart {
|
|
284
|
+
if (!this.multipartRequests) {
|
|
285
|
+
throw new Error("Multipart uploads need S3_ACCESS_KEY_ID and S3_SECRET_ACCESS_KEY (or the AWS_* equivalents)");
|
|
286
|
+
}
|
|
287
|
+
return this.multipartRequests;
|
|
288
|
+
}
|
|
146
289
|
}
|
|
@@ -23,7 +23,13 @@
|
|
|
23
23
|
],
|
|
24
24
|
"inject": [
|
|
25
25
|
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:imports", "text": "import { storageModule } from \"./storage/storage.module\";" },
|
|
26
|
-
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:modules", "text": "storageModule," }
|
|
26
|
+
{ "file": "apps/api/src/app.ts", "marker": "// podokit:end:modules", "text": "storageModule," },
|
|
27
|
+
{
|
|
28
|
+
"file": "AGENTS.md",
|
|
29
|
+
"marker": "<!-- podokit:end:agents-modules -->",
|
|
30
|
+
"optional": true,
|
|
31
|
+
"text": "### object-storage-s3\n- Store files through `OBJECT_STORAGE` from `@podosoft/podokit-runtime` (`services.resolve(OBJECT_STORAGE)`), not the provider directly. `put(key, body, { contentType })` streams a `ReadableStream` such as `request.body`, a `Blob` or file, an async iterable, bytes, or a string without buffering it; a failed body leaves no partial object.\n- For uploads split across requests (resumable or chunked), use `createMultipartUpload`, `uploadPart` (parts numbered 1 to 10,000, each but the last at least `MIN_MULTIPART_PART_BYTES` = 5 MiB), `completeMultipartUpload`, and `abortMultipartUpload`. Branch on `MultipartUploadError.code` (`UPLOAD_NOT_FOUND`, `INVALID_PART`, `PART_TOO_SMALL`).\n- Multipart requests are signed with the `S3_*` (or `AWS_*`) credentials. A part passed as a stream is held in memory while it is sent, so keep parts to tens of MiB. On AWS, add an `AbortIncompleteMultipartUpload` bucket lifecycle rule.\n"
|
|
32
|
+
}
|
|
27
33
|
],
|
|
28
34
|
"instructions": [
|
|
29
35
|
"Start Silo for local dev: docker compose -f infra/docker/docker-compose.yml -f infra/docker/minio.compose.yml up -d",
|
|
@@ -32,6 +38,8 @@
|
|
|
32
38
|
"Store an object: curl -b cookies.txt -XPUT localhost:5002/storage/hello -H 'content-type: application/json' -d '{\"content\":\"hi\"}'",
|
|
33
39
|
"Read it back: curl -b cookies.txt localhost:5002/storage/hello",
|
|
34
40
|
"Presigned URL: curl -b cookies.txt localhost:5002/storage/hello/presigned",
|
|
35
|
-
"For AWS S3: set STORAGE_PROVIDER=aws, remove S3_ENDPOINT, set S3_FORCE_PATH_STYLE=false and real credentials/bucket."
|
|
41
|
+
"For AWS S3: set STORAGE_PROVIDER=aws, remove S3_ENDPOINT, set S3_FORCE_PATH_STYLE=false and real credentials/bucket.",
|
|
42
|
+
"OBJECT_STORAGE.put streams large or streamed bodies as a multipart upload; uploads split across requests use createMultipartUpload, uploadPart, completeMultipartUpload, and abortMultipartUpload.",
|
|
43
|
+
"On AWS, add a bucket lifecycle rule with AbortIncompleteMultipartUpload so abandoned multipart uploads are removed."
|
|
36
44
|
]
|
|
37
45
|
}
|
package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.module.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { CACHE, type CacheStore } from "@podosoft/podokit-runtime";
|
|
1
|
+
import { CACHE, RATE_LIMIT_ROUTES, type CacheStore } from "@podosoft/podokit-runtime";
|
|
2
2
|
import { AppException } from "../common/app-exception";
|
|
3
3
|
import { API_KEY_VERIFIER } from "../api-key/api-key.module";
|
|
4
4
|
import {
|
|
@@ -13,6 +13,10 @@ import {
|
|
|
13
13
|
type RateLimitConfig,
|
|
14
14
|
} from "./rate-limit.config";
|
|
15
15
|
import { RateLimitIdentity } from "./rate-limit.identity";
|
|
16
|
+
import { defaultRateLimitRoutes, generalProfile, type RateLimitRoutes } from "./rate-limit.routes";
|
|
17
|
+
|
|
18
|
+
export { RATE_LIMIT_ROUTES } from "@podosoft/podokit-runtime";
|
|
19
|
+
export type { RateLimitProfile, RateLimitRoute, RateLimitRouteRegistry } from "@podosoft/podokit-runtime";
|
|
16
20
|
|
|
17
21
|
export interface RateLimitIncrement {
|
|
18
22
|
count: number;
|
|
@@ -38,13 +42,12 @@ export class CacheRateLimitStorage implements RateLimitStorage {
|
|
|
38
42
|
}
|
|
39
43
|
}
|
|
40
44
|
|
|
41
|
-
type LimitProfile = { name: string; ttlSeconds: number; limit: number };
|
|
42
|
-
|
|
43
45
|
export class RateLimiter {
|
|
44
46
|
constructor(
|
|
45
47
|
private readonly config: RateLimitConfig,
|
|
46
48
|
private readonly identity: RateLimitIdentity,
|
|
47
49
|
private readonly storage: RateLimitStorage,
|
|
50
|
+
readonly routes: RateLimitRoutes = defaultRateLimitRoutes(config),
|
|
48
51
|
) {}
|
|
49
52
|
|
|
50
53
|
async enforce(context: RequestGuardContext): Promise<void> {
|
|
@@ -55,8 +58,9 @@ export class RateLimiter {
|
|
|
55
58
|
["/health", "/health/ready", "/api-docs", "/api-docs-json", "/api-docs-elysia-json"].includes(path)
|
|
56
59
|
) return;
|
|
57
60
|
|
|
61
|
+
const profile = this.routes.resolve(request.method, path) ?? generalProfile(this.config);
|
|
62
|
+
if (profile === "exempt") return;
|
|
58
63
|
try {
|
|
59
|
-
const profile = this.profile(request.method, path);
|
|
60
64
|
const tracker = await this.identity.resolve(request, context.remoteAddress, this.config);
|
|
61
65
|
await this.enforceCustom(profile.name, tracker, profile.ttlSeconds, profile.limit, context.setHeader);
|
|
62
66
|
} catch (error) {
|
|
@@ -98,24 +102,6 @@ export class RateLimiter {
|
|
|
98
102
|
}
|
|
99
103
|
}
|
|
100
104
|
|
|
101
|
-
private profile(method: string, path: string): LimitProfile {
|
|
102
|
-
if (
|
|
103
|
-
method === "GET" &&
|
|
104
|
-
(path === "/api/auth/get-session" || path === "/site/settings")
|
|
105
|
-
) {
|
|
106
|
-
return { name: "runtime", ttlSeconds: this.config.ttlSeconds, limit: this.config.runtimeLimit };
|
|
107
|
-
}
|
|
108
|
-
// Admin reads need an admin session and are not credential attempts. The
|
|
109
|
-
// dashboard and organizations pages read users through Better Auth.
|
|
110
|
-
if (method === "GET" && path.startsWith("/api/auth/admin/")) {
|
|
111
|
-
return { name: "general", ttlSeconds: this.config.ttlSeconds, limit: this.config.limit };
|
|
112
|
-
}
|
|
113
|
-
if (path === "/api/auth" || path.startsWith("/api/auth/")) {
|
|
114
|
-
return { name: "auth", ttlSeconds: this.config.authTtlSeconds, limit: this.config.authLimit };
|
|
115
|
-
}
|
|
116
|
-
return { name: "general", ttlSeconds: this.config.ttlSeconds, limit: this.config.limit };
|
|
117
|
-
}
|
|
118
|
-
|
|
119
105
|
private async withTimeout(operation: Promise<RateLimitIncrement>): Promise<RateLimitIncrement> {
|
|
120
106
|
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
121
107
|
try {
|
|
@@ -149,6 +135,9 @@ export const rateLimitModule: PodokitModule = {
|
|
|
149
135
|
new CacheRateLimitStorage(services.resolve(CACHE)),
|
|
150
136
|
);
|
|
151
137
|
services.register(RATE_LIMITER, limiter);
|
|
138
|
+
// Other modules may configure before this one, so they register routes
|
|
139
|
+
// from their plugin factory, and applications from configureServices.
|
|
140
|
+
services.register(RATE_LIMIT_ROUTES, limiter.routes);
|
|
152
141
|
services.resolve(REQUEST_GUARDS).register((context) => limiter.enforce(context));
|
|
153
142
|
},
|
|
154
143
|
};
|
package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.routes.spec.ts
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
import { describe, expect, mock, test } from "bun:test";
|
|
2
|
+
import type { RequestGuardContext } from "../core/services";
|
|
3
|
+
import type { RateLimitIdentity } from "./rate-limit.identity";
|
|
4
|
+
import { RateLimiter } from "./rate-limit.module";
|
|
5
|
+
import { defaultRateLimitRoutes, RateLimitRoutes } from "./rate-limit.routes";
|
|
6
|
+
|
|
7
|
+
const config = {
|
|
8
|
+
keyPrefix: "podokit:test:rate-limit",
|
|
9
|
+
ttlSeconds: 60,
|
|
10
|
+
limit: 300,
|
|
11
|
+
authTtlSeconds: 60,
|
|
12
|
+
authLimit: 20,
|
|
13
|
+
runtimeLimit: 1000,
|
|
14
|
+
trustedProxyHops: 0,
|
|
15
|
+
proxyHeader: "x-forwarded-for",
|
|
16
|
+
storageTimeoutMs: 100,
|
|
17
|
+
unavailableRetryAfterSeconds: 1,
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
const uploads = { name: "uploads", limit: 5000, ttlSeconds: 60 };
|
|
21
|
+
|
|
22
|
+
function context(path: string, method = "GET"): { value: RequestGuardContext; headers: Record<string, string> } {
|
|
23
|
+
const headers: Record<string, string> = {};
|
|
24
|
+
return {
|
|
25
|
+
headers,
|
|
26
|
+
value: {
|
|
27
|
+
request: new Request(`http://localhost${path}`, { method }),
|
|
28
|
+
remoteAddress: "127.0.0.1",
|
|
29
|
+
setHeader: (name, value) => { headers[name.toLowerCase()] = value; },
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
describe("RateLimitRoutes", () => {
|
|
35
|
+
test("picks the longest prefix, then a matching method, then the latest registration", () => {
|
|
36
|
+
const routes = new RateLimitRoutes(60);
|
|
37
|
+
routes.register({ pathPrefix: "/files", profile: { name: "files", limit: 100 } });
|
|
38
|
+
routes.register({ pathPrefix: "/files/uploads", profile: uploads });
|
|
39
|
+
routes.register({ method: "put", pathPrefix: "/files/uploads/", profile: "exempt" });
|
|
40
|
+
routes.register({ pathPrefix: "/reports", profile: { name: "reports", limit: 10 } });
|
|
41
|
+
routes.register({ pathPrefix: "/reports", profile: { name: "reports-v2", limit: 20 } });
|
|
42
|
+
|
|
43
|
+
expect(routes.resolve("GET", "/files/a.txt")).toEqual({ name: "files", limit: 100, ttlSeconds: 60 });
|
|
44
|
+
expect(routes.resolve("POST", "/files/uploads")).toEqual(uploads);
|
|
45
|
+
expect(routes.resolve("POST", "/files/uploads/1/parts/2")).toEqual(uploads);
|
|
46
|
+
expect(routes.resolve("PUT", "/files/uploads/1/parts/2")).toBe("exempt");
|
|
47
|
+
expect(routes.resolve("GET", "/reports/daily")).toMatchObject({ name: "reports-v2" });
|
|
48
|
+
// Prefixes match whole segments only.
|
|
49
|
+
expect(routes.resolve("GET", "/files-archive")).toBeUndefined();
|
|
50
|
+
expect(routes.resolve("GET", "/other")).toBeUndefined();
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
test("rejects invalid routes and conflicting budgets under one name", () => {
|
|
54
|
+
const routes = new RateLimitRoutes(60);
|
|
55
|
+
expect(() => routes.register({ pathPrefix: "uploads", profile: "exempt" })).toThrow("absolute path");
|
|
56
|
+
expect(() => routes.register({ pathPrefix: "/a?b", profile: "exempt" })).toThrow("absolute path");
|
|
57
|
+
expect(() => routes.register({ pathPrefix: "/a//b", profile: "exempt" })).toThrow("absolute path");
|
|
58
|
+
expect(() => routes.register({ method: "GET /", pathPrefix: "/a", profile: "exempt" })).toThrow("method");
|
|
59
|
+
expect(() => routes.register({ pathPrefix: "/a", profile: { name: "a:b", limit: 1 } })).toThrow("name");
|
|
60
|
+
expect(() => routes.register({ pathPrefix: "/a", profile: { name: "a", limit: 0 } })).toThrow("limit");
|
|
61
|
+
expect(() => routes.register({ pathPrefix: "/a", profile: { name: "a", limit: 1, ttlSeconds: 1.5 } }))
|
|
62
|
+
.toThrow("ttlSeconds");
|
|
63
|
+
|
|
64
|
+
routes.register({ pathPrefix: "/a", profile: { name: "shared", limit: 10 } });
|
|
65
|
+
routes.register({ pathPrefix: "/b", profile: { name: "shared", limit: 10, ttlSeconds: 60 } });
|
|
66
|
+
expect(() => routes.register({ pathPrefix: "/c", profile: { name: "shared", limit: 11 } }))
|
|
67
|
+
.toThrow("already registered");
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
test("registers the built-in budgets as routes", () => {
|
|
71
|
+
const routes = defaultRateLimitRoutes(config);
|
|
72
|
+
expect(routes.resolve("GET", "/anything")).toMatchObject({ name: "general", limit: 300 });
|
|
73
|
+
expect(routes.resolve("POST", "/api/auth/sign-in/email")).toMatchObject({ name: "auth", limit: 20 });
|
|
74
|
+
expect(routes.resolve("GET", "/api/auth/admin/list-users")).toMatchObject({ name: "general" });
|
|
75
|
+
expect(routes.resolve("POST", "/api/auth/admin/ban-user")).toMatchObject({ name: "auth" });
|
|
76
|
+
expect(routes.resolve("GET", "/api/auth/get-session")).toMatchObject({ name: "runtime", limit: 1000 });
|
|
77
|
+
expect(routes.resolve("GET", "/site/settings")).toMatchObject({ name: "runtime" });
|
|
78
|
+
expect(routes.resolve("PUT", "/site/settings")).toMatchObject({ name: "general" });
|
|
79
|
+
// The built-in names keep their limits.
|
|
80
|
+
expect(() => routes.register({ pathPrefix: "/x", profile: { name: "auth", limit: 1000 } }))
|
|
81
|
+
.toThrow("already registered");
|
|
82
|
+
});
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
describe("RateLimiter route profiles", () => {
|
|
86
|
+
test("counts a registered route in its own bucket and skips exempt routes", async () => {
|
|
87
|
+
const increment = mock(async (_key: string, _ttlSeconds: number) => ({ count: 1, retryAfterSeconds: 30 }));
|
|
88
|
+
const limiter = new RateLimiter(
|
|
89
|
+
config,
|
|
90
|
+
{ resolve: async () => "session:test" } as unknown as RateLimitIdentity,
|
|
91
|
+
{ increment },
|
|
92
|
+
);
|
|
93
|
+
const shortWindow = { ...uploads, ttlSeconds: 30 };
|
|
94
|
+
limiter.routes.register({ pathPrefix: "/files/uploads", profile: shortWindow });
|
|
95
|
+
limiter.routes.register({ method: "GET", pathPrefix: "/files/uploads/status", profile: "exempt" });
|
|
96
|
+
// A more specific route can take a path out of the authentication budget.
|
|
97
|
+
limiter.routes.register({ method: "GET", pathPrefix: "/api/auth/list-sessions", profile: shortWindow });
|
|
98
|
+
|
|
99
|
+
const upload = context("/files/uploads/1/parts/1", "PUT");
|
|
100
|
+
await limiter.enforce(upload.value);
|
|
101
|
+
await limiter.enforce(context("/files/uploads/status").value);
|
|
102
|
+
await limiter.enforce(context("/api/auth/list-sessions").value);
|
|
103
|
+
await limiter.enforce(context("/api/auth/sign-out", "POST").value);
|
|
104
|
+
|
|
105
|
+
expect(increment.mock.calls).toEqual([
|
|
106
|
+
["podokit:test:rate-limit:uploads:session:test", 30],
|
|
107
|
+
["podokit:test:rate-limit:uploads:session:test", 30],
|
|
108
|
+
["podokit:test:rate-limit:auth:session:test", 60],
|
|
109
|
+
]);
|
|
110
|
+
expect(upload.headers["ratelimit-limit"]).toBe("5000");
|
|
111
|
+
expect(upload.headers["ratelimit-remaining"]).toBe("4999");
|
|
112
|
+
});
|
|
113
|
+
});
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
import type {
|
|
2
|
+
RateLimitProfile,
|
|
3
|
+
RateLimitRoute,
|
|
4
|
+
RateLimitRouteRegistry,
|
|
5
|
+
} from "@podosoft/podokit-runtime";
|
|
6
|
+
import type { RateLimitConfig } from "./rate-limit.config";
|
|
7
|
+
|
|
8
|
+
/** A budget with its window resolved, or "exempt". */
|
|
9
|
+
export type ResolvedRateLimitProfile = Required<RateLimitProfile> | "exempt";
|
|
10
|
+
|
|
11
|
+
interface RouteEntry {
|
|
12
|
+
method?: string;
|
|
13
|
+
prefix: string;
|
|
14
|
+
profile: ResolvedRateLimitProfile;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const PROFILE_NAME = /^[A-Za-z0-9_-]{1,64}$/;
|
|
18
|
+
const METHOD = /^[A-Z]+$/;
|
|
19
|
+
|
|
20
|
+
function positiveInteger(value: number | undefined, label: string): void {
|
|
21
|
+
if (value !== undefined && (!Number.isSafeInteger(value) || value < 1)) {
|
|
22
|
+
throw new Error(`${label} must be a positive integer`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function normalizePrefix(prefix: string): string {
|
|
27
|
+
if (!prefix.startsWith("/") || /[?#\s]/.test(prefix) || prefix.includes("//")) {
|
|
28
|
+
throw new Error(`Rate limit path prefix must be an absolute path: ${prefix}`);
|
|
29
|
+
}
|
|
30
|
+
return prefix.length > 1 && prefix.endsWith("/") ? prefix.slice(0, -1) : prefix;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function covers(prefix: string, path: string): boolean {
|
|
34
|
+
return prefix === "/" || path === prefix || path.startsWith(`${prefix}/`);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/**
|
|
38
|
+
* Route profiles for the rate limiter. The built-in authentication and runtime
|
|
39
|
+
* budgets are registered here too, so a module's more specific route takes
|
|
40
|
+
* precedence over them by the same rule as over any other route.
|
|
41
|
+
*/
|
|
42
|
+
export class RateLimitRoutes implements RateLimitRouteRegistry {
|
|
43
|
+
private readonly entries: RouteEntry[] = [];
|
|
44
|
+
private readonly budgets = new Map<string, Required<RateLimitProfile>>();
|
|
45
|
+
|
|
46
|
+
constructor(private readonly defaultTtlSeconds: number) {}
|
|
47
|
+
|
|
48
|
+
register(route: RateLimitRoute): void {
|
|
49
|
+
const method = route.method?.toUpperCase();
|
|
50
|
+
if (method !== undefined && !METHOD.test(method)) {
|
|
51
|
+
throw new Error(`Rate limit route method is invalid: ${route.method}`);
|
|
52
|
+
}
|
|
53
|
+
const prefix = normalizePrefix(route.pathPrefix);
|
|
54
|
+
const profile = route.profile === "exempt" ? "exempt" : this.budget(route.profile);
|
|
55
|
+
this.entries.push({ ...(method ? { method } : {}), prefix, profile });
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/** The most specific registered profile for a request, if any route matches. */
|
|
59
|
+
resolve(method: string, path: string): ResolvedRateLimitProfile | undefined {
|
|
60
|
+
let best: RouteEntry | undefined;
|
|
61
|
+
for (const entry of this.entries) {
|
|
62
|
+
if (entry.method !== undefined && entry.method !== method) continue;
|
|
63
|
+
if (!covers(entry.prefix, path)) continue;
|
|
64
|
+
// The longer prefix wins, then a route with a method, then the later one.
|
|
65
|
+
if (
|
|
66
|
+
!best ||
|
|
67
|
+
entry.prefix.length > best.prefix.length ||
|
|
68
|
+
(entry.prefix.length === best.prefix.length &&
|
|
69
|
+
(entry.method !== undefined || best.method === undefined))
|
|
70
|
+
) {
|
|
71
|
+
best = entry;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return best?.profile;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
private budget(profile: RateLimitProfile): Required<RateLimitProfile> {
|
|
78
|
+
if (!PROFILE_NAME.test(profile.name)) {
|
|
79
|
+
throw new Error(`Rate limit profile name must be 1-64 letters, digits, "_" or "-": ${profile.name}`);
|
|
80
|
+
}
|
|
81
|
+
positiveInteger(profile.limit, `Rate limit profile ${profile.name} limit`);
|
|
82
|
+
positiveInteger(profile.ttlSeconds, `Rate limit profile ${profile.name} ttlSeconds`);
|
|
83
|
+
const budget = {
|
|
84
|
+
name: profile.name,
|
|
85
|
+
limit: profile.limit,
|
|
86
|
+
ttlSeconds: profile.ttlSeconds ?? this.defaultTtlSeconds,
|
|
87
|
+
};
|
|
88
|
+
const existing = this.budgets.get(budget.name);
|
|
89
|
+
if (existing && (existing.limit !== budget.limit || existing.ttlSeconds !== budget.ttlSeconds)) {
|
|
90
|
+
throw new Error(
|
|
91
|
+
`Rate limit profile ${budget.name} is already registered with a limit of ${existing.limit} per ${existing.ttlSeconds}s`,
|
|
92
|
+
);
|
|
93
|
+
}
|
|
94
|
+
this.budgets.set(budget.name, budget);
|
|
95
|
+
return budget;
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
/** The general budget, which applies when no route matches. */
|
|
100
|
+
export function generalProfile(config: RateLimitConfig): Required<RateLimitProfile> {
|
|
101
|
+
return { name: "general", limit: config.limit, ttlSeconds: config.ttlSeconds };
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* The built-in profiles. Authentication writes get the strict auth budget,
|
|
106
|
+
* while administrator reads (the admin pages read users through Better Auth)
|
|
107
|
+
* stay on the general budget, and session and public site-settings reads get
|
|
108
|
+
* the higher runtime budget.
|
|
109
|
+
*/
|
|
110
|
+
export function defaultRateLimitRoutes(config: RateLimitConfig): RateLimitRoutes {
|
|
111
|
+
const routes = new RateLimitRoutes(config.ttlSeconds);
|
|
112
|
+
const runtime = { name: "runtime", limit: config.runtimeLimit, ttlSeconds: config.ttlSeconds };
|
|
113
|
+
routes.register({ pathPrefix: "/", profile: generalProfile(config) });
|
|
114
|
+
routes.register({
|
|
115
|
+
pathPrefix: "/api/auth",
|
|
116
|
+
profile: { name: "auth", limit: config.authLimit, ttlSeconds: config.authTtlSeconds },
|
|
117
|
+
});
|
|
118
|
+
routes.register({ method: "GET", pathPrefix: "/api/auth/admin", profile: generalProfile(config) });
|
|
119
|
+
routes.register({ method: "GET", pathPrefix: "/api/auth/get-session", profile: runtime });
|
|
120
|
+
routes.register({ method: "GET", pathPrefix: "/site/settings", profile: runtime });
|
|
121
|
+
return routes;
|
|
122
|
+
}
|
|
@@ -11,8 +11,13 @@ const origin = { origin: base };
|
|
|
11
11
|
// Honor RATE_LIMIT_MAX when the test runner has it (a dedicated app sets a small
|
|
12
12
|
// value so the window is crossed in a handful of requests); otherwise fall back
|
|
13
13
|
// to the module default and allow a little headroom.
|
|
14
|
+
// A local stack that raises the limits to run the whole suite quickly sets
|
|
15
|
+
// E2E_SKIP_RATE_LIMIT=true and runs this file separately against the defaults.
|
|
16
|
+
test.skip(process.env.E2E_SKIP_RATE_LIMIT === "true", "assumes the configured limits; run it on its own");
|
|
17
|
+
|
|
14
18
|
const limit = Number(process.env.RATE_LIMIT_MAX ?? 300);
|
|
15
19
|
const authLimit = Number(process.env.RATE_LIMIT_AUTH_MAX ?? 20);
|
|
20
|
+
const runtimeLimit = Number(process.env.RATE_LIMIT_RUNTIME_MAX ?? 1000);
|
|
16
21
|
|
|
17
22
|
async function session(
|
|
18
23
|
playwright: import("@playwright/test").PlaywrightWorkerArgs["playwright"],
|
|
@@ -121,3 +126,22 @@ test("rate limit: session reads do not consume the strict authentication budget"
|
|
|
121
126
|
}
|
|
122
127
|
await ctx.dispose();
|
|
123
128
|
});
|
|
129
|
+
|
|
130
|
+
test("rate limit: each route is counted in its most specific registered budget", async ({
|
|
131
|
+
playwright,
|
|
132
|
+
}) => {
|
|
133
|
+
const ctx = await session(playwright);
|
|
134
|
+
const probe = await ctx.get("/api/account/capabilities");
|
|
135
|
+
test.skip(probe.status() >= 500, "throttler storage (redis) not reachable");
|
|
136
|
+
const ceiling = (response: APIResponse): number => Number(response.headers()["ratelimit-limit"]);
|
|
137
|
+
|
|
138
|
+
// Ordinary routes fall back to the general budget registered at "/".
|
|
139
|
+
expect(ceiling(probe)).toBe(limit);
|
|
140
|
+
// Longer GET prefixes inside "/api/auth" win over its authentication budget:
|
|
141
|
+
// session reads use the runtime budget and administrator reads the general one.
|
|
142
|
+
expect(ceiling(await ctx.get("/api/auth/get-session"))).toBe(runtimeLimit);
|
|
143
|
+
expect(ceiling(await ctx.get("/api/auth/admin/list-users"))).toBe(limit);
|
|
144
|
+
// Other "/api/auth" requests carry the strict authentication budget.
|
|
145
|
+
expect(ceiling(await ctx.post("/api/auth/sign-out", { data: {} }))).toBe(authLimit);
|
|
146
|
+
await ctx.dispose();
|
|
147
|
+
});
|