@podosoft/podokit 1.3.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/dist/add.d.ts +10 -1
  2. package/dist/add.js +20 -7
  3. package/dist/api-paths.d.ts +34 -0
  4. package/dist/api-paths.js +103 -0
  5. package/dist/deploy-compose-profile.d.ts +4 -0
  6. package/dist/deploy-compose-profile.js +13 -2
  7. package/dist/deploy-compose-render.d.ts +1 -1
  8. package/dist/deploy-compose-render.js +23 -7
  9. package/dist/deploy-compose.js +2 -1
  10. package/dist/deploy-profile.d.ts +3 -0
  11. package/dist/deploy-profile.js +12 -4
  12. package/dist/deploy-render.js +54 -8
  13. package/dist/deploy.js +21 -1
  14. package/dist/dev.d.ts +14 -0
  15. package/dist/dev.js +152 -10
  16. package/dist/index.js +15 -2
  17. package/dist/lib.d.ts +2 -0
  18. package/dist/lib.js +4 -2
  19. package/dist/lockfile.d.ts +3 -0
  20. package/dist/lockfile.js +9 -0
  21. package/dist/module-api-paths.d.ts +52 -0
  22. package/dist/module-api-paths.js +189 -0
  23. package/dist/remove.d.ts +3 -0
  24. package/dist/remove.js +12 -3
  25. package/dist/templates/fullstack/AGENTS.md +17 -2
  26. package/dist/templates/fullstack/README.md +19 -5
  27. package/dist/templates/fullstack/apps/api/package.json +1 -1
  28. package/dist/templates/fullstack/apps/api/src/app.extensions.ts +7 -0
  29. package/dist/templates/fullstack/apps/api/src/app.spec.ts +20 -1
  30. package/dist/templates/fullstack/apps/api/src/app.ts +11 -0
  31. package/dist/templates/fullstack/apps/api/src/config/env.validation.ts +3 -0
  32. package/dist/templates/fullstack/apps/api/src/config/redis.connection.spec.ts +48 -2
  33. package/dist/templates/fullstack/apps/api/src/config/redis.connection.ts +45 -0
  34. package/dist/templates/fullstack/apps/api/src/config/server-options.spec.ts +46 -0
  35. package/dist/templates/fullstack/apps/api/src/config/server-options.ts +48 -0
  36. package/dist/templates/fullstack/apps/api/src/core/api-contract.spec.ts +1 -0
  37. package/dist/templates/fullstack/apps/api/src/core/api-contract.ts +9 -1
  38. package/dist/templates/fullstack/apps/api/src/core/services.spec.ts +196 -0
  39. package/dist/templates/fullstack/apps/api/src/core/services.ts +181 -2
  40. package/dist/templates/fullstack/apps/api/src/database/database.spec.ts +8 -1
  41. package/dist/templates/fullstack/apps/api/src/database/database.ts +29 -10
  42. package/dist/templates/fullstack/apps/api/src/database/pool-guard.spec.ts +188 -0
  43. package/dist/templates/fullstack/apps/api/src/database/pool-guard.ts +151 -0
  44. package/dist/templates/fullstack/apps/api/src/health/health.plugin.spec.ts +28 -4
  45. package/dist/templates/fullstack/apps/api/src/health/health.plugin.ts +10 -2
  46. package/dist/templates/fullstack/apps/api/src/main.ts +2 -1
  47. package/dist/templates/fullstack/apps/web/components.json +3 -1
  48. package/dist/templates/fullstack/apps/web/package.json +1 -1
  49. package/dist/templates/fullstack/apps/web/src/lib/components/ui/textarea/index.ts +7 -0
  50. package/dist/templates/fullstack/apps/web/src/lib/components/ui/textarea/textarea.svelte +22 -0
  51. package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.test.ts +276 -1
  52. package/dist/templates/fullstack/apps/web/src/lib/server/backend-proxy.ts +194 -9
  53. package/dist/templates/fullstack/apps/web/src/lib/server/body-limits.ts +15 -0
  54. package/dist/templates/fullstack/apps/web/src/lib/server/proxy-headers.ts +12 -0
  55. package/dist/templates/fullstack/dot-agents/skills/podokit-deploy/references/profile.md +10 -3
  56. package/dist/templates/fullstack/dot-claude/skills/podokit-elysia-endpoint/SKILL.md +5 -0
  57. package/dist/templates/fullstack/dot-claude/skills/podokit-sveltekit-route/SKILL.md +7 -1
  58. package/dist/templates/fullstack/dot-devcontainer/devcontainer.json +1 -0
  59. package/dist/templates/fullstack/{dot-env.docker → dot-env.docker.example} +7 -4
  60. package/dist/templates/fullstack/dot-env.example +10 -0
  61. package/dist/templates/fullstack/dot-gitignore +1 -0
  62. package/dist/templates/fullstack/dot-podokit/dev.json +2 -1
  63. package/dist/templates/fullstack/infra/k3s/README.md +3 -1
  64. package/dist/templates/modules/admin-dashboard/files/apps/api/src/bootstrap-admin-runtime.ts +1 -1
  65. package/dist/templates/modules/admin-dashboard/files/apps/web/src/hooks.server.ts +27 -9
  66. package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/asset-paths.ts +11 -0
  67. package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.test.ts +125 -0
  68. package/dist/templates/modules/admin-dashboard/files/apps/web/src/lib/server/runtime-context.ts +107 -0
  69. package/dist/templates/modules/admin-dashboard/files/tests/api/profile-image.api.spec.ts +2 -1
  70. package/dist/templates/modules/admin-dashboard/files/tests/api/site-settings.api.spec.ts +2 -1
  71. package/dist/templates/modules/admin-dashboard/files/tests/helpers/disposable-users.ts +10 -10
  72. package/dist/templates/modules/admin-dashboard/files/tests/helpers/object-response.ts +12 -0
  73. package/dist/templates/modules/admin-dashboard/module.manifest.json +3 -2
  74. package/dist/templates/modules/auth/files/apps/api/src/auth/auth-provider.spec.ts +26 -0
  75. package/dist/templates/modules/auth/files/apps/api/src/auth/auth-provider.ts +17 -2
  76. package/dist/templates/modules/auth/files/apps/api/src/auth/auth.service.ts +8 -4
  77. package/dist/templates/modules/auth/files/apps/api/src/auth/auth.ts +2 -2
  78. package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.spec.ts +114 -0
  79. package/dist/templates/modules/auth/files/apps/api/src/auth/database-outage.ts +50 -0
  80. package/dist/templates/modules/auth/files/apps/api/src/auth/db.spec.ts +4 -1
  81. package/dist/templates/modules/auth/files/apps/api/src/auth/db.ts +11 -2
  82. package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance-command.ts +5 -0
  83. package/dist/templates/modules/auth/files/apps/api/src/auth/maintenance.ts +14 -0
  84. package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.spec.ts +136 -0
  85. package/dist/templates/modules/auth/files/apps/api/src/auth/postgres-pool.ts +40 -0
  86. package/dist/templates/modules/auth/files/apps/api/src/migrate.ts +3 -2
  87. package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.spec.ts +121 -1
  88. package/dist/templates/modules/bullmq/files/apps/api/src/jobs/bullmq-job.queue.ts +108 -10
  89. package/dist/templates/modules/bullmq/files/apps/api/src/jobs/jobs.module.ts +3 -1
  90. package/dist/templates/modules/bullmq/files/apps/api/src/jobs/queue.ts +31 -3
  91. package/dist/templates/modules/bullmq/module.manifest.json +5 -1
  92. package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/events-redis.module.ts +2 -4
  93. package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.spec.ts +224 -0
  94. package/dist/templates/modules/events-redis/files/apps/api/src/events-provider-redis/redis-event.bus.ts +124 -35
  95. package/dist/templates/modules/events-redis/module.manifest.json +4 -0
  96. package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.routes.ts +136 -0
  97. package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.spec.ts +636 -0
  98. package/dist/templates/modules/file-upload/files/apps/api/src/files/chunked-uploads.ts +673 -0
  99. package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.spec.ts +10 -4
  100. package/dist/templates/modules/file-upload/files/apps/api/src/files/files.module.ts +37 -8
  101. package/dist/templates/modules/file-upload/files/apps/api/src/files/upload-config.ts +99 -0
  102. package/dist/templates/modules/file-upload/files/apps/api/src/migrations/1792100000000-InitFileUploadSessions.ts +49 -0
  103. package/dist/templates/modules/file-upload/files/apps/web/src/lib/uploads.test.ts +456 -0
  104. package/dist/templates/modules/file-upload/files/apps/web/src/lib/uploads.ts +356 -0
  105. package/dist/templates/modules/file-upload/files/tests/api/chunked-uploads.api.spec.ts +147 -0
  106. package/dist/templates/modules/file-upload/module.manifest.json +34 -6
  107. package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/jobs-local.module.ts +1 -1
  108. package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.spec.ts +23 -0
  109. package/dist/templates/modules/jobs-local/files/apps/api/src/jobs-local/local-job.queue.ts +30 -20
  110. package/dist/templates/modules/jobs-local/module.manifest.json +3 -0
  111. package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/download-url.ts +20 -0
  112. package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.spec.ts +79 -12
  113. package/dist/templates/modules/object-storage-local/files/apps/api/src/storage-local/storage-local.module.ts +16 -1
  114. package/dist/templates/modules/object-storage-local/module.manifest.json +11 -2
  115. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-multipart.spec.ts +115 -0
  116. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-multipart.ts +171 -0
  117. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.spec.ts +222 -9
  118. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-object.store.ts +55 -19
  119. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-signer.spec.ts +71 -0
  120. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/s3-signer.ts +98 -0
  121. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage-demo.spec.ts +6 -3
  122. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.config.ts +7 -2
  123. package/dist/templates/modules/object-storage-s3/files/apps/api/src/storage/storage.service.ts +187 -44
  124. package/dist/templates/modules/object-storage-s3/module.manifest.json +10 -2
  125. package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.module.ts +11 -22
  126. package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.routes.spec.ts +113 -0
  127. package/dist/templates/modules/rate-limit/files/apps/api/src/rate-limit/rate-limit.routes.ts +122 -0
  128. package/dist/templates/modules/rate-limit/files/tests/api/rate-limit.api.spec.ts +24 -0
  129. package/dist/templates/modules/rate-limit/module.manifest.json +7 -0
  130. package/dist/templates/modules/redis/files/apps/api/src/redis/redis-cache.store.spec.ts +14 -1
  131. package/dist/templates/modules/redis/files/apps/api/src/redis/redis-cache.store.ts +9 -12
  132. package/dist/templates/modules/redis/files/apps/api/src/redis/redis.module.ts +16 -4
  133. package/dist/templates/modules/redis/files/apps/api/src/redis/redis.service.spec.ts +233 -5
  134. package/dist/templates/modules/redis/files/apps/api/src/redis/redis.service.ts +134 -32
  135. package/dist/templates/modules/redis/module.manifest.json +4 -0
  136. package/dist/templates/modules/sse/files/apps/api/src/events/event-stream.ts +11 -4
  137. package/dist/templates/modules/sse/files/apps/api/src/events/events.service.spec.ts +33 -0
  138. package/dist/templates/modules/sse/files/apps/api/src/events/events.service.ts +7 -0
  139. package/dist/templates/todo/AGENTS.md +9 -1
  140. package/dist/templates/todo/README.md +18 -5
  141. package/dist/update.d.ts +5 -0
  142. package/dist/update.js +35 -0
  143. package/package.json +2 -2
  144. package/dist/templates/modules/auth/files/apps/api/src/auth/schema-validation.ts +0 -13
  145. package/dist/templates/modules/auth/files/apps/api/src/auth/skip-schema-validation.ts +0 -5
@@ -8,6 +8,9 @@
8
8
  "apps/api/src/config/providers.ts",
9
9
  "apps/api/src/database/database.ts"
10
10
  ],
11
+ "requiredProjectContents": {
12
+ "apps/api/src/core/services.ts": ["supervise(name: string, connection: SupervisedConnection): void {"]
13
+ },
11
14
  "managedGlobs": [
12
15
  "apps/api/src/jobs-local/**"
13
16
  ],
@@ -2,8 +2,11 @@ import { createHmac, randomBytes, randomUUID, timingSafeEqual } from "node:crypt
2
2
  import { link, mkdir, readFile, rm, writeFile } from "node:fs/promises";
3
3
  import { dirname } from "node:path";
4
4
  import type {
5
+ CompletedPart,
6
+ CreateMultipartUploadOptions,
5
7
  DownloadUrlOptions,
6
8
  ListObjectsOptions,
9
+ MultipartUpload,
7
10
  ObjectBody,
8
11
  ObjectData,
9
12
  ObjectInfo,
@@ -11,6 +14,7 @@ import type {
11
14
  ObjectStore,
12
15
  PutObjectOptions,
13
16
  StoredObject,
17
+ UploadedPart,
14
18
  } from "@podosoft/podokit-runtime";
15
19
 
16
20
  const KEY_BYTES = 32;
@@ -153,6 +157,22 @@ export class SignedDownloadStore implements ObjectStore {
153
157
  return `${path}?${new URLSearchParams({ expires, signature }).toString()}`;
154
158
  }
155
159
 
160
+ createMultipartUpload(key: string, options?: CreateMultipartUploadOptions): Promise<MultipartUpload> {
161
+ return this.store.createMultipartUpload(key, options);
162
+ }
163
+
164
+ uploadPart(key: string, uploadId: string, partNumber: number, body: ObjectBody): Promise<UploadedPart> {
165
+ return this.store.uploadPart(key, uploadId, partNumber, body);
166
+ }
167
+
168
+ completeMultipartUpload(key: string, uploadId: string, parts: readonly CompletedPart[]): Promise<ObjectInfo> {
169
+ return this.store.completeMultipartUpload(key, uploadId, parts);
170
+ }
171
+
172
+ abortMultipartUpload(key: string, uploadId: string): Promise<void> {
173
+ return this.store.abortMultipartUpload(key, uploadId);
174
+ }
175
+
156
176
  close(): void | Promise<void> {
157
177
  return this.store.close();
158
178
  }
@@ -1,14 +1,20 @@
1
1
  import { afterEach, describe, expect, it } from "bun:test";
2
+ import { randomBytes } from "node:crypto";
2
3
  import { existsSync } from "node:fs";
3
4
  import { mkdtemp, rm } from "node:fs/promises";
4
5
  import { tmpdir } from "node:os";
5
6
  import { join } from "node:path";
6
7
  import { AppException } from "@podosoft/podokit-contracts";
7
- import { LocalObjectStore } from "@podosoft/podokit-runtime";
8
+ import { IN_MEMORY_READ_BYTES, LocalObjectStore } from "@podosoft/podokit-runtime";
8
9
  import { Elysia } from "elysia";
9
10
  import type { RequestIdentity } from "../core/services";
10
11
  import { DownloadUrlSigner, SignedDownloadStore } from "./download-url";
11
- import { DOWNLOAD_BASE_URL, downloadSafetyHeaders, localStorageRoutes } from "./storage-local.module";
12
+ import {
13
+ DOWNLOAD_BASE_URL,
14
+ downloadSafetyHeaders,
15
+ localStorageRoutes,
16
+ multipartMaxAgeMs,
17
+ } from "./storage-local.module";
12
18
 
13
19
  let root: string | undefined;
14
20
 
@@ -51,15 +57,12 @@ interface Reply {
51
57
  headers: Headers;
52
58
  }
53
59
 
54
- async function call(
55
- { storage, signer }: Fixture,
56
- identity: RequestIdentity,
57
- method: "GET" | "PUT",
58
- path: string,
59
- content?: string,
60
- headers: Record<string, string> = {},
61
- ): Promise<Reply> {
62
- const app = new Elysia()
60
+ interface Handler {
61
+ handle(request: Request): Promise<Response>;
62
+ }
63
+
64
+ function routes({ storage, signer }: Fixture, identity: RequestIdentity): Handler {
65
+ return new Elysia()
63
66
  .onError(({ code, error, set }) => {
64
67
  if (error instanceof AppException) {
65
68
  set.status = error.statusCode;
@@ -76,7 +79,17 @@ async function call(
76
79
  return undefined;
77
80
  })
78
81
  .use(localStorageRoutes(storage, signer, identity));
79
- const response = await app.handle(new Request(`http://localhost${path}`, {
82
+ }
83
+
84
+ async function call(
85
+ local: Fixture,
86
+ identity: RequestIdentity,
87
+ method: "GET" | "PUT",
88
+ path: string,
89
+ content?: string,
90
+ headers: Record<string, string> = {},
91
+ ): Promise<Reply> {
92
+ const response = await routes(local, identity).handle(new Request(`http://localhost${path}`, {
80
93
  method,
81
94
  headers: { ...(content === undefined ? {} : { "content-type": "application/json" }), ...headers },
82
95
  ...(content === undefined ? {} : { body: JSON.stringify({ content }) }),
@@ -133,6 +146,31 @@ describe("local object storage provider", () => {
133
146
  expect(code(missing)).toBe("STORAGE_OBJECT_NOT_FOUND");
134
147
  });
135
148
 
149
+ it("serves objects through Bun with the Content-Length of those read into memory", async () => {
150
+ const local = await fixture();
151
+ const app = routes(local, caller(undefined));
152
+ const server = Bun.serve({ port: 0, fetch: (request) => app.handle(request) });
153
+ try {
154
+ for (const size of [68, 100 * 1024, IN_MEMORY_READ_BYTES, IN_MEMORY_READ_BYTES + 1]) {
155
+ const content = randomBytes(size);
156
+ const key = `uploads/served-${size}.bin`;
157
+ await local.storage.put(key, content, { contentType: "image/png" });
158
+ const response = await fetch(new URL(await local.storage.getDownloadUrl(key), server.url));
159
+ expect(response.status).toBe(200);
160
+ if (size <= IN_MEMORY_READ_BYTES) {
161
+ expect(response.headers.get("content-length")).toBe(String(size));
162
+ expect(response.headers.get("transfer-encoding")).toBeNull();
163
+ } else {
164
+ // Streamed from the file, which Bun may send chunked; a length must still be right.
165
+ expect(response.headers.get("content-length") ?? String(size)).toBe(String(size));
166
+ }
167
+ expect(Buffer.from(await response.arrayBuffer()).equals(content)).toBe(true);
168
+ }
169
+ } finally {
170
+ server.stop(true);
171
+ }
172
+ });
173
+
136
174
  it("limits the demo routes to administrators", async () => {
137
175
  const local = await fixture();
138
176
 
@@ -176,6 +214,35 @@ describe("local object storage provider", () => {
176
214
  });
177
215
  });
178
216
 
217
+ describe("local multipart uploads", () => {
218
+ it("assembles an object from parts through the signed store", async () => {
219
+ const local = await fixture();
220
+ const { uploadId } = await local.storage.createMultipartUpload("uploads/big.txt", { contentType: "text/plain" });
221
+ const second = await local.storage.uploadPart("uploads/big.txt", uploadId, 2, "world");
222
+ const first = await local.storage.uploadPart("uploads/big.txt", uploadId, 1, new Blob(["x".repeat(5 * 1024 * 1024)]));
223
+ const info = await local.storage.completeMultipartUpload("uploads/big.txt", uploadId, [
224
+ { partNumber: 1, etag: first.etag },
225
+ { partNumber: 2, etag: second.etag },
226
+ ]);
227
+ expect(info).toMatchObject({ size: 5 * 1024 * 1024 + 5, contentType: "text/plain" });
228
+ const object = await local.storage.get("uploads/big.txt");
229
+ expect((await new Response(object?.body).text()).endsWith("xworld")).toBe(true);
230
+
231
+ const aborted = await local.storage.createMultipartUpload("uploads/aborted.txt");
232
+ await local.storage.abortMultipartUpload("uploads/aborted.txt", aborted.uploadId);
233
+ expect(existsSync(join(local.directory, ".podokit", "multipart", aborted.uploadId))).toBe(false);
234
+ });
235
+
236
+ it("reads the idle multipart lifetime in seconds", () => {
237
+ expect(multipartMaxAgeMs(undefined)).toBe(24 * 60 * 60 * 1000);
238
+ expect(multipartMaxAgeMs("")).toBe(24 * 60 * 60 * 1000);
239
+ expect(multipartMaxAgeMs("3600")).toBe(3_600_000);
240
+ for (const value of ["0", "-1", "1.5", "a day"]) {
241
+ expect(() => multipartMaxAgeMs(value)).toThrow("LOCAL_STORAGE_MULTIPART_MAX_AGE");
242
+ }
243
+ });
244
+ });
245
+
179
246
  describe("local download safety headers", () => {
180
247
  it("renders only script-free types inline", () => {
181
248
  expect(downloadSafetyHeaders("image/png")["content-disposition"]).toBe("inline");
@@ -27,6 +27,17 @@ import {
27
27
  } from "./storage-demo";
28
28
 
29
29
  export const DOWNLOAD_BASE_URL = "/files/content";
30
+ const DEFAULT_MULTIPART_MAX_AGE_SECONDS = 24 * 60 * 60;
31
+
32
+ /** How long an unfinished multipart upload may sit idle before its parts are removed. */
33
+ export function multipartMaxAgeMs(value = process.env.LOCAL_STORAGE_MULTIPART_MAX_AGE): number {
34
+ if (value === undefined || value.trim() === "") return DEFAULT_MULTIPART_MAX_AGE_SECONDS * 1000;
35
+ const seconds = Number(value);
36
+ if (!Number.isInteger(seconds) || seconds < 1) {
37
+ throw new Error("LOCAL_STORAGE_MULTIPART_MAX_AGE must be a positive integer of seconds");
38
+ }
39
+ return seconds * 1000;
40
+ }
30
41
  export const LOCAL_DOWNLOAD_URLS = Symbol("local-download-urls") as ServiceKey<DownloadUrlSigner>;
31
42
 
32
43
  async function readObject(storage: ObjectStore, key: string): Promise<ObjectData> {
@@ -124,7 +135,11 @@ export const storageLocalModule: PodokitModule = {
124
135
  const root = resolve(process.env.LOCAL_STORAGE_PATH ?? "./data/files");
125
136
  const signer = new DownloadUrlSigner(join(root, ".podokit", "download-url.key"));
126
137
  const storage = new SignedDownloadStore(
127
- new LocalObjectStore({ rootDirectory: root, publicBaseUrl: DOWNLOAD_BASE_URL }),
138
+ new LocalObjectStore({
139
+ rootDirectory: root,
140
+ publicBaseUrl: DOWNLOAD_BASE_URL,
141
+ multipartMaxAgeMs: multipartMaxAgeMs(),
142
+ }),
128
143
  signer,
129
144
  DOWNLOAD_BASE_URL,
130
145
  );
@@ -16,7 +16,9 @@
16
16
  "targetApp": "api",
17
17
  "env": [
18
18
  "# Local object storage root. Use an absolute path in production and desktop packaging.",
19
- "LOCAL_STORAGE_PATH=./data/files"
19
+ "LOCAL_STORAGE_PATH=./data/files",
20
+ "# Seconds an unfinished multipart upload may sit idle before its parts are removed.",
21
+ "LOCAL_STORAGE_MULTIPART_MAX_AGE=86400"
20
22
  ],
21
23
  "inject": [
22
24
  {
@@ -28,12 +30,19 @@
28
30
  "file": "apps/api/src/app.ts",
29
31
  "marker": "// podokit:end:modules",
30
32
  "text": "storageLocalModule,"
33
+ },
34
+ {
35
+ "file": "AGENTS.md",
36
+ "marker": "<!-- podokit:end:agents-modules -->",
37
+ "optional": true,
38
+ "text": "### object-storage-local\n- Store files through `OBJECT_STORAGE` from `@podosoft/podokit-runtime` (`services.resolve(OBJECT_STORAGE)`), not the provider directly. `put(key, body, { contentType })` streams a `ReadableStream` such as `request.body`, a `Blob` or file, an async iterable, bytes, or a string without buffering it; a failed body leaves no partial object.\n- For uploads split across requests (resumable or chunked), use `createMultipartUpload`, `uploadPart` (parts numbered 1 to 10,000, each but the last at least `MIN_MULTIPART_PART_BYTES` = 5 MiB), `completeMultipartUpload`, and `abortMultipartUpload`. Branch on `MultipartUploadError.code` (`UPLOAD_NOT_FOUND`, `INVALID_PART`, `PART_TOO_SMALL`).\n- Unfinished multipart uploads live under `LOCAL_STORAGE_PATH/.podokit/multipart/` and are removed after `LOCAL_STORAGE_MULTIPART_MAX_AGE` seconds without activity.\n"
31
39
  }
32
40
  ],
33
41
  "instructions": [
34
42
  "Local object storage is single-host. Back up LOCAL_STORAGE_PATH together with the database.",
35
43
  "Keep the API at one replica or select the S3 provider.",
36
44
  "Download URLs are signed and expire after one hour by default. The signing key lives in LOCAL_STORAGE_PATH/.podokit/download-url.key; deleting it invalidates every issued URL.",
37
- "The /storage demo routes need the auth module and an administrator session, and only reach keys under demo/."
45
+ "The /storage demo routes need the auth module and an administrator session, and only reach keys under demo/.",
46
+ "Multipart upload parts live under LOCAL_STORAGE_PATH/.podokit/multipart until the upload completes or is aborted; uploads idle longer than LOCAL_STORAGE_MULTIPART_MAX_AGE seconds are removed."
38
47
  ]
39
48
  }
@@ -0,0 +1,115 @@
1
+ import { describe, expect, it } from "bun:test";
2
+ import { MultipartUploadError } from "@podosoft/podokit-runtime";
3
+ import { S3Multipart, type S3Target } from "./s3-multipart";
4
+
5
+ interface Call {
6
+ url: string;
7
+ method: string;
8
+ headers: Record<string, string>;
9
+ body: string | undefined;
10
+ }
11
+
12
+ function fakeS3(...responses: Response[]): { calls: Call[]; fetch: (input: string, init: RequestInit) => Promise<Response> } {
13
+ const calls: Call[] = [];
14
+ return {
15
+ calls,
16
+ fetch: async (input, init) => {
17
+ calls.push({
18
+ url: input,
19
+ method: init.method ?? "GET",
20
+ headers: init.headers as Record<string, string>,
21
+ body: init.body === undefined ? undefined : await new Response(init.body as BodyInit).text(),
22
+ });
23
+ const response = responses.shift();
24
+ if (!response) throw new Error("unexpected request");
25
+ return response;
26
+ },
27
+ };
28
+ }
29
+
30
+ function errorXml(code: string, status: number): Response {
31
+ return new Response(`<?xml version="1.0"?><Error><Code>${code}</Code><Message>${code} message</Message></Error>`, {
32
+ status,
33
+ });
34
+ }
35
+
36
+ const target: S3Target = {
37
+ bucket: "podokit",
38
+ region: "us-east-1",
39
+ endpoint: "http://localhost:9000",
40
+ virtualHostedStyle: false,
41
+ credentials: { accessKeyId: "podokit", secretAccessKey: "podokitsecret" },
42
+ };
43
+
44
+ describe("S3Multipart", () => {
45
+ it("creates, uploads, completes, and aborts with signed path-style requests", async () => {
46
+ const s3 = fakeS3(
47
+ new Response("<InitiateMultipartUploadResult><UploadId>id+1/2</UploadId></InitiateMultipartUploadResult>"),
48
+ new Response("", { headers: { etag: "\"abc\"" } }),
49
+ new Response("<CompleteMultipartUploadResult><ETag>&quot;x-1&quot;</ETag></CompleteMultipartUploadResult>"),
50
+ new Response(null, { status: 204 }),
51
+ );
52
+ const multipart = new S3Multipart(target, s3.fetch);
53
+
54
+ const uploadId = await multipart.create("videos/a b+c.mp4", "video/mp4");
55
+ expect(uploadId).toBe("id+1/2");
56
+ expect(await multipart.uploadPart("videos/a b+c.mp4", uploadId, 1, new TextEncoder().encode("part"))).toBe("\"abc\"");
57
+ await multipart.complete("videos/a b+c.mp4", uploadId, [{ partNumber: 1, etag: "\"abc\"" }]);
58
+ await multipart.abort("videos/a b+c.mp4", uploadId);
59
+
60
+ const object = "http://localhost:9000/podokit/videos/a%20b%2Bc.mp4";
61
+ expect(s3.calls.map((call) => [call.method, call.url])).toEqual([
62
+ ["POST", `${object}?uploads=`],
63
+ ["PUT", `${object}?partNumber=1&uploadId=id%2B1%2F2`],
64
+ ["POST", `${object}?uploadId=id%2B1%2F2`],
65
+ ["DELETE", `${object}?uploadId=id%2B1%2F2`],
66
+ ]);
67
+ expect(s3.calls[0]?.headers["content-type"]).toBe("video/mp4");
68
+ expect(s3.calls[1]?.headers["x-amz-content-sha256"]).toBe("UNSIGNED-PAYLOAD");
69
+ expect(s3.calls[1]?.body).toBe("part");
70
+ expect(s3.calls[2]?.body).toBe(
71
+ "<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>&quot;abc&quot;</ETag></Part></CompleteMultipartUpload>",
72
+ );
73
+ for (const call of s3.calls) {
74
+ expect(call.headers.authorization).toStartWith("AWS4-HMAC-SHA256 Credential=podokit/");
75
+ }
76
+ });
77
+
78
+ it("addresses the bucket by host name in virtual-hosted style and on AWS by default", async () => {
79
+ const s3 = fakeS3(
80
+ new Response("<UploadId>one</UploadId>"),
81
+ new Response("<UploadId>two</UploadId>"),
82
+ );
83
+ await new S3Multipart({ ...target, virtualHostedStyle: true }, s3.fetch).create("a.txt");
84
+ const { endpoint: _endpoint, ...aws } = target;
85
+ await new S3Multipart({ ...aws, region: "eu-west-1", virtualHostedStyle: true }, s3.fetch).create("a.txt");
86
+ expect(s3.calls.map((call) => call.url)).toEqual([
87
+ "http://podokit.localhost:9000/a.txt?uploads=",
88
+ "https://podokit.s3.eu-west-1.amazonaws.com/a.txt?uploads=",
89
+ ]);
90
+ });
91
+
92
+ it("maps S3 error codes to multipart error codes", async () => {
93
+ const s3 = fakeS3(
94
+ errorXml("NoSuchUpload", 404),
95
+ errorXml("InvalidPart", 400),
96
+ // S3 may report a failed completion in a 200 response.
97
+ new Response("<Error><Code>EntityTooSmall</Code><Message>small</Message></Error>"),
98
+ errorXml("AccessDenied", 403),
99
+ errorXml("NoSuchUpload", 404),
100
+ );
101
+ const multipart = new S3Multipart(target, s3.fetch);
102
+ const codeOf = (error: unknown): string | undefined =>
103
+ error instanceof MultipartUploadError ? error.code : undefined;
104
+
105
+ expect(codeOf(await multipart.uploadPart("a", "x", 1, new Uint8Array(1)).catch((error: unknown) => error)))
106
+ .toBe("UPLOAD_NOT_FOUND");
107
+ expect(codeOf(await multipart.complete("a", "x", [{ partNumber: 1, etag: "e" }]).catch((error: unknown) => error)))
108
+ .toBe("INVALID_PART");
109
+ expect(codeOf(await multipart.complete("a", "x", [{ partNumber: 1, etag: "e" }]).catch((error: unknown) => error)))
110
+ .toBe("PART_TOO_SMALL");
111
+ await expect(multipart.create("a")).rejects.toThrow("AccessDenied");
112
+ // Aborting an upload that is already gone succeeds.
113
+ await multipart.abort("a", "x");
114
+ });
115
+ });
@@ -0,0 +1,171 @@
1
+ import {
2
+ MultipartUploadError,
3
+ type CompletedPart,
4
+ type MultipartUploadErrorCode,
5
+ } from "@podosoft/podokit-runtime";
6
+ import {
7
+ EMPTY_PAYLOAD_HASH,
8
+ encodeRfc3986,
9
+ signS3Request,
10
+ UNSIGNED_PAYLOAD,
11
+ type S3Credentials,
12
+ } from "./s3-signer";
13
+
14
+ export interface S3Target {
15
+ bucket: string;
16
+ region: string;
17
+ /** Defaults to the AWS regional endpoint. */
18
+ endpoint?: string;
19
+ virtualHostedStyle: boolean;
20
+ credentials: S3Credentials;
21
+ }
22
+
23
+ type FetchLike = (input: string, init: RequestInit) => Promise<Response>;
24
+
25
+ const ERROR_CODES: Record<string, MultipartUploadErrorCode> = {
26
+ NoSuchUpload: "UPLOAD_NOT_FOUND",
27
+ InvalidPart: "INVALID_PART",
28
+ InvalidPartOrder: "INVALID_PART",
29
+ EntityTooSmall: "PART_TOO_SMALL",
30
+ };
31
+
32
+ function xmlValue(xml: string, element: string): string | undefined {
33
+ const match = new RegExp(`<${element}>([^<]*)</${element}>`).exec(xml);
34
+ return match?.[1]
35
+ ?.replaceAll("&quot;", "\"")
36
+ .replaceAll("&apos;", "'")
37
+ .replaceAll("&lt;", "<")
38
+ .replaceAll("&gt;", ">")
39
+ .replaceAll("&amp;", "&");
40
+ }
41
+
42
+ function escapeXml(value: string): string {
43
+ return value
44
+ .replaceAll("&", "&amp;")
45
+ .replaceAll("<", "&lt;")
46
+ .replaceAll(">", "&gt;")
47
+ .replaceAll("\"", "&quot;")
48
+ .replaceAll("'", "&apos;");
49
+ }
50
+
51
+ function s3Error(operation: string, status: number, xml: string): Error {
52
+ const code = xmlValue(xml, "Code");
53
+ const message = xmlValue(xml, "Message") ?? `status ${status}`;
54
+ const mapped = code ? ERROR_CODES[code] : undefined;
55
+ if (mapped) return new MultipartUploadError(mapped, `${operation} failed: ${message}`);
56
+ return new Error(`${operation} failed with ${code ?? `status ${status}`}: ${message}`);
57
+ }
58
+
59
+ /**
60
+ * S3 multipart operations over signed requests: create, upload part, complete,
61
+ * and abort. Works with AWS S3 and MinIO-compatible services such as Silo.
62
+ */
63
+ export class S3Multipart {
64
+ private readonly base: URL;
65
+
66
+ constructor(
67
+ private readonly target: S3Target,
68
+ private readonly fetchImpl: FetchLike = fetch,
69
+ ) {
70
+ this.base = new URL(target.endpoint ?? `https://s3.${target.region}.amazonaws.com`);
71
+ }
72
+
73
+ async create(key: string, contentType?: string): Promise<string> {
74
+ const response = await this.send("POST", key, [["uploads", ""]], {
75
+ ...(contentType ? { "content-type": contentType } : {}),
76
+ });
77
+ const xml = await response.text();
78
+ if (!response.ok) throw s3Error("CreateMultipartUpload", response.status, xml);
79
+ const uploadId = xmlValue(xml, "UploadId");
80
+ if (!uploadId) throw new Error("CreateMultipartUpload returned no upload id");
81
+ return uploadId;
82
+ }
83
+
84
+ /** Returns the part's entity tag. The body's length must be known, as S3 requires. */
85
+ async uploadPart(key: string, uploadId: string, partNumber: number, body: Uint8Array | Blob): Promise<string> {
86
+ const response = await this.send(
87
+ "PUT",
88
+ key,
89
+ [["partNumber", String(partNumber)], ["uploadId", uploadId]],
90
+ {},
91
+ body,
92
+ );
93
+ const xml = await response.text();
94
+ if (!response.ok) throw s3Error("UploadPart", response.status, xml);
95
+ const etag = response.headers.get("etag");
96
+ if (!etag) throw new Error("UploadPart returned no entity tag");
97
+ return etag;
98
+ }
99
+
100
+ async complete(key: string, uploadId: string, parts: readonly CompletedPart[]): Promise<void> {
101
+ const document = [
102
+ "<CompleteMultipartUpload>",
103
+ ...parts.map((part) =>
104
+ `<Part><PartNumber>${part.partNumber}</PartNumber><ETag>${escapeXml(part.etag)}</ETag></Part>`
105
+ ),
106
+ "</CompleteMultipartUpload>",
107
+ ].join("");
108
+ const response = await this.send(
109
+ "POST",
110
+ key,
111
+ [["uploadId", uploadId]],
112
+ { "content-type": "application/xml" },
113
+ new TextEncoder().encode(document),
114
+ );
115
+ const xml = await response.text();
116
+ // S3 can report a failed completion inside a 200 response.
117
+ if (!response.ok || xml.includes("<Error>")) {
118
+ throw s3Error("CompleteMultipartUpload", response.status, xml);
119
+ }
120
+ }
121
+
122
+ /** Resolves when the upload no longer exists, including when it never did. */
123
+ async abort(key: string, uploadId: string): Promise<void> {
124
+ const response = await this.send("DELETE", key, [["uploadId", uploadId]]);
125
+ const xml = await response.text();
126
+ if (response.ok) return;
127
+ const error = s3Error("AbortMultipartUpload", response.status, xml);
128
+ if (error instanceof MultipartUploadError && error.code === "UPLOAD_NOT_FOUND") return;
129
+ throw error;
130
+ }
131
+
132
+ private objectUrl(key: string): URL {
133
+ const url = new URL(this.base.href);
134
+ const basePath = url.pathname.replace(/\/$/, "");
135
+ const encodedKey = key.split("/").map(encodeRfc3986).join("/");
136
+ if (this.target.virtualHostedStyle) {
137
+ url.hostname = `${this.target.bucket}.${url.hostname}`;
138
+ url.pathname = `${basePath}/${encodedKey}`;
139
+ } else {
140
+ url.pathname = `${basePath}/${encodeRfc3986(this.target.bucket)}/${encodedKey}`;
141
+ }
142
+ return url;
143
+ }
144
+
145
+ private send(
146
+ method: string,
147
+ key: string,
148
+ query: ReadonlyArray<readonly [string, string]>,
149
+ headers: Record<string, string> = {},
150
+ body?: Uint8Array | Blob,
151
+ ): Promise<Response> {
152
+ const signed = signS3Request(
153
+ {
154
+ method,
155
+ url: this.objectUrl(key),
156
+ query,
157
+ headers,
158
+ // Payloads are sent unsigned so a part never has to be hashed before it is sent.
159
+ payloadHash: body === undefined ? EMPTY_PAYLOAD_HASH : UNSIGNED_PAYLOAD,
160
+ },
161
+ this.target.credentials,
162
+ this.target.region,
163
+ );
164
+ return this.fetchImpl(signed.url, {
165
+ method,
166
+ headers: signed.headers,
167
+ // Bytes on a SharedArrayBuffer are not a BodyInit in lib.dom; S3 parts never are.
168
+ ...(body === undefined ? {} : { body: body as BodyInit }),
169
+ });
170
+ }
171
+ }