@softure-ai/files 0.0.0-stage → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/CHANGELOG.md +26 -0
  2. package/LICENSE +21 -0
  3. package/README.md +213 -2
  4. package/dist/contract.d.ts +51 -0
  5. package/dist/contract.d.ts.map +1 -0
  6. package/dist/contract.js +4 -0
  7. package/dist/contract.js.map +1 -0
  8. package/dist/index.d.ts +45 -0
  9. package/dist/index.d.ts.map +1 -0
  10. package/dist/index.js +49 -0
  11. package/dist/index.js.map +1 -0
  12. package/dist/jobs/index.d.ts +12 -0
  13. package/dist/jobs/index.d.ts.map +1 -0
  14. package/dist/jobs/index.js +22 -0
  15. package/dist/jobs/index.js.map +1 -0
  16. package/dist/messages/en.d.ts +17 -0
  17. package/dist/messages/en.d.ts.map +1 -0
  18. package/dist/messages/en.js +17 -0
  19. package/dist/messages/en.js.map +1 -0
  20. package/dist/messages/index.d.ts +24 -0
  21. package/dist/messages/index.d.ts.map +1 -0
  22. package/dist/messages/index.js +10 -0
  23. package/dist/messages/index.js.map +1 -0
  24. package/dist/next/context.d.ts +4 -0
  25. package/dist/next/context.d.ts.map +1 -0
  26. package/dist/next/context.js +13 -0
  27. package/dist/next/context.js.map +1 -0
  28. package/dist/next/index.d.ts +3 -0
  29. package/dist/next/index.d.ts.map +1 -0
  30. package/dist/next/index.js +5 -0
  31. package/dist/next/index.js.map +1 -0
  32. package/dist/next/route.d.ts +3 -0
  33. package/dist/next/route.d.ts.map +1 -0
  34. package/dist/next/route.js +11 -0
  35. package/dist/next/route.js.map +1 -0
  36. package/dist/options.d.ts +26 -0
  37. package/dist/options.d.ts.map +1 -0
  38. package/dist/options.js +59 -0
  39. package/dist/options.js.map +1 -0
  40. package/dist/schema.d.ts +214 -0
  41. package/dist/schema.d.ts.map +1 -0
  42. package/dist/schema.js +19 -0
  43. package/dist/schema.js.map +1 -0
  44. package/dist/server/context.d.ts +13 -0
  45. package/dist/server/context.d.ts.map +1 -0
  46. package/dist/server/context.js +19 -0
  47. package/dist/server/context.js.map +1 -0
  48. package/dist/server/files.d.ts +48 -0
  49. package/dist/server/files.d.ts.map +1 -0
  50. package/dist/server/files.js +104 -0
  51. package/dist/server/files.js.map +1 -0
  52. package/dist/server/health.d.ts +4 -0
  53. package/dist/server/health.d.ts.map +1 -0
  54. package/dist/server/health.js +20 -0
  55. package/dist/server/health.js.map +1 -0
  56. package/dist/server/index.d.ts +15 -0
  57. package/dist/server/index.d.ts.map +1 -0
  58. package/dist/server/index.js +17 -0
  59. package/dist/server/index.js.map +1 -0
  60. package/dist/server/keys.d.ts +14 -0
  61. package/dist/server/keys.d.ts.map +1 -0
  62. package/dist/server/keys.js +47 -0
  63. package/dist/server/keys.js.map +1 -0
  64. package/dist/server/links.d.ts +30 -0
  65. package/dist/server/links.d.ts.map +1 -0
  66. package/dist/server/links.js +122 -0
  67. package/dist/server/links.js.map +1 -0
  68. package/dist/server/local-storage.d.ts +7 -0
  69. package/dist/server/local-storage.d.ts.map +1 -0
  70. package/dist/server/local-storage.js +83 -0
  71. package/dist/server/local-storage.js.map +1 -0
  72. package/dist/server/names.d.ts +11 -0
  73. package/dist/server/names.d.ts.map +1 -0
  74. package/dist/server/names.js +32 -0
  75. package/dist/server/names.js.map +1 -0
  76. package/dist/server/options.d.ts +8 -0
  77. package/dist/server/options.d.ts.map +1 -0
  78. package/dist/server/options.js +31 -0
  79. package/dist/server/options.js.map +1 -0
  80. package/dist/server/privacy.d.ts +16 -0
  81. package/dist/server/privacy.d.ts.map +1 -0
  82. package/dist/server/privacy.js +39 -0
  83. package/dist/server/privacy.js.map +1 -0
  84. package/dist/server/prune.d.ts +13 -0
  85. package/dist/server/prune.d.ts.map +1 -0
  86. package/dist/server/prune.js +34 -0
  87. package/dist/server/prune.js.map +1 -0
  88. package/dist/server/queries.d.ts +7 -0
  89. package/dist/server/queries.d.ts.map +1 -0
  90. package/dist/server/queries.js +17 -0
  91. package/dist/server/queries.js.map +1 -0
  92. package/dist/server/s3-storage.d.ts +33 -0
  93. package/dist/server/s3-storage.d.ts.map +1 -0
  94. package/dist/server/s3-storage.js +223 -0
  95. package/dist/server/s3-storage.js.map +1 -0
  96. package/dist/server/save.d.ts +29 -0
  97. package/dist/server/save.d.ts.map +1 -0
  98. package/dist/server/save.js +141 -0
  99. package/dist/server/save.js.map +1 -0
  100. package/dist/server/serve.d.ts +11 -0
  101. package/dist/server/serve.d.ts.map +1 -0
  102. package/dist/server/serve.js +108 -0
  103. package/dist/server/serve.js.map +1 -0
  104. package/dist/server/sigv4.d.ts +23 -0
  105. package/dist/server/sigv4.d.ts.map +1 -0
  106. package/dist/server/sigv4.js +57 -0
  107. package/dist/server/sigv4.js.map +1 -0
  108. package/dist/storage.d.ts +32 -0
  109. package/dist/storage.d.ts.map +1 -0
  110. package/dist/storage.js +10 -0
  111. package/dist/storage.js.map +1 -0
  112. package/dist/testing/index.d.ts +3 -0
  113. package/dist/testing/index.d.ts.map +1 -0
  114. package/dist/testing/index.js +4 -0
  115. package/dist/testing/index.js.map +1 -0
  116. package/dist/testing/links.d.ts +22 -0
  117. package/dist/testing/links.d.ts.map +1 -0
  118. package/dist/testing/links.js +30 -0
  119. package/dist/testing/links.js.map +1 -0
  120. package/dist/testing/memory-storage.d.ts +11 -0
  121. package/dist/testing/memory-storage.d.ts.map +1 -0
  122. package/dist/testing/memory-storage.js +52 -0
  123. package/dist/testing/memory-storage.js.map +1 -0
  124. package/migrations/0001_create_files.sql +32 -0
  125. package/module.json +24 -0
  126. package/package.json +72 -4
  127. package/src/contract.ts +59 -0
  128. package/src/index.ts +53 -0
  129. package/src/jobs/index.ts +23 -0
  130. package/src/messages/en.ts +16 -0
  131. package/src/messages/index.ts +13 -0
  132. package/src/next/context.ts +14 -0
  133. package/src/next/index.ts +4 -0
  134. package/src/next/route.ts +11 -0
  135. package/src/options.ts +64 -0
  136. package/src/schema.ts +22 -0
  137. package/src/server/context.ts +32 -0
  138. package/src/server/files.ts +133 -0
  139. package/src/server/health.ts +20 -0
  140. package/src/server/index.ts +45 -0
  141. package/src/server/keys.ts +56 -0
  142. package/src/server/links.ts +152 -0
  143. package/src/server/local-storage.ts +87 -0
  144. package/src/server/names.ts +35 -0
  145. package/src/server/options.ts +34 -0
  146. package/src/server/privacy.ts +53 -0
  147. package/src/server/prune.ts +46 -0
  148. package/src/server/queries.ts +19 -0
  149. package/src/server/s3-storage.ts +255 -0
  150. package/src/server/save.ts +167 -0
  151. package/src/server/serve.ts +106 -0
  152. package/src/server/sigv4.ts +82 -0
  153. package/src/storage.ts +43 -0
  154. package/src/testing/index.ts +3 -0
  155. package/src/testing/links.ts +47 -0
  156. package/src/testing/memory-storage.ts +64 -0
@@ -0,0 +1,34 @@
1
+ // The files options of the running app, read from the configuration in the module context.
2
+ import { getModule, type SoftureConfig } from "@softure-ai/core";
3
+ import type { FilesOptions } from "../options.js";
4
+
5
+ const MODULE_ID = "files";
6
+
7
+ /** The enabled module. Throws when the app did not enable it: calling its functions then is a bug. */
8
+ export function getFilesModule(config: SoftureConfig) {
9
+ const module = getModule(config, MODULE_ID);
10
+ if (module === undefined) {
11
+ throw new Error("@softure-ai/files: the module is not enabled; add files({ storage }) to modules in softure.config.ts");
12
+ }
13
+ return module;
14
+ }
15
+
16
+ export function getFilesOptions(config: SoftureConfig): FilesOptions {
17
+ // The module factory parsed these options with filesOptionsSchema.
18
+ return getFilesModule(config).options as FilesOptions;
19
+ }
20
+
21
+ /** The path prefix of public links after the app's override (`files({ routes: { links } })`), without a trailing slash. */
22
+ export function getLinksRoute(config: SoftureConfig): string {
23
+ const { links } = getFilesModule(config).routes;
24
+ // The manifest declares it, so a missing one means a broken module definition.
25
+ if (links === undefined) throw new Error("@softure-ai/files: the links route is missing from the module manifest");
26
+ return trimTrailingSlashes(links);
27
+ }
28
+
29
+ /** The path without trailing slashes; a loop, not `/\/+$/`, which backtracks quadratically on a run of slashes. */
30
+ function trimTrailingSlashes(path: string): string {
31
+ let end = path.length;
32
+ while (end > 0 && path[end - 1] === "/") end -= 1;
33
+ return path.slice(0, end);
34
+ }
@@ -0,0 +1,53 @@
1
+ // The files part of a GDPR export and deletion (`@softure-ai/privacy`). A file belongs to its owner: the export
2
+ // lists the owner's live files (never the bytes or the storage key; the app links them itself when it wants), and
3
+ // the deletion removes every file of the owner from storage and the table.
4
+ import { ok, type Ok, type PrivacyContributor } from "@softure-ai/core";
5
+ import type { Queryable } from "@softure-ai/db";
6
+ import { and, asc, eq } from "drizzle-orm";
7
+ import type { StoredFile } from "../contract.js";
8
+ import { files } from "../schema.js";
9
+ import { toStoredFile, type FilesContext } from "./context.js";
10
+ import { removeMarkedFile } from "./files.js";
11
+ import { isLive } from "./queries.js";
12
+
13
+ /** What files holds about one user, as it appears in their export. */
14
+ export interface FilesUserData {
15
+ readonly files: readonly StoredFile[];
16
+ }
17
+
18
+ export interface FilesDeletionReport {
19
+ readonly deletedFiles: number;
20
+ /** Files whose storage delete failed; their rows stay marked deleted and `pruneFiles` removes them. */
21
+ readonly pendingFiles: number;
22
+ }
23
+
24
+ export async function exportFilesUserData(ctx: FilesContext, userId: string): Promise<Ok<FilesUserData>> {
25
+ const rows = await ctx.db
26
+ .select()
27
+ .from(files)
28
+ .where(and(eq(files.ownerId, userId), isLive(ctx.clock.now())))
29
+ .orderBy(asc(files.createdAt), asc(files.id));
30
+ return ok({ files: rows.map(toStoredFile) });
31
+ }
32
+
33
+ export async function deleteFilesUserData(ctx: FilesContext, userId: string): Promise<Ok<FilesDeletionReport>> {
34
+ const now = ctx.clock.now();
35
+ // Marked first, in one statement: nothing of the owner is served from here on, whatever storage does next.
36
+ await ctx.db.update(files).set({ deletedAt: now }).where(eq(files.ownerId, userId));
37
+ const rows = await ctx.db.select().from(files).where(eq(files.ownerId, userId));
38
+ let deletedFiles = 0;
39
+ for (const row of rows) {
40
+ if (await removeMarkedFile(ctx, row)) deletedFiles += 1;
41
+ }
42
+ return ok({ deletedFiles, pendingFiles: rows.length - deletedFiles });
43
+ }
44
+
45
+ export const filesPrivacyContributor: PrivacyContributor = {
46
+ // Core types `db` as unknown; privacy passes the @softure-ai/db handle or its transaction.
47
+ exportUserData: (context, userId) => exportFilesUserData({ ...context, db: context.db as Queryable }, userId),
48
+ // Privacy's contract answers a deletion with an empty ok; the counts are for apps that call the helper.
49
+ deleteUserData: async (context, userId) => {
50
+ await deleteFilesUserData({ ...context, db: context.db as Queryable }, userId);
51
+ return ok();
52
+ },
53
+ };
@@ -0,0 +1,46 @@
1
+ // Retention: files past their expiry and files deleted but not yet out of storage leave storage, then the table.
2
+ // Run it from the `files.prune` job (`@softure-ai/files/jobs`) or a cron calling `pruneFiles(ctx)`.
3
+ import { asc, isNotNull, lte, or } from "drizzle-orm";
4
+ import { files } from "../schema.js";
5
+ import type { FilesContext } from "./context.js";
6
+ import { removeMarkedFile } from "./files.js";
7
+
8
+ const DEFAULT_BATCH_SIZE = 500;
9
+
10
+ export interface PruneFilesOptions {
11
+ /** Rows read per round; rounds repeat until one comes back short. */
12
+ readonly batchSize?: number;
13
+ }
14
+
15
+ export interface PruneFilesReport {
16
+ readonly deletedFiles: number;
17
+ readonly freedBytes: number;
18
+ /** Files whose storage delete failed; they stay for the next run. */
19
+ readonly failedFiles: number;
20
+ }
21
+
22
+ export async function pruneFiles(ctx: FilesContext, options: PruneFilesOptions = {}): Promise<PruneFilesReport> {
23
+ const batchSize = Math.max(1, Math.trunc(options.batchSize ?? DEFAULT_BATCH_SIZE));
24
+ const now = ctx.clock.now();
25
+ let deletedFiles = 0;
26
+ let freedBytes = 0;
27
+ let failedFiles = 0;
28
+ for (;;) {
29
+ const rows = await ctx.db
30
+ .select()
31
+ .from(files)
32
+ .where(or(isNotNull(files.deletedAt), lte(files.expiresAt, now)))
33
+ .orderBy(asc(files.createdAt), asc(files.id))
34
+ .limit(batchSize)
35
+ .offset(failedFiles);
36
+ for (const row of rows) {
37
+ if (await removeMarkedFile(ctx, row)) {
38
+ deletedFiles += 1;
39
+ freedBytes += row.size;
40
+ } else {
41
+ failedFiles += 1;
42
+ }
43
+ }
44
+ if (rows.length < batchSize) return { deletedFiles, freedBytes, failedFiles };
45
+ }
46
+ }
@@ -0,0 +1,19 @@
1
+ // Conditions and reads shared by the server functions.
2
+ import { and, eq, gt, isNull, or, sql, type SQL } from "drizzle-orm";
3
+ import type { Queryable } from "@softure-ai/db";
4
+ import { files } from "../schema.js";
5
+
6
+ /** A file that is neither deleted nor past its expiry at `now`. */
7
+ export function isLive(now: Date): SQL {
8
+ // `and` of two defined conditions is defined.
9
+ return and(isNull(files.deletedAt), or(isNull(files.expiresAt), gt(files.expiresAt, now))) as SQL;
10
+ }
11
+
12
+ /** The bytes an owner keeps in live files. */
13
+ export async function sumOwnerBytes(db: Queryable, ownerId: string, now: Date): Promise<number> {
14
+ const [row] = await db
15
+ .select({ total: sql<string>`coalesce(sum(${files.size}), 0)::text` })
16
+ .from(files)
17
+ .where(and(eq(files.ownerId, ownerId), isLive(now)));
18
+ return Number(row?.total ?? "0");
19
+ }
@@ -0,0 +1,255 @@
1
+ // The S3-compatible driver (Cloudflare R2, MinIO, Garage, AWS S3): path-style URLs, `<endpoint>/<bucket>/<key>`,
2
+ // signed with SigV4. A body that fits one part goes up in a single `PUT`; a longer one as a multipart upload, so
3
+ // memory holds one part at a time (`partSizeBytes`, 8 MiB by default) and a failure aborts the upload, leaving no
4
+ // object. The access keys are read from the environment on each request, so building the driver needs neither.
5
+ import { createHash, randomUUID } from "node:crypto";
6
+ import { errorLogLabel, type FetchFunction } from "@softure-ai/core";
7
+ import type { ByteRange } from "../contract.js";
8
+ import type { FileBody, FileStorage, StorageCheck } from "../storage.js";
9
+ import type { Env } from "./context.js";
10
+ import { EMPTY_PAYLOAD_SHA256, encodeRfc3986, signRequest, type SigV4Credentials } from "./sigv4.js";
11
+
12
+ const MIB = 1024 * 1024;
13
+ /** S3's smallest part (except the last one). */
14
+ const MIN_PART_BYTES = 5 * MIB;
15
+ const DEFAULT_PART_BYTES = 8 * MIB;
16
+ const DEFAULT_TIMEOUT_MS = 60_000;
17
+ const LOCAL_HOSTS = new Set(["localhost", "127.0.0.1", "[::1]"]);
18
+ const BUCKET_PATTERN = /^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$/;
19
+ const PREFIX_PATTERN = /^(?:[A-Za-z0-9][A-Za-z0-9._-]*\/)*$/;
20
+
21
+ export interface S3StorageOptions {
22
+ /** The service origin: `https://<account>.r2.cloudflarestorage.com`, `http://localhost:9000` for MinIO. */
23
+ readonly endpoint: string;
24
+ readonly bucket: string;
25
+ /** `auto` for R2; the bucket's region elsewhere (`us-east-1` for MinIO and Garage defaults). */
26
+ readonly region?: string;
27
+ /** A folder inside the bucket, ending in `/` (`files/`); none by default. */
28
+ readonly prefix?: string;
29
+ readonly accessKeyIdEnv?: string;
30
+ readonly secretAccessKeyEnv?: string;
31
+ /** The part size of a multipart upload, at least 5 MiB. */
32
+ readonly partSizeBytes?: number;
33
+ /** How long a request may wait for the service's answer (the headers). */
34
+ readonly timeoutMs?: number;
35
+ /** For tests: the `fetch` the driver calls, the environment the keys come from and the signing time. */
36
+ readonly fetch?: FetchFunction;
37
+ readonly env?: Env;
38
+ readonly now?: () => Date;
39
+ }
40
+
41
+ export const S3_ACCESS_KEY_ID_ENV = "SOFTURE_FILES_S3_ACCESS_KEY_ID";
42
+ export const S3_SECRET_ACCESS_KEY_ENV = "SOFTURE_FILES_S3_SECRET_ACCESS_KEY";
43
+
44
+ interface S3Request {
45
+ readonly method: string;
46
+ readonly key: string;
47
+ readonly query?: Readonly<Record<string, string>>;
48
+ readonly headers?: Readonly<Record<string, string>>;
49
+ readonly body?: Uint8Array;
50
+ }
51
+
52
+ /** Thrown for an answer the driver did not expect; carries the operation and status, never a URL or a key. */
53
+ export class S3StorageError extends Error {
54
+ constructor(
55
+ operation: string,
56
+ readonly status: number,
57
+ ) {
58
+ super(`@softure-ai/files: S3 ${operation} answered ${String(status)}`);
59
+ this.name = "S3StorageError";
60
+ }
61
+ }
62
+
63
+ export function createS3Storage(options: S3StorageOptions): FileStorage {
64
+ const endpoint = parseEndpoint(options.endpoint);
65
+ if (!BUCKET_PATTERN.test(options.bucket)) throw new RangeError("createS3Storage: bucket must be a valid S3 bucket name");
66
+ const prefix = options.prefix ?? "";
67
+ if (!PREFIX_PATTERN.test(prefix)) throw new RangeError("createS3Storage: prefix must be folder names ending in /, e.g. files/");
68
+ const partSize = options.partSizeBytes ?? DEFAULT_PART_BYTES;
69
+ if (!Number.isSafeInteger(partSize) || partSize < MIN_PART_BYTES) throw new RangeError("createS3Storage: partSizeBytes must be at least 5 MiB");
70
+ const region = options.region ?? "auto";
71
+ const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
72
+ const accessKeyIdEnv = options.accessKeyIdEnv ?? S3_ACCESS_KEY_ID_ENV;
73
+ const secretAccessKeyEnv = options.secretAccessKeyEnv ?? S3_SECRET_ACCESS_KEY_ENV;
74
+
75
+ function readCredentials(): SigV4Credentials {
76
+ const env = options.env ?? process.env;
77
+ const accessKeyId = env[accessKeyIdEnv]?.trim();
78
+ const secretAccessKey = env[secretAccessKeyEnv]?.trim();
79
+ if (accessKeyId === undefined || accessKeyId === "" || secretAccessKey === undefined || secretAccessKey === "") {
80
+ throw new Error(`@softure-ai/files: the S3 driver needs ${accessKeyIdEnv} and ${secretAccessKeyEnv}`);
81
+ }
82
+ return { accessKeyId, secretAccessKey };
83
+ }
84
+
85
+ async function send(request: S3Request): Promise<Response> {
86
+ const objectPath = `${prefix}${request.key}`.split("/").map(encodeRfc3986).join("/");
87
+ const url = new URL(`${endpoint.pathname}${encodeRfc3986(options.bucket)}/${objectPath}`, endpoint.origin);
88
+ for (const [name, value] of Object.entries(request.query ?? {})) url.searchParams.set(name, value);
89
+ const payloadSha256 = request.body === undefined ? EMPTY_PAYLOAD_SHA256 : createHash("sha256").update(request.body).digest("hex");
90
+ const signed = signRequest({
91
+ method: request.method,
92
+ url,
93
+ headers: request.headers,
94
+ payloadSha256,
95
+ region,
96
+ credentials: readCredentials(),
97
+ date: options.now?.() ?? new Date(),
98
+ });
99
+ // fetch sets Host from the URL itself (the same value that was signed).
100
+ const headers = Object.fromEntries(Object.entries(signed).filter(([name]) => name !== "host"));
101
+ const controller = new AbortController();
102
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
103
+ try {
104
+ // A Buffer is a Uint8Array over an ArrayBufferLike; fetch takes it as bytes, the DOM typing only names ArrayBuffer.
105
+ const body = request.body as Uint8Array<ArrayBuffer> | undefined;
106
+ return await (options.fetch ?? fetch)(url, {
107
+ method: request.method,
108
+ headers,
109
+ body,
110
+ signal: controller.signal,
111
+ redirect: "manual",
112
+ });
113
+ } finally {
114
+ clearTimeout(timer);
115
+ }
116
+ }
117
+
118
+ async function expectOk(response: Response, operation: string): Promise<string> {
119
+ const text = await response.text();
120
+ // CompleteMultipartUpload can answer 200 with an <Error> body.
121
+ if (!response.ok || text.includes("<Error>")) throw new S3StorageError(operation, response.status);
122
+ return text;
123
+ }
124
+
125
+ async function putMultipart(parts: AsyncIterator<Buffer>, first: Buffer, second: Buffer, key: string): Promise<void> {
126
+ const created = await expectOk(await send({ method: "POST", key, query: { uploads: "" } }), "CreateMultipartUpload");
127
+ const uploadId = /<UploadId>([^<]+)<\/UploadId>/.exec(created)?.[1];
128
+ if (uploadId === undefined) throw new S3StorageError("CreateMultipartUpload", 200);
129
+ const etags: string[] = [];
130
+ try {
131
+ const uploadPart = async (body: Buffer) => {
132
+ const response = await send({
133
+ method: "PUT",
134
+ key,
135
+ query: { partNumber: String(etags.length + 1), uploadId },
136
+ body,
137
+ });
138
+ await expectOk(response, "UploadPart");
139
+ const etag = response.headers.get("etag");
140
+ if (etag === null) throw new S3StorageError("UploadPart", response.status);
141
+ etags.push(etag);
142
+ };
143
+ await uploadPart(first);
144
+ await uploadPart(second);
145
+ for (let next = await parts.next(); next.done !== true; next = await parts.next()) await uploadPart(next.value);
146
+ const xml = `<CompleteMultipartUpload>${etags.map((etag, index) => `<Part><PartNumber>${String(index + 1)}</PartNumber><ETag>${escapeXml(etag)}</ETag></Part>`).join("")}</CompleteMultipartUpload>`;
147
+ await expectOk(
148
+ await send({
149
+ method: "POST",
150
+ key,
151
+ query: { uploadId },
152
+ headers: { "content-type": "application/xml" },
153
+ body: Buffer.from(xml),
154
+ }),
155
+ "CompleteMultipartUpload",
156
+ );
157
+ } catch (error) {
158
+ // Best effort: an upload left open is billed until the bucket's lifecycle rule removes it.
159
+ await send({ method: "DELETE", key, query: { uploadId } }).then(
160
+ (response) => response.body?.cancel(),
161
+ () => undefined,
162
+ );
163
+ throw error;
164
+ }
165
+ }
166
+
167
+ return {
168
+ kind: "s3",
169
+ async put(body: FileBody, key: string) {
170
+ const parts = splitIntoParts(body, partSize)[Symbol.asyncIterator]();
171
+ const first = await parts.next();
172
+ const second = first.done === true ? first : await parts.next();
173
+ if (second.done === true) {
174
+ const content = first.done === true ? Buffer.alloc(0) : first.value;
175
+ await expectOk(await send({ method: "PUT", key, body: content }), "PutObject");
176
+ return;
177
+ }
178
+ // Two parts read means the first one is a value too.
179
+ await putMultipart(parts, first.value as Buffer, second.value, key);
180
+ },
181
+ async get(key: string, range?: ByteRange) {
182
+ const headers = range === undefined ? undefined : { range: `bytes=${String(range.start)}-${String(range.end)}` };
183
+ const response = await send({ method: "GET", key, headers });
184
+ if (response.status === 404) {
185
+ await response.body?.cancel();
186
+ return null;
187
+ }
188
+ if ((response.status !== 200 && response.status !== 206) || response.body === null) {
189
+ await response.body?.cancel();
190
+ throw new S3StorageError("GetObject", response.status);
191
+ }
192
+ return response.body;
193
+ },
194
+ async head(key: string) {
195
+ const response = await send({ method: "HEAD", key });
196
+ await response.body?.cancel();
197
+ if (response.status === 404) return null;
198
+ if (!response.ok) throw new S3StorageError("HeadObject", response.status);
199
+ return { size: Number(response.headers.get("content-length") ?? "0") };
200
+ },
201
+ async delete(key: string) {
202
+ const response = await send({ method: "DELETE", key });
203
+ await response.body?.cancel();
204
+ if (!response.ok && response.status !== 404) throw new S3StorageError("DeleteObject", response.status);
205
+ },
206
+ async check(): Promise<StorageCheck> {
207
+ const key = `health-probe/${randomUUID()}`;
208
+ try {
209
+ await expectOk(await send({ method: "PUT", key, body: Buffer.from("ok") }), "PutObject");
210
+ const response = await send({ method: "DELETE", key });
211
+ await response.body?.cancel();
212
+ return { isWritable: true, freeBytes: null };
213
+ } catch (error) {
214
+ console.error(`@softure-ai/files: the S3 health probe failed: ${errorLogLabel(error)}`);
215
+ return { isWritable: false, freeBytes: null };
216
+ }
217
+ },
218
+ };
219
+ }
220
+
221
+ function parseEndpoint(value: string): URL {
222
+ let url: URL;
223
+ try {
224
+ url = new URL(value);
225
+ } catch {
226
+ throw new RangeError("createS3Storage: endpoint must be a URL such as https://<account>.r2.cloudflarestorage.com");
227
+ }
228
+ const isAllowed = url.protocol === "https:" || (url.protocol === "http:" && LOCAL_HOSTS.has(url.hostname));
229
+ if (!isAllowed || url.search !== "" || url.hash !== "") {
230
+ throw new RangeError("createS3Storage: endpoint must be an https origin (http only on localhost), without query or fragment");
231
+ }
232
+ if (!url.pathname.endsWith("/")) url.pathname = `${url.pathname}/`;
233
+ return url;
234
+ }
235
+
236
+ /** The body re-cut into parts of exactly `size` bytes (the last one shorter), holding at most one part plus a chunk. */
237
+ export async function* splitIntoParts(body: FileBody, size: number): AsyncGenerator<Buffer> {
238
+ let pending: Buffer[] = [];
239
+ let pendingBytes = 0;
240
+ for await (const chunk of body) {
241
+ pending.push(Buffer.from(chunk.buffer, chunk.byteOffset, chunk.byteLength));
242
+ pendingBytes += chunk.byteLength;
243
+ while (pendingBytes >= size) {
244
+ const joined = Buffer.concat(pending, pendingBytes);
245
+ yield joined.subarray(0, size);
246
+ pending = [joined.subarray(size)];
247
+ pendingBytes -= size;
248
+ }
249
+ }
250
+ if (pendingBytes > 0) yield Buffer.concat(pending, pendingBytes);
251
+ }
252
+
253
+ function escapeXml(value: string): string {
254
+ return value.replace(/&/g, "&amp;").replace(/</g, "&lt;").replace(/>/g, "&gt;").replace(/"/g, "&quot;");
255
+ }
@@ -0,0 +1,167 @@
1
+ // Saving a file: the body is hashed and counted while it streams into the storage driver, so memory holds one
2
+ // chunk at a time, and the size cap stops it at the first chunk over. The row goes in afterwards, under a per-owner
3
+ // advisory lock that serializes the quota check (and the dedupe lookup) with other saves for the same owner. Any
4
+ // refusal after the write deletes the blob it wrote.
5
+ import { createHash, randomUUID, type Hash } from "node:crypto";
6
+ import { createReadStream } from "node:fs";
7
+ import { err, errorLogLabel, ok, type Err, type Ok } from "@softure-ai/core";
8
+ import type { Queryable } from "@softure-ai/db";
9
+ import { and, eq, sql } from "drizzle-orm";
10
+ import type { FilesSaveErrorCode, StoredFile } from "../contract.js";
11
+ import { DEFAULT_FILE_KIND, FILE_KIND_PATTERN } from "../options.js";
12
+ import { files } from "../schema.js";
13
+ import type { FileBody, FileStorage } from "../storage.js";
14
+ import { getStorageKey, toStoredFile, type FilesContext } from "./context.js";
15
+ import { DEFAULT_CONTENT_TYPE, normalizeContentType, sanitizeFileName } from "./names.js";
16
+ import { getFilesOptions } from "./options.js";
17
+ import { isLive, sumOwnerBytes } from "./queries.js";
18
+
19
+ const MAX_OWNER_ID_LENGTH = 200;
20
+
21
+ interface SaveFileFields {
22
+ readonly ownerId: string;
23
+ /** The name the file came with; only its last path part is kept, sanitised. */
24
+ readonly name: string;
25
+ /** Defaults to `application/octet-stream`. */
26
+ readonly contentType?: string;
27
+ /** The app's own category; defaults to `file`. */
28
+ readonly kind?: string;
29
+ /** When the file stops being served and `pruneFiles` removes it; give this or `ttlSeconds`, or neither. */
30
+ readonly expiresAt?: Date;
31
+ readonly ttlSeconds?: number;
32
+ }
33
+
34
+ /** A file from a stream (a request body, a renderer's output) or from a path on the server's disk. */
35
+ export type SaveFileInput = SaveFileFields & ({ readonly body: FileBody; readonly path?: never } | { readonly path: string; readonly body?: never });
36
+
37
+ export type SaveFileResult = Ok<StoredFile> | Err<FilesSaveErrorCode>;
38
+
39
+ /** Thrown into the driver's write when the body passes the cap; the driver drops the partial object. */
40
+ class SizeLimitError extends Error {
41
+ constructor() {
42
+ super("@softure-ai/files: the body passed the size limit");
43
+ }
44
+ }
45
+
46
+ interface Meter {
47
+ size: number;
48
+ readonly hash: Hash;
49
+ }
50
+
51
+ async function* meterBody(body: FileBody, limit: number, meter: Meter): AsyncGenerator<Uint8Array> {
52
+ for await (const chunk of body as AsyncIterable<Uint8Array | string>) {
53
+ // A Node stream with an encoding set yields strings.
54
+ const bytes = typeof chunk === "string" ? Buffer.from(chunk) : chunk;
55
+ meter.size += bytes.byteLength;
56
+ if (meter.size > limit) throw new SizeLimitError();
57
+ meter.hash.update(bytes);
58
+ yield bytes;
59
+ }
60
+ }
61
+
62
+ /** Saves a file for `ownerId`. A storage failure propagates; refusals are codes. */
63
+ export async function saveFile(ctx: FilesContext, input: SaveFileInput): Promise<SaveFileResult> {
64
+ const options = getFilesOptions(ctx.config);
65
+ const now = ctx.clock.now();
66
+ const fields = validateFields(input, now, options.defaultTtlSeconds);
67
+ if (!fields.ok) return fields;
68
+
69
+ const remaining = options.quotaBytesPerOwner === null ? Infinity : options.quotaBytesPerOwner - (await sumOwnerBytes(ctx.db, input.ownerId, now));
70
+ if (remaining < 0) return err("files.quota_exceeded");
71
+ const limit = Math.min(options.maxFileBytes, remaining);
72
+
73
+ const id = randomUUID();
74
+ const storageKey = getStorageKey(id);
75
+ const meter: Meter = { size: 0, hash: createHash("sha256") };
76
+ const source: FileBody = input.body ?? createReadStream(input.path);
77
+ try {
78
+ await options.storage.put(meterBody(source, limit, meter), storageKey);
79
+ } catch (error) {
80
+ await deleteBlobQuietly(options.storage, storageKey);
81
+ if (error instanceof SizeLimitError) return err(meter.size > options.maxFileBytes ? "files.too_large" : "files.quota_exceeded");
82
+ throw error;
83
+ }
84
+
85
+ const sha256 = meter.hash.digest("hex");
86
+ let stored: Ok<{ row: StoredFile; isDuplicate: boolean }> | Err<"files.quota_exceeded">;
87
+ try {
88
+ stored = await ctx.db.transaction(async (tx: Queryable) => {
89
+ await tx.execute(sql`select pg_advisory_xact_lock(hashtextextended(${`files:${input.ownerId}`}, 0))`);
90
+ if (options.dedupe) {
91
+ const [existing] = await tx
92
+ .select()
93
+ .from(files)
94
+ .where(and(eq(files.ownerId, input.ownerId), eq(files.sha256, sha256), isLive(now)))
95
+ .limit(1);
96
+ if (existing !== undefined) return ok({ row: toStoredFile(existing), isDuplicate: true });
97
+ }
98
+ if (options.quotaBytesPerOwner !== null && (await sumOwnerBytes(tx, input.ownerId, now)) + meter.size > options.quotaBytesPerOwner) {
99
+ return err("files.quota_exceeded");
100
+ }
101
+ const [row] = await tx
102
+ .insert(files)
103
+ .values({
104
+ id,
105
+ ownerId: input.ownerId,
106
+ sha256,
107
+ size: meter.size,
108
+ ...fields.value,
109
+ storageKey,
110
+ linkGeneration: 1,
111
+ createdAt: now,
112
+ })
113
+ .returning();
114
+ // An insert with returning answers the row it wrote.
115
+ if (row === undefined) throw new Error("@softure-ai/files: the insert returned no row");
116
+ return ok({ row: toStoredFile(row), isDuplicate: false });
117
+ });
118
+ } catch (error) {
119
+ await deleteBlobQuietly(options.storage, storageKey);
120
+ throw error;
121
+ }
122
+ if (!stored.ok || stored.value.isDuplicate) await deleteBlobQuietly(options.storage, storageKey);
123
+ return stored.ok ? ok(stored.value.row) : stored;
124
+ }
125
+
126
+ interface ValidFields {
127
+ readonly originalName: string;
128
+ readonly contentType: string;
129
+ readonly kind: string;
130
+ readonly expiresAt: Date | null;
131
+ }
132
+
133
+ function validateFields(input: SaveFileInput, now: Date, defaultTtlSeconds: number | null): Ok<ValidFields> | Err<"files.invalid_input"> {
134
+ if (input.ownerId.length === 0 || input.ownerId.length > MAX_OWNER_ID_LENGTH) return err("files.invalid_input");
135
+ const kind = input.kind ?? DEFAULT_FILE_KIND;
136
+ if (!FILE_KIND_PATTERN.test(kind)) return err("files.invalid_input");
137
+ const contentType = normalizeContentType(input.contentType ?? DEFAULT_CONTENT_TYPE);
138
+ if (contentType === null) return err("files.invalid_input");
139
+ const expiresAt = resolveExpiry(input, now, defaultTtlSeconds);
140
+ if (expiresAt === false) return err("files.invalid_input");
141
+ return ok({
142
+ originalName: sanitizeFileName(input.name),
143
+ contentType,
144
+ kind,
145
+ expiresAt,
146
+ });
147
+ }
148
+
149
+ /** The expiry from `expiresAt`, `ttlSeconds` or the default; false when the input is contradictory or in the past. */
150
+ function resolveExpiry(input: SaveFileFields, now: Date, defaultTtlSeconds: number | null): Date | null | false {
151
+ if (input.expiresAt !== undefined && input.ttlSeconds !== undefined) return false;
152
+ if (input.expiresAt !== undefined) {
153
+ return Number.isNaN(input.expiresAt.getTime()) || input.expiresAt.getTime() <= now.getTime() ? false : input.expiresAt;
154
+ }
155
+ const ttlSeconds = input.ttlSeconds ?? defaultTtlSeconds;
156
+ if (ttlSeconds === null) return null;
157
+ return Number.isSafeInteger(ttlSeconds) && ttlSeconds > 0 ? new Date(now.getTime() + ttlSeconds * 1000) : false;
158
+ }
159
+
160
+ /** Deletes a blob after a refused or failed save; a failure here is logged, the row (if any) decides the rest. */
161
+ async function deleteBlobQuietly(storage: FileStorage, key: string): Promise<void> {
162
+ try {
163
+ await storage.delete(key);
164
+ } catch (error) {
165
+ console.error(`@softure-ai/files: deleting the blob of a refused save failed: ${errorLogLabel(error)}`);
166
+ }
167
+ }