@softure-ai/files 0.1.2 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/README.md +113 -10
- package/dist/contract.d.ts +41 -1
- package/dist/contract.d.ts.map +1 -1
- package/dist/index.d.ts +35 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +17 -4
- package/dist/index.js.map +1 -1
- package/dist/mcp/index.d.ts +41 -0
- package/dist/mcp/index.d.ts.map +1 -0
- package/dist/mcp/index.js +114 -0
- package/dist/mcp/index.js.map +1 -0
- package/dist/messages/en.d.ts +15 -0
- package/dist/messages/en.d.ts.map +1 -1
- package/dist/messages/en.js +14 -0
- package/dist/messages/en.js.map +1 -1
- package/dist/messages/index.d.ts +15 -0
- package/dist/messages/index.d.ts.map +1 -1
- package/dist/next/index.d.ts +1 -0
- package/dist/next/index.d.ts.map +1 -1
- package/dist/next/index.js +2 -1
- package/dist/next/index.js.map +1 -1
- package/dist/next/upload-route.d.ts +3 -0
- package/dist/next/upload-route.d.ts.map +1 -0
- package/dist/next/upload-route.js +11 -0
- package/dist/next/upload-route.js.map +1 -0
- package/dist/options.d.ts +6 -0
- package/dist/options.d.ts.map +1 -1
- package/dist/options.js +27 -0
- package/dist/options.js.map +1 -1
- package/dist/schema.d.ts +255 -0
- package/dist/schema.d.ts.map +1 -1
- package/dist/schema.js +17 -2
- package/dist/schema.js.map +1 -1
- package/dist/server/index.d.ts +6 -2
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +6 -2
- package/dist/server/index.js.map +1 -1
- package/dist/server/options.d.ts +2 -0
- package/dist/server/options.d.ts.map +1 -1
- package/dist/server/options.js +8 -0
- package/dist/server/options.js.map +1 -1
- package/dist/server/privacy.d.ts +6 -1
- package/dist/server/privacy.d.ts.map +1 -1
- package/dist/server/privacy.js +6 -3
- package/dist/server/privacy.js.map +1 -1
- package/dist/server/prune.d.ts +4 -0
- package/dist/server/prune.d.ts.map +1 -1
- package/dist/server/prune.js +4 -2
- package/dist/server/prune.js.map +1 -1
- package/dist/server/save.d.ts +15 -0
- package/dist/server/save.d.ts.map +1 -1
- package/dist/server/save.js +11 -9
- package/dist/server/save.js.map +1 -1
- package/dist/server/serve-upload.d.ts +6 -0
- package/dist/server/serve-upload.d.ts.map +1 -0
- package/dist/server/serve-upload.js +110 -0
- package/dist/server/serve-upload.js.map +1 -0
- package/dist/server/upload-page.d.ts +5 -0
- package/dist/server/upload-page.d.ts.map +1 -0
- package/dist/server/upload-page.js +94 -0
- package/dist/server/upload-page.js.map +1 -0
- package/dist/server/upload-tokens.d.ts +9 -0
- package/dist/server/upload-tokens.d.ts.map +1 -0
- package/dist/server/upload-tokens.js +43 -0
- package/dist/server/upload-tokens.js.map +1 -0
- package/dist/server/uploads.d.ts +73 -0
- package/dist/server/uploads.d.ts.map +1 -0
- package/dist/server/uploads.js +403 -0
- package/dist/server/uploads.js.map +1 -0
- package/migrations/0002_create_uploads.sql +33 -0
- package/module.json +9 -4
- package/package.json +12 -2
- package/src/contract.ts +45 -1
- package/src/index.ts +32 -5
- package/src/mcp/index.ts +169 -0
- package/src/messages/en.ts +14 -0
- package/src/next/index.ts +2 -1
- package/src/next/upload-route.ts +11 -0
- package/src/options.ts +28 -0
- package/src/schema.ts +20 -2
- package/src/server/index.ts +20 -2
- package/src/server/options.ts +8 -0
- package/src/server/privacy.ts +12 -4
- package/src/server/prune.ts +8 -2
- package/src/server/save.ts +25 -11
- package/src/server/serve-upload.ts +116 -0
- package/src/server/upload-page.ts +103 -0
- package/src/server/upload-tokens.ts +43 -0
- package/src/server/uploads.ts +437 -0
package/src/index.ts
CHANGED
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
// storage drivers and every database function are in `@softure-ai/files/server`, the link route in `/next`, the
|
|
3
3
|
// prune job in `/jobs` and the test helpers (an in-memory driver, link checks) in `/testing`.
|
|
4
4
|
import { defineModule, resolveMigrationsDir } from "@softure-ai/core";
|
|
5
|
+
import type { RateLimitBucketInput } from "@softure-ai/security";
|
|
5
6
|
import { filesMessages } from "./messages/index.js";
|
|
6
7
|
import { filesOptionsSchema } from "./options.js";
|
|
7
8
|
import { checkFilesHealth } from "./server/health.js";
|
|
@@ -9,6 +10,15 @@ import { filesPrivacyContributor } from "./server/privacy.js";
|
|
|
9
10
|
|
|
10
11
|
export const MODULE_ID = "files";
|
|
11
12
|
|
|
13
|
+
/**
|
|
14
|
+
* The rate limit bucket of the upload route, to spread into `security({ buckets })` when the app enables the
|
|
15
|
+
* security module: requests per upload session. 64 MiB chunks of a 2 GiB file are 32 requests; the rest is room for
|
|
16
|
+
* resumes and status checks.
|
|
17
|
+
*/
|
|
18
|
+
export const FILES_RATE_LIMIT_BUCKETS = {
|
|
19
|
+
"files-upload": { limit: 600, windowMinutes: 15, key: "subject" },
|
|
20
|
+
} as const satisfies Readonly<Record<string, RateLimitBucketInput>>;
|
|
21
|
+
|
|
12
22
|
/**
|
|
13
23
|
* Enables stored files in `softure.config.ts`:
|
|
14
24
|
* `files({ storage: createLocalStorage({ dir: "/data/files" }), quotaBytesPerOwner: 2 * 1024 ** 3 })`.
|
|
@@ -16,10 +26,10 @@ export const MODULE_ID = "files";
|
|
|
16
26
|
export const files = defineModule({
|
|
17
27
|
manifest: {
|
|
18
28
|
id: MODULE_ID,
|
|
19
|
-
version: "0.
|
|
20
|
-
dependsOn: { jobs: "^0.1.6?" },
|
|
29
|
+
version: "0.2.0",
|
|
30
|
+
dependsOn: { jobs: "^0.1.6?", security: "^0.2.5?" },
|
|
21
31
|
dbSchema: "files",
|
|
22
|
-
tables: ["files"],
|
|
32
|
+
tables: ["files", "uploads"],
|
|
23
33
|
env: [
|
|
24
34
|
{
|
|
25
35
|
name: "SOFTURE_FILES_LINK_KEYS",
|
|
@@ -29,13 +39,18 @@ export const files = defineModule({
|
|
|
29
39
|
},
|
|
30
40
|
],
|
|
31
41
|
switches: [],
|
|
32
|
-
routes: { links: "/f" },
|
|
42
|
+
routes: { links: "/f", uploads: "/u" },
|
|
33
43
|
mount: [
|
|
34
44
|
{
|
|
35
45
|
kind: "route-handler",
|
|
36
46
|
path: "app/f/[...link]/route.ts",
|
|
37
47
|
export: "serveFileLinkRoute",
|
|
38
48
|
},
|
|
49
|
+
{
|
|
50
|
+
kind: "route-handler",
|
|
51
|
+
path: "app/u/[...upload]/route.ts",
|
|
52
|
+
export: "serveUploadRoute",
|
|
53
|
+
},
|
|
39
54
|
],
|
|
40
55
|
privacy: { exports: true, deletes: true },
|
|
41
56
|
},
|
|
@@ -46,7 +61,19 @@ export const files = defineModule({
|
|
|
46
61
|
health: checkFilesHealth,
|
|
47
62
|
});
|
|
48
63
|
|
|
49
|
-
export type {
|
|
64
|
+
export type {
|
|
65
|
+
ByteRange,
|
|
66
|
+
CreatedUpload,
|
|
67
|
+
FileLink,
|
|
68
|
+
FilesErrorCode,
|
|
69
|
+
FilesKeyErrorCode,
|
|
70
|
+
FilesLookupErrorCode,
|
|
71
|
+
FilesSaveErrorCode,
|
|
72
|
+
FilesUploadErrorCode,
|
|
73
|
+
StoredFile,
|
|
74
|
+
UploadState,
|
|
75
|
+
UploadStatus,
|
|
76
|
+
} from "./contract.js";
|
|
50
77
|
export { filesMessages, getFilesErrorMessage, type FilesMessages } from "./messages/index.js";
|
|
51
78
|
export { DEFAULT_FILE_KIND, FILE_KIND_PATTERN, type FilesOptions, type FilesOptionsInput } from "./options.js";
|
|
52
79
|
export { files as filesTable } from "./schema.js";
|
package/src/mcp/index.ts
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
// The MCP adapter of @softure-ai/files: two tools around a plain HTTP upload, so an assistant hands a large file
|
|
2
|
+
// (a video) to the app without putting its bytes in a tool call. `create_upload` answers a signed upload URL with a
|
|
3
|
+
// ready `curl` loop for the chunks (and the browser page as a fallback); the assistant runs it in its own sandbox and
|
|
4
|
+
// then calls `finalize_upload`, which turns the chunks into a stored file.
|
|
5
|
+
//
|
|
6
|
+
// Both tools write, so they are registered only for an identity that can write (mcp-access's write switch). Their
|
|
7
|
+
// input schemas are plain objects; the app's factory makes them strict (`createMcpServerFactory({ strictToolInput })`).
|
|
8
|
+
//
|
|
9
|
+
// createMcpServerFactory({
|
|
10
|
+
// name: "acme",
|
|
11
|
+
// registerTools: (server, identity) => registerFileUploadTools(server, { identity, getContext: () => getFilesContext() }),
|
|
12
|
+
// });
|
|
13
|
+
// mcpAccess({ serverName: "acme", tools: [...FILE_UPLOAD_MCP_TOOLS] });
|
|
14
|
+
import type { CallToolResult, McpServer } from "@modelcontextprotocol/server";
|
|
15
|
+
import { errorLogLabel } from "@softure-ai/core";
|
|
16
|
+
import { z } from "zod";
|
|
17
|
+
import type { CreatedUpload, StoredFile } from "../contract.js";
|
|
18
|
+
import { filesMessages, getFilesErrorMessage } from "../messages/index.js";
|
|
19
|
+
import type { FilesContext } from "../server/context.js";
|
|
20
|
+
import { createUpload, finalizeUpload } from "../server/uploads.js";
|
|
21
|
+
|
|
22
|
+
export const CREATE_UPLOAD_TOOL_NAME = "create_upload";
|
|
23
|
+
export const FINALIZE_UPLOAD_TOOL_NAME = "finalize_upload";
|
|
24
|
+
|
|
25
|
+
const CREATE_UPLOAD_DESCRIPTION =
|
|
26
|
+
"Starts an upload of a file from your own environment (for a file too large for a tool call, such as a video). " +
|
|
27
|
+
"The file's bytes never go through this tool: it answers a one-time upload URL and a ready curl loop that sends the " +
|
|
28
|
+
"file in chunks; run it where the file is, then call finalize_upload with the upload_id. A person can open the " +
|
|
29
|
+
"same URL in a browser to upload the file there instead.";
|
|
30
|
+
const FINALIZE_UPLOAD_DESCRIPTION =
|
|
31
|
+
"Finishes an upload started with create_upload once every chunk is sent: checks the bytes and stores the file. " +
|
|
32
|
+
"Answers the stored file's id. Calling it again answers the same file.";
|
|
33
|
+
|
|
34
|
+
/** The catalog entries for `mcpAccess({ tools })`, so the token page and the catalog check know both tools. */
|
|
35
|
+
export const FILE_UPLOAD_MCP_TOOLS = [
|
|
36
|
+
{ name: CREATE_UPLOAD_TOOL_NAME, access: "write", description: { en: "Starts an upload of a large file to a one-time URL." } },
|
|
37
|
+
{ name: FINALIZE_UPLOAD_TOOL_NAME, access: "write", description: { en: "Stores a file once its upload is complete." } },
|
|
38
|
+
] as const;
|
|
39
|
+
|
|
40
|
+
/** What the tools need of the MCP request's identity (`McpServerIdentity` of `@softure-ai/mcp-access`). */
|
|
41
|
+
export interface FileUploadToolIdentity {
|
|
42
|
+
readonly userId: string;
|
|
43
|
+
readonly canWrite: boolean;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface FileUploadToolsOptions {
|
|
47
|
+
readonly identity: FileUploadToolIdentity;
|
|
48
|
+
/** The files context, e.g. `() => getFilesContext()` from `@softure-ai/files/next`. */
|
|
49
|
+
readonly getContext: () => FilesContext | Promise<FilesContext>;
|
|
50
|
+
/** Whose files the uploads become; the token's user by default. */
|
|
51
|
+
readonly getOwnerId?: (identity: FileUploadToolIdentity) => string;
|
|
52
|
+
/** The kind of the stored files; `file` by default. */
|
|
53
|
+
readonly kind?: string;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
const createUploadInput = z.object({
|
|
57
|
+
name: z.string().min(1).max(255).describe("The file's name, e.g. promo.mp4."),
|
|
58
|
+
size: z.number().int().min(1).describe("The file's exact size in bytes (e.g. from `wc -c < file`)."),
|
|
59
|
+
content_type: z.string().min(3).max(255).optional().describe("The media type, e.g. video/mp4."),
|
|
60
|
+
sha256: z
|
|
61
|
+
.string()
|
|
62
|
+
.regex(/^[0-9a-fA-F]{64}$/)
|
|
63
|
+
.optional()
|
|
64
|
+
.describe("The file's SHA-256 in hex; when given, the upload is refused unless the bytes match."),
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
const finalizeUploadInput = z.object({
|
|
68
|
+
upload_id: z.string().uuid().describe("The upload_id create_upload answered."),
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
/** Registers `create_upload` and `finalize_upload` for an identity that can write; does nothing for a read-only one. */
|
|
72
|
+
export function registerFileUploadTools(server: Pick<McpServer, "registerTool">, options: FileUploadToolsOptions): void {
|
|
73
|
+
if (!options.identity.canWrite) return;
|
|
74
|
+
const ownerId = (options.getOwnerId ?? ((identity) => identity.userId))(options.identity);
|
|
75
|
+
|
|
76
|
+
server.registerTool(
|
|
77
|
+
CREATE_UPLOAD_TOOL_NAME,
|
|
78
|
+
{
|
|
79
|
+
description: CREATE_UPLOAD_DESCRIPTION,
|
|
80
|
+
inputSchema: createUploadInput,
|
|
81
|
+
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
|
|
82
|
+
},
|
|
83
|
+
(args) =>
|
|
84
|
+
answer(CREATE_UPLOAD_TOOL_NAME, async () => {
|
|
85
|
+
const ctx = await options.getContext();
|
|
86
|
+
const created = await createUpload(ctx, {
|
|
87
|
+
ownerId,
|
|
88
|
+
name: args.name,
|
|
89
|
+
size: args.size,
|
|
90
|
+
...(args.content_type === undefined ? {} : { contentType: args.content_type }),
|
|
91
|
+
...(args.sha256 === undefined ? {} : { sha256: args.sha256 }),
|
|
92
|
+
...(options.kind === undefined ? {} : { kind: options.kind }),
|
|
93
|
+
});
|
|
94
|
+
return created.ok ? describeCreatedUpload(created.value, args.size) : created;
|
|
95
|
+
}),
|
|
96
|
+
);
|
|
97
|
+
|
|
98
|
+
server.registerTool(
|
|
99
|
+
FINALIZE_UPLOAD_TOOL_NAME,
|
|
100
|
+
{
|
|
101
|
+
description: FINALIZE_UPLOAD_DESCRIPTION,
|
|
102
|
+
inputSchema: finalizeUploadInput,
|
|
103
|
+
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
|
104
|
+
},
|
|
105
|
+
(args) =>
|
|
106
|
+
answer(FINALIZE_UPLOAD_TOOL_NAME, async () => {
|
|
107
|
+
const ctx = await options.getContext();
|
|
108
|
+
const finalized = await finalizeUpload(ctx, args.upload_id, { ownerId });
|
|
109
|
+
return finalized.ok ? describeStoredFile(finalized.value) : finalized;
|
|
110
|
+
}),
|
|
111
|
+
);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* A bash loop that sends `$FILE` to an upload URL in chunks with `dd` and `curl` (GNU and BSD alike). It stops at the
|
|
116
|
+
* first failed chunk; `CHUNKS="3 4"` before the loop resends only those (the URL answers `missingChunks` to a GET).
|
|
117
|
+
*/
|
|
118
|
+
export function buildCurlUploadScript(upload: Pick<CreatedUpload, "uploadUrl" | "chunkSize" | "chunkCount">, size: number): string {
|
|
119
|
+
const chunk = String(upload.chunkSize);
|
|
120
|
+
const total = String(size);
|
|
121
|
+
return [
|
|
122
|
+
`FILE='path/to/the/file'`,
|
|
123
|
+
`URL='${upload.uploadUrl}'`,
|
|
124
|
+
`for i in \${CHUNKS:-$(seq 0 ${String(upload.chunkCount - 1)})}; do`,
|
|
125
|
+
` start=$((i * ${chunk})); end=$((start + ${chunk} - 1)); [ "$end" -ge ${total} ] && end=$((${total} - 1))`,
|
|
126
|
+
` dd if="$FILE" bs=${chunk} skip="$i" count=1 2>/dev/null | curl -sS --fail -X PUT -H "Content-Range: bytes $start-$end/${total}" --data-binary @- "$URL" >/dev/null || { echo "chunk $i failed: run the loop again with CHUNKS=\\"$i ...\\""; break; }`,
|
|
127
|
+
`done`,
|
|
128
|
+
`curl -sS "$URL"`,
|
|
129
|
+
].join("\n");
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function describeCreatedUpload(upload: CreatedUpload, size: number) {
|
|
133
|
+
return {
|
|
134
|
+
upload_id: upload.uploadId,
|
|
135
|
+
upload_url: upload.uploadUrl,
|
|
136
|
+
chunk_size: upload.chunkSize,
|
|
137
|
+
chunk_count: upload.chunkCount,
|
|
138
|
+
expires_at: upload.expiresAt.toISOString(),
|
|
139
|
+
curl: buildCurlUploadScript(upload, size),
|
|
140
|
+
browser_page: upload.uploadUrl,
|
|
141
|
+
next_step:
|
|
142
|
+
"Set FILE in the curl loop to the file's path and run it; the last line prints the upload's state. When missingChunks is " +
|
|
143
|
+
"empty, call finalize_upload with this upload_id. A person without a shell can open browser_page instead.",
|
|
144
|
+
};
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function describeStoredFile(file: StoredFile) {
|
|
148
|
+
return { file_id: file.id, name: file.name, size: file.size, content_type: file.contentType, sha256: file.sha256 };
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/** The tool's answer: a value as JSON text, an error code as its English sentence, a thrown error as a fixed text. */
|
|
152
|
+
async function answer(tool: string, work: () => Promise<unknown>): Promise<CallToolResult> {
|
|
153
|
+
try {
|
|
154
|
+
const value = await work();
|
|
155
|
+
if (isFailure(value)) return failure(`${getFilesErrorMessage(filesMessages.en, value.error)} (${value.error})`);
|
|
156
|
+
return { content: [{ type: "text", text: JSON.stringify(value) }] };
|
|
157
|
+
} catch (error) {
|
|
158
|
+
console.error(`@softure-ai/files: tool ${tool} failed: ${errorLogLabel(error)}`);
|
|
159
|
+
return failure(filesMessages.en.errors.core.unexpected);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function failure(text: string): CallToolResult {
|
|
164
|
+
return { content: [{ type: "text", text }], isError: true };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
function isFailure(value: unknown): value is { readonly ok: false; readonly error: string } {
|
|
168
|
+
return typeof value === "object" && value !== null && "ok" in value && value.ok === false && "error" in value && typeof value.error === "string";
|
|
169
|
+
}
|
package/src/messages/en.ts
CHANGED
|
@@ -8,9 +8,23 @@ export const en = {
|
|
|
8
8
|
quota_exceeded: "There is no space left for this file.",
|
|
9
9
|
not_found: "The file was not found.",
|
|
10
10
|
range_not_satisfiable: "The requested part of the file is outside it.",
|
|
11
|
+
content_type_not_allowed: "This type of file cannot be uploaded.",
|
|
12
|
+
too_many_uploads: "Too many uploads are in progress. Finish or wait for one first.",
|
|
13
|
+
upload_incomplete: "The upload is not complete yet: some parts have not arrived.",
|
|
14
|
+
checksum_mismatch: "The uploaded file does not match its checksum.",
|
|
11
15
|
},
|
|
12
16
|
core: {
|
|
13
17
|
unexpected: "Something went wrong. Try again.",
|
|
14
18
|
},
|
|
19
|
+
}, uploadPage: {
|
|
20
|
+
title: "Upload a file",
|
|
21
|
+
intro: "Choose the file this upload is waiting for. It is sent in parts, and a broken connection resumes where it stopped.",
|
|
22
|
+
expected: "Expected: {name}, {size}.",
|
|
23
|
+
choose: "Choose a file",
|
|
24
|
+
start: "Upload",
|
|
25
|
+
progress: "Uploaded {done} of {total}.",
|
|
26
|
+
done: "Upload complete. Go back to where you started it to finish.",
|
|
27
|
+
wrongSize: "This file has a different size than the upload expects.",
|
|
28
|
+
failed: "The upload stopped. Press Upload again to resume.",
|
|
15
29
|
},
|
|
16
30
|
};
|
package/src/next/index.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
// The Next.js adapter of @softure-ai/files: the public link route and the request-side context
|
|
1
|
+
// The Next.js adapter of @softure-ai/files: the public link route, the upload route and the request-side context
|
|
2
2
|
// (docs/02-module-standard.md §8).
|
|
3
3
|
export { getFilesContext } from "./context.js";
|
|
4
4
|
export { serveFileLinkRoute } from "./route.js";
|
|
5
|
+
export { serveUploadRoute } from "./upload-route.js";
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// The upload route, mounted with one line:
|
|
2
|
+
// `app/u/[...upload]/route.ts → export { serveUploadRoute as GET, serveUploadRoute as HEAD, serveUploadRoute as PUT } from "@softure-ai/files/next";`
|
|
3
|
+
// It reads no session: an upload URL is protected by its signature, so the route must stay outside the app's
|
|
4
|
+
// sign-in guard and the edge's sign-in (README §4). The path is read from the request URL, not from Next's params.
|
|
5
|
+
import { serveUpload } from "../server/serve-upload.js";
|
|
6
|
+
import { getFilesContext } from "./context.js";
|
|
7
|
+
|
|
8
|
+
/** `GET`, `HEAD` and `PUT /u/<id>/<expiry>.<signature>`. */
|
|
9
|
+
export async function serveUploadRoute(request: Request): Promise<Response> {
|
|
10
|
+
return serveUpload(await getFilesContext(), request);
|
|
11
|
+
}
|
package/src/options.ts
CHANGED
|
@@ -9,6 +9,9 @@ const DAY_SECONDS = 24 * 60 * 60;
|
|
|
9
9
|
|
|
10
10
|
/** A file kind: lowercase letters, digits, `_` and `-`, starting with a letter, at most 32 characters. */
|
|
11
11
|
export const FILE_KIND_PATTERN = /^[a-z][a-z0-9_-]{0,31}$/;
|
|
12
|
+
/** An allowed upload media type: `type/subtype` or `type/*`, lowercase. */
|
|
13
|
+
const CONTENT_TYPE_RULE_PATTERN = /^[a-z0-9][a-z0-9!#$&^_.+-]*\/(\*|[a-z0-9][a-z0-9!#$&^_.+-]*)$/;
|
|
14
|
+
|
|
12
15
|
/** The kind of a file saved without one. */
|
|
13
16
|
export const DEFAULT_FILE_KIND = "file";
|
|
14
17
|
|
|
@@ -47,6 +50,31 @@ export const filesOptionsSchema = z.strictObject({
|
|
|
47
50
|
disposition: z.enum(["attachment", "inline"]).default("attachment"),
|
|
48
51
|
})
|
|
49
52
|
.default({ maxTtlSeconds: 7 * DAY_SECONDS, disposition: "attachment" }),
|
|
53
|
+
uploads: z
|
|
54
|
+
.strictObject({
|
|
55
|
+
/**
|
|
56
|
+
* The bytes of one upload request: a chunk. Keep it under the edge's body limit (Cloudflare refuses bodies over
|
|
57
|
+
* 100 MB on Free and Pro plans).
|
|
58
|
+
*/
|
|
59
|
+
chunkBytes: z
|
|
60
|
+
.number()
|
|
61
|
+
.int()
|
|
62
|
+
.min(1024)
|
|
63
|
+
.max(1024 * MIB)
|
|
64
|
+
.default(64 * MIB),
|
|
65
|
+
/** How long an upload session (and its URL) works; `createUpload` may ask for less. */
|
|
66
|
+
ttlSeconds: z
|
|
67
|
+
.number()
|
|
68
|
+
.int()
|
|
69
|
+
.min(60)
|
|
70
|
+
.max(7 * DAY_SECONDS)
|
|
71
|
+
.default(DAY_SECONDS),
|
|
72
|
+
/** Sessions one owner may have open at once (not finalized, not expired). */
|
|
73
|
+
maxOpenPerOwner: z.number().int().min(1).max(1000).default(5),
|
|
74
|
+
/** The media types an upload may declare (`video/mp4`, `video/*`); null: any. */
|
|
75
|
+
contentTypes: z.array(z.string().regex(CONTENT_TYPE_RULE_PATTERN, "must be type/subtype or type/*")).min(1).nullable().default(null),
|
|
76
|
+
})
|
|
77
|
+
.default({ chunkBytes: 64 * MIB, ttlSeconds: DAY_SECONDS, maxOpenPerOwner: 5, contentTypes: null }),
|
|
50
78
|
health: z
|
|
51
79
|
.strictObject({
|
|
52
80
|
/** Below this many free bytes (when the driver can tell) the health check fails. */
|
package/src/schema.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
// Drizzle view of the module's
|
|
2
|
-
//
|
|
1
|
+
// Drizzle view of the module's tables (migrations 0001 and 0002). The migrations are the source of truth; this file
|
|
2
|
+
// only types the queries.
|
|
3
3
|
import { bigint, integer, pgSchema, text, timestamp, uuid } from "drizzle-orm/pg-core";
|
|
4
4
|
|
|
5
5
|
export const filesSchema = pgSchema("files");
|
|
@@ -20,3 +20,21 @@ export const files = filesSchema.table("files", {
|
|
|
20
20
|
});
|
|
21
21
|
|
|
22
22
|
export type FileRow = typeof files.$inferSelect;
|
|
23
|
+
|
|
24
|
+
export const uploads = filesSchema.table("uploads", {
|
|
25
|
+
id: uuid("id").primaryKey(),
|
|
26
|
+
ownerId: text("owner_id").notNull(),
|
|
27
|
+
originalName: text("original_name").notNull(),
|
|
28
|
+
contentType: text("content_type").notNull(),
|
|
29
|
+
kind: text("kind").notNull(),
|
|
30
|
+
size: bigint("size", { mode: "number" }).notNull(),
|
|
31
|
+
sha256: text("sha256"),
|
|
32
|
+
chunkSize: integer("chunk_size").notNull(),
|
|
33
|
+
receivedChunks: integer("received_chunks").array().notNull(),
|
|
34
|
+
fileId: uuid("file_id"),
|
|
35
|
+
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
|
|
36
|
+
createdAt: timestamp("created_at", { withTimezone: true }).notNull(),
|
|
37
|
+
finalizedAt: timestamp("finalized_at", { withTimezone: true }),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export type UploadRow = typeof uploads.$inferSelect;
|
package/src/server/index.ts
CHANGED
|
@@ -29,7 +29,7 @@ export {
|
|
|
29
29
|
} from "./links.js";
|
|
30
30
|
export { createLocalStorage, type LocalStorageOptions } from "./local-storage.js";
|
|
31
31
|
export { DEFAULT_CONTENT_TYPE, formatContentDisposition, normalizeContentType, sanitizeFileName } from "./names.js";
|
|
32
|
-
export { getFilesModule, getFilesOptions, getLinksRoute } from "./options.js";
|
|
32
|
+
export { getFilesModule, getFilesOptions, getLinksRoute, getUploadsRoute } from "./options.js";
|
|
33
33
|
export { deleteFilesUserData, exportFilesUserData, filesPrivacyContributor, type FilesDeletionReport, type FilesUserData } from "./privacy.js";
|
|
34
34
|
export { pruneFiles, type PruneFilesOptions, type PruneFilesReport } from "./prune.js";
|
|
35
35
|
export {
|
|
@@ -40,6 +40,24 @@ export {
|
|
|
40
40
|
splitIntoParts,
|
|
41
41
|
type S3StorageOptions,
|
|
42
42
|
} from "./s3-storage.js";
|
|
43
|
-
export { saveFile, type SaveFileInput, type SaveFileResult } from "./save.js";
|
|
43
|
+
export { saveFile, validateFileDescription, type FileDescription, type FileDescriptionInput, type SaveFileInput, type SaveFileResult } from "./save.js";
|
|
44
|
+
export {
|
|
45
|
+
countChunks,
|
|
46
|
+
createUpload,
|
|
47
|
+
finalizeUpload,
|
|
48
|
+
getPartKey,
|
|
49
|
+
getUpload,
|
|
50
|
+
writeUploadChunk,
|
|
51
|
+
type CreateUploadInput,
|
|
52
|
+
type CreateUploadResult,
|
|
53
|
+
type FinalizeUploadResult,
|
|
54
|
+
type UploadChunkErrorCode,
|
|
55
|
+
type UploadLookupOptions,
|
|
56
|
+
type WriteUploadChunkInput,
|
|
57
|
+
type WriteUploadChunkResult,
|
|
58
|
+
} from "./uploads.js";
|
|
59
|
+
export { signUploadPath, verifyUploadPath } from "./upload-tokens.js";
|
|
60
|
+
export { FILES_UPLOAD_RATE_LIMIT_BUCKET, serveUpload } from "./serve-upload.js";
|
|
61
|
+
export { renderUploadPage } from "./upload-page.js";
|
|
44
62
|
export { parseRangeHeader, serveFileLink } from "./serve.js";
|
|
45
63
|
export { EMPTY_PAYLOAD_SHA256, encodeRfc3986, signRequest, type SigV4Credentials, type SignRequestInput } from "./sigv4.js";
|
package/src/server/options.ts
CHANGED
|
@@ -26,6 +26,14 @@ export function getLinksRoute(config: SoftureConfig): string {
|
|
|
26
26
|
return trimTrailingSlashes(links);
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
/** The path prefix of upload URLs after the app's override (`files({ routes: { uploads } })`), without a trailing slash. */
|
|
30
|
+
export function getUploadsRoute(config: SoftureConfig): string {
|
|
31
|
+
const { uploads } = getFilesModule(config).routes;
|
|
32
|
+
// The manifest declares it, so a missing one means a broken module definition.
|
|
33
|
+
if (uploads === undefined) throw new Error("@softure-ai/files: the uploads route is missing from the module manifest");
|
|
34
|
+
return trimTrailingSlashes(uploads);
|
|
35
|
+
}
|
|
36
|
+
|
|
29
37
|
/** The path without trailing slashes; a loop, not `/\/+$/`, which backtracks quadratically on a run of slashes. */
|
|
30
38
|
function trimTrailingSlashes(path: string): string {
|
|
31
39
|
let end = path.length;
|
package/src/server/privacy.ts
CHANGED
|
@@ -1,24 +1,31 @@
|
|
|
1
1
|
// The files part of a GDPR export and deletion (`@softure-ai/privacy`). A file belongs to its owner: the export
|
|
2
2
|
// lists the owner's live files (never the bytes or the storage key; the app links them itself when it wants), and
|
|
3
|
-
// the deletion removes every file of the owner from storage and the table.
|
|
3
|
+
// the deletion removes every file of the owner from storage and the table. Upload sessions are the owner's too: the
|
|
4
|
+
// export lists the open ones, the deletion removes all of them with their parts.
|
|
4
5
|
import { ok, type Ok, type PrivacyContributor } from "@softure-ai/core";
|
|
5
6
|
import type { Queryable } from "@softure-ai/db";
|
|
6
7
|
import { and, asc, eq } from "drizzle-orm";
|
|
7
|
-
import type { StoredFile } from "../contract.js";
|
|
8
|
+
import type { StoredFile, UploadStatus } from "../contract.js";
|
|
8
9
|
import { files } from "../schema.js";
|
|
9
10
|
import { toStoredFile, type FilesContext } from "./context.js";
|
|
10
11
|
import { removeMarkedFile } from "./files.js";
|
|
11
12
|
import { isLive } from "./queries.js";
|
|
13
|
+
import { deleteOwnerUploads, listOpenUploads } from "./uploads.js";
|
|
12
14
|
|
|
13
15
|
/** What files holds about one user, as it appears in their export. */
|
|
14
16
|
export interface FilesUserData {
|
|
15
17
|
readonly files: readonly StoredFile[];
|
|
18
|
+
/** Upload sessions still taking chunks. */
|
|
19
|
+
readonly uploads: readonly UploadStatus[];
|
|
16
20
|
}
|
|
17
21
|
|
|
18
22
|
export interface FilesDeletionReport {
|
|
19
23
|
readonly deletedFiles: number;
|
|
20
24
|
/** Files whose storage delete failed; their rows stay marked deleted and `pruneFiles` removes them. */
|
|
21
25
|
readonly pendingFiles: number;
|
|
26
|
+
readonly deletedUploads: number;
|
|
27
|
+
/** Sessions whose parts could not be deleted; `pruneFiles` removes them once they expire. */
|
|
28
|
+
readonly pendingUploads: number;
|
|
22
29
|
}
|
|
23
30
|
|
|
24
31
|
export async function exportFilesUserData(ctx: FilesContext, userId: string): Promise<Ok<FilesUserData>> {
|
|
@@ -27,7 +34,7 @@ export async function exportFilesUserData(ctx: FilesContext, userId: string): Pr
|
|
|
27
34
|
.from(files)
|
|
28
35
|
.where(and(eq(files.ownerId, userId), isLive(ctx.clock.now())))
|
|
29
36
|
.orderBy(asc(files.createdAt), asc(files.id));
|
|
30
|
-
return ok({ files: rows.map(toStoredFile) });
|
|
37
|
+
return ok({ files: rows.map(toStoredFile), uploads: await listOpenUploads(ctx, userId) });
|
|
31
38
|
}
|
|
32
39
|
|
|
33
40
|
export async function deleteFilesUserData(ctx: FilesContext, userId: string): Promise<Ok<FilesDeletionReport>> {
|
|
@@ -39,7 +46,8 @@ export async function deleteFilesUserData(ctx: FilesContext, userId: string): Pr
|
|
|
39
46
|
for (const row of rows) {
|
|
40
47
|
if (await removeMarkedFile(ctx, row)) deletedFiles += 1;
|
|
41
48
|
}
|
|
42
|
-
|
|
49
|
+
const removedUploads = await deleteOwnerUploads(ctx, userId);
|
|
50
|
+
return ok({ deletedFiles, pendingFiles: rows.length - deletedFiles, deletedUploads: removedUploads.deletedUploads, pendingUploads: removedUploads.failedUploads });
|
|
43
51
|
}
|
|
44
52
|
|
|
45
53
|
export const filesPrivacyContributor: PrivacyContributor = {
|
package/src/server/prune.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
// Retention: files past their expiry and files deleted but not yet out of storage leave storage, then the table
|
|
1
|
+
// Retention: files past their expiry and files deleted but not yet out of storage leave storage, then the table;
|
|
2
|
+
// upload sessions past their expiry leave with their parts.
|
|
2
3
|
// Run it from the `files.prune` job (`@softure-ai/files/jobs`) or a cron calling `pruneFiles(ctx)`.
|
|
3
4
|
import { asc, isNotNull, lte, or } from "drizzle-orm";
|
|
4
5
|
import { files } from "../schema.js";
|
|
5
6
|
import type { FilesContext } from "./context.js";
|
|
6
7
|
import { removeMarkedFile } from "./files.js";
|
|
8
|
+
import { pruneUploads } from "./uploads.js";
|
|
7
9
|
|
|
8
10
|
const DEFAULT_BATCH_SIZE = 500;
|
|
9
11
|
|
|
@@ -17,6 +19,10 @@ export interface PruneFilesReport {
|
|
|
17
19
|
readonly freedBytes: number;
|
|
18
20
|
/** Files whose storage delete failed; they stay for the next run. */
|
|
19
21
|
readonly failedFiles: number;
|
|
22
|
+
/** Expired upload sessions removed with their parts. */
|
|
23
|
+
readonly deletedUploads: number;
|
|
24
|
+
/** Expired upload sessions whose parts could not be deleted; they stay for the next run. */
|
|
25
|
+
readonly failedUploads: number;
|
|
20
26
|
}
|
|
21
27
|
|
|
22
28
|
export async function pruneFiles(ctx: FilesContext, options: PruneFilesOptions = {}): Promise<PruneFilesReport> {
|
|
@@ -41,6 +47,6 @@ export async function pruneFiles(ctx: FilesContext, options: PruneFilesOptions =
|
|
|
41
47
|
failedFiles += 1;
|
|
42
48
|
}
|
|
43
49
|
}
|
|
44
|
-
if (rows.length < batchSize) return { deletedFiles, freedBytes, failedFiles };
|
|
50
|
+
if (rows.length < batchSize) return { deletedFiles, freedBytes, failedFiles, ...(await pruneUploads(ctx, batchSize)) };
|
|
45
51
|
}
|
|
46
52
|
}
|
package/src/server/save.ts
CHANGED
|
@@ -123,27 +123,41 @@ export async function saveFile(ctx: FilesContext, input: SaveFileInput): Promise
|
|
|
123
123
|
return stored.ok ? ok(stored.value.row) : stored;
|
|
124
124
|
}
|
|
125
125
|
|
|
126
|
-
interface ValidFields {
|
|
126
|
+
interface ValidFields extends FileDescription {
|
|
127
|
+
readonly expiresAt: Date | null;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
function validateFields(input: SaveFileInput, now: Date, defaultTtlSeconds: number | null): Ok<ValidFields> | Err<"files.invalid_input"> {
|
|
131
|
+
const described = validateFileDescription(input);
|
|
132
|
+
if (!described.ok) return described;
|
|
133
|
+
const expiresAt = resolveExpiry(input, now, defaultTtlSeconds);
|
|
134
|
+
if (expiresAt === false) return err("files.invalid_input");
|
|
135
|
+
return ok({ ...described.value, expiresAt });
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/** What describes a file before its bytes, as a caller gives it. */
|
|
139
|
+
export interface FileDescriptionInput {
|
|
140
|
+
readonly ownerId: string;
|
|
141
|
+
readonly name: string;
|
|
142
|
+
readonly contentType?: string;
|
|
143
|
+
readonly kind?: string;
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
/** The description made safe to store: the owner checked, the name sanitised, the media type normalised. */
|
|
147
|
+
export interface FileDescription {
|
|
127
148
|
readonly originalName: string;
|
|
128
149
|
readonly contentType: string;
|
|
129
150
|
readonly kind: string;
|
|
130
|
-
readonly expiresAt: Date | null;
|
|
131
151
|
}
|
|
132
152
|
|
|
133
|
-
|
|
153
|
+
/** Checks the owner, kind and media type of a file to save or upload, and sanitises its name. */
|
|
154
|
+
export function validateFileDescription(input: FileDescriptionInput): Ok<FileDescription> | Err<"files.invalid_input"> {
|
|
134
155
|
if (input.ownerId.length === 0 || input.ownerId.length > MAX_OWNER_ID_LENGTH) return err("files.invalid_input");
|
|
135
156
|
const kind = input.kind ?? DEFAULT_FILE_KIND;
|
|
136
157
|
if (!FILE_KIND_PATTERN.test(kind)) return err("files.invalid_input");
|
|
137
158
|
const contentType = normalizeContentType(input.contentType ?? DEFAULT_CONTENT_TYPE);
|
|
138
159
|
if (contentType === null) return err("files.invalid_input");
|
|
139
|
-
|
|
140
|
-
if (expiresAt === false) return err("files.invalid_input");
|
|
141
|
-
return ok({
|
|
142
|
-
originalName: sanitizeFileName(input.name),
|
|
143
|
-
contentType,
|
|
144
|
-
kind,
|
|
145
|
-
expiresAt,
|
|
146
|
-
});
|
|
160
|
+
return ok({ originalName: sanitizeFileName(input.name), contentType, kind });
|
|
147
161
|
}
|
|
148
162
|
|
|
149
163
|
/** The expiry from `expiresAt`, `ttlSeconds` or the default; false when the input is contradictory or in the past. */
|