@softure-ai/files 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +127 -3
- package/dist/client/index.d.ts +3 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +6 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/send-upload.d.ts +26 -0
- package/dist/client/send-upload.d.ts.map +1 -0
- package/dist/client/send-upload.js +66 -0
- package/dist/client/send-upload.js.map +1 -0
- package/dist/client/upload-queue.d.ts +52 -0
- package/dist/client/upload-queue.d.ts.map +1 -0
- package/dist/client/upload-queue.js +164 -0
- package/dist/client/upload-queue.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/mcp/index.d.ts +25 -1
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +31 -14
- package/dist/mcp/index.js.map +1 -1
- package/dist/messages/en.d.ts +5 -0
- package/dist/messages/en.d.ts.map +1 -1
- package/dist/messages/en.js +7 -1
- package/dist/messages/en.js.map +1 -1
- package/dist/messages/index.d.ts +5 -0
- package/dist/messages/index.d.ts.map +1 -1
- package/dist/next/index.d.ts +1 -0
- package/dist/next/index.d.ts.map +1 -1
- package/dist/next/index.js +3 -2
- package/dist/next/index.js.map +1 -1
- package/dist/next/stored-file-route.d.ts +25 -0
- package/dist/next/stored-file-route.d.ts.map +1 -0
- package/dist/next/stored-file-route.js +14 -0
- package/dist/next/stored-file-route.js.map +1 -0
- package/dist/server/index.d.ts +1 -1
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +1 -1
- package/dist/server/index.js.map +1 -1
- package/dist/server/serve.d.ts +15 -0
- package/dist/server/serve.d.ts.map +1 -1
- package/dist/server/serve.js +38 -14
- package/dist/server/serve.js.map +1 -1
- package/dist/server/upload-page.d.ts.map +1 -1
- package/dist/server/upload-page.js +16 -23
- package/dist/server/upload-page.js.map +1 -1
- package/module.json +1 -1
- package/package.json +7 -2
- package/src/client/index.ts +12 -0
- package/src/client/send-upload.ts +89 -0
- package/src/client/upload-queue.ts +220 -0
- package/src/index.ts +1 -1
- package/src/mcp/index.ts +52 -15
- package/src/messages/en.ts +7 -1
- package/src/next/index.ts +9 -2
- package/src/next/stored-file-route.ts +43 -0
- package/src/server/index.ts +1 -1
- package/src/server/serve.ts +51 -14
- package/src/server/upload-page.ts +16 -23
package/module.json
CHANGED
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@softure-ai/files",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"description": "SOFTURE AI files module: an app's stored files (content hash, size, type, owner, kind, expiry, a per-owner quota, opt-in dedupe) on a storage driver (local disk with atomic writes, S3-compatible with SigV4 and multipart uploads, in memory for tests), streamed and never buffered, public links that expire (HMAC-signed, checked before any database read, revocable per file, a rotating keyring) served by a Next.js route with byte ranges and one 404 for every dead link, resumable chunked uploads through signed URLs (with a browser page and MCP upload tools), a prune job, a privacy contributor and a health check.",
|
|
3
|
+
"version": "0.3.0",
|
|
4
|
+
"description": "SOFTURE AI files module: an app's stored files (content hash, size, type, owner, kind, expiry, a per-owner quota, opt-in dedupe) on a storage driver (local disk with atomic writes, S3-compatible with SigV4 and multipart uploads, in memory for tests), streamed and never buffered, public links that expire (HMAC-signed, checked before any database read, revocable per file, a rotating keyring) served by a Next.js route with byte ranges and one 404 for every dead link, the same answer for the signed-in owner by id without a link, resumable chunked uploads through signed URLs (with a browser client, an upload list, a browser page and MCP upload tools), a prune job, a privacy contributor and a health check.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|
|
7
7
|
"sideEffects": false,
|
|
@@ -32,6 +32,11 @@
|
|
|
32
32
|
"types": "./dist/index.d.ts",
|
|
33
33
|
"default": "./dist/index.js"
|
|
34
34
|
},
|
|
35
|
+
"./client": {
|
|
36
|
+
"@softure-ai/source": "./src/client/index.ts",
|
|
37
|
+
"types": "./dist/client/index.d.ts",
|
|
38
|
+
"default": "./dist/client/index.js"
|
|
39
|
+
},
|
|
35
40
|
"./server": {
|
|
36
41
|
"@softure-ai/source": "./src/server/index.ts",
|
|
37
42
|
"types": "./dist/server/index.d.ts",
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
// The browser side of upload sessions: `sendUpload` sends a `File` to a session's signed URL in chunks, resumably,
|
|
2
|
+
// and `createUploadQueue` keeps a list of uploads for an upload screen (theme-notion's `UploadItem` takes its items
|
|
3
|
+
// as they are). No Node built-ins and no server code: safe to import from a client component.
|
|
4
|
+
export { sendUpload, type SendUploadErrorCode, type SendUploadOptions, type SendUploadResult, type UploadProgress } from "./send-upload.js";
|
|
5
|
+
export {
|
|
6
|
+
createUploadQueue,
|
|
7
|
+
type UploadQueue,
|
|
8
|
+
type UploadQueueItem,
|
|
9
|
+
type UploadQueueOptions,
|
|
10
|
+
type UploadQueueSession,
|
|
11
|
+
type UploadQueueState,
|
|
12
|
+
} from "./upload-queue.js";
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
// Sends a browser `File` (any `Blob`) to an upload session: reads what arrived from the session's `HEAD`, then `PUT`s
|
|
2
|
+
// only the missing chunks, in order, with `Content-Range`. The session decides the chunk size; the route refuses any
|
|
3
|
+
// other alignment. Every expected failure is a value, never a throw.
|
|
4
|
+
//
|
|
5
|
+
// `sendUpload` is self-contained on purpose: it references nothing outside its own body, so the package's upload
|
|
6
|
+
// page inlines its source (`sendUpload.toString()`) and runs the very same loop without a bundler or a second copy.
|
|
7
|
+
// Keep every helper inside the function and import types only.
|
|
8
|
+
import type { Err, Ok } from "@softure-ai/core";
|
|
9
|
+
|
|
10
|
+
/** Why a send stopped: the session is gone, the route refused it, the network failed, or the caller aborted. */
|
|
11
|
+
export type SendUploadErrorCode = "files.upload_expired" | "files.upload_rejected" | "files.network" | "files.upload_aborted";
|
|
12
|
+
|
|
13
|
+
/** Bytes the session holds so far out of the file's size. */
|
|
14
|
+
export interface UploadProgress {
|
|
15
|
+
readonly sentBytes: number;
|
|
16
|
+
readonly totalBytes: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export interface SendUploadOptions {
|
|
20
|
+
/** The session's signed upload URL, as `createUpload` returned it. */
|
|
21
|
+
readonly uploadUrl: string;
|
|
22
|
+
/** Aborts between chunks and the chunk in flight; the send answers `files.upload_aborted`. */
|
|
23
|
+
readonly signal?: AbortSignal;
|
|
24
|
+
/** Called once with what already arrived, then after every chunk. */
|
|
25
|
+
readonly onProgress?: (progress: UploadProgress) => void;
|
|
26
|
+
/** The `fetch` to use; the global one by default. */
|
|
27
|
+
readonly fetch?: (url: string, init: RequestInit) => Promise<Response>;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export type SendUploadResult = Ok<UploadProgress> | Err<SendUploadErrorCode>;
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Sends the chunks of `file` the session does not hold yet. Resuming is calling it again: a chunk that arrived is
|
|
34
|
+
* never sent twice. `404` answers `files.upload_expired`, another `4xx` (or a file whose size is not the session's)
|
|
35
|
+
* `files.upload_rejected`, a failed request, `429` or `5xx` `files.network`.
|
|
36
|
+
*/
|
|
37
|
+
export async function sendUpload(file: Blob, options: SendUploadOptions): Promise<SendUploadResult> {
|
|
38
|
+
const { uploadUrl, signal, onProgress } = options;
|
|
39
|
+
const request = options.fetch ?? ((url: string, init: RequestInit) => fetch(url, init));
|
|
40
|
+
const fail = (error: SendUploadErrorCode): Err<SendUploadErrorCode> => ({ ok: false, error });
|
|
41
|
+
const failForStatus = (status: number): Err<SendUploadErrorCode> => {
|
|
42
|
+
if (status === 404) return fail("files.upload_expired");
|
|
43
|
+
return fail(status === 429 || status >= 500 ? "files.network" : "files.upload_rejected");
|
|
44
|
+
};
|
|
45
|
+
const isAborted = (): boolean => signal?.aborted === true;
|
|
46
|
+
const call = async (init: RequestInit): Promise<Response | SendUploadErrorCode> => {
|
|
47
|
+
if (isAborted()) return "files.upload_aborted";
|
|
48
|
+
try {
|
|
49
|
+
return await request(uploadUrl, { ...init, signal: signal ?? null });
|
|
50
|
+
} catch {
|
|
51
|
+
// A thrown fetch is the network or the abort; both are expected and answered as a code.
|
|
52
|
+
return isAborted() ? "files.upload_aborted" : "files.network";
|
|
53
|
+
}
|
|
54
|
+
};
|
|
55
|
+
|
|
56
|
+
const head = await call({ method: "HEAD" });
|
|
57
|
+
if (typeof head === "string") return fail(head);
|
|
58
|
+
if (!head.ok) return failForStatus(head.status);
|
|
59
|
+
const totalBytes = Number(head.headers.get("upload-size"));
|
|
60
|
+
const chunkSize = Number(head.headers.get("upload-chunk-size"));
|
|
61
|
+
if (!Number.isSafeInteger(totalBytes) || !Number.isSafeInteger(chunkSize) || chunkSize <= 0 || totalBytes !== file.size) {
|
|
62
|
+
return fail("files.upload_rejected");
|
|
63
|
+
}
|
|
64
|
+
const received: { start: number; end: number }[] = [];
|
|
65
|
+
for (const part of (head.headers.get("upload-received") ?? "").split(",")) {
|
|
66
|
+
const match = /^(\d+)-(\d+)$/.exec(part.trim());
|
|
67
|
+
if (match !== null) received.push({ start: Number(match[1]), end: Number(match[2]) });
|
|
68
|
+
}
|
|
69
|
+
const missing: { start: number; end: number }[] = [];
|
|
70
|
+
for (let start = 0; start < totalBytes; start += chunkSize) {
|
|
71
|
+
const end = Math.min(start + chunkSize, totalBytes) - 1;
|
|
72
|
+
if (!received.some((range) => range.start <= start && range.end >= end)) missing.push({ start, end });
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
let sentBytes = totalBytes - missing.reduce((sum, chunk) => sum + chunk.end - chunk.start + 1, 0);
|
|
76
|
+
onProgress?.({ sentBytes, totalBytes });
|
|
77
|
+
for (const chunk of missing) {
|
|
78
|
+
const answer = await call({
|
|
79
|
+
method: "PUT",
|
|
80
|
+
headers: { "content-range": `bytes ${String(chunk.start)}-${String(chunk.end)}/${String(totalBytes)}` },
|
|
81
|
+
body: file.slice(chunk.start, chunk.end + 1),
|
|
82
|
+
});
|
|
83
|
+
if (typeof answer === "string") return fail(answer);
|
|
84
|
+
if (!answer.ok) return failForStatus(answer.status);
|
|
85
|
+
sentBytes += chunk.end - chunk.start + 1;
|
|
86
|
+
onProgress?.({ sentBytes, totalBytes });
|
|
87
|
+
}
|
|
88
|
+
return { ok: true, value: { sentBytes, totalBytes } };
|
|
89
|
+
}
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// A list of uploads for an upload screen, without a framework: files go in, each is opened as a session by the app,
|
|
2
|
+
// sent with `sendUpload` and finalized by the app, and the list says where each one stands. Its items fit
|
|
3
|
+
// theme-notion's `UploadItem` as they are (`UploadFileItem`: id, name, size, state, progress 0-100, error).
|
|
4
|
+
//
|
|
5
|
+
// React reads it with `useSyncExternalStore(queue.subscribe, queue.getItems)`: `getItems` answers the same array until
|
|
6
|
+
// something changes.
|
|
7
|
+
import type { ErrorCode, Result } from "@softure-ai/core";
|
|
8
|
+
import { filesMessages, getFilesErrorMessage } from "../messages/index.js";
|
|
9
|
+
import { sendUpload, type SendUploadOptions, type UploadProgress } from "./send-upload.js";
|
|
10
|
+
|
|
11
|
+
export type UploadQueueState = "queued" | "uploading" | "paused" | "done" | "failed";
|
|
12
|
+
|
|
13
|
+
/** One upload of the list. */
|
|
14
|
+
export interface UploadQueueItem {
|
|
15
|
+
readonly id: string;
|
|
16
|
+
readonly name: string;
|
|
17
|
+
/** In bytes. */
|
|
18
|
+
readonly size: number;
|
|
19
|
+
readonly state: UploadQueueState;
|
|
20
|
+
/** 0-100, what the session holds; 100 only once finalized. */
|
|
21
|
+
readonly progress: number;
|
|
22
|
+
/** The copy for `errorCode`, on a failed item. */
|
|
23
|
+
readonly error?: string;
|
|
24
|
+
/** Why it failed: a `files.*` code of the client or whatever code the app's `open` or `finalize` answered. */
|
|
25
|
+
readonly errorCode?: ErrorCode;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** What the app's `open` answers: at least the session's upload URL (`createUpload`'s answer fits). */
|
|
29
|
+
export interface UploadQueueSession {
|
|
30
|
+
readonly uploadUrl: string;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export interface UploadQueueOptions<TSession extends UploadQueueSession> {
|
|
34
|
+
/** Opens a session for a file, usually a server action around `createUpload`. */
|
|
35
|
+
readonly open: (file: File) => Promise<Result<TSession>>;
|
|
36
|
+
/** Turns a sent session into a stored file, usually a server action around `finalizeUpload`. */
|
|
37
|
+
readonly finalize: (session: TSession, file: File) => Promise<Result<unknown>>;
|
|
38
|
+
/** Uploads sent at the same time; 1 by default. */
|
|
39
|
+
readonly concurrency?: number;
|
|
40
|
+
/** The copy for a failure code; the built-in English `files` messages by default. */
|
|
41
|
+
readonly describeError?: (code: ErrorCode) => string;
|
|
42
|
+
/** The `fetch` the sender uses; the global one by default. */
|
|
43
|
+
readonly fetch?: SendUploadOptions["fetch"];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export interface UploadQueue {
|
|
47
|
+
/** Queues files (a `FileList` works) and answers their item ids. */
|
|
48
|
+
add(files: Iterable<File> | ArrayLike<File>): string[];
|
|
49
|
+
/** Queues a failed upload again; it keeps its session unless the session expired. */
|
|
50
|
+
retry(id: string): void;
|
|
51
|
+
/** Stops a queued or running upload; what arrived stays in the session. */
|
|
52
|
+
pause(id: string): void;
|
|
53
|
+
/** Queues a paused upload again; it sends only what is missing. */
|
|
54
|
+
resume(id: string): void;
|
|
55
|
+
/** Stops an upload and drops it from the list. An unfinished session expires and `files.prune` removes it. */
|
|
56
|
+
remove(id: string): void;
|
|
57
|
+
/** The list, in the order files were added; the same array until something changes. */
|
|
58
|
+
getItems(): readonly UploadQueueItem[];
|
|
59
|
+
/** Calls `listener` after every change; answers the function that stops it. */
|
|
60
|
+
subscribe(listener: () => void): () => void;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
interface Entry<TSession> {
|
|
64
|
+
item: UploadQueueItem;
|
|
65
|
+
readonly file: File;
|
|
66
|
+
session: TSession | null;
|
|
67
|
+
// Bumped by pause and remove: a run that settles under an older number changes nothing.
|
|
68
|
+
run: number;
|
|
69
|
+
controller: AbortController | null;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// A session the server no longer knows: a retry opens a new one.
|
|
73
|
+
const SESSION_GONE_CODES: ReadonlySet<string> = new Set(["files.upload_expired", "files.not_found"]);
|
|
74
|
+
|
|
75
|
+
/** Creates an upload list; `open` and `finalize` are the app's, the sending is the package's. */
|
|
76
|
+
export function createUploadQueue<TSession extends UploadQueueSession>(options: UploadQueueOptions<TSession>): UploadQueue {
|
|
77
|
+
const concurrency = Math.max(1, Math.floor(options.concurrency ?? 1));
|
|
78
|
+
const describeError = options.describeError ?? ((code: ErrorCode) => getFilesErrorMessage(filesMessages.en, code));
|
|
79
|
+
const entries = new Map<string, Entry<TSession>>();
|
|
80
|
+
const listeners = new Set<() => void>();
|
|
81
|
+
let items: readonly UploadQueueItem[] = [];
|
|
82
|
+
let nextId = 1;
|
|
83
|
+
|
|
84
|
+
function publish(): void {
|
|
85
|
+
items = [...entries.values()].map((entry) => entry.item);
|
|
86
|
+
for (const listener of [...listeners]) listener();
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function setItem(entry: Entry<TSession>, state: UploadQueueState, progress = entry.item.progress, errorCode?: ErrorCode): void {
|
|
90
|
+
const { id, name, size } = entry.item;
|
|
91
|
+
entry.item = { id, name, size, state, progress, ...(errorCode === undefined ? {} : { errorCode, error: describeError(errorCode) }) };
|
|
92
|
+
publish();
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function fail(entry: Entry<TSession>, code: ErrorCode): void {
|
|
96
|
+
if (SESSION_GONE_CODES.has(code)) entry.session = null;
|
|
97
|
+
setItem(entry, "failed", entry.item.progress, code);
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function stop(entry: Entry<TSession>): void {
|
|
101
|
+
entry.run += 1;
|
|
102
|
+
entry.controller?.abort();
|
|
103
|
+
entry.controller = null;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
function pump(): void {
|
|
107
|
+
for (const entry of entries.values()) {
|
|
108
|
+
if (countRunning() >= concurrency) return;
|
|
109
|
+
if (entry.item.state === "queued") void start(entry);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
function countRunning(): number {
|
|
114
|
+
let running = 0;
|
|
115
|
+
for (const entry of entries.values()) if (entry.item.state === "uploading") running += 1;
|
|
116
|
+
return running;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
async function start(entry: Entry<TSession>): Promise<void> {
|
|
120
|
+
entry.run += 1;
|
|
121
|
+
const run = entry.run;
|
|
122
|
+
const controller = new AbortController();
|
|
123
|
+
entry.controller = controller;
|
|
124
|
+
const isCurrent = (): boolean => entry.run === run && entries.get(entry.item.id) === entry;
|
|
125
|
+
setItem(entry, "uploading");
|
|
126
|
+
try {
|
|
127
|
+
let session = entry.session;
|
|
128
|
+
if (session === null) {
|
|
129
|
+
const opened = await options.open(entry.file);
|
|
130
|
+
// A session opened while the item was paused is kept, so the resume does not open a second one.
|
|
131
|
+
if (opened.ok) entry.session = opened.value;
|
|
132
|
+
if (!isCurrent()) return;
|
|
133
|
+
if (!opened.ok) return fail(entry, opened.error);
|
|
134
|
+
session = opened.value;
|
|
135
|
+
}
|
|
136
|
+
const sent = await sendUpload(entry.file, {
|
|
137
|
+
uploadUrl: session.uploadUrl,
|
|
138
|
+
signal: controller.signal,
|
|
139
|
+
onProgress: (progress) => {
|
|
140
|
+
if (isCurrent()) setItem(entry, "uploading", toPercent(progress));
|
|
141
|
+
},
|
|
142
|
+
...(options.fetch === undefined ? {} : { fetch: options.fetch }),
|
|
143
|
+
});
|
|
144
|
+
if (!isCurrent()) return;
|
|
145
|
+
if (!sent.ok) return fail(entry, sent.error);
|
|
146
|
+
const finalized = await options.finalize(session, entry.file);
|
|
147
|
+
if (!isCurrent()) return;
|
|
148
|
+
if (!finalized.ok) return fail(entry, finalized.error);
|
|
149
|
+
setItem(entry, "done", 100);
|
|
150
|
+
} catch (error) {
|
|
151
|
+
// The app's `open` or `finalize` threw, e.g. a server action whose request failed: the item fails with a
|
|
152
|
+
// retryable code, and the cause goes to the console for the developer.
|
|
153
|
+
console.error(`@softure-ai/files/client: an upload of "${entry.item.name}" stopped on a thrown open or finalize`, error);
|
|
154
|
+
if (isCurrent()) fail(entry, "files.network");
|
|
155
|
+
} finally {
|
|
156
|
+
if (entry.controller === controller) entry.controller = null;
|
|
157
|
+
pump();
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function find(id: string, states: readonly UploadQueueState[]): Entry<TSession> | null {
|
|
162
|
+
const entry = entries.get(id);
|
|
163
|
+
return entry !== undefined && states.includes(entry.item.state) ? entry : null;
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
return {
|
|
167
|
+
add(files) {
|
|
168
|
+
const ids: string[] = [];
|
|
169
|
+
for (const file of Array.from(files)) {
|
|
170
|
+
const id = `upload-${String(nextId)}`;
|
|
171
|
+
nextId += 1;
|
|
172
|
+
entries.set(id, { item: { id, name: file.name, size: file.size, state: "queued", progress: 0 }, file, session: null, run: 0, controller: null });
|
|
173
|
+
ids.push(id);
|
|
174
|
+
}
|
|
175
|
+
publish();
|
|
176
|
+
pump();
|
|
177
|
+
return ids;
|
|
178
|
+
},
|
|
179
|
+
retry(id) {
|
|
180
|
+
const entry = find(id, ["failed"]);
|
|
181
|
+
if (entry === null) return;
|
|
182
|
+
setItem(entry, "queued");
|
|
183
|
+
pump();
|
|
184
|
+
},
|
|
185
|
+
pause(id) {
|
|
186
|
+
const entry = find(id, ["queued", "uploading"]);
|
|
187
|
+
if (entry === null) return;
|
|
188
|
+
stop(entry);
|
|
189
|
+
setItem(entry, "paused");
|
|
190
|
+
pump();
|
|
191
|
+
},
|
|
192
|
+
resume(id) {
|
|
193
|
+
const entry = find(id, ["paused"]);
|
|
194
|
+
if (entry === null) return;
|
|
195
|
+
setItem(entry, "queued");
|
|
196
|
+
pump();
|
|
197
|
+
},
|
|
198
|
+
remove(id) {
|
|
199
|
+
const entry = entries.get(id);
|
|
200
|
+
if (entry === undefined) return;
|
|
201
|
+
stop(entry);
|
|
202
|
+
entries.delete(id);
|
|
203
|
+
publish();
|
|
204
|
+
pump();
|
|
205
|
+
},
|
|
206
|
+
getItems: () => items,
|
|
207
|
+
subscribe(listener) {
|
|
208
|
+
listeners.add(listener);
|
|
209
|
+
return () => {
|
|
210
|
+
listeners.delete(listener);
|
|
211
|
+
};
|
|
212
|
+
},
|
|
213
|
+
};
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
function toPercent(progress: UploadProgress): number {
|
|
217
|
+
// 100 is kept for a finalized upload: a sent but unfinalized one shows 99 at most.
|
|
218
|
+
if (progress.totalBytes === 0) return 99;
|
|
219
|
+
return Math.min(99, Math.floor((progress.sentBytes / progress.totalBytes) * 100));
|
|
220
|
+
}
|
package/src/index.ts
CHANGED
|
@@ -26,7 +26,7 @@ export const FILES_RATE_LIMIT_BUCKETS = {
|
|
|
26
26
|
export const files = defineModule({
|
|
27
27
|
manifest: {
|
|
28
28
|
id: MODULE_ID,
|
|
29
|
-
version: "0.
|
|
29
|
+
version: "0.3.0",
|
|
30
30
|
dependsOn: { jobs: "^0.1.6?", security: "^0.2.5?" },
|
|
31
31
|
dbSchema: "files",
|
|
32
32
|
tables: ["files", "uploads"],
|
package/src/mcp/index.ts
CHANGED
|
@@ -11,8 +11,11 @@
|
|
|
11
11
|
// registerTools: (server, identity) => registerFileUploadTools(server, { identity, getContext: () => getFilesContext() }),
|
|
12
12
|
// });
|
|
13
13
|
// mcpAccess({ serverName: "acme", tools: [...FILE_UPLOAD_MCP_TOOLS] });
|
|
14
|
+
//
|
|
15
|
+
// An app that turns the stored file into its own record passes `onFinalized` (its fields join the answer), its own
|
|
16
|
+
// tool texts in `descriptions`, and `hints` for the error codes it answers.
|
|
14
17
|
import type { CallToolResult, McpServer } from "@modelcontextprotocol/server";
|
|
15
|
-
import { errorLogLabel } from "@softure-ai/core";
|
|
18
|
+
import { errorLogLabel, type Err, type ErrorCode } from "@softure-ai/core";
|
|
16
19
|
import { z } from "zod";
|
|
17
20
|
import type { CreatedUpload, StoredFile } from "../contract.js";
|
|
18
21
|
import { filesMessages, getFilesErrorMessage } from "../messages/index.js";
|
|
@@ -22,14 +25,17 @@ import { createUpload, finalizeUpload } from "../server/uploads.js";
|
|
|
22
25
|
export const CREATE_UPLOAD_TOOL_NAME = "create_upload";
|
|
23
26
|
export const FINALIZE_UPLOAD_TOOL_NAME = "finalize_upload";
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
/** The served tool descriptions by default; an app prefixes or replaces them through `descriptions`. */
|
|
29
|
+
export const FILE_UPLOAD_TOOL_DESCRIPTIONS = {
|
|
30
|
+
create_upload:
|
|
31
|
+
"Starts an upload of a file from your own environment (for a file too large for a tool call, such as a video). " +
|
|
27
32
|
"The file's bytes never go through this tool: it answers a one-time upload URL and a ready curl loop that sends the " +
|
|
28
33
|
"file in chunks; run it where the file is, then call finalize_upload with the upload_id. A person can open the " +
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
34
|
+
"same URL in a browser to upload the file there instead.",
|
|
35
|
+
finalize_upload:
|
|
36
|
+
"Finishes an upload started with create_upload once every chunk is sent: checks the bytes and stores the file. " +
|
|
37
|
+
"Answers the stored file's id. Calling it again answers the same file.",
|
|
38
|
+
} as const;
|
|
33
39
|
|
|
34
40
|
/** The catalog entries for `mcpAccess({ tools })`, so the token page and the catalog check know both tools. */
|
|
35
41
|
export const FILE_UPLOAD_MCP_TOOLS = [
|
|
@@ -51,8 +57,24 @@ export interface FileUploadToolsOptions {
|
|
|
51
57
|
readonly getOwnerId?: (identity: FileUploadToolIdentity) => string;
|
|
52
58
|
/** The kind of the stored files; `file` by default. */
|
|
53
59
|
readonly kind?: string;
|
|
60
|
+
/**
|
|
61
|
+
* Runs after every successful `finalize_upload`, a repeated one for the same file included (so keep it idempotent,
|
|
62
|
+
* e.g. an upsert by `file.id`). Returned fields join the answer (the file's own fields win a name clash); a returned
|
|
63
|
+
* `err(code)` answers that code as a failure; a thrown error answers the generic failure and is logged.
|
|
64
|
+
*/
|
|
65
|
+
readonly onFinalized?: (file: StoredFile, ctx: FilesContext) => Promise<FileFinalizedAnswer>;
|
|
66
|
+
/** The served descriptions; a missing one keeps `FILE_UPLOAD_TOOL_DESCRIPTIONS`. */
|
|
67
|
+
readonly descriptions?: { readonly create_upload?: string; readonly finalize_upload?: string };
|
|
68
|
+
/**
|
|
69
|
+
* The text answered for an error code, the hook's codes and the package's `files.*` codes alike, followed by the code.
|
|
70
|
+
* Without a hint a `files.*` code answers its English sentence and any other code answers the code alone.
|
|
71
|
+
*/
|
|
72
|
+
readonly hints?: Readonly<Record<string, string>>;
|
|
54
73
|
}
|
|
55
74
|
|
|
75
|
+
/** What `onFinalized` answers: extra fields for the tool's answer, nothing, or a failure. */
|
|
76
|
+
export type FileFinalizedAnswer = Readonly<Record<string, unknown>> | Err<ErrorCode> | undefined | void;
|
|
77
|
+
|
|
56
78
|
const createUploadInput = z.object({
|
|
57
79
|
name: z.string().min(1).max(255).describe("The file's name, e.g. promo.mp4."),
|
|
58
80
|
size: z.number().int().min(1).describe("The file's exact size in bytes (e.g. from `wc -c < file`)."),
|
|
@@ -76,12 +98,12 @@ export function registerFileUploadTools(server: Pick<McpServer, "registerTool">,
|
|
|
76
98
|
server.registerTool(
|
|
77
99
|
CREATE_UPLOAD_TOOL_NAME,
|
|
78
100
|
{
|
|
79
|
-
description:
|
|
101
|
+
description: options.descriptions?.create_upload ?? FILE_UPLOAD_TOOL_DESCRIPTIONS.create_upload,
|
|
80
102
|
inputSchema: createUploadInput,
|
|
81
103
|
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
|
|
82
104
|
},
|
|
83
105
|
(args) =>
|
|
84
|
-
answer(CREATE_UPLOAD_TOOL_NAME, async () => {
|
|
106
|
+
answer(CREATE_UPLOAD_TOOL_NAME, options.hints, async () => {
|
|
85
107
|
const ctx = await options.getContext();
|
|
86
108
|
const created = await createUpload(ctx, {
|
|
87
109
|
ownerId,
|
|
@@ -98,15 +120,18 @@ export function registerFileUploadTools(server: Pick<McpServer, "registerTool">,
|
|
|
98
120
|
server.registerTool(
|
|
99
121
|
FINALIZE_UPLOAD_TOOL_NAME,
|
|
100
122
|
{
|
|
101
|
-
description:
|
|
123
|
+
description: options.descriptions?.finalize_upload ?? FILE_UPLOAD_TOOL_DESCRIPTIONS.finalize_upload,
|
|
102
124
|
inputSchema: finalizeUploadInput,
|
|
103
125
|
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
|
104
126
|
},
|
|
105
127
|
(args) =>
|
|
106
|
-
answer(FINALIZE_UPLOAD_TOOL_NAME, async () => {
|
|
128
|
+
answer(FINALIZE_UPLOAD_TOOL_NAME, options.hints, async () => {
|
|
107
129
|
const ctx = await options.getContext();
|
|
108
130
|
const finalized = await finalizeUpload(ctx, args.upload_id, { ownerId });
|
|
109
|
-
|
|
131
|
+
if (!finalized.ok) return finalized;
|
|
132
|
+
const extra = options.onFinalized === undefined ? undefined : await options.onFinalized(finalized.value, ctx);
|
|
133
|
+
if (isFailure(extra)) return extra;
|
|
134
|
+
return { ...extra, ...describeStoredFile(finalized.value) };
|
|
110
135
|
}),
|
|
111
136
|
);
|
|
112
137
|
}
|
|
@@ -148,11 +173,15 @@ function describeStoredFile(file: StoredFile) {
|
|
|
148
173
|
return { file_id: file.id, name: file.name, size: file.size, content_type: file.contentType, sha256: file.sha256 };
|
|
149
174
|
}
|
|
150
175
|
|
|
151
|
-
/** The tool's answer: a value as JSON text, an error code as its
|
|
152
|
-
async function answer(
|
|
176
|
+
/** The tool's answer: a value as JSON text, an error code as its hint or sentence, a thrown error as a fixed text. */
|
|
177
|
+
async function answer(
|
|
178
|
+
tool: string,
|
|
179
|
+
hints: Readonly<Record<string, string>> | undefined,
|
|
180
|
+
work: () => Promise<unknown>,
|
|
181
|
+
): Promise<CallToolResult> {
|
|
153
182
|
try {
|
|
154
183
|
const value = await work();
|
|
155
|
-
if (isFailure(value)) return failure(
|
|
184
|
+
if (isFailure(value)) return failure(describeErrorCode(value.error, hints));
|
|
156
185
|
return { content: [{ type: "text", text: JSON.stringify(value) }] };
|
|
157
186
|
} catch (error) {
|
|
158
187
|
console.error(`@softure-ai/files: tool ${tool} failed: ${errorLogLabel(error)}`);
|
|
@@ -160,6 +189,14 @@ async function answer(tool: string, work: () => Promise<unknown>): Promise<CallT
|
|
|
160
189
|
}
|
|
161
190
|
}
|
|
162
191
|
|
|
192
|
+
/** `<hint or files sentence> (<code>)`, or the code alone for another module's code without a hint. */
|
|
193
|
+
function describeErrorCode(code: string, hints: Readonly<Record<string, string>> | undefined): string {
|
|
194
|
+
const hint = hints !== undefined && Object.hasOwn(hints, code) ? hints[code] : undefined;
|
|
195
|
+
if (hint !== undefined) return `${hint} (${code})`;
|
|
196
|
+
if (code.startsWith("files.")) return `${getFilesErrorMessage(filesMessages.en, code)} (${code})`;
|
|
197
|
+
return code;
|
|
198
|
+
}
|
|
199
|
+
|
|
163
200
|
function failure(text: string): CallToolResult {
|
|
164
201
|
return { content: [{ type: "text", text }], isError: true };
|
|
165
202
|
}
|
package/src/messages/en.ts
CHANGED
|
@@ -12,11 +12,16 @@ export const en = {
|
|
|
12
12
|
too_many_uploads: "Too many uploads are in progress. Finish or wait for one first.",
|
|
13
13
|
upload_incomplete: "The upload is not complete yet: some parts have not arrived.",
|
|
14
14
|
checksum_mismatch: "The uploaded file does not match its checksum.",
|
|
15
|
+
upload_expired: "This upload has expired. Start it again.",
|
|
16
|
+
upload_rejected: "The upload was refused: the file does not match what the upload expects.",
|
|
17
|
+
network: "The connection dropped. Try again to resume where it stopped.",
|
|
18
|
+
upload_aborted: "The upload was stopped.",
|
|
15
19
|
},
|
|
16
20
|
core: {
|
|
17
21
|
unexpected: "Something went wrong. Try again.",
|
|
18
22
|
},
|
|
19
|
-
},
|
|
23
|
+
},
|
|
24
|
+
uploadPage: {
|
|
20
25
|
title: "Upload a file",
|
|
21
26
|
intro: "Choose the file this upload is waiting for. It is sent in parts, and a broken connection resumes where it stopped.",
|
|
22
27
|
expected: "Expected: {name}, {size}.",
|
|
@@ -26,5 +31,6 @@ export const en = {
|
|
|
26
31
|
done: "Upload complete. Go back to where you started it to finish.",
|
|
27
32
|
wrongSize: "This file has a different size than the upload expects.",
|
|
28
33
|
failed: "The upload stopped. Press Upload again to resume.",
|
|
34
|
+
expired: "This upload link has expired. Start a new upload where you began this one.",
|
|
29
35
|
},
|
|
30
36
|
};
|
package/src/next/index.ts
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
|
-
// The Next.js adapter of @softure-ai/files: the public link route, the upload route and the
|
|
2
|
-
// (docs/02-module-standard.md §8).
|
|
1
|
+
// The Next.js adapter of @softure-ai/files: the public link route, the owner route, the upload route and the
|
|
2
|
+
// request-side context (docs/02-module-standard.md §8).
|
|
3
3
|
export { getFilesContext } from "./context.js";
|
|
4
4
|
export { serveFileLinkRoute } from "./route.js";
|
|
5
|
+
export {
|
|
6
|
+
createStoredFileRoute,
|
|
7
|
+
type StoredFileRoute,
|
|
8
|
+
type StoredFileRouteContext,
|
|
9
|
+
type StoredFileRouteHandler,
|
|
10
|
+
type StoredFileRouteOptions,
|
|
11
|
+
} from "./stored-file-route.js";
|
|
5
12
|
export { serveUploadRoute } from "./upload-route.js";
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// The owner route of stored files, mounted behind the app's sign-in guard in `app/media/[id]/route.ts`:
|
|
2
|
+
// `export const { GET, HEAD } = createStoredFileRoute({ getOwnerId: async () => (await getCurrentUser())?.id ?? null });`
|
|
3
|
+
// The session check is the app's (files does not depend on auth); the owner check is the module's. Without a session,
|
|
4
|
+
// and for a file that is missing or another owner's, the route answers the same `404`, so it never tells whether a
|
|
5
|
+
// file exists.
|
|
6
|
+
import type { FilesContext } from "../server/context.js";
|
|
7
|
+
import { notFound, serveStoredFile } from "../server/serve.js";
|
|
8
|
+
import { getFilesContext } from "./context.js";
|
|
9
|
+
|
|
10
|
+
/** The second argument Next.js passes a handler of a `[id]` segment. */
|
|
11
|
+
export interface StoredFileRouteContext {
|
|
12
|
+
readonly params: Promise<{ readonly id: string }>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export type StoredFileRouteHandler = (request: Request, route: StoredFileRouteContext) => Promise<Response>;
|
|
16
|
+
|
|
17
|
+
export interface StoredFileRoute {
|
|
18
|
+
readonly GET: StoredFileRouteHandler;
|
|
19
|
+
readonly HEAD: StoredFileRouteHandler;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface StoredFileRouteOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The signed-in user's id, the same value the app passed to `saveFile({ ownerId })`; null when nobody is signed in
|
|
25
|
+
* (the route answers 404). Typically the app's session lookup through auth.
|
|
26
|
+
*/
|
|
27
|
+
readonly getOwnerId: (request: Request) => Promise<string | null>;
|
|
28
|
+
/** `inline` (the default: a player, an image) or `attachment` (a download). */
|
|
29
|
+
readonly disposition?: "attachment" | "inline";
|
|
30
|
+
/** The context the route reads; `getFilesContext()` (the registered config's database) by default. */
|
|
31
|
+
readonly getContext?: () => Promise<FilesContext>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createStoredFileRoute(options: StoredFileRouteOptions): StoredFileRoute {
|
|
35
|
+
const getContext = options.getContext ?? (() => getFilesContext());
|
|
36
|
+
const handle: StoredFileRouteHandler = async (request, route) => {
|
|
37
|
+
const ownerId = await options.getOwnerId(request);
|
|
38
|
+
if (ownerId === null) return notFound();
|
|
39
|
+
const { id } = await route.params;
|
|
40
|
+
return serveStoredFile(await getContext(), id, request, { ownerId, disposition: options.disposition });
|
|
41
|
+
};
|
|
42
|
+
return { GET: handle, HEAD: handle };
|
|
43
|
+
}
|
package/src/server/index.ts
CHANGED
|
@@ -59,5 +59,5 @@ export {
|
|
|
59
59
|
export { signUploadPath, verifyUploadPath } from "./upload-tokens.js";
|
|
60
60
|
export { FILES_UPLOAD_RATE_LIMIT_BUCKET, serveUpload } from "./serve-upload.js";
|
|
61
61
|
export { renderUploadPage } from "./upload-page.js";
|
|
62
|
-
export { parseRangeHeader, serveFileLink } from "./serve.js";
|
|
62
|
+
export { parseRangeHeader, serveFileLink, serveStoredFile, type ServeStoredFileOptions } from "./serve.js";
|
|
63
63
|
export { EMPTY_PAYLOAD_SHA256, encodeRfc3986, signRequest, type SigV4Credentials, type SignRequestInput } from "./sigv4.js";
|