@softure-ai/files 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +131 -4
- package/dist/client/index.d.ts +3 -0
- package/dist/client/index.d.ts.map +1 -0
- package/dist/client/index.js +6 -0
- package/dist/client/index.js.map +1 -0
- package/dist/client/send-upload.d.ts +26 -0
- package/dist/client/send-upload.d.ts.map +1 -0
- package/dist/client/send-upload.js +66 -0
- package/dist/client/send-upload.js.map +1 -0
- package/dist/client/upload-queue.d.ts +52 -0
- package/dist/client/upload-queue.d.ts.map +1 -0
- package/dist/client/upload-queue.js +164 -0
- package/dist/client/upload-queue.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/jobs/index.d.ts +23 -1
- package/dist/jobs/index.d.ts.map +1 -1
- package/dist/jobs/index.js +46 -11
- package/dist/jobs/index.js.map +1 -1
- package/dist/mcp/index.d.ts +25 -1
- package/dist/mcp/index.d.ts.map +1 -1
- package/dist/mcp/index.js +31 -14
- package/dist/mcp/index.js.map +1 -1
- package/dist/messages/en.d.ts +5 -0
- package/dist/messages/en.d.ts.map +1 -1
- package/dist/messages/en.js +7 -1
- package/dist/messages/en.js.map +1 -1
- package/dist/messages/index.d.ts +5 -0
- package/dist/messages/index.d.ts.map +1 -1
- package/dist/next/index.d.ts +1 -0
- package/dist/next/index.d.ts.map +1 -1
- package/dist/next/index.js +3 -2
- package/dist/next/index.js.map +1 -1
- package/dist/next/stored-file-route.d.ts +25 -0
- package/dist/next/stored-file-route.d.ts.map +1 -0
- package/dist/next/stored-file-route.js +14 -0
- package/dist/next/stored-file-route.js.map +1 -0
- package/dist/server/index.d.ts +2 -2
- package/dist/server/index.d.ts.map +1 -1
- package/dist/server/index.js +2 -2
- package/dist/server/index.js.map +1 -1
- package/dist/server/prune.d.ts +4 -0
- package/dist/server/prune.d.ts.map +1 -1
- package/dist/server/prune.js +8 -2
- package/dist/server/prune.js.map +1 -1
- package/dist/server/serve.d.ts +15 -0
- package/dist/server/serve.d.ts.map +1 -1
- package/dist/server/serve.js +38 -14
- package/dist/server/serve.js.map +1 -1
- package/dist/server/upload-page.d.ts.map +1 -1
- package/dist/server/upload-page.js +16 -23
- package/dist/server/upload-page.js.map +1 -1
- package/module.json +1 -1
- package/package.json +8 -3
- package/src/client/index.ts +12 -0
- package/src/client/send-upload.ts +89 -0
- package/src/client/upload-queue.ts +220 -0
- package/src/index.ts +1 -1
- package/src/jobs/index.ts +66 -13
- package/src/mcp/index.ts +52 -15
- package/src/messages/en.ts +7 -1
- package/src/next/index.ts +9 -2
- package/src/next/stored-file-route.ts +43 -0
- package/src/server/index.ts +2 -2
- package/src/server/prune.ts +10 -1
- package/src/server/serve.ts +51 -14
- package/src/server/upload-page.ts +16 -23
package/src/jobs/index.ts
CHANGED
|
@@ -3,21 +3,74 @@
|
|
|
3
3
|
import { ok } from "@softure-ai/core";
|
|
4
4
|
import { defineJobHandler, type JobHandlers, type JobSchedulesInput } from "@softure-ai/jobs";
|
|
5
5
|
import { z } from "zod";
|
|
6
|
-
import { pruneFiles } from "../server/prune.js";
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
6
|
+
import { PRUNE_REPORT_MAX_FILE_IDS, pruneFiles, type PruneFilesReport } from "../server/prune.js";
|
|
7
|
+
|
|
8
|
+
export type FilesJobsLogFields = Readonly<Record<string, unknown>>;
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Where the files jobs report a run. A subset of `@softure-ai/observability`'s `Logger`, so an app passes
|
|
12
|
+
* `createLogger("files")` as it is; without one, lines go to the console.
|
|
13
|
+
*/
|
|
14
|
+
export interface FilesJobsLogger {
|
|
15
|
+
info(message: string, fields?: FilesJobsLogFields): void;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface CreateFilesJobHandlersOptions {
|
|
19
|
+
readonly logger?: FilesJobsLogger;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const PREFIX = "@softure-ai/files:";
|
|
23
|
+
|
|
24
|
+
/** One `key=value` line per call; an array is joined with commas. */
|
|
25
|
+
export const consoleFilesJobsLogger: FilesJobsLogger = {
|
|
26
|
+
info: (message, fields) => console.log(formatLine(message, fields)),
|
|
27
|
+
};
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Handlers for the worker with a logger of the app's choice: `handlers: { ...createFilesJobHandlers({ logger }) }`.
|
|
31
|
+
* Every `files.prune` run logs one line `files.prune finished` with `removed`, `freedBytes`, `failedFiles`,
|
|
32
|
+
* `deletedUploads`, `failedUploads` and, when at most 10 files went, their ids as `fileIds`.
|
|
33
|
+
*/
|
|
34
|
+
export function createFilesJobHandlers(options: CreateFilesJobHandlersOptions = {}) {
|
|
35
|
+
const logger = options.logger ?? consoleFilesJobsLogger;
|
|
36
|
+
return {
|
|
37
|
+
/** Removes expired and deleted files from storage and the table (`pruneFiles`). */
|
|
38
|
+
"files.prune": defineJobHandler({
|
|
39
|
+
payload: z.object({}),
|
|
40
|
+
run: async (_payload, ctx) => {
|
|
41
|
+
logger.info("files.prune finished", getPruneLogFields(await pruneFiles(ctx)));
|
|
42
|
+
return ok();
|
|
43
|
+
},
|
|
44
|
+
}),
|
|
45
|
+
} satisfies JobHandlers;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/** Handlers for the worker: `handlers: { ...filesJobHandlers }`; they log to the console. */
|
|
49
|
+
export const filesJobHandlers = createFilesJobHandlers();
|
|
19
50
|
|
|
20
51
|
/** Schedules for `jobs({ schedules: { ...filesJobSchedules } })`: hourly. */
|
|
21
52
|
export const filesJobSchedules = {
|
|
22
53
|
"files.prune": { every: "1h" },
|
|
23
54
|
} satisfies JobSchedulesInput;
|
|
55
|
+
|
|
56
|
+
function getPruneLogFields(report: PruneFilesReport): FilesJobsLogFields {
|
|
57
|
+
const fields = {
|
|
58
|
+
removed: report.deletedFiles,
|
|
59
|
+
freedBytes: report.freedBytes,
|
|
60
|
+
failedFiles: report.failedFiles,
|
|
61
|
+
deletedUploads: report.deletedUploads,
|
|
62
|
+
failedUploads: report.failedUploads,
|
|
63
|
+
};
|
|
64
|
+
return report.deletedFiles > PRUNE_REPORT_MAX_FILE_IDS ? fields : { ...fields, fileIds: report.removedFileIds };
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function formatLine(message: string, fields: FilesJobsLogFields | undefined): string {
|
|
68
|
+
const parts = Object.entries(fields ?? {}).map(([key, value]) => `${key}=${formatValue(value)}`);
|
|
69
|
+
return [PREFIX, message, ...parts].join(" ");
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function formatValue(value: unknown): string {
|
|
73
|
+
if (Array.isArray(value)) return value.map(formatValue).join(",");
|
|
74
|
+
if (typeof value === "string") return value;
|
|
75
|
+
return JSON.stringify(value) ?? String(value);
|
|
76
|
+
}
|
package/src/mcp/index.ts
CHANGED
|
@@ -11,8 +11,11 @@
|
|
|
11
11
|
// registerTools: (server, identity) => registerFileUploadTools(server, { identity, getContext: () => getFilesContext() }),
|
|
12
12
|
// });
|
|
13
13
|
// mcpAccess({ serverName: "acme", tools: [...FILE_UPLOAD_MCP_TOOLS] });
|
|
14
|
+
//
|
|
15
|
+
// An app that turns the stored file into its own record passes `onFinalized` (its fields join the answer), its own
|
|
16
|
+
// tool texts in `descriptions`, and `hints` for the error codes it answers.
|
|
14
17
|
import type { CallToolResult, McpServer } from "@modelcontextprotocol/server";
|
|
15
|
-
import { errorLogLabel } from "@softure-ai/core";
|
|
18
|
+
import { errorLogLabel, type Err, type ErrorCode } from "@softure-ai/core";
|
|
16
19
|
import { z } from "zod";
|
|
17
20
|
import type { CreatedUpload, StoredFile } from "../contract.js";
|
|
18
21
|
import { filesMessages, getFilesErrorMessage } from "../messages/index.js";
|
|
@@ -22,14 +25,17 @@ import { createUpload, finalizeUpload } from "../server/uploads.js";
|
|
|
22
25
|
export const CREATE_UPLOAD_TOOL_NAME = "create_upload";
|
|
23
26
|
export const FINALIZE_UPLOAD_TOOL_NAME = "finalize_upload";
|
|
24
27
|
|
|
25
|
-
|
|
26
|
-
|
|
28
|
+
/** The served tool descriptions by default; an app prefixes or replaces them through `descriptions`. */
|
|
29
|
+
export const FILE_UPLOAD_TOOL_DESCRIPTIONS = {
|
|
30
|
+
create_upload:
|
|
31
|
+
"Starts an upload of a file from your own environment (for a file too large for a tool call, such as a video). " +
|
|
27
32
|
"The file's bytes never go through this tool: it answers a one-time upload URL and a ready curl loop that sends the " +
|
|
28
33
|
"file in chunks; run it where the file is, then call finalize_upload with the upload_id. A person can open the " +
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
34
|
+
"same URL in a browser to upload the file there instead.",
|
|
35
|
+
finalize_upload:
|
|
36
|
+
"Finishes an upload started with create_upload once every chunk is sent: checks the bytes and stores the file. " +
|
|
37
|
+
"Answers the stored file's id. Calling it again answers the same file.",
|
|
38
|
+
} as const;
|
|
33
39
|
|
|
34
40
|
/** The catalog entries for `mcpAccess({ tools })`, so the token page and the catalog check know both tools. */
|
|
35
41
|
export const FILE_UPLOAD_MCP_TOOLS = [
|
|
@@ -51,8 +57,24 @@ export interface FileUploadToolsOptions {
|
|
|
51
57
|
readonly getOwnerId?: (identity: FileUploadToolIdentity) => string;
|
|
52
58
|
/** The kind of the stored files; `file` by default. */
|
|
53
59
|
readonly kind?: string;
|
|
60
|
+
/**
|
|
61
|
+
* Runs after every successful `finalize_upload`, a repeated one for the same file included (so keep it idempotent,
|
|
62
|
+
* e.g. an upsert by `file.id`). Returned fields join the answer (the file's own fields win a name clash); a returned
|
|
63
|
+
* `err(code)` answers that code as a failure; a thrown error answers the generic failure and is logged.
|
|
64
|
+
*/
|
|
65
|
+
readonly onFinalized?: (file: StoredFile, ctx: FilesContext) => Promise<FileFinalizedAnswer>;
|
|
66
|
+
/** The served descriptions; a missing one keeps `FILE_UPLOAD_TOOL_DESCRIPTIONS`. */
|
|
67
|
+
readonly descriptions?: { readonly create_upload?: string; readonly finalize_upload?: string };
|
|
68
|
+
/**
|
|
69
|
+
* The text answered for an error code, the hook's codes and the package's `files.*` codes alike, followed by the code.
|
|
70
|
+
* Without a hint a `files.*` code answers its English sentence and any other code answers the code alone.
|
|
71
|
+
*/
|
|
72
|
+
readonly hints?: Readonly<Record<string, string>>;
|
|
54
73
|
}
|
|
55
74
|
|
|
75
|
+
/** What `onFinalized` answers: extra fields for the tool's answer, nothing, or a failure. */
|
|
76
|
+
export type FileFinalizedAnswer = Readonly<Record<string, unknown>> | Err<ErrorCode> | undefined | void;
|
|
77
|
+
|
|
56
78
|
const createUploadInput = z.object({
|
|
57
79
|
name: z.string().min(1).max(255).describe("The file's name, e.g. promo.mp4."),
|
|
58
80
|
size: z.number().int().min(1).describe("The file's exact size in bytes (e.g. from `wc -c < file`)."),
|
|
@@ -76,12 +98,12 @@ export function registerFileUploadTools(server: Pick<McpServer, "registerTool">,
|
|
|
76
98
|
server.registerTool(
|
|
77
99
|
CREATE_UPLOAD_TOOL_NAME,
|
|
78
100
|
{
|
|
79
|
-
description:
|
|
101
|
+
description: options.descriptions?.create_upload ?? FILE_UPLOAD_TOOL_DESCRIPTIONS.create_upload,
|
|
80
102
|
inputSchema: createUploadInput,
|
|
81
103
|
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
|
|
82
104
|
},
|
|
83
105
|
(args) =>
|
|
84
|
-
answer(CREATE_UPLOAD_TOOL_NAME, async () => {
|
|
106
|
+
answer(CREATE_UPLOAD_TOOL_NAME, options.hints, async () => {
|
|
85
107
|
const ctx = await options.getContext();
|
|
86
108
|
const created = await createUpload(ctx, {
|
|
87
109
|
ownerId,
|
|
@@ -98,15 +120,18 @@ export function registerFileUploadTools(server: Pick<McpServer, "registerTool">,
|
|
|
98
120
|
server.registerTool(
|
|
99
121
|
FINALIZE_UPLOAD_TOOL_NAME,
|
|
100
122
|
{
|
|
101
|
-
description:
|
|
123
|
+
description: options.descriptions?.finalize_upload ?? FILE_UPLOAD_TOOL_DESCRIPTIONS.finalize_upload,
|
|
102
124
|
inputSchema: finalizeUploadInput,
|
|
103
125
|
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
|
104
126
|
},
|
|
105
127
|
(args) =>
|
|
106
|
-
answer(FINALIZE_UPLOAD_TOOL_NAME, async () => {
|
|
128
|
+
answer(FINALIZE_UPLOAD_TOOL_NAME, options.hints, async () => {
|
|
107
129
|
const ctx = await options.getContext();
|
|
108
130
|
const finalized = await finalizeUpload(ctx, args.upload_id, { ownerId });
|
|
109
|
-
|
|
131
|
+
if (!finalized.ok) return finalized;
|
|
132
|
+
const extra = options.onFinalized === undefined ? undefined : await options.onFinalized(finalized.value, ctx);
|
|
133
|
+
if (isFailure(extra)) return extra;
|
|
134
|
+
return { ...extra, ...describeStoredFile(finalized.value) };
|
|
110
135
|
}),
|
|
111
136
|
);
|
|
112
137
|
}
|
|
@@ -148,11 +173,15 @@ function describeStoredFile(file: StoredFile) {
|
|
|
148
173
|
return { file_id: file.id, name: file.name, size: file.size, content_type: file.contentType, sha256: file.sha256 };
|
|
149
174
|
}
|
|
150
175
|
|
|
151
|
-
/** The tool's answer: a value as JSON text, an error code as its
|
|
152
|
-
async function answer(
|
|
176
|
+
/** The tool's answer: a value as JSON text, an error code as its hint or sentence, a thrown error as a fixed text. */
|
|
177
|
+
async function answer(
|
|
178
|
+
tool: string,
|
|
179
|
+
hints: Readonly<Record<string, string>> | undefined,
|
|
180
|
+
work: () => Promise<unknown>,
|
|
181
|
+
): Promise<CallToolResult> {
|
|
153
182
|
try {
|
|
154
183
|
const value = await work();
|
|
155
|
-
if (isFailure(value)) return failure(
|
|
184
|
+
if (isFailure(value)) return failure(describeErrorCode(value.error, hints));
|
|
156
185
|
return { content: [{ type: "text", text: JSON.stringify(value) }] };
|
|
157
186
|
} catch (error) {
|
|
158
187
|
console.error(`@softure-ai/files: tool ${tool} failed: ${errorLogLabel(error)}`);
|
|
@@ -160,6 +189,14 @@ async function answer(tool: string, work: () => Promise<unknown>): Promise<CallT
|
|
|
160
189
|
}
|
|
161
190
|
}
|
|
162
191
|
|
|
192
|
+
/** `<hint or files sentence> (<code>)`, or the code alone for another module's code without a hint. */
|
|
193
|
+
function describeErrorCode(code: string, hints: Readonly<Record<string, string>> | undefined): string {
|
|
194
|
+
const hint = hints !== undefined && Object.hasOwn(hints, code) ? hints[code] : undefined;
|
|
195
|
+
if (hint !== undefined) return `${hint} (${code})`;
|
|
196
|
+
if (code.startsWith("files.")) return `${getFilesErrorMessage(filesMessages.en, code)} (${code})`;
|
|
197
|
+
return code;
|
|
198
|
+
}
|
|
199
|
+
|
|
163
200
|
function failure(text: string): CallToolResult {
|
|
164
201
|
return { content: [{ type: "text", text }], isError: true };
|
|
165
202
|
}
|
package/src/messages/en.ts
CHANGED
|
@@ -12,11 +12,16 @@ export const en = {
|
|
|
12
12
|
too_many_uploads: "Too many uploads are in progress. Finish or wait for one first.",
|
|
13
13
|
upload_incomplete: "The upload is not complete yet: some parts have not arrived.",
|
|
14
14
|
checksum_mismatch: "The uploaded file does not match its checksum.",
|
|
15
|
+
upload_expired: "This upload has expired. Start it again.",
|
|
16
|
+
upload_rejected: "The upload was refused: the file does not match what the upload expects.",
|
|
17
|
+
network: "The connection dropped. Try again to resume where it stopped.",
|
|
18
|
+
upload_aborted: "The upload was stopped.",
|
|
15
19
|
},
|
|
16
20
|
core: {
|
|
17
21
|
unexpected: "Something went wrong. Try again.",
|
|
18
22
|
},
|
|
19
|
-
},
|
|
23
|
+
},
|
|
24
|
+
uploadPage: {
|
|
20
25
|
title: "Upload a file",
|
|
21
26
|
intro: "Choose the file this upload is waiting for. It is sent in parts, and a broken connection resumes where it stopped.",
|
|
22
27
|
expected: "Expected: {name}, {size}.",
|
|
@@ -26,5 +31,6 @@ export const en = {
|
|
|
26
31
|
done: "Upload complete. Go back to where you started it to finish.",
|
|
27
32
|
wrongSize: "This file has a different size than the upload expects.",
|
|
28
33
|
failed: "The upload stopped. Press Upload again to resume.",
|
|
34
|
+
expired: "This upload link has expired. Start a new upload where you began this one.",
|
|
29
35
|
},
|
|
30
36
|
};
|
package/src/next/index.ts
CHANGED
|
@@ -1,5 +1,12 @@
|
|
|
1
|
-
// The Next.js adapter of @softure-ai/files: the public link route, the upload route and the
|
|
2
|
-
// (docs/02-module-standard.md §8).
|
|
1
|
+
// The Next.js adapter of @softure-ai/files: the public link route, the owner route, the upload route and the
|
|
2
|
+
// request-side context (docs/02-module-standard.md §8).
|
|
3
3
|
export { getFilesContext } from "./context.js";
|
|
4
4
|
export { serveFileLinkRoute } from "./route.js";
|
|
5
|
+
export {
|
|
6
|
+
createStoredFileRoute,
|
|
7
|
+
type StoredFileRoute,
|
|
8
|
+
type StoredFileRouteContext,
|
|
9
|
+
type StoredFileRouteHandler,
|
|
10
|
+
type StoredFileRouteOptions,
|
|
11
|
+
} from "./stored-file-route.js";
|
|
5
12
|
export { serveUploadRoute } from "./upload-route.js";
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// The owner route of stored files, mounted behind the app's sign-in guard in `app/media/[id]/route.ts`:
|
|
2
|
+
// `export const { GET, HEAD } = createStoredFileRoute({ getOwnerId: async () => (await getCurrentUser())?.id ?? null });`
|
|
3
|
+
// The session check is the app's (files does not depend on auth); the owner check is the module's. Without a session,
|
|
4
|
+
// and for a file that is missing or another owner's, the route answers the same `404`, so it never tells whether a
|
|
5
|
+
// file exists.
|
|
6
|
+
import type { FilesContext } from "../server/context.js";
|
|
7
|
+
import { notFound, serveStoredFile } from "../server/serve.js";
|
|
8
|
+
import { getFilesContext } from "./context.js";
|
|
9
|
+
|
|
10
|
+
/** The second argument Next.js passes a handler of a `[id]` segment. */
|
|
11
|
+
export interface StoredFileRouteContext {
|
|
12
|
+
readonly params: Promise<{ readonly id: string }>;
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export type StoredFileRouteHandler = (request: Request, route: StoredFileRouteContext) => Promise<Response>;
|
|
16
|
+
|
|
17
|
+
export interface StoredFileRoute {
|
|
18
|
+
readonly GET: StoredFileRouteHandler;
|
|
19
|
+
readonly HEAD: StoredFileRouteHandler;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export interface StoredFileRouteOptions {
|
|
23
|
+
/**
|
|
24
|
+
* The signed-in user's id, the same value the app passed to `saveFile({ ownerId })`; null when nobody is signed in
|
|
25
|
+
* (the route answers 404). Typically the app's session lookup through auth.
|
|
26
|
+
*/
|
|
27
|
+
readonly getOwnerId: (request: Request) => Promise<string | null>;
|
|
28
|
+
/** `inline` (the default: a player, an image) or `attachment` (a download). */
|
|
29
|
+
readonly disposition?: "attachment" | "inline";
|
|
30
|
+
/** The context the route reads; `getFilesContext()` (the registered config's database) by default. */
|
|
31
|
+
readonly getContext?: () => Promise<FilesContext>;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export function createStoredFileRoute(options: StoredFileRouteOptions): StoredFileRoute {
|
|
35
|
+
const getContext = options.getContext ?? (() => getFilesContext());
|
|
36
|
+
const handle: StoredFileRouteHandler = async (request, route) => {
|
|
37
|
+
const ownerId = await options.getOwnerId(request);
|
|
38
|
+
if (ownerId === null) return notFound();
|
|
39
|
+
const { id } = await route.params;
|
|
40
|
+
return serveStoredFile(await getContext(), id, request, { ownerId, disposition: options.disposition });
|
|
41
|
+
};
|
|
42
|
+
return { GET: handle, HEAD: handle };
|
|
43
|
+
}
|
package/src/server/index.ts
CHANGED
|
@@ -31,7 +31,7 @@ export { createLocalStorage, type LocalStorageOptions } from "./local-storage.js
|
|
|
31
31
|
export { DEFAULT_CONTENT_TYPE, formatContentDisposition, normalizeContentType, sanitizeFileName } from "./names.js";
|
|
32
32
|
export { getFilesModule, getFilesOptions, getLinksRoute, getUploadsRoute } from "./options.js";
|
|
33
33
|
export { deleteFilesUserData, exportFilesUserData, filesPrivacyContributor, type FilesDeletionReport, type FilesUserData } from "./privacy.js";
|
|
34
|
-
export { pruneFiles, type PruneFilesOptions, type PruneFilesReport } from "./prune.js";
|
|
34
|
+
export { PRUNE_REPORT_MAX_FILE_IDS, pruneFiles, type PruneFilesOptions, type PruneFilesReport } from "./prune.js";
|
|
35
35
|
export {
|
|
36
36
|
createS3Storage,
|
|
37
37
|
S3_ACCESS_KEY_ID_ENV,
|
|
@@ -59,5 +59,5 @@ export {
|
|
|
59
59
|
export { signUploadPath, verifyUploadPath } from "./upload-tokens.js";
|
|
60
60
|
export { FILES_UPLOAD_RATE_LIMIT_BUCKET, serveUpload } from "./serve-upload.js";
|
|
61
61
|
export { renderUploadPage } from "./upload-page.js";
|
|
62
|
-
export { parseRangeHeader, serveFileLink } from "./serve.js";
|
|
62
|
+
export { parseRangeHeader, serveFileLink, serveStoredFile, type ServeStoredFileOptions } from "./serve.js";
|
|
63
63
|
export { EMPTY_PAYLOAD_SHA256, encodeRfc3986, signRequest, type SigV4Credentials, type SignRequestInput } from "./sigv4.js";
|
package/src/server/prune.ts
CHANGED
|
@@ -9,6 +9,9 @@ import { pruneUploads } from "./uploads.js";
|
|
|
9
9
|
|
|
10
10
|
const DEFAULT_BATCH_SIZE = 500;
|
|
11
11
|
|
|
12
|
+
/** How many removed file ids a prune report keeps (`removedFileIds`). */
|
|
13
|
+
export const PRUNE_REPORT_MAX_FILE_IDS = 10;
|
|
14
|
+
|
|
12
15
|
export interface PruneFilesOptions {
|
|
13
16
|
/** Rows read per round; rounds repeat until one comes back short. */
|
|
14
17
|
readonly batchSize?: number;
|
|
@@ -17,6 +20,8 @@ export interface PruneFilesOptions {
|
|
|
17
20
|
export interface PruneFilesReport {
|
|
18
21
|
readonly deletedFiles: number;
|
|
19
22
|
readonly freedBytes: number;
|
|
23
|
+
/** Ids of the first `PRUNE_REPORT_MAX_FILE_IDS` (10) removed files, in removal order; `deletedFiles` has the count. */
|
|
24
|
+
readonly removedFileIds: readonly string[];
|
|
20
25
|
/** Files whose storage delete failed; they stay for the next run. */
|
|
21
26
|
readonly failedFiles: number;
|
|
22
27
|
/** Expired upload sessions removed with their parts. */
|
|
@@ -31,6 +36,7 @@ export async function pruneFiles(ctx: FilesContext, options: PruneFilesOptions =
|
|
|
31
36
|
let deletedFiles = 0;
|
|
32
37
|
let freedBytes = 0;
|
|
33
38
|
let failedFiles = 0;
|
|
39
|
+
const removedFileIds: string[] = [];
|
|
34
40
|
for (;;) {
|
|
35
41
|
const rows = await ctx.db
|
|
36
42
|
.select()
|
|
@@ -43,10 +49,13 @@ export async function pruneFiles(ctx: FilesContext, options: PruneFilesOptions =
|
|
|
43
49
|
if (await removeMarkedFile(ctx, row)) {
|
|
44
50
|
deletedFiles += 1;
|
|
45
51
|
freedBytes += row.size;
|
|
52
|
+
if (removedFileIds.length < PRUNE_REPORT_MAX_FILE_IDS) removedFileIds.push(row.id);
|
|
46
53
|
} else {
|
|
47
54
|
failedFiles += 1;
|
|
48
55
|
}
|
|
49
56
|
}
|
|
50
|
-
if (rows.length < batchSize)
|
|
57
|
+
if (rows.length < batchSize) {
|
|
58
|
+
return { deletedFiles, freedBytes, removedFileIds, failedFiles, ...(await pruneUploads(ctx, batchSize)) };
|
|
59
|
+
}
|
|
51
60
|
}
|
|
52
61
|
}
|
package/src/server/serve.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
// The
|
|
2
|
-
//
|
|
3
|
-
//
|
|
1
|
+
// The answer for a stored file, framework-free, behind a public link (`serveFileLink`) or for its signed-in owner
|
|
2
|
+
// (`serveStoredFile`): `GET` and `HEAD` with `Content-Length`, `Accept-Ranges`, single byte ranges (`206`, `416`),
|
|
3
|
+
// `Content-Disposition`, and `Cache-Control: private, no-store` so no CDN keeps a copy. Every dead link or lookup
|
|
4
|
+
// (forged, expired, revoked, deleted, another owner's, missing from storage) gets the same `404`. A stored file is user
|
|
4
5
|
// content on the app's origin, so it is sent with `nosniff` and a sandboxing CSP: an HTML or SVG file never runs.
|
|
5
6
|
import { errorLogLabel } from "@softure-ai/core";
|
|
6
7
|
import type { ByteRange } from "../contract.js";
|
|
8
|
+
import type { FileRow } from "../schema.js";
|
|
7
9
|
import type { FilesContext } from "./context.js";
|
|
8
10
|
import { findLiveRow } from "./files.js";
|
|
9
11
|
import { verifyFileLink } from "./links.js";
|
|
@@ -19,8 +21,8 @@ const ALLOWED_METHODS = "GET, HEAD";
|
|
|
19
21
|
|
|
20
22
|
const NOT_FOUND_BODY = "Not found\n";
|
|
21
23
|
|
|
22
|
-
/** The one answer for every link that does not lead to a live file. */
|
|
23
|
-
function notFound(): Response {
|
|
24
|
+
/** The one answer for every link or lookup that does not lead to a live file. */
|
|
25
|
+
export function notFound(): Response {
|
|
24
26
|
return new Response(NOT_FOUND_BODY, {
|
|
25
27
|
status: 404,
|
|
26
28
|
headers: {
|
|
@@ -33,12 +35,7 @@ function notFound(): Response {
|
|
|
33
35
|
|
|
34
36
|
/** Answers a request for a public link (`GET` or `HEAD`). */
|
|
35
37
|
export async function serveFileLink(ctx: FilesContext, request: Request): Promise<Response> {
|
|
36
|
-
if (request
|
|
37
|
-
return new Response(null, {
|
|
38
|
-
status: 405,
|
|
39
|
-
headers: { ...BASE_HEADERS, allow: ALLOWED_METHODS },
|
|
40
|
-
});
|
|
41
|
-
}
|
|
38
|
+
if (!isReadMethod(request)) return methodNotAllowed();
|
|
42
39
|
const link = verifyFileLink(ctx, new URL(request.url).pathname);
|
|
43
40
|
if (!link.ok) {
|
|
44
41
|
if (link.error !== "files.not_found") console.error(`@softure-ai/files: a link could not be checked: ${link.error}`);
|
|
@@ -46,13 +43,53 @@ export async function serveFileLink(ctx: FilesContext, request: Request): Promis
|
|
|
46
43
|
}
|
|
47
44
|
const row = await findLiveRow(ctx, link.value.fileId);
|
|
48
45
|
if (row?.linkGeneration !== link.value.generation) return notFound();
|
|
46
|
+
const disposition = getFilesOptions(ctx.config).links.disposition;
|
|
47
|
+
return answerStoredFile(ctx, request, row, { name: link.value.name, disposition });
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export interface ServeStoredFileOptions {
|
|
51
|
+
/** The signed-in user the app's session names; another owner's file reads as missing. */
|
|
52
|
+
readonly ownerId: string;
|
|
53
|
+
/** `inline` (the default: shown in the app's own page, a player or an image) or `attachment` (a download). */
|
|
54
|
+
readonly disposition?: "attachment" | "inline";
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
/**
|
|
58
|
+
* Answers a request (`GET` or `HEAD`) for a stored file on behalf of its signed-in owner: the answer of a public link
|
|
59
|
+
* (single byte ranges, `private, no-store`, `nosniff`, the sandboxing CSP) under the file's stored name, with no link
|
|
60
|
+
* to mint or verify. The caller has already checked the session; a file that is missing, expired, deleted or another
|
|
61
|
+
* owner's gets the same `404`.
|
|
62
|
+
*/
|
|
63
|
+
export async function serveStoredFile(ctx: FilesContext, fileId: string, request: Request, options: ServeStoredFileOptions): Promise<Response> {
|
|
64
|
+
if (!isReadMethod(request)) return methodNotAllowed();
|
|
65
|
+
const row = await findLiveRow(ctx, fileId, { ownerId: options.ownerId });
|
|
66
|
+
if (row === undefined) return notFound();
|
|
67
|
+
return answerStoredFile(ctx, request, row, { name: row.originalName, disposition: options.disposition ?? "inline" });
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
function isReadMethod(request: Request): boolean {
|
|
71
|
+
return request.method === "GET" || request.method === "HEAD";
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function methodNotAllowed(): Response {
|
|
75
|
+
return new Response(null, {
|
|
76
|
+
status: 405,
|
|
77
|
+
headers: { ...BASE_HEADERS, allow: ALLOWED_METHODS },
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
interface StoredFileAnswerOptions {
|
|
82
|
+
readonly name: string;
|
|
83
|
+
readonly disposition: "attachment" | "inline";
|
|
84
|
+
}
|
|
49
85
|
|
|
50
|
-
|
|
86
|
+
/** The bytes of a live row as the answer to a `GET` or `HEAD`, honouring a single byte range. */
|
|
87
|
+
async function answerStoredFile(ctx: FilesContext, request: Request, row: FileRow, options: StoredFileAnswerOptions): Promise<Response> {
|
|
51
88
|
const headers: Record<string, string> = {
|
|
52
89
|
...BASE_HEADERS,
|
|
53
90
|
"content-type": row.contentType,
|
|
54
91
|
"accept-ranges": "bytes",
|
|
55
|
-
"content-disposition": formatContentDisposition(options.
|
|
92
|
+
"content-disposition": formatContentDisposition(options.disposition, options.name),
|
|
56
93
|
"content-security-policy": "sandbox; default-src 'none'",
|
|
57
94
|
};
|
|
58
95
|
const range = parseRangeHeader(request.headers.get("range"), row.size);
|
|
@@ -69,7 +106,7 @@ export async function serveFileLink(ctx: FilesContext, request: Request): Promis
|
|
|
69
106
|
|
|
70
107
|
let body: ReadableStream<Uint8Array> | null;
|
|
71
108
|
try {
|
|
72
|
-
body = await
|
|
109
|
+
body = await getFilesOptions(ctx.config).storage.get(row.storageKey, range ?? undefined);
|
|
73
110
|
} catch (error) {
|
|
74
111
|
console.error(`@softure-ai/files: reading file ${row.id} from storage failed: ${errorLogLabel(error)}`);
|
|
75
112
|
throw error;
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
// The browser fallback for an upload session: a small page on the upload URL itself, for a person on a phone or an
|
|
2
2
|
// assistant that cannot run `curl`. It sends the missing chunks of the chosen file one by one with `fetch`, so a
|
|
3
|
-
// broken connection resumes where it stopped.
|
|
4
|
-
// style, allowed by a per-response nonce. The copy
|
|
3
|
+
// broken connection resumes where it stopped. The loop is `sendUpload` of `@softure-ai/files/client`, inlined. No
|
|
4
|
+
// framework and no external asset: one inline script and one inline style, allowed by a per-response nonce. The copy
|
|
5
|
+
// comes from the module dictionary (`uploadPage`).
|
|
5
6
|
import { randomBytes } from "node:crypto";
|
|
6
7
|
import { selectMessages } from "@softure-ai/core";
|
|
8
|
+
import { sendUpload } from "../client/send-upload.js";
|
|
7
9
|
import type { UploadStatus } from "../contract.js";
|
|
8
10
|
import type { FilesMessages } from "../messages/index.js";
|
|
9
11
|
import type { FilesContext } from "./context.js";
|
|
@@ -13,38 +15,29 @@ const STYLE = `body{font:16px/1.5 system-ui,sans-serif;max-width:32rem;margin:2r
|
|
|
13
15
|
button,input{font:inherit}button{margin-top:1rem;padding:.5rem 1rem}progress{width:100%;margin-top:1rem}
|
|
14
16
|
p[role=status]{min-height:1.5em}`;
|
|
15
17
|
|
|
16
|
-
|
|
18
|
+
// The page's script: the client's own `sendUpload`, inlined from its source, then the few lines that wire it to the
|
|
19
|
+
// form. `sendUpload` references nothing outside its body (see `client/send-upload.ts`).
|
|
20
|
+
const SCRIPT = `${sendUpload.toString()}
|
|
21
|
+
const config = JSON.parse(document.getElementById("upload-config").textContent);
|
|
17
22
|
const input = document.getElementById("file");
|
|
18
23
|
const button = document.getElementById("start");
|
|
19
24
|
const bar = document.getElementById("bar");
|
|
20
25
|
const status = document.getElementById("status");
|
|
21
26
|
const megabytes = (bytes) => (bytes / 1048576).toFixed(1) + " MB";
|
|
22
27
|
function show(text) { status.textContent = text; }
|
|
28
|
+
function showProgress(progress) {
|
|
29
|
+
bar.value = progress.totalBytes === 0 ? 1 : progress.sentBytes / progress.totalBytes;
|
|
30
|
+
show(config.text.progress.replace("{done}", megabytes(progress.sentBytes)).replace("{total}", megabytes(progress.totalBytes)));
|
|
31
|
+
}
|
|
23
32
|
async function upload() {
|
|
24
33
|
const file = input.files[0];
|
|
25
34
|
if (!file) return;
|
|
26
35
|
if (file.size !== config.size) { show(config.text.wrongSize); return; }
|
|
27
36
|
button.disabled = true;
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
let done = state.receivedBytes;
|
|
33
|
-
for (const index of state.missingChunks) {
|
|
34
|
-
const start = index * config.chunkSize;
|
|
35
|
-
const end = Math.min(start + config.chunkSize, config.size) - 1;
|
|
36
|
-
const sent = await fetch(location.href, { method: "PUT", headers: { "content-range": "bytes " + start + "-" + end + "/" + config.size }, body: file.slice(start, end + 1) });
|
|
37
|
-
if (!sent.ok) throw new Error("status " + sent.status);
|
|
38
|
-
done += end - start + 1;
|
|
39
|
-
bar.value = done / config.size;
|
|
40
|
-
show(config.text.progress.replace("{done}", megabytes(done)).replace("{total}", megabytes(config.size)));
|
|
41
|
-
}
|
|
42
|
-
bar.value = 1;
|
|
43
|
-
show(config.text.done);
|
|
44
|
-
} catch {
|
|
45
|
-
show(config.text.failed);
|
|
46
|
-
button.disabled = false;
|
|
47
|
-
}
|
|
37
|
+
const sent = await sendUpload(file, { uploadUrl: location.href, onProgress: showProgress });
|
|
38
|
+
if (sent.ok) { bar.value = 1; show(config.text.done); return; }
|
|
39
|
+
show(sent.error === "files.upload_expired" ? config.text.expired : config.text.failed);
|
|
40
|
+
button.disabled = sent.error === "files.upload_expired";
|
|
48
41
|
}
|
|
49
42
|
button.addEventListener("click", upload);`;
|
|
50
43
|
|