@songsid/agend 2.2.0-beta.17 → 2.2.0-beta.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +18 -18
- package/README.zh-TW.md +18 -18
- package/dist/agent-whats-new.js +3 -1
- package/dist/agent-whats-new.js.map +1 -1
- package/dist/backend/kiro.d.ts +2 -0
- package/dist/backend/kiro.js +4 -0
- package/dist/backend/kiro.js.map +1 -1
- package/dist/backend/types.d.ts +5 -0
- package/dist/backend/types.js.map +1 -1
- package/dist/compact-watch.d.ts +9 -1
- package/dist/compact-watch.js +47 -6
- package/dist/compact-watch.js.map +1 -1
- package/dist/config-validator.js +26 -1
- package/dist/config-validator.js.map +1 -1
- package/dist/daemon.d.ts +8 -0
- package/dist/daemon.js +65 -5
- package/dist/daemon.js.map +1 -1
- package/dist/fleet-manager.js +49 -20
- package/dist/fleet-manager.js.map +1 -1
- package/dist/forged-envelope-scan.d.ts +4 -3
- package/dist/forged-envelope-scan.js +4 -3
- package/dist/forged-envelope-scan.js.map +1 -1
- package/dist/kiro-compact-pane.d.ts +38 -0
- package/dist/kiro-compact-pane.js +57 -0
- package/dist/kiro-compact-pane.js.map +1 -0
- package/dist/locale.js +4 -4
- package/dist/locale.js.map +1 -1
- package/dist/outbound-schemas.js +1 -1
- package/dist/outbound-schemas.js.map +1 -1
- package/dist/public-web-gateway.d.ts +2 -0
- package/dist/public-web-gateway.js +6 -1
- package/dist/public-web-gateway.js.map +1 -1
- package/dist/types.d.ts +8 -0
- package/dist/ui/chat-render.js +31 -6
- package/dist/ui/chat-store.js +5 -5
- package/dist/ui/chat-strings.js +6 -2
- package/dist/ui/chat-thread.js +15 -2
- package/dist/ui/panel-chat.js +1 -1
- package/dist/ui/panel-settings.js +1 -1
- package/dist/ui/shared/app.css +7 -0
- package/dist/web-api.d.ts +4 -0
- package/dist/web-api.js +135 -43
- package/dist/web-api.js.map +1 -1
- package/dist/web-chat-history.d.ts +14 -0
- package/dist/web-chat-history.js +18 -1
- package/dist/web-chat-history.js.map +1 -1
- package/dist/web-upload.d.ts +112 -1
- package/dist/web-upload.js +237 -36
- package/dist/web-upload.js.map +1 -1
- package/package.json +3 -3
package/dist/web-upload.d.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { openSync } from "node:fs";
|
|
2
|
+
/** #1937: the defaults; `web.upload_max_file_mb` / `web.upload_max_total_mb` change the two sizes (uploadLimits()). */
|
|
1
3
|
export declare const UPLOAD_LIMITS: {
|
|
2
4
|
/** One file. */
|
|
3
5
|
readonly maxFileBytes: number;
|
|
@@ -6,10 +8,79 @@ export declare const UPLOAD_LIMITS: {
|
|
|
6
8
|
/** All files on one message together. */
|
|
7
9
|
readonly maxTotalBytes: number;
|
|
8
10
|
};
|
|
11
|
+
/** #1937: what `web.upload_max_file_mb` and `web.upload_max_total_mb` may be — whole MB (MiB). */
|
|
12
|
+
export declare const UPLOAD_MB_RANGE: {
|
|
13
|
+
readonly min: 1;
|
|
14
|
+
readonly max: 500;
|
|
15
|
+
};
|
|
16
|
+
/**
|
|
17
|
+
* #1937: one file over the public link. A Cloudflare quick tunnel carries a request body of about 100 MB at most, and
|
|
18
|
+
* one upload is one request, so whatever the config says, a file through the tunnel stays under that.
|
|
19
|
+
*/
|
|
20
|
+
export declare const PUBLIC_LINK_MAX_FILE_BYTES: number;
|
|
21
|
+
export interface UploadLimits {
|
|
22
|
+
maxFileBytes: number;
|
|
23
|
+
maxFiles: number;
|
|
24
|
+
maxTotalBytes: number;
|
|
25
|
+
/** The public link lowered the per-file cap below the configured one (its 413 says why). */
|
|
26
|
+
publicLinkCapped?: true;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* #1937: the limits in force — the one source the upload route, the send, the served-file cap, the page's own check
|
|
30
|
+
* and its words all read. A file is never allowed more than a whole message is; over the public link a file is at most
|
|
31
|
+
* PUBLIC_LINK_MAX_FILE_BYTES. Read on every request, so `agend reload` applies a change to the next upload.
|
|
32
|
+
*/
|
|
33
|
+
export declare function uploadLimits(web: {
|
|
34
|
+
upload_max_file_mb?: unknown;
|
|
35
|
+
upload_max_total_mb?: unknown;
|
|
36
|
+
} | null | undefined, opts?: {
|
|
37
|
+
publicLink?: boolean;
|
|
38
|
+
}): UploadLimits;
|
|
39
|
+
/** A size in MB as the limits are worded ("50", "2.5"). */
|
|
40
|
+
export declare function mbText(bytes: number): string;
|
|
41
|
+
/** #1937: the 413 for one file, from the limits in force (never a constant's number). */
|
|
42
|
+
export declare function fileTooLargeText(limits: UploadLimits): string;
|
|
43
|
+
/** #1937: how long an upload may go without a byte before it is dropped (its partial file with it). */
|
|
44
|
+
export declare const UPLOAD_STALL_MS = 60000;
|
|
45
|
+
/** #1937: the longest one upload may take, start to end (at 95 MB, about 54 KB/s on average). */
|
|
46
|
+
export declare const UPLOAD_MAX_MS: number;
|
|
47
|
+
/** #1937: what a listener allows for receiving one request — above UPLOAD_MAX_MS, so an upload's own clocks decide. */
|
|
48
|
+
export declare const UPLOAD_REQUEST_TIMEOUT_MS: number;
|
|
49
|
+
/** #1937: the one route whose body may take long to arrive (its own clocks bound it). */
|
|
50
|
+
export declare function isUploadRequest(req: {
|
|
51
|
+
method?: string;
|
|
52
|
+
url?: string;
|
|
53
|
+
}): boolean;
|
|
54
|
+
/**
|
|
55
|
+
* #1937: every other request keeps the listener's old bound on receiving it — not whole within `ms`, its connection is
|
|
56
|
+
* dropped (what `server.requestTimeout` did before it was raised for uploads).
|
|
57
|
+
*/
|
|
58
|
+
export declare function limitRequestReceive(req: {
|
|
59
|
+
complete?: boolean;
|
|
60
|
+
socket?: {
|
|
61
|
+
destroy(): void;
|
|
62
|
+
} | null;
|
|
63
|
+
once?(event: "close", fn: () => void): unknown;
|
|
64
|
+
}, ms: number): void;
|
|
9
65
|
/** How long an uploaded file waits to be sent with a message before its id stops working. */
|
|
10
66
|
export declare const UPLOAD_TTL_MS: number;
|
|
11
67
|
/** An upload no message has taken yet is stored under this prefix; it drops the prefix when a message takes it. */
|
|
12
68
|
export declare const PENDING_PREFIX = "web-pending-";
|
|
69
|
+
/** #1937: an upload's bytes stream into this file first (pending prefix: a restart's sweep removes one left behind). */
|
|
70
|
+
export declare const PART_SUFFIX = ".part";
|
|
71
|
+
/**
|
|
72
|
+
* #1937: a new, empty part file in `inboxDir` (made private, 0700), created exclusively, 0600, never through a link.
|
|
73
|
+
* Its bytes are written through `fd` (never by reopening the name); the caller closes it. `discard()` removes the part
|
|
74
|
+
* and the directories made for it, when they are still empty — an upload that ends in nothing leaves nothing behind
|
|
75
|
+
* (an instance deleted while its body was arriving gets no inbox made back, #1252 review P2-3).
|
|
76
|
+
*/
|
|
77
|
+
export declare function createPartFile(inboxDir: string, open?: typeof openSync): {
|
|
78
|
+
path: string;
|
|
79
|
+
fd: number;
|
|
80
|
+
discard(): void;
|
|
81
|
+
};
|
|
82
|
+
/** Remove a file of ours, if it is still there; never throws. */
|
|
83
|
+
export declare function removeQuietly(path: string): void;
|
|
13
84
|
/** The largest file the dashboard will serve back (a reply attachment can be anything the agent made). */
|
|
14
85
|
export declare const MAX_SERVED_BYTES: number;
|
|
15
86
|
export type UploadKind = "photo" | "document";
|
|
@@ -28,6 +99,21 @@ export declare function storedExtension(name: string): string;
|
|
|
28
99
|
* is taken too, as a document the dashboard only offers as a download. Null only for an empty file.
|
|
29
100
|
*/
|
|
30
101
|
export declare function sniffUpload(bytes: Uint8Array, name: string): SniffedType | null;
|
|
102
|
+
/**
|
|
103
|
+
* #1937: sniffUpload() as the bytes stream in — the upload is written to disk chunk by chunk, never held whole. Keeps
|
|
104
|
+
* the first bytes (every signature is in the first 12), whether a NUL was seen, and whether everything so far is
|
|
105
|
+
* valid UTF-8 (a streaming decoder: a character split across two chunks is still one character).
|
|
106
|
+
*/
|
|
107
|
+
export declare class UploadSniffer {
|
|
108
|
+
private head;
|
|
109
|
+
private size;
|
|
110
|
+
private nul;
|
|
111
|
+
private utf8;
|
|
112
|
+
private readonly decoder;
|
|
113
|
+
push(chunk: Uint8Array): this;
|
|
114
|
+
/** The type of everything pushed (call once, at the end); null for an empty file. */
|
|
115
|
+
result(name: string): SniffedType | null;
|
|
116
|
+
}
|
|
31
117
|
/** A file name as a label only: no directory, no control characters, no leading dots, bounded. */
|
|
32
118
|
export declare function displayName(raw: string | undefined | null, fallback: string): string;
|
|
33
119
|
/** Any lone surrogate replaced by U+FFFD, so the name always encodes. */
|
|
@@ -85,10 +171,15 @@ export declare class WebFileLedger {
|
|
|
85
171
|
private readonly served;
|
|
86
172
|
private readonly now;
|
|
87
173
|
private readonly maxServed;
|
|
174
|
+
/** #1937: the limits in force (the fleet's config, read each time): the send's count and total, the served-file cap. */
|
|
175
|
+
readonly limits: () => UploadLimits;
|
|
88
176
|
constructor(opts?: {
|
|
89
177
|
now?: () => number;
|
|
90
178
|
maxServed?: number;
|
|
179
|
+
limits?: () => UploadLimits;
|
|
91
180
|
});
|
|
181
|
+
/** #1937: the largest file served back — MAX_SERVED_BYTES, or more when a file that large may be uploaded. */
|
|
182
|
+
maxServedBytes(): number;
|
|
92
183
|
/**
|
|
93
184
|
* Store an upload in `inboxDir` under a name chosen here and register it. The directory is made
|
|
94
185
|
* private (0700), the file 0600, written exclusively (never over an existing file).
|
|
@@ -100,6 +191,16 @@ export declare class WebFileLedger {
|
|
|
100
191
|
name: string;
|
|
101
192
|
type: SniffedType;
|
|
102
193
|
}): UploadEntry;
|
|
194
|
+
/**
|
|
195
|
+
* #1937: register an upload that was streamed to `partPath` (createPartFile(), in the inbox it belongs to): it is
|
|
196
|
+
* linked under its stored name — never over an existing file — and the part goes. The part is removed on failure too.
|
|
197
|
+
*/
|
|
198
|
+
storeUploadedFile(input: {
|
|
199
|
+
instance: string;
|
|
200
|
+
partPath: string;
|
|
201
|
+
name: string;
|
|
202
|
+
type: SniffedType;
|
|
203
|
+
}): UploadEntry;
|
|
103
204
|
/**
|
|
104
205
|
* The uploads a message names, all or nothing: each must exist, belong to this instance, not be
|
|
105
206
|
* expired and be waiting for a message; together within the per-message limits. On success they are
|
|
@@ -147,6 +248,16 @@ export declare class WebFileLedger {
|
|
|
147
248
|
file: ServedFile;
|
|
148
249
|
bytes: Buffer;
|
|
149
250
|
} | null;
|
|
251
|
+
/**
|
|
252
|
+
* #1937: a served file opened for streaming whole, with read()'s checks on the open descriptor (no symlink followed,
|
|
253
|
+
* a regular file, the device + inode and size it was registered with). The caller closes `fd`. Null when any fails.
|
|
254
|
+
*/
|
|
255
|
+
open(id: string): {
|
|
256
|
+
file: ServedFile;
|
|
257
|
+
fd: number;
|
|
258
|
+
} | null;
|
|
259
|
+
/** #1937: what is served under an id (no file is opened: open() and read() check the file itself). */
|
|
260
|
+
servedFile(id: string): ServedFile | null;
|
|
150
261
|
/** #1589: a served file that is audio or video (MEDIA_MIME), or null — the file route plays those by range. */
|
|
151
262
|
mediaOf(id: string): ServedFile | null;
|
|
152
263
|
/**
|
|
@@ -178,7 +289,7 @@ export declare class WebFileLedger {
|
|
|
178
289
|
* symlink, and a regular file within MAX_SERVED_BYTES; its real path inside this instance's inbox
|
|
179
290
|
* (`workspaces/<instance>/inbox`, itself not a link elsewhere) — or (#1770) inside its reply snapshots
|
|
180
291
|
* (`webFilesDir`, `workspaces/<instance>/web-files`, the same checks), then served as a snapshot again. Null when any
|
|
181
|
-
* check fails (the message then shows the name as unavailable). Fetched by id only, as ever: the stored path is the
|
|
292
|
+
* check fails (the message then shows the name as unavailable). (Within maxServedBytes(): #1937.) Fetched by id only, as ever: the stored path is the
|
|
182
293
|
* fleet's own file, never a request's.
|
|
183
294
|
*/
|
|
184
295
|
restoreServed(entry: {
|
package/dist/web-upload.js
CHANGED
|
@@ -12,22 +12,119 @@
|
|
|
12
12
|
* label, and a file can be fetched back only by an id this process issued for it — never by a path.
|
|
13
13
|
*/
|
|
14
14
|
import { randomBytes } from "node:crypto";
|
|
15
|
-
import { constants as fsConstants, closeSync, existsSync, fstatSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, realpathSync, renameSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
15
|
+
import { constants as fsConstants, closeSync, existsSync, fstatSync, linkSync, lstatSync, mkdirSync, openSync, readdirSync, readSync, realpathSync, renameSync, rmdirSync, statSync, unlinkSync, writeFileSync } from "node:fs";
|
|
16
16
|
import { basename, dirname, extname, join, sep } from "node:path";
|
|
17
|
+
const MB = 1024 * 1024;
|
|
18
|
+
/** #1937: the defaults; `web.upload_max_file_mb` / `web.upload_max_total_mb` change the two sizes (uploadLimits()). */
|
|
17
19
|
export const UPLOAD_LIMITS = {
|
|
18
20
|
/** One file. */
|
|
19
|
-
maxFileBytes:
|
|
21
|
+
maxFileBytes: 50 * MB,
|
|
20
22
|
/** Files on one message. */
|
|
21
23
|
maxFiles: 5,
|
|
22
24
|
/** All files on one message together. */
|
|
23
|
-
maxTotalBytes:
|
|
25
|
+
maxTotalBytes: 100 * MB,
|
|
24
26
|
};
|
|
27
|
+
/** #1937: what `web.upload_max_file_mb` and `web.upload_max_total_mb` may be — whole MB (MiB). */
|
|
28
|
+
export const UPLOAD_MB_RANGE = { min: 1, max: 500 };
|
|
29
|
+
/**
|
|
30
|
+
* #1937: one file over the public link. A Cloudflare quick tunnel carries a request body of about 100 MB at most, and
|
|
31
|
+
* one upload is one request, so whatever the config says, a file through the tunnel stays under that.
|
|
32
|
+
*/
|
|
33
|
+
export const PUBLIC_LINK_MAX_FILE_BYTES = 95 * MB;
|
|
34
|
+
/** A configured size in MB, or the default when it is absent or not a whole number in UPLOAD_MB_RANGE (the validator says so). */
|
|
35
|
+
function configuredMb(v, fallback) {
|
|
36
|
+
return typeof v === "number" && Number.isInteger(v) && v >= UPLOAD_MB_RANGE.min && v <= UPLOAD_MB_RANGE.max ? v * MB : fallback;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* #1937: the limits in force — the one source the upload route, the send, the served-file cap, the page's own check
|
|
40
|
+
* and its words all read. A file is never allowed more than a whole message is; over the public link a file is at most
|
|
41
|
+
* PUBLIC_LINK_MAX_FILE_BYTES. Read on every request, so `agend reload` applies a change to the next upload.
|
|
42
|
+
*/
|
|
43
|
+
export function uploadLimits(web, opts = {}) {
|
|
44
|
+
const total = configuredMb(web?.upload_max_total_mb, UPLOAD_LIMITS.maxTotalBytes);
|
|
45
|
+
const file = Math.min(configuredMb(web?.upload_max_file_mb, UPLOAD_LIMITS.maxFileBytes), total);
|
|
46
|
+
const capped = opts.publicLink === true && file > PUBLIC_LINK_MAX_FILE_BYTES;
|
|
47
|
+
return { maxFileBytes: capped ? PUBLIC_LINK_MAX_FILE_BYTES : file, maxFiles: UPLOAD_LIMITS.maxFiles, maxTotalBytes: total, ...(capped ? { publicLinkCapped: true } : {}) };
|
|
48
|
+
}
|
|
49
|
+
/** A size in MB as the limits are worded ("50", "2.5"). */
|
|
50
|
+
export function mbText(bytes) { return String(Math.round((bytes / MB) * 10) / 10); }
|
|
51
|
+
/** #1937: the 413 for one file, from the limits in force (never a constant's number). */
|
|
52
|
+
export function fileTooLargeText(limits) {
|
|
53
|
+
return limits.publicLinkCapped
|
|
54
|
+
? `a file can be at most ${mbText(limits.maxFileBytes)} MB over the public link (its Cloudflare tunnel carries about 100 MB per request)`
|
|
55
|
+
: `a file can be at most ${mbText(limits.maxFileBytes)} MB`;
|
|
56
|
+
}
|
|
57
|
+
/** #1937: how long an upload may go without a byte before it is dropped (its partial file with it). */
|
|
58
|
+
export const UPLOAD_STALL_MS = 60_000;
|
|
59
|
+
/** #1937: the longest one upload may take, start to end (at 95 MB, about 54 KB/s on average). */
|
|
60
|
+
export const UPLOAD_MAX_MS = 30 * 60 * 1000;
|
|
61
|
+
/** #1937: what a listener allows for receiving one request — above UPLOAD_MAX_MS, so an upload's own clocks decide. */
|
|
62
|
+
export const UPLOAD_REQUEST_TIMEOUT_MS = UPLOAD_MAX_MS + 60_000;
|
|
63
|
+
/** #1937: the one route whose body may take long to arrive (its own clocks bound it). */
|
|
64
|
+
export function isUploadRequest(req) {
|
|
65
|
+
return req.method === "POST" && (req.url ?? "").split("?")[0] === "/ui/upload";
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* #1937: every other request keeps the listener's old bound on receiving it — not whole within `ms`, its connection is
|
|
69
|
+
* dropped (what `server.requestTimeout` did before it was raised for uploads).
|
|
70
|
+
*/
|
|
71
|
+
export function limitRequestReceive(req, ms) {
|
|
72
|
+
if (req.complete !== false)
|
|
73
|
+
return; // whole already (a request object always says)
|
|
74
|
+
const t = setTimeout(() => { if (req.complete === false)
|
|
75
|
+
req.socket?.destroy(); }, ms);
|
|
76
|
+
t.unref?.();
|
|
77
|
+
req.once?.("close", () => clearTimeout(t));
|
|
78
|
+
}
|
|
25
79
|
/** How long an uploaded file waits to be sent with a message before its id stops working. */
|
|
26
80
|
export const UPLOAD_TTL_MS = 30 * 60 * 1000;
|
|
27
81
|
/** An upload no message has taken yet is stored under this prefix; it drops the prefix when a message takes it. */
|
|
28
82
|
export const PENDING_PREFIX = "web-pending-";
|
|
29
83
|
function sentPath(p) { const b = basename(p); return b.startsWith(PENDING_PREFIX) ? join(dirname(p), "web-" + b.slice(PENDING_PREFIX.length)) : p; }
|
|
30
84
|
function pendingPath(p) { const b = basename(p); return b.startsWith("web-") && !b.startsWith(PENDING_PREFIX) ? join(dirname(p), PENDING_PREFIX + b.slice(4)) : p; }
|
|
85
|
+
/** #1937: an upload's bytes stream into this file first (pending prefix: a restart's sweep removes one left behind). */
|
|
86
|
+
export const PART_SUFFIX = ".part";
|
|
87
|
+
/**
|
|
88
|
+
* #1937: a new, empty part file in `inboxDir` (made private, 0700), created exclusively, 0600, never through a link.
|
|
89
|
+
* Its bytes are written through `fd` (never by reopening the name); the caller closes it. `discard()` removes the part
|
|
90
|
+
* and the directories made for it, when they are still empty — an upload that ends in nothing leaves nothing behind
|
|
91
|
+
* (an instance deleted while its body was arriving gets no inbox made back, #1252 review P2-3).
|
|
92
|
+
*/
|
|
93
|
+
export function createPartFile(inboxDir, open = openSync) {
|
|
94
|
+
const made = mkdirSync(inboxDir, { recursive: true, mode: 0o700 });
|
|
95
|
+
const path = join(inboxDir, `${PENDING_PREFIX}${Date.now()}-${randomBytes(6).toString("hex")}${PART_SUFFIX}`);
|
|
96
|
+
const discard = () => {
|
|
97
|
+
removeQuietly(path);
|
|
98
|
+
if (!made)
|
|
99
|
+
return;
|
|
100
|
+
// From the inbox up to the first directory made here; rmdir only removes an empty one (never one that was there).
|
|
101
|
+
for (let d = inboxDir; d.length >= made.length; d = dirname(d)) {
|
|
102
|
+
try {
|
|
103
|
+
rmdirSync(d);
|
|
104
|
+
}
|
|
105
|
+
catch {
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
if (d === made)
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
111
|
+
};
|
|
112
|
+
// Prism #1941 P3-1: an open that fails (EMFILE, ENOSPC …) still takes back the directories made for it.
|
|
113
|
+
let fd;
|
|
114
|
+
try {
|
|
115
|
+
fd = open(path, fsConstants.O_WRONLY | fsConstants.O_CREAT | fsConstants.O_EXCL | fsConstants.O_NOFOLLOW, 0o600);
|
|
116
|
+
}
|
|
117
|
+
catch (err) {
|
|
118
|
+
discard();
|
|
119
|
+
throw err;
|
|
120
|
+
}
|
|
121
|
+
return { path, fd, discard };
|
|
122
|
+
}
|
|
123
|
+
/** Remove a file of ours, if it is still there; never throws. */
|
|
124
|
+
export function removeQuietly(path) { try {
|
|
125
|
+
unlinkSync(path);
|
|
126
|
+
}
|
|
127
|
+
catch { /* already gone */ } }
|
|
31
128
|
/** The largest file the dashboard will serve back (a reply attachment can be anything the agent made). */
|
|
32
129
|
export const MAX_SERVED_BYTES = 50 * 1024 * 1024;
|
|
33
130
|
/** Text files keep a known extension from their name; anything else is stored as .txt. */
|
|
@@ -53,29 +150,68 @@ const startsWith = (b, sig, at = 0) => sig.every((v, i) => b[at + i] === v);
|
|
|
53
150
|
* is taken too, as a document the dashboard only offers as a download. Null only for an empty file.
|
|
54
151
|
*/
|
|
55
152
|
export function sniffUpload(bytes, name) {
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
153
|
+
return new UploadSniffer().push(bytes).result(name);
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* #1937: sniffUpload() as the bytes stream in — the upload is written to disk chunk by chunk, never held whole. Keeps
|
|
157
|
+
* the first bytes (every signature is in the first 12), whether a NUL was seen, and whether everything so far is
|
|
158
|
+
* valid UTF-8 (a streaming decoder: a character split across two chunks is still one character).
|
|
159
|
+
*/
|
|
160
|
+
export class UploadSniffer {
|
|
161
|
+
head = Buffer.alloc(0);
|
|
162
|
+
size = 0;
|
|
163
|
+
nul = false;
|
|
164
|
+
utf8 = true;
|
|
165
|
+
decoder = new TextDecoder("utf-8", { fatal: true });
|
|
166
|
+
push(chunk) {
|
|
167
|
+
if (chunk.length === 0)
|
|
168
|
+
return this;
|
|
169
|
+
this.size += chunk.length;
|
|
170
|
+
if (this.head.length < 16)
|
|
171
|
+
this.head = Buffer.concat([this.head, Buffer.from(chunk.subarray(0, 16 - this.head.length))]);
|
|
172
|
+
if (!this.nul && chunk.includes(0))
|
|
173
|
+
this.nul = true;
|
|
174
|
+
if (this.utf8 && !this.nul) {
|
|
175
|
+
try {
|
|
176
|
+
this.decoder.decode(chunk, { stream: true });
|
|
177
|
+
}
|
|
178
|
+
catch {
|
|
179
|
+
this.utf8 = false;
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
return this;
|
|
74
183
|
}
|
|
75
|
-
|
|
76
|
-
|
|
184
|
+
/** The type of everything pushed (call once, at the end); null for an empty file. */
|
|
185
|
+
result(name) {
|
|
186
|
+
if (this.size === 0)
|
|
187
|
+
return null;
|
|
188
|
+
const b = this.head;
|
|
189
|
+
if (startsWith(b, [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))
|
|
190
|
+
return { kind: "photo", mime: "image/png", ext: ".png" };
|
|
191
|
+
if (startsWith(b, [0xff, 0xd8, 0xff]))
|
|
192
|
+
return { kind: "photo", mime: "image/jpeg", ext: ".jpg" };
|
|
193
|
+
if (startsWith(b, [0x47, 0x49, 0x46, 0x38]))
|
|
194
|
+
return { kind: "photo", mime: "image/gif", ext: ".gif" };
|
|
195
|
+
if (startsWith(b, [0x52, 0x49, 0x46, 0x46]) && startsWith(b, [0x57, 0x45, 0x42, 0x50], 8))
|
|
196
|
+
return { kind: "photo", mime: "image/webp", ext: ".webp" };
|
|
197
|
+
if (startsWith(b, [0x25, 0x50, 0x44, 0x46, 0x2d]))
|
|
198
|
+
return { kind: "document", mime: "application/pdf", ext: ".pdf" };
|
|
199
|
+
const other = { kind: "document", mime: OTHER_FILE_MIME, ext: storedExtension(name) };
|
|
200
|
+
const ext = extname(name).toLowerCase();
|
|
201
|
+
if (MARKUP_EXTENSIONS.has(ext) || this.nul)
|
|
202
|
+
return other;
|
|
203
|
+
if (this.utf8) {
|
|
204
|
+
try {
|
|
205
|
+
this.decoder.decode();
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
this.utf8 = false;
|
|
209
|
+
} // a character cut off at the very end
|
|
210
|
+
}
|
|
211
|
+
if (!this.utf8)
|
|
212
|
+
return other;
|
|
213
|
+
return { kind: "document", mime: "text/plain; charset=utf-8", ext: TEXT_EXTENSIONS.has(ext) ? ext : ".txt" };
|
|
77
214
|
}
|
|
78
|
-
return { kind: "document", mime: "text/plain; charset=utf-8", ext: TEXT_EXTENSIONS.has(ext) ? ext : ".txt" };
|
|
79
215
|
}
|
|
80
216
|
/** A file name as a label only: no directory, no control characters, no leading dots, bounded. */
|
|
81
217
|
export function displayName(raw, fallback) {
|
|
@@ -129,28 +265,58 @@ export class WebFileLedger {
|
|
|
129
265
|
served = new Map();
|
|
130
266
|
now;
|
|
131
267
|
maxServed;
|
|
268
|
+
/** #1937: the limits in force (the fleet's config, read each time): the send's count and total, the served-file cap. */
|
|
269
|
+
limits;
|
|
132
270
|
constructor(opts = {}) {
|
|
133
271
|
// Elapsed time, like the sweep timer below: a wall-clock jump must neither expire an upload early nor keep
|
|
134
272
|
// one past the timer that was meant to remove it (#1252 review).
|
|
135
273
|
this.now = opts.now ?? (() => performance.now());
|
|
136
274
|
this.maxServed = opts.maxServed ?? 2000;
|
|
275
|
+
this.limits = opts.limits ?? (() => uploadLimits(null));
|
|
137
276
|
}
|
|
277
|
+
/** #1937: the largest file served back — MAX_SERVED_BYTES, or more when a file that large may be uploaded. */
|
|
278
|
+
maxServedBytes() { return Math.max(MAX_SERVED_BYTES, this.limits().maxFileBytes); }
|
|
138
279
|
/**
|
|
139
280
|
* Store an upload in `inboxDir` under a name chosen here and register it. The directory is made
|
|
140
281
|
* private (0700), the file 0600, written exclusively (never over an existing file).
|
|
141
282
|
*/
|
|
142
283
|
storeUpload(input) {
|
|
143
|
-
|
|
284
|
+
const part = createPartFile(input.inboxDir);
|
|
285
|
+
try {
|
|
286
|
+
writeFileSync(part.fd, input.bytes);
|
|
287
|
+
}
|
|
288
|
+
catch (err) {
|
|
289
|
+
removeQuietly(part.path);
|
|
290
|
+
throw err;
|
|
291
|
+
}
|
|
292
|
+
finally {
|
|
293
|
+
closeSync(part.fd);
|
|
294
|
+
}
|
|
295
|
+
return this.storeUploadedFile({ instance: input.instance, partPath: part.path, name: input.name, type: input.type });
|
|
296
|
+
}
|
|
297
|
+
/**
|
|
298
|
+
* #1937: register an upload that was streamed to `partPath` (createPartFile(), in the inbox it belongs to): it is
|
|
299
|
+
* linked under its stored name — never over an existing file — and the part goes. The part is removed on failure too.
|
|
300
|
+
*/
|
|
301
|
+
storeUploadedFile(input) {
|
|
144
302
|
const id = randomBytes(16).toString("hex");
|
|
145
303
|
// Stored as web-pending-…: the name says on disk that no message has taken it yet, so a sweep after a restart
|
|
146
304
|
// (which loses this ledger) can still tell an abandoned upload from one an agent got (#1273).
|
|
147
|
-
const path = join(input.
|
|
148
|
-
|
|
305
|
+
const path = join(dirname(input.partPath), `${PENDING_PREFIX}${Date.now()}-${id.slice(0, 8)}${input.type.ext}`);
|
|
306
|
+
try {
|
|
307
|
+
const part = lstatSync(input.partPath);
|
|
308
|
+
if (!part.isFile())
|
|
309
|
+
throw new Error("the upload's part file is not a regular file");
|
|
310
|
+
linkSync(input.partPath, path);
|
|
311
|
+
}
|
|
312
|
+
finally {
|
|
313
|
+
removeQuietly(input.partPath);
|
|
314
|
+
}
|
|
315
|
+
const st = lstatSync(path);
|
|
149
316
|
const entry = {
|
|
150
317
|
id, instance: input.instance, path, kind: input.type.kind, mime: input.type.mime,
|
|
151
|
-
name: displayName(input.name, "file"), size:
|
|
318
|
+
name: displayName(input.name, "file"), size: st.size, expiresAt: this.now() + UPLOAD_TTL_MS, state: "pending",
|
|
152
319
|
};
|
|
153
|
-
const st = lstatSync(path);
|
|
154
320
|
this.written.set(entry, { dev: st.dev, ino: st.ino });
|
|
155
321
|
this.prune();
|
|
156
322
|
this.uploads.set(id, entry);
|
|
@@ -167,8 +333,9 @@ export class WebFileLedger {
|
|
|
167
333
|
* says how the delivery went: commit() when it was delivered, release() when it definitely was not.
|
|
168
334
|
*/
|
|
169
335
|
takeForMessage(instance, ids) {
|
|
170
|
-
|
|
171
|
-
|
|
336
|
+
const limits = this.limits();
|
|
337
|
+
if (ids.length > limits.maxFiles)
|
|
338
|
+
return { ok: false, error: `at most ${limits.maxFiles} files per message` };
|
|
172
339
|
if (new Set(ids).size !== ids.length)
|
|
173
340
|
return { ok: false, error: "the same file twice" };
|
|
174
341
|
const entries = [];
|
|
@@ -181,8 +348,8 @@ export class WebFileLedger {
|
|
|
181
348
|
total += e.size;
|
|
182
349
|
entries.push(e);
|
|
183
350
|
}
|
|
184
|
-
if (total >
|
|
185
|
-
return { ok: false, error: `the files together are over ${
|
|
351
|
+
if (total > limits.maxTotalBytes)
|
|
352
|
+
return { ok: false, error: `the files together are over ${mbText(limits.maxTotalBytes)} MB` };
|
|
186
353
|
// Each file gets its sent name now, before the delivery names it to the agent; all or nothing.
|
|
187
354
|
const moved = [];
|
|
188
355
|
for (const e of entries) {
|
|
@@ -282,7 +449,7 @@ export class WebFileLedger {
|
|
|
282
449
|
try {
|
|
283
450
|
realPath = realpathSync(input.path);
|
|
284
451
|
const st = statSync(realPath);
|
|
285
|
-
if (!st.isFile() || st.size >
|
|
452
|
+
if (!st.isFile() || st.size > this.maxServedBytes())
|
|
286
453
|
return null;
|
|
287
454
|
({ size, dev, ino } = st);
|
|
288
455
|
}
|
|
@@ -324,6 +491,40 @@ export class WebFileLedger {
|
|
|
324
491
|
const got = this.readSlice(id, 0, Number.MAX_SAFE_INTEGER);
|
|
325
492
|
return got && got.start === 0 && got.bytes.length === got.file.size ? { file: got.file, bytes: got.bytes } : null;
|
|
326
493
|
}
|
|
494
|
+
/**
|
|
495
|
+
* #1937: a served file opened for streaming whole, with read()'s checks on the open descriptor (no symlink followed,
|
|
496
|
+
* a regular file, the device + inode and size it was registered with). The caller closes `fd`. Null when any fails.
|
|
497
|
+
*/
|
|
498
|
+
open(id) {
|
|
499
|
+
const file = isFileId(id) ? this.served.get(id) : undefined;
|
|
500
|
+
if (!file)
|
|
501
|
+
return null;
|
|
502
|
+
let fd = null;
|
|
503
|
+
try {
|
|
504
|
+
fd = openSync(file.realPath, fsConstants.O_RDONLY | fsConstants.O_NOFOLLOW);
|
|
505
|
+
const st = fstatSync(fd);
|
|
506
|
+
if (st.isFile() && st.size === file.size && st.dev === file.dev && st.ino === file.ino) {
|
|
507
|
+
const out = { file, fd };
|
|
508
|
+
fd = null;
|
|
509
|
+
return out;
|
|
510
|
+
}
|
|
511
|
+
return null;
|
|
512
|
+
}
|
|
513
|
+
catch {
|
|
514
|
+
return null;
|
|
515
|
+
}
|
|
516
|
+
finally {
|
|
517
|
+
if (fd !== null)
|
|
518
|
+
try {
|
|
519
|
+
closeSync(fd);
|
|
520
|
+
}
|
|
521
|
+
catch { /* closed */ }
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
/** #1937: what is served under an id (no file is opened: open() and read() check the file itself). */
|
|
525
|
+
servedFile(id) {
|
|
526
|
+
return (isFileId(id) ? this.served.get(id) : undefined) ?? null;
|
|
527
|
+
}
|
|
327
528
|
/** #1589: a served file that is audio or video (MEDIA_MIME), or null — the file route plays those by range. */
|
|
328
529
|
mediaOf(id) {
|
|
329
530
|
const file = isFileId(id) ? this.served.get(id) : undefined;
|
|
@@ -408,7 +609,7 @@ export class WebFileLedger {
|
|
|
408
609
|
* symlink, and a regular file within MAX_SERVED_BYTES; its real path inside this instance's inbox
|
|
409
610
|
* (`workspaces/<instance>/inbox`, itself not a link elsewhere) — or (#1770) inside its reply snapshots
|
|
410
611
|
* (`webFilesDir`, `workspaces/<instance>/web-files`, the same checks), then served as a snapshot again. Null when any
|
|
411
|
-
* check fails (the message then shows the name as unavailable). Fetched by id only, as ever: the stored path is the
|
|
612
|
+
* check fails (the message then shows the name as unavailable). (Within maxServedBytes(): #1937.) Fetched by id only, as ever: the stored path is the
|
|
412
613
|
* fleet's own file, never a request's.
|
|
413
614
|
*/
|
|
414
615
|
restoreServed(entry, instance, inboxDir, webFilesDir) {
|
|
@@ -421,7 +622,7 @@ export class WebFileLedger {
|
|
|
421
622
|
let snapshot = false;
|
|
422
623
|
try {
|
|
423
624
|
own = lstatSync(entry.path);
|
|
424
|
-
if (own.isSymbolicLink() || !own.isFile() || own.size >
|
|
625
|
+
if (own.isSymbolicLink() || !own.isFile() || own.size > this.maxServedBytes())
|
|
425
626
|
return null;
|
|
426
627
|
real = realpathSync(entry.path);
|
|
427
628
|
// The directory, resolved, must be itself (not a link elsewhere); the file's real path directly under one of them.
|