@jini-ai/daemon 0.3.8 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +124 -0
- package/README.md +50 -8
- package/dist/agent-executor/contracts.d.ts +736 -0
- package/dist/agent-executor/contracts.js +28 -0
- package/dist/agent-executor/event-translation.d.ts +87 -0
- package/dist/agent-executor/event-translation.js +230 -0
- package/dist/agent-executor/events.d.ts +190 -0
- package/dist/agent-executor/events.js +848 -0
- package/dist/agent-executor/handoff.d.ts +18 -0
- package/dist/agent-executor/handoff.js +16 -0
- package/dist/agent-executor/index.d.ts +98 -0
- package/dist/agent-executor/index.js +98 -0
- package/dist/agent-executor/launch.d.ts +375 -0
- package/dist/agent-executor/launch.js +1001 -0
- package/dist/agent-executor/mcp-bridge.d.ts +338 -0
- package/dist/agent-executor/mcp-bridge.js +835 -0
- package/dist/agent-executor/values.d.ts +5 -0
- package/dist/agent-executor/values.js +13 -0
- package/dist/agent-executor.d.ts +375 -176
- package/dist/agent-executor.js +388 -335
- package/dist/close-status.d.ts +11 -8
- package/dist/close-status.js +31 -19
- package/dist/continuation/continuation-transport.d.ts +3 -1
- package/dist/continuation/continuation-transport.js +1 -1
- package/dist/continuation/journal.d.ts +12 -5
- package/dist/continuation/journal.js +4 -4
- package/dist/continuation/run-scoped-context-store.d.ts +13 -4
- package/dist/continuation/run-scoped-context-store.js +6 -6
- package/dist/continuation/run-start-handler.d.ts +1 -1
- package/dist/continuation/run-start-handler.js +3 -16
- package/dist/delegated-tool-bridge.d.ts +7 -3
- package/dist/delegated-tool-bridge.js +32 -31
- package/dist/event-log.d.ts +7 -5
- package/dist/event-log.js +12 -11
- package/dist/frontend-capability-tools.d.ts +1 -1
- package/dist/frontend-capability-tools.js +3 -2
- package/dist/frontend-session-registry.d.ts +32 -9
- package/dist/frontend-session-registry.js +12 -11
- package/dist/http/active-context.d.ts +105 -0
- package/dist/http/active-context.js +67 -0
- package/dist/http/agents.d.ts +68 -0
- package/dist/http/agents.js +12 -0
- package/dist/http/attachments.d.ts +618 -0
- package/dist/http/attachments.js +1034 -0
- package/dist/http/cancel-owned-runs.d.ts +49 -0
- package/dist/http/cancel-owned-runs.js +35 -0
- package/dist/http/component-catalog.d.ts +84 -0
- package/dist/http/component-catalog.js +64 -0
- package/dist/http/connectors.d.ts +209 -0
- package/dist/http/connectors.js +409 -0
- package/dist/http/daemon-status.d.ts +76 -0
- package/dist/http/daemon-status.js +55 -0
- package/dist/http/db-ops.d.ts +123 -0
- package/dist/http/db-ops.js +189 -0
- package/dist/http/delegated-tools.d.ts +113 -0
- package/dist/http/delegated-tools.js +230 -0
- package/dist/http/frontend-control.d.ts +136 -0
- package/dist/http/frontend-control.js +79 -0
- package/dist/http/frontend-sessions.d.ts +110 -0
- package/dist/http/frontend-sessions.js +180 -0
- package/dist/http/health.d.ts +95 -0
- package/dist/http/health.js +40 -0
- package/dist/http/host-tools.d.ts +214 -0
- package/dist/http/host-tools.js +329 -0
- package/dist/http/index.d.ts +52 -0
- package/dist/http/index.js +27 -0
- package/dist/http/media.d.ts +146 -0
- package/dist/http/media.js +346 -0
- package/dist/http/memory.d.ts +265 -0
- package/dist/http/memory.js +328 -0
- package/dist/http/model-proxy.d.ts +49 -0
- package/dist/http/model-proxy.js +642 -0
- package/dist/http/remote-run-events.d.ts +70 -0
- package/dist/http/remote-run-events.js +224 -0
- package/dist/http/research.d.ts +48 -0
- package/dist/http/research.js +315 -0
- package/dist/http/route-manifest.d.ts +35 -0
- package/dist/http/route-manifest.js +103 -0
- package/dist/http/routines.d.ts +120 -0
- package/dist/http/routines.js +298 -0
- package/dist/http/run-credentials.d.ts +73 -0
- package/dist/http/run-credentials.js +71 -0
- package/dist/http/runs.d.ts +91 -0
- package/dist/http/runs.js +221 -0
- package/dist/http/terminals.d.ts +138 -0
- package/dist/http/terminals.js +325 -0
- package/dist/http/tool-catalog.d.ts +78 -0
- package/dist/http/tool-catalog.js +66 -0
- package/dist/http/workspace-root.d.ts +77 -0
- package/dist/http/workspace-root.js +63 -0
- package/dist/http/xai.d.ts +90 -0
- package/dist/http/xai.js +655 -0
- package/dist/image-prompt-delivery.d.ts +13 -3
- package/dist/image-prompt-delivery.js +5 -5
- package/dist/index.d.ts +556 -4
- package/dist/index.js +556 -4
- package/dist/legacy-data-migration.d.ts +59 -8
- package/dist/legacy-data-migration.js +38 -33
- package/dist/read-only-tools.d.ts +130 -0
- package/dist/read-only-tools.js +115 -0
- package/dist/remote-tool-bridge.d.ts +11 -5
- package/dist/remote-tool-bridge.js +15 -15
- package/dist/routines/index.d.ts +1 -1
- package/dist/routines/index.js +1 -1
- package/dist/routines/routine-store.d.ts +34 -12
- package/dist/routines/routine-store.js +21 -20
- package/dist/routines/schedule.d.ts +48 -10
- package/dist/routines/schedule.js +21 -21
- package/dist/routines/scheduler.d.ts +32 -11
- package/dist/routines/scheduler.js +61 -57
- package/dist/routines/types.d.ts +10 -3
- package/dist/run/core/result.d.ts +6 -2
- package/dist/run/core/result.js +3 -3
- package/dist/run/core/retry.d.ts +17 -4
- package/dist/run/core/retry.js +7 -11
- package/dist/run/diagnostics/diagnostics.d.ts +14 -4
- package/dist/run/diagnostics/diagnostics.js +7 -7
- package/dist/run/index.d.ts +1 -1
- package/dist/run/index.js +1 -1
- package/dist/run-credentials.d.ts +138 -0
- package/dist/run-credentials.js +152 -0
- package/dist/run-lifecycle.d.ts +92 -33
- package/dist/run-lifecycle.js +81 -74
- package/dist/scheduler.d.ts +8 -0
- package/dist/scheduler.js +2 -0
- package/dist/session-coordination.d.ts +115 -0
- package/dist/session-coordination.js +111 -0
- package/dist/store/agent-sessions/errors.d.ts +8 -0
- package/dist/store/agent-sessions/errors.js +10 -0
- package/dist/store/agent-sessions/index.d.ts +12 -0
- package/dist/store/agent-sessions/index.js +19 -0
- package/dist/store/agent-sessions/pglite.d.ts +12 -0
- package/dist/store/agent-sessions/pglite.js +7 -0
- package/dist/store/agent-sessions/ports.d.ts +24 -0
- package/dist/store/agent-sessions/ports.js +2 -0
- package/dist/store/agent-sessions/postgres.d.ts +12 -0
- package/dist/store/agent-sessions/postgres.js +7 -0
- package/dist/store/agent-sessions/schema.d.ts +3 -0
- package/dist/store/agent-sessions/schema.js +23 -0
- package/dist/store/agent-sessions/sql.d.ts +35 -0
- package/dist/store/agent-sessions/sql.js +59 -0
- package/dist/store/agent-sessions/sqlite.d.ts +117 -0
- package/dist/store/agent-sessions/sqlite.js +128 -0
- package/dist/store/event-log/sqlite.d.ts +75 -0
- package/dist/store/event-log/sqlite.js +230 -0
- package/dist/surface-exchanges.d.ts +97 -0
- package/dist/surface-exchanges.js +153 -0
- package/dist/terminal-session.d.ts +43 -11
- package/dist/terminal-session.js +22 -20
- package/dist/tokens.js +4 -4
- package/dist/tool-audit.d.ts +114 -0
- package/dist/tool-audit.js +95 -0
- package/dist/tool-executor.d.ts +22 -7
- package/dist/tool-executor.js +16 -16
- package/dist/tool-result-media.d.ts +3 -1
- package/dist/tool-result-media.js +1 -1
- package/dist/tool-result-surfaces.d.ts +3 -1
- package/dist/tool-result-surfaces.js +1 -1
- package/package.json +127 -9
|
@@ -0,0 +1,1034 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @module attachments
|
|
3
|
+
*
|
|
4
|
+
* `POST /api/attachments`, `DELETE /api/attachments` — an HTTP route pack for user file/image
|
|
5
|
+
* uploads staged for one agent run, over a narrow, transport-owned `AttachmentStore` port plus a
|
|
6
|
+
* batteries-included disk implementation (`createDiskAttachmentStore`).
|
|
7
|
+
*
|
|
8
|
+
* **The capability this closes.** A chat composer that accepts drag-and-drop needs somewhere to put
|
|
9
|
+
* the bytes, a way to hand the agent a path it is actually allowed to read, and a guarantee that
|
|
10
|
+
* those bytes disappear when the run ends. That is three separate concerns (an upload endpoint, a
|
|
11
|
+
* trusted-metadata registry, and a run-scoped lifetime) which every host would otherwise hand-roll.
|
|
12
|
+
* This module owns all three; `@jini-ai/chat-react`'s `createDaemonAttachmentUploader` is the
|
|
13
|
+
* matching client half.
|
|
14
|
+
*
|
|
15
|
+
* **Trust model.** A renderer supplies bytes, a filename, and a batch id. None of that is trusted:
|
|
16
|
+
* - the filename is reduced to a basename over a conservative character allowlist
|
|
17
|
+
* (`sanitizeAttachmentName`) and is only ever used as *display* text plus a short extension
|
|
18
|
+
* suffix — never as the stored filename, which is a fresh `randomUUID()`;
|
|
19
|
+
* - the `kind` (`'image' | 'file'`) is sniffed from the leading bytes (`detectAttachmentKind`),
|
|
20
|
+
* never taken from a renderer-controlled MIME type or file extension;
|
|
21
|
+
* - the byte count is measured while streaming, never read from a client-supplied length;
|
|
22
|
+
* - the returned `path` is an opaque `attachment:<uuid>` capability id, not a filesystem path, so a
|
|
23
|
+
* renderer never learns where the upload root is and cannot name a file it did not upload.
|
|
24
|
+
* `claim()` is what exchanges those ids for real paths, server-side.
|
|
25
|
+
*
|
|
26
|
+
* **What `claim()` guarantees.** Exactly once per registered attachment, and only if the file is
|
|
27
|
+
* still the same file: `lstat` must report a regular non-symlink whose `dev`/`ino`/`size` match
|
|
28
|
+
* what was recorded at registration, and `realpath` must equal the recorded path (so neither the
|
|
29
|
+
* file nor any parent directory was swapped for a symlink between upload and run start). Every
|
|
30
|
+
* claimed attachment must belong to one batch, so the single `batchDirectory` a host grants the
|
|
31
|
+
* agent read access to cannot be widened by mixing batches.
|
|
32
|
+
*
|
|
33
|
+
* **Ownership is opt-in and host-attached, not self-asserted.** `register()`'s optional `ownerId`
|
|
34
|
+
* (and `listPendingForOwner`, the discovery method it enables) is never read from anything a
|
|
35
|
+
* renderer sends — a host wires `AttachmentsHttpDeps.resolveOwnerId` to pull it from a channel IT
|
|
36
|
+
* already trusts (its own session-verified principal header, say), never from `req` unauthenticated.
|
|
37
|
+
* A store with no `resolveOwnerId` wired registers every attachment ownerless, and
|
|
38
|
+
* `listPendingForOwner` never returns an ownerless record for any caller — see that method's own doc
|
|
39
|
+
* for why an absent `ownerId` must never become a wildcard match.
|
|
40
|
+
*
|
|
41
|
+
* **Storage lifetime is daemon-lifetime by default, and opt-in restart-surviving.** With
|
|
42
|
+
* `retainAcrossRestarts` left off (the default, and this pack's only behavior before that option
|
|
43
|
+
* existed), `createDiskAttachmentStore` empties its upload directory on construction: files left
|
|
44
|
+
* behind by an interrupted previous process cannot be authenticated against an in-memory registry
|
|
45
|
+
* that no longer exists, so they are removed rather than adopted. Unclaimed uploads also expire by
|
|
46
|
+
* TTL (`pruneExpired`), and a run's claimed files are deleted by `cleanupRun`.
|
|
47
|
+
*
|
|
48
|
+
* That default is wrong for any host whose process restarts while a person is still using the
|
|
49
|
+
* composer. A file-watching dev server restarting the daemon on an unrelated source edit is not a
|
|
50
|
+
* crash, but the construction wipe cannot tell the two apart, so it destroys uploads the user is
|
|
51
|
+
* about to send — real data loss, not a tidy-up. `retainAcrossRestarts: true` replaces "wipe
|
|
52
|
+
* everything, always" with "adopt exactly what can still be authenticated": see
|
|
53
|
+
* {@link prepareAttachmentStorage} for the whole mechanism and {@link loadPersistedAttachments} for
|
|
54
|
+
* why an adopted record is no less trusted than one this process registered itself.
|
|
55
|
+
*
|
|
56
|
+
* **This pack does not auto-wire itself into a run's lifecycle**, because no generic hook for that
|
|
57
|
+
* exists — the same deliberate choice `@jini-ai/daemon`'s `createRunScopedContextStore` makes. A
|
|
58
|
+
* host claims in its own `onRunStarted` and cleans up in a `finally`, roughly:
|
|
59
|
+
*
|
|
60
|
+
* ```ts
|
|
61
|
+
* const store = await createDiskAttachmentStore({ uploadDirectory });
|
|
62
|
+
* // ... httpExtensions: [(app, { adapter }) => registerAttachmentRoutes({ app, deps: { store }, adapter })]
|
|
63
|
+
* onRunStarted: (context) => {
|
|
64
|
+
* void (async () => {
|
|
65
|
+
* try {
|
|
66
|
+
* const claimed = await store.claim({ attachments: attachmentRefsFrom(context.request), runId: context.run.id });
|
|
67
|
+
* await executor.run({
|
|
68
|
+
* runId: context.run.id,
|
|
69
|
+
* // ... prompt, cwd, agentId
|
|
70
|
+
* }, {
|
|
71
|
+
* ...(claimed.batchDirectory === undefined ? {} : {
|
|
72
|
+
* imagePaths: claimed.attachments.filter((a) => a.kind === 'image').map((a) => a.path),
|
|
73
|
+
* extraAllowedDirs: [claimed.batchDirectory],
|
|
74
|
+
* uploadRoot: claimed.batchDirectory,
|
|
75
|
+
* }),
|
|
76
|
+
* });
|
|
77
|
+
* } finally {
|
|
78
|
+
* await store.cleanupRun({ runId: context.run.id });
|
|
79
|
+
* }
|
|
80
|
+
* })();
|
|
81
|
+
* }
|
|
82
|
+
* ```
|
|
83
|
+
*
|
|
84
|
+
* `imagePaths`/`extraAllowedDirs`/`uploadRoot` are pre-existing `AgentExecutor.run()` options; this
|
|
85
|
+
* module only produces real values for them.
|
|
86
|
+
*
|
|
87
|
+
* **Body-parser ordering (`POST` reads the raw request stream).** The upload route streams
|
|
88
|
+
* `request` straight to disk, so any body-parsing middleware that has already consumed the stream
|
|
89
|
+
* leaves nothing to write. The classic way to hit this is a JSON body parser mounted app-wide: a
|
|
90
|
+
* user drops a `.json` file, the browser sets `content-type: application/json`, and
|
|
91
|
+
* `express.json()` eats the body — which without a guard shows up as the deeply unhelpful
|
|
92
|
+
* "attachment is empty". `registerAttachmentRoutes` detects an already-consumed stream up front and
|
|
93
|
+
* reports it as a host misconfiguration (`'attachment-body-consumed'`) instead of failing silently.
|
|
94
|
+
* Mount this pack before any global body parser, or scope the parser so it skips this path.
|
|
95
|
+
*
|
|
96
|
+
* **SEC-005 redaction**: a store failure can carry filesystem paths a caller must not see, so
|
|
97
|
+
* anything that is not an explicitly-classified `AttachmentRejectedError` becomes a
|
|
98
|
+
* correlation-id-bearing generic `INTERNAL_ERROR`, with the real error reaching a host-owned sink —
|
|
99
|
+
* the same `reportInternalError` shape `media.ts`/`delegated-tools.ts` use.
|
|
100
|
+
*/
|
|
101
|
+
import { chmod, lstat, mkdir, open, readFile, readdir, realpath, rm, rmdir, writeFile } from 'node:fs/promises';
|
|
102
|
+
import { randomUUID } from 'node:crypto';
|
|
103
|
+
import { basename, dirname, extname, resolve } from 'node:path';
|
|
104
|
+
import { createApiError } from '@jini-ai/protocol';
|
|
105
|
+
import { guardSameOrigin } from '@jini-ai/http-kit';
|
|
106
|
+
import { sendApiError, sendJson } from '@jini-ai/http-kit';
|
|
107
|
+
/**
|
|
108
|
+
* A refusal this module classified itself, as opposed to an unexpected filesystem/programming
|
|
109
|
+
* error. The route pack turns the `reason` into a status code and lets the `message` through only
|
|
110
|
+
* for caller-safe rejections below 500. Classified server-side failures and unexpected errors are
|
|
111
|
+
* redacted to a generic `INTERNAL_ERROR`, preserving the server-state disclosure boundary.
|
|
112
|
+
*/
|
|
113
|
+
export class AttachmentRejectedError extends Error {
|
|
114
|
+
reason;
|
|
115
|
+
constructor({ reason, message }, _optional = {}) {
|
|
116
|
+
super(message);
|
|
117
|
+
this.name = 'AttachmentRejectedError';
|
|
118
|
+
this.reason = reason;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
/** The subset of `AttachmentRejectionReason` a caller is allowed to see a real message for. */
|
|
122
|
+
const REJECTION_STATUS = {
|
|
123
|
+
'invalid-batch': 400,
|
|
124
|
+
'attachment-too-large': 413,
|
|
125
|
+
'batch-count-exceeded': 413,
|
|
126
|
+
'batch-too-large': 413,
|
|
127
|
+
'storage-full': 413,
|
|
128
|
+
'empty-attachment': 400,
|
|
129
|
+
'too-many-concurrent-uploads': 429,
|
|
130
|
+
'invalid-cleanup-request': 400,
|
|
131
|
+
// The remaining reasons describe a broken or hostile server-side state. They are never given a
|
|
132
|
+
// real message over HTTP (see `respondToUploadFailure`); the entries exist so a host catching a
|
|
133
|
+
// rejection from `claim()` outside HTTP can still classify it.
|
|
134
|
+
'attachment-body-consumed': 500,
|
|
135
|
+
'attachment-integrity': 500,
|
|
136
|
+
'too-many-attachments': 400,
|
|
137
|
+
'duplicate-attachment': 400,
|
|
138
|
+
'attachment-unknown-or-claimed': 400,
|
|
139
|
+
'mixed-batch': 400,
|
|
140
|
+
};
|
|
141
|
+
/** `ApiError` code per refusal, so `sendApiError` produces the standard envelope. */
|
|
142
|
+
function apiErrorForRejection(error) {
|
|
143
|
+
const status = REJECTION_STATUS[error.reason];
|
|
144
|
+
if (status === 413)
|
|
145
|
+
return createApiError({ code: 'PAYLOAD_TOO_LARGE', message: error.message });
|
|
146
|
+
if (status === 429)
|
|
147
|
+
return createApiError({ code: 'RATE_LIMITED', message: error.message });
|
|
148
|
+
return createApiError({ code: 'BAD_REQUEST', message: error.message });
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* Accepted batch id shape. Deliberately narrow — long enough for a UUID, and admitting no `.`,
|
|
152
|
+
* `/`, or `\`, which is what makes `resolve(uploadRoot, batchId)` provably a single
|
|
153
|
+
* non-traversing path segment.
|
|
154
|
+
*/
|
|
155
|
+
const BATCH_ID_PATTERN = /^[a-zA-Z0-9-]{8,80}$/u;
|
|
156
|
+
/**
|
|
157
|
+
* `true` when `body`'s bytes 4-7 spell `ftyp` — the ISO-BMFF box tag AVIF, HEIC, and MP4 all
|
|
158
|
+
* share. This says only "some member of the ISO-BMFF family", never "MP4" or "AVIF" on its own:
|
|
159
|
+
* mirrors the CMS package's `content-type-sniffer.ts` `isIsoBmffFtyp`, which documents the same
|
|
160
|
+
* fact for the media route's own sniffer. `detectAttachmentKind` never returns `'image'` for this
|
|
161
|
+
* tag alone (see that function's doc) — doing so is exactly the 2026-09-06 regression that made an
|
|
162
|
+
* AVIF render as an unplayable `<video>` when the *media* sniffer briefly had the same bug.
|
|
163
|
+
*/
|
|
164
|
+
function hasIsoBmffFtypTag(body) {
|
|
165
|
+
return body.length >= 8 && new TextDecoder().decode(body.slice(4, 8)) === 'ftyp';
|
|
166
|
+
}
|
|
167
|
+
/** ISO-BMFF brands that identify AVIF: a still image (`avif`) or an image sequence (`avis`). Kept
|
|
168
|
+
* identical to `content-type-sniffer.ts`'s `AVIF_BRANDS` so the two sniffers can never disagree on
|
|
169
|
+
* which brands mean AVIF. */
|
|
170
|
+
const AVIF_FTYP_BRANDS = ['avif', 'avis'];
|
|
171
|
+
const FTYP_MAJOR_BRAND_OFFSET = 8;
|
|
172
|
+
/** Compatible brands follow the major brand (offset 8) and 4-byte minor version (offset 12). */
|
|
173
|
+
const FTYP_COMPATIBLE_BRANDS_OFFSET = 16;
|
|
174
|
+
/** Bound on the compatible-brand list scan — real `ftyp` boxes carry a handful, never dozens. */
|
|
175
|
+
const MAX_SCANNED_COMPATIBLE_BRANDS = 16;
|
|
176
|
+
function hasAvifBrandAt(body, offset) {
|
|
177
|
+
return body.length >= offset + 4
|
|
178
|
+
&& AVIF_FTYP_BRANDS.some((brand) => new TextDecoder().decode(body.slice(offset, offset + 4)) === brand);
|
|
179
|
+
}
|
|
180
|
+
/** The `ftyp` box's declared end offset (big-endian size at bytes 0-3), clamped to the bytes
|
|
181
|
+
* actually present so a truncated or size-inflating file can never push the brand scan past the
|
|
182
|
+
* captured signature window. */
|
|
183
|
+
function ftypBoxEnd(body) {
|
|
184
|
+
if (body.length < FTYP_MAJOR_BRAND_OFFSET)
|
|
185
|
+
return 0;
|
|
186
|
+
const declared = ((body[0] << 24) | (body[1] << 16) | (body[2] << 8) | body[3]) >>> 0;
|
|
187
|
+
return Math.min(declared, body.length);
|
|
188
|
+
}
|
|
189
|
+
/**
|
|
190
|
+
* `true` when `body` opens with an ISO-BMFF `ftyp` box whose major brand or compatible-brand list
|
|
191
|
+
* names AVIF. This is a line-for-line mirror of the CMS package's `content-type-sniffer.ts`
|
|
192
|
+
* `isAvif` (no shared dependency exists between HTTP kit and the CMS package today — see
|
|
193
|
+
* this module's handoff notes on whether extracting one is worth it): the major brand alone is not
|
|
194
|
+
* sufficient, because a great many real AVIF files declare the generic `mif1` (HEIF image) major
|
|
195
|
+
* brand and name `avif` only in the compatible-brand list, so both are checked. HEIC declares
|
|
196
|
+
* neither and is therefore never matched here — deliberate; nothing downstream of this attachment
|
|
197
|
+
* store has confirmed it can render a HEIC as an `<img>`, so labelling one `'image'` would risk the
|
|
198
|
+
* same "sniffed as a type the surface cannot actually display" failure this whole fix addresses.
|
|
199
|
+
*
|
|
200
|
+
* Must run before any generic `ftyp` check: {@link hasIsoBmffFtypTag} alone cannot tell AVIF, HEIC,
|
|
201
|
+
* and MP4 apart, so a brand-blind caller would misclassify every ISO-BMFF file it sees as one type.
|
|
202
|
+
*/
|
|
203
|
+
export function hasAvifSignature({ body }, _optional = {}) {
|
|
204
|
+
if (!hasIsoBmffFtypTag(body))
|
|
205
|
+
return false;
|
|
206
|
+
if (hasAvifBrandAt(body, FTYP_MAJOR_BRAND_OFFSET))
|
|
207
|
+
return true;
|
|
208
|
+
const boxEnd = ftypBoxEnd(body);
|
|
209
|
+
for (let index = 0; index < MAX_SCANNED_COMPATIBLE_BRANDS; index++) {
|
|
210
|
+
const offset = FTYP_COMPATIBLE_BRANDS_OFFSET + index * 4;
|
|
211
|
+
if (offset + 4 > boxEnd)
|
|
212
|
+
return false;
|
|
213
|
+
if (hasAvifBrandAt(body, offset))
|
|
214
|
+
return true;
|
|
215
|
+
}
|
|
216
|
+
return false;
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* Bytes of leading signature `detectAttachmentKind` needs. WEBP's marker ends at byte 12; AVIF
|
|
220
|
+
* needs more — {@link hasAvifSignature}'s compatible-brand scan can read as far as
|
|
221
|
+
* `FTYP_COMPATIBLE_BRANDS_OFFSET + MAX_SCANNED_COMPATIBLE_BRANDS * 4` (a great many real AVIF
|
|
222
|
+
* files declare the generic `mif1` major brand and name `avif` only in the compatible-brand list —
|
|
223
|
+
* see that function's doc), so this is sized to that worst case rather than to the smallest format.
|
|
224
|
+
*/
|
|
225
|
+
const SIGNATURE_BYTES = FTYP_COMPATIBLE_BRANDS_OFFSET + MAX_SCANNED_COMPATIBLE_BRANDS * 4;
|
|
226
|
+
/**
|
|
227
|
+
* `true` when the file on disk is still the same file registration accepted.
|
|
228
|
+
*
|
|
229
|
+
* Every condition is a distinct real attack: a regular file swapped for a symlink or directory, a
|
|
230
|
+
* parent directory swapped for a symlink pointing elsewhere, the file replaced by a different file
|
|
231
|
+
* at the same path (new inode), the same inode truncated or appended to (new size), or the path
|
|
232
|
+
* now resolving onto a different device.
|
|
233
|
+
*
|
|
234
|
+
* Pure and exported on purpose. Staging a *device* change for a file that keeps its path is not
|
|
235
|
+
* something a test can do on a real filesystem, and a check that can only be exercised in
|
|
236
|
+
* production is a check nobody knows works — a fake identity states each case directly.
|
|
237
|
+
*/
|
|
238
|
+
export function isUnchangedAttachment({ recorded, observed }, _optional = {}) {
|
|
239
|
+
return observed.isRegularFile
|
|
240
|
+
&& observed.canonicalPath === recorded.filePath
|
|
241
|
+
&& observed.dev === recorded.dev
|
|
242
|
+
&& observed.ino === recorded.ino
|
|
243
|
+
&& observed.size === recorded.size;
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* Phase 1 of `claim()`: synchronously reserves `attachments` for `runId`, or throws (releasing
|
|
247
|
+
* whatever it already reserved this call) the first time a requested attachment turns out to be
|
|
248
|
+
* unknown or already claimed.
|
|
249
|
+
*
|
|
250
|
+
* The reservation loop deliberately has no `await`: this store's exactly-once guarantee is what
|
|
251
|
+
* stops two runs being handed the same real path on disk, and `claim` is reachable concurrently
|
|
252
|
+
* (two run starts, one shared attachment). Nothing between `records.get` and the assignment of
|
|
253
|
+
* `record.claimedRunId` may ever become asynchronous — that window is exactly where a concurrent
|
|
254
|
+
* call would get its turn and could observe the same record as still unclaimed.
|
|
255
|
+
*
|
|
256
|
+
* Exported so this invariant can be exercised directly against a plain `Map` of fabricated
|
|
257
|
+
* records, without going through a disk-backed store.
|
|
258
|
+
*/
|
|
259
|
+
export function reserveAttachmentRecords({ attachments, records, runId, maxAttachments }, _optional = {}) {
|
|
260
|
+
if (attachments.length > maxAttachments) {
|
|
261
|
+
throw new AttachmentRejectedError({ reason: 'too-many-attachments', message: 'Too many attachments' });
|
|
262
|
+
}
|
|
263
|
+
const requestedPaths = new Set(attachments.map((attachment) => attachment.path));
|
|
264
|
+
if (requestedPaths.size !== attachments.length) {
|
|
265
|
+
throw new AttachmentRejectedError({ reason: 'duplicate-attachment', message: 'Duplicate attachment' });
|
|
266
|
+
}
|
|
267
|
+
const claimed = [];
|
|
268
|
+
for (const requested of attachments) {
|
|
269
|
+
const record = records.get(requested.path);
|
|
270
|
+
if (!record || record.claimedRunId !== undefined) {
|
|
271
|
+
for (const reserved of claimed)
|
|
272
|
+
delete reserved.claimedRunId;
|
|
273
|
+
throw new AttachmentRejectedError({ reason: 'attachment-unknown-or-claimed', message: 'Attachment is unknown or already claimed' });
|
|
274
|
+
}
|
|
275
|
+
record.claimedRunId = runId;
|
|
276
|
+
claimed.push(record);
|
|
277
|
+
}
|
|
278
|
+
return claimed;
|
|
279
|
+
}
|
|
280
|
+
/**
|
|
281
|
+
* Phase 2 of `claim()`: re-verifies every already-reserved record against the filesystem — not
|
|
282
|
+
* merely re-read, but re-checked against what registration recorded, so someone able to write into
|
|
283
|
+
* the batch directory between upload and run start cannot get the agent to read a file of their
|
|
284
|
+
* choosing — and checks every claimed attachment shares one batch. Returns the shared batch
|
|
285
|
+
* directory on success (or `''` for an empty `claimed`, which `claim()` never passes in — it
|
|
286
|
+
* returns before calling this — but which a direct caller can still treat as "no batch").
|
|
287
|
+
*
|
|
288
|
+
* Throws without releasing `claimed`'s reservations; releasing is the caller's job (`claim`'s own
|
|
289
|
+
* `catch`), so a rejected claim still leaves nothing half-claimed and the caller can retry with a
|
|
290
|
+
* corrected set.
|
|
291
|
+
*
|
|
292
|
+
* Exported so the integrity and mixed-batch checks can be exercised directly against fabricated
|
|
293
|
+
* records over real files on disk, without a full `createDiskAttachmentStore` around them.
|
|
294
|
+
*/
|
|
295
|
+
export async function verifyClaimedAttachments({ claimed }, _optional = {}) {
|
|
296
|
+
let batchDirectory = '';
|
|
297
|
+
for (const [index, record] of claimed.entries()) {
|
|
298
|
+
const info = await lstat(record.filePath);
|
|
299
|
+
const canonicalPath = await realpath(record.filePath);
|
|
300
|
+
if (!isUnchangedAttachment({
|
|
301
|
+
recorded: record, observed: {
|
|
302
|
+
isRegularFile: info.isFile(),
|
|
303
|
+
dev: info.dev,
|
|
304
|
+
ino: info.ino,
|
|
305
|
+
size: info.size,
|
|
306
|
+
canonicalPath,
|
|
307
|
+
}
|
|
308
|
+
})) {
|
|
309
|
+
throw new AttachmentRejectedError({ reason: 'attachment-integrity', message: 'Attachment changed after upload' });
|
|
310
|
+
}
|
|
311
|
+
if (index > 0 && record.batchDirectory !== batchDirectory) {
|
|
312
|
+
throw new AttachmentRejectedError({ reason: 'mixed-batch', message: 'Attachments must belong to one batch' });
|
|
313
|
+
}
|
|
314
|
+
batchDirectory = record.batchDirectory;
|
|
315
|
+
}
|
|
316
|
+
return batchDirectory;
|
|
317
|
+
}
|
|
318
|
+
/** Renders a byte cap the way a person would read it, for a message a user actually sees. */
|
|
319
|
+
function formatByteLimit(bytes) {
|
|
320
|
+
const megabytes = bytes / (1024 * 1024);
|
|
321
|
+
return Number.isInteger(megabytes) ? `${megabytes} MB` : `${bytes} bytes`;
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Reduces an untrusted upload filename to a display-only basename over a conservative allowlist.
|
|
325
|
+
* Never used as the name of the stored file — see this module's trust-model doc.
|
|
326
|
+
*/
|
|
327
|
+
export function sanitizeAttachmentName({ requestedName }, _optional = {}) {
|
|
328
|
+
if (typeof requestedName !== 'string')
|
|
329
|
+
return 'attachment';
|
|
330
|
+
return basename(requestedName).replaceAll(/[^a-zA-Z0-9._ -]/gu, '_') || 'attachment';
|
|
331
|
+
}
|
|
332
|
+
/** `true` when `body`'s first 8 bytes are the PNG signature. */
|
|
333
|
+
export function hasPngSignature({ body }, _optional = {}) {
|
|
334
|
+
return body.length >= 8
|
|
335
|
+
&& body[0] === 0x89
|
|
336
|
+
&& body[1] === 0x50
|
|
337
|
+
&& body[2] === 0x4e
|
|
338
|
+
&& body[3] === 0x47;
|
|
339
|
+
}
|
|
340
|
+
/** `true` when `body`'s first 3 bytes are the JPEG start-of-image marker. */
|
|
341
|
+
export function hasJpegSignature({ body }, _optional = {}) {
|
|
342
|
+
return body.length >= 3
|
|
343
|
+
&& body[0] === 0xff
|
|
344
|
+
&& body[1] === 0xd8
|
|
345
|
+
&& body[2] === 0xff;
|
|
346
|
+
}
|
|
347
|
+
/** `true` when `body`'s first 6 bytes spell either GIF version tag. */
|
|
348
|
+
export function hasGifSignature({ body }, _optional = {}) {
|
|
349
|
+
const signature = new TextDecoder().decode(body.slice(0, 6));
|
|
350
|
+
return signature === 'GIF87a' || signature === 'GIF89a';
|
|
351
|
+
}
|
|
352
|
+
/** `true` when `body` opens with a RIFF container whose form type is WEBP. */
|
|
353
|
+
export function hasWebpSignature({ body }, _optional = {}) {
|
|
354
|
+
return body.length >= 12
|
|
355
|
+
&& new TextDecoder().decode(body.slice(0, 4)) === 'RIFF'
|
|
356
|
+
&& new TextDecoder().decode(body.slice(8, 12)) === 'WEBP';
|
|
357
|
+
}
|
|
358
|
+
/** Every recognized image signature, checked in this order until one matches. */
|
|
359
|
+
const IMAGE_SIGNATURE_MATCHERS = [
|
|
360
|
+
hasPngSignature,
|
|
361
|
+
hasJpegSignature,
|
|
362
|
+
hasGifSignature,
|
|
363
|
+
hasWebpSignature,
|
|
364
|
+
hasAvifSignature,
|
|
365
|
+
];
|
|
366
|
+
/**
|
|
367
|
+
* Infers `'image'` from the leading bytes rather than from a renderer-controlled MIME type or file
|
|
368
|
+
* extension. PNG, JPEG, GIF87a/89a, WEBP, and AVIF (brand-checked ISO-BMFF, see
|
|
369
|
+
* {@link hasAvifSignature}) are recognized; everything else is `'file'` — including a plain MP4 or
|
|
370
|
+
* other non-AVIF ISO-BMFF file, which shares AVIF's `ftyp` tag but is deliberately NOT matched here
|
|
371
|
+
* (see {@link hasIsoBmffFtypTag}'s doc for why a brand-blind `ftyp` check would be wrong).
|
|
372
|
+
*
|
|
373
|
+
* `kind` decides whether a path is later passed to `AgentExecutor.run()`'s `imagePaths`, so letting
|
|
374
|
+
* a renderer assert it would let a renderer choose how the agent runtime parses the bytes.
|
|
375
|
+
*/
|
|
376
|
+
export function detectAttachmentKind({ body }, _optional = {}) {
|
|
377
|
+
return IMAGE_SIGNATURE_MATCHERS.some((matchesSignature) => matchesSignature({ body })) ? 'image' : 'file';
|
|
378
|
+
}
|
|
379
|
+
/**
|
|
380
|
+
* Streams a request body straight to a private file under a hard byte cap, keeping only the
|
|
381
|
+
* leading signature bytes in memory so an upload never costs memory proportional to its size.
|
|
382
|
+
*
|
|
383
|
+
* Opened `wx`, so this can never overwrite an existing file. A partial write is removed before the
|
|
384
|
+
* rejection propagates: the cap is enforced *during* the stream, which necessarily means some bytes
|
|
385
|
+
* already reached disk by the time it trips.
|
|
386
|
+
*/
|
|
387
|
+
export async function writeBoundedAttachmentBody(requiredArgs, optionalArgs = {}) {
|
|
388
|
+
const { request, filePath, maxBytes, mode = 0o600, } = { ...requiredArgs, ...optionalArgs };
|
|
389
|
+
const handle = await open(filePath, 'wx', mode);
|
|
390
|
+
let total = 0;
|
|
391
|
+
let signature = Buffer.alloc(0);
|
|
392
|
+
try {
|
|
393
|
+
for await (const chunk of request) {
|
|
394
|
+
const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk));
|
|
395
|
+
total += buffer.byteLength;
|
|
396
|
+
if (total > maxBytes) {
|
|
397
|
+
throw new AttachmentRejectedError({ reason: 'attachment-too-large', message: `Each attachment must be ${formatByteLimit(maxBytes)} or smaller` });
|
|
398
|
+
}
|
|
399
|
+
if (signature.byteLength < SIGNATURE_BYTES) {
|
|
400
|
+
signature = Buffer.concat([signature, buffer.subarray(0, SIGNATURE_BYTES - signature.byteLength)], Math.min(SIGNATURE_BYTES, signature.byteLength + buffer.byteLength));
|
|
401
|
+
}
|
|
402
|
+
await handle.write(buffer);
|
|
403
|
+
}
|
|
404
|
+
await handle.close();
|
|
405
|
+
await chmod(filePath, mode);
|
|
406
|
+
return { size: total, signature };
|
|
407
|
+
}
|
|
408
|
+
catch (error) {
|
|
409
|
+
// Reached both while the handle is still open (the byte cap tripped, or the request stream
|
|
410
|
+
// errored) and after it was already closed (`chmod` failed). No `.catch` is needed to tell those
|
|
411
|
+
// apart: Node's `FileHandle.close()` is idempotent — closing an already-closed handle resolves
|
|
412
|
+
// — so this can only reject for a handle that is genuinely still open and unclosable, which is
|
|
413
|
+
// a failure worth surfacing rather than swallowing.
|
|
414
|
+
await handle.close();
|
|
415
|
+
await rm(filePath, { force: true });
|
|
416
|
+
throw error;
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
// ---------------------------------------------------------------------------
|
|
420
|
+
// Restart survival (`retainAcrossRestarts`)
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
422
|
+
/**
|
|
423
|
+
* Directory, directly under the upload root, holding one small JSON sidecar per registered
|
|
424
|
+
* attachment. This is what makes a restart-surviving store possible at all: the upload path encodes
|
|
425
|
+
* the batch id and nothing else, so a directory scan alone can never recover an attachment's
|
|
426
|
+
* display `name`, its sniffed `kind`, its capability `id`, or — the one that decides whether
|
|
427
|
+
* `listPendingForOwner` can see it again — its `ownerId`. A scan-only "reconstruct from the paths"
|
|
428
|
+
* design therefore brings the *bytes* back and still shows the user an empty list, which is half a
|
|
429
|
+
* fix.
|
|
430
|
+
*
|
|
431
|
+
* The leading `.` is load-bearing: `BATCH_ID_PATTERN` admits no `.`, so `resolveBatchDirectory` can
|
|
432
|
+
* never produce this path and a batch can never collide with (or be made to write into) the sidecar
|
|
433
|
+
* directory. It also sits OUTSIDE every batch directory on purpose — a batch directory is the one
|
|
434
|
+
* directory a host grants the agent read access to (`AttachmentClaim.batchDirectory`), so sidecars
|
|
435
|
+
* kept next to their files would hand a run the `ownerId` and capability ids of every other
|
|
436
|
+
* attachment in the same batch.
|
|
437
|
+
*/
|
|
438
|
+
const SIDECAR_DIRECTORY_NAME = '.records';
|
|
439
|
+
/** Field types a sidecar must carry, checked as data so the validator stays one flat loop. */
|
|
440
|
+
const PERSISTED_FIELD_TYPES = {
|
|
441
|
+
id: 'string',
|
|
442
|
+
filePath: 'string',
|
|
443
|
+
name: 'string',
|
|
444
|
+
kind: 'string',
|
|
445
|
+
size: 'number',
|
|
446
|
+
batchId: 'string',
|
|
447
|
+
dev: 'number',
|
|
448
|
+
ino: 'number',
|
|
449
|
+
createdAt: 'number',
|
|
450
|
+
};
|
|
451
|
+
function hasPersistedFieldTypes(candidate) {
|
|
452
|
+
return Object.entries(PERSISTED_FIELD_TYPES)
|
|
453
|
+
.every(([field, type]) => typeof candidate[field] === type);
|
|
454
|
+
}
|
|
455
|
+
function isPersistedAttachmentShape(raw) {
|
|
456
|
+
if (typeof raw !== 'object' || raw === null)
|
|
457
|
+
return false;
|
|
458
|
+
const candidate = raw;
|
|
459
|
+
if (!hasPersistedFieldTypes(candidate))
|
|
460
|
+
return false;
|
|
461
|
+
if (candidate.ownerId !== undefined && typeof candidate.ownerId !== 'string')
|
|
462
|
+
return false;
|
|
463
|
+
return candidate.kind === 'image' || candidate.kind === 'file';
|
|
464
|
+
}
|
|
465
|
+
/**
|
|
466
|
+
* Turns one sidecar's parsed JSON back into an `AttachmentRecord`, or `undefined` for anything that
|
|
467
|
+
* is not a well-formed record naming a file inside its own batch directory.
|
|
468
|
+
*
|
|
469
|
+
* `batchDirectory` is re-derived here and `filePath`'s containment is re-checked against it with
|
|
470
|
+
* the same parent-equality test `register` uses — the two checks that make an adopted record no
|
|
471
|
+
* more trusted than one this process registered itself. Pure and exported so both can be exercised
|
|
472
|
+
* against hostile input directly, without staging files on disk.
|
|
473
|
+
*/
|
|
474
|
+
export function parsePersistedAttachment({ raw, canonicalUploadDirectory }, _optional = {}) {
|
|
475
|
+
if (!isPersistedAttachmentShape(raw))
|
|
476
|
+
return undefined;
|
|
477
|
+
if (!BATCH_ID_PATTERN.test(raw.batchId))
|
|
478
|
+
return undefined;
|
|
479
|
+
const batchDirectory = resolve(canonicalUploadDirectory, raw.batchId);
|
|
480
|
+
const filePath = resolve(raw.filePath);
|
|
481
|
+
if (dirname(filePath) !== batchDirectory)
|
|
482
|
+
return undefined;
|
|
483
|
+
return {
|
|
484
|
+
id: raw.id,
|
|
485
|
+
filePath,
|
|
486
|
+
name: raw.name,
|
|
487
|
+
kind: raw.kind,
|
|
488
|
+
size: raw.size,
|
|
489
|
+
batchId: raw.batchId,
|
|
490
|
+
batchDirectory,
|
|
491
|
+
dev: raw.dev,
|
|
492
|
+
ino: raw.ino,
|
|
493
|
+
createdAt: raw.createdAt,
|
|
494
|
+
...(raw.ownerId === undefined ? {} : { ownerId: raw.ownerId }),
|
|
495
|
+
};
|
|
496
|
+
}
|
|
497
|
+
/**
|
|
498
|
+
* Sidecar filename for a record id. Reduces the id to the same `[a-zA-Z0-9-]` allowlist
|
|
499
|
+
* `BATCH_ID_PATTERN` uses before appending the extension, so no id — however this store's own id
|
|
500
|
+
* format later changes — can put a `/`, a `..`, or a second extension into the path. Ids are
|
|
501
|
+
* `attachment:<uuid>`, so the reduction stays injective in practice: the UUID is what makes it
|
|
502
|
+
* unique and the UUID survives unchanged.
|
|
503
|
+
*/
|
|
504
|
+
export function attachmentSidecarFileName({ id }, _optional = {}) {
|
|
505
|
+
return `${id.replaceAll(/[^a-zA-Z0-9-]/gu, '_')}.json`;
|
|
506
|
+
}
|
|
507
|
+
async function readPersistedAttachment(sidecarPath, canonicalUploadDirectory) {
|
|
508
|
+
try {
|
|
509
|
+
const raw = JSON.parse(await readFile(sidecarPath, 'utf8'));
|
|
510
|
+
return parsePersistedAttachment({ raw, canonicalUploadDirectory });
|
|
511
|
+
}
|
|
512
|
+
catch {
|
|
513
|
+
// Unreadable, truncated by a kill mid-write, or not JSON at all. Indistinguishable from a
|
|
514
|
+
// sidecar that never described a real attachment, and handled identically: not adopted.
|
|
515
|
+
return undefined;
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
/**
|
|
519
|
+
* `true` when the file this record describes is still byte-for-byte the file registration accepted.
|
|
520
|
+
*
|
|
521
|
+
* Deliberately `verifyClaimedAttachments` rather than a looser existence check: adoption must apply
|
|
522
|
+
* the SAME `dev`/`ino`/`size`/`realpath` gate `claim()` applies, or a restart would become the one
|
|
523
|
+
* moment at which a file swapped underneath the store gets handed to an agent anyway.
|
|
524
|
+
*/
|
|
525
|
+
async function isAdoptableAttachment(record) {
|
|
526
|
+
try {
|
|
527
|
+
await verifyClaimedAttachments({ claimed: [record] });
|
|
528
|
+
return true;
|
|
529
|
+
}
|
|
530
|
+
catch {
|
|
531
|
+
return false;
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* Rebuilds the record map from the sidecar directory, adopting only records that are (a) well
|
|
536
|
+
* formed and contained, (b) still within the retention window, and (c) still backed by the exact
|
|
537
|
+
* file registration accepted. Anything failing any of those is deleted — sidecar first, then the
|
|
538
|
+
* file it named, and the file ONLY when containment passed, so an unparseable sidecar can never
|
|
539
|
+
* make this an arbitrary-file-delete primitive for whatever the daemon can unlink.
|
|
540
|
+
*
|
|
541
|
+
* **The retention consequence, stated rather than assumed harmless.** Because `claimedRunId` is not
|
|
542
|
+
* persisted (see {@link PersistedAttachmentRecord}), a file claimed by a run that died with the
|
|
543
|
+
* process is adopted as unclaimed instead of being wiped, so it now occupies disk and quota for up
|
|
544
|
+
* to `retentionMs` rather than until the next start. Bounded by the same `maxStoredAttachments` /
|
|
545
|
+
* `maxStoredBytes` quotas as any live upload, and cleared by the `pruneExpired` every upload
|
|
546
|
+
* already runs — but it is a real change in when that disk comes back.
|
|
547
|
+
*
|
|
548
|
+
* @complexity O(n) in sidecars present, each costing one read and one `lstat`/`realpath` pair.
|
|
549
|
+
*/
|
|
550
|
+
export async function loadPersistedAttachments(requiredArgs, optionalArgs = {}) {
|
|
551
|
+
const { canonicalUploadDirectory, retentionMs, now = Date.now(), } = { ...requiredArgs, ...optionalArgs };
|
|
552
|
+
const sidecarDirectory = resolve(canonicalUploadDirectory, SIDECAR_DIRECTORY_NAME);
|
|
553
|
+
await mkdir(sidecarDirectory, { recursive: true, mode: 0o700 });
|
|
554
|
+
const adopted = new Map();
|
|
555
|
+
for (const entry of await readdir(sidecarDirectory)) {
|
|
556
|
+
const sidecarPath = resolve(sidecarDirectory, entry);
|
|
557
|
+
const record = await readPersistedAttachment(sidecarPath, canonicalUploadDirectory);
|
|
558
|
+
if (record && now - record.createdAt < retentionMs && await isAdoptableAttachment(record)) {
|
|
559
|
+
adopted.set(record.id, record);
|
|
560
|
+
continue;
|
|
561
|
+
}
|
|
562
|
+
await rm(sidecarPath, { force: true });
|
|
563
|
+
if (record)
|
|
564
|
+
await rm(record.filePath, { force: true });
|
|
565
|
+
}
|
|
566
|
+
return adopted;
|
|
567
|
+
}
|
|
568
|
+
/** Deletes everything in one batch directory that no adopted record names, then tidies the
|
|
569
|
+
* directory away if that emptied it. A root entry that is not a directory at all is removed
|
|
570
|
+
* outright — the store never creates one, so it can only be debris. */
|
|
571
|
+
async function removeUnadoptedBatch(batchDirectory, keep) {
|
|
572
|
+
const entries = await readdir(batchDirectory).catch(() => undefined);
|
|
573
|
+
if (entries === undefined) {
|
|
574
|
+
await rm(batchDirectory, { force: true });
|
|
575
|
+
return;
|
|
576
|
+
}
|
|
577
|
+
for (const entry of entries) {
|
|
578
|
+
const filePath = resolve(batchDirectory, entry);
|
|
579
|
+
if (!keep.has(filePath))
|
|
580
|
+
await rm(filePath, { recursive: true, force: true });
|
|
581
|
+
}
|
|
582
|
+
await rmdir(batchDirectory).catch(() => undefined);
|
|
583
|
+
}
|
|
584
|
+
/**
|
|
585
|
+
* Removes every upload the sidecar pass did not adopt. Without this, a file whose sidecar was lost
|
|
586
|
+
* (or was never written, because it predates this option) would survive forever with nothing
|
|
587
|
+
* tracking it — trading the wipe's data loss for unbounded disk growth, which is not a trade worth
|
|
588
|
+
* making. This is what keeps "never wipe on construction" bounded.
|
|
589
|
+
*/
|
|
590
|
+
export async function removeUnadoptedUploads({ canonicalUploadDirectory, adopted }, _optional = {}) {
|
|
591
|
+
const keep = new Set([...adopted.values()].map((record) => record.filePath));
|
|
592
|
+
for (const entry of await readdir(canonicalUploadDirectory)) {
|
|
593
|
+
if (entry === SIDECAR_DIRECTORY_NAME)
|
|
594
|
+
continue;
|
|
595
|
+
await removeUnadoptedBatch(resolve(canonicalUploadDirectory, entry), keep);
|
|
596
|
+
}
|
|
597
|
+
}
|
|
598
|
+
/**
|
|
599
|
+
* Decides what a newly-constructed store starts holding: nothing (the default — empty the directory
|
|
600
|
+
* outright, this pack's behavior before `retainAcrossRestarts` existed), or the authenticated
|
|
601
|
+
* survivors of the previous process.
|
|
602
|
+
*
|
|
603
|
+
* Exported so the whole restart decision can be exercised without standing a store up around it.
|
|
604
|
+
*/
|
|
605
|
+
export async function prepareAttachmentStorage(requiredArgs, optionalArgs = {}) {
|
|
606
|
+
const { canonicalUploadDirectory, retainAcrossRestarts, retentionMs, now, } = { ...requiredArgs, ...optionalArgs };
|
|
607
|
+
if (!retainAcrossRestarts) {
|
|
608
|
+
for (const entry of await readdir(canonicalUploadDirectory)) {
|
|
609
|
+
await rm(resolve(canonicalUploadDirectory, entry), { recursive: true, force: true });
|
|
610
|
+
}
|
|
611
|
+
return new Map();
|
|
612
|
+
}
|
|
613
|
+
const adopted = await loadPersistedAttachments({ canonicalUploadDirectory, retentionMs, ...(now === undefined ? {} : { now }) });
|
|
614
|
+
await removeUnadoptedUploads({ canonicalUploadDirectory, adopted });
|
|
615
|
+
return adopted;
|
|
616
|
+
}
|
|
617
|
+
/**
|
|
618
|
+
* The disk-backed `AttachmentStore` this package ships. Every default matches what a chat composer
|
|
619
|
+
* needs out of the box; a host that wants different quotas passes them rather than reimplementing
|
|
620
|
+
* the port.
|
|
621
|
+
*
|
|
622
|
+
* @complexity `register`/`claim` are O(n) in the number of tracked records (a small bounded number
|
|
623
|
+
* — `maxStoredAttachments`), which is what keeps the quota decision synchronous; see `register`.
|
|
624
|
+
*/
|
|
625
|
+
export async function createDiskAttachmentStore(requiredArgs, optionalArgs = {}) {
|
|
626
|
+
const { uploadDirectory, maxAttachments = 10, maxBatchBytes = 50 * 1024 * 1024, maxStoredAttachments = 100, maxStoredBytes = 200 * 1024 * 1024, retentionMs = 60 * 60 * 1_000, retainAcrossRestarts = false, } = { ...requiredArgs, ...optionalArgs };
|
|
627
|
+
await mkdir(uploadDirectory, { recursive: true, mode: 0o700 });
|
|
628
|
+
await chmod(uploadDirectory, 0o700);
|
|
629
|
+
const canonicalUploadDirectory = await realpath(uploadDirectory);
|
|
630
|
+
// Either empties the directory (the default — a file left by an interrupted previous process has
|
|
631
|
+
// no record to authenticate it against) or adopts the previous process's authenticated survivors.
|
|
632
|
+
// See `prepareAttachmentStorage`.
|
|
633
|
+
const records = await prepareAttachmentStorage({ canonicalUploadDirectory, retainAcrossRestarts, retentionMs });
|
|
634
|
+
const sidecarDirectory = resolve(canonicalUploadDirectory, SIDECAR_DIRECTORY_NAME);
|
|
635
|
+
const sidecarPathFor = (record) => resolve(sidecarDirectory, attachmentSidecarFileName({ id: record.id }));
|
|
636
|
+
/**
|
|
637
|
+
* Writes one record's sidecar, or does nothing at all when this store was not asked to survive
|
|
638
|
+
* restarts. A failure rolls the record back out of the map before propagating: a registration
|
|
639
|
+
* whose sidecar could not be written would otherwise be silently non-surviving, which is the
|
|
640
|
+
* exact failure mode this whole option exists to remove. `register`'s own `catch` deletes the
|
|
641
|
+
* file, so the caller sees a failed upload rather than one that quietly evaporates later.
|
|
642
|
+
*/
|
|
643
|
+
const persistRecord = async (record) => {
|
|
644
|
+
if (!retainAcrossRestarts)
|
|
645
|
+
return;
|
|
646
|
+
const persisted = {
|
|
647
|
+
id: record.id,
|
|
648
|
+
filePath: record.filePath,
|
|
649
|
+
name: record.name,
|
|
650
|
+
kind: record.kind,
|
|
651
|
+
size: record.size,
|
|
652
|
+
batchId: record.batchId,
|
|
653
|
+
dev: record.dev,
|
|
654
|
+
ino: record.ino,
|
|
655
|
+
createdAt: record.createdAt,
|
|
656
|
+
...(record.ownerId === undefined ? {} : { ownerId: record.ownerId }),
|
|
657
|
+
};
|
|
658
|
+
try {
|
|
659
|
+
await writeFile(sidecarPathFor(record), JSON.stringify(persisted), { mode: 0o600 });
|
|
660
|
+
}
|
|
661
|
+
catch (error) {
|
|
662
|
+
records.delete(record.id);
|
|
663
|
+
throw error;
|
|
664
|
+
}
|
|
665
|
+
};
|
|
666
|
+
const forgetRecord = async (record) => {
|
|
667
|
+
if (retainAcrossRestarts)
|
|
668
|
+
await rm(sidecarPathFor(record), { force: true });
|
|
669
|
+
};
|
|
670
|
+
const resolveBatchDirectory = (batchId) => {
|
|
671
|
+
if (!BATCH_ID_PATTERN.test(batchId)) {
|
|
672
|
+
throw new AttachmentRejectedError({ reason: 'invalid-batch', message: 'Invalid attachment batch' });
|
|
673
|
+
}
|
|
674
|
+
// No containment re-check follows, and two things together are why — **both** are load-bearing:
|
|
675
|
+
// `BATCH_ID_PATTERN` admits only `[a-zA-Z0-9-]`, so `batchId` carries no `.`, `/`, or `\`; and
|
|
676
|
+
// `resolve` normalizes, so the result is exactly `<canonicalUploadDirectory><sep><batchId>`, one
|
|
677
|
+
// non-traversing segment deeper. Weakening the pattern (a `.` would be enough) or dropping the
|
|
678
|
+
// `resolve` reintroduces traversal here. Verified by sweeping 5,040,504 `(uploadRoot, batchId)`
|
|
679
|
+
// pairs on both posix and win32: zero escape the upload root.
|
|
680
|
+
return resolve(canonicalUploadDirectory, batchId);
|
|
681
|
+
};
|
|
682
|
+
const removeEmptyBatch = async (batchDirectory) => {
|
|
683
|
+
try {
|
|
684
|
+
await rmdir(batchDirectory);
|
|
685
|
+
}
|
|
686
|
+
catch {
|
|
687
|
+
// Non-empty (another unclaimed upload still occupies it) or already gone. Both are fine:
|
|
688
|
+
// this is opportunistic tidying, never the thing that makes a delete correct.
|
|
689
|
+
}
|
|
690
|
+
};
|
|
691
|
+
const deleteRecord = async (record) => {
|
|
692
|
+
records.delete(record.id);
|
|
693
|
+
await forgetRecord(record);
|
|
694
|
+
await rm(record.filePath, { force: true });
|
|
695
|
+
await removeEmptyBatch(record.batchDirectory);
|
|
696
|
+
};
|
|
697
|
+
const totalBytes = (candidates) => candidates.reduce((total, record) => total + record.size, 0);
|
|
698
|
+
return {
|
|
699
|
+
async createBatchDirectory({ batchId }, _optional = {}) {
|
|
700
|
+
const directory = resolveBatchDirectory(batchId);
|
|
701
|
+
await mkdir(directory, { recursive: true, mode: 0o700 });
|
|
702
|
+
await chmod(directory, 0o700);
|
|
703
|
+
return directory;
|
|
704
|
+
},
|
|
705
|
+
async register({ input }) {
|
|
706
|
+
const batchDirectory = resolveBatchDirectory(input.batchId);
|
|
707
|
+
// `resolve` is NOT redundant with the route's own `resolve`, and this line is the precondition
|
|
708
|
+
// the containment argument below depends on. Do not remove it or take a pre-resolved path from
|
|
709
|
+
// the caller: `register` is a public port method, so `input.path` is untrusted.
|
|
710
|
+
const filePath = resolve(input.path);
|
|
711
|
+
// Containment: the file's parent directory must be *exactly* the batch directory — stronger
|
|
712
|
+
// than a `path.relative`-based "is inside" test, which also admits nested subdirectories and
|
|
713
|
+
// the batch directory itself.
|
|
714
|
+
//
|
|
715
|
+
// Why parent-equality is sufficient: `resolve` above guarantees `filePath` is absolute and
|
|
716
|
+
// normalized with no `..` left in the path body, so `dirname(filePath) === batchDirectory`
|
|
717
|
+
// implies `filePath` is `batchDirectory + sep + basename` with a real, non-traversing
|
|
718
|
+
// basename. This is *not* an unconditional path-algebra identity — without the `resolve`,
|
|
719
|
+
// `dirname('/a/..') === '/a'` is true while `/a/..` escapes `/a`. The normalization is what
|
|
720
|
+
// makes the argument hold; a differential fuzz of 6,000,000 `input.path` values against the
|
|
721
|
+
// previous two-part check found zero inputs this accepts that containment rejected.
|
|
722
|
+
//
|
|
723
|
+
// Deliberately OUTSIDE the `try` below, whose `catch` unlinks `filePath`. A path that failed
|
|
724
|
+
// containment must never reach that cleanup, or this port becomes an arbitrary-file-delete
|
|
725
|
+
// primitive for anything the daemon can unlink.
|
|
726
|
+
if (dirname(filePath) !== batchDirectory) {
|
|
727
|
+
throw new AttachmentRejectedError({ reason: 'attachment-integrity', message: 'Attachment path is outside its batch' });
|
|
728
|
+
}
|
|
729
|
+
try {
|
|
730
|
+
const info = await lstat(filePath);
|
|
731
|
+
// `lstat` (not `stat`): a symlink must be seen as a symlink, not followed. `isFile()` is
|
|
732
|
+
// false for a symlink and for a directory, which is the whole check.
|
|
733
|
+
if (!info.isFile()) {
|
|
734
|
+
throw new AttachmentRejectedError({ reason: 'attachment-integrity', message: 'Attachment is not a regular file' });
|
|
735
|
+
}
|
|
736
|
+
const canonical = await realpath(filePath);
|
|
737
|
+
if (canonical !== filePath) {
|
|
738
|
+
throw new AttachmentRejectedError({ reason: 'attachment-integrity', message: 'Attachment path is not canonical' });
|
|
739
|
+
}
|
|
740
|
+
// Quotas are decided here, immediately before `records.set`, with no `await` in between —
|
|
741
|
+
// so two concurrent registrations cannot both observe the last free slot (or the last free
|
|
742
|
+
// bytes) and both commit. Moving any of this above the `await`s would reintroduce that
|
|
743
|
+
// race; adding an `await` below would too.
|
|
744
|
+
const batchRecords = [...records.values()]
|
|
745
|
+
.filter((record) => record.batchId === input.batchId);
|
|
746
|
+
if (batchRecords.length >= maxAttachments) {
|
|
747
|
+
throw new AttachmentRejectedError({ reason: 'batch-count-exceeded', message: `You can attach at most ${maxAttachments} files to one message` });
|
|
748
|
+
}
|
|
749
|
+
if (totalBytes(batchRecords) + info.size > maxBatchBytes) {
|
|
750
|
+
throw new AttachmentRejectedError({ reason: 'batch-too-large', message: `Attachments for one message must total ${formatByteLimit(maxBatchBytes)} or less` });
|
|
751
|
+
}
|
|
752
|
+
if (records.size >= maxStoredAttachments
|
|
753
|
+
|| totalBytes([...records.values()]) + info.size > maxStoredBytes) {
|
|
754
|
+
throw new AttachmentRejectedError({ reason: 'storage-full', message: 'Attachment storage is full' });
|
|
755
|
+
}
|
|
756
|
+
const id = `attachment:${randomUUID()}`;
|
|
757
|
+
const record = {
|
|
758
|
+
id,
|
|
759
|
+
filePath,
|
|
760
|
+
name: input.name,
|
|
761
|
+
kind: input.kind,
|
|
762
|
+
size: info.size,
|
|
763
|
+
batchId: input.batchId,
|
|
764
|
+
batchDirectory,
|
|
765
|
+
dev: info.dev,
|
|
766
|
+
ino: info.ino,
|
|
767
|
+
createdAt: Date.now(),
|
|
768
|
+
...(input.ownerId === undefined ? {} : { ownerId: input.ownerId }),
|
|
769
|
+
};
|
|
770
|
+
records.set(id, record);
|
|
771
|
+
// The first `await` after the synchronous quota window closes, so it cannot reopen the race
|
|
772
|
+
// the comment above guards. Rolls `records` back itself on failure — see `persistRecord`.
|
|
773
|
+
await persistRecord(record);
|
|
774
|
+
return { path: id, name: record.name, kind: record.kind, size: record.size };
|
|
775
|
+
}
|
|
776
|
+
catch (error) {
|
|
777
|
+
// A file this store refused to take ownership of must not be left behind. `rm` is
|
|
778
|
+
// deliberately non-recursive (never recurse over an attacker-influenced path) and its own
|
|
779
|
+
// failure is swallowed, so the real rejection above always propagates instead of being
|
|
780
|
+
// masked by e.g. EISDIR from a directory sitting at `filePath`.
|
|
781
|
+
await rm(filePath, { force: true }).catch(() => undefined);
|
|
782
|
+
await removeEmptyBatch(batchDirectory);
|
|
783
|
+
throw error;
|
|
784
|
+
}
|
|
785
|
+
},
|
|
786
|
+
async claim({ attachments, runId }, _optional = {}) {
|
|
787
|
+
if (attachments.length === 0)
|
|
788
|
+
return { attachments: [] };
|
|
789
|
+
// Phase 1 (`reserveAttachmentRecords`) reserves synchronously; Phase 2
|
|
790
|
+
// (`verifyClaimedAttachments`) re-validates what is now held exclusively. Any Phase 2 failure
|
|
791
|
+
// releases the whole reservation here, so a rejected claim still leaves nothing half-claimed
|
|
792
|
+
// and the caller can retry with a corrected set. Only reservations made by *this* call are
|
|
793
|
+
// released, so a concurrent winner's claim is never revoked by a loser's rollback.
|
|
794
|
+
const claimed = reserveAttachmentRecords({ attachments, records, runId, maxAttachments });
|
|
795
|
+
try {
|
|
796
|
+
const batchDirectory = await verifyClaimedAttachments({ claimed });
|
|
797
|
+
return {
|
|
798
|
+
attachments: claimed.map((record) => ({
|
|
799
|
+
path: record.filePath,
|
|
800
|
+
name: record.name,
|
|
801
|
+
kind: record.kind,
|
|
802
|
+
size: record.size,
|
|
803
|
+
})),
|
|
804
|
+
batchDirectory,
|
|
805
|
+
};
|
|
806
|
+
}
|
|
807
|
+
catch (error) {
|
|
808
|
+
for (const record of claimed)
|
|
809
|
+
delete record.claimedRunId;
|
|
810
|
+
throw error;
|
|
811
|
+
}
|
|
812
|
+
},
|
|
813
|
+
async resolveForRun({ ref, runId }, _optional = {}) {
|
|
814
|
+
// Synchronous check-then-reserve, same reasoning as `reserveAttachmentRecords`: nothing may
|
|
815
|
+
// `await` between reading `claimedRunId` and writing it, or a concurrent call could observe
|
|
816
|
+
// the same unclaimed record and both believe they reserved it.
|
|
817
|
+
const record = records.get(ref) ?? [...records.values()].find((candidate) => candidate.filePath === ref);
|
|
818
|
+
if (!record)
|
|
819
|
+
return undefined;
|
|
820
|
+
if (record.claimedRunId !== undefined && record.claimedRunId !== runId) {
|
|
821
|
+
throw new AttachmentRejectedError({ reason: 'attachment-unknown-or-claimed', message: 'Attachment is unknown or already claimed' });
|
|
822
|
+
}
|
|
823
|
+
const reservedNow = record.claimedRunId === undefined;
|
|
824
|
+
if (reservedNow)
|
|
825
|
+
record.claimedRunId = runId;
|
|
826
|
+
try {
|
|
827
|
+
await verifyClaimedAttachments({ claimed: [record] });
|
|
828
|
+
}
|
|
829
|
+
catch (error) {
|
|
830
|
+
// Mirrors `claim()`'s own rollback: a lookup that fails integrity must not leave a phantom
|
|
831
|
+
// reservation behind for a record that turned out to be unsafe to hand back.
|
|
832
|
+
if (reservedNow)
|
|
833
|
+
delete record.claimedRunId;
|
|
834
|
+
throw error;
|
|
835
|
+
}
|
|
836
|
+
return { path: record.filePath, name: record.name, kind: record.kind, size: record.size };
|
|
837
|
+
},
|
|
838
|
+
async listPendingForOwner({ ownerId }, _optional = {}) {
|
|
839
|
+
return [...records.values()]
|
|
840
|
+
// `record.ownerId !== undefined` first, short-circuiting before the comparison: this is
|
|
841
|
+
// what makes an ownerless record excluded rather than accidentally matched by a falsy-ish
|
|
842
|
+
// `ownerId` argument — see this method's own doc on why that must never be a wildcard.
|
|
843
|
+
.filter((record) => record.claimedRunId === undefined && record.ownerId !== undefined && record.ownerId === ownerId)
|
|
844
|
+
.sort((a, b) => a.createdAt - b.createdAt)
|
|
845
|
+
.map((record) => ({ ref: record.id, name: record.name, kind: record.kind, size: record.size, createdAt: record.createdAt }));
|
|
846
|
+
},
|
|
847
|
+
async deleteUnclaimed({ batchId, paths }, _optional = {}) {
|
|
848
|
+
const batchDirectory = resolveBatchDirectory(batchId);
|
|
849
|
+
for (const attachmentId of new Set(paths)) {
|
|
850
|
+
const record = records.get(attachmentId);
|
|
851
|
+
if (record && record.batchId === batchId && record.claimedRunId === undefined) {
|
|
852
|
+
await deleteRecord(record);
|
|
853
|
+
}
|
|
854
|
+
}
|
|
855
|
+
await removeEmptyBatch(batchDirectory);
|
|
856
|
+
},
|
|
857
|
+
async cleanupRun({ runId }, _optional = {}) {
|
|
858
|
+
for (const record of [...records.values()]) {
|
|
859
|
+
if (record.claimedRunId === runId)
|
|
860
|
+
await deleteRecord(record);
|
|
861
|
+
}
|
|
862
|
+
},
|
|
863
|
+
async pruneExpired(_required, { now = Date.now() } = {}) {
|
|
864
|
+
for (const record of [...records.values()]) {
|
|
865
|
+
if (record.claimedRunId === undefined && now - record.createdAt >= retentionMs) {
|
|
866
|
+
await deleteRecord(record);
|
|
867
|
+
}
|
|
868
|
+
}
|
|
869
|
+
},
|
|
870
|
+
async dispose() {
|
|
871
|
+
for (const record of [...records.values()])
|
|
872
|
+
await deleteRecord(record);
|
|
873
|
+
},
|
|
874
|
+
};
|
|
875
|
+
}
|
|
876
|
+
export const ATTACHMENTS_ROUTE_PATH = '/api/attachments';
|
|
877
|
+
function defaultInternalErrorSink(context) {
|
|
878
|
+
// eslint-disable-next-line no-console
|
|
879
|
+
console.error(`[@jini-ai/http-kit] internal error (${context.source}, correlationId=${context.correlationId})`, context.error);
|
|
880
|
+
}
|
|
881
|
+
function reportInternalError(deps, source, error, batchId) {
|
|
882
|
+
const correlationId = randomUUID();
|
|
883
|
+
const sink = deps.onInternalError ?? defaultInternalErrorSink;
|
|
884
|
+
sink({ source, batchId, correlationId, error });
|
|
885
|
+
return createApiError({ code: 'INTERNAL_ERROR', message: 'an internal error occurred' }, { requestId: correlationId });
|
|
886
|
+
}
|
|
887
|
+
/**
|
|
888
|
+
* Sends the right status for an upload failure: an explicitly-classified rejection keeps its real
|
|
889
|
+
* message (the composer shows it to the user, so "you can attach at most 10 files" has to survive),
|
|
890
|
+
* while an integrity failure or any unexpected error is redacted to a generic `INTERNAL_ERROR`.
|
|
891
|
+
*/
|
|
892
|
+
function respondToUploadFailure(res, deps, error, batchId, source) {
|
|
893
|
+
if (error instanceof AttachmentRejectedError && REJECTION_STATUS[error.reason] !== 500) {
|
|
894
|
+
sendApiError({ res, status: REJECTION_STATUS[error.reason], error: apiErrorForRejection(error) });
|
|
895
|
+
return;
|
|
896
|
+
}
|
|
897
|
+
sendApiError({ res, status: 500, error: reportInternalError(deps, source, error, batchId) });
|
|
898
|
+
}
|
|
899
|
+
/** `false` when the same-origin guard rejected the request (and already answered it). */
|
|
900
|
+
function passesOriginGuard(req, res, deps, adapter) {
|
|
901
|
+
if (deps.requireSameOrigin === false)
|
|
902
|
+
return true;
|
|
903
|
+
const origin = guardSameOrigin({ req, origin: adapter });
|
|
904
|
+
if (origin.ok)
|
|
905
|
+
return true;
|
|
906
|
+
sendApiError({ res, status: 403, error: origin.error });
|
|
907
|
+
return false;
|
|
908
|
+
}
|
|
909
|
+
/**
|
|
910
|
+
* Handles one upload: rate-limit, batch directory, bounded stream to disk, signature sniff,
|
|
911
|
+
* register. Exported so a host mounting its own path (or its own framework) can reuse the whole
|
|
912
|
+
* body without re-deriving the ordering, which is load-bearing — see the `finally`.
|
|
913
|
+
*/
|
|
914
|
+
export async function handleAttachmentUpload({ req, res, deps, state }, _optional = {}) {
|
|
915
|
+
const maxConcurrentUploads = deps.maxConcurrentUploads ?? 4;
|
|
916
|
+
const maxAttachmentBytes = deps.maxAttachmentBytes ?? 20 * 1024 * 1024;
|
|
917
|
+
const name = sanitizeAttachmentName({ requestedName: req.query.name });
|
|
918
|
+
const batchId = typeof req.query.batch === 'string' ? req.query.batch : '';
|
|
919
|
+
// Every refusal below goes through `respondToUploadFailure`, so the reason -> status/code mapping
|
|
920
|
+
// lives in exactly one place rather than being partly inlined here.
|
|
921
|
+
if (state.activeUploads >= maxConcurrentUploads) {
|
|
922
|
+
respondToUploadFailure(res, deps, new AttachmentRejectedError({ reason: 'too-many-concurrent-uploads', message: 'Too many attachment uploads are in progress' }), batchId, 'attachment-upload');
|
|
923
|
+
return;
|
|
924
|
+
}
|
|
925
|
+
// Checked before a single byte is written: a drained stream would otherwise produce a zero-byte
|
|
926
|
+
// file and the misleading "attachment is empty". See this module's body-parser doc.
|
|
927
|
+
if (req.readableEnded) {
|
|
928
|
+
respondToUploadFailure(res, deps, new AttachmentRejectedError({ reason: 'attachment-body-consumed', message: `the request body was already consumed before ${ATTACHMENTS_ROUTE_PATH} received it — mount this route pack before any global body parser, or scope that parser to skip this path` }), batchId, 'attachment-upload');
|
|
929
|
+
return;
|
|
930
|
+
}
|
|
931
|
+
state.activeUploads += 1;
|
|
932
|
+
try {
|
|
933
|
+
await deps.store.pruneExpired({});
|
|
934
|
+
const batchDirectory = await deps.store.createBatchDirectory({ batchId });
|
|
935
|
+
// A fresh UUID, never the client's filename. The extension is carried over (bounded) only
|
|
936
|
+
// because some agent runtimes decide how to read a file from its suffix.
|
|
937
|
+
const suffix = extname(name).slice(0, 12);
|
|
938
|
+
const path = resolve(batchDirectory, `${randomUUID()}${suffix}`);
|
|
939
|
+
const upload = await writeBoundedAttachmentBody({ request: req, filePath: path, maxBytes: maxAttachmentBytes });
|
|
940
|
+
if (upload.size === 0) {
|
|
941
|
+
await rm(path, { force: true });
|
|
942
|
+
sendApiError({ res, status: 400, error: createApiError({ code: 'BAD_REQUEST', message: 'Attachment is empty' }) });
|
|
943
|
+
return;
|
|
944
|
+
}
|
|
945
|
+
const ownerId = deps.resolveOwnerId?.({ req });
|
|
946
|
+
const attachment = await deps.store.register({
|
|
947
|
+
input: {
|
|
948
|
+
batchId,
|
|
949
|
+
path,
|
|
950
|
+
name,
|
|
951
|
+
kind: detectAttachmentKind({ body: upload.signature }),
|
|
952
|
+
size: upload.size,
|
|
953
|
+
...(ownerId === undefined ? {} : { ownerId }),
|
|
954
|
+
}
|
|
955
|
+
});
|
|
956
|
+
sendJson({ res, status: 201, body: { attachment } });
|
|
957
|
+
}
|
|
958
|
+
catch (error) {
|
|
959
|
+
respondToUploadFailure(res, deps, error, batchId, 'attachment-upload');
|
|
960
|
+
}
|
|
961
|
+
finally {
|
|
962
|
+
state.activeUploads -= 1;
|
|
963
|
+
// Removes the batch directory when this failed upload left it empty. Passing no paths is
|
|
964
|
+
// deliberate: a *successful* upload must not be deleted here, and `deleteUnclaimed` with an
|
|
965
|
+
// empty list does exactly the directory tidying and nothing else.
|
|
966
|
+
await deps.store.deleteUnclaimed({ batchId, paths: [] }).catch(() => undefined);
|
|
967
|
+
}
|
|
968
|
+
}
|
|
969
|
+
/**
|
|
970
|
+
* Handles a client abandoning a batch (a failed multi-file upload, a cleared composer). Only
|
|
971
|
+
* *unclaimed* uploads can be deleted this way, so a caller cannot use it to pull files out from
|
|
972
|
+
* under a run that already claimed them.
|
|
973
|
+
*/
|
|
974
|
+
export async function handleAttachmentCleanup({ req, res, deps }, _optional = {}) {
|
|
975
|
+
const maxCleanupPaths = deps.maxCleanupPaths ?? 10;
|
|
976
|
+
const body = req.body;
|
|
977
|
+
if (typeof body?.batchId !== 'string'
|
|
978
|
+
|| !Array.isArray(body.paths)
|
|
979
|
+
|| body.paths.length > maxCleanupPaths
|
|
980
|
+
|| !body.paths.every((path) => typeof path === 'string')) {
|
|
981
|
+
sendApiError({ res, status: 400, error: createApiError({ code: 'BAD_REQUEST', message: 'Invalid attachment cleanup request' }) });
|
|
982
|
+
return;
|
|
983
|
+
}
|
|
984
|
+
try {
|
|
985
|
+
await deps.store.deleteUnclaimed({ batchId: body.batchId, paths: body.paths });
|
|
986
|
+
res.status(204).end();
|
|
987
|
+
}
|
|
988
|
+
catch (error) {
|
|
989
|
+
respondToUploadFailure(res, deps, error, body.batchId, 'attachment-cleanup');
|
|
990
|
+
}
|
|
991
|
+
}
|
|
992
|
+
/**
|
|
993
|
+
* Mounts `POST`/`DELETE /api/attachments` on `app`. A pack's `http(app, services)` calls this
|
|
994
|
+
* directly.
|
|
995
|
+
*
|
|
996
|
+
* Hand-mounted rather than built from `defineJsonRoute`/`mountJsonRoute` for two concrete reasons:
|
|
997
|
+
* the upload reads the raw request stream (a JSON-parsed `req.body` is exactly what must not have
|
|
998
|
+
* happened), and the cleanup answers `204` with no body, which a JSON responder cannot express.
|
|
999
|
+
*/
|
|
1000
|
+
export function registerAttachmentRoutes({ app, deps, adapter }, _optional = {}) {
|
|
1001
|
+
// Per-registration, not module-level: two daemons in one process (a test harness, an embedded
|
|
1002
|
+
// second host) must not share one upload budget.
|
|
1003
|
+
const state = { activeUploads: 0 };
|
|
1004
|
+
app.post(ATTACHMENTS_ROUTE_PATH, async (req, res) => {
|
|
1005
|
+
try {
|
|
1006
|
+
if (!passesOriginGuard(req, res, deps, adapter))
|
|
1007
|
+
return;
|
|
1008
|
+
await handleAttachmentUpload({ req, res, deps, state });
|
|
1009
|
+
}
|
|
1010
|
+
catch (error) {
|
|
1011
|
+
// `handleAttachmentUpload` already catches everything inside its own body — this only ever
|
|
1012
|
+
// fires for `passesOriginGuard`/`guardSameOrigin` throwing, which is Result-returning by
|
|
1013
|
+
// contract but not guaranteed never to throw (`isLocalSameOrigin` really does throw on a
|
|
1014
|
+
// malformed `JINI_ALLOWED_ORIGINS` entry — see the paired regression test). Mounting this
|
|
1015
|
+
// route bypasses `mountJsonRoute`'s adapter (see this function's own doc), so nothing else
|
|
1016
|
+
// stood between that throw and an unhandled rejection with no process-level guard anywhere
|
|
1017
|
+
// in this package's path.
|
|
1018
|
+
if (!res.headersSent)
|
|
1019
|
+
respondToUploadFailure(res, deps, error, null, 'attachment-upload');
|
|
1020
|
+
}
|
|
1021
|
+
});
|
|
1022
|
+
app.delete(ATTACHMENTS_ROUTE_PATH, async (req, res) => {
|
|
1023
|
+
try {
|
|
1024
|
+
if (!passesOriginGuard(req, res, deps, adapter))
|
|
1025
|
+
return;
|
|
1026
|
+
await handleAttachmentCleanup({ req, res, deps });
|
|
1027
|
+
}
|
|
1028
|
+
catch (error) {
|
|
1029
|
+
if (!res.headersSent)
|
|
1030
|
+
sendApiError({ res, status: 500, error: reportInternalError(deps, 'attachment-cleanup', error, null) });
|
|
1031
|
+
}
|
|
1032
|
+
});
|
|
1033
|
+
}
|
|
1034
|
+
//# sourceMappingURL=attachments.js.map
|