@bevel-software/platform-core-backend 0.23.0 → 0.25.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/create-core-server.d.ts.map +1 -1
- package/dist/core/create-core-server.js +17 -3
- package/dist/core/create-core-server.js.map +1 -1
- package/dist/core/create-core-services.d.ts +3 -0
- package/dist/core/create-core-services.d.ts.map +1 -1
- package/dist/core/create-core-services.js +25 -1
- package/dist/core/create-core-services.js.map +1 -1
- package/dist/core/lifecycle.d.ts +12 -0
- package/dist/core/lifecycle.d.ts.map +1 -1
- package/dist/core/lifecycle.js +10 -0
- package/dist/core/lifecycle.js.map +1 -1
- package/dist/core-config.d.ts +16 -0
- package/dist/core-config.d.ts.map +1 -1
- package/dist/core-config.js +17 -0
- package/dist/core-config.js.map +1 -1
- package/dist/index.d.ts +3 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +13 -2
- package/dist/index.js.map +1 -1
- package/dist/modules/access/access.routes.d.ts.map +1 -1
- package/dist/modules/access/access.routes.js +4 -1
- package/dist/modules/access/access.routes.js.map +1 -1
- package/dist/modules/access/directory-sync-bot.d.ts.map +1 -1
- package/dist/modules/access/directory-sync-bot.js +7 -3
- package/dist/modules/access/directory-sync-bot.js.map +1 -1
- package/dist/modules/agent-instructions/agent-instructions.routes.d.ts +8 -1
- package/dist/modules/agent-instructions/agent-instructions.routes.d.ts.map +1 -1
- package/dist/modules/agent-instructions/agent-instructions.routes.js +8 -2
- package/dist/modules/agent-instructions/agent-instructions.routes.js.map +1 -1
- package/dist/modules/agent-instructions/compose.d.ts +38 -6
- package/dist/modules/agent-instructions/compose.d.ts.map +1 -1
- package/dist/modules/agent-instructions/compose.js +39 -6
- package/dist/modules/agent-instructions/compose.js.map +1 -1
- package/dist/modules/agent-instructions/index.d.ts +2 -1
- package/dist/modules/agent-instructions/index.d.ts.map +1 -1
- package/dist/modules/agent-instructions/index.js +2 -1
- package/dist/modules/agent-instructions/index.js.map +1 -1
- package/dist/modules/agent-instructions/shared-file-rules.d.ts +115 -0
- package/dist/modules/agent-instructions/shared-file-rules.d.ts.map +1 -0
- package/dist/modules/agent-instructions/shared-file-rules.js +272 -0
- package/dist/modules/agent-instructions/shared-file-rules.js.map +1 -0
- package/dist/modules/audit/agent-audit.service.d.ts.map +1 -1
- package/dist/modules/audit/agent-audit.service.js +4 -2
- package/dist/modules/audit/agent-audit.service.js.map +1 -1
- package/dist/modules/auth/account-erasure.service.d.ts.map +1 -1
- package/dist/modules/auth/account-erasure.service.js +57 -16
- package/dist/modules/auth/account-erasure.service.js.map +1 -1
- package/dist/modules/auth/auth.service.d.ts.map +1 -1
- package/dist/modules/auth/auth.service.js +25 -15
- package/dist/modules/auth/auth.service.js.map +1 -1
- package/dist/modules/code-mode/code-mode.tool.d.ts +20 -2
- package/dist/modules/code-mode/code-mode.tool.d.ts.map +1 -1
- package/dist/modules/code-mode/code-mode.tool.js +66 -35
- package/dist/modules/code-mode/code-mode.tool.js.map +1 -1
- package/dist/modules/database/connection.d.ts +16 -0
- package/dist/modules/database/connection.d.ts.map +1 -1
- package/dist/modules/database/connection.js +117 -0
- package/dist/modules/database/connection.js.map +1 -1
- package/dist/modules/database/core-schema.d.ts +296 -96
- package/dist/modules/database/core-schema.d.ts.map +1 -1
- package/dist/modules/database/core-schema.js +81 -31
- package/dist/modules/database/core-schema.js.map +1 -1
- package/dist/modules/database/migrate.d.ts +95 -1
- package/dist/modules/database/migrate.d.ts.map +1 -1
- package/dist/modules/database/migrate.js +390 -2
- package/dist/modules/database/migrate.js.map +1 -1
- package/dist/modules/kb-fs/locking-filesystem.d.ts +17 -0
- package/dist/modules/kb-fs/locking-filesystem.d.ts.map +1 -1
- package/dist/modules/kb-fs/locking-filesystem.js +29 -0
- package/dist/modules/kb-fs/locking-filesystem.js.map +1 -1
- package/dist/modules/mcp/mcp.service.d.ts +8 -0
- package/dist/modules/mcp/mcp.service.d.ts.map +1 -1
- package/dist/modules/mcp/mcp.service.js +38 -8
- package/dist/modules/mcp/mcp.service.js.map +1 -1
- package/dist/modules/plugins/join-request-records.store.d.ts.map +1 -1
- package/dist/modules/plugins/join-request-records.store.js +7 -4
- package/dist/modules/plugins/join-request-records.store.js.map +1 -1
- package/dist/modules/tool-auth/external-api-key.service.d.ts.map +1 -1
- package/dist/modules/tool-auth/external-api-key.service.js +8 -2
- package/dist/modules/tool-auth/external-api-key.service.js.map +1 -1
- package/dist/modules/tool-manuals/tool-manuals.service.d.ts.map +1 -1
- package/dist/modules/tool-manuals/tool-manuals.service.js +30 -0
- package/dist/modules/tool-manuals/tool-manuals.service.js.map +1 -1
- package/dist/modules/tool-manuals/tool-manuals.tools.d.ts.map +1 -1
- package/dist/modules/tool-manuals/tool-manuals.tools.js +42 -31
- package/dist/modules/tool-manuals/tool-manuals.tools.js.map +1 -1
- package/dist/modules/tool-registry/description-length.d.ts +80 -0
- package/dist/modules/tool-registry/description-length.d.ts.map +1 -0
- package/dist/modules/tool-registry/description-length.js +108 -0
- package/dist/modules/tool-registry/description-length.js.map +1 -0
- package/dist/modules/workflow/git/git.service.d.ts +25 -0
- package/dist/modules/workflow/git/git.service.d.ts.map +1 -1
- package/dist/modules/workflow/git/git.service.js +40 -1
- package/dist/modules/workflow/git/git.service.js.map +1 -1
- package/dist/modules/workflow/pending-commits.service.d.ts.map +1 -1
- package/dist/modules/workflow/pending-commits.service.js +5 -1
- package/dist/modules/workflow/pending-commits.service.js.map +1 -1
- package/dist/modules/workflow/recovery-bot.d.ts.map +1 -1
- package/dist/modules/workflow/recovery-bot.js +7 -3
- package/dist/modules/workflow/recovery-bot.js.map +1 -1
- package/dist/modules/workflow/review-workflow/review-workflow.service.d.ts.map +1 -1
- package/dist/modules/workflow/review-workflow/review-workflow.service.js +12 -3
- package/dist/modules/workflow/review-workflow/review-workflow.service.js.map +1 -1
- package/dist/modules/workflow/workflow.service.d.ts.map +1 -1
- package/dist/modules/workflow/workflow.service.js +4 -1
- package/dist/modules/workflow/workflow.service.js.map +1 -1
- package/dist/modules/workspace/agent-upload.routes.d.ts +77 -0
- package/dist/modules/workspace/agent-upload.routes.d.ts.map +1 -0
- package/dist/modules/workspace/agent-upload.routes.js +210 -0
- package/dist/modules/workspace/agent-upload.routes.js.map +1 -0
- package/dist/modules/workspace/agent-upload.store.d.ts +284 -0
- package/dist/modules/workspace/agent-upload.store.d.ts.map +1 -0
- package/dist/modules/workspace/agent-upload.store.js +553 -0
- package/dist/modules/workspace/agent-upload.store.js.map +1 -0
- package/dist/modules/workspace/startup/steps/seed-tree.d.ts.map +1 -1
- package/dist/modules/workspace/startup/steps/seed-tree.js +3 -3
- package/dist/modules/workspace/startup/steps/seed-tree.js.map +1 -1
- package/dist/modules/workspace/startup/steps/template-source.d.ts +40 -0
- package/dist/modules/workspace/startup/steps/template-source.d.ts.map +1 -1
- package/dist/modules/workspace/startup/steps/template-source.js +46 -4
- package/dist/modules/workspace/startup/steps/template-source.js.map +1 -1
- package/dist/modules/workspace/upload-limits.d.ts +13 -0
- package/dist/modules/workspace/upload-limits.d.ts.map +1 -0
- package/dist/modules/workspace/upload-limits.js +13 -0
- package/dist/modules/workspace/upload-limits.js.map +1 -0
- package/dist/modules/workspace/workspace.routes.d.ts.map +1 -1
- package/dist/modules/workspace/workspace.routes.js +1 -1
- package/dist/modules/workspace/workspace.routes.js.map +1 -1
- package/dist/modules/workspace/workspace.service.d.ts +13 -0
- package/dist/modules/workspace/workspace.service.d.ts.map +1 -1
- package/dist/modules/workspace/workspace.service.js +61 -33
- package/dist/modules/workspace/workspace.service.js.map +1 -1
- package/dist/modules/workspace/workspace.tools.d.ts +11 -9
- package/dist/modules/workspace/workspace.tools.d.ts.map +1 -1
- package/dist/modules/workspace/workspace.tools.js +570 -134
- package/dist/modules/workspace/workspace.tools.js.map +1 -1
- package/dist/modules/workspace/write-denial.d.ts +0 -6
- package/dist/modules/workspace/write-denial.d.ts.map +1 -1
- package/dist/modules/workspace/write-denial.js +0 -6
- package/dist/modules/workspace/write-denial.js.map +1 -1
- package/dist/modules/workspace/zip-entry-rules.d.ts +114 -0
- package/dist/modules/workspace/zip-entry-rules.d.ts.map +1 -0
- package/dist/modules/workspace/zip-entry-rules.js +154 -0
- package/dist/modules/workspace/zip-entry-rules.js.map +1 -0
- package/dist/shared/column-crypto.d.ts +194 -0
- package/dist/shared/column-crypto.d.ts.map +1 -0
- package/dist/shared/column-crypto.js +144 -0
- package/dist/shared/column-crypto.js.map +1 -0
- package/dist/shared/token-crypto.d.ts.map +1 -1
- package/dist/shared/token-crypto.js +25 -1
- package/dist/shared/token-crypto.js.map +1 -1
- package/dist/tenancy/static-tenant-source.d.ts.map +1 -1
- package/dist/tenancy/static-tenant-source.js +1 -0
- package/dist/tenancy/static-tenant-source.js.map +1 -1
- package/dist/tenancy/tenant-secrets.d.ts +5 -1
- package/dist/tenancy/tenant-secrets.d.ts.map +1 -1
- package/dist/tenancy/tenant-secrets.js +4 -0
- package/dist/tenancy/tenant-secrets.js.map +1 -1
- package/kb-template/AGENTS.md +42 -0
- package/migrations/0016_pii_encryption.sql +20 -0
- package/migrations/meta/0016_snapshot.json +2327 -0
- package/migrations/meta/_journal.json +7 -0
- package/package.json +3 -3
- package/src/core/__tests__/lifecycle.test.ts +71 -4
- package/src/core/create-core-server.ts +21 -3
- package/src/core/create-core-services.ts +27 -1
- package/src/core/lifecycle.ts +19 -0
- package/src/core-config.ts +18 -0
- package/src/index.ts +25 -0
- package/src/modules/access/__tests__/users-db-double.ts +21 -12
- package/src/modules/access/access.routes.ts +4 -1
- package/src/modules/access/directory-sync-bot.ts +7 -3
- package/src/modules/agent-instructions/__tests__/agent-instructions.route.test.ts +8 -4
- package/src/modules/agent-instructions/__tests__/compose.test.ts +19 -10
- package/src/modules/agent-instructions/__tests__/shared-file-rules.test.ts +238 -0
- package/src/modules/agent-instructions/agent-instructions.routes.ts +12 -2
- package/src/modules/agent-instructions/compose.ts +50 -7
- package/src/modules/agent-instructions/index.ts +12 -0
- package/src/modules/agent-instructions/shared-file-rules.ts +314 -0
- package/src/modules/audit/agent-audit.service.ts +4 -2
- package/src/modules/auth/__tests__/account-deactivation.test.ts +2 -1
- package/src/modules/auth/__tests__/account-erasure.approval-gate.test.ts +6 -2
- package/src/modules/auth/__tests__/account.routes.test.ts +6 -3
- package/src/modules/auth/__tests__/auth.service.test.ts +3 -1
- package/src/modules/auth/account-erasure.service.ts +69 -18
- package/src/modules/auth/auth.service.ts +33 -23
- package/src/modules/code-mode/__tests__/chain-runtime.e2e.test.ts +335 -0
- package/src/modules/code-mode/__tests__/code-mode.tool.test.ts +46 -2
- package/src/modules/code-mode/code-mode.tool.ts +80 -34
- package/src/modules/database/__tests__/connection.test.ts +12 -0
- package/src/modules/database/__tests__/pii-backfill.pg.test.ts +780 -0
- package/src/modules/database/connection.ts +117 -0
- package/src/modules/database/core-schema.ts +81 -31
- package/src/modules/database/migrate.ts +540 -2
- package/src/modules/kb-fs/__tests__/locking-filesystem.test.ts +93 -0
- package/src/modules/kb-fs/locking-filesystem.ts +37 -0
- package/src/modules/mcp/__tests__/mcp.e2e.test.ts +4 -3
- package/src/modules/mcp/__tests__/mcp.service.test.ts +47 -7
- package/src/modules/mcp/mcp.service.ts +46 -7
- package/src/modules/plugins/join-request-records.store.ts +7 -4
- package/src/modules/tool-auth/external-api-key.service.ts +8 -2
- package/src/modules/tool-manuals/__tests__/tool-manuals.service.test.ts +151 -0
- package/src/modules/tool-manuals/tool-manuals.service.ts +37 -0
- package/src/modules/tool-manuals/tool-manuals.tools.ts +44 -31
- package/src/modules/tool-registry/__tests__/tool-description-length.test.ts +378 -0
- package/src/modules/tool-registry/description-length.ts +111 -0
- package/src/modules/workflow/git/__tests__/git.service.prFetchFailure.test.ts +170 -0
- package/src/modules/workflow/git/git.service.ts +47 -1
- package/src/modules/workflow/pending-commits.service.ts +5 -1
- package/src/modules/workflow/recovery-bot.ts +7 -3
- package/src/modules/workflow/review-workflow/__tests__/carry-approvals-forward.test.ts +4 -0
- package/src/modules/workflow/review-workflow/__tests__/erase-approver.test.ts +25 -3
- package/src/modules/workflow/review-workflow/review-workflow.service.ts +12 -3
- package/src/modules/workflow/workflow.service.ts +4 -1
- package/src/modules/workspace/__tests__/agent-uploads.test.ts +1604 -0
- package/src/modules/workspace/__tests__/escape-sequences.routes.test.ts +24 -14
- package/src/modules/workspace/__tests__/workspace.service.any-workspace-credentials.test.ts +156 -0
- package/src/modules/workspace/__tests__/workspace.service.replaced-repository.test.ts +77 -1
- package/src/modules/workspace/__tests__/workspace.service.test.ts +57 -0
- package/src/modules/workspace/__tests__/workspace.tools.agents-file.test.ts +39 -36
- package/src/modules/workspace/__tests__/workspace.tools.test.ts +196 -46
- package/src/modules/workspace/agent-upload.routes.ts +214 -0
- package/src/modules/workspace/agent-upload.store.ts +668 -0
- package/src/modules/workspace/startup/steps/__tests__/steps.test.ts +9 -9
- package/src/modules/workspace/startup/steps/seed-tree.ts +3 -6
- package/src/modules/workspace/startup/steps/template-source.ts +53 -5
- package/src/modules/workspace/upload-limits.ts +12 -0
- package/src/modules/workspace/workspace.routes.ts +1 -2
- package/src/modules/workspace/workspace.service.ts +63 -37
- package/src/modules/workspace/workspace.tools.ts +647 -148
- package/src/modules/workspace/write-denial.ts +0 -8
- package/src/modules/workspace/zip-entry-rules.ts +173 -0
- package/src/shared/__tests__/column-crypto.test.ts +217 -0
- package/src/shared/column-crypto.ts +218 -0
- package/src/shared/token-crypto.ts +28 -1
- package/src/tenancy/__tests__/static-tenant-source.test.ts +4 -0
- package/src/tenancy/static-tenant-source.ts +1 -0
- package/src/tenancy/tenant-secrets.ts +5 -1
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import express from 'express';
|
|
2
|
+
import { validateFilename } from '@bevel-software/platform-shared';
|
|
3
|
+
import { logger } from '../../shared/logging.js';
|
|
4
|
+
import { UploadTokenError, overLimit } from './agent-upload.store.js';
|
|
5
|
+
const log = logger('agent-uploads');
|
|
6
|
+
/** The route's own prefix, under `/api`. One spelling, shared with the raw-body test below. */
|
|
7
|
+
export const AGENT_UPLOAD_ROUTE = '/agent/uploads/:token';
|
|
8
|
+
/**
|
|
9
|
+
* The same endpoint with the token in the `x-upload-token` HEADER instead of
|
|
10
|
+
* the path — the address `uploadUrl` is, without its last segment.
|
|
11
|
+
*
|
|
12
|
+
* Two spellings because a token in a URL is a credential in a place that keeps
|
|
13
|
+
* copies: an access log, a proxy log, a shell history, a `ps` listing of the
|
|
14
|
+
* `curl` that sent it. The path form is what `request_file_upload` answers,
|
|
15
|
+
* because one address an agent can paste into any client is the thing that
|
|
16
|
+
* makes this route usable at all; the header form is for a caller that would
|
|
17
|
+
* rather its credential not be written down on the way. Both reach the same
|
|
18
|
+
* handler and are judged identically — same token, same single use.
|
|
19
|
+
*/
|
|
20
|
+
export const AGENT_UPLOAD_HEADER_ROUTE = '/agent/uploads';
|
|
21
|
+
/**
|
|
22
|
+
* Whether `path` is the agent upload route, and so must reach its handler with
|
|
23
|
+
* the body still a STREAM.
|
|
24
|
+
*
|
|
25
|
+
* The app installs a global `express.json()`. It only claims a JSON
|
|
26
|
+
* content-type, but `curl --data-binary @file.zip -H 'content-type:
|
|
27
|
+
* application/json'` is a request an agent can and will make — and once the
|
|
28
|
+
* parser has drained the stream there are no bytes left to store, so the
|
|
29
|
+
* upload would answer "0 bytes received" for a file that was sent in full.
|
|
30
|
+
* Exempting the path is the same move `/api/sync` makes for its HMAC body, and
|
|
31
|
+
* for the same reason: whoever needs the exact bytes has to see them first.
|
|
32
|
+
*
|
|
33
|
+
* Case-insensitive, because Express routing is: `/api/Agent/uploads/x` reaches
|
|
34
|
+
* this router, and a check that said no would let the parser eat that body.
|
|
35
|
+
*/
|
|
36
|
+
export function isAgentUploadRawBodyPath(path) {
|
|
37
|
+
// Trailing slashes trimmed first, so the header form's bare address matches
|
|
38
|
+
// in every spelling Express routes to it (`/api/agent/uploads` and
|
|
39
|
+
// `/api/agent/uploads/` are one route): missing one of them would hand that
|
|
40
|
+
// request to the JSON parser, and the bytes it drains are gone.
|
|
41
|
+
const lower = path.toLowerCase().replace(/\/+$/, '');
|
|
42
|
+
return lower === '/api/agent/uploads' || lower.startsWith('/api/agent/uploads/');
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* `POST /api/agent/uploads/:token` — the one endpoint on this server
|
|
46
|
+
* authenticated by a single-use token and nothing else.
|
|
47
|
+
*
|
|
48
|
+
* It exists because MCP tool arguments are JSON, so every byte an agent sends
|
|
49
|
+
* through a tool passes through the model first. A 37 KB page gets truncated
|
|
50
|
+
* mid-response; a file of regex backslashes fails to parse as a JSON string; a
|
|
51
|
+
* PNG cannot be sent at all. Here the agent asks for a token, sends the file
|
|
52
|
+
* with any HTTP client it has, and then names the token in
|
|
53
|
+
* `apply_file_upload`. The bytes never enter a prompt.
|
|
54
|
+
*
|
|
55
|
+
* What the route itself may do is deliberately almost nothing: it stores bytes
|
|
56
|
+
* against a token, in a directory beside the workspaces root, and answers what
|
|
57
|
+
* it received. It resolves no workspace, writes nothing into one, and commits
|
|
58
|
+
* nothing — every access, platform-file and branch rule is applied later, by
|
|
59
|
+
* the apply tool, against the branch the caller then names. A token that is
|
|
60
|
+
* unknown, spent, expired or someone else's gets one 404 that says which of
|
|
61
|
+
* those it was: none of them.
|
|
62
|
+
*/
|
|
63
|
+
export function createAgentUploadRoutes(deps) {
|
|
64
|
+
const router = express.Router();
|
|
65
|
+
const { uploads } = deps;
|
|
66
|
+
const handle = async (req, res) => {
|
|
67
|
+
const token = tokenOf(req);
|
|
68
|
+
try {
|
|
69
|
+
// THE TOKEN FIRST, before anything about the request is read, parsed,
|
|
70
|
+
// judged or quoted back. This route is authenticated by the token and
|
|
71
|
+
// nothing else, which cuts two ways. Without the check up here, anyone
|
|
72
|
+
// could make the process buffer the deployment's whole upload limit per
|
|
73
|
+
// request against tokens they invented, as many at a time as they liked;
|
|
74
|
+
// and a caller holding no token could learn which of its OTHER guesses
|
|
75
|
+
// were well formed — "that is not a usable file name" is an answer only
|
|
76
|
+
// somebody entitled to send a file should get. One refusal, nothing else.
|
|
77
|
+
// `receive` below asks again, under the same record, because that is
|
|
78
|
+
// where the token is actually spent.
|
|
79
|
+
uploads.assertOpen(token);
|
|
80
|
+
const filename = fileNameOf(req);
|
|
81
|
+
if (filename === null) {
|
|
82
|
+
res.status(400).json({
|
|
83
|
+
error: 'Name the file you are sending: add `?filename=<name>` to the upload URL (or send it as the ' +
|
|
84
|
+
'`x-upload-filename` header). A single file lands under that name, and a name ending in `.zip` is ' +
|
|
85
|
+
'read as an archive.',
|
|
86
|
+
});
|
|
87
|
+
return;
|
|
88
|
+
}
|
|
89
|
+
const invalid = validateFilename(filename);
|
|
90
|
+
if (invalid !== null || filename.includes('/')) {
|
|
91
|
+
res.status(400).json({
|
|
92
|
+
error: `"${filename}" is not a usable file name: ${invalid ?? 'a name cannot contain "/"'}. Send one plain file name.`,
|
|
93
|
+
});
|
|
94
|
+
return;
|
|
95
|
+
}
|
|
96
|
+
// Then the declared length, so a caller sending something far too large
|
|
97
|
+
// is told the limit before it spends the bandwidth. The real total is
|
|
98
|
+
// counted by the store as the bytes arrive — `content-length` is the
|
|
99
|
+
// sender's claim, not a fact.
|
|
100
|
+
const declared = Number.parseInt(req.headers['content-length'] ?? '', 10);
|
|
101
|
+
if (Number.isFinite(declared) && declared > uploads.maxBytes) {
|
|
102
|
+
res.status(413).json({ error: overLimit(declared, uploads.maxBytes) });
|
|
103
|
+
return;
|
|
104
|
+
}
|
|
105
|
+
// The request itself, as a stream: the store writes the bytes to disk as
|
|
106
|
+
// they arrive and never holds the body. It refuses an empty body, one
|
|
107
|
+
// past the limit and an unreadable archive with its own status.
|
|
108
|
+
res.json(await uploads.receive(token, filename, req));
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
if (err instanceof UploadTokenError) {
|
|
112
|
+
res.status(err.status).json({ error: err.message });
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
// The detail stays in the log. An unexpected failure here is a
|
|
116
|
+
// filesystem error, and its message quotes the absolute path of the
|
|
117
|
+
// store's root — a place the caller is told nothing else about.
|
|
118
|
+
log.error('upload failed:', { err });
|
|
119
|
+
res.status(500).json({ error: 'Upload failed' });
|
|
120
|
+
}
|
|
121
|
+
};
|
|
122
|
+
// Both spellings of the one endpoint: the token in the path, or in the
|
|
123
|
+
// `x-upload-token` header on the bare address.
|
|
124
|
+
router.post(AGENT_UPLOAD_ROUTE, handle);
|
|
125
|
+
router.post(AGENT_UPLOAD_HEADER_ROUTE, handle);
|
|
126
|
+
return router;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* The token the sender presented: the `:token` path segment, or the
|
|
130
|
+
* `x-upload-token` header when the bytes went to the bare address.
|
|
131
|
+
*
|
|
132
|
+
* The path wins when both are present — it is the address the sender actually
|
|
133
|
+
* POSTed to, and a header left over from an earlier upload must not quietly
|
|
134
|
+
* redirect these bytes onto a different token.
|
|
135
|
+
*
|
|
136
|
+
* No token at all answers the empty string rather than its own refusal, so it
|
|
137
|
+
* goes through `assertOpen` like any other unusable token and gets the same
|
|
138
|
+
* single 404. A caller holding nothing is told nothing it did not already
|
|
139
|
+
* know — not even whether this endpoint wanted a header.
|
|
140
|
+
*/
|
|
141
|
+
function tokenOf(req) {
|
|
142
|
+
const inPath = req.params.token;
|
|
143
|
+
if (typeof inPath === 'string' && inPath.trim() !== '')
|
|
144
|
+
return inPath.trim();
|
|
145
|
+
const header = req.headers['x-upload-token'];
|
|
146
|
+
if (typeof header === 'string' && header.trim() !== '')
|
|
147
|
+
return header.trim();
|
|
148
|
+
return '';
|
|
149
|
+
}
|
|
150
|
+
/**
|
|
151
|
+
* The name the sender gave the file: the `filename` query parameter, the
|
|
152
|
+
* `x-upload-filename` header, or a `content-disposition`'s own `filename=`.
|
|
153
|
+
* Three spellings because three kinds of client are expected to use this —
|
|
154
|
+
* `curl` with a query string, a scripted `fetch` with a header, and a client
|
|
155
|
+
* that sends the disposition it would send to any upload endpoint.
|
|
156
|
+
*/
|
|
157
|
+
function fileNameOf(req) {
|
|
158
|
+
const query = req.query.filename;
|
|
159
|
+
if (typeof query === 'string' && query.trim() !== '')
|
|
160
|
+
return query.trim();
|
|
161
|
+
const header = req.headers['x-upload-filename'];
|
|
162
|
+
if (typeof header === 'string' && header.trim() !== '')
|
|
163
|
+
return header.trim();
|
|
164
|
+
const disposition = req.headers['content-disposition'];
|
|
165
|
+
if (typeof disposition === 'string')
|
|
166
|
+
return dispositionFilename(disposition);
|
|
167
|
+
return null;
|
|
168
|
+
}
|
|
169
|
+
/**
|
|
170
|
+
* The `filename` a `content-disposition` names, or null when it names none.
|
|
171
|
+
*
|
|
172
|
+
* A QUOTED value is read to its closing quote, not to the first semicolon: a
|
|
173
|
+
* semicolon separates the header's parameters only OUTSIDE the quotes, and
|
|
174
|
+
* `filename="report;final.md"` is one perfectly ordinary name that a
|
|
175
|
+
* semicolon-first reading landed as `report`. An unquoted value ends at the
|
|
176
|
+
* next parameter, as it must.
|
|
177
|
+
*
|
|
178
|
+
* Percent-decoded only in the extended `filename*=UTF-8''…` form, which is the
|
|
179
|
+
* only one where the encoding is part of the grammar. A plain `filename=` value
|
|
180
|
+
* is the name itself, so `50%20off.md` stays `50%20off.md` rather than losing
|
|
181
|
+
* its `%20` to a decode nobody asked for.
|
|
182
|
+
*
|
|
183
|
+
* The parameter name is matched at a boundary — the start of the header or a
|
|
184
|
+
* `;` — because `filename` is a suffix of other perfectly legal parameter
|
|
185
|
+
* names: without it, `inline; xfilename=wrong.md` read `wrong.md` as the name
|
|
186
|
+
* the sender gave, from a parameter that says nothing of the kind.
|
|
187
|
+
*/
|
|
188
|
+
export function dispositionFilename(disposition) {
|
|
189
|
+
const match = /(?:^|;)\s*filename(\*?)\s*=\s*(?:"([^"]*)"|([^;]*))/i.exec(disposition);
|
|
190
|
+
if (!match)
|
|
191
|
+
return null;
|
|
192
|
+
const extended = match[1] === '*';
|
|
193
|
+
const raw = (match[2] ?? match[3] ?? '').trim();
|
|
194
|
+
if (!extended)
|
|
195
|
+
return raw === '' ? null : raw;
|
|
196
|
+
// `UTF-8''name`, or any other charset and language the sender declares.
|
|
197
|
+
const encoded = raw.replace(/^[^']*'[^']*'/, '');
|
|
198
|
+
try {
|
|
199
|
+
const decoded = decodeURIComponent(encoded).trim();
|
|
200
|
+
if (decoded !== '')
|
|
201
|
+
return decoded;
|
|
202
|
+
}
|
|
203
|
+
catch {
|
|
204
|
+
const kept = encoded.trim();
|
|
205
|
+
if (kept !== '')
|
|
206
|
+
return kept;
|
|
207
|
+
}
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
//# sourceMappingURL=agent-upload.routes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-upload.routes.js","sourceRoot":"","sources":["../../../src/modules/workspace/agent-upload.routes.ts"],"names":[],"mappings":"AAAA,OAAO,OAAO,MAAM,SAAS,CAAC;AAC9B,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AACnE,OAAO,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AACjD,OAAO,EAAoB,gBAAgB,EAAE,SAAS,EAAE,MAAM,yBAAyB,CAAC;AAExF,MAAM,GAAG,GAAG,MAAM,CAAC,eAAe,CAAC,CAAC;AAEpC,+FAA+F;AAC/F,MAAM,CAAC,MAAM,kBAAkB,GAAG,uBAAuB,CAAC;AAE1D;;;;;;;;;;;GAWG;AACH,MAAM,CAAC,MAAM,yBAAyB,GAAG,gBAAgB,CAAC;AAE1D;;;;;;;;;;;;;;GAcG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAAY;IACnD,4EAA4E;IAC5E,mEAAmE;IACnE,4EAA4E;IAC5E,gEAAgE;IAChE,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IACrD,OAAO,KAAK,KAAK,oBAAoB,IAAI,KAAK,CAAC,UAAU,CAAC,qBAAqB,CAAC,CAAC;AACnF,CAAC;AAMD;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,uBAAuB,CAAC,IAA0B;IAChE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAChC,MAAM,EAAE,OAAO,EAAE,GAAG,IAAI,CAAC;IAEzB,MAAM,MAAM,GAA2B,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC;QAC3B,IAAI,CAAC;YACH,sEAAsE;YACtE,sEAAsE;YACtE,uEAAuE;YACvE,wEAAwE;YACxE,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,0EAA0E;YAC1E,qEAAqE;YACrE,qCAAqC;YACrC,OAAO,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC1B,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YACjC,IAAI,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EACH,6FAA6F;wBAC7F,mGAAmG;wBACnG,qBAAqB;iBACxB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,MAAM,OAAO,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;YAC3C,IAAI,OAAO,KAAK,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC/C,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,IAAI,QAAQ,gCAAgC,OAAO,IAAI,2BAA2B,6BAA6B;iBACvH,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YACD,wEAAwE;YACxE,sEAAsE;YACtE,qEAAqE;YACrE,8BAA8B;YAC9B,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC1E,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;gBAC7D,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvE,OAAO;YACT,CAAC;YACD,yEAAyE;YACzE,sEAAsE;YACtE,gEAAgE;YAChE,GAAG,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,OAAO,CAAC,KAAK,EAAE,QAAQ,EAAE,GAAG,CAAC,CAAC,CAAC;QACxD,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,IAAI,GAAG,YAAY,gBAAgB,EAAE,CAAC;gBACpC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;gBACpD,OAAO;YACT,CAAC;YACD,+DAA+D;YAC/D,oEAAoE;YACpE,gEAAgE;YAChE,GAAG,CAAC,KAAK,CAAC,gBAAgB,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;YACrC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAC;QACnD,CAAC;IACH,CAAC,CAAC;IAEF,uEAAuE;IACvE,+CAA+C;IAC/C,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;IACxC,MAAM,CAAC,IAAI,CAAC,yBAAyB,EAAE,MAAM,CAAC,CAAC;IAE/C,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,OAAO,CAAC,GAAoB;IACnC,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC;IAChC,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7E,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAC7C,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7E,OAAO,EAAE,CAAC;AACZ,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,GAAoB;IACtC,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC;IACjC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,KAAK,CAAC,IAAI,EAAE,CAAC;IAC1E,MAAM,MAAM,GAAG,GAAG,CAAC,OAAO,CAAC,mBAAmB,CAAC,CAAC;IAChD,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,MAAM,CAAC,IAAI,EAAE,KAAK,EAAE;QAAE,OAAO,MAAM,CAAC,IAAI,EAAE,CAAC;IAC7E,MAAM,WAAW,GAAG,GAAG,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC;IACvD,IAAI,OAAO,WAAW,KAAK,QAAQ;QAAE,OAAO,mBAAmB,CAAC,WAAW,CAAC,CAAC;IAC7E,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;;;;;;;;;;;;;;GAkBG;AACH,MAAM,UAAU,mBAAmB,CAAC,WAAmB;IACrD,MAAM,KAAK,GAAG,sDAAsD,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IACvF,IAAI,CAAC,KAAK;QAAE,OAAO,IAAI,CAAC;IACxB,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC;IAClC,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,IAAI,CAAC,QAAQ;QAAE,OAAO,GAAG,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC;IAC9C,wEAAwE;IACxE,MAAM,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,EAAE,CAAC,CAAC;IACjD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,OAAO,KAAK,EAAE;YAAE,OAAO,OAAO,CAAC;IACrC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC5B,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;IAC/B,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/** How long an issued token, and the bytes sent against it, stay usable. */
|
|
2
|
+
export declare const UPLOAD_TOKEN_TTL_MS: number;
|
|
3
|
+
/**
|
|
4
|
+
* THE one refusal for every way a token can fail to be usable: it was never
|
|
5
|
+
* issued, it has already been applied, it has expired, or it belongs to
|
|
6
|
+
* somebody else. One sentence for all four, deliberately — an answer that
|
|
7
|
+
* distinguished "no such token" from "not yours" would let anyone holding a
|
|
8
|
+
* guess learn which guesses exist, and the token is the whole credential this
|
|
9
|
+
* route has.
|
|
10
|
+
*/
|
|
11
|
+
export declare const UPLOAD_TOKEN_REFUSAL: string;
|
|
12
|
+
/**
|
|
13
|
+
* How many tokens one user may hold open at once.
|
|
14
|
+
*
|
|
15
|
+
* A token is permission to put the deployment's whole upload limit on its
|
|
16
|
+
* disk for a TTL, and to hold a connection open while it arrives. Without a
|
|
17
|
+
* bound, one caller asks for a hundred and sends against all of them
|
|
18
|
+
* together. Ten is far more than the route's own use needs — one token carries
|
|
19
|
+
* a zip of any number of files — and a token is given back the moment it is
|
|
20
|
+
* applied or expires.
|
|
21
|
+
*/
|
|
22
|
+
export declare const MAX_OPEN_UPLOADS_PER_USER = 10;
|
|
23
|
+
/** The refusal an over-limit upload gets, naming the limit that applied. */
|
|
24
|
+
export declare function overLimit(bytes: number, maxBytes: number): string;
|
|
25
|
+
/** A refusal with the HTTP status the upload route and the apply tool both answer. */
|
|
26
|
+
export declare class UploadTokenError extends Error {
|
|
27
|
+
readonly status: number;
|
|
28
|
+
constructor(message: string, status: number);
|
|
29
|
+
}
|
|
30
|
+
/** What `request_file_upload` answers: where to send the bytes, and the terms. */
|
|
31
|
+
export interface IssuedUpload {
|
|
32
|
+
/** The absolute URL the bytes are POSTed to. Carries the token in its path. */
|
|
33
|
+
uploadUrl: string;
|
|
34
|
+
/**
|
|
35
|
+
* The token itself — what `apply_file_upload` takes, and the credential the
|
|
36
|
+
* upload route is authenticated by.
|
|
37
|
+
*
|
|
38
|
+
* Named separately from `uploadUrl` for two reasons: the apply needs it on
|
|
39
|
+
* its own, and the route takes it either way round. The bytes can go to
|
|
40
|
+
* `uploadUrl`, which carries the token in its last path segment, or to that
|
|
41
|
+
* address WITHOUT that segment with the token in the `x-upload-token`
|
|
42
|
+
* header — the spelling for a caller that would rather its credential not
|
|
43
|
+
* land in an access log or a shell history on the way.
|
|
44
|
+
*/
|
|
45
|
+
token: string;
|
|
46
|
+
/** ISO-8601 instant after which the token and any bytes sent with it are gone. */
|
|
47
|
+
expiresAt: string;
|
|
48
|
+
/** Seconds from now until `expiresAt`, so a caller need not parse the date. */
|
|
49
|
+
expiresInSeconds: number;
|
|
50
|
+
/** The largest upload this deployment accepts, in bytes. */
|
|
51
|
+
maxBytes: number;
|
|
52
|
+
}
|
|
53
|
+
/** What the upload route answers, and what `apply_file_upload` reads back. */
|
|
54
|
+
export interface ReceivedUpload {
|
|
55
|
+
/** The name the sender gave the file. For a single file, the name it lands under. */
|
|
56
|
+
filename: string;
|
|
57
|
+
bytes: number;
|
|
58
|
+
/** `zip` when the name ends in `.zip` and the bytes parse as an archive. */
|
|
59
|
+
kind: 'file' | 'zip';
|
|
60
|
+
/** Present for a zip: how many members the archive holds. */
|
|
61
|
+
entries?: number;
|
|
62
|
+
}
|
|
63
|
+
/** A received upload, with the bytes on disk, as `apply_file_upload` claims it. */
|
|
64
|
+
export interface ClaimedUpload extends ReceivedUpload {
|
|
65
|
+
/** Absolute path of the stored bytes — OUTSIDE every workspace. */
|
|
66
|
+
absolutePath: string;
|
|
67
|
+
}
|
|
68
|
+
export interface AgentUploadStoreOptions {
|
|
69
|
+
/** Directory the bytes are written to. A sibling of `workspacesRoot`, never inside one. */
|
|
70
|
+
root: string;
|
|
71
|
+
/** Absolute base URL of this deployment's API, e.g. `https://core.example.com`. */
|
|
72
|
+
publicBaseUrl: string;
|
|
73
|
+
/** Token prefix, so a leaked credential can be recognised by shape. */
|
|
74
|
+
tokenPrefix?: string;
|
|
75
|
+
ttlMs?: number;
|
|
76
|
+
maxBytes?: number;
|
|
77
|
+
/** How many tokens one user may hold open at once. Defaults to {@link MAX_OPEN_UPLOADS_PER_USER}. */
|
|
78
|
+
maxOpenPerUser?: number;
|
|
79
|
+
/**
|
|
80
|
+
* How the staging root is listed. Test seam — defaults to `fs.readdir`.
|
|
81
|
+
*
|
|
82
|
+
* The ONE thing about the sweep a suite needs to control. Whether a token
|
|
83
|
+
* issued and uploaded while the sweep is running survives depends on the
|
|
84
|
+
* order of two steps inside {@link AgentUploadStore.sweepNow} — the listing
|
|
85
|
+
* and the live-id set — and the gap between them is a filesystem round-trip
|
|
86
|
+
* no test can otherwise sit inside. Given a listing it can hold open, a test
|
|
87
|
+
* can put a whole upload in that gap and assert the bytes are still there.
|
|
88
|
+
*/
|
|
89
|
+
listRoot?: (root: string) => Promise<string[]>;
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* The upload tokens an agent lands files with, and the bytes sent against
|
|
93
|
+
* them.
|
|
94
|
+
*
|
|
95
|
+
* An agent that wants to put 27 files on a branch has no way to do it through
|
|
96
|
+
* a tool argument: the content would have to pass through the model, which
|
|
97
|
+
* truncates, mangles escape characters and cannot carry a PNG at all. So the
|
|
98
|
+
* bytes take a route of their own — this store issues a one-time token, the
|
|
99
|
+
* upload route attaches bytes to it, and `apply_file_upload` lands them.
|
|
100
|
+
*
|
|
101
|
+
* Three properties the whole design rests on, because the upload route is the
|
|
102
|
+
* one endpoint on this server that authenticates by a token alone:
|
|
103
|
+
*
|
|
104
|
+
* - **Unguessable.** 32 random bytes, base64url. Nothing about the token is
|
|
105
|
+
* derived from the user, the time or a counter.
|
|
106
|
+
* - **Never stored in the clear.** Records are keyed by the token's SHA-256,
|
|
107
|
+
* so the store's own memory (and anything that dumps it) holds no usable
|
|
108
|
+
* credential.
|
|
109
|
+
* - **Bound and single-use.** A record carries the id of the user it was
|
|
110
|
+
* issued to and is consumed by the apply that lands it; a second apply, an
|
|
111
|
+
* apply by anyone else, and an apply after the expiry all meet
|
|
112
|
+
* {@link UPLOAD_TOKEN_REFUSAL}.
|
|
113
|
+
*
|
|
114
|
+
* And the bytes land in `root` — a directory beside the workspaces root, not
|
|
115
|
+
* inside one. No file tool can reach it: every workspace path is resolved
|
|
116
|
+
* against a branch's checkout, so there is no spelling of a tool argument that
|
|
117
|
+
* names a file in here. An upload nobody applies is deleted when its token
|
|
118
|
+
* expires (see {@link startSweeping}), so an abandoned drop costs disk for at
|
|
119
|
+
* most one TTL.
|
|
120
|
+
*/
|
|
121
|
+
export declare class AgentUploadStore {
|
|
122
|
+
private readonly records;
|
|
123
|
+
private readonly root;
|
|
124
|
+
private readonly publicBaseUrl;
|
|
125
|
+
private readonly tokenPrefix;
|
|
126
|
+
private readonly ttlMs;
|
|
127
|
+
private readonly listRoot;
|
|
128
|
+
readonly maxBytes: number;
|
|
129
|
+
private readonly maxOpenPerUser;
|
|
130
|
+
private sweepTimer;
|
|
131
|
+
/** The sweep in flight, so a shutdown can wait for it — see {@link drainSweep}. */
|
|
132
|
+
private sweeping;
|
|
133
|
+
/** Set by {@link stopSweeping}: no further sweep deletes anything. */
|
|
134
|
+
private stopped;
|
|
135
|
+
constructor(options: AgentUploadStoreOptions);
|
|
136
|
+
/** The URL a token's bytes are sent to. The one spelling of this route's address. */
|
|
137
|
+
uploadUrlFor(token: string): string;
|
|
138
|
+
/**
|
|
139
|
+
* Mint a token for `user` and answer the terms it is good for. Nothing is
|
|
140
|
+
* written to disk yet — a token nobody uploads against costs one map entry
|
|
141
|
+
* until the sweep drops it.
|
|
142
|
+
*/
|
|
143
|
+
issue(user: {
|
|
144
|
+
id: string;
|
|
145
|
+
}): IssuedUpload;
|
|
146
|
+
/**
|
|
147
|
+
* Store the bytes of `body` against `token` and say what was received. One
|
|
148
|
+
* file per token: a second upload against the same token is refused, so a
|
|
149
|
+
* token cannot be used to keep replacing bytes an apply is about to land.
|
|
150
|
+
*
|
|
151
|
+
* WRITTEN AS THEY ARRIVE, never gathered first. The body is up to the
|
|
152
|
+
* deployment's whole upload limit, and this is the one route a caller can
|
|
153
|
+
* send to without a session: held in memory, a handful of uploads at once
|
|
154
|
+
* cost the process several times that limit each (the chunks, the buffer
|
|
155
|
+
* they were joined into, the archive read over it). Streamed, an upload in
|
|
156
|
+
* flight costs one chunk.
|
|
157
|
+
*
|
|
158
|
+
* A name ending in `.zip` is read as an archive HERE, once the bytes are on
|
|
159
|
+
* disk, so the answer can carry the entry count and so a corrupt archive is
|
|
160
|
+
* refused while the caller is still holding the file — rather than at apply
|
|
161
|
+
* time, when its token would already be spent.
|
|
162
|
+
*/
|
|
163
|
+
receive(token: string, filename: string, body: AsyncIterable<Buffer | string>): Promise<ReceivedUpload>;
|
|
164
|
+
/**
|
|
165
|
+
* Refuse `token` if it cannot accept bytes — unknown, expired, somebody
|
|
166
|
+
* else's doing, or already holding a file — and answer its record if it can.
|
|
167
|
+
*
|
|
168
|
+
* Exists as its own step so the upload route can ask BEFORE it reads the
|
|
169
|
+
* body. The route is the one endpoint here authenticated by a token alone,
|
|
170
|
+
* and buffering up to the deployment's whole upload limit for an invented
|
|
171
|
+
* token would let a handful of concurrent requests spend the process's
|
|
172
|
+
* memory on bytes that were never going to be stored.
|
|
173
|
+
*/
|
|
174
|
+
assertOpen(token: string): void;
|
|
175
|
+
/** {@link assertOpen}, with the record it found — the store's own view of it. */
|
|
176
|
+
private openRecord;
|
|
177
|
+
/**
|
|
178
|
+
* Hold `token` for an apply by `userId`, and answer where its bytes are.
|
|
179
|
+
*
|
|
180
|
+
* A CLAIM rather than a consume, because an apply can be refused whole —
|
|
181
|
+
* the destination is on a protected branch the caller may not write, the
|
|
182
|
+
* archive turned out unreadable — and a token spent on a refusal would make
|
|
183
|
+
* the agent send the same 40 MB again to try a different destination. The
|
|
184
|
+
* claim is what keeps it single-use meanwhile: a second apply arriving while
|
|
185
|
+
* the first runs finds the token in use and is refused. The caller
|
|
186
|
+
* {@link consume}s it once an answer exists, or {@link release}s it on a
|
|
187
|
+
* refusal that landed nothing.
|
|
188
|
+
*/
|
|
189
|
+
claim(token: string, userId: string): ClaimedUpload;
|
|
190
|
+
/** Give a claimed token back, unused — the apply refused without landing anything. */
|
|
191
|
+
release(token: string): void;
|
|
192
|
+
/** Spend the token and delete its bytes. Idempotent. */
|
|
193
|
+
consume(token: string): Promise<void>;
|
|
194
|
+
/**
|
|
195
|
+
* Delete every record whose token has expired, with the bytes it was
|
|
196
|
+
* holding — then delete any file in the root that no LIVE record claims and
|
|
197
|
+
* that has outlived every token that could name it.
|
|
198
|
+
*
|
|
199
|
+
* The second half is what makes the first one true. A process killed between
|
|
200
|
+
* the write and the apply leaves a file no map will ever mention again, and
|
|
201
|
+
* sweeping by what the records DON'T name is the honest reading of the
|
|
202
|
+
* promise: an upload nobody applied does not stay. By AGE rather than at
|
|
203
|
+
* once, because a file this map does not name may be another process's (see
|
|
204
|
+
* the loop below).
|
|
205
|
+
*
|
|
206
|
+
* ONE record is kept past its expiry: one an apply has CLAIMED. Those bytes
|
|
207
|
+
* are being read right now, and a sweep that deleted them would make the
|
|
208
|
+
* commit land short of what the answer promised. The pin lasts
|
|
209
|
+
* {@link CLAIM_GRACE_MS}, after which a claim nobody consumed or released
|
|
210
|
+
* belongs to a dead process and is reclaimed.
|
|
211
|
+
*/
|
|
212
|
+
sweepNow(): Promise<void>;
|
|
213
|
+
/**
|
|
214
|
+
* Whether the stored file `name` is older than any token that could still be
|
|
215
|
+
* naming it, in this process or another. A file that cannot be examined is
|
|
216
|
+
* left for the next sweep rather than judged.
|
|
217
|
+
*/
|
|
218
|
+
private outlivedEveryToken;
|
|
219
|
+
/**
|
|
220
|
+
* Whether an expired record is held open by work that is still running:
|
|
221
|
+
* an UPLOAD still writing its bytes, or an APPLY still reading them.
|
|
222
|
+
*
|
|
223
|
+
* Both windows can outlast a TTL — a large upload over a slow link, an apply
|
|
224
|
+
* that clones a branch before it reads — and in both the record's own file is
|
|
225
|
+
* being written or read right now. A sweep that deleted either would leave a
|
|
226
|
+
* file no map mentions (the upload writes after the delete) or an apply
|
|
227
|
+
* reading a path that has gone. The apply's pin has a grace, because a claim
|
|
228
|
+
* can be abandoned by a process that dies; the upload's needs none, because
|
|
229
|
+
* `receive` always ends — it clears `attaching` on failure and sets
|
|
230
|
+
* `received` on success, and one whose TTL passed meanwhile deletes the
|
|
231
|
+
* record itself rather than leaving it pinned.
|
|
232
|
+
*/
|
|
233
|
+
private pinned;
|
|
234
|
+
/**
|
|
235
|
+
* Sweep now, and keep sweeping. Started by the first {@link issue} rather
|
|
236
|
+
* than at boot, so a deployment nobody uploads to runs no timer; idempotent,
|
|
237
|
+
* and the timer is `unref`'d because nothing here is worth keeping a process
|
|
238
|
+
* alive for — the records are in memory and go with it.
|
|
239
|
+
*
|
|
240
|
+
* The first sweep runs IMMEDIATELY, not one interval later, because the
|
|
241
|
+
* records are in memory: a process that restarted holds no record of what
|
|
242
|
+
* the previous one stored, and what that one left behind long enough ago
|
|
243
|
+
* has no reason to wait a further interval.
|
|
244
|
+
*/
|
|
245
|
+
startSweeping(intervalMs?: number): void;
|
|
246
|
+
/**
|
|
247
|
+
* Stop sweeping, for good: the timer is cleared AND a sweep already running
|
|
248
|
+
* abandons the rest of its work.
|
|
249
|
+
*
|
|
250
|
+
* Both halves matter when a graph is stopped — a tenant evicted, the process
|
|
251
|
+
* shutting down. The root is this tenant's, and a reactivation builds a new
|
|
252
|
+
* store over the same directory with an empty record map: a sweep left
|
|
253
|
+
* running from the old store would find the new store's files named by no
|
|
254
|
+
* record of ITS own and delete them, under an apply that is about to read
|
|
255
|
+
* them. {@link drainSweep} is how a caller waits for the abandonment to
|
|
256
|
+
* actually have happened.
|
|
257
|
+
*/
|
|
258
|
+
stopSweeping(): void;
|
|
259
|
+
/** Wait for the sweep in flight, if any. Pairs with {@link stopSweeping} on shutdown. */
|
|
260
|
+
drainSweep(): Promise<void>;
|
|
261
|
+
/** One sweep, with its failure logged and its promise kept for {@link drainSweep}. */
|
|
262
|
+
private sweep;
|
|
263
|
+
/** The live record for `token`, or the one refusal. Expiry is judged here. */
|
|
264
|
+
private find;
|
|
265
|
+
/** Best-effort delete of one stored upload. Never throws. */
|
|
266
|
+
private remove;
|
|
267
|
+
}
|
|
268
|
+
/**
|
|
269
|
+
* Refuse a boot whose upload root is not OUTSIDE every workspace.
|
|
270
|
+
*
|
|
271
|
+
* The whole safety of this route rests on where the bytes land: they are a
|
|
272
|
+
* buffer somebody sent, judged by nothing until `apply_file_upload` judges each
|
|
273
|
+
* path against a branch. A root configured inside `workspacesRoot` would put
|
|
274
|
+
* that unjudged buffer where the file tools read — `read_file`, `grep`, the
|
|
275
|
+
* download route — bypassing the access and platform-file rules the apply
|
|
276
|
+
* exists to apply. The invariant is documented on `AGENT_UPLOADS_ROOT`; this is
|
|
277
|
+
* it checked, at boot, naming the variable, rather than trusted.
|
|
278
|
+
*
|
|
279
|
+
* Both directions and both spellings: equal paths, either containing the
|
|
280
|
+
* other, and the real paths, so a root that is a LINK into the workspaces tree
|
|
281
|
+
* is refused too.
|
|
282
|
+
*/
|
|
283
|
+
export declare function assertUploadsRootOutsideWorkspaces(uploadsRoot: string, workspacesRoot: string): Promise<void>;
|
|
284
|
+
//# sourceMappingURL=agent-upload.store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"agent-upload.store.d.ts","sourceRoot":"","sources":["../../../src/modules/workspace/agent-upload.store.ts"],"names":[],"mappings":"AASA,4EAA4E;AAC5E,eAAO,MAAM,mBAAmB,QAAiB,CAAC;AAmBlD;;;;;;;GAOG;AACH,eAAO,MAAM,oBAAoB,QAEY,CAAC;AAE9C;;;;;;;;;GASG;AACH,eAAO,MAAM,yBAAyB,KAAK,CAAC;AAE5C,4EAA4E;AAC5E,wBAAgB,SAAS,CAAC,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,GAAG,MAAM,CAKjE;AAOD,sFAAsF;AACtF,qBAAa,gBAAiB,SAAQ,KAAK;IAGvC,QAAQ,CAAC,MAAM,EAAE,MAAM;gBADvB,OAAO,EAAE,MAAM,EACN,MAAM,EAAE,MAAM;CAK1B;AAED,kFAAkF;AAClF,MAAM,WAAW,YAAY;IAC3B,+EAA+E;IAC/E,SAAS,EAAE,MAAM,CAAC;IAClB;;;;;;;;;;OAUG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,kFAAkF;IAClF,SAAS,EAAE,MAAM,CAAC;IAClB,+EAA+E;IAC/E,gBAAgB,EAAE,MAAM,CAAC;IACzB,4DAA4D;IAC5D,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,8EAA8E;AAC9E,MAAM,WAAW,cAAc;IAC7B,qFAAqF;IACrF,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,4EAA4E;IAC5E,IAAI,EAAE,MAAM,GAAG,KAAK,CAAC;IACrB,6DAA6D;IAC7D,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,mFAAmF;AACnF,MAAM,WAAW,aAAc,SAAQ,cAAc;IACnD,mEAAmE;IACnE,YAAY,EAAE,MAAM,CAAC;CACtB;AAyBD,MAAM,WAAW,uBAAuB;IACtC,2FAA2F;IAC3F,IAAI,EAAE,MAAM,CAAC;IACb,mFAAmF;IACnF,aAAa,EAAE,MAAM,CAAC;IACtB,uEAAuE;IACvE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qGAAqG;IACrG,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB;;;;;;;;;OASG;IACH,QAAQ,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAChD;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA6BG;AACH,qBAAa,gBAAgB;IAC3B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAmC;IAC3D,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;IACrC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAsC;IAC/D,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAS;IACxC,OAAO,CAAC,UAAU,CAA+C;IACjE,mFAAmF;IACnF,OAAO,CAAC,QAAQ,CAA8B;IAC9C,sEAAsE;IACtE,OAAO,CAAC,OAAO,CAAS;gBAEZ,OAAO,EAAE,uBAAuB;IAU5C,qFAAqF;IACrF,YAAY,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAInC;;;;OAIG;IACH,KAAK,CAAC,IAAI,EAAE;QAAE,EAAE,EAAE,MAAM,CAAA;KAAE,GAAG,YAAY;IA2CzC;;;;;;;;;;;;;;;;OAgBG;IACG,OAAO,CACX,KAAK,EAAE,MAAM,EACb,QAAQ,EAAE,MAAM,EAChB,IAAI,EAAE,aAAa,CAAC,MAAM,GAAG,MAAM,CAAC,GACnC,OAAO,CAAC,cAAc,CAAC;IAqE1B;;;;;;;;;OASG;IACH,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAI/B,iFAAiF;IACjF,OAAO,CAAC,UAAU;IAQlB;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,GAAG,aAAa;IAUnD,sFAAsF;IACtF,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAQ5B,wDAAwD;IAClD,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ3C;;;;;;;;;;;;;;;;;OAiBG;IACG,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;IA6C/B;;;;OAIG;YACW,kBAAkB;IAShC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,MAAM;IAKd;;;;;;;;;;OAUG;IACH,aAAa,CAAC,UAAU,GAAE,MAA0B,GAAG,IAAI;IAQ3D;;;;;;;;;;;OAWG;IACH,YAAY,IAAI,IAAI;IAMpB,yFAAyF;IACnF,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAIjC,sFAAsF;IACtF,OAAO,CAAC,KAAK;IAWb,8EAA8E;IAC9E,OAAO,CAAC,IAAI;IAkBZ,6DAA6D;YAC/C,MAAM;CAOrB;AAMD;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,kCAAkC,CACtD,WAAW,EAAE,MAAM,EACnB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,IAAI,CAAC,CAaf"}
|