@needmoretruth/nmts-cli 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +591 -0
- package/LICENSE +202 -0
- package/LICENSING.md +49 -0
- package/README.ko.md +641 -0
- package/README.md +657 -0
- package/dist/account-proof.js +78 -0
- package/dist/account.js +75 -0
- package/dist/api-key.js +310 -0
- package/dist/api.js +271 -0
- package/dist/args.js +137 -0
- package/dist/artifact-about.js +70 -0
- package/dist/autonomy.js +98 -0
- package/dist/code-access.js +76 -0
- package/dist/code-vault.js +233 -0
- package/dist/collision.js +133 -0
- package/dist/commands/balance.js +93 -0
- package/dist/commands/consent.js +77 -0
- package/dist/commands/create.js +356 -0
- package/dist/commands/env.js +136 -0
- package/dist/commands/expiring.js +167 -0
- package/dist/commands/extend.js +236 -0
- package/dist/commands/get.js +130 -0
- package/dist/commands/kit.js +136 -0
- package/dist/commands/listfile.js +105 -0
- package/dist/commands/login.js +335 -0
- package/dist/commands/logout.js +33 -0
- package/dist/commands/ls.js +202 -0
- package/dist/commands/marks.js +176 -0
- package/dist/commands/mcp.js +139 -0
- package/dist/commands/mode.js +54 -0
- package/dist/commands/on-collision.js +45 -0
- package/dist/commands/organise.js +296 -0
- package/dist/commands/public-code.js +89 -0
- package/dist/commands/pull.js +185 -0
- package/dist/commands/push.js +278 -0
- package/dist/commands/put.js +282 -0
- package/dist/commands/rebuild.js +160 -0
- package/dist/commands/receive.js +125 -0
- package/dist/commands/recovery-list.js +104 -0
- package/dist/commands/recovery.js +291 -0
- package/dist/commands/s3.js +241 -0
- package/dist/commands/settings.js +28 -0
- package/dist/commands/share.js +276 -0
- package/dist/commands/sweep.js +240 -0
- package/dist/commands/trash.js +288 -0
- package/dist/commands/trial.js +264 -0
- package/dist/commands/update.js +169 -0
- package/dist/commands/usage.js +84 -0
- package/dist/commands/verify.js +331 -0
- package/dist/commands/wallet.js +122 -0
- package/dist/commands/whoami.js +33 -0
- package/dist/consent.js +200 -0
- package/dist/credentials.js +311 -0
- package/dist/crypto-surface.js +54 -0
- package/dist/crypto.js +137 -0
- package/dist/download-part.js +144 -0
- package/dist/download-sink.js +213 -0
- package/dist/download.js +163 -0
- package/dist/drive-paths.js +170 -0
- package/dist/environment.js +190 -0
- package/dist/errors.js +71 -0
- package/dist/exit.js +107 -0
- package/dist/expiry.js +115 -0
- package/dist/extend-chain.js +137 -0
- package/dist/extend-plan.js +149 -0
- package/dist/extend-sign.js +130 -0
- package/dist/guards.js +15 -0
- package/dist/help.js +192 -0
- package/dist/human-check.js +54 -0
- package/dist/item-trash.js +25 -0
- package/dist/kit-file.js +133 -0
- package/dist/list-file.js +71 -0
- package/dist/list-view-find.js +71 -0
- package/dist/list-view-order.js +96 -0
- package/dist/main.js +394 -0
- package/dist/manifest-create.js +81 -0
- package/dist/manifest-write.js +244 -0
- package/dist/manifest.js +213 -0
- package/dist/mark-render.js +45 -0
- package/dist/mcp-args.js +154 -0
- package/dist/mcp-tools/context.js +50 -0
- package/dist/mcp-tools/files.js +133 -0
- package/dist/mcp-tools/organise.js +128 -0
- package/dist/mcp-tools/reads.js +89 -0
- package/dist/mcp-tools/share.js +61 -0
- package/dist/mcp.js +142 -0
- package/dist/net-retry.js +84 -0
- package/dist/network.js +43 -0
- package/dist/notice.js +45 -0
- package/dist/product.js +38 -0
- package/dist/progress.js +110 -0
- package/dist/prompt.js +145 -0
- package/dist/rebuild.js +255 -0
- package/dist/recovery-assemble.js +148 -0
- package/dist/recovery-build.js +260 -0
- package/dist/recovery-map-file.js +114 -0
- package/dist/recovery-map.js +191 -0
- package/dist/recovery-release.js +229 -0
- package/dist/recovery-seq.js +63 -0
- package/dist/recovery-source.js +143 -0
- package/dist/registration.js +83 -0
- package/dist/s3/listing.js +134 -0
- package/dist/s3/multipart.js +104 -0
- package/dist/s3/response-sink.js +41 -0
- package/dist/s3/same-file.js +117 -0
- package/dist/s3/server.js +252 -0
- package/dist/s3/sigv4.js +168 -0
- package/dist/s3/staging.js +87 -0
- package/dist/s3/xml.js +80 -0
- package/dist/safe-path.js +89 -0
- package/dist/seal.js +206 -0
- package/dist/secret-reader.js +112 -0
- package/dist/server.js +33 -0
- package/dist/session.js +49 -0
- package/dist/setup-questions.js +31 -0
- package/dist/share.js +195 -0
- package/dist/shared/lib/crypto/size-padding.js +126 -0
- package/dist/shared/lib/drive/manifest-codec.js +287 -0
- package/dist/shared/lib/drive/manifest-index.js +235 -0
- package/dist/shared/lib/drive/manifest-ops.js +267 -0
- package/dist/shared/lib/drive/manifest-settings.js +65 -0
- package/dist/shared/lib/drive/name-conflict.js +121 -0
- package/dist/shared/lib/drive/unique-name.js +48 -0
- package/dist/shared/lib/extend/epochs.js +256 -0
- package/dist/shared/lib/net/retry-budget.js +95 -0
- package/dist/shared/lib/share/shared-file-info.js +65 -0
- package/dist/shared/lib/storage-network.js +64 -0
- package/dist/shared/lib/upload/part-plan.js +28 -0
- package/dist/stdout.js +109 -0
- package/dist/trash-sweep.js +123 -0
- package/dist/units.js +19 -0
- package/dist/update-check.js +164 -0
- package/dist/update-source.js +143 -0
- package/dist/upload-api.js +98 -0
- package/dist/upload-file.js +242 -0
- package/dist/upload-price.js +117 -0
- package/dist/upload-steps.js +111 -0
- package/dist/upload-store.js +252 -0
- package/dist/upload-wire.js +25 -0
- package/dist/upload.js +294 -0
- package/dist/usage-report.js +54 -0
- package/dist/wallet-chain.js +76 -0
- package/dist/wallet.js +209 -0
- package/dist/walrus-write.js +188 -0
- package/dist/walrus.js +156 -0
- package/package.json +56 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm.d.ts +469 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm.js +1300 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm +0 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm.d.ts +55 -0
- package/vendor/nmts-crypto/package.json +17 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// The gateway itself: an S3 request in, this account's drive out.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ LOOPBACK ONLY, AND NO OPTION TO CHANGE IT. The machine running this already holds the account
|
|
4
|
+
// code, and one signature is all that stands between a request and every file in the account.
|
|
5
|
+
// Bound to an address other people can reach, that one signature becomes the whole lock on the
|
|
6
|
+
// account -- and the key it checks was printed on somebody's terminal. This is the same call the
|
|
7
|
+
// rest of the system made on 2026-08-20 when every container port was pulled back to loopback.
|
|
8
|
+
//
|
|
9
|
+
// ⛔ WHAT IS NOT ANSWERED IS REFUSED, LOUDLY. An S3 client that asks for something this gateway does
|
|
10
|
+
// not do gets 501 and a sentence naming what it does do. The alternative -- answering an empty
|
|
11
|
+
// listing, or a 200 with nothing behind it -- is how a backup tool reports success over a backup
|
|
12
|
+
// that never happened.
|
|
13
|
+
import { createServer } from "node:http";
|
|
14
|
+
import { randomBytes } from "node:crypto";
|
|
15
|
+
import { BUCKET, listObjects, objectsOf, folderPrefixesOf, MAX_KEYS_LIMIT } from "./listing.js";
|
|
16
|
+
import { handleMultipart, isMultipartRequest } from "./multipart.js";
|
|
17
|
+
import { responseSink } from "./response-sink.js";
|
|
18
|
+
import { isKeyConflict } from "./same-file.js";
|
|
19
|
+
import { STREAMING_PAYLOAD, STREAMING_PAYLOAD_TRAILER, verifySignature, } from "./sigv4.js";
|
|
20
|
+
import { errorXml, listBucketsXml, listObjectsXml } from "./xml.js";
|
|
21
|
+
/** Where the drive is served. Loopback, always — see the note above. */
|
|
22
|
+
export const BIND_ADDRESS = "127.0.0.1";
|
|
23
|
+
/** A random pair, made fresh every time the gateway starts and stored nowhere. */
|
|
24
|
+
export function newCredential() {
|
|
25
|
+
const letters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
|
|
26
|
+
const raw = randomBytes(20);
|
|
27
|
+
let id = "NMTS";
|
|
28
|
+
for (const byte of raw)
|
|
29
|
+
id += letters[byte % letters.length] ?? "A";
|
|
30
|
+
return { accessKeyId: id.slice(0, 20), secretAccessKey: randomBytes(30).toString("base64url") };
|
|
31
|
+
}
|
|
32
|
+
function fail(res, status, code, message, resource) {
|
|
33
|
+
const body = errorXml(code, message, resource);
|
|
34
|
+
res.writeHead(status, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
35
|
+
res.end(body);
|
|
36
|
+
}
|
|
37
|
+
/** `/drive/photos/a.jpg` → bucket `drive`, key `photos/a.jpg`. */
|
|
38
|
+
function splitPath(pathname) {
|
|
39
|
+
const trimmed = pathname.replace(/^\//, "");
|
|
40
|
+
const at = trimmed.indexOf("/");
|
|
41
|
+
if (at < 0)
|
|
42
|
+
return { bucket: decodeURIComponent(trimmed), key: "" };
|
|
43
|
+
return { bucket: decodeURIComponent(trimmed.slice(0, at)), key: decodeURIComponent(trimmed.slice(at + 1)) };
|
|
44
|
+
}
|
|
45
|
+
function headerOf(req, name) {
|
|
46
|
+
const raw = req.headers[name];
|
|
47
|
+
return Array.isArray(raw) ? raw.join(",") : raw;
|
|
48
|
+
}
|
|
49
|
+
/** The one sentence a write gets when this machine has not agreed to spending. */
|
|
50
|
+
function readOnlyBecause() {
|
|
51
|
+
return ("This gateway is read only. Uploading spends credits, and this machine has not agreed to " +
|
|
52
|
+
"spending — `nmts consent grant spend`, run by the person whose account this is, is what " +
|
|
53
|
+
"changes that. Nothing was written.");
|
|
54
|
+
}
|
|
55
|
+
function objectHeaders(object) {
|
|
56
|
+
return {
|
|
57
|
+
"content-type": "application/octet-stream",
|
|
58
|
+
"last-modified": new Date(object.entry.updatedAt).toUTCString(),
|
|
59
|
+
etag: object.etag,
|
|
60
|
+
"accept-ranges": "none",
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
async function handle(req, res, options) {
|
|
64
|
+
const url = req.url ?? "/";
|
|
65
|
+
const at = url.indexOf("?");
|
|
66
|
+
const pathname = at < 0 ? url : url.slice(0, at);
|
|
67
|
+
const query = new URLSearchParams(at < 0 ? "" : url.slice(at + 1));
|
|
68
|
+
const method = (req.method ?? "GET").toUpperCase();
|
|
69
|
+
const verdict = verifySignature({ method, url, headers: req.headers }, options.credential, options.now?.() ?? Date.now());
|
|
70
|
+
if (!verdict.ok) {
|
|
71
|
+
fail(res, verdict.code === "InvalidAccessKeyId" ? 403 : 403, verdict.code, verdict.message, pathname);
|
|
72
|
+
return;
|
|
73
|
+
}
|
|
74
|
+
const { bucket, key } = splitPath(pathname);
|
|
75
|
+
if (pathname === "/" && (method === "GET" || method === "HEAD")) {
|
|
76
|
+
const body = listBucketsXml(BUCKET, new Date(0).toISOString());
|
|
77
|
+
res.writeHead(200, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
78
|
+
res.end(method === "HEAD" ? undefined : body);
|
|
79
|
+
return;
|
|
80
|
+
}
|
|
81
|
+
if (bucket !== BUCKET) {
|
|
82
|
+
fail(res, 404, "NoSuchBucket", `This gateway serves one bucket, named ${BUCKET}.`, pathname);
|
|
83
|
+
return;
|
|
84
|
+
}
|
|
85
|
+
// ⛔ MEASURED, NOT GUESSED: rclone's first act when copying a file is to create the bucket, and a
|
|
86
|
+
// refusal here ends the copy before the upload is ever attempted. The bucket exists, so the
|
|
87
|
+
// honest answer to "make it" is that it is made.
|
|
88
|
+
if (key === "" && method === "PUT") {
|
|
89
|
+
res.writeHead(200, { "content-length": "0" });
|
|
90
|
+
res.end();
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
const entries = await options.source.entries();
|
|
94
|
+
if (key === "" && (method === "GET" || method === "HEAD")) {
|
|
95
|
+
const objects = objectsOf(entries);
|
|
96
|
+
const listing = listObjects(objects, folderPrefixesOf(entries), {
|
|
97
|
+
prefix: query.get("prefix") ?? "",
|
|
98
|
+
delimiter: query.get("delimiter") ?? "",
|
|
99
|
+
maxKeys: Number(query.get("max-keys") ?? MAX_KEYS_LIMIT) || MAX_KEYS_LIMIT,
|
|
100
|
+
after: query.get("continuation-token") ?? query.get("start-after") ?? query.get("marker"),
|
|
101
|
+
});
|
|
102
|
+
const body = listObjectsXml({
|
|
103
|
+
bucket: BUCKET,
|
|
104
|
+
prefix: query.get("prefix") ?? "",
|
|
105
|
+
delimiter: query.get("delimiter") ?? "",
|
|
106
|
+
maxKeys: Number(query.get("max-keys") ?? MAX_KEYS_LIMIT) || MAX_KEYS_LIMIT,
|
|
107
|
+
v2: query.get("list-type") === "2",
|
|
108
|
+
contents: listing.contents,
|
|
109
|
+
commonPrefixes: listing.commonPrefixes,
|
|
110
|
+
truncated: listing.truncated,
|
|
111
|
+
next: listing.next,
|
|
112
|
+
encodingType: query.get("encoding-type"),
|
|
113
|
+
});
|
|
114
|
+
res.writeHead(200, { "content-type": "application/xml", "content-length": String(Buffer.byteLength(body)) });
|
|
115
|
+
res.end(method === "HEAD" ? undefined : body);
|
|
116
|
+
options.log?.(`${method} list prefix=${query.get("prefix") ?? ""} → ${listing.contents.length} keys`);
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
if (method === "HEAD" || method === "GET") {
|
|
120
|
+
const object = objectsOf(entries).find((o) => o.key === key);
|
|
121
|
+
if (object === undefined) {
|
|
122
|
+
fail(res, 404, "NoSuchKey", "This account's file list has no such file.", pathname);
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
if (method === "HEAD") {
|
|
126
|
+
res.writeHead(200, { ...objectHeaders(object), "content-length": String(object.size) });
|
|
127
|
+
res.end();
|
|
128
|
+
options.log?.(`HEAD ${key}`);
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (object.entry.dekWrapped === undefined) {
|
|
132
|
+
fail(res, 500, "InternalError", "That entry has no key in the file list.", pathname);
|
|
133
|
+
return;
|
|
134
|
+
}
|
|
135
|
+
const sink = responseSink(res, { headers: objectHeaders(object) });
|
|
136
|
+
try {
|
|
137
|
+
await options.source.fetch(object, sink);
|
|
138
|
+
options.log?.(`GET ${key} → ${object.size} bytes`);
|
|
139
|
+
}
|
|
140
|
+
catch (error) {
|
|
141
|
+
await sink.abandon();
|
|
142
|
+
if (!res.headersSent) {
|
|
143
|
+
fail(res, 502, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
144
|
+
}
|
|
145
|
+
options.log?.(`GET ${key} → failed`);
|
|
146
|
+
}
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
const writer = options.source.write;
|
|
150
|
+
// ⛔ WHETHER A TAKEN KEY IS A CONFLICT IS NOT DECIDED HERE.
|
|
151
|
+
// It used to be, on the strength of the NAME alone, and both upload paths carried their own
|
|
152
|
+
// copy of that check. The question is now about CONTENT — is the file arriving the file
|
|
153
|
+
// already there — and it cannot be answered until the bytes have arrived, so it is answered
|
|
154
|
+
// once, by the writer, at the point both paths meet. What reaches this layer is the verdict:
|
|
155
|
+
// a writer that returns normally means the key now holds these bytes (whether it had to send
|
|
156
|
+
// them or they were already there), and one that throws a conflict means something else is at
|
|
157
|
+
// that key. ⭐ The status matters: 409 is a request the drive declined, 500 is a fault of ours.
|
|
158
|
+
const refuseConflict = (error) => {
|
|
159
|
+
fail(res, 409, "InvalidRequest", error instanceof Error ? error.message : String(error), pathname);
|
|
160
|
+
};
|
|
161
|
+
if (isMultipartRequest(method, query) && key !== "") {
|
|
162
|
+
if (writer === undefined) {
|
|
163
|
+
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
const handled = await handleMultipart({
|
|
167
|
+
req,
|
|
168
|
+
res,
|
|
169
|
+
bucket: BUCKET,
|
|
170
|
+
key,
|
|
171
|
+
method,
|
|
172
|
+
query,
|
|
173
|
+
writer,
|
|
174
|
+
payloadHash: /^[0-9a-f]{64}$/.test(verdict.payloadHash) ? verdict.payloadHash : null,
|
|
175
|
+
fail: (status, code, message) => fail(res, status, code, message, pathname),
|
|
176
|
+
...(options.log === undefined ? {} : { log: options.log }),
|
|
177
|
+
});
|
|
178
|
+
if (handled)
|
|
179
|
+
return;
|
|
180
|
+
}
|
|
181
|
+
if (method === "PUT" && key !== "") {
|
|
182
|
+
if (writer === undefined) {
|
|
183
|
+
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
184
|
+
return;
|
|
185
|
+
}
|
|
186
|
+
const declared = headerOf(req, "x-amz-content-sha256");
|
|
187
|
+
if (declared === STREAMING_PAYLOAD || declared === STREAMING_PAYLOAD_TRAILER) {
|
|
188
|
+
fail(res, 501, "NotImplemented", "This gateway does not read chunk-signed uploads yet. Tell the client to send the body " +
|
|
189
|
+
"unsigned (the AWS CLI calls this --no-sign-payload on http endpoints; rclone already " +
|
|
190
|
+
"does it).", pathname);
|
|
191
|
+
return;
|
|
192
|
+
}
|
|
193
|
+
const length = Number(headerOf(req, "content-length") ?? "");
|
|
194
|
+
if (!Number.isInteger(length) || length < 0) {
|
|
195
|
+
fail(res, 411, "MissingContentLength", "This gateway needs to know the size before it starts.", pathname);
|
|
196
|
+
return;
|
|
197
|
+
}
|
|
198
|
+
try {
|
|
199
|
+
await writer.put(key, req, length);
|
|
200
|
+
}
|
|
201
|
+
catch (error) {
|
|
202
|
+
if (isKeyConflict(error)) {
|
|
203
|
+
refuseConflict(error);
|
|
204
|
+
return;
|
|
205
|
+
}
|
|
206
|
+
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
207
|
+
return;
|
|
208
|
+
}
|
|
209
|
+
res.writeHead(200, { "content-length": "0" });
|
|
210
|
+
res.end();
|
|
211
|
+
options.log?.(`PUT ${key} → ${length} bytes`);
|
|
212
|
+
return;
|
|
213
|
+
}
|
|
214
|
+
if (method === "DELETE" && key !== "") {
|
|
215
|
+
if (writer === undefined) {
|
|
216
|
+
fail(res, 501, "NotImplemented", readOnlyBecause(), pathname);
|
|
217
|
+
return;
|
|
218
|
+
}
|
|
219
|
+
const object = objectsOf(entries).find((o) => o.key === key);
|
|
220
|
+
if (object === undefined) {
|
|
221
|
+
// S3 answers 204 for a key that is not there, and clients rely on it: a sync that deletes
|
|
222
|
+
// the same key twice must not fail the second time.
|
|
223
|
+
res.writeHead(204);
|
|
224
|
+
res.end();
|
|
225
|
+
return;
|
|
226
|
+
}
|
|
227
|
+
try {
|
|
228
|
+
await writer.trash(object);
|
|
229
|
+
}
|
|
230
|
+
catch (error) {
|
|
231
|
+
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), pathname);
|
|
232
|
+
return;
|
|
233
|
+
}
|
|
234
|
+
res.writeHead(204);
|
|
235
|
+
res.end();
|
|
236
|
+
options.log?.(`DELETE ${key} → trash`);
|
|
237
|
+
return;
|
|
238
|
+
}
|
|
239
|
+
fail(res, 501, "NotImplemented", `This gateway does not answer ${method} on that address.`, pathname);
|
|
240
|
+
}
|
|
241
|
+
export function createGateway(options) {
|
|
242
|
+
return createServer((req, res) => {
|
|
243
|
+
handle(req, res, options).catch((error) => {
|
|
244
|
+
if (!res.headersSent) {
|
|
245
|
+
fail(res, 500, "InternalError", error instanceof Error ? error.message : String(error), req.url ?? "/");
|
|
246
|
+
}
|
|
247
|
+
else {
|
|
248
|
+
res.destroy();
|
|
249
|
+
}
|
|
250
|
+
});
|
|
251
|
+
});
|
|
252
|
+
}
|
package/dist/s3/sigv4.js
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
// AWS Signature Version 4, the verifying half.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ WHY A LOCAL SERVER CHECKS SIGNATURES AT ALL. This gateway listens on the loopback address of a
|
|
4
|
+
// machine that already holds an account code, and a request that reaches it can upload, read and
|
|
5
|
+
// delete that account's files. "It is only local" is not an argument: every other program on the
|
|
6
|
+
// machine, and every page a browser on it loads, can also reach 127.0.0.1. The signature is the
|
|
7
|
+
// one thing that separates the tool the person started from everything else running as them.
|
|
8
|
+
//
|
|
9
|
+
// ⛔ WHAT IS DELIBERATELY NOT HERE. The body is not hashed here. The signature covers a payload hash
|
|
10
|
+
// the client DECLARES in `x-amz-content-sha256`, and whether the bytes that follow really hash to
|
|
11
|
+
// that value can only be known once they have all arrived. This module returns the declared value
|
|
12
|
+
// and the caller enforces it while streaming -- doing it here would mean holding whole uploads in
|
|
13
|
+
// memory before a single byte reached the storage network.
|
|
14
|
+
//
|
|
15
|
+
// The rules implemented are S3's, which differ from the generic SigV4 ones in one way that matters:
|
|
16
|
+
// the canonical path is the request path EXACTLY as it arrived, neither normalised nor re-encoded.
|
|
17
|
+
import { createHash, createHmac, timingSafeEqual } from "node:crypto";
|
|
18
|
+
/** How far a request's own timestamp may sit from ours before it is refused. AWS uses the same. */
|
|
19
|
+
export const MAX_CLOCK_SKEW_MS = 15 * 60 * 1000;
|
|
20
|
+
/** The literals a client may put in `x-amz-content-sha256` instead of a hex digest. */
|
|
21
|
+
export const UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD";
|
|
22
|
+
export const STREAMING_PAYLOAD = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
|
|
23
|
+
export const STREAMING_PAYLOAD_TRAILER = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER";
|
|
24
|
+
function refuse(code, message) {
|
|
25
|
+
return { ok: false, code, message };
|
|
26
|
+
}
|
|
27
|
+
function headerValue(headers, name) {
|
|
28
|
+
const raw = headers[name];
|
|
29
|
+
if (raw === undefined)
|
|
30
|
+
return undefined;
|
|
31
|
+
return Array.isArray(raw) ? raw.join(",") : String(raw);
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* RFC 3986 encoding, which is what SigV4 means by "URI-encode".
|
|
35
|
+
*
|
|
36
|
+
* ⚠ `encodeURIComponent` leaves `!'()*` alone and AWS does not, so those four are finished by hand.
|
|
37
|
+
* A query string that contains one of them and is encoded the JavaScript way produces a different
|
|
38
|
+
* canonical request from the client's, and the request is refused for no reason a person can see.
|
|
39
|
+
*/
|
|
40
|
+
function uriEncode(value) {
|
|
41
|
+
return encodeURIComponent(value).replace(/[!'()*]/g, (c) => `%${c.charCodeAt(0).toString(16).toUpperCase()}`);
|
|
42
|
+
}
|
|
43
|
+
/** `k=v&k2=v2` in the order AWS wants: encoded, sorted by key and then by value. */
|
|
44
|
+
export function canonicalQuery(rawQuery) {
|
|
45
|
+
if (rawQuery.length === 0)
|
|
46
|
+
return "";
|
|
47
|
+
const pairs = [];
|
|
48
|
+
for (const part of rawQuery.split("&")) {
|
|
49
|
+
if (part.length === 0)
|
|
50
|
+
continue;
|
|
51
|
+
const at = part.indexOf("=");
|
|
52
|
+
const key = at < 0 ? part : part.slice(0, at);
|
|
53
|
+
const value = at < 0 ? "" : part.slice(at + 1);
|
|
54
|
+
pairs.push([uriEncode(decodeURIComponent(key)), uriEncode(decodeURIComponent(value))]);
|
|
55
|
+
}
|
|
56
|
+
pairs.sort((a, b) => (a[0] === b[0] ? (a[1] < b[1] ? -1 : 1) : a[0] < b[0] ? -1 : 1));
|
|
57
|
+
return pairs.map(([k, v]) => `${k}=${v}`).join("&");
|
|
58
|
+
}
|
|
59
|
+
/** Pull apart `AWS4-HMAC-SHA256 Credential=…, SignedHeaders=…, Signature=…`. */
|
|
60
|
+
export function parseAuthorization(header) {
|
|
61
|
+
if (header === undefined || !header.startsWith("AWS4-HMAC-SHA256 "))
|
|
62
|
+
return null;
|
|
63
|
+
const fields = new Map();
|
|
64
|
+
for (const part of header.slice("AWS4-HMAC-SHA256 ".length).split(",")) {
|
|
65
|
+
const at = part.indexOf("=");
|
|
66
|
+
if (at < 0)
|
|
67
|
+
return null;
|
|
68
|
+
fields.set(part.slice(0, at).trim(), part.slice(at + 1).trim());
|
|
69
|
+
}
|
|
70
|
+
const credential = fields.get("Credential");
|
|
71
|
+
const signedHeaders = fields.get("SignedHeaders");
|
|
72
|
+
const signature = fields.get("Signature");
|
|
73
|
+
if (credential === undefined || signedHeaders === undefined || signature === undefined)
|
|
74
|
+
return null;
|
|
75
|
+
const scope = credential.split("/");
|
|
76
|
+
if (scope.length !== 5 || scope[4] !== "aws4_request")
|
|
77
|
+
return null;
|
|
78
|
+
const [accessKeyId, date, region, service] = scope;
|
|
79
|
+
if (accessKeyId === undefined || date === undefined || region === undefined || service === undefined) {
|
|
80
|
+
return null;
|
|
81
|
+
}
|
|
82
|
+
return {
|
|
83
|
+
accessKeyId,
|
|
84
|
+
date,
|
|
85
|
+
region,
|
|
86
|
+
service,
|
|
87
|
+
signedHeaders: signedHeaders.split(";").filter((h) => h.length > 0),
|
|
88
|
+
signature,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
function signingKey(secret, date, region, service) {
|
|
92
|
+
const kDate = createHmac("sha256", `AWS4${secret}`).update(date).digest();
|
|
93
|
+
const kRegion = createHmac("sha256", kDate).update(region).digest();
|
|
94
|
+
const kService = createHmac("sha256", kRegion).update(service).digest();
|
|
95
|
+
return createHmac("sha256", kService).update("aws4_request").digest();
|
|
96
|
+
}
|
|
97
|
+
/** `20260824T232759Z` → epoch milliseconds, or null if it is not that shape. */
|
|
98
|
+
export function amzDateToMs(stamp) {
|
|
99
|
+
if (stamp === undefined)
|
|
100
|
+
return null;
|
|
101
|
+
const m = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
|
|
102
|
+
if (m === null)
|
|
103
|
+
return null;
|
|
104
|
+
return Date.UTC(Number(m[1]), Number(m[2]) - 1, Number(m[3]), Number(m[4]), Number(m[5]), Number(m[6]));
|
|
105
|
+
}
|
|
106
|
+
/**
|
|
107
|
+
* Rebuild the string the client signed and check that the signature matches.
|
|
108
|
+
*
|
|
109
|
+
* The clock is passed in rather than read here: a test that cannot choose "now" cannot check the
|
|
110
|
+
* skew rule at all, and that rule is the one that stops a captured request being replayed tomorrow.
|
|
111
|
+
*/
|
|
112
|
+
export function verifySignature(request, credential, now) {
|
|
113
|
+
const auth = parseAuthorization(headerValue(request.headers, "authorization"));
|
|
114
|
+
if (auth === null)
|
|
115
|
+
return refuse("AccessDenied", "no AWS Signature Version 4 authorization header");
|
|
116
|
+
if (auth.accessKeyId !== credential.accessKeyId) {
|
|
117
|
+
return refuse("InvalidAccessKeyId", "that access key is not the one this gateway printed");
|
|
118
|
+
}
|
|
119
|
+
const stamp = headerValue(request.headers, "x-amz-date");
|
|
120
|
+
const signedAt = amzDateToMs(stamp);
|
|
121
|
+
if (signedAt === null)
|
|
122
|
+
return refuse("AccessDenied", "missing or malformed x-amz-date");
|
|
123
|
+
if (Math.abs(now - signedAt) > MAX_CLOCK_SKEW_MS) {
|
|
124
|
+
return refuse("RequestTimeTooSkewed", "the request's own timestamp is too far from this clock");
|
|
125
|
+
}
|
|
126
|
+
if (stamp !== undefined && !stamp.startsWith(auth.date)) {
|
|
127
|
+
return refuse("AccessDenied", "the signature's date does not match x-amz-date");
|
|
128
|
+
}
|
|
129
|
+
const payloadHash = headerValue(request.headers, "x-amz-content-sha256");
|
|
130
|
+
if (payloadHash === undefined) {
|
|
131
|
+
return refuse("AccessDenied", "missing x-amz-content-sha256");
|
|
132
|
+
}
|
|
133
|
+
const canonicalHeaders = [];
|
|
134
|
+
for (const name of auth.signedHeaders) {
|
|
135
|
+
const value = headerValue(request.headers, name);
|
|
136
|
+
if (value === undefined) {
|
|
137
|
+
return refuse("AccessDenied", `the signature covers a header that is not here: ${name}`);
|
|
138
|
+
}
|
|
139
|
+
canonicalHeaders.push(`${name}:${value.trim().replace(/\s+/g, " ")}\n`);
|
|
140
|
+
}
|
|
141
|
+
const at = request.url.indexOf("?");
|
|
142
|
+
const path = at < 0 ? request.url : request.url.slice(0, at);
|
|
143
|
+
const query = at < 0 ? "" : request.url.slice(at + 1);
|
|
144
|
+
const canonicalRequest = [
|
|
145
|
+
request.method.toUpperCase(),
|
|
146
|
+
path,
|
|
147
|
+
canonicalQuery(query),
|
|
148
|
+
canonicalHeaders.join(""),
|
|
149
|
+
auth.signedHeaders.join(";"),
|
|
150
|
+
payloadHash,
|
|
151
|
+
].join("\n");
|
|
152
|
+
const scope = `${auth.date}/${auth.region}/${auth.service}/aws4_request`;
|
|
153
|
+
const stringToSign = [
|
|
154
|
+
"AWS4-HMAC-SHA256",
|
|
155
|
+
stamp,
|
|
156
|
+
scope,
|
|
157
|
+
createHash("sha256").update(canonicalRequest).digest("hex"),
|
|
158
|
+
].join("\n");
|
|
159
|
+
const expected = createHmac("sha256", signingKey(credential.secretAccessKey, auth.date, auth.region, auth.service))
|
|
160
|
+
.update(stringToSign)
|
|
161
|
+
.digest("hex");
|
|
162
|
+
const given = Buffer.from(auth.signature, "utf8");
|
|
163
|
+
const mine = Buffer.from(expected, "utf8");
|
|
164
|
+
if (given.length !== mine.length || !timingSafeEqual(given, mine)) {
|
|
165
|
+
return refuse("SignatureDoesNotMatch", "the signature does not match what was signed");
|
|
166
|
+
}
|
|
167
|
+
return { ok: true, payloadHash };
|
|
168
|
+
}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
// Where the pieces of a multipart upload wait until they are one file.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ ITS OWN MODULE SO IT CAN BE TESTED WITHOUT AN ACCOUNT. What goes wrong here is ordering and
|
|
4
|
+
// integrity -- pieces arrive at the same time and out of order, and one that changed on the way
|
|
5
|
+
// becomes part of a file that opens and is wrong. Both are testable against a real directory
|
|
6
|
+
// with a stub for the one thing that costs money, and neither is testable through a command that
|
|
7
|
+
// starts by opening a session.
|
|
8
|
+
//
|
|
9
|
+
// ⛔ ONE DIRECTORY PER RUN, 0700, AND EVERY PIECE 0600. The pieces are somebody's plaintext; left
|
|
10
|
+
// in a shared temporary directory under a predictable name they would be readable by every other
|
|
11
|
+
// account on the machine, for as long as the upload takes.
|
|
12
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
13
|
+
import { createReadStream, createWriteStream } from "node:fs";
|
|
14
|
+
import { mkdir, rm, stat } from "node:fs/promises";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { Transform } from "node:stream";
|
|
17
|
+
import { pipeline } from "node:stream/promises";
|
|
18
|
+
import { NmtsError } from "../errors.js";
|
|
19
|
+
export function createStaging(root, store) {
|
|
20
|
+
const inFlight = new Map();
|
|
21
|
+
const dirOf = (uploadId) => join(root, uploadId);
|
|
22
|
+
const openUpload = (uploadId) => {
|
|
23
|
+
const upload = inFlight.get(uploadId);
|
|
24
|
+
if (upload === undefined)
|
|
25
|
+
throw new NmtsError("No upload is in progress with that id.");
|
|
26
|
+
return upload;
|
|
27
|
+
};
|
|
28
|
+
return {
|
|
29
|
+
async begin(key) {
|
|
30
|
+
const uploadId = randomUUID();
|
|
31
|
+
await mkdir(dirOf(uploadId), { recursive: true, mode: 0o700 });
|
|
32
|
+
inFlight.set(uploadId, { key, parts: new Set() });
|
|
33
|
+
return uploadId;
|
|
34
|
+
},
|
|
35
|
+
// ⛔ EACH PIECE IS ITS OWN FILE NAMED BY ITS NUMBER, and nothing is appended: a real client
|
|
36
|
+
// sends them concurrently and out of order (measured from rclone: 1, 3, 2).
|
|
37
|
+
async part(uploadId, partNumber, body, size, expectedSha256) {
|
|
38
|
+
const upload = openUpload(uploadId);
|
|
39
|
+
const path = join(dirOf(uploadId), String(partNumber));
|
|
40
|
+
const digest = createHash("sha256");
|
|
41
|
+
const hashing = new Transform({
|
|
42
|
+
transform(chunk, _encoding, next) {
|
|
43
|
+
digest.update(chunk);
|
|
44
|
+
next(null, chunk);
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
await pipeline(body, hashing, createWriteStream(path, { mode: 0o600 }));
|
|
48
|
+
const written = (await stat(path)).size;
|
|
49
|
+
// ⛔ WHAT THE CLIENT SIGNED FOR IS WHAT MUST HAVE ARRIVED. A piece that changed on the way is
|
|
50
|
+
// the one failure a backup cannot notice later.
|
|
51
|
+
if (expectedSha256 !== null && digest.digest("hex") !== expectedSha256) {
|
|
52
|
+
await rm(path, { force: true });
|
|
53
|
+
throw new NmtsError("That part's bytes do not hash to what the request declared.");
|
|
54
|
+
}
|
|
55
|
+
if (written !== size) {
|
|
56
|
+
await rm(path, { force: true });
|
|
57
|
+
throw new NmtsError(`The part said ${size} bytes and ${written} arrived.`);
|
|
58
|
+
}
|
|
59
|
+
upload.parts.add(partNumber);
|
|
60
|
+
return `"${partNumber.toString(16).padStart(32, "0")}-1"`;
|
|
61
|
+
},
|
|
62
|
+
async complete(uploadId) {
|
|
63
|
+
const upload = openUpload(uploadId);
|
|
64
|
+
const numbers = [...upload.parts].sort((a, b) => a - b);
|
|
65
|
+
if (numbers.length === 0)
|
|
66
|
+
throw new NmtsError("That upload has no parts to finish.");
|
|
67
|
+
const whole = join(dirOf(uploadId), "whole");
|
|
68
|
+
const out = createWriteStream(whole, { mode: 0o600 });
|
|
69
|
+
try {
|
|
70
|
+
for (const number of numbers) {
|
|
71
|
+
await pipeline(createReadStream(join(dirOf(uploadId), String(number))), out, { end: false });
|
|
72
|
+
}
|
|
73
|
+
await new Promise((resolve) => out.end(resolve));
|
|
74
|
+
await store(upload.key, whole);
|
|
75
|
+
}
|
|
76
|
+
finally {
|
|
77
|
+
inFlight.delete(uploadId);
|
|
78
|
+
await rm(dirOf(uploadId), { recursive: true, force: true });
|
|
79
|
+
}
|
|
80
|
+
return `"${uploadId.replace(/-/g, "").slice(0, 32)}-${numbers.length}"`;
|
|
81
|
+
},
|
|
82
|
+
async abort(uploadId) {
|
|
83
|
+
inFlight.delete(uploadId);
|
|
84
|
+
await rm(dirOf(uploadId), { recursive: true, force: true });
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
package/dist/s3/xml.js
ADDED
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
// The XML the S3 protocol answers in.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ WRITTEN BY HAND, ON PURPOSE. Pulling in an XML builder to emit five fixed shapes would add a
|
|
4
|
+
// dependency to a tool whose whole dependency list is auditable in one screen, and this is the
|
|
5
|
+
// one place where the output is dictated by somebody else's specification -- there is nothing to
|
|
6
|
+
// design, only to match.
|
|
7
|
+
//
|
|
8
|
+
// ⚠ EVERY VALUE THAT CAME FROM A FILE NAME IS ESCAPED. Names in this drive are whatever a person
|
|
9
|
+
// typed, including `&` and `<`, and an unescaped one produces XML the client cannot parse -- a
|
|
10
|
+
// listing that fails for one badly-named file and works for everything else.
|
|
11
|
+
/** The five characters XML cannot carry raw. */
|
|
12
|
+
export function escapeXml(value) {
|
|
13
|
+
return value
|
|
14
|
+
.replace(/&/g, "&")
|
|
15
|
+
.replace(/</g, "<")
|
|
16
|
+
.replace(/>/g, ">")
|
|
17
|
+
.replace(/"/g, """)
|
|
18
|
+
.replace(/'/g, "'");
|
|
19
|
+
}
|
|
20
|
+
const HEAD = `<?xml version="1.0" encoding="UTF-8"?>`;
|
|
21
|
+
const NS = `http://s3.amazonaws.com/doc/2006-03-01/`;
|
|
22
|
+
export function errorXml(code, message, resource) {
|
|
23
|
+
return (`${HEAD}<Error><Code>${escapeXml(code)}</Code><Message>${escapeXml(message)}</Message>` +
|
|
24
|
+
`<Resource>${escapeXml(resource)}</Resource></Error>`);
|
|
25
|
+
}
|
|
26
|
+
export function listBucketsXml(bucket, createdAt) {
|
|
27
|
+
return (`${HEAD}<ListAllMyBucketsResult xmlns="${NS}"><Owner><ID>nmts</ID>` +
|
|
28
|
+
`<DisplayName>nmts</DisplayName></Owner><Buckets><Bucket>` +
|
|
29
|
+
`<Name>${escapeXml(bucket)}</Name><CreationDate>${escapeXml(createdAt)}</CreationDate>` +
|
|
30
|
+
`</Bucket></Buckets></ListAllMyBucketsResult>`);
|
|
31
|
+
}
|
|
32
|
+
export function initiateUploadXml(bucket, key, uploadId) {
|
|
33
|
+
return (`${HEAD}<InitiateMultipartUploadResult xmlns="${NS}">` +
|
|
34
|
+
`<Bucket>${escapeXml(bucket)}</Bucket><Key>${escapeXml(key)}</Key>` +
|
|
35
|
+
`<UploadId>${escapeXml(uploadId)}</UploadId></InitiateMultipartUploadResult>`);
|
|
36
|
+
}
|
|
37
|
+
export function completeUploadXml(bucket, key, etag) {
|
|
38
|
+
return (`${HEAD}<CompleteMultipartUploadResult xmlns="${NS}">` +
|
|
39
|
+
`<Location>/${escapeXml(bucket)}/${escapeXml(key)}</Location>` +
|
|
40
|
+
`<Bucket>${escapeXml(bucket)}</Bucket><Key>${escapeXml(key)}</Key>` +
|
|
41
|
+
`<ETag>${escapeXml(etag)}</ETag></CompleteMultipartUploadResult>`);
|
|
42
|
+
}
|
|
43
|
+
/** `encoding-type=url` means every name in the answer comes back percent-encoded. */
|
|
44
|
+
function out(value, encodingType) {
|
|
45
|
+
return escapeXml(encodingType === "url" ? encodeURIComponent(value) : value);
|
|
46
|
+
}
|
|
47
|
+
export function listObjectsXml(listing) {
|
|
48
|
+
const enc = listing.encodingType;
|
|
49
|
+
const parts = [
|
|
50
|
+
HEAD,
|
|
51
|
+
`<ListBucketResult xmlns="${NS}">`,
|
|
52
|
+
`<Name>${escapeXml(listing.bucket)}</Name>`,
|
|
53
|
+
`<Prefix>${out(listing.prefix, enc)}</Prefix>`,
|
|
54
|
+
listing.delimiter.length > 0 ? `<Delimiter>${out(listing.delimiter, enc)}</Delimiter>` : "",
|
|
55
|
+
`<MaxKeys>${listing.maxKeys}</MaxKeys>`,
|
|
56
|
+
enc === null ? "" : `<EncodingType>${escapeXml(enc)}</EncodingType>`,
|
|
57
|
+
`<IsTruncated>${listing.truncated ? "true" : "false"}</IsTruncated>`,
|
|
58
|
+
];
|
|
59
|
+
if (listing.v2) {
|
|
60
|
+
parts.push(`<KeyCount>${listing.contents.length + listing.commonPrefixes.length}</KeyCount>`);
|
|
61
|
+
if (listing.next !== null) {
|
|
62
|
+
parts.push(`<NextContinuationToken>${escapeXml(listing.next)}</NextContinuationToken>`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
else if (listing.next !== null) {
|
|
66
|
+
parts.push(`<NextMarker>${out(listing.next, enc)}</NextMarker>`);
|
|
67
|
+
}
|
|
68
|
+
for (const row of listing.contents) {
|
|
69
|
+
parts.push(`<Contents><Key>${out(row.key, enc)}</Key>` +
|
|
70
|
+
`<LastModified>${escapeXml(row.lastModified)}</LastModified>` +
|
|
71
|
+
`<ETag>${escapeXml(row.etag)}</ETag>` +
|
|
72
|
+
`<Size>${row.size}</Size>` +
|
|
73
|
+
`<StorageClass>STANDARD</StorageClass></Contents>`);
|
|
74
|
+
}
|
|
75
|
+
for (const prefix of listing.commonPrefixes) {
|
|
76
|
+
parts.push(`<CommonPrefixes><Prefix>${out(prefix, enc)}</Prefix></CommonPrefixes>`);
|
|
77
|
+
}
|
|
78
|
+
parts.push(`</ListBucketResult>`);
|
|
79
|
+
return parts.join("");
|
|
80
|
+
}
|