@needmoretruth/nmts-cli 0.17.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. package/AGENTS.md +591 -0
  2. package/LICENSE +202 -0
  3. package/LICENSING.md +49 -0
  4. package/README.ko.md +641 -0
  5. package/README.md +657 -0
  6. package/dist/account-proof.js +78 -0
  7. package/dist/account.js +75 -0
  8. package/dist/api-key.js +310 -0
  9. package/dist/api.js +271 -0
  10. package/dist/args.js +137 -0
  11. package/dist/artifact-about.js +70 -0
  12. package/dist/autonomy.js +98 -0
  13. package/dist/code-access.js +76 -0
  14. package/dist/code-vault.js +233 -0
  15. package/dist/collision.js +133 -0
  16. package/dist/commands/balance.js +93 -0
  17. package/dist/commands/consent.js +77 -0
  18. package/dist/commands/create.js +356 -0
  19. package/dist/commands/env.js +136 -0
  20. package/dist/commands/expiring.js +167 -0
  21. package/dist/commands/extend.js +236 -0
  22. package/dist/commands/get.js +130 -0
  23. package/dist/commands/kit.js +136 -0
  24. package/dist/commands/listfile.js +105 -0
  25. package/dist/commands/login.js +335 -0
  26. package/dist/commands/logout.js +33 -0
  27. package/dist/commands/ls.js +202 -0
  28. package/dist/commands/marks.js +176 -0
  29. package/dist/commands/mcp.js +139 -0
  30. package/dist/commands/mode.js +54 -0
  31. package/dist/commands/on-collision.js +45 -0
  32. package/dist/commands/organise.js +296 -0
  33. package/dist/commands/public-code.js +89 -0
  34. package/dist/commands/pull.js +185 -0
  35. package/dist/commands/push.js +278 -0
  36. package/dist/commands/put.js +282 -0
  37. package/dist/commands/rebuild.js +160 -0
  38. package/dist/commands/receive.js +125 -0
  39. package/dist/commands/recovery-list.js +104 -0
  40. package/dist/commands/recovery.js +291 -0
  41. package/dist/commands/s3.js +241 -0
  42. package/dist/commands/settings.js +28 -0
  43. package/dist/commands/share.js +276 -0
  44. package/dist/commands/sweep.js +240 -0
  45. package/dist/commands/trash.js +288 -0
  46. package/dist/commands/trial.js +264 -0
  47. package/dist/commands/update.js +169 -0
  48. package/dist/commands/usage.js +84 -0
  49. package/dist/commands/verify.js +331 -0
  50. package/dist/commands/wallet.js +122 -0
  51. package/dist/commands/whoami.js +33 -0
  52. package/dist/consent.js +200 -0
  53. package/dist/credentials.js +311 -0
  54. package/dist/crypto-surface.js +54 -0
  55. package/dist/crypto.js +137 -0
  56. package/dist/download-part.js +144 -0
  57. package/dist/download-sink.js +213 -0
  58. package/dist/download.js +163 -0
  59. package/dist/drive-paths.js +170 -0
  60. package/dist/environment.js +190 -0
  61. package/dist/errors.js +71 -0
  62. package/dist/exit.js +107 -0
  63. package/dist/expiry.js +115 -0
  64. package/dist/extend-chain.js +137 -0
  65. package/dist/extend-plan.js +149 -0
  66. package/dist/extend-sign.js +130 -0
  67. package/dist/guards.js +15 -0
  68. package/dist/help.js +192 -0
  69. package/dist/human-check.js +54 -0
  70. package/dist/item-trash.js +25 -0
  71. package/dist/kit-file.js +133 -0
  72. package/dist/list-file.js +71 -0
  73. package/dist/list-view-find.js +71 -0
  74. package/dist/list-view-order.js +96 -0
  75. package/dist/main.js +394 -0
  76. package/dist/manifest-create.js +81 -0
  77. package/dist/manifest-write.js +244 -0
  78. package/dist/manifest.js +213 -0
  79. package/dist/mark-render.js +45 -0
  80. package/dist/mcp-args.js +154 -0
  81. package/dist/mcp-tools/context.js +50 -0
  82. package/dist/mcp-tools/files.js +133 -0
  83. package/dist/mcp-tools/organise.js +128 -0
  84. package/dist/mcp-tools/reads.js +89 -0
  85. package/dist/mcp-tools/share.js +61 -0
  86. package/dist/mcp.js +142 -0
  87. package/dist/net-retry.js +84 -0
  88. package/dist/network.js +43 -0
  89. package/dist/notice.js +45 -0
  90. package/dist/product.js +38 -0
  91. package/dist/progress.js +110 -0
  92. package/dist/prompt.js +145 -0
  93. package/dist/rebuild.js +255 -0
  94. package/dist/recovery-assemble.js +148 -0
  95. package/dist/recovery-build.js +260 -0
  96. package/dist/recovery-map-file.js +114 -0
  97. package/dist/recovery-map.js +191 -0
  98. package/dist/recovery-release.js +229 -0
  99. package/dist/recovery-seq.js +63 -0
  100. package/dist/recovery-source.js +143 -0
  101. package/dist/registration.js +83 -0
  102. package/dist/s3/listing.js +134 -0
  103. package/dist/s3/multipart.js +104 -0
  104. package/dist/s3/response-sink.js +41 -0
  105. package/dist/s3/same-file.js +117 -0
  106. package/dist/s3/server.js +252 -0
  107. package/dist/s3/sigv4.js +168 -0
  108. package/dist/s3/staging.js +87 -0
  109. package/dist/s3/xml.js +80 -0
  110. package/dist/safe-path.js +89 -0
  111. package/dist/seal.js +206 -0
  112. package/dist/secret-reader.js +112 -0
  113. package/dist/server.js +33 -0
  114. package/dist/session.js +49 -0
  115. package/dist/setup-questions.js +31 -0
  116. package/dist/share.js +195 -0
  117. package/dist/shared/lib/crypto/size-padding.js +126 -0
  118. package/dist/shared/lib/drive/manifest-codec.js +287 -0
  119. package/dist/shared/lib/drive/manifest-index.js +235 -0
  120. package/dist/shared/lib/drive/manifest-ops.js +267 -0
  121. package/dist/shared/lib/drive/manifest-settings.js +65 -0
  122. package/dist/shared/lib/drive/name-conflict.js +121 -0
  123. package/dist/shared/lib/drive/unique-name.js +48 -0
  124. package/dist/shared/lib/extend/epochs.js +256 -0
  125. package/dist/shared/lib/net/retry-budget.js +95 -0
  126. package/dist/shared/lib/share/shared-file-info.js +65 -0
  127. package/dist/shared/lib/storage-network.js +64 -0
  128. package/dist/shared/lib/upload/part-plan.js +28 -0
  129. package/dist/stdout.js +109 -0
  130. package/dist/trash-sweep.js +123 -0
  131. package/dist/units.js +19 -0
  132. package/dist/update-check.js +164 -0
  133. package/dist/update-source.js +143 -0
  134. package/dist/upload-api.js +98 -0
  135. package/dist/upload-file.js +242 -0
  136. package/dist/upload-price.js +117 -0
  137. package/dist/upload-steps.js +111 -0
  138. package/dist/upload-store.js +252 -0
  139. package/dist/upload-wire.js +25 -0
  140. package/dist/upload.js +294 -0
  141. package/dist/usage-report.js +54 -0
  142. package/dist/wallet-chain.js +76 -0
  143. package/dist/wallet.js +209 -0
  144. package/dist/walrus-write.js +188 -0
  145. package/dist/walrus.js +156 -0
  146. package/package.json +56 -0
  147. package/vendor/nmts-crypto/nmts_crypto_wasm.d.ts +469 -0
  148. package/vendor/nmts-crypto/nmts_crypto_wasm.js +1300 -0
  149. package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm +0 -0
  150. package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm.d.ts +55 -0
  151. package/vendor/nmts-crypto/package.json +17 -0
@@ -0,0 +1,143 @@
1
+ // Reading the account's storage dump — every live file and where its bytes are — one page at a time.
2
+ //
3
+ // ⛔ WHY A DUMP AND NOT THE ORDINARY LISTING. The drive listing is per folder and carries no
4
+ // storage parts; building a list from it would cost one request per folder plus one per file.
5
+ // This returns the whole account, parts included, 500 at a time.
6
+ //
7
+ // ⛔ EVERY PAGE CARRIES THE ACCOUNT PROOF. The route's verdict is `NeedsAccountProof`: an API key
8
+ // alone is refused, whatever its permissions. `account-proof.ts` says what the value is, why
9
+ // sending it is safe, and what it can still do if it is stolen.
10
+ //
11
+ // ⛔ NOTHING HERE TRUSTS THE ANSWER'S SHAPE. It arrives as `unknown` and is narrowed by predicates
12
+ // rather than asserted, because the whole point of the artefact being built from it is that the
13
+ // server may one day be hostile or broken. A row this cannot read is a refusal, never a skip: a
14
+ // list quietly missing a file tells somebody they are covered when they are not.
15
+ //
16
+ // ⚠ ITEMS CREATED DURING THE WALK may or may not appear, depending on where their id sorts. That
17
+ // is why the caller stamps the moment the walk BEGAN and sends that as the capture time — a file
18
+ // made while the pages were being read then counts as outside the list rather than falling into
19
+ // the gap between the two.
20
+ import { request } from "./api.js";
21
+ import { NmtsError } from "./errors.js";
22
+ import { isRecord } from "./guards.js";
23
+ import { plaintextLenFromSealed } from "./seal.js";
24
+ /** `file_parts.storage_kind` for a quilt patch. 0 is a dedicated blob. */
25
+ export const STORAGE_QUILT = 1;
26
+ function unreadable(what) {
27
+ return new NmtsError(`The server described ${what} in a shape this version cannot read.`, {
28
+ exitCode: 1,
29
+ nextStep: "Nothing was written. Update this tool, or build the recovery list from the account " +
30
+ "screen in a browser — a list that skipped what it could not read would claim to cover " +
31
+ "files it does not.",
32
+ });
33
+ }
34
+ function str(row, name, what) {
35
+ const value = row[name];
36
+ if (typeof value !== "string" || value.length === 0)
37
+ throw unreadable(`${what} (${name})`);
38
+ return value;
39
+ }
40
+ function optionalStr(row, name) {
41
+ const value = row[name];
42
+ return typeof value === "string" && value.length > 0 ? value : undefined;
43
+ }
44
+ function whole(row, name, what) {
45
+ const value = row[name];
46
+ if (typeof value !== "number" || !Number.isSafeInteger(value))
47
+ throw unreadable(`${what} (${name})`);
48
+ return value;
49
+ }
50
+ function partOf(value, itemId) {
51
+ if (!isRecord(value))
52
+ throw unreadable(`a stored part of file ${itemId}`);
53
+ const what = `a stored part of file ${itemId}`;
54
+ const sealedLen = whole(value, "sealed_len", what);
55
+ const part = {
56
+ part_index: whole(value, "part_index", what),
57
+ storage_kind: whole(value, "storage_kind", what),
58
+ blob_id: str(value, "blob_id", what),
59
+ sealed_len: sealedLen,
60
+ // Throws on a length no honest upload could have produced, which is exactly the number a
61
+ // recovery list must not record.
62
+ streamPlaintextLen: plaintextLenFromSealed(sealedLen),
63
+ };
64
+ const network = value["network"];
65
+ if (typeof network === "number" && Number.isSafeInteger(network))
66
+ part.network = network;
67
+ const patch = optionalStr(value, "patch_id");
68
+ if (patch !== undefined)
69
+ part.patch_id = patch;
70
+ const object = optionalStr(value, "sui_object_id");
71
+ if (object !== undefined)
72
+ part.sui_object_id = object;
73
+ return part;
74
+ }
75
+ function itemOf(value) {
76
+ if (!isRecord(value))
77
+ throw unreadable("a stored file");
78
+ const id = str(value, "id", "a stored file");
79
+ const parts = value["parts"];
80
+ if (!Array.isArray(parts))
81
+ throw unreadable(`the stored parts of file ${id}`);
82
+ const item = {
83
+ id,
84
+ size: whole(value, "size", `stored file ${id}`),
85
+ createdAt: str(value, "created_at", `stored file ${id}`),
86
+ updatedAt: str(value, "updated_at", `stored file ${id}`),
87
+ parts: parts.map((p) => partOf(p, id)),
88
+ };
89
+ const dek = optionalStr(value, "dek_wrapped");
90
+ if (dek !== undefined)
91
+ item.dekWrapped = dek;
92
+ const hash = optionalStr(value, "content_hash_ct");
93
+ if (hash !== undefined)
94
+ item.contentHashCt = hash;
95
+ return item;
96
+ }
97
+ /**
98
+ * Read every page of the dump, oldest cursor first.
99
+ *
100
+ * ⛔ A REPEATED CURSOR IS A REFUSAL, NOT A LOOP. The cursor is server-issued and strictly
101
+ * increasing, but a bug — or a hand-edited answer — that repeated one would spin here forever
102
+ * with nothing to show for it.
103
+ */
104
+ export async function readAllRecoverySource(options) {
105
+ const items = [];
106
+ const seen = new Set();
107
+ let cursor;
108
+ do {
109
+ // ⚠ `after` IS ALWAYS SPELLED OUT, empty on the first page. The server reads an empty value as
110
+ // "from the beginning" — and one shape for the address is what lets the route gate read this
111
+ // call and compare it against what the server actually registers.
112
+ // ⚠ THE VALUE IS PREPARED OUTSIDE THE ADDRESS, and not for tidiness: `check:cli-routes` reads
113
+ // the literal handed to `request` and its reader stops at a quote character, so a `""`
114
+ // written inside the template would take this call out of the gate's sight entirely — the
115
+ // exact failure that gate was built for. Proved by breaking the address and watching it go
116
+ // red.
117
+ const after = cursor === undefined ? "" : cursor;
118
+ const answer = await request(options.server, `/v1/account/recovery-source?after=${encodeURIComponent(after)}`, { token: options.apiKey, accountProof: options.accountProof });
119
+ if (!isRecord(answer))
120
+ throw unreadable("this account's stored files");
121
+ const page = answer["items"];
122
+ if (!Array.isArray(page))
123
+ throw unreadable("this account's stored files");
124
+ for (const row of page)
125
+ items.push(itemOf(row));
126
+ options.onProgress?.(items.length);
127
+ const next = answer["next_cursor"];
128
+ if (next === undefined || next === null)
129
+ return items;
130
+ if (typeof next !== "string" || next.length === 0)
131
+ throw unreadable("the next page of files");
132
+ if (seen.has(next)) {
133
+ throw new NmtsError("The account's stored files repeated a page marker.", {
134
+ exitCode: 1,
135
+ nextStep: "Nothing was written. The listing cannot be read to the end, so a list built from it " +
136
+ "would be missing files without saying so.",
137
+ });
138
+ }
139
+ seen.add(next);
140
+ cursor = next;
141
+ } while (cursor !== undefined);
142
+ return items;
143
+ }
@@ -0,0 +1,83 @@
1
+ // Making an account exist: the one derivation this tool performs that it does not keep.
2
+ //
3
+ // ⛔ WHY THIS IS NOT IN `account.ts`. That module's header says `authSecret` and `dataKey` are not
4
+ // exposed, and it is right to say so — `identityOf` hands back only the two public values.
5
+ // Registering a new account is the single call in this program that needs a THIRD one: the
6
+ // server has to be given `authSecret` so it can store the argon2id verifier of it, exactly as
7
+ // the browser does at sign-up (`web/src/lib/auth/account-service.ts`). Widening `identityOf`
8
+ // for that would have made a promise false everywhere it is read; a separate module keeps the
9
+ // exception in one file, next to the reason for it.
10
+ //
11
+ // ⛔ `authSecret` IS NOT THE ACCOUNT CODE AND CANNOT BE TURNED BACK INTO ONE. It is 32 bytes out
12
+ // of a one-way derivation (NCF-3 §1.2), it opens no file, and the same value goes over the
13
+ // wire on every sign-in the browser makes. What it can do is prove ownership, so it is built
14
+ // here, sent once, and never written down, printed or returned to a caller that did not ask.
15
+ //
16
+ // ⛔ THE DERIVED BUFFER IS WIPED. `kdf_derive` returns EVERY key in the account — the sign-in
17
+ // secret, the key that opens the files, the file-list key, the wallet root. Two public fields
18
+ // are copied out and the rest is zeroed before this returns, the same discipline `identityOf`
19
+ // keeps and for the same reason: a live copy of that buffer is a live copy of the account.
20
+ import { DERIVED, loadCrypto } from "./crypto.js";
21
+ import { NmtsError } from "./errors.js";
22
+ /**
23
+ * A brand-new account code, from the engine.
24
+ *
25
+ * ⛔ THE RETURNED STRING IS THE ONLY COPY THAT WILL EVER EXIST. The server keeps a verifier of a
26
+ * value derived from it and nothing else, so a caller that loses this has destroyed an account
27
+ * and nobody — not the holder, not NMTS — can bring it back. Every caller of this owes the
28
+ * person a way to keep it before anything else happens.
29
+ *
30
+ * ⛔ AND THE RANDOMNESS IS THE ENGINE'S, NEVER NODE'S — the reason `generate_dek` gives, with more
31
+ * at stake. These twenty bytes are the seed every key in an account descends from, so a second
32
+ * source of them would be a source the conformance vectors say nothing about, in the one place
33
+ * where a weak draw loses an entire account rather than one file. `crypto.ts` declares the
34
+ * engine call; this is the only thing in the program that makes it.
35
+ */
36
+ export async function newAccountCode() {
37
+ const glue = await loadCrypto();
38
+ const code = glue.account_code_generate();
39
+ if (typeof code !== "string" || code.length === 0) {
40
+ // ⛔ Refused rather than passed on. An empty code would be registered as a real account, and
41
+ // the failure would surface much later as an account nobody can open.
42
+ throw new NmtsError("The NMTS crypto engine did not produce an account code.", {
43
+ exitCode: 1,
44
+ nextStep: "Nothing was created. Reinstall the package and try again.",
45
+ });
46
+ }
47
+ return code;
48
+ }
49
+ /**
50
+ * The pair the server is told about a code, derived here and nowhere else.
51
+ *
52
+ * ⚠ It re-parses the code rather than taking bytes: the parser checks the trailing check symbol,
53
+ * so a code that arrived through anything but `newAccountCode` is refused before it is used to
54
+ * claim an account id.
55
+ */
56
+ export async function registrationProofOf(code) {
57
+ const glue = await loadCrypto();
58
+ let bytes;
59
+ try {
60
+ bytes = glue.account_code_parse(code);
61
+ }
62
+ catch {
63
+ // ⛔ The engine's own message is not repeated: it can contain the input, and the input is the
64
+ // account code.
65
+ throw new NmtsError("That is not a valid NMTS account code.", {
66
+ exitCode: 2,
67
+ nextStep: "The last character is a check symbol, and it does not match the rest.",
68
+ });
69
+ }
70
+ const derived = glue.kdf_derive(bytes);
71
+ try {
72
+ const [idFrom, idTo] = DERIVED.accountId;
73
+ const [secretFrom, secretTo] = DERIVED.authSecret;
74
+ return {
75
+ accountId: Buffer.from(derived.slice(idFrom, idTo)).toString("base64url"),
76
+ authSecret: Buffer.from(derived.slice(secretFrom, secretTo)).toString("base64url"),
77
+ };
78
+ }
79
+ finally {
80
+ derived.fill(0);
81
+ bytes.fill(0);
82
+ }
83
+ }
@@ -0,0 +1,134 @@
1
+ // The drive as an S3 client sees it: one bucket, and a key for every live file.
2
+ //
3
+ // ⛔ THE MAPPING IS THE WHOLE DESIGN, so it is written down here rather than spread across the
4
+ // server. A key is the file's path with the leading slash removed -- `photos/a.jpg` -- because
5
+ // that is what every S3 tool will put back on the disk when it syncs. Nothing else in this
6
+ // account's shape is exposed: not item ids, not the trash, not the marks.
7
+ //
8
+ // ⛔ TRASHED FILES ARE NOT KEYS. Being in the trash is inherited from a folder, so filtering on the
9
+ // entry's own field alone would list files whose bytes the server already refuses -- an S3 client
10
+ // would see them, ask for them, and get a failure for every one.
11
+ //
12
+ // ⚠ ONE DELIBERATE DIFFERENCE FROM S3: a folder holding no files still comes back as a common
13
+ // prefix. Real S3 has no folders, so an empty one cannot exist there; this drive does have them,
14
+ // and hiding them would make `rclone lsd` describe a drive that is not the one in the browser.
15
+ import { buildIndex, fullPathOf, isLive, KIND_FOLDER } from "../drive-paths.js";
16
+ /** The one bucket. Named for what it is, and not configurable: two names for one drive is worse. */
17
+ export const BUCKET = "drive";
18
+ /** S3's own ceiling, and the default when a client does not ask for one. */
19
+ export const MAX_KEYS_LIMIT = 1000;
20
+ /**
21
+ * An ETag that is stable for a file and changes when the file does.
22
+ *
23
+ * ⛔ IT ENDS IN `-1` FOR A REASON. S3 clients treat an ETag that looks like a hex digest as the
24
+ * MD5 of the object and check downloads against it; this drive has no MD5 of anything -- the
25
+ * bytes are encrypted before they leave the machine and the digest it does keep is a different
26
+ * function. The `-N` suffix is S3's own mark for "assembled from parts, not an MD5", and every
27
+ * client already knows to skip the check when it sees one. Without it a correct download is
28
+ * reported as corrupt.
29
+ */
30
+ export function etagOf(entry) {
31
+ const id = entry.id.replace(/[^0-9a-zA-Z]/g, "");
32
+ const stamp = entry.updatedAt.toString(16);
33
+ return `"${(id + stamp).slice(0, 32).padEnd(32, "0")}-1"`;
34
+ }
35
+ /** Every live file in the account, as keys, in the order S3 promises: ascending by key. */
36
+ export function objectsOf(entries) {
37
+ const index = buildIndex(entries);
38
+ const rows = [];
39
+ for (const entry of entries) {
40
+ if (entry.kind === KIND_FOLDER)
41
+ continue;
42
+ if (!isLive(index, entry))
43
+ continue;
44
+ rows.push({
45
+ key: fullPathOf(index, entry).replace(/^\//, ""),
46
+ lastModified: new Date(entry.updatedAt).toISOString(),
47
+ etag: etagOf(entry),
48
+ size: entry.size,
49
+ entry,
50
+ });
51
+ }
52
+ rows.sort((a, b) => (a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
53
+ return rows;
54
+ }
55
+ /** Every live folder, as a key ending in the delimiter — see the note at the top of this file. */
56
+ export function folderPrefixesOf(entries) {
57
+ const index = buildIndex(entries);
58
+ const out = [];
59
+ for (const entry of entries) {
60
+ if (entry.kind !== KIND_FOLDER)
61
+ continue;
62
+ if (!isLive(index, entry))
63
+ continue;
64
+ out.push(`${fullPathOf(index, entry).replace(/^\//, "")}/`);
65
+ }
66
+ return out;
67
+ }
68
+ /**
69
+ * Apply prefix, delimiter and paging the way `ListObjects` does.
70
+ *
71
+ * The rules are S3's: a key is returned whole unless it holds the delimiter after the prefix, in
72
+ * which case everything up to and including that delimiter becomes a common prefix and the key
73
+ * itself is not listed. Common prefixes and keys share one page budget and one cursor.
74
+ */
75
+ export function listObjects(objects, folders, query) {
76
+ const maxKeys = Math.max(0, Math.min(query.maxKeys, MAX_KEYS_LIMIT));
77
+ const seenPrefix = new Set();
78
+ const rows = [];
79
+ for (const object of objects) {
80
+ if (!object.key.startsWith(query.prefix))
81
+ continue;
82
+ if (query.delimiter.length > 0) {
83
+ const rest = object.key.slice(query.prefix.length);
84
+ const at = rest.indexOf(query.delimiter);
85
+ if (at >= 0) {
86
+ const prefix = query.prefix + rest.slice(0, at + query.delimiter.length);
87
+ if (!seenPrefix.has(prefix)) {
88
+ seenPrefix.add(prefix);
89
+ rows.push({ sort: prefix, row: null, prefix });
90
+ }
91
+ continue;
92
+ }
93
+ }
94
+ rows.push({ sort: object.key, row: object, prefix: null });
95
+ }
96
+ // Folders that hold no listed file still belong in the answer — the note at the top says why.
97
+ if (query.delimiter.length > 0) {
98
+ for (const folder of folders) {
99
+ if (!folder.startsWith(query.prefix))
100
+ continue;
101
+ const rest = folder.slice(query.prefix.length);
102
+ const at = rest.indexOf(query.delimiter);
103
+ if (at < 0)
104
+ continue;
105
+ const prefix = query.prefix + rest.slice(0, at + query.delimiter.length);
106
+ if (seenPrefix.has(prefix))
107
+ continue;
108
+ seenPrefix.add(prefix);
109
+ rows.push({ sort: prefix, row: null, prefix });
110
+ }
111
+ }
112
+ rows.sort((a, b) => (a.sort < b.sort ? -1 : a.sort > b.sort ? 1 : 0));
113
+ const started = query.after === null ? rows : rows.filter((r) => r.sort > (query.after ?? ""));
114
+ const page = started.slice(0, maxKeys);
115
+ const truncated = started.length > page.length;
116
+ const last = page[page.length - 1];
117
+ // ⛔ Split in a loop rather than two filter-and-map passes: a `map` over a filtered array cannot
118
+ // convince the type checker that the field is there, and the usual way round that is to invent
119
+ // an empty row for a case that cannot happen — which is how an empty key reaches a client.
120
+ const contents = [];
121
+ const commonPrefixes = [];
122
+ for (const item of page) {
123
+ if (item.row !== null)
124
+ contents.push(item.row);
125
+ else if (item.prefix !== null)
126
+ commonPrefixes.push(item.prefix);
127
+ }
128
+ return {
129
+ contents,
130
+ commonPrefixes,
131
+ truncated,
132
+ next: truncated && last !== undefined ? last.sort : null,
133
+ };
134
+ }
@@ -0,0 +1,104 @@
1
+ // Uploads that arrive in pieces.
2
+ //
3
+ // ⛔ WHY IT IS NOT OPTIONAL. Every S3 client switches to this above a size of its own choosing --
4
+ // rclone's default is 200 MiB -- so a gateway without it works on small files and fails on the
5
+ // large ones, which is the half of a backup that matters most.
6
+ //
7
+ // ⛔ MEASURED FROM A REAL CLIENT, NOT FROM THE SPECIFICATION. What rclone actually sends is:
8
+ // `POST ?uploads=` to begin · `PUT ?partNumber=N&uploadId=…` for each piece, CONCURRENTLY AND
9
+ // OUT OF ORDER (1, 3, 2 in the capture) · `POST ?uploadId=…` carrying the part list to finish.
10
+ // The out-of-order part is the one a from-the-specification implementation gets wrong, because
11
+ // reading the spec top to bottom suggests a sequence.
12
+ //
13
+ // ⚠ THE PIECES ARE STAGED, and staging is the caller's business rather than this file's: this
14
+ // module speaks the protocol and the writer it is handed does the filesystem.
15
+ import { isKeyConflict } from "./same-file.js";
16
+ import { completeUploadXml, initiateUploadXml } from "./xml.js";
17
+ /** S3's own ceiling, and a bound on what one client can stage on this machine. */
18
+ export const MAX_PARTS = 10_000;
19
+ /** True when this request belongs to a multipart upload rather than a plain object call. */
20
+ export function isMultipartRequest(method, query) {
21
+ return query.has("uploads") || query.has("uploadId") || method === "POST";
22
+ }
23
+ function send(res, body) {
24
+ res.writeHead(200, {
25
+ "content-type": "application/xml",
26
+ "content-length": String(Buffer.byteLength(body)),
27
+ });
28
+ res.end(body);
29
+ }
30
+ /** Answer one multipart request. Returns false when the shape is not one this gateway knows. */
31
+ export async function handleMultipart(context) {
32
+ const { method, query, writer, key, res, req } = context;
33
+ const staging = writer.multipart;
34
+ if (staging === undefined) {
35
+ context.fail(501, "NotImplemented", "This gateway does not stage multipart uploads.");
36
+ return true;
37
+ }
38
+ if (method === "POST" && query.has("uploads")) {
39
+ const uploadId = await staging.begin(key);
40
+ send(res, initiateUploadXml(context.bucket, key, uploadId));
41
+ context.log?.(`multipart begin ${key}`);
42
+ return true;
43
+ }
44
+ const uploadId = query.get("uploadId");
45
+ if (uploadId === null)
46
+ return false;
47
+ if (method === "PUT") {
48
+ const partNumber = Number(query.get("partNumber") ?? "");
49
+ if (!Number.isInteger(partNumber) || partNumber < 1 || partNumber > MAX_PARTS) {
50
+ context.fail(400, "InvalidPart", `A part number must be between 1 and ${MAX_PARTS}.`);
51
+ return true;
52
+ }
53
+ const size = Number(req.headers["content-length"] ?? "");
54
+ if (!Number.isInteger(size) || size < 0) {
55
+ context.fail(411, "MissingContentLength", "This gateway needs the part's size before it starts.");
56
+ return true;
57
+ }
58
+ try {
59
+ const etag = await staging.part(uploadId, partNumber, req, size, context.payloadHash);
60
+ res.writeHead(200, { etag, "content-length": "0" });
61
+ res.end();
62
+ }
63
+ catch (error) {
64
+ context.fail(400, "InvalidPart", error instanceof Error ? error.message : String(error));
65
+ }
66
+ return true;
67
+ }
68
+ if (method === "POST") {
69
+ // ⛔ THE PART LIST IN THE BODY IS NOT READ. It carries the tags this gateway itself handed back,
70
+ // so believing it would be believing our own echo; what the file is made of is what was
71
+ // staged, in part-number order. Reading it would only add a way to disagree with ourselves.
72
+ await drain(req);
73
+ try {
74
+ const etag = await staging.complete(uploadId);
75
+ send(res, completeUploadXml(context.bucket, key, etag));
76
+ context.log?.(`multipart complete ${key}`);
77
+ }
78
+ catch (error) {
79
+ // ⛔ A KEY THAT HOLDS A DIFFERENT FILE IS A 409, NOT A 500 — the request was well formed and
80
+ // the drive declined it. Told 500, a sync tool retries the whole upload forever; told 409
81
+ // it records a conflict and moves on. The verdict itself is the writer's (`same-file.ts`),
82
+ // reached only once the pieces are one file, because until then there is nothing to hash.
83
+ if (isKeyConflict(error)) {
84
+ context.fail(409, "InvalidRequest", error instanceof Error ? error.message : String(error));
85
+ }
86
+ else {
87
+ context.fail(500, "InternalError", error instanceof Error ? error.message : String(error));
88
+ }
89
+ }
90
+ return true;
91
+ }
92
+ if (method === "DELETE") {
93
+ await staging.abort(uploadId);
94
+ res.writeHead(204);
95
+ res.end();
96
+ context.log?.(`multipart abort ${key}`);
97
+ return true;
98
+ }
99
+ return false;
100
+ }
101
+ async function drain(req) {
102
+ for await (const chunk of req)
103
+ void chunk;
104
+ }
@@ -0,0 +1,41 @@
1
+ // Where a decrypted file goes when the reader is an HTTP client rather than a disk.
2
+ //
3
+ // ⛔ THE INTEGRITY PROMISE IS DIFFERENT HERE, AND THE DIFFERENCE IS SAID OUT LOUD. Every other sink
4
+ // in this tool makes a file visible only after the whole of it has been proved. A response
5
+ // cannot do that: S3 clients want the body to start arriving immediately, and the status line
6
+ // and length go out before the first byte. So bytes reach the client as they are decrypted, and
7
+ // the check at the end can no longer withhold them.
8
+ //
9
+ // ⛔ WHAT IT DOES INSTEAD: if anything fails after the response began, the connection is DESTROYED
10
+ // rather than ended. A client that was promised `Content-Length` bytes and gets fewer, with no
11
+ // clean end, reports a failed transfer — which is the truth. Ending the response normally would
12
+ // hand over a short file that every client would file away as complete.
13
+ /** A sink that writes one file into an HTTP response and never leaves a short body looking whole. */
14
+ export function responseSink(res, options) {
15
+ let started = false;
16
+ return {
17
+ expect(size) {
18
+ res.writeHead(200, { ...options.headers, "content-length": String(size) });
19
+ started = true;
20
+ },
21
+ async write(bytes) {
22
+ if (res.writableEnded || res.destroyed)
23
+ return;
24
+ await new Promise((resolve, reject) => {
25
+ res.write(bytes, (error) => (error === null || error === undefined ? resolve() : reject(error)));
26
+ });
27
+ },
28
+ async commit() {
29
+ if (res.destroyed)
30
+ return false;
31
+ await new Promise((resolve) => res.end(resolve));
32
+ return true;
33
+ },
34
+ async abandon() {
35
+ // Nothing was written yet: the caller can still answer with a proper S3 error document.
36
+ if (!started)
37
+ return;
38
+ res.destroy();
39
+ },
40
+ };
41
+ }
@@ -0,0 +1,117 @@
1
+ // Is the file arriving at this key the file already there?
2
+ //
3
+ // ⛔ WHY THE QUESTION IS ABOUT CONTENT AND NOT ABOUT NAMES. The gateway used to answer a taken key
4
+ // with 409 on the strength of the name alone. A backup program's whole job is to send the same
5
+ // names again, so every run failed on every file it had already stored — and a sync tool writes
6
+ // a 409 down as a failure, which is the wrong word for "it is already there".
7
+ //
8
+ // ⛔ WHAT IS COMPARED IS THE PLAINTEXT'S SHA-256, AND THE STORED COPY OF IT IS SEALED. Every
9
+ // upload already records one (`contentHashCt`, sealed under the account's data key), so nothing
10
+ // about the format changes here. It is sealed rather than stored bare for a reason worth
11
+ // repeating: a bare hash is the SAME NUMBER for everybody who holds that file, so a server
12
+ // keeping them could match its users against a published list of hashes. Sealed, only the
13
+ // account that wrote it can compare.
14
+ //
15
+ // ⛔ "REFUSE" MEANS THE UPLOAD, NOT THE REQUEST. Identical content is answered 200 with nothing
16
+ // sent and nothing charged, because from the client's side the statement "that file is at that
17
+ // key" is true. Only DIFFERENT content is a conflict.
18
+ import { createHash } from "node:crypto";
19
+ import { createReadStream } from "node:fs";
20
+ import { AAD, DERIVED, loadCrypto } from "../crypto.js";
21
+ import { objectsOf } from "./listing.js";
22
+ import { NmtsError } from "../errors.js";
23
+ /**
24
+ * Thrown when a key holds a DIFFERENT file. The gateway answers 409 rather than 500: the request
25
+ * was well formed and the drive declined it, which is what that status is for.
26
+ */
27
+ export class KeyConflict extends NmtsError {
28
+ constructor(message) {
29
+ super(message);
30
+ this.name = "KeyConflict";
31
+ }
32
+ }
33
+ /** True when a thrown value is that refusal, without importing the class into the protocol layer. */
34
+ export function isKeyConflict(error) {
35
+ return error instanceof Error && error.name === "KeyConflict";
36
+ }
37
+ /** The SHA-256 of a file on this machine, read in pieces so a large one costs no memory. */
38
+ export async function hashOfFile(path) {
39
+ const digest = createHash("sha256");
40
+ for await (const chunk of createReadStream(path))
41
+ digest.update(chunk);
42
+ return new Uint8Array(digest.digest());
43
+ }
44
+ /**
45
+ * Open the hash this drive recorded for a file, with the account's own key.
46
+ *
47
+ * `null` when the entry carries none — files stored before the field existed do not have one, and
48
+ * that is a real state rather than an error. A hash that is present and will not open IS an error:
49
+ * it means the file list was written by another account or altered, and answering "no hash" there
50
+ * would quietly turn a tampered list into an upload.
51
+ */
52
+ export async function recordedHash(accountCode, contentHashCt) {
53
+ if (contentHashCt === undefined || contentHashCt === "")
54
+ return null;
55
+ const crypt = await loadCrypto();
56
+ const [from, to] = DERIVED.dataKey;
57
+ const derived = crypt.kdf_derive(crypt.account_code_parse(accountCode));
58
+ const dataKey = derived.slice(from, to);
59
+ derived.fill(0);
60
+ try {
61
+ return crypt.envelope_open(dataKey, new TextEncoder().encode(AAD.contentHash), Buffer.from(contentHashCt, "base64url"));
62
+ }
63
+ catch {
64
+ throw new NmtsError("The recorded hash of the file at that key did not open with this account's key.", {
65
+ nextStep: "Either the file list belongs to another account, or it has been altered.",
66
+ });
67
+ }
68
+ finally {
69
+ dataKey.fill(0);
70
+ }
71
+ }
72
+ /**
73
+ * The whole question, for one key: what does this drive hold there, and is the file on disk it?
74
+ *
75
+ * ⛔ IT IS ONE FUNCTION SO THERE IS ONE ANSWER. Both ways of uploading — a single PUT, and pieces
76
+ * staged and joined — reach the drive through the same store, and this is what that store asks.
77
+ * Written at the two call sites instead, the two would differ the first time one of them
78
+ * changed, and the difference would show up only above whatever size the client switches at.
79
+ *
80
+ * ⚠ THE FILE IS HASHED ONLY WHEN THE KEY IS TAKEN. An upload onto a free key is the common case
81
+ * and pays nothing for this.
82
+ */
83
+ export async function verdictForKey(entries, key, accountCode, path) {
84
+ const standing = objectsOf(entries).find((o) => o.key === key);
85
+ if (standing === undefined)
86
+ return "free";
87
+ return compare(await recordedHash(accountCode, standing.entry.contentHashCt), await hashOfFile(path));
88
+ }
89
+ /** The verdict, given what is on record and what arrived. */
90
+ export function compare(recorded, arriving) {
91
+ if (recorded === undefined)
92
+ return "free";
93
+ if (recorded === null)
94
+ return "unknown";
95
+ if (recorded.length !== arriving.length)
96
+ return "differs";
97
+ for (let i = 0; i < recorded.length; i += 1) {
98
+ if (recorded[i] !== arriving[i])
99
+ return "differs";
100
+ }
101
+ return "same";
102
+ }
103
+ /**
104
+ * What to tell a client whose upload was declined.
105
+ *
106
+ * ⛔ IT NAMES WHICH OF THE TWO HAPPENED. "There is already a file there" is the same sentence for a
107
+ * file that changed and for a file this drive cannot compare, and the two need different things
108
+ * from the person reading the log.
109
+ */
110
+ export function refusalFor(verdict, key) {
111
+ return new KeyConflict(verdict === "differs"
112
+ ? `A different file is already at ${key}, and this drive does not replace files. Delete it ` +
113
+ "first — a delete puts it in the trash, where it stays recoverable for thirty days."
114
+ : `A file is already at ${key} and this drive has no recorded hash for it, so it cannot tell ` +
115
+ "whether yours is the same one. Files stored before hashes were recorded are in this state. " +
116
+ "Delete it first if you mean to replace it.");
117
+ }