@needmoretruth/nmts-cli 0.17.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS.md +591 -0
- package/LICENSE +202 -0
- package/LICENSING.md +49 -0
- package/README.ko.md +641 -0
- package/README.md +657 -0
- package/dist/account-proof.js +78 -0
- package/dist/account.js +75 -0
- package/dist/api-key.js +310 -0
- package/dist/api.js +271 -0
- package/dist/args.js +137 -0
- package/dist/artifact-about.js +70 -0
- package/dist/autonomy.js +98 -0
- package/dist/code-access.js +76 -0
- package/dist/code-vault.js +233 -0
- package/dist/collision.js +133 -0
- package/dist/commands/balance.js +93 -0
- package/dist/commands/consent.js +77 -0
- package/dist/commands/create.js +356 -0
- package/dist/commands/env.js +136 -0
- package/dist/commands/expiring.js +167 -0
- package/dist/commands/extend.js +236 -0
- package/dist/commands/get.js +130 -0
- package/dist/commands/kit.js +136 -0
- package/dist/commands/listfile.js +105 -0
- package/dist/commands/login.js +335 -0
- package/dist/commands/logout.js +33 -0
- package/dist/commands/ls.js +202 -0
- package/dist/commands/marks.js +176 -0
- package/dist/commands/mcp.js +139 -0
- package/dist/commands/mode.js +54 -0
- package/dist/commands/on-collision.js +45 -0
- package/dist/commands/organise.js +296 -0
- package/dist/commands/public-code.js +89 -0
- package/dist/commands/pull.js +185 -0
- package/dist/commands/push.js +278 -0
- package/dist/commands/put.js +282 -0
- package/dist/commands/rebuild.js +160 -0
- package/dist/commands/receive.js +125 -0
- package/dist/commands/recovery-list.js +104 -0
- package/dist/commands/recovery.js +291 -0
- package/dist/commands/s3.js +241 -0
- package/dist/commands/settings.js +28 -0
- package/dist/commands/share.js +276 -0
- package/dist/commands/sweep.js +240 -0
- package/dist/commands/trash.js +288 -0
- package/dist/commands/trial.js +264 -0
- package/dist/commands/update.js +169 -0
- package/dist/commands/usage.js +84 -0
- package/dist/commands/verify.js +331 -0
- package/dist/commands/wallet.js +122 -0
- package/dist/commands/whoami.js +33 -0
- package/dist/consent.js +200 -0
- package/dist/credentials.js +311 -0
- package/dist/crypto-surface.js +54 -0
- package/dist/crypto.js +137 -0
- package/dist/download-part.js +144 -0
- package/dist/download-sink.js +213 -0
- package/dist/download.js +163 -0
- package/dist/drive-paths.js +170 -0
- package/dist/environment.js +190 -0
- package/dist/errors.js +71 -0
- package/dist/exit.js +107 -0
- package/dist/expiry.js +115 -0
- package/dist/extend-chain.js +137 -0
- package/dist/extend-plan.js +149 -0
- package/dist/extend-sign.js +130 -0
- package/dist/guards.js +15 -0
- package/dist/help.js +192 -0
- package/dist/human-check.js +54 -0
- package/dist/item-trash.js +25 -0
- package/dist/kit-file.js +133 -0
- package/dist/list-file.js +71 -0
- package/dist/list-view-find.js +71 -0
- package/dist/list-view-order.js +96 -0
- package/dist/main.js +394 -0
- package/dist/manifest-create.js +81 -0
- package/dist/manifest-write.js +244 -0
- package/dist/manifest.js +213 -0
- package/dist/mark-render.js +45 -0
- package/dist/mcp-args.js +154 -0
- package/dist/mcp-tools/context.js +50 -0
- package/dist/mcp-tools/files.js +133 -0
- package/dist/mcp-tools/organise.js +128 -0
- package/dist/mcp-tools/reads.js +89 -0
- package/dist/mcp-tools/share.js +61 -0
- package/dist/mcp.js +142 -0
- package/dist/net-retry.js +84 -0
- package/dist/network.js +43 -0
- package/dist/notice.js +45 -0
- package/dist/product.js +38 -0
- package/dist/progress.js +110 -0
- package/dist/prompt.js +145 -0
- package/dist/rebuild.js +255 -0
- package/dist/recovery-assemble.js +148 -0
- package/dist/recovery-build.js +260 -0
- package/dist/recovery-map-file.js +114 -0
- package/dist/recovery-map.js +191 -0
- package/dist/recovery-release.js +229 -0
- package/dist/recovery-seq.js +63 -0
- package/dist/recovery-source.js +143 -0
- package/dist/registration.js +83 -0
- package/dist/s3/listing.js +134 -0
- package/dist/s3/multipart.js +104 -0
- package/dist/s3/response-sink.js +41 -0
- package/dist/s3/same-file.js +117 -0
- package/dist/s3/server.js +252 -0
- package/dist/s3/sigv4.js +168 -0
- package/dist/s3/staging.js +87 -0
- package/dist/s3/xml.js +80 -0
- package/dist/safe-path.js +89 -0
- package/dist/seal.js +206 -0
- package/dist/secret-reader.js +112 -0
- package/dist/server.js +33 -0
- package/dist/session.js +49 -0
- package/dist/setup-questions.js +31 -0
- package/dist/share.js +195 -0
- package/dist/shared/lib/crypto/size-padding.js +126 -0
- package/dist/shared/lib/drive/manifest-codec.js +287 -0
- package/dist/shared/lib/drive/manifest-index.js +235 -0
- package/dist/shared/lib/drive/manifest-ops.js +267 -0
- package/dist/shared/lib/drive/manifest-settings.js +65 -0
- package/dist/shared/lib/drive/name-conflict.js +121 -0
- package/dist/shared/lib/drive/unique-name.js +48 -0
- package/dist/shared/lib/extend/epochs.js +256 -0
- package/dist/shared/lib/net/retry-budget.js +95 -0
- package/dist/shared/lib/share/shared-file-info.js +65 -0
- package/dist/shared/lib/storage-network.js +64 -0
- package/dist/shared/lib/upload/part-plan.js +28 -0
- package/dist/stdout.js +109 -0
- package/dist/trash-sweep.js +123 -0
- package/dist/units.js +19 -0
- package/dist/update-check.js +164 -0
- package/dist/update-source.js +143 -0
- package/dist/upload-api.js +98 -0
- package/dist/upload-file.js +242 -0
- package/dist/upload-price.js +117 -0
- package/dist/upload-steps.js +111 -0
- package/dist/upload-store.js +252 -0
- package/dist/upload-wire.js +25 -0
- package/dist/upload.js +294 -0
- package/dist/usage-report.js +54 -0
- package/dist/wallet-chain.js +76 -0
- package/dist/wallet.js +209 -0
- package/dist/walrus-write.js +188 -0
- package/dist/walrus.js +156 -0
- package/package.json +56 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm.d.ts +469 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm.js +1300 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm +0 -0
- package/vendor/nmts-crypto/nmts_crypto_wasm_bg.wasm.d.ts +55 -0
- package/vendor/nmts-crypto/package.json +17 -0
|
@@ -0,0 +1,252 @@
|
|
|
1
|
+
// Where an unfinished PAID upload waits.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ THIS FILE EXISTS BECAUSE THE CREDITS MOVE BEFORE THE BYTES DO. `POST /v1/sponsored/reserve`
|
|
4
|
+
// spends the account's credits and buys storage on the network; only after that do the bytes
|
|
5
|
+
// go to the relay. A process that dies in between has bought storage it never filled, and a
|
|
6
|
+
// plain retry would buy a second lot. What makes the retry free instead is sending the SAME
|
|
7
|
+
// idempotency key with the SAME blob — and the blob id is a function of the sealed bytes and a
|
|
8
|
+
// random tip nonce, neither of which can be re-derived once they are gone.
|
|
9
|
+
//
|
|
10
|
+
// So they are written down BEFORE the money moves. That ordering is the whole design.
|
|
11
|
+
//
|
|
12
|
+
// ⛔ WHAT IS ON DISK IS ALREADY PUBLIC. The `.bin` is the sealed NCF-3 stream — the exact bytes
|
|
13
|
+
// about to be handed to a public storage network. It is still written 0600, because "already
|
|
14
|
+
// public" is about the CONTENT and the file's presence would otherwise say which files this
|
|
15
|
+
// account uploaded and when.
|
|
16
|
+
//
|
|
17
|
+
// ⛔ THE FILE NAME IS NOT A CONTENT FINGERPRINT. Keying by SHA-256 of the plaintext would leave a
|
|
18
|
+
// directory of hashes matchable against published hash sets — the very thing sealing the
|
|
19
|
+
// content hash avoids. The key mixes the account's data key in, so it identifies the file only
|
|
20
|
+
// to somebody who already holds the account.
|
|
21
|
+
import { createHash } from "node:crypto";
|
|
22
|
+
import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
|
|
23
|
+
import { join } from "node:path";
|
|
24
|
+
import { configDir, modesAreEnforced } from "./credentials.js";
|
|
25
|
+
import { chmodSync } from "node:fs";
|
|
26
|
+
import { NmtsError } from "./errors.js";
|
|
27
|
+
/** Where unfinished uploads live. */
|
|
28
|
+
export function uploadsDir() {
|
|
29
|
+
return join(configDir(), "uploads");
|
|
30
|
+
}
|
|
31
|
+
/**
|
|
32
|
+
* A stable, account-scoped name for one file's upload attempt.
|
|
33
|
+
*
|
|
34
|
+
* Same account, same bytes, same destination give the same key — which is what lets a second `put`
|
|
35
|
+
* of a file whose first attempt died find the reservation instead of paying again.
|
|
36
|
+
*
|
|
37
|
+
* ⛔ THE DESTINATION IS PART OF IT, and leaving it out was a real defect. Two local files with
|
|
38
|
+
* identical content — a copy, a re-download, `a.bin` and `backup/a.bin` — would otherwise share
|
|
39
|
+
* one reservation, and putting the second one would silently resume the first: it would report
|
|
40
|
+
* success, spend nothing, and write a SECOND file-list entry pointing at the FIRST file's item.
|
|
41
|
+
* Deleting either would then break the other. They are two files in the drive, so they are two
|
|
42
|
+
* uploads.
|
|
43
|
+
*
|
|
44
|
+
* ⚠ It is the destination AS TYPED, not the folder id, because the id needs a network round trip
|
|
45
|
+
* and this key is wanted before one. Two spellings of one folder therefore make two reservations
|
|
46
|
+
* — one extra unfinished record, never a wrong file.
|
|
47
|
+
*/
|
|
48
|
+
export function reservationKey(dataKey, plaintext, name, destination) {
|
|
49
|
+
return reservationKeyStreamed(dataKey, [plaintext], name, destination);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* The same key, computed without ever holding the file.
|
|
53
|
+
*
|
|
54
|
+
* ⛔ IT MUST HASH THE IDENTICAL BYTE SEQUENCE. A file large enough to need several parts is a file
|
|
55
|
+
* too large to hand `reservationKey` as one array, and a second construction here — hashing a
|
|
56
|
+
* hash, hashing the parts' digests — would give the same file two different keys depending on
|
|
57
|
+
* how it was read. The one above is written in terms of this one so there is only ever one
|
|
58
|
+
* answer, and a test feeds the same file both ways to hold that.
|
|
59
|
+
*/
|
|
60
|
+
export function reservationKeyStreamed(dataKey, plaintextChunks, name, destination) {
|
|
61
|
+
const hash = startReservationKey(dataKey);
|
|
62
|
+
for (const chunk of plaintextChunks)
|
|
63
|
+
hash.update(chunk);
|
|
64
|
+
return finishReservationKey(hash, name, destination);
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Begin the hash a reservation key is made from.
|
|
68
|
+
*
|
|
69
|
+
* ⛔ TWO HASHES, ONE READ. A file large enough to need parts is read once to work out its key AND
|
|
70
|
+
* the SHA-256 the account checks its contents against; handing the caller the running hash is
|
|
71
|
+
* what lets both come out of a single pass instead of two reads of a very large file.
|
|
72
|
+
*/
|
|
73
|
+
export function startReservationKey(dataKey) {
|
|
74
|
+
return createHash("sha256").update(dataKey);
|
|
75
|
+
}
|
|
76
|
+
/** Finish it. The name and the destination go in last, exactly as the one-shot form does. */
|
|
77
|
+
export function finishReservationKey(hash, name, destination) {
|
|
78
|
+
const digest = hash.update(new TextEncoder().encode(`\u0000${name}\u0000${destination}`)).digest();
|
|
79
|
+
return Buffer.from(digest).toString("base64url").slice(0, 32);
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* The record name for ONE part of a file.
|
|
83
|
+
*
|
|
84
|
+
* ⛔ EVERY PART IS ITS OWN RESERVATION. Each one buys its own storage, under its own idempotency
|
|
85
|
+
* key, and can fail or resume on its own — so each one is written down on its own. Sharing a
|
|
86
|
+
* record between parts would mean a run that died between part 3 and part 4 could not say which
|
|
87
|
+
* of them the credits had already been spent on.
|
|
88
|
+
*/
|
|
89
|
+
export function partKey(fileKey, partIndex) {
|
|
90
|
+
return `${fileKey}~p${partIndex}`;
|
|
91
|
+
}
|
|
92
|
+
function paths(key) {
|
|
93
|
+
const dir = uploadsDir();
|
|
94
|
+
return { json: join(dir, `${key}.json`), bin: join(dir, `${key}.bin`) };
|
|
95
|
+
}
|
|
96
|
+
function isReservation(value) {
|
|
97
|
+
if (typeof value !== "object" || value === null)
|
|
98
|
+
return false;
|
|
99
|
+
for (const name of ["blobId", "nonceB64", "rootHashB64", "relayUrl", "dekWrapped", "contentHashCt", "name"]) {
|
|
100
|
+
if (typeof Reflect.get(value, name) !== "string")
|
|
101
|
+
return false;
|
|
102
|
+
}
|
|
103
|
+
for (const name of ["epochs", "sealedLen", "plaintextLen", "partPlaintextLen", "partIndex", "partTotal", "attempt"]) {
|
|
104
|
+
if (typeof Reflect.get(value, name) !== "number")
|
|
105
|
+
return false;
|
|
106
|
+
}
|
|
107
|
+
const parent = Reflect.get(value, "parentId");
|
|
108
|
+
return parent === null || typeof parent === "string";
|
|
109
|
+
}
|
|
110
|
+
/**
|
|
111
|
+
* The waiting reservation's RECORD, without its sealed bytes.
|
|
112
|
+
*
|
|
113
|
+
* ⛔ THE BYTES ARE THE FILE. Reading them to answer "has this part been paid for?" would mean a
|
|
114
|
+
* resume that only needs to commit still reads every byte of a very large upload off the disk.
|
|
115
|
+
* They are fetched separately, by the one step that actually pushes them.
|
|
116
|
+
*/
|
|
117
|
+
export function readReservationRecord(key) {
|
|
118
|
+
const { json, bin } = paths(key);
|
|
119
|
+
if (!existsSync(json) || !existsSync(bin))
|
|
120
|
+
return null;
|
|
121
|
+
let parsed;
|
|
122
|
+
try {
|
|
123
|
+
parsed = JSON.parse(readFileSync(json, "utf8"));
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
// ⛔ Unreadable is not the same as absent, and treating it as absent would buy storage twice.
|
|
127
|
+
throw new NmtsError(`An unfinished upload record at ${json} could not be read.`, {
|
|
128
|
+
nextStep: "It names storage this account may already have paid for. Move it aside rather than " +
|
|
129
|
+
"deleting it if the upload matters, then try again.",
|
|
130
|
+
});
|
|
131
|
+
}
|
|
132
|
+
if (!isReservation(parsed)) {
|
|
133
|
+
throw new NmtsError(`The unfinished upload record at ${json} is not in a shape this version knows.`, {
|
|
134
|
+
nextStep: "Move it aside and try again. Nothing was sent.",
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return parsed;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* The sealed bytes a reservation bought.
|
|
141
|
+
*
|
|
142
|
+
* ⛔ NEVER RE-SEALED ONES. Sealing is non-deterministic, so bytes produced by a later run are a
|
|
143
|
+
* different blob from the one the treasury registered — the relay refuses them, forever, and
|
|
144
|
+
* the credits are gone.
|
|
145
|
+
*/
|
|
146
|
+
export function readReservationBytes(key) {
|
|
147
|
+
const { bin } = paths(key);
|
|
148
|
+
return new Uint8Array(readFileSync(bin));
|
|
149
|
+
}
|
|
150
|
+
/** The record and its bytes together, for the callers that need both. */
|
|
151
|
+
export function readReservation(key) {
|
|
152
|
+
const record = readReservationRecord(key);
|
|
153
|
+
if (record === null)
|
|
154
|
+
return null;
|
|
155
|
+
return { record, sealed: readReservationBytes(key) };
|
|
156
|
+
}
|
|
157
|
+
/** Write the record and its sealed bytes. Called BEFORE the reserve, and again after it answers. */
|
|
158
|
+
export function writeReservation(key, record, sealed) {
|
|
159
|
+
const dir = uploadsDir();
|
|
160
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
161
|
+
if (modesAreEnforced())
|
|
162
|
+
chmodSync(dir, 0o700);
|
|
163
|
+
const { json, bin } = paths(key);
|
|
164
|
+
// ⛔ WRITTEN ASIDE AND RENAMED OVER, never truncated in place. A rename within one directory is
|
|
165
|
+
// atomic, so a reader sees the old record or the new one and never a half-written one. The
|
|
166
|
+
// write that matters is the LAST one — the one that adds the item id — because a truncated
|
|
167
|
+
// file there is the only local pointer to a file that is already paid for and committed, and
|
|
168
|
+
// losing it makes that file invisible.
|
|
169
|
+
atomically(bin, sealed);
|
|
170
|
+
atomically(json, Buffer.from(`${JSON.stringify(record, null, 2)}\n`, "utf8"));
|
|
171
|
+
}
|
|
172
|
+
function atomically(target, bytes) {
|
|
173
|
+
const scratch = `${target}.${process.pid}.tmp`;
|
|
174
|
+
writeFileSync(scratch, bytes, { mode: 0o600 });
|
|
175
|
+
if (modesAreEnforced())
|
|
176
|
+
chmodSync(scratch, 0o600);
|
|
177
|
+
renameSync(scratch, target);
|
|
178
|
+
}
|
|
179
|
+
/**
|
|
180
|
+
* Forget a reservation.
|
|
181
|
+
*
|
|
182
|
+
* ⚠ Called on success AND on a couple of failure paths, so the comment that used to say "the
|
|
183
|
+
* upload already succeeded" was not true of every caller. What IS true of all of them is that
|
|
184
|
+
* nothing further depends on the record, which is why it never throws.
|
|
185
|
+
*/
|
|
186
|
+
export function clearReservation(key) {
|
|
187
|
+
const { json, bin } = paths(key);
|
|
188
|
+
for (const path of [json, bin]) {
|
|
189
|
+
try {
|
|
190
|
+
rmSync(path, { force: true });
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// ⚠ A record that cannot be removed is left where it is. That is not free -- a record
|
|
194
|
+
// carrying an item id is READ before anything is written, so a stale one would be
|
|
195
|
+
// resumed rather than overwritten. It is still better than failing a finished upload
|
|
196
|
+
// over a file that could not be deleted.
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
function itemPath(fileKey) {
|
|
201
|
+
return join(uploadsDir(), `${fileKey}.item.json`);
|
|
202
|
+
}
|
|
203
|
+
function isItemRecord(value) {
|
|
204
|
+
if (typeof value !== "object" || value === null)
|
|
205
|
+
return false;
|
|
206
|
+
if (typeof Reflect.get(value, "attempt") !== "number")
|
|
207
|
+
return false;
|
|
208
|
+
const id = Reflect.get(value, "itemId");
|
|
209
|
+
return id === undefined || typeof id === "string";
|
|
210
|
+
}
|
|
211
|
+
/** What is known about this file's commit, or `null` when it has not been attempted. */
|
|
212
|
+
export function readItemRecord(fileKey) {
|
|
213
|
+
const path = itemPath(fileKey);
|
|
214
|
+
if (!existsSync(path))
|
|
215
|
+
return null;
|
|
216
|
+
let parsed;
|
|
217
|
+
try {
|
|
218
|
+
parsed = JSON.parse(readFileSync(path, "utf8"));
|
|
219
|
+
}
|
|
220
|
+
catch {
|
|
221
|
+
// ⛔ Same reasoning as an unreadable reservation: unreadable is not absent. This file is the
|
|
222
|
+
// only local pointer to storage that is already bought and possibly already committed.
|
|
223
|
+
throw new NmtsError(`An unfinished upload record at ${path} could not be read.`, {
|
|
224
|
+
nextStep: "It names a file this account may already have paid for. Move it aside rather than " +
|
|
225
|
+
"deleting it if the upload matters, then try again.",
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
if (!isItemRecord(parsed)) {
|
|
229
|
+
throw new NmtsError(`The unfinished upload record at ${path} is not in a shape this version knows.`, {
|
|
230
|
+
nextStep: "Move it aside and try again. Nothing was sent.",
|
|
231
|
+
});
|
|
232
|
+
}
|
|
233
|
+
return parsed;
|
|
234
|
+
}
|
|
235
|
+
/** Write the file-level record. Called before the commit, and again once it has an id. */
|
|
236
|
+
export function writeItemRecord(fileKey, record) {
|
|
237
|
+
const dir = uploadsDir();
|
|
238
|
+
mkdirSync(dir, { recursive: true, mode: 0o700 });
|
|
239
|
+
if (modesAreEnforced())
|
|
240
|
+
chmodSync(dir, 0o700);
|
|
241
|
+
atomically(itemPath(fileKey), Buffer.from(`${JSON.stringify(record, null, 2)}\n`, "utf8"));
|
|
242
|
+
}
|
|
243
|
+
/** Forget the file-level record. Never throws, for the same reason `clearReservation` does not. */
|
|
244
|
+
export function clearItemRecord(fileKey) {
|
|
245
|
+
try {
|
|
246
|
+
rmSync(itemPath(fileKey), { force: true });
|
|
247
|
+
}
|
|
248
|
+
catch {
|
|
249
|
+
// ⚠ Left where it is. A stale one carrying an item id would be resumed rather than
|
|
250
|
+
// overwritten, which is still better than failing a finished upload over a stuck file.
|
|
251
|
+
}
|
|
252
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
// The shapes the credit-paid upload speaks in — the storage-network protocol, the api calls, and
|
|
2
|
+
// what a failure is allowed to claim about money.
|
|
3
|
+
//
|
|
4
|
+
// ⛔ SEPARATE FROM THE MACHINE ON PURPOSE. `upload.ts` is a sequence of decisions about spending;
|
|
5
|
+
// keeping the vocabulary here means a test can name every seam without importing that sequence,
|
|
6
|
+
// and means the file that DOES spend stays short enough to read in one sitting.
|
|
7
|
+
import { NmtsError } from "./errors.js";
|
|
8
|
+
/**
|
|
9
|
+
* A failure that names its phase — and, crucially, whether the account has already paid.
|
|
10
|
+
*
|
|
11
|
+
* ⛔ `paid` IS NOT COSMETIC. Before the reserve, a failure costs nothing and "try again" is honest
|
|
12
|
+
* advice. After it, the credits are gone and the storage exists; the honest advice is that the
|
|
13
|
+
* same command will FINISH it rather than buy it again, and that saying otherwise would send
|
|
14
|
+
* somebody to spend twice.
|
|
15
|
+
*/
|
|
16
|
+
export class UploadError extends NmtsError {
|
|
17
|
+
phase;
|
|
18
|
+
paid;
|
|
19
|
+
constructor(input) {
|
|
20
|
+
super(input.message, { exitCode: 1, nextStep: input.nextStep ?? null });
|
|
21
|
+
this.name = "UploadError";
|
|
22
|
+
this.phase = input.phase;
|
|
23
|
+
this.paid = input.paid;
|
|
24
|
+
}
|
|
25
|
+
}
|
package/dist/upload.js
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
1
|
+
// The credit-paid upload, end to end, for ONE file.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ THIS IS THE ONLY PATH IN THIS TOOL THAT SPENDS. Everything else reads. The order of the steps
|
|
4
|
+
// below is not a style choice — it is what stands between an interruption and money that bought
|
|
5
|
+
// nothing:
|
|
6
|
+
//
|
|
7
|
+
// seal → encode → WRITE IT DOWN → reserve (credits move here) → write it down again
|
|
8
|
+
// → relay → certify … once per part …
|
|
9
|
+
// → commit the file (every part named at once) → write the file list → forget the records
|
|
10
|
+
//
|
|
11
|
+
// The two writes bracket the spend. Before it, so a retry has the sealed bytes and the tip
|
|
12
|
+
// nonce that reproduce the blob the reservation bought. After it, so the transaction the relay
|
|
13
|
+
// checks its tip in is not lost with the process.
|
|
14
|
+
//
|
|
15
|
+
// ⛔ A RESUME READS THE RECORD AND NOT THE CALLER. Sealing is non-deterministic, so a run that
|
|
16
|
+
// re-sealed and then resumed would push bytes that are not the blob the treasury registered —
|
|
17
|
+
// the relay refuses them, forever, and the credits are gone. Everything the reservation bought
|
|
18
|
+
// is a function of ONE PARTICULAR SEALING of the file: the blob id, the wrapped file key, the
|
|
19
|
+
// content hash, the relay that was tipped. All of it comes back out of the record.
|
|
20
|
+
//
|
|
21
|
+
// ⛔ NO WALLET IS TOUCHED AND NOTHING IS SIGNED HERE. The treasury pays for the storage against
|
|
22
|
+
// credits the account already holds; this tool encodes bytes, asks the server to buy, pushes
|
|
23
|
+
// the bytes at the relay and reports what the storage nodes signed. There is no signer in this
|
|
24
|
+
// file and no transaction is built.
|
|
25
|
+
//
|
|
26
|
+
// ⛔ EVERY SEAM IS INJECTED. The storage-network protocol and the api are interfaces, so the tests
|
|
27
|
+
// drive the real decisions — including every failure branch — without a network and without
|
|
28
|
+
// spending anything. The seams themselves are in `upload-wire.ts`.
|
|
29
|
+
import { NmtsError } from "./errors.js";
|
|
30
|
+
import { ServerError } from "./api.js";
|
|
31
|
+
import { readReservationBytes, readReservationRecord, writeReservation, } from "./upload-store.js";
|
|
32
|
+
import { pushPart } from "./upload-steps.js";
|
|
33
|
+
import { UploadError, } from "./upload-wire.js";
|
|
34
|
+
export * from "./upload-wire.js";
|
|
35
|
+
/** A reservation state that can still become storage. Anything else is dead. */
|
|
36
|
+
function isLive(state) {
|
|
37
|
+
return state === "registered" || state === "certified";
|
|
38
|
+
}
|
|
39
|
+
function why(error) {
|
|
40
|
+
return error instanceof Error ? error.message : String(error);
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Buy storage for ONE PART, put its bytes on the network, and stop there.
|
|
44
|
+
*
|
|
45
|
+
* ⛔ IT DOES NOT MAKE A FILE. Committing is one act for the whole file — `commitItem` names every
|
|
46
|
+
* part at once — and a part that returns from here is bought, filled and certified storage that
|
|
47
|
+
* nothing in the account can see yet. Splitting it this way is what lets a file be larger than
|
|
48
|
+
* memory: the parts are bought one at a time, each written down before its own money moves.
|
|
49
|
+
*
|
|
50
|
+
* ⛔ IT DOES NOT WRITE THE FILE LIST either. That is the caller's step, and it must happen before
|
|
51
|
+
* the records are cleared — a committed file the list does not name is invisible and, to the
|
|
52
|
+
* person, indistinguishable from one that never uploaded.
|
|
53
|
+
*/
|
|
54
|
+
export async function buyAndPushPart(input) {
|
|
55
|
+
const { api, protocol, key, sealed, onStep } = input;
|
|
56
|
+
const existing = readReservationRecord(key);
|
|
57
|
+
// ⛔ THE STORED PLACEMENT WINS THE ARGUMENT, AND A DISAGREEMENT STOPS THE RUN. The bytes on disk
|
|
58
|
+
// were sealed as part i of n and paid for as that; pushing them while this run believes the
|
|
59
|
+
// file splits some other way would file storage under the wrong position in the file. It
|
|
60
|
+
// happens for one ordinary reason — the same file put again with a different part size — so
|
|
61
|
+
// it is said plainly rather than left to fail later as an unreadable download.
|
|
62
|
+
if (existing !== null) {
|
|
63
|
+
const record = existing;
|
|
64
|
+
if (record.partIndex !== input.part.index || record.partTotal !== input.part.total) {
|
|
65
|
+
throw new UploadError({
|
|
66
|
+
phase: "reserve",
|
|
67
|
+
message: `This upload was started as part ${record.partIndex + 1} of ${record.partTotal} and this ` +
|
|
68
|
+
`run is treating it as part ${input.part.index + 1} of ${input.part.total}.`,
|
|
69
|
+
paid: record.ledgerId !== undefined,
|
|
70
|
+
nextStep: "Run it again with the part size the first attempt used, or move the records in the " +
|
|
71
|
+
"uploads directory aside to start over. Nothing was sent.",
|
|
72
|
+
});
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// ── the credits already moved: ask where the reservation stands before doing anything ──
|
|
76
|
+
if (existing?.ledgerId !== undefined) {
|
|
77
|
+
const record = existing;
|
|
78
|
+
const ledgerId = record.ledgerId;
|
|
79
|
+
if (ledgerId === undefined)
|
|
80
|
+
throw new NmtsError("unreachable: a record with no ledger id");
|
|
81
|
+
let status;
|
|
82
|
+
try {
|
|
83
|
+
status = await api.status(ledgerId);
|
|
84
|
+
}
|
|
85
|
+
catch (error) {
|
|
86
|
+
throw new UploadError({
|
|
87
|
+
phase: "reserve",
|
|
88
|
+
message: `Could not ask about the paid reservation ${ledgerId}: ${why(error)}`,
|
|
89
|
+
paid: true,
|
|
90
|
+
nextStep: "Nothing more was spent. The storage this account paid for is still bought — run the " +
|
|
91
|
+
"same command again when the server answers.",
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
onStep?.({ step: "resuming", ledgerId, state: status.state });
|
|
95
|
+
if (status.state === "certified") {
|
|
96
|
+
return paidPart(record, ledgerId, true);
|
|
97
|
+
}
|
|
98
|
+
if (!isLive(status.state)) {
|
|
99
|
+
// ⛔ THE RECORD IS KEPT AND ITS ATTEMPT NUMBER GOES UP. Deleting it looks tidier and is the
|
|
100
|
+
// trap: the idempotency key is derived from a key that is a pure function of the account,
|
|
101
|
+
// the bytes and the destination, and the server replays a reservation row under its key
|
|
102
|
+
// whatever state it is in — `failed` included. A cleared record means the next run
|
|
103
|
+
// rebuilds the same key, is handed the same dead row, and is told to start over into it.
|
|
104
|
+
// Forever. Counting up is what starting over actually means.
|
|
105
|
+
writeReservation(key, { ...stripReservation(record), attempt: record.attempt + 1 }, readReservationBytes(key));
|
|
106
|
+
throw new UploadError({
|
|
107
|
+
phase: "reserve",
|
|
108
|
+
message: `The credit reservation for this file ended as "${status.state}" and cannot be used.`,
|
|
109
|
+
paid: true,
|
|
110
|
+
nextStep: "Running the same command again asks for a NEW reservation, which will spend credits. " +
|
|
111
|
+
"Nothing was uploaded under the one that failed.",
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
if (!status.register_tx_digest || !status.blob_object_id) {
|
|
115
|
+
throw new UploadError({
|
|
116
|
+
phase: "reserve",
|
|
117
|
+
message: `Reservation ${ledgerId} is registered but the server did not say which blob it bought.`,
|
|
118
|
+
paid: true,
|
|
119
|
+
nextStep: "Nothing was uploaded and nothing more was spent.",
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
await pushPart(input, {
|
|
123
|
+
ledgerId,
|
|
124
|
+
blobId: record.blobId,
|
|
125
|
+
nonce: Buffer.from(record.nonceB64, "base64url"),
|
|
126
|
+
registerTxDigest: status.register_tx_digest,
|
|
127
|
+
blobObjectId: status.blob_object_id,
|
|
128
|
+
// ⛔ THE STORED BYTES AND THE STORED RELAY. Not the caller's — see the module header.
|
|
129
|
+
// Read here, and only here: a part that came back certified never needs them at all.
|
|
130
|
+
sealed: readReservationBytes(key),
|
|
131
|
+
relayUrl: record.relayUrl,
|
|
132
|
+
});
|
|
133
|
+
return paidPart(record, ledgerId, true);
|
|
134
|
+
}
|
|
135
|
+
// ── fresh, or interrupted before the money moved ──
|
|
136
|
+
onStep?.({ step: "encoding", bytes: sealed.length });
|
|
137
|
+
let meta;
|
|
138
|
+
try {
|
|
139
|
+
meta = await protocol.computeMetadata({
|
|
140
|
+
bytes: sealed,
|
|
141
|
+
// Re-feeding a stored nonce is what makes the retry reproduce the digest a paid tip was
|
|
142
|
+
// computed for. A fresh random one would strand the reservation.
|
|
143
|
+
nonce: existing ? new Uint8Array(Buffer.from(existing.nonceB64, "base64url")) : undefined,
|
|
144
|
+
});
|
|
145
|
+
}
|
|
146
|
+
catch (error) {
|
|
147
|
+
throw new UploadError({
|
|
148
|
+
phase: "encoding",
|
|
149
|
+
message: `Could not prepare this file for the storage network: ${why(error)}`,
|
|
150
|
+
paid: false,
|
|
151
|
+
nextStep: "Nothing was sent and nothing was spent.",
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
const nonceB64 = Buffer.from(meta.nonce).toString("base64url");
|
|
155
|
+
const record = {
|
|
156
|
+
// ⛔ The attempt number is CARRIED FORWARD, not reset. A record that survives a dead
|
|
157
|
+
// reservation is exactly the case that needs a different idempotency key.
|
|
158
|
+
attempt: existing?.attempt ?? 0,
|
|
159
|
+
blobId: meta.blobId,
|
|
160
|
+
nonceB64,
|
|
161
|
+
rootHashB64: Buffer.from(meta.rootHash).toString("base64url"),
|
|
162
|
+
relayUrl: input.relayUrl,
|
|
163
|
+
epochs: input.epochs,
|
|
164
|
+
sealedLen: sealed.length,
|
|
165
|
+
plaintextLen: input.entry.plaintextLen,
|
|
166
|
+
partPlaintextLen: input.part.plaintextLen,
|
|
167
|
+
partIndex: input.part.index,
|
|
168
|
+
partTotal: input.part.total,
|
|
169
|
+
dekWrapped: input.entry.dekWrapped,
|
|
170
|
+
contentHashCt: input.entry.contentHashCt,
|
|
171
|
+
name: input.entry.name,
|
|
172
|
+
parentId: input.entry.parentId,
|
|
173
|
+
};
|
|
174
|
+
// ⛔ BEFORE THE MONEY. See the module header.
|
|
175
|
+
writeReservation(key, record, sealed);
|
|
176
|
+
onStep?.({ step: "reserving" });
|
|
177
|
+
let reply;
|
|
178
|
+
try {
|
|
179
|
+
reply = await api.reserve({
|
|
180
|
+
idempotency_key: idempotencyKey(key, record.attempt),
|
|
181
|
+
blob_id: meta.blobId,
|
|
182
|
+
root_hash_b64: record.rootHashB64,
|
|
183
|
+
// One number, measured: what the treasury is asked to buy and what the credits are charged
|
|
184
|
+
// on are the same sealed bytes.
|
|
185
|
+
size: sealed.length,
|
|
186
|
+
epochs: input.epochs,
|
|
187
|
+
relay: {
|
|
188
|
+
host: input.relayUrl,
|
|
189
|
+
blob_digest_b64: Buffer.from(meta.blobDigest).toString("base64url"),
|
|
190
|
+
nonce_b64: nonceB64,
|
|
191
|
+
},
|
|
192
|
+
});
|
|
193
|
+
}
|
|
194
|
+
catch (error) {
|
|
195
|
+
throw new UploadError({
|
|
196
|
+
phase: "reserve",
|
|
197
|
+
message: why(error),
|
|
198
|
+
// ⛔ A refusal did not spend. A request that never got an answer MIGHT have — and the record
|
|
199
|
+
// is on disk either way, so the next run asks the server instead of guessing here.
|
|
200
|
+
paid: false,
|
|
201
|
+
nextStep: error instanceof ServerError
|
|
202
|
+
? "Nothing was uploaded. No credits were spent on a refused reservation."
|
|
203
|
+
: "If the request reached the server, credits may have been spent — running the same " +
|
|
204
|
+
"command again finds that reservation rather than making a second one.",
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
if (!isLive(reply.state)) {
|
|
208
|
+
// Same reason as the resumed branch above: keep the record, count up, so the next run does not
|
|
209
|
+
// ask under a key the server has already settled.
|
|
210
|
+
writeReservation(key, { ...stripReservation(record), attempt: record.attempt + 1 }, sealed);
|
|
211
|
+
throw new UploadError({
|
|
212
|
+
phase: "reserve",
|
|
213
|
+
message: `The reservation came back as "${reply.state}", which cannot become storage.`,
|
|
214
|
+
paid: reply.credits_spent > 0,
|
|
215
|
+
nextStep: "Nothing was uploaded.",
|
|
216
|
+
});
|
|
217
|
+
}
|
|
218
|
+
record.ledgerId = reply.ledger_id;
|
|
219
|
+
if (reply.register_tx_digest)
|
|
220
|
+
record.registerTxDigest = reply.register_tx_digest;
|
|
221
|
+
if (reply.blob_object_id)
|
|
222
|
+
record.blobObjectId = reply.blob_object_id;
|
|
223
|
+
writeReservation(key, record, sealed);
|
|
224
|
+
if (reply.state === "certified") {
|
|
225
|
+
return paidPart(record, reply.ledger_id, false);
|
|
226
|
+
}
|
|
227
|
+
if (!reply.register_tx_digest || !reply.blob_object_id) {
|
|
228
|
+
throw new UploadError({
|
|
229
|
+
phase: "reserve",
|
|
230
|
+
message: "The storage was bought but the reply did not say which blob it bought.",
|
|
231
|
+
paid: true,
|
|
232
|
+
nextStep: "Nothing was uploaded. Running the same command again asks the server again.",
|
|
233
|
+
});
|
|
234
|
+
}
|
|
235
|
+
await pushPart(input, {
|
|
236
|
+
ledgerId: reply.ledger_id,
|
|
237
|
+
blobId: meta.blobId,
|
|
238
|
+
nonce: meta.nonce,
|
|
239
|
+
registerTxDigest: reply.register_tx_digest,
|
|
240
|
+
blobObjectId: reply.blob_object_id,
|
|
241
|
+
sealed,
|
|
242
|
+
relayUrl: input.relayUrl,
|
|
243
|
+
});
|
|
244
|
+
return paidPart(record, reply.ledger_id, false);
|
|
245
|
+
}
|
|
246
|
+
/**
|
|
247
|
+
* What the commit needs about a finished part — read off the RECORD, never off the run.
|
|
248
|
+
*
|
|
249
|
+
* ⛔ THE SAME REASON THE FILE LIST ENTRY COMES FROM THE RECORD. What is stored on the network is
|
|
250
|
+
* one particular sealing; a resumed run that named its own freshly computed blob id or length
|
|
251
|
+
* would commit a part that does not exist.
|
|
252
|
+
*/
|
|
253
|
+
function paidPart(record, ledgerId, resumed) {
|
|
254
|
+
return {
|
|
255
|
+
partIndex: record.partIndex,
|
|
256
|
+
ledgerId,
|
|
257
|
+
blobId: record.blobId,
|
|
258
|
+
sealedLen: record.sealedLen,
|
|
259
|
+
resumed,
|
|
260
|
+
};
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* The idempotency key for one attempt at one file.
|
|
264
|
+
*
|
|
265
|
+
* ⛔ THE ATTEMPT NUMBER IS IN IT. Without it the key is a pure function of the account, the bytes
|
|
266
|
+
* and the destination — and the server replays whatever row it has under a key, including a
|
|
267
|
+
* settled one that can never become storage.
|
|
268
|
+
*/
|
|
269
|
+
function idempotencyKey(key, attempt) {
|
|
270
|
+
return `nmts-cli-${key}-${attempt}`;
|
|
271
|
+
}
|
|
272
|
+
/** What the file list must record about this upload. Always from the record, never from a run. */
|
|
273
|
+
export function entryOf(record) {
|
|
274
|
+
return {
|
|
275
|
+
name: record.name,
|
|
276
|
+
parentId: record.parentId,
|
|
277
|
+
plaintextLen: record.plaintextLen,
|
|
278
|
+
dekWrapped: record.dekWrapped,
|
|
279
|
+
contentHashCt: record.contentHashCt,
|
|
280
|
+
};
|
|
281
|
+
}
|
|
282
|
+
/**
|
|
283
|
+
* The record with everything the dead reservation put on it removed.
|
|
284
|
+
*
|
|
285
|
+
* ⛔ Explicit, not `delete`: a field left behind here is a resumed run believing it holds a
|
|
286
|
+
* transaction the relay will accept, and the failure would arrive after the money moved.
|
|
287
|
+
*/
|
|
288
|
+
function stripReservation(record) {
|
|
289
|
+
const { ledgerId, registerTxDigest, blobObjectId, ...rest } = record;
|
|
290
|
+
void ledgerId;
|
|
291
|
+
void registerTxDigest;
|
|
292
|
+
void blobObjectId;
|
|
293
|
+
return rest;
|
|
294
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// What an account is holding, worked out from the decrypted list and nothing else.
|
|
2
|
+
//
|
|
3
|
+
// ⛔ EVERY NUMBER IS EXACT, NOT ESTIMATED. The sealed list IS the drive — it is what every device
|
|
4
|
+
// reads to know what exists — so counting it is counting the account. Nothing here samples,
|
|
5
|
+
// rounds or asks the server: the server holds sizes and times, and the names and the tree it
|
|
6
|
+
// cannot read at all, so "how many folders do I have" has no server-side answer to fetch.
|
|
7
|
+
//
|
|
8
|
+
// ⛔ THE TRASH IS COUNTED SEPARATELY AND KEPT OUT OF THE TOTAL. The two figures answer different
|
|
9
|
+
// questions: the total is what the drive holds, and the trash figure is what is still stored
|
|
10
|
+
// and still paid for while being no longer in the drive. Folding them together would tell
|
|
11
|
+
// somebody who just deleted 4 GB that nothing changed; leaving the trash out silently would
|
|
12
|
+
// tell them the 4 GB is gone when the storage is still bought. It is the same split the account
|
|
13
|
+
// screen in a browser shows, and for the same reason.
|
|
14
|
+
//
|
|
15
|
+
// ⚠ IT COUNTS PLAINTEXT BYTES — what the files hold. What the storage network holds for the
|
|
16
|
+
// account is larger: sealing adds bytes to every file, and storage is bought in fixed units.
|
|
17
|
+
// This is not a bill, and nothing here should be printed as one.
|
|
18
|
+
import { fullPathOf, isLive, KIND_FILE } from "./drive-paths.js";
|
|
19
|
+
import { totalsOf } from "./shared/lib/drive/manifest-index.js";
|
|
20
|
+
/** How many of the largest files a report carries. */
|
|
21
|
+
export const BIGGEST_SHOWN = 5;
|
|
22
|
+
/**
|
|
23
|
+
* The whole report, in one pass over the list.
|
|
24
|
+
*
|
|
25
|
+
* The counts come from the same `totalsOf` the browser's own usage figures are built on — this
|
|
26
|
+
* package carries that module as a byte-for-byte copy — so the two cannot drift into disagreeing
|
|
27
|
+
* about what "in the trash" means. Being in the trash is INHERITED: a file under a trashed folder
|
|
28
|
+
* is trashed even though nothing marked the file itself, and a count that read only the file's own
|
|
29
|
+
* mark would report bytes as live that the server has already stopped serving.
|
|
30
|
+
*/
|
|
31
|
+
export function computeUsage(index) {
|
|
32
|
+
const totals = totalsOf(index);
|
|
33
|
+
const biggest = index.all
|
|
34
|
+
.filter((entry) => entry.kind === KIND_FILE && isLive(index, entry))
|
|
35
|
+
// Largest first, ties broken by name — so two runs over one unchanged list name the same
|
|
36
|
+
// files in the same order instead of reshuffling whatever the list order happened to be.
|
|
37
|
+
.slice()
|
|
38
|
+
.sort((a, b) => b.size - a.size || a.name.localeCompare(b.name))
|
|
39
|
+
.slice(0, BIGGEST_SHOWN)
|
|
40
|
+
.map((entry) => ({
|
|
41
|
+
id: entry.id,
|
|
42
|
+
name: entry.name,
|
|
43
|
+
path: fullPathOf(index, entry),
|
|
44
|
+
size: entry.size,
|
|
45
|
+
}));
|
|
46
|
+
return {
|
|
47
|
+
files: totals.files,
|
|
48
|
+
folders: totals.folders,
|
|
49
|
+
bytes: totals.bytes,
|
|
50
|
+
trashedFiles: totals.trashedFiles,
|
|
51
|
+
trashedBytes: totals.trashedBytes,
|
|
52
|
+
biggest,
|
|
53
|
+
};
|
|
54
|
+
}
|