@needmoretruth/nmts-cli 0.17.2

package/dist/upload-file.js

@@ -0,0 +1,242 @@
+// One local file, from a path to a committed item — however many parts that takes.
+//
+// ⛔ THIS IS WHERE A FILE BECOMES SEVERAL BLOBS. `upload.ts` buys and fills ONE part; this drives
+//    the plan, one part at a time, and then commits them together. Splitting it that way is what
+//    lets a file be larger than memory: the file is read a slice at a time and only one part's
+//    sealed bytes are ever live.
+//
+// ⛔ ONE FILE IS ONE COMMIT. The server derives an item's size from the sum of its parts and
+//    refuses a set whose indices are not a contiguous 0..n, so there is no such thing as
+//    committing half a file. A run that stops between parts has bought and filled storage that
+//    nothing in the account can see yet, and the record on disk is what turns that into a resume
+//    rather than a loss.
+//
+// ⛔ THE FILE IS READ TWICE AND THAT IS DELIBERATE. The first pass computes two things at once —
+//    the reservation key, and the SHA-256 the account will check the contents against — because
+//    both are needed BEFORE the first part is sealed. The second pass is the sealing itself. The
+//    alternative is holding the file, which is the thing this module exists to avoid.
+import { createHash } from "node:crypto";
+import { open } from "node:fs/promises";
+import { AAD } from "./crypto.js";
+import { NmtsError } from "./errors.js";
+import { fileSecrets, sealPart } from "./seal.js";
+import { planParts } from "./shared/lib/upload/part-plan.js";
+import { paddedPlaintextLen, } from "./shared/lib/crypto/size-padding.js";
+import { NCF3_SHAPE } from "./seal.js";
+import { buyAndPushPart, entryOf } from "./upload.js";
+import { commitItem } from "./upload-steps.js";
+import { finishReservationKey, partKey, readItemRecord, readReservationBytes, readReservationRecord, startReservationKey, } from "./upload-store.js";
+/** How much plaintext is handed to the engine at a time. Matches the format's own chunk size. */
+const READ_CHUNK_BYTES = 4 * 2 ** 20;
+/** Read a file off the disk, a chunk at a time. */
+export function fileSource(path, size) {
+    return {
+        size,
+        async *read(offset, length) {
+            const handle = await open(path, "r");
+            try {
+                const buffer = Buffer.allocUnsafe(Math.min(READ_CHUNK_BYTES, length));
+                let at = 0;
+                while (at < length) {
+                    const want = Math.min(buffer.length, length - at);
+                    const { bytesRead } = await handle.read(buffer, 0, want, offset + at);
+                    if (bytesRead === 0) {
+                        // ⛔ SHORT IS NOT DONE. The plan was made from the size this file had when it was
+                        //    measured; a read that ends early means it shrank underneath us, and sealing what
+                        //    arrived would declare a length the bytes do not match.
+                        throw new NmtsError(`${path} ended after ${at} of ${length} bytes.`, {
+                            nextStep: "Nothing was sent. The file changed while it was being read.",
+                        });
+                    }
+                    at += bytesRead;
+                    yield new Uint8Array(buffer.subarray(0, bytesRead));
+                }
+            }
+            finally {
+                await handle.close();
+            }
+        },
+    };
+}
+/**
+ * Upload one file and return what the caller must write into the account's file list.
+ *
+ * ⛔ IT DOES NOT WRITE THE FILE LIST, and the caller must — before clearing the records. A
+ *    committed file the list does not name is invisible and, to the person, indistinguishable from
+ *    one that never uploaded.
+ */
+export async function uploadFile(input) {
+    const { dataKey, source, onStep } = input;
+    if (source.size <= 0) {
+        throw new NmtsError("An empty file cannot be uploaded.", {
+            nextStep: "The storage network has nothing to store and would refuse the reservation.",
+        });
+    }
+    const plan = planParts(source.size, input.partSize);
+    onStep?.({ step: "planning", parts: plan.length, partSize: input.partSize });
+    // ── pass one: the reservation key and the content hash, from a single read ──
+    onStep?.({ step: "hashing", parts: plan.length });
+    const keyHash = startReservationKey(dataKey);
+    const contentDigest = await hashWhole(source, keyHash);
+    const fileKey = finishReservationKey(keyHash, input.name, input.destination);
+    // ⛔ ALREADY COMMITTED STOPS HERE, BEFORE ANY PART IS TOUCHED. A file that got as far as
+    //    `POST /v1/items` exists and is paid for; all that can still be missing is the account's own
+    //    list. Asking the server about every part again would be a round trip per part to learn
+    //    something the record already says.
+    const committed = readItemRecord(fileKey);
+    if (committed?.itemId !== undefined) {
+        const entry = recordedEntry(fileKey, plan.length, source.size);
+        if (entry !== null) {
+            return { itemId: committed.itemId, resumed: true, ledgerIds: [], fileKey, parts: plan.length, entry };
+        }
+    }
+    // ── the file's secrets: from the record if one exists, otherwise made now ──
+    const secrets = openSecrets(input, fileKey, plan.length, contentDigest);
+    try {
+        const entry = {
+            name: input.name,
+            parentId: input.parentId,
+            plaintextLen: source.size,
+            dekWrapped: secrets.dekWrapped,
+            contentHashCt: secrets.contentHashCt,
+        };
+        const paid = [];
+        for (const range of plan) {
+            const key = partKey(fileKey, range.partIndex);
+            const stored = readReservationRecord(key);
+            // ⛔ A PART THAT IS WRITTEN DOWN IS NEVER SEALED AGAIN. Its bytes are a particular sealing
+            //    the treasury may already have paid to register; a fresh one is a different blob.
+            // ⛔ THE LAST PART ONLY. Every reader recovers the parts' real lengths from the file's size
+            //    and what each stream declares, and that answer is unique only because the earlier parts
+            //    are exactly full. Padding one of them would not fail here — it would fail years later,
+            //    as a download that wrote padding into the middle of a file.
+            const isLast = range.partIndex === plan.length - 1;
+            const sealFrom = isLast
+                ? paddedPlaintextLen(range.length, input.padding.rule, {
+                    unitBytes: input.padding.unitBytes,
+                    shape: NCF3_SHAPE,
+                })
+                : range.length;
+            const sealed = stored === null
+                ? await sealPartOf(input, secrets.dek, range, plan.length, sealFrom)
+                : readReservationBytes(key);
+            paid.push(await buyAndPushPart({
+                api: input.api,
+                protocol: input.protocol,
+                key,
+                sealed,
+                relayUrl: input.relayUrl,
+                epochs: input.epochs,
+                currentEpoch: input.currentEpoch,
+                // ⚠ The length the STREAM declares, which for a padded last part is more than the file
+                //   contributes. The list entry keeps the file's real size; this is what was sealed.
+                part: { index: range.partIndex, total: plan.length, plaintextLen: sealFrom },
+                entry,
+                onStep: (step) => onStep?.({ ...step, partIndex: range.partIndex, parts: plan.length }),
+            }));
+        }
+        const itemId = await commitItem({
+            api: input.api,
+            epochs: input.epochs,
+            currentEpoch: input.currentEpoch,
+            entry,
+            onStep: (step) => onStep?.({ ...step, partIndex: plan.length - 1, parts: plan.length }),
+        }, fileKey, paid);
+        return {
+            itemId,
+            resumed: paid.every((part) => part.resumed),
+            ledgerIds: paid.map((part) => part.ledgerId),
+            fileKey,
+            parts: plan.length,
+            entry,
+        };
+    }
+    finally {
+        secrets.dek.fill(0);
+    }
+}
+/** The file key's records, so a caller can clear them once the list is written. */
+export function partKeysOf(fileKey, parts) {
+    return Array.from({ length: parts }, (_, index) => partKey(fileKey, index));
+}
+/** Read the whole file once, feeding the key hash and returning the content digest. */
+async function hashWhole(source, keyHash) {
+    const content = createHash("sha256");
+    for await (const chunk of source.read(0, source.size)) {
+        keyHash.update(chunk);
+        content.update(chunk);
+    }
+    return new Uint8Array(content.digest());
+}
+/**
+ * The file's key and sealed content hash — from a written-down part if there is one.
+ *
+ * ⛔ A RESUMED RUN MUST NOT MAKE A NEW KEY. Every part of this file was sealed under one key, and
+ *    the bytes already on the network cannot be re-sealed. Unwrapping the recorded one is what
+ *    makes the parts still to come belong to the same file.
+ */
+function openSecrets(input, fileKey, parts, contentDigest) {
+    for (const key of partKeysOf(fileKey, parts)) {
+        const record = readReservationRecord(key);
+        if (record === null)
+            continue;
+        const wrapped = new Uint8Array(Buffer.from(record.dekWrapped, "base64url"));
+        const dek = input.crypt.envelope_open(input.dataKey, new TextEncoder().encode(AAD.dekWrap), wrapped);
+        return { dek, dekWrapped: record.dekWrapped, contentHashCt: record.contentHashCt };
+    }
+    return fileSecrets(input.crypt, input.dataKey, contentDigest);
+}
+async function sealPartOf(input, dek, range, parts, sealFrom) {
+    input.onStep?.({
+        step: "sealing",
+        partIndex: range.partIndex,
+        parts,
+        bytes: sealFrom,
+    });
+    return sealPart(input.crypt, dek, padded(input.source.read(range.offset, range.length), range.length, sealFrom), {
+        index: range.partIndex,
+        total: parts,
+        plaintextLen: sealFrom,
+    });
+}
+/**
+ * The part's bytes, followed by zeros up to the length it will declare.
+ *
+ * ⛔ ZEROS APPENDED TO THE PLAINTEXT, NOT BYTES TACKED ONTO THE STORED STREAM. A stream's header is
+ *    authenticated but not encrypted, so padding added after sealing would leave the real length
+ *    legible in the header of a public object — which is the exact thing this is for.
+ */
+async function* padded(chunks, real, sealFrom) {
+    for await (const chunk of chunks)
+        yield chunk;
+    let left = sealFrom - real;
+    const zeros = new Uint8Array(Math.min(left, 1 << 20));
+    while (left > 0) {
+        const take = Math.min(left, zeros.length);
+        yield zeros.subarray(0, take);
+        left -= take;
+    }
+}
+export { entryOf };
+/**
+ * The file list entry a written-down part remembers.
+ *
+ * ⛔ FROM THE RECORD, NEVER FROM THE RUN. The key that opens the stored bytes is the key they were
+ *    sealed with. A run that wrote its own freshly generated one into the list would produce a
+ *    file that is paid for, present, correctly named and impossible to open.
+ */
+function recordedEntry(fileKey, parts, size) {
+    for (const key of partKeysOf(fileKey, parts)) {
+        const record = readReservationRecord(key);
+        if (record === null)
+            continue;
+        return {
+            name: record.name,
+            parentId: record.parentId,
+            plaintextLen: size,
+            dekWrapped: record.dekWrapped,
+            contentHashCt: record.contentHashCt,
+        };
+    }
+    return null;
+}

package/dist/upload-price.js

@@ -0,0 +1,117 @@
+// What an upload will cost, and how the file is cut up to pay for it.
+//
+// ⛔ SPLIT OUT OF `put.ts` SO THAT FILE STAYS READABLE IN ONE SITTING. Everything here is
+//    arithmetic over numbers — no network, no crypto, no disk beyond one `stat` — which is what
+//    lets `--dry-run` answer without reading a very large file, and what lets a test drive the
+//    price without an account.
+import { statSync } from "node:fs";
+import { NmtsError } from "./errors.js";
+import { DEFAULT_PART_BYTES, NCF3_SHAPE, sealedLenFor } from "./seal.js";
+import { paddedPlaintextLen } from "./shared/lib/crypto/size-padding.js";
+import { planParts } from "./shared/lib/upload/part-plan.js";
+/**
+ * Storage term, in storage-network epochs.
+ *
+ * ⛔ NOT CHOOSABLE HERE, and that is a property of credits rather than of this tool: one credit is
+ *    defined as one mebibyte for exactly this term. Offering a duration picker would be offering
+ *    to spend a multiple of a unit the credit surface does not speak in. A browser's own-wallet
+ *    upload has a picker because it is paying in WAL, which does divide.
+ */
+export const UPLOAD_EPOCHS = 2;
+/** Bytes one credit stores for `UPLOAD_EPOCHS`. */
+export const CREDIT_BYTES = 1024 * 1024;
+/**
+ * What this upload will cost, in credits — the same arithmetic the server does.
+ *
+ * Printed BEFORE anything is spent so the number can be compared with what the account screen
+ * shows afterwards. The server is still the authority; this is a quote, not a promise.
+ */
+export function creditsFor(sealedBytes, epochs = UPLOAD_EPOCHS) {
+    const mib = Math.ceil(sealedBytes / CREDIT_BYTES);
+    return Math.ceil((mib * epochs) / UPLOAD_EPOCHS);
+}
+/**
+ * How big the local file is, with the two refusals that are worth their own words.
+ *
+ * ⛔ IT IS NOT READ HERE. A file large enough to need several parts is a file too large to hold,
+ *    and the size is all that is needed to plan the upload and quote its price. The bytes are read
+ *    later, a slice at a time, by the part that is being sealed.
+ */
+export function measureLocal(path) {
+    let stat;
+    try {
+        stat = statSync(path);
+    }
+    catch {
+        throw new NmtsError(`There is no file at ${path}.`, { exitCode: 4 });
+    }
+    if (stat.isDirectory()) {
+        throw new NmtsError(`${path} is a folder.`, {
+            exitCode: 4,
+            nextStep: "This version uploads one file at a time.",
+        });
+    }
+    if (stat.size === 0) {
+        throw new NmtsError(`${path} is empty.`, {
+            exitCode: 4,
+            nextStep: "The storage network has nothing to store and would refuse the reservation.",
+        });
+    }
+    return stat.size;
+}
+/**
+ * How much of the file goes into one part.
+ *
+ * ⛔ THE SAME NUMBER ON A RESUME OR NOTHING MATCHES. The parts already written down were sealed
+ *    and paid for at one size; a second run that split the file differently would be asking to
+ *    push different bytes under reservations that bought the first ones. `buyAndPushPart` refuses
+ *    that rather than doing it, and this is where the number comes from.
+ */
+export function partSizeFor(chosen) {
+    if (chosen === undefined)
+        return DEFAULT_PART_BYTES;
+    const bytes = typeof chosen === "number" ? chosen : parseSize(chosen);
+    if (!Number.isSafeInteger(bytes) || bytes <= 0) {
+        throw new NmtsError(`A part size must be a positive whole number of bytes: ${String(chosen)}.`, {
+            exitCode: 2,
+            nextStep: "Nothing was sent. Give a number of bytes, or a number with KiB, MiB or GiB.",
+        });
+    }
+    return bytes;
+}
+/**
+ * A byte count, plainly or with a unit.
+ *
+ * ⛔ EXACT NUMBERS ARE ALWAYS ACCEPTED. The units are a convenience on top, never instead: a person
+ *    who knows exactly what their machine can hold has to be able to say exactly that.
+ */
+function parseSize(text) {
+    const match = /^(\d+)\s*(B|KiB|MiB|GiB|K|M|G)?$/i.exec(text.trim());
+    if (match === null || match[1] === undefined)
+        return Number.NaN;
+    const scale = { b: 1, k: 2 ** 10, kib: 2 ** 10, m: 2 ** 20, mib: 2 ** 20, g: 2 ** 30, gib: 2 ** 30 };
+    const unit = (match[2] ?? "B").toLowerCase();
+    const factor = scale[unit];
+    if (factor === undefined)
+        return Number.NaN;
+    return Number(match[1]) * factor;
+}
+/**
+ * The plan, and what it will cost — one function so the price and the sealing cannot disagree.
+ *
+ * ⛔ ONLY THE LAST PART IS ROUNDED UP. The earlier ones are exactly the part size, which is what
+ *    lets a reader work out where the padding is. Quoting any other way would price a file
+ *    differently from how it is actually stored.
+ */
+export function planAndPrice(size, partSize, rule) {
+    const plan = planParts(size, partSize);
+    const sealFor = (range) => range.partIndex === plan.length - 1
+        ? paddedPlaintextLen(range.length, rule, { unitBytes: CREDIT_BYTES, shape: NCF3_SHAPE })
+        : range.length;
+    return {
+        plan,
+        sealFor,
+        sealedBytes: plan.reduce((sum, range) => sum + sealedLenFor(sealFor(range)), 0),
+        credits: plan.reduce((sum, range) => sum + creditsFor(sealedLenFor(sealFor(range))), 0),
+    };
+}

package/dist/upload-steps.js

@@ -0,0 +1,111 @@
+// The two steps after the money has moved: get the bytes onto the network, and make the file real.
+//
+// ⛔ SPLIT OUT OF `upload.ts` SO THAT FILE STAYS READABLE IN ONE SITTING. It is the file that
+//    decides what gets bought; these are the steps it drives afterwards, and both of them run in a
+//    world where the account has ALREADY PAID. Every failure message below therefore has to say so
+//    — "try again" is honest advice before the reserve and misleading after it.
+import { readItemRecord, writeItemRecord } from "./upload-store.js";
+import { UploadError, } from "./upload-wire.js";
+/** ITEM PARTS: the storage shape this tool writes — one blob per part, the file its own set. */
+const STORAGE_KIND_DEDICATED_BLOB = 0;
+/** Walrus. The only network with an upload path. */
+const NETWORK_WALRUS = 0;
+/** Only this account can read it. Sharing is a separate act, made in a browser. */
+const VISIBILITY_PERSONAL = 0;
+function why(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+/** Push ONE part's bytes at the relay and report the certificate. Commits nothing. */
+export async function pushPart(input, step) {
+    input.onStep?.({ step: "uploading", relayUrl: step.relayUrl, bytes: step.sealed.length });
+    let certificate;
+    try {
+        certificate = await input.protocol.uploadToRelay({
+            blobId: step.blobId,
+            bytes: step.sealed,
+            nonce: step.nonce,
+            registerTxDigest: step.registerTxDigest,
+            blobObjectId: step.blobObjectId,
+        });
+    }
+    catch (error) {
+        throw new UploadError({
+            phase: "uploading",
+            message: `Uploading to the storage network failed: ${why(error)}`,
+            paid: true,
+            nextStep: "The storage is already bought. Running the same command again pushes the same bytes to " +
+                "the same relay and costs nothing more.",
+        });
+    }
+    input.onStep?.({ step: "certifying" });
+    try {
+        await input.api.uploaded(step.ledgerId, certificate);
+    }
+    catch (error) {
+        throw new UploadError({
+            phase: "certify",
+            message: why(error),
+            paid: true,
+            nextStep: "The bytes are on the network. Certifying moves no money, so running the same command " +
+                "again simply finishes the job.",
+        });
+    }
+}
+/**
+ * Make the file real: one `POST /v1/items` naming EVERY part and the reservation that paid for it.
+ *
+ * ⛔ ONE CALL FOR THE WHOLE FILE. The server derives the item's size as the sum of the parts'
+ *    sealed lengths and refuses a set whose indices are not a contiguous 0..n — so a file cannot
+ *    be committed a piece at a time, and a part left out is not a smaller file, it is a rejection.
+ *
+ * ⛔ EVERY FACT HERE COMES OFF A RECORD. The parts were read back from what was written down
+ *    before each reservation; the key and the content hash come from whichever record the caller
+ *    read them from. A resumed run that used its own freshly generated key would produce a file
+ *    that is paid for, present, correctly named — and impossible to open.
+ */
+export async function commitItem(input, fileKey, parts) {
+    input.onStep?.({ step: "committing" });
+    // Advisory only — the chain is the authority on a blob's life. 0 when this machine could not
+    // read the epoch clock: a number we do not have is not a number to invent.
+    const expiryEpoch = input.currentEpoch === null ? 0 : input.currentEpoch + input.epochs;
+    const previous = readItemRecord(fileKey);
+    // ⛔ ALREADY COMMITTED IS NOT COMMITTED AGAIN. The record outlives the commit precisely so a run
+    //    that died before writing the file list does not make a second file out of storage that is
+    //    already named.
+    if (previous?.itemId !== undefined)
+        return previous.itemId;
+    const attempt = previous?.attempt ?? 0;
+    writeItemRecord(fileKey, { attempt });
+    let view;
+    try {
+        view = await input.api.createItem({
+            size: parts.reduce((sum, part) => sum + part.sealedLen, 0),
+            dek_wrapped: input.entry.dekWrapped,
+            content_hash_ct: input.entry.contentHashCt,
+            visibility: VISIBILITY_PERSONAL,
+            parts: parts.map((part) => ({
+                part_index: part.partIndex,
+                storage_kind: STORAGE_KIND_DEDICATED_BLOB,
+                network: NETWORK_WALRUS,
+                blob_id: part.blobId,
+                sealed_len: part.sealedLen,
+                expiry_epoch: expiryEpoch,
+                sponsored_ledger_id: part.ledgerId,
+            })),
+        }, `nmts-cli-commit-${fileKey}-${attempt}`);
+    }
+    catch (error) {
+        throw new UploadError({
+            phase: "committing",
+            message: `The file is stored but saving it to the drive failed: ${why(error)}`,
+            paid: true,
+            nextStep: "Nothing more will be spent. Running the same command again commits the same stored " +
+                "bytes — the retry is recognised and does not make a second file.",
+        });
+    }
+    // ⛔ Written down before returning: from here on the file EXISTS and is paid for, and the only
+    //    thing still missing is the account's own list. Losing the record now would make it
+    //    unreachable.
+    writeItemRecord(fileKey, { attempt, itemId: view.id });
+    return view.id;
+}