@vdoninja/ninja-p2p 0.1.2 → 0.2.0

package/dist/swarm-wire.d.ts

@@ -0,0 +1,48 @@
+/**
+ * Swarm binary wire format
+ *
+ * Chunks used to travel as base64 inside a JSON envelope, because the SDK
+ * stringified everything before it reached the data channel. `sendBinary` gives
+ * us a real byte lane, so a chunk now goes out as a 40-byte header followed by
+ * the payload untouched — 0.06% overhead on a 64 KB chunk instead of 33%.
+ *
+ * The lane carries no routing of its own: `binaryReceived` tells us which peer
+ * sent bytes and nothing else. So the frame has to identify the file and the
+ * chunk itself, and has to be self-describing enough that a future frame type
+ * can share the lane without ambiguity.
+ *
+ *   0       magic 'N'
+ *   1       magic 'J'
+ *   2       version
+ *   3       frame type
+ *   4..35   fileId, 32 raw bytes (the sha256 the hex id encodes)
+ *   36..39  chunk index, uint32 big-endian
+ *   40..    payload
+ */
+export declare const SWARM_WIRE_MAGIC_0 = 78;
+export declare const SWARM_WIRE_MAGIC_1 = 74;
+export declare const SWARM_WIRE_VERSION = 1;
+export declare const SWARM_FRAME_CHUNK = 1;
+export declare const SWARM_WIRE_HEADER_BYTES = 40;
+export type SwarmChunkFrame = {
+    fileId: string;
+    index: number;
+    data: Uint8Array;
+};
+/**
+ * Whether a content id can ride the binary lane.
+ *
+ * Ids are sha256 hex, so this is true for anything we produce. A peer that
+ * offers something else is not lied to or dropped — it just gets the base64
+ * path, which imposes no constraints on the id at all.
+ */
+export declare function isBinaryFileId(fileId: string): boolean;
+export declare function encodeChunkFrame(fileId: string, index: number, data: Uint8Array): Uint8Array;
+/**
+ * Parse a frame, or return null if it is not one of ours.
+ *
+ * Null rather than throwing: the binary lane is shared with anything else the
+ * application chooses to put on it, and a frame we do not recognise is somebody
+ * else's business, not an error.
+ */
+export declare function decodeChunkFrame(bytes: Uint8Array): SwarmChunkFrame | null;

package/dist/swarm-wire.js

@@ -0,0 +1,86 @@
+/**
+ * Swarm binary wire format
+ *
+ * Chunks used to travel as base64 inside a JSON envelope, because the SDK
+ * stringified everything before it reached the data channel. `sendBinary` gives
+ * us a real byte lane, so a chunk now goes out as a 40-byte header followed by
+ * the payload untouched — 0.06% overhead on a 64 KB chunk instead of 33%.
+ *
+ * The lane carries no routing of its own: `binaryReceived` tells us which peer
+ * sent bytes and nothing else. So the frame has to identify the file and the
+ * chunk itself, and has to be self-describing enough that a future frame type
+ * can share the lane without ambiguity.
+ *
+ *   0       magic 'N'
+ *   1       magic 'J'
+ *   2       version
+ *   3       frame type
+ *   4..35   fileId, 32 raw bytes (the sha256 the hex id encodes)
+ *   36..39  chunk index, uint32 big-endian
+ *   40..    payload
+ */
+export const SWARM_WIRE_MAGIC_0 = 0x4e; // 'N'
+export const SWARM_WIRE_MAGIC_1 = 0x4a; // 'J'
+export const SWARM_WIRE_VERSION = 1;
+export const SWARM_FRAME_CHUNK = 1;
+export const SWARM_WIRE_HEADER_BYTES = 40;
+const FILE_ID_BYTES = 32;
+const HEX_FILE_ID = /^[0-9a-f]{64}$/;
+/**
+ * Whether a content id can ride the binary lane.
+ *
+ * Ids are sha256 hex, so this is true for anything we produce. A peer that
+ * offers something else is not lied to or dropped — it just gets the base64
+ * path, which imposes no constraints on the id at all.
+ */
+export function isBinaryFileId(fileId) {
+    return HEX_FILE_ID.test(fileId);
+}
+export function encodeChunkFrame(fileId, index, data) {
+    if (!isBinaryFileId(fileId)) {
+        throw new Error(`fileId is not a 64-character hex id: ${fileId}`);
+    }
+    if (!Number.isInteger(index) || index < 0 || index > 0xffffffff) {
+        throw new Error(`chunk index out of range: ${index}`);
+    }
+    const frame = new Uint8Array(SWARM_WIRE_HEADER_BYTES + data.length);
+    frame[0] = SWARM_WIRE_MAGIC_0;
+    frame[1] = SWARM_WIRE_MAGIC_1;
+    frame[2] = SWARM_WIRE_VERSION;
+    frame[3] = SWARM_FRAME_CHUNK;
+    for (let i = 0; i < FILE_ID_BYTES; i += 1) {
+        frame[4 + i] = Number.parseInt(fileId.slice(i * 2, i * 2 + 2), 16);
+    }
+    const view = new DataView(frame.buffer, frame.byteOffset);
+    view.setUint32(36, index, false);
+    frame.set(data, SWARM_WIRE_HEADER_BYTES);
+    return frame;
+}
+/**
+ * Parse a frame, or return null if it is not one of ours.
+ *
+ * Null rather than throwing: the binary lane is shared with anything else the
+ * application chooses to put on it, and a frame we do not recognise is somebody
+ * else's business, not an error.
+ */
+export function decodeChunkFrame(bytes) {
+    if (bytes.length < SWARM_WIRE_HEADER_BYTES)
+        return null;
+    if (bytes[0] !== SWARM_WIRE_MAGIC_0 || bytes[1] !== SWARM_WIRE_MAGIC_1)
+        return null;
+    if (bytes[2] !== SWARM_WIRE_VERSION)
+        return null;
+    if (bytes[3] !== SWARM_FRAME_CHUNK)
+        return null;
+    let fileId = "";
+    for (let i = 0; i < FILE_ID_BYTES; i += 1) {
+        fileId += bytes[4 + i].toString(16).padStart(2, "0");
+    }
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const index = view.getUint32(36, false);
+    // Copy rather than subarray: the payload outlives the frame — it is hashed,
+    // written to disk, and may be re-served — and holding a view would pin the
+    // whole received buffer alive for as long as any chunk of it is referenced.
+    const data = bytes.slice(SWARM_WIRE_HEADER_BYTES);
+    return { fileId, index, data };
+}

package/dist/swarm.d.ts

@@ -0,0 +1,258 @@
+/**
+ * Swarm Transfer
+ *
+ * The original file transfer was one sender pushing base64 chunks in strict
+ * order to one receiver. That caps throughput at a single peer's upload, breaks
+ * if a chunk arrives out of order, and cannot resume from anyone else.
+ *
+ * This is the torrent-shaped replacement. Files are content-addressed by their
+ * sha256, so every peer holding the same bytes is interchangeable. Peers publish
+ * a bitfield of the chunks they hold, chunks are pulled rather than pushed, and
+ * a peer can start serving a chunk the moment it has verified it — while still
+ * downloading the rest.
+ *
+ * The SDK gives one data channel per peer, so parallelism comes from two places:
+ * requesting from several peers at once, and keeping several requests in flight
+ * on each channel rather than waiting for each to land.
+ *
+ * This module is deliberately pure and I/O-light so the hard parts — bitfields,
+ * piece selection, verification — are testable without a network.
+ */
+export declare const DEFAULT_SWARM_CHUNK_SIZE = 64000;
+export declare const DEFAULT_MAX_IN_FLIGHT_PER_PEER = 4;
+export declare const DEFAULT_MAX_IN_FLIGHT_TOTAL = 24;
+/** Keep each paged manifest message comfortably below any SCTP limit. */
+export declare const SWARM_MANIFEST_PAGE_HASHES = 256;
+/** Small files stay one-message simple; large files request hash pages on demand. */
+export declare const SWARM_INLINE_MANIFEST_HASHES = 256;
+/** Bounds allocations from a peer-controlled manifest while still allowing ~64 GB at 64 KB chunks. */
+export declare const MAX_SWARM_TOTAL_CHUNKS = 1000000;
+/** A chunk must fit in memory and, in practice, one data-channel message. */
+export declare const MAX_SWARM_CHUNK_SIZE: number;
+export declare const SHA256_HEX_PATTERN: RegExp;
+/** Room left for the JSON envelope wrapped around a base64 chunk. */
+export declare const SWARM_ENVELOPE_HEADROOM = 4096;
+/** After this long with no progress, a lock is treated as abandoned. */
+export declare const SWARM_LOCK_STALE_MS: number;
+/**
+ * Where an in-progress download is written.
+ *
+ * Keyed by content **and destination**, not content alone. Two `fetch` runs of
+ * the same file into different folders are a perfectly ordinary thing to do,
+ * and sharing one `.part` between them meant they wrote over each other and the
+ * first to finish renamed the file out from under the rest — which surfaced as
+ * an ENOENT crash, having already corrupted whatever the others had written.
+ *
+ * Keeping the destination in the key also preserves resume: the same fetch, run
+ * again, lands on the same part file and picks up where it left off.
+ */
+export declare function partPathFor(workDir: string, fileId: string, savedPath: string): string;
+/**
+ * Largest chunk that still fits one SCTP message once base64-encoded.
+ *
+ * Only the fallback path needs this. Binary chunks add a 40-byte header, so a
+ * 64 KB chunk fits inside even the 65536 every WebRTC implementation must
+ * support. Base64 inflates by 4/3, so the same chunk becomes 85 KB and would be
+ * refused by a peer that negotiated only that minimum — a failure that would
+ * only ever show up against an implementation we had not tested with.
+ *
+ * Every implementation measured here negotiates 262144, which is why this has
+ * never bitten. It is a guard, not a tuning knob: chunk size was measured
+ * against throughput at 64 KB, 128 KB and 192 KB and moved it by under 2%.
+ */
+export declare function maxSafeChunkSize(messageLimit: number): number;
+/** Immutable description of a file, shared by every peer in its swarm. */
+export type SwarmManifest = {
+    /** sha256 of the whole file. Also the swarm identity. */
+    fileId: string;
+    name: string;
+    mimeType: string;
+    size: number;
+    chunkSize: number;
+    totalChunks: number;
+    /** sha256 per chunk, so a bad chunk can be blamed on the peer that sent it. */
+    chunkHashes: string[];
+    /** sha256 over the raw chunk hashes, used to authenticate paged manifests. */
+    chunkHashesHash: string;
+};
+/** Metadata that is safe to advertise before the full chunk-hash manifest arrives. */
+export type SwarmManifestSummary = Omit<SwarmManifest, "chunkHashes">;
+/**
+ * Which chunks a peer holds.
+ *
+ * A bitfield rather than a list: a 4 GB file at 64 KB chunks is ~65k chunks,
+ * which is 8 KB as bits and would be megabytes as JSON indices. Peers exchange
+ * these constantly, so the compact form matters.
+ */
+export declare class ChunkMap {
+    readonly totalChunks: number;
+    private readonly bits;
+    private present;
+    constructor(totalChunks: number, bits?: Uint8Array);
+    static full(totalChunks: number): ChunkMap;
+    static fromBase64(value: string, totalChunks: number): ChunkMap;
+    toBase64(): string;
+    has(index: number): boolean;
+    set(index: number): boolean;
+    count(): number;
+    isComplete(): boolean;
+    missing(): number[];
+    clone(): ChunkMap;
+    private recount;
+}
+/**
+ * What we have actually observed about a peer.
+ *
+ * Every field here is measured from real traffic — no declared capacity, no
+ * trust tiers, nothing a peer can assert about itself. A peer that claims to be
+ * fast and then is not will simply score worse after a few chunks.
+ */
+export type SwarmPeerStats = {
+    /** Exponentially weighted round-trip time for chunk requests, or null until measured. */
+    rttMs: number | null;
+    /** Requests that timed out or returned a chunk failing its hash. */
+    failures: number;
+    /** Chunks delivered and verified. */
+    delivered: number;
+};
+export type SwarmPeerState = {
+    peerId: string;
+    chunks: ChunkMap;
+    /** How many requests are already outstanding with this peer. */
+    inFlight: number;
+    stats?: SwarmPeerStats;
+};
+/** RTT assumed for a peer we have not measured yet, so new peers still get tried. */
+export declare const UNKNOWN_PEER_RTT_MS = 250;
+export declare function createPeerStats(): SwarmPeerStats;
+/** Fold a new RTT sample in. Weighted toward history so one spike cannot swing it. */
+export declare function recordPeerRtt(stats: SwarmPeerStats, sampleMs: number): SwarmPeerStats;
+/**
+ * Rank a peer for serving the next chunk. Lower is better, and the units are
+ * milliseconds throughout so the number stays explainable: it is roughly
+ * "expected delay if we ask this peer".
+ */
+export declare function scorePeer(peer: SwarmPeerState): number;
+export type ChunkRequestPlan = {
+    peerId: string;
+    index: number;
+};
+export type PlanOptions = {
+    maxInFlightPerPeer?: number;
+    maxInFlightTotal?: number;
+    /**
+     * Tie-break among equally rare chunks. Injectable so tests stay deterministic;
+     * everything else about planning already is.
+     */
+    random?: () => number;
+};
+/**
+ * Decide what to ask for next.
+ *
+ * Rarest-first: chunks held by the fewest peers are fetched first, so a swarm
+ * does not lose the only copy of a chunk when the peer holding it leaves. Among
+ * peers that can serve a chunk, the best-scoring one wins — measured latency,
+ * observed failures, and current queue depth — which spreads load and routes
+ * around bad peers without any central coordination.
+ *
+ * Ties are broken randomly, and that is load-bearing rather than a detail. Ties
+ * are the normal case at the start of a download, when every chunk has exactly
+ * the same rarity. Breaking them by index meant every downloader asked for the
+ * same chunks in the same order, so downloaders never held anything the others
+ * lacked and could never serve each other — three leechers against one seeder
+ * measured at exactly one third the speed of one, with no peer-to-peer traffic
+ * at all. Random selection makes them diverge immediately.
+ */
+export declare function planChunkRequests(have: ChunkMap, peers: SwarmPeerState[], inFlightIndices: ReadonlySet<number>, options?: PlanOptions): ChunkRequestPlan[];
+export declare function chunkOffset(index: number, chunkSize: number): number;
+export declare function chunkLength(index: number, manifest: Pick<SwarmManifest, "size" | "chunkSize">): number;
+/**
+ * A partially downloaded file backed by a sparse `.part` on disk.
+ *
+ * Chunks are written at their byte offset rather than appended, which is what
+ * makes out-of-order and multi-source delivery possible at all.
+ */
+export declare class ChunkFile {
+    readonly filePath: string;
+    readonly manifest: SwarmManifest;
+    private fd;
+    private lockPath;
+    private lockToken;
+    /**
+     * @param exclusive Refuse to open if another live process holds this file.
+     * Only a download needs it — a seed session opens the user's own file
+     * read-mostly and several of those coexisting is fine.
+     */
+    constructor(filePath: string, manifest: SwarmManifest, exclusive?: boolean);
+    open(): void;
+    /**
+     * Work out which chunks this file already holds, by hashing them.
+     *
+     * Keeping the bytes across a restart is only half of resuming; without this
+     * the bitfield starts empty and every chunk on disk is fetched again. That is
+     * what the code did, so "resume after an interruption" was true of the file
+     * and false of the transfer.
+     *
+     * Every chunk is verified rather than inferred from the file's length. A part
+     * file is written at byte offsets, so a gap reads back as zeros and a
+     * truncated write leaves a partial chunk — neither is distinguishable from
+     * real data by size alone, and trusting either would silently corrupt the
+     * result. Hashing costs one pass over the file, which is only paid when a
+     * part file already exists.
+     */
+    scanVerified(): ChunkMap;
+    close(): void;
+    /** Close the data handle while retaining ownership through final verification. */
+    closeHandle(): void;
+    /** Release a download lock after its part file has been finalized or abandoned. */
+    releaseExclusiveLock(): void;
+    /**
+     * Claim the part file, or explain who has it.
+     *
+     * Two downloads of the same file to the same folder is the one case the
+     * destination-keyed path cannot separate, and it is a genuine conflict rather
+     * than something to paper over — so fail loudly instead of interleaving
+     * writes. A lock left behind by a killed process goes stale rather than
+     * blocking that destination forever.
+     */
+    private acquireLock;
+    /** Refresh the lock so a long, healthy transfer is never mistaken for stale. */
+    touchLock(): void;
+    /**
+     * Verify and store one chunk. Returns false when the bytes do not match the
+     * manifest hash, which means the peer that sent them is wrong or hostile —
+     * the chunk is dropped and can be re-fetched from someone else.
+     */
+    writeChunk(index: number, bytes: Uint8Array): boolean;
+    readChunk(index: number): Uint8Array | null;
+}
+/** Build a manifest by hashing a file's bytes chunk by chunk. */
+export declare function buildManifest(bytes: Uint8Array, name: string, mimeType: string, chunkSize?: number): SwarmManifest;
+/**
+ * Build a manifest without loading the file into memory.
+ *
+ * The old path copied the whole file into a Buffer and then a Uint8Array before
+ * hashing it, and completion did the same again. That made the "large file"
+ * command consume roughly twice the file size in memory. This keeps the peak at
+ * one chunk while producing the same content and chunk digests.
+ */
+export declare function buildManifestFromFile(filePath: string, name: string, mimeType: string, chunkSize?: number): SwarmManifest;
+/** sha256 a file with bounded memory. */
+export declare function sha256FileHex(filePath: string): {
+    sha256: string;
+    size: number;
+};
+/** Stable digest for a paged chunk-hash manifest. */
+export declare function hashChunkHashes(chunkHashes: readonly string[]): string;
+export declare function toManifestSummary(manifest: SwarmManifest): SwarmManifestSummary;
+/** Return a useful reason rather than letting peer-controlled values reach I/O. */
+export declare function validateManifestSummary(manifest: SwarmManifestSummary): string | null;
+export declare function validateSwarmManifest(manifest: SwarmManifest): string | null;
+export declare function isSha256Hex(value: unknown): value is string;
+/**
+ * Confirm two peers are describing the same bytes before trading chunks.
+ * Without this a peer could advertise a familiar fileId while serving a
+ * different file's chunk hashes.
+ */
+export declare function manifestsAgree(a: SwarmManifest, b: SwarmManifest): boolean;
+export declare function sha256Hex(bytes: Uint8Array): string;