@vdoninja/ninja-p2p 0.1.4 → 0.2.0

package/dist/wake.d.ts

@@ -0,0 +1,83 @@
+/**
+ * Wake Hooks
+ *
+ * Turn-based agents (Claude Code, Codex CLI) only act when something hands them
+ * a turn. The sidecar will happily hold a message forever while the agent sits
+ * idle and never notices. A wake hook closes that gap: when real messages land
+ * in the inbox, run a shell command so the agent gets a turn.
+ *
+ * Safety matters more than latency here. Two agents that both wake on message
+ * and both reply will ping-pong forever, burning tokens with nobody watching.
+ * So runs are coalesced into batches, never overlapped, and rate limited.
+ */
+import type { MessageEnvelope } from "./protocol.js";
+export declare const DEFAULT_WAKE_DEBOUNCE_MS = 750;
+export declare const DEFAULT_WAKE_LIMIT_PER_MINUTE = 30;
+/** Keep peer-controlled preview text well below Windows' process environment limit. */
+export declare const MAX_WAKE_TEXT_LENGTH = 4096;
+export declare const MAX_WAKE_CONTEXT_LENGTH = 4096;
+export type WakeConfig = {
+    command: string;
+    debounceMs: number;
+    limitPerMinute: number;
+};
+/** Minimal shape of a spawned process, so tests can inject a fake. */
+export type WakeChild = {
+    on(event: "exit", listener: () => void): unknown;
+    on(event: "error", listener: (error: Error) => void): unknown;
+};
+export type WakeSpawn = (command: string, env: NodeJS.ProcessEnv) => WakeChild;
+export type WakeContext = {
+    streamId: string;
+    room: string;
+    stateDir: string;
+};
+export type WakeRunnerOptions = {
+    config: WakeConfig;
+    context: WakeContext;
+    spawnFn?: WakeSpawn;
+    now?: () => number;
+    log?: (message: string) => void;
+};
+/**
+ * Environment handed to the wake command.
+ *
+ * NINJA_ID and NINJA_STATE_DIR are set on purpose: they let the woken command
+ * run bare `ninja-p2p read` / `ninja-p2p dm <peer> <text>` and have those route
+ * through this sidecar's local state.
+ *
+ * NINJA_ROOM is deliberately NOT set. The CLI treats a room as "one-shot mode"
+ * and would open a second WebRTC connection under the same streamId instead of
+ * queueing through the running sidecar. The room is exposed as NINJA_WAKE_ROOM
+ * for display purposes only.
+ */
+export declare function buildWakeEnv(batch: MessageEnvelope[], context: WakeContext): Record<string, string>;
+export declare class WakeRunner {
+    private readonly config;
+    private readonly context;
+    private readonly spawnFn;
+    private readonly now;
+    private readonly log;
+    private pending;
+    private timer;
+    private running;
+    private rerunRequested;
+    private recentRuns;
+    private disposed;
+    constructor(options: WakeRunnerOptions);
+    /** Record an inbox message and schedule a wake. */
+    notify(envelope: MessageEnvelope): void;
+    dispose(): void;
+    /** Test seam: whether a wake command is currently executing. */
+    isRunning(): boolean;
+    /** Test seam: how many messages are waiting for the next wake. */
+    pendingCount(): number;
+    private schedule;
+    private clearTimer;
+    private fire;
+    /** Milliseconds to wait before another run is allowed, or 0 if one is. */
+    private throttleDelayMs;
+    private run;
+    /** After a run finishes, start another if messages arrived meanwhile. */
+    private settle;
+}

package/dist/wake.js

@@ -0,0 +1,206 @@
+/**
+ * Wake Hooks
+ *
+ * Turn-based agents (Claude Code, Codex CLI) only act when something hands them
+ * a turn. The sidecar will happily hold a message forever while the agent sits
+ * idle and never notices. A wake hook closes that gap: when real messages land
+ * in the inbox, run a shell command so the agent gets a turn.
+ *
+ * Safety matters more than latency here. Two agents that both wake on message
+ * and both reply will ping-pong forever, burning tokens with nobody watching.
+ * So runs are coalesced into batches, never overlapped, and rate limited.
+ */
+import { spawn } from "node:child_process";
+export const DEFAULT_WAKE_DEBOUNCE_MS = 750;
+export const DEFAULT_WAKE_LIMIT_PER_MINUTE = 30;
+/** Keep peer-controlled preview text well below Windows' process environment limit. */
+export const MAX_WAKE_TEXT_LENGTH = 4_096;
+export const MAX_WAKE_CONTEXT_LENGTH = 4_096;
+/**
+ * Environment handed to the wake command.
+ *
+ * NINJA_ID and NINJA_STATE_DIR are set on purpose: they let the woken command
+ * run bare `ninja-p2p read` / `ninja-p2p dm <peer> <text>` and have those route
+ * through this sidecar's local state.
+ *
+ * NINJA_ROOM is deliberately NOT set. The CLI treats a room as "one-shot mode"
+ * and would open a second WebRTC connection under the same streamId instead of
+ * queueing through the running sidecar. The room is exposed as NINJA_WAKE_ROOM
+ * for display purposes only.
+ */
+export function buildWakeEnv(batch, context) {
+    return {
+        NINJA_ID: context.streamId,
+        NINJA_STATE_DIR: context.stateDir,
+        NINJA_WAKE_ROOM: context.room,
+        NINJA_WAKE_COUNT: String(batch.length),
+        NINJA_WAKE_FROM: joinBounded(unique(batch.map((envelope) => envelope.from.streamId))),
+        NINJA_WAKE_TYPES: joinBounded(unique(batch.map((envelope) => envelope.type))),
+        NINJA_WAKE_TEXT: firstText(batch),
+    };
+}
+export class WakeRunner {
+    config;
+    context;
+    spawnFn;
+    now;
+    log;
+    pending = [];
+    timer = null;
+    running = false;
+    rerunRequested = false;
+    recentRuns = [];
+    disposed = false;
+    constructor(options) {
+        this.config = options.config;
+        this.context = options.context;
+        this.spawnFn = options.spawnFn ?? defaultWakeSpawn;
+        this.now = options.now ?? (() => Date.now());
+        this.log = options.log ?? (() => { });
+    }
+    /** Record an inbox message and schedule a wake. */
+    notify(envelope) {
+        if (this.disposed)
+            return;
+        this.pending.push(envelope);
+        this.schedule(this.config.debounceMs);
+    }
+    dispose() {
+        this.disposed = true;
+        this.clearTimer();
+        this.pending = [];
+    }
+    /** Test seam: whether a wake command is currently executing. */
+    isRunning() {
+        return this.running;
+    }
+    /** Test seam: how many messages are waiting for the next wake. */
+    pendingCount() {
+        return this.pending.length;
+    }
+    schedule(delayMs) {
+        if (this.disposed || this.timer)
+            return;
+        // Fixed window rather than a resetting debounce. A resetting timer can be
+        // starved forever by a steady message stream; this bounds wake latency to
+        // roughly debounceMs no matter how chatty the room gets.
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            this.fire();
+        }, delayMs);
+        if (typeof this.timer.unref === "function")
+            this.timer.unref();
+    }
+    clearTimer() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+    fire() {
+        if (this.disposed || this.pending.length === 0)
+            return;
+        if (this.running) {
+            // Do not run two wake commands at once; an agent turn is not reentrant.
+            this.rerunRequested = true;
+            this.log(`[wake] busy, ${this.pending.length} message(s) will wait for the current run`);
+            return;
+        }
+        const waitMs = this.throttleDelayMs();
+        if (waitMs > 0) {
+            this.log(`[wake] rate limit of ${this.config.limitPerMinute}/min reached, deferring ${waitMs}ms`);
+            this.schedule(waitMs);
+            return;
+        }
+        this.run();
+    }
+    /** Milliseconds to wait before another run is allowed, or 0 if one is. */
+    throttleDelayMs() {
+        const limit = this.config.limitPerMinute;
+        if (limit <= 0)
+            return 0;
+        const now = this.now();
+        this.recentRuns = this.recentRuns.filter((at) => now - at < 60_000);
+        if (this.recentRuns.length < limit)
+            return 0;
+        return 60_000 - (now - this.recentRuns[0]) + 1;
+    }
+    run() {
+        const batch = this.pending;
+        this.pending = [];
+        this.running = true;
+        this.recentRuns.push(this.now());
+        const env = { ...process.env, ...buildWakeEnv(batch, this.context) };
+        this.log(`[wake] ${batch.length} message(s) from ${env.NINJA_WAKE_FROM || "unknown"} -> ${this.config.command}`);
+        let child;
+        try {
+            child = this.spawnFn(this.config.command, env);
+        }
+        catch (error) {
+            this.running = false;
+            this.log(`[wake] failed to start: ${error instanceof Error ? error.message : String(error)}`);
+            this.settle();
+            return;
+        }
+        let settled = false;
+        const finish = (error) => {
+            if (settled)
+                return;
+            settled = true;
+            this.running = false;
+            if (error)
+                this.log(`[wake] process error: ${error.message}`);
+            this.settle();
+        };
+        // A failed asynchronous spawn emits `error`; without a listener Node treats
+        // it as uncaught, and without settling the runner stays busy forever.
+        child.on("error", (error) => finish(error));
+        child.on("exit", () => finish());
+    }
+    /** After a run finishes, start another if messages arrived meanwhile. */
+    settle() {
+        if (this.disposed)
+            return;
+        if (this.rerunRequested || this.pending.length > 0) {
+            this.rerunRequested = false;
+            this.schedule(this.config.debounceMs);
+        }
+    }
+}
+function defaultWakeSpawn(command, env) {
+    // Inherit stdio so the wake command's output lands in the sidecar log,
+    // which is the only place a detached agent can report for itself.
+    return spawn(command, {
+        shell: true,
+        env,
+        stdio: ["ignore", "inherit", "inherit"],
+        windowsHide: true,
+    });
+}
+function unique(values) {
+    return [...new Set(values.filter(Boolean))];
+}
+function joinBounded(values) {
+    let joined = "";
+    for (const value of values) {
+        const next = joined ? `${joined},${value}` : value;
+        if (next.length > MAX_WAKE_CONTEXT_LENGTH)
+            break;
+        joined = next;
+    }
+    return joined;
+}
+function firstText(batch) {
+    for (const envelope of batch) {
+        const payload = envelope.payload;
+        if (typeof payload !== "object" || payload === null)
+            continue;
+        const text = payload.text;
+        if (typeof text === "string" && text.trim()) {
+            return text.length <= MAX_WAKE_TEXT_LENGTH
+                ? text
+                : `${text.slice(0, MAX_WAKE_TEXT_LENGTH - 1)}…`;
+        }
+    }
+    return "";
+}

package/docs/protocol-and-reliability.md

@@ -1,38 +1,231 @@
-# Protocol and reliability
-
-`ninja-p2p` is an agent coordination layer over VDO.Ninja WebRTC data channels. It does not replace or extend VDO.Ninja signaling.
-
-## Layers
-
-1. The VDO.Ninja SDK joins a room, discovers published stream IDs, and establishes WebRTC peer connections and data channels.
-2. `VDOBridge` asks the SDK to view discovered peers as data-only connections.
-3. `MessageBus` sends a versioned JSON envelope through those channels.
-4. The optional sidecar stores inbox and outbox records on the local machine so a turn-based agent can read them later.
-
-The application envelope contains an ID, timestamp, sender identity, message type, optional target/topic, and payload. It is data carried through VDO.Ninja, not a new WebSocket command.
-
-## Delivery semantics
-
-- Direct messages are sent when the SDK accepts them for an open data channel.
-- If a known peer is offline, or the channel exists but is not ready yet, the message is queued in memory.
-- A queue flush retries the original envelope, including its message ID and type, and removes only messages the SDK accepted. A failed flush leaves the remaining messages in order for the next channel-open or announce event.
-- The persistent CLI sidecar additionally stores actions on disk before its live process sends them.
-- There is no durable remote broker and no exactly-once guarantee. Applications that require confirmation should use message IDs with `ack` or `command_response` and make handlers idempotent.
-- Room broadcasts are best-effort. They are not retained independently for every absent peer.
-
-## Connection race this prevents
-
-WebRTC emits several milestones. `peerConnected` can occur before `dataChannelOpen`. Older code treated the first event as send-ready, so an agent replying in that interval could receive a normal-looking local envelope even though the SDK rejected the send. The bridge now observes the SDK's boolean send result and queues an explicitly rejected direct message.
-
-A second race occurred during reconnect: the queue was deleted before replay attempts. If the data channel closed between the open event and the first replay, every queued message was lost locally. Queue entries now remain until each replay is accepted.
-
-## Compatibility
-
-- The wire envelope format is unchanged.
-- Existing send callbacks that return `void` remain accepted; only an explicit `false` means the transport rejected a send.
-- SDK targets use the SDK's documented `{ uuid }` or `{ streamID }` forms.
-- Existing VDO.Ninja room and signaling behavior remains authoritative.
-
-## Security boundary
-
-WebRTC encrypts data in transit. Room names and optional VDO.Ninja passwords control discovery/connection behavior, but they are not an application authorization system. Validate commands at the receiving agent, expose only intentional shared folders, and do not treat peer-supplied names or capabilities as trusted identity claims.
+# Protocol and reliability
+
+`ninja-p2p` is an agent coordination layer over VDO.Ninja WebRTC data channels. It does not replace or extend VDO.Ninja signaling.
+
+## Layers
+
+1. The VDO.Ninja SDK joins a room, discovers published stream IDs, and establishes WebRTC peer connections and data channels.
+2. `VDOBridge` asks the SDK to view discovered peers as data-only connections.
+3. `MessageBus` sends a versioned JSON envelope through those channels.
+4. The optional sidecar stores inbox and outbox records on the local machine so a turn-based agent can read them later.
+
+The application envelope contains an ID, timestamp, sender identity, message type, optional target/topic, and payload. It is data carried through VDO.Ninja, not a new WebSocket command.
+
+## Delivery semantics
+
+- Direct messages are sent when the SDK accepts them for an open data channel.
+- If a known peer is offline, or the channel exists but is not ready yet, the message is queued in memory.
+- A queue flush retries the original envelope, including its message ID and type, and removes only messages the SDK accepted. A failed flush leaves the remaining messages in order for the next channel-open or announce event.
+- The persistent CLI sidecar additionally stores actions on disk before its live process sends them.
+- There is no durable remote broker and no exactly-once guarantee. Applications that require confirmation should use message IDs with `ack` or `command_response` and make handlers idempotent.
+- Room broadcasts are best-effort. They are not retained independently for every absent peer.
+- A history request is capped at 200 entries (invalid or non-positive counts
+  default to 50) and can replay broadcasts plus messages sent to or by the
+  requester. Direct messages involving other peers are filtered before replay.
+
+## Connection race this prevents
+
+WebRTC emits several milestones. `peerConnected` can occur before `dataChannelOpen`. Older code treated the first event as send-ready, so an agent replying in that interval could receive a normal-looking local envelope even though the SDK rejected the send. The bridge now observes the SDK's boolean send result and queues an explicitly rejected direct message.
+
+A second race occurred during reconnect: the queue was deleted before replay attempts. If the data channel closed between the open event and the first replay, every queued message was lost locally. Queue entries now remain until each replay is accepted.
+
+## Swarm transfer
+
+Bulk file transfer runs its own protocol on top of the same envelope, with one
+exception: chunk payloads travel as raw bytes on a dedicated binary channel
+rather than inside an envelope, when both peers can.
+
+### Two lanes, deliberately
+
+Control messages — offers, bitfields, requests, progress — stay on the shared
+control channel. Chunk bytes go on a separate reserved `x-bin` channel.
+
+That split is the single largest thing separating this from the original
+transfer path, and not for the reason that looks obvious. Base64 costs 33% in
+bandwidth, which is real but modest. The larger cost was that a 64 KB chunk on
+the control channel sat in front of every chunk request queued behind it. With
+two downloaders sharing what they had, the control channel became the
+bottleneck: measured at 627 KB/s each without the binary lane against 7.1 MB/s
+with it.
+
+Binary requires `@vdoninja/sdk` 1.4.1 or newer. Older peers are not excluded —
+each request states whether its sender can receive bytes, so the answer is
+chosen per request and a room can mix versions freely.
+
+### Why the manifest is paged
+
+The offer originally carried one 64-character hash per chunk. At 250 MB with
+the default 64 KB chunks, that produces an offer around 275 KB — already larger
+than a common 256 KB SCTP message limit. The transfer was valid but impossible
+to announce.
+
+Offers now carry the file shape plus a digest of the ordered chunk-hash list.
+Small lists remain inline. Large lists are requested in pages of at most 256
+hashes, and the assembled list is checked against its digest before a part file
+is opened. This bounds every control message and also keeps an unsolicited
+large offer from allocating a full manifest until somebody asks for the file.
+
+### Why ties are broken at random
+
+Rarest-chunk-first is the standard rule, but at the start of a download every
+chunk is equally rare. Breaking those ties by index meant every downloader
+requested the same chunks in the same order, so no downloader ever held anything
+another lacked, and the "a partial downloader is already a source" property was
+true in code and worthless in practice. Three downloaders against one seeder
+measured at exactly one third the speed of one, with no peer-to-peer traffic at
+all.
+
+Random tie-breaking makes downloaders diverge from the first request. Total
+throughput across three downloaders went from 7.8 MB/s to 14.1 MB/s.
+
+### Startup
+
+A downloader that has just learned a file exists knows nothing about who holds
+it. Waiting for the next periodic bitfield broadcast cost a fixed 2.6 s of a
+4.0 s transfer. A peer now answers another peer's bitfield with its own,
+unicast, but only when it holds something that peer lacks — which both makes the
+reply useful and guarantees the exchange terminates after one round.
+
+### What was measured and deliberately not built
+
+- **Adaptive in-flight windows.** Throughput was identical at 4, 16 and 48
+  outstanding requests per peer, so the window is not the constraint and an
+  adaptive one would be complexity without a benefit. The SDK's own drain wait
+  already bounds the send buffer, measured at 1.08 MB against a 1 MB
+  high-water mark.
+- **Larger chunks.** 64 KB, 128 KB and 192 KB were within 2% of each other. The
+  negotiated SCTP limit is used as a safety check instead: at the 64 KB default
+  a base64 chunk is 85 KB, which would be refused by a peer negotiating the
+  65536 every implementation must support.
+
+### Losing a chunk on a healthy link
+
+Chunks are occasionally dropped with nothing wrong anywhere: measured on an idle
+machine with three downloaders, one lost a chunk in roughly one run in three.
+The only thing that notices is the request timeout, so its length is the whole
+cost of a loss — a flat 15s turned a 2s transfer into 17s, which is where the
+worst outliers in the table above come from.
+
+The timeout is therefore scaled to the peer's measured round-trip time, with a
+3s floor and the 15s ceiling kept for any peer we have not timed yet. The floor
+is the important half: a peer serving several downloaders is slow rather than
+broken, and expiring it early costs twice, because the chunk is re-requested
+*and* the peer is charged a failure it did not earn.
+
+This was built, reverted, and reinstated. The revert was based on a measured
+2.5x throughput regression that turned out to be an artefact — over a hundred
+stray test daemons from earlier runs were still sitting in rooms on the same
+machine, because killing a shell job on Windows does not reach the node process
+it spawned. Re-measured on a genuinely idle machine, the regression did not
+exist. Any benchmark here is worth nothing without checking what else is
+running first.
+
+### Resume
+
+Keeping a part file's bytes across a restart is only half of resuming. The
+bitfield was rebuilt empty, so every chunk already on disk was fetched again —
+"resumable" was true of the file and false of the transfer, and no test caught it
+because none ever constructed a second session over an existing part file.
+
+A download now hashes its part file on startup and credits the chunks that
+verify. Verification is per chunk rather than inferred from the file's length:
+chunks are written at byte offsets, so a gap reads back as zeros and an
+interrupted write leaves a partial chunk, and neither is distinguishable from
+real data by size. The cost is one pass over the file, paid only when a part
+file already exists.
+
+A part file that turns out to be complete — a run interrupted between the last
+chunk and the rename — is finished immediately, since nothing else would ever
+drive it to completion.
+
+### Surviving a network drop
+
+Closing a downloader's signalling socket mid-transfer produced the most
+surprising result of any test here: chunk requests still arrived at the seeder,
+the seeder's `send()` returned success, the data channel reported `open` at both
+ends — and the bytes never landed. Measured directly: the seeder handed 331
+chunks to the transport with zero failures and a worst case of 81 ms, and the
+downloader received 327.
+
+**No layer below the application notices a path like that.** Everything reports
+health. The only evidence is a peer that stops delivering.
+
+What the transfer does about it:
+
+- Swarm messages are transient (above), so a request lost to the outage is not
+  replayed later asking for a chunk we now hold.
+- A request is only counted as outstanding if the transport accepted it.
+  Assuming otherwise meant a chunk was held hostage for a full timeout by a
+  request that never left.
+- On reconnect, everything outstanding is abandoned and re-planned rather than
+  timed out one chunk at a time, and no peer is charged a failure — the outage
+  was ours.
+- `viewedStreamIds` is cleared on reconnect. It was only ever cleared on
+  shutdown, so after a blip every peer looked "already viewed" and nothing we
+  did could re-establish one.
+- A peer that misses several requests in a row **with nothing delivered in
+  between** has its connection torn down and rebuilt. Measured with healthy
+  signalling, a rebuild restores a working path in ~260 ms.
+
+Rebuilding is deliberately held off while the SDK is itself reconnecting, plus a
+short grace afterwards. It replays its own view intent then, and rebuilding
+underneath that races it — ungated, three seeders sometimes never recovered at
+all; gated, every configuration tested recovers.
+
+**The honest limit:** a transfer always recovers, but not quickly. A 60 MB
+transfer interrupted at 30% completed in about 55 s against roughly 5 s
+uninterrupted, and more peers makes it slower rather than faster, since each one
+has its own dead path to notice. The remaining cost is in re-establishing peer
+connections after a signalling reconnect, which is below this layer.
+
+### Concurrent downloads of the same file
+
+In-progress files are keyed by content **and** destination. Keying by content
+alone meant two `fetch` runs of the same file on one machine wrote into a single
+part file and the first to finish renamed it out from under the others. A part
+file is also locked while in use, so the one genuinely ambiguous case — two
+downloads of the same file into the same folder — fails with a clear message
+instead of interleaving writes. Locks left by a killed process go stale after
+five minutes.
+
+## Simple one-to-one transfer
+
+The older one-to-one file offer, chunk, and completion messages are transient
+too. Replaying a partial run after reconnect cannot reconstruct the missing
+transport state, and retaining hundreds of chunks used to evict actual
+conversation from the history ring. The sender now fails when the transport
+rejects any step, and receivers validate transfer IDs, sender ownership, size,
+chunk count, chunk length, and final sha256 before moving a file into place.
+
+The simple path is capped at 256 MiB because the sender buffers the file. A
+sidecar accepts at most 16 incomplete transfers and 512 MiB of promised
+incomplete data. Destinations are chosen without overwriting existing files and
+the final move is exclusive. The browser uses the same wire messages but caps
+in-memory receives at 64 MiB.
+
+## Untrusted wire input
+
+The room is not a trust boundary, so parsing is deliberately stricter than the
+TypeScript types:
+
+- envelope identity, route, topic, name, and type strings are bounded before an
+  envelope is accepted
+- a data connection is bound to its observed stream identity; it cannot start
+  sending envelopes as another established peer
+- swarm summaries must have internally consistent size, chunk size, and chunk
+  count fields
+- a swarm may describe at most 1,000,000 chunks, each hash must be lowercase
+  sha256, and paged manifests are authenticated by a digest over the ordered
+  hash list before disk I/O starts
+- malformed messages are rejected locally and do not tear down the room
+
+## Compatibility
+
+- The wire envelope format is unchanged.
+- Existing send callbacks that return `void` remain accepted; only an explicit `false` means the transport rejected a send.
+- SDK targets use the SDK's documented `{ uuid }` or `{ streamID }` forms.
+- Existing VDO.Ninja room and signaling behavior remains authoritative.
+
+## Security boundary
+
+WebRTC encrypts data in transit. Room names and optional VDO.Ninja passwords control discovery/connection behavior, but they are not an application authorization system. Validate commands at the receiving agent, expose only intentional shared folders, and do not treat peer-supplied names or capabilities as trusted identity claims.