@freqhole/midden 0.1.32 → 0.2.1

package/README.md

@@ -1,82 +1,105 @@
 # midden
 
-browser WASM client for freqhole P2P federation.
-
-## what it does
-
-midden provides a `MiddenNode` class that lets browsers connect to freqhole peers over iroh P2P. just need the peer's node_id (public key) to connect - iroh handles discovery via relay.
-
-## building
-
-requires wasm-pack and LLVM toolchain:
-
-```bash
-# install wasm-pack
-cargo install wasm-pack
+the unified wasm iroh node for the freqhole world of apps: a `MiddenNode` class that lets
+browsers talk to freqhole peers over iroh p2p - api requests, verified blob streaming,
+opfs-backed persistent blob storage, chunked import sessions, radio, and raw bidirectional
+streams for app protocols. rust crate built with wasm-bindgen/wasm-pack; npm package
+`@freqhole/midden`.
+
+## architecture
+
+```mermaid
+graph LR
+    subgraph browser
+        APP[your app / worker]
+        MN[MiddenNode wasm]
+        OPFS[(opfs blob store)]
+    end
+    subgraph peers
+        P1[always-on peer<br/>grimoire / tumulus / any]
+        P2[another browser]
+    end
+    APP --> MN
+    MN --- OPFS
+    MN <-- "iroh p2p (relay discovery + direct)" --> P1 & P2
+```
 
-# on macOS, ensure LLVM is installed
-brew install llvm
+the default registered ALPN set is the app-neutral base (`freqhole/1`, automerge sync,
+friendz, admin, events, iroh-blobs). app-specific ALPNs are passed via `extra_alpns` -
+never hardcoded here.
 
-# build (dev mode)
-make build
+## structure
 
-# build (release mode, optimized)
-make build-release
+```
+src/
+  lib.rs           MiddenNode + options, bi-streams, blob transfer, import sessions
+  opfs_store/      persistent browser blob store (storage-generic core, native tests)
+  radio.rs         radio streaming
+pkg/               committed wasm-pack output - consumers file:-dep straight at it
+Makefile           build targets
 ```
 
-output goes to `pkg/` directory.
+`pkg/` is checked into git: rebuild it whenever `src/` changes or consumers won't see the
+change.
 
-## usage
+## getting started
 
 ```typescript
-import { MiddenNode } from "midden";
-
-// create node (waits for relay connection)
-const node = await MiddenNode.create();
-console.log("my node_id:", node.node_id());
-
-// make API request to peer - accepts plain node_id or full endpoint JSON
-const response = await node.proxy_request(
-  peerNodeId, // e.g. "abc123def456..." or '{"id":"...","addrs":[...]}'
-  "GET",
-  "/api/music/songs?limit=10",
-  null,
+import { MiddenNode, MiddenNodeOptions } from "@freqhole/midden";
+
+const options = new MiddenNodeOptions();
+options.set_secret_key(mySecretKeyBytes); // omit to generate fresh
+options.set_opfs_store_dir("myapp-blobs"); // persistent blob store
+options.set_extra_alpns(["myapp/1"]); // app protocol ALPNs
+const node = await MiddenNode.create_with_options(options);
+console.log("node id:", node.node_id());
+
+// request/response to a peer (json over iroh; same dispatch shape as http)
+const resp = await node.api_request(peerAddr, "GET", "/api/hello", null);
+
+// verified blob download with progress
+const bytes = await node.download_verified_with_ensure_progress(
+  peerAddr,
+  blake3Hash,
+  totalSize,
+  (fraction) => {},
+  downloadId,
 );
-console.log(response.status, response.body);
 
-// fetch blob from peer
-const blob = await node.fetch_blob(peerNodeId, blobId);
-console.log(blob.size(), blob.content_type());
-// blob.data() returns Uint8Array
+// raw bidirectional stream for your own protocol
+const stream = await node.open_bi(peerAddr, "myapp/1");
+await stream.write_raw_and_finish(encodedRequest);
+const reply = await stream.read_to_end(64 * 1024);
 ```
 
-### peer address formats
-
-midden accepts two formats for `peer_addr`:
+`peer_addr` accepts a bare 64-hex node id (relay discovery) or a full endpoint json
+(`{"id":"...","addrs":[...]}` with relay/ip hints).
 
-1. **plain node_id** (64 hex chars): uses iroh relay for discovery
+other surface: `import_blob`/`release_blob` (stage bytes for peers), `ImportSession`
+(chunked streaming import with incremental blake3), `Blake3Hasher`, `CancelToken` +
+`download_cancel` (pause/resume), `protect_blob`/`unprotect_blob` (gc pins),
+`has_complete_blob`, opfs selftests, radio.
 
-   ```
-   13a257b5367d6b5b7ceb67ec6246c3dafbe886af8ed429408cd7619c7a4787b1
-   ```
+## developer quick start
 
-2. **full endpoint JSON**: includes relay URL and/or direct IP hints
-   ```json
-   {
-     "id": "13a257b5...",
-     "addrs": [{ "Relay": "https://..." }, { "Ip": "192.168.1.100:57383" }]
-   }
-   ```
+```bash
+cargo install wasm-pack
+brew install llvm          # macos: wasm toolchain needs it
 
-## protocol
+make build                 # dev build -> pkg/
+make build-release         # optimized
+cargo test                 # native tests (opfs_store core runs without a browser)
+```
 
-uses same protocol as grimoire's federation transport:
+consumers today: tomb/spume + rathole, skein/loam, playlistz - all via `file:` deps at
+`pkg/` (e.g. `file:../../midden/pkg`). vite consumers bundling this inside a worker need
+the resolveId-plugin pattern (see @freqhole/reliquary's README).
 
-- ALPN: `freqhole/1`
-- messages: `ProxyRequest`, `ProxyResponse`, `BlobStreamRequest`, `BlobStreamResponse`
-- blob streaming: length-prefixed header followed by raw bytes
+## protocol
 
-see `grimoire/src/federation/transport/protocol.rs` for details.
+same wire protocol as grimoire's federation transport: ALPN `freqhole/1`, messages
+`ApiRequest`/`ApiResponse`, `EnsureBlobRequest`/`EnsureBlobResponse`, blob streaming as a
+length-prefixed header + raw bytes. see `grimoire/src/federation/transport/protocol.rs`.
 
 ---
 

package/midden.d.ts

@@ -6,7 +6,7 @@
  * *This API requires the following crate features to be activated: `ReadableStreamType`*
  */
 
-type ReadableStreamType = "bytes";
+export type ReadableStreamType = "bytes";
 
 /**
  * a bidirectional QUIC stream for length-delimited message exchange.
@@ -90,11 +90,57 @@ export class BiStream {
      * race the QUIC flush -- the peer's `read_to_end` then errors with
      * "connection lost" mid-payload because the in-flight frames are
      * torn down with the connection. matters most for large payloads
-     * (e.g. base64-encoded blob bodies in `proxy_response`).
+     * (e.g. base64-encoded blob bodies in `api_response`).
      */
     write_raw_and_finish(data: Uint8Array): Promise<void>;
 }
 
+/**
+ * incremental blake3 hasher for streaming uploads — feed fixed-size chunks
+ * via update() and read the final hex hash from finalize(). lets JS hash a
+ * File while streaming it (file.stream() reader loop) instead of holding
+ * the whole payload in memory for a one-shot hash_blake3().
+ */
+export class Blake3Hasher {
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * finish and return the hash as a 64-char hex string. the hasher can
+     * keep absorbing after this (blake3 finalize is non-destructive), but
+     * callers should treat the session as done.
+     */
+    finalize(): string;
+    constructor();
+    /**
+     * absorb the next chunk of data.
+     */
+    update(chunk: Uint8Array): void;
+}
+
+/**
+ * cooperative cancellation for in-flight downloads (pause/cancel from JS).
+ * the download loops select on `cancelled()` between progress events —
+ * cancellation takes effect at the next event boundary, and the partial
+ * data stays in the store, so a later download of the same hash resumes
+ * from the persisted bitfield (only missing ranges transfer).
+ */
+export class CancelToken {
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * request cancellation. idempotent.
+     */
+    cancel(): void;
+    /**
+     * return a new CancelToken sharing the same cancellation state.
+     * needed because passing a wasm class by value consumes the JS handle —
+     * callers keep the original and pass a clone into download calls.
+     */
+    clone_token(): CancelToken;
+    is_cancelled(): boolean;
+    constructor();
+}
+
 /**
  * result from fetching the server hello image from a peer
  */
@@ -106,6 +152,38 @@ export class HelloImageResult {
     readonly data: Uint8Array;
 }
 
+/**
+ * chunked import session — the streaming counterpart to import_blob.
+ *
+ * created via MiddenNode::start_import(). JS feeds fixed-size chunks with
+ * push() (backpressured: the promise resolves only once the chunk is
+ * queued), then finish() completes the import and returns the blake3 hash.
+ * the wasm boundary never sees the whole payload at once; the store's
+ * ImportByteStream machinery computes the bao tree incrementally.
+ *
+ * the finished blob is pinned in the node's active_tags (same as
+ * import_blob) until release_blob() is called.
+ */
+export class ImportSession {
+    private constructor();
+    free(): void;
+    [Symbol.dispose](): void;
+    /**
+     * abort the import. any partially-imported data is left to GC.
+     */
+    abort(): void;
+    /**
+     * signal end-of-stream, wait for the import to complete, pin the
+     * resulting blob, and return its blake3 hash as a hex string.
+     */
+    finish(): Promise<string>;
+    /**
+     * queue the next chunk. resolves once the chunk has been accepted by
+     * the import stream (bounded channel — this is the backpressure point).
+     */
+    push(chunk: Uint8Array): Promise<void>;
+}
+
 export class IntoUnderlyingByteSource {
     private constructor();
     free(): void;
@@ -138,7 +216,7 @@ export class IntoUnderlyingSource {
  * browser P2P node for freqhole federation
  *
  * supports two protocols:
- * - freqhole/1: API proxying and small blob streaming
+ * - freqhole/1: API requests and small blob streaming
  * - iroh-blobs: verified streaming for audio files
  */
 export class MiddenNode {
@@ -161,6 +239,16 @@ export class MiddenNode {
      * return the number of blobs currently held in the store via active TempTags.
      */
     active_blob_count(): number;
+    /**
+     * send an API request to a peer
+     * peer_addr can be plain node_id or full endpoint JSON with relay/IP hints
+     */
+    api_request(peer_addr: string, method: string, path: string, body?: string | null): Promise<any>;
+    /**
+     * PROTOTYPE: remove a hash's restriction, returning it to the default
+     * (served to anyone) state.
+     */
+    clear_blob_restriction(blake3_hash: string): void;
     /**
      * compute blake3 hash for a blob on demand
      *
@@ -172,8 +260,10 @@ export class MiddenNode {
      */
     compute_blake3(peer_addr: string, blob_id: string): Promise<string | undefined>;
     /**
-     * create a new node with random identity
-     * waits for relay connection before returning.
+     * create a new node with random identity, an in-memory blob store, and
+     * the default ALPN set. waits for relay connection before returning.
+     *
+     * deprecated: use `create_with_options` instead.
      *
      * `connect_timeout_ms` is an optional per-dial timeout for `open_bi`
      * (defaults to 10s when omitted/undefined).
@@ -183,6 +273,8 @@ export class MiddenNode {
      * create a node from existing secret key bytes (for persistence)
      * key_bytes must be exactly 32 bytes.
      *
+     * deprecated: use `create_with_options` instead.
+     *
      * `connect_timeout_ms` is an optional per-dial timeout for `open_bi`
      * (defaults to 10s when omitted/undefined).
      */
@@ -190,13 +282,21 @@ export class MiddenNode {
     /**
      * create a node from existing secret key with additional ALPN protocols.
      *
+     * deprecated: use `create_with_options` instead.
+     *
      * `extra_alpns` is a JS array of strings (e.g. ["iroh/automerge-repo/1"]).
-     * the node always registers "freqhole/1" plus whatever extra ALPNs are given.
+     * the node always registers the default ALPN set plus whatever extra ALPNs are given.
      *
      * `connect_timeout_ms` is an optional per-dial timeout for `open_bi`
      * (defaults to 10s when omitted/undefined).
      */
     static create_with_alpns(key_bytes: Uint8Array, extra_alpns: Array<any>, connect_timeout_ms?: number | null): Promise<MiddenNode>;
+    /**
+     * create a node from an options bag. this is the single canonical
+     * constructor — `create`/`create_from_key`/`create_with_alpns` below
+     * are deprecated wrappers kept for existing callers (spume, playlistz).
+     */
+    static create_with_options(options: MiddenNodeOptions): Promise<MiddenNode>;
     /**
      * download a blob using iroh-blobs verified streaming
      *
@@ -218,6 +318,13 @@ export class MiddenNode {
      * returns (blob_data, blake3_hash) for caching the hash for future requests.
      */
     download_verified_by_id(peer_addr: string, blob_id: string): Promise<Array<any>>;
+    /**
+     * full pipeline from blob_id with progress reporting.
+     *
+     * computes blake3 on demand, then uses verified download with progress.
+     * returns [data: Uint8Array, blake3: string].
+     */
+    download_verified_by_id_progress(peer_addr: string, blob_id: string, total_size: number, on_progress: Function): Promise<Array<any>>;
     /**
      * download a verified blob and stream chunks to JS via callback
      *
@@ -238,13 +345,14 @@ export class MiddenNode {
      *
      * returns total bytes streamed.
      */
-    download_verified_streaming(peer_addr: string, blake3_hash: string, total_size: number, on_chunk: Function, on_progress: Function): Promise<number>;
+    download_verified_streaming(peer_addr: string, blake3_hash: string, total_size: number, on_chunk: Function, on_progress: Function, cancel?: CancelToken | null): Promise<number>;
     /**
      * streaming download with auto ensure+retry. first attempts the streaming
      * download; if the verified download fails (blob not in peer's store), calls
-     * ensure_blob to load it, then retries.
+     * ensure_blob to load it, then retries. a deliberate cancellation is NOT
+     * retried — it propagates immediately with the "download cancelled" message.
      */
-    download_verified_streaming_with_ensure(peer_addr: string, blake3_hash: string, total_size: number, on_chunk: Function, on_progress: Function): Promise<number>;
+    download_verified_streaming_with_ensure(peer_addr: string, blake3_hash: string, total_size: number, on_chunk: Function, on_progress: Function, cancel?: CancelToken | null): Promise<number>;
     /**
      * download a blob using iroh-blobs with automatic ensure + retry
      *
@@ -252,6 +360,35 @@ export class MiddenNode {
      * calls ensure_blob to load it, then retries.
      */
     download_verified_with_ensure(peer_addr: string, blake3_hash: string): Promise<Uint8Array>;
+    /**
+     * download with ensure + retry and progress reporting.
+     *
+     * tries download first; if blob not in peer's FsStore, calls ensure_blob
+     * then retries. progress callback receives fraction (0.0 to 1.0).
+     * `cancel`: optional cooperative cancellation (pause) — a deliberate
+     * cancellation is NOT retried, it propagates immediately.
+     *
+     * NOTE: any failure on the first attempt triggers this same
+     * ensure-then-retry fallback, not just the "blob not in FsStore yet"
+     * case the fallback was designed for. for a large blob, the first
+     * attempt can stream a substantial fraction of the bytes (driving
+     * `on_progress` most/all of the way to 1.0) before failing late, so the
+     * caller-visible symptom is a full 0->100% progress cycle that silently
+     * restarts from 0 for a second full cycle. logging the first attempt's
+     * error and explicitly resetting progress to 0 here makes this restart
+     * visible/diagnosable instead of looking like a silent glitch.
+     */
+    download_verified_with_ensure_progress(peer_addr: string, blake3_hash: string, total_size: number, on_progress: Function, cancel?: CancelToken | null): Promise<Uint8Array>;
+    /**
+     * download a blob with progress reporting via JS callback
+     *
+     * same as download_verified but calls on_progress(fraction) where
+     * fraction is bytes_received / total_size (0.0 to 1.0).
+     * total_size should come from the caller's known size field.
+     * `cancel`: optional cooperative cancellation (pause) — see
+     * download_verified_streaming for the semantics.
+     */
+    download_verified_with_progress(peer_addr: string, blake3_hash: string, total_size: number, on_progress: Function, cancel?: CancelToken | null): Promise<Uint8Array>;
     /**
      * ensure a blob is loaded into the peer's FsStore by blake3 hash
      *
@@ -268,11 +405,18 @@ export class MiddenNode {
      */
     fetch_hello_image(peer_addr: string): Promise<HelloImageResult>;
     /**
-     * check whether a blob with the given blake3 hash is currently held in the MemStore
+     * check whether a blob with the given blake3 hash is currently held in the store
      * via an active TempTag. avoids expensive OPFS read + bao recomputation when the
      * blob is already loaded.
      */
     has_active_blob(blake3_hash: string): boolean;
+    /**
+     * check whether a COMPLETE blob with this hash exists in the blob store
+     * itself — with the persistent opfs store this is true across reloads,
+     * even when no TempTag pins it. lets serving paths skip re-imports
+     * entirely.
+     */
+    has_complete_blob(blake3_hash: string): Promise<boolean>;
     /**
      * import a blob from its pre-computed bao-encoded bytes, skipping the
      * expensive bao tree computation. `blake3_hash` is the 64-char hex hash,
@@ -288,7 +432,7 @@ export class MiddenNode {
      * import raw bytes into the iroh-blobs store, returning the blake3 hash.
      * this makes the blob available for verified download by peers.
      * the blob stays in the store as long as its TempTag is held in active_tags.
-     * call release_blob() to allow GC, or it will be evicted when the map exceeds 3 entries.
+     * call release_blob() to allow GC.
      */
     import_blob(data: Uint8Array): Promise<string>;
     /**
@@ -317,12 +461,18 @@ export class MiddenNode {
      * open a bidirectional stream to a peer on a specific ALPN.
      *
      * `peer_addr` can be a plain node_id hex string or a full endpoint
-     * address JSON (same format as proxy_request). `alpn` is the protocol
+     * address JSON (same format as api_request). `alpn` is the protocol
      * to negotiate (e.g. "iroh/automerge-repo/1").
      *
      * returns a BiStream for length-delimited message exchange.
      */
     open_bi(peer_addr: string, alpn: string): Promise<BiStream>;
+    /**
+     * pin a hash so gc won't sweep it (e.g. a paused partial download).
+     * idempotent. pair with unprotect_blob when the partial is resumed to
+     * completion or discarded.
+     */
+    protect_blob(blake3_hash: string): void;
     /**
      * dispatch a typed admin command to a peer over the freqhole-admin/1 ALPN.
      *
@@ -332,16 +482,23 @@ export class MiddenNode {
      * schema happens in the spume `AdminClient`.
      */
     proxy_admin(peer_addr: string, command: string, args: string): Promise<any>;
-    /**
-     * send an API request to a peer
-     * peer_addr can be plain node_id or full endpoint JSON with relay/IP hints
-     */
-    proxy_request(peer_addr: string, method: string, path: string, body?: string | null): Promise<any>;
     /**
      * release a blob's TempTag, allowing the store to garbage-collect it.
      * blake3_hash should be the 64-char hex string returned by import_blob.
      */
     release_blob(blake3_hash: string): void;
+    /**
+     * PROTOTYPE: restrict a blob (by blake3 hex hash) so only the given
+     * peer node ids may fetch it over the `iroh-blobs/*` ALPN. a hash with
+     * no restriction registered is served to anyone (today's default
+     * behavior, unchanged) — calling this is what opts a specific hash
+     * into gating.
+     *
+     * this is a stopgap/demo hook, not the real canvas-ACL integration: it
+     * has to be called explicitly, from JS, with an already-resolved list
+     * of allowed peer node ids for this one hash.
+     */
+    restrict_blob_to_peers(blake3_hash: string, peer_node_ids: Array<any>): void;
     /**
      * get the secret key bytes for persistence (32 bytes)
      * store this in IndexedDB to maintain the same identity across sessions
@@ -365,6 +522,12 @@ export class MiddenNode {
      * they already know the hash of. peer filtering can be added later if needed.
      */
     start_blob_server(): void;
+    /**
+     * begin a chunked import — the streaming counterpart to import_blob for
+     * payloads that shouldn't be materialized as one contiguous &[u8] across
+     * the wasm boundary. see ImportSession for the push/finish protocol.
+     */
+    start_import(): ImportSession;
     /**
      * connect to a freqhole radio broadcaster.
      *
@@ -381,6 +544,53 @@ export class MiddenNode {
      * playback and closes the iroh connection.
      */
     tune_radio(peer_addr: string, station_id: string | null | undefined, on_hello: Function, on_meta: Function, on_chunk: Function): Promise<RadioHandle>;
+    /**
+     * remove a gc pin added by protect_blob (or by a cancelled download).
+     */
+    unprotect_blob(blake3_hash: string): void;
+}
+
+/**
+ * options bag for `MiddenNode::create_with_options`, the single canonical
+ * constructor. build one, set whichever fields are needed, and pass it in:
+ *
+ * ```js
+ * const opts = new MiddenNodeOptions();
+ * opts.opfs_store_dir = "midden-blob-store";
+ * opts.connect_timeout_ms = 5000;
+ * const node = await MiddenNode.create_with_options(opts);
+ * ```
+ *
+ * `create`/`create_from_key`/`create_with_alpns` remain as deprecated
+ * wrappers over this constructor for existing callers (spume, playlistz).
+ */
+export class MiddenNodeOptions {
+    free(): void;
+    [Symbol.dispose](): void;
+    constructor();
+    /**
+     * per-dial timeout (ms) for `open_bi`/`connect` (defaults to 10s).
+     */
+    get connect_timeout_ms(): number | undefined;
+    set connect_timeout_ms(value: number | null | undefined);
+    /**
+     * additional ALPN protocols to register beyond the default set.
+     */
+    get extra_alpns(): string[] | undefined;
+    set extra_alpns(value: string[] | null | undefined);
+    /**
+     * when given, blobs persist in an OPFS-backed store under this
+     * directory (worker context required); otherwise (or when OPFS is
+     * unavailable) an in-memory store is used.
+     */
+    get opfs_store_dir(): string | undefined;
+    set opfs_store_dir(value: string | null | undefined);
+    /**
+     * the node's secret key (32 raw bytes). omit (or pass null/undefined)
+     * to generate a random identity.
+     */
+    get secret_key(): Uint8Array | undefined;
+    set secret_key(value: Uint8Array | null | undefined);
 }
 
 /**
@@ -404,4 +614,19 @@ export class RadioHandle {
  */
 export function hash_blake3(data: Uint8Array): string;
 
+/**
+ * opfs store selftest — runs the full import/export round trip against
+ * real OPFS through the real iroh-blobs api. worker context required
+ * (sync access handles). wasm-only debug helper, used for manual
+ * debugging from the blob worker, not from automated tests.
+ */
+export function opfs_store_selftest(): Promise<string>;
+
+/**
+ * persistence selftest: blobs + tags survive a store shutdown/reopen over
+ * the same OPFS directory. worker context required. wasm-only debug
+ * helper, used for manual debugging from the blob worker.
+ */
+export function opfs_store_selftest_persistence(): Promise<string>;
+
 export function start(): void;

package/midden.js

@@ -1,9 +1,9 @@
 /* @ts-self-types="./midden.d.ts" */
-
 import * as wasm from "./midden_bg.wasm";
 import { __wbg_set_wasm } from "./midden_bg.js";
+
 __wbg_set_wasm(wasm);
 wasm.__wbindgen_start();
 export {
-    BiStream, HelloImageResult, IntoUnderlyingByteSource, IntoUnderlyingSink, IntoUnderlyingSource, MiddenNode, RadioHandle, hash_blake3, start
+    BiStream, Blake3Hasher, CancelToken, HelloImageResult, ImportSession, IntoUnderlyingByteSource, IntoUnderlyingSink, IntoUnderlyingSource, MiddenNode, MiddenNodeOptions, RadioHandle, hash_blake3, opfs_store_selftest, opfs_store_selftest_persistence, start
 } from "./midden_bg.js";