@bcts/hubert 1.0.0-alpha.17

package/src/error.ts

@@ -0,0 +1,89 @@
+/**
+ * Top-level error types for the hubert library.
+ *
+ * Port of error.rs from hubert-rust.
+ *
+ * @module
+ */
+
+/**
+ * Base error class for all Hubert errors.
+ *
+ * @category Errors
+ */
+export class HubertError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "HubertError";
+    // Maintains proper stack trace for where error was thrown (V8 only)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, this.constructor);
+    }
+  }
+}
+
+/**
+ * Error thrown when attempting to store at an ARID that already exists.
+ *
+ * Port of `Error::AlreadyExists { arid }` from error.rs line 6.
+ *
+ * @category Errors
+ */
+export class AlreadyExistsError extends HubertError {
+  /** The ARID that already exists */
+  readonly arid: string;
+
+  constructor(arid: string) {
+    super(`${arid} already exists`);
+    this.name = "AlreadyExistsError";
+    this.arid = arid;
+  }
+}
+
+/**
+ * Error thrown when an ARID is not found.
+ *
+ * Port of `Error::NotFound` from error.rs line 8.
+ *
+ * @category Errors
+ */
+export class NotFoundError extends HubertError {
+  constructor() {
+    super("Not found");
+    this.name = "NotFoundError";
+  }
+}
+
+/**
+ * Error thrown when an ARID format is invalid.
+ *
+ * Port of `Error::InvalidArid` from error.rs line 11.
+ *
+ * @category Errors
+ */
+export class InvalidAridError extends HubertError {
+  constructor() {
+    super("Invalid ARID format");
+    this.name = "InvalidAridError";
+  }
+}
+
+/**
+ * Error thrown for I/O operations.
+ *
+ * Port of `Error::Io(e)` from error.rs line 35.
+ *
+ * @category Errors
+ */
+export class IoError extends HubertError {
+  /** The underlying error */
+  override readonly cause?: Error;
+
+  constructor(message: string, cause?: Error) {
+    super(`IO error: ${message}`);
+    this.name = "IoError";
+    if (cause !== undefined) {
+      this.cause = cause;
+    }
+  }
+}

package/src/hybrid/error.ts

@@ -0,0 +1,77 @@
+/**
+ * Hybrid-specific errors.
+ *
+ * Port of hybrid/error.rs from hubert-rust.
+ *
+ * @module
+ */
+
+import { HubertError } from "../error.js";
+
+/**
+ * Base class for Hybrid-specific errors.
+ *
+ * @category Hybrid
+ */
+export class HybridError extends HubertError {
+  constructor(message: string) {
+    super(message);
+    this.name = "HybridError";
+  }
+}
+
+/**
+ * Referenced IPFS content not found.
+ *
+ * Port of `Error::ContentNotFound` from hybrid/error.rs line 4-5.
+ *
+ * @category Hybrid
+ */
+export class ContentNotFoundError extends HybridError {
+  constructor() {
+    super("Referenced IPFS content not found");
+    this.name = "ContentNotFoundError";
+  }
+}
+
+/**
+ * Not a reference envelope.
+ *
+ * Port of `Error::NotReferenceEnvelope` from hybrid/error.rs line 7-8.
+ *
+ * @category Hybrid
+ */
+export class NotReferenceEnvelopeError extends HybridError {
+  constructor() {
+    super("Not a reference envelope");
+    this.name = "NotReferenceEnvelopeError";
+  }
+}
+
+/**
+ * Invalid ARID in reference envelope.
+ *
+ * Port of `Error::InvalidReferenceArid` from hybrid/error.rs line 10-11.
+ *
+ * @category Hybrid
+ */
+export class InvalidReferenceAridError extends HybridError {
+  constructor() {
+    super("Invalid ARID in reference envelope");
+    this.name = "InvalidReferenceAridError";
+  }
+}
+
+/**
+ * No id assertion found in reference envelope.
+ *
+ * Port of `Error::NoIdAssertion` from hybrid/error.rs line 13-14.
+ *
+ * @category Hybrid
+ */
+export class NoIdAssertionError extends HybridError {
+  constructor() {
+    super("No id assertion found in reference envelope");
+    this.name = "NoIdAssertionError";
+  }
+}

package/src/hybrid/index.ts

@@ -0,0 +1,24 @@
+/**
+ * Hybrid module for Hubert distributed storage.
+ *
+ * This module combines Mainline DHT and IPFS for optimal storage.
+ *
+ * Port of hybrid/mod.rs from hubert-rust.
+ *
+ * @module
+ */
+
+// Error types
+export {
+  HybridError,
+  ContentNotFoundError,
+  NotReferenceEnvelopeError,
+  InvalidReferenceAridError,
+  NoIdAssertionError,
+} from "./error.js";
+
+// Reference envelope utilities
+export { createReferenceEnvelope, isReferenceEnvelope, extractReferenceArid } from "./reference.js";
+
+// Hybrid KvStore implementation
+export { HybridKv } from "./kv.js";

package/src/hybrid/kv.ts

@@ -0,0 +1,236 @@
+/**
+ * Hybrid storage layer combining Mainline DHT and IPFS.
+ *
+ * Port of hybrid/kv.rs from hubert-rust.
+ *
+ * @module
+ */
+
+import { ARID } from "@bcts/components";
+import { type Envelope } from "@bcts/envelope";
+
+import { type KvStore } from "../kv-store.js";
+import { verbosePrintln } from "../logging.js";
+import { IpfsKv } from "../ipfs/kv.js";
+import { MainlineDhtKv } from "../mainline/kv.js";
+import { ContentNotFoundError } from "./error.js";
+import { createReferenceEnvelope, extractReferenceArid, isReferenceEnvelope } from "./reference.js";
+
+/**
+ * Hybrid storage layer combining Mainline DHT and IPFS.
+ *
+ * Automatically optimizes storage based on envelope size:
+ * - **Small envelopes (≤1000 bytes)**: Stored directly in DHT
+ * - **Large envelopes (>1000 bytes)**: Reference in DHT → actual envelope in IPFS
+ *
+ * This provides the best of both worlds:
+ * - Fast lookups for small messages via DHT
+ * - Large capacity for big messages via IPFS
+ * - Transparent indirection handled automatically
+ *
+ * Port of `struct HybridKv` from hybrid/kv.rs lines 59-63.
+ *
+ * # Requirements
+ *
+ * - No external daemon for DHT (embedded client)
+ * - Requires Kubo daemon for IPFS (http://127.0.0.1:5001)
+ *
+ * @category Hybrid Backend
+ *
+ * @example
+ * ```typescript
+ * const store = await HybridKv.create("http://127.0.0.1:5001");
+ *
+ * // Small envelope → DHT only
+ * const arid1 = ARID.new();
+ * const small = Envelope.new("Small message");
+ * await store.put(arid1, small);
+ *
+ * // Large envelope → DHT reference + IPFS
+ * const arid2 = ARID.new();
+ * const large = Envelope.new("x".repeat(2000));
+ * await store.put(arid2, large);
+ *
+ * // Get works the same for both
+ * const retrieved1 = await store.get(arid1);
+ * const retrieved2 = await store.get(arid2);
+ * ```
+ */
+export class HybridKv implements KvStore {
+  private readonly dht: MainlineDhtKv;
+  private ipfs: IpfsKv;
+  private dhtSizeLimit: number;
+
+  /**
+   * Private constructor - use `create()` factory method.
+   */
+  private constructor(dht: MainlineDhtKv, ipfs: IpfsKv) {
+    this.dht = dht;
+    this.ipfs = ipfs;
+    this.dhtSizeLimit = 1000; // Conservative DHT limit
+  }
+
+  /**
+   * Create a new Hybrid KV store with default settings.
+   *
+   * Port of `HybridKv::new()` from hybrid/kv.rs lines 75-84.
+   *
+   * @param ipfsRpcUrl - IPFS RPC endpoint (e.g., "http://127.0.0.1:5001")
+   */
+  static async create(ipfsRpcUrl: string): Promise<HybridKv> {
+    const dht = await MainlineDhtKv.create();
+    const ipfs = new IpfsKv(ipfsRpcUrl);
+
+    return new HybridKv(dht, ipfs);
+  }
+
+  /**
+   * Set custom DHT size limit (default: 1000 bytes).
+   *
+   * Envelopes larger than this will use IPFS indirection.
+   *
+   * Port of `HybridKv::with_dht_size_limit()` from hybrid/kv.rs lines 89-92.
+   */
+  withDhtSizeLimit(limit: number): this {
+    this.dhtSizeLimit = limit;
+    return this;
+  }
+
+  /**
+   * Set whether to pin content in IPFS (default: false).
+   *
+   * Only affects envelopes stored in IPFS (when larger than DHT limit).
+   *
+   * Port of `HybridKv::with_pin_content()` from hybrid/kv.rs lines 97-100.
+   */
+  withPinContent(pin: boolean): this {
+    this.ipfs = this.ipfs.withPinContent(pin);
+    return this;
+  }
+
+  /**
+   * Check if an envelope fits in the DHT.
+   *
+   * Port of `HybridKv::fits_in_dht()` from hybrid/kv.rs lines 103-106.
+   *
+   * @internal
+   */
+  private fitsInDht(envelope: Envelope): boolean {
+    const serialized = envelope.taggedCborData;
+    return serialized.length <= this.dhtSizeLimit;
+  }
+
+  /**
+   * Store an envelope at the given ARID.
+   *
+   * Port of `KvStore::put()` implementation from hybrid/kv.rs lines 109-168.
+   */
+  async put(
+    arid: ARID,
+    envelope: Envelope,
+    ttlSeconds?: number,
+    verbose?: boolean,
+  ): Promise<string> {
+    // Check if it fits in DHT
+    if (this.fitsInDht(envelope)) {
+      // Store directly in DHT (DHT handles obfuscation)
+      if (verbose) {
+        verbosePrintln(`Storing envelope in DHT (size ≤ ${this.dhtSizeLimit} bytes)`);
+      }
+      await this.dht.put(arid, envelope, ttlSeconds, verbose);
+      return `Stored in DHT at ARID: ${arid.urString()}`;
+    } else {
+      // Use IPFS with DHT reference
+      if (verbose) {
+        verbosePrintln("Envelope too large for DHT, using IPFS indirection");
+      }
+
+      // 1. Store actual envelope in IPFS with a new ARID
+      //    (IPFS handles obfuscation with reference_arid)
+      const referenceArid = ARID.new();
+      if (verbose) {
+        verbosePrintln(
+          `Storing actual envelope in IPFS with reference ARID: ${referenceArid.urString()}`,
+        );
+      }
+      await this.ipfs.put(referenceArid, envelope, ttlSeconds, verbose);
+
+      // 2. Create reference envelope
+      const envelopeSize = envelope.taggedCborData.length;
+      const reference = createReferenceEnvelope(referenceArid, envelopeSize);
+
+      // 3. Store reference envelope in DHT at original ARID
+      //    (DHT handles obfuscation with original arid)
+      if (verbose) {
+        verbosePrintln("Storing reference envelope in DHT at original ARID");
+      }
+      await this.dht.put(arid, reference, ttlSeconds, verbose);
+
+      return `Stored in IPFS (ref: ${referenceArid.urString()}) via DHT at ARID: ${arid.urString()}`;
+    }
+  }
+
+  /**
+   * Retrieve an envelope for the given ARID.
+   *
+   * Port of `KvStore::get()` implementation from hybrid/kv.rs lines 171-230.
+   */
+  async get(arid: ARID, timeoutSeconds?: number, verbose?: boolean): Promise<Envelope | null> {
+    // 1. Try to get from DHT (DHT handles deobfuscation)
+    const dhtEnvelope = await this.dht.get(arid, timeoutSeconds, verbose);
+
+    if (dhtEnvelope === null) {
+      return null;
+    }
+
+    // 2. Check if the envelope is a reference envelope
+    if (isReferenceEnvelope(dhtEnvelope)) {
+      if (verbose) {
+        verbosePrintln("Found reference envelope, fetching actual envelope from IPFS");
+      }
+
+      // 3. Extract reference ARID
+      const referenceArid = extractReferenceArid(dhtEnvelope);
+
+      if (verbose) {
+        verbosePrintln(`Reference ARID: ${referenceArid.urString()}`);
+      }
+
+      // 4. Retrieve actual envelope from IPFS
+      //    (IPFS handles deobfuscation with reference_arid)
+      const ipfsEnvelope = await this.ipfs.get(referenceArid, timeoutSeconds, verbose);
+
+      if (ipfsEnvelope === null) {
+        throw new ContentNotFoundError();
+      }
+
+      if (verbose) {
+        verbosePrintln("Successfully retrieved actual envelope from IPFS");
+      }
+      return ipfsEnvelope;
+    } else {
+      // Not a reference envelope, return it directly
+      if (verbose) {
+        verbosePrintln("Envelope is not a reference, treating as direct payload");
+      }
+      return dhtEnvelope;
+    }
+  }
+
+  /**
+   * Check if an envelope exists at the given ARID.
+   *
+   * Port of `KvStore::exists()` implementation from hybrid/kv.rs lines 254-257.
+   */
+  async exists(arid: ARID): Promise<boolean> {
+    // Check DHT only (references count as existing)
+    return await this.dht.exists(arid);
+  }
+
+  /**
+   * Destroy the hybrid store and release resources.
+   */
+  async destroy(): Promise<void> {
+    await this.dht.destroy();
+  }
+}

package/src/hybrid/reference.ts

@@ -0,0 +1,176 @@
+/**
+ * Reference envelope utilities for hybrid storage.
+ *
+ * Port of hybrid/reference.rs from hubert-rust.
+ *
+ * @module
+ */
+
+import { ARID } from "@bcts/components";
+import { Envelope } from "@bcts/envelope";
+import { KnownValue } from "@bcts/known-values";
+
+import {
+  InvalidReferenceAridError,
+  NoIdAssertionError,
+  NotReferenceEnvelopeError,
+} from "./error.js";
+
+/**
+ * Known value for dereferenceVia predicate.
+ * @internal
+ */
+const DEREFERENCE_VIA = new KnownValue(8);
+
+/**
+ * Known value for id predicate.
+ * @internal
+ */
+const ID = new KnownValue(2);
+
+/**
+ * Creates a reference envelope that points to content stored in IPFS.
+ *
+ * Reference envelopes are small envelopes stored in the DHT that contain
+ * a pointer to the actual envelope stored in IPFS. This allows the hybrid
+ * storage layer to transparently handle large envelopes that exceed the
+ * DHT size limit.
+ *
+ * Port of `create_reference_envelope()` from hybrid/reference.rs lines 31-39.
+ *
+ * # Format
+ *
+ * ```text
+ * '' [
+ *     'dereferenceVia': "ipfs",
+ *     'id': <ARID>,
+ *     "size": <number>
+ * ]
+ * ```
+ *
+ * @param referenceArid - The ARID used to look up the actual envelope in IPFS
+ * @param actualSize - Size of the actual envelope in bytes (for diagnostics)
+ * @returns A reference envelope that can be stored in the DHT
+ *
+ * @category Hybrid
+ */
+export function createReferenceEnvelope(referenceArid: ARID, actualSize: number): Envelope {
+  return Envelope.unit()
+    .addAssertion(DEREFERENCE_VIA, "ipfs")
+    .addAssertion(ID, referenceArid)
+    .addAssertion("size", actualSize);
+}
+
+/**
+ * Checks if an envelope is a reference envelope.
+ *
+ * A reference envelope contains `dereferenceVia: "ipfs"` and an `id` assertion.
+ *
+ * Port of `is_reference_envelope()` from hybrid/reference.rs lines 53-91.
+ *
+ * @param envelope - The envelope to check
+ * @returns `true` if this is a reference envelope, `false` otherwise
+ *
+ * @category Hybrid
+ */
+export function isReferenceEnvelope(envelope: Envelope): boolean {
+  // Check if subject is the unit value
+  if (!envelope.isSubjectUnit) {
+    return false;
+  }
+
+  const assertions = envelope.assertions();
+
+  // Check for dereferenceVia: "ipfs" assertion
+  let hasDereferenceVia = false;
+  let hasId = false;
+
+  for (const assertion of assertions) {
+    try {
+      const predicate = assertion.predicate();
+
+      // Check if predicate is a known value
+      const predicateSubject = predicate.subject();
+      if (predicateSubject instanceof KnownValue) {
+        const kv = predicateSubject;
+
+        // Check for dereferenceVia
+        if (kv.value === DEREFERENCE_VIA.value) {
+          const object = assertion.object();
+          const objectSubject = object.subject();
+          if (typeof objectSubject === "string" && objectSubject === "ipfs") {
+            hasDereferenceVia = true;
+          }
+        }
+
+        // Check for id
+        if (kv.value === ID.value) {
+          hasId = true;
+        }
+      }
+    } catch {
+      // Skip assertions that can't be parsed
+      continue;
+    }
+  }
+
+  return hasDereferenceVia && hasId;
+}
+
+/**
+ * Extracts the reference ARID from a reference envelope.
+ *
+ * Port of `extract_reference_arid()` from hybrid/reference.rs lines 104-129.
+ *
+ * @param envelope - The reference envelope
+ * @returns The reference ARID
+ * @throws {NotReferenceEnvelopeError} If the envelope is not a reference envelope
+ * @throws {InvalidReferenceAridError} If the ARID cannot be extracted
+ * @throws {NoIdAssertionError} If no id assertion is found
+ *
+ * @category Hybrid
+ */
+export function extractReferenceArid(envelope: Envelope): ARID {
+  if (!isReferenceEnvelope(envelope)) {
+    throw new NotReferenceEnvelopeError();
+  }
+
+  const assertions = envelope.assertions();
+
+  // Find the id assertion and extract the ARID
+  for (const assertion of assertions) {
+    try {
+      const predicate = assertion.predicate();
+      const predicateSubject = predicate.subject();
+
+      if (predicateSubject instanceof KnownValue) {
+        const kv = predicateSubject;
+
+        // Check for id
+        if (kv.value === ID.value) {
+          const object = assertion.object();
+          const objectSubject = object.subject();
+
+          if (objectSubject instanceof ARID) {
+            return objectSubject;
+          }
+
+          // Try to extract ARID from CBOR
+          throw new InvalidReferenceAridError();
+        }
+      }
+    } catch (error) {
+      if (
+        error instanceof NotReferenceEnvelopeError ||
+        error instanceof InvalidReferenceAridError ||
+        error instanceof NoIdAssertionError
+      ) {
+        throw error;
+      }
+      // Continue searching
+      continue;
+    }
+  }
+
+  throw new NoIdAssertionError();
+}