@waku/core 0.0.1

package/src/lib/enr/multiaddrs_codec.ts

@@ -0,0 +1,50 @@
+import { multiaddr } from "@multiformats/multiaddr";
+import type { Multiaddr } from "@multiformats/multiaddr";
+
+import { MULTIADDR_LENGTH_SIZE } from "./constants";
+
+export function decodeMultiaddrs(bytes: Uint8Array): Multiaddr[] {
+  const multiaddrs = [];
+
+  let index = 0;
+
+  while (index < bytes.length) {
+    const sizeDataView = new DataView(
+      bytes.buffer,
+      index,
+      MULTIADDR_LENGTH_SIZE
+    );
+    const size = sizeDataView.getUint16(0);
+    index += MULTIADDR_LENGTH_SIZE;
+
+    const multiaddrBytes = bytes.slice(index, index + size);
+    index += size;
+
+    multiaddrs.push(multiaddr(multiaddrBytes));
+  }
+  return multiaddrs;
+}
+
+export function encodeMultiaddrs(multiaddrs: Multiaddr[]): Uint8Array {
+  const totalLength = multiaddrs.reduce(
+    (acc, ma) => acc + MULTIADDR_LENGTH_SIZE + ma.bytes.length,
+    0
+  );
+  const bytes = new Uint8Array(totalLength);
+  const dataView = new DataView(bytes.buffer);
+
+  let index = 0;
+  multiaddrs.forEach((multiaddr) => {
+    if (multiaddr.getPeerId())
+      throw new Error("`multiaddr` field MUST not contain peer id");
+
+    // Prepend the size of the next entry
+    dataView.setUint16(index, multiaddr.bytes.length);
+    index += MULTIADDR_LENGTH_SIZE;
+
+    bytes.set(multiaddr.bytes, index);
+    index += multiaddr.bytes.length;
+  });
+
+  return bytes;
+}

package/src/lib/enr/types.ts

@@ -0,0 +1,11 @@
+// Custom and aliased types for ENRs
+
+/**
+ * We represent NodeId as a hex string, since node equality is used very heavily
+ * and it is convenient to index data by NodeId
+ */
+export type NodeId = string;
+export type SequenceNumber = bigint;
+
+export type ENRKey = string;
+export type ENRValue = Uint8Array;

package/src/lib/enr/v4.ts

@@ -0,0 +1,22 @@
+import * as secp from "@noble/secp256k1";
+
+import { keccak256 } from "../crypto";
+import { bytesToHex } from "../utils";
+
+import { NodeId } from "./types";
+
+export async function sign(
+  privKey: Uint8Array,
+  msg: Uint8Array
+): Promise<Uint8Array> {
+  return secp.sign(keccak256(msg), privKey, {
+    der: false,
+  });
+}
+
+export function nodeId(pubKey: Uint8Array): NodeId {
+  const publicKey = secp.Point.fromHex(pubKey);
+  const uncompressedPubkey = publicKey.toRawBytes(false);
+
+  return bytesToHex(keccak256(uncompressedPubkey.slice(1)));
+}

package/src/lib/enr/waku2_codec.ts

@@ -0,0 +1,39 @@
+export interface Waku2 {
+  relay: boolean;
+  store: boolean;
+  filter: boolean;
+  lightPush: boolean;
+}
+
+export function encodeWaku2(protocols: Waku2): number {
+  let byte = 0;
+
+  if (protocols.lightPush) byte += 1;
+  byte = byte << 1;
+  if (protocols.filter) byte += 1;
+  byte = byte << 1;
+  if (protocols.store) byte += 1;
+  byte = byte << 1;
+  if (protocols.relay) byte += 1;
+
+  return byte;
+}
+
+export function decodeWaku2(byte: number): Waku2 {
+  const waku2 = {
+    relay: false,
+    store: false,
+    filter: false,
+    lightPush: false,
+  };
+
+  if (byte % 2) waku2.relay = true;
+  byte = byte >> 1;
+  if (byte % 2) waku2.store = true;
+  byte = byte >> 1;
+  if (byte % 2) waku2.filter = true;
+  byte = byte >> 1;
+  if (byte % 2) waku2.lightPush = true;
+
+  return waku2;
+}

package/src/lib/group_by.ts

@@ -0,0 +1,14 @@
+export function groupByContentTopic<T extends { contentTopic: string }>(
+  values: T[]
+): Map<string, Array<T>> {
+  const groupedDecoders = new Map();
+  values.forEach((value) => {
+    let decs = groupedDecoders.get(value.contentTopic);
+    if (!decs) {
+      groupedDecoders.set(value.contentTopic, []);
+      decs = groupedDecoders.get(value.contentTopic);
+    }
+    decs.push(value);
+  });
+  return groupedDecoders;
+}

package/src/lib/multiaddr_to_peer_info.ts

@@ -0,0 +1,17 @@
+import { PeerInfo } from "@libp2p/interface-peer-info";
+import { peerIdFromString } from "@libp2p/peer-id";
+import { Multiaddr } from "@multiformats/multiaddr";
+
+export function multiaddrsToPeerInfo(mas: Multiaddr[]): PeerInfo[] {
+  return mas
+    .map((ma) => {
+      const peerIdStr = ma.getPeerId();
+      const protocols: string[] = [];
+      return {
+        id: peerIdStr ? peerIdFromString(peerIdStr) : null,
+        multiaddrs: [ma.decapsulateCode(421)],
+        protocols,
+      };
+    })
+    .filter((peerInfo): peerInfo is PeerInfo => peerInfo.id !== null);
+}

package/src/lib/peer_discovery_dns/dns.ts

@@ -0,0 +1,223 @@
+import debug from "debug";
+
+import { ENR } from "../enr";
+
+import { DnsOverHttps } from "./dns_over_https";
+import { ENRTree } from "./enrtree";
+import {
+  fetchNodesUntilCapabilitiesFulfilled,
+  yieldNodesUntilCapabilitiesFulfilled,
+} from "./fetch_nodes";
+
+const log = debug("waku:discovery:dns");
+
+export type SearchContext = {
+  domain: string;
+  publicKey: string;
+  visits: { [key: string]: boolean };
+};
+
+export interface DnsClient {
+  resolveTXT: (domain: string) => Promise<string[]>;
+}
+
+export interface NodeCapabilityCount {
+  relay: number;
+  store: number;
+  filter: number;
+  lightPush: number;
+}
+
+export class DnsNodeDiscovery {
+  private readonly dns: DnsClient;
+  private readonly _DNSTreeCache: { [key: string]: string };
+  private readonly _errorTolerance: number = 10;
+
+  public static dnsOverHttp(dnsClient?: DnsClient): DnsNodeDiscovery {
+    if (!dnsClient) {
+      dnsClient = new DnsOverHttps();
+    }
+    return new DnsNodeDiscovery(dnsClient);
+  }
+
+  /**
+   * Returns a list of verified peers listed in an EIP-1459 DNS tree. Method may
+   * return fewer peers than requested if [[wantedNodeCapabilityCount]] requires
+   * larger quantity of peers than available or the number of errors/duplicate
+   * peers encountered by randomized search exceeds the sum of the fields of
+   * [[wantedNodeCapabilityCount]] plus the [[_errorTolerance]] factor.
+   */
+  async getPeers(
+    enrTreeUrls: string[],
+    wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
+  ): Promise<ENR[]> {
+    const networkIndex = Math.floor(Math.random() * enrTreeUrls.length);
+    const { publicKey, domain } = ENRTree.parseTree(enrTreeUrls[networkIndex]);
+    const context: SearchContext = {
+      domain,
+      publicKey,
+      visits: {},
+    };
+
+    const peers = await fetchNodesUntilCapabilitiesFulfilled(
+      wantedNodeCapabilityCount,
+      this._errorTolerance,
+      () => this._search(domain, context)
+    );
+    log(
+      "retrieved peers: ",
+      peers.map((peer) => {
+        return {
+          id: peer.peerId?.toString(),
+          multiaddrs: peer.multiaddrs?.map((ma) => ma.toString()),
+        };
+      })
+    );
+    return peers;
+  }
+
+  public constructor(dns: DnsClient) {
+    this._DNSTreeCache = {};
+    this.dns = dns;
+  }
+
+  /**
+   * {@docInherit getPeers}
+   */
+  async *getNextPeer(
+    enrTreeUrls: string[],
+    wantedNodeCapabilityCount: Partial<NodeCapabilityCount>
+  ): AsyncGenerator<ENR> {
+    const networkIndex = Math.floor(Math.random() * enrTreeUrls.length);
+    const { publicKey, domain } = ENRTree.parseTree(enrTreeUrls[networkIndex]);
+    const context: SearchContext = {
+      domain,
+      publicKey,
+      visits: {},
+    };
+
+    for await (const peer of yieldNodesUntilCapabilitiesFulfilled(
+      wantedNodeCapabilityCount,
+      this._errorTolerance,
+      () => this._search(domain, context)
+    )) {
+      yield peer;
+    }
+  }
+
+  /**
+   * Runs a recursive, randomized descent of the DNS tree to retrieve a single
+   * ENR record as an ENR. Returns null if parsing or DNS resolution fails.
+   */
+  private async _search(
+    subdomain: string,
+    context: SearchContext
+  ): Promise<ENR | null> {
+    try {
+      const entry = await this._getTXTRecord(subdomain, context);
+      context.visits[subdomain] = true;
+
+      let next: string;
+      let branches: string[];
+
+      const entryType = getEntryType(entry);
+      try {
+        switch (entryType) {
+          case ENRTree.ROOT_PREFIX:
+            next = ENRTree.parseAndVerifyRoot(entry, context.publicKey);
+            return await this._search(next, context);
+          case ENRTree.BRANCH_PREFIX:
+            branches = ENRTree.parseBranch(entry);
+            next = selectRandomPath(branches, context);
+            return await this._search(next, context);
+          case ENRTree.RECORD_PREFIX:
+            return ENR.decodeTxt(entry);
+          default:
+            return null;
+        }
+      } catch (error) {
+        log(
+          `Failed to search DNS tree ${entryType} at subdomain ${subdomain}: ${error}`
+        );
+        return null;
+      }
+    } catch (error) {
+      log(`Failed to retrieve TXT record at subdomain ${subdomain}: ${error}`);
+      return null;
+    }
+  }
+
+  /**
+   * Retrieves the TXT record stored at a location from either
+   * this DNS tree cache or via DNS query.
+   *
+   * @throws if the TXT Record contains non-UTF-8 values.
+   */
+  private async _getTXTRecord(
+    subdomain: string,
+    context: SearchContext
+  ): Promise<string> {
+    if (this._DNSTreeCache[subdomain]) {
+      return this._DNSTreeCache[subdomain];
+    }
+
+    // Location is either the top level tree entry host or a subdomain of it.
+    const location =
+      subdomain !== context.domain
+        ? `${subdomain}.${context.domain}`
+        : context.domain;
+
+    const response = await this.dns.resolveTXT(location);
+
+    if (!response.length)
+      throw new Error("Received empty result array while fetching TXT record");
+    if (!response[0].length) throw new Error("Received empty TXT record");
+
+    // Branch entries can be an array of strings of comma delimited subdomains, with
+    // some subdomain strings split across the array elements
+    const result = response.join("");
+
+    this._DNSTreeCache[subdomain] = result;
+    return result;
+  }
+}
+
+function getEntryType(entry: string): string {
+  if (entry.startsWith(ENRTree.ROOT_PREFIX)) return ENRTree.ROOT_PREFIX;
+  if (entry.startsWith(ENRTree.BRANCH_PREFIX)) return ENRTree.BRANCH_PREFIX;
+  if (entry.startsWith(ENRTree.RECORD_PREFIX)) return ENRTree.RECORD_PREFIX;
+
+  return "";
+}
+
+/**
+ * Returns a randomly selected subdomain string from the list provided by a branch
+ * entry record.
+ *
+ * The client must track subdomains which are already resolved to avoid
+ * going into an infinite loop b/c branch entries can contain
+ * circular references. It’s in the client’s best interest to traverse the
+ * tree in random order.
+ */
+function selectRandomPath(branches: string[], context: SearchContext): string {
+  // Identify domains already visited in this traversal of the DNS tree.
+  // Then filter against them to prevent cycles.
+  const circularRefs: { [key: number]: boolean } = {};
+  for (const [idx, subdomain] of branches.entries()) {
+    if (context.visits[subdomain]) {
+      circularRefs[idx] = true;
+    }
+  }
+  // If all possible paths are circular...
+  if (Object.keys(circularRefs).length === branches.length) {
+    throw new Error("Unresolvable circular path detected");
+  }
+
+  // Randomly select a viable path
+  let index;
+  do {
+    index = Math.floor(Math.random() * branches.length);
+  } while (circularRefs[index]);
+
+  return branches[index];
+}

package/src/lib/peer_discovery_dns/dns_over_https.ts

@@ -0,0 +1,98 @@
+import debug from "debug";
+import { Endpoint, query, toEndpoint } from "dns-query";
+
+import { bytesToUtf8 } from "../utils";
+
+import { DnsClient } from "./dns";
+
+const log = debug("waku:dns-over-https");
+
+export class DnsOverHttps implements DnsClient {
+  /**
+   * Default endpoints to use for DNS queries.
+   * Taken from https://github.com/martinheidegger/dns-query as static data
+   * to avoid dynamic queries.
+   *
+   * To dynamically retrieve other endpoints, use https://github.com/martinheidegger/dns-query#well-known-endpoints
+   */
+  static DefaultEndpoints: Endpoint[] = [
+    toEndpoint({
+      name: "cisco-doh",
+      protocol: "https:",
+      host: "doh.opendns.com",
+      ipv4: "146.112.41.2",
+    }),
+    toEndpoint({
+      name: "cloudflare",
+      protocol: "https:",
+      host: "dns.cloudflare.com",
+      ipv4: "1.0.0.1",
+    }),
+  ];
+
+  /**
+   * Create new Dns-Over-Http DNS client.
+   *
+   * @param endpoints The endpoints for Dns-Over-Https queries;
+   * Defaults to [[DnsOverHttps.DefaultEndpoints]].
+   * @param retries Retries if a given endpoint fails.
+   *
+   * @throws {code: string} If DNS query fails.
+   */
+  public constructor(
+    private endpoints: Endpoint[] = DnsOverHttps.DefaultEndpoints,
+    private retries: number = 3
+  ) {}
+
+  /**
+   * Resolves a TXT record
+   *
+   * @param domain The domain name
+   *
+   * @throws if the query fails
+   */
+  async resolveTXT(domain: string): Promise<string[]> {
+    let answers;
+    try {
+      const res = await query(
+        {
+          question: { type: "TXT", name: domain },
+        },
+        {
+          endpoints: this.endpoints,
+          retries: this.retries,
+        }
+      );
+      answers = res.answers;
+    } catch (error) {
+      log("query failed: ", error);
+      throw new Error("DNS query failed");
+    }
+
+    if (!answers) throw new Error(`Could not resolve ${domain}`);
+
+    const data = answers.map((a) => a.data) as
+      | Array<string | Uint8Array>
+      | Array<Array<string | Uint8Array>>;
+
+    const result: string[] = [];
+
+    data.forEach((d) => {
+      if (typeof d === "string") {
+        result.push(d);
+      } else if (Array.isArray(d)) {
+        d.forEach((sd) => {
+          if (typeof sd === "string") {
+            result.push(sd);
+          } else {
+            result.push(bytesToUtf8(sd));
+          }
+        });
+      } else {
+        result.push(bytesToUtf8(d));
+      }
+    });
+
+    return result;
+  }
+}

package/src/lib/peer_discovery_dns/enrtree.ts

@@ -0,0 +1,123 @@
+import base32 from "hi-base32";
+import { fromString } from "uint8arrays/from-string";
+
+import { keccak256, verifySignature } from "../crypto";
+import { ENR } from "../enr";
+import { utf8ToBytes } from "../utils";
+
+export type ENRRootValues = {
+  eRoot: string;
+  lRoot: string;
+  seq: number;
+  signature: string;
+};
+
+export type ENRTreeValues = {
+  publicKey: string;
+  domain: string;
+};
+
+export class ENRTree {
+  public static readonly RECORD_PREFIX = ENR.RECORD_PREFIX;
+  public static readonly TREE_PREFIX = "enrtree:";
+  public static readonly BRANCH_PREFIX = "enrtree-branch:";
+  public static readonly ROOT_PREFIX = "enrtree-root:";
+
+  /**
+   * Extracts the branch subdomain referenced by a DNS tree root string after verifying
+   * the root record signature with its base32 compressed public key.
+   */
+  static parseAndVerifyRoot(root: string, publicKey: string): string {
+    if (!root.startsWith(this.ROOT_PREFIX))
+      throw new Error(
+        `ENRTree root entry must start with '${this.ROOT_PREFIX}'`
+      );
+
+    const rootValues = ENRTree.parseRootValues(root);
+    const decodedPublicKey = base32.decode.asBytes(publicKey);
+
+    // The signature is a 65-byte secp256k1 over the keccak256 hash
+    // of the record content, excluding the `sig=` part, encoded as URL-safe base64 string
+    // (Trailing recovery bit must be trimmed to pass `ecdsaVerify` method)
+    const signedComponent = root.split(" sig")[0];
+    const signedComponentBuffer = utf8ToBytes(signedComponent);
+    const signatureBuffer = fromString(rootValues.signature, "base64url").slice(
+      0,
+      64
+    );
+
+    const isVerified = verifySignature(
+      signatureBuffer,
+      keccak256(signedComponentBuffer),
+      new Uint8Array(decodedPublicKey)
+    );
+
+    if (!isVerified) throw new Error("Unable to verify ENRTree root signature");
+
+    return rootValues.eRoot;
+  }
+
+  static parseRootValues(txt: string): ENRRootValues {
+    const matches = txt.match(
+      /^enrtree-root:v1 e=([^ ]+) l=([^ ]+) seq=(\d+) sig=([^ ]+)$/
+    );
+
+    if (!Array.isArray(matches))
+      throw new Error("Could not parse ENRTree root entry");
+
+    matches.shift(); // The first entry is the full match
+    const [eRoot, lRoot, seq, signature] = matches;
+
+    if (!eRoot)
+      throw new Error("Could not parse 'e' value from ENRTree root entry");
+    if (!lRoot)
+      throw new Error("Could not parse 'l' value from ENRTree root entry");
+
+    if (!seq)
+      throw new Error("Could not parse 'seq' value from ENRTree root entry");
+    if (!signature)
+      throw new Error("Could not parse 'sig' value from ENRTree root entry");
+
+    return { eRoot, lRoot, seq: Number(seq), signature };
+  }
+
+  /**
+   * Returns the public key and top level domain of an ENR tree entry.
+   * The domain is the starting point for traversing a set of linked DNS TXT records
+   * and the public key is used to verify the root entry record
+   */
+  static parseTree(tree: string): ENRTreeValues {
+    if (!tree.startsWith(this.TREE_PREFIX))
+      throw new Error(
+        `ENRTree tree entry must start with '${this.TREE_PREFIX}'`
+      );
+
+    const matches = tree.match(/^enrtree:\/\/([^@]+)@(.+)$/);
+
+    if (!Array.isArray(matches))
+      throw new Error("Could not parse ENRTree tree entry");
+
+    matches.shift(); // The first entry is the full match
+    const [publicKey, domain] = matches;
+
+    if (!publicKey)
+      throw new Error("Could not parse public key from ENRTree tree entry");
+    if (!domain)
+      throw new Error("Could not parse domain from ENRTree tree entry");
+
+    return { publicKey, domain };
+  }
+
+  /**
+   * Returns subdomains listed in an ENR branch entry. These in turn lead to
+   * either further branch entries or ENR records.
+   */
+  static parseBranch(branch: string): string[] {
+    if (!branch.startsWith(this.BRANCH_PREFIX))
+      throw new Error(
+        `ENRTree branch entry must start with '${this.BRANCH_PREFIX}'`
+      );
+
+    return branch.split(this.BRANCH_PREFIX)[1].split(",");
+  }
+}