@enbox/dwn-sdk-js 0.0.1

package/src/types/records-types.ts

@@ -0,0 +1,263 @@
+import type { EncryptionAlgorithm } from '../utils/encryption.js';
+import type { GeneralJws } from './jws-types.js';
+import type { KeyDerivationScheme } from '../utils/hd-key.js';
+import type { PublicJwk } from './jose-types.js';
+import type { Readable } from 'readable-stream';
+import type { AuthorizationModel, GenericMessage, GenericMessageReply, GenericSignaturePayload, MessageSubscription, Pagination } from './message-types.js';
+import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
+import type { PaginationCursor, RangeCriterion, RangeFilter, StartsWithFilter } from './query-types.js';
+
+export enum DateSort {
+  CreatedAscending = 'createdAscending',
+  CreatedDescending = 'createdDescending',
+  PublishedAscending = 'publishedAscending',
+  PublishedDescending = 'publishedDescending'
+}
+
+export type RecordsWriteTagValue = string | number | boolean | string[] | number[];
+export type RecordsWriteTags = {
+  [property: string]: RecordsWriteTagValue;
+};
+
+export type RecordsWriteTagsFilter = StartsWithFilter | RangeFilter | string | number | boolean;
+
+export type RecordsWriteDescriptor = {
+  interface: DwnInterfaceName.Records;
+  method: DwnMethodName.Write;
+  protocol?: string;
+  protocolPath?: string;
+  recipient?: string;
+  schema?: string;
+  tags?: RecordsWriteTags;
+  parentId?: string;
+  dataCid: string;
+  dataSize: number;
+  dateCreated: string;
+  messageTimestamp: string;
+  published?: boolean;
+  datePublished?: string;
+  dataFormat: string;
+};
+
+export type RecordsWriteMessageOptions = {
+  dataStream?: Readable;
+};
+
+/**
+ * Internal RecordsWrite message representation that can be in an incomplete state.
+ */
+export type InternalRecordsWriteMessage = GenericMessage & {
+  recordId?: string,
+  contextId?: string;
+  descriptor: RecordsWriteDescriptor;
+  attestation?: GeneralJws;
+  encryption?: EncryptionProperty;
+};
+
+export type RecordsWriteMessage = {
+  authorization: AuthorizationModel; // overriding `GenericMessage` with `authorization` being required
+  recordId: string,
+  contextId?: string;
+  descriptor: RecordsWriteDescriptor;
+  attestation?: GeneralJws;
+  encryption?: EncryptionProperty;
+};
+
+export type EncryptionProperty = {
+  algorithm: EncryptionAlgorithm;
+  initializationVector: string;
+  keyEncryption: EncryptedKey[]
+};
+
+export type EncryptedKey = {
+  /**
+   * The fully qualified key ID (e.g. did:example:abc#encryption-key-id) of the root public key used to encrypt the symmetric encryption key.
+   */
+  rootKeyId: string;
+
+  /**
+   * The actual derived public key.
+   */
+  derivedPublicKey?: PublicJwk;
+  derivationScheme: KeyDerivationScheme;
+  algorithm: EncryptionAlgorithm;
+  initializationVector: string;
+  ephemeralPublicKey: PublicJwk;
+  messageAuthenticationCode: string;
+  encryptedKey: string;
+};
+
+/**
+ * Data structure returned in a `RecordsQuery` reply entry.
+ * NOTE: the message structure is a modified version of the message received, the most notable differences are:
+ * 1. May include an initial RecordsWrite message
+ * 2. May include encoded data
+ */
+export type RecordsQueryReplyEntry = RecordsWriteMessage & {
+  /**
+   * The initial write of the record if the returned RecordsWrite message itself is not the initial write.
+   */
+  initialWrite?: RecordsWriteMessage;
+
+  /**
+   * The encoded data of the record if the data associated with the record is equal or smaller than `DwnConstant.maxDataSizeAllowedToBeEncoded`.
+   */
+  encodedData?: string;
+};
+
+/**
+ * Represents a RecordsWrite message with encoded data attached.
+ */
+export type DataEncodedRecordsWriteMessage = RecordsWriteMessage & {
+  /**
+   * The encoded data of the record if the data associated with the record is equal or smaller than `DwnConstant.maxDataSizeAllowedToBeEncoded`.
+   */
+  encodedData: string;
+};
+
+export type RecordsQueryDescriptor = {
+  interface: DwnInterfaceName.Records;
+  method: DwnMethodName.Query;
+  messageTimestamp: string;
+  filter: RecordsFilter;
+  dateSort?: DateSort;
+  pagination?: Pagination;
+};
+
+export type RecordsSubscribeDescriptor = {
+  interface: DwnInterfaceName.Records;
+  method: DwnMethodName.Subscribe;
+  messageTimestamp: string;
+  filter: RecordsFilter;
+};
+
+export type RecordsFilter = {
+  /**
+   * The logical author of the record
+   */
+  author?: string | string[];
+  attester?: string;
+  recipient?: string | string[];
+  protocol?: string;
+  protocolPath?: string;
+  published?: boolean;
+
+  /**
+   * When given all Records message under the context of the given `contextId` will be returned.
+   */
+  contextId?: string;
+  schema?: string;
+  tags?: { [property:string]: RecordsWriteTagsFilter }
+  recordId?: string;
+  parentId?: string;
+  dataFormat?: string;
+  dataSize?: RangeFilter;
+  dataCid?: string;
+  dateCreated?: RangeCriterion;
+  datePublished?: RangeCriterion;
+  dateUpdated?: RangeCriterion;
+};
+
+export type RecordsWriteAttestationPayload = {
+  descriptorCid: string;
+};
+
+export type RecordsWriteSignaturePayload = GenericSignaturePayload & {
+  recordId: string;
+  contextId?: string;
+  attestationCid?: string;
+  encryptionCid?: string;
+};
+
+export type RecordsQueryMessage = GenericMessage & {
+  descriptor: RecordsQueryDescriptor;
+};
+
+export type RecordsQueryReply = GenericMessageReply & {
+  entries?: RecordsQueryReplyEntry[];
+  cursor?: PaginationCursor;
+};
+
+export type RecordEvent = {
+  message: RecordsWriteMessage | RecordsDeleteMessage
+  initialWrite?: RecordsWriteMessage;
+};
+
+export type RecordSubscriptionHandler = (event: RecordEvent) => void;
+
+export type RecordsSubscribeMessageOptions = {
+  subscriptionHandler: RecordSubscriptionHandler;
+};
+
+export type RecordsSubscribeMessage = GenericMessage & {
+  descriptor: RecordsSubscribeDescriptor;
+};
+
+export type RecordsSubscribeReply = GenericMessageReply & {
+  subscription?: MessageSubscription;
+};
+
+export type RecordsReadMessage = {
+  authorization?: AuthorizationModel;
+  descriptor: RecordsReadDescriptor;
+};
+
+/**
+ * The reply to a RecordsRead message.
+ */
+export type RecordsReadReply = GenericMessageReply & {
+  /**
+   * A container for the data returned from a `RecordsRead`.
+   * `undefined` if no data needs to be returned.
+   */
+  entry?: RecordsReadReplyEntry;
+};
+
+/**
+ * The structure of the `entry` container property in `RecordsReadReplyEntry`.
+ */
+export type RecordsReadReplyEntry = {
+  /**
+   * The latest RecordsWrite message of the record if record exists (not deleted).
+   */
+  recordsWrite?: RecordsWriteMessage;
+
+  /**
+   * The RecordsDelete if the record is deleted.
+   */
+  recordsDelete?: RecordsDeleteMessage;
+
+  /**
+   * The initial write of the record if the returned RecordsWrite message itself is not the initial write or if a RecordsDelete is returned.
+   */
+  initialWrite?: RecordsWriteMessage;
+
+  /**
+   * The data stream associated with the record if the records exists (not deleted).
+   */
+  data?: Readable;
+};
+
+export type RecordsReadDescriptor = {
+  interface: DwnInterfaceName.Records;
+  method: DwnMethodName.Read;
+  filter: RecordsFilter;
+  messageTimestamp: string;
+};
+
+export type RecordsDeleteMessage = GenericMessage & {
+  authorization: AuthorizationModel; // overriding `GenericMessage` with `authorization` being required
+  descriptor: RecordsDeleteDescriptor;
+};
+
+export type RecordsDeleteDescriptor = {
+  interface: DwnInterfaceName.Records;
+  method: DwnMethodName.Delete;
+  messageTimestamp: string;
+  recordId: string;
+
+  /**
+   * Denotes if all the descendent records should be purged.
+   */
+  prune: boolean
+};

package/src/types/resumable-task-store.ts

@@ -0,0 +1,96 @@
+/**
+ * An managed resumable task model.
+ */
+export type ManagedResumableTask = {
+  /** Globally unique ID. Used to extend or delete the task. */
+  id: string;
+  /** Task specific data. This is deliberately of type `any` because this store should not have to be ware of its type. */
+  task: any;
+  /** Task timeout in Epoch Time. */
+  timeout: number;
+  /** Number of retries */
+  retryCount: number;
+};
+
+/**
+ * Interface for interacting with the resumable task store.
+ *
+ * Implementer's Note:
+ * The store implementation used in a horizontally scalable deployment, such as in a Kubernetes cluster,
+ * must account for concurrent access by multiple `ResumableTaskStore` instances.
+ * It would be undesirable to have many kubernetes pods all trying to handle the same resumable task.
+ * A minimal viable implementation can use a per tenant exclusive lock on the store when `grab()` and  method is called.
+ * This would prevent issues that occur from concurrent modification to the same task to the store,
+ * but negatively impacts the throughput performance of the DWN.
+ * Requirements for a more performant implementation that allows distributed processing of resumable tasks across multiple clients:
+ * 1. The implementation probably requires both:
+ *   a. a persistent store for storing the data of each resumable task; and
+ *   b. an message streaming queue/service for distributing the each task exclusively to one of multiple handling clients.
+ * 2. The `grab()` and/or `open()` implementation will need to copy the timed-out tasks from persistent store into the message queue/service
+ *    for distributed processing by multiple clients when there is no resumable tasks to grab in the message queue.
+ *    During the move, the persistent store should be locked to prevent multiple copies of the same tasks from being copied.
+ * 3. Both Google's pub-sub and Amazon's SQS require an Ack ID / Receipt Handle to acknowledge the message,
+ *    The value of Ack ID / Receipt Handle is not known until the message is received,
+ *    therefore the implementer will likely need to have an in-memory mapping of task ID -> Ack ID / Receipt Handle
+ *    so that `delete()` can be called with task ID.
+ */
+export interface ResumableTaskStore {
+  /**
+   * Opens a connection to the underlying store and initializes it.
+   */
+  open(): Promise<void>;
+
+  /**
+   * Closes the connection to the underlying store.
+   */
+  close(): Promise<void>;
+
+  /**
+   * Registers a new resumable task that is currently in-flight/under processing to the store.
+   * If the task is timed out, a client will be able to grab it through the `grab()` method and resume the task.
+   * @param task Task specific data.  This is deliberately of type `any` because this store should not have to be ware of its type.
+   * @param timeoutInSeconds Timeout in seconds from the current time.
+   * @returns A `ManagedResumableTask` object that can be used to extend or delete the task.
+   * @throws {Error} with `code` set to `ResumableTaskAlreadyExists` if the same task is already registered.
+   */
+  register(task: any, timeoutInSeconds: number): Promise<ManagedResumableTask>;
+
+  /**
+   * Grabs a number of unhandled tasks from the store.
+   * Unhandled tasks are tasks that are not currently in-flight/under processing (ie. tasks that have timed-out).
+   * NOTE: The implementation must make sure that once a task is grabbed by a client,
+   * tis timeout must be updated so that it is considered in-flight/under processing
+   * and cannot be grabbed by another client until it is timed-out.
+   * @param count Desired number of tasks to grab.
+   * @returns A list of tasks exclusive for the caller to handle; or empty array if there is no tasks, or if all tasks are already grabbed by others.
+   */
+  grab(count: number): Promise<ManagedResumableTask[]>;
+
+  /**
+   * Reads the task associated with the task ID provided regardless of whether it is in-flight/under processing or not.
+   * This is mainly introduced for testing purposes: ie. to check the status of a task for easy test verification.
+   * @param taskId ID of the task to read.
+   */
+  read(taskId: string): Promise<ManagedResumableTask | undefined>;
+
+  /**
+   * Extends the timeout of the task associated with the task ID provided.
+   * No-op if the task is not found, as this implies that the task has already been completed.
+   * This allows the client that is executing the task to continue working on it before the task is considered timed out.
+   * @param taskId ID of the task to extend the timeout for.
+   * @param timeoutInSeconds Timeout in seconds from the current time.
+   */
+  extend(taskId: string, timeoutInSeconds: number): Promise<void>;
+
+  /**
+   * Deletes the task associated with the task ID provided.
+   * No-op if the task is not found, as this implies that the task has already been completed.
+   * Called when the task has been successfully completed.
+   */
+  delete(taskId: string): Promise<void>;
+
+  /**
+   * Clears the entire store. Mainly used for cleaning up in test environment.
+   */
+  clear(): Promise<void>;
+}

package/src/types/signer.ts

@@ -0,0 +1,27 @@
+/**
+ * A signer that is capable of generating a digital signature over any given bytes.
+ */
+export interface Signer {
+  /**
+   * The ID of the key used by this signer.
+   * This needs to be a fully-qualified ID (ie. prefixed with DID) so that author can be parsed out for processing such as `recordId` computation.
+   * Example: did:example:alice#key1
+   * This value will be used as the "kid" parameter in JWS produced.
+   * While this property is not a required property per JWS specification, it is required for DWN authentication.
+   */
+  keyId: string
+
+  /**
+   * The name of the signature algorithm used by this signer.
+   * This value will be used as the "alg" parameter in JWS produced.
+   * This parameter is not used by the DWN but is unfortunately a required header property for a JWS as per:
+   * https://datatracker.ietf.org/doc/html/rfc7515#section-4.1.1
+   * Valid signature algorithm values can be found at https://www.iana.org/assignments/jose/jose.xhtml
+   */
+  algorithm: string;
+
+  /**
+   * Signs the given content and returns the signature as bytes.
+   */
+  sign (content: Uint8Array): Promise<Uint8Array>;
+}

package/src/types/subscriptions.ts

@@ -0,0 +1,34 @@
+import type { GenericMessageReply } from '../types/message-types.js';
+import type { KeyValues } from './query-types.js';
+import type { RecordsWriteMessage } from './records-types.js';
+import type { GenericMessage, MessageSubscription } from './message-types.js';
+
+export type EventListener = (tenant: string, event: MessageEvent, indexes: KeyValues) => void;
+
+/**
+ * MessageEvent contains the message being emitted and an optional initial write message.
+ */
+export type MessageEvent = {
+  message: GenericMessage;
+  /** the initial write of the RecordsWrite or RecordsDelete message */
+  initialWrite?: RecordsWriteMessage
+};
+
+/**
+ * The EventStream interface implements a pub/sub system based on Message filters.
+ */
+export interface EventStream {
+  subscribe(tenant: string, id: string, listener: EventListener): Promise<EventSubscription>;
+  emit(tenant: string, event: MessageEvent, indexes: KeyValues): void;
+  open(): Promise<void>;
+  close(): Promise<void>;
+}
+
+export interface EventSubscription {
+  id: string;
+  close: () => Promise<void>;
+}
+
+export type SubscriptionReply = GenericMessageReply & {
+  subscription?: MessageSubscription;
+};

package/src/utils/abort.ts

@@ -0,0 +1,31 @@
+/**
+ * Wraps the given `AbortSignal` in a `Promise` that rejects if it is programmatically triggered,
+ * otherwise the promise will remain in await state (will never resolve).
+ */
+function promisifySignal<T>(signal: AbortSignal): Promise<T> {
+  return new Promise((resolve, reject) => {
+    // immediately reject if the given is signal is already aborted
+    if (signal.aborted) {
+      reject(signal.reason);
+      return;
+    }
+
+    signal.addEventListener('abort', () => {
+      reject(signal.reason);
+    });
+  });
+}
+
+/**
+ * Wraps the given `Promise` such that it will reject if the `AbortSignal` is triggered.
+ */
+export async function executeUnlessAborted<T>(promise: Promise<T>, signal: AbortSignal | undefined): Promise<T> {
+  if (!signal) {
+    return promise;
+  }
+
+  return Promise.race([
+    promise,
+    promisifySignal<T>(signal),
+  ]);
+}

package/src/utils/array.ts

@@ -0,0 +1,39 @@
+/**
+ * Array utility methods.
+ */
+export class ArrayUtility {
+  /**
+   * Returns `true` if content of the two given byte arrays are equal; `false` otherwise.
+   */
+  public static byteArraysEqual(array1: Uint8Array, array2:Uint8Array): boolean {
+    const equal = array1.length === array2.length && array1.every((value, index) => value === array2[index]);
+    return equal;
+  }
+
+  /**
+   * Asynchronously iterates an {AsyncGenerator} to return all the values in an array.
+   */
+  public static async fromAsyncGenerator<T>(iterator: AsyncGenerator<T>): Promise<Array<T>> {
+    const array: Array<T> = [ ];
+    for await (const value of iterator) {
+      array.push(value);
+    }
+    return array;
+  }
+
+  /**
+   * Generic asynchronous sort method.
+   */
+  public static async asyncSort<T>(array: T[], asyncComparer: (a: T, b: T) => Promise<number>): Promise<T[]> {
+    // this is a bubble sort implementation
+    for (let i = 0; i < array.length; i++) {
+      for (let j = i + 1; j < array.length; j++) {
+        const comparison = await asyncComparer(array[i], array[j]);
+        if (comparison > 0) {
+          [array[i], array[j]] = [array[j], array[i]]; // Swap
+        }
+      }
+    }
+    return array;
+  }
+}

package/src/utils/cid.ts

@@ -0,0 +1,101 @@
+import * as cbor from '@ipld/dag-cbor';
+
+import type { Readable } from 'readable-stream';
+
+import { BlockstoreMock } from '../store/blockstore-mock.js';
+import { CID } from 'multiformats/cid';
+import { importer } from 'ipfs-unixfs-importer';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { DwnError, DwnErrorCode } from '../core/dwn-error.js';
+
+// a map of all supported CID hashing algorithms. This map is used to select the appropriate hasher
+// when generating a CID to compare against a provided CID
+const hashers = {
+  [sha256.code as number]: sha256,
+};
+
+// a map of all support codecs.This map is used to select the appropriate codec
+// when generating a CID to compare against a provided CID
+const codecs = {
+  [cbor.code as number]: cbor
+};
+
+/**
+ * Utility class for creating CIDs. Exported for the convenience of developers.
+ */
+export class Cid {
+
+  /**
+   * Computes a V1 CID for the provided payload
+   * @param codecCode - the codec to use. Defaults to cbor
+   * @param multihashCode - the multihasher to use. Defaults to sha256
+   * @returns payload CID
+   * @throws {Error} codec is not supported
+   * @throws {Error} encoding fails
+   * @throws {Error} if hasher is not supported
+   */
+  public static async computeCid(
+    payload: any,
+    codecCode: number = cbor.code,
+    multihashCode: number = sha256.code
+  ): Promise<string> {
+    const codec = codecs[codecCode];
+    if (!codec) {
+      throw new DwnError(DwnErrorCode.ComputeCidCodecNotSupported, `codec [${codecCode}] not supported`);
+    }
+
+    const hasher = hashers[multihashCode];
+    if (!hasher) {
+      throw new DwnError(DwnErrorCode.ComputeCidMultihashNotSupported, `multihash code [${multihashCode}] not supported`);
+    }
+
+    const payloadBytes = codec.encode(payload);
+    const payloadHash = await hasher.digest(payloadBytes);
+
+    const cid = await CID.createV1(codec.code, payloadHash);
+    return cid.toString();
+  }
+
+  /**
+   * Parses the given CID string into a {CID}.
+   */
+  public static parseCid(str: string): CID {
+    const cid: CID = CID.parse(str).toV1();
+
+    if (!codecs[cid.code]) {
+      throw new DwnError(DwnErrorCode.ParseCidCodecNotSupported, `codec [${cid.code}] not supported`);
+    }
+
+    if (!hashers[cid.multihash.code]) {
+      throw new DwnError(DwnErrorCode.ParseCidMultihashNotSupported, `multihash code [${cid.multihash.code}] not supported`);
+    }
+
+    return cid;
+  }
+
+  /**
+   * @returns V1 CID of the DAG comprised by chunking data into unixfs DAG-PB encoded blocks
+   */
+  public static async computeDagPbCidFromBytes(content: Uint8Array): Promise<string> {
+    const asyncDataBlocks = importer([{ content }], new BlockstoreMock(), { cidVersion: 1 });
+
+    // NOTE: the last block contains the root CID
+    let block;
+    for await (block of asyncDataBlocks) { ; }
+
+    return block ? block.cid.toString() : '';
+  }
+
+  /**
+   * @returns V1 CID of the DAG comprised by chunking data into unixfs DAG-PB encoded blocks
+   */
+  public static async computeDagPbCidFromStream(dataStream: Readable): Promise<string> {
+    const asyncDataBlocks = importer([{ content: dataStream }], new BlockstoreMock(), { cidVersion: 1 });
+
+    // NOTE: the last block contains the root CID
+    let block;
+    for await (block of asyncDataBlocks) { ; }
+
+    return block ? block.cid.toString() : '';
+  }
+}

package/src/utils/data-stream.ts

@@ -0,0 +1,99 @@
+import { Encoder } from './encoder.js';
+import { PassThrough, Readable } from 'readable-stream';
+
+/**
+ * Utility class for readable data stream, intentionally named to disambiguate from ReadableStream, readable-stream, Readable etc.
+ */
+export class DataStream {
+  /**
+   * Reads the entire readable stream given into array of bytes.
+   */
+  public static async toBytes(readableStream: Readable): Promise<Uint8Array> {
+    return new Promise((resolve, reject) => {
+      const chunks: any[] = [];
+      readableStream.on('data', chunk => {
+        chunks.push(chunk);
+      });
+
+      readableStream.on('end', () => {
+        const uint8Array = DataStream.concatenateArrayOfBytes(chunks);
+        resolve(uint8Array);
+      });
+
+      readableStream.on('error', reject);
+    });
+  }
+
+  /**
+   * Reads the entire readable stream and JSON parses it into an object.
+   */
+  public static async toObject(readableStream: Readable): Promise<object> {
+    const contentBytes = await DataStream.toBytes(readableStream);
+    const contentObject = Encoder.bytesToObject(contentBytes);
+    return contentObject;
+  }
+
+  /**
+   * Concatenates the array of bytes given into one Uint8Array.
+   */
+  private static concatenateArrayOfBytes(arrayOfBytes: Uint8Array[]): Uint8Array {
+    // sum of individual array lengths
+    const totalLength = arrayOfBytes.reduce((accumulatedValue, currentValue) => accumulatedValue + currentValue.length, 0);
+
+    const result = new Uint8Array(totalLength);
+
+    let length = 0;
+    for (const bytes of arrayOfBytes) {
+      result.set(bytes, length);
+      length += bytes.length;
+    }
+
+    return result;
+  }
+
+  /**
+   * Creates a readable stream from the bytes given.
+   */
+  public static fromBytes(bytes: Uint8Array): Readable {
+    // chunk up the bytes to simulate a more real-world like behavior
+    const chunkLength = 100_000;
+    let currentIndex = 0;
+    const readableStream = new Readable({
+      read(_size): void {
+        // if this is the last chunk
+        if (currentIndex + chunkLength > bytes.length) {
+          this.push(bytes.subarray(currentIndex));
+          this.push(null);
+        } else {
+          this.push(bytes.subarray(currentIndex, currentIndex + chunkLength));
+
+          currentIndex = currentIndex + chunkLength;
+        }
+      }
+    });
+
+    return readableStream;
+  }
+
+  /**
+   * Creates a readable stream from the object given.
+   */
+  public static fromObject(object: Record<string, any>): Readable {
+    const bytes = Encoder.objectToBytes(object);
+    return DataStream.fromBytes(bytes);
+  }
+
+  /**
+   * Duplicates the given data stream into the number of streams specified so that multiple handlers can consume the same data stream.
+   */
+  public static duplicateDataStream(dataStream: Readable, count: number): Readable[] {
+    const streams: Readable[] = [];
+    for (let i = 0; i < count; i++) {
+      const passThrough = new PassThrough();
+      dataStream.pipe(passThrough);
+      streams.push(passThrough as unknown as Readable);
+    }
+
+    return streams;
+  }
+}