@enbox/dwn-sdk-js 0.0.1

package/src/store/storage-controller.ts

@@ -0,0 +1,240 @@
+import type { DataStore } from '../types/data-store.js';
+import type { EventLog } from '../types/event-log.js';
+import type { EventStream } from '../types/subscriptions.js';
+import type { GenericMessage } from '../types/message-types.js';
+import type { MessageStore } from '../types/message-store.js';
+import type { RecordsDeleteMessage, RecordsQueryReplyEntry, RecordsWriteMessage } from '../types/records-types.js';
+
+import { DwnConstant } from '../core/dwn-constant.js';
+import { Message } from '../core/message.js';
+import { Records } from '../utils/records.js';
+import { RecordsDelete } from '../interfaces/records-delete.js';
+import { RecordsWrite } from '../interfaces/records-write.js';
+import { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
+
+
+export type ResumableRecordsDeleteData = {
+  tenant: string;
+  message: RecordsDeleteMessage;
+};
+
+/**
+ * A class that provides an abstraction for the usage of MessageStore, DataStore, and EventLog.
+ */
+export class StorageController {
+
+  private messageStore: MessageStore;
+  private dataStore: DataStore;
+  private eventLog: EventLog;
+  private eventStream?: EventStream;
+
+  public constructor({ messageStore, dataStore, eventLog, eventStream }: {
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    eventLog: EventLog,
+    eventStream?: EventStream}
+  ) {
+    this.messageStore = messageStore;
+    this.dataStore = dataStore;
+    this.eventLog = eventLog;
+    this.eventStream = eventStream;
+  }
+
+  public async performRecordsDelete({ tenant, message }: ResumableRecordsDeleteData): Promise<void> {
+    // get existing records matching the `recordId`
+    const query = {
+      interface : DwnInterfaceName.Records,
+      recordId  : message.descriptor.recordId
+    };
+    const { messages: existingMessages } = await this.messageStore.query(tenant, [ query ]);
+
+    // find which message is the newest, and if the incoming message is the newest
+    const newestExistingMessage = await Message.getNewestMessage(existingMessages);
+
+    if (!Records.canPerformDeleteAgainstRecord(message, newestExistingMessage)) {
+      return;
+    }
+
+    // NOTE: code above is duplicated from `RecordsDeleteHandler` and is already performed if this was invoked by the `RecordsDeleteHandler`,
+    // But we repeat the logic for the code path when the ResumableTaskManager resumes the task.
+    // We make the two different code paths to share this same method to reduce code duplication, there might be a better way to refactor this.
+
+    const recordsDelete = await RecordsDelete.parse(message);
+    const initialWrite = await RecordsWrite.getInitialWrite(existingMessages);
+    const indexes = recordsDelete.constructIndexes(initialWrite);
+    const messageCid = await Message.getCid(message);
+    await this.messageStore.put(tenant, message, indexes);
+    await this.eventLog.append(tenant, messageCid, indexes);
+
+    // only emit if the event stream is set
+    if (this.eventStream !== undefined) {
+      this.eventStream.emit(tenant, { message, initialWrite }, indexes);
+    }
+
+    if (message.descriptor.prune) {
+      // purge/hard-delete all descendent records
+      await StorageController.purgeRecordDescendants(tenant, message.descriptor.recordId, this.messageStore, this.dataStore, this.eventLog);
+    }
+
+    // delete all existing messages that are not newest, except for the initial write
+    await StorageController.deleteAllOlderMessagesButKeepInitialWrite(
+      tenant, existingMessages, message, this.messageStore, this.dataStore, this.eventLog
+    );
+  }
+
+  /**
+   * Deletes the data referenced by the given message if needed.
+   * @param message The message to check if the data it references should be deleted.
+   */
+  private static async deleteFromDataStoreIfNeeded(
+    dataStore: DataStore,
+    tenant: string,
+    message: GenericMessage,
+    newestMessage: GenericMessage
+  ): Promise<void> {
+    if (message.descriptor.method !== DwnMethodName.Write) {
+      return;
+    }
+
+    const recordsWriteMessage = message as RecordsWriteMessage;
+
+    // Optional short-circuit optimization to avoid unnecessary data store call since the data should be encoded with the message itself in this case,
+    // but data store call is a no-op thus code still works correctly even if this short-circuit is removed.
+    if (recordsWriteMessage.descriptor.dataSize <= DwnConstant.maxDataSizeAllowedToBeEncoded) {
+      return;
+    }
+
+    // We must still keep the data if the newest message still references the same data.
+    if (recordsWriteMessage.descriptor.dataCid === (newestMessage as RecordsWriteMessage).descriptor.dataCid) {
+      return;
+    }
+
+    // Else we delete the data from the data store.
+    await dataStore.delete(tenant, recordsWriteMessage.recordId, recordsWriteMessage.descriptor.dataCid);
+  }
+
+  /**
+   * Purges (permanent hard-delete) all descendant's data of the given `recordId`.
+   */
+  public static async purgeRecordDescendants(
+    tenant: string,
+    recordId: string,
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    eventLog: EventLog
+  ): Promise<void> {
+    const filter = {
+      interface : DwnInterfaceName.Records,
+      parentId  : recordId
+    };
+    const { messages: childMessages } = await messageStore.query(tenant, [filter]);
+
+    // group the child messages by `recordId`
+    const recordIdToMessagesMap = new Map<string, GenericMessage[]>();
+    for (const message of childMessages) {
+      // get the recordId
+      let recordId;
+      if (Records.isRecordsWrite(message)) {
+        recordId = message.recordId;
+      } else {
+        recordId = (message as RecordsDeleteMessage).descriptor.recordId;
+      }
+
+      if (!recordIdToMessagesMap.has(recordId)) {
+        recordIdToMessagesMap.set(recordId, []);
+      }
+      recordIdToMessagesMap.get(recordId)!.push(message);
+    }
+
+    // purge all child's descendants first
+    for (const childRecordId of recordIdToMessagesMap.keys()) {
+      // purge the child's descendent messages first
+      await StorageController.purgeRecordDescendants(tenant, childRecordId, messageStore, dataStore, eventLog);
+    }
+
+    // then purge the child messages themselves
+    for (const childRecordId of recordIdToMessagesMap.keys()) {
+      await StorageController.purgeRecordMessages(tenant, recordIdToMessagesMap.get(childRecordId)!, messageStore, dataStore, eventLog);
+    }
+  }
+
+  /**
+   * Purges (permanent hard-delete) all messages of the SAME `recordId` given and their associated data and events.
+   * Assumes that the given `recordMessages` are all of the same `recordId`.
+   */
+  private static async purgeRecordMessages(
+    tenant: string,
+    recordMessages: GenericMessage[],
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    eventLog: EventLog
+  ): Promise<void> {
+    // delete the data from the data store first so no chance of orphaned data (not having a message referencing it) in case of server crash
+    // NOTE: only the `RecordsWrite` with latest timestamp can possibly have data associated with it so we do this filtering as an optimization
+    // NOTE: however there could still be no data associated with the `RecordsWrite` with newest timestamp, because either:
+    //       1. the data is encoded with the message itself; or
+    //       2. the newest `RecordsWrite` may not be the "true" latest state due to:
+    //          a. sync has yet to write the latest `RecordsWrite`; or
+    //          b. `recordMessages` maybe an incomplete page of results if the caller uses the paging in its query
+    // Calling dataStore.delete() is a no-op if the data is not found, so we are safe to call it redundantly.
+    const recordsWrites = recordMessages.filter((message) => message.descriptor.method === DwnMethodName.Write);
+    const newestRecordsWrite = (await Message.getNewestMessage(recordsWrites)) as RecordsWriteMessage;
+    await dataStore.delete(tenant, newestRecordsWrite.recordId, newestRecordsWrite.descriptor.dataCid);
+
+    // then delete all events associated with the record messages before deleting the messages so we don't have orphaned events
+    const messageCids = await Promise.all(recordMessages.map((message) => Message.getCid(message)));
+    await eventLog.deleteEventsByCid(tenant, messageCids);
+
+    // finally delete all record messages
+    await Promise.all(messageCids.map((messageCid) => messageStore.delete(tenant, messageCid)));
+  }
+
+  /**
+   * Deletes all messages in `existingMessages` that are older than the `newestMessage` in the given tenant,
+   * but keep the initial write write for future processing by ensuring its `isLatestBaseState` index is "false".
+   */
+  public static async deleteAllOlderMessagesButKeepInitialWrite(
+    tenant: string,
+    existingMessages: GenericMessage[],
+    newestMessage: GenericMessage,
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    eventLog: EventLog
+  ): Promise<void> {
+    const deletedMessageCids: string[] = [];
+
+    // NOTE: under normal operation, there should only be at most two existing records per `recordId` (initial + a potential subsequent write/delete),
+    // but the DWN may crash before `delete()` is called below, so we use a loop as a tactic to clean up lingering data as needed
+    for (const message of existingMessages) {
+      const messageIsOld = await Message.isOlder(message, newestMessage);
+      if (messageIsOld) {
+        // the easiest implementation here is delete each old messages
+        // and re-create it with the right index (isLatestBaseState = 'false') if the message is the initial write,
+        // but there is room for better/more efficient implementation here
+
+        await StorageController.deleteFromDataStoreIfNeeded(dataStore, tenant, message, newestMessage);
+
+        // delete message from message store
+        const messageCid = await Message.getCid(message);
+        await messageStore.delete(tenant, messageCid);
+
+        // if the existing message is the initial write
+        // we actually need to keep it BUT, need to ensure the message is no longer marked as the latest state
+        const existingMessageIsInitialWrite = await RecordsWrite.isInitialWrite(message);
+        if (existingMessageIsInitialWrite) {
+          const existingRecordsWrite = await RecordsWrite.parse(message as RecordsWriteMessage);
+          const isLatestBaseState = false;
+          const indexes = await existingRecordsWrite.constructIndexes(isLatestBaseState);
+          const writeMessage = message as RecordsQueryReplyEntry;
+          delete writeMessage.encodedData;
+          await messageStore.put(tenant, writeMessage, indexes);
+        } else {
+          const messageCid = await Message.getCid(message);
+          deletedMessageCids.push(messageCid);
+        }
+      }
+
+      await eventLog.deleteEventsByCid(tenant, deletedMessageCids);
+    }
+  }
+}

package/src/types/cache.ts

@@ -0,0 +1,16 @@
+/**
+ * A generalized cache interface.
+ * The motivation behind this interface is so that code that depend on the cache can remain independent to the underlying implementation.
+ */
+export interface Cache {
+  /**
+   * Sets a key-value pair. Does not throw error.
+   */
+  set(key: string, value: any): Promise<void>;
+
+  /**
+   * Gets the value corresponding to the given key.
+   * @returns value stored corresponding to the given key; `undefined` if key is not found or expired
+   */
+  get(key: string): Promise<any | undefined>;
+}

package/src/types/data-store.ts

@@ -0,0 +1,64 @@
+import type { Readable } from 'readable-stream';
+
+/**
+ * The interface that defines how to store and fetch data associated with a message.
+ */
+export interface DataStore {
+  /**
+   * Opens a connection to the underlying store.
+   */
+  open(): Promise<void>;
+
+  /**
+   * Closes the connection to the underlying store.
+   */
+  close(): Promise<void>;
+
+  /**
+   * Stores the given data.
+   * @param recordId The logical ID of the record that references the data.
+   * @param dataCid The IPFS CID of the data.
+   */
+  put(tenant: string, recordId: string, dataCid: string, dataStream: Readable): Promise<DataStorePutResult>;
+
+  /**
+   * Fetches the specified data.
+   * @param recordId The logical ID of the record that references the data.
+   * @param dataCid The IPFS CID of the data.
+   * @returns the data size and data stream if found, otherwise `undefined`.
+   */
+  get(tenant: string, recordId: string, dataCid: string): Promise<DataStoreGetResult | undefined>;
+
+  /**
+   * Deletes the specified data. No-op if the data does not exist.
+   * @param recordId The logical ID of the record that references the data.
+   * @param dataCid The IPFS CID of the data.
+   */
+  delete(tenant: string, recordId: string, dataCid: string): Promise<void>;
+
+  /**
+   * Clears the entire store. Mainly used for testing to cleaning up in test environments.
+   */
+  clear(): Promise<void>;
+}
+
+/**
+ * Result of a data store `put()` method call.
+ */
+export type DataStorePutResult = {
+  /**
+   * The number of bytes of the data stored.
+   */
+  dataSize: number;
+};
+
+/**
+ * Result of a data store `get()` method call if the data exists.
+ */
+export type DataStoreGetResult = {
+  /**
+   * The number of bytes of the data stored.
+   */
+  dataSize: number;
+  dataStream: Readable;
+};

package/src/types/event-log.ts

@@ -0,0 +1,52 @@
+import type { Filter, KeyValues, PaginationCursor } from './query-types.js';
+
+export interface EventLog {
+ /**
+  * opens a connection to the underlying store
+  */
+  open(): Promise<void>;
+
+  /**
+   * closes the connection to the underlying store
+   */
+  close(): Promise<void>;
+
+  /**
+   * adds an event to a tenant's event log
+   * @param tenant - the tenant's DID
+   * @param messageCid - the CID of the message
+   * @param indexes - (key-value pairs) to be included as part of indexing this event.
+   */
+  append(tenant: string, messageCid: string, indexes: KeyValues): Promise<void>
+
+  /**
+   * Retrieves all of a tenant's events that occurred after the cursor provided.
+   * If no cursor is provided, all events for a given tenant will be returned.
+   *
+   * The cursor is a messageCid.
+   *
+   * Returns an array of messageCids that represent the events.
+   */
+  getEvents(tenant: string, cursor?: PaginationCursor): Promise<{ events: string[], cursor?: PaginationCursor }>
+
+  /**
+   * retrieves a filtered set of events that occurred after a the cursor provided, accepts multiple filters.
+   *
+   * If no cursor is provided, all events for a given tenant and filter combo will be returned.
+   * The cursor is a messageCid.
+   *
+   * Returns an array of messageCids that represent the events.
+   */
+  queryEvents(tenant: string, filters: Filter[], cursor?: PaginationCursor): Promise<{ events: string[], cursor?: PaginationCursor }>
+
+  /**
+   * deletes any events that have any of the messageCids provided
+   * @returns {Promise<number>} the number of events deleted
+   */
+  deleteEventsByCid(tenant: string, messageCids: Array<string>): Promise<void>
+
+  /**
+   * Clears the entire store. Mainly used for cleaning up in test environment.
+   */
+  clear(): Promise<void>;
+}

package/src/types/jose-types.ts

@@ -0,0 +1,76 @@
+/**
+ * Contains a public-private key pair and the associated key ID.
+ */
+export type KeyMaterial = {
+  keyId: string,
+  keyPair: { publicJwk: PublicJwk, privateJwk: PrivateJwk }
+};
+
+export type Jwk = {
+  /** The "alg" (algorithm) parameter identifies the algorithm intended for use with the key. */
+  alg?: string;
+  /** The "alg" (algorithm) parameter identifies the algorithm intended for use with the key. */
+  kid?: string;
+  /** identifies the cryptographic algorithm family used with the key, such "EC". */
+  kty: string;
+};
+
+export type PublicJwk = Jwk & {
+  /** The "crv" (curve) parameter identifies the cryptographic curve used with the key.
+   * MUST be present for all EC public keys
+   */
+  crv: 'Ed25519' | 'secp256k1' | 'P-256';
+  /**
+   * the x coordinate for the Elliptic Curve point.
+   * Represented as the base64url encoding of the octet string representation of the coordinate.
+   * MUST be present for all EC public keys
+   */
+  x: string;
+  /**
+   * the y coordinate for the Elliptic Curve point.
+   * Represented as the base64url encoding of the octet string representation of the coordinate.
+   */
+  y?: string;
+};
+
+export type PrivateJwk = PublicJwk & {
+  /**
+   * the Elliptic Curve private key value.
+   * It is represented as the base64url encoding of the octet string representation of the private key value
+   * MUST be present to represent Elliptic Curve private keys.
+   */
+  d: string;
+};
+
+export interface SignatureAlgorithm {
+  /**
+   * signs the provided payload using the provided JWK
+   * @param content - the content to sign
+   * @param privateJwk - the key to sign with
+   * @returns the signed content (aka signature)
+   */
+  sign(content: Uint8Array, privateJwk: PrivateJwk): Promise<Uint8Array>;
+
+  /**
+   * Verifies a signature against the provided payload hash and public key.
+   * @param content - the content to verify with
+   * @param signature - the signature to verify against
+   * @param publicJwk - the key to verify with
+   * @returns a boolean indicating whether the signature matches
+   */
+  verify(content: Uint8Array, signature: Uint8Array, publicJwk: PublicJwk): Promise<boolean>;
+
+  /**
+   * generates a random key pair
+   * @returns the public and private keys as JWKs
+   */
+  generateKeyPair(): Promise<{ publicJwk: PublicJwk, privateJwk: PrivateJwk }>
+
+
+  /**
+   * converts public key in bytes into a JWK
+   * @param publicKeyBytes - the public key to convert into JWK
+   * @returns the public key in JWK format
+   */
+  publicKeyToJwk(publicKeyBytes: Uint8Array): Promise<PublicJwk>
+}

package/src/types/jws-types.ts

@@ -0,0 +1,28 @@
+/**
+ * General JWS definition. Payload is returned as an empty
+ * string when JWS Unencoded Payload Option
+ * [RFC7797](https://www.rfc-editor.org/rfc/rfc7797) is used.
+ */
+export type GeneralJws = {
+  payload: string
+  signatures: SignatureEntry[]
+};
+
+/**
+ * An entry of the `signatures` array in a general JWS.
+ */
+export type SignatureEntry = {
+  /**
+   * The "protected" member MUST be present and contain the value
+   * BASE64URL(UTF8(JWS Protected Header)) when the JWS Protected
+   * Header value is non-empty; otherwise, it MUST be absent.  These
+   * Header Parameter values are integrity protected.
+   */
+  protected: string
+
+  /**
+   * The "signature" member MUST be present and contain the value
+   * BASE64URL(JWS Signature).
+   */
+  signature: string
+};

package/src/types/message-interface.ts

@@ -0,0 +1,30 @@
+import type { GenericMessage, GenericSignaturePayload } from './message-types.js';
+
+/**
+ * An generic interface that represents a DWN message and convenience methods for working with it.
+ */
+export interface MessageInterface<M extends GenericMessage> {
+  /**
+   * Valid JSON message representing this DWN message.
+   */
+  get message(): M;
+
+  /**
+   * Gets the signer of this message.
+   * This is not to be confused with the logical author of the message.
+   */
+  get signer(): string | undefined;
+
+  /**
+   * DID of the logical author of this message.
+   * NOTE: we say "logical" author because a message can be signed by a delegate of the actual author,
+   * in which case the author DID would not be the same as the signer/delegate DID,
+   * but be the DID of the grantor (`grantedBy`) of the delegated grant presented.
+   */
+  get author(): string | undefined;
+
+  /**
+   * Decoded payload of the signature of this message.
+   */
+  get signaturePayload(): GenericSignaturePayload | undefined;
+}

package/src/types/message-store.ts

@@ -0,0 +1,57 @@
+import type { Filter, KeyValues, PaginationCursor } from './query-types.js';
+import type { GenericMessage, MessageSort, Pagination } from './message-types.js';
+
+export interface MessageStoreOptions {
+  signal?: AbortSignal;
+}
+
+export interface MessageStore {
+  /**
+   * opens a connection to the underlying store
+   */
+  open(): Promise<void>;
+
+  /**
+   * closes the connection to the underlying store
+   */
+  close(): Promise<void>;
+
+  /**
+   * adds a message to the underlying store. Uses the message's cid as the key
+   * @param indexes indexes (key-value pairs) to be included as part of this put operation
+   */
+  put(
+    tenant: string,
+    message: GenericMessage,
+    indexes: KeyValues,
+    options?: MessageStoreOptions
+  ): Promise<void>;
+
+  /**
+   * Fetches a single message by `cid` from the underlying store.
+   * Returns `undefined` no message was found.
+   */
+  get(tenant: string, cid: string, options?: MessageStoreOptions): Promise<GenericMessage | undefined>;
+
+  /**
+   * Queries the underlying store for messages that matches the provided filters.
+   * Supplying multiple filters establishes an OR condition between the filters.
+   */
+  query(
+    tenant: string,
+    filters: Filter[],
+    messageSort?: MessageSort,
+    pagination?: Pagination,
+    options?: MessageStoreOptions
+  ): Promise<{ messages: GenericMessage[], cursor?: PaginationCursor}>;
+
+  /**
+   * Deletes the message associated with the id provided.
+   */
+  delete(tenant: string, cid: string, options?: MessageStoreOptions): Promise<void>;
+
+  /**
+   * Clears the entire store. Mainly used for cleaning up in test environment.
+   */
+  clear(): Promise<void>;
+}