@abaxxtech/id 0.0.0

package/src/store/message-store-level.ts

@@ -0,0 +1,284 @@
+
+import type { RecordsWriteMessage } from '../types/records-types.js';
+import type { Filter, GenericMessage, MessageSort, Pagination } from '../types/message-types.js';
+import type { MessageStore, MessageStoreOptions } from '../types/message-store.js';
+
+import * as block from 'multiformats/block';
+import * as cbor from '@ipld/dag-cbor';
+
+import { ArrayUtility } from '../utils/array.js';
+import { BlockstoreLevel } from './blockstore-level.js';
+import { Cid } from '../utils/cid.js';
+import { CID } from 'multiformats/cid';
+import { createLevelDatabase } from './level-wrapper.js';
+import { executeUnlessAborted } from '../utils/abort.js';
+import { IndexLevel } from './index-level.js';
+import { Message } from '../core/message.js';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { SortOrder } from '../types/message-types.js';
+
+
+/**
+ * A simple implementation of {@link MessageStore} that works in both the browser and server-side.
+ * Leverages LevelDB under the hood.
+ */
+export class MessageStoreLevel implements MessageStore {
+  config: MessageStoreLevelConfig;
+
+  blockstore: BlockstoreLevel;
+
+  index: IndexLevel;
+
+  /**
+   * @param {MessageStoreLevelConfig} config
+   * @param {string} config.blockstoreLocation - must be a directory path (relative or absolute) where
+   *  LevelDB will store its files, or in browsers, the name of the
+   * {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} to be opened.
+   * @param {string} config.indexLocation - same as config.blockstoreLocation
+   */
+  constructor(config: MessageStoreLevelConfig = {}) {
+    this.config = {
+      blockstoreLocation : 'data/MESSAGESTORE',
+      indexLocation      : 'data/INDEX',
+      createLevelDatabase,
+      ...config
+    };
+
+    this.blockstore = new BlockstoreLevel({
+      location            : this.config.blockstoreLocation!,
+      createLevelDatabase : this.config.createLevelDatabase,
+    });
+
+    this.index = new IndexLevel({
+      location            : this.config.indexLocation!,
+      createLevelDatabase : this.config.createLevelDatabase,
+    });
+  }
+
+  async open(): Promise<void> {
+    await this.blockstore.open();
+    await this.index.open();
+  }
+
+  async close(): Promise<void> {
+    await this.blockstore.close();
+    await this.index.close();
+  }
+
+  async get(tenant: string, cidString: string, options?: MessageStoreOptions): Promise<GenericMessage | undefined> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const cid = CID.parse(cidString);
+    const bytes = await partition.get(cid, options);
+
+    if (!bytes) {
+      return undefined;
+    }
+
+    const decodedBlock = await executeUnlessAborted(block.decode({ bytes, codec: cbor, hasher: sha256 }), options?.signal);
+
+    const message = decodedBlock.value as GenericMessage;
+    return message;
+  }
+
+  async query(
+    tenant: string,
+    filters: Filter[],
+    messageSort?: MessageSort,
+    pagination?: Pagination,
+    options?: MessageStoreOptions
+  ): Promise<{ messages: GenericMessage[], cursor?: string }> {
+    options?.signal?.throwIfAborted();
+
+    const messages: GenericMessage[] = [];
+    // note: injecting tenant into filters to allow querying with an "empty" filter.
+    // if there are no other filters present it will return all the messages the tenant.
+    const resultIds = await this.index.query(tenant, filters.map(f => ({ ...f, tenant })), options);
+
+    // as an optimization for large data sets, we are finding the message object which matches the cursor here.
+    // we can use this within the pagination function after sorting to determine the starting point of the array in a more efficient way.
+    let paginationMessage: GenericMessage | undefined;
+    for (const id of resultIds) {
+      const message = await this.get(tenant, id, options);
+      if (message) { messages.push(message); }
+      if (pagination?.cursor && pagination.cursor === id) {
+        paginationMessage = message;
+      }
+    }
+
+    if (pagination?.cursor !== undefined && paginationMessage === undefined) {
+      return { messages: [] }; //if paginationMessage is not found, do not return any results
+    }
+
+    const sortedRecords = await MessageStoreLevel.sortMessages(messages, messageSort);
+    return this.paginateMessages(sortedRecords, paginationMessage, pagination);
+  }
+
+  private async paginateMessages(
+    messages: GenericMessage[],
+    paginationMessage?: GenericMessage,
+    pagination: Pagination = { }
+  ): Promise<{ messages: GenericMessage[], cursor?: string } > {
+    const { limit } = pagination;
+    if (paginationMessage === undefined && limit === undefined) {
+      return { messages }; // return all without pagination pointer.
+    }
+
+    // we are passing the pagination message object for an easier lookup
+    // since we know this object exists within the array if passed, we can assume that it will always have a value greater than -1
+    const cursorIndex = paginationMessage ? messages.indexOf(paginationMessage) : undefined;
+
+    // the first element of the returned results is always the message immediately following the cursor.
+    const start = cursorIndex === undefined ? 0 : cursorIndex + 1;
+    const end = limit === undefined ? undefined : start + limit;
+    const results = messages.slice(start, end);
+
+    // we only return a cursor cursor if there are more results
+    const hasMoreResults = end !== undefined && end < messages.length;
+    let cursor: string|undefined;
+    if (hasMoreResults) {
+      // we extract the cid of the last message in the result set.
+      const lastMessage = results.at(-1);
+      cursor = await Message.getCid(lastMessage!);
+    }
+
+    return { messages: results, cursor };
+  }
+
+  /**
+   * Compares the chosen property of two messages in lexicographical order.
+   * When the value is the same between the two objects, `messageCid` comparison is used to tiebreak.
+   * tiebreaker always compares messageA to messageB
+   *
+   * @returns if SortOrder is Ascending:
+   *            1 if the chosen property of `messageA` is larger than of `messageB`;
+   *           -1 if the chosen property `messageA` is smaller/older than of `messageB`;
+   *            0 otherwise
+   *          if SortOrder is Descending:
+   *            1 if the chosen property of `messageB` is larger than of `messageA`;
+   *           -1 if the chosen property `messageB` is smaller/older than of `messageA`;
+   *            0 otherwise
+   */
+  static async lexicographicalCompare(
+    messageA: GenericMessage,
+    messageB: GenericMessage,
+    comparedPropertyName: string,
+    sortOrder: SortOrder): Promise<number>
+  {
+    const a = (messageA.descriptor as any)[comparedPropertyName];
+    const b = (messageB.descriptor as any)[comparedPropertyName];
+
+    if (sortOrder === SortOrder.Ascending) {
+      if (a > b) {
+        return 1;
+      } else if (a < b) {
+        return -1;
+      }
+    } else {
+      // descending order
+      if (b > a) {
+        return 1;
+      } else if (b < a) {
+        return -1;
+      }
+    }
+
+    // if we reach here it means the compared properties have the same values, we need to fall back to compare the `messageCid` instead
+    return await Message.compareCid(messageA, messageB);
+  }
+
+  /**
+   * This is a temporary naive sort, it will eventually be done within the underlying data store.
+   *
+   * If sorting is based on date published, records that are not published are filtered out.
+   * @param messages - Messages to be sorted if dateSort is present
+   * @param sort - Sorting scheme
+   * @returns Sorted Messages
+   */
+  public static async sortMessages(
+    messages: GenericMessage[],
+    messageSort: MessageSort = { }
+  ): Promise<GenericMessage[]> {
+    const { dateCreated, datePublished, messageTimestamp } = messageSort;
+
+    let sortOrder = SortOrder.Ascending; // default
+    let messagesToSort = messages; // default
+    let propertyToCompare: keyof MessageSort | undefined; // `keyof MessageSort` = name of all properties of `MessageSort`
+
+    if (dateCreated !== undefined) {
+      propertyToCompare = 'dateCreated';
+    } else if (datePublished !== undefined) {
+      propertyToCompare = 'datePublished';
+      messagesToSort = (messages as RecordsWriteMessage[]).filter(message => message.descriptor.published);
+    } else if (messageTimestamp !== undefined) {
+      propertyToCompare = 'messageTimestamp';
+    }
+
+    if (propertyToCompare !== undefined) {
+      sortOrder = messageSort[propertyToCompare]!;
+    } else {
+      propertyToCompare = 'messageTimestamp';
+    }
+
+    const asyncComparer = (a: GenericMessage, b: GenericMessage): Promise<number> => {
+      return MessageStoreLevel.lexicographicalCompare(a, b, propertyToCompare!, sortOrder);
+    };
+
+    // NOTE: we needed to implement our own asynchronous sort method because Array.sort() does not take an async comparer
+    return await ArrayUtility.asyncSort(messagesToSort, asyncComparer);
+  }
+
+  async delete(tenant: string, cidString: string, options?: MessageStoreOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const cid = CID.parse(cidString);
+    await partition.delete(cid, options);
+    await this.index.delete(tenant, cidString, options);
+  }
+
+  async put(
+    tenant: string,
+    message: GenericMessage,
+    indexes: { [key: string]: string | boolean },
+    options?: MessageStoreOptions
+  ): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const encodedMessageBlock = await executeUnlessAborted(block.encode({ value: message, codec: cbor, hasher: sha256 }), options?.signal);
+
+    // MessageStore data may contain `encodedData` which is not taken into account when calculating the blockCID as it is optional data.
+    const messageCid = Cid.parseCid(await Message.getCid(message));
+    await partition.put(messageCid, encodedMessageBlock.bytes, options);
+
+    const messageCidString = messageCid.toString();
+
+    // note: leaving the additional tenant indexing to allow for querying with an "empty" filter.
+    // when querying, we also inject a filter for the specific tenant.
+    // if there are no other filters present it will return all the messages for that tenant.
+    const indexDocument = {
+      ...indexes,
+      tenant,
+    };
+    await this.index.put(tenant, messageCidString, indexDocument, options);
+  }
+
+  /**
+   * deletes everything in the underlying blockstore and indices.
+   */
+  async clear(): Promise<void> {
+    await this.blockstore.clear();
+    await this.index.clear();
+  }
+}
+
+type MessageStoreLevelConfig = {
+  blockstoreLocation?: string,
+  indexLocation?: string,
+  createLevelDatabase?: typeof createLevelDatabase,
+};

package/src/store/storage-controller.ts

@@ -0,0 +1,80 @@
+import type { DataStore } from '../types/data-store.js';
+import type { EventLog } from '../types/event-log.js';
+import type { GenericMessage } from '../types/message-types.js';
+import type { MessageStore } from '../types/message-store.js';
+import type { RecordsWriteMessage, RecordsWriteMessageWithOptionalEncodedData } from '../types/records-types.js';
+
+import { DwnConstant } from '../core/dwn-constant.js';
+import { DwnMethodName } from '../enums/dwn-interface-method.js';
+import { Message } from '../core/message.js';
+import { RecordsWrite } from '../interfaces/records-write.js';
+
+/**
+ * A class that provides an abstraction for the usage of MessageStore, DataStore, and EventLog.
+ */
+export class StorageController {
+  /**
+   * Deletes a message.
+   */
+  private static async delete(
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    tenant: string,
+    message: GenericMessage
+  ): Promise<void> {
+    const messageCid = await Message.getCid(message);
+
+    if (message.descriptor.method === DwnMethodName.Write &&
+        (message as RecordsWriteMessage).descriptor.dataSize > DwnConstant.maxDataSizeAllowedToBeEncoded) {
+      const recordsWriteMessage = message as RecordsWriteMessage;
+      await dataStore.delete(tenant, messageCid, recordsWriteMessage.descriptor.dataCid);
+    }
+
+    await messageStore.delete(tenant, messageCid);
+  }
+
+
+  /**
+   * Deletes all messages in `existingMessages` that are older than the `comparedToMessage` in the given tenant,
+   * but keep the initial write write for future processing by ensuring its `isLatestBaseState` index is "false".
+   */
+  public static async deleteAllOlderMessagesButKeepInitialWrite(
+    tenant: string,
+    existingMessages: GenericMessage[],
+    comparedToMessage: GenericMessage,
+    messageStore: MessageStore,
+    dataStore: DataStore,
+    eventLog: EventLog
+  ): Promise<void> {
+    const deletedMessageCids: string[] = [];
+
+    // NOTE: under normal operation, there should only be at most two existing records per `recordId` (initial + a potential subsequent write/delete),
+    // but the DWN may crash before `delete()` is called below, so we use a loop as a tactic to clean up lingering data as needed
+    for (const message of existingMessages) {
+      const messageIsOld = await Message.isOlder(message, comparedToMessage);
+      if (messageIsOld) {
+      // the easiest implementation here is delete each old messages
+      // and re-create it with the right index (isLatestBaseState = 'false') if the message is the initial write,
+      // but there is room for better/more efficient implementation here
+        await StorageController.delete(messageStore, dataStore, tenant, message);
+
+        // if the existing message is the initial write
+        // we actually need to keep it BUT, need to ensure the message is no longer marked as the latest state
+        const existingMessageIsInitialWrite = await RecordsWrite.isInitialWrite(message);
+        if (existingMessageIsInitialWrite) {
+          const existingRecordsWrite = await RecordsWrite.parse(message as RecordsWriteMessage);
+          const isLatestBaseState = false;
+          const indexes = await existingRecordsWrite.constructRecordsWriteIndexes(isLatestBaseState);
+          const writeMessage = message as RecordsWriteMessageWithOptionalEncodedData;
+          delete writeMessage.encodedData;
+          await messageStore.put(tenant, writeMessage, indexes);
+        } else {
+          const messageCid = await Message.getCid(message);
+          deletedMessageCids.push(messageCid);
+        }
+      }
+
+      await eventLog.deleteEventsByCid(tenant, deletedMessageCids);
+    }
+  }
+}

package/src/types/cache.ts

@@ -0,0 +1,16 @@
+/**
+ * A generalized cache interface.
+ * The motivation behind this interface is so that code that depend on the cache can remain independent to the underlying implementation.
+ */
+export interface Cache {
+  /**
+   * Sets a key-value pair. Does not throw error.
+   */
+  set(key: string, value: any): Promise<void>;
+
+  /**
+   * Gets the value corresponding to the given key.
+   * @returns value stored corresponding to the given key; `undefined` if key is not found or expired
+   */
+  get(key: string): Promise<any | undefined>;
+}

package/src/types/data-store.ts

@@ -0,0 +1,78 @@
+import type { Readable } from 'readable-stream';
+
+/**
+ * The interface that defines how to store and fetch data associated with a message
+ */
+export interface DataStore {
+  /**
+   * Opens a connection to the underlying store.
+   */
+  open(): Promise<void>;
+
+  /**
+   * Closes the connection to the underlying store.
+   */
+  close(): Promise<void>;
+
+  /**
+   * Puts the given data in store.
+   * It is expected that the CID of the dataStream matches the given dataCid.
+   * The returned dataCid and returned dataSize will be verified against the given dataCid (and inferred dataSize).
+   * @param messageCid CID of the message that references the data.
+   * @returns The CID and size in number of bytes of the data stored.
+   */
+  put(tenant: string, messageCid: string, dataCid: string, dataStream: Readable): Promise<PutResult>;
+
+  /**
+   * Fetches the specified data.
+   * The returned dataCid and returned dataSize will be verified against the given dataCid (and inferred dataSize).
+   * @param messageCid CID of the message that references the data.
+   */
+  get(tenant: string, messageCid: string, dataCid: string): Promise<GetResult | undefined>;
+
+  /**
+   * Associates dataCid of existing data with the given messageCid.
+   * The returned dataCid and returned dataSize will be verified against the given dataCid (and inferred dataSize).
+   * @param tenant The tenant in which the data must exist under for the association to occur.
+   * @param messageCid CID of the message that references the data.
+   * @param dataCid The CID of the data stored.
+   * @returns {AssociateResult} if association succeeds. `undefined` if data to be associated is not found.
+   */
+  associate(tenant: string, messageCid: string, dataCid: string): Promise<AssociateResult | undefined>;
+
+  /**
+   * Deletes the specified data.
+   * @param messageCid CID of the message that references the data.
+   */
+  delete(tenant: string, messageCid: string, dataCid: string): Promise<void>;
+
+  /**
+   * Clears the entire store. Mainly used for cleaning up in test environment.
+   */
+  clear(): Promise<void>;
+}
+
+/**
+ * Result of a data store `put()` method call.
+ */
+export type PutResult = {
+  dataCid: string;
+  dataSize: number;
+};
+
+/**
+ * Result of a data store `get()` method call.
+ */
+export type GetResult = {
+  dataCid: string;
+  dataSize: number;
+  dataStream: Readable;
+};
+
+/**
+ * Result of a data store `associate()` method call.
+ */
+export type AssociateResult = {
+  dataCid: string;
+  dataSize: number;
+};

package/src/types/delegated-grant-message.ts

@@ -0,0 +1,15 @@
+import type { GeneralJws } from './jws-types.js';
+import type { PermissionsGrantDescriptor } from './permissions-grant-descriptor.js';
+
+export type DelegatedGrantMessage = {
+  authorization: {
+    /**
+     * The signature of the message signer.
+     */
+    signature: GeneralJws;
+  };
+
+  descriptor: PermissionsGrantDescriptor & {
+    delegated: true;
+  };
+};

package/src/types/did-types.ts

@@ -0,0 +1,95 @@
+import type { PublicJwk } from './jose-types.js';
+
+/**
+ * A generalized interface that can be implemented for individual
+ * DID methods
+ */
+export interface DidMethodResolver {
+  /**
+   * @returns the DID method supported by {@link DidMethodResolver.resolve}
+   */
+  method(): string;
+
+  /**
+   * attempts to resolve the DID provided into its respective DID Document.
+   * More info on resolving DIDs can be found
+   * {@link https://www.w3.org/TR/did-core/#resolution here}
+   * @param did - the DID to resolve
+   * @throws {Error} if unable to resolve the DID
+   */
+  resolve(did: string): Promise<DidResolutionResult>;
+}
+
+
+
+export type DidDocument = {
+  '@context'?: 'https://www.w3.org/ns/did/v1' | string | string[]
+  id: string
+  alsoKnownAs?: string[]
+  controller?: string | string[]
+  verificationMethod?: VerificationMethod[]
+  service?: ServiceEndpoint[]
+  authentication?: VerificationMethod[] | string[]
+  assertionMethod?: VerificationMethod[] | string[]
+  keyAgreement?: VerificationMethod[] | string[]
+  capabilityInvocation?: VerificationMethod[] | string[]
+  capabilityDelegation?: VerificationMethod[] | string[]
+};
+
+export type DwnServiceEndpoint = {
+  nodes: string[]
+};
+
+export type ServiceEndpoint = {
+  id: string
+  type: string
+  serviceEndpoint: string | DwnServiceEndpoint
+  description?: string
+};
+
+export type VerificationMethod = {
+  id: string
+  // one of the valid verification method types as per
+  // https://www.w3.org/TR/did-spec-registries/#verification-method-types
+  type: string
+  // DID of the key's controller
+  controller: string
+  // a JSON Web Key that conforms to https://datatracker.ietf.org/doc/html/rfc7517
+  publicKeyJwk?: PublicJwk
+};
+
+export type DidResolutionResult = {
+  '@context'?: 'https://w3id.org/did-resolution/v1' | string | string[]
+  didResolutionMetadata: DidResolutionMetadata
+  didDocument?: DidDocument
+  didDocumentMetadata: DidDocumentMetadata
+};
+
+export type DidResolutionMetadata = {
+  contentType?: string
+  error?: 'invalidDid' | 'notFound' | 'representationNotSupported' |
+  'unsupportedDidMethod' | string
+};
+
+export type DidDocumentMetadata = {
+  // indicates the timestamp of the Create operation. ISO8601 timestamp
+  created?: string
+  // indicates the timestamp of the last Update operation for the document version which was
+  // resolved. ISO8601 timestamp
+  updated?: string
+  // indicates whether the DID has been deactivated
+  deactivated?: boolean
+  // indicates the version of the last Update operation for the document version which
+  // was resolved
+  versionId?: string
+  // indicates the timestamp of the next Update operation if the resolved document version
+  // is not the latest version of the document.
+  nextUpdate?: string
+  // indicates the version of the next Update operation if the resolved document version
+  // is not the latest version of the document.
+  nextVersionId?: string
+  // @see https://www.w3.org/TR/did-core/#dfn-equivalentid
+  equivalentId?: string
+  // @see https://www.w3.org/TR/did-core/#dfn-canonicalid
+  canonicalId?: string
+};

package/src/types/event-log.ts

@@ -0,0 +1,46 @@
+export type Event = {
+  watermark: string,
+  messageCid: string
+};
+
+
+export type GetEventsOptions = {
+  gt: string
+};
+
+export interface EventLog {
+ /**
+  * opens a connection to the underlying store
+  */
+  open(): Promise<void>;
+
+  /**
+   * closes the connection to the underlying store
+   */
+  close(): Promise<void>;
+
+  /**
+   * adds an event to a tenant's event log
+   * @param tenant - the tenant's DID
+   * @param messageCid - the CID of the message
+   * @returns {Promise<string>} watermark
+   */
+  append(tenant: string, messageCid: string): Promise<string>
+
+  /**
+   * retrieves all of a tenant's events that occurred after the watermark provided.
+   * If no watermark is provided, all events for a given tenant will be returned.
+   */
+  getEvents(tenant: string, options?: GetEventsOptions): Promise<Array<Event>>
+
+  /**
+   * deletes any events that have any of the cids provided
+   * @returns {Promise<number>} the number of events deleted
+   */
+  deleteEventsByCid(tenant: string, cids: Array<string>): Promise<number>
+
+  /**
+   * Clears the entire store. Mainly used for cleaning up in test environment.
+   */
+  clear(): Promise<void>;
+}

package/src/types/event-types.ts

@@ -0,0 +1,20 @@
+import type { Event } from './event-log.js';
+import type { GenericMessageReply } from '../core/message-reply.js';
+import type { AuthorizationModel, GenericMessage } from './message-types.js';
+import type { DwnInterfaceName, DwnMethodName } from '../enums/dwn-interface-method.js';
+
+export type EventsGetDescriptor = {
+  interface : DwnInterfaceName.Events;
+  method: DwnMethodName.Get;
+  watermark?: string;
+  messageTimestamp: string;
+};
+
+export type EventsGetMessage = GenericMessage & {
+  authorization: AuthorizationModel; // overriding `GenericMessage` with `authorization` being required
+  descriptor: EventsGetDescriptor;
+};
+
+export type EventsGetReply = GenericMessageReply & {
+  events?: Event[];
+};