@enbox/dwn-sdk-js 0.0.1

package/src/store/level-wrapper.ts

@@ -0,0 +1,272 @@
+import type { AbstractBatchOperation, AbstractDatabaseOptions, AbstractIteratorOptions, AbstractLevel } from 'abstract-level';
+
+import { executeUnlessAborted } from '../utils/abort.js';
+import { Level } from 'level';
+
+export type CreateLevelDatabaseOptions<V> = AbstractDatabaseOptions<string, V>;
+
+export type LevelDatabase<V> = Level<string, V>;
+
+export async function createLevelDatabase<V>(location: string, options?: CreateLevelDatabaseOptions<V>): Promise<LevelDatabase<V>> {
+  // Only import `'level'` when it's actually necessary (i.e. only when the default `createLevelDatabase` is used).
+  // Overriding `createLevelDatabase` will prevent this from happening.
+  return new Level(location, { ...options, keyEncoding: 'utf8' });
+}
+
+export interface LevelWrapperOptions {
+  signal?: AbortSignal;
+}
+
+export type LevelWrapperBatchOperation<V> = AbstractBatchOperation<Level<string, V>, string, V>;
+
+export type LevelWrapperIteratorOptions<V> = AbstractIteratorOptions<string, V>;
+
+// `Level` works in Node.js 12+ and Electron 5+ on Linux, Mac OS, Windows and FreeBSD, including any
+// future Node.js and Electron release thanks to Node-API, including ARM platforms like Raspberry Pi
+// and Android, as well as in Chrome, Firefox, Edge, Safari, iOS Safari and Chrome for Android.
+export class LevelWrapper<V> {
+  config: LevelWrapperConfig<V>;
+
+  db: LevelDatabase<V>;
+
+  /**
+   * @param config.location - must be a directory path (relative or absolute) where `Level`` will
+   * store its files, or in browsers, the name of the {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase `IDBDatabase`}
+   * to be opened.
+   */
+  constructor(config: LevelWrapperConfig<V>, db?: LevelDatabase<V>) {
+    this.config = {
+      createLevelDatabase,
+      ...config
+    };
+
+    this.db = db!;
+  }
+
+  async open(): Promise<void> {
+    await this.createLevelDatabase();
+
+    // `db.open()` is automatically called by the database constructor. We may need to call it explicitly
+    // in order to explicitly catch an error that would otherwise not surface until another method
+    // like `db.get()` is called.  Once `db.open()` has then been called, any read & write
+    // operations will again be queued internally until opening has finished.
+
+    switch (this.db.status) {
+    // If db is open, we are done.
+    case 'open':
+      return;
+
+    // If db is still opening, wait until the 'open' event is emitted
+    case 'opening':
+      return new Promise((resolve) => {
+        this.db.once('open', resolve);
+      });
+
+    // If db is closing, wait until it is closed then await `db.open()`
+    case 'closing':
+      return new Promise((resolve, reject) => {
+        const onClosed = (): void => {
+          // Make sure that errors from `db.open()` propogate up
+          this.db.open().then(resolve).catch(reject);;
+        };
+        this.db.once('closed', onClosed);
+      });
+
+    // If db is closed, `db.open`
+    case 'closed':
+      return this.db.open();
+    }
+  }
+
+  async close(): Promise<void> {
+    if (!this.db) {
+      return;
+    }
+
+    switch (this.db.status) {
+    // If db is open, we `db.close`.
+    case 'open':
+      return this.db.close();
+
+    // If db is still opening, wait until it is open then await `db.close()`
+    case 'opening':
+      return new Promise((resolve, reject) => {
+        const onOpen = (): void => {
+          // Make sure that errors from `db.open()` propogate up
+          this.db.close().then(resolve).catch(reject);;
+        };
+        this.db.once('open', onOpen);
+      });
+
+    // If db is closing, wait until the 'closed' event is emitted
+    case 'closing':
+      return new Promise((resolve) => {
+        this.db.once('closed', resolve);
+      });
+
+    // If db is closed, we are done
+    case 'closed':
+      return;
+    }
+  }
+
+  async partition(name: string): Promise<LevelWrapper<V>> {
+    await this.createLevelDatabase();
+
+    return new LevelWrapper(this.config, (this.db as any).sublevel(name, {
+      keyEncoding   : 'utf8',
+      valueEncoding : this.config.valueEncoding
+    }));
+  }
+
+  async get(key: string, options?: LevelWrapperOptions): Promise<V|undefined>{
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    try {
+      const value = await executeUnlessAborted(this.db.get(String(key)), options?.signal);
+      return value;
+    } catch (error) {
+      const e = error as { code: string };
+      // `Level`` throws an error if the key is not present.  Return `undefined` in this case.
+      if (e.code === 'LEVEL_NOT_FOUND') {
+        return undefined;
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  async has(key: string, options?: LevelWrapperOptions): Promise<boolean> {
+    return !! await this.get(key, options);
+  }
+
+  async * keys(options?: LevelWrapperOptions): AsyncGenerator<string> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    for await (const key of this.db.keys()) {
+      options?.signal?.throwIfAborted();
+
+      yield key;
+    }
+  }
+
+  async * iterator(iteratorOptions?: LevelWrapperIteratorOptions<V>, options?: LevelWrapperOptions): AsyncGenerator<[string, V]> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    for await (const entry of this.db.iterator(iteratorOptions!)) {
+      options?.signal?.throwIfAborted();
+
+      yield entry;
+    }
+  }
+
+  async put(key: string, value: V, options?: LevelWrapperOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    return executeUnlessAborted(this.db.put(String(key), value), options?.signal);
+  }
+
+  async delete(key: string, options?: LevelWrapperOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    return executeUnlessAborted(this.db.del(String(key)), options?.signal);
+  }
+
+  async isEmpty(options?: LevelWrapperOptions): Promise<boolean> {
+    for await (const _key of this.keys(options)) {
+      return false;
+    }
+    return true;
+  }
+
+  async clear(): Promise<void> {
+    await this.createLevelDatabase();
+
+    await this.db.clear();
+
+    await this.compactUnderlyingStorage();
+  }
+
+  async batch(operations: Array<LevelWrapperBatchOperation<V>>, options?: LevelWrapperOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    return executeUnlessAborted(this.db.batch(operations), options?.signal);
+  }
+
+  /**
+   * Wraps the given LevelWrapperBatchOperation as an operation for the specified partition.
+   */
+  createPartitionOperation(partitionName: string, operation: LevelWrapperBatchOperation<V>): LevelWrapperBatchOperation<V> {
+    return { ...operation, sublevel: (this.db as any).sublevel(partitionName, {
+      keyEncoding   : 'utf8',
+      valueEncoding : this.config.valueEncoding
+    }) };
+  }
+
+  private async compactUnderlyingStorage(options?: LevelWrapperOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    await executeUnlessAborted(this.createLevelDatabase(), options?.signal);
+
+    const range = this.sublevelRange;
+    if (!range) {
+      return;
+    }
+
+    // additional methods are only available on the root API instance
+    const root = this.root;
+
+    if (root.db.supports.additionalMethods.compactRange) {
+      return executeUnlessAborted((root.db as any).compactRange?.(...range), options?.signal);
+    }
+  }
+
+  /**
+   * Gets the min and max key value of this partition.
+   */
+  private get sublevelRange(): [ string, string ] | undefined {
+    const prefix = (this.db as any).prefix as string;
+    if (!prefix) {
+      return undefined;
+    }
+
+    // derive an exclusive `maxKey` by changing the last prefix character to the immediate succeeding character in unicode
+    // (which matches how `abstract-level` creates a `boundary`)
+    const maxKey = prefix.slice(0, -1) + String.fromCharCode(prefix.charCodeAt(prefix.length - 1) + 1);
+    const minKey = prefix;
+
+    return [minKey, maxKey];
+  }
+
+  private get root(): LevelWrapper<V> {
+    let db = this.db;
+    for (const parent = (db as any).db; parent && parent !== db; ) {
+      db = parent;
+    }
+    return new LevelWrapper(this.config, db);
+  }
+
+  private async createLevelDatabase(): Promise<void> {
+    this.db ??= await this.config.createLevelDatabase!<V>(this.config.location, {
+      keyEncoding   : 'utf8',
+      valueEncoding : this.config.valueEncoding
+    });
+  }
+}
+
+export type LevelWrapperConfig<V> = CreateLevelDatabaseOptions<V> & {
+  location: string,
+  createLevelDatabase?: typeof createLevelDatabase,
+};

package/src/store/message-store-level.ts

@@ -0,0 +1,195 @@
+
+import type { Filter, KeyValues, PaginationCursor, QueryOptions } from '../types/query-types.js';
+import type { GenericMessage, MessageSort, Pagination } from '../types/message-types.js';
+import type { MessageStore, MessageStoreOptions } from '../types/message-store.js';
+
+import * as block from 'multiformats/block';
+import * as cbor from '@ipld/dag-cbor';
+
+import { BlockstoreLevel } from './blockstore-level.js';
+import { Cid } from '../utils/cid.js';
+import { CID } from 'multiformats/cid';
+import { createLevelDatabase } from './level-wrapper.js';
+import { executeUnlessAborted } from '../utils/abort.js';
+import { IndexLevel } from './index-level.js';
+import { Message } from '../core/message.js';
+import { sha256 } from 'multiformats/hashes/sha2';
+import { SortDirection } from '../types/query-types.js';
+
+
+/**
+ * A simple implementation of {@link MessageStore} that works in both the browser and server-side.
+ * Leverages LevelDB under the hood.
+ */
+export class MessageStoreLevel implements MessageStore {
+  config: MessageStoreLevelConfig;
+
+  blockstore: BlockstoreLevel;
+  index: IndexLevel;
+
+  /**
+   * @param {MessageStoreLevelConfig} config
+   * @param {string} config.blockstoreLocation - must be a directory path (relative or absolute) where
+   *  LevelDB will store its files, or in browsers, the name of the
+   * {@link https://developer.mozilla.org/en-US/docs/Web/API/IDBDatabase IDBDatabase} to be opened.
+   * @param {string} config.indexLocation - same as config.blockstoreLocation
+   */
+  constructor(config: MessageStoreLevelConfig = {}) {
+    this.config = {
+      blockstoreLocation : 'MESSAGESTORE',
+      indexLocation      : 'INDEX',
+      createLevelDatabase,
+      ...config
+    };
+
+    this.blockstore = new BlockstoreLevel({
+      location            : this.config.blockstoreLocation!,
+      createLevelDatabase : this.config.createLevelDatabase,
+    });
+
+    this.index = new IndexLevel({
+      location            : this.config.indexLocation!,
+      createLevelDatabase : this.config.createLevelDatabase,
+    });
+  }
+
+  async open(): Promise<void> {
+    await this.blockstore.open();
+    await this.index.open();
+  }
+
+  async close(): Promise<void> {
+    await this.blockstore.close();
+    await this.index.close();
+  }
+
+  async get(tenant: string, cidString: string, options?: MessageStoreOptions): Promise<GenericMessage | undefined> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const cid = CID.parse(cidString);
+    const bytes = await partition.get(cid, options);
+
+    if (!bytes) {
+      return undefined;
+    }
+
+    const decodedBlock = await executeUnlessAborted(block.decode({ bytes, codec: cbor, hasher: sha256 }), options?.signal);
+
+    const message = decodedBlock.value as GenericMessage;
+    return message;
+  }
+
+  async query(
+    tenant: string,
+    filters: Filter[],
+    messageSort?: MessageSort,
+    pagination?: Pagination,
+    options?: MessageStoreOptions
+  ): Promise<{ messages: GenericMessage[], cursor?: PaginationCursor}> {
+    options?.signal?.throwIfAborted();
+
+    // creates the query options including sorting and pagination.
+    // this adds 1 to the limit if provided, that way we can check to see if there are additional results and provide a return cursor.
+    const queryOptions = MessageStoreLevel.buildQueryOptions(messageSort, pagination);
+    const results = await this.index.query(tenant, filters, queryOptions, options);
+
+    let cursor: PaginationCursor | undefined;
+    // checks to see if the returned results are greater than the limit, which would indicate additional results.
+    if (pagination?.limit !== undefined && pagination.limit < results.length) {
+      // has additional records, remove last record and set cursor
+      results.splice(-1);
+
+      // set cursor to the last item remaining after the spliced result.
+      cursor = IndexLevel.createCursorFromLastArrayItem(results, queryOptions.sortProperty);
+    }
+
+    const messages: GenericMessage[] = [];
+    for (let i = 0; i < results.length; i++) {
+      const { messageCid } = results[i];
+      const message = await this.get(tenant, messageCid, options);
+      if (message) { messages.push(message); }
+    }
+
+    return { messages, cursor };
+  }
+
+  /**
+   * Builds the IndexLevel QueryOptions object given MessageStore sort and pagination parameters.
+   */
+  static buildQueryOptions(messageSort: MessageSort = {}, pagination: Pagination = {}): QueryOptions {
+    let { limit, cursor } = pagination;
+    const { dateCreated, datePublished, messageTimestamp } = messageSort;
+
+    let sortDirection = SortDirection.Ascending; // default
+    // `keyof MessageSort` = name of all properties of `MessageSort` defaults to messageTimestamp
+    let sortProperty: keyof MessageSort = 'messageTimestamp';
+
+    // set the sort property
+    if (dateCreated !== undefined) {
+      sortProperty = 'dateCreated';
+    } else if (datePublished !== undefined) {
+      sortProperty = 'datePublished';
+    } else if (messageTimestamp !== undefined) {
+      sortProperty = 'messageTimestamp';
+    }
+
+    if (messageSort[sortProperty] !== undefined) {
+      sortDirection = messageSort[sortProperty]!;
+    }
+
+    // we add one more to the limit to determine whether there are additional results and to return a cursor.
+    if (limit !== undefined && limit > 0) {
+      limit = limit + 1;
+    }
+
+    return { sortDirection, sortProperty, limit, cursor };
+  }
+
+  async delete(tenant: string, cidString: string, options?: MessageStoreOptions): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const cid = CID.parse(cidString);
+    await partition.delete(cid, options);
+    await this.index.delete(tenant, cidString, options);
+  }
+
+  async put(
+    tenant: string,
+    message: GenericMessage,
+    indexes: KeyValues,
+    options?: MessageStoreOptions
+  ): Promise<void> {
+    options?.signal?.throwIfAborted();
+
+    const partition = await executeUnlessAborted(this.blockstore.partition(tenant), options?.signal);
+
+    const encodedMessageBlock = await executeUnlessAborted(block.encode({ value: message, codec: cbor, hasher: sha256 }), options?.signal);
+
+    // MessageStore data may contain `encodedData` which is not taken into account when calculating the blockCID as it is optional data.
+    const messageCid = Cid.parseCid(await Message.getCid(message));
+    await partition.put(messageCid, encodedMessageBlock.bytes, options);
+
+    const messageCidString = messageCid.toString();
+
+    await this.index.put(tenant, messageCidString, indexes, options);
+  }
+
+
+  /**
+   * deletes everything in the underlying blockstore and indices.
+   */
+  async clear(): Promise<void> {
+    await this.blockstore.clear();
+    await this.index.clear();
+  }
+}
+
+export type MessageStoreLevelConfig = {
+  blockstoreLocation?: string,
+  indexLocation?: string,
+  createLevelDatabase?: typeof createLevelDatabase,
+};

package/src/store/resumable-task-store-level.ts

@@ -0,0 +1,120 @@
+import type { ManagedResumableTask, ResumableTaskStore } from '../types/resumable-task-store.js';
+
+import { Cid } from '../utils/cid.js';
+import { createLevelDatabase, LevelWrapper } from './level-wrapper.js';
+
+export type ResumableTaskStoreLevelConfig = {
+  location?: string,
+  createLevelDatabase?: typeof createLevelDatabase
+};
+
+/**
+ * A simple single-instance implementation of {@link ResumableTaskStore} that works in both browsers and node.js.
+ * Leverages LevelDB under the hood.
+ */
+export class ResumableTaskStoreLevel implements ResumableTaskStore {
+  private static readonly taskTimeoutInSeconds = 60;
+
+  db: LevelWrapper<string>;
+  config: ResumableTaskStoreLevelConfig;
+
+  constructor(config: ResumableTaskStoreLevelConfig) {
+    this.config = {
+      // defaults:
+      location: 'RESUMABLE-TASK-STORE',
+      createLevelDatabase,
+      // user-provided overrides:
+      ...config,
+    };
+
+    this.db = new LevelWrapper<string>({
+      location            : this.config.location!,
+      createLevelDatabase : this.config.createLevelDatabase,
+      keyEncoding         : 'utf8'
+    });
+  }
+
+  async open(): Promise<void> {
+    await this.db.open();
+  }
+
+  async close(): Promise<void> {
+    await this.db.close();
+  }
+
+  public async register(task: any, timeoutInSeconds: number): Promise<ManagedResumableTask> {
+    const taskId = await Cid.computeCid(task);
+
+    const managedResumableTask: ManagedResumableTask = {
+      id         : taskId,
+      timeout    : Date.now() + (timeoutInSeconds * 1000),
+      retryCount : 0,
+      task,
+    };
+
+    await this.db.put(taskId, JSON.stringify(managedResumableTask));
+
+    return managedResumableTask;
+  }
+
+  public async grab(count: number): Promise<ManagedResumableTask[]> {
+    const tasks: ManagedResumableTask[] = [];
+
+    // iterate over the tasks to find unhandled (timed-out) tasks to return to the caller,
+    // tasks that are not timed-out are considered in-flight/under processing
+    // NOTE: there is an opportunity here to introduce an additional index where we can query by timed-out tasks,
+    // but it requires an additional index thus more complexity
+    for await (const [ _, value ] of this.db.iterator()) {
+      const task = JSON.parse(value) as ManagedResumableTask;
+
+      // if the task is timed-out, we can give it to the caller to handle
+      if (Date.now() >= task.timeout) {
+        // update the task metadata first before adding to list of tasks to return
+        task.timeout = Date.now() + (ResumableTaskStoreLevel.taskTimeoutInSeconds * 1000);
+        task.retryCount++;
+        await this.db.put(task.id, JSON.stringify(task));
+
+        tasks.push(task);
+      }
+
+      if (tasks.length >= count) {
+        break;
+      }
+    }
+
+    return tasks;
+  }
+
+  public async read(taskId: string): Promise<ManagedResumableTask | undefined> {
+    const value = await this.db.get(taskId);
+
+    if (value) {
+      const task = JSON.parse(value) as ManagedResumableTask;
+      return task;
+    } else {
+      return undefined;
+    }
+  }
+
+  public async extend(taskId: string, timeoutInSeconds: number): Promise<void> {
+    const value = await this.db.get(taskId);
+
+    if (value) {
+      const task = JSON.parse(value) as ManagedResumableTask;
+      task.timeout = Date.now() + (timeoutInSeconds * 1000);
+
+      await this.db.put(task.id, JSON.stringify(task));
+    }
+  }
+
+  public async delete(taskId: string): Promise<void> {
+    return this.db.delete(taskId);
+  }
+
+  /**
+   * Deletes everything in the store. Mainly used in tests.
+   */
+  public async clear(): Promise<void> {
+    await this.db.clear();
+  }
+}