@enbox/dwn-sql-store 0.0.1

package/src/resumable-task-store-sql.ts

@@ -0,0 +1,174 @@
+import { DwnDatabaseType } from './types.js';
+import { Dialect } from './dialect/dialect.js';
+import { executeWithRetryIfDatabaseIsLocked } from './utils/transaction.js';
+import { Kysely } from 'kysely';
+import { Cid, ManagedResumableTask, ResumableTaskStore } from '@enbox/dwn-sdk-js';
+
+export class ResumableTaskStoreSql implements ResumableTaskStore {
+  private static readonly taskTimeoutInSeconds = 60;
+
+  #dialect: Dialect;
+  #db: Kysely<DwnDatabaseType> | null = null;
+
+  constructor(dialect: Dialect) {
+    this.#dialect = dialect;
+  }
+
+  async open(): Promise<void> {
+    if (this.#db) {
+      return;
+    }
+
+    this.#db = new Kysely<DwnDatabaseType>({ dialect: this.#dialect });
+
+    // if table already exists, there is no more things todo
+    const tableName = 'resumableTasks';
+    const tableExists = await this.#dialect.hasTable(this.#db, tableName);
+    if (tableExists) {
+      return;
+    }
+
+    // else create the table and corresponding indexes
+
+    let table = this.#db.schema
+      .createTable(tableName)
+      .ifNotExists() // kept to show supported by all dialects in contrast to ifNotExists() below, though not needed due to hasTable() check above
+      .addColumn('id', 'varchar(255)', (col) => col.primaryKey())
+      .addColumn('task', 'text')
+      .addColumn('timeout', 'bigint')
+      .addColumn('retryCount', 'integer');
+
+    await table.execute();
+
+    await this.#db.schema
+      .createIndex('index_timeout')
+      // .ifNotExists() // intentionally kept commented out code to show that it is not supported by all dialects (ie. MySQL)
+      .on('resumableTasks')
+      .column('timeout')
+      .execute();
+  }
+
+  async close(): Promise<void> {
+    await this.#db?.destroy();
+    this.#db = null;
+  }
+
+  async register(task: any, timeoutInSeconds: number): Promise<ManagedResumableTask> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `register`.');
+    }
+
+    const id = await Cid.computeCid(task);
+    const timeout = Date.now() + timeoutInSeconds * 1000;
+    const taskString = JSON.stringify(task);
+    const retryCount = 0;
+    const taskEntryInDatabase: ManagedResumableTask = { id, task: taskString, timeout, retryCount };
+    await this.#db.insertInto('resumableTasks').values(taskEntryInDatabase).execute();
+
+    return {
+      id,
+      task,
+      retryCount,
+      timeout,
+    };
+  }
+
+  async grab(count: number): Promise<ManagedResumableTask[]> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `grab`.');
+    }
+
+    const now = Date.now();
+    const newTimeout = now + (ResumableTaskStoreSql.taskTimeoutInSeconds * 1000);
+
+    let tasks: DwnDatabaseType['resumableTasks'][] = [];
+
+    const operation = async (transaction) => {
+      tasks = await transaction
+        .selectFrom('resumableTasks')
+        .selectAll()
+        .where('timeout', '<=', now)
+        .limit(count)
+        .execute();
+
+      if (tasks.length > 0) {
+        const ids = tasks.map((task) => task.id);
+        await transaction
+          .updateTable('resumableTasks')
+          .set({ timeout: newTimeout })
+          .where((eb) => eb('id', 'in', ids))
+          .execute();
+      }
+    };
+
+    await executeWithRetryIfDatabaseIsLocked(this.#db, operation);
+
+    const tasksToReturn = tasks.map((task) => {
+      return {
+        id         : task.id,
+        task       : JSON.parse(task.task),
+        retryCount : task.retryCount,
+        timeout    : task.timeout,
+      };
+    });
+
+    return tasksToReturn;
+  }
+
+  async read(taskId: string): Promise<ManagedResumableTask | undefined> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `read`.');
+    }
+
+    const task = await this.#db
+      .selectFrom('resumableTasks')
+      .selectAll()
+      .where('id', '=', taskId)
+      .executeTakeFirst();
+
+    if (task !== undefined) {
+      // NOTE: special handling ONLY for PostgreSQL:
+      // Even though PostgreSQL stores `bigint` as a 64 bit number, the `pg` library we depend on returns it as a string, hence the conversion.
+      if (typeof task.timeout !== 'number') {
+        task.timeout = parseInt(task.timeout, 10);
+      }
+    }
+
+    return task;
+  }
+
+  async extend(taskId: string, timeoutInSeconds: number): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `extend`.');
+    }
+
+    const timeout = Date.now() + (timeoutInSeconds * 1000);
+
+    await this.#db
+      .updateTable('resumableTasks')
+      .set({ timeout })
+      .where('id', '=', taskId)
+      .execute();
+  }
+
+  async delete(taskId: string): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `delete`.');
+    }
+
+    await this.#db
+      .deleteFrom('resumableTasks')
+      .where('id', '=', taskId)
+      .execute();
+  }
+
+  async clear(): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `clear`.');
+    }
+
+    await this.#db
+      .deleteFrom('resumableTasks')
+      .execute();
+  }
+}

package/src/types.ts

@@ -0,0 +1,109 @@
+import type { Generated } from 'kysely';
+
+export type KeyValues = { [key:string]: string | number | boolean | string[] | number[] };
+
+type EventLogTable = {
+  watermark: Generated<number>;
+  tenant: string;
+  messageCid: string;
+
+  // "indexes" start
+  interface: string | null;
+  method: string | null;
+  schema: string | null;
+  dataCid: string | null;
+  dataSize: number | null;
+  dateCreated: string | null;
+  messageTimestamp: string | null;
+  dataFormat: string | null;
+  isLatestBaseState: boolean | null;
+  published: boolean | null;
+  author: string | null;
+  recordId: string | null;
+  entryId: string | null;
+  datePublished: string | null;
+  latest: string | null;
+  protocol: string | null;
+  permissionsRequestId: string | null;
+  attester: string | null;
+  protocolPath: string | null;
+  recipient: string | null;
+  contextId: string | null;
+  parentId: string | null;
+  permissionGrantId: string | null;
+  prune: boolean | null;
+  // "indexes" end
+}
+
+type MessageStoreTable = {
+  id: Generated<number>;
+  tenant: string;
+  messageCid: string;
+  encodedMessageBytes: Uint8Array;
+  encodedData: string | null;
+  // "indexes" start
+  interface: string | null;
+  method: string | null;
+  schema: string | null;
+  dataCid: string | null;
+  dataSize: number | null;
+  dateCreated: string | null;
+  messageTimestamp: string | null;
+  dataFormat: string | null;
+  isLatestBaseState: boolean | null;
+  published: boolean | null;
+  author: string | null;
+  recordId: string | null;
+  entryId: string | null;
+  datePublished: string | null;
+  protocol: string | null;
+  permissionsRequestId: string | null;
+  attester: string | null;
+  protocolPath: string | null;
+  recipient: string | null;
+  contextId: string | null;
+  parentId: string | null;
+  permissionGrantId: string | null;
+  prune: boolean | null;
+  // "indexes" end
+}
+
+type MessageStoreRecordsTagsTable = {
+  id: Generated<number>;
+  tag: string;
+  messageInsertId: number;
+  valueString: string | null;
+  valueNumber: number | null;
+};
+
+type EventLogRecordsTagsTable = {
+  id: Generated<number>;
+  tag: string;
+  eventWatermark: number;
+  valueString: string | null;
+  valueNumber: number | null;
+};
+
+type DataStoreTable = {
+  id: Generated<number>;
+  tenant: string;
+  recordId: string;
+  dataCid: string;
+  data: Uint8Array;
+}
+
+type ResumableTaskTable = {
+  id: string;
+  task: string;
+  timeout: number;
+  retryCount: number;
+}
+
+export type DwnDatabaseType = {
+  eventLogMessages: EventLogTable;
+  eventLogRecordsTags: EventLogRecordsTagsTable;
+  messageStoreMessages: MessageStoreTable;
+  messageStoreRecordsTags: MessageStoreRecordsTagsTable;
+  dataStore: DataStoreTable;
+  resumableTasks: ResumableTaskTable;
+}

package/src/utils/filter.ts

@@ -0,0 +1,136 @@
+import { Filter } from '@enbox/dwn-sdk-js';
+import { DynamicModule, ExpressionBuilder, OperandExpression, SelectQueryBuilder, SqlBool } from 'kysely';
+import { sanitizeFiltersAndSeparateTags, sanitizedValue } from './sanitize.js';
+import { DwnDatabaseType } from '../types.js';
+
+/**
+ * Takes multiple Filters and returns a single query.
+ * Each filter is evaluated as an OR operation.
+ *
+ * @param filters Array of filters to be evaluated as OR operations
+ * @param query the incoming QueryBuilder.
+ * @returns The modified QueryBuilder respecting the provided filters.
+ */
+export function filterSelectQuery<DB = DwnDatabaseType, TB extends keyof DB = keyof DB, O = unknown>(
+  filters: Filter[],
+  query: SelectQueryBuilder<DB, TB, O>
+): SelectQueryBuilder<DB, TB, O> {
+  const sanitizedFilters = sanitizeFiltersAndSeparateTags(filters);
+
+  return query.where((eb) =>
+    // evaluate the filters as an OR expression.
+    eb.or(sanitizedFilters.map(({ filter, tags }) => {
+      // evaluate each filter + tags tuple as an AND expression.
+      const andOperands: OperandExpression<SqlBool>[] = [];
+
+      processFilter(eb, andOperands, filter);
+      processTags(eb, andOperands, tags);
+
+      return eb.and(andOperands);
+    }))
+  );
+}
+
+/**
+ * Processes each property in the non-tags filter as an AND operand and adds it to the `andOperands` array.
+ * If a property has an array of values it will treat it as a OneOf (IN) within the overall AND query.
+ *
+ * @param eb The ExpressionBuilder from the query.
+ * @param andOperands The array of AND operands to append to.
+ * @param filter The filter to be evaluated.
+ */
+function processFilter<DB = DwnDatabaseType, TB extends keyof DB = keyof DB>(
+  eb: ExpressionBuilder<DB, TB>,
+  andOperands: OperandExpression<SqlBool>[],
+  filter: Filter
+): void {
+  for (let property in filter) {
+    const value = filter[property];
+    const column = new DynamicModule().ref(property);
+    if (Array.isArray(value)) { // OneOfFilter
+      andOperands.push(eb(column, 'in', value));
+    } else if (typeof value === 'object') { // RangeFilter
+      if (value.gt) {
+        andOperands.push(eb(column, '>', sanitizedValue(value.gt)));
+      }
+      if (value.gte) {
+        andOperands.push(eb(column, '>=', sanitizedValue(value.gte)));
+      }
+      if (value.lt) {
+        andOperands.push(eb(column, '<', sanitizedValue(value.lt)));
+      }
+      if (value.lte) {
+        andOperands.push(eb(column, '<=', sanitizedValue(value.lte)));
+      }
+    } else { // EqualFilter
+      andOperands.push(eb(column, '=', sanitizedValue(value)));
+    }
+  }
+}
+
+/**
+ * Processes each property in the tags filter as an AND operand and adds it to the `andOperands` array.
+ * If a property has an array of values it will treat it as a OneOf (IN) within the overall AND query.
+ *
+ * @param eb The ExpressionBuilder from the query.
+ * @param andOperands The array of AND operands to append to.
+ * @param tag The tags filter to be evaluated.
+ */
+function processTags<DB = DwnDatabaseType, TB extends keyof DB = keyof DB>(
+  eb: ExpressionBuilder<DB, TB>,
+  andOperands: OperandExpression<SqlBool>[],
+  tags: Filter
+): void {
+
+  const tagColumn = new DynamicModule().ref('tag');
+  const valueNumber = new DynamicModule().ref('valueNumber');
+  const valueString = new DynamicModule().ref('valueString');
+
+  // process each tag and add it to the andOperands from the rest of the filters
+  for (let property in tags) {
+    andOperands.push(eb(tagColumn, '=', property));
+    const value = tags[property];
+    if (Array.isArray(value)) { // OneOfFilter
+      if (value.some(val => typeof val === 'number')) {
+        andOperands.push(eb(valueNumber, 'in', value));
+      } else {
+        andOperands.push(eb(valueString, 'in', value.map(v => String(v))));
+      }
+    } else if (typeof value === 'object') { // RangeFilter
+      if (value.gt) {
+        if (typeof value.gt === 'number') {
+          andOperands.push(eb(valueNumber, '>', value.gt));
+        } else {
+          andOperands.push(eb(valueString, '>', String(value.gt)));
+        }
+      }
+      if (value.gte) {
+        if (typeof value.gte === 'number') {
+          andOperands.push(eb(valueNumber, '>=', value.gte));
+        } else {
+          andOperands.push(eb(valueString, '>=', String(value.gte)));
+        }
+      }
+      if (value.lt) {
+        if (typeof value.lt === 'number') {
+          andOperands.push(eb(valueNumber, '<', value.lt));
+        } else {
+          andOperands.push(eb(valueString, '<', String(value.lt)));
+        }
+      }
+      if (value.lte) {
+        if (typeof value.lte === 'number') {
+          andOperands.push(eb(valueNumber, '<=', value.lte));
+        } else {
+          andOperands.push(eb(valueString, '<=', String(value.lte)));
+        }
+      }
+    } else { // EqualFilter
+      if (typeof value === 'number') {
+        andOperands.push(eb(valueNumber, '=', value));
+      } else {
+        andOperands.push(eb(valueString, '=', String(value)));
+      }
+    }
+  }
+}

package/src/utils/sanitize.ts

@@ -0,0 +1,117 @@
+import { Filter } from '@enbox/dwn-sdk-js';
+import { KeyValues } from '../types.js';
+
+export function extractTagsAndSanitizeIndexes(records: KeyValues): {
+  tags: KeyValues;
+  indexes: KeyValues;
+} {
+
+  const tags = {};
+  const indexes = { ...records };
+
+  sanitizeIndexes(indexes);
+
+  // tag values are prefixed with 'tag.', we extract them to be inserted separately into the tags reference tables.
+  // we delete them from the `indexes` object so they are not included in the main insert.
+  for (let key in indexes) {
+    if (key.startsWith('tag.')) {
+      let value = indexes[key];
+      delete indexes[key];
+      tags[key.slice(4)] = value;
+    }
+  }
+
+  return { tags, indexes };
+}
+
+export function sanitizeIndexes(records: KeyValues) {
+  for (let key in records) {
+    let value = records[key];
+    if (Array.isArray(value)) {
+      const sanitizedValues: any[] = [];
+      for (const valueItem of value) {
+        sanitizedValues.push(sanitizedValue(valueItem));
+      }
+      records[key] = sanitizedValues;
+      continue;
+    }
+
+    records[key] = sanitizedValue(value);
+  }
+}
+
+/**
+ * Sanitizes the given value into a string or number.
+ * NOTE: sqlite3 we use does not support inserting boolean values, so we convert them to a number.
+ */
+export function sanitizedValue(value: string | number | boolean): string | number {
+  switch (typeof value) {
+  case 'boolean':
+    return value ? 1 : 0;
+  default:
+    return value;
+  }
+}
+
+/**
+ * Sanitizes the filters and separates tags from non-tags filter.
+ */
+export function sanitizeFiltersAndSeparateTags(filters: Filter[]): {
+  tags: Filter;
+  filter: Filter;
+}[] {
+
+  const extractedFilters: { tags: Filter, filter: Filter }[] = [];
+
+  for (const filter of filters) {
+    const tagFilter = {};
+    const nonTagFilter = {};
+
+    // tag values are prefixed with 'tag.', we extract them to be queried separately in the tags tables.
+    for (let key in filter) {
+      const value = sanitizeFilterValue(filter[key]);
+
+      if (key.startsWith('tag.')) {
+        tagFilter[key.slice(4)] = value;
+      } else {
+        nonTagFilter[key] = value;
+      }
+    }
+
+    extractedFilters.push({
+      tags   : tagFilter,
+      filter : nonTagFilter,
+    });
+  }
+
+  return extractedFilters;
+}
+
+// we sanitize the filter value for a number representation of the boolean
+
+/**
+ * Sanitizes the given filter value to align with the value conversions done during insertions/updates.
+ * NOTE: sqlite3 we use does not support inserting boolean values,
+ *       so we convert them to a number during insertions/updates, as a result we need to align the filter values in queries.
+*/
+// TODO: export filter types from `dwn-sdk-js`
+export function sanitizeFilterValue(value: any): any {
+  switch (typeof value) {
+  case 'boolean':
+    return value ? 1 : 0;
+  default:
+    return value;
+  }
+}
+
+export function sanitizeFilters(filters: Filter[]) {
+  filters.forEach(sanitizeFilter);
+}
+
+export function sanitizeFilter(filter: Filter): Filter {
+  for (let key in filter) {
+    let value = filter[key];
+    filter[key] = sanitizeFilterValue(value);
+  }
+  return filter;
+}

package/src/utils/tags.ts

@@ -0,0 +1,46 @@
+import { Transaction } from 'kysely';
+
+import type { DwnDatabaseType, KeyValues } from '../types.js';
+
+import { Dialect } from '../dialect/dialect.js';
+import { sanitizedValue } from './sanitize.js';
+
+/**
+ * Helper class to manage adding indexes for `RecordsWrite` messages which contain `tags`.
+ */
+export class TagTables {
+
+  /**
+   * @param dialect the target dialect, necessary for returning the `insertId`
+   * @param table the DB Table in order to index the tags and values in the correct tables. Choice between `messageStoreMessages` and `eventLogMessages`
+   */
+  constructor(private dialect: Dialect, private table: 'messageStoreMessages' | 'eventLogMessages'){}
+
+  /**
+   * Inserts the given tags associated with the given foreign `insertId`.
+   */
+  async executeTagsInsert(
+    foreignInsertId: number,
+    tags: KeyValues,
+    tx: Transaction<DwnDatabaseType>,
+  ):Promise<void> {
+    const tagTable = this.table === 'messageStoreMessages' ? 'messageStoreRecordsTags' : 'eventLogRecordsTags';
+    const foreignKeyReference = tagTable === 'messageStoreRecordsTags' ? { messageInsertId: foreignInsertId } : { eventWatermark: foreignInsertId };
+
+    for (const tag in tags) {
+      const tagValues = tags[tag];
+      const values = Array.isArray(tagValues) ? tagValues : [ tagValues ];
+
+      for(const value of values) {
+        const tagInsertValue = sanitizedValue(value);
+        const insertValues = {
+          tag,
+          valueNumber : typeof tagInsertValue === 'number' ? tagInsertValue : null,
+          valueString : typeof tagInsertValue === 'string' ? tagInsertValue : null,
+          ...foreignKeyReference,
+        };
+        await this.dialect.insertThenReturnId(tx, tagTable, insertValues, 'id as insertId').executeTakeFirstOrThrow();
+      }
+    }
+  }
+}

package/src/utils/transaction.ts

@@ -0,0 +1,28 @@
+import { DwnDatabaseType } from '../types.js';
+import { Kysely, Transaction } from 'kysely';
+
+/**
+ * Executes the provided transactional operation with retry if the database is locked.
+ */
+export async function executeWithRetryIfDatabaseIsLocked(
+  database: Kysely<DwnDatabaseType>,
+  operation: (transaction: Transaction<DwnDatabaseType>) => Promise<void>
+): Promise<void>{
+  let retryCount = 0;
+  let success = false;
+  while (!success) {
+    try {
+      await database.transaction().execute(operation);
+      success = true;
+    } catch (error) {
+      // if error is "database is locked", we retry the transaction
+      // this mainly happens when multiple transactions are trying to access the database at the same time in SQLite implementation.
+      if (error.code === 'SQLITE_BUSY') {
+        retryCount++;
+        console.log(`Database is locked when attempting SQL operation, retrying #${retryCount}...`);
+      } else {
+        throw error;
+      }
+    }
+  }
+}