@enbox/dwn-sql-store 0.0.1 → 0.0.3

package/src/data-store-sql.ts

@@ -1,8 +1,9 @@
-import { DataStore, DataStream, DataStoreGetResult, DataStorePutResult } from '@enbox/dwn-sdk-js';
+import type { Dialect } from './dialect/dialect.js';
+import type { DwnDatabaseType } from './types.js';
+import type { DataStore, DataStoreGetResult, DataStorePutResult } from '@enbox/dwn-sdk-js';
+
+import { DataStream } from '@enbox/dwn-sdk-js';
 import { Kysely } from 'kysely';
-import { Readable } from 'readable-stream';
-import { DwnDatabaseType } from './types.js';
-import { Dialect } from './dialect/dialect.js';
 
 export class DataStoreSql implements DataStore {
   #dialect: Dialect;
@@ -78,12 +79,13 @@ export class DataStoreSql implements DataStore {
       return undefined;
     }
 
+    const dataBytes = new Uint8Array(result.data);
     return {
       dataSize   : result.data.length,
-      dataStream : new Readable({
-        read() {
-          this.push(Buffer.from(result.data));
-          this.push(null);
+      dataStream : new ReadableStream<Uint8Array>({
+        start(controller): void {
+          controller.enqueue(dataBytes);
+          controller.close();
         }
       }),
     };
@@ -93,7 +95,7 @@ export class DataStoreSql implements DataStore {
     tenant: string,
     recordId: string,
     dataCid: string,
-    dataStream: Readable
+    dataStream: ReadableStream<Uint8Array>
   ): Promise<DataStorePutResult> {
     if (!this.#db) {
       throw new Error(

package/src/dialect/bun-sqlite-adapter.ts

@@ -0,0 +1,82 @@
+/**
+ * Adapter that wraps Bun's built-in SQLite (`bun:sqlite`) to conform to Kysely's
+ * `SqliteDatabase` / `SqliteStatement` interfaces, enabling it as a drop-in
+ * replacement for `better-sqlite3`.
+ */
+import { Database as BunDatabase } from 'bun:sqlite';
+
+/**
+ * Matches Kysely's SqliteStatement interface.
+ */
+interface KyselySqliteStatement {
+  readonly reader: boolean;
+  all(parameters: ReadonlyArray<unknown>): unknown[];
+  run(parameters: ReadonlyArray<unknown>): {
+    changes: number | bigint;
+    lastInsertRowid: number | bigint;
+  };
+}
+
+/**
+ * Matches Kysely's SqliteDatabase interface.
+ */
+interface KyselySqliteDatabase {
+  close(): void;
+  prepare(sql: string): KyselySqliteStatement;
+}
+
+/** SQL command prefixes that indicate a read/query operation. */
+const READER_PREFIXES = /^\s*(SELECT|PRAGMA|EXPLAIN|WITH)\b/i;
+
+/**
+ * Detects `RETURNING` clause in DML statements (INSERT/UPDATE/DELETE ... RETURNING).
+ * These produce rows like a SELECT, so the statement must use `all()` not `run()`.
+ */
+const HAS_RETURNING = /\bRETURNING\b/i;
+
+/**
+ * Creates a Kysely-compatible SQLite database backed by `bun:sqlite`.
+ *
+ * @param path - File path or `":memory:"` for in-memory database.
+ * @param options - Options forwarded to `bun:sqlite`'s Database constructor.
+ *   - `readonly`:  Open in read-only mode.
+ *   - `create`:    Create the file if it doesn't exist (default: true).
+ * @returns An object implementing Kysely's `SqliteDatabase` interface.
+ */
+export function createBunSqliteDatabase(
+  path: string,
+  options?: { readonly?: boolean; create?: boolean },
+): KyselySqliteDatabase {
+  const db = new BunDatabase(path, options);
+
+  return {
+    close(): void {
+      db.close();
+    },
+
+    prepare(sql: string): KyselySqliteStatement {
+      const stmt = db.prepare(sql);
+      const isReader = READER_PREFIXES.test(sql) || HAS_RETURNING.test(sql);
+
+      return {
+        get reader(): boolean {
+          return isReader;
+        },
+
+        all(parameters: ReadonlyArray<unknown>): unknown[] {
+          return stmt.all(...(parameters as any[]));
+        },
+
+        run(parameters: ReadonlyArray<unknown>): {
+          changes: number | bigint;
+          lastInsertRowid: number | bigint;
+        } {
+          return stmt.run(...(parameters as any[])) as {
+            changes: number | bigint;
+            lastInsertRowid: number | bigint;
+          };
+        },
+      };
+    },
+  };
+}

package/src/dialect/dialect.ts

@@ -1,13 +1,13 @@
-import {
+import type {
   ColumnBuilderCallback,
   ColumnDataType,
   CreateTableBuilder,
-  Dialect as KyselyDialect,
-  Kysely,
   InsertObject,
   InsertQueryBuilder,
-  Selection,
+  Kysely,
+  Dialect as KyselyDialect,
   SelectExpression,
+  Selection,
   Transaction,
 } from 'kysely';
 
@@ -64,7 +64,6 @@ export interface Dialect extends KyselyDialect {
    *
    *  NOTE: the `returning` value must be formatted to return an insertId value.
    *  ex. if the generated key is `id` the string should be `id as insertId`.
-   *      if the generated key is `watermark` the string should be `watermark as insertId`.
    *
    * @returns {InsertQueryBuilder} object to further modify the query or execute it.
    */

package/src/dialect/mysql-dialect.ts

@@ -1,16 +1,18 @@
-import { Dialect } from './dialect.js';
-import {
+import type { Dialect } from './dialect.js';
+import type {
   AnyColumn,
+  ColumnBuilderCallback,
   ColumnDataType,
   CreateTableBuilder,
-  ColumnBuilderCallback,
   InsertObject,
   InsertQueryBuilder,
+  Kysely,
   SelectExpression,
   Selection,
-  Transaction,
-  MysqlDialect as KyselyMysqlDialect,
-  Kysely,
+  Transaction } from 'kysely';
+
+import {
+  MysqlDialect as KyselyMysqlDialect
 } from 'kysely';
 
 export class MysqlDialect extends KyselyMysqlDialect implements Dialect {

package/src/dialect/postgres-dialect.ts

@@ -1,15 +1,17 @@
-import { Dialect } from './dialect.js';
-import {
-  ColumnDataType,
+import type { Dialect } from './dialect.js';
+import type {
   ColumnBuilderCallback,
+  ColumnDataType,
   CreateTableBuilder,
   InsertObject,
   InsertQueryBuilder,
   Kysely,
-  PostgresDialect as KyselyPostgresDialect,
   SelectExpression,
   Selection,
-  Transaction,
+  Transaction } from 'kysely';
+
+import {
+  PostgresDialect as KyselyPostgresDialect
 } from 'kysely';
 
 export class PostgresDialect extends KyselyPostgresDialect implements Dialect {
@@ -51,7 +53,10 @@ export class PostgresDialect extends KyselyPostgresDialect implements Dialect {
     referenceColumnName: string,
     onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
   ): CreateTableBuilder<TB & string> {
-    return builder.addColumn(columnName, columnType, (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction));
+    return builder.addColumn(
+      columnName, columnType,
+      (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction),
+    );
   }
 
   insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = any>(

package/src/dialect/sqlite-dialect.ts

@@ -1,15 +1,17 @@
-import { Dialect } from './dialect.js';
-import {
+import type { Dialect } from './dialect.js';
+import type {
   ColumnBuilderCallback,
   ColumnDataType,
   CreateTableBuilder,
-  Kysely,
   InsertObject,
   InsertQueryBuilder,
+  Kysely,
   SelectExpression,
   Selection,
-  SqliteDialect as KyselySqliteDialect,
-  Transaction,
+  Transaction } from 'kysely';
+
+import {
+  SqliteDialect as KyselySqliteDialect
 } from 'kysely';
 
 export class SqliteDialect extends KyselySqliteDialect implements Dialect {
@@ -58,7 +60,10 @@ export class SqliteDialect extends KyselySqliteDialect implements Dialect {
     referenceColumnName: string,
     onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
   ): CreateTableBuilder<TB & string> {
-    return builder.addColumn(columnName, columnType, (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction));
+    return builder.addColumn(
+      columnName, columnType,
+      (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction),
+    );
   }
 
   insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = any>(

package/src/main.ts

@@ -1,8 +1,10 @@
 export * from './dialect/dialect.js';
+export * from './dialect/bun-sqlite-adapter.js';
 export * from './dialect/mysql-dialect.js';
 export * from './dialect/postgres-dialect.js';
 export * from './dialect/sqlite-dialect.js';
 export * from './data-store-sql.js';
-export * from './event-log-sql.js';
+export * from './state-index-sql.js';
 export * from './message-store-sql.js';
-export * from './resumable-task-store-sql.js';
+export * from './resumable-task-store-sql.js';
+export * from './smt-store-sql.js';

package/src/message-store-sql.ts

@@ -1,27 +1,29 @@
-import {
-  DwnInterfaceName,
-  DwnMethodName,
-  executeUnlessAborted,
+import type { Dialect } from './dialect/dialect.js';
+import type { Transaction } from 'kysely';
+import type { DwnDatabaseType, KeyValues } from './types.js';
+import type {
   Filter,
   GenericMessage,
+  MessageSort,
   MessageStore,
   MessageStoreOptions,
-  MessageSort,
   Pagination,
-  SortDirection,
-  PaginationCursor,
-} from '@enbox/dwn-sdk-js';
+  PaginationCursor } from '@enbox/dwn-sdk-js';
 
-import { Kysely, Transaction } from 'kysely';
-import { DwnDatabaseType, KeyValues } from './types.js';
 import * as block from 'multiformats/block';
 import * as cbor from '@ipld/dag-cbor';
-import { Dialect } from './dialect/dialect.js';
-import { executeWithRetryIfDatabaseIsLocked } from './utils/transaction.js';
+import { executeWithTransaction } from './utils/transaction.js';
 import { extractTagsAndSanitizeIndexes } from './utils/sanitize.js';
 import { filterSelectQuery } from './utils/filter.js';
 import { sha256 } from 'multiformats/hashes/sha2';
 import { TagTables } from './utils/tags.js';
+import {
+  DwnInterfaceName,
+  DwnMethodName,
+  executeUnlessAborted,
+  SortDirection
+} from '@enbox/dwn-sdk-js';
+import { Kysely, sql } from 'kysely';
 
 
 export class MessageStoreSql implements MessageStore {
@@ -31,7 +33,7 @@ export class MessageStoreSql implements MessageStore {
 
   constructor(dialect: Dialect) {
     this.#dialect = dialect;
-    this.#tags = new TagTables(dialect, 'messageStoreMessages');
+    this.#tags = new TagTables(dialect);
   }
 
   async open(): Promise<void> {
@@ -57,7 +59,7 @@ export class MessageStoreSql implements MessageStore {
         .addColumn('parentId', 'varchar(60)')
         .addColumn('protocol', 'varchar(200)')
         .addColumn('protocolPath', 'varchar(200)')
-        .addColumn('contextId', 'varchar(500)')
+        .addColumn('contextId', 'varchar(600)')
         .addColumn('schema', 'varchar(200)')
         .addColumn('author', 'varchar(255)')
         .addColumn('recipient', 'varchar(255)')
@@ -72,32 +74,48 @@ export class MessageStoreSql implements MessageStore {
         .addColumn('dataSize', 'integer')
         .addColumn('encodedData', 'text') // we optionally store encoded data if it is below a threshold
         .addColumn('attester', 'text')
-        .addColumn('permissionGrantId', 'varchar(60)')
-        .addColumn('latest', 'text'); // TODO: obsolete, remove once `dwn-sdk-js` tests are updated
+        .addColumn('permissionGrantId', 'varchar(60)');
 
       // Add columns that have dialect-specific constraints
       createMessagesTable = this.#dialect.addAutoIncrementingColumn(createMessagesTable, 'id', (col) => col.primaryKey());
       createMessagesTable = this.#dialect.addBlobColumn(createMessagesTable, 'encodedMessageBytes', (col) => col.notNull());
       await createMessagesTable.execute();
 
+      // add unique index for get() and delete() by messageCid — the most fundamental lookup path
+      await this.#db.schema
+        .createIndex('index_tenant_messageCid')
+        .on(messagesTableName)
+        .columns(['tenant', 'messageCid'])
+        .unique()
+        .execute();
+
       // add indexes to the table
       await this.createIndexes(this.#db, messagesTableName, [
-        ['tenant'], // baseline protection to prevent full table scans across all tenants
         ['tenant', 'recordId'], // multiple uses, notably heavily depended by record chain construction for protocol authorization
+        ['tenant', 'entryId'], // used by fetchInitialRecordsWriteMessage in RecordsRead, RecordsQuery, and RecordsDelete
         ['tenant', 'parentId'], // used to walk down hierarchy of records, use cases include purging of records
-        ['tenant', 'protocol', 'published', 'messageTimestamp'], // index used for basically every external query.
+        ['tenant', 'protocol', 'published', 'messageTimestamp'], // index used for basically every external query
         ['tenant', 'interface'], // mainly for fast fetch of ProtocolsConfigure for authorization, not needed if protocol was a DWN Record
-        ['tenant', 'contextId', 'messageTimestamp'], // expected to be used for common query pattern
-        ['tenant', 'permissionGrantId'], // for deleting grant-authorized messages though pending https://github.com/TBD54566975/dwn-sdk-js/issues/716
-        // other potential indexes
-        // ['tenant', 'author'],
-        // ['tenant', 'recipient'],
-        // ['tenant', 'schema', 'dataFormat'],
-        // ['tenant', 'dateCreated'],
-        // ['tenant', 'datePublished'],
-        // ['tenant', 'messageCid'],
-        // ['tenant', 'protocolPath'],
+        ['tenant', 'permissionGrantId'], // for deleting grant-authorized messages though pending https://github.com/enboxorg/enbox/issues/716
+        ['tenant', 'dateCreated'], // sort optimization for RecordsQuery with DateSort.CreatedAscending/Descending
+        ['tenant', 'datePublished'], // sort optimization for RecordsQuery with DateSort.PublishedAscending/Descending
       ]);
+
+      // contextId index created separately because MySQL requires a prefix length to fit within
+      // the 3072-byte InnoDB index key limit. contextId is varchar(600) × 4 bytes (utf8mb4) = 2400 bytes,
+      // which combined with tenant (255 × 4 = 1020) and messageTimestamp (30 × 4 = 120) = 3540 bytes,
+      // exceeding the limit. A prefix of 480 chars (1920 bytes) brings the total to 3060 bytes.
+      // contextId values only contain ASCII chars [a-zA-Z0-9/], so a 480-char prefix is sufficient
+      // to distinguish most records (covers ~8 nesting levels of 59-char CID segments).
+      if (this.#dialect.name === 'MySQL') {
+        await sql`CREATE INDEX index_tenant_contextId_messageTimestamp
+          ON ${sql.table(messagesTableName)} (tenant, contextId(480), messageTimestamp)`
+          .execute(this.#db);
+      } else {
+        await this.createIndexes(this.#db, messagesTableName, [
+          ['tenant', 'contextId', 'messageTimestamp'], // expected to be used for common query pattern
+        ]);
+      }
     }
 
     // create tags table
@@ -169,19 +187,19 @@ export class MessageStoreSql implements MessageStore {
     const getEncodedData = (message: GenericMessage): { message: GenericMessage, encodedData: string|null} => {
       let encodedData: string|null = null;
       if (message.descriptor.interface === DwnInterfaceName.Records && message.descriptor.method === DwnMethodName.Write) {
-        const data = (message as any).encodedData as string|undefined;
-        if(data) {
-          delete (message as any).encodedData;
+        const data = message.encodedData;
+        if (data) {
+          delete message.encodedData;
           encodedData = data;
         }
       }
       return { message, encodedData };
     };
 
-    const { message: messageToProcess, encodedData} = getEncodedData(message);
+    const { message: messageToProcess, encodedData } = getEncodedData(message);
 
     const encodedMessageBlock = await executeUnlessAborted(
-      block.encode({ value: messageToProcess, codec: cbor, hasher: sha256}),
+      block.encode({ value: messageToProcess, codec: cbor, hasher: sha256 }),
       options?.signal
     );
 
@@ -192,7 +210,7 @@ export class MessageStoreSql implements MessageStore {
     // if any of these inserts would throw, the whole transaction would be rolled back.
     // otherwise it is committed.
     const putMessageOperation = this.constructPutMessageOperation({ tenant, messageCid, encodedMessageBytes, encodedData, indexes });
-    await executeWithRetryIfDatabaseIsLocked(this.#db, putMessageOperation);
+    await executeWithTransaction(this.#db, putMessageOperation);
   }
 
   /**
@@ -297,9 +315,9 @@ export class MessageStoreSql implements MessageStore {
     // filter sanitization takes place within `filterSelectQuery`
     query = filterSelectQuery(filters, query);
 
-    if(pagination?.cursor !== undefined) {
+    if (pagination?.cursor !== undefined) {
       // currently the sort property is explicitly either `dateCreated` | `messageTimestamp` | `datePublished` which are all strings
-      // TODO: https://github.com/TBD54566975/dwn-sdk-js/issues/664 to handle the edge case
+      // TODO: https://github.com/enboxorg/enbox/issues/664 to handle the edge case
       const cursorValue = pagination.cursor.value as string;
       const cursorMessageId = pagination.cursor.messageCid;
 
@@ -312,7 +330,7 @@ export class MessageStoreSql implements MessageStore {
 
     const orderDirection = sortDirection === SortDirection.Ascending ? 'asc' : 'desc';
     // sorting by the provided sort property, the tiebreak is always in ascending order regardless of sort
-    query =  query
+    query = query
       .orderBy(sortProperty, orderDirection)
       .orderBy('messageCid', orderDirection);
 
@@ -331,6 +349,33 @@ export class MessageStoreSql implements MessageStore {
     return this.processPaginationResults(results, sortProperty, pagination?.limit, options);
   }
 
+  async count(
+    tenant: string,
+    filters: Filter[],
+    messageSort?: MessageSort,
+    options?: MessageStoreOptions
+  ): Promise<number> {
+    if (!this.#db) {
+      throw new Error(
+        'Connection to database not open. Call `open` before using `count`.'
+      );
+    }
+
+    options?.signal?.throwIfAborted();
+
+    let query = this.#db
+      .selectFrom('messageStoreMessages')
+      .leftJoin('messageStoreRecordsTags', 'messageStoreRecordsTags.messageInsertId', 'messageStoreMessages.id')
+      .select(sql<number>`count(distinct ${sql.ref('messageStoreMessages.messageCid')})`.as('count'))
+      .where('tenant', '=', tenant);
+
+    query = filterSelectQuery(filters, query);
+
+    const result = await executeUnlessAborted(query.executeTakeFirstOrThrow(), options?.signal);
+
+    return Number(result.count);
+  }
+
   async delete(
     tenant: string,
     cid: string,
@@ -382,9 +427,9 @@ export class MessageStoreSql implements MessageStore {
     const message = decodedBlock.value as GenericMessage;
     // If encodedData is stored within the MessageStore we include it in the response.
     // We store encodedData when the data is below a certain threshold.
-    // https://github.com/TBD54566975/dwn-sdk-js/pull/456
+    // https://github.com/enboxorg/enbox/pull/456
     if (message !== undefined && encodedData !== undefined && encodedData !== null) {
-      (message as any).encodedData = encodedData;
+      message.encodedData = encodedData;
     }
     return message;
   }
@@ -427,14 +472,14 @@ export class MessageStoreSql implements MessageStore {
   private extractSortProperties(
     messageSort?: MessageSort
   ):{ property: 'dateCreated' | 'datePublished' | 'messageTimestamp', direction: SortDirection } {
-    if(messageSort?.dateCreated !== undefined)  {
-      return  { property: 'dateCreated', direction: messageSort.dateCreated };
-    } else if(messageSort?.datePublished !== undefined) {
-      return  { property: 'datePublished', direction: messageSort.datePublished };
+    if (messageSort?.dateCreated !== undefined) {
+      return { property: 'dateCreated', direction: messageSort.dateCreated };
+    } else if (messageSort?.datePublished !== undefined) {
+      return { property: 'datePublished', direction: messageSort.datePublished };
     } else if (messageSort?.messageTimestamp !== undefined) {
-      return  { property: 'messageTimestamp', direction: messageSort.messageTimestamp };
+      return { property: 'messageTimestamp', direction: messageSort.messageTimestamp };
     } else {
-      return  { property: 'messageTimestamp', direction: SortDirection.Ascending };
+      return { property: 'messageTimestamp', direction: SortDirection.Ascending };
     }
   }
 }

package/src/resumable-task-store-sql.ts

@@ -1,8 +1,10 @@
-import { DwnDatabaseType } from './types.js';
-import { Dialect } from './dialect/dialect.js';
-import { executeWithRetryIfDatabaseIsLocked } from './utils/transaction.js';
+import type { Dialect } from './dialect/dialect.js';
+import type { DwnDatabaseType } from './types.js';
+import type { ManagedResumableTask, ResumableTaskStore } from '@enbox/dwn-sdk-js';
+
+import { Cid } from '@enbox/dwn-sdk-js';
+import { executeWithTransaction } from './utils/transaction.js';
 import { Kysely } from 'kysely';
-import { Cid, ManagedResumableTask, ResumableTaskStore } from '@enbox/dwn-sdk-js';
 
 export class ResumableTaskStoreSql implements ResumableTaskStore {
   private static readonly taskTimeoutInSeconds = 60;
@@ -30,7 +32,7 @@ export class ResumableTaskStoreSql implements ResumableTaskStore {
 
     // else create the table and corresponding indexes
 
-    let table = this.#db.schema
+    const table = this.#db.schema
       .createTable(tableName)
       .ifNotExists() // kept to show supported by all dialects in contrast to ifNotExists() below, though not needed due to hasTable() check above
       .addColumn('id', 'varchar(255)', (col) => col.primaryKey())
@@ -83,7 +85,7 @@ export class ResumableTaskStoreSql implements ResumableTaskStore {
 
     let tasks: DwnDatabaseType['resumableTasks'][] = [];
 
-    const operation = async (transaction) => {
+    const operation = async (transaction): Promise<void> => {
       tasks = await transaction
         .selectFrom('resumableTasks')
         .selectAll()
@@ -101,7 +103,7 @@ export class ResumableTaskStoreSql implements ResumableTaskStore {
       }
     };
 
-    await executeWithRetryIfDatabaseIsLocked(this.#db, operation);
+    await executeWithTransaction(this.#db, operation);
 
     const tasksToReturn = tasks.map((task) => {
       return {