@enbox/dwn-sql-store 0.0.1 → 0.0.3

package/src/smt-store-sql.ts

@@ -0,0 +1,206 @@
+/**
+ * SQL-backed implementation of SMTNodeStore.
+ *
+ * Storage layout:
+ * - Nodes are stored in the `stateIndexNodes` table, keyed by hex-encoded hash
+ * - The root hash is stored in the `stateIndexRoots` table
+ * - Node values are serialized with Uint8Array fields encoded as hex strings
+ *
+ * Each SMTStoreSql instance represents a single tree (identified by a tenant + scope key).
+ * Multiple instances share the same underlying Kysely database connection.
+ */
+
+import type { DwnDatabaseType } from './types.js';
+import type { Kysely } from 'kysely';
+
+import type { Hash, SMTInternalNode, SMTLeafNode, SMTNode, SMTNodeStore } from '@enbox/dwn-sdk-js';
+
+import { hashToHex, hexToHash } from '@enbox/dwn-sdk-js';
+
+export type SMTStoreSqlParams = {
+  /** The shared Kysely database instance. */
+  db: Kysely<DwnDatabaseType>;
+  /** The tenant DID that owns this tree. */
+  tenant: string;
+  /** The scope key for this tree (e.g. '' for global, protocol URI for protocol-scoped). */
+  scope: string;
+};
+
+export class SMTStoreSql implements SMTNodeStore {
+  #db: Kysely<DwnDatabaseType>;
+  #tenant: string;
+  #scope: string;
+
+  constructor({ db, tenant, scope }: SMTStoreSqlParams) {
+    this.#db = db;
+    this.#tenant = tenant;
+    this.#scope = scope;
+  }
+
+  async open(): Promise<void> {
+    // No-op: the Kysely DB is already open and tables are created by StateIndexSql.
+  }
+
+  async close(): Promise<void> {
+    // No-op: the Kysely DB lifecycle is managed by StateIndexSql.
+  }
+
+  async clear(): Promise<void> {
+    await this.#db
+      .deleteFrom('stateIndexNodes')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .execute();
+
+    await this.#db
+      .deleteFrom('stateIndexRoots')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .execute();
+  }
+
+  async getNode(hash: Hash): Promise<SMTNode | undefined> {
+    const hexKey = hashToHex(hash);
+
+    const result = await this.#db
+      .selectFrom('stateIndexNodes')
+      .selectAll()
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .where('nodeHash', '=', hexKey)
+      .executeTakeFirst();
+
+    if (!result) {
+      return undefined;
+    }
+
+    return this.deserializeNode(result);
+  }
+
+  async putNode(hash: Hash, node: SMTNode): Promise<void> {
+    const hexKey = hashToHex(hash);
+    const values = this.serializeNodeToRow(hexKey, node);
+
+    // Use INSERT ... ON CONFLICT REPLACE pattern.
+    // Since different SQL dialects handle upsert differently, we use
+    // delete-then-insert as a portable approach.
+    await this.#db
+      .deleteFrom('stateIndexNodes')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .where('nodeHash', '=', hexKey)
+      .execute();
+
+    await this.#db
+      .insertInto('stateIndexNodes')
+      .values(values)
+      .execute();
+  }
+
+  async deleteNode(hash: Hash): Promise<void> {
+    const hexKey = hashToHex(hash);
+
+    await this.#db
+      .deleteFrom('stateIndexNodes')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .where('nodeHash', '=', hexKey)
+      .execute();
+  }
+
+  async getRoot(): Promise<Hash | undefined> {
+    const result = await this.#db
+      .selectFrom('stateIndexRoots')
+      .select('rootHash')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .executeTakeFirst();
+
+    if (!result) {
+      return undefined;
+    }
+
+    return hexToHash(result.rootHash);
+  }
+
+  async setRoot(hash: Hash): Promise<void> {
+    const hexRoot = hashToHex(hash);
+
+    // Delete-then-insert as a portable upsert.
+    await this.#db
+      .deleteFrom('stateIndexRoots')
+      .where('tenant', '=', this.#tenant)
+      .where('scope', '=', this.#scope)
+      .execute();
+
+    await this.#db
+      .insertInto('stateIndexRoots')
+      .values({
+        tenant   : this.#tenant,
+        scope    : this.#scope,
+        rootHash : hexRoot,
+      })
+      .execute();
+  }
+
+  // ─── Serialization helpers ──────────────────────────────────────────────
+
+  private serializeNodeToRow(hexKey: string, node: SMTNode): {
+    tenant: string;
+    scope: string;
+    nodeHash: string;
+    nodeType: string;
+    leftHash: string | null;
+    rightHash: string | null;
+    leafKeyHash: string | null;
+    leafValueCid: string | null;
+  } {
+    if (node.type === 'internal') {
+      return {
+        tenant       : this.#tenant,
+        scope        : this.#scope,
+        nodeHash     : hexKey,
+        nodeType     : 'internal',
+        leftHash     : hashToHex(node.leftHash),
+        rightHash    : hashToHex(node.rightHash),
+        leafKeyHash  : null,
+        leafValueCid : null,
+      };
+    }
+
+    return {
+      tenant       : this.#tenant,
+      scope        : this.#scope,
+      nodeHash     : hexKey,
+      nodeType     : 'leaf',
+      leftHash     : null,
+      rightHash    : null,
+      leafKeyHash  : hashToHex(node.keyHash),
+      leafValueCid : node.valueCid,
+    };
+  }
+
+  private deserializeNode(row: {
+    nodeType: string;
+    leftHash: string | null;
+    rightHash: string | null;
+    leafKeyHash: string | null;
+    leafValueCid: string | null;
+  }): SMTNode {
+    if (row.nodeType === 'internal') {
+      const node: SMTInternalNode = {
+        type      : 'internal',
+        leftHash  : hexToHash(row.leftHash!),
+        rightHash : hexToHash(row.rightHash!),
+      };
+      return node;
+    }
+
+    const node: SMTLeafNode = {
+      type     : 'leaf',
+      keyHash  : hexToHash(row.leafKeyHash!),
+      valueCid : row.leafValueCid!,
+    };
+    return node;
+  }
+}

package/src/state-index-sql.ts

@@ -0,0 +1,283 @@
+/**
+ * SQL-backed implementation of the StateIndex interface.
+ *
+ * Manages per-tenant Sparse Merkle Trees (global + per-protocol sub-trees) backed by SQL tables.
+ *
+ * Tables:
+ * - `stateIndexNodes`: stores SMT nodes (internal + leaf), keyed by (tenant, scope, nodeHash)
+ * - `stateIndexRoots`: stores the current root hash per (tenant, scope)
+ * - `stateIndexMeta`:  reverse lookup from messageCid → protocol (for deletion)
+ */
+
+import type { Dialect } from './dialect/dialect.js';
+import type { DwnDatabaseType } from './types.js';
+import type { Hash } from '@enbox/dwn-sdk-js';
+import type { KeyValues } from '@enbox/dwn-sdk-js';
+import type { StateIndex } from '@enbox/dwn-sdk-js';
+
+import { initDefaultHashes } from '@enbox/dwn-sdk-js';
+import { Kysely } from 'kysely';
+import { SMTStoreSql } from './smt-store-sql.js';
+import { SparseMerkleTree } from '@enbox/dwn-sdk-js';
+
+export class StateIndexSql implements StateIndex {
+  #dialect: Dialect;
+  #db: Kysely<DwnDatabaseType> | null = null;
+
+  /**
+   * Cache of per-tenant global SMTs. Lazily populated on first access.
+   * Stores promises to avoid race conditions when multiple concurrent operations
+   * trigger lazy initialization for the same tenant.
+   */
+  #globalTrees: Map<string, Promise<SparseMerkleTree>> = new Map();
+
+  /**
+   * Cache of per-tenant, per-protocol SMTs. Key format: `{tenant}\x00{protocol}`.
+   * Stores promises to avoid race conditions.
+   */
+  #protocolTrees: Map<string, Promise<SparseMerkleTree>> = new Map();
+
+  constructor(dialect: Dialect) {
+    this.#dialect = dialect;
+  }
+
+  async open(): Promise<void> {
+    if (this.#db) {
+      return;
+    }
+
+    this.#db = new Kysely<DwnDatabaseType>({ dialect: this.#dialect });
+
+    // Ensure default hashes are initialized for the SMT
+    await initDefaultHashes();
+
+    // ─── Create stateIndexNodes table ─────────────────────────────────────
+    const nodesTableName = 'stateIndexNodes';
+    const nodesTableExists = await this.#dialect.hasTable(this.#db, nodesTableName);
+    if (!nodesTableExists) {
+      await this.#db.schema
+        .createTable(nodesTableName)
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('scope', 'varchar(200)', (col) => col.notNull())
+        .addColumn('nodeHash', 'varchar(64)', (col) => col.notNull())
+        .addColumn('nodeType', 'varchar(10)', (col) => col.notNull())
+        .addColumn('leftHash', 'varchar(64)')
+        .addColumn('rightHash', 'varchar(64)')
+        .addColumn('leafKeyHash', 'varchar(64)')
+        .addColumn('leafValueCid', 'varchar(60)')
+        .execute();
+
+      // Not UNIQUE because the delete-then-insert upsert pattern in SMTStoreSql
+      // can race under concurrent access, causing duplicate key violations.
+      await this.#db.schema
+        .createIndex('index_stateIndexNodes_tenant_scope_nodeHash')
+        .on(nodesTableName)
+        .columns(['tenant', 'scope', 'nodeHash'])
+        .execute();
+    }
+
+    // ─── Create stateIndexRoots table ─────────────────────────────────────
+    const rootsTableName = 'stateIndexRoots';
+    const rootsTableExists = await this.#dialect.hasTable(this.#db, rootsTableName);
+    if (!rootsTableExists) {
+      await this.#db.schema
+        .createTable(rootsTableName)
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('scope', 'varchar(200)', (col) => col.notNull())
+        .addColumn('rootHash', 'varchar(64)', (col) => col.notNull())
+        .execute();
+
+      await this.#db.schema
+        .createIndex('index_stateIndexRoots_tenant_scope')
+        .on(rootsTableName)
+        .columns(['tenant', 'scope'])
+        .execute();
+    }
+
+    // ─── Create stateIndexMeta table ──────────────────────────────────────
+    const metaTableName = 'stateIndexMeta';
+    const metaTableExists = await this.#dialect.hasTable(this.#db, metaTableName);
+    if (!metaTableExists) {
+      await this.#db.schema
+        .createTable(metaTableName)
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('messageCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('protocol', 'varchar(200)')
+        .execute();
+
+      await this.#db.schema
+        .createIndex('index_stateIndexMeta_tenant_messageCid')
+        .on(metaTableName)
+        .columns(['tenant', 'messageCid'])
+        .execute();
+    }
+  }
+
+  async close(): Promise<void> {
+    this.#globalTrees.clear();
+    this.#protocolTrees.clear();
+    await this.#db?.destroy();
+    this.#db = null;
+  }
+
+  async clear(): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `clear`.');
+    }
+
+    this.#globalTrees.clear();
+    this.#protocolTrees.clear();
+
+    await this.#db.deleteFrom('stateIndexNodes').execute();
+    await this.#db.deleteFrom('stateIndexRoots').execute();
+    await this.#db.deleteFrom('stateIndexMeta').execute();
+  }
+
+  async insert(tenant: string, messageCid: string, indexes: KeyValues): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `insert`.');
+    }
+
+    // Insert into the global tree
+    const globalSmt = await this.getGlobalTree(tenant);
+    await globalSmt.insert(messageCid);
+
+    // If the message is associated with a protocol, insert into the protocol-scoped tree
+    const protocol = indexes.protocol as string | undefined;
+    if (protocol !== undefined) {
+      const protoSmt = await this.getProtocolTree(tenant, protocol);
+      await protoSmt.insert(messageCid);
+    }
+
+    // Store reverse lookup metadata for deletion
+    await this.#db
+      .insertInto('stateIndexMeta')
+      .values({
+        tenant,
+        messageCid,
+        protocol: protocol ?? null,
+      })
+      .execute();
+  }
+
+  async delete(tenant: string, messageCids: string[]): Promise<void> {
+    if (!this.#db) {
+      throw new Error('Connection to database not open. Call `open` before using `delete`.');
+    }
+
+    if (messageCids.length === 0) {
+      return;
+    }
+
+    const globalSmt = await this.getGlobalTree(tenant);
+
+    for (const messageCid of messageCids) {
+      // Look up stored metadata to find the protocol
+      const meta = await this.#db
+        .selectFrom('stateIndexMeta')
+        .select('protocol')
+        .where('tenant', '=', tenant)
+        .where('messageCid', '=', messageCid)
+        .executeTakeFirst();
+
+      // Delete from global tree
+      await globalSmt.delete(messageCid);
+
+      // Delete from protocol tree if applicable
+      if (meta?.protocol) {
+        const protoSmt = await this.getProtocolTree(tenant, meta.protocol);
+        await protoSmt.delete(messageCid);
+      }
+
+      // Remove the reverse lookup
+      await this.#db
+        .deleteFrom('stateIndexMeta')
+        .where('tenant', '=', tenant)
+        .where('messageCid', '=', messageCid)
+        .execute();
+    }
+  }
+
+  async getRoot(tenant: string): Promise<Hash> {
+    const smt = await this.getGlobalTree(tenant);
+    return smt.getRoot();
+  }
+
+  async getProtocolRoot(tenant: string, protocol: string): Promise<Hash> {
+    const smt = await this.getProtocolTree(tenant, protocol);
+    return smt.getRoot();
+  }
+
+  async getSubtreeHash(tenant: string, prefix: boolean[]): Promise<Hash> {
+    const smt = await this.getGlobalTree(tenant);
+    return smt.getSubtreeHash(prefix);
+  }
+
+  async getProtocolSubtreeHash(tenant: string, protocol: string, prefix: boolean[]): Promise<Hash> {
+    const smt = await this.getProtocolTree(tenant, protocol);
+    return smt.getSubtreeHash(prefix);
+  }
+
+  async getLeaves(tenant: string, prefix: boolean[]): Promise<string[]> {
+    const smt = await this.getGlobalTree(tenant);
+    return smt.getLeaves(prefix);
+  }
+
+  async getProtocolLeaves(tenant: string, protocol: string, prefix: boolean[]): Promise<string[]> {
+    const smt = await this.getProtocolTree(tenant, protocol);
+    return smt.getLeaves(prefix);
+  }
+
+  // ─── Private helpers ────────────────────────────────────────────────────
+
+  /**
+   * Get or create the global SMT for a tenant.
+   * Uses a promise-based cache to prevent concurrent callers from racing.
+   */
+  private getGlobalTree(tenant: string): Promise<SparseMerkleTree> {
+    let smtPromise = this.#globalTrees.get(tenant);
+    if (smtPromise !== undefined) {
+      return smtPromise;
+    }
+
+    smtPromise = this.createTree(tenant, '');
+    this.#globalTrees.set(tenant, smtPromise);
+    return smtPromise;
+  }
+
+  /**
+   * Get or create a protocol-scoped SMT for a tenant.
+   * Uses a promise-based cache to prevent concurrent callers from racing.
+   */
+  private getProtocolTree(tenant: string, protocol: string): Promise<SparseMerkleTree> {
+    const cacheKey = `${tenant}\x00${protocol}`;
+    let smtPromise = this.#protocolTrees.get(cacheKey);
+    if (smtPromise !== undefined) {
+      return smtPromise;
+    }
+
+    smtPromise = this.createTree(tenant, protocol);
+    this.#protocolTrees.set(cacheKey, smtPromise);
+    return smtPromise;
+  }
+
+  /**
+   * Create and initialize a new SparseMerkleTree backed by SQL via SMTStoreSql.
+   */
+  private async createTree(tenant: string, scope: string): Promise<SparseMerkleTree> {
+    const store = new SMTStoreSql({
+      db: this.#db!,
+      tenant,
+      scope,
+    });
+    const smt = new SparseMerkleTree(store);
+    await smt.initialize();
+    return smt;
+  }
+
+}
+
+

package/src/types.ts

@@ -1,39 +1,7 @@
 import type { Generated } from 'kysely';
+import type { KeyValues } from '@enbox/dwn-sdk-js';
 
-export type KeyValues = { [key:string]: string | number | boolean | string[] | number[] };
-
-type EventLogTable = {
-  watermark: Generated<number>;
-  tenant: string;
-  messageCid: string;
-
-  // "indexes" start
-  interface: string | null;
-  method: string | null;
-  schema: string | null;
-  dataCid: string | null;
-  dataSize: number | null;
-  dateCreated: string | null;
-  messageTimestamp: string | null;
-  dataFormat: string | null;
-  isLatestBaseState: boolean | null;
-  published: boolean | null;
-  author: string | null;
-  recordId: string | null;
-  entryId: string | null;
-  datePublished: string | null;
-  latest: string | null;
-  protocol: string | null;
-  permissionsRequestId: string | null;
-  attester: string | null;
-  protocolPath: string | null;
-  recipient: string | null;
-  contextId: string | null;
-  parentId: string | null;
-  permissionGrantId: string | null;
-  prune: boolean | null;
-  // "indexes" end
-}
+export type { KeyValues };
 
 type MessageStoreTable = {
   id: Generated<number>;
@@ -57,7 +25,6 @@ type MessageStoreTable = {
   entryId: string | null;
   datePublished: string | null;
   protocol: string | null;
-  permissionsRequestId: string | null;
   attester: string | null;
   protocolPath: string | null;
   recipient: string | null;
@@ -66,7 +33,7 @@ type MessageStoreTable = {
   permissionGrantId: string | null;
   prune: boolean | null;
   // "indexes" end
-}
+};
 
 type MessageStoreRecordsTagsTable = {
   id: Generated<number>;
@@ -76,12 +43,29 @@ type MessageStoreRecordsTagsTable = {
   valueNumber: number | null;
 };
 
-type EventLogRecordsTagsTable = {
-  id: Generated<number>;
-  tag: string;
-  eventWatermark: number;
-  valueString: string | null;
-  valueNumber: number | null;
+// ─── StateIndex SMT tables ────────────────────────────────────────────────
+
+type StateIndexNodeTable = {
+  tenant: string;
+  scope: string;
+  nodeHash: string;
+  nodeType: string;
+  leftHash: string | null;
+  rightHash: string | null;
+  leafKeyHash: string | null;
+  leafValueCid: string | null;
+};
+
+type StateIndexRootTable = {
+  tenant: string;
+  scope: string;
+  rootHash: string;
+};
+
+type StateIndexMetaTable = {
+  tenant: string;
+  messageCid: string;
+  protocol: string | null;
 };
 
 type DataStoreTable = {
@@ -90,20 +74,21 @@ type DataStoreTable = {
   recordId: string;
   dataCid: string;
   data: Uint8Array;
-}
+};
 
 type ResumableTaskTable = {
   id: string;
   task: string;
   timeout: number;
   retryCount: number;
-}
+};
 
 export type DwnDatabaseType = {
-  eventLogMessages: EventLogTable;
-  eventLogRecordsTags: EventLogRecordsTagsTable;
   messageStoreMessages: MessageStoreTable;
   messageStoreRecordsTags: MessageStoreRecordsTagsTable;
   dataStore: DataStoreTable;
   resumableTasks: ResumableTaskTable;
-}
+  stateIndexNodes: StateIndexNodeTable;
+  stateIndexRoots: StateIndexRootTable;
+  stateIndexMeta: StateIndexMetaTable;
+};

package/src/utils/filter.ts

@@ -1,7 +1,9 @@
-import { Filter } from '@enbox/dwn-sdk-js';
-import { DynamicModule, ExpressionBuilder, OperandExpression, SelectQueryBuilder, SqlBool } from 'kysely';
-import { sanitizeFiltersAndSeparateTags, sanitizedValue } from './sanitize.js';
-import { DwnDatabaseType } from '../types.js';
+import type { DwnDatabaseType } from '../types.js';
+import type { Filter } from '@enbox/dwn-sdk-js';
+import type { ExpressionBuilder, OperandExpression, SelectQueryBuilder, SqlBool } from 'kysely';
+
+import { DynamicModule } from 'kysely';
+import { sanitizedValue, sanitizeFiltersAndSeparateTags } from './sanitize.js';
 
 /**
  * Takes multiple Filters and returns a single query.
@@ -44,7 +46,7 @@ function processFilter<DB = DwnDatabaseType, TB extends keyof DB = keyof DB>(
   andOperands: OperandExpression<SqlBool>[],
   filter: Filter
 ): void {
-  for (let property in filter) {
+  for (const property in filter) {
     const value = filter[property];
     const column = new DynamicModule().ref(property);
     if (Array.isArray(value)) { // OneOfFilter
@@ -87,7 +89,7 @@ function processTags<DB = DwnDatabaseType, TB extends keyof DB = keyof DB>(
   const valueString = new DynamicModule().ref('valueString');
 
   // process each tag and add it to the andOperands from the rest of the filters
-  for (let property in tags) {
+  for (const property in tags) {
     andOperands.push(eb(tagColumn, '=', property));
     const value = tags[property];
     if (Array.isArray(value)) { // OneOfFilter