@enbox/dwn-sql-store 0.0.7 → 0.0.8

package/src/main.ts

@@ -3,8 +3,12 @@ export * from './dialect/bun-sqlite-adapter.js';
 export * from './dialect/mysql-dialect.js';
 export * from './dialect/postgres-dialect.js';
 export * from './dialect/sqlite-dialect.js';
+export * from './blockstore-sql.js';
+export * from './data-store-s3.js';
 export * from './data-store-sql.js';
 export * from './state-index-sql.js';
 export * from './message-store-sql.js';
+export * from './migration-runner.js';
+export * from './migrations/index.js';
 export * from './resumable-task-store-sql.js';
 export * from './smt-store-sql.js';

package/src/migration-runner.ts

@@ -0,0 +1,137 @@
+import type { Dialect } from './dialect/dialect.js';
+import type { Kysely } from 'kysely';
+
+import { allMigrations } from './migrations/index.js';
+
+/**
+ * A single migration step. Migrations are TypeScript functions (not raw SQL)
+ * so they can use the Dialect abstraction for cross-dialect column types.
+ */
+export type Migration = {
+  /** Unique sequential name, e.g. '001-initial-schema'. */
+  name: string;
+  /**
+   * Apply this migration. Receives the Kysely instance and dialect for
+   * dialect-aware DDL (blob types, auto-increment, etc.).
+   */
+  up(db: Kysely<any>, dialect: Dialect): Promise<void>;
+};
+
+type MigrationRecord = {
+  name: string;
+  appliedAt: string;
+};
+
+type MigrationDatabaseType = {
+  dwn_migrations: MigrationRecord;
+};
+
+/**
+ * Minimal forward-only migration runner for dwn-sql-store.
+ *
+ * Tracks applied migrations in a `dwn_migrations` table and applies
+ * pending migrations in sequential order on each call to `run()`.
+ *
+ * Design decisions:
+ * - Forward-only: no rollback support. Keep migrations simple and additive.
+ * - TypeScript migrations: use the Dialect interface for cross-dialect DDL.
+ * - Idempotent: calling `run()` on an up-to-date database is a no-op.
+ * - Transaction per migration: each migration runs in its own transaction
+ *   so a failure leaves the database in the last known-good state.
+ */
+/**
+ * Convenience function to run all DWN store migrations against a database.
+ *
+ * Creates a `MigrationRunner` with the full set of built-in migrations and
+ * runs them. Call this once during application startup, before opening any
+ * stores — e.g. in `getDwnConfig()` or equivalent initialization code.
+ *
+ * @param db - An open Kysely instance connected to the target database.
+ * @param dialect - The dialect for the target database.
+ * @returns The names of newly applied migrations (empty if already up-to-date).
+ */
+export async function runDwnStoreMigrations(db: Kysely<any>, dialect: Dialect): Promise<string[]> {
+  const runner = new MigrationRunner(db, dialect, allMigrations);
+  return runner.run();
+}
+
+export class MigrationRunner {
+  #db: Kysely<MigrationDatabaseType>;
+  #dialect: Dialect;
+  #migrations: Migration[];
+
+  constructor(db: Kysely<any>, dialect: Dialect, migrations: Migration[]) {
+    this.#db = db as Kysely<MigrationDatabaseType>;
+    this.#dialect = dialect;
+    this.#migrations = migrations;
+  }
+
+  /**
+   * Ensure the `dwn_migrations` tracking table exists, then apply any
+   * pending migrations in order. Returns the names of newly applied migrations.
+   */
+  public async run(): Promise<string[]> {
+    await this.#ensureMigrationTable();
+
+    const applied = await this.#getAppliedMigrations();
+    const appliedSet = new Set(applied);
+    const pending = this.#migrations.filter((m) => !appliedSet.has(m.name));
+
+    const newlyApplied: string[] = [];
+    for (const migration of pending) {
+      await this.#applyMigration(migration);
+      newlyApplied.push(migration.name);
+    }
+
+    return newlyApplied;
+  }
+
+  /**
+   * Create the `dwn_migrations` table if it does not already exist.
+   */
+  async #ensureMigrationTable(): Promise<void> {
+    const exists = await this.#dialect.hasTable(this.#db, 'dwn_migrations');
+    if (exists) {
+      return;
+    }
+
+    await this.#db.schema
+      .createTable('dwn_migrations')
+      .ifNotExists()
+      .addColumn('name', 'varchar(255)', (col) => col.primaryKey().notNull())
+      .addColumn('appliedAt', 'varchar(30)', (col) => col.notNull())
+      .execute();
+  }
+
+  /**
+   * Get the list of migration names that have already been applied.
+   */
+  async #getAppliedMigrations(): Promise<string[]> {
+    const rows = await this.#db
+      .selectFrom('dwn_migrations')
+      .select('name')
+      .orderBy('name', 'asc')
+      .execute();
+
+    return rows.map((r) => r.name);
+  }
+
+  /**
+   * Apply a single migration within a transaction and record it.
+   */
+  async #applyMigration(migration: Migration): Promise<void> {
+    await this.#db.transaction().execute(async (trx) => {
+      // Run the migration
+      await migration.up(trx as unknown as Kysely<any>, this.#dialect);
+
+      // Record it as applied
+      await (trx as unknown as Kysely<MigrationDatabaseType>)
+        .insertInto('dwn_migrations')
+        .values({
+          name      : migration.name,
+          appliedAt : new Date().toISOString(),
+        })
+        .execute();
+    });
+  }
+}

package/src/migrations/001-initial-schema.ts

@@ -0,0 +1,190 @@
+import type { Dialect } from '../dialect/dialect.js';
+import type { Kysely } from 'kysely';
+import type { Migration } from '../migration-runner.js';
+
+import { sql } from 'kysely';
+
+/**
+ * Baseline migration: captures the schema as of the pre-migration era.
+ *
+ * For existing databases that already have these tables, this migration is
+ * detected as "already applied" during the adoption bootstrap (see MigrationRunner).
+ * For new databases, this creates the full initial schema.
+ */
+export const migration001InitialSchema: Migration = {
+  name: '001-initial-schema',
+
+  async up(db: Kysely<any>, dialect: Dialect): Promise<void> {
+
+    // ─── messageStoreMessages ───────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'messageStoreMessages'))) {
+      let table = db.schema
+        .createTable('messageStoreMessages')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('messageCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('interface', 'varchar(20)')
+        .addColumn('method', 'varchar(20)')
+        .addColumn('recordId', 'varchar(60)')
+        .addColumn('entryId', 'varchar(60)')
+        .addColumn('parentId', 'varchar(60)')
+        .addColumn('protocol', 'varchar(200)')
+        .addColumn('protocolPath', 'varchar(200)')
+        .addColumn('contextId', 'varchar(600)')
+        .addColumn('schema', 'varchar(200)')
+        .addColumn('author', 'varchar(255)')
+        .addColumn('recipient', 'varchar(255)')
+        .addColumn('messageTimestamp', 'varchar(30)')
+        .addColumn('dateCreated', 'varchar(30)')
+        .addColumn('datePublished', 'varchar(30)')
+        .addColumn('isLatestBaseState', 'boolean')
+        .addColumn('published', 'boolean')
+        .addColumn('prune', 'boolean')
+        .addColumn('dataFormat', 'varchar(30)')
+        .addColumn('dataCid', 'varchar(60)')
+        .addColumn('dataSize', 'integer')
+        .addColumn('encodedData', 'text')
+        .addColumn('attester', 'text')
+        .addColumn('permissionGrantId', 'varchar(60)');
+
+      table = dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
+      table = dialect.addBlobColumn(table, 'encodedMessageBytes', (col) => col.notNull());
+      await table.execute();
+
+      await db.schema.createIndex('index_tenant_messageCid')
+        .on('messageStoreMessages').columns(['tenant', 'messageCid']).unique().execute();
+
+      const indexes = [
+        ['tenant', 'recordId'],
+        ['tenant', 'entryId'],
+        ['tenant', 'parentId'],
+        ['tenant', 'protocol', 'published', 'messageTimestamp'],
+        ['tenant', 'interface'],
+        ['tenant', 'permissionGrantId'],
+        ['tenant', 'dateCreated'],
+        ['tenant', 'datePublished'],
+      ];
+
+      for (const cols of indexes) {
+        await db.schema.createIndex('index_' + cols.join('_'))
+          .on('messageStoreMessages').columns(cols).execute();
+      }
+
+      // MySQL needs prefix length for contextId
+      if (dialect.name === 'MySQL') {
+        await sql`CREATE INDEX index_tenant_contextId_messageTimestamp
+          ON ${sql.table('messageStoreMessages')} (tenant, contextId(480), messageTimestamp)`
+          .execute(db);
+      } else {
+        await db.schema.createIndex('index_tenant_contextId_messageTimestamp')
+          .on('messageStoreMessages').columns(['tenant', 'contextId', 'messageTimestamp']).execute();
+      }
+    }
+
+    // ─── messageStoreRecordsTags ─────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'messageStoreRecordsTags'))) {
+      let table = db.schema
+        .createTable('messageStoreRecordsTags')
+        .ifNotExists()
+        .addColumn('tag', 'varchar(30)', (col) => col.notNull())
+        .addColumn('valueString', 'varchar(200)')
+        .addColumn('valueNumber', 'decimal');
+
+      table = dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
+      table = dialect.addReferencedColumn(
+        table, 'messageStoreRecordsTags', 'messageInsertId', 'integer',
+        'messageStoreMessages', 'id', 'cascade'
+      );
+      await table.execute();
+
+      const tagIndexes = [
+        ['messageInsertId'],
+        ['tag', 'valueString'],
+        ['tag', 'valueNumber'],
+      ];
+      for (const cols of tagIndexes) {
+        await db.schema.createIndex('index_' + cols.join('_'))
+          .on('messageStoreRecordsTags').columns(cols).execute();
+      }
+    }
+
+    // ─── dataStore ──────────────────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'dataStore'))) {
+      let table = db.schema
+        .createTable('dataStore')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataCid', 'varchar(60)', (col) => col.notNull());
+
+      table = dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
+      table = dialect.addBlobColumn(table, 'data', (col) => col.notNull());
+      await table.execute();
+
+      await db.schema.createIndex('tenant_recordId_dataCid')
+        .on('dataStore').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+    }
+
+    // ─── resumableTasks ─────────────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'resumableTasks'))) {
+      await db.schema
+        .createTable('resumableTasks')
+        .ifNotExists()
+        .addColumn('id', 'varchar(255)', (col) => col.primaryKey())
+        .addColumn('task', 'text')
+        .addColumn('timeout', 'bigint')
+        .addColumn('retryCount', 'integer')
+        .execute();
+
+      await db.schema.createIndex('index_timeout')
+        .on('resumableTasks').column('timeout').execute();
+    }
+
+    // ─── stateIndexNodes ────────────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'stateIndexNodes'))) {
+      await db.schema
+        .createTable('stateIndexNodes')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('scope', 'varchar(200)', (col) => col.notNull())
+        .addColumn('nodeHash', 'varchar(64)', (col) => col.notNull())
+        .addColumn('nodeType', 'varchar(10)', (col) => col.notNull())
+        .addColumn('leftHash', 'varchar(64)')
+        .addColumn('rightHash', 'varchar(64)')
+        .addColumn('leafKeyHash', 'varchar(64)')
+        .addColumn('leafValueCid', 'varchar(60)')
+        .execute();
+
+      await db.schema.createIndex('index_stateIndexNodes_tenant_scope_nodeHash')
+        .on('stateIndexNodes').columns(['tenant', 'scope', 'nodeHash']).execute();
+    }
+
+    // ─── stateIndexRoots ────────────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'stateIndexRoots'))) {
+      await db.schema
+        .createTable('stateIndexRoots')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('scope', 'varchar(200)', (col) => col.notNull())
+        .addColumn('rootHash', 'varchar(64)', (col) => col.notNull())
+        .execute();
+
+      await db.schema.createIndex('index_stateIndexRoots_tenant_scope')
+        .on('stateIndexRoots').columns(['tenant', 'scope']).execute();
+    }
+
+    // ─── stateIndexMeta ─────────────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'stateIndexMeta'))) {
+      await db.schema
+        .createTable('stateIndexMeta')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('messageCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('protocol', 'varchar(200)')
+        .execute();
+
+      await db.schema.createIndex('index_stateIndexMeta_tenant_messageCid')
+        .on('stateIndexMeta').columns(['tenant', 'messageCid']).execute();
+    }
+  },
+};

package/src/migrations/002-content-addressed-datastore.ts

@@ -0,0 +1,140 @@
+import type { Dialect } from '../dialect/dialect.js';
+import type { Kysely } from 'kysely';
+import type { Migration } from '../migration-runner.js';
+
+import { sql } from 'kysely';
+
+/**
+ * Migration 002: Content-addressed DataStore with deduplication.
+ *
+ * Replaces the monolithic `dataStore` table (which stores entire blobs per
+ * tenant+recordId+dataCid) with two tables that enable whole-file dedup:
+ *
+ * - `dataRefs`: reference table linking (tenant, recordId) to a dataCid.
+ *   Multiple tenant/record pairs can reference the same dataCid. Includes
+ *   `dataSize` for efficient size queries (fixes the existing admin store bug).
+ *
+ * - `dataBlocks`: content storage table keyed by (rootDataCid, blockCid).
+ *   Stores individual ~256KB DAG-PB blocks produced by ipfs-unixfs-importer.
+ *   Content is shared across all references to the same dataCid.
+ *
+ * Data migration strategy:
+ * - For each row in the old `dataStore`, insert a ref into `dataRefs` and a
+ *   single block into `dataBlocks` with blockCid = dataCid (treating the
+ *   existing assembled blob as one block). This is a safe migration because
+ *   the new DataStoreSql code will re-chunk via ipfs-unixfs-importer on the
+ *   next write. Reads of migrated data use a fast path that detects the
+ *   single-block case and returns the data directly without the exporter.
+ *
+ * NOTE: For large databases, the data migration may take significant time.
+ * The migration runs in a single transaction for atomicity.
+ */
+export const migration002ContentAddressedDatastore: Migration = {
+  name: '002-content-addressed-datastore',
+
+  async up(db: Kysely<any>, dialect: Dialect): Promise<void> {
+
+    // ─── Create dataRefs table ──────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'dataRefs'))) {
+      await db.schema
+        .createTable('dataRefs')
+        .ifNotExists()
+        .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+        .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('dataSize', 'bigint', (col) => col.notNull())
+        .execute();
+
+      // Unique constraint: one ref per (tenant, recordId, dataCid)
+      await db.schema.createIndex('index_dataRefs_tenant_recordId_dataCid')
+        .on('dataRefs').columns(['tenant', 'recordId', 'dataCid']).unique().execute();
+
+      // Index for dataCid lookups (refcount queries, GC)
+      await db.schema.createIndex('index_dataRefs_dataCid')
+        .on('dataRefs').column('dataCid').execute();
+
+      // Index for tenant-scoped size aggregation (admin queries)
+      await db.schema.createIndex('index_dataRefs_tenant')
+        .on('dataRefs').column('tenant').execute();
+    }
+
+    // ─── Create dataBlocks table ────────────────────────────────────────
+    if (!(await dialect.hasTable(db, 'dataBlocks'))) {
+      let table = db.schema
+        .createTable('dataBlocks')
+        .ifNotExists()
+        .addColumn('rootDataCid', 'varchar(60)', (col) => col.notNull())
+        .addColumn('blockCid', 'varchar(60)', (col) => col.notNull());
+
+      table = dialect.addBlobColumn(table, 'data', (col) => col.notNull());
+      await table.execute();
+
+      // Primary-like unique index on (rootDataCid, blockCid)
+      await db.schema.createIndex('index_dataBlocks_rootDataCid_blockCid')
+        .on('dataBlocks').columns(['rootDataCid', 'blockCid']).unique().execute();
+    }
+
+    // ─── Migrate data from old dataStore table ──────────────────────────
+    const oldTableExists = await dialect.hasTable(db, 'dataStore');
+    if (oldTableExists) {
+      // Check if old table has any data to migrate
+      const countResult = await sql`SELECT COUNT(*) as cnt FROM ${sql.table('dataStore')}`
+        .execute(db);
+
+      const count = Number((countResult.rows[0] as any)?.cnt ?? 0);
+      if (count > 0) {
+        // Column references must be quoted to preserve camelCase in PostgreSQL.
+        const recordId = sql.ref('recordId');
+        const dataCid = sql.ref('dataCid');
+        const dataSize = sql.ref('dataSize');
+        const rootDataCid = sql.ref('rootDataCid');
+        const blockCid = sql.ref('blockCid');
+
+        // Migrate refs: insert into dataRefs from dataStore
+        // Use LENGTH/OCTET_LENGTH depending on dialect for blob size
+        if (dialect.name === 'SQLite') {
+          await sql`
+            INSERT INTO ${sql.table('dataRefs')} (tenant, ${recordId}, ${dataCid}, ${dataSize})
+            SELECT tenant, ${recordId}, ${dataCid}, LENGTH(data)
+            FROM ${sql.table('dataStore')}
+          `.execute(db);
+        } else {
+          // PostgreSQL uses OCTET_LENGTH for bytea, MySQL uses LENGTH for blob
+          await sql`
+            INSERT INTO ${sql.table('dataRefs')} (tenant, ${recordId}, ${dataCid}, ${dataSize})
+            SELECT tenant, ${recordId}, ${dataCid}, OCTET_LENGTH(data)
+            FROM ${sql.table('dataStore')}
+          `.execute(db);
+        }
+
+        // Migrate blocks: treat each existing blob as a single block
+        // where blockCid = dataCid (the root CID is the only block)
+        // Skip duplicates: only insert blocks for dataCids not already in dataBlocks
+        if (dialect.name === 'MySQL') {
+          await sql`
+            INSERT IGNORE INTO ${sql.table('dataBlocks')} (${rootDataCid}, ${blockCid}, data)
+            SELECT ${dataCid}, ${dataCid}, data
+            FROM ${sql.table('dataStore')}
+          `.execute(db);
+        } else if (dialect.name === 'SQLite') {
+          await sql`
+            INSERT OR IGNORE INTO ${sql.table('dataBlocks')} (${rootDataCid}, ${blockCid}, data)
+            SELECT ${dataCid}, ${dataCid}, data
+            FROM ${sql.table('dataStore')}
+          `.execute(db);
+        } else {
+          // PostgreSQL
+          await sql`
+            INSERT INTO ${sql.table('dataBlocks')} (${rootDataCid}, ${blockCid}, data)
+            SELECT ${dataCid}, ${dataCid}, data
+            FROM ${sql.table('dataStore')}
+            ON CONFLICT (${rootDataCid}, ${blockCid}) DO NOTHING
+          `.execute(db);
+        }
+      }
+
+      // Drop the old table after migration
+      await db.schema.dropTable('dataStore').execute();
+    }
+  },
+};

package/src/migrations/index.ts

@@ -0,0 +1,13 @@
+import type { Migration } from '../migration-runner.js';
+
+import { migration001InitialSchema } from './001-initial-schema.js';
+import { migration002ContentAddressedDatastore } from './002-content-addressed-datastore.js';
+
+/**
+ * All migrations in sequential order. The MigrationRunner applies them
+ * in array order, skipping any that have already been recorded.
+ */
+export const allMigrations: Migration[] = [
+  migration001InitialSchema,
+  migration002ContentAddressedDatastore,
+];

package/src/state-index-sql.ts

@@ -145,7 +145,8 @@ export class StateIndexSql implements StateIndex {
     const globalSmt = await this.getGlobalTree(tenant);
     await globalSmt.insert(messageCid);
 
-    // If the message is associated with a protocol, insert into the protocol-scoped tree
+    // Insert into the protocol-scoped tree if the message has a protocol (e.g. RecordsWrite).
+    // Non-record messages like ProtocolsConfigure do not have a protocol.
     const protocol = indexes.protocol as string | undefined;
     if (protocol !== undefined) {
       const protoSmt = await this.getProtocolTree(tenant, protocol);
@@ -186,8 +187,8 @@ export class StateIndexSql implements StateIndex {
       // Delete from global tree
       await globalSmt.delete(messageCid);
 
-      // Delete from protocol tree if applicable
-      if (meta?.protocol) {
+      // Delete from protocol tree if the message had a protocol
+      if (meta && meta.protocol !== null) {
         const protoSmt = await this.getProtocolTree(tenant, meta.protocol);
         await protoSmt.delete(messageCid);
       }

package/src/types.ts

@@ -68,6 +68,12 @@ type StateIndexMetaTable = {
   protocol: string | null;
 };
 
+// ─── DataStore tables (legacy + content-addressed) ────────────────────────
+
+/**
+ * @deprecated Legacy monolithic data table. Replaced by `dataRefs` + `dataBlocks`
+ * in migration 002. Retained for type compatibility during migration.
+ */
 type DataStoreTable = {
   id: Generated<number>;
   tenant: string;
@@ -76,6 +82,27 @@ type DataStoreTable = {
   data: Uint8Array;
 };
 
+/**
+ * Reference table linking (tenant, recordId) to a content-addressed dataCid.
+ * Multiple tenant/record pairs can reference the same dataCid for dedup.
+ */
+type DataRefsTable = {
+  tenant: string;
+  recordId: string;
+  dataCid: string;
+  dataSize: number;
+};
+
+/**
+ * Content storage table holding individual DAG-PB blocks (~256KB each)
+ * keyed by (rootDataCid, blockCid). Shared across all refs to the same dataCid.
+ */
+type DataBlocksTable = {
+  rootDataCid: string;
+  blockCid: string;
+  data: Uint8Array;
+};
+
 type ResumableTaskTable = {
   id: string;
   task: string;
@@ -87,6 +114,8 @@ export type DwnDatabaseType = {
   messageStoreMessages: MessageStoreTable;
   messageStoreRecordsTags: MessageStoreRecordsTagsTable;
   dataStore: DataStoreTable;
+  dataRefs: DataRefsTable;
+  dataBlocks: DataBlocksTable;
   resumableTasks: ResumableTaskTable;
   stateIndexNodes: StateIndexNodeTable;
   stateIndexRoots: StateIndexRootTable;