@enbox/dwn-server 0.0.2 → 0.0.4

package/src/storage.ts

@@ -1,43 +1,47 @@
+import type { Dialect } from '@enbox/dwn-sql-store';
 import type { DidResolver } from '@enbox/dids';
+import type { DwnServerConfig } from './config.js';
 import type {
   DataStore,
   DwnConfig,
   EventLog,
-  EventStream,
   MessageStore,
   ResumableTaskStore,
+  StateIndex,
   TenantGate,
 } from '@enbox/dwn-sdk-js';
-import type { Dialect } from '@enbox/dwn-sql-store';
-import type { DwnServerConfig } from './config.js';
 
 import * as fs from 'fs';
 import Cursor from 'pg-cursor';
-import Database from 'better-sqlite3';
-import pg from 'pg';
 import { createPool as MySQLCreatePool } from 'mysql2';
+import pg from 'pg';
+
+import { Kysely } from 'kysely';
+
+import { createBunSqliteDatabase } from '@enbox/dwn-sql-store';
 import { PluginLoader } from './plugin-loader.js';
 
 import {
   DataStoreLevel,
-  EventLogLevel,
   MessageStoreLevel,
   ResumableTaskStoreLevel,
+  StateIndexLevel,
 } from '@enbox/dwn-sdk-js';
 import {
   DataStoreSql,
-  EventLogSql,
   MessageStoreSql,
   MysqlDialect,
   PostgresDialect,
   ResumableTaskStoreSql,
+  runDwnStoreMigrations,
   SqliteDialect,
+  StateIndexSql,
 } from '@enbox/dwn-sql-store';
 
 export enum StoreType {
   DataStore,
   MessageStore,
-  EventLog,
+  StateIndex,
   ResumableTaskStore,
 }
 
@@ -48,23 +52,113 @@ export enum BackendTypes {
   POSTGRES = 'postgres',
 }
 
-export type DwnStore = DataStore | EventLog | MessageStore | ResumableTaskStore;
+export type DwnStore = DataStore | StateIndex | MessageStore | ResumableTaskStore;
+
+/**
+ * Cache of shared PostgreSQL dialects keyed by connection URL. When multiple
+ * DWN stores share the same Postgres URL, they reuse a single dialect (and
+ * thus a single `pg.Pool`) instead of each creating their own. This reduces
+ * connection count from 4 × pool_max to 1 × pool_max per DWN process.
+ */
+const sharedDialectCache: Map<string, Dialect> = new Map();
+
+/**
+ * Returns a (potentially cached) dialect for the given Postgres connection URL.
+ * Non-Postgres URLs always return a fresh dialect (no caching).
+ */
+function getOrCreateDialect(connectionUrl: URL, config: DwnServerConfig): Dialect {
+  const protocol = connectionUrl.protocol.slice(0, -1);
+
+  if (protocol !== BackendTypes.POSTGRES) {
+    return getDialectFromUrl(connectionUrl);
+  }
+
+  const key = connectionUrl.toString();
+  const cached = sharedDialectCache.get(key);
+  if (cached !== undefined) {
+    return cached;
+  }
+
+  // Create a single pg.Pool instance with configurable sizing.
+  const pool = new pg.Pool({
+    connectionString  : connectionUrl.toString(),
+    min               : config.pgPoolMin,
+    max               : config.pgPoolMax,
+    idleTimeoutMillis : config.pgPoolIdleTimeout,
+  });
+
+  const dialect = new PostgresDialect({
+    pool   : async (): Promise<pg.Pool> => pool,
+    cursor : Cursor,
+  });
+
+  sharedDialectCache.set(key, dialect);
+  return dialect;
+}
 
 export async function getDwnConfig(
-  config  : DwnServerConfig,
+  config : DwnServerConfig,
   options : {
     didResolver? : DidResolver,
-    tenantGate?  : TenantGate,
-    eventStream? : EventStream,
+    tenantGate? : TenantGate,
+    eventLog? : EventLog,
   }
 ): Promise<DwnConfig> {
-  const { tenantGate, eventStream, didResolver } = options;
-  const dataStore: DataStore = await getStore(config.dataStore, StoreType.DataStore);
-  const eventLog: EventLog = await getStore(config.eventLog, StoreType.EventLog);
-  const messageStore: MessageStore = await getStore(config.messageStore, StoreType.MessageStore);
-  const resumableTaskStore: ResumableTaskStore = await getStore(config.resumableTaskStore, StoreType.ResumableTaskStore);
+  const { tenantGate, eventLog, didResolver } = options;
+
+  // Run SQL schema migrations before creating stores. Uses the data store
+  // connection to determine the dialect — all SQL stores typically share the
+  // same database. Non-SQL backends (level://) are skipped.
+  await runSqlMigrationsIfNeeded(config);
 
-  return { didResolver, eventStream, eventLog, dataStore, messageStore, resumableTaskStore, tenantGate };
+  const dataStore: DataStore = await getStore(config, config.dataStore, StoreType.DataStore);
+  const stateIndex: StateIndex = await getStore(config, config.stateIndex, StoreType.StateIndex);
+  const messageStore: MessageStore = await getStore(config, config.messageStore, StoreType.MessageStore);
+  const resumableTaskStore: ResumableTaskStore = await getStore(config, config.resumableTaskStore, StoreType.ResumableTaskStore);
+
+  return { didResolver, eventLog, stateIndex, dataStore, messageStore, resumableTaskStore, tenantGate };
+}
+
+/**
+ * Runs DWN SQL schema migrations if the data store is configured with a SQL
+ * backend. Creates a temporary Kysely instance, runs all pending migrations,
+ * then destroys it. The subsequent store `open()` calls will reuse the shared
+ * dialect/pool and find the schema already in place.
+ */
+async function runSqlMigrationsIfNeeded(config: DwnServerConfig): Promise<void> {
+  // Skip if the data store config is a file path (plugin) or non-SQL backend
+  if (isFilePath(config.dataStore)) {
+    return;
+  }
+
+  let storeUrl: URL;
+  try {
+    storeUrl = new URL(config.dataStore);
+  } catch {
+    return; // Not a valid URL — skip
+  }
+
+  const protocol = storeUrl.protocol.slice(0, -1);
+  const sqlBackends: string[] = [BackendTypes.SQLITE, BackendTypes.MYSQL, BackendTypes.POSTGRES];
+  if (!sqlBackends.includes(protocol)) {
+    return;
+  }
+
+  const dialect = getOrCreateDialect(storeUrl, config);
+  const db = new Kysely<Record<string, unknown>>({ dialect });
+  try {
+    const applied = await runDwnStoreMigrations(db, dialect);
+    if (applied.length > 0) {
+      console.log(`DWN migrations applied: ${applied.join(', ')}`);
+    }
+  } finally {
+    // Don't destroy the Kysely instance if using a shared Postgres pool —
+    // the pool is cached in sharedDialectCache and will be reused by stores.
+    // Only destroy for non-cached dialects (SQLite, MySQL).
+    if (protocol !== BackendTypes.POSTGRES) {
+      await db.destroy();
+    }
+  }
 }
 
 function getLevelStore(
@@ -78,12 +172,12 @@ function getLevelStore(
       });
     case StoreType.MessageStore:
       return new MessageStoreLevel({
-        blockstoreLocation: storeURI.host + storeURI.pathname + '/MESSAGESTORE',
-        indexLocation: storeURI.host + storeURI.pathname + '/INDEX',
+        blockstoreLocation : storeURI.host + storeURI.pathname + '/MESSAGESTORE',
+        indexLocation      : storeURI.host + storeURI.pathname + '/INDEX',
       });
-    case StoreType.EventLog:
-      return new EventLogLevel({
-        location: storeURI.host + storeURI.pathname + '/EVENTLOG',
+    case StoreType.StateIndex:
+      return new StateIndexLevel({
+        location: storeURI.host + storeURI.pathname + '/STATEINDEX',
       });
     case StoreType.ResumableTaskStore:
       return new ResumableTaskStoreLevel({
@@ -95,18 +189,19 @@ function getLevelStore(
 }
 
 function getSqlStore(
+  config: DwnServerConfig,
   connectionUrl: URL,
   storeType: StoreType,
 ): DwnStore {
-  const dialect = getDialectFromUrl(connectionUrl);
+  const dialect = getOrCreateDialect(connectionUrl, config);
 
   switch (storeType) {
     case StoreType.DataStore:
       return new DataStoreSql(dialect);
     case StoreType.MessageStore:
       return new MessageStoreSql(dialect);
-    case StoreType.EventLog:
-      return new EventLogSql(dialect);
+    case StoreType.StateIndex:
+      return new StateIndexSql(dialect);
     case StoreType.ResumableTaskStore:
       return new ResumableTaskStoreSql(dialect);
     default:
@@ -122,16 +217,16 @@ function isFilePath(configString: string): boolean {
   return filePathPrefixes.some(prefix => configString.startsWith(prefix));
 }
 
-async function getStore(storeString: string, storeType: StoreType.DataStore): Promise<DataStore>;
-async function getStore(storeString: string, storeType: StoreType.EventLog): Promise<EventLog>;
-async function getStore(storeString: string, storeType: StoreType.MessageStore): Promise<MessageStore>;
-async function getStore(storeString: string, storeType: StoreType.ResumableTaskStore): Promise<ResumableTaskStore>;
-async function getStore(storeConfigString: string, storeType: StoreType): Promise<DwnStore> {
+async function getStore(config: DwnServerConfig, storeString: string, storeType: StoreType.DataStore): Promise<DataStore>;
+async function getStore(config: DwnServerConfig, storeString: string, storeType: StoreType.StateIndex): Promise<StateIndex>;
+async function getStore(config: DwnServerConfig, storeString: string, storeType: StoreType.MessageStore): Promise<MessageStore>;
+async function getStore(config: DwnServerConfig, storeString: string, storeType: StoreType.ResumableTaskStore): Promise<ResumableTaskStore>;
+async function getStore(config: DwnServerConfig, storeConfigString: string, storeType: StoreType): Promise<DwnStore> {
   if (isFilePath(storeConfigString)) {
     return await loadStoreFromFilePath(storeConfigString, storeType);
   }
   // else treat the `storeConfigString` as a connection string
-  
+
   const storeURI = new URL(storeConfigString);
 
   switch (storeURI.protocol.slice(0, -1)) {
@@ -141,7 +236,7 @@ async function getStore(storeConfigString: string, storeType: StoreType): Promis
     case BackendTypes.SQLITE:
     case BackendTypes.MYSQL:
     case BackendTypes.POSTGRES:
-      return getSqlStore(storeURI, storeType);
+      return getSqlStore(config, storeURI, storeType);
 
     default:
       throw invalidStorageSchemeMessage(storeURI.protocol);
@@ -158,8 +253,8 @@ async function loadStoreFromFilePath(
   switch (storeType) {
     case StoreType.DataStore:
       return await PluginLoader.loadPlugin<DataStore>(filePath);
-    case StoreType.EventLog:
-      return await PluginLoader.loadPlugin<EventLog>(filePath);
+    case StoreType.StateIndex:
+      return await PluginLoader.loadPlugin<StateIndex>(filePath);
     case StoreType.MessageStore:
       return await PluginLoader.loadPlugin<MessageStore>(filePath);
     case StoreType.ResumableTaskStore:
@@ -171,7 +266,7 @@ async function loadStoreFromFilePath(
 
 export function getDialectFromUrl(connectionUrl: URL): Dialect {
   switch (connectionUrl.protocol.slice(0, -1)) {
-    case BackendTypes.SQLITE:
+    case BackendTypes.SQLITE: {
       const path = connectionUrl.host + connectionUrl.pathname;
       console.log('SQL-lite relative path:', path ? path : undefined); // NOTE, using ? for lose equality comparison
 
@@ -184,16 +279,17 @@ export function getDialectFromUrl(connectionUrl: URL): Dialect {
       const dbPath = path || ':memory:';
 
       return new SqliteDialect({
-        database: async () => new Database(dbPath),
+        database: async () => createBunSqliteDatabase(dbPath),
       });
+    }
     case BackendTypes.MYSQL:
       return new MysqlDialect({
         pool: async () => MySQLCreatePool(connectionUrl.toString()),
       });
     case BackendTypes.POSTGRES:
       return new PostgresDialect({
-        pool: async () => new pg.Pool({ connectionString: connectionUrl.toString() }),
-        cursor: Cursor,
+        pool   : async () => new pg.Pool({ connectionString: connectionUrl.toString() }),
+        cursor : Cursor,
       });
   }
 }
@@ -205,7 +301,7 @@ function invalidStorageSchemeMessage(protocol: string): string {
   }
   return (
     'Unknown storage protocol ' +
-    protocol.slice(0, 1) +
+    protocol.slice(0, -1) +
     '! Please use one of: ' +
     schemes.join(', ') +
     '. For details, see README'

package/src/web5-connect/sql-ttl-cache.ts

@@ -39,9 +39,9 @@ export class SqlTtlCache {
         // 512 chars to accommodate potentially large `state` in Web5 Connect flow
         .addColumn('key', 'varchar(512)', (column) => column.primaryKey())
         .addColumn('value', 'text', (column) => column.notNull())
-        .addColumn('expiry', 'integer', (column) => column.notNull())
+        .addColumn('expiry', 'bigint', (column) => column.notNull())
         .execute();
-  
+
       await this.db.schema
         .createIndex('index_expiry')
         // .ifNotExists() // intentionally kept commented out code to show that it is not supported by all dialects (ie. MySQL)
@@ -97,12 +97,19 @@ export class SqlTtlCache {
     const entry = result[0];
 
     // if the entry is expired, don't return it and delete it
-    if (Date.now() >= entry.expiry) {
+    const expiry = typeof entry.expiry === 'string' ? parseInt(entry.expiry, 10) : entry.expiry;
+    if (Date.now() >= expiry) {
       this.delete(key); // no need to await
       return undefined;
     }
 
-    return JSON.parse(entry.value);
+    try {
+      return JSON.parse(entry.value);
+    } catch {
+      // Corrupt entry — remove it and return undefined.
+      this.delete(key); // no need to await
+      return undefined;
+    }
   }
 
   /**
@@ -129,7 +136,7 @@ export class SqlTtlCache {
 interface CacheEntry {
   key: string;
   value: string;
-  expiry: number;
+  expiry: number | string; // bigint is returned as string by PostgreSQL driver
 }
 
 interface CacheDatabase {

package/src/web5-connect/web5-connect-server.ts

@@ -1,16 +1,17 @@
-import { getDialectFromUrl } from "../storage.js";
 import { CryptoUtils } from '@enbox/crypto';
-import { SqlTtlCache } from "./sql-ttl-cache.js";
+
+import { getDialectFromUrl } from '../storage.js';
+import { SqlTtlCache } from './sql-ttl-cache.js';
 
 /**
  * The Web5 Connect Request object.
  */
-export type Web5ConnectRequest = any; // TODO: define type in common repo for reuse (https://github.com/TBD54566975/dwn-server/issues/138)
+export type Web5ConnectRequest = any; // TODO: define type in common repo for reuse (https://github.com/enboxorg/enbox/issues/138)
 
 /**
  * The Web5 Connect Response object, which is also an OIDC ID token
  */
-export type Web5ConnectResponse = any; // TODO: define type in common repo for reuse (https://github.com/TBD54566975/dwn-server/issues/138)
+export type Web5ConnectResponse = any; // TODO: define type in common repo for reuse (https://github.com/enboxorg/enbox/issues/138)
 
 /**
  * The result of the setWeb5ConnectRequest() method.
@@ -25,7 +26,7 @@ export type SetWeb5ConnectRequestResult = {
    * The time in seconds that the Request URI is valid for.
    */
   expires_in: number;
-}
+};
 
 /**
  * The Web5 Connect Server is responsible for handling the Web5 Connect flow.
@@ -69,13 +70,13 @@ export class Web5ConnectServer {
     // Generate a request URI
     const requestId = CryptoUtils.randomUuid();
     const request_uri = `${this.baseUrl}/connect/authorize/${requestId}.jwt`;
-  
+
     // Store the Request Object.
     this.cache.insert(`request:${requestId}`, request, Web5ConnectServer.ttlInSeconds);
-  
+
     return {
       request_uri,
-      expires_in  : Web5ConnectServer.ttlInSeconds,
+      expires_in: Web5ConnectServer.ttlInSeconds,
     };
   }
 

package/src/ws-api.ts

@@ -1,45 +1,58 @@
+import type { Dwn } from '@enbox/dwn-sdk-js';
+import type { ServerWebSocket } from 'bun';
 
-import type {
-  Dwn,
-} from '@enbox/dwn-sdk-js';
-
-import type { Server } from 'http';
-
-import { WebSocketServer } from 'ws';
-
+import type { ActivityLog } from './admin/activity-log.js';
+import type { AdminStore } from './admin/admin-store.js';
 import type { ConnectionManager } from './connection/connection-manager.js';
+import type { DwnServerConfig } from './config.js';
+import type { MessageProcessedHook } from './message-processed-hook.js';
+import type { RateLimiter } from './rate-limiter.js';
+import type { RegistrationStore } from './registration/registration-store.js';
+import type { HttpApi, WsData } from './http-api.js';
+
 import { InMemoryConnectionManager } from './connection/connection-manager.js';
 
 export class WsApi {
-  #wsServer: WebSocketServer;
   dwn: Dwn;
-  #connectionManager: ConnectionManager
-
-  constructor(server: Server, dwn: Dwn, connectionManager?: ConnectionManager) {
+  #connectionManager: ConnectionManager;
+
+  constructor(
+    httpApi: HttpApi, dwn: Dwn, connectionManager?: ConnectionManager,
+    maxInFlight?: number, activityLog?: ActivityLog,
+    options?: {
+      adminStore? : AdminStore;
+      registrationStore? : RegistrationStore;
+      config? : DwnServerConfig;
+      tenantRateLimiter? : RateLimiter;
+      messageProcessedHooks? : MessageProcessedHook[];
+    },
+  ) {
     this.dwn = dwn;
-    this.#connectionManager = connectionManager || new InMemoryConnectionManager(dwn);
-    this.#wsServer = new WebSocketServer({ server });
+    this.#connectionManager = connectionManager ||
+      new InMemoryConnectionManager(
+        dwn, new Map(), maxInFlight, activityLog,
+        options?.adminStore, options?.registrationStore, options?.config, options?.tenantRateLimiter,
+        options?.messageProcessedHooks,
+      );
+
+    // Wire up the WebSocket open event from Bun.serve() to the connection manager.
+    httpApi.onWebSocketConnection = (ws: ServerWebSocket<WsData>): void => {
+      this.#connectionManager.connect(ws);
+    };
   }
 
-  get server(): WebSocketServer {
-    return this.#wsServer;
+  start(): void {
+    // No additional setup needed — Bun.serve() handles WebSocket lifecycle.
   }
 
   /**
-   * Handler for starting a WebSocket.
-   * Sets listeners for `connection`, `close` events.
+   * Returns the connection manager. Used by the admin API for connection introspection.
    */
-  #setupWebSocket(): void {
-    this.#wsServer.on('connection', (socket, request) => this.#connectionManager.connect(socket, request));
-    this.#wsServer.on('close', () => this.#connectionManager.closeAll());
-  }
-
-  start(): void {
-    this.#setupWebSocket();
+  get connectionManager(): ConnectionManager {
+    return this.#connectionManager;
   }
 
   async close(): Promise<void> {
-    this.#wsServer.close();
     await this.#connectionManager.closeAll();
   }
 }