@enbox/dwn-sql-store 0.0.1

package/package.json

@@ -0,0 +1,91 @@
+{
+  "name": "@enbox/dwn-sql-store",
+  "version": "0.0.1",
+  "description": "SQL backed implementations of DWN MessageStore, DataStore, and EventLog",
+  "type": "module",
+  "license": "Apache-2.0",
+  "homepage": "https://github.com/enboxorg/enbox/tree/main/packages/dwn-sql-store#readme",
+  "bugs": "https://github.com/enboxorg/enbox/issues",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/enboxorg/enbox.git",
+    "directory": "packages/dwn-sql-store"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "main": "./dist/cjs/main.js",
+  "module": "./dist/esm/src/main.js",
+  "types": "./dist/types/src/main.d.ts",
+  "exports": {
+    "import": "./dist/esm/src/main.js",
+    "require": "./dist/cjs/main.js",
+    "types": "./dist/types/src/main.d.ts"
+  },
+  "react-native": "./dist/esm/src/main.js",
+  "dependencies": {
+    "@enbox/dwn-sdk-js": "workspace:*",
+    "@ipld/dag-cbor": "9.0.5",
+    "kysely": "0.26.3",
+    "multiformats": "12.0.1",
+    "readable-stream": "4.4.2"
+  },
+  "devDependencies": {
+    "@types/better-sqlite3": "7.6.4",
+    "@types/chai": "4.3.4",
+    "@types/chai-as-promised": "7.1.5",
+    "@types/express": "4.17.17",
+    "@types/mocha": "10.0.1",
+    "@types/node": "20.3.1",
+    "@types/pg": "8.10.2",
+    "@types/pg-cursor": "2.7.0",
+    "@types/readable-stream": "4.0.6",
+    "@types/supertest": "2.0.12",
+    "@typescript-eslint/eslint-plugin": "5.59.11",
+    "@typescript-eslint/parser": "5.59.11",
+    "better-sqlite3": "8.4.0",
+    "c8": "8.0.0",
+    "chai": "4.3.6",
+    "chai-as-promised": "7.1.1",
+    "esbuild": "^0.19.3",
+    "eslint": "8.42.0",
+    "mocha": "10.2.0",
+    "mysql2": "3.9.8",
+    "pg": "8.11.1",
+    "pg-cursor": "2.10.1",
+    "rimraf": "5.0.1",
+    "sinon": "^18.0.1",
+    "typescript": "5.0.4"
+  },
+  "scripts": {
+    "build:esm": "pnpm clean & tsc",
+    "build:cjs": "pnpm build:esm && node build/create-cjs-bundle.cjs && echo '{\"type\": \"commonjs\"}' > ./dist/cjs/package.json",
+    "build": "pnpm clean && pnpm build:esm && pnpm build:cjs",
+    "clean": "rimraf dist compiled",
+    "lint": "eslint . --ext .ts --max-warnings 0",
+    "lint:fix": "eslint . --ext .ts --fix",
+    "test": "rimraf compiled && tsc -p tests/tsconfig.json && mocha",
+    "test-coverage": "rimraf compiled && tsc -p tests/tsconfig.json && c8 mocha"
+  },
+  "files": [
+    "dist",
+    "src"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "contributors": [
+    {
+      "name": "Adam Mika",
+      "url": "https://github.com/amika-sq"
+    },
+    {
+      "name": "Moe Jangda",
+      "url": "https://github.com/mistermoe"
+    },
+    {
+      "name": "Liran Cohen",
+      "url": "https://github.com/lirancohen"
+    }
+  ]
+}

package/src/data-store-sql.ts

@@ -0,0 +1,148 @@
+import { DataStore, DataStream, DataStoreGetResult, DataStorePutResult } from '@enbox/dwn-sdk-js';
+import { Kysely } from 'kysely';
+import { Readable } from 'readable-stream';
+import { DwnDatabaseType } from './types.js';
+import { Dialect } from './dialect/dialect.js';
+
+export class DataStoreSql implements DataStore {
+  #dialect: Dialect;
+  #db: Kysely<DwnDatabaseType> | null = null;
+
+  constructor(dialect: Dialect) {
+    this.#dialect = dialect;
+  }
+
+  async open(): Promise<void> {
+    if (this.#db) {
+      return;
+    }
+
+    this.#db = new Kysely<DwnDatabaseType>({ dialect: this.#dialect });
+
+    // if table already exists, there is no more things todo
+    const tableName = 'dataStore';
+    const tableExists = await this.#dialect.hasTable(this.#db, tableName);
+    if (tableExists) {
+      return;
+    }
+
+    // else create the table and corresponding indexes
+
+    let table = this.#db.schema
+      .createTable(tableName)
+      .ifNotExists()// kept to show supported by all dialects in contrast to ifNotExists() below, though not needed due to hasTable() check above
+      .addColumn('tenant', 'varchar(255)', (col) => col.notNull())
+      .addColumn('recordId', 'varchar(60)', (col) => col.notNull())
+      .addColumn('dataCid', 'varchar(60)', (col) => col.notNull());
+
+    // Add columns that have dialect-specific constraints
+    table = this.#dialect.addAutoIncrementingColumn(table, 'id', (col) => col.primaryKey());
+    table = this.#dialect.addBlobColumn(table, 'data', (col) => col.notNull());
+    await table.execute();
+
+    // Add index for efficient lookups.
+    await this.#db.schema
+      .createIndex('tenant_recordId_dataCid')
+      // .ifNotExists() // intentionally kept commented out code to show that it is not supported by all dialects (ie. MySQL)
+      .on(tableName)
+      .columns(['tenant', 'recordId', 'dataCid'])
+      .unique()
+      .execute();
+  }
+
+  async close(): Promise<void> {
+    await this.#db?.destroy();
+    this.#db = null;
+  }
+
+  async get(
+    tenant: string,
+    recordId: string,
+    dataCid: string
+  ): Promise<DataStoreGetResult | undefined> {
+    if (!this.#db) {
+      throw new Error(
+        'Connection to database not open. Call `open` before using `get`.'
+      );
+    }
+
+    const result = await this.#db
+      .selectFrom('dataStore')
+      .selectAll()
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .executeTakeFirst();
+
+    if (!result) {
+      return undefined;
+    }
+
+    return {
+      dataSize   : result.data.length,
+      dataStream : new Readable({
+        read() {
+          this.push(Buffer.from(result.data));
+          this.push(null);
+        }
+      }),
+    };
+  }
+
+  async put(
+    tenant: string,
+    recordId: string,
+    dataCid: string,
+    dataStream: Readable
+  ): Promise<DataStorePutResult> {
+    if (!this.#db) {
+      throw new Error(
+        'Connection to database not open. Call `open` before using `put`.'
+      );
+    }
+
+    const bytes = await DataStream.toBytes(dataStream);
+    const data = Buffer.from(bytes);
+
+    await this.#db
+      .insertInto('dataStore')
+      .values({ tenant, recordId, dataCid, data })
+      .executeTakeFirstOrThrow();
+
+    return {
+      dataSize: bytes.length
+    };
+  }
+
+  async delete(
+    tenant: string,
+    recordId: string,
+    dataCid: string
+  ): Promise<void> {
+    if (!this.#db) {
+      throw new Error(
+        'Connection to database not open. Call `open` before using `delete`.'
+      );
+    }
+
+    await this.#db
+      .deleteFrom('dataStore')
+      .where('tenant', '=', tenant)
+      .where('recordId', '=', recordId)
+      .where('dataCid', '=', dataCid)
+      .execute();
+  }
+
+  async clear(): Promise<void> {
+    if (!this.#db) {
+      throw new Error(
+        'Connection to database not open. Call `open` before using `clear`.'
+      );
+    }
+
+    await this.#db
+      .deleteFrom('dataStore')
+      .execute();
+  }
+
+}

package/src/dialect/dialect.ts

@@ -0,0 +1,77 @@
+import {
+  ColumnBuilderCallback,
+  ColumnDataType,
+  CreateTableBuilder,
+  Dialect as KyselyDialect,
+  Kysely,
+  InsertObject,
+  InsertQueryBuilder,
+  Selection,
+  SelectExpression,
+  Transaction,
+} from 'kysely';
+
+export interface Dialect extends KyselyDialect {
+  readonly name: string;
+  readonly isStreamingSupported: boolean;
+
+  hasTable(db: Kysely<any>, tableName: string): Promise<boolean>;
+
+  addAutoIncrementingColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB>;
+
+  addBlobColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB>;
+
+  /**
+   * This is a helper method to add a column with foreign key constraints.
+   * This is primarily useful because the `mySQL` dialect adds the constraints in a different way than `sqlite` and `postgres`.
+   *
+   * @param builder the CreateTableBuilder to add the column to.
+   * @param tableName the name of the table to add the column to.
+   * @param columnName the name of the column to add.
+   * @param columnType the type of the column to add.
+   * @param referenceTable the foreign table to reference.
+   * @param referenceColumnName the foreign column to reference.
+   * @param onDeleteAction the action to take when the referenced row is deleted.
+   *
+   * @returns {CreateTableBuilder} the CreateTableBuilder with the added column.
+   */
+  addReferencedColumn<TB extends string>(
+    builder: CreateTableBuilder<TB & string>,
+    tableName: TB,
+    columnName: string,
+    columnType: ColumnDataType,
+    referenceTable: string,
+    referenceColumnName: string,
+    onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
+  ): CreateTableBuilder<TB & string>;
+
+  /**
+   * This is a helper method to return an `insertId` across all dialects after inserting values.
+   * `postgres` and `sqlite` both support the `returning` clause, however `mysql` does not and instead returns the last inserted id.
+   *
+   * @param db the Kysely DB object or a DB Transaction.
+   * @param table the table to insert into.
+   * @param values the values to insert.
+   * @param returning a string representing the generated key you'd like returned as an insertId.
+   *
+   *  NOTE: the `returning` value must be formatted to return an insertId value.
+   *  ex. if the generated key is `id` the string should be `id as insertId`.
+   *      if the generated key is `watermark` the string should be `watermark as insertId`.
+   *
+   * @returns {InsertQueryBuilder} object to further modify the query or execute it.
+   */
+  insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = any>(
+    db: Transaction<DB> | Kysely<DB>,
+    table: TB & string,
+    values: InsertObject<DB, TB & string>,
+    returning: SE & `${string} as insertId`,
+  ): InsertQueryBuilder<DB, TB & string, Selection<DB, TB & string, SE & `${string} as insertId`>>;
+}

package/src/dialect/mysql-dialect.ts

@@ -0,0 +1,86 @@
+import { Dialect } from './dialect.js';
+import {
+  AnyColumn,
+  ColumnDataType,
+  CreateTableBuilder,
+  ColumnBuilderCallback,
+  InsertObject,
+  InsertQueryBuilder,
+  SelectExpression,
+  Selection,
+  Transaction,
+  MysqlDialect as KyselyMysqlDialect,
+  Kysely,
+} from 'kysely';
+
+export class MysqlDialect extends KyselyMysqlDialect implements Dialect {
+  name = 'MySQL';
+  isStreamingSupported = true;
+
+  async hasTable(db: Kysely<any>, tableName: string): Promise<boolean> {
+    const result = await db
+      .selectFrom('information_schema.tables')
+      .select('table_name')
+      .where('table_name', '=', tableName)
+      .execute();
+
+    return result.length > 0;
+  }
+
+  addAutoIncrementingColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'integer', (col) => {
+      col = col.autoIncrement();
+      if (callback) {
+        col = callback(col);
+      }
+      return col;
+    });
+  }
+
+  addBlobColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'blob', callback);
+  }
+
+  /**
+   * In MySQL, the ForeignKey name it creates in `mysql` will be in the following format:
+   * `${referenceTable}_${referenceColumnName}__${tableName}_${columnName}`
+   * ex: if the reference table is `users` and the reference column is `id` and the table is `profiles` and the column is `userId`,
+   * the resulting name for the foreign key is: `users_id__profiles_userId`
+   */
+  addReferencedColumn<TB extends string>(
+    builder: CreateTableBuilder<TB & string>,
+    tableName: TB,
+    columnName: string,
+    columnType: ColumnDataType,
+    referenceTable: string,
+    referenceColumnName: string,
+    onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
+  ): CreateTableBuilder<TB & string> {
+    return builder
+      .addColumn(columnName, columnType, (col) => col.notNull())
+      .addForeignKeyConstraint(
+        `${referenceTable}_${referenceColumnName}__${tableName}_${columnName}`,
+        [columnName],
+        referenceTable,
+        [referenceColumnName],
+        (constraint) => constraint.onDelete(onDeleteAction)
+      );
+  }
+
+  insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = AnyColumn<DB, TB>>(
+    db: Transaction<DB> | Kysely<DB>,
+    table: TB & string,
+    values: InsertObject<DB, TB & string>,
+    _returning: SE & `${string} as insertId`,
+  ): InsertQueryBuilder<DB, TB & string, Selection<DB, TB & string, SE & `${string} as insertId`>> {
+    return db.insertInto(table).values(values);
+  }
+}

package/src/dialect/postgres-dialect.ts

@@ -0,0 +1,66 @@
+import { Dialect } from './dialect.js';
+import {
+  ColumnDataType,
+  ColumnBuilderCallback,
+  CreateTableBuilder,
+  InsertObject,
+  InsertQueryBuilder,
+  Kysely,
+  PostgresDialect as KyselyPostgresDialect,
+  SelectExpression,
+  Selection,
+  Transaction,
+} from 'kysely';
+
+export class PostgresDialect extends KyselyPostgresDialect implements Dialect {
+  name = 'PostgreSQL';
+  isStreamingSupported = true;
+
+  async hasTable(db: Kysely<any>, tableName: string): Promise<boolean> {
+    const result = await db
+      .selectFrom('information_schema.tables')
+      .select('table_name')
+      .where('table_name', '=', tableName)
+      .execute();
+
+    return result.length > 0;
+  }
+
+  addAutoIncrementingColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'serial', callback);
+  }
+
+  addBlobColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'bytea', callback);
+  }
+
+  addReferencedColumn<TB extends string>(
+    builder: CreateTableBuilder<TB & string>,
+    _tableName: TB,
+    columnName: string,
+    columnType: ColumnDataType,
+    referenceTable: string,
+    referenceColumnName: string,
+    onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
+  ): CreateTableBuilder<TB & string> {
+    return builder.addColumn(columnName, columnType, (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction));
+  }
+
+  insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = any>(
+    db: Transaction<DB> | Kysely<DB>,
+    table: TB & string,
+    values: InsertObject<DB, TB & string>,
+    returning: SE & `${string} as insertId`,
+  ): InsertQueryBuilder<DB, TB & string, Selection<DB, TB & string, SE & `${string} as insertId`>> {
+    return db.insertInto(table).values(values).returning(returning);
+  }
+
+}

package/src/dialect/sqlite-dialect.ts

@@ -0,0 +1,72 @@
+import { Dialect } from './dialect.js';
+import {
+  ColumnBuilderCallback,
+  ColumnDataType,
+  CreateTableBuilder,
+  Kysely,
+  InsertObject,
+  InsertQueryBuilder,
+  SelectExpression,
+  Selection,
+  SqliteDialect as KyselySqliteDialect,
+  Transaction,
+} from 'kysely';
+
+export class SqliteDialect extends KyselySqliteDialect implements Dialect {
+  name = 'SQLite';
+  isStreamingSupported = false;
+
+  async hasTable(db: Kysely<any>, tableName: string): Promise<boolean> {
+    const result = await db
+      .selectFrom('sqlite_master')
+      .select('name')
+      .where('type', '=', 'table')
+      .where('name', '=', tableName)
+      .execute();
+
+    return result.length > 0;
+  }
+
+  addAutoIncrementingColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'integer', (col) => {
+      col = col.autoIncrement();
+      if (callback) {
+        col = callback(col);
+      }
+      return col;
+    });
+  }
+
+  addBlobColumn<TB extends string>(
+    builder: CreateTableBuilder<TB>,
+    columnName: string,
+    callback?: ColumnBuilderCallback
+  ): CreateTableBuilder<TB> {
+    return builder.addColumn(columnName, 'blob', callback);
+  }
+
+  addReferencedColumn<TB extends string>(
+    builder: CreateTableBuilder<TB & string>,
+    _tableName: TB,
+    columnName: string,
+    columnType: ColumnDataType,
+    referenceTable: string,
+    referenceColumnName: string,
+    onDeleteAction: 'cascade' | 'no action' | 'restrict' | 'set null' | 'set default',
+  ): CreateTableBuilder<TB & string> {
+    return builder.addColumn(columnName, columnType, (col) => col.notNull().references(`${referenceTable}.${referenceColumnName}`).onDelete(onDeleteAction));
+  }
+
+  insertThenReturnId<DB, TB extends keyof DB = keyof DB, SE extends SelectExpression<DB, TB & string> = any>(
+    db: Transaction<DB> | Kysely<DB>,
+    table: TB & string,
+    values: InsertObject<DB, TB & string>,
+    returning: SE & `${string} as insertId`,
+  ): InsertQueryBuilder<DB, TB & string, Selection<DB, TB & string, SE & `${string} as insertId`>> {
+    return db.insertInto(table).values(values).returning(returning);
+  }
+}