@dbos-inc/knex-datasource 3.0.6-preview

package/index.ts

@@ -0,0 +1,230 @@
+// using https://github.com/knex/knex
+
+import { DBOS, DBOSWorkflowConflictError } from '@dbos-inc/dbos-sdk';
+import {
+  type DataSourceTransactionHandler,
+  isPGRetriableTransactionError,
+  isPGKeyConflictError,
+  registerTransaction,
+  runTransaction,
+  DBOSDataSource,
+  registerDataSource,
+} from '@dbos-inc/dbos-sdk/datasource';
+import { AsyncLocalStorage } from 'async_hooks';
+import knex, { type Knex } from 'knex';
+import { SuperJSON } from 'superjson';
+
+interface transaction_completion {
+  workflow_id: string;
+  function_num: number;
+  output: string | null;
+}
+
+interface KnexDataSourceContext {
+  client: Knex.Transaction;
+}
+
+export type TransactionConfig = Pick<Knex.TransactionConfig, 'isolationLevel' | 'readOnly'>;
+
+const asyncLocalCtx = new AsyncLocalStorage<KnexDataSourceContext>();
+
+class KnexDSTH implements DataSourceTransactionHandler {
+  readonly name: string;
+  readonly dsType = 'KnexDataSource';
+  readonly #knexDB: Knex;
+
+  constructor(name: string, config: Knex.Config) {
+    this.name = name;
+    this.#knexDB = knex(config);
+  }
+
+  initialize(): Promise<void> {
+    return Promise.resolve();
+  }
+
+  destroy(): Promise<void> {
+    return this.#knexDB.destroy();
+  }
+
+  static async #checkExecution(
+    client: Knex.Transaction,
+    workflowID: string,
+    functionNum: number,
+  ): Promise<{ output: string | null } | undefined> {
+    const result = await client<transaction_completion>('transaction_completion')
+      .withSchema('dbos')
+      .select('output')
+      .where({
+        workflow_id: workflowID,
+        function_num: functionNum,
+      })
+      .first();
+    return result === undefined ? undefined : { output: result.output };
+  }
+
+  static async #recordOutput(
+    client: Knex.Transaction,
+    workflowID: string,
+    functionNum: number,
+    output: string | null,
+  ): Promise<void> {
+    try {
+      await client<transaction_completion>('transaction_completion').withSchema('dbos').insert({
+        workflow_id: workflowID,
+        function_num: functionNum,
+        output,
+      });
+    } catch (error) {
+      if (isPGKeyConflictError(error)) {
+        throw new DBOSWorkflowConflictError(workflowID);
+      } else {
+        throw error;
+      }
+    }
+  }
+
+  async invokeTransactionFunction<This, Args extends unknown[], Return>(
+    config: TransactionConfig | undefined,
+    target: This,
+    func: (this: This, ...args: Args) => Promise<Return>,
+    ...args: Args
+  ): Promise<Return> {
+    const workflowID = DBOS.workflowID;
+    if (workflowID === undefined) {
+      throw new Error('Workflow ID is not set.');
+    }
+    const functionNum = DBOS.stepID;
+    if (functionNum === undefined) {
+      throw new Error('Function Number is not set.');
+    }
+
+    const readOnly = config?.readOnly ?? false;
+    let retryWaitMS = 1;
+    const backoffFactor = 1.5;
+    const maxRetryWaitMS = 2000;
+
+    while (true) {
+      try {
+        const result = await this.#knexDB.transaction<Return>(
+          async (client) => {
+            // Check to see if this tx has already been executed
+            const previousResult =
+              readOnly || !workflowID ? undefined : await KnexDSTH.#checkExecution(client, workflowID, functionNum);
+            if (previousResult) {
+              return (previousResult.output ? SuperJSON.parse(previousResult.output) : null) as Return;
+            }
+
+            // execute user's transaction function
+            const result = await asyncLocalCtx.run({ client }, async () => {
+              return (await func.call(target, ...args)) as Return;
+            });
+
+            // save the output of read/write transactions
+            if (!readOnly && workflowID) {
+              await KnexDSTH.#recordOutput(client, workflowID, functionNum, SuperJSON.stringify(result));
+
+              // Note, existing code wraps #recordOutput call in a try/catch block that
+              // converts DB error with code 25P02 to DBOSFailedSqlTransactionError.
+              // However, existing code doesn't make any logic decisions based on that error type.
+              // DBOSFailedSqlTransactionError does stored WF ID and function name, so I assume that info is logged out somewhere
+            }
+
+            return result;
+          },
+          { isolationLevel: config?.isolationLevel, readOnly: config?.readOnly },
+        );
+        // TODO: span.setStatus({ code: SpanStatusCode.OK });
+        // TODO: this.tracer.endSpan(span);
+
+        return result;
+      } catch (error) {
+        if (isPGRetriableTransactionError(error)) {
+          // TODO: span.addEvent('TXN SERIALIZATION FAILURE', { retryWaitMillis: retryWaitMillis }, performance.now());
+          await new Promise((resolve) => setTimeout(resolve, retryWaitMS));
+          retryWaitMS = Math.min(retryWaitMS * backoffFactor, maxRetryWaitMS);
+          continue;
+        } else {
+          // TODO: span.setStatus({ code: SpanStatusCode.ERROR, message: e.message });
+          // TODO: this.tracer.endSpan(span);
+
+          // TODO: currently, we are *not* recording errors in the txOutput table.
+          // For normal execution, this is fine because we also store tx step results (output and error) in the sysdb operation output table.
+          // However, I'm concerned that we have a dueling execution hole where one tx fails while another succeeds.
+          // This implies that we can end up in a situation where the step output records an error but the txOutput table records success.
+
+          throw error;
+        }
+      }
+    }
+  }
+}
+
+export class KnexDataSource implements DBOSDataSource<TransactionConfig> {
+  static get client(): Knex.Transaction {
+    if (!DBOS.isInTransaction()) {
+      throw new Error('invalid use of KnexDataSource.client outside of a DBOS transaction.');
+    }
+    const ctx = asyncLocalCtx.getStore();
+    if (!ctx) {
+      throw new Error('No async local context found.');
+    }
+    return ctx.client;
+  }
+
+  static async initializeInternalSchema(config: Knex.Config) {
+    const knexDB = knex(config);
+    try {
+      await knexDB.schema.createSchemaIfNotExists('dbos');
+      const exists = await knexDB.schema.withSchema('dbos').hasTable('transaction_completion');
+      if (!exists) {
+        await knexDB.schema.withSchema('dbos').createTable('transaction_completion', (table) => {
+          table.string('workflow_id').notNullable();
+          table.integer('function_num').notNullable();
+          table.string('output').nullable();
+          table.primary(['workflow_id', 'function_num']);
+        });
+      }
+    } finally {
+      await knexDB.destroy();
+    }
+  }
+
+  readonly name: string;
+  #provider: KnexDSTH;
+
+  constructor(name: string, config: Knex.Config) {
+    this.name = name;
+    this.#provider = new KnexDSTH(name, config);
+    registerDataSource(this.#provider);
+  }
+
+  async runTransaction<T>(callback: () => Promise<T>, funcName: string, config?: TransactionConfig) {
+    return await runTransaction(callback, funcName, { dsName: this.name, config });
+  }
+
+  registerTransaction<This, Args extends unknown[], Return>(
+    func: (this: This, ...args: Args) => Promise<Return>,
+    name: string,
+    config?: TransactionConfig,
+  ): (this: This, ...args: Args) => Promise<Return> {
+    return registerTransaction(this.name, func, { name }, config);
+  }
+
+  transaction(config?: TransactionConfig) {
+    // eslint-disable-next-line @typescript-eslint/no-this-alias
+    const ds = this;
+    return function decorator<This, Args extends unknown[], Return>(
+      _target: object,
+      propertyKey: string,
+      descriptor: TypedPropertyDescriptor<(this: This, ...args: Args) => Promise<Return>>,
+    ) {
+      if (!descriptor.value) {
+        throw Error('Use of decorator when original method is undefined');
+      }
+
+      descriptor.value = ds.registerTransaction(descriptor.value, propertyKey.toString(), config);
+
+      return descriptor;
+    };
+  }
+}

package/jest.config.js

@@ -0,0 +1,8 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} */
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testRegex: '((\\.|/)(test|spec))\\.ts?$',
+  moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node'],
+  modulePaths: ['./'],
+};

package/package.json

@@ -0,0 +1,31 @@
+{
+  "name": "@dbos-inc/knex-datasource",
+  "version": "3.0.6-preview",
+  "description": "",
+  "license": "MIT",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/dbos-inc/dbos-transact-ts",
+    "directory": "packages/knex-datasource"
+  },
+  "homepage": "https://docs.dbos.dev/",
+  "main": "index.js",
+  "scripts": {
+    "build": "tsc --project tsconfig.json",
+    "test": "jest --detectOpenHandles"
+  },
+  "dependencies": {
+    "knex": "^3.1.0",
+    "superjson": "^1.13"
+  },
+  "peerDependencies": {
+    "@dbos-inc/dbos-sdk": "*"
+  },
+  "devDependencies": {
+    "@types/jest": "^29.5.14",
+    "@types/pg": "^8.15.2",
+    "jest": "^29.7.0",
+    "pg": "^8.11.3",
+    "typescript": "^5.4.5"
+  }
+}

package/tests/config.test.ts

@@ -0,0 +1,31 @@
+import { Client } from 'pg';
+import { KnexDataSource } from '../index';
+import { dropDB, ensureDB } from './test-helpers';
+
+describe('KnexDataSource.configure', () => {
+  const config = { user: 'postgres', database: 'knex_ds_config_test' };
+
+  beforeAll(async () => {
+    const client = new Client({ ...config, database: 'postgres' });
+    try {
+      await client.connect();
+      await dropDB(client, config.database);
+      await ensureDB(client, config.database);
+    } finally {
+      await client.end();
+    }
+  });
+
+  test('configure creates tx outputs table', async () => {
+    await KnexDataSource.initializeInternalSchema({ client: 'pg', connection: config });
+
+    const client = new Client(config);
+    try {
+      await client.connect();
+      const result = await client.query('SELECT workflow_id, function_num, output FROM dbos.transaction_completion');
+      expect(result.rows.length).toBe(0);
+    } finally {
+      await client.end();
+    }
+  });
+});

package/tests/datasource.test.ts

@@ -0,0 +1,310 @@
+import { DBOS } from '@dbos-inc/dbos-sdk';
+import { Client, Pool } from 'pg';
+import { KnexDataSource } from '..';
+import { dropDB, ensureDB } from './test-helpers';
+import { randomUUID } from 'crypto';
+import SuperJSON from 'superjson';
+
+const config = { client: 'pg', connection: { user: 'postgres', database: 'knex_ds_test_userdb' } };
+const dataSource = new KnexDataSource('app-db', config);
+
+interface transaction_completion {
+  workflow_id: string;
+  function_num: number;
+  output: string | null;
+}
+
+describe('KnexDataSource', () => {
+  const userDB = new Pool(config.connection);
+
+  beforeAll(async () => {
+    {
+      const client = new Client({ ...config.connection, database: 'postgres' });
+      try {
+        await client.connect();
+        await dropDB(client, 'knex_ds_test');
+        await dropDB(client, 'knex_ds_test_dbos_sys');
+        await dropDB(client, config.connection.database);
+        await ensureDB(client, config.connection.database);
+      } finally {
+        await client.end();
+      }
+    }
+
+    {
+      const client = await userDB.connect();
+      try {
+        await client.query(
+          'CREATE TABLE greetings(name text NOT NULL, greet_count integer DEFAULT 0, PRIMARY KEY(name))',
+        );
+      } finally {
+        client.release();
+      }
+    }
+
+    await KnexDataSource.initializeInternalSchema(config);
+    DBOS.setConfig({ name: 'knex-ds-test' });
+    await DBOS.launch();
+  });
+
+  afterAll(async () => {
+    await DBOS.shutdown();
+    await userDB.end();
+  });
+
+  test('insert dataSource.register function', async () => {
+    const user = 'helloTest1';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    const workflowID = randomUUID();
+
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regInsertWorfklowReg(user))).resolves.toEqual({
+      user,
+      greet_count: 1,
+    });
+
+    const { rows } = await userDB.query<transaction_completion>(
+      'SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1',
+      [workflowID],
+    );
+    expect(rows.length).toBe(1);
+    expect(rows[0].workflow_id).toBe(workflowID);
+    expect(rows[0].function_num).toBe(0);
+    expect(rows[0].output).not.toBeNull();
+    expect(SuperJSON.parse(rows[0].output!)).toEqual({ user, greet_count: 1 });
+  });
+
+  test('insert dataSource.runAsTx function', async () => {
+    const user = 'helloTest2';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    const workflowID = randomUUID();
+
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regInsertWorfklowRunTx(user))).resolves.toEqual({
+      user,
+      greet_count: 1,
+    });
+
+    const { rows } = await userDB.query<transaction_completion>(
+      'SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1',
+      [workflowID],
+    );
+    expect(rows.length).toBe(1);
+    expect(rows[0].workflow_id).toBe(workflowID);
+    expect(rows[0].function_num).toBe(0);
+    expect(rows[0].output).not.toBeNull();
+    expect(SuperJSON.parse(rows[0].output!)).toEqual({ user, greet_count: 1 });
+  });
+
+  test('error dataSource.register function', async () => {
+    const user = 'errorTest1';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    await userDB.query('INSERT INTO greetings("name","greet_count") VALUES($1,10);', [user]);
+    const workflowID = randomUUID();
+
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regErrorWorkflowReg(user))).rejects.toThrow('test error');
+
+    const { rows: txOutput } = await userDB.query<transaction_completion>(
+      'SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1',
+      [workflowID],
+    );
+    expect(txOutput.length).toBe(0);
+
+    const { rows } = await userDB.query<greetings>('SELECT * FROM greetings WHERE name = $1', [user]);
+    expect(rows.length).toBe(1);
+    expect(rows[0].greet_count).toBe(10);
+  });
+
+  test('error dataSource.runAsTx function', async () => {
+    const user = 'errorTest2';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    await userDB.query('INSERT INTO greetings("name","greet_count") VALUES($1,10);', [user]);
+    const workflowID = randomUUID();
+
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regErrorWorkflowRunTx(user))).rejects.toThrow('test error');
+
+    const { rows: txOutput } = await userDB.query<transaction_completion>(
+      'SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1',
+      [workflowID],
+    );
+    expect(txOutput.length).toBe(0);
+
+    const { rows } = await userDB.query<greetings>('SELECT * FROM greetings WHERE name = $1', [user]);
+    expect(rows.length).toBe(1);
+    expect(rows[0].greet_count).toBe(10);
+  });
+
+  test('readonly dataSource.register function', async () => {
+    const user = 'readTest1';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    await userDB.query('INSERT INTO greetings("name","greet_count") VALUES($1,10);', [user]);
+
+    const workflowID = randomUUID();
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regReadWorkflowReg(user))).resolves.toEqual({
+      user,
+      greet_count: 10,
+    });
+
+    const { rows } = await userDB.query<transaction_completion>(
+      'SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1',
+      [workflowID],
+    );
+    expect(rows.length).toBe(0);
+  });
+
+  test('readonly dataSource.runAsTx function', async () => {
+    const user = 'readTest2';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+    await userDB.query('INSERT INTO greetings("name","greet_count") VALUES($1,10);', [user]);
+
+    const workflowID = randomUUID();
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regReadWorkflowRunTx(user))).resolves.toEqual({
+      user,
+      greet_count: 10,
+    });
+
+    const { rows } = await userDB.query('SELECT * FROM dbos.transaction_completion WHERE workflow_id = $1', [
+      workflowID,
+    ]);
+    expect(rows.length).toBe(0);
+  });
+
+  test('static dataSource.register methods', async () => {
+    const user = 'staticTest1';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+
+    const workflowID = randomUUID();
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regStaticWorkflow(user))).resolves.toEqual([
+      { user, greet_count: 1 },
+      { user, greet_count: 1 },
+    ]);
+  });
+
+  test('instance dataSource.register methods', async () => {
+    const user = 'instanceTest1';
+
+    await userDB.query('DELETE FROM greetings WHERE name = $1', [user]);
+
+    const workflowID = randomUUID();
+    await expect(DBOS.withNextWorkflowID(workflowID, () => regInstanceWorkflow(user))).resolves.toEqual([
+      { user, greet_count: 1 },
+      { user, greet_count: 1 },
+    ]);
+  });
+});
+
+export interface greetings {
+  name: string;
+  greet_count: number;
+}
+
+async function insertFunction(user: string) {
+  const rows = await KnexDataSource.client<greetings>('greetings')
+    .insert({ name: user, greet_count: 1 })
+    .onConflict('name')
+    .merge({ greet_count: KnexDataSource.client.raw('greetings.greet_count + 1') })
+    .returning('greet_count');
+  const row = rows.length > 0 ? rows[0] : undefined;
+
+  return { user, greet_count: row?.greet_count };
+}
+
+async function errorFunction(user: string) {
+  const result = await insertFunction(user);
+  throw new Error('test error');
+  return result;
+}
+
+async function readFunction(user: string) {
+  const row = await KnexDataSource.client<greetings>('greetings').select('greet_count').where('name', user).first();
+  return { user, greet_count: row?.greet_count };
+}
+
+const regInsertFunction = dataSource.registerTransaction(insertFunction, 'insertFunction');
+const regErrorFunction = dataSource.registerTransaction(errorFunction, 'errorFunction');
+const regReadFunction = dataSource.registerTransaction(readFunction, 'readFunction', { readOnly: true });
+
+class StaticClass {
+  static async insertFunction(user: string) {
+    return await insertFunction(user);
+  }
+
+  static async readFunction(user: string) {
+    return await readFunction(user);
+  }
+}
+
+StaticClass.insertFunction = dataSource.registerTransaction(StaticClass.insertFunction, 'insertFunction');
+StaticClass.readFunction = dataSource.registerTransaction(StaticClass.readFunction, 'readFunction');
+
+class InstanceClass {
+  async insertFunction(user: string) {
+    return await insertFunction(user);
+  }
+
+  async readFunction(user: string) {
+    return await readFunction(user);
+  }
+}
+
+InstanceClass.prototype.insertFunction = dataSource.registerTransaction(
+  // eslint-disable-next-line @typescript-eslint/unbound-method
+  InstanceClass.prototype.insertFunction,
+  'insertFunction',
+);
+InstanceClass.prototype.readFunction = dataSource.registerTransaction(
+  // eslint-disable-next-line @typescript-eslint/unbound-method
+  InstanceClass.prototype.readFunction,
+  'readFunction',
+);
+
+async function insertWorkflowReg(user: string) {
+  return await regInsertFunction(user);
+}
+
+async function insertWorkflowRunTx(user: string) {
+  return await dataSource.runTransaction(() => insertFunction(user), 'insertFunction');
+}
+
+async function errorWorkflowReg(user: string) {
+  return await regErrorFunction(user);
+}
+
+async function errorWorkflowRunTx(user: string) {
+  return await dataSource.runTransaction(() => errorFunction(user), 'errorFunction');
+}
+
+async function readWorkflowReg(user: string) {
+  return await regReadFunction(user);
+}
+
+async function readWorkflowRunTx(user: string) {
+  return await dataSource.runTransaction(() => readFunction(user), 'readFunction', { readOnly: true });
+}
+
+async function staticWorkflow(user: string) {
+  const result = await StaticClass.insertFunction(user);
+  const readResult = await StaticClass.readFunction(user);
+  return [result, readResult];
+}
+
+async function instanceWorkflow(user: string) {
+  const instance = new InstanceClass();
+  const result = await instance.insertFunction(user);
+  const readResult = await instance.readFunction(user);
+  return [result, readResult];
+}
+
+const regInsertWorfklowReg = DBOS.registerWorkflow(insertWorkflowReg, 'insertWorkflowReg');
+const regInsertWorfklowRunTx = DBOS.registerWorkflow(insertWorkflowRunTx, 'insertWorkflowRunTx');
+const regErrorWorkflowReg = DBOS.registerWorkflow(errorWorkflowReg, 'errorWorkflowReg');
+const regErrorWorkflowRunTx = DBOS.registerWorkflow(errorWorkflowRunTx, 'errorWorkflowRunTx');
+const regReadWorkflowReg = DBOS.registerWorkflow(readWorkflowReg, 'readWorkflowReg');
+const regReadWorkflowRunTx = DBOS.registerWorkflow(readWorkflowRunTx, 'readWorkflowRunTx');
+const regStaticWorkflow = DBOS.registerWorkflow(staticWorkflow, 'staticWorkflow');
+const regInstanceWorkflow = DBOS.registerWorkflow(instanceWorkflow, 'instanceWorkflow');

package/tests/test-helpers.ts

@@ -0,0 +1,12 @@
+import { Client } from 'pg';
+
+export async function ensureDB(client: Client, name: string) {
+  const results = await client.query('SELECT 1 FROM pg_database WHERE datname = $1', [name]);
+  if (results.rows.length === 0) {
+    await client.query(`CREATE DATABASE ${name}`);
+  }
+}
+
+export async function dropDB(client: Client, name: string) {
+  await client.query(`DROP DATABASE IF EXISTS ${name}`);
+}

package/tsconfig.json

@@ -0,0 +1,9 @@
+/* Visit https://aka.ms/tsconfig to read more about this file */
+{
+  "extends": "../../tsconfig.shared.json",
+  "compilerOptions": {
+    "outDir": "./dist"
+  },
+  "include": ["index.ts"],
+  "exclude": ["dist", "tests"]
+}