@powersync/node 0.12.0 → 0.14.0

package/src/db/RemoteConnection.ts

@@ -0,0 +1,129 @@
+import { Worker } from 'node:worker_threads';
+import { LockContext, QueryResult } from '@powersync/common';
+import { releaseProxy, Remote } from 'comlink';
+import { AsyncDatabase, AsyncDatabaseOpener, ProxiedQueryResult } from './AsyncDatabase.js';
+
+/**
+ * A PowerSync database connection implemented with RPC calls to a background worker.
+ */
+export class RemoteConnection implements LockContext {
+  isBusy = false;
+
+  private readonly worker: Worker;
+  private readonly comlink: Remote<AsyncDatabaseOpener>;
+  private readonly database: Remote<AsyncDatabase>;
+
+  private readonly notifyWorkerClosed = new AbortController();
+
+  constructor(worker: Worker, comlink: Remote<AsyncDatabaseOpener>, database: Remote<AsyncDatabase>) {
+    this.worker = worker;
+    this.comlink = comlink;
+    this.database = database;
+
+    this.worker.once('exit', (_) => {
+      this.notifyWorkerClosed.abort();
+    });
+  }
+
+  /**
+   * Runs the inner function, but appends the stack trace where this function was called. This is useful for workers
+   * because stack traces from worker errors are otherwise unrelated to the application issue that has caused them.
+   */
+  private withRemote<T>(inner: () => Promise<T>): Promise<T> {
+    const trace = {};
+    Error.captureStackTrace(trace);
+    const controller = this.notifyWorkerClosed;
+
+    return new Promise((resolve, reject) => {
+      if (controller.signal.aborted) {
+        reject(new Error('Called operation on closed remote'));
+      }
+
+      function handleAbort() {
+        reject(new Error('Remote peer closed with request in flight'));
+      }
+
+      function completePromise(action: () => void) {
+        controller!.signal.removeEventListener('abort', handleAbort);
+        action();
+      }
+
+      controller.signal.addEventListener('abort', handleAbort);
+
+      inner()
+        .then((data) => completePromise(() => resolve(data)))
+        .catch((e) => {
+          if (e instanceof Error && e.stack) {
+            e.stack += (trace as any).stack;
+          }
+
+          return completePromise(() => reject(e));
+        });
+    });
+  }
+
+  executeBatch(query: string, params: any[][] = []): Promise<QueryResult> {
+    return this.withRemote(async () => {
+      const result = await this.database.executeBatch(query, params ?? []);
+      return RemoteConnection.wrapQueryResult(result);
+    });
+  }
+
+  execute(query: string, params?: any[] | undefined): Promise<QueryResult> {
+    return this.withRemote(async () => {
+      const result = await this.database.execute(query, params ?? []);
+      return RemoteConnection.wrapQueryResult(result);
+    });
+  }
+
+  executeRaw(query: string, params?: any[] | undefined): Promise<any[][]> {
+    return this.withRemote(async () => {
+      return await this.database.executeRaw(query, params ?? []);
+    });
+  }
+
+  async getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+    const res = await this.execute(sql, parameters);
+    return res.rows?._array ?? [];
+  }
+
+  async getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+    const res = await this.execute(sql, parameters);
+    return res.rows?.item(0) ?? null;
+  }
+
+  async get<T>(sql: string, parameters?: any[]): Promise<T> {
+    const res = await this.execute(sql, parameters);
+    const first = res.rows?.item(0);
+    if (!first) {
+      throw new Error('Result set is empty');
+    }
+    return first;
+  }
+
+  async refreshSchema() {
+    await this.execute("pragma table_info('sqlite_master')");
+  }
+
+  async close() {
+    await this.database.close();
+    this.database[releaseProxy]();
+    this.comlink[releaseProxy]();
+    await this.worker.terminate();
+  }
+
+  static wrapQueryResult(result: ProxiedQueryResult): QueryResult {
+    let rows: QueryResult['rows'] | undefined = undefined;
+    if (result.rows) {
+      rows = {
+        ...result.rows,
+        item: (idx) => result.rows?._array[idx]
+      } satisfies QueryResult['rows'];
+    }
+
+    return {
+      ...result,
+      rows
+    };
+  }
+}

package/src/db/SqliteWorker.ts

@@ -0,0 +1,119 @@
+import * as Comlink from 'comlink';
+import OS from 'node:os';
+import * as path from 'node:path';
+import url from 'node:url';
+import { parentPort } from 'node:worker_threads';
+import { dynamicImport, isBundledToCommonJs } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { openDatabase as openBetterSqliteDatabase } from './BetterSqliteWorker.js';
+import { openDatabase as openNodeDatabase } from './NodeSqliteWorker.js';
+
+export interface PowerSyncWorkerOptions {
+  /**
+   * A function responsible for finding the powersync DLL/so/dylib file.
+   *
+   * @returns The absolute path of the PowerSync SQLite core extensions library.
+   */
+  extensionPath: () => string;
+
+  /**
+   * A function that returns the `Database` constructor from the `better-sqlite3` package.
+   */
+  loadBetterSqlite3: () => Promise<any>;
+}
+
+/**
+ * @returns The relevant PowerSync extension binary filename for the current platform and architecture
+ */
+export function getPowerSyncExtensionFilename() {
+  const platform = OS.platform();
+  const arch = OS.arch();
+  let extensionFile: string;
+
+  if (platform == 'win32') {
+    if (arch == 'x64') {
+      extensionFile = 'powersync_x64.dll';
+    } else if (arch == 'ia32') {
+      extensionFile = 'powersync_x86.dll';
+    } else if (arch == 'arm64') {
+      extensionFile = 'powersync_aarch64.dll';
+    } else {
+      throw new Error('Windows platform only supports arm64, ia32 and x64 architecture.');
+    }
+  } else if (platform == 'linux') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.so';
+    } else if (arch == 'arm64') {
+      // TODO detect armv7 as an option
+      extensionFile = 'libpowersync_aarch64.so';
+    } else if (arch == 'riscv64') {
+      extensionFile = 'libpowersync_riscv64gc.so';
+    } else {
+      throw new Error('Linux platform only supports x64, arm64 and riscv64 architectures.');
+    }
+  } else if (platform == 'darwin') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.dylib';
+    } else if (arch == 'arm64') {
+      extensionFile = 'libpowersync_aarch64.dylib';
+    } else {
+      throw new Error('macOS platform only supports x64 and arm64 architectures.');
+    }
+  } else {
+    throw new Error(
+      `Unknown platform: ${platform}, PowerSync for Node.js currently supports Windows, Linux and macOS.`
+    );
+  }
+
+  return extensionFile;
+}
+
+export function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>) {
+  const resolvedOptions: PowerSyncWorkerOptions = {
+    extensionPath() {
+      const isCommonJsModule = isBundledToCommonJs;
+      const extensionFilename = getPowerSyncExtensionFilename();
+      let resolved: string;
+      if (isCommonJsModule) {
+        resolved = path.resolve(__dirname, '../lib/', extensionFilename);
+      } else {
+        resolved = url.fileURLToPath(new URL(`../${extensionFilename}`, import.meta.url));
+      }
+
+      return resolved;
+    },
+    async loadBetterSqlite3() {
+      const module = await dynamicImport('better-sqlite3');
+      return module.default;
+    },
+    ...options
+  };
+
+  Comlink.expose(new DatabaseOpenHelper(resolvedOptions), parentPort! as Comlink.Endpoint);
+}
+
+class DatabaseOpenHelper implements AsyncDatabaseOpener {
+  private options: PowerSyncWorkerOptions;
+
+  constructor(options: PowerSyncWorkerOptions) {
+    this.options = options;
+  }
+
+  async open(options: AsyncDatabaseOpenOptions): Promise<AsyncDatabase> {
+    let database: AsyncDatabase;
+
+    const implementation = options.implementation;
+    switch (implementation.type) {
+      case 'better-sqlite3':
+        database = await openBetterSqliteDatabase(this.options, options);
+        break;
+      case 'node:sqlite':
+        database = await openNodeDatabase(this.options, options);
+        break;
+      default:
+        throw new Error(`Unknown database implementation: ${options.implementation}.`);
+    }
+
+    return Comlink.proxy(database);
+  }
+}

package/src/db/WorkerConnectionPool.ts

@@ -0,0 +1,328 @@
+import * as Comlink from 'comlink';
+import fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { Worker } from 'node:worker_threads';
+
+import {
+  BaseObserver,
+  BatchedUpdateNotification,
+  DBAdapter,
+  DBAdapterListener,
+  DBLockOptions,
+  LockContext,
+  QueryResult,
+  Transaction
+} from '@powersync/common';
+import { Remote } from 'comlink';
+import { AsyncResource } from 'node:async_hooks';
+import { isBundledToCommonJs } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpener } from './AsyncDatabase.js';
+import { RemoteConnection } from './RemoteConnection.js';
+import { NodeDatabaseImplementation, NodeSQLOpenOptions } from './options.js';
+
+export type BetterSQLite3LockContext = LockContext & {
+  executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
+};
+
+export type BetterSQLite3Transaction = Transaction & BetterSQLite3LockContext;
+
+const READ_CONNECTIONS = 5;
+
+const defaultDatabaseImplementation: NodeDatabaseImplementation = {
+  type: 'better-sqlite3'
+};
+
+/**
+ * Adapter for better-sqlite3
+ */
+export class WorkerConnectionPool extends BaseObserver<DBAdapterListener> implements DBAdapter {
+  private readonly options: NodeSQLOpenOptions;
+  public readonly name: string;
+
+  private readConnections: RemoteConnection[];
+  private writeConnection: RemoteConnection;
+
+  private readonly readQueue: Array<(connection: RemoteConnection) => void> = [];
+  private readonly writeQueue: Array<() => void> = [];
+
+  constructor(options: NodeSQLOpenOptions) {
+    super();
+
+    if (options.readWorkerCount != null && options.readWorkerCount < 1) {
+      throw `Needs at least one worker for reads, got ${options.readWorkerCount}`;
+    }
+
+    this.options = options;
+    this.name = options.dbFilename;
+  }
+
+  async initialize() {
+    let dbFilePath = this.options.dbFilename;
+    if (this.options.dbLocation !== undefined) {
+      // Make sure the dbLocation exists, we get a TypeError from better-sqlite3 otherwise.
+      let directoryExists = false;
+      try {
+        const stat = await fs.stat(this.options.dbLocation);
+        directoryExists = stat.isDirectory();
+      } catch (_) {
+        // If we can't even stat, the directory won't be accessible to SQLite either.
+      }
+
+      if (!directoryExists) {
+        throw new Error(
+          `The dbLocation directory at "${this.options.dbLocation}" does not exist. Please create it before opening the PowerSync database!`
+        );
+      }
+
+      dbFilePath = path.join(this.options.dbLocation, dbFilePath);
+    }
+
+    const openWorker = async (isWriter: boolean) => {
+      const isCommonJsModule = isBundledToCommonJs;
+      let worker: Worker;
+      const workerName = isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`;
+
+      const workerFactory = this.options.openWorker ?? ((...args) => new Worker(...args));
+      if (isCommonJsModule) {
+        worker = workerFactory(path.resolve(__dirname, 'DefaultWorker.cjs'), { name: workerName });
+      } else {
+        worker = workerFactory(new URL('./DefaultWorker.js', import.meta.url), { name: workerName });
+      }
+
+      const listeners = new WeakMap<EventListenerOrEventListenerObject, (e: any) => void>();
+
+      const comlink = Comlink.wrap<AsyncDatabaseOpener>({
+        postMessage: worker.postMessage.bind(worker),
+        addEventListener: (type, listener) => {
+          let resolved: (event: any) => void =
+            'handleEvent' in listener ? listener.handleEvent.bind(listener) : listener;
+
+          // Comlink wants message events, but the message event on workers in Node returns the data only.
+          if (type === 'message') {
+            const original = resolved;
+
+            resolved = (data) => {
+              original({ data });
+            };
+          }
+
+          listeners.set(listener, resolved);
+          worker.addListener(type, resolved);
+        },
+        removeEventListener: (type, listener) => {
+          const resolved = listeners.get(listener);
+          if (!resolved) {
+            return;
+          }
+          worker.removeListener(type, resolved);
+        }
+      });
+
+      worker.once('error', (e) => {
+        console.error('Unexpected PowerSync database worker error', e);
+      });
+
+      const database = (await comlink.open({
+        path: dbFilePath,
+        isWriter,
+        implementation: this.options.implementation ?? defaultDatabaseImplementation
+      })) as Remote<AsyncDatabase>;
+      if (isWriter) {
+        await database.execute("SELECT powersync_update_hooks('install');", []);
+      }
+
+      const connection = new RemoteConnection(worker, comlink, database);
+      if (this.options.initializeConnection) {
+        await this.options.initializeConnection(connection, isWriter);
+      }
+      if (!isWriter) {
+        await connection.execute('pragma query_only = true');
+      } else {
+        // We only need to enable this on the writer connection.
+        // We can get `database is locked` errors if we enable this on concurrently opening read connections.
+        await connection.execute('pragma journal_mode = WAL');
+      }
+
+      return connection;
+    };
+
+    // Open the writer first to avoid multiple threads enabling WAL concurrently (causing "database is locked" errors).
+    this.writeConnection = await openWorker(true);
+    const createWorkers: Promise<RemoteConnection>[] = [];
+    const amountOfReaders = this.options.readWorkerCount ?? READ_CONNECTIONS;
+    for (let i = 0; i < amountOfReaders; i++) {
+      createWorkers.push(openWorker(false));
+    }
+    this.readConnections = await Promise.all(createWorkers);
+  }
+
+  async close() {
+    await this.writeConnection.close();
+    for (const connection of this.readConnections) {
+      await connection.close();
+    }
+  }
+
+  readLock<T>(fn: (tx: BetterSQLite3LockContext) => Promise<T>, _options?: DBLockOptions | undefined): Promise<T> {
+    let resolveConnectionPromise!: (connection: RemoteConnection) => void;
+    const connectionPromise = new Promise<RemoteConnection>((resolve, _reject) => {
+      resolveConnectionPromise = AsyncResource.bind(resolve);
+    });
+
+    const connection = this.readConnections.find((connection) => !connection.isBusy);
+    if (connection) {
+      connection.isBusy = true;
+      resolveConnectionPromise(connection);
+    } else {
+      this.readQueue.push(resolveConnectionPromise);
+    }
+
+    return (async () => {
+      const connection = await connectionPromise;
+
+      try {
+        return await fn(connection);
+      } finally {
+        const next = this.readQueue.shift();
+        if (next) {
+          next(connection);
+        } else {
+          connection.isBusy = false;
+        }
+      }
+    })();
+  }
+
+  writeLock<T>(fn: (tx: BetterSQLite3LockContext) => Promise<T>, _options?: DBLockOptions | undefined): Promise<T> {
+    let resolveLockPromise!: () => void;
+    const lockPromise = new Promise<void>((resolve, _reject) => {
+      resolveLockPromise = AsyncResource.bind(resolve);
+    });
+
+    if (!this.writeConnection.isBusy) {
+      this.writeConnection.isBusy = true;
+      resolveLockPromise();
+    } else {
+      this.writeQueue.push(resolveLockPromise);
+    }
+
+    return (async () => {
+      await lockPromise;
+
+      try {
+        try {
+          return await fn(this.writeConnection);
+        } finally {
+          const serializedUpdates = await this.writeConnection.executeRaw("SELECT powersync_update_hooks('get');", []);
+          const updates = JSON.parse(serializedUpdates[0][0] as string) as string[];
+
+          if (updates.length > 0) {
+            const event: BatchedUpdateNotification = {
+              tables: updates,
+              groupedUpdates: {},
+              rawUpdates: []
+            };
+            this.iterateListeners((cb) => cb.tablesUpdated?.(event));
+          }
+        }
+      } finally {
+        const next = this.writeQueue.shift();
+        if (next) {
+          next();
+        } else {
+          this.writeConnection.isBusy = false;
+        }
+      }
+    })();
+  }
+
+  readTransaction<T>(
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>,
+    _options?: DBLockOptions | undefined
+  ): Promise<T> {
+    return this.readLock((ctx) => this.internalTransaction(ctx as RemoteConnection, fn));
+  }
+
+  writeTransaction<T>(
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>,
+    _options?: DBLockOptions | undefined
+  ): Promise<T> {
+    return this.writeLock((ctx) => this.internalTransaction(ctx as RemoteConnection, fn));
+  }
+
+  private async internalTransaction<T>(
+    connection: RemoteConnection,
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>
+  ): Promise<T> {
+    let finalized = false;
+    const commit = async (): Promise<QueryResult> => {
+      if (!finalized) {
+        finalized = true;
+        await connection.execute('COMMIT');
+      }
+      return { rowsAffected: 0 };
+    };
+    const rollback = async (): Promise<QueryResult> => {
+      if (!finalized) {
+        finalized = true;
+        await connection.execute('ROLLBACK');
+      }
+      return { rowsAffected: 0 };
+    };
+    try {
+      await connection.execute('BEGIN');
+      const result = await fn({
+        execute: (query, params) => connection.execute(query, params),
+        executeRaw: (query, params) => connection.executeRaw(query, params),
+        executeBatch: (query, params) => connection.executeBatch(query, params),
+        get: (query, params) => connection.get(query, params),
+        getAll: (query, params) => connection.getAll(query, params),
+        getOptional: (query, params) => connection.getOptional(query, params),
+        commit,
+        rollback
+      });
+      await commit();
+      return result;
+    } catch (ex) {
+      try {
+        await rollback();
+      } catch (ex2) {
+        // In rare cases, a rollback may fail.
+        // Safe to ignore.
+      }
+      throw ex;
+    }
+  }
+
+  getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+    return this.readLock((ctx) => ctx.getAll(sql, parameters));
+  }
+
+  getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+    return this.readLock((ctx) => ctx.getOptional(sql, parameters));
+  }
+
+  get<T>(sql: string, parameters?: any[]): Promise<T> {
+    return this.readLock((ctx) => ctx.get(sql, parameters));
+  }
+
+  execute(query: string, params?: any[] | undefined): Promise<QueryResult> {
+    return this.writeLock((ctx) => ctx.execute(query, params));
+  }
+
+  executeRaw(query: string, params?: any[] | undefined): Promise<any[][]> {
+    return this.writeLock((ctx) => ctx.executeRaw(query, params));
+  }
+
+  executeBatch(query: string, params?: any[][]): Promise<QueryResult> {
+    return this.writeTransaction((ctx) => ctx.executeBatch(query, params));
+  }
+
+  async refreshSchema() {
+    await this.writeConnection.refreshSchema();
+
+    for (const readConnection of this.readConnections) {
+      await readConnection.refreshSchema();
+    }
+  }
+}

package/src/db/options.ts

@@ -0,0 +1,51 @@
+import { type Worker } from 'node:worker_threads';
+import { LockContext, SQLOpenOptions } from '@powersync/common';
+
+export type WorkerOpener = (...args: ConstructorParameters<typeof Worker>) => InstanceType<typeof Worker>;
+
+/**
+ * Use the [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) package as a SQLite driver for PowerSync.
+ */
+export interface BetterSqlite3Options {
+  type: 'better-sqlite3';
+}
+
+/**
+ * Use the experimental `node:sqlite` interface as a SQLite driver for PowerSync.
+ *
+ * Note that this option is not currently tested and highly unstable.
+ */
+export interface NodeSqliteOptions {
+  type: 'node:sqlite';
+}
+
+export type NodeDatabaseImplementation = BetterSqlite3Options | NodeSqliteOptions;
+
+/**
+ * The {@link SQLOpenOptions} available across all PowerSync SDKs for JavaScript extended with
+ * Node.JS-specific options.
+ */
+export interface NodeSQLOpenOptions extends SQLOpenOptions {
+  implementation?: NodeDatabaseImplementation;
+
+  /**
+   * The Node.JS SDK will use one worker to run writing queries and additional workers to run reads.
+   * This option controls how many workers to use for reads.
+   */
+  readWorkerCount?: number;
+  /**
+   * A callback to allow customizing how the Node.JS SDK loads workers. This can be customized to
+   * use workers at different paths.
+   *
+   * @param args The arguments that would otherwise be passed to the {@link Worker} constructor.
+   * @returns the resolved worker.
+   */
+  openWorker?: WorkerOpener;
+
+  /**
+   * Initializes a created database connection.
+   *
+   * This can be used to e.g. set encryption keys, if an encrypted database should be used.
+   */
+  initializeConnection?: (db: LockContext, isWriter: boolean) => Promise<void>;
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+// Re export to only require one import in client side code
+export * from '@powersync/common';
+
+export * from './db/PowerSyncDatabase.js';