@powersync/node 0.13.0 → 0.14.1

package/src/db/SqliteWorker.ts

@@ -0,0 +1,119 @@
+import * as Comlink from 'comlink';
+import OS from 'node:os';
+import * as path from 'node:path';
+import url from 'node:url';
+import { parentPort } from 'node:worker_threads';
+import { dynamicImport, isBundledToCommonJs } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { openDatabase as openBetterSqliteDatabase } from './BetterSqliteWorker.js';
+import { openDatabase as openNodeDatabase } from './NodeSqliteWorker.js';
+
+export interface PowerSyncWorkerOptions {
+  /**
+   * A function responsible for finding the powersync DLL/so/dylib file.
+   *
+   * @returns The absolute path of the PowerSync SQLite core extensions library.
+   */
+  extensionPath: () => string;
+
+  /**
+   * A function that returns the `Database` constructor from the `better-sqlite3` package.
+   */
+  loadBetterSqlite3: () => Promise<any>;
+}
+
+/**
+ * @returns The relevant PowerSync extension binary filename for the current platform and architecture
+ */
+export function getPowerSyncExtensionFilename() {
+  const platform = OS.platform();
+  const arch = OS.arch();
+  let extensionFile: string;
+
+  if (platform == 'win32') {
+    if (arch == 'x64') {
+      extensionFile = 'powersync_x64.dll';
+    } else if (arch == 'ia32') {
+      extensionFile = 'powersync_x86.dll';
+    } else if (arch == 'arm64') {
+      extensionFile = 'powersync_aarch64.dll';
+    } else {
+      throw new Error('Windows platform only supports arm64, ia32 and x64 architecture.');
+    }
+  } else if (platform == 'linux') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.so';
+    } else if (arch == 'arm64') {
+      // TODO detect armv7 as an option
+      extensionFile = 'libpowersync_aarch64.so';
+    } else if (arch == 'riscv64') {
+      extensionFile = 'libpowersync_riscv64gc.so';
+    } else {
+      throw new Error('Linux platform only supports x64, arm64 and riscv64 architectures.');
+    }
+  } else if (platform == 'darwin') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.dylib';
+    } else if (arch == 'arm64') {
+      extensionFile = 'libpowersync_aarch64.dylib';
+    } else {
+      throw new Error('macOS platform only supports x64 and arm64 architectures.');
+    }
+  } else {
+    throw new Error(
+      `Unknown platform: ${platform}, PowerSync for Node.js currently supports Windows, Linux and macOS.`
+    );
+  }
+
+  return extensionFile;
+}
+
+export function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>) {
+  const resolvedOptions: PowerSyncWorkerOptions = {
+    extensionPath() {
+      const isCommonJsModule = isBundledToCommonJs;
+      const extensionFilename = getPowerSyncExtensionFilename();
+      let resolved: string;
+      if (isCommonJsModule) {
+        resolved = path.resolve(__dirname, '../lib/', extensionFilename);
+      } else {
+        resolved = url.fileURLToPath(new URL(`../${extensionFilename}`, import.meta.url));
+      }
+
+      return resolved;
+    },
+    async loadBetterSqlite3() {
+      const module = await dynamicImport('better-sqlite3');
+      return module.default;
+    },
+    ...options
+  };
+
+  Comlink.expose(new DatabaseOpenHelper(resolvedOptions), parentPort! as Comlink.Endpoint);
+}
+
+class DatabaseOpenHelper implements AsyncDatabaseOpener {
+  private options: PowerSyncWorkerOptions;
+
+  constructor(options: PowerSyncWorkerOptions) {
+    this.options = options;
+  }
+
+  async open(options: AsyncDatabaseOpenOptions): Promise<AsyncDatabase> {
+    let database: AsyncDatabase;
+
+    const implementation = options.implementation;
+    switch (implementation.type) {
+      case 'better-sqlite3':
+        database = await openBetterSqliteDatabase(this.options, options);
+        break;
+      case 'node:sqlite':
+        database = await openNodeDatabase(this.options, options);
+        break;
+      default:
+        throw new Error(`Unknown database implementation: ${options.implementation}.`);
+    }
+
+    return Comlink.proxy(database);
+  }
+}

package/src/db/WorkerConnectionPool.ts

@@ -0,0 +1,328 @@
+import * as Comlink from 'comlink';
+import fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { Worker } from 'node:worker_threads';
+
+import {
+  BaseObserver,
+  BatchedUpdateNotification,
+  DBAdapter,
+  DBAdapterListener,
+  DBLockOptions,
+  LockContext,
+  QueryResult,
+  Transaction
+} from '@powersync/common';
+import { Remote } from 'comlink';
+import { AsyncResource } from 'node:async_hooks';
+import { isBundledToCommonJs } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpener } from './AsyncDatabase.js';
+import { RemoteConnection } from './RemoteConnection.js';
+import { NodeDatabaseImplementation, NodeSQLOpenOptions } from './options.js';
+
+export type BetterSQLite3LockContext = LockContext & {
+  executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
+};
+
+export type BetterSQLite3Transaction = Transaction & BetterSQLite3LockContext;
+
+const READ_CONNECTIONS = 5;
+
+const defaultDatabaseImplementation: NodeDatabaseImplementation = {
+  type: 'better-sqlite3'
+};
+
+/**
+ * Adapter for better-sqlite3
+ */
+export class WorkerConnectionPool extends BaseObserver<DBAdapterListener> implements DBAdapter {
+  private readonly options: NodeSQLOpenOptions;
+  public readonly name: string;
+
+  private readConnections: RemoteConnection[];
+  private writeConnection: RemoteConnection;
+
+  private readonly readQueue: Array<(connection: RemoteConnection) => void> = [];
+  private readonly writeQueue: Array<() => void> = [];
+
+  constructor(options: NodeSQLOpenOptions) {
+    super();
+
+    if (options.readWorkerCount != null && options.readWorkerCount < 1) {
+      throw `Needs at least one worker for reads, got ${options.readWorkerCount}`;
+    }
+
+    this.options = options;
+    this.name = options.dbFilename;
+  }
+
+  async initialize() {
+    let dbFilePath = this.options.dbFilename;
+    if (this.options.dbLocation !== undefined) {
+      // Make sure the dbLocation exists, we get a TypeError from better-sqlite3 otherwise.
+      let directoryExists = false;
+      try {
+        const stat = await fs.stat(this.options.dbLocation);
+        directoryExists = stat.isDirectory();
+      } catch (_) {
+        // If we can't even stat, the directory won't be accessible to SQLite either.
+      }
+
+      if (!directoryExists) {
+        throw new Error(
+          `The dbLocation directory at "${this.options.dbLocation}" does not exist. Please create it before opening the PowerSync database!`
+        );
+      }
+
+      dbFilePath = path.join(this.options.dbLocation, dbFilePath);
+    }
+
+    const openWorker = async (isWriter: boolean) => {
+      const isCommonJsModule = isBundledToCommonJs;
+      let worker: Worker;
+      const workerName = isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`;
+
+      const workerFactory = this.options.openWorker ?? ((...args) => new Worker(...args));
+      if (isCommonJsModule) {
+        worker = workerFactory(path.resolve(__dirname, 'DefaultWorker.cjs'), { name: workerName });
+      } else {
+        worker = workerFactory(new URL('./DefaultWorker.js', import.meta.url), { name: workerName });
+      }
+
+      const listeners = new WeakMap<EventListenerOrEventListenerObject, (e: any) => void>();
+
+      const comlink = Comlink.wrap<AsyncDatabaseOpener>({
+        postMessage: worker.postMessage.bind(worker),
+        addEventListener: (type, listener) => {
+          let resolved: (event: any) => void =
+            'handleEvent' in listener ? listener.handleEvent.bind(listener) : listener;
+
+          // Comlink wants message events, but the message event on workers in Node returns the data only.
+          if (type === 'message') {
+            const original = resolved;
+
+            resolved = (data) => {
+              original({ data });
+            };
+          }
+
+          listeners.set(listener, resolved);
+          worker.addListener(type, resolved);
+        },
+        removeEventListener: (type, listener) => {
+          const resolved = listeners.get(listener);
+          if (!resolved) {
+            return;
+          }
+          worker.removeListener(type, resolved);
+        }
+      });
+
+      worker.once('error', (e) => {
+        console.error('Unexpected PowerSync database worker error', e);
+      });
+
+      const database = (await comlink.open({
+        path: dbFilePath,
+        isWriter,
+        implementation: this.options.implementation ?? defaultDatabaseImplementation
+      })) as Remote<AsyncDatabase>;
+      if (isWriter) {
+        await database.execute("SELECT powersync_update_hooks('install');", []);
+      }
+
+      const connection = new RemoteConnection(worker, comlink, database);
+      if (this.options.initializeConnection) {
+        await this.options.initializeConnection(connection, isWriter);
+      }
+      if (!isWriter) {
+        await connection.execute('pragma query_only = true');
+      } else {
+        // We only need to enable this on the writer connection.
+        // We can get `database is locked` errors if we enable this on concurrently opening read connections.
+        await connection.execute('pragma journal_mode = WAL');
+      }
+
+      return connection;
+    };
+
+    // Open the writer first to avoid multiple threads enabling WAL concurrently (causing "database is locked" errors).
+    this.writeConnection = await openWorker(true);
+    const createWorkers: Promise<RemoteConnection>[] = [];
+    const amountOfReaders = this.options.readWorkerCount ?? READ_CONNECTIONS;
+    for (let i = 0; i < amountOfReaders; i++) {
+      createWorkers.push(openWorker(false));
+    }
+    this.readConnections = await Promise.all(createWorkers);
+  }
+
+  async close() {
+    await this.writeConnection.close();
+    for (const connection of this.readConnections) {
+      await connection.close();
+    }
+  }
+
+  readLock<T>(fn: (tx: BetterSQLite3LockContext) => Promise<T>, _options?: DBLockOptions | undefined): Promise<T> {
+    let resolveConnectionPromise!: (connection: RemoteConnection) => void;
+    const connectionPromise = new Promise<RemoteConnection>((resolve, _reject) => {
+      resolveConnectionPromise = AsyncResource.bind(resolve);
+    });
+
+    const connection = this.readConnections.find((connection) => !connection.isBusy);
+    if (connection) {
+      connection.isBusy = true;
+      resolveConnectionPromise(connection);
+    } else {
+      this.readQueue.push(resolveConnectionPromise);
+    }
+
+    return (async () => {
+      const connection = await connectionPromise;
+
+      try {
+        return await fn(connection);
+      } finally {
+        const next = this.readQueue.shift();
+        if (next) {
+          next(connection);
+        } else {
+          connection.isBusy = false;
+        }
+      }
+    })();
+  }
+
+  writeLock<T>(fn: (tx: BetterSQLite3LockContext) => Promise<T>, _options?: DBLockOptions | undefined): Promise<T> {
+    let resolveLockPromise!: () => void;
+    const lockPromise = new Promise<void>((resolve, _reject) => {
+      resolveLockPromise = AsyncResource.bind(resolve);
+    });
+
+    if (!this.writeConnection.isBusy) {
+      this.writeConnection.isBusy = true;
+      resolveLockPromise();
+    } else {
+      this.writeQueue.push(resolveLockPromise);
+    }
+
+    return (async () => {
+      await lockPromise;
+
+      try {
+        try {
+          return await fn(this.writeConnection);
+        } finally {
+          const serializedUpdates = await this.writeConnection.executeRaw("SELECT powersync_update_hooks('get');", []);
+          const updates = JSON.parse(serializedUpdates[0][0] as string) as string[];
+
+          if (updates.length > 0) {
+            const event: BatchedUpdateNotification = {
+              tables: updates,
+              groupedUpdates: {},
+              rawUpdates: []
+            };
+            this.iterateListeners((cb) => cb.tablesUpdated?.(event));
+          }
+        }
+      } finally {
+        const next = this.writeQueue.shift();
+        if (next) {
+          next();
+        } else {
+          this.writeConnection.isBusy = false;
+        }
+      }
+    })();
+  }
+
+  readTransaction<T>(
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>,
+    _options?: DBLockOptions | undefined
+  ): Promise<T> {
+    return this.readLock((ctx) => this.internalTransaction(ctx as RemoteConnection, fn));
+  }
+
+  writeTransaction<T>(
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>,
+    _options?: DBLockOptions | undefined
+  ): Promise<T> {
+    return this.writeLock((ctx) => this.internalTransaction(ctx as RemoteConnection, fn));
+  }
+
+  private async internalTransaction<T>(
+    connection: RemoteConnection,
+    fn: (tx: BetterSQLite3Transaction) => Promise<T>
+  ): Promise<T> {
+    let finalized = false;
+    const commit = async (): Promise<QueryResult> => {
+      if (!finalized) {
+        finalized = true;
+        await connection.execute('COMMIT');
+      }
+      return { rowsAffected: 0 };
+    };
+    const rollback = async (): Promise<QueryResult> => {
+      if (!finalized) {
+        finalized = true;
+        await connection.execute('ROLLBACK');
+      }
+      return { rowsAffected: 0 };
+    };
+    try {
+      await connection.execute('BEGIN');
+      const result = await fn({
+        execute: (query, params) => connection.execute(query, params),
+        executeRaw: (query, params) => connection.executeRaw(query, params),
+        executeBatch: (query, params) => connection.executeBatch(query, params),
+        get: (query, params) => connection.get(query, params),
+        getAll: (query, params) => connection.getAll(query, params),
+        getOptional: (query, params) => connection.getOptional(query, params),
+        commit,
+        rollback
+      });
+      await commit();
+      return result;
+    } catch (ex) {
+      try {
+        await rollback();
+      } catch (ex2) {
+        // In rare cases, a rollback may fail.
+        // Safe to ignore.
+      }
+      throw ex;
+    }
+  }
+
+  getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+    return this.readLock((ctx) => ctx.getAll(sql, parameters));
+  }
+
+  getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+    return this.readLock((ctx) => ctx.getOptional(sql, parameters));
+  }
+
+  get<T>(sql: string, parameters?: any[]): Promise<T> {
+    return this.readLock((ctx) => ctx.get(sql, parameters));
+  }
+
+  execute(query: string, params?: any[] | undefined): Promise<QueryResult> {
+    return this.writeLock((ctx) => ctx.execute(query, params));
+  }
+
+  executeRaw(query: string, params?: any[] | undefined): Promise<any[][]> {
+    return this.writeLock((ctx) => ctx.executeRaw(query, params));
+  }
+
+  executeBatch(query: string, params?: any[][]): Promise<QueryResult> {
+    return this.writeTransaction((ctx) => ctx.executeBatch(query, params));
+  }
+
+  async refreshSchema() {
+    await this.writeConnection.refreshSchema();
+
+    for (const readConnection of this.readConnections) {
+      await readConnection.refreshSchema();
+    }
+  }
+}

package/src/db/options.ts

@@ -0,0 +1,51 @@
+import { type Worker } from 'node:worker_threads';
+import { LockContext, SQLOpenOptions } from '@powersync/common';
+
+export type WorkerOpener = (...args: ConstructorParameters<typeof Worker>) => InstanceType<typeof Worker>;
+
+/**
+ * Use the [better-sqlite3](https://github.com/WiseLibs/better-sqlite3) package as a SQLite driver for PowerSync.
+ */
+export interface BetterSqlite3Options {
+  type: 'better-sqlite3';
+}
+
+/**
+ * Use the experimental `node:sqlite` interface as a SQLite driver for PowerSync.
+ *
+ * Note that this option is not currently tested and highly unstable.
+ */
+export interface NodeSqliteOptions {
+  type: 'node:sqlite';
+}
+
+export type NodeDatabaseImplementation = BetterSqlite3Options | NodeSqliteOptions;
+
+/**
+ * The {@link SQLOpenOptions} available across all PowerSync SDKs for JavaScript extended with
+ * Node.JS-specific options.
+ */
+export interface NodeSQLOpenOptions extends SQLOpenOptions {
+  implementation?: NodeDatabaseImplementation;
+
+  /**
+   * The Node.JS SDK will use one worker to run writing queries and additional workers to run reads.
+   * This option controls how many workers to use for reads.
+   */
+  readWorkerCount?: number;
+  /**
+   * A callback to allow customizing how the Node.JS SDK loads workers. This can be customized to
+   * use workers at different paths.
+   *
+   * @param args The arguments that would otherwise be passed to the {@link Worker} constructor.
+   * @returns the resolved worker.
+   */
+  openWorker?: WorkerOpener;
+
+  /**
+   * Initializes a created database connection.
+   *
+   * This can be used to e.g. set encryption keys, if an encrypted database should be used.
+   */
+  initializeConnection?: (db: LockContext, isWriter: boolean) => Promise<void>;
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+// Re export to only require one import in client side code
+export * from '@powersync/common';
+
+export * from './db/PowerSyncDatabase.js';

package/src/sync/stream/NodeRemote.ts

@@ -0,0 +1,123 @@
+import * as os from 'node:os';
+
+import {
+  type ILogger,
+  AbstractRemote,
+  AbstractRemoteOptions,
+  BSONImplementation,
+  DEFAULT_REMOTE_LOGGER,
+  FetchImplementation,
+  FetchImplementationProvider,
+  RemoteConnector
+} from '@powersync/common';
+import { BSON } from 'bson';
+import {
+  Dispatcher,
+  EnvHttpProxyAgent,
+  ErrorEvent,
+  getGlobalDispatcher,
+  ProxyAgent,
+  WebSocket as UndiciWebSocket
+} from 'undici';
+
+export const STREAMING_POST_TIMEOUT_MS = 30_000;
+
+class NodeFetchProvider extends FetchImplementationProvider {
+  getFetch(): FetchImplementation {
+    return fetch.bind(globalThis);
+  }
+}
+
+export type NodeCustomConnectionOptions = {
+  /**
+   * Optional custom dispatcher for HTTP or WEB_SOCKET connections.
+   *
+   * This can be used to customize proxy usage (using undici ProxyAgent),
+   * or other connection options.
+   */
+  dispatcher?: Dispatcher;
+};
+
+export type NodeRemoteOptions = AbstractRemoteOptions & NodeCustomConnectionOptions;
+
+export class NodeRemote extends AbstractRemote {
+  private wsDispatcher: Dispatcher | undefined;
+
+  constructor(
+    protected connector: RemoteConnector,
+    protected logger: ILogger = DEFAULT_REMOTE_LOGGER,
+    options?: Partial<NodeRemoteOptions>
+  ) {
+    const fetchDispatcher = options?.dispatcher ?? defaultFetchDispatcher();
+
+    super(connector, logger, {
+      fetchImplementation: options?.fetchImplementation ?? new NodeFetchProvider(),
+      fetchOptions: {
+        dispatcher: fetchDispatcher
+      },
+      ...(options ?? {})
+    });
+
+    this.wsDispatcher = options?.dispatcher;
+  }
+
+  protected createSocket(url: string): globalThis.WebSocket {
+    // Create dedicated dispatcher for this WebSocket
+    const baseDispatcher = this.getWebsocketDispatcher(url);
+
+    // Create WebSocket with dedicated dispatcher
+    const ws = new UndiciWebSocket(url, {
+      dispatcher: baseDispatcher,
+      headers: {
+        'User-Agent': this.getUserAgent()
+      }
+    });
+
+    return ws as globalThis.WebSocket;
+  }
+
+  protected getWebsocketDispatcher(url: string) {
+    if (this.wsDispatcher != null) {
+      return this.wsDispatcher;
+    }
+
+    const protocol = new URL(url).protocol.replace(':', '');
+    const proxy = getProxyForProtocol(protocol);
+    if (proxy != null) {
+      return new ProxyAgent(proxy);
+    } else {
+      return getGlobalDispatcher();
+    }
+  }
+
+  getUserAgent(): string {
+    return [
+      super.getUserAgent(),
+      `powersync-node`,
+      `node/${process.versions.node}`,
+      `${os.platform()}/${os.release()}`
+    ].join(' ');
+  }
+
+  async getBSON(): Promise<BSONImplementation> {
+    return BSON;
+  }
+}
+
+function defaultFetchDispatcher(): Dispatcher {
+  // EnvHttpProxyAgent automatically uses HTTP_PROXY, HTTPS_PROXY and NO_PROXY env vars by default.
+  // We add ALL_PROXY support.
+  return new EnvHttpProxyAgent({
+    httpProxy: getProxyForProtocol('http'),
+    httpsProxy: getProxyForProtocol('https')
+  });
+}
+
+function getProxyForProtocol(protocol: string): string | undefined {
+  return (
+    process.env[`${protocol.toLowerCase()}_proxy`] ??
+    process.env[`${protocol.toUpperCase()}_PROXY`] ??
+    process.env[`all_proxy`] ??
+    process.env[`ALL_PROXY`]
+  );
+}

package/src/sync/stream/NodeStreamingSyncImplementation.ts

@@ -0,0 +1,57 @@
+import {
+  AbstractStreamingSyncImplementation,
+  AbstractStreamingSyncImplementationOptions,
+  LockOptions,
+  LockType
+} from '@powersync/common';
+import { Mutex } from 'async-mutex';
+
+/**
+ * Global locks which prevent multiple instances from syncing
+ * concurrently.
+ */
+const LOCKS = new Map<string, Map<LockType, Mutex>>();
+
+const lockTypes = new Set(Object.values(LockType));
+
+export class NodeStreamingSyncImplementation extends AbstractStreamingSyncImplementation {
+  locks: Map<LockType, Mutex>;
+
+  constructor(options: AbstractStreamingSyncImplementationOptions) {
+    super(options);
+    this.initLocks();
+  }
+
+  /**
+   *  Configures global locks on sync process
+   */
+  initLocks() {
+    const { identifier } = this.options;
+    if (identifier && LOCKS.has(identifier)) {
+      this.locks = LOCKS.get(identifier)!;
+      return;
+    }
+
+    this.locks = new Map<LockType, Mutex>();
+    this.locks.set(LockType.CRUD, new Mutex());
+    this.locks.set(LockType.SYNC, new Mutex());
+
+    if (identifier) {
+      LOCKS.set(identifier, this.locks);
+    }
+  }
+
+  obtainLock<T>(lockOptions: LockOptions<T>): Promise<T> {
+    const lock = this.locks.get(lockOptions.type);
+    if (!lock) {
+      throw new Error(`Lock type ${lockOptions.type} not found`);
+    }
+    return lock.runExclusive(async () => {
+      if (lockOptions.signal?.aborted) {
+        throw new Error('Aborted');
+      }
+
+      return lockOptions.callback();
+    });
+  }
+}

package/src/utils/modules.ts

@@ -0,0 +1,6 @@
+// Note: When updating this file, always update module_commonjs.ts as well.
+export const isBundledToCommonJs: boolean = false;
+
+export async function dynamicImport(path: string) {
+  return await import(path);
+}

package/src/utils/modules_commonjs.ts

@@ -0,0 +1,7 @@
+// NOTE! Do not import this file directly! We have a rollup plugin that rewrites imports to modules.js to this file when
+// bundling to CommonJS.
+export const isBundledToCommonJs: boolean = true;
+
+export async function dynamicImport(path: string) {
+  return require(path);
+}

package/src/worker.ts

@@ -0,0 +1 @@
+export * from './db/SqliteWorker.js';