@powersync/node 0.0.0-dev-20251030153614 → 0.0.0-dev-20251106124255

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@powersync/node",
-  "version": "0.0.0-dev-20251030153614",
+  "version": "0.0.0-dev-20251106124255",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
@@ -10,7 +10,8 @@
   "files": [
     "lib",
     "dist",
-    "download_core.js"
+    "download_core.js",
+    "src"
   ],
   "type": "module",
   "exports": {
@@ -47,7 +48,7 @@
   },
   "homepage": "https://docs.powersync.com/",
   "peerDependencies": {
-    "@powersync/common": "0.0.0-dev-20251030153614",
+    "@powersync/common": "0.0.0-dev-20251106124255",
     "better-sqlite3": "12.x"
   },
   "peerDependenciesMeta": {
@@ -56,21 +57,20 @@
     }
   },
   "dependencies": {
-    "async-lock": "^1.4.0",
-    "bson": "^6.6.0",
+    "async-mutex": "^0.5.0",
     "comlink": "^4.4.2",
     "undici": "^7.11.0",
-    "@powersync/common": "0.0.0-dev-20251030153614"
+    "bson": "^6.10.4",
+    "@powersync/common": "0.0.0-dev-20251106124255"
   },
   "devDependencies": {
-    "@types/async-lock": "^1.4.0",
     "@types/node": "^24.2.0",
     "better-sqlite3": "^12.2.0",
     "drizzle-orm": "^0.35.2",
     "rollup": "4.14.3",
     "typescript": "^5.5.3",
     "vitest": "^3.2.4",
-    "@powersync/drizzle-driver": "0.6.0"
+    "@powersync/drizzle-driver": "0.0.0-dev-20251106124255"
   },
   "keywords": [
     "data sync",
@@ -82,7 +82,7 @@
   "scripts": {
     "prepare:core": "node download_core.js",
     "build": " pnpm prepare:core && tsc -b && rollup --config",
-    "build:prod": "pnpm prepare:core && tsc -b --sourceMap false && rollup --config",
+    "build:prod": "pnpm prepare:core && tsc -b && rollup --config",
     "clean": "rm -rf lib dist tsconfig.tsbuildinfo",
     "watch": "tsc -b -w",
     "test": " pnpm prepare:core && vitest",

package/src/db/AsyncDatabase.ts

@@ -0,0 +1,26 @@
+import { QueryResult } from '@powersync/common';
+import { NodeDatabaseImplementation } from './options.js';
+
+export type ProxiedQueryResult = Omit<QueryResult, 'rows'> & {
+  rows?: {
+    _array: any[];
+    length: number;
+  };
+};
+
+export interface AsyncDatabaseOpenOptions {
+  path: string;
+  isWriter: boolean;
+  implementation: NodeDatabaseImplementation;
+}
+
+export interface AsyncDatabaseOpener {
+  open(options: AsyncDatabaseOpenOptions): Promise<AsyncDatabase>;
+}
+
+export interface AsyncDatabase {
+  execute: (query: string, params: any[]) => Promise<ProxiedQueryResult>;
+  executeRaw: (query: string, params: any[]) => Promise<any[][]>;
+  executeBatch: (query: string, params: any[][]) => Promise<ProxiedQueryResult>;
+  close: () => Promise<void>;
+}

package/src/db/BetterSqliteWorker.ts

@@ -0,0 +1,72 @@
+import type { Database } from 'better-sqlite3';
+import { AsyncDatabase, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { PowerSyncWorkerOptions } from './SqliteWorker.js';
+import { threadId } from 'node:worker_threads';
+
+class BlockingAsyncDatabase implements AsyncDatabase {
+  private readonly db: Database;
+
+  constructor(db: Database) {
+    this.db = db;
+
+    db.function('node_thread_id', () => threadId);
+  }
+
+  async close() {
+    this.db.close();
+  }
+
+  async execute(query: string, params: any[]) {
+    const stmt = this.db.prepare(query);
+    if (stmt.reader) {
+      const rows = stmt.all(params);
+      return {
+        rowsAffected: 0,
+        rows: {
+          _array: rows,
+          length: rows.length
+        }
+      };
+    } else {
+      const info = stmt.run(params);
+      return {
+        rowsAffected: info.changes,
+        insertId: Number(info.lastInsertRowid)
+      };
+    }
+  }
+
+  async executeRaw(query: string, params: any[]) {
+    const stmt = this.db.prepare(query);
+
+    if (stmt.reader) {
+      return stmt.raw().all(params) as any[][];
+    } else {
+      stmt.raw().run(params);
+      return [];
+    }
+  }
+
+  async executeBatch(query: string, params: any[][]) {
+    params = params ?? [];
+
+    let rowsAffected = 0;
+
+    const stmt = this.db.prepare(query);
+    for (const paramSet of params) {
+      const info = stmt.run(paramSet);
+      rowsAffected += info.changes;
+    }
+
+    return { rowsAffected };
+  }
+}
+
+export async function openDatabase(worker: PowerSyncWorkerOptions, options: AsyncDatabaseOpenOptions) {
+  const BetterSQLite3Database = await worker.loadBetterSqlite3();
+  const baseDB = new BetterSQLite3Database(options.path);
+  baseDB.loadExtension(worker.extensionPath(), 'sqlite3_powersync_init');
+
+  const asyncDb = new BlockingAsyncDatabase(baseDB);
+  return asyncDb;
+}

package/src/db/DefaultWorker.ts

@@ -0,0 +1,3 @@
+import { startPowerSyncWorker } from './SqliteWorker.js';
+
+startPowerSyncWorker();

package/src/db/NodeSqliteWorker.ts

@@ -0,0 +1,63 @@
+import type { DatabaseSync } from 'node:sqlite';
+import { threadId } from 'node:worker_threads';
+
+import { dynamicImport } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { PowerSyncWorkerOptions } from './SqliteWorker.js';
+
+class BlockingNodeDatabase implements AsyncDatabase {
+  private readonly db: DatabaseSync;
+
+  constructor(db: DatabaseSync, write: boolean) {
+    this.db = db;
+
+    db.function('node_thread_id', () => threadId);
+  }
+
+  async close() {
+    this.db.close();
+  }
+
+  async execute(query: string, params: any[]) {
+    const stmt = this.db.prepare(query);
+    const rows = stmt.all(...params);
+    return {
+      rowsAffected: 0,
+      rows: {
+        _array: rows,
+        length: rows.length
+      }
+    };
+  }
+
+  async executeRaw(query: string, params: any[]) {
+    const stmt = this.db.prepare(query);
+    (stmt as any).setReturnArrays(true); // Missing in @types/node, https://nodejs.org/api/sqlite.html#statementsetreturnarraysenabled
+    return stmt.all(...params) as any as any[][];
+  }
+
+  async executeBatch(query: string, params: any[][]) {
+    params = params ?? [];
+
+    let rowsAffected = 0;
+
+    const stmt = this.db.prepare(query);
+    for (const paramSet of params) {
+      const info = stmt.run(...paramSet);
+      rowsAffected += info.changes as number;
+    }
+
+    return { rowsAffected };
+  }
+}
+
+export async function openDatabase(worker: PowerSyncWorkerOptions, options: AsyncDatabaseOpenOptions) {
+  // NOTE: We want to import node:sqlite dynamically, to avoid bundlers unconditionally requiring node:sqlite in the
+  // end, since that would make us incompatible with older Node.JS versions.
+  const { DatabaseSync } = await dynamicImport('node:sqlite');
+
+  const baseDB = new DatabaseSync(options.path, { allowExtension: true });
+  baseDB.loadExtension(worker.extensionPath(), 'sqlite3_powersync_init');
+
+  return new BlockingNodeDatabase(baseDB, options.isWriter);
+}

package/src/db/PowerSyncDatabase.ts

@@ -0,0 +1,100 @@
+import {
+  AbstractPowerSyncDatabase,
+  AbstractRemoteOptions,
+  AbstractStreamingSyncImplementation,
+  AdditionalConnectionOptions,
+  BucketStorageAdapter,
+  DBAdapter,
+  PowerSyncBackendConnector,
+  PowerSyncConnectionOptions,
+  PowerSyncDatabaseOptions,
+  PowerSyncDatabaseOptionsWithSettings,
+  RequiredAdditionalConnectionOptions,
+  SqliteBucketStorage,
+  SQLOpenFactory
+} from '@powersync/common';
+
+import { NodeCustomConnectionOptions, NodeRemote } from '../sync/stream/NodeRemote.js';
+import { NodeStreamingSyncImplementation } from '../sync/stream/NodeStreamingSyncImplementation.js';
+
+import { WorkerConnectionPool } from './WorkerConnectionPool.js';
+import { NodeSQLOpenOptions } from './options.js';
+
+export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & {
+  database: DBAdapter | SQLOpenFactory | NodeSQLOpenOptions;
+  /**
+   * Options to override how the SDK will connect to the sync service.
+   *
+   * This option is intended to be used for internal tests.
+   */
+  remoteOptions?: Partial<AbstractRemoteOptions>;
+};
+
+export type NodeAdditionalConnectionOptions = AdditionalConnectionOptions & NodeCustomConnectionOptions;
+
+export type NodePowerSyncConnectionOptions = PowerSyncConnectionOptions & NodeAdditionalConnectionOptions;
+
+/**
+ * A PowerSync database which provides SQLite functionality
+ * which is automatically synced.
+ *
+ * @example
+ * ```typescript
+ * export const db = new PowerSyncDatabase({
+ *  schema: AppSchema,
+ *  database: {
+ *    dbFilename: 'example.db'
+ *  }
+ * });
+ * ```
+ */
+export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
+  constructor(options: NodePowerSyncDatabaseOptions) {
+    super(options);
+  }
+
+  async _initialize(): Promise<void> {
+    await (this.database as WorkerConnectionPool).initialize();
+  }
+
+  /**
+   * Opens a DBAdapter using better-sqlite3 as the default SQLite open factory.
+   */
+  protected openDBAdapter(options: PowerSyncDatabaseOptionsWithSettings): DBAdapter {
+    return new WorkerConnectionPool(options.database);
+  }
+
+  protected generateBucketStorageAdapter(): BucketStorageAdapter {
+    return new SqliteBucketStorage(this.database, this.logger);
+  }
+
+  connect(
+    connector: PowerSyncBackendConnector,
+    options?: PowerSyncConnectionOptions & NodeCustomConnectionOptions
+  ): Promise<void> {
+    return super.connect(connector, options);
+  }
+
+  protected generateSyncStreamImplementation(
+    connector: PowerSyncBackendConnector,
+    options: RequiredAdditionalConnectionOptions & NodeAdditionalConnectionOptions
+  ): AbstractStreamingSyncImplementation {
+    const logger = this.logger;
+    const remote = new NodeRemote(connector, logger, {
+      dispatcher: options.dispatcher,
+      ...(this.options as NodePowerSyncDatabaseOptions).remoteOptions
+    });
+
+    return new NodeStreamingSyncImplementation({
+      adapter: this.bucketStorageAdapter,
+      remote,
+      uploadCrud: async () => {
+        await this.waitForReady();
+        await connector.uploadData(this);
+      },
+      ...options,
+      identifier: this.database.name,
+      logger
+    });
+  }
+}

package/src/db/RemoteConnection.ts

@@ -0,0 +1,105 @@
+import { Worker } from 'node:worker_threads';
+import { LockContext, QueryResult } from '@powersync/common';
+import { releaseProxy, Remote } from 'comlink';
+import { AsyncDatabase, AsyncDatabaseOpener, ProxiedQueryResult } from './AsyncDatabase.js';
+
+/**
+ * A PowerSync database connection implemented with RPC calls to a background worker.
+ */
+export class RemoteConnection implements LockContext {
+  isBusy = false;
+
+  private readonly worker: Worker;
+  private readonly comlink: Remote<AsyncDatabaseOpener>;
+  readonly database: Remote<AsyncDatabase>;
+
+  constructor(worker: Worker, comlink: Remote<AsyncDatabaseOpener>, database: Remote<AsyncDatabase>) {
+    this.worker = worker;
+    this.comlink = comlink;
+    this.database = database;
+  }
+
+  /**
+   * Runs the inner function, but appends the stack trace where this function was called. This is useful for workers
+   * because stack traces from worker errors are otherwise unrelated to the application issue that has caused them.
+   */
+  private async recoverTrace<T>(inner: () => Promise<T>): Promise<T> {
+    const trace = {};
+    Error.captureStackTrace(trace);
+
+    try {
+      return await inner();
+    } catch (e) {
+      if (e instanceof Error && e.stack) {
+        e.stack += (trace as any).stack;
+      }
+
+      throw e;
+    }
+  }
+
+  executeBatch(query: string, params: any[][] = []): Promise<QueryResult> {
+    return this.recoverTrace(async () => {
+      const result = await this.database.executeBatch(query, params ?? []);
+      return RemoteConnection.wrapQueryResult(result);
+    });
+  }
+
+  execute(query: string, params?: any[] | undefined): Promise<QueryResult> {
+    return this.recoverTrace(async () => {
+      const result = await this.database.execute(query, params ?? []);
+      return RemoteConnection.wrapQueryResult(result);
+    });
+  }
+
+  executeRaw(query: string, params?: any[] | undefined): Promise<any[][]> {
+    return this.recoverTrace(async () => {
+      return await this.database.executeRaw(query, params ?? []);
+    });
+  }
+
+  async getAll<T>(sql: string, parameters?: any[]): Promise<T[]> {
+    const res = await this.execute(sql, parameters);
+    return res.rows?._array ?? [];
+  }
+
+  async getOptional<T>(sql: string, parameters?: any[]): Promise<T | null> {
+    const res = await this.execute(sql, parameters);
+    return res.rows?.item(0) ?? null;
+  }
+
+  async get<T>(sql: string, parameters?: any[]): Promise<T> {
+    const res = await this.execute(sql, parameters);
+    const first = res.rows?.item(0);
+    if (!first) {
+      throw new Error('Result set is empty');
+    }
+    return first;
+  }
+
+  async refreshSchema() {
+    await this.execute("pragma table_info('sqlite_master')");
+  }
+
+  async close() {
+    await this.database.close();
+    this.database[releaseProxy]();
+    this.comlink[releaseProxy]();
+    await this.worker.terminate();
+  }
+
+  static wrapQueryResult(result: ProxiedQueryResult): QueryResult {
+    let rows: QueryResult['rows'] | undefined = undefined;
+    if (result.rows) {
+      rows = {
+        ...result.rows,
+        item: (idx) => result.rows?._array[idx]
+      } satisfies QueryResult['rows'];
+    }
+
+    return {
+      ...result,
+      rows
+    };
+  }
+}

package/src/db/SqliteWorker.ts

@@ -0,0 +1,119 @@
+import * as Comlink from 'comlink';
+import OS from 'node:os';
+import * as path from 'node:path';
+import url from 'node:url';
+import { parentPort } from 'node:worker_threads';
+import { dynamicImport, isBundledToCommonJs } from '../utils/modules.js';
+import { AsyncDatabase, AsyncDatabaseOpener, AsyncDatabaseOpenOptions } from './AsyncDatabase.js';
+import { openDatabase as openBetterSqliteDatabase } from './BetterSqliteWorker.js';
+import { openDatabase as openNodeDatabase } from './NodeSqliteWorker.js';
+
+export interface PowerSyncWorkerOptions {
+  /**
+   * A function responsible for finding the powersync DLL/so/dylib file.
+   *
+   * @returns The absolute path of the PowerSync SQLite core extensions library.
+   */
+  extensionPath: () => string;
+
+  /**
+   * A function that returns the `Database` constructor from the `better-sqlite3` package.
+   */
+  loadBetterSqlite3: () => Promise<any>;
+}
+
+/**
+ * @returns The relevant PowerSync extension binary filename for the current platform and architecture
+ */
+export function getPowerSyncExtensionFilename() {
+  const platform = OS.platform();
+  const arch = OS.arch();
+  let extensionFile: string;
+
+  if (platform == 'win32') {
+    if (arch == 'x64') {
+      extensionFile = 'powersync_x64.dll';
+    } else if (arch == 'ia32') {
+      extensionFile = 'powersync_x86.dll';
+    } else if (arch == 'arm64') {
+      extensionFile = 'powersync_aarch64.dll';
+    } else {
+      throw new Error('Windows platform only supports arm64, ia32 and x64 architecture.');
+    }
+  } else if (platform == 'linux') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.so';
+    } else if (arch == 'arm64') {
+      // TODO detect armv7 as an option
+      extensionFile = 'libpowersync_aarch64.so';
+    } else if (arch == 'riscv64') {
+      extensionFile = 'libpowersync_riscv64gc.so';
+    } else {
+      throw new Error('Linux platform only supports x64, arm64 and riscv64 architectures.');
+    }
+  } else if (platform == 'darwin') {
+    if (arch == 'x64') {
+      extensionFile = 'libpowersync_x64.dylib';
+    } else if (arch == 'arm64') {
+      extensionFile = 'libpowersync_aarch64.dylib';
+    } else {
+      throw new Error('macOS platform only supports x64 and arm64 architectures.');
+    }
+  } else {
+    throw new Error(
+      `Unknown platform: ${platform}, PowerSync for Node.js currently supports Windows, Linux and macOS.`
+    );
+  }
+
+  return extensionFile;
+}
+
+export function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>) {
+  const resolvedOptions: PowerSyncWorkerOptions = {
+    extensionPath() {
+      const isCommonJsModule = isBundledToCommonJs;
+      const extensionFilename = getPowerSyncExtensionFilename();
+      let resolved: string;
+      if (isCommonJsModule) {
+        resolved = path.resolve(__dirname, '../lib/', extensionFilename);
+      } else {
+        resolved = url.fileURLToPath(new URL(`../${extensionFilename}`, import.meta.url));
+      }
+
+      return resolved;
+    },
+    async loadBetterSqlite3() {
+      const module = await dynamicImport('better-sqlite3');
+      return module.default;
+    },
+    ...options
+  };
+
+  Comlink.expose(new DatabaseOpenHelper(resolvedOptions), parentPort! as Comlink.Endpoint);
+}
+
+class DatabaseOpenHelper implements AsyncDatabaseOpener {
+  private options: PowerSyncWorkerOptions;
+
+  constructor(options: PowerSyncWorkerOptions) {
+    this.options = options;
+  }
+
+  async open(options: AsyncDatabaseOpenOptions): Promise<AsyncDatabase> {
+    let database: AsyncDatabase;
+
+    const implementation = options.implementation;
+    switch (implementation.type) {
+      case 'better-sqlite3':
+        database = await openBetterSqliteDatabase(this.options, options);
+        break;
+      case 'node:sqlite':
+        database = await openNodeDatabase(this.options, options);
+        break;
+      default:
+        throw new Error(`Unknown database implementation: ${options.implementation}.`);
+    }
+
+    return Comlink.proxy(database);
+  }
+}