@powersync/node 0.1.1 → 0.2.1

package/README.md

@@ -26,6 +26,27 @@ npm install @powersync/node
 Both `@powersync/node` and the `better-sqlite3` packages have install scripts that need to run to compile
 or download sqlite3 and PowerSync binaries.
 
+### Common Installation Issues
+
+The `better-sqlite` package requires native compilation, which depends on certain system tools. This compilation process is handled by `node-gyp` and may fail if required dependencies are missing or misconfigured.
+
+#### Node-gyp Version Conflicts
+
+`better-sqlite` depends on `node-gyp@^11`, but some project configurations may introduce multiple versions of `node-gyp`, potentially causing build issues.
+
+#### Python Dependency Issues
+
+`node-gyp` requires Python for compilation. If your project uses `node-gyp` below version `10` and your system has Python `3.12` or later, you may encounter the following error:
+
+```python
+ModuleNotFoundError: No module named 'distutils'
+```
+
+To resolve this, either:
+
+- Upgrade `node-gyp` to version 10 or later.
+- Install Python [setuptools](https://pypi.org/project/setuptools/), which includes `distutils`.
+
 # Getting Started
 
 The [Node.js SDK reference](https://docs.powersync.com/client-sdk-references/node)

package/lib/db/AsyncDatabase.d.ts

@@ -10,6 +10,7 @@ export interface AsyncDatabaseOpener {
 }
 export interface AsyncDatabase {
     execute: (query: string, params: any[]) => Promise<ProxiedQueryResult>;
+    executeRaw: (query: string, params: any[]) => Promise<any[][]>;
     executeBatch: (query: string, params: any[][]) => Promise<ProxiedQueryResult>;
     close: () => Promise<void>;
     collectCommittedUpdates: () => Promise<string[]>;

package/lib/db/BetterSQLite3DBAdapter.d.ts

@@ -1,4 +1,5 @@
-import { BaseObserver, DBAdapter, DBAdapterListener, LockContext, Transaction, DBLockOptions, QueryResult, SQLOpenOptions } from '@powersync/common';
+import { BaseObserver, DBAdapter, DBAdapterListener, LockContext, Transaction, DBLockOptions, QueryResult } from '@powersync/common';
+import { NodeSQLOpenOptions } from './options.js';
 export type BetterSQLite3LockContext = LockContext & {
     executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
 };
@@ -13,7 +14,7 @@ export declare class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListen
     private writeConnection;
     private readonly readQueue;
     private readonly writeQueue;
-    constructor(options: SQLOpenOptions);
+    constructor(options: NodeSQLOpenOptions);
     initialize(): Promise<void>;
     close(): Promise<void>;
     readLock<T>(fn: (tx: BetterSQLite3LockContext) => Promise<T>, _options?: DBLockOptions | undefined): Promise<T>;
@@ -25,6 +26,7 @@ export declare class BetterSQLite3DBAdapter extends BaseObserver<DBAdapterListen
     getOptional<T>(sql: string, parameters?: any[]): Promise<T | null>;
     get<T>(sql: string, parameters?: any[]): Promise<T>;
     execute(query: string, params?: any[] | undefined): Promise<QueryResult>;
+    executeRaw(query: string, params?: any[] | undefined): Promise<any[][]>;
     executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
     refreshSchema(): Promise<void>;
 }

package/lib/db/BetterSQLite3DBAdapter.js

@@ -1,3 +1,4 @@
+import fs from 'node:fs/promises';
 import * as path from 'node:path';
 import { Worker } from 'node:worker_threads';
 import * as Comlink from 'comlink';
@@ -17,16 +18,40 @@ export class BetterSQLite3DBAdapter extends BaseObserver {
     writeQueue = [];
     constructor(options) {
         super();
+        if (options.readWorkerCount != null && options.readWorkerCount < 1) {
+            throw `Needs at least one worker for reads, got ${options.readWorkerCount}`;
+        }
         this.options = options;
         this.name = options.dbFilename;
     }
     async initialize() {
         let dbFilePath = this.options.dbFilename;
         if (this.options.dbLocation !== undefined) {
+            // Make sure the dbLocation exists, we get a TypeError from better-sqlite3 otherwise.
+            let directoryExists = false;
+            try {
+                const stat = await fs.stat(this.options.dbLocation);
+                directoryExists = stat.isDirectory();
+            }
+            catch (_) {
+                // If we can't even stat, the directory won't be accessible to SQLite either.
+            }
+            if (!directoryExists) {
+                throw new Error(`The dbLocation directory at "${this.options.dbLocation}" does not exist. Please create it before opening the PowerSync database!`);
+            }
             dbFilePath = path.join(this.options.dbLocation, dbFilePath);
         }
         const openWorker = async (isWriter) => {
-            const worker = new Worker(new URL('./SqliteWorker.js', import.meta.url), { name: isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}` });
+            const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+            let worker;
+            const workerName = isWriter ? `write ${dbFilePath}` : `read ${dbFilePath}`;
+            const workerFactory = this.options.openWorker ?? ((...args) => new Worker(...args));
+            if (isCommonJsModule) {
+                worker = workerFactory(path.resolve(__dirname, 'DefaultWorker.cjs'), { name: workerName });
+            }
+            else {
+                worker = workerFactory(new URL('./DefaultWorker.js', import.meta.url), { name: workerName });
+            }
             const listeners = new WeakMap();
             const comlink = Comlink.wrap({
                 postMessage: worker.postMessage.bind(worker),
@@ -59,7 +84,8 @@ export class BetterSQLite3DBAdapter extends BaseObserver {
         // Open the writer first to avoid multiple threads enabling WAL concurrently (causing "database is locked" errors).
         this.writeConnection = await openWorker(true);
         const createWorkers = [];
-        for (let i = 0; i < READ_CONNECTIONS; i++) {
+        const amountOfReaders = this.options.readWorkerCount ?? READ_CONNECTIONS;
+        for (let i = 0; i < amountOfReaders; i++) {
             createWorkers.push(openWorker(false));
         }
         this.readConnections = await Promise.all(createWorkers);
@@ -166,6 +192,7 @@ export class BetterSQLite3DBAdapter extends BaseObserver {
             await connection.execute('BEGIN');
             const result = await fn({
                 execute: (query, params) => connection.execute(query, params),
+                executeRaw: (query, params) => connection.executeRaw(query, params),
                 executeBatch: (query, params) => connection.executeBatch(query, params),
                 get: (query, params) => connection.get(query, params),
                 getAll: (query, params) => connection.getAll(query, params),
@@ -199,6 +226,9 @@ export class BetterSQLite3DBAdapter extends BaseObserver {
     execute(query, params) {
         return this.writeLock((ctx) => ctx.execute(query, params));
     }
+    executeRaw(query, params) {
+        return this.writeLock((ctx) => ctx.executeRaw(query, params));
+    }
     executeBatch(query, params) {
         return this.writeTransaction((ctx) => ctx.executeBatch(query, params));
     }

package/lib/db/DefaultWorker.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/db/DefaultWorker.js

@@ -0,0 +1,2 @@
+import { startPowerSyncWorker } from './SqliteWorker.js';
+startPowerSyncWorker();

package/lib/db/PowerSyncDatabase.d.ts

@@ -1,4 +1,8 @@
-import { AbstractPowerSyncDatabase, AbstractStreamingSyncImplementation, BucketStorageAdapter, DBAdapter, PowerSyncBackendConnector, PowerSyncDatabaseOptionsWithSettings } from '@powersync/common';
+import { AbstractPowerSyncDatabase, AbstractStreamingSyncImplementation, BucketStorageAdapter, DBAdapter, PowerSyncBackendConnector, PowerSyncDatabaseOptions, PowerSyncDatabaseOptionsWithSettings, SQLOpenFactory } from '@powersync/common';
+import { NodeSQLOpenOptions } from './options.js';
+export type NodePowerSyncDatabaseOptions = PowerSyncDatabaseOptions & {
+    database: DBAdapter | SQLOpenFactory | NodeSQLOpenOptions;
+};
 /**
  * A PowerSync database which provides SQLite functionality
  * which is automatically synced.
@@ -14,6 +18,7 @@ import { AbstractPowerSyncDatabase, AbstractStreamingSyncImplementation, BucketS
  * ```
  */
 export declare class PowerSyncDatabase extends AbstractPowerSyncDatabase {
+    constructor(options: NodePowerSyncDatabaseOptions);
     _initialize(): Promise<void>;
     /**
      * Opens a DBAdapter using better-sqlite3 as the default SQLite open factory.

package/lib/db/PowerSyncDatabase.js

@@ -17,6 +17,9 @@ import { BetterSQLite3DBAdapter } from './BetterSQLite3DBAdapter.js';
  * ```
  */
 export class PowerSyncDatabase extends AbstractPowerSyncDatabase {
+    constructor(options) {
+        super(options);
+    }
     async _initialize() {
         await this.database.initialize();
     }

package/lib/db/RemoteConnection.d.ts

@@ -13,6 +13,7 @@ export declare class RemoteConnection implements LockContext {
     constructor(worker: Worker, comlink: Remote<AsyncDatabaseOpener>, database: Remote<AsyncDatabase>);
     executeBatch(query: string, params?: any[][]): Promise<QueryResult>;
     execute(query: string, params?: any[] | undefined): Promise<QueryResult>;
+    executeRaw(query: string, params?: any[] | undefined): Promise<any[][]>;
     getAll<T>(sql: string, parameters?: any[]): Promise<T[]>;
     getOptional<T>(sql: string, parameters?: any[]): Promise<T | null>;
     get<T>(sql: string, parameters?: any[]): Promise<T>;

package/lib/db/RemoteConnection.js

@@ -20,6 +20,9 @@ export class RemoteConnection {
         const result = await this.database.execute(query, params ?? []);
         return RemoteConnection.wrapQueryResult(result);
     }
+    async executeRaw(query, params) {
+        return await this.database.executeRaw(query, params ?? []);
+    }
     async getAll(sql, parameters) {
         const res = await this.execute(sql, parameters);
         return res.rows?._array ?? [];

package/lib/db/SqliteWorker.d.ts

@@ -1 +1,9 @@
-export {};
+export interface PowerSyncWorkerOptions {
+    /**
+     * A function responsible for finding the powersync DLL/so/dylib file.
+     *
+     * @returns The absolute path of the PowerSync SQLite core extensions library.
+     */
+    extensionPath: () => string;
+}
+export declare function startPowerSyncWorker(options?: Partial<PowerSyncWorkerOptions>): void;

package/lib/db/SqliteWorker.js

@@ -1,3 +1,4 @@
+import * as path from 'node:path';
 import BetterSQLite3Database from '@powersync/better-sqlite3';
 import * as Comlink from 'comlink';
 import { parentPort, threadId } from 'node:worker_threads';
@@ -54,6 +55,16 @@ class BlockingAsyncDatabase {
             };
         }
     }
+    async executeRaw(query, params) {
+        const stmt = this.db.prepare(query);
+        if (stmt.reader) {
+            return stmt.raw().all(params);
+        }
+        else {
+            stmt.raw().run(params);
+            return [];
+        }
+    }
     async executeBatch(query, params) {
         params = params ?? [];
         let rowsAffected = 0;
@@ -66,10 +77,14 @@ class BlockingAsyncDatabase {
     }
 }
 class BetterSqliteWorker {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
     async open(path, isWriter) {
         const baseDB = new BetterSQLite3Database(path);
         baseDB.pragma('journal_mode = WAL');
-        loadExtension(baseDB);
+        baseDB.loadExtension(this.options.extensionPath(), 'sqlite3_powersync_init');
         if (!isWriter) {
             baseDB.pragma('query_only = true');
         }
@@ -78,22 +93,34 @@ class BetterSqliteWorker {
         return Comlink.proxy(asyncDb);
     }
 }
-const loadExtension = (db) => {
-    const platform = OS.platform();
-    let extensionPath;
-    if (platform === 'win32') {
-        extensionPath = 'powersync.dll';
-    }
-    else if (platform === 'linux') {
-        extensionPath = 'libpowersync.so';
-    }
-    else if (platform === 'darwin') {
-        extensionPath = 'libpowersync.dylib';
-    }
-    else {
-        throw 'Unknown platform, PowerSync for Node.js currently supports Windows, Linux and macOS.';
-    }
-    const resolved = url.fileURLToPath(new URL(`../${extensionPath}`, import.meta.url));
-    db.loadExtension(resolved, 'sqlite3_powersync_init');
-};
-Comlink.expose(new BetterSqliteWorker(), parentPort);
+export function startPowerSyncWorker(options) {
+    const resolvedOptions = {
+        extensionPath() {
+            const isCommonJsModule = import.meta.isBundlingToCommonJs ?? false;
+            const platform = OS.platform();
+            let extensionPath;
+            if (platform === 'win32') {
+                extensionPath = 'powersync.dll';
+            }
+            else if (platform === 'linux') {
+                extensionPath = 'libpowersync.so';
+            }
+            else if (platform === 'darwin') {
+                extensionPath = 'libpowersync.dylib';
+            }
+            else {
+                throw 'Unknown platform, PowerSync for Node.js currently supports Windows, Linux and macOS.';
+            }
+            let resolved;
+            if (isCommonJsModule) {
+                resolved = path.resolve(__dirname, '../lib/', extensionPath);
+            }
+            else {
+                resolved = url.fileURLToPath(new URL(`../${extensionPath}`, import.meta.url));
+            }
+            return resolved;
+        },
+        ...options
+    };
+    Comlink.expose(new BetterSqliteWorker(resolvedOptions), parentPort);
+}

package/lib/db/options.d.ts

@@ -0,0 +1,22 @@
+import { type Worker } from 'node:worker_threads';
+import { SQLOpenOptions } from '@powersync/common';
+export type WorkerOpener = (...args: ConstructorParameters<typeof Worker>) => InstanceType<typeof Worker>;
+/**
+ * The {@link SQLOpenOptions} available across all PowerSync SDKs for JavaScript extended with
+ * Node.JS-specific options.
+ */
+export interface NodeSQLOpenOptions extends SQLOpenOptions {
+    /**
+     * The Node.JS SDK will use one worker to run writing queries and additional workers to run reads.
+     * This option controls how many workers to use for reads.
+     */
+    readWorkerCount?: number;
+    /**
+     * A callback to allow customizing how the Node.JS SDK loads workers. This can be customized to
+     * use workers at different paths.
+     *
+     * @param args The arguments that would otherwise be passed to the {@link Worker} constructor.
+     * @returns the resolved worker.
+     */
+    openWorker?: WorkerOpener;
+}

package/lib/db/options.js

@@ -0,0 +1 @@
+export {};

package/lib/worker.d.ts

@@ -0,0 +1 @@
+export * from './db/SqliteWorker.js';

package/lib/worker.js

@@ -0,0 +1 @@
+export * from './db/SqliteWorker.js';

package/package.json

@@ -1,20 +1,30 @@
 {
   "name": "@powersync/node",
-  "version": "0.1.1",
+  "version": "0.2.1",
   "publishConfig": {
     "registry": "https://registry.npmjs.org/",
     "access": "public"
   },
   "description": "PowerSync Node.js SDK. Sync Postgres, MongoDB or MySQL with SQLite in your Node.js app",
-  "main": "./lib/index.js",
-  "module": "./lib/index.js",
-  "types": "./lib/index.d.ts",
   "files": [
     "lib",
     "dist",
     "download_core.js"
   ],
   "type": "module",
+  "exports": {
+    ".": {
+      "import": "./lib/index.js",
+      "require": "./dist/bundle.cjs",
+      "types": "./lib/index.d.ts"
+    },
+    "./worker.js": {
+      "import": "./lib/worker.js",
+      "require": "./dist/worker.cjs",
+      "types": "./lib/worker.d.ts"
+    },
+    "./package.json": "./package.json"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/powersync-ja/powersync-js.git"
@@ -26,19 +36,22 @@
   },
   "homepage": "https://docs.powersync.com/",
   "peerDependencies": {
-    "@powersync/common": "^1.22.0",
-    "@powersync/better-sqlite3": "^0.1.0"
+    "@powersync/common": "^1.27.0"
   },
   "dependencies": {
+    "@powersync/better-sqlite3": "^0.1.1",
     "async-lock": "^1.4.0",
     "bson": "^6.6.0",
     "comlink": "^4.4.2",
-    "@powersync/common": "1.25.0"
+    "@powersync/common": "1.27.0"
   },
   "devDependencies": {
     "@types/async-lock": "^1.4.0",
+    "drizzle-orm": "^0.35.2",
+    "rollup": "4.14.3",
     "typescript": "^5.5.3",
-    "vitest": "^3.0.5"
+    "vitest": "^3.0.5",
+    "@powersync/drizzle-driver": "0.4.0"
   },
   "keywords": [
     "data sync",
@@ -49,7 +62,7 @@
   ],
   "scripts": {
     "install": "node download_core.js",
-    "build": "tsc -b",
+    "build": "tsc -b && rollup --config",
     "build:prod": "tsc -b --sourceMap false",
     "clean": "rm -rf lib dist tsconfig.tsbuildinfo dist",
     "watch": "tsc -b -w",