@velarscript-labs/sqlite 0.3.3 → 0.3.4

package/src/index.vel

@@ -1,6 +1,9 @@
 /// Per-connection bounds. Every operation is serialized through one Worker;
 /// queueCapacity bounds admitted operations waiting for that connection.
 import {DatabaseExecutor, DatabaseStatement, trustedSql} from "@velarscript-labs/database"
+import {sqliteWorkerSource, sqliteWorkerSourceFormatVersion} from "./generated/worker-source.vel"
+
+assert sqliteWorkerSourceFormatVersion == 1 else "SQLite Worker source module has an unsupported format"
 
 export enum SqliteJournalMode:
     delete
@@ -80,235 +83,6 @@ extern js`
       };
     }
 
-    function workerMain() {
-      const { Buffer } = require("node:buffer");
-      const { parentPort, workerData } = require("node:worker_threads");
-      const { DatabaseSync, constants } = require("node:sqlite");
-      const { parseStmt } = require("sqlite3-parser");
-      let database = null;
-      let statements = null;
-      let closed = false;
-
-      function failure(message, code, operation, retryable = false) {
-        const error = new Error(message);
-        error.code = code;
-        error.operation = operation;
-        error.retryable = retryable;
-        return error;
-      }
-
-      function errorData(error, operation) {
-        const message = error instanceof Error ? error.message : String(error);
-        const rawCode = error && typeof error === "object" ? error.errstr ?? error.code ?? null : null;
-        const code = rawCode == null ? null : String(rawCode);
-        const upper = code == null ? "" : code.toUpperCase();
-        return {
-          message,
-          sqliteCode: code,
-          operation: error && typeof error.operation === "string" ? error.operation : operation,
-          retryable: error && error.retryable === true || upper.includes("BUSY") || upper.includes("LOCKED"),
-        };
-      }
-
-      function parameter(value, budget) {
-        if (value === null) return null;
-        if (typeof value === "boolean") return value ? 1 : 0;
-        if (typeof value === "string") {
-          budget.value += Buffer.byteLength(value, "utf8");
-          return value;
-        }
-        if (typeof value === "number") {
-          if (!Number.isFinite(value)) throw failure("SQLite number parameters must be finite", "SQLITE_INPUT", "params");
-          if (Number.isInteger(value) && !Number.isSafeInteger(value)) throw failure("SQLite integer parameters must be safe integers", "SQLITE_INPUT", "params");
-          budget.value += 8;
-          return value;
-        }
-        if (value instanceof Uint8Array) {
-          budget.value += value.byteLength;
-          return value;
-        }
-        throw failure("SQLite parameters support only null, bool, string, finite number, and Bytes", "SQLITE_INPUT", "params");
-      }
-
-      function parameters(value) {
-        if (!Array.isArray(value) || value.length > 999) throw failure("SQLite parameters must be a List with at most 999 values", "SQLITE_INPUT", "params");
-        const budget = { value: 0 };
-        const output = value.map(item => parameter(item, budget));
-        if (budget.value > workerData.maxResultBytes) throw failure("SQLite parameters exceed the connection byte limit", "SQLITE_INPUT", "params");
-        return output;
-      }
-
-      function statementParts(rawFragments, rawParameters, operation) {
-        if (!Array.isArray(rawFragments) || rawFragments.length !== rawParameters.length + 1 || rawFragments.length > 1000) {
-          throw failure("SQLite statements require exactly one more SQL fragment than bound parameters", "SQLITE_INPUT", operation);
-        }
-        let sourceBytes = 0;
-        const fragments = rawFragments.map(fragment => {
-          if (typeof fragment !== "string" || fragment.includes("\0") || fragment.includes("?")) {
-            throw failure("SQLite SQL fragments must be text without NUL or raw parameter placeholders", "SQLITE_INPUT", operation);
-          }
-          sourceBytes += Buffer.byteLength(fragment, "utf8");
-          return fragment;
-        });
-        if (sourceBytes === 0 || sourceBytes > 1024 * 1024) throw failure("SQLite SQL must be non-empty text no longer than 1 MiB", "SQLITE_INPUT", operation);
-        const bound = parameters(rawParameters);
-        const source = fragments.join("?");
-        const parsed = parseStmt(source);
-        if (parsed.status !== "ok") {
-          const message = parsed.errors.length === 0 ? "invalid statement" : parsed.errors[0].message;
-          throw failure("SQLite requires exactly one valid statement: " + message, "SQLITE_INPUT", operation);
-        }
-        return { fragments, bound };
-      }
-
-      function count(value, label) {
-        const number = typeof value === "bigint" ? Number(value) : value;
-        if (!Number.isSafeInteger(number) || number < 0) throw failure(label + " is outside VelarScript's safe integer range", "SQLITE_INTEGER_RANGE", "result");
-        return number;
-      }
-
-      function resultValue(value, budget, field) {
-        if (value === null) return null;
-        if (typeof value === "string") {
-          budget.value += Buffer.byteLength(value, "utf8");
-          return value;
-        }
-        if (typeof value === "number") {
-          if (!Number.isFinite(value)) throw failure("SQLite returned a non-finite number in " + field, "SQLITE_RESULT", "result");
-          budget.value += 8;
-          return value;
-        }
-        if (typeof value === "bigint") {
-          const number = Number(value);
-          if (!Number.isSafeInteger(number)) throw failure("SQLite returned an integer outside VelarScript's safe range in " + field, "SQLITE_INTEGER_RANGE", "result");
-          budget.value += 8;
-          return number;
-        }
-        if (value instanceof Uint8Array) {
-          budget.value += value.byteLength;
-          return value;
-        }
-        throw failure("SQLite returned an unsupported value in " + field, "SQLITE_RESULT", "result");
-      }
-
-      function resultRow(value, budget) {
-        if (value === null || typeof value !== "object" || Array.isArray(value)) throw failure("SQLite returned a non-record row", "SQLITE_RESULT", "result");
-        const output = Object.create(null);
-        for (const key of Object.keys(value)) {
-          budget.value += Buffer.byteLength(key, "utf8");
-          output[key] = resultValue(value[key], budget, key);
-          if (budget.value > workerData.maxResultBytes) throw failure("SQLite result exceeds the connection byte limit", "SQLITE_RESULT_LIMIT", "result");
-        }
-        return output;
-      }
-
-      function execute(message) {
-        const statement = statementParts(message.fragments, message.params, "execute");
-        const result = statements.run(statement.fragments, ...statement.bound);
-        return count(result.changes, "SQLite affected row count");
-      }
-
-      function one(message) {
-        const statement = statementParts(message.fragments, message.params, "one");
-        const row = statements.get(statement.fragments, ...statement.bound);
-        if (row === undefined) return null;
-        return resultRow(row, { value: 0 });
-      }
-
-      function all(message) {
-        const statement = statementParts(message.fragments, message.params, "all");
-        const rows = [];
-        const budget = { value: 0 };
-        for (const row of statements.iterate(statement.fragments, ...statement.bound)) {
-          if (rows.length >= workerData.maxRows) throw failure("SQLite result exceeds the connection row limit", "SQLITE_ROW_LIMIT", "all");
-          rows.push(resultRow(row, budget));
-        }
-        return rows;
-      }
-
-      function dispatch(message) {
-        if (closed) throw failure("SQLite connection is closed", "SQLITE_CLOSED", message.operation);
-        if (message.operation === "execute") return execute(message);
-        if (message.operation === "one") return one(message);
-        if (message.operation === "all") return all(message);
-        if (message.operation === "begin") {
-          if (database.isTransaction) throw failure("SQLite transaction is already active", "SQLITE_TRANSACTION", "begin", true);
-          database.exec("BEGIN IMMEDIATE");
-          return null;
-        }
-        if (message.operation === "commit") {
-          if (!database.isTransaction) throw failure("SQLite transaction is not active", "SQLITE_TRANSACTION", "commit");
-          database.exec("COMMIT");
-          return null;
-        }
-        if (message.operation === "rollback") {
-          if (database.isTransaction) database.exec("ROLLBACK");
-          return null;
-        }
-        if (message.operation === "close") {
-          try {
-            if (database.isTransaction) database.exec("ROLLBACK");
-          } finally {
-            database.close();
-            closed = true;
-          }
-          return null;
-        }
-        throw failure("Unknown SQLite operation", "SQLITE_PROTOCOL", String(message.operation));
-      }
-
-      try {
-        database = new DatabaseSync(workerData.path, {
-          timeout: workerData.busyTimeoutMilliseconds,
-          readOnly: workerData.readOnly,
-          enableForeignKeyConstraints: true,
-          enableDoubleQuotedStringLiterals: false,
-          allowExtension: false,
-          readBigInts: true,
-          allowBareNamedParameters: false,
-          allowUnknownNamedParameters: false,
-          defensive: true,
-          limits: {
-            length: workerData.maxResultBytes,
-            sqlLength: 1024 * 1024,
-            column: 2000,
-            exprDepth: 1000,
-            compoundSelect: 500,
-            vdbeOp: 250000,
-            functionArg: 1000,
-            attach: 0,
-            likePatternLength: 50000,
-            variableNumber: 999,
-            triggerDepth: 100,
-          },
-        });
-        database.enableDefensive(true);
-        database.enableLoadExtension(false);
-        database.setAuthorizer((actionCode, arg1, arg2) => {
-          if (actionCode === constants.SQLITE_ATTACH || actionCode === constants.SQLITE_DETACH) return constants.SQLITE_DENY;
-          if (actionCode === constants.SQLITE_FUNCTION && typeof arg2 === "string" && arg2.toLowerCase() === "load_extension") return constants.SQLITE_DENY;
-          return constants.SQLITE_OK;
-        });
-        statements = database.createTagStore(workerData.statementCacheCapacity);
-        if (workerData.journalMode !== null) database.exec("PRAGMA journal_mode = " + workerData.journalMode.toUpperCase());
-        parentPort.postMessage({ kind: "ready" });
-      } catch (error) {
-        parentPort.postMessage({ kind: "ready", error: errorData(error, "open") });
-        parentPort.close();
-        return;
-      }
-
-      parentPort.on("message", message => {
-        try {
-          const value = dispatch(message);
-          parentPort.postMessage({ kind: "response", id: message.id, value });
-          if (message.operation === "close") parentPort.close();
-        } catch (error) {
-          parentPort.postMessage({ kind: "response", id: message.id, error: errorData(error, message.operation) });
-        }
-      });
-    }
-
     function nativeError(value) {
       return new NativeSqliteError(value.message, value.sqliteCode, value.operation, value.retryable);
     }
@@ -323,11 +97,18 @@ extern js`
       state.pending.clear();
     }
 
+    function failWorker(state, error) {
+      if (state.workerFailure !== null) return;
+      state.workerFailure = error;
+      state.closed = true;
+      failPending(state, error);
+    }
+
     function request(state, operation, fragments = [""], params = []) {
       if (state.closed) return Promise.reject(new NativeSqliteError("SQLite connection is closed", "SQLITE_CLOSED", operation));
       const id = state.nextId++;
       return new Promise((resolve, reject) => {
-        state.pending.set(id, { resolve, reject });
+        state.pending.set(id, { resolve, reject, operation });
         try {
           state.worker.postMessage({ id, operation, fragments, params });
         } catch (error) {
@@ -430,9 +211,16 @@ extern js`
         if (transactionContext.getStore() === state) return Promise.reject(new NativeSqliteConcurrencyError("SQLite connection cannot close inside its transaction callback"));
         state.closing = true;
         state.closePromise = state.tail.then(async () => {
+          let closeError = null;
           try {
             if (!state.closed) await request(state, "close");
+          } catch (error) {
+            closeError = error;
+          }
+          try {
             await state.exited;
+            if (closeError !== null) throw closeError;
+            if (state.workerFailure !== null) throw state.workerFailure;
           } finally {
             state.closed = true;
           }
@@ -441,11 +229,13 @@ extern js`
       }
     }
 
-    export async function openNativeSqlite(path, options = {}) {
+    export async function openNativeSqlite(path, options = {}, workerSource = "") {
       if (typeof path !== "string" || path.trim() === "" || path.length > 4096 || path.includes("\0")) throw new TypeError("SQLite path must be non-empty bounded text");
+      if (typeof workerSource !== "string") throw new TypeError("SQLite Worker source must be text");
+      if (workerSource.length === 0 || workerSource.length > 4 * 1024 * 1024) throw new TypeError("SQLite Worker source must be non-empty bounded text");
       const checked = checkedOptions(options);
-      const source = "(" + workerMain.toString() + ")()";
-      const worker = new Worker(source, { eval: true, workerData: { path, ...checked } });
+      const workerUrl = new URL("data:text/javascript;charset=utf-8," + encodeURIComponent(workerSource));
+      const worker = new Worker(workerUrl, { type: "module", workerData: { path, ...checked } });
       let readyResolve;
       let readyReject;
       let exitResolve;
@@ -463,10 +253,14 @@ extern js`
         tail: Promise.resolve(),
         closing: false,
         closed: false,
+        closeAcknowledged: false,
+        workerFailure: null,
+        exitCode: null,
         closePromise: null,
       };
       worker.on("message", message => {
         if (message.kind === "ready") {
+          if (state.readyResolve === null) return;
           const resolve = state.readyResolve;
           const reject = state.readyReject;
           state.readyResolve = null;
@@ -479,21 +273,29 @@ extern js`
         const pending = state.pending.get(message.id);
         if (!pending) return;
         state.pending.delete(message.id);
+        if (pending.operation === "close") state.closeAcknowledged = true;
         if (message.error) pending.reject(nativeError(message.error));
         else pending.resolve(message.value);
       });
       worker.on("error", error => {
-        state.closed = true;
-        failPending(state, new NativeSqliteError(error.message, "SQLITE_WORKER", "worker"));
+        failWorker(state, new NativeSqliteError(error.message, "SQLITE_WORKER", "worker"));
       });
       worker.on("exit", code => {
-        exitResolve(code);
-        if (code !== 0 && !state.closed) {
-          state.closed = true;
-          failPending(state, new NativeSqliteError("SQLite Worker exited with code " + code, "SQLITE_WORKER", "worker"));
+        state.exitCode = code;
+        const expected = code === 0 && state.closeAcknowledged && state.workerFailure === null;
+        if (!expected) {
+          failWorker(state, new NativeSqliteError("SQLite Worker exited unexpectedly with code " + code, "SQLITE_WORKER", "worker"));
         }
+        state.closed = true;
+        exitResolve(code);
       });
-      await ready;
+      try {
+        await ready;
+      } catch (error) {
+        if (state.exitCode === null) await worker.terminate();
+        await exited;
+        throw error;
+      }
       return new NativeSqliteConnection(TOKEN, state);
     }
 `:
@@ -521,7 +323,7 @@ extern js`
         async def transaction(operation: (NativeSqliteTransaction) -> Promise<null>) -> null
         async def close() -> null
 
-    export async def openNativeSqlite(path: string, options: SqliteOptions = {}) -> NativeSqliteConnection
+    export async def openNativeSqlite(path: string, options: SqliteOptions = {}, workerSource: string = "") -> NativeSqliteConnection
 
 /// A checked SQLite failure. sqliteCode preserves the driver result code when
 /// Node exposes one; retryable is true for busy, locked, and queue pressure.
@@ -686,7 +488,7 @@ export class SqliteConnection:
     @dispose: await self.close()
 
 export async def openSqlite(path: string, options: SqliteOptions = {}) -> SqliteConnection:
-    try: return SqliteConnection(await openNativeSqlite(path, options))
+    try: return SqliteConnection(await openNativeSqlite(path, options, sqliteWorkerSource))
     catch error:
         if error is NativeSqliteError: throw sqliteFailure(error)
         throw SqliteError(error.message)