@noormdev/sdk 1.0.0-alpha.17 → 1.0.0-alpha.18

package/dist/index.js

@@ -1,19 +1,22 @@
-import { processFile, DtReader, isTemplate, FORMAT_VERSION, DT_EXTENSIONS, encryptWithPassphrase, ENCODED_TYPES, GZIP_THRESHOLD, GZIP_RATIO_THRESHOLD } from './chunk-BTXUEWYQ.js';
+import { processFile, DtReader, isTemplate, FORMAT_VERSION, DT_EXTENSIONS, encryptWithPassphrase, ENCODED_TYPES } from './chunk-KF7GE2VM.js';
 import { initializeVault, getVaultStatus, setVaultSecret, getVaultSecret, getAllVaultSecrets, listVaultSecretKeys, deleteVaultSecret, vaultSecretExists, getVaultKey, noormDb, getNoormTables, encryptVaultKey } from './chunk-LJDHQEWW.js';
 import { observer } from './chunk-4HW4CS2C.js';
 import { constants, existsSync, readFileSync, mkdirSync, writeFileSync, createWriteStream } from 'fs';
-import path, { join, dirname } from 'path';
+import path, { join, dirname, resolve } from 'path';
 import { makeNestedConfig, gigabytes, attempt, retry, clone, merge, attemptSync } from '@logosdx/utils';
-import { homedir, userInfo } from 'os';
+import { homedir, availableParallelism, userInfo } from 'os';
 import { stat, mkdir, writeFile, unlink, rename, rm, readdir, readFile, access } from 'fs/promises';
-import { createHash, createDecipheriv, randomBytes, createCipheriv, hkdfSync } from 'crypto';
+import { createHash, createDecipheriv, randomBytes, createCipheriv, randomUUID, hkdfSync } from 'crypto';
 import { z } from 'zod';
 import { parse, stringify } from 'yaml';
 import { execSync } from 'child_process';
 import { sql } from 'kysely';
 import 'dayjs';
 import ansis from 'ansis';
-import { createGzip, gunzipSync, gzipSync } from 'zlib';
+import { Worker, parentPort, workerData } from 'worker_threads';
+import { ObserverRelay } from '@logosdx/observer';
+import { fileURLToPath } from 'url';
+import { createGzip } from 'zlib';
 import { pipeline } from 'stream/promises';
 import { PassThrough } from 'stream';
 import JSON5 from 'json5';
@@ -1672,6 +1675,7 @@ async function loadExistingIdentity() {
 var ConnectionManager = class {
   #cached = /* @__PURE__ */ new Map();
   #tracked = /* @__PURE__ */ new Map();
+  #bridges = /* @__PURE__ */ new Map();
   #nextId = 1;
   #shuttingDown = false;
   #unsubscribe = null;
@@ -1755,7 +1759,7 @@ var ConnectionManager = class {
     for (const [id, entry] of trackedEntries) {
       const destroyWithTimeout = Promise.race([
         entry.conn.destroy(),
-        new Promise((resolve) => setTimeout(resolve, CLOSE_TIMEOUT))
+        new Promise((resolve2) => setTimeout(resolve2, CLOSE_TIMEOUT))
       ]);
       const [, err] = await attempt(() => destroyWithTimeout);
       this.#tracked.delete(id);
@@ -1765,6 +1769,28 @@ var ConnectionManager = class {
         observer.emit("connection:close", { configName: entry.configName });
       }
     }
+    const bridgeNames = Array.from(this.#bridges.keys());
+    for (const name of bridgeNames) {
+      await this.closeBridge(name);
+    }
+  }
+  /**
+   * Track a WorkerBridge connection for a config.
+   *
+   * The bridge is shut down alongside regular connections on app:shutdown.
+   */
+  trackBridge(configName, bridge) {
+    this.#bridges.set(configName, bridge);
+  }
+  /**
+   * Close a specific bridge by config name.
+   */
+  async closeBridge(configName) {
+    const bridge = this.#bridges.get(configName);
+    if (bridge) {
+      await bridge.shutdown();
+      this.#bridges.delete(configName);
+    }
   }
   /**
    * Check if a cached connection exists for a config name.
@@ -1776,7 +1802,7 @@ var ConnectionManager = class {
    * Get total number of active connections.
    */
   get size() {
-    return this.#cached.size + this.#tracked.size;
+    return this.#cached.size + this.#tracked.size + this.#bridges.size;
   }
   /**
    * Check if shutdown is in progress.
@@ -4546,7 +4572,7 @@ function getLockManager() {
   return instance3;
 }
 function sleep(ms) {
-  return new Promise((resolve) => setTimeout(resolve, ms));
+  return new Promise((resolve2) => setTimeout(resolve2, ms));
 }
 
 // src/core/change/executor.ts
@@ -9494,7 +9520,7 @@ async function copyVaultSecrets(sourceConfig, destConfig, keys, identityHash, pr
         throw new Error(`No vault access on source config "${sourceConfig.name}"`);
       }
       const destStatus = await getVaultStatus(ctx.destination.db, identityHash, ctx.destination.dialect);
-      let destVaultKey = null;
+      let destVaultKey;
       if (!destStatus.isInitialized) {
         const [newKey, initErr] = await initializeVault(
           ctx.destination.db,
@@ -10681,16 +10707,16 @@ async function planTransfer(ctx, options = {}) {
   const orderedNames = sortErr ? allTables.map((t) => t.name) : sortedNames;
   const tablePlans = [];
   for (const name of orderedNames) {
-    const meta = tablesByName[name];
-    if (!meta) continue;
+    const meta2 = tablesByName[name];
+    if (!meta2) continue;
     tablePlans.push({
-      name: meta.name,
-      schema: meta.schema,
-      rowCount: meta.rowCount,
-      hasIdentity: meta.hasIdentity,
-      identityColumn: meta.identityColumn,
-      primaryKey: meta.primaryKey,
-      columns: meta.columns,
+      name: meta2.name,
+      schema: meta2.schema,
+      rowCount: meta2.rowCount,
+      hasIdentity: meta2.hasIdentity,
+      identityColumn: meta2.identityColumn,
+      primaryKey: meta2.primaryKey,
+      columns: meta2.columns,
       dependsOn: dependencyMap.get(name) ?? []
     });
   }
@@ -11234,8 +11260,8 @@ async function executeTransfer(ctx, plan, options = {}) {
     const strategy = options.onConflict ?? "fail";
     const useSameServer = plan.sameServer && strategy === "fail" && !plan.crossDialect;
     const useCrossDialect = plan.crossDialect && tablePlan.columnTypes;
-    let tableResult = null;
-    let tableErr = null;
+    let tableResult;
+    let tableErr;
     if (useSameServer) {
       [tableResult, tableErr] = await transferTableSameServer(
         ctx,
@@ -11864,6 +11890,174 @@ var TransferNamespace = class {
     return getTransferPlan(this.#state.config, destConfig, options);
   }
 };
+
+// src/core/worker-bridge/pool.ts
+var WorkerPool = class {
+  #workers;
+  #nextIndex = 0;
+  #isShutdown = false;
+  constructor(script, options) {
+    const size = Math.max(1, options.size);
+    this.#workers = Array.from(
+      { length: size },
+      () => new WorkerBridge(script)
+    );
+  }
+  /**
+   * Returns the number of workers in the pool.
+   */
+  get size() {
+    return this.#workers.length;
+  }
+  /**
+   * Returns true if the pool has been shut down.
+   */
+  get isShutdown() {
+    return this.#isShutdown;
+  }
+  /**
+   * Sends a request to the next available worker (round-robin) and awaits its response.
+   *
+   * Injects a correlation ID so concurrent requests on the same event type
+   * can be matched to their originating worker.
+   *
+   * @example
+   * const { values } = await pool.request('serialize', { row, columns, index: 0 })
+   */
+  async request(event, data2, options) {
+    const worker = this.#workers[this.#nextIndex];
+    this.#nextIndex = (this.#nextIndex + 1) % this.#workers.length;
+    return worker.request(event, data2, options);
+  }
+  /**
+   * Registers a listener on all workers for the given event.
+   *
+   * Useful for broadcast-style events that every worker may emit.
+   *
+   * @example
+   * pool.on('serialize:res', ({ data }) => console.log(data.values))
+   */
+  on(event, callback, options) {
+    for (const worker of this.#workers) {
+      worker.on(event, callback, options);
+    }
+  }
+  /**
+   * Terminates all workers in the pool.
+   *
+   * Marks the pool as shut down and waits for every worker thread to exit
+   * before resolving.
+   *
+   * @example
+   * await pool.shutdown()
+   */
+  async shutdown() {
+    this.#isShutdown = true;
+    await Promise.all(this.#workers.map((w) => w.shutdown()));
+  }
+};
+
+// src/core/worker-bridge/types.ts
+function isTransferable(data2) {
+  return !!data2 && typeof data2 === "object" && "__transfer" in data2;
+}
+
+// src/core/worker-bridge/bridge.ts
+var WorkerBridge = class extends ObserverRelay {
+  #port;
+  #ownsWorker;
+  constructor(script, data2) {
+    const isParent = !!script;
+    super({ name: isParent ? "bridge:parent" : "bridge:worker" });
+    if (script) {
+      this.#port = new Worker(script, { workerData: data2 });
+      this.#ownsWorker = true;
+      this.#port.on("exit", (code) => {
+        if (!this.isShutdown && code !== 0) {
+          this.receive("worker:exit", { code, error: `Worker exited with code ${code}` }, {});
+        }
+      });
+    } else {
+      if (!parentPort) {
+        throw new Error(
+          "WorkerBridge: no script provided and not in a worker thread"
+        );
+      }
+      this.#port = parentPort;
+      this.#ownsWorker = false;
+    }
+    this.#port.on("message", (msg) => {
+      this.receive(msg.event, msg.data, {});
+    });
+  }
+  send(event, data2) {
+    if (isTransferable(data2)) {
+      this.#port.postMessage({ event, data: data2 }, data2.__transfer);
+    } else {
+      this.#port.postMessage({ event, data: data2 }, []);
+    }
+  }
+  async request(event, data2, options) {
+    const cid = randomUUID();
+    const pending = this.once(new RegExp(`^${event}:res:${cid}$`), options);
+    this.send(event, { ...data2, __cid: cid });
+    const { data: { data: result } } = await pending;
+    return result;
+  }
+  transfer(event, data2, transferables) {
+    this.#port.postMessage({ event, data: data2 }, transferables);
+  }
+  static pool(script, options) {
+    return new WorkerPool(script, options);
+  }
+  static get workerData() {
+    return workerData;
+  }
+  async shutdown() {
+    super.shutdown();
+    if (this.#ownsWorker && "terminate" in this.#port) {
+      await this.#port.terminate();
+    }
+  }
+};
+
+// src/core/worker-bridge/order-buffer.ts
+var OrderBuffer = class {
+  #buffer = /* @__PURE__ */ new Map();
+  #nextIndex = 0;
+  #flush;
+  constructor(flush) {
+    this.#flush = flush;
+  }
+  get nextIndex() {
+    return this.#nextIndex;
+  }
+  get pending() {
+    return this.#buffer.size;
+  }
+  add(index, item) {
+    this.#buffer.set(index, item);
+    this.#drain();
+  }
+  #drain() {
+    while (this.#buffer.has(this.#nextIndex)) {
+      const item = this.#buffer.get(this.#nextIndex);
+      this.#buffer.delete(this.#nextIndex);
+      this.#nextIndex++;
+      this.#flush(item);
+    }
+  }
+};
+var IS_COMPILED = import.meta.url.includes("$bunfs");
+var meta = import.meta;
+var MODULE_DIR = typeof meta.dir === "string" ? meta.dir : dirname(fileURLToPath(import.meta.url));
+var WORKER_DIR = resolve(MODULE_DIR, "../../workers");
+function resolveWorker(name) {
+  if (IS_COMPILED) {
+    return new URL(`./workers/${name}.js`, import.meta.url);
+  }
+  return resolve(WORKER_DIR, `${name}.js`);
+}
 var DtWriter = class {
   #filepath;
   #schema;
@@ -11967,8 +12161,8 @@ var DtWriter = class {
       this.#passthrough.end();
       await this.#pipelinePromise;
     } else if (this.#stream) {
-      await new Promise((resolve, reject) => {
-        this.#stream.end(() => resolve());
+      await new Promise((resolve2, reject) => {
+        this.#stream.end(() => resolve2());
         this.#stream.on("error", reject);
       });
     }
@@ -11985,211 +12179,26 @@ var DtWriter = class {
     this.#stream.write(buf);
   }
 };
-function serializeRow(options) {
-  const { row, columns } = options;
-  const result = [];
-  for (const col of columns) {
-    const value = row[col.name];
-    result.push(serializeValue(value, col));
-  }
-  return result;
-}
-function serializeValue(value, column) {
-  if (value === null || value === void 0) {
-    return null;
-  }
-  if (isEncodedType(column.type)) {
-    return encodeValue(value, column.type);
-  }
-  return serializeSimple(value, column.type);
-}
-function serializeSimple(value, type) {
-  switch (type) {
-    case "bigint":
-      return String(value);
-    case "decimal":
-      return String(value);
-    case "bool":
-      if (typeof value === "number") return value !== 0;
-      if (typeof value === "string") return value === "1" || value.toLowerCase() === "true";
-      return Boolean(value);
-    case "timestamp": {
-      if (value instanceof Date) return value.toISOString();
-      return String(value);
-    }
-    case "date": {
-      if (value instanceof Date) return value.toISOString().split("T")[0];
-      return String(value);
-    }
-    case "uuid":
-      return String(value);
-    case "int":
-    case "float":
-      return typeof value === "string" ? Number(value) : value;
-    case "string":
-    default:
-      return String(value);
-  }
-}
-function encodeValue(value, type) {
-  if (type === "binary") {
-    return encodeBinary(value);
-  }
-  return encodeJsonLike(value);
-}
-function encodeBinary(value) {
-  const buf = toBuffer(value);
-  const byteLength = buf.length;
-  if (byteLength < GZIP_THRESHOLD) {
-    return [buf.toString("base64"), "b64"];
-  }
-  const compressed = gzipSync(buf);
-  if (compressed.length < byteLength * GZIP_RATIO_THRESHOLD) {
-    return [compressed.toString("base64"), "gz64"];
-  }
-  return [buf.toString("base64"), "b64"];
-}
-function encodeJsonLike(value) {
-  const jsonStr = JSON.stringify(value);
-  const byteLength = Buffer.byteLength(jsonStr, "utf8");
-  if (byteLength < GZIP_THRESHOLD) {
-    return [value, "raw"];
-  }
-  const buf = Buffer.from(jsonStr, "utf8");
-  const compressed = gzipSync(buf);
-  if (compressed.length < byteLength * GZIP_RATIO_THRESHOLD) {
-    return [compressed.toString("base64"), "gz64"];
-  }
-  return [value, "raw"];
-}
-function toBuffer(value) {
-  if (Buffer.isBuffer(value)) return value;
-  if (value instanceof Uint8Array) return Buffer.from(value);
-  if (typeof value === "string") return Buffer.from(value, "base64");
-  return Buffer.from(String(value), "utf8");
-}
-function deserializeRow(options) {
-  const { values, columns, targetDialect, targetVersion } = options;
-  const result = {};
-  for (let i = 0; i < columns.length; i++) {
-    const col = columns[i];
-    const value = values[i];
-    result[col.name] = deserializeValue(value, col, targetDialect, targetVersion);
-  }
-  return result;
-}
-function deserializeValue(value, column, targetDialect, targetVersion) {
-  if (value === null || value === void 0) {
-    return null;
-  }
-  if (isEncodedType(column.type)) {
-    const decoded = decodeTuple(value);
-    return convertEncodedForTarget(decoded, column.type, targetDialect, targetVersion);
-  }
-  return convertSimpleForTarget(value, column.type, targetDialect);
-}
-function decodeTuple(tuple) {
-  if (!Array.isArray(tuple) || tuple.length < 2) {
-    return tuple;
-  }
-  const [value, encoding] = tuple;
-  switch (encoding) {
-    case "raw":
-      return value;
-    case "b64":
-      return Buffer.from(value, "base64");
-    case "gz64": {
-      const compressed = Buffer.from(value, "base64");
-      const decompressed = gunzipSync(compressed);
-      const str = decompressed.toString("utf8");
-      if (/^[[{"\d]/.test(str)) {
-        return JSON.parse(str);
-      }
-      return decompressed;
-    }
-    default:
-      return value;
-  }
-}
-function convertEncodedForTarget(decoded, type, targetDialect, targetVersion) {
-  switch (type) {
-    case "json":
-      return convertJsonForTarget(decoded, targetDialect, targetVersion);
-    case "text":
-      return decoded;
-    case "binary":
-      return decoded;
-    case "vector":
-      return convertVectorForTarget(decoded, targetDialect, targetVersion);
-    case "array":
-      return convertArrayForTarget(decoded, targetDialect);
-    case "custom":
-      if (typeof decoded === "object") return JSON.stringify(decoded);
-      return decoded;
-    default:
-      return decoded;
-  }
-}
-function convertJsonForTarget(decoded, targetDialect, targetVersion) {
-  if (targetDialect === "mssql") {
-    const major = targetVersion?.major ?? 2022;
-    if (major < 2025) {
-      return typeof decoded === "string" ? decoded : JSON.stringify(decoded);
-    }
-  }
-  return decoded;
-}
-function convertVectorForTarget(decoded, targetDialect, targetVersion) {
-  const arr = Array.isArray(decoded) ? decoded : decoded;
-  if (targetDialect === "postgres") {
-    return Array.isArray(arr) ? `[${arr.join(",")}]` : String(arr);
-  }
-  if (targetDialect === "mysql") {
-    const major = targetVersion?.major ?? 8;
-    if (major >= 9) {
-      return Array.isArray(arr) ? `[${arr.join(",")}]` : String(arr);
-    }
-    return Array.isArray(arr) ? JSON.stringify(arr) : String(arr);
-  }
-  return Array.isArray(arr) ? JSON.stringify(arr) : String(arr);
-}
-function convertArrayForTarget(decoded, targetDialect) {
-  if (targetDialect === "postgres") {
-    return decoded;
-  }
-  return Array.isArray(decoded) ? JSON.stringify(decoded) : String(decoded);
+
+// src/core/dt/index.ts
+var CONNECTION_WORKER = resolveWorker("connection");
+var COMPUTE_WORKER = resolveWorker("compute");
+function createDefaultComputePool() {
+  return WorkerBridge.pool(COMPUTE_WORKER, {
+    size: Math.max(1, availableParallelism() - 2)
+  });
 }
-function convertSimpleForTarget(value, type, targetDialect) {
-  switch (type) {
-    case "bigint":
-      if (typeof value === "string") {
-        const num = Number(value);
-        if (Number.isSafeInteger(num)) return num;
-      }
-      return value;
-    case "decimal":
-      return value;
-    case "bool":
-      if (targetDialect === "mssql") return value ? 1 : 0;
-      if (targetDialect === "mysql") return value ? 1 : 0;
-      return value;
-    case "timestamp":
-      if (typeof value === "string") return new Date(value);
-      return value;
-    case "date":
-      return value;
-    case "uuid":
-      return value;
-    case "int":
-    case "float":
-      return value;
-    case "string":
-    default:
-      return value;
+async function createDefaultConnectionBridge(dialect, connectionString) {
+  const bridge = new WorkerBridge(CONNECTION_WORKER);
+  const [, connectErr] = await attempt(
+    () => bridge.request("connect", { dialect, connectionString })
+  );
+  if (connectErr) {
+    await attempt(() => bridge.shutdown());
+    return [null, connectErr];
   }
+  return [bridge, null];
 }
-
-// src/core/dt/index.ts
 async function exportTable(options) {
   const startTime = Date.now();
   const { db, dialect, tableName, filepath, passphrase, schema: schemaName } = options;
@@ -12215,58 +12224,178 @@ async function exportTable(options) {
   if (openErr) {
     return [null, openErr];
   }
-  let offset = 0;
+  let ownedConnectionBridge = null;
+  let ownedComputePool = null;
+  let connectionBridge = options.connectionBridge;
+  let computePool = options.computePool;
+  if (!connectionBridge && options.connectionString) {
+    const [bridge, bridgeErr] = await createDefaultConnectionBridge(dialect, options.connectionString);
+    if (bridgeErr) {
+      return [null, bridgeErr];
+    }
+    connectionBridge = bridge;
+    ownedConnectionBridge = bridge;
+  }
+  if (!computePool) {
+    computePool = createDefaultComputePool();
+    ownedComputePool = computePool;
+  }
+  const [result, workerErr] = await exportTableWithWorkers({
+    writer,
+    dtSchema,
+    tableName,
+    filepath,
+    dialect,
+    batchSize,
+    connectionBridge,
+    computePool,
+    kyselyDb
+  });
+  if (ownedComputePool) {
+    await attempt(() => ownedComputePool.shutdown());
+  }
+  if (ownedConnectionBridge) {
+    await attempt(() => ownedConnectionBridge.shutdown());
+  }
+  if (workerErr) {
+    return [null, workerErr];
+  }
+  const durationMs = Date.now() - startTime;
+  observer.emit("dt:export:complete", {
+    filepath,
+    table: tableName,
+    rowsWritten: result.rowsWritten,
+    bytesWritten: result.bytesWritten,
+    durationMs
+  });
+  return [result, null];
+}
+function getQuoteIdent(dialect) {
+  if (dialect === "mssql") return (c) => `[${c}]`;
+  if (dialect === "mysql") return (c) => `\`${c}\``;
+  return (c) => `"${c}"`;
+}
+function buildBatchSql(dialect, tableName, columnList, orderCol, batchSize, offset) {
+  if (dialect === "mssql") {
+    return `SELECT ${columnList} FROM [${tableName}] ORDER BY ${orderCol} OFFSET ${offset} ROWS FETCH NEXT ${batchSize} ROWS ONLY`;
+  }
+  return `SELECT ${columnList} FROM "${tableName}" LIMIT ${batchSize} OFFSET ${offset}`;
+}
+async function exportTableWithWorkers(ctx) {
+  const { writer, dtSchema, tableName, filepath, dialect, batchSize } = ctx;
+  const { connectionBridge, computePool, kyselyDb } = ctx;
+  const backpressureLimit = batchSize * 3;
+  let totalRows = 0;
+  if (connectionBridge) {
+    const countSql = dialect === "mssql" ? `SELECT COUNT(*) AS cnt FROM [${tableName}]` : `SELECT COUNT(*) AS cnt FROM "${tableName}"`;
+    const [countResult, countErr] = await attempt(
+      () => connectionBridge.request("query", { sql: countSql })
+    );
+    if (countErr) {
+      return [null, countErr];
+    }
+    const firstRow = countResult.rows[0];
+    totalRows = Number(firstRow?.["cnt"] ?? 0);
+  }
   const columns = dtSchema.columns.map((c) => c.name);
-  const quoteIdent4 = dialect === "mssql" ? (c) => `[${c}]` : dialect === "mysql" ? (c) => `\`${c}\`` : (c) => `"${c}"`;
+  const quoteIdent4 = getQuoteIdent(dialect);
   const columnList = columns.map(quoteIdent4).join(", ");
+  const orderCol = quoteIdent4(columns[0]);
+  let globalIndex = 0;
+  let loaded = 0;
+  let processed = 0;
+  let saved = 0;
+  let inFlight = 0;
+  let pipelineError = null;
+  const orderBuffer = new OrderBuffer((values) => {
+    writer.writeRow(values);
+    saved++;
+    inFlight--;
+    observer.emit("dt:export:saved", { table: tableName, saved, totalRows });
+  });
+  let offset = 0;
   while (true) {
-    const [rows, fetchErr] = await attempt(() => {
-      if (dialect === "mssql") {
-        const orderCol = quoteIdent4(columns[0]);
+    while (inFlight >= backpressureLimit) {
+      await new Promise((resolve2) => setTimeout(resolve2, 1));
+    }
+    let batchRows;
+    if (connectionBridge) {
+      const batchSql = buildBatchSql(dialect, tableName, columnList, orderCol, batchSize, offset);
+      const [queryResult, queryErr] = await attempt(
+        () => connectionBridge.request("query", { sql: batchSql })
+      );
+      if (queryErr) {
+        pipelineError = queryErr;
+        break;
+      }
+      if (queryResult.error) {
+        pipelineError = new Error(queryResult.error);
+        break;
+      }
+      batchRows = queryResult.rows;
+    } else {
+      const [rows, fetchErr] = await attempt(() => {
+        if (dialect === "mssql") {
+          return sql`
+                        SELECT ${sql.raw(columnList)}
+                        FROM ${sql.table(tableName)}
+                        ORDER BY ${sql.raw(orderCol)}
+                        OFFSET ${offset} ROWS
+                        FETCH NEXT ${batchSize} ROWS ONLY
+                    `.execute(kyselyDb);
+        }
         return sql`
                     SELECT ${sql.raw(columnList)}
                     FROM ${sql.table(tableName)}
-                    ORDER BY ${sql.raw(orderCol)}
-                    OFFSET ${offset} ROWS
-                    FETCH NEXT ${batchSize} ROWS ONLY
+                    LIMIT ${batchSize}
+                    OFFSET ${offset}
                 `.execute(kyselyDb);
+      });
+      if (fetchErr) {
+        pipelineError = fetchErr;
+        break;
       }
-      return sql`
-                SELECT ${sql.raw(columnList)}
-                FROM ${sql.table(tableName)}
-                LIMIT ${batchSize}
-                OFFSET ${offset}
-            `.execute(kyselyDb);
-    });
-    if (fetchErr) {
-      return [null, fetchErr];
-    }
-    if (rows.rows.length === 0) break;
-    for (const row of rows.rows) {
-      const values = serializeRow({ row, columns: dtSchema.columns });
-      writer.writeRow(values);
+      batchRows = rows.rows;
+    }
+    if (batchRows.length === 0) break;
+    loaded += batchRows.length;
+    observer.emit("dt:export:loaded", { table: tableName, loaded, totalRows });
+    for (const row of batchRows) {
+      const rowIndex = globalIndex++;
+      inFlight++;
+      computePool.request("serialize", {
+        row,
+        columns: dtSchema.columns,
+        index: rowIndex
+      }).then((result) => {
+        processed++;
+        observer.emit("dt:export:processed", { table: tableName, processed, totalRows });
+        if (result.error) {
+          pipelineError = pipelineError ?? new Error(result.error);
+          return;
+        }
+        orderBuffer.add(result.index, result.values);
+      });
     }
-    offset += rows.rows.length;
+    offset += batchRows.length;
     observer.emit("dt:export:progress", {
       filepath,
       table: tableName,
       rowsWritten: writer.rowsWritten,
       bytesWritten: writer.bytesWritten
     });
-    if (rows.rows.length < batchSize) break;
+    if (batchRows.length < batchSize) break;
+  }
+  while (inFlight > 0) {
+    await new Promise((resolve2) => setTimeout(resolve2, 1));
+  }
+  if (pipelineError) {
+    return [null, pipelineError];
   }
   const [, closeErr] = await attempt(() => writer.close());
   if (closeErr) {
     return [null, closeErr];
   }
-  const durationMs = Date.now() - startTime;
-  observer.emit("dt:export:complete", {
-    filepath,
-    table: tableName,
-    rowsWritten: writer.rowsWritten,
-    bytesWritten: writer.bytesWritten,
-    durationMs
-  });
   return [{ rowsWritten: writer.rowsWritten, bytesWritten: writer.bytesWritten }, null];
 }
 async function importDtFile(options) {
@@ -12326,70 +12455,158 @@ async function importDtFile(options) {
       return [null, err];
     }
   }
+  let ownedComputePool = null;
+  let computePool = options.computePool;
+  if (!computePool) {
+    computePool = createDefaultComputePool();
+    ownedComputePool = computePool;
+  }
+  const [result, workerErr] = await importFileWithWorkers({
+    reader,
+    dtSchema,
+    tableName,
+    filepath,
+    dialect,
+    version,
+    batchSize,
+    onConflict: options.onConflict ?? "fail",
+    computePool,
+    kyselyDb
+  });
+  if (ownedComputePool) {
+    await attempt(() => ownedComputePool.shutdown());
+  }
+  if (workerErr) {
+    reader.close();
+    observer.emit("error", { source: "dt:import", error: workerErr, context: { filepath, table: tableName } });
+    return [null, workerErr];
+  }
+  reader.close();
+  const durationMs = Date.now() - startTime;
+  observer.emit("dt:import:complete", {
+    filepath,
+    table: tableName,
+    rowsImported: result.rowsImported,
+    rowsSkipped: result.rowsSkipped,
+    durationMs
+  });
+  return [result, null];
+}
+async function importFileWithWorkers(ctx) {
+  const { reader, dtSchema, tableName, filepath, dialect, version, batchSize } = ctx;
+  const { onConflict, computePool, kyselyDb } = ctx;
   let rowsImported = 0;
   let rowsSkipped = 0;
-  let batch = [];
-  for await (const values of reader.rows()) {
-    const row = deserializeRow({
-      values,
-      columns: dtSchema.columns,
-      targetDialect: dialect,
-      targetVersion: version
-    });
-    batch.push(row);
-    if (batch.length >= batchSize) {
-      const columnNames = dtSchema.columns.map((c) => c.name);
-      const [batchResult, insertErr] = await insertImportBatch(
-        kyselyDb,
-        tableName,
-        batch,
-        options.onConflict ?? "fail",
-        dialect,
-        columnNames
-      );
-      if (insertErr) {
-        reader.close();
-        observer.emit("error", { source: "dt:import", error: insertErr, context: { filepath, table: tableName } });
-        return [null, insertErr];
-      }
-      rowsImported += batchResult.inserted + batchResult.updated;
-      rowsSkipped += batchResult.skipped;
-      batch = [];
-      observer.emit("dt:import:progress", {
-        filepath,
-        table: tableName,
-        rowsImported,
-        rowsSkipped
-      });
-    }
-  }
-  if (batch.length > 0) {
-    const columnNames = dtSchema.columns.map((c) => c.name);
+  let loaded = 0;
+  let processed = 0;
+  let saved = 0;
+  let globalIndex = 0;
+  let inFlight = 0;
+  let pipelineError = null;
+  let totalRows = 0;
+  let insertBatch2 = [];
+  const columnNames = dtSchema.columns.map((c) => c.name);
+  const flushInsertBatch = async () => {
+    if (insertBatch2.length === 0) return null;
     const [batchResult, insertErr] = await insertImportBatch(
       kyselyDb,
       tableName,
-      batch,
-      options.onConflict ?? "fail",
+      insertBatch2,
+      onConflict,
       dialect,
       columnNames
     );
     if (insertErr) {
-      reader.close();
-      observer.emit("error", { source: "dt:import", error: insertErr, context: { filepath, table: tableName } });
-      return [null, insertErr];
+      return insertErr;
     }
     rowsImported += batchResult.inserted + batchResult.updated;
     rowsSkipped += batchResult.skipped;
-  }
-  reader.close();
-  const durationMs = Date.now() - startTime;
-  observer.emit("dt:import:complete", {
-    filepath,
-    table: tableName,
-    rowsImported,
-    rowsSkipped,
-    durationMs
+    insertBatch2 = [];
+    observer.emit("dt:import:progress", {
+      filepath,
+      table: tableName,
+      rowsImported,
+      rowsSkipped
+    });
+    return null;
+  };
+  const orderBuffer = new OrderBuffer((record) => {
+    insertBatch2.push(record);
+    saved++;
+    inFlight--;
+    observer.emit("dt:import:saved", { table: tableName, saved, totalRows });
   });
+  let readBatch = [];
+  for await (const values of reader.rows()) {
+    readBatch.push(values);
+    loaded++;
+    totalRows = Math.max(totalRows, loaded);
+    if (readBatch.length >= batchSize) {
+      observer.emit("dt:import:loaded", { table: tableName, loaded, totalRows });
+      for (const rowValues of readBatch) {
+        const rowIndex = globalIndex++;
+        inFlight++;
+        computePool.request("deserialize", {
+          values: rowValues,
+          columns: dtSchema.columns,
+          targetDialect: dialect,
+          targetVersion: version ? `${version.major}.${version.minor}` : void 0,
+          index: rowIndex
+        }).then((result) => {
+          processed++;
+          observer.emit("dt:import:processed", { table: tableName, processed, totalRows });
+          if (result.error) {
+            pipelineError = pipelineError ?? new Error(result.error);
+            return;
+          }
+          orderBuffer.add(result.index, result.record);
+        });
+      }
+      readBatch = [];
+      while (inFlight > 0) {
+        await new Promise((resolve2) => setTimeout(resolve2, 1));
+      }
+      if (pipelineError) {
+        return [null, pipelineError];
+      }
+      const flushErr = await flushInsertBatch();
+      if (flushErr) {
+        return [null, flushErr];
+      }
+    }
+  }
+  if (readBatch.length > 0) {
+    observer.emit("dt:import:loaded", { table: tableName, loaded, totalRows });
+    for (const rowValues of readBatch) {
+      const rowIndex = globalIndex++;
+      inFlight++;
+      computePool.request("deserialize", {
+        values: rowValues,
+        columns: dtSchema.columns,
+        targetDialect: dialect,
+        targetVersion: version ? `${version.major}.${version.minor}` : void 0,
+        index: rowIndex
+      }).then((result) => {
+        processed++;
+        observer.emit("dt:import:processed", { table: tableName, processed, totalRows });
+        if (result.error) {
+          pipelineError = pipelineError ?? new Error(result.error);
+          return;
+        }
+        orderBuffer.add(result.index, result.record);
+      });
+    }
+    while (inFlight > 0) {
+      await new Promise((resolve2) => setTimeout(resolve2, 1));
+    }
+    if (pipelineError) {
+      return [null, pipelineError];
+    }
+    const flushErr = await flushInsertBatch();
+    if (flushErr) {
+      return [null, flushErr];
+    }
+  }
   return [{ rowsImported, rowsSkipped }, null];
 }
 async function insertImportBatch(db, table, rows, onConflict, dialect, columns) {