dbgate-api-premium 7.1.10 → 7.1.12

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "dbgate-api-premium",
   "main": "src/index.js",
-  "version": "7.1.10",
+  "version": "7.1.12",
   "homepage": "https://www.dbgate.io/",
   "repository": {
     "type": "git",
@@ -30,11 +30,11 @@
     "compare-versions": "^3.6.0",
     "cors": "^2.8.5",
     "cross-env": "^6.0.3",
-    "dbgate-datalib": "7.1.10",
+    "dbgate-datalib": "7.1.12",
     "dbgate-query-splitter": "^4.12.0",
-    "dbgate-rest": "7.1.10",
-    "dbgate-sqltree": "7.1.10",
-    "dbgate-tools": "7.1.10",
+    "dbgate-rest": "7.1.12",
+    "dbgate-sqltree": "7.1.12",
+    "dbgate-tools": "7.1.12",
     "debug": "^4.3.4",
     "diff": "^5.0.0",
     "diff2html": "^3.4.13",
@@ -42,7 +42,6 @@
     "express": "^4.17.1",
     "express-basic-auth": "^1.2.0",
     "express-fileupload": "^1.2.0",
-    "external-sorting": "^1.3.1",
     "fs-extra": "^9.1.0",
     "fs-reverse": "^0.0.3",
     "get-port": "^5.1.1",
@@ -81,15 +80,25 @@
     "start:singleconn": "env-cmd node src/index.js --server localhost --user root --port 3307 --engine mysql@dbgate-plugin-mysql --password test --listen-api",
     "start:azure": "env-cmd -f env/azure/.env node src/index.js --listen-api",
     "start:e2e:team": "cross-env DEVWEB=1 DEVMODE=1 env-cmd -f ../../e2e-tests/env/team/.env node src/index.js --listen-api",
+    "test": "jest",
+    "test:ci": "jest --json --outputFile=result.json --testLocationInResults",
     "ts": "tsc",
     "build": "webpack",
     "build:doc": "jsdoc2md --template doctpl.hbs  ./src/shell/* > ../../../dbgate.github.io/_docs/apidoc.md"
   },
+  "jest": {
+    "testMatch": [
+      "**/*Test.js",
+      "**/*.test.js"
+    ]
+  },
   "devDependencies": {
     "@types/fs-extra": "^9.0.11",
+    "@types/jest": "^30.0.0",
     "@types/lodash": "^4.14.149",
-    "dbgate-types": "7.1.10",
+    "dbgate-types": "7.1.12",
     "env-cmd": "^10.1.0",
+    "jest": "^30.4.2",
     "jsdoc-to-markdown": "^9.0.5",
     "node-loader": "^1.0.2",
     "nodemon": "^2.0.2",

package/src/controllers/config.js

@@ -264,7 +264,7 @@ module.exports = {
     try {
       const fingerprint = await getPublicHardwareFingerprint();
 
-      const resp = await axios.default.post(`${getAuthProxyUrl()}/trial-license`, {
+      const resp = await axios.default.post(`https://api.dbgate.cloud/trial-license`, {
         type: 'premium-trial',
         days: 30,
         fingerprint,

package/src/controllers/databaseConnections.js

@@ -15,6 +15,7 @@ const {
   getLogger,
   extractErrorLogData,
   filterStructureBySchema,
+  isCompositeDbName,
   serializeJsTypesForJsonStringify,
 } = require('dbgate-tools');
 const { html, parse } = require('diff2html');
@@ -194,6 +195,8 @@ module.exports = {
     );
     pipeForkLogs(subprocess);
     const lastClosed = this.closed[`${conid}/${database}`];
+    const initialStatusName =
+      !lastClosed && !(connection.useSeparateSchemas && !isCompositeDbName(database)) ? 'loadStructure' : 'pending';
     const newOpened = {
       conid,
       database,
@@ -201,7 +204,7 @@ module.exports = {
       structure: lastClosed ? lastClosed.structure : DatabaseAnalyser.createEmptyStructure(),
       serverVersion: lastClosed ? lastClosed.serverVersion : null,
       connection,
-      status: { name: 'pending' },
+      status: { name: initialStatusName },
     };
     this.opened.push(newOpened);
     subprocess.on('message', message => {

package/src/currentVersion.js

@@ -1,5 +1,5 @@
 
 module.exports = { 
-  version: '7.1.10',
-  buildTime: '2026-04-29T12:39:25.294Z'
+  version: '7.1.12',
+  buildTime: '2026-05-20T09:12:47.123Z'
 };

package/src/proc/databaseConnectionProcess.js

@@ -69,6 +69,16 @@ async function checkedAsyncCall(promise) {
 }
 
 let loadingModel = false;
+let queuedSyncModelFullRefresh = null;
+
+function finishLoadingModel() {
+  loadingModel = false;
+  if (queuedSyncModelFullRefresh != null) {
+    const isFullRefresh = queuedSyncModelFullRefresh;
+    queuedSyncModelFullRefresh = null;
+    handleSyncModel({ isFullRefresh });
+  }
+}
 
 async function handleFullRefresh() {
   if (storedConnection.useSeparateSchemas && !isCompositeDbName(dbhan?.database)) {
@@ -86,7 +96,7 @@ async function handleFullRefresh() {
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
 
-  loadingModel = false;
+  finishLoadingModel();
   resolveAnalysedPromises();
 }
 
@@ -111,12 +121,15 @@ async function handleIncrementalRefresh(forceSend) {
 
   process.send({ msgtype: 'structureTime', analysedTime });
   setStatusName('ok');
-  loadingModel = false;
+  finishLoadingModel();
   resolveAnalysedPromises();
 }
 
 function handleSyncModel({ isFullRefresh }) {
-  if (loadingModel) return;
+  if (loadingModel) {
+    queuedSyncModelFullRefresh = queuedSyncModelFullRefresh || !!isFullRefresh;
+    return;
+  }
   if (isFullRefresh) handleFullRefresh();
   else handleIncrementalRefresh();
 }

package/src/utility/JsonLinesDatastore.js

@@ -1,15 +1,16 @@
 const crypto = require('crypto');
 const fs = require('fs');
-const os = require('os');
-const rimraf = require('rimraf');
 const path = require('path');
 const AsyncLock = require('async-lock');
 const lock = new AsyncLock();
 const stableStringify = require('json-stable-stringify');
 const { evaluateCondition } = require('dbgate-sqltree');
-const esort = require('external-sorting');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
 const { jsldir } = require('./directories');
 const LineReader = require('./LineReader');
+const { sortFile } = require('./externalSort');
+
+const logger = getLogger('JsonLinesDatastore');
 
 class JsonLinesDatastore {
   constructor(file, formatterFunction) {
@@ -30,33 +31,7 @@ class JsonLinesDatastore {
   }
 
   static async sortFile(infile, outfile, sort) {
-    const tempDir = path.join(os.tmpdir(), crypto.randomUUID());
-    fs.mkdirSync(tempDir);
-
-    await esort
-      .default({
-        input: fs.createReadStream(infile),
-        output: fs.createWriteStream(outfile),
-        deserializer: JSON.parse,
-        serializer: JSON.stringify,
-        tempDir,
-        maxHeap: 100,
-        comparer: (a, b) => {
-          for (const item of sort) {
-            const { uniqueName, order } = item;
-            if (a[uniqueName] < b[uniqueName]) {
-              return order == 'ASC' ? -1 : 1;
-            }
-            if (a[uniqueName] > b[uniqueName]) {
-              return order == 'ASC' ? 1 : -1;
-            }
-          }
-          return 0;
-        },
-      })
-      .asc();
-
-    await new Promise(resolve => rimraf(tempDir, resolve));
+    return sortFile(infile, outfile, sort);
   }
 
   async _closeReader() {
@@ -214,8 +189,16 @@ class JsonLinesDatastore {
     if (sort && !this.sortedFiles[stableStringify(sort)]) {
       const jslid = crypto.randomUUID();
       const sortedFile = path.join(jsldir(), `${jslid}.jsonl`);
-      await JsonLinesDatastore.sortFile(this.file, sortedFile, sort);
-      this.sortedFiles[stableStringify(sort)] = sortedFile;
+      try {
+        await JsonLinesDatastore.sortFile(this.file, sortedFile, sort);
+        this.sortedFiles[stableStringify(sort)] = sortedFile;
+      } catch (e) {
+        logger.error(extractErrorLogData(e), 'DBGM-00000 Failed to sort data file, returning unsorted results');
+        // Remove any partial output file left by the failed sort so it does
+        // not accumulate in jsldir() across repeated failures.
+        try { fs.unlinkSync(sortedFile); } catch { /* best-effort */ }
+        sort = null;
+      }
     }
     await lock.acquire('reader', async () => {
       await this._ensureReader(offset, filter, sort);

package/src/utility/authProxy.js

@@ -21,7 +21,7 @@ const AI_GATEWAY_URL = process.env.LOCAL_AI_GATEWAY
   ? 'http://localhost:3110'
   : process.env.DEVWEB || process.env.DEVMODE
   ? 'https://aigw.dbgate.udolni.net'
-  : 'https://aigw.dbgate.io';
+  : 'https://api.dbgate.cloud';
 
 const DBGATE_API_URL = process.env.LOCAL_DBGATE_API
   ? 'http://localhost:3115'
@@ -177,7 +177,7 @@ async function obtainRefreshedLicense() {
 
     try {
       const respToken = await axios.default.post(
-        `${AUTH_PROXY_URL}/refresh-license`,
+        `https://api.dbgate.cloud/refresh-license`,
         {},
         {
           headers: {
@@ -199,7 +199,7 @@ async function obtainRefreshedLicense() {
 async function tryToGetRefreshedLicense(oldLicenseKey) {
   try {
     const respToken = await axios.default.post(
-      `${AUTH_PROXY_URL}/refresh-license`,
+      `https://api.dbgate.cloud/refresh-license`,
       {},
       {
         headers: {

package/src/utility/cloudIntf.js

@@ -34,7 +34,7 @@ const DBGATE_CLOUD_URL = process.env.LOCAL_DBGATE_CLOUD
   : process.env.PROD_DBGATE_CLOUD
   ? 'https://cloud.dbgate.io'
   : process.env.DEVWEB || process.env.DEVMODE
-  ? 'https://cloud.dbgate.udolni.net'
+  ? 'https://dev.dbgate.cloud'
   : 'https://cloud.dbgate.io';
 
 
@@ -297,7 +297,7 @@ async function updatePremiumPromoWidget(language) {
   const tags = (await collectCloudFilesSearchTags()).join(',');
 
   const resp = await axios.default.get(
-    `${DBGATE_CLOUD_URL}/premium-promo-widget?identifier=${promoWidgetData?.identifier ?? 'empty'}&tags=${tags}`,
+    `${DBGATE_PUBLIC_CLOUD_URL}/premium-promo-widget?identifier=${promoWidgetData?.identifier ?? 'empty'}&tags=${tags}`,
     {
       headers: {
         ...getLicenseHttpHeaders(),
@@ -508,14 +508,14 @@ async function getPromoWidgetData() {
 
 async function getPromoWidgetPreview(campaign, variant) {
   const resp = await axios.default.get(
-    `${DBGATE_CLOUD_URL}/premium-promo-widget-preview/${campaign}/${variant}`,
+    `${DBGATE_PUBLIC_CLOUD_URL}/premium-promo-widget-preview/${campaign}/${variant}`,
     stageAxiosConfig
   );
   return resp.data;
 }
 
 async function getPromoWidgetList() {
-  const resp = await axios.default.get(`${DBGATE_CLOUD_URL}/promo-widget-list`, stageAxiosConfig);
+  const resp = await axios.default.get(`${DBGATE_PUBLIC_CLOUD_URL}/promo-widget-list`, stageAxiosConfig);
   return resp.data;
 }
 

package/src/utility/externalSort.js

@@ -0,0 +1,366 @@
+const crypto = require('crypto');
+const fs = require('fs');
+const { pipeline } = require('stream');
+const os = require('os');
+const readline = require('readline');
+const path = require('path');
+const { getLogger, extractErrorLogData } = require('dbgate-tools');
+
+const logger = getLogger('externalSort');
+
+// Number of rows accumulated per sorted temp-chunk during external sort.
+// Capped so that a single chunk never exceeds ~50 MB for typical row sizes.
+const SORT_CHUNK_SIZE = 50_000;
+
+// Maximum number of chunk files merged simultaneously in one pass.
+// Limits the number of concurrently open file descriptors during merge.
+const MAX_MERGE_FAN_IN = 16;
+
+// Async generator that yields parsed JSON objects from an internally-generated
+// sorted chunk file. Parse errors are thrown immediately — chunk files are
+// always written by this module, so a parse error indicates corruption.
+async function* readChunkLines(file) {
+  const stream = fs.createReadStream(file);
+  const rl = readline.createInterface({ input: stream, crlfDelay: Infinity });
+  try {
+    for await (const line of rl) {
+      if (!line.trim()) continue;
+      yield JSON.parse(line); // intentionally throws on bad JSON
+    }
+  } finally {
+    rl.close();
+    stream.destroy();
+  }
+}
+
+// Cross-device-safe rename: tries renameSync first; on EXDEV (cross-filesystem)
+// falls back to a streaming copy followed by unlinking the source.
+// Uses stream.pipeline() so both streams are destroyed and the partial
+// destination file is removed if an error occurs mid-copy.
+async function safeRename(src, dest) {
+  try {
+    fs.renameSync(src, dest);
+  } catch (e) {
+    if (e.code !== 'EXDEV') throw e;
+    await new Promise((resolve, reject) => {
+      const rs = fs.createReadStream(src);
+      const ws = fs.createWriteStream(dest);
+      pipeline(rs, ws, err => {
+        if (err) {
+          try { fs.unlinkSync(dest); } catch { /* best-effort */ }
+          reject(err);
+        } else {
+          resolve();
+        }
+      });
+    });
+    fs.unlinkSync(src);
+  }
+}
+
+// Write an array of rows to a JSON-lines file, respecting stream backpressure.
+function writeChunkFile(filePath, rows) {
+  return new Promise((resolve, reject) => {
+    const ws = fs.createWriteStream(filePath);
+    ws.on('error', reject);
+    ws.on('finish', resolve);
+    const writeNext = i => {
+      for (; i < rows.length; i++) {
+        const ok = ws.write(JSON.stringify(rows[i]) + '\n');
+        if (!ok) {
+          ws.once('drain', () => writeNext(i + 1));
+          return;
+        }
+      }
+      ws.end();
+    };
+    writeNext(0);
+  });
+}
+
+// Min-heap used for k-way merge.  Each item stored in the heap is
+// { row: object, iter: AsyncGenerator }.
+class _SortMinHeap {
+  constructor(comparator) {
+    this._data = [];
+    this._cmp = comparator;
+  }
+  get size() { return this._data.length; }
+  push(item) {
+    this._data.push(item);
+    let i = this._data.length - 1;
+    while (i > 0) {
+      const p = (i - 1) >> 1;
+      if (this._cmp(this._data[i].row, this._data[p].row) < 0) {
+        [this._data[i], this._data[p]] = [this._data[p], this._data[i]];
+        i = p;
+      } else break;
+    }
+  }
+  pop() {
+    const top = this._data[0];
+    const last = this._data.pop();
+    if (this._data.length > 0) {
+      this._data[0] = last;
+      let i = 0;
+      for (;;) {
+        const l = 2 * i + 1, r = 2 * i + 2, n = this._data.length;
+        let min = i;
+        if (l < n && this._cmp(this._data[l].row, this._data[min].row) < 0) min = l;
+        if (r < n && this._cmp(this._data[r].row, this._data[min].row) < 0) min = r;
+        if (min === i) break;
+        [this._data[i], this._data[min]] = [this._data[min], this._data[i]];
+        i = min;
+      }
+    }
+    return top;
+  }
+}
+
+// Merge exactly inputFiles.length (≤ MAX_MERGE_FAN_IN) sorted chunk files into
+// one output file. Opens exactly inputFiles.length file descriptors at once.
+async function mergeBatch(inputFiles, outfile, comparator) {
+  const ws = fs.createWriteStream(outfile);
+  const iters = inputFiles.map(f => readChunkLines(f));
+
+  const cleanup = () => {
+    ws.destroy();
+    for (const it of iters) it.return();
+  };
+
+  try {
+    await new Promise((resolve, reject) => {
+      ws.on('error', err => { cleanup(); reject(err); });
+
+      const heap = new _SortMinHeap(comparator);
+      let settled = false;
+
+      const fail = err => {
+        if (settled) return;
+        settled = true;
+        cleanup();
+        reject(err);
+      };
+
+      const advance = async iter => {
+        const { value, done } = await iter.next();
+        if (!done) heap.push({ row: value, iter });
+      };
+
+      const drain = async () => {
+        try {
+          await Promise.all(iters.map(advance));
+
+          while (heap.size > 0) {
+            if (settled) return;
+            const { row, iter } = heap.pop();
+            const line = JSON.stringify(row) + '\n';
+            const ok = ws.write(line);
+            if (!ok) await new Promise(r => ws.once('drain', r));
+            await advance(iter);
+          }
+
+          if (settled) return;
+          ws.end();
+          ws.once('finish', () => { settled = true; resolve(); });
+        } catch (e) {
+          fail(e);
+        }
+      };
+
+      drain();
+    });
+  } catch (e) {
+    // cleanup() was already called inside the Promise; re-throw so callers see the error.
+    throw e;
+  }
+}
+
+// Multi-pass k-way merge. Each pass merges groups of ≤ MAX_MERGE_FAN_IN files,
+// writing intermediates via nextTmpFile() so they are tracked for cleanup.
+// The very last pass writes directly to outfile (no cross-fs rename needed).
+async function multiPassMerge(inputFiles, outfile, comparator, nextTmpFile) {
+  let current = inputFiles;
+  while (current.length > MAX_MERGE_FAN_IN) {
+    const next = [];
+    for (let i = 0; i < current.length; i += MAX_MERGE_FAN_IN) {
+      const batch = current.slice(i, i + MAX_MERGE_FAN_IN);
+      const merged = nextTmpFile();
+      next.push(merged);
+      await mergeBatch(batch, merged, comparator);
+    }
+    current = next;
+  }
+  await mergeBatch(current, outfile, comparator);
+}
+
+async function sortFile(infile, outfile, sort) {
+  const comparator = (a, b) => {
+    for (const { uniqueName, order } of sort) {
+      const av = a[uniqueName], bv = b[uniqueName];
+      if (av < bv) return order === 'ASC' ? -1 : 1;
+      if (av > bv) return order === 'ASC' ? 1 : -1;
+    }
+    return 0;
+  };
+
+  const tmpDir = path.join(os.tmpdir(), `dbgate-sort-${crypto.randomUUID()}`);
+  fs.mkdirSync(tmpDir, { recursive: true });
+
+  // All tmp paths are registered here BEFORE any write attempt so that the
+  // finally block can unlink partial files even when a write fails mid-way.
+  const createdTmpFiles = new Set();
+  let tmpCounter = 0;
+  const nextTmpFile = () => {
+    const f = path.join(tmpDir, `es-${tmpCounter++}.jsonl`);
+    createdTmpFiles.add(f);
+    return f;
+  };
+
+  try {
+    // ── Phase 1: generate sorted runs ──────────────────────────────────────
+    // Read the input line by line; accumulate SORT_CHUNK_SIZE rows, sort
+    // them, write to a temp file, then discard the chunk from memory.
+    // Peak memory ≈ one chunk (SORT_CHUNK_SIZE rows).
+    //
+    // The first non-empty line is inspected for __isStreamHeader. If found,
+    // it is saved and excluded from sorting so it can be written back as the
+    // very first line of the output file.
+    let chunk = [];
+    const runFiles = [];
+    let headerRow = null;
+    let isFirstNonEmptyLine = true;
+
+    const flushChunk = async () => {
+      if (chunk.length === 0) return;
+      chunk.sort(comparator);
+      // Register the path BEFORE writing so the finally block can always
+      // clean it up even if writeChunkFile throws partway through.
+      const tmpFile = nextTmpFile();
+      runFiles.push(tmpFile);
+      await writeChunkFile(tmpFile, chunk);
+      chunk = [];
+    };
+
+    await new Promise((resolve, reject) => {
+      const inputStream = fs.createReadStream(infile);
+      const rl = readline.createInterface({ input: inputStream, crlfDelay: Infinity });
+      let pendingFlush = Promise.resolve();
+      let settled = false;
+
+      const fail = err => {
+        if (settled) return;
+        settled = true;
+        // Destroy both the readline interface and the underlying stream so
+        // no file descriptors are leaked when we reject while paused.
+        rl.close();
+        inputStream.destroy();
+        reject(err);
+      };
+
+      // Attach directly to the stream — readline.Interface does not reliably
+      // forward the underlying stream's 'error' event.
+      inputStream.on('error', fail);
+      rl.on('error', fail);
+
+      rl.on('line', line => {
+        if (!line.trim()) return;
+        let parsed;
+        try {
+          parsed = JSON.parse(line);
+        } catch (e) {
+          logger.warn(extractErrorLogData(e), 'DBGM-00000 Skipping invalid JSON line during sort');
+          return;
+        }
+        // Detect and capture the stream header; do not include it in the sort.
+        if (isFirstNonEmptyLine) {
+          isFirstNonEmptyLine = false;
+          if (parsed.__isStreamHeader) {
+            headerRow = parsed;
+            return;
+          }
+        }
+        chunk.push(parsed);
+        if (chunk.length >= SORT_CHUNK_SIZE) {
+          rl.pause();
+          pendingFlush = pendingFlush
+            .then(() => flushChunk())
+            .then(() => rl.resume())
+            .catch(fail);
+        }
+      });
+
+      rl.on('close', () => {
+        if (settled) return;
+        pendingFlush
+          .then(() => flushChunk())
+          .then(() => { settled = true; resolve(); })
+          .catch(fail);
+      });
+    });
+
+    // ── Phase 2: k-way streaming merge ─────────────────────────────────────
+    if (headerRow !== null) {
+      // There is a stream header that must appear as the first line of outfile.
+      // Merge all data runs into a single intermediate file (or use the single
+      // run directly), then write outfile = header + merged data.
+      let mergedDataFile = null;
+      if (runFiles.length === 1) {
+        mergedDataFile = runFiles[0]; // cleaned up by the finally block
+      } else if (runFiles.length > 1) {
+        mergedDataFile = nextTmpFile();
+        await multiPassMerge(runFiles, mergedDataFile, comparator, nextTmpFile);
+      }
+
+      await new Promise((resolve, reject) => {
+        const ws = fs.createWriteStream(outfile);
+        let settled = false;
+        const fail = err => {
+          if (settled) return;
+          settled = true;
+          ws.destroy();
+          try { fs.unlinkSync(outfile); } catch { /* best-effort */ }
+          reject(err);
+        };
+        ws.on('error', fail);
+
+        const headerLine = JSON.stringify(headerRow) + '\n';
+        if (!mergedDataFile) {
+          // Header only — no data rows.
+          ws.end(headerLine);
+          ws.once('finish', () => { settled = true; resolve(); });
+        } else {
+          ws.write(headerLine, writeErr => {
+            if (writeErr) return fail(writeErr);
+            const rs = fs.createReadStream(mergedDataFile);
+            rs.on('error', fail);
+            rs.pipe(ws);
+            ws.once('finish', () => { settled = true; resolve(); });
+          });
+        }
+      });
+    } else if (runFiles.length === 0) {
+      fs.writeFileSync(outfile, '');
+    } else if (runFiles.length === 1) {
+      // safeRename handles EXDEV (cross-filesystem) by falling back to
+      // stream-copy + unlink, so outfile is always populated correctly.
+      await safeRename(runFiles[0], outfile);
+      // The file is now at outfile (or already unlinked on EXDEV path),
+      // so remove it from the cleanup set to avoid a spurious unlink error.
+      createdTmpFiles.delete(runFiles[0]);
+    } else {
+      // multiPassMerge batches the fan-in to MAX_MERGE_FAN_IN files per pass,
+      // bounding the number of concurrently open file descriptors.
+      // Intermediate files are allocated via nextTmpFile() so they are always
+      // tracked in createdTmpFiles for cleanup.
+      await multiPassMerge(runFiles, outfile, comparator, nextTmpFile);
+    }
+  } finally {
+    for (const f of createdTmpFiles) {
+      try { fs.unlinkSync(f); } catch { /* best-effort cleanup */ }
+    }
+    try { fs.rmdirSync(tmpDir); } catch { /* best-effort cleanup */ }
+  }
+}
+
+module.exports = { sortFile };

package/src/utility/externalSortTest.js

@@ -0,0 +1,275 @@
+'use strict';
+
+const fs = require('fs');
+const os = require('os');
+const path = require('path');
+const { sortFile } = require('./externalSort');
+
+// ── helpers ──────────────────────────────────────────────────────────────────
+
+function tmpFile() {
+  return path.join(os.tmpdir(), `externalSortTest-${Date.now()}-${Math.random().toString(36).slice(2)}.jsonl`);
+}
+
+function writeJsonl(filePath, rows) {
+  const content = rows.map(r => JSON.stringify(r)).join('\n');
+  fs.writeFileSync(filePath, rows.length ? content + '\n' : '');
+}
+
+function readJsonl(filePath) {
+  return fs
+    .readFileSync(filePath, 'utf8')
+    .split('\n')
+    .filter(l => l.trim())
+    .map(l => JSON.parse(l));
+}
+
+async function withTmpPair(fn) {
+  const inFile = tmpFile();
+  const outFile = tmpFile();
+  try {
+    await fn(inFile, outFile);
+  } finally {
+    for (const f of [inFile, outFile]) {
+      try { fs.unlinkSync(f); } catch { /* best-effort */ }
+    }
+  }
+}
+
+// ── tests ─────────────────────────────────────────────────────────────────────
+
+describe('externalSort', () => {
+
+  // ── Basic sorts ────────────────────────────────────────────────────────────
+
+  test('sorts numbers ASC', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ id: 3 }, { id: 1 }, { id: 2 }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      expect(readJsonl(outFile).map(r => r.id)).toEqual([1, 2, 3]);
+    });
+  });
+
+  test('sorts numbers DESC', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ id: 1 }, { id: 3 }, { id: 2 }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'DESC' }]);
+      expect(readJsonl(outFile).map(r => r.id)).toEqual([3, 2, 1]);
+    });
+  });
+
+  test('sorts strings ASC', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ name: 'Zebra' }, { name: 'Apple' }, { name: 'Mango' }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'name', order: 'ASC' }]);
+      expect(readJsonl(outFile).map(r => r.name)).toEqual(['Apple', 'Mango', 'Zebra']);
+    });
+  });
+
+  test('sorts strings DESC', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ name: 'Zebra' }, { name: 'Apple' }, { name: 'Mango' }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'name', order: 'DESC' }]);
+      expect(readJsonl(outFile).map(r => r.name)).toEqual(['Zebra', 'Mango', 'Apple']);
+    });
+  });
+
+  // ── Multi-field sort ────────────────────────────────────────────────────────
+
+  test('sorts by multiple fields (primary ASC, secondary DESC)', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { category: 'B', value: 1 },
+        { category: 'A', value: 2 },
+        { category: 'A', value: 5 },
+        { category: 'B', value: 3 },
+      ]);
+      await sortFile(inFile, outFile, [
+        { uniqueName: 'category', order: 'ASC' },
+        { uniqueName: 'value', order: 'DESC' },
+      ]);
+      const rows = readJsonl(outFile);
+      expect(rows.map(r => [r.category, r.value])).toEqual([['A', 5], ['A', 2], ['B', 3], ['B', 1]]);
+    });
+  });
+
+  // ── With stream header ──────────────────────────────────────────────────────
+
+  test('with header: header is first row, data rows are sorted', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { __isStreamHeader: true, title: 'dataset' },
+        { id: 3, val: 'c' },
+        { id: 1, val: 'a' },
+        { id: 2, val: 'b' },
+      ]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows[0].__isStreamHeader).toBe(true);
+      expect(rows.slice(1).map(r => r.id)).toEqual([1, 2, 3]);
+    });
+  });
+
+  test('with header: header fields are preserved intact', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { __isStreamHeader: true, columns: ['a', 'b'], extra: 42 },
+        { a: 2, b: 'x' },
+        { a: 1, b: 'y' },
+      ]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'a', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows[0]).toEqual({ __isStreamHeader: true, columns: ['a', 'b'], extra: 42 });
+    });
+  });
+
+  test('with header + multi-field sort', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { __isStreamHeader: true },
+        { dept: 'HR', salary: 50000 },
+        { dept: 'IT', salary: 80000 },
+        { dept: 'HR', salary: 70000 },
+        { dept: 'IT', salary: 60000 },
+      ]);
+      await sortFile(inFile, outFile, [
+        { uniqueName: 'dept', order: 'ASC' },
+        { uniqueName: 'salary', order: 'DESC' },
+      ]);
+      const rows = readJsonl(outFile);
+      expect(rows[0].__isStreamHeader).toBe(true);
+      expect(rows.slice(1).map(r => [r.dept, r.salary])).toEqual([
+        ['HR', 70000],
+        ['HR', 50000],
+        ['IT', 80000],
+        ['IT', 60000],
+      ]);
+    });
+  });
+
+  // ── Without stream header ───────────────────────────────────────────────────
+
+  test('without header: all rows sorted, no header injected', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ x: 30 }, { x: 10 }, { x: 20 }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'x', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(3);
+      expect(rows[0].__isStreamHeader).toBeFalsy();
+      expect(rows.map(r => r.x)).toEqual([10, 20, 30]);
+    });
+  });
+
+  // ── Edge cases ───────────────────────────────────────────────────────────────
+
+  test('edge: empty file produces empty output', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      fs.writeFileSync(inFile, '');
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      expect(fs.readFileSync(outFile, 'utf8')).toBe('');
+    });
+  });
+
+  test('edge: single data row is passed through unchanged', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ id: 42, msg: 'only one' }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(1);
+      expect(rows[0].id).toBe(42);
+    });
+  });
+
+  test('edge: header-only file (no data rows) produces only header', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [{ __isStreamHeader: true, columns: ['a', 'b'] }]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'a', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(1);
+      expect(rows[0].__isStreamHeader).toBe(true);
+    });
+  });
+
+  test('edge: all rows have equal sort key — all rows preserved', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { score: 10, label: 'a' },
+        { score: 10, label: 'b' },
+        { score: 10, label: 'c' },
+      ]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'score', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(3);
+      expect(rows.every(r => r.score === 10)).toBe(true);
+    });
+  });
+
+  test('edge: invalid JSON lines are skipped gracefully', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      fs.writeFileSync(inFile, '{"id":3}\nNOT_JSON\n{"id":1}\n{"id":2}\n');
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.map(r => r.id)).toEqual([1, 2, 3]);
+    });
+  });
+
+  test('edge: blank lines in input are ignored', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      fs.writeFileSync(inFile, '{"id":2}\n\n{"id":1}\n\n');
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.map(r => r.id)).toEqual([1, 2]);
+    });
+  });
+
+  test('edge: rows missing sort key field are included in output', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { id: 2, name: 'Bob' },
+        { name: 'Alice' },         // no id field
+        { id: 1, name: 'Charlie' },
+      ]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'id', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(3);
+      expect(rows.find(r => r.name === 'Alice')).toBeTruthy();
+      expect(rows.find(r => r.id === 1)).toBeTruthy();
+      expect(rows.find(r => r.id === 2)).toBeTruthy();
+    });
+  });
+
+  test('edge: unicode values are preserved correctly', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { word: 'über' },
+        { word: 'apple' },
+        { word: 'éclair' },
+      ]);
+      await sortFile(inFile, outFile, [{ uniqueName: 'word', order: 'ASC' }]);
+      const rows = readJsonl(outFile);
+      expect(rows.length).toBe(3);
+      const words = new Set(rows.map(r => r.word));
+      expect(words.has('über')).toBe(true);
+      expect(words.has('apple')).toBe(true);
+      expect(words.has('éclair')).toBe(true);
+    });
+  });
+
+  test('edge: two rows with same primary key, different secondary key', async () => {
+    await withTmpPair(async (inFile, outFile) => {
+      writeJsonl(inFile, [
+        { pk: 'a', sk: 2 },
+        { pk: 'b', sk: 1 },
+        { pk: 'a', sk: 1 },
+      ]);
+      await sortFile(inFile, outFile, [
+        { uniqueName: 'pk', order: 'ASC' },
+        { uniqueName: 'sk', order: 'ASC' },
+      ]);
+      const rows = readJsonl(outFile);
+      expect(rows.map(r => [r.pk, r.sk])).toEqual([['a', 1], ['a', 2], ['b', 1]]);
+    });
+  });
+
+});
+