@delali/narsil 0.3.1-next.72 → 0.3.1-next.73

package/dist/{chunk-RV77FR2F.mjs → chunk-I64LXKO3.mjs}

@@ -72,7 +72,6 @@ var ErrorCodes = {
   SNAPSHOT_SYNC_UNAUTHORIZED: "SNAPSHOT_SYNC_UNAUTHORIZED",
   SNAPSHOT_SYNC_REQUEST_INVALID: "SNAPSHOT_SYNC_REQUEST_INVALID",
   SNAPSHOT_SYNC_INDEX_NOT_FOUND: "SNAPSHOT_SYNC_INDEX_NOT_FOUND",
-  SNAPSHOT_SYNC_TOO_LARGE: "SNAPSHOT_SYNC_TOO_LARGE",
   SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: "SNAPSHOT_SYNC_CAPACITY_EXHAUSTED",
   SNAPSHOT_SYNC_SNAPSHOT_FAILED: "SNAPSHOT_SYNC_SNAPSHOT_FAILED",
   SNAPSHOT_SYNC_DECODE_FAILED: "SNAPSHOT_SYNC_DECODE_FAILED",

package/dist/{chunk-EDMCZLSJ.mjs → chunk-LPBWCHBE.mjs}

@@ -1,6 +1,9 @@
-import { MAX_SORT_FIELDS } from './chunk-M6XF5HDE.mjs';
-import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-P2ZZN32C.mjs';
-import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-RV77FR2F.mjs';
+import { MAX_SORT_FIELDS } from './chunk-AK3BLGCI.mjs';
+import { nativeSearchCoreIsEnabled, loadNativeCore } from './chunk-PU43WDY4.mjs';
+import { NarsilError, ServerErrorCodes, ErrorCodes } from './chunk-I64LXKO3.mjs';
+import { mkdtemp, open, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import path from 'path';
 
 // src/server/capabilities.ts
 var ASYNC_IMPORT_CAPABILITY = "documents.import.async";
@@ -8,12 +11,14 @@ var TASK_CANCEL_CAPABILITY = "tasks.cancel";
 var TASK_FILTER_CAPABILITY = "tasks.filter";
 var REBUILD_ANALYSIS_CAPABILITY = "indexes.rebuildAnalysis";
 var INDEX_LIFECYCLE_CAPABILITY = "indexes.lifecycle";
+var DOCUMENT_EXPORT_CAPABILITY = "indexes.documentExport";
 var SERVER_CAPABILITIES = [
   ASYNC_IMPORT_CAPABILITY,
   TASK_CANCEL_CAPABILITY,
   TASK_FILTER_CAPABILITY,
   REBUILD_ANALYSIS_CAPABILITY,
-  INDEX_LIFECYCLE_CAPABILITY
+  INDEX_LIFECYCLE_CAPABILITY,
+  DOCUMENT_EXPORT_CAPABILITY
 ];
 
 // src/server/concurrency-gate.ts
@@ -171,7 +176,6 @@ var STATUS_BY_CODE = {
   SNAPSHOT_SYNC_HEADER_INVALID: 400,
   SNAPSHOT_SYNC_FRAME_INVALID: 400,
   SNAPSHOT_SYNC_INDEX_NOT_FOUND: 404,
-  SNAPSHOT_SYNC_TOO_LARGE: 413,
   SNAPSHOT_SYNC_CHUNK_SIZE_EXCEEDED: 413,
   SNAPSHOT_SYNC_ABORTED: 409,
   SNAPSHOT_SYNC_CAPACITY_EXHAUSTED: 503,
@@ -226,6 +230,11 @@ function encodeJson(value) {
 var BYTES_PER_MB = 1024 * 1024;
 var DEFAULT_MAX_BODY_BYTES = 16 * BYTES_PER_MB;
 var DEFAULT_MAX_IMPORT_BYTES = 100 * BYTES_PER_MB;
+var DEFAULT_MAX_RESTORE_BYTES = 64 * 1024 * BYTES_PER_MB;
+var SPOOL_PAUSE_ABOVE_BYTES = 8 * BYTES_PER_MB;
+var SNAPSHOT_ARCHIVE_CONTENT_TYPE = "application/x-tar";
+var DOCUMENT_EXPORT_CONTENT_TYPE = "application/x-ndjson";
+var RELAY_STREAM_CHUNKS_IN_FLIGHT = 4;
 var DEFAULT_MAX_LINE_BYTES = 4 * BYTES_PER_MB;
 var DEFAULT_IMPORT_BATCH_SIZE = 1e3;
 var IMPORT_FLUSHES_IN_FLIGHT = 4;
@@ -300,8 +309,56 @@ function sendError(res, status, code, message, details) {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
 }
-function sendBinary(res, data, abort, status = 200) {
-  streamBody(res, abort, data, status, [["Content-Type", "application/octet-stream"]]);
+async function sendStream(res, abort, source, headers) {
+  const reader = source.getReader();
+  let wake = null;
+  abort.onAbort(() => wake?.());
+  const waitUntilWritable = () => new Promise((resolve) => {
+    wake = () => {
+      wake = null;
+      resolve();
+    };
+    res.onWritable(() => {
+      wake?.();
+      return true;
+    });
+  });
+  let started = false;
+  const begin = () => {
+    if (started) return;
+    started = true;
+    res.writeStatus(statusLine(200));
+    for (const [key, value] of headers) res.writeHeader(key, value);
+  };
+  try {
+    for (; ; ) {
+      const step = await reader.read();
+      if (abort.aborted) {
+        await reader.cancel().catch(() => void 0);
+        return;
+      }
+      if (step.done) {
+        res.cork(() => {
+          begin();
+          res.end();
+        });
+        return;
+      }
+      let writable = true;
+      res.cork(() => {
+        begin();
+        writable = res.write(step.value);
+      });
+      if (!writable && !abort.aborted) await waitUntilWritable();
+    }
+  } catch (err) {
+    await reader.cancel().catch(() => void 0);
+    if (abort.aborted) return;
+    if (!started) throw err;
+    res.cork(() => {
+      res.close();
+    });
+  }
 }
 function sendRelayed(res, abort, status, headers, body) {
   if (abort.aborted) return;
@@ -496,7 +553,14 @@ function validateRebalance(body) {
   }
   return null;
 }
-var TASK_TYPES = ["optimizeVectors", "rebalance", "restore", "import", "rebuildAnalysis"];
+var TASK_TYPES = [
+  "optimizeVectors",
+  "rebalance",
+  "restore",
+  "import",
+  "importExport",
+  "rebuildAnalysis"
+];
 var TASK_STATUSES = ["running", "succeeded", "failed", "cancelled"];
 function parseCountParam(raw, field, minimum) {
   if (raw === null) return minimum;
@@ -631,6 +695,87 @@ function createVersionHandler(deps) {
   }
   return { report };
 }
+var NotAdmittedError = class extends Error {
+};
+async function removeDirectory(directory) {
+  if (directory !== null) await rm(directory, { recursive: true, force: true });
+}
+function spoolBody(res, maxBytes, abort, admitted) {
+  return new Promise((resolve, reject) => {
+    const state = {
+      directory: null,
+      handle: null,
+      chain: Promise.resolve(),
+      queued: 0,
+      total: 0,
+      paused: true,
+      done: false
+    };
+    const fail = (error, answer) => {
+      if (state.done) return;
+      state.done = true;
+      if (answer && !abort.aborted) {
+        sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "The server could not store the request body");
+      }
+      void state.chain.catch(() => void 0).then(async () => {
+        await state.handle?.close().catch(() => void 0);
+        await removeDirectory(state.directory).catch(() => void 0);
+      }).finally(() => reject(error));
+    };
+    const failFromChain = (error) => {
+      if (error instanceof NotAdmittedError) fail(error, false);
+      else fail(error instanceof Error ? error : new Error(String(error)), true);
+    };
+    res.pause();
+    state.chain = (async () => {
+      if (!await admitted) throw new NotAdmittedError("the request was not admitted");
+      state.directory = await mkdtemp(path.join(tmpdir(), "narsil-spool-"));
+      state.handle = await open(path.join(state.directory, "body"), "w");
+      if (!abort.aborted && !state.done && state.queued <= SPOOL_PAUSE_ABOVE_BYTES) {
+        state.paused = false;
+        res.resume();
+      }
+    })();
+    state.chain.catch(failFromChain);
+    abort.onAbort(() => fail(new Error("aborted"), false));
+    res.onData((chunk, isLast) => {
+      if (state.done || abort.aborted) return;
+      state.total += chunk.byteLength;
+      if (maxBytes > 0 && state.total > maxBytes) {
+        sendError(res, 413, ServerErrorCodes.PAYLOAD_TOO_LARGE, "Request body exceeds the configured size limit");
+        fail(new Error("payload too large"), false);
+        return;
+      }
+      const copy = new Uint8Array(chunk.slice(0));
+      state.queued += copy.byteLength;
+      if (!state.paused && state.queued > SPOOL_PAUSE_ABOVE_BYTES) {
+        state.paused = true;
+        res.pause();
+      }
+      state.chain = state.chain.then(async () => {
+        if (state.handle === null) throw new Error("the spool file is not open");
+        await state.handle.write(copy);
+        state.queued -= copy.byteLength;
+        if (state.paused && state.queued <= SPOOL_PAUSE_ABOVE_BYTES && !abort.aborted && !state.done) {
+          state.paused = false;
+          res.resume();
+        }
+      });
+      state.chain.catch(failFromChain);
+      if (!isLast) return;
+      state.chain.then(async () => {
+        if (state.done) return;
+        state.done = true;
+        await state.handle?.close();
+        resolve({
+          path: path.join(state.directory ?? "", "body"),
+          bytes: state.total,
+          remove: () => removeDirectory(state.directory)
+        });
+      }, failFromChain);
+    });
+  });
+}
 
 // src/server/request.ts
 function initAbortHandler(res) {
@@ -732,6 +877,19 @@ function sinkWritingAfterStatus(res, headers) {
       res.end(body);
       return sink;
     },
+    write(chunk) {
+      return res.write(chunk);
+    },
+    pause() {
+      res.pause();
+    },
+    resume() {
+      res.resume();
+    },
+    close() {
+      res.close();
+      return sink;
+    },
     endWithoutBody() {
       res.endWithoutBody();
       return sink;
@@ -830,18 +988,33 @@ function createRouteRunner(deps) {
       }
       const bodyPromise = needsBody ? readBody(res, opts.maxBytes, abort, declaredBytes) : Promise.resolve(null);
       const hookPromise = hookContext !== null && authorize !== void 0 ? authorize(hookContext) : Promise.resolve({ allowed: true });
-      Promise.all([bodyPromise, hookPromise]).then(async ([rawBody, authorization]) => {
-        if (abort.aborted) return;
+      const spoolPromise = opts.spoolsBody === true ? spoolBody(
+        res,
+        opts.maxBytes,
+        abort,
+        hookPromise.then((authorization) => authorization.allowed)
+      ).catch(() => null) : Promise.resolve(null);
+      Promise.all([bodyPromise, spoolPromise, hookPromise]).then(async ([rawBody, spooledBody, authorization]) => {
+        const discardSpool = () => {
+          void spooledBody?.remove().catch(() => void 0);
+        };
+        if (abort.aborted) {
+          discardSpool();
+          return;
+        }
         if (!authorization.allowed) {
+          discardSpool();
           sendDenial(res, authorization);
           return;
         }
+        if (opts.spoolsBody === true && spooledBody === null) return;
         if (gated && !gate.tryAcquire()) {
+          discardSpool();
           sendError(res, 429, ServerErrorCodes.TOO_MANY_REQUESTS, "The server is at capacity; retry shortly");
           return;
         }
         try {
-          await handler({ res, params, query, contentType, rawBody, abort, hookContext });
+          await handler({ res, params, query, contentType, rawBody, spooledBody, abort, hookContext });
         } catch (err) {
           if (!abort.aborted) {
             sendError(res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
@@ -916,7 +1089,14 @@ function serverRoutes(limits) {
       method: "post",
       path: "/indexes/:name/restore",
       handler: "restore",
-      options: { paramCount: 1, needsBody: true, maxBytes: maxImportBytes }
+      options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
+    },
+    { method: "get", path: "/indexes/:name/export", handler: "exportDocuments", options: named },
+    {
+      method: "post",
+      path: "/indexes/:name/import",
+      handler: "importExport",
+      options: { paramCount: 1, spoolsBody: true, maxBytes: limits.maxRestoreBytes }
     }
   ];
 }
@@ -975,4 +1155,4 @@ function childAppOf(app) {
   return candidate;
 }
 
-export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendBinary, sendError, sendRelayed, serializeBatchResult, serializeNarsilError, serverRoutes, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };
+export { ASYNC_IMPORT_CAPABILITY, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_BODY_BYTES, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_RETAINED_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_TASK_PAGE_SIZE, DOCUMENT_EXPORT_CAPABILITY, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, INDEX_LIFECYCLE_CAPABILITY, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, MAIN_THREAD_GATE_SLOT, MAX_INDEX_TOUCH_INTERVAL_MS, MIN_COUNTED_SERVER_LIMIT, MIN_UNBOUNDED_SERVER_LIMIT, REBUILD_ANALYSIS_CAPABILITY, RELAY_STREAM_CHUNKS_IN_FLIGHT, RUNNING_TASK_TTL_MS, SERVER_CAPABILITIES, SNAPSHOT_ARCHIVE_CONTENT_TYPE, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, TASK_PROGRESS_WRITE_INTERVAL_MS, TERMINAL_TASK_TTL_MS, acceptorOf, authorizerFor, badRequest, childAppOf, clearGateSlot, corsHeaderReader, createDocumentReadHandlers, createRouteRunner, createSharedGateBuffer, createSharedRequestGate, createVersionHandler, gateSlotOfWorker, httpStatusForNarsilError, loadUWebSockets, parseJson, parseJsonOptional, parseTaskListQuery, registerServerRoutes, rejectInvalid, resolveCors, respondError, respondJson, sendError, sendRelayed, sendStream, serializeBatchResult, serializeNarsilError, serverRoutes, statusLine, toHttpError, validateBatch, validateCreateIndex, validateDocumentBody, validateQuery, validateRebalance, validateSuggest, versionReport };

package/dist/{chunk-P2ZZN32C.mjs → chunk-PU43WDY4.mjs}

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
+import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 import { existsSync } from 'fs';
 import { createRequire } from 'module';
 import { join, dirname } from 'path';
@@ -6,6 +6,8 @@ import { join, dirname } from 'path';
 // src/vector/hnsw/constants.ts
 var MAX_LAYER_CAP = 32;
 var MAX_M = 512;
+var MIN_M = 2;
+var EARLIER_RELEASE_MIN_M = 1;
 var INITIAL_UPPER_SLOTS = 64;
 var INITIAL_LIST_CAPACITY = 64;
 var PLACEMENT_ROOM_REQUESTS = 8;
@@ -136,4 +138,4 @@ function loadNativeCore() {
   return loaded;
 }
 
-export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, PLACEMENT_ROOM_REQUESTS, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback, requireSearchBackendSettings, stopUsingTheNativeSearchCore };
+export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, DEFAULT_EF_SEARCH, EARLIER_RELEASE_MIN_M, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, MIN_M, PLACEMENT_ROOM_REQUESTS, loadNativeCore, nativeSearchCoreIsEnabled, noteFallback, requireSearchBackendSettings, stopUsingTheNativeSearchCore };