@delali/narsil 0.3.1-next.71 → 0.3.1-next.73

package/dist/server.mjs

@@ -1,21 +1,22 @@
-import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, parseTaskListQuery, parseJsonOptional, validateRebalance, sendBinary, IMPORT_FLUSHES_IN_FLIGHT, badRequest, SERVER_CAPABILITIES, toHttpError, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-2STVUTNA.mjs';
-export { ASYNC_IMPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-2STVUTNA.mjs';
-import { validateIndexName } from './chunk-3JGWULDF.mjs';
-import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-FVLQ63B7.mjs';
-import { detectRuntime } from './chunk-HWQLTLNG.mjs';
-import './chunk-GIVAGD3Q.mjs';
-import './chunk-653BNMAF.mjs';
+import { DEFAULT_MAX_RETAINED_TASKS, loadUWebSockets, resolveCors, authorizerFor, createSharedGateBuffer, gateSlotOfWorker, createRouteRunner, corsHeaderReader, createSharedRequestGate, MAIN_THREAD_GATE_SLOT, registerServerRoutes, DEFAULT_MAX_CONCURRENT_TASKS, DEFAULT_MAX_TASK_PAGE_SIZE, DEFAULT_MAX_IMPORT_ERRORS, DEFAULT_MAX_FETCH_DOCUMENTS, DEFAULT_MAX_RESULT_WINDOW, DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_IMPORT_BATCH_SIZE, DEFAULT_MAX_LINE_BYTES, DEFAULT_MAX_RESTORE_BYTES, DEFAULT_MAX_IMPORT_BYTES, DEFAULT_MAX_BODY_BYTES, RUNNING_TASK_TTL_MS, DEFAULT_TASK_PAGE_SIZE, TERMINAL_TASK_TTL_MS, TASK_PROGRESS_WRITE_INTERVAL_MS, serializeNarsilError, createVersionHandler, acceptorOf, createDocumentReadHandlers, sendError, respondError, respondJson, MAX_INDEX_TOUCH_INTERVAL_MS, INDEX_TOUCH_INTERVALS_PER_IDLE_TIMEOUT, clearGateSlot, parseJson, validateCreateIndex, rejectInvalid, validateBatch, serializeBatchResult, validateDocumentBody, validateSuggest, validateQuery, parseTaskListQuery, parseJsonOptional, validateRebalance, sendStream, SNAPSHOT_ARCHIVE_CONTENT_TYPE, DOCUMENT_EXPORT_CONTENT_TYPE, IMPORT_FLUSHES_IN_FLIGHT, badRequest, SERVER_CAPABILITIES, toHttpError, RELAY_STREAM_CHUNKS_IN_FLIGHT, MIN_UNBOUNDED_SERVER_LIMIT, MIN_COUNTED_SERVER_LIMIT } from './chunk-LPBWCHBE.mjs';
+export { ASYNC_IMPORT_CAPABILITY, DOCUMENT_EXPORT_CAPABILITY, INDEX_LIFECYCLE_CAPABILITY, REBUILD_ANALYSIS_CAPABILITY, SERVER_CAPABILITIES, TASK_CANCEL_CAPABILITY, TASK_FILTER_CAPABILITY, httpStatusForNarsilError } from './chunk-LPBWCHBE.mjs';
+import { validateIndexName } from './chunk-2WLD2NFW.mjs';
+import { engineCoreOf, providedDocId, resolveVectorText } from './chunk-AFKH4ZK3.mjs';
+import { detectRuntime } from './chunk-4MQ5F4IL.mjs';
+import './chunk-AK3BLGCI.mjs';
 import './chunk-TGP5M5R7.mjs';
-import './chunk-YNVIUGF5.mjs';
+import './chunk-PU43WDY4.mjs';
 import './chunk-U2BKYEC7.mjs';
-import './chunk-OI42GWLR.mjs';
+import './chunk-LJYTW7GT.mjs';
 import { startIdleHeapCleanup } from './chunk-Y3L5MDRV.mjs';
-import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-USKFU7J5.mjs';
-export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-USKFU7J5.mjs';
+import { NarsilError, ErrorCodes, ServerErrorCodes, isNarsilError } from './chunk-I64LXKO3.mjs';
+export { ClientErrorCodes, ErrorCodes, NarsilError, ServerErrorCodes, isNarsilError } from './chunk-I64LXKO3.mjs';
 import { randomUUID } from 'crypto';
+import { createReadStream } from 'fs';
+import { rm } from 'fs/promises';
+import path from 'path';
 import { MessageChannel } from 'worker_threads';
 
-// src/server/handlers/admin.ts
 function createAdminHandlers(deps) {
   const { engine, tasks, limits } = deps;
   async function checkpoint(ctx) {
@@ -28,9 +29,12 @@ function createAdminHandlers(deps) {
   }
   async function snapshot(ctx) {
     try {
-      const bytes = await engine.snapshot(ctx.params[0]);
-      if (ctx.abort.aborted) return;
-      sendBinary(ctx.res, bytes, ctx.abort);
+      const archive = await engine.snapshot(ctx.params[0]);
+      if (ctx.abort.aborted) {
+        await archive.cancel();
+        return;
+      }
+      await sendStream(ctx.res, ctx.abort, archive, [["Content-Type", SNAPSHOT_ARCHIVE_CONTENT_TYPE]]);
     } catch (err) {
       respondError(ctx, err);
     }
@@ -118,15 +122,26 @@ function createAdminHandlers(deps) {
     respondJson(ctx, record, 202);
   }
   async function restore(ctx) {
-    const raw = ctx.rawBody;
-    if (!raw || raw.length === 0) {
+    const spooled = ctx.spooledBody;
+    if (spooled === null || spooled.bytes === 0) {
+      await spooled?.remove();
       sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
       return;
     }
     const name = ctx.params[0];
-    const bytes = new Uint8Array(raw);
-    const record = await tasks.start("restore", name, () => engine.restore(name, bytes));
-    respondJson(ctx, record, 202);
+    try {
+      const record = await tasks.start("restore", name, async () => {
+        try {
+          await engine.restore(name, createReadStream(spooled.path));
+        } finally {
+          await spooled.remove();
+        }
+      });
+      respondJson(ctx, record, 202);
+    } catch (err) {
+      await spooled.remove();
+      respondError(ctx, err);
+    }
   }
   async function getTask(ctx) {
     const record = await tasks.get(ctx.params[0]);
@@ -270,6 +285,54 @@ function createClusterHandlers(deps) {
   }
   return { topology, allocation };
 }
+function importResultOf(result, maxErrors) {
+  const errors = result.refused.slice(0, maxErrors);
+  return {
+    indexed: result.imported,
+    failed: result.refusedCount,
+    errors,
+    errorsTruncated: result.refusedCount > errors.length
+  };
+}
+function createDocumentExportHandlers(deps) {
+  const { engine, tasks, limits } = deps;
+  async function exportDocuments(ctx) {
+    try {
+      const lines = await engine.exportDocuments(ctx.params[0]);
+      if (ctx.abort.aborted) {
+        await lines.cancel();
+        return;
+      }
+      await sendStream(ctx.res, ctx.abort, lines, [["Content-Type", DOCUMENT_EXPORT_CONTENT_TYPE]]);
+    } catch (err) {
+      respondError(ctx, err);
+    }
+  }
+  async function importExport(ctx) {
+    const spooled = ctx.spooledBody;
+    if (spooled === null || spooled.bytes === 0) {
+      await spooled?.remove();
+      sendError(ctx.res, 400, ServerErrorCodes.EMPTY_BODY, "Request body is empty");
+      return;
+    }
+    const name = ctx.params[0];
+    try {
+      const record = await tasks.start("importExport", name, async (context) => {
+        try {
+          const result = await engine.importDocumentExport(name, createReadStream(spooled.path));
+          context.reportResult(importResultOf(result, limits.maxImportErrors));
+        } finally {
+          await spooled.remove();
+        }
+      });
+      respondJson(ctx, record, 202);
+    } catch (err) {
+      await spooled.remove();
+      respondError(ctx, err);
+    }
+  }
+  return { exportDocuments, importExport };
+}
 
 // src/server/handlers/documents.ts
 function createDocumentHandlers(deps) {
@@ -757,6 +820,7 @@ var ENGINE_METHOD_SAMPLE = ["createIndex", "listIndexes", "insert", "query", "ge
 var LIMITS_UNBOUNDED_AT_ZERO = [
   "maxBodyBytes",
   "maxImportBytes",
+  "maxRestoreBytes",
   "maxLineBytes",
   "maxImportErrors",
   "maxConcurrentRequests",
@@ -808,10 +872,32 @@ function bodyBytes(body) {
   if (typeof body === "string") return Buffer.from(body, "utf8");
   return body;
 }
-function createCapturingResponse(remoteAddress) {
+function createRelayAbort() {
+  const listeners = [];
+  let aborted = false;
+  return {
+    get aborted() {
+      return aborted;
+    },
+    onAbort(fn) {
+      if (aborted) fn();
+      else listeners.push(fn);
+    },
+    trigger() {
+      if (aborted) return;
+      aborted = true;
+      for (const fn of listeners) fn();
+    }
+  };
+}
+function createCapturingResponse(remoteAddress, requestId, sendPart) {
   let status = DEFAULT_STATUS;
   const headers = [];
   let finished = false;
+  let streaming = false;
+  let inFlight = 0;
+  let writable = null;
+  const abort = createRelayAbort();
   let resolveDone = () => void 0;
   const done = new Promise((resolve) => {
     resolveDone = resolve;
@@ -819,25 +905,57 @@ function createCapturingResponse(remoteAddress) {
   function finish(body) {
     if (finished) return;
     finished = true;
+    if (streaming) {
+      sendPart({ type: "stream-end", requestId, failed: false }, []);
+      resolveDone(null);
+      return;
+    }
     resolveDone({ status, headers, body });
   }
+  function startStreaming() {
+    if (streaming) return;
+    streaming = true;
+    sendPart({ type: "stream-start", requestId, status, headers }, []);
+  }
   const sink = {
     cork(callback) {
       callback();
       return sink;
     },
     writeStatus(line) {
-      if (!finished) status = statusOf(line);
+      if (!finished && !streaming) status = statusOf(line);
       return sink;
     },
     writeHeader(key, value) {
-      if (!finished) headers.push([key, value]);
+      if (!finished && !streaming) headers.push([key, value]);
       return sink;
     },
     end(body) {
       finish(bodyBytes(body));
       return sink;
     },
+    write(chunk) {
+      if (finished || abort.aborted) return true;
+      startStreaming();
+      const copy = chunk.slice();
+      inFlight += 1;
+      sendPart({ type: "stream-chunk", requestId, chunk: copy }, [copy.buffer]);
+      return inFlight < RELAY_STREAM_CHUNKS_IN_FLIGHT;
+    },
+    pause: () => void 0,
+    resume: () => void 0,
+    close() {
+      if (finished) return sink;
+      finished = true;
+      if (streaming) {
+        sendPart({ type: "stream-end", requestId, failed: true }, []);
+        resolveDone(null);
+      } else {
+        status = NO_RESPONSE_STATUS;
+        resolveDone({ status, headers, body: new Uint8Array(0) });
+      }
+      return sink;
+    },
     endWithoutBody() {
       finish(new Uint8Array(0));
       return sink;
@@ -847,16 +965,45 @@ function createCapturingResponse(remoteAddress) {
       return [true, true];
     },
     getWriteOffset: () => 0,
-    onWritable: () => sink,
+    onWritable(handler) {
+      writable = handler;
+      return sink;
+    },
     onAborted: () => sink,
     onData: () => sink,
     getRemoteAddressAsText: () => new TextEncoder().encode(remoteAddress).buffer
   };
   return {
     sink,
+    abort,
     done,
+    credit() {
+      inFlight = Math.max(0, inFlight - 1);
+      if (inFlight >= RELAY_STREAM_CHUNKS_IN_FLIGHT || writable === null) return;
+      const handler = writable;
+      writable = null;
+      handler(0);
+    },
+    cancel() {
+      abort.trigger();
+      if (writable !== null) {
+        const handler = writable;
+        writable = null;
+        handler(0);
+      }
+      if (!finished) {
+        finished = true;
+        resolveDone(null);
+      }
+    },
     finishUnanswered() {
       if (finished) return;
+      if (streaming) {
+        finished = true;
+        sendPart({ type: "stream-end", requestId, failed: true }, []);
+        resolveDone(null);
+        return;
+      }
       status = NO_RESPONSE_STATUS;
       finish(new Uint8Array(0));
     }
@@ -867,22 +1014,36 @@ function toRelayedResponse(requestId, captured) {
 }
 
 // src/server/request-threads/host.ts
-var NO_ABORT = { aborted: false, onAbort: () => void 0 };
+var RELAYED_TYPES = /* @__PURE__ */ new Set(["request", "authorize", "embed", "stream-credit", "stream-cancel"]);
 function isRelayRequest(message) {
   if (typeof message !== "object" || message === null) return false;
   const { type, requestId } = message;
   if (type === "touch") return true;
-  return typeof requestId === "number" && (type === "request" || type === "authorize" || type === "embed");
+  return typeof requestId === "number" && typeof type === "string" && RELAYED_TYPES.has(type);
 }
-async function answerRequest(handlers, request) {
-  const captured = createCapturingResponse(request.hookContext?.remoteAddress ?? "");
+function spooledBodyOf(relayed) {
+  if (relayed === null) return null;
+  return {
+    path: relayed.path,
+    bytes: relayed.bytes,
+    remove: () => rm(path.dirname(relayed.path), { recursive: true, force: true })
+  };
+}
+async function answerRequest(handlers, request, port, streams) {
+  const captured = createCapturingResponse(
+    request.hookContext?.remoteAddress ?? "",
+    request.requestId,
+    (part, moved) => port.postMessage(part, moved)
+  );
+  streams.set(request.requestId, captured);
   const ctx = {
     res: captured.sink,
     params: request.params,
     query: new URLSearchParams(request.query),
     contentType: request.contentType,
     rawBody: request.rawBody === null ? null : Buffer.from(request.rawBody.buffer, request.rawBody.byteOffset, request.rawBody.byteLength),
-    abort: NO_ABORT,
+    spooledBody: spooledBodyOf(request.spooledBody),
+    abort: captured.abort,
     hookContext: request.hookContext
   };
   try {
@@ -891,7 +1052,9 @@ async function answerRequest(handlers, request) {
     sendError(ctx.res, 500, ServerErrorCodes.INTERNAL_ERROR, "An unexpected error occurred");
   }
   captured.finishUnanswered();
-  return toRelayedResponse(request.requestId, await captured.done);
+  const answered = await captured.done;
+  streams.delete(request.requestId);
+  return answered === null ? null : toRelayedResponse(request.requestId, answered);
 }
 async function answerAuthorize(authorize, request) {
   if (authorize === void 0) {
@@ -942,14 +1105,25 @@ async function startRequestThreads(options) {
   const threads = /* @__PURE__ */ new Map();
   let stopped = false;
   function relayFrom(port) {
+    const streams = /* @__PURE__ */ new Map();
     port.on("message", (message) => {
       if (!isRelayRequest(message)) return;
       if (message.type === "touch") {
         core.orchestrator.noteAccess(message.indexName);
         return;
       }
-      const reply = message.type === "request" ? answerRequest(handlers, message) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
-      void reply.then((answer) => port.postMessage(answer));
+      if (message.type === "stream-credit") {
+        streams.get(message.requestId)?.credit();
+        return;
+      }
+      if (message.type === "stream-cancel") {
+        streams.get(message.requestId)?.cancel();
+        return;
+      }
+      const reply = message.type === "request" ? answerRequest(handlers, message, port, streams) : message.type === "authorize" ? answerAuthorize(authorize, message) : answerEmbed(core, message);
+      void reply.then((answer) => {
+        if (answer !== null) port.postMessage(answer);
+      });
     });
   }
   async function stopThread(workerId, thread) {
@@ -982,7 +1156,8 @@ async function startRequestThreads(options) {
               limits: options.limits,
               build: options.build,
               authorizes: authorize !== void 0,
-              searchHooks
+              searchHooks,
+              patternWorkCap: options.core.patternWorkCap
             }
           },
           [channel.port1]
@@ -1253,6 +1428,7 @@ function resolveLimits(limits) {
   return {
     maxBodyBytes: limits?.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES,
     maxImportBytes: limits?.maxImportBytes ?? DEFAULT_MAX_IMPORT_BYTES,
+    maxRestoreBytes: limits?.maxRestoreBytes ?? DEFAULT_MAX_RESTORE_BYTES,
     maxLineBytes: limits?.maxLineBytes ?? DEFAULT_MAX_LINE_BYTES,
     importBatchSize: limits?.importBatchSize ?? DEFAULT_IMPORT_BATCH_SIZE,
     maxConcurrentRequests: limits?.maxConcurrentRequests ?? DEFAULT_MAX_CONCURRENT_REQUESTS,
@@ -1286,6 +1462,7 @@ function mainThreadHandlers(deps) {
   const doc = createDocumentHandlers(deps);
   const search = createSearchHandlers(deps);
   const admin = createAdminHandlers(deps);
+  const documentExport = createDocumentExportHandlers(deps);
   const health = createHealthHandlers(deps);
   const cluster = createClusterHandlers(deps);
   const version = createVersionHandler(deps);
@@ -1332,7 +1509,9 @@ function mainThreadHandlers(deps) {
     optimize: admin.optimize,
     rebalance: admin.rebalance,
     partitionConfig: admin.partitionConfig,
-    restore: admin.restore
+    restore: admin.restore,
+    exportDocuments: documentExport.exportDocuments,
+    importExport: documentExport.importExport
   };
 }
 var NarsilHttpServer = class {

package/dist/{tasks-BZt_zes6.d.ts → tasks-DpgCpW1x.d.ts}

@@ -1,4 +1,4 @@
-import { M as MemoryStats } from './memory-C7UMCMH3.js';
+import { M as MemoryStats } from './memory-B4jMmygw.js';
 
 /** Declarative index configuration accepted over HTTP. Function-valued engine
  * options (custom tokenizer, stopWords-as-function, group reducer, embedding
@@ -142,7 +142,7 @@ interface MemoryStatsResponse extends MemoryStats {
  *
  * @public
  */
-type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'rebuildAnalysis';
+type TaskType = 'optimizeVectors' | 'rebalance' | 'restore' | 'import' | 'importExport' | 'rebuildAnalysis';
 /**
  * Where a long-running operation stands.
  *
@@ -207,7 +207,7 @@ interface TaskRecord {
     cancelRequestedAt?: number;
     /** How far the work has gone, which an import reports and other task types omit. */
     progress?: TaskProgress;
-    /** What the work produced, which an import reports once it finishes. */
+    /** What the work produced, which an `import` or `importExport` task reports once it finishes. */
     result?: ImportResult;
     /** This says why the task failed, and a failed task alone carries it. */
     error?: {

package/dist/{thread-JEVAYETR.mjs → thread-3LNVA43V.mjs}

@@ -1,12 +1,12 @@
-import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery } from './chunk-2STVUTNA.mjs';
-import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-P4DSQ6M4.mjs';
-import './chunk-GIVAGD3Q.mjs';
-import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-LMO5TJUS.mjs';
-import { extractVectorFieldsFromSchema } from './chunk-653BNMAF.mjs';
+import { loadUWebSockets, childAppOf, createRouteRunner, corsHeaderReader, createSharedRequestGate, registerServerRoutes, sendRelayed, createDocumentReadHandlers, respondJson, SERVER_CAPABILITIES, versionReport, statusLine, sendError, serverRoutes, parseJson, validateSuggest, rejectInvalid, respondError, validateQuery } from './chunk-LPBWCHBE.mjs';
+import { executeListDocuments, executeSuggest, executePreflight, queryBindingOf, executeQuery } from './chunk-ZULSEZCE.mjs';
+import { resolveProjection, projectionKeepsField, setNestedValue } from './chunk-3YKWSO54.mjs';
+import { extractVectorFieldsFromSchema } from './chunk-AK3BLGCI.mjs';
 import './chunk-ZJ4CSAB2.mjs';
-import './chunk-YNVIUGF5.mjs';
-import './chunk-OI42GWLR.mjs';
-import { ServerErrorCodes, NarsilError, ErrorCodes } from './chunk-USKFU7J5.mjs';
+import './chunk-PU43WDY4.mjs';
+import './chunk-ZKRV5M2P.mjs';
+import './chunk-LJYTW7GT.mjs';
+import { ServerErrorCodes, NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
 
 // src/server/request-threads/held-vectors.ts
 function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
@@ -28,7 +28,7 @@ function managerWithHeldVectors(manager, fieldPaths, vectorOf) {
 
 // src/server/request-threads/read-engine.ts
 function createThreadReadEngine(options) {
-  const { executor, relay, searchHooks } = options;
+  const { executor, relay, searchHooks, patternWorkCap } = options;
   function vectorFieldsOf(context) {
     return [...extractVectorFieldsFromSchema(context.config.schema).keys()];
   }
@@ -110,7 +110,8 @@ function createThreadReadEngine(options) {
         config: context.config,
         indexName,
         cursorBinding: queryBindingOf(params),
-        vectorSearchers: context.vectorSearchers
+        vectorSearchers: context.vectorSearchers,
+        patternWorkCap
       });
       if (context.analysisStale) result.analysisStale = true;
       return result;
@@ -123,7 +124,8 @@ function createThreadReadEngine(options) {
         config: context.config,
         indexName,
         cursorBinding: queryBindingOf(params),
-        vectorSearchers: context.vectorSearchers
+        vectorSearchers: context.vectorSearchers,
+        patternWorkCap
       });
       if (context.analysisStale) result.analysisStale = true;
       return result;
@@ -157,7 +159,8 @@ function createThreadReadEngine(options) {
       return executeListDocuments(params ?? {}, {
         manager,
         schema: context.config.schema,
-        strict: context.config.strict
+        strict: context.config.strict,
+        patternWorkCap
       });
     }
   };
@@ -169,6 +172,46 @@ function isRelayReply(message) {
   const { type, requestId } = message;
   return typeof requestId === "number" && (type === "response" || type === "authorized" || type === "embedded");
 }
+function isStreamPart(message) {
+  if (typeof message !== "object" || message === null) return false;
+  const { type, requestId } = message;
+  return typeof requestId === "number" && (type === "stream-start" || type === "stream-chunk" || type === "stream-end");
+}
+function writeStreamPart(port, writer, part) {
+  const { ctx } = writer;
+  if (part.type === "stream-end") {
+    if (!ctx.abort.aborted) {
+      ctx.res.cork(() => {
+        if (part.failed) ctx.res.close();
+        else ctx.res.end();
+      });
+    }
+    writer.finish();
+    return;
+  }
+  if (ctx.abort.aborted) return;
+  if (part.type === "stream-start") {
+    writer.started = true;
+    ctx.res.cork(() => {
+      ctx.res.writeStatus(statusLine(part.status));
+      for (const [key, value] of part.headers) ctx.res.writeHeader(key, value);
+    });
+    return;
+  }
+  let writable = true;
+  ctx.res.cork(() => {
+    writable = ctx.res.write(part.chunk);
+  });
+  const credit = () => port.postMessage({ type: "stream-credit", requestId: part.requestId });
+  if (writable) {
+    credit();
+    return;
+  }
+  ctx.res.onWritable(() => {
+    credit();
+    return true;
+  });
+}
 function errorOf(reply) {
   const error = reply.error ?? { code: ErrorCodes.EMBEDDING_FAILED, message: "The main thread returned no vector" };
   return new NarsilError(error.code, error.message, error.details);
@@ -201,11 +244,18 @@ function createRelayClient(port, touchIntervalMs) {
     pending.clear();
     for (const entry of waiting) entry.reject(portClosedError());
   }
+  const writers = /* @__PURE__ */ new Map();
   port.on("message", (message) => {
+    if (isStreamPart(message)) {
+      const writer = writers.get(message.requestId);
+      if (writer !== void 0) writeStreamPart(port, writer, message);
+      return;
+    }
     if (!isRelayReply(message)) return;
     const waiting = pending.get(message.requestId);
     if (waiting === void 0) return;
     pending.delete(message.requestId);
+    writers.delete(message.requestId);
     waiting.resolve(message);
   });
   port.on("close", failAll);
@@ -240,24 +290,36 @@ function createRelayClient(port, touchIntervalMs) {
       let reply;
       const body = relayedBody(ctx.rawBody);
       try {
-        reply = await send(
-          (requestId) => ({
-            type: "request",
-            requestId,
-            route,
-            params: ctx.params,
-            query: ctx.query.toString(),
-            contentType: ctx.contentType,
-            rawBody: body.bytes,
-            hookContext: ctx.hookContext
-          }),
-          body.moved
-        );
+        reply = await new Promise((resolve, reject) => {
+          void send((requestId) => {
+            writers.set(requestId, {
+              ctx,
+              started: false,
+              finish: () => {
+                writers.delete(requestId);
+                pending.delete(requestId);
+                resolve(null);
+              }
+            });
+            ctx.abort.onAbort(() => port.postMessage({ type: "stream-cancel", requestId }));
+            return {
+              type: "request",
+              requestId,
+              route,
+              params: ctx.params,
+              query: ctx.query.toString(),
+              contentType: ctx.contentType,
+              rawBody: body.bytes,
+              spooledBody: ctx.spooledBody === null ? null : { path: ctx.spooledBody.path, bytes: ctx.spooledBody.bytes },
+              hookContext: ctx.hookContext
+            };
+          }, body.moved).then(resolve, reject);
+        });
       } catch {
         if (!ctx.abort.aborted) sendUnavailable(ctx.res);
         return;
       }
-      if (reply.type !== "response" || ctx.abort.aborted) return;
+      if (reply === null || reply.type !== "response" || ctx.abort.aborted) return;
       sendRelayed(ctx.res, ctx.abort, reply.status, reply.headers, reply.body);
     },
     async embed(indexName, vector) {
@@ -381,7 +443,12 @@ async function serveRequests(executor, settings) {
   const child = childAppOf(app);
   if (child === null) throw new Error("This uWebSockets.js build offers no child apps");
   const relay = createRelayClient(settings.port, settings.touchIntervalMs);
-  const local = createThreadReadEngine({ executor, relay, searchHooks: settings.searchHooks });
+  const local = createThreadReadEngine({
+    executor,
+    relay,
+    searchHooks: settings.searchHooks,
+    patternWorkCap: settings.patternWorkCap
+  });
   const run = createRouteRunner({
     authorize: settings.authorizes ? (context) => relay.authorize(context) : void 0,
     gate: createSharedRequestGate(settings.gate, settings.maxConcurrentRequests, settings.gateSlot),

package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts}

@@ -1,4 +1,4 @@
-import { T as TransportConfig } from './types-D5OwhtsQ.js';
+import { T as TransportConfig } from './types-Cb9kiY5P.js';
 
 /**
  * The certificate material one node presents to its peers.

package/dist/{types-gOUqhs2F.d.ts → types-BiUmaiWT.d.ts}

@@ -1,4 +1,4 @@
-import { S as SchemaDefinition } from './schema-Bgzr4g7b.js';
+import { S as SchemaDefinition } from './schema-BBOCdhND.js';
 
 /**
  * What a node does in a cluster. A node may hold several roles at once.

package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts}

@@ -31,7 +31,7 @@ interface TransportConfig {
     requestTimeout: number;
     /** A replication entry may take this long to reach a replica and come back acknowledged. */
     replicationTimeout: number;
-    /** A whole snapshot transfer, which moves a partition between nodes, may take this long. */
+    /** A node waits this long for each next chunk of a snapshot transfer, which moves a partition or an index between nodes, so that the node can receive a transfer of any size while the sender keeps sending. */
     snapshotTimeout: number;
 }
 /**

package/dist/vector/search-worker.d.ts

@@ -1,4 +1,4 @@
-import { e as VectorQuantizationMode } from '../schema-Bgzr4g7b.js';
+import { e as VectorQuantizationMode } from '../schema-BBOCdhND.js';
 import { A as ArenaSimd } from '../simd-DW5AtJ6B.js';
 import '../adapters-79OIa4xm.js';
 

package/dist/vector/search-worker.mjs

@@ -1,8 +1,8 @@
-import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-YJDQ7NLA.mjs';
+import { openSharedVectorField, MAX_WORKER_COPIES, createVectorStore, osqBitsOf, createHNSWIndex, createOsqQuantizer, osqRecordBytes } from '../chunk-WVDFG45M.mjs';
 import '../chunk-ZJ4CSAB2.mjs';
-import '../chunk-YNVIUGF5.mjs';
-import '../chunk-OI42GWLR.mjs';
-import '../chunk-USKFU7J5.mjs';
+import '../chunk-PU43WDY4.mjs';
+import '../chunk-LJYTW7GT.mjs';
+import '../chunk-I64LXKO3.mjs';
 
 // src/vector/worker-copy.ts
 function restoreQuantizer(snapshot, store) {