@delali/narsil 0.1.14 → 0.1.15

package/README.md

@@ -560,13 +560,13 @@ await narsil.optimizeVectors('docs', 'embedding')
 
 ## Hybrid search
 
-Hybrid mode runs the full-text and vector searches in one query and fuses the two rankings.
+Hybrid mode runs the full-text and vector searches in one query and fuses the two rankings. The vector side needs a query vector: pass a precomputed `value` array, or a `text` string that the index or instance embedding adapter turns into a vector. The `text` form needs an embedding adapter configured first, so passing `text` without one fails with `EMBEDDING_CONFIG_INVALID`; see [Embedding adapters](#embedding-adapters).
 
 ```ts
 const results = await narsil.query('docs', {
   mode: 'hybrid',
   term: 'how do search engines scale',
-  vector: { field: 'embedding', text: 'how do search engines scale' },
+  vector: { field: 'embedding', value: myQueryVector },
   hybrid: { strategy: 'rrf', k: 60 },
   limit: 10,
 })
@@ -581,14 +581,14 @@ Two fusion strategies are available:
 const weighted = await narsil.query('docs', {
   mode: 'hybrid',
   term: 'partition rebalancing',
-  vector: { field: 'embedding', text: 'partition rebalancing' },
+  vector: { field: 'embedding', value: myQueryVector },
   hybrid: { strategy: 'linear', alpha: 0.7 },
 })
 ```
 
 ## Geosearch
 
-Declare a `geopoint` field and insert documents with `{ lat, lon }` values. Geo conditions are filters, so they compose with terms, other filters, facets, and every other query feature.
+Declare a `geopoint` field and insert documents with `{ lat, lon }` values. Geo conditions are filters that refine a search, so a geo query pairs a `term` (or a vector query) with a location filter, and it composes with other filters, facets, and every other query feature. A query with only a location filter and no `term` matches nothing, the same way any other filter refines a term search.
 
 ```ts
 await narsil.createIndex('stores', {
@@ -604,6 +604,7 @@ await narsil.insert('stores', {
 })
 
 const nearby = await narsil.query('stores', {
+  term: 'market',
   filters: {
     fields: {
       location: {
@@ -614,6 +615,7 @@ const nearby = await narsil.query('stores', {
 })
 
 const inArea = await narsil.query('stores', {
+  term: 'market',
   filters: {
     fields: {
       location: {
@@ -707,7 +709,7 @@ engine.registerEmbeddingAdapter('openai-small', myReplacementAdapter)
 
 `registerEmbeddingAdapter` rebinds every index referencing that name, which lets you rotate credentials or swap models without recreating indexes.
 
-### Shipped adapters
+### Bundled adapters
 
 | Adapter | Package | Dependencies |
 | --- | --- | --- |
@@ -733,7 +735,7 @@ When `embedBatch` is missing, Narsil falls back to parallel `embed()` calls with
 
 ## Persistence
 
-Persistence stores serialized partitions through a pluggable adapter. Three adapters ship with the package:
+Persistence stores serialized partitions through a pluggable adapter. The package includes three adapters:
 
 | Adapter | Import | Environment |
 | --- | --- | --- |
@@ -848,7 +850,7 @@ Promotion emits the `workerPromote` event, and a crashed worker emits `workerCra
 
 ## Multi-instance invalidation
 
-When several engine instances share one persistence backend, the invalidation adapter tells the others which partitions changed so they evict stale cache instead of serving old data. Two adapters ship with the package, and `@delali/narsil/invalidation/noop` stubs the interface for single-instance deployments:
+When several engine instances share one persistence backend, the invalidation adapter tells the others which partitions changed so they evict stale cache instead of serving old data. The package includes two adapters, and `@delali/narsil/invalidation/noop` stubs the interface for single-instance deployments:
 
 | Adapter | Import | Use case |
 | --- | --- | --- |
@@ -986,7 +988,7 @@ console.log(memory.workers)
 
 ## Language support
 
-Narsil ships with language modules for tokenization, stemming (Snowball algorithm), and stop word removal.
+Narsil includes language modules for tokenization, stemming (Snowball algorithm), and stop word removal.
 
 ### Full support (tokenizer + stemmer + stop words)
 
@@ -998,17 +1000,25 @@ Chinese (Mandarin), Japanese
 
 ### African language support
 
-Full stemmer support ships for Swahili. Tokenization and stop word support is available for Dagbani, Ewe, Ga, Hausa, Igbo, Twi (Akan), Yoruba, and Zulu.
+Swahili has full stemmer support. Tokenization and stop word support is available for Dagbani, Ewe, Ga, Hausa, Igbo, Twi (Akan), Yoruba, and Zulu.
 
-Each language module is a separate entry point, so you only bundle the languages your application needs:
+Each language module is a separate entry point, so you only bundle the languages your application needs. Import the module and register it before you create an index that names it:
 
 ```ts
-import '@delali/narsil/languages/french'
-import '@delali/narsil/languages/swahili'
-import '@delali/narsil/languages/twi'
+import { createNarsil, registerLanguage } from '@delali/narsil'
+import { french } from '@delali/narsil/languages/french'
+import { swahili } from '@delali/narsil/languages/swahili'
+import { twi } from '@delali/narsil/languages/twi'
+
+registerLanguage(french)
+registerLanguage(swahili)
+registerLanguage(twi)
+
+const narsil = await createNarsil()
+await narsil.createIndex('articles', { schema: { title: 'string' }, language: 'french' })
 ```
 
-Import the module before creating an index that names it; naming a language that was never imported fails with `LANGUAGE_NOT_SUPPORTED`. English is always available. `registerLanguage(module)` adds a language of your own: a module carries a name, a tokenizer, a stemmer, and a stop word set, and any of the shipped modules serves as a reference.
+English is registered by default, so it needs no import. Naming a language you have not registered fails with `LANGUAGE_NOT_SUPPORTED`. `registerLanguage(module)` also adds a language of your own: a module carries a name, a tokenizer, a stemmer, and a stop word set, and any of the built-in language modules serves as a reference.
 
 ## HTTP server
 
@@ -1032,7 +1042,7 @@ const server = createServer(engine, {
 await server.listen()
 ```
 
-`ServerOptions` also accepts `cors`, an `onRequest` hook for authentication, `limits` for body-size and concurrency caps, `embeddingAdapters` that JSON index configs reference by name, a `taskStore` that keeps long-running task status across restarts, an `instanceId` for task recovery, and `allowInsecure` for trusted private networks. The server refuses to bind a non-loopback address without an `onRequest` hook, because the admin endpoints can destroy data.
+`ServerOptions` also accepts `cors`, an `onRequest` hook for authentication, `limits` for body-size, concurrency, result-window, and fetch-count caps, `embeddingAdapters` that JSON index configs reference by name, a `taskStore` that keeps long-running task status across restarts, an `instanceId` for task recovery, and `allowInsecure` for trusted private networks. The server refuses to bind a non-loopback address without an `onRequest` hook, because the admin endpoints can destroy data.
 
 The full surface:
 

package/dist/server.d.ts

@@ -45,6 +45,14 @@ interface ServerLimits {
     importBatchSize?: number;
     /** Maximum requests executing engine work at once; excess is shed with 503. Omit or 0 to disable. */
     maxConcurrentRequests?: number;
+    /** Ceiling for a search's `limit`, `offset`, and `group.maxPerGroup`, so one
+     * request cannot ask for an unbounded result set. Excess → 400. Defaults to
+     * 10000, matching the cluster query result window. */
+    maxResultWindow?: number;
+    /** Ceiling for the number of document ids in one multi-get request, so one
+     * request cannot pull an unbounded number of documents. Excess → 400.
+     * Defaults to 10000. */
+    maxFetchDocuments?: number;
 }
 interface ServerOptions {
     host?: string;

package/dist/server.mjs

@@ -118,11 +118,31 @@ var STATUS_TEXT = {
   500: "500 Internal Server Error",
   503: "503 Service Unavailable"
 };
+var STREAM_THRESHOLD = 64 * 1024;
 function statusLine(status) {
   return STATUS_TEXT[status] ?? `${status}`;
 }
-function sendJson(res, data, status = 200) {
+function streamBody(res, abort, body, status, contentType) {
+  if (abort.aborted) return;
+  const totalSize = body.byteLength;
+  res.cork(() => {
+    res.writeStatus(statusLine(status)).writeHeader("Content-Type", contentType);
+    const startOffset = res.getWriteOffset();
+    const [, done] = res.tryEnd(body, totalSize);
+    if (done) return;
+    res.onWritable((offset) => {
+      if (abort.aborted) return true;
+      const [retryOk] = res.tryEnd(body.subarray(offset - startOffset), totalSize);
+      return retryOk;
+    });
+  });
+}
+function sendJson(res, data, status = 200, abort) {
   const payload = JSON.stringify(data);
+  if (abort && payload !== void 0 && Buffer.byteLength(payload) >= STREAM_THRESHOLD) {
+    streamBody(res, abort, Buffer.from(payload, "utf8"), status, "application/json");
+    return;
+  }
   res.cork(() => {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
@@ -134,10 +154,8 @@ function sendError(res, status, code, message, details) {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
 }
-function sendBinary(res, data, status = 200) {
-  res.cork(() => {
-    res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/octet-stream").end(data);
-  });
+function sendBinary(res, data, abort, status = 200) {
+  streamBody(res, abort, data, status, "application/octet-stream");
 }
 
 // src/server/handler-utils.ts
@@ -171,11 +189,15 @@ function respondError(ctx, err) {
 }
 function respondJson(ctx, data, status = 200) {
   if (ctx.abort.aborted) return;
-  sendJson(ctx.res, data, status);
+  sendJson(ctx.res, data, status, ctx.abort);
 }
 function badRequest(res, message, details) {
   sendError(res, 400, ServerErrorCodes.INVALID_REQUEST, message, details);
 }
+function rejectInvalid(ctx, failure) {
+  if (ctx.abort.aborted) return;
+  badRequest(ctx.res, failure.message, failure.details);
+}
 function serializeBatchResult(result) {
   return {
     succeeded: result.succeeded,
@@ -186,6 +208,98 @@ function serializeBatchResult(result) {
   };
 }
 
+// src/server/validation.ts
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function overWindow(field, value, maxWindow) {
+  if (typeof value === "number" && value > maxWindow) {
+    return {
+      message: `Field "${field}" exceeds the maximum result window of ${maxWindow}`,
+      details: { value, limit: maxWindow }
+    };
+  }
+  return null;
+}
+function validateQuery(params, maxWindow) {
+  if (params.group && typeof params.group === "object" && "reduce" in params.group) {
+    return {
+      message: 'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
+    };
+  }
+  const rowBound = overWindow("limit", params.limit, maxWindow) ?? overWindow("offset", params.offset, maxWindow);
+  if (rowBound) return rowBound;
+  if (params.group && typeof params.group === "object") {
+    const grouped = overWindow("group.maxPerGroup", params.group.maxPerGroup, maxWindow);
+    if (grouped) return grouped;
+  }
+  if (params.facets && typeof params.facets === "object") {
+    for (const [field, facet] of Object.entries(params.facets)) {
+      if (facet && typeof facet === "object") {
+        const faceted = overWindow(`facets.${field}.limit`, facet.limit, maxWindow);
+        if (faceted) return faceted;
+      }
+    }
+  }
+  return null;
+}
+function validateSuggest(params) {
+  if (typeof params.prefix !== "string") {
+    return { message: 'Field "prefix" is required and must be a string' };
+  }
+  return null;
+}
+function validateDocumentBody(body) {
+  if (!isPlainObject(body.document)) {
+    return { message: 'Field "document" is required and must be an object' };
+  }
+  return null;
+}
+function validateMultiGet(body, maxFetch) {
+  if (!Array.isArray(body.docIds)) {
+    return { message: 'Field "docIds" is required and must be an array of strings' };
+  }
+  if (body.docIds.length > maxFetch) {
+    return {
+      message: `Field "docIds" exceeds the maximum of ${maxFetch} ids per request`,
+      details: { count: body.docIds.length, limit: maxFetch }
+    };
+  }
+  return null;
+}
+function validateBatch(body) {
+  const action = body.action ?? "insert";
+  if (action === "insert") {
+    if (!Array.isArray(body.documents)) return { message: 'Batch insert requires a "documents" array' };
+    return null;
+  }
+  if (action === "update") {
+    if (!Array.isArray(body.updates)) return { message: 'Batch update requires an "updates" array' };
+    return null;
+  }
+  if (action === "delete") {
+    if (!Array.isArray(body.docIds)) return { message: 'Batch delete requires a "docIds" array' };
+    return null;
+  }
+  return { message: 'Field "action" must be one of "insert", "update", or "delete"' };
+}
+function validateCreateIndex(body) {
+  if (typeof body.name !== "string" || body.name.length === 0) {
+    return { message: 'Field "name" is required and must be a non-empty string' };
+  }
+  if (!isPlainObject(body.config)) {
+    return { message: 'Field "config" is required and must be an object' };
+  }
+  return null;
+}
+function validateRebalance(body) {
+  const target = body.targetPartitionCount;
+  if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
+    return { message: 'Field "targetPartitionCount" is required and must be a positive integer' };
+  }
+  return null;
+}
+
 // src/server/handlers/admin.ts
 function taskResponse(record) {
   return { taskId: record.id, type: record.type, status: record.status };
@@ -204,7 +318,7 @@ function createAdminHandlers(deps) {
     try {
       const bytes = await engine.snapshot(ctx.params[0]);
       if (ctx.abort.aborted) return;
-      sendBinary(ctx.res, bytes);
+      sendBinary(ctx.res, bytes, ctx.abort);
     } catch (err) {
       respondError(ctx, err);
     }
@@ -262,11 +376,12 @@ function createAdminHandlers(deps) {
   async function rebalance(ctx) {
     const body = parseJsonOptional(ctx);
     if (!body) return;
-    const target = body.targetPartitionCount;
-    if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
-      badRequest(ctx.res, 'Field "targetPartitionCount" is required and must be a positive integer');
+    const failure = validateRebalance(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
+    const target = body.targetPartitionCount;
     const name = ctx.params[0];
     try {
       requireIndexExists(name);
@@ -315,16 +430,14 @@ function createAdminHandlers(deps) {
 }
 
 // src/server/handlers/documents.ts
-function isPlainObject(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 function createDocumentHandlers(deps) {
-  const { engine } = deps;
+  const { engine, limits } = deps;
   async function insert(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -356,8 +469,9 @@ function createDocumentHandlers(deps) {
   async function put(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     const [name, id] = ctx.params;
@@ -376,8 +490,9 @@ function createDocumentHandlers(deps) {
   async function patch(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -405,8 +520,9 @@ function createDocumentHandlers(deps) {
   async function multiGet(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!Array.isArray(body.docIds)) {
-      badRequest(ctx.res, 'Field "docIds" is required and must be an array of strings');
+    const failure = validateMultiGet(body, limits.maxFetchDocuments);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -421,28 +537,22 @@ function createDocumentHandlers(deps) {
   async function batch(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
+    const failure = validateBatch(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     const action = body.action ?? "insert";
     try {
-      if (action === "insert") {
-        if (!Array.isArray(body.documents)) {
-          badRequest(ctx.res, 'Batch insert requires a "documents" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents, body.options)));
-      } else if (action === "update") {
-        if (!Array.isArray(body.updates)) {
-          badRequest(ctx.res, 'Batch update requires an "updates" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates)));
+      if (action === "update") {
+        respondJson(ctx, serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [])));
       } else if (action === "delete") {
-        if (!Array.isArray(body.docIds)) {
-          badRequest(ctx.res, 'Batch delete requires a "docIds" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds)));
+        respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [])));
       } else {
-        badRequest(ctx.res, 'Field "action" must be one of "insert", "update", or "delete"');
+        respondJson(
+          ctx,
+          serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
+        );
       }
     } catch (err) {
       respondError(ctx, err);
@@ -627,12 +737,9 @@ function createIndexHandlers(deps) {
   async function create(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (typeof body.name !== "string" || body.name.length === 0) {
-      badRequest(ctx.res, 'Field "name" is required and must be a non-empty string');
-      return;
-    }
-    if (typeof body.config !== "object" || body.config === null) {
-      badRequest(ctx.res, 'Field "config" is required and must be an object');
+    const failure = validateCreateIndex(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -684,22 +791,16 @@ function createIndexHandlers(deps) {
 }
 
 // src/server/handlers/search.ts
-function rejectsCustomReducer(ctx, params) {
-  if (params.group && typeof params.group === "object" && "reduce" in params.group) {
-    badRequest(
-      ctx.res,
-      'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
-    );
-    return true;
-  }
-  return false;
-}
 function createSearchHandlers(deps) {
-  const { engine } = deps;
+  const { engine, limits } = deps;
   async function search(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (rejectsCustomReducer(ctx, params)) return;
+    const failure = validateQuery(params, limits.maxResultWindow);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     try {
       respondJson(ctx, await engine.query(ctx.params[0], params));
     } catch (err) {
@@ -709,7 +810,11 @@ function createSearchHandlers(deps) {
   async function preflight(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (rejectsCustomReducer(ctx, params)) return;
+    const failure = validateQuery(params, limits.maxResultWindow);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     try {
       respondJson(ctx, await engine.preflight(ctx.params[0], params));
     } catch (err) {
@@ -719,8 +824,9 @@ function createSearchHandlers(deps) {
   async function suggest(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (typeof params.prefix !== "string") {
-      badRequest(ctx.res, 'Field "prefix" is required and must be a string');
+    const failure = validateSuggest(params);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -1007,7 +1113,9 @@ function resolveLimits(limits) {
     maxImportBytes: limits?.maxImportBytes ?? 100 * MB,
     maxLineBytes: limits?.maxLineBytes ?? 4 * MB,
     importBatchSize: limits?.importBatchSize ?? 1e3,
-    maxConcurrentRequests: limits?.maxConcurrentRequests ?? 0
+    maxConcurrentRequests: limits?.maxConcurrentRequests ?? 0,
+    maxResultWindow: limits?.maxResultWindow ?? 1e4,
+    maxFetchDocuments: limits?.maxFetchDocuments ?? 1e4
   };
 }
 function resolveBuild(build) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.1.14",
+  "version": "0.1.15",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",