@delali/narsil 0.1.14 → 0.2.0

package/dist/server.mjs

@@ -1,4 +1,4 @@
-import { NarsilError, ErrorCodes } from './chunk-ALB35PRQ.mjs';
+import { NarsilError, ErrorCodes } from './chunk-M2EDG55Y.mjs';
 import { randomUUID } from 'crypto';
 
 // src/server/cors.ts
@@ -81,7 +81,18 @@ var STATUS_BY_CODE = {
   QUERY_ROUTING_FAILED: 503,
   QUERY_PARTIAL_FAILURE: 503,
   QUERY_NODE_TIMEOUT: 503,
-  QUERY_NO_ACTIVE_REPLICA: 503
+  QUERY_NO_ACTIVE_REPLICA: 503,
+  REPLICATION_ENTRY_INVALID: 503,
+  REPLICATION_INSYNC_REMOVAL_FAILED: 503,
+  REPLICATION_ROLLBACK_FAILED: 503,
+  REPLICATION_LOG_FULL: 503,
+  REPLICATION_ENTRY_CORRUPT: 503,
+  REPLICATION_SNAPSHOT_CORRUPT: 503,
+  REPLICATION_TERM_MISMATCH: 503,
+  REPLICATION_SYNC_FAILED: 503,
+  PARTITION_NOT_PRIMARY: 503,
+  PARTITION_UNASSIGNED: 503,
+  INSUFFICIENT_REPLICAS: 503
 };
 function httpStatusForNarsilError(code) {
   return STATUS_BY_CODE[code] ?? 500;
@@ -118,11 +129,31 @@ var STATUS_TEXT = {
   500: "500 Internal Server Error",
   503: "503 Service Unavailable"
 };
+var STREAM_THRESHOLD = 64 * 1024;
 function statusLine(status) {
   return STATUS_TEXT[status] ?? `${status}`;
 }
-function sendJson(res, data, status = 200) {
+function streamBody(res, abort, body, status, contentType) {
+  if (abort.aborted) return;
+  const totalSize = body.byteLength;
+  res.cork(() => {
+    res.writeStatus(statusLine(status)).writeHeader("Content-Type", contentType);
+    const startOffset = res.getWriteOffset();
+    const [, done] = res.tryEnd(body, totalSize);
+    if (done) return;
+    res.onWritable((offset) => {
+      if (abort.aborted) return true;
+      const [retryOk] = res.tryEnd(body.subarray(offset - startOffset), totalSize);
+      return retryOk;
+    });
+  });
+}
+function sendJson(res, data, status = 200, abort) {
   const payload = JSON.stringify(data);
+  if (abort && payload !== void 0 && Buffer.byteLength(payload) >= STREAM_THRESHOLD) {
+    streamBody(res, abort, Buffer.from(payload, "utf8"), status, "application/json");
+    return;
+  }
   res.cork(() => {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
@@ -134,10 +165,8 @@ function sendError(res, status, code, message, details) {
     res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/json").end(payload);
   });
 }
-function sendBinary(res, data, status = 200) {
-  res.cork(() => {
-    res.writeStatus(statusLine(status)).writeHeader("Content-Type", "application/octet-stream").end(data);
-  });
+function sendBinary(res, data, abort, status = 200) {
+  streamBody(res, abort, data, status, "application/octet-stream");
 }
 
 // src/server/handler-utils.ts
@@ -171,11 +200,15 @@ function respondError(ctx, err) {
 }
 function respondJson(ctx, data, status = 200) {
   if (ctx.abort.aborted) return;
-  sendJson(ctx.res, data, status);
+  sendJson(ctx.res, data, status, ctx.abort);
 }
 function badRequest(res, message, details) {
   sendError(res, 400, ServerErrorCodes.INVALID_REQUEST, message, details);
 }
+function rejectInvalid(ctx, failure) {
+  if (ctx.abort.aborted) return;
+  badRequest(ctx.res, failure.message, failure.details);
+}
 function serializeBatchResult(result) {
   return {
     succeeded: result.succeeded,
@@ -186,6 +219,98 @@ function serializeBatchResult(result) {
   };
 }
 
+// src/server/validation.ts
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function overWindow(field, value, maxWindow) {
+  if (typeof value === "number" && value > maxWindow) {
+    return {
+      message: `Field "${field}" exceeds the maximum result window of ${maxWindow}`,
+      details: { value, limit: maxWindow }
+    };
+  }
+  return null;
+}
+function validateQuery(params, maxWindow) {
+  if (params.group && typeof params.group === "object" && "reduce" in params.group) {
+    return {
+      message: 'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
+    };
+  }
+  const rowBound = overWindow("limit", params.limit, maxWindow) ?? overWindow("offset", params.offset, maxWindow);
+  if (rowBound) return rowBound;
+  if (params.group && typeof params.group === "object") {
+    const grouped = overWindow("group.maxPerGroup", params.group.maxPerGroup, maxWindow);
+    if (grouped) return grouped;
+  }
+  if (params.facets && typeof params.facets === "object") {
+    for (const [field, facet] of Object.entries(params.facets)) {
+      if (facet && typeof facet === "object") {
+        const faceted = overWindow(`facets.${field}.limit`, facet.limit, maxWindow);
+        if (faceted) return faceted;
+      }
+    }
+  }
+  return null;
+}
+function validateSuggest(params) {
+  if (typeof params.prefix !== "string") {
+    return { message: 'Field "prefix" is required and must be a string' };
+  }
+  return null;
+}
+function validateDocumentBody(body) {
+  if (!isPlainObject(body.document)) {
+    return { message: 'Field "document" is required and must be an object' };
+  }
+  return null;
+}
+function validateMultiGet(body, maxFetch) {
+  if (!Array.isArray(body.docIds)) {
+    return { message: 'Field "docIds" is required and must be an array of strings' };
+  }
+  if (body.docIds.length > maxFetch) {
+    return {
+      message: `Field "docIds" exceeds the maximum of ${maxFetch} ids per request`,
+      details: { count: body.docIds.length, limit: maxFetch }
+    };
+  }
+  return null;
+}
+function validateBatch(body) {
+  const action = body.action ?? "insert";
+  if (action === "insert") {
+    if (!Array.isArray(body.documents)) return { message: 'Batch insert requires a "documents" array' };
+    return null;
+  }
+  if (action === "update") {
+    if (!Array.isArray(body.updates)) return { message: 'Batch update requires an "updates" array' };
+    return null;
+  }
+  if (action === "delete") {
+    if (!Array.isArray(body.docIds)) return { message: 'Batch delete requires a "docIds" array' };
+    return null;
+  }
+  return { message: 'Field "action" must be one of "insert", "update", or "delete"' };
+}
+function validateCreateIndex(body) {
+  if (typeof body.name !== "string" || body.name.length === 0) {
+    return { message: 'Field "name" is required and must be a non-empty string' };
+  }
+  if (!isPlainObject(body.config)) {
+    return { message: 'Field "config" is required and must be an object' };
+  }
+  return null;
+}
+function validateRebalance(body) {
+  const target = body.targetPartitionCount;
+  if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
+    return { message: 'Field "targetPartitionCount" is required and must be a positive integer' };
+  }
+  return null;
+}
+
 // src/server/handlers/admin.ts
 function taskResponse(record) {
   return { taskId: record.id, type: record.type, status: record.status };
@@ -204,7 +329,7 @@ function createAdminHandlers(deps) {
     try {
       const bytes = await engine.snapshot(ctx.params[0]);
       if (ctx.abort.aborted) return;
-      sendBinary(ctx.res, bytes);
+      sendBinary(ctx.res, bytes, ctx.abort);
     } catch (err) {
       respondError(ctx, err);
     }
@@ -262,11 +387,12 @@ function createAdminHandlers(deps) {
   async function rebalance(ctx) {
     const body = parseJsonOptional(ctx);
     if (!body) return;
-    const target = body.targetPartitionCount;
-    if (typeof target !== "number" || !Number.isInteger(target) || target <= 0) {
-      badRequest(ctx.res, 'Field "targetPartitionCount" is required and must be a positive integer');
+    const failure = validateRebalance(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
+    const target = body.targetPartitionCount;
     const name = ctx.params[0];
     try {
       requireIndexExists(name);
@@ -315,16 +441,14 @@ function createAdminHandlers(deps) {
 }
 
 // src/server/handlers/documents.ts
-function isPlainObject(value) {
-  return typeof value === "object" && value !== null && !Array.isArray(value);
-}
 function createDocumentHandlers(deps) {
-  const { engine } = deps;
+  const { engine, limits } = deps;
   async function insert(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -356,8 +480,9 @@ function createDocumentHandlers(deps) {
   async function put(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     const [name, id] = ctx.params;
@@ -376,8 +501,9 @@ function createDocumentHandlers(deps) {
   async function patch(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!isPlainObject(body.document)) {
-      badRequest(ctx.res, 'Field "document" is required and must be an object');
+    const failure = validateDocumentBody(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -405,8 +531,9 @@ function createDocumentHandlers(deps) {
   async function multiGet(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (!Array.isArray(body.docIds)) {
-      badRequest(ctx.res, 'Field "docIds" is required and must be an array of strings');
+    const failure = validateMultiGet(body, limits.maxFetchDocuments);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -421,28 +548,22 @@ function createDocumentHandlers(deps) {
   async function batch(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
+    const failure = validateBatch(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     const action = body.action ?? "insert";
     try {
-      if (action === "insert") {
-        if (!Array.isArray(body.documents)) {
-          badRequest(ctx.res, 'Batch insert requires a "documents" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents, body.options)));
-      } else if (action === "update") {
-        if (!Array.isArray(body.updates)) {
-          badRequest(ctx.res, 'Batch update requires an "updates" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates)));
+      if (action === "update") {
+        respondJson(ctx, serializeBatchResult(await engine.updateBatch(ctx.params[0], body.updates ?? [])));
       } else if (action === "delete") {
-        if (!Array.isArray(body.docIds)) {
-          badRequest(ctx.res, 'Batch delete requires a "docIds" array');
-          return;
-        }
-        respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds)));
+        respondJson(ctx, serializeBatchResult(await engine.removeBatch(ctx.params[0], body.docIds ?? [])));
       } else {
-        badRequest(ctx.res, 'Field "action" must be one of "insert", "update", or "delete"');
+        respondJson(
+          ctx,
+          serializeBatchResult(await engine.insertBatch(ctx.params[0], body.documents ?? [], body.options))
+        );
       }
     } catch (err) {
       respondError(ctx, err);
@@ -627,12 +748,9 @@ function createIndexHandlers(deps) {
   async function create(ctx) {
     const body = parseJson(ctx);
     if (!body) return;
-    if (typeof body.name !== "string" || body.name.length === 0) {
-      badRequest(ctx.res, 'Field "name" is required and must be a non-empty string');
-      return;
-    }
-    if (typeof body.config !== "object" || body.config === null) {
-      badRequest(ctx.res, 'Field "config" is required and must be an object');
+    const failure = validateCreateIndex(body);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -684,22 +802,16 @@ function createIndexHandlers(deps) {
 }
 
 // src/server/handlers/search.ts
-function rejectsCustomReducer(ctx, params) {
-  if (params.group && typeof params.group === "object" && "reduce" in params.group) {
-    badRequest(
-      ctx.res,
-      'Custom group reducers are not available over HTTP; use "group.fields" and "group.maxPerGroup" only'
-    );
-    return true;
-  }
-  return false;
-}
 function createSearchHandlers(deps) {
-  const { engine } = deps;
+  const { engine, limits } = deps;
   async function search(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (rejectsCustomReducer(ctx, params)) return;
+    const failure = validateQuery(params, limits.maxResultWindow);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     try {
       respondJson(ctx, await engine.query(ctx.params[0], params));
     } catch (err) {
@@ -709,7 +821,11 @@ function createSearchHandlers(deps) {
   async function preflight(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (rejectsCustomReducer(ctx, params)) return;
+    const failure = validateQuery(params, limits.maxResultWindow);
+    if (failure) {
+      rejectInvalid(ctx, failure);
+      return;
+    }
     try {
       respondJson(ctx, await engine.preflight(ctx.params[0], params));
     } catch (err) {
@@ -719,8 +835,9 @@ function createSearchHandlers(deps) {
   async function suggest(ctx) {
     const params = parseJson(ctx);
     if (!params) return;
-    if (typeof params.prefix !== "string") {
-      badRequest(ctx.res, 'Field "prefix" is required and must be a string');
+    const failure = validateSuggest(params);
+    if (failure) {
+      rejectInvalid(ctx, failure);
       return;
     }
     try {
@@ -1007,7 +1124,9 @@ function resolveLimits(limits) {
     maxImportBytes: limits?.maxImportBytes ?? 100 * MB,
     maxLineBytes: limits?.maxLineBytes ?? 4 * MB,
     importBatchSize: limits?.importBatchSize ?? 1e3,
-    maxConcurrentRequests: limits?.maxConcurrentRequests ?? 0
+    maxConcurrentRequests: limits?.maxConcurrentRequests ?? 0,
+    maxResultWindow: limits?.maxResultWindow ?? 1e4,
+    maxFetchDocuments: limits?.maxFetchDocuments ?? 1e4
   };
 }
 function resolveBuild(build) {

package/dist/{types-DHX6zGeD.d.ts → types-BRTcBgAp.d.ts}

@@ -1,4 +1,4 @@
-import { S as SchemaDefinition } from './schema-DCgkO1j4.js';
+import { b as SchemaDefinition } from './schema-BnvLvjRp.js';
 
 type NodeRole = 'data' | 'coordinator' | 'controller';
 type PartitionState = 'UNASSIGNED' | 'INITIALISING' | 'ACTIVE' | 'MIGRATING' | 'DECOMMISSIONING';

package/dist/workers/entry.mjs

@@ -1,4 +1,4 @@
-import { isValidWorkerAction } from '../chunk-HRABJLPB.mjs';
+import { isValidWorkerAction } from '../chunk-BKAI2WXI.mjs';
 
 // src/workers/worker-entry.ts
 function startWorker() {
@@ -15,7 +15,7 @@ async function setup() {
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-E52FFGDE.mjs');
+  const { createDirectExecutor } = await import('../direct-executor-2FV3MEDD.mjs');
   const executor = createDirectExecutor();
   function buildErrorResponse(requestId, code, message) {
     return { type: "error", requestId, code, message };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.1.14",
+  "version": "0.2.0",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -109,7 +109,8 @@
   },
   "files": [
     "dist/",
-    "LICENSE"
+    "LICENSE",
+    "NOTICE"
   ],
   "dependencies": {
     "@msgpack/msgpack": "3.1.1"
@@ -122,6 +123,7 @@
     "fflate": "0.8.3",
     "selfsigned": "5.5.0",
     "tsup": "8.5.1",
+    "tsx": "4.21.0",
     "typescript": "5.9.3",
     "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.58.0",
     "vitest": "4.0.18",
@@ -149,10 +151,15 @@
     "test:recall": "vitest run src/__tests__/vector/recall.test.ts",
     "test:embeddings": "NARSIL_TEST_EMBEDDINGS=1 vitest run embedding-e2e",
     "bench": "vitest bench",
+    "stemmers": "tsx snowball/build.ts",
+    "revisions": "tsx scripts/analysis-revisions.ts",
+    "revisions:write": "tsx scripts/analysis-revisions.ts --write",
+    "lookalikes": "tsx scripts/lookalike-counts.ts",
+    "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
     "lint": "biome check src/",
     "typecheck:src": "tsc --noEmit",
     "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
-    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json",
+    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json && tsc --noEmit -p snowball/tsconfig.json",
     "typecheck": "pnpm run typecheck:src && pnpm run typecheck:tests && pnpm run typecheck:tooling"
   }
 }