@delali/narsil 0.1.6 → 0.1.8

package/dist/types-Bt7c4hDF.d.ts

@@ -0,0 +1,73 @@
+import { S as SchemaDefinition } from './schema-BVaw7nqz.js';
+
+type NodeRole = 'data' | 'coordinator' | 'controller';
+type PartitionState = 'UNASSIGNED' | 'INITIALISING' | 'ACTIVE' | 'MIGRATING' | 'DECOMMISSIONING';
+interface NodeCapacity {
+    memoryBytes: number;
+    cpuCores: number;
+    diskBytes: number | null;
+}
+interface NodeRegistration {
+    nodeId: string;
+    address: string;
+    roles: NodeRole[];
+    capacity: NodeCapacity;
+    startedAt: string;
+    version: string;
+    metadata?: Record<string, string>;
+}
+interface PartitionAssignment {
+    primary: string | null;
+    replicas: string[];
+    inSyncSet: string[];
+    state: PartitionState;
+    primaryTerm: number;
+}
+interface AllocationTable {
+    indexName: string;
+    version: number;
+    replicationFactor: number;
+    assignments: Map<number, PartitionAssignment>;
+}
+interface NodeEvent {
+    type: 'node_joined' | 'node_left';
+    nodeId: string;
+    registration: NodeRegistration | null;
+}
+interface AllocationEvent {
+    indexName: string;
+    table: AllocationTable;
+}
+interface SchemaEvent {
+    type: 'schema_created' | 'schema_dropped';
+    indexName: string;
+    schema: SchemaDefinition | null;
+}
+interface AllocationConstraints {
+    zoneAwareness: boolean;
+    zoneAttribute: string;
+    maxShardsPerNode: number | null;
+}
+interface ClusterCoordinator {
+    registerNode(registration: NodeRegistration): Promise<void>;
+    deregisterNode(nodeId: string): Promise<void>;
+    listNodes(): Promise<NodeRegistration[]>;
+    watchNodes(handler: (event: NodeEvent) => void): Promise<() => void>;
+    getAllocation(indexName: string): Promise<AllocationTable | null>;
+    putAllocation(indexName: string, table: AllocationTable, expectedVersion?: number | null): Promise<boolean>;
+    watchAllocation(handler: (event: AllocationEvent) => void): Promise<() => void>;
+    getPartitionState(indexName: string, partitionId: number): Promise<PartitionState>;
+    putPartitionState(indexName: string, partitionId: number, state: PartitionState): Promise<void>;
+    acquireLease(key: string, nodeId: string, ttlMs: number): Promise<boolean>;
+    renewLease(key: string, nodeId: string, ttlMs: number): Promise<boolean>;
+    releaseLease(key: string): Promise<void>;
+    get(key: string): Promise<Uint8Array | null>;
+    compareAndSet(key: string, expected: Uint8Array | null, value: Uint8Array): Promise<boolean>;
+    getSchema(indexName: string): Promise<SchemaDefinition | null>;
+    putSchema(indexName: string, schema: SchemaDefinition): Promise<void>;
+    watchSchemas(handler: (event: SchemaEvent) => void): Promise<() => void>;
+    getLeaseHolder(key: string): Promise<string | null>;
+    shutdown(): Promise<void>;
+}
+
+export type { AllocationTable as A, ClusterCoordinator as C, NodeRegistration as N, PartitionAssignment as P, SchemaEvent as S, NodeRole as a, NodeCapacity as b, AllocationConstraints as c, AllocationEvent as d, NodeEvent as e, PartitionState as f };

package/dist/vector/hnsw-build-worker.d.ts

@@ -6,7 +6,7 @@ interface HNSWConfig {
     metric?: VectorMetric;
 }
 interface SerializedHNSWGraph {
-    entryPoint: string;
+    entryPoint: string | null;
     maxLayer: number;
     m: number;
     efConstruction: number;
@@ -23,6 +23,13 @@ interface HNSWBuildRequest {
     dimension: number;
     config: HNSWConfig;
 }
+interface HNSWBuildRequestBinary {
+    type: 'build-binary';
+    docIds: string[];
+    vectorData: Float32Array;
+    dimension: number;
+    config: HNSWConfig;
+}
 interface HNSWBuildResponse {
     type: 'success';
     graph: SerializedHNSWGraph;
@@ -34,4 +41,4 @@ interface HNSWBuildError {
 type HNSWWorkerMessage = HNSWBuildResponse | HNSWBuildError;
 declare function handleBuildRequest(request: HNSWBuildRequest): SerializedHNSWGraph;
 
-export { type HNSWBuildError, type HNSWBuildRequest, type HNSWBuildResponse, type HNSWWorkerMessage, handleBuildRequest };
+export { type HNSWBuildError, type HNSWBuildRequest, type HNSWBuildRequestBinary, type HNSWBuildResponse, type HNSWWorkerMessage, handleBuildRequest };

package/dist/vector/hnsw-build-worker.mjs

@@ -1,4 +1,4 @@
-import { createHNSWIndex } from '../chunk-GVN2W4VA.mjs';
+import { createVectorStore, createHNSWIndex } from '../chunk-S7RJJMHI.mjs';
 
 // src/vector/hnsw-build-worker.ts
 var MAX_WORKER_DIMENSION = 8192;
@@ -23,15 +23,63 @@ function validateBuildRequest(request) {
     }
   }
 }
+function clampHNSWConfig(config) {
+  return {
+    ...config,
+    m: config.m != null ? Math.max(2, Math.min(64, config.m)) : void 0,
+    efConstruction: config.efConstruction != null ? Math.max(10, Math.min(1e3, config.efConstruction)) : void 0
+  };
+}
 function handleBuildRequest(request) {
   validateBuildRequest(request);
-  const hnsw = createHNSWIndex(request.dimension, request.config);
+  const safeConfig = clampHNSWConfig(request.config);
+  const store = createVectorStore();
+  for (const entry of request.vectors) {
+    store.insert(entry.docId, new Float32Array(entry.values));
+  }
+  const hnsw = createHNSWIndex(request.dimension, store, safeConfig);
   for (const entry of request.vectors) {
-    const vec = new Float32Array(entry.values);
-    hnsw.insert(entry.docId, vec);
+    hnsw.insertNode(entry.docId);
   }
   return hnsw.serialize();
 }
+function handleBuildRequestBinary(request) {
+  const { docIds, vectorData, dimension, config } = request;
+  if (!Number.isInteger(dimension) || dimension <= 0 || dimension > MAX_WORKER_DIMENSION) {
+    throw new Error(`Invalid dimension: ${dimension} (must be 1..${MAX_WORKER_DIMENSION})`);
+  }
+  if (docIds.length > MAX_WORKER_VECTORS) {
+    throw new Error(`Too many vectors: ${docIds.length} (max ${MAX_WORKER_VECTORS})`);
+  }
+  const expectedLength = docIds.length * dimension;
+  if (vectorData.length !== expectedLength) {
+    throw new Error(
+      `vectorData length ${vectorData.length} does not match ${docIds.length} * ${dimension} = ${expectedLength}`
+    );
+  }
+  const safeConfig = clampHNSWConfig(config);
+  const store = createVectorStore();
+  for (let i = 0; i < docIds.length; i++) {
+    const offset = i * dimension;
+    const vec = vectorData.subarray(offset, offset + dimension);
+    store.insert(docIds[i], new Float32Array(vec));
+  }
+  const hnsw = createHNSWIndex(dimension, store, safeConfig);
+  for (const docId of docIds) {
+    hnsw.insertNode(docId);
+  }
+  return hnsw.serialize();
+}
+function handleAnyRequest(raw) {
+  const request = raw;
+  if (request.type === "build") {
+    return handleBuildRequest(raw);
+  }
+  if (request.type === "build-binary") {
+    return handleBuildRequestBinary(raw);
+  }
+  throw new Error(`Unknown request type: ${request.type}`);
+}
 function setupWorker() {
   setupAsync().catch((err) => {
     console.error("HNSW build worker setup failed:", err);
@@ -48,10 +96,8 @@ async function setupAsync() {
   if (parentPort) {
     const port = parentPort;
     port.on("message", (raw) => {
-      const request = raw;
-      if (request.type !== "build") return;
       try {
-        const graph = handleBuildRequest(request);
+        const graph = handleAnyRequest(raw);
         port.postMessage({ type: "success", graph });
       } catch (err) {
         const message = err instanceof Error ? err.message : String(err);
@@ -64,10 +110,8 @@ async function setupAsync() {
   if (globalSelf && typeof globalSelf.postMessage === "function") {
     const webSelf = globalSelf;
     webSelf.onmessage = (event) => {
-      const request = event.data;
-      if (request.type !== "build") return;
       try {
-        const graph = handleBuildRequest(request);
+        const graph = handleAnyRequest(event.data);
         webSelf.postMessage({ type: "success", graph });
       } catch (err) {
         const message = err instanceof Error ? err.message : String(err);

package/dist/workers/entry.mjs

@@ -15,7 +15,7 @@ async function setup() {
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-6TSG7GWH.mjs');
+  const { createDirectExecutor } = await import('../direct-executor-FX56DYD3.mjs');
   const executor = createDirectExecutor();
   function buildErrorResponse(requestId, code, message) {
     return { type: "error", requestId, code, message };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -74,9 +74,29 @@
       "import": "./dist/vector/hnsw-build-worker.mjs",
       "types": "./dist/vector/hnsw-build-worker.d.ts"
     },
+    "./embeddings/openai": {
+      "import": "./dist/embeddings/openai.mjs",
+      "types": "./dist/embeddings/openai.d.ts"
+    },
     "./languages/*": {
       "import": "./dist/languages/*.mjs",
       "types": "./dist/languages/*.d.ts"
+    },
+    "./distribution": {
+      "import": "./dist/distribution.mjs",
+      "types": "./dist/distribution.d.ts"
+    },
+    "./distribution/coordinator/in-memory": {
+      "import": "./dist/distribution/coordinator/in-memory.mjs",
+      "types": "./dist/distribution/coordinator/in-memory.d.ts"
+    },
+    "./distribution/coordinator/etcd": {
+      "import": "./dist/distribution/coordinator/etcd.mjs",
+      "types": "./dist/distribution/coordinator/etcd.d.ts"
+    },
+    "./server": {
+      "import": "./dist/server.mjs",
+      "types": "./dist/server.d.ts"
     }
   },
   "files": [
@@ -90,18 +110,40 @@
     "@biomejs/biome": "2.4.4",
     "@types/node": "22.15.32",
     "@vitest/coverage-v8": "4.0.18",
+    "etcd3": "1.1.2",
+    "selfsigned": "5.5.0",
     "tsup": "8.5.1",
     "typescript": "5.9.3",
+    "uWebSockets.js": "github:uNetworking/uWebSockets.js#v20.58.0",
     "vitest": "4.0.18",
-    "wabt": "1.0.39"
+    "wabt": "1.0.39",
+    "@delali/narsil-embeddings-transformers": "0.1.6"
+  },
+  "peerDependencies": {
+    "etcd3": "1.1.2",
+    "uWebSockets.js": ">=20.0.0"
+  },
+  "peerDependenciesMeta": {
+    "etcd3": {
+      "optional": true
+    },
+    "uWebSockets.js": {
+      "optional": true
+    }
   },
   "scripts": {
     "build": "tsup",
-    "test": "vitest run",
+    "test": "vitest run --exclude='**/vector/recall.test.ts'",
+    "test:etcd": "vitest run src/__tests__/distribution/coordinator/etcd-integration/*.test.ts",
     "test:watch": "vitest",
     "test:coverage": "vitest run --coverage",
+    "test:recall": "vitest run src/__tests__/vector/recall.test.ts",
+    "test:embeddings": "NARSIL_TEST_EMBEDDINGS=1 vitest run embedding-e2e",
     "bench": "vitest bench",
     "lint": "biome check src/",
-    "typecheck": "tsc --noEmit"
+    "typecheck:src": "tsc --noEmit",
+    "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",
+    "typecheck:tooling": "tsc --noEmit -p scripts/tsconfig.json && tsc --noEmit -p benchmarks/tsconfig.json && tsc --noEmit -p wasm/tsconfig.json",
+    "typecheck": "pnpm run typecheck:src && pnpm run typecheck:tests && pnpm run typecheck:tooling"
   }
 }

package/dist/chunk-7GKPWGCN.mjs

@@ -1,43 +0,0 @@
-// src/errors.ts
-var ErrorCodes = {
-  SCHEMA_INVALID_TYPE: "SCHEMA_INVALID_TYPE",
-  SCHEMA_MISSING_FIELD: "SCHEMA_MISSING_FIELD",
-  SCHEMA_DEPTH_EXCEEDED: "SCHEMA_DEPTH_EXCEEDED",
-  SCHEMA_INVALID_VECTOR_DIMENSION: "SCHEMA_INVALID_VECTOR_DIMENSION",
-  SCHEMA_INVALID_GEOPOINT: "SCHEMA_INVALID_GEOPOINT",
-  DOC_NOT_FOUND: "DOC_NOT_FOUND",
-  DOC_ALREADY_EXISTS: "DOC_ALREADY_EXISTS",
-  DOC_VALIDATION_FAILED: "DOC_VALIDATION_FAILED",
-  INDEX_NOT_FOUND: "INDEX_NOT_FOUND",
-  INDEX_ALREADY_EXISTS: "INDEX_ALREADY_EXISTS",
-  PARTITION_CORRUPTED: "PARTITION_CORRUPTED",
-  PARTITION_REBALANCING_BACKPRESSURE: "PARTITION_REBALANCING_BACKPRESSURE",
-  WORKER_CRASHED: "WORKER_CRASHED",
-  WORKER_BUSY: "WORKER_BUSY",
-  WORKER_TIMEOUT: "WORKER_TIMEOUT",
-  PERSISTENCE_SAVE_FAILED: "PERSISTENCE_SAVE_FAILED",
-  PERSISTENCE_LOAD_FAILED: "PERSISTENCE_LOAD_FAILED",
-  PERSISTENCE_DELETE_FAILED: "PERSISTENCE_DELETE_FAILED",
-  PERSISTENCE_CRC_MISMATCH: "PERSISTENCE_CRC_MISMATCH",
-  SEARCH_INVALID_FIELD: "SEARCH_INVALID_FIELD",
-  SEARCH_INVALID_VECTOR_SIZE: "SEARCH_INVALID_VECTOR_SIZE",
-  SEARCH_INVALID_FILTER: "SEARCH_INVALID_FILTER",
-  SEARCH_INVALID_MODE: "SEARCH_INVALID_MODE",
-  SEARCH_INVALID_CURSOR: "SEARCH_INVALID_CURSOR",
-  LANGUAGE_NOT_SUPPORTED: "LANGUAGE_NOT_SUPPORTED",
-  ENVELOPE_VERSION_MISMATCH: "ENVELOPE_VERSION_MISMATCH",
-  ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
-  PARTITION_CAPACITY_EXCEEDED: "PARTITION_CAPACITY_EXCEEDED"
-};
-var NarsilError = class extends Error {
-  code;
-  details;
-  constructor(code, message, details) {
-    super(message);
-    this.name = "NarsilError";
-    this.code = code;
-    this.details = details ?? {};
-  }
-};
-
-export { ErrorCodes, NarsilError };