@delali/narsil 0.2.2 → 0.2.3-next.33

package/dist/workers/entry.mjs

@@ -1,4 +1,44 @@
-import { isValidWorkerAction } from '../chunk-BKAI2WXI.mjs';
+import { isValidWorkerAction } from '../chunk-WTONLDU3.mjs';
+import { NarsilError, ErrorCodes } from '../chunk-EMI63EXP.mjs';
+
+// src/workers/action-handler.ts
+function buildErrorResponse(requestId, code, message) {
+  return { type: "error", requestId, code, message };
+}
+function buildSuccessResponse(requestId, data) {
+  return { type: "success", requestId, data };
+}
+async function importBootstrapModule(moduleUrl) {
+  if (typeof moduleUrl !== "string" || moduleUrl.trim().length === 0) {
+    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "A bootstrap module needs a non-empty module URL", {
+      moduleUrl
+    });
+  }
+  await import(moduleUrl);
+}
+function createActionHandler(executor) {
+  return async function handleAction(action, post) {
+    if (action.type === "shutdown") {
+      await executor.shutdown();
+      post(buildSuccessResponse(action.requestId, void 0));
+      return true;
+    }
+    try {
+      if (action.type === "bootstrap") {
+        await importBootstrapModule(action.moduleUrl);
+        post(buildSuccessResponse(action.requestId, void 0));
+        return false;
+      }
+      const result = await executor.execute(action);
+      post(buildSuccessResponse(action.requestId, result));
+    } catch (err) {
+      const code = err?.code ?? "UNKNOWN_ERROR";
+      const message = err instanceof Error ? err.message : String(err);
+      post(buildErrorResponse(action.requestId, code, message));
+    }
+    return false;
+  };
+}
 
 // src/workers/worker-entry.ts
 function startWorker() {
@@ -15,30 +55,8 @@ async function setup() {
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-3W4QRA4E.mjs');
-  const executor = createDirectExecutor();
-  function buildErrorResponse(requestId, code, message) {
-    return { type: "error", requestId, code, message };
-  }
-  function buildSuccessResponse(requestId, data) {
-    return { type: "success", requestId, data };
-  }
-  async function handleAction(action, postFn) {
-    if (action.type === "shutdown") {
-      await executor.shutdown();
-      postFn(buildSuccessResponse(action.requestId, void 0));
-      return true;
-    }
-    try {
-      const result = await executor.execute(action);
-      postFn(buildSuccessResponse(action.requestId, result));
-    } catch (err) {
-      const code = err?.code ?? "UNKNOWN_ERROR";
-      const message = err instanceof Error ? err.message : String(err);
-      postFn(buildErrorResponse(action.requestId, code, message));
-    }
-    return false;
-  }
+  const { createDirectExecutor } = await import('../direct-executor-YARCDMJF.mjs');
+  const handleAction = createActionHandler(createDirectExecutor());
   if (parentPort) {
     const port = parentPort;
     port.on("message", (raw) => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.2.2",
+  "version": "0.2.3-next.33",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -82,6 +82,10 @@
       "import": "./dist/vector/hnsw-build-worker.mjs",
       "types": "./dist/vector/hnsw-build-worker.d.ts"
     },
+    "./vector/search-worker": {
+      "import": "./dist/vector/search-worker.mjs",
+      "types": "./dist/vector/search-worker.d.ts"
+    },
     "./embeddings/openai": {
       "import": "./dist/embeddings/openai.mjs",
       "types": "./dist/embeddings/openai.d.ts"
@@ -102,9 +106,29 @@
       "import": "./dist/distribution/coordinator/etcd.mjs",
       "types": "./dist/distribution/coordinator/etcd.d.ts"
     },
+    "./distribution/transport/tcp": {
+      "import": "./dist/distribution/transport/tcp.mjs",
+      "types": "./dist/distribution/transport/tcp.d.ts"
+    },
+    "./distribution/transport/in-memory": {
+      "import": "./dist/distribution/transport/in-memory.mjs",
+      "types": "./dist/distribution/transport/in-memory.d.ts"
+    },
+    "./distribution/transport/grpc": {
+      "import": "./dist/distribution/transport/grpc.mjs",
+      "types": "./dist/distribution/transport/grpc.d.ts"
+    },
     "./server": {
       "import": "./dist/server.mjs",
       "types": "./dist/server.d.ts"
+    },
+    "./client": {
+      "import": "./dist/client.mjs",
+      "types": "./dist/client.d.ts"
+    },
+    "./react": {
+      "import": "./dist/react.mjs",
+      "types": "./dist/react.d.ts"
     }
   },
   "files": [
@@ -117,11 +141,17 @@
   },
   "devDependencies": {
     "@biomejs/biome": "2.4.4",
+    "@grpc/grpc-js": "1.14.4",
     "@microsoft/api-extractor": "7.58.12",
     "@types/node": "22.15.32",
+    "@types/react": "19.2.14",
+    "@types/react-dom": "19.2.3",
     "@vitest/coverage-v8": "4.0.18",
     "etcd3": "1.1.2",
     "fflate": "0.8.3",
+    "jsdom": "28.1.0",
+    "react": "19.2.4",
+    "react-dom": "19.2.4",
     "selfsigned": "5.5.0",
     "tsup": "8.5.1",
     "tsx": "4.21.0",
@@ -132,19 +162,27 @@
     "@delali/narsil-embeddings-transformers": "0.1.6"
   },
   "peerDependencies": {
+    "@grpc/grpc-js": "1.14.4",
     "etcd3": "1.1.2",
+    "react": ">=19.2.0",
     "uWebSockets.js": ">=20.0.0"
   },
   "peerDependenciesMeta": {
+    "@grpc/grpc-js": {
+      "optional": true
+    },
     "etcd3": {
       "optional": true
     },
+    "react": {
+      "optional": true
+    },
     "uWebSockets.js": {
       "optional": true
     }
   },
   "scripts": {
-    "build": "tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
+    "build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
     "test": "vitest run --exclude='**/vector/recall.test.ts'",
     "test:etcd": "vitest run src/__tests__/distribution/coordinator/etcd-integration/*.test.ts",
     "test:watch": "vitest",
@@ -159,6 +197,8 @@
     "api:write": "tsx scripts/api-surface.ts --write",
     "lookalikes": "tsx scripts/lookalike-counts.ts",
     "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
+    "fold-table": "tsx scripts/fold-table.ts",
+    "fold-table:write": "tsx scripts/fold-table.ts --write",
     "lint": "biome check src/",
     "typecheck:src": "tsc --noEmit",
     "typecheck:tests": "tsc --noEmit -p tsconfig.test.json",

package/dist/chunk-FKK76OR7.mjs

@@ -1,456 +0,0 @@
-import { NarsilError, ErrorCodes } from './chunk-UNJZ7ZDB.mjs';
-import { decode, encode } from '@msgpack/msgpack';
-
-// src/distribution/coordinator/etcd/types.ts
-var DEFAULT_ETCD_CONFIG = {
-  endpoints: ["http://localhost:2379"],
-  keyPrefix: "_narsil",
-  nodeHeartbeatTtlSeconds: 30,
-  leaseTtlSeconds: 15
-};
-var ETCD_KEY_NODES = "nodes";
-var ETCD_KEY_ALLOCATION = "allocation";
-var ETCD_KEY_PARTITION = "partition";
-var ETCD_KEY_SCHEMA = "schema";
-function buildKey(prefix, ...segments) {
-  return [prefix, ...segments].join("/");
-}
-
-// src/distribution/coordinator/etcd/keys.ts
-var ETCD_KEY_LEASE = "lease";
-var ETCD_KEY_GENERIC = "kv";
-function coordinatorKeys(config) {
-  const prefix = config.keyPrefix;
-  return {
-    node: (nodeId) => buildKey(prefix, ETCD_KEY_NODES, nodeId),
-    nodePrefix: () => buildKey(prefix, ETCD_KEY_NODES),
-    allocation: (indexName) => buildKey(prefix, ETCD_KEY_ALLOCATION, indexName),
-    allocationPrefix: () => buildKey(prefix, ETCD_KEY_ALLOCATION),
-    partition: (indexName, partitionId) => buildKey(prefix, ETCD_KEY_PARTITION, indexName, String(partitionId)),
-    schema: (indexName) => buildKey(prefix, ETCD_KEY_SCHEMA, indexName),
-    schemaPrefix: () => buildKey(prefix, ETCD_KEY_SCHEMA),
-    lease: (key) => buildKey(prefix, ETCD_KEY_LEASE, key),
-    generic: (key) => buildKey(prefix, ETCD_KEY_GENERIC, key)
-  };
-}
-function extractSuffix(fullKey, prefix) {
-  return fullKey.slice(prefix.length + 1);
-}
-
-// src/distribution/coordinator/etcd/leases.ts
-var LeaseManager = class {
-  leases = /* @__PURE__ */ new Map();
-  track(key, lease, nodeId) {
-    this.leases.set(key, { lease, nodeId, key });
-  }
-  get(key) {
-    return this.leases.get(key);
-  }
-  getByNodeId(key, nodeId) {
-    const entry = this.leases.get(key);
-    if (entry === void 0 || entry.nodeId !== nodeId) {
-      return void 0;
-    }
-    return entry;
-  }
-  remove(key) {
-    const entry = this.leases.get(key);
-    if (entry !== void 0) {
-      this.leases.delete(key);
-    }
-    return entry;
-  }
-  async revokeAll() {
-    const revokePromises = [];
-    for (const [, entry] of this.leases) {
-      revokePromises.push(
-        entry.lease.revoke().catch(() => {
-        })
-      );
-    }
-    this.leases.clear();
-    await Promise.all(revokePromises);
-  }
-  entries() {
-    return this.leases.entries();
-  }
-};
-
-// src/distribution/coordinator/etcd/loader.ts
-async function loadEtcd3Module() {
-  try {
-    return await import('etcd3');
-  } catch (err) {
-    throw new NarsilError(
-      ErrorCodes.COORDINATOR_DEPENDENCY_MISSING,
-      "The `etcd3` package is not installed. Install it with `pnpm add etcd3` (or the npm or yarn equivalent) to use createEtcdCoordinator.",
-      { cause: err instanceof Error ? err.message : String(err) }
-    );
-  }
-}
-function serializeAllocationTable(table) {
-  const serializable = {
-    indexName: table.indexName,
-    version: table.version,
-    replicationFactor: table.replicationFactor,
-    assignments: Array.from(table.assignments.entries())
-  };
-  return new Uint8Array(encode(serializable));
-}
-function deserializeAllocationTable(data) {
-  const raw = decode(data);
-  if (typeof raw.indexName !== "string") {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable missing or invalid indexName field");
-  }
-  if (typeof raw.version !== "number") {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable missing or invalid version field");
-  }
-  if (!Array.isArray(raw.assignments)) {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable missing or invalid assignments field");
-  }
-  return {
-    indexName: raw.indexName,
-    version: raw.version,
-    replicationFactor: typeof raw.replicationFactor === "number" ? raw.replicationFactor : 0,
-    assignments: new Map(raw.assignments)
-  };
-}
-function serializeNodeRegistration(reg) {
-  return new Uint8Array(encode(reg));
-}
-function deserializeNodeRegistration(data) {
-  const raw = decode(data);
-  if (typeof raw.nodeId !== "string") {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "NodeRegistration missing or invalid nodeId field");
-  }
-  if (typeof raw.address !== "string") {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "NodeRegistration missing or invalid address field");
-  }
-  return raw;
-}
-function serializeSchema(schema) {
-  return Buffer.from(new Uint8Array(encode(schema)));
-}
-function deserializeSchema(data) {
-  return decode(data);
-}
-
-// src/distribution/coordinator/etcd/validation.ts
-var MAX_NODE_ID_LENGTH = 255;
-var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
-var MAX_WATCHERS = 64;
-function validateNodeId(nodeId) {
-  if (nodeId.length === 0) {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Node ID must not be empty");
-  }
-  if (nodeId.length > MAX_NODE_ID_LENGTH) {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Node ID exceeds the ${MAX_NODE_ID_LENGTH} character limit`, {
-      nodeId,
-      length: nodeId.length
-    });
-  }
-  if (nodeId.includes("/") || nodeId.includes("\\") || nodeId.includes("..") || nodeId.includes("\0")) {
-    throw new NarsilError(
-      ErrorCodes.CONFIG_INVALID,
-      "Node ID contains forbidden characters (/, \\, .., or null bytes)",
-      {
-        nodeId
-      }
-    );
-  }
-}
-function validatePartitionState(value) {
-  if (!VALID_PARTITION_STATES.has(value)) {
-    throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Unknown partition state '${value}'`, {
-      state: value,
-      validStates: Array.from(VALID_PARTITION_STATES)
-    });
-  }
-  return value;
-}
-
-// src/distribution/coordinator/etcd/index.ts
-async function createEtcdCoordinator(config) {
-  const resolvedConfig = {
-    ...DEFAULT_ETCD_CONFIG,
-    ...config
-  };
-  const etcd3Module = await loadEtcd3Module();
-  const client = new etcd3Module.Etcd3({ hosts: resolvedConfig.endpoints });
-  const leaseManager = new LeaseManager();
-  const watchers = /* @__PURE__ */ new Set();
-  let isShutdown = false;
-  function assertNotShutdown() {
-    if (isShutdown) {
-      throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Coordinator has been shut down");
-    }
-  }
-  const keys = coordinatorKeys(resolvedConfig);
-  function addWatcher(watcher) {
-    if (watchers.size >= MAX_WATCHERS) {
-      watcher.cancel().catch(() => {
-      });
-      throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Maximum watcher limit (${MAX_WATCHERS}) reached`, {
-        currentCount: watchers.size
-      });
-    }
-    watchers.add(watcher);
-  }
-  function removeWatcher(watcher) {
-    watcher.cancel().catch(() => {
-    });
-    watchers.delete(watcher);
-  }
-  const coordinator = {
-    async registerNode(registration) {
-      assertNotShutdown();
-      validateNodeId(registration.nodeId);
-      const ttl = resolvedConfig.nodeHeartbeatTtlSeconds;
-      const lease = client.lease(ttl);
-      await lease.grant();
-      const key = keys.node(registration.nodeId);
-      const data = Buffer.from(serializeNodeRegistration(registration));
-      await lease.put(key).value(data).exec();
-      const existing = leaseManager.get(key);
-      if (existing !== void 0) {
-        await existing.lease.revoke().catch(() => {
-        });
-      }
-      leaseManager.track(key, lease, registration.nodeId);
-    },
-    async deregisterNode(nodeId) {
-      assertNotShutdown();
-      validateNodeId(nodeId);
-      const key = keys.node(nodeId);
-      const entry = leaseManager.remove(key);
-      if (entry !== void 0) {
-        await entry.lease.revoke().catch(() => {
-        });
-      }
-      await client.delete().key(key).exec();
-    },
-    async listNodes() {
-      assertNotShutdown();
-      const prefix = keys.nodePrefix();
-      const result = await client.getAll().prefix(prefix).buffers();
-      const nodes = [];
-      for (const key of Object.keys(result)) {
-        const buf = result[key];
-        if (buf !== void 0) {
-          nodes.push(deserializeNodeRegistration(buf));
-        }
-      }
-      return nodes;
-    },
-    async watchNodes(handler) {
-      assertNotShutdown();
-      const prefix = keys.nodePrefix();
-      const watcher = await client.watch().prefix(prefix).create();
-      addWatcher(watcher);
-      watcher.on("put", (kv) => {
-        try {
-          const registration = deserializeNodeRegistration(kv.value);
-          handler({ type: "node_joined", nodeId: registration.nodeId, registration });
-        } catch (_) {
-        }
-      });
-      watcher.on("delete", (kv) => {
-        const fullKey = kv.key.toString();
-        const nodeId = extractSuffix(fullKey, prefix);
-        handler({ type: "node_left", nodeId, registration: null });
-      });
-      return () => removeWatcher(watcher);
-    },
-    async getAllocation(indexName) {
-      assertNotShutdown();
-      const key = keys.allocation(indexName);
-      const buf = await client.get(key).buffer();
-      if (buf === null) {
-        return null;
-      }
-      return deserializeAllocationTable(buf);
-    },
-    async putAllocation(indexName, table, expectedVersion) {
-      assertNotShutdown();
-      const key = keys.allocation(indexName);
-      const data = Buffer.from(serializeAllocationTable(table));
-      if (expectedVersion === void 0) {
-        await client.put(key).value(data).exec();
-        return true;
-      }
-      if (expectedVersion === null) {
-        const txn2 = await client.if(key, "Create", "==", 0).then(client.put(key).value(data)).commit();
-        return txn2.succeeded;
-      }
-      const response = await client.get(key).exec();
-      if (response.kvs.length === 0) {
-        return false;
-      }
-      const kv = response.kvs[0];
-      const currentTable = deserializeAllocationTable(Buffer.from(kv.value));
-      if (currentTable.version !== expectedVersion) {
-        return false;
-      }
-      const txn = await client.if(key, "Mod", "==", kv.mod_revision).then(client.put(key).value(data)).commit();
-      return txn.succeeded;
-    },
-    async watchAllocation(handler) {
-      assertNotShutdown();
-      const prefix = keys.allocationPrefix();
-      const watcher = await client.watch().prefix(prefix).create();
-      addWatcher(watcher);
-      watcher.on("put", (kv) => {
-        try {
-          const table = deserializeAllocationTable(kv.value);
-          handler({ indexName: table.indexName, table });
-        } catch (_) {
-        }
-      });
-      return () => removeWatcher(watcher);
-    },
-    async getPartitionState(indexName, partitionId) {
-      assertNotShutdown();
-      const key = keys.partition(indexName, partitionId);
-      const value = await client.get(key).string();
-      if (value === null) {
-        return "UNASSIGNED";
-      }
-      return validatePartitionState(value);
-    },
-    async putPartitionState(indexName, partitionId, state) {
-      assertNotShutdown();
-      const key = keys.partition(indexName, partitionId);
-      await client.put(key).value(state).exec();
-    },
-    async acquireLease(key, nodeId, ttlMs) {
-      assertNotShutdown();
-      const etcdKey = keys.lease(key);
-      const ttlSeconds = Math.max(1, Math.ceil(ttlMs / 1e3));
-      const existing = leaseManager.get(etcdKey);
-      if (existing !== void 0 && existing.nodeId === nodeId) {
-        try {
-          await existing.lease.keepaliveOnce();
-          return true;
-        } catch (_) {
-          leaseManager.remove(etcdKey);
-        }
-      }
-      const lease = client.lease(ttlSeconds);
-      await lease.grant();
-      const txn = await client.if(etcdKey, "Create", "==", 0).then(lease.put(etcdKey).value(nodeId)).commit();
-      if (!txn.succeeded) {
-        await lease.revoke().catch(() => {
-        });
-        const currentValue = await client.get(etcdKey).string();
-        return currentValue === nodeId;
-      }
-      leaseManager.track(etcdKey, lease, nodeId);
-      return true;
-    },
-    async renewLease(key, nodeId, _ttlMs) {
-      assertNotShutdown();
-      const etcdKey = keys.lease(key);
-      const entry = leaseManager.getByNodeId(etcdKey, nodeId);
-      if (entry === void 0) {
-        return false;
-      }
-      try {
-        await entry.lease.keepaliveOnce();
-        return true;
-      } catch (_) {
-        leaseManager.remove(etcdKey);
-        return false;
-      }
-    },
-    async releaseLease(key) {
-      assertNotShutdown();
-      const etcdKey = keys.lease(key);
-      const entry = leaseManager.remove(etcdKey);
-      if (entry !== void 0) {
-        await entry.lease.revoke().catch(() => {
-        });
-      }
-      await client.delete().key(etcdKey).exec();
-    },
-    async get(key) {
-      assertNotShutdown();
-      const etcdKey = keys.generic(key);
-      const buf = await client.get(etcdKey).buffer();
-      if (buf === null) {
-        return null;
-      }
-      return new Uint8Array(buf);
-    },
-    async compareAndSet(key, expected, value) {
-      assertNotShutdown();
-      const etcdKey = keys.generic(key);
-      const newValue = Buffer.from(value);
-      if (expected === null) {
-        const txn2 = await client.if(etcdKey, "Create", "==", 0).then(client.put(etcdKey).value(newValue)).commit();
-        return txn2.succeeded;
-      }
-      const expectedBuf = Buffer.from(expected);
-      const txn = await client.if(etcdKey, "Value", "==", expectedBuf).then(client.put(etcdKey).value(newValue)).commit();
-      return txn.succeeded;
-    },
-    async getSchema(indexName) {
-      assertNotShutdown();
-      const key = keys.schema(indexName);
-      const buf = await client.get(key).buffer();
-      if (buf === null) {
-        return null;
-      }
-      return deserializeSchema(buf);
-    },
-    async putSchema(indexName, schema) {
-      assertNotShutdown();
-      const key = keys.schema(indexName);
-      await client.put(key).value(serializeSchema(schema)).exec();
-    },
-    async watchSchemas(handler) {
-      assertNotShutdown();
-      const prefix = keys.schemaPrefix();
-      const watcher = await client.watch().prefix(prefix).create();
-      addWatcher(watcher);
-      watcher.on("put", (kv) => {
-        try {
-          const schema = deserializeSchema(kv.value);
-          const fullKey = kv.key.toString();
-          const indexName = extractSuffix(fullKey, prefix);
-          handler({ type: "schema_created", indexName, schema });
-        } catch (_) {
-        }
-      });
-      watcher.on("delete", (kv) => {
-        const fullKey = kv.key.toString();
-        const indexName = extractSuffix(fullKey, prefix);
-        handler({ type: "schema_dropped", indexName, schema: null });
-      });
-      return () => removeWatcher(watcher);
-    },
-    async getLeaseHolder(key) {
-      assertNotShutdown();
-      const etcdKey = keys.lease(key);
-      const value = await client.get(etcdKey).string();
-      return value ?? null;
-    },
-    async shutdown() {
-      if (isShutdown) {
-        return;
-      }
-      isShutdown = true;
-      const cancelPromises = [];
-      for (const watcher of watchers) {
-        cancelPromises.push(watcher.cancel().catch(() => {
-        }));
-      }
-      await Promise.all(cancelPromises);
-      watchers.clear();
-      await leaseManager.revokeAll();
-      client.close();
-    }
-  };
-  return coordinator;
-}
-
-export { createEtcdCoordinator, validateNodeId };