@delali/narsil 0.2.3-next.32 → 0.2.3-next.34

package/dist/{chunk-C6VVLMRX.mjs → chunk-36ER4WCG.mjs}

@@ -35,6 +35,7 @@ var ReplicationMessageTypes = {
   SNAPSHOT_CHUNK: "replication.snapshot_chunk",
   SNAPSHOT_END: "replication.snapshot_end",
   SNAPSHOT_SYNC_REQUEST: "replication.snapshot_sync_request",
+  INSYNC_ADD: "replication.insync_add",
   INSYNC_REMOVE: "replication.insync_remove",
   INSYNC_CONFIRM: "replication.insync_confirm"
 };

package/dist/{chunk-TEAJO7CU.mjs → chunk-GN7KF675.mjs}

@@ -1,4 +1,4 @@
-import { ReplicationMessageTypes } from './chunk-C6VVLMRX.mjs';
+import { ReplicationMessageTypes } from './chunk-36ER4WCG.mjs';
 import { getNestedValue, getLanguage, extractVectorFieldsFromSchema, createDirectExecutor, createPartitionRouter, clampRowCount, DEFAULT_PAGE_SIZE, validateSchema, validateVectorPromotion, collectQueryTermStats, createPartitionManager, createVectorIndex, flattenSchema, isTextFieldType, requireWithinResultWindow, normalizeSort, sortSignatureOf, requireSortableFields, fanOutQuery, applySorting, applyPagination, resolveProjection, tokenize, decodePageCursor, requireMatchingCursor, encodePageCursor, resolveIndexAnalysis, deserializePayloadV2, deserializePayloadV1, validateRequiredFields, mergePartitionStats, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, createPartitionIndex, fnv1a, createWorkerPool, validateDocument, validateDocumentStrict, createFrozenSegment, detectRuntime, serializeMetadata, isNodeMainThread, spawnNodeWorker, createSharedFrozenSegment, buildCompactedSegmentPayload, freezeSegmentShared, deserializeMetadata, isCompositePartition, kWayMerge } from './chunk-RO4EFWE6.mjs';
 import { crc32, crc32Init, crc32Update, crc32Final } from './chunk-KMMJMDUA.mjs';
 import { compareCodePoints, compareComparableKeys } from './chunk-BFIH7OV5.mjs';
@@ -2565,6 +2565,14 @@ function createForwardBatchResultMessage(payload, sourceId, requestId) {
     payload: encode(payload)
   };
 }
+function createInsyncAddMessage(payload, sourceId) {
+  return {
+    type: ReplicationMessageTypes.INSYNC_ADD,
+    sourceId,
+    requestId: generateId(),
+    payload: encode(payload)
+  };
+}
 function createInsyncRemoveMessage(payload, sourceId) {
   return {
     type: ReplicationMessageTypes.INSYNC_REMOVE,
@@ -9321,4 +9329,4 @@ async function createNarsil(config) {
 // src/index.ts
 var VERSION = "0.1.0";
 
-export { DEFAULT_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, VERSION, WIRE_BATCH_BUDGET, applyDeleteEntry, applyIndexEntry, chunkByBudget, clampLimit, computeEntryChecksum, createAckMessage, createDurableDirectory, createEngineCore, createEngineIndex, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createNarsil, createNarsilFromCore, getIndexMetadata, indexConfigKey, now, putIndexMetadata, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateIndexName, validateInsyncConfirmPayload, validateReplicationEntry, writeSegmentedCheckpoint };
+export { DEFAULT_SUGGEST_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_PARTITION_COUNT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, VERSION, WIRE_BATCH_BUDGET, applyDeleteEntry, applyIndexEntry, chunkByBudget, clampLimit, computeEntryChecksum, createAckMessage, createDurableDirectory, createEngineCore, createEngineIndex, createEntryBatchMessage, createEntryMessage, createForwardBatchMessage, createForwardBatchResultMessage, createForwardMessage, createInsyncAddMessage, createInsyncConfirmMessage, createInsyncRemoveMessage, createNarsil, createNarsilFromCore, getIndexMetadata, indexConfigKey, now, putIndexMetadata, runEngineListDocuments, runEnginePreflight, runEngineQuery, runEngineQueryStats, runEngineSuggest, validateAckPayload, validateEntryBatchPayload, validateEntryPayload, validateForwardBatchPayload, validateForwardBatchResultPayload, validateIndexName, validateInsyncConfirmPayload, validateReplicationEntry, writeSegmentedCheckpoint };

package/dist/{chunk-MFFAJ2NR.mjs → chunk-GODECMDT.mjs}

@@ -1,4 +1,4 @@
-import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from './chunk-C6VVLMRX.mjs';
+import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from './chunk-36ER4WCG.mjs';
 import { encode, decode } from '@msgpack/msgpack';
 
 // src/distribution/transport/tcp/types.ts

package/dist/distribution/coordinator/etcd.d.ts

@@ -1,4 +1,4 @@
-import { C as ClusterCoordinator } from '../../types-Bfhhnh3G.js';
+import { C as ClusterCoordinator } from '../../types-BhdINMtE.js';
 import '../../schema-emTQfEiX.js';
 import '../../adapters-79OIa4xm.js';
 

package/dist/distribution/coordinator/etcd.mjs

@@ -39,10 +39,22 @@ function extractSuffix(fullKey, prefix) {
 }
 
 // src/distribution/coordinator/etcd/leases.ts
+function payloadsMatch(left, right) {
+  if (left === null || left.byteLength !== right.byteLength) {
+    return false;
+  }
+  for (let index = 0; index < left.byteLength; index++) {
+    if (left[index] !== right[index]) {
+      return false;
+    }
+  }
+  return true;
+}
 var LeaseManager = class {
   leases = /* @__PURE__ */ new Map();
-  track(key, lease, nodeId) {
-    this.leases.set(key, { lease, nodeId, key });
+  keyLocks = /* @__PURE__ */ new Map();
+  track(key, lease, nodeId, payload = null) {
+    this.leases.set(key, { lease, nodeId, key, payload });
   }
   get(key) {
     return this.leases.get(key);
@@ -54,6 +66,10 @@ var LeaseManager = class {
     }
     return entry;
   }
+  holdsPayload(key, payload) {
+    const entry = this.leases.get(key);
+    return entry !== void 0 && payloadsMatch(entry.payload, payload);
+  }
   remove(key) {
     const entry = this.leases.get(key);
     if (entry !== void 0) {
@@ -61,6 +77,23 @@ var LeaseManager = class {
     }
     return entry;
   }
+  runExclusively(key, work) {
+    const previous = this.keyLocks.get(key) ?? Promise.resolve();
+    const result = previous.then(work, work);
+    const settled = result.then(
+      () => {
+      },
+      () => {
+      }
+    );
+    this.keyLocks.set(key, settled);
+    void settled.then(() => {
+      if (this.keyLocks.get(key) === settled) {
+        this.keyLocks.delete(key);
+      }
+    });
+    return result;
+  }
   async revokeAll() {
     const revokePromises = [];
     for (const [, entry] of this.leases) {
@@ -113,9 +146,26 @@ function deserializeAllocationTable(data) {
     indexName: raw.indexName,
     version: raw.version,
     replicationFactor: typeof raw.replicationFactor === "number" ? raw.replicationFactor : 0,
-    assignments: new Map(raw.assignments)
+    assignments: normaliseAssignments(raw.assignments)
   };
 }
+function normaliseAssignments(entries) {
+  const assignments = /* @__PURE__ */ new Map();
+  for (const pair of entries) {
+    if (!Array.isArray(pair) || pair.length < 2) {
+      throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable holds a malformed assignment entry");
+    }
+    const [partitionId, assignment] = pair;
+    if (typeof partitionId !== "number" || assignment === null || typeof assignment !== "object") {
+      throw new NarsilError(ErrorCodes.CONFIG_INVALID, "AllocationTable holds a malformed partition assignment");
+    }
+    const candidate = assignment;
+    const storedCommitPoint = candidate.commitPoint;
+    const commitPoint = typeof storedCommitPoint === "number" && Number.isSafeInteger(storedCommitPoint) && storedCommitPoint >= 0 ? storedCommitPoint : 0;
+    assignments.set(partitionId, { ...assignment, commitPoint });
+  }
+  return assignments;
+}
 function serializeNodeRegistration(reg) {
   return new Uint8Array(encode(reg));
 }
@@ -136,6 +186,38 @@ function deserializeSchema(data) {
   return decode(data);
 }
 
+// src/distribution/coordinator/etcd/registration.ts
+function writeNodeRegistration(context, registration) {
+  return context.leaseManager.runExclusively(context.key, () => writeUnderLease(context, registration));
+}
+async function writeUnderLease(context, registration) {
+  const payload = serializeNodeRegistration(registration);
+  const data = Buffer.from(payload);
+  const tracked = context.leaseManager.getByNodeId(context.key, registration.nodeId);
+  if (tracked !== void 0) {
+    try {
+      await tracked.lease.keepaliveOnce();
+      if (context.leaseManager.holdsPayload(context.key, payload)) {
+        return;
+      }
+      await tracked.lease.put(context.key).value(data).exec();
+      context.leaseManager.track(context.key, tracked.lease, registration.nodeId, payload);
+      return;
+    } catch (_) {
+      context.leaseManager.remove(context.key);
+    }
+  }
+  const lease = context.client.lease(context.ttlSeconds);
+  await lease.grant();
+  await lease.put(context.key).value(data).exec();
+  const existing = context.leaseManager.get(context.key);
+  if (existing !== void 0) {
+    await existing.lease.revoke().catch(() => {
+    });
+  }
+  context.leaseManager.track(context.key, lease, registration.nodeId, payload);
+}
+
 // src/distribution/coordinator/etcd/validation.ts
 var MAX_NODE_ID_LENGTH = 255;
 var VALID_PARTITION_STATES = /* @__PURE__ */ new Set(["UNASSIGNED", "INITIALISING", "ACTIVE", "MIGRATING", "DECOMMISSIONING"]);
@@ -206,18 +288,15 @@ async function createEtcdCoordinator(config) {
     async registerNode(registration) {
       assertNotShutdown();
       validateNodeId(registration.nodeId);
-      const ttl = resolvedConfig.nodeHeartbeatTtlSeconds;
-      const lease = client.lease(ttl);
-      await lease.grant();
-      const key = keys.node(registration.nodeId);
-      const data = Buffer.from(serializeNodeRegistration(registration));
-      await lease.put(key).value(data).exec();
-      const existing = leaseManager.get(key);
-      if (existing !== void 0) {
-        await existing.lease.revoke().catch(() => {
-        });
-      }
-      leaseManager.track(key, lease, registration.nodeId);
+      await writeNodeRegistration(
+        {
+          client,
+          leaseManager,
+          key: keys.node(registration.nodeId),
+          ttlSeconds: resolvedConfig.nodeHeartbeatTtlSeconds
+        },
+        registration
+      );
     },
     async deregisterNode(nodeId) {
       assertNotShutdown();

package/dist/distribution/coordinator/in-memory.d.ts

@@ -1,4 +1,4 @@
-import { C as ClusterCoordinator } from '../../types-Bfhhnh3G.js';
+import { C as ClusterCoordinator } from '../../types-BhdINMtE.js';
 import '../../schema-emTQfEiX.js';
 import '../../adapters-79OIa4xm.js';
 

package/dist/distribution/coordinator/in-memory.mjs

@@ -2,6 +2,7 @@ import { compareCodePoints } from '../../chunk-BFIH7OV5.mjs';
 
 // src/distribution/coordinator/in-memory.ts
 var NODE_HEARTBEAT_PREFIX = "_narsil/node/";
+var NODE_HEARTBEAT_TTL_MS = 3e4;
 function bytesEqual(a, b) {
   if (a.byteLength !== b.byteLength) {
     return false;
@@ -80,16 +81,19 @@ function createInMemoryCoordinator() {
   const coordinator = {
     async registerNode(registration) {
       assertNotShutdown();
+      const previous = nodes.get(registration.nodeId);
       nodes.set(registration.nodeId, registration);
       const heartbeatKey = `${NODE_HEARTBEAT_PREFIX}${registration.nodeId}`;
-      const defaultTtlMs = 3e4;
       clearLeaseEntry(heartbeatKey);
-      const timer = createLeaseTimer(heartbeatKey, defaultTtlMs);
+      const timer = createLeaseTimer(heartbeatKey, NODE_HEARTBEAT_TTL_MS);
       leases.set(heartbeatKey, {
         nodeId: registration.nodeId,
-        expiresAt: Date.now() + defaultTtlMs,
+        expiresAt: Date.now() + NODE_HEARTBEAT_TTL_MS,
         timer
       });
+      if (previous !== void 0 && registrationsMatch(previous, registration)) {
+        return;
+      }
       emitNodeEvent({ type: "node_joined", nodeId: registration.nodeId, registration });
     },
     async deregisterNode(nodeId) {
@@ -282,5 +286,8 @@ function createInMemoryCoordinator() {
   };
   return coordinator;
 }
+function registrationsMatch(left, right) {
+  return JSON.stringify(left) === JSON.stringify(right);
+}
 
 export { createInMemoryCoordinator };

package/dist/distribution/transport/grpc.mjs

@@ -1,5 +1,5 @@
-import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-MFFAJ2NR.mjs';
-import { TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-C6VVLMRX.mjs';
+import { decodeTransportMessage, encodeTransportMessage } from '../../chunk-GODECMDT.mjs';
+import { TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-36ER4WCG.mjs';
 import { NarsilError, ErrorCodes } from '../../chunk-EMI63EXP.mjs';
 
 // src/distribution/transport/grpc/codec.ts

package/dist/distribution/transport/in-memory.mjs

@@ -1,4 +1,4 @@
-import { DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-C6VVLMRX.mjs';
+import { DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES } from '../../chunk-36ER4WCG.mjs';
 
 // src/distribution/transport/in-memory.ts
 function createInMemoryNetwork() {

package/dist/distribution/transport/tcp.mjs

@@ -1,5 +1,5 @@
-import { DEFAULT_TCP_CONFIG, encodeTransportMessage, encodeFrame, FRAME_TYPE_REQUEST, FrameParser, FRAME_TYPE_RESPONSE, decodeTransportMessage, FRAME_TYPE_STREAM_CHUNK, FRAME_TYPE_STREAM_END } from '../../chunk-MFFAJ2NR.mjs';
-import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from '../../chunk-C6VVLMRX.mjs';
+import { DEFAULT_TCP_CONFIG, encodeTransportMessage, encodeFrame, FRAME_TYPE_REQUEST, FrameParser, FRAME_TYPE_RESPONSE, decodeTransportMessage, FRAME_TYPE_STREAM_CHUNK, FRAME_TYPE_STREAM_END } from '../../chunk-GODECMDT.mjs';
+import { MAX_MESSAGE_SIZE_BYTES, TransportError, TransportErrorCodes } from '../../chunk-36ER4WCG.mjs';
 import { connect as connect$1, createServer as createServer$1 } from 'net';
 import { connect, createServer } from 'tls';
 

package/dist/distribution.d.ts

@@ -1,6 +1,6 @@
 import { N as NarsilConfig } from './config-ChZ7lfqc.js';
-import { A as AllocationTable, N as NodeRole, C as ClusterCoordinator, a as NodeCapacity } from './types-Bfhhnh3G.js';
-export { b as AllocationConstraints, c as AllocationEvent, d as NodeEvent, e as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-Bfhhnh3G.js';
+import { A as AllocationTable, N as NodeRole, C as ClusterCoordinator, a as NodeCapacity } from './types-BhdINMtE.js';
+export { b as AllocationConstraints, c as AllocationEvent, d as NodeEvent, e as NodeRegistration, P as PartitionAssignment, f as PartitionState, S as SchemaEvent } from './types-BhdINMtE.js';
 import { N as NodeTransport } from './types-D5OwhtsQ.js';
 export { L as ListenHandler, R as RespondFn, T as TransportConfig, a as TransportMessage } from './types-D5OwhtsQ.js';
 import { a as NarsilEventMap, N as Narsil } from './engine-DScGlT2Z.js';