@delali/narsil 0.2.0 → 0.2.1-next.25

package/dist/types-BV3nAKTL.d.ts

@@ -0,0 +1,302 @@
+import { b as SchemaDefinition } from './schema-CE6S_aAr.js';
+
+/**
+ * What a node does in a cluster. A node may hold several roles at once.
+ *
+ * A `data` node holds partitions and answers queries, a `coordinator` node
+ * routes requests without holding data, and a `controller` node runs the
+ * allocation decisions for the whole cluster.
+ *
+ * @public
+ */
+type NodeRole = 'data' | 'coordinator' | 'controller';
+/**
+ * Where one partition stands in its move between nodes.
+ *
+ * `ACTIVE` is the steady state. `INITIALISING` means a node is still filling
+ * it, `MIGRATING` means it is moving, `DECOMMISSIONING` means it is being
+ * drained, and `UNASSIGNED` means no node holds it.
+ *
+ * @public
+ */
+type PartitionState = 'UNASSIGNED' | 'INITIALISING' | 'ACTIVE' | 'MIGRATING' | 'DECOMMISSIONING';
+/**
+ * What one node has to offer, which the allocator weighs when it places
+ * partitions.
+ *
+ * @public
+ */
+interface NodeCapacity {
+    /** The node can give this much memory to indexes. */
+    memoryBytes: number;
+    /** The node has this many cores available. */
+    cpuCores: number;
+    /** The node can give this much disk to durability, and reports `null` when it keeps nothing on disk. */
+    diskBytes: number | null;
+}
+/**
+ * What a node publishes about itself when it joins a cluster.
+ *
+ * @public
+ */
+interface NodeRegistration {
+    /** This identifies the node across the cluster, and must stay stable across restarts. */
+    nodeId: string;
+    /** Peers reach the node at this address, as the transport writes it. */
+    address: string;
+    /** The node holds these roles in the cluster. */
+    roles: NodeRole[];
+    /** The node offers this much capacity. */
+    capacity: NodeCapacity;
+    /** The node started at this ISO 8601 timestamp. */
+    startedAt: string;
+    /** The node runs this engine version, which is what a mixed-version cluster reads. */
+    version: string;
+    /** These free-form labels are where a zone or rack goes for zone-aware allocation. */
+    metadata?: Record<string, string>;
+}
+/**
+ * Which nodes hold one partition, and where that partition stands.
+ *
+ * @public
+ */
+interface PartitionAssignment {
+    /** This node accepts writes, and it is `null` while no node holds the partition. */
+    primary: string | null;
+    /** These nodes each hold a copy, including the primary. */
+    replicas: string[];
+    /** These replicas are current enough to be promoted, which is the set a failover picks from. */
+    inSyncSet: string[];
+    /** This says where the partition stands in its move between nodes. */
+    state: PartitionState;
+    /** This rises each time the primary changes, which is what makes a stale primary's writes detectable. */
+    primaryTerm: number;
+}
+/**
+ * How one index's partitions are spread across the cluster.
+ *
+ * @public
+ */
+interface AllocationTable {
+    /** The table covers this index. */
+    indexName: string;
+    /** This rises on every change, so a writer detects that another controller got there first. */
+    version: number;
+    /** The cluster keeps this many copies of each partition. */
+    replicationFactor: number;
+    /** These assignments are keyed by partition id. */
+    assignments: Map<number, PartitionAssignment>;
+}
+/**
+ * Tells watchers that a node joined or left.
+ *
+ * @public
+ */
+interface NodeEvent {
+    /** This says which change it is. */
+    type: 'node_joined' | 'node_left';
+    /** This node joined or left. */
+    nodeId: string;
+    /** The node published this on joining, and it is `null` when the node left. */
+    registration: NodeRegistration | null;
+}
+/**
+ * Tells watchers that an index's allocation changed.
+ *
+ * @public
+ */
+interface AllocationEvent {
+    /** This index's allocation changed. */
+    indexName: string;
+    /** The allocation now stands like this. */
+    table: AllocationTable;
+}
+/**
+ * Tells watchers that an index's schema was published or dropped.
+ *
+ * @public
+ */
+interface SchemaEvent {
+    /** This says which change it is. */
+    type: 'schema_created' | 'schema_dropped';
+    /** This index's schema changed. */
+    indexName: string;
+    /** This is the schema, and it is `null` when the index was dropped. */
+    schema: SchemaDefinition | null;
+}
+/**
+ * The rules the allocator respects when it places partitions.
+ *
+ * The controller stores these with each index's metadata, so every node
+ * allocating that index applies the same rules.
+ *
+ * @public
+ */
+interface AllocationConstraints {
+    /** Setting this spreads an index's replicas across zones, so losing one zone never costs every copy. */
+    zoneAwareness: boolean;
+    /** This names the key in a node's `metadata` that carries its zone. */
+    zoneAttribute: string;
+    /** One node may hold this many partitions of an index, and `null` lifts the ceiling. */
+    maxShardsPerNode: number | null;
+}
+/**
+ * The shared state every cluster node reads and writes: which nodes exist,
+ * which node holds each partition, who holds which lease, and what each
+ * index's schema is.
+ *
+ * The package includes an etcd-backed coordinator for real deployments and an
+ * in-memory one for tests, and this is the contract either satisfies. Write
+ * your own to run a cluster on another store, so long as
+ * {@link ClusterCoordinator.compareAndSet} and the lease calls are atomic,
+ * because the controller election depends on them.
+ *
+ * @public
+ */
+interface ClusterCoordinator {
+    /**
+     * Publishes a node so that peers find it, and keeps it published while the
+     * node heartbeats.
+     *
+     * @param registration - What the node publishes about itself.
+     */
+    registerNode(registration: NodeRegistration): Promise<void>;
+    /**
+     * Removes a node, so the controller reallocates whatever it held.
+     *
+     * @param nodeId - The node to remove.
+     */
+    deregisterNode(nodeId: string): Promise<void>;
+    /**
+     * Lists the nodes currently published.
+     *
+     * @returns One registration per live node.
+     */
+    listNodes(): Promise<NodeRegistration[]>;
+    /**
+     * Watches nodes joining and leaving.
+     *
+     * @param handler - Called once per change.
+     * @returns A function that ends the watch.
+     */
+    watchNodes(handler: (event: NodeEvent) => void): Promise<() => void>;
+    /**
+     * Reads one index's allocation.
+     *
+     * @param indexName - The index to read.
+     * @returns Its allocation, or `null` when the index has none yet.
+     */
+    getAllocation(indexName: string): Promise<AllocationTable | null>;
+    /**
+     * Writes one index's allocation, optionally only while nobody else has
+     * changed it first.
+     *
+     * @param indexName - The index to write.
+     * @param table - The allocation to store.
+     * @param expectedVersion - The version the caller read, `null` to write only
+     * while no allocation exists, or omit it to write unconditionally.
+     * @returns True when the write went through, and false when the expected
+     * version no longer matched.
+     */
+    putAllocation(indexName: string, table: AllocationTable, expectedVersion?: number | null): Promise<boolean>;
+    /**
+     * Watches allocations changing.
+     *
+     * @param handler - Called once per change.
+     * @returns A function that ends the watch.
+     */
+    watchAllocation(handler: (event: AllocationEvent) => void): Promise<() => void>;
+    /**
+     * Reads where one partition stands.
+     *
+     * @param indexName - Index the partition belongs to.
+     * @param partitionId - The partition to read.
+     * @returns Its state, or `UNASSIGNED` when nothing is recorded.
+     */
+    getPartitionState(indexName: string, partitionId: number): Promise<PartitionState>;
+    /**
+     * Records where one partition stands.
+     *
+     * @param indexName - Index the partition belongs to.
+     * @param partitionId - The partition to record.
+     * @param state - Its new state.
+     */
+    putPartitionState(indexName: string, partitionId: number, state: PartitionState): Promise<void>;
+    /**
+     * Takes a lease, which is how one node wins the controller election.
+     *
+     * @param key - The lease to take.
+     * @param nodeId - The node taking it.
+     * @param ttlMs - Milliseconds the lease survives without renewal.
+     * @returns True when this node now holds the lease.
+     */
+    acquireLease(key: string, nodeId: string, ttlMs: number): Promise<boolean>;
+    /**
+     * Extends a lease this node already holds.
+     *
+     * @param key - The lease to extend.
+     * @param nodeId - The node that holds it.
+     * @param ttlMs - Milliseconds the lease survives from now.
+     * @returns True when the lease was extended, and false once it has been
+     * lost, which means the node has to step down.
+     */
+    renewLease(key: string, nodeId: string, ttlMs: number): Promise<boolean>;
+    /**
+     * Gives a lease up straight away, rather than waiting for it to expire.
+     *
+     * @param key - The lease to release.
+     */
+    releaseLease(key: string): Promise<void>;
+    /**
+     * Reads one key of cluster metadata.
+     *
+     * @param key - The key to read.
+     * @returns Its bytes, or `null` when the key holds nothing.
+     */
+    get(key: string): Promise<Uint8Array | null>;
+    /**
+     * Writes one key of cluster metadata only while it still holds what the
+     * caller read, which is what makes concurrent controllers safe.
+     *
+     * @param key - The key to write.
+     * @param expected - The bytes the caller read, or `null` to write only while
+     * the key holds nothing.
+     * @param value - The bytes to store.
+     * @returns True when the write went through.
+     */
+    compareAndSet(key: string, expected: Uint8Array | null, value: Uint8Array): Promise<boolean>;
+    /**
+     * Reads one index's schema, which is how a node joining later learns the
+     * layout of an index it has never held.
+     *
+     * @param indexName - The index to read.
+     * @returns Its schema, or `null` when the index is unknown.
+     */
+    getSchema(indexName: string): Promise<SchemaDefinition | null>;
+    /**
+     * Publishes one index's schema to the cluster.
+     *
+     * @param indexName - The index to publish.
+     * @param schema - Its field layout.
+     */
+    putSchema(indexName: string, schema: SchemaDefinition): Promise<void>;
+    /**
+     * Watches schemas being published and dropped.
+     *
+     * @param handler - Called once per change.
+     * @returns A function that ends the watch.
+     */
+    watchSchemas(handler: (event: SchemaEvent) => void): Promise<() => void>;
+    /**
+     * Reads which node holds a lease, which is how a node finds the active
+     * controller.
+     *
+     * @param key - The lease to look up.
+     * @returns The holder's node id, or `null` when nobody holds it.
+     */
+    getLeaseHolder(key: string): Promise<string | null>;
+    /** Releases every lease and watch this node opened, then closes the connection. */
+    shutdown(): Promise<void>;
+}
+
+export type { AllocationTable as A, ClusterCoordinator as C, NodeRole as N, PartitionAssignment as P, SchemaEvent as S, NodeCapacity as a, AllocationConstraints as b, AllocationEvent as c, NodeEvent as d, NodeRegistration as e, PartitionState as f };

package/dist/vector/hnsw-build-worker.d.ts

@@ -1,44 +1,110 @@
+/**
+ * Similarity metric a vector search ranks by.
+ *
+ * @internal
+ */
 type VectorMetric = 'cosine' | 'dotProduct' | 'euclidean';
 
+/**
+ * How an HNSW graph is built.
+ *
+ * @internal
+ */
 interface HNSWConfig {
+    /** Each node keeps this many neighbours per layer. */
     m?: number;
+    /** The builder explores this many candidates while placing each node. */
     efConstruction?: number;
+    /** The graph ranks by this metric. */
     metric?: VectorMetric;
 }
+/**
+ * An HNSW graph in the form the engine passes between threads.
+ *
+ * @internal
+ */
 interface SerializedHNSWGraph {
+    /** Every search starts at this node, and it is `null` while the graph is empty. */
     entryPoint: string | null;
+    /** The graph reaches this many layers. */
     maxLayer: number;
+    /** Each node keeps this many neighbours per layer. */
     m: number;
+    /** The builder explored this many candidates while placing each node. */
     efConstruction: number;
+    /** The graph ranks by this metric. */
     metric?: VectorMetric;
+    /** Each entry holds a document id, its top layer, and its neighbours per layer. */
     nodes: Array<[string, number, Array<[number, string[]]>]>;
 }
 
+/**
+ * Asks the build worker to construct a graph from vectors sent as arrays.
+ *
+ * This module is the worker's own entry point, and the engine posts these
+ * messages to it. It exists so that a bundler can resolve the worker file, and
+ * nothing here is part of the engine's supported surface.
+ *
+ * @internal
+ */
 interface HNSWBuildRequest {
+    /** This marks the message as an array-form build request. */
     type: 'build';
+    /** The worker indexes these vectors, each carrying the document id it belongs to. */
     vectors: Array<{
         docId: string;
         values: number[];
     }>;
+    /** Each vector carries this many components. */
     dimension: number;
+    /** The worker builds to this graph shape, clamped to safe bounds first. */
     config: HNSWConfig;
 }
+/**
+ * Asks the build worker to construct a graph from vectors packed into one
+ * transferable buffer, which is what the engine sends for a large field.
+ *
+ * @internal
+ */
 interface HNSWBuildRequestBinary {
+    /** This marks the message as a packed-buffer build request. */
     type: 'build-binary';
+    /** These document ids run in the order their vectors appear in the buffer. */
     docIds: string[];
+    /** This holds every vector end to end, `docIds.length * dimension` components long. */
     vectorData: Float32Array;
+    /** Each vector carries this many components. */
     dimension: number;
+    /** The worker builds to this graph shape, clamped to safe bounds first. */
     config: HNSWConfig;
 }
+/**
+ * What the build worker returns once a graph is built.
+ *
+ * @internal
+ */
 interface HNSWBuildResponse {
+    /** This marks the message as a completed build. */
     type: 'success';
+    /** This graph is ready for the engine to adopt. */
     graph: SerializedHNSWGraph;
 }
+/**
+ * What the build worker returns when a build fails.
+ *
+ * @internal
+ */
 interface HNSWBuildError {
+    /** This marks the message as a failed build. */
     type: 'error';
+    /** This says why the build failed. */
     message: string;
 }
+/**
+ * Either message the build worker posts back.
+ *
+ * @internal
+ */
 type HNSWWorkerMessage = HNSWBuildResponse | HNSWBuildError;
-declare function handleBuildRequest(request: HNSWBuildRequest): SerializedHNSWGraph;
 
-export { type HNSWBuildError, type HNSWBuildRequest, type HNSWBuildRequestBinary, type HNSWBuildResponse, type HNSWWorkerMessage, handleBuildRequest };
+export type { HNSWBuildError, HNSWBuildRequest, HNSWBuildRequestBinary, HNSWBuildResponse, HNSWConfig, HNSWWorkerMessage, SerializedHNSWGraph, VectorMetric };

package/dist/vector/hnsw-build-worker.mjs

@@ -121,5 +121,3 @@ async function setupAsync() {
   }
 }
 setupWorker();
-
-export { handleBuildRequest };

package/dist/workers/entry.mjs

@@ -15,7 +15,7 @@ async function setup() {
   } catch {
     parentPort = null;
   }
-  const { createDirectExecutor } = await import('../direct-executor-2FV3MEDD.mjs');
+  const { createDirectExecutor } = await import('../direct-executor-3W4QRA4E.mjs');
   const executor = createDirectExecutor();
   function buildErrorResponse(requestId, code, message) {
     return { type: "error", requestId, code, message };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@delali/narsil",
-  "version": "0.2.0",
+  "version": "0.2.1-next.25",
   "type": "module",
   "description": "A distributed full-text search engine with cross-language portable binary format.",
   "author": "Delali (https://sondelali.com)",
@@ -117,6 +117,7 @@
   },
   "devDependencies": {
     "@biomejs/biome": "2.4.4",
+    "@microsoft/api-extractor": "7.58.12",
     "@types/node": "22.15.32",
     "@vitest/coverage-v8": "4.0.18",
     "etcd3": "1.1.2",
@@ -143,7 +144,7 @@
     }
   },
   "scripts": {
-    "build": "tsup && node scripts/check-browser-bundle.mjs",
+    "build": "tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
     "test": "vitest run --exclude='**/vector/recall.test.ts'",
     "test:etcd": "vitest run src/__tests__/distribution/coordinator/etcd-integration/*.test.ts",
     "test:watch": "vitest",
@@ -154,6 +155,8 @@
     "stemmers": "tsx snowball/build.ts",
     "revisions": "tsx scripts/analysis-revisions.ts",
     "revisions:write": "tsx scripts/analysis-revisions.ts --write",
+    "api": "tsx scripts/api-surface.ts",
+    "api:write": "tsx scripts/api-surface.ts --write",
     "lookalikes": "tsx scripts/lookalike-counts.ts",
     "lookalikes:collect": "tsx scripts/lookalike-counts.ts --collect",
     "lint": "biome check src/",

package/dist/adapters-Bo2JQg8b.d.ts

@@ -1,37 +0,0 @@
-interface PersistenceAdapter {
-    save(key: string, data: Uint8Array): Promise<void>;
-    load(key: string): Promise<Uint8Array | null>;
-    delete(key: string): Promise<void>;
-    list(prefix: string): Promise<string[]>;
-    readonly directory?: string;
-}
-interface InvalidationAdapter {
-    publish(event: InvalidationEvent): Promise<void>;
-    subscribe(handler: (event: InvalidationEvent) => void): Promise<void>;
-    shutdown(): Promise<void>;
-}
-type InvalidationEvent = {
-    type: 'partition';
-    indexName: string;
-    partitions: number[];
-    timestamp: number;
-    sourceInstanceId: string;
-} | {
-    type: 'statistics';
-    indexName: string;
-    instanceId: string;
-    stats: PartitionStatistics;
-};
-interface PartitionStatistics {
-    totalDocs: number;
-    docFrequencies: Record<string, number>;
-    totalFieldLengths: Record<string, number>;
-}
-interface EmbeddingAdapter {
-    embed(input: string, purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array>;
-    embedBatch?(inputs: string[], purpose: 'document' | 'query', signal?: AbortSignal): Promise<Float32Array[]>;
-    readonly dimensions: number;
-    shutdown?(): Promise<void>;
-}
-
-export type { EmbeddingAdapter as E, InvalidationAdapter as I, PartitionStatistics as P, InvalidationEvent as a, PersistenceAdapter as b };