@delali/narsil 0.2.4-next.48 → 0.2.4-next.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/indexeddb.mjs +15 -10
- package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
- package/dist/{chunk-UZTT2BKQ.mjs → chunk-56PJBIJ6.mjs} +891 -542
- package/dist/chunk-CBLS46IE.mjs +10 -0
- package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
- package/dist/{chunk-4COAUBLJ.mjs → chunk-HGJC5L65.mjs} +4 -7
- package/dist/{chunk-DFZDV5FV.mjs → chunk-JXHUHFJ5.mjs} +27 -14
- package/dist/{chunk-XWICBGYT.mjs → chunk-KGRYQVZ5.mjs} +2 -5
- package/dist/{chunk-EV45L35L.mjs → chunk-M6JGMKA5.mjs} +391 -183
- package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
- package/dist/chunk-ORBLD3EL.mjs +6 -0
- package/dist/chunk-OWCFK42Q.mjs +30 -0
- package/dist/{chunk-WTONLDU3.mjs → chunk-RDGXYRF4.mjs} +1 -0
- package/dist/chunk-UNEEFTEU.mjs +7 -0
- package/dist/chunk-YEO6G2NG.mjs +6 -0
- package/dist/{client-C5BkUqB8.d.ts → client-DX7xPcNA.d.ts} +1 -1
- package/dist/client.d.ts +5 -5
- package/dist/client.mjs +224 -222
- package/dist/{config-B-ZI-ft5.d.ts → config-B41W23Us.d.ts} +1 -1
- package/dist/{config-BZRExglL.d.ts → config-Co-zAXqO.d.ts} +27 -11
- package/dist/direct-executor-HRTDWMVB.mjs +9 -0
- package/dist/distribution/coordinator/etcd.mjs +2 -3
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution/transport/grpc.mjs +7 -9
- package/dist/distribution/transport/in-memory.mjs +2 -1
- package/dist/distribution/transport/tcp.mjs +3 -2
- package/dist/distribution.d.ts +5 -5
- package/dist/distribution.mjs +121 -97
- package/dist/embeddings/openai.mjs +5 -3
- package/dist/{engine-Cjr1uBLa.d.ts → engine-CABZg25p.d.ts} +12 -8
- package/dist/index.browser.mjs +1514 -894
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +9 -8
- package/dist/invalidation/broadcast-channel.mjs +2 -1
- package/dist/invalidation/filesystem.mjs +3 -4
- package/dist/{options-BC68VPoE.d.ts → options-Dtk99qUs.d.ts} +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +9 -8
- package/dist/react.d.ts +3 -3
- package/dist/react.mjs +7 -5
- package/dist/{search-DDHjfXhJ.d.ts → search-FjWhZN3f.d.ts} +20 -3
- package/dist/server.d.ts +6 -6
- package/dist/server.mjs +43 -30
- package/dist/vector/hnsw-build-worker.mjs +2 -4
- package/dist/vector/search-worker.mjs +3 -4
- package/dist/workers/entry.mjs +2 -2
- package/package.json +1 -1
- package/dist/direct-executor-EOTFGXTO.mjs +0 -8
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// src/search/constants.ts
|
|
2
|
+
var RESULT_WINDOW = 1e4;
|
|
3
|
+
var DEFAULT_PAGE_SIZE = 10;
|
|
4
|
+
var MAX_CURSOR_LENGTH = 40960;
|
|
5
|
+
var MAX_SORT_FIELDS = 8;
|
|
6
|
+
var MAX_SORT_FIELD_NAME_LENGTH = 255;
|
|
7
|
+
var MAX_CURSOR_ANCHOR_CODE_POINTS = 512;
|
|
8
|
+
var PREFIX_MAX_EXPANSIONS = 50;
|
|
9
|
+
|
|
10
|
+
export { DEFAULT_PAGE_SIZE, MAX_CURSOR_ANCHOR_CODE_POINTS, MAX_CURSOR_LENGTH, MAX_SORT_FIELDS, MAX_SORT_FIELD_NAME_LENGTH, PREFIX_MAX_EXPANSIONS, RESULT_WINDOW };
|
|
@@ -5,7 +5,6 @@ var DEFAULT_TRANSPORT_CONFIG = {
|
|
|
5
5
|
replicationTimeout: 1e4,
|
|
6
6
|
snapshotTimeout: 3e5
|
|
7
7
|
};
|
|
8
|
-
var MAX_MESSAGE_SIZE_BYTES = 67108864;
|
|
9
8
|
var TransportErrorCodes = {
|
|
10
9
|
CONNECT_FAILED: "TRANSPORT_CONNECT_FAILED",
|
|
11
10
|
TIMEOUT: "TRANSPORT_TIMEOUT",
|
|
@@ -62,4 +61,4 @@ var ClusterMessageTypes = {
|
|
|
62
61
|
PARTITION_STORES: "cluster.partition_stores"
|
|
63
62
|
};
|
|
64
63
|
|
|
65
|
-
export { ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG,
|
|
64
|
+
export { ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, QueryMessageTypes, ReplicationMessageTypes, TransportError, TransportErrorCodes };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createArenaSimd } from './chunk-
|
|
1
|
+
import { createArenaSimd, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, QUANTIZATION_PADDING_FACTOR } from './chunk-JXHUHFJ5.mjs';
|
|
2
2
|
|
|
3
3
|
// src/vector/scalar-quantization-math.ts
|
|
4
4
|
function deriveSq8Constants(alpha, offset, dimensions) {
|
|
@@ -69,7 +69,6 @@ function arenaQuantizedDistance(simd, queryByteOffset, codeByteOffset, dimension
|
|
|
69
69
|
}
|
|
70
70
|
|
|
71
71
|
// src/vector/scalar-quantization-calibration.ts
|
|
72
|
-
var PADDING_FACTOR = 0.01;
|
|
73
72
|
function computeCalibrationBounds(vectors, dimensions) {
|
|
74
73
|
let globalMin = Number.POSITIVE_INFINITY;
|
|
75
74
|
let globalMax = Number.NEGATIVE_INFINITY;
|
|
@@ -84,7 +83,7 @@ function computeCalibrationBounds(vectors, dimensions) {
|
|
|
84
83
|
}
|
|
85
84
|
if (count === 0) return null;
|
|
86
85
|
const range = globalMax - globalMin;
|
|
87
|
-
const pad = range *
|
|
86
|
+
const pad = range * QUANTIZATION_PADDING_FACTOR;
|
|
88
87
|
globalMin -= pad;
|
|
89
88
|
globalMax += pad;
|
|
90
89
|
if (globalMin === globalMax) {
|
|
@@ -95,8 +94,6 @@ function computeCalibrationBounds(vectors, dimensions) {
|
|
|
95
94
|
}
|
|
96
95
|
|
|
97
96
|
// src/vector/scalar-quantization.ts
|
|
98
|
-
var INITIAL_CAPACITY = 16;
|
|
99
|
-
var PAGE_BYTES = 65536;
|
|
100
97
|
function createScalarQuantizer(dimensions, ordinalSource) {
|
|
101
98
|
const docToOrd = /* @__PURE__ */ new Map();
|
|
102
99
|
let simd = createArenaSimd();
|
|
@@ -114,7 +111,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
|
|
|
114
111
|
let calibrated = false;
|
|
115
112
|
function ensureCapacity(needed) {
|
|
116
113
|
if (needed <= capacity) return;
|
|
117
|
-
let newCap = capacity === 0 ?
|
|
114
|
+
let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
|
|
118
115
|
while (newCap < needed) newCap *= 2;
|
|
119
116
|
const nextSums = new Float64Array(newCap);
|
|
120
117
|
nextSums.set(sums);
|
|
@@ -133,7 +130,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
|
|
|
133
130
|
const have = simd.memory.buffer.byteLength;
|
|
134
131
|
if (requiredBytes > have) {
|
|
135
132
|
try {
|
|
136
|
-
simd.memory.grow(Math.ceil((requiredBytes - have) /
|
|
133
|
+
simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
|
|
137
134
|
} catch {
|
|
138
135
|
const migrated = new Uint8Array(newCap * dimensions);
|
|
139
136
|
migrated.set(quantizedArena.subarray(0, capacity * dimensions));
|
|
@@ -1,4 +1,16 @@
|
|
|
1
|
-
import { compareCodePoints } from './chunk-
|
|
1
|
+
import { compareCodePoints } from './chunk-2GWVQWGA.mjs';
|
|
2
|
+
|
|
3
|
+
// src/vector/constants.ts
|
|
4
|
+
var WASM_PAGE_BYTES = 65536;
|
|
5
|
+
var MAX_WASM_PAGES = 65536;
|
|
6
|
+
var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
7
|
+
var MAX_WORKER_DIMENSION = 8192;
|
|
8
|
+
var MAX_WORKER_VECTORS = 1e6;
|
|
9
|
+
var MAX_WORKER_COPIES = 64;
|
|
10
|
+
var HNSW_BUILD_TIMEOUT_MS = 12e4;
|
|
11
|
+
var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
|
|
12
|
+
var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
|
|
13
|
+
var QUANTIZATION_PADDING_FACTOR = 0.01;
|
|
2
14
|
|
|
3
15
|
// src/vector/simd-wasm-binary.ts
|
|
4
16
|
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABFQNgA39/fwF9YAJ/fwF9YAN/f38BfwMHBgABAAACAgUDAQABB2sHBm1lbW9yeQIAC2RvdF9wcm9kdWN0AAAJbWFnbml0dWRlAAEac3F1YXJlZF9ldWNsaWRlYW5fZGlzdGFuY2UAAhJldWNsaWRlYW5fZGlzdGFuY2UAAwZkb3RfdTgABAlzcWRpc3RfdTgABQqvBwauAQQBfwF7AX0BfyACQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhBEEAIQMCQANAIAMgBk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAE/R8AIAT9HwGSIAT9HwIgBP0fA5KSIQUCQANAIAMgAk8NASAFIAAgA0ECdGoqAgAgASADQQJ0aioCAJSSIQUgA0EBaiEDDAALCyAFC6kBBAF/AnsBfQF/IAFBfHEhBv0MAAAAAAAAAAAAAAAAAAAAACEDQQAhAgJAA0AgAiAGTw0BIAAgAkECdGr9AAQAIQQgAyAEIAT95gH95AEhAyACQQRqIQIMAAsLIAP9HwAgA/0fAZIgA/0fAiAD/R8DkpIhBQJAA0AgAiABTw0BIAUgACACQQJ0aioCACAAIAJBAnRqKgIAlJIhBSACQQFqIQIMAAsLIAWRC8ABBQF/AnsBfQF/AX0gAkF8cSEH/QwAAAAAAAAAAAAAAAAAAAAAIQRBACEDAkADQCADIAdPDQEgACADQQJ0av0ABAAgASADQQJ0av0ABAD95QEhBSAEIAUgBf3mAf3kASEEIANBBGohAwwACwsgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEGAkADQCADIAJPDQEgACADQQJ0aioCACABIANBAnRqKgIAkyEIIAYgCCAIlJIhBiADQQFqIQMMAAsLIAYLCwAgACABIAIQApELtAEDAX8DewJ/IAJBcHEhCEEA/REhBEEAIQMCQANAIAMgCE8NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBCAF/YkBIAb9iQH9ugH9rgEhBCAEIAX9igEgBv2KAf26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEHAkADQCADIAJPDQEgByAAIANqLQAAIAEgA2otAABsaiEHIANBAWohAwwACwsgBwvNAQMBfwV7A38gAkFwcSELQQD9ESEEQQAhAwJAA0AgAyALTw0BIAAgA2r9AAQAIQUgASADav0ABAAhBiAF/YkBIAb9iQH9kQEhByAF/YoBIAb9igH9kQEhCCAEIAcgB/26Af2uASEEIAQgCCAI/boB/a4BIQQgA0EQaiEDDAALCyAE/RsAIAT9GwFqIAT9GwIgBP0bA2pqIQkCQANAIAMgAk8NASAAIANqLQAAIAEgA2otAABrIQogCSAKIApsaiEJIANBAWohAwwACwsgCQs=";
|
|
@@ -226,8 +238,6 @@ function euclideanDistance(a, b) {
|
|
|
226
238
|
}
|
|
227
239
|
|
|
228
240
|
// src/vector/vector-store/index.ts
|
|
229
|
-
var INITIAL_CAPACITY = 16;
|
|
230
|
-
var PAGE_BYTES = 65536;
|
|
231
241
|
var UNKNOWN_PARTITION = -1;
|
|
232
242
|
var NO_PARTITION = -2;
|
|
233
243
|
function createVectorStore() {
|
|
@@ -254,7 +264,7 @@ function createVectorStore() {
|
|
|
254
264
|
}
|
|
255
265
|
function ensureCapacity2(needed) {
|
|
256
266
|
if (needed <= capacity) return;
|
|
257
|
-
let newCap = capacity === 0 ?
|
|
267
|
+
let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
|
|
258
268
|
while (newCap < needed) newCap *= 2;
|
|
259
269
|
const nextMags = new Float64Array(newCap);
|
|
260
270
|
nextMags.set(mags);
|
|
@@ -264,7 +274,7 @@ function createVectorStore() {
|
|
|
264
274
|
const have = simd.memory.buffer.byteLength;
|
|
265
275
|
if (requiredBytes > have) {
|
|
266
276
|
try {
|
|
267
|
-
simd.memory.grow(Math.ceil((requiredBytes - have) /
|
|
277
|
+
simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
|
|
268
278
|
} catch {
|
|
269
279
|
const migrated = new Float32Array(newCap * dimension);
|
|
270
280
|
migrated.set(arena.subarray(scratchFloatLength, scratchFloatLength + capacity * dimension));
|
|
@@ -294,7 +304,7 @@ function createVectorStore() {
|
|
|
294
304
|
}
|
|
295
305
|
function ensurePartitionSlots(ord) {
|
|
296
306
|
if (ord < partitionOf.length) return;
|
|
297
|
-
let length = partitionOf.length === 0 ?
|
|
307
|
+
let length = partitionOf.length === 0 ? VECTOR_STORE_INITIAL_CAPACITY : partitionOf.length;
|
|
298
308
|
while (length <= ord) length *= 2;
|
|
299
309
|
const grown = new Int32Array(length).fill(UNKNOWN_PARTITION);
|
|
300
310
|
grown.set(partitionOf);
|
|
@@ -536,11 +546,18 @@ function createVectorStore() {
|
|
|
536
546
|
};
|
|
537
547
|
}
|
|
538
548
|
|
|
539
|
-
// src/vector/hnsw/
|
|
549
|
+
// src/vector/hnsw/constants.ts
|
|
540
550
|
var MAX_LAYER_CAP = 32;
|
|
541
551
|
var MAX_M = 512;
|
|
542
|
-
var ABSENT = -1;
|
|
543
552
|
var INITIAL_UPPER_SLOTS = 64;
|
|
553
|
+
var INITIAL_LIST_CAPACITY = 64;
|
|
554
|
+
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
555
|
+
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
556
|
+
var SQ8_OVERSELECTION_FACTOR = 2;
|
|
557
|
+
var SQ8_RERANK_FLOOR = 10;
|
|
558
|
+
|
|
559
|
+
// src/vector/hnsw/adjacency.ts
|
|
560
|
+
var ABSENT = -1;
|
|
544
561
|
function createAdjacency(m, mMax0) {
|
|
545
562
|
return {
|
|
546
563
|
m,
|
|
@@ -726,9 +743,6 @@ function estimateAdjacencyBytes(adj) {
|
|
|
726
743
|
}
|
|
727
744
|
|
|
728
745
|
// src/vector/hnsw/shared.ts
|
|
729
|
-
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
730
|
-
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
731
|
-
var SQ8_OVERSELECTION_FACTOR = 2;
|
|
732
746
|
function ensureCapacity(state, needed) {
|
|
733
747
|
ensureAdjacencyCapacity(state.adjacency, needed);
|
|
734
748
|
if (needed <= state.capacity) return;
|
|
@@ -802,7 +816,6 @@ function addConnection(connections, ord) {
|
|
|
802
816
|
}
|
|
803
817
|
|
|
804
818
|
// src/vector/hnsw/workspace.ts
|
|
805
|
-
var INITIAL_LIST_CAPACITY = 64;
|
|
806
819
|
function createList(capacity) {
|
|
807
820
|
return { ords: new Int32Array(capacity), distances: new Float64Array(capacity), size: 0 };
|
|
808
821
|
}
|
|
@@ -1110,7 +1123,7 @@ function collectHits(state, query, k, searchMetric, minSimilarity, filter, efSea
|
|
|
1110
1123
|
}
|
|
1111
1124
|
function rerankWithFullPrecision(state, candidates, query, qMag, arenaQuery, k, metric, minSimilarity, filter, hasDocument) {
|
|
1112
1125
|
const reranked = [];
|
|
1113
|
-
const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR,
|
|
1126
|
+
const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR);
|
|
1114
1127
|
for (let i = 0; i < candidates.size; i++) {
|
|
1115
1128
|
const ord = candidates.ords[i];
|
|
1116
1129
|
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
@@ -1565,4 +1578,4 @@ function createHNSWIndex(dimension, store, config, quantizer) {
|
|
|
1565
1578
|
};
|
|
1566
1579
|
}
|
|
1567
1580
|
|
|
1568
|
-
export { ABSENT_DOCUMENT_RANK, MAX_M, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toScore };
|
|
1581
|
+
export { ABSENT_DOCUMENT_RANK, HNSW_BUILD_TIMEOUT_MS, MAX_M, MAX_WASM_PAGES, MAX_WORKER_COPIES, MAX_WORKER_DIMENSION, MAX_WORKER_VECTORS, QUANTIZATION_PADDING_FACTOR, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toScore };
|
|
@@ -1,12 +1,9 @@
|
|
|
1
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR } from './chunk-OWCFK42Q.mjs';
|
|
1
2
|
import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
|
|
2
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
3
4
|
|
|
4
5
|
var INDEX_CONFIG_PREFIX = "_narsil/index/";
|
|
5
6
|
var INDEX_CONFIG_SUFFIX = "/config";
|
|
6
|
-
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
7
|
-
var MAX_INDEX_NAME_LENGTH = 255;
|
|
8
|
-
var MAX_PARTITION_COUNT = 65536;
|
|
9
|
-
var MAX_REPLICATION_FACTOR = 255;
|
|
10
7
|
function truncateForDisplay(value) {
|
|
11
8
|
const str = String(value);
|
|
12
9
|
if (str.length > 100) {
|
|
@@ -134,4 +131,4 @@ async function getIndexMetadata(coordinator, indexName) {
|
|
|
134
131
|
return validateDecodedMetadata(decoded, indexName);
|
|
135
132
|
}
|
|
136
133
|
|
|
137
|
-
export {
|
|
134
|
+
export { getIndexMetadata, indexConfigKey, putIndexMetadata, validateIndexName };
|