@delali/narsil 0.2.4-next.48 → 0.2.4-next.50
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/indexeddb.mjs +15 -10
- package/dist/{options-BC68VPoE.d.ts → capabilities-JrVQOwi4.d.ts} +55 -47
- package/dist/{chunk-BFIH7OV5.mjs → chunk-2GWVQWGA.mjs} +3 -1
- package/dist/chunk-AV4IJKD2.mjs +35 -0
- package/dist/{chunk-EV45L35L.mjs → chunk-BEIK3EOZ.mjs} +845 -2407
- package/dist/chunk-CBLS46IE.mjs +10 -0
- package/dist/chunk-CMVG2UEF.mjs +2038 -0
- package/dist/chunk-CV5H7XL7.mjs +876 -0
- package/dist/{chunk-JQUL2T6I.mjs → chunk-GCM3JFVA.mjs} +1 -2
- package/dist/{chunk-XWICBGYT.mjs → chunk-MHE7YWAU.mjs} +3 -5
- package/dist/{chunk-UZTT2BKQ.mjs → chunk-ND34I3SV.mjs} +1202 -1881
- package/dist/{chunk-PM52SMBT.mjs → chunk-OOSIDHA3.mjs} +2 -1
- package/dist/chunk-ORBLD3EL.mjs +6 -0
- package/dist/chunk-PIWKZPP7.mjs +64 -0
- package/dist/{chunk-4COAUBLJ.mjs → chunk-PMBHH44R.mjs} +124 -8
- package/dist/chunk-RP23PWMV.mjs +11 -0
- package/dist/chunk-UNEEFTEU.mjs +7 -0
- package/dist/{chunk-WTONLDU3.mjs → chunk-WYJXSLER.mjs} +9 -19
- package/dist/{chunk-DFZDV5FV.mjs → chunk-X6YZO4MQ.mjs} +19 -15
- package/dist/chunk-XE6XR2OE.mjs +23 -0
- package/dist/chunk-YEO6G2NG.mjs +6 -0
- package/dist/chunk-ZHUQKZBE.mjs +1292 -0
- package/dist/{client-C5BkUqB8.d.ts → client-CBOVHdjp.d.ts} +31 -7
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +250 -232
- package/dist/{config-B-ZI-ft5.d.ts → config-Czh2YNDi.d.ts} +2 -2
- package/dist/{config-BZRExglL.d.ts → config-OON7IJ_B.d.ts} +50 -13
- package/dist/direct-executor-OREXU72C.mjs +12 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -3
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.mjs +2 -2
- package/dist/distribution/transport/grpc.mjs +7 -9
- package/dist/distribution/transport/in-memory.mjs +2 -1
- package/dist/distribution/transport/tcp.mjs +3 -2
- package/dist/distribution.d.ts +8 -8
- package/dist/distribution.mjs +142 -101
- package/dist/embeddings/openai.mjs +5 -3
- package/dist/{engine-Cjr1uBLa.d.ts → engine-9PtFY50q.d.ts} +125 -69
- package/dist/index.browser.mjs +4689 -3330
- package/dist/index.d.ts +7 -7
- package/dist/index.mjs +14 -8
- package/dist/invalidation/broadcast-channel.mjs +2 -1
- package/dist/invalidation/filesystem.mjs +3 -4
- package/dist/{search-DDHjfXhJ.d.ts → memory-Bzd5xcEW.d.ts} +89 -63
- package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -8
- package/dist/react.d.ts +4 -4
- package/dist/react.mjs +7 -5
- package/dist/{schema-B7juWg8Q.d.ts → schema-BqRx9t4E.d.ts} +22 -3
- package/dist/server.d.ts +47 -42
- package/dist/server.mjs +338 -821
- package/dist/thread-PZLEO6JL.mjs +310 -0
- package/dist/{types-DsSSjY4d.d.ts → types-DXPxaXPO.d.ts} +1 -1
- package/dist/vector/hnsw-build-worker.mjs +3 -4
- package/dist/vector/search-worker.mjs +4 -123
- package/dist/workers/entry.mjs +47 -3
- package/package.json +5 -1
- package/dist/direct-executor-EOTFGXTO.mjs +0 -8
|
@@ -1,4 +1,5 @@
|
|
|
1
|
-
import { MAX_MESSAGE_SIZE_BYTES
|
|
1
|
+
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
+
import { TransportError, TransportErrorCodes } from './chunk-GCM3JFVA.mjs';
|
|
2
3
|
import { encode, decode } from '@msgpack/msgpack';
|
|
3
4
|
|
|
4
5
|
// src/distribution/transport/tcp/types.ts
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { NarsilError, ErrorCodes } from './chunk-P4ELXVJQ.mjs';
|
|
2
|
+
|
|
3
|
+
// src/narsil/internals.ts
|
|
4
|
+
var engineCores = /* @__PURE__ */ new WeakMap();
|
|
5
|
+
function bindEngineCore(engine, core) {
|
|
6
|
+
engineCores.set(engine, core);
|
|
7
|
+
}
|
|
8
|
+
function engineCoreOf(engine) {
|
|
9
|
+
return engineCores.get(engine);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
// src/distribution/constants.ts
|
|
13
|
+
var WIRE_BATCH_BUDGET = {
|
|
14
|
+
maxCount: 1e3,
|
|
15
|
+
maxBytes: 8388608,
|
|
16
|
+
overheadBytes: 256
|
|
17
|
+
};
|
|
18
|
+
var MAX_PARTITION_COUNT = 65536;
|
|
19
|
+
var MAX_REPLICATION_FACTOR = 255;
|
|
20
|
+
|
|
21
|
+
// src/engine/resolve-vector-text.ts
|
|
22
|
+
async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdapterName) {
|
|
23
|
+
if (!params.vector) return params;
|
|
24
|
+
if (params.vector.text === void 0 && params.vector.value === void 0) return params;
|
|
25
|
+
if (params.vector.value !== void 0 && params.vector.text === void 0) return params;
|
|
26
|
+
if (params.vector.text !== void 0 && params.vector.value !== void 0) {
|
|
27
|
+
throw new NarsilError(ErrorCodes.EMBEDDING_CONFIG_INVALID, "Vector query cannot have both 'text' and 'value'");
|
|
28
|
+
}
|
|
29
|
+
if (!embeddingAdapter) {
|
|
30
|
+
throw new NarsilError(
|
|
31
|
+
ErrorCodes.EMBEDDING_CONFIG_INVALID,
|
|
32
|
+
embeddingAdapterName ? `Vector query with 'text' needs embedding adapter "${embeddingAdapterName}", which is not registered on this engine. Register it with registerEmbeddingAdapter or supply a raw vector.` : "Vector query with 'text' requires an embedding adapter on the index or instance",
|
|
33
|
+
embeddingAdapterName ? { adapter: embeddingAdapterName } : void 0
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
if (typeof params.vector.text !== "string") {
|
|
37
|
+
throw new NarsilError(
|
|
38
|
+
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
39
|
+
`Vector query 'text' must be a string, got ${typeof params.vector.text}`
|
|
40
|
+
);
|
|
41
|
+
}
|
|
42
|
+
if (params.vector.text.length === 0) {
|
|
43
|
+
throw new NarsilError(ErrorCodes.DOC_VALIDATION_FAILED, "Vector query 'text' must not be empty");
|
|
44
|
+
}
|
|
45
|
+
const raw = await embeddingAdapter.embed(params.vector.text, "query", signal);
|
|
46
|
+
if (!(raw instanceof Float32Array)) {
|
|
47
|
+
throw new NarsilError(
|
|
48
|
+
ErrorCodes.EMBEDDING_FAILED,
|
|
49
|
+
`Adapter returned ${typeof raw} for query embedding, expected Float32Array`
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
if (raw.length !== embeddingAdapter.dimensions) {
|
|
53
|
+
throw new NarsilError(
|
|
54
|
+
ErrorCodes.EMBEDDING_DIMENSION_MISMATCH,
|
|
55
|
+
`Adapter returned ${raw.length}-dimensional vector for query, expected ${embeddingAdapter.dimensions}`,
|
|
56
|
+
{ expected: embeddingAdapter.dimensions, actual: raw.length }
|
|
57
|
+
);
|
|
58
|
+
}
|
|
59
|
+
const resolved = { ...params.vector, value: Array.from(raw) };
|
|
60
|
+
delete resolved.text;
|
|
61
|
+
return { ...params, vector: resolved };
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, bindEngineCore, engineCoreOf, resolveVectorText };
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { createArenaSimd } from './chunk-
|
|
1
|
+
import { createSharedArenaSimd, importAdjacency, createHNSWWorkspace, createArenaSimd, WASM_PAGE_BYTES, VECTOR_STORE_INITIAL_CAPACITY, arenaFloat32Distance, ABSENT_DOCUMENT_RANK, QUANTIZATION_PADDING_FACTOR } from './chunk-X6YZO4MQ.mjs';
|
|
2
2
|
|
|
3
3
|
// src/vector/scalar-quantization-math.ts
|
|
4
4
|
function deriveSq8Constants(alpha, offset, dimensions) {
|
|
@@ -68,8 +68,126 @@ function arenaQuantizedDistance(simd, queryByteOffset, codeByteOffset, dimension
|
|
|
68
68
|
return 1 - realDot / (queryMagnitude * documentMagnitude);
|
|
69
69
|
}
|
|
70
70
|
|
|
71
|
+
// src/vector/shared-generation/worker-view.ts
|
|
72
|
+
function createVectorReader(snapshot, simd, scratchSlot) {
|
|
73
|
+
const { layout, magnitudes, rankByOrdinal } = snapshot;
|
|
74
|
+
const { dimension, slots } = layout;
|
|
75
|
+
const vectors = new Float32Array(snapshot.memory.buffer, layout.vectorsOffset, slots * dimension);
|
|
76
|
+
const scratchByteOffset = scratchSlot * layout.float32ScratchStride;
|
|
77
|
+
const scratch = simd ? new Float32Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
|
|
78
|
+
const holdsDocument = (ordinal) => ordinal >= 0 && ordinal < slots && rankByOrdinal[ordinal] !== ABSENT_DOCUMENT_RANK;
|
|
79
|
+
return {
|
|
80
|
+
entryForOrdinal(ordinal) {
|
|
81
|
+
if (!holdsDocument(ordinal)) return void 0;
|
|
82
|
+
const base = ordinal * dimension;
|
|
83
|
+
return { vector: vectors.subarray(base, base + dimension), magnitude: magnitudes[ordinal] };
|
|
84
|
+
},
|
|
85
|
+
prepareQueryArena(query) {
|
|
86
|
+
if (!simd || !scratch || query.length !== dimension) return null;
|
|
87
|
+
scratch.set(query, 0);
|
|
88
|
+
return { magnitude: simd.magnitude(scratchByteOffset, dimension) };
|
|
89
|
+
},
|
|
90
|
+
distanceFromArena(prepared, ordinal, metric) {
|
|
91
|
+
if (!simd || !holdsDocument(ordinal)) return Number.POSITIVE_INFINITY;
|
|
92
|
+
const byteOffset = layout.vectorsOffset + ordinal * dimension * 4;
|
|
93
|
+
return arenaFloat32Distance(
|
|
94
|
+
simd,
|
|
95
|
+
scratchByteOffset,
|
|
96
|
+
byteOffset,
|
|
97
|
+
dimension,
|
|
98
|
+
metric,
|
|
99
|
+
prepared.magnitude,
|
|
100
|
+
magnitudes[ordinal]
|
|
101
|
+
);
|
|
102
|
+
}
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
function createQuantizerReader(snapshot, simd, scratchSlot) {
|
|
106
|
+
if (snapshot.quantization !== "sq8" || snapshot.calibration === null) return void 0;
|
|
107
|
+
const { layout, codeSums, codeMagnitudes, codePresent } = snapshot;
|
|
108
|
+
const { dimension, slots } = layout;
|
|
109
|
+
const constants = deriveSq8Constants(snapshot.calibration.alpha, snapshot.calibration.offset, dimension);
|
|
110
|
+
const codes = new Uint8Array(snapshot.memory.buffer, layout.codesOffset, slots * dimension);
|
|
111
|
+
const scratchByteOffset = layout.codeScratchOffset + scratchSlot * layout.codeScratchStride;
|
|
112
|
+
const scratch = simd ? new Uint8Array(snapshot.memory.buffer, scratchByteOffset, dimension) : null;
|
|
113
|
+
let liveCodes = 0;
|
|
114
|
+
for (let ordinal = 0; ordinal < codePresent.length; ordinal++) {
|
|
115
|
+
if (codePresent[ordinal] === 1) liveCodes++;
|
|
116
|
+
}
|
|
117
|
+
const holdsCodes = (ordinal) => ordinal >= 0 && ordinal < slots && codePresent[ordinal] === 1;
|
|
118
|
+
return {
|
|
119
|
+
get size() {
|
|
120
|
+
return liveCodes;
|
|
121
|
+
},
|
|
122
|
+
isCalibrated: () => true,
|
|
123
|
+
prepareQuery(query) {
|
|
124
|
+
const quantized = new Uint8Array(dimension);
|
|
125
|
+
const { sum, sumSq } = quantizeVectorInto(quantized, query, dimension, constants);
|
|
126
|
+
return { quantized, sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
|
|
127
|
+
},
|
|
128
|
+
distanceFromPreparedByOrdinal(prepared, ordinal, metric) {
|
|
129
|
+
if (!holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
|
|
130
|
+
return scalarQuantizedDistance(
|
|
131
|
+
codes,
|
|
132
|
+
codeSums,
|
|
133
|
+
codeMagnitudes,
|
|
134
|
+
dimension,
|
|
135
|
+
constants,
|
|
136
|
+
prepared.quantized,
|
|
137
|
+
prepared.sum,
|
|
138
|
+
prepared.magnitude,
|
|
139
|
+
ordinal,
|
|
140
|
+
metric
|
|
141
|
+
);
|
|
142
|
+
},
|
|
143
|
+
prepareQueryArena(query) {
|
|
144
|
+
if (!simd || !scratch) return null;
|
|
145
|
+
const { sum, sumSq } = quantizeVectorInto(scratch, query, dimension, constants);
|
|
146
|
+
return { sum, sumSq, magnitude: quantizedMagnitude(constants, sumSq, sum) };
|
|
147
|
+
},
|
|
148
|
+
distanceFromArena(prepared, ordinal, metric) {
|
|
149
|
+
if (!simd || !holdsCodes(ordinal)) return Number.POSITIVE_INFINITY;
|
|
150
|
+
return arenaQuantizedDistance(
|
|
151
|
+
simd,
|
|
152
|
+
scratchByteOffset,
|
|
153
|
+
layout.codesOffset + ordinal * dimension,
|
|
154
|
+
dimension,
|
|
155
|
+
constants,
|
|
156
|
+
prepared.sum,
|
|
157
|
+
prepared.magnitude,
|
|
158
|
+
codeSums[ordinal],
|
|
159
|
+
codeMagnitudes[ordinal],
|
|
160
|
+
metric
|
|
161
|
+
);
|
|
162
|
+
}
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
function openSharedWorkerCopy(snapshot, scratchSlot) {
|
|
166
|
+
if (scratchSlot < 0 || scratchSlot >= snapshot.layout.workerSlots) {
|
|
167
|
+
throw new Error(`Scratch slot ${scratchSlot} is outside the ${snapshot.layout.workerSlots} reserved slots`);
|
|
168
|
+
}
|
|
169
|
+
const simd = createSharedArenaSimd(snapshot.memory);
|
|
170
|
+
const adjacency = importAdjacency(snapshot.graph.adjacency);
|
|
171
|
+
const capacity = Math.max(snapshot.graph.capacity, adjacency.capacity);
|
|
172
|
+
const searchState = {
|
|
173
|
+
dimension: snapshot.dimension,
|
|
174
|
+
store: createVectorReader(snapshot, simd, scratchSlot),
|
|
175
|
+
quantizer: createQuantizerReader(snapshot, simd, scratchSlot),
|
|
176
|
+
adjacency,
|
|
177
|
+
tombstones: snapshot.graph.tombstones,
|
|
178
|
+
tombstoneCount: snapshot.graph.tombstoneCount,
|
|
179
|
+
nodeCount: snapshot.graph.nodeCount,
|
|
180
|
+
capacity,
|
|
181
|
+
visited: new Uint32Array(capacity),
|
|
182
|
+
visitStamp: 0,
|
|
183
|
+
entryPointOrd: snapshot.graph.entryPointOrd,
|
|
184
|
+
topLayer: snapshot.graph.topLayer,
|
|
185
|
+
workspace: createHNSWWorkspace()
|
|
186
|
+
};
|
|
187
|
+
return { searchState, rankByOrdinal: snapshot.rankByOrdinal };
|
|
188
|
+
}
|
|
189
|
+
|
|
71
190
|
// src/vector/scalar-quantization-calibration.ts
|
|
72
|
-
var PADDING_FACTOR = 0.01;
|
|
73
191
|
function computeCalibrationBounds(vectors, dimensions) {
|
|
74
192
|
let globalMin = Number.POSITIVE_INFINITY;
|
|
75
193
|
let globalMax = Number.NEGATIVE_INFINITY;
|
|
@@ -84,7 +202,7 @@ function computeCalibrationBounds(vectors, dimensions) {
|
|
|
84
202
|
}
|
|
85
203
|
if (count === 0) return null;
|
|
86
204
|
const range = globalMax - globalMin;
|
|
87
|
-
const pad = range *
|
|
205
|
+
const pad = range * QUANTIZATION_PADDING_FACTOR;
|
|
88
206
|
globalMin -= pad;
|
|
89
207
|
globalMax += pad;
|
|
90
208
|
if (globalMin === globalMax) {
|
|
@@ -95,8 +213,6 @@ function computeCalibrationBounds(vectors, dimensions) {
|
|
|
95
213
|
}
|
|
96
214
|
|
|
97
215
|
// src/vector/scalar-quantization.ts
|
|
98
|
-
var INITIAL_CAPACITY = 16;
|
|
99
|
-
var PAGE_BYTES = 65536;
|
|
100
216
|
function createScalarQuantizer(dimensions, ordinalSource) {
|
|
101
217
|
const docToOrd = /* @__PURE__ */ new Map();
|
|
102
218
|
let simd = createArenaSimd();
|
|
@@ -114,7 +230,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
|
|
|
114
230
|
let calibrated = false;
|
|
115
231
|
function ensureCapacity(needed) {
|
|
116
232
|
if (needed <= capacity) return;
|
|
117
|
-
let newCap = capacity === 0 ?
|
|
233
|
+
let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
|
|
118
234
|
while (newCap < needed) newCap *= 2;
|
|
119
235
|
const nextSums = new Float64Array(newCap);
|
|
120
236
|
nextSums.set(sums);
|
|
@@ -133,7 +249,7 @@ function createScalarQuantizer(dimensions, ordinalSource) {
|
|
|
133
249
|
const have = simd.memory.buffer.byteLength;
|
|
134
250
|
if (requiredBytes > have) {
|
|
135
251
|
try {
|
|
136
|
-
simd.memory.grow(Math.ceil((requiredBytes - have) /
|
|
252
|
+
simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
|
|
137
253
|
} catch {
|
|
138
254
|
const migrated = new Uint8Array(newCap * dimensions);
|
|
139
255
|
migrated.set(quantizedArena.subarray(0, capacity * dimensions));
|
|
@@ -373,4 +489,4 @@ function createScalarQuantizer(dimensions, ordinalSource) {
|
|
|
373
489
|
};
|
|
374
490
|
}
|
|
375
491
|
|
|
376
|
-
export {
|
|
492
|
+
export { createScalarQuantizer, openSharedWorkerCopy };
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
// src/vector/hnsw/constants.ts
|
|
2
|
+
var MAX_LAYER_CAP = 32;
|
|
3
|
+
var MAX_M = 512;
|
|
4
|
+
var INITIAL_UPPER_SLOTS = 64;
|
|
5
|
+
var INITIAL_LIST_CAPACITY = 64;
|
|
6
|
+
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
7
|
+
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
8
|
+
var SQ8_OVERSELECTION_FACTOR = 2;
|
|
9
|
+
var SQ8_RERANK_FLOOR = 10;
|
|
10
|
+
|
|
11
|
+
export { COMPACTION_ABSOLUTE_THRESHOLD, COMPACTION_TOMBSTONE_RATIO, INITIAL_LIST_CAPACITY, INITIAL_UPPER_SLOTS, MAX_LAYER_CAP, MAX_M, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR };
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
// src/distribution/transport/constants.ts
|
|
2
|
+
var MAX_MESSAGE_SIZE_BYTES = 67108864;
|
|
3
|
+
var GRPC_ENVELOPE_OVERHEAD_BYTES = 1024;
|
|
4
|
+
var GRPC_KEEPALIVE_TIME_MS = 3e4;
|
|
5
|
+
var GRPC_KEEPALIVE_TIMEOUT_MS = 1e4;
|
|
6
|
+
|
|
7
|
+
export { GRPC_ENVELOPE_OVERHEAD_BYTES, GRPC_KEEPALIVE_TIMEOUT_MS, GRPC_KEEPALIVE_TIME_MS, MAX_MESSAGE_SIZE_BYTES };
|
|
@@ -1,20 +1,3 @@
|
|
|
1
|
-
// src/core/id-generator.ts
|
|
2
|
-
function generateId() {
|
|
3
|
-
const bytes = new Uint8Array(16);
|
|
4
|
-
crypto.getRandomValues(bytes);
|
|
5
|
-
const now = Date.now();
|
|
6
|
-
bytes[0] = Math.floor(now / 2 ** 40);
|
|
7
|
-
bytes[1] = Math.floor(now / 2 ** 32) % 256;
|
|
8
|
-
bytes[2] = Math.floor(now / 2 ** 24) % 256;
|
|
9
|
-
bytes[3] = Math.floor(now / 2 ** 16) % 256;
|
|
10
|
-
bytes[4] = Math.floor(now / 2 ** 8) % 256;
|
|
11
|
-
bytes[5] = now % 256;
|
|
12
|
-
bytes[6] = bytes[6] & 15 | 112;
|
|
13
|
-
bytes[8] = bytes[8] & 63 | 128;
|
|
14
|
-
const hex = Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
|
|
15
|
-
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
|
16
|
-
}
|
|
17
|
-
|
|
18
1
|
// src/workers/protocol.ts
|
|
19
2
|
var KNOWN_ACTION_TYPES = [
|
|
20
3
|
"insert",
|
|
@@ -26,6 +9,10 @@ var KNOWN_ACTION_TYPES = [
|
|
|
26
9
|
"has",
|
|
27
10
|
"count",
|
|
28
11
|
"createIndex",
|
|
12
|
+
"loadVectorCopy",
|
|
13
|
+
"dropVectorCopy",
|
|
14
|
+
"serveRequests",
|
|
15
|
+
"stopServing",
|
|
29
16
|
"dropIndex",
|
|
30
17
|
"getStats",
|
|
31
18
|
"clear",
|
|
@@ -36,12 +23,15 @@ var KNOWN_ACTION_TYPES = [
|
|
|
36
23
|
"attachSegments",
|
|
37
24
|
"compactSegments",
|
|
38
25
|
"swapSegments",
|
|
26
|
+
"freezeLiveTail",
|
|
39
27
|
"memoryReport",
|
|
40
28
|
"bootstrap",
|
|
41
29
|
"shutdown"
|
|
42
30
|
];
|
|
31
|
+
var nextRequestSequence = 0;
|
|
43
32
|
function createRequestId() {
|
|
44
|
-
|
|
33
|
+
nextRequestSequence += 1;
|
|
34
|
+
return String(nextRequestSequence);
|
|
45
35
|
}
|
|
46
36
|
function isValidWorkerAction(msg) {
|
|
47
37
|
if (msg === null || msg === void 0 || typeof msg !== "object") {
|
|
@@ -67,4 +57,4 @@ function isValidWorkerResponse(msg) {
|
|
|
67
57
|
return true;
|
|
68
58
|
}
|
|
69
59
|
|
|
70
|
-
export { createRequestId,
|
|
60
|
+
export { createRequestId, isValidWorkerAction, isValidWorkerResponse };
|
|
@@ -1,4 +1,17 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { INITIAL_LIST_CAPACITY, COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, MAX_LAYER_CAP, SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR, INITIAL_UPPER_SLOTS } from './chunk-RP23PWMV.mjs';
|
|
2
|
+
import { compareCodePoints } from './chunk-2GWVQWGA.mjs';
|
|
3
|
+
|
|
4
|
+
// src/vector/constants.ts
|
|
5
|
+
var WASM_PAGE_BYTES = 65536;
|
|
6
|
+
var MAX_WASM_PAGES = 65536;
|
|
7
|
+
var VECTOR_STORE_INITIAL_CAPACITY = 16;
|
|
8
|
+
var MAX_WORKER_DIMENSION = 8192;
|
|
9
|
+
var MAX_WORKER_VECTORS = 1e6;
|
|
10
|
+
var MAX_WORKER_COPIES = 64;
|
|
11
|
+
var HNSW_BUILD_TIMEOUT_MS = 12e4;
|
|
12
|
+
var VECTOR_SEARCH_TIMEOUT_MS = 3e4;
|
|
13
|
+
var VECTOR_SEARCH_LOAD_TIMEOUT_MS = 3e5;
|
|
14
|
+
var QUANTIZATION_PADDING_FACTOR = 0.01;
|
|
2
15
|
|
|
3
16
|
// src/vector/simd-wasm-binary.ts
|
|
4
17
|
var SIMD_DISTANCE_WASM_BASE64 = "AGFzbQEAAAABFQNgA39/fwF9YAJ/fwF9YAN/f38BfwMHBgABAAACAgUDAQABB2sHBm1lbW9yeQIAC2RvdF9wcm9kdWN0AAAJbWFnbml0dWRlAAEac3F1YXJlZF9ldWNsaWRlYW5fZGlzdGFuY2UAAhJldWNsaWRlYW5fZGlzdGFuY2UAAwZkb3RfdTgABAlzcWRpc3RfdTgABQqvBwauAQQBfwF7AX0BfyACQXxxIQb9DAAAAAAAAAAAAAAAAAAAAAAhBEEAIQMCQANAIAMgBk8NASAEIAAgA0ECdGr9AAQAIAEgA0ECdGr9AAQA/eYB/eQBIQQgA0EEaiEDDAALCyAE/R8AIAT9HwGSIAT9HwIgBP0fA5KSIQUCQANAIAMgAk8NASAFIAAgA0ECdGoqAgAgASADQQJ0aioCAJSSIQUgA0EBaiEDDAALCyAFC6kBBAF/AnsBfQF/IAFBfHEhBv0MAAAAAAAAAAAAAAAAAAAAACEDQQAhAgJAA0AgAiAGTw0BIAAgAkECdGr9AAQAIQQgAyAEIAT95gH95AEhAyACQQRqIQIMAAsLIAP9HwAgA/0fAZIgA/0fAiAD/R8DkpIhBQJAA0AgAiABTw0BIAUgACACQQJ0aioCACAAIAJBAnRqKgIAlJIhBSACQQFqIQIMAAsLIAWRC8ABBQF/AnsBfQF/AX0gAkF8cSEH/QwAAAAAAAAAAAAAAAAAAAAAIQRBACEDAkADQCADIAdPDQEgACADQQJ0av0ABAAgASADQQJ0av0ABAD95QEhBSAEIAUgBf3mAf3kASEEIANBBGohAwwACwsgBP0fACAE/R8BkiAE/R8CIAT9HwOSkiEGAkADQCADIAJPDQEgACADQQJ0aioCACABIANBAnRqKgIAkyEIIAYgCCAIlJIhBiADQQFqIQMMAAsLIAYLCwAgACABIAIQApELtAEDAX8DewJ/IAJBcHEhCEEA/REhBEEAIQMCQANAIAMgCE8NASAAIANq/QAEACEFIAEgA2r9AAQAIQYgBCAF/YkBIAb9iQH9ugH9rgEhBCAEIAX9igEgBv2KAf26Af2uASEEIANBEGohAwwACwsgBP0bACAE/RsBaiAE/RsCIAT9GwNqaiEHAkADQCADIAJPDQEgByAAIANqLQAAIAEgA2otAABsaiEHIANBAWohAwwACwsgBwvNAQMBfwV7A38gAkFwcSELQQD9ESEEQQAhAwJAA0AgAyALTw0BIAAgA2r9AAQAIQUgASADav0ABAAhBiAF/YkBIAb9iQH9kQEhByAF/YoBIAb9igH9kQEhCCAEIAcgB/26Af2uASEEIAQgCCAI/boB/a4BIQQgA0EQaiEDDAALCyAE/RsAIAT9GwFqIAT9GwIgBP0bA2pqIQkCQANAIAMgAk8NASAAIANqLQAAIAEgA2otAABrIQogCSAKIApsaiEJIANBAWohAwwACwsgCQs=";
|
|
@@ -226,8 +239,6 @@ function euclideanDistance(a, b) {
|
|
|
226
239
|
}
|
|
227
240
|
|
|
228
241
|
// src/vector/vector-store/index.ts
|
|
229
|
-
var INITIAL_CAPACITY = 16;
|
|
230
|
-
var PAGE_BYTES = 65536;
|
|
231
242
|
var UNKNOWN_PARTITION = -1;
|
|
232
243
|
var NO_PARTITION = -2;
|
|
233
244
|
function createVectorStore() {
|
|
@@ -254,7 +265,7 @@ function createVectorStore() {
|
|
|
254
265
|
}
|
|
255
266
|
function ensureCapacity2(needed) {
|
|
256
267
|
if (needed <= capacity) return;
|
|
257
|
-
let newCap = capacity === 0 ?
|
|
268
|
+
let newCap = capacity === 0 ? VECTOR_STORE_INITIAL_CAPACITY : capacity;
|
|
258
269
|
while (newCap < needed) newCap *= 2;
|
|
259
270
|
const nextMags = new Float64Array(newCap);
|
|
260
271
|
nextMags.set(mags);
|
|
@@ -264,7 +275,7 @@ function createVectorStore() {
|
|
|
264
275
|
const have = simd.memory.buffer.byteLength;
|
|
265
276
|
if (requiredBytes > have) {
|
|
266
277
|
try {
|
|
267
|
-
simd.memory.grow(Math.ceil((requiredBytes - have) /
|
|
278
|
+
simd.memory.grow(Math.ceil((requiredBytes - have) / WASM_PAGE_BYTES));
|
|
268
279
|
} catch {
|
|
269
280
|
const migrated = new Float32Array(newCap * dimension);
|
|
270
281
|
migrated.set(arena.subarray(scratchFloatLength, scratchFloatLength + capacity * dimension));
|
|
@@ -294,7 +305,7 @@ function createVectorStore() {
|
|
|
294
305
|
}
|
|
295
306
|
function ensurePartitionSlots(ord) {
|
|
296
307
|
if (ord < partitionOf.length) return;
|
|
297
|
-
let length = partitionOf.length === 0 ?
|
|
308
|
+
let length = partitionOf.length === 0 ? VECTOR_STORE_INITIAL_CAPACITY : partitionOf.length;
|
|
298
309
|
while (length <= ord) length *= 2;
|
|
299
310
|
const grown = new Int32Array(length).fill(UNKNOWN_PARTITION);
|
|
300
311
|
grown.set(partitionOf);
|
|
@@ -537,10 +548,7 @@ function createVectorStore() {
|
|
|
537
548
|
}
|
|
538
549
|
|
|
539
550
|
// src/vector/hnsw/adjacency.ts
|
|
540
|
-
var MAX_LAYER_CAP = 32;
|
|
541
|
-
var MAX_M = 512;
|
|
542
551
|
var ABSENT = -1;
|
|
543
|
-
var INITIAL_UPPER_SLOTS = 64;
|
|
544
552
|
function createAdjacency(m, mMax0) {
|
|
545
553
|
return {
|
|
546
554
|
m,
|
|
@@ -726,9 +734,6 @@ function estimateAdjacencyBytes(adj) {
|
|
|
726
734
|
}
|
|
727
735
|
|
|
728
736
|
// src/vector/hnsw/shared.ts
|
|
729
|
-
var COMPACTION_TOMBSTONE_RATIO = 0.1;
|
|
730
|
-
var COMPACTION_ABSOLUTE_THRESHOLD = 1e3;
|
|
731
|
-
var SQ8_OVERSELECTION_FACTOR = 2;
|
|
732
737
|
function ensureCapacity(state, needed) {
|
|
733
738
|
ensureAdjacencyCapacity(state.adjacency, needed);
|
|
734
739
|
if (needed <= state.capacity) return;
|
|
@@ -802,7 +807,6 @@ function addConnection(connections, ord) {
|
|
|
802
807
|
}
|
|
803
808
|
|
|
804
809
|
// src/vector/hnsw/workspace.ts
|
|
805
|
-
var INITIAL_LIST_CAPACITY = 64;
|
|
806
810
|
function createList(capacity) {
|
|
807
811
|
return { ords: new Int32Array(capacity), distances: new Float64Array(capacity), size: 0 };
|
|
808
812
|
}
|
|
@@ -1110,7 +1114,7 @@ function collectHits(state, query, k, searchMetric, minSimilarity, filter, efSea
|
|
|
1110
1114
|
}
|
|
1111
1115
|
function rerankWithFullPrecision(state, candidates, query, qMag, arenaQuery, k, metric, minSimilarity, filter, hasDocument) {
|
|
1112
1116
|
const reranked = [];
|
|
1113
|
-
const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR,
|
|
1117
|
+
const rerankLimit = Math.max(k * SQ8_OVERSELECTION_FACTOR, SQ8_RERANK_FLOOR);
|
|
1114
1118
|
for (let i = 0; i < candidates.size; i++) {
|
|
1115
1119
|
const ord = candidates.ords[i];
|
|
1116
1120
|
if (filter && !ordinalFilterHas(filter, ord)) continue;
|
|
@@ -1565,4 +1569,4 @@ function createHNSWIndex(dimension, store, config, quantizer) {
|
|
|
1565
1569
|
};
|
|
1566
1570
|
}
|
|
1567
1571
|
|
|
1568
|
-
export { ABSENT_DOCUMENT_RANK,
|
|
1572
|
+
export { ABSENT_DOCUMENT_RANK, HNSW_BUILD_TIMEOUT_MS, MAX_WASM_PAGES, MAX_WORKER_COPIES, MAX_WORKER_DIMENSION, MAX_WORKER_VECTORS, QUANTIZATION_PADDING_FACTOR, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_STORE_INITIAL_CAPACITY, WASM_PAGE_BYTES, addToOrdinalFilter, arenaFloat32Distance, cosineSimilarityWithMagnitudes, createArenaSimd, createHNSWIndex, createHNSWWorkspace, createOrdinalFilter, createSharedArenaSimd, createVectorStore, dotProduct, entryForOrd, euclideanDistance, importAdjacency, isSimdAvailable, magnitude, ordinalFilterHas, ordinalFilterValues, removeFromOrdinalFilter, searchOrdinals, toDistance, toScore };
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { DEFAULT_PAGE_SIZE } from './chunk-CBLS46IE.mjs';
|
|
2
|
+
|
|
3
|
+
// src/engine/constants.ts
|
|
4
|
+
var MIN_DOCUMENTS_FOR_SEGMENTS = 64;
|
|
5
|
+
var BATCH_CHUNK_SIZE = 1e3;
|
|
6
|
+
var DEFAULT_LIMIT = DEFAULT_PAGE_SIZE;
|
|
7
|
+
var DEFAULT_OFFSET = 0;
|
|
8
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
9
|
+
var MAX_INDEX_NAME_LENGTH = 255;
|
|
10
|
+
var MAX_DOC_ID_LENGTH = 512;
|
|
11
|
+
var MIN_SWEEP_INTERVAL_MS = 100;
|
|
12
|
+
var MAX_SWEEP_INTERVAL_MS = 6e4;
|
|
13
|
+
var HEAP_PRESSURE_WARNING_FRACTION = 0.9;
|
|
14
|
+
var HEAP_PRESSURE_REARM_FRACTION = 0.8;
|
|
15
|
+
var STATISTICS_BROADCAST_INTERVAL_MS = 5e3;
|
|
16
|
+
var FOREIGN_STATS_TTL_MS = 6e4;
|
|
17
|
+
var DEFAULT_SUGGEST_LIMIT = 10;
|
|
18
|
+
var MAX_SUGGEST_LIMIT = 100;
|
|
19
|
+
var SUGGEST_OVERSAMPLE_FACTOR = 1.5;
|
|
20
|
+
var SUGGEST_OVERSAMPLE_PADDING = 10;
|
|
21
|
+
var MAX_SUGGEST_SCATTER_LIMIT = Math.ceil(MAX_SUGGEST_LIMIT * SUGGEST_OVERSAMPLE_FACTOR) + SUGGEST_OVERSAMPLE_PADDING;
|
|
22
|
+
|
|
23
|
+
export { BATCH_CHUNK_SIZE, DEFAULT_LIMIT, DEFAULT_OFFSET, DEFAULT_SUGGEST_LIMIT, FOREIGN_STATS_TTL_MS, HEAP_PRESSURE_REARM_FRACTION, HEAP_PRESSURE_WARNING_FRACTION, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MAX_INDEX_NAME_LENGTH, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_SWEEP_INTERVAL_MS, MIN_DOCUMENTS_FOR_SEGMENTS, MIN_SWEEP_INTERVAL_MS, STATISTICS_BROADCAST_INTERVAL_MS, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING };
|