@delali/narsil 0.3.1-next.72 → 0.3.1-next.73
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-D6gjjQyA.d.ts → capabilities-CYeAWYxQ.d.ts} +19 -4
- package/dist/{chunk-Y52WZWAU.mjs → chunk-2WLD2NFW.mjs} +4 -4
- package/dist/{chunk-QR4OKRVY.mjs → chunk-3YKWSO54.mjs} +8 -3
- package/dist/{chunk-KZTI25D3.mjs → chunk-4MQ5F4IL.mjs} +1 -1
- package/dist/{chunk-CVPA5Q7R.mjs → chunk-AFKH4ZK3.mjs} +9 -6
- package/dist/{chunk-M6XF5HDE.mjs → chunk-AK3BLGCI.mjs} +141 -11
- package/dist/{chunk-7QMTA7AA.mjs → chunk-CVFAFZSN.mjs} +7771 -4271
- package/dist/{chunk-RV77FR2F.mjs → chunk-I64LXKO3.mjs} +0 -1
- package/dist/{chunk-EDMCZLSJ.mjs → chunk-LPBWCHBE.mjs} +193 -13
- package/dist/{chunk-P2ZZN32C.mjs → chunk-PU43WDY4.mjs} +4 -2
- package/dist/{chunk-HVJYTKK7.mjs → chunk-THMSFOJO.mjs} +365 -774
- package/dist/{chunk-LCCL6EEH.mjs → chunk-WVDFG45M.mjs} +92 -94
- package/dist/{chunk-3HQPUCCT.mjs → chunk-XW25X6AU.mjs} +66 -237
- package/dist/{chunk-2X2ZQOAB.mjs → chunk-ZULSEZCE.mjs} +3 -4
- package/dist/{client-C2TiW29y.d.ts → client-D698E5ge.d.ts} +76 -25
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +44 -22
- package/dist/{config-BLguB-om.d.ts → config-Bv8etwUZ.d.ts} +10 -3
- package/dist/{config-C7Ip8Ife.d.ts → config-QXU3HMpq.d.ts} +10 -1
- package/dist/direct-executor-5SRDFZAS.mjs +13 -0
- package/dist/distribution/coordinator/etcd.d.ts +27 -14
- package/dist/distribution/coordinator/etcd.mjs +162 -75
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +22 -2
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +6 -3
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +40 -16
- package/dist/distribution.d.ts +6 -6
- package/dist/distribution.mjs +637 -366
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-Dm4446j1.d.ts → engine-D_OzKZ-6.d.ts} +96 -187
- package/dist/index.browser.mjs +7179 -4031
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +11 -14
- package/dist/{memory-CKHTU1CL.d.ts → memory-B4jMmygw.d.ts} +213 -2
- package/dist/pattern.mjs +1 -1
- package/dist/persistence/durability/checkpoint-worker.d.ts +24 -16
- package/dist/persistence/durability/checkpoint-worker.mjs +22 -16
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +0 -1
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +210 -32
- package/dist/{tasks-BECd0b__.d.ts → tasks-DpgCpW1x.d.ts} +3 -3
- package/dist/{thread-IBSIZZ6W.mjs → thread-3LNVA43V.mjs} +79 -21
- package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
- package/dist/vector/search-worker.mjs +3 -3
- package/dist/workers/entry.mjs +4 -4
- package/package.json +7 -7
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-XZ4PAREY.mjs +0 -25
- package/dist/direct-executor-SVCHXR74.mjs +0 -14
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
|
|
2
|
-
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-
|
|
2
|
+
import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-PU43WDY4.mjs';
|
|
3
3
|
import { compareCodePoints } from './chunk-LJYTW7GT.mjs';
|
|
4
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
5
5
|
import { readSync, closeSync, openSync } from 'fs';
|
|
6
6
|
|
|
7
7
|
// src/vector/constants.ts
|
|
@@ -3651,6 +3651,13 @@ function createVectorStore(options) {
|
|
|
3651
3651
|
yield [docId, entryAt(open, ordinal)];
|
|
3652
3652
|
}
|
|
3653
3653
|
},
|
|
3654
|
+
*docIds() {
|
|
3655
|
+
if (open === null) return;
|
|
3656
|
+
for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
|
|
3657
|
+
const docId = ordToDoc[ordinal];
|
|
3658
|
+
if (docId !== void 0) yield docId;
|
|
3659
|
+
}
|
|
3660
|
+
},
|
|
3654
3661
|
clear() {
|
|
3655
3662
|
forgetDocuments();
|
|
3656
3663
|
if (open === null) return;
|
|
@@ -3789,30 +3796,24 @@ function savedTopLayer(state, ordinal) {
|
|
|
3789
3796
|
if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
|
|
3790
3797
|
return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
|
|
3791
3798
|
}
|
|
3792
|
-
function
|
|
3799
|
+
function neighbourValueCount(state, topLayers, start, end) {
|
|
3793
3800
|
let values = 0;
|
|
3794
|
-
for (let number =
|
|
3801
|
+
for (let number = start; number < end; number++) {
|
|
3795
3802
|
const top = topLayers[number];
|
|
3796
3803
|
if (top < 0) continue;
|
|
3797
3804
|
values += state.adjacency.level0Stride + top * state.adjacency.upperStride;
|
|
3798
3805
|
}
|
|
3799
3806
|
return values;
|
|
3800
3807
|
}
|
|
3801
|
-
function
|
|
3808
|
+
function serializePart(state, numbers, start, end) {
|
|
3809
|
+
const { numberOfOrdinal, ordinalOfNumber, topLayers } = numbers;
|
|
3802
3810
|
const adjacency = state.adjacency;
|
|
3803
|
-
const
|
|
3804
|
-
const
|
|
3805
|
-
const levels = new Uint8Array(ordinalOfNumber.length);
|
|
3806
|
-
for (let number = 0; number < ordinalOfNumber.length; number++) {
|
|
3807
|
-
const ordinal = ordinalOfNumber[number];
|
|
3808
|
-
if (ordinal >= slots) continue;
|
|
3809
|
-
topLayers[number] = savedTopLayer(state, ordinal);
|
|
3810
|
-
levels[number] = topLayers[number] + 1;
|
|
3811
|
-
}
|
|
3812
|
-
const values = new Uint32Array(largestNeighbourValueCount(state, topLayers));
|
|
3811
|
+
const levels = new Uint8Array(end - start);
|
|
3812
|
+
const values = new Uint32Array(neighbourValueCount(state, topLayers, start, end));
|
|
3813
3813
|
let cursor = 0;
|
|
3814
|
-
for (let number =
|
|
3814
|
+
for (let number = start; number < end; number++) {
|
|
3815
3815
|
const top = topLayers[number];
|
|
3816
|
+
levels[number - start] = top + 1;
|
|
3816
3817
|
for (let layer = 0; layer <= top; layer++) {
|
|
3817
3818
|
const base = layerBase(adjacency, ordinalOfNumber[number], layer);
|
|
3818
3819
|
const array = layerArray(adjacency, layer);
|
|
@@ -3830,18 +3831,33 @@ function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
|
|
|
3830
3831
|
values[countAt] = cursor - countAt - 1;
|
|
3831
3832
|
}
|
|
3832
3833
|
}
|
|
3834
|
+
return { levels, neighbours: littleEndianBytesOf(values.subarray(0, cursor)) };
|
|
3835
|
+
}
|
|
3836
|
+
function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize) {
|
|
3837
|
+
const slots = adjacencySlots(state.adjacency);
|
|
3838
|
+
const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
|
|
3839
|
+
for (let number = 0; number < ordinalOfNumber.length; number++) {
|
|
3840
|
+
const ordinal = ordinalOfNumber[number];
|
|
3841
|
+
if (ordinal >= slots) continue;
|
|
3842
|
+
topLayers[number] = savedTopLayer(state, ordinal);
|
|
3843
|
+
}
|
|
3833
3844
|
const entryOrdinal = entryPointOf(state);
|
|
3834
3845
|
const entryNumber = entryOrdinal >= 0 && entryOrdinal < numberOfOrdinal.length ? numberOfOrdinal[entryOrdinal] : NO_NUMBER;
|
|
3835
3846
|
const entryHoldsNode = entryNumber !== NO_NUMBER && topLayers[entryNumber] >= 0;
|
|
3836
|
-
|
|
3847
|
+
const header = {
|
|
3837
3848
|
entryPoint: entryHoldsNode ? entryNumber : null,
|
|
3838
3849
|
maxLayer: Math.max(0, Math.min(topLayerOf(state), MAX_LAYER_CAP)),
|
|
3839
3850
|
m: state.M,
|
|
3840
3851
|
efConstruction: state.efCons,
|
|
3841
|
-
metric: state.buildMetric
|
|
3842
|
-
levels,
|
|
3843
|
-
neighbours: littleEndianBytesOf(values.subarray(0, cursor))
|
|
3852
|
+
metric: state.buildMetric
|
|
3844
3853
|
};
|
|
3854
|
+
const numbers = { numberOfOrdinal, ordinalOfNumber, topLayers };
|
|
3855
|
+
const parts = [];
|
|
3856
|
+
for (let start = 0; start < ordinalOfNumber.length; start += partSize) {
|
|
3857
|
+
const end = Math.min(start + partSize, ordinalOfNumber.length);
|
|
3858
|
+
parts.push({ ...header, ...serializePart(state, numbers, start, end) });
|
|
3859
|
+
}
|
|
3860
|
+
return parts;
|
|
3845
3861
|
}
|
|
3846
3862
|
function endsInsideAList() {
|
|
3847
3863
|
throw new NarsilError(
|
|
@@ -3862,95 +3878,78 @@ function requireWholeNeighbourLists(graph) {
|
|
|
3862
3878
|
}
|
|
3863
3879
|
}
|
|
3864
3880
|
}
|
|
3865
|
-
function
|
|
3866
|
-
|
|
3881
|
+
function topLayerInParts(graph, number) {
|
|
3882
|
+
const part = graph.parts[Math.floor(number / graph.partSize)];
|
|
3883
|
+
if (part === void 0) return -1;
|
|
3884
|
+
const at = number % graph.partSize;
|
|
3885
|
+
return at < part.levels.length ? part.levels[at] - 1 : -1;
|
|
3886
|
+
}
|
|
3887
|
+
function highestNodeOrdinal(graph, ordinalAt) {
|
|
3888
|
+
let highest = -1;
|
|
3889
|
+
for (let p = 0; p < graph.parts.length; p++) {
|
|
3890
|
+
const levels = graph.parts[p].levels;
|
|
3891
|
+
for (let at = 0; at < levels.length; at++) {
|
|
3892
|
+
const ordinal = ordinalAt(p * graph.partSize + at);
|
|
3893
|
+
if (levels[at] > 0 && ordinal > highest) highest = ordinal;
|
|
3894
|
+
}
|
|
3895
|
+
}
|
|
3896
|
+
return highest;
|
|
3867
3897
|
}
|
|
3868
3898
|
function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
|
|
3869
|
-
|
|
3899
|
+
for (const part of graph.parts) requireWholeNeighbourLists(part);
|
|
3870
3900
|
resetGraph(state);
|
|
3871
|
-
const values = valuesOf(graph.neighbours);
|
|
3872
3901
|
const ordinalAt = (number) => number < ordinalOfNumber.length ? ordinalOfNumber[number] : NO_ORDINAL;
|
|
3873
|
-
|
|
3874
|
-
for (let number = 0; number < graph.levels.length; number++) {
|
|
3875
|
-
if (graph.levels[number] > 0 && ordinalAt(number) > highestOrdinal) highestOrdinal = ordinalAt(number);
|
|
3876
|
-
}
|
|
3902
|
+
const highestOrdinal = highestNodeOrdinal(graph, ordinalAt);
|
|
3877
3903
|
if (highestOrdinal >= 0) ensureCapacity(state, highestOrdinal + 1);
|
|
3878
3904
|
const kept = [];
|
|
3879
|
-
let cursor = 0;
|
|
3880
3905
|
let highestLayer = -1;
|
|
3881
3906
|
let nodeOnHighestLayer = NO_ORDINAL;
|
|
3882
|
-
for (let
|
|
3883
|
-
const
|
|
3884
|
-
|
|
3885
|
-
|
|
3886
|
-
|
|
3887
|
-
|
|
3888
|
-
|
|
3889
|
-
|
|
3890
|
-
|
|
3891
|
-
|
|
3892
|
-
|
|
3893
|
-
|
|
3894
|
-
|
|
3895
|
-
|
|
3896
|
-
|
|
3897
|
-
|
|
3898
|
-
|
|
3899
|
-
for (let i = 1; i <= count && kept.length < limit; i++) {
|
|
3900
|
-
const neighbour = values[cursor + i];
|
|
3901
|
-
const neighbourOrdinal = ordinalAt(neighbour);
|
|
3902
|
-
if (neighbourOrdinal === NO_ORDINAL || neighbour === number || topLayerIn(graph, neighbour) < layer) continue;
|
|
3903
|
-
if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
|
|
3907
|
+
for (let p = 0; p < graph.parts.length; p++) {
|
|
3908
|
+
const part = graph.parts[p];
|
|
3909
|
+
const values = valuesOf(part.neighbours);
|
|
3910
|
+
let cursor = 0;
|
|
3911
|
+
for (let at = 0; at < part.levels.length; at++) {
|
|
3912
|
+
const number = p * graph.partSize + at;
|
|
3913
|
+
const top = part.levels[at] - 1;
|
|
3914
|
+
if (top < 0) continue;
|
|
3915
|
+
const ordinal = ordinalAt(number);
|
|
3916
|
+
const keptTop = Math.min(top, MAX_LAYER_CAP);
|
|
3917
|
+
if (ordinal !== NO_ORDINAL) {
|
|
3918
|
+
createNode(state.adjacency, ordinal, keptTop);
|
|
3919
|
+
Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
|
|
3920
|
+
if (keptTop > highestLayer) {
|
|
3921
|
+
highestLayer = keptTop;
|
|
3922
|
+
nodeOnHighestLayer = ordinal;
|
|
3923
|
+
}
|
|
3904
3924
|
}
|
|
3905
|
-
|
|
3906
|
-
|
|
3907
|
-
|
|
3925
|
+
for (let layer = 0; layer <= top; layer++) {
|
|
3926
|
+
const count = values[cursor];
|
|
3927
|
+
kept.length = 0;
|
|
3928
|
+
const limit = maxConns(state, layer);
|
|
3929
|
+
for (let i = 1; i <= count && kept.length < limit; i++) {
|
|
3930
|
+
const neighbour = values[cursor + i];
|
|
3931
|
+
const neighbourOrdinal = ordinalAt(neighbour);
|
|
3932
|
+
if (neighbourOrdinal === NO_ORDINAL || neighbour === number) continue;
|
|
3933
|
+
if (topLayerInParts(graph, neighbour) < layer) continue;
|
|
3934
|
+
if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
|
|
3935
|
+
}
|
|
3936
|
+
cursor += count + 1;
|
|
3937
|
+
if (ordinal !== NO_ORDINAL && layer <= keptTop) {
|
|
3938
|
+
replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
|
|
3939
|
+
}
|
|
3908
3940
|
}
|
|
3909
3941
|
}
|
|
3910
3942
|
}
|
|
3911
|
-
const
|
|
3912
|
-
const
|
|
3943
|
+
const first = graph.parts[0];
|
|
3944
|
+
const entryPoint = first === void 0 ? null : first.entryPoint;
|
|
3945
|
+
const entryOrdinal = entryPoint === null ? NO_ORDINAL : ordinalAt(entryPoint);
|
|
3946
|
+
const entryTop = entryPoint === null ? -1 : Math.min(topLayerInParts(graph, entryPoint), MAX_LAYER_CAP);
|
|
3913
3947
|
const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
|
|
3914
3948
|
Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
|
|
3915
3949
|
Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
|
|
3916
3950
|
}
|
|
3917
3951
|
|
|
3918
3952
|
// src/vector/hnsw/persistence.ts
|
|
3919
|
-
function serializeGraph(state, store) {
|
|
3920
|
-
const nodeArray = [];
|
|
3921
|
-
const slots = adjacencySlots(state.adjacency);
|
|
3922
|
-
for (let ord = 0; ord < slots; ord++) {
|
|
3923
|
-
const maxLayer = nodeMaxLayer(state, ord);
|
|
3924
|
-
if (maxLayer === -1) continue;
|
|
3925
|
-
if (isTombstoned(state, ord)) continue;
|
|
3926
|
-
const docId = store.docIdForOrdinal(ord);
|
|
3927
|
-
if (docId === void 0) continue;
|
|
3928
|
-
const layerConns = [];
|
|
3929
|
-
for (let layer = 0; layer <= maxLayer; layer++) {
|
|
3930
|
-
const liveNeighbors = [];
|
|
3931
|
-
for (const neighborOrd of collectNeighbors(state.adjacency, ord, layer)) {
|
|
3932
|
-
if (isTombstoned(state, neighborOrd)) continue;
|
|
3933
|
-
const neighborDoc = store.docIdForOrdinal(neighborOrd);
|
|
3934
|
-
if (neighborDoc === void 0) continue;
|
|
3935
|
-
liveNeighbors.push(neighborDoc);
|
|
3936
|
-
}
|
|
3937
|
-
if (liveNeighbors.length > 0) {
|
|
3938
|
-
layerConns.push([layer, liveNeighbors]);
|
|
3939
|
-
}
|
|
3940
|
-
}
|
|
3941
|
-
nodeArray.push([docId, maxLayer, layerConns]);
|
|
3942
|
-
}
|
|
3943
|
-
const entryOrd = entryPointOf(state);
|
|
3944
|
-
const entryPoint = entryOrd === -1 ? null : store.docIdForOrdinal(entryOrd) ?? null;
|
|
3945
|
-
return {
|
|
3946
|
-
entryPoint,
|
|
3947
|
-
maxLayer: topLayerOf(state),
|
|
3948
|
-
m: state.M,
|
|
3949
|
-
efConstruction: state.efCons,
|
|
3950
|
-
metric: state.buildMetric,
|
|
3951
|
-
nodes: nodeArray
|
|
3952
|
-
};
|
|
3953
|
-
}
|
|
3954
3953
|
function resolveNodes(state, store, data) {
|
|
3955
3954
|
const nodes = [];
|
|
3956
3955
|
let maxOrd = -1;
|
|
@@ -4186,13 +4185,12 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
|
|
|
4186
4185
|
compactTombstones: () => exclusively(() => compactTombstones(state)),
|
|
4187
4186
|
rebuild: () => exclusively(() => rebuild(state)),
|
|
4188
4187
|
exclusively,
|
|
4189
|
-
serialize: () => serializeGraph(state, store),
|
|
4190
4188
|
deserialize: (data) => exclusively(() => deserializeGraph(state, store, data)),
|
|
4191
|
-
serializeNumbered: (numberOfOrdinal, ordinalOfNumber) =>
|
|
4189
|
+
serializeNumbered: (numberOfOrdinal, ordinalOfNumber, partSize) => serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize),
|
|
4192
4190
|
deserializeNumbered: (graph, ordinalOfNumber) => exclusively(() => deserializeNumberedGraph(state, graph, ordinalOfNumber)),
|
|
4193
4191
|
exportSnapshot: () => exportSnapshot(state),
|
|
4194
4192
|
restoreSnapshot: (snapshot) => exclusively(() => restoreSnapshot(state, snapshot))
|
|
4195
4193
|
};
|
|
4196
4194
|
}
|
|
4197
4195
|
|
|
4198
|
-
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };
|
|
4196
|
+
export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
|
|
@@ -1,55 +1,6 @@
|
|
|
1
|
-
import {
|
|
2
|
-
|
|
3
|
-
// src/core/pattern-index/matcher-budget.ts
|
|
4
|
-
var POSITIONS_PER_WORD = 32;
|
|
5
|
-
function positionWords(positions) {
|
|
6
|
-
return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
|
|
7
|
-
}
|
|
8
|
-
function automatonBytes(positions) {
|
|
9
|
-
return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
|
|
10
|
-
}
|
|
11
|
-
function smallestMatcherBytes(positions) {
|
|
12
|
-
return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
|
|
13
|
-
}
|
|
14
|
-
function widestPatternSplit(positions) {
|
|
15
|
-
return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
|
|
16
|
-
}
|
|
17
|
-
|
|
18
|
-
// src/pattern/automaton.ts
|
|
19
|
-
function setPosition(words, position) {
|
|
20
|
-
words[position >>> 5] |= 1 << (position & 31);
|
|
21
|
-
}
|
|
22
|
-
function hasPosition(words, position) {
|
|
23
|
-
return (words[position >>> 5] & 1 << (position & 31)) !== 0;
|
|
24
|
-
}
|
|
25
|
-
function literalAutomaton(codePoints, anchoredStart, anchoredEnd) {
|
|
26
|
-
const size = codePoints.length;
|
|
27
|
-
const words = positionWords(size);
|
|
28
|
-
const first = new Uint32Array(words);
|
|
29
|
-
const last = new Uint32Array(words);
|
|
30
|
-
if (size > 0) {
|
|
31
|
-
setPosition(first, 0);
|
|
32
|
-
setPosition(last, size - 1);
|
|
33
|
-
}
|
|
34
|
-
return {
|
|
35
|
-
size,
|
|
36
|
-
words,
|
|
37
|
-
first,
|
|
38
|
-
last,
|
|
39
|
-
nullable: size === 0,
|
|
40
|
-
anchoredStart,
|
|
41
|
-
anchoredEnd,
|
|
42
|
-
bytes: automatonBytes(size),
|
|
43
|
-
accepts: (position, codePoint) => codePoints[position] === codePoint,
|
|
44
|
-
followerCount: (position) => position + 1 < size ? 1 : 0,
|
|
45
|
-
follower: (position) => position + 1
|
|
46
|
-
};
|
|
47
|
-
}
|
|
1
|
+
import { bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS } from './chunk-ZKRV5M2P.mjs';
|
|
48
2
|
|
|
49
3
|
// src/pattern/constants.ts
|
|
50
|
-
var MATCHER_TRANSITION_BYTES = 48;
|
|
51
|
-
var MATCHER_KEY_CHARACTER_BYTES = 2;
|
|
52
|
-
var MATCHER_MIN_CODE_POINTS_PER_STATE = 10;
|
|
53
4
|
var SHORTLIST_RUN_LOOKUPS = 16;
|
|
54
5
|
var SHORTLIST_INTERSECT_RATIO = 8;
|
|
55
6
|
var RUN_LIST_INITIAL_CAPACITY = 4;
|
|
@@ -57,205 +8,86 @@ var RUN_LIST_COMPACTION_FRACTION = 0.5;
|
|
|
57
8
|
var RUN_ENTRY_OVERHEAD_BYTES = 140;
|
|
58
9
|
var DOCUMENT_ENTRY_BYTES = 24;
|
|
59
10
|
|
|
60
|
-
// src/pattern/
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
11
|
+
// src/pattern/literal-matcher.ts
|
|
12
|
+
function bordersOf(codePoints) {
|
|
13
|
+
const length = codePoints.length;
|
|
14
|
+
const borders = new Uint32Array(length + 1);
|
|
15
|
+
let border = 0;
|
|
16
|
+
for (let at = 1; at < length; at++) {
|
|
17
|
+
while (border > 0 && codePoints[at] !== codePoints[border]) border = borders[border];
|
|
18
|
+
if (codePoints[at] === codePoints[border]) border++;
|
|
19
|
+
borders[at + 1] = border;
|
|
20
|
+
}
|
|
21
|
+
return borders;
|
|
22
|
+
}
|
|
23
|
+
function reachablePositionsOf(borders, length) {
|
|
24
|
+
const reachable = new Uint32Array(length + 1);
|
|
25
|
+
for (let prefix = 1; prefix <= length; prefix++) {
|
|
26
|
+
reachable[prefix] = (prefix < length ? 1 : 0) + reachable[borders[prefix]];
|
|
27
|
+
}
|
|
28
|
+
return reachable;
|
|
29
|
+
}
|
|
30
|
+
var LiteralMatcher = class {
|
|
31
|
+
constructor(codePoints, anchoredStart, anchoredEnd) {
|
|
32
|
+
this.codePoints = codePoints;
|
|
33
|
+
this.anchoredStart = anchoredStart;
|
|
34
|
+
this.anchoredEnd = anchoredEnd;
|
|
35
|
+
this.borders = bordersOf(codePoints);
|
|
36
|
+
this.reachable = reachablePositionsOf(this.borders, codePoints.length);
|
|
37
|
+
}
|
|
38
|
+
borders;
|
|
39
|
+
reachable;
|
|
40
|
+
prefix = 0;
|
|
41
|
+
alive = true;
|
|
74
42
|
accepting = false;
|
|
75
|
-
active;
|
|
76
|
-
following;
|
|
77
|
-
marks;
|
|
78
|
-
first;
|
|
79
|
-
activeCount = 0;
|
|
80
|
-
stamp = 0;
|
|
81
|
-
get steps() {
|
|
82
|
-
return this.activeCount;
|
|
83
|
-
}
|
|
84
|
-
get bytes() {
|
|
85
|
-
return this.automaton.size * MATCHER_SIMULATION_BYTES_PER_POSITION;
|
|
86
|
-
}
|
|
87
|
-
reset() {
|
|
88
|
-
this.active.set(this.first);
|
|
89
|
-
this.activeCount = this.first.length;
|
|
90
|
-
this.accepting = this.automaton.nullable;
|
|
91
|
-
}
|
|
92
|
-
get firstPositions() {
|
|
93
|
-
return this.first;
|
|
94
|
-
}
|
|
95
|
-
load(active, accepting) {
|
|
96
|
-
this.active.set(active);
|
|
97
|
-
this.activeCount = active.length;
|
|
98
|
-
this.accepting = accepting;
|
|
99
|
-
}
|
|
100
|
-
sortedActive() {
|
|
101
|
-
return this.active.slice(0, this.activeCount).sort();
|
|
102
|
-
}
|
|
103
|
-
advance(codePoint) {
|
|
104
|
-
const { automaton, marks } = this;
|
|
105
|
-
const following = this.following;
|
|
106
|
-
const stamp = this.nextStamp();
|
|
107
|
-
let count = 0;
|
|
108
|
-
let accepting = false;
|
|
109
|
-
for (let at = 0; at < this.activeCount; at++) {
|
|
110
|
-
const position = this.active[at];
|
|
111
|
-
if (!automaton.accepts(position, codePoint)) continue;
|
|
112
|
-
if (hasPosition(automaton.last, position)) accepting = true;
|
|
113
|
-
const followers = automaton.followerCount(position);
|
|
114
|
-
for (let index = 0; index < followers; index++) {
|
|
115
|
-
const follower = automaton.follower(position, index);
|
|
116
|
-
if (marks[follower] === stamp) continue;
|
|
117
|
-
marks[follower] = stamp;
|
|
118
|
-
following[count++] = follower;
|
|
119
|
-
}
|
|
120
|
-
}
|
|
121
|
-
if (!automaton.anchoredStart) {
|
|
122
|
-
for (const position of this.first) {
|
|
123
|
-
if (marks[position] === stamp) continue;
|
|
124
|
-
marks[position] = stamp;
|
|
125
|
-
following[count++] = position;
|
|
126
|
-
}
|
|
127
|
-
if (automaton.nullable) accepting = true;
|
|
128
|
-
}
|
|
129
|
-
this.following = this.active;
|
|
130
|
-
this.active = following;
|
|
131
|
-
this.activeCount = count;
|
|
132
|
-
this.accepting = accepting;
|
|
133
|
-
}
|
|
134
|
-
nextStamp() {
|
|
135
|
-
if (this.stamp === LAST_STAMP) {
|
|
136
|
-
this.marks.fill(0);
|
|
137
|
-
this.stamp = 0;
|
|
138
|
-
}
|
|
139
|
-
this.stamp++;
|
|
140
|
-
return this.stamp;
|
|
141
|
-
}
|
|
142
|
-
};
|
|
143
|
-
|
|
144
|
-
// src/pattern/lazy-dfa.ts
|
|
145
|
-
var UNCACHED_GENERATION = -1;
|
|
146
|
-
function stateKey(active, accepting) {
|
|
147
|
-
return `${active.join(",")}${accepting ? "+" : "-"}`;
|
|
148
|
-
}
|
|
149
|
-
function stateBytes(active, key) {
|
|
150
|
-
return MATCHER_STATE_OVERHEAD_BYTES + active.byteLength + key.length * MATCHER_KEY_CHARACTER_BYTES;
|
|
151
|
-
}
|
|
152
|
-
var LazyDfa = class {
|
|
153
|
-
constructor(automaton, stateLimitBytes) {
|
|
154
|
-
this.automaton = automaton;
|
|
155
|
-
this.simulation = new PositionSimulation(automaton);
|
|
156
|
-
this.cacheLimitBytes = stateLimitBytes - automaton.bytes - this.simulation.bytes;
|
|
157
|
-
this.start = this.intern(this.simulation.firstPositions, automaton.nullable);
|
|
158
|
-
this.state = this.start;
|
|
159
|
-
if (this.wantsSimulation()) this.simulate(this.start);
|
|
160
|
-
}
|
|
161
|
-
states = /* @__PURE__ */ new Map();
|
|
162
|
-
generation = 0;
|
|
163
|
-
cachedBytes = 0;
|
|
164
|
-
codePointsSinceReset = 0;
|
|
165
|
-
start;
|
|
166
|
-
state;
|
|
167
|
-
simulating = false;
|
|
168
43
|
decided = null;
|
|
169
|
-
simulation;
|
|
170
|
-
cacheLimitBytes;
|
|
171
44
|
begin() {
|
|
172
45
|
this.decided = null;
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
46
|
+
this.prefix = 0;
|
|
47
|
+
this.alive = this.codePoints.length > 0;
|
|
48
|
+
this.accepting = this.codePoints.length === 0;
|
|
49
|
+
if (!this.anchoredEnd && this.accepting) return this.decide(true);
|
|
176
50
|
return true;
|
|
177
51
|
}
|
|
178
52
|
feed(codePoint, tally) {
|
|
179
|
-
const steps = this.
|
|
180
|
-
if (steps === 0) return this.decide(!this.
|
|
53
|
+
const steps = this.steps();
|
|
54
|
+
if (steps === 0) return this.decide(!this.anchoredStart && this.codePoints.length === 0);
|
|
181
55
|
tally.add(steps);
|
|
182
|
-
if (this.
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
this.codePointsSinceReset++;
|
|
186
|
-
this.state = this.step(this.state, codePoint);
|
|
187
|
-
if (this.wantsSimulation()) this.simulate(this.state);
|
|
188
|
-
}
|
|
189
|
-
if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
|
|
56
|
+
if (this.anchoredStart) this.advanceAnchored(codePoint);
|
|
57
|
+
else this.advance(codePoint);
|
|
58
|
+
if (!this.anchoredEnd && this.accepting) return this.decide(true);
|
|
190
59
|
return true;
|
|
191
60
|
}
|
|
192
61
|
matched() {
|
|
193
|
-
return this.decided ?? (this.
|
|
62
|
+
return this.decided ?? (this.anchoredEnd && this.accepting);
|
|
63
|
+
}
|
|
64
|
+
steps() {
|
|
65
|
+
const length = this.codePoints.length;
|
|
66
|
+
if (length === 0) return 0;
|
|
67
|
+
if (this.anchoredStart) return this.alive && this.prefix < length ? 1 : 0;
|
|
68
|
+
return 1 + this.reachable[this.prefix];
|
|
69
|
+
}
|
|
70
|
+
advanceAnchored(codePoint) {
|
|
71
|
+
if (this.codePoints[this.prefix] === codePoint) {
|
|
72
|
+
this.prefix++;
|
|
73
|
+
this.accepting = this.prefix === this.codePoints.length;
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.alive = false;
|
|
77
|
+
this.accepting = false;
|
|
194
78
|
}
|
|
195
|
-
|
|
196
|
-
|
|
79
|
+
advance(codePoint) {
|
|
80
|
+
const { codePoints, borders } = this;
|
|
81
|
+
let prefix = this.prefix === codePoints.length ? borders[this.prefix] : this.prefix;
|
|
82
|
+
while (prefix > 0 && codePoints[prefix] !== codePoint) prefix = borders[prefix];
|
|
83
|
+
if (codePoints[prefix] === codePoint) prefix++;
|
|
84
|
+
this.prefix = prefix;
|
|
85
|
+
this.accepting = prefix === codePoints.length;
|
|
197
86
|
}
|
|
198
87
|
decide(outcome) {
|
|
199
88
|
this.decided = outcome;
|
|
200
89
|
return false;
|
|
201
90
|
}
|
|
202
|
-
wantsSimulation() {
|
|
203
|
-
return !this.simulating && this.state.generation === UNCACHED_GENERATION;
|
|
204
|
-
}
|
|
205
|
-
simulate(from) {
|
|
206
|
-
this.dropCache();
|
|
207
|
-
this.state = this.start;
|
|
208
|
-
this.simulation.load(from.active, from.accepting);
|
|
209
|
-
this.simulating = true;
|
|
210
|
-
}
|
|
211
|
-
dropCache() {
|
|
212
|
-
for (const state of this.states.values()) state.next.clear();
|
|
213
|
-
this.start.next.clear();
|
|
214
|
-
this.state.next.clear();
|
|
215
|
-
this.states = /* @__PURE__ */ new Map();
|
|
216
|
-
this.cachedBytes = 0;
|
|
217
|
-
}
|
|
218
|
-
step(state, codePoint) {
|
|
219
|
-
const current = state.generation === this.generation;
|
|
220
|
-
if (current) {
|
|
221
|
-
const cached = state.next.get(codePoint);
|
|
222
|
-
if (cached !== void 0) return cached;
|
|
223
|
-
}
|
|
224
|
-
const { simulation } = this;
|
|
225
|
-
simulation.load(state.active, state.accepting);
|
|
226
|
-
simulation.advance(codePoint);
|
|
227
|
-
const next = this.intern(simulation.sortedActive(), simulation.accepting);
|
|
228
|
-
if (!current || state.generation !== this.generation || next.generation !== this.generation) return next;
|
|
229
|
-
if (!this.reserve(MATCHER_TRANSITION_BYTES) || state.generation !== this.generation) return next;
|
|
230
|
-
state.next.set(codePoint, next);
|
|
231
|
-
return next;
|
|
232
|
-
}
|
|
233
|
-
intern(active, accepting) {
|
|
234
|
-
const key = stateKey(active, accepting);
|
|
235
|
-
const existing = this.states.get(key);
|
|
236
|
-
if (existing !== void 0) return existing;
|
|
237
|
-
if (!this.reserve(stateBytes(active, key))) {
|
|
238
|
-
return { active, accepting, generation: UNCACHED_GENERATION, next: /* @__PURE__ */ new Map() };
|
|
239
|
-
}
|
|
240
|
-
const state = { active, accepting, generation: this.generation, next: /* @__PURE__ */ new Map() };
|
|
241
|
-
this.states.set(key, state);
|
|
242
|
-
return state;
|
|
243
|
-
}
|
|
244
|
-
reserve(bytes) {
|
|
245
|
-
if (this.cachedBytes + bytes <= this.cacheLimitBytes) {
|
|
246
|
-
this.cachedBytes += bytes;
|
|
247
|
-
return true;
|
|
248
|
-
}
|
|
249
|
-
if (this.states.size === 0 || this.codePointsSinceReset < MATCHER_MIN_CODE_POINTS_PER_STATE * this.states.size) {
|
|
250
|
-
return false;
|
|
251
|
-
}
|
|
252
|
-
this.dropCache();
|
|
253
|
-
this.codePointsSinceReset = 0;
|
|
254
|
-
this.generation++;
|
|
255
|
-
this.start = this.intern(this.start.active, this.start.accepting);
|
|
256
|
-
if (this.start.generation === UNCACHED_GENERATION) return false;
|
|
257
|
-
return this.reserve(bytes);
|
|
258
|
-
}
|
|
259
91
|
};
|
|
260
92
|
|
|
261
93
|
// src/pattern/text.ts
|
|
@@ -391,10 +223,7 @@ function matcherFor(request) {
|
|
|
391
223
|
const { operator, text, caseFold, host } = request;
|
|
392
224
|
const testCodePoints = [];
|
|
393
225
|
appendTextCodePoints(text, caseFold, host, testCodePoints);
|
|
394
|
-
return new
|
|
395
|
-
literalAutomaton(testCodePoints, ANCHORED_START.has(operator), ANCHORED_END.has(operator)),
|
|
396
|
-
request.meter.matcherStateBytes
|
|
397
|
-
);
|
|
226
|
+
return new LiteralMatcher(Uint32Array.from(testCodePoints), ANCHORED_START.has(operator), ANCHORED_END.has(operator));
|
|
398
227
|
}
|
|
399
228
|
function matchesText(matcher, text, caseFold, host, folded, tally) {
|
|
400
229
|
if (!matcher.begin()) return matcher.matched();
|
|
@@ -782,4 +611,4 @@ var patternSearchModule = {
|
|
|
782
611
|
matchInto: matchPattern
|
|
783
612
|
};
|
|
784
613
|
|
|
785
|
-
export { GrowableUint32, patternSearchModule
|
|
614
|
+
export { GrowableUint32, patternSearchModule };
|
|
@@ -1,9 +1,8 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
3
|
-
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-M6XF5HDE.mjs';
|
|
1
|
+
import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-3YKWSO54.mjs';
|
|
2
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_OFFSET, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-AK3BLGCI.mjs';
|
|
4
3
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
5
4
|
import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys } from './chunk-LJYTW7GT.mjs';
|
|
6
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
5
|
+
import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
|
|
7
6
|
|
|
8
7
|
// src/search/cursor-binding.ts
|
|
9
8
|
var ABSENT_TAG = 0;
|