@delali/narsil 0.3.1-next.72 → 0.3.1-next.73

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. package/README.md +2 -2
  2. package/dist/adapters/filesystem.mjs +1 -1
  3. package/dist/adapters/indexeddb.mjs +1 -1
  4. package/dist/{capabilities-D6gjjQyA.d.ts → capabilities-CYeAWYxQ.d.ts} +19 -4
  5. package/dist/{chunk-Y52WZWAU.mjs → chunk-2WLD2NFW.mjs} +4 -4
  6. package/dist/{chunk-QR4OKRVY.mjs → chunk-3YKWSO54.mjs} +8 -3
  7. package/dist/{chunk-KZTI25D3.mjs → chunk-4MQ5F4IL.mjs} +1 -1
  8. package/dist/{chunk-CVPA5Q7R.mjs → chunk-AFKH4ZK3.mjs} +9 -6
  9. package/dist/{chunk-M6XF5HDE.mjs → chunk-AK3BLGCI.mjs} +141 -11
  10. package/dist/{chunk-7QMTA7AA.mjs → chunk-CVFAFZSN.mjs} +7771 -4271
  11. package/dist/{chunk-RV77FR2F.mjs → chunk-I64LXKO3.mjs} +0 -1
  12. package/dist/{chunk-EDMCZLSJ.mjs → chunk-LPBWCHBE.mjs} +193 -13
  13. package/dist/{chunk-P2ZZN32C.mjs → chunk-PU43WDY4.mjs} +4 -2
  14. package/dist/{chunk-HVJYTKK7.mjs → chunk-THMSFOJO.mjs} +365 -774
  15. package/dist/{chunk-LCCL6EEH.mjs → chunk-WVDFG45M.mjs} +92 -94
  16. package/dist/{chunk-3HQPUCCT.mjs → chunk-XW25X6AU.mjs} +66 -237
  17. package/dist/{chunk-2X2ZQOAB.mjs → chunk-ZULSEZCE.mjs} +3 -4
  18. package/dist/{client-C2TiW29y.d.ts → client-D698E5ge.d.ts} +76 -25
  19. package/dist/client.d.ts +7 -7
  20. package/dist/client.mjs +44 -22
  21. package/dist/{config-BLguB-om.d.ts → config-Bv8etwUZ.d.ts} +10 -3
  22. package/dist/{config-C7Ip8Ife.d.ts → config-QXU3HMpq.d.ts} +10 -1
  23. package/dist/direct-executor-5SRDFZAS.mjs +13 -0
  24. package/dist/distribution/coordinator/etcd.d.ts +27 -14
  25. package/dist/distribution/coordinator/etcd.mjs +162 -75
  26. package/dist/distribution/transport/grpc.d.ts +2 -2
  27. package/dist/distribution/transport/grpc.mjs +22 -2
  28. package/dist/distribution/transport/in-memory.d.ts +2 -2
  29. package/dist/distribution/transport/in-memory.mjs +6 -3
  30. package/dist/distribution/transport/tcp.d.ts +2 -2
  31. package/dist/distribution/transport/tcp.mjs +40 -16
  32. package/dist/distribution.d.ts +6 -6
  33. package/dist/distribution.mjs +637 -366
  34. package/dist/embeddings/openai.mjs +1 -1
  35. package/dist/{engine-Dm4446j1.d.ts → engine-D_OzKZ-6.d.ts} +96 -187
  36. package/dist/index.browser.mjs +7179 -4031
  37. package/dist/index.d.ts +5 -5
  38. package/dist/index.mjs +11 -14
  39. package/dist/{memory-CKHTU1CL.d.ts → memory-B4jMmygw.d.ts} +213 -2
  40. package/dist/pattern.mjs +1 -1
  41. package/dist/persistence/durability/checkpoint-worker.d.ts +24 -16
  42. package/dist/persistence/durability/checkpoint-worker.mjs +22 -16
  43. package/dist/react.d.ts +6 -6
  44. package/dist/react.mjs +0 -1
  45. package/dist/server.d.ts +9 -9
  46. package/dist/server.mjs +210 -32
  47. package/dist/{tasks-BECd0b__.d.ts → tasks-DpgCpW1x.d.ts} +3 -3
  48. package/dist/{thread-IBSIZZ6W.mjs → thread-3LNVA43V.mjs} +79 -21
  49. package/dist/{types-BQDlIvfS.d.ts → types-B3b68nfF.d.ts} +1 -1
  50. package/dist/{types-D5OwhtsQ.d.ts → types-Cb9kiY5P.d.ts} +1 -1
  51. package/dist/vector/search-worker.mjs +3 -3
  52. package/dist/workers/entry.mjs +4 -4
  53. package/package.json +7 -7
  54. package/dist/chunk-3L4U6EJQ.mjs +0 -49
  55. package/dist/chunk-IFBKLBGB.mjs +0 -7
  56. package/dist/chunk-XZ4PAREY.mjs +0 -25
  57. package/dist/direct-executor-SVCHXR74.mjs +0 -14
  58. package/dist/serialization/crc32-worker.d.ts +0 -20
  59. package/dist/serialization/crc32-worker.mjs +0 -62
@@ -1,7 +1,7 @@
1
1
  import { OSQ_STEPS, OSQ_GRID, OSQ_REFINE_ROUNDS, OSQ_REFINE_TOLERANCE, OSQ4_NARROW_DIMENSION_LIMIT, DEFAULT_OSQ4_NARROW_OVERSAMPLE, DEFAULT_OVERSAMPLE_BY_BITS, OSQ_QUERY_BITS, OSQ_LAMBDA } from './chunk-ZJ4CSAB2.mjs';
2
- import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-P2ZZN32C.mjs';
2
+ import { COMPACTION_TOMBSTONE_RATIO, COMPACTION_ABSOLUTE_THRESHOLD, nativeSearchCoreIsEnabled, loadNativeCore, stopUsingTheNativeSearchCore, noteFallback, MAX_LAYER_CAP, PLACEMENT_ROOM_REQUESTS, INITIAL_LIST_CAPACITY, DEFAULT_EF_SEARCH, INITIAL_UPPER_SLOTS } from './chunk-PU43WDY4.mjs';
3
3
  import { compareCodePoints } from './chunk-LJYTW7GT.mjs';
4
- import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
4
+ import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
5
5
  import { readSync, closeSync, openSync } from 'fs';
6
6
 
7
7
  // src/vector/constants.ts
@@ -3651,6 +3651,13 @@ function createVectorStore(options) {
3651
3651
  yield [docId, entryAt(open, ordinal)];
3652
3652
  }
3653
3653
  },
3654
+ *docIds() {
3655
+ if (open === null) return;
3656
+ for (let ordinal = 0; ordinal < ordToDoc.length; ordinal++) {
3657
+ const docId = ordToDoc[ordinal];
3658
+ if (docId !== void 0) yield docId;
3659
+ }
3660
+ },
3654
3661
  clear() {
3655
3662
  forgetDocuments();
3656
3663
  if (open === null) return;
@@ -3789,30 +3796,24 @@ function savedTopLayer(state, ordinal) {
3789
3796
  if (ordinal === NO_ORDINAL || isTombstoned(state, ordinal)) return NO_LAYER;
3790
3797
  return Math.min(nodeMaxLayer(state, ordinal), MAX_LAYER_CAP);
3791
3798
  }
3792
- function largestNeighbourValueCount(state, topLayers) {
3799
+ function neighbourValueCount(state, topLayers, start, end) {
3793
3800
  let values = 0;
3794
- for (let number = 0; number < topLayers.length; number++) {
3801
+ for (let number = start; number < end; number++) {
3795
3802
  const top = topLayers[number];
3796
3803
  if (top < 0) continue;
3797
3804
  values += state.adjacency.level0Stride + top * state.adjacency.upperStride;
3798
3805
  }
3799
3806
  return values;
3800
3807
  }
3801
- function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
3808
+ function serializePart(state, numbers, start, end) {
3809
+ const { numberOfOrdinal, ordinalOfNumber, topLayers } = numbers;
3802
3810
  const adjacency = state.adjacency;
3803
- const slots = adjacencySlots(adjacency);
3804
- const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
3805
- const levels = new Uint8Array(ordinalOfNumber.length);
3806
- for (let number = 0; number < ordinalOfNumber.length; number++) {
3807
- const ordinal = ordinalOfNumber[number];
3808
- if (ordinal >= slots) continue;
3809
- topLayers[number] = savedTopLayer(state, ordinal);
3810
- levels[number] = topLayers[number] + 1;
3811
- }
3812
- const values = new Uint32Array(largestNeighbourValueCount(state, topLayers));
3811
+ const levels = new Uint8Array(end - start);
3812
+ const values = new Uint32Array(neighbourValueCount(state, topLayers, start, end));
3813
3813
  let cursor = 0;
3814
- for (let number = 0; number < ordinalOfNumber.length; number++) {
3814
+ for (let number = start; number < end; number++) {
3815
3815
  const top = topLayers[number];
3816
+ levels[number - start] = top + 1;
3816
3817
  for (let layer = 0; layer <= top; layer++) {
3817
3818
  const base = layerBase(adjacency, ordinalOfNumber[number], layer);
3818
3819
  const array = layerArray(adjacency, layer);
@@ -3830,18 +3831,33 @@ function serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber) {
3830
3831
  values[countAt] = cursor - countAt - 1;
3831
3832
  }
3832
3833
  }
3834
+ return { levels, neighbours: littleEndianBytesOf(values.subarray(0, cursor)) };
3835
+ }
3836
+ function serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize) {
3837
+ const slots = adjacencySlots(state.adjacency);
3838
+ const topLayers = new Int8Array(ordinalOfNumber.length).fill(-1);
3839
+ for (let number = 0; number < ordinalOfNumber.length; number++) {
3840
+ const ordinal = ordinalOfNumber[number];
3841
+ if (ordinal >= slots) continue;
3842
+ topLayers[number] = savedTopLayer(state, ordinal);
3843
+ }
3833
3844
  const entryOrdinal = entryPointOf(state);
3834
3845
  const entryNumber = entryOrdinal >= 0 && entryOrdinal < numberOfOrdinal.length ? numberOfOrdinal[entryOrdinal] : NO_NUMBER;
3835
3846
  const entryHoldsNode = entryNumber !== NO_NUMBER && topLayers[entryNumber] >= 0;
3836
- return {
3847
+ const header = {
3837
3848
  entryPoint: entryHoldsNode ? entryNumber : null,
3838
3849
  maxLayer: Math.max(0, Math.min(topLayerOf(state), MAX_LAYER_CAP)),
3839
3850
  m: state.M,
3840
3851
  efConstruction: state.efCons,
3841
- metric: state.buildMetric,
3842
- levels,
3843
- neighbours: littleEndianBytesOf(values.subarray(0, cursor))
3852
+ metric: state.buildMetric
3844
3853
  };
3854
+ const numbers = { numberOfOrdinal, ordinalOfNumber, topLayers };
3855
+ const parts = [];
3856
+ for (let start = 0; start < ordinalOfNumber.length; start += partSize) {
3857
+ const end = Math.min(start + partSize, ordinalOfNumber.length);
3858
+ parts.push({ ...header, ...serializePart(state, numbers, start, end) });
3859
+ }
3860
+ return parts;
3845
3861
  }
3846
3862
  function endsInsideAList() {
3847
3863
  throw new NarsilError(
@@ -3862,95 +3878,78 @@ function requireWholeNeighbourLists(graph) {
3862
3878
  }
3863
3879
  }
3864
3880
  }
3865
- function topLayerIn(graph, number) {
3866
- return number < graph.levels.length ? graph.levels[number] - 1 : -1;
3881
+ function topLayerInParts(graph, number) {
3882
+ const part = graph.parts[Math.floor(number / graph.partSize)];
3883
+ if (part === void 0) return -1;
3884
+ const at = number % graph.partSize;
3885
+ return at < part.levels.length ? part.levels[at] - 1 : -1;
3886
+ }
3887
+ function highestNodeOrdinal(graph, ordinalAt) {
3888
+ let highest = -1;
3889
+ for (let p = 0; p < graph.parts.length; p++) {
3890
+ const levels = graph.parts[p].levels;
3891
+ for (let at = 0; at < levels.length; at++) {
3892
+ const ordinal = ordinalAt(p * graph.partSize + at);
3893
+ if (levels[at] > 0 && ordinal > highest) highest = ordinal;
3894
+ }
3895
+ }
3896
+ return highest;
3867
3897
  }
3868
3898
  function deserializeNumberedGraph(state, graph, ordinalOfNumber) {
3869
- requireWholeNeighbourLists(graph);
3899
+ for (const part of graph.parts) requireWholeNeighbourLists(part);
3870
3900
  resetGraph(state);
3871
- const values = valuesOf(graph.neighbours);
3872
3901
  const ordinalAt = (number) => number < ordinalOfNumber.length ? ordinalOfNumber[number] : NO_ORDINAL;
3873
- let highestOrdinal = -1;
3874
- for (let number = 0; number < graph.levels.length; number++) {
3875
- if (graph.levels[number] > 0 && ordinalAt(number) > highestOrdinal) highestOrdinal = ordinalAt(number);
3876
- }
3902
+ const highestOrdinal = highestNodeOrdinal(graph, ordinalAt);
3877
3903
  if (highestOrdinal >= 0) ensureCapacity(state, highestOrdinal + 1);
3878
3904
  const kept = [];
3879
- let cursor = 0;
3880
3905
  let highestLayer = -1;
3881
3906
  let nodeOnHighestLayer = NO_ORDINAL;
3882
- for (let number = 0; number < graph.levels.length; number++) {
3883
- const top = graph.levels[number] - 1;
3884
- if (top < 0) continue;
3885
- const ordinal = ordinalAt(number);
3886
- const keptTop = Math.min(top, MAX_LAYER_CAP);
3887
- if (ordinal !== NO_ORDINAL) {
3888
- createNode(state.adjacency, ordinal, keptTop);
3889
- Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
3890
- if (keptTop > highestLayer) {
3891
- highestLayer = keptTop;
3892
- nodeOnHighestLayer = ordinal;
3893
- }
3894
- }
3895
- for (let layer = 0; layer <= top; layer++) {
3896
- const count = values[cursor];
3897
- kept.length = 0;
3898
- const limit = maxConns(state, layer);
3899
- for (let i = 1; i <= count && kept.length < limit; i++) {
3900
- const neighbour = values[cursor + i];
3901
- const neighbourOrdinal = ordinalAt(neighbour);
3902
- if (neighbourOrdinal === NO_ORDINAL || neighbour === number || topLayerIn(graph, neighbour) < layer) continue;
3903
- if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
3907
+ for (let p = 0; p < graph.parts.length; p++) {
3908
+ const part = graph.parts[p];
3909
+ const values = valuesOf(part.neighbours);
3910
+ let cursor = 0;
3911
+ for (let at = 0; at < part.levels.length; at++) {
3912
+ const number = p * graph.partSize + at;
3913
+ const top = part.levels[at] - 1;
3914
+ if (top < 0) continue;
3915
+ const ordinal = ordinalAt(number);
3916
+ const keptTop = Math.min(top, MAX_LAYER_CAP);
3917
+ if (ordinal !== NO_ORDINAL) {
3918
+ createNode(state.adjacency, ordinal, keptTop);
3919
+ Atomics.add(state.header, GRAPH_NODE_COUNT, 1);
3920
+ if (keptTop > highestLayer) {
3921
+ highestLayer = keptTop;
3922
+ nodeOnHighestLayer = ordinal;
3923
+ }
3904
3924
  }
3905
- cursor += count + 1;
3906
- if (ordinal !== NO_ORDINAL && layer <= keptTop) {
3907
- replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
3925
+ for (let layer = 0; layer <= top; layer++) {
3926
+ const count = values[cursor];
3927
+ kept.length = 0;
3928
+ const limit = maxConns(state, layer);
3929
+ for (let i = 1; i <= count && kept.length < limit; i++) {
3930
+ const neighbour = values[cursor + i];
3931
+ const neighbourOrdinal = ordinalAt(neighbour);
3932
+ if (neighbourOrdinal === NO_ORDINAL || neighbour === number) continue;
3933
+ if (topLayerInParts(graph, neighbour) < layer) continue;
3934
+ if (!kept.includes(neighbourOrdinal)) kept.push(neighbourOrdinal);
3935
+ }
3936
+ cursor += count + 1;
3937
+ if (ordinal !== NO_ORDINAL && layer <= keptTop) {
3938
+ replaceNeighbors(state.adjacency, ordinal, layer, kept, kept.length);
3939
+ }
3908
3940
  }
3909
3941
  }
3910
3942
  }
3911
- const entryOrdinal = graph.entryPoint === null ? NO_ORDINAL : ordinalAt(graph.entryPoint);
3912
- const entryTop = graph.entryPoint === null ? -1 : Math.min(topLayerIn(graph, graph.entryPoint), MAX_LAYER_CAP);
3943
+ const first = graph.parts[0];
3944
+ const entryPoint = first === void 0 ? null : first.entryPoint;
3945
+ const entryOrdinal = entryPoint === null ? NO_ORDINAL : ordinalAt(entryPoint);
3946
+ const entryTop = entryPoint === null ? -1 : Math.min(topLayerInParts(graph, entryPoint), MAX_LAYER_CAP);
3913
3947
  const entryIsUsable = entryOrdinal !== NO_ORDINAL && entryTop >= 0;
3914
3948
  Atomics.store(state.header, GRAPH_ENTRY_POINT, entryIsUsable ? entryOrdinal : nodeOnHighestLayer);
3915
3949
  Atomics.store(state.header, GRAPH_TOP_LAYER, entryIsUsable ? entryTop : highestLayer);
3916
3950
  }
3917
3951
 
3918
3952
  // src/vector/hnsw/persistence.ts
3919
- function serializeGraph(state, store) {
3920
- const nodeArray = [];
3921
- const slots = adjacencySlots(state.adjacency);
3922
- for (let ord = 0; ord < slots; ord++) {
3923
- const maxLayer = nodeMaxLayer(state, ord);
3924
- if (maxLayer === -1) continue;
3925
- if (isTombstoned(state, ord)) continue;
3926
- const docId = store.docIdForOrdinal(ord);
3927
- if (docId === void 0) continue;
3928
- const layerConns = [];
3929
- for (let layer = 0; layer <= maxLayer; layer++) {
3930
- const liveNeighbors = [];
3931
- for (const neighborOrd of collectNeighbors(state.adjacency, ord, layer)) {
3932
- if (isTombstoned(state, neighborOrd)) continue;
3933
- const neighborDoc = store.docIdForOrdinal(neighborOrd);
3934
- if (neighborDoc === void 0) continue;
3935
- liveNeighbors.push(neighborDoc);
3936
- }
3937
- if (liveNeighbors.length > 0) {
3938
- layerConns.push([layer, liveNeighbors]);
3939
- }
3940
- }
3941
- nodeArray.push([docId, maxLayer, layerConns]);
3942
- }
3943
- const entryOrd = entryPointOf(state);
3944
- const entryPoint = entryOrd === -1 ? null : store.docIdForOrdinal(entryOrd) ?? null;
3945
- return {
3946
- entryPoint,
3947
- maxLayer: topLayerOf(state),
3948
- m: state.M,
3949
- efConstruction: state.efCons,
3950
- metric: state.buildMetric,
3951
- nodes: nodeArray
3952
- };
3953
- }
3954
3953
  function resolveNodes(state, store, data) {
3955
3954
  const nodes = [];
3956
3955
  let maxOrd = -1;
@@ -4186,13 +4185,12 @@ function createHNSWIndex(dimension, store, config, quantizer, handles) {
4186
4185
  compactTombstones: () => exclusively(() => compactTombstones(state)),
4187
4186
  rebuild: () => exclusively(() => rebuild(state)),
4188
4187
  exclusively,
4189
- serialize: () => serializeGraph(state, store),
4190
4188
  deserialize: (data) => exclusively(() => deserializeGraph(state, store, data)),
4191
- serializeNumbered: (numberOfOrdinal, ordinalOfNumber) => serializeNumberedGraph(state, numberOfOrdinal, ordinalOfNumber),
4189
+ serializeNumbered: (numberOfOrdinal, ordinalOfNumber, partSize) => serializeNumberedGraphParts(state, numberOfOrdinal, ordinalOfNumber, partSize),
4192
4190
  deserializeNumbered: (graph, ordinalOfNumber) => exclusively(() => deserializeNumberedGraph(state, graph, ordinalOfNumber)),
4193
4191
  exportSnapshot: () => exportSnapshot(state),
4194
4192
  restoreSnapshot: (snapshot) => exclusively(() => restoreSnapshot(state, snapshot))
4195
4193
  };
4196
4194
  }
4197
4195
 
4198
- export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore };
4196
+ export { DISK_STORAGE_BLOCK_BYTES, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, MAX_WORKER_COPIES, SCRATCH_SLOTS_PER_THREAD_POOL, STORE_CALIBRATION_GENERATION, VECTOR_INSERT_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SEARCH_TIMEOUT_MS, addToOrdinalFilter, cosineSimilarityWithMagnitudes, createHNSWIndex, createOrdinalFilter, createOsqQuantizer, createVectorStore, dotProduct, entryForOrd, euclideanDistance, isSimdAvailable, isTombstoned, magnitude, nativeScoresOf, nativeStoreScratchBytes, openSharedVectorField, ordinalFilterHas, ordinalFilterValues, osqBitsOf, osqRecordBytes, releaseLocksHeldBy, removeFromOrdinalFilter, requireWholeNeighbourLists, sharedMemoryAvailable, toDistance, toScore, topLayerInParts };
@@ -1,55 +1,6 @@
1
- import { MATCHER_STATE_BUDGET_BYTES, bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS, MATCHER_SIMULATION_BYTES_PER_POSITION, MATCHER_STATE_OVERHEAD_BYTES, MATCHER_CODE_POINT_BYTES } from './chunk-ZKRV5M2P.mjs';
2
-
3
- // src/core/pattern-index/matcher-budget.ts
4
- var POSITIONS_PER_WORD = 32;
5
- function positionWords(positions) {
6
- return Math.max(1, Math.ceil(positions / POSITIONS_PER_WORD));
7
- }
8
- function automatonBytes(positions) {
9
- return 2 * positionWords(positions) * Uint32Array.BYTES_PER_ELEMENT + positions * MATCHER_CODE_POINT_BYTES;
10
- }
11
- function smallestMatcherBytes(positions) {
12
- return automatonBytes(positions) + positions * MATCHER_SIMULATION_BYTES_PER_POSITION + MATCHER_STATE_OVERHEAD_BYTES + positions * Uint32Array.BYTES_PER_ELEMENT;
13
- }
14
- function widestPatternSplit(positions) {
15
- return Math.max(1, Math.floor(MATCHER_STATE_BUDGET_BYTES / smallestMatcherBytes(positions)));
16
- }
17
-
18
- // src/pattern/automaton.ts
19
- function setPosition(words, position) {
20
- words[position >>> 5] |= 1 << (position & 31);
21
- }
22
- function hasPosition(words, position) {
23
- return (words[position >>> 5] & 1 << (position & 31)) !== 0;
24
- }
25
- function literalAutomaton(codePoints, anchoredStart, anchoredEnd) {
26
- const size = codePoints.length;
27
- const words = positionWords(size);
28
- const first = new Uint32Array(words);
29
- const last = new Uint32Array(words);
30
- if (size > 0) {
31
- setPosition(first, 0);
32
- setPosition(last, size - 1);
33
- }
34
- return {
35
- size,
36
- words,
37
- first,
38
- last,
39
- nullable: size === 0,
40
- anchoredStart,
41
- anchoredEnd,
42
- bytes: automatonBytes(size),
43
- accepts: (position, codePoint) => codePoints[position] === codePoint,
44
- followerCount: (position) => position + 1 < size ? 1 : 0,
45
- follower: (position) => position + 1
46
- };
47
- }
1
+ import { bitsetHas, bitsetSet, PATTERN_RUN_CODE_POINTS } from './chunk-ZKRV5M2P.mjs';
48
2
 
49
3
  // src/pattern/constants.ts
50
- var MATCHER_TRANSITION_BYTES = 48;
51
- var MATCHER_KEY_CHARACTER_BYTES = 2;
52
- var MATCHER_MIN_CODE_POINTS_PER_STATE = 10;
53
4
  var SHORTLIST_RUN_LOOKUPS = 16;
54
5
  var SHORTLIST_INTERSECT_RATIO = 8;
55
6
  var RUN_LIST_INITIAL_CAPACITY = 4;
@@ -57,205 +8,86 @@ var RUN_LIST_COMPACTION_FRACTION = 0.5;
57
8
  var RUN_ENTRY_OVERHEAD_BYTES = 140;
58
9
  var DOCUMENT_ENTRY_BYTES = 24;
59
10
 
60
- // src/pattern/position-simulation.ts
61
- var LAST_STAMP = 4294967295;
62
- var PositionSimulation = class {
63
- constructor(automaton) {
64
- this.automaton = automaton;
65
- this.active = new Uint32Array(automaton.size);
66
- this.following = new Uint32Array(automaton.size);
67
- this.marks = new Uint32Array(automaton.size);
68
- const first = [];
69
- for (let position = 0; position < automaton.size; position++) {
70
- if (hasPosition(automaton.first, position)) first.push(position);
71
- }
72
- this.first = Uint32Array.from(first);
73
- }
11
+ // src/pattern/literal-matcher.ts
12
+ function bordersOf(codePoints) {
13
+ const length = codePoints.length;
14
+ const borders = new Uint32Array(length + 1);
15
+ let border = 0;
16
+ for (let at = 1; at < length; at++) {
17
+ while (border > 0 && codePoints[at] !== codePoints[border]) border = borders[border];
18
+ if (codePoints[at] === codePoints[border]) border++;
19
+ borders[at + 1] = border;
20
+ }
21
+ return borders;
22
+ }
23
+ function reachablePositionsOf(borders, length) {
24
+ const reachable = new Uint32Array(length + 1);
25
+ for (let prefix = 1; prefix <= length; prefix++) {
26
+ reachable[prefix] = (prefix < length ? 1 : 0) + reachable[borders[prefix]];
27
+ }
28
+ return reachable;
29
+ }
30
+ var LiteralMatcher = class {
31
+ constructor(codePoints, anchoredStart, anchoredEnd) {
32
+ this.codePoints = codePoints;
33
+ this.anchoredStart = anchoredStart;
34
+ this.anchoredEnd = anchoredEnd;
35
+ this.borders = bordersOf(codePoints);
36
+ this.reachable = reachablePositionsOf(this.borders, codePoints.length);
37
+ }
38
+ borders;
39
+ reachable;
40
+ prefix = 0;
41
+ alive = true;
74
42
  accepting = false;
75
- active;
76
- following;
77
- marks;
78
- first;
79
- activeCount = 0;
80
- stamp = 0;
81
- get steps() {
82
- return this.activeCount;
83
- }
84
- get bytes() {
85
- return this.automaton.size * MATCHER_SIMULATION_BYTES_PER_POSITION;
86
- }
87
- reset() {
88
- this.active.set(this.first);
89
- this.activeCount = this.first.length;
90
- this.accepting = this.automaton.nullable;
91
- }
92
- get firstPositions() {
93
- return this.first;
94
- }
95
- load(active, accepting) {
96
- this.active.set(active);
97
- this.activeCount = active.length;
98
- this.accepting = accepting;
99
- }
100
- sortedActive() {
101
- return this.active.slice(0, this.activeCount).sort();
102
- }
103
- advance(codePoint) {
104
- const { automaton, marks } = this;
105
- const following = this.following;
106
- const stamp = this.nextStamp();
107
- let count = 0;
108
- let accepting = false;
109
- for (let at = 0; at < this.activeCount; at++) {
110
- const position = this.active[at];
111
- if (!automaton.accepts(position, codePoint)) continue;
112
- if (hasPosition(automaton.last, position)) accepting = true;
113
- const followers = automaton.followerCount(position);
114
- for (let index = 0; index < followers; index++) {
115
- const follower = automaton.follower(position, index);
116
- if (marks[follower] === stamp) continue;
117
- marks[follower] = stamp;
118
- following[count++] = follower;
119
- }
120
- }
121
- if (!automaton.anchoredStart) {
122
- for (const position of this.first) {
123
- if (marks[position] === stamp) continue;
124
- marks[position] = stamp;
125
- following[count++] = position;
126
- }
127
- if (automaton.nullable) accepting = true;
128
- }
129
- this.following = this.active;
130
- this.active = following;
131
- this.activeCount = count;
132
- this.accepting = accepting;
133
- }
134
- nextStamp() {
135
- if (this.stamp === LAST_STAMP) {
136
- this.marks.fill(0);
137
- this.stamp = 0;
138
- }
139
- this.stamp++;
140
- return this.stamp;
141
- }
142
- };
143
-
144
- // src/pattern/lazy-dfa.ts
145
- var UNCACHED_GENERATION = -1;
146
- function stateKey(active, accepting) {
147
- return `${active.join(",")}${accepting ? "+" : "-"}`;
148
- }
149
- function stateBytes(active, key) {
150
- return MATCHER_STATE_OVERHEAD_BYTES + active.byteLength + key.length * MATCHER_KEY_CHARACTER_BYTES;
151
- }
152
- var LazyDfa = class {
153
- constructor(automaton, stateLimitBytes) {
154
- this.automaton = automaton;
155
- this.simulation = new PositionSimulation(automaton);
156
- this.cacheLimitBytes = stateLimitBytes - automaton.bytes - this.simulation.bytes;
157
- this.start = this.intern(this.simulation.firstPositions, automaton.nullable);
158
- this.state = this.start;
159
- if (this.wantsSimulation()) this.simulate(this.start);
160
- }
161
- states = /* @__PURE__ */ new Map();
162
- generation = 0;
163
- cachedBytes = 0;
164
- codePointsSinceReset = 0;
165
- start;
166
- state;
167
- simulating = false;
168
43
  decided = null;
169
- simulation;
170
- cacheLimitBytes;
171
44
  begin() {
172
45
  this.decided = null;
173
- if (this.simulating) this.simulation.reset();
174
- else this.state = this.start;
175
- if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
46
+ this.prefix = 0;
47
+ this.alive = this.codePoints.length > 0;
48
+ this.accepting = this.codePoints.length === 0;
49
+ if (!this.anchoredEnd && this.accepting) return this.decide(true);
176
50
  return true;
177
51
  }
178
52
  feed(codePoint, tally) {
179
- const steps = this.simulating ? this.simulation.steps : this.state.active.length;
180
- if (steps === 0) return this.decide(!this.automaton.anchoredStart && this.automaton.nullable);
53
+ const steps = this.steps();
54
+ if (steps === 0) return this.decide(!this.anchoredStart && this.codePoints.length === 0);
181
55
  tally.add(steps);
182
- if (this.simulating) {
183
- this.simulation.advance(codePoint);
184
- } else {
185
- this.codePointsSinceReset++;
186
- this.state = this.step(this.state, codePoint);
187
- if (this.wantsSimulation()) this.simulate(this.state);
188
- }
189
- if (!this.automaton.anchoredEnd && this.accepting()) return this.decide(true);
56
+ if (this.anchoredStart) this.advanceAnchored(codePoint);
57
+ else this.advance(codePoint);
58
+ if (!this.anchoredEnd && this.accepting) return this.decide(true);
190
59
  return true;
191
60
  }
192
61
  matched() {
193
- return this.decided ?? (this.automaton.anchoredEnd && this.accepting());
62
+ return this.decided ?? (this.anchoredEnd && this.accepting);
63
+ }
64
+ steps() {
65
+ const length = this.codePoints.length;
66
+ if (length === 0) return 0;
67
+ if (this.anchoredStart) return this.alive && this.prefix < length ? 1 : 0;
68
+ return 1 + this.reachable[this.prefix];
69
+ }
70
+ advanceAnchored(codePoint) {
71
+ if (this.codePoints[this.prefix] === codePoint) {
72
+ this.prefix++;
73
+ this.accepting = this.prefix === this.codePoints.length;
74
+ return;
75
+ }
76
+ this.alive = false;
77
+ this.accepting = false;
194
78
  }
195
- accepting() {
196
- return this.simulating ? this.simulation.accepting : this.state.accepting;
79
+ advance(codePoint) {
80
+ const { codePoints, borders } = this;
81
+ let prefix = this.prefix === codePoints.length ? borders[this.prefix] : this.prefix;
82
+ while (prefix > 0 && codePoints[prefix] !== codePoint) prefix = borders[prefix];
83
+ if (codePoints[prefix] === codePoint) prefix++;
84
+ this.prefix = prefix;
85
+ this.accepting = prefix === codePoints.length;
197
86
  }
198
87
  decide(outcome) {
199
88
  this.decided = outcome;
200
89
  return false;
201
90
  }
202
- wantsSimulation() {
203
- return !this.simulating && this.state.generation === UNCACHED_GENERATION;
204
- }
205
- simulate(from) {
206
- this.dropCache();
207
- this.state = this.start;
208
- this.simulation.load(from.active, from.accepting);
209
- this.simulating = true;
210
- }
211
- dropCache() {
212
- for (const state of this.states.values()) state.next.clear();
213
- this.start.next.clear();
214
- this.state.next.clear();
215
- this.states = /* @__PURE__ */ new Map();
216
- this.cachedBytes = 0;
217
- }
218
- step(state, codePoint) {
219
- const current = state.generation === this.generation;
220
- if (current) {
221
- const cached = state.next.get(codePoint);
222
- if (cached !== void 0) return cached;
223
- }
224
- const { simulation } = this;
225
- simulation.load(state.active, state.accepting);
226
- simulation.advance(codePoint);
227
- const next = this.intern(simulation.sortedActive(), simulation.accepting);
228
- if (!current || state.generation !== this.generation || next.generation !== this.generation) return next;
229
- if (!this.reserve(MATCHER_TRANSITION_BYTES) || state.generation !== this.generation) return next;
230
- state.next.set(codePoint, next);
231
- return next;
232
- }
233
- intern(active, accepting) {
234
- const key = stateKey(active, accepting);
235
- const existing = this.states.get(key);
236
- if (existing !== void 0) return existing;
237
- if (!this.reserve(stateBytes(active, key))) {
238
- return { active, accepting, generation: UNCACHED_GENERATION, next: /* @__PURE__ */ new Map() };
239
- }
240
- const state = { active, accepting, generation: this.generation, next: /* @__PURE__ */ new Map() };
241
- this.states.set(key, state);
242
- return state;
243
- }
244
- reserve(bytes) {
245
- if (this.cachedBytes + bytes <= this.cacheLimitBytes) {
246
- this.cachedBytes += bytes;
247
- return true;
248
- }
249
- if (this.states.size === 0 || this.codePointsSinceReset < MATCHER_MIN_CODE_POINTS_PER_STATE * this.states.size) {
250
- return false;
251
- }
252
- this.dropCache();
253
- this.codePointsSinceReset = 0;
254
- this.generation++;
255
- this.start = this.intern(this.start.active, this.start.accepting);
256
- if (this.start.generation === UNCACHED_GENERATION) return false;
257
- return this.reserve(bytes);
258
- }
259
91
  };
260
92
 
261
93
  // src/pattern/text.ts
@@ -391,10 +223,7 @@ function matcherFor(request) {
391
223
  const { operator, text, caseFold, host } = request;
392
224
  const testCodePoints = [];
393
225
  appendTextCodePoints(text, caseFold, host, testCodePoints);
394
- return new LazyDfa(
395
- literalAutomaton(testCodePoints, ANCHORED_START.has(operator), ANCHORED_END.has(operator)),
396
- request.meter.matcherStateBytes
397
- );
226
+ return new LiteralMatcher(Uint32Array.from(testCodePoints), ANCHORED_START.has(operator), ANCHORED_END.has(operator));
398
227
  }
399
228
  function matchesText(matcher, text, caseFold, host, folded, tally) {
400
229
  if (!matcher.begin()) return matcher.matched();
@@ -782,4 +611,4 @@ var patternSearchModule = {
782
611
  matchInto: matchPattern
783
612
  };
784
613
 
785
- export { GrowableUint32, patternSearchModule, widestPatternSplit };
614
+ export { GrowableUint32, patternSearchModule };
@@ -1,9 +1,8 @@
1
- import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-XZ4PAREY.mjs';
2
- import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-QR4OKRVY.mjs';
3
- import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-M6XF5HDE.mjs';
1
+ import { clampRowCount, sortSignatureOf, requireSortableFields, decodePageCursor, requireMatchingCursor, resolveProjection, encodePageCursor, requireWithinResultWindow, normalizeSort, fanOutQuery, sortModesOf, applySorting, applyPagination, projectionKeepsField, countsWithoutScores, fanOutMatchCount, tokenize, fnv1aBytes, requireValidFilter, createPatternWorkMeter, everyValueFacetConfig, fulltextMatches, mergeFacets, pruneStatsToQueryTerms, partitionsIn, undeclaredFieldOnStrictIndex, fuzzyTermMatches } from './chunk-3YKWSO54.mjs';
2
+ import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, DEFAULT_LIMIT, DEFAULT_MAX_PER_GROUP, flattenSchema, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, DEFAULT_OFFSET, DEFAULT_FUZZY_PREFIX_LENGTH, RESULT_WINDOW, isWordIndexedFieldType, isGeopointOrVectorType, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-AK3BLGCI.mjs';
4
3
  import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
5
4
  import { holdsUnpairedSurrogate, compareCodePoints, compareComparableKeys } from './chunk-LJYTW7GT.mjs';
6
- import { NarsilError, ErrorCodes } from './chunk-RV77FR2F.mjs';
5
+ import { NarsilError, ErrorCodes } from './chunk-I64LXKO3.mjs';
7
6
 
8
7
  // src/search/cursor-binding.ts
9
8
  var ABSENT_TAG = 0;