@delali/narsil 0.3.1-next.90 → 0.3.1-next.94

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/adapters/filesystem.mjs +1 -1
  2. package/dist/adapters/indexeddb.mjs +1 -1
  3. package/dist/browser/{chunk-3GZQKGVQ.mjs → chunk-4JTBL63P.mjs} +13 -5
  4. package/dist/browser/{chunk-3ZRMZIH4.mjs → chunk-5S74HQRT.mjs} +208 -78
  5. package/dist/browser/{chunk-FHFFWLEK.mjs → chunk-HARBSUWP.mjs} +3 -4
  6. package/dist/browser/{chunk-RAHUQOJL.mjs → chunk-HC6DMR4T.mjs} +2445 -2107
  7. package/dist/browser/{chunk-JACCR3BJ.mjs → chunk-PP6N5CCS.mjs} +2 -2
  8. package/dist/browser/{chunk-NQW2YXEX.mjs → chunk-WKPJ4E73.mjs} +1 -1
  9. package/dist/browser/{durability-wiring-VSUBLUDK.mjs → durability-wiring-7DKYHKT5.mjs} +498 -71
  10. package/dist/browser/{module-E25UDADM.mjs → module-O5WH32EL.mjs} +102 -54
  11. package/dist/browser/release-76MXRPGV.mjs +33 -0
  12. package/dist/browser/replace-RURXTGVG.mjs +57 -0
  13. package/dist/browser/restore-queue-Z7FZZNKH.mjs +35 -0
  14. package/dist/browser/{snapshot-5MXBEDXF.mjs → snapshot-PPTZ565G.mjs} +368 -103
  15. package/dist/browser/{write-6SBYTS62.mjs → write-2ZAA7UDH.mjs} +4 -4
  16. package/dist/{capabilities-SJPttcgQ.d.ts → capabilities-xsJfdFoQ.d.ts} +11 -3
  17. package/dist/chunk-2GZGBOQI.mjs +426 -0
  18. package/dist/{chunk-CVQWQVFX.mjs → chunk-4ZJWRETC.mjs} +2 -2
  19. package/dist/chunk-7KC56574.mjs +507 -0
  20. package/dist/{chunk-MXS5HWIH.mjs → chunk-7VALMHZF.mjs} +89 -17
  21. package/dist/chunk-7X5JST5L.mjs +114 -0
  22. package/dist/{chunk-UKSUENHK.mjs → chunk-AF2CIKYC.mjs} +40 -26
  23. package/dist/{chunk-BVOJIYRV.mjs → chunk-BB7MASRQ.mjs} +1 -1
  24. package/dist/{chunk-R6BMMQXH.mjs → chunk-C45FUYPI.mjs} +193 -79
  25. package/dist/{chunk-EMXAYPTQ.mjs → chunk-CHOGQSHK.mjs} +2326 -3503
  26. package/dist/{chunk-QZN3MIJ3.mjs → chunk-ES33L3HO.mjs} +2 -2
  27. package/dist/{chunk-VY4DXZ5P.mjs → chunk-EWBXO45L.mjs} +3 -3
  28. package/dist/{chunk-QAMUC2BJ.mjs → chunk-EX62BQT6.mjs} +10 -9
  29. package/dist/{chunk-MWJIAREY.mjs → chunk-EZD4MWOU.mjs} +4 -4
  30. package/dist/{chunk-YCJLVNWR.mjs → chunk-F52QGNZ2.mjs} +1 -1
  31. package/dist/{chunk-URGLSNKR.mjs → chunk-GJZHU4FC.mjs} +202 -16
  32. package/dist/{chunk-S4LQDSUL.mjs → chunk-J53IHLG7.mjs} +7 -7
  33. package/dist/{chunk-R7OSDAFK.mjs → chunk-K6A4FCVR.mjs} +1 -1
  34. package/dist/chunk-KOKM4PQN.mjs +52 -0
  35. package/dist/{chunk-GCM3JFVA.mjs → chunk-OHP7QR33.mjs} +2 -1
  36. package/dist/{chunk-T6CNO72J.mjs → chunk-QM3VPX4Z.mjs} +2 -2
  37. package/dist/{chunk-V7YUSKKZ.mjs → chunk-QWWCLRXX.mjs} +10 -6
  38. package/dist/chunk-QY7GC2MW.mjs +33 -0
  39. package/dist/{chunk-DCK4P5BI.mjs → chunk-R36MIGBU.mjs} +100 -59
  40. package/dist/{chunk-YRX6XMOQ.mjs → chunk-R6PR4WYZ.mjs} +1 -0
  41. package/dist/{chunk-RVM7AODE.mjs → chunk-RDJKYKMQ.mjs} +2 -2
  42. package/dist/{chunk-OOSIDHA3.mjs → chunk-RSRVGAIQ.mjs} +1 -1
  43. package/dist/{chunk-XSZVS4S3.mjs → chunk-RUIXODYU.mjs} +9 -4
  44. package/dist/{chunk-A3UB2MXL.mjs → chunk-SBCEHIPL.mjs} +4 -507
  45. package/dist/{chunk-TPJU7NDJ.mjs → chunk-SYWGORQB.mjs} +1 -0
  46. package/dist/chunk-UAUZGXJP.mjs +3591 -0
  47. package/dist/{chunk-CL4OM7GB.mjs → chunk-WBS3EYC4.mjs} +12 -8
  48. package/dist/{chunk-J2BPXTD7.mjs → chunk-WLOF6HSP.mjs} +3 -3
  49. package/dist/chunk-X5CV2E2B.mjs +1 -0
  50. package/dist/{chunk-ULRPNYAG.mjs → chunk-Z3MOCHG5.mjs} +120 -62
  51. package/dist/chunk-ZHS44YKF.mjs +35 -0
  52. package/dist/{client-BMK53IR2.d.ts → client-CgayPw2R.d.ts} +27 -3
  53. package/dist/client.d.ts +9 -8
  54. package/dist/client.mjs +16 -2
  55. package/dist/{config-UJx1dyX1.d.ts → config-3kBsQ0Pi.d.ts} +2 -2
  56. package/dist/{config-BUR-7eVQ.d.ts → config-B-RnO6dr.d.ts} +2 -2
  57. package/dist/direct-executor-IJTRU3YN.mjs +22 -0
  58. package/dist/distribution/coordinator/etcd.d.ts +3 -2
  59. package/dist/distribution/coordinator/etcd.mjs +5 -1
  60. package/dist/distribution/coordinator/in-memory.d.ts +3 -2
  61. package/dist/distribution/coordinator/in-memory.mjs +4 -0
  62. package/dist/distribution/transport/grpc.mjs +3 -3
  63. package/dist/distribution/transport/in-memory.mjs +1 -1
  64. package/dist/distribution/transport/tcp.mjs +2 -2
  65. package/dist/distribution.d.ts +43 -9
  66. package/dist/distribution.mjs +4620 -4293
  67. package/dist/{durability-wiring-ZLRQU6FG.mjs → durability-wiring-Y55SCPFC.mjs} +524 -93
  68. package/dist/embeddings/openai.mjs +1 -1
  69. package/dist/{engine-9X6W5naj.d.ts → engine-DSUfY2F0.d.ts} +53 -17
  70. package/dist/{index-vCMu8OeQ.d.ts → index-ONAEaqtx.d.ts} +8 -1
  71. package/dist/index.browser.mjs +2 -2
  72. package/dist/index.d.ts +44 -25
  73. package/dist/index.mjs +22 -18
  74. package/dist/{memory-CV9NAxLY.d.ts → memory-wuXcYOD1.d.ts} +99 -79
  75. package/dist/{module-WIDYIJLY.mjs → module-VITS4ZR5.mjs} +72 -38
  76. package/dist/partial-words.mjs +5 -4
  77. package/dist/pattern.mjs +3 -3
  78. package/dist/persistence/durability/checkpoint-worker.d.ts +60 -58
  79. package/dist/persistence/durability/checkpoint-worker.mjs +27 -23
  80. package/dist/react.d.ts +8 -7
  81. package/dist/react.mjs +1 -0
  82. package/dist/release-BH3DKW3T.mjs +33 -0
  83. package/dist/replace-CEO5KJJ3.mjs +57 -0
  84. package/dist/restore-queue-4SG45BLF.mjs +35 -0
  85. package/dist/{schema-DtsgmNfU.d.ts → schema-DttzM-ln.d.ts} +12 -0
  86. package/dist/server.d.ts +11 -10
  87. package/dist/server.mjs +38 -11
  88. package/dist/snapshot-XBKE7PUG.mjs +738 -0
  89. package/dist/synonyms-CpOAHPa-.d.ts +52 -0
  90. package/dist/synonyms.d.ts +22 -0
  91. package/dist/synonyms.mjs +12 -0
  92. package/dist/{tasks-BGaZJyoY.d.ts → tasks-Bj9WBlFG.d.ts} +4 -1
  93. package/dist/{thread-ZP3UMIZT.mjs → thread-U7XYJWGG.mjs} +17 -12
  94. package/dist/vector/search-worker.d.ts +2 -1
  95. package/dist/vector/search-worker.mjs +6 -6
  96. package/dist/workers/entry.mjs +6 -6
  97. package/dist/{write-PXDKPG2R.mjs → write-BMNV7P2H.mjs} +24 -20
  98. package/package.json +15 -7
  99. package/dist/chunk-PR4O7CAP.mjs +0 -18
  100. package/dist/chunk-X4SEXL4U.mjs +0 -2310
  101. package/dist/chunk-Y2F66IQJ.mjs +0 -74
  102. package/dist/chunk-ZJ4CSAB2.mjs +0 -13
  103. package/dist/direct-executor-EGESD26S.mjs +0 -18
  104. package/dist/snapshot-22CUEYYW.mjs +0 -469
@@ -0,0 +1,114 @@
1
+ import { encode, decode } from '@msgpack/msgpack';
2
+
3
+ // src/core/synonyms/rules.ts
4
+
5
+ // src/core/synonyms/constants.ts
6
+ var MAX_SYNONYM_RULES = 1e5;
7
+ var MAX_SYNONYM_WORDS = 5e5;
8
+ var MAX_SYNONYM_WORD_BYTES = 4194304;
9
+ var MAX_SYNONYM_RULE_WORDS = 100;
10
+ var MIN_TWO_WAY_SYNONYM_WORDS = 2;
11
+ var MIN_ONE_WAY_SYNONYMS = 1;
12
+ var SYNONYMS_ADDED_PER_QUERY = 50;
13
+ var SYNONYM_EXPANSION_CACHE_ENTRIES = 1024;
14
+
15
+ // src/core/synonyms/rules.ts
16
+ var FIRST_PRINTABLE_CODE = 32;
17
+ function holdsControlCharacter(word) {
18
+ for (let i = 0; i < word.length; i++) if (word.charCodeAt(i) < FIRST_PRINTABLE_CODE) return true;
19
+ return false;
20
+ }
21
+ function utf8Length(word) {
22
+ let bytes = 0;
23
+ for (let i = 0; i < word.length; i++) {
24
+ const unit = word.charCodeAt(i);
25
+ if (unit < 128) bytes += 1;
26
+ else if (unit < 2048) bytes += 2;
27
+ else if (unit >= 55296 && unit < 56320 && i + 1 < word.length) {
28
+ const low = word.charCodeAt(i + 1);
29
+ if (low >= 56320 && low < 57344) {
30
+ bytes += 4;
31
+ i++;
32
+ } else bytes += 3;
33
+ } else bytes += 3;
34
+ }
35
+ return bytes;
36
+ }
37
+ function hasOnlyKeys(value, keys) {
38
+ const own = Object.keys(value);
39
+ return own.length === keys.length && keys.every((key) => Object.hasOwn(value, key));
40
+ }
41
+ function readWord(value, rule, tally, raise) {
42
+ if (typeof value !== "string") {
43
+ raise(`Synonym rule ${rule} holds a word that is not a string`, { rule });
44
+ }
45
+ if (holdsControlCharacter(value)) {
46
+ raise(`Synonym rule ${rule} holds a word with a control character`, { rule });
47
+ }
48
+ tally.words++;
49
+ tally.bytes += utf8Length(value);
50
+ if (tally.words > MAX_SYNONYM_WORDS) {
51
+ raise(`A synonym set holds at most ${MAX_SYNONYM_WORDS} words`, { maxWords: MAX_SYNONYM_WORDS });
52
+ }
53
+ if (tally.bytes > MAX_SYNONYM_WORD_BYTES) {
54
+ raise(`The words of a synonym set hold at most ${MAX_SYNONYM_WORD_BYTES} bytes in UTF-8`, {
55
+ maxWordBytes: MAX_SYNONYM_WORD_BYTES
56
+ });
57
+ }
58
+ return value;
59
+ }
60
+ function readWords(value, rule, key, minimum, maximum, tally, raise) {
61
+ if (!Array.isArray(value) || value.length < minimum || value.length > maximum) {
62
+ raise(`Synonym rule ${rule} needs "${key}" as a list of ${minimum} to ${maximum} words`, { rule });
63
+ }
64
+ const words = new Array(value.length);
65
+ for (let i = 0; i < value.length; i++) words[i] = readWord(value[i], rule, tally, raise);
66
+ return words;
67
+ }
68
+ function readRule(value, rule, tally, raise) {
69
+ if (value === null || typeof value !== "object" || Array.isArray(value)) {
70
+ raise(`Synonym rule ${rule} must be an object`, { rule });
71
+ }
72
+ if (hasOnlyKeys(value, ["words"])) {
73
+ const words = Reflect.get(value, "words");
74
+ return { words: readWords(words, rule, "words", MIN_TWO_WAY_SYNONYM_WORDS, MAX_SYNONYM_RULE_WORDS, tally, raise) };
75
+ }
76
+ if (hasOnlyKeys(value, ["word", "synonyms"])) {
77
+ const word = readWord(Reflect.get(value, "word"), rule, tally, raise);
78
+ const synonyms = readWords(
79
+ Reflect.get(value, "synonyms"),
80
+ rule,
81
+ "synonyms",
82
+ MIN_ONE_WAY_SYNONYMS,
83
+ MAX_SYNONYM_RULE_WORDS - 1,
84
+ tally,
85
+ raise
86
+ );
87
+ return { word, synonyms };
88
+ }
89
+ raise(`Synonym rule ${rule} must hold either "words" alone, or "word" and "synonyms"`, { rule });
90
+ }
91
+ function parseSynonymRules(value, raise) {
92
+ if (!Array.isArray(value)) raise("A synonym set must be a list of rules", {});
93
+ if (value.length > MAX_SYNONYM_RULES) {
94
+ raise(`A synonym set holds at most ${MAX_SYNONYM_RULES} rules`, { maxRules: MAX_SYNONYM_RULES });
95
+ }
96
+ const tally = { words: 0, bytes: 0 };
97
+ const rules = new Array(value.length);
98
+ for (let rule = 0; rule < value.length; rule++) rules[rule] = readRule(value[rule], rule, tally, raise);
99
+ return rules;
100
+ }
101
+ function encodeSynonymRules(rules) {
102
+ return encode(rules);
103
+ }
104
+ function decodeSynonymRules(bytes, raise) {
105
+ let decoded;
106
+ try {
107
+ decoded = decode(bytes);
108
+ } catch {
109
+ raise("A stored synonym set is not valid MessagePack", {});
110
+ }
111
+ return parseSynonymRules(decoded, raise);
112
+ }
113
+
114
+ export { SYNONYMS_ADDED_PER_QUERY, SYNONYM_EXPANSION_CACHE_ENTRIES, decodeSynonymRules, encodeSynonymRules, parseSynonymRules };
@@ -1,6 +1,6 @@
1
- import { openSharedQuantizer, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, STORE_RELEASED_VECTORS_TO_DISK, STORE_CODE_COUNT, STORE_CALIBRATED, STORE_CALIBRATION_GENERATION, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-ULRPNYAG.mjs';
2
- import { MAX_VECTOR_ORDINALS } from './chunk-Y2F66IQJ.mjs';
3
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
1
+ import { openSharedQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, searchOrdinals, insertNode, GRAPH_RETIRED, nodeCountOf, tombstoneCountOf, BLOCK_MAX_BYTES, sharedVectorStoreBytes, STORE_SLOTS, STORE_LIVE_COUNT, IN_MEMORY, resetStoreHeader, STORE_RELEASED_VECTORS_TO_DISK, magnitude, openGraphState, fixedView, createSharedVectorStoreHandles, growBufferTo, unplacedChunkCount, STORE_DOC_ID_BYTES, createOrdinalFilter, addToOrdinalFilter, slotByteOffset, openVectorBlock, NOTHING_STAGED, arenaFloat32Distance, detachNativeStore, STORE_BLOCK_COUNT, STORE_CODE_BLOCK_COUNT, createVectorBlock, ensureBlockSlots, QUERY_STAGED, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes } from './chunk-Z3MOCHG5.mjs';
2
+ import { MAX_VECTOR_ORDINALS } from './chunk-QY7GC2MW.mjs';
3
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
4
4
  import { readSync, closeSync, openSync } from 'fs';
5
5
 
6
6
  function createVectorFileReader() {
@@ -404,6 +404,7 @@ function openSharedVectorField(initial, threadSlot) {
404
404
  let graph = null;
405
405
  let ordinals = null;
406
406
  let scanned = 0;
407
+ let scannedLayoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
407
408
  let placed = 0;
408
409
  function openGraph(next) {
409
410
  graph = next.graph === null ? null : openGraphState(next.graph, next.dimension, store, quantizer, threadSlot);
@@ -414,6 +415,12 @@ function openSharedVectorField(initial, threadSlot) {
414
415
  return store.docIdAt(ordinal);
415
416
  }
416
417
  function ordinalOf(docId) {
418
+ const layoutGeneration = Atomics.load(store.handles.header, STORE_ORDINAL_LAYOUT_GENERATION);
419
+ if (layoutGeneration !== scannedLayoutGeneration) {
420
+ ordinals = null;
421
+ scanned = 0;
422
+ scannedLayoutGeneration = layoutGeneration;
423
+ }
417
424
  const slots = store.slots;
418
425
  if (ordinals === null) ordinals = /* @__PURE__ */ new Map();
419
426
  for (; scanned < slots; scanned++) {
@@ -581,6 +588,31 @@ function relayout(store) {
581
588
  store.handles.layoutRevision += 1;
582
589
  store.view.adoptHandles(store.handles);
583
590
  }
591
+ function claimSlot(store, docId, cold) {
592
+ const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
593
+ ensureOrdinal(store, ordinal, cold);
594
+ store.diskFile[ordinal] = IN_MEMORY;
595
+ appendDocId(store.handles, ordinal, docId);
596
+ return ordinal;
597
+ }
598
+ function publishOrdinal(store, ordinal) {
599
+ Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
600
+ Atomics.store(store.present, ordinal, 1);
601
+ }
602
+ function publishOnDisk(store, ordinal, vectorMagnitude, location) {
603
+ store.magnitudes[ordinal] = vectorMagnitude;
604
+ store.diskOffset[ordinal] = location.offset;
605
+ store.diskFile[ordinal] = location.fileIndex;
606
+ publishOrdinal(store, ordinal);
607
+ }
608
+ function relocateFiles(store, rewrite) {
609
+ const files = store.handles.vectorFiles;
610
+ if (files.length === 0) return;
611
+ for (let file = 0; file < files.length; file++) {
612
+ if (files[file].length > 0) files[file] = rewrite(files[file]);
613
+ }
614
+ relayout(store);
615
+ }
584
616
  function ensureCodeSlot(store, ordinal) {
585
617
  const { handles } = store;
586
618
  if (handles.codeLayout === null) return;
@@ -749,13 +781,6 @@ function createVectorStore(options) {
749
781
  Atomics.store(store.present, ordinal, 1);
750
782
  replacedOrdinals.add(ordinal);
751
783
  }
752
- function claimSlot(store, docId, cold) {
753
- const ordinal = Atomics.load(store.handles.header, STORE_SLOTS);
754
- ensureOrdinal(store, ordinal, cold);
755
- store.diskFile[ordinal] = IN_MEMORY;
756
- appendDocId(store.handles, ordinal, docId);
757
- return ordinal;
758
- }
759
784
  function claimOrdinal(store, docId, cold, partitionId) {
760
785
  const ordinal = claimSlot(store, docId, cold);
761
786
  recordPartition(store, ordinal, partitionId);
@@ -763,16 +788,6 @@ function createVectorStore(options) {
763
788
  docToOrd.set(docId, ordinal);
764
789
  return ordinal;
765
790
  }
766
- function publishOrdinal(store, ordinal) {
767
- Atomics.store(store.handles.header, STORE_SLOTS, ordinal + 1);
768
- Atomics.store(store.present, ordinal, 1);
769
- }
770
- function publishOnDisk(store, ordinal, vectorMagnitude, location) {
771
- store.magnitudes[ordinal] = vectorMagnitude;
772
- store.diskOffset[ordinal] = location.offset;
773
- store.diskFile[ordinal] = location.fileIndex;
774
- publishOrdinal(store, ordinal);
775
- }
776
791
  function appendOrdinal(store, docId, vector, partitionId) {
777
792
  const ordinal = claimOrdinal(store, docId, false, partitionId);
778
793
  writeVector(store, ordinal, vector);
@@ -943,19 +958,18 @@ function createVectorStore(options) {
943
958
  relayout(open);
944
959
  }
945
960
  resetDocIds(open.handles);
946
- Atomics.store(open.handles.header, STORE_SLOTS, 0);
947
- Atomics.store(open.handles.header, STORE_RELEASED_VECTORS_TO_DISK, 0);
948
- Atomics.store(open.handles.header, STORE_LIVE_COUNT, 0);
949
- Atomics.store(open.handles.header, STORE_CODE_COUNT, 0);
950
- Atomics.store(open.handles.header, STORE_CALIBRATED, 0);
951
- Atomics.add(open.handles.header, STORE_CALIBRATION_GENERATION, 1);
961
+ resetStoreHeader(open.handles.header);
952
962
  },
953
963
  release() {
954
964
  forgetDocuments();
965
+ if (open !== null) Atomics.add(open.handles.header, STORE_ORDINAL_LAYOUT_GENERATION, 1);
955
966
  open?.view.close();
956
967
  open = null;
957
968
  },
958
969
  closeFiles: () => open?.view.close(),
970
+ relocateVectorFiles(rewrite) {
971
+ if (open !== null) relocateFiles(open, rewrite);
972
+ },
959
973
  getOrdinal(docId) {
960
974
  return docToOrd.get(docId);
961
975
  },
@@ -1,5 +1,5 @@
1
1
  import { PARSED_FIELD_TYPE_CACHE_ENTRIES, VECTOR_PATTERN, MAX_STORED_VECTOR_DIMENSION } from './chunk-L5ORED7V.mjs';
2
- import { NarsilError, ErrorCodes } from './chunk-TPJU7NDJ.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-SYWGORQB.mjs';
3
3
 
4
4
  // src/schema/validator/field-type.ts
5
5
  var FIELD_OPTION_ORDER = ["sortable", "pattern", "partial"];