@delali/narsil 0.3.1-next.163 → 0.3.1-next.169

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +24 -8
  2. package/dist/browser/{chunk-MZHQ65VI.mjs → chunk-2Q45HI5H.mjs} +14 -10
  3. package/dist/browser/{chunk-MMHIIL5B.mjs → chunk-4ZICWS4V.mjs} +1 -1
  4. package/dist/browser/{chunk-KQXKCMGQ.mjs → chunk-JHFBQKUF.mjs} +119 -90
  5. package/dist/browser/{core-leaf-SIWXFZLV.mjs → core-leaf-WYFANMKC.mjs} +61 -13
  6. package/dist/browser/custom-token-results-R5BIX7SL.mjs +59 -0
  7. package/dist/browser/{embed-RF2MPGAO.mjs → embed-24IPR4LP.mjs} +2 -2
  8. package/dist/browser/{module-PMQBRESM.mjs → module-XVLHJ7T7.mjs} +358 -62
  9. package/dist/browser/{optional-writes-WMZX4OEC.mjs → optional-writes-TB34FYUW.mjs} +118 -66
  10. package/dist/browser/{phrase-module-22TKMXWI.mjs → phrase-module-6UGFVW5J.mjs} +1 -0
  11. package/dist/browser/{resolve-vector-text-KI65IHMD.mjs → resolve-vector-text-EJ6DHFF5.mjs} +8 -0
  12. package/dist/browser/{storage-leaf-VGR2BWJJ.mjs → storage-leaf-QFXIWDFK.mjs} +208 -87
  13. package/dist/browser/{worker-copies-WQCPP3TS.mjs → worker-copies-ICHLYYXM.mjs} +1 -0
  14. package/dist/{capabilities-DkXAkb-X.d.ts → capabilities-BQIdsaWy.d.ts} +2 -2
  15. package/dist/{chunk-BKOMO7HT.mjs → chunk-26E5MO3Y.mjs} +5 -5
  16. package/dist/{chunk-EEYDJB4Z.mjs → chunk-2C3BXZGA.mjs} +213 -149
  17. package/dist/{chunk-NXMGUDZA.mjs → chunk-2KWTQ2P7.mjs} +2 -2
  18. package/dist/{chunk-TW543F33.mjs → chunk-34IOOYXA.mjs} +1 -1
  19. package/dist/{chunk-D5GNFN6X.mjs → chunk-7FC2ZILY.mjs} +98 -40
  20. package/dist/{chunk-XRBPHCCT.mjs → chunk-A4XYDUVN.mjs} +5 -5
  21. package/dist/{chunk-R4F7ZTVG.mjs → chunk-BIIROQ5Q.mjs} +2 -2
  22. package/dist/{chunk-Q2WQSAYH.mjs → chunk-CQX5B6L2.mjs} +54 -11
  23. package/dist/{chunk-XXWEUXE4.mjs → chunk-D6FYUTFN.mjs} +1 -1
  24. package/dist/{chunk-OUWZKNO3.mjs → chunk-DG6WIEB3.mjs} +1 -1
  25. package/dist/{chunk-GJKFXKMI.mjs → chunk-F3FDHXCJ.mjs} +12 -9
  26. package/dist/{chunk-3UGZBRDN.mjs → chunk-GO6TRSJ6.mjs} +1 -1
  27. package/dist/{react-native/chunks/chunk-OK3WUA6N.mjs → chunk-I2WJCRIH.mjs} +1 -1
  28. package/dist/{chunk-Q7CUUSJ2.mjs → chunk-ISJ35523.mjs} +2 -2
  29. package/dist/{chunk-GQLOEACG.mjs → chunk-JCWC75NH.mjs} +2 -2
  30. package/dist/{chunk-ZGYF37MG.mjs → chunk-JDIATDPY.mjs} +1 -1
  31. package/dist/{chunk-3B775F3C.mjs → chunk-JP77OWMI.mjs} +3 -1
  32. package/dist/{chunk-KSOZJEYU.mjs → chunk-KHJPMZE2.mjs} +1 -1
  33. package/dist/{chunk-ZQ6QY42L.mjs → chunk-KL6W73BZ.mjs} +8 -0
  34. package/dist/{chunk-ES5EOXT6.mjs → chunk-LOQHA6QK.mjs} +3 -3
  35. package/dist/{chunk-AB27FWZM.mjs → chunk-LPRRFCVU.mjs} +2 -2
  36. package/dist/{chunk-EIR3IQCL.mjs → chunk-QFJMRB7D.mjs} +2 -1
  37. package/dist/{chunk-S3BSDBF7.mjs → chunk-QJGDB3WR.mjs} +1 -1
  38. package/dist/{chunk-EI7U43MX.mjs → chunk-R7PLSVK3.mjs} +2 -2
  39. package/dist/{chunk-5DUTOEK5.mjs → chunk-TBOZHYCF.mjs} +2 -2
  40. package/dist/{chunk-VJOS6MHF.mjs → chunk-TNEU32EN.mjs} +64 -16
  41. package/dist/{chunk-RXMR2SIQ.mjs → chunk-UISV46WX.mjs} +4 -4
  42. package/dist/{chunk-4CAMH7FO.mjs → chunk-X4UMDSB4.mjs} +2 -1
  43. package/dist/{chunk-R4CE4IVU.mjs → chunk-X5NHWOXT.mjs} +15 -11
  44. package/dist/{react-native/chunks/chunk-HEELWLKL.mjs → chunk-XFOL77WH.mjs} +1 -1
  45. package/dist/{chunk-TOZENMM7.mjs → chunk-XITVDRRH.mjs} +5 -5
  46. package/dist/{chunk-FSLQHOKZ.mjs → chunk-XNFBYAXI.mjs} +1 -1
  47. package/dist/{chunk-V4YIEMAI.mjs → chunk-XQQKETAF.mjs} +4 -4
  48. package/dist/{chunk-DSXFN5BF.mjs → chunk-XTSFRWAS.mjs} +8 -8
  49. package/dist/{chunk-K4BMDPPX.mjs → chunk-Y27E2MGC.mjs} +2 -2
  50. package/dist/{chunk-JV74RKCY.mjs → chunk-YM2HI5N5.mjs} +1 -1
  51. package/dist/{chunk-7OE73OO4.mjs → chunk-YPV2L7M5.mjs} +56 -6
  52. package/dist/{chunk-STLNPJXU.mjs → chunk-YXZDTKQS.mjs} +2 -2
  53. package/dist/{chunk-5P72V66D.mjs → chunk-ZB5X5WAK.mjs} +24 -20
  54. package/dist/{client-VJmV_AjZ.d.ts → client-CeUxvIvd.d.ts} +4 -4
  55. package/dist/client.d.ts +8 -8
  56. package/dist/{config-ZZg4cwQz.d.ts → config-CIpLXXxm.d.ts} +2 -2
  57. package/dist/{config-Cb3FOlE_.d.ts → config-DehU-VGL.d.ts} +2 -2
  58. package/dist/{core-leaf-LOXCD7MY.mjs → core-leaf-VIMO377O.mjs} +3 -3
  59. package/dist/custom-token-results-NXHBVY7U.mjs +59 -0
  60. package/dist/{direct-executor-PF4UDSUW.mjs → direct-executor-2WQSGBN3.mjs} +7 -7
  61. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  62. package/dist/distribution/coordinator/etcd.mjs +1 -1
  63. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  64. package/dist/distribution/transport/grpc.mjs +1 -1
  65. package/dist/distribution/transport/tcp.mjs +1 -1
  66. package/dist/distribution.d.ts +7 -7
  67. package/dist/distribution.mjs +39 -39
  68. package/dist/{embed-4H7RTBVL.mjs → embed-UNHDFGDU.mjs} +2 -2
  69. package/dist/{engine-DdOs682x.d.ts → engine-P-4xhUe4.d.ts} +6 -2
  70. package/dist/{index-Ck9w5WHz.d.ts → index-CBUl4z3E.d.ts} +1 -1
  71. package/dist/index.browser.mjs +3 -3
  72. package/dist/index.d.ts +6 -6
  73. package/dist/index.mjs +7 -7
  74. package/dist/index.react-native.mjs +6 -6
  75. package/dist/{long-query-thread-OTVLP2OI.mjs → long-query-thread-AAYTLYZK.mjs} +19 -19
  76. package/dist/{module-UIKSHY3G.mjs → module-5SXNUCD5.mjs} +315 -64
  77. package/dist/{msgpack-leaf-5WFRP6JM.mjs → msgpack-leaf-7KBYOVTF.mjs} +2 -2
  78. package/dist/{optional-reads-RSLNSGNG.mjs → optional-reads-ENKMVQ4Q.mjs} +1 -1
  79. package/dist/{optional-writes-4ITCV7YT.mjs → optional-writes-N2DFWWZ3.mjs} +39 -26
  80. package/dist/{paged-partition-archive-VMJ2AUA6.mjs → paged-partition-archive-X7L27E5Z.mjs} +19 -19
  81. package/dist/partial-words.browser.mjs +3 -3
  82. package/dist/partial-words.mjs +7 -7
  83. package/dist/pattern.browser.mjs +2 -2
  84. package/dist/pattern.mjs +2 -2
  85. package/dist/persistence/durability/checkpoint-worker.d.ts +1 -1
  86. package/dist/persistence/durability/checkpoint-worker.mjs +17 -17
  87. package/dist/{phrase-module-IWSG2DDZ.mjs → phrase-module-ZY3TLTQA.mjs} +2 -1
  88. package/dist/{query-actions-EW72OQXY.mjs → query-actions-2JOLPELC.mjs} +7 -7
  89. package/dist/query-suggestions/merge-worker.mjs +10 -10
  90. package/dist/query-suggestions.browser.mjs +3 -3
  91. package/dist/query-suggestions.d.ts +2 -2
  92. package/dist/query-suggestions.mjs +8 -8
  93. package/dist/{chunk-ZBVHS5KF.mjs → react-native/chunks/chunk-57R74JWC.mjs} +1 -1
  94. package/dist/react-native/chunks/{chunk-AUXXPSFB.mjs → chunk-F4MIGBBP.mjs} +214 -150
  95. package/dist/react-native/chunks/{chunk-3RUXKZJG.mjs → chunk-F6BJYCF2.mjs} +2 -2
  96. package/dist/react-native/chunks/{chunk-RVHCQJQM.mjs → chunk-GGBATERA.mjs} +3 -3
  97. package/dist/react-native/chunks/{chunk-I5W7A2JF.mjs → chunk-GVWLACAT.mjs} +2 -2
  98. package/dist/react-native/chunks/{chunk-XOCM3BFU.mjs → chunk-H344ZXRH.mjs} +2 -2
  99. package/dist/react-native/chunks/{chunk-VIPNDN4B.mjs → chunk-I7DTUOK2.mjs} +65 -15
  100. package/dist/react-native/chunks/{chunk-2JMAUU26.mjs → chunk-J2BQTJJE.mjs} +1 -1
  101. package/dist/{chunk-QBVTYY4J.mjs → react-native/chunks/chunk-OZ3XEM6D.mjs} +1 -1
  102. package/dist/react-native/chunks/{chunk-4EXKQ2IB.mjs → chunk-SK3DPAI4.mjs} +1 -1
  103. package/dist/react-native/chunks/{chunk-XBTFK3AZ.mjs → chunk-WFQ6YQZA.mjs} +1 -1
  104. package/dist/react-native/chunks/{chunk-FTQ7JNMU.mjs → chunk-ZDDJVR7X.mjs} +15 -11
  105. package/dist/react-native/chunks/{core-leaf-O3236QY3.mjs → core-leaf-UDUMVC3M.mjs} +2 -2
  106. package/dist/react-native/chunks/custom-token-results-3OZEMRE7.mjs +59 -0
  107. package/dist/react-native/chunks/{embed-3WQ2CCR5.mjs → embed-U7CIKEY4.mjs} +2 -2
  108. package/dist/react-native/chunks/{module-LRR7D5ZO.mjs → module-7BJ3VOWQ.mjs} +360 -66
  109. package/dist/react-native/chunks/{msgpack-leaf-TR66ZVAX.mjs → msgpack-leaf-GTKLF6GJ.mjs} +2 -2
  110. package/dist/react-native/chunks/{optional-reads-TOWGNEDH.mjs → optional-reads-74HYI4DO.mjs} +1 -1
  111. package/dist/react-native/chunks/{optional-writes-7AFBIZMN.mjs → optional-writes-TXTJ6HUZ.mjs} +37 -24
  112. package/dist/react-native/chunks/{phrase-module-3A4AKMKL.mjs → phrase-module-UC4YGEBB.mjs} +2 -1
  113. package/dist/react-native/chunks/{resolve-vector-text-LXRYXF7M.mjs → resolve-vector-text-FTPD5EY6.mjs} +8 -0
  114. package/dist/react-native/chunks/{restore-queue-7FSHTGJ5.mjs → restore-queue-SZLWIL3B.mjs} +9 -9
  115. package/dist/react-native/chunks/{segment-codec-leaf-4QV4HO4R.mjs → segment-codec-leaf-WMUZ5P4F.mjs} +7 -7
  116. package/dist/react-native/chunks/{segment-merge-leaf-5DUZZSXJ.mjs → segment-merge-leaf-Q2OEJIQ3.mjs} +8 -8
  117. package/dist/react-native/chunks/{storage-leaf-PNU6HWY7.mjs → storage-leaf-OXYV6O6P.mjs} +213 -100
  118. package/dist/react-native/chunks/{store-leaf-ESVUGYYO.mjs → store-leaf-ZMDXYK4E.mjs} +8 -8
  119. package/dist/react-native/chunks/{suggestions-leaf-MOVOPBGD.mjs → suggestions-leaf-FIMGQZ2T.mjs} +8 -8
  120. package/dist/react-native/chunks/{worker-copies-VQGF3WZX.mjs → worker-copies-72TGFYQB.mjs} +8 -7
  121. package/dist/react-native/partial-words.mjs +6 -6
  122. package/dist/react-native/pattern.mjs +3 -3
  123. package/dist/react-native/query-suggestions.mjs +6 -6
  124. package/dist/react-native/sync.mjs +6 -6
  125. package/dist/react-native/synonyms.mjs +6 -6
  126. package/dist/react.d.ts +9 -9
  127. package/dist/{rerank-QHJXPSAK.mjs → rerank-E2HFKKGJ.mjs} +7 -7
  128. package/dist/{resolve-vector-text-4PDRTPNQ.mjs → resolve-vector-text-6WPFFIX5.mjs} +1 -1
  129. package/dist/{restore-queue-5W5HX44H.mjs → restore-queue-PUVBN6L3.mjs} +10 -10
  130. package/dist/{schema-C_7yRJLt.d.ts → schema-CzG9OUw5.d.ts} +5 -1
  131. package/dist/{segment-codec-leaf-7VLVXKWO.mjs → segment-codec-leaf-24Z3MMQI.mjs} +8 -8
  132. package/dist/{segment-merge-leaf-22XR3IGA.mjs → segment-merge-leaf-FQQDU45E.mjs} +8 -8
  133. package/dist/server.d.ts +10 -10
  134. package/dist/server.mjs +26 -26
  135. package/dist/{storage-leaf-ZZ77C3DX.mjs → storage-leaf-KIA7XO6Y.mjs} +79 -81
  136. package/dist/{store-leaf-LJ6D2PMF.mjs → store-leaf-ESVLL4JO.mjs} +9 -9
  137. package/dist/{suggestions-leaf-KTQHO3PY.mjs → suggestions-leaf-FXF4AHAY.mjs} +11 -11
  138. package/dist/sync/filesystem.d.ts +3 -3
  139. package/dist/sync/filesystem.mjs +15 -15
  140. package/dist/sync/indexeddb.browser.mjs +1 -1
  141. package/dist/sync/indexeddb.d.ts +3 -3
  142. package/dist/sync/indexeddb.mjs +2 -2
  143. package/dist/sync.browser.mjs +3 -3
  144. package/dist/sync.d.ts +4 -4
  145. package/dist/sync.mjs +7 -7
  146. package/dist/synonyms.browser.mjs +3 -3
  147. package/dist/synonyms.mjs +7 -7
  148. package/dist/{tasks-CxA-J_Ty.d.ts → tasks-CXtQ3Zpp.d.ts} +4 -4
  149. package/dist/thread-OWXVP2ZA.mjs +30 -0
  150. package/dist/{types-BboB278g.d.ts → types-CDN70Y_C.d.ts} +2 -2
  151. package/dist/vector/search-worker.d.ts +1 -1
  152. package/dist/vector/search-worker.mjs +5 -5
  153. package/dist/{worker-copies-ODAAGQRC.mjs → worker-copies-XXX7PF4J.mjs} +10 -10
  154. package/dist/workers/entry.mjs +13 -13
  155. package/package.json +9 -8
  156. package/dist/thread-XWDK6TAR.mjs +0 -30
  157. /package/dist/{chunk-77AUVDQ5.mjs → chunk-CV3Q2EKO.mjs} +0 -0
@@ -1646,6 +1646,7 @@ var extractVectorFromDoc;
1646
1646
  var fnv1a;
1647
1647
  var generateId;
1648
1648
  var getLanguage;
1649
+ var getNestedValue;
1649
1650
  var getVectorFieldPaths;
1650
1651
  var indexChangeOutsideExternalWaits;
1651
1652
  var insertDocument;
@@ -1720,6 +1721,7 @@ function bindOptionalWritesHost(host, segmentCodec) {
1720
1721
  fnv1a,
1721
1722
  generateId,
1722
1723
  getLanguage,
1724
+ getNestedValue,
1723
1725
  getVectorFieldPaths,
1724
1726
  indexChangeOutsideExternalWaits,
1725
1727
  insertDocument,
@@ -1886,21 +1888,26 @@ function recoveryFailure(metadata, err) {
1886
1888
  stopWords: metadata.stopWords
1887
1889
  });
1888
1890
  }
1889
- async function restorePartitionCount(deps, metadata) {
1890
- if (!Number.isInteger(metadata.partitionCount) || metadata.partitionCount < 1 || metadata.partitionCount > 4294967295) {
1891
- throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, "The saved index has an invalid partition count");
1891
+ async function savedPartitionCountOf(metadata) {
1892
+ const count = metadata.partitionCount;
1893
+ if (typeof count !== "number" || !Number.isSafeInteger(count) || count < 1) {
1894
+ const held = typeof count === "number" ? String(count) : `a ${typeof count} as its count of`;
1895
+ throw new NarsilError(
1896
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
1897
+ `Recovery of index "${metadata.indexName}" failed: its metadata holds ${held} partitions, and an index holds a whole number above zero`,
1898
+ { indexName: metadata.indexName, partitionCount: typeof count === "number" ? count : typeof count }
1899
+ );
1892
1900
  }
1893
- const manager = deps.executor.getManager(metadata.indexName);
1894
- if (manager === void 0) return;
1895
- await manager.publication.write(() => {
1896
- while (manager.partitionCount < metadata.partitionCount) manager.addPartition();
1897
- if (manager.partitionCount > metadata.partitionCount) {
1898
- manager.setPartitions(manager.getAllPartitions().slice(0, metadata.partitionCount));
1899
- }
1900
- });
1901
+ const { requireSavedPartitionCount } = await loadStorageCode();
1902
+ requireSavedPartitionCount(count, "metadata", metadata.indexName);
1903
+ return count;
1901
1904
  }
1902
1905
  async function createIndexFromMetadata(deps, metadata, loadData) {
1903
- const loadingSchemaCode = loadData ? loadSchemaCodeFor(metadata.schema) : null;
1906
+ const loadingSchemaCode = loadSchemaCodeFor(
1907
+ loadData ? metadata.schema : null,
1908
+ metadata.tokenizer,
1909
+ metadata.pendingAnalysis?.tokenizer
1910
+ );
1904
1911
  if (loadingSchemaCode !== null) await loadingSchemaCode;
1905
1912
  const existing = deps.indexRegistry.get(metadata.indexName);
1906
1913
  if (existing !== void 0) {
@@ -1909,14 +1916,15 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1909
1916
  adoptSavedAnalysis(deps.executor.getManager(metadata.indexName), existing, metadata);
1910
1917
  }
1911
1918
  if (loadData && deps.executor.getManager(metadata.indexName) === void 0) {
1919
+ const partitionCount2 = await savedPartitionCountOf(metadata);
1912
1920
  deps.executor.createIndex(
1913
1921
  metadata.indexName,
1914
1922
  existing.config,
1915
1923
  existing.language,
1916
1924
  false,
1917
- lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language)
1925
+ lookupOfStoredSynonyms(existing.synonyms, existing.config, existing.language),
1926
+ partitionCount2
1918
1927
  );
1919
- await restorePartitionCount(deps, metadata);
1920
1928
  }
1921
1929
  return;
1922
1930
  }
@@ -1929,6 +1937,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1929
1937
  } catch (err) {
1930
1938
  throw recoveryFailure(metadata, err);
1931
1939
  }
1940
+ const partitionCount = await savedPartitionCountOf(metadata);
1932
1941
  const embedding = resolveEmbeddingAdapter(deps, metadata, indexConfig.schema);
1933
1942
  const prepared = prepareStoredSynonyms(
1934
1943
  metadata.synonyms,
@@ -1941,8 +1950,14 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1941
1950
  validateVectorStorage(indexConfig.vectorPromotion, deps.filesystemDurability);
1942
1951
  validateTextStorage(indexConfig.textStorage, deps.filesystemDurability);
1943
1952
  if (loadData) {
1944
- deps.executor.createIndex(metadata.indexName, indexConfig, language, false, prepared?.lookup ?? null);
1945
- await restorePartitionCount(deps, metadata);
1953
+ deps.executor.createIndex(
1954
+ metadata.indexName,
1955
+ indexConfig,
1956
+ language,
1957
+ false,
1958
+ prepared?.lookup ?? null,
1959
+ partitionCount
1960
+ );
1946
1961
  }
1947
1962
  } catch (err) {
1948
1963
  throw recoveryFailure(metadata, err);
@@ -1956,7 +1971,7 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1956
1971
  indexUuid: metadata.indexUuid ?? null,
1957
1972
  heldPartitions: metadata.heldPartitions ?? null,
1958
1973
  documentCount: metadata.documentCount ?? 0,
1959
- partitionCount: metadata.partitionCount,
1974
+ partitionCount,
1960
1975
  analysisRevision: metadata.analysisRevision ?? null,
1961
1976
  arrayPositionGap: metadata.arrayPositionGap ?? null,
1962
1977
  synonyms,
@@ -1979,6 +1994,20 @@ async function createIndexFromMetadata(deps, metadata, loadData) {
1979
1994
  }
1980
1995
 
1981
1996
  // src/engine/mutations/durable-rollback.ts
1997
+ function holdVectorsForRollback(indexes, docId) {
1998
+ const releases = [];
1999
+ for (const index of indexes.values()) releases.push(index.holdForRollback(docId));
2000
+ return () => {
2001
+ for (const release of releases) release();
2002
+ };
2003
+ }
2004
+ function restoreDocumentVectors(docId, vectors, indexes, partitionId) {
2005
+ for (const [fieldPath, index] of indexes) {
2006
+ const vector = vectors.get(fieldPath);
2007
+ if (vector !== void 0) index.restore(docId, vector, partitionId);
2008
+ else if (index.has(docId)) index.remove(docId);
2009
+ }
2010
+ }
1982
2011
  function rollbackFailed(docId, originalError, rollbackError) {
1983
2012
  throw new NarsilError(
1984
2013
  ErrorCodes.REPLICATION_ROLLBACK_FAILED,
@@ -2027,7 +2056,7 @@ async function rollbackRemovedDocument(ctx, indexName, docId, restoreDoc, origin
2027
2056
  manager.insert(docId, previous.partitionDoc, {
2028
2057
  preparedText: { after: preparedText?.before }
2029
2058
  });
2030
- insertDocumentVectors(
2059
+ restoreDocumentVectors(
2031
2060
  docId,
2032
2061
  previous.extractedVectors,
2033
2062
  manager.getVectorIndexes(),
@@ -2061,8 +2090,12 @@ async function rollbackUpdatedDocument(ctx, indexName, docId, restoreDoc, origin
2061
2090
  skipClone: rollbackRef !== void 0,
2062
2091
  preparedText: { before: preparedText?.after, after: preparedText?.before }
2063
2092
  });
2064
- const vectors = new Map([...fields].map((field) => [field, previous.extractedVectors.get(field) ?? null]));
2065
- updateDocumentVectors(docId, vectors, manager.getVectorIndexes(), manager.partitionIdOf(docId));
2093
+ restoreDocumentVectors(
2094
+ docId,
2095
+ previous.extractedVectors,
2096
+ manager.getVectorIndexes(),
2097
+ manager.partitionIdOf(docId)
2098
+ );
2066
2099
  ctx.orchestrator.publishToWorkers({
2067
2100
  type: "update",
2068
2101
  indexName,
@@ -2761,6 +2794,7 @@ function applyOfRemoval(ctx, indexName, prepared) {
2761
2794
  return queued;
2762
2795
  }
2763
2796
  prepared.restoreDoc = ctx.durability ? manager.get(docId) : void 0;
2797
+ if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
2764
2798
  manager.remove(docId, void 0, {
2765
2799
  onRemovedSpellings: (spellings) => {
2766
2800
  prepared.removedSpellings = spellings;
@@ -2801,6 +2835,8 @@ async function removeDocument2(ctx, indexName, docId, options) {
2801
2835
  } catch (err) {
2802
2836
  await rollbackRemovedDocument(ctx, indexName, docId, prepared.restoreDoc, err, prepared.preparedText);
2803
2837
  throw err;
2838
+ } finally {
2839
+ prepared.releaseVectorHolds?.();
2804
2840
  }
2805
2841
  } else {
2806
2842
  await apply();
@@ -2829,29 +2865,35 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
2829
2865
  }
2830
2866
  }
2831
2867
  const applies = prepared.map((removal) => applyOfRemoval(ctx, indexName, removal));
2832
- const failures = await recordChunk(ctx, indexName, prepared, applies);
2833
- for (let i = 0; i < prepared.length; i++) {
2834
- const removal = prepared[i];
2835
- const failure = failures[i];
2836
- if (failure !== null) {
2837
- let error = failure.error;
2838
- try {
2839
- await rollbackRemovedDocument(
2840
- ctx,
2841
- indexName,
2842
- removal.docId,
2843
- removal.restoreDoc,
2844
- error,
2845
- removal.preparedText
2846
- );
2847
- } catch (rollbackError) {
2848
- error = rollbackError;
2868
+ try {
2869
+ const failures = await recordChunk(ctx, indexName, prepared, applies);
2870
+ for (let i = 0; i < prepared.length; i++) {
2871
+ const removal = prepared[i];
2872
+ const failure = failures[i];
2873
+ if (failure !== null) {
2874
+ let error = failure.error;
2875
+ try {
2876
+ await rollbackRemovedDocument(
2877
+ ctx,
2878
+ indexName,
2879
+ removal.docId,
2880
+ removal.restoreDoc,
2881
+ error,
2882
+ removal.preparedText
2883
+ );
2884
+ } catch (rollbackError) {
2885
+ error = rollbackError;
2886
+ }
2887
+ removal.releaseVectorHolds?.();
2888
+ failed.push({ docId: removal.docId, error: asRemoveError(error) });
2889
+ continue;
2849
2890
  }
2850
- failed.push({ docId: removal.docId, error: asRemoveError(error) });
2851
- continue;
2891
+ removal.releaseVectorHolds?.();
2892
+ if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
2893
+ succeeded.push(removal.docId);
2852
2894
  }
2853
- if (!removal.buffered) await settleRemoval(ctx, indexName, removal);
2854
- succeeded.push(removal.docId);
2895
+ } finally {
2896
+ for (const removal of prepared) removal.releaseVectorHolds?.();
2855
2897
  }
2856
2898
  if (chunkEnd < docIds.length) {
2857
2899
  await new Promise((r) => setTimeout(r, 0));
@@ -2866,9 +2908,6 @@ async function removeDocumentBatch(ctx, indexName, docIds, options) {
2866
2908
  }
2867
2909
 
2868
2910
  // src/engine/mutations/update.ts
2869
- function extractVectorFromDocForUpdate(document, fieldPath) {
2870
- return extractVectorFromDoc(document, fieldPath);
2871
- }
2872
2911
  function prepareUpdatePartitionDoc(document, extractedVectors) {
2873
2912
  if (extractedVectors.size === 0) {
2874
2913
  return { partitionDoc: document };
@@ -2892,8 +2931,8 @@ function prepareUpdatePartitionDoc(document, extractedVectors) {
2892
2931
  for (const fieldPath of extractedVectors.keys()) delete source[fieldPath];
2893
2932
  }
2894
2933
  const partitionDoc = structuredClone(source);
2895
- for (const fieldPath of extractedVectors.keys()) {
2896
- deleteNestedValue(partitionDoc, fieldPath);
2934
+ for (const [fieldPath, vector] of extractedVectors) {
2935
+ if (vector !== null || getNestedValue(partitionDoc, fieldPath) == null) deleteNestedValue(partitionDoc, fieldPath);
2897
2936
  }
2898
2937
  return { partitionDoc };
2899
2938
  }
@@ -2958,7 +2997,7 @@ async function prepareUpdate(ctx, indexName, entry, docId, document) {
2958
2997
  if (vecIndexes.size > 0) {
2959
2998
  const dimensionCheckVectors = /* @__PURE__ */ new Map();
2960
2999
  for (const fieldPath of entry.vectorFieldPaths) {
2961
- const newVec = extractVectorFromDocForUpdate(document, fieldPath);
3000
+ const newVec = extractVectorFromDoc(document, fieldPath);
2962
3001
  extractedVectors.set(fieldPath, newVec);
2963
3002
  if (newVec) {
2964
3003
  dimensionCheckVectors.set(fieldPath, newVec);
@@ -2996,9 +3035,15 @@ function applyOfUpdate(ctx, indexName, prepared) {
2996
3035
  prepared.buffered = true;
2997
3036
  return queued;
2998
3037
  }
3038
+ if (manager.has(docId)) {
3039
+ for (const [fieldPath, vector] of extractedVectors) {
3040
+ if (vector !== null) manager.getVectorIndexes().get(fieldPath)?.requireRoomFor(docId, vector);
3041
+ }
3042
+ }
2999
3043
  prepared.oldDocument = extractedVectors.size > 0 || ctx.durability !== null || ctx.pluginRegistry.hasHooks("afterUpdate") ? manager.get(docId) : void 0;
3000
3044
  prepared.rollbackDoc = prepared.oldDocument;
3001
3045
  prepared.rollbackRef = ctx.durability === null ? void 0 : manager.getRef(docId);
3046
+ if (ctx.durability !== null) prepared.releaseVectorHolds = holdVectorsForRollback(manager.getVectorIndexes(), docId);
3002
3047
  const stored = manager.update(docId, partitionDoc, {
3003
3048
  onRemovedSpellings: (spellings) => {
3004
3049
  prepared.removedSpellings = spellings;
@@ -3060,6 +3105,8 @@ async function updateDocument(ctx, indexName, docId, document, options) {
3060
3105
  prepared.rollbackRef
3061
3106
  );
3062
3107
  throw err;
3108
+ } finally {
3109
+ prepared.releaseVectorHolds?.();
3063
3110
  }
3064
3111
  } else {
3065
3112
  await apply();
@@ -3108,25 +3155,30 @@ async function updateDocumentBatch(ctx, indexName, updates, options) {
3108
3155
  }
3109
3156
  }
3110
3157
  const applies = prepared.map((update) => applyOfUpdate(ctx, indexName, update));
3111
- const failures = await recordChunk(ctx, indexName, prepared, applies);
3112
- for (let i = prepared.length - 1; i >= 0; i--) {
3113
- const failure = failures[i];
3114
- if (failure === null) continue;
3115
- const update = prepared[i];
3116
- try {
3117
- await rollbackUpdatedDocument(
3118
- ctx,
3119
- indexName,
3120
- update.docId,
3121
- update.rollbackDoc,
3122
- failure.error,
3123
- update.preparedText,
3124
- update.appliedRef,
3125
- update.rollbackRef
3126
- );
3127
- } catch (error) {
3128
- failure.error = error;
3158
+ let failures;
3159
+ try {
3160
+ failures = await recordChunk(ctx, indexName, prepared, applies);
3161
+ for (let i = prepared.length - 1; i >= 0; i--) {
3162
+ const failure = failures[i];
3163
+ if (failure === null) continue;
3164
+ const update = prepared[i];
3165
+ try {
3166
+ await rollbackUpdatedDocument(
3167
+ ctx,
3168
+ indexName,
3169
+ update.docId,
3170
+ update.rollbackDoc,
3171
+ failure.error,
3172
+ update.preparedText,
3173
+ update.appliedRef,
3174
+ update.rollbackRef
3175
+ );
3176
+ } catch (error) {
3177
+ failure.error = error;
3178
+ }
3129
3179
  }
3180
+ } finally {
3181
+ for (const update of prepared) update.releaseVectorHolds?.();
3130
3182
  }
3131
3183
  for (let i = 0; i < prepared.length; i++) {
3132
3184
  const update = prepared[i];
@@ -3193,4 +3245,4 @@ async function updateEnginePartitionConfig(core, indexName, partitionConfig) {
3193
3245
  core.watermarkNotifier.check(indexName);
3194
3246
  }
3195
3247
 
3196
- export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
3248
+ export { applyStoredIndexBm25, bindOptionalWritesHost, createIndexFromMetadata, holdVectorsForRollback, insertDocument, insertDocumentBatch, readProcessMemory, removeDocument2 as removeDocument, removeDocumentBatch, rollbackInsertedDocument, rollbackRemovedDocument, rollbackUpdatedDocument, updateDocument, updateDocumentBatch, updateEnginePartitionConfig, updateIndexBm25 };
@@ -560,6 +560,7 @@ function analyseQueryPhrases(parts, analyseText) {
560
560
  wordsAfterLastPhrase = "";
561
561
  if (size > largestPhraseSize) largestPhraseSize = size;
562
562
  if (ordered.length > largestTokenCount) largestTokenCount = ordered.length;
563
+ if (size > MAX_PHRASE_WORDS || ordered.length > MAX_PHRASE_WORDS) continue;
563
564
  const tokens = ordered.map((entry) => entry.token);
564
565
  const offsets = ordered.map((entry) => entry.position - firstPosition);
565
566
  const identity = identityOf(tokens, offsets);
@@ -52,6 +52,14 @@ async function resolveVectorText(params, embeddingAdapter, signal, embeddingAdap
52
52
  { expected: embeddingAdapter.dimensions, actual: raw.length }
53
53
  );
54
54
  }
55
+ for (let index = 0; index < raw.length; index++) {
56
+ if (Number.isFinite(raw[index])) continue;
57
+ throw new NarsilError(
58
+ ErrorCodes.EMBEDDING_FAILED,
59
+ `Adapter returned vector with ${raw[index]} at index ${index} for query embedding`,
60
+ { index }
61
+ );
62
+ }
55
63
  const resolved = { ...params.vector, value: Array.from(raw) };
56
64
  delete resolved.text;
57
65
  return { ...params, vector: resolved };