@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,18 +1,19 @@
1
1
  import { noteWaitingForMemory, admitsBackgroundBytes } from './chunk-PELHVTO7.mjs';
2
- import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, threadsHoldCurrentLayout, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, WORKER_COPY_MIN_VECTORS, allLiveDocIds, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-6VHWXAOI.mjs';
3
- import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-ICW45TTU.mjs';
2
+ import { VECTOR_WORKER_COPIES_ALLOWED, DEFAULT_PROMOTION_THRESHOLD, DEFAULT_FILTER_THRESHOLD, noBuildHistory, recordCheckpoint, planCheckpoint, deserialize, assignStorePartitions, liveSize, ordinalFilterForSubsets, handOverBuildTurns, filterForOptions, OSQ4_MIN_DIMENSION, emptyFieldBeforeRestore, ESTIMATED_MS_PER_TOMBSTONE, graphNeedsRebuild, ESTIMATED_MS_PER_VECTOR_REBUILD, releaseReplacedVectors, yieldBuildTurn, allLiveDocIds, threadsHoldCurrentLayout, WORKER_COPY_MIN_VECTORS, SHUFFLED_REBUILD_MEMORY_RETRY_MS, recalibrateFromStore, signatureOf, adoptGraph, BUILD_RETRY_MAX_DELAY_MS, BUILD_RETRY_FIRST_DELAY_MS, fieldHandlesOf, liveOrdinals, RECALIBRATION_CHUNK_VECTORS, BUILD_CHUNK_SIZE, DRIFT_CHECK_GROWTH_FACTOR, DISK_ADOPTION_YIELD_INTERVAL, adoptLoadedCalibration, RECALIBRATION_DRIFT_RATIO } from './chunk-Z4HFMYNX.mjs';
4
3
  import { chargeHeapBytes } from './chunk-IWVEG5CX.mjs';
5
- import { createVectorStore } from './chunk-CCCGEPJM.mjs';
6
- export { openSharedVectorField } from './chunk-CCCGEPJM.mjs';
7
- import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-K3AQOZVM.mjs';
8
- export { releaseLocksHeldBy } from './chunk-K3AQOZVM.mjs';
4
+ import { isDead, NO_LIVE_VECTOR_AT_POSITION, bytesToVectors } from './chunk-ICW45TTU.mjs';
5
+ import { createVectorStore } from './chunk-XZ3SRNDJ.mjs';
6
+ export { openSharedVectorField } from './chunk-XZ3SRNDJ.mjs';
7
+ import './chunk-FOCV4CBZ.mjs';
8
+ import { osqBitsOf, createOsqQuantizer, STORE_ORDINAL_LAYOUT_GENERATION, ordinalFilterValues, STORE_UNPLACED_COUNT, STORE_LIVE_COUNT, STORE_REMOVED_COUNT, createOrdinalFilter, addToOrdinalFilter, nativeScoresOf, toScore, ordinalFilterHas, magnitude, nativeStoreScratchBytes, toDistance, euclideanDistance, dotProduct, cosineSimilarityWithMagnitudes, createHNSWIndex, topLayerInParts, sharedMemoryAvailable, requireWholeNeighbourLists, shuffleInPlace, createCentroidSums, centroidOfCentroidSums, meanOfCentroidSums, countOfCentroidSums, squaredLengthsOfCentroidSums, releaseLocksHeldBy } from './chunk-TNBIK2LQ.mjs';
9
+ export { releaseLocksHeldBy } from './chunk-TNBIK2LQ.mjs';
9
10
  import './chunk-37CD3OKS.mjs';
10
11
  import { noteFallback } from './chunk-DJHHF2L5.mjs';
11
12
  import { spawnNodeWorker } from './chunk-W55X2BYN.mjs';
12
13
  import { resolveWorkerCount, resolveWorkerEntry, workerResourceLimits } from './chunk-UIKN2OTQ.mjs';
13
14
  import { detectRuntime } from './chunk-AV4IJKD2.mjs';
14
- import './chunk-EBIHRL7B.mjs';
15
15
  import './chunk-EEBCDP36.mjs';
16
+ import './chunk-EBIHRL7B.mjs';
16
17
  import './chunk-KOKM4PQN.mjs';
17
18
  import { MAX_RENUMBERED_SEARCH_ATTEMPTS, DISK_STORAGE_BLOCK_BYTES, UNPLACED_FLAG_CHUNK_ORDINALS, GRAPH_BUILD_CHUNKS_IN_FLIGHT_PER_WORKER, GRAPH_BUILD_DISPATCH_CHUNK, VECTOR_SEARCH_TIMEOUT_MS, VECTOR_SEARCH_LOAD_TIMEOUT_MS, VECTOR_SCRATCH_SLOTS, VECTOR_INSERT_TIMEOUT_MS } from './chunk-QSK35UC2.mjs';
18
19
  import './chunk-74XB2SXI.mjs';
@@ -1607,19 +1608,25 @@ function beginCheckpointRestore(state, shape) {
1607
1608
  const dead = isDead(file.dead, i);
1608
1609
  if (dead && !graphHoldsNodeFor(firstNumber + i)) continue;
1609
1610
  const docId = payload.docIds[i];
1610
- const vector = vectors.subarray(i * dimension, (i + 1) * dimension);
1611
1611
  const offset = onDisk ? location.vectorsOffset + i * dimension * 4 : 0;
1612
+ const vectorHere = () => file.byParts === void 0 ? vectors.subarray(i * dimension, (i + 1) * dimension) : file.byParts.vectorAt(i);
1612
1613
  if (dead) {
1613
- ordinals[i] = state.store.insertReplaced(docId, vector, fileIndex === null ? void 0 : { fileIndex, offset });
1614
+ ordinals[i] = state.store.insertReplaced(
1615
+ docId,
1616
+ vectorHere(),
1617
+ fileIndex === null ? void 0 : { fileIndex, offset }
1618
+ );
1614
1619
  tombstoned.add(ordinals[i]);
1615
1620
  continue;
1616
1621
  }
1617
1622
  const replaced = state.store.getOrdinal(docId);
1618
1623
  if (replaced !== void 0) state.osq?.removeOrdinal(replaced);
1619
1624
  if (fileIndex !== null) {
1620
- ordinals[i] = state.store.insertCold(docId, magnitude(vector), { fileIndex, offset });
1625
+ const known = file.byParts?.magnitudes[i];
1626
+ ordinals[i] = state.store.insertCold(docId, known ?? magnitude(vectorHere()), { fileIndex, offset });
1621
1627
  continue;
1622
1628
  }
1629
+ const vector = vectorHere();
1623
1630
  ordinals[i] = state.store.insert(docId, vector);
1624
1631
  if (onDisk) state.pendingLocations.set(docId, { path: location.path, offset });
1625
1632
  }
@@ -1661,6 +1668,223 @@ function beginCheckpointRestore(state, shape) {
1661
1668
  return { addFile, finish };
1662
1669
  }
1663
1670
 
1671
+ // src/vector/vector-index/cold.ts
1672
+ function beginColdHold(state, field, loaded, searchAnswered) {
1673
+ const hold = {
1674
+ field,
1675
+ loaded,
1676
+ searchAnswered,
1677
+ written: /* @__PURE__ */ new Map(),
1678
+ removedNumbers: /* @__PURE__ */ new Set(),
1679
+ removedIds: /* @__PURE__ */ new Set(),
1680
+ writes: [],
1681
+ live: field.liveCount
1682
+ };
1683
+ state.cold = hold;
1684
+ return hold;
1685
+ }
1686
+ function endColdHold(state) {
1687
+ const hold = state.cold;
1688
+ if (hold === null) return;
1689
+ state.cold = null;
1690
+ for (const write of hold.writes) write();
1691
+ hold.field.close();
1692
+ }
1693
+ function cancelColdHold(state) {
1694
+ const hold = state.cold;
1695
+ if (hold === null) return;
1696
+ state.cold = null;
1697
+ hold.field.close();
1698
+ }
1699
+ function openColdField(state, field, load) {
1700
+ cancelColdHold(state);
1701
+ let finished = () => void 0;
1702
+ let failed = () => void 0;
1703
+ const loaded = new Promise((resolve, reject) => {
1704
+ finished = resolve;
1705
+ failed = reject;
1706
+ });
1707
+ let searchAnswered = () => void 0;
1708
+ const firstSearchAnswered = new Promise((resolve) => {
1709
+ searchAnswered = resolve;
1710
+ });
1711
+ const hold = beginColdHold(state, field, loaded, searchAnswered);
1712
+ void load(
1713
+ (shape) => beginCheckpointRestore(state, shape),
1714
+ () => state.cold === hold,
1715
+ firstSearchAnswered
1716
+ ).then(
1717
+ () => {
1718
+ if (state.cold === hold) endColdHold(state);
1719
+ finished();
1720
+ },
1721
+ (error) => failed(error)
1722
+ );
1723
+ loaded.catch(() => void 0);
1724
+ }
1725
+ function retireOlderCopy(hold, docId) {
1726
+ if (hold.written.delete(docId)) return true;
1727
+ if (hold.removedIds.has(docId)) return false;
1728
+ const number = hold.field.numberOf(docId);
1729
+ if (number === void 0) return false;
1730
+ hold.removedIds.add(docId);
1731
+ hold.removedNumbers.add(number);
1732
+ return true;
1733
+ }
1734
+ function coldInsert(hold, docId, vector, partitionId) {
1735
+ const replaced = retireOlderCopy(hold, docId);
1736
+ if (!replaced) hold.live += 1;
1737
+ hold.written.set(docId, { vector: vector.slice(), partitionId });
1738
+ return replaced;
1739
+ }
1740
+ function coldRemove(hold, docId) {
1741
+ const removed = retireOlderCopy(hold, docId);
1742
+ if (removed) hold.live -= 1;
1743
+ return removed;
1744
+ }
1745
+ function coldHas(hold, docId) {
1746
+ if (hold.written.has(docId)) return true;
1747
+ if (hold.removedIds.has(docId)) return false;
1748
+ return hold.field.numberOf(docId) !== void 0;
1749
+ }
1750
+ function coldVectorOf(hold, docId) {
1751
+ const written = hold.written.get(docId);
1752
+ if (written !== void 0) return written.vector.slice();
1753
+ if (hold.removedIds.has(docId)) return null;
1754
+ const number = hold.field.numberOf(docId);
1755
+ return number === void 0 ? null : hold.field.vectorOfNumber(number);
1756
+ }
1757
+ function isAdmitted(subsets, docId) {
1758
+ for (const subset of subsets) {
1759
+ const { docStore } = subset.owner;
1760
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
1761
+ const internalId = docStore.getInternalId(docId);
1762
+ if (internalId === void 0) continue;
1763
+ const word = subset.bits[internalId >>> 5];
1764
+ if (word !== void 0 && (word & 1 << (internalId & 31)) !== 0 && docStore.resolver().isLive(internalId)) {
1765
+ return true;
1766
+ }
1767
+ }
1768
+ return false;
1769
+ }
1770
+ function admissionOf(options, hold) {
1771
+ const { filterSubsets, filterPartitions } = options;
1772
+ if (filterSubsets !== void 0) return (docId) => isAdmitted(filterSubsets, docId);
1773
+ if (filterPartitions !== void 0) {
1774
+ return (docId) => {
1775
+ const partition = hold.field.partitionOf(docId);
1776
+ return partition !== void 0 && filterPartitions.has(partition);
1777
+ };
1778
+ }
1779
+ return null;
1780
+ }
1781
+ function bitCount(word) {
1782
+ let value = word - (word >>> 1 & 1431655765);
1783
+ value = (value & 858993459) + (value >>> 2 & 858993459);
1784
+ return (value + (value >>> 4) & 252645135) * 16843009 >>> 24;
1785
+ }
1786
+ function admittedBySubsets(subsets) {
1787
+ let admitted = 0;
1788
+ for (const subset of subsets) {
1789
+ for (let at = 0; at < subset.bits.length; at++) admitted += bitCount(subset.bits[at]);
1790
+ }
1791
+ return admitted;
1792
+ }
1793
+ function* idsOfSubsets(subsets) {
1794
+ for (const subset of subsets) {
1795
+ const { bits } = subset;
1796
+ const { docStore } = subset.owner;
1797
+ if (docStore.numberingGeneration() !== subset.numberingGeneration) continue;
1798
+ for (let wordIndex = 0; wordIndex < bits.length; wordIndex++) {
1799
+ let word = bits[wordIndex];
1800
+ while (word !== 0) {
1801
+ const internalId = (wordIndex << 5) + (Math.clz32(word & -word) ^ 31);
1802
+ word &= word - 1;
1803
+ if (!docStore.resolver().isLive(internalId)) continue;
1804
+ const docId = docStore.getExternalId(internalId);
1805
+ if (docId !== void 0) yield docId;
1806
+ }
1807
+ }
1808
+ }
1809
+ }
1810
+ function scoreOf(query, qMag, vector, metric) {
1811
+ const vectorMagnitude = metric === "cosine" ? magnitude(vector) : 0;
1812
+ return toScore(toDistance(query, vector, qMag, vectorMagnitude, metric), metric);
1813
+ }
1814
+ function scanWritten(hold, query, k, options, admits) {
1815
+ if (hold.written.size === 0) return [];
1816
+ const qMag = magnitude(query);
1817
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
1818
+ for (const [docId, { vector }] of hold.written) {
1819
+ if (admits !== null && !admits(docId)) continue;
1820
+ const score = scoreOf(query, qMag, vector, options.metric);
1821
+ if (score >= options.minSimilarity) heap.push({ docId, score });
1822
+ }
1823
+ return heap.toSortedArray().reverse();
1824
+ }
1825
+ function scanSubsets(hold, query, k, options, subsets) {
1826
+ const qMag = magnitude(query);
1827
+ const heap = createBoundedMaxHeap(highScoreFirst, Math.max(k, 1));
1828
+ let matched = 0;
1829
+ for (const docId of idsOfSubsets(subsets)) {
1830
+ const written = hold.written.get(docId);
1831
+ let score;
1832
+ if (written !== void 0) {
1833
+ score = scoreOf(query, qMag, written.vector, options.metric);
1834
+ } else {
1835
+ if (hold.removedIds.has(docId)) continue;
1836
+ const number = hold.field.numberOf(docId);
1837
+ if (number === void 0) continue;
1838
+ score = scoreOf(query, qMag, hold.field.reusedVectorOfNumber(number), options.metric);
1839
+ }
1840
+ if (score < options.minSimilarity) continue;
1841
+ matched += 1;
1842
+ heap.push({ docId, score });
1843
+ }
1844
+ return { results: heap.toSortedArray().reverse(), matched };
1845
+ }
1846
+ function outcomeWithoutGraphWalk(state, hold, query, k, options) {
1847
+ if (hold.live === 0) return { results: [], matched: 0, matchedExact: true };
1848
+ const subsets = options.filterSubsets;
1849
+ let admitted = hold.live;
1850
+ if (subsets !== void 0) {
1851
+ admitted = admittedBySubsets(subsets);
1852
+ if (admitted === 0) return { results: [], matched: 0, matchedExact: true };
1853
+ if (admitted / hold.live < state.filterThreshold) {
1854
+ const { results, matched } = scanSubsets(hold, query, k, options, subsets);
1855
+ return { results: results.slice(0, Math.max(k, 0)), matched, matchedExact: true };
1856
+ }
1857
+ }
1858
+ if (k <= 0) {
1859
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results: [], matched: admitted, matchedExact: true } : { results: [], matched: 0, matchedExact: false };
1860
+ }
1861
+ return { admits: admissionOf(options, hold), admitted };
1862
+ }
1863
+ function outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options) {
1864
+ const results = mergeResults(fromGraph, scanWritten(hold, query, k, options, walk.admits), k);
1865
+ return options.minSimilarity === Number.NEGATIVE_INFINITY ? { results, matched: walk.admitted, matchedExact: true } : { results, matched: results.length, matchedExact: false };
1866
+ }
1867
+ function coldSearch(state, hold, query, k, options) {
1868
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
1869
+ if ("results" in walk) {
1870
+ hold.searchAnswered();
1871
+ return walk;
1872
+ }
1873
+ const fromGraph = hold.field.search(query, k, options, walk.admits, hold.removedNumbers);
1874
+ hold.searchAnswered();
1875
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
1876
+ }
1877
+ async function coldSearchConcurrently(state, hold, query, k, options) {
1878
+ const walk = outcomeWithoutGraphWalk(state, hold, query, k, options);
1879
+ if ("results" in walk) {
1880
+ hold.searchAnswered();
1881
+ return walk;
1882
+ }
1883
+ const fromGraph = await hold.field.searchConcurrently(query, k, options, walk.admits, hold.removedNumbers);
1884
+ hold.searchAnswered();
1885
+ return outcomeOfGraphWalk(hold, walk, fromGraph, query, k, options);
1886
+ }
1887
+
1664
1888
  // src/vector/vector-index/flagged-documents.ts
1665
1889
  var FlaggedDocuments = class extends Set {
1666
1890
  constructor(store, layout) {
@@ -1812,7 +2036,7 @@ function completeGraph(state) {
1812
2036
  }
1813
2037
  function maintenanceStatus(state) {
1814
2038
  const storeSize = state.store.size;
1815
- const graphCount = state.hnsw ? 1 : 0;
2039
+ const graphCount = state.hnsw || state.cold ? 1 : 0;
1816
2040
  const graphNodes = state.hnsw === null ? storeSize : state.hnsw.size + state.hnsw.tombstoneCount;
1817
2041
  const tombstoned = state.tombstones.size + state.store.replacedVectorCount;
1818
2042
  const tombstoneRatio = graphNodes > 0 ? tombstoned / graphNodes : 0;
@@ -1975,6 +2199,91 @@ function withBufferedVectors(state, query, k, options, filter, graphResults, buf
1975
2199
  return mergeResults(graphResults, scan.results, k);
1976
2200
  }
1977
2201
 
2202
+ // src/vector/vector-index/writes.ts
2203
+ function createVectorWrites(state) {
2204
+ function validateDimension(vector) {
2205
+ if (vector.length !== state.dimension) {
2206
+ throw new NarsilError(
2207
+ ErrorCodes.VECTOR_DIMENSION_MISMATCH,
2208
+ `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
2209
+ { expected: state.dimension, received: vector.length }
2210
+ );
2211
+ }
2212
+ }
2213
+ function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
2214
+ try {
2215
+ return state.store.entryForOrdinal(ordinal)?.vector;
2216
+ } catch {
2217
+ return void 0;
2218
+ }
2219
+ }
2220
+ function holdsThisVectorAlready(docId, ordinal, vector) {
2221
+ if (state.tombstones.has(docId)) return false;
2222
+ const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
2223
+ if (held === void 0) return false;
2224
+ for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
2225
+ return true;
2226
+ }
2227
+ function applyInsert(docId, vector, partitionId) {
2228
+ handOverBuildTurns(state);
2229
+ const previous = state.store.getOrdinal(docId);
2230
+ if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
2231
+ if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
2232
+ noteWrite(state);
2233
+ state.store.setPartition(docId, partitionId);
2234
+ }
2235
+ return;
2236
+ }
2237
+ noteWrite(state);
2238
+ state.tombstones.delete(docId);
2239
+ const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
2240
+ state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
2241
+ if (previous !== void 0) {
2242
+ state.hnsw?.markTombstoneOrdinal(previous);
2243
+ state.freshGraph?.markTombstoneOrdinal(previous);
2244
+ if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
2245
+ }
2246
+ state.buffer.add(docId);
2247
+ state.pendingLocations.delete(docId);
2248
+ if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
2249
+ }
2250
+ function applyRemove(docId) {
2251
+ handOverBuildTurns(state);
2252
+ if (!state.store.has(docId)) return;
2253
+ noteWrite(state);
2254
+ state.tombstones.add(docId);
2255
+ state.buffer.delete(docId);
2256
+ state.pendingLocations.delete(docId);
2257
+ if (state.hnsw) {
2258
+ state.hnsw.markTombstone(docId);
2259
+ }
2260
+ }
2261
+ return {
2262
+ insert(docId, vector, partitionId) {
2263
+ validateDimension(vector);
2264
+ const hold = state.cold;
2265
+ if (hold === null) {
2266
+ applyInsert(docId, vector, partitionId);
2267
+ return;
2268
+ }
2269
+ noteWrite(state);
2270
+ coldInsert(hold, docId, vector, partitionId);
2271
+ const held = vector.slice();
2272
+ hold.writes.push(() => applyInsert(docId, held, partitionId));
2273
+ },
2274
+ remove(docId) {
2275
+ const hold = state.cold;
2276
+ if (hold === null) {
2277
+ applyRemove(docId);
2278
+ return;
2279
+ }
2280
+ if (!coldRemove(hold, docId)) return;
2281
+ noteWrite(state);
2282
+ hold.writes.push(() => applyRemove(docId));
2283
+ }
2284
+ };
2285
+ }
2286
+
1978
2287
  // src/vector/vector-index/index.ts
1979
2288
  function defaultQuantizationFor(dimension) {
1980
2289
  if (dimension >= OSQ4_MIN_DIMENSION) return "osq4";
@@ -2045,76 +2354,26 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2045
2354
  sharedHandles: /* @__PURE__ */ new Map(),
2046
2355
  sharedLayoutRevision: 0,
2047
2356
  sharing: Promise.resolve(),
2048
- releaseToFilesInFlight: Promise.resolve()
2357
+ releaseToFilesInFlight: Promise.resolve(),
2358
+ cold: null
2049
2359
  };
2050
- function validateDimension(vector) {
2051
- if (vector.length !== state.dimension) {
2052
- throw new NarsilError(
2053
- ErrorCodes.VECTOR_DIMENSION_MISMATCH,
2054
- `Vector dimension mismatch: expected ${state.dimension}, got ${vector.length}`,
2055
- { expected: state.dimension, received: vector.length }
2056
- );
2057
- }
2058
- }
2059
- function heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal) {
2060
- try {
2061
- return state.store.entryForOrdinal(ordinal)?.vector;
2062
- } catch {
2063
- return void 0;
2064
- }
2065
- }
2066
- function holdsThisVectorAlready(docId, ordinal, vector) {
2067
- if (state.tombstones.has(docId)) return false;
2068
- const held = heldVectorOrUndefinedWhereItsFileFailsToRead(ordinal);
2069
- if (held === void 0) return false;
2070
- for (let i = 0; i < vector.length; i++) if (held[i] !== vector[i]) return false;
2071
- return true;
2072
- }
2073
- function insert(docId, vector, partitionId) {
2074
- validateDimension(vector);
2075
- handOverBuildTurns(state);
2076
- const previous = state.store.getOrdinal(docId);
2077
- if (previous !== void 0 && holdsThisVectorAlready(docId, previous, vector)) {
2078
- if (partitionId !== void 0 && state.store.partitionOfOrdinal(previous) !== partitionId) {
2079
- noteWrite(state);
2080
- state.store.setPartition(docId, partitionId);
2081
- }
2082
- return;
2083
- }
2084
- noteWrite(state);
2085
- state.tombstones.delete(docId);
2086
- const graphHoldsPrevious = previous !== void 0 && ((state.hnsw?.holdsNode(previous) ?? false) || (state.freshGraph?.holdsNode(previous) ?? false));
2087
- state.store.insert(docId, vector, partitionId, graphHoldsPrevious);
2088
- if (previous !== void 0) {
2089
- state.hnsw?.markTombstoneOrdinal(previous);
2090
- state.freshGraph?.markTombstoneOrdinal(previous);
2091
- if (!graphHoldsPrevious) state.osq?.removeOrdinal(previous);
2092
- }
2093
- state.buffer.add(docId);
2094
- state.pendingLocations.delete(docId);
2095
- if (!threadsHoldCurrentLayout(state)) scheduleWorkerCopyLoad(state);
2096
- }
2097
- function remove(docId) {
2098
- handOverBuildTurns(state);
2099
- if (!state.store.has(docId)) return;
2100
- noteWrite(state);
2101
- state.tombstones.add(docId);
2102
- state.buffer.delete(docId);
2103
- state.pendingLocations.delete(docId);
2104
- if (state.hnsw) {
2105
- state.hnsw.markTombstone(docId);
2106
- }
2107
- }
2360
+ const { insert, remove } = createVectorWrites(state);
2108
2361
  function getVector(docId) {
2362
+ if (state.cold !== null) return coldVectorOf(state.cold, docId);
2109
2363
  if (state.tombstones.has(docId)) return null;
2110
2364
  const entry = state.store.get(docId);
2111
2365
  if (!entry) return null;
2112
2366
  return new Float32Array(entry.vector);
2113
2367
  }
2114
2368
  function has(docId) {
2369
+ if (state.cold !== null) return coldHas(state.cold, docId);
2115
2370
  return state.store.has(docId) && !state.tombstones.has(docId);
2116
2371
  }
2372
+ async function whenLoaded() {
2373
+ while (state.cold !== null) await state.cold.loaded;
2374
+ }
2117
2375
  async function searchParallel(query, k, options) {
2376
+ if (state.cold !== null) return coldSearchConcurrently(state, state.cold, query, k, options);
2118
2377
  handOverBuildTurns(state);
2119
2378
  const confined = options.filterSubsets !== void 0 || options.filterPartitions !== void 0;
2120
2379
  let filter = filterForOptions(state, options);
@@ -2161,44 +2420,73 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2161
2420
  get storage() {
2162
2421
  return state.storage;
2163
2422
  },
2423
+ get metric() {
2424
+ return state.metric;
2425
+ },
2164
2426
  insert,
2165
2427
  remove,
2166
- partitionsKnown: () => state.store.partitionsKnown,
2167
- assignPartitions: (resolve) => assignStorePartitions(state, resolve),
2168
- scheduleBuild: () => scheduleBuild(state),
2169
- awaitPendingBuild: () => awaitBuilds(state),
2170
- completeGraph: () => completeGraph(state),
2171
- dispose: () => disposeIndex(state),
2172
- search: (query, k, options) => search(state, query, k, options),
2428
+ partitionsKnown: () => state.cold !== null || state.store.partitionsKnown,
2429
+ assignPartitions: (resolve) => {
2430
+ if (state.cold === null) assignStorePartitions(state, resolve);
2431
+ },
2432
+ scheduleBuild: () => {
2433
+ if (state.cold === null) scheduleBuild(state);
2434
+ },
2435
+ awaitPendingBuild: async () => {
2436
+ await whenLoaded();
2437
+ await awaitBuilds(state);
2438
+ },
2439
+ completeGraph: async () => {
2440
+ await whenLoaded();
2441
+ await completeGraph(state);
2442
+ },
2443
+ dispose: () => {
2444
+ cancelColdHold(state);
2445
+ disposeIndex(state);
2446
+ },
2447
+ search: (query, k, options) => state.cold === null ? search(state, query, k, options) : coldSearch(state, state.cold, query, k, options),
2173
2448
  searchParallel,
2174
- refreshWorkerCopies: () => refreshWorkerCopies(state),
2449
+ refreshWorkerCopies: () => {
2450
+ if (state.cold === null) refreshWorkerCopies(state);
2451
+ },
2175
2452
  getVector,
2176
2453
  has,
2177
2454
  compact: () => {
2455
+ if (state.cold !== null) return;
2178
2456
  noteWrite(state);
2179
2457
  compact(state);
2180
2458
  },
2181
2459
  optimize: async () => {
2460
+ await whenLoaded();
2182
2461
  noteWrite(state);
2183
2462
  await optimize(state);
2184
2463
  },
2185
- maintenanceStatus: () => maintenanceStatus(state),
2186
- estimateMemoryBytes: () => estimateMemoryBytes(state),
2464
+ maintenanceStatus: () => ({ ...maintenanceStatus(state), loadingFromFiles: state.cold !== null }),
2465
+ estimateMemoryBytes: () => estimateMemoryBytes(state) + (state.cold?.field.residentBytes ?? 0),
2187
2466
  growthBytesFor: (vectors) => state.store.growthBytesFor(vectors),
2188
2467
  deserialize: (parts) => {
2468
+ cancelColdHold(state);
2189
2469
  invalidateWorkerCopies(state);
2190
2470
  deserialize(state, parts);
2191
2471
  },
2192
2472
  planCheckpoint: (listedKeys, rewritesGraph) => planCheckpoint(state, listedKeys, rewritesGraph),
2193
2473
  recordCheckpoint: (plan, written) => recordCheckpoint(state, plan, written),
2194
2474
  restoreCheckpoint: (shape) => {
2475
+ cancelColdHold(state);
2195
2476
  invalidateWorkerCopies(state);
2196
2477
  return beginCheckpointRestore(state, shape);
2197
2478
  },
2198
- startShuffledRebuildWhenDue: () => startShuffledRebuild(state),
2199
- vectorFilesInUse: () => state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== ""),
2479
+ openColdField: (field, load) => openColdField(state, field, load),
2480
+ startShuffledRebuildWhenDue: () => {
2481
+ if (state.cold === null) startShuffledRebuild(state);
2482
+ },
2483
+ vectorFilesInUse: () => {
2484
+ const stored = state.store.dimension === 0 ? [] : state.store.handles.vectorFiles.filter((path) => path !== "");
2485
+ return state.cold === null ? stored : [...stored, ...state.cold.field.filePaths()];
2486
+ },
2200
2487
  adoptDiskLayout: (layout) => adoptDiskLayout(state, layout),
2201
2488
  releaseVectorFiles: async () => {
2489
+ await whenLoaded();
2202
2490
  await withdrawWorkerCopies(state);
2203
2491
  state.store.closeFiles();
2204
2492
  },
@@ -2208,7 +2496,10 @@ function createVectorIndex(fieldName, dimension, config, workerCopies = VECTOR_W
2208
2496
  state.pendingLocations.set(docId, { ...location, path: rewrite(location.path) });
2209
2497
  }
2210
2498
  },
2211
- holdWorkerCopies: () => holdWorkerCopies(state),
2499
+ holdWorkerCopies: async () => {
2500
+ await whenLoaded();
2501
+ await holdWorkerCopies(state);
2502
+ },
2212
2503
  resumeWorkerCopies: () => {
2213
2504
  state.workerCopiesHeld = false;
2214
2505
  }
@@ -0,0 +1,48 @@
1
+ export { actionPartitionsOf, adoptSegmentsOnWorker, bytesThatSegmentsHoldInMemory, checkPagedSegmentFully, closeUnusedSegments, countsAsLatestHolder, createPagedFrozenSegment, isPagedSegmentFile, keysThatSegmentsStillRead, mergePagedWindow, openFileVectorReads, openPagedSegmentScan, openPagedSnapshot, openRewriteSpill, pagedListingOf, pagedPartitionLines, pagedTombstonesFor, prepareDiskAdoption, startReadingOpeningSectionsAhead, swapPreparedPartitions, writePagedSegment } from './chunk-GWOS5DYC.mjs';
2
+ import './chunk-NN33CZRE.mjs';
3
+ import './chunk-BOJZGUS6.mjs';
4
+ import './chunk-W2COJ5AH.mjs';
5
+ import './chunk-FOCV4CBZ.mjs';
6
+ import './chunk-7AUASRE7.mjs';
7
+ import './chunk-UUZCQQG2.mjs';
8
+ import './chunk-KEIYVZUX.mjs';
9
+ import './chunk-IO5F7ZBE.mjs';
10
+ import './chunk-VEFN6X6Q.mjs';
11
+ import './chunk-5WYVUPGD.mjs';
12
+ import './chunk-HJSN7UYG.mjs';
13
+ import './chunk-UIKN2OTQ.mjs';
14
+ import './chunk-AV4IJKD2.mjs';
15
+ import './chunk-EEBCDP36.mjs';
16
+ import './chunk-WJGO5PGV.mjs';
17
+ import './chunk-ZPWS2SKJ.mjs';
18
+ import './chunk-N37VMC62.mjs';
19
+ import './chunk-ZATRDAYU.mjs';
20
+ import './chunk-EBIHRL7B.mjs';
21
+ import './chunk-KOKM4PQN.mjs';
22
+ import './chunk-QSK35UC2.mjs';
23
+ import './chunk-74XB2SXI.mjs';
24
+ import './chunk-RKAVHNRW.mjs';
25
+ import './chunk-WQBCBKY3.mjs';
26
+ import './chunk-JUKYYXSP.mjs';
27
+ import './chunk-DQNKVOJ2.mjs';
28
+ import './chunk-KGOXHLLZ.mjs';
29
+ import './chunk-U476IXKQ.mjs';
30
+ import './chunk-FQBFZAWG.mjs';
31
+ import './chunk-VJS4JFBQ.mjs';
32
+ import './chunk-GES7U2PM.mjs';
33
+ import './chunk-ZSWHPKGM.mjs';
34
+ import './chunk-O4LJQI4G.mjs';
35
+ import './chunk-X5CV2E2B.mjs';
36
+ import './chunk-WNBRW4Q2.mjs';
37
+ import './chunk-TGP5M5R7.mjs';
38
+ import './chunk-U2BKYEC7.mjs';
39
+ import './chunk-NPNK57CA.mjs';
40
+ import './chunk-4UMZ7KQL.mjs';
41
+ import './chunk-R6A773WK.mjs';
42
+ import './chunk-YTJCGKM4.mjs';
43
+ import './chunk-QHPYS427.mjs';
44
+ import './chunk-GLZHNSGB.mjs';
45
+ import './chunk-22YPZVMG.mjs';
46
+ import './chunk-RFBLQLLZ.mjs';
47
+ import './chunk-L5ORED7V.mjs';
48
+ import './chunk-HNHCNTVG.mjs';