@delali/narsil 0.3.1-next.140 → 0.3.1-next.143

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -1,8 +1,8 @@
1
- import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-B0H-vp4U.js';
2
- import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-BTXbHCqA.js';
1
+ import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-BhZxIeb1.js';
2
+ import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-B-Runv0n.js';
3
3
  import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
4
- import { N as NodeRole, c as NodeMemoryReport } from './index-HYnqS1SE.js';
5
- import { a as ClusterNodeInfo, N as NodeReadiness } from './config-Brysl9NO.js';
4
+ import { N as NodeRole, c as NodeMemoryReport } from './index-DsEurhSl.js';
5
+ import { a as ClusterNodeInfo, N as NodeReadiness } from './config-CIG5yDyg.js';
6
6
 
7
7
  /**
8
8
  * The body of `POST /indexes`, which the server sends once the index exists.
@@ -479,6 +479,8 @@ interface HttpIndexConfig {
479
479
  stopWords?: string | string[];
480
480
  /** Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it. */
481
481
  trackPositions?: boolean;
482
+ /** Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`. */
483
+ textStorage?: 'memory' | 'disk';
482
484
  /** Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed. */
483
485
  surfaceForms?: boolean;
484
486
  /** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
@@ -1,22 +1,22 @@
1
- export { closeRequestThread, searchHandler, serveRequests } from './chunk-B5I3IL4H.mjs';
2
- import './chunk-SBMAWTRG.mjs';
1
+ export { closeRequestThread, searchHandler, serveRequests } from './chunk-YLNSJQLX.mjs';
3
2
  import './chunk-KZ3NX2P7.mjs';
3
+ import './chunk-SBMAWTRG.mjs';
4
4
  import './chunk-DD2673TX.mjs';
5
5
  import './chunk-IWVEG5CX.mjs';
6
6
  import './chunk-DJHHF2L5.mjs';
7
- import './chunk-XMQNOD5V.mjs';
8
- import './chunk-U4X4AWKL.mjs';
9
- import './chunk-ZATRDAYU.mjs';
10
- import './chunk-EBIHRL7B.mjs';
7
+ import './chunk-VEFN6X6Q.mjs';
11
8
  import './chunk-HJSN7UYG.mjs';
12
9
  import './chunk-EEBCDP36.mjs';
10
+ import './chunk-ZPWS2SKJ.mjs';
11
+ import './chunk-ZATRDAYU.mjs';
12
+ import './chunk-EBIHRL7B.mjs';
13
13
  import './chunk-QSK35UC2.mjs';
14
14
  import './chunk-74XB2SXI.mjs';
15
15
  import './chunk-RKAVHNRW.mjs';
16
16
  import './chunk-WQBCBKY3.mjs';
17
17
  import './chunk-JUKYYXSP.mjs';
18
- import './chunk-J3EHATEZ.mjs';
19
- import './chunk-UBXJQJII.mjs';
18
+ import './chunk-DQNKVOJ2.mjs';
19
+ import './chunk-KGOXHLLZ.mjs';
20
20
  import './chunk-U476IXKQ.mjs';
21
21
  import './chunk-FQBFZAWG.mjs';
22
22
  import './chunk-VJS4JFBQ.mjs';
@@ -1,4 +1,4 @@
1
- import { f as VectorQuantizationMode } from '../schema-BTXbHCqA.js';
1
+ import { f as VectorQuantizationMode } from '../schema-B-Runv0n.js';
2
2
  import '../adapters-BQFQ4PRz.js';
3
3
  import '../synonyms-CpOAHPa-.js';
4
4
 
@@ -1,5 +1,6 @@
1
- import { openSharedVectorField, createVectorStore } from '../chunk-CCCGEPJM.mjs';
2
- import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-K3AQOZVM.mjs';
1
+ import { openSharedVectorField, createVectorStore } from '../chunk-XZ3SRNDJ.mjs';
2
+ import '../chunk-FOCV4CBZ.mjs';
3
+ import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-TNBIK2LQ.mjs';
3
4
  import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
4
5
  import '../chunk-DJHHF2L5.mjs';
5
6
  import '../chunk-KOKM4PQN.mjs';
@@ -1,6 +1,6 @@
1
1
  import { startIdleHeapCleanup } from '../chunk-RIIUZJQ3.mjs';
2
- import { isValidWorkerAction } from '../chunk-UJWGQIXU.mjs';
3
- import { loadVectorCode } from '../chunk-FFVLPCLH.mjs';
2
+ import { isValidWorkerAction } from '../chunk-QHPYS427.mjs';
3
+ import { loadVectorCode } from '../chunk-GLZHNSGB.mjs';
4
4
  import '../chunk-22YPZVMG.mjs';
5
5
  import { threadSlotOfWorker } from '../chunk-RFBLQLLZ.mjs';
6
6
  import '../chunk-L5ORED7V.mjs';
@@ -22,9 +22,9 @@ async function importBootstrapModule(moduleUrl) {
22
22
  await import(moduleUrl);
23
23
  }
24
24
  async function stopServingRequests() {
25
- const { closeRequestThread } = await import('../thread-USB3GOPY.mjs');
25
+ const { closeRequestThread } = await import('../thread-KGNAIKPD.mjs');
26
26
  closeRequestThread();
27
- const { closeLongQueryThread } = await import('../long-query-thread-U7K5A72G.mjs');
27
+ const { closeLongQueryThread } = await import('../long-query-thread-4SW7RWDE.mjs');
28
28
  closeLongQueryThread();
29
29
  }
30
30
  function holdsCopies(executor) {
@@ -55,7 +55,7 @@ function createActionHandler(executor) {
55
55
  if (!holdsCopies(executor)) {
56
56
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
57
57
  }
58
- const starting = import('../thread-USB3GOPY.mjs').then(
58
+ const starting = import('../thread-KGNAIKPD.mjs').then(
59
59
  ({ serveRequests }) => serveRequests(executor, action.settings)
60
60
  );
61
61
  serving.start = starting.then(
@@ -72,7 +72,7 @@ function createActionHandler(executor) {
72
72
  if (!holdsCopies(executor)) {
73
73
  throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer long queries from");
74
74
  }
75
- const { serveLongQueries } = await import('../long-query-thread-U7K5A72G.mjs');
75
+ const { serveLongQueries } = await import('../long-query-thread-4SW7RWDE.mjs');
76
76
  serveLongQueries(executor, action.settings);
77
77
  servingRequests = true;
78
78
  post(buildSuccessResponse(action.requestId, void 0));
@@ -118,7 +118,7 @@ async function setup() {
118
118
  } catch {
119
119
  parentPort = null;
120
120
  }
121
- const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-J5UDLPL3.mjs'), loadVectorCode()]);
121
+ const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-7XOQ7VQP.mjs'), loadVectorCode()]);
122
122
  const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
123
123
  if (parentPort) {
124
124
  const port = parentPort;
@@ -1,29 +1,31 @@
1
- import { NUMBER_TAG, DATE_TAG, BYTES_TAG, encodeBase64, startsWithTagPrefix, OBJECT_TAG, MILLISECONDS_PER_SECOND, NANOSECONDS_PER_MILLISECOND, DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-BDIDUW2B.mjs';
1
+ import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-6SNBQESJ.mjs';
2
+ import { encodeExactJson, encodeDocumentJson } from './chunk-NN33CZRE.mjs';
3
+ import './chunk-BOJZGUS6.mjs';
2
4
  import './chunk-CZQ33DLW.mjs';
3
- import './chunk-TCWQJL7O.mjs';
5
+ import './chunk-L76W3IZJ.mjs';
4
6
  import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-ICW45TTU.mjs';
5
7
  import './chunk-37CD3OKS.mjs';
6
- import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-LLDFIKGP.mjs';
7
- import { openChecksummedEnvelope } from './chunk-W2XDK66Z.mjs';
8
- import './chunk-XMQNOD5V.mjs';
9
- import './chunk-XXKWEG53.mjs';
8
+ import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-KEIYVZUX.mjs';
9
+ import { openChecksummedEnvelope } from './chunk-IO5F7ZBE.mjs';
10
+ import './chunk-VEFN6X6Q.mjs';
11
+ import './chunk-5WYVUPGD.mjs';
12
+ import './chunk-HJSN7UYG.mjs';
10
13
  import './chunk-UIKN2OTQ.mjs';
11
14
  import './chunk-AV4IJKD2.mjs';
12
- import { decodeStoredDocument } from './chunk-IBATGS3J.mjs';
13
- import './chunk-U4X4AWKL.mjs';
15
+ import { EXPORT_CHUNK_BYTES } from './chunk-EEBCDP36.mjs';
16
+ import { decodeStoredDocument } from './chunk-WJGO5PGV.mjs';
17
+ import './chunk-ZPWS2SKJ.mjs';
14
18
  import './chunk-N37VMC62.mjs';
15
19
  import './chunk-ZATRDAYU.mjs';
16
20
  import './chunk-EBIHRL7B.mjs';
17
- import './chunk-HJSN7UYG.mjs';
18
- import { EXPORT_CHUNK_BYTES } from './chunk-EEBCDP36.mjs';
19
21
  import './chunk-KOKM4PQN.mjs';
20
22
  import './chunk-QSK35UC2.mjs';
21
23
  import './chunk-74XB2SXI.mjs';
22
24
  import './chunk-RKAVHNRW.mjs';
23
25
  import './chunk-WQBCBKY3.mjs';
24
26
  import './chunk-JUKYYXSP.mjs';
25
- import './chunk-J3EHATEZ.mjs';
26
- import './chunk-UBXJQJII.mjs';
27
+ import './chunk-DQNKVOJ2.mjs';
28
+ import './chunk-KGOXHLLZ.mjs';
27
29
  import './chunk-U476IXKQ.mjs';
28
30
  import './chunk-FQBFZAWG.mjs';
29
31
  import './chunk-VJS4JFBQ.mjs';
@@ -38,99 +40,17 @@ import './chunk-NPNK57CA.mjs';
38
40
  import './chunk-4UMZ7KQL.mjs';
39
41
  import './chunk-R6A773WK.mjs';
40
42
  import './chunk-YTJCGKM4.mjs';
41
- import './chunk-UJWGQIXU.mjs';
42
- import './chunk-FFVLPCLH.mjs';
43
+ import './chunk-QHPYS427.mjs';
44
+ import './chunk-GLZHNSGB.mjs';
43
45
  import './chunk-22YPZVMG.mjs';
44
46
  import './chunk-RFBLQLLZ.mjs';
45
47
  import './chunk-L5ORED7V.mjs';
46
- import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
48
+ import './chunk-HNHCNTVG.mjs';
47
49
  import { decode } from '@msgpack/msgpack';
48
50
 
49
- // src/engine/export/json-line.ts
50
- function refuse(path, kind, docId) {
51
- throw new NarsilError(
52
- ErrorCodes.DOC_VALIDATION_FAILED,
53
- `Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
54
- { docId, field: path }
55
- );
56
- }
57
- function encodeNumber(value, path, target) {
58
- if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
59
- if (!target.tagged) refuse(path, String(value), target.docId);
60
- return `{"${NUMBER_TAG}":"${String(value)}"}`;
61
- }
62
- function encodeDate(value, path, target) {
63
- const milliseconds = value.getTime();
64
- if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
65
- const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
66
- const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
67
- return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
68
- }
69
- function encodeBytes(value, path, target) {
70
- if (!target.tagged) refuse(path, "bytes", target.docId);
71
- return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
72
- }
73
- function encodeList(value, path, target) {
74
- target.parts.push("[");
75
- for (let at = 0; at < value.length; at++) {
76
- if (at > 0) target.parts.push(",");
77
- encodeValue(value[at], `${path}[${at}]`, target);
78
- }
79
- target.parts.push("]");
80
- }
81
- function encodeMap(value, path, target) {
82
- const proto = Object.getPrototypeOf(value);
83
- if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
84
- const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
85
- const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
86
- target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
87
- for (let at = 0; at < entries.length; at++) {
88
- const [key, entry] = entries[at];
89
- if (at > 0) target.parts.push(",");
90
- target.parts.push(JSON.stringify(key), ":");
91
- encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
92
- }
93
- target.parts.push(wrapped ? "}}" : "}");
94
- }
95
- function encodeValue(value, path, target) {
96
- if (value === null) {
97
- target.parts.push("null");
98
- return;
99
- }
100
- switch (typeof value) {
101
- case "string":
102
- target.parts.push(JSON.stringify(value));
103
- return;
104
- case "boolean":
105
- target.parts.push(value ? "true" : "false");
106
- return;
107
- case "number":
108
- target.parts.push(encodeNumber(value, path, target));
109
- return;
110
- case "object":
111
- break;
112
- default:
113
- refuse(path, `a ${typeof value}`, target.docId);
114
- }
115
- if (Array.isArray(value)) encodeList(value, path, target);
116
- else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
117
- else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
118
- else encodeMap(value, path, target);
119
- }
120
- function encodeExactJson(value, docId) {
121
- const target = { docId, tagged: false, parts: [] };
122
- encodeValue(value, "", target);
123
- return target.parts.join("");
124
- }
125
- function encodeDocumentJson(document, docId) {
126
- const target = { docId, tagged: true, parts: [] };
127
- encodeValue(document, "", target);
128
- return target.parts.join("");
129
- }
130
-
131
- // src/engine/export/write.ts
132
51
  var VECTOR_FILE_ENVELOPE_VERSION = 2;
133
52
  var FLOAT32_BYTES = 4;
53
+ var ENVELOPE_VERSION_OFFSET = 4;
134
54
  async function locateVectors(checkpoint) {
135
55
  const located = /* @__PURE__ */ new Map();
136
56
  for (const field of checkpoint.manifest.vectors) {
@@ -178,7 +98,59 @@ async function winnersOf(checkpoint, partition) {
178
98
  }
179
99
  return winners;
180
100
  }
101
+ async function wholeSegmentScan(checkpoint, ref) {
102
+ const contents = await readSegment(checkpoint, ref);
103
+ const table = contents.arrays.documentTable;
104
+ const ids = /* @__PURE__ */ new Set();
105
+ for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
106
+ return {
107
+ count: contents.arrays.documentCount,
108
+ schema: contents.schema,
109
+ tombstones: new Set(contents.tombstones),
110
+ idAt: (ordinal) => idAt(contents, ordinal),
111
+ holds: (docId) => ids.has(docId),
112
+ storedDocumentAt: (ordinal) => decodeStoredDocument(
113
+ table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
114
+ table.discardedFields
115
+ ),
116
+ close: () => void 0
117
+ };
118
+ }
119
+ async function wholeFileVectorReads(checkpoint) {
120
+ const located = await locateVectors(checkpoint);
121
+ return {
122
+ async vectorOf(fieldPath, docId) {
123
+ const location = located.get(fieldPath)?.get(docId);
124
+ return location === void 0 ? null : readVector(checkpoint, location);
125
+ },
126
+ close: () => located.clear()
127
+ };
128
+ }
129
+ async function* pagedDocumentLines(checkpoint) {
130
+ const { openFileVectorReads, pagedPartitionLines } = await import('./module-IYPLWHGC.mjs');
131
+ const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
132
+ try {
133
+ for (const partition of checkpoint.manifest.partitions) {
134
+ yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
135
+ }
136
+ } finally {
137
+ vectors.close();
138
+ }
139
+ }
140
+ async function anyPartitionIsPaged(checkpoint) {
141
+ for (const partition of checkpoint.manifest.partitions) {
142
+ for (const ref of partition.segments) {
143
+ const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
144
+ if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
145
+ }
146
+ }
147
+ return false;
148
+ }
181
149
  async function* documentLines(checkpoint) {
150
+ if (await anyPartitionIsPaged(checkpoint)) {
151
+ yield* pagedDocumentLines(checkpoint);
152
+ return;
153
+ }
182
154
  const vectors = await locateVectors(checkpoint);
183
155
  for (const partition of checkpoint.manifest.partitions) {
184
156
  const winners = await winnersOf(checkpoint, partition);
package/openapi.json CHANGED
@@ -3228,6 +3228,14 @@
3228
3228
  "type": "boolean",
3229
3229
  "description": "Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it."
3230
3230
  },
3231
+ "textStorage": {
3232
+ "type": "string",
3233
+ "enum": [
3234
+ "memory",
3235
+ "disk"
3236
+ ],
3237
+ "description": "Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`."
3238
+ },
3231
3239
  "surfaceForms": {
3232
3240
  "type": "boolean",
3233
3241
  "description": "Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed."
@@ -5142,6 +5150,10 @@
5142
5150
  "type": "boolean",
5143
5151
  "description": "This turns true while a rebuild that keeps recall up waits for the node to have room for a second graph beside the one it serves, and false once it starts."
5144
5152
  },
5153
+ "loadingFromFiles": {
5154
+ "type": "boolean",
5155
+ "description": "This stays true after a restart while a field kept on disk loads its codes and graph in the background, during which a search reads them from the field's files."
5156
+ },
5145
5157
  "estimatedCompactMs": {
5146
5158
  "type": "number",
5147
5159
  "description": "Narsil.compactVectors would take about this many milliseconds."
@@ -5158,6 +5170,7 @@
5158
5170
  "bufferSize",
5159
5171
  "building",
5160
5172
  "waitingForMemory",
5173
+ "loadingFromFiles",
5161
5174
  "estimatedCompactMs",
5162
5175
  "estimatedOptimizeMs"
5163
5176
  ],
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@delali/narsil",
3
- "version": "0.3.1-next.140",
3
+ "version": "0.3.1-next.143",
4
4
  "type": "module",
5
5
  "description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
6
6
  "author": "Delali (https://sondelali.com)",
@@ -51,6 +51,10 @@
51
51
  "browser": "./src/vector/vector-store/vector-file.browser.ts",
52
52
  "default": "./src/vector/vector-store/vector-file.ts"
53
53
  },
54
+ "#platform/raw-deflate": {
55
+ "browser": "./src/persistence/paged/raw-deflate.browser.ts",
56
+ "default": "./src/persistence/paged/raw-deflate.ts"
57
+ },
54
58
  "#platform/native-core": {
55
59
  "browser": "./src/vector/native/loader.browser.ts",
56
60
  "default": "./src/vector/native/loader.ts"
@@ -90,7 +94,7 @@
90
94
  "node": ">=22"
91
95
  },
92
96
  "scripts": {
93
- "build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
97
+ "build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && node scripts/check-vector-chunk.mjs && tsx scripts/api-surface.ts",
94
98
  "build:code": "rm -rf dist && NARSIL_BUILD_CODE_ONLY=1 tsup",
95
99
  "test": "vitest run --exclude='**/vector/recall.test.ts' --exclude='**/e2e/cluster-memory/**' --exclude='**/e2e/cluster-faults/**' --exclude='**/frozen-removal-memory/container-removes.test.ts'",
96
100
  "test:native": "vitest run src/__tests__/vector/native src/__tests__/integration/filtered-vector-recall/native src/__tests__/integration/filtered-vectors/mutations-native src/__tests__/integration/removed-vectors/main-thread",
@@ -279,11 +283,11 @@
279
283
  }
280
284
  },
281
285
  "optionalDependencies": {
282
- "@delali/narsil-native-darwin-arm64": "0.3.1-next.140",
283
- "@delali/narsil-native-darwin-x64": "0.3.1-next.140",
284
- "@delali/narsil-native-linux-arm64": "0.3.1-next.140",
285
- "@delali/narsil-native-linux-x64": "0.3.1-next.140",
286
- "@delali/narsil-native-win32-arm64": "0.3.1-next.140",
287
- "@delali/narsil-native-win32-x64": "0.3.1-next.140"
286
+ "@delali/narsil-native-darwin-arm64": "0.3.1-next.143",
287
+ "@delali/narsil-native-darwin-x64": "0.3.1-next.143",
288
+ "@delali/narsil-native-linux-arm64": "0.3.1-next.143",
289
+ "@delali/narsil-native-linux-x64": "0.3.1-next.143",
290
+ "@delali/narsil-native-win32-arm64": "0.3.1-next.143",
291
+ "@delali/narsil-native-win32-x64": "0.3.1-next.143"
288
292
  }
289
293
  }