@delali/narsil 0.3.1-next.140 → 0.3.1-next.141

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
  2. package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
  3. package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
  4. package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
  5. package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
  6. package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
  7. package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
  8. package/dist/browser/chunk-CCOAOO73.mjs +86 -0
  9. package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
  10. package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
  11. package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
  12. package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
  13. package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
  14. package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
  15. package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
  16. package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
  17. package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
  18. package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
  19. package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
  20. package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
  21. package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
  22. package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
  23. package/dist/browser/module-BUFNZMEU.mjs +12 -0
  24. package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
  25. package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
  26. package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
  27. package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
  28. package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
  29. package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
  30. package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
  31. package/dist/chunk-6SNBQESJ.mjs +214 -0
  32. package/dist/chunk-7AUASRE7.mjs +653 -0
  33. package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
  34. package/dist/chunk-BOJZGUS6.mjs +177 -0
  35. package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
  36. package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
  37. package/dist/chunk-FOCV4CBZ.mjs +87 -0
  38. package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
  39. package/dist/chunk-GWOS5DYC.mjs +2637 -0
  40. package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
  41. package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
  42. package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
  43. package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
  44. package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
  45. package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
  46. package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
  47. package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
  48. package/dist/chunk-NN33CZRE.mjs +86 -0
  49. package/dist/chunk-NRUXBYPH.mjs +121 -0
  50. package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
  51. package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
  52. package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
  53. package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
  54. package/dist/chunk-UUZCQQG2.mjs +209 -0
  55. package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
  56. package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
  57. package/dist/chunk-W2COJ5AH.mjs +148 -0
  58. package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
  59. package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
  60. package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
  61. package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
  62. package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
  63. package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
  64. package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
  65. package/dist/client.d.ts +7 -7
  66. package/dist/cold-module-4X7WJCFM.mjs +783 -0
  67. package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
  68. package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
  69. package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
  70. package/dist/distribution/coordinator/etcd.d.ts +2 -2
  71. package/dist/distribution/coordinator/in-memory.d.ts +2 -2
  72. package/dist/distribution.d.ts +7 -7
  73. package/dist/distribution.mjs +123 -42
  74. package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
  75. package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
  76. package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
  77. package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
  78. package/dist/index.browser.mjs +3 -3
  79. package/dist/index.d.ts +6 -6
  80. package/dist/index.mjs +11 -11
  81. package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
  82. package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
  83. package/dist/module-IYPLWHGC.mjs +48 -0
  84. package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
  85. package/dist/partial-words.mjs +2 -2
  86. package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
  87. package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
  88. package/dist/react.d.ts +9 -9
  89. package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
  90. package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
  91. package/dist/server.d.ts +10 -10
  92. package/dist/server.mjs +8 -7
  93. package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
  94. package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
  95. package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
  96. package/dist/vector/search-worker.d.ts +1 -1
  97. package/dist/vector/search-worker.mjs +3 -2
  98. package/dist/workers/entry.mjs +7 -7
  99. package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
  100. package/openapi.json +13 -0
  101. package/package.json +12 -8
  102. package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
  103. package/dist/chunk-BDIDUW2B.mjs +0 -387
  104. package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
@@ -0,0 +1,209 @@
1
+ import { packChecksummedEnvelope } from './chunk-IO5F7ZBE.mjs';
2
+ import { fnv1a } from './chunk-ZPWS2SKJ.mjs';
3
+ import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
4
+ import { encode } from '@msgpack/msgpack';
5
+
6
+ // src/persistence/durability/segment/store.ts
7
+ function directoryCheckpointStore(directory) {
8
+ const store = {
9
+ read: (key) => directory.read(key),
10
+ write: (key, chunks) => directory.atomicWrite(key, chunks),
11
+ remove: (key) => directory.remove(key),
12
+ list: (prefix) => directory.list(prefix),
13
+ pathOf: (key) => directory.pathOf(key),
14
+ directory
15
+ };
16
+ const byteLength = directory.byteLength;
17
+ if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
18
+ return store;
19
+ }
20
+ function joined(chunks) {
21
+ if (chunks.length === 1) return chunks[0];
22
+ let total = 0;
23
+ for (const chunk of chunks) total += chunk.byteLength;
24
+ const bytes = new Uint8Array(total);
25
+ let offset = 0;
26
+ for (const chunk of chunks) {
27
+ bytes.set(chunk, offset);
28
+ offset += chunk.byteLength;
29
+ }
30
+ return bytes;
31
+ }
32
+ function adapterFailure(code, verb, key, error) {
33
+ if (error instanceof NarsilError) return error;
34
+ const cause = error instanceof Error ? error.message : String(error);
35
+ return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
36
+ }
37
+ async function guarded(run, code, verb, key) {
38
+ try {
39
+ return await run();
40
+ } catch (error) {
41
+ throw adapterFailure(code, verb, key, error);
42
+ }
43
+ }
44
+ function adapterCheckpointStore(adapter) {
45
+ return {
46
+ read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
47
+ write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
48
+ remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
49
+ list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
50
+ };
51
+ }
52
+ async function requireListedSize(store, key, expected) {
53
+ const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
54
+ if (length === null) missingFile(expected.what, key);
55
+ if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
56
+ }
57
+ async function readListedFile(store, key, expected) {
58
+ await requireListedSize(store, key, expected);
59
+ const bytes = await store.read(key);
60
+ if (bytes === null) missingFile(expected.what, key);
61
+ requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
62
+ return bytes;
63
+ }
64
+ function missingFile(what, key) {
65
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
66
+ key
67
+ });
68
+ }
69
+ function requireListedLength(what, key, found, listed) {
70
+ if (found === listed) return;
71
+ throw new NarsilError(
72
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
73
+ `The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
74
+ { key, bytes: found, listed }
75
+ );
76
+ }
77
+ function requireListedChecksum(what, key, found, listed) {
78
+ if (found === listed) return;
79
+ throw new NarsilError(
80
+ ErrorCodes.PERSISTENCE_LOAD_FAILED,
81
+ `The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
82
+ { key, checksum: found, listed }
83
+ );
84
+ }
85
+
86
+ // src/persistence/durability/segment/layout.ts
87
+ var SEGMENT_ID_WIDTH = 16;
88
+ function manifestKey(indexName) {
89
+ return `${indexName}/manifest`;
90
+ }
91
+ function snapshotBundleKey(indexName) {
92
+ return `${indexName}/snapshot`;
93
+ }
94
+ function segmentPrefix(indexName, partitionId) {
95
+ return `${indexName}/segments/${partitionId}/`;
96
+ }
97
+ function segmentKey(indexName, partitionId, segmentId) {
98
+ return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
99
+ }
100
+ function segmentsPrefix(indexName) {
101
+ return `${indexName}/segments/`;
102
+ }
103
+ function vectorFileKey(indexName, fieldPath, fileId) {
104
+ return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
105
+ }
106
+ function requireGeneration(generation) {
107
+ if (Number.isSafeInteger(generation) && generation > 0) return;
108
+ throw new NarsilError(
109
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
110
+ `Vector graph generation ${generation} must be a positive integer`,
111
+ { generation }
112
+ );
113
+ }
114
+ function graphPartKey(indexName, fieldPath, generation, part) {
115
+ requireGeneration(generation);
116
+ return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
117
+ }
118
+ function formatSegmentId(segmentId) {
119
+ if (!Number.isInteger(segmentId) || segmentId < 0) {
120
+ throw new NarsilError(
121
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
122
+ `Segment id ${segmentId} must be a non-negative integer`,
123
+ { segmentId }
124
+ );
125
+ }
126
+ return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
127
+ }
128
+ var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
129
+ function encodeFieldPath(fieldPath) {
130
+ if (!FIELD_PATH_PATTERN.test(fieldPath)) {
131
+ throw new NarsilError(
132
+ ErrorCodes.PERSISTENCE_SAVE_FAILED,
133
+ `Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
134
+ { fieldPath }
135
+ );
136
+ }
137
+ const fingerprint = fnv1a(fieldPath).toString(36);
138
+ return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
139
+ }
140
+ var SEGMENT_MANIFEST_VERSION = 7;
141
+ var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
142
+ var MANIFEST_ENVELOPE_VERSION = 2;
143
+ var MAX_SEGMENTS_PER_PARTITION = 65536;
144
+ var MAX_SEGMENT_ID = 281474976710655;
145
+ function fileRefOnTheWire(ref) {
146
+ return {
147
+ key: ref.key,
148
+ bytes: ref.bytes,
149
+ checksum: ref.checksum,
150
+ ...ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum }
151
+ };
152
+ }
153
+ function encodeSegmentManifest(manifest) {
154
+ const payload = encode({
155
+ version: SEGMENT_MANIFEST_VERSION,
156
+ schema: manifest.schema,
157
+ language: manifest.language,
158
+ ...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
159
+ checkpoint: manifest.checkpoint.map((c) => ({
160
+ partitionId: c.partitionId,
161
+ lastSeqNo: c.lastSeqNo,
162
+ primaryTerm: c.primaryTerm,
163
+ ...c.termHistory === void 0 ? {} : {
164
+ termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
165
+ }
166
+ })),
167
+ partitions: manifest.partitions.map((p) => ({
168
+ partitionId: p.partitionId,
169
+ nextSegmentId: p.nextSegmentId,
170
+ segments: p.segments.map((s) => ({
171
+ id: s.id,
172
+ key: s.key,
173
+ docCount: s.docCount,
174
+ tombstoneCount: s.tombstoneCount,
175
+ bytes: s.bytes,
176
+ checksum: s.checksum
177
+ }))
178
+ })),
179
+ vectors: manifest.vectors.map((v) => ({
180
+ fieldPath: v.fieldPath,
181
+ nextFileId: v.nextFileId,
182
+ files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
183
+ graphGeneration: v.graphGeneration,
184
+ graphParts: v.graphParts.map(fileRefOnTheWire)
185
+ }))
186
+ });
187
+ return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
188
+ }
189
+ function manifestReferencedKeys(manifest) {
190
+ const keys = /* @__PURE__ */ new Set();
191
+ for (const partition of manifest.partitions) {
192
+ for (const segment of partition.segments) {
193
+ keys.add(segment.key);
194
+ }
195
+ }
196
+ for (const vector of manifest.vectors) {
197
+ for (const file of vector.files) keys.add(file.key);
198
+ for (const part of vector.graphParts) keys.add(part.key);
199
+ }
200
+ return keys;
201
+ }
202
+ function manifestFileRefs(manifest) {
203
+ const refs = [];
204
+ for (const partition of manifest.partitions) refs.push(...partition.segments);
205
+ for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
206
+ return refs;
207
+ }
208
+
209
+ export { EARLIER_SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION, MAX_SEGMENTS_PER_PARTITION, MAX_SEGMENT_ID, SEGMENT_MANIFEST_VERSION, adapterCheckpointStore, directoryCheckpointStore, encodeSegmentManifest, graphPartKey, manifestFileRefs, manifestKey, manifestReferencedKeys, readListedFile, requireListedChecksum, requireListedSize, segmentKey, segmentsPrefix, snapshotBundleKey, vectorFileKey };
@@ -1,7 +1,7 @@
1
- import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-U4X4AWKL.mjs';
2
1
  import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
2
+ import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-ZPWS2SKJ.mjs';
3
3
  import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
4
- import { flattenSchema, partialFieldsOf } from './chunk-UBXJQJII.mjs';
4
+ import { flattenSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
5
5
  import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
6
6
  import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
7
7
  import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-ZSWHPKGM.mjs';
@@ -1,5 +1,5 @@
1
1
  import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
2
- import { extractVectorFieldsFromSchema } from './chunk-UBXJQJII.mjs';
2
+ import { extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
3
3
  import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
4
4
  import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
5
5
 
@@ -106,11 +106,14 @@ function deadCountOf(dead) {
106
106
  }
107
107
  return count;
108
108
  }
109
- function archiveFootprint(prepared, manifest, replacesIndex) {
110
- let segmentBytes = 0;
109
+ function storedBytesOfSegments(manifest) {
110
+ let bytes = 0;
111
111
  for (const partition of manifest.partitions) {
112
- for (const segment of partition.segments) segmentBytes += segment.bytes;
112
+ for (const segment of partition.segments) bytes += segment.bytes;
113
113
  }
114
+ return bytes;
115
+ }
116
+ function archiveFootprint(prepared, manifest, replacesIndex, segmentBytes) {
114
117
  const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
115
118
  const vectors = [];
116
119
  for (const field of manifest.vectors) {
@@ -171,4 +174,4 @@ function requireRestoreMemory(indexName, footprint, replacesIndex) {
171
174
  if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
172
175
  }
173
176
 
174
- export { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, archiveFootprint, earlierSnapshotFootprint, requireAnalysisRebuildMemory, requireRestoreMemory };
177
+ export { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, archiveFootprint, earlierSnapshotFootprint, requireAnalysisRebuildMemory, requireRestoreMemory, storedBytesOfSegments };
@@ -0,0 +1,148 @@
1
+ import { crc32 } from './chunk-IO5F7ZBE.mjs';
2
+ import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
3
+ import { decode, encode } from '@msgpack/msgpack';
4
+
5
+ var PAGED_SEGMENT_ENVELOPE_VERSION = 6;
6
+ var PAGED_SEGMENT_HEAD_VERSION = 1;
7
+ var SEGMENT_PAGE_BYTES = 65536;
8
+ var RECORD_PAGE_BYTES = 16384;
9
+ var SECTION_ALIGNMENT_BYTES = 4;
10
+ var PAYLOAD_PREFIX_BYTES = 8;
11
+ var POSTING_WIDTH_BYTES = 8;
12
+ var POSITION_WIDTH_BYTES = 4;
13
+ var SMALLEST_BLOCK_POSTINGS = 128;
14
+ var DOCUMENT_BLOCK_MIN_RAW_BYTES = 65536;
15
+ var MAX_DOCUMENT_BLOCK_RAW_BYTES = 16842752;
16
+ var ID_SAMPLE_INTERVAL = 256;
17
+ var BLOCK_SUMMARY_BYTES = 22;
18
+ var DOCUMENT_BLOCK_BYTES = 20;
19
+ function loadFailed(message, details) {
20
+ throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
21
+ }
22
+ function isByteCount(value) {
23
+ return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
24
+ }
25
+ function decodeSection(raw, what, allowedPageBytes) {
26
+ if (raw === null || typeof raw !== "object") loadFailed(`The ${what} lists a section that is not a map`);
27
+ const entry = raw;
28
+ const { name, offset, length, pageBytes, checksums } = entry;
29
+ if (typeof name !== "string" || name.length === 0) loadFailed(`The ${what} lists a section without a name`);
30
+ if (!isByteCount(offset) || !isByteCount(length)) {
31
+ loadFailed(`The ${what} gives section "${name}" an invalid place`, { section: name, offset, length });
32
+ }
33
+ if (typeof pageBytes !== "number" || !allowedPageBytes.has(pageBytes)) {
34
+ loadFailed(`The ${what} gives section "${name}" a page size of ${String(pageBytes)}`, { section: name, pageBytes });
35
+ }
36
+ if (!(checksums instanceof Uint8Array)) loadFailed(`The ${what} gives section "${name}" no checksums`);
37
+ if (checksums.byteLength !== Math.ceil(length / pageBytes) * 4) {
38
+ loadFailed(`The ${what} gives section "${name}" ${checksums.byteLength} bytes of checksums`, {
39
+ section: name,
40
+ checksumBytes: checksums.byteLength
41
+ });
42
+ }
43
+ return { name, offset, length, pageBytes, checksums };
44
+ }
45
+ function decodeSections(raw, payloadLength, what, allowedPageBytes) {
46
+ if (!Array.isArray(raw)) loadFailed(`The ${what} holds no section list`);
47
+ const sections = raw.map((entry) => decodeSection(entry, what, allowedPageBytes));
48
+ const names = /* @__PURE__ */ new Set();
49
+ for (const section of sections) {
50
+ if (names.has(section.name)) loadFailed(`The ${what} repeats section "${section.name}"`, { section: section.name });
51
+ names.add(section.name);
52
+ if (section.offset + section.length > payloadLength) {
53
+ loadFailed(`The ${what} ends section "${section.name}" outside the payload`, { section: section.name });
54
+ }
55
+ }
56
+ const byOffset = [...sections].sort((left, right) => left.offset - right.offset);
57
+ for (let at = 1; at < byOffset.length; at++) {
58
+ const before = byOffset[at - 1];
59
+ if (before.offset + before.length > byOffset[at].offset) {
60
+ loadFailed(`The ${what} overlaps section "${before.name}" with "${byOffset[at].name}"`, {
61
+ section: before.name
62
+ });
63
+ }
64
+ }
65
+ return sections;
66
+ }
67
+ var SEGMENT_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
68
+ function isStringRecord(value) {
69
+ if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
70
+ return Object.values(value).every((entry) => typeof entry === "string");
71
+ }
72
+ function isStringList(value) {
73
+ return Array.isArray(value) && value.every((entry) => typeof entry === "string");
74
+ }
75
+ function decodePagedSegmentHead(head, payloadLength, what) {
76
+ let raw;
77
+ try {
78
+ raw = decode(head);
79
+ } catch (error) {
80
+ loadFailed(`The head of ${what} is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
81
+ }
82
+ if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
83
+ const entry = raw;
84
+ if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
85
+ loadFailed(`The head of ${what} holds version ${String(entry.v)}`, { version: entry.v });
86
+ }
87
+ if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
88
+ if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
89
+ if (!isStringList(entry.fieldNames)) loadFailed(`The head of ${what} holds no field names`);
90
+ const { documentCount } = entry;
91
+ if (typeof documentCount !== "number" || !Number.isInteger(documentCount) || documentCount < 0) {
92
+ loadFailed(`The head of ${what} holds an invalid document count`, { documentCount });
93
+ }
94
+ if (documentCount > 4294967295) loadFailed(`The head of ${what} holds a document count above uint32`);
95
+ return {
96
+ v: PAGED_SEGMENT_HEAD_VERSION,
97
+ schema: entry.schema,
98
+ tombstones: entry.tombstones,
99
+ documentCount,
100
+ fieldNames: entry.fieldNames,
101
+ sections: decodeSections(entry.sections, payloadLength, `head of ${what}`, SEGMENT_PAGE_SIZES)
102
+ };
103
+ }
104
+ function alignedUp(offset) {
105
+ const over = offset % SECTION_ALIGNMENT_BYTES;
106
+ return over === 0 ? offset : offset + SECTION_ALIGNMENT_BYTES - over;
107
+ }
108
+ function placeSections(planned, sectionsStart) {
109
+ let cursor = sectionsStart;
110
+ return planned.map((section) => {
111
+ const offset = alignedUp(cursor);
112
+ cursor = offset + section.length;
113
+ return {
114
+ name: section.name,
115
+ offset,
116
+ length: section.length,
117
+ pageBytes: SEGMENT_PAGE_BYTES,
118
+ checksums: section.checksums
119
+ };
120
+ });
121
+ }
122
+ function encodePagedSegmentHead(fields, planned) {
123
+ let headLength = 0;
124
+ for (; ; ) {
125
+ const sectionsStart = alignedUp(PAYLOAD_PREFIX_BYTES + headLength);
126
+ const sections = placeSections(planned, sectionsStart);
127
+ const head = encode({ v: PAGED_SEGMENT_HEAD_VERSION, ...fields, sections });
128
+ if (head.byteLength !== headLength) {
129
+ headLength = head.byteLength;
130
+ continue;
131
+ }
132
+ const prefix = new Uint8Array(PAYLOAD_PREFIX_BYTES);
133
+ const view = new DataView(prefix.buffer);
134
+ view.setUint32(0, head.byteLength, true);
135
+ view.setUint32(4, crc32(head), true);
136
+ const last = sections[sections.length - 1];
137
+ const payloadLength = last === void 0 ? sectionsStart : last.offset + last.length;
138
+ return { head, prefix, sections, payloadLength };
139
+ }
140
+ }
141
+ function sectionNamed(sections, name) {
142
+ for (const section of sections) {
143
+ if (section.name === name) return section;
144
+ }
145
+ return void 0;
146
+ }
147
+
148
+ export { BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, MAX_DOCUMENT_BLOCK_RAW_BYTES, PAGED_SEGMENT_ENVELOPE_VERSION, PAYLOAD_PREFIX_BYTES, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, SMALLEST_BLOCK_POSTINGS, decodePagedSegmentHead, decodeSections, encodePagedSegmentHead, loadFailed, sectionNamed };