@delali/narsil 0.3.1-next.131 → 0.3.1-next.141
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-HQUT2Z3I.mjs +43 -0
- package/dist/analysis-change-entry-TZTG4AZB.mjs +1 -0
- package/dist/browser/analysis-change-JPV775LR.mjs +900 -0
- package/dist/browser/analysis-change-entry-7HR4LWL3.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-5DB2VDGI.mjs} +1149 -1853
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-D33A5VMU.mjs} +16 -16
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-HKX2G6WH.mjs} +1520 -2159
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-KTOK275N.mjs} +606 -830
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-LVY2LI35.mjs} +2 -2
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-R4WUDZQV.mjs +176 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-TCLEOZ4P.mjs} +3 -3
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/chunk-XNWPP5NS.mjs +21 -0
- package/dist/browser/chunk-ZEV26AQ6.mjs +270 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-SNOWIT6Z.mjs} +370 -95
- package/dist/browser/import-TLB2A5LJ.mjs +9 -0
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-BVP4TKZU.mjs} +397 -109
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-ONYNYUDE.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-QXBKHJAG.mjs} +135 -183
- package/dist/browser/{write-YIC2MKRY.mjs → write-WHFBDWQS.mjs} +64 -90
- package/dist/{capabilities-B6WxrWxt.d.ts → capabilities-BCdUwK_L.d.ts} +10 -3
- package/dist/chunk-46NZOPHS.mjs +21 -0
- package/dist/{chunk-JT3YZGCE.mjs → chunk-5WYVUPGD.mjs} +3 -188
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-AST3SHFA.mjs} +27 -25
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-BRUUBXCB.mjs} +92 -17
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-EUCX5K74.mjs → chunk-DQNKVOJ2.mjs} +65 -65
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-HCGJKDZV.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-TGHNXN7W.mjs → chunk-IO5F7ZBE.mjs} +261 -948
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-KEIYVZUX.mjs} +5 -5
- package/dist/{chunk-PU6SKH7O.mjs → chunk-KGOXHLLZ.mjs} +7 -3
- package/dist/{chunk-WVRDRLIC.mjs → chunk-KZL72VTC.mjs} +70 -18
- package/dist/chunk-L76W3IZJ.mjs +193 -0
- package/dist/{chunk-2JWNJSNE.mjs → chunk-LA4WNDHL.mjs} +88 -801
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-F6AAXI7T.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-RCMNL7RI.mjs → chunk-SBMAWTRG.mjs} +41 -4
- package/dist/chunk-SRJBFG2T.mjs +881 -0
- package/dist/{chunk-X5CNUKD5.mjs → chunk-TNBIK2LQ.mjs} +1663 -1652
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-6M6SDH3S.mjs → chunk-VEFN6X6Q.mjs} +8 -8
- package/dist/chunk-VJEGOELZ.mjs +177 -0
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-WJGO5PGV.mjs} +317 -174
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-XZ3SRNDJ.mjs} +25 -64
- package/dist/{chunk-K77FJXKS.mjs → chunk-YHUTF6EA.mjs} +474 -93
- package/dist/{chunk-OGLV4I3L.mjs → chunk-YLNSJQLX.mjs} +12 -7
- package/dist/{chunk-SOHMMNCH.mjs → chunk-Z4HFMYNX.mjs} +32 -27
- package/dist/{chunk-5W7AEVTH.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-2MKX2SCO.mjs → chunk-ZPWS2SKJ.mjs} +297 -274
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-gNAfaHcE.d.ts → client-B12YFALP.d.ts} +18 -3
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +11 -1
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-D1cDv3sE.d.ts → config-BhZxIeb1.d.ts} +10 -1
- package/dist/{config-CmYJO34X.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-7XOQ7VQP.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -8
- package/dist/distribution.mjs +982 -211
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-324D2DDD.mjs} +362 -94
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-m_Ddr1bz.d.ts → engine-CtzbHOmG.d.ts} +64 -7
- package/dist/import-OOBUW3RR.mjs +44 -0
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +5 -5
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +22 -22
- package/dist/{long-query-thread-BSL25PD6.mjs → long-query-thread-4SW7RWDE.mjs} +15 -15
- package/dist/{module-2E4XYSGT.mjs → module-DXSJ5LT2.mjs} +378 -87
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +16 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +28 -26
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +9 -9
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-FEYAC4SB.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +12 -10
- package/dist/server.mjs +54 -21
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-CGM65VCD.mjs} +101 -197
- package/dist/synonyms.mjs +6 -4
- package/dist/{tasks-Co8HfRD7.d.ts → tasks-CjRJRGTb.d.ts} +26 -8
- package/dist/{thread-SKO7CN3R.mjs → thread-KGNAIKPD.mjs} +14 -14
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +6 -5
- package/dist/workers/entry.mjs +8 -8
- package/dist/{write-ZKR7SNRT.mjs → write-2DZPEQL2.mjs} +82 -107
- package/openapi.json +114 -6
- package/package.json +12 -8
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { SCRATCH_SLOTS_PER_THREAD_POOL } from './chunk-QSK35UC2.mjs';
|
|
2
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
3
3
|
|
|
4
4
|
// src/workers/constants.ts
|
|
5
5
|
var WORKER_YOUNG_GENERATION_MB = 24;
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
import { packChecksummedEnvelope } from './chunk-IO5F7ZBE.mjs';
|
|
2
|
+
import { fnv1a } from './chunk-ZPWS2SKJ.mjs';
|
|
3
|
+
import { ErrorCodes, NarsilError } from './chunk-HNHCNTVG.mjs';
|
|
4
|
+
import { encode } from '@msgpack/msgpack';
|
|
5
|
+
|
|
6
|
+
// src/persistence/durability/segment/store.ts
|
|
7
|
+
function directoryCheckpointStore(directory) {
|
|
8
|
+
const store = {
|
|
9
|
+
read: (key) => directory.read(key),
|
|
10
|
+
write: (key, chunks) => directory.atomicWrite(key, chunks),
|
|
11
|
+
remove: (key) => directory.remove(key),
|
|
12
|
+
list: (prefix) => directory.list(prefix),
|
|
13
|
+
pathOf: (key) => directory.pathOf(key),
|
|
14
|
+
directory
|
|
15
|
+
};
|
|
16
|
+
const byteLength = directory.byteLength;
|
|
17
|
+
if (byteLength !== void 0) store.byteLength = (key) => byteLength.call(directory, key);
|
|
18
|
+
return store;
|
|
19
|
+
}
|
|
20
|
+
function joined(chunks) {
|
|
21
|
+
if (chunks.length === 1) return chunks[0];
|
|
22
|
+
let total = 0;
|
|
23
|
+
for (const chunk of chunks) total += chunk.byteLength;
|
|
24
|
+
const bytes = new Uint8Array(total);
|
|
25
|
+
let offset = 0;
|
|
26
|
+
for (const chunk of chunks) {
|
|
27
|
+
bytes.set(chunk, offset);
|
|
28
|
+
offset += chunk.byteLength;
|
|
29
|
+
}
|
|
30
|
+
return bytes;
|
|
31
|
+
}
|
|
32
|
+
function adapterFailure(code, verb, key, error) {
|
|
33
|
+
if (error instanceof NarsilError) return error;
|
|
34
|
+
const cause = error instanceof Error ? error.message : String(error);
|
|
35
|
+
return new NarsilError(code, `The persistence adapter failed to ${verb} "${key}": ${cause}`, { key, cause });
|
|
36
|
+
}
|
|
37
|
+
async function guarded(run, code, verb, key) {
|
|
38
|
+
try {
|
|
39
|
+
return await run();
|
|
40
|
+
} catch (error) {
|
|
41
|
+
throw adapterFailure(code, verb, key, error);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
function adapterCheckpointStore(adapter) {
|
|
45
|
+
return {
|
|
46
|
+
read: (key) => guarded(() => adapter.load(key), ErrorCodes.PERSISTENCE_LOAD_FAILED, "load", key),
|
|
47
|
+
write: (key, chunks) => guarded(() => adapter.save(key, joined(chunks)), ErrorCodes.PERSISTENCE_SAVE_FAILED, "save", key),
|
|
48
|
+
remove: (key) => guarded(() => adapter.delete(key), ErrorCodes.PERSISTENCE_DELETE_FAILED, "delete", key),
|
|
49
|
+
list: (prefix) => guarded(() => adapter.list(prefix), ErrorCodes.PERSISTENCE_LOAD_FAILED, "list", prefix)
|
|
50
|
+
};
|
|
51
|
+
}
|
|
52
|
+
async function requireListedSize(store, key, expected) {
|
|
53
|
+
const length = store.byteLength === void 0 ? void 0 : await store.byteLength(key);
|
|
54
|
+
if (length === null) missingFile(expected.what, key);
|
|
55
|
+
if (length !== void 0) requireListedLength(expected.what, key, length, expected.bytes);
|
|
56
|
+
}
|
|
57
|
+
async function readListedFile(store, key, expected) {
|
|
58
|
+
await requireListedSize(store, key, expected);
|
|
59
|
+
const bytes = await store.read(key);
|
|
60
|
+
if (bytes === null) missingFile(expected.what, key);
|
|
61
|
+
requireListedLength(expected.what, key, bytes.byteLength, expected.bytes);
|
|
62
|
+
return bytes;
|
|
63
|
+
}
|
|
64
|
+
function missingFile(what, key) {
|
|
65
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, `The ${what} "${key}" that the manifest lists is missing`, {
|
|
66
|
+
key
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
function requireListedLength(what, key, found, listed) {
|
|
70
|
+
if (found === listed) return;
|
|
71
|
+
throw new NarsilError(
|
|
72
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
73
|
+
`The ${what} "${key}" holds ${found} bytes where the manifest lists ${listed}`,
|
|
74
|
+
{ key, bytes: found, listed }
|
|
75
|
+
);
|
|
76
|
+
}
|
|
77
|
+
function requireListedChecksum(what, key, found, listed) {
|
|
78
|
+
if (found === listed) return;
|
|
79
|
+
throw new NarsilError(
|
|
80
|
+
ErrorCodes.PERSISTENCE_LOAD_FAILED,
|
|
81
|
+
`The ${what} "${key}" carries checksum ${found} where the manifest lists ${listed}`,
|
|
82
|
+
{ key, checksum: found, listed }
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// src/persistence/durability/segment/layout.ts
|
|
87
|
+
var SEGMENT_ID_WIDTH = 16;
|
|
88
|
+
function manifestKey(indexName) {
|
|
89
|
+
return `${indexName}/manifest`;
|
|
90
|
+
}
|
|
91
|
+
function snapshotBundleKey(indexName) {
|
|
92
|
+
return `${indexName}/snapshot`;
|
|
93
|
+
}
|
|
94
|
+
function segmentPrefix(indexName, partitionId) {
|
|
95
|
+
return `${indexName}/segments/${partitionId}/`;
|
|
96
|
+
}
|
|
97
|
+
function segmentKey(indexName, partitionId, segmentId) {
|
|
98
|
+
return `${segmentPrefix(indexName, partitionId)}s${formatSegmentId(segmentId)}`;
|
|
99
|
+
}
|
|
100
|
+
function segmentsPrefix(indexName) {
|
|
101
|
+
return `${indexName}/segments/`;
|
|
102
|
+
}
|
|
103
|
+
function vectorFileKey(indexName, fieldPath, fileId) {
|
|
104
|
+
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-f${formatSegmentId(fileId)}`;
|
|
105
|
+
}
|
|
106
|
+
function requireGeneration(generation) {
|
|
107
|
+
if (Number.isSafeInteger(generation) && generation > 0) return;
|
|
108
|
+
throw new NarsilError(
|
|
109
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
110
|
+
`Vector graph generation ${generation} must be a positive integer`,
|
|
111
|
+
{ generation }
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
function graphPartKey(indexName, fieldPath, generation, part) {
|
|
115
|
+
requireGeneration(generation);
|
|
116
|
+
return `${segmentsPrefix(indexName)}vec-${encodeFieldPath(fieldPath)}-graph-g${generation}-p${formatSegmentId(part)}`;
|
|
117
|
+
}
|
|
118
|
+
function formatSegmentId(segmentId) {
|
|
119
|
+
if (!Number.isInteger(segmentId) || segmentId < 0) {
|
|
120
|
+
throw new NarsilError(
|
|
121
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
122
|
+
`Segment id ${segmentId} must be a non-negative integer`,
|
|
123
|
+
{ segmentId }
|
|
124
|
+
);
|
|
125
|
+
}
|
|
126
|
+
return segmentId.toString().padStart(SEGMENT_ID_WIDTH, "0");
|
|
127
|
+
}
|
|
128
|
+
var FIELD_PATH_PATTERN = /^[A-Za-z0-9_.]+$/;
|
|
129
|
+
function encodeFieldPath(fieldPath) {
|
|
130
|
+
if (!FIELD_PATH_PATTERN.test(fieldPath)) {
|
|
131
|
+
throw new NarsilError(
|
|
132
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
133
|
+
`Vector field path "${fieldPath}" contains characters that cannot be encoded in a segment key`,
|
|
134
|
+
{ fieldPath }
|
|
135
|
+
);
|
|
136
|
+
}
|
|
137
|
+
const fingerprint = fnv1a(fieldPath).toString(36);
|
|
138
|
+
return `${fieldPath.replace(/\./g, "_")}-${fingerprint}`;
|
|
139
|
+
}
|
|
140
|
+
var SEGMENT_MANIFEST_VERSION = 7;
|
|
141
|
+
var EARLIER_SEGMENT_MANIFEST_VERSION = 6;
|
|
142
|
+
var MANIFEST_ENVELOPE_VERSION = 2;
|
|
143
|
+
var MAX_SEGMENTS_PER_PARTITION = 65536;
|
|
144
|
+
var MAX_SEGMENT_ID = 281474976710655;
|
|
145
|
+
function fileRefOnTheWire(ref) {
|
|
146
|
+
return {
|
|
147
|
+
key: ref.key,
|
|
148
|
+
bytes: ref.bytes,
|
|
149
|
+
checksum: ref.checksum,
|
|
150
|
+
...ref.headOffset === void 0 ? {} : { headOffset: ref.headOffset, headLength: ref.headLength, headChecksum: ref.headChecksum }
|
|
151
|
+
};
|
|
152
|
+
}
|
|
153
|
+
function encodeSegmentManifest(manifest) {
|
|
154
|
+
const payload = encode({
|
|
155
|
+
version: SEGMENT_MANIFEST_VERSION,
|
|
156
|
+
schema: manifest.schema,
|
|
157
|
+
language: manifest.language,
|
|
158
|
+
...manifest.analysisGeneration ? { analysisGeneration: manifest.analysisGeneration } : {},
|
|
159
|
+
checkpoint: manifest.checkpoint.map((c) => ({
|
|
160
|
+
partitionId: c.partitionId,
|
|
161
|
+
lastSeqNo: c.lastSeqNo,
|
|
162
|
+
primaryTerm: c.primaryTerm,
|
|
163
|
+
...c.termHistory === void 0 ? {} : {
|
|
164
|
+
termHistory: c.termHistory.map((start) => ({ primaryTerm: start.primaryTerm, firstSeqNo: start.firstSeqNo }))
|
|
165
|
+
}
|
|
166
|
+
})),
|
|
167
|
+
partitions: manifest.partitions.map((p) => ({
|
|
168
|
+
partitionId: p.partitionId,
|
|
169
|
+
nextSegmentId: p.nextSegmentId,
|
|
170
|
+
segments: p.segments.map((s) => ({
|
|
171
|
+
id: s.id,
|
|
172
|
+
key: s.key,
|
|
173
|
+
docCount: s.docCount,
|
|
174
|
+
tombstoneCount: s.tombstoneCount,
|
|
175
|
+
bytes: s.bytes,
|
|
176
|
+
checksum: s.checksum
|
|
177
|
+
}))
|
|
178
|
+
})),
|
|
179
|
+
vectors: manifest.vectors.map((v) => ({
|
|
180
|
+
fieldPath: v.fieldPath,
|
|
181
|
+
nextFileId: v.nextFileId,
|
|
182
|
+
files: v.files.map((f) => ({ id: f.id, ...fileRefOnTheWire(f), count: f.count, dead: f.dead })),
|
|
183
|
+
graphGeneration: v.graphGeneration,
|
|
184
|
+
graphParts: v.graphParts.map(fileRefOnTheWire)
|
|
185
|
+
}))
|
|
186
|
+
});
|
|
187
|
+
return packChecksummedEnvelope([payload], MANIFEST_ENVELOPE_VERSION, "segment manifest");
|
|
188
|
+
}
|
|
189
|
+
function manifestReferencedKeys(manifest) {
|
|
190
|
+
const keys = /* @__PURE__ */ new Set();
|
|
191
|
+
for (const partition of manifest.partitions) {
|
|
192
|
+
for (const segment of partition.segments) {
|
|
193
|
+
keys.add(segment.key);
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
for (const vector of manifest.vectors) {
|
|
197
|
+
for (const file of vector.files) keys.add(file.key);
|
|
198
|
+
for (const part of vector.graphParts) keys.add(part.key);
|
|
199
|
+
}
|
|
200
|
+
return keys;
|
|
201
|
+
}
|
|
202
|
+
function manifestFileRefs(manifest) {
|
|
203
|
+
const refs = [];
|
|
204
|
+
for (const partition of manifest.partitions) refs.push(...partition.segments);
|
|
205
|
+
for (const vector of manifest.vectors) refs.push(...vector.files, ...vector.graphParts);
|
|
206
|
+
return refs;
|
|
207
|
+
}
|
|
208
|
+
|
|
209
|
+
export { EARLIER_SEGMENT_MANIFEST_VERSION, MANIFEST_ENVELOPE_VERSION, MAX_SEGMENTS_PER_PARTITION, MAX_SEGMENT_ID, SEGMENT_MANIFEST_VERSION, adapterCheckpointStore, directoryCheckpointStore, encodeSegmentManifest, graphPartKey, manifestFileRefs, manifestKey, manifestReferencedKeys, readListedFile, requireListedChecksum, requireListedSize, segmentKey, segmentsPrefix, snapshotBundleKey, vectorFileKey };
|
|
@@ -1,15 +1,15 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import {
|
|
1
|
+
import { MAX_INDEX_NAME_LENGTH, INDEX_NAME_PATTERN, MAX_DOC_ID_LENGTH, DEFAULT_LIMIT, DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, ABSOLUTE_WINDOWS_PATH_PATTERN, MODULE_URL_SCHEME_PATTERN, DEFAULT_OFFSET } from './chunk-EEBCDP36.mjs';
|
|
2
|
+
import { normalizeSort, requireMatchingCursor, encodePageCursor, decodePageCursor, projectionKeepsField, resolveProjection, clampRowCount, bm25SetsValue, sortSignatureOf, requireSortableFields, analyseQueryPhrases, requireLoadedPhraseCode, fanOutQuery, loadPhraseCodeFor, requireWithinResultWindow, sortModesOf, applySorting, applyPagination, countsWithoutScores, fanOutMatchCount, FNV_OFFSET_BASIS, validatedFilterOf, createPatternWorkMeter, partialWordsToHighlight, partialWordHighlighter, pruneStatsToQueryTerms, mergeBM25Levels, everyValueFacetConfig, fulltextMatches, mergeFacets, applyPartitionFiltersBitset, fnv1aContinue, partitionsIn, fnv1aContinueTaggedNumbers, undeclaredFieldOnStrictIndex, MAX_BM25_K1, MAX_BM25_B } from './chunk-ZPWS2SKJ.mjs';
|
|
3
3
|
import { MIN_OVERSAMPLE, MAX_RENUMBERED_SEARCH_ATTEMPTS } from './chunk-QSK35UC2.mjs';
|
|
4
|
-
import { flattenSchema, partialFieldsOf } from './chunk-
|
|
4
|
+
import { flattenSchema, partialFieldsOf } from './chunk-KGOXHLLZ.mjs';
|
|
5
5
|
import { DEFAULT_MAX_PER_GROUP, DEFAULT_FUZZY_PREFIX_LENGTH, BM25_DISPLAYED_NAME_LENGTH, DEFAULT_LINEAR_ALPHA, MIN_LINEAR_ALPHA, MAX_LINEAR_ALPHA, DEFAULT_RANK_CONSTANT, MIN_RANK_CONSTANT, RESULT_WINDOW, MAX_TOLERANCE, MAX_PREFIX_LENGTH } from './chunk-FQBFZAWG.mjs';
|
|
6
6
|
import { fuzzyTermMatches } from './chunk-GES7U2PM.mjs';
|
|
7
|
-
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-
|
|
8
|
-
import { tokenize } from './chunk-
|
|
7
|
+
import { isStringValuedBase, parsedTypeOf, isWordIndexedFieldType } from './chunk-ZSWHPKGM.mjs';
|
|
8
|
+
import { tokenize } from './chunk-NPNK57CA.mjs';
|
|
9
9
|
import { compareComparableKeys } from './chunk-R6A773WK.mjs';
|
|
10
10
|
import { compareCodePoints, holdsUnpairedSurrogate } from './chunk-YTJCGKM4.mjs';
|
|
11
11
|
import { MIN_PATTERN_VALUE_LIMIT, MAX_PATTERN_VALUE_LIMIT, isPlainObject, isGeopointOrVectorType, VECTOR_PATTERN } from './chunk-L5ORED7V.mjs';
|
|
12
|
-
import { NarsilError, ErrorCodes } from './chunk-
|
|
12
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
13
13
|
|
|
14
14
|
// src/search/cursor-binding.ts
|
|
15
15
|
var ABSENT_TAG = 0;
|
|
@@ -1550,7 +1550,7 @@ function requireCursorDepth(cursor, signature, binding) {
|
|
|
1550
1550
|
}
|
|
1551
1551
|
function hybridSearchOf(params, context, limit, offset, cursorDepth) {
|
|
1552
1552
|
if ((params.hybrid?.depth ?? null) === null) return executeHybridSearch(params, context, limit, offset, cursorDepth);
|
|
1553
|
-
return import('./runtime-
|
|
1553
|
+
return import('./runtime-DYV75DCB.mjs').then(
|
|
1554
1554
|
(runtime) => runtime.searchFixedDepthHybrid(params, context, { cursorDepth: cursorDepth ?? 0, offset, limit }, runtimeHost)
|
|
1555
1555
|
);
|
|
1556
1556
|
}
|
|
@@ -1560,7 +1560,7 @@ function executeQuery(request, callerContext) {
|
|
|
1560
1560
|
if (loadingPhraseCode === null) return runQuery(request, callerContext);
|
|
1561
1561
|
return loadingPhraseCode.then(() => runQuery(request, callerContext));
|
|
1562
1562
|
}
|
|
1563
|
-
return import('./runtime-
|
|
1563
|
+
return import('./runtime-DYV75DCB.mjs').then(
|
|
1564
1564
|
(runtime) => runtime.executeRerankedQuery(request, callerContext, executeQuery, runtimeHost)
|
|
1565
1565
|
);
|
|
1566
1566
|
}
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
import { readHeapStatistics, readResidentMemory } from './chunk-ZATRDAYU.mjs';
|
|
2
|
+
import { extractVectorFieldsFromSchema } from './chunk-KGOXHLLZ.mjs';
|
|
3
|
+
import { collectAndReturnPages } from './chunk-RIIUZJQ3.mjs';
|
|
4
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
5
|
+
|
|
6
|
+
// src/engine/snapshot/constants.ts
|
|
7
|
+
var RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE = 0.2;
|
|
8
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 7.86;
|
|
9
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR = 264;
|
|
10
|
+
var RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.24;
|
|
11
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 738;
|
|
12
|
+
var RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.2;
|
|
13
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE = 4.06;
|
|
14
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 14.5;
|
|
15
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR = 1482;
|
|
16
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 9.29;
|
|
17
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 2141;
|
|
18
|
+
var RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 24.15;
|
|
19
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE = 0.16;
|
|
20
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 6.16;
|
|
21
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR = 204;
|
|
22
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION = 0.17;
|
|
23
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 695;
|
|
24
|
+
var RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 0.04;
|
|
25
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE = 2.14;
|
|
26
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE = 5.05;
|
|
27
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR = 1080;
|
|
28
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION = 0.72;
|
|
29
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR = 1744;
|
|
30
|
+
var RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION = 20.05;
|
|
31
|
+
var RESTORE_ESTIMATE_MARGIN = 1.25;
|
|
32
|
+
var MAX_SNAPSHOT_ANALYSIS_ATTEMPTS = 3;
|
|
33
|
+
|
|
34
|
+
// src/engine/snapshot/memory-budget.ts
|
|
35
|
+
var MEGABYTE_BYTES = 1048576;
|
|
36
|
+
var ARCHIVE_RATES = {
|
|
37
|
+
worst: {
|
|
38
|
+
heap: {
|
|
39
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_STORED_BYTE,
|
|
40
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_VECTOR,
|
|
41
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
42
|
+
},
|
|
43
|
+
external: {
|
|
44
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
45
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR,
|
|
46
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
floor: {
|
|
50
|
+
heap: {
|
|
51
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_STORED_BYTE,
|
|
52
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR,
|
|
53
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_VECTOR_DIMENSION
|
|
54
|
+
},
|
|
55
|
+
external: {
|
|
56
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_STORED_BYTE,
|
|
57
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR,
|
|
58
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_VECTOR_DIMENSION
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
};
|
|
62
|
+
var EARLIER_SNAPSHOT_RATES = {
|
|
63
|
+
worst: {
|
|
64
|
+
heap: {
|
|
65
|
+
perStoredByte: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
66
|
+
perVector: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
67
|
+
perVectorDimension: RESTORE_WORST_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
68
|
+
},
|
|
69
|
+
external: {
|
|
70
|
+
perStoredByte: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
71
|
+
perVector: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
72
|
+
perVectorDimension: RESTORE_WORST_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
73
|
+
}
|
|
74
|
+
},
|
|
75
|
+
floor: {
|
|
76
|
+
heap: {
|
|
77
|
+
perStoredByte: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
78
|
+
perVector: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
79
|
+
perVectorDimension: RESTORE_FLOOR_HEAP_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
80
|
+
},
|
|
81
|
+
external: {
|
|
82
|
+
perStoredByte: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_BYTE,
|
|
83
|
+
perVector: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR,
|
|
84
|
+
perVectorDimension: RESTORE_FLOOR_EXTERNAL_BYTES_PER_EARLIER_SNAPSHOT_VECTOR_DIMENSION
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
};
|
|
88
|
+
function bytesAt(rate, storedBytes, vectors) {
|
|
89
|
+
let bytes = storedBytes * rate.perStoredByte;
|
|
90
|
+
for (const { dimension, count } of vectors) bytes += count * (rate.perVector + dimension * rate.perVectorDimension);
|
|
91
|
+
return bytes;
|
|
92
|
+
}
|
|
93
|
+
function footprintAt(rates, replacesIndex, storedBytes, vectors) {
|
|
94
|
+
const chosen = replacesIndex ? rates.worst : rates.floor;
|
|
95
|
+
const scale = replacesIndex ? RESTORE_ESTIMATE_MARGIN : 1 / RESTORE_ESTIMATE_MARGIN;
|
|
96
|
+
return {
|
|
97
|
+
heapBytes: bytesAt(chosen.heap, storedBytes, vectors) * scale,
|
|
98
|
+
externalBytes: bytesAt(chosen.external, storedBytes, vectors) * scale
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
function deadCountOf(dead) {
|
|
102
|
+
if (dead === null) return 0;
|
|
103
|
+
let count = 0;
|
|
104
|
+
for (const byte of dead) {
|
|
105
|
+
for (let bits = byte; bits !== 0; bits &= bits - 1) count += 1;
|
|
106
|
+
}
|
|
107
|
+
return count;
|
|
108
|
+
}
|
|
109
|
+
function storedBytesOfSegments(manifest) {
|
|
110
|
+
let bytes = 0;
|
|
111
|
+
for (const partition of manifest.partitions) {
|
|
112
|
+
for (const segment of partition.segments) bytes += segment.bytes;
|
|
113
|
+
}
|
|
114
|
+
return bytes;
|
|
115
|
+
}
|
|
116
|
+
function archiveFootprint(prepared, manifest, replacesIndex, segmentBytes) {
|
|
117
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
118
|
+
const vectors = [];
|
|
119
|
+
for (const field of manifest.vectors) {
|
|
120
|
+
const dimension = dimensions.get(field.fieldPath) ?? 0;
|
|
121
|
+
for (const file of field.files) vectors.push({ dimension, count: Math.max(0, file.count - deadCountOf(file.dead)) });
|
|
122
|
+
}
|
|
123
|
+
return footprintAt(ARCHIVE_RATES, replacesIndex, segmentBytes, vectors);
|
|
124
|
+
}
|
|
125
|
+
function earlierSnapshotFootprint(prepared, partitionBytes, vectorCounts, replacesIndex) {
|
|
126
|
+
const dimensions = extractVectorFieldsFromSchema(prepared.config.schema);
|
|
127
|
+
const vectors = [...vectorCounts].map(([fieldPath, count]) => ({ dimension: dimensions.get(fieldPath) ?? 0, count }));
|
|
128
|
+
return footprintAt(EARLIER_SNAPSHOT_RATES, replacesIndex, partitionBytes, vectors);
|
|
129
|
+
}
|
|
130
|
+
function inMegabytes(bytes) {
|
|
131
|
+
return Math.ceil(bytes / MEGABYTE_BYTES);
|
|
132
|
+
}
|
|
133
|
+
function shortfallOf(heapBytes, processBytes, countInUse) {
|
|
134
|
+
const heap = readHeapStatistics();
|
|
135
|
+
if (heap !== null) {
|
|
136
|
+
const limitBytes = heap.configuredLimitBytes ?? heap.limitBytes;
|
|
137
|
+
const availableBytes = countInUse ? limitBytes - heap.usedBytes : limitBytes;
|
|
138
|
+
if (heapBytes > availableBytes) return { budget: "heap", neededBytes: heapBytes, availableBytes };
|
|
139
|
+
}
|
|
140
|
+
const resident = readResidentMemory();
|
|
141
|
+
if (resident !== null) {
|
|
142
|
+
const availableBytes = countInUse ? resident.limitBytes - resident.residentBytes : resident.limitBytes;
|
|
143
|
+
if (processBytes > availableBytes) return { budget: "process", neededBytes: processBytes, availableBytes };
|
|
144
|
+
}
|
|
145
|
+
return null;
|
|
146
|
+
}
|
|
147
|
+
function refuse(indexName, replacesIndex, shortfall) {
|
|
148
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
149
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
150
|
+
const outcome = replacesIndex ? "so the index keeps its earlier contents. Drop the index first to free the memory that it holds, or give the process more memory, then restore again" : "so the engine creates no index. Give the process more memory, then restore again";
|
|
151
|
+
throw new NarsilError(
|
|
152
|
+
ErrorCodes.RESTORE_MEMORY_EXCEEDED,
|
|
153
|
+
`The restore of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where}, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, ${outcome}`,
|
|
154
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
155
|
+
);
|
|
156
|
+
}
|
|
157
|
+
function requireAnalysisRebuildMemory(indexName, termBytes) {
|
|
158
|
+
if (shortfallOf(termBytes, termBytes, true) === null) return;
|
|
159
|
+
const shortfall = shortfallOf(termBytes, termBytes, collectAndReturnPages());
|
|
160
|
+
if (shortfall === null) return;
|
|
161
|
+
const { budget, neededBytes, availableBytes } = shortfall;
|
|
162
|
+
const where = budget === "heap" ? "of JavaScript heap" : "of process memory";
|
|
163
|
+
throw new NarsilError(
|
|
164
|
+
ErrorCodes.ANALYSIS_REBUILD_MEMORY_EXCEEDED,
|
|
165
|
+
`The analysis change of index "${indexName}" needs about ${inMegabytes(neededBytes)} MB ${where} for a second copy of its terms, and this process has ${inMegabytes(Math.max(0, availableBytes))} MB free, so the index keeps its current settings. Give the process more memory, then change the settings again`,
|
|
166
|
+
{ indexName, budget, neededBytes, availableBytes }
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
function requireRestoreMemory(indexName, footprint, replacesIndex) {
|
|
170
|
+
const processBytes = footprint.heapBytes + footprint.externalBytes;
|
|
171
|
+
if (shortfallOf(footprint.heapBytes, processBytes, true) === null) return;
|
|
172
|
+
const collected = collectAndReturnPages();
|
|
173
|
+
const shortfall = shortfallOf(footprint.heapBytes, processBytes, collected || replacesIndex);
|
|
174
|
+
if (shortfall !== null) refuse(indexName, replacesIndex, shortfall);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export { MAX_SNAPSHOT_ANALYSIS_ATTEMPTS, archiveFootprint, earlierSnapshotFootprint, requireAnalysisRebuildMemory, requireRestoreMemory, storedBytesOfSegments };
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { crc32 } from './chunk-IO5F7ZBE.mjs';
|
|
2
|
+
import { NarsilError, ErrorCodes } from './chunk-HNHCNTVG.mjs';
|
|
3
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
4
|
+
|
|
5
|
+
var PAGED_SEGMENT_ENVELOPE_VERSION = 6;
|
|
6
|
+
var PAGED_SEGMENT_HEAD_VERSION = 1;
|
|
7
|
+
var SEGMENT_PAGE_BYTES = 65536;
|
|
8
|
+
var RECORD_PAGE_BYTES = 16384;
|
|
9
|
+
var SECTION_ALIGNMENT_BYTES = 4;
|
|
10
|
+
var PAYLOAD_PREFIX_BYTES = 8;
|
|
11
|
+
var POSTING_WIDTH_BYTES = 8;
|
|
12
|
+
var POSITION_WIDTH_BYTES = 4;
|
|
13
|
+
var SMALLEST_BLOCK_POSTINGS = 128;
|
|
14
|
+
var DOCUMENT_BLOCK_MIN_RAW_BYTES = 65536;
|
|
15
|
+
var MAX_DOCUMENT_BLOCK_RAW_BYTES = 16842752;
|
|
16
|
+
var ID_SAMPLE_INTERVAL = 256;
|
|
17
|
+
var BLOCK_SUMMARY_BYTES = 22;
|
|
18
|
+
var DOCUMENT_BLOCK_BYTES = 20;
|
|
19
|
+
function loadFailed(message, details) {
|
|
20
|
+
throw new NarsilError(ErrorCodes.PERSISTENCE_LOAD_FAILED, message, details);
|
|
21
|
+
}
|
|
22
|
+
function isByteCount(value) {
|
|
23
|
+
return typeof value === "number" && Number.isSafeInteger(value) && value >= 0;
|
|
24
|
+
}
|
|
25
|
+
function decodeSection(raw, what, allowedPageBytes) {
|
|
26
|
+
if (raw === null || typeof raw !== "object") loadFailed(`The ${what} lists a section that is not a map`);
|
|
27
|
+
const entry = raw;
|
|
28
|
+
const { name, offset, length, pageBytes, checksums } = entry;
|
|
29
|
+
if (typeof name !== "string" || name.length === 0) loadFailed(`The ${what} lists a section without a name`);
|
|
30
|
+
if (!isByteCount(offset) || !isByteCount(length)) {
|
|
31
|
+
loadFailed(`The ${what} gives section "${name}" an invalid place`, { section: name, offset, length });
|
|
32
|
+
}
|
|
33
|
+
if (typeof pageBytes !== "number" || !allowedPageBytes.has(pageBytes)) {
|
|
34
|
+
loadFailed(`The ${what} gives section "${name}" a page size of ${String(pageBytes)}`, { section: name, pageBytes });
|
|
35
|
+
}
|
|
36
|
+
if (!(checksums instanceof Uint8Array)) loadFailed(`The ${what} gives section "${name}" no checksums`);
|
|
37
|
+
if (checksums.byteLength !== Math.ceil(length / pageBytes) * 4) {
|
|
38
|
+
loadFailed(`The ${what} gives section "${name}" ${checksums.byteLength} bytes of checksums`, {
|
|
39
|
+
section: name,
|
|
40
|
+
checksumBytes: checksums.byteLength
|
|
41
|
+
});
|
|
42
|
+
}
|
|
43
|
+
return { name, offset, length, pageBytes, checksums };
|
|
44
|
+
}
|
|
45
|
+
function decodeSections(raw, payloadLength, what, allowedPageBytes) {
|
|
46
|
+
if (!Array.isArray(raw)) loadFailed(`The ${what} holds no section list`);
|
|
47
|
+
const sections = raw.map((entry) => decodeSection(entry, what, allowedPageBytes));
|
|
48
|
+
const names = /* @__PURE__ */ new Set();
|
|
49
|
+
for (const section of sections) {
|
|
50
|
+
if (names.has(section.name)) loadFailed(`The ${what} repeats section "${section.name}"`, { section: section.name });
|
|
51
|
+
names.add(section.name);
|
|
52
|
+
if (section.offset + section.length > payloadLength) {
|
|
53
|
+
loadFailed(`The ${what} ends section "${section.name}" outside the payload`, { section: section.name });
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const byOffset = [...sections].sort((left, right) => left.offset - right.offset);
|
|
57
|
+
for (let at = 1; at < byOffset.length; at++) {
|
|
58
|
+
const before = byOffset[at - 1];
|
|
59
|
+
if (before.offset + before.length > byOffset[at].offset) {
|
|
60
|
+
loadFailed(`The ${what} overlaps section "${before.name}" with "${byOffset[at].name}"`, {
|
|
61
|
+
section: before.name
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
return sections;
|
|
66
|
+
}
|
|
67
|
+
var SEGMENT_PAGE_SIZES = /* @__PURE__ */ new Set([SEGMENT_PAGE_BYTES]);
|
|
68
|
+
function isStringRecord(value) {
|
|
69
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return false;
|
|
70
|
+
return Object.values(value).every((entry) => typeof entry === "string");
|
|
71
|
+
}
|
|
72
|
+
function isStringList(value) {
|
|
73
|
+
return Array.isArray(value) && value.every((entry) => typeof entry === "string");
|
|
74
|
+
}
|
|
75
|
+
function decodePagedSegmentHead(head, payloadLength, what) {
|
|
76
|
+
let raw;
|
|
77
|
+
try {
|
|
78
|
+
raw = decode(head);
|
|
79
|
+
} catch (error) {
|
|
80
|
+
loadFailed(`The head of ${what} is not MessagePack: ${error instanceof Error ? error.message : String(error)}`);
|
|
81
|
+
}
|
|
82
|
+
if (raw === null || typeof raw !== "object") loadFailed(`The head of ${what} is not a map`);
|
|
83
|
+
const entry = raw;
|
|
84
|
+
if (entry.v !== PAGED_SEGMENT_HEAD_VERSION) {
|
|
85
|
+
loadFailed(`The head of ${what} holds version ${String(entry.v)}`, { version: entry.v });
|
|
86
|
+
}
|
|
87
|
+
if (!isStringRecord(entry.schema)) loadFailed(`The head of ${what} holds no schema`);
|
|
88
|
+
if (!isStringList(entry.tombstones)) loadFailed(`The head of ${what} holds no tombstone list`);
|
|
89
|
+
if (!isStringList(entry.fieldNames)) loadFailed(`The head of ${what} holds no field names`);
|
|
90
|
+
const { documentCount } = entry;
|
|
91
|
+
if (typeof documentCount !== "number" || !Number.isInteger(documentCount) || documentCount < 0) {
|
|
92
|
+
loadFailed(`The head of ${what} holds an invalid document count`, { documentCount });
|
|
93
|
+
}
|
|
94
|
+
if (documentCount > 4294967295) loadFailed(`The head of ${what} holds a document count above uint32`);
|
|
95
|
+
return {
|
|
96
|
+
v: PAGED_SEGMENT_HEAD_VERSION,
|
|
97
|
+
schema: entry.schema,
|
|
98
|
+
tombstones: entry.tombstones,
|
|
99
|
+
documentCount,
|
|
100
|
+
fieldNames: entry.fieldNames,
|
|
101
|
+
sections: decodeSections(entry.sections, payloadLength, `head of ${what}`, SEGMENT_PAGE_SIZES)
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
function alignedUp(offset) {
|
|
105
|
+
const over = offset % SECTION_ALIGNMENT_BYTES;
|
|
106
|
+
return over === 0 ? offset : offset + SECTION_ALIGNMENT_BYTES - over;
|
|
107
|
+
}
|
|
108
|
+
function placeSections(planned, sectionsStart) {
|
|
109
|
+
let cursor = sectionsStart;
|
|
110
|
+
return planned.map((section) => {
|
|
111
|
+
const offset = alignedUp(cursor);
|
|
112
|
+
cursor = offset + section.length;
|
|
113
|
+
return {
|
|
114
|
+
name: section.name,
|
|
115
|
+
offset,
|
|
116
|
+
length: section.length,
|
|
117
|
+
pageBytes: SEGMENT_PAGE_BYTES,
|
|
118
|
+
checksums: section.checksums
|
|
119
|
+
};
|
|
120
|
+
});
|
|
121
|
+
}
|
|
122
|
+
function encodePagedSegmentHead(fields, planned) {
|
|
123
|
+
let headLength = 0;
|
|
124
|
+
for (; ; ) {
|
|
125
|
+
const sectionsStart = alignedUp(PAYLOAD_PREFIX_BYTES + headLength);
|
|
126
|
+
const sections = placeSections(planned, sectionsStart);
|
|
127
|
+
const head = encode({ v: PAGED_SEGMENT_HEAD_VERSION, ...fields, sections });
|
|
128
|
+
if (head.byteLength !== headLength) {
|
|
129
|
+
headLength = head.byteLength;
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
const prefix = new Uint8Array(PAYLOAD_PREFIX_BYTES);
|
|
133
|
+
const view = new DataView(prefix.buffer);
|
|
134
|
+
view.setUint32(0, head.byteLength, true);
|
|
135
|
+
view.setUint32(4, crc32(head), true);
|
|
136
|
+
const last = sections[sections.length - 1];
|
|
137
|
+
const payloadLength = last === void 0 ? sectionsStart : last.offset + last.length;
|
|
138
|
+
return { head, prefix, sections, payloadLength };
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
function sectionNamed(sections, name) {
|
|
142
|
+
for (const section of sections) {
|
|
143
|
+
if (section.name === name) return section;
|
|
144
|
+
}
|
|
145
|
+
return void 0;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export { BLOCK_SUMMARY_BYTES, DOCUMENT_BLOCK_BYTES, DOCUMENT_BLOCK_MIN_RAW_BYTES, ID_SAMPLE_INTERVAL, MAX_DOCUMENT_BLOCK_RAW_BYTES, PAGED_SEGMENT_ENVELOPE_VERSION, PAYLOAD_PREFIX_BYTES, POSITION_WIDTH_BYTES, POSTING_WIDTH_BYTES, RECORD_PAGE_BYTES, SECTION_ALIGNMENT_BYTES, SEGMENT_PAGE_BYTES, SMALLEST_BLOCK_POSTINGS, decodePagedSegmentHead, decodeSections, encodePagedSegmentHead, loadFailed, sectionNamed };
|