@delali/narsil 0.3.1-next.140 → 0.3.1-next.143
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{analysis-change-JN23AZQP.mjs → analysis-change-HQUT2Z3I.mjs} +13 -13
- package/dist/{analysis-change-entry-5JSIMEKG.mjs → analysis-change-entry-TZTG4AZB.mjs} +1 -1
- package/dist/browser/{analysis-change-57MIRJWR.mjs → analysis-change-JPV775LR.mjs} +407 -6
- package/dist/browser/{analysis-change-entry-ICGRR3IN.mjs → analysis-change-entry-7HR4LWL3.mjs} +1 -1
- package/dist/browser/chunk-3ZBJYAS4.mjs +19 -0
- package/dist/browser/{chunk-ZAIZQOTM.mjs → chunk-5DB2VDGI.mjs} +951 -1134
- package/dist/browser/chunk-6RHTAVZX.mjs +656 -0
- package/dist/browser/chunk-CCOAOO73.mjs +86 -0
- package/dist/browser/{chunk-3EYDURFD.mjs → chunk-HKX2G6WH.mjs} +1615 -2266
- package/dist/browser/{chunk-CGVBGHBX.mjs → chunk-KTOK275N.mjs} +531 -830
- package/dist/browser/{chunk-5LYWZJE2.mjs → chunk-LVY2LI35.mjs} +1 -1
- package/dist/browser/chunk-MVM33ZUL.mjs +993 -0
- package/dist/browser/chunk-MWUK3DA5.mjs +2634 -0
- package/dist/browser/{chunk-67KSBUYR.mjs → chunk-R4WUDZQV.mjs} +8 -5
- package/dist/browser/{chunk-6BSB4N6B.mjs → chunk-TCLEOZ4P.mjs} +2 -2
- package/dist/browser/chunk-VFYMPCOV.mjs +212 -0
- package/dist/browser/chunk-WCFCHIBB.mjs +177 -0
- package/dist/browser/{chunk-E5SFTOQR.mjs → chunk-XNWPP5NS.mjs} +2 -2
- package/dist/browser/{chunk-QHNNULPW.mjs → chunk-ZEV26AQ6.mjs} +5 -0
- package/dist/browser/cold-module-MJMKNUYJ.mjs +745 -0
- package/dist/browser/{durability-wiring-MTIAN6VT.mjs → durability-wiring-SNOWIT6Z.mjs} +231 -51
- package/dist/browser/{import-Q7MXD6BS.mjs → import-TLB2A5LJ.mjs} +6 -5
- package/dist/browser/module-BUFNZMEU.mjs +12 -0
- package/dist/browser/{module-5J3PEUEB.mjs → module-BVP4TKZU.mjs} +395 -108
- package/dist/{restore-queue-RYSRRFS4.mjs → browser/restore-queue-ONYNYUDE.mjs} +2 -2
- package/dist/browser/{snapshot-YBJ5MF2M.mjs → snapshot-QXBKHJAG.mjs} +108 -25
- package/dist/browser/{write-RTWI5MXL.mjs → write-WHFBDWQS.mjs} +63 -91
- package/dist/{capabilities-DXAmysqJ.d.ts → capabilities-BCdUwK_L.d.ts} +2 -2
- package/dist/{chunk-HMDT3I63.mjs → chunk-46NZOPHS.mjs} +2 -2
- package/dist/{chunk-XXKWEG53.mjs → chunk-5WYVUPGD.mjs} +1 -1
- package/dist/chunk-6SNBQESJ.mjs +214 -0
- package/dist/chunk-7AUASRE7.mjs +653 -0
- package/dist/{chunk-4GCN34JV.mjs → chunk-AST3SHFA.mjs} +26 -24
- package/dist/chunk-BOJZGUS6.mjs +177 -0
- package/dist/{chunk-IP4HJOGC.mjs → chunk-BRUUBXCB.mjs} +4 -3
- package/dist/{chunk-J3EHATEZ.mjs → chunk-DQNKVOJ2.mjs} +63 -63
- package/dist/chunk-FOCV4CBZ.mjs +87 -0
- package/dist/{chunk-FFVLPCLH.mjs → chunk-GLZHNSGB.mjs} +1 -1
- package/dist/chunk-GWOS5DYC.mjs +2637 -0
- package/dist/{chunk-W2XDK66Z.mjs → chunk-IO5F7ZBE.mjs} +112 -438
- package/dist/{chunk-P36MUFEJ.mjs → chunk-JDDXV6UF.mjs} +2 -1
- package/dist/{chunk-LLDFIKGP.mjs → chunk-KEIYVZUX.mjs} +4 -4
- package/dist/{chunk-UBXJQJII.mjs → chunk-KGOXHLLZ.mjs} +5 -1
- package/dist/{chunk-FKARMUWH.mjs → chunk-KZL72VTC.mjs} +68 -16
- package/dist/{chunk-TCWQJL7O.mjs → chunk-L76W3IZJ.mjs} +5 -0
- package/dist/{chunk-CPCGEIOM.mjs → chunk-LA4WNDHL.mjs} +83 -799
- package/dist/{chunk-PNNVTKMT.mjs → chunk-NJJCPFXN.mjs} +2 -2
- package/dist/chunk-NN33CZRE.mjs +86 -0
- package/dist/chunk-NRUXBYPH.mjs +121 -0
- package/dist/{chunk-UJWGQIXU.mjs → chunk-QHPYS427.mjs} +1 -0
- package/dist/{chunk-VSUYTPME.mjs → chunk-RHKRTYGO.mjs} +13 -4
- package/dist/{chunk-UEV53BBO.mjs → chunk-SRJBFG2T.mjs} +406 -5
- package/dist/{chunk-K3AQOZVM.mjs → chunk-TNBIK2LQ.mjs} +1638 -1644
- package/dist/chunk-UUZCQQG2.mjs +209 -0
- package/dist/{chunk-XMQNOD5V.mjs → chunk-VEFN6X6Q.mjs} +2 -2
- package/dist/{chunk-LK3MOTXF.mjs → chunk-VJEGOELZ.mjs} +8 -5
- package/dist/chunk-W2COJ5AH.mjs +148 -0
- package/dist/{chunk-IBATGS3J.mjs → chunk-WJGO5PGV.mjs} +241 -124
- package/dist/{chunk-CCCGEPJM.mjs → chunk-XZ3SRNDJ.mjs} +24 -63
- package/dist/{chunk-MTQTX4BP.mjs → chunk-YHUTF6EA.mjs} +400 -92
- package/dist/{chunk-B5I3IL4H.mjs → chunk-YLNSJQLX.mjs} +4 -4
- package/dist/{chunk-6VHWXAOI.mjs → chunk-Z4HFMYNX.mjs} +29 -24
- package/dist/{chunk-U4X4AWKL.mjs → chunk-ZPWS2SKJ.mjs} +293 -270
- package/dist/{client-DApBeLbk.d.ts → client-B12YFALP.d.ts} +3 -3
- package/dist/client.d.ts +7 -7
- package/dist/cold-module-4X7WJCFM.mjs +783 -0
- package/dist/{config-B0H-vp4U.d.ts → config-BhZxIeb1.d.ts} +6 -1
- package/dist/{config-Brysl9NO.d.ts → config-CIG5yDyg.d.ts} +2 -2
- package/dist/{direct-executor-J5UDLPL3.mjs → direct-executor-7XOQ7VQP.mjs} +5 -5
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +123 -42
- package/dist/{durability-wiring-AYOTAPNQ.mjs → durability-wiring-324D2DDD.mjs} +213 -40
- package/dist/{engine-CuLMbH1-.d.ts → engine-CtzbHOmG.d.ts} +2 -2
- package/dist/{import-QRZAQ5RH.mjs → import-OOBUW3RR.mjs} +14 -13
- package/dist/{index-HYnqS1SE.d.ts → index-DsEurhSl.d.ts} +1 -1
- package/dist/index.browser.mjs +3 -3
- package/dist/index.d.ts +6 -6
- package/dist/index.mjs +11 -11
- package/dist/{long-query-thread-U7K5A72G.mjs → long-query-thread-4SW7RWDE.mjs} +8 -8
- package/dist/{module-L2OZO5PQ.mjs → module-DXSJ5LT2.mjs} +374 -83
- package/dist/module-IYPLWHGC.mjs +48 -0
- package/dist/paged-partition-archive-HFP45PLU.mjs +269 -0
- package/dist/partial-words.mjs +2 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +5 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +9 -9
- package/dist/{browser/restore-queue-DISFECE4.mjs → restore-queue-FEYAC4SB.mjs} +2 -2
- package/dist/{schema-BTXbHCqA.d.ts → schema-B-Runv0n.d.ts} +7 -0
- package/dist/server.d.ts +10 -10
- package/dist/server.mjs +8 -7
- package/dist/{snapshot-BTRZ3MZK.mjs → snapshot-CGM65VCD.mjs} +60 -25
- package/dist/{tasks-DjGjK70Y.d.ts → tasks-CjRJRGTb.d.ts} +6 -4
- package/dist/{thread-USB3GOPY.mjs → thread-KGNAIKPD.mjs} +8 -8
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +3 -2
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-GHTA56OW.mjs → write-2DZPEQL2.mjs} +70 -98
- package/openapi.json +13 -0
- package/package.json +12 -8
- package/dist/browser/chunk-XVMOQQ34.mjs +0 -384
- package/dist/chunk-BDIDUW2B.mjs +0 -387
- package/dist/browser/{chunk-4MFH226Q.mjs → chunk-D33A5VMU.mjs} +15 -15
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-
|
|
2
|
-
import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-
|
|
1
|
+
import { I as IndexInfo, P as PartitionStatsResult, V as VectorMaintenanceResult, M as MemoryStats } from './config-BhZxIeb1.js';
|
|
2
|
+
import { A as AnyDocument, I as InsertOptions, B as BM25Params, W as WriteOptions } from './schema-B-Runv0n.js';
|
|
3
3
|
import { a as SynonymRule } from './synonyms-CpOAHPa-.js';
|
|
4
|
-
import { N as NodeRole, c as NodeMemoryReport } from './index-
|
|
5
|
-
import { a as ClusterNodeInfo, N as NodeReadiness } from './config-
|
|
4
|
+
import { N as NodeRole, c as NodeMemoryReport } from './index-DsEurhSl.js';
|
|
5
|
+
import { a as ClusterNodeInfo, N as NodeReadiness } from './config-CIG5yDyg.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* The body of `POST /indexes`, which the server sends once the index exists.
|
|
@@ -479,6 +479,8 @@ interface HttpIndexConfig {
|
|
|
479
479
|
stopWords?: string | string[];
|
|
480
480
|
/** Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it. */
|
|
481
481
|
trackPositions?: boolean;
|
|
482
|
+
/** Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`. */
|
|
483
|
+
textStorage?: 'memory' | 'disk';
|
|
482
484
|
/** Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed. */
|
|
483
485
|
surfaceForms?: boolean;
|
|
484
486
|
/** These settings control when vector fields move to an HNSW graph, and how that graph is built. */
|
|
@@ -1,22 +1,22 @@
|
|
|
1
|
-
export { closeRequestThread, searchHandler, serveRequests } from './chunk-
|
|
2
|
-
import './chunk-SBMAWTRG.mjs';
|
|
1
|
+
export { closeRequestThread, searchHandler, serveRequests } from './chunk-YLNSJQLX.mjs';
|
|
3
2
|
import './chunk-KZ3NX2P7.mjs';
|
|
3
|
+
import './chunk-SBMAWTRG.mjs';
|
|
4
4
|
import './chunk-DD2673TX.mjs';
|
|
5
5
|
import './chunk-IWVEG5CX.mjs';
|
|
6
6
|
import './chunk-DJHHF2L5.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import './chunk-U4X4AWKL.mjs';
|
|
9
|
-
import './chunk-ZATRDAYU.mjs';
|
|
10
|
-
import './chunk-EBIHRL7B.mjs';
|
|
7
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
11
8
|
import './chunk-HJSN7UYG.mjs';
|
|
12
9
|
import './chunk-EEBCDP36.mjs';
|
|
10
|
+
import './chunk-ZPWS2SKJ.mjs';
|
|
11
|
+
import './chunk-ZATRDAYU.mjs';
|
|
12
|
+
import './chunk-EBIHRL7B.mjs';
|
|
13
13
|
import './chunk-QSK35UC2.mjs';
|
|
14
14
|
import './chunk-74XB2SXI.mjs';
|
|
15
15
|
import './chunk-RKAVHNRW.mjs';
|
|
16
16
|
import './chunk-WQBCBKY3.mjs';
|
|
17
17
|
import './chunk-JUKYYXSP.mjs';
|
|
18
|
-
import './chunk-
|
|
19
|
-
import './chunk-
|
|
18
|
+
import './chunk-DQNKVOJ2.mjs';
|
|
19
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
20
20
|
import './chunk-U476IXKQ.mjs';
|
|
21
21
|
import './chunk-FQBFZAWG.mjs';
|
|
22
22
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
import { openSharedVectorField, createVectorStore } from '../chunk-
|
|
2
|
-
import
|
|
1
|
+
import { openSharedVectorField, createVectorStore } from '../chunk-XZ3SRNDJ.mjs';
|
|
2
|
+
import '../chunk-FOCV4CBZ.mjs';
|
|
3
|
+
import { osqBitsOf, createHNSWIndex, createOsqQuantizer } from '../chunk-TNBIK2LQ.mjs';
|
|
3
4
|
import { osqRecordBytes } from '../chunk-37CD3OKS.mjs';
|
|
4
5
|
import '../chunk-DJHHF2L5.mjs';
|
|
5
6
|
import '../chunk-KOKM4PQN.mjs';
|
package/dist/workers/entry.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { startIdleHeapCleanup } from '../chunk-RIIUZJQ3.mjs';
|
|
2
|
-
import { isValidWorkerAction } from '../chunk-
|
|
3
|
-
import { loadVectorCode } from '../chunk-
|
|
2
|
+
import { isValidWorkerAction } from '../chunk-QHPYS427.mjs';
|
|
3
|
+
import { loadVectorCode } from '../chunk-GLZHNSGB.mjs';
|
|
4
4
|
import '../chunk-22YPZVMG.mjs';
|
|
5
5
|
import { threadSlotOfWorker } from '../chunk-RFBLQLLZ.mjs';
|
|
6
6
|
import '../chunk-L5ORED7V.mjs';
|
|
@@ -22,9 +22,9 @@ async function importBootstrapModule(moduleUrl) {
|
|
|
22
22
|
await import(moduleUrl);
|
|
23
23
|
}
|
|
24
24
|
async function stopServingRequests() {
|
|
25
|
-
const { closeRequestThread } = await import('../thread-
|
|
25
|
+
const { closeRequestThread } = await import('../thread-KGNAIKPD.mjs');
|
|
26
26
|
closeRequestThread();
|
|
27
|
-
const { closeLongQueryThread } = await import('../long-query-thread-
|
|
27
|
+
const { closeLongQueryThread } = await import('../long-query-thread-4SW7RWDE.mjs');
|
|
28
28
|
closeLongQueryThread();
|
|
29
29
|
}
|
|
30
30
|
function holdsCopies(executor) {
|
|
@@ -55,7 +55,7 @@ function createActionHandler(executor) {
|
|
|
55
55
|
if (!holdsCopies(executor)) {
|
|
56
56
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer requests from");
|
|
57
57
|
}
|
|
58
|
-
const starting = import('../thread-
|
|
58
|
+
const starting = import('../thread-KGNAIKPD.mjs').then(
|
|
59
59
|
({ serveRequests }) => serveRequests(executor, action.settings)
|
|
60
60
|
);
|
|
61
61
|
serving.start = starting.then(
|
|
@@ -72,7 +72,7 @@ function createActionHandler(executor) {
|
|
|
72
72
|
if (!holdsCopies(executor)) {
|
|
73
73
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "This executor holds no copies to answer long queries from");
|
|
74
74
|
}
|
|
75
|
-
const { serveLongQueries } = await import('../long-query-thread-
|
|
75
|
+
const { serveLongQueries } = await import('../long-query-thread-4SW7RWDE.mjs');
|
|
76
76
|
serveLongQueries(executor, action.settings);
|
|
77
77
|
servingRequests = true;
|
|
78
78
|
post(buildSuccessResponse(action.requestId, void 0));
|
|
@@ -118,7 +118,7 @@ async function setup() {
|
|
|
118
118
|
} catch {
|
|
119
119
|
parentPort = null;
|
|
120
120
|
}
|
|
121
|
-
const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-
|
|
121
|
+
const [{ createDirectExecutor }] = await Promise.all([import('../direct-executor-7XOQ7VQP.mjs'), loadVectorCode()]);
|
|
122
122
|
const handleAction = createActionHandler(createDirectExecutor({ threadSlot }));
|
|
123
123
|
if (parentPort) {
|
|
124
124
|
const port = parentPort;
|
|
@@ -1,29 +1,31 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { DOCUMENT_EXPORT_VERSION, DOCUMENT_EXPORT_FORMAT } from './chunk-6SNBQESJ.mjs';
|
|
2
|
+
import { encodeExactJson, encodeDocumentJson } from './chunk-NN33CZRE.mjs';
|
|
3
|
+
import './chunk-BOJZGUS6.mjs';
|
|
2
4
|
import './chunk-CZQ33DLW.mjs';
|
|
3
|
-
import './chunk-
|
|
5
|
+
import './chunk-L76W3IZJ.mjs';
|
|
4
6
|
import { decodeVectorFilePayload, isDead, bytesToVectors } from './chunk-ICW45TTU.mjs';
|
|
5
7
|
import './chunk-37CD3OKS.mjs';
|
|
6
|
-
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-
|
|
7
|
-
import { openChecksummedEnvelope } from './chunk-
|
|
8
|
-
import './chunk-
|
|
9
|
-
import './chunk-
|
|
8
|
+
import { setNestedValue, KEYWORD_SEGMENT_ENVELOPE_VERSION, decodeKeywordSegment } from './chunk-KEIYVZUX.mjs';
|
|
9
|
+
import { openChecksummedEnvelope } from './chunk-IO5F7ZBE.mjs';
|
|
10
|
+
import './chunk-VEFN6X6Q.mjs';
|
|
11
|
+
import './chunk-5WYVUPGD.mjs';
|
|
12
|
+
import './chunk-HJSN7UYG.mjs';
|
|
10
13
|
import './chunk-UIKN2OTQ.mjs';
|
|
11
14
|
import './chunk-AV4IJKD2.mjs';
|
|
12
|
-
import {
|
|
13
|
-
import './chunk-
|
|
15
|
+
import { EXPORT_CHUNK_BYTES } from './chunk-EEBCDP36.mjs';
|
|
16
|
+
import { decodeStoredDocument } from './chunk-WJGO5PGV.mjs';
|
|
17
|
+
import './chunk-ZPWS2SKJ.mjs';
|
|
14
18
|
import './chunk-N37VMC62.mjs';
|
|
15
19
|
import './chunk-ZATRDAYU.mjs';
|
|
16
20
|
import './chunk-EBIHRL7B.mjs';
|
|
17
|
-
import './chunk-HJSN7UYG.mjs';
|
|
18
|
-
import { EXPORT_CHUNK_BYTES } from './chunk-EEBCDP36.mjs';
|
|
19
21
|
import './chunk-KOKM4PQN.mjs';
|
|
20
22
|
import './chunk-QSK35UC2.mjs';
|
|
21
23
|
import './chunk-74XB2SXI.mjs';
|
|
22
24
|
import './chunk-RKAVHNRW.mjs';
|
|
23
25
|
import './chunk-WQBCBKY3.mjs';
|
|
24
26
|
import './chunk-JUKYYXSP.mjs';
|
|
25
|
-
import './chunk-
|
|
26
|
-
import './chunk-
|
|
27
|
+
import './chunk-DQNKVOJ2.mjs';
|
|
28
|
+
import './chunk-KGOXHLLZ.mjs';
|
|
27
29
|
import './chunk-U476IXKQ.mjs';
|
|
28
30
|
import './chunk-FQBFZAWG.mjs';
|
|
29
31
|
import './chunk-VJS4JFBQ.mjs';
|
|
@@ -38,99 +40,17 @@ import './chunk-NPNK57CA.mjs';
|
|
|
38
40
|
import './chunk-4UMZ7KQL.mjs';
|
|
39
41
|
import './chunk-R6A773WK.mjs';
|
|
40
42
|
import './chunk-YTJCGKM4.mjs';
|
|
41
|
-
import './chunk-
|
|
42
|
-
import './chunk-
|
|
43
|
+
import './chunk-QHPYS427.mjs';
|
|
44
|
+
import './chunk-GLZHNSGB.mjs';
|
|
43
45
|
import './chunk-22YPZVMG.mjs';
|
|
44
46
|
import './chunk-RFBLQLLZ.mjs';
|
|
45
47
|
import './chunk-L5ORED7V.mjs';
|
|
46
|
-
import
|
|
48
|
+
import './chunk-HNHCNTVG.mjs';
|
|
47
49
|
import { decode } from '@msgpack/msgpack';
|
|
48
50
|
|
|
49
|
-
// src/engine/export/json-line.ts
|
|
50
|
-
function refuse(path, kind, docId) {
|
|
51
|
-
throw new NarsilError(
|
|
52
|
-
ErrorCodes.DOC_VALIDATION_FAILED,
|
|
53
|
-
`Document "${docId}" holds ${kind} at "${path || "(document)"}", which a document export cannot carry as JSON`,
|
|
54
|
-
{ docId, field: path }
|
|
55
|
-
);
|
|
56
|
-
}
|
|
57
|
-
function encodeNumber(value, path, target) {
|
|
58
|
-
if (Number.isFinite(value)) return Object.is(value, -0) ? "-0" : JSON.stringify(value);
|
|
59
|
-
if (!target.tagged) refuse(path, String(value), target.docId);
|
|
60
|
-
return `{"${NUMBER_TAG}":"${String(value)}"}`;
|
|
61
|
-
}
|
|
62
|
-
function encodeDate(value, path, target) {
|
|
63
|
-
const milliseconds = value.getTime();
|
|
64
|
-
if (!target.tagged || Number.isNaN(milliseconds)) refuse(path, "a date", target.docId);
|
|
65
|
-
const seconds = Math.floor(milliseconds / MILLISECONDS_PER_SECOND);
|
|
66
|
-
const nanoseconds = (milliseconds - seconds * MILLISECONDS_PER_SECOND) * NANOSECONDS_PER_MILLISECOND;
|
|
67
|
-
return `{"${DATE_TAG}":[${seconds},${nanoseconds}]}`;
|
|
68
|
-
}
|
|
69
|
-
function encodeBytes(value, path, target) {
|
|
70
|
-
if (!target.tagged) refuse(path, "bytes", target.docId);
|
|
71
|
-
return `{"${BYTES_TAG}":"${encodeBase64(value)}"}`;
|
|
72
|
-
}
|
|
73
|
-
function encodeList(value, path, target) {
|
|
74
|
-
target.parts.push("[");
|
|
75
|
-
for (let at = 0; at < value.length; at++) {
|
|
76
|
-
if (at > 0) target.parts.push(",");
|
|
77
|
-
encodeValue(value[at], `${path}[${at}]`, target);
|
|
78
|
-
}
|
|
79
|
-
target.parts.push("]");
|
|
80
|
-
}
|
|
81
|
-
function encodeMap(value, path, target) {
|
|
82
|
-
const proto = Object.getPrototypeOf(value);
|
|
83
|
-
if (proto !== Object.prototype && proto !== null) refuse(path, "a value other than a plain object", target.docId);
|
|
84
|
-
const entries = Object.entries(value).filter(([, entry]) => entry !== void 0);
|
|
85
|
-
const wrapped = target.tagged && entries.length === 1 && startsWithTagPrefix(entries[0][0]);
|
|
86
|
-
target.parts.push(wrapped ? `{"${OBJECT_TAG}":{` : "{");
|
|
87
|
-
for (let at = 0; at < entries.length; at++) {
|
|
88
|
-
const [key, entry] = entries[at];
|
|
89
|
-
if (at > 0) target.parts.push(",");
|
|
90
|
-
target.parts.push(JSON.stringify(key), ":");
|
|
91
|
-
encodeValue(entry, path === "" ? key : `${path}.${key}`, target);
|
|
92
|
-
}
|
|
93
|
-
target.parts.push(wrapped ? "}}" : "}");
|
|
94
|
-
}
|
|
95
|
-
function encodeValue(value, path, target) {
|
|
96
|
-
if (value === null) {
|
|
97
|
-
target.parts.push("null");
|
|
98
|
-
return;
|
|
99
|
-
}
|
|
100
|
-
switch (typeof value) {
|
|
101
|
-
case "string":
|
|
102
|
-
target.parts.push(JSON.stringify(value));
|
|
103
|
-
return;
|
|
104
|
-
case "boolean":
|
|
105
|
-
target.parts.push(value ? "true" : "false");
|
|
106
|
-
return;
|
|
107
|
-
case "number":
|
|
108
|
-
target.parts.push(encodeNumber(value, path, target));
|
|
109
|
-
return;
|
|
110
|
-
case "object":
|
|
111
|
-
break;
|
|
112
|
-
default:
|
|
113
|
-
refuse(path, `a ${typeof value}`, target.docId);
|
|
114
|
-
}
|
|
115
|
-
if (Array.isArray(value)) encodeList(value, path, target);
|
|
116
|
-
else if (value instanceof Date) target.parts.push(encodeDate(value, path, target));
|
|
117
|
-
else if (value instanceof Uint8Array) target.parts.push(encodeBytes(value, path, target));
|
|
118
|
-
else encodeMap(value, path, target);
|
|
119
|
-
}
|
|
120
|
-
function encodeExactJson(value, docId) {
|
|
121
|
-
const target = { docId, tagged: false, parts: [] };
|
|
122
|
-
encodeValue(value, "", target);
|
|
123
|
-
return target.parts.join("");
|
|
124
|
-
}
|
|
125
|
-
function encodeDocumentJson(document, docId) {
|
|
126
|
-
const target = { docId, tagged: true, parts: [] };
|
|
127
|
-
encodeValue(document, "", target);
|
|
128
|
-
return target.parts.join("");
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// src/engine/export/write.ts
|
|
132
51
|
var VECTOR_FILE_ENVELOPE_VERSION = 2;
|
|
133
52
|
var FLOAT32_BYTES = 4;
|
|
53
|
+
var ENVELOPE_VERSION_OFFSET = 4;
|
|
134
54
|
async function locateVectors(checkpoint) {
|
|
135
55
|
const located = /* @__PURE__ */ new Map();
|
|
136
56
|
for (const field of checkpoint.manifest.vectors) {
|
|
@@ -178,7 +98,59 @@ async function winnersOf(checkpoint, partition) {
|
|
|
178
98
|
}
|
|
179
99
|
return winners;
|
|
180
100
|
}
|
|
101
|
+
async function wholeSegmentScan(checkpoint, ref) {
|
|
102
|
+
const contents = await readSegment(checkpoint, ref);
|
|
103
|
+
const table = contents.arrays.documentTable;
|
|
104
|
+
const ids = /* @__PURE__ */ new Set();
|
|
105
|
+
for (let ordinal = 0; ordinal < contents.arrays.documentCount; ordinal++) ids.add(idAt(contents, ordinal));
|
|
106
|
+
return {
|
|
107
|
+
count: contents.arrays.documentCount,
|
|
108
|
+
schema: contents.schema,
|
|
109
|
+
tombstones: new Set(contents.tombstones),
|
|
110
|
+
idAt: (ordinal) => idAt(contents, ordinal),
|
|
111
|
+
holds: (docId) => ids.has(docId),
|
|
112
|
+
storedDocumentAt: (ordinal) => decodeStoredDocument(
|
|
113
|
+
table.blob.subarray(table.offsets[ordinal], table.offsets[ordinal + 1]),
|
|
114
|
+
table.discardedFields
|
|
115
|
+
),
|
|
116
|
+
close: () => void 0
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
async function wholeFileVectorReads(checkpoint) {
|
|
120
|
+
const located = await locateVectors(checkpoint);
|
|
121
|
+
return {
|
|
122
|
+
async vectorOf(fieldPath, docId) {
|
|
123
|
+
const location = located.get(fieldPath)?.get(docId);
|
|
124
|
+
return location === void 0 ? null : readVector(checkpoint, location);
|
|
125
|
+
},
|
|
126
|
+
close: () => located.clear()
|
|
127
|
+
};
|
|
128
|
+
}
|
|
129
|
+
async function* pagedDocumentLines(checkpoint) {
|
|
130
|
+
const { openFileVectorReads, pagedPartitionLines } = await import('./module-IYPLWHGC.mjs');
|
|
131
|
+
const vectors = await openFileVectorReads(checkpoint, () => wholeFileVectorReads(checkpoint));
|
|
132
|
+
try {
|
|
133
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
134
|
+
yield* pagedPartitionLines(checkpoint, partition, vectors, (ref) => wholeSegmentScan(checkpoint, ref));
|
|
135
|
+
}
|
|
136
|
+
} finally {
|
|
137
|
+
vectors.close();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
async function anyPartitionIsPaged(checkpoint) {
|
|
141
|
+
for (const partition of checkpoint.manifest.partitions) {
|
|
142
|
+
for (const ref of partition.segments) {
|
|
143
|
+
const [version] = await checkpoint.readRange(ref, ENVELOPE_VERSION_OFFSET, 1);
|
|
144
|
+
if (version !== KEYWORD_SEGMENT_ENVELOPE_VERSION) return true;
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return false;
|
|
148
|
+
}
|
|
181
149
|
async function* documentLines(checkpoint) {
|
|
150
|
+
if (await anyPartitionIsPaged(checkpoint)) {
|
|
151
|
+
yield* pagedDocumentLines(checkpoint);
|
|
152
|
+
return;
|
|
153
|
+
}
|
|
182
154
|
const vectors = await locateVectors(checkpoint);
|
|
183
155
|
for (const partition of checkpoint.manifest.partitions) {
|
|
184
156
|
const winners = await winnersOf(checkpoint, partition);
|
package/openapi.json
CHANGED
|
@@ -3228,6 +3228,14 @@
|
|
|
3228
3228
|
"type": "boolean",
|
|
3229
3229
|
"description": "Setting this records term positions in each posting, which the `.nrsl` format carries for readers that match phrases. Highlighting works without it."
|
|
3230
3230
|
},
|
|
3231
|
+
"textStorage": {
|
|
3232
|
+
"type": "string",
|
|
3233
|
+
"enum": [
|
|
3234
|
+
"memory",
|
|
3235
|
+
"disk"
|
|
3236
|
+
],
|
|
3237
|
+
"description": "Setting this to `disk` lets the index answer from segment files that the server reads by position, which needs filesystem durability. The default is `memory`."
|
|
3238
|
+
},
|
|
3231
3239
|
"surfaceForms": {
|
|
3232
3240
|
"type": "boolean",
|
|
3233
3241
|
"description": "Setting this to false returns index stems from suggestions and prefix expansion, instead of the words users typed."
|
|
@@ -5142,6 +5150,10 @@
|
|
|
5142
5150
|
"type": "boolean",
|
|
5143
5151
|
"description": "This turns true while a rebuild that keeps recall up waits for the node to have room for a second graph beside the one it serves, and false once it starts."
|
|
5144
5152
|
},
|
|
5153
|
+
"loadingFromFiles": {
|
|
5154
|
+
"type": "boolean",
|
|
5155
|
+
"description": "This stays true after a restart while a field kept on disk loads its codes and graph in the background, during which a search reads them from the field's files."
|
|
5156
|
+
},
|
|
5145
5157
|
"estimatedCompactMs": {
|
|
5146
5158
|
"type": "number",
|
|
5147
5159
|
"description": "Narsil.compactVectors would take about this many milliseconds."
|
|
@@ -5158,6 +5170,7 @@
|
|
|
5158
5170
|
"bufferSize",
|
|
5159
5171
|
"building",
|
|
5160
5172
|
"waitingForMemory",
|
|
5173
|
+
"loadingFromFiles",
|
|
5161
5174
|
"estimatedCompactMs",
|
|
5162
5175
|
"estimatedOptimizeMs"
|
|
5163
5176
|
],
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@delali/narsil",
|
|
3
|
-
"version": "0.3.1-next.
|
|
3
|
+
"version": "0.3.1-next.143",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Distributed search engine with full-text, vector, hybrid, and geosearch. One engine, embedded or on a server, with a cross-language .nrsl format.",
|
|
6
6
|
"author": "Delali (https://sondelali.com)",
|
|
@@ -51,6 +51,10 @@
|
|
|
51
51
|
"browser": "./src/vector/vector-store/vector-file.browser.ts",
|
|
52
52
|
"default": "./src/vector/vector-store/vector-file.ts"
|
|
53
53
|
},
|
|
54
|
+
"#platform/raw-deflate": {
|
|
55
|
+
"browser": "./src/persistence/paged/raw-deflate.browser.ts",
|
|
56
|
+
"default": "./src/persistence/paged/raw-deflate.ts"
|
|
57
|
+
},
|
|
54
58
|
"#platform/native-core": {
|
|
55
59
|
"browser": "./src/vector/native/loader.browser.ts",
|
|
56
60
|
"default": "./src/vector/native/loader.ts"
|
|
@@ -90,7 +94,7 @@
|
|
|
90
94
|
"node": ">=22"
|
|
91
95
|
},
|
|
92
96
|
"scripts": {
|
|
93
|
-
"build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && tsx scripts/api-surface.ts",
|
|
97
|
+
"build": "rm -rf dist && tsup && node scripts/check-browser-bundle.mjs && node scripts/check-vector-chunk.mjs && tsx scripts/api-surface.ts",
|
|
94
98
|
"build:code": "rm -rf dist && NARSIL_BUILD_CODE_ONLY=1 tsup",
|
|
95
99
|
"test": "vitest run --exclude='**/vector/recall.test.ts' --exclude='**/e2e/cluster-memory/**' --exclude='**/e2e/cluster-faults/**' --exclude='**/frozen-removal-memory/container-removes.test.ts'",
|
|
96
100
|
"test:native": "vitest run src/__tests__/vector/native src/__tests__/integration/filtered-vector-recall/native src/__tests__/integration/filtered-vectors/mutations-native src/__tests__/integration/removed-vectors/main-thread",
|
|
@@ -279,11 +283,11 @@
|
|
|
279
283
|
}
|
|
280
284
|
},
|
|
281
285
|
"optionalDependencies": {
|
|
282
|
-
"@delali/narsil-native-darwin-arm64": "0.3.1-next.
|
|
283
|
-
"@delali/narsil-native-darwin-x64": "0.3.1-next.
|
|
284
|
-
"@delali/narsil-native-linux-arm64": "0.3.1-next.
|
|
285
|
-
"@delali/narsil-native-linux-x64": "0.3.1-next.
|
|
286
|
-
"@delali/narsil-native-win32-arm64": "0.3.1-next.
|
|
287
|
-
"@delali/narsil-native-win32-x64": "0.3.1-next.
|
|
286
|
+
"@delali/narsil-native-darwin-arm64": "0.3.1-next.143",
|
|
287
|
+
"@delali/narsil-native-darwin-x64": "0.3.1-next.143",
|
|
288
|
+
"@delali/narsil-native-linux-arm64": "0.3.1-next.143",
|
|
289
|
+
"@delali/narsil-native-linux-x64": "0.3.1-next.143",
|
|
290
|
+
"@delali/narsil-native-win32-arm64": "0.3.1-next.143",
|
|
291
|
+
"@delali/narsil-native-win32-x64": "0.3.1-next.143"
|
|
288
292
|
}
|
|
289
293
|
}
|