@delali/narsil 0.3.0 → 0.3.1-next.101
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +9 -8
- package/dist/adapters/filesystem.d.ts +1 -1
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.d.ts +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/adapters/memory.d.ts +1 -1
- package/dist/{adapters-79OIa4xm.d.ts → adapters-BQFQ4PRz.d.ts} +27 -1
- package/dist/browser/chunk-BDOSMEEP.mjs +493 -0
- package/dist/browser/chunk-EJDUFWSO.mjs +602 -0
- package/dist/browser/chunk-HOZW3PKP.mjs +9 -0
- package/dist/browser/chunk-JOVZ6HMT.mjs +4358 -0
- package/dist/browser/chunk-JPTQZWJQ.mjs +10 -0
- package/dist/browser/chunk-KPJMDPHG.mjs +24558 -0
- package/dist/browser/chunk-PGIM3HIM.mjs +2743 -0
- package/dist/browser/chunk-RHG4O77A.mjs +206 -0
- package/dist/browser/chunk-RIJSLY2I.mjs +360 -0
- package/dist/browser/durability-wiring-SRUYT3VV.mjs +3875 -0
- package/dist/browser/module-CBB2ETYP.mjs +3111 -0
- package/dist/browser/release-76MXRPGV.mjs +33 -0
- package/dist/browser/replace-RURXTGVG.mjs +57 -0
- package/dist/browser/restore-queue-EEZQO6RF.mjs +35 -0
- package/dist/browser/runtime-7ZAHCHWR.mjs +557 -0
- package/dist/browser/snapshot-OAAA5UJ7.mjs +1210 -0
- package/dist/browser/write-7D6VIK6O.mjs +208 -0
- package/dist/{capabilities-DSsOGz6F.d.ts → capabilities-DiGlB6Sr.d.ts} +68 -7
- package/dist/chunk-2RIRGOUL.mjs +113 -0
- package/dist/chunk-37CD3OKS.mjs +87 -0
- package/dist/chunk-44W3CBNG.mjs +7 -0
- package/dist/chunk-46D6JQIG.mjs +426 -0
- package/dist/{chunk-VMRR3DQU.mjs → chunk-5JNA5SKS.mjs} +546 -35
- package/dist/chunk-747OHOUA.mjs +11 -0
- package/dist/chunk-75XIEF6H.mjs +126 -0
- package/dist/chunk-7X5JST5L.mjs +114 -0
- package/dist/chunk-BOFBLJFZ.mjs +209 -0
- package/dist/{chunk-USKFU7J5.mjs → chunk-BWJVNZPR.mjs} +8 -1
- package/dist/chunk-C7SLA6CM.mjs +19 -0
- package/dist/{chunk-POWOTUKJ.mjs → chunk-CQUMNIIG.mjs} +3 -6
- package/dist/chunk-CVXLLEQ3.mjs +1011 -0
- package/dist/chunk-DJHHF2L5.mjs +17 -0
- package/dist/chunk-E74YLAIG.mjs +24 -0
- package/dist/{chunk-UXNWQZST.mjs → chunk-EMRMGYRE.mjs} +236 -18
- package/dist/chunk-EWZLIYHV.mjs +57 -0
- package/dist/chunk-GBEEG5AN.mjs +35 -0
- package/dist/chunk-GES7U2PM.mjs +68 -0
- package/dist/chunk-HJSN7UYG.mjs +10 -0
- package/dist/chunk-HLPRHOSO.mjs +17 -0
- package/dist/chunk-HMFNFVP7.mjs +14 -0
- package/dist/chunk-J5QP52HA.mjs +1751 -0
- package/dist/{chunk-YNVIUGF5.mjs → chunk-JZVARCRZ.mjs} +3 -28
- package/dist/chunk-KOKM4PQN.mjs +52 -0
- package/dist/{chunk-B2TFROFX.mjs → chunk-KWTW6VTG.mjs} +465 -250
- package/dist/chunk-KZ3NX2P7.mjs +21 -0
- package/dist/chunk-L5ORED7V.mjs +25 -0
- package/dist/{chunk-THYRHHO2.mjs → chunk-LGAMKHA2.mjs} +2009 -2351
- package/dist/chunk-LKOB5COA.mjs +117 -0
- package/dist/chunk-LPQOEU5M.mjs +96 -0
- package/dist/{chunk-3LHGNRYE.mjs → chunk-LXWDQLTW.mjs} +7 -2
- package/dist/{chunk-L6TNPLAR.mjs → chunk-OEYI3KRB.mjs} +270 -108
- package/dist/chunk-OM362BRO.mjs +948 -0
- package/dist/chunk-OZDFA4WC.mjs +67 -0
- package/dist/chunk-P36MUFEJ.mjs +10 -0
- package/dist/chunk-PC2G76VW.mjs +2540 -0
- package/dist/{chunk-GCM3JFVA.mjs → chunk-PNEYIHA4.mjs} +11 -2
- package/dist/chunk-PNFKUCG3.mjs +1130 -0
- package/dist/chunk-Q6XWVUZX.mjs +44 -0
- package/dist/chunk-R27EMH5M.mjs +617 -0
- package/dist/{chunk-YMZCMH5M.mjs → chunk-R6A773WK.mjs} +19 -24
- package/dist/chunk-RF7PLG43.mjs +106 -0
- package/dist/chunk-RFBLQLLZ.mjs +16 -0
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-RFVUDM5E.mjs} +17 -2
- package/dist/chunk-S7FL6UAH.mjs +521 -0
- package/dist/chunk-S7GNDRPS.mjs +651 -0
- package/dist/chunk-SP3NM3ZG.mjs +3537 -0
- package/dist/chunk-U476IXKQ.mjs +10 -0
- package/dist/chunk-UA3AUGNV.mjs +462 -0
- package/dist/chunk-UTG5YNXV.mjs +507 -0
- package/dist/chunk-VI4R6W25.mjs +8484 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-24RXHMEL.mjs → chunk-WVCOCKBU.mjs} +3794 -7570
- package/dist/chunk-X5CV2E2B.mjs +1 -0
- package/dist/chunk-XADZPPZI.mjs +614 -0
- package/dist/chunk-YEEQHT4V.mjs +212 -0
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-YYQIODMF.mjs +275 -0
- package/dist/chunk-ZI34TVOA.mjs +144 -0
- package/dist/{chunk-OOSIDHA3.mjs → chunk-ZKWYWW4Y.mjs} +1 -1
- package/dist/chunk-ZM7KU4YW.mjs +35 -0
- package/dist/{client-BQK6Tf6A.d.ts → client-BOStAkc5.d.ts} +160 -37
- package/dist/client.d.ts +11 -10
- package/dist/client.mjs +209 -45
- package/dist/{config-DmXULPa-.d.ts → config-Cjvzbs_j.d.ts} +51 -3
- package/dist/config-DffQlmRq.d.ts +421 -0
- package/dist/direct-executor-ZD64PPEZ.mjs +24 -0
- package/dist/distribution/coordinator/etcd.d.ts +79 -18
- package/dist/distribution/coordinator/etcd.mjs +553 -109
- package/dist/distribution/coordinator/in-memory.d.ts +4 -3
- package/dist/distribution/coordinator/in-memory.mjs +22 -3
- package/dist/distribution/transport/grpc.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +79 -14
- package/dist/distribution/transport/in-memory.d.ts +2 -2
- package/dist/distribution/transport/in-memory.mjs +12 -7
- package/dist/distribution/transport/tcp.d.ts +2 -2
- package/dist/distribution/transport/tcp.mjs +50 -28
- package/dist/distribution.d.ts +144 -23
- package/dist/distribution.mjs +14479 -8173
- package/dist/durability-wiring-FEUBR5BT.mjs +3330 -0
- package/dist/embeddings/openai.d.ts +2 -2
- package/dist/embeddings/openai.mjs +22 -133
- package/dist/{engine-CIOyFozK.d.ts → engine-rC0vjjY-.d.ts} +165 -141
- package/dist/{memory-yFaVn2xZ.d.ts → events-DqOIg5xa.d.ts} +645 -136
- package/dist/{types-DAK7mnUZ.d.ts → index-Bkdt8nDw.d.ts} +182 -45
- package/dist/index.browser.mjs +4 -32180
- package/dist/index.d.ts +91 -20
- package/dist/index.mjs +33 -16
- package/dist/invalidation/broadcast-channel.d.ts +1 -1
- package/dist/invalidation/filesystem.d.ts +1 -1
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/invalidation/noop.d.ts +1 -1
- package/dist/module-3S65MVCM.mjs +2151 -0
- package/dist/partial-words-DTIHO8G4.d.ts +19 -0
- package/dist/partial-words.d.ts +22 -0
- package/dist/partial-words.mjs +17 -0
- package/dist/pattern-Cledccbl.d.ts +18 -0
- package/dist/pattern.d.ts +22 -0
- package/dist/pattern.mjs +12 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +119 -72
- package/dist/persistence/durability/checkpoint-worker.mjs +51 -18
- package/dist/react.d.ts +279 -183
- package/dist/react.mjs +201 -48
- package/dist/release-BH3DKW3T.mjs +33 -0
- package/dist/replace-CEO5KJJ3.mjs +57 -0
- package/dist/rerank-K7C4WZFG.mjs +60 -0
- package/dist/rerankers/http.d.ts +50 -0
- package/dist/rerankers/http.mjs +104 -0
- package/dist/restore-queue-KQHA7ZV2.mjs +35 -0
- package/dist/runtime-6Y4ZYXOG.mjs +177 -0
- package/dist/{schema-d5Afu03A.d.ts → schema-4W_7tk1f.d.ts} +81 -11
- package/dist/server.d.ts +13 -12
- package/dist/server.mjs +611 -80
- package/dist/snapshot-QGZPODM5.mjs +799 -0
- package/dist/synonyms-CpOAHPa-.d.ts +52 -0
- package/dist/synonyms.d.ts +22 -0
- package/dist/synonyms.mjs +12 -0
- package/dist/{tasks-CH2Dod8w.d.ts → tasks-CwPTz-tP.d.ts} +10 -3
- package/dist/thread-N3IUGUS3.mjs +697 -0
- package/dist/{types-BQDlIvfS.d.ts → types-2qoitTXg.d.ts} +1 -1
- package/dist/{types-D5OwhtsQ.d.ts → types-Cz5okLml.d.ts} +2 -2
- package/dist/vector/search-worker.d.ts +46 -4
- package/dist/vector/search-worker.mjs +21 -11
- package/dist/workers/entry.mjs +9 -6
- package/dist/write-U5RI7MAX.mjs +241 -0
- package/package.json +55 -9
- package/dist/chunk-3L4U6EJQ.mjs +0 -49
- package/dist/chunk-IFBKLBGB.mjs +0 -7
- package/dist/chunk-ORBLD3EL.mjs +0 -6
- package/dist/chunk-U7KYW4BA.mjs +0 -135
- package/dist/chunk-ZJ4CSAB2.mjs +0 -13
- package/dist/chunk-ZNZBXMP5.mjs +0 -2117
- package/dist/chunk-ZZT3EAUC.mjs +0 -11168
- package/dist/config-BhldZgaK.d.ts +0 -194
- package/dist/direct-executor-ALAVE7FC.mjs +0 -12
- package/dist/serialization/crc32-worker.d.ts +0 -20
- package/dist/serialization/crc32-worker.mjs +0 -62
- package/dist/simd-DW5AtJ6B.d.ts +0 -41
- package/dist/thread-DYHEQRVY.mjs +0 -402
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* An app passes an object of this type to {@link registerPartialWordSearch}
|
|
3
|
+
* to give the engine its partial-word search code. The engine uses that code
|
|
4
|
+
* to match each query word inside the longer words of a field whose type
|
|
5
|
+
* includes `partial`, whenever a query has {@link QueryParams.partialWords}
|
|
6
|
+
* set.
|
|
7
|
+
*
|
|
8
|
+
* A Node process loads this code by itself. A browser bundle contains it only
|
|
9
|
+
* when the app imports `partialWordSearch` from `@delali/narsil/partial-words`,
|
|
10
|
+
* so a bundler can leave it out of an app that has no `partial` field.
|
|
11
|
+
*
|
|
12
|
+
* @public
|
|
13
|
+
*/
|
|
14
|
+
interface PartialWordSearch {
|
|
15
|
+
/** The engine checks that this name is `'partialWords'` when an app registers the object. */
|
|
16
|
+
readonly name: 'partialWords';
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export type { PartialWordSearch as P };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { P as PartialWordSearch } from './partial-words-DTIHO8G4.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This object contains the partial-word search code. A browser app has to
|
|
5
|
+
* pass it to {@link registerPartialWordSearch} before the app creates or
|
|
6
|
+
* loads an index with a `partial` field.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { registerPartialWordSearch } from '@delali/narsil'
|
|
10
|
+
* import { partialWordSearch } from '@delali/narsil/partial-words'
|
|
11
|
+
*
|
|
12
|
+
* registerPartialWordSearch(partialWordSearch)
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* A Node process loads that code by itself, so only a browser app has to make
|
|
16
|
+
* this call.
|
|
17
|
+
*
|
|
18
|
+
* @public
|
|
19
|
+
*/
|
|
20
|
+
declare const partialWordSearch: PartialWordSearch;
|
|
21
|
+
|
|
22
|
+
export { PartialWordSearch, partialWordSearch };
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { partialWordSearchModule } from './chunk-OM362BRO.mjs';
|
|
2
|
+
import './chunk-OEYI3KRB.mjs';
|
|
3
|
+
import './chunk-U476IXKQ.mjs';
|
|
4
|
+
import './chunk-E74YLAIG.mjs';
|
|
5
|
+
import './chunk-HMFNFVP7.mjs';
|
|
6
|
+
import './chunk-GES7U2PM.mjs';
|
|
7
|
+
import './chunk-LKOB5COA.mjs';
|
|
8
|
+
import './chunk-UTG5YNXV.mjs';
|
|
9
|
+
import './chunk-R6A773WK.mjs';
|
|
10
|
+
import './chunk-YTJCGKM4.mjs';
|
|
11
|
+
import './chunk-L5ORED7V.mjs';
|
|
12
|
+
import './chunk-BWJVNZPR.mjs';
|
|
13
|
+
|
|
14
|
+
// src/partial-words/index.ts
|
|
15
|
+
var partialWordSearch = partialWordSearchModule;
|
|
16
|
+
|
|
17
|
+
export { partialWordSearch };
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* This object holds the code with which the engine builds the pattern index of
|
|
3
|
+
* each pattern field and answers `contains` and the other text tests on that
|
|
4
|
+
* field.
|
|
5
|
+
*
|
|
6
|
+
* A Node process loads this code by itself. A browser bundle includes it only
|
|
7
|
+
* where the app imports `patternSearch` from `@delali/narsil/pattern` and
|
|
8
|
+
* passes it to {@link registerPatternSearch}, so a bundler can leave it out of
|
|
9
|
+
* an app that has no pattern field.
|
|
10
|
+
*
|
|
11
|
+
* @public
|
|
12
|
+
*/
|
|
13
|
+
interface PatternSearch {
|
|
14
|
+
/** This is always `'pattern'`, the name of the code that the object holds. */
|
|
15
|
+
readonly name: 'pattern';
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export type { PatternSearch as P };
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { P as PatternSearch } from './pattern-Cledccbl.js';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* This object holds the pattern search code, which a browser app has to pass
|
|
5
|
+
* to {@link registerPatternSearch} before it creates or loads an index with a
|
|
6
|
+
* pattern field.
|
|
7
|
+
*
|
|
8
|
+
* ```ts
|
|
9
|
+
* import { registerPatternSearch } from '@delali/narsil'
|
|
10
|
+
* import { patternSearch } from '@delali/narsil/pattern'
|
|
11
|
+
*
|
|
12
|
+
* registerPatternSearch(patternSearch)
|
|
13
|
+
* ```
|
|
14
|
+
*
|
|
15
|
+
* A Node process loads the pattern search code by itself, so it needs no such
|
|
16
|
+
* call.
|
|
17
|
+
*
|
|
18
|
+
* @public
|
|
19
|
+
*/
|
|
20
|
+
declare const patternSearch: PatternSearch;
|
|
21
|
+
|
|
22
|
+
export { PatternSearch, patternSearch };
|
package/dist/pattern.mjs
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { patternSearchModule } from './chunk-PC2G76VW.mjs';
|
|
2
|
+
import './chunk-WQBCBKY3.mjs';
|
|
3
|
+
import './chunk-GES7U2PM.mjs';
|
|
4
|
+
import './chunk-LKOB5COA.mjs';
|
|
5
|
+
import './chunk-YTJCGKM4.mjs';
|
|
6
|
+
import './chunk-L5ORED7V.mjs';
|
|
7
|
+
import './chunk-BWJVNZPR.mjs';
|
|
8
|
+
|
|
9
|
+
// src/pattern/index.ts
|
|
10
|
+
var patternSearch = patternSearchModule;
|
|
11
|
+
|
|
12
|
+
export { patternSearch };
|
|
@@ -1,55 +1,39 @@
|
|
|
1
|
-
import { P as PartitionConfig,
|
|
2
|
-
import '../../adapters-
|
|
1
|
+
import { P as PartitionConfig, d as ScoringMode, V as VectorIndexConfig } from '../../schema-4W_7tk1f.js';
|
|
2
|
+
import '../../adapters-BQFQ4PRz.js';
|
|
3
|
+
import '../../synonyms-CpOAHPa-.js';
|
|
3
4
|
|
|
4
|
-
interface
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
/** The partitions this copy holds, which a cluster node answers a partition-stores request from. */
|
|
11
|
-
heldPartitions?: number[];
|
|
12
|
-
schema: Record<string, string>;
|
|
13
|
-
language: string;
|
|
14
|
-
partitionCount: number;
|
|
15
|
-
bm25Params: {
|
|
16
|
-
k1: number;
|
|
17
|
-
b: number;
|
|
18
|
-
};
|
|
19
|
-
createdAt: number;
|
|
20
|
-
engineVersion: string;
|
|
21
|
-
vectorFields?: Record<string, {
|
|
22
|
-
dimension: number;
|
|
23
|
-
metric: string;
|
|
24
|
-
quantization: string;
|
|
25
|
-
}>;
|
|
26
|
-
embedding?: IndexEmbeddingMetadata;
|
|
27
|
-
/** Persisted so recovered indexes keep collecting surface forms. */
|
|
28
|
-
surfaceForms?: boolean;
|
|
29
|
-
analysisRevision?: string;
|
|
30
|
-
tokenizer?: string;
|
|
31
|
-
stopWords?: string;
|
|
32
|
-
stopWordList?: string[];
|
|
33
|
-
partitionLimits?: PartitionConfig;
|
|
34
|
-
defaultScoring?: ScoringMode;
|
|
35
|
-
trackPositions?: boolean;
|
|
36
|
-
strict?: boolean;
|
|
37
|
-
required?: string[];
|
|
38
|
-
vectorPromotion?: VectorIndexConfig;
|
|
5
|
+
interface PatternIndexArrays {
|
|
6
|
+
fieldPath: string;
|
|
7
|
+
docIds: Uint32Array;
|
|
8
|
+
runs: Uint32Array;
|
|
9
|
+
offsets: Uint32Array;
|
|
10
|
+
postings: Uint32Array;
|
|
39
11
|
}
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
12
|
+
|
|
13
|
+
interface FrozenTokenTableData {
|
|
14
|
+
blob: Uint8Array;
|
|
15
|
+
offsets: Uint32Array;
|
|
16
|
+
payloadSlots: Uint32Array;
|
|
17
|
+
documentFrequencies: Uint32Array;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
interface PartialSpellings {
|
|
21
|
+
fields: string[];
|
|
22
|
+
tokens: string[];
|
|
23
|
+
spellings: string[];
|
|
24
|
+
counts: number[];
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
interface PartialWordTableData {
|
|
28
|
+
fields: string[];
|
|
29
|
+
spellings: Uint8Array;
|
|
30
|
+
spellingOffsets: Uint32Array;
|
|
31
|
+
spellingTokens: Uint32Array;
|
|
32
|
+
spellingCounts: Uint32Array;
|
|
33
|
+
runs: Uint32Array;
|
|
34
|
+
runOffsets: Uint32Array;
|
|
35
|
+
runEntries: Uint32Array;
|
|
45
36
|
}
|
|
46
|
-
/**
|
|
47
|
-
* Persisted surface-form dictionary of one partition. Keys are the
|
|
48
|
-
* unstemmed display tokens; the value is the occurrence count when the
|
|
49
|
-
* surface equals its index token, or `[count, indexToken]` when stemming
|
|
50
|
-
* changed it.
|
|
51
|
-
*/
|
|
52
|
-
type SerializedSurfaceForms = Record<string, number | [number, string]>;
|
|
53
37
|
|
|
54
38
|
interface SegmentPayload {
|
|
55
39
|
documentCount: number;
|
|
@@ -59,7 +43,7 @@ interface SegmentPayload {
|
|
|
59
43
|
postingOffsets: Uint32Array;
|
|
60
44
|
postingDocIds: Uint32Array;
|
|
61
45
|
postingFrequencies: Uint16Array;
|
|
62
|
-
postingFieldIndices:
|
|
46
|
+
postingFieldIndices: Uint16Array;
|
|
63
47
|
positionOffsets: Uint32Array | null;
|
|
64
48
|
positionValues: Uint32Array | null;
|
|
65
49
|
fieldLengthNames: string[];
|
|
@@ -67,6 +51,7 @@ interface SegmentPayload {
|
|
|
67
51
|
totalFieldLengths: Record<string, number>;
|
|
68
52
|
docFrequencies: Record<string, number>;
|
|
69
53
|
surfaceForms: SerializedSurfaceForms | null;
|
|
54
|
+
partialSpellings: PartialSpellings | null;
|
|
70
55
|
numeric: Array<{
|
|
71
56
|
fieldPath: string;
|
|
72
57
|
docIds: Uint32Array;
|
|
@@ -89,13 +74,19 @@ interface SegmentPayload {
|
|
|
89
74
|
latitudes: Float64Array;
|
|
90
75
|
longitudes: Float64Array;
|
|
91
76
|
}>;
|
|
77
|
+
pattern: PatternIndexArrays[];
|
|
78
|
+
sizes: Array<{
|
|
79
|
+
fieldPath: string;
|
|
80
|
+
docIds: Uint32Array;
|
|
81
|
+
values: Float64Array;
|
|
82
|
+
}>;
|
|
92
83
|
}
|
|
93
84
|
|
|
94
85
|
interface SegmentPostingColumns {
|
|
95
86
|
postingOffsets: Uint32Array;
|
|
96
87
|
postingDocIds: Uint32Array;
|
|
97
88
|
postingFrequencies: Uint16Array;
|
|
98
|
-
postingFieldIndices:
|
|
89
|
+
postingFieldIndices: Uint16Array;
|
|
99
90
|
positionOffsets: Uint32Array | null;
|
|
100
91
|
positionValues: Uint32Array | null;
|
|
101
92
|
}
|
|
@@ -109,12 +100,15 @@ interface SegmentColumns extends SegmentPostingColumns {
|
|
|
109
100
|
boolean: SegmentPayload['boolean'];
|
|
110
101
|
enums: SegmentPayload['enums'];
|
|
111
102
|
geo: SegmentPayload['geo'];
|
|
103
|
+
pattern: SegmentPayload['pattern'];
|
|
104
|
+
sizes: SegmentPayload['sizes'];
|
|
112
105
|
surfaceForms: SerializedSurfaceForms | null;
|
|
113
106
|
}
|
|
114
107
|
|
|
115
108
|
interface EncodedDocumentTableData {
|
|
116
109
|
blob: Uint8Array;
|
|
117
110
|
offsets: Uint32Array;
|
|
111
|
+
discardedFields?: readonly string[];
|
|
118
112
|
}
|
|
119
113
|
|
|
120
114
|
interface ExternalIdTableData {
|
|
@@ -129,13 +123,6 @@ interface SurfaceTableData {
|
|
|
129
123
|
counts: Uint32Array;
|
|
130
124
|
}
|
|
131
125
|
|
|
132
|
-
interface FrozenTokenTableData {
|
|
133
|
-
blob: Uint8Array;
|
|
134
|
-
offsets: Uint32Array;
|
|
135
|
-
payloadSlots: Uint32Array;
|
|
136
|
-
documentFrequencies: Uint32Array;
|
|
137
|
-
}
|
|
138
|
-
|
|
139
126
|
interface SharedSegmentSnapshot extends Omit<SegmentColumns, 'surfaceForms'> {
|
|
140
127
|
segmentId: string;
|
|
141
128
|
tokenTable: FrozenTokenTableData;
|
|
@@ -143,18 +130,7 @@ interface SharedSegmentSnapshot extends Omit<SegmentColumns, 'surfaceForms'> {
|
|
|
143
130
|
documentTable: EncodedDocumentTableData;
|
|
144
131
|
/** The surface forms packed flat, or null where the segment holds none. */
|
|
145
132
|
surfaceTable: SurfaceTableData | null;
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
interface CapturedFrozenSegment {
|
|
149
|
-
snapshot: SharedSegmentSnapshot;
|
|
150
|
-
tombstonedDocIds: string[];
|
|
151
|
-
}
|
|
152
|
-
interface CapturedWholePartition {
|
|
153
|
-
partitionId: number;
|
|
154
|
-
totalPartitions: number;
|
|
155
|
-
docCount: number;
|
|
156
|
-
liveTail: Uint8Array | null;
|
|
157
|
-
frozen: CapturedFrozenSegment[];
|
|
133
|
+
partialWords: PartialWordTableData | null;
|
|
158
134
|
}
|
|
159
135
|
|
|
160
136
|
interface PartitionCheckpoint {
|
|
@@ -163,9 +139,13 @@ interface PartitionCheckpoint {
|
|
|
163
139
|
primaryTerm: number;
|
|
164
140
|
}
|
|
165
141
|
|
|
166
|
-
interface
|
|
167
|
-
id: number;
|
|
142
|
+
interface FileRef {
|
|
168
143
|
key: string;
|
|
144
|
+
bytes: number;
|
|
145
|
+
checksum: number;
|
|
146
|
+
}
|
|
147
|
+
interface SegmentRef extends FileRef {
|
|
148
|
+
id: number;
|
|
169
149
|
docCount: number;
|
|
170
150
|
tombstoneCount: number;
|
|
171
151
|
}
|
|
@@ -175,6 +155,72 @@ interface PartitionManifestEntry {
|
|
|
175
155
|
segments: SegmentRef[];
|
|
176
156
|
}
|
|
177
157
|
|
|
158
|
+
interface IndexMetadata {
|
|
159
|
+
indexName: string;
|
|
160
|
+
/** Number of documents represented by the last completed checkpoint. */
|
|
161
|
+
documentCount?: number;
|
|
162
|
+
/** The identity a cluster gave the index at creation; absent on a single engine. */
|
|
163
|
+
indexUuid?: string;
|
|
164
|
+
/** The partitions this copy holds, which a cluster node answers a partition-stores request from. */
|
|
165
|
+
heldPartitions?: number[];
|
|
166
|
+
schema: Record<string, string>;
|
|
167
|
+
language: string;
|
|
168
|
+
partitionCount: number;
|
|
169
|
+
bm25Params: {
|
|
170
|
+
k1: number;
|
|
171
|
+
b: number;
|
|
172
|
+
};
|
|
173
|
+
createdAt: number;
|
|
174
|
+
engineVersion: string;
|
|
175
|
+
vectorFields?: Record<string, {
|
|
176
|
+
dimension: number;
|
|
177
|
+
metric: string;
|
|
178
|
+
quantization: string;
|
|
179
|
+
}>;
|
|
180
|
+
embedding?: IndexEmbeddingMetadata;
|
|
181
|
+
/** Persisted so recovered indexes keep collecting surface forms. */
|
|
182
|
+
surfaceForms?: boolean;
|
|
183
|
+
analysisRevision?: string;
|
|
184
|
+
tokenizer?: string;
|
|
185
|
+
stopWords?: string;
|
|
186
|
+
stopWordList?: string[];
|
|
187
|
+
partitionLimits?: PartitionConfig;
|
|
188
|
+
defaultScoring?: ScoringMode;
|
|
189
|
+
trackPositions?: boolean;
|
|
190
|
+
strict?: boolean;
|
|
191
|
+
required?: string[];
|
|
192
|
+
vectorPromotion?: VectorIndexConfig;
|
|
193
|
+
patternValueLimit?: number;
|
|
194
|
+
synonyms?: unknown;
|
|
195
|
+
}
|
|
196
|
+
/** Persisted so recovery can restore field mappings and rebind the adapter by
|
|
197
|
+
* name; the name is absent when the index was configured with a bare instance. */
|
|
198
|
+
interface IndexEmbeddingMetadata {
|
|
199
|
+
adapter?: string;
|
|
200
|
+
fields: Record<string, string | string[]>;
|
|
201
|
+
}
|
|
202
|
+
/**
|
|
203
|
+
* Persisted surface-form dictionary of one partition. Keys are the
|
|
204
|
+
* unstemmed display tokens; the value is the occurrence count when the
|
|
205
|
+
* surface equals its index token, or `[count, indexToken]` when stemming
|
|
206
|
+
* changed it.
|
|
207
|
+
*/
|
|
208
|
+
type SerializedSurfaceForms = Record<string, number | [number, string]>;
|
|
209
|
+
|
|
210
|
+
interface CapturedSegment {
|
|
211
|
+
snapshot: SharedSegmentSnapshot;
|
|
212
|
+
keep: Uint8Array;
|
|
213
|
+
}
|
|
214
|
+
interface CapturedWholePartition {
|
|
215
|
+
partitionId: number;
|
|
216
|
+
docCount: number;
|
|
217
|
+
segments: CapturedSegment[];
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
interface PriorCheckpoint {
|
|
221
|
+
checkpoint: PartitionCheckpoint[];
|
|
222
|
+
partitions: PartitionManifestEntry[];
|
|
223
|
+
}
|
|
178
224
|
interface CheckpointSegmentsWritten {
|
|
179
225
|
checkpoint: PartitionCheckpoint[];
|
|
180
226
|
partitions: PartitionManifestEntry[];
|
|
@@ -185,6 +231,7 @@ interface CheckpointWorkerRequest {
|
|
|
185
231
|
metadata: IndexMetadata;
|
|
186
232
|
targets: PartitionCheckpoint[];
|
|
187
233
|
compactionThreshold: number;
|
|
234
|
+
prior: PriorCheckpoint | null;
|
|
188
235
|
capturedPartitions?: CapturedWholePartition[];
|
|
189
236
|
}
|
|
190
237
|
interface CheckpointWorkerSuccess {
|
|
@@ -1,27 +1,59 @@
|
|
|
1
|
-
import {
|
|
2
|
-
import '../../chunk-
|
|
3
|
-
import '../../chunk-
|
|
4
|
-
import '../../chunk-
|
|
1
|
+
import { createDurableDirectory } from '../../chunk-R27EMH5M.mjs';
|
|
2
|
+
import '../../chunk-C7SLA6CM.mjs';
|
|
3
|
+
import '../../chunk-HLPRHOSO.mjs';
|
|
4
|
+
import { CHECKPOINT_WORKER_HEARTBEAT_MS, writeCheckpointSegments, directoryCheckpointStore } from '../../chunk-J5QP52HA.mjs';
|
|
5
|
+
import '../../chunk-XADZPPZI.mjs';
|
|
6
|
+
import '../../chunk-P36MUFEJ.mjs';
|
|
7
|
+
import '../../chunk-PNEYIHA4.mjs';
|
|
8
|
+
import '../../chunk-WVCOCKBU.mjs';
|
|
9
|
+
import '../../chunk-KWTW6VTG.mjs';
|
|
10
|
+
import '../../chunk-EMRMGYRE.mjs';
|
|
11
|
+
import '../../chunk-OZDFA4WC.mjs';
|
|
5
12
|
import '../../chunk-AV4IJKD2.mjs';
|
|
6
|
-
import '../../chunk-
|
|
7
|
-
import '../../chunk-
|
|
8
|
-
import '../../chunk-
|
|
9
|
-
import '../../chunk-
|
|
13
|
+
import '../../chunk-SP3NM3ZG.mjs';
|
|
14
|
+
import '../../chunk-VI4R6W25.mjs';
|
|
15
|
+
import '../../chunk-ZM7KU4YW.mjs';
|
|
16
|
+
import '../../chunk-LPQOEU5M.mjs';
|
|
17
|
+
import '../../chunk-HJSN7UYG.mjs';
|
|
18
|
+
import '../../chunk-LXWDQLTW.mjs';
|
|
10
19
|
import '../../chunk-TGP5M5R7.mjs';
|
|
11
|
-
import '../../chunk-THYRHHO2.mjs';
|
|
12
|
-
import '../../chunk-ZJ4CSAB2.mjs';
|
|
13
|
-
import '../../chunk-YNVIUGF5.mjs';
|
|
14
|
-
import '../../chunk-3L4U6EJQ.mjs';
|
|
15
|
-
import '../../chunk-IFBKLBGB.mjs';
|
|
16
20
|
import '../../chunk-U2BKYEC7.mjs';
|
|
17
|
-
import '../../chunk-
|
|
18
|
-
import '../../chunk-
|
|
19
|
-
import '../../chunk-
|
|
21
|
+
import '../../chunk-KOKM4PQN.mjs';
|
|
22
|
+
import '../../chunk-GBEEG5AN.mjs';
|
|
23
|
+
import '../../chunk-JZVARCRZ.mjs';
|
|
24
|
+
import '../../chunk-PC2G76VW.mjs';
|
|
25
|
+
import '../../chunk-WQBCBKY3.mjs';
|
|
26
|
+
import '../../chunk-OM362BRO.mjs';
|
|
27
|
+
import '../../chunk-OEYI3KRB.mjs';
|
|
28
|
+
import '../../chunk-U476IXKQ.mjs';
|
|
29
|
+
import '../../chunk-E74YLAIG.mjs';
|
|
30
|
+
import '../../chunk-HMFNFVP7.mjs';
|
|
31
|
+
import '../../chunk-GES7U2PM.mjs';
|
|
32
|
+
import '../../chunk-LKOB5COA.mjs';
|
|
33
|
+
import '../../chunk-46D6JQIG.mjs';
|
|
34
|
+
import '../../chunk-X5CV2E2B.mjs';
|
|
35
|
+
import '../../chunk-7X5JST5L.mjs';
|
|
36
|
+
import '../../chunk-UTG5YNXV.mjs';
|
|
37
|
+
import '../../chunk-R6A773WK.mjs';
|
|
38
|
+
import '../../chunk-YTJCGKM4.mjs';
|
|
39
|
+
import '../../chunk-CQUMNIIG.mjs';
|
|
40
|
+
import '../../chunk-Q6XWVUZX.mjs';
|
|
41
|
+
import '../../chunk-RFBLQLLZ.mjs';
|
|
42
|
+
import '../../chunk-L5ORED7V.mjs';
|
|
43
|
+
import '../../chunk-BWJVNZPR.mjs';
|
|
20
44
|
|
|
21
45
|
// src/persistence/durability/rebuild.ts
|
|
22
|
-
async function rebuildSegmentsFromDurable(root, metadata, targets, compactionThreshold, capturedPartitions = []) {
|
|
46
|
+
async function rebuildSegmentsFromDurable(root, metadata, targets, compactionThreshold, prior, capturedPartitions = []) {
|
|
23
47
|
const directory = createDurableDirectory(root);
|
|
24
|
-
return writeCheckpointSegments({
|
|
48
|
+
return writeCheckpointSegments({
|
|
49
|
+
store: directoryCheckpointStore(directory),
|
|
50
|
+
directory,
|
|
51
|
+
metadata,
|
|
52
|
+
targets,
|
|
53
|
+
compactionThreshold,
|
|
54
|
+
prior,
|
|
55
|
+
capturedPartitions
|
|
56
|
+
});
|
|
25
57
|
}
|
|
26
58
|
|
|
27
59
|
// src/persistence/durability/checkpoint-worker.ts
|
|
@@ -47,6 +79,7 @@ async function handleRequest(raw) {
|
|
|
47
79
|
request.metadata,
|
|
48
80
|
request.targets,
|
|
49
81
|
request.compactionThreshold,
|
|
82
|
+
request.prior ?? null,
|
|
50
83
|
request.capturedPartitions
|
|
51
84
|
);
|
|
52
85
|
return { type: "success", segments };
|