@delali/narsil 0.3.1-next.130 → 0.3.1-next.140
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/analysis-change-JN23AZQP.mjs +43 -0
- package/dist/analysis-change-entry-5JSIMEKG.mjs +1 -0
- package/dist/browser/analysis-change-57MIRJWR.mjs +499 -0
- package/dist/browser/analysis-change-entry-ICGRR3IN.mjs +1 -0
- package/dist/browser/{chunk-UDGB2MWT.mjs → chunk-35B42UFM.mjs} +4 -0
- package/dist/browser/{chunk-VBBLTD7O.mjs → chunk-3EYDURFD.mjs} +39 -27
- package/dist/browser/{chunk-AIIJDWMG.mjs → chunk-4MFH226Q.mjs} +1 -1
- package/dist/browser/{chunk-D7WQF6DV.mjs → chunk-5LYWZJE2.mjs} +2 -2
- package/dist/browser/chunk-67KSBUYR.mjs +173 -0
- package/dist/browser/{chunk-M5UAAXGU.mjs → chunk-6BSB4N6B.mjs} +2 -2
- package/dist/browser/{chunk-NSCSR6MA.mjs → chunk-CGVBGHBX.mjs} +81 -6
- package/dist/browser/chunk-E5SFTOQR.mjs +21 -0
- package/dist/browser/{chunk-VEGHFP2P.mjs → chunk-MZI3TEOW.mjs} +4 -2
- package/dist/browser/chunk-OIZS5KIX.mjs +6 -0
- package/dist/browser/chunk-QHNNULPW.mjs +265 -0
- package/dist/browser/chunk-XVMOQQ34.mjs +384 -0
- package/dist/browser/{chunk-2Y3CO4EQ.mjs → chunk-ZAIZQOTM.mjs} +256 -777
- package/dist/browser/{durability-wiring-FNZSRH4N.mjs → durability-wiring-MTIAN6VT.mjs} +145 -50
- package/dist/browser/import-Q7MXD6BS.mjs +8 -0
- package/dist/browser/{module-4HIMOQM5.mjs → module-5J3PEUEB.mjs} +7 -6
- package/dist/browser/{phrase-module-WESYVU7K.mjs → phrase-module-HGVBO24W.mjs} +1 -1
- package/dist/{restore-queue-JAMNRC6T.mjs → browser/restore-queue-DISFECE4.mjs} +8 -2
- package/dist/browser/{runtime-3X7ZJ5T3.mjs → runtime-VEUH674B.mjs} +2 -2
- package/dist/browser/{snapshot-V7VF6BN6.mjs → snapshot-YBJ5MF2M.mjs} +36 -167
- package/dist/browser/{write-YIC2MKRY.mjs → write-RTWI5MXL.mjs} +8 -6
- package/dist/{capabilities-Bl8G9taj.d.ts → capabilities-DXAmysqJ.d.ts} +11 -11
- package/dist/{chunk-NMXRXFGZ.mjs → chunk-4GCN34JV.mjs} +2 -2
- package/dist/{chunk-SOHMMNCH.mjs → chunk-6VHWXAOI.mjs} +4 -4
- package/dist/{chunk-2C3WXBQS.mjs → chunk-74XB2SXI.mjs} +1 -1
- package/dist/{chunk-QQTXT65X.mjs → chunk-B5I3IL4H.mjs} +12 -7
- package/dist/chunk-BDIDUW2B.mjs +387 -0
- package/dist/{chunk-5LX6MOIB.mjs → chunk-C3YDYX6C.mjs} +1 -1
- package/dist/{chunk-Y4BVTQ53.mjs → chunk-CCCGEPJM.mjs} +2 -2
- package/dist/{chunk-2JWNJSNE.mjs → chunk-CPCGEIOM.mjs} +11 -8
- package/dist/chunk-CZQ33DLW.mjs +78 -0
- package/dist/{chunk-4GDLFDPZ.mjs → chunk-DD2673TX.mjs} +3 -2
- package/dist/{chunk-RNSC4NFI.mjs → chunk-EEBCDP36.mjs} +3 -1
- package/dist/{chunk-HCGJKDZV.mjs → chunk-FFVLPCLH.mjs} +1 -1
- package/dist/{chunk-WVRDRLIC.mjs → chunk-FKARMUWH.mjs} +6 -6
- package/dist/{chunk-PATMDIMN.mjs → chunk-HJJWY6PT.mjs} +1 -1
- package/dist/chunk-HMDT3I63.mjs +21 -0
- package/dist/{chunk-CZ7ISLXA.mjs → chunk-HNHCNTVG.mjs} +4 -0
- package/dist/{chunk-RWESFSJI.mjs → chunk-IBATGS3J.mjs} +87 -61
- package/dist/{chunk-Y2U5VVWX.mjs → chunk-ICW45TTU.mjs} +1 -1
- package/dist/{chunk-T4OFG4IQ.mjs → chunk-IP4HJOGC.mjs} +90 -16
- package/dist/{chunk-EUCX5K74.mjs → chunk-J3EHATEZ.mjs} +3 -3
- package/dist/{chunk-X5CNUKD5.mjs → chunk-K3AQOZVM.mjs} +37 -20
- package/dist/chunk-LK3MOTXF.mjs +174 -0
- package/dist/{chunk-7OIQ2SIO.mjs → chunk-LLDFIKGP.mjs} +4 -4
- package/dist/{chunk-K77FJXKS.mjs → chunk-MTQTX4BP.mjs} +81 -8
- package/dist/{chunk-ZR4VM4A6.mjs → chunk-N37VMC62.mjs} +1 -18
- package/dist/{chunk-2WBY4Y4M.mjs → chunk-NOT5QOHU.mjs} +1 -1
- package/dist/{chunk-D5TFRB5G.mjs → chunk-NPNK57CA.mjs} +1 -1
- package/dist/{chunk-6SSVIW7Y.mjs → chunk-O4LJQI4G.mjs} +31 -5
- package/dist/{chunk-JJ6EUMXB.mjs → chunk-PNNVTKMT.mjs} +2 -2
- package/dist/{chunk-2SEL6SBS.mjs → chunk-RKAVHNRW.mjs} +2 -2
- package/dist/{chunk-XABMCBHA.mjs → chunk-SBMAWTRG.mjs} +684 -99
- package/dist/chunk-TCWQJL7O.mjs +188 -0
- package/dist/{chunk-2MKX2SCO.mjs → chunk-U4X4AWKL.mjs} +7 -7
- package/dist/{chunk-PU6SKH7O.mjs → chunk-UBXJQJII.mjs} +2 -2
- package/dist/chunk-UEV53BBO.mjs +480 -0
- package/dist/{chunk-NNO7CO3A.mjs → chunk-UIKN2OTQ.mjs} +1 -1
- package/dist/{chunk-F6AAXI7T.mjs → chunk-VSUYTPME.mjs} +3 -3
- package/dist/{chunk-TGHNXN7W.mjs → chunk-W2XDK66Z.mjs} +180 -541
- package/dist/{chunk-7X5JST5L.mjs → chunk-WNBRW4Q2.mjs} +16 -2
- package/dist/{chunk-6M6SDH3S.mjs → chunk-XMQNOD5V.mjs} +8 -8
- package/dist/{chunk-SZGRYEIC.mjs → chunk-XS2VM7UE.mjs} +3 -3
- package/dist/{chunk-JT3YZGCE.mjs → chunk-XXKWEG53.mjs} +3 -188
- package/dist/{chunk-YBWSV5DV.mjs → chunk-ZMBFWQXN.mjs} +2 -2
- package/dist/{chunk-7XY6VD7B.mjs → chunk-ZSWHPKGM.mjs} +1 -1
- package/dist/{client-Cx-TiqPE.d.ts → client-DApBeLbk.d.ts} +18 -3
- package/dist/client.d.ts +7 -8
- package/dist/client.mjs +11 -1
- package/dist/{events-DErop1LM.d.ts → config-B0H-vp4U.d.ts} +424 -2
- package/dist/{config-CzB9YvF_.d.ts → config-Brysl9NO.d.ts} +3 -3
- package/dist/{direct-executor-HQ4QVDX7.mjs → direct-executor-J5UDLPL3.mjs} +14 -14
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +24 -9
- package/dist/distribution.mjs +882 -192
- package/dist/{durability-wiring-S7OX2MAI.mjs → durability-wiring-AYOTAPNQ.mjs} +165 -70
- package/dist/embeddings/openai.mjs +2 -2
- package/dist/{engine-B_W9sdZK.d.ts → engine-CuLMbH1-.d.ts} +64 -7
- package/dist/import-QRZAQ5RH.mjs +43 -0
- package/dist/{index-CKI1fDEl.d.ts → index-HYnqS1SE.d.ts} +2 -2
- package/dist/index.browser.mjs +4 -4
- package/dist/index.d.ts +6 -7
- package/dist/index.mjs +20 -20
- package/dist/{long-query-thread-WVSMIBYP.mjs → long-query-thread-U7K5A72G.mjs} +14 -14
- package/dist/{module-2E4XYSGT.mjs → module-L2OZO5PQ.mjs} +11 -11
- package/dist/partial-words.mjs +5 -5
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.d.ts +12 -1
- package/dist/persistence/durability/checkpoint-worker.mjs +23 -23
- package/dist/{phrase-module-JERF2IDG.mjs → phrase-module-5QHBD2VJ.mjs} +1 -1
- package/dist/react.d.ts +11 -8
- package/dist/react.mjs +4 -0
- package/dist/{rerank-DMMT4FZA.mjs → rerank-KTLC3E6N.mjs} +5 -5
- package/dist/rerankers/http.mjs +2 -2
- package/dist/{browser/restore-queue-GEZ54CA5.mjs → restore-queue-RYSRRFS4.mjs} +8 -2
- package/dist/{runtime-7MLQ7J64.mjs → runtime-DYV75DCB.mjs} +6 -6
- package/dist/{schema-CS-clLun.d.ts → schema-BTXbHCqA.d.ts} +4 -1
- package/dist/server.d.ts +200 -11
- package/dist/server.mjs +66 -23
- package/dist/{snapshot-DOQLGGEX.mjs → snapshot-BTRZ3MZK.mjs} +59 -190
- package/dist/synonyms.mjs +6 -4
- package/dist/tasks-DjGjK70Y.d.ts +720 -0
- package/dist/{thread-DNK5XWXG.mjs → thread-USB3GOPY.mjs} +13 -13
- package/dist/vector/search-worker.d.ts +1 -1
- package/dist/vector/search-worker.mjs +5 -5
- package/dist/workers/entry.mjs +7 -7
- package/dist/{write-ZKR7SNRT.mjs → write-GHTA56OW.mjs} +25 -22
- package/openapi.json +5310 -0
- package/package.json +16 -8
- package/dist/config-hOWGBeC6.d.ts +0 -414
- package/dist/tasks-BfF-VRhm.d.ts +0 -302
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration } from './chunk-UEV53BBO.mjs';
|
|
2
|
+
import './chunk-LK3MOTXF.mjs';
|
|
3
|
+
import './chunk-W2XDK66Z.mjs';
|
|
4
|
+
import './chunk-XMQNOD5V.mjs';
|
|
5
|
+
import './chunk-XXKWEG53.mjs';
|
|
6
|
+
import './chunk-UIKN2OTQ.mjs';
|
|
7
|
+
import './chunk-AV4IJKD2.mjs';
|
|
8
|
+
import './chunk-IBATGS3J.mjs';
|
|
9
|
+
import './chunk-U4X4AWKL.mjs';
|
|
10
|
+
import './chunk-N37VMC62.mjs';
|
|
11
|
+
import './chunk-ZATRDAYU.mjs';
|
|
12
|
+
import './chunk-EBIHRL7B.mjs';
|
|
13
|
+
import './chunk-HJSN7UYG.mjs';
|
|
14
|
+
import './chunk-EEBCDP36.mjs';
|
|
15
|
+
import './chunk-KOKM4PQN.mjs';
|
|
16
|
+
import './chunk-QSK35UC2.mjs';
|
|
17
|
+
import './chunk-74XB2SXI.mjs';
|
|
18
|
+
import './chunk-RKAVHNRW.mjs';
|
|
19
|
+
import './chunk-WQBCBKY3.mjs';
|
|
20
|
+
import './chunk-JUKYYXSP.mjs';
|
|
21
|
+
import './chunk-J3EHATEZ.mjs';
|
|
22
|
+
import './chunk-UBXJQJII.mjs';
|
|
23
|
+
import './chunk-U476IXKQ.mjs';
|
|
24
|
+
import './chunk-FQBFZAWG.mjs';
|
|
25
|
+
import './chunk-VJS4JFBQ.mjs';
|
|
26
|
+
import './chunk-GES7U2PM.mjs';
|
|
27
|
+
import './chunk-ZSWHPKGM.mjs';
|
|
28
|
+
import './chunk-O4LJQI4G.mjs';
|
|
29
|
+
import './chunk-X5CV2E2B.mjs';
|
|
30
|
+
import './chunk-WNBRW4Q2.mjs';
|
|
31
|
+
import './chunk-TGP5M5R7.mjs';
|
|
32
|
+
import './chunk-U2BKYEC7.mjs';
|
|
33
|
+
import './chunk-NPNK57CA.mjs';
|
|
34
|
+
import './chunk-4UMZ7KQL.mjs';
|
|
35
|
+
import './chunk-R6A773WK.mjs';
|
|
36
|
+
import './chunk-YTJCGKM4.mjs';
|
|
37
|
+
import './chunk-RIIUZJQ3.mjs';
|
|
38
|
+
import './chunk-UJWGQIXU.mjs';
|
|
39
|
+
import './chunk-FFVLPCLH.mjs';
|
|
40
|
+
import './chunk-22YPZVMG.mjs';
|
|
41
|
+
import './chunk-RFBLQLLZ.mjs';
|
|
42
|
+
import './chunk-L5ORED7V.mjs';
|
|
43
|
+
import './chunk-HNHCNTVG.mjs';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-HMDT3I63.mjs';
|
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
import { requireAnalysisRebuildMemory } from './chunk-67KSBUYR.mjs';
|
|
2
|
+
import './chunk-OIZS5KIX.mjs';
|
|
3
|
+
import { reconstructSchemaFromMetadata, getLanguage, resolveIndexAnalysis, prepareStoredSynonyms, rulesOfStoredSynonyms, resolvePartialWordAnalysis, emitEngineEvent, requireConfigurationWithinSizeLimit, prepareConfiguredSynonyms, rebuildTextIndexes, textRebuildTargetOf, ARRAY_POSITION_GAP } from './chunk-ZAIZQOTM.mjs';
|
|
4
|
+
import './chunk-26A5JFT2.mjs';
|
|
5
|
+
import { compareCodePoints } from './chunk-4MFH226Q.mjs';
|
|
6
|
+
import './chunk-JPTQZWJQ.mjs';
|
|
7
|
+
import { ANALYSIS_CHANGE_MEMBERS, ANALYSIS_HOLD_APPLY_INTERVAL_MS } from './chunk-MZI3TEOW.mjs';
|
|
8
|
+
import { NarsilError, ErrorCodes } from './chunk-35B42UFM.mjs';
|
|
9
|
+
|
|
10
|
+
// src/engine/analysis-change/pending-target.ts
|
|
11
|
+
function configUnderAnalysis(base, language, tokenizer, stopWords) {
|
|
12
|
+
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = base;
|
|
13
|
+
return {
|
|
14
|
+
...rest,
|
|
15
|
+
language,
|
|
16
|
+
...tokenizer !== void 0 ? { tokenizer } : {},
|
|
17
|
+
...stopWords !== void 0 ? { stopWords } : {}
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
function configOfNamedSettings(base, settings) {
|
|
21
|
+
const { language, tokenizer, stopWords, stopWordList } = settings;
|
|
22
|
+
const listed = stopWordList === void 0 ? void 0 : new Set(stopWordList);
|
|
23
|
+
return configUnderAnalysis(base, language, tokenizer, stopWords ?? listed);
|
|
24
|
+
}
|
|
25
|
+
function targetOfPending(entry, pending) {
|
|
26
|
+
const config = configOfNamedSettings(entry.config, pending);
|
|
27
|
+
return { config, language: getLanguage(pending.language), analysis: resolveIndexAnalysis(config) };
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// src/engine/analysis-change/run.ts
|
|
31
|
+
function createHold(generation) {
|
|
32
|
+
let markReady = () => void 0;
|
|
33
|
+
let refuse = () => void 0;
|
|
34
|
+
let wake = () => void 0;
|
|
35
|
+
const ready = new Promise((resolve, reject) => {
|
|
36
|
+
markReady = resolve;
|
|
37
|
+
refuse = reject;
|
|
38
|
+
});
|
|
39
|
+
ready.catch(() => void 0);
|
|
40
|
+
return {
|
|
41
|
+
generation,
|
|
42
|
+
built: false,
|
|
43
|
+
released: false,
|
|
44
|
+
ready,
|
|
45
|
+
markReady,
|
|
46
|
+
refuse,
|
|
47
|
+
wake: () => wake(),
|
|
48
|
+
pause: () => new Promise((resolve) => {
|
|
49
|
+
const timer = setTimeout(resolve, ANALYSIS_HOLD_APPLY_INTERVAL_MS);
|
|
50
|
+
wake = () => {
|
|
51
|
+
clearTimeout(timer);
|
|
52
|
+
resolve();
|
|
53
|
+
};
|
|
54
|
+
})
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
var runsByCore = /* @__PURE__ */ new WeakMap();
|
|
58
|
+
function runsOf(core) {
|
|
59
|
+
let runs = runsByCore.get(core);
|
|
60
|
+
if (runs === void 0) {
|
|
61
|
+
runs = /* @__PURE__ */ new Map();
|
|
62
|
+
runsByCore.set(core, runs);
|
|
63
|
+
}
|
|
64
|
+
return runs;
|
|
65
|
+
}
|
|
66
|
+
function createRun(target, hold = null) {
|
|
67
|
+
let finish = () => void 0;
|
|
68
|
+
let rejectFinished = () => void 0;
|
|
69
|
+
let settle = () => void 0;
|
|
70
|
+
const finished = new Promise((resolve, reject) => {
|
|
71
|
+
finish = resolve;
|
|
72
|
+
rejectFinished = reject;
|
|
73
|
+
});
|
|
74
|
+
finished.catch(() => void 0);
|
|
75
|
+
const settled = new Promise((resolve) => {
|
|
76
|
+
settle = resolve;
|
|
77
|
+
});
|
|
78
|
+
const fail = (error) => {
|
|
79
|
+
rejectFinished(error);
|
|
80
|
+
hold?.refuse(error);
|
|
81
|
+
hold?.wake();
|
|
82
|
+
};
|
|
83
|
+
return { target, stopReason: null, switched: false, settled, finish, fail, finished, settle, hold };
|
|
84
|
+
}
|
|
85
|
+
async function waitForRelease(run, hold, keepCurrent) {
|
|
86
|
+
hold.built = true;
|
|
87
|
+
hold.markReady();
|
|
88
|
+
while (!hold.released) {
|
|
89
|
+
if (run.stopReason !== null || !keepCurrent()) {
|
|
90
|
+
hold.built = false;
|
|
91
|
+
return false;
|
|
92
|
+
}
|
|
93
|
+
await hold.pause();
|
|
94
|
+
}
|
|
95
|
+
return true;
|
|
96
|
+
}
|
|
97
|
+
function stopRun(run, indexName) {
|
|
98
|
+
run.stopReason = new NarsilError(
|
|
99
|
+
ErrorCodes.ANALYSIS_CHANGE_REPLACED,
|
|
100
|
+
`A later change or a restore replaced this analysis change of index "${indexName}"`,
|
|
101
|
+
{ indexName }
|
|
102
|
+
);
|
|
103
|
+
run.fail(run.stopReason);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// src/engine/analysis-change/settings.ts
|
|
107
|
+
function refuseTokenizer(indexName) {
|
|
108
|
+
throw new NarsilError(
|
|
109
|
+
ErrorCodes.CONFIG_INVALID,
|
|
110
|
+
`The tokenizer of index "${indexName}" must change to a name registered with registerTokenizer, because an instance survives neither a worker boundary nor a restart`,
|
|
111
|
+
{ indexName }
|
|
112
|
+
);
|
|
113
|
+
}
|
|
114
|
+
function refuseStopWords(indexName) {
|
|
115
|
+
throw new NarsilError(
|
|
116
|
+
ErrorCodes.CONFIG_INVALID,
|
|
117
|
+
`The stop words of index "${indexName}" must change to a name registered with registerStopWords or to a list of words, because a function survives neither a worker boundary nor a restart`,
|
|
118
|
+
{ indexName }
|
|
119
|
+
);
|
|
120
|
+
}
|
|
121
|
+
function stopWordsOfChange(indexName, value) {
|
|
122
|
+
if (value === void 0 || value === null || typeof value === "string") return value;
|
|
123
|
+
const words = value instanceof Set ? [...value] : value;
|
|
124
|
+
if (!Array.isArray(words) || words.some((word) => typeof word !== "string")) refuseStopWords(indexName);
|
|
125
|
+
return new Set(words);
|
|
126
|
+
}
|
|
127
|
+
function tokenizerOfChange(indexName, value) {
|
|
128
|
+
if (value === void 0 || value === null || typeof value === "string") return value;
|
|
129
|
+
refuseTokenizer(indexName);
|
|
130
|
+
}
|
|
131
|
+
function requireRegisteredLanguage(name) {
|
|
132
|
+
if (typeof name !== "string") {
|
|
133
|
+
throw new NarsilError(ErrorCodes.LANGUAGE_NOT_SUPPORTED, "The language of an analysis change must be a name");
|
|
134
|
+
}
|
|
135
|
+
return getLanguage(name);
|
|
136
|
+
}
|
|
137
|
+
function resolveAnalysisTarget(indexName, base, change) {
|
|
138
|
+
if (change === null || typeof change !== "object" || Array.isArray(change)) {
|
|
139
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "An analysis change must be an object", { indexName });
|
|
140
|
+
}
|
|
141
|
+
const unknownMember = Object.keys(change).find((member) => !ANALYSIS_CHANGE_MEMBERS.includes(member));
|
|
142
|
+
if (unknownMember !== void 0) {
|
|
143
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `"${unknownMember}" is not an analysis setting`, { indexName });
|
|
144
|
+
}
|
|
145
|
+
const language = change.language === void 0 ? getLanguage(base.language ?? "english") : requireRegisteredLanguage(change.language);
|
|
146
|
+
const tokenizer = tokenizerOfChange(indexName, change.tokenizer);
|
|
147
|
+
const stopWords = stopWordsOfChange(indexName, change.stopWords);
|
|
148
|
+
const config = configUnderAnalysis(
|
|
149
|
+
base,
|
|
150
|
+
language.name,
|
|
151
|
+
tokenizer === void 0 ? base.tokenizer : tokenizer ?? void 0,
|
|
152
|
+
stopWords === void 0 ? base.stopWords : stopWords ?? void 0
|
|
153
|
+
);
|
|
154
|
+
requireConfigurationWithinSizeLimit(config, language);
|
|
155
|
+
return { config, language, analysis: resolveIndexAnalysis(config) };
|
|
156
|
+
}
|
|
157
|
+
function currentConfigUnderTarget(current, target) {
|
|
158
|
+
return configUnderAnalysis(current, target.language.name, target.config.tokenizer, target.config.stopWords);
|
|
159
|
+
}
|
|
160
|
+
function sameStopWords(left, right) {
|
|
161
|
+
if (left === right) return true;
|
|
162
|
+
if (!(left instanceof Set) || !(right instanceof Set) || left.size !== right.size) return false;
|
|
163
|
+
for (const word of left) if (!right.has(word)) return false;
|
|
164
|
+
return true;
|
|
165
|
+
}
|
|
166
|
+
function holdsSettings(config, target) {
|
|
167
|
+
return (config.language ?? "english") === target.language.name && config.tokenizer === target.config.tokenizer && sameStopWords(config.stopWords, target.config.stopWords);
|
|
168
|
+
}
|
|
169
|
+
function namedSettingsOf(config, language) {
|
|
170
|
+
return {
|
|
171
|
+
language,
|
|
172
|
+
...typeof config.tokenizer === "string" ? { tokenizer: config.tokenizer } : {},
|
|
173
|
+
...typeof config.stopWords === "string" ? { stopWords: config.stopWords } : {},
|
|
174
|
+
...config.stopWords instanceof Set ? { stopWordList: [...config.stopWords].sort(compareCodePoints) } : {}
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
function durableSettingsOf(entry) {
|
|
178
|
+
return {
|
|
179
|
+
...namedSettingsOf(entry.config, entry.language.name),
|
|
180
|
+
generation: entry.analysisGeneration,
|
|
181
|
+
analysisRevision: entry.analysisRevision,
|
|
182
|
+
arrayPositionGap: entry.arrayPositionGap
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
function requireSynonymsUnderTarget(entry, target) {
|
|
186
|
+
if (entry.synonyms === null) return;
|
|
187
|
+
prepareConfiguredSynonyms({ ...target.config, synonyms: rulesOfStoredSynonyms(entry.synonyms) }, target.language);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
// src/engine/analysis-change/adopt-saved.ts
|
|
191
|
+
function adoptSavedAnalysis(manager, entry, metadata) {
|
|
192
|
+
const saved = reconstructSchemaFromMetadata(metadata);
|
|
193
|
+
const { tokenizer: _tokenizer, stopWords: _stopWords, ...rest } = entry.config;
|
|
194
|
+
const config = {
|
|
195
|
+
...rest,
|
|
196
|
+
language: saved.language,
|
|
197
|
+
...saved.tokenizer !== void 0 ? { tokenizer: saved.tokenizer } : {},
|
|
198
|
+
...saved.stopWords !== void 0 ? { stopWords: saved.stopWords } : {}
|
|
199
|
+
};
|
|
200
|
+
const language = getLanguage(config.language ?? "english");
|
|
201
|
+
const analysis = resolveIndexAnalysis(config);
|
|
202
|
+
const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
|
|
203
|
+
entry.config = config;
|
|
204
|
+
entry.language = language;
|
|
205
|
+
entry.analysisGeneration = metadata.analysisGeneration ?? 0;
|
|
206
|
+
entry.analysisRevision = metadata.analysisRevision ?? null;
|
|
207
|
+
entry.arrayPositionGap = metadata.arrayPositionGap ?? null;
|
|
208
|
+
manager?.adoptAnalysis({
|
|
209
|
+
config,
|
|
210
|
+
language,
|
|
211
|
+
analysis,
|
|
212
|
+
partialWordAnalysis: resolvePartialWordAnalysis(config, language, analysis),
|
|
213
|
+
synonyms: synonyms?.lookup ?? null
|
|
214
|
+
});
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
// src/engine/analysis-change/index.ts
|
|
218
|
+
function termBytesOf(core, indexName) {
|
|
219
|
+
let bytes = 0;
|
|
220
|
+
for (const partition of core.requireManager(indexName).getAllPartitions()) {
|
|
221
|
+
bytes += partition.estimateMemoryBytes();
|
|
222
|
+
for (const state of partition.readStates()) bytes -= state.docStore.contentBytes();
|
|
223
|
+
}
|
|
224
|
+
return Math.max(0, bytes);
|
|
225
|
+
}
|
|
226
|
+
function toError(value) {
|
|
227
|
+
return value instanceof Error ? value : new Error(String(value));
|
|
228
|
+
}
|
|
229
|
+
async function requireNoUndurableSwitch(core, indexName, entry) {
|
|
230
|
+
if (entry.durableAnalysis === null) return;
|
|
231
|
+
const durable = core.durability?.manager;
|
|
232
|
+
await (durable?.checkpointFromMemory?.(indexName) ?? durable?.checkpoint(indexName));
|
|
233
|
+
if (entry.durableAnalysis === null) return;
|
|
234
|
+
throw new NarsilError(
|
|
235
|
+
ErrorCodes.ANALYSIS_CHANGE_NOT_READY,
|
|
236
|
+
`The previous analysis change of index "${indexName}" has no durable checkpoint yet, so try this change again once a checkpoint succeeds`,
|
|
237
|
+
{ indexName }
|
|
238
|
+
);
|
|
239
|
+
}
|
|
240
|
+
function switchIndex({ core, indexName, entry, run, generation }) {
|
|
241
|
+
const { language, analysis } = run.target;
|
|
242
|
+
const config = currentConfigUnderTarget(entry.config, run.target);
|
|
243
|
+
const synonyms = prepareStoredSynonyms(rulesOfStoredSynonyms(entry.synonyms), config, language);
|
|
244
|
+
return () => {
|
|
245
|
+
run.switched = true;
|
|
246
|
+
entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
|
|
247
|
+
entry.config = config;
|
|
248
|
+
entry.language = language;
|
|
249
|
+
entry.analysisGeneration = generation;
|
|
250
|
+
entry.analysisRevision = language.revision;
|
|
251
|
+
entry.arrayPositionGap = ARRAY_POSITION_GAP;
|
|
252
|
+
entry.pendingAnalysis = null;
|
|
253
|
+
entry.synonyms = synonyms?.encoded ?? null;
|
|
254
|
+
core.requireManager(indexName).adoptAnalysis({
|
|
255
|
+
...textRebuildTargetOf(config, language, analysis),
|
|
256
|
+
synonyms: synonyms?.lookup ?? null
|
|
257
|
+
});
|
|
258
|
+
core.analysisRebuild.clearStale(indexName);
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
async function buildAndSwitch(scope) {
|
|
262
|
+
const { core, indexName, entry, run } = scope;
|
|
263
|
+
const { hold } = run;
|
|
264
|
+
const manager = core.requireManager(indexName);
|
|
265
|
+
const emit = (status, partitionsRebuilt2, error) => {
|
|
266
|
+
emitEngineEvent(core.eventHandlers, "analysisRebuild", {
|
|
267
|
+
indexName,
|
|
268
|
+
status,
|
|
269
|
+
partitionsRebuilt: partitionsRebuilt2,
|
|
270
|
+
partitionCount: manager.partitionCount,
|
|
271
|
+
...error !== void 0 ? { error } : {}
|
|
272
|
+
});
|
|
273
|
+
};
|
|
274
|
+
let partitionsRebuilt = 0;
|
|
275
|
+
let adopt = () => void 0;
|
|
276
|
+
const isCurrent = () => run.stopReason === null && !core.shutdownState.isShutdown && core.indexRegistry.get(indexName) === entry && core.executor.getManager(indexName) === manager;
|
|
277
|
+
emit("started", 0);
|
|
278
|
+
const wasPromoted = core.orchestrator.desyncIndex(indexName);
|
|
279
|
+
try {
|
|
280
|
+
const durable = core.durability?.manager;
|
|
281
|
+
const rebuilt = await rebuildTextIndexes(
|
|
282
|
+
manager,
|
|
283
|
+
{
|
|
284
|
+
isCurrent,
|
|
285
|
+
onPartitionBuilt(built) {
|
|
286
|
+
partitionsRebuilt = built;
|
|
287
|
+
},
|
|
288
|
+
onSwitch: () => adopt(),
|
|
289
|
+
...hold === null ? {} : { holdBeforeSwitch: (keepCurrent) => waitForRelease(run, hold, keepCurrent) },
|
|
290
|
+
runSwitch(swap) {
|
|
291
|
+
const swapWithSynonyms = () => {
|
|
292
|
+
if (!isCurrent()) return false;
|
|
293
|
+
adopt = switchIndex(scope);
|
|
294
|
+
return swap();
|
|
295
|
+
};
|
|
296
|
+
return durable?.checkpointAfterSwitch?.(indexName, swapWithSynonyms) ?? Promise.resolve(swapWithSynonyms());
|
|
297
|
+
}
|
|
298
|
+
},
|
|
299
|
+
textRebuildTargetOf(run.target.config, run.target.language, run.target.analysis)
|
|
300
|
+
);
|
|
301
|
+
if (!rebuilt) throw run.stopReason ?? stoppedError(core, indexName);
|
|
302
|
+
if (entry.durableAnalysis !== null) {
|
|
303
|
+
throw new NarsilError(
|
|
304
|
+
ErrorCodes.PERSISTENCE_SAVE_FAILED,
|
|
305
|
+
`The new terms of index "${indexName}" answer queries, but they have no durable checkpoint yet`,
|
|
306
|
+
{ indexName }
|
|
307
|
+
);
|
|
308
|
+
}
|
|
309
|
+
emit("completed", partitionsRebuilt);
|
|
310
|
+
} catch (error) {
|
|
311
|
+
emit("failed", partitionsRebuilt, toError(error));
|
|
312
|
+
throw error;
|
|
313
|
+
} finally {
|
|
314
|
+
if (core.executor.getManager(indexName) === manager) {
|
|
315
|
+
await core.orchestrator.resyncIndex(indexName, wasPromoted).catch(() => void 0);
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
}
|
|
319
|
+
function stoppedError(core, indexName) {
|
|
320
|
+
if (core.shutdownState.isShutdown) {
|
|
321
|
+
return new NarsilError(ErrorCodes.INSTANCE_SHUT_DOWN, "This Narsil instance has been shut down");
|
|
322
|
+
}
|
|
323
|
+
return new NarsilError(ErrorCodes.INDEX_NOT_FOUND, `Index "${indexName}" closed during its analysis change`, {
|
|
324
|
+
indexName
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
async function clearPending(core, indexName, entry) {
|
|
328
|
+
entry.pendingAnalysis = null;
|
|
329
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
330
|
+
}
|
|
331
|
+
function requireNotRebalancing(core, indexName) {
|
|
332
|
+
if (!core.rebalancingIndexes.has(indexName)) return;
|
|
333
|
+
throw new NarsilError(
|
|
334
|
+
ErrorCodes.PARTITION_REBALANCING_BACKPRESSURE,
|
|
335
|
+
`Index "${indexName}" is being rebalanced, so try its analysis change again once the rebalance ends`,
|
|
336
|
+
{ indexName }
|
|
337
|
+
);
|
|
338
|
+
}
|
|
339
|
+
async function driveRun(core, indexName, entry, run, previous, resumedGeneration) {
|
|
340
|
+
const runs = runsOf(core);
|
|
341
|
+
try {
|
|
342
|
+
if (previous !== void 0) await previous.settled;
|
|
343
|
+
if (runs.get(indexName) !== run) return;
|
|
344
|
+
await requireNoUndurableSwitch(core, indexName, entry);
|
|
345
|
+
if (run.stopReason !== null) return;
|
|
346
|
+
if (core.indexRegistry.get(indexName) !== entry) throw stoppedError(core, indexName);
|
|
347
|
+
requireNotRebalancing(core, indexName);
|
|
348
|
+
if (resumedGeneration === null && holdsSettings(entry.config, run.target)) {
|
|
349
|
+
if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
|
|
350
|
+
run.finish();
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
const generation = resumedGeneration ?? Math.max(entry.analysisGeneration, entry.pendingAnalysis?.generation ?? 0) + 1;
|
|
354
|
+
if (entry.pendingAnalysis?.generation !== generation || run.hold !== null) {
|
|
355
|
+
entry.pendingAnalysis = { generation, ...namedSettingsOf(run.target.config, run.target.language.name) };
|
|
356
|
+
await core.durability?.manager.persistMetadata(indexName);
|
|
357
|
+
}
|
|
358
|
+
if (run.stopReason !== null) return;
|
|
359
|
+
await core.analysisRebuild.abandon(indexName);
|
|
360
|
+
await core.analysisRebuild.runExclusive(
|
|
361
|
+
indexName,
|
|
362
|
+
() => buildAndSwitch({ core, indexName, entry, run, generation }),
|
|
363
|
+
run.hold?.ready
|
|
364
|
+
);
|
|
365
|
+
run.finish();
|
|
366
|
+
} catch (error) {
|
|
367
|
+
run.fail(error);
|
|
368
|
+
const keepsPending = resumedGeneration !== null || core.shutdownState.isShutdown;
|
|
369
|
+
const stillOwnsPending = runs.get(indexName) === run && run.stopReason === null;
|
|
370
|
+
if (!keepsPending && stillOwnsPending && core.indexRegistry.get(indexName) === entry) {
|
|
371
|
+
if (entry.pendingAnalysis !== null) await clearPending(core, indexName, entry).catch(() => void 0);
|
|
372
|
+
}
|
|
373
|
+
} finally {
|
|
374
|
+
if (runs.get(indexName) === run) runs.delete(indexName);
|
|
375
|
+
run.settle();
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
async function changeIndexAnalysis(core, indexName, change) {
|
|
379
|
+
const entry = core.requireIndex(indexName);
|
|
380
|
+
requireNotRebalancing(core, indexName);
|
|
381
|
+
const target = resolveAnalysisTarget(indexName, entry.config, change);
|
|
382
|
+
const runs = runsOf(core);
|
|
383
|
+
const previous = runs.get(indexName);
|
|
384
|
+
if (previous !== void 0 && holdsSettings(previous.target.config, target)) return previous.finished;
|
|
385
|
+
const heldAlready = holdsSettings(entry.config, target);
|
|
386
|
+
if (previous === void 0 && entry.durableAnalysis === null && heldAlready) return;
|
|
387
|
+
if (!heldAlready) {
|
|
388
|
+
requireSynonymsUnderTarget(entry, target);
|
|
389
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
390
|
+
}
|
|
391
|
+
const run = createRun(target);
|
|
392
|
+
if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
|
|
393
|
+
runs.set(indexName, run);
|
|
394
|
+
void driveRun(core, indexName, entry, run, previous, null);
|
|
395
|
+
return run.finished;
|
|
396
|
+
}
|
|
397
|
+
function resumeAnalysisChange(core, indexName) {
|
|
398
|
+
const entry = core.requireIndex(indexName);
|
|
399
|
+
const runs = runsOf(core);
|
|
400
|
+
const running = runs.get(indexName);
|
|
401
|
+
if (running !== void 0) return running.finished;
|
|
402
|
+
const pending = entry.pendingAnalysis;
|
|
403
|
+
if (pending === null) return Promise.resolve();
|
|
404
|
+
let target;
|
|
405
|
+
try {
|
|
406
|
+
target = targetOfPending(entry, pending);
|
|
407
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
408
|
+
} catch (error) {
|
|
409
|
+
emitEngineEvent(core.eventHandlers, "analysisRebuild", {
|
|
410
|
+
indexName,
|
|
411
|
+
status: "failed",
|
|
412
|
+
partitionsRebuilt: 0,
|
|
413
|
+
partitionCount: entry.partitionCount,
|
|
414
|
+
error: toError(error)
|
|
415
|
+
});
|
|
416
|
+
return Promise.reject(error);
|
|
417
|
+
}
|
|
418
|
+
const run = createRun(target);
|
|
419
|
+
runs.set(indexName, run);
|
|
420
|
+
void driveRun(core, indexName, entry, run, void 0, pending.generation);
|
|
421
|
+
return run.finished;
|
|
422
|
+
}
|
|
423
|
+
async function abandonAnalysisChange(core, indexName) {
|
|
424
|
+
const run = runsOf(core).get(indexName);
|
|
425
|
+
if (run === void 0) return;
|
|
426
|
+
stopRun(run, indexName);
|
|
427
|
+
await run.settled;
|
|
428
|
+
}
|
|
429
|
+
function holdServes(run, entry, pending) {
|
|
430
|
+
return run.hold?.generation === pending.generation && holdsSettings(run.target.config, targetOfPending(entry, pending));
|
|
431
|
+
}
|
|
432
|
+
function startHeldRun(core, indexName, pending) {
|
|
433
|
+
const entry = core.requireIndex(indexName);
|
|
434
|
+
if (entry.analysisGeneration >= pending.generation) return null;
|
|
435
|
+
const runs = runsOf(core);
|
|
436
|
+
const previous = runs.get(indexName);
|
|
437
|
+
if (previous !== void 0 && holdServes(previous, entry, pending)) return previous;
|
|
438
|
+
const target = targetOfPending(entry, pending);
|
|
439
|
+
requireSynonymsUnderTarget(entry, target);
|
|
440
|
+
requireAnalysisRebuildMemory(indexName, termBytesOf(core, indexName));
|
|
441
|
+
const run = createRun(target, createHold(pending.generation));
|
|
442
|
+
if (previous !== void 0 && !previous.switched) stopRun(previous, indexName);
|
|
443
|
+
runs.set(indexName, run);
|
|
444
|
+
void driveRun(core, indexName, entry, run, previous, pending.generation);
|
|
445
|
+
return run;
|
|
446
|
+
}
|
|
447
|
+
async function buildHeldAnalysisChange(core, indexName, pending) {
|
|
448
|
+
const running = runsOf(core).get(indexName);
|
|
449
|
+
const entry = core.indexRegistry.get(indexName);
|
|
450
|
+
if (running?.hold != null && entry !== void 0 && holdServes(running, entry, pending)) return running.hold.ready;
|
|
451
|
+
const release = await core.indexState.acquire(indexName, false);
|
|
452
|
+
let run;
|
|
453
|
+
try {
|
|
454
|
+
run = startHeldRun(core, indexName, pending);
|
|
455
|
+
} catch (error) {
|
|
456
|
+
release();
|
|
457
|
+
throw error;
|
|
458
|
+
}
|
|
459
|
+
if (run?.hold === void 0 || run.hold === null) {
|
|
460
|
+
release();
|
|
461
|
+
return;
|
|
462
|
+
}
|
|
463
|
+
void run.settled.finally(release);
|
|
464
|
+
return run.hold.ready;
|
|
465
|
+
}
|
|
466
|
+
function heldAnalysisIsBuilt(core, indexName, generation) {
|
|
467
|
+
const hold = runsOf(core).get(indexName)?.hold;
|
|
468
|
+
return hold !== void 0 && hold !== null && hold.generation === generation && hold.built;
|
|
469
|
+
}
|
|
470
|
+
async function releaseHeldAnalysisChange(core, indexName, generation) {
|
|
471
|
+
const run = runsOf(core).get(indexName);
|
|
472
|
+
if (run?.hold?.generation !== generation) return;
|
|
473
|
+
run.hold.released = true;
|
|
474
|
+
run.hold.wake();
|
|
475
|
+
await run.finished;
|
|
476
|
+
}
|
|
477
|
+
async function dropHeldAnalysisChange(core, indexName) {
|
|
478
|
+
await abandonAnalysisChange(core, indexName);
|
|
479
|
+
const entry = core.indexRegistry.get(indexName);
|
|
480
|
+
if (entry !== void 0 && entry.pendingAnalysis !== null) await clearPending(core, indexName, entry);
|
|
481
|
+
}
|
|
482
|
+
function holdsNamedSettings(entry, settings) {
|
|
483
|
+
return holdsSettings(entry.config, targetOfPending(entry, { ...settings }));
|
|
484
|
+
}
|
|
485
|
+
async function stampAnalysisGeneration(core, indexName, generation) {
|
|
486
|
+
const entry = core.requireIndex(indexName);
|
|
487
|
+
if (entry.analysisGeneration >= generation) return;
|
|
488
|
+
await requireNoUndurableSwitch(core, indexName, entry);
|
|
489
|
+
const durable = core.durability?.manager;
|
|
490
|
+
const stamp = () => {
|
|
491
|
+
entry.durableAnalysis = core.durability === null ? null : durableSettingsOf(entry);
|
|
492
|
+
entry.analysisGeneration = generation;
|
|
493
|
+
entry.pendingAnalysis = null;
|
|
494
|
+
return true;
|
|
495
|
+
};
|
|
496
|
+
await (durable?.checkpointAfterSwitch?.(indexName, stamp) ?? Promise.resolve(stamp()));
|
|
497
|
+
}
|
|
498
|
+
|
|
499
|
+
export { abandonAnalysisChange, adoptSavedAnalysis, buildHeldAnalysisChange, changeIndexAnalysis, dropHeldAnalysisChange, heldAnalysisIsBuilt, holdsNamedSettings, releaseHeldAnalysisChange, resumeAnalysisChange, stampAnalysisGeneration };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { abandonChangeOf, resumeChangeOf, resumeEveryChange } from './chunk-E5SFTOQR.mjs';
|
|
@@ -40,6 +40,10 @@ var ErrorCodes = {
|
|
|
40
40
|
ENVELOPE_INVALID_MAGIC: "ENVELOPE_INVALID_MAGIC",
|
|
41
41
|
PARTITION_CAPACITY_EXCEEDED: "PARTITION_CAPACITY_EXCEEDED",
|
|
42
42
|
RESTORE_MEMORY_EXCEEDED: "RESTORE_MEMORY_EXCEEDED",
|
|
43
|
+
ANALYSIS_CHANGE_REPLACED: "ANALYSIS_CHANGE_REPLACED",
|
|
44
|
+
ANALYSIS_CHANGE_NOT_READY: "ANALYSIS_CHANGE_NOT_READY",
|
|
45
|
+
ANALYSIS_GENERATION_MISSING: "ANALYSIS_GENERATION_MISSING",
|
|
46
|
+
ANALYSIS_REBUILD_MEMORY_EXCEEDED: "ANALYSIS_REBUILD_MEMORY_EXCEEDED",
|
|
43
47
|
EMBEDDING_FAILED: "EMBEDDING_FAILED",
|
|
44
48
|
EMBEDDING_DIMENSION_MISMATCH: "EMBEDDING_DIMENSION_MISMATCH",
|
|
45
49
|
EMBEDDING_NO_SOURCE: "EMBEDDING_NO_SOURCE",
|