@delali/narsil 0.1.15 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/NOTICE +54 -0
- package/README.md +38 -982
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{chunk-WISGW24C.mjs → chunk-AUMKFYTG.mjs} +1 -1
- package/dist/{chunk-HRABJLPB.mjs → chunk-BKAI2WXI.mjs} +1 -0
- package/dist/chunk-GHJWC2E5.mjs +9 -0
- package/dist/chunk-KH3YGMVT.mjs +15 -0
- package/dist/{chunk-ALB35PRQ.mjs → chunk-M2EDG55Y.mjs} +8 -0
- package/dist/{chunk-HMO6OMZ3.mjs → chunk-S6IAXOBA.mjs} +1073 -506
- package/dist/chunk-TGP5M5R7.mjs +880 -0
- package/dist/chunk-U2BKYEC7.mjs +254 -0
- package/dist/{chunk-MEW7HD75.mjs → chunk-XAKNSWIK.mjs} +2543 -1154
- package/dist/{config-CojEPOgS.d.ts → config-P27AtSn4.d.ts} +28 -7
- package/dist/direct-executor-2FV3MEDD.mjs +5 -0
- package/dist/distribution/coordinator/etcd.d.ts +2 -2
- package/dist/distribution/coordinator/etcd.mjs +2 -2
- package/dist/distribution/coordinator/in-memory.d.ts +2 -2
- package/dist/distribution.d.ts +10 -4
- package/dist/distribution.mjs +1875 -1772
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/index.browser.mjs +4566 -1783
- package/dist/index.d.ts +14 -5
- package/dist/index.mjs +6 -5
- package/dist/{language-ONQ3iVDf.d.ts → language-CmJBHf1j.d.ts} +4 -1
- package/dist/languages/albanian.d.ts +5 -0
- package/dist/languages/albanian.mjs +103 -0
- package/dist/languages/amharic.d.ts +5 -0
- package/dist/languages/amharic.mjs +240 -0
- package/dist/languages/arabic.d.ts +1 -1
- package/dist/languages/arabic.mjs +939 -154
- package/dist/languages/armenian.d.ts +1 -1
- package/dist/languages/armenian.mjs +275 -243
- package/dist/languages/azerbaijani.d.ts +5 -0
- package/dist/languages/azerbaijani.mjs +152 -0
- package/dist/languages/bambara.d.ts +5 -0
- package/dist/languages/bambara.mjs +60 -0
- package/dist/languages/basque.d.ts +5 -0
- package/dist/languages/basque.mjs +726 -0
- package/dist/languages/belarusian.d.ts +5 -0
- package/dist/languages/belarusian.mjs +100 -0
- package/dist/languages/bengali.d.ts +5 -0
- package/dist/languages/bengali.mjs +128 -0
- package/dist/languages/bosnian.d.ts +5 -0
- package/dist/languages/bosnian.mjs +197 -0
- package/dist/languages/breton.d.ts +5 -0
- package/dist/languages/breton.mjs +96 -0
- package/dist/languages/bulgarian.d.ts +1 -1
- package/dist/languages/bulgarian.mjs +11 -2
- package/dist/languages/burmese.d.ts +5 -0
- package/dist/languages/burmese.mjs +57 -0
- package/dist/languages/catalan.d.ts +5 -0
- package/dist/languages/catalan.mjs +987 -0
- package/dist/languages/chinese.d.ts +1 -1
- package/dist/languages/chinese.mjs +4 -3
- package/dist/languages/croatian.d.ts +5 -0
- package/dist/languages/croatian.mjs +191 -0
- package/dist/languages/czech.d.ts +5 -0
- package/dist/languages/czech.mjs +628 -0
- package/dist/languages/dagbani.d.ts +1 -1
- package/dist/languages/dagbani.mjs +11 -2
- package/dist/languages/danish.d.ts +1 -1
- package/dist/languages/danish.mjs +215 -110
- package/dist/languages/dutch.d.ts +1 -1
- package/dist/languages/dutch.mjs +1248 -163
- package/dist/languages/english.d.ts +1 -1
- package/dist/languages/english.mjs +2 -1
- package/dist/languages/esperanto.d.ts +5 -0
- package/dist/languages/esperanto.mjs +689 -0
- package/dist/languages/estonian.d.ts +5 -0
- package/dist/languages/estonian.mjs +2160 -0
- package/dist/languages/ewe.d.ts +1 -1
- package/dist/languages/ewe.mjs +11 -2
- package/dist/languages/faroese.d.ts +5 -0
- package/dist/languages/faroese.mjs +93 -0
- package/dist/languages/fijian.d.ts +5 -0
- package/dist/languages/fijian.mjs +55 -0
- package/dist/languages/finnish.d.ts +1 -1
- package/dist/languages/finnish.mjs +528 -263
- package/dist/languages/french.d.ts +1 -1
- package/dist/languages/french.mjs +950 -437
- package/dist/languages/ga.d.ts +1 -1
- package/dist/languages/ga.mjs +2 -1
- package/dist/languages/galician.d.ts +5 -0
- package/dist/languages/galician.mjs +172 -0
- package/dist/languages/georgian.d.ts +5 -0
- package/dist/languages/georgian.mjs +91 -0
- package/dist/languages/german.d.ts +1 -1
- package/dist/languages/german.mjs +432 -162
- package/dist/languages/greek.d.ts +1 -1
- package/dist/languages/greek.mjs +1970 -281
- package/dist/languages/guarani.d.ts +5 -0
- package/dist/languages/guarani.mjs +132 -0
- package/dist/languages/gujarati.d.ts +5 -0
- package/dist/languages/gujarati.mjs +234 -0
- package/dist/languages/haitian-creole.d.ts +5 -0
- package/dist/languages/haitian-creole.mjs +106 -0
- package/dist/languages/hausa.d.ts +1 -1
- package/dist/languages/hausa.mjs +2 -1
- package/dist/languages/hawaiian.d.ts +5 -0
- package/dist/languages/hawaiian.mjs +69 -0
- package/dist/languages/hebrew.d.ts +5 -0
- package/dist/languages/hebrew.mjs +213 -0
- package/dist/languages/hindi.d.ts +1 -1
- package/dist/languages/hindi.mjs +175 -97
- package/dist/languages/hungarian.d.ts +1 -1
- package/dist/languages/hungarian.mjs +412 -326
- package/dist/languages/icelandic.d.ts +5 -0
- package/dist/languages/icelandic.mjs +161 -0
- package/dist/languages/igbo.d.ts +1 -1
- package/dist/languages/igbo.mjs +2 -1
- package/dist/languages/indonesian.d.ts +1 -1
- package/dist/languages/indonesian.mjs +329 -137
- package/dist/languages/irish.d.ts +1 -1
- package/dist/languages/irish.mjs +223 -139
- package/dist/languages/italian.d.ts +1 -1
- package/dist/languages/italian.mjs +703 -406
- package/dist/languages/japanese.d.ts +1 -1
- package/dist/languages/japanese.mjs +5 -108
- package/dist/languages/kannada.d.ts +5 -0
- package/dist/languages/kannada.mjs +94 -0
- package/dist/languages/kazakh.d.ts +5 -0
- package/dist/languages/kazakh.mjs +90 -0
- package/dist/languages/khmer.d.ts +5 -0
- package/dist/languages/khmer.mjs +181 -0
- package/dist/languages/kinyarwanda.d.ts +5 -0
- package/dist/languages/kinyarwanda.mjs +99 -0
- package/dist/languages/kirundi.d.ts +5 -0
- package/dist/languages/kirundi.mjs +92 -0
- package/dist/languages/korean.d.ts +5 -0
- package/dist/languages/korean.mjs +252 -0
- package/dist/languages/kurmanji.d.ts +5 -0
- package/dist/languages/kurmanji.mjs +50 -0
- package/dist/languages/kyrgyz.d.ts +5 -0
- package/dist/languages/kyrgyz.mjs +108 -0
- package/dist/languages/lao.d.ts +5 -0
- package/dist/languages/lao.mjs +63 -0
- package/dist/languages/latin.d.ts +5 -0
- package/dist/languages/latin.mjs +69 -0
- package/dist/languages/latvian.d.ts +5 -0
- package/dist/languages/latvian.mjs +173 -0
- package/dist/languages/lingala.d.ts +5 -0
- package/dist/languages/lingala.mjs +61 -0
- package/dist/languages/lithuanian.d.ts +5 -0
- package/dist/languages/lithuanian.mjs +571 -0
- package/dist/languages/luxembourgish.d.ts +5 -0
- package/dist/languages/luxembourgish.mjs +214 -0
- package/dist/languages/macedonian.d.ts +5 -0
- package/dist/languages/macedonian.mjs +808 -0
- package/dist/languages/malagasy.d.ts +5 -0
- package/dist/languages/malagasy.mjs +67 -0
- package/dist/languages/malay.d.ts +5 -0
- package/dist/languages/malay.mjs +482 -0
- package/dist/languages/malayalam.d.ts +5 -0
- package/dist/languages/malayalam.mjs +122 -0
- package/dist/languages/maltese.d.ts +5 -0
- package/dist/languages/maltese.mjs +139 -0
- package/dist/languages/maori.d.ts +5 -0
- package/dist/languages/maori.mjs +79 -0
- package/dist/languages/marathi.d.ts +5 -0
- package/dist/languages/marathi.mjs +111 -0
- package/dist/languages/nepali.d.ts +1 -1
- package/dist/languages/nepali.mjs +209 -99
- package/dist/languages/norwegian.d.ts +1 -1
- package/dist/languages/norwegian.mjs +218 -85
- package/dist/languages/oromo.d.ts +5 -0
- package/dist/languages/oromo.mjs +78 -0
- package/dist/languages/persian.d.ts +5 -0
- package/dist/languages/persian.mjs +710 -0
- package/dist/languages/polish.d.ts +5 -0
- package/dist/languages/polish.mjs +614 -0
- package/dist/languages/portuguese.d.ts +1 -1
- package/dist/languages/portuguese.mjs +613 -380
- package/dist/languages/punjabi.d.ts +5 -0
- package/dist/languages/punjabi.mjs +197 -0
- package/dist/languages/romanian.d.ts +1 -1
- package/dist/languages/romanian.mjs +561 -382
- package/dist/languages/russian.d.ts +1 -1
- package/dist/languages/russian.mjs +373 -244
- package/dist/languages/samoan.d.ts +5 -0
- package/dist/languages/samoan.mjs +63 -0
- package/dist/languages/sanskrit.d.ts +1 -1
- package/dist/languages/sanskrit.mjs +2 -2
- package/dist/languages/scottish-gaelic.d.ts +5 -0
- package/dist/languages/scottish-gaelic.mjs +370 -0
- package/dist/languages/serbian.d.ts +1 -1
- package/dist/languages/serbian.mjs +3635 -155
- package/dist/languages/shona.d.ts +5 -0
- package/dist/languages/shona.mjs +91 -0
- package/dist/languages/sinhala.d.ts +5 -0
- package/dist/languages/sinhala.mjs +182 -0
- package/dist/languages/slovak.d.ts +5 -0
- package/dist/languages/slovak.mjs +430 -0
- package/dist/languages/slovenian.d.ts +1 -1
- package/dist/languages/slovenian.mjs +11 -2
- package/dist/languages/sorani.d.ts +5 -0
- package/dist/languages/sorani.mjs +156 -0
- package/dist/languages/spanish.d.ts +1 -1
- package/dist/languages/spanish.mjs +676 -374
- package/dist/languages/swahili.d.ts +1 -1
- package/dist/languages/swahili.mjs +2 -1
- package/dist/languages/swedish.d.ts +1 -1
- package/dist/languages/swedish.mjs +237 -95
- package/dist/languages/tagalog.d.ts +5 -0
- package/dist/languages/tagalog.mjs +159 -0
- package/dist/languages/tamil.d.ts +1 -1
- package/dist/languages/tamil.mjs +836 -64
- package/dist/languages/tatar.d.ts +5 -0
- package/dist/languages/tatar.mjs +1017 -0
- package/dist/languages/telugu.d.ts +5 -0
- package/dist/languages/telugu.mjs +62 -0
- package/dist/languages/thai.d.ts +5 -0
- package/dist/languages/thai.mjs +65 -0
- package/dist/languages/tibetan.d.ts +5 -0
- package/dist/languages/tibetan.mjs +119 -0
- package/dist/languages/tigrinya.d.ts +5 -0
- package/dist/languages/tigrinya.mjs +194 -0
- package/dist/languages/tongan.d.ts +5 -0
- package/dist/languages/tongan.mjs +56 -0
- package/dist/languages/turkish.d.ts +1 -1
- package/dist/languages/turkish.mjs +1246 -393
- package/dist/languages/twi.d.ts +1 -1
- package/dist/languages/twi.mjs +13 -2
- package/dist/languages/ukrainian.d.ts +1 -1
- package/dist/languages/ukrainian.mjs +125 -47
- package/dist/languages/urdu.d.ts +5 -0
- package/dist/languages/urdu.mjs +528 -0
- package/dist/languages/vietnamese.d.ts +5 -0
- package/dist/languages/vietnamese.mjs +277 -0
- package/dist/languages/welsh.d.ts +5 -0
- package/dist/languages/welsh.mjs +446 -0
- package/dist/languages/wolof.d.ts +5 -0
- package/dist/languages/wolof.mjs +79 -0
- package/dist/languages/xhosa.d.ts +5 -0
- package/dist/languages/xhosa.mjs +95 -0
- package/dist/languages/yoruba.d.ts +1 -1
- package/dist/languages/yoruba.mjs +1 -0
- package/dist/languages/zulu.d.ts +1 -1
- package/dist/languages/zulu.mjs +2 -1
- package/dist/narsil-CMZRPDlQ.d.ts +159 -0
- package/dist/persistence/durability/checkpoint-worker.d.ts +13 -0
- package/dist/persistence/durability/checkpoint-worker.mjs +6 -5
- package/dist/{schema-DCgkO1j4.d.ts → schema-BnvLvjRp.d.ts} +5 -3
- package/dist/server.d.ts +4 -3
- package/dist/server.mjs +13 -2
- package/dist/{types-DHX6zGeD.d.ts → types-BRTcBgAp.d.ts} +1 -1
- package/dist/workers/entry.mjs +2 -2
- package/package.json +10 -3
- package/dist/chunk-TW6BXTOL.mjs +0 -338
- package/dist/direct-executor-E52FFGDE.mjs +0 -4
- package/dist/narsil-D-84Nvm4.d.ts +0 -72
package/dist/distribution.mjs
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
export { createInMemoryCoordinator } from './chunk-RQLXC2YS.mjs';
|
|
2
|
-
import './chunk-
|
|
3
|
-
import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-
|
|
4
|
-
import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-
|
|
2
|
+
import './chunk-AUMKFYTG.mjs';
|
|
3
|
+
import { ReplicationMessageTypes, ClusterMessageTypes, DEFAULT_TRANSPORT_CONFIG, TransportError, TransportErrorCodes, MAX_MESSAGE_SIZE_BYTES, createEngineCore, createNarsilFromCore, QueryMessageTypes, applyIndexEntry, applyDeleteEntry, validateEntryPayload, createAckMessage, validateReplicationEntry, MAX_PARTITION_COUNT, MAX_INDEX_NAME_LENGTH, validateIndexName, computeEntryChecksum, createInsyncConfirmMessage, createEntryMessage, getIndexMetadata, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, putIndexMetadata, createForwardMessage } from './chunk-XAKNSWIK.mjs';
|
|
4
|
+
import { deserializePayloadV2, createPartitionIndex, getLanguage, fnv1a, mergePartitionStats, MAX_NESTING_DEPTH } from './chunk-S6IAXOBA.mjs';
|
|
5
|
+
import './chunk-TGP5M5R7.mjs';
|
|
5
6
|
import './chunk-S7RJJMHI.mjs';
|
|
6
7
|
import { crc32 } from './chunk-KMMJMDUA.mjs';
|
|
7
|
-
import './chunk-
|
|
8
|
-
import { ErrorCodes, NarsilError } from './chunk-
|
|
9
|
-
import { generateId } from './chunk-
|
|
8
|
+
import './chunk-U2BKYEC7.mjs';
|
|
9
|
+
import { ErrorCodes, NarsilError } from './chunk-M2EDG55Y.mjs';
|
|
10
|
+
import { generateId } from './chunk-BKAI2WXI.mjs';
|
|
10
11
|
import { encode, decode } from '@msgpack/msgpack';
|
|
11
12
|
|
|
12
13
|
// src/distribution/cluster/controller/types.ts
|
|
@@ -1474,6 +1475,90 @@ function createDataNodeLifecycle(config) {
|
|
|
1474
1475
|
return handle;
|
|
1475
1476
|
}
|
|
1476
1477
|
|
|
1478
|
+
// src/distribution/query/cursor.ts
|
|
1479
|
+
var MAX_CURSOR_LENGTH = 4096;
|
|
1480
|
+
var MAX_CURSOR_DOC_ID_LENGTH = 512;
|
|
1481
|
+
function truncateCursorForError(encoded) {
|
|
1482
|
+
if (encoded.length > 100) {
|
|
1483
|
+
return `${encoded.slice(0, 100)}...`;
|
|
1484
|
+
}
|
|
1485
|
+
return encoded;
|
|
1486
|
+
}
|
|
1487
|
+
function encodeDistributedCursor(score, docId) {
|
|
1488
|
+
const json = JSON.stringify({ s: score, d: docId });
|
|
1489
|
+
if (typeof Buffer !== "undefined") {
|
|
1490
|
+
return Buffer.from(json).toString("base64");
|
|
1491
|
+
}
|
|
1492
|
+
return btoa(json);
|
|
1493
|
+
}
|
|
1494
|
+
function decodeDistributedCursor(encoded) {
|
|
1495
|
+
if (encoded.length > MAX_CURSOR_LENGTH) {
|
|
1496
|
+
throw new NarsilError(
|
|
1497
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1498
|
+
`Distributed cursor exceeds maximum length of ${MAX_CURSOR_LENGTH}`,
|
|
1499
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1500
|
+
);
|
|
1501
|
+
}
|
|
1502
|
+
let json;
|
|
1503
|
+
try {
|
|
1504
|
+
if (typeof Buffer !== "undefined") {
|
|
1505
|
+
json = Buffer.from(encoded, "base64").toString("utf-8");
|
|
1506
|
+
} else {
|
|
1507
|
+
json = atob(encoded);
|
|
1508
|
+
}
|
|
1509
|
+
} catch {
|
|
1510
|
+
throw new NarsilError(
|
|
1511
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1512
|
+
"Failed to decode distributed cursor: invalid base64 encoding",
|
|
1513
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1514
|
+
);
|
|
1515
|
+
}
|
|
1516
|
+
let parsed;
|
|
1517
|
+
try {
|
|
1518
|
+
parsed = JSON.parse(json);
|
|
1519
|
+
} catch {
|
|
1520
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode distributed cursor: invalid JSON", {
|
|
1521
|
+
cursor: truncateCursorForError(encoded)
|
|
1522
|
+
});
|
|
1523
|
+
}
|
|
1524
|
+
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1525
|
+
throw new NarsilError(
|
|
1526
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1527
|
+
'Invalid distributed cursor structure: expected an object with "s" and "d" fields',
|
|
1528
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1529
|
+
);
|
|
1530
|
+
}
|
|
1531
|
+
const record = parsed;
|
|
1532
|
+
const { s, d } = record;
|
|
1533
|
+
if (typeof s !== "number" || !Number.isFinite(s)) {
|
|
1534
|
+
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, 'Invalid distributed cursor: "s" must be a finite number', {
|
|
1535
|
+
cursor: truncateCursorForError(encoded)
|
|
1536
|
+
});
|
|
1537
|
+
}
|
|
1538
|
+
if (typeof d !== "string" || d.length === 0) {
|
|
1539
|
+
throw new NarsilError(
|
|
1540
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1541
|
+
'Invalid distributed cursor: "d" must be a non-empty string',
|
|
1542
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1543
|
+
);
|
|
1544
|
+
}
|
|
1545
|
+
if (d.length > MAX_CURSOR_DOC_ID_LENGTH) {
|
|
1546
|
+
throw new NarsilError(
|
|
1547
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1548
|
+
`Invalid distributed cursor: "d" exceeds maximum length of ${MAX_CURSOR_DOC_ID_LENGTH}`,
|
|
1549
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1550
|
+
);
|
|
1551
|
+
}
|
|
1552
|
+
if (d.includes("\0")) {
|
|
1553
|
+
throw new NarsilError(
|
|
1554
|
+
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1555
|
+
'Invalid distributed cursor: "d" must not contain null bytes',
|
|
1556
|
+
{ cursor: truncateCursorForError(encoded) }
|
|
1557
|
+
);
|
|
1558
|
+
}
|
|
1559
|
+
return { s, d };
|
|
1560
|
+
}
|
|
1561
|
+
|
|
1477
1562
|
// src/distribution/query/validators/common.ts
|
|
1478
1563
|
var MAX_PARTITION_IDS = MAX_PARTITION_COUNT;
|
|
1479
1564
|
var MAX_TERM_LENGTH = 1024;
|
|
@@ -1682,90 +1767,6 @@ function validateFetchResultPayload(decoded) {
|
|
|
1682
1767
|
return decoded;
|
|
1683
1768
|
}
|
|
1684
1769
|
|
|
1685
|
-
// src/distribution/query/cursor.ts
|
|
1686
|
-
var MAX_CURSOR_LENGTH = 4096;
|
|
1687
|
-
var MAX_CURSOR_DOC_ID_LENGTH = 512;
|
|
1688
|
-
function truncateCursorForError(encoded) {
|
|
1689
|
-
if (encoded.length > 100) {
|
|
1690
|
-
return `${encoded.slice(0, 100)}...`;
|
|
1691
|
-
}
|
|
1692
|
-
return encoded;
|
|
1693
|
-
}
|
|
1694
|
-
function encodeDistributedCursor(score, docId) {
|
|
1695
|
-
const json = JSON.stringify({ s: score, d: docId });
|
|
1696
|
-
if (typeof Buffer !== "undefined") {
|
|
1697
|
-
return Buffer.from(json).toString("base64");
|
|
1698
|
-
}
|
|
1699
|
-
return btoa(json);
|
|
1700
|
-
}
|
|
1701
|
-
function decodeDistributedCursor(encoded) {
|
|
1702
|
-
if (encoded.length > MAX_CURSOR_LENGTH) {
|
|
1703
|
-
throw new NarsilError(
|
|
1704
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1705
|
-
`Distributed cursor exceeds maximum length of ${MAX_CURSOR_LENGTH}`,
|
|
1706
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1707
|
-
);
|
|
1708
|
-
}
|
|
1709
|
-
let json;
|
|
1710
|
-
try {
|
|
1711
|
-
if (typeof Buffer !== "undefined") {
|
|
1712
|
-
json = Buffer.from(encoded, "base64").toString("utf-8");
|
|
1713
|
-
} else {
|
|
1714
|
-
json = atob(encoded);
|
|
1715
|
-
}
|
|
1716
|
-
} catch {
|
|
1717
|
-
throw new NarsilError(
|
|
1718
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1719
|
-
"Failed to decode distributed cursor: invalid base64 encoding",
|
|
1720
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1721
|
-
);
|
|
1722
|
-
}
|
|
1723
|
-
let parsed;
|
|
1724
|
-
try {
|
|
1725
|
-
parsed = JSON.parse(json);
|
|
1726
|
-
} catch {
|
|
1727
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, "Failed to decode distributed cursor: invalid JSON", {
|
|
1728
|
-
cursor: truncateCursorForError(encoded)
|
|
1729
|
-
});
|
|
1730
|
-
}
|
|
1731
|
-
if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
|
|
1732
|
-
throw new NarsilError(
|
|
1733
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1734
|
-
'Invalid distributed cursor structure: expected an object with "s" and "d" fields',
|
|
1735
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1736
|
-
);
|
|
1737
|
-
}
|
|
1738
|
-
const record = parsed;
|
|
1739
|
-
const { s, d } = record;
|
|
1740
|
-
if (typeof s !== "number" || !Number.isFinite(s)) {
|
|
1741
|
-
throw new NarsilError(ErrorCodes.SEARCH_INVALID_CURSOR, 'Invalid distributed cursor: "s" must be a finite number', {
|
|
1742
|
-
cursor: truncateCursorForError(encoded)
|
|
1743
|
-
});
|
|
1744
|
-
}
|
|
1745
|
-
if (typeof d !== "string" || d.length === 0) {
|
|
1746
|
-
throw new NarsilError(
|
|
1747
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1748
|
-
'Invalid distributed cursor: "d" must be a non-empty string',
|
|
1749
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1750
|
-
);
|
|
1751
|
-
}
|
|
1752
|
-
if (d.length > MAX_CURSOR_DOC_ID_LENGTH) {
|
|
1753
|
-
throw new NarsilError(
|
|
1754
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1755
|
-
`Invalid distributed cursor: "d" exceeds maximum length of ${MAX_CURSOR_DOC_ID_LENGTH}`,
|
|
1756
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1757
|
-
);
|
|
1758
|
-
}
|
|
1759
|
-
if (d.includes("\0")) {
|
|
1760
|
-
throw new NarsilError(
|
|
1761
|
-
ErrorCodes.SEARCH_INVALID_CURSOR,
|
|
1762
|
-
'Invalid distributed cursor: "d" must not contain null bytes',
|
|
1763
|
-
{ cursor: truncateCursorForError(encoded) }
|
|
1764
|
-
);
|
|
1765
|
-
}
|
|
1766
|
-
return { s, d };
|
|
1767
|
-
}
|
|
1768
|
-
|
|
1769
1770
|
// src/distribution/query/validators/filters.ts
|
|
1770
1771
|
var COMPARISON_NUMERIC_OPS = ["gt", "lt", "gte", "lte"];
|
|
1771
1772
|
var COMPARISON_EQ_OPS = ["eq", "ne"];
|
|
@@ -2493,6 +2494,8 @@ function createStatsMessage(payload, sourceId) {
|
|
|
2493
2494
|
payload: encode(payload)
|
|
2494
2495
|
};
|
|
2495
2496
|
}
|
|
2497
|
+
|
|
2498
|
+
// src/distribution/query/fan-out.ts
|
|
2496
2499
|
async function collectDistributedStats(indexName, params, nodeToPartitions, deps, config) {
|
|
2497
2500
|
const terms = params.term !== null ? params.term.split(/\s+/).filter((t) => t.length > 0) : [];
|
|
2498
2501
|
const statsPromises = [];
|
|
@@ -2849,8 +2852,8 @@ function mergeDistributedFacets(allFacets, maxBuckets = DEFAULT_MAX_FACET_BUCKET
|
|
|
2849
2852
|
}
|
|
2850
2853
|
|
|
2851
2854
|
// src/distribution/query/selection.ts
|
|
2852
|
-
function
|
|
2853
|
-
return candidates[
|
|
2855
|
+
function randomSelector(candidates, _partitionId) {
|
|
2856
|
+
return candidates[Math.floor(Math.random() * candidates.length)];
|
|
2854
2857
|
}
|
|
2855
2858
|
function collectActiveCandidates(assignment) {
|
|
2856
2859
|
if (assignment.state !== "ACTIVE") {
|
|
@@ -2868,7 +2871,7 @@ function collectActiveCandidates(assignment) {
|
|
|
2868
2871
|
candidates.sort();
|
|
2869
2872
|
return candidates;
|
|
2870
2873
|
}
|
|
2871
|
-
function selectReplica(assignment, localNodeId, selector =
|
|
2874
|
+
function selectReplica(assignment, localNodeId, selector = randomSelector, partitionId = 0) {
|
|
2872
2875
|
const candidates = collectActiveCandidates(assignment);
|
|
2873
2876
|
if (candidates.length === 0) {
|
|
2874
2877
|
return null;
|
|
@@ -2878,7 +2881,7 @@ function selectReplica(assignment, localNodeId, selector = hashBasedSelector, pa
|
|
|
2878
2881
|
}
|
|
2879
2882
|
return selector(candidates, partitionId);
|
|
2880
2883
|
}
|
|
2881
|
-
function selectReplicasForQuery(allocationTable, localNodeId, selector =
|
|
2884
|
+
function selectReplicasForQuery(allocationTable, localNodeId, selector = randomSelector) {
|
|
2882
2885
|
const nodeToPartitions = /* @__PURE__ */ new Map();
|
|
2883
2886
|
const unavailablePartitions = [];
|
|
2884
2887
|
for (const [partitionId, assignment] of allocationTable.assignments) {
|
|
@@ -2936,7 +2939,7 @@ async function distributedQuery(indexName, params, deps, config, selector) {
|
|
|
2936
2939
|
coverage: { totalPartitions: 0, queriedPartitions: 0, timedOutPartitions: 0, failedPartitions: 0 }
|
|
2937
2940
|
};
|
|
2938
2941
|
}
|
|
2939
|
-
const routing = selectReplicasForQuery(allocationTable, deps.sourceNodeId,
|
|
2942
|
+
const routing = selectReplicasForQuery(allocationTable, deps.sourceNodeId, randomSelector);
|
|
2940
2943
|
const totalPartitions = allocationTable.assignments.size;
|
|
2941
2944
|
if (routing.unavailablePartitions.length > 0 && !resolvedConfig.allowPartialResults) {
|
|
2942
2945
|
throw new NarsilError("QUERY_NO_ACTIVE_REPLICA", "No active replica for one or more partitions", {
|
|
@@ -3100,242 +3103,53 @@ function worstCaseCoverage(a, b) {
|
|
|
3100
3103
|
};
|
|
3101
3104
|
}
|
|
3102
3105
|
|
|
3103
|
-
// src/distribution/
|
|
3104
|
-
|
|
3105
|
-
|
|
3106
|
-
|
|
3107
|
-
|
|
3108
|
-
|
|
3109
|
-
|
|
3106
|
+
// src/distribution/cluster-node/bootstrap-cleanup.ts
|
|
3107
|
+
async function cleanupRemovedPartition(indexName, partitionId, deps) {
|
|
3108
|
+
try {
|
|
3109
|
+
const existing = deps.engine.listIndexes().find((idx) => idx.name === indexName);
|
|
3110
|
+
if (existing === void 0) {
|
|
3111
|
+
return;
|
|
3112
|
+
}
|
|
3113
|
+
if (await nodeHasOtherPartitions(indexName, partitionId, deps)) {
|
|
3114
|
+
return;
|
|
3115
|
+
}
|
|
3116
|
+
await deps.engine.dropIndex(indexName);
|
|
3117
|
+
} catch (err) {
|
|
3118
|
+
if (deps.onError === void 0) {
|
|
3119
|
+
return;
|
|
3120
|
+
}
|
|
3121
|
+
if (err instanceof NarsilError && err.code === ErrorCodes.INDEX_NOT_FOUND) {
|
|
3122
|
+
return;
|
|
3123
|
+
}
|
|
3124
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
3125
|
+
deps.onError(
|
|
3126
|
+
new NarsilError(
|
|
3127
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
3128
|
+
`failed to drop local index '${indexName}' after partition removal: ${cause}`,
|
|
3129
|
+
{ indexName, partitionId, cause }
|
|
3130
|
+
)
|
|
3131
|
+
);
|
|
3132
|
+
}
|
|
3110
3133
|
}
|
|
3111
|
-
|
|
3112
|
-
|
|
3113
|
-
|
|
3114
|
-
|
|
3115
|
-
|
|
3116
|
-
|
|
3117
|
-
let entries = [];
|
|
3118
|
-
let headIndex = 0;
|
|
3119
|
-
let totalSizeBytes = 0;
|
|
3120
|
-
function liveCount() {
|
|
3121
|
-
return entries.length - headIndex;
|
|
3134
|
+
async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
|
|
3135
|
+
let allocation;
|
|
3136
|
+
try {
|
|
3137
|
+
allocation = await deps.coordinator.getAllocation(indexName);
|
|
3138
|
+
} catch (_) {
|
|
3139
|
+
return true;
|
|
3122
3140
|
}
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
entries = entries.slice(headIndex);
|
|
3126
|
-
headIndex = 0;
|
|
3127
|
-
}
|
|
3141
|
+
if (allocation === null) {
|
|
3142
|
+
return false;
|
|
3128
3143
|
}
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
while (low <= high) {
|
|
3133
|
-
const mid = low + high >>> 1;
|
|
3134
|
-
const midSeqNo = entries[mid].seqNo;
|
|
3135
|
-
if (midSeqNo < seqNo) {
|
|
3136
|
-
low = mid + 1;
|
|
3137
|
-
} else if (midSeqNo > seqNo) {
|
|
3138
|
-
high = mid - 1;
|
|
3139
|
-
} else {
|
|
3140
|
-
return mid;
|
|
3141
|
-
}
|
|
3144
|
+
for (const [partitionId, assignment] of allocation.assignments) {
|
|
3145
|
+
if (partitionId === removedPartitionId) {
|
|
3146
|
+
continue;
|
|
3142
3147
|
}
|
|
3143
|
-
|
|
3144
|
-
|
|
3145
|
-
function evictOldEntries() {
|
|
3146
|
-
while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
|
|
3147
|
-
totalSizeBytes -= estimateEntrySize(entries[headIndex]);
|
|
3148
|
-
entries[headIndex] = null;
|
|
3149
|
-
headIndex += 1;
|
|
3148
|
+
if (assignment.primary === deps.nodeId) {
|
|
3149
|
+
return true;
|
|
3150
3150
|
}
|
|
3151
|
-
if (
|
|
3152
|
-
|
|
3153
|
-
}
|
|
3154
|
-
if (headIndex >= 256 || headIndex > liveCount()) {
|
|
3155
|
-
compact();
|
|
3156
|
-
}
|
|
3157
|
-
}
|
|
3158
|
-
function storeEntry(entry) {
|
|
3159
|
-
entries.push(entry);
|
|
3160
|
-
if (entry.seqNo >= committedSeqNo) {
|
|
3161
|
-
committedSeqNo = entry.seqNo;
|
|
3162
|
-
committedPrimaryTerm = entry.primaryTerm;
|
|
3163
|
-
}
|
|
3164
|
-
totalSizeBytes += estimateEntrySize(entry);
|
|
3165
|
-
evictOldEntries();
|
|
3166
|
-
return { ...entry };
|
|
3167
|
-
}
|
|
3168
|
-
function readEntry(seqNo) {
|
|
3169
|
-
if (liveCount() === 0) return void 0;
|
|
3170
|
-
const idx = findIndexBySeqNo(seqNo);
|
|
3171
|
-
if (idx < entries.length && entries[idx].seqNo === seqNo) {
|
|
3172
|
-
return { ...entries[idx] };
|
|
3173
|
-
}
|
|
3174
|
-
return void 0;
|
|
3175
|
-
}
|
|
3176
|
-
function verifyEntryChecksum(entry) {
|
|
3177
|
-
const expected = computeChecksum({
|
|
3178
|
-
seqNo: entry.seqNo,
|
|
3179
|
-
primaryTerm: entry.primaryTerm,
|
|
3180
|
-
operation: entry.operation,
|
|
3181
|
-
partitionId: entry.partitionId,
|
|
3182
|
-
indexName: entry.indexName,
|
|
3183
|
-
documentId: entry.documentId,
|
|
3184
|
-
document: entry.document
|
|
3185
|
-
});
|
|
3186
|
-
return expected === entry.checksum;
|
|
3187
|
-
}
|
|
3188
|
-
return {
|
|
3189
|
-
append(partial) {
|
|
3190
|
-
const seqNo = nextSeqNo;
|
|
3191
|
-
nextSeqNo += 1;
|
|
3192
|
-
const entryWithoutChecksum = {
|
|
3193
|
-
seqNo,
|
|
3194
|
-
primaryTerm: partial.primaryTerm,
|
|
3195
|
-
operation: partial.operation,
|
|
3196
|
-
partitionId,
|
|
3197
|
-
indexName: partial.indexName,
|
|
3198
|
-
documentId: partial.documentId,
|
|
3199
|
-
document: partial.document
|
|
3200
|
-
};
|
|
3201
|
-
const checksum = computeChecksum(entryWithoutChecksum);
|
|
3202
|
-
const entry = { ...entryWithoutChecksum, checksum };
|
|
3203
|
-
return storeEntry(entry);
|
|
3204
|
-
},
|
|
3205
|
-
appendCommitted(entry) {
|
|
3206
|
-
if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
|
|
3207
|
-
throw new NarsilError(
|
|
3208
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
3209
|
-
`Invalid replication sequence number ${entry.seqNo}`,
|
|
3210
|
-
{
|
|
3211
|
-
seqNo: entry.seqNo,
|
|
3212
|
-
partitionId
|
|
3213
|
-
}
|
|
3214
|
-
);
|
|
3215
|
-
}
|
|
3216
|
-
if (entry.partitionId !== partitionId) {
|
|
3217
|
-
throw new NarsilError(
|
|
3218
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
3219
|
-
`Replication entry partition ${entry.partitionId} does not match log partition ${partitionId}`,
|
|
3220
|
-
{ entryPartitionId: entry.partitionId, logPartitionId: partitionId }
|
|
3221
|
-
);
|
|
3222
|
-
}
|
|
3223
|
-
const existing = readEntry(entry.seqNo);
|
|
3224
|
-
if (existing !== void 0) {
|
|
3225
|
-
if (existing.checksum !== entry.checksum) {
|
|
3226
|
-
throw new NarsilError(
|
|
3227
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
3228
|
-
`Conflicting replication entry at sequence number ${entry.seqNo}`,
|
|
3229
|
-
{ seqNo: entry.seqNo, partitionId }
|
|
3230
|
-
);
|
|
3231
|
-
}
|
|
3232
|
-
return existing;
|
|
3233
|
-
}
|
|
3234
|
-
if (entry.seqNo !== nextSeqNo) {
|
|
3235
|
-
throw new NarsilError(
|
|
3236
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
3237
|
-
`Out-of-order replication entry ${entry.seqNo}; expected ${nextSeqNo}`,
|
|
3238
|
-
{ seqNo: entry.seqNo, expectedSeqNo: nextSeqNo, partitionId }
|
|
3239
|
-
);
|
|
3240
|
-
}
|
|
3241
|
-
if (!verifyEntryChecksum(entry)) {
|
|
3242
|
-
throw new NarsilError(
|
|
3243
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
3244
|
-
`Invalid checksum for replication entry ${entry.seqNo}`,
|
|
3245
|
-
{ seqNo: entry.seqNo, partitionId }
|
|
3246
|
-
);
|
|
3247
|
-
}
|
|
3248
|
-
nextSeqNo = entry.seqNo + 1;
|
|
3249
|
-
return storeEntry({ ...entry });
|
|
3250
|
-
},
|
|
3251
|
-
getEntriesFrom(fromSeqNo) {
|
|
3252
|
-
if (liveCount() === 0) return [];
|
|
3253
|
-
const idx = findIndexBySeqNo(fromSeqNo);
|
|
3254
|
-
if (idx >= entries.length) return [];
|
|
3255
|
-
return entries.slice(idx).map((e) => ({ ...e }));
|
|
3256
|
-
},
|
|
3257
|
-
getEntry(seqNo) {
|
|
3258
|
-
return readEntry(seqNo);
|
|
3259
|
-
},
|
|
3260
|
-
verifyChecksum(entry) {
|
|
3261
|
-
return verifyEntryChecksum(entry);
|
|
3262
|
-
},
|
|
3263
|
-
get committedSeqNo() {
|
|
3264
|
-
return committedSeqNo;
|
|
3265
|
-
},
|
|
3266
|
-
get committedPrimaryTerm() {
|
|
3267
|
-
return committedPrimaryTerm;
|
|
3268
|
-
},
|
|
3269
|
-
get oldestSeqNo() {
|
|
3270
|
-
if (liveCount() === 0) return void 0;
|
|
3271
|
-
return entries[headIndex].seqNo;
|
|
3272
|
-
},
|
|
3273
|
-
get newestSeqNo() {
|
|
3274
|
-
if (liveCount() === 0) return void 0;
|
|
3275
|
-
return entries[entries.length - 1].seqNo;
|
|
3276
|
-
},
|
|
3277
|
-
get entryCount() {
|
|
3278
|
-
return liveCount();
|
|
3279
|
-
},
|
|
3280
|
-
get sizeBytes() {
|
|
3281
|
-
return totalSizeBytes;
|
|
3282
|
-
},
|
|
3283
|
-
clear() {
|
|
3284
|
-
entries = [];
|
|
3285
|
-
headIndex = 0;
|
|
3286
|
-
totalSizeBytes = 0;
|
|
3287
|
-
committedSeqNo = nextSeqNo - 1;
|
|
3288
|
-
}
|
|
3289
|
-
};
|
|
3290
|
-
}
|
|
3291
|
-
|
|
3292
|
-
// src/distribution/cluster-node/bootstrap-cleanup.ts
|
|
3293
|
-
async function cleanupRemovedPartition(indexName, partitionId, deps) {
|
|
3294
|
-
try {
|
|
3295
|
-
const existing = deps.engine.listIndexes().find((idx) => idx.name === indexName);
|
|
3296
|
-
if (existing === void 0) {
|
|
3297
|
-
return;
|
|
3298
|
-
}
|
|
3299
|
-
if (await nodeHasOtherPartitions(indexName, partitionId, deps)) {
|
|
3300
|
-
return;
|
|
3301
|
-
}
|
|
3302
|
-
await deps.engine.dropIndex(indexName);
|
|
3303
|
-
} catch (err) {
|
|
3304
|
-
if (deps.onError === void 0) {
|
|
3305
|
-
return;
|
|
3306
|
-
}
|
|
3307
|
-
if (err instanceof NarsilError && err.code === ErrorCodes.INDEX_NOT_FOUND) {
|
|
3308
|
-
return;
|
|
3309
|
-
}
|
|
3310
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
3311
|
-
deps.onError(
|
|
3312
|
-
new NarsilError(
|
|
3313
|
-
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
3314
|
-
`failed to drop local index '${indexName}' after partition removal: ${cause}`,
|
|
3315
|
-
{ indexName, partitionId, cause }
|
|
3316
|
-
)
|
|
3317
|
-
);
|
|
3318
|
-
}
|
|
3319
|
-
}
|
|
3320
|
-
async function nodeHasOtherPartitions(indexName, removedPartitionId, deps) {
|
|
3321
|
-
let allocation;
|
|
3322
|
-
try {
|
|
3323
|
-
allocation = await deps.coordinator.getAllocation(indexName);
|
|
3324
|
-
} catch (_) {
|
|
3325
|
-
return true;
|
|
3326
|
-
}
|
|
3327
|
-
if (allocation === null) {
|
|
3328
|
-
return false;
|
|
3329
|
-
}
|
|
3330
|
-
for (const [partitionId, assignment] of allocation.assignments) {
|
|
3331
|
-
if (partitionId === removedPartitionId) {
|
|
3332
|
-
continue;
|
|
3333
|
-
}
|
|
3334
|
-
if (assignment.primary === deps.nodeId) {
|
|
3335
|
-
return true;
|
|
3336
|
-
}
|
|
3337
|
-
if (assignment.replicas.includes(deps.nodeId)) {
|
|
3338
|
-
return true;
|
|
3151
|
+
if (assignment.replicas.includes(deps.nodeId)) {
|
|
3152
|
+
return true;
|
|
3339
3153
|
}
|
|
3340
3154
|
}
|
|
3341
3155
|
return false;
|
|
@@ -3662,6 +3476,8 @@ function toStreamFailure(err, defaultCode, defaultMessage) {
|
|
|
3662
3476
|
details: { cause: message }
|
|
3663
3477
|
};
|
|
3664
3478
|
}
|
|
3479
|
+
|
|
3480
|
+
// src/distribution/cluster-node/bootstrap-fetch.ts
|
|
3665
3481
|
var CAPACITY_EXHAUSTED_BACKOFF_BASE_MS = 100;
|
|
3666
3482
|
var CAPACITY_EXHAUSTED_BACKOFF_MAX_MS = 500;
|
|
3667
3483
|
var RETRY_ANY_TARGET_CODES = /* @__PURE__ */ new Set([
|
|
@@ -3963,10 +3779,10 @@ function validateArguments(indexName, partitionId, primaryNodeId) {
|
|
|
3963
3779
|
}
|
|
3964
3780
|
return null;
|
|
3965
3781
|
}
|
|
3966
|
-
async function resolveTransportTargets(indexName, primaryNodeId,
|
|
3782
|
+
async function resolveTransportTargets(indexName, primaryNodeId, resolveNodeTargets3) {
|
|
3967
3783
|
let targets;
|
|
3968
3784
|
try {
|
|
3969
|
-
targets = await
|
|
3785
|
+
targets = await resolveNodeTargets3(primaryNodeId);
|
|
3970
3786
|
} catch (err) {
|
|
3971
3787
|
const cause = err instanceof Error ? err.message : String(err);
|
|
3972
3788
|
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_NO_TARGETS, `resolveNodeTargets failed: ${cause}`, {
|
|
@@ -4128,7 +3944,7 @@ async function dropExistingIndex(engine, indexName, primaryNodeId) {
|
|
|
4128
3944
|
}
|
|
4129
3945
|
}
|
|
4130
3946
|
|
|
4131
|
-
// src/distribution/cluster-node/bootstrap-sync.ts
|
|
3947
|
+
// src/distribution/cluster-node/bootstrap-sync/state.ts
|
|
4132
3948
|
var DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS = 6e5;
|
|
4133
3949
|
function createBootstrapSyncState() {
|
|
4134
3950
|
return {
|
|
@@ -4160,40 +3976,6 @@ function clearBootstrapSyncIndex(state, indexName, partitionId) {
|
|
|
4160
3976
|
function hasCompletedBootstrapSync(state, indexName, partitionId) {
|
|
4161
3977
|
return state.completed.has(entryKey(indexName, partitionId));
|
|
4162
3978
|
}
|
|
4163
|
-
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
4164
|
-
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
4165
|
-
if (validationError !== null) {
|
|
4166
|
-
surfaceError(deps, indexName, primaryNodeId, validationError);
|
|
4167
|
-
return false;
|
|
4168
|
-
}
|
|
4169
|
-
const key = entryKey(indexName, partitionId);
|
|
4170
|
-
if (state.completed.has(key)) {
|
|
4171
|
-
return true;
|
|
4172
|
-
}
|
|
4173
|
-
const existing = state.inFlight.get(key);
|
|
4174
|
-
if (existing !== void 0) {
|
|
4175
|
-
const result = await Promise.race([existing.promise, existing.abortPromise]);
|
|
4176
|
-
if (result === ABORT_SENTINEL) {
|
|
4177
|
-
return false;
|
|
4178
|
-
}
|
|
4179
|
-
return result;
|
|
4180
|
-
}
|
|
4181
|
-
const entry = createEntry(state, indexName, partitionId, key);
|
|
4182
|
-
entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
|
|
4183
|
-
const current = state.inFlight.get(key);
|
|
4184
|
-
if (current === entry) {
|
|
4185
|
-
state.inFlight.delete(key);
|
|
4186
|
-
}
|
|
4187
|
-
if (!state.completed.has(key) && !state.inFlight.has(key)) {
|
|
4188
|
-
const currentGen = state.generations.get(key);
|
|
4189
|
-
if (currentGen !== void 0 && currentGen !== entry.generation) {
|
|
4190
|
-
state.generations.delete(key);
|
|
4191
|
-
}
|
|
4192
|
-
}
|
|
4193
|
-
});
|
|
4194
|
-
state.inFlight.set(key, entry);
|
|
4195
|
-
return entry.promise;
|
|
4196
|
-
}
|
|
4197
3979
|
function createEntry(state, indexName, partitionId, key) {
|
|
4198
3980
|
const startGeneration = state.generations.get(key) ?? 0;
|
|
4199
3981
|
let abortResolve = () => {
|
|
@@ -4211,401 +3993,100 @@ function createEntry(state, indexName, partitionId, key) {
|
|
|
4211
3993
|
abortPromise
|
|
4212
3994
|
};
|
|
4213
3995
|
}
|
|
4214
|
-
|
|
4215
|
-
const
|
|
4216
|
-
|
|
4217
|
-
const key = entryKey(indexName, partitionId);
|
|
4218
|
-
const abortCheck = () => {
|
|
4219
|
-
const currentGeneration = state.generations.get(key) ?? 0;
|
|
4220
|
-
if (currentGeneration !== entry.generation) {
|
|
4221
|
-
entry.aborted = true;
|
|
4222
|
-
}
|
|
4223
|
-
return entry.aborted;
|
|
4224
|
-
};
|
|
4225
|
-
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
4226
|
-
return executeLiveBootstrapSync(
|
|
4227
|
-
state,
|
|
4228
|
-
entry,
|
|
4229
|
-
indexName,
|
|
4230
|
-
partitionId,
|
|
4231
|
-
primaryNodeId,
|
|
4232
|
-
deps,
|
|
4233
|
-
deadlineMs,
|
|
4234
|
-
deadline,
|
|
4235
|
-
abortCheck
|
|
4236
|
-
);
|
|
4237
|
-
}
|
|
4238
|
-
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
4239
|
-
if (schemaResult === "aborted") {
|
|
4240
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4241
|
-
return false;
|
|
4242
|
-
}
|
|
4243
|
-
if (schemaResult === null) {
|
|
3996
|
+
function anotherBootstrapOwnsKey(state, key, self) {
|
|
3997
|
+
const current = state.inFlight.get(key);
|
|
3998
|
+
if (current === void 0) {
|
|
4244
3999
|
return false;
|
|
4245
4000
|
}
|
|
4246
|
-
|
|
4247
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4248
|
-
return false;
|
|
4249
|
-
}
|
|
4250
|
-
if (Date.now() >= deadline) {
|
|
4251
|
-
surfaceError(
|
|
4252
|
-
deps,
|
|
4253
|
-
indexName,
|
|
4254
|
-
primaryNodeId,
|
|
4255
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
4256
|
-
indexName,
|
|
4257
|
-
deadlineMs
|
|
4258
|
-
})
|
|
4259
|
-
);
|
|
4260
|
-
return false;
|
|
4261
|
-
}
|
|
4262
|
-
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
4263
|
-
if (targetsResult instanceof NarsilError) {
|
|
4264
|
-
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
4265
|
-
return false;
|
|
4266
|
-
}
|
|
4267
|
-
const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
|
|
4268
|
-
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
4269
|
-
indexName,
|
|
4270
|
-
primaryNodeId,
|
|
4271
|
-
targetsResult,
|
|
4272
|
-
deadline,
|
|
4273
|
-
fetchDeps,
|
|
4274
|
-
abortCheck,
|
|
4275
|
-
partitionId
|
|
4276
|
-
);
|
|
4277
|
-
if (!fetchResult.ok) {
|
|
4278
|
-
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
4279
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4280
|
-
return false;
|
|
4281
|
-
}
|
|
4282
|
-
surfaceError(
|
|
4283
|
-
deps,
|
|
4284
|
-
indexName,
|
|
4285
|
-
primaryNodeId,
|
|
4286
|
-
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
4287
|
-
indexName,
|
|
4288
|
-
primaryNodeId,
|
|
4289
|
-
...fetchResult.details
|
|
4290
|
-
})
|
|
4291
|
-
);
|
|
4292
|
-
return false;
|
|
4293
|
-
}
|
|
4294
|
-
if (abortCheck()) {
|
|
4295
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4296
|
-
return false;
|
|
4297
|
-
}
|
|
4298
|
-
const restoreSucceeded = await applyRestore(
|
|
4299
|
-
state,
|
|
4300
|
-
entry,
|
|
4301
|
-
indexName,
|
|
4302
|
-
primaryNodeId,
|
|
4303
|
-
fetchResult.bytes,
|
|
4304
|
-
schemaResult,
|
|
4305
|
-
deadline,
|
|
4306
|
-
deps
|
|
4307
|
-
);
|
|
4308
|
-
if (!restoreSucceeded) {
|
|
4309
|
-
return false;
|
|
4310
|
-
}
|
|
4311
|
-
if (abortCheck()) {
|
|
4312
|
-
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
4313
|
-
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
4314
|
-
}
|
|
4315
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4316
|
-
return false;
|
|
4317
|
-
}
|
|
4318
|
-
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
4319
|
-
state.completed.add(key);
|
|
4320
|
-
return true;
|
|
4321
|
-
}
|
|
4322
|
-
async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
|
|
4323
|
-
try {
|
|
4324
|
-
const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
|
|
4325
|
-
if (winner === ABORT_SENTINEL) {
|
|
4326
|
-
return "aborted";
|
|
4327
|
-
}
|
|
4328
|
-
const allocation = winner;
|
|
4329
|
-
if (allocation === null) {
|
|
4330
|
-
return {
|
|
4331
|
-
error: new NarsilError(
|
|
4332
|
-
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
4333
|
-
`coordinator has no allocation for index '${indexName}'`,
|
|
4334
|
-
{ indexName, primaryNodeId }
|
|
4335
|
-
)
|
|
4336
|
-
};
|
|
4337
|
-
}
|
|
4338
|
-
const assignment = allocation.assignments.get(partitionId);
|
|
4339
|
-
if (assignment === void 0) {
|
|
4340
|
-
return {
|
|
4341
|
-
error: new NarsilError(
|
|
4342
|
-
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
4343
|
-
`No assignment exists for partition ${partitionId} of index '${indexName}'`,
|
|
4344
|
-
{ indexName, partitionId, primaryNodeId }
|
|
4345
|
-
)
|
|
4346
|
-
};
|
|
4347
|
-
}
|
|
4348
|
-
if (assignment.primary !== primaryNodeId) {
|
|
4349
|
-
return {
|
|
4350
|
-
error: new NarsilError(
|
|
4351
|
-
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
4352
|
-
`Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
|
|
4353
|
-
{ indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
|
|
4354
|
-
)
|
|
4355
|
-
};
|
|
4356
|
-
}
|
|
4357
|
-
return { partitionCount: allocation.assignments.size };
|
|
4358
|
-
} catch (err) {
|
|
4359
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
4360
|
-
return {
|
|
4361
|
-
error: new NarsilError(
|
|
4362
|
-
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
4363
|
-
`coordinator.getAllocation failed: ${cause}`,
|
|
4364
|
-
{
|
|
4365
|
-
indexName,
|
|
4366
|
-
partitionId,
|
|
4367
|
-
primaryNodeId,
|
|
4368
|
-
cause
|
|
4369
|
-
}
|
|
4370
|
-
)
|
|
4371
|
-
};
|
|
4372
|
-
}
|
|
4001
|
+
return current !== self;
|
|
4373
4002
|
}
|
|
4374
|
-
|
|
4375
|
-
|
|
4376
|
-
|
|
4377
|
-
|
|
4378
|
-
|
|
4379
|
-
|
|
4380
|
-
if ("error" in schemaResult) {
|
|
4381
|
-
surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
|
|
4382
|
-
return false;
|
|
4383
|
-
}
|
|
4384
|
-
if (abortCheck()) {
|
|
4385
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4386
|
-
return false;
|
|
4387
|
-
}
|
|
4388
|
-
const allocationResult = await resolveLivePartitionCount(
|
|
4389
|
-
deps.coordinator,
|
|
4390
|
-
indexName,
|
|
4391
|
-
partitionId,
|
|
4392
|
-
primaryNodeId,
|
|
4393
|
-
entry.abortPromise
|
|
4394
|
-
);
|
|
4395
|
-
if (allocationResult === "aborted") {
|
|
4396
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4397
|
-
return false;
|
|
4398
|
-
}
|
|
4399
|
-
if ("error" in allocationResult) {
|
|
4400
|
-
surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
|
|
4401
|
-
return false;
|
|
4402
|
-
}
|
|
4403
|
-
if (Date.now() >= deadline) {
|
|
4404
|
-
surfaceError(
|
|
4405
|
-
deps,
|
|
4406
|
-
indexName,
|
|
4407
|
-
primaryNodeId,
|
|
4408
|
-
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
|
|
4409
|
-
indexName,
|
|
4410
|
-
deadlineMs
|
|
4411
|
-
})
|
|
4412
|
-
);
|
|
4413
|
-
return false;
|
|
4414
|
-
}
|
|
4415
|
-
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
4416
|
-
if (targetsResult instanceof NarsilError) {
|
|
4417
|
-
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
4418
|
-
return false;
|
|
4419
|
-
}
|
|
4420
|
-
const syncResult = await syncFromAnyTarget(
|
|
4421
|
-
state,
|
|
4422
|
-
entry,
|
|
4423
|
-
indexName,
|
|
4424
|
-
partitionId,
|
|
4425
|
-
primaryNodeId,
|
|
4426
|
-
targetsResult,
|
|
4427
|
-
schemaResult.schema,
|
|
4428
|
-
allocationResult.partitionCount,
|
|
4429
|
-
deadline,
|
|
4430
|
-
deps,
|
|
4431
|
-
abortCheck
|
|
4432
|
-
);
|
|
4433
|
-
if (!syncResult.ok) {
|
|
4434
|
-
if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
4435
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4436
|
-
return false;
|
|
4437
|
-
}
|
|
4438
|
-
if (syncResult.error.details.alreadySurfaced !== true) {
|
|
4439
|
-
surfaceError(deps, indexName, primaryNodeId, syncResult.error);
|
|
4440
|
-
}
|
|
4441
|
-
return false;
|
|
4442
|
-
}
|
|
4443
|
-
if (abortCheck()) {
|
|
4444
|
-
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4445
|
-
return false;
|
|
4446
|
-
}
|
|
4447
|
-
if (syncResult.snapshotHeader !== null) {
|
|
4448
|
-
deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
|
|
4003
|
+
|
|
4004
|
+
// src/distribution/cluster-node/bootstrap-sync/entries.ts
|
|
4005
|
+
async function applySyncEntries(indexName, partitionId, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, deps) {
|
|
4006
|
+
const terminalError = validateSyncEntryBatchTermination(entriesPayloads);
|
|
4007
|
+
if (terminalError !== null) {
|
|
4008
|
+
return { ok: false, error: terminalError };
|
|
4449
4009
|
}
|
|
4450
|
-
|
|
4451
|
-
|
|
4452
|
-
|
|
4453
|
-
|
|
4454
|
-
|
|
4455
|
-
|
|
4456
|
-
|
|
4457
|
-
|
|
4458
|
-
|
|
4459
|
-
|
|
4460
|
-
|
|
4461
|
-
|
|
4462
|
-
|
|
4463
|
-
|
|
4464
|
-
|
|
4465
|
-
|
|
4466
|
-
|
|
4467
|
-
|
|
4468
|
-
|
|
4469
|
-
|
|
4470
|
-
|
|
4471
|
-
|
|
4472
|
-
|
|
4473
|
-
|
|
4474
|
-
|
|
4475
|
-
|
|
4476
|
-
if (Date.now() >= deadline) {
|
|
4477
|
-
return {
|
|
4478
|
-
ok: false,
|
|
4479
|
-
error: new NarsilError(
|
|
4480
|
-
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
4481
|
-
"bootstrap sync exceeded deadline while iterating live targets",
|
|
4482
|
-
{ indexName, partitionId, primaryNodeId, targets }
|
|
4483
|
-
)
|
|
4484
|
-
};
|
|
4485
|
-
}
|
|
4486
|
-
const attempt = await syncFromTarget(
|
|
4487
|
-
state,
|
|
4488
|
-
entry,
|
|
4489
|
-
indexName,
|
|
4490
|
-
partitionId,
|
|
4491
|
-
target,
|
|
4492
|
-
coordinatorSchema,
|
|
4493
|
-
partitionCount,
|
|
4494
|
-
deadline,
|
|
4495
|
-
deps,
|
|
4496
|
-
abortCheck
|
|
4497
|
-
);
|
|
4498
|
-
if (attempt.ok) {
|
|
4499
|
-
return attempt;
|
|
4500
|
-
}
|
|
4501
|
-
lastError = new NarsilError(attempt.error.code, attempt.error.message, {
|
|
4502
|
-
...attempt.error.details,
|
|
4503
|
-
target
|
|
4504
|
-
});
|
|
4505
|
-
if (!isTransientFailure(attempt.error.code, attempt.error.details)) {
|
|
4506
|
-
return { ok: false, error: lastError };
|
|
4010
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4011
|
+
let expectedSeqNo = expectedFirstSeqNo;
|
|
4012
|
+
let entriesApplied = 0;
|
|
4013
|
+
for (const payload of entriesPayloads) {
|
|
4014
|
+
for (const entry of payload.entries) {
|
|
4015
|
+
const entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
4016
|
+
if (entryError !== null) {
|
|
4017
|
+
return { ok: false, error: entryError };
|
|
4018
|
+
}
|
|
4019
|
+
try {
|
|
4020
|
+
await deps.applyReplicationEntry(entry);
|
|
4021
|
+
log.appendCommitted(entry);
|
|
4022
|
+
} catch (err) {
|
|
4023
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
4024
|
+
return {
|
|
4025
|
+
ok: false,
|
|
4026
|
+
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
4027
|
+
indexName,
|
|
4028
|
+
partitionId,
|
|
4029
|
+
seqNo: entry.seqNo,
|
|
4030
|
+
cause
|
|
4031
|
+
})
|
|
4032
|
+
};
|
|
4033
|
+
}
|
|
4034
|
+
expectedSeqNo = entry.seqNo + 1;
|
|
4035
|
+
entriesApplied += 1;
|
|
4507
4036
|
}
|
|
4508
4037
|
}
|
|
4509
|
-
return {
|
|
4510
|
-
|
|
4511
|
-
|
|
4512
|
-
|
|
4513
|
-
const requestPayload = {
|
|
4514
|
-
indexName,
|
|
4515
|
-
partitionId,
|
|
4516
|
-
lastSeqNo: logState.lastSeqNo,
|
|
4517
|
-
lastPrimaryTerm: logState.lastPrimaryTerm
|
|
4518
|
-
};
|
|
4519
|
-
const request = {
|
|
4520
|
-
type: ReplicationMessageTypes.SYNC_REQUEST,
|
|
4521
|
-
sourceId: deps.sourceNodeId,
|
|
4522
|
-
requestId: generateId(),
|
|
4523
|
-
payload: encode(requestPayload)
|
|
4038
|
+
return {
|
|
4039
|
+
ok: true,
|
|
4040
|
+
entriesApplied,
|
|
4041
|
+
newSeqNo: expectedSeqNo - 1
|
|
4524
4042
|
};
|
|
4525
|
-
|
|
4526
|
-
|
|
4527
|
-
if (
|
|
4528
|
-
return
|
|
4529
|
-
ok: false,
|
|
4530
|
-
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "deadline expired before live sync stream", {
|
|
4531
|
-
indexName,
|
|
4532
|
-
partitionId,
|
|
4533
|
-
target
|
|
4534
|
-
})
|
|
4535
|
-
};
|
|
4043
|
+
}
|
|
4044
|
+
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
4045
|
+
if (entriesPayloads.length === 0) {
|
|
4046
|
+
return null;
|
|
4536
4047
|
}
|
|
4537
|
-
|
|
4538
|
-
|
|
4539
|
-
|
|
4540
|
-
}
|
|
4541
|
-
await withDeadline(streamWork, remainingMs, indexName, "live-sync");
|
|
4542
|
-
} catch (err) {
|
|
4543
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
4544
|
-
return {
|
|
4545
|
-
ok: false,
|
|
4546
|
-
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, `live sync transport failed: ${cause}`, {
|
|
4547
|
-
indexName,
|
|
4548
|
-
partitionId,
|
|
4549
|
-
target,
|
|
4550
|
-
cause
|
|
4551
|
-
})
|
|
4552
|
-
};
|
|
4048
|
+
for (let i = 0; i < entriesPayloads.length - 1; i += 1) {
|
|
4049
|
+
if (entriesPayloads[i].isLast) {
|
|
4050
|
+
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID, "sync_entries marked isLast before final batch");
|
|
4051
|
+
}
|
|
4553
4052
|
}
|
|
4554
|
-
if (
|
|
4555
|
-
return
|
|
4053
|
+
if (!entriesPayloads[entriesPayloads.length - 1].isLast) {
|
|
4054
|
+
return new NarsilError(
|
|
4055
|
+
ErrorCodes.SNAPSHOT_SYNC_FRAME_INVALID,
|
|
4056
|
+
"live sync stream ended before final sync_entries batch"
|
|
4057
|
+
);
|
|
4556
4058
|
}
|
|
4557
|
-
|
|
4558
|
-
|
|
4559
|
-
|
|
4560
|
-
|
|
4561
|
-
|
|
4562
|
-
|
|
4563
|
-
|
|
4564
|
-
|
|
4565
|
-
|
|
4059
|
+
return null;
|
|
4060
|
+
}
|
|
4061
|
+
function validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log) {
|
|
4062
|
+
if (entry.indexName !== indexName || entry.partitionId !== partitionId) {
|
|
4063
|
+
return new NarsilError(
|
|
4064
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
4065
|
+
"replication entry scope does not match bootstrap target",
|
|
4066
|
+
{
|
|
4067
|
+
expectedIndexName: indexName,
|
|
4068
|
+
expectedPartitionId: partitionId,
|
|
4069
|
+
entryIndexName: entry.indexName,
|
|
4070
|
+
entryPartitionId: entry.partitionId
|
|
4071
|
+
}
|
|
4072
|
+
);
|
|
4566
4073
|
}
|
|
4567
|
-
if (
|
|
4568
|
-
return
|
|
4569
|
-
|
|
4570
|
-
entry
|
|
4571
|
-
indexName,
|
|
4572
|
-
partitionId,
|
|
4573
|
-
target,
|
|
4574
|
-
frameState,
|
|
4575
|
-
coordinatorSchema,
|
|
4576
|
-
partitionCount,
|
|
4577
|
-
deadline,
|
|
4578
|
-
deps
|
|
4074
|
+
if (entry.seqNo !== expectedSeqNo) {
|
|
4075
|
+
return new NarsilError(
|
|
4076
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
4077
|
+
`Out-of-order sync entry ${entry.seqNo}; expected ${expectedSeqNo}`,
|
|
4078
|
+
{ indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
4579
4079
|
);
|
|
4580
4080
|
}
|
|
4581
|
-
|
|
4582
|
-
|
|
4583
|
-
|
|
4584
|
-
|
|
4585
|
-
|
|
4586
|
-
|
|
4587
|
-
logState.lastSeqNo + 1,
|
|
4588
|
-
logState.lastPrimaryTerm,
|
|
4589
|
-
coordinatorSchema,
|
|
4590
|
-
partitionCount,
|
|
4591
|
-
deps
|
|
4081
|
+
const validation = validateReplicationEntry(entry, localPrimaryTerm, log);
|
|
4082
|
+
if (!validation.valid) {
|
|
4083
|
+
return new NarsilError(
|
|
4084
|
+
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
4085
|
+
`Invalid sync entry ${entry.seqNo}`,
|
|
4086
|
+
{ indexName, partitionId, seqNo: entry.seqNo }
|
|
4592
4087
|
);
|
|
4593
4088
|
}
|
|
4594
|
-
return
|
|
4595
|
-
ok: false,
|
|
4596
|
-
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING, "live sync stream ended without sync frames", {
|
|
4597
|
-
indexName,
|
|
4598
|
-
partitionId,
|
|
4599
|
-
target
|
|
4600
|
-
})
|
|
4601
|
-
};
|
|
4602
|
-
}
|
|
4603
|
-
function readLocalLogState(indexName, partitionId, deps) {
|
|
4604
|
-
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4605
|
-
const lastSeqNo = log.committedSeqNo;
|
|
4606
|
-
const lastPrimaryTerm = log.committedPrimaryTerm;
|
|
4607
|
-
const newestEntry = log.getEntry(lastSeqNo);
|
|
4608
|
-
return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
|
|
4089
|
+
return null;
|
|
4609
4090
|
}
|
|
4610
4091
|
function createLiveSyncFrameState(indexName, partitionId) {
|
|
4611
4092
|
return {
|
|
@@ -4679,6 +4160,11 @@ function decodeSyncEntriesRecord(record) {
|
|
|
4679
4160
|
}
|
|
4680
4161
|
return { entries, isLast: record.isLast };
|
|
4681
4162
|
}
|
|
4163
|
+
function streamFailureToError(failure) {
|
|
4164
|
+
return new NarsilError(failure.code, failure.message, failure.details);
|
|
4165
|
+
}
|
|
4166
|
+
|
|
4167
|
+
// src/distribution/cluster-node/bootstrap-sync/apply.ts
|
|
4682
4168
|
async function applyLiveIncrementalSync(indexName, partitionId, target, entriesPayloads, expectedFirstSeqNo, localPrimaryTerm, coordinatorSchema, partitionCount, deps) {
|
|
4683
4169
|
const ensureResult = await ensureLocalIndexForIncremental(indexName, target, coordinatorSchema, partitionCount, deps);
|
|
4684
4170
|
if (ensureResult instanceof NarsilError) {
|
|
@@ -4838,136 +4324,334 @@ async function ensureLocalIndexForIncremental(indexName, primaryNodeId, coordina
|
|
|
4838
4324
|
const cause = err instanceof Error ? err.message : String(err);
|
|
4839
4325
|
return new NarsilError(ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED, `engine.createIndex failed: ${cause}`, {
|
|
4840
4326
|
indexName,
|
|
4841
|
-
primaryNodeId,
|
|
4842
|
-
cause
|
|
4843
|
-
});
|
|
4327
|
+
primaryNodeId,
|
|
4328
|
+
cause
|
|
4329
|
+
});
|
|
4330
|
+
}
|
|
4331
|
+
return { created: true };
|
|
4332
|
+
}
|
|
4333
|
+
function validateLocalSchema(engine, indexName, primaryNodeId, coordinatorSchema) {
|
|
4334
|
+
const schemaError = validateRestoredSchema(engine, indexName, primaryNodeId, coordinatorSchema);
|
|
4335
|
+
if (schemaError !== null) {
|
|
4336
|
+
return schemaError;
|
|
4337
|
+
}
|
|
4338
|
+
return { created: false };
|
|
4339
|
+
}
|
|
4340
|
+
function validateLocalPartitionCount(engine, indexName, primaryNodeId, partitionCount) {
|
|
4341
|
+
let localPartitionCount;
|
|
4342
|
+
try {
|
|
4343
|
+
localPartitionCount = engine.getStats(indexName).partitionCount;
|
|
4344
|
+
} catch (err) {
|
|
4345
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
4346
|
+
return new NarsilError(
|
|
4347
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
4348
|
+
`failed to read local partition count for '${indexName}': ${cause}`,
|
|
4349
|
+
{ indexName, primaryNodeId, cause }
|
|
4350
|
+
);
|
|
4351
|
+
}
|
|
4352
|
+
if (localPartitionCount !== partitionCount) {
|
|
4353
|
+
return new NarsilError(
|
|
4354
|
+
ErrorCodes.SNAPSHOT_SYNC_RESTORE_FAILED,
|
|
4355
|
+
`local partition count ${localPartitionCount} disagrees with coordinator partition count ${partitionCount}`,
|
|
4356
|
+
{ indexName, primaryNodeId, localPartitionCount, partitionCount }
|
|
4357
|
+
);
|
|
4358
|
+
}
|
|
4359
|
+
return { created: false };
|
|
4360
|
+
}
|
|
4361
|
+
|
|
4362
|
+
// src/distribution/cluster-node/bootstrap-sync/targets.ts
|
|
4363
|
+
function readLocalLogState(indexName, partitionId, deps) {
|
|
4364
|
+
const log = deps.getReplicationLog(indexName, partitionId);
|
|
4365
|
+
const lastSeqNo = log.committedSeqNo;
|
|
4366
|
+
const lastPrimaryTerm = log.committedPrimaryTerm;
|
|
4367
|
+
const newestEntry = log.getEntry(lastSeqNo);
|
|
4368
|
+
return { lastSeqNo, lastPrimaryTerm: newestEntry?.primaryTerm ?? lastPrimaryTerm };
|
|
4369
|
+
}
|
|
4370
|
+
async function syncFromAnyTarget(state, entry, indexName, partitionId, primaryNodeId, targets, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
|
|
4371
|
+
let lastError = new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, "no live sync attempts were made", {
|
|
4372
|
+
indexName,
|
|
4373
|
+
partitionId,
|
|
4374
|
+
primaryNodeId
|
|
4375
|
+
});
|
|
4376
|
+
const attempted = /* @__PURE__ */ new Set();
|
|
4377
|
+
for (const target of targets) {
|
|
4378
|
+
if (attempted.has(target)) {
|
|
4379
|
+
continue;
|
|
4380
|
+
}
|
|
4381
|
+
attempted.add(target);
|
|
4382
|
+
if (abortCheck()) {
|
|
4383
|
+
return {
|
|
4384
|
+
ok: false,
|
|
4385
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted before next live target", {
|
|
4386
|
+
indexName,
|
|
4387
|
+
partitionId,
|
|
4388
|
+
primaryNodeId,
|
|
4389
|
+
target
|
|
4390
|
+
})
|
|
4391
|
+
};
|
|
4392
|
+
}
|
|
4393
|
+
if (Date.now() >= deadline) {
|
|
4394
|
+
return {
|
|
4395
|
+
ok: false,
|
|
4396
|
+
error: new NarsilError(
|
|
4397
|
+
ErrorCodes.SNAPSHOT_SYNC_TIMEOUT,
|
|
4398
|
+
"bootstrap sync exceeded deadline while iterating live targets",
|
|
4399
|
+
{ indexName, partitionId, primaryNodeId, targets }
|
|
4400
|
+
)
|
|
4401
|
+
};
|
|
4402
|
+
}
|
|
4403
|
+
const attempt = await syncFromTarget(
|
|
4404
|
+
state,
|
|
4405
|
+
entry,
|
|
4406
|
+
indexName,
|
|
4407
|
+
partitionId,
|
|
4408
|
+
target,
|
|
4409
|
+
coordinatorSchema,
|
|
4410
|
+
partitionCount,
|
|
4411
|
+
deadline,
|
|
4412
|
+
deps,
|
|
4413
|
+
abortCheck
|
|
4414
|
+
);
|
|
4415
|
+
if (attempt.ok) {
|
|
4416
|
+
return attempt;
|
|
4417
|
+
}
|
|
4418
|
+
lastError = new NarsilError(attempt.error.code, attempt.error.message, {
|
|
4419
|
+
...attempt.error.details,
|
|
4420
|
+
target
|
|
4421
|
+
});
|
|
4422
|
+
if (!isTransientFailure(attempt.error.code, attempt.error.details)) {
|
|
4423
|
+
return { ok: false, error: lastError };
|
|
4424
|
+
}
|
|
4425
|
+
}
|
|
4426
|
+
return { ok: false, error: lastError };
|
|
4427
|
+
}
|
|
4428
|
+
async function syncFromTarget(state, entry, indexName, partitionId, target, coordinatorSchema, partitionCount, deadline, deps, abortCheck) {
|
|
4429
|
+
const logState = readLocalLogState(indexName, partitionId, deps);
|
|
4430
|
+
const requestPayload = {
|
|
4431
|
+
indexName,
|
|
4432
|
+
partitionId,
|
|
4433
|
+
lastSeqNo: logState.lastSeqNo,
|
|
4434
|
+
lastPrimaryTerm: logState.lastPrimaryTerm
|
|
4435
|
+
};
|
|
4436
|
+
const request = {
|
|
4437
|
+
type: ReplicationMessageTypes.SYNC_REQUEST,
|
|
4438
|
+
sourceId: deps.sourceNodeId,
|
|
4439
|
+
requestId: generateId(),
|
|
4440
|
+
payload: encode(requestPayload)
|
|
4441
|
+
};
|
|
4442
|
+
const frameState = createLiveSyncFrameState(indexName, partitionId);
|
|
4443
|
+
const remainingMs = deadline - Date.now();
|
|
4444
|
+
if (remainingMs <= 0) {
|
|
4445
|
+
return {
|
|
4446
|
+
ok: false,
|
|
4447
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "deadline expired before live sync stream", {
|
|
4448
|
+
indexName,
|
|
4449
|
+
partitionId,
|
|
4450
|
+
target
|
|
4451
|
+
})
|
|
4452
|
+
};
|
|
4453
|
+
}
|
|
4454
|
+
try {
|
|
4455
|
+
const streamWork = deps.transport.stream(target, request, (chunk) => {
|
|
4456
|
+
handleLiveSyncFrame(frameState, chunk);
|
|
4457
|
+
});
|
|
4458
|
+
await withDeadline(streamWork, remainingMs, indexName, "live-sync");
|
|
4459
|
+
} catch (err) {
|
|
4460
|
+
const cause = err instanceof Error ? err.message : String(err);
|
|
4461
|
+
return {
|
|
4462
|
+
ok: false,
|
|
4463
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TRANSPORT_FAILED, `live sync transport failed: ${cause}`, {
|
|
4464
|
+
indexName,
|
|
4465
|
+
partitionId,
|
|
4466
|
+
target,
|
|
4467
|
+
cause
|
|
4468
|
+
})
|
|
4469
|
+
};
|
|
4470
|
+
}
|
|
4471
|
+
if (frameState.error !== null) {
|
|
4472
|
+
return { ok: false, error: frameState.error };
|
|
4473
|
+
}
|
|
4474
|
+
if (abortCheck()) {
|
|
4475
|
+
return {
|
|
4476
|
+
ok: false,
|
|
4477
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_ABORTED, "bootstrap sync aborted after live stream", {
|
|
4478
|
+
indexName,
|
|
4479
|
+
partitionId,
|
|
4480
|
+
target
|
|
4481
|
+
})
|
|
4482
|
+
};
|
|
4483
|
+
}
|
|
4484
|
+
if (frameState.sawSnapshotFrame) {
|
|
4485
|
+
return applyLiveSnapshotSync(
|
|
4486
|
+
state,
|
|
4487
|
+
entry,
|
|
4488
|
+
indexName,
|
|
4489
|
+
partitionId,
|
|
4490
|
+
target,
|
|
4491
|
+
frameState,
|
|
4492
|
+
coordinatorSchema,
|
|
4493
|
+
partitionCount,
|
|
4494
|
+
deadline,
|
|
4495
|
+
deps
|
|
4496
|
+
);
|
|
4844
4497
|
}
|
|
4845
|
-
|
|
4846
|
-
|
|
4847
|
-
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4498
|
+
if (frameState.syncEntries.length > 0) {
|
|
4499
|
+
return applyLiveIncrementalSync(
|
|
4500
|
+
indexName,
|
|
4501
|
+
partitionId,
|
|
4502
|
+
target,
|
|
4503
|
+
frameState.syncEntries,
|
|
4504
|
+
logState.lastSeqNo + 1,
|
|
4505
|
+
logState.lastPrimaryTerm,
|
|
4506
|
+
coordinatorSchema,
|
|
4507
|
+
partitionCount,
|
|
4508
|
+
deps
|
|
4509
|
+
);
|
|
4851
4510
|
}
|
|
4852
|
-
return {
|
|
4511
|
+
return {
|
|
4512
|
+
ok: false,
|
|
4513
|
+
error: new NarsilError(ErrorCodes.SNAPSHOT_SYNC_CHUNK_MISSING, "live sync stream ended without sync frames", {
|
|
4514
|
+
indexName,
|
|
4515
|
+
partitionId,
|
|
4516
|
+
target
|
|
4517
|
+
})
|
|
4518
|
+
};
|
|
4853
4519
|
}
|
|
4854
|
-
|
|
4855
|
-
|
|
4520
|
+
|
|
4521
|
+
// src/distribution/cluster-node/bootstrap-sync/live.ts
|
|
4522
|
+
async function resolveLivePartitionCount(coordinator, indexName, partitionId, primaryNodeId, abortPromise) {
|
|
4856
4523
|
try {
|
|
4857
|
-
|
|
4524
|
+
const winner = await Promise.race([coordinator.getAllocation(indexName), abortPromise]);
|
|
4525
|
+
if (winner === ABORT_SENTINEL) {
|
|
4526
|
+
return "aborted";
|
|
4527
|
+
}
|
|
4528
|
+
const allocation = winner;
|
|
4529
|
+
if (allocation === null) {
|
|
4530
|
+
return {
|
|
4531
|
+
error: new NarsilError(
|
|
4532
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
4533
|
+
`coordinator has no allocation for index '${indexName}'`,
|
|
4534
|
+
{ indexName, primaryNodeId }
|
|
4535
|
+
)
|
|
4536
|
+
};
|
|
4537
|
+
}
|
|
4538
|
+
const assignment = allocation.assignments.get(partitionId);
|
|
4539
|
+
if (assignment === void 0) {
|
|
4540
|
+
return {
|
|
4541
|
+
error: new NarsilError(
|
|
4542
|
+
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
4543
|
+
`No assignment exists for partition ${partitionId} of index '${indexName}'`,
|
|
4544
|
+
{ indexName, partitionId, primaryNodeId }
|
|
4545
|
+
)
|
|
4546
|
+
};
|
|
4547
|
+
}
|
|
4548
|
+
if (assignment.primary !== primaryNodeId) {
|
|
4549
|
+
return {
|
|
4550
|
+
error: new NarsilError(
|
|
4551
|
+
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
4552
|
+
`Partition ${partitionId} of index '${indexName}' is no longer primary on '${primaryNodeId}'`,
|
|
4553
|
+
{ indexName, partitionId, primaryNodeId, currentPrimary: assignment.primary }
|
|
4554
|
+
)
|
|
4555
|
+
};
|
|
4556
|
+
}
|
|
4557
|
+
return { partitionCount: allocation.assignments.size };
|
|
4858
4558
|
} catch (err) {
|
|
4859
4559
|
const cause = err instanceof Error ? err.message : String(err);
|
|
4860
|
-
return
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
4867
|
-
|
|
4868
|
-
|
|
4869
|
-
|
|
4870
|
-
|
|
4871
|
-
|
|
4560
|
+
return {
|
|
4561
|
+
error: new NarsilError(
|
|
4562
|
+
ErrorCodes.SNAPSHOT_SYNC_ALLOCATION_UNAVAILABLE,
|
|
4563
|
+
`coordinator.getAllocation failed: ${cause}`,
|
|
4564
|
+
{
|
|
4565
|
+
indexName,
|
|
4566
|
+
partitionId,
|
|
4567
|
+
primaryNodeId,
|
|
4568
|
+
cause
|
|
4569
|
+
}
|
|
4570
|
+
)
|
|
4571
|
+
};
|
|
4872
4572
|
}
|
|
4873
|
-
return { created: false };
|
|
4874
4573
|
}
|
|
4875
|
-
async function
|
|
4876
|
-
const
|
|
4877
|
-
if (
|
|
4878
|
-
|
|
4574
|
+
async function executeLiveBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps, deadlineMs, deadline, abortCheck) {
|
|
4575
|
+
const schemaResult = await loadCoordinatorSchema(deps.coordinator, indexName, primaryNodeId, entry.abortPromise);
|
|
4576
|
+
if (schemaResult === "aborted") {
|
|
4577
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4578
|
+
return false;
|
|
4879
4579
|
}
|
|
4880
|
-
|
|
4881
|
-
|
|
4882
|
-
|
|
4883
|
-
for (const payload of entriesPayloads) {
|
|
4884
|
-
for (const entry of payload.entries) {
|
|
4885
|
-
const entryError = validateSyncEntryForApply(entry, indexName, partitionId, expectedSeqNo, localPrimaryTerm, log);
|
|
4886
|
-
if (entryError !== null) {
|
|
4887
|
-
return { ok: false, error: entryError };
|
|
4888
|
-
}
|
|
4889
|
-
try {
|
|
4890
|
-
await deps.applyReplicationEntry(entry);
|
|
4891
|
-
log.appendCommitted(entry);
|
|
4892
|
-
} catch (err) {
|
|
4893
|
-
const cause = err instanceof Error ? err.message : String(err);
|
|
4894
|
-
return {
|
|
4895
|
-
ok: false,
|
|
4896
|
-
error: new NarsilError(ErrorCodes.REPLICATION_ENTRY_INVALID, `failed to apply sync entry: ${cause}`, {
|
|
4897
|
-
indexName,
|
|
4898
|
-
partitionId,
|
|
4899
|
-
seqNo: entry.seqNo,
|
|
4900
|
-
cause
|
|
4901
|
-
})
|
|
4902
|
-
};
|
|
4903
|
-
}
|
|
4904
|
-
expectedSeqNo = entry.seqNo + 1;
|
|
4905
|
-
entriesApplied += 1;
|
|
4906
|
-
}
|
|
4580
|
+
if ("error" in schemaResult) {
|
|
4581
|
+
surfaceError(deps, indexName, primaryNodeId, schemaResult.error);
|
|
4582
|
+
return false;
|
|
4907
4583
|
}
|
|
4908
|
-
|
|
4909
|
-
|
|
4910
|
-
|
|
4911
|
-
newSeqNo: expectedSeqNo - 1
|
|
4912
|
-
};
|
|
4913
|
-
}
|
|
4914
|
-
function validateSyncEntryBatchTermination(entriesPayloads) {
|
|
4915
|
-
if (entriesPayloads.length === 0) {
|
|
4916
|
-
return null;
|
|
4584
|
+
if (abortCheck()) {
|
|
4585
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4586
|
+
return false;
|
|
4917
4587
|
}
|
|
4918
|
-
|
|
4919
|
-
|
|
4920
|
-
|
|
4921
|
-
|
|
4588
|
+
const allocationResult = await resolveLivePartitionCount(
|
|
4589
|
+
deps.coordinator,
|
|
4590
|
+
indexName,
|
|
4591
|
+
partitionId,
|
|
4592
|
+
primaryNodeId,
|
|
4593
|
+
entry.abortPromise
|
|
4594
|
+
);
|
|
4595
|
+
if (allocationResult === "aborted") {
|
|
4596
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4597
|
+
return false;
|
|
4922
4598
|
}
|
|
4923
|
-
if (
|
|
4924
|
-
|
|
4925
|
-
|
|
4926
|
-
"live sync stream ended before final sync_entries batch"
|
|
4927
|
-
);
|
|
4599
|
+
if ("error" in allocationResult) {
|
|
4600
|
+
surfaceError(deps, indexName, primaryNodeId, allocationResult.error);
|
|
4601
|
+
return false;
|
|
4928
4602
|
}
|
|
4929
|
-
|
|
4930
|
-
|
|
4931
|
-
|
|
4932
|
-
|
|
4933
|
-
|
|
4934
|
-
ErrorCodes.
|
|
4935
|
-
|
|
4936
|
-
|
|
4937
|
-
|
|
4938
|
-
expectedPartitionId: partitionId,
|
|
4939
|
-
entryIndexName: entry.indexName,
|
|
4940
|
-
entryPartitionId: entry.partitionId
|
|
4941
|
-
}
|
|
4603
|
+
if (Date.now() >= deadline) {
|
|
4604
|
+
surfaceError(
|
|
4605
|
+
deps,
|
|
4606
|
+
indexName,
|
|
4607
|
+
primaryNodeId,
|
|
4608
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before starting live sync", {
|
|
4609
|
+
indexName,
|
|
4610
|
+
deadlineMs
|
|
4611
|
+
})
|
|
4942
4612
|
);
|
|
4613
|
+
return false;
|
|
4943
4614
|
}
|
|
4944
|
-
|
|
4945
|
-
|
|
4946
|
-
|
|
4947
|
-
|
|
4948
|
-
{ indexName, partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
4949
|
-
);
|
|
4615
|
+
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
4616
|
+
if (targetsResult instanceof NarsilError) {
|
|
4617
|
+
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
4618
|
+
return false;
|
|
4950
4619
|
}
|
|
4951
|
-
const
|
|
4952
|
-
|
|
4953
|
-
|
|
4954
|
-
|
|
4955
|
-
|
|
4956
|
-
|
|
4957
|
-
|
|
4620
|
+
const syncResult = await syncFromAnyTarget(
|
|
4621
|
+
state,
|
|
4622
|
+
entry,
|
|
4623
|
+
indexName,
|
|
4624
|
+
partitionId,
|
|
4625
|
+
primaryNodeId,
|
|
4626
|
+
targetsResult,
|
|
4627
|
+
schemaResult.schema,
|
|
4628
|
+
allocationResult.partitionCount,
|
|
4629
|
+
deadline,
|
|
4630
|
+
deps,
|
|
4631
|
+
abortCheck
|
|
4632
|
+
);
|
|
4633
|
+
if (!syncResult.ok) {
|
|
4634
|
+
if (syncResult.error.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
4635
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4636
|
+
return false;
|
|
4637
|
+
}
|
|
4638
|
+
if (syncResult.error.details.alreadySurfaced !== true) {
|
|
4639
|
+
surfaceError(deps, indexName, primaryNodeId, syncResult.error);
|
|
4640
|
+
}
|
|
4641
|
+
return false;
|
|
4958
4642
|
}
|
|
4959
|
-
|
|
4960
|
-
|
|
4961
|
-
function streamFailureToError(failure) {
|
|
4962
|
-
return new NarsilError(failure.code, failure.message, failure.details);
|
|
4963
|
-
}
|
|
4964
|
-
function anotherBootstrapOwnsKey(state, key, self) {
|
|
4965
|
-
const current = state.inFlight.get(key);
|
|
4966
|
-
if (current === void 0) {
|
|
4643
|
+
if (abortCheck()) {
|
|
4644
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4967
4645
|
return false;
|
|
4968
4646
|
}
|
|
4969
|
-
|
|
4647
|
+
if (syncResult.snapshotHeader !== null) {
|
|
4648
|
+
deps.onSnapshotApplied?.(indexName, partitionId, syncResult.snapshotHeader);
|
|
4649
|
+
}
|
|
4650
|
+
state.completed.add(entryKey(indexName, partitionId));
|
|
4651
|
+
return true;
|
|
4970
4652
|
}
|
|
4653
|
+
|
|
4654
|
+
// src/distribution/cluster-node/bootstrap-sync/snapshot-restore.ts
|
|
4971
4655
|
async function fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps) {
|
|
4972
4656
|
const dropError = await dropExistingIndex(deps.engine, indexName, primaryNodeId);
|
|
4973
4657
|
if (dropError !== null) {
|
|
@@ -5028,6 +4712,150 @@ async function applyRestore(state, entry, indexName, primaryNodeId, bytes, coord
|
|
|
5028
4712
|
return true;
|
|
5029
4713
|
}
|
|
5030
4714
|
|
|
4715
|
+
// src/distribution/cluster-node/bootstrap-sync/index.ts
|
|
4716
|
+
async function runBootstrapSync(state, indexName, partitionId, primaryNodeId, deps) {
|
|
4717
|
+
const validationError = validateArguments(indexName, partitionId, primaryNodeId);
|
|
4718
|
+
if (validationError !== null) {
|
|
4719
|
+
surfaceError(deps, indexName, primaryNodeId, validationError);
|
|
4720
|
+
return false;
|
|
4721
|
+
}
|
|
4722
|
+
const key = entryKey(indexName, partitionId);
|
|
4723
|
+
if (state.completed.has(key)) {
|
|
4724
|
+
return true;
|
|
4725
|
+
}
|
|
4726
|
+
const existing = state.inFlight.get(key);
|
|
4727
|
+
if (existing !== void 0) {
|
|
4728
|
+
const result = await Promise.race([existing.promise, existing.abortPromise]);
|
|
4729
|
+
if (result === ABORT_SENTINEL) {
|
|
4730
|
+
return false;
|
|
4731
|
+
}
|
|
4732
|
+
return result;
|
|
4733
|
+
}
|
|
4734
|
+
const entry = createEntry(state, indexName, partitionId, key);
|
|
4735
|
+
entry.promise = executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps).finally(() => {
|
|
4736
|
+
const current = state.inFlight.get(key);
|
|
4737
|
+
if (current === entry) {
|
|
4738
|
+
state.inFlight.delete(key);
|
|
4739
|
+
}
|
|
4740
|
+
if (!state.completed.has(key) && !state.inFlight.has(key)) {
|
|
4741
|
+
const currentGen = state.generations.get(key);
|
|
4742
|
+
if (currentGen !== void 0 && currentGen !== entry.generation) {
|
|
4743
|
+
state.generations.delete(key);
|
|
4744
|
+
}
|
|
4745
|
+
}
|
|
4746
|
+
});
|
|
4747
|
+
state.inFlight.set(key, entry);
|
|
4748
|
+
return entry.promise;
|
|
4749
|
+
}
|
|
4750
|
+
async function executeBootstrapSync(state, entry, indexName, partitionId, primaryNodeId, deps) {
|
|
4751
|
+
const deadlineMs = deps.deadlineMs ?? DEFAULT_BOOTSTRAP_SYNC_DEADLINE_MS;
|
|
4752
|
+
const deadline = Date.now() + deadlineMs;
|
|
4753
|
+
const key = entryKey(indexName, partitionId);
|
|
4754
|
+
const abortCheck = () => {
|
|
4755
|
+
const currentGeneration = state.generations.get(key) ?? 0;
|
|
4756
|
+
if (currentGeneration !== entry.generation) {
|
|
4757
|
+
entry.aborted = true;
|
|
4758
|
+
}
|
|
4759
|
+
return entry.aborted;
|
|
4760
|
+
};
|
|
4761
|
+
if (hasLiveBootstrapSyncDeps(deps)) {
|
|
4762
|
+
return executeLiveBootstrapSync(
|
|
4763
|
+
state,
|
|
4764
|
+
entry,
|
|
4765
|
+
indexName,
|
|
4766
|
+
partitionId,
|
|
4767
|
+
primaryNodeId,
|
|
4768
|
+
deps,
|
|
4769
|
+
deadlineMs,
|
|
4770
|
+
deadline,
|
|
4771
|
+
abortCheck
|
|
4772
|
+
);
|
|
4773
|
+
}
|
|
4774
|
+
const schemaResult = await fetchSchemaAndPrepare(entry, indexName, primaryNodeId, deps);
|
|
4775
|
+
if (schemaResult === "aborted") {
|
|
4776
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4777
|
+
return false;
|
|
4778
|
+
}
|
|
4779
|
+
if (schemaResult === null) {
|
|
4780
|
+
return false;
|
|
4781
|
+
}
|
|
4782
|
+
if (abortCheck()) {
|
|
4783
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4784
|
+
return false;
|
|
4785
|
+
}
|
|
4786
|
+
if (Date.now() >= deadline) {
|
|
4787
|
+
surfaceError(
|
|
4788
|
+
deps,
|
|
4789
|
+
indexName,
|
|
4790
|
+
primaryNodeId,
|
|
4791
|
+
new NarsilError(ErrorCodes.SNAPSHOT_SYNC_TIMEOUT, "bootstrap sync exceeded deadline before fetching snapshot", {
|
|
4792
|
+
indexName,
|
|
4793
|
+
deadlineMs
|
|
4794
|
+
})
|
|
4795
|
+
);
|
|
4796
|
+
return false;
|
|
4797
|
+
}
|
|
4798
|
+
const targetsResult = await resolveTransportTargets(indexName, primaryNodeId, deps.resolveNodeTargets);
|
|
4799
|
+
if (targetsResult instanceof NarsilError) {
|
|
4800
|
+
surfaceError(deps, indexName, primaryNodeId, targetsResult);
|
|
4801
|
+
return false;
|
|
4802
|
+
}
|
|
4803
|
+
const fetchDeps = { transport: deps.transport, sourceNodeId: deps.sourceNodeId };
|
|
4804
|
+
const fetchResult = await fetchSnapshotFromAnyTarget(
|
|
4805
|
+
indexName,
|
|
4806
|
+
primaryNodeId,
|
|
4807
|
+
targetsResult,
|
|
4808
|
+
deadline,
|
|
4809
|
+
fetchDeps,
|
|
4810
|
+
abortCheck,
|
|
4811
|
+
partitionId
|
|
4812
|
+
);
|
|
4813
|
+
if (!fetchResult.ok) {
|
|
4814
|
+
if (fetchResult.code === ErrorCodes.SNAPSHOT_SYNC_ABORTED) {
|
|
4815
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4816
|
+
return false;
|
|
4817
|
+
}
|
|
4818
|
+
surfaceError(
|
|
4819
|
+
deps,
|
|
4820
|
+
indexName,
|
|
4821
|
+
primaryNodeId,
|
|
4822
|
+
new NarsilError(fetchResult.code, fetchResult.message, {
|
|
4823
|
+
indexName,
|
|
4824
|
+
primaryNodeId,
|
|
4825
|
+
...fetchResult.details
|
|
4826
|
+
})
|
|
4827
|
+
);
|
|
4828
|
+
return false;
|
|
4829
|
+
}
|
|
4830
|
+
if (abortCheck()) {
|
|
4831
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4832
|
+
return false;
|
|
4833
|
+
}
|
|
4834
|
+
const restoreSucceeded = await applyRestore(
|
|
4835
|
+
state,
|
|
4836
|
+
entry,
|
|
4837
|
+
indexName,
|
|
4838
|
+
primaryNodeId,
|
|
4839
|
+
fetchResult.bytes,
|
|
4840
|
+
schemaResult,
|
|
4841
|
+
deadline,
|
|
4842
|
+
deps
|
|
4843
|
+
);
|
|
4844
|
+
if (!restoreSucceeded) {
|
|
4845
|
+
return false;
|
|
4846
|
+
}
|
|
4847
|
+
if (abortCheck()) {
|
|
4848
|
+
if (!anotherBootstrapOwnsKey(state, key, entry)) {
|
|
4849
|
+
await dropRestoredIndexQuietly(deps.engine, indexName, deps);
|
|
4850
|
+
}
|
|
4851
|
+
surfaceAborted(deps, indexName, primaryNodeId);
|
|
4852
|
+
return false;
|
|
4853
|
+
}
|
|
4854
|
+
deps.onSnapshotApplied?.(indexName, partitionId, fetchResult.header);
|
|
4855
|
+
state.completed.add(key);
|
|
4856
|
+
return true;
|
|
4857
|
+
}
|
|
4858
|
+
|
|
5031
4859
|
// src/distribution/cluster-node/local-engine.ts
|
|
5032
4860
|
async function createClusterLocalEngine(config) {
|
|
5033
4861
|
const core = createEngineCore(config);
|
|
@@ -5257,38 +5085,6 @@ async function applyReplicationEntry(core, entry) {
|
|
|
5257
5085
|
requestId: `replicate-entry-remove-${entry.indexName}-${entry.partitionId}-${entry.seqNo}`
|
|
5258
5086
|
});
|
|
5259
5087
|
}
|
|
5260
|
-
function decideSyncTier(log, replicaLastSeqNo) {
|
|
5261
|
-
const oldest = log.oldestSeqNo;
|
|
5262
|
-
if (oldest === void 0) {
|
|
5263
|
-
if (log.committedSeqNo === 0) {
|
|
5264
|
-
return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
|
|
5265
|
-
}
|
|
5266
|
-
return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
|
|
5267
|
-
}
|
|
5268
|
-
if (oldest <= replicaLastSeqNo + 1) {
|
|
5269
|
-
return "incremental";
|
|
5270
|
-
}
|
|
5271
|
-
return "snapshot";
|
|
5272
|
-
}
|
|
5273
|
-
function validateSyncRequest(decoded) {
|
|
5274
|
-
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
5275
|
-
throw new Error("Invalid SyncRequestPayload: expected an object");
|
|
5276
|
-
}
|
|
5277
|
-
const record = decoded;
|
|
5278
|
-
if (typeof record.indexName !== "string") {
|
|
5279
|
-
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
5280
|
-
}
|
|
5281
|
-
if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
|
|
5282
|
-
throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
|
|
5283
|
-
}
|
|
5284
|
-
if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
|
|
5285
|
-
throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
|
|
5286
|
-
}
|
|
5287
|
-
if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
|
|
5288
|
-
throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
|
|
5289
|
-
}
|
|
5290
|
-
return record;
|
|
5291
|
-
}
|
|
5292
5088
|
|
|
5293
5089
|
// src/distribution/cluster-node/snapshot-auth.ts
|
|
5294
5090
|
var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
|
|
@@ -5319,133 +5115,35 @@ async function authorizeSnapshotRequest(coordinator, indexName, sourceId) {
|
|
|
5319
5115
|
continue;
|
|
5320
5116
|
}
|
|
5321
5117
|
sawAssignment = true;
|
|
5322
|
-
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
5323
|
-
continue;
|
|
5324
|
-
}
|
|
5325
|
-
if (isPrimary) {
|
|
5326
|
-
return { outcome: "authorized" };
|
|
5327
|
-
}
|
|
5328
|
-
if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
|
|
5329
|
-
continue;
|
|
5330
|
-
}
|
|
5331
|
-
return { outcome: "authorized" };
|
|
5332
|
-
}
|
|
5333
|
-
if (sawAssignment) {
|
|
5334
|
-
return {
|
|
5335
|
-
outcome: "denied",
|
|
5336
|
-
code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
5337
|
-
reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
|
|
5338
|
-
};
|
|
5339
|
-
}
|
|
5340
|
-
return {
|
|
5341
|
-
outcome: "denied",
|
|
5342
|
-
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
5343
|
-
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
5344
|
-
};
|
|
5345
|
-
}
|
|
5346
|
-
function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
|
|
5347
|
-
if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
|
|
5348
|
-
return true;
|
|
5349
|
-
}
|
|
5350
|
-
return assignment.inSyncSet.includes(sourceId);
|
|
5351
|
-
}
|
|
5352
|
-
var SNAPSHOT_SYNC_ERROR_TYPE = `${ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST}.error`;
|
|
5353
|
-
var CHUNK_YIELD_INTERVAL_MS = 8;
|
|
5354
|
-
function createSingleResponseSink(respond) {
|
|
5355
|
-
const sink = ((response) => {
|
|
5356
|
-
if (sink.closed) {
|
|
5357
|
-
return;
|
|
5358
|
-
}
|
|
5359
|
-
respond(response);
|
|
5360
|
-
});
|
|
5361
|
-
sink.closed = false;
|
|
5362
|
-
return sink;
|
|
5363
|
-
}
|
|
5364
|
-
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, options = {}) {
|
|
5365
|
-
const snapshotBytes = build.bytes;
|
|
5366
|
-
const totalBytes = snapshotBytes.byteLength;
|
|
5367
|
-
const checksum = build.checksum;
|
|
5368
|
-
const header = {
|
|
5369
|
-
lastSeqNo: metadata.lastSeqNo,
|
|
5370
|
-
primaryTerm: metadata.primaryTerm,
|
|
5371
|
-
partitionId: metadata.partitionId,
|
|
5372
|
-
indexName,
|
|
5373
|
-
checksum
|
|
5374
|
-
};
|
|
5375
|
-
const startPayload = { header, totalBytes };
|
|
5376
|
-
const startBytes = encode(startPayload);
|
|
5377
|
-
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
5378
|
-
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
5379
|
-
await yieldToEventLoop();
|
|
5380
|
-
let offset = 0;
|
|
5381
|
-
let lastYieldAt = now();
|
|
5382
|
-
while (offset < totalBytes) {
|
|
5383
|
-
const end = Math.min(offset + SNAPSHOT_CHUNK_SIZE, totalBytes);
|
|
5384
|
-
const chunk = snapshotBytes.subarray(offset, end);
|
|
5385
|
-
const chunkPayload = {
|
|
5386
|
-
partitionId: metadata.partitionId,
|
|
5387
|
-
indexName,
|
|
5388
|
-
offset,
|
|
5389
|
-
data: chunk
|
|
5390
|
-
};
|
|
5391
|
-
const chunkBytes = encode(chunkPayload);
|
|
5392
|
-
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
5393
|
-
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
5394
|
-
offset = end;
|
|
5395
|
-
if (offset < totalBytes && now() - lastYieldAt >= CHUNK_YIELD_INTERVAL_MS) {
|
|
5396
|
-
await yieldToEventLoop();
|
|
5397
|
-
lastYieldAt = now();
|
|
5118
|
+
if (!BOOTSTRAPPABLE_STATES.has(assignment.state)) {
|
|
5119
|
+
continue;
|
|
5398
5120
|
}
|
|
5121
|
+
if (isPrimary) {
|
|
5122
|
+
return { outcome: "authorized" };
|
|
5123
|
+
}
|
|
5124
|
+
if (!assignmentAllowsReplicaBootstrap(assignment, sourceId)) {
|
|
5125
|
+
continue;
|
|
5126
|
+
}
|
|
5127
|
+
return { outcome: "authorized" };
|
|
5399
5128
|
}
|
|
5400
|
-
|
|
5401
|
-
|
|
5402
|
-
|
|
5403
|
-
|
|
5404
|
-
|
|
5405
|
-
|
|
5406
|
-
};
|
|
5407
|
-
const endBytes = encode(endPayload);
|
|
5408
|
-
assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
|
|
5409
|
-
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
|
|
5410
|
-
if (options.closeOnEnd !== false) {
|
|
5411
|
-
sink.closed = true;
|
|
5412
|
-
}
|
|
5413
|
-
}
|
|
5414
|
-
function now() {
|
|
5415
|
-
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
5416
|
-
return performance.now();
|
|
5129
|
+
if (sawAssignment) {
|
|
5130
|
+
return {
|
|
5131
|
+
outcome: "denied",
|
|
5132
|
+
code: ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
5133
|
+
reason: `node '${sourceId}' is listed for index '${indexName}' but not in a bootstrappable state`
|
|
5134
|
+
};
|
|
5417
5135
|
}
|
|
5418
|
-
return
|
|
5419
|
-
|
|
5420
|
-
|
|
5421
|
-
|
|
5422
|
-
|
|
5423
|
-
setImmediate(resolve);
|
|
5424
|
-
return;
|
|
5425
|
-
}
|
|
5426
|
-
setTimeout(resolve, 0);
|
|
5427
|
-
});
|
|
5428
|
-
}
|
|
5429
|
-
function respondMessage(sink, type, sourceId, requestId, payload) {
|
|
5430
|
-
sink({ type, sourceId, requestId, payload });
|
|
5136
|
+
return {
|
|
5137
|
+
outcome: "denied",
|
|
5138
|
+
code: ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
5139
|
+
reason: `node '${sourceId}' is not an assigned replica for index '${indexName}'`
|
|
5140
|
+
};
|
|
5431
5141
|
}
|
|
5432
|
-
function
|
|
5433
|
-
if (
|
|
5434
|
-
|
|
5435
|
-
ErrorCodes.CONFIG_INVALID,
|
|
5436
|
-
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
5437
|
-
);
|
|
5142
|
+
function assignmentAllowsReplicaBootstrap(assignment, sourceId) {
|
|
5143
|
+
if (assignment.state === "INITIALISING" || assignment.state === "MIGRATING") {
|
|
5144
|
+
return true;
|
|
5438
5145
|
}
|
|
5439
|
-
|
|
5440
|
-
function respondError(sink, sourceId, requestId, code, message) {
|
|
5441
|
-
const response = {
|
|
5442
|
-
type: SNAPSHOT_SYNC_ERROR_TYPE,
|
|
5443
|
-
sourceId,
|
|
5444
|
-
requestId,
|
|
5445
|
-
payload: encode({ error: true, code, message })
|
|
5446
|
-
};
|
|
5447
|
-
sink(response);
|
|
5448
|
-
sink.closed = true;
|
|
5146
|
+
return assignment.inSyncSet.includes(sourceId);
|
|
5449
5147
|
}
|
|
5450
5148
|
|
|
5451
5149
|
// src/distribution/cluster-node/snapshot-cache.ts
|
|
@@ -5576,37 +5274,106 @@ async function acquireSnapshotBuild(state, indexName, builder) {
|
|
|
5576
5274
|
state.cache.set(indexName, entry);
|
|
5577
5275
|
return entry.promise;
|
|
5578
5276
|
}
|
|
5579
|
-
|
|
5580
|
-
|
|
5581
|
-
|
|
5582
|
-
|
|
5583
|
-
|
|
5584
|
-
|
|
5585
|
-
const code = value.charCodeAt(i);
|
|
5586
|
-
if (code < 32) {
|
|
5587
|
-
return true;
|
|
5277
|
+
var SNAPSHOT_SYNC_ERROR_TYPE = `${ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST}.error`;
|
|
5278
|
+
var CHUNK_YIELD_INTERVAL_MS = 8;
|
|
5279
|
+
function createSingleResponseSink(respond) {
|
|
5280
|
+
const sink = ((response) => {
|
|
5281
|
+
if (sink.closed) {
|
|
5282
|
+
return;
|
|
5588
5283
|
}
|
|
5589
|
-
|
|
5590
|
-
|
|
5284
|
+
respond(response);
|
|
5285
|
+
});
|
|
5286
|
+
sink.closed = false;
|
|
5287
|
+
return sink;
|
|
5288
|
+
}
|
|
5289
|
+
async function streamSnapshotToReplica(sink, nodeId, requestId, indexName, build, metadata, options = {}) {
|
|
5290
|
+
const snapshotBytes = build.bytes;
|
|
5291
|
+
const totalBytes = snapshotBytes.byteLength;
|
|
5292
|
+
const checksum = build.checksum;
|
|
5293
|
+
const header = {
|
|
5294
|
+
lastSeqNo: metadata.lastSeqNo,
|
|
5295
|
+
primaryTerm: metadata.primaryTerm,
|
|
5296
|
+
partitionId: metadata.partitionId,
|
|
5297
|
+
indexName,
|
|
5298
|
+
checksum
|
|
5299
|
+
};
|
|
5300
|
+
const startPayload = { header, totalBytes };
|
|
5301
|
+
const startBytes = encode(startPayload);
|
|
5302
|
+
assertMessageWithinLimit(startBytes, "SNAPSHOT_START");
|
|
5303
|
+
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_START, nodeId, requestId, startBytes);
|
|
5304
|
+
await yieldToEventLoop();
|
|
5305
|
+
let offset = 0;
|
|
5306
|
+
let lastYieldAt = now();
|
|
5307
|
+
while (offset < totalBytes) {
|
|
5308
|
+
const end = Math.min(offset + SNAPSHOT_CHUNK_SIZE, totalBytes);
|
|
5309
|
+
const chunk = snapshotBytes.subarray(offset, end);
|
|
5310
|
+
const chunkPayload = {
|
|
5311
|
+
partitionId: metadata.partitionId,
|
|
5312
|
+
indexName,
|
|
5313
|
+
offset,
|
|
5314
|
+
data: chunk
|
|
5315
|
+
};
|
|
5316
|
+
const chunkBytes = encode(chunkPayload);
|
|
5317
|
+
assertMessageWithinLimit(chunkBytes, "SNAPSHOT_CHUNK");
|
|
5318
|
+
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_CHUNK, nodeId, requestId, chunkBytes);
|
|
5319
|
+
offset = end;
|
|
5320
|
+
if (offset < totalBytes && now() - lastYieldAt >= CHUNK_YIELD_INTERVAL_MS) {
|
|
5321
|
+
await yieldToEventLoop();
|
|
5322
|
+
lastYieldAt = now();
|
|
5591
5323
|
}
|
|
5592
|
-
|
|
5593
|
-
|
|
5324
|
+
}
|
|
5325
|
+
await yieldToEventLoop();
|
|
5326
|
+
const endPayload = {
|
|
5327
|
+
partitionId: metadata.partitionId,
|
|
5328
|
+
indexName,
|
|
5329
|
+
totalBytes,
|
|
5330
|
+
checksum
|
|
5331
|
+
};
|
|
5332
|
+
const endBytes = encode(endPayload);
|
|
5333
|
+
assertMessageWithinLimit(endBytes, "SNAPSHOT_END");
|
|
5334
|
+
respondMessage(sink, ReplicationMessageTypes.SNAPSHOT_END, nodeId, requestId, endBytes);
|
|
5335
|
+
if (options.closeOnEnd !== false) {
|
|
5336
|
+
sink.closed = true;
|
|
5337
|
+
}
|
|
5338
|
+
}
|
|
5339
|
+
function now() {
|
|
5340
|
+
if (typeof performance !== "undefined" && typeof performance.now === "function") {
|
|
5341
|
+
return performance.now();
|
|
5342
|
+
}
|
|
5343
|
+
return Date.now();
|
|
5344
|
+
}
|
|
5345
|
+
function yieldToEventLoop() {
|
|
5346
|
+
return new Promise((resolve) => {
|
|
5347
|
+
if (typeof setImmediate === "function") {
|
|
5348
|
+
setImmediate(resolve);
|
|
5349
|
+
return;
|
|
5594
5350
|
}
|
|
5351
|
+
setTimeout(resolve, 0);
|
|
5352
|
+
});
|
|
5353
|
+
}
|
|
5354
|
+
function respondMessage(sink, type, sourceId, requestId, payload) {
|
|
5355
|
+
sink({ type, sourceId, requestId, payload });
|
|
5356
|
+
}
|
|
5357
|
+
function assertMessageWithinLimit(bytes, label) {
|
|
5358
|
+
if (bytes.byteLength > MAX_MESSAGE_SIZE_BYTES) {
|
|
5359
|
+
throw new NarsilError(
|
|
5360
|
+
ErrorCodes.CONFIG_INVALID,
|
|
5361
|
+
`${label} payload (${bytes.byteLength} bytes) exceeds transport message limit (${MAX_MESSAGE_SIZE_BYTES})`
|
|
5362
|
+
);
|
|
5595
5363
|
}
|
|
5596
|
-
return false;
|
|
5597
5364
|
}
|
|
5598
|
-
|
|
5599
|
-
|
|
5600
|
-
|
|
5601
|
-
|
|
5602
|
-
|
|
5603
|
-
|
|
5604
|
-
|
|
5605
|
-
|
|
5606
|
-
|
|
5607
|
-
function createSnapshotSyncHandlerState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
|
|
5608
|
-
return createSnapshotCacheState(maxConcurrent, maxPerSource, maxStreamsPerIndex);
|
|
5365
|
+
function respondError(sink, sourceId, requestId, code, message) {
|
|
5366
|
+
const response = {
|
|
5367
|
+
type: SNAPSHOT_SYNC_ERROR_TYPE,
|
|
5368
|
+
sourceId,
|
|
5369
|
+
requestId,
|
|
5370
|
+
payload: encode({ error: true, code, message })
|
|
5371
|
+
};
|
|
5372
|
+
sink(response);
|
|
5373
|
+
sink.closed = true;
|
|
5609
5374
|
}
|
|
5375
|
+
|
|
5376
|
+
// src/distribution/cluster-node/snapshot-sync-handler/metadata.ts
|
|
5610
5377
|
async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
|
|
5611
5378
|
try {
|
|
5612
5379
|
const allocation = await coordinator.getAllocation(indexName);
|
|
@@ -5628,81 +5395,24 @@ async function defaultSnapshotHeaderMetadataProvider(coordinator, indexName) {
|
|
|
5628
5395
|
primaryTerm: 0,
|
|
5629
5396
|
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
5630
5397
|
};
|
|
5631
|
-
}
|
|
5632
|
-
async function
|
|
5633
|
-
const
|
|
5634
|
-
|
|
5635
|
-
|
|
5636
|
-
respondError(
|
|
5637
|
-
sink,
|
|
5638
|
-
deps.nodeId,
|
|
5639
|
-
message.requestId,
|
|
5640
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5641
|
-
`SNAPSHOT_SYNC_REQUEST payload (${message.payload.byteLength} bytes) exceeds the ${MAX_SNAPSHOT_SYNC_REQUEST_BYTES} byte limit`
|
|
5642
|
-
);
|
|
5643
|
-
return;
|
|
5644
|
-
}
|
|
5645
|
-
await runSnapshotSyncRequest(message, sink, deps);
|
|
5646
|
-
} catch (err) {
|
|
5647
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
5648
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
5649
|
-
respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
5650
|
-
}
|
|
5651
|
-
}
|
|
5652
|
-
async function runSnapshotSyncRequest(message, sink, deps) {
|
|
5653
|
-
const request = decodeRequest(message, sink, deps);
|
|
5654
|
-
if (request === null) {
|
|
5655
|
-
return;
|
|
5656
|
-
}
|
|
5657
|
-
await streamValidatedSnapshotRequest(message, sink, request, deps);
|
|
5658
|
-
}
|
|
5659
|
-
async function streamValidatedSnapshotRequest(message, sink, request, deps, options = {}) {
|
|
5660
|
-
const sourceIdError = validateSourceId(message.sourceId);
|
|
5661
|
-
if (sourceIdError !== null) {
|
|
5662
|
-
respondError(sink, deps.nodeId, message.requestId, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
|
|
5663
|
-
return;
|
|
5664
|
-
}
|
|
5665
|
-
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
5666
|
-
if (authResult.outcome === "denied") {
|
|
5667
|
-
respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
|
|
5668
|
-
return;
|
|
5669
|
-
}
|
|
5670
|
-
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
|
|
5671
|
-
if (existingIndex === void 0) {
|
|
5672
|
-
respondError(
|
|
5673
|
-
sink,
|
|
5674
|
-
deps.nodeId,
|
|
5675
|
-
message.requestId,
|
|
5676
|
-
ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
|
|
5677
|
-
`Index '${request.indexName}' is not hosted on this node`
|
|
5678
|
-
);
|
|
5679
|
-
return;
|
|
5680
|
-
}
|
|
5681
|
-
await acquireAndStream(message, sink, request, deps, options);
|
|
5682
|
-
}
|
|
5683
|
-
function decodeRequest(message, sink, deps) {
|
|
5684
|
-
try {
|
|
5685
|
-
const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
|
|
5686
|
-
return validateSnapshotSyncRequestPayload(decoded);
|
|
5687
|
-
} catch (err) {
|
|
5688
|
-
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
5689
|
-
const errMessage = err instanceof Error ? err.message : String(err);
|
|
5690
|
-
respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
5691
|
-
return null;
|
|
5692
|
-
}
|
|
5693
|
-
}
|
|
5694
|
-
function validateSourceId(sourceId) {
|
|
5695
|
-
if (typeof sourceId !== "string" || sourceId.length === 0) {
|
|
5696
|
-
return "request sourceId is missing";
|
|
5697
|
-
}
|
|
5698
|
-
if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
|
|
5699
|
-
return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
|
|
5398
|
+
}
|
|
5399
|
+
async function resolveHeaderMetadata(deps, indexName, partitionId) {
|
|
5400
|
+
const provider = deps.resolveHeaderMetadata;
|
|
5401
|
+
if (provider === void 0) {
|
|
5402
|
+
return defaultSnapshotHeaderMetadataProvider(deps.coordinator, indexName);
|
|
5700
5403
|
}
|
|
5701
|
-
|
|
5702
|
-
return
|
|
5404
|
+
try {
|
|
5405
|
+
return await provider(indexName, partitionId);
|
|
5406
|
+
} catch (_) {
|
|
5407
|
+
return {
|
|
5408
|
+
partitionId: SNAPSHOT_HEADER_SENTINEL_PARTITION_ID,
|
|
5409
|
+
primaryTerm: 0,
|
|
5410
|
+
lastSeqNo: SNAPSHOT_HEADER_SENTINEL_SEQNO
|
|
5411
|
+
};
|
|
5703
5412
|
}
|
|
5704
|
-
return null;
|
|
5705
5413
|
}
|
|
5414
|
+
|
|
5415
|
+
// src/distribution/cluster-node/snapshot-sync-handler/stream.ts
|
|
5706
5416
|
async function acquireAndStream(message, sink, request, deps, options) {
|
|
5707
5417
|
const { indexName } = request;
|
|
5708
5418
|
let sourceHandle;
|
|
@@ -5783,241 +5493,411 @@ function buildSnapshotForRequest(deps, indexName, options) {
|
|
|
5783
5493
|
}
|
|
5784
5494
|
return buildSnapshot(deps, indexName);
|
|
5785
5495
|
}
|
|
5786
|
-
|
|
5787
|
-
|
|
5788
|
-
|
|
5789
|
-
|
|
5496
|
+
var INDEX_NAME_PATTERN = /^[a-zA-Z0-9][a-zA-Z0-9._-]*$/;
|
|
5497
|
+
var MAX_SOURCE_ID_LENGTH = 256;
|
|
5498
|
+
function containsControlCharacter(value) {
|
|
5499
|
+
for (let i = 0; i < value.length; i++) {
|
|
5500
|
+
const code = value.charCodeAt(i);
|
|
5501
|
+
if (code < 32) {
|
|
5502
|
+
return true;
|
|
5503
|
+
}
|
|
5504
|
+
if (code >= 127 && code <= 159) {
|
|
5505
|
+
return true;
|
|
5506
|
+
}
|
|
5507
|
+
if (code === 8232 || code === 8233 || code === 65279) {
|
|
5508
|
+
return true;
|
|
5509
|
+
}
|
|
5790
5510
|
}
|
|
5511
|
+
return false;
|
|
5512
|
+
}
|
|
5513
|
+
var MAX_SNAPSHOT_SYNC_REQUEST_BYTES = 4096;
|
|
5514
|
+
var REQUEST_DECODE_OPTIONS = {
|
|
5515
|
+
maxMapLength: 16,
|
|
5516
|
+
maxArrayLength: 16,
|
|
5517
|
+
maxStrLength: MAX_INDEX_NAME_LENGTH,
|
|
5518
|
+
maxBinLength: 0,
|
|
5519
|
+
maxExtLength: 0,
|
|
5520
|
+
keyDecoder: null
|
|
5521
|
+
};
|
|
5522
|
+
function decodeRequest(message, sink, deps) {
|
|
5791
5523
|
try {
|
|
5792
|
-
|
|
5793
|
-
|
|
5794
|
-
|
|
5795
|
-
|
|
5796
|
-
|
|
5797
|
-
|
|
5798
|
-
|
|
5524
|
+
const decoded = decode(message.payload, REQUEST_DECODE_OPTIONS);
|
|
5525
|
+
return validateSnapshotSyncRequestPayload(decoded);
|
|
5526
|
+
} catch (err) {
|
|
5527
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
5528
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
5529
|
+
respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
5530
|
+
return null;
|
|
5531
|
+
}
|
|
5532
|
+
}
|
|
5533
|
+
function validateSourceId(sourceId) {
|
|
5534
|
+
if (typeof sourceId !== "string" || sourceId.length === 0) {
|
|
5535
|
+
return "request sourceId is missing";
|
|
5536
|
+
}
|
|
5537
|
+
if (sourceId.length > MAX_SOURCE_ID_LENGTH) {
|
|
5538
|
+
return `request sourceId exceeds ${MAX_SOURCE_ID_LENGTH} characters`;
|
|
5539
|
+
}
|
|
5540
|
+
if (containsControlCharacter(sourceId)) {
|
|
5541
|
+
return "request sourceId contains control characters";
|
|
5542
|
+
}
|
|
5543
|
+
return null;
|
|
5544
|
+
}
|
|
5545
|
+
function validateSnapshotSyncRequestPayload(decoded) {
|
|
5546
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
5547
|
+
throw new NarsilError(
|
|
5548
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5549
|
+
"Invalid SnapshotSyncRequestPayload: expected an object"
|
|
5550
|
+
);
|
|
5551
|
+
}
|
|
5552
|
+
const record = decoded;
|
|
5553
|
+
const indexName = record.indexName;
|
|
5554
|
+
if (typeof indexName !== "string" || indexName.length === 0) {
|
|
5555
|
+
throw new NarsilError(
|
|
5556
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5557
|
+
'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
|
|
5558
|
+
);
|
|
5559
|
+
}
|
|
5560
|
+
if (indexName.length > MAX_INDEX_NAME_LENGTH) {
|
|
5561
|
+
throw new NarsilError(
|
|
5562
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5563
|
+
`Invalid SnapshotSyncRequestPayload: "indexName" must be at most ${MAX_INDEX_NAME_LENGTH} characters`,
|
|
5564
|
+
{ length: indexName.length, limit: MAX_INDEX_NAME_LENGTH }
|
|
5565
|
+
);
|
|
5566
|
+
}
|
|
5567
|
+
if (!INDEX_NAME_PATTERN.test(indexName) || indexName.includes("..")) {
|
|
5568
|
+
throw new NarsilError(
|
|
5569
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5570
|
+
'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
|
|
5571
|
+
);
|
|
5572
|
+
}
|
|
5573
|
+
const partitionId = record.partitionId;
|
|
5574
|
+
if (partitionId !== void 0 && partitionId !== null) {
|
|
5575
|
+
if (typeof partitionId !== "number" || !Number.isInteger(partitionId) || partitionId < 0) {
|
|
5576
|
+
throw new NarsilError(
|
|
5577
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5578
|
+
'Invalid SnapshotSyncRequestPayload: "partitionId" must be a non-negative integer when provided'
|
|
5579
|
+
);
|
|
5580
|
+
}
|
|
5581
|
+
return { indexName, partitionId };
|
|
5582
|
+
}
|
|
5583
|
+
return { indexName, partitionId: null };
|
|
5584
|
+
}
|
|
5585
|
+
|
|
5586
|
+
// src/distribution/cluster-node/snapshot-sync-handler/index.ts
|
|
5587
|
+
function createSnapshotSyncHandlerState(maxConcurrent = DEFAULT_MAX_CONCURRENT_SNAPSHOTS, maxPerSource = DEFAULT_MAX_PER_SOURCE_SNAPSHOTS, maxStreamsPerIndex = DEFAULT_MAX_STREAMS_PER_INDEX) {
|
|
5588
|
+
return createSnapshotCacheState(maxConcurrent, maxPerSource, maxStreamsPerIndex);
|
|
5589
|
+
}
|
|
5590
|
+
async function handleSnapshotSyncRequest(message, respond, deps) {
|
|
5591
|
+
const sink = createSingleResponseSink(respond);
|
|
5592
|
+
try {
|
|
5593
|
+
if (message.payload.byteLength > MAX_SNAPSHOT_SYNC_REQUEST_BYTES) {
|
|
5594
|
+
respondError(
|
|
5595
|
+
sink,
|
|
5596
|
+
deps.nodeId,
|
|
5597
|
+
message.requestId,
|
|
5598
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5599
|
+
`SNAPSHOT_SYNC_REQUEST payload (${message.payload.byteLength} bytes) exceeds the ${MAX_SNAPSHOT_SYNC_REQUEST_BYTES} byte limit`
|
|
5600
|
+
);
|
|
5601
|
+
return;
|
|
5602
|
+
}
|
|
5603
|
+
await runSnapshotSyncRequest(message, sink, deps);
|
|
5604
|
+
} catch (err) {
|
|
5605
|
+
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
5606
|
+
const errMessage = err instanceof Error ? err.message : String(err);
|
|
5607
|
+
respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
5608
|
+
}
|
|
5609
|
+
}
|
|
5610
|
+
async function runSnapshotSyncRequest(message, sink, deps) {
|
|
5611
|
+
const request = decodeRequest(message, sink, deps);
|
|
5612
|
+
if (request === null) {
|
|
5613
|
+
return;
|
|
5614
|
+
}
|
|
5615
|
+
await streamValidatedSnapshotRequest(message, sink, request, deps);
|
|
5616
|
+
}
|
|
5617
|
+
async function streamValidatedSnapshotRequest(message, sink, request, deps, options = {}) {
|
|
5618
|
+
const sourceIdError = validateSourceId(message.sourceId);
|
|
5619
|
+
if (sourceIdError !== null) {
|
|
5620
|
+
respondError(sink, deps.nodeId, message.requestId, ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID, sourceIdError);
|
|
5621
|
+
return;
|
|
5622
|
+
}
|
|
5623
|
+
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
5624
|
+
if (authResult.outcome === "denied") {
|
|
5625
|
+
respondError(sink, deps.nodeId, message.requestId, authResult.code, authResult.reason);
|
|
5626
|
+
return;
|
|
5627
|
+
}
|
|
5628
|
+
const existingIndex = deps.engine.listIndexes().find((idx) => idx.name === request.indexName);
|
|
5629
|
+
if (existingIndex === void 0) {
|
|
5630
|
+
respondError(
|
|
5631
|
+
sink,
|
|
5632
|
+
deps.nodeId,
|
|
5633
|
+
message.requestId,
|
|
5634
|
+
ErrorCodes.SNAPSHOT_SYNC_INDEX_NOT_FOUND,
|
|
5635
|
+
`Index '${request.indexName}' is not hosted on this node`
|
|
5636
|
+
);
|
|
5637
|
+
return;
|
|
5638
|
+
}
|
|
5639
|
+
await acquireAndStream(message, sink, request, deps, options);
|
|
5640
|
+
}
|
|
5641
|
+
async function handleSearch(message, respond, deps) {
|
|
5642
|
+
const decoded = decode(message.payload);
|
|
5643
|
+
const payload = validateSearchPayload(decoded);
|
|
5644
|
+
const queryParams = {
|
|
5645
|
+
term: payload.params.term ?? void 0,
|
|
5646
|
+
fields: payload.params.fields ?? void 0,
|
|
5647
|
+
filters: payload.params.filters ?? void 0,
|
|
5648
|
+
boost: payload.params.boost ?? void 0,
|
|
5649
|
+
scoring: payload.params.scoring,
|
|
5650
|
+
tolerance: payload.params.tolerance ?? void 0,
|
|
5651
|
+
minScore: payload.params.threshold ?? void 0,
|
|
5652
|
+
limit: payload.params.limit,
|
|
5653
|
+
offset: payload.params.offset,
|
|
5654
|
+
searchAfter: payload.params.searchAfter ?? void 0,
|
|
5655
|
+
sort: convertWireSortToLocal(payload.params.sort),
|
|
5656
|
+
group: payload.params.group !== null ? { fields: [payload.params.group.field], maxPerGroup: payload.params.group.maxPerGroup } : void 0,
|
|
5657
|
+
facets: convertWireFacetsToLocal(payload.params.facets, payload.facetShardSize ?? payload.params.facetSize),
|
|
5658
|
+
vector: payload.params.vector !== null ? {
|
|
5659
|
+
field: payload.params.vector.field,
|
|
5660
|
+
value: payload.params.vector.value ?? void 0,
|
|
5661
|
+
text: payload.params.vector.text ?? void 0,
|
|
5662
|
+
similarity: payload.params.vector.similarity ?? void 0
|
|
5663
|
+
} : void 0,
|
|
5664
|
+
hybrid: payload.params.hybrid !== null ? { strategy: payload.params.hybrid.strategy, k: payload.params.hybrid.k, alpha: payload.params.hybrid.alpha } : void 0
|
|
5665
|
+
};
|
|
5666
|
+
const queryResult = await deps.engine.query(payload.indexName, queryParams);
|
|
5667
|
+
const scored = queryResult.hits.map((hit) => ({
|
|
5668
|
+
docId: hit.id,
|
|
5669
|
+
score: hit.score,
|
|
5670
|
+
sortValues: null
|
|
5671
|
+
}));
|
|
5672
|
+
const results = payload.partitionIds.map((partitionId) => ({
|
|
5673
|
+
partitionId,
|
|
5674
|
+
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
5675
|
+
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
5676
|
+
}));
|
|
5677
|
+
const resultPayload = { results, facets: convertLocalFacetsToWire(queryResult.facets) };
|
|
5678
|
+
respond({
|
|
5679
|
+
type: QueryMessageTypes.SEARCH_RESULT,
|
|
5680
|
+
sourceId: deps.nodeId,
|
|
5681
|
+
requestId: message.requestId,
|
|
5682
|
+
payload: encode(resultPayload)
|
|
5683
|
+
});
|
|
5684
|
+
}
|
|
5685
|
+
async function handleFetch(message, respond, deps) {
|
|
5686
|
+
const decoded = decode(message.payload);
|
|
5687
|
+
const payload = validateFetchPayload(decoded);
|
|
5688
|
+
const documents = [];
|
|
5689
|
+
for (const docRef of payload.documentIds) {
|
|
5690
|
+
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
5691
|
+
if (doc !== void 0) {
|
|
5692
|
+
documents.push({
|
|
5693
|
+
docId: docRef.docId,
|
|
5694
|
+
document: doc,
|
|
5695
|
+
highlights: null
|
|
5696
|
+
});
|
|
5697
|
+
}
|
|
5698
|
+
}
|
|
5699
|
+
const resultPayload = { documents };
|
|
5700
|
+
respond({
|
|
5701
|
+
type: QueryMessageTypes.FETCH_RESULT,
|
|
5702
|
+
sourceId: deps.nodeId,
|
|
5703
|
+
requestId: message.requestId,
|
|
5704
|
+
payload: encode(resultPayload)
|
|
5705
|
+
});
|
|
5706
|
+
}
|
|
5707
|
+
async function handleStats(message, respond, deps) {
|
|
5708
|
+
const decoded = decode(message.payload);
|
|
5709
|
+
const payload = validateStatsPayload(decoded);
|
|
5710
|
+
const stats = deps.engine.getStats(payload.indexName);
|
|
5711
|
+
const resultPayload = {
|
|
5712
|
+
totalDocuments: stats.documentCount,
|
|
5713
|
+
docFrequencies: {},
|
|
5714
|
+
totalFieldLengths: {}
|
|
5715
|
+
};
|
|
5716
|
+
respond({
|
|
5717
|
+
type: QueryMessageTypes.STATS_RESULT,
|
|
5718
|
+
sourceId: deps.nodeId,
|
|
5719
|
+
requestId: message.requestId,
|
|
5720
|
+
payload: encode(resultPayload)
|
|
5721
|
+
});
|
|
5722
|
+
}
|
|
5723
|
+
function convertWireSortToLocal(wireSort) {
|
|
5724
|
+
if (wireSort === null || wireSort.length === 0) {
|
|
5725
|
+
return void 0;
|
|
5726
|
+
}
|
|
5727
|
+
const result = {};
|
|
5728
|
+
for (const entry of wireSort) {
|
|
5729
|
+
result[entry.field] = entry.direction;
|
|
5799
5730
|
}
|
|
5731
|
+
return result;
|
|
5800
5732
|
}
|
|
5801
|
-
function
|
|
5802
|
-
if (
|
|
5803
|
-
|
|
5804
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5805
|
-
"Invalid SnapshotSyncRequestPayload: expected an object"
|
|
5806
|
-
);
|
|
5733
|
+
function convertWireFacetsToLocal(facets, limit) {
|
|
5734
|
+
if (facets === null || facets.length === 0) {
|
|
5735
|
+
return void 0;
|
|
5807
5736
|
}
|
|
5808
|
-
const
|
|
5809
|
-
const
|
|
5810
|
-
|
|
5811
|
-
throw new NarsilError(
|
|
5812
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5813
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" must be a non-empty string'
|
|
5814
|
-
);
|
|
5737
|
+
const result = {};
|
|
5738
|
+
for (const field of facets) {
|
|
5739
|
+
result[field] = limit !== null ? { limit } : {};
|
|
5815
5740
|
}
|
|
5816
|
-
|
|
5817
|
-
|
|
5818
|
-
|
|
5819
|
-
|
|
5820
|
-
|
|
5821
|
-
);
|
|
5741
|
+
return result;
|
|
5742
|
+
}
|
|
5743
|
+
function convertLocalFacetsToWire(facets) {
|
|
5744
|
+
if (facets === void 0) {
|
|
5745
|
+
return null;
|
|
5822
5746
|
}
|
|
5823
|
-
|
|
5824
|
-
|
|
5825
|
-
|
|
5826
|
-
'Invalid SnapshotSyncRequestPayload: "indexName" contains invalid characters'
|
|
5827
|
-
);
|
|
5747
|
+
const result = {};
|
|
5748
|
+
for (const [field, facet] of Object.entries(facets)) {
|
|
5749
|
+
result[field] = Object.entries(facet.values).map(([value, count]) => ({ value, count }));
|
|
5828
5750
|
}
|
|
5829
|
-
|
|
5830
|
-
|
|
5831
|
-
|
|
5832
|
-
|
|
5833
|
-
|
|
5834
|
-
|
|
5835
|
-
|
|
5751
|
+
return result;
|
|
5752
|
+
}
|
|
5753
|
+
function decideSyncTier(log, replicaLastSeqNo) {
|
|
5754
|
+
const oldest = log.oldestSeqNo;
|
|
5755
|
+
if (oldest === void 0) {
|
|
5756
|
+
if (log.committedSeqNo === 0) {
|
|
5757
|
+
return replicaLastSeqNo === 0 ? "incremental" : "snapshot";
|
|
5836
5758
|
}
|
|
5837
|
-
return
|
|
5759
|
+
return replicaLastSeqNo >= log.committedSeqNo ? "incremental" : "snapshot";
|
|
5838
5760
|
}
|
|
5839
|
-
|
|
5761
|
+
if (oldest <= replicaLastSeqNo + 1) {
|
|
5762
|
+
return "incremental";
|
|
5763
|
+
}
|
|
5764
|
+
return "snapshot";
|
|
5840
5765
|
}
|
|
5841
|
-
|
|
5842
|
-
|
|
5843
|
-
|
|
5844
|
-
return { acknowledged: [], failed: [] };
|
|
5766
|
+
function validateSyncRequest(decoded) {
|
|
5767
|
+
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
5768
|
+
throw new Error("Invalid SyncRequestPayload: expected an object");
|
|
5845
5769
|
}
|
|
5846
|
-
const
|
|
5847
|
-
|
|
5848
|
-
|
|
5849
|
-
);
|
|
5850
|
-
const acknowledged = [];
|
|
5851
|
-
const failed = [];
|
|
5852
|
-
for (let i = 0; i < sendResults.length; i++) {
|
|
5853
|
-
const result = sendResults[i];
|
|
5854
|
-
const replicaNodeId = uniqueReplicas[i];
|
|
5855
|
-
if (result.status === "rejected") {
|
|
5856
|
-
if (result.reason instanceof TransportError) {
|
|
5857
|
-
failed.push(replicaNodeId);
|
|
5858
|
-
continue;
|
|
5859
|
-
}
|
|
5860
|
-
throw result.reason;
|
|
5861
|
-
}
|
|
5862
|
-
const response = result.value;
|
|
5863
|
-
if (isMatchingAck(response, entry)) {
|
|
5864
|
-
acknowledged.push(replicaNodeId);
|
|
5865
|
-
} else {
|
|
5866
|
-
failed.push(replicaNodeId);
|
|
5867
|
-
}
|
|
5770
|
+
const record = decoded;
|
|
5771
|
+
if (typeof record.indexName !== "string") {
|
|
5772
|
+
throw new Error('Invalid SyncRequestPayload: "indexName" must be a string');
|
|
5868
5773
|
}
|
|
5869
|
-
|
|
5870
|
-
|
|
5871
|
-
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets2) {
|
|
5872
|
-
const targets = resolveNodeTargets2 === void 0 ? [replicaNodeId] : await resolveNodeTargets2(replicaNodeId);
|
|
5873
|
-
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
5874
|
-
let lastError;
|
|
5875
|
-
for (const target of resolvedTargets) {
|
|
5876
|
-
try {
|
|
5877
|
-
return await transport.send(target, message);
|
|
5878
|
-
} catch (error) {
|
|
5879
|
-
lastError = error;
|
|
5880
|
-
}
|
|
5774
|
+
if (typeof record.partitionId !== "number" || !Number.isInteger(record.partitionId) || record.partitionId < 0) {
|
|
5775
|
+
throw new Error('Invalid SyncRequestPayload: "partitionId" must be a non-negative integer');
|
|
5881
5776
|
}
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
function isMatchingAck(response, entry) {
|
|
5885
|
-
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
5886
|
-
return false;
|
|
5777
|
+
if (typeof record.lastSeqNo !== "number" || !Number.isInteger(record.lastSeqNo) || record.lastSeqNo < 0) {
|
|
5778
|
+
throw new Error('Invalid SyncRequestPayload: "lastSeqNo" must be a non-negative integer');
|
|
5887
5779
|
}
|
|
5888
|
-
|
|
5889
|
-
|
|
5890
|
-
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
5891
|
-
} catch {
|
|
5892
|
-
return false;
|
|
5780
|
+
if (typeof record.lastPrimaryTerm !== "number" || !Number.isInteger(record.lastPrimaryTerm) || record.lastPrimaryTerm < 0) {
|
|
5781
|
+
throw new Error('Invalid SyncRequestPayload: "lastPrimaryTerm" must be a non-negative integer');
|
|
5893
5782
|
}
|
|
5783
|
+
return record;
|
|
5894
5784
|
}
|
|
5895
5785
|
|
|
5896
|
-
// src/distribution/cluster-node/
|
|
5897
|
-
|
|
5898
|
-
|
|
5899
|
-
|
|
5900
|
-
|
|
5901
|
-
|
|
5902
|
-
|
|
5903
|
-
|
|
5904
|
-
|
|
5905
|
-
|
|
5906
|
-
|
|
5907
|
-
|
|
5908
|
-
|
|
5786
|
+
// src/distribution/cluster-node/message-handler/sync.ts
|
|
5787
|
+
async function handleSyncRequestMessage(message, respond, deps) {
|
|
5788
|
+
const request = validateSyncRequest(decode(message.payload));
|
|
5789
|
+
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
5790
|
+
if (authResult.outcome === "denied") {
|
|
5791
|
+
throw new NarsilError(authResult.code, authResult.reason, {
|
|
5792
|
+
indexName: request.indexName,
|
|
5793
|
+
partitionId: request.partitionId,
|
|
5794
|
+
sourceNodeId: message.sourceId
|
|
5795
|
+
});
|
|
5796
|
+
}
|
|
5797
|
+
const table = await deps.coordinator.getAllocation(request.indexName);
|
|
5798
|
+
const assignment = table?.assignments.get(request.partitionId);
|
|
5799
|
+
if (assignment === void 0) {
|
|
5909
5800
|
throw new NarsilError(
|
|
5910
|
-
ErrorCodes.
|
|
5911
|
-
`
|
|
5912
|
-
{
|
|
5801
|
+
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
5802
|
+
`No assignment exists for sync request partition ${request.partitionId} of index '${request.indexName}'`,
|
|
5803
|
+
{ indexName: request.indexName, partitionId: request.partitionId }
|
|
5913
5804
|
);
|
|
5914
5805
|
}
|
|
5915
|
-
if (
|
|
5806
|
+
if (assignment.primary !== deps.nodeId) {
|
|
5916
5807
|
throw new NarsilError(
|
|
5917
|
-
ErrorCodes.
|
|
5918
|
-
`
|
|
5919
|
-
{
|
|
5808
|
+
ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
5809
|
+
`Node '${deps.nodeId}' is not primary for sync request partition ${request.partitionId} of index '${request.indexName}'`,
|
|
5810
|
+
{
|
|
5811
|
+
indexName: request.indexName,
|
|
5812
|
+
partitionId: request.partitionId,
|
|
5813
|
+
localNodeId: deps.nodeId,
|
|
5814
|
+
primaryNodeId: assignment.primary
|
|
5815
|
+
}
|
|
5920
5816
|
);
|
|
5921
5817
|
}
|
|
5922
|
-
|
|
5923
|
-
|
|
5924
|
-
validateIndexName(name);
|
|
5925
|
-
const partitionCount = options?.partitionCount ?? DEFAULT_PARTITION_COUNT;
|
|
5926
|
-
const replicationFactor = options?.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
|
|
5927
|
-
validateCreateIndexOptions(partitionCount, replicationFactor);
|
|
5928
|
-
const constraints = {
|
|
5929
|
-
zoneAwareness: false,
|
|
5930
|
-
zoneAttribute: "zone",
|
|
5931
|
-
maxShardsPerNode: null
|
|
5932
|
-
};
|
|
5933
|
-
const metadata = {
|
|
5934
|
-
indexName: name,
|
|
5935
|
-
partitionCount,
|
|
5936
|
-
replicationFactor,
|
|
5937
|
-
constraints
|
|
5938
|
-
};
|
|
5939
|
-
const stored = await putIndexMetadata(coordinator, metadata);
|
|
5940
|
-
if (!stored) {
|
|
5818
|
+
const sourceAssigned = assignment.primary === message.sourceId || assignment.replicas.includes(message.sourceId);
|
|
5819
|
+
if (!sourceAssigned) {
|
|
5941
5820
|
throw new NarsilError(
|
|
5942
|
-
ErrorCodes.
|
|
5943
|
-
`
|
|
5944
|
-
{ indexName:
|
|
5821
|
+
ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
5822
|
+
`Node '${message.sourceId}' is not assigned to partition ${request.partitionId} of index '${request.indexName}'`,
|
|
5823
|
+
{ indexName: request.indexName, partitionId: request.partitionId, sourceNodeId: message.sourceId }
|
|
5945
5824
|
);
|
|
5946
5825
|
}
|
|
5947
|
-
|
|
5948
|
-
|
|
5949
|
-
|
|
5950
|
-
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5826
|
+
if (request.lastPrimaryTerm > assignment.primaryTerm) {
|
|
5827
|
+
throw new NarsilError(
|
|
5828
|
+
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
5829
|
+
`Replica term ${request.lastPrimaryTerm} is newer than primary term ${assignment.primaryTerm}`,
|
|
5830
|
+
{
|
|
5831
|
+
indexName: request.indexName,
|
|
5832
|
+
partitionId: request.partitionId,
|
|
5833
|
+
lastPrimaryTerm: request.lastPrimaryTerm,
|
|
5834
|
+
primaryTerm: assignment.primaryTerm
|
|
5835
|
+
}
|
|
5836
|
+
);
|
|
5837
|
+
}
|
|
5838
|
+
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
5839
|
+
const tier = decideSyncTier(log, request.lastSeqNo);
|
|
5840
|
+
if (tier === "incremental") {
|
|
5841
|
+
sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
5842
|
+
return;
|
|
5843
|
+
}
|
|
5844
|
+
const snapshotSeqNo = log.committedSeqNo;
|
|
5845
|
+
const sink = createSingleResponseSink(respond);
|
|
5846
|
+
await streamValidatedSnapshotRequest(
|
|
5847
|
+
message,
|
|
5848
|
+
sink,
|
|
5849
|
+
{ indexName: request.indexName, partitionId: request.partitionId },
|
|
5850
|
+
{
|
|
5851
|
+
nodeId: deps.nodeId,
|
|
5852
|
+
engine: deps.engine,
|
|
5853
|
+
coordinator: deps.coordinator,
|
|
5854
|
+
state: deps.snapshotSyncState,
|
|
5855
|
+
resolveHeaderMetadata: deps.resolveHeaderMetadata
|
|
5856
|
+
},
|
|
5857
|
+
{
|
|
5858
|
+
metadata: {
|
|
5859
|
+
partitionId: request.partitionId,
|
|
5860
|
+
lastSeqNo: snapshotSeqNo,
|
|
5861
|
+
primaryTerm: assignment.primaryTerm
|
|
5862
|
+
},
|
|
5863
|
+
closeOnEnd: false,
|
|
5864
|
+
disableBuildCache: true,
|
|
5865
|
+
buildSnapshot: async () => {
|
|
5866
|
+
const bytes = await deps.engine.serializeReplicationPartition(request.indexName, request.partitionId);
|
|
5867
|
+
return { bytes, checksum: crc32(bytes) };
|
|
5868
|
+
},
|
|
5869
|
+
afterSnapshot: (trailingSink) => {
|
|
5870
|
+
const entries = log.getEntriesFrom(snapshotSeqNo + 1);
|
|
5871
|
+
sendSyncEntriesResponse(message, trailingSink, deps, entries);
|
|
5957
5872
|
}
|
|
5958
|
-
} catch (cleanErr) {
|
|
5959
|
-
cleanupFailed = true;
|
|
5960
|
-
cleanupError = cleanErr;
|
|
5961
|
-
}
|
|
5962
|
-
if (cleanupFailed) {
|
|
5963
|
-
throw new NarsilError(
|
|
5964
|
-
ErrorCodes.CONFIG_INVALID,
|
|
5965
|
-
`Index creation for '${name}' failed and metadata cleanup also failed. The index may be in a partial state and require manual intervention.`,
|
|
5966
|
-
{
|
|
5967
|
-
indexName: name,
|
|
5968
|
-
createError: createErr instanceof Error ? createErr.message : String(createErr),
|
|
5969
|
-
cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
5970
|
-
}
|
|
5971
|
-
);
|
|
5972
5873
|
}
|
|
5973
|
-
|
|
5974
|
-
}
|
|
5975
|
-
}
|
|
5976
|
-
function resolvePartitionId(docId, partitionCount) {
|
|
5977
|
-
return fnv1a(docId) % partitionCount;
|
|
5978
|
-
}
|
|
5979
|
-
async function forwardInsertToRemote(indexName, document, docId, primaryNodeId, deps) {
|
|
5980
|
-
const payload = {
|
|
5981
|
-
indexName,
|
|
5982
|
-
documentId: docId,
|
|
5983
|
-
operation: "insert",
|
|
5984
|
-
document: encode(document),
|
|
5985
|
-
updateFields: null
|
|
5986
|
-
};
|
|
5987
|
-
const message = createForwardMessage(payload, deps.nodeId);
|
|
5988
|
-
const response = await sendToNode(primaryNodeId, message, deps);
|
|
5989
|
-
const decoded = decode(response.payload);
|
|
5990
|
-
if (typeof decoded.documentId !== "string") {
|
|
5991
|
-
throw new NarsilError(
|
|
5992
|
-
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5993
|
-
`Remote primary returned invalid forward response for index '${indexName}'`,
|
|
5994
|
-
{ indexName, primaryNodeId }
|
|
5995
|
-
);
|
|
5996
|
-
}
|
|
5997
|
-
return decoded.documentId;
|
|
5874
|
+
);
|
|
5998
5875
|
}
|
|
5999
|
-
|
|
5876
|
+
function sendSyncEntriesResponse(message, respond, deps, entries) {
|
|
6000
5877
|
const payload = {
|
|
6001
|
-
|
|
6002
|
-
|
|
6003
|
-
operation: "remove",
|
|
6004
|
-
document: null,
|
|
6005
|
-
updateFields: null
|
|
5878
|
+
entries,
|
|
5879
|
+
isLast: true
|
|
6006
5880
|
};
|
|
6007
|
-
|
|
6008
|
-
|
|
5881
|
+
respond({
|
|
5882
|
+
type: ReplicationMessageTypes.SYNC_ENTRIES,
|
|
5883
|
+
sourceId: deps.nodeId,
|
|
5884
|
+
requestId: message.requestId,
|
|
5885
|
+
payload: encode(payload)
|
|
5886
|
+
});
|
|
6009
5887
|
}
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
|
|
6015
|
-
|
|
6016
|
-
|
|
6017
|
-
|
|
6018
|
-
|
|
6019
|
-
|
|
6020
|
-
|
|
5888
|
+
|
|
5889
|
+
// src/distribution/cluster-node/types.ts
|
|
5890
|
+
var DEFAULT_PARTITION_COUNT = 5;
|
|
5891
|
+
var DEFAULT_REPLICATION_FACTOR = 1;
|
|
5892
|
+
var DEFAULT_CAPACITY = {
|
|
5893
|
+
memoryBytes: 8e9,
|
|
5894
|
+
cpuCores: 4,
|
|
5895
|
+
diskBytes: null
|
|
5896
|
+
};
|
|
5897
|
+
|
|
5898
|
+
// src/distribution/cluster-node/write-routing/assignment.ts
|
|
5899
|
+
function resolvePartitionId(docId, partitionCount) {
|
|
5900
|
+
return fnv1a(docId) % partitionCount;
|
|
6021
5901
|
}
|
|
6022
5902
|
async function resolveNodeTargets(nodeId, deps) {
|
|
6023
5903
|
if (deps.resolveNodeTargets === void 0) {
|
|
@@ -6029,7 +5909,7 @@ async function resolveNodeTargets(nodeId, deps) {
|
|
|
6029
5909
|
function requireAssignedPrimary(assignment, indexName, partitionId) {
|
|
6030
5910
|
if (assignment === void 0 || assignment.primary === null) {
|
|
6031
5911
|
throw new NarsilError(
|
|
6032
|
-
ErrorCodes.
|
|
5912
|
+
ErrorCodes.PARTITION_UNASSIGNED,
|
|
6033
5913
|
`No primary assigned for partition ${partitionId} of index '${indexName}'`,
|
|
6034
5914
|
{ indexName, partitionId }
|
|
6035
5915
|
);
|
|
@@ -6053,13 +5933,27 @@ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrim
|
|
|
6053
5933
|
const assignment = requireAssignedPrimary(table.assignments.get(partitionId), indexName, partitionId);
|
|
6054
5934
|
if (requireLocalPrimary && assignment.primary !== deps.nodeId) {
|
|
6055
5935
|
throw new NarsilError(
|
|
6056
|
-
ErrorCodes.
|
|
5936
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
6057
5937
|
`Node '${deps.nodeId}' is not primary for partition ${partitionId} of index '${indexName}'`,
|
|
6058
5938
|
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
6059
5939
|
);
|
|
6060
5940
|
}
|
|
6061
5941
|
return { partitionId, assignment };
|
|
6062
5942
|
}
|
|
5943
|
+
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
5944
|
+
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
5945
|
+
if (requiredActiveCopies <= 1) {
|
|
5946
|
+
return;
|
|
5947
|
+
}
|
|
5948
|
+
const activeCopies = 1 + getInSyncReplicaTargets(assignment, deps.nodeId).length;
|
|
5949
|
+
if (activeCopies < requiredActiveCopies) {
|
|
5950
|
+
throw new NarsilError(
|
|
5951
|
+
ErrorCodes.INSUFFICIENT_REPLICAS,
|
|
5952
|
+
`Write to partition ${partitionId} of index '${indexName}' requires ${requiredActiveCopies} active copies but only ${activeCopies} are available`,
|
|
5953
|
+
{ indexName, partitionId, requiredActiveCopies, activeCopies }
|
|
5954
|
+
);
|
|
5955
|
+
}
|
|
5956
|
+
}
|
|
6063
5957
|
function getInSyncReplicaTargets(assignment, localNodeId) {
|
|
6064
5958
|
const configuredReplicas = new Set(assignment.replicas);
|
|
6065
5959
|
const targets = [];
|
|
@@ -6070,6 +5964,51 @@ function getInSyncReplicaTargets(assignment, localNodeId) {
|
|
|
6070
5964
|
}
|
|
6071
5965
|
return targets;
|
|
6072
5966
|
}
|
|
5967
|
+
async function forwardInsertToRemote(indexName, document, docId, primaryNodeId, deps) {
|
|
5968
|
+
const payload = {
|
|
5969
|
+
indexName,
|
|
5970
|
+
documentId: docId,
|
|
5971
|
+
operation: "insert",
|
|
5972
|
+
document: encode(document),
|
|
5973
|
+
updateFields: null
|
|
5974
|
+
};
|
|
5975
|
+
const message = createForwardMessage(payload, deps.nodeId);
|
|
5976
|
+
const response = await sendToNode(primaryNodeId, message, deps);
|
|
5977
|
+
const decoded = decode(response.payload);
|
|
5978
|
+
if (typeof decoded.documentId !== "string") {
|
|
5979
|
+
throw new NarsilError(
|
|
5980
|
+
ErrorCodes.QUERY_ROUTING_FAILED,
|
|
5981
|
+
`Remote primary returned invalid forward response for index '${indexName}'`,
|
|
5982
|
+
{ indexName, primaryNodeId }
|
|
5983
|
+
);
|
|
5984
|
+
}
|
|
5985
|
+
return decoded.documentId;
|
|
5986
|
+
}
|
|
5987
|
+
async function forwardRemoveToRemote(indexName, docId, primaryNodeId, deps) {
|
|
5988
|
+
const payload = {
|
|
5989
|
+
indexName,
|
|
5990
|
+
documentId: docId,
|
|
5991
|
+
operation: "remove",
|
|
5992
|
+
document: null,
|
|
5993
|
+
updateFields: null
|
|
5994
|
+
};
|
|
5995
|
+
const message = createForwardMessage(payload, deps.nodeId);
|
|
5996
|
+
await sendToNode(primaryNodeId, message, deps);
|
|
5997
|
+
}
|
|
5998
|
+
async function sendToNode(nodeId, message, deps) {
|
|
5999
|
+
const targets = await resolveNodeTargets(nodeId, deps);
|
|
6000
|
+
let lastError;
|
|
6001
|
+
for (const target of targets) {
|
|
6002
|
+
try {
|
|
6003
|
+
return await deps.transport.send(target, message);
|
|
6004
|
+
} catch (error) {
|
|
6005
|
+
lastError = error;
|
|
6006
|
+
}
|
|
6007
|
+
}
|
|
6008
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6009
|
+
}
|
|
6010
|
+
|
|
6011
|
+
// src/distribution/cluster-node/write-routing/failures.ts
|
|
6073
6012
|
function errorMessage(error) {
|
|
6074
6013
|
return error instanceof Error ? error.message : String(error);
|
|
6075
6014
|
}
|
|
@@ -6121,6 +6060,62 @@ async function rollbackPrimaryRemove(indexName, partitionId, documentId, previou
|
|
|
6121
6060
|
}
|
|
6122
6061
|
throwWriteFailure(originalError);
|
|
6123
6062
|
}
|
|
6063
|
+
async function replicateToReplicas(entry, inSyncReplicas, transport, sourceNodeId, resolveNodeTargets3) {
|
|
6064
|
+
const uniqueReplicas = [...new Set(inSyncReplicas)];
|
|
6065
|
+
if (uniqueReplicas.length === 0) {
|
|
6066
|
+
return { acknowledged: [], failed: [] };
|
|
6067
|
+
}
|
|
6068
|
+
const message = createEntryMessage(entry, sourceNodeId);
|
|
6069
|
+
const sendResults = await Promise.allSettled(
|
|
6070
|
+
uniqueReplicas.map((replicaNodeId) => sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3))
|
|
6071
|
+
);
|
|
6072
|
+
const acknowledged = [];
|
|
6073
|
+
const failed = [];
|
|
6074
|
+
for (let i = 0; i < sendResults.length; i++) {
|
|
6075
|
+
const result = sendResults[i];
|
|
6076
|
+
const replicaNodeId = uniqueReplicas[i];
|
|
6077
|
+
if (result.status === "rejected") {
|
|
6078
|
+
if (result.reason instanceof TransportError) {
|
|
6079
|
+
failed.push(replicaNodeId);
|
|
6080
|
+
continue;
|
|
6081
|
+
}
|
|
6082
|
+
throw result.reason;
|
|
6083
|
+
}
|
|
6084
|
+
const response = result.value;
|
|
6085
|
+
if (isMatchingAck(response, entry)) {
|
|
6086
|
+
acknowledged.push(replicaNodeId);
|
|
6087
|
+
} else {
|
|
6088
|
+
failed.push(replicaNodeId);
|
|
6089
|
+
}
|
|
6090
|
+
}
|
|
6091
|
+
return { acknowledged, failed };
|
|
6092
|
+
}
|
|
6093
|
+
async function sendToReplica(transport, replicaNodeId, message, resolveNodeTargets3) {
|
|
6094
|
+
const targets = resolveNodeTargets3 === void 0 ? [replicaNodeId] : await resolveNodeTargets3(replicaNodeId);
|
|
6095
|
+
const resolvedTargets = targets.length > 0 ? targets : [replicaNodeId];
|
|
6096
|
+
let lastError;
|
|
6097
|
+
for (const target of resolvedTargets) {
|
|
6098
|
+
try {
|
|
6099
|
+
return await transport.send(target, message);
|
|
6100
|
+
} catch (error) {
|
|
6101
|
+
lastError = error;
|
|
6102
|
+
}
|
|
6103
|
+
}
|
|
6104
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6105
|
+
}
|
|
6106
|
+
function isMatchingAck(response, entry) {
|
|
6107
|
+
if (response.type !== ReplicationMessageTypes.ACK) {
|
|
6108
|
+
return false;
|
|
6109
|
+
}
|
|
6110
|
+
try {
|
|
6111
|
+
const payload = validateAckPayload(decode(response.payload));
|
|
6112
|
+
return payload.seqNo === entry.seqNo && payload.partitionId === entry.partitionId && payload.indexName === entry.indexName;
|
|
6113
|
+
} catch {
|
|
6114
|
+
return false;
|
|
6115
|
+
}
|
|
6116
|
+
}
|
|
6117
|
+
|
|
6118
|
+
// src/distribution/cluster-node/write-routing/replication.ts
|
|
6124
6119
|
async function assertPrimaryWriteAuthority(entry, deps) {
|
|
6125
6120
|
const table = await deps.coordinator.getAllocation(entry.indexName);
|
|
6126
6121
|
const currentAssignment = table?.assignments.get(entry.partitionId);
|
|
@@ -6130,7 +6125,7 @@ async function assertPrimaryWriteAuthority(entry, deps) {
|
|
|
6130
6125
|
return;
|
|
6131
6126
|
}
|
|
6132
6127
|
throw new NarsilError(
|
|
6133
|
-
ErrorCodes.
|
|
6128
|
+
ErrorCodes.PARTITION_NOT_PRIMARY,
|
|
6134
6129
|
`Primary authority changed before acknowledging write for index '${entry.indexName}' partition ${entry.partitionId}`,
|
|
6135
6130
|
{
|
|
6136
6131
|
indexName: entry.indexName,
|
|
@@ -6221,7 +6216,10 @@ async function replicateEntry(entry, assignment, deps) {
|
|
|
6221
6216
|
await removeFailedReplicasFromInsync(entry, result.failed, deps);
|
|
6222
6217
|
await assertPrimaryWriteAuthority(entry, deps);
|
|
6223
6218
|
}
|
|
6219
|
+
|
|
6220
|
+
// src/distribution/cluster-node/write-routing/primary-writes.ts
|
|
6224
6221
|
async function applyPrimaryInsert(indexName, document, docId, partitionId, assignment, deps) {
|
|
6222
|
+
assertSufficientActiveReplicas(indexName, partitionId, assignment, deps);
|
|
6225
6223
|
const insertedDocId = await deps.engine.insert(indexName, document, docId);
|
|
6226
6224
|
const storedDocument = await deps.engine.get(indexName, insertedDocId);
|
|
6227
6225
|
if (storedDocument === void 0) {
|
|
@@ -6240,6 +6238,7 @@ async function applyPrimaryInsert(indexName, document, docId, partitionId, assig
|
|
|
6240
6238
|
return insertedDocId;
|
|
6241
6239
|
}
|
|
6242
6240
|
async function applyPrimaryRemove(indexName, docId, partitionId, assignment, deps) {
|
|
6241
|
+
assertSufficientActiveReplicas(indexName, partitionId, assignment, deps);
|
|
6243
6242
|
const previousDocument = await deps.engine.get(indexName, docId);
|
|
6244
6243
|
await deps.engine.remove(indexName, docId);
|
|
6245
6244
|
try {
|
|
@@ -6262,21 +6261,93 @@ async function applyForwardedWrite(payload, deps) {
|
|
|
6262
6261
|
if (payload.document === null) {
|
|
6263
6262
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Invalid ForwardPayload: insert requires a document");
|
|
6264
6263
|
}
|
|
6265
|
-
const document = decode(payload.document);
|
|
6266
|
-
return applyPrimaryInsert(
|
|
6267
|
-
payload.indexName,
|
|
6268
|
-
document,
|
|
6269
|
-
payload.documentId,
|
|
6270
|
-
resolution.partitionId,
|
|
6271
|
-
resolution.assignment,
|
|
6272
|
-
deps
|
|
6273
|
-
);
|
|
6274
|
-
}
|
|
6275
|
-
if (payload.operation === "remove") {
|
|
6276
|
-
await applyPrimaryRemove(payload.indexName, payload.documentId, resolution.partitionId, resolution.assignment, deps);
|
|
6277
|
-
return payload.documentId;
|
|
6264
|
+
const document = decode(payload.document);
|
|
6265
|
+
return applyPrimaryInsert(
|
|
6266
|
+
payload.indexName,
|
|
6267
|
+
document,
|
|
6268
|
+
payload.documentId,
|
|
6269
|
+
resolution.partitionId,
|
|
6270
|
+
resolution.assignment,
|
|
6271
|
+
deps
|
|
6272
|
+
);
|
|
6273
|
+
}
|
|
6274
|
+
if (payload.operation === "remove") {
|
|
6275
|
+
await applyPrimaryRemove(payload.indexName, payload.documentId, resolution.partitionId, resolution.assignment, deps);
|
|
6276
|
+
return payload.documentId;
|
|
6277
|
+
}
|
|
6278
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Forward update operations are not supported yet");
|
|
6279
|
+
}
|
|
6280
|
+
|
|
6281
|
+
// src/distribution/cluster-node/write-routing/index.ts
|
|
6282
|
+
var MAX_REPLICATION_FACTOR = 255;
|
|
6283
|
+
function validateCreateIndexOptions(partitionCount, replicationFactor) {
|
|
6284
|
+
if (!Number.isInteger(partitionCount) || partitionCount < 1 || partitionCount > MAX_PARTITION_COUNT) {
|
|
6285
|
+
throw new NarsilError(
|
|
6286
|
+
ErrorCodes.CONFIG_INVALID,
|
|
6287
|
+
`partitionCount must be an integer between 1 and ${MAX_PARTITION_COUNT}, received ${partitionCount}`,
|
|
6288
|
+
{ partitionCount }
|
|
6289
|
+
);
|
|
6290
|
+
}
|
|
6291
|
+
if (!Number.isInteger(replicationFactor) || replicationFactor < 0 || replicationFactor > MAX_REPLICATION_FACTOR) {
|
|
6292
|
+
throw new NarsilError(
|
|
6293
|
+
ErrorCodes.CONFIG_INVALID,
|
|
6294
|
+
`replicationFactor must be an integer between 0 and ${MAX_REPLICATION_FACTOR}, received ${replicationFactor}`,
|
|
6295
|
+
{ replicationFactor }
|
|
6296
|
+
);
|
|
6297
|
+
}
|
|
6298
|
+
}
|
|
6299
|
+
async function routeCreateIndex(name, config, options, coordinator, engine) {
|
|
6300
|
+
validateIndexName(name);
|
|
6301
|
+
const partitionCount = options?.partitionCount ?? DEFAULT_PARTITION_COUNT;
|
|
6302
|
+
const replicationFactor = options?.replicationFactor ?? DEFAULT_REPLICATION_FACTOR;
|
|
6303
|
+
validateCreateIndexOptions(partitionCount, replicationFactor);
|
|
6304
|
+
const constraints = {
|
|
6305
|
+
zoneAwareness: false,
|
|
6306
|
+
zoneAttribute: "zone",
|
|
6307
|
+
maxShardsPerNode: null
|
|
6308
|
+
};
|
|
6309
|
+
const metadata = {
|
|
6310
|
+
indexName: name,
|
|
6311
|
+
partitionCount,
|
|
6312
|
+
replicationFactor,
|
|
6313
|
+
constraints
|
|
6314
|
+
};
|
|
6315
|
+
const stored = await putIndexMetadata(coordinator, metadata);
|
|
6316
|
+
if (!stored) {
|
|
6317
|
+
throw new NarsilError(
|
|
6318
|
+
ErrorCodes.INDEX_ALREADY_EXISTS,
|
|
6319
|
+
`Index '${name}' already exists in the cluster or a partial creation is pending`,
|
|
6320
|
+
{ indexName: name }
|
|
6321
|
+
);
|
|
6322
|
+
}
|
|
6323
|
+
try {
|
|
6324
|
+
await coordinator.putSchema(name, config.schema);
|
|
6325
|
+
await engine.createIndex(name, config);
|
|
6326
|
+
} catch (createErr) {
|
|
6327
|
+
let cleanupFailed = false;
|
|
6328
|
+
let cleanupError;
|
|
6329
|
+
try {
|
|
6330
|
+
const currentBytes = await coordinator.get(`_narsil/index/${name}/config`);
|
|
6331
|
+
if (currentBytes !== null) {
|
|
6332
|
+
await coordinator.compareAndSet(`_narsil/index/${name}/config`, currentBytes, new Uint8Array(0));
|
|
6333
|
+
}
|
|
6334
|
+
} catch (cleanErr) {
|
|
6335
|
+
cleanupFailed = true;
|
|
6336
|
+
cleanupError = cleanErr;
|
|
6337
|
+
}
|
|
6338
|
+
if (cleanupFailed) {
|
|
6339
|
+
throw new NarsilError(
|
|
6340
|
+
ErrorCodes.CONFIG_INVALID,
|
|
6341
|
+
`Index creation for '${name}' failed and metadata cleanup also failed. The index may be in a partial state and require manual intervention.`,
|
|
6342
|
+
{
|
|
6343
|
+
indexName: name,
|
|
6344
|
+
createError: createErr instanceof Error ? createErr.message : String(createErr),
|
|
6345
|
+
cleanupError: cleanupError instanceof Error ? cleanupError.message : String(cleanupError)
|
|
6346
|
+
}
|
|
6347
|
+
);
|
|
6348
|
+
}
|
|
6349
|
+
throw createErr;
|
|
6278
6350
|
}
|
|
6279
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Forward update operations are not supported yet");
|
|
6280
6351
|
}
|
|
6281
6352
|
async function routeInsert(indexName, document, docId, deps) {
|
|
6282
6353
|
const resolvedDocId = docId ?? generateId();
|
|
@@ -6376,158 +6447,7 @@ async function routeRemoveBatch(indexName, docIds, deps) {
|
|
|
6376
6447
|
return { succeeded, failed };
|
|
6377
6448
|
}
|
|
6378
6449
|
|
|
6379
|
-
// src/distribution/cluster-node/message-handler.ts
|
|
6380
|
-
function createDataNodeHandler(deps) {
|
|
6381
|
-
return async (message, respond) => {
|
|
6382
|
-
if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
|
|
6383
|
-
await handleSnapshotSyncRequest(message, respond, {
|
|
6384
|
-
nodeId: deps.nodeId,
|
|
6385
|
-
engine: deps.engine,
|
|
6386
|
-
coordinator: deps.coordinator,
|
|
6387
|
-
state: deps.snapshotSyncState,
|
|
6388
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata
|
|
6389
|
-
});
|
|
6390
|
-
return;
|
|
6391
|
-
}
|
|
6392
|
-
try {
|
|
6393
|
-
switch (message.type) {
|
|
6394
|
-
case ReplicationMessageTypes.SYNC_REQUEST:
|
|
6395
|
-
await handleSyncRequestMessage(message, respond, deps);
|
|
6396
|
-
return;
|
|
6397
|
-
case ReplicationMessageTypes.FORWARD:
|
|
6398
|
-
await handleForward(message, respond, deps);
|
|
6399
|
-
return;
|
|
6400
|
-
case ReplicationMessageTypes.ENTRY:
|
|
6401
|
-
await handleReplicationEntry(message, respond, deps);
|
|
6402
|
-
return;
|
|
6403
|
-
case QueryMessageTypes.SEARCH:
|
|
6404
|
-
await handleSearch(message, respond, deps);
|
|
6405
|
-
return;
|
|
6406
|
-
case QueryMessageTypes.FETCH:
|
|
6407
|
-
await handleFetch(message, respond, deps);
|
|
6408
|
-
return;
|
|
6409
|
-
case QueryMessageTypes.STATS:
|
|
6410
|
-
await handleStats(message, respond, deps);
|
|
6411
|
-
return;
|
|
6412
|
-
default:
|
|
6413
|
-
return;
|
|
6414
|
-
}
|
|
6415
|
-
} catch (err) {
|
|
6416
|
-
const errorPayload = encode({
|
|
6417
|
-
error: true,
|
|
6418
|
-
code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
|
|
6419
|
-
message: err instanceof Error ? err.message : String(err)
|
|
6420
|
-
});
|
|
6421
|
-
respond({
|
|
6422
|
-
type: `${message.type}.error`,
|
|
6423
|
-
sourceId: deps.nodeId,
|
|
6424
|
-
requestId: message.requestId,
|
|
6425
|
-
payload: errorPayload
|
|
6426
|
-
});
|
|
6427
|
-
}
|
|
6428
|
-
};
|
|
6429
|
-
}
|
|
6430
|
-
async function handleSyncRequestMessage(message, respond, deps) {
|
|
6431
|
-
const request = validateSyncRequest(decode(message.payload));
|
|
6432
|
-
const authResult = await authorizeSnapshotRequest(deps.coordinator, request.indexName, message.sourceId);
|
|
6433
|
-
if (authResult.outcome === "denied") {
|
|
6434
|
-
throw new NarsilError(authResult.code, authResult.reason, {
|
|
6435
|
-
indexName: request.indexName,
|
|
6436
|
-
partitionId: request.partitionId,
|
|
6437
|
-
sourceNodeId: message.sourceId
|
|
6438
|
-
});
|
|
6439
|
-
}
|
|
6440
|
-
const table = await deps.coordinator.getAllocation(request.indexName);
|
|
6441
|
-
const assignment = table?.assignments.get(request.partitionId);
|
|
6442
|
-
if (assignment === void 0) {
|
|
6443
|
-
throw new NarsilError(
|
|
6444
|
-
ErrorCodes.SNAPSHOT_SYNC_NOT_ASSIGNED,
|
|
6445
|
-
`No assignment exists for sync request partition ${request.partitionId} of index '${request.indexName}'`,
|
|
6446
|
-
{ indexName: request.indexName, partitionId: request.partitionId }
|
|
6447
|
-
);
|
|
6448
|
-
}
|
|
6449
|
-
if (assignment.primary !== deps.nodeId) {
|
|
6450
|
-
throw new NarsilError(
|
|
6451
|
-
ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
6452
|
-
`Node '${deps.nodeId}' is not primary for sync request partition ${request.partitionId} of index '${request.indexName}'`,
|
|
6453
|
-
{
|
|
6454
|
-
indexName: request.indexName,
|
|
6455
|
-
partitionId: request.partitionId,
|
|
6456
|
-
localNodeId: deps.nodeId,
|
|
6457
|
-
primaryNodeId: assignment.primary
|
|
6458
|
-
}
|
|
6459
|
-
);
|
|
6460
|
-
}
|
|
6461
|
-
const sourceAssigned = assignment.primary === message.sourceId || assignment.replicas.includes(message.sourceId);
|
|
6462
|
-
if (!sourceAssigned) {
|
|
6463
|
-
throw new NarsilError(
|
|
6464
|
-
ErrorCodes.SNAPSHOT_SYNC_UNAUTHORIZED,
|
|
6465
|
-
`Node '${message.sourceId}' is not assigned to partition ${request.partitionId} of index '${request.indexName}'`,
|
|
6466
|
-
{ indexName: request.indexName, partitionId: request.partitionId, sourceNodeId: message.sourceId }
|
|
6467
|
-
);
|
|
6468
|
-
}
|
|
6469
|
-
if (request.lastPrimaryTerm > assignment.primaryTerm) {
|
|
6470
|
-
throw new NarsilError(
|
|
6471
|
-
ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID,
|
|
6472
|
-
`Replica term ${request.lastPrimaryTerm} is newer than primary term ${assignment.primaryTerm}`,
|
|
6473
|
-
{
|
|
6474
|
-
indexName: request.indexName,
|
|
6475
|
-
partitionId: request.partitionId,
|
|
6476
|
-
lastPrimaryTerm: request.lastPrimaryTerm,
|
|
6477
|
-
primaryTerm: assignment.primaryTerm
|
|
6478
|
-
}
|
|
6479
|
-
);
|
|
6480
|
-
}
|
|
6481
|
-
const log = deps.writeDeps.getReplicationLog(request.indexName, request.partitionId);
|
|
6482
|
-
const tier = decideSyncTier(log, request.lastSeqNo);
|
|
6483
|
-
if (tier === "incremental") {
|
|
6484
|
-
sendSyncEntriesResponse(message, respond, deps, log.getEntriesFrom(request.lastSeqNo + 1));
|
|
6485
|
-
return;
|
|
6486
|
-
}
|
|
6487
|
-
const snapshotSeqNo = log.committedSeqNo;
|
|
6488
|
-
const sink = createSingleResponseSink(respond);
|
|
6489
|
-
await streamValidatedSnapshotRequest(
|
|
6490
|
-
message,
|
|
6491
|
-
sink,
|
|
6492
|
-
{ indexName: request.indexName, partitionId: request.partitionId },
|
|
6493
|
-
{
|
|
6494
|
-
nodeId: deps.nodeId,
|
|
6495
|
-
engine: deps.engine,
|
|
6496
|
-
coordinator: deps.coordinator,
|
|
6497
|
-
state: deps.snapshotSyncState,
|
|
6498
|
-
resolveHeaderMetadata: deps.resolveHeaderMetadata
|
|
6499
|
-
},
|
|
6500
|
-
{
|
|
6501
|
-
metadata: {
|
|
6502
|
-
partitionId: request.partitionId,
|
|
6503
|
-
lastSeqNo: snapshotSeqNo,
|
|
6504
|
-
primaryTerm: assignment.primaryTerm
|
|
6505
|
-
},
|
|
6506
|
-
closeOnEnd: false,
|
|
6507
|
-
disableBuildCache: true,
|
|
6508
|
-
buildSnapshot: async () => {
|
|
6509
|
-
const bytes = await deps.engine.serializeReplicationPartition(request.indexName, request.partitionId);
|
|
6510
|
-
return { bytes, checksum: crc32(bytes) };
|
|
6511
|
-
},
|
|
6512
|
-
afterSnapshot: (trailingSink) => {
|
|
6513
|
-
const entries = log.getEntriesFrom(snapshotSeqNo + 1);
|
|
6514
|
-
sendSyncEntriesResponse(message, trailingSink, deps, entries);
|
|
6515
|
-
}
|
|
6516
|
-
}
|
|
6517
|
-
);
|
|
6518
|
-
}
|
|
6519
|
-
function sendSyncEntriesResponse(message, respond, deps, entries) {
|
|
6520
|
-
const payload = {
|
|
6521
|
-
entries,
|
|
6522
|
-
isLast: true
|
|
6523
|
-
};
|
|
6524
|
-
respond({
|
|
6525
|
-
type: ReplicationMessageTypes.SYNC_ENTRIES,
|
|
6526
|
-
sourceId: deps.nodeId,
|
|
6527
|
-
requestId: message.requestId,
|
|
6528
|
-
payload: encode(payload)
|
|
6529
|
-
});
|
|
6530
|
-
}
|
|
6450
|
+
// src/distribution/cluster-node/message-handler/writes.ts
|
|
6531
6451
|
async function handleForward(message, respond, deps) {
|
|
6532
6452
|
const decoded = decode(message.payload);
|
|
6533
6453
|
const payload = validateForwardPayload(decoded);
|
|
@@ -6572,7 +6492,7 @@ async function handleReplicationEntry(message, respond, deps) {
|
|
|
6572
6492
|
}
|
|
6573
6493
|
if (assignment.primaryTerm !== entry.primaryTerm) {
|
|
6574
6494
|
throw new NarsilError(
|
|
6575
|
-
ErrorCodes.
|
|
6495
|
+
ErrorCodes.REPLICATION_TERM_MISMATCH,
|
|
6576
6496
|
`Replication entry term ${entry.primaryTerm} does not match allocation term ${assignment.primaryTerm}`,
|
|
6577
6497
|
{ indexName: entry.indexName, partitionId: entry.partitionId }
|
|
6578
6498
|
);
|
|
@@ -6591,118 +6511,36 @@ async function handleReplicationEntry(message, respond, deps) {
|
|
|
6591
6511
|
throw new NarsilError(
|
|
6592
6512
|
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6593
6513
|
`Conflicting replication entry at sequence number ${entry.seqNo}`,
|
|
6594
|
-
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
|
|
6595
|
-
);
|
|
6596
|
-
}
|
|
6597
|
-
respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
6598
|
-
return;
|
|
6599
|
-
}
|
|
6600
|
-
const expectedSeqNo = (log.newestSeqNo ?? 0) + 1;
|
|
6601
|
-
if (entry.seqNo !== expectedSeqNo) {
|
|
6602
|
-
const canSeedFromCompletedBootstrap = log.entryCount === 0 && entry.seqNo > expectedSeqNo && deps.isBootstrapSynced?.(entry.indexName, entry.partitionId) === true;
|
|
6603
|
-
if (!canSeedFromCompletedBootstrap) {
|
|
6604
|
-
throw new NarsilError(
|
|
6605
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6606
|
-
`Out-of-order replication entry ${entry.seqNo}; expected ${expectedSeqNo}`,
|
|
6607
|
-
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
6608
|
-
);
|
|
6609
|
-
}
|
|
6610
|
-
deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
|
|
6611
|
-
log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
6612
|
-
}
|
|
6613
|
-
const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
|
|
6614
|
-
if (!validation.valid) {
|
|
6615
|
-
throw new NarsilError(
|
|
6616
|
-
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6617
|
-
`Invalid replication entry ${entry.seqNo}: ${validation.error ?? "unknown validation error"}`,
|
|
6618
|
-
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
|
|
6619
|
-
);
|
|
6620
|
-
}
|
|
6621
|
-
await deps.engine.applyReplicationEntry(entry);
|
|
6622
|
-
log.appendCommitted(entry);
|
|
6623
|
-
respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
6624
|
-
}
|
|
6625
|
-
async function handleSearch(message, respond, deps) {
|
|
6626
|
-
const decoded = decode(message.payload);
|
|
6627
|
-
const payload = validateSearchPayload(decoded);
|
|
6628
|
-
const queryParams = {
|
|
6629
|
-
term: payload.params.term ?? void 0,
|
|
6630
|
-
fields: payload.params.fields ?? void 0,
|
|
6631
|
-
filters: payload.params.filters ?? void 0,
|
|
6632
|
-
boost: payload.params.boost ?? void 0,
|
|
6633
|
-
scoring: payload.params.scoring,
|
|
6634
|
-
tolerance: payload.params.tolerance ?? void 0,
|
|
6635
|
-
minScore: payload.params.threshold ?? void 0,
|
|
6636
|
-
limit: payload.params.limit,
|
|
6637
|
-
offset: payload.params.offset,
|
|
6638
|
-
searchAfter: payload.params.searchAfter ?? void 0,
|
|
6639
|
-
sort: convertWireSortToLocal(payload.params.sort),
|
|
6640
|
-
group: payload.params.group !== null ? { fields: [payload.params.group.field], maxPerGroup: payload.params.group.maxPerGroup } : void 0,
|
|
6641
|
-
facets: convertWireFacetsToLocal(payload.params.facets, payload.facetShardSize ?? payload.params.facetSize),
|
|
6642
|
-
vector: payload.params.vector !== null ? {
|
|
6643
|
-
field: payload.params.vector.field,
|
|
6644
|
-
value: payload.params.vector.value ?? void 0,
|
|
6645
|
-
text: payload.params.vector.text ?? void 0,
|
|
6646
|
-
similarity: payload.params.vector.similarity ?? void 0
|
|
6647
|
-
} : void 0,
|
|
6648
|
-
hybrid: payload.params.hybrid !== null ? { strategy: payload.params.hybrid.strategy, k: payload.params.hybrid.k, alpha: payload.params.hybrid.alpha } : void 0
|
|
6649
|
-
};
|
|
6650
|
-
const queryResult = await deps.engine.query(payload.indexName, queryParams);
|
|
6651
|
-
const scored = queryResult.hits.map((hit) => ({
|
|
6652
|
-
docId: hit.id,
|
|
6653
|
-
score: hit.score,
|
|
6654
|
-
sortValues: null
|
|
6655
|
-
}));
|
|
6656
|
-
const results = payload.partitionIds.map((partitionId) => ({
|
|
6657
|
-
partitionId,
|
|
6658
|
-
scored: partitionId === payload.partitionIds[0] ? scored : [],
|
|
6659
|
-
totalHits: partitionId === payload.partitionIds[0] ? queryResult.count : 0
|
|
6660
|
-
}));
|
|
6661
|
-
const resultPayload = { results, facets: convertLocalFacetsToWire(queryResult.facets) };
|
|
6662
|
-
respond({
|
|
6663
|
-
type: QueryMessageTypes.SEARCH_RESULT,
|
|
6664
|
-
sourceId: deps.nodeId,
|
|
6665
|
-
requestId: message.requestId,
|
|
6666
|
-
payload: encode(resultPayload)
|
|
6667
|
-
});
|
|
6668
|
-
}
|
|
6669
|
-
async function handleFetch(message, respond, deps) {
|
|
6670
|
-
const decoded = decode(message.payload);
|
|
6671
|
-
const payload = validateFetchPayload(decoded);
|
|
6672
|
-
const documents = [];
|
|
6673
|
-
for (const docRef of payload.documentIds) {
|
|
6674
|
-
const doc = await deps.engine.get(payload.indexName, docRef.docId);
|
|
6675
|
-
if (doc !== void 0) {
|
|
6676
|
-
documents.push({
|
|
6677
|
-
docId: docRef.docId,
|
|
6678
|
-
document: doc,
|
|
6679
|
-
highlights: null
|
|
6680
|
-
});
|
|
6514
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
|
|
6515
|
+
);
|
|
6681
6516
|
}
|
|
6517
|
+
respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
6518
|
+
return;
|
|
6682
6519
|
}
|
|
6683
|
-
const
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
}
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
|
|
6694
|
-
|
|
6695
|
-
|
|
6696
|
-
|
|
6697
|
-
|
|
6698
|
-
|
|
6699
|
-
|
|
6700
|
-
|
|
6701
|
-
|
|
6702
|
-
|
|
6703
|
-
|
|
6704
|
-
|
|
6705
|
-
|
|
6520
|
+
const expectedSeqNo = (log.newestSeqNo ?? 0) + 1;
|
|
6521
|
+
if (entry.seqNo !== expectedSeqNo) {
|
|
6522
|
+
const canSeedFromCompletedBootstrap = log.entryCount === 0 && entry.seqNo > expectedSeqNo && deps.isBootstrapSynced?.(entry.indexName, entry.partitionId) === true;
|
|
6523
|
+
if (!canSeedFromCompletedBootstrap) {
|
|
6524
|
+
throw new NarsilError(
|
|
6525
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6526
|
+
`Out-of-order replication entry ${entry.seqNo}; expected ${expectedSeqNo}`,
|
|
6527
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo, expectedSeqNo }
|
|
6528
|
+
);
|
|
6529
|
+
}
|
|
6530
|
+
deps.writeDeps.resetReplicationLog(entry.indexName, entry.partitionId, entry.seqNo, entry.primaryTerm);
|
|
6531
|
+
log = deps.writeDeps.getReplicationLog(entry.indexName, entry.partitionId);
|
|
6532
|
+
}
|
|
6533
|
+
const validation = validateReplicationEntry(entry, assignment.primaryTerm, log);
|
|
6534
|
+
if (!validation.valid) {
|
|
6535
|
+
throw new NarsilError(
|
|
6536
|
+
validation.error ?? ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6537
|
+
`Invalid replication entry ${entry.seqNo}`,
|
|
6538
|
+
{ indexName: entry.indexName, partitionId: entry.partitionId, seqNo: entry.seqNo }
|
|
6539
|
+
);
|
|
6540
|
+
}
|
|
6541
|
+
await deps.engine.applyReplicationEntry(entry);
|
|
6542
|
+
log.appendCommitted(entry);
|
|
6543
|
+
respond(createAckMessage(entry.seqNo, entry.partitionId, entry.indexName, deps.nodeId, message.requestId));
|
|
6706
6544
|
}
|
|
6707
6545
|
function validateForwardPayload(decoded) {
|
|
6708
6546
|
if (decoded === null || typeof decoded !== "object" || Array.isArray(decoded)) {
|
|
@@ -6723,35 +6561,127 @@ function validateForwardPayload(decoded) {
|
|
|
6723
6561
|
}
|
|
6724
6562
|
return decoded;
|
|
6725
6563
|
}
|
|
6726
|
-
|
|
6727
|
-
|
|
6728
|
-
|
|
6729
|
-
|
|
6730
|
-
|
|
6731
|
-
|
|
6732
|
-
|
|
6733
|
-
|
|
6734
|
-
|
|
6564
|
+
|
|
6565
|
+
// src/distribution/cluster-node/message-handler/index.ts
|
|
6566
|
+
function createDataNodeHandler(deps) {
|
|
6567
|
+
return async (message, respond) => {
|
|
6568
|
+
if (message.type === ReplicationMessageTypes.SNAPSHOT_SYNC_REQUEST) {
|
|
6569
|
+
await handleSnapshotSyncRequest(message, respond, {
|
|
6570
|
+
nodeId: deps.nodeId,
|
|
6571
|
+
engine: deps.engine,
|
|
6572
|
+
coordinator: deps.coordinator,
|
|
6573
|
+
state: deps.snapshotSyncState,
|
|
6574
|
+
resolveHeaderMetadata: deps.resolveHeaderMetadata
|
|
6575
|
+
});
|
|
6576
|
+
return;
|
|
6577
|
+
}
|
|
6578
|
+
try {
|
|
6579
|
+
switch (message.type) {
|
|
6580
|
+
case ReplicationMessageTypes.SYNC_REQUEST:
|
|
6581
|
+
await handleSyncRequestMessage(message, respond, deps);
|
|
6582
|
+
return;
|
|
6583
|
+
case ReplicationMessageTypes.FORWARD:
|
|
6584
|
+
await handleForward(message, respond, deps);
|
|
6585
|
+
return;
|
|
6586
|
+
case ReplicationMessageTypes.ENTRY:
|
|
6587
|
+
await handleReplicationEntry(message, respond, deps);
|
|
6588
|
+
return;
|
|
6589
|
+
case QueryMessageTypes.SEARCH:
|
|
6590
|
+
await handleSearch(message, respond, deps);
|
|
6591
|
+
return;
|
|
6592
|
+
case QueryMessageTypes.FETCH:
|
|
6593
|
+
await handleFetch(message, respond, deps);
|
|
6594
|
+
return;
|
|
6595
|
+
case QueryMessageTypes.STATS:
|
|
6596
|
+
await handleStats(message, respond, deps);
|
|
6597
|
+
return;
|
|
6598
|
+
default:
|
|
6599
|
+
return;
|
|
6600
|
+
}
|
|
6601
|
+
} catch (err) {
|
|
6602
|
+
const errorPayload = encode({
|
|
6603
|
+
error: true,
|
|
6604
|
+
code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
|
|
6605
|
+
message: err instanceof Error ? err.message : String(err)
|
|
6606
|
+
});
|
|
6607
|
+
respond({
|
|
6608
|
+
type: `${message.type}.error`,
|
|
6609
|
+
sourceId: deps.nodeId,
|
|
6610
|
+
requestId: message.requestId,
|
|
6611
|
+
payload: errorPayload
|
|
6612
|
+
});
|
|
6613
|
+
}
|
|
6614
|
+
};
|
|
6735
6615
|
}
|
|
6736
|
-
function
|
|
6737
|
-
|
|
6738
|
-
|
|
6616
|
+
async function resolveNodeTargets2(config, targetNodeId) {
|
|
6617
|
+
const targets = [targetNodeId];
|
|
6618
|
+
const nodes = await config.coordinator.listNodes();
|
|
6619
|
+
const registration = nodes.find((node) => node.nodeId === targetNodeId);
|
|
6620
|
+
if (registration !== void 0 && registration.address.length > 0 && registration.address !== targetNodeId) {
|
|
6621
|
+
targets.push(registration.address);
|
|
6739
6622
|
}
|
|
6740
|
-
|
|
6741
|
-
|
|
6742
|
-
|
|
6623
|
+
return targets;
|
|
6624
|
+
}
|
|
6625
|
+
async function sendToNode2(config, targetNodeId, message) {
|
|
6626
|
+
const targets = await resolveNodeTargets2(config, targetNodeId);
|
|
6627
|
+
let lastError;
|
|
6628
|
+
for (const target of targets) {
|
|
6629
|
+
try {
|
|
6630
|
+
return await config.transport.send(target, message);
|
|
6631
|
+
} catch (error) {
|
|
6632
|
+
lastError = error;
|
|
6633
|
+
}
|
|
6743
6634
|
}
|
|
6744
|
-
|
|
6635
|
+
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
6745
6636
|
}
|
|
6746
|
-
function
|
|
6747
|
-
|
|
6748
|
-
|
|
6637
|
+
async function fetchDistributedDocuments(config, nodeId, engine, indexName, result, allocation) {
|
|
6638
|
+
const partitionCount = allocation.assignments.size;
|
|
6639
|
+
const nodeToDocumentIds = /* @__PURE__ */ new Map();
|
|
6640
|
+
for (const entry of result.scored) {
|
|
6641
|
+
const partitionId = resolvePartitionId(entry.docId, partitionCount);
|
|
6642
|
+
const assignment = allocation.assignments.get(partitionId);
|
|
6643
|
+
if (assignment === void 0) {
|
|
6644
|
+
continue;
|
|
6645
|
+
}
|
|
6646
|
+
const selectedNodeId = selectReplica(assignment, nodeId, void 0, partitionId);
|
|
6647
|
+
if (selectedNodeId === null) {
|
|
6648
|
+
continue;
|
|
6649
|
+
}
|
|
6650
|
+
let documentIds = nodeToDocumentIds.get(selectedNodeId);
|
|
6651
|
+
if (documentIds === void 0) {
|
|
6652
|
+
documentIds = [];
|
|
6653
|
+
nodeToDocumentIds.set(selectedNodeId, documentIds);
|
|
6654
|
+
}
|
|
6655
|
+
documentIds.push({ docId: entry.docId, partitionId });
|
|
6749
6656
|
}
|
|
6750
|
-
const
|
|
6751
|
-
for (const [
|
|
6752
|
-
|
|
6657
|
+
const documents = /* @__PURE__ */ new Map();
|
|
6658
|
+
for (const [targetNodeId, documentIds] of nodeToDocumentIds) {
|
|
6659
|
+
if (targetNodeId === nodeId) {
|
|
6660
|
+
for (const { docId } of documentIds) {
|
|
6661
|
+
const document = await engine.get(indexName, docId);
|
|
6662
|
+
if (document !== void 0) {
|
|
6663
|
+
documents.set(docId, document);
|
|
6664
|
+
}
|
|
6665
|
+
}
|
|
6666
|
+
continue;
|
|
6667
|
+
}
|
|
6668
|
+
const fetchMessage = createFetchMessage(
|
|
6669
|
+
{
|
|
6670
|
+
indexName,
|
|
6671
|
+
documentIds,
|
|
6672
|
+
fields: null,
|
|
6673
|
+
highlight: null
|
|
6674
|
+
},
|
|
6675
|
+
nodeId
|
|
6676
|
+
);
|
|
6677
|
+
const response = await sendToNode2(config, targetNodeId, fetchMessage);
|
|
6678
|
+
const decoded = decode(response.payload);
|
|
6679
|
+
const payload = validateFetchResultPayload(decoded);
|
|
6680
|
+
for (const fetched of payload.documents) {
|
|
6681
|
+
documents.set(fetched.docId, fetched.document);
|
|
6682
|
+
}
|
|
6753
6683
|
}
|
|
6754
|
-
return
|
|
6684
|
+
return documents;
|
|
6755
6685
|
}
|
|
6756
6686
|
|
|
6757
6687
|
// src/distribution/cluster-node/query-conversion.ts
|
|
@@ -6798,50 +6728,275 @@ function convertLocalGroupToWire(group) {
|
|
|
6798
6728
|
if (group === void 0) {
|
|
6799
6729
|
return null;
|
|
6800
6730
|
}
|
|
6801
|
-
return {
|
|
6802
|
-
field: group.fields[0],
|
|
6803
|
-
maxPerGroup: group.maxPerGroup ?? 1
|
|
6804
|
-
};
|
|
6805
|
-
}
|
|
6806
|
-
function convertLocalFacetsToWire2(facets) {
|
|
6807
|
-
if (facets === void 0) {
|
|
6808
|
-
return null;
|
|
6731
|
+
return {
|
|
6732
|
+
field: group.fields[0],
|
|
6733
|
+
maxPerGroup: group.maxPerGroup ?? 1
|
|
6734
|
+
};
|
|
6735
|
+
}
|
|
6736
|
+
function convertLocalFacetsToWire2(facets) {
|
|
6737
|
+
if (facets === void 0) {
|
|
6738
|
+
return null;
|
|
6739
|
+
}
|
|
6740
|
+
return Object.keys(facets);
|
|
6741
|
+
}
|
|
6742
|
+
function convertLocalVectorToWire(vector) {
|
|
6743
|
+
if (vector === void 0) {
|
|
6744
|
+
return null;
|
|
6745
|
+
}
|
|
6746
|
+
return {
|
|
6747
|
+
field: vector.field,
|
|
6748
|
+
value: vector.value ?? null,
|
|
6749
|
+
text: vector.text ?? null,
|
|
6750
|
+
similarity: vector.similarity ?? null
|
|
6751
|
+
};
|
|
6752
|
+
}
|
|
6753
|
+
function convertLocalHybridToWire(hybrid) {
|
|
6754
|
+
if (hybrid === void 0) {
|
|
6755
|
+
return null;
|
|
6756
|
+
}
|
|
6757
|
+
return {
|
|
6758
|
+
strategy: hybrid.strategy ?? "rrf",
|
|
6759
|
+
k: hybrid.k ?? 60,
|
|
6760
|
+
alpha: hybrid.alpha ?? 0.5
|
|
6761
|
+
};
|
|
6762
|
+
}
|
|
6763
|
+
function convertWireFacetsToLocal2(wireFacets) {
|
|
6764
|
+
const result = {};
|
|
6765
|
+
for (const [field, buckets] of Object.entries(wireFacets)) {
|
|
6766
|
+
const values = {};
|
|
6767
|
+
let totalCount = 0;
|
|
6768
|
+
for (const bucket of buckets) {
|
|
6769
|
+
values[bucket.value] = bucket.count;
|
|
6770
|
+
totalCount += bucket.count;
|
|
6771
|
+
}
|
|
6772
|
+
result[field] = { values, count: totalCount };
|
|
6773
|
+
}
|
|
6774
|
+
return result;
|
|
6775
|
+
}
|
|
6776
|
+
|
|
6777
|
+
// src/distribution/replication/types.ts
|
|
6778
|
+
var DEFAULT_LOG_RETENTION_BYTES = 268435456;
|
|
6779
|
+
|
|
6780
|
+
// src/distribution/replication/log.ts
|
|
6781
|
+
var ENTRY_FIXED_OVERHEAD_BYTES = 40;
|
|
6782
|
+
function estimateEntrySize(entry) {
|
|
6783
|
+
return ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
6784
|
+
}
|
|
6785
|
+
var computeChecksum = computeEntryChecksum;
|
|
6786
|
+
function createReplicationLog(partitionId, config) {
|
|
6787
|
+
const retentionBytes = config?.logRetentionBytes ?? DEFAULT_LOG_RETENTION_BYTES;
|
|
6788
|
+
let nextSeqNo = config?.startSeqNo ?? 1;
|
|
6789
|
+
let committedSeqNo = nextSeqNo - 1;
|
|
6790
|
+
let committedPrimaryTerm = config?.lastPrimaryTerm ?? 0;
|
|
6791
|
+
let entries = [];
|
|
6792
|
+
let headIndex = 0;
|
|
6793
|
+
let totalSizeBytes = 0;
|
|
6794
|
+
function liveCount() {
|
|
6795
|
+
return entries.length - headIndex;
|
|
6796
|
+
}
|
|
6797
|
+
function compact() {
|
|
6798
|
+
if (headIndex > 0) {
|
|
6799
|
+
entries = entries.slice(headIndex);
|
|
6800
|
+
headIndex = 0;
|
|
6801
|
+
}
|
|
6802
|
+
}
|
|
6803
|
+
function findIndexBySeqNo(seqNo) {
|
|
6804
|
+
let low = headIndex;
|
|
6805
|
+
let high = entries.length - 1;
|
|
6806
|
+
while (low <= high) {
|
|
6807
|
+
const mid = low + high >>> 1;
|
|
6808
|
+
const midSeqNo = entries[mid].seqNo;
|
|
6809
|
+
if (midSeqNo < seqNo) {
|
|
6810
|
+
low = mid + 1;
|
|
6811
|
+
} else if (midSeqNo > seqNo) {
|
|
6812
|
+
high = mid - 1;
|
|
6813
|
+
} else {
|
|
6814
|
+
return mid;
|
|
6815
|
+
}
|
|
6816
|
+
}
|
|
6817
|
+
return low;
|
|
6818
|
+
}
|
|
6819
|
+
function evictOldEntries() {
|
|
6820
|
+
while (liveCount() > 0 && totalSizeBytes > retentionBytes) {
|
|
6821
|
+
totalSizeBytes -= estimateEntrySize(entries[headIndex]);
|
|
6822
|
+
entries[headIndex] = null;
|
|
6823
|
+
headIndex += 1;
|
|
6824
|
+
}
|
|
6825
|
+
if (totalSizeBytes < 0) {
|
|
6826
|
+
totalSizeBytes = 0;
|
|
6827
|
+
}
|
|
6828
|
+
if (headIndex >= 256 || headIndex > liveCount()) {
|
|
6829
|
+
compact();
|
|
6830
|
+
}
|
|
6831
|
+
}
|
|
6832
|
+
function assertEntryFitsRetention(entry) {
|
|
6833
|
+
const entrySize = ENTRY_FIXED_OVERHEAD_BYTES + entry.indexName.length + entry.documentId.length + (entry.document?.byteLength ?? 0);
|
|
6834
|
+
if (entrySize > retentionBytes) {
|
|
6835
|
+
throw new NarsilError(
|
|
6836
|
+
ErrorCodes.REPLICATION_LOG_FULL,
|
|
6837
|
+
`Replication log entry of ${entrySize} bytes exceeds the log retention limit of ${retentionBytes} bytes and cannot be buffered`,
|
|
6838
|
+
{ partitionId, entrySize, retentionBytes, seqNo: entry.seqNo }
|
|
6839
|
+
);
|
|
6840
|
+
}
|
|
6841
|
+
}
|
|
6842
|
+
function storeEntry(entry) {
|
|
6843
|
+
entries.push(entry);
|
|
6844
|
+
if (entry.seqNo >= committedSeqNo) {
|
|
6845
|
+
committedSeqNo = entry.seqNo;
|
|
6846
|
+
committedPrimaryTerm = entry.primaryTerm;
|
|
6847
|
+
}
|
|
6848
|
+
totalSizeBytes += estimateEntrySize(entry);
|
|
6849
|
+
evictOldEntries();
|
|
6850
|
+
return { ...entry };
|
|
6851
|
+
}
|
|
6852
|
+
function readEntry(seqNo) {
|
|
6853
|
+
if (liveCount() === 0) return void 0;
|
|
6854
|
+
const idx = findIndexBySeqNo(seqNo);
|
|
6855
|
+
if (idx < entries.length && entries[idx].seqNo === seqNo) {
|
|
6856
|
+
return { ...entries[idx] };
|
|
6857
|
+
}
|
|
6858
|
+
return void 0;
|
|
6809
6859
|
}
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
|
|
6813
|
-
|
|
6814
|
-
|
|
6860
|
+
function verifyEntryChecksum(entry) {
|
|
6861
|
+
const expected = computeChecksum({
|
|
6862
|
+
seqNo: entry.seqNo,
|
|
6863
|
+
primaryTerm: entry.primaryTerm,
|
|
6864
|
+
operation: entry.operation,
|
|
6865
|
+
partitionId: entry.partitionId,
|
|
6866
|
+
indexName: entry.indexName,
|
|
6867
|
+
documentId: entry.documentId,
|
|
6868
|
+
document: entry.document
|
|
6869
|
+
});
|
|
6870
|
+
return expected === entry.checksum;
|
|
6815
6871
|
}
|
|
6816
6872
|
return {
|
|
6817
|
-
|
|
6818
|
-
|
|
6819
|
-
|
|
6820
|
-
|
|
6873
|
+
append(partial) {
|
|
6874
|
+
assertEntryFitsRetention(partial);
|
|
6875
|
+
const seqNo = nextSeqNo;
|
|
6876
|
+
nextSeqNo += 1;
|
|
6877
|
+
const entryWithoutChecksum = {
|
|
6878
|
+
seqNo,
|
|
6879
|
+
primaryTerm: partial.primaryTerm,
|
|
6880
|
+
operation: partial.operation,
|
|
6881
|
+
partitionId,
|
|
6882
|
+
indexName: partial.indexName,
|
|
6883
|
+
documentId: partial.documentId,
|
|
6884
|
+
document: partial.document
|
|
6885
|
+
};
|
|
6886
|
+
const checksum = computeChecksum(entryWithoutChecksum);
|
|
6887
|
+
const entry = { ...entryWithoutChecksum, checksum };
|
|
6888
|
+
return storeEntry(entry);
|
|
6889
|
+
},
|
|
6890
|
+
appendCommitted(entry) {
|
|
6891
|
+
if (!Number.isSafeInteger(entry.seqNo) || entry.seqNo < 1) {
|
|
6892
|
+
throw new NarsilError(
|
|
6893
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6894
|
+
`Invalid replication sequence number ${entry.seqNo}`,
|
|
6895
|
+
{
|
|
6896
|
+
seqNo: entry.seqNo,
|
|
6897
|
+
partitionId
|
|
6898
|
+
}
|
|
6899
|
+
);
|
|
6900
|
+
}
|
|
6901
|
+
if (entry.partitionId !== partitionId) {
|
|
6902
|
+
throw new NarsilError(
|
|
6903
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6904
|
+
`Replication entry partition ${entry.partitionId} does not match log partition ${partitionId}`,
|
|
6905
|
+
{ entryPartitionId: entry.partitionId, logPartitionId: partitionId }
|
|
6906
|
+
);
|
|
6907
|
+
}
|
|
6908
|
+
const existing = readEntry(entry.seqNo);
|
|
6909
|
+
if (existing !== void 0) {
|
|
6910
|
+
if (existing.checksum !== entry.checksum) {
|
|
6911
|
+
throw new NarsilError(
|
|
6912
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6913
|
+
`Conflicting replication entry at sequence number ${entry.seqNo}`,
|
|
6914
|
+
{ seqNo: entry.seqNo, partitionId }
|
|
6915
|
+
);
|
|
6916
|
+
}
|
|
6917
|
+
return existing;
|
|
6918
|
+
}
|
|
6919
|
+
if (entry.seqNo !== nextSeqNo) {
|
|
6920
|
+
throw new NarsilError(
|
|
6921
|
+
ErrorCodes.REPLICATION_ENTRY_INVALID,
|
|
6922
|
+
`Out-of-order replication entry ${entry.seqNo}; expected ${nextSeqNo}`,
|
|
6923
|
+
{ seqNo: entry.seqNo, expectedSeqNo: nextSeqNo, partitionId }
|
|
6924
|
+
);
|
|
6925
|
+
}
|
|
6926
|
+
if (!verifyEntryChecksum(entry)) {
|
|
6927
|
+
throw new NarsilError(
|
|
6928
|
+
ErrorCodes.REPLICATION_ENTRY_CORRUPT,
|
|
6929
|
+
`Invalid checksum for replication entry ${entry.seqNo}`,
|
|
6930
|
+
{ seqNo: entry.seqNo, partitionId }
|
|
6931
|
+
);
|
|
6932
|
+
}
|
|
6933
|
+
assertEntryFitsRetention(entry);
|
|
6934
|
+
nextSeqNo = entry.seqNo + 1;
|
|
6935
|
+
return storeEntry({ ...entry });
|
|
6936
|
+
},
|
|
6937
|
+
getEntriesFrom(fromSeqNo) {
|
|
6938
|
+
if (liveCount() === 0) return [];
|
|
6939
|
+
const idx = findIndexBySeqNo(fromSeqNo);
|
|
6940
|
+
if (idx >= entries.length) return [];
|
|
6941
|
+
return entries.slice(idx).map((e) => ({ ...e }));
|
|
6942
|
+
},
|
|
6943
|
+
getEntry(seqNo) {
|
|
6944
|
+
return readEntry(seqNo);
|
|
6945
|
+
},
|
|
6946
|
+
verifyChecksum(entry) {
|
|
6947
|
+
return verifyEntryChecksum(entry);
|
|
6948
|
+
},
|
|
6949
|
+
get committedSeqNo() {
|
|
6950
|
+
return committedSeqNo;
|
|
6951
|
+
},
|
|
6952
|
+
get committedPrimaryTerm() {
|
|
6953
|
+
return committedPrimaryTerm;
|
|
6954
|
+
},
|
|
6955
|
+
get oldestSeqNo() {
|
|
6956
|
+
if (liveCount() === 0) return void 0;
|
|
6957
|
+
return entries[headIndex].seqNo;
|
|
6958
|
+
},
|
|
6959
|
+
get newestSeqNo() {
|
|
6960
|
+
if (liveCount() === 0) return void 0;
|
|
6961
|
+
return entries[entries.length - 1].seqNo;
|
|
6962
|
+
},
|
|
6963
|
+
get entryCount() {
|
|
6964
|
+
return liveCount();
|
|
6965
|
+
},
|
|
6966
|
+
get sizeBytes() {
|
|
6967
|
+
return totalSizeBytes;
|
|
6968
|
+
},
|
|
6969
|
+
clear() {
|
|
6970
|
+
entries = [];
|
|
6971
|
+
headIndex = 0;
|
|
6972
|
+
totalSizeBytes = 0;
|
|
6973
|
+
committedSeqNo = nextSeqNo - 1;
|
|
6974
|
+
}
|
|
6821
6975
|
};
|
|
6822
6976
|
}
|
|
6823
|
-
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
}
|
|
6827
|
-
return {
|
|
6828
|
-
strategy: hybrid.strategy ?? "rrf",
|
|
6829
|
-
k: hybrid.k ?? 60,
|
|
6830
|
-
alpha: hybrid.alpha ?? 0.5
|
|
6831
|
-
};
|
|
6977
|
+
|
|
6978
|
+
// src/distribution/cluster-node/replication-logs.ts
|
|
6979
|
+
function replicationLogKey(indexName, partitionId) {
|
|
6980
|
+
return `${indexName}:${partitionId}`;
|
|
6832
6981
|
}
|
|
6833
|
-
function
|
|
6834
|
-
const
|
|
6835
|
-
|
|
6836
|
-
|
|
6837
|
-
|
|
6838
|
-
|
|
6839
|
-
|
|
6840
|
-
|
|
6841
|
-
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6982
|
+
function getReplicationLog(replicationLogs, indexName, partitionId, replicationConfig) {
|
|
6983
|
+
const key = replicationLogKey(indexName, partitionId);
|
|
6984
|
+
let log = replicationLogs.get(key);
|
|
6985
|
+
if (log === void 0) {
|
|
6986
|
+
log = createReplicationLog(partitionId, { logRetentionBytes: replicationConfig?.logRetentionBytes });
|
|
6987
|
+
replicationLogs.set(key, log);
|
|
6988
|
+
}
|
|
6989
|
+
return log;
|
|
6990
|
+
}
|
|
6991
|
+
function seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm = 0, replicationConfig) {
|
|
6992
|
+
replicationLogs.set(
|
|
6993
|
+
replicationLogKey(indexName, partitionId),
|
|
6994
|
+
createReplicationLog(partitionId, {
|
|
6995
|
+
startSeqNo,
|
|
6996
|
+
lastPrimaryTerm,
|
|
6997
|
+
logRetentionBytes: replicationConfig?.logRetentionBytes
|
|
6998
|
+
})
|
|
6999
|
+
);
|
|
6845
7000
|
}
|
|
6846
7001
|
|
|
6847
7002
|
// src/distribution/cluster-node/transport-listener.ts
|
|
@@ -6897,6 +7052,53 @@ function createSingleResponseRespond(respond) {
|
|
|
6897
7052
|
};
|
|
6898
7053
|
}
|
|
6899
7054
|
|
|
7055
|
+
// src/distribution/cluster-node/validate-config.ts
|
|
7056
|
+
var MAX_WAIT_FOR_ACTIVE_REPLICAS = 255;
|
|
7057
|
+
function validateClusterNodeConfig(config) {
|
|
7058
|
+
if (config.address.length === 0) {
|
|
7059
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
|
|
7060
|
+
}
|
|
7061
|
+
if (config.roles !== void 0) {
|
|
7062
|
+
if (config.roles.length === 0) {
|
|
7063
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.roles must contain at least one role");
|
|
7064
|
+
}
|
|
7065
|
+
const validRoles = /* @__PURE__ */ new Set(["data", "coordinator", "controller"]);
|
|
7066
|
+
for (const role of config.roles) {
|
|
7067
|
+
if (!validRoles.has(role)) {
|
|
7068
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid role: '${role}'`, { role });
|
|
7069
|
+
}
|
|
7070
|
+
}
|
|
7071
|
+
}
|
|
7072
|
+
if (config.capacity !== void 0) {
|
|
7073
|
+
if (config.capacity.memoryBytes <= 0 || !Number.isFinite(config.capacity.memoryBytes)) {
|
|
7074
|
+
throw new NarsilError(
|
|
7075
|
+
ErrorCodes.CONFIG_INVALID,
|
|
7076
|
+
"ClusterNodeConfig.capacity.memoryBytes must be a positive finite number"
|
|
7077
|
+
);
|
|
7078
|
+
}
|
|
7079
|
+
if (config.capacity.cpuCores <= 0 || !Number.isInteger(config.capacity.cpuCores)) {
|
|
7080
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.capacity.cpuCores must be a positive integer");
|
|
7081
|
+
}
|
|
7082
|
+
}
|
|
7083
|
+
if (config.replication !== void 0) {
|
|
7084
|
+
const { logRetentionBytes, waitForActiveReplicas } = config.replication;
|
|
7085
|
+
if (logRetentionBytes !== void 0 && (!Number.isInteger(logRetentionBytes) || logRetentionBytes <= 0)) {
|
|
7086
|
+
throw new NarsilError(
|
|
7087
|
+
ErrorCodes.CONFIG_INVALID,
|
|
7088
|
+
"ClusterNodeConfig.replication.logRetentionBytes must be a positive integer",
|
|
7089
|
+
{ logRetentionBytes }
|
|
7090
|
+
);
|
|
7091
|
+
}
|
|
7092
|
+
if (waitForActiveReplicas !== void 0 && (!Number.isInteger(waitForActiveReplicas) || waitForActiveReplicas < 1 || waitForActiveReplicas > MAX_WAIT_FOR_ACTIVE_REPLICAS)) {
|
|
7093
|
+
throw new NarsilError(
|
|
7094
|
+
ErrorCodes.CONFIG_INVALID,
|
|
7095
|
+
`ClusterNodeConfig.replication.waitForActiveReplicas must be an integer between 1 and ${MAX_WAIT_FOR_ACTIVE_REPLICAS}`,
|
|
7096
|
+
{ waitForActiveReplicas }
|
|
7097
|
+
);
|
|
7098
|
+
}
|
|
7099
|
+
}
|
|
7100
|
+
}
|
|
7101
|
+
|
|
6900
7102
|
// src/distribution/cluster-node/index.ts
|
|
6901
7103
|
var SPEC_VERSION = "1.0";
|
|
6902
7104
|
async function createClusterNode(config) {
|
|
@@ -6908,6 +7110,21 @@ async function createClusterNode(config) {
|
|
|
6908
7110
|
const bootstrapSyncState = createBootstrapSyncState();
|
|
6909
7111
|
const snapshotSyncHandlerState = createSnapshotSyncHandlerState();
|
|
6910
7112
|
const replicationLogs = /* @__PURE__ */ new Map();
|
|
7113
|
+
function replicationLogKey2(indexName, partitionId) {
|
|
7114
|
+
return replicationLogKey(indexName, partitionId);
|
|
7115
|
+
}
|
|
7116
|
+
function getReplicationLog2(indexName, partitionId) {
|
|
7117
|
+
return getReplicationLog(replicationLogs, indexName, partitionId, config.replication);
|
|
7118
|
+
}
|
|
7119
|
+
function seedReplicationLog2(indexName, partitionId, startSeqNo, lastPrimaryTerm = 0) {
|
|
7120
|
+
seedReplicationLog(replicationLogs, indexName, partitionId, startSeqNo, lastPrimaryTerm, config.replication);
|
|
7121
|
+
}
|
|
7122
|
+
async function resolveNodeTargets3(targetNodeId) {
|
|
7123
|
+
return resolveNodeTargets2(config, targetNodeId);
|
|
7124
|
+
}
|
|
7125
|
+
async function fetchDistributedDocuments2(indexName, result, allocation) {
|
|
7126
|
+
return fetchDistributedDocuments(config, nodeId, engine, indexName, result, allocation);
|
|
7127
|
+
}
|
|
6911
7128
|
const forwardOnError = (error) => {
|
|
6912
7129
|
if (config.onError === void 0) {
|
|
6913
7130
|
return;
|
|
@@ -6931,9 +7148,10 @@ async function createClusterNode(config) {
|
|
|
6931
7148
|
coordinator: config.coordinator,
|
|
6932
7149
|
engine,
|
|
6933
7150
|
transport: config.transport,
|
|
6934
|
-
getReplicationLog,
|
|
6935
|
-
resetReplicationLog:
|
|
6936
|
-
resolveNodeTargets:
|
|
7151
|
+
getReplicationLog: getReplicationLog2,
|
|
7152
|
+
resetReplicationLog: seedReplicationLog2,
|
|
7153
|
+
resolveNodeTargets: resolveNodeTargets3,
|
|
7154
|
+
waitForActiveReplicas: config.replication?.waitForActiveReplicas
|
|
6937
7155
|
};
|
|
6938
7156
|
let lifecycle = null;
|
|
6939
7157
|
let controller = null;
|
|
@@ -6956,16 +7174,16 @@ async function createClusterNode(config) {
|
|
|
6956
7174
|
coordinator: config.coordinator,
|
|
6957
7175
|
transport: config.transport,
|
|
6958
7176
|
sourceNodeId: nodeId,
|
|
6959
|
-
resolveNodeTargets:
|
|
6960
|
-
getReplicationLog,
|
|
6961
|
-
resetReplicationLog:
|
|
7177
|
+
resolveNodeTargets: resolveNodeTargets3,
|
|
7178
|
+
getReplicationLog: getReplicationLog2,
|
|
7179
|
+
resetReplicationLog: seedReplicationLog2,
|
|
6962
7180
|
applyReplicationEntry: engine.applyReplicationEntry,
|
|
6963
7181
|
restoreReplicationPartition: engine.restoreReplicationPartition,
|
|
6964
7182
|
onError: forwardOnError
|
|
6965
7183
|
}),
|
|
6966
7184
|
onRemovePartition: (indexName, partitionId) => {
|
|
6967
7185
|
clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
|
|
6968
|
-
replicationLogs.delete(
|
|
7186
|
+
replicationLogs.delete(replicationLogKey2(indexName, partitionId));
|
|
6969
7187
|
void cleanupRemovedPartition(indexName, partitionId, {
|
|
6970
7188
|
engine,
|
|
6971
7189
|
coordinator: config.coordinator,
|
|
@@ -7061,10 +7279,10 @@ async function createClusterNode(config) {
|
|
|
7061
7279
|
transport: config.transport,
|
|
7062
7280
|
sourceNodeId: nodeId,
|
|
7063
7281
|
getAllocation: (idx) => config.coordinator.getAllocation(idx),
|
|
7064
|
-
resolveNodeTargets:
|
|
7282
|
+
resolveNodeTargets: resolveNodeTargets3
|
|
7065
7283
|
};
|
|
7066
7284
|
const distributed = await distributedQuery(indexName, wireParams, queryDeps);
|
|
7067
|
-
const documents = await
|
|
7285
|
+
const documents = await fetchDistributedDocuments2(indexName, distributed, allocation);
|
|
7068
7286
|
return distributedResultToLocal(distributed, documents);
|
|
7069
7287
|
});
|
|
7070
7288
|
},
|
|
@@ -7115,24 +7333,6 @@ async function createClusterNode(config) {
|
|
|
7115
7333
|
}
|
|
7116
7334
|
};
|
|
7117
7335
|
return node;
|
|
7118
|
-
function replicationLogKey(indexName, partitionId) {
|
|
7119
|
-
return `${indexName}:${partitionId}`;
|
|
7120
|
-
}
|
|
7121
|
-
function getReplicationLog(indexName, partitionId) {
|
|
7122
|
-
const key = replicationLogKey(indexName, partitionId);
|
|
7123
|
-
let log = replicationLogs.get(key);
|
|
7124
|
-
if (log === void 0) {
|
|
7125
|
-
log = createReplicationLog(partitionId);
|
|
7126
|
-
replicationLogs.set(key, log);
|
|
7127
|
-
}
|
|
7128
|
-
return log;
|
|
7129
|
-
}
|
|
7130
|
-
function seedReplicationLog(indexName, partitionId, startSeqNo, lastPrimaryTerm = 0) {
|
|
7131
|
-
replicationLogs.set(
|
|
7132
|
-
replicationLogKey(indexName, partitionId),
|
|
7133
|
-
createReplicationLog(partitionId, { startSeqNo, lastPrimaryTerm })
|
|
7134
|
-
);
|
|
7135
|
-
}
|
|
7136
7336
|
async function resolveSnapshotHeaderMetadata(indexName, partitionId) {
|
|
7137
7337
|
const fallback = await defaultSnapshotHeaderMetadataProvider(config.coordinator, indexName);
|
|
7138
7338
|
if (partitionId === null) {
|
|
@@ -7140,110 +7340,13 @@ async function createClusterNode(config) {
|
|
|
7140
7340
|
}
|
|
7141
7341
|
const allocation = await config.coordinator.getAllocation(indexName);
|
|
7142
7342
|
const assignment = allocation?.assignments.get(partitionId);
|
|
7143
|
-
const log =
|
|
7343
|
+
const log = getReplicationLog2(indexName, partitionId);
|
|
7144
7344
|
return {
|
|
7145
7345
|
partitionId,
|
|
7146
7346
|
primaryTerm: assignment?.primaryTerm ?? fallback.primaryTerm,
|
|
7147
7347
|
lastSeqNo: log.newestSeqNo ?? 0
|
|
7148
7348
|
};
|
|
7149
7349
|
}
|
|
7150
|
-
async function resolveNodeTargets2(targetNodeId) {
|
|
7151
|
-
const targets = [targetNodeId];
|
|
7152
|
-
const nodes = await config.coordinator.listNodes();
|
|
7153
|
-
const registration2 = nodes.find((node2) => node2.nodeId === targetNodeId);
|
|
7154
|
-
if (registration2 !== void 0 && registration2.address.length > 0 && registration2.address !== targetNodeId) {
|
|
7155
|
-
targets.push(registration2.address);
|
|
7156
|
-
}
|
|
7157
|
-
return targets;
|
|
7158
|
-
}
|
|
7159
|
-
async function fetchDistributedDocuments(indexName, result, allocation) {
|
|
7160
|
-
const partitionCount = allocation.assignments.size;
|
|
7161
|
-
const nodeToDocumentIds = /* @__PURE__ */ new Map();
|
|
7162
|
-
for (const entry of result.scored) {
|
|
7163
|
-
const partitionId = resolvePartitionId(entry.docId, partitionCount);
|
|
7164
|
-
const assignment = allocation.assignments.get(partitionId);
|
|
7165
|
-
if (assignment === void 0) {
|
|
7166
|
-
continue;
|
|
7167
|
-
}
|
|
7168
|
-
const selectedNodeId = selectReplica(assignment, nodeId, void 0, partitionId);
|
|
7169
|
-
if (selectedNodeId === null) {
|
|
7170
|
-
continue;
|
|
7171
|
-
}
|
|
7172
|
-
let documentIds = nodeToDocumentIds.get(selectedNodeId);
|
|
7173
|
-
if (documentIds === void 0) {
|
|
7174
|
-
documentIds = [];
|
|
7175
|
-
nodeToDocumentIds.set(selectedNodeId, documentIds);
|
|
7176
|
-
}
|
|
7177
|
-
documentIds.push({ docId: entry.docId, partitionId });
|
|
7178
|
-
}
|
|
7179
|
-
const documents = /* @__PURE__ */ new Map();
|
|
7180
|
-
for (const [targetNodeId, documentIds] of nodeToDocumentIds) {
|
|
7181
|
-
if (targetNodeId === nodeId) {
|
|
7182
|
-
for (const { docId } of documentIds) {
|
|
7183
|
-
const document = await engine.get(indexName, docId);
|
|
7184
|
-
if (document !== void 0) {
|
|
7185
|
-
documents.set(docId, document);
|
|
7186
|
-
}
|
|
7187
|
-
}
|
|
7188
|
-
continue;
|
|
7189
|
-
}
|
|
7190
|
-
const fetchMessage = createFetchMessage(
|
|
7191
|
-
{
|
|
7192
|
-
indexName,
|
|
7193
|
-
documentIds,
|
|
7194
|
-
fields: null,
|
|
7195
|
-
highlight: null
|
|
7196
|
-
},
|
|
7197
|
-
nodeId
|
|
7198
|
-
);
|
|
7199
|
-
const response = await sendToNode2(targetNodeId, fetchMessage);
|
|
7200
|
-
const decoded = decode(response.payload);
|
|
7201
|
-
const payload = validateFetchResultPayload(decoded);
|
|
7202
|
-
for (const fetched of payload.documents) {
|
|
7203
|
-
documents.set(fetched.docId, fetched.document);
|
|
7204
|
-
}
|
|
7205
|
-
}
|
|
7206
|
-
return documents;
|
|
7207
|
-
}
|
|
7208
|
-
async function sendToNode2(targetNodeId, message) {
|
|
7209
|
-
const targets = await resolveNodeTargets2(targetNodeId);
|
|
7210
|
-
let lastError;
|
|
7211
|
-
for (const target of targets) {
|
|
7212
|
-
try {
|
|
7213
|
-
return await config.transport.send(target, message);
|
|
7214
|
-
} catch (error) {
|
|
7215
|
-
lastError = error;
|
|
7216
|
-
}
|
|
7217
|
-
}
|
|
7218
|
-
throw lastError instanceof Error ? lastError : new Error(String(lastError));
|
|
7219
|
-
}
|
|
7220
|
-
}
|
|
7221
|
-
function validateClusterNodeConfig(config) {
|
|
7222
|
-
if (config.address.length === 0) {
|
|
7223
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
|
|
7224
|
-
}
|
|
7225
|
-
if (config.roles !== void 0) {
|
|
7226
|
-
if (config.roles.length === 0) {
|
|
7227
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.roles must contain at least one role");
|
|
7228
|
-
}
|
|
7229
|
-
const validRoles = /* @__PURE__ */ new Set(["data", "coordinator", "controller"]);
|
|
7230
|
-
for (const role of config.roles) {
|
|
7231
|
-
if (!validRoles.has(role)) {
|
|
7232
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, `Invalid role: '${role}'`, { role });
|
|
7233
|
-
}
|
|
7234
|
-
}
|
|
7235
|
-
}
|
|
7236
|
-
if (config.capacity !== void 0) {
|
|
7237
|
-
if (config.capacity.memoryBytes <= 0 || !Number.isFinite(config.capacity.memoryBytes)) {
|
|
7238
|
-
throw new NarsilError(
|
|
7239
|
-
ErrorCodes.CONFIG_INVALID,
|
|
7240
|
-
"ClusterNodeConfig.capacity.memoryBytes must be a positive finite number"
|
|
7241
|
-
);
|
|
7242
|
-
}
|
|
7243
|
-
if (config.capacity.cpuCores <= 0 || !Number.isInteger(config.capacity.cpuCores)) {
|
|
7244
|
-
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.capacity.cpuCores must be a positive integer");
|
|
7245
|
-
}
|
|
7246
|
-
}
|
|
7247
7350
|
}
|
|
7248
7351
|
|
|
7249
7352
|
// src/distribution/transport/in-memory.ts
|