@delali/narsil 0.3.1-next.79 → 0.3.1-next.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-1lEYL3WK.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-XSZTF4ZQ.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-DFW3U5UW.mjs → chunk-4CWHWOBI.mjs} +24 -195
- package/dist/{chunk-R6WBLBTC.mjs → chunk-4D6OH4EF.mjs} +10 -46
- package/dist/{chunk-RTR44V2A.mjs → chunk-4VBG6PEH.mjs} +586 -4
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-PJWG547Z.mjs → chunk-GNYXCLJS.mjs} +5 -5
- package/dist/{chunk-55WEJKWT.mjs → chunk-IH3GKNDQ.mjs} +197 -93
- package/dist/{chunk-AMGDG4ZJ.mjs → chunk-IN5CH4P2.mjs} +3 -3
- package/dist/{chunk-PXDQFYD5.mjs → chunk-KJ7LB3TR.mjs} +24 -140
- package/dist/chunk-LS5IRM23.mjs +140 -0
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-IBR6634K.mjs → chunk-R6A773WK.mjs} +2 -69
- package/dist/{chunk-4344I4MR.mjs → chunk-SREPGC6O.mjs} +336 -13
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/chunk-WQBCBKY3.mjs +8 -0
- package/dist/{chunk-P2BEMMOD.mjs → chunk-XYDP7CCJ.mjs} +151 -204
- package/dist/{chunk-Z3BXCZXS.mjs → chunk-XZF4LZ3X.mjs} +5 -3
- package/dist/chunk-YTJCGKM4.mjs +56 -0
- package/dist/chunk-ZCKPRCYI.mjs +30 -0
- package/dist/{client-ZXcbSViH.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-DHL2KsfD.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-DTe5EbWW.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-JVXDR5Y5.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +3 -2
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/coordinator/in-memory.mjs +2 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +759 -65
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-BjPBzWkA.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +138 -539
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +17 -15
- package/dist/invalidation/filesystem.mjs +2 -1
- package/dist/{memory-DW3OJ0im.d.ts → memory-BPKNx9Tl.d.ts} +1288 -1287
- package/dist/partial-words.mjs +6 -5
- package/dist/pattern.mjs +5 -2
- package/dist/persistence/durability/checkpoint-worker.d.ts +7 -7
- package/dist/persistence/durability/checkpoint-worker.mjs +17 -15
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +21 -13
- package/dist/{tasks-BTbwBqar.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-XBDVXKUE.mjs → thread-33M5AOEI.mjs} +13 -10
- package/dist/vector/search-worker.mjs +5 -4
- package/dist/workers/entry.mjs +5 -5
- package/package.json +13 -7
- package/dist/chunk-IOZPBNSG.mjs +0 -55
- package/dist/chunk-WFVOBZS4.mjs +0 -485
- package/dist/direct-executor-WB2HKU4N.mjs +0 -17
package/dist/distribution.mjs
CHANGED
|
@@ -1,37 +1,40 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
4
|
-
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
2
|
+
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-IH3GKNDQ.mjs';
|
|
4
|
+
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
|
|
5
5
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import { generateId, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
8
|
-
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-
|
|
9
|
-
import { getLanguage } from './chunk-
|
|
10
|
-
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT,
|
|
6
|
+
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
|
|
7
|
+
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-4CWHWOBI.mjs';
|
|
8
|
+
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
|
|
9
|
+
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
10
|
+
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
11
11
|
import './chunk-TGP5M5R7.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-IN5CH4P2.mjs';
|
|
13
13
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import { isPatternTolerance } from './chunk-
|
|
16
|
-
import './chunk-
|
|
17
|
-
import './chunk-
|
|
18
|
-
import { validateSchema, requireCreatableSchema, encodeDocument,
|
|
14
|
+
import './chunk-B3UOQJD3.mjs';
|
|
15
|
+
import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
|
|
16
|
+
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
17
|
+
import './chunk-4VBG6PEH.mjs';
|
|
18
|
+
import { validateSchema, requireCreatableSchema, DEFAULT_PAGE_SIZE, encodeDocument, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-KJ7LB3TR.mjs';
|
|
19
19
|
import './chunk-KEHK2O6P.mjs';
|
|
20
20
|
import './chunk-GES7U2PM.mjs';
|
|
21
|
+
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
21
22
|
import './chunk-U2BKYEC7.mjs';
|
|
22
|
-
import {
|
|
23
|
+
import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
24
|
+
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
25
|
+
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
23
26
|
import './chunk-POWOTUKJ.mjs';
|
|
24
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
25
|
-
import {
|
|
27
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-TPJU7NDJ.mjs';
|
|
28
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
26
29
|
import { randomUUID } from 'crypto';
|
|
27
30
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
28
31
|
import { mkdtemp, rm, mkdir } from 'fs/promises';
|
|
29
|
-
import { tmpdir } from 'os';
|
|
32
|
+
import { tmpdir, totalmem } from 'os';
|
|
30
33
|
import { join } from 'path';
|
|
31
34
|
|
|
32
35
|
// src/distribution/cluster/constants.ts
|
|
33
36
|
var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
|
|
34
|
-
var
|
|
37
|
+
var ESTIMATED_PARTITION_BYTES = 50 * 1024 * 1024;
|
|
35
38
|
var REBALANCE_THRESHOLD = 0.1;
|
|
36
39
|
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
37
40
|
var ALLOCATION_CAS_ATTEMPTS = 5;
|
|
@@ -132,22 +135,87 @@ var balanceDecider = {
|
|
|
132
135
|
}
|
|
133
136
|
};
|
|
134
137
|
|
|
138
|
+
// src/distribution/cluster/allocator/partition-sizes.ts
|
|
139
|
+
function isByteCount(value) {
|
|
140
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
141
|
+
}
|
|
142
|
+
function reportedEntriesOf(node) {
|
|
143
|
+
const reported = node.partitionBytes;
|
|
144
|
+
if (!Array.isArray(reported)) {
|
|
145
|
+
return [];
|
|
146
|
+
}
|
|
147
|
+
const entries = [];
|
|
148
|
+
for (const index of reported) {
|
|
149
|
+
if (index === null || typeof index !== "object" || typeof index.indexName !== "string") {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (!Array.isArray(index.partitions)) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
for (const pair of index.partitions) {
|
|
156
|
+
if (!Array.isArray(pair) || !Number.isSafeInteger(pair[0]) || pair[0] < 0 || !isByteCount(pair[1])) {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
entries.push({ indexName: index.indexName, partitionId: pair[0], bytes: pair[1] });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return entries;
|
|
163
|
+
}
|
|
164
|
+
function readPartitionSizes(nodes, indexName) {
|
|
165
|
+
const largestByPartition = /* @__PURE__ */ new Map();
|
|
166
|
+
const reportedByNode = /* @__PURE__ */ new Map();
|
|
167
|
+
const reportedBytesByNode = /* @__PURE__ */ new Map();
|
|
168
|
+
for (const node of nodes) {
|
|
169
|
+
const reportedHere = /* @__PURE__ */ new Set();
|
|
170
|
+
let reportedBytes = 0;
|
|
171
|
+
for (const entry of reportedEntriesOf(node)) {
|
|
172
|
+
reportedBytes += entry.bytes;
|
|
173
|
+
if (entry.indexName !== indexName) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
reportedHere.add(entry.partitionId);
|
|
177
|
+
largestByPartition.set(entry.partitionId, Math.max(largestByPartition.get(entry.partitionId) ?? 0, entry.bytes));
|
|
178
|
+
}
|
|
179
|
+
reportedByNode.set(node.nodeId, reportedHere);
|
|
180
|
+
reportedBytesByNode.set(node.nodeId, reportedBytes);
|
|
181
|
+
}
|
|
182
|
+
function sizeOf(partitionId) {
|
|
183
|
+
return largestByPartition.get(partitionId) ?? ESTIMATED_PARTITION_BYTES;
|
|
184
|
+
}
|
|
185
|
+
function reports(nodeId, partitionId) {
|
|
186
|
+
return reportedByNode.get(nodeId)?.has(partitionId) ?? false;
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
sizeOf,
|
|
190
|
+
reports,
|
|
191
|
+
heldBytesOf(nodeId, assignments) {
|
|
192
|
+
let heldBytes = reportedBytesByNode.get(nodeId) ?? 0;
|
|
193
|
+
for (const [partitionId, assignment] of assignments) {
|
|
194
|
+
const holdsCopy = assignment.primary === nodeId || assignment.replicas.includes(nodeId);
|
|
195
|
+
if (holdsCopy && !reports(nodeId, partitionId)) {
|
|
196
|
+
heldBytes += sizeOf(partitionId);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return heldBytes;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
135
204
|
// src/distribution/cluster/allocator/deciders/capacity.ts
|
|
136
|
-
function createCapacityDecider(
|
|
137
|
-
const bytesPerPartition = DEFAULT_ESTIMATED_PARTITION_BYTES;
|
|
205
|
+
function createCapacityDecider(sizes = readPartitionSizes([], "")) {
|
|
138
206
|
return {
|
|
139
207
|
name: "capacity",
|
|
140
208
|
canAllocate(context) {
|
|
141
|
-
const { candidateNodeId,
|
|
209
|
+
const { candidateNodeId, partitionId, allAssignments, nodes } = context;
|
|
142
210
|
const node = nodes.get(candidateNodeId);
|
|
143
211
|
if (node === void 0) {
|
|
144
212
|
return "NO";
|
|
145
213
|
}
|
|
146
|
-
if (node.capacity.memoryBytes === 0) {
|
|
214
|
+
if (node.capacity.memoryBytes === 0 || node.memory?.full === true) {
|
|
147
215
|
return "NO";
|
|
148
216
|
}
|
|
149
|
-
const
|
|
150
|
-
const projectedBytes = (
|
|
217
|
+
const addedBytes = sizes.reports(candidateNodeId, partitionId) ? 0 : sizes.sizeOf(partitionId);
|
|
218
|
+
const projectedBytes = sizes.heldBytesOf(candidateNodeId, allAssignments) + addedBytes;
|
|
151
219
|
if (projectedBytes > node.capacity.memoryBytes) {
|
|
152
220
|
return "NO";
|
|
153
221
|
}
|
|
@@ -255,11 +323,11 @@ function createZoneDecider(zoneAttribute) {
|
|
|
255
323
|
}
|
|
256
324
|
|
|
257
325
|
// src/distribution/cluster/allocator/deciders/index.ts
|
|
258
|
-
function createDeciderChain(constraints,
|
|
326
|
+
function createDeciderChain(constraints, partitionSizes) {
|
|
259
327
|
const deciders = [
|
|
260
328
|
colocationDecider,
|
|
261
329
|
createMaxShardsDecider(constraints.maxShardsPerNode),
|
|
262
|
-
createCapacityDecider()
|
|
330
|
+
createCapacityDecider(partitionSizes)
|
|
263
331
|
];
|
|
264
332
|
if (constraints.zoneAwareness) {
|
|
265
333
|
deciders.push(createZoneDecider(constraints.zoneAttribute));
|
|
@@ -789,7 +857,7 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
|
|
|
789
857
|
totalNodes: nodes.length
|
|
790
858
|
});
|
|
791
859
|
}
|
|
792
|
-
const deciders = createDeciderChain(constraints);
|
|
860
|
+
const deciders = createDeciderChain(constraints, readPartitionSizes(nodes, indexName));
|
|
793
861
|
if (currentTable === null) {
|
|
794
862
|
return initialAllocate(dataNodes, indexName, partitionCount, replicationFactor, constraints, deciders);
|
|
795
863
|
}
|
|
@@ -1620,6 +1688,7 @@ function handleQueuedBootstrapComplete(state, message, respond, coordinator, nod
|
|
|
1620
1688
|
function createEventLoopState(initialIndexNames) {
|
|
1621
1689
|
return {
|
|
1622
1690
|
unwatchNodes: null,
|
|
1691
|
+
placementByNodeId: /* @__PURE__ */ new Map(),
|
|
1623
1692
|
unwatchSchemas: null,
|
|
1624
1693
|
unwatchTransport: null,
|
|
1625
1694
|
knownIndexes: new Set(initialIndexNames),
|
|
@@ -1635,6 +1704,7 @@ function clearEventLoopWatchers(state) {
|
|
|
1635
1704
|
state.unwatchNodes();
|
|
1636
1705
|
state.unwatchNodes = null;
|
|
1637
1706
|
}
|
|
1707
|
+
state.placementByNodeId.clear();
|
|
1638
1708
|
if (state.unwatchSchemas !== null) {
|
|
1639
1709
|
state.unwatchSchemas();
|
|
1640
1710
|
state.unwatchSchemas = null;
|
|
@@ -1652,6 +1722,22 @@ function clearEventLoopWatchers(state) {
|
|
|
1652
1722
|
}
|
|
1653
1723
|
state.teardownTimers.clear();
|
|
1654
1724
|
}
|
|
1725
|
+
function placementOf(registration) {
|
|
1726
|
+
const { memory: _memory, partitionBytes: _partitionBytes, ...placement } = registration;
|
|
1727
|
+
return JSON.stringify(placement);
|
|
1728
|
+
}
|
|
1729
|
+
function changesTopology(state, event) {
|
|
1730
|
+
if (event.type === "node_left" || event.registration === null) {
|
|
1731
|
+
state.placementByNodeId.delete(event.nodeId);
|
|
1732
|
+
return true;
|
|
1733
|
+
}
|
|
1734
|
+
const placement = placementOf(event.registration);
|
|
1735
|
+
if (state.placementByNodeId.get(event.nodeId) === placement) {
|
|
1736
|
+
return false;
|
|
1737
|
+
}
|
|
1738
|
+
state.placementByNodeId.set(event.nodeId, placement);
|
|
1739
|
+
return true;
|
|
1740
|
+
}
|
|
1655
1741
|
|
|
1656
1742
|
// src/distribution/cluster/controller/event-loop/index.ts
|
|
1657
1743
|
async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
|
|
@@ -1659,8 +1745,10 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
1659
1745
|
state.transport = transport;
|
|
1660
1746
|
state.controllerNodeId = nodeId;
|
|
1661
1747
|
try {
|
|
1662
|
-
state.unwatchNodes = await coordinator.watchNodes((
|
|
1663
|
-
|
|
1748
|
+
state.unwatchNodes = await coordinator.watchNodes((event) => {
|
|
1749
|
+
if (changesTopology(state, event)) {
|
|
1750
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1751
|
+
}
|
|
1664
1752
|
});
|
|
1665
1753
|
state.unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
1666
1754
|
handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
|
|
@@ -2369,17 +2457,43 @@ function clearHeartbeatTimer(state) {
|
|
|
2369
2457
|
state.timer = null;
|
|
2370
2458
|
}
|
|
2371
2459
|
}
|
|
2460
|
+
function registrationToWrite(config) {
|
|
2461
|
+
return config.currentRegistration?.() ?? config.registration;
|
|
2462
|
+
}
|
|
2463
|
+
function refusedForSize(error) {
|
|
2464
|
+
return error instanceof NarsilError && error.code === ErrorCodes.CONFIG_INVALID;
|
|
2465
|
+
}
|
|
2466
|
+
async function writeRegistration(config) {
|
|
2467
|
+
const registration = registrationToWrite(config);
|
|
2468
|
+
try {
|
|
2469
|
+
await config.coordinator.registerNode(registration);
|
|
2470
|
+
} catch (error) {
|
|
2471
|
+
if (registration.partitionBytes === void 0 || !refusedForSize(error)) {
|
|
2472
|
+
throw error;
|
|
2473
|
+
}
|
|
2474
|
+
const { partitionBytes: _, ...withoutPartitionBytes } = registration;
|
|
2475
|
+
await config.coordinator.registerNode(withoutPartitionBytes);
|
|
2476
|
+
throw error;
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
function queueRegistrationWrite(state, config) {
|
|
2480
|
+
state.inFlight = state.inFlight.then(() => writeRegistration(config)).catch((error) => {
|
|
2481
|
+
if (config.onError !== void 0) {
|
|
2482
|
+
config.onError(error);
|
|
2483
|
+
}
|
|
2484
|
+
});
|
|
2485
|
+
}
|
|
2372
2486
|
function startRegistrationHeartbeat(state, config) {
|
|
2373
2487
|
clearHeartbeatTimer(state);
|
|
2374
|
-
state.timer = setInterval(() =>
|
|
2375
|
-
state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
|
|
2376
|
-
if (config.onError !== void 0) {
|
|
2377
|
-
config.onError(error);
|
|
2378
|
-
}
|
|
2379
|
-
});
|
|
2380
|
-
}, config.nodeHeartbeatIntervalMs);
|
|
2488
|
+
state.timer = setInterval(() => queueRegistrationWrite(state, config), config.nodeHeartbeatIntervalMs);
|
|
2381
2489
|
state.timer.unref?.();
|
|
2382
2490
|
}
|
|
2491
|
+
function renewRegistrationNow(state, config) {
|
|
2492
|
+
if (state.timer === null) {
|
|
2493
|
+
return;
|
|
2494
|
+
}
|
|
2495
|
+
queueRegistrationWrite(state, config);
|
|
2496
|
+
}
|
|
2383
2497
|
async function stopRegistrationHeartbeat(state) {
|
|
2384
2498
|
clearHeartbeatTimer(state);
|
|
2385
2499
|
await state.inFlight;
|
|
@@ -2387,7 +2501,7 @@ async function stopRegistrationHeartbeat(state) {
|
|
|
2387
2501
|
|
|
2388
2502
|
// src/distribution/cluster/node-lifecycle/join.ts
|
|
2389
2503
|
async function joinCluster(config, watcherState, heartbeatState) {
|
|
2390
|
-
await config.coordinator.registerNode(config
|
|
2504
|
+
await config.coordinator.registerNode(registrationToWrite(config));
|
|
2391
2505
|
config.onRegistered?.();
|
|
2392
2506
|
startRegistrationHeartbeat(heartbeatState, config);
|
|
2393
2507
|
const initialTables = await loadInitialAllocations(config);
|
|
@@ -2483,6 +2597,11 @@ function createDataNodeLifecycle(config) {
|
|
|
2483
2597
|
}
|
|
2484
2598
|
});
|
|
2485
2599
|
},
|
|
2600
|
+
renewRegistration() {
|
|
2601
|
+
if (status === "active" || status === "joining") {
|
|
2602
|
+
renewRegistrationNow(heartbeatState, config);
|
|
2603
|
+
}
|
|
2604
|
+
},
|
|
2486
2605
|
leave() {
|
|
2487
2606
|
return withLock(async () => {
|
|
2488
2607
|
if (status !== "active") {
|
|
@@ -3085,6 +3204,18 @@ var MAX_SNAPSHOT_SYNC_REQUEST_BYTES = 4096;
|
|
|
3085
3204
|
var MAX_SOURCE_ID_LENGTH = 256;
|
|
3086
3205
|
var MAX_WAIT_FOR_ACTIVE_REPLICAS = 255;
|
|
3087
3206
|
var MIN_CONTROLLER_LEASE_TTL_MS = 1e3;
|
|
3207
|
+
var DEFAULT_BLOCK_WRITES_ABOVE = 0.9;
|
|
3208
|
+
var DEFAULT_RESUME_WRITES_BELOW = 0.8;
|
|
3209
|
+
var MEMORY_SAMPLE_INTERVAL_MS = 1e3;
|
|
3210
|
+
var FULL_MEMORY_COLLECTION_INTERVAL_MS = 5e3;
|
|
3211
|
+
var PARTITION_BYTES_REPORT_BUDGET_BYTES = 262144;
|
|
3212
|
+
var PARTITION_BYTES_ENTRY_BYTES = 16;
|
|
3213
|
+
var PARTITION_BYTES_INDEX_OVERHEAD_BYTES = 32;
|
|
3214
|
+
var DEFAULT_IN_FLIGHT_WRITE_FRACTION = 0.02;
|
|
3215
|
+
var WRITE_MEMORY_SAMPLE_MAX_AGE_MS = 100;
|
|
3216
|
+
var DOCUMENT_NUMBER_BYTES = 8;
|
|
3217
|
+
var DOCUMENT_SCALAR_BYTES = 4;
|
|
3218
|
+
var DOCUMENT_ENTRY_OVERHEAD_BYTES = 2;
|
|
3088
3219
|
|
|
3089
3220
|
// src/distribution/cluster-node/bootstrap-fetch.ts
|
|
3090
3221
|
var RETRY_ANY_TARGET_CODES = /* @__PURE__ */ new Set([
|
|
@@ -4604,7 +4735,7 @@ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrim
|
|
|
4604
4735
|
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
4605
4736
|
);
|
|
4606
4737
|
}
|
|
4607
|
-
return { partitionId, assignment };
|
|
4738
|
+
return { table, partitionId, assignment };
|
|
4608
4739
|
}
|
|
4609
4740
|
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
4610
4741
|
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
@@ -5761,6 +5892,443 @@ function report(deps, indexName, reason) {
|
|
|
5761
5892
|
)
|
|
5762
5893
|
);
|
|
5763
5894
|
}
|
|
5895
|
+
function processLimitBytes() {
|
|
5896
|
+
const hostBytes = totalmem();
|
|
5897
|
+
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
5898
|
+
return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
|
|
5899
|
+
}
|
|
5900
|
+
function readMemoryFigures() {
|
|
5901
|
+
try {
|
|
5902
|
+
const heap = readHeapStatistics();
|
|
5903
|
+
const processBytes = process.memoryUsage.rss();
|
|
5904
|
+
const limitBytes = processLimitBytes();
|
|
5905
|
+
if (heap === null || !Number.isFinite(processBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) {
|
|
5906
|
+
return null;
|
|
5907
|
+
}
|
|
5908
|
+
return {
|
|
5909
|
+
processBytes,
|
|
5910
|
+
processLimitBytes: limitBytes,
|
|
5911
|
+
heapBytes: heap.usedBytes,
|
|
5912
|
+
heapLimitBytes: heap.configuredLimitBytes ?? heap.limitBytes
|
|
5913
|
+
};
|
|
5914
|
+
} catch {
|
|
5915
|
+
return null;
|
|
5916
|
+
}
|
|
5917
|
+
}
|
|
5918
|
+
|
|
5919
|
+
// src/distribution/cluster-node/memory/monitor.ts
|
|
5920
|
+
function fractionOf(usedBytes, limitBytes) {
|
|
5921
|
+
return limitBytes > 0 ? usedBytes / limitBytes : 0;
|
|
5922
|
+
}
|
|
5923
|
+
function nextFullState(wasFull, figures, options) {
|
|
5924
|
+
const processFraction = fractionOf(figures.processBytes, figures.processLimitBytes);
|
|
5925
|
+
const heapFraction = fractionOf(figures.heapBytes, figures.heapLimitBytes);
|
|
5926
|
+
if (!wasFull) {
|
|
5927
|
+
return processFraction > options.blockWritesAbove || heapFraction > options.blockWritesAbove;
|
|
5928
|
+
}
|
|
5929
|
+
return processFraction >= options.resumeWritesBelow || heapFraction >= options.resumeWritesBelow;
|
|
5930
|
+
}
|
|
5931
|
+
function createMemoryMonitor(options) {
|
|
5932
|
+
let latest = null;
|
|
5933
|
+
let timer = null;
|
|
5934
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
5935
|
+
let lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
5936
|
+
function figuresWithoutGarbage(figures) {
|
|
5937
|
+
const now3 = Date.now();
|
|
5938
|
+
if (now3 - lastCollectionAt < options.collectionIntervalMs) {
|
|
5939
|
+
return figures;
|
|
5940
|
+
}
|
|
5941
|
+
lastCollectionAt = now3;
|
|
5942
|
+
if (!options.collectGarbage()) {
|
|
5943
|
+
return figures;
|
|
5944
|
+
}
|
|
5945
|
+
return options.read() ?? figures;
|
|
5946
|
+
}
|
|
5947
|
+
function sample() {
|
|
5948
|
+
lastSampleAt = Date.now();
|
|
5949
|
+
let figures = options.read();
|
|
5950
|
+
if (figures === null) {
|
|
5951
|
+
return;
|
|
5952
|
+
}
|
|
5953
|
+
const wasFull = latest?.full ?? false;
|
|
5954
|
+
let full = nextFullState(wasFull, figures, options);
|
|
5955
|
+
if (full) {
|
|
5956
|
+
figures = figuresWithoutGarbage(figures);
|
|
5957
|
+
full = nextFullState(wasFull, figures, options);
|
|
5958
|
+
}
|
|
5959
|
+
latest = { ...figures, full };
|
|
5960
|
+
if (full !== wasFull) {
|
|
5961
|
+
options.onFullChange(full);
|
|
5962
|
+
}
|
|
5963
|
+
}
|
|
5964
|
+
return {
|
|
5965
|
+
report: () => latest,
|
|
5966
|
+
isFull: () => latest?.full ?? false,
|
|
5967
|
+
sample,
|
|
5968
|
+
sampleIfOlderThan(maxAgeMs) {
|
|
5969
|
+
if (timer !== null && Date.now() - lastSampleAt >= maxAgeMs) {
|
|
5970
|
+
sample();
|
|
5971
|
+
}
|
|
5972
|
+
},
|
|
5973
|
+
start() {
|
|
5974
|
+
if (timer !== null) {
|
|
5975
|
+
return;
|
|
5976
|
+
}
|
|
5977
|
+
sample();
|
|
5978
|
+
timer = setInterval(sample, options.intervalMs);
|
|
5979
|
+
timer.unref?.();
|
|
5980
|
+
},
|
|
5981
|
+
stop() {
|
|
5982
|
+
if (timer !== null) {
|
|
5983
|
+
clearInterval(timer);
|
|
5984
|
+
timer = null;
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
5987
|
+
};
|
|
5988
|
+
}
|
|
5989
|
+
|
|
5990
|
+
// src/distribution/cluster-node/memory/partition-bytes.ts
|
|
5991
|
+
function heldPartitionSizes(engine, indexName) {
|
|
5992
|
+
const held = new Set(engine.heldPartitionsOf(indexName) ?? []);
|
|
5993
|
+
if (held.size === 0) {
|
|
5994
|
+
return [];
|
|
5995
|
+
}
|
|
5996
|
+
const partitions = engine.getPartitionStats(indexName);
|
|
5997
|
+
const indexBytes = engine.getStats(indexName).estimatedMemoryBytes;
|
|
5998
|
+
let partitionBytes = 0;
|
|
5999
|
+
let documents = 0;
|
|
6000
|
+
for (const partition of partitions) {
|
|
6001
|
+
partitionBytes += partition.estimatedMemoryBytes;
|
|
6002
|
+
documents += partition.documentCount;
|
|
6003
|
+
}
|
|
6004
|
+
const sharedBytes = Math.max(0, indexBytes - partitionBytes);
|
|
6005
|
+
const sizes = [];
|
|
6006
|
+
for (const partition of partitions) {
|
|
6007
|
+
if (!held.has(partition.partitionId)) {
|
|
6008
|
+
continue;
|
|
6009
|
+
}
|
|
6010
|
+
const share = documents > 0 ? sharedBytes * partition.documentCount / documents : 0;
|
|
6011
|
+
sizes.push({
|
|
6012
|
+
indexName,
|
|
6013
|
+
partitionId: partition.partitionId,
|
|
6014
|
+
bytes: Math.round(partition.estimatedMemoryBytes + share)
|
|
6015
|
+
});
|
|
6016
|
+
}
|
|
6017
|
+
return sizes;
|
|
6018
|
+
}
|
|
6019
|
+
function largestWithinBudget(sizes) {
|
|
6020
|
+
const ordered = [...sizes].sort((left, right) => right.bytes - left.bytes);
|
|
6021
|
+
const reportedIndexes = /* @__PURE__ */ new Set();
|
|
6022
|
+
const kept = [];
|
|
6023
|
+
let budgetBytes = PARTITION_BYTES_REPORT_BUDGET_BYTES;
|
|
6024
|
+
for (const size of ordered) {
|
|
6025
|
+
const indexCost = reportedIndexes.has(size.indexName) ? 0 : PARTITION_BYTES_INDEX_OVERHEAD_BYTES + Buffer.byteLength(size.indexName);
|
|
6026
|
+
const cost = indexCost + PARTITION_BYTES_ENTRY_BYTES;
|
|
6027
|
+
if (cost > budgetBytes) {
|
|
6028
|
+
break;
|
|
6029
|
+
}
|
|
6030
|
+
budgetBytes -= cost;
|
|
6031
|
+
reportedIndexes.add(size.indexName);
|
|
6032
|
+
kept.push(size);
|
|
6033
|
+
}
|
|
6034
|
+
return kept;
|
|
6035
|
+
}
|
|
6036
|
+
function collectPartitionBytes(engine) {
|
|
6037
|
+
const sizes = [];
|
|
6038
|
+
for (const index of engine.listIndexes()) {
|
|
6039
|
+
try {
|
|
6040
|
+
sizes.push(...heldPartitionSizes(engine, index.name));
|
|
6041
|
+
} catch {
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
const byIndex = /* @__PURE__ */ new Map();
|
|
6045
|
+
for (const size of largestWithinBudget(sizes)) {
|
|
6046
|
+
let entry = byIndex.get(size.indexName);
|
|
6047
|
+
if (entry === void 0) {
|
|
6048
|
+
entry = { indexName: size.indexName, partitions: [] };
|
|
6049
|
+
byIndex.set(size.indexName, entry);
|
|
6050
|
+
}
|
|
6051
|
+
entry.partitions.push([size.partitionId, size.bytes]);
|
|
6052
|
+
}
|
|
6053
|
+
for (const entry of byIndex.values()) {
|
|
6054
|
+
entry.partitions.sort((left, right) => left[0] - right[0]);
|
|
6055
|
+
}
|
|
6056
|
+
return [...byIndex.values()];
|
|
6057
|
+
}
|
|
6058
|
+
|
|
6059
|
+
// src/distribution/cluster-node/memory/write-block.ts
|
|
6060
|
+
var NODE_MEMORY_FULL_REASON = "node_memory_full";
|
|
6061
|
+
function createNodeMemoryView(options) {
|
|
6062
|
+
const registeredFull = /* @__PURE__ */ new Set();
|
|
6063
|
+
let unwatch = null;
|
|
6064
|
+
function record(nodeId, full) {
|
|
6065
|
+
if (full) {
|
|
6066
|
+
registeredFull.add(nodeId);
|
|
6067
|
+
} else {
|
|
6068
|
+
registeredFull.delete(nodeId);
|
|
6069
|
+
}
|
|
6070
|
+
}
|
|
6071
|
+
return {
|
|
6072
|
+
fullNodeIds() {
|
|
6073
|
+
const fullNodeIds = new Set(registeredFull);
|
|
6074
|
+
fullNodeIds.delete(options.localNodeId);
|
|
6075
|
+
if (options.localIsFull()) {
|
|
6076
|
+
fullNodeIds.add(options.localNodeId);
|
|
6077
|
+
}
|
|
6078
|
+
return fullNodeIds;
|
|
6079
|
+
},
|
|
6080
|
+
async start() {
|
|
6081
|
+
if (unwatch !== null) {
|
|
6082
|
+
return;
|
|
6083
|
+
}
|
|
6084
|
+
const reportedByWatch = /* @__PURE__ */ new Set();
|
|
6085
|
+
unwatch = await options.coordinator.watchNodes((event) => {
|
|
6086
|
+
reportedByWatch.add(event.nodeId);
|
|
6087
|
+
record(event.nodeId, event.registration?.memory?.full === true);
|
|
6088
|
+
});
|
|
6089
|
+
for (const node of await options.coordinator.listNodes()) {
|
|
6090
|
+
if (!reportedByWatch.has(node.nodeId)) {
|
|
6091
|
+
record(node.nodeId, node.memory?.full === true);
|
|
6092
|
+
}
|
|
6093
|
+
}
|
|
6094
|
+
},
|
|
6095
|
+
stop() {
|
|
6096
|
+
if (unwatch !== null) {
|
|
6097
|
+
unwatch();
|
|
6098
|
+
unwatch = null;
|
|
6099
|
+
}
|
|
6100
|
+
registeredFull.clear();
|
|
6101
|
+
}
|
|
6102
|
+
};
|
|
6103
|
+
}
|
|
6104
|
+
function copyHoldersOf(table) {
|
|
6105
|
+
const holders = /* @__PURE__ */ new Set();
|
|
6106
|
+
for (const assignment of table.assignments.values()) {
|
|
6107
|
+
if (assignment.primary !== null) {
|
|
6108
|
+
holders.add(assignment.primary);
|
|
6109
|
+
}
|
|
6110
|
+
for (const replica of assignment.replicas) {
|
|
6111
|
+
holders.add(replica);
|
|
6112
|
+
}
|
|
6113
|
+
}
|
|
6114
|
+
return holders;
|
|
6115
|
+
}
|
|
6116
|
+
function findWriteBlock(table, view) {
|
|
6117
|
+
if (view === void 0) {
|
|
6118
|
+
return null;
|
|
6119
|
+
}
|
|
6120
|
+
const fullNodeIds = view.fullNodeIds();
|
|
6121
|
+
if (fullNodeIds.size === 0) {
|
|
6122
|
+
return null;
|
|
6123
|
+
}
|
|
6124
|
+
const blockingNodeIds = [...copyHoldersOf(table)].filter((nodeId) => fullNodeIds.has(nodeId)).sort(compareCodePoints);
|
|
6125
|
+
if (blockingNodeIds.length === 0) {
|
|
6126
|
+
return null;
|
|
6127
|
+
}
|
|
6128
|
+
const quotedNodeIds = blockingNodeIds.map((nodeId) => `'${nodeId}'`).join(", ");
|
|
6129
|
+
const holders = blockingNodeIds.length === 1 ? `node ${quotedNodeIds} holds a copy of it and reports` : `nodes ${quotedNodeIds} hold copies of it and report`;
|
|
6130
|
+
return new NarsilError(
|
|
6131
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
6132
|
+
`Index '${table.indexName}' refuses inserts and updates because ${holders} full memory. Remove documents from the indexes on that node or give it more memory, and writes resume once its memory falls below the resume threshold.`,
|
|
6133
|
+
{ reason: NODE_MEMORY_FULL_REASON, indexName: table.indexName, fullNodeIds: blockingNodeIds }
|
|
6134
|
+
);
|
|
6135
|
+
}
|
|
6136
|
+
function requireWritableIndex(table, view) {
|
|
6137
|
+
const block = findWriteBlock(table, view);
|
|
6138
|
+
if (block !== null) {
|
|
6139
|
+
throw block;
|
|
6140
|
+
}
|
|
6141
|
+
}
|
|
6142
|
+
|
|
6143
|
+
// src/distribution/cluster-node/memory/write-pressure.ts
|
|
6144
|
+
var WRITE_BYTES_IN_FLIGHT_REASON = "write_bytes_in_flight";
|
|
6145
|
+
function createWritePressure(options) {
|
|
6146
|
+
let inFlight = 0;
|
|
6147
|
+
return {
|
|
6148
|
+
admit(indexName, bytes) {
|
|
6149
|
+
options.beforeAdmit();
|
|
6150
|
+
const requestBytes = Math.max(0, Math.ceil(bytes));
|
|
6151
|
+
if (inFlight + requestBytes > options.limitBytes) {
|
|
6152
|
+
throw new NarsilError(
|
|
6153
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
6154
|
+
`Node '${options.nodeId}' refuses this write to index '${indexName}' because its ${requestBytes} bytes would take the writes in flight on the node from ${inFlight} bytes past the limit of ${options.limitBytes} bytes. Retry once the writes in flight finish, or send smaller batches.`,
|
|
6155
|
+
{
|
|
6156
|
+
reason: WRITE_BYTES_IN_FLIGHT_REASON,
|
|
6157
|
+
indexName,
|
|
6158
|
+
nodeId: options.nodeId,
|
|
6159
|
+
requestBytes,
|
|
6160
|
+
inFlightBytes: inFlight,
|
|
6161
|
+
limitBytes: options.limitBytes
|
|
6162
|
+
}
|
|
6163
|
+
);
|
|
6164
|
+
}
|
|
6165
|
+
inFlight += requestBytes;
|
|
6166
|
+
let released = false;
|
|
6167
|
+
return () => {
|
|
6168
|
+
if (released) {
|
|
6169
|
+
return;
|
|
6170
|
+
}
|
|
6171
|
+
released = true;
|
|
6172
|
+
inFlight -= requestBytes;
|
|
6173
|
+
};
|
|
6174
|
+
}
|
|
6175
|
+
};
|
|
6176
|
+
}
|
|
6177
|
+
async function withAdmittedWrite(pressure, indexName, bytes, work) {
|
|
6178
|
+
if (pressure === void 0) {
|
|
6179
|
+
return work();
|
|
6180
|
+
}
|
|
6181
|
+
const release = pressure.admit(indexName, bytes());
|
|
6182
|
+
try {
|
|
6183
|
+
return await work();
|
|
6184
|
+
} finally {
|
|
6185
|
+
release();
|
|
6186
|
+
}
|
|
6187
|
+
}
|
|
6188
|
+
function estimateDocumentBytes(document) {
|
|
6189
|
+
let bytes = 0;
|
|
6190
|
+
const pending = [document];
|
|
6191
|
+
const visited = /* @__PURE__ */ new Set();
|
|
6192
|
+
while (pending.length > 0) {
|
|
6193
|
+
const value = pending.pop();
|
|
6194
|
+
if (typeof value === "string") {
|
|
6195
|
+
bytes += value.length;
|
|
6196
|
+
continue;
|
|
6197
|
+
}
|
|
6198
|
+
if (typeof value === "number" || typeof value === "bigint") {
|
|
6199
|
+
bytes += DOCUMENT_NUMBER_BYTES;
|
|
6200
|
+
continue;
|
|
6201
|
+
}
|
|
6202
|
+
if (value === null || typeof value !== "object") {
|
|
6203
|
+
bytes += DOCUMENT_SCALAR_BYTES;
|
|
6204
|
+
continue;
|
|
6205
|
+
}
|
|
6206
|
+
if (visited.has(value)) {
|
|
6207
|
+
continue;
|
|
6208
|
+
}
|
|
6209
|
+
visited.add(value);
|
|
6210
|
+
if (ArrayBuffer.isView(value)) {
|
|
6211
|
+
bytes += value.byteLength;
|
|
6212
|
+
} else if (Array.isArray(value)) {
|
|
6213
|
+
bytes += DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
6214
|
+
for (const item of value) {
|
|
6215
|
+
pending.push(item);
|
|
6216
|
+
}
|
|
6217
|
+
} else {
|
|
6218
|
+
for (const [key, item] of Object.entries(value)) {
|
|
6219
|
+
bytes += key.length + DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
6220
|
+
pending.push(item);
|
|
6221
|
+
}
|
|
6222
|
+
}
|
|
6223
|
+
}
|
|
6224
|
+
return bytes;
|
|
6225
|
+
}
|
|
6226
|
+
function estimateDocumentsBytes(documents) {
|
|
6227
|
+
let bytes = 0;
|
|
6228
|
+
for (const document of documents) {
|
|
6229
|
+
bytes += estimateDocumentBytes(document);
|
|
6230
|
+
}
|
|
6231
|
+
return bytes;
|
|
6232
|
+
}
|
|
6233
|
+
function estimateForwardedBytes(operation) {
|
|
6234
|
+
const documentBytes = operation.document?.byteLength ?? 0;
|
|
6235
|
+
return operation.updateFields === null ? documentBytes : documentBytes + estimateDocumentBytes(operation.updateFields);
|
|
6236
|
+
}
|
|
6237
|
+
|
|
6238
|
+
// src/distribution/cluster-node/memory/node-memory.ts
|
|
6239
|
+
function partitionBytesOf(engine) {
|
|
6240
|
+
try {
|
|
6241
|
+
return collectPartitionBytes(engine);
|
|
6242
|
+
} catch {
|
|
6243
|
+
return [];
|
|
6244
|
+
}
|
|
6245
|
+
}
|
|
6246
|
+
function createNodeMemoryReporting(options) {
|
|
6247
|
+
const monitor = createMemoryMonitor({
|
|
6248
|
+
blockWritesAbove: options.config?.blockWritesAbove ?? DEFAULT_BLOCK_WRITES_ABOVE,
|
|
6249
|
+
resumeWritesBelow: options.config?.resumeWritesBelow ?? DEFAULT_RESUME_WRITES_BELOW,
|
|
6250
|
+
intervalMs: MEMORY_SAMPLE_INTERVAL_MS,
|
|
6251
|
+
collectionIntervalMs: FULL_MEMORY_COLLECTION_INTERVAL_MS,
|
|
6252
|
+
read: readMemoryFigures,
|
|
6253
|
+
collectGarbage: collectAndReturnPages,
|
|
6254
|
+
onFullChange: options.onFullChange
|
|
6255
|
+
});
|
|
6256
|
+
const view = createNodeMemoryView({
|
|
6257
|
+
coordinator: options.coordinator,
|
|
6258
|
+
localNodeId: options.registration.nodeId,
|
|
6259
|
+
localIsFull: monitor.isFull
|
|
6260
|
+
});
|
|
6261
|
+
const heapLimitBytes = readMemoryFigures()?.heapLimitBytes ?? Number.POSITIVE_INFINITY;
|
|
6262
|
+
const writePressure = createWritePressure({
|
|
6263
|
+
nodeId: options.registration.nodeId,
|
|
6264
|
+
limitBytes: Math.floor(
|
|
6265
|
+
heapLimitBytes * (options.config?.inFlightWriteFraction ?? DEFAULT_IN_FLIGHT_WRITE_FRACTION)
|
|
6266
|
+
),
|
|
6267
|
+
beforeAdmit: () => monitor.sampleIfOlderThan(WRITE_MEMORY_SAMPLE_MAX_AGE_MS)
|
|
6268
|
+
});
|
|
6269
|
+
return {
|
|
6270
|
+
view,
|
|
6271
|
+
writePressure,
|
|
6272
|
+
async start() {
|
|
6273
|
+
monitor.start();
|
|
6274
|
+
await view.start();
|
|
6275
|
+
},
|
|
6276
|
+
stop() {
|
|
6277
|
+
monitor.stop();
|
|
6278
|
+
view.stop();
|
|
6279
|
+
},
|
|
6280
|
+
currentRegistration() {
|
|
6281
|
+
const registration = { ...options.registration };
|
|
6282
|
+
const memory = monitor.report();
|
|
6283
|
+
if (memory !== null) {
|
|
6284
|
+
registration.memory = { ...memory };
|
|
6285
|
+
}
|
|
6286
|
+
const partitionBytes = partitionBytesOf(options.engine);
|
|
6287
|
+
if (partitionBytes.length > 0) {
|
|
6288
|
+
registration.partitionBytes = partitionBytes;
|
|
6289
|
+
}
|
|
6290
|
+
return registration;
|
|
6291
|
+
}
|
|
6292
|
+
};
|
|
6293
|
+
}
|
|
6294
|
+
function transportableDetails(error) {
|
|
6295
|
+
if (!(error instanceof NarsilError)) {
|
|
6296
|
+
return void 0;
|
|
6297
|
+
}
|
|
6298
|
+
try {
|
|
6299
|
+
encode(error.details);
|
|
6300
|
+
return error.details;
|
|
6301
|
+
} catch {
|
|
6302
|
+
return void 0;
|
|
6303
|
+
}
|
|
6304
|
+
}
|
|
6305
|
+
function encodeErrorReply(code, message, details) {
|
|
6306
|
+
if (details !== void 0) {
|
|
6307
|
+
try {
|
|
6308
|
+
return encode({ error: true, code, message, details });
|
|
6309
|
+
} catch {
|
|
6310
|
+
return encode({ error: true, code, message });
|
|
6311
|
+
}
|
|
6312
|
+
}
|
|
6313
|
+
return encode({ error: true, code, message });
|
|
6314
|
+
}
|
|
6315
|
+
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
6316
|
+
return encodeErrorReply(
|
|
6317
|
+
error instanceof NarsilError ? error.code : fallbackCode,
|
|
6318
|
+
error instanceof Error ? error.message : String(error),
|
|
6319
|
+
error instanceof NarsilError ? error.details : void 0
|
|
6320
|
+
);
|
|
6321
|
+
}
|
|
6322
|
+
function isPlainRecord(value) {
|
|
6323
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
6324
|
+
return false;
|
|
6325
|
+
}
|
|
6326
|
+
const prototype = Object.getPrototypeOf(value);
|
|
6327
|
+
return prototype === Object.prototype || prototype === null;
|
|
6328
|
+
}
|
|
6329
|
+
function remoteErrorDetails(value) {
|
|
6330
|
+
return isPlainRecord(value) ? value : {};
|
|
6331
|
+
}
|
|
5764
6332
|
|
|
5765
6333
|
// src/distribution/cluster-node/snapshot-auth.ts
|
|
5766
6334
|
var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
|
|
@@ -6040,12 +6608,12 @@ function assertMessageWithinLimit(bytes, label) {
|
|
|
6040
6608
|
);
|
|
6041
6609
|
}
|
|
6042
6610
|
}
|
|
6043
|
-
async function respondError(sink, sourceId, requestId, code, message) {
|
|
6611
|
+
async function respondError(sink, sourceId, requestId, code, message, details) {
|
|
6044
6612
|
const response = {
|
|
6045
6613
|
type: SNAPSHOT_SYNC_ERROR_TYPE,
|
|
6046
6614
|
sourceId,
|
|
6047
6615
|
requestId,
|
|
6048
|
-
payload:
|
|
6616
|
+
payload: encodeErrorReply(code, message, details)
|
|
6049
6617
|
};
|
|
6050
6618
|
await sink(response);
|
|
6051
6619
|
sink.closed = true;
|
|
@@ -6123,7 +6691,7 @@ async function acquireAndStream(message, sink, request, deps, options) {
|
|
|
6123
6691
|
} catch (err) {
|
|
6124
6692
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
6125
6693
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6126
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6694
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6127
6695
|
return;
|
|
6128
6696
|
}
|
|
6129
6697
|
try {
|
|
@@ -6134,7 +6702,7 @@ async function acquireAndStream(message, sink, request, deps, options) {
|
|
|
6134
6702
|
} catch (err) {
|
|
6135
6703
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
6136
6704
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6137
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6705
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6138
6706
|
return;
|
|
6139
6707
|
}
|
|
6140
6708
|
try {
|
|
@@ -6154,7 +6722,7 @@ async function streamLeasedSnapshot(message, sink, request, deps, options, build
|
|
|
6154
6722
|
} catch (err) {
|
|
6155
6723
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
6156
6724
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6157
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6725
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6158
6726
|
return;
|
|
6159
6727
|
}
|
|
6160
6728
|
try {
|
|
@@ -6224,7 +6792,7 @@ async function decodeRequest(message, sink, deps) {
|
|
|
6224
6792
|
} catch (err) {
|
|
6225
6793
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
6226
6794
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6227
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6795
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6228
6796
|
return null;
|
|
6229
6797
|
}
|
|
6230
6798
|
}
|
|
@@ -6302,7 +6870,7 @@ async function handleSnapshotSyncRequest(message, respond, deps) {
|
|
|
6302
6870
|
} catch (err) {
|
|
6303
6871
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
6304
6872
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6305
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6873
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6306
6874
|
}
|
|
6307
6875
|
}
|
|
6308
6876
|
async function runSnapshotSyncRequest(message, sink, deps) {
|
|
@@ -8598,7 +9166,7 @@ function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
|
8598
9166
|
throw new NarsilError(
|
|
8599
9167
|
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
8600
9168
|
typeof decoded.message === "string" ? decoded.message : `Remote primary rejected a forwarded write for index '${indexName}'`,
|
|
8601
|
-
{ indexName, primaryNodeId }
|
|
9169
|
+
{ ...remoteErrorDetails(decoded.details), indexName, primaryNodeId }
|
|
8602
9170
|
);
|
|
8603
9171
|
}
|
|
8604
9172
|
return decoded;
|
|
@@ -9102,7 +9670,18 @@ async function replicateAppendedWrites(assignment, appended, failed, rollback, d
|
|
|
9102
9670
|
}
|
|
9103
9671
|
return succeeded;
|
|
9104
9672
|
}
|
|
9105
|
-
|
|
9673
|
+
function applyForwardedWrite(payload, deps) {
|
|
9674
|
+
if (payload.operation === "remove") {
|
|
9675
|
+
return applyAdmittedForwardedWrite(payload, deps);
|
|
9676
|
+
}
|
|
9677
|
+
return withAdmittedWrite(
|
|
9678
|
+
deps.writePressure,
|
|
9679
|
+
payload.indexName,
|
|
9680
|
+
() => estimateForwardedBytes(payload),
|
|
9681
|
+
() => applyAdmittedForwardedWrite(payload, deps)
|
|
9682
|
+
);
|
|
9683
|
+
}
|
|
9684
|
+
async function applyAdmittedForwardedWrite(payload, deps) {
|
|
9106
9685
|
const resolution = await resolvePrimaryAssignment(payload.indexName, payload.documentId, deps, true);
|
|
9107
9686
|
if (resolution === null) {
|
|
9108
9687
|
throw new NarsilError(
|
|
@@ -9111,6 +9690,9 @@ async function applyForwardedWrite(payload, deps) {
|
|
|
9111
9690
|
{ indexName: payload.indexName }
|
|
9112
9691
|
);
|
|
9113
9692
|
}
|
|
9693
|
+
if (payload.operation !== "remove") {
|
|
9694
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9695
|
+
}
|
|
9114
9696
|
if (payload.operation === "insert") {
|
|
9115
9697
|
if (payload.document === null) {
|
|
9116
9698
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Invalid ForwardPayload: insert requires a document");
|
|
@@ -9251,7 +9833,7 @@ async function forwardBatchToRemote(indexName, items, primaryNodeId, deps) {
|
|
|
9251
9833
|
error: new NarsilError(
|
|
9252
9834
|
operationResult.errorCode ?? ErrorCodes.QUERY_ROUTING_FAILED,
|
|
9253
9835
|
operationResult.errorMessage ?? `Remote primary rejected forwarded operation '${operationResult.documentId}'`,
|
|
9254
|
-
{ indexName, primaryNodeId }
|
|
9836
|
+
{ ...operationResult.errorDetails ?? {}, indexName, primaryNodeId }
|
|
9255
9837
|
)
|
|
9256
9838
|
});
|
|
9257
9839
|
}
|
|
@@ -9285,7 +9867,36 @@ function groupIntoRuns(payload, partitionCount) {
|
|
|
9285
9867
|
}
|
|
9286
9868
|
return runsByPartition;
|
|
9287
9869
|
}
|
|
9870
|
+
function admitForwardedWrites(payload, deps) {
|
|
9871
|
+
let bytes = 0;
|
|
9872
|
+
for (const operation of payload.operations) {
|
|
9873
|
+
if (operation.operation !== "remove") {
|
|
9874
|
+
bytes += estimateForwardedBytes(operation);
|
|
9875
|
+
}
|
|
9876
|
+
}
|
|
9877
|
+
if (deps.writePressure === void 0 || bytes === 0) {
|
|
9878
|
+
return { release: () => {
|
|
9879
|
+
}, refusal: null };
|
|
9880
|
+
}
|
|
9881
|
+
try {
|
|
9882
|
+
return { release: deps.writePressure.admit(payload.indexName, bytes), refusal: null };
|
|
9883
|
+
} catch (error) {
|
|
9884
|
+
if (!(error instanceof NarsilError)) {
|
|
9885
|
+
throw error;
|
|
9886
|
+
}
|
|
9887
|
+
return { release: () => {
|
|
9888
|
+
}, refusal: error };
|
|
9889
|
+
}
|
|
9890
|
+
}
|
|
9288
9891
|
async function applyForwardedBatch(payload, deps) {
|
|
9892
|
+
const admission = admitForwardedWrites(payload, deps);
|
|
9893
|
+
try {
|
|
9894
|
+
return await applyAdmittedForwardedBatch(payload, deps, admission.refusal);
|
|
9895
|
+
} finally {
|
|
9896
|
+
admission.release();
|
|
9897
|
+
}
|
|
9898
|
+
}
|
|
9899
|
+
async function applyAdmittedForwardedBatch(payload, deps, refusal) {
|
|
9289
9900
|
const results = new Array(payload.operations.length);
|
|
9290
9901
|
const table = await deps.coordinator.getAllocation(payload.indexName);
|
|
9291
9902
|
if (table === null || table.assignments.size === 0) {
|
|
@@ -9300,9 +9911,17 @@ async function applyForwardedBatch(payload, deps) {
|
|
|
9300
9911
|
return results;
|
|
9301
9912
|
}
|
|
9302
9913
|
const runsByPartition = groupIntoRuns(payload, table.assignments.size);
|
|
9914
|
+
const writesIndex = payload.operations.some((operation) => operation.operation !== "remove");
|
|
9915
|
+
const block = writesIndex ? refusal ?? findWriteBlock(table, deps.memoryView) : null;
|
|
9303
9916
|
await Promise.all(
|
|
9304
9917
|
Array.from(runsByPartition.entries()).map(async ([partitionId, runs]) => {
|
|
9305
9918
|
for (const run of runs) {
|
|
9919
|
+
if (block !== null && run.operation !== "remove") {
|
|
9920
|
+
for (const { index, wire } of run.operations) {
|
|
9921
|
+
results[index] = failureResult(wire.documentId, block);
|
|
9922
|
+
}
|
|
9923
|
+
continue;
|
|
9924
|
+
}
|
|
9306
9925
|
await applyRun(payload.indexName, partitionId, run, table.assignments.get(partitionId), deps, results);
|
|
9307
9926
|
}
|
|
9308
9927
|
})
|
|
@@ -9402,7 +10021,8 @@ function failureResult(documentId, error) {
|
|
|
9402
10021
|
documentId,
|
|
9403
10022
|
success: false,
|
|
9404
10023
|
errorCode: error.code,
|
|
9405
|
-
errorMessage: error.message
|
|
10024
|
+
errorMessage: error.message,
|
|
10025
|
+
errorDetails: transportableDetails(error) ?? null
|
|
9406
10026
|
};
|
|
9407
10027
|
}
|
|
9408
10028
|
|
|
@@ -9450,11 +10070,20 @@ function collectGroupFailure(failed, docIds, err, indexName, fallbackCode) {
|
|
|
9450
10070
|
failed.push({ docId, error: narsilErr });
|
|
9451
10071
|
}
|
|
9452
10072
|
}
|
|
9453
|
-
|
|
10073
|
+
function routeInsertBatch(indexName, documents, deps, options) {
|
|
10074
|
+
return withAdmittedWrite(
|
|
10075
|
+
deps.writePressure,
|
|
10076
|
+
indexName,
|
|
10077
|
+
() => estimateDocumentsBytes(documents),
|
|
10078
|
+
() => routeAdmittedInsertBatch(indexName, documents, deps, options)
|
|
10079
|
+
);
|
|
10080
|
+
}
|
|
10081
|
+
async function routeAdmittedInsertBatch(indexName, documents, deps, options) {
|
|
9454
10082
|
const table = await routableAllocation(deps.coordinator, indexName);
|
|
9455
10083
|
if (table === null) {
|
|
9456
10084
|
return deps.engine.insertBatch(indexName, documents, options);
|
|
9457
10085
|
}
|
|
10086
|
+
requireWritableIndex(table, deps.memoryView);
|
|
9458
10087
|
const withIds = documents.map((doc) => ({
|
|
9459
10088
|
doc,
|
|
9460
10089
|
docId: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : generateId()
|
|
@@ -9518,11 +10147,20 @@ async function routeRemoveBatch(indexName, docIds, deps) {
|
|
|
9518
10147
|
}
|
|
9519
10148
|
return { succeeded, failed };
|
|
9520
10149
|
}
|
|
9521
|
-
|
|
10150
|
+
function routeUpdateBatch(indexName, updates, deps) {
|
|
10151
|
+
return withAdmittedWrite(
|
|
10152
|
+
deps.writePressure,
|
|
10153
|
+
indexName,
|
|
10154
|
+
() => estimateDocumentsBytes(updates.map((update) => update.document)),
|
|
10155
|
+
() => routeAdmittedUpdateBatch(indexName, updates, deps)
|
|
10156
|
+
);
|
|
10157
|
+
}
|
|
10158
|
+
async function routeAdmittedUpdateBatch(indexName, updates, deps) {
|
|
9522
10159
|
const table = await routableAllocation(deps.coordinator, indexName);
|
|
9523
10160
|
if (table === null) {
|
|
9524
10161
|
return deps.engine.updateBatch(indexName, updates);
|
|
9525
10162
|
}
|
|
10163
|
+
requireWritableIndex(table, deps.memoryView);
|
|
9526
10164
|
const routed = routeBatchItems(table, indexName, updates, (update) => update.docId, deps.nodeId);
|
|
9527
10165
|
const succeeded = [];
|
|
9528
10166
|
const failed = routed.failed;
|
|
@@ -9675,12 +10313,21 @@ async function routeDropIndex(name, coordinator, engine) {
|
|
|
9675
10313
|
}
|
|
9676
10314
|
await coordinator.dropSchema(name);
|
|
9677
10315
|
}
|
|
9678
|
-
|
|
10316
|
+
function routeInsert(indexName, document, docId, deps, options) {
|
|
10317
|
+
return withAdmittedWrite(
|
|
10318
|
+
deps.writePressure,
|
|
10319
|
+
indexName,
|
|
10320
|
+
() => estimateDocumentBytes(document),
|
|
10321
|
+
() => routeAdmittedInsert(indexName, document, docId, deps, options)
|
|
10322
|
+
);
|
|
10323
|
+
}
|
|
10324
|
+
async function routeAdmittedInsert(indexName, document, docId, deps, options) {
|
|
9679
10325
|
const resolvedDocId = docId ?? providedDocId(document) ?? generateId();
|
|
9680
10326
|
const resolution = await resolvePrimaryAssignment(indexName, resolvedDocId, deps, false);
|
|
9681
10327
|
if (resolution === null) {
|
|
9682
10328
|
return deps.engine.insert(indexName, document, resolvedDocId, options);
|
|
9683
10329
|
}
|
|
10330
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9684
10331
|
const primaryNodeId = resolution.assignment.primary;
|
|
9685
10332
|
if (primaryNodeId === deps.nodeId) {
|
|
9686
10333
|
return applyPrimaryInsert(
|
|
@@ -9706,11 +10353,20 @@ async function routeRemove(indexName, docId, deps) {
|
|
|
9706
10353
|
}
|
|
9707
10354
|
return forwardRemoveToRemote(indexName, docId, primaryNodeId, deps);
|
|
9708
10355
|
}
|
|
9709
|
-
|
|
10356
|
+
function routeUpdate(indexName, docId, document, deps) {
|
|
10357
|
+
return withAdmittedWrite(
|
|
10358
|
+
deps.writePressure,
|
|
10359
|
+
indexName,
|
|
10360
|
+
() => estimateDocumentBytes(document),
|
|
10361
|
+
() => routeAdmittedUpdate(indexName, docId, document, deps)
|
|
10362
|
+
);
|
|
10363
|
+
}
|
|
10364
|
+
async function routeAdmittedUpdate(indexName, docId, document, deps) {
|
|
9710
10365
|
const resolution = await resolvePrimaryAssignment(indexName, docId, deps, false);
|
|
9711
10366
|
if (resolution === null) {
|
|
9712
10367
|
return deps.engine.update(indexName, docId, document);
|
|
9713
10368
|
}
|
|
10369
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9714
10370
|
const primaryNodeId = resolution.assignment.primary;
|
|
9715
10371
|
if (primaryNodeId === deps.nodeId) {
|
|
9716
10372
|
return applyPrimaryUpdate(indexName, docId, document, resolution.partitionId, resolution.assignment, deps);
|
|
@@ -9915,11 +10571,7 @@ function createDataNodeHandler(deps) {
|
|
|
9915
10571
|
return;
|
|
9916
10572
|
}
|
|
9917
10573
|
} catch (err) {
|
|
9918
|
-
const errorPayload =
|
|
9919
|
-
error: true,
|
|
9920
|
-
code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
|
|
9921
|
-
message: err instanceof Error ? err.message : String(err)
|
|
9922
|
-
});
|
|
10574
|
+
const errorPayload = encodeErrorReplyFrom(err, "INTERNAL_ERROR");
|
|
9923
10575
|
await respond({
|
|
9924
10576
|
type: `${message.type}.error`,
|
|
9925
10577
|
sourceId: deps.nodeId,
|
|
@@ -11635,6 +12287,8 @@ function resumeNumberingAboveOwnCopy(indexName, partitionId, assignment, deps) {
|
|
|
11635
12287
|
}
|
|
11636
12288
|
deps.seedReplicationLog(indexName, partitionId, floor + 1, assignment.primaryTerm);
|
|
11637
12289
|
}
|
|
12290
|
+
|
|
12291
|
+
// src/distribution/cluster-node/transport-listener.ts
|
|
11638
12292
|
var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
11639
12293
|
ReplicationMessageTypes.INSYNC_ADD,
|
|
11640
12294
|
ReplicationMessageTypes.INSYNC_REMOVE,
|
|
@@ -11645,7 +12299,7 @@ function refuseMessage(nodeId, message, respond, code, text) {
|
|
|
11645
12299
|
type: `${message.type}.error`,
|
|
11646
12300
|
sourceId: nodeId,
|
|
11647
12301
|
requestId: message.requestId,
|
|
11648
|
-
payload:
|
|
12302
|
+
payload: encodeErrorReply(code, text, { nodeId })
|
|
11649
12303
|
});
|
|
11650
12304
|
}
|
|
11651
12305
|
function createNotReadyHandler(nodeId) {
|
|
@@ -11714,6 +12368,34 @@ function createSingleResponseRespond(respond) {
|
|
|
11714
12368
|
}
|
|
11715
12369
|
|
|
11716
12370
|
// src/distribution/cluster-node/validate-config.ts
|
|
12371
|
+
function validateMemoryConfig(memory) {
|
|
12372
|
+
if (typeof memory !== "object" || memory === null) {
|
|
12373
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.memory must be an object", { memory });
|
|
12374
|
+
}
|
|
12375
|
+
const { blockWritesAbove, resumeWritesBelow, inFlightWriteFraction } = memory;
|
|
12376
|
+
for (const [name, value] of [
|
|
12377
|
+
["blockWritesAbove", blockWritesAbove],
|
|
12378
|
+
["resumeWritesBelow", resumeWritesBelow],
|
|
12379
|
+
["inFlightWriteFraction", inFlightWriteFraction]
|
|
12380
|
+
]) {
|
|
12381
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1)) {
|
|
12382
|
+
throw new NarsilError(
|
|
12383
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12384
|
+
`ClusterNodeConfig.memory.${name} must be a number above 0 and at most 1`,
|
|
12385
|
+
{ [name]: value }
|
|
12386
|
+
);
|
|
12387
|
+
}
|
|
12388
|
+
}
|
|
12389
|
+
const block = typeof blockWritesAbove === "number" ? blockWritesAbove : DEFAULT_BLOCK_WRITES_ABOVE;
|
|
12390
|
+
const resume = typeof resumeWritesBelow === "number" ? resumeWritesBelow : DEFAULT_RESUME_WRITES_BELOW;
|
|
12391
|
+
if (resume >= block) {
|
|
12392
|
+
throw new NarsilError(
|
|
12393
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12394
|
+
`ClusterNodeConfig.memory.resumeWritesBelow (${resume}) must lie below blockWritesAbove (${block})`,
|
|
12395
|
+
{ blockWritesAbove: block, resumeWritesBelow: resume }
|
|
12396
|
+
);
|
|
12397
|
+
}
|
|
12398
|
+
}
|
|
11717
12399
|
function validateClusterNodeConfig(config) {
|
|
11718
12400
|
if (config.address.length === 0) {
|
|
11719
12401
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
|
|
@@ -11776,6 +12458,9 @@ function validateClusterNodeConfig(config) {
|
|
|
11776
12458
|
);
|
|
11777
12459
|
}
|
|
11778
12460
|
}
|
|
12461
|
+
if (config.memory !== void 0) {
|
|
12462
|
+
validateMemoryConfig(config.memory);
|
|
12463
|
+
}
|
|
11779
12464
|
if (config.controller !== void 0) {
|
|
11780
12465
|
const { leaseTtlMs, standbyRetryMs } = config.controller;
|
|
11781
12466
|
if (leaseTtlMs !== void 0 && (!Number.isInteger(leaseTtlMs) || leaseTtlMs < MIN_CONTROLLER_LEASE_TTL_MS)) {
|
|
@@ -11825,9 +12510,6 @@ async function createClusterNode(config) {
|
|
|
11825
12510
|
await engine.shutdown();
|
|
11826
12511
|
throw err;
|
|
11827
12512
|
});
|
|
11828
|
-
function replicationLogKey2(indexName, partitionId) {
|
|
11829
|
-
return replicationLogKey(indexName, partitionId);
|
|
11830
|
-
}
|
|
11831
12513
|
function getReplicationLog2(indexName, partitionId) {
|
|
11832
12514
|
return getReplicationLog(replicationLogs, indexName, partitionId, config.replication);
|
|
11833
12515
|
}
|
|
@@ -11847,6 +12529,13 @@ async function createClusterNode(config) {
|
|
|
11847
12529
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11848
12530
|
version: SPEC_VERSION
|
|
11849
12531
|
};
|
|
12532
|
+
const memoryReporting = createNodeMemoryReporting({
|
|
12533
|
+
registration,
|
|
12534
|
+
engine,
|
|
12535
|
+
coordinator: config.coordinator,
|
|
12536
|
+
config: config.memory,
|
|
12537
|
+
onFullChange: () => lifecycle.renewRegistration()
|
|
12538
|
+
});
|
|
11850
12539
|
const hasDataRole = roles.includes("data");
|
|
11851
12540
|
const hasControllerRole = roles.includes("controller");
|
|
11852
12541
|
const controllerTransport = hasDataRole ? createMultiplexedControllerTransport(config.transport, nodeId) : null;
|
|
@@ -11860,7 +12549,9 @@ async function createClusterNode(config) {
|
|
|
11860
12549
|
partitionWriteQueues: createPartitionWriteQueues(),
|
|
11861
12550
|
catchUp: createCatchUpState(),
|
|
11862
12551
|
resolveNodeTargets: resolveNodeTargets3,
|
|
11863
|
-
waitForActiveReplicas: config.replication?.waitForActiveReplicas
|
|
12552
|
+
waitForActiveReplicas: config.replication?.waitForActiveReplicas,
|
|
12553
|
+
memoryView: memoryReporting.view,
|
|
12554
|
+
writePressure: memoryReporting.writePressure
|
|
11864
12555
|
};
|
|
11865
12556
|
const orphanedIndexes = /* @__PURE__ */ new Set();
|
|
11866
12557
|
const heldPartitionWrites = /* @__PURE__ */ new Set();
|
|
@@ -11905,6 +12596,7 @@ async function createClusterNode(config) {
|
|
|
11905
12596
|
}
|
|
11906
12597
|
const lifecycle = createDataNodeLifecycle({
|
|
11907
12598
|
registration,
|
|
12599
|
+
currentRegistration: memoryReporting.currentRegistration,
|
|
11908
12600
|
coordinator: config.coordinator,
|
|
11909
12601
|
transport: config.transport,
|
|
11910
12602
|
knownIndexNames: [],
|
|
@@ -11920,7 +12612,7 @@ async function createClusterNode(config) {
|
|
|
11920
12612
|
retainedPartitionIds: (indexName) => engine.heldPartitionsOf(indexName) ?? [],
|
|
11921
12613
|
onRemovePartition: (indexName, partitionId) => {
|
|
11922
12614
|
clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
|
|
11923
|
-
replicationLogs.delete(
|
|
12615
|
+
replicationLogs.delete(replicationLogKey(indexName, partitionId));
|
|
11924
12616
|
trackHeldPartitionWrite(engine.forgetHeldPartition(indexName, partitionId));
|
|
11925
12617
|
void cleanupRemovedPartition(indexName, partitionId, {
|
|
11926
12618
|
engine,
|
|
@@ -12006,6 +12698,7 @@ async function createClusterNode(config) {
|
|
|
12006
12698
|
if (hasDataRole) {
|
|
12007
12699
|
unregisterNotReadyHandler = await config.transport.listen(createNotReadyHandler(nodeId));
|
|
12008
12700
|
}
|
|
12701
|
+
await memoryReporting.start();
|
|
12009
12702
|
await lifecycle.join();
|
|
12010
12703
|
if (controller !== null) {
|
|
12011
12704
|
await controller.start();
|
|
@@ -12044,6 +12737,7 @@ async function createClusterNode(config) {
|
|
|
12044
12737
|
unregisterNotReadyHandler = null;
|
|
12045
12738
|
}
|
|
12046
12739
|
await lifecycle.shutdown();
|
|
12740
|
+
memoryReporting.stop();
|
|
12047
12741
|
if (controller !== null) {
|
|
12048
12742
|
await controller.shutdown();
|
|
12049
12743
|
}
|