@delali/narsil 0.3.1-next.80 → 0.3.1-next.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/filesystem.mjs +1 -1
- package/dist/adapters/indexeddb.mjs +1 -1
- package/dist/{capabilities-CcTDWh_P.d.ts → capabilities-CAdkp2P-.d.ts} +2 -2
- package/dist/{chunk-HJASKX4F.mjs → chunk-2453RJYK.mjs} +2 -2
- package/dist/{chunk-K4FZ55Q2.mjs → chunk-4CWHWOBI.mjs} +10 -10
- package/dist/{chunk-4QDQEJDV.mjs → chunk-4D6OH4EF.mjs} +6 -6
- package/dist/{chunk-P2PYEZZE.mjs → chunk-4VBG6PEH.mjs} +3 -3
- package/dist/{chunk-Y3L5MDRV.mjs → chunk-5XNIRLFC.mjs} +2 -2
- package/dist/{chunk-2E35UM3G.mjs → chunk-B3UOQJD3.mjs} +1 -1
- package/dist/{chunk-MEOOYKWL.mjs → chunk-GNYXCLJS.mjs} +4 -4
- package/dist/{chunk-NSXQCE4K.mjs → chunk-IH3GKNDQ.mjs} +190 -40
- package/dist/{chunk-23HJ5QXQ.mjs → chunk-IN5CH4P2.mjs} +2 -2
- package/dist/{chunk-OSF3SGJV.mjs → chunk-KJ7LB3TR.mjs} +2 -2
- package/dist/{chunk-DQABW5UQ.mjs → chunk-LS5IRM23.mjs} +1 -1
- package/dist/{chunk-4MQ5F4IL.mjs → chunk-OLXM65YH.mjs} +1 -1
- package/dist/{chunk-R2XCJIO7.mjs → chunk-SREPGC6O.mjs} +2 -2
- package/dist/{chunk-I64LXKO3.mjs → chunk-TPJU7NDJ.mjs} +1 -0
- package/dist/{chunk-YGLSR7BC.mjs → chunk-XYDP7CCJ.mjs} +5 -5
- package/dist/{chunk-QPYV22XH.mjs → chunk-XZF4LZ3X.mjs} +4 -3
- package/dist/{chunk-GG2FBGVZ.mjs → chunk-ZCKPRCYI.mjs} +1 -1
- package/dist/{client-CsybN673.d.ts → client-CKUtkLz3.d.ts} +2 -2
- package/dist/client.d.ts +7 -7
- package/dist/client.mjs +1 -0
- package/dist/{config-CFonR_gY.d.ts → config-BuIkeM5C.d.ts} +39 -3
- package/dist/{config-BCNQtszf.d.ts → config-CSDEPraV.d.ts} +1 -1
- package/dist/direct-executor-JVXDR5Y5.mjs +19 -0
- package/dist/distribution/coordinator/etcd.d.ts +1 -1
- package/dist/distribution/coordinator/etcd.mjs +1 -1
- package/dist/distribution/coordinator/in-memory.d.ts +1 -1
- package/dist/distribution/transport/grpc.mjs +1 -1
- package/dist/distribution.d.ts +7 -7
- package/dist/distribution.mjs +756 -64
- package/dist/embeddings/openai.mjs +1 -1
- package/dist/{engine-B0oIF1_r.d.ts → engine-DMNZJAUq.d.ts} +1 -1
- package/dist/{types-ChrVy4lH.d.ts → index-DzONZUKE.d.ts} +79 -37
- package/dist/index.browser.mjs +25 -23
- package/dist/index.d.ts +5 -5
- package/dist/index.mjs +14 -14
- package/dist/{memory-B6SBOKlS.d.ts → memory-BPKNx9Tl.d.ts} +1 -0
- package/dist/partial-words.mjs +4 -4
- package/dist/pattern.mjs +3 -3
- package/dist/persistence/durability/checkpoint-worker.mjs +14 -14
- package/dist/react.d.ts +6 -6
- package/dist/react.mjs +1 -0
- package/dist/server.d.ts +9 -9
- package/dist/server.mjs +18 -13
- package/dist/{tasks-CYAQe6qd.d.ts → tasks-DGyz586z.d.ts} +1 -1
- package/dist/{thread-P6QJ2Q5O.mjs → thread-33M5AOEI.mjs} +10 -10
- package/dist/vector/search-worker.mjs +3 -3
- package/dist/workers/entry.mjs +5 -5
- package/package.json +11 -7
- package/dist/direct-executor-VNMQ6NIH.mjs +0 -19
package/dist/distribution.mjs
CHANGED
|
@@ -1,39 +1,40 @@
|
|
|
1
1
|
import { MAX_MESSAGE_SIZE_BYTES } from './chunk-UNEEFTEU.mjs';
|
|
2
|
-
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-
|
|
3
|
-
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-
|
|
4
|
-
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-
|
|
2
|
+
import { validateIndexName, indexConfigKey, withPartitionCount, indexSettingsOf, putIndexMetadata, getIndexMetadata, getClusterIndexConfig } from './chunk-GNYXCLJS.mjs';
|
|
3
|
+
import { requireKnownConfig, resolveRunnableConfig, createEngineCore, startEngineCore, createNarsilFromCore, runEngineQueryStats, runEngineListDocuments, runEngineSuggest, runEnginePreflight, runEngineQuery, createEngineIndex, buildArchiveFromMemory, indexMetadataOf, encodeIndexConfiguration, archiveStream, applyIndexEntry, applyDeleteEntry, requireConfigurationWithinSizeLimit, validateEntryBatchPayload, createAckMessage, validateEntryPayload, validateForwardBatchPayload, createForwardBatchResultMessage, memoryCheckpointStore, readSnapshotArchive, createByteReader, openChecksummedEnvelope, loadPartitionSegments, readListedFile, createForwardBatchMessage, validateForwardBatchResultPayload, createForwardMessage, validateReplicationEntry, crc32Init, crc32Final, crc32Update, computeEntryChecksum, createEntryMessage, createEntryBatchMessage, createInsyncConfirmMessage, validateAckPayload, createInsyncRemoveMessage, validateInsyncConfirmPayload, createInsyncAddMessage } from './chunk-IH3GKNDQ.mjs';
|
|
4
|
+
import { now, listBindingOf, requireSurfaceFormsForPartialFields, queryBindingOf, hitsKeptPerGroup, clampLimit, resolveHybridFusion, dedupePinnedEntries, foldGroupReducer } from './chunk-4D6OH4EF.mjs';
|
|
5
5
|
import { ReplicationMessageTypes, TransportErrorCodes, ClusterMessageTypes, QueryMessageTypes, TransportError } from './chunk-GCM3JFVA.mjs';
|
|
6
|
-
import {
|
|
7
|
-
import { generateId, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-
|
|
8
|
-
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-
|
|
9
|
-
import { getLanguage } from './chunk-
|
|
10
|
-
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-
|
|
6
|
+
import { MAX_PARTITION_COUNT, MAX_REPLICATION_FACTOR, WIRE_BATCH_BUDGET, providedDocId } from './chunk-2453RJYK.mjs';
|
|
7
|
+
import { generateId, readHeapStatistics, decodeVectorFilePayload, bytesToVectors, isDead } from './chunk-4CWHWOBI.mjs';
|
|
8
|
+
import { oversampledShardSize, resolveProjection, clampRowCount, sortSignatureOf, requireMatchingCursor, decodePageCursor, normalizeSort, sortModesOf, encodePageCursor, applyProjection, readSortValues, requireWithinResultWindow, fnv1a, sortSignatureEntry, FILTER_EXPRESSION_KEYS, filterTestCount, fnv1aUint32s, mergePartitionStats, FIELD_FILTER_OPERATORS } from './chunk-XYDP7CCJ.mjs';
|
|
9
|
+
import { getLanguage } from './chunk-OLXM65YH.mjs';
|
|
10
|
+
import { DEFAULT_SUGGEST_LIMIT, MAX_SUGGEST_LIMIT, MAX_SUGGEST_SCATTER_LIMIT, MAX_INDEX_NAME_LENGTH, MAX_DOC_ID_LENGTH, SUGGEST_OVERSAMPLE_FACTOR, SUGGEST_OVERSAMPLE_PADDING, INDEX_NAME_PATTERN } from './chunk-ZCKPRCYI.mjs';
|
|
11
11
|
import './chunk-TGP5M5R7.mjs';
|
|
12
|
-
import './chunk-
|
|
12
|
+
import './chunk-IN5CH4P2.mjs';
|
|
13
13
|
import { MIN_OVERSAMPLE } from './chunk-ZJ4CSAB2.mjs';
|
|
14
|
-
import './chunk-
|
|
15
|
-
import { isPatternTolerance } from './chunk-
|
|
14
|
+
import './chunk-B3UOQJD3.mjs';
|
|
15
|
+
import { isPatternTolerance } from './chunk-SREPGC6O.mjs';
|
|
16
16
|
import { MAX_FILTER_DEPTH, MAX_FILTER_TESTS, MAX_FILTER_ARRAY_SIZE } from './chunk-WQBCBKY3.mjs';
|
|
17
|
-
import './chunk-
|
|
18
|
-
import { validateSchema, requireCreatableSchema, DEFAULT_PAGE_SIZE, encodeDocument, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-
|
|
17
|
+
import './chunk-4VBG6PEH.mjs';
|
|
18
|
+
import { validateSchema, requireCreatableSchema, DEFAULT_PAGE_SIZE, encodeDocument, MAX_SORT_FIELDS, MAX_CURSOR_LENGTH, MAX_TOLERANCE, MAX_PREFIX_LENGTH, RESULT_WINDOW, MIN_POLYGON_POINTS, isLatitude, MAX_LATITUDE_DEGREES, isLongitude, MAX_LONGITUDE_DEGREES } from './chunk-KJ7LB3TR.mjs';
|
|
19
19
|
import './chunk-KEHK2O6P.mjs';
|
|
20
20
|
import './chunk-GES7U2PM.mjs';
|
|
21
|
-
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-
|
|
21
|
+
import { MAX_STORED_VECTOR_DIMENSION, MAX_NESTING_DEPTH } from './chunk-LS5IRM23.mjs';
|
|
22
22
|
import './chunk-U2BKYEC7.mjs';
|
|
23
23
|
import { toComparableSortValue, compareComparableKeys, isSortMode, SORT_MODES, compareSortValues } from './chunk-R6A773WK.mjs';
|
|
24
24
|
import { compareCodePoints, exceedsCodePoints } from './chunk-YTJCGKM4.mjs';
|
|
25
|
+
import { collectAndReturnPages } from './chunk-5XNIRLFC.mjs';
|
|
25
26
|
import './chunk-POWOTUKJ.mjs';
|
|
26
|
-
import { ErrorCodes, NarsilError, describeError } from './chunk-
|
|
27
|
-
import {
|
|
27
|
+
import { ErrorCodes, NarsilError, describeError } from './chunk-TPJU7NDJ.mjs';
|
|
28
|
+
import { decode, encode } from '@msgpack/msgpack';
|
|
28
29
|
import { randomUUID } from 'crypto';
|
|
29
30
|
import { openSync, writeSync, closeSync, rmSync, readSync } from 'fs';
|
|
30
31
|
import { mkdtemp, rm, mkdir } from 'fs/promises';
|
|
31
|
-
import { tmpdir } from 'os';
|
|
32
|
+
import { tmpdir, totalmem } from 'os';
|
|
32
33
|
import { join } from 'path';
|
|
33
34
|
|
|
34
35
|
// src/distribution/cluster/constants.ts
|
|
35
36
|
var LEADERSHIP_IMBALANCE_THRESHOLD = 2;
|
|
36
|
-
var
|
|
37
|
+
var ESTIMATED_PARTITION_BYTES = 50 * 1024 * 1024;
|
|
37
38
|
var REBALANCE_THRESHOLD = 0.1;
|
|
38
39
|
var BOOTSTRAP_CAS_ATTEMPTS = 5;
|
|
39
40
|
var ALLOCATION_CAS_ATTEMPTS = 5;
|
|
@@ -134,22 +135,87 @@ var balanceDecider = {
|
|
|
134
135
|
}
|
|
135
136
|
};
|
|
136
137
|
|
|
138
|
+
// src/distribution/cluster/allocator/partition-sizes.ts
|
|
139
|
+
function isByteCount(value) {
|
|
140
|
+
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
141
|
+
}
|
|
142
|
+
function reportedEntriesOf(node) {
|
|
143
|
+
const reported = node.partitionBytes;
|
|
144
|
+
if (!Array.isArray(reported)) {
|
|
145
|
+
return [];
|
|
146
|
+
}
|
|
147
|
+
const entries = [];
|
|
148
|
+
for (const index of reported) {
|
|
149
|
+
if (index === null || typeof index !== "object" || typeof index.indexName !== "string") {
|
|
150
|
+
continue;
|
|
151
|
+
}
|
|
152
|
+
if (!Array.isArray(index.partitions)) {
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
for (const pair of index.partitions) {
|
|
156
|
+
if (!Array.isArray(pair) || !Number.isSafeInteger(pair[0]) || pair[0] < 0 || !isByteCount(pair[1])) {
|
|
157
|
+
continue;
|
|
158
|
+
}
|
|
159
|
+
entries.push({ indexName: index.indexName, partitionId: pair[0], bytes: pair[1] });
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
return entries;
|
|
163
|
+
}
|
|
164
|
+
function readPartitionSizes(nodes, indexName) {
|
|
165
|
+
const largestByPartition = /* @__PURE__ */ new Map();
|
|
166
|
+
const reportedByNode = /* @__PURE__ */ new Map();
|
|
167
|
+
const reportedBytesByNode = /* @__PURE__ */ new Map();
|
|
168
|
+
for (const node of nodes) {
|
|
169
|
+
const reportedHere = /* @__PURE__ */ new Set();
|
|
170
|
+
let reportedBytes = 0;
|
|
171
|
+
for (const entry of reportedEntriesOf(node)) {
|
|
172
|
+
reportedBytes += entry.bytes;
|
|
173
|
+
if (entry.indexName !== indexName) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
reportedHere.add(entry.partitionId);
|
|
177
|
+
largestByPartition.set(entry.partitionId, Math.max(largestByPartition.get(entry.partitionId) ?? 0, entry.bytes));
|
|
178
|
+
}
|
|
179
|
+
reportedByNode.set(node.nodeId, reportedHere);
|
|
180
|
+
reportedBytesByNode.set(node.nodeId, reportedBytes);
|
|
181
|
+
}
|
|
182
|
+
function sizeOf(partitionId) {
|
|
183
|
+
return largestByPartition.get(partitionId) ?? ESTIMATED_PARTITION_BYTES;
|
|
184
|
+
}
|
|
185
|
+
function reports(nodeId, partitionId) {
|
|
186
|
+
return reportedByNode.get(nodeId)?.has(partitionId) ?? false;
|
|
187
|
+
}
|
|
188
|
+
return {
|
|
189
|
+
sizeOf,
|
|
190
|
+
reports,
|
|
191
|
+
heldBytesOf(nodeId, assignments) {
|
|
192
|
+
let heldBytes = reportedBytesByNode.get(nodeId) ?? 0;
|
|
193
|
+
for (const [partitionId, assignment] of assignments) {
|
|
194
|
+
const holdsCopy = assignment.primary === nodeId || assignment.replicas.includes(nodeId);
|
|
195
|
+
if (holdsCopy && !reports(nodeId, partitionId)) {
|
|
196
|
+
heldBytes += sizeOf(partitionId);
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
return heldBytes;
|
|
200
|
+
}
|
|
201
|
+
};
|
|
202
|
+
}
|
|
203
|
+
|
|
137
204
|
// src/distribution/cluster/allocator/deciders/capacity.ts
|
|
138
|
-
function createCapacityDecider(
|
|
139
|
-
const bytesPerPartition = DEFAULT_ESTIMATED_PARTITION_BYTES;
|
|
205
|
+
function createCapacityDecider(sizes = readPartitionSizes([], "")) {
|
|
140
206
|
return {
|
|
141
207
|
name: "capacity",
|
|
142
208
|
canAllocate(context) {
|
|
143
|
-
const { candidateNodeId,
|
|
209
|
+
const { candidateNodeId, partitionId, allAssignments, nodes } = context;
|
|
144
210
|
const node = nodes.get(candidateNodeId);
|
|
145
211
|
if (node === void 0) {
|
|
146
212
|
return "NO";
|
|
147
213
|
}
|
|
148
|
-
if (node.capacity.memoryBytes === 0) {
|
|
214
|
+
if (node.capacity.memoryBytes === 0 || node.memory?.full === true) {
|
|
149
215
|
return "NO";
|
|
150
216
|
}
|
|
151
|
-
const
|
|
152
|
-
const projectedBytes = (
|
|
217
|
+
const addedBytes = sizes.reports(candidateNodeId, partitionId) ? 0 : sizes.sizeOf(partitionId);
|
|
218
|
+
const projectedBytes = sizes.heldBytesOf(candidateNodeId, allAssignments) + addedBytes;
|
|
153
219
|
if (projectedBytes > node.capacity.memoryBytes) {
|
|
154
220
|
return "NO";
|
|
155
221
|
}
|
|
@@ -257,11 +323,11 @@ function createZoneDecider(zoneAttribute) {
|
|
|
257
323
|
}
|
|
258
324
|
|
|
259
325
|
// src/distribution/cluster/allocator/deciders/index.ts
|
|
260
|
-
function createDeciderChain(constraints,
|
|
326
|
+
function createDeciderChain(constraints, partitionSizes) {
|
|
261
327
|
const deciders = [
|
|
262
328
|
colocationDecider,
|
|
263
329
|
createMaxShardsDecider(constraints.maxShardsPerNode),
|
|
264
|
-
createCapacityDecider()
|
|
330
|
+
createCapacityDecider(partitionSizes)
|
|
265
331
|
];
|
|
266
332
|
if (constraints.zoneAwareness) {
|
|
267
333
|
deciders.push(createZoneDecider(constraints.zoneAttribute));
|
|
@@ -791,7 +857,7 @@ function allocate(nodes, currentTable, indexName, partitionCount, replicationFac
|
|
|
791
857
|
totalNodes: nodes.length
|
|
792
858
|
});
|
|
793
859
|
}
|
|
794
|
-
const deciders = createDeciderChain(constraints);
|
|
860
|
+
const deciders = createDeciderChain(constraints, readPartitionSizes(nodes, indexName));
|
|
795
861
|
if (currentTable === null) {
|
|
796
862
|
return initialAllocate(dataNodes, indexName, partitionCount, replicationFactor, constraints, deciders);
|
|
797
863
|
}
|
|
@@ -1622,6 +1688,7 @@ function handleQueuedBootstrapComplete(state, message, respond, coordinator, nod
|
|
|
1622
1688
|
function createEventLoopState(initialIndexNames) {
|
|
1623
1689
|
return {
|
|
1624
1690
|
unwatchNodes: null,
|
|
1691
|
+
placementByNodeId: /* @__PURE__ */ new Map(),
|
|
1625
1692
|
unwatchSchemas: null,
|
|
1626
1693
|
unwatchTransport: null,
|
|
1627
1694
|
knownIndexes: new Set(initialIndexNames),
|
|
@@ -1637,6 +1704,7 @@ function clearEventLoopWatchers(state) {
|
|
|
1637
1704
|
state.unwatchNodes();
|
|
1638
1705
|
state.unwatchNodes = null;
|
|
1639
1706
|
}
|
|
1707
|
+
state.placementByNodeId.clear();
|
|
1640
1708
|
if (state.unwatchSchemas !== null) {
|
|
1641
1709
|
state.unwatchSchemas();
|
|
1642
1710
|
state.unwatchSchemas = null;
|
|
@@ -1654,6 +1722,22 @@ function clearEventLoopWatchers(state) {
|
|
|
1654
1722
|
}
|
|
1655
1723
|
state.teardownTimers.clear();
|
|
1656
1724
|
}
|
|
1725
|
+
function placementOf(registration) {
|
|
1726
|
+
const { memory: _memory, partitionBytes: _partitionBytes, ...placement } = registration;
|
|
1727
|
+
return JSON.stringify(placement);
|
|
1728
|
+
}
|
|
1729
|
+
function changesTopology(state, event) {
|
|
1730
|
+
if (event.type === "node_left" || event.registration === null) {
|
|
1731
|
+
state.placementByNodeId.delete(event.nodeId);
|
|
1732
|
+
return true;
|
|
1733
|
+
}
|
|
1734
|
+
const placement = placementOf(event.registration);
|
|
1735
|
+
if (state.placementByNodeId.get(event.nodeId) === placement) {
|
|
1736
|
+
return false;
|
|
1737
|
+
}
|
|
1738
|
+
state.placementByNodeId.set(event.nodeId, placement);
|
|
1739
|
+
return true;
|
|
1740
|
+
}
|
|
1657
1741
|
|
|
1658
1742
|
// src/distribution/cluster/controller/event-loop/index.ts
|
|
1659
1743
|
async function startEventLoop(state, coordinator, transport, nodeId, isActive, onError) {
|
|
@@ -1661,8 +1745,10 @@ async function startEventLoop(state, coordinator, transport, nodeId, isActive, o
|
|
|
1661
1745
|
state.transport = transport;
|
|
1662
1746
|
state.controllerNodeId = nodeId;
|
|
1663
1747
|
try {
|
|
1664
|
-
state.unwatchNodes = await coordinator.watchNodes((
|
|
1665
|
-
|
|
1748
|
+
state.unwatchNodes = await coordinator.watchNodes((event) => {
|
|
1749
|
+
if (changesTopology(state, event)) {
|
|
1750
|
+
scheduleDebouncedAllocation(state, coordinator, isActive, onError);
|
|
1751
|
+
}
|
|
1666
1752
|
});
|
|
1667
1753
|
state.unwatchSchemas = await coordinator.watchSchemas((event) => {
|
|
1668
1754
|
handleSchemaEvent(event, coordinator, state, isActive, onError).catch((error) => {
|
|
@@ -2371,17 +2457,43 @@ function clearHeartbeatTimer(state) {
|
|
|
2371
2457
|
state.timer = null;
|
|
2372
2458
|
}
|
|
2373
2459
|
}
|
|
2460
|
+
function registrationToWrite(config) {
|
|
2461
|
+
return config.currentRegistration?.() ?? config.registration;
|
|
2462
|
+
}
|
|
2463
|
+
function refusedForSize(error) {
|
|
2464
|
+
return error instanceof NarsilError && error.code === ErrorCodes.CONFIG_INVALID;
|
|
2465
|
+
}
|
|
2466
|
+
async function writeRegistration(config) {
|
|
2467
|
+
const registration = registrationToWrite(config);
|
|
2468
|
+
try {
|
|
2469
|
+
await config.coordinator.registerNode(registration);
|
|
2470
|
+
} catch (error) {
|
|
2471
|
+
if (registration.partitionBytes === void 0 || !refusedForSize(error)) {
|
|
2472
|
+
throw error;
|
|
2473
|
+
}
|
|
2474
|
+
const { partitionBytes: _, ...withoutPartitionBytes } = registration;
|
|
2475
|
+
await config.coordinator.registerNode(withoutPartitionBytes);
|
|
2476
|
+
throw error;
|
|
2477
|
+
}
|
|
2478
|
+
}
|
|
2479
|
+
function queueRegistrationWrite(state, config) {
|
|
2480
|
+
state.inFlight = state.inFlight.then(() => writeRegistration(config)).catch((error) => {
|
|
2481
|
+
if (config.onError !== void 0) {
|
|
2482
|
+
config.onError(error);
|
|
2483
|
+
}
|
|
2484
|
+
});
|
|
2485
|
+
}
|
|
2374
2486
|
function startRegistrationHeartbeat(state, config) {
|
|
2375
2487
|
clearHeartbeatTimer(state);
|
|
2376
|
-
state.timer = setInterval(() =>
|
|
2377
|
-
state.inFlight = state.inFlight.then(() => config.coordinator.registerNode(config.registration)).catch((error) => {
|
|
2378
|
-
if (config.onError !== void 0) {
|
|
2379
|
-
config.onError(error);
|
|
2380
|
-
}
|
|
2381
|
-
});
|
|
2382
|
-
}, config.nodeHeartbeatIntervalMs);
|
|
2488
|
+
state.timer = setInterval(() => queueRegistrationWrite(state, config), config.nodeHeartbeatIntervalMs);
|
|
2383
2489
|
state.timer.unref?.();
|
|
2384
2490
|
}
|
|
2491
|
+
function renewRegistrationNow(state, config) {
|
|
2492
|
+
if (state.timer === null) {
|
|
2493
|
+
return;
|
|
2494
|
+
}
|
|
2495
|
+
queueRegistrationWrite(state, config);
|
|
2496
|
+
}
|
|
2385
2497
|
async function stopRegistrationHeartbeat(state) {
|
|
2386
2498
|
clearHeartbeatTimer(state);
|
|
2387
2499
|
await state.inFlight;
|
|
@@ -2389,7 +2501,7 @@ async function stopRegistrationHeartbeat(state) {
|
|
|
2389
2501
|
|
|
2390
2502
|
// src/distribution/cluster/node-lifecycle/join.ts
|
|
2391
2503
|
async function joinCluster(config, watcherState, heartbeatState) {
|
|
2392
|
-
await config.coordinator.registerNode(config
|
|
2504
|
+
await config.coordinator.registerNode(registrationToWrite(config));
|
|
2393
2505
|
config.onRegistered?.();
|
|
2394
2506
|
startRegistrationHeartbeat(heartbeatState, config);
|
|
2395
2507
|
const initialTables = await loadInitialAllocations(config);
|
|
@@ -2485,6 +2597,11 @@ function createDataNodeLifecycle(config) {
|
|
|
2485
2597
|
}
|
|
2486
2598
|
});
|
|
2487
2599
|
},
|
|
2600
|
+
renewRegistration() {
|
|
2601
|
+
if (status === "active" || status === "joining") {
|
|
2602
|
+
renewRegistrationNow(heartbeatState, config);
|
|
2603
|
+
}
|
|
2604
|
+
},
|
|
2488
2605
|
leave() {
|
|
2489
2606
|
return withLock(async () => {
|
|
2490
2607
|
if (status !== "active") {
|
|
@@ -3087,6 +3204,18 @@ var MAX_SNAPSHOT_SYNC_REQUEST_BYTES = 4096;
|
|
|
3087
3204
|
var MAX_SOURCE_ID_LENGTH = 256;
|
|
3088
3205
|
var MAX_WAIT_FOR_ACTIVE_REPLICAS = 255;
|
|
3089
3206
|
var MIN_CONTROLLER_LEASE_TTL_MS = 1e3;
|
|
3207
|
+
var DEFAULT_BLOCK_WRITES_ABOVE = 0.9;
|
|
3208
|
+
var DEFAULT_RESUME_WRITES_BELOW = 0.8;
|
|
3209
|
+
var MEMORY_SAMPLE_INTERVAL_MS = 1e3;
|
|
3210
|
+
var FULL_MEMORY_COLLECTION_INTERVAL_MS = 5e3;
|
|
3211
|
+
var PARTITION_BYTES_REPORT_BUDGET_BYTES = 262144;
|
|
3212
|
+
var PARTITION_BYTES_ENTRY_BYTES = 16;
|
|
3213
|
+
var PARTITION_BYTES_INDEX_OVERHEAD_BYTES = 32;
|
|
3214
|
+
var DEFAULT_IN_FLIGHT_WRITE_FRACTION = 0.02;
|
|
3215
|
+
var WRITE_MEMORY_SAMPLE_MAX_AGE_MS = 100;
|
|
3216
|
+
var DOCUMENT_NUMBER_BYTES = 8;
|
|
3217
|
+
var DOCUMENT_SCALAR_BYTES = 4;
|
|
3218
|
+
var DOCUMENT_ENTRY_OVERHEAD_BYTES = 2;
|
|
3090
3219
|
|
|
3091
3220
|
// src/distribution/cluster-node/bootstrap-fetch.ts
|
|
3092
3221
|
var RETRY_ANY_TARGET_CODES = /* @__PURE__ */ new Set([
|
|
@@ -4606,7 +4735,7 @@ async function resolvePrimaryAssignment(indexName, docId, deps, requireLocalPrim
|
|
|
4606
4735
|
{ indexName, partitionId, primaryNodeId: assignment.primary, localNodeId: deps.nodeId }
|
|
4607
4736
|
);
|
|
4608
4737
|
}
|
|
4609
|
-
return { partitionId, assignment };
|
|
4738
|
+
return { table, partitionId, assignment };
|
|
4610
4739
|
}
|
|
4611
4740
|
function assertSufficientActiveReplicas(indexName, partitionId, assignment, deps) {
|
|
4612
4741
|
const requiredActiveCopies = deps.waitForActiveReplicas ?? 1;
|
|
@@ -5763,6 +5892,443 @@ function report(deps, indexName, reason) {
|
|
|
5763
5892
|
)
|
|
5764
5893
|
);
|
|
5765
5894
|
}
|
|
5895
|
+
function processLimitBytes() {
|
|
5896
|
+
const hostBytes = totalmem();
|
|
5897
|
+
const constrained = typeof process.constrainedMemory === "function" ? process.constrainedMemory() : 0;
|
|
5898
|
+
return Number.isFinite(constrained) && constrained > 0 && constrained < hostBytes ? constrained : hostBytes;
|
|
5899
|
+
}
|
|
5900
|
+
function readMemoryFigures() {
|
|
5901
|
+
try {
|
|
5902
|
+
const heap = readHeapStatistics();
|
|
5903
|
+
const processBytes = process.memoryUsage.rss();
|
|
5904
|
+
const limitBytes = processLimitBytes();
|
|
5905
|
+
if (heap === null || !Number.isFinite(processBytes) || !Number.isFinite(limitBytes) || limitBytes <= 0) {
|
|
5906
|
+
return null;
|
|
5907
|
+
}
|
|
5908
|
+
return {
|
|
5909
|
+
processBytes,
|
|
5910
|
+
processLimitBytes: limitBytes,
|
|
5911
|
+
heapBytes: heap.usedBytes,
|
|
5912
|
+
heapLimitBytes: heap.configuredLimitBytes ?? heap.limitBytes
|
|
5913
|
+
};
|
|
5914
|
+
} catch {
|
|
5915
|
+
return null;
|
|
5916
|
+
}
|
|
5917
|
+
}
|
|
5918
|
+
|
|
5919
|
+
// src/distribution/cluster-node/memory/monitor.ts
|
|
5920
|
+
function fractionOf(usedBytes, limitBytes) {
|
|
5921
|
+
return limitBytes > 0 ? usedBytes / limitBytes : 0;
|
|
5922
|
+
}
|
|
5923
|
+
function nextFullState(wasFull, figures, options) {
|
|
5924
|
+
const processFraction = fractionOf(figures.processBytes, figures.processLimitBytes);
|
|
5925
|
+
const heapFraction = fractionOf(figures.heapBytes, figures.heapLimitBytes);
|
|
5926
|
+
if (!wasFull) {
|
|
5927
|
+
return processFraction > options.blockWritesAbove || heapFraction > options.blockWritesAbove;
|
|
5928
|
+
}
|
|
5929
|
+
return processFraction >= options.resumeWritesBelow || heapFraction >= options.resumeWritesBelow;
|
|
5930
|
+
}
|
|
5931
|
+
function createMemoryMonitor(options) {
|
|
5932
|
+
let latest = null;
|
|
5933
|
+
let timer = null;
|
|
5934
|
+
let lastCollectionAt = Number.NEGATIVE_INFINITY;
|
|
5935
|
+
let lastSampleAt = Number.NEGATIVE_INFINITY;
|
|
5936
|
+
function figuresWithoutGarbage(figures) {
|
|
5937
|
+
const now3 = Date.now();
|
|
5938
|
+
if (now3 - lastCollectionAt < options.collectionIntervalMs) {
|
|
5939
|
+
return figures;
|
|
5940
|
+
}
|
|
5941
|
+
lastCollectionAt = now3;
|
|
5942
|
+
if (!options.collectGarbage()) {
|
|
5943
|
+
return figures;
|
|
5944
|
+
}
|
|
5945
|
+
return options.read() ?? figures;
|
|
5946
|
+
}
|
|
5947
|
+
function sample() {
|
|
5948
|
+
lastSampleAt = Date.now();
|
|
5949
|
+
let figures = options.read();
|
|
5950
|
+
if (figures === null) {
|
|
5951
|
+
return;
|
|
5952
|
+
}
|
|
5953
|
+
const wasFull = latest?.full ?? false;
|
|
5954
|
+
let full = nextFullState(wasFull, figures, options);
|
|
5955
|
+
if (full) {
|
|
5956
|
+
figures = figuresWithoutGarbage(figures);
|
|
5957
|
+
full = nextFullState(wasFull, figures, options);
|
|
5958
|
+
}
|
|
5959
|
+
latest = { ...figures, full };
|
|
5960
|
+
if (full !== wasFull) {
|
|
5961
|
+
options.onFullChange(full);
|
|
5962
|
+
}
|
|
5963
|
+
}
|
|
5964
|
+
return {
|
|
5965
|
+
report: () => latest,
|
|
5966
|
+
isFull: () => latest?.full ?? false,
|
|
5967
|
+
sample,
|
|
5968
|
+
sampleIfOlderThan(maxAgeMs) {
|
|
5969
|
+
if (timer !== null && Date.now() - lastSampleAt >= maxAgeMs) {
|
|
5970
|
+
sample();
|
|
5971
|
+
}
|
|
5972
|
+
},
|
|
5973
|
+
start() {
|
|
5974
|
+
if (timer !== null) {
|
|
5975
|
+
return;
|
|
5976
|
+
}
|
|
5977
|
+
sample();
|
|
5978
|
+
timer = setInterval(sample, options.intervalMs);
|
|
5979
|
+
timer.unref?.();
|
|
5980
|
+
},
|
|
5981
|
+
stop() {
|
|
5982
|
+
if (timer !== null) {
|
|
5983
|
+
clearInterval(timer);
|
|
5984
|
+
timer = null;
|
|
5985
|
+
}
|
|
5986
|
+
}
|
|
5987
|
+
};
|
|
5988
|
+
}
|
|
5989
|
+
|
|
5990
|
+
// src/distribution/cluster-node/memory/partition-bytes.ts
|
|
5991
|
+
function heldPartitionSizes(engine, indexName) {
|
|
5992
|
+
const held = new Set(engine.heldPartitionsOf(indexName) ?? []);
|
|
5993
|
+
if (held.size === 0) {
|
|
5994
|
+
return [];
|
|
5995
|
+
}
|
|
5996
|
+
const partitions = engine.getPartitionStats(indexName);
|
|
5997
|
+
const indexBytes = engine.getStats(indexName).estimatedMemoryBytes;
|
|
5998
|
+
let partitionBytes = 0;
|
|
5999
|
+
let documents = 0;
|
|
6000
|
+
for (const partition of partitions) {
|
|
6001
|
+
partitionBytes += partition.estimatedMemoryBytes;
|
|
6002
|
+
documents += partition.documentCount;
|
|
6003
|
+
}
|
|
6004
|
+
const sharedBytes = Math.max(0, indexBytes - partitionBytes);
|
|
6005
|
+
const sizes = [];
|
|
6006
|
+
for (const partition of partitions) {
|
|
6007
|
+
if (!held.has(partition.partitionId)) {
|
|
6008
|
+
continue;
|
|
6009
|
+
}
|
|
6010
|
+
const share = documents > 0 ? sharedBytes * partition.documentCount / documents : 0;
|
|
6011
|
+
sizes.push({
|
|
6012
|
+
indexName,
|
|
6013
|
+
partitionId: partition.partitionId,
|
|
6014
|
+
bytes: Math.round(partition.estimatedMemoryBytes + share)
|
|
6015
|
+
});
|
|
6016
|
+
}
|
|
6017
|
+
return sizes;
|
|
6018
|
+
}
|
|
6019
|
+
function largestWithinBudget(sizes) {
|
|
6020
|
+
const ordered = [...sizes].sort((left, right) => right.bytes - left.bytes);
|
|
6021
|
+
const reportedIndexes = /* @__PURE__ */ new Set();
|
|
6022
|
+
const kept = [];
|
|
6023
|
+
let budgetBytes = PARTITION_BYTES_REPORT_BUDGET_BYTES;
|
|
6024
|
+
for (const size of ordered) {
|
|
6025
|
+
const indexCost = reportedIndexes.has(size.indexName) ? 0 : PARTITION_BYTES_INDEX_OVERHEAD_BYTES + Buffer.byteLength(size.indexName);
|
|
6026
|
+
const cost = indexCost + PARTITION_BYTES_ENTRY_BYTES;
|
|
6027
|
+
if (cost > budgetBytes) {
|
|
6028
|
+
break;
|
|
6029
|
+
}
|
|
6030
|
+
budgetBytes -= cost;
|
|
6031
|
+
reportedIndexes.add(size.indexName);
|
|
6032
|
+
kept.push(size);
|
|
6033
|
+
}
|
|
6034
|
+
return kept;
|
|
6035
|
+
}
|
|
6036
|
+
function collectPartitionBytes(engine) {
|
|
6037
|
+
const sizes = [];
|
|
6038
|
+
for (const index of engine.listIndexes()) {
|
|
6039
|
+
try {
|
|
6040
|
+
sizes.push(...heldPartitionSizes(engine, index.name));
|
|
6041
|
+
} catch {
|
|
6042
|
+
}
|
|
6043
|
+
}
|
|
6044
|
+
const byIndex = /* @__PURE__ */ new Map();
|
|
6045
|
+
for (const size of largestWithinBudget(sizes)) {
|
|
6046
|
+
let entry = byIndex.get(size.indexName);
|
|
6047
|
+
if (entry === void 0) {
|
|
6048
|
+
entry = { indexName: size.indexName, partitions: [] };
|
|
6049
|
+
byIndex.set(size.indexName, entry);
|
|
6050
|
+
}
|
|
6051
|
+
entry.partitions.push([size.partitionId, size.bytes]);
|
|
6052
|
+
}
|
|
6053
|
+
for (const entry of byIndex.values()) {
|
|
6054
|
+
entry.partitions.sort((left, right) => left[0] - right[0]);
|
|
6055
|
+
}
|
|
6056
|
+
return [...byIndex.values()];
|
|
6057
|
+
}
|
|
6058
|
+
|
|
6059
|
+
// src/distribution/cluster-node/memory/write-block.ts
|
|
6060
|
+
var NODE_MEMORY_FULL_REASON = "node_memory_full";
|
|
6061
|
+
function createNodeMemoryView(options) {
|
|
6062
|
+
const registeredFull = /* @__PURE__ */ new Set();
|
|
6063
|
+
let unwatch = null;
|
|
6064
|
+
function record(nodeId, full) {
|
|
6065
|
+
if (full) {
|
|
6066
|
+
registeredFull.add(nodeId);
|
|
6067
|
+
} else {
|
|
6068
|
+
registeredFull.delete(nodeId);
|
|
6069
|
+
}
|
|
6070
|
+
}
|
|
6071
|
+
return {
|
|
6072
|
+
fullNodeIds() {
|
|
6073
|
+
const fullNodeIds = new Set(registeredFull);
|
|
6074
|
+
fullNodeIds.delete(options.localNodeId);
|
|
6075
|
+
if (options.localIsFull()) {
|
|
6076
|
+
fullNodeIds.add(options.localNodeId);
|
|
6077
|
+
}
|
|
6078
|
+
return fullNodeIds;
|
|
6079
|
+
},
|
|
6080
|
+
async start() {
|
|
6081
|
+
if (unwatch !== null) {
|
|
6082
|
+
return;
|
|
6083
|
+
}
|
|
6084
|
+
const reportedByWatch = /* @__PURE__ */ new Set();
|
|
6085
|
+
unwatch = await options.coordinator.watchNodes((event) => {
|
|
6086
|
+
reportedByWatch.add(event.nodeId);
|
|
6087
|
+
record(event.nodeId, event.registration?.memory?.full === true);
|
|
6088
|
+
});
|
|
6089
|
+
for (const node of await options.coordinator.listNodes()) {
|
|
6090
|
+
if (!reportedByWatch.has(node.nodeId)) {
|
|
6091
|
+
record(node.nodeId, node.memory?.full === true);
|
|
6092
|
+
}
|
|
6093
|
+
}
|
|
6094
|
+
},
|
|
6095
|
+
stop() {
|
|
6096
|
+
if (unwatch !== null) {
|
|
6097
|
+
unwatch();
|
|
6098
|
+
unwatch = null;
|
|
6099
|
+
}
|
|
6100
|
+
registeredFull.clear();
|
|
6101
|
+
}
|
|
6102
|
+
};
|
|
6103
|
+
}
|
|
6104
|
+
function copyHoldersOf(table) {
|
|
6105
|
+
const holders = /* @__PURE__ */ new Set();
|
|
6106
|
+
for (const assignment of table.assignments.values()) {
|
|
6107
|
+
if (assignment.primary !== null) {
|
|
6108
|
+
holders.add(assignment.primary);
|
|
6109
|
+
}
|
|
6110
|
+
for (const replica of assignment.replicas) {
|
|
6111
|
+
holders.add(replica);
|
|
6112
|
+
}
|
|
6113
|
+
}
|
|
6114
|
+
return holders;
|
|
6115
|
+
}
|
|
6116
|
+
function findWriteBlock(table, view) {
|
|
6117
|
+
if (view === void 0) {
|
|
6118
|
+
return null;
|
|
6119
|
+
}
|
|
6120
|
+
const fullNodeIds = view.fullNodeIds();
|
|
6121
|
+
if (fullNodeIds.size === 0) {
|
|
6122
|
+
return null;
|
|
6123
|
+
}
|
|
6124
|
+
const blockingNodeIds = [...copyHoldersOf(table)].filter((nodeId) => fullNodeIds.has(nodeId)).sort(compareCodePoints);
|
|
6125
|
+
if (blockingNodeIds.length === 0) {
|
|
6126
|
+
return null;
|
|
6127
|
+
}
|
|
6128
|
+
const quotedNodeIds = blockingNodeIds.map((nodeId) => `'${nodeId}'`).join(", ");
|
|
6129
|
+
const holders = blockingNodeIds.length === 1 ? `node ${quotedNodeIds} holds a copy of it and reports` : `nodes ${quotedNodeIds} hold copies of it and report`;
|
|
6130
|
+
return new NarsilError(
|
|
6131
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
6132
|
+
`Index '${table.indexName}' refuses inserts and updates because ${holders} full memory. Remove documents from the indexes on that node or give it more memory, and writes resume once its memory falls below the resume threshold.`,
|
|
6133
|
+
{ reason: NODE_MEMORY_FULL_REASON, indexName: table.indexName, fullNodeIds: blockingNodeIds }
|
|
6134
|
+
);
|
|
6135
|
+
}
|
|
6136
|
+
function requireWritableIndex(table, view) {
|
|
6137
|
+
const block = findWriteBlock(table, view);
|
|
6138
|
+
if (block !== null) {
|
|
6139
|
+
throw block;
|
|
6140
|
+
}
|
|
6141
|
+
}
|
|
6142
|
+
|
|
6143
|
+
// src/distribution/cluster-node/memory/write-pressure.ts
|
|
6144
|
+
var WRITE_BYTES_IN_FLIGHT_REASON = "write_bytes_in_flight";
|
|
6145
|
+
function createWritePressure(options) {
|
|
6146
|
+
let inFlight = 0;
|
|
6147
|
+
return {
|
|
6148
|
+
admit(indexName, bytes) {
|
|
6149
|
+
options.beforeAdmit();
|
|
6150
|
+
const requestBytes = Math.max(0, Math.ceil(bytes));
|
|
6151
|
+
if (inFlight + requestBytes > options.limitBytes) {
|
|
6152
|
+
throw new NarsilError(
|
|
6153
|
+
ErrorCodes.CLUSTER_WRITE_BLOCKED,
|
|
6154
|
+
`Node '${options.nodeId}' refuses this write to index '${indexName}' because its ${requestBytes} bytes would take the writes in flight on the node from ${inFlight} bytes past the limit of ${options.limitBytes} bytes. Retry once the writes in flight finish, or send smaller batches.`,
|
|
6155
|
+
{
|
|
6156
|
+
reason: WRITE_BYTES_IN_FLIGHT_REASON,
|
|
6157
|
+
indexName,
|
|
6158
|
+
nodeId: options.nodeId,
|
|
6159
|
+
requestBytes,
|
|
6160
|
+
inFlightBytes: inFlight,
|
|
6161
|
+
limitBytes: options.limitBytes
|
|
6162
|
+
}
|
|
6163
|
+
);
|
|
6164
|
+
}
|
|
6165
|
+
inFlight += requestBytes;
|
|
6166
|
+
let released = false;
|
|
6167
|
+
return () => {
|
|
6168
|
+
if (released) {
|
|
6169
|
+
return;
|
|
6170
|
+
}
|
|
6171
|
+
released = true;
|
|
6172
|
+
inFlight -= requestBytes;
|
|
6173
|
+
};
|
|
6174
|
+
}
|
|
6175
|
+
};
|
|
6176
|
+
}
|
|
6177
|
+
async function withAdmittedWrite(pressure, indexName, bytes, work) {
|
|
6178
|
+
if (pressure === void 0) {
|
|
6179
|
+
return work();
|
|
6180
|
+
}
|
|
6181
|
+
const release = pressure.admit(indexName, bytes());
|
|
6182
|
+
try {
|
|
6183
|
+
return await work();
|
|
6184
|
+
} finally {
|
|
6185
|
+
release();
|
|
6186
|
+
}
|
|
6187
|
+
}
|
|
6188
|
+
function estimateDocumentBytes(document) {
|
|
6189
|
+
let bytes = 0;
|
|
6190
|
+
const pending = [document];
|
|
6191
|
+
const visited = /* @__PURE__ */ new Set();
|
|
6192
|
+
while (pending.length > 0) {
|
|
6193
|
+
const value = pending.pop();
|
|
6194
|
+
if (typeof value === "string") {
|
|
6195
|
+
bytes += value.length;
|
|
6196
|
+
continue;
|
|
6197
|
+
}
|
|
6198
|
+
if (typeof value === "number" || typeof value === "bigint") {
|
|
6199
|
+
bytes += DOCUMENT_NUMBER_BYTES;
|
|
6200
|
+
continue;
|
|
6201
|
+
}
|
|
6202
|
+
if (value === null || typeof value !== "object") {
|
|
6203
|
+
bytes += DOCUMENT_SCALAR_BYTES;
|
|
6204
|
+
continue;
|
|
6205
|
+
}
|
|
6206
|
+
if (visited.has(value)) {
|
|
6207
|
+
continue;
|
|
6208
|
+
}
|
|
6209
|
+
visited.add(value);
|
|
6210
|
+
if (ArrayBuffer.isView(value)) {
|
|
6211
|
+
bytes += value.byteLength;
|
|
6212
|
+
} else if (Array.isArray(value)) {
|
|
6213
|
+
bytes += DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
6214
|
+
for (const item of value) {
|
|
6215
|
+
pending.push(item);
|
|
6216
|
+
}
|
|
6217
|
+
} else {
|
|
6218
|
+
for (const [key, item] of Object.entries(value)) {
|
|
6219
|
+
bytes += key.length + DOCUMENT_ENTRY_OVERHEAD_BYTES;
|
|
6220
|
+
pending.push(item);
|
|
6221
|
+
}
|
|
6222
|
+
}
|
|
6223
|
+
}
|
|
6224
|
+
return bytes;
|
|
6225
|
+
}
|
|
6226
|
+
function estimateDocumentsBytes(documents) {
|
|
6227
|
+
let bytes = 0;
|
|
6228
|
+
for (const document of documents) {
|
|
6229
|
+
bytes += estimateDocumentBytes(document);
|
|
6230
|
+
}
|
|
6231
|
+
return bytes;
|
|
6232
|
+
}
|
|
6233
|
+
function estimateForwardedBytes(operation) {
|
|
6234
|
+
const documentBytes = operation.document?.byteLength ?? 0;
|
|
6235
|
+
return operation.updateFields === null ? documentBytes : documentBytes + estimateDocumentBytes(operation.updateFields);
|
|
6236
|
+
}
|
|
6237
|
+
|
|
6238
|
+
// src/distribution/cluster-node/memory/node-memory.ts
|
|
6239
|
+
function partitionBytesOf(engine) {
|
|
6240
|
+
try {
|
|
6241
|
+
return collectPartitionBytes(engine);
|
|
6242
|
+
} catch {
|
|
6243
|
+
return [];
|
|
6244
|
+
}
|
|
6245
|
+
}
|
|
6246
|
+
function createNodeMemoryReporting(options) {
|
|
6247
|
+
const monitor = createMemoryMonitor({
|
|
6248
|
+
blockWritesAbove: options.config?.blockWritesAbove ?? DEFAULT_BLOCK_WRITES_ABOVE,
|
|
6249
|
+
resumeWritesBelow: options.config?.resumeWritesBelow ?? DEFAULT_RESUME_WRITES_BELOW,
|
|
6250
|
+
intervalMs: MEMORY_SAMPLE_INTERVAL_MS,
|
|
6251
|
+
collectionIntervalMs: FULL_MEMORY_COLLECTION_INTERVAL_MS,
|
|
6252
|
+
read: readMemoryFigures,
|
|
6253
|
+
collectGarbage: collectAndReturnPages,
|
|
6254
|
+
onFullChange: options.onFullChange
|
|
6255
|
+
});
|
|
6256
|
+
const view = createNodeMemoryView({
|
|
6257
|
+
coordinator: options.coordinator,
|
|
6258
|
+
localNodeId: options.registration.nodeId,
|
|
6259
|
+
localIsFull: monitor.isFull
|
|
6260
|
+
});
|
|
6261
|
+
const heapLimitBytes = readMemoryFigures()?.heapLimitBytes ?? Number.POSITIVE_INFINITY;
|
|
6262
|
+
const writePressure = createWritePressure({
|
|
6263
|
+
nodeId: options.registration.nodeId,
|
|
6264
|
+
limitBytes: Math.floor(
|
|
6265
|
+
heapLimitBytes * (options.config?.inFlightWriteFraction ?? DEFAULT_IN_FLIGHT_WRITE_FRACTION)
|
|
6266
|
+
),
|
|
6267
|
+
beforeAdmit: () => monitor.sampleIfOlderThan(WRITE_MEMORY_SAMPLE_MAX_AGE_MS)
|
|
6268
|
+
});
|
|
6269
|
+
return {
|
|
6270
|
+
view,
|
|
6271
|
+
writePressure,
|
|
6272
|
+
async start() {
|
|
6273
|
+
monitor.start();
|
|
6274
|
+
await view.start();
|
|
6275
|
+
},
|
|
6276
|
+
stop() {
|
|
6277
|
+
monitor.stop();
|
|
6278
|
+
view.stop();
|
|
6279
|
+
},
|
|
6280
|
+
currentRegistration() {
|
|
6281
|
+
const registration = { ...options.registration };
|
|
6282
|
+
const memory = monitor.report();
|
|
6283
|
+
if (memory !== null) {
|
|
6284
|
+
registration.memory = { ...memory };
|
|
6285
|
+
}
|
|
6286
|
+
const partitionBytes = partitionBytesOf(options.engine);
|
|
6287
|
+
if (partitionBytes.length > 0) {
|
|
6288
|
+
registration.partitionBytes = partitionBytes;
|
|
6289
|
+
}
|
|
6290
|
+
return registration;
|
|
6291
|
+
}
|
|
6292
|
+
};
|
|
6293
|
+
}
|
|
6294
|
+
function transportableDetails(error) {
|
|
6295
|
+
if (!(error instanceof NarsilError)) {
|
|
6296
|
+
return void 0;
|
|
6297
|
+
}
|
|
6298
|
+
try {
|
|
6299
|
+
encode(error.details);
|
|
6300
|
+
return error.details;
|
|
6301
|
+
} catch {
|
|
6302
|
+
return void 0;
|
|
6303
|
+
}
|
|
6304
|
+
}
|
|
6305
|
+
function encodeErrorReply(code, message, details) {
|
|
6306
|
+
if (details !== void 0) {
|
|
6307
|
+
try {
|
|
6308
|
+
return encode({ error: true, code, message, details });
|
|
6309
|
+
} catch {
|
|
6310
|
+
return encode({ error: true, code, message });
|
|
6311
|
+
}
|
|
6312
|
+
}
|
|
6313
|
+
return encode({ error: true, code, message });
|
|
6314
|
+
}
|
|
6315
|
+
function encodeErrorReplyFrom(error, fallbackCode) {
|
|
6316
|
+
return encodeErrorReply(
|
|
6317
|
+
error instanceof NarsilError ? error.code : fallbackCode,
|
|
6318
|
+
error instanceof Error ? error.message : String(error),
|
|
6319
|
+
error instanceof NarsilError ? error.details : void 0
|
|
6320
|
+
);
|
|
6321
|
+
}
|
|
6322
|
+
function isPlainRecord(value) {
|
|
6323
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) {
|
|
6324
|
+
return false;
|
|
6325
|
+
}
|
|
6326
|
+
const prototype = Object.getPrototypeOf(value);
|
|
6327
|
+
return prototype === Object.prototype || prototype === null;
|
|
6328
|
+
}
|
|
6329
|
+
function remoteErrorDetails(value) {
|
|
6330
|
+
return isPlainRecord(value) ? value : {};
|
|
6331
|
+
}
|
|
5766
6332
|
|
|
5767
6333
|
// src/distribution/cluster-node/snapshot-auth.ts
|
|
5768
6334
|
var BOOTSTRAPPABLE_STATES = /* @__PURE__ */ new Set(["ACTIVE", "INITIALISING", "MIGRATING"]);
|
|
@@ -6042,12 +6608,12 @@ function assertMessageWithinLimit(bytes, label) {
|
|
|
6042
6608
|
);
|
|
6043
6609
|
}
|
|
6044
6610
|
}
|
|
6045
|
-
async function respondError(sink, sourceId, requestId, code, message) {
|
|
6611
|
+
async function respondError(sink, sourceId, requestId, code, message, details) {
|
|
6046
6612
|
const response = {
|
|
6047
6613
|
type: SNAPSHOT_SYNC_ERROR_TYPE,
|
|
6048
6614
|
sourceId,
|
|
6049
6615
|
requestId,
|
|
6050
|
-
payload:
|
|
6616
|
+
payload: encodeErrorReply(code, message, details)
|
|
6051
6617
|
};
|
|
6052
6618
|
await sink(response);
|
|
6053
6619
|
sink.closed = true;
|
|
@@ -6125,7 +6691,7 @@ async function acquireAndStream(message, sink, request, deps, options) {
|
|
|
6125
6691
|
} catch (err) {
|
|
6126
6692
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
6127
6693
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6128
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6694
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6129
6695
|
return;
|
|
6130
6696
|
}
|
|
6131
6697
|
try {
|
|
@@ -6136,7 +6702,7 @@ async function acquireAndStream(message, sink, request, deps, options) {
|
|
|
6136
6702
|
} catch (err) {
|
|
6137
6703
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
6138
6704
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6139
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6705
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6140
6706
|
return;
|
|
6141
6707
|
}
|
|
6142
6708
|
try {
|
|
@@ -6156,7 +6722,7 @@ async function streamLeasedSnapshot(message, sink, request, deps, options, build
|
|
|
6156
6722
|
} catch (err) {
|
|
6157
6723
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_CAPACITY_EXHAUSTED;
|
|
6158
6724
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6159
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6725
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6160
6726
|
return;
|
|
6161
6727
|
}
|
|
6162
6728
|
try {
|
|
@@ -6226,7 +6792,7 @@ async function decodeRequest(message, sink, deps) {
|
|
|
6226
6792
|
} catch (err) {
|
|
6227
6793
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_REQUEST_INVALID;
|
|
6228
6794
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6229
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6795
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6230
6796
|
return null;
|
|
6231
6797
|
}
|
|
6232
6798
|
}
|
|
@@ -6304,7 +6870,7 @@ async function handleSnapshotSyncRequest(message, respond, deps) {
|
|
|
6304
6870
|
} catch (err) {
|
|
6305
6871
|
const code = err instanceof NarsilError ? err.code : ErrorCodes.SNAPSHOT_SYNC_SNAPSHOT_FAILED;
|
|
6306
6872
|
const errMessage = err instanceof Error ? err.message : String(err);
|
|
6307
|
-
await respondError(sink, deps.nodeId, message.requestId, code, errMessage);
|
|
6873
|
+
await respondError(sink, deps.nodeId, message.requestId, code, errMessage, transportableDetails(err));
|
|
6308
6874
|
}
|
|
6309
6875
|
}
|
|
6310
6876
|
async function runSnapshotSyncRequest(message, sink, deps) {
|
|
@@ -8600,7 +9166,7 @@ function assertForwardResponse(response, indexName, primaryNodeId) {
|
|
|
8600
9166
|
throw new NarsilError(
|
|
8601
9167
|
typeof decoded.code === "string" ? decoded.code : ErrorCodes.QUERY_ROUTING_FAILED,
|
|
8602
9168
|
typeof decoded.message === "string" ? decoded.message : `Remote primary rejected a forwarded write for index '${indexName}'`,
|
|
8603
|
-
{ indexName, primaryNodeId }
|
|
9169
|
+
{ ...remoteErrorDetails(decoded.details), indexName, primaryNodeId }
|
|
8604
9170
|
);
|
|
8605
9171
|
}
|
|
8606
9172
|
return decoded;
|
|
@@ -9104,7 +9670,18 @@ async function replicateAppendedWrites(assignment, appended, failed, rollback, d
|
|
|
9104
9670
|
}
|
|
9105
9671
|
return succeeded;
|
|
9106
9672
|
}
|
|
9107
|
-
|
|
9673
|
+
function applyForwardedWrite(payload, deps) {
|
|
9674
|
+
if (payload.operation === "remove") {
|
|
9675
|
+
return applyAdmittedForwardedWrite(payload, deps);
|
|
9676
|
+
}
|
|
9677
|
+
return withAdmittedWrite(
|
|
9678
|
+
deps.writePressure,
|
|
9679
|
+
payload.indexName,
|
|
9680
|
+
() => estimateForwardedBytes(payload),
|
|
9681
|
+
() => applyAdmittedForwardedWrite(payload, deps)
|
|
9682
|
+
);
|
|
9683
|
+
}
|
|
9684
|
+
async function applyAdmittedForwardedWrite(payload, deps) {
|
|
9108
9685
|
const resolution = await resolvePrimaryAssignment(payload.indexName, payload.documentId, deps, true);
|
|
9109
9686
|
if (resolution === null) {
|
|
9110
9687
|
throw new NarsilError(
|
|
@@ -9113,6 +9690,9 @@ async function applyForwardedWrite(payload, deps) {
|
|
|
9113
9690
|
{ indexName: payload.indexName }
|
|
9114
9691
|
);
|
|
9115
9692
|
}
|
|
9693
|
+
if (payload.operation !== "remove") {
|
|
9694
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9695
|
+
}
|
|
9116
9696
|
if (payload.operation === "insert") {
|
|
9117
9697
|
if (payload.document === null) {
|
|
9118
9698
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "Invalid ForwardPayload: insert requires a document");
|
|
@@ -9253,7 +9833,7 @@ async function forwardBatchToRemote(indexName, items, primaryNodeId, deps) {
|
|
|
9253
9833
|
error: new NarsilError(
|
|
9254
9834
|
operationResult.errorCode ?? ErrorCodes.QUERY_ROUTING_FAILED,
|
|
9255
9835
|
operationResult.errorMessage ?? `Remote primary rejected forwarded operation '${operationResult.documentId}'`,
|
|
9256
|
-
{ indexName, primaryNodeId }
|
|
9836
|
+
{ ...operationResult.errorDetails ?? {}, indexName, primaryNodeId }
|
|
9257
9837
|
)
|
|
9258
9838
|
});
|
|
9259
9839
|
}
|
|
@@ -9287,7 +9867,36 @@ function groupIntoRuns(payload, partitionCount) {
|
|
|
9287
9867
|
}
|
|
9288
9868
|
return runsByPartition;
|
|
9289
9869
|
}
|
|
9870
|
+
function admitForwardedWrites(payload, deps) {
|
|
9871
|
+
let bytes = 0;
|
|
9872
|
+
for (const operation of payload.operations) {
|
|
9873
|
+
if (operation.operation !== "remove") {
|
|
9874
|
+
bytes += estimateForwardedBytes(operation);
|
|
9875
|
+
}
|
|
9876
|
+
}
|
|
9877
|
+
if (deps.writePressure === void 0 || bytes === 0) {
|
|
9878
|
+
return { release: () => {
|
|
9879
|
+
}, refusal: null };
|
|
9880
|
+
}
|
|
9881
|
+
try {
|
|
9882
|
+
return { release: deps.writePressure.admit(payload.indexName, bytes), refusal: null };
|
|
9883
|
+
} catch (error) {
|
|
9884
|
+
if (!(error instanceof NarsilError)) {
|
|
9885
|
+
throw error;
|
|
9886
|
+
}
|
|
9887
|
+
return { release: () => {
|
|
9888
|
+
}, refusal: error };
|
|
9889
|
+
}
|
|
9890
|
+
}
|
|
9290
9891
|
async function applyForwardedBatch(payload, deps) {
|
|
9892
|
+
const admission = admitForwardedWrites(payload, deps);
|
|
9893
|
+
try {
|
|
9894
|
+
return await applyAdmittedForwardedBatch(payload, deps, admission.refusal);
|
|
9895
|
+
} finally {
|
|
9896
|
+
admission.release();
|
|
9897
|
+
}
|
|
9898
|
+
}
|
|
9899
|
+
async function applyAdmittedForwardedBatch(payload, deps, refusal) {
|
|
9291
9900
|
const results = new Array(payload.operations.length);
|
|
9292
9901
|
const table = await deps.coordinator.getAllocation(payload.indexName);
|
|
9293
9902
|
if (table === null || table.assignments.size === 0) {
|
|
@@ -9302,9 +9911,17 @@ async function applyForwardedBatch(payload, deps) {
|
|
|
9302
9911
|
return results;
|
|
9303
9912
|
}
|
|
9304
9913
|
const runsByPartition = groupIntoRuns(payload, table.assignments.size);
|
|
9914
|
+
const writesIndex = payload.operations.some((operation) => operation.operation !== "remove");
|
|
9915
|
+
const block = writesIndex ? refusal ?? findWriteBlock(table, deps.memoryView) : null;
|
|
9305
9916
|
await Promise.all(
|
|
9306
9917
|
Array.from(runsByPartition.entries()).map(async ([partitionId, runs]) => {
|
|
9307
9918
|
for (const run of runs) {
|
|
9919
|
+
if (block !== null && run.operation !== "remove") {
|
|
9920
|
+
for (const { index, wire } of run.operations) {
|
|
9921
|
+
results[index] = failureResult(wire.documentId, block);
|
|
9922
|
+
}
|
|
9923
|
+
continue;
|
|
9924
|
+
}
|
|
9308
9925
|
await applyRun(payload.indexName, partitionId, run, table.assignments.get(partitionId), deps, results);
|
|
9309
9926
|
}
|
|
9310
9927
|
})
|
|
@@ -9404,7 +10021,8 @@ function failureResult(documentId, error) {
|
|
|
9404
10021
|
documentId,
|
|
9405
10022
|
success: false,
|
|
9406
10023
|
errorCode: error.code,
|
|
9407
|
-
errorMessage: error.message
|
|
10024
|
+
errorMessage: error.message,
|
|
10025
|
+
errorDetails: transportableDetails(error) ?? null
|
|
9408
10026
|
};
|
|
9409
10027
|
}
|
|
9410
10028
|
|
|
@@ -9452,11 +10070,20 @@ function collectGroupFailure(failed, docIds, err, indexName, fallbackCode) {
|
|
|
9452
10070
|
failed.push({ docId, error: narsilErr });
|
|
9453
10071
|
}
|
|
9454
10072
|
}
|
|
9455
|
-
|
|
10073
|
+
function routeInsertBatch(indexName, documents, deps, options) {
|
|
10074
|
+
return withAdmittedWrite(
|
|
10075
|
+
deps.writePressure,
|
|
10076
|
+
indexName,
|
|
10077
|
+
() => estimateDocumentsBytes(documents),
|
|
10078
|
+
() => routeAdmittedInsertBatch(indexName, documents, deps, options)
|
|
10079
|
+
);
|
|
10080
|
+
}
|
|
10081
|
+
async function routeAdmittedInsertBatch(indexName, documents, deps, options) {
|
|
9456
10082
|
const table = await routableAllocation(deps.coordinator, indexName);
|
|
9457
10083
|
if (table === null) {
|
|
9458
10084
|
return deps.engine.insertBatch(indexName, documents, options);
|
|
9459
10085
|
}
|
|
10086
|
+
requireWritableIndex(table, deps.memoryView);
|
|
9460
10087
|
const withIds = documents.map((doc) => ({
|
|
9461
10088
|
doc,
|
|
9462
10089
|
docId: typeof doc.id === "string" && doc.id.length > 0 ? doc.id : generateId()
|
|
@@ -9520,11 +10147,20 @@ async function routeRemoveBatch(indexName, docIds, deps) {
|
|
|
9520
10147
|
}
|
|
9521
10148
|
return { succeeded, failed };
|
|
9522
10149
|
}
|
|
9523
|
-
|
|
10150
|
+
function routeUpdateBatch(indexName, updates, deps) {
|
|
10151
|
+
return withAdmittedWrite(
|
|
10152
|
+
deps.writePressure,
|
|
10153
|
+
indexName,
|
|
10154
|
+
() => estimateDocumentsBytes(updates.map((update) => update.document)),
|
|
10155
|
+
() => routeAdmittedUpdateBatch(indexName, updates, deps)
|
|
10156
|
+
);
|
|
10157
|
+
}
|
|
10158
|
+
async function routeAdmittedUpdateBatch(indexName, updates, deps) {
|
|
9524
10159
|
const table = await routableAllocation(deps.coordinator, indexName);
|
|
9525
10160
|
if (table === null) {
|
|
9526
10161
|
return deps.engine.updateBatch(indexName, updates);
|
|
9527
10162
|
}
|
|
10163
|
+
requireWritableIndex(table, deps.memoryView);
|
|
9528
10164
|
const routed = routeBatchItems(table, indexName, updates, (update) => update.docId, deps.nodeId);
|
|
9529
10165
|
const succeeded = [];
|
|
9530
10166
|
const failed = routed.failed;
|
|
@@ -9677,12 +10313,21 @@ async function routeDropIndex(name, coordinator, engine) {
|
|
|
9677
10313
|
}
|
|
9678
10314
|
await coordinator.dropSchema(name);
|
|
9679
10315
|
}
|
|
9680
|
-
|
|
10316
|
+
function routeInsert(indexName, document, docId, deps, options) {
|
|
10317
|
+
return withAdmittedWrite(
|
|
10318
|
+
deps.writePressure,
|
|
10319
|
+
indexName,
|
|
10320
|
+
() => estimateDocumentBytes(document),
|
|
10321
|
+
() => routeAdmittedInsert(indexName, document, docId, deps, options)
|
|
10322
|
+
);
|
|
10323
|
+
}
|
|
10324
|
+
async function routeAdmittedInsert(indexName, document, docId, deps, options) {
|
|
9681
10325
|
const resolvedDocId = docId ?? providedDocId(document) ?? generateId();
|
|
9682
10326
|
const resolution = await resolvePrimaryAssignment(indexName, resolvedDocId, deps, false);
|
|
9683
10327
|
if (resolution === null) {
|
|
9684
10328
|
return deps.engine.insert(indexName, document, resolvedDocId, options);
|
|
9685
10329
|
}
|
|
10330
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9686
10331
|
const primaryNodeId = resolution.assignment.primary;
|
|
9687
10332
|
if (primaryNodeId === deps.nodeId) {
|
|
9688
10333
|
return applyPrimaryInsert(
|
|
@@ -9708,11 +10353,20 @@ async function routeRemove(indexName, docId, deps) {
|
|
|
9708
10353
|
}
|
|
9709
10354
|
return forwardRemoveToRemote(indexName, docId, primaryNodeId, deps);
|
|
9710
10355
|
}
|
|
9711
|
-
|
|
10356
|
+
function routeUpdate(indexName, docId, document, deps) {
|
|
10357
|
+
return withAdmittedWrite(
|
|
10358
|
+
deps.writePressure,
|
|
10359
|
+
indexName,
|
|
10360
|
+
() => estimateDocumentBytes(document),
|
|
10361
|
+
() => routeAdmittedUpdate(indexName, docId, document, deps)
|
|
10362
|
+
);
|
|
10363
|
+
}
|
|
10364
|
+
async function routeAdmittedUpdate(indexName, docId, document, deps) {
|
|
9712
10365
|
const resolution = await resolvePrimaryAssignment(indexName, docId, deps, false);
|
|
9713
10366
|
if (resolution === null) {
|
|
9714
10367
|
return deps.engine.update(indexName, docId, document);
|
|
9715
10368
|
}
|
|
10369
|
+
requireWritableIndex(resolution.table, deps.memoryView);
|
|
9716
10370
|
const primaryNodeId = resolution.assignment.primary;
|
|
9717
10371
|
if (primaryNodeId === deps.nodeId) {
|
|
9718
10372
|
return applyPrimaryUpdate(indexName, docId, document, resolution.partitionId, resolution.assignment, deps);
|
|
@@ -9917,11 +10571,7 @@ function createDataNodeHandler(deps) {
|
|
|
9917
10571
|
return;
|
|
9918
10572
|
}
|
|
9919
10573
|
} catch (err) {
|
|
9920
|
-
const errorPayload =
|
|
9921
|
-
error: true,
|
|
9922
|
-
code: err instanceof NarsilError ? err.code : "INTERNAL_ERROR",
|
|
9923
|
-
message: err instanceof Error ? err.message : String(err)
|
|
9924
|
-
});
|
|
10574
|
+
const errorPayload = encodeErrorReplyFrom(err, "INTERNAL_ERROR");
|
|
9925
10575
|
await respond({
|
|
9926
10576
|
type: `${message.type}.error`,
|
|
9927
10577
|
sourceId: deps.nodeId,
|
|
@@ -11637,6 +12287,8 @@ function resumeNumberingAboveOwnCopy(indexName, partitionId, assignment, deps) {
|
|
|
11637
12287
|
}
|
|
11638
12288
|
deps.seedReplicationLog(indexName, partitionId, floor + 1, assignment.primaryTerm);
|
|
11639
12289
|
}
|
|
12290
|
+
|
|
12291
|
+
// src/distribution/cluster-node/transport-listener.ts
|
|
11640
12292
|
var CONTROLLER_MESSAGE_TYPES = /* @__PURE__ */ new Set([
|
|
11641
12293
|
ReplicationMessageTypes.INSYNC_ADD,
|
|
11642
12294
|
ReplicationMessageTypes.INSYNC_REMOVE,
|
|
@@ -11647,7 +12299,7 @@ function refuseMessage(nodeId, message, respond, code, text) {
|
|
|
11647
12299
|
type: `${message.type}.error`,
|
|
11648
12300
|
sourceId: nodeId,
|
|
11649
12301
|
requestId: message.requestId,
|
|
11650
|
-
payload:
|
|
12302
|
+
payload: encodeErrorReply(code, text, { nodeId })
|
|
11651
12303
|
});
|
|
11652
12304
|
}
|
|
11653
12305
|
function createNotReadyHandler(nodeId) {
|
|
@@ -11716,6 +12368,34 @@ function createSingleResponseRespond(respond) {
|
|
|
11716
12368
|
}
|
|
11717
12369
|
|
|
11718
12370
|
// src/distribution/cluster-node/validate-config.ts
|
|
12371
|
+
function validateMemoryConfig(memory) {
|
|
12372
|
+
if (typeof memory !== "object" || memory === null) {
|
|
12373
|
+
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.memory must be an object", { memory });
|
|
12374
|
+
}
|
|
12375
|
+
const { blockWritesAbove, resumeWritesBelow, inFlightWriteFraction } = memory;
|
|
12376
|
+
for (const [name, value] of [
|
|
12377
|
+
["blockWritesAbove", blockWritesAbove],
|
|
12378
|
+
["resumeWritesBelow", resumeWritesBelow],
|
|
12379
|
+
["inFlightWriteFraction", inFlightWriteFraction]
|
|
12380
|
+
]) {
|
|
12381
|
+
if (value !== void 0 && (typeof value !== "number" || !Number.isFinite(value) || value <= 0 || value > 1)) {
|
|
12382
|
+
throw new NarsilError(
|
|
12383
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12384
|
+
`ClusterNodeConfig.memory.${name} must be a number above 0 and at most 1`,
|
|
12385
|
+
{ [name]: value }
|
|
12386
|
+
);
|
|
12387
|
+
}
|
|
12388
|
+
}
|
|
12389
|
+
const block = typeof blockWritesAbove === "number" ? blockWritesAbove : DEFAULT_BLOCK_WRITES_ABOVE;
|
|
12390
|
+
const resume = typeof resumeWritesBelow === "number" ? resumeWritesBelow : DEFAULT_RESUME_WRITES_BELOW;
|
|
12391
|
+
if (resume >= block) {
|
|
12392
|
+
throw new NarsilError(
|
|
12393
|
+
ErrorCodes.CONFIG_INVALID,
|
|
12394
|
+
`ClusterNodeConfig.memory.resumeWritesBelow (${resume}) must lie below blockWritesAbove (${block})`,
|
|
12395
|
+
{ blockWritesAbove: block, resumeWritesBelow: resume }
|
|
12396
|
+
);
|
|
12397
|
+
}
|
|
12398
|
+
}
|
|
11719
12399
|
function validateClusterNodeConfig(config) {
|
|
11720
12400
|
if (config.address.length === 0) {
|
|
11721
12401
|
throw new NarsilError(ErrorCodes.CONFIG_INVALID, "ClusterNodeConfig.address must not be empty");
|
|
@@ -11778,6 +12458,9 @@ function validateClusterNodeConfig(config) {
|
|
|
11778
12458
|
);
|
|
11779
12459
|
}
|
|
11780
12460
|
}
|
|
12461
|
+
if (config.memory !== void 0) {
|
|
12462
|
+
validateMemoryConfig(config.memory);
|
|
12463
|
+
}
|
|
11781
12464
|
if (config.controller !== void 0) {
|
|
11782
12465
|
const { leaseTtlMs, standbyRetryMs } = config.controller;
|
|
11783
12466
|
if (leaseTtlMs !== void 0 && (!Number.isInteger(leaseTtlMs) || leaseTtlMs < MIN_CONTROLLER_LEASE_TTL_MS)) {
|
|
@@ -11827,9 +12510,6 @@ async function createClusterNode(config) {
|
|
|
11827
12510
|
await engine.shutdown();
|
|
11828
12511
|
throw err;
|
|
11829
12512
|
});
|
|
11830
|
-
function replicationLogKey2(indexName, partitionId) {
|
|
11831
|
-
return replicationLogKey(indexName, partitionId);
|
|
11832
|
-
}
|
|
11833
12513
|
function getReplicationLog2(indexName, partitionId) {
|
|
11834
12514
|
return getReplicationLog(replicationLogs, indexName, partitionId, config.replication);
|
|
11835
12515
|
}
|
|
@@ -11849,6 +12529,13 @@ async function createClusterNode(config) {
|
|
|
11849
12529
|
startedAt: (/* @__PURE__ */ new Date()).toISOString(),
|
|
11850
12530
|
version: SPEC_VERSION
|
|
11851
12531
|
};
|
|
12532
|
+
const memoryReporting = createNodeMemoryReporting({
|
|
12533
|
+
registration,
|
|
12534
|
+
engine,
|
|
12535
|
+
coordinator: config.coordinator,
|
|
12536
|
+
config: config.memory,
|
|
12537
|
+
onFullChange: () => lifecycle.renewRegistration()
|
|
12538
|
+
});
|
|
11852
12539
|
const hasDataRole = roles.includes("data");
|
|
11853
12540
|
const hasControllerRole = roles.includes("controller");
|
|
11854
12541
|
const controllerTransport = hasDataRole ? createMultiplexedControllerTransport(config.transport, nodeId) : null;
|
|
@@ -11862,7 +12549,9 @@ async function createClusterNode(config) {
|
|
|
11862
12549
|
partitionWriteQueues: createPartitionWriteQueues(),
|
|
11863
12550
|
catchUp: createCatchUpState(),
|
|
11864
12551
|
resolveNodeTargets: resolveNodeTargets3,
|
|
11865
|
-
waitForActiveReplicas: config.replication?.waitForActiveReplicas
|
|
12552
|
+
waitForActiveReplicas: config.replication?.waitForActiveReplicas,
|
|
12553
|
+
memoryView: memoryReporting.view,
|
|
12554
|
+
writePressure: memoryReporting.writePressure
|
|
11866
12555
|
};
|
|
11867
12556
|
const orphanedIndexes = /* @__PURE__ */ new Set();
|
|
11868
12557
|
const heldPartitionWrites = /* @__PURE__ */ new Set();
|
|
@@ -11907,6 +12596,7 @@ async function createClusterNode(config) {
|
|
|
11907
12596
|
}
|
|
11908
12597
|
const lifecycle = createDataNodeLifecycle({
|
|
11909
12598
|
registration,
|
|
12599
|
+
currentRegistration: memoryReporting.currentRegistration,
|
|
11910
12600
|
coordinator: config.coordinator,
|
|
11911
12601
|
transport: config.transport,
|
|
11912
12602
|
knownIndexNames: [],
|
|
@@ -11922,7 +12612,7 @@ async function createClusterNode(config) {
|
|
|
11922
12612
|
retainedPartitionIds: (indexName) => engine.heldPartitionsOf(indexName) ?? [],
|
|
11923
12613
|
onRemovePartition: (indexName, partitionId) => {
|
|
11924
12614
|
clearBootstrapSyncIndex(bootstrapSyncState, indexName, partitionId);
|
|
11925
|
-
replicationLogs.delete(
|
|
12615
|
+
replicationLogs.delete(replicationLogKey(indexName, partitionId));
|
|
11926
12616
|
trackHeldPartitionWrite(engine.forgetHeldPartition(indexName, partitionId));
|
|
11927
12617
|
void cleanupRemovedPartition(indexName, partitionId, {
|
|
11928
12618
|
engine,
|
|
@@ -12008,6 +12698,7 @@ async function createClusterNode(config) {
|
|
|
12008
12698
|
if (hasDataRole) {
|
|
12009
12699
|
unregisterNotReadyHandler = await config.transport.listen(createNotReadyHandler(nodeId));
|
|
12010
12700
|
}
|
|
12701
|
+
await memoryReporting.start();
|
|
12011
12702
|
await lifecycle.join();
|
|
12012
12703
|
if (controller !== null) {
|
|
12013
12704
|
await controller.start();
|
|
@@ -12046,6 +12737,7 @@ async function createClusterNode(config) {
|
|
|
12046
12737
|
unregisterNotReadyHandler = null;
|
|
12047
12738
|
}
|
|
12048
12739
|
await lifecycle.shutdown();
|
|
12740
|
+
memoryReporting.stop();
|
|
12049
12741
|
if (controller !== null) {
|
|
12050
12742
|
await controller.shutdown();
|
|
12051
12743
|
}
|