apache-iggy 0.10.0 → 0.10.1-edge.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +248 -28
- package/dist/client/client.config.d.ts +2 -0
- package/dist/client/client.config.js +13 -2
- package/dist/client/client.config.test.js +15 -2
- package/dist/client/client.connection-string.js +13 -1
- package/dist/client/client.connection-string.test.js +43 -1
- package/dist/client/client.connection.d.ts +33 -135
- package/dist/client/client.connection.js +190 -453
- package/dist/client/client.connection.test.js +244 -669
- package/dist/client/client.d.ts +4 -35
- package/dist/client/client.js +8 -101
- package/dist/client/client.operation.d.ts +120 -0
- package/dist/client/client.operation.js +428 -0
- package/dist/client/client.operation.test.d.ts +2 -0
- package/dist/client/client.operation.test.js +285 -0
- package/dist/client/client.socket.d.ts +92 -209
- package/dist/client/client.socket.js +1205 -981
- package/dist/client/client.socket.test.js +4865 -679
- package/dist/client/client.type.d.ts +65 -22
- package/dist/client/index.d.ts +1 -0
- package/dist/client/index.js +1 -0
- package/dist/client.e2e.js +3 -5
- package/dist/e2e/resource-options.e2e.d.ts +2 -0
- package/dist/{wire/stream/purge-stream.command.js → e2e/resource-options.e2e.js} +12 -20
- package/dist/e2e/resource-options.suite.d.ts +15 -0
- package/dist/e2e/resource-options.suite.js +272 -0
- package/dist/e2e/tcp.cluster.e2e.js +1 -0
- package/dist/e2e/tcp.consumer-stream.e2e.js +4 -0
- package/dist/e2e/tcp.parallel.e2e.js +5 -6
- package/dist/e2e/tcp.stream.e2e.js +0 -3
- package/dist/e2e/tcp.system.e2e.js +10 -1
- package/dist/e2e/tcp.token.e2e.js +8 -2
- package/dist/e2e/tcp.topic.e2e.js +0 -3
- package/dist/e2e/tcp.user.e2e.js +46 -3
- package/dist/e2e/tls.system.e2e.js +21 -12
- package/dist/index.d.ts +2 -1
- package/dist/index.js +1 -1
- package/dist/stream/consumer-stream.js +97 -12
- package/dist/wire/cluster/cluster.type.d.ts +2 -2
- package/dist/wire/cluster/cluster.utils.d.ts +1 -0
- package/dist/wire/cluster/cluster.utils.js +4 -1
- package/dist/wire/command-set.d.ts +8 -8
- package/dist/wire/command-set.js +18 -9
- package/dist/wire/command.code.d.ts +3 -10
- package/dist/wire/command.code.js +9 -11
- package/dist/wire/command.utils.d.ts +3 -2
- package/dist/wire/command.utils.js +3 -2
- package/dist/wire/error.code.js +17 -1
- package/dist/wire/error.code.test.js +28 -1
- package/dist/wire/error.utils.d.ts +4 -2
- package/dist/wire/error.utils.js +6 -3
- package/dist/wire/identifier.utils.d.ts +2 -0
- package/dist/wire/identifier.utils.js +2 -0
- package/dist/wire/index.d.ts +1 -0
- package/dist/wire/index.js +1 -0
- package/dist/wire/message/header.type.d.ts +1 -1
- package/dist/wire/message/message-batch.test.js +11 -5
- package/dist/wire/message/message.utils.d.ts +0 -10
- package/dist/wire/message/message.utils.js +1 -22
- package/dist/wire/message/poll-messages.command.d.ts +12 -1
- package/dist/wire/message/poll-messages.command.js +64 -16
- package/dist/wire/message/poll-messages.command.test.js +143 -5
- package/dist/wire/message/poll.utils.d.ts +4 -0
- package/dist/wire/message/poll.utils.js +6 -1
- package/dist/wire/message/send-messages.command.d.ts +3 -3
- package/dist/wire/message/send-messages.command.js +103 -3
- package/dist/wire/message/send-messages.command.test.js +113 -2
- package/dist/wire/offset/delete-offset.command.d.ts +11 -4
- package/dist/wire/offset/delete-offset.command.js +1 -1
- package/dist/wire/offset/get-offset.command.d.ts +1 -1
- package/dist/wire/offset/offset-write.command.test.d.ts +2 -0
- package/dist/wire/offset/offset-write.command.test.js +43 -0
- package/dist/wire/offset/offset.utils.js +10 -6
- package/dist/wire/offset/store-offset.command.d.ts +10 -0
- package/dist/wire/offset/store-offset.command.js +1 -1
- package/dist/wire/options.utils.d.ts +46 -0
- package/dist/wire/options.utils.js +57 -0
- package/dist/wire/partition/partition-route.d.ts +12 -0
- package/dist/wire/partition/partition-route.js +40 -0
- package/dist/wire/partition/partition-route.test.d.ts +2 -0
- package/dist/wire/partition/partition-route.test.js +42 -0
- package/dist/wire/session/login-with-token.command.d.ts +2 -2
- package/dist/wire/session/login-with-token.command.js +7 -2
- package/dist/wire/session/login.command.d.ts +2 -2
- package/dist/wire/session/login.command.js +7 -2
- package/dist/wire/stream/create-stream.command.d.ts +5 -2
- package/dist/wire/stream/create-stream.command.js +20 -2
- package/dist/wire/stream/index.d.ts +1 -1
- package/dist/wire/stream/index.js +1 -1
- package/dist/wire/stream/stream-options.d.ts +77 -0
- package/dist/wire/stream/stream-options.js +101 -0
- package/dist/wire/stream/stream-options.test.d.ts +2 -0
- package/dist/wire/stream/stream-options.test.js +103 -0
- package/dist/wire/stream/stream.utils.d.ts +9 -0
- package/dist/wire/stream/stream.utils.js +17 -3
- package/dist/wire/stream/update-stream.command.d.ts +23 -5
- package/dist/wire/stream/update-stream.command.js +10 -17
- package/dist/wire/system/describe-options.command.d.ts +23 -6
- package/dist/wire/system/describe-options.command.js +21 -2
- package/dist/wire/system/get-stats.command.d.ts +17 -0
- package/dist/wire/system/get-stats.command.js +69 -16
- package/dist/wire/system/get-stats.command.test.d.ts +2 -0
- package/dist/wire/system/get-stats.command.test.js +97 -0
- package/dist/wire/system/index.d.ts +2 -0
- package/dist/wire/system/index.js +2 -0
- package/dist/wire/system/snapshot.command.d.ts +64 -0
- package/dist/wire/system/snapshot.command.js +76 -0
- package/dist/wire/system/snapshot.command.test.d.ts +2 -0
- package/dist/wire/system/snapshot.command.test.js +86 -0
- package/dist/wire/system/update-resource.command.d.ts +53 -0
- package/dist/wire/system/update-resource.command.js +130 -0
- package/dist/wire/system/update-resource.command.test.d.ts +2 -0
- package/dist/wire/system/update-resource.command.test.js +497 -0
- package/dist/wire/token/create-token.command.d.ts +1 -1
- package/dist/wire/token/create-token.command.js +1 -1
- package/dist/wire/token/create-token.command.test.js +6 -0
- package/dist/wire/topic/create-topic.command.d.ts +20 -12
- package/dist/wire/topic/create-topic.command.js +11 -4
- package/dist/wire/topic/index.d.ts +1 -2
- package/dist/wire/topic/index.js +1 -2
- package/dist/wire/topic/topic.utils.d.ts +14 -5
- package/dist/wire/topic/topic.utils.js +26 -11
- package/dist/wire/topic/topic.utils.test.js +13 -13
- package/dist/wire/topic/update-topic.command.d.ts +22 -15
- package/dist/wire/topic/update-topic.command.js +12 -34
- package/dist/wire/user/create-user.command.d.ts +12 -2
- package/dist/wire/user/create-user.command.js +7 -3
- package/dist/wire/user/index.d.ts +1 -0
- package/dist/wire/user/index.js +1 -0
- package/dist/wire/user/update-user.command.d.ts +25 -7
- package/dist/wire/user/update-user.command.js +18 -31
- package/dist/wire/user/user.utils.d.ts +2 -0
- package/dist/wire/user/user.utils.js +5 -3
- package/dist/wire/vsr/header.d.ts +23 -8
- package/dist/wire/vsr/header.js +32 -10
- package/dist/wire/vsr/header.test.js +5 -1
- package/dist/wire/vsr/index.d.ts +8 -2
- package/dist/wire/vsr/index.js +44 -11
- package/dist/wire/vsr/operation.d.ts +18 -6
- package/dist/wire/vsr/operation.js +46 -16
- package/dist/wire/vsr/operation.test.js +32 -8
- package/dist/wire/vsr/register.d.ts +10 -6
- package/dist/wire/vsr/register.js +24 -7
- package/dist/wire/vsr/register.test.js +28 -8
- package/dist/wire/vsr/reply.d.ts +4 -1
- package/dist/wire/vsr/reply.js +42 -7
- package/dist/wire/vsr/reply.test.js +11 -1
- package/dist/wire/vsr/session.d.ts +9 -10
- package/dist/wire/vsr/session.js +32 -23
- package/dist/wire/vsr/session.test.js +33 -7
- package/dist/wire/vsr/vsr.test.js +57 -6
- package/package.json +6 -7
- package/dist/wire/message/flush-unsaved-buffers.command.d.ts +0 -26
- package/dist/wire/message/flush-unsaved-buffers.command.js +0 -36
- package/dist/wire/stream/purge-stream.command.d.ts +0 -22
- package/dist/wire/topic/purge-topic.command.d.ts +0 -24
- package/dist/wire/topic/purge-topic.command.js +0 -39
|
@@ -15,981 +15,1126 @@
|
|
|
15
15
|
// specific language governing permissions and limitations
|
|
16
16
|
// under the License.
|
|
17
17
|
import { EventEmitter } from 'node:events';
|
|
18
|
-
import { ResponseError, responseError } from '../wire/error.utils.js';
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
18
|
+
import { DeserializeError, ResponseError, responseError } from '../wire/error.utils.js';
|
|
19
|
+
import { IggyConnection, endpointKey } from './client.connection.js';
|
|
20
|
+
import { normalizeClientConfig } from './client.config.js';
|
|
21
|
+
import { ChangeSignal, MutationLanes, NotConnectedError, OperationUncertainError, OperationNotRetainedError, OPERATION_TIMEOUT_MS, VsrResponseTimeoutError, pause, waitWithin } from './client.operation.js';
|
|
21
22
|
import { LOGIN, LOGIN_WITH_TOKEN, LOGOUT, PING } from '../wire/index.js';
|
|
22
23
|
import { GET_CLUSTER_METADATA } from '../wire/cluster/get-cluster-metadata.command.js';
|
|
23
|
-
import {
|
|
24
|
+
import { deserializePartitionRoute } from '../wire/partition/partition-route.js';
|
|
25
|
+
import { ConsumerKind } from '../wire/offset/offset.utils.js';
|
|
24
26
|
import { COMMAND_CODE } from '../wire/command.code.js';
|
|
25
27
|
import { serializeIdentifier } from '../wire/identifier.utils.js';
|
|
26
|
-
import {
|
|
27
|
-
import {
|
|
28
|
-
import {
|
|
29
|
-
import {
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
28
|
+
import { deserializeOptions } from '../wire/options.utils.js';
|
|
29
|
+
import { OptionsScope } from '../wire/system/describe-options.command.js';
|
|
30
|
+
import { PartitionKind } from '../wire/message/partitioning.utils.js';
|
|
31
|
+
import { PollingStrategyKind } from '../wire/message/poll.utils.js';
|
|
32
|
+
import { EvictionReason, HEADER_SIZE, REPLY_OFFSET, hasCommittedResult, EMPTY_PARTITION_CONTEXT, readReplyOperation } from '../wire/vsr/header.js';
|
|
33
|
+
import { Operation, isMetadata, operationForCode } from '../wire/vsr/operation.js';
|
|
34
|
+
import { decodeVsrResponse, prepareVsrCommand, readRegisteredSession, readWireName, VsrEvictionError, VsrSession } from '../wire/vsr/index.js';
|
|
35
|
+
import { validateReplyForRequest } from '../wire/vsr/reply.js';
|
|
36
|
+
import { debug } from './client.debug.js';
|
|
37
|
+
export { LaneBlockedError, NotConnectedError, OperationNotRetainedError, OperationUncertainError, VsrResponseTimeoutError } from './client.operation.js';
|
|
38
|
+
const RETRY_INTERVAL_MS = 50;
|
|
39
|
+
const MAX_RETRY_INTERVAL_MS = 1_000;
|
|
40
|
+
const MAX_LIFECYCLE_INTERVAL_MS = 1_000;
|
|
41
|
+
const MAX_RECONNECT_INTERVAL_MS = 10_000;
|
|
42
|
+
const MAX_REPLICAS = 256;
|
|
43
|
+
const MAX_CONCURRENT_DIALS = 4;
|
|
44
|
+
const MAX_PARTITION_ROUTES = 4096;
|
|
35
45
|
const TRANSIENT_NOT_COMMITTED = 57;
|
|
36
46
|
const TRANSIENT_NOT_ACCEPTED = 58;
|
|
37
47
|
const UNAUTHENTICATED = 40;
|
|
38
48
|
const STALE_CLIENT = 30;
|
|
49
|
+
const OPERATION_NOT_RETAINED = 85;
|
|
50
|
+
const HISTORY_UNAVAILABLE = 87;
|
|
51
|
+
const LIFECYCLE_BUSY = 88;
|
|
52
|
+
const NOT_PRIMARY = 89;
|
|
53
|
+
const INVALID_COMMAND = 3;
|
|
39
54
|
const FEATURE_UNAVAILABLE = 5;
|
|
40
|
-
const
|
|
41
|
-
const
|
|
42
|
-
const CONSUMER_SESSION_SIZE = 32;
|
|
43
|
-
const METADATA_WATERMARK_OFFSET = 24;
|
|
55
|
+
const INVALID_SESSION_BINDING = 14004;
|
|
56
|
+
const ROUTE_NAMESPACE_MISSING = new Set([1009, 1010, 2010, 2011, 3007]);
|
|
44
57
|
const POLL_OPTIONS_SIZE = 9 + 4 + 1;
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
const
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
constructor(
|
|
66
|
-
super('
|
|
67
|
-
this.refusal = refusal;
|
|
58
|
+
const IDENTIFIER_PREFIX_SIZE = 2;
|
|
59
|
+
const SEND_METADATA_LENGTH_SIZE = 4;
|
|
60
|
+
const PARTITION_ID_SIZE = 4;
|
|
61
|
+
const STORE_OFFSET_SUFFIX_SIZE = 8 + 1;
|
|
62
|
+
const DELETE_OFFSET_SUFFIX_SIZE = 1;
|
|
63
|
+
const RAW_SEND_NEEDS_PARTITION = 'Raw sends need an explicit partition id. Use the typed send API for Balanced or MessageKey partitioning.';
|
|
64
|
+
const TOPIC_DISCOVERY_RESETS = new Set([
|
|
65
|
+
COMMAND_CODE.DeleteStream, COMMAND_CODE.DeleteTopic,
|
|
66
|
+
COMMAND_CODE.CreatePartitions, COMMAND_CODE.DeletePartitions
|
|
67
|
+
]);
|
|
68
|
+
const READ_COMMANDS = new Set([
|
|
69
|
+
COMMAND_CODE.Ping, COMMAND_CODE.GetStats, COMMAND_CODE.GetClusterMetadata,
|
|
70
|
+
COMMAND_CODE.DescribeOptions, COMMAND_CODE.GetMe, COMMAND_CODE.GetClient,
|
|
71
|
+
COMMAND_CODE.GetClients, COMMAND_CODE.GetUser, COMMAND_CODE.GetUsers,
|
|
72
|
+
COMMAND_CODE.GetAccessTokens, COMMAND_CODE.GetPartitionRoute, COMMAND_CODE.GetOffset,
|
|
73
|
+
COMMAND_CODE.GetStream, COMMAND_CODE.GetStreams, COMMAND_CODE.GetTopic,
|
|
74
|
+
COMMAND_CODE.GetTopics, COMMAND_CODE.GetGroup, COMMAND_CODE.GetGroups, COMMAND_CODE.SyncGroup
|
|
75
|
+
]);
|
|
76
|
+
/** A replica link used its retry budget, and nothing restarts it. */
|
|
77
|
+
class ReconnectExhaustedError extends Error {
|
|
78
|
+
constructor() {
|
|
79
|
+
super('replica reconnect attempts exhausted');
|
|
68
80
|
}
|
|
69
81
|
}
|
|
70
|
-
/**
|
|
71
|
-
* Command codes that can be executed without authentication.
|
|
72
|
-
*/
|
|
73
|
-
const UNLOGGED_COMMAND_CODE = [
|
|
74
|
-
PING.code,
|
|
75
|
-
LOGIN.code,
|
|
76
|
-
LOGIN_WITH_TOKEN.code
|
|
77
|
-
];
|
|
78
|
-
export class VsrResponseTimeoutError extends Error {
|
|
79
|
-
constructor(timeout) {
|
|
80
|
-
super(`timed out after ${timeout} ms waiting for VSR response`);
|
|
81
|
-
this.name = 'VsrResponseTimeoutError';
|
|
82
|
-
Object.setPrototypeOf(this, VsrResponseTimeoutError.prototype);
|
|
83
|
-
}
|
|
84
|
-
}
|
|
85
|
-
/**
|
|
86
|
-
* Manages command execution and response handling for the Iggy server.
|
|
87
|
-
* Implements command queuing, authentication, and heartbeat functionality.
|
|
88
|
-
*/
|
|
82
|
+
/** One logical session owns every link, mutation lane and retained operation. */
|
|
89
83
|
export class CommandResponseStream extends EventEmitter {
|
|
90
|
-
/** Client configuration */
|
|
91
84
|
options;
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
settlingLeader;
|
|
85
|
+
vsrSession = new VsrSession();
|
|
86
|
+
lanes;
|
|
87
|
+
replicas = new Map();
|
|
88
|
+
partitionRoutes = new Map();
|
|
89
|
+
changed = new ChangeSignal();
|
|
90
|
+
lifetime = new AbortController();
|
|
91
|
+
sessionLifetime = new AbortController();
|
|
92
|
+
bootstrap;
|
|
93
|
+
bootstrapPromise;
|
|
102
94
|
/**
|
|
103
|
-
*
|
|
104
|
-
*
|
|
95
|
+
* What the running bootstrap reports when it gives up: the bind failure of
|
|
96
|
+
* the last endpoint that accepted a dial, else NotConnectedError with the
|
|
97
|
+
* last dial failure.
|
|
105
98
|
*/
|
|
106
|
-
|
|
99
|
+
bootstrapFailure;
|
|
100
|
+
authenticationPromise;
|
|
101
|
+
sessionAction = Promise.resolve();
|
|
102
|
+
recoveryPromise;
|
|
103
|
+
rosterPromise;
|
|
107
104
|
/**
|
|
108
|
-
*
|
|
109
|
-
*
|
|
110
|
-
* the
|
|
105
|
+
* Whom the client signs in as by itself: the caller's last sign-in, else the
|
|
106
|
+
* configured credentials while the caller never signed in. A logout, a
|
|
107
|
+
* sign-in of the caller that fails and a refused sign-in clear it.
|
|
111
108
|
*/
|
|
112
|
-
|
|
109
|
+
signInCredentials;
|
|
113
110
|
/**
|
|
114
|
-
*
|
|
115
|
-
*
|
|
116
|
-
* command re-checks the leader, and a command written in the meantime goes
|
|
117
|
-
* out on the socket that check is about to replace.
|
|
111
|
+
* Raised by each sign-in and sign-out of the caller. A call continues under
|
|
112
|
+
* a new session only while the lineage it started with is unchanged.
|
|
118
113
|
*/
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
leaderlessPollInterval;
|
|
124
|
-
/** Calls that have acquired this stream but have not fully settled */
|
|
125
|
-
pendingSubmissions;
|
|
126
|
-
/** Whether the stream is currently processing a command */
|
|
127
|
-
busy;
|
|
128
|
-
/** Whether the client has been authenticated */
|
|
129
|
-
isAuthenticated;
|
|
130
|
-
/** Authenticated user ID */
|
|
131
|
-
userId;
|
|
132
|
-
/** Heartbeat interval timer handle */
|
|
133
|
-
heartbeatIntervalHandler;
|
|
134
|
-
/** Whether a heartbeat ping is still awaiting its response */
|
|
135
|
-
heartbeatInFlight;
|
|
136
|
-
rememberedCredentials;
|
|
137
|
-
clustered;
|
|
114
|
+
sessionLineage = 0;
|
|
115
|
+
knownEndpoints = [];
|
|
116
|
+
configuredEndpoint;
|
|
117
|
+
metadataReplica;
|
|
138
118
|
metadataWatermark = 0n;
|
|
139
119
|
routingGeneration = 0;
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
*/
|
|
147
|
-
constructor(options) {
|
|
120
|
+
activeDials = 0;
|
|
121
|
+
pendingSubmissions = 0;
|
|
122
|
+
heartbeatIntervalHandler;
|
|
123
|
+
isAuthenticated = false;
|
|
124
|
+
userId;
|
|
125
|
+
constructor(config) {
|
|
148
126
|
super();
|
|
149
|
-
|
|
150
|
-
this.
|
|
151
|
-
this.
|
|
152
|
-
this.
|
|
153
|
-
this.isAuthenticated = false;
|
|
154
|
-
this._execQueue = [];
|
|
155
|
-
this.vsrSession = new VsrSession();
|
|
156
|
-
this.authenticationPromise = undefined;
|
|
157
|
-
this.settlingLeader = false;
|
|
158
|
-
this.leaderMovesUndecided = 0;
|
|
159
|
-
this.leaderlessWaitBudget = LEADERLESS_WAIT_BUDGET_MS;
|
|
160
|
-
this.leaderlessPollInterval = LEADERLESS_POLL_INTERVAL_MS;
|
|
161
|
-
this.pendingSubmissions = 0;
|
|
162
|
-
this.heartbeatInFlight = false;
|
|
163
|
-
this._init();
|
|
127
|
+
this.options = normalizeClientConfig(config);
|
|
128
|
+
this.signInCredentials = this.options.credentials;
|
|
129
|
+
this.configuredEndpoint = { host: this.options.options.host ?? '127.0.0.1', port: this.options.options.port };
|
|
130
|
+
this.lanes = new MutationLanes(this.options.maxPendingOperations, this.options.maxPendingOperationBytes);
|
|
164
131
|
}
|
|
165
|
-
;
|
|
166
|
-
|
|
167
|
-
* Initializes the stream by setting up heartbeat and connection event handlers.
|
|
168
|
-
*/
|
|
169
|
-
_init() {
|
|
170
|
-
this.heartbeat(this.options.heartbeatInterval);
|
|
171
|
-
this.connection.on('error', (error) => {
|
|
172
|
-
this._failQueue(error);
|
|
173
|
-
});
|
|
174
|
-
this.connection.on('eviction', (error) => {
|
|
175
|
-
this._resetSession();
|
|
176
|
-
this.emit('eviction', error);
|
|
177
|
-
});
|
|
178
|
-
this.connection.on('disconnected', () => {
|
|
179
|
-
this._resetSession();
|
|
180
|
-
if (this.connection.redirecting) {
|
|
181
|
-
// The client is moving to the leader, which is its own doing: a queued
|
|
182
|
-
// command has not been written, so it belongs on the node being moved
|
|
183
|
-
// to rather than in an error.
|
|
184
|
-
this._reissueQueue();
|
|
185
|
-
return;
|
|
186
|
-
}
|
|
187
|
-
this._failQueue(new Error('connection closed before queued commands were sent'));
|
|
188
|
-
});
|
|
189
|
-
}
|
|
190
|
-
/**
|
|
191
|
-
* Re-submits queued commands through the full send path, so each one
|
|
192
|
-
* reconnects, re-authenticates and re-checks the leader as if it had just
|
|
193
|
-
* been called.
|
|
194
|
-
*
|
|
195
|
-
* Only for a drop the client caused. Nothing here was written, so there is no
|
|
196
|
-
* outcome in doubt: a command still in the queue when the socket is replaced
|
|
197
|
-
* would otherwise fail with a lost-connection error the caller can do nothing
|
|
198
|
-
* about.
|
|
199
|
-
*/
|
|
200
|
-
_reissueQueue() {
|
|
201
|
-
const queued = this._execQueue;
|
|
202
|
-
this._execQueue = [];
|
|
203
|
-
for (const job of queued) {
|
|
204
|
-
debug('re-issuing a queued command after a leader move', job.command);
|
|
205
|
-
// The whole job, not just the payload: a fresh budget would let a command
|
|
206
|
-
// caught in a move take twice the response timeout, and a roster read
|
|
207
|
-
// re-issued as leader-following would answer a leader check with another
|
|
208
|
-
// leader check.
|
|
209
|
-
this.sendCommand(job.command, job.payload, {
|
|
210
|
-
handleResponse: job.handleResponse,
|
|
211
|
-
last: job.last,
|
|
212
|
-
followsLeaderMoves: job.followsLeaderMoves,
|
|
213
|
-
deadline: job.deadline
|
|
214
|
-
}).then(job.resolve, job.reject);
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
/**
|
|
218
|
-
* Sends a command to the server.
|
|
219
|
-
* Automatically handles connection, authentication and leader settlement.
|
|
220
|
-
*
|
|
221
|
-
* @param command - Command code to send
|
|
222
|
-
* @param payload - Command payload buffer
|
|
223
|
-
* @param options - Response and queue options
|
|
224
|
-
* @returns Promise resolving to the command response
|
|
225
|
-
*/
|
|
132
|
+
get lineage() { return this.sessionLineage; }
|
|
133
|
+
get busy() { return this.pendingSubmissions > 0; }
|
|
226
134
|
async sendCommand(command, payload, options = {}) {
|
|
227
|
-
|
|
135
|
+
if (isLoginCommand(command))
|
|
136
|
+
return this.loginByCaller(command, () => Buffer.from(payload), options);
|
|
137
|
+
const lineage = options.lineage ?? this.lineage;
|
|
138
|
+
payload = Buffer.from(payload);
|
|
139
|
+
const release = this.hold();
|
|
140
|
+
const deadline = options.deadline ?? Date.now() + OPERATION_TIMEOUT_MS;
|
|
141
|
+
const signal = AbortSignal.any([this.lifetime.signal, ...(options.signal ? [options.signal] : [])]);
|
|
228
142
|
try {
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
: undefined;
|
|
235
|
-
if (!this.connection.connected)
|
|
236
|
-
await (pollDeadline === undefined ? this.connection.connect()
|
|
237
|
-
: withinDeadline(this.connection.connect(), pollDeadline));
|
|
238
|
-
if (!this.isAuthenticated && !this.isUnloggedCommand(command))
|
|
239
|
-
await (pollDeadline === undefined ? this.authenticate(this._signInCredentials())
|
|
240
|
-
: withinDeadline(this.authenticate(this._signInCredentials()), pollDeadline));
|
|
241
|
-
if (pollDeadline !== undefined) {
|
|
242
|
-
const deadline = pollDeadline;
|
|
243
|
-
if (this.clustered === undefined)
|
|
244
|
-
await withinDeadline(this.sendCommand(GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), { deadline, followsLeaderMoves: false }), deadline);
|
|
245
|
-
if (this.clustered === undefined)
|
|
246
|
-
throw new Error('cannot determine the poll routing topology');
|
|
247
|
-
if (this.clustered)
|
|
248
|
-
return await this._pollOnPrimary(payload, deadline, handleResponse);
|
|
249
|
-
}
|
|
250
|
-
// The roster read is itself a queued command and the queue is
|
|
251
|
-
// single-flighted, so the leader re-check cannot happen inside
|
|
252
|
-
// `_processVsr`. The refusal comes back out here instead, where the
|
|
253
|
-
// queue is free, and the command is re-issued on the node that now
|
|
254
|
-
// leads.
|
|
255
|
-
//
|
|
256
|
-
// A not-admitted refusal means the request was never applied, so it is
|
|
257
|
-
// re-issued for the whole request budget rather than given up on after
|
|
258
|
-
// one window: the roster can still name this node -- an election in
|
|
259
|
-
// flight, a leader that has not moved yet -- and that is a wait, not a
|
|
260
|
-
// verdict.
|
|
261
|
-
//
|
|
262
|
-
// One budget for the whole request: the transient replays on a
|
|
263
|
-
// connection, the leader re-checks, and the re-issues after a move all
|
|
264
|
-
// spend it, so a request cannot outlive it by moving. A command re-issued
|
|
265
|
-
// after a move keeps the budget it was first submitted with, rather than
|
|
266
|
-
// opening a second one.
|
|
267
|
-
const deadline = options.deadline ?? Date.now() + VSR_RESPONSE_TIMEOUT_MS;
|
|
268
|
-
let response;
|
|
269
|
-
let walkingRoster = false;
|
|
270
|
-
const visitedRosterEndpoints = new Set();
|
|
271
|
-
for (;;) {
|
|
272
|
-
try {
|
|
273
|
-
response = await this._queueCommand(command, payload, handleResponse, last, followsLeaderMoves, deadline);
|
|
274
|
-
break;
|
|
275
|
-
}
|
|
276
|
-
catch (error) {
|
|
277
|
-
if (!(error instanceof LeaderMovedError))
|
|
278
|
-
throw error;
|
|
279
|
-
// The roster read that a re-check runs is itself a command that can
|
|
280
|
-
// be refused this way, and answering a leader check with another
|
|
281
|
-
// leader check would recurse. Its caller reads a failure as "stay
|
|
282
|
-
// where you are".
|
|
283
|
-
//
|
|
284
|
-
// A budget too small to carry another attempt ends it here, with the
|
|
285
|
-
// refusal the server actually gave: re-issued into what is left, the
|
|
286
|
-
// request would time out instead and the caller would see a timeout
|
|
287
|
-
// where the answer was "not admitted".
|
|
288
|
-
let moved = false;
|
|
143
|
+
if (command === LOGOUT.code) {
|
|
144
|
+
const signedOut = ++this.sessionLineage;
|
|
145
|
+
this.signInCredentials = undefined;
|
|
146
|
+
return await this.serializeSession(async () => {
|
|
147
|
+
this.ensureLineage(signedOut, command);
|
|
289
148
|
try {
|
|
290
|
-
if (!
|
|
291
|
-
throw responseError(command,
|
|
292
|
-
|
|
293
|
-
// leader recheck between hops would put it straight back on the
|
|
294
|
-
// metadata leader whose partition replica refused it, and the
|
|
295
|
-
// walk would bounce between two nodes without reaching the rest.
|
|
296
|
-
moved = await this._followLeaderMove(walkingRoster, visitedRosterEndpoints);
|
|
297
|
-
if (typeof moved === 'object') {
|
|
298
|
-
walkingRoster = true;
|
|
299
|
-
visitedRosterEndpoints.add(endpointKey(moved.endpoint));
|
|
300
|
-
}
|
|
149
|
+
if (!this.isAuthenticated)
|
|
150
|
+
throw responseError(command, UNAUTHENTICATED);
|
|
151
|
+
return await this.drive(command, payload, options, deadline, signal, signedOut);
|
|
301
152
|
}
|
|
302
153
|
finally {
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
if (followsLeaderMoves)
|
|
308
|
-
this._releaseUndecidedMove();
|
|
309
|
-
}
|
|
310
|
-
const connectionMoved = moved === 'leader' ||
|
|
311
|
-
(typeof moved === 'object' && moved.moved);
|
|
312
|
-
if (!connectionMoved) {
|
|
313
|
-
// Nowhere else to go yet: the roster still names this node, or it
|
|
314
|
-
// could not be read. Paced, because the in-connection replay
|
|
315
|
-
// window belongs to the request's budget and has already been
|
|
316
|
-
// spent -- re-issuing straight away would spin.
|
|
317
|
-
await delay(Math.min(VSR_FAILOVER_CHECK_MS, Math.max(0, deadline - Date.now())));
|
|
154
|
+
if (signedOut === this.lineage) {
|
|
155
|
+
this.signInCredentials = undefined;
|
|
156
|
+
this.resetSession();
|
|
157
|
+
}
|
|
318
158
|
}
|
|
319
|
-
|
|
320
|
-
throw responseError(command, error.refusal.errorCode);
|
|
321
|
-
// A move drops the session with the socket it was bound to, so the
|
|
322
|
-
// re-issue would otherwise go out under no session: a replicated
|
|
323
|
-
// command fails client-side, a non-replicated one goes out with
|
|
324
|
-
// session 0.
|
|
325
|
-
if (!this.isAuthenticated && !this.isUnloggedCommand(command))
|
|
326
|
-
await this.authenticate(this._signInCredentials());
|
|
327
|
-
}
|
|
159
|
+
});
|
|
328
160
|
}
|
|
329
|
-
if (
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
if (this.walkSettleSuppressed) {
|
|
335
|
-
this.walkSettleSuppressed = false;
|
|
336
|
-
return response;
|
|
161
|
+
if (lineage !== this.lineage)
|
|
162
|
+
throw responseError(command, UNAUTHENTICATED);
|
|
163
|
+
if (!this.isAuthenticated && command !== PING.code) {
|
|
164
|
+
void this.signIn();
|
|
165
|
+
await this.nextSession(command, lineage, deadline, signal);
|
|
337
166
|
}
|
|
338
|
-
this.
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
167
|
+
if (this.isAuthenticated && this.replicas.size === 0) {
|
|
168
|
+
const generation = this.routingGeneration;
|
|
169
|
+
// The call waits for a session lost meanwhile itself.
|
|
170
|
+
await this.refreshRoster(deadline, signal).catch((error) => {
|
|
171
|
+
if (generation !== this.routingGeneration)
|
|
172
|
+
return;
|
|
173
|
+
// The call sent nothing yet, so a deadline leaves it not accepted.
|
|
174
|
+
throw error instanceof VsrResponseTimeoutError ? responseError(command, TRANSIENT_NOT_ACCEPTED) : error;
|
|
175
|
+
});
|
|
345
176
|
}
|
|
177
|
+
return await this.drive(command, payload, options, deadline, signal, lineage);
|
|
346
178
|
}
|
|
347
179
|
finally {
|
|
348
|
-
|
|
349
|
-
this._emitFinishQueue();
|
|
180
|
+
release();
|
|
350
181
|
}
|
|
351
182
|
}
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
362
|
-
reject
|
|
363
|
-
};
|
|
364
|
-
if (last)
|
|
365
|
-
this._execQueue.push(job);
|
|
366
|
-
else
|
|
367
|
-
this._execQueue.unshift(job);
|
|
368
|
-
this._processQueue();
|
|
369
|
-
});
|
|
183
|
+
async resolveOperation(operationId, options = {}) {
|
|
184
|
+
try {
|
|
185
|
+
await this.replay(this.lanes.retained(operationId), options);
|
|
186
|
+
}
|
|
187
|
+
catch (error) {
|
|
188
|
+
// A replay that proved the outcome released the operation.
|
|
189
|
+
if (!(error instanceof OperationNotRetainedError))
|
|
190
|
+
throw error;
|
|
191
|
+
this.lanes.provedOutcome(operationId);
|
|
192
|
+
}
|
|
370
193
|
}
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
194
|
+
abandonOperation(operationId) {
|
|
195
|
+
this.lanes.abandon(operationId);
|
|
196
|
+
}
|
|
197
|
+
async replay(operation, options) {
|
|
198
|
+
const lineage = options.lineage ?? this.lineage;
|
|
199
|
+
operation.frame.retain();
|
|
200
|
+
const release = this.hold();
|
|
201
|
+
const deadline = options.deadline ?? Date.now() + OPERATION_TIMEOUT_MS;
|
|
202
|
+
const signal = AbortSignal.any([this.lifetime.signal, this.sessionLifetime.signal,
|
|
203
|
+
operation.lifetime.signal, ...(options.signal ? [options.signal] : [])]);
|
|
204
|
+
let unlock;
|
|
205
|
+
try {
|
|
206
|
+
unlock = await this.lanes.acquireResolution(operation, deadline, signal);
|
|
207
|
+
await this.drive(operation.command, operation.frame.buffer.subarray(HEADER_SIZE), { context: operation.context }, deadline, signal, lineage, operation);
|
|
208
|
+
}
|
|
209
|
+
finally {
|
|
210
|
+
unlock?.();
|
|
211
|
+
operation.frame.release();
|
|
212
|
+
release();
|
|
213
|
+
}
|
|
214
|
+
}
|
|
215
|
+
async drive(command, body, options, deadline, signal, lineage, retained) {
|
|
216
|
+
/** The session that the frame of the call belongs to. */
|
|
217
|
+
let generation = this.routingGeneration;
|
|
218
|
+
const target = partitionRoute(command, body);
|
|
219
|
+
const replicated = command !== LOGOUT.code && operationForCode(command) !== Operation.NonReplicated;
|
|
220
|
+
const progress = command === COMMAND_CODE.PollMessages &&
|
|
221
|
+
(body[body.length - 1] !== 0 || body[body.length - POLL_OPTIONS_SIZE] === PollingStrategyKind.Next);
|
|
222
|
+
const safeRead = READ_COMMANDS.has(command) || (command === COMMAND_CODE.PollMessages && !progress);
|
|
223
|
+
// A replay and a logout belong to their session. Any other call that
|
|
224
|
+
// needs a session waits for the one that replaces a lost one.
|
|
225
|
+
const restartable = !retained && command !== LOGOUT.code && (command !== PING.code || this.isAuthenticated);
|
|
226
|
+
let operation = retained;
|
|
227
|
+
let frameLease = false;
|
|
228
|
+
/** The frame of the operation belongs to a replaced session, or an 88 spent its request id. */
|
|
229
|
+
let staleFrame = false;
|
|
230
|
+
let frame;
|
|
231
|
+
let context = options.context ? { ...options.context } : undefined;
|
|
232
|
+
let selected;
|
|
233
|
+
let historyRefreshed = false;
|
|
234
|
+
let lifecycleDelay = RETRY_INTERVAL_MS;
|
|
235
|
+
let retryDelay = RETRY_INTERVAL_MS;
|
|
236
|
+
let notPrimary = false;
|
|
237
|
+
let refresh = false;
|
|
238
|
+
let lastRefusal;
|
|
239
|
+
let previousLink;
|
|
240
|
+
let sent = false;
|
|
241
|
+
try {
|
|
242
|
+
for (;;) {
|
|
243
|
+
this.ensureLineage(lineage, command);
|
|
244
|
+
signal.throwIfAborted();
|
|
245
|
+
if (Date.now() >= deadline)
|
|
246
|
+
throw new VsrResponseTimeoutError(0);
|
|
247
|
+
if (generation !== this.routingGeneration || (restartable && !this.isAuthenticated)) {
|
|
248
|
+
// No session but the one of a frame can apply it. A frame that can
|
|
249
|
+
// have taken effect ends uncertain and never goes out again.
|
|
250
|
+
if (operation?.uncertain)
|
|
251
|
+
throw new OperationUncertainError(operation.id, responseError(command, TRANSIENT_NOT_ACCEPTED));
|
|
252
|
+
if (!restartable)
|
|
253
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
254
|
+
await this.nextSession(command, lineage, deadline, signal);
|
|
255
|
+
generation = this.routingGeneration;
|
|
256
|
+
selected = undefined;
|
|
257
|
+
previousLink = undefined;
|
|
258
|
+
refresh = false;
|
|
259
|
+
frame = undefined;
|
|
260
|
+
// A frame of the new session carries the context of the route that
|
|
261
|
+
// the new session looks up.
|
|
262
|
+
context = options.context ? { ...options.context } : undefined;
|
|
263
|
+
if (operation) {
|
|
264
|
+
if (frameLease)
|
|
265
|
+
operation.frame.release();
|
|
266
|
+
frameLease = false;
|
|
267
|
+
staleFrame = true;
|
|
397
268
|
}
|
|
398
269
|
}
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
this.
|
|
270
|
+
const sessionSignal = AbortSignal.any([signal, this.sessionLifetime.signal]);
|
|
271
|
+
if (target && !selected) {
|
|
272
|
+
try {
|
|
273
|
+
selected = await this.routeFor(target, deadline, sessionSignal, generation);
|
|
274
|
+
}
|
|
275
|
+
catch (error) {
|
|
276
|
+
// A session failure answered the lookup of the call itself.
|
|
277
|
+
if (restartable && generation !== this.routingGeneration && !isSessionFailure(error))
|
|
278
|
+
continue;
|
|
279
|
+
throw error;
|
|
280
|
+
}
|
|
281
|
+
context ??= { ...selected.route.context };
|
|
282
|
+
}
|
|
283
|
+
context ??= { ...EMPTY_PARTITION_CONTEXT, metadataOp: this.metadataWatermark };
|
|
284
|
+
this.ensureLineage(lineage, command);
|
|
285
|
+
const lane = target ? groupKey(context, target) : 'metadata';
|
|
286
|
+
if (operation && staleFrame && operation.lane !== lane) {
|
|
287
|
+
// A definitive refusal released the old history. The replacement
|
|
288
|
+
// must enter the lane that orders writes to its current incarnation.
|
|
289
|
+
this.lanes.complete(operation);
|
|
290
|
+
operation = undefined;
|
|
291
|
+
}
|
|
292
|
+
if (operation && staleFrame) {
|
|
293
|
+
staleFrame = false;
|
|
294
|
+
try {
|
|
295
|
+
await this.lanes.restart(operation, body, context, this.vsrSession, deadline, signal);
|
|
296
|
+
}
|
|
297
|
+
catch (error) {
|
|
298
|
+
if (generation === this.routingGeneration)
|
|
299
|
+
throw error;
|
|
300
|
+
this.lanes.complete(operation);
|
|
301
|
+
operation = undefined;
|
|
302
|
+
continue;
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
else if (!operation && (replicated || progress)) {
|
|
306
|
+
try {
|
|
307
|
+
operation = await this.lanes.enter(lane, command, body, context, replicated, this.vsrSession, deadline, signal);
|
|
308
|
+
staleFrame = false;
|
|
309
|
+
}
|
|
310
|
+
catch (error) {
|
|
311
|
+
// A lane can free while no session is bound to encode the frame.
|
|
312
|
+
if (restartable && generation !== this.routingGeneration)
|
|
313
|
+
continue;
|
|
314
|
+
throw error;
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
if (operation && !frameLease) {
|
|
318
|
+
operation.frame.retain();
|
|
319
|
+
frameLease = true;
|
|
320
|
+
}
|
|
321
|
+
const attemptSignal = operation
|
|
322
|
+
? AbortSignal.any([sessionSignal, operation.lifetime.signal]) : sessionSignal;
|
|
323
|
+
let attempt;
|
|
324
|
+
let exchanged = false;
|
|
413
325
|
try {
|
|
414
|
-
|
|
326
|
+
if (refresh && target)
|
|
327
|
+
selected = await this.routeFor(target, deadline, attemptSignal, generation, true);
|
|
328
|
+
else if (refresh)
|
|
329
|
+
await this.refreshRoster(deadline, attemptSignal);
|
|
330
|
+
refresh = false;
|
|
331
|
+
const link = target
|
|
332
|
+
? await this.primaryLink(selected, deadline, attemptSignal)
|
|
333
|
+
: await this.readyLink(deadline, attemptSignal, safeRead ? previousLink : undefined);
|
|
334
|
+
previousLink = link;
|
|
335
|
+
this.ensureGeneration(generation, command);
|
|
336
|
+
attempt = await this.exchange(link, command, () => {
|
|
337
|
+
this.ensureGeneration(generation, command);
|
|
338
|
+
this.ensureLineage(lineage, command);
|
|
339
|
+
return operation?.frame.buffer ?? (frame ??= this.vsrSession.encode(command, body, context));
|
|
340
|
+
}, deadline, attemptSignal, operation);
|
|
341
|
+
exchanged = true;
|
|
415
342
|
}
|
|
416
343
|
catch (error) {
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
344
|
+
attempt = { error, committed: false, started: false };
|
|
345
|
+
}
|
|
346
|
+
sent ||= attempt.started;
|
|
347
|
+
if (generation !== this.routingGeneration && attempt.started && (replicated || progress))
|
|
348
|
+
throw uncertainOutcome(command, operation, attempt.error ?? responseError(command, TRANSIENT_NOT_ACCEPTED));
|
|
349
|
+
// A committed rejection proves the outcome even when it also ends the session.
|
|
350
|
+
if (operation?.uncertain && attempt.committed && attempt.error) {
|
|
351
|
+
operation.uncertain = false;
|
|
352
|
+
this.lanes.prove(operation, attempt.error);
|
|
353
|
+
if (isSessionFailure(attempt.error))
|
|
354
|
+
this.recoverSession(generation);
|
|
355
|
+
throw attempt.error;
|
|
356
|
+
}
|
|
357
|
+
if (isSessionFailure(attempt.error)) {
|
|
358
|
+
// An eviction does not say whether a written request was applied,
|
|
359
|
+
// while a 40 refusal or a committed 30 answers this exact request.
|
|
360
|
+
const uncertain = operation?.uncertain ||
|
|
361
|
+
(attempt.started && (replicated || progress) && attempt.error instanceof VsrEvictionError);
|
|
362
|
+
this.recoverSession(generation);
|
|
363
|
+
if (uncertain)
|
|
364
|
+
throw uncertainOutcome(command, operation, attempt.error);
|
|
365
|
+
// An eviction fails every exchange of its link, also one never written.
|
|
366
|
+
if (restartable && exchanged && !attempt.started && generation !== this.routingGeneration)
|
|
367
|
+
continue;
|
|
368
|
+
throw attempt.error;
|
|
369
|
+
}
|
|
370
|
+
if (generation !== this.routingGeneration) {
|
|
371
|
+
if (restartable && (!attempt.started || safeRead))
|
|
372
|
+
continue;
|
|
373
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
374
|
+
}
|
|
375
|
+
if (!attempt.error) {
|
|
376
|
+
if (operation?.uncertain && !attempt.committed)
|
|
377
|
+
throw new OperationUncertainError(operation.id, new Error('reply does not prove the retained result'));
|
|
378
|
+
if (command === COMMAND_CODE.GetPartitionRoute &&
|
|
379
|
+
!this.rememberRoute({ query: body, key: body.toString('hex') }, attempt.response)) {
|
|
380
|
+
await waitWithin(pause(Math.floor(Math.random() * (retryDelay + 1)), signal), deadline, signal);
|
|
381
|
+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_INTERVAL_MS);
|
|
382
|
+
continue;
|
|
383
|
+
}
|
|
384
|
+
if (lineage === this.lineage)
|
|
385
|
+
this._rememberCredentials(command, body);
|
|
386
|
+
if (command === GET_CLUSTER_METADATA.code)
|
|
387
|
+
this.installRoster(attempt.response);
|
|
388
|
+
if (TOPIC_DISCOVERY_RESETS.has(command)) {
|
|
389
|
+
for (const route of this.partitionRoutes.values())
|
|
390
|
+
route.stale = true;
|
|
391
|
+
this.emit('topicDiscoveryReset');
|
|
392
|
+
}
|
|
393
|
+
if (operation)
|
|
394
|
+
this.lanes.prove(operation);
|
|
395
|
+
return options.handleResponse === false ? attempt.raw : attempt.response;
|
|
396
|
+
}
|
|
397
|
+
const error = attempt.error;
|
|
398
|
+
if (error instanceof ResponseError && error.fromServer)
|
|
399
|
+
lastRefusal = error;
|
|
400
|
+
const code = error instanceof ResponseError && error.fromServer ? error.errorCode : undefined;
|
|
401
|
+
const repeatedNotPrimary = notPrimary;
|
|
402
|
+
notPrimary = code === NOT_PRIMARY;
|
|
403
|
+
if (code === TRANSIENT_NOT_COMMITTED) {
|
|
404
|
+
if (operation?.retainable)
|
|
405
|
+
operation.uncertain = true;
|
|
406
|
+
else if (!safeRead)
|
|
407
|
+
throw error;
|
|
408
|
+
}
|
|
409
|
+
else if (code === OPERATION_NOT_RETAINED) {
|
|
410
|
+
if (operation?.retainable)
|
|
411
|
+
operation.uncertain = true;
|
|
412
|
+
throw uncertainOutcome(command, operation, error);
|
|
413
|
+
}
|
|
414
|
+
else if (code === NOT_PRIMARY || code === TRANSIENT_NOT_ACCEPTED) {
|
|
415
|
+
if (selected)
|
|
416
|
+
this.invalidateRoute(selected, code === NOT_PRIMARY);
|
|
417
|
+
refresh = true;
|
|
418
|
+
if (code === NOT_PRIMARY) {
|
|
419
|
+
// Neither 58 nor 89 admitted the call, so the deadline reports both as 58.
|
|
420
|
+
lastRefusal = responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
421
|
+
lastRefusal.fromServer = true;
|
|
422
|
+
}
|
|
423
|
+
if (code === NOT_PRIMARY && !repeatedNotPrimary)
|
|
424
|
+
continue;
|
|
425
|
+
}
|
|
426
|
+
else if (code === HISTORY_UNAVAILABLE && !operation?.uncertain) {
|
|
427
|
+
if (selected)
|
|
428
|
+
this.invalidateRoute(selected, false);
|
|
429
|
+
if (target && command === COMMAND_CODE.SendMessages && !options.context && !historyRefreshed) {
|
|
430
|
+
historyRefreshed = true;
|
|
431
|
+
if (operation) {
|
|
432
|
+
this.lanes.complete(operation);
|
|
433
|
+
operation.frame.release();
|
|
434
|
+
frameLease = false;
|
|
435
|
+
operation = undefined;
|
|
436
|
+
}
|
|
437
|
+
selected = undefined;
|
|
438
|
+
context = undefined;
|
|
439
|
+
continue;
|
|
440
|
+
}
|
|
441
|
+
throw error;
|
|
442
|
+
}
|
|
443
|
+
else if (code === LIFECYCLE_BUSY && !operation?.uncertain) {
|
|
444
|
+
// The refusal spent the request id, so the call goes out again as a
|
|
445
|
+
// new operation in its lane, with the context of its route by then.
|
|
446
|
+
if (operation) {
|
|
447
|
+
operation.frame.release();
|
|
448
|
+
frameLease = false;
|
|
449
|
+
staleFrame = true;
|
|
450
|
+
}
|
|
451
|
+
frame = undefined;
|
|
452
|
+
selected = undefined;
|
|
453
|
+
context = options.context ? { ...options.context } : undefined;
|
|
454
|
+
await waitWithin(pause(lifecycleDelay, signal), deadline, signal);
|
|
455
|
+
lifecycleDelay = Math.min(lifecycleDelay * 2, MAX_LIFECYCLE_INTERVAL_MS);
|
|
456
|
+
continue;
|
|
457
|
+
}
|
|
458
|
+
else if (code !== undefined && ROUTE_NAMESPACE_MISSING.has(code) && attempt.started &&
|
|
459
|
+
(selected || command === COMMAND_CODE.GetPartitionRoute) &&
|
|
460
|
+
!(await this.isMetadataPrimary(previousLink, generation, deadline, attemptSignal))) {
|
|
461
|
+
// A backup can lag the metadata create that the metadata primary
|
|
462
|
+
// committed. A partition call retries with a new route, and a route
|
|
463
|
+
// lookup asks the next replica. A miss of the lookup that routes a
|
|
464
|
+
// partition call came before the request started and stays final.
|
|
465
|
+
if (selected) {
|
|
466
|
+
this.invalidateRoute(selected, false);
|
|
467
|
+
refresh = true;
|
|
468
|
+
}
|
|
469
|
+
lastRefusal = responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
470
|
+
lastRefusal.fromServer = true;
|
|
422
471
|
}
|
|
423
|
-
|
|
472
|
+
else if (code !== undefined) {
|
|
473
|
+
if (operation?.uncertain)
|
|
474
|
+
throw new OperationUncertainError(operation.id, error);
|
|
475
|
+
throw error;
|
|
476
|
+
}
|
|
477
|
+
else {
|
|
478
|
+
if (attempt.started) {
|
|
479
|
+
if (operation?.retainable)
|
|
480
|
+
operation.uncertain = true;
|
|
481
|
+
else if (!safeRead)
|
|
482
|
+
throw uncertainOutcome(command, operation, error);
|
|
483
|
+
}
|
|
484
|
+
// NotConnectedError and an exhausted link leave no link up or
|
|
485
|
+
// redialing, so the call fails now instead of at its deadline.
|
|
486
|
+
if (error instanceof TypeError || error instanceof ResponseError || error instanceof VsrEvictionError ||
|
|
487
|
+
signal.aborted || error instanceof NotConnectedError || error instanceof ReconnectExhaustedError)
|
|
488
|
+
throw error;
|
|
489
|
+
refresh = true;
|
|
490
|
+
}
|
|
491
|
+
await waitWithin(pause(Math.floor(Math.random() * (retryDelay + 1)), signal), deadline, signal);
|
|
492
|
+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_INTERVAL_MS);
|
|
424
493
|
}
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
// A call that sent nothing and saw no refusal was not accepted either.
|
|
497
|
+
if (error instanceof VsrResponseTimeoutError && (lastRefusal || !sent))
|
|
498
|
+
error = lastRefusal ?? responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
499
|
+
if (operation?.uncertain) {
|
|
500
|
+
// A frame of a replaced session must not block the lane of the new one.
|
|
501
|
+
if (generation === this.routingGeneration && this.lanes.owns(operation))
|
|
502
|
+
this.lanes.retain(operation);
|
|
503
|
+
throw error instanceof OperationUncertainError ? error : new OperationUncertainError(operation.id, error);
|
|
428
504
|
}
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
return this.leaderMovesUndecided > 0 || this.leaderMoveInFlight !== undefined;
|
|
446
|
-
}
|
|
447
|
-
/**
|
|
448
|
-
* Releases the queue hold one refusal took, and drains what was held back
|
|
449
|
-
* once the last of them is decided.
|
|
450
|
-
*/
|
|
451
|
-
_releaseUndecidedMove() {
|
|
452
|
-
if (this.leaderMovesUndecided > 0)
|
|
453
|
-
this.leaderMovesUndecided -= 1;
|
|
454
|
-
if (this._movePending() || this.connection.redirecting)
|
|
455
|
-
return;
|
|
456
|
-
void this._processQueue();
|
|
505
|
+
if (retained && !this.lanes.owns(retained))
|
|
506
|
+
throw error instanceof ResponseError ? error : new OperationNotRetainedError(retained.id);
|
|
507
|
+
if (error instanceof ResponseError && !(error instanceof VsrEvictionError) && error.commandCode !== command) {
|
|
508
|
+
const remapped = responseError(command, error.errorCode);
|
|
509
|
+
remapped.fromServer = error.fromServer;
|
|
510
|
+
remapped.committed = error.committed;
|
|
511
|
+
throw remapped;
|
|
512
|
+
}
|
|
513
|
+
throw error;
|
|
514
|
+
}
|
|
515
|
+
finally {
|
|
516
|
+
if (operation && !operation.retained)
|
|
517
|
+
this.lanes.complete(operation);
|
|
518
|
+
if (operation && frameLease)
|
|
519
|
+
operation.frame.release();
|
|
520
|
+
}
|
|
457
521
|
}
|
|
458
|
-
|
|
522
|
+
async exchange(link, command, frame, deadline, signal, operation) {
|
|
523
|
+
const generation = this.routingGeneration;
|
|
524
|
+
let request;
|
|
525
|
+
const attempt = await link.exchange(() => request = frame(), deadline, signal, operation?.frame);
|
|
526
|
+
if (attempt.error || !attempt.response) {
|
|
527
|
+
const error = this.isAuthenticated && command !== COMMAND_CODE.BindSession &&
|
|
528
|
+
isDetachedFailure(attempt.error)
|
|
529
|
+
? new Error('replica link detached from its session', { cause: attempt.error }) : attempt.error;
|
|
530
|
+
return { error, committed: false, started: attempt.started };
|
|
531
|
+
}
|
|
459
532
|
try {
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
.filter((node) => node.endpoints.tcp !== 0)
|
|
466
|
-
.map((node) => ({ host: node.ip, port: node.endpoints.tcp })));
|
|
533
|
+
validateReplyForRequest(attempt.response, request, command);
|
|
534
|
+
const response = decodeVsrResponse(attempt.response, command);
|
|
535
|
+
if (generation === this.routingGeneration)
|
|
536
|
+
this.observeMetadata(attempt.response);
|
|
537
|
+
return { response, raw: attempt.response, committed: hasCommittedResult(attempt.response), started: attempt.started };
|
|
467
538
|
}
|
|
468
539
|
catch (error) {
|
|
469
|
-
|
|
540
|
+
if (this.isAuthenticated && command !== COMMAND_CODE.BindSession && isDetachedFailure(error) &&
|
|
541
|
+
error instanceof ResponseError && error.errorCode === UNAUTHENTICATED) {
|
|
542
|
+
link._destroy();
|
|
543
|
+
const refusal = responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
544
|
+
refusal.fromServer = true;
|
|
545
|
+
return { error: refusal, committed: false, started: attempt.started };
|
|
546
|
+
}
|
|
547
|
+
if (error instanceof ResponseError && error.committed)
|
|
548
|
+
if (generation === this.routingGeneration)
|
|
549
|
+
this.observeMetadata(attempt.response);
|
|
550
|
+
return { error, committed: error instanceof ResponseError && error.committed, started: attempt.started };
|
|
470
551
|
}
|
|
471
552
|
}
|
|
472
|
-
async
|
|
473
|
-
|
|
474
|
-
|
|
553
|
+
async control(link, command, body, deadline, signal) {
|
|
554
|
+
let frame;
|
|
555
|
+
let retryDelay = RETRY_INTERVAL_MS;
|
|
556
|
+
for (;;) {
|
|
557
|
+
const attempt = await this.exchange(link, command, () => frame ??= this.vsrSession.encode(command, body, { ...EMPTY_PARTITION_CONTEXT, metadataOp: this.metadataWatermark }), deadline, signal);
|
|
558
|
+
if (!attempt.error)
|
|
559
|
+
return attempt.response;
|
|
560
|
+
if (!(attempt.error instanceof ResponseError) || !attempt.error.fromServer ||
|
|
561
|
+
!isTransient(attempt.error.errorCode))
|
|
562
|
+
throw attempt.error;
|
|
475
563
|
try {
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
if (this.pollConnections.size >= MAX_POLL_CONNECTIONS) {
|
|
506
|
-
const idle = Array.from(this.pollConnections).find(([, value]) => !value.busy);
|
|
507
|
-
if (!idle)
|
|
508
|
-
throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
|
|
509
|
-
this.pollConnections.delete(idle[0]);
|
|
510
|
-
idle[1].client.destroy();
|
|
511
|
-
}
|
|
512
|
-
entry = {
|
|
513
|
-
client: new CommandResponseStream({
|
|
514
|
-
...this.options,
|
|
515
|
-
options: { ...this.options.options, ...route.endpoint },
|
|
516
|
-
heartbeatInterval: 0,
|
|
517
|
-
reconnect: { enabled: false, interval: 0, maxRetries: 0 }
|
|
518
|
-
})
|
|
519
|
-
};
|
|
520
|
-
this.pollConnections.set(endpoint, entry);
|
|
521
|
-
}
|
|
522
|
-
while (entry.busy)
|
|
523
|
-
await withinDeadline(entry.busy, deadline);
|
|
524
|
-
if (generation !== this.routingGeneration ||
|
|
525
|
-
this.pollConnections.get(endpoint) !== entry ||
|
|
526
|
-
route.attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) <
|
|
527
|
-
this.metadataWatermark)
|
|
528
|
-
throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
|
|
529
|
-
let release;
|
|
530
|
-
entry.busy = new Promise((resolve) => { release = resolve; });
|
|
531
|
-
let polling = false;
|
|
564
|
+
await waitWithin(pause(Math.floor(Math.random() * (retryDelay + 1)), signal), deadline, signal);
|
|
565
|
+
}
|
|
566
|
+
catch (error) {
|
|
567
|
+
throw error instanceof VsrResponseTimeoutError ? attempt.error : error;
|
|
568
|
+
}
|
|
569
|
+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_INTERVAL_MS);
|
|
570
|
+
if (command === GET_CLUSTER_METADATA.code) {
|
|
571
|
+
const alternative = [...this.replicas.values()].find((slot) => slot.attached && slot.connection?.connected && slot.connection !== link);
|
|
572
|
+
link = alternative?.connection ?? link;
|
|
573
|
+
}
|
|
574
|
+
}
|
|
575
|
+
}
|
|
576
|
+
async routeFor(target, deadline, signal, generation, force = false) {
|
|
577
|
+
const cached = this.partitionRoutes.get(target.key);
|
|
578
|
+
if (!force && cached && !cached.stale && cached.route.context.metadataOp >= this.metadataWatermark)
|
|
579
|
+
return cached;
|
|
580
|
+
let lastRefusal;
|
|
581
|
+
let refreshedRoster = false;
|
|
582
|
+
let previous;
|
|
583
|
+
let retryDelay = RETRY_INTERVAL_MS;
|
|
584
|
+
const visited = new Set();
|
|
585
|
+
for (;;) {
|
|
586
|
+
this.ensureGeneration(generation, COMMAND_CODE.GetPartitionRoute);
|
|
587
|
+
const link = await this.readyLink(deadline, signal, previous);
|
|
588
|
+
if (visited.has(link)) {
|
|
589
|
+
visited.clear();
|
|
590
|
+
// Each pass starts at the metadata primary, which a roster read after
|
|
591
|
+
// a namespace miss can have moved.
|
|
592
|
+
previous = undefined;
|
|
532
593
|
try {
|
|
533
|
-
|
|
534
|
-
entry.attachment = undefined;
|
|
535
|
-
await withinDeadline(entry.client.connection.connect(true), deadline, () => entry.client.destroy());
|
|
536
|
-
const credentials = this._signInCredentials();
|
|
537
|
-
const login = 'token' in credentials ? LOGIN_WITH_TOKEN : LOGIN;
|
|
538
|
-
const loginPayload = 'token' in credentials
|
|
539
|
-
? LOGIN_WITH_TOKEN.serialize(credentials)
|
|
540
|
-
: LOGIN.serialize(credentials);
|
|
541
|
-
await entry.client._queueCommand(login.code, loginPayload, true, true, false, deadline);
|
|
542
|
-
}
|
|
543
|
-
if (!entry.attachment?.equals(route.attachment)) {
|
|
544
|
-
await entry.client._queueCommand(COMMAND_CODE.AttachConsumerSession, route.attachment, true, true, false, deadline);
|
|
545
|
-
entry.attachment = route.attachment;
|
|
546
|
-
}
|
|
547
|
-
if (generation !== this.routingGeneration ||
|
|
548
|
-
route.attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) <
|
|
549
|
-
this.metadataWatermark)
|
|
550
|
-
throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
|
|
551
|
-
polling = true;
|
|
552
|
-
return await entry.client._queueCommand(COMMAND_CODE.PollMessagesOnPrimary, payload, handleResponse, true, false, deadline);
|
|
594
|
+
await waitWithin(pause(Math.floor(Math.random() * (retryDelay + 1)), signal), deadline, signal);
|
|
553
595
|
}
|
|
554
596
|
catch (error) {
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
597
|
+
throw error instanceof VsrResponseTimeoutError && lastRefusal ? lastRefusal : error;
|
|
598
|
+
}
|
|
599
|
+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_INTERVAL_MS);
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
visited.add(link);
|
|
603
|
+
previous = link;
|
|
604
|
+
try {
|
|
605
|
+
const attempt = await this.exchange(link, COMMAND_CODE.GetPartitionRoute, () => this.vsrSession.encode(COMMAND_CODE.GetPartitionRoute, target.query, { ...EMPTY_PARTITION_CONTEXT, metadataOp: this.metadataWatermark }), deadline, signal);
|
|
606
|
+
if (attempt.error)
|
|
607
|
+
throw attempt.error;
|
|
608
|
+
this.ensureGeneration(generation, COMMAND_CODE.GetPartitionRoute);
|
|
609
|
+
const entry = this.rememberRoute(target, attempt.response);
|
|
610
|
+
if (entry) {
|
|
611
|
+
if (!this.replicas.has(entry.route.primaryReplica) && !refreshedRoster) {
|
|
612
|
+
refreshedRoster = true;
|
|
613
|
+
await this.refreshRoster(deadline, signal);
|
|
563
614
|
}
|
|
564
|
-
if (
|
|
565
|
-
|
|
566
|
-
|
|
615
|
+
if (this.replicas.has(entry.route.primaryReplica))
|
|
616
|
+
return entry;
|
|
617
|
+
const error = responseError(COMMAND_CODE.GetPartitionRoute, TRANSIENT_NOT_ACCEPTED);
|
|
618
|
+
error.fromServer = true;
|
|
567
619
|
throw error;
|
|
568
620
|
}
|
|
569
|
-
finally {
|
|
570
|
-
entry.busy = undefined;
|
|
571
|
-
release();
|
|
572
|
-
}
|
|
573
621
|
}
|
|
574
622
|
catch (error) {
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
throw
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
623
|
+
if (isSessionFailure(error)) {
|
|
624
|
+
this.recoverSession(generation);
|
|
625
|
+
throw error;
|
|
626
|
+
}
|
|
627
|
+
this.ensureGeneration(generation, COMMAND_CODE.GetPartitionRoute);
|
|
628
|
+
if (error instanceof ResponseError && ROUTE_NAMESPACE_MISSING.has(error.errorCode) &&
|
|
629
|
+
!(await this.isMetadataPrimary(link, generation, deadline, signal))) {
|
|
630
|
+
// A backup can lag a metadata create while the primary materializes the route.
|
|
631
|
+
lastRefusal = responseError(COMMAND_CODE.GetPartitionRoute, TRANSIENT_NOT_ACCEPTED);
|
|
632
|
+
lastRefusal.fromServer = error.fromServer;
|
|
633
|
+
continue;
|
|
634
|
+
}
|
|
635
|
+
if (error instanceof DeserializeError || (error instanceof ResponseError && !isTransient(error.errorCode)))
|
|
636
|
+
throw error;
|
|
637
|
+
if (error instanceof ResponseError)
|
|
638
|
+
lastRefusal = error;
|
|
586
639
|
}
|
|
587
640
|
}
|
|
588
|
-
throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
|
|
589
641
|
}
|
|
590
|
-
|
|
591
|
-
|
|
642
|
+
/**
|
|
643
|
+
* Whether the replica that answered a namespace miss on `link` is the
|
|
644
|
+
* metadata primary. The belief can predate a view change, so one roster read
|
|
645
|
+
* on `link`, without a retry or another link, must still name that replica.
|
|
646
|
+
* No lookup reaches a believed metadata primary without a link, so then that
|
|
647
|
+
* roster also moves the belief.
|
|
648
|
+
*/
|
|
649
|
+
async isMetadataPrimary(link, generation, deadline, signal) {
|
|
650
|
+
const replica = [...this.replicas.values()].find((slot) => slot.connection === link)?.replica;
|
|
651
|
+
const believed = this.metadataReplica === undefined ? undefined : this.replicas.get(this.metadataReplica);
|
|
652
|
+
if (replica !== this.metadataReplica && believed?.attached && believed.connection?.connected)
|
|
653
|
+
return false;
|
|
592
654
|
try {
|
|
593
|
-
|
|
655
|
+
const attempt = await this.exchange(link, GET_CLUSTER_METADATA.code, () => this.vsrSession.encode(GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), { ...EMPTY_PARTITION_CONTEXT, metadataOp: this.metadataWatermark }), deadline, signal);
|
|
656
|
+
if (attempt.error)
|
|
657
|
+
throw attempt.error;
|
|
658
|
+
this.ensureGeneration(generation, GET_CLUSTER_METADATA.code);
|
|
659
|
+
this.installRoster(attempt.response);
|
|
594
660
|
}
|
|
595
661
|
catch (error) {
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
throw error;
|
|
599
|
-
if (Date.now() >= deadline)
|
|
600
|
-
throw error;
|
|
601
|
-
// Only a failed coordinator exchange uses normal session recovery.
|
|
602
|
-
// A healthy coordinator refusing a route must retain its group membership.
|
|
603
|
-
if (error instanceof ResponseError) {
|
|
604
|
-
this.connection.abort();
|
|
605
|
-
this.connection.connected = false;
|
|
606
|
-
this._resetSession();
|
|
607
|
-
}
|
|
608
|
-
await this.sendCommand(PING.code, PING.serialize(), { deadline });
|
|
609
|
-
return this.sendCommand(COMMAND_CODE.GetPollRouting, payload, options);
|
|
662
|
+
debug('roster read after a namespace miss failed', error);
|
|
663
|
+
return false;
|
|
610
664
|
}
|
|
665
|
+
return replica !== undefined && replica === this.metadataReplica;
|
|
611
666
|
}
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
const
|
|
616
|
-
if (
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
if (command === LOGIN.code) {
|
|
626
|
-
const username = readWireName(payload, 0);
|
|
627
|
-
this.rememberedCredentials = {
|
|
628
|
-
username: username.value, password: readWireName(payload, username.next).value
|
|
629
|
-
};
|
|
667
|
+
rememberRoute(target, response) {
|
|
668
|
+
const route = deserializePartitionRoute(response.data);
|
|
669
|
+
const group = groupKey(route.context, target);
|
|
670
|
+
const cached = this.partitionRoutes.get(target.key);
|
|
671
|
+
if (cached && cached.group !== group && route.context.metadataOp < cached.route.context.metadataOp)
|
|
672
|
+
return undefined;
|
|
673
|
+
let minimumView = cached?.group === group ? cached.minimumView : 0;
|
|
674
|
+
for (const alias of this.partitionRoutes.values()) {
|
|
675
|
+
if (alias.group !== group)
|
|
676
|
+
continue;
|
|
677
|
+
minimumView = Math.max(minimumView, alias.minimumView, alias.route.view);
|
|
678
|
+
if (alias.route.view === route.view && alias.route.primaryReplica !== route.primaryReplica)
|
|
679
|
+
minimumView = Math.max(minimumView, route.view + 1);
|
|
630
680
|
}
|
|
631
|
-
|
|
632
|
-
|
|
681
|
+
if (route.context.metadataOp < this.metadataWatermark || route.view < minimumView)
|
|
682
|
+
return undefined;
|
|
683
|
+
const entry = { route, group, minimumView: route.view, stale: false };
|
|
684
|
+
for (const alias of this.partitionRoutes.values()) {
|
|
685
|
+
if (alias.group !== group)
|
|
686
|
+
continue;
|
|
687
|
+
alias.minimumView = Math.max(alias.minimumView, route.view);
|
|
688
|
+
if (alias.route.view < route.view)
|
|
689
|
+
alias.stale = true;
|
|
633
690
|
}
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
691
|
+
this.partitionRoutes.delete(target.key);
|
|
692
|
+
if (this.partitionRoutes.size >= MAX_PARTITION_ROUTES)
|
|
693
|
+
this.partitionRoutes.delete(this.partitionRoutes.keys().next().value);
|
|
694
|
+
this.partitionRoutes.set(target.key, entry);
|
|
695
|
+
this.emit('partitionRoute', { topic: target.query.subarray(0, -PARTITION_ID_SIZE).toString('hex'), count: route.partitionsCount });
|
|
696
|
+
return entry;
|
|
697
|
+
}
|
|
698
|
+
invalidateRoute(refused, advanceView) {
|
|
699
|
+
const floor = refused.route.view + (advanceView ? 1 : 0);
|
|
700
|
+
for (const route of this.partitionRoutes.values()) {
|
|
701
|
+
if (route.group !== refused.group || route.route.view > refused.route.view)
|
|
702
|
+
continue;
|
|
703
|
+
route.stale = true;
|
|
704
|
+
route.minimumView = Math.max(route.minimumView, floor);
|
|
705
|
+
}
|
|
706
|
+
}
|
|
707
|
+
async primaryLink(route, deadline, signal) {
|
|
708
|
+
for (;;) {
|
|
709
|
+
const slot = this.replicas.get(route.route.primaryReplica);
|
|
710
|
+
if (slot?.attached && slot.connection?.connected)
|
|
711
|
+
return slot.connection;
|
|
712
|
+
if (slot?.failure instanceof ResponseError)
|
|
713
|
+
throw slot.failure;
|
|
714
|
+
if (slot?.failure instanceof TypeError)
|
|
715
|
+
throw slot.failure;
|
|
716
|
+
if (!slot || !slot.initial) {
|
|
717
|
+
route.stale = true;
|
|
718
|
+
const error = responseError(COMMAND_CODE.GetPartitionRoute, TRANSIENT_NOT_ACCEPTED);
|
|
719
|
+
error.fromServer = true;
|
|
720
|
+
throw error;
|
|
648
721
|
}
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
722
|
+
try {
|
|
723
|
+
await waitWithin(this.changed.promise, deadline, signal);
|
|
724
|
+
}
|
|
725
|
+
catch (error) {
|
|
726
|
+
if (!(error instanceof VsrResponseTimeoutError))
|
|
727
|
+
throw error;
|
|
728
|
+
const refusal = responseError(COMMAND_CODE.GetPartitionRoute, TRANSIENT_NOT_ACCEPTED);
|
|
729
|
+
refusal.fromServer = true;
|
|
730
|
+
throw refusal;
|
|
654
731
|
}
|
|
655
732
|
}
|
|
656
733
|
}
|
|
657
|
-
|
|
658
|
-
|
|
734
|
+
async readyLink(deadline, signal, avoid) {
|
|
735
|
+
for (;;) {
|
|
736
|
+
if (avoid) {
|
|
737
|
+
const ready = [...this.replicas.values()].filter((slot) => slot.attached && slot.connection?.connected);
|
|
738
|
+
const previous = ready.findIndex((slot) => slot.connection === avoid);
|
|
739
|
+
if (previous >= 0)
|
|
740
|
+
return ready[(previous + 1) % ready.length].connection;
|
|
741
|
+
}
|
|
742
|
+
const primary = this.metadataReplica === undefined ? undefined : this.replicas.get(this.metadataReplica);
|
|
743
|
+
if (primary?.attached && primary.connection?.connected && primary.connection !== avoid)
|
|
744
|
+
return primary.connection;
|
|
745
|
+
for (const slot of this.replicas.values())
|
|
746
|
+
if (slot.attached && slot.connection?.connected && slot.connection !== avoid)
|
|
747
|
+
return slot.connection;
|
|
748
|
+
if (this.bootstrap?.connected && !this.bootstrapPromise)
|
|
749
|
+
return this.bootstrap;
|
|
750
|
+
if (!this.isAuthenticated || this.replicas.size === 0)
|
|
751
|
+
return await this.connectBootstrap(deadline, signal);
|
|
752
|
+
if ([...this.replicas.values()].every((slot) => slot.failure))
|
|
753
|
+
throw [...this.replicas.values()][0]?.failure ?? new Error('no replica endpoints');
|
|
754
|
+
await waitWithin(this.changed.promise, deadline, signal);
|
|
755
|
+
}
|
|
659
756
|
}
|
|
660
|
-
|
|
661
|
-
this.
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
757
|
+
async refreshRoster(deadline, signal) {
|
|
758
|
+
if (!this.rosterPromise) {
|
|
759
|
+
const generation = this.routingGeneration;
|
|
760
|
+
const ownerSignal = AbortSignal.any([this.lifetime.signal, this.sessionLifetime.signal]);
|
|
761
|
+
const pending = (async () => {
|
|
762
|
+
const link = await this.readyLink(Date.now() + OPERATION_TIMEOUT_MS, ownerSignal);
|
|
763
|
+
const response = await this.control(link, GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), Date.now() + OPERATION_TIMEOUT_MS, ownerSignal);
|
|
764
|
+
this.ensureGeneration(generation, GET_CLUSTER_METADATA.code);
|
|
765
|
+
this.installRoster(response);
|
|
766
|
+
})().finally(() => { if (this.rosterPromise === pending)
|
|
767
|
+
this.rosterPromise = undefined; });
|
|
768
|
+
this.rosterPromise = pending;
|
|
769
|
+
}
|
|
770
|
+
await waitWithin(this.rosterPromise, deadline, signal);
|
|
666
771
|
}
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
this.
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
// command in flight when that happens dies with a lost-connection error
|
|
682
|
-
// instead of being re-issued on the node the move lands on.
|
|
683
|
-
const index = this._movePending()
|
|
684
|
-
? this._execQueue.findIndex((job) => !job.followsLeaderMoves)
|
|
685
|
-
: 0;
|
|
686
|
-
if (index < 0)
|
|
687
|
-
break;
|
|
688
|
-
const [next] = this._execQueue.splice(index, 1);
|
|
689
|
-
if (!next)
|
|
690
|
-
break;
|
|
691
|
-
const { command, payload, handleResponse, deadline, resolve, reject } = next;
|
|
692
|
-
try {
|
|
693
|
-
resolve(await this._processNext(command, payload, handleResponse, deadline));
|
|
772
|
+
installRoster(response) {
|
|
773
|
+
const metadata = GET_CLUSTER_METADATA.deserialize(response);
|
|
774
|
+
if (metadata.nodes.length === 0 || metadata.nodes.length > MAX_REPLICAS || new Set(metadata.nodes.map((node) => node.replicaId)).size !== metadata.nodes.length)
|
|
775
|
+
throw responseError(GET_CLUSTER_METADATA.code, INVALID_COMMAND, 'invalid replica roster');
|
|
776
|
+
metadata.nodes = metadata.nodes.filter((node) => node.endpoints.tcp !== 0);
|
|
777
|
+
this.metadataReplica = metadata.nodes.find((node) => node.role === 'Leader' && node.status === 'Healthy')?.replicaId;
|
|
778
|
+
this.knownEndpoints = metadata.nodes.map((node) => ({ host: node.ip, port: node.endpoints.tcp }));
|
|
779
|
+
const present = new Set(metadata.nodes.map((node) => node.replicaId));
|
|
780
|
+
for (const [replica, slot] of this.replicas) {
|
|
781
|
+
const endpoint = metadata.nodes.find((node) => node.replicaId === replica);
|
|
782
|
+
if (!present.has(replica) || !endpoint || endpointKey(slot.endpoint) !== endpointKey({ host: endpoint.ip, port: endpoint.endpoints.tcp })) {
|
|
783
|
+
slot.lifetime.abort();
|
|
784
|
+
slot.connection?._destroy();
|
|
785
|
+
this.replicas.delete(replica);
|
|
694
786
|
}
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
787
|
+
}
|
|
788
|
+
// A single node can advertise an address the client cannot reach, such as
|
|
789
|
+
// one behind a port mapping or a port-forward, so its slot dials the
|
|
790
|
+
// configured endpoint.
|
|
791
|
+
const single = metadata.nodes.length === 1;
|
|
792
|
+
for (const node of metadata.nodes) {
|
|
793
|
+
if (this.replicas.has(node.replicaId))
|
|
794
|
+
continue;
|
|
795
|
+
const endpoint = { host: node.ip, port: node.endpoints.tcp };
|
|
796
|
+
const slot = {
|
|
797
|
+
replica: node.replicaId, endpoint, dialEndpoint: single ? this.configuredEndpoint : endpoint,
|
|
798
|
+
generation: this.routingGeneration, lifetime: new AbortController(), attached: false, initial: true,
|
|
799
|
+
heartbeat: false, restartable: false
|
|
800
|
+
};
|
|
801
|
+
if (this.bootstrap?.connected && (single || this.bootstrap.isConnectedTo(endpoint.host, endpoint.port))) {
|
|
802
|
+
slot.connection = this.bootstrap;
|
|
803
|
+
slot.attached = true;
|
|
804
|
+
slot.initial = false;
|
|
805
|
+
this.bootstrap = undefined;
|
|
702
806
|
}
|
|
807
|
+
this.replicas.set(slot.replica, slot);
|
|
808
|
+
void this.runReplica(slot);
|
|
703
809
|
}
|
|
704
|
-
if (this.
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
// node being moved to.
|
|
708
|
-
if (this.connection.redirecting)
|
|
709
|
-
this._reissueQueue();
|
|
710
|
-
else if (!this._movePending())
|
|
711
|
-
this._failQueue(new Error('connection is not writable'));
|
|
712
|
-
// Otherwise the move is still being decided: these commands were never
|
|
713
|
-
// written, and they are drained again once it settles -- here if the
|
|
714
|
-
// client stays, on the new socket if it moves.
|
|
810
|
+
if (this.bootstrap && metadata.nodes.length > 0) {
|
|
811
|
+
this.bootstrap._destroy();
|
|
812
|
+
this.bootstrap = undefined;
|
|
715
813
|
}
|
|
716
|
-
this.
|
|
717
|
-
this._emitFinishQueue();
|
|
814
|
+
this.changed.pulse();
|
|
718
815
|
}
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
816
|
+
async runReplica(slot) {
|
|
817
|
+
const signal = AbortSignal.any([this.lifetime.signal, this.sessionLifetime.signal, slot.lifetime.signal]);
|
|
818
|
+
const reconnect = this.options.reconnect ?? { enabled: true, interval: 5_000, maxRetries: 12 };
|
|
819
|
+
let failures = 0;
|
|
820
|
+
try {
|
|
821
|
+
for (;;) {
|
|
822
|
+
signal.throwIfAborted();
|
|
823
|
+
if (!slot.connection) {
|
|
824
|
+
let permit = false;
|
|
825
|
+
try {
|
|
826
|
+
while (this.activeDials >= MAX_CONCURRENT_DIALS)
|
|
827
|
+
await waitWithin(this.changed.promise, Date.now() + OPERATION_TIMEOUT_MS, signal);
|
|
828
|
+
signal.throwIfAborted();
|
|
829
|
+
this.activeDials += 1;
|
|
830
|
+
permit = true;
|
|
831
|
+
const connection = this.connectionFor(slot.dialEndpoint);
|
|
832
|
+
slot.connection = connection;
|
|
833
|
+
await connection.connect(signal);
|
|
834
|
+
this.activeDials -= 1;
|
|
835
|
+
permit = false;
|
|
836
|
+
this.changed.pulse();
|
|
837
|
+
const body = this.vsrSession.resumePayload(this.metadataWatermark);
|
|
838
|
+
if (!body)
|
|
839
|
+
throw responseError(COMMAND_CODE.BindSession, UNAUTHENTICATED);
|
|
840
|
+
const response = await this.control(connection, COMMAND_CODE.BindSession, body, Date.now() + OPERATION_TIMEOUT_MS, signal);
|
|
841
|
+
this.ensureGeneration(slot.generation, COMMAND_CODE.BindSession);
|
|
842
|
+
if (readRegisteredSession(response) !== this.vsrSession.sessionId)
|
|
843
|
+
throw responseError(COMMAND_CODE.BindSession, INVALID_SESSION_BINDING);
|
|
844
|
+
slot.attached = true;
|
|
845
|
+
failures = 0;
|
|
846
|
+
this.changed.pulse();
|
|
847
|
+
}
|
|
848
|
+
catch (error) {
|
|
849
|
+
slot.connection?._destroy();
|
|
850
|
+
if (isSessionFailure(error, true)) {
|
|
851
|
+
this.recoverSession(slot.generation);
|
|
852
|
+
return;
|
|
853
|
+
}
|
|
854
|
+
// A 30 refusal failed only the link, so the slot binds again.
|
|
855
|
+
if (error instanceof TypeError || (error instanceof ResponseError &&
|
|
856
|
+
!isTransient(error.errorCode) && error.errorCode !== STALE_CLIENT)) {
|
|
857
|
+
slot.failure = error;
|
|
858
|
+
return;
|
|
859
|
+
}
|
|
860
|
+
}
|
|
861
|
+
finally {
|
|
862
|
+
if (permit)
|
|
863
|
+
this.activeDials -= 1;
|
|
864
|
+
slot.initial = false;
|
|
865
|
+
this.changed.pulse();
|
|
866
|
+
}
|
|
867
|
+
}
|
|
868
|
+
if (slot.attached && slot.connection) {
|
|
869
|
+
await waitWithin(slot.connection.closed, Number.MAX_SAFE_INTEGER, signal);
|
|
870
|
+
if (isSessionFailure(slot.connection.failure)) {
|
|
871
|
+
this.recoverSession(slot.generation);
|
|
872
|
+
return;
|
|
873
|
+
}
|
|
874
|
+
}
|
|
875
|
+
slot.attached = false;
|
|
876
|
+
slot.connection?._destroy();
|
|
877
|
+
slot.connection = undefined;
|
|
878
|
+
this.changed.pulse();
|
|
879
|
+
if (!reconnect.enabled || failures >= reconnect.maxRetries) {
|
|
880
|
+
slot.restartable = reconnect.enabled;
|
|
881
|
+
slot.failure = this.budgetFailure(slot);
|
|
882
|
+
return;
|
|
883
|
+
}
|
|
884
|
+
const base = Math.min(reconnect.interval * 2 ** Math.min(failures, 16), MAX_RECONNECT_INTERVAL_MS);
|
|
885
|
+
failures += 1;
|
|
886
|
+
await pause(Math.floor(Math.random() * (base + 1)), signal, true);
|
|
887
|
+
}
|
|
888
|
+
}
|
|
889
|
+
catch (error) {
|
|
890
|
+
if (!signal.aborted)
|
|
891
|
+
slot.failure = error;
|
|
892
|
+
}
|
|
893
|
+
finally {
|
|
894
|
+
slot.attached = false;
|
|
895
|
+
slot.connection?._destroy();
|
|
896
|
+
this.changed.pulse();
|
|
897
|
+
}
|
|
724
898
|
}
|
|
725
899
|
/**
|
|
726
|
-
*
|
|
727
|
-
*
|
|
728
|
-
* @param command - Command code
|
|
729
|
-
* @param payload - Command payload
|
|
730
|
-
* @param handleResp - Whether to parse the response
|
|
731
|
-
* @param deadline - When the whole request gives up, shared with the leader
|
|
732
|
-
* re-checks and the re-issues after a move
|
|
733
|
-
* @returns Promise resolving to the command response
|
|
900
|
+
* Only a heartbeat restarts a link that used its budget, so without one the
|
|
901
|
+
* link stays down for good.
|
|
734
902
|
*/
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
return this._processVsrLogin(command, payload, handleResp, deadline);
|
|
738
|
-
return this._processVsr(command, payload, handleResp, deadline);
|
|
903
|
+
budgetFailure(slot) {
|
|
904
|
+
return slot.restartable && this.heartbeatIntervalHandler ? new NotConnectedError() : new ReconnectExhaustedError();
|
|
739
905
|
}
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
906
|
+
connectionFor(endpoint) {
|
|
907
|
+
const connection = new IggyConnection({ ...this.options, options: { ...this.options.options, ...endpoint } });
|
|
908
|
+
connection.on('eviction', (error) => this.emit('eviction', error));
|
|
909
|
+
return connection;
|
|
743
910
|
}
|
|
744
|
-
async
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
const
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
// A not-admitted refusal is a statement about who leads, not about
|
|
793
|
-
// load: a node that stopped being primary refuses forever, so
|
|
794
|
-
// replaying on this connection never recovers. Hand it back for a
|
|
795
|
-
// roster re-read once the window is spent. Not-committed (57) stays
|
|
796
|
-
// here: the request is in flight on this very node, and its outcome
|
|
797
|
-
// is unknown anywhere else.
|
|
798
|
-
if (error.errorCode === TRANSIENT_NOT_ACCEPTED &&
|
|
799
|
-
!isLoginCommand(command) &&
|
|
800
|
-
Date.now() >= notAcceptedDeadline)
|
|
801
|
-
throw new LeaderMovedError(error);
|
|
802
|
-
const retryDelay = Math.min(VSR_RETRY_INTERVAL_MS, Math.max(0, deadline - Date.now()));
|
|
803
|
-
if (retryDelay === 0)
|
|
804
|
-
throw error;
|
|
805
|
-
await delay(retryDelay);
|
|
911
|
+
async connectBootstrap(deadline, signal) {
|
|
912
|
+
if (this.bootstrap?.connected && !this.bootstrapPromise)
|
|
913
|
+
return this.bootstrap;
|
|
914
|
+
if (!this.bootstrapPromise) {
|
|
915
|
+
const generation = this.routingGeneration;
|
|
916
|
+
const ownerSignal = AbortSignal.any([this.lifetime.signal, this.sessionLifetime.signal]);
|
|
917
|
+
this.bootstrapFailure = undefined;
|
|
918
|
+
const pending = (async () => {
|
|
919
|
+
const endpoints = [...new Map([this.configuredEndpoint, ...this.knownEndpoints].map((endpoint) => [endpointKey(endpoint), endpoint])).values()];
|
|
920
|
+
const reconnect = this.options.reconnect ?? { enabled: true, interval: 5_000, maxRetries: 12 };
|
|
921
|
+
let dialed = false;
|
|
922
|
+
for (let pass = 0;; pass += 1) {
|
|
923
|
+
for (const endpoint of endpoints) {
|
|
924
|
+
const link = this.connectionFor(endpoint);
|
|
925
|
+
this.bootstrap = link;
|
|
926
|
+
let accepted = false;
|
|
927
|
+
try {
|
|
928
|
+
await link.connect(ownerSignal);
|
|
929
|
+
accepted = dialed = true;
|
|
930
|
+
this.bootstrapFailure = undefined;
|
|
931
|
+
if (this.isAuthenticated) {
|
|
932
|
+
const body = this.vsrSession.resumePayload(this.metadataWatermark);
|
|
933
|
+
if (!body)
|
|
934
|
+
throw responseError(COMMAND_CODE.BindSession, UNAUTHENTICATED);
|
|
935
|
+
const response = await this.control(link, COMMAND_CODE.BindSession, body, Date.now() + OPERATION_TIMEOUT_MS, ownerSignal);
|
|
936
|
+
if (readRegisteredSession(response) !== this.vsrSession.sessionId)
|
|
937
|
+
throw responseError(COMMAND_CODE.BindSession, INVALID_SESSION_BINDING);
|
|
938
|
+
}
|
|
939
|
+
return link;
|
|
940
|
+
}
|
|
941
|
+
catch (error) {
|
|
942
|
+
link._destroy();
|
|
943
|
+
if (this.isAuthenticated && isSessionFailure(error, true)) {
|
|
944
|
+
this.recoverSession(generation);
|
|
945
|
+
throw error;
|
|
946
|
+
}
|
|
947
|
+
if (error instanceof TypeError || ownerSignal.aborted)
|
|
948
|
+
throw error;
|
|
949
|
+
if (accepted)
|
|
950
|
+
this.bootstrapFailure = error;
|
|
951
|
+
else if (!dialed)
|
|
952
|
+
this.bootstrapFailure = new NotConnectedError(error);
|
|
953
|
+
}
|
|
954
|
+
}
|
|
955
|
+
if (!reconnect.enabled || pass >= reconnect.maxRetries)
|
|
956
|
+
throw this.bootstrapFailure;
|
|
957
|
+
const base = Math.min(reconnect.interval * 2 ** Math.min(pass, 16), MAX_RECONNECT_INTERVAL_MS);
|
|
958
|
+
await pause(Math.floor(Math.random() * (base + 1)), ownerSignal, true);
|
|
806
959
|
}
|
|
807
|
-
}
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
}
|
|
814
|
-
this._rememberCredentials(command, payload);
|
|
815
|
-
if (prepared.command === COMMAND_CODE.LogoutUser) {
|
|
816
|
-
this.rememberedCredentials = undefined;
|
|
817
|
-
this._resetSession();
|
|
818
|
-
}
|
|
819
|
-
// Every roster read feeds the redial candidates, whoever asked for it
|
|
820
|
-
// and whatever it says: a node dies together with its address, the
|
|
821
|
-
// roster is unreachable exactly when it is needed, and reading it only
|
|
822
|
-
// during a login would leave the candidates stale between logins.
|
|
823
|
-
if (handleResp && command === GET_CLUSTER_METADATA.code)
|
|
824
|
-
this._rememberRoster(parsed);
|
|
825
|
-
return parsed;
|
|
960
|
+
})().finally(() => { if (this.bootstrapPromise === pending)
|
|
961
|
+
this.bootstrapPromise = undefined; });
|
|
962
|
+
this.bootstrapPromise = pending;
|
|
963
|
+
}
|
|
964
|
+
try {
|
|
965
|
+
return await waitWithin(this.bootstrapPromise, deadline, signal);
|
|
826
966
|
}
|
|
827
967
|
catch (error) {
|
|
828
|
-
//
|
|
829
|
-
//
|
|
830
|
-
if (error instanceof
|
|
831
|
-
throw
|
|
832
|
-
// Once bytes were handed to the socket, a local transport or decode
|
|
833
|
-
// failure leaves the request outcome ambiguous. Register a fresh session
|
|
834
|
-
// rather than replaying that request under a different client identity.
|
|
835
|
-
if (!(error instanceof ResponseError) && requestWritten)
|
|
836
|
-
this._resetSession();
|
|
837
|
-
if (error instanceof VsrEvictionError)
|
|
838
|
-
throw error;
|
|
839
|
-
if (error instanceof ResponseError)
|
|
840
|
-
throw responseError(command, error.errorCode);
|
|
968
|
+
// With the default reconnect values the deadline usually ends the dials
|
|
969
|
+
// before the retry budget does, so it reports the last failure the same way.
|
|
970
|
+
if (error instanceof VsrResponseTimeoutError && this.bootstrapFailure !== undefined)
|
|
971
|
+
throw this.bootstrapFailure;
|
|
841
972
|
throw error;
|
|
842
973
|
}
|
|
843
974
|
}
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
}, timeout);
|
|
877
|
-
}
|
|
878
|
-
this.connection.once('error', errorCallback);
|
|
879
|
-
this.connection.once('disconnected', disconnectedCallback);
|
|
880
|
-
this.connection.once('eviction', evictionCallback);
|
|
881
|
-
this.connection.once('response', responseCallback);
|
|
882
|
-
try {
|
|
883
|
-
write();
|
|
884
|
-
if (state)
|
|
885
|
-
state.written = true;
|
|
975
|
+
serializeSession(action) {
|
|
976
|
+
const pending = this.sessionAction.then(action, action);
|
|
977
|
+
this.sessionAction = pending.catch(() => undefined);
|
|
978
|
+
return pending;
|
|
979
|
+
}
|
|
980
|
+
async login(command, body, deadline, signal, lineage) {
|
|
981
|
+
this.ensureLineage(lineage, command);
|
|
982
|
+
signal.throwIfAborted();
|
|
983
|
+
if (this.isAuthenticated)
|
|
984
|
+
await this.drive(LOGOUT.code, LOGOUT.serialize(), {}, deadline, AbortSignal.any([signal, this.sessionLifetime.signal]), lineage);
|
|
985
|
+
this.ensureLineage(lineage, command);
|
|
986
|
+
this.resetSession();
|
|
987
|
+
const generation = this.routingGeneration;
|
|
988
|
+
const sessionSignal = AbortSignal.any([signal, this.sessionLifetime.signal]);
|
|
989
|
+
const prepared = prepareVsrCommand(command, body, this.vsrSession.bindSecret);
|
|
990
|
+
let frame;
|
|
991
|
+
let retryDelay = RETRY_INTERVAL_MS;
|
|
992
|
+
for (;;) {
|
|
993
|
+
const link = await this.connectBootstrap(deadline, sessionSignal);
|
|
994
|
+
this.ensureLineage(lineage, command);
|
|
995
|
+
const attempt = await this.exchange(link, prepared.command, () => frame ??= this.vsrSession.encode(prepared.command, prepared.payload), deadline, sessionSignal);
|
|
996
|
+
this.ensureLineage(lineage, command);
|
|
997
|
+
this.ensureGeneration(generation, command);
|
|
998
|
+
if (!attempt.error) {
|
|
999
|
+
this.vsrSession.bind(readRegisteredSession(attempt.response));
|
|
1000
|
+
this.userId = attempt.response.data.readUInt32LE(0);
|
|
1001
|
+
this.isAuthenticated = true;
|
|
1002
|
+
this.heartbeat(this.options.heartbeatInterval);
|
|
1003
|
+
await this.refreshRoster(deadline, sessionSignal);
|
|
1004
|
+
this.ensureLineage(lineage, command);
|
|
1005
|
+
this._rememberCredentials(command, body);
|
|
1006
|
+
return attempt.response;
|
|
886
1007
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
reject(error);
|
|
1008
|
+
if (attempt.error instanceof ResponseError && !isTransient(attempt.error.errorCode)) {
|
|
1009
|
+
throw attempt.error;
|
|
890
1010
|
}
|
|
891
|
-
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
// at a backup still succeeds because the server forwards the register to
|
|
896
|
-
// the primary.
|
|
897
|
-
isUnloggedCommand(command) {
|
|
898
|
-
return UNLOGGED_COMMAND_CODE.includes(command);
|
|
899
|
-
}
|
|
900
|
-
/**
|
|
901
|
-
* Moves a freshly authenticated session to the cluster leader.
|
|
902
|
-
*
|
|
903
|
-
* Only the leader accepts replicated commands, and the roster read is
|
|
904
|
-
* auth-gated, so the topology cannot be inspected before a login binds a
|
|
905
|
-
* session. The redirect drops that session along with the socket, so the
|
|
906
|
-
* login is replayed on the leader and its answer supersedes the one from the
|
|
907
|
-
* node the client dialed. Leadership can move between the roster read and
|
|
908
|
-
* the replay, so each freshly bound hop rechecks the roster under a bounded
|
|
909
|
-
* redirect budget.
|
|
910
|
-
*
|
|
911
|
-
* @returns The leader's login response, or undefined when the client stays
|
|
912
|
-
*/
|
|
913
|
-
async _settleOnLeader(loginCommand, loginPayload) {
|
|
914
|
-
let settledResponse;
|
|
915
|
-
for (let redirects = 0; redirects < MAX_LEADER_REDIRECTS; redirects += 1) {
|
|
916
|
-
const leader = await this._readLeaderEndpoint();
|
|
917
|
-
if (!leader || this.connection.isConnectedTo(leader.host, leader.port))
|
|
918
|
-
return settledResponse;
|
|
919
|
-
await this.connection.redirect(leader.host, leader.port);
|
|
920
|
-
settledResponse = await this.sendCommand(loginCommand, loginPayload, { last: false });
|
|
1011
|
+
if (attempt.error instanceof VsrEvictionError || attempt.error instanceof TypeError)
|
|
1012
|
+
throw attempt.error;
|
|
1013
|
+
await waitWithin(pause(Math.floor(Math.random() * (retryDelay + 1)), sessionSignal), deadline, sessionSignal);
|
|
1014
|
+
retryDelay = Math.min(retryDelay * 2, MAX_RETRY_INTERVAL_MS);
|
|
921
1015
|
}
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
1016
|
+
}
|
|
1017
|
+
recoverSession(generation) {
|
|
1018
|
+
if (generation !== this.routingGeneration || this.recoveryPromise || this.lifetime.signal.aborted)
|
|
1019
|
+
return;
|
|
1020
|
+
this.resetSession();
|
|
1021
|
+
this.authenticationPromise = undefined;
|
|
1022
|
+
this.recoveryPromise = this.signIn().then(() => undefined)
|
|
1023
|
+
.catch((error) => { debug('session recovery failed', error); })
|
|
1024
|
+
.finally(() => { this.recoveryPromise = undefined; });
|
|
925
1025
|
}
|
|
926
1026
|
/**
|
|
927
|
-
*
|
|
928
|
-
*
|
|
929
|
-
*
|
|
930
|
-
*
|
|
931
|
-
* node instead of failing a login that already succeeded.
|
|
1027
|
+
* Waits for the sign-in that runs or is queued now, and fails unless it binds
|
|
1028
|
+
* a session of the call's lineage. Only the client's own sign-in continues a
|
|
1029
|
+
* call: after a sign-in or sign-out of the caller, or a refused sign-in, the
|
|
1030
|
+
* call ends with 40, and at its deadline with 58.
|
|
932
1031
|
*/
|
|
933
|
-
async
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
// until the peers' election completes. That window is roughly one heartbeat
|
|
937
|
-
// timeout, so poll through it rather than settling on a replica that denies
|
|
938
|
-
// every replicated command for its whole retry budget.
|
|
939
|
-
const deadline = Date.now() + this.leaderlessWaitBudget;
|
|
940
|
-
while (true) {
|
|
941
|
-
// Reading without a session would re-enter authentication, which awaits
|
|
942
|
-
// the very login this settlement runs inside of. The session can also die
|
|
943
|
-
// between polls, so this holds for every pass, not just the first.
|
|
944
|
-
if (!this.isAuthenticated)
|
|
945
|
-
return undefined;
|
|
1032
|
+
async nextSession(command, lineage, deadline, signal) {
|
|
1033
|
+
let failure;
|
|
1034
|
+
if (lineage === this.lineage) {
|
|
946
1035
|
try {
|
|
947
|
-
|
|
948
|
-
// would race an in-flight exchange and both would wake on the same
|
|
949
|
-
// response event.
|
|
950
|
-
const response = await this.sendCommand(GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), { last: false, followsLeaderMoves: false });
|
|
951
|
-
// The redial candidates are fed by `_processVsr` for every roster
|
|
952
|
-
// read, leaderless ones included: a roster with no leader still names
|
|
953
|
-
// where the nodes are.
|
|
954
|
-
const metadata = GET_CLUSTER_METADATA.deserialize(response);
|
|
955
|
-
if (metadata.nodes.length <= 1)
|
|
956
|
-
return undefined;
|
|
957
|
-
const leader = metadata.nodes.find((node) => node.role === 'Leader' && node.status === 'Healthy');
|
|
958
|
-
if (leader)
|
|
959
|
-
return { host: leader.ip, port: leader.endpoints.tcp };
|
|
1036
|
+
await waitWithin(this.authenticationPromise ?? this.sessionAction, deadline, signal);
|
|
960
1037
|
}
|
|
961
1038
|
catch (error) {
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
}
|
|
965
|
-
if (Date.now() >= deadline) {
|
|
966
|
-
debug('cluster metadata named no healthy leader within ' +
|
|
967
|
-
`${this.leaderlessWaitBudget} ms, staying on this node`);
|
|
968
|
-
return undefined;
|
|
1039
|
+
signal.throwIfAborted();
|
|
1040
|
+
failure = error;
|
|
969
1041
|
}
|
|
970
|
-
await delay(this.leaderlessPollInterval);
|
|
971
1042
|
}
|
|
1043
|
+
if (lineage !== this.lineage)
|
|
1044
|
+
throw unauthenticated(command, failure);
|
|
1045
|
+
if (failure instanceof VsrResponseTimeoutError)
|
|
1046
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
1047
|
+
if (!this.isAuthenticated && !this.signInCredentials)
|
|
1048
|
+
throw unauthenticated(command, failure);
|
|
1049
|
+
if (failure !== undefined)
|
|
1050
|
+
throw failure;
|
|
1051
|
+
if (!this.isAuthenticated)
|
|
1052
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
972
1053
|
}
|
|
973
1054
|
/**
|
|
974
|
-
*
|
|
975
|
-
*
|
|
976
|
-
* @param err - Error to reject all queued commands with
|
|
1055
|
+
* The `sessionReset` event says whether the reset ended a signed-in session.
|
|
1056
|
+
* A reset after a refused sign-in ends none.
|
|
977
1057
|
*/
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
this.
|
|
984
|
-
this.
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
1058
|
+
resetSession() {
|
|
1059
|
+
const signedIn = this.isAuthenticated;
|
|
1060
|
+
const hadActivity = signedIn || this.vsrSession.hasActivity;
|
|
1061
|
+
this.routingGeneration += 1;
|
|
1062
|
+
this.sessionLifetime.abort(responseError(0, TRANSIENT_NOT_ACCEPTED));
|
|
1063
|
+
this.sessionLifetime = new AbortController();
|
|
1064
|
+
this.lanes.reset();
|
|
1065
|
+
for (const slot of this.replicas.values()) {
|
|
1066
|
+
slot.lifetime.abort();
|
|
1067
|
+
slot.connection?._destroy();
|
|
1068
|
+
}
|
|
1069
|
+
this.replicas.clear();
|
|
1070
|
+
this.bootstrap?._destroy();
|
|
1071
|
+
this.bootstrap = undefined;
|
|
1072
|
+
this.bootstrapPromise = undefined;
|
|
1073
|
+
this.rosterPromise = undefined;
|
|
1074
|
+
this.partitionRoutes.clear();
|
|
1075
|
+
this.metadataReplica = undefined;
|
|
989
1076
|
this.isAuthenticated = false;
|
|
990
1077
|
this.userId = undefined;
|
|
991
1078
|
this.vsrSession.reset();
|
|
992
|
-
this.
|
|
1079
|
+
clearInterval(this.heartbeatIntervalHandler);
|
|
1080
|
+
this.heartbeatIntervalHandler = undefined;
|
|
1081
|
+
this.changed.pulse();
|
|
1082
|
+
if (hadActivity)
|
|
1083
|
+
this.emit('sessionReset', signedIn);
|
|
1084
|
+
}
|
|
1085
|
+
ensureLineage(lineage, command) {
|
|
1086
|
+
if (lineage !== this.lineage)
|
|
1087
|
+
throw responseError(command, UNAUTHENTICATED);
|
|
1088
|
+
}
|
|
1089
|
+
ensureGeneration(generation, command) {
|
|
1090
|
+
if (generation !== this.routingGeneration || this.lifetime.signal.aborted)
|
|
1091
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
1092
|
+
}
|
|
1093
|
+
observeMetadata(frame) {
|
|
1094
|
+
const operation = readReplyOperation(frame);
|
|
1095
|
+
if (!hasCommittedResult(frame) || !(isMetadata(operation) || operation === Operation.DeleteSegments || operation === Operation.Register))
|
|
1096
|
+
return;
|
|
1097
|
+
const commit = frame.readBigUInt64LE(REPLY_OFFSET.commit);
|
|
1098
|
+
if (commit > this.metadataWatermark)
|
|
1099
|
+
this.metadataWatermark = commit;
|
|
1100
|
+
}
|
|
1101
|
+
_rememberCredentials(command, payload) {
|
|
1102
|
+
if (command === LOGIN.code) {
|
|
1103
|
+
const username = readWireName(payload, 0);
|
|
1104
|
+
this.signInCredentials = {
|
|
1105
|
+
username: username.value, password: readWireName(payload, username.next).value
|
|
1106
|
+
};
|
|
1107
|
+
}
|
|
1108
|
+
else if (command === LOGIN_WITH_TOKEN.code) {
|
|
1109
|
+
this.signInCredentials = { token: readWireName(payload, 0).value };
|
|
1110
|
+
}
|
|
1111
|
+
else if ((command === COMMAND_CODE.ChangePassword ||
|
|
1112
|
+
(command === COMMAND_CODE.UpdateResource && payload[0] === OptionsScope.User)) &&
|
|
1113
|
+
this.userId !== undefined && this.signInCredentials &&
|
|
1114
|
+
'username' in this.signInCredentials) {
|
|
1115
|
+
const credentials = this.signInCredentials;
|
|
1116
|
+
// An update leads with its scope byte, a password change with the user id.
|
|
1117
|
+
const start = command === COMMAND_CODE.ChangePassword ? 0 : 1;
|
|
1118
|
+
const identifier = [serializeIdentifier(this.userId), serializeIdentifier(credentials.username)]
|
|
1119
|
+
.find((encoded) => payload.subarray(start, start + encoded.length).equals(encoded));
|
|
1120
|
+
if (!identifier)
|
|
1121
|
+
return;
|
|
1122
|
+
if (command === COMMAND_CODE.ChangePassword) {
|
|
1123
|
+
const currentPassword = readWireName(payload, identifier.length);
|
|
1124
|
+
const password = readWireName(payload, currentPassword.next).value;
|
|
1125
|
+
this.signInCredentials = { ...credentials, password };
|
|
1126
|
+
return;
|
|
1127
|
+
}
|
|
1128
|
+
let position = start + identifier.length;
|
|
1129
|
+
const resetCount = payload.readUInt8(position);
|
|
1130
|
+
position += 1;
|
|
1131
|
+
for (let index = 0; index < resetCount; index++)
|
|
1132
|
+
position = readWireName(payload, position).next;
|
|
1133
|
+
const username = deserializeOptions(payload, position).username;
|
|
1134
|
+
if (typeof username !== 'string')
|
|
1135
|
+
return;
|
|
1136
|
+
this.signInCredentials = { ...credentials, username };
|
|
1137
|
+
}
|
|
993
1138
|
}
|
|
994
1139
|
hold() {
|
|
995
1140
|
this.pendingSubmissions += 1;
|
|
@@ -999,146 +1144,225 @@ export class CommandResponseStream extends EventEmitter {
|
|
|
999
1144
|
return;
|
|
1000
1145
|
released = true;
|
|
1001
1146
|
this.pendingSubmissions -= 1;
|
|
1002
|
-
this.
|
|
1147
|
+
if (this.pendingSubmissions === 0)
|
|
1148
|
+
this.emit('finishQueue');
|
|
1003
1149
|
};
|
|
1004
1150
|
}
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1151
|
+
async authenticate(credentials) {
|
|
1152
|
+
if (credentials && typeof credentials === 'object' && 'token' in credentials)
|
|
1153
|
+
await this._authWithToken(credentials);
|
|
1154
|
+
else
|
|
1155
|
+
await this._authWithPassword(credentials);
|
|
1156
|
+
return true;
|
|
1157
|
+
}
|
|
1158
|
+
async loginByCaller(command, payload, options = {}) {
|
|
1159
|
+
const lineage = ++this.sessionLineage;
|
|
1160
|
+
this.signInCredentials = undefined;
|
|
1161
|
+
const release = this.hold();
|
|
1162
|
+
const deadline = options.deadline ?? Date.now() + OPERATION_TIMEOUT_MS;
|
|
1163
|
+
const signal = AbortSignal.any([this.lifetime.signal, ...(options.signal ? [options.signal] : [])]);
|
|
1017
1164
|
try {
|
|
1018
|
-
|
|
1165
|
+
const body = payload();
|
|
1166
|
+
let admit;
|
|
1167
|
+
const admitted = new Promise((resolve) => { admit = resolve; });
|
|
1168
|
+
const pending = this.serializeSession(() => {
|
|
1169
|
+
admit();
|
|
1170
|
+
return this.login(command, body, deadline, signal, lineage);
|
|
1171
|
+
});
|
|
1172
|
+
const authentication = pending.then(() => true)
|
|
1173
|
+
.finally(() => { if (this.authenticationPromise === authentication)
|
|
1174
|
+
this.authenticationPromise = undefined; });
|
|
1175
|
+
this.authenticationPromise = authentication;
|
|
1176
|
+
void authentication.catch(() => undefined);
|
|
1177
|
+
// Bound the queue wait separately so dial timeouts keep their last transport error.
|
|
1178
|
+
await waitWithin(admitted, deadline, signal);
|
|
1179
|
+
return await pending;
|
|
1180
|
+
}
|
|
1181
|
+
catch (error) {
|
|
1182
|
+
if (lineage === this.lineage) {
|
|
1183
|
+
this.sessionLineage += 1;
|
|
1184
|
+
this.signInCredentials = undefined;
|
|
1185
|
+
this.resetSession();
|
|
1186
|
+
}
|
|
1187
|
+
if (error instanceof VsrResponseTimeoutError)
|
|
1188
|
+
throw responseError(command, TRANSIENT_NOT_ACCEPTED);
|
|
1189
|
+
throw error;
|
|
1019
1190
|
}
|
|
1020
1191
|
finally {
|
|
1021
|
-
|
|
1192
|
+
release();
|
|
1022
1193
|
}
|
|
1023
1194
|
}
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
1028
|
-
this.
|
|
1029
|
-
|
|
1030
|
-
|
|
1195
|
+
/** Only SDK recovery shares a sign-in; callers always own a fresh attempt. */
|
|
1196
|
+
signIn() {
|
|
1197
|
+
if (this.isAuthenticated)
|
|
1198
|
+
return Promise.resolve(true);
|
|
1199
|
+
if (!this.authenticationPromise) {
|
|
1200
|
+
const lineage = this.lineage;
|
|
1201
|
+
const identity = this.signInCredentials;
|
|
1202
|
+
const pending = this.serializeSession(async () => {
|
|
1203
|
+
this.ensureLineage(lineage, LOGIN.code);
|
|
1204
|
+
if (this.isAuthenticated)
|
|
1205
|
+
return true;
|
|
1206
|
+
if (!identity)
|
|
1207
|
+
return false;
|
|
1208
|
+
const command = 'token' in identity ? LOGIN_WITH_TOKEN : LOGIN;
|
|
1209
|
+
try {
|
|
1210
|
+
const body = 'token' in identity ? LOGIN_WITH_TOKEN.serialize(identity) : LOGIN.serialize(identity);
|
|
1211
|
+
await this.login(command.code, body, Date.now() + OPERATION_TIMEOUT_MS, this.lifetime.signal, lineage);
|
|
1212
|
+
return this.isAuthenticated;
|
|
1213
|
+
}
|
|
1214
|
+
catch (error) {
|
|
1215
|
+
if (lineage === this.lineage && (error instanceof TypeError || error instanceof VsrEvictionError ||
|
|
1216
|
+
(error instanceof ResponseError && !isTransient(error.errorCode))))
|
|
1217
|
+
this.signInCredentials = undefined;
|
|
1218
|
+
throw error;
|
|
1219
|
+
}
|
|
1220
|
+
}).finally(() => { if (this.authenticationPromise === pending)
|
|
1221
|
+
this.authenticationPromise = undefined; });
|
|
1222
|
+
this.authenticationPromise = pending;
|
|
1223
|
+
}
|
|
1224
|
+
return this.authenticationPromise;
|
|
1031
1225
|
}
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
*
|
|
1035
|
-
* @param creds - Password credentials
|
|
1036
|
-
* @returns Login response with user ID
|
|
1037
|
-
*/
|
|
1038
|
-
async _authWithPassword(creds) {
|
|
1039
|
-
const pl = LOGIN.serialize(creds);
|
|
1040
|
-
const logr = await this.sendCommand(LOGIN.code, pl, { last: false });
|
|
1041
|
-
return LOGIN.deserialize(logr);
|
|
1226
|
+
async _authWithPassword(credentials) {
|
|
1227
|
+
return LOGIN.deserialize(await this.loginByCaller(LOGIN.code, () => LOGIN.serialize(credentials)));
|
|
1042
1228
|
}
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
*
|
|
1046
|
-
* @param creds - Token credentials
|
|
1047
|
-
* @returns Login response with user ID
|
|
1048
|
-
*/
|
|
1049
|
-
async _authWithToken(creds) {
|
|
1050
|
-
const pl = LOGIN_WITH_TOKEN.serialize(creds);
|
|
1051
|
-
const logr = await this.sendCommand(LOGIN_WITH_TOKEN.code, pl, { last: false });
|
|
1052
|
-
return LOGIN_WITH_TOKEN.deserialize(logr);
|
|
1229
|
+
async _authWithToken(credentials) {
|
|
1230
|
+
return LOGIN_WITH_TOKEN.deserialize(await this.loginByCaller(LOGIN_WITH_TOKEN.code, () => LOGIN_WITH_TOKEN.serialize(credentials)));
|
|
1053
1231
|
}
|
|
1054
|
-
/**
|
|
1055
|
-
* Sends a ping command to the server.
|
|
1056
|
-
*
|
|
1057
|
-
* @returns Ping response
|
|
1058
|
-
*/
|
|
1059
1232
|
async ping() {
|
|
1060
|
-
|
|
1061
|
-
const pingR = await this.sendCommand(PING.code, pl);
|
|
1062
|
-
return PING.deserialize(pingR);
|
|
1233
|
+
return PING.deserialize(await this.sendCommand(PING.code, PING.serialize()));
|
|
1063
1234
|
}
|
|
1064
|
-
/**
|
|
1065
|
-
* Starts sending periodic heartbeat pings to keep the connection alive.
|
|
1066
|
-
*
|
|
1067
|
-
* @param interval - Heartbeat interval in milliseconds
|
|
1068
|
-
*/
|
|
1069
1235
|
heartbeat(interval) {
|
|
1070
|
-
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
this.
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1236
|
+
clearInterval(this.heartbeatIntervalHandler);
|
|
1237
|
+
this.heartbeatIntervalHandler = undefined;
|
|
1238
|
+
if (interval && this.isAuthenticated) {
|
|
1239
|
+
this.heartbeatIntervalHandler = setInterval(() => {
|
|
1240
|
+
for (const slot of this.replicas.values()) {
|
|
1241
|
+
// A retry budget limits one reconnect burst, not the life of the
|
|
1242
|
+
// session, so a long outage cannot leave the client down for good.
|
|
1243
|
+
if (slot.restartable) {
|
|
1244
|
+
slot.restartable = false;
|
|
1245
|
+
slot.failure = undefined;
|
|
1246
|
+
void this.runReplica(slot);
|
|
1247
|
+
}
|
|
1248
|
+
if (!slot.attached || !slot.connection?.connected || slot.heartbeat)
|
|
1249
|
+
continue;
|
|
1250
|
+
slot.heartbeat = true;
|
|
1251
|
+
const signal = AbortSignal.any([this.lifetime.signal, this.sessionLifetime.signal, slot.lifetime.signal]);
|
|
1252
|
+
void this.control(slot.connection, PING.code, PING.serialize(), Date.now() + OPERATION_TIMEOUT_MS, signal)
|
|
1253
|
+
.then(() => { this.emit('heartbeat'); })
|
|
1254
|
+
.catch((error) => {
|
|
1255
|
+
if (isSessionFailure(error))
|
|
1256
|
+
this.recoverSession(slot.generation);
|
|
1257
|
+
debug('replica heartbeat failed', error);
|
|
1258
|
+
}).finally(() => { slot.heartbeat = false; });
|
|
1085
1259
|
}
|
|
1086
|
-
}
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
// that
|
|
1090
|
-
this.
|
|
1260
|
+
}, interval);
|
|
1261
|
+
this.heartbeatIntervalHandler.unref();
|
|
1262
|
+
}
|
|
1263
|
+
// A link that used its budget reports whether a heartbeat restarts it.
|
|
1264
|
+
for (const slot of this.replicas.values())
|
|
1265
|
+
if (slot.restartable)
|
|
1266
|
+
slot.failure = this.budgetFailure(slot);
|
|
1091
1267
|
}
|
|
1092
|
-
/**
|
|
1093
|
-
* Returns the underlying socket as a readable stream.
|
|
1094
|
-
*
|
|
1095
|
-
* @returns The connection socket
|
|
1096
|
-
*/
|
|
1097
1268
|
getReadStream() {
|
|
1098
|
-
|
|
1269
|
+
const connection = this.bootstrap ?? [...this.replicas.values()].find((slot) => slot.attached)?.connection;
|
|
1270
|
+
if (!connection?.socket)
|
|
1271
|
+
throw new Error('client has no connected replica');
|
|
1272
|
+
return connection.socket;
|
|
1099
1273
|
}
|
|
1100
|
-
/**
|
|
1101
|
-
* Destroys the stream and cleans up resources.
|
|
1102
|
-
* Stops heartbeat and destroys the connection.
|
|
1103
|
-
*/
|
|
1104
1274
|
destroy() {
|
|
1105
|
-
this.
|
|
1106
|
-
this.
|
|
1107
|
-
|
|
1108
|
-
clearInterval(this.heartbeatIntervalHandler);
|
|
1109
|
-
return this.connection._destroy();
|
|
1275
|
+
this.signInCredentials = undefined;
|
|
1276
|
+
this.lifetime.abort(new Error('client destroyed'));
|
|
1277
|
+
this.resetSession();
|
|
1110
1278
|
}
|
|
1111
1279
|
}
|
|
1112
|
-
;
|
|
1113
|
-
/**
|
|
1114
|
-
* Creates a new RawClient instance.
|
|
1115
|
-
*
|
|
1116
|
-
* @param options - Client configuration
|
|
1117
|
-
* @returns RawClient instance
|
|
1118
|
-
*/
|
|
1119
1280
|
export function getRawClient(options) {
|
|
1120
1281
|
return new CommandResponseStream(options);
|
|
1121
1282
|
}
|
|
1122
|
-
const isLoginCommand = (command) => command === COMMAND_CODE.LoginUser ||
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
const
|
|
1126
|
-
|
|
1127
|
-
|
|
1128
|
-
|
|
1129
|
-
|
|
1130
|
-
|
|
1131
|
-
|
|
1132
|
-
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1283
|
+
const isLoginCommand = (command) => command === COMMAND_CODE.LoginUser || command === COMMAND_CODE.LoginWithAccessToken;
|
|
1284
|
+
const isTransient = (code) => code === TRANSIENT_NOT_COMMITTED || code === TRANSIENT_NOT_ACCEPTED || code === NOT_PRIMARY;
|
|
1285
|
+
/** A 40 that keeps why the call has no session, such as a refused sign-in. */
|
|
1286
|
+
const unauthenticated = (command, cause) => {
|
|
1287
|
+
const error = responseError(command, UNAUTHENTICATED);
|
|
1288
|
+
if (cause !== undefined)
|
|
1289
|
+
error.cause = cause;
|
|
1290
|
+
return error;
|
|
1291
|
+
};
|
|
1292
|
+
/** An unknown outcome. Only a retainable operation has an ID to resolve, any other call ends with 57. */
|
|
1293
|
+
const uncertainOutcome = (command, operation, cause) => {
|
|
1294
|
+
if (operation?.retainable)
|
|
1295
|
+
return new OperationUncertainError(operation.id, cause);
|
|
1296
|
+
const error = responseError(command, TRANSIENT_NOT_COMMITTED);
|
|
1297
|
+
error.cause = cause;
|
|
1298
|
+
return error;
|
|
1299
|
+
};
|
|
1300
|
+
const isDetachedFailure = (error) => error instanceof VsrEvictionError
|
|
1301
|
+
? error.reason === EvictionReason.NoSession || error.reason === EvictionReason.StaleClient
|
|
1302
|
+
: error instanceof ResponseError && error.fromServer && !error.committed &&
|
|
1303
|
+
(error.errorCode === UNAUTHENTICATED || error.errorCode === STALE_CLIENT);
|
|
1304
|
+
const isSessionFailure = (error, binding = false) => (error instanceof VsrEvictionError && (binding || !isDetachedFailure(error))) ||
|
|
1305
|
+
(error instanceof ResponseError && error.fromServer &&
|
|
1306
|
+
((error.errorCode === UNAUTHENTICATED && (binding || error.committed)) ||
|
|
1307
|
+
(error.errorCode === STALE_CLIENT && error.committed)));
|
|
1308
|
+
const groupKey = (context, target) => `${context.incarnation}:${target.query.readUInt32LE(target.query.length - PARTITION_ID_SIZE)}`;
|
|
1309
|
+
const partitionRoute = (command, payload) => {
|
|
1310
|
+
if (command === COMMAND_CODE.SendMessages) {
|
|
1311
|
+
const { query } = sendTarget(payload);
|
|
1312
|
+
return { query, key: query.toString('hex') };
|
|
1137
1313
|
}
|
|
1138
|
-
|
|
1139
|
-
|
|
1314
|
+
if (command !== COMMAND_CODE.PollMessages && command !== COMMAND_CODE.GetOffset &&
|
|
1315
|
+
command !== COMMAND_CODE.StoreOffset && command !== COMMAND_CODE.DeleteConsumerOffset)
|
|
1316
|
+
return undefined;
|
|
1317
|
+
let position = 1;
|
|
1318
|
+
const consumerEnd = identifierEnd(payload, position, command);
|
|
1319
|
+
const streamEnd = identifierEnd(payload, consumerEnd, command);
|
|
1320
|
+
position = identifierEnd(payload, streamEnd, command);
|
|
1321
|
+
const suffix = command === COMMAND_CODE.PollMessages ? POLL_OPTIONS_SIZE
|
|
1322
|
+
: command === COMMAND_CODE.StoreOffset ? STORE_OFFSET_SUFFIX_SIZE
|
|
1323
|
+
: command === COMMAND_CODE.DeleteConsumerOffset ? DELETE_OFFSET_SUFFIX_SIZE : 0;
|
|
1324
|
+
if (payload.length !== position + 1 + PARTITION_ID_SIZE + suffix || payload[position] > 1)
|
|
1325
|
+
throw responseError(command, INVALID_COMMAND);
|
|
1326
|
+
if (payload[position] === 0) {
|
|
1327
|
+
if (command === COMMAND_CODE.StoreOffset || command === COMMAND_CODE.DeleteConsumerOffset ||
|
|
1328
|
+
(command === COMMAND_CODE.PollMessages && payload[0] === ConsumerKind.Group))
|
|
1329
|
+
throw responseError(command, INVALID_COMMAND);
|
|
1330
|
+
}
|
|
1331
|
+
const query = Buffer.concat([payload.subarray(consumerEnd, position),
|
|
1332
|
+
payload[position] === 0 ? Buffer.alloc(PARTITION_ID_SIZE)
|
|
1333
|
+
: payload.subarray(position + 1, position + 1 + PARTITION_ID_SIZE)]);
|
|
1334
|
+
return { query, key: query.toString('hex') };
|
|
1335
|
+
};
|
|
1336
|
+
const identifierEnd = (payload, position, command) => {
|
|
1337
|
+
if (position + IDENTIFIER_PREFIX_SIZE > payload.length)
|
|
1338
|
+
throw responseError(command, INVALID_COMMAND);
|
|
1339
|
+
const end = position + IDENTIFIER_PREFIX_SIZE + payload[position + 1];
|
|
1340
|
+
if (end > payload.length)
|
|
1341
|
+
throw responseError(command, INVALID_COMMAND);
|
|
1342
|
+
return end;
|
|
1343
|
+
};
|
|
1344
|
+
/**
|
|
1345
|
+
* Reads the explicit partition a raw send writes to. A Balanced or MessageKey
|
|
1346
|
+
* send names no partition, so there is no context to capture for it, and the
|
|
1347
|
+
* server refuses a send without one.
|
|
1348
|
+
*/
|
|
1349
|
+
const sendTarget = (payload) => {
|
|
1350
|
+
let position = SEND_METADATA_LENGTH_SIZE;
|
|
1351
|
+
for (let identifier = 0; identifier < 2; identifier++) {
|
|
1352
|
+
if (position + IDENTIFIER_PREFIX_SIZE > payload.length)
|
|
1353
|
+
throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND);
|
|
1354
|
+
position += IDENTIFIER_PREFIX_SIZE + payload[position + 1];
|
|
1140
1355
|
}
|
|
1356
|
+
if (position + IDENTIFIER_PREFIX_SIZE > payload.length)
|
|
1357
|
+
throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND);
|
|
1358
|
+
if (payload[position] !== PartitionKind.PartitionId)
|
|
1359
|
+
throw responseError(COMMAND_CODE.SendMessages, FEATURE_UNAVAILABLE, RAW_SEND_NEEDS_PARTITION);
|
|
1360
|
+
const partition = position + IDENTIFIER_PREFIX_SIZE;
|
|
1361
|
+
if (payload[position + 1] !== PARTITION_ID_SIZE || partition + PARTITION_ID_SIZE > payload.length)
|
|
1362
|
+
throw responseError(COMMAND_CODE.SendMessages, INVALID_COMMAND);
|
|
1363
|
+
const identifiers = payload.subarray(SEND_METADATA_LENGTH_SIZE, position);
|
|
1364
|
+
return {
|
|
1365
|
+
query: Buffer.concat([identifiers, payload.subarray(partition, partition + PARTITION_ID_SIZE)])
|
|
1366
|
+
};
|
|
1141
1367
|
};
|
|
1142
|
-
const isTransientVsrError = (errorCode) => errorCode === TRANSIENT_NOT_COMMITTED ||
|
|
1143
|
-
errorCode === TRANSIENT_NOT_ACCEPTED;
|
|
1144
1368
|
//# sourceMappingURL=client.socket.js.map
|