@laserdata/laser-sdk 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +49 -3
- package/dist/agent/reliable-consumer.d.ts.map +1 -1
- package/dist/agent/reliable-consumer.js +5 -1
- package/dist/agent/reliable-consumer.js.map +1 -1
- package/dist/bridges/a2a.d.ts +1 -1
- package/dist/bridges/a2a.js +1 -1
- package/dist/client/capabilities.d.ts +15 -2
- package/dist/client/capabilities.d.ts.map +1 -1
- package/dist/client/capabilities.js +22 -1
- package/dist/client/capabilities.js.map +1 -1
- package/dist/client/connect-options.d.ts +5 -0
- package/dist/client/connect-options.d.ts.map +1 -0
- package/dist/client/connect-options.js +18 -0
- package/dist/client/connect-options.js.map +1 -0
- package/dist/client/errors.d.ts +23 -1
- package/dist/client/errors.d.ts.map +1 -1
- package/dist/client/errors.js +34 -0
- package/dist/client/errors.js.map +1 -1
- package/dist/client/laser.d.ts +17 -1
- package/dist/client/laser.d.ts.map +1 -1
- package/dist/client/laser.js +96 -26
- package/dist/client/laser.js.map +1 -1
- package/dist/client/managed.d.ts +1 -0
- package/dist/client/managed.d.ts.map +1 -1
- package/dist/client/managed.js +15 -2
- package/dist/client/managed.js.map +1 -1
- package/dist/iggy/apache-iggy.d.ts +69 -4
- package/dist/iggy/apache-iggy.d.ts.map +1 -1
- package/dist/iggy/apache-iggy.js +416 -23
- package/dist/iggy/apache-iggy.js.map +1 -1
- package/dist/index.d.ts +8 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +4 -1
- package/dist/index.js.map +1 -1
- package/dist/managed/filters.d.ts +358 -0
- package/dist/managed/filters.d.ts.map +1 -0
- package/dist/managed/filters.js +1913 -0
- package/dist/managed/filters.js.map +1 -0
- package/dist/stream/consumer.d.ts +11 -1
- package/dist/stream/consumer.d.ts.map +1 -1
- package/dist/stream/consumer.js +82 -9
- package/dist/stream/consumer.js.map +1 -1
- package/dist/stream/stream.d.ts +4 -1
- package/dist/stream/stream.d.ts.map +1 -1
- package/dist/stream/stream.js +11 -1
- package/dist/stream/stream.js.map +1 -1
- package/dist/stream/topic.d.ts +2 -0
- package/dist/stream/topic.d.ts.map +1 -1
- package/dist/stream/topic.js +6 -0
- package/dist/stream/topic.js.map +1 -1
- package/dist/wire/authz.d.ts +1 -1
- package/dist/wire/authz.d.ts.map +1 -1
- package/dist/wire/authz.js +12 -3
- package/dist/wire/authz.js.map +1 -1
- package/dist/wire/codes.d.ts +17 -0
- package/dist/wire/codes.d.ts.map +1 -1
- package/dist/wire/codes.js +17 -0
- package/dist/wire/codes.js.map +1 -1
- package/dist/wire/commands.d.ts +12 -1
- package/dist/wire/commands.d.ts.map +1 -1
- package/dist/wire/commands.js +13 -1
- package/dist/wire/commands.js.map +1 -1
- package/dist/wire/control.d.ts +4 -0
- package/dist/wire/control.d.ts.map +1 -1
- package/dist/wire/control.js +5 -0
- package/dist/wire/control.js.map +1 -1
- package/dist/wire/filter-codecs.d.ts +16 -0
- package/dist/wire/filter-codecs.d.ts.map +1 -0
- package/dist/wire/filter-codecs.js +519 -0
- package/dist/wire/filter-codecs.js.map +1 -0
- package/dist/wire/filter-eval.d.ts +75 -0
- package/dist/wire/filter-eval.d.ts.map +1 -0
- package/dist/wire/filter-eval.js +899 -0
- package/dist/wire/filter-eval.js.map +1 -0
- package/dist/wire/filter.d.ts +772 -0
- package/dist/wire/filter.d.ts.map +1 -0
- package/dist/wire/filter.js +2869 -0
- package/dist/wire/filter.js.map +1 -0
- package/dist/wire/hello.d.ts +18 -0
- package/dist/wire/hello.d.ts.map +1 -1
- package/dist/wire/hello.js +29 -3
- package/dist/wire/hello.js.map +1 -1
- package/dist/wire/http.d.ts +22 -0
- package/dist/wire/http.d.ts.map +1 -1
- package/dist/wire/http.js +54 -47
- package/dist/wire/http.js.map +1 -1
- package/dist/wire/limits.d.ts +25 -0
- package/dist/wire/limits.d.ts.map +1 -1
- package/dist/wire/limits.js +25 -0
- package/dist/wire/limits.js.map +1 -1
- package/dist/wire/result.d.ts +4 -0
- package/dist/wire/result.d.ts.map +1 -1
- package/dist/wire/result.js +50 -0
- package/dist/wire/result.js.map +1 -1
- package/package.json +6 -6
package/dist/iggy/apache-iggy.js
CHANGED
|
@@ -1,8 +1,9 @@
|
|
|
1
|
+
import { connectOptions } from "../client/connect-options.js";
|
|
1
2
|
import { publishOptions, publishWithin } from "../client/publish-options.js";
|
|
2
|
-
import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy, SimpleClient, getRawClient } from "apache-iggy";
|
|
3
|
+
import { Consumer, DeserializeError, HeaderKeyFactory, HeaderValue as IggyHeaderValueFactory, Partitioning, PollingStrategy as IggyPollingStrategy, ResponseError, SimpleClient, getRawClient } from "apache-iggy";
|
|
3
4
|
import { readFileSync } from "node:fs";
|
|
4
5
|
import { isIP } from "node:net";
|
|
5
|
-
import { AmbiguousMutationError, ConfigError, TimeoutError, TransportError } from "../client/errors.js";
|
|
6
|
+
import { AmbiguousMutationError, ConfigError, ProtocolError, TimeoutError, TransportError, UnsupportedError } from "../client/errors.js";
|
|
6
7
|
import { LASERDATA_ROOT_CA } from "../client/laserdata-ca.js";
|
|
7
8
|
import { Mutex } from "../runtime/mutex.js";
|
|
8
9
|
import { mintUlidValue } from "../runtime/ulid.js";
|
|
@@ -10,12 +11,21 @@ import { encodeNamed } from "../wire/cbor.js";
|
|
|
10
11
|
import { isIdempotentManagedRequest } from "../wire/codes.js";
|
|
11
12
|
import { MANAGED_REQUEST_VERSION, encodeManagedRequestEnvelope } from "../wire/mutation.js";
|
|
12
13
|
const DEFAULT_RECONNECT_INTERVAL_MS = 1_000;
|
|
14
|
+
const ACCEPT_STAGE = "Iggy server to accept the connection";
|
|
15
|
+
const LOGIN_STAGE = "Iggy login reply";
|
|
13
16
|
const VSR_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
14
17
|
const TRANSIENT_NOT_COMMITTED = 57;
|
|
15
18
|
const TRANSIENT_NOT_ACCEPTED = 58;
|
|
19
|
+
const POLLED_HEAD_BYTES = 16;
|
|
20
|
+
const BATCH_HEADER_BYTES = 256;
|
|
21
|
+
const FRAME_HEADER_BYTES = 48;
|
|
22
|
+
const STRING_HEADER_KIND = 2;
|
|
23
|
+
const CONTENT_TYPE_HEADER = "agdx.ct";
|
|
24
|
+
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true });
|
|
16
25
|
export function toNodeBuffer(bytes) {
|
|
17
26
|
return Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
18
27
|
}
|
|
28
|
+
const SYNC_CONSUMER_GROUP_CODE = 606;
|
|
19
29
|
function toIggyConsumer(target) {
|
|
20
30
|
return target.kind === "single"
|
|
21
31
|
? target.name === undefined
|
|
@@ -186,6 +196,162 @@ function parsedHeadersToMap(entries) {
|
|
|
186
196
|
}
|
|
187
197
|
return map;
|
|
188
198
|
}
|
|
199
|
+
/** The Iggy error code a server reply carried, through any transport wrapping. */
|
|
200
|
+
export function serverErrorCode(error) {
|
|
201
|
+
for (let current = error; current !== undefined;) {
|
|
202
|
+
if (current instanceof ResponseError)
|
|
203
|
+
return current.errorCode;
|
|
204
|
+
current = current instanceof Error ? current.cause : undefined;
|
|
205
|
+
}
|
|
206
|
+
return undefined;
|
|
207
|
+
}
|
|
208
|
+
/**
|
|
209
|
+
* The records of a standard polled-messages body, decoded exactly as stored:
|
|
210
|
+
* original offsets, per-record microsecond timestamps, payloads, and typed
|
|
211
|
+
* user headers.
|
|
212
|
+
*/
|
|
213
|
+
export function decodePolledBody(body) {
|
|
214
|
+
const malformed = (what) => new ProtocolError(`the polled record body has a malformed ${what}`);
|
|
215
|
+
if (body.byteLength < POLLED_HEAD_BYTES)
|
|
216
|
+
throw malformed("head");
|
|
217
|
+
const view = new DataView(body.buffer, body.byteOffset, body.byteLength);
|
|
218
|
+
const partitionId = view.getUint32(0, true);
|
|
219
|
+
const messages = [];
|
|
220
|
+
let position = POLLED_HEAD_BYTES;
|
|
221
|
+
while (position < body.byteLength) {
|
|
222
|
+
if (position + BATCH_HEADER_BYTES > body.byteLength)
|
|
223
|
+
throw malformed("batch header");
|
|
224
|
+
const baseOffset = view.getBigUint64(position + 8, true);
|
|
225
|
+
const baseTimestamp = view.getBigUint64(position + 16, true);
|
|
226
|
+
const batchLength = view.getBigUint64(position + 32, true);
|
|
227
|
+
if (batchLength < BigInt(BATCH_HEADER_BYTES) ||
|
|
228
|
+
batchLength > BigInt(body.byteLength - position))
|
|
229
|
+
throw malformed("batch length");
|
|
230
|
+
const batchEnd = position + Number(batchLength);
|
|
231
|
+
position += BATCH_HEADER_BYTES;
|
|
232
|
+
while (position < batchEnd) {
|
|
233
|
+
if (position + FRAME_HEADER_BYTES > batchEnd)
|
|
234
|
+
throw malformed("message frame");
|
|
235
|
+
const offsetDelta = view.getUint32(position + 24, true);
|
|
236
|
+
const headersLength = view.getUint32(position + 32, true);
|
|
237
|
+
const payloadLength = view.getUint32(position + 36, true);
|
|
238
|
+
const payloadStart = position + FRAME_HEADER_BYTES;
|
|
239
|
+
const headersStart = payloadStart + payloadLength;
|
|
240
|
+
const frameEnd = headersStart + headersLength;
|
|
241
|
+
if (frameEnd > batchEnd)
|
|
242
|
+
throw malformed("message frame");
|
|
243
|
+
const block = body.subarray(headersStart, frameEnd);
|
|
244
|
+
const headers = decodeUserHeaders(block);
|
|
245
|
+
const contentType = headers === "entry" ? decodeUserHeaders(block, true) : headers;
|
|
246
|
+
messages.push({
|
|
247
|
+
payload: body.subarray(payloadStart, headersStart),
|
|
248
|
+
partitionId,
|
|
249
|
+
offset: baseOffset + BigInt(offsetDelta),
|
|
250
|
+
timestampMicros: baseTimestamp,
|
|
251
|
+
...(typeof headers === "string"
|
|
252
|
+
? {
|
|
253
|
+
headers: typeof contentType === "string" ? new Map() : contentType,
|
|
254
|
+
headersMalformed: typeof contentType === "string" ? contentType : headers
|
|
255
|
+
}
|
|
256
|
+
: { headers })
|
|
257
|
+
});
|
|
258
|
+
position = frameEnd;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
return messages;
|
|
262
|
+
}
|
|
263
|
+
// The typed user headers as the server reads them for a filter: string keys
|
|
264
|
+
// only, a kind this build does not know kept raw, a value that does not fit
|
|
265
|
+
// its kind faulting the record, and the last of a repeated key kept. A structurally broken
|
|
266
|
+
// block names what did not decode, which the server treats as a malformed
|
|
267
|
+
// record when its filter needs that part.
|
|
268
|
+
function decodeUserHeaders(block, contentTypeOnly = false) {
|
|
269
|
+
const headers = new Map();
|
|
270
|
+
const view = new DataView(block.buffer, block.byteOffset, block.byteLength);
|
|
271
|
+
let position = 0;
|
|
272
|
+
while (position < block.byteLength) {
|
|
273
|
+
if (position + 5 > block.byteLength)
|
|
274
|
+
return "structure";
|
|
275
|
+
const keyKind = view.getUint8(position);
|
|
276
|
+
const keyStart = position + 5;
|
|
277
|
+
const keyEnd = keyStart + view.getUint32(position + 1, true);
|
|
278
|
+
if (keyKind === 0 ||
|
|
279
|
+
keyEnd === keyStart ||
|
|
280
|
+
keyEnd - keyStart > 255 ||
|
|
281
|
+
keyEnd + 5 > block.byteLength)
|
|
282
|
+
return "structure";
|
|
283
|
+
const valueKind = view.getUint8(keyEnd);
|
|
284
|
+
const valueStart = keyEnd + 5;
|
|
285
|
+
const valueEnd = valueStart + view.getUint32(keyEnd + 1, true);
|
|
286
|
+
if (valueKind === 0 ||
|
|
287
|
+
valueEnd === valueStart ||
|
|
288
|
+
valueEnd - valueStart > 255 ||
|
|
289
|
+
valueEnd > block.byteLength)
|
|
290
|
+
return "structure";
|
|
291
|
+
position = valueEnd;
|
|
292
|
+
if (keyKind !== STRING_HEADER_KIND)
|
|
293
|
+
continue;
|
|
294
|
+
if (contentTypeOnly &&
|
|
295
|
+
(keyEnd - keyStart !== CONTENT_TYPE_HEADER.length ||
|
|
296
|
+
CONTENT_TYPE_HEADER.split("").some((character, index) => block[keyStart + index] !== character.charCodeAt(0))))
|
|
297
|
+
continue;
|
|
298
|
+
let key;
|
|
299
|
+
try {
|
|
300
|
+
key = STRICT_UTF8.decode(block.subarray(keyStart, keyEnd));
|
|
301
|
+
}
|
|
302
|
+
catch {
|
|
303
|
+
return "entry";
|
|
304
|
+
}
|
|
305
|
+
const value = headerValueOf(valueKind, block.subarray(valueStart, valueEnd));
|
|
306
|
+
if (value === undefined)
|
|
307
|
+
return key === CONTENT_TYPE_HEADER ? "content_type" : "entry";
|
|
308
|
+
headers.set(key, value);
|
|
309
|
+
}
|
|
310
|
+
return headers;
|
|
311
|
+
}
|
|
312
|
+
function headerValueOf(kind, bytes) {
|
|
313
|
+
const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
314
|
+
const sized = (size) => bytes.byteLength === size;
|
|
315
|
+
switch (kind) {
|
|
316
|
+
case STRING_HEADER_KIND:
|
|
317
|
+
try {
|
|
318
|
+
return { kind: "string", value: STRICT_UTF8.decode(bytes) };
|
|
319
|
+
}
|
|
320
|
+
catch {
|
|
321
|
+
return undefined;
|
|
322
|
+
}
|
|
323
|
+
case 3:
|
|
324
|
+
return sized(1) && (bytes[0] === 0 || bytes[0] === 1)
|
|
325
|
+
? { kind: "bool", value: bytes[0] === 1 }
|
|
326
|
+
: undefined;
|
|
327
|
+
case 4:
|
|
328
|
+
return sized(1) ? { kind: "int8", value: view.getInt8(0) } : undefined;
|
|
329
|
+
case 5:
|
|
330
|
+
return sized(2) ? { kind: "int16", value: view.getInt16(0, true) } : undefined;
|
|
331
|
+
case 6:
|
|
332
|
+
return sized(4) ? { kind: "int32", value: view.getInt32(0, true) } : undefined;
|
|
333
|
+
case 7:
|
|
334
|
+
return sized(8) ? { kind: "int64", value: view.getBigInt64(0, true) } : undefined;
|
|
335
|
+
case 8:
|
|
336
|
+
return sized(16) ? { kind: "int128", value: bytes.slice() } : undefined;
|
|
337
|
+
case 9:
|
|
338
|
+
return sized(1) ? { kind: "uint8", value: view.getUint8(0) } : undefined;
|
|
339
|
+
case 10:
|
|
340
|
+
return sized(2) ? { kind: "uint16", value: view.getUint16(0, true) } : undefined;
|
|
341
|
+
case 11:
|
|
342
|
+
return sized(4) ? { kind: "uint32", value: view.getUint32(0, true) } : undefined;
|
|
343
|
+
case 12:
|
|
344
|
+
return sized(8) ? { kind: "uint64", value: view.getBigUint64(0, true) } : undefined;
|
|
345
|
+
case 13:
|
|
346
|
+
return sized(16) ? { kind: "uint128", value: bytes.slice() } : undefined;
|
|
347
|
+
case 14:
|
|
348
|
+
return sized(4) ? { kind: "float", value: view.getFloat32(0, true) } : undefined;
|
|
349
|
+
case 15:
|
|
350
|
+
return sized(8) ? { kind: "double", value: view.getFloat64(0, true) } : undefined;
|
|
351
|
+
default:
|
|
352
|
+
return { kind: "raw", value: bytes.slice() };
|
|
353
|
+
}
|
|
354
|
+
}
|
|
189
355
|
function laserDataHost(host) {
|
|
190
356
|
const normalized = host.toLowerCase();
|
|
191
357
|
return (normalized === "laserdata.cloud" ||
|
|
@@ -292,7 +458,7 @@ export function watchConnectionLoss(connection, onLost) {
|
|
|
292
458
|
onLost();
|
|
293
459
|
});
|
|
294
460
|
}
|
|
295
|
-
async function connectSimpleClient(parsed,
|
|
461
|
+
async function connectSimpleClient(parsed, deadline, pinned = false) {
|
|
296
462
|
const config = parsed.tls
|
|
297
463
|
? {
|
|
298
464
|
heartbeatInterval: VSR_HEARTBEAT_INTERVAL_MS,
|
|
@@ -300,7 +466,11 @@ async function connectSimpleClient(parsed, timeoutMs) {
|
|
|
300
466
|
options: {
|
|
301
467
|
port: parsed.port,
|
|
302
468
|
host: parsed.host,
|
|
303
|
-
...(
|
|
469
|
+
...(parsed.servername !== undefined
|
|
470
|
+
? { servername: parsed.servername }
|
|
471
|
+
: isIP(parsed.host) === 0
|
|
472
|
+
? { servername: parsed.host }
|
|
473
|
+
: {}),
|
|
304
474
|
...(parsed.ca !== undefined ? { ca: parsed.ca } : {})
|
|
305
475
|
},
|
|
306
476
|
credentials: parsed.credentials,
|
|
@@ -322,28 +492,96 @@ async function connectSimpleClient(parsed, timeoutMs) {
|
|
|
322
492
|
}
|
|
323
493
|
try {
|
|
324
494
|
const connection = raw.connection;
|
|
325
|
-
const client = new SimpleClient(raw);
|
|
495
|
+
const client = pinned ? pinnedClient(raw, deadline) : new SimpleClient(raw);
|
|
496
|
+
// The client emits `connect` only once the transport, TLS included, is up, so it
|
|
497
|
+
// separates a server that never accepts the socket from one that never answers login.
|
|
498
|
+
let accepted = false;
|
|
499
|
+
const markAccepted = () => {
|
|
500
|
+
accepted = true;
|
|
501
|
+
};
|
|
502
|
+
connection.once("connect", markAccepted);
|
|
326
503
|
const ready = new Promise((resolve, reject) => {
|
|
327
504
|
const failed = (cause) => {
|
|
328
505
|
reject(cause instanceof Error ? cause : new Error(String(cause)));
|
|
329
506
|
};
|
|
330
507
|
connection.once("error", failed);
|
|
331
|
-
|
|
508
|
+
const login = async () => {
|
|
509
|
+
if (pinned) {
|
|
510
|
+
const connection = raw.connection;
|
|
511
|
+
await connection.connect(true);
|
|
512
|
+
if ("token" in parsed.credentials)
|
|
513
|
+
await client.session.loginWithToken(parsed.credentials);
|
|
514
|
+
else
|
|
515
|
+
await client.session.login(parsed.credentials);
|
|
516
|
+
}
|
|
517
|
+
await client.client.getMe();
|
|
518
|
+
};
|
|
519
|
+
login().then(() => {
|
|
332
520
|
connection.off("error", failed);
|
|
333
521
|
resolve();
|
|
334
522
|
}, reject);
|
|
335
523
|
});
|
|
336
|
-
|
|
524
|
+
void ready.catch(() => undefined);
|
|
525
|
+
let timer;
|
|
526
|
+
try {
|
|
527
|
+
await (deadline === undefined
|
|
528
|
+
? ready
|
|
529
|
+
: Promise.race([
|
|
530
|
+
ready,
|
|
531
|
+
new Promise((_, reject) => {
|
|
532
|
+
timer = setTimeout(() => {
|
|
533
|
+
reject(new TimeoutError(accepted ? LOGIN_STAGE : ACCEPT_STAGE));
|
|
534
|
+
}, Math.max(0, deadline - Date.now()));
|
|
535
|
+
})
|
|
536
|
+
]));
|
|
537
|
+
}
|
|
538
|
+
finally {
|
|
539
|
+
clearTimeout(timer);
|
|
540
|
+
connection.off("connect", markAccepted);
|
|
541
|
+
}
|
|
337
542
|
connection.on("error", () => undefined);
|
|
338
543
|
return { client, raw };
|
|
339
544
|
}
|
|
340
545
|
catch (cause) {
|
|
341
546
|
raw.destroy();
|
|
547
|
+
if (cause instanceof TimeoutError || cause instanceof UnsupportedError)
|
|
548
|
+
throw cause;
|
|
342
549
|
throw new TransportError(`failed to connect to ${parsed.host}:${String(parsed.port)}`, true, {
|
|
343
550
|
cause
|
|
344
551
|
});
|
|
345
552
|
}
|
|
346
553
|
}
|
|
554
|
+
// Use the same request queue the Apache Iggy SDK uses for its native partition
|
|
555
|
+
// data connections. This bypasses coordinator login settlement on this connection.
|
|
556
|
+
function pinnedClient(raw, loginDeadline) {
|
|
557
|
+
const data = raw;
|
|
558
|
+
const queue = data._queueCommand;
|
|
559
|
+
if (typeof queue !== "function")
|
|
560
|
+
throw new UnsupportedError("this Apache Iggy SDK does not expose the partition data request queue required by filtered Primary reads");
|
|
561
|
+
const facade = {
|
|
562
|
+
sendCommand(code, payload, options) {
|
|
563
|
+
return queue.call(data, code, payload, options?.handleResponse ?? true, options?.last ?? true, false, options?.deadline ??
|
|
564
|
+
(raw.isAuthenticated
|
|
565
|
+
? Date.now() + connectOptions().timeoutMs
|
|
566
|
+
: (loginDeadline ?? Date.now() + connectOptions().timeoutMs)));
|
|
567
|
+
},
|
|
568
|
+
get isAuthenticated() {
|
|
569
|
+
return raw.isAuthenticated;
|
|
570
|
+
},
|
|
571
|
+
authenticate: (credentials) => raw.authenticate(credentials),
|
|
572
|
+
destroy: () => {
|
|
573
|
+
raw.destroy();
|
|
574
|
+
},
|
|
575
|
+
on: (event, callback) => {
|
|
576
|
+
raw.on(event, callback);
|
|
577
|
+
},
|
|
578
|
+
once: (event, callback) => {
|
|
579
|
+
raw.once(event, callback);
|
|
580
|
+
},
|
|
581
|
+
getReadStream: () => raw.getReadStream()
|
|
582
|
+
};
|
|
583
|
+
return new SimpleClient(facade);
|
|
584
|
+
}
|
|
347
585
|
function serverResponseError(error) {
|
|
348
586
|
let current = error;
|
|
349
587
|
for (let depth = 0; depth < 8; depth += 1) {
|
|
@@ -357,16 +595,16 @@ function serverResponseError(error) {
|
|
|
357
595
|
}
|
|
358
596
|
return undefined;
|
|
359
597
|
}
|
|
360
|
-
async function connectWithRetry(parsed) {
|
|
598
|
+
async function connectWithRetry(parsed, deadline) {
|
|
361
599
|
let retries = 0;
|
|
362
600
|
let lastError;
|
|
363
601
|
while (parsed.reconnection.maxRetries === undefined ||
|
|
364
602
|
retries <= parsed.reconnection.maxRetries) {
|
|
365
603
|
try {
|
|
366
|
-
return await connectSimpleClient(parsed);
|
|
604
|
+
return await connectSimpleClient(parsed, deadline);
|
|
367
605
|
}
|
|
368
606
|
catch (error) {
|
|
369
|
-
if (error instanceof ConfigError)
|
|
607
|
+
if (error instanceof ConfigError || error instanceof TimeoutError)
|
|
370
608
|
throw error;
|
|
371
609
|
lastError = error;
|
|
372
610
|
if (serverResponseError(error) !== undefined) {
|
|
@@ -377,7 +615,10 @@ async function connectWithRetry(parsed) {
|
|
|
377
615
|
break;
|
|
378
616
|
}
|
|
379
617
|
retries += 1;
|
|
380
|
-
|
|
618
|
+
const remaining = deadline - Date.now();
|
|
619
|
+
if (remaining <= 0)
|
|
620
|
+
throw new TimeoutError(ACCEPT_STAGE, { cause: error });
|
|
621
|
+
await new Promise((resolve) => setTimeout(resolve, Math.min(parsed.reconnection.intervalMs, remaining)));
|
|
381
622
|
}
|
|
382
623
|
}
|
|
383
624
|
const responseError = serverResponseError(lastError);
|
|
@@ -396,6 +637,8 @@ export class ApacheIggyTransport {
|
|
|
396
637
|
reconnectLock = new Mutex();
|
|
397
638
|
publishLane = new Mutex();
|
|
398
639
|
disconnected = new WeakSet();
|
|
640
|
+
// Groups this transport rejoins after a reconnect. Setup helpers allow
|
|
641
|
+
// recreation, while joining an existing group keeps `create: false`.
|
|
399
642
|
consumerGroups = new Map();
|
|
400
643
|
partitionCounts = new Map();
|
|
401
644
|
balancedCursors = new Map();
|
|
@@ -409,24 +652,43 @@ export class ApacheIggyTransport {
|
|
|
409
652
|
get iggyClient() {
|
|
410
653
|
return this.client;
|
|
411
654
|
}
|
|
412
|
-
|
|
655
|
+
get connectsNodes() {
|
|
656
|
+
return this.connection !== undefined;
|
|
657
|
+
}
|
|
658
|
+
static async connect(connectionString, options, deadline = Date.now() + connectOptions().timeoutMs) {
|
|
413
659
|
const config = publishOptions(options);
|
|
414
660
|
const parsed = parseConnectionString(connectionString);
|
|
415
|
-
const connected = await connectWithRetry(parsed);
|
|
661
|
+
const connected = await connectWithRetry(parsed, deadline);
|
|
416
662
|
const transport = new ApacheIggyTransport(connected.client, parsed, "owned", config);
|
|
417
663
|
transport.watch(connected);
|
|
418
664
|
return transport;
|
|
419
665
|
}
|
|
420
|
-
static async fromClient(client, ownership = "borrowed", options) {
|
|
666
|
+
static async fromClient(client, ownership = "borrowed", options, deadline = Date.now() + connectOptions().timeoutMs) {
|
|
421
667
|
const config = publishOptions(options);
|
|
422
|
-
|
|
668
|
+
try {
|
|
669
|
+
await publishWithin(client.clientProvider(), Math.max(0, deadline - Date.now()));
|
|
670
|
+
}
|
|
671
|
+
catch (cause) {
|
|
672
|
+
if (ownership === "owned")
|
|
673
|
+
void client.destroy().catch(() => undefined);
|
|
674
|
+
if (cause instanceof TimeoutError)
|
|
675
|
+
throw new TimeoutError("Iggy client readiness", { cause });
|
|
676
|
+
throw cause;
|
|
677
|
+
}
|
|
423
678
|
return new ApacheIggyTransport(client, undefined, ownership, config);
|
|
424
679
|
}
|
|
425
680
|
async execute(operation, message, retryAfterReconnect = true) {
|
|
681
|
+
if (this.closed)
|
|
682
|
+
throw new TransportError("transport is closed", false);
|
|
426
683
|
const stale = this.client;
|
|
427
684
|
if (this.disconnected.has(stale)) {
|
|
428
685
|
await this.reconnect(stale);
|
|
429
|
-
|
|
686
|
+
try {
|
|
687
|
+
return await operation(this.client);
|
|
688
|
+
}
|
|
689
|
+
catch (cause) {
|
|
690
|
+
throw new TransportError(message, serverResponseError(cause) === undefined, { cause });
|
|
691
|
+
}
|
|
430
692
|
}
|
|
431
693
|
try {
|
|
432
694
|
return await operation(stale);
|
|
@@ -478,10 +740,23 @@ export class ApacheIggyTransport {
|
|
|
478
740
|
const remaining = deadline === undefined ? undefined : deadline - Date.now();
|
|
479
741
|
if (remaining !== undefined && remaining <= 0)
|
|
480
742
|
throw new TimeoutError("Iggy publish reconnect");
|
|
481
|
-
const connected = await connectSimpleClient(this.connection,
|
|
743
|
+
const connected = await connectSimpleClient(this.connection, deadline);
|
|
482
744
|
try {
|
|
483
|
-
for (const group of this.consumerGroups
|
|
484
|
-
const join =
|
|
745
|
+
for (const [key, group] of [...this.consumerGroups]) {
|
|
746
|
+
const join = group.create && typeof group.name === "string"
|
|
747
|
+
? connected.client.group.ensureAndJoin(group.streamId, group.topicId, group.name)
|
|
748
|
+
: connected.client.group
|
|
749
|
+
.join({
|
|
750
|
+
streamId: group.streamId,
|
|
751
|
+
topicId: group.topicId,
|
|
752
|
+
groupId: group.name
|
|
753
|
+
})
|
|
754
|
+
.then(() => undefined,
|
|
755
|
+
// A deleted group is not recreated: its member finds out
|
|
756
|
+
// on its next assignment read.
|
|
757
|
+
() => {
|
|
758
|
+
this.consumerGroups.delete(key);
|
|
759
|
+
});
|
|
485
760
|
await (deadline === undefined
|
|
486
761
|
? join
|
|
487
762
|
: publishWithin(join, Math.max(1, deadline - Date.now())));
|
|
@@ -537,6 +812,30 @@ export class ApacheIggyTransport {
|
|
|
537
812
|
async ensureStream(name) {
|
|
538
813
|
await this.execute((client) => client.stream.ensure(name), `failed to ensure stream \`${name}\``);
|
|
539
814
|
}
|
|
815
|
+
async deleteStream(name) {
|
|
816
|
+
const deleted = await this.execute(async (client) => {
|
|
817
|
+
if ((await client.stream.get({ streamId: name })) === null)
|
|
818
|
+
return false;
|
|
819
|
+
try {
|
|
820
|
+
await client.stream.delete({ streamId: name });
|
|
821
|
+
}
|
|
822
|
+
catch (cause) {
|
|
823
|
+
if ((await client.stream.get({ streamId: name })) !== null)
|
|
824
|
+
throw cause;
|
|
825
|
+
}
|
|
826
|
+
return true;
|
|
827
|
+
}, `failed to delete stream \`${name}\``);
|
|
828
|
+
const prefix = `${name}\0`;
|
|
829
|
+
for (const cache of [this.partitionCounts, this.balancedCursors, this.consumerGroups]) {
|
|
830
|
+
for (const key of cache.keys())
|
|
831
|
+
if (key.startsWith(prefix))
|
|
832
|
+
cache.delete(key);
|
|
833
|
+
}
|
|
834
|
+
return deleted;
|
|
835
|
+
}
|
|
836
|
+
async ensureConsumerGroup(streamId, topicId, name) {
|
|
837
|
+
await this.execute((client) => client.group.ensure(streamId, topicId, name), `failed to ensure consumer group \`${name}\``);
|
|
838
|
+
}
|
|
540
839
|
async ensureTopic(streamId, topicId, partitions) {
|
|
541
840
|
const topic = await this.execute((client) => client.topic.ensure(streamId, topicId, partitions), `failed to ensure topic \`${topicId}\` on stream \`${streamId}\``);
|
|
542
841
|
this.partitionCounts.set(this.topicKey(streamId, topicId), topic.partitionsCount);
|
|
@@ -701,7 +1000,7 @@ export class ApacheIggyTransport {
|
|
|
701
1000
|
const reply = await this.execute((client) => client.message.poll({
|
|
702
1001
|
streamId,
|
|
703
1002
|
topicId,
|
|
704
|
-
partitionId: target.
|
|
1003
|
+
partitionId: target.partitionId ?? null,
|
|
705
1004
|
consumer: toIggyConsumer(target),
|
|
706
1005
|
pollingStrategy: toIggyPollingStrategy(strategy),
|
|
707
1006
|
count,
|
|
@@ -737,11 +1036,105 @@ export class ApacheIggyTransport {
|
|
|
737
1036
|
}
|
|
738
1037
|
async joinConsumerGroup(streamId, topicId, name) {
|
|
739
1038
|
await this.execute((client) => client.group.ensureAndJoin(streamId, topicId, name), `failed to join consumer group \`${name}\``);
|
|
740
|
-
this.consumerGroups.set(`${streamId}\0${topicId}\0${name}`, {
|
|
1039
|
+
this.consumerGroups.set(`${streamId}\0${topicId}\0${name}`, {
|
|
1040
|
+
streamId,
|
|
1041
|
+
topicId,
|
|
1042
|
+
name,
|
|
1043
|
+
create: true
|
|
1044
|
+
});
|
|
1045
|
+
}
|
|
1046
|
+
async syncConsumerGroup(streamId, topicId, name) {
|
|
1047
|
+
const identifiers = [streamId, topicId, name].map((value) => new TextEncoder().encode(value));
|
|
1048
|
+
if (identifiers.some((value) => value.byteLength === 0 || value.byteLength > 255))
|
|
1049
|
+
throw new ConfigError("consumer group identifiers must contain 1 to 255 UTF-8 bytes");
|
|
1050
|
+
const payload = new Uint8Array(identifiers.reduce((size, value) => size + 2 + value.byteLength, 0));
|
|
1051
|
+
let offset = 0;
|
|
1052
|
+
for (const value of identifiers) {
|
|
1053
|
+
payload.set([2, value.byteLength], offset);
|
|
1054
|
+
payload.set(value, offset + 2);
|
|
1055
|
+
offset += value.byteLength + 2;
|
|
1056
|
+
}
|
|
1057
|
+
let reply = await this.sendManaged(SYNC_CONSUMER_GROUP_CODE, payload, {
|
|
1058
|
+
retryAfterReconnect: true
|
|
1059
|
+
});
|
|
1060
|
+
let rejoined = false;
|
|
1061
|
+
if (reply.byteLength === 0) {
|
|
1062
|
+
await this.joinConsumerGroup(streamId, topicId, name);
|
|
1063
|
+
rejoined = true;
|
|
1064
|
+
reply = await this.sendManaged(SYNC_CONSUMER_GROUP_CODE, payload, {
|
|
1065
|
+
retryAfterReconnect: true
|
|
1066
|
+
});
|
|
1067
|
+
if (reply.byteLength === 0)
|
|
1068
|
+
return undefined;
|
|
1069
|
+
}
|
|
1070
|
+
if (reply.byteLength < 12)
|
|
1071
|
+
throw new ProtocolError("the consumer group assignment is truncated");
|
|
1072
|
+
const view = new DataView(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1073
|
+
const count = view.getUint32(8, true);
|
|
1074
|
+
if (reply.byteLength !== 12 + count * 4)
|
|
1075
|
+
throw new ProtocolError("the consumer group assignment length is invalid");
|
|
1076
|
+
return {
|
|
1077
|
+
generation: view.getBigUint64(0, true),
|
|
1078
|
+
rejoined,
|
|
1079
|
+
partitions: Array.from({ length: count }, (_, index) => view.getUint32(12 + index * 4, true))
|
|
1080
|
+
};
|
|
741
1081
|
}
|
|
742
1082
|
async leaveConsumerGroup(streamId, topicId, name) {
|
|
743
|
-
await this.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${name}\``);
|
|
744
|
-
this.consumerGroups.delete(`${streamId}\0${topicId}\0${name}`);
|
|
1083
|
+
await this.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${String(name)}\``);
|
|
1084
|
+
this.consumerGroups.delete(`${streamId}\0${topicId}\0${String(name)}`);
|
|
1085
|
+
}
|
|
1086
|
+
async joinExistingConsumerGroup(streamId, topicId, name) {
|
|
1087
|
+
await this.execute((client) => client.group.join({ streamId, topicId, groupId: name }), `failed to join consumer group \`${String(name)}\``);
|
|
1088
|
+
this.consumerGroups.set(`${streamId}\0${topicId}\0${String(name)}`, {
|
|
1089
|
+
streamId,
|
|
1090
|
+
topicId,
|
|
1091
|
+
name,
|
|
1092
|
+
create: false
|
|
1093
|
+
});
|
|
1094
|
+
}
|
|
1095
|
+
async openNodeConnection(ip, port) {
|
|
1096
|
+
if (this.connection === undefined) {
|
|
1097
|
+
throw new ConfigError("a node connection needs a Laser connected from a connection string, not an injected client");
|
|
1098
|
+
}
|
|
1099
|
+
const unspecified = ip.length === 0 || ip === "0.0.0.0" || ip === "::" || ip === "[::]";
|
|
1100
|
+
const host = unspecified ? this.connection.host : ip;
|
|
1101
|
+
const connected = await connectSimpleClient({
|
|
1102
|
+
...this.connection,
|
|
1103
|
+
host,
|
|
1104
|
+
port,
|
|
1105
|
+
...(isIP(this.connection.host) === 0 ? { servername: this.connection.host } : {})
|
|
1106
|
+
}, Date.now() + connectOptions().timeoutMs, true);
|
|
1107
|
+
const client = connected.client;
|
|
1108
|
+
return {
|
|
1109
|
+
async send(code, payload) {
|
|
1110
|
+
const reply = await client.sendBinaryRequest(code, toNodeBuffer(payload));
|
|
1111
|
+
return new Uint8Array(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1112
|
+
},
|
|
1113
|
+
async close() {
|
|
1114
|
+
await client.destroy().catch(() => undefined);
|
|
1115
|
+
}
|
|
1116
|
+
};
|
|
1117
|
+
}
|
|
1118
|
+
async openCoordinator() {
|
|
1119
|
+
if (this.connection === undefined) {
|
|
1120
|
+
throw new ConfigError("a coordinator connection needs a Laser connected from a connection string, not an injected client");
|
|
1121
|
+
}
|
|
1122
|
+
const connected = await connectSimpleClient(this.connection, Date.now() + connectOptions().timeoutMs);
|
|
1123
|
+
const coordinator = new ApacheIggyTransport(connected.client, this.connection, "owned", this.publishConfig);
|
|
1124
|
+
coordinator.watch(connected);
|
|
1125
|
+
return {
|
|
1126
|
+
async send(code, payload) {
|
|
1127
|
+
const reply = await coordinator.execute((client) => client.sendBinaryRequest(code, toNodeBuffer(payload)), `coordinator command ${String(code)} failed`);
|
|
1128
|
+
return new Uint8Array(reply.buffer, reply.byteOffset, reply.byteLength);
|
|
1129
|
+
},
|
|
1130
|
+
async joinConsumerGroup(streamId, topicId, name) {
|
|
1131
|
+
await coordinator.execute((client) => client.group.join({ streamId, topicId, groupId: name }), `failed to join consumer group \`${String(name)}\``);
|
|
1132
|
+
},
|
|
1133
|
+
async leaveConsumerGroup(streamId, topicId, name) {
|
|
1134
|
+
await coordinator.execute((client) => client.group.leave({ streamId, topicId, groupId: name }), `failed to leave consumer group \`${String(name)}\``);
|
|
1135
|
+
},
|
|
1136
|
+
close: () => coordinator.close()
|
|
1137
|
+
};
|
|
745
1138
|
}
|
|
746
1139
|
async close() {
|
|
747
1140
|
this.closed = true;
|