@thru/replay 0.2.20 → 0.2.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +448 -181
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +33 -4
- package/dist/index.d.ts +33 -4
- package/dist/index.mjs +450 -183
- package/dist/index.mjs.map +1 -1
- package/package.json +11 -12
package/dist/index.mjs
CHANGED
|
@@ -1,21 +1,50 @@
|
|
|
1
1
|
import { create } from '@bufbuild/protobuf';
|
|
2
2
|
import { createClient } from '@connectrpc/connect';
|
|
3
|
-
import { createGrpcTransport } from '@connectrpc/connect-node';
|
|
4
|
-
import { QueryService, StreamingService, GetAccountRequestSchema, ListAccountsRequestSchema, ListBlocksRequestSchema, StreamBlocksRequestSchema, ListTransactionsRequestSchema, StreamTransactionsRequestSchema, ListEventsRequestSchema, StreamEventsRequestSchema, StreamAccountUpdatesRequestSchema, GetHeightRequestSchema, AccountView, PageRequestSchema, PubkeySchema, FilterParamValueSchema, FilterSchema, EventSchema } from '@thru/proto';
|
|
5
|
-
export { AccountView, FilterParamValueSchema, FilterSchema } from '@thru/proto';
|
|
3
|
+
import { Http2SessionManager, createGrpcTransport } from '@connectrpc/connect-node';
|
|
4
|
+
import { QueryService, StreamingService, GetAccountRequestSchema, ListAccountsRequestSchema, ListBlocksRequestSchema, StreamBlocksRequestSchema, ListTransactionsRequestSchema, StreamTransactionsRequestSchema, ListEventsRequestSchema, StreamEventsRequestSchema, StreamAccountUpdatesRequestSchema, GetHeightRequestSchema, GetChainInfoRequestSchema, AccountView, PageRequestSchema, PubkeySchema, FilterParamValueSchema, FilterSchema, EventSchema } from '@thru/sdk/proto';
|
|
5
|
+
export { AccountView, FilterParamValueSchema, FilterSchema } from '@thru/sdk/proto';
|
|
6
6
|
|
|
7
7
|
// src/chain-client.ts
|
|
8
8
|
var ChainClient = class {
|
|
9
9
|
constructor(options) {
|
|
10
10
|
this.options = options;
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
11
|
+
if (options.transport) {
|
|
12
|
+
this.sessionManager = null;
|
|
13
|
+
this.query = createClient(QueryService, options.transport);
|
|
14
|
+
this.streaming = createClient(StreamingService, options.transport);
|
|
15
|
+
} else {
|
|
16
|
+
const { transport, sessionManager } = this.createOwnedTransport();
|
|
17
|
+
this.sessionManager = sessionManager;
|
|
18
|
+
this.query = createClient(QueryService, transport);
|
|
19
|
+
this.streaming = createClient(StreamingService, transport);
|
|
20
|
+
}
|
|
14
21
|
this.callOptions = options.callOptions;
|
|
15
22
|
}
|
|
23
|
+
options;
|
|
16
24
|
query;
|
|
17
25
|
streaming;
|
|
18
26
|
callOptions;
|
|
27
|
+
/**
|
|
28
|
+
* The HTTP/2 session manager owned by this client. Only set when the client
|
|
29
|
+
* created its own gRPC transport (i.e., `options.transport` was not provided).
|
|
30
|
+
* `close()` uses this to tear down the underlying persistent connection.
|
|
31
|
+
*/
|
|
32
|
+
sessionManager;
|
|
33
|
+
closed = false;
|
|
34
|
+
/**
|
|
35
|
+
* Close the underlying HTTP/2 session, if this client owns one. Idempotent.
|
|
36
|
+
*
|
|
37
|
+
* Callers are responsible for ensuring that no in-flight RPCs or streams
|
|
38
|
+
* are still being awaited on this client — pending requests will fail.
|
|
39
|
+
*
|
|
40
|
+
* If the client was constructed with an externally-supplied `transport`,
|
|
41
|
+
* `close()` is a no-op; the caller owns the transport's lifecycle.
|
|
42
|
+
*/
|
|
43
|
+
close() {
|
|
44
|
+
if (this.closed) return;
|
|
45
|
+
this.closed = true;
|
|
46
|
+
this.sessionManager?.abort();
|
|
47
|
+
}
|
|
19
48
|
getAccount(request) {
|
|
20
49
|
return this.query.getAccount(create(GetAccountRequestSchema, request), this.callOptions);
|
|
21
50
|
}
|
|
@@ -40,7 +69,7 @@ var ChainClient = class {
|
|
|
40
69
|
streamEvents(request) {
|
|
41
70
|
return this.streaming.streamEvents(create(StreamEventsRequestSchema, request), this.callOptions);
|
|
42
71
|
}
|
|
43
|
-
|
|
72
|
+
createOwnedTransport() {
|
|
44
73
|
if (!this.options.baseUrl) {
|
|
45
74
|
throw new Error("ChainClient requires baseUrl when no transport is provided");
|
|
46
75
|
}
|
|
@@ -50,15 +79,19 @@ var ChainClient = class {
|
|
|
50
79
|
...userInterceptors,
|
|
51
80
|
...headerInterceptor ? [headerInterceptor] : []
|
|
52
81
|
];
|
|
53
|
-
|
|
54
|
-
baseUrl: this.options.baseUrl,
|
|
55
|
-
useBinaryFormat: this.options.useBinaryFormat ?? true,
|
|
56
|
-
interceptors: mergedInterceptors.length ? mergedInterceptors : void 0,
|
|
82
|
+
const sessionManager = new Http2SessionManager(this.options.baseUrl, {
|
|
57
83
|
pingIntervalMs: 3e4,
|
|
58
84
|
pingIdleConnection: true,
|
|
59
85
|
pingTimeoutMs: 1e4,
|
|
60
86
|
idleConnectionTimeoutMs: 0
|
|
61
87
|
});
|
|
88
|
+
const transport = createGrpcTransport({
|
|
89
|
+
baseUrl: this.options.baseUrl,
|
|
90
|
+
useBinaryFormat: this.options.useBinaryFormat ?? true,
|
|
91
|
+
interceptors: mergedInterceptors.length ? mergedInterceptors : void 0,
|
|
92
|
+
sessionManager
|
|
93
|
+
});
|
|
94
|
+
return { transport, sessionManager };
|
|
62
95
|
}
|
|
63
96
|
createHeaderInterceptor() {
|
|
64
97
|
const headers = {};
|
|
@@ -76,6 +109,9 @@ var ChainClient = class {
|
|
|
76
109
|
getHeight() {
|
|
77
110
|
return this.query.getHeight(create(GetHeightRequestSchema, {}), this.callOptions);
|
|
78
111
|
}
|
|
112
|
+
getChainInfo(request = {}) {
|
|
113
|
+
return this.query.getChainInfo(create(GetChainInfoRequestSchema, request), this.callOptions);
|
|
114
|
+
}
|
|
79
115
|
};
|
|
80
116
|
|
|
81
117
|
// src/async-queue.ts
|
|
@@ -238,15 +274,16 @@ var LivePump = class {
|
|
|
238
274
|
buffer;
|
|
239
275
|
slotOf;
|
|
240
276
|
keyOf;
|
|
241
|
-
|
|
277
|
+
sourceIterator;
|
|
242
278
|
logger;
|
|
243
279
|
mode;
|
|
244
280
|
minSlotSeen = null;
|
|
245
281
|
maxSlotSeen = null;
|
|
246
282
|
minEmitSlot = null;
|
|
247
283
|
pumpPromise;
|
|
284
|
+
closing = false;
|
|
248
285
|
constructor(options) {
|
|
249
|
-
this.
|
|
286
|
+
this.sourceIterator = options.source[Symbol.asyncIterator]();
|
|
250
287
|
this.slotOf = options.slotOf;
|
|
251
288
|
this.keyOf = options.keyOf ?? ((item) => options.slotOf(item).toString());
|
|
252
289
|
this.logger = options.logger ?? NOOP_LOGGER;
|
|
@@ -290,12 +327,19 @@ var LivePump = class {
|
|
|
290
327
|
return this.queue.next();
|
|
291
328
|
}
|
|
292
329
|
async close() {
|
|
330
|
+
this.closing = true;
|
|
293
331
|
this.queue.close();
|
|
294
|
-
await
|
|
332
|
+
await Promise.allSettled([
|
|
333
|
+
this.closeSourceIterator(),
|
|
334
|
+
this.pumpPromise
|
|
335
|
+
]);
|
|
295
336
|
}
|
|
296
337
|
async start() {
|
|
297
338
|
try {
|
|
298
|
-
|
|
339
|
+
while (!this.closing) {
|
|
340
|
+
const next = await this.sourceIterator.next();
|
|
341
|
+
if (next.done || this.closing) break;
|
|
342
|
+
const item = next.value;
|
|
299
343
|
const slot = this.slotOf(item);
|
|
300
344
|
if (this.minSlotSeen === null || slot < this.minSlotSeen) this.minSlotSeen = slot;
|
|
301
345
|
if (this.maxSlotSeen === null || slot > this.maxSlotSeen) this.maxSlotSeen = slot;
|
|
@@ -305,9 +349,21 @@ var LivePump = class {
|
|
|
305
349
|
this.queue.push(item);
|
|
306
350
|
}
|
|
307
351
|
}
|
|
308
|
-
this.queue.close();
|
|
309
352
|
} catch (err) {
|
|
310
|
-
this.
|
|
353
|
+
if (!this.closing) {
|
|
354
|
+
this.queue.fail(err);
|
|
355
|
+
}
|
|
356
|
+
} finally {
|
|
357
|
+
this.queue.close();
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
async closeSourceIterator() {
|
|
361
|
+
if (typeof this.sourceIterator.return !== "function") {
|
|
362
|
+
return;
|
|
363
|
+
}
|
|
364
|
+
try {
|
|
365
|
+
await this.sourceIterator.return();
|
|
366
|
+
} catch {
|
|
311
367
|
}
|
|
312
368
|
}
|
|
313
369
|
};
|
|
@@ -327,6 +383,7 @@ function withTimeout(promise, timeoutMs) {
|
|
|
327
383
|
const timer = setTimeout(() => {
|
|
328
384
|
reject(new TimeoutError(`Operation timed out after ${timeoutMs}ms`));
|
|
329
385
|
}, timeoutMs);
|
|
386
|
+
timer.unref?.();
|
|
330
387
|
promise.then((value) => {
|
|
331
388
|
clearTimeout(timer);
|
|
332
389
|
resolve(value);
|
|
@@ -345,6 +402,37 @@ var TimeoutError = class extends Error {
|
|
|
345
402
|
function delay(ms) {
|
|
346
403
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
347
404
|
}
|
|
405
|
+
function abortableDelay(ms, signal) {
|
|
406
|
+
if (ms <= 0 || signal?.aborted) {
|
|
407
|
+
return Promise.resolve();
|
|
408
|
+
}
|
|
409
|
+
const abortSignal = signal;
|
|
410
|
+
return new Promise((resolve) => {
|
|
411
|
+
if (!abortSignal) {
|
|
412
|
+
const timer2 = setTimeout(resolve, ms);
|
|
413
|
+
timer2.unref?.();
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
416
|
+
const onAbort = () => {
|
|
417
|
+
clearTimeout(timer);
|
|
418
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
419
|
+
resolve();
|
|
420
|
+
};
|
|
421
|
+
const timer = setTimeout(() => {
|
|
422
|
+
abortSignal.removeEventListener("abort", onAbort);
|
|
423
|
+
resolve();
|
|
424
|
+
}, ms);
|
|
425
|
+
timer.unref?.();
|
|
426
|
+
abortSignal.addEventListener("abort", onAbort, { once: true });
|
|
427
|
+
});
|
|
428
|
+
}
|
|
429
|
+
function isAbortError(err) {
|
|
430
|
+
if (!err || typeof err !== "object") return false;
|
|
431
|
+
const name = typeof err.name === "string" ? err.name.toLowerCase() : "";
|
|
432
|
+
const message = typeof err.message === "string" ? err.message.toLowerCase() : "";
|
|
433
|
+
const code = String(err.code ?? "").toLowerCase();
|
|
434
|
+
return name.includes("abort") || name.includes("cancel") || code === "canceled" || code === "cancelled" || message.includes("aborted") || message.includes("canceled") || message.includes("cancelled");
|
|
435
|
+
}
|
|
348
436
|
|
|
349
437
|
// src/replay-stream.ts
|
|
350
438
|
var DEFAULT_METRICS = {
|
|
@@ -358,6 +446,27 @@ function compareBigint(a, b) {
|
|
|
358
446
|
if (a === b) return 0;
|
|
359
447
|
return a < b ? -1 : 1;
|
|
360
448
|
}
|
|
449
|
+
function isNonDecreasing(items, extractSlot) {
|
|
450
|
+
for (let idx = 1; idx < items.length; idx += 1) {
|
|
451
|
+
if (extractSlot(items[idx]) < extractSlot(items[idx - 1])) return false;
|
|
452
|
+
}
|
|
453
|
+
return true;
|
|
454
|
+
}
|
|
455
|
+
function assertBackfillPageOrder(previousPage, currentPage, extractSlot) {
|
|
456
|
+
if (!isNonDecreasing(currentPage, extractSlot)) {
|
|
457
|
+
throw new Error(
|
|
458
|
+
"backfill source returned a page that is not ordered by ascending slot"
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
if (!previousPage?.length || !currentPage.length) return;
|
|
462
|
+
const previousMaxSlot = extractSlot(previousPage[previousPage.length - 1]);
|
|
463
|
+
const currentMinSlot = extractSlot(currentPage[0]);
|
|
464
|
+
if (currentMinSlot < previousMaxSlot) {
|
|
465
|
+
throw new Error(
|
|
466
|
+
`backfill source returned pages out of ascending slot order: page minimum slot ${currentMinSlot} is before previous page maximum slot ${previousMaxSlot}`
|
|
467
|
+
);
|
|
468
|
+
}
|
|
469
|
+
}
|
|
361
470
|
var ReplayStream = class {
|
|
362
471
|
config;
|
|
363
472
|
logger;
|
|
@@ -381,12 +490,17 @@ var ReplayStream = class {
|
|
|
381
490
|
extractKey,
|
|
382
491
|
safetyMargin,
|
|
383
492
|
resubscribeOnEnd,
|
|
384
|
-
onReconnect
|
|
493
|
+
onReconnect,
|
|
494
|
+
signal,
|
|
495
|
+
dispose
|
|
385
496
|
} = this.config;
|
|
386
497
|
const shouldResubscribeOnEnd = resubscribeOnEnd ?? true;
|
|
387
498
|
const keyOf = extractKey ?? ((item) => extractSlot(item).toString());
|
|
499
|
+
const shouldStop = (err) => signal?.aborted === true || isAbortError(err);
|
|
388
500
|
let currentSubscribeLive = subscribeLive;
|
|
389
501
|
let currentFetchBackfill = fetchBackfill;
|
|
502
|
+
let currentDispose = dispose ?? (() => {
|
|
503
|
+
});
|
|
390
504
|
const createLivePump = (slot, startStreaming = false, emitFloor) => new LivePump({
|
|
391
505
|
source: currentSubscribeLive(slot),
|
|
392
506
|
slotOf: extractSlot,
|
|
@@ -418,91 +532,103 @@ var ReplayStream = class {
|
|
|
418
532
|
this.logger.info(
|
|
419
533
|
`replay entering BACKFILLING state (startSlot=${startSlot}, safetyMargin=${safetyMargin})`
|
|
420
534
|
);
|
|
421
|
-
let
|
|
422
|
-
const
|
|
423
|
-
|
|
424
|
-
const page = await currentFetchBackfill({ startSlot, cursor });
|
|
425
|
-
if (!page.items.length && !page.cursor && !page.done) {
|
|
426
|
-
emptyPageRetries++;
|
|
427
|
-
if (emptyPageRetries > MAX_EMPTY_PAGE_RETRIES) {
|
|
428
|
-
this.logger.error(
|
|
429
|
-
`backfill returned ${MAX_EMPTY_PAGE_RETRIES} consecutive empty pages; treating as done`
|
|
430
|
-
);
|
|
431
|
-
break;
|
|
432
|
-
}
|
|
433
|
-
const backoffMs = calculateBackoff(emptyPageRetries - 1, DEFAULT_RETRY_CONFIG);
|
|
434
|
-
this.logger.warn(
|
|
435
|
-
`empty backfill page without cursor; retrying in ${backoffMs}ms (${emptyPageRetries}/${MAX_EMPTY_PAGE_RETRIES})`
|
|
436
|
-
);
|
|
437
|
-
await delay(backoffMs);
|
|
438
|
-
continue;
|
|
439
|
-
}
|
|
440
|
-
emptyPageRetries = 0;
|
|
441
|
-
const sorted = [...page.items].sort(
|
|
442
|
-
(a, b) => compareBigint(extractSlot(a), extractSlot(b))
|
|
443
|
-
);
|
|
444
|
-
for (const item of sorted) {
|
|
535
|
+
let pendingOrderedPage = null;
|
|
536
|
+
const emitBackfillItems = async function* (self, items) {
|
|
537
|
+
for (const item of items) {
|
|
445
538
|
const slot = extractSlot(item);
|
|
446
539
|
const key = keyOf(item);
|
|
447
540
|
if (slot < startSlot) continue;
|
|
448
541
|
if (seenItem(slot, key)) {
|
|
449
|
-
|
|
542
|
+
self.metrics.discardedDuplicates += 1;
|
|
450
543
|
continue;
|
|
451
544
|
}
|
|
452
545
|
currentSlot = slot;
|
|
453
546
|
recordEmission(slot, key);
|
|
454
|
-
|
|
547
|
+
self.metrics.emittedBackfill += 1;
|
|
548
|
+
yield item;
|
|
549
|
+
}
|
|
550
|
+
};
|
|
551
|
+
const flushPendingBackfill = async function* (self) {
|
|
552
|
+
if (!pendingOrderedPage) return;
|
|
553
|
+
for await (const item of emitBackfillItems(self, pendingOrderedPage)) {
|
|
455
554
|
yield item;
|
|
456
555
|
}
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
if (
|
|
464
|
-
|
|
465
|
-
|
|
556
|
+
pendingOrderedPage = null;
|
|
557
|
+
};
|
|
558
|
+
let emptyPageRetries = 0;
|
|
559
|
+
const MAX_EMPTY_PAGE_RETRIES = 10;
|
|
560
|
+
try {
|
|
561
|
+
while (!backfillDone) {
|
|
562
|
+
if (shouldStop()) return;
|
|
563
|
+
let page;
|
|
564
|
+
try {
|
|
565
|
+
page = await currentFetchBackfill({ startSlot, cursor });
|
|
566
|
+
} catch (err) {
|
|
567
|
+
if (shouldStop(err)) return;
|
|
568
|
+
throw err;
|
|
569
|
+
}
|
|
570
|
+
if (!page.items.length && !page.cursor && !page.done) {
|
|
571
|
+
emptyPageRetries++;
|
|
572
|
+
if (emptyPageRetries > MAX_EMPTY_PAGE_RETRIES) {
|
|
573
|
+
this.logger.error(
|
|
574
|
+
`backfill returned ${MAX_EMPTY_PAGE_RETRIES} consecutive empty pages; treating as done`
|
|
575
|
+
);
|
|
576
|
+
for await (const item of flushPendingBackfill(this)) {
|
|
577
|
+
yield item;
|
|
578
|
+
}
|
|
579
|
+
break;
|
|
580
|
+
}
|
|
581
|
+
const backoffMs = calculateBackoff(emptyPageRetries - 1, DEFAULT_RETRY_CONFIG);
|
|
582
|
+
this.logger.warn(
|
|
583
|
+
`empty backfill page without cursor; retrying in ${backoffMs}ms (${emptyPageRetries}/${MAX_EMPTY_PAGE_RETRIES})`
|
|
466
584
|
);
|
|
467
|
-
|
|
585
|
+
await abortableDelay(backoffMs, signal);
|
|
586
|
+
continue;
|
|
468
587
|
}
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
recordEmission(slot, key);
|
|
485
|
-
this.metrics.emittedLive += 1;
|
|
486
|
-
yield item;
|
|
487
|
-
livePump.updateEmitFloor(currentSlot);
|
|
488
|
-
}
|
|
489
|
-
if (!drained.length) livePump.updateEmitFloor(currentSlot);
|
|
490
|
-
this.logger.info("replay entering STREAMING state");
|
|
491
|
-
const retryConfig = DEFAULT_RETRY_CONFIG;
|
|
492
|
-
let retryAttempt = 0;
|
|
493
|
-
while (true) {
|
|
494
|
-
try {
|
|
495
|
-
const next = await withTimeout(
|
|
496
|
-
livePump.next(),
|
|
497
|
-
retryConfig.connectionTimeoutMs
|
|
498
|
-
);
|
|
499
|
-
retryAttempt = 0;
|
|
500
|
-
if (next.done) {
|
|
501
|
-
if (!shouldResubscribeOnEnd) break;
|
|
502
|
-
throw new Error("stream ended");
|
|
588
|
+
emptyPageRetries = 0;
|
|
589
|
+
assertBackfillPageOrder(pendingOrderedPage, page.items, extractSlot);
|
|
590
|
+
if (pendingOrderedPage !== null) {
|
|
591
|
+
for await (const item of flushPendingBackfill(this)) {
|
|
592
|
+
if (shouldStop()) return;
|
|
593
|
+
yield item;
|
|
594
|
+
}
|
|
595
|
+
}
|
|
596
|
+
pendingOrderedPage = [...page.items];
|
|
597
|
+
const reachedEnd = page.done || page.cursor === void 0;
|
|
598
|
+
if (reachedEnd) {
|
|
599
|
+
for await (const item of flushPendingBackfill(this)) {
|
|
600
|
+
if (shouldStop()) return;
|
|
601
|
+
yield item;
|
|
602
|
+
}
|
|
503
603
|
}
|
|
504
|
-
const
|
|
505
|
-
|
|
604
|
+
const duplicatesTrimmed = livePump.discardBufferedUpTo(currentSlot);
|
|
605
|
+
this.metrics.discardedDuplicates += duplicatesTrimmed;
|
|
606
|
+
cursor = page.cursor;
|
|
607
|
+
const maxStreamSlot = livePump.maxSlot();
|
|
608
|
+
if (maxStreamSlot !== null) {
|
|
609
|
+
const catchUpSlot = maxStreamSlot > safetyMargin ? maxStreamSlot - safetyMargin : 0n;
|
|
610
|
+
if (currentSlot >= catchUpSlot) {
|
|
611
|
+
for await (const item of flushPendingBackfill(this)) {
|
|
612
|
+
if (shouldStop()) return;
|
|
613
|
+
yield item;
|
|
614
|
+
}
|
|
615
|
+
this.logger.info(
|
|
616
|
+
`replay reached SWITCHING threshold (currentSlot=${currentSlot}, maxStreamSlot=${maxStreamSlot}, catchUpSlot=${catchUpSlot})`
|
|
617
|
+
);
|
|
618
|
+
backfillDone = true;
|
|
619
|
+
}
|
|
620
|
+
}
|
|
621
|
+
if (reachedEnd) backfillDone = true;
|
|
622
|
+
}
|
|
623
|
+
if (shouldStop()) return;
|
|
624
|
+
this.logger.info(`replay entering SWITCHING state (currentSlot=${currentSlot})`);
|
|
625
|
+
const { drained, discarded } = livePump.enableStreaming(currentSlot);
|
|
626
|
+
this.metrics.bufferedItems = drained.length;
|
|
627
|
+
this.metrics.discardedDuplicates += discarded;
|
|
628
|
+
for (const item of drained) {
|
|
629
|
+
if (shouldStop()) return;
|
|
630
|
+
const slot = extractSlot(item);
|
|
631
|
+
const key = keyOf(item);
|
|
506
632
|
if (seenItem(slot, key)) {
|
|
507
633
|
this.metrics.discardedDuplicates += 1;
|
|
508
634
|
continue;
|
|
@@ -510,57 +636,97 @@ var ReplayStream = class {
|
|
|
510
636
|
currentSlot = slot;
|
|
511
637
|
recordEmission(slot, key);
|
|
512
638
|
this.metrics.emittedLive += 1;
|
|
513
|
-
yield
|
|
639
|
+
yield item;
|
|
514
640
|
livePump.updateEmitFloor(currentSlot);
|
|
515
|
-
}
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
641
|
+
}
|
|
642
|
+
if (!drained.length) livePump.updateEmitFloor(currentSlot);
|
|
643
|
+
this.logger.info("replay entering STREAMING state");
|
|
644
|
+
const retryConfig = DEFAULT_RETRY_CONFIG;
|
|
645
|
+
let retryAttempt = 0;
|
|
646
|
+
while (true) {
|
|
647
|
+
if (shouldStop()) return;
|
|
648
|
+
try {
|
|
649
|
+
const next = await withTimeout(
|
|
650
|
+
livePump.next(),
|
|
651
|
+
retryConfig.connectionTimeoutMs
|
|
652
|
+
);
|
|
653
|
+
retryAttempt = 0;
|
|
654
|
+
if (next.done) {
|
|
655
|
+
if (shouldStop()) return;
|
|
656
|
+
if (!shouldResubscribeOnEnd) break;
|
|
657
|
+
throw new Error("stream ended");
|
|
658
|
+
}
|
|
659
|
+
const slot = extractSlot(next.value);
|
|
660
|
+
const key = keyOf(next.value);
|
|
661
|
+
if (seenItem(slot, key)) {
|
|
662
|
+
this.metrics.discardedDuplicates += 1;
|
|
663
|
+
continue;
|
|
664
|
+
}
|
|
665
|
+
currentSlot = slot;
|
|
666
|
+
recordEmission(slot, key);
|
|
667
|
+
this.metrics.emittedLive += 1;
|
|
668
|
+
yield next.value;
|
|
669
|
+
livePump.updateEmitFloor(currentSlot);
|
|
670
|
+
} catch (err) {
|
|
671
|
+
if (shouldStop(err)) return;
|
|
672
|
+
const errMsg = err instanceof Error ? err.message : String(err);
|
|
673
|
+
const backoffMs = calculateBackoff(retryAttempt, retryConfig);
|
|
674
|
+
this.logger.warn(
|
|
675
|
+
`live stream disconnected (${errMsg}); reconnecting in ${backoffMs}ms from slot ${currentSlot} (attempt ${retryAttempt + 1})`
|
|
676
|
+
);
|
|
677
|
+
await abortableDelay(backoffMs, signal);
|
|
678
|
+
if (shouldStop()) return;
|
|
679
|
+
currentDispose();
|
|
680
|
+
await safeClose(livePump);
|
|
681
|
+
retryAttempt++;
|
|
682
|
+
if (onReconnect) {
|
|
683
|
+
try {
|
|
684
|
+
const fresh = onReconnect();
|
|
685
|
+
currentSubscribeLive = fresh.subscribeLive;
|
|
686
|
+
if (fresh.fetchBackfill) {
|
|
687
|
+
currentFetchBackfill = fresh.fetchBackfill;
|
|
688
|
+
}
|
|
689
|
+
currentDispose = fresh.dispose ?? (() => {
|
|
690
|
+
});
|
|
691
|
+
this.logger.info("created fresh client for reconnection");
|
|
692
|
+
} catch (factoryErr) {
|
|
693
|
+
this.logger.error(
|
|
694
|
+
`failed to create fresh client: ${factoryErr instanceof Error ? factoryErr.message : String(factoryErr)}; using existing`
|
|
695
|
+
);
|
|
530
696
|
}
|
|
531
|
-
this.logger.info("created fresh client for reconnection");
|
|
532
|
-
} catch (factoryErr) {
|
|
533
|
-
this.logger.error(
|
|
534
|
-
`failed to create fresh client: ${factoryErr instanceof Error ? factoryErr.message : String(factoryErr)}; using existing`
|
|
535
|
-
);
|
|
536
697
|
}
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
currentSlot
|
|
698
|
+
if (onReconnect && currentSlot > 0n) {
|
|
699
|
+
for await (const item of this.miniBackfill(
|
|
700
|
+
currentSlot,
|
|
701
|
+
currentFetchBackfill,
|
|
702
|
+
extractSlot,
|
|
703
|
+
keyOf,
|
|
704
|
+
seenItem,
|
|
705
|
+
recordEmission,
|
|
706
|
+
signal
|
|
707
|
+
)) {
|
|
708
|
+
if (shouldStop()) return;
|
|
709
|
+
const itemSlot = extractSlot(item);
|
|
710
|
+
if (itemSlot > currentSlot) {
|
|
711
|
+
currentSlot = itemSlot;
|
|
712
|
+
}
|
|
713
|
+
yield item;
|
|
550
714
|
}
|
|
551
|
-
yield item;
|
|
552
715
|
}
|
|
716
|
+
const resumeSlot = currentSlot > 0n ? currentSlot : 0n;
|
|
717
|
+
livePump = createLivePump(resumeSlot, true, currentSlot);
|
|
553
718
|
}
|
|
554
|
-
const resumeSlot = currentSlot > 0n ? currentSlot : 0n;
|
|
555
|
-
livePump = createLivePump(resumeSlot, true, currentSlot);
|
|
556
719
|
}
|
|
720
|
+
} finally {
|
|
721
|
+
currentDispose();
|
|
722
|
+
await safeClose(livePump);
|
|
557
723
|
}
|
|
558
724
|
}
|
|
559
725
|
/**
|
|
560
726
|
* Perform mini-backfill from lastProcessedSlot to catch up after reconnection.
|
|
561
727
|
* Ensures no data gaps from events that occurred during disconnection.
|
|
562
728
|
*/
|
|
563
|
-
async *miniBackfill(fromSlot, fetchBackfill, extractSlot, keyOf, seenItem, recordEmission) {
|
|
729
|
+
async *miniBackfill(fromSlot, fetchBackfill, extractSlot, keyOf, seenItem, recordEmission, signal) {
|
|
564
730
|
this.logger.info(`mini-backfill starting from slot ${fromSlot}`);
|
|
565
731
|
const MINI_BACKFILL_TIMEOUT = 3e4;
|
|
566
732
|
let lastProgressTime = Date.now();
|
|
@@ -568,6 +734,7 @@ var ReplayStream = class {
|
|
|
568
734
|
let itemsYielded = 0;
|
|
569
735
|
try {
|
|
570
736
|
while (true) {
|
|
737
|
+
if (signal?.aborted) return;
|
|
571
738
|
if (Date.now() - lastProgressTime > MINI_BACKFILL_TIMEOUT) {
|
|
572
739
|
this.logger.warn(`mini-backfill timed out after ${MINI_BACKFILL_TIMEOUT}ms with no progress`);
|
|
573
740
|
break;
|
|
@@ -578,6 +745,7 @@ var ReplayStream = class {
|
|
|
578
745
|
);
|
|
579
746
|
let pageYielded = 0;
|
|
580
747
|
for (const item of sorted) {
|
|
748
|
+
if (signal?.aborted) return;
|
|
581
749
|
const slot = extractSlot(item);
|
|
582
750
|
const key = keyOf(item);
|
|
583
751
|
if (seenItem(slot, key)) {
|
|
@@ -596,6 +764,9 @@ var ReplayStream = class {
|
|
|
596
764
|
}
|
|
597
765
|
this.logger.info(`mini-backfill complete: ${itemsYielded} items yielded`);
|
|
598
766
|
} catch (err) {
|
|
767
|
+
if (signal?.aborted || isAbortError(err)) {
|
|
768
|
+
return;
|
|
769
|
+
}
|
|
599
770
|
this.logger.warn(
|
|
600
771
|
`mini-backfill failed: ${err instanceof Error ? err.message : String(err)}; proceeding with live stream`
|
|
601
772
|
);
|
|
@@ -603,14 +774,22 @@ var ReplayStream = class {
|
|
|
603
774
|
}
|
|
604
775
|
};
|
|
605
776
|
async function safeClose(pump) {
|
|
777
|
+
let timeoutId;
|
|
606
778
|
try {
|
|
779
|
+
const timeout = new Promise((resolve) => {
|
|
780
|
+
timeoutId = setTimeout(() => resolve("timeout"), 5e3);
|
|
781
|
+
});
|
|
607
782
|
const result = await Promise.race([
|
|
608
783
|
pump.close().then(() => "closed"),
|
|
609
|
-
|
|
784
|
+
timeout
|
|
610
785
|
]);
|
|
611
786
|
if (result === "timeout") {
|
|
612
787
|
}
|
|
613
788
|
} catch {
|
|
789
|
+
} finally {
|
|
790
|
+
if (timeoutId !== void 0) {
|
|
791
|
+
clearTimeout(timeoutId);
|
|
792
|
+
}
|
|
614
793
|
}
|
|
615
794
|
}
|
|
616
795
|
function combineFilters(base, user) {
|
|
@@ -706,15 +885,38 @@ function createBlockReplay(options) {
|
|
|
706
885
|
subscribeLive,
|
|
707
886
|
extractSlot: (block) => block.header?.slot ?? 0n,
|
|
708
887
|
logger: options.logger,
|
|
709
|
-
resubscribeOnEnd: options.resubscribeOnEnd
|
|
888
|
+
resubscribeOnEnd: options.resubscribeOnEnd,
|
|
889
|
+
signal: options.signal
|
|
710
890
|
});
|
|
711
891
|
}
|
|
892
|
+
|
|
893
|
+
// src/types.ts
|
|
894
|
+
function resolveClient(opts, optionsName) {
|
|
895
|
+
if (opts.clientFactory) {
|
|
896
|
+
return opts.clientFactory();
|
|
897
|
+
}
|
|
898
|
+
if (!opts.client) {
|
|
899
|
+
throw new Error(`${optionsName} requires either client or clientFactory`);
|
|
900
|
+
}
|
|
901
|
+
return opts.client;
|
|
902
|
+
}
|
|
903
|
+
function closeIfCloseable(value) {
|
|
904
|
+
if (value && typeof value.close === "function") {
|
|
905
|
+
try {
|
|
906
|
+
value.close();
|
|
907
|
+
} catch {
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
}
|
|
911
|
+
|
|
912
|
+
// src/replay/transaction-replay.ts
|
|
712
913
|
var DEFAULT_PAGE_SIZE2 = 256;
|
|
713
914
|
var DEFAULT_SAFETY_MARGIN2 = 64n;
|
|
714
915
|
var PAGE_ORDER_ASC2 = "slot asc";
|
|
715
916
|
function createTransactionReplay(options) {
|
|
716
917
|
const safetyMargin = options.safetyMargin ?? DEFAULT_SAFETY_MARGIN2;
|
|
717
|
-
|
|
918
|
+
let currentClient = resolveClient(options, "TransactionReplayOptions");
|
|
919
|
+
const createFetchBackfill = (client) => async ({
|
|
718
920
|
startSlot,
|
|
719
921
|
cursor
|
|
720
922
|
}) => {
|
|
@@ -724,7 +926,7 @@ function createTransactionReplay(options) {
|
|
|
724
926
|
pageToken: cursor
|
|
725
927
|
});
|
|
726
928
|
const mergedFilter = combineFilters(slotLiteralFilter("transaction.slot", startSlot), options.filter);
|
|
727
|
-
const response = await
|
|
929
|
+
const response = await client.listTransactions(
|
|
728
930
|
create(ListTransactionsRequestSchema, {
|
|
729
931
|
filter: mergedFilter,
|
|
730
932
|
page,
|
|
@@ -734,26 +936,38 @@ function createTransactionReplay(options) {
|
|
|
734
936
|
);
|
|
735
937
|
return backfillPage(response.transactions, response.page);
|
|
736
938
|
};
|
|
737
|
-
const
|
|
939
|
+
const createSubscribeLive = (client) => (startSlot) => {
|
|
738
940
|
const mergedFilter = combineFilters(slotLiteralFilter("transaction.slot", startSlot), options.filter);
|
|
739
941
|
const request = create(StreamTransactionsRequestSchema, {
|
|
740
942
|
filter: mergedFilter,
|
|
741
943
|
minConsensus: options.minConsensus
|
|
742
944
|
});
|
|
743
945
|
return mapAsyncIterable(
|
|
744
|
-
|
|
946
|
+
client.streamTransactions(request),
|
|
745
947
|
(resp) => resp.transaction
|
|
746
948
|
);
|
|
747
949
|
};
|
|
950
|
+
const onReconnect = options.clientFactory ? () => {
|
|
951
|
+
const newClient = options.clientFactory();
|
|
952
|
+
currentClient = newClient;
|
|
953
|
+
return {
|
|
954
|
+
subscribeLive: createSubscribeLive(currentClient),
|
|
955
|
+
fetchBackfill: createFetchBackfill(currentClient),
|
|
956
|
+
dispose: () => closeIfCloseable(newClient)
|
|
957
|
+
};
|
|
958
|
+
} : void 0;
|
|
748
959
|
return new ReplayStream({
|
|
749
960
|
startSlot: options.startSlot,
|
|
750
961
|
safetyMargin,
|
|
751
|
-
fetchBackfill,
|
|
752
|
-
subscribeLive,
|
|
962
|
+
fetchBackfill: createFetchBackfill(currentClient),
|
|
963
|
+
subscribeLive: createSubscribeLive(currentClient),
|
|
753
964
|
extractSlot: (tx) => tx.slot ?? 0n,
|
|
754
965
|
extractKey: transactionKey,
|
|
755
966
|
logger: options.logger,
|
|
756
|
-
resubscribeOnEnd: options.resubscribeOnEnd
|
|
967
|
+
resubscribeOnEnd: options.resubscribeOnEnd,
|
|
968
|
+
signal: options.signal,
|
|
969
|
+
onReconnect,
|
|
970
|
+
dispose: options.clientFactory ? () => closeIfCloseable(currentClient) : void 0
|
|
757
971
|
});
|
|
758
972
|
}
|
|
759
973
|
function transactionKey(tx) {
|
|
@@ -768,19 +982,6 @@ function bytesToHex(bytes) {
|
|
|
768
982
|
for (const byte of bytes) hex += byte.toString(16).padStart(2, "0");
|
|
769
983
|
return hex;
|
|
770
984
|
}
|
|
771
|
-
|
|
772
|
-
// src/types.ts
|
|
773
|
-
function resolveClient(opts, optionsName) {
|
|
774
|
-
if (opts.clientFactory) {
|
|
775
|
-
return opts.clientFactory();
|
|
776
|
-
}
|
|
777
|
-
if (!opts.client) {
|
|
778
|
-
throw new Error(`${optionsName} requires either client or clientFactory`);
|
|
779
|
-
}
|
|
780
|
-
return opts.client;
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
// src/replay/event-replay.ts
|
|
784
985
|
var DEFAULT_PAGE_SIZE3 = 512;
|
|
785
986
|
var DEFAULT_SAFETY_MARGIN3 = 64n;
|
|
786
987
|
var PAGE_ORDER_ASC3 = "slot asc";
|
|
@@ -823,10 +1024,12 @@ function createEventReplay(options) {
|
|
|
823
1024
|
);
|
|
824
1025
|
};
|
|
825
1026
|
const onReconnect = options.clientFactory ? () => {
|
|
826
|
-
|
|
1027
|
+
const newClient = options.clientFactory();
|
|
1028
|
+
currentClient = newClient;
|
|
827
1029
|
return {
|
|
828
1030
|
subscribeLive: createSubscribeLive(currentClient),
|
|
829
|
-
fetchBackfill: createFetchBackfill(currentClient)
|
|
1031
|
+
fetchBackfill: createFetchBackfill(currentClient),
|
|
1032
|
+
dispose: () => closeIfCloseable(newClient)
|
|
830
1033
|
};
|
|
831
1034
|
} : void 0;
|
|
832
1035
|
return new ReplayStream({
|
|
@@ -838,7 +1041,9 @@ function createEventReplay(options) {
|
|
|
838
1041
|
extractKey: eventKey,
|
|
839
1042
|
logger: options.logger,
|
|
840
1043
|
resubscribeOnEnd: options.resubscribeOnEnd,
|
|
841
|
-
|
|
1044
|
+
signal: options.signal,
|
|
1045
|
+
onReconnect,
|
|
1046
|
+
dispose: options.clientFactory ? () => closeIfCloseable(currentClient) : void 0
|
|
842
1047
|
});
|
|
843
1048
|
}
|
|
844
1049
|
function streamResponseToEvent(resp) {
|
|
@@ -863,7 +1068,7 @@ function eventBackfillFilter(startSlot, resumeAfter) {
|
|
|
863
1068
|
return slotLiteralFilter("event.slot", startSlot);
|
|
864
1069
|
}
|
|
865
1070
|
return create(FilterSchema, {
|
|
866
|
-
expression: `event.slot > uint(${boundary.slot.toString()}) || (event.slot == uint(${boundary.slot.toString()}) && (event.block_offset > uint(${boundary.blockOffset.toString()}) || (event.block_offset == uint(${boundary.blockOffset.toString()}) && event.call_idx
|
|
1071
|
+
expression: `event.slot > uint(${boundary.slot.toString()}) || (event.slot == uint(${boundary.slot.toString()}) && (event.block_offset > uint(${boundary.blockOffset.toString()}) || (event.block_offset == uint(${boundary.blockOffset.toString()}) && event.call_idx >= uint(${boundary.callIdx.toString()}))))`
|
|
867
1072
|
});
|
|
868
1073
|
}
|
|
869
1074
|
function parseEventId(resumeAfter) {
|
|
@@ -872,23 +1077,27 @@ function parseEventId(resumeAfter) {
|
|
|
872
1077
|
}
|
|
873
1078
|
function parseCanonicalEventId(eventId, expectedSlot) {
|
|
874
1079
|
if (!eventId) return null;
|
|
875
|
-
const
|
|
876
|
-
if (
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
1080
|
+
const parts = eventId.split(":");
|
|
1081
|
+
if (parts.length === 5) {
|
|
1082
|
+
const [, slotPart, blockOffsetPart, callIdxPart, eventIdxPart] = parts;
|
|
1083
|
+
const slot = BigInt(slotPart);
|
|
1084
|
+
if (slot !== expectedSlot) return null;
|
|
1085
|
+
return {
|
|
1086
|
+
slot,
|
|
1087
|
+
blockOffset: BigInt(blockOffsetPart),
|
|
1088
|
+
callIdx: BigInt(callIdxPart),
|
|
1089
|
+
eventIdx: BigInt(eventIdxPart)
|
|
1090
|
+
};
|
|
1091
|
+
}
|
|
1092
|
+
return null;
|
|
885
1093
|
}
|
|
886
1094
|
function isAfterBoundary(event, boundary) {
|
|
887
1095
|
if (event.slot !== boundary.slot) return event.slot > boundary.slot;
|
|
888
1096
|
if (event.blockOffset !== boundary.blockOffset) {
|
|
889
1097
|
return event.blockOffset > boundary.blockOffset;
|
|
890
1098
|
}
|
|
891
|
-
return event.callIdx > boundary.callIdx;
|
|
1099
|
+
if (event.callIdx !== boundary.callIdx) return event.callIdx > boundary.callIdx;
|
|
1100
|
+
return event.eventIdx > boundary.eventIdx;
|
|
892
1101
|
}
|
|
893
1102
|
function shouldEmitLiveEvent(event, startSlot, resumeAfter) {
|
|
894
1103
|
const boundary = parseEventId(resumeAfter);
|
|
@@ -1059,6 +1268,15 @@ var PageAssembler = class {
|
|
|
1059
1268
|
};
|
|
1060
1269
|
|
|
1061
1270
|
// src/account-replay.ts
|
|
1271
|
+
async function closeAsyncIterator(iterator) {
|
|
1272
|
+
if (!iterator || typeof iterator.return !== "function") {
|
|
1273
|
+
return;
|
|
1274
|
+
}
|
|
1275
|
+
try {
|
|
1276
|
+
await iterator.return();
|
|
1277
|
+
} catch {
|
|
1278
|
+
}
|
|
1279
|
+
}
|
|
1062
1280
|
function bytesToHex2(bytes) {
|
|
1063
1281
|
return Array.from(bytes).map((b) => b.toString(16).padStart(2, "0")).join("");
|
|
1064
1282
|
}
|
|
@@ -1133,9 +1351,12 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1133
1351
|
cleanupInterval = 1e4,
|
|
1134
1352
|
onBackfillComplete,
|
|
1135
1353
|
clientFactory,
|
|
1136
|
-
logger = NOOP_LOGGER
|
|
1354
|
+
logger = NOOP_LOGGER,
|
|
1355
|
+
signal
|
|
1137
1356
|
} = options;
|
|
1357
|
+
const shouldStop = (err) => signal?.aborted === true || isAbortError(err);
|
|
1138
1358
|
let client = resolveClient(options, "AccountsByOwnerReplayOptions");
|
|
1359
|
+
const ownsClient = Boolean(clientFactory);
|
|
1139
1360
|
const seenFromStream = /* @__PURE__ */ new Set();
|
|
1140
1361
|
const fetchQueue = [];
|
|
1141
1362
|
let highestSlotSeen = minUpdatedSlot ?? 0n;
|
|
@@ -1145,15 +1366,24 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1145
1366
|
let streamDone = false;
|
|
1146
1367
|
let streamError = null;
|
|
1147
1368
|
let lastActivityTime = Date.now();
|
|
1369
|
+
let activeStreamIterator = null;
|
|
1370
|
+
let activeStreamProcessor = null;
|
|
1148
1371
|
try {
|
|
1372
|
+
if (shouldStop()) return;
|
|
1149
1373
|
cleanupTimer = setInterval(() => {
|
|
1150
1374
|
assembler.cleanup();
|
|
1151
1375
|
}, cleanupInterval);
|
|
1376
|
+
cleanupTimer.unref?.();
|
|
1152
1377
|
const streamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, minUpdatedSlot);
|
|
1153
1378
|
const stream = client.streamAccountUpdates({ view, filter: streamFilter });
|
|
1154
|
-
const
|
|
1379
|
+
const streamIterator = stream[Symbol.asyncIterator]();
|
|
1380
|
+
activeStreamIterator = streamIterator;
|
|
1381
|
+
activeStreamProcessor = (async () => {
|
|
1155
1382
|
try {
|
|
1156
|
-
|
|
1383
|
+
while (true) {
|
|
1384
|
+
const next = await streamIterator.next();
|
|
1385
|
+
if (next.done) break;
|
|
1386
|
+
const response = next.value;
|
|
1157
1387
|
lastActivityTime = Date.now();
|
|
1158
1388
|
const event = processResponseMulti(response, assembler);
|
|
1159
1389
|
if (event) {
|
|
@@ -1184,6 +1414,7 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1184
1414
|
const backfillFilter = buildListAccountsOwnerFilter(owner, dataSizes, minUpdatedSlot);
|
|
1185
1415
|
let pageToken;
|
|
1186
1416
|
do {
|
|
1417
|
+
if (shouldStop()) return;
|
|
1187
1418
|
const request = {
|
|
1188
1419
|
view: AccountView.META_ONLY,
|
|
1189
1420
|
// Address + metadata only, no data
|
|
@@ -1193,7 +1424,13 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1193
1424
|
pageToken
|
|
1194
1425
|
})
|
|
1195
1426
|
};
|
|
1196
|
-
|
|
1427
|
+
let response;
|
|
1428
|
+
try {
|
|
1429
|
+
response = await client.listAccounts(request);
|
|
1430
|
+
} catch (err) {
|
|
1431
|
+
if (shouldStop(err)) return;
|
|
1432
|
+
throw err;
|
|
1433
|
+
}
|
|
1197
1434
|
for (const account of response.accounts) {
|
|
1198
1435
|
if (account.address?.value) {
|
|
1199
1436
|
fetchQueue.push(account.address.value);
|
|
@@ -1203,6 +1440,7 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1203
1440
|
yield* yieldStreamBuffer();
|
|
1204
1441
|
} while (pageToken);
|
|
1205
1442
|
for (const address of fetchQueue) {
|
|
1443
|
+
if (shouldStop()) return;
|
|
1206
1444
|
const addressHex = bytesToHex2(address);
|
|
1207
1445
|
if (seenFromStream.has(addressHex)) {
|
|
1208
1446
|
continue;
|
|
@@ -1220,10 +1458,11 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1220
1458
|
});
|
|
1221
1459
|
break;
|
|
1222
1460
|
} catch (err) {
|
|
1461
|
+
if (shouldStop(err)) return;
|
|
1223
1462
|
if (attempt === maxRetries - 1) {
|
|
1224
1463
|
logger.error(`[backfill] failed to fetch account ${addressHex} after ${maxRetries} attempts`, { error: err });
|
|
1225
1464
|
} else {
|
|
1226
|
-
await
|
|
1465
|
+
await abortableDelay(100 * (attempt + 1), signal);
|
|
1227
1466
|
}
|
|
1228
1467
|
}
|
|
1229
1468
|
}
|
|
@@ -1242,13 +1481,13 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1242
1481
|
}
|
|
1243
1482
|
const retryConfig = DEFAULT_RETRY_CONFIG;
|
|
1244
1483
|
let retryAttempt = 0;
|
|
1245
|
-
let currentStream = stream;
|
|
1246
|
-
let currentStreamProcessor = streamProcessor;
|
|
1247
1484
|
lastActivityTime = Date.now();
|
|
1248
1485
|
const createStreamProcessor = () => {
|
|
1249
1486
|
if (clientFactory) {
|
|
1250
1487
|
try {
|
|
1251
|
-
|
|
1488
|
+
const newClient = clientFactory();
|
|
1489
|
+
closeIfCloseable(client);
|
|
1490
|
+
client = newClient;
|
|
1252
1491
|
logger.info("[account-stream] created fresh client for reconnection");
|
|
1253
1492
|
} catch (err) {
|
|
1254
1493
|
logger.error("[account-stream] failed to create fresh client", { error: err });
|
|
@@ -1256,9 +1495,14 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1256
1495
|
}
|
|
1257
1496
|
const newStreamFilter = buildOwnerFilterWithMinSlot(owner, dataSizes, highestSlotSeen > 0n ? highestSlotSeen : minUpdatedSlot);
|
|
1258
1497
|
const newStream = client.streamAccountUpdates({ view, filter: newStreamFilter });
|
|
1498
|
+
const newStreamIterator = newStream[Symbol.asyncIterator]();
|
|
1499
|
+
activeStreamIterator = newStreamIterator;
|
|
1259
1500
|
const newProcessor = (async () => {
|
|
1260
1501
|
try {
|
|
1261
|
-
|
|
1502
|
+
while (true) {
|
|
1503
|
+
const next = await newStreamIterator.next();
|
|
1504
|
+
if (next.done) break;
|
|
1505
|
+
const response = next.value;
|
|
1262
1506
|
retryAttempt = 0;
|
|
1263
1507
|
lastActivityTime = Date.now();
|
|
1264
1508
|
const event = processResponseMulti(response, assembler);
|
|
@@ -1278,9 +1522,10 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1278
1522
|
streamDone = true;
|
|
1279
1523
|
}
|
|
1280
1524
|
})();
|
|
1281
|
-
return {
|
|
1525
|
+
return { iterator: newStreamIterator, processor: newProcessor };
|
|
1282
1526
|
};
|
|
1283
1527
|
while (true) {
|
|
1528
|
+
if (shouldStop()) return;
|
|
1284
1529
|
const hadEvents = streamBuffer.length > 0;
|
|
1285
1530
|
yield* yieldStreamBuffer();
|
|
1286
1531
|
if (hadEvents) {
|
|
@@ -1295,36 +1540,56 @@ async function* createAccountsByOwnerReplay(options) {
|
|
|
1295
1540
|
}
|
|
1296
1541
|
if (streamDone) {
|
|
1297
1542
|
if (streamError) {
|
|
1543
|
+
if (shouldStop(streamError)) return;
|
|
1298
1544
|
const backoffMs = calculateBackoff(retryAttempt, retryConfig);
|
|
1299
1545
|
logger.warn(
|
|
1300
1546
|
`[account-stream] disconnected (${streamError.message}); reconnecting in ${backoffMs}ms (attempt ${retryAttempt + 1})`
|
|
1301
1547
|
);
|
|
1302
|
-
await
|
|
1548
|
+
await abortableDelay(backoffMs, signal);
|
|
1549
|
+
if (shouldStop()) return;
|
|
1303
1550
|
retryAttempt++;
|
|
1304
1551
|
streamDone = false;
|
|
1305
1552
|
streamError = null;
|
|
1306
1553
|
streamBuffer.length = 0;
|
|
1307
1554
|
lastActivityTime = Date.now();
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1555
|
+
await closeAsyncIterator(activeStreamIterator);
|
|
1556
|
+
if (activeStreamProcessor) {
|
|
1557
|
+
await Promise.allSettled([activeStreamProcessor]);
|
|
1558
|
+
}
|
|
1559
|
+
const { iterator: newIterator, processor: newProcessor } = createStreamProcessor();
|
|
1560
|
+
activeStreamIterator = newIterator;
|
|
1561
|
+
activeStreamProcessor = newProcessor;
|
|
1311
1562
|
continue;
|
|
1312
1563
|
} else {
|
|
1564
|
+
if (shouldStop()) return;
|
|
1313
1565
|
logger.warn("[account-stream] stream ended unexpectedly; reconnecting...");
|
|
1314
1566
|
streamDone = false;
|
|
1315
1567
|
lastActivityTime = Date.now();
|
|
1316
|
-
|
|
1317
|
-
|
|
1318
|
-
|
|
1568
|
+
await closeAsyncIterator(activeStreamIterator);
|
|
1569
|
+
if (activeStreamProcessor) {
|
|
1570
|
+
await Promise.allSettled([activeStreamProcessor]);
|
|
1571
|
+
}
|
|
1572
|
+
const { iterator: newIterator, processor: newProcessor } = createStreamProcessor();
|
|
1573
|
+
activeStreamIterator = newIterator;
|
|
1574
|
+
activeStreamProcessor = newProcessor;
|
|
1319
1575
|
continue;
|
|
1320
1576
|
}
|
|
1321
1577
|
}
|
|
1322
|
-
await
|
|
1578
|
+
await abortableDelay(10, signal);
|
|
1323
1579
|
}
|
|
1324
1580
|
} finally {
|
|
1325
1581
|
if (cleanupTimer) {
|
|
1326
1582
|
clearInterval(cleanupTimer);
|
|
1327
1583
|
}
|
|
1584
|
+
const closeIteratorPromise = closeAsyncIterator(activeStreamIterator);
|
|
1585
|
+
if (ownsClient) {
|
|
1586
|
+
closeIfCloseable(client);
|
|
1587
|
+
}
|
|
1588
|
+
if (activeStreamProcessor) {
|
|
1589
|
+
await Promise.allSettled([closeIteratorPromise, activeStreamProcessor]);
|
|
1590
|
+
} else {
|
|
1591
|
+
await closeIteratorPromise;
|
|
1592
|
+
}
|
|
1328
1593
|
assembler.clear();
|
|
1329
1594
|
}
|
|
1330
1595
|
}
|
|
@@ -1360,6 +1625,7 @@ async function* createAccountReplay(options) {
|
|
|
1360
1625
|
cleanupTimer = setInterval(() => {
|
|
1361
1626
|
assembler.cleanup();
|
|
1362
1627
|
}, cleanupInterval);
|
|
1628
|
+
cleanupTimer.unref?.();
|
|
1363
1629
|
const filterParams = {
|
|
1364
1630
|
address: create(FilterParamValueSchema, { kind: { case: "bytesValue", value: new Uint8Array(address) } })
|
|
1365
1631
|
};
|
|
@@ -1566,6 +1832,7 @@ var ConsoleSink = class {
|
|
|
1566
1832
|
constructor(prefix = "ReplaySink") {
|
|
1567
1833
|
this.prefix = prefix;
|
|
1568
1834
|
}
|
|
1835
|
+
prefix;
|
|
1569
1836
|
open(meta) {
|
|
1570
1837
|
const suffix = meta?.stream ? ` (${meta.stream})` : "";
|
|
1571
1838
|
console.info(`${this.prefix}${suffix} opened`, meta?.label ?? "");
|