@durable-streams/client 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -10
- package/dist/index.cjs +954 -795
- package/dist/index.d.cts +63 -25
- package/dist/index.d.ts +63 -25
- package/dist/index.js +954 -795
- package/package.json +2 -2
- package/src/idempotent-producer.ts +51 -38
- package/src/response.ts +258 -23
- package/src/sse.ts +17 -4
- package/src/stream-api.ts +22 -9
- package/src/stream.ts +77 -56
- package/src/types.ts +24 -12
package/dist/index.js
CHANGED
|
@@ -482,7 +482,10 @@ async function* parseSSEStream(stream$1, signal) {
|
|
|
482
482
|
streamCursor: control.streamCursor,
|
|
483
483
|
upToDate: control.upToDate
|
|
484
484
|
};
|
|
485
|
-
} catch {
|
|
485
|
+
} catch (err) {
|
|
486
|
+
const preview = dataStr.length > 100 ? dataStr.slice(0, 100) + `...` : dataStr;
|
|
487
|
+
throw new DurableStreamError(`Failed to parse SSE control event: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`, `PARSE_ERROR`);
|
|
488
|
+
}
|
|
486
489
|
}
|
|
487
490
|
currentEvent = { data: [] };
|
|
488
491
|
} else if (line.startsWith(`event:`)) currentEvent.type = line.slice(6).trim();
|
|
@@ -507,7 +510,10 @@ async function* parseSSEStream(stream$1, signal) {
|
|
|
507
510
|
streamCursor: control.streamCursor,
|
|
508
511
|
upToDate: control.upToDate
|
|
509
512
|
};
|
|
510
|
-
} catch {
|
|
513
|
+
} catch (err) {
|
|
514
|
+
const preview = dataStr.length > 100 ? dataStr.slice(0, 100) + `...` : dataStr;
|
|
515
|
+
throw new DurableStreamError(`Failed to parse SSE control event: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`, `PARSE_ERROR`);
|
|
516
|
+
}
|
|
511
517
|
}
|
|
512
518
|
} finally {
|
|
513
519
|
reader.releaseLock();
|
|
@@ -517,6 +523,10 @@ async function* parseSSEStream(stream$1, signal) {
|
|
|
517
523
|
//#endregion
|
|
518
524
|
//#region src/response.ts
|
|
519
525
|
/**
|
|
526
|
+
* Constant used as abort reason when pausing the stream due to visibility change.
|
|
527
|
+
*/
|
|
528
|
+
const PAUSE_STREAM = `PAUSE_STREAM`;
|
|
529
|
+
/**
|
|
520
530
|
* Implementation of the StreamResponse interface.
|
|
521
531
|
*/
|
|
522
532
|
var StreamResponseImpl = class {
|
|
@@ -529,9 +539,9 @@ var StreamResponseImpl = class {
|
|
|
529
539
|
#statusText;
|
|
530
540
|
#ok;
|
|
531
541
|
#isLoading;
|
|
532
|
-
offset;
|
|
533
|
-
cursor;
|
|
534
|
-
upToDate;
|
|
542
|
+
#offset;
|
|
543
|
+
#cursor;
|
|
544
|
+
#upToDate;
|
|
535
545
|
#isJsonMode;
|
|
536
546
|
#abortController;
|
|
537
547
|
#fetchNext;
|
|
@@ -541,6 +551,12 @@ var StreamResponseImpl = class {
|
|
|
541
551
|
#closed;
|
|
542
552
|
#stopAfterUpToDate = false;
|
|
543
553
|
#consumptionMethod = null;
|
|
554
|
+
#state = `active`;
|
|
555
|
+
#requestAbortController;
|
|
556
|
+
#unsubscribeFromVisibilityChanges;
|
|
557
|
+
#pausePromise;
|
|
558
|
+
#pauseResolve;
|
|
559
|
+
#justResumedFromPause = false;
|
|
544
560
|
#sseResilience;
|
|
545
561
|
#lastSSEConnectionStartTime;
|
|
546
562
|
#consecutiveShortSSEConnections = 0;
|
|
@@ -551,9 +567,9 @@ var StreamResponseImpl = class {
|
|
|
551
567
|
this.contentType = config.contentType;
|
|
552
568
|
this.live = config.live;
|
|
553
569
|
this.startOffset = config.startOffset;
|
|
554
|
-
this
|
|
555
|
-
this
|
|
556
|
-
this
|
|
570
|
+
this.#offset = config.initialOffset;
|
|
571
|
+
this.#cursor = config.initialCursor;
|
|
572
|
+
this.#upToDate = config.initialUpToDate;
|
|
557
573
|
this.#headers = config.firstResponse.headers;
|
|
558
574
|
this.#status = config.firstResponse.status;
|
|
559
575
|
this.#statusText = config.firstResponse.statusText;
|
|
@@ -575,6 +591,59 @@ var StreamResponseImpl = class {
|
|
|
575
591
|
this.#closedReject = reject;
|
|
576
592
|
});
|
|
577
593
|
this.#responseStream = this.#createResponseStream(config.firstResponse);
|
|
594
|
+
this.#abortController.signal.addEventListener(`abort`, () => {
|
|
595
|
+
this.#requestAbortController?.abort(this.#abortController.signal.reason);
|
|
596
|
+
this.#pauseResolve?.();
|
|
597
|
+
this.#pausePromise = void 0;
|
|
598
|
+
this.#pauseResolve = void 0;
|
|
599
|
+
}, { once: true });
|
|
600
|
+
this.#subscribeToVisibilityChanges();
|
|
601
|
+
}
|
|
602
|
+
/**
|
|
603
|
+
* Subscribe to document visibility changes to pause/resume syncing.
|
|
604
|
+
* When the page is hidden, we pause to save battery and bandwidth.
|
|
605
|
+
* When visible again, we resume syncing.
|
|
606
|
+
*/
|
|
607
|
+
#subscribeToVisibilityChanges() {
|
|
608
|
+
if (typeof document === `object` && typeof document.hidden === `boolean` && typeof document.addEventListener === `function`) {
|
|
609
|
+
const visibilityHandler = () => {
|
|
610
|
+
if (document.hidden) this.#pause();
|
|
611
|
+
else this.#resume();
|
|
612
|
+
};
|
|
613
|
+
document.addEventListener(`visibilitychange`, visibilityHandler);
|
|
614
|
+
this.#unsubscribeFromVisibilityChanges = () => {
|
|
615
|
+
if (typeof document === `object`) document.removeEventListener(`visibilitychange`, visibilityHandler);
|
|
616
|
+
};
|
|
617
|
+
if (document.hidden) this.#pause();
|
|
618
|
+
}
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* Pause the stream when page becomes hidden.
|
|
622
|
+
* Aborts any in-flight request to free resources.
|
|
623
|
+
* Creates a promise that pull() will await while paused.
|
|
624
|
+
*/
|
|
625
|
+
#pause() {
|
|
626
|
+
if (this.#state === `active`) {
|
|
627
|
+
this.#state = `pause-requested`;
|
|
628
|
+
this.#pausePromise = new Promise((resolve) => {
|
|
629
|
+
this.#pauseResolve = resolve;
|
|
630
|
+
});
|
|
631
|
+
this.#requestAbortController?.abort(PAUSE_STREAM);
|
|
632
|
+
}
|
|
633
|
+
}
|
|
634
|
+
/**
|
|
635
|
+
* Resume the stream when page becomes visible.
|
|
636
|
+
* Resolves the pause promise to unblock pull().
|
|
637
|
+
*/
|
|
638
|
+
#resume() {
|
|
639
|
+
if (this.#state === `paused` || this.#state === `pause-requested`) {
|
|
640
|
+
if (this.#abortController.signal.aborted) return;
|
|
641
|
+
this.#state = `active`;
|
|
642
|
+
this.#justResumedFromPause = true;
|
|
643
|
+
this.#pauseResolve?.();
|
|
644
|
+
this.#pausePromise = void 0;
|
|
645
|
+
this.#pauseResolve = void 0;
|
|
646
|
+
}
|
|
578
647
|
}
|
|
579
648
|
get headers() {
|
|
580
649
|
return this.#headers;
|
|
@@ -591,13 +660,24 @@ var StreamResponseImpl = class {
|
|
|
591
660
|
get isLoading() {
|
|
592
661
|
return this.#isLoading;
|
|
593
662
|
}
|
|
663
|
+
get offset() {
|
|
664
|
+
return this.#offset;
|
|
665
|
+
}
|
|
666
|
+
get cursor() {
|
|
667
|
+
return this.#cursor;
|
|
668
|
+
}
|
|
669
|
+
get upToDate() {
|
|
670
|
+
return this.#upToDate;
|
|
671
|
+
}
|
|
594
672
|
#ensureJsonMode() {
|
|
595
673
|
if (!this.#isJsonMode) throw new DurableStreamError(`JSON methods are only valid for JSON-mode streams. Content-Type is "${this.contentType}" and json hint was not set.`, `BAD_REQUEST`);
|
|
596
674
|
}
|
|
597
675
|
#markClosed() {
|
|
676
|
+
this.#unsubscribeFromVisibilityChanges?.();
|
|
598
677
|
this.#closedResolve();
|
|
599
678
|
}
|
|
600
679
|
#markError(err) {
|
|
680
|
+
this.#unsubscribeFromVisibilityChanges?.();
|
|
601
681
|
this.#closedReject(err);
|
|
602
682
|
}
|
|
603
683
|
/**
|
|
@@ -622,10 +702,10 @@ var StreamResponseImpl = class {
|
|
|
622
702
|
*/
|
|
623
703
|
#updateStateFromResponse(response) {
|
|
624
704
|
const offset = response.headers.get(STREAM_OFFSET_HEADER);
|
|
625
|
-
if (offset) this
|
|
705
|
+
if (offset) this.#offset = offset;
|
|
626
706
|
const cursor = response.headers.get(STREAM_CURSOR_HEADER);
|
|
627
|
-
if (cursor) this
|
|
628
|
-
this
|
|
707
|
+
if (cursor) this.#cursor = cursor;
|
|
708
|
+
this.#upToDate = response.headers.has(STREAM_UP_TO_DATE_HEADER);
|
|
629
709
|
this.#headers = response.headers;
|
|
630
710
|
this.#status = response.status;
|
|
631
711
|
this.#statusText = response.statusText;
|
|
@@ -667,9 +747,9 @@ var StreamResponseImpl = class {
|
|
|
667
747
|
* Update instance state from an SSE control event.
|
|
668
748
|
*/
|
|
669
749
|
#updateStateFromSSEControl(controlEvent) {
|
|
670
|
-
this
|
|
671
|
-
if (controlEvent.streamCursor) this
|
|
672
|
-
if (controlEvent.upToDate !== void 0) this
|
|
750
|
+
this.#offset = controlEvent.streamNextOffset;
|
|
751
|
+
if (controlEvent.streamCursor) this.#cursor = controlEvent.streamCursor;
|
|
752
|
+
if (controlEvent.upToDate !== void 0) this.#upToDate = controlEvent.upToDate;
|
|
673
753
|
}
|
|
674
754
|
/**
|
|
675
755
|
* Mark the start of an SSE connection for duration tracking.
|
|
@@ -710,8 +790,9 @@ var StreamResponseImpl = class {
|
|
|
710
790
|
const delayOrNull = await this.#handleSSEConnectionEnd();
|
|
711
791
|
if (delayOrNull === null) return null;
|
|
712
792
|
this.#markSSEConnectionStart();
|
|
713
|
-
|
|
714
|
-
|
|
793
|
+
this.#requestAbortController = new AbortController();
|
|
794
|
+
const newSSEResponse = await this.#startSSE(this.offset, this.cursor, this.#requestAbortController.signal);
|
|
795
|
+
if (newSSEResponse.body) return parseSSEStream(newSSEResponse.body, this.#requestAbortController.signal);
|
|
715
796
|
return null;
|
|
716
797
|
}
|
|
717
798
|
/**
|
|
@@ -797,7 +878,8 @@ var StreamResponseImpl = class {
|
|
|
797
878
|
const isSSE = firstResponse.headers.get(`content-type`)?.includes(`text/event-stream`) ?? false;
|
|
798
879
|
if (isSSE && firstResponse.body) {
|
|
799
880
|
this.#markSSEConnectionStart();
|
|
800
|
-
|
|
881
|
+
this.#requestAbortController = new AbortController();
|
|
882
|
+
sseEventIterator = parseSSEStream(firstResponse.body, this.#requestAbortController.signal);
|
|
801
883
|
} else {
|
|
802
884
|
controller.enqueue(firstResponse);
|
|
803
885
|
if (this.upToDate && !this.#shouldContinueLive()) {
|
|
@@ -808,33 +890,63 @@ var StreamResponseImpl = class {
|
|
|
808
890
|
return;
|
|
809
891
|
}
|
|
810
892
|
}
|
|
811
|
-
if (sseEventIterator)
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
controller.enqueue(result.response);
|
|
817
|
-
return;
|
|
818
|
-
case `closed`:
|
|
893
|
+
if (sseEventIterator) {
|
|
894
|
+
if (this.#state === `pause-requested` || this.#state === `paused`) {
|
|
895
|
+
this.#state = `paused`;
|
|
896
|
+
if (this.#pausePromise) await this.#pausePromise;
|
|
897
|
+
if (this.#abortController.signal.aborted) {
|
|
819
898
|
this.#markClosed();
|
|
820
899
|
controller.close();
|
|
821
900
|
return;
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
901
|
+
}
|
|
902
|
+
const newIterator = await this.#trySSEReconnect();
|
|
903
|
+
if (newIterator) sseEventIterator = newIterator;
|
|
904
|
+
else {
|
|
905
|
+
this.#markClosed();
|
|
906
|
+
controller.close();
|
|
825
907
|
return;
|
|
826
|
-
|
|
827
|
-
|
|
828
|
-
|
|
908
|
+
}
|
|
909
|
+
}
|
|
910
|
+
while (true) {
|
|
911
|
+
const result = await this.#processSSEEvents(sseEventIterator);
|
|
912
|
+
switch (result.type) {
|
|
913
|
+
case `response`:
|
|
914
|
+
if (result.newIterator) sseEventIterator = result.newIterator;
|
|
915
|
+
controller.enqueue(result.response);
|
|
916
|
+
return;
|
|
917
|
+
case `closed`:
|
|
918
|
+
this.#markClosed();
|
|
919
|
+
controller.close();
|
|
920
|
+
return;
|
|
921
|
+
case `error`:
|
|
922
|
+
this.#markError(result.error);
|
|
923
|
+
controller.error(result.error);
|
|
924
|
+
return;
|
|
925
|
+
case `continue`:
|
|
926
|
+
if (result.newIterator) sseEventIterator = result.newIterator;
|
|
927
|
+
continue;
|
|
928
|
+
}
|
|
829
929
|
}
|
|
830
930
|
}
|
|
831
931
|
if (this.#shouldContinueLive()) {
|
|
932
|
+
if (this.#state === `pause-requested` || this.#state === `paused`) {
|
|
933
|
+
this.#state = `paused`;
|
|
934
|
+
if (this.#pausePromise) await this.#pausePromise;
|
|
935
|
+
if (this.#abortController.signal.aborted) {
|
|
936
|
+
this.#markClosed();
|
|
937
|
+
controller.close();
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
940
|
+
}
|
|
832
941
|
if (this.#abortController.signal.aborted) {
|
|
833
942
|
this.#markClosed();
|
|
834
943
|
controller.close();
|
|
835
944
|
return;
|
|
836
945
|
}
|
|
837
|
-
const
|
|
946
|
+
const resumingFromPause = this.#justResumedFromPause;
|
|
947
|
+
this.#justResumedFromPause = false;
|
|
948
|
+
this.#requestAbortController = new AbortController();
|
|
949
|
+
const response = await this.#fetchNext(this.offset, this.cursor, this.#requestAbortController.signal, resumingFromPause);
|
|
838
950
|
this.#updateStateFromResponse(response);
|
|
839
951
|
controller.enqueue(response);
|
|
840
952
|
return;
|
|
@@ -842,6 +954,10 @@ var StreamResponseImpl = class {
|
|
|
842
954
|
this.#markClosed();
|
|
843
955
|
controller.close();
|
|
844
956
|
} catch (err) {
|
|
957
|
+
if (this.#requestAbortController?.signal.aborted && this.#requestAbortController.signal.reason === PAUSE_STREAM) {
|
|
958
|
+
if (this.#state === `pause-requested`) this.#state = `paused`;
|
|
959
|
+
return;
|
|
960
|
+
}
|
|
845
961
|
if (this.#abortController.signal.aborted) {
|
|
846
962
|
this.#markClosed();
|
|
847
963
|
controller.close();
|
|
@@ -853,6 +969,7 @@ var StreamResponseImpl = class {
|
|
|
853
969
|
},
|
|
854
970
|
cancel: () => {
|
|
855
971
|
this.#abortController.abort();
|
|
972
|
+
this.#unsubscribeFromVisibilityChanges?.();
|
|
856
973
|
this.#markClosed();
|
|
857
974
|
}
|
|
858
975
|
});
|
|
@@ -898,7 +1015,13 @@ var StreamResponseImpl = class {
|
|
|
898
1015
|
const wasUpToDate = this.upToDate;
|
|
899
1016
|
const text = await result.value.text();
|
|
900
1017
|
const content = text.trim() || `[]`;
|
|
901
|
-
|
|
1018
|
+
let parsed;
|
|
1019
|
+
try {
|
|
1020
|
+
parsed = JSON.parse(content);
|
|
1021
|
+
} catch (err) {
|
|
1022
|
+
const preview = content.length > 100 ? content.slice(0, 100) + `...` : content;
|
|
1023
|
+
throw new DurableStreamError(`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`, `PARSE_ERROR`);
|
|
1024
|
+
}
|
|
902
1025
|
if (Array.isArray(parsed)) items.push(...parsed);
|
|
903
1026
|
else items.push(parsed);
|
|
904
1027
|
if (wasUpToDate) break;
|
|
@@ -995,7 +1118,13 @@ var StreamResponseImpl = class {
|
|
|
995
1118
|
}
|
|
996
1119
|
const text = await response.text();
|
|
997
1120
|
const content = text.trim() || `[]`;
|
|
998
|
-
|
|
1121
|
+
let parsed;
|
|
1122
|
+
try {
|
|
1123
|
+
parsed = JSON.parse(content);
|
|
1124
|
+
} catch (err) {
|
|
1125
|
+
const preview = content.length > 100 ? content.slice(0, 100) + `...` : content;
|
|
1126
|
+
throw new DurableStreamError(`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`, `PARSE_ERROR`);
|
|
1127
|
+
}
|
|
999
1128
|
pendingItems = Array.isArray(parsed) ? parsed : [parsed];
|
|
1000
1129
|
if (pendingItems.length > 0) controller.enqueue(pendingItems.shift());
|
|
1001
1130
|
},
|
|
@@ -1034,7 +1163,13 @@ var StreamResponseImpl = class {
|
|
|
1034
1163
|
const { offset, cursor, upToDate } = this.#getMetadataFromResponse(response);
|
|
1035
1164
|
const text = await response.text();
|
|
1036
1165
|
const content = text.trim() || `[]`;
|
|
1037
|
-
|
|
1166
|
+
let parsed;
|
|
1167
|
+
try {
|
|
1168
|
+
parsed = JSON.parse(content);
|
|
1169
|
+
} catch (err) {
|
|
1170
|
+
const preview = content.length > 100 ? content.slice(0, 100) + `...` : content;
|
|
1171
|
+
throw new DurableStreamError(`Failed to parse JSON response: ${err instanceof Error ? err.message : String(err)}. Data: ${preview}`, `PARSE_ERROR`);
|
|
1172
|
+
}
|
|
1038
1173
|
const items = Array.isArray(parsed) ? parsed : [parsed];
|
|
1039
1174
|
await subscriber({
|
|
1040
1175
|
items,
|
|
@@ -1134,6 +1269,7 @@ var StreamResponseImpl = class {
|
|
|
1134
1269
|
}
|
|
1135
1270
|
cancel(reason) {
|
|
1136
1271
|
this.#abortController.abort(reason);
|
|
1272
|
+
this.#unsubscribeFromVisibilityChanges?.();
|
|
1137
1273
|
this.#markClosed();
|
|
1138
1274
|
}
|
|
1139
1275
|
get closed() {
|
|
@@ -1278,7 +1414,7 @@ function _resetHttpWarningForTesting() {
|
|
|
1278
1414
|
* url,
|
|
1279
1415
|
* auth,
|
|
1280
1416
|
* offset: savedOffset,
|
|
1281
|
-
* live:
|
|
1417
|
+
* live: true,
|
|
1282
1418
|
* })
|
|
1283
1419
|
* live.subscribeJson(async (batch) => {
|
|
1284
1420
|
* for (const item of batch.items) {
|
|
@@ -1319,10 +1455,11 @@ async function stream(options) {
|
|
|
1319
1455
|
*/
|
|
1320
1456
|
async function streamInternal(options) {
|
|
1321
1457
|
const url = options.url instanceof URL ? options.url.toString() : options.url;
|
|
1458
|
+
warnIfUsingHttpInBrowser(url, options.warnOnHttp);
|
|
1322
1459
|
const fetchUrl = new URL(url);
|
|
1323
1460
|
const startOffset = options.offset ?? `-1`;
|
|
1324
1461
|
fetchUrl.searchParams.set(OFFSET_QUERY_PARAM, startOffset);
|
|
1325
|
-
const live = options.live ??
|
|
1462
|
+
const live = options.live ?? true;
|
|
1326
1463
|
if (live === `long-poll` || live === `sse`) fetchUrl.searchParams.set(LIVE_QUERY_PARAM, live);
|
|
1327
1464
|
const params = await resolveParams(options.params);
|
|
1328
1465
|
for (const [key, value] of Object.entries(params)) fetchUrl.searchParams.set(key, value);
|
|
@@ -1348,11 +1485,13 @@ async function streamInternal(options) {
|
|
|
1348
1485
|
const initialCursor = firstResponse.headers.get(STREAM_CURSOR_HEADER) ?? void 0;
|
|
1349
1486
|
const initialUpToDate = firstResponse.headers.has(STREAM_UP_TO_DATE_HEADER);
|
|
1350
1487
|
const isJsonMode = options.json === true || (contentType?.includes(`application/json`) ?? false);
|
|
1351
|
-
const fetchNext = async (offset, cursor, signal) => {
|
|
1488
|
+
const fetchNext = async (offset, cursor, signal, resumingFromPause) => {
|
|
1352
1489
|
const nextUrl = new URL(url);
|
|
1353
1490
|
nextUrl.searchParams.set(OFFSET_QUERY_PARAM, offset);
|
|
1354
|
-
if (
|
|
1355
|
-
|
|
1491
|
+
if (!resumingFromPause) {
|
|
1492
|
+
if (live === `sse`) nextUrl.searchParams.set(LIVE_QUERY_PARAM, `sse`);
|
|
1493
|
+
else if (live === true || live === `long-poll`) nextUrl.searchParams.set(LIVE_QUERY_PARAM, `long-poll`);
|
|
1494
|
+
}
|
|
1356
1495
|
if (cursor) nextUrl.searchParams.set(`cursor`, cursor);
|
|
1357
1496
|
const nextParams = await resolveParams(options.params);
|
|
1358
1497
|
for (const [key, value] of Object.entries(nextParams)) nextUrl.searchParams.set(key, value);
|
|
@@ -1399,927 +1538,947 @@ async function streamInternal(options) {
|
|
|
1399
1538
|
}
|
|
1400
1539
|
|
|
1401
1540
|
//#endregion
|
|
1402
|
-
//#region src/
|
|
1541
|
+
//#region src/idempotent-producer.ts
|
|
1542
|
+
/**
|
|
1543
|
+
* Error thrown when a producer's epoch is stale (zombie fencing).
|
|
1544
|
+
*/
|
|
1545
|
+
var StaleEpochError = class extends Error {
|
|
1546
|
+
/**
|
|
1547
|
+
* The current epoch on the server.
|
|
1548
|
+
*/
|
|
1549
|
+
currentEpoch;
|
|
1550
|
+
constructor(currentEpoch) {
|
|
1551
|
+
super(`Producer epoch is stale. Current server epoch: ${currentEpoch}. Call restart() or create a new producer with a higher epoch.`);
|
|
1552
|
+
this.name = `StaleEpochError`;
|
|
1553
|
+
this.currentEpoch = currentEpoch;
|
|
1554
|
+
}
|
|
1555
|
+
};
|
|
1556
|
+
/**
|
|
1557
|
+
* Error thrown when an unrecoverable sequence gap is detected.
|
|
1558
|
+
*
|
|
1559
|
+
* With maxInFlight > 1, HTTP requests can arrive out of order at the server,
|
|
1560
|
+
* causing temporary 409 responses. The client automatically handles these
|
|
1561
|
+
* by waiting for earlier sequences to complete, then retrying.
|
|
1562
|
+
*
|
|
1563
|
+
* This error is only thrown when the gap cannot be resolved (e.g., the
|
|
1564
|
+
* expected sequence is >= our sequence, indicating a true protocol violation).
|
|
1565
|
+
*/
|
|
1566
|
+
var SequenceGapError = class extends Error {
|
|
1567
|
+
expectedSeq;
|
|
1568
|
+
receivedSeq;
|
|
1569
|
+
constructor(expectedSeq, receivedSeq) {
|
|
1570
|
+
super(`Producer sequence gap: expected ${expectedSeq}, received ${receivedSeq}`);
|
|
1571
|
+
this.name = `SequenceGapError`;
|
|
1572
|
+
this.expectedSeq = expectedSeq;
|
|
1573
|
+
this.receivedSeq = receivedSeq;
|
|
1574
|
+
}
|
|
1575
|
+
};
|
|
1403
1576
|
/**
|
|
1404
1577
|
* Normalize content-type by extracting the media type (before any semicolon).
|
|
1405
|
-
* Handles cases like "application/json; charset=utf-8".
|
|
1406
1578
|
*/
|
|
1407
1579
|
function normalizeContentType$1(contentType) {
|
|
1408
1580
|
if (!contentType) return ``;
|
|
1409
1581
|
return contentType.split(`;`)[0].trim().toLowerCase();
|
|
1410
1582
|
}
|
|
1411
1583
|
/**
|
|
1412
|
-
*
|
|
1413
|
-
*/
|
|
1414
|
-
function isPromiseLike(value) {
|
|
1415
|
-
return value !== null && typeof value === `object` && `then` in value && typeof value.then === `function`;
|
|
1416
|
-
}
|
|
1417
|
-
/**
|
|
1418
|
-
* A handle to a remote durable stream for read/write operations.
|
|
1584
|
+
* An idempotent producer for exactly-once writes to a durable stream.
|
|
1419
1585
|
*
|
|
1420
|
-
*
|
|
1421
|
-
*
|
|
1422
|
-
*
|
|
1586
|
+
* Features:
|
|
1587
|
+
* - Fire-and-forget: append() returns immediately, batches in background
|
|
1588
|
+
* - Exactly-once: server deduplicates using (producerId, epoch, seq)
|
|
1589
|
+
* - Batching: multiple appends batched into single HTTP request
|
|
1590
|
+
* - Pipelining: up to maxInFlight concurrent batches
|
|
1591
|
+
* - Zombie fencing: stale producers rejected via epoch validation
|
|
1423
1592
|
*
|
|
1424
1593
|
* @example
|
|
1425
1594
|
* ```typescript
|
|
1426
|
-
*
|
|
1427
|
-
* const
|
|
1428
|
-
*
|
|
1429
|
-
*
|
|
1430
|
-
* contentType: "application/json"
|
|
1595
|
+
* const stream = new DurableStream({ url: "https://..." });
|
|
1596
|
+
* const producer = new IdempotentProducer(stream, "order-service-1", {
|
|
1597
|
+
* epoch: 0,
|
|
1598
|
+
* autoClaim: true,
|
|
1431
1599
|
* });
|
|
1432
1600
|
*
|
|
1433
|
-
* //
|
|
1434
|
-
*
|
|
1601
|
+
* // Fire-and-forget writes (synchronous, returns immediately)
|
|
1602
|
+
* producer.append("message 1");
|
|
1603
|
+
* producer.append("message 2");
|
|
1435
1604
|
*
|
|
1436
|
-
* //
|
|
1437
|
-
*
|
|
1438
|
-
*
|
|
1439
|
-
* for (const item of batch.items) {
|
|
1440
|
-
* console.log(item.message);
|
|
1441
|
-
* }
|
|
1442
|
-
* });
|
|
1605
|
+
* // Ensure all messages are delivered before shutdown
|
|
1606
|
+
* await producer.flush();
|
|
1607
|
+
* await producer.close();
|
|
1443
1608
|
* ```
|
|
1444
1609
|
*/
|
|
1445
|
-
var
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
contentType;
|
|
1454
|
-
#options;
|
|
1610
|
+
var IdempotentProducer = class {
|
|
1611
|
+
#stream;
|
|
1612
|
+
#producerId;
|
|
1613
|
+
#epoch;
|
|
1614
|
+
#nextSeq = 0;
|
|
1615
|
+
#autoClaim;
|
|
1616
|
+
#maxBatchBytes;
|
|
1617
|
+
#lingerMs;
|
|
1455
1618
|
#fetchClient;
|
|
1619
|
+
#signal;
|
|
1456
1620
|
#onError;
|
|
1457
|
-
#
|
|
1621
|
+
#pendingBatch = [];
|
|
1622
|
+
#batchBytes = 0;
|
|
1623
|
+
#lingerTimeout = null;
|
|
1458
1624
|
#queue;
|
|
1459
|
-
#
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
*/
|
|
1464
|
-
constructor(opts) {
|
|
1465
|
-
validateOptions(opts);
|
|
1466
|
-
const urlStr = opts.url instanceof URL ? opts.url.toString() : opts.url;
|
|
1467
|
-
this.url = urlStr;
|
|
1468
|
-
this.#options = {
|
|
1469
|
-
...opts,
|
|
1470
|
-
url: urlStr
|
|
1471
|
-
};
|
|
1472
|
-
this.#onError = opts.onError;
|
|
1473
|
-
if (opts.contentType) this.contentType = opts.contentType;
|
|
1474
|
-
this.#batchingEnabled = opts.batching !== false;
|
|
1475
|
-
if (this.#batchingEnabled) this.#queue = fastq.promise(this.#batchWorker.bind(this), 1);
|
|
1476
|
-
const baseFetchClient = opts.fetch ?? ((...args) => fetch(...args));
|
|
1477
|
-
const backOffOpts = { ...opts.backoffOptions ?? BackoffDefaults };
|
|
1478
|
-
const fetchWithBackoffClient = createFetchWithBackoff(baseFetchClient, backOffOpts);
|
|
1479
|
-
this.#fetchClient = createFetchWithConsumedBody(fetchWithBackoffClient);
|
|
1480
|
-
}
|
|
1625
|
+
#maxInFlight;
|
|
1626
|
+
#closed = false;
|
|
1627
|
+
#epochClaimed;
|
|
1628
|
+
#seqState = new Map();
|
|
1481
1629
|
/**
|
|
1482
|
-
* Create
|
|
1483
|
-
*
|
|
1630
|
+
* Create an idempotent producer for a stream.
|
|
1631
|
+
*
|
|
1632
|
+
* @param stream - The DurableStream to write to
|
|
1633
|
+
* @param producerId - Stable identifier for this producer (e.g., "order-service-1")
|
|
1634
|
+
* @param opts - Producer options
|
|
1484
1635
|
*/
|
|
1485
|
-
|
|
1486
|
-
const
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1491
|
-
|
|
1492
|
-
|
|
1493
|
-
|
|
1636
|
+
constructor(stream$1, producerId, opts) {
|
|
1637
|
+
const epoch = opts?.epoch ?? 0;
|
|
1638
|
+
const maxBatchBytes = opts?.maxBatchBytes ?? 1024 * 1024;
|
|
1639
|
+
const maxInFlight = opts?.maxInFlight ?? 5;
|
|
1640
|
+
const lingerMs = opts?.lingerMs ?? 5;
|
|
1641
|
+
if (epoch < 0) throw new Error(`epoch must be >= 0`);
|
|
1642
|
+
if (maxBatchBytes <= 0) throw new Error(`maxBatchBytes must be > 0`);
|
|
1643
|
+
if (maxInFlight <= 0) throw new Error(`maxInFlight must be > 0`);
|
|
1644
|
+
if (lingerMs < 0) throw new Error(`lingerMs must be >= 0`);
|
|
1645
|
+
this.#stream = stream$1;
|
|
1646
|
+
this.#producerId = producerId;
|
|
1647
|
+
this.#epoch = epoch;
|
|
1648
|
+
this.#autoClaim = opts?.autoClaim ?? false;
|
|
1649
|
+
this.#maxBatchBytes = maxBatchBytes;
|
|
1650
|
+
this.#lingerMs = lingerMs;
|
|
1651
|
+
this.#signal = opts?.signal;
|
|
1652
|
+
this.#onError = opts?.onError;
|
|
1653
|
+
this.#fetchClient = opts?.fetch ?? ((...args) => fetch(...args));
|
|
1654
|
+
this.#maxInFlight = maxInFlight;
|
|
1655
|
+
this.#epochClaimed = !this.#autoClaim;
|
|
1656
|
+
this.#queue = fastq.promise(this.#batchWorker.bind(this), this.#maxInFlight);
|
|
1657
|
+
if (this.#signal) this.#signal.addEventListener(`abort`, () => {
|
|
1658
|
+
this.#rejectPendingBatch(new DurableStreamError(`Producer aborted`, `ALREADY_CLOSED`, void 0, void 0));
|
|
1659
|
+
}, { once: true });
|
|
1494
1660
|
}
|
|
1495
1661
|
/**
|
|
1496
|
-
*
|
|
1497
|
-
* Returns a handle with contentType populated (if sent by server).
|
|
1662
|
+
* Append data to the stream.
|
|
1498
1663
|
*
|
|
1499
|
-
*
|
|
1500
|
-
*
|
|
1501
|
-
*
|
|
1664
|
+
* This is fire-and-forget: returns immediately after adding to the batch.
|
|
1665
|
+
* The message is batched and sent when:
|
|
1666
|
+
* - maxBatchBytes is reached
|
|
1667
|
+
* - lingerMs elapses
|
|
1668
|
+
* - flush() is called
|
|
1669
|
+
*
|
|
1670
|
+
* Errors are reported via onError callback if configured. Use flush() to
|
|
1671
|
+
* wait for all pending messages to be sent.
|
|
1672
|
+
*
|
|
1673
|
+
* For JSON streams, pass pre-serialized JSON strings.
|
|
1674
|
+
* For byte streams, pass string or Uint8Array.
|
|
1675
|
+
*
|
|
1676
|
+
* @param body - Data to append (string or Uint8Array)
|
|
1502
1677
|
*
|
|
1503
1678
|
* @example
|
|
1504
1679
|
* ```typescript
|
|
1505
|
-
* //
|
|
1506
|
-
*
|
|
1507
|
-
*
|
|
1680
|
+
* // JSON stream
|
|
1681
|
+
* producer.append(JSON.stringify({ message: "hello" }));
|
|
1682
|
+
*
|
|
1683
|
+
* // Byte stream
|
|
1684
|
+
* producer.append("raw text data");
|
|
1685
|
+
* producer.append(new Uint8Array([1, 2, 3]));
|
|
1508
1686
|
* ```
|
|
1509
1687
|
*/
|
|
1510
|
-
|
|
1511
|
-
|
|
1512
|
-
|
|
1513
|
-
|
|
1688
|
+
append(body) {
|
|
1689
|
+
if (this.#closed) throw new DurableStreamError(`Producer is closed`, `ALREADY_CLOSED`, void 0, void 0);
|
|
1690
|
+
let bytes;
|
|
1691
|
+
if (typeof body === `string`) bytes = new TextEncoder().encode(body);
|
|
1692
|
+
else if (body instanceof Uint8Array) bytes = body;
|
|
1693
|
+
else throw new DurableStreamError(`append() requires string or Uint8Array. For objects, use JSON.stringify().`, `BAD_REQUEST`, 400, void 0);
|
|
1694
|
+
this.#pendingBatch.push({ body: bytes });
|
|
1695
|
+
this.#batchBytes += bytes.length;
|
|
1696
|
+
if (this.#batchBytes >= this.#maxBatchBytes) this.#enqueuePendingBatch();
|
|
1697
|
+
else if (!this.#lingerTimeout) this.#lingerTimeout = setTimeout(() => {
|
|
1698
|
+
this.#lingerTimeout = null;
|
|
1699
|
+
if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
|
|
1700
|
+
}, this.#lingerMs);
|
|
1514
1701
|
}
|
|
1515
1702
|
/**
|
|
1516
|
-
*
|
|
1703
|
+
* Send any pending batch immediately and wait for all in-flight batches.
|
|
1704
|
+
*
|
|
1705
|
+
* Call this before shutdown to ensure all messages are delivered.
|
|
1517
1706
|
*/
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1707
|
+
async flush() {
|
|
1708
|
+
if (this.#lingerTimeout) {
|
|
1709
|
+
clearTimeout(this.#lingerTimeout);
|
|
1710
|
+
this.#lingerTimeout = null;
|
|
1711
|
+
}
|
|
1712
|
+
if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
|
|
1713
|
+
await this.#queue.drained();
|
|
1521
1714
|
}
|
|
1522
1715
|
/**
|
|
1523
|
-
*
|
|
1716
|
+
* Flush pending messages and close the producer.
|
|
1717
|
+
*
|
|
1718
|
+
* After calling close(), further append() calls will throw.
|
|
1524
1719
|
*/
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1720
|
+
async close() {
|
|
1721
|
+
if (this.#closed) return;
|
|
1722
|
+
this.#closed = true;
|
|
1723
|
+
try {
|
|
1724
|
+
await this.flush();
|
|
1725
|
+
} catch {}
|
|
1528
1726
|
}
|
|
1529
1727
|
/**
|
|
1530
|
-
*
|
|
1728
|
+
* Increment epoch and reset sequence.
|
|
1729
|
+
*
|
|
1730
|
+
* Call this when restarting the producer to establish a new session.
|
|
1731
|
+
* Flushes any pending messages first.
|
|
1531
1732
|
*/
|
|
1532
|
-
async
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
|
|
1536
|
-
headers: requestHeaders,
|
|
1537
|
-
signal: opts?.signal ?? this.#options.signal
|
|
1538
|
-
});
|
|
1539
|
-
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
1540
|
-
const contentType = response.headers.get(`content-type`) ?? void 0;
|
|
1541
|
-
const offset = response.headers.get(STREAM_OFFSET_HEADER) ?? void 0;
|
|
1542
|
-
const etag = response.headers.get(`etag`) ?? void 0;
|
|
1543
|
-
const cacheControl = response.headers.get(`cache-control`) ?? void 0;
|
|
1544
|
-
if (contentType) this.contentType = contentType;
|
|
1545
|
-
return {
|
|
1546
|
-
exists: true,
|
|
1547
|
-
contentType,
|
|
1548
|
-
offset,
|
|
1549
|
-
etag,
|
|
1550
|
-
cacheControl
|
|
1551
|
-
};
|
|
1733
|
+
async restart() {
|
|
1734
|
+
await this.flush();
|
|
1735
|
+
this.#epoch++;
|
|
1736
|
+
this.#nextSeq = 0;
|
|
1552
1737
|
}
|
|
1553
1738
|
/**
|
|
1554
|
-
*
|
|
1739
|
+
* Current epoch for this producer.
|
|
1555
1740
|
*/
|
|
1556
|
-
|
|
1557
|
-
|
|
1558
|
-
const contentType = opts?.contentType ?? this.#options.contentType;
|
|
1559
|
-
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
1560
|
-
if (opts?.ttlSeconds !== void 0) requestHeaders[STREAM_TTL_HEADER] = String(opts.ttlSeconds);
|
|
1561
|
-
if (opts?.expiresAt) requestHeaders[STREAM_EXPIRES_AT_HEADER] = opts.expiresAt;
|
|
1562
|
-
const body = encodeBody(opts?.body);
|
|
1563
|
-
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
1564
|
-
method: `PUT`,
|
|
1565
|
-
headers: requestHeaders,
|
|
1566
|
-
body,
|
|
1567
|
-
signal: this.#options.signal
|
|
1568
|
-
});
|
|
1569
|
-
if (!response.ok) await handleErrorResponse(response, this.url, { operation: `create` });
|
|
1570
|
-
const responseContentType = response.headers.get(`content-type`);
|
|
1571
|
-
if (responseContentType) this.contentType = responseContentType;
|
|
1572
|
-
else if (contentType) this.contentType = contentType;
|
|
1573
|
-
return this;
|
|
1741
|
+
get epoch() {
|
|
1742
|
+
return this.#epoch;
|
|
1574
1743
|
}
|
|
1575
1744
|
/**
|
|
1576
|
-
*
|
|
1745
|
+
* Next sequence number to be assigned.
|
|
1577
1746
|
*/
|
|
1578
|
-
|
|
1579
|
-
|
|
1580
|
-
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
1581
|
-
method: `DELETE`,
|
|
1582
|
-
headers: requestHeaders,
|
|
1583
|
-
signal: opts?.signal ?? this.#options.signal
|
|
1584
|
-
});
|
|
1585
|
-
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
1747
|
+
get nextSeq() {
|
|
1748
|
+
return this.#nextSeq;
|
|
1586
1749
|
}
|
|
1587
1750
|
/**
|
|
1588
|
-
*
|
|
1589
|
-
*
|
|
1590
|
-
* When batching is enabled (default), multiple append() calls made while
|
|
1591
|
-
* a POST is in-flight will be batched together into a single request.
|
|
1592
|
-
* This significantly improves throughput for high-frequency writes.
|
|
1593
|
-
*
|
|
1594
|
-
* - `body` may be Uint8Array, string, or any JSON-serializable value (for JSON streams).
|
|
1595
|
-
* - `body` may also be a Promise that resolves to any of the above types.
|
|
1596
|
-
* - Strings are encoded as UTF-8.
|
|
1597
|
-
* - `seq` (if provided) is sent as stream-seq (writer coordination).
|
|
1598
|
-
*
|
|
1599
|
-
* @example
|
|
1600
|
-
* ```typescript
|
|
1601
|
-
* // Direct value
|
|
1602
|
-
* await stream.append({ message: "hello" });
|
|
1603
|
-
*
|
|
1604
|
-
* // Promise value - awaited before buffering
|
|
1605
|
-
* await stream.append(fetchData());
|
|
1606
|
-
* await stream.append(Promise.all([a, b, c]));
|
|
1607
|
-
* ```
|
|
1751
|
+
* Number of messages in the current pending batch.
|
|
1608
1752
|
*/
|
|
1609
|
-
|
|
1610
|
-
|
|
1611
|
-
if (this.#batchingEnabled && this.#queue) return this.#appendWithBatching(resolvedBody, opts);
|
|
1612
|
-
return this.#appendDirect(resolvedBody, opts);
|
|
1753
|
+
get pendingCount() {
|
|
1754
|
+
return this.#pendingBatch.length;
|
|
1613
1755
|
}
|
|
1614
1756
|
/**
|
|
1615
|
-
*
|
|
1757
|
+
* Number of batches currently in flight.
|
|
1616
1758
|
*/
|
|
1617
|
-
|
|
1618
|
-
|
|
1619
|
-
const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
1620
|
-
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
1621
|
-
if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
|
|
1622
|
-
const isJson = normalizeContentType$1(contentType) === `application/json`;
|
|
1623
|
-
const bodyToEncode = isJson ? [body] : body;
|
|
1624
|
-
const encodedBody = encodeBody(bodyToEncode);
|
|
1625
|
-
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
1626
|
-
method: `POST`,
|
|
1627
|
-
headers: requestHeaders,
|
|
1628
|
-
body: encodedBody,
|
|
1629
|
-
signal: opts?.signal ?? this.#options.signal
|
|
1630
|
-
});
|
|
1631
|
-
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
1759
|
+
get inFlightCount() {
|
|
1760
|
+
return this.#queue.length();
|
|
1632
1761
|
}
|
|
1633
1762
|
/**
|
|
1634
|
-
*
|
|
1763
|
+
* Enqueue the current pending batch for processing.
|
|
1635
1764
|
*/
|
|
1636
|
-
|
|
1637
|
-
|
|
1638
|
-
|
|
1639
|
-
|
|
1640
|
-
|
|
1641
|
-
|
|
1642
|
-
|
|
1643
|
-
|
|
1644
|
-
|
|
1645
|
-
|
|
1646
|
-
|
|
1647
|
-
|
|
1648
|
-
this.#queue.push(batch).catch((err) => {
|
|
1649
|
-
for (const msg of batch) msg.reject(err);
|
|
1650
|
-
});
|
|
1651
|
-
}
|
|
1765
|
+
#enqueuePendingBatch() {
|
|
1766
|
+
if (this.#pendingBatch.length === 0) return;
|
|
1767
|
+
const batch = this.#pendingBatch;
|
|
1768
|
+
const seq = this.#nextSeq;
|
|
1769
|
+
this.#pendingBatch = [];
|
|
1770
|
+
this.#batchBytes = 0;
|
|
1771
|
+
this.#nextSeq++;
|
|
1772
|
+
if (this.#autoClaim && !this.#epochClaimed && this.#queue.length() > 0) this.#queue.drained().then(() => {
|
|
1773
|
+
this.#queue.push({
|
|
1774
|
+
batch,
|
|
1775
|
+
seq
|
|
1776
|
+
}).catch(() => {});
|
|
1652
1777
|
});
|
|
1778
|
+
else this.#queue.push({
|
|
1779
|
+
batch,
|
|
1780
|
+
seq
|
|
1781
|
+
}).catch(() => {});
|
|
1653
1782
|
}
|
|
1654
1783
|
/**
|
|
1655
|
-
* Batch worker - processes batches
|
|
1784
|
+
* Batch worker - processes batches via fastq.
|
|
1656
1785
|
*/
|
|
1657
|
-
async #batchWorker(
|
|
1786
|
+
async #batchWorker(task) {
|
|
1787
|
+
const { batch, seq } = task;
|
|
1788
|
+
const epoch = this.#epoch;
|
|
1658
1789
|
try {
|
|
1659
|
-
await this.#
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
const nextBatch = this.#buffer.splice(0);
|
|
1663
|
-
this.#queue.push(nextBatch).catch((err) => {
|
|
1664
|
-
for (const msg of nextBatch) msg.reject(err);
|
|
1665
|
-
});
|
|
1666
|
-
}
|
|
1790
|
+
await this.#doSendBatch(batch, seq, epoch);
|
|
1791
|
+
if (!this.#epochClaimed) this.#epochClaimed = true;
|
|
1792
|
+
this.#signalSeqComplete(epoch, seq, void 0);
|
|
1667
1793
|
} catch (error) {
|
|
1668
|
-
|
|
1669
|
-
|
|
1670
|
-
this.#buffer = [];
|
|
1794
|
+
this.#signalSeqComplete(epoch, seq, error);
|
|
1795
|
+
if (this.#onError) this.#onError(error);
|
|
1671
1796
|
throw error;
|
|
1672
1797
|
}
|
|
1673
1798
|
}
|
|
1674
1799
|
/**
|
|
1675
|
-
*
|
|
1800
|
+
* Signal that a sequence has completed (success or failure).
|
|
1676
1801
|
*/
|
|
1677
|
-
|
|
1678
|
-
|
|
1679
|
-
|
|
1680
|
-
|
|
1681
|
-
|
|
1682
|
-
let highestSeq;
|
|
1683
|
-
for (let i = batch.length - 1; i >= 0; i--) if (batch[i].seq !== void 0) {
|
|
1684
|
-
highestSeq = batch[i].seq;
|
|
1685
|
-
break;
|
|
1802
|
+
#signalSeqComplete(epoch, seq, error) {
|
|
1803
|
+
let epochMap = this.#seqState.get(epoch);
|
|
1804
|
+
if (!epochMap) {
|
|
1805
|
+
epochMap = new Map();
|
|
1806
|
+
this.#seqState.set(epoch, epochMap);
|
|
1686
1807
|
}
|
|
1687
|
-
|
|
1808
|
+
const state = epochMap.get(seq);
|
|
1809
|
+
if (state) {
|
|
1810
|
+
state.resolved = true;
|
|
1811
|
+
state.error = error;
|
|
1812
|
+
for (const waiter of state.waiters) waiter(error);
|
|
1813
|
+
state.waiters = [];
|
|
1814
|
+
} else epochMap.set(seq, {
|
|
1815
|
+
resolved: true,
|
|
1816
|
+
error,
|
|
1817
|
+
waiters: []
|
|
1818
|
+
});
|
|
1819
|
+
const cleanupThreshold = seq - this.#maxInFlight * 3;
|
|
1820
|
+
if (cleanupThreshold > 0) {
|
|
1821
|
+
for (const oldSeq of epochMap.keys()) if (oldSeq < cleanupThreshold) epochMap.delete(oldSeq);
|
|
1822
|
+
}
|
|
1823
|
+
}
|
|
1824
|
+
/**
|
|
1825
|
+
* Wait for a specific sequence to complete.
|
|
1826
|
+
* Returns immediately if already completed.
|
|
1827
|
+
* Throws if the sequence failed.
|
|
1828
|
+
*/
|
|
1829
|
+
#waitForSeq(epoch, seq) {
|
|
1830
|
+
let epochMap = this.#seqState.get(epoch);
|
|
1831
|
+
if (!epochMap) {
|
|
1832
|
+
epochMap = new Map();
|
|
1833
|
+
this.#seqState.set(epoch, epochMap);
|
|
1834
|
+
}
|
|
1835
|
+
const state = epochMap.get(seq);
|
|
1836
|
+
if (state?.resolved) {
|
|
1837
|
+
if (state.error) return Promise.reject(state.error);
|
|
1838
|
+
return Promise.resolve();
|
|
1839
|
+
}
|
|
1840
|
+
return new Promise((resolve, reject) => {
|
|
1841
|
+
const waiter = (err) => {
|
|
1842
|
+
if (err) reject(err);
|
|
1843
|
+
else resolve();
|
|
1844
|
+
};
|
|
1845
|
+
if (state) state.waiters.push(waiter);
|
|
1846
|
+
else epochMap.set(seq, {
|
|
1847
|
+
resolved: false,
|
|
1848
|
+
waiters: [waiter]
|
|
1849
|
+
});
|
|
1850
|
+
});
|
|
1851
|
+
}
|
|
1852
|
+
/**
|
|
1853
|
+
* Actually send the batch to the server.
|
|
1854
|
+
* Handles auto-claim retry on 403 (stale epoch) if autoClaim is enabled.
|
|
1855
|
+
* Does NOT implement general retry/backoff for network errors or 5xx responses.
|
|
1856
|
+
*/
|
|
1857
|
+
async #doSendBatch(batch, seq, epoch) {
|
|
1858
|
+
const contentType = this.#stream.contentType ?? `application/octet-stream`;
|
|
1688
1859
|
const isJson = normalizeContentType$1(contentType) === `application/json`;
|
|
1689
1860
|
let batchedBody;
|
|
1690
1861
|
if (isJson) {
|
|
1691
|
-
const
|
|
1692
|
-
batchedBody =
|
|
1862
|
+
const jsonStrings = batch.map((e) => new TextDecoder().decode(e.body));
|
|
1863
|
+
batchedBody = `[${jsonStrings.join(`,`)}]`;
|
|
1693
1864
|
} else {
|
|
1694
|
-
const totalSize = batch.reduce((sum,
|
|
1695
|
-
const size = typeof m.data === `string` ? new TextEncoder().encode(m.data).length : m.data.length;
|
|
1696
|
-
return sum + size;
|
|
1697
|
-
}, 0);
|
|
1865
|
+
const totalSize = batch.reduce((sum, e) => sum + e.body.length, 0);
|
|
1698
1866
|
const concatenated = new Uint8Array(totalSize);
|
|
1699
1867
|
let offset = 0;
|
|
1700
|
-
for (const
|
|
1701
|
-
|
|
1702
|
-
|
|
1703
|
-
offset += bytes.length;
|
|
1868
|
+
for (const entry of batch) {
|
|
1869
|
+
concatenated.set(entry.body, offset);
|
|
1870
|
+
offset += entry.body.length;
|
|
1704
1871
|
}
|
|
1705
1872
|
batchedBody = concatenated;
|
|
1706
1873
|
}
|
|
1707
|
-
const
|
|
1708
|
-
|
|
1709
|
-
|
|
1710
|
-
|
|
1711
|
-
|
|
1874
|
+
const url = this.#stream.url;
|
|
1875
|
+
const headers = {
|
|
1876
|
+
"content-type": contentType,
|
|
1877
|
+
[PRODUCER_ID_HEADER]: this.#producerId,
|
|
1878
|
+
[PRODUCER_EPOCH_HEADER]: epoch.toString(),
|
|
1879
|
+
[PRODUCER_SEQ_HEADER]: seq.toString()
|
|
1880
|
+
};
|
|
1881
|
+
const response = await this.#fetchClient(url, {
|
|
1712
1882
|
method: `POST`,
|
|
1713
|
-
headers
|
|
1883
|
+
headers,
|
|
1714
1884
|
body: batchedBody,
|
|
1715
|
-
signal:
|
|
1885
|
+
signal: this.#signal
|
|
1716
1886
|
});
|
|
1717
|
-
if (
|
|
1718
|
-
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
1725
|
-
|
|
1726
|
-
|
|
1727
|
-
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
* });
|
|
1737
|
-
* await stream.appendStream(readable);
|
|
1738
|
-
*
|
|
1739
|
-
* // Pipe from an async generator
|
|
1740
|
-
* async function* generate() {
|
|
1741
|
-
* yield "line 1\n";
|
|
1742
|
-
* yield "line 2\n";
|
|
1743
|
-
* }
|
|
1744
|
-
* await stream.appendStream(generate());
|
|
1745
|
-
*
|
|
1746
|
-
* // Pipe from fetch response body
|
|
1747
|
-
* const response = await fetch("https://example.com/data");
|
|
1748
|
-
* await stream.appendStream(response.body!);
|
|
1749
|
-
* ```
|
|
1750
|
-
*/
|
|
1751
|
-
async appendStream(source, opts) {
|
|
1752
|
-
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
1753
|
-
const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
1754
|
-
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
1755
|
-
if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
|
|
1756
|
-
const body = toReadableStream(source);
|
|
1757
|
-
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
1758
|
-
method: `POST`,
|
|
1759
|
-
headers: requestHeaders,
|
|
1760
|
-
body,
|
|
1761
|
-
duplex: `half`,
|
|
1762
|
-
signal: opts?.signal ?? this.#options.signal
|
|
1763
|
-
});
|
|
1764
|
-
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
1765
|
-
}
|
|
1766
|
-
/**
|
|
1767
|
-
* Create a writable stream that pipes data to this durable stream.
|
|
1768
|
-
*
|
|
1769
|
-
* Returns a WritableStream that can be used with `pipeTo()` or
|
|
1770
|
-
* `pipeThrough()` from any ReadableStream source.
|
|
1771
|
-
*
|
|
1772
|
-
* @example
|
|
1773
|
-
* ```typescript
|
|
1774
|
-
* // Pipe from fetch response
|
|
1775
|
-
* const response = await fetch("https://example.com/data");
|
|
1776
|
-
* await response.body!.pipeTo(stream.writable());
|
|
1777
|
-
*
|
|
1778
|
-
* // Pipe through a transform
|
|
1779
|
-
* const readable = someStream.pipeThrough(new TextEncoderStream());
|
|
1780
|
-
* await readable.pipeTo(stream.writable());
|
|
1781
|
-
* ```
|
|
1782
|
-
*/
|
|
1783
|
-
writable(opts) {
|
|
1784
|
-
const chunks = [];
|
|
1785
|
-
const stream$1 = this;
|
|
1786
|
-
return new WritableStream({
|
|
1787
|
-
write(chunk) {
|
|
1788
|
-
chunks.push(chunk);
|
|
1789
|
-
},
|
|
1790
|
-
async close() {
|
|
1791
|
-
if (chunks.length > 0) {
|
|
1792
|
-
const readable = new ReadableStream({ start(controller) {
|
|
1793
|
-
for (const chunk of chunks) controller.enqueue(chunk);
|
|
1794
|
-
controller.close();
|
|
1795
|
-
} });
|
|
1796
|
-
await stream$1.appendStream(readable, opts);
|
|
1797
|
-
}
|
|
1798
|
-
},
|
|
1799
|
-
abort(reason) {
|
|
1800
|
-
console.error(`WritableStream aborted:`, reason);
|
|
1887
|
+
if (response.status === 204) return {
|
|
1888
|
+
offset: ``,
|
|
1889
|
+
duplicate: true
|
|
1890
|
+
};
|
|
1891
|
+
if (response.status === 200) {
|
|
1892
|
+
const resultOffset = response.headers.get(STREAM_OFFSET_HEADER) ?? ``;
|
|
1893
|
+
return {
|
|
1894
|
+
offset: resultOffset,
|
|
1895
|
+
duplicate: false
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
if (response.status === 403) {
|
|
1899
|
+
const currentEpochStr = response.headers.get(PRODUCER_EPOCH_HEADER);
|
|
1900
|
+
const currentEpoch = currentEpochStr ? parseInt(currentEpochStr, 10) : epoch;
|
|
1901
|
+
if (this.#autoClaim) {
|
|
1902
|
+
const newEpoch = currentEpoch + 1;
|
|
1903
|
+
this.#epoch = newEpoch;
|
|
1904
|
+
this.#nextSeq = 1;
|
|
1905
|
+
return this.#doSendBatch(batch, 0, newEpoch);
|
|
1801
1906
|
}
|
|
1802
|
-
|
|
1803
|
-
}
|
|
1804
|
-
/**
|
|
1805
|
-
* Start a fetch-like streaming session against this handle's URL/headers/params.
|
|
1806
|
-
* The first request is made inside this method; it resolves when we have
|
|
1807
|
-
* a valid first response, or rejects on errors.
|
|
1808
|
-
*
|
|
1809
|
-
* Call-specific headers and params are merged with handle-level ones,
|
|
1810
|
-
* with call-specific values taking precedence.
|
|
1811
|
-
*
|
|
1812
|
-
* @example
|
|
1813
|
-
* ```typescript
|
|
1814
|
-
* const handle = await DurableStream.connect({
|
|
1815
|
-
* url,
|
|
1816
|
-
* headers: { Authorization: `Bearer ${token}` }
|
|
1817
|
-
* });
|
|
1818
|
-
* const res = await handle.stream<{ message: string }>();
|
|
1819
|
-
*
|
|
1820
|
-
* // Accumulate all JSON items
|
|
1821
|
-
* const items = await res.json();
|
|
1822
|
-
*
|
|
1823
|
-
* // Or stream live with ReadableStream
|
|
1824
|
-
* const reader = res.jsonStream().getReader();
|
|
1825
|
-
* let result = await reader.read();
|
|
1826
|
-
* while (!result.done) {
|
|
1827
|
-
* console.log(result.value);
|
|
1828
|
-
* result = await reader.read();
|
|
1829
|
-
* }
|
|
1830
|
-
*
|
|
1831
|
-
* // Or use subscriber for backpressure-aware consumption
|
|
1832
|
-
* res.subscribeJson(async (batch) => {
|
|
1833
|
-
* for (const item of batch.items) {
|
|
1834
|
-
* console.log(item);
|
|
1835
|
-
* }
|
|
1836
|
-
* });
|
|
1837
|
-
* ```
|
|
1838
|
-
*/
|
|
1839
|
-
async stream(options) {
|
|
1840
|
-
if (options?.live === `sse` && this.contentType) {
|
|
1841
|
-
const isSSECompatible = SSE_COMPATIBLE_CONTENT_TYPES.some((prefix) => this.contentType.startsWith(prefix));
|
|
1842
|
-
if (!isSSECompatible) throw new DurableStreamError(`SSE is not supported for content-type: ${this.contentType}`, `SSE_NOT_SUPPORTED`, 400);
|
|
1907
|
+
throw new StaleEpochError(currentEpoch);
|
|
1843
1908
|
}
|
|
1844
|
-
|
|
1845
|
-
|
|
1846
|
-
|
|
1847
|
-
|
|
1848
|
-
|
|
1849
|
-
|
|
1850
|
-
|
|
1851
|
-
|
|
1852
|
-
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
1863
|
-
warnOnHttp: options?.warnOnHttp ?? this.#options.warnOnHttp
|
|
1864
|
-
});
|
|
1909
|
+
if (response.status === 409) {
|
|
1910
|
+
const expectedSeqStr = response.headers.get(PRODUCER_EXPECTED_SEQ_HEADER);
|
|
1911
|
+
const expectedSeq = expectedSeqStr ? parseInt(expectedSeqStr, 10) : 0;
|
|
1912
|
+
if (expectedSeq < seq) {
|
|
1913
|
+
const waitPromises = [];
|
|
1914
|
+
for (let s = expectedSeq; s < seq; s++) waitPromises.push(this.#waitForSeq(epoch, s));
|
|
1915
|
+
await Promise.all(waitPromises);
|
|
1916
|
+
return this.#doSendBatch(batch, seq, epoch);
|
|
1917
|
+
}
|
|
1918
|
+
const receivedSeqStr = response.headers.get(PRODUCER_RECEIVED_SEQ_HEADER);
|
|
1919
|
+
const receivedSeq = receivedSeqStr ? parseInt(receivedSeqStr, 10) : seq;
|
|
1920
|
+
throw new SequenceGapError(expectedSeq, receivedSeq);
|
|
1921
|
+
}
|
|
1922
|
+
if (response.status === 400) {
|
|
1923
|
+
const error$1 = await DurableStreamError.fromResponse(response, url);
|
|
1924
|
+
throw error$1;
|
|
1925
|
+
}
|
|
1926
|
+
const error = await FetchError.fromResponse(response, url);
|
|
1927
|
+
throw error;
|
|
1865
1928
|
}
|
|
1866
1929
|
/**
|
|
1867
|
-
*
|
|
1930
|
+
* Clear pending batch and report error.
|
|
1868
1931
|
*/
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
};
|
|
1932
|
+
#rejectPendingBatch(error) {
|
|
1933
|
+
if (this.#onError && this.#pendingBatch.length > 0) this.#onError(error);
|
|
1934
|
+
this.#pendingBatch = [];
|
|
1935
|
+
this.#batchBytes = 0;
|
|
1936
|
+
if (this.#lingerTimeout) {
|
|
1937
|
+
clearTimeout(this.#lingerTimeout);
|
|
1938
|
+
this.#lingerTimeout = null;
|
|
1939
|
+
}
|
|
1878
1940
|
}
|
|
1879
1941
|
};
|
|
1880
|
-
/**
|
|
1881
|
-
* Encode a body value to the appropriate format.
|
|
1882
|
-
* Strings are encoded as UTF-8.
|
|
1883
|
-
* Objects are JSON-serialized.
|
|
1884
|
-
*/
|
|
1885
|
-
function encodeBody(body) {
|
|
1886
|
-
if (body === void 0) return void 0;
|
|
1887
|
-
if (typeof body === `string`) return new TextEncoder().encode(body);
|
|
1888
|
-
if (body instanceof Uint8Array) return body;
|
|
1889
|
-
if (body instanceof Blob || body instanceof FormData || body instanceof ReadableStream || body instanceof ArrayBuffer || ArrayBuffer.isView(body)) return body;
|
|
1890
|
-
return new TextEncoder().encode(JSON.stringify(body));
|
|
1891
|
-
}
|
|
1892
|
-
/**
|
|
1893
|
-
* Convert an async iterable to a ReadableStream.
|
|
1894
|
-
*/
|
|
1895
|
-
function toReadableStream(source) {
|
|
1896
|
-
if (source instanceof ReadableStream) return source.pipeThrough(new TransformStream({ transform(chunk, controller) {
|
|
1897
|
-
if (typeof chunk === `string`) controller.enqueue(new TextEncoder().encode(chunk));
|
|
1898
|
-
else controller.enqueue(chunk);
|
|
1899
|
-
} }));
|
|
1900
|
-
const encoder = new TextEncoder();
|
|
1901
|
-
const iterator = source[Symbol.asyncIterator]();
|
|
1902
|
-
return new ReadableStream({
|
|
1903
|
-
async pull(controller) {
|
|
1904
|
-
try {
|
|
1905
|
-
const { done, value } = await iterator.next();
|
|
1906
|
-
if (done) controller.close();
|
|
1907
|
-
else if (typeof value === `string`) controller.enqueue(encoder.encode(value));
|
|
1908
|
-
else controller.enqueue(value);
|
|
1909
|
-
} catch (e) {
|
|
1910
|
-
controller.error(e);
|
|
1911
|
-
}
|
|
1912
|
-
},
|
|
1913
|
-
cancel() {
|
|
1914
|
-
iterator.return?.();
|
|
1915
|
-
}
|
|
1916
|
-
});
|
|
1917
|
-
}
|
|
1918
|
-
/**
|
|
1919
|
-
* Validate stream options.
|
|
1920
|
-
*/
|
|
1921
|
-
function validateOptions(options) {
|
|
1922
|
-
if (!options.url) throw new MissingStreamUrlError();
|
|
1923
|
-
if (options.signal && !(options.signal instanceof AbortSignal)) throw new InvalidSignalError();
|
|
1924
|
-
warnIfUsingHttpInBrowser(options.url, options.warnOnHttp);
|
|
1925
|
-
}
|
|
1926
1942
|
|
|
1927
1943
|
//#endregion
|
|
1928
|
-
//#region src/
|
|
1929
|
-
/**
|
|
1930
|
-
* Error thrown when a producer's epoch is stale (zombie fencing).
|
|
1931
|
-
*/
|
|
1932
|
-
var StaleEpochError = class extends Error {
|
|
1933
|
-
/**
|
|
1934
|
-
* The current epoch on the server.
|
|
1935
|
-
*/
|
|
1936
|
-
currentEpoch;
|
|
1937
|
-
constructor(currentEpoch) {
|
|
1938
|
-
super(`Producer epoch is stale. Current server epoch: ${currentEpoch}. Call restart() or create a new producer with a higher epoch.`);
|
|
1939
|
-
this.name = `StaleEpochError`;
|
|
1940
|
-
this.currentEpoch = currentEpoch;
|
|
1941
|
-
}
|
|
1942
|
-
};
|
|
1943
|
-
/**
|
|
1944
|
-
* Error thrown when an unrecoverable sequence gap is detected.
|
|
1945
|
-
*
|
|
1946
|
-
* With maxInFlight > 1, HTTP requests can arrive out of order at the server,
|
|
1947
|
-
* causing temporary 409 responses. The client automatically handles these
|
|
1948
|
-
* by waiting for earlier sequences to complete, then retrying.
|
|
1949
|
-
*
|
|
1950
|
-
* This error is only thrown when the gap cannot be resolved (e.g., the
|
|
1951
|
-
* expected sequence is >= our sequence, indicating a true protocol violation).
|
|
1952
|
-
*/
|
|
1953
|
-
var SequenceGapError = class extends Error {
|
|
1954
|
-
expectedSeq;
|
|
1955
|
-
receivedSeq;
|
|
1956
|
-
constructor(expectedSeq, receivedSeq) {
|
|
1957
|
-
super(`Producer sequence gap: expected ${expectedSeq}, received ${receivedSeq}`);
|
|
1958
|
-
this.name = `SequenceGapError`;
|
|
1959
|
-
this.expectedSeq = expectedSeq;
|
|
1960
|
-
this.receivedSeq = receivedSeq;
|
|
1961
|
-
}
|
|
1962
|
-
};
|
|
1944
|
+
//#region src/stream.ts
|
|
1963
1945
|
/**
|
|
1964
1946
|
* Normalize content-type by extracting the media type (before any semicolon).
|
|
1947
|
+
* Handles cases like "application/json; charset=utf-8".
|
|
1965
1948
|
*/
|
|
1966
1949
|
function normalizeContentType(contentType) {
|
|
1967
1950
|
if (!contentType) return ``;
|
|
1968
1951
|
return contentType.split(`;`)[0].trim().toLowerCase();
|
|
1969
1952
|
}
|
|
1970
1953
|
/**
|
|
1971
|
-
*
|
|
1954
|
+
* Check if a value is a Promise or Promise-like (thenable).
|
|
1955
|
+
*/
|
|
1956
|
+
function isPromiseLike(value) {
|
|
1957
|
+
return value != null && typeof value.then === `function`;
|
|
1958
|
+
}
|
|
1959
|
+
/**
|
|
1960
|
+
* A handle to a remote durable stream for read/write operations.
|
|
1972
1961
|
*
|
|
1973
|
-
*
|
|
1974
|
-
*
|
|
1975
|
-
*
|
|
1976
|
-
* - Batching: multiple appends batched into single HTTP request
|
|
1977
|
-
* - Pipelining: up to maxInFlight concurrent batches
|
|
1978
|
-
* - Zombie fencing: stale producers rejected via epoch validation
|
|
1962
|
+
* This is a lightweight, reusable handle - not a persistent connection.
|
|
1963
|
+
* It does not automatically start reading or listening.
|
|
1964
|
+
* Create sessions as needed via stream().
|
|
1979
1965
|
*
|
|
1980
1966
|
* @example
|
|
1981
1967
|
* ```typescript
|
|
1982
|
-
*
|
|
1983
|
-
* const
|
|
1984
|
-
*
|
|
1985
|
-
*
|
|
1968
|
+
* // Create a new stream
|
|
1969
|
+
* const stream = await DurableStream.create({
|
|
1970
|
+
* url: "https://streams.example.com/my-stream",
|
|
1971
|
+
* headers: { Authorization: "Bearer my-token" },
|
|
1972
|
+
* contentType: "application/json"
|
|
1986
1973
|
* });
|
|
1987
1974
|
*
|
|
1988
|
-
* //
|
|
1989
|
-
*
|
|
1990
|
-
* producer.append("message 2");
|
|
1975
|
+
* // Write data
|
|
1976
|
+
* await stream.append(JSON.stringify({ message: "hello" }));
|
|
1991
1977
|
*
|
|
1992
|
-
* //
|
|
1993
|
-
* await
|
|
1994
|
-
*
|
|
1978
|
+
* // Read with the new API
|
|
1979
|
+
* const res = await stream.stream<{ message: string }>();
|
|
1980
|
+
* res.subscribeJson(async (batch) => {
|
|
1981
|
+
* for (const item of batch.items) {
|
|
1982
|
+
* console.log(item.message);
|
|
1983
|
+
* }
|
|
1984
|
+
* });
|
|
1995
1985
|
* ```
|
|
1996
1986
|
*/
|
|
1997
|
-
var
|
|
1998
|
-
|
|
1999
|
-
|
|
2000
|
-
|
|
2001
|
-
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
1987
|
+
var DurableStream = class DurableStream {
|
|
1988
|
+
/**
|
|
1989
|
+
* The URL of the durable stream.
|
|
1990
|
+
*/
|
|
1991
|
+
url;
|
|
1992
|
+
/**
|
|
1993
|
+
* The content type of the stream (populated after connect/head/read).
|
|
1994
|
+
*/
|
|
1995
|
+
contentType;
|
|
1996
|
+
#options;
|
|
2005
1997
|
#fetchClient;
|
|
2006
|
-
#signal;
|
|
2007
1998
|
#onError;
|
|
2008
|
-
#
|
|
2009
|
-
#batchBytes = 0;
|
|
2010
|
-
#lingerTimeout = null;
|
|
1999
|
+
#batchingEnabled;
|
|
2011
2000
|
#queue;
|
|
2012
|
-
#
|
|
2013
|
-
#closed = false;
|
|
2014
|
-
#epochClaimed;
|
|
2015
|
-
#seqState = new Map();
|
|
2001
|
+
#buffer = [];
|
|
2016
2002
|
/**
|
|
2017
|
-
* Create
|
|
2018
|
-
*
|
|
2019
|
-
* @param stream - The DurableStream to write to
|
|
2020
|
-
* @param producerId - Stable identifier for this producer (e.g., "order-service-1")
|
|
2021
|
-
* @param opts - Producer options
|
|
2003
|
+
* Create a cold handle to a stream.
|
|
2004
|
+
* No network IO is performed by the constructor.
|
|
2022
2005
|
*/
|
|
2023
|
-
constructor(
|
|
2024
|
-
|
|
2025
|
-
|
|
2026
|
-
this
|
|
2027
|
-
this.#
|
|
2028
|
-
|
|
2029
|
-
|
|
2030
|
-
|
|
2031
|
-
this.#onError = opts
|
|
2032
|
-
this
|
|
2033
|
-
this.#
|
|
2034
|
-
this.#
|
|
2035
|
-
|
|
2036
|
-
|
|
2037
|
-
|
|
2038
|
-
|
|
2006
|
+
constructor(opts) {
|
|
2007
|
+
validateOptions(opts);
|
|
2008
|
+
const urlStr = opts.url instanceof URL ? opts.url.toString() : opts.url;
|
|
2009
|
+
this.url = urlStr;
|
|
2010
|
+
this.#options = {
|
|
2011
|
+
...opts,
|
|
2012
|
+
url: urlStr
|
|
2013
|
+
};
|
|
2014
|
+
this.#onError = opts.onError;
|
|
2015
|
+
if (opts.contentType) this.contentType = opts.contentType;
|
|
2016
|
+
this.#batchingEnabled = opts.batching !== false;
|
|
2017
|
+
if (this.#batchingEnabled) this.#queue = fastq.promise(this.#batchWorker.bind(this), 1);
|
|
2018
|
+
const baseFetchClient = opts.fetch ?? ((...args) => fetch(...args));
|
|
2019
|
+
const backOffOpts = { ...opts.backoffOptions ?? BackoffDefaults };
|
|
2020
|
+
const fetchWithBackoffClient = createFetchWithBackoff(baseFetchClient, backOffOpts);
|
|
2021
|
+
this.#fetchClient = createFetchWithConsumedBody(fetchWithBackoffClient);
|
|
2039
2022
|
}
|
|
2040
2023
|
/**
|
|
2041
|
-
*
|
|
2042
|
-
*
|
|
2043
|
-
* This is fire-and-forget: returns immediately after adding to the batch.
|
|
2044
|
-
* The message is batched and sent when:
|
|
2045
|
-
* - maxBatchBytes is reached
|
|
2046
|
-
* - lingerMs elapses
|
|
2047
|
-
* - flush() is called
|
|
2048
|
-
*
|
|
2049
|
-
* Errors are reported via onError callback if configured. Use flush() to
|
|
2050
|
-
* wait for all pending messages to be sent.
|
|
2051
|
-
*
|
|
2052
|
-
* For JSON streams, pass native objects (which will be serialized internally).
|
|
2053
|
-
* For byte streams, pass string or Uint8Array.
|
|
2054
|
-
*
|
|
2055
|
-
* @param body - Data to append (object for JSON streams, string or Uint8Array for byte streams)
|
|
2024
|
+
* Create a new stream (create-only PUT) and return a handle.
|
|
2025
|
+
* Fails with DurableStreamError(code="CONFLICT_EXISTS") if it already exists.
|
|
2056
2026
|
*/
|
|
2057
|
-
|
|
2058
|
-
|
|
2059
|
-
|
|
2060
|
-
|
|
2061
|
-
|
|
2062
|
-
|
|
2063
|
-
|
|
2064
|
-
bytes = new TextEncoder().encode(json);
|
|
2065
|
-
data = body;
|
|
2066
|
-
} else {
|
|
2067
|
-
if (typeof body === `string`) bytes = new TextEncoder().encode(body);
|
|
2068
|
-
else if (body instanceof Uint8Array) bytes = body;
|
|
2069
|
-
else throw new DurableStreamError(`Non-JSON streams require string or Uint8Array`, `BAD_REQUEST`, 400, void 0);
|
|
2070
|
-
data = bytes;
|
|
2071
|
-
}
|
|
2072
|
-
this.#pendingBatch.push({
|
|
2073
|
-
data,
|
|
2074
|
-
body: bytes
|
|
2027
|
+
static async create(opts) {
|
|
2028
|
+
const stream$1 = new DurableStream(opts);
|
|
2029
|
+
await stream$1.create({
|
|
2030
|
+
contentType: opts.contentType,
|
|
2031
|
+
ttlSeconds: opts.ttlSeconds,
|
|
2032
|
+
expiresAt: opts.expiresAt,
|
|
2033
|
+
body: opts.body
|
|
2075
2034
|
});
|
|
2076
|
-
|
|
2077
|
-
if (this.#batchBytes >= this.#maxBatchBytes) this.#enqueuePendingBatch();
|
|
2078
|
-
else if (!this.#lingerTimeout) this.#lingerTimeout = setTimeout(() => {
|
|
2079
|
-
this.#lingerTimeout = null;
|
|
2080
|
-
if (this.#pendingBatch.length > 0) this.#enqueuePendingBatch();
|
|
2081
|
-
}, this.#lingerMs);
|
|
2035
|
+
return stream$1;
|
|
2082
2036
|
}
|
|
2083
2037
|
/**
|
|
2084
|
-
*
|
|
2038
|
+
* Validate that a stream exists and fetch metadata via HEAD.
|
|
2039
|
+
* Returns a handle with contentType populated (if sent by server).
|
|
2085
2040
|
*
|
|
2086
|
-
*
|
|
2041
|
+
* **Important**: This only performs a HEAD request for validation - it does
|
|
2042
|
+
* NOT open a session or start reading data. To read from the stream, call
|
|
2043
|
+
* `stream()` on the returned handle.
|
|
2044
|
+
*
|
|
2045
|
+
* @example
|
|
2046
|
+
* ```typescript
|
|
2047
|
+
* // Validate stream exists before reading
|
|
2048
|
+
* const handle = await DurableStream.connect({ url })
|
|
2049
|
+
* const res = await handle.stream() // Now actually read
|
|
2050
|
+
* ```
|
|
2087
2051
|
*/
|
|
2088
|
-
async
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2092
|
-
|
|
2093
|
-
|
|
2094
|
-
|
|
2052
|
+
static async connect(opts) {
|
|
2053
|
+
const stream$1 = new DurableStream(opts);
|
|
2054
|
+
await stream$1.head();
|
|
2055
|
+
return stream$1;
|
|
2056
|
+
}
|
|
2057
|
+
/**
|
|
2058
|
+
* HEAD metadata for a stream without creating a handle.
|
|
2059
|
+
*/
|
|
2060
|
+
static async head(opts) {
|
|
2061
|
+
const stream$1 = new DurableStream(opts);
|
|
2062
|
+
return stream$1.head();
|
|
2095
2063
|
}
|
|
2096
2064
|
/**
|
|
2097
|
-
*
|
|
2098
|
-
*
|
|
2099
|
-
* After calling close(), further append() calls will throw.
|
|
2065
|
+
* Delete a stream without creating a handle.
|
|
2100
2066
|
*/
|
|
2101
|
-
async
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
try {
|
|
2105
|
-
await this.flush();
|
|
2106
|
-
} catch {}
|
|
2067
|
+
static async delete(opts) {
|
|
2068
|
+
const stream$1 = new DurableStream(opts);
|
|
2069
|
+
return stream$1.delete();
|
|
2107
2070
|
}
|
|
2108
2071
|
/**
|
|
2109
|
-
*
|
|
2110
|
-
*
|
|
2111
|
-
* Call this when restarting the producer to establish a new session.
|
|
2112
|
-
* Flushes any pending messages first.
|
|
2072
|
+
* HEAD metadata for this stream.
|
|
2113
2073
|
*/
|
|
2114
|
-
async
|
|
2115
|
-
await this
|
|
2116
|
-
this.#
|
|
2117
|
-
|
|
2074
|
+
async head(opts) {
|
|
2075
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2076
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2077
|
+
method: `HEAD`,
|
|
2078
|
+
headers: requestHeaders,
|
|
2079
|
+
signal: opts?.signal ?? this.#options.signal
|
|
2080
|
+
});
|
|
2081
|
+
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
2082
|
+
const contentType = response.headers.get(`content-type`) ?? void 0;
|
|
2083
|
+
const offset = response.headers.get(STREAM_OFFSET_HEADER) ?? void 0;
|
|
2084
|
+
const etag = response.headers.get(`etag`) ?? void 0;
|
|
2085
|
+
const cacheControl = response.headers.get(`cache-control`) ?? void 0;
|
|
2086
|
+
if (contentType) this.contentType = contentType;
|
|
2087
|
+
return {
|
|
2088
|
+
exists: true,
|
|
2089
|
+
contentType,
|
|
2090
|
+
offset,
|
|
2091
|
+
etag,
|
|
2092
|
+
cacheControl
|
|
2093
|
+
};
|
|
2118
2094
|
}
|
|
2119
2095
|
/**
|
|
2120
|
-
*
|
|
2096
|
+
* Create this stream (create-only PUT) using the URL/auth from the handle.
|
|
2121
2097
|
*/
|
|
2122
|
-
|
|
2123
|
-
|
|
2098
|
+
async create(opts) {
|
|
2099
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2100
|
+
const contentType = opts?.contentType ?? this.#options.contentType;
|
|
2101
|
+
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
2102
|
+
if (opts?.ttlSeconds !== void 0) requestHeaders[STREAM_TTL_HEADER] = String(opts.ttlSeconds);
|
|
2103
|
+
if (opts?.expiresAt) requestHeaders[STREAM_EXPIRES_AT_HEADER] = opts.expiresAt;
|
|
2104
|
+
const body = encodeBody(opts?.body);
|
|
2105
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2106
|
+
method: `PUT`,
|
|
2107
|
+
headers: requestHeaders,
|
|
2108
|
+
body,
|
|
2109
|
+
signal: this.#options.signal
|
|
2110
|
+
});
|
|
2111
|
+
if (!response.ok) await handleErrorResponse(response, this.url, { operation: `create` });
|
|
2112
|
+
const responseContentType = response.headers.get(`content-type`);
|
|
2113
|
+
if (responseContentType) this.contentType = responseContentType;
|
|
2114
|
+
else if (contentType) this.contentType = contentType;
|
|
2115
|
+
return this;
|
|
2124
2116
|
}
|
|
2125
2117
|
/**
|
|
2126
|
-
*
|
|
2118
|
+
* Delete this stream.
|
|
2127
2119
|
*/
|
|
2128
|
-
|
|
2129
|
-
|
|
2120
|
+
async delete(opts) {
|
|
2121
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2122
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2123
|
+
method: `DELETE`,
|
|
2124
|
+
headers: requestHeaders,
|
|
2125
|
+
signal: opts?.signal ?? this.#options.signal
|
|
2126
|
+
});
|
|
2127
|
+
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
2130
2128
|
}
|
|
2131
2129
|
/**
|
|
2132
|
-
*
|
|
2130
|
+
* Append a single payload to the stream.
|
|
2131
|
+
*
|
|
2132
|
+
* When batching is enabled (default), multiple append() calls made while
|
|
2133
|
+
* a POST is in-flight will be batched together into a single request.
|
|
2134
|
+
* This significantly improves throughput for high-frequency writes.
|
|
2135
|
+
*
|
|
2136
|
+
* - `body` must be string or Uint8Array.
|
|
2137
|
+
* - For JSON streams, pass pre-serialized JSON strings.
|
|
2138
|
+
* - `body` may also be a Promise that resolves to string or Uint8Array.
|
|
2139
|
+
* - Strings are encoded as UTF-8.
|
|
2140
|
+
* - `seq` (if provided) is sent as stream-seq (writer coordination).
|
|
2141
|
+
*
|
|
2142
|
+
* @example
|
|
2143
|
+
* ```typescript
|
|
2144
|
+
* // JSON stream - pass pre-serialized JSON
|
|
2145
|
+
* await stream.append(JSON.stringify({ message: "hello" }));
|
|
2146
|
+
*
|
|
2147
|
+
* // Byte stream
|
|
2148
|
+
* await stream.append("raw text data");
|
|
2149
|
+
* await stream.append(new Uint8Array([1, 2, 3]));
|
|
2150
|
+
*
|
|
2151
|
+
* // Promise value - awaited before buffering
|
|
2152
|
+
* await stream.append(fetchData());
|
|
2153
|
+
* ```
|
|
2133
2154
|
*/
|
|
2134
|
-
|
|
2135
|
-
|
|
2155
|
+
async append(body, opts) {
|
|
2156
|
+
const resolvedBody = isPromiseLike(body) ? await body : body;
|
|
2157
|
+
if (this.#batchingEnabled && this.#queue) return this.#appendWithBatching(resolvedBody, opts);
|
|
2158
|
+
return this.#appendDirect(resolvedBody, opts);
|
|
2136
2159
|
}
|
|
2137
2160
|
/**
|
|
2138
|
-
*
|
|
2161
|
+
* Direct append without batching (used when batching is disabled).
|
|
2139
2162
|
*/
|
|
2140
|
-
|
|
2141
|
-
|
|
2163
|
+
async #appendDirect(body, opts) {
|
|
2164
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2165
|
+
const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
2166
|
+
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
2167
|
+
if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
|
|
2168
|
+
const isJson = normalizeContentType(contentType) === `application/json`;
|
|
2169
|
+
const bodyStr = typeof body === `string` ? body : new TextDecoder().decode(body);
|
|
2170
|
+
const encodedBody = isJson ? `[${bodyStr}]` : bodyStr;
|
|
2171
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2172
|
+
method: `POST`,
|
|
2173
|
+
headers: requestHeaders,
|
|
2174
|
+
body: encodedBody,
|
|
2175
|
+
signal: opts?.signal ?? this.#options.signal
|
|
2176
|
+
});
|
|
2177
|
+
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
2142
2178
|
}
|
|
2143
2179
|
/**
|
|
2144
|
-
*
|
|
2180
|
+
* Append with batching - buffers messages and sends them in batches.
|
|
2145
2181
|
*/
|
|
2146
|
-
#
|
|
2147
|
-
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2182
|
+
async #appendWithBatching(body, opts) {
|
|
2183
|
+
return new Promise((resolve, reject) => {
|
|
2184
|
+
this.#buffer.push({
|
|
2185
|
+
data: body,
|
|
2186
|
+
seq: opts?.seq,
|
|
2187
|
+
contentType: opts?.contentType,
|
|
2188
|
+
signal: opts?.signal,
|
|
2189
|
+
resolve,
|
|
2190
|
+
reject
|
|
2191
|
+
});
|
|
2192
|
+
if (this.#queue.idle()) {
|
|
2193
|
+
const batch = this.#buffer.splice(0);
|
|
2194
|
+
this.#queue.push(batch).catch((err) => {
|
|
2195
|
+
for (const msg of batch) msg.reject(err);
|
|
2196
|
+
});
|
|
2197
|
+
}
|
|
2158
2198
|
});
|
|
2159
|
-
else this.#queue.push({
|
|
2160
|
-
batch,
|
|
2161
|
-
seq
|
|
2162
|
-
}).catch(() => {});
|
|
2163
2199
|
}
|
|
2164
2200
|
/**
|
|
2165
|
-
* Batch worker - processes batches
|
|
2201
|
+
* Batch worker - processes batches of messages.
|
|
2166
2202
|
*/
|
|
2167
|
-
async #batchWorker(
|
|
2168
|
-
const { batch, seq } = task;
|
|
2169
|
-
const epoch = this.#epoch;
|
|
2203
|
+
async #batchWorker(batch) {
|
|
2170
2204
|
try {
|
|
2171
|
-
await this.#
|
|
2172
|
-
|
|
2173
|
-
this.#
|
|
2205
|
+
await this.#sendBatch(batch);
|
|
2206
|
+
for (const msg of batch) msg.resolve();
|
|
2207
|
+
if (this.#buffer.length > 0) {
|
|
2208
|
+
const nextBatch = this.#buffer.splice(0);
|
|
2209
|
+
this.#queue.push(nextBatch).catch((err) => {
|
|
2210
|
+
for (const msg of nextBatch) msg.reject(err);
|
|
2211
|
+
});
|
|
2212
|
+
}
|
|
2174
2213
|
} catch (error) {
|
|
2175
|
-
|
|
2176
|
-
|
|
2214
|
+
for (const msg of batch) msg.reject(error);
|
|
2215
|
+
for (const msg of this.#buffer) msg.reject(error);
|
|
2216
|
+
this.#buffer = [];
|
|
2177
2217
|
throw error;
|
|
2178
2218
|
}
|
|
2179
2219
|
}
|
|
2180
2220
|
/**
|
|
2181
|
-
*
|
|
2221
|
+
* Send a batch of messages as a single POST request.
|
|
2182
2222
|
*/
|
|
2183
|
-
#
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2223
|
+
async #sendBatch(batch) {
|
|
2224
|
+
if (batch.length === 0) return;
|
|
2225
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2226
|
+
const contentType = batch[0]?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
2227
|
+
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
2228
|
+
let highestSeq;
|
|
2229
|
+
for (let i = batch.length - 1; i >= 0; i--) if (batch[i].seq !== void 0) {
|
|
2230
|
+
highestSeq = batch[i].seq;
|
|
2231
|
+
break;
|
|
2188
2232
|
}
|
|
2189
|
-
|
|
2190
|
-
|
|
2191
|
-
|
|
2192
|
-
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
} else
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2233
|
+
if (highestSeq) requestHeaders[STREAM_SEQ_HEADER] = highestSeq;
|
|
2234
|
+
const isJson = normalizeContentType(contentType) === `application/json`;
|
|
2235
|
+
let batchedBody;
|
|
2236
|
+
if (isJson) {
|
|
2237
|
+
const jsonStrings = batch.map((m) => typeof m.data === `string` ? m.data : new TextDecoder().decode(m.data));
|
|
2238
|
+
batchedBody = `[${jsonStrings.join(`,`)}]`;
|
|
2239
|
+
} else {
|
|
2240
|
+
const strings = batch.map((m) => typeof m.data === `string` ? m.data : new TextDecoder().decode(m.data));
|
|
2241
|
+
batchedBody = strings.join(``);
|
|
2242
|
+
}
|
|
2243
|
+
const signals = [];
|
|
2244
|
+
if (this.#options.signal) signals.push(this.#options.signal);
|
|
2245
|
+
for (const msg of batch) if (msg.signal) signals.push(msg.signal);
|
|
2246
|
+
const combinedSignal = signals.length > 0 ? AbortSignal.any(signals) : void 0;
|
|
2247
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2248
|
+
method: `POST`,
|
|
2249
|
+
headers: requestHeaders,
|
|
2250
|
+
body: batchedBody,
|
|
2251
|
+
signal: combinedSignal
|
|
2252
|
+
});
|
|
2253
|
+
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
2254
|
+
}
|
|
2255
|
+
/**
|
|
2256
|
+
* Append a streaming body to the stream.
|
|
2257
|
+
*
|
|
2258
|
+
* Supports piping from any ReadableStream or async iterable:
|
|
2259
|
+
* - `source` yields Uint8Array or string chunks.
|
|
2260
|
+
* - Strings are encoded as UTF-8; no delimiters are added.
|
|
2261
|
+
* - Internally uses chunked transfer or HTTP/2 streaming.
|
|
2262
|
+
*
|
|
2263
|
+
* @example
|
|
2264
|
+
* ```typescript
|
|
2265
|
+
* // Pipe from a ReadableStream
|
|
2266
|
+
* const readable = new ReadableStream({
|
|
2267
|
+
* start(controller) {
|
|
2268
|
+
* controller.enqueue("chunk 1");
|
|
2269
|
+
* controller.enqueue("chunk 2");
|
|
2270
|
+
* controller.close();
|
|
2271
|
+
* }
|
|
2272
|
+
* });
|
|
2273
|
+
* await stream.appendStream(readable);
|
|
2274
|
+
*
|
|
2275
|
+
* // Pipe from an async generator
|
|
2276
|
+
* async function* generate() {
|
|
2277
|
+
* yield "line 1\n";
|
|
2278
|
+
* yield "line 2\n";
|
|
2279
|
+
* }
|
|
2280
|
+
* await stream.appendStream(generate());
|
|
2281
|
+
*
|
|
2282
|
+
* // Pipe from fetch response body
|
|
2283
|
+
* const response = await fetch("https://example.com/data");
|
|
2284
|
+
* await stream.appendStream(response.body!);
|
|
2285
|
+
* ```
|
|
2286
|
+
*/
|
|
2287
|
+
async appendStream(source, opts) {
|
|
2288
|
+
const { requestHeaders, fetchUrl } = await this.#buildRequest();
|
|
2289
|
+
const contentType = opts?.contentType ?? this.#options.contentType ?? this.contentType;
|
|
2290
|
+
if (contentType) requestHeaders[`content-type`] = contentType;
|
|
2291
|
+
if (opts?.seq) requestHeaders[STREAM_SEQ_HEADER] = opts.seq;
|
|
2292
|
+
const body = toReadableStream(source);
|
|
2293
|
+
const response = await this.#fetchClient(fetchUrl.toString(), {
|
|
2294
|
+
method: `POST`,
|
|
2295
|
+
headers: requestHeaders,
|
|
2296
|
+
body,
|
|
2297
|
+
duplex: `half`,
|
|
2298
|
+
signal: opts?.signal ?? this.#options.signal
|
|
2199
2299
|
});
|
|
2200
|
-
|
|
2201
|
-
if (cleanupThreshold > 0) {
|
|
2202
|
-
for (const oldSeq of epochMap.keys()) if (oldSeq < cleanupThreshold) epochMap.delete(oldSeq);
|
|
2203
|
-
}
|
|
2300
|
+
if (!response.ok) await handleErrorResponse(response, this.url);
|
|
2204
2301
|
}
|
|
2205
2302
|
/**
|
|
2206
|
-
*
|
|
2207
|
-
*
|
|
2208
|
-
*
|
|
2303
|
+
* Create a writable stream that pipes data to this durable stream.
|
|
2304
|
+
*
|
|
2305
|
+
* Returns a WritableStream that can be used with `pipeTo()` or
|
|
2306
|
+
* `pipeThrough()` from any ReadableStream source.
|
|
2307
|
+
*
|
|
2308
|
+
* Uses IdempotentProducer internally for:
|
|
2309
|
+
* - Automatic batching (controlled by lingerMs, maxBatchBytes)
|
|
2310
|
+
* - Exactly-once delivery semantics
|
|
2311
|
+
* - Streaming writes (doesn't buffer entire content in memory)
|
|
2312
|
+
*
|
|
2313
|
+
* @example
|
|
2314
|
+
* ```typescript
|
|
2315
|
+
* // Pipe from fetch response
|
|
2316
|
+
* const response = await fetch("https://example.com/data");
|
|
2317
|
+
* await response.body!.pipeTo(stream.writable());
|
|
2318
|
+
*
|
|
2319
|
+
* // Pipe through a transform
|
|
2320
|
+
* const readable = someStream.pipeThrough(new TextEncoderStream());
|
|
2321
|
+
* await readable.pipeTo(stream.writable());
|
|
2322
|
+
*
|
|
2323
|
+
* // With custom producer options
|
|
2324
|
+
* await source.pipeTo(stream.writable({
|
|
2325
|
+
* producerId: "my-producer",
|
|
2326
|
+
* lingerMs: 10,
|
|
2327
|
+
* maxBatchBytes: 64 * 1024,
|
|
2328
|
+
* }));
|
|
2329
|
+
* ```
|
|
2209
2330
|
*/
|
|
2210
|
-
|
|
2211
|
-
|
|
2212
|
-
|
|
2213
|
-
|
|
2214
|
-
|
|
2215
|
-
|
|
2216
|
-
|
|
2217
|
-
|
|
2218
|
-
|
|
2219
|
-
|
|
2220
|
-
|
|
2221
|
-
|
|
2222
|
-
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2331
|
+
writable(opts) {
|
|
2332
|
+
const producerId = opts?.producerId ?? `writable-${crypto.randomUUID().slice(0, 8)}`;
|
|
2333
|
+
let writeError = null;
|
|
2334
|
+
const producer = new IdempotentProducer(this, producerId, {
|
|
2335
|
+
autoClaim: true,
|
|
2336
|
+
lingerMs: opts?.lingerMs,
|
|
2337
|
+
maxBatchBytes: opts?.maxBatchBytes,
|
|
2338
|
+
onError: (error) => {
|
|
2339
|
+
if (!writeError) writeError = error;
|
|
2340
|
+
opts?.onError?.(error);
|
|
2341
|
+
},
|
|
2342
|
+
signal: opts?.signal ?? this.#options.signal
|
|
2343
|
+
});
|
|
2344
|
+
return new WritableStream({
|
|
2345
|
+
write(chunk) {
|
|
2346
|
+
producer.append(chunk);
|
|
2347
|
+
},
|
|
2348
|
+
async close() {
|
|
2349
|
+
await producer.flush();
|
|
2350
|
+
await producer.close();
|
|
2351
|
+
if (writeError) throw writeError;
|
|
2352
|
+
},
|
|
2353
|
+
abort(_reason) {
|
|
2354
|
+
producer.close().catch((err) => {
|
|
2355
|
+
opts?.onError?.(err);
|
|
2356
|
+
});
|
|
2357
|
+
}
|
|
2231
2358
|
});
|
|
2232
2359
|
}
|
|
2233
2360
|
/**
|
|
2234
|
-
*
|
|
2235
|
-
*
|
|
2236
|
-
*
|
|
2361
|
+
* Start a fetch-like streaming session against this handle's URL/headers/params.
|
|
2362
|
+
* The first request is made inside this method; it resolves when we have
|
|
2363
|
+
* a valid first response, or rejects on errors.
|
|
2364
|
+
*
|
|
2365
|
+
* Call-specific headers and params are merged with handle-level ones,
|
|
2366
|
+
* with call-specific values taking precedence.
|
|
2367
|
+
*
|
|
2368
|
+
* @example
|
|
2369
|
+
* ```typescript
|
|
2370
|
+
* const handle = await DurableStream.connect({
|
|
2371
|
+
* url,
|
|
2372
|
+
* headers: { Authorization: `Bearer ${token}` }
|
|
2373
|
+
* });
|
|
2374
|
+
* const res = await handle.stream<{ message: string }>();
|
|
2375
|
+
*
|
|
2376
|
+
* // Accumulate all JSON items
|
|
2377
|
+
* const items = await res.json();
|
|
2378
|
+
*
|
|
2379
|
+
* // Or stream live with ReadableStream
|
|
2380
|
+
* const reader = res.jsonStream().getReader();
|
|
2381
|
+
* let result = await reader.read();
|
|
2382
|
+
* while (!result.done) {
|
|
2383
|
+
* console.log(result.value);
|
|
2384
|
+
* result = await reader.read();
|
|
2385
|
+
* }
|
|
2386
|
+
*
|
|
2387
|
+
* // Or use subscriber for backpressure-aware consumption
|
|
2388
|
+
* res.subscribeJson(async (batch) => {
|
|
2389
|
+
* for (const item of batch.items) {
|
|
2390
|
+
* console.log(item);
|
|
2391
|
+
* }
|
|
2392
|
+
* });
|
|
2393
|
+
* ```
|
|
2237
2394
|
*/
|
|
2238
|
-
async
|
|
2239
|
-
|
|
2240
|
-
|
|
2241
|
-
|
|
2242
|
-
if (isJson) {
|
|
2243
|
-
const values = batch.map((e) => e.data);
|
|
2244
|
-
batchedBody = JSON.stringify(values);
|
|
2245
|
-
} else {
|
|
2246
|
-
const totalSize = batch.reduce((sum, e) => sum + e.body.length, 0);
|
|
2247
|
-
const concatenated = new Uint8Array(totalSize);
|
|
2248
|
-
let offset = 0;
|
|
2249
|
-
for (const entry of batch) {
|
|
2250
|
-
concatenated.set(entry.body, offset);
|
|
2251
|
-
offset += entry.body.length;
|
|
2252
|
-
}
|
|
2253
|
-
batchedBody = concatenated;
|
|
2395
|
+
async stream(options) {
|
|
2396
|
+
if (options?.live === `sse` && this.contentType) {
|
|
2397
|
+
const isSSECompatible = SSE_COMPATIBLE_CONTENT_TYPES.some((prefix) => this.contentType.startsWith(prefix));
|
|
2398
|
+
if (!isSSECompatible) throw new DurableStreamError(`SSE is not supported for content-type: ${this.contentType}`, `SSE_NOT_SUPPORTED`, 400);
|
|
2254
2399
|
}
|
|
2255
|
-
const
|
|
2256
|
-
|
|
2257
|
-
|
|
2258
|
-
[PRODUCER_ID_HEADER]: this.#producerId,
|
|
2259
|
-
[PRODUCER_EPOCH_HEADER]: epoch.toString(),
|
|
2260
|
-
[PRODUCER_SEQ_HEADER]: seq.toString()
|
|
2400
|
+
const mergedHeaders = {
|
|
2401
|
+
...this.#options.headers,
|
|
2402
|
+
...options?.headers
|
|
2261
2403
|
};
|
|
2262
|
-
const
|
|
2263
|
-
|
|
2264
|
-
|
|
2265
|
-
body: batchedBody,
|
|
2266
|
-
signal: this.#signal
|
|
2267
|
-
});
|
|
2268
|
-
if (response.status === 204) return {
|
|
2269
|
-
offset: ``,
|
|
2270
|
-
duplicate: true
|
|
2404
|
+
const mergedParams = {
|
|
2405
|
+
...this.#options.params,
|
|
2406
|
+
...options?.params
|
|
2271
2407
|
};
|
|
2272
|
-
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
this.#nextSeq = 1;
|
|
2286
|
-
return this.#doSendBatch(batch, 0, newEpoch);
|
|
2287
|
-
}
|
|
2288
|
-
throw new StaleEpochError(currentEpoch);
|
|
2289
|
-
}
|
|
2290
|
-
if (response.status === 409) {
|
|
2291
|
-
const expectedSeqStr = response.headers.get(PRODUCER_EXPECTED_SEQ_HEADER);
|
|
2292
|
-
const expectedSeq = expectedSeqStr ? parseInt(expectedSeqStr, 10) : 0;
|
|
2293
|
-
if (expectedSeq < seq) {
|
|
2294
|
-
const waitPromises = [];
|
|
2295
|
-
for (let s = expectedSeq; s < seq; s++) waitPromises.push(this.#waitForSeq(epoch, s));
|
|
2296
|
-
await Promise.all(waitPromises);
|
|
2297
|
-
return this.#doSendBatch(batch, seq, epoch);
|
|
2298
|
-
}
|
|
2299
|
-
const receivedSeqStr = response.headers.get(PRODUCER_RECEIVED_SEQ_HEADER);
|
|
2300
|
-
const receivedSeq = receivedSeqStr ? parseInt(receivedSeqStr, 10) : seq;
|
|
2301
|
-
throw new SequenceGapError(expectedSeq, receivedSeq);
|
|
2302
|
-
}
|
|
2303
|
-
if (response.status === 400) {
|
|
2304
|
-
const error$1 = await DurableStreamError.fromResponse(response, url);
|
|
2305
|
-
throw error$1;
|
|
2306
|
-
}
|
|
2307
|
-
const error = await FetchError.fromResponse(response, url);
|
|
2308
|
-
throw error;
|
|
2408
|
+
return stream({
|
|
2409
|
+
url: this.url,
|
|
2410
|
+
headers: mergedHeaders,
|
|
2411
|
+
params: mergedParams,
|
|
2412
|
+
signal: options?.signal ?? this.#options.signal,
|
|
2413
|
+
fetch: this.#options.fetch,
|
|
2414
|
+
backoffOptions: this.#options.backoffOptions,
|
|
2415
|
+
offset: options?.offset,
|
|
2416
|
+
live: options?.live,
|
|
2417
|
+
json: options?.json,
|
|
2418
|
+
onError: options?.onError ?? this.#onError,
|
|
2419
|
+
warnOnHttp: options?.warnOnHttp ?? this.#options.warnOnHttp
|
|
2420
|
+
});
|
|
2309
2421
|
}
|
|
2310
2422
|
/**
|
|
2311
|
-
*
|
|
2423
|
+
* Build request headers and URL.
|
|
2312
2424
|
*/
|
|
2313
|
-
#
|
|
2314
|
-
|
|
2315
|
-
|
|
2316
|
-
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2425
|
+
async #buildRequest() {
|
|
2426
|
+
const requestHeaders = await resolveHeaders(this.#options.headers);
|
|
2427
|
+
const fetchUrl = new URL(this.url);
|
|
2428
|
+
const params = await resolveParams(this.#options.params);
|
|
2429
|
+
for (const [key, value] of Object.entries(params)) fetchUrl.searchParams.set(key, value);
|
|
2430
|
+
return {
|
|
2431
|
+
requestHeaders,
|
|
2432
|
+
fetchUrl
|
|
2433
|
+
};
|
|
2321
2434
|
}
|
|
2322
2435
|
};
|
|
2436
|
+
/**
|
|
2437
|
+
* Encode a body value to the appropriate format.
|
|
2438
|
+
* Strings are encoded as UTF-8.
|
|
2439
|
+
* Objects are JSON-serialized.
|
|
2440
|
+
*/
|
|
2441
|
+
function encodeBody(body) {
|
|
2442
|
+
if (body === void 0) return void 0;
|
|
2443
|
+
if (typeof body === `string`) return new TextEncoder().encode(body);
|
|
2444
|
+
if (body instanceof Uint8Array) return body;
|
|
2445
|
+
if (body instanceof Blob || body instanceof FormData || body instanceof ReadableStream || body instanceof ArrayBuffer || ArrayBuffer.isView(body)) return body;
|
|
2446
|
+
return new TextEncoder().encode(JSON.stringify(body));
|
|
2447
|
+
}
|
|
2448
|
+
/**
|
|
2449
|
+
* Convert an async iterable to a ReadableStream.
|
|
2450
|
+
*/
|
|
2451
|
+
function toReadableStream(source) {
|
|
2452
|
+
if (source instanceof ReadableStream) return source.pipeThrough(new TransformStream({ transform(chunk, controller) {
|
|
2453
|
+
if (typeof chunk === `string`) controller.enqueue(new TextEncoder().encode(chunk));
|
|
2454
|
+
else controller.enqueue(chunk);
|
|
2455
|
+
} }));
|
|
2456
|
+
const encoder = new TextEncoder();
|
|
2457
|
+
const iterator = source[Symbol.asyncIterator]();
|
|
2458
|
+
return new ReadableStream({
|
|
2459
|
+
async pull(controller) {
|
|
2460
|
+
try {
|
|
2461
|
+
const { done, value } = await iterator.next();
|
|
2462
|
+
if (done) controller.close();
|
|
2463
|
+
else if (typeof value === `string`) controller.enqueue(encoder.encode(value));
|
|
2464
|
+
else controller.enqueue(value);
|
|
2465
|
+
} catch (e) {
|
|
2466
|
+
controller.error(e);
|
|
2467
|
+
}
|
|
2468
|
+
},
|
|
2469
|
+
cancel() {
|
|
2470
|
+
iterator.return?.();
|
|
2471
|
+
}
|
|
2472
|
+
});
|
|
2473
|
+
}
|
|
2474
|
+
/**
|
|
2475
|
+
* Validate stream options.
|
|
2476
|
+
*/
|
|
2477
|
+
function validateOptions(options) {
|
|
2478
|
+
if (!options.url) throw new MissingStreamUrlError();
|
|
2479
|
+
if (options.signal && !(options.signal instanceof AbortSignal)) throw new InvalidSignalError();
|
|
2480
|
+
warnIfUsingHttpInBrowser(options.url, options.warnOnHttp);
|
|
2481
|
+
}
|
|
2323
2482
|
|
|
2324
2483
|
//#endregion
|
|
2325
2484
|
export { BackoffDefaults, CURSOR_QUERY_PARAM, DURABLE_STREAM_PROTOCOL_QUERY_PARAMS, DurableStream, DurableStreamError, FetchBackoffAbortError, FetchError, IdempotentProducer, InvalidSignalError, LIVE_QUERY_PARAM, MissingStreamUrlError, OFFSET_QUERY_PARAM, PRODUCER_EPOCH_HEADER, PRODUCER_EXPECTED_SEQ_HEADER, PRODUCER_ID_HEADER, PRODUCER_RECEIVED_SEQ_HEADER, PRODUCER_SEQ_HEADER, SSE_COMPATIBLE_CONTENT_TYPES, STREAM_CURSOR_HEADER, STREAM_EXPIRES_AT_HEADER, STREAM_OFFSET_HEADER, STREAM_SEQ_HEADER, STREAM_TTL_HEADER, STREAM_UP_TO_DATE_HEADER, SequenceGapError, StaleEpochError, _resetHttpWarningForTesting, asAsyncIterableReadableStream, createFetchWithBackoff, createFetchWithConsumedBody, stream, warnIfUsingHttpInBrowser };
|