@s2-dev/streamstore 0.17.6 → 0.18.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (176) hide show
  1. package/README.md +69 -1
  2. package/dist/cjs/accessTokens.d.ts +3 -2
  3. package/dist/cjs/accessTokens.d.ts.map +1 -1
  4. package/dist/cjs/accessTokens.js +22 -37
  5. package/dist/cjs/accessTokens.js.map +1 -1
  6. package/dist/cjs/basin.d.ts +4 -3
  7. package/dist/cjs/basin.d.ts.map +1 -1
  8. package/dist/cjs/basin.js +16 -6
  9. package/dist/cjs/basin.js.map +1 -1
  10. package/dist/cjs/basins.d.ts +10 -10
  11. package/dist/cjs/basins.d.ts.map +1 -1
  12. package/dist/cjs/basins.js +36 -64
  13. package/dist/cjs/basins.js.map +1 -1
  14. package/dist/cjs/batch-transform.d.ts +1 -1
  15. package/dist/cjs/batch-transform.d.ts.map +1 -1
  16. package/dist/cjs/batch-transform.js +36 -5
  17. package/dist/cjs/batch-transform.js.map +1 -1
  18. package/dist/cjs/common.d.ts +42 -0
  19. package/dist/cjs/common.d.ts.map +1 -1
  20. package/dist/cjs/error.d.ts +40 -2
  21. package/dist/cjs/error.d.ts.map +1 -1
  22. package/dist/cjs/error.js +268 -2
  23. package/dist/cjs/error.js.map +1 -1
  24. package/dist/cjs/generated/client/types.gen.d.ts +7 -0
  25. package/dist/cjs/generated/client/types.gen.d.ts.map +1 -1
  26. package/dist/cjs/generated/client/utils.gen.d.ts +1 -0
  27. package/dist/cjs/generated/client/utils.gen.d.ts.map +1 -1
  28. package/dist/cjs/generated/client/utils.gen.js.map +1 -1
  29. package/dist/cjs/generated/core/types.gen.d.ts +2 -0
  30. package/dist/cjs/generated/core/types.gen.d.ts.map +1 -1
  31. package/dist/cjs/index.d.ts +46 -3
  32. package/dist/cjs/index.d.ts.map +1 -1
  33. package/dist/cjs/index.js +28 -2
  34. package/dist/cjs/index.js.map +1 -1
  35. package/dist/cjs/lib/result.d.ts +57 -0
  36. package/dist/cjs/lib/result.d.ts.map +1 -0
  37. package/dist/cjs/lib/result.js +43 -0
  38. package/dist/cjs/lib/result.js.map +1 -0
  39. package/dist/cjs/lib/retry.d.ts +151 -0
  40. package/dist/cjs/lib/retry.d.ts.map +1 -0
  41. package/dist/cjs/lib/retry.js +839 -0
  42. package/dist/cjs/lib/retry.js.map +1 -0
  43. package/dist/cjs/lib/stream/factory.d.ts +0 -1
  44. package/dist/cjs/lib/stream/factory.d.ts.map +1 -1
  45. package/dist/cjs/lib/stream/factory.js +0 -1
  46. package/dist/cjs/lib/stream/factory.js.map +1 -1
  47. package/dist/cjs/lib/stream/runtime.d.ts +14 -0
  48. package/dist/cjs/lib/stream/runtime.d.ts.map +1 -1
  49. package/dist/cjs/lib/stream/runtime.js +18 -3
  50. package/dist/cjs/lib/stream/runtime.js.map +1 -1
  51. package/dist/cjs/lib/stream/transport/fetch/index.d.ts +24 -32
  52. package/dist/cjs/lib/stream/transport/fetch/index.d.ts.map +1 -1
  53. package/dist/cjs/lib/stream/transport/fetch/index.js +260 -187
  54. package/dist/cjs/lib/stream/transport/fetch/index.js.map +1 -1
  55. package/dist/cjs/lib/stream/transport/fetch/shared.d.ts +1 -2
  56. package/dist/cjs/lib/stream/transport/fetch/shared.d.ts.map +1 -1
  57. package/dist/cjs/lib/stream/transport/fetch/shared.js +49 -72
  58. package/dist/cjs/lib/stream/transport/fetch/shared.js.map +1 -1
  59. package/dist/cjs/lib/stream/transport/s2s/index.d.ts +0 -1
  60. package/dist/cjs/lib/stream/transport/s2s/index.d.ts.map +1 -1
  61. package/dist/cjs/lib/stream/transport/s2s/index.js +312 -352
  62. package/dist/cjs/lib/stream/transport/s2s/index.js.map +1 -1
  63. package/dist/cjs/lib/stream/types.d.ts +102 -8
  64. package/dist/cjs/lib/stream/types.d.ts.map +1 -1
  65. package/dist/cjs/metrics.d.ts +3 -2
  66. package/dist/cjs/metrics.d.ts.map +1 -1
  67. package/dist/cjs/metrics.js +24 -39
  68. package/dist/cjs/metrics.js.map +1 -1
  69. package/dist/cjs/s2.d.ts +1 -0
  70. package/dist/cjs/s2.d.ts.map +1 -1
  71. package/dist/cjs/s2.js +20 -3
  72. package/dist/cjs/s2.js.map +1 -1
  73. package/dist/cjs/stream.d.ts +5 -3
  74. package/dist/cjs/stream.d.ts.map +1 -1
  75. package/dist/cjs/stream.js +29 -18
  76. package/dist/cjs/stream.js.map +1 -1
  77. package/dist/cjs/streams.d.ts +10 -10
  78. package/dist/cjs/streams.d.ts.map +1 -1
  79. package/dist/cjs/streams.js +36 -64
  80. package/dist/cjs/streams.js.map +1 -1
  81. package/dist/cjs/utils.d.ts +3 -3
  82. package/dist/cjs/utils.d.ts.map +1 -1
  83. package/dist/cjs/utils.js +3 -3
  84. package/dist/cjs/utils.js.map +1 -1
  85. package/dist/cjs/version.d.ts +8 -0
  86. package/dist/cjs/version.d.ts.map +1 -0
  87. package/dist/cjs/version.js +11 -0
  88. package/dist/cjs/version.js.map +1 -0
  89. package/dist/esm/accessTokens.d.ts +3 -2
  90. package/dist/esm/accessTokens.d.ts.map +1 -1
  91. package/dist/esm/accessTokens.js +23 -38
  92. package/dist/esm/accessTokens.js.map +1 -1
  93. package/dist/esm/basin.d.ts +4 -3
  94. package/dist/esm/basin.d.ts.map +1 -1
  95. package/dist/esm/basin.js +16 -6
  96. package/dist/esm/basin.js.map +1 -1
  97. package/dist/esm/basins.d.ts +10 -10
  98. package/dist/esm/basins.d.ts.map +1 -1
  99. package/dist/esm/basins.js +37 -65
  100. package/dist/esm/basins.js.map +1 -1
  101. package/dist/esm/batch-transform.d.ts +1 -1
  102. package/dist/esm/batch-transform.d.ts.map +1 -1
  103. package/dist/esm/batch-transform.js +37 -6
  104. package/dist/esm/batch-transform.js.map +1 -1
  105. package/dist/esm/common.d.ts +42 -0
  106. package/dist/esm/common.d.ts.map +1 -1
  107. package/dist/esm/error.d.ts +40 -2
  108. package/dist/esm/error.d.ts.map +1 -1
  109. package/dist/esm/error.js +260 -2
  110. package/dist/esm/error.js.map +1 -1
  111. package/dist/esm/generated/client/types.gen.d.ts +7 -0
  112. package/dist/esm/generated/client/types.gen.d.ts.map +1 -1
  113. package/dist/esm/generated/client/utils.gen.d.ts +1 -0
  114. package/dist/esm/generated/client/utils.gen.d.ts.map +1 -1
  115. package/dist/esm/generated/client/utils.gen.js.map +1 -1
  116. package/dist/esm/generated/core/types.gen.d.ts +2 -0
  117. package/dist/esm/generated/core/types.gen.d.ts.map +1 -1
  118. package/dist/esm/index.d.ts +46 -3
  119. package/dist/esm/index.d.ts.map +1 -1
  120. package/dist/esm/index.js +23 -1
  121. package/dist/esm/index.js.map +1 -1
  122. package/dist/esm/lib/result.d.ts +57 -0
  123. package/dist/esm/lib/result.d.ts.map +1 -0
  124. package/dist/esm/lib/result.js +37 -0
  125. package/dist/esm/lib/result.js.map +1 -0
  126. package/dist/esm/lib/retry.d.ts +151 -0
  127. package/dist/esm/lib/retry.d.ts.map +1 -0
  128. package/dist/esm/lib/retry.js +830 -0
  129. package/dist/esm/lib/retry.js.map +1 -0
  130. package/dist/esm/lib/stream/factory.d.ts +0 -1
  131. package/dist/esm/lib/stream/factory.d.ts.map +1 -1
  132. package/dist/esm/lib/stream/factory.js +0 -1
  133. package/dist/esm/lib/stream/factory.js.map +1 -1
  134. package/dist/esm/lib/stream/runtime.d.ts +14 -0
  135. package/dist/esm/lib/stream/runtime.d.ts.map +1 -1
  136. package/dist/esm/lib/stream/runtime.js +23 -3
  137. package/dist/esm/lib/stream/runtime.js.map +1 -1
  138. package/dist/esm/lib/stream/transport/fetch/index.d.ts +24 -32
  139. package/dist/esm/lib/stream/transport/fetch/index.d.ts.map +1 -1
  140. package/dist/esm/lib/stream/transport/fetch/index.js +260 -187
  141. package/dist/esm/lib/stream/transport/fetch/index.js.map +1 -1
  142. package/dist/esm/lib/stream/transport/fetch/shared.d.ts +1 -2
  143. package/dist/esm/lib/stream/transport/fetch/shared.d.ts.map +1 -1
  144. package/dist/esm/lib/stream/transport/fetch/shared.js +51 -74
  145. package/dist/esm/lib/stream/transport/fetch/shared.js.map +1 -1
  146. package/dist/esm/lib/stream/transport/s2s/index.d.ts +0 -1
  147. package/dist/esm/lib/stream/transport/s2s/index.d.ts.map +1 -1
  148. package/dist/esm/lib/stream/transport/s2s/index.js +313 -353
  149. package/dist/esm/lib/stream/transport/s2s/index.js.map +1 -1
  150. package/dist/esm/lib/stream/types.d.ts +102 -8
  151. package/dist/esm/lib/stream/types.d.ts.map +1 -1
  152. package/dist/esm/metrics.d.ts +3 -2
  153. package/dist/esm/metrics.d.ts.map +1 -1
  154. package/dist/esm/metrics.js +25 -40
  155. package/dist/esm/metrics.js.map +1 -1
  156. package/dist/esm/s2.d.ts +1 -0
  157. package/dist/esm/s2.d.ts.map +1 -1
  158. package/dist/esm/s2.js +20 -3
  159. package/dist/esm/s2.js.map +1 -1
  160. package/dist/esm/stream.d.ts +5 -3
  161. package/dist/esm/stream.d.ts.map +1 -1
  162. package/dist/esm/stream.js +30 -19
  163. package/dist/esm/stream.js.map +1 -1
  164. package/dist/esm/streams.d.ts +10 -10
  165. package/dist/esm/streams.d.ts.map +1 -1
  166. package/dist/esm/streams.js +37 -65
  167. package/dist/esm/streams.js.map +1 -1
  168. package/dist/esm/utils.d.ts +3 -3
  169. package/dist/esm/utils.d.ts.map +1 -1
  170. package/dist/esm/utils.js +2 -2
  171. package/dist/esm/utils.js.map +1 -1
  172. package/dist/esm/version.d.ts +8 -0
  173. package/dist/esm/version.d.ts.map +1 -0
  174. package/dist/esm/version.js +8 -0
  175. package/dist/esm/version.js.map +1 -0
  176. package/package.json +7 -4
@@ -5,12 +5,16 @@
5
5
  * This file should only be imported in Node.js environments
6
6
  */
7
7
  import * as http2 from "node:http2";
8
- import { createClient, createConfig, } from "../../../../generated/client/index.js";
8
+ import createDebug from "debug";
9
+ import { makeAppendPreconditionError, makeServerError, RangeNotSatisfiableError, S2Error, } from "../../../../error.js";
9
10
  import { AppendAck as ProtoAppendAck, AppendInput as ProtoAppendInput, ReadBatch as ProtoReadBatch, } from "../../../../generated/proto/s2.js";
10
- import { S2Error } from "../../../../index.js";
11
- import { meteredSizeBytes } from "../../../../utils.js";
11
+ import { meteredBytes } from "../../../../utils.js";
12
12
  import * as Redacted from "../../../redacted.js";
13
+ import { err, errClose, ok, okClose } from "../../../result.js";
14
+ import { RetryAppendSession as AppendSessionImpl, RetryReadSession as ReadSessionImpl, } from "../../../retry.js";
15
+ import { DEFAULT_USER_AGENT } from "../../runtime.js";
13
16
  import { frameMessage, S2SFrameParser } from "./framing.js";
17
+ const debug = createDebug("s2:s2s");
14
18
  export function buildProtoAppendInput(records, args) {
15
19
  const textEncoder = new TextEncoder();
16
20
  return ProtoAppendInput.create({
@@ -40,22 +44,21 @@ export function buildProtoAppendInput(records, args) {
40
44
  });
41
45
  }
42
46
  export class S2STransport {
43
- client;
44
47
  transportConfig;
45
48
  connection;
46
49
  connectionPromise;
47
50
  constructor(config) {
48
- this.client = createClient(createConfig({
49
- baseUrl: config.baseUrl,
50
- auth: () => Redacted.value(config.accessToken),
51
- }));
52
51
  this.transportConfig = config;
53
52
  }
54
53
  async makeAppendSession(stream, sessionOptions, requestOptions) {
55
- return S2SAppendSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, () => this.getConnection(), sessionOptions, requestOptions);
54
+ return AppendSessionImpl.create((myOptions) => {
55
+ return S2SAppendSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, () => this.getConnection(), this.transportConfig.basinName, myOptions, requestOptions);
56
+ }, sessionOptions, this.transportConfig.retry);
56
57
  }
57
58
  async makeReadSession(stream, args, options) {
58
- return S2SReadSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, args, options, () => this.getConnection());
59
+ return ReadSessionImpl.create((myArgs) => {
60
+ return S2SReadSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, myArgs, options, () => this.getConnection(), this.transportConfig.basinName);
61
+ }, args, this.transportConfig.retry);
59
62
  }
60
63
  /**
61
64
  * Get or create HTTP/2 connection (one per transport)
@@ -117,25 +120,36 @@ class S2SReadSession extends ReadableStream {
117
120
  url;
118
121
  options;
119
122
  getConnection;
123
+ basinName;
120
124
  http2Stream;
121
125
  _lastReadPosition;
126
+ _nextReadPosition;
127
+ _lastObservedTail;
122
128
  parser = new S2SFrameParser();
123
- static async create(baseUrl, bearerToken, streamName, args, options, getConnection) {
129
+ static async create(baseUrl, bearerToken, streamName, args, options, getConnection, basinName) {
124
130
  const url = new URL(baseUrl);
125
- return new S2SReadSession(streamName, args, bearerToken, url, options, getConnection);
131
+ return new S2SReadSession(streamName, args, bearerToken, url, options, getConnection, basinName);
126
132
  }
127
- constructor(streamName, args, authToken, url, options, getConnection) {
133
+ constructor(streamName, args, authToken, url, options, getConnection, basinName) {
128
134
  // Initialize parser and textDecoder before super() call
129
135
  const parser = new S2SFrameParser();
130
136
  const textDecoder = new TextDecoder();
131
137
  let http2Stream;
132
138
  let lastReadPosition;
139
+ // Track timeout for detecting when server stops sending data
140
+ const TAIL_TIMEOUT_MS = 20000; // 20 seconds
141
+ let timeoutTimer;
133
142
  super({
134
143
  start: async (controller) => {
135
144
  let controllerClosed = false;
145
+ let responseCode;
136
146
  const safeClose = () => {
137
147
  if (!controllerClosed) {
138
148
  controllerClosed = true;
149
+ if (timeoutTimer) {
150
+ clearTimeout(timeoutTimer);
151
+ timeoutTimer = undefined;
152
+ }
139
153
  try {
140
154
  controller.close();
141
155
  }
@@ -147,10 +161,37 @@ class S2SReadSession extends ReadableStream {
147
161
  const safeError = (err) => {
148
162
  if (!controllerClosed) {
149
163
  controllerClosed = true;
150
- controller.error(err);
164
+ if (timeoutTimer) {
165
+ clearTimeout(timeoutTimer);
166
+ timeoutTimer = undefined;
167
+ }
168
+ // Convert error to S2Error and enqueue as error result
169
+ const s2Err = err instanceof S2Error
170
+ ? err
171
+ : new S2Error({ message: String(err), status: 500 });
172
+ controller.enqueue({ ok: false, error: s2Err });
173
+ controller.close();
174
+ }
175
+ };
176
+ // Helper to start/reset the timeout timer
177
+ // Resets on every tail received, fires only if no tail for 20s
178
+ const resetTimeoutTimer = () => {
179
+ if (timeoutTimer) {
180
+ clearTimeout(timeoutTimer);
151
181
  }
182
+ timeoutTimer = setTimeout(() => {
183
+ const timeoutError = new S2Error({
184
+ message: `No tail received for ${TAIL_TIMEOUT_MS / 1000}s`,
185
+ status: 408, // Request Timeout
186
+ code: "TIMEOUT",
187
+ });
188
+ debug("tail timeout detected");
189
+ safeError(timeoutError);
190
+ }, TAIL_TIMEOUT_MS);
152
191
  };
153
192
  try {
193
+ // Start the timeout timer - will fire in 20s if no tail received
194
+ resetTimeoutTimer();
154
195
  const connection = await getConnection();
155
196
  // Build query string
156
197
  const queryParams = new URLSearchParams();
@@ -177,9 +218,11 @@ class S2SReadSession extends ReadableStream {
177
218
  ":path": path,
178
219
  ":scheme": url.protocol.slice(0, -1),
179
220
  ":authority": url.host,
221
+ "user-agent": DEFAULT_USER_AGENT,
180
222
  authorization: `Bearer ${Redacted.value(authToken)}`,
181
223
  accept: "application/protobuf",
182
224
  "content-type": "s2s/proto",
225
+ ...(basinName ? { "s2-basin": basinName } : {}),
183
226
  });
184
227
  http2Stream = stream;
185
228
  options?.signal?.addEventListener("abort", () => {
@@ -187,64 +230,139 @@ class S2SReadSession extends ReadableStream {
187
230
  stream.close();
188
231
  }
189
232
  });
233
+ stream.on("response", (headers) => {
234
+ // Cache the status.
235
+ // This informs whether we should attempt to parse s2s frames in the "data" handler.
236
+ responseCode = headers[":status"] ?? 500;
237
+ });
238
+ connection.on("goaway", (errorCode, lastStreamID, opaqueData) => {
239
+ debug("received GOAWAY from server");
240
+ });
241
+ stream.on("error", (err) => {
242
+ safeError(err);
243
+ });
190
244
  stream.on("data", (chunk) => {
191
- // Buffer already extends Uint8Array in Node.js, no need to convert
192
- parser.push(chunk);
193
- let frame = parser.parseFrame();
194
- while (frame) {
195
- if (frame.terminal) {
196
- if (frame.statusCode && frame.statusCode >= 400) {
197
- const errorText = textDecoder.decode(frame.body);
198
- try {
199
- const errorJson = JSON.parse(errorText);
200
- safeError(new S2Error({
201
- message: errorJson.message ?? "Unknown error",
202
- code: errorJson.code,
203
- status: frame.statusCode,
204
- }));
205
- }
206
- catch {
207
- safeError(new S2Error({
208
- message: errorText || "Unknown error",
209
- status: frame.statusCode,
210
- }));
211
- }
245
+ try {
246
+ if ((responseCode ?? 500) >= 400) {
247
+ const errorText = textDecoder.decode(chunk);
248
+ try {
249
+ const errorJson = JSON.parse(errorText);
250
+ safeError(new S2Error({
251
+ message: errorJson.message ?? "Unknown error",
252
+ code: errorJson.code,
253
+ status: responseCode,
254
+ origin: "server",
255
+ }));
212
256
  }
213
- else {
214
- safeClose();
257
+ catch {
258
+ safeError(new S2Error({
259
+ message: errorText || "Unknown error",
260
+ status: responseCode,
261
+ origin: "server",
262
+ }));
215
263
  }
216
- stream.close();
264
+ return;
217
265
  }
218
- else {
219
- // Parse ReadBatch
220
- try {
221
- const protoBatch = ProtoReadBatch.fromBinary(frame.body);
222
- // Update position from tail
223
- if (protoBatch.tail) {
224
- lastReadPosition = convertStreamPosition(protoBatch.tail);
225
- // Assign to instance property
226
- this._lastReadPosition = lastReadPosition;
266
+ // Buffer already extends Uint8Array in Node.js, no need to convert
267
+ parser.push(chunk);
268
+ let frame = parser.parseFrame();
269
+ while (frame) {
270
+ if (frame.terminal) {
271
+ if (frame.statusCode && frame.statusCode >= 400) {
272
+ const errorText = textDecoder.decode(frame.body);
273
+ try {
274
+ const errorJson = JSON.parse(errorText);
275
+ const status = frame.statusCode ?? 500;
276
+ // Map known read errors
277
+ if (status === 416) {
278
+ safeError(new RangeNotSatisfiableError({ status }));
279
+ }
280
+ else {
281
+ safeError(makeServerError({ status, statusText: undefined }, errorJson));
282
+ }
283
+ }
284
+ catch {
285
+ safeError(makeServerError({
286
+ status: frame.statusCode ?? 500,
287
+ statusText: undefined,
288
+ }, errorText));
289
+ }
227
290
  }
228
- // Enqueue each record
229
- for (const record of protoBatch.records) {
230
- const converted = this.convertRecord(record, as ?? "string", textDecoder);
231
- controller.enqueue(converted);
291
+ else {
292
+ safeClose();
232
293
  }
294
+ stream.close();
233
295
  }
234
- catch (err) {
235
- safeError(new S2Error({
236
- message: `Failed to parse ReadBatch: ${err}`,
237
- }));
296
+ else {
297
+ // Parse ReadBatch
298
+ try {
299
+ const protoBatch = ProtoReadBatch.fromBinary(frame.body);
300
+ resetTimeoutTimer();
301
+ // Update tail from batch
302
+ if (protoBatch.tail) {
303
+ const tail = convertStreamPosition(protoBatch.tail);
304
+ lastReadPosition = tail;
305
+ this._lastReadPosition = tail;
306
+ this._lastObservedTail = tail;
307
+ debug("received tail");
308
+ }
309
+ // Enqueue each record and track next position
310
+ for (const record of protoBatch.records) {
311
+ const converted = this.convertRecord(record, as ?? "string", textDecoder);
312
+ controller.enqueue({ ok: true, value: converted });
313
+ // Update next read position to after this record
314
+ if (record.seqNum !== undefined) {
315
+ this._nextReadPosition = {
316
+ seq_num: Number(record.seqNum) + 1,
317
+ timestamp: Number(record.timestamp ?? 0n),
318
+ };
319
+ }
320
+ }
321
+ }
322
+ catch (err) {
323
+ safeError(new S2Error({
324
+ message: `Failed to parse ReadBatch: ${err}`,
325
+ status: 500,
326
+ origin: "sdk",
327
+ }));
328
+ }
238
329
  }
330
+ frame = parser.parseFrame();
239
331
  }
240
- frame = parser.parseFrame();
332
+ }
333
+ catch (error) {
334
+ safeError(error instanceof S2Error
335
+ ? error
336
+ : new S2Error({
337
+ message: `Failed to process read data: ${error}`,
338
+ status: 500,
339
+ origin: "sdk",
340
+ }));
241
341
  }
242
342
  });
243
- stream.on("error", (err) => {
244
- safeError(err);
343
+ stream.on("end", () => {
344
+ if (stream.rstCode != 0) {
345
+ debug("stream reset code=%d", stream.rstCode);
346
+ safeError(new S2Error({
347
+ message: `Stream ended with error: ${stream.rstCode}`,
348
+ status: 500,
349
+ code: "stream reset",
350
+ origin: "sdk",
351
+ }));
352
+ }
245
353
  });
246
354
  stream.on("close", () => {
247
- safeClose();
355
+ if (parser.hasData()) {
356
+ safeError(new S2Error({
357
+ message: "Stream closed with unparsed data remaining",
358
+ status: 500,
359
+ code: "STREAM_CLOSED_PREMATURELY",
360
+ origin: "sdk",
361
+ }));
362
+ }
363
+ else {
364
+ safeClose();
365
+ }
248
366
  });
249
367
  }
250
368
  catch (err) {
@@ -263,6 +381,7 @@ class S2SReadSession extends ReadableStream {
263
381
  this.url = url;
264
382
  this.options = options;
265
383
  this.getConnection = getConnection;
384
+ this.basinName = basinName;
266
385
  // Assign parser to instance property after super() completes
267
386
  this.parser = parser;
268
387
  this.http2Stream = http2Stream;
@@ -325,163 +444,46 @@ class S2SReadSession extends ReadableStream {
325
444
  },
326
445
  };
327
446
  }
328
- lastReadPosition() {
329
- return this._lastReadPosition;
447
+ nextReadPosition() {
448
+ return this._nextReadPosition;
449
+ }
450
+ lastObservedTail() {
451
+ return this._lastObservedTail;
330
452
  }
331
453
  }
332
454
  /**
333
455
  * AcksStream for S2S append session
334
456
  */
335
- class S2SAcksStream extends ReadableStream {
336
- constructor(setController) {
337
- super({
338
- start: (controller) => {
339
- setController(controller);
340
- },
341
- });
342
- }
343
- async [Symbol.asyncDispose]() {
344
- await this.cancel("disposed");
345
- }
346
- // Polyfill for older browsers
347
- [Symbol.asyncIterator]() {
348
- const fn = ReadableStream.prototype[Symbol.asyncIterator];
349
- if (typeof fn === "function")
350
- return fn.call(this);
351
- const reader = this.getReader();
352
- return {
353
- next: async () => {
354
- const r = await reader.read();
355
- if (r.done) {
356
- reader.releaseLock();
357
- return { done: true, value: undefined };
358
- }
359
- return { done: false, value: r.value };
360
- },
361
- throw: async (e) => {
362
- await reader.cancel(e);
363
- reader.releaseLock();
364
- return { done: true, value: undefined };
365
- },
366
- return: async () => {
367
- await reader.cancel("done");
368
- reader.releaseLock();
369
- return { done: true, value: undefined };
370
- },
371
- [Symbol.asyncIterator]() {
372
- return this;
373
- },
374
- };
375
- }
376
- }
457
+ // Removed S2SAcksStream - transport sessions no longer expose streams
377
458
  /**
378
- * S2S Append Session for pipelined writes
379
- * Unlike fetch-based append, writes don't block on acks - only on submission
459
+ * Fetch-based transport session for appending records via HTTP/2.
460
+ * Pipelined: multiple requests can be in-flight simultaneously.
461
+ * No backpressure, no retry logic, no streams - just submit/close with value-encoded errors.
380
462
  */
381
463
  class S2SAppendSession {
382
464
  baseUrl;
383
465
  authToken;
384
466
  streamName;
385
467
  getConnection;
468
+ basinName;
386
469
  options;
387
470
  http2Stream;
388
- _lastAckedPosition;
389
471
  parser = new S2SFrameParser();
390
- acksController;
391
- _readable;
392
- _writable;
393
472
  closed = false;
394
- queuedBytes = 0;
395
- maxQueuedBytes;
396
- waitingForCapacity = [];
397
473
  pendingAcks = [];
398
474
  initPromise;
399
- readable;
400
- writable;
401
- static async create(baseUrl, bearerToken, streamName, getConnection, sessionOptions, requestOptions) {
402
- return new S2SAppendSession(baseUrl, bearerToken, streamName, getConnection, sessionOptions, requestOptions);
475
+ static async create(baseUrl, bearerToken, streamName, getConnection, basinName, sessionOptions, requestOptions) {
476
+ return new S2SAppendSession(baseUrl, bearerToken, streamName, getConnection, basinName, sessionOptions, requestOptions);
403
477
  }
404
- constructor(baseUrl, authToken, streamName, getConnection, sessionOptions, options) {
478
+ constructor(baseUrl, authToken, streamName, getConnection, basinName, sessionOptions, options) {
405
479
  this.baseUrl = baseUrl;
406
480
  this.authToken = authToken;
407
481
  this.streamName = streamName;
408
482
  this.getConnection = getConnection;
483
+ this.basinName = basinName;
409
484
  this.options = options;
410
- this.maxQueuedBytes = sessionOptions?.maxQueuedBytes ?? 10 * 1024 * 1024; // 10 MiB default
411
- // Create the readable stream for acks
412
- this._readable = new S2SAcksStream((controller) => {
413
- this.acksController = controller;
414
- });
415
- this.readable = this._readable;
416
- // Create the writable stream
417
- this._writable = new WritableStream({
418
- start: async (controller) => {
419
- this.initPromise = this.initializeStream();
420
- await this.initPromise;
421
- },
422
- write: async (chunk) => {
423
- if (this.closed) {
424
- throw new S2Error({ message: "AppendSession is closed" });
425
- }
426
- const recordsArray = Array.isArray(chunk.records)
427
- ? chunk.records
428
- : [chunk.records];
429
- // Validate batch size limits
430
- if (recordsArray.length > 1000) {
431
- throw new S2Error({
432
- message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`,
433
- });
434
- }
435
- // Calculate metered size
436
- let batchMeteredSize = 0;
437
- for (const record of recordsArray) {
438
- batchMeteredSize += meteredSizeBytes(record);
439
- }
440
- if (batchMeteredSize > 1024 * 1024) {
441
- throw new S2Error({
442
- message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`,
443
- });
444
- }
445
- // Wait for capacity if needed (backpressure)
446
- while (this.queuedBytes + batchMeteredSize > this.maxQueuedBytes &&
447
- !this.closed) {
448
- await new Promise((resolve) => {
449
- this.waitingForCapacity.push(resolve);
450
- });
451
- }
452
- if (this.closed) {
453
- throw new S2Error({ message: "AppendSession is closed" });
454
- }
455
- // Send the batch immediately (pipelined)
456
- // Returns when frame is sent, not when ack is received
457
- await this.sendBatchNonBlocking(recordsArray, chunk, batchMeteredSize);
458
- },
459
- close: async () => {
460
- this.closed = true;
461
- await this.closeStream();
462
- },
463
- abort: async (reason) => {
464
- this.closed = true;
465
- this.queuedBytes = 0;
466
- // Reject all pending acks
467
- const error = new S2Error({
468
- message: `AppendSession was aborted: ${reason}`,
469
- });
470
- for (const pending of this.pendingAcks) {
471
- pending.reject(error);
472
- }
473
- this.pendingAcks = [];
474
- // Wake up all waiting for capacity
475
- for (const resolver of this.waitingForCapacity) {
476
- resolver();
477
- }
478
- this.waitingForCapacity = [];
479
- if (this.http2Stream && !this.http2Stream.closed) {
480
- this.http2Stream.close();
481
- }
482
- },
483
- });
484
- this.writable = this._writable;
485
+ // No stream setup
486
+ // Initialization happens lazily on first submit
485
487
  }
486
488
  async initializeStream() {
487
489
  const url = new URL(this.baseUrl);
@@ -492,9 +494,11 @@ class S2SAppendSession {
492
494
  ":path": path,
493
495
  ":scheme": url.protocol.slice(0, -1),
494
496
  ":authority": url.host,
497
+ "user-agent": DEFAULT_USER_AGENT,
495
498
  authorization: `Bearer ${Redacted.value(this.authToken)}`,
496
499
  "content-type": "s2s/proto",
497
500
  accept: "application/protobuf",
501
+ ...(this.basinName ? { "s2-basin": this.basinName } : {}),
498
502
  });
499
503
  this.http2Stream = stream;
500
504
  this.options?.signal?.addEventListener("abort", () => {
@@ -503,145 +507,87 @@ class S2SAppendSession {
503
507
  }
504
508
  });
505
509
  const textDecoder = new TextDecoder();
506
- let controllerClosed = false;
507
- const safeClose = () => {
508
- if (!controllerClosed && this.acksController) {
509
- controllerClosed = true;
510
- try {
511
- this.acksController.close();
512
- }
513
- catch {
514
- // Controller may already be closed, ignore
515
- }
516
- }
517
- };
518
- const safeError = (err) => {
519
- if (!controllerClosed && this.acksController) {
520
- controllerClosed = true;
521
- this.acksController.error(err);
522
- }
523
- // Reject all pending acks
510
+ const safeError = (error) => {
511
+ const s2Err = error instanceof S2Error
512
+ ? error
513
+ : new S2Error({ message: String(error), status: 502 });
514
+ // Resolve all pending acks with error result
524
515
  for (const pending of this.pendingAcks) {
525
- pending.reject(err);
516
+ pending.resolve(err(s2Err));
526
517
  }
527
518
  this.pendingAcks = [];
528
519
  };
529
520
  // Handle incoming data (acks)
530
521
  stream.on("data", (chunk) => {
531
- this.parser.push(chunk);
532
- let frame = this.parser.parseFrame();
533
- while (frame) {
534
- if (frame.terminal) {
535
- if (frame.statusCode && frame.statusCode >= 400) {
536
- const errorText = textDecoder.decode(frame.body);
537
- try {
538
- const errorJson = JSON.parse(errorText);
539
- safeError(new S2Error({
540
- message: errorJson.message ?? "Unknown error",
541
- code: errorJson.code,
542
- status: frame.statusCode,
543
- }));
544
- }
545
- catch {
546
- safeError(new S2Error({
547
- message: errorText || "Unknown error",
548
- status: frame.statusCode,
549
- }));
522
+ try {
523
+ this.parser.push(chunk);
524
+ let frame = this.parser.parseFrame();
525
+ while (frame) {
526
+ if (frame.terminal) {
527
+ if (frame.statusCode && frame.statusCode >= 400) {
528
+ const errorText = textDecoder.decode(frame.body);
529
+ const status = frame.statusCode ?? 500;
530
+ try {
531
+ const errorJson = JSON.parse(errorText);
532
+ const err = status === 412
533
+ ? makeAppendPreconditionError(status, errorJson)
534
+ : makeServerError({ status, statusText: undefined }, errorJson);
535
+ queueMicrotask(() => safeError(err));
536
+ }
537
+ catch {
538
+ const err = makeServerError({ status, statusText: undefined }, errorText);
539
+ queueMicrotask(() => safeError(err));
540
+ }
550
541
  }
542
+ stream.close();
551
543
  }
552
544
  else {
553
- safeClose();
554
- }
555
- stream.close();
556
- }
557
- else {
558
- // Parse AppendAck
559
- try {
560
- const protoAck = ProtoAppendAck.fromBinary(frame.body);
561
- const ack = convertAppendAck(protoAck);
562
- this._lastAckedPosition = ack;
563
- // Enqueue to readable stream
564
- if (this.acksController) {
565
- this.acksController.enqueue(ack);
566
- }
567
- // Resolve the pending ack promise
568
- const pending = this.pendingAcks.shift();
569
- if (pending) {
570
- pending.resolve(ack);
571
- // Release capacity
572
- this.queuedBytes -= pending.batchSize;
573
- // Wake up one waiting writer
574
- if (this.waitingForCapacity.length > 0) {
575
- const waiter = this.waitingForCapacity.shift();
576
- waiter();
545
+ // Parse AppendAck
546
+ try {
547
+ const protoAck = ProtoAppendAck.fromBinary(frame.body);
548
+ const ack = convertAppendAck(protoAck);
549
+ // Resolve the pending ack promise (FIFO)
550
+ const pending = this.pendingAcks.shift();
551
+ if (pending) {
552
+ pending.resolve(ok(ack));
577
553
  }
578
554
  }
555
+ catch (parseErr) {
556
+ queueMicrotask(() => safeError(new S2Error({
557
+ message: `Failed to parse AppendAck: ${parseErr}`,
558
+ status: 500,
559
+ })));
560
+ }
579
561
  }
580
- catch (err) {
581
- safeError(new S2Error({
582
- message: `Failed to parse AppendAck: ${err}`,
583
- }));
584
- }
562
+ frame = this.parser.parseFrame();
585
563
  }
586
- frame = this.parser.parseFrame();
564
+ }
565
+ catch (error) {
566
+ queueMicrotask(() => safeError(error));
587
567
  }
588
568
  });
589
- stream.on("error", (err) => {
590
- safeError(err);
569
+ stream.on("error", (streamErr) => {
570
+ queueMicrotask(() => safeError(streamErr));
591
571
  });
592
572
  stream.on("close", () => {
593
- safeClose();
594
- });
595
- }
596
- /**
597
- * Send a batch non-blocking (returns when frame is sent, not when ack is received)
598
- */
599
- sendBatchNonBlocking(records, args, batchMeteredSize) {
600
- if (!this.http2Stream || this.http2Stream.closed) {
601
- return Promise.reject(new S2Error({ message: "HTTP/2 stream is not open" }));
602
- }
603
- // Convert to protobuf AppendInput
604
- const protoInput = buildProtoAppendInput(records, args);
605
- const bodyBytes = ProtoAppendInput.toBinary(protoInput);
606
- // Frame the message
607
- const frame = frameMessage({
608
- terminal: false,
609
- body: bodyBytes,
610
- });
611
- // This promise resolves when the frame is written (not when ack is received)
612
- return new Promise((resolve, reject) => {
613
- // Track pending ack - will be resolved when ack arrives
614
- const ackPromise = {
615
- resolve: () => { },
616
- reject,
617
- batchSize: batchMeteredSize,
618
- };
619
- this.pendingAcks.push(ackPromise);
620
- this.queuedBytes += batchMeteredSize;
621
- // Send the frame (pipelined)
622
- this.http2Stream.write(frame, (err) => {
623
- if (err) {
624
- // Remove from pending acks on write error
625
- const idx = this.pendingAcks.indexOf(ackPromise);
626
- if (idx !== -1) {
627
- this.pendingAcks.splice(idx, 1);
628
- this.queuedBytes -= batchMeteredSize;
629
- }
630
- reject(err);
631
- }
632
- else {
633
- // Frame written successfully - resolve immediately (pipelined)
634
- resolve();
635
- }
636
- });
573
+ // Stream closed - resolve any remaining pending acks with error
574
+ // This can happen if the server closes the stream without sending all acks
575
+ if (this.pendingAcks.length > 0) {
576
+ queueMicrotask(() => safeError(new S2Error({
577
+ message: "Stream closed with pending acks",
578
+ status: 502,
579
+ code: "BAD_GATEWAY",
580
+ })));
581
+ }
637
582
  });
638
583
  }
639
584
  /**
640
- * Send a batch and wait for ack (used by submit method)
585
+ * Send a batch and wait for ack. Returns AppendResult (never throws).
586
+ * Pipelined: multiple sends can be in-flight; acks resolve FIFO.
641
587
  */
642
588
  sendBatch(records, args, batchMeteredSize) {
643
589
  if (!this.http2Stream || this.http2Stream.closed) {
644
- return Promise.reject(new S2Error({ message: "HTTP/2 stream is not open" }));
590
+ return Promise.resolve(err(new S2Error({ message: "HTTP/2 stream is not open", status: 502 })));
645
591
  }
646
592
  // Convert to protobuf AppendInput
647
593
  const protoInput = buildProtoAppendInput(records, args);
@@ -651,82 +597,99 @@ class S2SAppendSession {
651
597
  terminal: false,
652
598
  body: bodyBytes,
653
599
  });
654
- // Track pending ack - this promise resolves when the ack is received
655
- return new Promise((resolve, reject) => {
600
+ // Track pending ack - this promise resolves when the ack is received (FIFO)
601
+ return new Promise((resolve) => {
656
602
  this.pendingAcks.push({
657
603
  resolve,
658
- reject,
659
604
  batchSize: batchMeteredSize,
660
605
  });
661
- this.queuedBytes += batchMeteredSize;
662
- // Send the frame (non-blocking - pipelined)
663
- this.http2Stream.write(frame, (err) => {
664
- if (err) {
606
+ // Send the frame (pipelined - non-blocking)
607
+ this.http2Stream.write(frame, (writeErr) => {
608
+ if (writeErr) {
665
609
  // Remove from pending acks on write error
666
- const idx = this.pendingAcks.findIndex((p) => p.reject === reject);
610
+ const idx = this.pendingAcks.findIndex((p) => p.resolve === resolve);
667
611
  if (idx !== -1) {
668
612
  this.pendingAcks.splice(idx, 1);
669
- this.queuedBytes -= batchMeteredSize;
670
613
  }
671
- reject(err);
614
+ // Resolve with error result
615
+ const s2Err = writeErr instanceof S2Error
616
+ ? writeErr
617
+ : new S2Error({ message: String(writeErr), status: 502 });
618
+ resolve(err(s2Err));
672
619
  }
673
- // Write completed, but promise resolves when ack is received
620
+ // Write completed successfully - promise resolves later when ack is received
674
621
  });
675
622
  });
676
623
  }
677
- async closeStream() {
678
- // Wait for all pending acks
679
- while (this.pendingAcks.length > 0) {
680
- await new Promise((resolve) => setTimeout(resolve, 10));
681
- }
682
- // Close the HTTP/2 stream (client doesn't send terminal frame for clean close)
683
- if (this.http2Stream && !this.http2Stream.closed) {
684
- this.http2Stream.end();
685
- }
686
- }
687
- async [Symbol.asyncDispose]() {
688
- await this.close();
689
- }
690
- /**
691
- * Get a stream of acknowledgements for appends.
692
- */
693
- acks() {
694
- return this._readable;
695
- }
696
624
  /**
697
625
  * Close the append session.
698
626
  * Waits for all pending appends to complete before resolving.
627
+ * Never throws - returns CloseResult.
699
628
  */
700
629
  async close() {
701
- await this.writable.close();
630
+ try {
631
+ this.closed = true;
632
+ // Wait for all pending acks to complete
633
+ while (this.pendingAcks.length > 0) {
634
+ await new Promise((resolve) => setTimeout(resolve, 10));
635
+ }
636
+ // Close the HTTP/2 stream (client doesn't send terminal frame for clean close)
637
+ if (this.http2Stream && !this.http2Stream.closed) {
638
+ this.http2Stream.end();
639
+ }
640
+ return okClose();
641
+ }
642
+ catch (error) {
643
+ const s2Err = error instanceof S2Error
644
+ ? error
645
+ : new S2Error({ message: String(error), status: 500 });
646
+ return errClose(s2Err);
647
+ }
702
648
  }
703
649
  /**
704
650
  * Submit an append request to the session.
705
- * Returns a promise that resolves with the ack when received.
651
+ * Returns AppendResult (never throws).
652
+ * Pipelined: multiple submits can be in-flight; acks resolve FIFO.
706
653
  */
707
654
  async submit(records, args) {
655
+ // Validate closed state
708
656
  if (this.closed) {
709
- return Promise.reject(new S2Error({ message: "AppendSession is closed" }));
657
+ return err(new S2Error({ message: "AppendSession is closed", status: 400 }));
710
658
  }
711
- // Wait for initialization
712
- if (this.initPromise) {
659
+ // Lazy initialize HTTP/2 stream on first submit
660
+ if (!this.initPromise) {
661
+ this.initPromise = this.initializeStream();
662
+ }
663
+ try {
713
664
  await this.initPromise;
714
665
  }
666
+ catch (initErr) {
667
+ const s2Err = initErr instanceof S2Error
668
+ ? initErr
669
+ : new S2Error({ message: String(initErr), status: 502 });
670
+ return err(s2Err);
671
+ }
715
672
  const recordsArray = Array.isArray(records) ? records : [records];
716
- // Validate batch size limits
673
+ // Validate batch size limits (non-retryable 400-level error)
717
674
  if (recordsArray.length > 1000) {
718
- return Promise.reject(new S2Error({
675
+ return err(new S2Error({
719
676
  message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`,
677
+ status: 400,
678
+ code: "INVALID_ARGUMENT",
720
679
  }));
721
680
  }
722
- // Calculate metered size
723
- let batchMeteredSize = 0;
724
- for (const record of recordsArray) {
725
- batchMeteredSize += meteredSizeBytes(record);
681
+ // Calculate metered size (use precalculated if provided)
682
+ let batchMeteredSize = args?.precalculatedSize ?? 0;
683
+ if (batchMeteredSize === 0) {
684
+ for (const record of recordsArray) {
685
+ batchMeteredSize += meteredBytes(record);
686
+ }
726
687
  }
727
688
  if (batchMeteredSize > 1024 * 1024) {
728
- return Promise.reject(new S2Error({
689
+ return err(new S2Error({
729
690
  message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`,
691
+ status: 400,
692
+ code: "INVALID_ARGUMENT",
730
693
  }));
731
694
  }
732
695
  return this.sendBatch(recordsArray, {
@@ -735,9 +698,6 @@ class S2SAppendSession {
735
698
  match_seq_num: args?.match_seq_num,
736
699
  }, batchMeteredSize);
737
700
  }
738
- lastAckedPosition() {
739
- return this._lastAckedPosition;
740
- }
741
701
  }
742
702
  /**
743
703
  * Convert protobuf StreamPosition to OpenAPI StreamPosition