@s2-dev/streamstore 0.17.6 → 0.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -1
- package/dist/cjs/accessTokens.d.ts +3 -2
- package/dist/cjs/accessTokens.d.ts.map +1 -1
- package/dist/cjs/accessTokens.js +22 -37
- package/dist/cjs/accessTokens.js.map +1 -1
- package/dist/cjs/basin.d.ts +4 -3
- package/dist/cjs/basin.d.ts.map +1 -1
- package/dist/cjs/basin.js +16 -6
- package/dist/cjs/basin.js.map +1 -1
- package/dist/cjs/basins.d.ts +10 -10
- package/dist/cjs/basins.d.ts.map +1 -1
- package/dist/cjs/basins.js +36 -64
- package/dist/cjs/basins.js.map +1 -1
- package/dist/cjs/batch-transform.d.ts +1 -1
- package/dist/cjs/batch-transform.d.ts.map +1 -1
- package/dist/cjs/batch-transform.js +36 -5
- package/dist/cjs/batch-transform.js.map +1 -1
- package/dist/cjs/common.d.ts +42 -0
- package/dist/cjs/common.d.ts.map +1 -1
- package/dist/cjs/error.d.ts +40 -2
- package/dist/cjs/error.d.ts.map +1 -1
- package/dist/cjs/error.js +268 -2
- package/dist/cjs/error.js.map +1 -1
- package/dist/cjs/generated/client/types.gen.d.ts +7 -0
- package/dist/cjs/generated/client/types.gen.d.ts.map +1 -1
- package/dist/cjs/generated/client/utils.gen.d.ts +1 -0
- package/dist/cjs/generated/client/utils.gen.d.ts.map +1 -1
- package/dist/cjs/generated/client/utils.gen.js.map +1 -1
- package/dist/cjs/generated/core/types.gen.d.ts +2 -0
- package/dist/cjs/generated/core/types.gen.d.ts.map +1 -1
- package/dist/cjs/index.d.ts +46 -3
- package/dist/cjs/index.d.ts.map +1 -1
- package/dist/cjs/index.js +28 -2
- package/dist/cjs/index.js.map +1 -1
- package/dist/cjs/lib/result.d.ts +57 -0
- package/dist/cjs/lib/result.d.ts.map +1 -0
- package/dist/cjs/lib/result.js +43 -0
- package/dist/cjs/lib/result.js.map +1 -0
- package/dist/cjs/lib/retry.d.ts +151 -0
- package/dist/cjs/lib/retry.d.ts.map +1 -0
- package/dist/cjs/lib/retry.js +839 -0
- package/dist/cjs/lib/retry.js.map +1 -0
- package/dist/cjs/lib/stream/factory.d.ts +0 -1
- package/dist/cjs/lib/stream/factory.d.ts.map +1 -1
- package/dist/cjs/lib/stream/factory.js +0 -1
- package/dist/cjs/lib/stream/factory.js.map +1 -1
- package/dist/cjs/lib/stream/runtime.d.ts +14 -0
- package/dist/cjs/lib/stream/runtime.d.ts.map +1 -1
- package/dist/cjs/lib/stream/runtime.js +18 -3
- package/dist/cjs/lib/stream/runtime.js.map +1 -1
- package/dist/cjs/lib/stream/transport/fetch/index.d.ts +24 -32
- package/dist/cjs/lib/stream/transport/fetch/index.d.ts.map +1 -1
- package/dist/cjs/lib/stream/transport/fetch/index.js +260 -187
- package/dist/cjs/lib/stream/transport/fetch/index.js.map +1 -1
- package/dist/cjs/lib/stream/transport/fetch/shared.d.ts +1 -2
- package/dist/cjs/lib/stream/transport/fetch/shared.d.ts.map +1 -1
- package/dist/cjs/lib/stream/transport/fetch/shared.js +49 -72
- package/dist/cjs/lib/stream/transport/fetch/shared.js.map +1 -1
- package/dist/cjs/lib/stream/transport/s2s/index.d.ts +0 -1
- package/dist/cjs/lib/stream/transport/s2s/index.d.ts.map +1 -1
- package/dist/cjs/lib/stream/transport/s2s/index.js +312 -352
- package/dist/cjs/lib/stream/transport/s2s/index.js.map +1 -1
- package/dist/cjs/lib/stream/types.d.ts +102 -8
- package/dist/cjs/lib/stream/types.d.ts.map +1 -1
- package/dist/cjs/metrics.d.ts +3 -2
- package/dist/cjs/metrics.d.ts.map +1 -1
- package/dist/cjs/metrics.js +24 -39
- package/dist/cjs/metrics.js.map +1 -1
- package/dist/cjs/s2.d.ts +1 -0
- package/dist/cjs/s2.d.ts.map +1 -1
- package/dist/cjs/s2.js +20 -3
- package/dist/cjs/s2.js.map +1 -1
- package/dist/cjs/stream.d.ts +5 -3
- package/dist/cjs/stream.d.ts.map +1 -1
- package/dist/cjs/stream.js +29 -18
- package/dist/cjs/stream.js.map +1 -1
- package/dist/cjs/streams.d.ts +10 -10
- package/dist/cjs/streams.d.ts.map +1 -1
- package/dist/cjs/streams.js +36 -64
- package/dist/cjs/streams.js.map +1 -1
- package/dist/cjs/utils.d.ts +3 -3
- package/dist/cjs/utils.d.ts.map +1 -1
- package/dist/cjs/utils.js +3 -3
- package/dist/cjs/utils.js.map +1 -1
- package/dist/cjs/version.d.ts +8 -0
- package/dist/cjs/version.d.ts.map +1 -0
- package/dist/cjs/version.js +11 -0
- package/dist/cjs/version.js.map +1 -0
- package/dist/esm/accessTokens.d.ts +3 -2
- package/dist/esm/accessTokens.d.ts.map +1 -1
- package/dist/esm/accessTokens.js +23 -38
- package/dist/esm/accessTokens.js.map +1 -1
- package/dist/esm/basin.d.ts +4 -3
- package/dist/esm/basin.d.ts.map +1 -1
- package/dist/esm/basin.js +16 -6
- package/dist/esm/basin.js.map +1 -1
- package/dist/esm/basins.d.ts +10 -10
- package/dist/esm/basins.d.ts.map +1 -1
- package/dist/esm/basins.js +37 -65
- package/dist/esm/basins.js.map +1 -1
- package/dist/esm/batch-transform.d.ts +1 -1
- package/dist/esm/batch-transform.d.ts.map +1 -1
- package/dist/esm/batch-transform.js +37 -6
- package/dist/esm/batch-transform.js.map +1 -1
- package/dist/esm/common.d.ts +42 -0
- package/dist/esm/common.d.ts.map +1 -1
- package/dist/esm/error.d.ts +40 -2
- package/dist/esm/error.d.ts.map +1 -1
- package/dist/esm/error.js +260 -2
- package/dist/esm/error.js.map +1 -1
- package/dist/esm/generated/client/types.gen.d.ts +7 -0
- package/dist/esm/generated/client/types.gen.d.ts.map +1 -1
- package/dist/esm/generated/client/utils.gen.d.ts +1 -0
- package/dist/esm/generated/client/utils.gen.d.ts.map +1 -1
- package/dist/esm/generated/client/utils.gen.js.map +1 -1
- package/dist/esm/generated/core/types.gen.d.ts +2 -0
- package/dist/esm/generated/core/types.gen.d.ts.map +1 -1
- package/dist/esm/index.d.ts +46 -3
- package/dist/esm/index.d.ts.map +1 -1
- package/dist/esm/index.js +23 -1
- package/dist/esm/index.js.map +1 -1
- package/dist/esm/lib/result.d.ts +57 -0
- package/dist/esm/lib/result.d.ts.map +1 -0
- package/dist/esm/lib/result.js +37 -0
- package/dist/esm/lib/result.js.map +1 -0
- package/dist/esm/lib/retry.d.ts +151 -0
- package/dist/esm/lib/retry.d.ts.map +1 -0
- package/dist/esm/lib/retry.js +830 -0
- package/dist/esm/lib/retry.js.map +1 -0
- package/dist/esm/lib/stream/factory.d.ts +0 -1
- package/dist/esm/lib/stream/factory.d.ts.map +1 -1
- package/dist/esm/lib/stream/factory.js +0 -1
- package/dist/esm/lib/stream/factory.js.map +1 -1
- package/dist/esm/lib/stream/runtime.d.ts +14 -0
- package/dist/esm/lib/stream/runtime.d.ts.map +1 -1
- package/dist/esm/lib/stream/runtime.js +23 -3
- package/dist/esm/lib/stream/runtime.js.map +1 -1
- package/dist/esm/lib/stream/transport/fetch/index.d.ts +24 -32
- package/dist/esm/lib/stream/transport/fetch/index.d.ts.map +1 -1
- package/dist/esm/lib/stream/transport/fetch/index.js +260 -187
- package/dist/esm/lib/stream/transport/fetch/index.js.map +1 -1
- package/dist/esm/lib/stream/transport/fetch/shared.d.ts +1 -2
- package/dist/esm/lib/stream/transport/fetch/shared.d.ts.map +1 -1
- package/dist/esm/lib/stream/transport/fetch/shared.js +51 -74
- package/dist/esm/lib/stream/transport/fetch/shared.js.map +1 -1
- package/dist/esm/lib/stream/transport/s2s/index.d.ts +0 -1
- package/dist/esm/lib/stream/transport/s2s/index.d.ts.map +1 -1
- package/dist/esm/lib/stream/transport/s2s/index.js +313 -353
- package/dist/esm/lib/stream/transport/s2s/index.js.map +1 -1
- package/dist/esm/lib/stream/types.d.ts +102 -8
- package/dist/esm/lib/stream/types.d.ts.map +1 -1
- package/dist/esm/metrics.d.ts +3 -2
- package/dist/esm/metrics.d.ts.map +1 -1
- package/dist/esm/metrics.js +25 -40
- package/dist/esm/metrics.js.map +1 -1
- package/dist/esm/s2.d.ts +1 -0
- package/dist/esm/s2.d.ts.map +1 -1
- package/dist/esm/s2.js +20 -3
- package/dist/esm/s2.js.map +1 -1
- package/dist/esm/stream.d.ts +5 -3
- package/dist/esm/stream.d.ts.map +1 -1
- package/dist/esm/stream.js +30 -19
- package/dist/esm/stream.js.map +1 -1
- package/dist/esm/streams.d.ts +10 -10
- package/dist/esm/streams.d.ts.map +1 -1
- package/dist/esm/streams.js +37 -65
- package/dist/esm/streams.js.map +1 -1
- package/dist/esm/utils.d.ts +3 -3
- package/dist/esm/utils.d.ts.map +1 -1
- package/dist/esm/utils.js +2 -2
- package/dist/esm/utils.js.map +1 -1
- package/dist/esm/version.d.ts +8 -0
- package/dist/esm/version.d.ts.map +1 -0
- package/dist/esm/version.js +8 -0
- package/dist/esm/version.js.map +1 -0
- package/package.json +7 -4
|
@@ -5,12 +5,16 @@
|
|
|
5
5
|
* This file should only be imported in Node.js environments
|
|
6
6
|
*/
|
|
7
7
|
import * as http2 from "node:http2";
|
|
8
|
-
import
|
|
8
|
+
import createDebug from "debug";
|
|
9
|
+
import { makeAppendPreconditionError, makeServerError, RangeNotSatisfiableError, S2Error, } from "../../../../error.js";
|
|
9
10
|
import { AppendAck as ProtoAppendAck, AppendInput as ProtoAppendInput, ReadBatch as ProtoReadBatch, } from "../../../../generated/proto/s2.js";
|
|
10
|
-
import {
|
|
11
|
-
import { meteredSizeBytes } from "../../../../utils.js";
|
|
11
|
+
import { meteredBytes } from "../../../../utils.js";
|
|
12
12
|
import * as Redacted from "../../../redacted.js";
|
|
13
|
+
import { err, errClose, ok, okClose } from "../../../result.js";
|
|
14
|
+
import { RetryAppendSession as AppendSessionImpl, RetryReadSession as ReadSessionImpl, } from "../../../retry.js";
|
|
15
|
+
import { DEFAULT_USER_AGENT } from "../../runtime.js";
|
|
13
16
|
import { frameMessage, S2SFrameParser } from "./framing.js";
|
|
17
|
+
const debug = createDebug("s2:s2s");
|
|
14
18
|
export function buildProtoAppendInput(records, args) {
|
|
15
19
|
const textEncoder = new TextEncoder();
|
|
16
20
|
return ProtoAppendInput.create({
|
|
@@ -40,22 +44,21 @@ export function buildProtoAppendInput(records, args) {
|
|
|
40
44
|
});
|
|
41
45
|
}
|
|
42
46
|
export class S2STransport {
|
|
43
|
-
client;
|
|
44
47
|
transportConfig;
|
|
45
48
|
connection;
|
|
46
49
|
connectionPromise;
|
|
47
50
|
constructor(config) {
|
|
48
|
-
this.client = createClient(createConfig({
|
|
49
|
-
baseUrl: config.baseUrl,
|
|
50
|
-
auth: () => Redacted.value(config.accessToken),
|
|
51
|
-
}));
|
|
52
51
|
this.transportConfig = config;
|
|
53
52
|
}
|
|
54
53
|
async makeAppendSession(stream, sessionOptions, requestOptions) {
|
|
55
|
-
return
|
|
54
|
+
return AppendSessionImpl.create((myOptions) => {
|
|
55
|
+
return S2SAppendSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, () => this.getConnection(), this.transportConfig.basinName, myOptions, requestOptions);
|
|
56
|
+
}, sessionOptions, this.transportConfig.retry);
|
|
56
57
|
}
|
|
57
58
|
async makeReadSession(stream, args, options) {
|
|
58
|
-
return
|
|
59
|
+
return ReadSessionImpl.create((myArgs) => {
|
|
60
|
+
return S2SReadSession.create(this.transportConfig.baseUrl, this.transportConfig.accessToken, stream, myArgs, options, () => this.getConnection(), this.transportConfig.basinName);
|
|
61
|
+
}, args, this.transportConfig.retry);
|
|
59
62
|
}
|
|
60
63
|
/**
|
|
61
64
|
* Get or create HTTP/2 connection (one per transport)
|
|
@@ -117,25 +120,36 @@ class S2SReadSession extends ReadableStream {
|
|
|
117
120
|
url;
|
|
118
121
|
options;
|
|
119
122
|
getConnection;
|
|
123
|
+
basinName;
|
|
120
124
|
http2Stream;
|
|
121
125
|
_lastReadPosition;
|
|
126
|
+
_nextReadPosition;
|
|
127
|
+
_lastObservedTail;
|
|
122
128
|
parser = new S2SFrameParser();
|
|
123
|
-
static async create(baseUrl, bearerToken, streamName, args, options, getConnection) {
|
|
129
|
+
static async create(baseUrl, bearerToken, streamName, args, options, getConnection, basinName) {
|
|
124
130
|
const url = new URL(baseUrl);
|
|
125
|
-
return new S2SReadSession(streamName, args, bearerToken, url, options, getConnection);
|
|
131
|
+
return new S2SReadSession(streamName, args, bearerToken, url, options, getConnection, basinName);
|
|
126
132
|
}
|
|
127
|
-
constructor(streamName, args, authToken, url, options, getConnection) {
|
|
133
|
+
constructor(streamName, args, authToken, url, options, getConnection, basinName) {
|
|
128
134
|
// Initialize parser and textDecoder before super() call
|
|
129
135
|
const parser = new S2SFrameParser();
|
|
130
136
|
const textDecoder = new TextDecoder();
|
|
131
137
|
let http2Stream;
|
|
132
138
|
let lastReadPosition;
|
|
139
|
+
// Track timeout for detecting when server stops sending data
|
|
140
|
+
const TAIL_TIMEOUT_MS = 20000; // 20 seconds
|
|
141
|
+
let timeoutTimer;
|
|
133
142
|
super({
|
|
134
143
|
start: async (controller) => {
|
|
135
144
|
let controllerClosed = false;
|
|
145
|
+
let responseCode;
|
|
136
146
|
const safeClose = () => {
|
|
137
147
|
if (!controllerClosed) {
|
|
138
148
|
controllerClosed = true;
|
|
149
|
+
if (timeoutTimer) {
|
|
150
|
+
clearTimeout(timeoutTimer);
|
|
151
|
+
timeoutTimer = undefined;
|
|
152
|
+
}
|
|
139
153
|
try {
|
|
140
154
|
controller.close();
|
|
141
155
|
}
|
|
@@ -147,10 +161,37 @@ class S2SReadSession extends ReadableStream {
|
|
|
147
161
|
const safeError = (err) => {
|
|
148
162
|
if (!controllerClosed) {
|
|
149
163
|
controllerClosed = true;
|
|
150
|
-
|
|
164
|
+
if (timeoutTimer) {
|
|
165
|
+
clearTimeout(timeoutTimer);
|
|
166
|
+
timeoutTimer = undefined;
|
|
167
|
+
}
|
|
168
|
+
// Convert error to S2Error and enqueue as error result
|
|
169
|
+
const s2Err = err instanceof S2Error
|
|
170
|
+
? err
|
|
171
|
+
: new S2Error({ message: String(err), status: 500 });
|
|
172
|
+
controller.enqueue({ ok: false, error: s2Err });
|
|
173
|
+
controller.close();
|
|
174
|
+
}
|
|
175
|
+
};
|
|
176
|
+
// Helper to start/reset the timeout timer
|
|
177
|
+
// Resets on every tail received, fires only if no tail for 20s
|
|
178
|
+
const resetTimeoutTimer = () => {
|
|
179
|
+
if (timeoutTimer) {
|
|
180
|
+
clearTimeout(timeoutTimer);
|
|
151
181
|
}
|
|
182
|
+
timeoutTimer = setTimeout(() => {
|
|
183
|
+
const timeoutError = new S2Error({
|
|
184
|
+
message: `No tail received for ${TAIL_TIMEOUT_MS / 1000}s`,
|
|
185
|
+
status: 408, // Request Timeout
|
|
186
|
+
code: "TIMEOUT",
|
|
187
|
+
});
|
|
188
|
+
debug("tail timeout detected");
|
|
189
|
+
safeError(timeoutError);
|
|
190
|
+
}, TAIL_TIMEOUT_MS);
|
|
152
191
|
};
|
|
153
192
|
try {
|
|
193
|
+
// Start the timeout timer - will fire in 20s if no tail received
|
|
194
|
+
resetTimeoutTimer();
|
|
154
195
|
const connection = await getConnection();
|
|
155
196
|
// Build query string
|
|
156
197
|
const queryParams = new URLSearchParams();
|
|
@@ -177,9 +218,11 @@ class S2SReadSession extends ReadableStream {
|
|
|
177
218
|
":path": path,
|
|
178
219
|
":scheme": url.protocol.slice(0, -1),
|
|
179
220
|
":authority": url.host,
|
|
221
|
+
"user-agent": DEFAULT_USER_AGENT,
|
|
180
222
|
authorization: `Bearer ${Redacted.value(authToken)}`,
|
|
181
223
|
accept: "application/protobuf",
|
|
182
224
|
"content-type": "s2s/proto",
|
|
225
|
+
...(basinName ? { "s2-basin": basinName } : {}),
|
|
183
226
|
});
|
|
184
227
|
http2Stream = stream;
|
|
185
228
|
options?.signal?.addEventListener("abort", () => {
|
|
@@ -187,64 +230,139 @@ class S2SReadSession extends ReadableStream {
|
|
|
187
230
|
stream.close();
|
|
188
231
|
}
|
|
189
232
|
});
|
|
233
|
+
stream.on("response", (headers) => {
|
|
234
|
+
// Cache the status.
|
|
235
|
+
// This informs whether we should attempt to parse s2s frames in the "data" handler.
|
|
236
|
+
responseCode = headers[":status"] ?? 500;
|
|
237
|
+
});
|
|
238
|
+
connection.on("goaway", (errorCode, lastStreamID, opaqueData) => {
|
|
239
|
+
debug("received GOAWAY from server");
|
|
240
|
+
});
|
|
241
|
+
stream.on("error", (err) => {
|
|
242
|
+
safeError(err);
|
|
243
|
+
});
|
|
190
244
|
stream.on("data", (chunk) => {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
code: errorJson.code,
|
|
203
|
-
status: frame.statusCode,
|
|
204
|
-
}));
|
|
205
|
-
}
|
|
206
|
-
catch {
|
|
207
|
-
safeError(new S2Error({
|
|
208
|
-
message: errorText || "Unknown error",
|
|
209
|
-
status: frame.statusCode,
|
|
210
|
-
}));
|
|
211
|
-
}
|
|
245
|
+
try {
|
|
246
|
+
if ((responseCode ?? 500) >= 400) {
|
|
247
|
+
const errorText = textDecoder.decode(chunk);
|
|
248
|
+
try {
|
|
249
|
+
const errorJson = JSON.parse(errorText);
|
|
250
|
+
safeError(new S2Error({
|
|
251
|
+
message: errorJson.message ?? "Unknown error",
|
|
252
|
+
code: errorJson.code,
|
|
253
|
+
status: responseCode,
|
|
254
|
+
origin: "server",
|
|
255
|
+
}));
|
|
212
256
|
}
|
|
213
|
-
|
|
214
|
-
|
|
257
|
+
catch {
|
|
258
|
+
safeError(new S2Error({
|
|
259
|
+
message: errorText || "Unknown error",
|
|
260
|
+
status: responseCode,
|
|
261
|
+
origin: "server",
|
|
262
|
+
}));
|
|
215
263
|
}
|
|
216
|
-
|
|
264
|
+
return;
|
|
217
265
|
}
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
if (
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
266
|
+
// Buffer already extends Uint8Array in Node.js, no need to convert
|
|
267
|
+
parser.push(chunk);
|
|
268
|
+
let frame = parser.parseFrame();
|
|
269
|
+
while (frame) {
|
|
270
|
+
if (frame.terminal) {
|
|
271
|
+
if (frame.statusCode && frame.statusCode >= 400) {
|
|
272
|
+
const errorText = textDecoder.decode(frame.body);
|
|
273
|
+
try {
|
|
274
|
+
const errorJson = JSON.parse(errorText);
|
|
275
|
+
const status = frame.statusCode ?? 500;
|
|
276
|
+
// Map known read errors
|
|
277
|
+
if (status === 416) {
|
|
278
|
+
safeError(new RangeNotSatisfiableError({ status }));
|
|
279
|
+
}
|
|
280
|
+
else {
|
|
281
|
+
safeError(makeServerError({ status, statusText: undefined }, errorJson));
|
|
282
|
+
}
|
|
283
|
+
}
|
|
284
|
+
catch {
|
|
285
|
+
safeError(makeServerError({
|
|
286
|
+
status: frame.statusCode ?? 500,
|
|
287
|
+
statusText: undefined,
|
|
288
|
+
}, errorText));
|
|
289
|
+
}
|
|
227
290
|
}
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
const converted = this.convertRecord(record, as ?? "string", textDecoder);
|
|
231
|
-
controller.enqueue(converted);
|
|
291
|
+
else {
|
|
292
|
+
safeClose();
|
|
232
293
|
}
|
|
294
|
+
stream.close();
|
|
233
295
|
}
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
296
|
+
else {
|
|
297
|
+
// Parse ReadBatch
|
|
298
|
+
try {
|
|
299
|
+
const protoBatch = ProtoReadBatch.fromBinary(frame.body);
|
|
300
|
+
resetTimeoutTimer();
|
|
301
|
+
// Update tail from batch
|
|
302
|
+
if (protoBatch.tail) {
|
|
303
|
+
const tail = convertStreamPosition(protoBatch.tail);
|
|
304
|
+
lastReadPosition = tail;
|
|
305
|
+
this._lastReadPosition = tail;
|
|
306
|
+
this._lastObservedTail = tail;
|
|
307
|
+
debug("received tail");
|
|
308
|
+
}
|
|
309
|
+
// Enqueue each record and track next position
|
|
310
|
+
for (const record of protoBatch.records) {
|
|
311
|
+
const converted = this.convertRecord(record, as ?? "string", textDecoder);
|
|
312
|
+
controller.enqueue({ ok: true, value: converted });
|
|
313
|
+
// Update next read position to after this record
|
|
314
|
+
if (record.seqNum !== undefined) {
|
|
315
|
+
this._nextReadPosition = {
|
|
316
|
+
seq_num: Number(record.seqNum) + 1,
|
|
317
|
+
timestamp: Number(record.timestamp ?? 0n),
|
|
318
|
+
};
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
catch (err) {
|
|
323
|
+
safeError(new S2Error({
|
|
324
|
+
message: `Failed to parse ReadBatch: ${err}`,
|
|
325
|
+
status: 500,
|
|
326
|
+
origin: "sdk",
|
|
327
|
+
}));
|
|
328
|
+
}
|
|
238
329
|
}
|
|
330
|
+
frame = parser.parseFrame();
|
|
239
331
|
}
|
|
240
|
-
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
safeError(error instanceof S2Error
|
|
335
|
+
? error
|
|
336
|
+
: new S2Error({
|
|
337
|
+
message: `Failed to process read data: ${error}`,
|
|
338
|
+
status: 500,
|
|
339
|
+
origin: "sdk",
|
|
340
|
+
}));
|
|
241
341
|
}
|
|
242
342
|
});
|
|
243
|
-
stream.on("
|
|
244
|
-
|
|
343
|
+
stream.on("end", () => {
|
|
344
|
+
if (stream.rstCode != 0) {
|
|
345
|
+
debug("stream reset code=%d", stream.rstCode);
|
|
346
|
+
safeError(new S2Error({
|
|
347
|
+
message: `Stream ended with error: ${stream.rstCode}`,
|
|
348
|
+
status: 500,
|
|
349
|
+
code: "stream reset",
|
|
350
|
+
origin: "sdk",
|
|
351
|
+
}));
|
|
352
|
+
}
|
|
245
353
|
});
|
|
246
354
|
stream.on("close", () => {
|
|
247
|
-
|
|
355
|
+
if (parser.hasData()) {
|
|
356
|
+
safeError(new S2Error({
|
|
357
|
+
message: "Stream closed with unparsed data remaining",
|
|
358
|
+
status: 500,
|
|
359
|
+
code: "STREAM_CLOSED_PREMATURELY",
|
|
360
|
+
origin: "sdk",
|
|
361
|
+
}));
|
|
362
|
+
}
|
|
363
|
+
else {
|
|
364
|
+
safeClose();
|
|
365
|
+
}
|
|
248
366
|
});
|
|
249
367
|
}
|
|
250
368
|
catch (err) {
|
|
@@ -263,6 +381,7 @@ class S2SReadSession extends ReadableStream {
|
|
|
263
381
|
this.url = url;
|
|
264
382
|
this.options = options;
|
|
265
383
|
this.getConnection = getConnection;
|
|
384
|
+
this.basinName = basinName;
|
|
266
385
|
// Assign parser to instance property after super() completes
|
|
267
386
|
this.parser = parser;
|
|
268
387
|
this.http2Stream = http2Stream;
|
|
@@ -325,163 +444,46 @@ class S2SReadSession extends ReadableStream {
|
|
|
325
444
|
},
|
|
326
445
|
};
|
|
327
446
|
}
|
|
328
|
-
|
|
329
|
-
return this.
|
|
447
|
+
nextReadPosition() {
|
|
448
|
+
return this._nextReadPosition;
|
|
449
|
+
}
|
|
450
|
+
lastObservedTail() {
|
|
451
|
+
return this._lastObservedTail;
|
|
330
452
|
}
|
|
331
453
|
}
|
|
332
454
|
/**
|
|
333
455
|
* AcksStream for S2S append session
|
|
334
456
|
*/
|
|
335
|
-
|
|
336
|
-
constructor(setController) {
|
|
337
|
-
super({
|
|
338
|
-
start: (controller) => {
|
|
339
|
-
setController(controller);
|
|
340
|
-
},
|
|
341
|
-
});
|
|
342
|
-
}
|
|
343
|
-
async [Symbol.asyncDispose]() {
|
|
344
|
-
await this.cancel("disposed");
|
|
345
|
-
}
|
|
346
|
-
// Polyfill for older browsers
|
|
347
|
-
[Symbol.asyncIterator]() {
|
|
348
|
-
const fn = ReadableStream.prototype[Symbol.asyncIterator];
|
|
349
|
-
if (typeof fn === "function")
|
|
350
|
-
return fn.call(this);
|
|
351
|
-
const reader = this.getReader();
|
|
352
|
-
return {
|
|
353
|
-
next: async () => {
|
|
354
|
-
const r = await reader.read();
|
|
355
|
-
if (r.done) {
|
|
356
|
-
reader.releaseLock();
|
|
357
|
-
return { done: true, value: undefined };
|
|
358
|
-
}
|
|
359
|
-
return { done: false, value: r.value };
|
|
360
|
-
},
|
|
361
|
-
throw: async (e) => {
|
|
362
|
-
await reader.cancel(e);
|
|
363
|
-
reader.releaseLock();
|
|
364
|
-
return { done: true, value: undefined };
|
|
365
|
-
},
|
|
366
|
-
return: async () => {
|
|
367
|
-
await reader.cancel("done");
|
|
368
|
-
reader.releaseLock();
|
|
369
|
-
return { done: true, value: undefined };
|
|
370
|
-
},
|
|
371
|
-
[Symbol.asyncIterator]() {
|
|
372
|
-
return this;
|
|
373
|
-
},
|
|
374
|
-
};
|
|
375
|
-
}
|
|
376
|
-
}
|
|
457
|
+
// Removed S2SAcksStream - transport sessions no longer expose streams
|
|
377
458
|
/**
|
|
378
|
-
*
|
|
379
|
-
*
|
|
459
|
+
* Fetch-based transport session for appending records via HTTP/2.
|
|
460
|
+
* Pipelined: multiple requests can be in-flight simultaneously.
|
|
461
|
+
* No backpressure, no retry logic, no streams - just submit/close with value-encoded errors.
|
|
380
462
|
*/
|
|
381
463
|
class S2SAppendSession {
|
|
382
464
|
baseUrl;
|
|
383
465
|
authToken;
|
|
384
466
|
streamName;
|
|
385
467
|
getConnection;
|
|
468
|
+
basinName;
|
|
386
469
|
options;
|
|
387
470
|
http2Stream;
|
|
388
|
-
_lastAckedPosition;
|
|
389
471
|
parser = new S2SFrameParser();
|
|
390
|
-
acksController;
|
|
391
|
-
_readable;
|
|
392
|
-
_writable;
|
|
393
472
|
closed = false;
|
|
394
|
-
queuedBytes = 0;
|
|
395
|
-
maxQueuedBytes;
|
|
396
|
-
waitingForCapacity = [];
|
|
397
473
|
pendingAcks = [];
|
|
398
474
|
initPromise;
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
static async create(baseUrl, bearerToken, streamName, getConnection, sessionOptions, requestOptions) {
|
|
402
|
-
return new S2SAppendSession(baseUrl, bearerToken, streamName, getConnection, sessionOptions, requestOptions);
|
|
475
|
+
static async create(baseUrl, bearerToken, streamName, getConnection, basinName, sessionOptions, requestOptions) {
|
|
476
|
+
return new S2SAppendSession(baseUrl, bearerToken, streamName, getConnection, basinName, sessionOptions, requestOptions);
|
|
403
477
|
}
|
|
404
|
-
constructor(baseUrl, authToken, streamName, getConnection, sessionOptions, options) {
|
|
478
|
+
constructor(baseUrl, authToken, streamName, getConnection, basinName, sessionOptions, options) {
|
|
405
479
|
this.baseUrl = baseUrl;
|
|
406
480
|
this.authToken = authToken;
|
|
407
481
|
this.streamName = streamName;
|
|
408
482
|
this.getConnection = getConnection;
|
|
483
|
+
this.basinName = basinName;
|
|
409
484
|
this.options = options;
|
|
410
|
-
|
|
411
|
-
//
|
|
412
|
-
this._readable = new S2SAcksStream((controller) => {
|
|
413
|
-
this.acksController = controller;
|
|
414
|
-
});
|
|
415
|
-
this.readable = this._readable;
|
|
416
|
-
// Create the writable stream
|
|
417
|
-
this._writable = new WritableStream({
|
|
418
|
-
start: async (controller) => {
|
|
419
|
-
this.initPromise = this.initializeStream();
|
|
420
|
-
await this.initPromise;
|
|
421
|
-
},
|
|
422
|
-
write: async (chunk) => {
|
|
423
|
-
if (this.closed) {
|
|
424
|
-
throw new S2Error({ message: "AppendSession is closed" });
|
|
425
|
-
}
|
|
426
|
-
const recordsArray = Array.isArray(chunk.records)
|
|
427
|
-
? chunk.records
|
|
428
|
-
: [chunk.records];
|
|
429
|
-
// Validate batch size limits
|
|
430
|
-
if (recordsArray.length > 1000) {
|
|
431
|
-
throw new S2Error({
|
|
432
|
-
message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`,
|
|
433
|
-
});
|
|
434
|
-
}
|
|
435
|
-
// Calculate metered size
|
|
436
|
-
let batchMeteredSize = 0;
|
|
437
|
-
for (const record of recordsArray) {
|
|
438
|
-
batchMeteredSize += meteredSizeBytes(record);
|
|
439
|
-
}
|
|
440
|
-
if (batchMeteredSize > 1024 * 1024) {
|
|
441
|
-
throw new S2Error({
|
|
442
|
-
message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`,
|
|
443
|
-
});
|
|
444
|
-
}
|
|
445
|
-
// Wait for capacity if needed (backpressure)
|
|
446
|
-
while (this.queuedBytes + batchMeteredSize > this.maxQueuedBytes &&
|
|
447
|
-
!this.closed) {
|
|
448
|
-
await new Promise((resolve) => {
|
|
449
|
-
this.waitingForCapacity.push(resolve);
|
|
450
|
-
});
|
|
451
|
-
}
|
|
452
|
-
if (this.closed) {
|
|
453
|
-
throw new S2Error({ message: "AppendSession is closed" });
|
|
454
|
-
}
|
|
455
|
-
// Send the batch immediately (pipelined)
|
|
456
|
-
// Returns when frame is sent, not when ack is received
|
|
457
|
-
await this.sendBatchNonBlocking(recordsArray, chunk, batchMeteredSize);
|
|
458
|
-
},
|
|
459
|
-
close: async () => {
|
|
460
|
-
this.closed = true;
|
|
461
|
-
await this.closeStream();
|
|
462
|
-
},
|
|
463
|
-
abort: async (reason) => {
|
|
464
|
-
this.closed = true;
|
|
465
|
-
this.queuedBytes = 0;
|
|
466
|
-
// Reject all pending acks
|
|
467
|
-
const error = new S2Error({
|
|
468
|
-
message: `AppendSession was aborted: ${reason}`,
|
|
469
|
-
});
|
|
470
|
-
for (const pending of this.pendingAcks) {
|
|
471
|
-
pending.reject(error);
|
|
472
|
-
}
|
|
473
|
-
this.pendingAcks = [];
|
|
474
|
-
// Wake up all waiting for capacity
|
|
475
|
-
for (const resolver of this.waitingForCapacity) {
|
|
476
|
-
resolver();
|
|
477
|
-
}
|
|
478
|
-
this.waitingForCapacity = [];
|
|
479
|
-
if (this.http2Stream && !this.http2Stream.closed) {
|
|
480
|
-
this.http2Stream.close();
|
|
481
|
-
}
|
|
482
|
-
},
|
|
483
|
-
});
|
|
484
|
-
this.writable = this._writable;
|
|
485
|
+
// No stream setup
|
|
486
|
+
// Initialization happens lazily on first submit
|
|
485
487
|
}
|
|
486
488
|
async initializeStream() {
|
|
487
489
|
const url = new URL(this.baseUrl);
|
|
@@ -492,9 +494,11 @@ class S2SAppendSession {
|
|
|
492
494
|
":path": path,
|
|
493
495
|
":scheme": url.protocol.slice(0, -1),
|
|
494
496
|
":authority": url.host,
|
|
497
|
+
"user-agent": DEFAULT_USER_AGENT,
|
|
495
498
|
authorization: `Bearer ${Redacted.value(this.authToken)}`,
|
|
496
499
|
"content-type": "s2s/proto",
|
|
497
500
|
accept: "application/protobuf",
|
|
501
|
+
...(this.basinName ? { "s2-basin": this.basinName } : {}),
|
|
498
502
|
});
|
|
499
503
|
this.http2Stream = stream;
|
|
500
504
|
this.options?.signal?.addEventListener("abort", () => {
|
|
@@ -503,145 +507,87 @@ class S2SAppendSession {
|
|
|
503
507
|
}
|
|
504
508
|
});
|
|
505
509
|
const textDecoder = new TextDecoder();
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
this.acksController.close();
|
|
512
|
-
}
|
|
513
|
-
catch {
|
|
514
|
-
// Controller may already be closed, ignore
|
|
515
|
-
}
|
|
516
|
-
}
|
|
517
|
-
};
|
|
518
|
-
const safeError = (err) => {
|
|
519
|
-
if (!controllerClosed && this.acksController) {
|
|
520
|
-
controllerClosed = true;
|
|
521
|
-
this.acksController.error(err);
|
|
522
|
-
}
|
|
523
|
-
// Reject all pending acks
|
|
510
|
+
const safeError = (error) => {
|
|
511
|
+
const s2Err = error instanceof S2Error
|
|
512
|
+
? error
|
|
513
|
+
: new S2Error({ message: String(error), status: 502 });
|
|
514
|
+
// Resolve all pending acks with error result
|
|
524
515
|
for (const pending of this.pendingAcks) {
|
|
525
|
-
pending.
|
|
516
|
+
pending.resolve(err(s2Err));
|
|
526
517
|
}
|
|
527
518
|
this.pendingAcks = [];
|
|
528
519
|
};
|
|
529
520
|
// Handle incoming data (acks)
|
|
530
521
|
stream.on("data", (chunk) => {
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
if (frame.
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
const
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
}
|
|
522
|
+
try {
|
|
523
|
+
this.parser.push(chunk);
|
|
524
|
+
let frame = this.parser.parseFrame();
|
|
525
|
+
while (frame) {
|
|
526
|
+
if (frame.terminal) {
|
|
527
|
+
if (frame.statusCode && frame.statusCode >= 400) {
|
|
528
|
+
const errorText = textDecoder.decode(frame.body);
|
|
529
|
+
const status = frame.statusCode ?? 500;
|
|
530
|
+
try {
|
|
531
|
+
const errorJson = JSON.parse(errorText);
|
|
532
|
+
const err = status === 412
|
|
533
|
+
? makeAppendPreconditionError(status, errorJson)
|
|
534
|
+
: makeServerError({ status, statusText: undefined }, errorJson);
|
|
535
|
+
queueMicrotask(() => safeError(err));
|
|
536
|
+
}
|
|
537
|
+
catch {
|
|
538
|
+
const err = makeServerError({ status, statusText: undefined }, errorText);
|
|
539
|
+
queueMicrotask(() => safeError(err));
|
|
540
|
+
}
|
|
550
541
|
}
|
|
542
|
+
stream.close();
|
|
551
543
|
}
|
|
552
544
|
else {
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
const ack = convertAppendAck(protoAck);
|
|
562
|
-
this._lastAckedPosition = ack;
|
|
563
|
-
// Enqueue to readable stream
|
|
564
|
-
if (this.acksController) {
|
|
565
|
-
this.acksController.enqueue(ack);
|
|
566
|
-
}
|
|
567
|
-
// Resolve the pending ack promise
|
|
568
|
-
const pending = this.pendingAcks.shift();
|
|
569
|
-
if (pending) {
|
|
570
|
-
pending.resolve(ack);
|
|
571
|
-
// Release capacity
|
|
572
|
-
this.queuedBytes -= pending.batchSize;
|
|
573
|
-
// Wake up one waiting writer
|
|
574
|
-
if (this.waitingForCapacity.length > 0) {
|
|
575
|
-
const waiter = this.waitingForCapacity.shift();
|
|
576
|
-
waiter();
|
|
545
|
+
// Parse AppendAck
|
|
546
|
+
try {
|
|
547
|
+
const protoAck = ProtoAppendAck.fromBinary(frame.body);
|
|
548
|
+
const ack = convertAppendAck(protoAck);
|
|
549
|
+
// Resolve the pending ack promise (FIFO)
|
|
550
|
+
const pending = this.pendingAcks.shift();
|
|
551
|
+
if (pending) {
|
|
552
|
+
pending.resolve(ok(ack));
|
|
577
553
|
}
|
|
578
554
|
}
|
|
555
|
+
catch (parseErr) {
|
|
556
|
+
queueMicrotask(() => safeError(new S2Error({
|
|
557
|
+
message: `Failed to parse AppendAck: ${parseErr}`,
|
|
558
|
+
status: 500,
|
|
559
|
+
})));
|
|
560
|
+
}
|
|
579
561
|
}
|
|
580
|
-
|
|
581
|
-
safeError(new S2Error({
|
|
582
|
-
message: `Failed to parse AppendAck: ${err}`,
|
|
583
|
-
}));
|
|
584
|
-
}
|
|
562
|
+
frame = this.parser.parseFrame();
|
|
585
563
|
}
|
|
586
|
-
|
|
564
|
+
}
|
|
565
|
+
catch (error) {
|
|
566
|
+
queueMicrotask(() => safeError(error));
|
|
587
567
|
}
|
|
588
568
|
});
|
|
589
|
-
stream.on("error", (
|
|
590
|
-
safeError(
|
|
569
|
+
stream.on("error", (streamErr) => {
|
|
570
|
+
queueMicrotask(() => safeError(streamErr));
|
|
591
571
|
});
|
|
592
572
|
stream.on("close", () => {
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
}
|
|
603
|
-
// Convert to protobuf AppendInput
|
|
604
|
-
const protoInput = buildProtoAppendInput(records, args);
|
|
605
|
-
const bodyBytes = ProtoAppendInput.toBinary(protoInput);
|
|
606
|
-
// Frame the message
|
|
607
|
-
const frame = frameMessage({
|
|
608
|
-
terminal: false,
|
|
609
|
-
body: bodyBytes,
|
|
610
|
-
});
|
|
611
|
-
// This promise resolves when the frame is written (not when ack is received)
|
|
612
|
-
return new Promise((resolve, reject) => {
|
|
613
|
-
// Track pending ack - will be resolved when ack arrives
|
|
614
|
-
const ackPromise = {
|
|
615
|
-
resolve: () => { },
|
|
616
|
-
reject,
|
|
617
|
-
batchSize: batchMeteredSize,
|
|
618
|
-
};
|
|
619
|
-
this.pendingAcks.push(ackPromise);
|
|
620
|
-
this.queuedBytes += batchMeteredSize;
|
|
621
|
-
// Send the frame (pipelined)
|
|
622
|
-
this.http2Stream.write(frame, (err) => {
|
|
623
|
-
if (err) {
|
|
624
|
-
// Remove from pending acks on write error
|
|
625
|
-
const idx = this.pendingAcks.indexOf(ackPromise);
|
|
626
|
-
if (idx !== -1) {
|
|
627
|
-
this.pendingAcks.splice(idx, 1);
|
|
628
|
-
this.queuedBytes -= batchMeteredSize;
|
|
629
|
-
}
|
|
630
|
-
reject(err);
|
|
631
|
-
}
|
|
632
|
-
else {
|
|
633
|
-
// Frame written successfully - resolve immediately (pipelined)
|
|
634
|
-
resolve();
|
|
635
|
-
}
|
|
636
|
-
});
|
|
573
|
+
// Stream closed - resolve any remaining pending acks with error
|
|
574
|
+
// This can happen if the server closes the stream without sending all acks
|
|
575
|
+
if (this.pendingAcks.length > 0) {
|
|
576
|
+
queueMicrotask(() => safeError(new S2Error({
|
|
577
|
+
message: "Stream closed with pending acks",
|
|
578
|
+
status: 502,
|
|
579
|
+
code: "BAD_GATEWAY",
|
|
580
|
+
})));
|
|
581
|
+
}
|
|
637
582
|
});
|
|
638
583
|
}
|
|
639
584
|
/**
|
|
640
|
-
* Send a batch and wait for ack
|
|
585
|
+
* Send a batch and wait for ack. Returns AppendResult (never throws).
|
|
586
|
+
* Pipelined: multiple sends can be in-flight; acks resolve FIFO.
|
|
641
587
|
*/
|
|
642
588
|
sendBatch(records, args, batchMeteredSize) {
|
|
643
589
|
if (!this.http2Stream || this.http2Stream.closed) {
|
|
644
|
-
return Promise.
|
|
590
|
+
return Promise.resolve(err(new S2Error({ message: "HTTP/2 stream is not open", status: 502 })));
|
|
645
591
|
}
|
|
646
592
|
// Convert to protobuf AppendInput
|
|
647
593
|
const protoInput = buildProtoAppendInput(records, args);
|
|
@@ -651,82 +597,99 @@ class S2SAppendSession {
|
|
|
651
597
|
terminal: false,
|
|
652
598
|
body: bodyBytes,
|
|
653
599
|
});
|
|
654
|
-
// Track pending ack - this promise resolves when the ack is received
|
|
655
|
-
return new Promise((resolve
|
|
600
|
+
// Track pending ack - this promise resolves when the ack is received (FIFO)
|
|
601
|
+
return new Promise((resolve) => {
|
|
656
602
|
this.pendingAcks.push({
|
|
657
603
|
resolve,
|
|
658
|
-
reject,
|
|
659
604
|
batchSize: batchMeteredSize,
|
|
660
605
|
});
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
if (err) {
|
|
606
|
+
// Send the frame (pipelined - non-blocking)
|
|
607
|
+
this.http2Stream.write(frame, (writeErr) => {
|
|
608
|
+
if (writeErr) {
|
|
665
609
|
// Remove from pending acks on write error
|
|
666
|
-
const idx = this.pendingAcks.findIndex((p) => p.
|
|
610
|
+
const idx = this.pendingAcks.findIndex((p) => p.resolve === resolve);
|
|
667
611
|
if (idx !== -1) {
|
|
668
612
|
this.pendingAcks.splice(idx, 1);
|
|
669
|
-
this.queuedBytes -= batchMeteredSize;
|
|
670
613
|
}
|
|
671
|
-
|
|
614
|
+
// Resolve with error result
|
|
615
|
+
const s2Err = writeErr instanceof S2Error
|
|
616
|
+
? writeErr
|
|
617
|
+
: new S2Error({ message: String(writeErr), status: 502 });
|
|
618
|
+
resolve(err(s2Err));
|
|
672
619
|
}
|
|
673
|
-
// Write completed
|
|
620
|
+
// Write completed successfully - promise resolves later when ack is received
|
|
674
621
|
});
|
|
675
622
|
});
|
|
676
623
|
}
|
|
677
|
-
async closeStream() {
|
|
678
|
-
// Wait for all pending acks
|
|
679
|
-
while (this.pendingAcks.length > 0) {
|
|
680
|
-
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
681
|
-
}
|
|
682
|
-
// Close the HTTP/2 stream (client doesn't send terminal frame for clean close)
|
|
683
|
-
if (this.http2Stream && !this.http2Stream.closed) {
|
|
684
|
-
this.http2Stream.end();
|
|
685
|
-
}
|
|
686
|
-
}
|
|
687
|
-
async [Symbol.asyncDispose]() {
|
|
688
|
-
await this.close();
|
|
689
|
-
}
|
|
690
|
-
/**
|
|
691
|
-
* Get a stream of acknowledgements for appends.
|
|
692
|
-
*/
|
|
693
|
-
acks() {
|
|
694
|
-
return this._readable;
|
|
695
|
-
}
|
|
696
624
|
/**
|
|
697
625
|
* Close the append session.
|
|
698
626
|
* Waits for all pending appends to complete before resolving.
|
|
627
|
+
* Never throws - returns CloseResult.
|
|
699
628
|
*/
|
|
700
629
|
async close() {
|
|
701
|
-
|
|
630
|
+
try {
|
|
631
|
+
this.closed = true;
|
|
632
|
+
// Wait for all pending acks to complete
|
|
633
|
+
while (this.pendingAcks.length > 0) {
|
|
634
|
+
await new Promise((resolve) => setTimeout(resolve, 10));
|
|
635
|
+
}
|
|
636
|
+
// Close the HTTP/2 stream (client doesn't send terminal frame for clean close)
|
|
637
|
+
if (this.http2Stream && !this.http2Stream.closed) {
|
|
638
|
+
this.http2Stream.end();
|
|
639
|
+
}
|
|
640
|
+
return okClose();
|
|
641
|
+
}
|
|
642
|
+
catch (error) {
|
|
643
|
+
const s2Err = error instanceof S2Error
|
|
644
|
+
? error
|
|
645
|
+
: new S2Error({ message: String(error), status: 500 });
|
|
646
|
+
return errClose(s2Err);
|
|
647
|
+
}
|
|
702
648
|
}
|
|
703
649
|
/**
|
|
704
650
|
* Submit an append request to the session.
|
|
705
|
-
* Returns
|
|
651
|
+
* Returns AppendResult (never throws).
|
|
652
|
+
* Pipelined: multiple submits can be in-flight; acks resolve FIFO.
|
|
706
653
|
*/
|
|
707
654
|
async submit(records, args) {
|
|
655
|
+
// Validate closed state
|
|
708
656
|
if (this.closed) {
|
|
709
|
-
return
|
|
657
|
+
return err(new S2Error({ message: "AppendSession is closed", status: 400 }));
|
|
710
658
|
}
|
|
711
|
-
//
|
|
712
|
-
if (this.initPromise) {
|
|
659
|
+
// Lazy initialize HTTP/2 stream on first submit
|
|
660
|
+
if (!this.initPromise) {
|
|
661
|
+
this.initPromise = this.initializeStream();
|
|
662
|
+
}
|
|
663
|
+
try {
|
|
713
664
|
await this.initPromise;
|
|
714
665
|
}
|
|
666
|
+
catch (initErr) {
|
|
667
|
+
const s2Err = initErr instanceof S2Error
|
|
668
|
+
? initErr
|
|
669
|
+
: new S2Error({ message: String(initErr), status: 502 });
|
|
670
|
+
return err(s2Err);
|
|
671
|
+
}
|
|
715
672
|
const recordsArray = Array.isArray(records) ? records : [records];
|
|
716
|
-
// Validate batch size limits
|
|
673
|
+
// Validate batch size limits (non-retryable 400-level error)
|
|
717
674
|
if (recordsArray.length > 1000) {
|
|
718
|
-
return
|
|
675
|
+
return err(new S2Error({
|
|
719
676
|
message: `Batch of ${recordsArray.length} exceeds maximum batch size of 1000 records`,
|
|
677
|
+
status: 400,
|
|
678
|
+
code: "INVALID_ARGUMENT",
|
|
720
679
|
}));
|
|
721
680
|
}
|
|
722
|
-
// Calculate metered size
|
|
723
|
-
let batchMeteredSize = 0;
|
|
724
|
-
|
|
725
|
-
|
|
681
|
+
// Calculate metered size (use precalculated if provided)
|
|
682
|
+
let batchMeteredSize = args?.precalculatedSize ?? 0;
|
|
683
|
+
if (batchMeteredSize === 0) {
|
|
684
|
+
for (const record of recordsArray) {
|
|
685
|
+
batchMeteredSize += meteredBytes(record);
|
|
686
|
+
}
|
|
726
687
|
}
|
|
727
688
|
if (batchMeteredSize > 1024 * 1024) {
|
|
728
|
-
return
|
|
689
|
+
return err(new S2Error({
|
|
729
690
|
message: `Batch size ${batchMeteredSize} bytes exceeds maximum of 1 MiB (1048576 bytes)`,
|
|
691
|
+
status: 400,
|
|
692
|
+
code: "INVALID_ARGUMENT",
|
|
730
693
|
}));
|
|
731
694
|
}
|
|
732
695
|
return this.sendBatch(recordsArray, {
|
|
@@ -735,9 +698,6 @@ class S2SAppendSession {
|
|
|
735
698
|
match_seq_num: args?.match_seq_num,
|
|
736
699
|
}, batchMeteredSize);
|
|
737
700
|
}
|
|
738
|
-
lastAckedPosition() {
|
|
739
|
-
return this._lastAckedPosition;
|
|
740
|
-
}
|
|
741
701
|
}
|
|
742
702
|
/**
|
|
743
703
|
* Convert protobuf StreamPosition to OpenAPI StreamPosition
|