@trigger.dev/core 4.0.7 → 4.1.1

package/dist/esm/v3/realtimeStreams/streamsWriterV1.js

@@ -0,0 +1,378 @@
+import { request as httpsRequest } from "node:https";
+import { request as httpRequest } from "node:http";
+import { URL } from "node:url";
+import { randomBytes } from "node:crypto";
+export class StreamsWriterV1 {
+    options;
+    controller = new AbortController();
+    serverStream;
+    consumerStream;
+    streamPromise;
+    retryCount = 0;
+    maxRetries;
+    currentChunkIndex = 0;
+    baseDelayMs = 1000; // 1 second base delay
+    maxDelayMs = 30000; // 30 seconds max delay
+    maxBufferSize;
+    clientId;
+    ringBuffer = []; // Ring buffer for recent chunks
+    bufferStartIndex = 0; // Index of the oldest chunk in buffer
+    highestBufferedIndex = -1; // Highest chunk index that's been buffered
+    streamReader = null;
+    bufferReaderTask = null;
+    streamComplete = false;
+    constructor(options) {
+        this.options = options;
+        const [serverStream, consumerStream] = this.options.source.tee();
+        this.serverStream = serverStream;
+        this.consumerStream = consumerStream;
+        this.maxRetries = options.maxRetries ?? 10;
+        this.maxBufferSize = options.maxBufferSize ?? 10000; // Default 10000 chunks
+        this.clientId = options.clientId || this.generateClientId();
+        // Start background task to continuously read from stream into ring buffer
+        this.startBuffering();
+        this.streamPromise = this.initializeServerStream();
+    }
+    generateClientId() {
+        return randomBytes(4).toString("hex");
+    }
+    startBuffering() {
+        this.streamReader = this.serverStream.getReader();
+        this.bufferReaderTask = (async () => {
+            try {
+                let chunkIndex = 0;
+                while (true) {
+                    const { done, value } = await this.streamReader.read();
+                    if (done) {
+                        this.streamComplete = true;
+                        break;
+                    }
+                    // Add to ring buffer
+                    this.addToRingBuffer(chunkIndex, value);
+                    this.highestBufferedIndex = chunkIndex;
+                    chunkIndex++;
+                }
+            }
+            catch (error) {
+                throw error;
+            }
+        })();
+    }
+    async makeRequest(startFromChunk = 0) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(this.buildUrl());
+            const timeout = 15 * 60 * 1000; // 15 minutes
+            const requestFn = url.protocol === "https:" ? httpsRequest : httpRequest;
+            const req = requestFn({
+                method: "POST",
+                hostname: url.hostname,
+                port: url.port || (url.protocol === "https:" ? 443 : 80),
+                path: url.pathname + url.search,
+                headers: {
+                    ...this.options.headers,
+                    "Content-Type": "application/json",
+                    "X-Client-Id": this.clientId,
+                    "X-Resume-From-Chunk": startFromChunk.toString(),
+                    "X-Stream-Version": this.options.version ?? "v1",
+                },
+                timeout,
+            });
+            req.on("error", async (error) => {
+                const errorCode = "code" in error ? error.code : undefined;
+                const errorMsg = error instanceof Error ? error.message : String(error);
+                // Check if this is a retryable connection error
+                if (this.isRetryableError(error)) {
+                    if (this.retryCount < this.maxRetries) {
+                        this.retryCount++;
+                        // Clean up the current request to avoid socket leaks
+                        req.destroy();
+                        const delayMs = this.calculateBackoffDelay();
+                        await this.delay(delayMs);
+                        // Query server to find out what the last chunk it received was
+                        const serverLastChunk = await this.queryServerLastChunkIndex();
+                        // Resume from the next chunk after what the server has
+                        const resumeFromChunk = serverLastChunk + 1;
+                        resolve(this.makeRequest(resumeFromChunk));
+                        return;
+                    }
+                }
+                reject(error);
+            });
+            req.on("timeout", async () => {
+                // Timeout is retryable
+                if (this.retryCount < this.maxRetries) {
+                    this.retryCount++;
+                    // Clean up the current request to avoid socket leaks
+                    req.destroy();
+                    const delayMs = this.calculateBackoffDelay();
+                    await this.delay(delayMs);
+                    // Query server to find where to resume
+                    const serverLastChunk = await this.queryServerLastChunkIndex();
+                    const resumeFromChunk = serverLastChunk + 1;
+                    resolve(this.makeRequest(resumeFromChunk));
+                    return;
+                }
+                req.destroy();
+                reject(new Error("Request timed out"));
+            });
+            req.on("response", async (res) => {
+                // Check for retryable status codes (408, 429, 5xx)
+                if (res.statusCode && this.isRetryableStatusCode(res.statusCode)) {
+                    if (this.retryCount < this.maxRetries) {
+                        this.retryCount++;
+                        // Drain and destroy the response and request to avoid socket leaks
+                        // We need to consume the response before destroying it
+                        res.resume(); // Start draining the response
+                        res.destroy(); // Destroy the response to free the socket
+                        req.destroy(); // Destroy the request as well
+                        const delayMs = this.calculateBackoffDelay();
+                        await this.delay(delayMs);
+                        // Query server to find where to resume (in case some data was written)
+                        const serverLastChunk = await this.queryServerLastChunkIndex();
+                        const resumeFromChunk = serverLastChunk + 1;
+                        resolve(this.makeRequest(resumeFromChunk));
+                        return;
+                    }
+                    res.destroy();
+                    req.destroy();
+                    reject(new Error(`Max retries (${this.maxRetries}) exceeded for status code ${res.statusCode}`));
+                    return;
+                }
+                // Non-retryable error status
+                if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
+                    res.destroy();
+                    req.destroy();
+                    const error = new Error(`HTTP error! status: ${res.statusCode}`);
+                    reject(error);
+                    return;
+                }
+                // Success! Reset retry count
+                this.retryCount = 0;
+                res.on("end", () => {
+                    resolve();
+                });
+                res.resume();
+            });
+            if (this.options.signal) {
+                this.options.signal.addEventListener("abort", () => {
+                    req.destroy(new Error("Request aborted"));
+                });
+            }
+            const processStream = async () => {
+                try {
+                    let lastSentIndex = startFromChunk - 1;
+                    while (true) {
+                        // Send all chunks that are available in buffer
+                        while (lastSentIndex < this.highestBufferedIndex) {
+                            lastSentIndex++;
+                            const chunk = this.ringBuffer.find((c) => c.index === lastSentIndex);
+                            if (chunk) {
+                                const stringified = JSON.stringify(chunk.data) + "\n";
+                                req.write(stringified);
+                                this.currentChunkIndex = lastSentIndex + 1;
+                            }
+                        }
+                        // If stream is complete and we've sent all buffered chunks, we're done
+                        if (this.streamComplete && lastSentIndex >= this.highestBufferedIndex) {
+                            req.end();
+                            break;
+                        }
+                        // Wait a bit for more chunks to be buffered
+                        await this.delay(10);
+                    }
+                }
+                catch (error) {
+                    reject(error);
+                }
+            };
+            processStream().catch((error) => {
+                reject(error);
+            });
+        });
+    }
+    async initializeServerStream() {
+        await this.makeRequest(0);
+    }
+    async wait() {
+        return this.streamPromise;
+    }
+    [Symbol.asyncIterator]() {
+        return streamToAsyncIterator(this.consumerStream);
+    }
+    buildUrl() {
+        return `${this.options.baseUrl}/realtime/v1/streams/${this.options.runId}/${this.options.target ?? "self"}/${this.options.key}`;
+    }
+    isRetryableError(error) {
+        if (!error)
+            return false;
+        // Connection errors that are safe to retry
+        const retryableErrors = [
+            "ECONNRESET", // Connection reset by peer
+            "ECONNREFUSED", // Connection refused
+            "ETIMEDOUT", // Connection timed out
+            "ENOTFOUND", // DNS lookup failed
+            "EPIPE", // Broken pipe
+            "EHOSTUNREACH", // Host unreachable
+            "ENETUNREACH", // Network unreachable
+            "socket hang up", // Socket hang up
+        ];
+        // Check error code
+        if (error.code && retryableErrors.includes(error.code)) {
+            return true;
+        }
+        // Check error message for socket hang up
+        if (error.message && error.message.includes("socket hang up")) {
+            return true;
+        }
+        return false;
+    }
+    isRetryableStatusCode(statusCode) {
+        // Retry on transient server errors
+        if (statusCode === 408)
+            return true; // Request Timeout
+        if (statusCode === 429)
+            return true; // Rate Limit
+        if (statusCode === 500)
+            return true; // Internal Server Error
+        if (statusCode === 502)
+            return true; // Bad Gateway
+        if (statusCode === 503)
+            return true; // Service Unavailable
+        if (statusCode === 504)
+            return true; // Gateway Timeout
+        return false;
+    }
+    async delay(ms) {
+        return new Promise((resolve) => setTimeout(resolve, ms));
+    }
+    calculateBackoffDelay() {
+        // Exponential backoff with jitter: baseDelay * 2^retryCount + random jitter
+        const exponentialDelay = this.baseDelayMs * Math.pow(2, this.retryCount);
+        const jitter = Math.random() * 1000; // 0-1000ms jitter
+        return Math.min(exponentialDelay + jitter, this.maxDelayMs);
+    }
+    addToRingBuffer(index, data) {
+        const chunk = { index, data };
+        if (this.ringBuffer.length < this.maxBufferSize) {
+            // Buffer not full yet, just append
+            this.ringBuffer.push(chunk);
+        }
+        else {
+            // Buffer full, replace oldest chunk (ring buffer behavior)
+            const bufferIndex = index % this.maxBufferSize;
+            this.ringBuffer[bufferIndex] = chunk;
+            this.bufferStartIndex = Math.max(this.bufferStartIndex, index - this.maxBufferSize + 1);
+        }
+    }
+    getChunksFromBuffer(startIndex) {
+        const result = [];
+        for (const chunk of this.ringBuffer) {
+            if (chunk.index >= startIndex) {
+                result.push(chunk);
+            }
+        }
+        // Sort by index to ensure correct order
+        result.sort((a, b) => a.index - b.index);
+        return result;
+    }
+    async queryServerLastChunkIndex(attempt = 0) {
+        return new Promise((resolve, reject) => {
+            const url = new URL(this.buildUrl());
+            const maxHeadRetries = 3; // Separate retry limit for HEAD requests
+            const requestFn = url.protocol === "https:" ? httpsRequest : httpRequest;
+            const req = requestFn({
+                method: "HEAD",
+                hostname: url.hostname,
+                port: url.port || (url.protocol === "https:" ? 443 : 80),
+                path: url.pathname + url.search,
+                headers: {
+                    ...this.options.headers,
+                    "X-Client-Id": this.clientId,
+                    "X-Stream-Version": this.options.version ?? "v1",
+                },
+                timeout: 5000, // 5 second timeout for HEAD request
+            });
+            req.on("error", async (error) => {
+                if (this.isRetryableError(error) && attempt < maxHeadRetries) {
+                    // Clean up the current request to avoid socket leaks
+                    req.destroy();
+                    await this.delay(1000 * (attempt + 1)); // Simple linear backoff
+                    const result = await this.queryServerLastChunkIndex(attempt + 1);
+                    resolve(result);
+                    return;
+                }
+                req.destroy();
+                // Return -1 to indicate we don't know what the server has (resume from 0)
+                resolve(-1);
+            });
+            req.on("timeout", async () => {
+                req.destroy();
+                if (attempt < maxHeadRetries) {
+                    await this.delay(1000 * (attempt + 1));
+                    const result = await this.queryServerLastChunkIndex(attempt + 1);
+                    resolve(result);
+                    return;
+                }
+                resolve(-1);
+            });
+            req.on("response", async (res) => {
+                // Retry on 5xx errors
+                if (res.statusCode && this.isRetryableStatusCode(res.statusCode)) {
+                    if (attempt < maxHeadRetries) {
+                        // Drain and destroy the response and request to avoid socket leaks
+                        res.resume();
+                        res.destroy();
+                        req.destroy();
+                        await this.delay(1000 * (attempt + 1));
+                        const result = await this.queryServerLastChunkIndex(attempt + 1);
+                        resolve(result);
+                        return;
+                    }
+                    res.destroy();
+                    req.destroy();
+                    resolve(-1);
+                    return;
+                }
+                // Non-retryable error
+                if (res.statusCode && (res.statusCode < 200 || res.statusCode >= 300)) {
+                    res.destroy();
+                    req.destroy();
+                    resolve(-1);
+                    return;
+                }
+                // Success - extract chunk index
+                const lastChunkHeader = res.headers["x-last-chunk-index"];
+                if (lastChunkHeader) {
+                    const lastChunkIndex = parseInt(Array.isArray(lastChunkHeader) ? lastChunkHeader[0] ?? "0" : lastChunkHeader ?? "0", 10);
+                    resolve(lastChunkIndex);
+                }
+                else {
+                    resolve(-1);
+                }
+                res.resume(); // Consume response
+            });
+            req.end();
+        });
+    }
+}
+async function* streamToAsyncIterator(stream) {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            yield value;
+        }
+    }
+    finally {
+        safeReleaseLock(reader);
+    }
+}
+function safeReleaseLock(reader) {
+    try {
+        reader.releaseLock();
+    }
+    catch (error) { }
+}
+//# sourceMappingURL=streamsWriterV1.js.map

package/dist/esm/v3/realtimeStreams/streamsWriterV1.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamsWriterV1.js","sourceRoot":"","sources":["../../../../src/v3/realtimeStreams/streamsWriterV1.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,IAAI,YAAY,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,OAAO,IAAI,WAAW,EAAE,MAAM,WAAW,CAAC;AACnD,OAAO,EAAE,GAAG,EAAE,MAAM,UAAU,CAAC;AAC/B,OAAO,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAsB1C,MAAM,OAAO,eAAe;IAmBN;IAlBZ,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACnC,YAAY,CAAoB;IAChC,cAAc,CAAoB;IAClC,aAAa,CAAgB;IAC7B,UAAU,GAAG,CAAC,CAAC;IACN,UAAU,CAAS;IAC5B,iBAAiB,GAAG,CAAC,CAAC;IACb,WAAW,GAAG,IAAI,CAAC,CAAC,sBAAsB;IAC1C,UAAU,GAAG,KAAK,CAAC,CAAC,uBAAuB;IAC3C,aAAa,CAAS;IACtB,QAAQ,CAAS;IAC1B,UAAU,GAAuB,EAAE,CAAC,CAAC,gCAAgC;IACrE,gBAAgB,GAAG,CAAC,CAAC,CAAC,sCAAsC;IAC5D,oBAAoB,GAAG,CAAC,CAAC,CAAC,CAAC,2CAA2C;IACtE,YAAY,GAA0C,IAAI,CAAC;IAC3D,gBAAgB,GAAyB,IAAI,CAAC;IAC9C,cAAc,GAAG,KAAK,CAAC;IAE/B,YAAoB,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QACpD,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACjE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,EAAE,CAAC;QAC3C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,KAAK,CAAC,CAAC,uBAAuB;QAC5E,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAE5D,0EAA0E;QAC1E,IAAI,CAAC,cAAc,EAAE,CAAC;QAEtB,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACrD,CAAC;IAEO,gBAAgB;QACtB,OAAO,WAAW,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;IACxC,CAAC;IAEO,cAAc;QACpB,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,YAAY,CAAC,SAAS,EAAE,CAAC;QAElD,IAAI,CAAC,gBAAgB,GAAG,CAAC,KAAK,IAAI,EAAE;YAClC,IAAI,CAAC;gBACH,IAAI,UAAU,GAAG,CAAC,CAAC;gBACnB,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,IAAI,CAAC,YAAa,CAAC,IAAI,EAAE,CAAC;oBAExD,IAAI,IAAI,EAAE,CAAC;wBACT,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC;wBAC3B,MAAM;oBACR,CAAC;oBAED,qBAAqB;oBACrB,IAAI,CAAC,eAAe,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;oBACxC,IAAI,CAAC,oBAAoB,GAAG,UAAU,CAAC;oBACvC,UAAU,EAAE,CAAC;gBACf,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC,CAAC,EAAE,CAAC;IACP,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,iBAAyB,CAAC;QAClD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrC,MAAM,OAAO,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC,aAAa;YAE7C,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;YACzE,MAAM,GAAG,GAAG,SAAS,CAAC;gBACpB,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,IAAI,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;gBAC/B,OAAO,EAAE;oBACP,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;oBACvB,cAAc,EAAE,kBAAkB;oBAClC,aAAa,EAAE,IAAI,CAAC,QAAQ;oBAC5B,qBAAqB,EAAE,cAAc,CAAC,QAAQ,EAAE;oBAChD,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI;iBACjD;gBACD,OAAO;aACR,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBAC9B,MAAM,SAAS,GAAG,MAAM,IAAI,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC3D,MAAM,QAAQ,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAExE,gDAAgD;gBAChD,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,EAAE,CAAC;oBACjC,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;wBACtC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAElB,qDAAqD;wBACrD,GAAG,CAAC,OAAO,EAAE,CAAC;wBAEd,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;wBAE7C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAE1B,+DAA+D;wBAC/D,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;wBAE/D,uDAAuD;wBACvD,MAAM,eAAe,GAAG,eAAe,GAAG,CAAC,CAAC;wBAE5C,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC;wBAC3C,OAAO;oBACT,CAAC;gBACH,CAAC;gBAED,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAC3B,uBAAuB;gBACvB,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;oBACtC,IAAI,CAAC,UAAU,EAAE,CAAC;oBAElB,qDAAqD;oBACrD,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;oBAE7C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;oBAE1B,uCAAuC;oBACvC,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;oBAC/D,MAAM,eAAe,GAAG,eAAe,GAAG,CAAC,CAAC;oBAE5C,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC;oBAC3C,OAAO;gBACT,CAAC;gBAED,GAAG,CAAC,OAAO,EAAE,CAAC;gBACd,MAAM,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;YACzC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC/B,mDAAmD;gBACnD,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBACjE,IAAI,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,CAAC;wBACtC,IAAI,CAAC,UAAU,EAAE,CAAC;wBAElB,mEAAmE;wBACnE,uDAAuD;wBACvD,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,8BAA8B;wBAC5C,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,0CAA0C;wBACzD,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,8BAA8B;wBAE7C,MAAM,OAAO,GAAG,IAAI,CAAC,qBAAqB,EAAE,CAAC;wBAE7C,MAAM,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;wBAE1B,uEAAuE;wBACvE,MAAM,eAAe,GAAG,MAAM,IAAI,CAAC,yBAAyB,EAAE,CAAC;wBAC/D,MAAM,eAAe,GAAG,eAAe,GAAG,CAAC,CAAC;wBAE5C,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,eAAe,CAAC,CAAC,CAAC;wBAC3C,OAAO;oBACT,CAAC;oBAED,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,MAAM,CACJ,IAAI,KAAK,CAAC,gBAAgB,IAAI,CAAC,UAAU,8BAA8B,GAAG,CAAC,UAAU,EAAE,CAAC,CACzF,CAAC;oBACF,OAAO;gBACT,CAAC;gBAED,6BAA6B;gBAC7B,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;oBACtE,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,uBAAuB,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;oBACjE,MAAM,CAAC,KAAK,CAAC,CAAC;oBACd,OAAO;gBACT,CAAC;gBAED,6BAA6B;gBAC7B,IAAI,CAAC,UAAU,GAAG,CAAC,CAAC;gBAEpB,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE;oBACjB,OAAO,EAAE,CAAC;gBACZ,CAAC,CAAC,CAAC;gBAEH,GAAG,CAAC,MAAM,EAAE,CAAC;YACf,CAAC,CAAC,CAAC;YAEH,IAAI,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;gBACxB,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;oBACjD,GAAG,CAAC,OAAO,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;gBAC5C,CAAC,CAAC,CAAC;YACL,CAAC;YAED,MAAM,aAAa,GAAG,KAAK,IAAI,EAAE;gBAC/B,IAAI,CAAC;oBACH,IAAI,aAAa,GAAG,cAAc,GAAG,CAAC,CAAC;oBAEvC,OAAO,IAAI,EAAE,CAAC;wBACZ,+CAA+C;wBAC/C,OAAO,aAAa,GAAG,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BACjD,aAAa,EAAE,CAAC;4BAChB,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,aAAa,CAAC,CAAC;4BAErE,IAAI,KAAK,EAAE,CAAC;gCACV,MAAM,WAAW,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;gCACtD,GAAG,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gCACvB,IAAI,CAAC,iBAAiB,GAAG,aAAa,GAAG,CAAC,CAAC;4BAC7C,CAAC;wBACH,CAAC;wBAED,uEAAuE;wBACvE,IAAI,IAAI,CAAC,cAAc,IAAI,aAAa,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;4BACtE,GAAG,CAAC,GAAG,EAAE,CAAC;4BACV,MAAM;wBACR,CAAC;wBAED,4CAA4C;wBAC5C,MAAM,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,CAAC,KAAK,CAAC,CAAC;gBAChB,CAAC;YACH,CAAC,CAAC;YAEF,aAAa,EAAE,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC9B,MAAM,CAAC,KAAK,CAAC,CAAC;YAChB,CAAC,CAAC,CAAC;QACL,CAAC,CAAC,CAAC;IACL,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,MAAM,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,OAAO,IAAI,CAAC,aAAa,CAAC;IAC5B,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAEO,QAAQ;QACd,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,wBAAwB,IAAI,CAAC,OAAO,CAAC,KAAK,IACtE,IAAI,CAAC,OAAO,CAAC,MAAM,IAAI,MACzB,IAAI,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC;IACzB,CAAC;IAEO,gBAAgB,CAAC,KAAU;QACjC,IAAI,CAAC,KAAK;YAAE,OAAO,KAAK,CAAC;QAEzB,2CAA2C;QAC3C,MAAM,eAAe,GAAG;YACtB,YAAY,EAAE,2BAA2B;YACzC,cAAc,EAAE,qBAAqB;YACrC,WAAW,EAAE,uBAAuB;YACpC,WAAW,EAAE,oBAAoB;YACjC,OAAO,EAAE,cAAc;YACvB,cAAc,EAAE,mBAAmB;YACnC,aAAa,EAAE,sBAAsB;YACrC,gBAAgB,EAAE,iBAAiB;SACpC,CAAC;QAEF,mBAAmB;QACnB,IAAI,KAAK,CAAC,IAAI,IAAI,eAAe,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;YACvD,OAAO,IAAI,CAAC;QACd,CAAC;QAED,yCAAyC;QACzC,IAAI,KAAK,CAAC,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC;YAC9D,OAAO,IAAI,CAAC;QACd,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,qBAAqB,CAAC,UAAkB;QAC9C,mCAAmC;QACnC,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;QACvD,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,aAAa;QAClD,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,wBAAwB;QAC7D,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,cAAc;QACnD,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,sBAAsB;QAC3D,IAAI,UAAU,KAAK,GAAG;YAAE,OAAO,IAAI,CAAC,CAAC,kBAAkB;QAEvD,OAAO,KAAK,CAAC;IACf,CAAC;IAEO,KAAK,CAAC,KAAK,CAAC,EAAU;QAC5B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAEO,qBAAqB;QAC3B,4EAA4E;QAC5E,MAAM,gBAAgB,GAAG,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QACzE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,kBAAkB;QACvD,OAAO,IAAI,CAAC,GAAG,CAAC,gBAAgB,GAAG,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC9D,CAAC;IAEO,eAAe,CAAC,KAAa,EAAE,IAAO;QAC5C,MAAM,KAAK,GAAqB,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;QAEhD,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,aAAa,EAAE,CAAC;YAChD,mCAAmC;YACnC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC;aAAM,CAAC;YACN,2DAA2D;YAC3D,MAAM,WAAW,GAAG,KAAK,GAAG,IAAI,CAAC,aAAa,CAAC;YAC/C,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,GAAG,KAAK,CAAC;YACrC,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,gBAAgB,EAAE,KAAK,GAAG,IAAI,CAAC,aAAa,GAAG,CAAC,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;IAEO,mBAAmB,CAAC,UAAkB;QAC5C,MAAM,MAAM,GAAuB,EAAE,CAAC;QAEtC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACpC,IAAI,KAAK,CAAC,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC9B,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;QACH,CAAC;QAED,wCAAwC;QACxC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;QACzC,OAAO,MAAM,CAAC;IAChB,CAAC;IAEO,KAAK,CAAC,yBAAyB,CAAC,UAAkB,CAAC;QACzD,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;YACrC,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,QAAQ,EAAE,CAAC,CAAC;YACrC,MAAM,cAAc,GAAG,CAAC,CAAC,CAAC,yCAAyC;YAEnE,MAAM,SAAS,GAAG,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,WAAW,CAAC;YACzE,MAAM,GAAG,GAAG,SAAS,CAAC;gBACpB,MAAM,EAAE,MAAM;gBACd,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,IAAI,EAAE,GAAG,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,IAAI,EAAE,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;gBAC/B,OAAO,EAAE;oBACP,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;oBACvB,aAAa,EAAE,IAAI,CAAC,QAAQ;oBAC5B,kBAAkB,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,IAAI;iBACjD;gBACD,OAAO,EAAE,IAAI,EAAE,oCAAoC;aACpD,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE;gBAC9B,IAAI,IAAI,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,OAAO,GAAG,cAAc,EAAE,CAAC;oBAC7D,qDAAqD;oBACrD,GAAG,CAAC,OAAO,EAAE,CAAC;oBAEd,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,wBAAwB;oBAChE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;oBACjE,OAAO,CAAC,MAAM,CAAC,CAAC;oBAChB,OAAO;gBACT,CAAC;gBAED,GAAG,CAAC,OAAO,EAAE,CAAC;gBACd,0EAA0E;gBAC1E,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,KAAK,IAAI,EAAE;gBAC3B,GAAG,CAAC,OAAO,EAAE,CAAC;gBAEd,IAAI,OAAO,GAAG,cAAc,EAAE,CAAC;oBAC7B,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;oBACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;oBACjE,OAAO,CAAC,MAAM,CAAC,CAAC;oBAChB,OAAO;gBACT,CAAC;gBAED,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACd,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,EAAE,CAAC,UAAU,EAAE,KAAK,EAAE,GAAG,EAAE,EAAE;gBAC/B,sBAAsB;gBACtB,IAAI,GAAG,CAAC,UAAU,IAAI,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;oBACjE,IAAI,OAAO,GAAG,cAAc,EAAE,CAAC;wBAC7B,mEAAmE;wBACnE,GAAG,CAAC,MAAM,EAAE,CAAC;wBACb,GAAG,CAAC,OAAO,EAAE,CAAC;wBACd,GAAG,CAAC,OAAO,EAAE,CAAC;wBAEd,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,GAAG,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC;wBACvC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,yBAAyB,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;wBACjE,OAAO,CAAC,MAAM,CAAC,CAAC;wBAChB,OAAO;oBACT,CAAC;oBAED,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACZ,OAAO;gBACT,CAAC;gBAED,sBAAsB;gBACtB,IAAI,GAAG,CAAC,UAAU,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,IAAI,GAAG,CAAC,UAAU,IAAI,GAAG,CAAC,EAAE,CAAC;oBACtE,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,GAAG,CAAC,OAAO,EAAE,CAAC;oBACd,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;oBACZ,OAAO;gBACT,CAAC;gBAED,gCAAgC;gBAChC,MAAM,eAAe,GAAG,GAAG,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;gBAC1D,IAAI,eAAe,EAAE,CAAC;oBACpB,MAAM,cAAc,GAAG,QAAQ,CAC7B,KAAK,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,CAAC,eAAe,IAAI,GAAG,EACnF,EAAE,CACH,CAAC;oBACF,OAAO,CAAC,cAAc,CAAC,CAAC;gBAC1B,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;gBACd,CAAC;gBAED,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,mBAAmB;YACnC,CAAC,CAAC,CAAC;YAEH,GAAG,CAAC,GAAG,EAAE,CAAC;QACZ,CAAC,CAAC,CAAC;IACL,CAAC;CACF;AAED,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAAI,MAAyB;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,OAAO;YACjB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAwC;IAC/D,IAAI,CAAC;QACH,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC,CAAA,CAAC;AACpB,CAAC"}

package/dist/esm/v3/realtimeStreams/streamsWriterV2.d.ts

@@ -0,0 +1,60 @@
+import { StreamsWriter } from "./types.js";
+export type StreamsWriterV2Options<T = any> = {
+    basin: string;
+    stream: string;
+    accessToken: string;
+    source: ReadableStream<T>;
+    signal?: AbortSignal;
+    flushIntervalMs?: number;
+    maxRetries?: number;
+    debug?: boolean;
+    maxQueuedBytes?: number;
+};
+/**
+ * StreamsWriterV2 writes metadata stream data directly to S2 (https://s2.dev).
+ *
+ * Features:
+ * - Direct streaming: Uses S2's appendSession for efficient streaming
+ * - Automatic batching: Uses BatchTransform to batch records
+ * - No manual buffering: S2 handles buffering internally
+ * - Debug logging: Enable with debug: true to see detailed operation logs
+ *
+ * Example usage:
+ * ```typescript
+ * const stream = new StreamsWriterV2({
+ *   basin: "my-basin",
+ *   stream: "my-stream",
+ *   accessToken: "s2-token-here",
+ *   source: myAsyncIterable,
+ *   flushIntervalMs: 200, // Optional: batch linger duration in ms
+ *   debug: true, // Optional: enable debug logging
+ * });
+ *
+ * // Wait for streaming to complete
+ * await stream.wait();
+ *
+ * // Or consume the stream
+ * for await (const value of stream) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+export declare class StreamsWriterV2<T = any> implements StreamsWriter {
+    private options;
+    private s2Client;
+    private serverStream;
+    private consumerStream;
+    private streamPromise;
+    private readonly flushIntervalMs;
+    private readonly debug;
+    private readonly maxQueuedBytes;
+    private aborted;
+    private sessionWritable;
+    constructor(options: StreamsWriterV2Options<T>);
+    private handleAbort;
+    private initializeServerStream;
+    wait(): Promise<void>;
+    [Symbol.asyncIterator](): AsyncIterableIterator<T>;
+    private log;
+    private logError;
+}

package/dist/esm/v3/realtimeStreams/streamsWriterV2.js

@@ -0,0 +1,175 @@
+import { S2, AppendRecord, BatchTransform } from "@s2-dev/streamstore";
+import { nanoid } from "nanoid";
+/**
+ * StreamsWriterV2 writes metadata stream data directly to S2 (https://s2.dev).
+ *
+ * Features:
+ * - Direct streaming: Uses S2's appendSession for efficient streaming
+ * - Automatic batching: Uses BatchTransform to batch records
+ * - No manual buffering: S2 handles buffering internally
+ * - Debug logging: Enable with debug: true to see detailed operation logs
+ *
+ * Example usage:
+ * ```typescript
+ * const stream = new StreamsWriterV2({
+ *   basin: "my-basin",
+ *   stream: "my-stream",
+ *   accessToken: "s2-token-here",
+ *   source: myAsyncIterable,
+ *   flushIntervalMs: 200, // Optional: batch linger duration in ms
+ *   debug: true, // Optional: enable debug logging
+ * });
+ *
+ * // Wait for streaming to complete
+ * await stream.wait();
+ *
+ * // Or consume the stream
+ * for await (const value of stream) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+export class StreamsWriterV2 {
+    options;
+    s2Client;
+    serverStream;
+    consumerStream;
+    streamPromise;
+    flushIntervalMs;
+    debug;
+    maxQueuedBytes;
+    aborted = false;
+    sessionWritable = null;
+    constructor(options) {
+        this.options = options;
+        this.debug = options.debug ?? false;
+        this.s2Client = new S2({ accessToken: options.accessToken });
+        this.flushIntervalMs = options.flushIntervalMs ?? 200;
+        this.maxQueuedBytes = options.maxQueuedBytes ?? 1024 * 1024 * 10; // 10MB default
+        this.log(`[S2MetadataStream] Initializing: basin=${options.basin}, stream=${options.stream}, flushIntervalMs=${this.flushIntervalMs}, maxQueuedBytes=${this.maxQueuedBytes}`);
+        // Check if already aborted
+        if (options.signal?.aborted) {
+            this.aborted = true;
+            this.log("[S2MetadataStream] Signal already aborted, skipping initialization");
+            this.serverStream = new ReadableStream();
+            this.consumerStream = new ReadableStream();
+            this.streamPromise = Promise.resolve();
+            return;
+        }
+        // Set up abort signal handler
+        if (options.signal) {
+            options.signal.addEventListener("abort", () => {
+                this.log("[S2MetadataStream] Abort signal received");
+                this.handleAbort();
+            });
+        }
+        const [serverStream, consumerStream] = this.options.source.tee();
+        this.serverStream = serverStream;
+        this.consumerStream = consumerStream;
+        this.streamPromise = this.initializeServerStream();
+    }
+    handleAbort() {
+        if (this.aborted) {
+            return; // Already aborted
+        }
+        this.aborted = true;
+        this.log("[S2MetadataStream] Handling abort - cleaning up resources");
+        // Abort the writable stream if it exists
+        if (this.sessionWritable) {
+            this.sessionWritable
+                .abort("Aborted")
+                .catch((error) => {
+                this.logError("[S2MetadataStream] Error aborting writable stream:", error);
+            })
+                .finally(() => {
+                this.log("[S2MetadataStream] Writable stream aborted");
+            });
+        }
+        this.log("[S2MetadataStream] Abort cleanup complete");
+    }
+    async initializeServerStream() {
+        try {
+            if (this.aborted) {
+                this.log("[S2MetadataStream] Stream initialization aborted");
+                return;
+            }
+            this.log("[S2MetadataStream] Getting S2 basin and stream");
+            const basin = this.s2Client.basin(this.options.basin);
+            const stream = basin.stream(this.options.stream);
+            const session = await stream.appendSession({
+                maxQueuedBytes: this.maxQueuedBytes,
+            });
+            this.sessionWritable = session.writable;
+            this.log(`[S2MetadataStream] Starting stream pipeline`);
+            // Convert source stream to AppendRecord format and pipe to S2
+            await this.serverStream
+                .pipeThrough(new TransformStream({
+                transform: (chunk, controller) => {
+                    if (this.aborted) {
+                        controller.error(new Error("Stream aborted"));
+                        return;
+                    }
+                    // Convert each chunk to JSON string and wrap in AppendRecord
+                    controller.enqueue(AppendRecord.make(JSON.stringify({ data: chunk, id: nanoid(7) })));
+                },
+            }))
+                .pipeThrough(new BatchTransform({
+                lingerDurationMillis: this.flushIntervalMs,
+            }))
+                .pipeTo(session.writable);
+            this.log("[S2MetadataStream] Stream pipeline completed successfully");
+            // Get final position to verify completion
+            const lastAcked = session.lastAckedPosition();
+            if (lastAcked?.end) {
+                const recordsWritten = lastAcked.end.seq_num;
+                this.log(`[S2MetadataStream] Written ${recordsWritten} records, ending at seq_num=${lastAcked.end.seq_num}`);
+            }
+        }
+        catch (error) {
+            if (this.aborted) {
+                this.log("[S2MetadataStream] Stream error occurred but stream was aborted");
+                return;
+            }
+            this.logError("[S2MetadataStream] Error in stream pipeline:", error);
+            throw error;
+        }
+    }
+    async wait() {
+        await this.streamPromise;
+    }
+    [Symbol.asyncIterator]() {
+        return streamToAsyncIterator(this.consumerStream);
+    }
+    // Helper methods
+    log(message) {
+        if (this.debug) {
+            console.log(message);
+        }
+    }
+    logError(message, error) {
+        if (this.debug) {
+            console.error(message, error);
+        }
+    }
+}
+async function* streamToAsyncIterator(stream) {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            yield value;
+        }
+    }
+    finally {
+        safeReleaseLock(reader);
+    }
+}
+function safeReleaseLock(reader) {
+    try {
+        reader.releaseLock();
+    }
+    catch (error) { }
+}
+//# sourceMappingURL=streamsWriterV2.js.map

package/dist/esm/v3/realtimeStreams/streamsWriterV2.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamsWriterV2.js","sourceRoot":"","sources":["../../../../src/v3/realtimeStreams/streamsWriterV2.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAEvE,OAAO,EAAE,MAAM,EAAE,MAAM,QAAQ,CAAC;AAchC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH,MAAM,OAAO,eAAe;IAWN;IAVZ,QAAQ,CAAK;IACb,YAAY,CAAoB;IAChC,cAAc,CAAoB;IAClC,aAAa,CAAgB;IACpB,eAAe,CAAS;IACxB,KAAK,CAAU;IACf,cAAc,CAAS;IAChC,OAAO,GAAG,KAAK,CAAC;IAChB,eAAe,GAA+B,IAAI,CAAC;IAE3D,YAAoB,OAAkC;QAAlC,YAAO,GAAP,OAAO,CAA2B;QACpD,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,EAAE,CAAC,EAAE,WAAW,EAAE,OAAO,CAAC,WAAW,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,GAAG,CAAC;QACtD,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,eAAe;QAEjF,IAAI,CAAC,GAAG,CACN,0CAA0C,OAAO,CAAC,KAAK,YAAY,OAAO,CAAC,MAAM,qBAAqB,IAAI,CAAC,eAAe,oBAAoB,IAAI,CAAC,cAAc,EAAE,CACpK,CAAC;QAEF,2BAA2B;QAC3B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;YAC/E,IAAI,CAAC,YAAY,GAAG,IAAI,cAAc,EAAK,CAAC;YAC5C,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAK,CAAC;YAC9C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACvC,OAAO;QACT,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC5C,IAAI,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACrD,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACjE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;IACrD,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QAEtE,yCAAyC;QACzC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,IAAI,CAAC,eAAe;iBACjB,KAAK,CAAC,SAAS,CAAC;iBAChB,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC,QAAQ,CAAC,oDAAoD,EAAE,KAAK,CAAC,CAAC;YAC7E,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;gBAC7D,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;gBACzC,cAAc,EAAE,IAAI,CAAC,cAAc;aACpC,CAAC,CAAC;YAEH,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC;YAExC,IAAI,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAExD,8DAA8D;YAC9D,MAAM,IAAI,CAAC,YAAY;iBACpB,WAAW,CACV,IAAI,eAAe,CAAkB;gBACnC,SAAS,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;oBAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACjB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;wBAC9C,OAAO;oBACT,CAAC;oBACD,6DAA6D;oBAC7D,UAAU,CAAC,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC;gBACxF,CAAC;aACF,CAAC,CACH;iBACA,WAAW,CACV,IAAI,cAAc,CAAC;gBACjB,oBAAoB,EAAE,IAAI,CAAC,eAAe;aAC3C,CAAC,CACH;iBACA,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAE5B,IAAI,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;YAEtE,0CAA0C;YAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAE9C,IAAI,SAAS,EAAE,GAAG,EAAE,CAAC;gBACnB,MAAM,cAAc,GAAG,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC;gBAC7C,IAAI,CAAC,GAAG,CACN,8BAA8B,cAAc,+BAA+B,SAAS,CAAC,GAAG,CAAC,OAAO,EAAE,CACnG,CAAC;YACJ,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;gBAC5E,OAAO;YACT,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,aAAa,CAAC;IAC3B,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAED,iBAAiB;IAET,GAAG,CAAC,OAAe;QACzB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,OAAe,EAAE,KAAW;QAC3C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF;AAED,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAAI,MAAyB;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,OAAO;YACjB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAwC;IAC/D,IAAI,CAAC;QACH,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC,CAAA,CAAC;AACpB,CAAC"}