@trigger.dev/core 0.0.0-python-preview-20250226140121 → 0.0.0-rc-4-8-0-20261021151928

package/dist/commonjs/v3/realtimeStreams/streamsWriterV2.js

@@ -0,0 +1,254 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StreamsWriterV2 = void 0;
+exports.encodeChunkOrError = encodeChunkOrError;
+const streamstore_1 = require("@s2-dev/streamstore");
+const errors_js_1 = require("../errors.js");
+const nanoid_1 = require("nanoid");
+// S2 caps a single record at 1 MiB of metered bytes (body + headers + 8 byte
+// overhead). We give ourselves ~1 KiB of headroom for the JSON envelope and
+// metering bytes so the check fires before the SDK's internal `BatchTransform`
+// rejects the record with an opaque `S2Error`.
+const RECORD_BODY_MAX_BYTES = 1024 * 1024 - 1024;
+const utf8Encoder = new TextEncoder();
+/**
+ * StreamsWriterV2 writes metadata stream data directly to S2 (https://s2.dev).
+ *
+ * Features:
+ * - Direct streaming: Uses S2's appendSession for efficient streaming
+ * - Automatic batching: Uses BatchTransform to batch records
+ * - No manual buffering: S2 handles buffering internally
+ * - Debug logging: Enable with debug: true to see detailed operation logs
+ *
+ * Example usage:
+ * ```typescript
+ * const stream = new StreamsWriterV2({
+ *   basin: "my-basin",
+ *   stream: "my-stream",
+ *   accessToken: "s2-token-here",
+ *   source: myAsyncIterable,
+ *   flushIntervalMs: 200, // Optional: batch linger duration in ms
+ *   debug: true, // Optional: enable debug logging
+ * });
+ *
+ * // Wait for streaming to complete
+ * await stream.wait();
+ *
+ * // Or consume the stream
+ * for await (const value of stream) {
+ *   console.log(value);
+ * }
+ * ```
+ */
+class StreamsWriterV2 {
+    options;
+    s2Client;
+    serverStream;
+    consumerStream;
+    streamPromise;
+    flushIntervalMs;
+    debug;
+    maxInflightBytes;
+    aborted = false;
+    sessionWritable = null;
+    lastSeqNum;
+    constructor(options) {
+        this.options = options;
+        this.debug = options.debug ?? false;
+        this.s2Client = new streamstore_1.S2({
+            accessToken: options.accessToken,
+            ...(options.endpoint
+                ? {
+                    endpoints: {
+                        account: options.endpoint,
+                        basin: options.endpoint,
+                    },
+                }
+                : {}),
+        });
+        this.flushIntervalMs = options.flushIntervalMs ?? 200;
+        this.maxInflightBytes = options.maxInflightBytes ?? 1024 * 1024 * 10; // 10MB default
+        this.log(`[S2MetadataStream] Initializing: basin=${options.basin}, stream=${options.stream}, flushIntervalMs=${this.flushIntervalMs}, maxInflightBytes=${this.maxInflightBytes}`);
+        // Check if already aborted
+        if (options.signal?.aborted) {
+            this.aborted = true;
+            this.log("[S2MetadataStream] Signal already aborted, skipping initialization");
+            this.serverStream = new ReadableStream();
+            this.consumerStream = new ReadableStream();
+            this.streamPromise = Promise.resolve();
+            return;
+        }
+        // Set up abort signal handler
+        if (options.signal) {
+            options.signal.addEventListener("abort", () => {
+                this.log("[S2MetadataStream] Abort signal received");
+                this.handleAbort();
+            });
+        }
+        const [serverStream, consumerStream] = this.options.source.tee();
+        this.serverStream = serverStream;
+        this.consumerStream = consumerStream;
+        this.streamPromise = this.initializeServerStream();
+        // Detached writers (e.g. the head-start drain) call `wait()` long after a
+        // failed append rejects; without this the rejection is unhandled and Node
+        // takes the process down. `wait()` still surfaces the error.
+        this.streamPromise.catch(() => { });
+    }
+    handleAbort() {
+        if (this.aborted) {
+            return; // Already aborted
+        }
+        this.aborted = true;
+        this.log("[S2MetadataStream] Handling abort - cleaning up resources");
+        // Abort the writable stream if it exists
+        if (this.sessionWritable) {
+            this.sessionWritable
+                .abort("Aborted")
+                .catch((error) => {
+                this.logError("[S2MetadataStream] Error aborting writable stream:", error);
+            })
+                .finally(() => {
+                this.log("[S2MetadataStream] Writable stream aborted");
+            });
+        }
+        this.log("[S2MetadataStream] Abort cleanup complete");
+    }
+    async initializeServerStream() {
+        try {
+            if (this.aborted) {
+                this.log("[S2MetadataStream] Stream initialization aborted");
+                return;
+            }
+            this.log("[S2MetadataStream] Getting S2 basin and stream");
+            const basin = this.s2Client.basin(this.options.basin);
+            const stream = basin.stream(this.options.stream);
+            const session = await stream.appendSession({
+                maxInflightBytes: this.maxInflightBytes,
+            });
+            this.sessionWritable = session.writable;
+            this.log(`[S2MetadataStream] Starting stream pipeline`);
+            // Convert source stream to AppendRecord format and pipe to S2
+            await this.serverStream
+                .pipeThrough(new TransformStream({
+                transform: (chunk, controller) => {
+                    if (this.aborted) {
+                        controller.error(new Error("Stream aborted"));
+                        return;
+                    }
+                    const encoded = encodeChunkOrError(chunk);
+                    if (!encoded.ok) {
+                        controller.error(encoded.error);
+                        return;
+                    }
+                    controller.enqueue(streamstore_1.AppendRecord.string({ body: encoded.body }));
+                },
+            }))
+                .pipeThrough(new streamstore_1.BatchTransform({
+                lingerDurationMillis: this.flushIntervalMs,
+            }))
+                .pipeTo(session.writable);
+            this.log("[S2MetadataStream] Stream pipeline completed successfully");
+            // Get final position to verify completion
+            const lastAcked = session.lastAckedPosition();
+            if (lastAcked?.end) {
+                /**
+                 * S2's ack `end.seqNum` is exclusive: the seq AFTER the last record
+                 * written (equal to the tail). Report the last record's own seq
+                 * (`end - 1`) as `lastEventId` so a resume subscribe, which reads
+                 * from `lastEventId + 1`, lands on the next record instead of
+                 * skipping one. Matches the inclusive seq the one-shot control
+                 * writer returns.
+                 */
+                this.lastSeqNum = lastAcked.end.seqNum - 1;
+                this.log(`[S2MetadataStream] Wrote through seqNum=${this.lastSeqNum}`);
+            }
+        }
+        catch (error) {
+            if (this.aborted) {
+                this.log("[S2MetadataStream] Stream error occurred but stream was aborted");
+                return;
+            }
+            this.logError("[S2MetadataStream] Error in stream pipeline:", error);
+            throw error;
+        }
+    }
+    async wait() {
+        await this.streamPromise;
+        return { lastEventId: this.lastSeqNum?.toString() };
+    }
+    [Symbol.asyncIterator]() {
+        return streamToAsyncIterator(this.consumerStream);
+    }
+    // Helper methods
+    log(message) {
+        if (this.debug) {
+            console.log(message);
+        }
+    }
+    logError(message, error) {
+        if (this.debug) {
+            console.error(message, error);
+        }
+    }
+}
+exports.StreamsWriterV2 = StreamsWriterV2;
+async function* streamToAsyncIterator(stream) {
+    const reader = stream.getReader();
+    try {
+        while (true) {
+            const { done, value } = await reader.read();
+            if (done)
+                return;
+            yield value;
+        }
+    }
+    finally {
+        safeReleaseLock(reader);
+    }
+}
+function safeReleaseLock(reader) {
+    try {
+        reader.releaseLock();
+    }
+    catch (_error) { }
+}
+// chat.agent emits two chunk shapes through this writer:
+//   - UIMessageChunks + custom data parts: `{ type: "tool-output-available" | "data-..." | ... }`
+//   - ChatInputChunks (mostly seen on `.in`, but reused as the discriminant
+//     elsewhere): `{ kind: "message" | "stop" | "action" }`
+// Surfacing whichever discriminant exists turns "chunk too large" into
+// "tool-output-available chunk too large", which is what users actually need.
+function extractChunkType(chunk) {
+    if (!chunk || typeof chunk !== "object")
+        return undefined;
+    const c = chunk;
+    if (typeof c.type === "string")
+        return c.type;
+    if (typeof c.kind === "string")
+        return c.kind;
+    return undefined;
+}
+/**
+ * Encode a chunk as a JSON record body for S2, enforcing the per-record
+ * size cap. Exported so the size/discriminant logic can be unit-tested
+ * directly without spinning up an S2 client or mocking `@s2-dev/streamstore`.
+ *
+ * Returns `{ ok: true, body }` when the encoded chunk fits within
+ * `RECORD_BODY_MAX_BYTES`, or `{ ok: false, error }` carrying a
+ * `ChatChunkTooLargeError` annotated with the chunk's discriminant
+ * (`type` or `kind`, whichever is present) so the surfaced error is
+ * useful — "tool-output-available chunk too large" beats a bare
+ * "chunk too large" by a lot.
+ */
+function encodeChunkOrError(chunk) {
+    const body = JSON.stringify({ data: chunk, id: (0, nanoid_1.nanoid)(7) });
+    const size = utf8Encoder.encode(body).length;
+    if (size > RECORD_BODY_MAX_BYTES) {
+        return {
+            ok: false,
+            error: new errors_js_1.ChatChunkTooLargeError(size, RECORD_BODY_MAX_BYTES, extractChunkType(chunk)),
+        };
+    }
+    return { ok: true, body };
+}
+//# sourceMappingURL=streamsWriterV2.js.map

package/dist/commonjs/v3/realtimeStreams/streamsWriterV2.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"streamsWriterV2.js","sourceRoot":"","sources":["../../../../src/v3/realtimeStreams/streamsWriterV2.ts"],"names":[],"mappings":";;;;AAAA,qDAAuE;AACvE,4CAAsD;AAEtD,mCAAgC;AAEhC,6EAA6E;AAC7E,4EAA4E;AAC5E,+EAA+E;AAC/E,+CAA+C;AAC/C,MAAM,qBAAqB,GAAG,IAAI,GAAG,IAAI,GAAG,IAAI,CAAC;AAEjD,MAAM,WAAW,GAAG,IAAI,WAAW,EAAE,CAAC;AAetC;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4BG;AACH;IAYsB,OAAO;IAXnB,QAAQ,CAAK;IACb,YAAY,CAAoB;IAChC,cAAc,CAAoB;IAClC,aAAa,CAAgB;IACpB,eAAe,CAAS;IACxB,KAAK,CAAU;IACf,gBAAgB,CAAS;IAClC,OAAO,GAAG,KAAK,CAAC;IAChB,eAAe,GAA+B,IAAI,CAAC;IACnD,UAAU,CAAqB;IAEvC,YAAoB,OAAkC;uBAAlC,OAAO;QACzB,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;QACpC,IAAI,CAAC,QAAQ,GAAG,IAAI,gBAAE,CAAC;YACrB,WAAW,EAAE,OAAO,CAAC,WAAW;YAChC,GAAG,CAAC,OAAO,CAAC,QAAQ;gBAClB,CAAC,CAAC;oBACE,SAAS,EAAE;wBACT,OAAO,EAAE,OAAO,CAAC,QAAQ;wBACzB,KAAK,EAAE,OAAO,CAAC,QAAQ;qBACxB;iBACF;gBACH,CAAC,CAAC,EAAE,CAAC;SACR,CAAC,CAAC;QACH,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,IAAI,GAAG,CAAC;QACtD,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,gBAAgB,IAAI,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,CAAC,eAAe;QAErF,IAAI,CAAC,GAAG,CACN,0CAA0C,OAAO,CAAC,KAAK,YAAY,OAAO,CAAC,MAAM,qBAAqB,IAAI,CAAC,eAAe,sBAAsB,IAAI,CAAC,gBAAgB,EAAE,CACxK,CAAC;QAEF,2BAA2B;QAC3B,IAAI,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5B,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,oEAAoE,CAAC,CAAC;YAC/E,IAAI,CAAC,YAAY,GAAG,IAAI,cAAc,EAAK,CAAC;YAC5C,IAAI,CAAC,cAAc,GAAG,IAAI,cAAc,EAAK,CAAC;YAC9C,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,OAAO,EAAE,CAAC;YACvC,OAAO;QACT,CAAC;QAED,8BAA8B;QAC9B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;YACnB,OAAO,CAAC,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,GAAG,EAAE;gBAC5C,IAAI,CAAC,GAAG,CAAC,0CAA0C,CAAC,CAAC;gBACrD,IAAI,CAAC,WAAW,EAAE,CAAC;YACrB,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,CAAC,YAAY,EAAE,cAAc,CAAC,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QACjE,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QAErC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,sBAAsB,EAAE,CAAC;QACnD,0EAA0E;QAC1E,0EAA0E;QAC1E,6DAA6D;QAC7D,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IACrC,CAAC;IAEO,WAAW;QACjB,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;YACjB,OAAO,CAAC,kBAAkB;QAC5B,CAAC;QAED,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;QAEtE,yCAAyC;QACzC,IAAI,IAAI,CAAC,eAAe,EAAE,CAAC;YACzB,IAAI,CAAC,eAAe;iBACjB,KAAK,CAAC,SAAS,CAAC;iBAChB,KAAK,CAAC,CAAC,KAAK,EAAE,EAAE;gBACf,IAAI,CAAC,QAAQ,CAAC,oDAAoD,EAAE,KAAK,CAAC,CAAC;YAC7E,CAAC,CAAC;iBACD,OAAO,CAAC,GAAG,EAAE;gBACZ,IAAI,CAAC,GAAG,CAAC,4CAA4C,CAAC,CAAC;YACzD,CAAC,CAAC,CAAC;QACP,CAAC;QAED,IAAI,CAAC,GAAG,CAAC,2CAA2C,CAAC,CAAC;IACxD,CAAC;IAEO,KAAK,CAAC,sBAAsB;QAClC,IAAI,CAAC;YACH,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,kDAAkD,CAAC,CAAC;gBAC7D,OAAO;YACT,CAAC;YAED,IAAI,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACtD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;YAEjD,MAAM,OAAO,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC;gBACzC,gBAAgB,EAAE,IAAI,CAAC,gBAAgB;aACxC,CAAC,CAAC;YAEH,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC;YAExC,IAAI,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;YAExD,8DAA8D;YAC9D,MAAM,IAAI,CAAC,YAAY;iBACpB,WAAW,CACV,IAAI,eAAe,CAAkB;gBACnC,SAAS,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;oBAC/B,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;wBACjB,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC,CAAC;wBAC9C,OAAO;oBACT,CAAC;oBACD,MAAM,OAAO,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;oBAC1C,IAAI,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC;wBAChB,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;wBAChC,OAAO;oBACT,CAAC;oBACD,UAAU,CAAC,OAAO,CAAC,0BAAY,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,OAAO,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;gBAClE,CAAC;aACF,CAAC,CACH;iBACA,WAAW,CACV,IAAI,4BAAc,CAAC;gBACjB,oBAAoB,EAAE,IAAI,CAAC,eAAe;aAC3C,CAAC,CACH;iBACA,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;YAE5B,IAAI,CAAC,GAAG,CAAC,2DAA2D,CAAC,CAAC;YAEtE,0CAA0C;YAC1C,MAAM,SAAS,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;YAE9C,IAAI,SAAS,EAAE,GAAG,EAAE,CAAC;gBACnB;;;;;;;mBAOG;gBACH,IAAI,CAAC,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC;gBAC3C,IAAI,CAAC,GAAG,CAAC,2CAA2C,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YACzE,CAAC;QACH,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACjB,IAAI,CAAC,GAAG,CAAC,iEAAiE,CAAC,CAAC;gBAC5E,OAAO;YACT,CAAC;YACD,IAAI,CAAC,QAAQ,CAAC,8CAA8C,EAAE,KAAK,CAAC,CAAC;YACrE,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAEM,KAAK,CAAC,IAAI;QACf,MAAM,IAAI,CAAC,aAAa,CAAC;QACzB,OAAO,EAAE,WAAW,EAAE,IAAI,CAAC,UAAU,EAAE,QAAQ,EAAE,EAAE,CAAC;IACtD,CAAC;IAEM,CAAC,MAAM,CAAC,aAAa,CAAC;QAC3B,OAAO,qBAAqB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IACpD,CAAC;IAED,iBAAiB;IAET,GAAG,CAAC,OAAe;QACzB,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAEO,QAAQ,CAAC,OAAe,EAAE,KAAW;QAC3C,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAChC,CAAC;IACH,CAAC;CACF;;AAED,KAAK,SAAS,CAAC,CAAC,qBAAqB,CAAI,MAAyB;IAChE,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,EAAE,CAAC;IAClC,IAAI,CAAC;QACH,OAAO,IAAI,EAAE,CAAC;YACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;YAC5C,IAAI,IAAI;gBAAE,OAAO;YACjB,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;YAAS,CAAC;QACT,eAAe,CAAC,MAAM,CAAC,CAAC;IAC1B,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,MAAwC;IAC/D,IAAI,CAAC;QACH,MAAM,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAAC,OAAO,MAAM,EAAE,CAAC,CAAA,CAAC;AACrB,CAAC;AAED,yDAAyD;AACzD,kGAAkG;AAClG,4EAA4E;AAC5E,4DAA4D;AAC5D,uEAAuE;AACvE,8EAA8E;AAC9E,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,CAAC,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,SAAS,CAAC;IAC1D,MAAM,CAAC,GAAG,KAA2C,CAAC;IACtD,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC9C,IAAI,OAAO,CAAC,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC,IAAI,CAAC;IAC9C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;GAWG;AACH,4BACE,KAAc;IAEd,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,EAAE,EAAE,IAAA,eAAM,EAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC5D,MAAM,IAAI,GAAG,WAAW,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC7C,IAAI,IAAI,GAAG,qBAAqB,EAAE,CAAC;QACjC,OAAO;YACL,EAAE,EAAE,KAAK;YACT,KAAK,EAAE,IAAI,kCAAsB,CAAC,IAAI,EAAE,qBAAqB,EAAE,gBAAgB,CAAC,KAAK,CAAC,CAAC;SACxF,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC"}

package/dist/commonjs/v3/realtimeStreams/types.d.ts

@@ -0,0 +1,261 @@
+import type { AnyZodFetchOptions, ApiRequestOptions } from "../apiClient/core.js";
+import type { InputStreamOncePromise } from "../inputStreams/types.js";
+export { InputStreamOncePromise, InputStreamTimeoutError } from "../inputStreams/types.js";
+export type { InputStreamOnceResult } from "../inputStreams/types.js";
+import type { AsyncIterableStream } from "../streams/asyncIterableStream.js";
+import type { Prettify } from "../types/utils.js";
+import type { ManualWaitpointPromise } from "../waitpoints/index.js";
+export type RealtimeStreamOperationOptions = {
+    signal?: AbortSignal;
+    target?: string;
+    requestOptions?: AnyZodFetchOptions;
+};
+export interface RealtimeStreamsManager {
+    pipe<T>(key: string, source: AsyncIterable<T> | ReadableStream<T>, options?: RealtimeStreamOperationOptions): RealtimeStreamInstance<T>;
+    append<TPart extends BodyInit>(key: string, part: TPart, options?: RealtimeStreamOperationOptions): Promise<void>;
+}
+export type StreamWriteResult = {
+    lastEventId?: string;
+};
+export interface RealtimeStreamInstance<T> {
+    wait(): Promise<StreamWriteResult>;
+    get stream(): AsyncIterableStream<T>;
+}
+export interface StreamsWriter {
+    wait(): Promise<StreamWriteResult>;
+}
+export type RealtimeDefinedStream<TPart> = {
+    id: string;
+    pipe: (value: AsyncIterable<TPart> | ReadableStream<TPart>, options?: PipeStreamOptions) => PipeStreamResult<TPart>;
+    read: (runId: string, options?: ReadStreamOptions) => Promise<AsyncIterableStream<TPart>>;
+    append: (value: TPart, options?: AppendStreamOptions) => Promise<void>;
+    writer: (options: WriterStreamOptions<TPart>) => PipeStreamResult<TPart>;
+};
+export type InferStreamType<T> = T extends RealtimeDefinedStream<infer TPart> ? TPart : unknown;
+/**
+ * Options for appending data to a realtime stream.
+ */
+export type PipeStreamOptions = {
+    /**
+     * An AbortSignal that can be used to cancel the stream operation.
+     * If the signal is aborted, the stream will be closed.
+     */
+    signal?: AbortSignal;
+    /**
+     * The target run ID to pipe the stream to. Can be:
+     * - `"self"` - Pipe to the current run (default)
+     * - `"parent"` - Pipe to the parent run
+     * - `"root"` - Pipe to the root run
+     * - A specific run ID string
+     *
+     * If not provided and not called from within a task, an error will be thrown.
+     */
+    target?: string;
+    /**
+     * Additional request options for the API call.
+     */
+    requestOptions?: ApiRequestOptions;
+    /** Override the default span name for this operation. */
+    spanName?: string;
+    /** When true, the span will be collapsed in the dashboard. */
+    collapsed?: boolean;
+};
+/**
+ * The result of piping data to a realtime stream.
+ *
+ * @template T - The type of data chunks in the stream
+ */
+export type PipeStreamResult<T> = {
+    /**
+     * The original stream that was piped. You can consume this stream in your task
+     * to process the data locally while it's also being piped to the realtime stream.
+     */
+    stream: AsyncIterableStream<T>;
+    /**
+     * A function that returns a promise which resolves when all data has been piped
+     * to the realtime stream. Use this to wait for the stream to complete before
+     * finishing your task.
+     */
+    waitUntilComplete: () => Promise<StreamWriteResult>;
+};
+/**
+ * Options for reading data from a realtime stream.
+ */
+export type ReadStreamOptions = {
+    /**
+     * An AbortSignal that can be used to cancel the stream reading operation.
+     * If the signal is aborted, the stream will be closed.
+     */
+    signal?: AbortSignal;
+    /**
+     * The number of seconds to wait for new data to be available.
+     * If no data arrives within the timeout, the stream will be closed.
+     *
+     * @default 60 seconds
+     */
+    timeoutInSeconds?: number;
+    /**
+     * The index to start reading from (1-based).
+     * If not provided, the stream will start from the beginning.
+     * Use this to resume reading from a specific position.
+     *
+     * @default 0 (start from beginning)
+     */
+    startIndex?: number;
+    /**
+     * Where a fresh read starts.
+     *
+     * - `"beginning"` (default): replay the full stream history, then live-tail.
+     * - `"latest"`: skip history and start at the current tail — only records
+     *   appended after this read connects are delivered (a last-value / live view).
+     *
+     * Ignored when `startIndex` is set (which pins an absolute start position).
+     */
+    from?: "beginning" | "latest";
+};
+/**
+ * Options for appending data to a realtime stream.
+ */
+export type AppendStreamOptions = {
+    /**
+     * The target run ID to append the stream to. Can be:
+     * - `"self"` - Pipe to the current run (default)
+     * - `"parent"` - Pipe to the parent run
+     * - `"root"` - Pipe to the root run
+     * - A specific run ID string
+     *
+     * If not provided and not called from within a task, an error will be thrown.
+     */
+    target?: string;
+    /**
+     * Additional request options for the API call.
+     */
+    requestOptions?: ApiRequestOptions;
+};
+export type WriterStreamOptions<TPart> = Prettify<PipeStreamOptions & {
+    execute: (options: {
+        write: (part: TPart) => void;
+        merge(stream: ReadableStream<TPart>): void;
+    }) => Promise<void> | void;
+}>;
+/**
+ * A defined input stream that can receive typed data from external callers.
+ *
+ * Inside a task, use `.on()`, `.once()`, or `.peek()` to receive data.
+ * Outside a task, use `.send()` to send data to a running task.
+ */
+export type RealtimeDefinedInputStream<TData> = {
+    id: string;
+    /**
+     * Register a handler that fires every time data arrives on this input stream.
+     * Handlers are automatically cleaned up when the task run completes, so calling
+     * `.off()` is optional. Returns a subscription with `.off()` for early unsubscription.
+     */
+    on: (handler: (data: TData) => void | Promise<void>) => InputStreamSubscription;
+    /**
+     * Wait for the next piece of data on this input stream.
+     * Returns a result object `{ ok, output }` or `{ ok, error }`.
+     * Chain `.unwrap()` to get the data directly or throw on timeout.
+     */
+    once: (options?: InputStreamOnceOptions) => InputStreamOncePromise<TData>;
+    /**
+     * Non-blocking peek at the most recent data received on this input stream.
+     * Returns `undefined` if no data has been received yet.
+     */
+    peek: () => TData | undefined;
+    /**
+     * Suspend the task until data arrives on this input stream.
+     *
+     * Unlike `.once()` which keeps the task process alive while waiting,
+     * `.wait()` suspends the task entirely — freeing compute resources.
+     * The task resumes when data is sent via `.send()`.
+     *
+     * Uses a waitpoint token internally. Can only be called inside a task.run().
+     */
+    wait: (options?: InputStreamWaitOptions) => ManualWaitpointPromise<TData>;
+    /**
+     * Wait for data with an idle phase before suspending.
+     *
+     * Keeps the task active (using compute) for `idleTimeoutInSeconds`,
+     * then suspends via `.wait()` if no data arrives. If data arrives during
+     * the idle phase the task responds instantly without suspending.
+     */
+    waitWithIdleTimeout: (options: InputStreamWaitWithIdleTimeoutOptions) => Promise<{
+        ok: true;
+        output: TData;
+    } | {
+        ok: false;
+        error?: any;
+    }>;
+    /**
+     * Send data to this input stream on a specific run.
+     * This is used from outside the task (e.g., from your backend or another task).
+     */
+    send: (runId: string, data: TData, options?: SendInputStreamOptions) => Promise<void>;
+};
+export type InputStreamSubscription = {
+    off: () => void;
+};
+export type InputStreamOnceOptions = {
+    signal?: AbortSignal;
+    timeoutMs?: number;
+    /** Override the default span name for this operation. */
+    spanName?: string;
+};
+export type SendInputStreamOptions = {
+    requestOptions?: ApiRequestOptions;
+};
+export type InputStreamWaitOptions = {
+    /**
+     * Maximum time to wait before the waitpoint times out.
+     * Uses the same period format as `wait.createToken()`.
+     * If the timeout is reached, the result will be `{ ok: false, error }`.
+     *
+     * @example "30s", "5m", "1h", "24h", "7d"
+     */
+    timeout?: string;
+    /**
+     * Idempotency key for the underlying waitpoint token.
+     * If the same key is used again (and hasn't expired), the existing
+     * waitpoint is reused. This means if the task retries, it will
+     * resume waiting on the same waitpoint rather than creating a new one.
+     */
+    idempotencyKey?: string;
+    /**
+     * TTL for the idempotency key. After this period, the same key
+     * will create a new waitpoint.
+     */
+    idempotencyKeyTTL?: string;
+    /**
+     * Tags for the underlying waitpoint token, useful for querying
+     * and filtering waitpoints via `wait.listTokens()`.
+     */
+    tags?: string[];
+    /** Override the default span name for this operation. */
+    spanName?: string;
+};
+export type InputStreamWaitWithIdleTimeoutOptions = {
+    /** Seconds to keep the task idle (active, using compute) before suspending. */
+    idleTimeoutInSeconds: number;
+    /** Maximum time to wait after suspending (duration string, e.g. "1h"). */
+    timeout?: string;
+    /** Override the default span name for the outer operation. */
+    spanName?: string;
+    /** Called right before suspending (after idle phase times out). Not called if data arrives during idle. */
+    onSuspend?: () => void | Promise<void>;
+    /** Called right after resuming from suspension with data. Not called if data arrived during idle or on timeout. */
+    onResume?: () => void | Promise<void>;
+    /** When true, skip the suspend phase entirely. If idle times out, return `{ ok: false }` immediately. */
+    skipSuspend?: boolean;
+};
+export type InferInputStreamType<T> = T extends RealtimeDefinedInputStream<infer TData> ? TData : unknown;
+/**
+ * Internal record format for multiplexed input stream data on S2.
+ * All input streams for a run share a single S2 stream, demuxed by `stream` field.
+ */
+export type InputStreamRecord = {
+    stream: string;
+    data: unknown;
+    ts: number;
+    id: string;
+};

package/dist/commonjs/v3/realtimeStreams/types.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InputStreamTimeoutError = exports.InputStreamOncePromise = void 0;
+var types_js_1 = require("../inputStreams/types.js");
+Object.defineProperty(exports, "InputStreamOncePromise", { enumerable: true, get: function () { return types_js_1.InputStreamOncePromise; } });
+Object.defineProperty(exports, "InputStreamTimeoutError", { enumerable: true, get: function () { return types_js_1.InputStreamTimeoutError; } });
+//# sourceMappingURL=types.js.map

package/dist/commonjs/v3/realtimeStreams/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../../src/v3/realtimeStreams/types.ts"],"names":[],"mappings":";;;AAEA,qDAA2F;AAAlF,kHAAA,sBAAsB,OAAA;AAAE,mHAAA,uBAAuB,OAAA"}

package/dist/commonjs/v3/resource-catalog/catalog.d.ts

@@ -0,0 +1,41 @@
+import type { PromptManifest, QueueManifest, SkillManifest, SkillMetadata, TaskManifest, WebhookEndpointManifest, WebhookEndpointMetadata, WebhookSubscriberManifest, WorkerManifest, ConcurrencyLimitManifest } from "../schemas/index.js";
+import type { PromptMetadataWithFunctions, TaskMetadataWithFunctions, TaskSchema } from "../types/index.js";
+export interface ResourceCatalog {
+    setCurrentFileContext(filePath: string, entryPoint: string): void;
+    clearCurrentFileContext(): void;
+    registerTaskMetadata(task: TaskMetadataWithFunctions): void;
+    updateTaskMetadata(id: string, task: Partial<TaskMetadataWithFunctions>): void;
+    listTaskManifests(): Array<TaskManifest>;
+    listTaskIdCollisions(): Array<{
+        id: string;
+        filePaths: string[];
+    }>;
+    getTaskManifest(id: string): TaskManifest | undefined;
+    getTask(id: string): TaskMetadataWithFunctions | undefined;
+    taskExists(id: string): boolean;
+    registerWorkerManifest(workerManifest: WorkerManifest): void;
+    registerQueueMetadata(queue: QueueManifest): void;
+    listQueueManifests(): Array<QueueManifest>;
+    registerConcurrencyLimitMetadata(limit: ConcurrencyLimitManifest): void;
+    listConcurrencyLimitManifests(): Array<ConcurrencyLimitManifest>;
+    getTaskSchema(id: string): TaskSchema | undefined;
+    registerPromptMetadata(prompt: PromptMetadataWithFunctions): void;
+    listPromptManifests(): Array<PromptManifest>;
+    getPrompt(id: string): PromptMetadataWithFunctions | undefined;
+    getPromptSchema(id: string): TaskSchema | undefined;
+    registerSkillMetadata(skill: SkillMetadata): void;
+    listSkillManifests(): Array<SkillManifest>;
+    getSkillManifest(id: string): SkillManifest | undefined;
+    registerWebhookEndpointMetadata(endpoint: WebhookEndpointMetadata): void;
+    listWebhookEndpointManifests(): Array<WebhookEndpointManifest>;
+    getWebhookEndpointManifest(id: string): WebhookEndpointManifest | undefined;
+    listWebhookEndpointIdCollisions(): Array<{
+        id: string;
+        filePaths: string[];
+    }>;
+    registerWebhookSubscriber(subscriber: WebhookSubscriberManifest): void;
+    listWebhookSubscribers(): Array<WebhookSubscriberManifest>;
+    registerDeclaredSessionWebhook(id: string): void;
+    markSessionWebhookClaimed(id: string): void;
+    listUnclaimedSessionWebhooks(): Array<string>;
+}

package/dist/commonjs/v3/resource-catalog/catalog.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=catalog.js.map

package/dist/commonjs/v3/resource-catalog/catalog.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"catalog.js","sourceRoot":"","sources":["../../../../src/v3/resource-catalog/catalog.ts"],"names":[],"mappings":""}

package/dist/commonjs/v3/resource-catalog/index.d.ts

@@ -0,0 +1,48 @@
+import type { PromptManifest, QueueManifest, SkillManifest, SkillMetadata, TaskManifest, WebhookEndpointManifest, WebhookEndpointMetadata, WebhookSubscriberManifest, WorkerManifest, ConcurrencyLimitManifest } from "../schemas/index.js";
+import type { PromptMetadataWithFunctions, TaskMetadataWithFunctions, TaskSchema } from "../types/index.js";
+import { type ResourceCatalog } from "./catalog.js";
+export declare class ResourceCatalogAPI {
+    #private;
+    private static _instance?;
+    private constructor();
+    static getInstance(): ResourceCatalogAPI;
+    setGlobalResourceCatalog(resourceCatalog: ResourceCatalog): boolean;
+    disable(): void;
+    registerQueueMetadata(queue: QueueManifest): void;
+    registerTaskMetadata(task: TaskMetadataWithFunctions): void;
+    updateTaskMetadata(id: string, updates: Partial<TaskMetadataWithFunctions>): void;
+    setCurrentFileContext(filePath: string, entryPoint: string): void;
+    clearCurrentFileContext(): void;
+    registerWorkerManifest(workerManifest: WorkerManifest): void;
+    listTaskManifests(): Array<TaskManifest>;
+    listTaskIdCollisions(): Array<{
+        id: string;
+        filePaths: string[];
+    }>;
+    getTaskManifest(id: string): TaskManifest | undefined;
+    getTask(id: string): TaskMetadataWithFunctions | undefined;
+    getTaskSchema(id: string): TaskSchema | undefined;
+    taskExists(id: string): boolean;
+    listQueueManifests(): Array<QueueManifest>;
+    registerConcurrencyLimitMetadata(limit: ConcurrencyLimitManifest): void;
+    listConcurrencyLimitManifests(): Array<ConcurrencyLimitManifest>;
+    registerPromptMetadata(prompt: PromptMetadataWithFunctions): void;
+    listPromptManifests(): Array<PromptManifest>;
+    getPrompt(id: string): PromptMetadataWithFunctions | undefined;
+    getPromptSchema(id: string): TaskSchema | undefined;
+    registerSkillMetadata(skill: SkillMetadata): void;
+    listSkillManifests(): Array<SkillManifest>;
+    getSkillManifest(id: string): SkillManifest | undefined;
+    registerWebhookEndpointMetadata(endpoint: WebhookEndpointMetadata): void;
+    listWebhookEndpointManifests(): Array<WebhookEndpointManifest>;
+    getWebhookEndpointManifest(id: string): WebhookEndpointManifest | undefined;
+    listWebhookEndpointIdCollisions(): Array<{
+        id: string;
+        filePaths: string[];
+    }>;
+    registerWebhookSubscriber(subscriber: WebhookSubscriberManifest): void;
+    listWebhookSubscribers(): Array<WebhookSubscriberManifest>;
+    registerDeclaredSessionWebhook(id: string): void;
+    markSessionWebhookClaimed(id: string): void;
+    listUnclaimedSessionWebhooks(): Array<string>;
+}