@langchain/google-common 0.1.0 → 0.1.2

package/dist/utils/stream.cjs

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.ReadableJsonStream = exports.ComplexJsonStream = exports.JsonStream = exports.simpleValue = exports.complexValue = void 0;
+exports.ReadableSseJsonStream = exports.SseJsonStream = exports.ReadableSseStream = exports.SseStream = exports.ReadableJsonStream = exports.ReadableAbstractStream = exports.ComplexJsonStream = exports.JsonStream = exports.simpleValue = exports.complexValue = void 0;
 function complexValue(value) {
     if (value === null || typeof value === "undefined") {
         // I dunno what to put here. An error, probably
@@ -268,15 +268,21 @@ class ComplexJsonStream extends JsonStream {
     }
 }
 exports.ComplexJsonStream = ComplexJsonStream;
-class ReadableJsonStream extends JsonStream {
-    constructor(body) {
-        super();
+class ReadableAbstractStream {
+    constructor(baseStream, body) {
+        Object.defineProperty(this, "baseStream", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "decoder", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        this.baseStream = baseStream;
         this.decoder = new TextDecoder("utf-8");
         if (body) {
             void this.run(body);
@@ -285,6 +291,19 @@ class ReadableJsonStream extends JsonStream {
             console.error("Unexpected empty body while streaming");
         }
     }
+    appendBuffer(data) {
+        return this.baseStream.appendBuffer(data);
+    }
+    closeBuffer() {
+        return this.baseStream.closeBuffer();
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    nextChunk() {
+        return this.baseStream.nextChunk();
+    }
+    get streamDone() {
+        return this.baseStream.streamDone;
+    }
     async run(body) {
         const reader = body.getReader();
         let isDone = false;
@@ -301,4 +320,165 @@ class ReadableJsonStream extends JsonStream {
         }
     }
 }
+exports.ReadableAbstractStream = ReadableAbstractStream;
+class ReadableJsonStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new JsonStream(), body);
+    }
+}
 exports.ReadableJsonStream = ReadableJsonStream;
+class SseStream {
+    constructor() {
+        Object.defineProperty(this, "_buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ""
+        });
+        Object.defineProperty(this, "_bufferOpen", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        // Set up a potential Promise that the handler can resolve.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkResolution", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // If there is no Promise (it is null), the handler must add it to the queue
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkPending", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        // A queue that will collect chunks while there is no Promise
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkQueue", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+    }
+    appendBuffer(data) {
+        this._buffer += data;
+        this._parseBuffer();
+    }
+    closeBuffer() {
+        this._bufferOpen = false;
+        this._parseBuffer();
+    }
+    /**
+     * Attempt to load an entire event.
+     * For each entire event we load,
+     * send them to be handled.
+     */
+    _parseBuffer() {
+        const events = this._buffer.split(/\n\n/);
+        this._buffer = events.pop() ?? "";
+        events.forEach((event) => this._handleEvent(event.trim()));
+        if (!this._bufferOpen) {
+            // No more data will be added, and we have parsed
+            // everything. So dump the rest.
+            this._handleEvent(null);
+            this._buffer = "";
+        }
+    }
+    /**
+     * Given an event string, get all the fields
+     * in the event. It is assumed there is one field
+     * per line, but that field names can be duplicated,
+     * indicating to append the new value to the previous value
+     * @param event
+     */
+    _parseEvent(event) {
+        if (!event || event.trim() === "") {
+            return null;
+        }
+        const ret = {};
+        const lines = event.split(/\n/);
+        lines.forEach((line) => {
+            const match = line.match(/^([^:]+): \s*(.+)\n*$/);
+            if (match && match.length === 3) {
+                const key = match[1];
+                const val = match[2];
+                const cur = ret[key] ?? "";
+                ret[key] = `${cur}${val}`;
+            }
+        });
+        return ret;
+    }
+    _handleEvent(event) {
+        const chunk = this._parseEvent(event);
+        if (this._chunkPending) {
+            this._chunkResolution(chunk);
+            this._chunkPending = null;
+        }
+        else {
+            this._chunkQueue.push(chunk);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async nextChunk() {
+        if (this._chunkQueue.length > 0) {
+            // If there is data in the queue, return the next queue chunk
+            return this._chunkQueue.shift();
+        }
+        else {
+            // Otherwise, set up a promise that handleChunk will cause to be resolved
+            this._chunkPending = new Promise((resolve) => {
+                this._chunkResolution = resolve;
+            });
+            return this._chunkPending;
+        }
+    }
+    get streamDone() {
+        return (!this._bufferOpen &&
+            this._buffer.length === 0 &&
+            this._chunkQueue.length === 0 &&
+            this._chunkPending === null);
+    }
+}
+exports.SseStream = SseStream;
+class ReadableSseStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new SseStream(), body);
+    }
+}
+exports.ReadableSseStream = ReadableSseStream;
+class SseJsonStream extends SseStream {
+    constructor(jsonAttribute) {
+        super();
+        Object.defineProperty(this, "_jsonAttribute", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "data"
+        });
+        this._jsonAttribute = jsonAttribute ?? this._jsonAttribute;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async nextChunk() {
+        const eventRecord = (await super.nextChunk());
+        const json = eventRecord?.[this._jsonAttribute];
+        if (!json) {
+            return null;
+        }
+        else {
+            return JSON.parse(json);
+        }
+    }
+}
+exports.SseJsonStream = SseJsonStream;
+class ReadableSseJsonStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new SseJsonStream(), body);
+    }
+}
+exports.ReadableSseJsonStream = ReadableSseJsonStream;

package/dist/utils/stream.d.ts

@@ -1,6 +1,31 @@
+export interface AbstractStream {
+    /**
+     * Add more text to the buffer
+     * @param data
+     */
+    appendBuffer(data: string): void;
+    /**
+     * Indicate that there is no more text to be added to the buffer
+     * (ie - our source material is done)
+     */
+    closeBuffer(): void;
+    /**
+     * Get the next chunk that is coming from the stream.
+     * This chunk may be null, usually indicating the last chunk in the stream.
+     */
+    nextChunk(): Promise<any>;
+    /**
+     * Is the stream done?
+     * A stream is only done if all of the following are true:
+     * - There is no more data to be added to the text buffer
+     * - There is no more data in the text buffer
+     * - There are no chunks that are waiting to be consumed
+     */
+    get streamDone(): boolean;
+}
 export declare function complexValue(value: unknown): unknown;
 export declare function simpleValue(val: unknown): unknown;
-export declare class JsonStream {
+export declare class JsonStream implements AbstractStream {
     _buffer: string;
     _bufferOpen: boolean;
     _firstRun: boolean;
@@ -63,8 +88,53 @@ export declare class JsonStream {
 export declare class ComplexJsonStream extends JsonStream {
     _simplifyObject(obj: unknown): object;
 }
-export declare class ReadableJsonStream extends JsonStream {
+export declare class ReadableAbstractStream implements AbstractStream {
+    private baseStream;
     decoder: TextDecoder;
-    constructor(body: ReadableStream | null);
+    constructor(baseStream: AbstractStream, body: ReadableStream | null);
+    appendBuffer(data: string): void;
+    closeBuffer(): void;
+    nextChunk(): Promise<any>;
+    get streamDone(): boolean;
     run(body: ReadableStream): Promise<void>;
 }
+export declare class ReadableJsonStream extends ReadableAbstractStream {
+    constructor(body: ReadableStream | null);
+}
+export declare class SseStream implements AbstractStream {
+    _buffer: string;
+    _bufferOpen: boolean;
+    appendBuffer(data: string): void;
+    closeBuffer(): void;
+    /**
+     * Attempt to load an entire event.
+     * For each entire event we load,
+     * send them to be handled.
+     */
+    _parseBuffer(): void;
+    /**
+     * Given an event string, get all the fields
+     * in the event. It is assumed there is one field
+     * per line, but that field names can be duplicated,
+     * indicating to append the new value to the previous value
+     * @param event
+     */
+    _parseEvent(event: string | null): Record<string, string> | null;
+    _chunkResolution: (chunk: any) => void;
+    _chunkPending: Promise<any> | null;
+    _chunkQueue: any[];
+    _handleEvent(event: string | null): void;
+    nextChunk(): Promise<any>;
+    get streamDone(): boolean;
+}
+export declare class ReadableSseStream extends ReadableAbstractStream {
+    constructor(body: ReadableStream | null);
+}
+export declare class SseJsonStream extends SseStream {
+    _jsonAttribute: string;
+    constructor(jsonAttribute?: string);
+    nextChunk(): Promise<any>;
+}
+export declare class ReadableSseJsonStream extends ReadableAbstractStream {
+    constructor(body: ReadableStream | null);
+}

package/dist/utils/stream.js

@@ -261,15 +261,21 @@ export class ComplexJsonStream extends JsonStream {
         return simpleValue(obj);
     }
 }
-export class ReadableJsonStream extends JsonStream {
-    constructor(body) {
-        super();
+export class ReadableAbstractStream {
+    constructor(baseStream, body) {
+        Object.defineProperty(this, "baseStream", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
         Object.defineProperty(this, "decoder", {
             enumerable: true,
             configurable: true,
             writable: true,
             value: void 0
         });
+        this.baseStream = baseStream;
         this.decoder = new TextDecoder("utf-8");
         if (body) {
             void this.run(body);
@@ -278,6 +284,19 @@ export class ReadableJsonStream extends JsonStream {
             console.error("Unexpected empty body while streaming");
         }
     }
+    appendBuffer(data) {
+        return this.baseStream.appendBuffer(data);
+    }
+    closeBuffer() {
+        return this.baseStream.closeBuffer();
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    nextChunk() {
+        return this.baseStream.nextChunk();
+    }
+    get streamDone() {
+        return this.baseStream.streamDone;
+    }
     async run(body) {
         const reader = body.getReader();
         let isDone = false;
@@ -294,3 +313,159 @@ export class ReadableJsonStream extends JsonStream {
         }
     }
 }
+export class ReadableJsonStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new JsonStream(), body);
+    }
+}
+export class SseStream {
+    constructor() {
+        Object.defineProperty(this, "_buffer", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: ""
+        });
+        Object.defineProperty(this, "_bufferOpen", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: true
+        });
+        // Set up a potential Promise that the handler can resolve.
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkResolution", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        // If there is no Promise (it is null), the handler must add it to the queue
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkPending", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: null
+        });
+        // A queue that will collect chunks while there is no Promise
+        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+        Object.defineProperty(this, "_chunkQueue", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: []
+        });
+    }
+    appendBuffer(data) {
+        this._buffer += data;
+        this._parseBuffer();
+    }
+    closeBuffer() {
+        this._bufferOpen = false;
+        this._parseBuffer();
+    }
+    /**
+     * Attempt to load an entire event.
+     * For each entire event we load,
+     * send them to be handled.
+     */
+    _parseBuffer() {
+        const events = this._buffer.split(/\n\n/);
+        this._buffer = events.pop() ?? "";
+        events.forEach((event) => this._handleEvent(event.trim()));
+        if (!this._bufferOpen) {
+            // No more data will be added, and we have parsed
+            // everything. So dump the rest.
+            this._handleEvent(null);
+            this._buffer = "";
+        }
+    }
+    /**
+     * Given an event string, get all the fields
+     * in the event. It is assumed there is one field
+     * per line, but that field names can be duplicated,
+     * indicating to append the new value to the previous value
+     * @param event
+     */
+    _parseEvent(event) {
+        if (!event || event.trim() === "") {
+            return null;
+        }
+        const ret = {};
+        const lines = event.split(/\n/);
+        lines.forEach((line) => {
+            const match = line.match(/^([^:]+): \s*(.+)\n*$/);
+            if (match && match.length === 3) {
+                const key = match[1];
+                const val = match[2];
+                const cur = ret[key] ?? "";
+                ret[key] = `${cur}${val}`;
+            }
+        });
+        return ret;
+    }
+    _handleEvent(event) {
+        const chunk = this._parseEvent(event);
+        if (this._chunkPending) {
+            this._chunkResolution(chunk);
+            this._chunkPending = null;
+        }
+        else {
+            this._chunkQueue.push(chunk);
+        }
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async nextChunk() {
+        if (this._chunkQueue.length > 0) {
+            // If there is data in the queue, return the next queue chunk
+            return this._chunkQueue.shift();
+        }
+        else {
+            // Otherwise, set up a promise that handleChunk will cause to be resolved
+            this._chunkPending = new Promise((resolve) => {
+                this._chunkResolution = resolve;
+            });
+            return this._chunkPending;
+        }
+    }
+    get streamDone() {
+        return (!this._bufferOpen &&
+            this._buffer.length === 0 &&
+            this._chunkQueue.length === 0 &&
+            this._chunkPending === null);
+    }
+}
+export class ReadableSseStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new SseStream(), body);
+    }
+}
+export class SseJsonStream extends SseStream {
+    constructor(jsonAttribute) {
+        super();
+        Object.defineProperty(this, "_jsonAttribute", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: "data"
+        });
+        this._jsonAttribute = jsonAttribute ?? this._jsonAttribute;
+    }
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    async nextChunk() {
+        const eventRecord = (await super.nextChunk());
+        const json = eventRecord?.[this._jsonAttribute];
+        if (!json) {
+            return null;
+        }
+        else {
+            return JSON.parse(json);
+        }
+    }
+}
+export class ReadableSseJsonStream extends ReadableAbstractStream {
+    constructor(body) {
+        super(new SseJsonStream(), body);
+    }
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@langchain/google-common",
-  "version": "0.1.0",
+  "version": "0.1.2",
   "description": "Core types and classes for Google services.",
   "type": "module",
   "engines": {