@xsai/stream-object 0.0.25

package/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Moeru AI
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/index.d.ts

@@ -0,0 +1,149 @@
+import { Schema, Infer } from '@typeschema/main';
+import { StreamTextOptions, StreamTextResult } from '@xsai/stream-text';
+
+/**
+Matches any [primitive value](https://developer.mozilla.org/en-US/docs/Glossary/Primitive).
+
+@category Type
+*/
+type Primitive =
+	| null
+	| undefined
+	| string
+	| number
+	| boolean
+	| symbol
+	| bigint;
+
+declare global {
+	// eslint-disable-next-line @typescript-eslint/consistent-type-definitions -- It has to be an `interface` so that it can be merged.
+	interface SymbolConstructor {
+		readonly observable: symbol;
+	}
+}
+
+/**
+Matches any primitive, `void`, `Date`, or `RegExp` value.
+*/
+type BuiltIns = Primitive | void | Date | RegExp;
+
+/**
+@see PartialDeep
+*/
+type PartialDeepOptions = {
+	/**
+	Whether to affect the individual elements of arrays and tuples.
+
+	@default false
+	*/
+	readonly recurseIntoArrays?: boolean;
+};
+
+/**
+Create a type from another type with all keys and nested keys set to optional.
+
+Use-cases:
+- Merging a default settings/config object with another object, the second object would be a deep partial of the default object.
+- Mocking and testing complex entities, where populating an entire object with its keys would be redundant in terms of the mock or test.
+
+@example
+```
+import type {PartialDeep} from 'type-fest';
+
+const settings: Settings = {
+	textEditor: {
+		fontSize: 14;
+		fontColor: '#000000';
+		fontWeight: 400;
+	}
+	autocomplete: false;
+	autosave: true;
+};
+
+const applySavedSettings = (savedSettings: PartialDeep<Settings>) => {
+	return {...settings, ...savedSettings};
+}
+
+settings = applySavedSettings({textEditor: {fontWeight: 500}});
+```
+
+By default, this does not affect elements in array and tuple types. You can change this by passing `{recurseIntoArrays: true}` as the second type argument:
+
+```
+import type {PartialDeep} from 'type-fest';
+
+interface Settings {
+	languages: string[];
+}
+
+const partialSettings: PartialDeep<Settings, {recurseIntoArrays: true}> = {
+	languages: [undefined]
+};
+```
+
+@category Object
+@category Array
+@category Set
+@category Map
+*/
+type PartialDeep<T, Options extends PartialDeepOptions = {}> = T extends BuiltIns | (((...arguments_: any[]) => unknown)) | (new (...arguments_: any[]) => unknown)
+	? T
+	: T extends Map<infer KeyType, infer ValueType>
+		? PartialMapDeep<KeyType, ValueType, Options>
+		: T extends Set<infer ItemType>
+			? PartialSetDeep<ItemType, Options>
+			: T extends ReadonlyMap<infer KeyType, infer ValueType>
+				? PartialReadonlyMapDeep<KeyType, ValueType, Options>
+				: T extends ReadonlySet<infer ItemType>
+					? PartialReadonlySetDeep<ItemType, Options>
+					: T extends object
+						? T extends ReadonlyArray<infer ItemType> // Test for arrays/tuples, per https://github.com/microsoft/TypeScript/issues/35156
+							? Options['recurseIntoArrays'] extends true
+								? ItemType[] extends T // Test for arrays (non-tuples) specifically
+									? readonly ItemType[] extends T // Differentiate readonly and mutable arrays
+										? ReadonlyArray<PartialDeep<ItemType | undefined, Options>>
+										: Array<PartialDeep<ItemType | undefined, Options>>
+									: PartialObjectDeep<T, Options> // Tuples behave properly
+								: T // If they don't opt into array testing, just use the original type
+							: PartialObjectDeep<T, Options>
+						: unknown;
+
+/**
+Same as `PartialDeep`, but accepts only `Map`s and as inputs. Internal helper for `PartialDeep`.
+*/
+type PartialMapDeep<KeyType, ValueType, Options extends PartialDeepOptions> = {} & Map<PartialDeep<KeyType, Options>, PartialDeep<ValueType, Options>>;
+
+/**
+Same as `PartialDeep`, but accepts only `Set`s as inputs. Internal helper for `PartialDeep`.
+*/
+type PartialSetDeep<T, Options extends PartialDeepOptions> = {} & Set<PartialDeep<T, Options>>;
+
+/**
+Same as `PartialDeep`, but accepts only `ReadonlyMap`s as inputs. Internal helper for `PartialDeep`.
+*/
+type PartialReadonlyMapDeep<KeyType, ValueType, Options extends PartialDeepOptions> = {} & ReadonlyMap<PartialDeep<KeyType, Options>, PartialDeep<ValueType, Options>>;
+
+/**
+Same as `PartialDeep`, but accepts only `ReadonlySet`s as inputs. Internal helper for `PartialDeep`.
+*/
+type PartialReadonlySetDeep<T, Options extends PartialDeepOptions> = {} & ReadonlySet<PartialDeep<T, Options>>;
+
+/**
+Same as `PartialDeep`, but accepts only `object`s as inputs. Internal helper for `PartialDeep`.
+*/
+type PartialObjectDeep<ObjectType extends object, Options extends PartialDeepOptions> = {
+	[KeyType in keyof ObjectType]?: PartialDeep<ObjectType[KeyType], Options>
+};
+
+interface StreamObjectOptions<T extends Schema> extends StreamTextOptions {
+    schema: T;
+    schemaDescription?: string;
+    schemaName?: string;
+}
+interface StreamObjectResult<T extends Schema> extends StreamTextResult {
+    partialObjectStream: ReadableStream<PartialDeep<Infer<T>>>;
+}
+/** @experimental WIP */
+declare const streamObject: <T extends Schema>(options: StreamObjectOptions<T>) => Promise<StreamObjectResult<T>>;
+
+export { type StreamObjectOptions, type StreamObjectResult, streamObject as default, streamObject };

package/dist/index.js

@@ -0,0 +1,271 @@
+import { toJSONSchema } from '@typeschema/main';
+import { clean } from '@xsai/shared';
+import { streamText } from '@xsai/stream-text';
+
+var parse = {};
+
+(function (exports) {
+	Object.defineProperty(exports, "__esModule", { value: true });
+	exports.parse = void 0;
+	function parse(s) {
+	    if (s === undefined) {
+	        return undefined;
+	    }
+	    if (s === null) {
+	        return null;
+	    }
+	    if (s === '') {
+	        return '';
+	    }
+	    // remove incomplete escaped characters at the end of the string
+	    s = s.replace(/\\+$/, match => match.length % 2 === 0 ? match : match.slice(0, -1));
+	    try {
+	        return JSON.parse(s);
+	    }
+	    catch (e) {
+	        const [data, reminding] = s.trimLeft()[0] === ':'
+	            ? parseAny(s, e)
+	            : parseAny(s, e, parseStringWithoutQuote);
+	        parse.lastParseReminding = reminding;
+	        if (parse.onExtraToken && reminding.length > 0) {
+	            parse.onExtraToken(s, data, reminding);
+	        }
+	        return data;
+	    }
+	}
+	exports.parse = parse;
+	(function (parse) {
+	    parse.onExtraToken = (text, data, reminding) => {
+	        console.error('parsed json with extra tokens:', {
+	            text,
+	            data,
+	            reminding,
+	        });
+	    };
+	})(parse = exports.parse || (exports.parse = {}));
+	function parseAny(s, e, fallback) {
+	    const parser = parsers[s[0]] || fallback;
+	    if (!parser) {
+	        console.error(`no parser registered for ${JSON.stringify(s[0])}:`, { s });
+	        throw e;
+	    }
+	    return parser(s, e);
+	}
+	function parseStringCasual(s, e, delimiters) {
+	    if (s[0] === '"') {
+	        return parseString(s);
+	    }
+	    if (s[0] === "'") {
+	        return parseSingleQuoteString(s);
+	    }
+	    return parseStringWithoutQuote(s, e, delimiters);
+	}
+	const parsers = {};
+	function skipSpace(s) {
+	    return s.trimLeft();
+	}
+	parsers[' '] = parseSpace;
+	parsers['\r'] = parseSpace;
+	parsers['\n'] = parseSpace;
+	parsers['\t'] = parseSpace;
+	function parseSpace(s, e) {
+	    s = skipSpace(s);
+	    return parseAny(s, e);
+	}
+	parsers['['] = parseArray;
+	function parseArray(s, e) {
+	    s = s.substr(1); // skip starting '['
+	    const acc = [];
+	    s = skipSpace(s);
+	    for (; s.length > 0;) {
+	        if (s[0] === ']') {
+	            s = s.substr(1); // skip ending ']'
+	            break;
+	        }
+	        const res = parseAny(s, e, (s, e) => parseStringWithoutQuote(s, e, [',', ']']));
+	        acc.push(res[0]);
+	        s = res[1];
+	        s = skipSpace(s);
+	        if (s[0] === ',') {
+	            s = s.substring(1);
+	            s = skipSpace(s);
+	        }
+	    }
+	    return [acc, s];
+	}
+	for (const c of '0123456789.-'.slice()) {
+	    parsers[c] = parseNumber;
+	}
+	function parseNumber(s) {
+	    for (let i = 0; i < s.length; i++) {
+	        const c = s[i];
+	        if (parsers[c] === parseNumber) {
+	            continue;
+	        }
+	        const num = s.substring(0, i);
+	        s = s.substring(i);
+	        return [numToStr(num), s];
+	    }
+	    return [numToStr(s), ''];
+	}
+	function numToStr(s) {
+	    if (s === '-') {
+	        return -0;
+	    }
+	    const num = +s;
+	    if (Number.isNaN(num)) {
+	        return s;
+	    }
+	    return num;
+	}
+	parsers['"'] = parseString;
+	function parseString(s) {
+	    for (let i = 1; i < s.length; i++) {
+	        const c = s[i];
+	        if (c === '\\') {
+	            i++;
+	            continue;
+	        }
+	        if (c === '"') {
+	            const str = fixEscapedCharacters(s.substring(0, i + 1));
+	            s = s.substring(i + 1);
+	            return [JSON.parse(str), s];
+	        }
+	    }
+	    return [JSON.parse(fixEscapedCharacters(s) + '"'), ''];
+	}
+	function fixEscapedCharacters(s) {
+	    return s.replace(/\n/g, '\\n').replace(/\t/g, '\\t').replace(/\r/g, '\\r');
+	}
+	parsers["'"] = parseSingleQuoteString;
+	function parseSingleQuoteString(s) {
+	    for (let i = 1; i < s.length; i++) {
+	        const c = s[i];
+	        if (c === '\\') {
+	            i++;
+	            continue;
+	        }
+	        if (c === "'") {
+	            const str = fixEscapedCharacters(s.substring(0, i + 1));
+	            s = s.substring(i + 1);
+	            return [JSON.parse('"' + str.slice(1, -1) + '"'), s];
+	        }
+	    }
+	    return [JSON.parse('"' + fixEscapedCharacters(s.slice(1)) + '"'), ''];
+	}
+	function parseStringWithoutQuote(s, e, delimiters = [' ']) {
+	    const index = Math.min(...delimiters.map(delimiter => {
+	        const index = s.indexOf(delimiter);
+	        return index === -1 ? s.length : index;
+	    }));
+	    const value = s.substring(0, index).trim();
+	    const rest = s.substring(index);
+	    return [value, rest];
+	}
+	parsers['{'] = parseObject;
+	function parseObject(s, e) {
+	    s = s.substr(1); // skip starting '{'
+	    const acc = {};
+	    s = skipSpace(s);
+	    for (; s.length > 0;) {
+	        if (s[0] === '}') {
+	            s = s.substr(1); // skip ending '}'
+	            break;
+	        }
+	        const keyRes = parseStringCasual(s, e, [':', '}']);
+	        const key = keyRes[0];
+	        s = keyRes[1];
+	        s = skipSpace(s);
+	        if (s[0] !== ':') {
+	            acc[key] = undefined;
+	            break;
+	        }
+	        s = s.substr(1); // skip ':'
+	        s = skipSpace(s);
+	        if (s.length === 0) {
+	            acc[key] = undefined;
+	            break;
+	        }
+	        const valueRes = parseAny(s, e);
+	        acc[key] = valueRes[0];
+	        s = valueRes[1];
+	        s = skipSpace(s);
+	        if (s[0] === ',') {
+	            s = s.substr(1);
+	            s = skipSpace(s);
+	        }
+	    }
+	    return [acc, s];
+	}
+	parsers['t'] = parseTrue;
+	function parseTrue(s, e) {
+	    return parseToken(s, `true`, true, e);
+	}
+	parsers['f'] = parseFalse;
+	function parseFalse(s, e) {
+	    return parseToken(s, `false`, false, e);
+	}
+	parsers['n'] = parseNull;
+	function parseNull(s, e) {
+	    return parseToken(s, `null`, null, e);
+	}
+	function parseToken(s, tokenStr, tokenVal, e) {
+	    for (let i = tokenStr.length; i >= 1; i--) {
+	        if (s.startsWith(tokenStr.slice(0, i))) {
+	            return [tokenVal, s.slice(i)];
+	        }
+	    }
+	    /* istanbul ignore next */
+	    {
+	        const prefix = JSON.stringify(s.slice(0, tokenStr.length));
+	        console.error(`unknown token starting with ${prefix}:`, { s });
+	        throw e;
+	    }
+	}
+	
+} (parse));
+
+const streamObject = async (options) => await streamText({
+  ...options,
+  response_format: {
+    json_schema: {
+      description: options.schemaDescription,
+      name: options.schemaName ?? "json_schema",
+      schema: await toJSONSchema(options.schema).then((json) => clean({
+        ...json,
+        $schema: void 0
+      })),
+      strict: true
+    },
+    type: "json_schema"
+  },
+  schema: void 0,
+  schemaDescription: void 0,
+  schemaName: void 0
+}).then(({ chunkStream, finishReason, textStream: rawTextStream, usage }) => {
+  const [textStream, rawPartialObjectStream] = rawTextStream.tee();
+  let partialObjectData = "";
+  let partialObjectSnapshot = {};
+  const partialObjectStream = rawPartialObjectStream.pipeThrough(new TransformStream({
+    transform: (chunk, controller) => {
+      partialObjectData += chunk;
+      try {
+        const data = parse.parse(partialObjectData);
+        if (JSON.stringify(partialObjectSnapshot) !== JSON.stringify(data)) {
+          partialObjectSnapshot = data;
+          controller.enqueue(data);
+        }
+      } catch {
+      }
+    }
+  }));
+  return {
+    chunkStream,
+    finishReason,
+    partialObjectStream,
+    textStream,
+    usage
+  };
+});
+
+export { streamObject as default, streamObject };

package/package.json

@@ -0,0 +1,50 @@
+{
+  "name": "@xsai/stream-object",
+  "version": "0.0.25",
+  "type": "module",
+  "author": "Moeru AI",
+  "license": "MIT",
+  "homepage": "https://xsai.js.org",
+  "description": "extra-small AI SDK for Browser, Node.js, Deno, Bun or Edge Runtime.",
+  "keywords": [
+    "xsai",
+    "openai",
+    "ai"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/moeru-ai/xsai.git",
+    "directory": "packages/stream-object"
+  },
+  "bugs": "https://github.com/moeru-ai/xsai/issues",
+  "sideEffects": false,
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist"
+  ],
+  "dependencies": {
+    "@typeschema/main": "^0.14.1",
+    "@xsai/shared": "",
+    "@xsai/stream-text": ""
+  },
+  "devDependencies": {
+    "@gcornut/valibot-json-schema": "^0.42.0",
+    "@xsai/providers": "",
+    "best-effort-json-parser": "^1.1.2",
+    "type-fest": "^4.31.0",
+    "valibot": "^0.42.1"
+  },
+  "scripts": {
+    "build": "pkgroll",
+    "build:watch": "pkgroll --watch",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  }
+}