@aigne/core 1.72.0 → 1.74.0-beta

package/lib/cjs/utils/stream-utils.js

@@ -1,235 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.objectToAgentResponseStream = objectToAgentResponseStream;
-exports.mergeAgentResponseChunk = mergeAgentResponseChunk;
-exports.agentResponseStreamToObject = agentResponseStreamToObject;
-exports.asyncGeneratorToReadableStream = asyncGeneratorToReadableStream;
-exports.onAgentResponseStreamEnd = onAgentResponseStreamEnd;
-exports.isAsyncGenerator = isAsyncGenerator;
-exports.arrayToAgentProcessAsyncGenerator = arrayToAgentProcessAsyncGenerator;
-exports.arrayToReadableStream = arrayToReadableStream;
-exports.readableStreamToArray = readableStreamToArray;
-exports.stringToAgentResponseStream = stringToAgentResponseStream;
-exports.toReadableStream = toReadableStream;
-exports.readAllString = readAllString;
-exports.mergeReadableStreams = mergeReadableStreams;
-const fast_deep_equal_1 = __importDefault(require("fast-deep-equal"));
-const agent_js_1 = require("../agents/agent.js");
-const type_utils_js_1 = require("./type-utils.js");
-require("./stream-polyfill.js");
-function objectToAgentResponseStream(json) {
-    if (!(0, type_utils_js_1.isRecord)(json)) {
-        throw new Error(`expect to return a record type such as {result: ...}, but got (${typeof json}): ${json}`);
-    }
-    return new ReadableStream({
-        pull(controller) {
-            controller.enqueue({ delta: { json } });
-            controller.close();
-        },
-    });
-}
-function mergeAgentResponseChunk(output, chunk) {
-    if ((0, agent_js_1.isAgentResponseDelta)(chunk)) {
-        if (chunk.delta.text) {
-            for (const [key, text] of Object.entries(chunk.delta.text)) {
-                const original = output[key];
-                const t = (original || "") + (text || "");
-                if (t)
-                    Object.assign(output, { [key]: t });
-            }
-        }
-        if (chunk.delta.json) {
-            Object.assign(output, (0, type_utils_js_1.omitBy)(chunk.delta.json, (v) => v === undefined));
-        }
-    }
-    return output;
-}
-async function agentResponseStreamToObject(stream) {
-    const json = {};
-    if (stream instanceof ReadableStream) {
-        for await (const value of stream) {
-            mergeAgentResponseChunk(json, value);
-        }
-    }
-    else {
-        for (;;) {
-            const chunk = await stream.next();
-            if (chunk.value) {
-                if (chunk.done) {
-                    Object.assign(json, chunk.value);
-                }
-                else {
-                    mergeAgentResponseChunk(json, chunk.value);
-                }
-            }
-            if (chunk.done)
-                break;
-        }
-    }
-    return json;
-}
-function asyncGeneratorToReadableStream(generator) {
-    return new ReadableStream({
-        async pull(controller) {
-            try {
-                const chunk = await generator.next();
-                if (chunk.value) {
-                    if (chunk.done) {
-                        controller.enqueue({ delta: { json: chunk.value } });
-                    }
-                    else {
-                        controller.enqueue(chunk.value);
-                    }
-                }
-                if (chunk.done) {
-                    controller.close();
-                }
-            }
-            catch (error) {
-                controller.error(error);
-            }
-        },
-    });
-}
-function onAgentResponseStreamEnd(stream, options) {
-    const json = {};
-    const reader = stream.getReader();
-    return new ReadableStream({
-        async pull(controller) {
-            try {
-                while (true) {
-                    const { value, done } = await reader.read();
-                    if (done) {
-                        const result = (await options?.onResult?.(json)) ?? json;
-                        if (result && !(0, fast_deep_equal_1.default)(result, json)) {
-                            let chunk = { delta: { json: result } };
-                            chunk = (await options?.onChunk?.(chunk)) ?? chunk;
-                            controller.enqueue(chunk);
-                        }
-                        controller.close();
-                        return;
-                    }
-                    mergeAgentResponseChunk(json, value);
-                    const chunk = (await options?.onChunk?.(value)) ?? value;
-                    if (!(0, agent_js_1.isEmptyChunk)(chunk)) {
-                        controller.enqueue(chunk);
-                        break;
-                    }
-                }
-            }
-            catch (error) {
-                controller.error((await options?.onError?.(error)) ?? error);
-            }
-        },
-    });
-}
-function isAsyncGenerator(value) {
-    return typeof value === "object" && value !== null && Symbol.asyncIterator in value;
-}
-async function* arrayToAgentProcessAsyncGenerator(chunks, result) {
-    for (const chunk of chunks) {
-        if (chunk instanceof Error)
-            throw chunk;
-        yield chunk;
-    }
-    if (result !== undefined)
-        return result;
-}
-function arrayToReadableStream(chunks) {
-    const list = [...chunks];
-    return new ReadableStream({
-        pull(controller) {
-            const item = list.shift();
-            if (!item) {
-                controller.close();
-                return;
-            }
-            if (item instanceof Error) {
-                controller.error(item);
-                return;
-            }
-            controller.enqueue(item);
-        },
-    });
-}
-async function readableStreamToArray(stream, options) {
-    const result = [];
-    try {
-        for await (const value of stream) {
-            result.push(value);
-        }
-    }
-    catch (error) {
-        if (!options?.catchError)
-            throw error;
-        result.push(error);
-    }
-    return result;
-}
-function stringToAgentResponseStream(str, key = "text") {
-    const segmenter = new Intl.Segmenter(undefined, { granularity: "word" });
-    const segments = segmenter.segment(str);
-    return arrayToReadableStream(Array.from(segments).map((segment) => ({ delta: { text: { [key]: segment.segment } } })));
-}
-function toReadableStream(stream) {
-    return new ReadableStream({
-        start(controller) {
-            const onData = (chunk) => {
-                controller.enqueue(new Uint8Array(chunk));
-            };
-            const onEnd = () => {
-                cleanup();
-                controller.close();
-            };
-            const onError = (err) => {
-                cleanup();
-                controller.error(err);
-            };
-            function cleanup() {
-                stream.off("data", onData);
-                stream.off("end", onEnd);
-                stream.off("error", onError);
-            }
-            stream.on("data", onData);
-            stream.on("end", onEnd);
-            stream.on("error", onError);
-        },
-    });
-}
-async function readAllString(stream) {
-    return (await readableStreamToArray((stream instanceof ReadableStream ? stream : toReadableStream(stream)).pipeThrough(new TextDecoderStream()))).join("");
-}
-function mergeReadableStreams(...streams) {
-    let readers;
-    return new ReadableStream({
-        async pull(controller) {
-            try {
-                readers ??= streams.filter(type_utils_js_1.isNonNullable).map((s) => ({ reader: s.getReader(), data: [] }));
-                while (readers.length) {
-                    const chunk = await Promise.race(readers.map((i) => {
-                        i.reading ??= i.reader.read().then((result) => ({ result, item: i }));
-                        return i.reading;
-                    }));
-                    if (chunk.result.value) {
-                        controller.enqueue(chunk.result.value);
-                        chunk.item.reading = undefined;
-                        return;
-                    }
-                    if (chunk.result.done) {
-                        readers = readers.filter((i) => i !== chunk.item);
-                    }
-                }
-                controller.close();
-            }
-            catch (error) {
-                controller.error(error);
-                if (readers)
-                    for (const item of readers)
-                        item.reader.releaseLock();
-            }
-        },
-    });
-}

package/lib/cjs/utils/structured-stream-extractor.d.ts

@@ -1,14 +0,0 @@
-import { type AgentResponseChunk } from "../agents/agent.js";
-import type { ChatModelOutput } from "../agents/chat-model.js";
-export declare class ExtractMetadataTransform extends TransformStream<AgentResponseChunk<ChatModelOutput>, AgentResponseChunk<ChatModelOutput & {
-    metadata?: string;
-}>> {
-    private buffer;
-    private cursor;
-    private state;
-    constructor({ start, end, parse, }: {
-        start: string;
-        end: string;
-        parse: (raw: string) => object;
-    });
-}

package/lib/cjs/utils/structured-stream-extractor.js

@@ -1,68 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ExtractMetadataTransform = void 0;
-const agent_js_1 = require("../agents/agent.js");
-class ExtractMetadataTransform extends TransformStream {
-    buffer = "";
-    cursor = 0;
-    state = "none";
-    constructor({ start, end, parse, }) {
-        super({
-            transform: async (chunk, controller) => {
-                if ((0, agent_js_1.isAgentResponseDelta)(chunk) && chunk.delta.text?.text) {
-                    const text = chunk.delta.text.text;
-                    this.buffer += text;
-                    for (;;) {
-                        if (this.state === "none") {
-                            const found = findMatchIndex(this.buffer, this.cursor, start);
-                            if (found.start > this.cursor) {
-                                let text = this.buffer.slice(this.cursor, found.start);
-                                this.cursor = found.start;
-                                // Trim trailing whitespace from the text
-                                const whitespace = text.match(/(?<whitespace>\s+)$/)?.groups?.whitespace;
-                                if (whitespace) {
-                                    text = text.slice(0, -whitespace.length);
-                                    this.cursor -= whitespace.length;
-                                }
-                                if (text) {
-                                    controller.enqueue({ delta: { text: { text } } });
-                                }
-                            }
-                            if (found.end) {
-                                this.state = "start";
-                                this.cursor = found.end;
-                            }
-                        }
-                        if (this.state === "start") {
-                            const found = findMatchIndex(this.buffer, this.cursor, end);
-                            if (found.end) {
-                                const metadata = this.buffer.slice(this.cursor, found.start);
-                                const json = parse(metadata);
-                                controller.enqueue({ delta: { json: { json } } });
-                                this.state = "none";
-                                this.cursor = found.end;
-                                continue;
-                            }
-                        }
-                        break;
-                    }
-                    return;
-                }
-                controller.enqueue(chunk);
-            },
-        });
-    }
-}
-exports.ExtractMetadataTransform = ExtractMetadataTransform;
-function findMatchIndex(str, position, match) {
-    const i = str.indexOf(match, position);
-    if (i >= 0)
-        return { start: i, end: i + match.length };
-    for (let i = match.length - 1; i > 0; i--) {
-        const m = match.slice(0, i);
-        if (str.endsWith(m)) {
-            return { start: str.length - m.length };
-        }
-    }
-    return { start: str.length };
-}

package/lib/cjs/utils/token-estimator.d.ts

@@ -1,9 +0,0 @@
-/**
- * Estimate tokens in text by analyzing character types
- * This function handles mixed-language text (Chinese and English) by counting
- * different character types and applying appropriate token ratios for each type
- *
- * @param text - The text to estimate
- * @returns Estimated token count
- */
-export declare function estimateTokens(text: string): number;

package/lib/cjs/utils/token-estimator.js

@@ -1,66 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.estimateTokens = estimateTokens;
-/**
- * Token estimation ratios for different character types
- * Based on empirical data from various tokenizers
- */
-const CHAR_TYPE_RATIOS = {
-    chinese: 1.5, // Chinese characters: ~1.5 characters per token
-    word: 0.75, // English words: ~0.75 tokens per word (accounting for subword tokenization)
-    other: 1, // Other characters (punctuation, numbers, etc.): ~1 character per token
-};
-/**
- * Regular expressions for character type detection
- */
-const CHAR_PATTERNS = {
-    // CJK characters (Chinese, Japanese Kanji, etc.)
-    chinese: /[\u4e00-\u9fff\u3400-\u4dbf\u3040-\u309f\u30a0-\u30ff]/g,
-    // English words (sequences of letters)
-    word: /[a-zA-Z]+/g,
-};
-/**
- * Estimate tokens in text by analyzing character types
- * This function handles mixed-language text (Chinese and English) by counting
- * different character types and applying appropriate token ratios for each type
- *
- * @param text - The text to estimate
- * @returns Estimated token count
- */
-function estimateTokens(text) {
-    if (!text)
-        return 0;
-    let tokens = 0;
-    const processedIndices = new Set();
-    // Count Chinese characters (including CJK)
-    const chineseMatches = text.match(CHAR_PATTERNS.chinese);
-    if (chineseMatches) {
-        tokens += chineseMatches.length / CHAR_TYPE_RATIOS.chinese;
-        // Mark processed positions
-        const chineseRegex = new RegExp(CHAR_PATTERNS.chinese.source, "g");
-        let match = chineseRegex.exec(text);
-        while (match !== null) {
-            processedIndices.add(match.index);
-            match = chineseRegex.exec(text);
-        }
-    }
-    // Count English words
-    const wordMatches = text.match(CHAR_PATTERNS.word);
-    if (wordMatches) {
-        tokens += wordMatches.length * CHAR_TYPE_RATIOS.word;
-        const wordRegex = new RegExp(CHAR_PATTERNS.word.source, "g");
-        let match = wordRegex.exec(text);
-        while (match !== null) {
-            for (let i = 0; i < match[0].length; i++) {
-                processedIndices.add(match.index + i);
-            }
-            match = wordRegex.exec(text);
-        }
-    }
-    // Count remaining characters (punctuation, numbers, whitespace, etc.)
-    const remainingChars = text.length - processedIndices.size;
-    if (remainingChars > 0) {
-        tokens += remainingChars / CHAR_TYPE_RATIOS.other;
-    }
-    return Math.ceil(tokens);
-}

package/lib/cjs/utils/type-utils.d.ts

@@ -1,31 +0,0 @@
-import { type ZodType, z } from "zod";
-export type PromiseOrValue<T> = T | Promise<T>;
-export type Nullish<T> = T | null | undefined | void;
-export type OmitPropertiesFromArrayFirstElement<T extends unknown[], K extends string | number | symbol> = T extends [infer U, ...infer Rest] ? [Omit<U, K>, ...Rest] : never;
-export type XOr<T, K extends keyof T, O extends keyof T> = (Omit<T, O> & {
-    [key in O]?: undefined;
-}) | (Omit<T, K> & {
-    [key in K]?: undefined;
-});
-export declare function isNil(value: unknown): value is null | undefined;
-export declare function isRecord<T>(value: unknown): value is Record<string, T>;
-export declare function isEmpty(obj: unknown): boolean;
-export declare function isNonNullable<T>(value: T): value is NonNullable<T>;
-export declare function isNotEmpty<T>(arr: T[]): arr is [T, ...T[]];
-export declare function duplicates<T>(arr: T[], key?: (item: T) => unknown): T[];
-export declare function remove<T>(arr: T[], remove: T[] | ((item: T) => boolean)): T[];
-export declare function unique<T>(arr: T[], key?: (item: T) => unknown): T[];
-export declare function get(obj: any, path: (string | number | symbol)[]): any;
-export declare function pick<T extends object, K extends keyof T>(obj: T, ...keys: (K | K[])[]): Pick<T, K>;
-export declare function omit<T extends object, K extends keyof T>(obj: T, ...keys: (K | K[])[]): Omit<T, K>;
-export declare function omitDeep<T, K>(obj: T, ...keys: (K | K[])[]): unknown;
-export declare function omitBy<T extends object, K extends keyof T>(obj: T, predicate: (value: T[K], key: K) => boolean): Partial<T>;
-export declare function omitByDeep(obj: any, predicate: (value: any, key: any) => boolean): any;
-export declare function flat<T>(...value: (T | T[])[]): NonNullable<T>[];
-export declare function createAccessorArray<T>(array: T[], accessor: (array: T[], name: string) => T | undefined): T[] & {
-    [key: string]: T;
-};
-export declare function checkArguments<T extends ZodType>(prefix: string, schema: T, args: unknown): z.infer<T>;
-export declare function tryOrThrow<P extends PromiseOrValue<unknown>>(fn: () => P, error: string | Error | ((error: Error) => Error)): P;
-export declare function tryOrThrow<P extends PromiseOrValue<unknown>>(fn: () => P, error?: Nullish<string | Error | ((error: Error) => Nullish<Error>)>): P | undefined;
-export declare function partition<T>(array: T[], predicate: (item: T) => boolean): [T[], T[]];

package/lib/cjs/utils/type-utils.js

@@ -1,202 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.isNil = isNil;
-exports.isRecord = isRecord;
-exports.isEmpty = isEmpty;
-exports.isNonNullable = isNonNullable;
-exports.isNotEmpty = isNotEmpty;
-exports.duplicates = duplicates;
-exports.remove = remove;
-exports.unique = unique;
-exports.get = get;
-exports.pick = pick;
-exports.omit = omit;
-exports.omitDeep = omitDeep;
-exports.omitBy = omitBy;
-exports.omitByDeep = omitByDeep;
-exports.flat = flat;
-exports.createAccessorArray = createAccessorArray;
-exports.checkArguments = checkArguments;
-exports.tryOrThrow = tryOrThrow;
-exports.partition = partition;
-const zod_1 = require("zod");
-function isNil(value) {
-    return value === null || value === undefined;
-}
-function isRecord(value) {
-    return !!value && typeof value === "object" && !Array.isArray(value);
-}
-function isEmpty(obj) {
-    if (isNil(obj))
-        return true;
-    if (typeof obj === "string" || Array.isArray(obj))
-        return obj.length === 0;
-    if (typeof obj === "object")
-        return Object.keys(obj).length === 0;
-    return false;
-}
-function isNonNullable(value) {
-    return !isNil(value);
-}
-function isNotEmpty(arr) {
-    return arr.length > 0;
-}
-function duplicates(arr, key = (item) => item) {
-    const seen = new Set();
-    const duplicates = new Set();
-    for (const item of arr) {
-        const k = key(item);
-        if (seen.has(k)) {
-            duplicates.add(item);
-        }
-        else {
-            seen.add(k);
-        }
-    }
-    return Array.from(duplicates);
-}
-function remove(arr, remove) {
-    const removed = [];
-    for (let i = 0; i < arr.length; i++) {
-        const item = arr[i];
-        if ((Array.isArray(remove) && remove.includes(item)) ||
-            (typeof remove === "function" && remove(item))) {
-            removed.push(...arr.splice(i, 1));
-            i--;
-        }
-    }
-    return removed;
-}
-function unique(arr, key = (item) => item) {
-    const seen = new Set();
-    return arr.filter((item) => {
-        const k = key(item);
-        if (seen.has(k)) {
-            return false;
-        }
-        seen.add(k);
-        return true;
-    });
-}
-function get(obj, path) {
-    return path.reduce((acc, key) => (isRecord(acc) ? Reflect.get(acc, key) : undefined), obj);
-}
-function pick(obj, ...keys) {
-    const flattenedKeys = new Set(keys.flat());
-    return Object.fromEntries(Object.entries(obj).filter(([key]) => flattenedKeys.has(key)));
-}
-function omit(obj, ...keys) {
-    const flattenedKeys = new Set(keys.flat());
-    return Object.fromEntries(Object.entries(obj).filter(([key]) => !flattenedKeys.has(key)));
-}
-function omitDeep(obj, ...keys) {
-    if (Array.isArray(obj)) {
-        return obj.map((item) => omitDeep(item, ...keys));
-    }
-    if (isRecord(obj)) {
-        const flattenedKeys = new Set(keys.flat());
-        return Object.fromEntries(Object.entries(obj)
-            .filter(([key]) => !flattenedKeys.has(key))
-            .map(([key, value]) => [key, omitDeep(value, ...keys)]));
-    }
-    return obj;
-}
-function omitBy(obj, predicate) {
-    return Object.fromEntries(Object.entries(obj).filter(([key, value]) => {
-        const k = key;
-        return !predicate(value, k);
-    }));
-}
-function omitByDeep(obj, predicate) {
-    if (obj === null || obj === undefined)
-        return obj;
-    if (Array.isArray(obj)) {
-        return obj.map((item) => omitByDeep(item, predicate));
-    }
-    if (typeof obj === "object") {
-        const result = {};
-        for (const [key, value] of Object.entries(obj)) {
-            const newValue = omitByDeep(value, predicate);
-            if (!predicate(newValue, key)) {
-                result[key] = newValue;
-            }
-        }
-        return result;
-    }
-    return obj;
-}
-function flat(...value) {
-    return value.flat().filter(isNonNullable);
-}
-function createAccessorArray(array, accessor) {
-    return new Proxy(array, {
-        get: (t, p, r) => Reflect.get(t, p, r) ?? accessor(array, p),
-    });
-}
-function checkArguments(prefix, schema, args) {
-    try {
-        return schema.parse(args);
-    }
-    catch (error) {
-        if (error instanceof zod_1.z.ZodError) {
-            const message = error.issues
-                .map((i) => {
-                const msg = i.code === "invalid_union"
-                    ? `${i.message} expect ${getExpectedTypes(i)} got ${get(args, i.path)}`
-                    : i.message;
-                return `${i.path}: ${msg}`;
-            })
-                .join(", ");
-            throw new Error(`${prefix} check arguments error: ${message}`);
-        }
-        throw error;
-    }
-}
-function getExpectedTypes(issue) {
-    if (Array.isArray(issue))
-        return issue.flatMap((issue) => getExpectedTypes(issue));
-    if ("expected" in issue)
-        return [String(issue.expected)];
-    if (issue.code === "invalid_union") {
-        return issue.unionErrors.flatMap((e) => getExpectedTypes(e.issues));
-    }
-    return [];
-}
-function tryOrThrow(fn, error) {
-    const createError = (e) => {
-        return typeof error === "function"
-            ? error(e)
-            : typeof error === "string"
-                ? new Error(error)
-                : error;
-    };
-    try {
-        const result = fn();
-        if (result instanceof Promise) {
-            return result.catch((e) => {
-                const error = createError(e);
-                if (error)
-                    throw error;
-            });
-        }
-        return result;
-    }
-    catch (e) {
-        const error = createError(e);
-        if (error)
-            throw error;
-    }
-}
-function partition(array, predicate) {
-    const pass = [];
-    const fail = [];
-    for (const item of array) {
-        if (predicate(item)) {
-            pass.push(item);
-        }
-        else {
-            fail.push(item);
-        }
-    }
-    return [pass, fail];
-}

package/lib/cjs/utils/typed-event-emitter.d.ts

@@ -1,10 +0,0 @@
-type EventMap<T> = Record<keyof T, unknown[]>;
-export type Args<K, T> = K extends keyof T ? T[K] : never;
-export type Listener<K, T> = K extends keyof T ? T[K] extends unknown[] ? (...args: T[K]) => void : never : never;
-export interface TypedEventEmitter<T extends EventMap<T>, E extends EventMap<E> = T> {
-    emit<K extends keyof E>(eventName: K, ...args: Args<K, E>): boolean;
-    on<K extends keyof T>(eventName: K, listener: Listener<K, T>): this;
-    once<K extends keyof T>(eventName: K, listener: Listener<K, T>): this;
-    off<K extends keyof T>(eventName: K, listener: Listener<K, T>): this;
-}
-export {};

package/lib/cjs/utils/typed-event-emitter.js

@@ -1,2 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });