node-llama-cpp 3.4.3 → 3.6.0

package/dist/evaluator/LlamaChat/LlamaChat.d.ts

@@ -1,12 +1,13 @@
 import { EventRelay } from "lifecycle-utils";
 import { ChatWrapper } from "../../ChatWrapper.js";
 import { LlamaContextSequence } from "../LlamaContext/LlamaContext.js";
-import { ChatHistoryItem, ChatModelFunctions, LLamaContextualRepeatPenalty, Token, Tokenizer } from "../../types.js";
+import { ChatHistoryItem, ChatModelFunctions, ChatModelSegmentType, LLamaContextualRepeatPenalty, Token, Tokenizer } from "../../types.js";
 import { GbnfJsonSchemaToType } from "../../utils/gbnfJson/types.js";
 import { LlamaGrammar } from "../LlamaGrammar.js";
 import { LlamaText, LlamaTextJSON } from "../../utils/LlamaText.js";
 import { EvaluationPriority } from "../LlamaContext/types.js";
 import { TokenBias } from "../TokenBias.js";
+import { LlamaModel } from "../LlamaModel/LlamaModel.js";
 export type LlamaChatOptions = {
     contextSequence: LlamaContextSequence;
     /** `"auto"` is used by default */
@@ -18,19 +19,84 @@ export type LlamaChatOptions = {
      */
     autoDisposeSequence?: boolean;
 };
+export type LlamaChatResponseChunk = LlamaChatResponseTextChunk | LlamaChatResponseSegmentChunk;
+export type LlamaChatResponseTextChunk = {
+    /** When `type` is `undefined`, the chunk is part of the main response and is not a segment */
+    type: undefined;
+    /**
+     * `segmentType` has no purpose when `type` is `undefined` (meaning that this chunk is part of the main response and is not a segment).
+     */
+    segmentType: undefined;
+    /**
+     * The generated text chunk.
+     *
+     * Detokenized from the `tokens` property,
+     * but with the context of the previous generation (for better spacing of the text with some models).
+     *
+     * Prefer using this property over `tokens` when streaming the generated response as text.
+     */
+    text: string;
+    /** The generated tokens */
+    tokens: Token[];
+};
+export type LlamaChatResponseSegmentChunk = {
+    type: "segment";
+    /** Segment type */
+    segmentType: ChatModelSegmentType;
+    /**
+     * The generated text chunk.
+     *
+     * Detokenized from the `tokens` property,
+     * but with the context of the previous generation (for better spacing of the text with some models).
+     *
+     * Prefer using this property over `tokens` when streaming the generated response as text.
+     */
+    text: string;
+    /** The generated tokens */
+    tokens: Token[];
+    /**
+     * When the current chunk is the start of a segment, this field will be set.
+     *
+     * It's possible that a chunk with no tokens and empty text will be emitted just to set this field
+     * to signify that the segment has started.
+     */
+    segmentStartTime?: Date;
+    /**
+     * When the current chunk is the last one of a segment (meaning the current segment has ended), this field will be set.
+     *
+     * It's possible that a chunk with no tokens and empty text will be emitted just to set this field
+     * to signify that the segment has ended.
+     */
+    segmentEndTime?: Date;
+};
 export type LLamaChatGenerateResponseOptions<Functions extends ChatModelFunctions | undefined = undefined> = {
     /**
-     * Called as the model generates a response with the generated text chunk.
+     * Called as the model generates the main response with the generated text chunk.
      *
      * Useful for streaming the generated response as it's being generated.
+     *
+     * Includes only the main response without any text segments (like thoughts).
+     * For streaming the response with segments, use {@link onResponseChunk `onResponseChunk`}.
      */
     onTextChunk?: (text: string) => void;
     /**
-     * Called as the model generates a response with the generated tokens.
+     * Called as the model generates the main response with the generated tokens.
      *
-     * Preferably, you'd want to use `onTextChunk` instead of this.
+     * Preferably, you'd want to use {@link onTextChunk `onTextChunk`} instead of this.
+     *
+     * Includes only the main response without any segments (like thoughts).
+     * For streaming the response with segments, use {@link onResponseChunk `onResponseChunk`}.
      */
     onToken?: (tokens: Token[]) => void;
+    /**
+     * Called as the model generates a response with the generated text and tokens,
+     * including segment information (when the generated output is part of a segment).
+     *
+     * Useful for streaming the generated response as it's being generated, including the main response and all segments.
+     *
+     * Only use this function when you need the segmented texts, like thought segments (chain of thought text).
+     */
+    onResponseChunk?: (chunk: LlamaChatResponseChunk) => void;
     signal?: AbortSignal;
     /**
      * When a response already started being generated and then the signal is aborted,
@@ -247,12 +313,19 @@ export declare class LlamaChat {
     get chatWrapper(): ChatWrapper;
     get sequence(): LlamaContextSequence;
     get context(): import("../LlamaContext/LlamaContext.js").LlamaContext;
-    get model(): import("../LlamaModel/LlamaModel.js").LlamaModel;
+    get model(): LlamaModel;
     generateResponse<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], options?: LLamaChatGenerateResponseOptions<Functions>): Promise<LlamaChatResponse<Functions>>;
     loadChatAndCompleteUserMessage<const Functions extends ChatModelFunctions | undefined = undefined>(history: ChatHistoryItem[], options?: LLamaChatLoadAndCompleteUserMessageOptions<Functions>): Promise<LlamaChatLoadAndCompleteUserResponse>;
 }
 export type LlamaChatResponse<Functions extends ChatModelFunctions | undefined = undefined> = {
+    /**
+     * The response text only, _without_ any text segments (like thoughts).
+     */
     response: string;
+    /**
+     * The full response, including all text and text segments (like thoughts).
+     */
+    fullResponse: Array<string | LlamaChatResponseSegment>;
     functionCalls?: Functions extends ChatModelFunctions ? LlamaChatResponseFunctionCall<Functions>[] : never;
     lastEvaluation: {
         cleanHistory: ChatHistoryItem[];
@@ -273,6 +346,15 @@ export type LlamaChatResponseFunctionCall<Functions extends ChatModelFunctions,
     params: Params;
     raw: LlamaTextJSON;
 };
+export type LlamaChatResponseSegment = {
+    type: "segment";
+    segmentType: ChatModelSegmentType;
+    text: string;
+    ended: boolean;
+    raw: LlamaTextJSON;
+    startTime?: string;
+    endTime?: string;
+};
 export type LlamaChatLoadAndCompleteUserResponse = {
     completion: string;
     lastEvaluation: {