node-llama-cpp 3.0.0-beta.14 → 3.0.0-beta.15

package/dist/index.d.ts

@@ -2,7 +2,7 @@ import { DisposedError } from "lifecycle-utils";
 import { Llama } from "./bindings/Llama.js";
 import { getLlama, LlamaOptions } from "./bindings/getLlama.js";
 import { NoBinaryFoundError } from "./bindings/utils/NoBinaryFoundError.js";
-import { LlamaLogLevel } from "./bindings/types.js";
+import { LlamaLogLevel, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, LlamaVocabularyType } from "./bindings/types.js";
 import { LlamaModel, LlamaModelInfillTokens, type LlamaModelOptions, LlamaModelTokens } from "./evaluator/LlamaModel.js";
 import { LlamaGrammar, type LlamaGrammarOptions } from "./evaluator/LlamaGrammar.js";
 import { LlamaJsonSchemaGrammar } from "./evaluator/LlamaJsonSchemaGrammar.js";
@@ -10,12 +10,14 @@ import { LlamaJsonSchemaValidationError } from "./utils/gbnfJson/utils/validateO
 import { LlamaGrammarEvaluationState, LlamaGrammarEvaluationStateOptions } from "./evaluator/LlamaGrammarEvaluationState.js";
 import { LlamaContext, LlamaContextSequence } from "./evaluator/LlamaContext/LlamaContext.js";
 import { LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON } from "./evaluator/LlamaEmbeddingContext.js";
-import { type LlamaContextOptions, type BatchingOptions, type LlamaContextSequenceRepeatPenalty, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizeStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority } from "./evaluator/LlamaContext/types.js";
+import { type LlamaContextOptions, type BatchingOptions, type LlamaContextSequenceRepeatPenalty, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority } from "./evaluator/LlamaContext/types.js";
 import { LlamaChatSession, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty } from "./evaluator/LlamaChatSession/LlamaChatSession.js";
 import { defineChatSessionFunction } from "./evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
 import { LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LlamaChatResponse, type LlamaChatResponseFunctionCall } from "./evaluator/LlamaChat/LlamaChat.js";
 import { LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions } from "./evaluator/LlamaCompletion.js";
+import { TokenMeter, type TokenMeterState } from "./evaluator/TokenMeter.js";
 import { UnsupportedError } from "./utils/UnsupportedError.js";
+import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper, type ChatWrapperSettings } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
 import { LlamaChatWrapper } from "./chatWrappers/LlamaChatWrapper.js";
@@ -25,12 +27,16 @@ import { FalconChatWrapper } from "./chatWrappers/FalconChatWrapper.js";
 import { AlpacaChatWrapper } from "./chatWrappers/AlpacaChatWrapper.js";
 import { FunctionaryChatWrapper } from "./chatWrappers/FunctionaryChatWrapper.js";
 import { GemmaChatWrapper } from "./chatWrappers/GemmaChatWrapper.js";
-import { TemplateChatWrapper, type TemplateChatWrapperOptions } from "./TemplateChatWrapper.js";
-import { resolveChatWrapperBasedOnModel } from "./chatWrappers/resolveChatWrapperBasedOnModel.js";
-import { resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, type ChatWrapperTypeName } from "./bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
-import { LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokenJSON } from "./utils/LlamaText.js";
+import { TemplateChatWrapper, type TemplateChatWrapperOptions } from "./chatWrappers/generic/TemplateChatWrapper.js";
+import { JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions } from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
+import { resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, resolveChatWrapper, type ResolveChatWrapperOptions } from "./chatWrappers/utils/resolveChatWrapper.js";
+import { LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON } from "./utils/LlamaText.js";
 import { appendUserMessageToChatHistory } from "./utils/appendUserMessageToChatHistory.js";
 import { getModuleVersion } from "./utils/getModuleVersion.js";
+import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type LLamaContextualRepeatPenalty } from "./types.js";
 import { type GbnfJsonArraySchema, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonObjectSchema, type GbnfJsonOneOfSchema, type GbnfJsonSchema, type GbnfJsonSchemaImmutableType, type GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
-export { Llama, getLlama, type LlamaOptions, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizeStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, UnsupportedError, DisposedError, ChatWrapper, type ChatWrapperSettings, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, type TemplateChatWrapperOptions, resolveChatWrapperBasedOnModel, resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, type ChatWrapperTypeName, LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokenJSON, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema };
+import { type GgufFileInfo } from "./gguf/types/GgufFileInfoTypes.js";
+import { type GgufMetadata, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, type GgufMetadataRWKV, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
+import { GgmlType, type GgufTensorInfo } from "./gguf/types/GgufTensorInfoTypes.js";
+export { Llama, getLlama, type LlamaOptions, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, TokenMeter, type TokenMeterState, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, type ChatWrapperSettings, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, type TemplateChatWrapperOptions, JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions, resolveChatWrapper, type ResolveChatWrapperOptions, resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, type GgufFileInfo, type GgufMetadata, type GgufTensorInfo, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, type GgufMetadataRWKV, GgmlType, isGgufMetadataOfArchitectureType };

package/dist/index.js

@@ -2,7 +2,7 @@ import { DisposedError } from "lifecycle-utils";
 import { Llama } from "./bindings/Llama.js";
 import { getLlama } from "./bindings/getLlama.js";
 import { NoBinaryFoundError } from "./bindings/utils/NoBinaryFoundError.js";
-import { LlamaLogLevel } from "./bindings/types.js";
+import { LlamaLogLevel, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, LlamaVocabularyType } from "./bindings/types.js";
 import { LlamaModel, LlamaModelInfillTokens, LlamaModelTokens } from "./evaluator/LlamaModel.js";
 import { LlamaGrammar } from "./evaluator/LlamaGrammar.js";
 import { LlamaJsonSchemaGrammar } from "./evaluator/LlamaJsonSchemaGrammar.js";
@@ -14,7 +14,9 @@ import { LlamaChatSession } from "./evaluator/LlamaChatSession/LlamaChatSession.
 import { defineChatSessionFunction } from "./evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
 import { LlamaChat } from "./evaluator/LlamaChat/LlamaChat.js";
 import { LlamaCompletion } from "./evaluator/LlamaCompletion.js";
+import { TokenMeter } from "./evaluator/TokenMeter.js";
 import { UnsupportedError } from "./utils/UnsupportedError.js";
+import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
 import { LlamaChatWrapper } from "./chatWrappers/LlamaChatWrapper.js";
@@ -24,12 +26,15 @@ import { FalconChatWrapper } from "./chatWrappers/FalconChatWrapper.js";
 import { AlpacaChatWrapper } from "./chatWrappers/AlpacaChatWrapper.js";
 import { FunctionaryChatWrapper } from "./chatWrappers/FunctionaryChatWrapper.js";
 import { GemmaChatWrapper } from "./chatWrappers/GemmaChatWrapper.js";
-import { TemplateChatWrapper } from "./TemplateChatWrapper.js";
-import { resolveChatWrapperBasedOnModel } from "./chatWrappers/resolveChatWrapperBasedOnModel.js";
-import { resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames } from "./bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
-import { LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText } from "./utils/LlamaText.js";
+import { TemplateChatWrapper } from "./chatWrappers/generic/TemplateChatWrapper.js";
+import { JinjaTemplateChatWrapper } from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
+import { resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, resolveChatWrapper } from "./chatWrappers/utils/resolveChatWrapper.js";
+import { LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText } from "./utils/LlamaText.js";
 import { appendUserMessageToChatHistory } from "./utils/appendUserMessageToChatHistory.js";
 import { getModuleVersion } from "./utils/getModuleVersion.js";
+import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { isChatModelResponseFunctionCall } from "./types.js";
-export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, LlamaEmbeddingContext, LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaCompletion, UnsupportedError, DisposedError, ChatWrapper, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, resolveChatWrapperBasedOnModel, resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall };
+import { GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
+import { GgmlType } from "./gguf/types/GgufTensorInfoTypes.js";
+export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, LlamaEmbeddingContext, LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaCompletion, TokenMeter, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, JinjaTemplateChatWrapper, resolveChatWrapper, resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, GgmlType, isGgufMetadataOfArchitectureType };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAe,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EACH,qBAAqB,EAAqC,cAAc,EAC3E,MAAM,sCAAsC,CAAC;AAM9C,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,eAAe,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,WAAW,EAA2B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAkC,MAAM,0BAA0B,CAAC;AAC9F,OAAO,EAAC,8BAA8B,EAAC,MAAM,kDAAkD,CAAC;AAChG,OAAO,EACH,wCAAwC,EAAE,oBAAoB,EACjE,MAAM,8DAA8D,CAAC;AACtE,OAAO,EACH,SAAS,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,YAAY,EAE1E,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAGS,+BAA+B,EAC9C,MAAM,YAAY,CAAC;AAOpB,OAAO,EACH,KAAK,EACL,QAAQ,EAER,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EAEtB,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,qBAAqB,EAErB,cAAc,EAEd,gBAAgB,EAChB,yBAAyB,EAKzB,SAAS,EAOT,eAAe,EAIf,gBAAgB,EAChB,aAAa,EACb,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EAEnB,8BAA8B,EAC9B,wCAAwC,EACxC,oBAAoB,EAEpB,SAAS,EACT,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,YAAY,EAIZ,8BAA8B,EAC9B,gBAAgB,EAUhB,+BAA+B,EAUlC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAe,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EACH,qBAAqB,EAAqC,cAAc,EAC3E,MAAM,sCAAsC,CAAC;AAM9C,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,eAAe,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,UAAU,EAAuB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,WAAW,EAA2B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAkC,MAAM,+CAA+C,CAAC;AACnH,OAAO,EAAC,wBAAwB,EAAuC,MAAM,oDAAoD,CAAC;AAClI,OAAO,EACH,8BAA8B,EAAsC,+BAA+B,EAC9D,4BAA4B,EAAoC,kBAAkB,EAE1H,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACH,SAAS,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAExE,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,gBAAgB,EAAC,MAAM,4BAA4B,CAAC;AAE5D,OAAO,EAGS,+BAA+B,EAC9C,MAAM,YAAY,CAAC;AAMpB,OAAO,EAC4C,oBAAoB,EAAE,YAAY,EAAE,8BAA8B,EACjH,mCAAmC,EAE6D,gCAAgC,EACnI,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAC,QAAQ,EAAsB,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EACH,KAAK,EACL,QAAQ,EAER,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EAEtB,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,qBAAqB,EAErB,cAAc,EAEd,gBAAgB,EAChB,yBAAyB,EAKzB,SAAS,EAOT,eAAe,EAIf,UAAU,EAEV,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EAEnB,wBAAwB,EAExB,kBAAkB,EAElB,8BAA8B,EAE9B,+BAA+B,EAE/B,4BAA4B,EAE5B,SAAS,EACT,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,YAAY,EAKZ,8BAA8B,EAC9B,gBAAgB,EAUhB,+BAA+B,EAU/B,mBAAmB,EACnB,wBAAwB,EACxB,+BAA+B,EAC/B,gBAAgB,EAKhB,oBAAoB,EACpB,YAAY,EACZ,8BAA8B,EAC9B,mCAAmC,EAanC,QAAQ,EACR,gCAAgC,EACnC,CAAC"}

package/dist/types.d.ts

@@ -4,7 +4,7 @@ export type Token = number & {
     __token: never;
 };
 export type Tokenizer = {
-    tokenize(text: string, specialTokens?: boolean): Token[];
+    tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
     tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
 }["tokenize"];
 export type ChatHistoryItem = ChatSystemMessage | ChatUserMessage | ChatModelResponse;

package/dist/utils/InsufficientMemoryError.d.ts

@@ -0,0 +1,3 @@
+export declare class InsufficientMemoryError extends Error {
+    constructor(message?: string);
+}

package/dist/utils/InsufficientMemoryError.js

@@ -0,0 +1,6 @@
+export class InsufficientMemoryError extends Error {
+    constructor(message = "Insufficient memory") {
+        super(message);
+    }
+}
+//# sourceMappingURL=InsufficientMemoryError.js.map

package/dist/utils/InsufficientMemoryError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InsufficientMemoryError.js","sourceRoot":"","sources":["../../src/utils/InsufficientMemoryError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAC9C,YAAmB,UAAkB,qBAAqB;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

package/dist/utils/LlamaText.d.ts

@@ -2,6 +2,7 @@ import { Token, Tokenizer } from "../types.js";
 export type LlamaTextClass = {
     <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText<V> | readonly (LlamaText<V> | V)[], ...values: readonly (V | LlamaText<V> | V2 | LlamaText<V2> | number | boolean | readonly (LlamaText<V> | V | LlamaText<V2> | V2)[])[]): LlamaText<V | V2>;
     fromJSON(json: LlamaTextJSON): LlamaText;
+    compare(a: LlamaText, b: LlamaText): boolean;
 };
 export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
     <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText | readonly (LlamaText | V)[], ...values: readonly (V | V2 | LlamaText | number | boolean | readonly (LlamaText | V | V2)[])[]): LlamaText<T | V>;
@@ -11,32 +12,46 @@ export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
     joinValues<V extends LlamaTextValue = LlamaTextValue>(separator: LlamaText<V> | V): LlamaText<T | V>;
     toString(): string;
     toJSON(): LlamaTextJSON;
-    tokenize(tokenizer: Tokenizer): Token[];
+    tokenize(tokenizer: Tokenizer, options?: "trimLeadingSpace"): Token[];
+    compare(other: LlamaText): boolean;
+    trimStart(): LlamaText<T>;
+    trimEnd(): LlamaText<T>;
+    includes(value: LlamaText): boolean;
 };
-export type LlamaTextValue = string | SpecialToken;
+export type LlamaTextValue = string | SpecialTokensText | SpecialToken;
 export type LlamaTextJSON = Array<LlamaTextJSONValue>;
-export type LlamaTextJSONValue = string | LlamaTextSpecialTokenJSON;
+export type LlamaTextJSONValue = string | LlamaTextSpecialTokensTextJSON | LlamaTextSpecialTokenJSON;
+export type LlamaTextSpecialTokensTextJSON = {
+    type: "specialTokensText";
+    value: string;
+};
 export type LlamaTextSpecialTokenJSON = {
     type: "specialToken";
     value: string;
-    builtin?: true;
 };
 export declare const LlamaText: LlamaTextClass;
-export declare class SpecialToken {
+export declare class SpecialTokensText {
     readonly value: string;
     constructor(value: string);
     toString(): string;
-    tokenize(tokenizer: Tokenizer): Token[];
-    toJSON(): LlamaTextSpecialTokenJSON;
-    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
-    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
+    tokenize(tokenizer: Tokenizer, trimLeadingSpace?: boolean): Token[];
+    toJSON(): LlamaTextSpecialTokensTextJSON;
+    static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText;
+    static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON;
+    /**
+     * Wraps the value with a `SpecialTokensText` only if `shouldWrap` is true
+     */
+    static wrapIf(shouldWrap: boolean, value: string): SpecialTokensText | string;
 }
 export type BuiltinSpecialTokenValue = "BOS" | "EOS" | "NL";
-export declare class BuiltinSpecialToken extends SpecialToken {
+export declare class SpecialToken {
     readonly value: BuiltinSpecialTokenValue;
     constructor(value: BuiltinSpecialTokenValue);
+    toString(): BuiltinSpecialTokenValue;
     tokenize(tokenizer: Tokenizer): Token[];
     toJSON(): LlamaTextSpecialTokenJSON;
+    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
+    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
 }
 export declare function isLlamaText(value: unknown): value is LlamaText;
 export declare function tokenizeText(text: string | LlamaText, tokenizer: Tokenizer): Token[];

package/dist/utils/LlamaText.js

@@ -7,13 +7,26 @@ LlamaText.fromJSON = function fromJSON(json) {
             return value;
         else if (SpecialToken.isSpecialTokenJSON(value))
             return SpecialToken.fromJSON(value);
+        else if (SpecialTokensText.isSpecialTokensTextJSON(value))
+            return SpecialTokensText.fromJSON(value);
         else {
             void (value);
             throw new Error(`Unknown value type: ${value}`);
         }
     }));
 };
-export class SpecialToken {
+LlamaText.compare = function compare(a, b) {
+    if (!isLlamaText(a) || !isLlamaText(b))
+        return false;
+    if (a.values.length !== b.values.length)
+        return false;
+    for (let i = 0; i < a.values.length; i++) {
+        if (!compareLlamaTextValues(a.values[i], b.values[i]))
+            return false;
+    }
+    return true;
+};
+export class SpecialTokensText {
     value;
     constructor(value) {
         this.value = value;
@@ -21,41 +34,58 @@ export class SpecialToken {
     toString() {
         return this.value;
     }
-    tokenize(tokenizer) {
-        return tokenizer(this.value, true);
+    tokenize(tokenizer, trimLeadingSpace = false) {
+        return tokenizer(this.value, true, trimLeadingSpace ? "trimLeadingSpace" : undefined);
     }
     toJSON() {
         return {
-            type: "specialToken",
+            type: "specialTokensText",
             value: this.value
         };
     }
     static fromJSON(json) {
-        if (json.builtin)
-            return new BuiltinSpecialToken(json.value);
-        else
-            return new SpecialToken(json.value);
+        if (SpecialTokensText.isSpecialTokensTextJSON(json))
+            return new SpecialTokensText(json.value);
+        throw new Error(`Invalid JSON for SpecialTokensText: ${JSON.stringify(json)}`);
     }
-    static isSpecialTokenJSON(value) {
-        return value != null && typeof value === "object" && value.type === "specialToken";
+    static isSpecialTokensTextJSON(value) {
+        return value != null && typeof value === "object" && value.type === "specialTokensText";
+    }
+    /**
+     * Wraps the value with a `SpecialTokensText` only if `shouldWrap` is true
+     */
+    static wrapIf(shouldWrap, value) {
+        if (shouldWrap)
+            return new SpecialTokensText(value);
+        else
+            return value;
     }
 }
-export class BuiltinSpecialToken extends SpecialToken {
+export class SpecialToken {
     value;
     constructor(value) {
-        super(value);
         this.value = value;
     }
+    toString() {
+        return this.value;
+    }
     tokenize(tokenizer) {
         return tokenizer(this.value, "builtin");
     }
     toJSON() {
         return {
             type: "specialToken",
-            value: this.value,
-            builtin: true
+            value: this.value
         };
     }
+    static fromJSON(json) {
+        if (SpecialToken.isSpecialTokenJSON(json))
+            return new SpecialToken(json.value);
+        throw new Error(`Invalid JSON for SpecialToken: ${JSON.stringify(json)}`);
+    }
+    static isSpecialTokenJSON(value) {
+        return value != null && typeof value === "object" && value.type === "specialToken";
+    }
 }
 export function isLlamaText(value) {
     return typeof value === "function" && "type" in value && value.type === "LlamaText";
@@ -88,32 +118,129 @@ const LlamaTextPrototypeFunctions = {
             .map((value) => {
             if (value instanceof SpecialToken)
                 return value.toString();
+            else if (value instanceof SpecialTokensText)
+                return value.toString();
             else
                 return value;
         })
             .join("");
     },
-    tokenize(tokenizer) {
+    tokenize(tokenizer, options) {
         let textToTokenize = "";
         const res = [];
+        const hasContent = () => (res.length > 0 || textToTokenize.length > 0);
+        const resolveTokenizerOptions = () => (hasContent() ? "trimLeadingSpace" : options);
         for (const value of this.values) {
             if (value instanceof SpecialToken) {
-                res.push(...tokenizer(textToTokenize, false), ...value.tokenize(tokenizer));
+                res.push(...tokenizer(textToTokenize, false, resolveTokenizerOptions()), ...value.tokenize(tokenizer));
+                textToTokenize = "";
+            }
+            else if (value instanceof SpecialTokensText) {
+                res.push(...tokenizer(textToTokenize, false, resolveTokenizerOptions()), ...value.tokenize(tokenizer, hasContent() || options === "trimLeadingSpace"));
                 textToTokenize = "";
             }
             else
                 textToTokenize += value;
         }
-        res.push(...tokenizer(textToTokenize, false));
+        res.push(...tokenizer(textToTokenize, false, resolveTokenizerOptions()));
         return res;
     },
     toJSON() {
         return this.values.map((value) => {
             if (value instanceof SpecialToken)
-                return { type: "specialToken", value: value.value };
+                return value.toJSON();
+            else if (value instanceof SpecialTokensText)
+                return value.toJSON();
             else
                 return value;
         });
+    },
+    compare(other) {
+        return LlamaText.compare(this, other);
+    },
+    trimStart() {
+        const newValues = this.values.slice();
+        while (newValues.length > 0) {
+            const firstValue = newValues[0];
+            if (firstValue instanceof SpecialToken)
+                break;
+            if (firstValue instanceof SpecialTokensText) {
+                const newValue = firstValue.value.trimStart();
+                if (newValue === "") {
+                    newValues.shift();
+                    continue;
+                }
+                else if (newValue !== firstValue.value) {
+                    newValues[0] = new SpecialTokensText(newValue);
+                    break;
+                }
+                break;
+            }
+            else if (typeof firstValue === "string") {
+                const newValue = firstValue.trimStart();
+                if (newValue === "") {
+                    newValues.shift();
+                    continue;
+                }
+                else if (newValue !== firstValue) {
+                    newValues[0] = newValue;
+                    break;
+                }
+                break;
+            }
+            else
+                void (firstValue);
+        }
+        return createLlamaText(newValues);
+    },
+    trimEnd() {
+        const newValues = this.values.slice();
+        while (newValues.length > 0) {
+            const lastValue = newValues[newValues.length - 1];
+            if (lastValue instanceof SpecialToken)
+                break;
+            if (lastValue instanceof SpecialTokensText) {
+                const newValue = lastValue.value.trimEnd();
+                if (newValue === "") {
+                    newValues.pop();
+                    continue;
+                }
+                else if (newValue !== lastValue.value) {
+                    newValues[newValues.length - 1] = new SpecialTokensText(newValue);
+                    break;
+                }
+                break;
+            }
+            else if (typeof lastValue === "string") {
+                const newValue = lastValue.trimEnd();
+                if (newValue === "") {
+                    newValues.pop();
+                    continue;
+                }
+                else if (newValue !== lastValue) {
+                    newValues[newValues.length - 1] = newValue;
+                    break;
+                }
+                break;
+            }
+            else
+                void (lastValue);
+        }
+        return createLlamaText(newValues);
+    },
+    includes(value) {
+        for (let i = 0; i < this.values.length; i++) {
+            if (compareLlamaTextValues(this.values[i], value.values[0])) {
+                let j = 1;
+                for (; j < value.values.length; j++) {
+                    if (!compareLlamaTextValues(this.values[i + j], value.values[j]))
+                        break;
+                }
+                if (j === value.values.length)
+                    return true;
+            }
+        }
+        return false;
     }
 };
 function createLlamaText(history) {
@@ -162,15 +289,39 @@ function createLlamaText(history) {
             writable: false,
             configurable: false,
             enumerable: false
+        },
+        ["compare"]: {
+            value: LlamaTextPrototypeFunctions.compare,
+            writable: false,
+            configurable: false,
+            enumerable: false
+        },
+        ["trimStart"]: {
+            value: LlamaTextPrototypeFunctions.trimStart,
+            writable: false,
+            configurable: false,
+            enumerable: false
+        },
+        ["trimEnd"]: {
+            value: LlamaTextPrototypeFunctions.trimEnd,
+            writable: false,
+            configurable: false,
+            enumerable: false
+        },
+        ["includes"]: {
+            value: LlamaTextPrototypeFunctions.includes,
+            writable: false,
+            configurable: false,
+            enumerable: false
         }
     });
     return llamaText;
 }
 function createHistoryFromStringsAndValues(strings, values) {
     function addItemToRes(res, item) {
-        if (item === undefined || item === "" || (item instanceof SpecialToken && item.value === ""))
+        if (item === undefined || item === "" || (item instanceof SpecialTokensText && item.value === ""))
             return res;
-        else if (typeof item === "string" || item instanceof SpecialToken)
+        else if (typeof item === "string" || item instanceof SpecialTokensText || item instanceof SpecialToken)
             return res.concat([item]);
         else if (isLlamaText(item))
             return res.concat(item.values);
@@ -178,7 +329,7 @@ function createHistoryFromStringsAndValues(strings, values) {
             return res.concat(item.reduce((res, value) => {
                 if (isLlamaText(value))
                     return res.concat(value.values);
-                else if (value === "" || (value instanceof SpecialToken && value.value === ""))
+                else if (value === "" || (value instanceof SpecialTokensText && value.value === ""))
                     return res;
                 return res.concat([value]);
             }, []));
@@ -187,10 +338,32 @@ function createHistoryFromStringsAndValues(strings, values) {
             return res.concat([String(item)]);
         return item;
     }
+    function squashAdjacentItems(res, item) {
+        if (res.length === 0) {
+            res.push(item);
+            return res;
+        }
+        const lastItem = res[res.length - 1];
+        if (lastItem instanceof SpecialToken || item instanceof SpecialToken) {
+            res.push(item);
+            return res;
+        }
+        if (typeof lastItem === "string" && typeof item === "string") {
+            res[res.length - 1] += item;
+            return res;
+        }
+        else if (lastItem instanceof SpecialTokensText && item instanceof SpecialTokensText) {
+            res[res.length - 1] = new SpecialTokensText(lastItem.value + item.value);
+            return res;
+        }
+        res.push(item);
+        return res;
+    }
     if (!isTemplateStringsArray(strings)) {
         return [strings]
             .concat(values)
-            .reduce(addItemToRes, []);
+            .reduce(addItemToRes, [])
+            .reduce(squashAdjacentItems, []);
     }
     let res = [];
     for (let i = 0; i < strings.length; i++) {
@@ -198,10 +371,19 @@ function createHistoryFromStringsAndValues(strings, values) {
         if (i < values.length)
             res = addItemToRes(res, values[i]);
     }
-    return res;
+    return res.reduce(squashAdjacentItems, []);
 }
 function isTemplateStringsArray(value) {
     return value instanceof Array && value.raw instanceof Array &&
         value.length === value.raw.length;
 }
+function compareLlamaTextValues(a, b) {
+    if (a instanceof SpecialTokensText && b instanceof SpecialTokensText)
+        return a.value === b.value;
+    else if (a instanceof SpecialToken && b instanceof SpecialToken)
+        return a.value === b.value;
+    else if (a !== a)
+        return false;
+    return true;
+}
 //# sourceMappingURL=LlamaText.js.map

package/dist/utils/LlamaText.js.map

@@ -1 +1 @@
-{"version":3,"file":"LlamaText.js","sourceRoot":"","sources":["../../src/utils/LlamaText.ts"],"names":[],"mappings":"AA+BA,MAAM,CAAC,MAAM,SAAS,GAAmB,SAAS,SAAS,CACvD,OAA0F,EAC1F,GAAG,MAAyF;IAE5F,OAAO,eAAe,CAAC,iCAAiC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/E,CAAmB,CAAC;AACpB,SAAS,CAAC,QAAQ,GAAG,SAAS,QAAQ,CAAC,IAAmB;IACtD,OAAO,eAAe,CAClB,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,IAAI,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,KAAK,CAAC;aACZ,IAAI,YAAY,CAAC,kBAAkB,CAAC,KAAK,CAAC;YAC3C,OAAO,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACnC;YACD,KAAK,CAAC,KAAqB,CAAC,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,EAAE,CAAC,CAAC;SACnD;IACL,CAAC,CAAC,CACL,CAAC;AACN,CAAC,CAAC;AAEF,MAAM,OAAO,YAAY;IACL,KAAK,CAAS;IAE9B,YAAmB,KAAa;QAC5B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,QAAQ;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEM,QAAQ,CAAC,SAAoB;QAChC,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;IACvC,CAAC;IAEM,MAAM;QACT,OAAO;YACH,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,IAAI,CAAC,KAAK;SACpB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,IAA+B;QAClD,IAAI,IAAI,CAAC,OAAO;YACZ,OAAO,IAAI,mBAAmB,CAAC,IAAI,CAAC,KAAiC,CAAC,CAAC;;YAEvE,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC5C,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAC,KAAyB;QACtD,OAAO,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;IACvF,CAAC;CACJ;AAGD,MAAM,OAAO,mBAAoB,SAAQ,YAAY;IACxB,KAAK,CAA2B;IAEzD,YAAmB,KAA+B;QAC9C,KAAK,CAAC,KAAK,CAAC,CAAC;QAEb,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEe,QAAQ,CAAC,SAAoB;QACzC,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,CAAC;IAEe,MAAM;QAClB,OAAO;YACH,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,OAAO,EAAE,IAAI;SAChB,CAAC;IACN,CAAC;CACJ;AAED,MAAM,UAAU,WAAW,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,UAAU,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAwB,EAAE,SAAoB;IACvE,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;;QAE9B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AACxC,CAAC;AAOD,MAAM,2BAA2B,GAAuB;IACpD,SAAS,EAAE,SAAS,SAAS,CAAkB,MAAM;QACjD,OAAO,eAAe,CAClB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAC1B,CAAC;IACN,CAA2B;IAC3B,UAAU,CAAkB,SAAqC;QAC7D,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAE/B,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC9B,IAAI,WAAW,CAAC,SAAS,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;;oBAEpC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACjC;SACJ;QAED,OAAO,eAAe,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM;aACb,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACX,IAAI,KAAK,YAAY,YAAY;gBAC7B,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;;gBAExB,OAAO,KAAK,CAAC;QACrB,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IACD,QAAQ,CAAkB,SAAS;QAC/B,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAC7B,IAAI,KAAK,YAAY,YAAY,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBAC5E,cAAc,GAAG,EAAE,CAAC;aACvB;;gBACG,cAAc,IAAI,KAAK,CAAC;SAC/B;QAED,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,KAAK,CAAC,CAAC,CAAC;QAE9C,OAAO,GAAG,CAAC;IACf,CAAC;IACD,MAAM;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,IAAI,KAAK,YAAY,YAAY;gBAC7B,OAAO,EAAC,IAAI,EAAE,cAAc,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAA8B,CAAC;;gBAE/E,OAAO,KAAkC,CAAC;QAClD,CAAC,CAAC,CAAC;IACP,CAAC;CACJ,CAAC;AAEF,SAAS,eAAe,CAAC,OAAkC;IACvD,MAAM,SAAS,GAAuB,SAAS,SAAS,CACpD,OAAgF,EAChF,GAAG,MAAgC;QAEnC,OAAO,eAAe,CAClB,SAAS,CAAC,MAAM,CAAC,MAAM,CACnB,iCAAiC,CAAC,OAAO,EAAE,MAAM,CAAC,CACrD,CACJ,CAAC;IACN,CAAc,CAAC;IAEf,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE;QAC/B,CAAC,MAAgC,CAAC,EAAE;YAChC,KAAK,EAAE,WAAuC;YAC9C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,IAAI;SACnB;QACD,CAAC,QAAkC,CAAC,EAAE;YAClC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAA+B;YACnE,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,IAAI;SACnB;QACD,CAAC,WAAqC,CAAC,EAAE;YACrC,KAAK,EAAE,2BAA2B,CAAC,SAAS;YAC5C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,YAAsC,CAAC,EAAE;YACtC,KAAK,EAAE,2BAA2B,CAAC,UAAU;YAC7C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,UAAoC,CAAC,EAAE;YACpC,KAAK,EAAE,2BAA2B,CAAC,QAAQ;YAC3C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,UAAoC,CAAC,EAAE;YACpC,KAAK,EAAE,2BAA2B,CAAC,QAAQ;YAC3C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,QAAkC,CAAC,EAAE;YAClC,KAAK,EAAE,2BAA2B,CAAC,MAAM;YACzC,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;KACJ,CAAC,CAAC;IAEH,OAAO,SAAsB,CAAC;AAClC,CAAC;AAED,SAAS,iCAAiC,CACtC,OAAgF,EAChF,MAAgC;IAEhC,SAAS,YAAY,CAAC,GAA0B,EAAE,IAAyB;QACvE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,YAAY,YAAY,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;YACxF,OAAO,GAAG,CAAC;aACV,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,YAAY,YAAY;YAC7D,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;aACzB,IAAI,WAAW,CAAC,IAAI,CAAC;YACtB,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9B,IAAI,IAAI,YAAY,KAAK,EAAE;YAC5B,OAAO,GAAG,CAAC,MAAM,CACb,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,WAAW,CAAC,KAAK,CAAC;oBAClB,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;qBAC/B,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,KAAK,YAAY,YAAY,IAAI,KAAK,CAAC,KAAK,KAAK,EAAE,CAAC;oBAC1E,OAAO,GAAG,CAAC;gBAEf,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC,EAAE,EAA2B,CAAC,CAClC,CAAC;SACL;aAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,SAAS;YAC5D,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEtC,OAAO,IAAoB,CAAC;IAChC,CAAC;IAED,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE;QAClC,OAAQ,CAAC,OAAO,CAA2B;aACtC,MAAM,CAAC,MAAM,CAAC;aACd,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC,CAAC;KACjC;IAGD,IAAI,GAAG,GAA0B,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM;YACjB,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;KAC1C;IAED,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC1C,OAAO,KAAK,YAAY,KAAK,IAAK,KAAqC,CAAC,GAAG,YAAY,KAAK;QACxF,KAAK,CAAC,MAAM,KAAM,KAAqC,CAAC,GAAG,CAAC,MAAM,CAAC;AAC3E,CAAC"}
+{"version":3,"file":"LlamaText.js","sourceRoot":"","sources":["../../src/utils/LlamaText.ts"],"names":[],"mappings":"AAqCA,MAAM,CAAC,MAAM,SAAS,GAAmB,SAAS,SAAS,CACvD,OAA8G,EAC9G,GAAG,MAA6G;IAEhH,OAAO,eAAe,CAAC,iCAAiC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC;AAC/E,CAAmB,CAAC;AACpB,SAAS,CAAC,QAAQ,GAAG,SAAS,QAAQ,CAAC,IAAmB;IACtD,OAAO,eAAe,CAClB,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACf,IAAI,OAAO,KAAK,KAAK,QAAQ;YACzB,OAAO,KAAK,CAAC;aACZ,IAAI,YAAY,CAAC,kBAAkB,CAAC,KAAK,CAAC;YAC3C,OAAO,YAAY,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACnC,IAAI,iBAAiB,CAAC,uBAAuB,CAAC,KAAK,CAAC;YACrD,OAAO,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;aACxC;YACD,KAAK,CAAC,KAAqB,CAAC,CAAC;YAC7B,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,EAAE,CAAC,CAAC;SACnD;IACL,CAAC,CAAC,CACL,CAAC;AACN,CAAC,CAAC;AACF,SAAS,CAAC,OAAO,GAAG,SAAS,OAAO,CAAC,CAAY,EAAE,CAAY;IAC3D,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC;QAClC,OAAO,KAAK,CAAC;IAEjB,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM,CAAC,MAAM;QACnC,OAAO,KAAK,CAAC;IAEjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACtC,IAAI,CAAC,sBAAsB,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YACjD,OAAO,KAAK,CAAC;KACpB;IAED,OAAO,IAAI,CAAC;AAChB,CAAC,CAAC;AAEF,MAAM,OAAO,iBAAiB;IACV,KAAK,CAAS;IAE9B,YAAmB,KAAa;QAC5B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,QAAQ;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEM,QAAQ,CAAC,SAAoB,EAAE,mBAA4B,KAAK;QACnE,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,EAAE,gBAAgB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IAC1F,CAAC;IAEM,MAAM;QACT,OAAO;YACH,IAAI,EAAE,mBAAmB;YACzB,KAAK,EAAE,IAAI,CAAC,KAAK;SACpB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,IAAoC;QACvD,IAAI,iBAAiB,CAAC,uBAAuB,CAAC,IAAI,CAAC;YAC/C,OAAO,IAAI,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7C,MAAM,IAAI,KAAK,CAAC,uCAAuC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnF,CAAC;IAEM,MAAM,CAAC,uBAAuB,CAAC,KAAyB;QAC3D,OAAO,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,mBAAmB,CAAC;IAC5F,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,MAAM,CAAC,UAAmB,EAAE,KAAa;QACnD,IAAI,UAAU;YACV,OAAO,IAAI,iBAAiB,CAAC,KAAK,CAAC,CAAC;;YAEpC,OAAO,KAAK,CAAC;IACrB,CAAC;CACJ;AAGD,MAAM,OAAO,YAAY;IACL,KAAK,CAA2B;IAEhD,YAAmB,KAA+B;QAC9C,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;IACvB,CAAC;IAEM,QAAQ;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAEM,QAAQ,CAAC,SAAoB;QAChC,OAAO,SAAS,CAAC,IAAI,CAAC,KAAK,EAAE,SAAS,CAAC,CAAC;IAC5C,CAAC;IAEM,MAAM;QACT,OAAO;YACH,IAAI,EAAE,cAAc;YACpB,KAAK,EAAE,IAAI,CAAC,KAAK;SACpB,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,QAAQ,CAAC,IAA+B;QAClD,IAAI,YAAY,CAAC,kBAAkB,CAAC,IAAI,CAAC;YACrC,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,KAAiC,CAAC,CAAC;QAEpE,MAAM,IAAI,KAAK,CAAC,kCAAkC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC9E,CAAC;IAEM,MAAM,CAAC,kBAAkB,CAAC,KAAyB;QACtD,OAAO,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,cAAc,CAAC;IACvF,CAAC;CACJ;AAED,MAAM,UAAU,WAAW,CAAC,KAAc;IACtC,OAAO,OAAO,KAAK,KAAK,UAAU,IAAI,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,CAAC;AACxF,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAwB,EAAE,SAAoB;IACvE,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,SAAS,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;;QAE9B,OAAO,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;AACxC,CAAC;AAOD,MAAM,2BAA2B,GAAuB;IACpD,SAAS,EAAE,SAAS,SAAS,CAAkB,MAAM;QACjD,OAAO,eAAe,CAClB,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAC1B,CAAC;IACN,CAA2B;IAC3B,UAAU,CAAkB,SAAqC;QAC7D,MAAM,SAAS,GAAqB,EAAE,CAAC;QAEvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;YAE/B,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE;gBAC9B,IAAI,WAAW,CAAC,SAAS,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;;oBAEpC,SAAS,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;aACjC;SACJ;QAED,OAAO,eAAe,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,QAAQ;QACJ,OAAO,IAAI,CAAC,MAAM;aACb,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACX,IAAI,KAAK,YAAY,YAAY;gBAC7B,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;iBACvB,IAAI,KAAK,YAAY,iBAAiB;gBACvC,OAAO,KAAK,CAAC,QAAQ,EAAE,CAAC;;gBAExB,OAAO,KAAK,CAAC;QACrB,CAAC,CAAC;aACD,IAAI,CAAC,EAAE,CAAC,CAAC;IAClB,CAAC;IACD,QAAQ,CAAkB,SAAS,EAAE,OAA4B;QAC7D,IAAI,cAAc,GAAG,EAAE,CAAC;QACxB,MAAM,GAAG,GAAY,EAAE,CAAC;QACxB,MAAM,UAAU,GAAG,GAAG,EAAE,CAAC,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QACvE,MAAM,uBAAuB,GAAG,GAAG,EAAE,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QAEpF,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,MAAM,EAAE;YAC7B,IAAI,KAAK,YAAY,YAAY,EAAE;gBAC/B,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC;gBACvG,cAAc,GAAG,EAAE,CAAC;aACvB;iBAAM,IAAI,KAAK,YAAY,iBAAiB,EAAE;gBAC3C,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,EAAE,GAAG,KAAK,CAAC,QAAQ,CAAC,SAAS,EAAE,UAAU,EAAE,IAAI,OAAO,KAAK,kBAAkB,CAAC,CAAC,CAAC;gBACvJ,cAAc,GAAG,EAAE,CAAC;aACvB;;gBACG,cAAc,IAAI,KAAK,CAAC;SAC/B;QAED,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,cAAc,EAAE,KAAK,EAAE,uBAAuB,EAAE,CAAC,CAAC,CAAC;QAEzE,OAAO,GAAG,CAAC;IACf,CAAC;IACD,MAAM;QACF,OAAO,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YAC7B,IAAI,KAAK,YAAY,YAAY;gBAC7B,OAAO,KAAK,CAAC,MAAM,EAA+B,CAAC;iBAClD,IAAI,KAAK,YAAY,iBAAiB;gBACvC,OAAO,KAAK,CAAC,MAAM,EAA+B,CAAC;;gBAEnD,OAAO,KAAkC,CAAC;QAClD,CAAC,CAAC,CAAC;IACP,CAAC;IACD,OAAO,CAAkB,KAAgB;QACrC,OAAO,SAAS,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;IAC1C,CAAC;IACD,SAAS;QACL,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAEtC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,MAAM,UAAU,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;YAEhC,IAAI,UAAU,YAAY,YAAY;gBAClC,MAAM;YAEV,IAAI,UAAU,YAAY,iBAAiB,EAAE;gBACzC,MAAM,QAAQ,GAAG,UAAU,CAAC,KAAK,CAAC,SAAS,EAAE,CAAC;gBAC9C,IAAI,QAAQ,KAAK,EAAE,EAAE;oBACjB,SAAS,CAAC,KAAK,EAAE,CAAC;oBAClB,SAAS;iBACZ;qBAAM,IAAI,QAAQ,KAAK,UAAU,CAAC,KAAK,EAAE;oBACtC,SAAS,CAAC,CAAC,CAAC,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAAC,CAAC;oBAC/C,MAAM;iBACT;gBAED,MAAM;aACT;iBAAM,IAAI,OAAO,UAAU,KAAK,QAAQ,EAAE;gBACvC,MAAM,QAAQ,GAAG,UAAU,CAAC,SAAS,EAAE,CAAC;gBACxC,IAAI,QAAQ,KAAK,EAAE,EAAE;oBACjB,SAAS,CAAC,KAAK,EAAE,CAAC;oBAClB,SAAS;iBACZ;qBAAM,IAAI,QAAQ,KAAK,UAAU,EAAE;oBAChC,SAAS,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC;oBACxB,MAAM;iBACT;gBAED,MAAM;aACT;;gBACG,KAAK,CAAC,UAA0B,CAAC,CAAC;SACzC;QAED,OAAO,eAAe,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,OAAO;QACH,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAEtC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE;YACzB,MAAM,SAAS,GAAG,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YAElD,IAAI,SAAS,YAAY,YAAY;gBACjC,MAAM;YAEV,IAAI,SAAS,YAAY,iBAAiB,EAAE;gBACxC,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,OAAO,EAAE,CAAC;gBAC3C,IAAI,QAAQ,KAAK,EAAE,EAAE;oBACjB,SAAS,CAAC,GAAG,EAAE,CAAC;oBAChB,SAAS;iBACZ;qBAAM,IAAI,QAAQ,KAAK,SAAS,CAAC,KAAK,EAAE;oBACrC,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAAC,CAAC;oBAClE,MAAM;iBACT;gBAED,MAAM;aACT;iBAAM,IAAI,OAAO,SAAS,KAAK,QAAQ,EAAE;gBACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,OAAO,EAAE,CAAC;gBACrC,IAAI,QAAQ,KAAK,EAAE,EAAE;oBACjB,SAAS,CAAC,GAAG,EAAE,CAAC;oBAChB,SAAS;iBACZ;qBAAM,IAAI,QAAQ,KAAK,SAAS,EAAE;oBAC/B,SAAS,CAAC,SAAS,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,QAAQ,CAAC;oBAC3C,MAAM;iBACT;gBAED,MAAM;aACT;;gBACG,KAAK,CAAC,SAAyB,CAAC,CAAC;SACxC;QAED,OAAO,eAAe,CAAC,SAAS,CAAC,CAAC;IACtC,CAAC;IACD,QAAQ,CAAkB,KAAgB;QACtC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACzC,IAAI,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE;gBACzD,IAAI,CAAC,GAAG,CAAC,CAAC;gBACV,OAAO,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;oBACjC,IAAI,CAAC,sBAAsB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;wBAC5D,MAAM;iBACb;gBAED,IAAI,CAAC,KAAK,KAAK,CAAC,MAAM,CAAC,MAAM;oBACzB,OAAO,IAAI,CAAC;aACnB;SACJ;QAED,OAAO,KAAK,CAAC;IACjB,CAAC;CACJ,CAAC;AAEF,SAAS,eAAe,CAAC,OAAkC;IACvD,MAAM,SAAS,GAAuB,SAAS,SAAS,CACpD,OAAgF,EAChF,GAAG,MAAgC;QAEnC,OAAO,eAAe,CAClB,SAAS,CAAC,MAAM,CAAC,MAAM,CACnB,iCAAiC,CAAC,OAAO,EAAE,MAAM,CAAC,CACrD,CACJ,CAAC;IACN,CAAc,CAAC;IAEf,MAAM,CAAC,gBAAgB,CAAC,SAAS,EAAE;QAC/B,CAAC,MAAgC,CAAC,EAAE;YAChC,KAAK,EAAE,WAAuC;YAC9C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,IAAI;SACnB;QACD,CAAC,QAAkC,CAAC,EAAE;YAClC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,EAAE,CAA+B;YACnE,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,IAAI;SACnB;QACD,CAAC,WAAqC,CAAC,EAAE;YACrC,KAAK,EAAE,2BAA2B,CAAC,SAAS;YAC5C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,YAAsC,CAAC,EAAE;YACtC,KAAK,EAAE,2BAA2B,CAAC,UAAU;YAC7C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,UAAoC,CAAC,EAAE;YACpC,KAAK,EAAE,2BAA2B,CAAC,QAAQ;YAC3C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,UAAoC,CAAC,EAAE;YACpC,KAAK,EAAE,2BAA2B,CAAC,QAAQ;YAC3C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,QAAkC,CAAC,EAAE;YAClC,KAAK,EAAE,2BAA2B,CAAC,MAAM;YACzC,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,SAAmC,CAAC,EAAE;YACnC,KAAK,EAAE,2BAA2B,CAAC,OAAO;YAC1C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,WAAqC,CAAC,EAAE;YACrC,KAAK,EAAE,2BAA2B,CAAC,SAAS;YAC5C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,SAAmC,CAAC,EAAE;YACnC,KAAK,EAAE,2BAA2B,CAAC,OAAO;YAC1C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;QACD,CAAC,UAAoC,CAAC,EAAE;YACpC,KAAK,EAAE,2BAA2B,CAAC,QAAQ;YAC3C,QAAQ,EAAE,KAAK;YACf,YAAY,EAAE,KAAK;YACnB,UAAU,EAAE,KAAK;SACpB;KACJ,CAAC,CAAC;IAEH,OAAO,SAAsB,CAAC;AAClC,CAAC;AAED,SAAS,iCAAiC,CACtC,OAAgF,EAChF,MAAgC;IAEhC,SAAS,YAAY,CAAC,GAA0B,EAAE,IAAyB;QACvE,IAAI,IAAI,KAAK,SAAS,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,YAAY,iBAAiB,IAAI,IAAI,CAAC,KAAK,KAAK,EAAE,CAAC;YAC7F,OAAO,GAAG,CAAC;aACV,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,YAAY,iBAAiB,IAAI,IAAI,YAAY,YAAY;YAClG,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;aACzB,IAAI,WAAW,CAAC,IAAI,CAAC;YACtB,OAAO,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;aAC9B,IAAI,IAAI,YAAY,KAAK,EAAE;YAC5B,OAAO,GAAG,CAAC,MAAM,CACb,IAAI,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBACvB,IAAI,WAAW,CAAC,KAAK,CAAC;oBAClB,OAAO,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;qBAC/B,IAAI,KAAK,KAAK,EAAE,IAAI,CAAC,KAAK,YAAY,iBAAiB,IAAI,KAAK,CAAC,KAAK,KAAK,EAAE,CAAC;oBAC/E,OAAO,GAAG,CAAC;gBAEf,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;YAC/B,CAAC,EAAE,EAA2B,CAAC,CAClC,CAAC;SACL;aAAM,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,SAAS;YAC5D,OAAO,GAAG,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAEtC,OAAO,IAAoB,CAAC;IAChC,CAAC;IAED,SAAS,mBAAmB,CAAC,GAA0B,EAAE,IAAoB;QACzE,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE;YAClB,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,GAAG,CAAC;SACd;QAED,MAAM,QAAQ,GAAG,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAErC,IAAI,QAAQ,YAAY,YAAY,IAAI,IAAI,YAAY,YAAY,EAAE;YAClE,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,OAAO,GAAG,CAAC;SACd;QAED,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE;YAC1D,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC;YAC5B,OAAO,GAAG,CAAC;SACd;aAAM,IAAI,QAAQ,YAAY,iBAAiB,IAAI,IAAI,YAAY,iBAAiB,EAAE;YACnF,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,GAAG,IAAI,iBAAiB,CAAC,QAAQ,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACzE,OAAO,GAAG,CAAC;SACd;QAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACf,OAAO,GAAG,CAAC;IACf,CAAC;IAED,IAAI,CAAC,sBAAsB,CAAC,OAAO,CAAC,EAAE;QAClC,OAAQ,CAAC,OAAO,CAA2B;aACtC,MAAM,CAAC,MAAM,CAAC;aACd,MAAM,CAAC,YAAY,EAAE,EAAE,CAAC;aACxB,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;KACxC;IAGD,IAAI,GAAG,GAA0B,EAAE,CAAC;IAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;QACrC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAErB,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM;YACjB,GAAG,GAAG,YAAY,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;KAC1C;IAED,OAAO,GAAG,CAAC,MAAM,CAAC,mBAAmB,EAAE,EAAE,CAAC,CAAC;AAC/C,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc;IAC1C,OAAO,KAAK,YAAY,KAAK,IAAK,KAAqC,CAAC,GAAG,YAAY,KAAK;QACxF,KAAK,CAAC,MAAM,KAAM,KAAqC,CAAC,GAAG,CAAC,MAAM,CAAC;AAC3E,CAAC;AAED,SAAS,sBAAsB,CAAC,CAAiB,EAAE,CAAiB;IAChE,IAAI,CAAC,YAAY,iBAAiB,IAAI,CAAC,YAAY,iBAAiB;QAChE,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC;SAC1B,IAAI,CAAC,YAAY,YAAY,IAAI,CAAC,YAAY,YAAY;QAC3D,OAAO,CAAC,CAAC,KAAK,KAAK,CAAC,CAAC,KAAK,CAAC;SAC1B,IAAI,CAAC,KAAK,CAAC;QACZ,OAAO,KAAK,CAAC;IAEjB,OAAO,IAAI,CAAC;AAChB,CAAC"}

package/dist/utils/StopGenerationDetector.js

@@ -1,4 +1,4 @@
-import { isLlamaText, SpecialToken } from "./LlamaText.js";
+import { SpecialToken, isLlamaText, SpecialTokensText } from "./LlamaText.js";
 export class StopGenerationDetector {
     /** @internal */ _stopTriggers = new Map();
     /** @internal */ _activeChecks = new Set();
@@ -179,6 +179,8 @@ export class StopGenerationDetector {
                 return [value];
             else if (value instanceof SpecialToken)
                 return value.tokenize(tokenizer);
+            else if (value instanceof SpecialTokensText)
+                return value.tokenize(tokenizer, true);
             return value;
         })
             .flat(1));