node-llama-cpp 3.0.0-beta.13 → 3.0.0-beta.15

package/dist/index.d.ts

@@ -2,7 +2,7 @@ import { DisposedError } from "lifecycle-utils";
 import { Llama } from "./bindings/Llama.js";
 import { getLlama, LlamaOptions } from "./bindings/getLlama.js";
 import { NoBinaryFoundError } from "./bindings/utils/NoBinaryFoundError.js";
-import { LlamaLogLevel } from "./bindings/types.js";
+import { LlamaLogLevel, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, LlamaVocabularyType } from "./bindings/types.js";
 import { LlamaModel, LlamaModelInfillTokens, type LlamaModelOptions, LlamaModelTokens } from "./evaluator/LlamaModel.js";
 import { LlamaGrammar, type LlamaGrammarOptions } from "./evaluator/LlamaGrammar.js";
 import { LlamaJsonSchemaGrammar } from "./evaluator/LlamaJsonSchemaGrammar.js";
@@ -10,12 +10,14 @@ import { LlamaJsonSchemaValidationError } from "./utils/gbnfJson/utils/validateO
 import { LlamaGrammarEvaluationState, LlamaGrammarEvaluationStateOptions } from "./evaluator/LlamaGrammarEvaluationState.js";
 import { LlamaContext, LlamaContextSequence } from "./evaluator/LlamaContext/LlamaContext.js";
 import { LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON } from "./evaluator/LlamaEmbeddingContext.js";
-import { type LlamaContextOptions, type BatchingOptions, type LlamaContextSequenceRepeatPenalty, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizeStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority } from "./evaluator/LlamaContext/types.js";
+import { type LlamaContextOptions, type BatchingOptions, type LlamaContextSequenceRepeatPenalty, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority } from "./evaluator/LlamaContext/types.js";
 import { LlamaChatSession, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty } from "./evaluator/LlamaChatSession/LlamaChatSession.js";
 import { defineChatSessionFunction } from "./evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
 import { LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LlamaChatResponse, type LlamaChatResponseFunctionCall } from "./evaluator/LlamaChat/LlamaChat.js";
 import { LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions } from "./evaluator/LlamaCompletion.js";
+import { TokenMeter, type TokenMeterState } from "./evaluator/TokenMeter.js";
 import { UnsupportedError } from "./utils/UnsupportedError.js";
+import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper, type ChatWrapperSettings } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
 import { LlamaChatWrapper } from "./chatWrappers/LlamaChatWrapper.js";
@@ -25,12 +27,16 @@ import { FalconChatWrapper } from "./chatWrappers/FalconChatWrapper.js";
 import { AlpacaChatWrapper } from "./chatWrappers/AlpacaChatWrapper.js";
 import { FunctionaryChatWrapper } from "./chatWrappers/FunctionaryChatWrapper.js";
 import { GemmaChatWrapper } from "./chatWrappers/GemmaChatWrapper.js";
-import { TemplateChatWrapper } from "./TemplateChatWrapper.js";
-import { resolveChatWrapperBasedOnModel } from "./chatWrappers/resolveChatWrapperBasedOnModel.js";
-import { resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, type ChatWrapperTypeName } from "./bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
-import { LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokenJSON } from "./utils/LlamaText.js";
+import { TemplateChatWrapper, type TemplateChatWrapperOptions } from "./chatWrappers/generic/TemplateChatWrapper.js";
+import { JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions } from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
+import { resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, resolveChatWrapper, type ResolveChatWrapperOptions } from "./chatWrappers/utils/resolveChatWrapper.js";
+import { LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON } from "./utils/LlamaText.js";
 import { appendUserMessageToChatHistory } from "./utils/appendUserMessageToChatHistory.js";
 import { getModuleVersion } from "./utils/getModuleVersion.js";
+import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type LLamaContextualRepeatPenalty } from "./types.js";
 import { type GbnfJsonArraySchema, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonObjectSchema, type GbnfJsonOneOfSchema, type GbnfJsonSchema, type GbnfJsonSchemaImmutableType, type GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
-export { Llama, getLlama, type LlamaOptions, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizeStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, UnsupportedError, DisposedError, ChatWrapper, type ChatWrapperSettings, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, resolveChatWrapperBasedOnModel, resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, type ChatWrapperTypeName, LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokenJSON, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema };
+import { type GgufFileInfo } from "./gguf/types/GgufFileInfoTypes.js";
+import { type GgufMetadata, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, type GgufMetadataRWKV, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
+import { GgmlType, type GgufTensorInfo } from "./gguf/types/GgufTensorInfoTypes.js";
+export { Llama, getLlama, type LlamaOptions, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, LlamaEmbedding, type LlamaEmbeddingJSON, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LlamaChatSessionRepeatPenalty, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, TokenMeter, type TokenMeterState, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, type ChatWrapperSettings, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, type TemplateChatWrapperOptions, JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions, resolveChatWrapper, type ResolveChatWrapperOptions, resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, type GgufFileInfo, type GgufMetadata, type GgufTensorInfo, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, type GgufMetadataRWKV, GgmlType, isGgufMetadataOfArchitectureType };

package/dist/index.js

@@ -2,7 +2,7 @@ import { DisposedError } from "lifecycle-utils";
 import { Llama } from "./bindings/Llama.js";
 import { getLlama } from "./bindings/getLlama.js";
 import { NoBinaryFoundError } from "./bindings/utils/NoBinaryFoundError.js";
-import { LlamaLogLevel } from "./bindings/types.js";
+import { LlamaLogLevel, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, LlamaVocabularyType } from "./bindings/types.js";
 import { LlamaModel, LlamaModelInfillTokens, LlamaModelTokens } from "./evaluator/LlamaModel.js";
 import { LlamaGrammar } from "./evaluator/LlamaGrammar.js";
 import { LlamaJsonSchemaGrammar } from "./evaluator/LlamaJsonSchemaGrammar.js";
@@ -14,7 +14,9 @@ import { LlamaChatSession } from "./evaluator/LlamaChatSession/LlamaChatSession.
 import { defineChatSessionFunction } from "./evaluator/LlamaChatSession/utils/defineChatSessionFunction.js";
 import { LlamaChat } from "./evaluator/LlamaChat/LlamaChat.js";
 import { LlamaCompletion } from "./evaluator/LlamaCompletion.js";
+import { TokenMeter } from "./evaluator/TokenMeter.js";
 import { UnsupportedError } from "./utils/UnsupportedError.js";
+import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
 import { LlamaChatWrapper } from "./chatWrappers/LlamaChatWrapper.js";
@@ -24,12 +26,15 @@ import { FalconChatWrapper } from "./chatWrappers/FalconChatWrapper.js";
 import { AlpacaChatWrapper } from "./chatWrappers/AlpacaChatWrapper.js";
 import { FunctionaryChatWrapper } from "./chatWrappers/FunctionaryChatWrapper.js";
 import { GemmaChatWrapper } from "./chatWrappers/GemmaChatWrapper.js";
-import { TemplateChatWrapper } from "./TemplateChatWrapper.js";
-import { resolveChatWrapperBasedOnModel } from "./chatWrappers/resolveChatWrapperBasedOnModel.js";
-import { resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames } from "./bindings/utils/resolveChatWrapperBasedOnWrapperTypeName.js";
-import { LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText } from "./utils/LlamaText.js";
+import { TemplateChatWrapper } from "./chatWrappers/generic/TemplateChatWrapper.js";
+import { JinjaTemplateChatWrapper } from "./chatWrappers/generic/JinjaTemplateChatWrapper.js";
+import { resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, resolveChatWrapper } from "./chatWrappers/utils/resolveChatWrapper.js";
+import { LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText } from "./utils/LlamaText.js";
 import { appendUserMessageToChatHistory } from "./utils/appendUserMessageToChatHistory.js";
 import { getModuleVersion } from "./utils/getModuleVersion.js";
+import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { isChatModelResponseFunctionCall } from "./types.js";
-export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, LlamaEmbeddingContext, LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaCompletion, UnsupportedError, DisposedError, ChatWrapper, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, resolveChatWrapperBasedOnModel, resolveChatWrapperBasedOnWrapperTypeName, chatWrapperTypeNames, LlamaText, SpecialToken, BuiltinSpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall };
+import { GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
+import { GgmlType } from "./gguf/types/GgufTensorInfoTypes.js";
+export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, LlamaEmbeddingContext, LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaCompletion, TokenMeter, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, EmptyChatWrapper, LlamaChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, JinjaTemplateChatWrapper, resolveChatWrapper, resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, GgmlType, isGgufMetadataOfArchitectureType };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAe,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAC,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EACH,qBAAqB,EAAqC,cAAc,EAC3E,MAAM,sCAAsC,CAAC;AAM9C,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,eAAe,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,WAAW,EAA2B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAC,MAAM,0BAA0B,CAAC;AAC7D,OAAO,EAAC,8BAA8B,EAAC,MAAM,kDAAkD,CAAC;AAChG,OAAO,EACH,wCAAwC,EAAE,oBAAoB,EACjE,MAAM,8DAA8D,CAAC;AACtE,OAAO,EACH,SAAS,EAAE,YAAY,EAAE,mBAAmB,EAAE,WAAW,EAAE,YAAY,EAE1E,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAE7D,OAAO,EAGS,+BAA+B,EAC9C,MAAM,YAAY,CAAC;AAOpB,OAAO,EACH,KAAK,EACL,QAAQ,EAER,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EAEtB,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,qBAAqB,EAErB,cAAc,EAEd,gBAAgB,EAChB,yBAAyB,EAKzB,SAAS,EAOT,eAAe,EAIf,gBAAgB,EAChB,aAAa,EACb,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EACnB,8BAA8B,EAC9B,wCAAwC,EACxC,oBAAoB,EAEpB,SAAS,EACT,YAAY,EACZ,mBAAmB,EACnB,WAAW,EACX,YAAY,EAIZ,8BAA8B,EAC9B,gBAAgB,EAUhB,+BAA+B,EAUlC,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAAe,MAAM,wBAAwB,CAAC;AAC9D,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EAAC,aAAa,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,mBAAmB,EAAC,MAAM,qBAAqB,CAAC;AAClI,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,2BAA2B,CAAC;AACvH,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EACH,qBAAqB,EAAqC,cAAc,EAC3E,MAAM,sCAAsC,CAAC;AAM9C,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,eAAe,EAClB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,UAAU,EAAuB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,WAAW,EAA2B,MAAM,kBAAkB,CAAC;AACvE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAkC,MAAM,+CAA+C,CAAC;AACnH,OAAO,EAAC,wBAAwB,EAAuC,MAAM,oDAAoD,CAAC;AAClI,OAAO,EACH,8BAA8B,EAAsC,+BAA+B,EAC9D,4BAA4B,EAAoC,kBAAkB,EAE1H,MAAM,4CAA4C,CAAC;AACpD,OAAO,EACH,SAAS,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAExE,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,gBAAgB,EAAC,MAAM,4BAA4B,CAAC;AAE5D,OAAO,EAGS,+BAA+B,EAC9C,MAAM,YAAY,CAAC;AAMpB,OAAO,EAC4C,oBAAoB,EAAE,YAAY,EAAE,8BAA8B,EACjH,mCAAmC,EAE6D,gCAAgC,EACnI,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAC,QAAQ,EAAsB,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EACH,KAAK,EACL,QAAQ,EAER,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EAEtB,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,qBAAqB,EAErB,cAAc,EAEd,gBAAgB,EAChB,yBAAyB,EAKzB,SAAS,EAOT,eAAe,EAIf,UAAU,EAEV,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,WAAW,EAEX,gBAAgB,EAChB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EAEnB,wBAAwB,EAExB,kBAAkB,EAElB,8BAA8B,EAE9B,+BAA+B,EAE/B,4BAA4B,EAE5B,SAAS,EACT,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,YAAY,EAKZ,8BAA8B,EAC9B,gBAAgB,EAUhB,+BAA+B,EAU/B,mBAAmB,EACnB,wBAAwB,EACxB,+BAA+B,EAC/B,gBAAgB,EAKhB,oBAAoB,EACpB,YAAY,EACZ,8BAA8B,EAC9B,mCAAmC,EAanC,QAAQ,EACR,gCAAgC,EACnC,CAAC"}

package/dist/state.d.ts

@@ -2,3 +2,5 @@ export declare function getIsInDocumentationMode(): boolean;
 export declare function setIsInDocumentationMode(value: boolean): void;
 export declare function getIsRunningFromCLI(): boolean;
 export declare function setIsRunningFromCLI(value: boolean): void;
+export declare function getForceShowConsoleLogPrefix(): boolean;
+export declare function setForceShowConsoleLogPrefix(value: boolean): void;

package/dist/state.js

@@ -1,5 +1,6 @@
 let isInDocumentationMode = false;
 let isInCLI = false;
+let forceShowConsoleLogPrefix = false;
 export function getIsInDocumentationMode() {
     return isInDocumentationMode;
 }
@@ -12,4 +13,10 @@ export function getIsRunningFromCLI() {
 export function setIsRunningFromCLI(value) {
     isInCLI = value;
 }
+export function getForceShowConsoleLogPrefix() {
+    return forceShowConsoleLogPrefix;
+}
+export function setForceShowConsoleLogPrefix(value) {
+    forceShowConsoleLogPrefix = value;
+}
 //# sourceMappingURL=state.js.map

package/dist/state.js.map

@@ -1 +1 @@
-{"version":3,"file":"state.js","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA,IAAI,qBAAqB,GAAG,KAAK,CAAC;AAClC,IAAI,OAAO,GAAG,KAAK,CAAC;AAEpB,MAAM,UAAU,wBAAwB;IACpC,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACnD,qBAAqB,GAAG,KAAK,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,mBAAmB;IAC/B,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAC9C,OAAO,GAAG,KAAK,CAAC;AACpB,CAAC"}
+{"version":3,"file":"state.js","sourceRoot":"","sources":["../src/state.ts"],"names":[],"mappings":"AAAA,IAAI,qBAAqB,GAAG,KAAK,CAAC;AAClC,IAAI,OAAO,GAAG,KAAK,CAAC;AACpB,IAAI,yBAAyB,GAAG,KAAK,CAAC;AAEtC,MAAM,UAAU,wBAAwB;IACpC,OAAO,qBAAqB,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,KAAc;IACnD,qBAAqB,GAAG,KAAK,CAAC;AAClC,CAAC;AAED,MAAM,UAAU,mBAAmB;IAC/B,OAAO,OAAO,CAAC;AACnB,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,KAAc;IAC9C,OAAO,GAAG,KAAK,CAAC;AACpB,CAAC;AAED,MAAM,UAAU,4BAA4B;IACxC,OAAO,yBAAyB,CAAC;AACrC,CAAC;AAED,MAAM,UAAU,4BAA4B,CAAC,KAAc;IACvD,yBAAyB,GAAG,KAAK,CAAC;AACtC,CAAC"}

package/dist/types.d.ts

@@ -4,7 +4,7 @@ export type Token = number & {
     __token: never;
 };
 export type Tokenizer = {
-    tokenize(text: string, specialTokens?: boolean): Token[];
+    tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
     tokenize(text: BuiltinSpecialTokenValue, specialTokens: "builtin"): Token[];
 }["tokenize"];
 export type ChatHistoryItem = ChatSystemMessage | ChatUserMessage | ChatModelResponse;

package/dist/utils/DisposeGuard.d.ts

@@ -0,0 +1,13 @@
+export declare class DisposeGuard {
+    constructor(parentDisposeGuards?: DisposeGuard[]);
+    addParentDisposeGuard(parent: DisposeGuard): void;
+    removeParentDisposeGuard(parent: DisposeGuard): void;
+    acquireDisposeLock(): Promise<void>;
+    createPreventDisposalHandle(ignoreAwaitingDispose?: boolean): DisposalPreventionHandle;
+}
+export declare class DisposalPreventionHandle {
+    private constructor();
+    dispose(): void;
+    [Symbol.dispose](): void;
+    get disposed(): boolean;
+}

package/dist/utils/DisposeGuard.js

@@ -0,0 +1,120 @@
+import { DisposedError } from "lifecycle-utils";
+export class DisposeGuard {
+    /** @internal */ _preventionHandles = 0;
+    /** @internal */ _awaitingDisposeLockCallbacks = [];
+    /** @internal */ _disposeActivated = false;
+    /** @internal */ _parentDisposeGuardsLocks = new Map();
+    constructor(parentDisposeGuards = []) {
+        for (const parent of parentDisposeGuards)
+            this._parentDisposeGuardsLocks.set(parent, null);
+    }
+    addParentDisposeGuard(parent) {
+        if (this._parentDisposeGuardsLocks.has(parent))
+            return;
+        this._parentDisposeGuardsLocks.set(parent, null);
+        if (this._preventionHandles > 0)
+            this._parentDisposeGuardsLocks.set(parent, parent.createPreventDisposalHandle(true));
+    }
+    removeParentDisposeGuard(parent) {
+        const parentLock = this._parentDisposeGuardsLocks.get(parent);
+        if (parentLock != null) {
+            parentLock.dispose();
+            this._parentDisposeGuardsLocks.delete(parent);
+        }
+    }
+    async acquireDisposeLock() {
+        return new Promise((accept) => {
+            if (this._preventionHandles > 0)
+                this._awaitingDisposeLockCallbacks.push(accept);
+            else {
+                this._disposeActivated = true;
+                accept();
+            }
+        });
+    }
+    createPreventDisposalHandle(ignoreAwaitingDispose = false) {
+        if (this._isDisposeActivated() || (!ignoreAwaitingDispose && this._hasAwaitingDisposeLocks()))
+            throw new DisposedError();
+        this._preventionHandles++;
+        try {
+            this._updateParentDisposeGuardLocks();
+        }
+        catch (err) {
+            this._preventionHandles--;
+            if (this._preventionHandles === 0)
+                this._updateParentDisposeGuardLocks();
+            throw err;
+        }
+        return DisposalPreventionHandle._create(() => {
+            this._preventionHandles--;
+            this._activateLocksIfNeeded();
+            this._updateParentDisposeGuardLocks(true);
+        });
+    }
+    /** @internal */
+    _isDisposeActivated() {
+        if (this._disposeActivated)
+            return true;
+        return [...this._parentDisposeGuardsLocks.keys()].some((parent) => parent._isDisposeActivated());
+    }
+    /** @internal */
+    _activateLocksIfNeeded() {
+        if (this._preventionHandles > 0)
+            return;
+        while (this._awaitingDisposeLockCallbacks.length > 0) {
+            this._disposeActivated = true;
+            this._awaitingDisposeLockCallbacks.shift()();
+        }
+    }
+    /** @internal */
+    _updateParentDisposeGuardLocks(onlyAllowRemoval = false) {
+        if (this._preventionHandles === 0) {
+            for (const parent of this._parentDisposeGuardsLocks.keys()) {
+                const parentLock = this._parentDisposeGuardsLocks.get(parent);
+                if (parentLock == null)
+                    continue;
+                parentLock.dispose();
+                this._parentDisposeGuardsLocks.set(parent, null);
+            }
+        }
+        else if (!onlyAllowRemoval) {
+            for (const parent of this._parentDisposeGuardsLocks.keys()) {
+                if (this._parentDisposeGuardsLocks.get(parent) != null)
+                    continue;
+                this._parentDisposeGuardsLocks.set(parent, parent.createPreventDisposalHandle(true));
+            }
+        }
+    }
+    /** @internal */
+    _hasAwaitingDisposeLocks() {
+        if (this._awaitingDisposeLockCallbacks.length > 0)
+            return true;
+        return [...this._parentDisposeGuardsLocks.keys()].some((parent) => parent._hasAwaitingDisposeLocks());
+    }
+}
+export class DisposalPreventionHandle {
+    /** @internal */
+    _dispose;
+    constructor(dispose) {
+        this._dispose = dispose;
+        this.dispose = this.dispose.bind(this);
+        this[Symbol.dispose] = this[Symbol.dispose].bind(this);
+    }
+    dispose() {
+        if (this._dispose != null) {
+            this._dispose();
+            this._dispose = null;
+        }
+    }
+    [Symbol.dispose]() {
+        this.dispose();
+    }
+    get disposed() {
+        return this._dispose == null;
+    }
+    /** @internal */
+    static _create(dispose) {
+        return new DisposalPreventionHandle(dispose);
+    }
+}
+//# sourceMappingURL=DisposeGuard.js.map

package/dist/utils/DisposeGuard.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DisposeGuard.js","sourceRoot":"","sources":["../../src/utils/DisposeGuard.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,MAAM,OAAO,YAAY;IACrB,gBAAgB,CAAS,kBAAkB,GAAW,CAAC,CAAC;IACxD,gBAAgB,CAAS,6BAA6B,GAAmB,EAAE,CAAC;IAC5E,gBAAgB,CAAS,iBAAiB,GAAY,KAAK,CAAC;IAC5D,gBAAgB,CAAS,yBAAyB,GAAuD,IAAI,GAAG,EAAE,CAAC;IAEnH,YAAmB,sBAAsC,EAAE;QACvD,KAAK,MAAM,MAAM,IAAI,mBAAmB;YACpC,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IACzD,CAAC;IAEM,qBAAqB,CAAC,MAAoB;QAC7C,IAAI,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC;YAC1C,OAAO;QAEX,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAEjD,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC3B,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;IAC7F,CAAC;IAEM,wBAAwB,CAAC,MAAoB;QAChD,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAE9D,IAAI,UAAU,IAAI,IAAI,EAAE;YACpB,UAAU,CAAC,OAAO,EAAE,CAAC;YACrB,IAAI,CAAC,yBAAyB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;SACjD;IACL,CAAC;IAEM,KAAK,CAAC,kBAAkB;QAC3B,OAAO,IAAI,OAAO,CAAO,CAAC,MAAM,EAAE,EAAE;YAChC,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;gBAC3B,IAAI,CAAC,6BAA6B,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;iBAC/C;gBACD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;gBAC9B,MAAM,EAAE,CAAC;aACZ;QACL,CAAC,CAAC,CAAC;IACP,CAAC;IAEM,2BAA2B,CAAC,wBAAiC,KAAK;QACrE,IAAI,IAAI,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC,qBAAqB,IAAI,IAAI,CAAC,wBAAwB,EAAE,CAAC;YACzF,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,IAAI,CAAC,kBAAkB,EAAE,CAAC;QAC1B,IAAI;YACA,IAAI,CAAC,8BAA8B,EAAE,CAAC;SACzC;QAAC,OAAO,GAAG,EAAE;YACV,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,IAAI,CAAC,kBAAkB,KAAK,CAAC;gBAC7B,IAAI,CAAC,8BAA8B,EAAE,CAAC;YAE1C,MAAM,GAAG,CAAC;SACb;QAED,OAAO,wBAAwB,CAAC,OAAO,CAAC,GAAG,EAAE;YACzC,IAAI,CAAC,kBAAkB,EAAE,CAAC;YAE1B,IAAI,CAAC,sBAAsB,EAAE,CAAC;YAC9B,IAAI,CAAC,8BAA8B,CAAC,IAAI,CAAC,CAAC;QAC9C,CAAC,CAAC,CAAC;IACP,CAAC;IAED,gBAAgB;IACR,mBAAmB;QACvB,IAAI,IAAI,CAAC,iBAAiB;YACtB,OAAO,IAAI,CAAC;QAEhB,OAAO,CAAC,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,mBAAmB,EAAE,CAAC,CAAC;IACrG,CAAC;IAED,gBAAgB;IACR,sBAAsB;QAC1B,IAAI,IAAI,CAAC,kBAAkB,GAAG,CAAC;YAC3B,OAAO;QAEX,OAAO,IAAI,CAAC,6BAA6B,CAAC,MAAM,GAAG,CAAC,EAAE;YAClD,IAAI,CAAC,iBAAiB,GAAG,IAAI,CAAC;YAC9B,IAAI,CAAC,6BAA6B,CAAC,KAAK,EAAG,EAAE,CAAC;SACjD;IACL,CAAC;IAED,gBAAgB;IACR,8BAA8B,CAAC,mBAA4B,KAAK;QACpE,IAAI,IAAI,CAAC,kBAAkB,KAAK,CAAC,EAAE;YAC/B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,EAAE;gBACxD,MAAM,UAAU,GAAG,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAE9D,IAAI,UAAU,IAAI,IAAI;oBAClB,SAAS;gBAEb,UAAU,CAAC,OAAO,EAAE,CAAC;gBACrB,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;aACpD;SACJ;aAAM,IAAI,CAAC,gBAAgB,EAAE;YAC1B,KAAK,MAAM,MAAM,IAAI,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,EAAE;gBACxD,IAAI,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,IAAI;oBAClD,SAAS;gBAEb,IAAI,CAAC,yBAAyB,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,2BAA2B,CAAC,IAAI,CAAC,CAAC,CAAC;aACxF;SACJ;IACL,CAAC;IAED,gBAAgB;IACR,wBAAwB;QAC5B,IAAI,IAAI,CAAC,6BAA6B,CAAC,MAAM,GAAG,CAAC;YAC7C,OAAO,IAAI,CAAC;QAEhB,OAAO,CAAC,GAAG,IAAI,CAAC,yBAAyB,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC,CAAC;IAC1G,CAAC;CACJ;AAED,MAAM,OAAO,wBAAwB;IACjC,gBAAgB;IACR,QAAQ,CAAsB;IAEtC,YAAoB,OAAmB;QACnC,IAAI,CAAC,QAAQ,GAAG,OAAO,CAAC;QAExB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3D,CAAC;IAEM,OAAO;QACV,IAAI,IAAI,CAAC,QAAQ,IAAI,IAAI,EAAE;YACvB,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;SACxB;IACL,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC;IACjC,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,OAAmB;QACrC,OAAO,IAAI,wBAAwB,CAAC,OAAO,CAAC,CAAC;IACjD,CAAC;CACJ"}

package/dist/utils/InsufficientMemoryError.d.ts

@@ -0,0 +1,3 @@
+export declare class InsufficientMemoryError extends Error {
+    constructor(message?: string);
+}

package/dist/utils/InsufficientMemoryError.js

@@ -0,0 +1,6 @@
+export class InsufficientMemoryError extends Error {
+    constructor(message = "Insufficient memory") {
+        super(message);
+    }
+}
+//# sourceMappingURL=InsufficientMemoryError.js.map

package/dist/utils/InsufficientMemoryError.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"InsufficientMemoryError.js","sourceRoot":"","sources":["../../src/utils/InsufficientMemoryError.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,uBAAwB,SAAQ,KAAK;IAC9C,YAAmB,UAAkB,qBAAqB;QACtD,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

package/dist/utils/LlamaText.d.ts

@@ -2,6 +2,7 @@ import { Token, Tokenizer } from "../types.js";
 export type LlamaTextClass = {
     <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText<V> | readonly (LlamaText<V> | V)[], ...values: readonly (V | LlamaText<V> | V2 | LlamaText<V2> | number | boolean | readonly (LlamaText<V> | V | LlamaText<V2> | V2)[])[]): LlamaText<V | V2>;
     fromJSON(json: LlamaTextJSON): LlamaText;
+    compare(a: LlamaText, b: LlamaText): boolean;
 };
 export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
     <const V extends LlamaTextValue = LlamaTextValue, const V2 extends LlamaTextValue = LlamaTextValue>(strings: TemplateStringsArray | V | LlamaText | readonly (LlamaText | V)[], ...values: readonly (V | V2 | LlamaText | number | boolean | readonly (LlamaText | V | V2)[])[]): LlamaText<T | V>;
@@ -11,32 +12,46 @@ export type LlamaText<T extends LlamaTextValue = LlamaTextValue> = {
     joinValues<V extends LlamaTextValue = LlamaTextValue>(separator: LlamaText<V> | V): LlamaText<T | V>;
     toString(): string;
     toJSON(): LlamaTextJSON;
-    tokenize(tokenizer: Tokenizer): Token[];
+    tokenize(tokenizer: Tokenizer, options?: "trimLeadingSpace"): Token[];
+    compare(other: LlamaText): boolean;
+    trimStart(): LlamaText<T>;
+    trimEnd(): LlamaText<T>;
+    includes(value: LlamaText): boolean;
 };
-export type LlamaTextValue = string | SpecialToken;
+export type LlamaTextValue = string | SpecialTokensText | SpecialToken;
 export type LlamaTextJSON = Array<LlamaTextJSONValue>;
-export type LlamaTextJSONValue = string | LlamaTextSpecialTokenJSON;
+export type LlamaTextJSONValue = string | LlamaTextSpecialTokensTextJSON | LlamaTextSpecialTokenJSON;
+export type LlamaTextSpecialTokensTextJSON = {
+    type: "specialTokensText";
+    value: string;
+};
 export type LlamaTextSpecialTokenJSON = {
     type: "specialToken";
     value: string;
-    builtin?: true;
 };
 export declare const LlamaText: LlamaTextClass;
-export declare class SpecialToken {
+export declare class SpecialTokensText {
     readonly value: string;
     constructor(value: string);
     toString(): string;
-    tokenize(tokenizer: Tokenizer): Token[];
-    toJSON(): LlamaTextSpecialTokenJSON;
-    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
-    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
+    tokenize(tokenizer: Tokenizer, trimLeadingSpace?: boolean): Token[];
+    toJSON(): LlamaTextSpecialTokensTextJSON;
+    static fromJSON(json: LlamaTextSpecialTokensTextJSON): SpecialTokensText;
+    static isSpecialTokensTextJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokensTextJSON;
+    /**
+     * Wraps the value with a `SpecialTokensText` only if `shouldWrap` is true
+     */
+    static wrapIf(shouldWrap: boolean, value: string): SpecialTokensText | string;
 }
 export type BuiltinSpecialTokenValue = "BOS" | "EOS" | "NL";
-export declare class BuiltinSpecialToken extends SpecialToken {
+export declare class SpecialToken {
     readonly value: BuiltinSpecialTokenValue;
     constructor(value: BuiltinSpecialTokenValue);
+    toString(): BuiltinSpecialTokenValue;
     tokenize(tokenizer: Tokenizer): Token[];
     toJSON(): LlamaTextSpecialTokenJSON;
+    static fromJSON(json: LlamaTextSpecialTokenJSON): SpecialToken;
+    static isSpecialTokenJSON(value: LlamaTextJSONValue): value is LlamaTextSpecialTokenJSON;
 }
 export declare function isLlamaText(value: unknown): value is LlamaText;
 export declare function tokenizeText(text: string | LlamaText, tokenizer: Tokenizer): Token[];