node-llama-cpp 3.0.0-beta.37 → 3.0.0-beta.39

package/dist/gguf/types/GgufMetadataTypes.js.map

@@ -1 +1 @@
-{"version":3,"file":"GgufMetadataTypes.js","sourceRoot":"","sources":["../../../src/gguf/types/GgufMetadataTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAkB,oBAyCjB;AAzCD,WAAkB,oBAAoB;IAClC,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,2CAAmB,CAAA;IACnB,mCAAW,CAAA;IACX,6CAAqB,CAAA;IACrB,+CAAuB,CAAA;IACvB,yCAAiB,CAAA;IACjB,qCAAa,CAAA;IACb,gDAAwB,CAAA;IACxB,mDAA2B,CAAA;IAC3B,uCAAe,CAAA;IACf,6CAAqB,CAAA;IACrB,qCAAa,CAAA;IACb,uCAAe,CAAA;IACf,6CAAqB,CAAA;IACrB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,uCAAe,CAAA;IACf,+CAAuB,CAAA;IACvB,uCAAe,CAAA;IACf,+CAAuB,CAAA;IACvB,2CAAmB,CAAA;IACnB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,iDAAyB,CAAA;IACzB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,8CAAsB,CAAA;IACtB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,2CAAmB,CAAA;IACnB,yCAAiB,CAAA;IACjB,+CAAuB,CAAA;IACvB,yCAAiB,CAAA;IACjB,iCAAS,CAAA;IACT,qCAAa,CAAA;IACb,6CAAqB,CAAA;AACzB,CAAC,EAzCiB,oBAAoB,KAApB,oBAAoB,QAyCrC;AA8BD,yEAAyE;AACzE,MAAM,CAAN,IAAY,YAgCX;AAhCD,WAAY,YAAY;IACpB,qDAAW,CAAA;IACX,2DAAc,CAAA;IACd,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,+EAAwB,CAAA;IACxB,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,8DAAgB,CAAA;IAChB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,8DAAgB,CAAA;IAChB,oEAAmB,CAAA;IACnB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,oEAAmB,CAAA;IACnB,gEAAiB,CAAA;IACjB,kEAAkB,CAAA;IAClB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,kEAAkB,CAAA;AACtB,CAAC,EAhCW,YAAY,KAAZ,YAAY,QAgCvB;AAyED,MAAM,CAAN,IAAkB,8BAQjB;AARD,WAAkB,8BAA8B;IAC5C,6FAAa,CAAA;IACb,uFAAU,CAAA;IACV,yFAAW,CAAA;IACX,yFAAW,CAAA;IACX,iGAAe,CAAA;IACf,uFAAU,CAAA;IACV,mFAAQ,CAAA;AACZ,CAAC,EARiB,8BAA8B,KAA9B,8BAA8B,QAQ/C;AAiCD,MAAM,CAAN,IAAkB,mCAKjB;AALD,WAAkB,mCAAmC;IACjD,4GAAgB,CAAA;IAChB,6FAAQ,CAAA;IACR,6FAAQ,CAAA;IACR,2FAAO,CAAA;AACX,CAAC,EALiB,mCAAmC,KAAnC,mCAAmC,QAKpD;AA8MD,MAAM,UAAU,gCAAgC,CAC5C,QAAsB,EAAE,IAAO;IAE/B,OAAO,QAAQ,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;AACpD,CAAC"}
+{"version":3,"file":"GgufMetadataTypes.js","sourceRoot":"","sources":["../../../src/gguf/types/GgufMetadataTypes.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAkB,oBAyCjB;AAzCD,WAAkB,oBAAoB;IAClC,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,2CAAmB,CAAA;IACnB,mCAAW,CAAA;IACX,6CAAqB,CAAA;IACrB,+CAAuB,CAAA;IACvB,yCAAiB,CAAA;IACjB,qCAAa,CAAA;IACb,gDAAwB,CAAA;IACxB,mDAA2B,CAAA;IAC3B,uCAAe,CAAA;IACf,6CAAqB,CAAA;IACrB,qCAAa,CAAA;IACb,uCAAe,CAAA;IACf,6CAAqB,CAAA;IACrB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,uCAAe,CAAA;IACf,+CAAuB,CAAA;IACvB,uCAAe,CAAA;IACf,+CAAuB,CAAA;IACvB,2CAAmB,CAAA;IACnB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,iDAAyB,CAAA;IACzB,uCAAe,CAAA;IACf,yCAAiB,CAAA;IACjB,8CAAsB,CAAA;IACtB,qCAAa,CAAA;IACb,qCAAa,CAAA;IACb,2CAAmB,CAAA;IACnB,yCAAiB,CAAA;IACjB,+CAAuB,CAAA;IACvB,yCAAiB,CAAA;IACjB,iCAAS,CAAA;IACT,qCAAa,CAAA;IACb,6CAAqB,CAAA;AACzB,CAAC,EAzCiB,oBAAoB,KAApB,oBAAoB,QAyCrC;AA8BD,yEAAyE;AACzE,MAAM,CAAN,IAAY,YAgCX;AAhCD,WAAY,YAAY;IACpB,qDAAW,CAAA;IACX,2DAAc,CAAA;IACd,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,+EAAwB,CAAA;IACxB,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,6DAAe,CAAA;IACf,8DAAgB,CAAA;IAChB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,8DAAgB,CAAA;IAChB,oEAAmB,CAAA;IACnB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,kEAAkB,CAAA;IAClB,oEAAmB,CAAA;IACnB,gEAAiB,CAAA;IACjB,kEAAkB,CAAA;IAClB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,gEAAiB,CAAA;IACjB,kEAAkB,CAAA;AACtB,CAAC,EAhCW,YAAY,KAAZ,YAAY,QAgCvB;AA0FD,MAAM,CAAN,IAAkB,8BAQjB;AARD,WAAkB,8BAA8B;IAC5C,6FAAa,CAAA;IACb,uFAAU,CAAA;IACV,yFAAW,CAAA;IACX,yFAAW,CAAA;IACX,iGAAe,CAAA;IACf,uFAAU,CAAA;IACV,mFAAQ,CAAA;AACZ,CAAC,EARiB,8BAA8B,KAA9B,8BAA8B,QAQ/C;AAiCD,MAAM,CAAN,IAAkB,mCAKjB;AALD,WAAkB,mCAAmC;IACjD,4GAAgB,CAAA;IAChB,6FAAQ,CAAA;IACR,6FAAQ,CAAA;IACR,2FAAO,CAAA;AACX,CAAC,EALiB,mCAAmC,KAAnC,mCAAmC,QAKpD;AA8MD,MAAM,UAAU,gCAAgC,CAC5C,QAAsB,EAAE,IAAO;IAE/B,OAAO,QAAQ,EAAE,OAAO,EAAE,YAAY,KAAK,IAAI,CAAC;AACpD,CAAC"}

package/dist/gguf/utils/convertMetadataKeyValueRecordToNestedObject.d.ts

@@ -1,5 +1,6 @@
 import { MetadataKeyValueRecord, MetadataValue } from "../types/GgufFileInfoTypes.js";
-export declare function convertMetadataKeyValueRecordToNestedObject(keyValueRecord: MetadataKeyValueRecord, { logOverrideWarnings, ignoreKeys }?: {
+export declare function convertMetadataKeyValueRecordToNestedObject(keyValueRecord: MetadataKeyValueRecord, { logOverrideWarnings, ignoreKeys, noDirectSubNestingKeys }?: {
     logOverrideWarnings?: boolean;
-    ignoreKeys?: string[];
+    ignoreKeys?: readonly string[];
+    noDirectSubNestingKeys?: readonly string[];
 }): Record<string, MetadataValue>;

package/dist/gguf/utils/convertMetadataKeyValueRecordToNestedObject.js

@@ -1,23 +1,36 @@
 import { getConsoleLogPrefix } from "../../utils/getConsoleLogPrefix.js";
-export function convertMetadataKeyValueRecordToNestedObject(keyValueRecord, { logOverrideWarnings = true, ignoreKeys = [] } = {}) {
+export function convertMetadataKeyValueRecordToNestedObject(keyValueRecord, { logOverrideWarnings = true, ignoreKeys = [], noDirectSubNestingKeys } = {}) {
     const nestedObject = {};
     const ignoreKeySet = new Set(ignoreKeys);
+    const noDirectSubNestingKeysSet = new Set(noDirectSubNestingKeys);
     for (const [key, value] of Object.entries(keyValueRecord)) {
         if (ignoreKeySet.has(key))
             continue;
-        const { lastObject, lastKey } = getNestedObject(key, nestedObject);
-        if (Object.hasOwn(lastObject, lastKey) && logOverrideWarnings)
-            console.warn(getConsoleLogPrefix() + `Metadata key "${key}" is already occupied by a value. Overwriting it.`);
+        const { lastObject, lastKey } = getNestedObject(key, nestedObject, noDirectSubNestingKeysSet);
+        if (Object.hasOwn(lastObject, lastKey)) {
+            const currentValue = lastObject[lastKey];
+            delete lastObject[lastKey];
+            flattenNestedKeys(lastObject, lastKey, currentValue, logOverrideWarnings);
+            if (Object.hasOwn(lastObject, lastKey) && logOverrideWarnings)
+                console.warn(getConsoleLogPrefix() + `Metadata key "${key}" is already occupied by a value. Overwriting it.`);
+        }
         lastObject[lastKey] = value;
     }
     return nestedObject;
 }
-function getNestedObject(key, nestedObject) {
+function getNestedObject(key, nestedObject, noDirectSubNestingKeysSet) {
     const nestedKey = key.split(".");
-    const lastKey = nestedKey.pop();
+    let lastKey = "";
     let currentObject = nestedObject;
+    const previousKeys = [];
     while (nestedKey.length > 0) {
-        const currentKey = nestedKey.shift();
+        let currentKey = nestedKey.shift();
+        while (noDirectSubNestingKeysSet.has([...previousKeys, currentKey].join(".")) && nestedKey.length > 0)
+            currentKey += "." + nestedKey.shift();
+        if (nestedKey.length === 0) {
+            lastKey = currentKey;
+            break;
+        }
         if (!Object.hasOwn(currentObject, currentKey)) {
             const nextCurrentObject = {};
             currentObject[currentKey] = nextCurrentObject;
@@ -25,14 +38,37 @@ function getNestedObject(key, nestedObject) {
         }
         else {
             const value = currentObject[currentKey];
-            if (value instanceof Array || value == null || typeof value !== "object")
+            if (value instanceof Array || value == null || typeof value !== "object") {
+                if (nestedKey.length > 0) {
+                    nestedKey.unshift(currentKey + "." + nestedKey.shift());
+                    continue;
+                }
                 throw new Error(`Cannot create nested object for key "${key}". The key "${currentKey}" is already occupied by a non-object value.`);
+            }
             currentObject = value;
         }
+        previousKeys.push(currentKey);
     }
     return {
         lastObject: currentObject,
         lastKey
     };
 }
+function flattenNestedKeys(parent, newParentKey, keyValue, logOverrideWarnings = false) {
+    if (typeof keyValue !== "object" || keyValue instanceof Array) {
+        parent[newParentKey] = keyValue;
+        return;
+    }
+    for (const [key, subValue] of Object.entries(keyValue)) {
+        const newKey = newParentKey + "." + key;
+        if (Object.hasOwn(parent, newKey)) {
+            const currentValue = parent[newKey];
+            delete parent[newKey];
+            flattenNestedKeys(parent, newKey, currentValue, logOverrideWarnings);
+            if (Object.hasOwn(parent, newKey) && logOverrideWarnings)
+                console.warn(getConsoleLogPrefix() + `Metadata key "${newKey}" is already occupied by a value. Overwriting it.`);
+        }
+        parent[newKey] = subValue;
+    }
+}
 //# sourceMappingURL=convertMetadataKeyValueRecordToNestedObject.js.map

package/dist/gguf/utils/convertMetadataKeyValueRecordToNestedObject.js.map

@@ -1 +1 @@
-{"version":3,"file":"convertMetadataKeyValueRecordToNestedObject.js","sourceRoot":"","sources":["../../../src/gguf/utils/convertMetadataKeyValueRecordToNestedObject.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAGvE,MAAM,UAAU,2CAA2C,CACvD,cAAsC,EACtC,EACI,mBAAmB,GAAG,IAAI,EAC1B,UAAU,GAAG,EAAE,KAIf,EAAE;IAEN,MAAM,YAAY,GAAkC,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IAEzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QACxD,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;YACrB,SAAS;QAEb,MAAM,EAAC,UAAU,EAAE,OAAO,EAAC,GAAG,eAAe,CAAC,GAAG,EAAE,YAAY,CAAC,CAAC;QACjE,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,mBAAmB;YACzD,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,iBAAiB,GAAG,mDAAmD,CAAC,CAAC;QAElH,UAAU,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW,EAAE,YAAkC;IACpE,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,MAAM,OAAO,GAAG,SAAS,CAAC,GAAG,EAAG,CAAC;IAEjC,IAAI,aAAa,GAAG,YAAY,CAAC;IAEjC,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,EAAG,CAAC;QACtC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5C,MAAM,iBAAiB,GAAG,EAAE,CAAC;YAC7B,aAAa,CAAC,UAAU,CAAC,GAAG,iBAAiB,CAAC;YAE9C,aAAa,GAAG,iBAAiB,CAAC;QACtC,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;gBACpE,MAAM,IAAI,KAAK,CACX,wCAAwC,GAAG,eAAe,UAAU,8CAA8C,CACrH,CAAC;YAEN,aAAa,GAAG,KAAK,CAAC;QAC1B,CAAC;IACL,CAAC;IAED,OAAO;QACH,UAAU,EAAE,aAAa;QACzB,OAAO;KACV,CAAC;AACN,CAAC"}
+{"version":3,"file":"convertMetadataKeyValueRecordToNestedObject.js","sourceRoot":"","sources":["../../../src/gguf/utils/convertMetadataKeyValueRecordToNestedObject.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,mBAAmB,EAAC,MAAM,oCAAoC,CAAC;AAGvE,MAAM,UAAU,2CAA2C,CACvD,cAAsC,EACtC,EACI,mBAAmB,GAAG,IAAI,EAC1B,UAAU,GAAG,EAAE,EACf,sBAAsB,KAKtB,EAAE;IAEN,MAAM,YAAY,GAAkC,EAAE,CAAC;IACvD,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,yBAAyB,GAAG,IAAI,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAElE,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;QACxD,IAAI,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC;YACrB,SAAS;QAEb,MAAM,EAAC,UAAU,EAAE,OAAO,EAAC,GAAG,eAAe,CAAC,GAAG,EAAE,YAAY,EAAE,yBAAyB,CAAC,CAAC;QAC5F,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,EAAE,CAAC;YACrC,MAAM,YAAY,GAAG,UAAU,CAAC,OAAO,CAAC,CAAC;YACzC,OAAO,UAAU,CAAC,OAAO,CAAC,CAAC;YAC3B,iBAAiB,CAAC,UAAU,EAAE,OAAO,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;YAE1E,IAAI,MAAM,CAAC,MAAM,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,mBAAmB;gBACzD,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,iBAAiB,GAAG,mDAAmD,CAAC,CAAC;QACtH,CAAC;QAED,UAAU,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC;IAChC,CAAC;IAED,OAAO,YAAY,CAAC;AACxB,CAAC;AAED,SAAS,eAAe,CAAC,GAAW,EAAE,YAAkC,EAAE,yBAAsC;IAC5G,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,OAAO,GAAG,EAAE,CAAC;IAEjB,IAAI,aAAa,GAAG,YAAY,CAAC;IAEjC,MAAM,YAAY,GAAG,EAAE,CAAC;IACxB,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC1B,IAAI,UAAU,GAAG,SAAS,CAAC,KAAK,EAAG,CAAC;QAEpC,OAAO,yBAAyB,CAAC,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC;YACjG,UAAU,IAAI,GAAG,GAAG,SAAS,CAAC,KAAK,EAAG,CAAC;QAE3C,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,OAAO,GAAG,UAAU,CAAC;YACrB,MAAM;QACV,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,aAAa,EAAE,UAAU,CAAC,EAAE,CAAC;YAC5C,MAAM,iBAAiB,GAAG,EAAE,CAAC;YAC7B,aAAa,CAAC,UAAU,CAAC,GAAG,iBAAiB,CAAC;YAE9C,aAAa,GAAG,iBAAiB,CAAC;QACtC,CAAC;aAAM,CAAC;YACJ,MAAM,KAAK,GAAG,aAAa,CAAC,UAAU,CAAC,CAAC;YACxC,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,IAAI,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvE,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACvB,SAAS,CAAC,OAAO,CAAC,UAAU,GAAG,GAAG,GAAG,SAAS,CAAC,KAAK,EAAG,CAAC,CAAC;oBACzD,SAAS;gBACb,CAAC;gBAED,MAAM,IAAI,KAAK,CACX,wCAAwC,GAAG,eAAe,UAAU,8CAA8C,CACrH,CAAC;YACN,CAAC;YAED,aAAa,GAAG,KAAK,CAAC;QAC1B,CAAC;QAED,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAClC,CAAC;IAED,OAAO;QACH,UAAU,EAAE,aAAa;QACzB,OAAO;KACV,CAAC;AACN,CAAC;AAED,SAAS,iBAAiB,CACtB,MAA4B,EAC5B,YAAoB,EACpB,QAA8C,EAC9C,sBAA+B,KAAK;IAEpC,IAAI,OAAO,QAAQ,KAAK,QAAQ,IAAI,QAAQ,YAAY,KAAK,EAAE,CAAC;QAC5D,MAAM,CAAC,YAAY,CAAC,GAAG,QAAQ,CAAC;QAChC,OAAO;IACX,CAAC;IAED,KAAK,MAAM,CAAC,GAAG,EAAE,QAAQ,CAAC,IAAK,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAsD,EAAE,CAAC;QAC3G,MAAM,MAAM,GAAG,YAAY,GAAG,GAAG,GAAG,GAAG,CAAC;QAExC,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;YACpC,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC;YACtB,iBAAiB,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,mBAAmB,CAAC,CAAC;YAErE,IAAI,MAAM,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,mBAAmB;gBACpD,OAAO,CAAC,IAAI,CAAC,mBAAmB,EAAE,GAAG,iBAAiB,MAAM,mDAAmD,CAAC,CAAC;QACzH,CAAC;QAED,MAAM,CAAC,MAAM,CAAC,GAAG,QAAQ,CAAC;IAC9B,CAAC;AACL,CAAC"}

package/dist/index.d.ts

@@ -23,6 +23,7 @@ import { UnsupportedError } from "./utils/UnsupportedError.js";
 import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
+import { Llama3_1ChatWrapper } from "./chatWrappers/Llama3_1ChatWrapper.js";
 import { Llama3ChatWrapper } from "./chatWrappers/Llama3ChatWrapper.js";
 import { Llama2ChatWrapper } from "./chatWrappers/Llama2ChatWrapper.js";
 import { GeneralChatWrapper } from "./chatWrappers/GeneralChatWrapper.js";
@@ -43,9 +44,10 @@ import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { GgufInsights, type GgufInsightsResourceRequirements } from "./gguf/insights/GgufInsights.js";
 import { GgufInsightsConfigurationResolver } from "./gguf/insights/GgufInsightsConfigurationResolver.js";
 import { createModelDownloader, ModelDownloader, type ModelDownloaderOptions } from "./utils/createModelDownloader.js";
-import { type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, type Tokenizer, type Detokenizer, isChatModelResponseFunctionCall, type LLamaContextualRepeatPenalty, type ChatWrapperSettings, type ChatWrapperGenerateContextStateOptions, type ChatWrapperGeneratedContextState } from "./types.js";
+import { jsonDumps } from "./chatWrappers/utils/jsonDumps.js";
+import { type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, type Tokenizer, type Detokenizer, isChatModelResponseFunctionCall, type LLamaContextualRepeatPenalty, type ChatWrapperSettings, type ChatWrapperGenerateContextStateOptions, type ChatWrapperGeneratedContextState, type ChatWrapperGenerateInitialHistoryOptions } from "./types.js";
 import { type GbnfJsonArraySchema, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonObjectSchema, type GbnfJsonOneOfSchema, type GbnfJsonSchema, type GbnfJsonSchemaImmutableType, type GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
 import { type GgufFileInfo } from "./gguf/types/GgufFileInfoTypes.js";
 import { type GgufMetadata, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
 import { GgmlType, type GgufTensorInfo } from "./gguf/types/GgufTensorInfoTypes.js";
-export { Llama, getLlama, type LlamaOptions, type LastBuildOptions, type LlamaGpuType, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, TokenAttributes, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, TokenBias, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, type LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LLamaChatCompletePromptOptions, type LlamaChatSessionRepeatPenalty, type LLamaChatPreloadPromptOptions, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatLoadAndCompleteUserMessageOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, type LlamaChatLoadAndCompleteUserResponse, LlamaChatSessionPromptCompletionEngine, type LLamaChatPromptCompletionEngineOptions, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, type LlamaCompletionResponse, TokenMeter, type TokenMeterState, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, type ChatWrapperSettings, type ChatWrapperGenerateContextStateOptions, type ChatWrapperGeneratedContextState, EmptyChatWrapper, Llama3ChatWrapper, Llama2ChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, type TemplateChatWrapperOptions, JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions, type JinjaTemplateChatWrapperOptionsConvertMessageFormat, type ChatHistoryFunctionCallMessageTemplate, resolveChatWrapper, type ResolveChatWrapperOptions, resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, ChatModelFunctionsDocumentationGenerator, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextValue, type LlamaTextInputValue, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON, type BuiltinSpecialTokenValue, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, type Tokenizer, type Detokenizer, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, type GgufFileInfo, type GgufMetadata, type GgufTensorInfo, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, GgmlType, isGgufMetadataOfArchitectureType, GgufInsights, type GgufInsightsResourceRequirements, GgufInsightsConfigurationResolver, createModelDownloader, ModelDownloader, type ModelDownloaderOptions };
+export { Llama, getLlama, type LlamaOptions, type LastBuildOptions, type LlamaGpuType, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, TokenAttributes, type LlamaModelOptions, LlamaGrammar, type LlamaGrammarOptions, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, type LlamaGrammarEvaluationStateOptions, LlamaContext, LlamaContextSequence, type LlamaContextOptions, type BatchingOptions, type CustomBatchingDispatchSchedule, type CustomBatchingPrioritizationStrategy, type BatchItem, type PrioritizedBatchItem, type ContextShiftOptions, type ContextTokensDeleteRange, type EvaluationPriority, type LlamaContextSequenceRepeatPenalty, TokenBias, LlamaEmbeddingContext, type LlamaEmbeddingContextOptions, type LlamaEmbedding, LlamaChatSession, defineChatSessionFunction, type LlamaChatSessionOptions, type LlamaChatSessionContextShiftOptions, type LLamaChatPromptOptions, type LLamaChatCompletePromptOptions, type LlamaChatSessionRepeatPenalty, type LLamaChatPreloadPromptOptions, LlamaChat, type LlamaChatOptions, type LLamaChatGenerateResponseOptions, type LLamaChatLoadAndCompleteUserMessageOptions, type LLamaChatContextShiftOptions, type LLamaContextualRepeatPenalty, type LlamaChatResponse, type LlamaChatResponseFunctionCall, type LlamaChatLoadAndCompleteUserResponse, LlamaChatSessionPromptCompletionEngine, type LLamaChatPromptCompletionEngineOptions, LlamaCompletion, type LlamaCompletionOptions, type LlamaCompletionGenerationOptions, type LlamaInfillGenerationOptions, type LlamaCompletionResponse, TokenMeter, type TokenMeterState, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, type ChatWrapperSettings, type ChatWrapperGenerateContextStateOptions, type ChatWrapperGeneratedContextState, type ChatWrapperGenerateInitialHistoryOptions, EmptyChatWrapper, Llama3_1ChatWrapper, Llama3ChatWrapper, Llama2ChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, type TemplateChatWrapperOptions, JinjaTemplateChatWrapper, type JinjaTemplateChatWrapperOptions, type JinjaTemplateChatWrapperOptionsConvertMessageFormat, type ChatHistoryFunctionCallMessageTemplate, resolveChatWrapper, type ResolveChatWrapperOptions, resolvableChatWrapperTypeNames, type ResolvableChatWrapperTypeName, specializedChatWrapperTypeNames, type SpecializedChatWrapperTypeName, templateChatWrapperTypeNames, type TemplateChatWrapperTypeName, ChatModelFunctionsDocumentationGenerator, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, type LlamaTextValue, type LlamaTextInputValue, type LlamaTextJSON, type LlamaTextJSONValue, type LlamaTextSpecialTokensTextJSON, type LlamaTextSpecialTokenJSON, type BuiltinSpecialTokenValue, appendUserMessageToChatHistory, getModuleVersion, type ChatHistoryItem, type ChatModelFunctionCall, type ChatModelFunctions, type ChatModelResponse, type ChatSessionModelFunction, type ChatSessionModelFunctions, type ChatSystemMessage, type ChatUserMessage, type Token, type Tokenizer, type Detokenizer, isChatModelResponseFunctionCall, type GbnfJsonSchema, type GbnfJsonSchemaToType, type GbnfJsonSchemaImmutableType, type GbnfJsonBasicSchema, type GbnfJsonConstSchema, type GbnfJsonEnumSchema, type GbnfJsonOneOfSchema, type GbnfJsonObjectSchema, type GbnfJsonArraySchema, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, type GgufFileInfo, type GgufMetadata, type GgufTensorInfo, type GgufMetadataLlmToType, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, type GgufMetadataGeneral, type GgufMetadataTokenizer, type GgufMetadataDefaultArchitectureType, type GgufMetadataLlmLLaMA, type GgufMetadataMPT, type GgufMetadataGPTNeoX, type GgufMetadataGPTJ, type GgufMetadataGPT2, type GgufMetadataBloom, type GgufMetadataFalcon, type GgufMetadataMamba, GgmlType, isGgufMetadataOfArchitectureType, GgufInsights, type GgufInsightsResourceRequirements, GgufInsightsConfigurationResolver, createModelDownloader, ModelDownloader, type ModelDownloaderOptions, jsonDumps };

package/dist/index.js

@@ -22,6 +22,7 @@ import { UnsupportedError } from "./utils/UnsupportedError.js";
 import { InsufficientMemoryError } from "./utils/InsufficientMemoryError.js";
 import { ChatWrapper } from "./ChatWrapper.js";
 import { EmptyChatWrapper } from "./chatWrappers/EmptyChatWrapper.js";
+import { Llama3_1ChatWrapper } from "./chatWrappers/Llama3_1ChatWrapper.js";
 import { Llama3ChatWrapper } from "./chatWrappers/Llama3ChatWrapper.js";
 import { Llama2ChatWrapper } from "./chatWrappers/Llama2ChatWrapper.js";
 import { GeneralChatWrapper } from "./chatWrappers/GeneralChatWrapper.js";
@@ -41,8 +42,9 @@ import { readGgufFileInfo } from "./gguf/readGgufFileInfo.js";
 import { GgufInsights } from "./gguf/insights/GgufInsights.js";
 import { GgufInsightsConfigurationResolver } from "./gguf/insights/GgufInsightsConfigurationResolver.js";
 import { createModelDownloader, ModelDownloader } from "./utils/createModelDownloader.js";
+import { jsonDumps } from "./chatWrappers/utils/jsonDumps.js";
 import { isChatModelResponseFunctionCall } from "./types.js";
 import { GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, isGgufMetadataOfArchitectureType } from "./gguf/types/GgufMetadataTypes.js";
 import { GgmlType } from "./gguf/types/GgufTensorInfoTypes.js";
-export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, TokenAttributes, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, TokenBias, LlamaEmbeddingContext, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaChatSessionPromptCompletionEngine, LlamaCompletion, TokenMeter, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, EmptyChatWrapper, Llama3ChatWrapper, Llama2ChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, JinjaTemplateChatWrapper, resolveChatWrapper, resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, ChatModelFunctionsDocumentationGenerator, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, GgmlType, isGgufMetadataOfArchitectureType, GgufInsights, GgufInsightsConfigurationResolver, createModelDownloader, ModelDownloader };
+export { Llama, getLlama, LlamaLogLevel, NoBinaryFoundError, LlamaModel, LlamaModelTokens, LlamaModelInfillTokens, TokenAttributes, LlamaGrammar, LlamaJsonSchemaGrammar, LlamaJsonSchemaValidationError, LlamaGrammarEvaluationState, LlamaContext, LlamaContextSequence, TokenBias, LlamaEmbeddingContext, LlamaChatSession, defineChatSessionFunction, LlamaChat, LlamaChatSessionPromptCompletionEngine, LlamaCompletion, TokenMeter, UnsupportedError, InsufficientMemoryError, DisposedError, ChatWrapper, EmptyChatWrapper, Llama3_1ChatWrapper, Llama3ChatWrapper, Llama2ChatWrapper, GeneralChatWrapper, ChatMLChatWrapper, FalconChatWrapper, AlpacaChatWrapper, FunctionaryChatWrapper, GemmaChatWrapper, TemplateChatWrapper, JinjaTemplateChatWrapper, resolveChatWrapper, resolvableChatWrapperTypeNames, specializedChatWrapperTypeNames, templateChatWrapperTypeNames, ChatModelFunctionsDocumentationGenerator, LlamaText, SpecialTokensText, SpecialToken, isLlamaText, tokenizeText, appendUserMessageToChatHistory, getModuleVersion, isChatModelResponseFunctionCall, LlamaVocabularyType, LlamaLogLevelGreaterThan, LlamaLogLevelGreaterThanOrEqual, readGgufFileInfo, GgufArchitectureType, GgufFileType, GgufMetadataTokenizerTokenType, GgufMetadataArchitecturePoolingType, GgmlType, isGgufMetadataOfArchitectureType, GgufInsights, GgufInsightsConfigurationResolver, createModelDownloader, ModelDownloader, jsonDumps };
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAA2C,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EACgB,aAAa,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,mBAAmB,EACnH,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAClI,OAAO,EAAC,eAAe,EAAC,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EAAC,qBAAqB,EAAyD,MAAM,sCAAsC,CAAC;AAMnI,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,sCAAsC,EACzC,MAAM,8EAA8E,CAAC;AACtF,OAAO,EACH,eAAe,EAElB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,UAAU,EAAuB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAkC,MAAM,+CAA+C,CAAC;AACnH,OAAO,EACH,wBAAwB,EAC3B,MAAM,oDAAoD,CAAC;AAE5D,OAAO,EACH,8BAA8B,EAAsC,+BAA+B,EAC9D,4BAA4B,EAAoC,kBAAkB,EAE1H,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAC,wCAAwC,EAAC,MAAM,kEAAkE,CAAC;AAC1H,OAAO,EACH,SAAS,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAGxE,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,gBAAgB,EAAC,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAC,YAAY,EAAwC,MAAM,iCAAiC,CAAC;AACpG,OAAO,EAAC,iCAAiC,EAAC,MAAM,sDAAsD,CAAC;AACvG,OAAO,EAAC,qBAAqB,EAAE,eAAe,EAA8B,MAAM,kCAAkC,CAAC;AAErH,OAAO,EAG2C,+BAA+B,EAEhF,MAAM,YAAY,CAAC;AAMpB,OAAO,EAC4C,oBAAoB,EAAE,YAAY,EAAE,8BAA8B,EACjH,mCAAmC,EAEsC,gCAAgC,EAC5G,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAC,QAAQ,EAAsB,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EACH,KAAK,EACL,QAAQ,EAIR,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EAEf,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,SAAS,EACT,qBAAqB,EAGrB,gBAAgB,EAChB,yBAAyB,EAOzB,SAAS,EAST,sCAAsC,EAEtC,eAAe,EAKf,UAAU,EAEV,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,WAAW,EAIX,gBAAgB,EAChB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EAEnB,wBAAwB,EAIxB,kBAAkB,EAElB,8BAA8B,EAE9B,+BAA+B,EAE/B,4BAA4B,EAE5B,wCAAwC,EACxC,SAAS,EACT,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,YAAY,EAQZ,8BAA8B,EAC9B,gBAAgB,EAYhB,+BAA+B,EAU/B,mBAAmB,EACnB,wBAAwB,EACxB,+BAA+B,EAC/B,gBAAgB,EAKhB,oBAAoB,EACpB,YAAY,EACZ,8BAA8B,EAC9B,mCAAmC,EAYnC,QAAQ,EACR,gCAAgC,EAChC,YAAY,EAEZ,iCAAiC,EACjC,qBAAqB,EACrB,eAAe,EAElB,CAAC"}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAC9C,OAAO,EAAC,KAAK,EAAC,MAAM,qBAAqB,CAAC;AAC1C,OAAO,EAAC,QAAQ,EAA2C,MAAM,wBAAwB,CAAC;AAC1F,OAAO,EAAC,kBAAkB,EAAC,MAAM,wCAAwC,CAAC;AAC1E,OAAO,EACgB,aAAa,EAAE,wBAAwB,EAAE,+BAA+B,EAAE,mBAAmB,EACnH,MAAM,qBAAqB,CAAC;AAC7B,OAAO,EAAC,UAAU,EAAE,sBAAsB,EAA0B,gBAAgB,EAAC,MAAM,sCAAsC,CAAC;AAClI,OAAO,EAAC,eAAe,EAAC,MAAM,iDAAiD,CAAC;AAChF,OAAO,EAAC,YAAY,EAA2B,MAAM,6BAA6B,CAAC;AACnF,OAAO,EAAC,sBAAsB,EAAC,MAAM,uCAAuC,CAAC;AAC7E,OAAO,EAAC,8BAA8B,EAAC,MAAM,2DAA2D,CAAC;AACzG,OAAO,EAAC,2BAA2B,EAAqC,MAAM,4CAA4C,CAAC;AAC3H,OAAO,EAAC,YAAY,EAAE,oBAAoB,EAAC,MAAM,0CAA0C,CAAC;AAC5F,OAAO,EAAC,qBAAqB,EAAyD,MAAM,sCAAsC,CAAC;AAMnI,OAAO,EAAC,SAAS,EAAC,MAAM,0BAA0B,CAAC;AACnD,OAAO,EACH,gBAAgB,EAEnB,MAAM,kDAAkD,CAAC;AAC1D,OAAO,EAAC,yBAAyB,EAAC,MAAM,iEAAiE,CAAC;AAC1G,OAAO,EACH,SAAS,EAEZ,MAAM,oCAAoC,CAAC;AAC5C,OAAO,EACH,sCAAsC,EACzC,MAAM,8EAA8E,CAAC;AACtF,OAAO,EACH,eAAe,EAElB,MAAM,gCAAgC,CAAC;AACxC,OAAO,EAAC,UAAU,EAAuB,MAAM,2BAA2B,CAAC;AAC3E,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,uBAAuB,EAAC,MAAM,oCAAoC,CAAC;AAC3E,OAAO,EAAC,WAAW,EAAC,MAAM,kBAAkB,CAAC;AAC7C,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAC,MAAM,uCAAuC,CAAC;AAC1E,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,kBAAkB,EAAC,MAAM,sCAAsC,CAAC;AACxE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,iBAAiB,EAAC,MAAM,qCAAqC,CAAC;AACtE,OAAO,EAAC,sBAAsB,EAAC,MAAM,0CAA0C,CAAC;AAChF,OAAO,EAAC,gBAAgB,EAAC,MAAM,oCAAoC,CAAC;AACpE,OAAO,EAAC,mBAAmB,EAAkC,MAAM,+CAA+C,CAAC;AACnH,OAAO,EACH,wBAAwB,EAC3B,MAAM,oDAAoD,CAAC;AAE5D,OAAO,EACH,8BAA8B,EAAsC,+BAA+B,EAC9D,4BAA4B,EAAoC,kBAAkB,EAE1H,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAAC,wCAAwC,EAAC,MAAM,kEAAkE,CAAC;AAC1H,OAAO,EACH,SAAS,EAAE,iBAAiB,EAAE,YAAY,EAAE,WAAW,EAAE,YAAY,EAGxE,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EAAC,8BAA8B,EAAC,MAAM,2CAA2C,CAAC;AACzF,OAAO,EAAC,gBAAgB,EAAC,MAAM,6BAA6B,CAAC;AAC7D,OAAO,EAAC,gBAAgB,EAAC,MAAM,4BAA4B,CAAC;AAC5D,OAAO,EAAC,YAAY,EAAwC,MAAM,iCAAiC,CAAC;AACpG,OAAO,EAAC,iCAAiC,EAAC,MAAM,sDAAsD,CAAC;AACvG,OAAO,EAAC,qBAAqB,EAAE,eAAe,EAA8B,MAAM,kCAAkC,CAAC;AACrH,OAAO,EAAC,SAAS,EAAC,MAAM,mCAAmC,CAAC;AAE5D,OAAO,EAG2C,+BAA+B,EAGhF,MAAM,YAAY,CAAC;AAMpB,OAAO,EAC4C,oBAAoB,EAAE,YAAY,EAAE,8BAA8B,EACjH,mCAAmC,EAEsC,gCAAgC,EAC5G,MAAM,mCAAmC,CAAC;AAC3C,OAAO,EAAC,QAAQ,EAAsB,MAAM,qCAAqC,CAAC;AAGlF,OAAO,EACH,KAAK,EACL,QAAQ,EAIR,aAAa,EACb,kBAAkB,EAClB,UAAU,EACV,gBAAgB,EAChB,sBAAsB,EACtB,eAAe,EAEf,YAAY,EAEZ,sBAAsB,EACtB,8BAA8B,EAC9B,2BAA2B,EAE3B,YAAY,EACZ,oBAAoB,EAWpB,SAAS,EACT,qBAAqB,EAGrB,gBAAgB,EAChB,yBAAyB,EAOzB,SAAS,EAST,sCAAsC,EAEtC,eAAe,EAKf,UAAU,EAEV,gBAAgB,EAChB,uBAAuB,EACvB,aAAa,EACb,WAAW,EAKX,gBAAgB,EAChB,mBAAmB,EACnB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,iBAAiB,EACjB,sBAAsB,EACtB,gBAAgB,EAChB,mBAAmB,EAEnB,wBAAwB,EAIxB,kBAAkB,EAElB,8BAA8B,EAE9B,+BAA+B,EAE/B,4BAA4B,EAE5B,wCAAwC,EACxC,SAAS,EACT,iBAAiB,EACjB,YAAY,EACZ,WAAW,EACX,YAAY,EAQZ,8BAA8B,EAC9B,gBAAgB,EAYhB,+BAA+B,EAU/B,mBAAmB,EACnB,wBAAwB,EACxB,+BAA+B,EAC/B,gBAAgB,EAKhB,oBAAoB,EACpB,YAAY,EACZ,8BAA8B,EAC9B,mCAAmC,EAYnC,QAAQ,EACR,gCAAgC,EAChC,YAAY,EAEZ,iCAAiC,EACjC,qBAAqB,EACrB,eAAe,EAEf,SAAS,EACZ,CAAC"}

package/dist/types.d.ts

@@ -1,10 +1,11 @@
 import { GbnfJsonSchema, GbnfJsonSchemaToType } from "./utils/gbnfJson/types.js";
 import { LlamaText, BuiltinSpecialTokenValue, LlamaTextJSON } from "./utils/LlamaText.js";
+import type { GgufFileInfo } from "./gguf/types/GgufFileInfoTypes.js";
 export type Token = number & {
     __token: never;
 };
 export type Detokenizer = {
-    detokenize(tokens: readonly Token[], specialTokens?: boolean): string;
+    detokenize(tokens: readonly Token[], specialTokens?: boolean, lastTokens?: readonly Token[]): string;
 }["detokenize"];
 export type Tokenizer = {
     tokenize(text: string, specialTokens?: boolean, options?: "trimLeadingSpace"): Token[];
@@ -78,6 +79,10 @@ export type ChatWrapperGenerateContextStateOptions = {
     availableFunctions?: ChatModelFunctions;
     documentFunctionParams?: boolean;
 };
+export type ChatWrapperCheckModelCompatibilityParams = {
+    tokenizer?: Tokenizer;
+    fileInfo?: GgufFileInfo;
+};
 export type ChatWrapperGeneratedContextState = {
     contextText: LlamaText;
     stopGenerationTriggers: LlamaText[];
@@ -87,6 +92,9 @@ export type ChatWrapperGeneratedContextState = {
         disengageInitiallyEngaged: LlamaText[];
     };
 };
+export type ChatWrapperGenerateInitialHistoryOptions = {
+    systemPrompt?: string;
+};
 export type ChatHistoryItem = ChatSystemMessage | ChatUserMessage | ChatModelResponse;
 export type ChatSystemMessage = {
     type: "system";
@@ -107,6 +115,12 @@ export type ChatModelFunctionCall = {
     params: any;
     result: any;
     rawCall?: LlamaTextJSON;
+    /**
+     * Whether this function call starts a new function calling chunk.
+     *
+     * Relevant only when parallel function calling is supported.
+     */
+    startsNewChunk?: boolean;
 };
 export type ChatModelFunctions = {
     readonly [name: string]: {

package/dist/types.js.map

@@ -1 +1 @@
-{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA2IA,MAAM,UAAU,+BAA+B,CAAC,IAA2C;IACvF,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,KAAK,CAAC;IAEjB,OAAO,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;AACxC,CAAC"}
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AA4JA,MAAM,UAAU,+BAA+B,CAAC,IAA2C;IACvF,IAAI,OAAO,IAAI,KAAK,QAAQ;QACxB,OAAO,KAAK,CAAC;IAEjB,OAAO,IAAI,CAAC,IAAI,KAAK,cAAc,CAAC;AACxC,CAAC"}

package/dist/utils/DeepPartialObject.d.ts

@@ -0,0 +1,3 @@
+export type DeepPartialObject<T, AllowedValueTypes> = T extends object ? {
+    [P in keyof T]?: DeepPartialObject<T[P], AllowedValueTypes>;
+} : T extends Array<infer I> ? AllowedValueTypes extends Array<any> ? Array<DeepPartialObject<I, AllowedValueTypes>> : never : T extends ReadonlyArray<infer I> ? AllowedValueTypes extends ReadonlyArray<any> ? ReadonlyArray<DeepPartialObject<I, AllowedValueTypes>> : never : AllowedValueTypes extends T ? T : never;

package/dist/utils/DeepPartialObject.js

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=DeepPartialObject.js.map

package/dist/utils/DeepPartialObject.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"DeepPartialObject.js","sourceRoot":"","sources":["../../src/utils/DeepPartialObject.ts"],"names":[],"mappings":""}

package/dist/utils/StopGenerationDetector.d.ts

@@ -1,4 +1,4 @@
-import { Detokenizer, Token, Tokenizer } from "../types.js";
+import { Token, Tokenizer } from "../types.js";
 import { LlamaText } from "./LlamaText.js";
 import { QueuedTokenRelease, QueuedTokenReleaseLock } from "./TokenStreamRegulator.js";
 export type StopGenerationTrigger = (string | Token)[];
@@ -33,8 +33,11 @@ export declare class StopGenerationDetector<T extends string = string> {
     }): number;
     static resolveStopTriggers(stopTriggers: readonly (string | Readonly<StopGenerationTrigger> | LlamaText)[], tokenizer: Tokenizer): StopGenerationTrigger[];
     static resolveLlamaTextTrigger(llamaText: LlamaText, tokenizer: Tokenizer): StopGenerationTrigger;
-    static getFirstRemainingGenerationAfterStop(triggeredStops: TriggeredStop[]): string | Token[] | undefined;
-    static detokenizeRemainingGeneration(remainingGeneration: string | Token[] | undefined, detokenizer: Detokenizer, specialTokens?: boolean): string;
+    static getFirstRemainingGenerationAfterStop(triggeredStops: TriggeredStop[]): {
+        stopTrigger: StopGenerationTrigger | undefined;
+        firstRemainingGenerationAfterStop: string | Token[] | undefined;
+    };
+    static detokenizeRemainingGeneration(remainingGeneration: string | Token[] | undefined, stopTrigger: StopGenerationTrigger | undefined, tokenizer: Tokenizer, specialTokens?: boolean): string;
 }
 export type TriggeredStop<T extends string = string> = {
     stopTrigger: StopGenerationTrigger;

package/dist/utils/StopGenerationDetector.js

@@ -241,18 +241,19 @@ export class StopGenerationDetector {
             .flat(1));
     }
     static getFirstRemainingGenerationAfterStop(triggeredStops) {
-        const [firstRemainingGenerationAfterStop] = triggeredStops
-            .map((stopTrigger) => stopTrigger.remainingGeneration)
-            .filter((remainingGenerations) => remainingGenerations.length > 0)
-            .flat(1);
-        return firstRemainingGenerationAfterStop;
+        const [stopTrigger] = triggeredStops
+            .filter((stopTrigger) => (stopTrigger.remainingGeneration.some((remainingGeneration) => remainingGeneration.length > 0)));
+        return {
+            stopTrigger: stopTrigger?.stopTrigger ?? triggeredStops?.[0]?.stopTrigger,
+            firstRemainingGenerationAfterStop: stopTrigger?.remainingGeneration?.filter((remainingGeneration) => remainingGeneration.length > 0)?.[0]
+        };
     }
-    static detokenizeRemainingGeneration(remainingGeneration, detokenizer, specialTokens = false) {
+    static detokenizeRemainingGeneration(remainingGeneration, stopTrigger, tokenizer, specialTokens = false) {
         if (remainingGeneration == null || remainingGeneration.length === 0)
             return "";
         if (typeof remainingGeneration === "string")
             return remainingGeneration;
-        return detokenizer(remainingGeneration, specialTokens);
+        return tokenizer.detokenize(remainingGeneration, specialTokens, tokenizeStopTrigger(stopTrigger, tokenizer, specialTokens));
     }
 }
 function simplifyStopTrigger(stopTrigger) {
@@ -273,4 +274,18 @@ function simplifyStopTrigger(stopTrigger) {
         res.push(text);
     return res;
 }
+function tokenizeStopTrigger(stopTrigger, tokenizer, specialTokens = false) {
+    if (stopTrigger == null)
+        return [];
+    const res = [];
+    for (const item of stopTrigger) {
+        if (typeof item === "string") {
+            const tokens = tokenizer(item, specialTokens, "trimLeadingSpace");
+            res.push(...tokens);
+        }
+        else
+            res.push(item);
+    }
+    return res;
+}
 //# sourceMappingURL=StopGenerationDetector.js.map

package/dist/utils/StopGenerationDetector.js.map

@@ -1 +1 @@
-{"version":3,"file":"StopGenerationDetector.js","sourceRoot":"","sources":["../../src/utils/StopGenerationDetector.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAE,WAAW,EAAa,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAKvF,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAS,aAAa,GAAG,IAAI,GAAG,EAAkC,CAAC;IACnF,gBAAgB,CAAS,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;IACpE,gBAAgB,CAAS,eAAe,GAAG,IAAI,GAAG,EAG9C,CAAC;IAEE,gBAAgB,CAAC,EACpB,IAAI,EACJ,MAAM,EACN,kBAAkB,EAClB,cAAc,GAAG,IAAI,EACrB,8BAA8B,GAAG,KAAK,EAOzC;QACG,MAAM,mBAAmB,GAAG,IAAI,CAAC,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,mBAAmB,EAAE,CAAC;YACtC,IAAI,SAAS,GAAG,KAAK,CAAC;YAEtB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBACf,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACnC,CAAC;gBACF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBACjB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;iBACrC,CAAC;gBACF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YAED,IAAI,CAAC,SAAS;gBACV,KAAK,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,cAAc;YACf,OAAO;QAEX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnF,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAEjD,IAAI,WAAW,IAAI,IAAI;gBACnB,SAAS;YAEb,MAAM,SAAS,GAAoB;gBAC/B,sBAAsB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAClE,WAAW;aACd,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAErD,SAAS,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAChD,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACrF,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAElD,IAAI,WAAW,IAAI,IAAI;gBACnB,SAAS;YAEb,MAAM,UAAU,GAAoB;gBAChC,sBAAsB,EAAE,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CAAC;gBACnE,WAAW;aACd,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAExD,UAAU,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QACjD,CAAC;IACL,CAAC;IAEM,cAAc,CAAC,WAAkC,EAAE,aAAiB;QACvE,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,iBAAiB;aAClC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,OAAO,IAAI,KAAK,QAAQ;gBACxB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;;gBAEtB,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,CAAC;QAEb,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,KAAK,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE;oBAClB,IAAI,EAAE,IAAI,GAAG,EAAE;iBAClB,CAAC,CAAC;YACP,CAAC;YAED,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;gBACtB,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,GAAG,EAAE,CAAC;gBAEvD,IAAI,aAAa,IAAI,IAAI;oBACrB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;gBACxB,MAAM;;gBAEN,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,gFAAgF;IAChF,IAAW,kBAAkB;QACzB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,yDAAyD;IAClD,iBAAiB;QACpB,MAAM,GAAG,GAAuB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YACxE,GAAG,CAAC,IAAI,CAAC;gBACL,WAAW,EAAE,WAAW,CAAC,gBAAiB;gBAC1C,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,IAAI,IAAI,GAAG,EAAE,CAAC;gBAC3D,mBAAmB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC;gBACnE,uBAAuB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC;aAC7E,CAAC,CAAC;QACP,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,mBAAmB;QACtB,KAAK,MAAM,aAAa,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YACxD,KAAK,MAAM,sBAAsB,IAAI,aAAa,CAAC,uBAAuB;gBACtE,sBAAsB,CAAC,OAAO,EAAE,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAEM,oBAAoB;QACvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa;YAClC,KAAK,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAE5C,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACI,8CAA8C,CAAC,EAClD,IAAI,EAAE,MAAM,EAAE,cAAc,EAM/B;QACG,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,MAAM,4BAA4B,GAAG,IAAI,CAAC,0CAA0C,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAC9G,MAAM,6BAA6B,GAAG,IAAI,CAAC,0CAA0C,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAEjH,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,4BAA4B,EAAE,6BAA6B,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,4BAA4B,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;gBAChD,CAAC,CAAC,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjG,CAAC,CAAC,IAAI,CAAC;YACX,MAAM,6BAA6B,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrG,CAAC,CAAC,IAAI,CAAC;YAEX,IAAI,4BAA4B,IAAI,IAAI,IAAI,6BAA6B,IAAI,IAAI;gBAC7E,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,4BAA4B,EAAE,6BAA6B,CAAC,CAAC;iBAC5E,IAAI,4BAA4B,IAAI,IAAI;gBACzC,GAAG,IAAI,4BAA4B,CAAC;iBACnC,IAAI,6BAA6B,IAAI,IAAI;gBAC1C,GAAG,IAAI,6BAA6B,CAAC;QAC7C,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,gBAAgB;IACR,aAAa,CACjB,IAAoB,EACpB,mBAAsC,EACtC,sBAA+C;QAE/C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;YAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE;gBAC3B,oBAAoB,EAAE,IAAI,GAAG,EAAE;gBAC/B,uBAAuB,EAAE,IAAI,GAAG,EAAE;aACrC,CAAC,CAAC;QAEP,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAEtD,IAAI,mBAAmB,IAAI,IAAI;YAC3B,aAAa,CAAC,oBAAoB,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAEhE,IAAI,sBAAsB,IAAI,IAAI;YAC9B,aAAa,CAAC,uBAAuB,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAC1E,CAAC;IAED,gBAAgB;IACR,0CAA0C,CAAC,WAAuC,EAAE,KAAuB;QAC/G,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,CAAC,CAAC;QAEb,IAAI,IAAI,GAA+B,WAAW,CAAC;QACnD,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;gBACjB,OAAO,GAAG,GAAG,CAAC,CAAC;YAEnB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC1B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YAED,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;YACjC,OAAO,GAAG,GAAG,CAAC,CAAC;QAEnB,OAAO,GAAG,CAAC;IACf,CAAC;IAED,gBAAgB;IACR,iBAAiB,CAAC,KAAkC,EAAE,KAAuB;QACjF,IAAI,KAAK,IAAI,IAAI;YACb,OAAO,KAAK,CAAC;QAEjB,IAAI,IAAI,GAA+B,KAAK,CAAC,WAAW,CAAC;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;gBACtF,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YAED,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,IAAI,IAAI;YACZ,OAAO,KAAK,CAAC;QAEjB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACjF,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;gBACnB,GAAG,KAAK;gBACR,WAAW,EAAE,IAAI;gBACjB,sBAAsB,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE;aACtE,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,mBAAmB,CAC7B,YAA+E,EAC/E,SAAoB;QAEpB,OAAO,YAAY;aACd,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YACjB,IAAI,WAAW,CAAC,WAAW,CAAC;gBACxB,OAAO,sBAAsB,CAAC,uBAAuB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;iBAC7E,IAAI,OAAO,WAAW,KAAK,QAAQ;gBACpC,OAAO,mBAAmB,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;;gBAE1C,OAAO,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAEM,MAAM,CAAC,uBAAuB,CACjC,SAAoB,EACpB,SAAoB;QAEpB,OAAO,mBAAmB,CACtB,SAAS,CAAC,MAAM;aACX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;aAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACX,IAAI,OAAO,KAAK,KAAK,QAAQ;gBACzB,OAAO,CAAC,KAAK,CAAC,CAAC;iBACd,IAAI,KAAK,YAAY,YAAY;gBAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;iBAChC,IAAI,KAAK,YAAY,iBAAiB;gBACvC,OAAO,KAAK,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC;YAEtD,OAAO,KAAqB,CAAC;QACjC,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,CACf,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,oCAAoC,CAAC,cAA+B;QAC9E,MAAM,CAAC,iCAAiC,CAAC,GAAG,cAAc;aACrD,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,mBAAmB,CAAC;aACrD,MAAM,CAAC,CAAC,oBAAoB,EAAE,EAAE,CAAC,oBAAoB,CAAC,MAAM,GAAG,CAAC,CAAC;aACjE,IAAI,CAAC,CAAC,CAAC,CAAC;QAEb,OAAO,iCAAiC,CAAC;IAC7C,CAAC;IAEM,MAAM,CAAC,6BAA6B,CACvC,mBAAiD,EACjD,WAAwB,EACxB,gBAAyB,KAAK;QAE9B,IAAI,mBAAmB,IAAI,IAAI,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC;YAC/D,OAAO,EAAE,CAAC;QAEd,IAAI,OAAO,mBAAmB,KAAK,QAAQ;YACvC,OAAO,mBAAmB,CAAC;QAE/B,OAAO,WAAW,CAAC,mBAAmB,EAAE,aAAa,CAAC,CAAC;IAC3D,CAAC;CACJ;AAED,SAAS,mBAAmB,CAAC,WAA4C;IACrE,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,GAAG,GAA0B,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,IAAI,IAAI,CAAC;YACb,SAAS;QACb,CAAC;QAED,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,IAAI,GAAG,EAAE,CAAC;QACd,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,KAAK,EAAE;QACX,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnB,OAAO,GAAG,CAAC;AACf,CAAC"}
+{"version":3,"file":"StopGenerationDetector.js","sourceRoot":"","sources":["../../src/utils/StopGenerationDetector.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAE,WAAW,EAAa,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAKvF,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAS,aAAa,GAAG,IAAI,GAAG,EAAkC,CAAC;IACnF,gBAAgB,CAAS,aAAa,GAAG,IAAI,GAAG,EAAmB,CAAC;IACpE,gBAAgB,CAAS,eAAe,GAAG,IAAI,GAAG,EAG9C,CAAC;IAEE,gBAAgB,CAAC,EACpB,IAAI,EACJ,MAAM,EACN,kBAAkB,EAClB,cAAc,GAAG,IAAI,EACrB,8BAA8B,GAAG,KAAK,EAOzC;QACG,MAAM,mBAAmB,GAAG,IAAI,CAAC,aAAa,CAAC;QAC/C,IAAI,CAAC,aAAa,GAAG,IAAI,GAAG,EAAE,CAAC;QAE/B,KAAK,MAAM,KAAK,IAAI,mBAAmB,EAAE,CAAC;YACtC,IAAI,SAAS,GAAG,KAAK,CAAC;YAEtB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBACf,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;iBACnC,CAAC;gBACF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;gBACjB,IAAI,CAAC,iBAAiB,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;iBACrC,CAAC;gBACF,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;gBAC9B,SAAS,GAAG,IAAI,CAAC;YACrB,CAAC;YAED,IAAI,CAAC,SAAS;gBACV,KAAK,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAChD,CAAC;QAED,IAAI,CAAC,cAAc;YACf,OAAO;QAEX,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACnF,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;YACrB,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAEjD,IAAI,WAAW,IAAI,IAAI;gBACnB,SAAS;YAEb,MAAM,SAAS,GAAoB;gBAC/B,sBAAsB,EAAE,kBAAkB,EAAE,mBAAmB,CAAC,CAAC,CAAC;gBAClE,WAAW;aACd,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,SAAS,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAErD,SAAS,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAChD,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,IAAI,CAAC,CAAC,8BAA8B,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACrF,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACxB,MAAM,WAAW,GAAG,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAElD,IAAI,WAAW,IAAI,IAAI;gBACnB,SAAS;YAEb,MAAM,UAAU,GAAoB;gBAChC,sBAAsB,EAAE,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CAAC;gBACnE,WAAW;aACd,CAAC;YACF,IAAI,CAAC,iBAAiB,CAAC,UAAU,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAExD,UAAU,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QACjD,CAAC;IACL,CAAC;IAEM,cAAc,CAAC,WAAkC,EAAE,aAAiB;QACvE,MAAM,iBAAiB,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAC3D,MAAM,aAAa,GAAG,iBAAiB;aAClC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE;YACV,IAAI,OAAO,IAAI,KAAK,QAAQ;gBACxB,OAAO,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;;gBAEtB,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,CAAC;QAEb,IAAI,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC;QAEpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5C,MAAM,KAAK,GAAG,aAAa,CAAC,CAAC,CAAC,CAAC;YAC/B,MAAM,MAAM,GAAG,CAAC,KAAK,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC;YAE9C,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,UAAU,CAAC,GAAG,CAAC,KAAK,EAAE;oBAClB,IAAI,EAAE,IAAI,GAAG,EAAE;iBAClB,CAAC,CAAC;YACP,CAAC;YAED,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAE,CAAC;YACpC,IAAI,MAAM,EAAE,CAAC;gBACT,IAAI,CAAC,IAAI,GAAG,SAAS,CAAC;gBACtB,IAAI,CAAC,gBAAgB,GAAG,iBAAiB,CAAC;gBAC1C,IAAI,CAAC,cAAc,GAAG,IAAI,CAAC,cAAc,IAAI,IAAI,GAAG,EAAE,CAAC;gBAEvD,IAAI,aAAa,IAAI,IAAI;oBACrB,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;YAC/C,CAAC;iBAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;gBACxB,MAAM;;gBAEN,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,uEAAuE;IACvE,IAAW,iBAAiB;QACxB,OAAO,IAAI,CAAC,eAAe,CAAC,IAAI,GAAG,CAAC,CAAC;IACzC,CAAC;IAED,gFAAgF;IAChF,IAAW,kBAAkB;QACzB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;IACvC,CAAC;IAED,yDAAyD;IAClD,iBAAiB;QACpB,MAAM,GAAG,GAAuB,EAAE,CAAC;QAEnC,KAAK,MAAM,CAAC,WAAW,EAAE,aAAa,CAAC,IAAI,IAAI,CAAC,eAAe,CAAC,OAAO,EAAE,EAAE,CAAC;YACxE,GAAG,CAAC,IAAI,CAAC;gBACL,WAAW,EAAE,WAAW,CAAC,gBAAiB;gBAC1C,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,cAAc,IAAI,IAAI,GAAG,EAAE,CAAC;gBAC3D,mBAAmB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,oBAAoB,CAAC;gBACnE,uBAAuB,EAAE,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,uBAAuB,CAAC;aAC7E,CAAC,CAAC;QACP,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,mBAAmB;QACtB,KAAK,MAAM,aAAa,IAAI,IAAI,CAAC,eAAe,CAAC,MAAM,EAAE,EAAE,CAAC;YACxD,KAAK,MAAM,sBAAsB,IAAI,aAAa,CAAC,uBAAuB;gBACtE,sBAAsB,CAAC,OAAO,EAAE,CAAC;QACzC,CAAC;QAED,IAAI,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;IACjC,CAAC;IAEM,oBAAoB;QACvB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa;YAClC,KAAK,CAAC,sBAAsB,EAAE,OAAO,EAAE,CAAC;QAE5C,IAAI,CAAC,aAAa,CAAC,KAAK,EAAE,CAAC;IAC/B,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,aAAa,CAAC,IAAI,GAAG,CAAC,CAAC;IACvC,CAAC;IAED;;;;OAIG;IACI,8CAA8C,CAAC,EAClD,IAAI,EAAE,MAAM,EAAE,cAAc,EAM/B;QACG,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,EAAE,CAAC;YACrC,MAAM,4BAA4B,GAAG,IAAI,CAAC,0CAA0C,CAAC,KAAK,CAAC,WAAW,EAAE,IAAI,CAAC,CAAC;YAC9G,MAAM,6BAA6B,GAAG,IAAI,CAAC,0CAA0C,CAAC,KAAK,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;YAEjH,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,4BAA4B,EAAE,6BAA6B,CAAC,CAAC;QACjF,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACjB,MAAM,4BAA4B,GAAG,IAAI,CAAC,MAAM,GAAG,CAAC;gBAChD,CAAC,CAAC,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjG,CAAC,CAAC,IAAI,CAAC;YACX,MAAM,6BAA6B,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC;gBACnD,CAAC,CAAC,IAAI,CAAC,0CAA0C,CAAC,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACrG,CAAC,CAAC,IAAI,CAAC;YAEX,IAAI,4BAA4B,IAAI,IAAI,IAAI,6BAA6B,IAAI,IAAI;gBAC7E,GAAG,IAAI,IAAI,CAAC,GAAG,CAAC,4BAA4B,EAAE,6BAA6B,CAAC,CAAC;iBAC5E,IAAI,4BAA4B,IAAI,IAAI;gBACzC,GAAG,IAAI,4BAA4B,CAAC;iBACnC,IAAI,6BAA6B,IAAI,IAAI;gBAC1C,GAAG,IAAI,6BAA6B,CAAC;QAC7C,CAAC;QAED,OAAO,GAAG,CAAC;IACf,CAAC;IAED,gBAAgB;IACR,aAAa,CACjB,IAAoB,EACpB,mBAAsC,EACtC,sBAA+C;QAE/C,IAAI,CAAC,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC;YAC/B,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,EAAE;gBAC3B,oBAAoB,EAAE,IAAI,GAAG,EAAE;gBAC/B,uBAAuB,EAAE,IAAI,GAAG,EAAE;aACrC,CAAC,CAAC;QAEP,MAAM,aAAa,GAAG,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;QAEtD,IAAI,mBAAmB,IAAI,IAAI;YAC3B,aAAa,CAAC,oBAAoB,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QAEhE,IAAI,sBAAsB,IAAI,IAAI;YAC9B,aAAa,CAAC,uBAAuB,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IAC1E,CAAC;IAED,gBAAgB;IACR,0CAA0C,CAAC,WAAuC,EAAE,KAAuB;QAC/G,IAAI,WAAW,IAAI,IAAI;YACnB,OAAO,CAAC,CAAC;QAEb,IAAI,IAAI,GAA+B,WAAW,CAAC;QACnD,IAAI,GAAG,GAAG,CAAC,CAAC;QAEZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;gBACjB,OAAO,GAAG,GAAG,CAAC,CAAC;YAEnB,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;gBAC1B,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YAED,OAAO,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC;QAChC,CAAC;QAED,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI;YACjC,OAAO,GAAG,GAAG,CAAC,CAAC;QAEnB,OAAO,GAAG,CAAC;IACf,CAAC;IAED,gBAAgB;IACR,iBAAiB,CAAC,KAAkC,EAAE,KAAuB;QACjF,IAAI,KAAK,IAAI,IAAI;YACb,OAAO,KAAK,CAAC;QAEjB,IAAI,IAAI,GAA+B,KAAK,CAAC,WAAW,CAAC;QAEzD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC,EAAE,EAAE,CAAC;YACpD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAEtB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;gBACpB,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;gBACtF,OAAO,IAAI,CAAC;YAChB,CAAC;YAED,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACtB,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBAC3B,SAAS;YACb,CAAC;YAED,OAAO,KAAK,CAAC;QACjB,CAAC;QAED,IAAI,IAAI,IAAI,IAAI;YACZ,OAAO,KAAK,CAAC;QAEjB,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,EAAE,CAAC;YACpB,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,SAAS,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE,CAAC,CAAC;YACjF,OAAO,IAAI,CAAC;QAChB,CAAC;aAAM,CAAC;YACJ,IAAI,CAAC,aAAa,CAAC,GAAG,CAAC;gBACnB,GAAG,KAAK;gBACR,WAAW,EAAE,IAAI;gBACjB,sBAAsB,EAAE,KAAK,CAAC,sBAAsB,EAAE,SAAS,EAAE,EAAE;aACtE,CAAC,CAAC;YACH,OAAO,IAAI,CAAC;QAChB,CAAC;IACL,CAAC;IAEM,MAAM,CAAC,mBAAmB,CAC7B,YAA+E,EAC/E,SAAoB;QAEpB,OAAO,YAAY;aACd,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE;YACjB,IAAI,WAAW,CAAC,WAAW,CAAC;gBACxB,OAAO,sBAAsB,CAAC,uBAAuB,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;iBAC7E,IAAI,OAAO,WAAW,KAAK,QAAQ;gBACpC,OAAO,mBAAmB,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC;;gBAE1C,OAAO,mBAAmB,CAAC,WAAW,CAAC,CAAC;QAChD,CAAC,CAAC;aACD,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAEM,MAAM,CAAC,uBAAuB,CACjC,SAAoB,EACpB,SAAoB;QAEpB,OAAO,mBAAmB,CACtB,SAAS,CAAC,MAAM;aACX,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,KAAK,EAAE,CAAC;aAC7B,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;YACX,IAAI,OAAO,KAAK,KAAK,QAAQ;gBACzB,OAAO,CAAC,KAAK,CAAC,CAAC;iBACd,IAAI,KAAK,YAAY,YAAY;gBAClC,OAAO,KAAK,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;iBAChC,IAAI,KAAK,YAAY,iBAAiB;gBACvC,OAAO,KAAK,CAAC,yBAAyB,CAAC,SAAS,CAAC,CAAC;YAEtD,OAAO,KAAqB,CAAC;QACjC,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,CACf,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,oCAAoC,CAAC,cAA+B;QAI9E,MAAM,CAAC,WAAW,CAAC,GAAG,cAAc;aAC/B,MAAM,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CACrB,WAAW,CAAC,mBAAmB,CAAC,IAAI,CAAC,CAAC,mBAAmB,EAAE,EAAE,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,CAChG,CAAC,CAAC;QAEP,OAAO;YACH,WAAW,EAAE,WAAW,EAAE,WAAW,IAAI,cAAc,EAAE,CAAC,CAAC,CAAC,EAAE,WAAW;YACzE,iCAAiC,EAC7B,WAAW,EAAE,mBAAmB,EAAE,MAAM,CAAC,CAAC,mBAAmB,EAAE,EAAE,CAAC,mBAAmB,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;SAC7G,CAAC;IACN,CAAC;IAEM,MAAM,CAAC,6BAA6B,CACvC,mBAAiD,EACjD,WAA8C,EAC9C,SAAoB,EACpB,gBAAyB,KAAK;QAE9B,IAAI,mBAAmB,IAAI,IAAI,IAAI,mBAAmB,CAAC,MAAM,KAAK,CAAC;YAC/D,OAAO,EAAE,CAAC;QAEd,IAAI,OAAO,mBAAmB,KAAK,QAAQ;YACvC,OAAO,mBAAmB,CAAC;QAE/B,OAAO,SAAS,CAAC,UAAU,CAAC,mBAAmB,EAAE,aAAa,EAAE,mBAAmB,CAAC,WAAW,EAAE,SAAS,EAAE,aAAa,CAAC,CAAC,CAAC;IAChI,CAAC;CACJ;AAED,SAAS,mBAAmB,CAAC,WAA4C;IACrE,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,MAAM,GAAG,GAA0B,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,IAAI,IAAI,IAAI,CAAC;YACb,SAAS;QACb,CAAC;QAED,IAAI,IAAI,KAAK,EAAE,EAAE,CAAC;YACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACf,IAAI,GAAG,EAAE,CAAC;QACd,CAAC;QAED,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnB,CAAC;IAED,IAAI,IAAI,KAAK,EAAE;QACX,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAEnB,OAAO,GAAG,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CACxB,WAA8C,EAC9C,SAAoB,EACpB,gBAAyB,KAAK;IAE9B,IAAI,WAAW,IAAI,IAAI;QACnB,OAAO,EAAE,CAAC;IAEd,MAAM,GAAG,GAAY,EAAE,CAAC;IAExB,KAAK,MAAM,IAAI,IAAI,WAAW,EAAE,CAAC;QAC7B,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC3B,MAAM,MAAM,GAAG,SAAS,CAAC,IAAI,EAAE,aAAa,EAAE,kBAAkB,CAAC,CAAC;YAClE,GAAG,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;QACxB,CAAC;;YACG,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,GAAG,CAAC;AACf,CAAC"}

package/dist/utils/TokenStreamRegulator.d.ts

@@ -10,6 +10,7 @@ export declare class TokenStreamRegulator {
         text: string;
     };
     getAllQueuedChunkTokens(): Token[];
+    getLastQueuedChunkTokens(maxTokens?: number): Token[];
     clearQueue(): void;
 }
 export declare class QueuedTokenRelease {

package/dist/utils/TokenStreamRegulator.js

@@ -1,7 +1,9 @@
 import { DisposedError } from "lifecycle-utils";
+import { maxRecentDetokenizerTokens } from "../consts.js";
 import { pushAll } from "./pushAll.js";
 export class TokenStreamRegulator {
     /** @internal */ _queue = [];
+    /** @internal */ _LastTokens = [];
     addChunk({ tokens, text }) {
         const queuedRelease = QueuedTokenRelease._create(tokens, text);
         this._queue.push(queuedRelease);
@@ -9,8 +11,13 @@ export class TokenStreamRegulator {
     }
     popFreeChunkTokens() {
         const res = [];
-        while (this._queue.length > 0 && this._queue[0].isFree)
-            pushAll(res, this._queue.shift().tokens);
+        while (this._queue.length > 0 && this._queue[0].isFree) {
+            const tokens = this._queue.shift().tokens;
+            pushAll(res, tokens);
+            pushAll(this._LastTokens, tokens);
+        }
+        if (this._LastTokens.length > maxRecentDetokenizerTokens)
+            this._LastTokens.splice(0, this._LastTokens.length - maxRecentDetokenizerTokens);
         return res;
     }
     getPartiallyFreeChunk(tokenizer) {
@@ -25,12 +32,12 @@ export class TokenStreamRegulator {
                 const tokens = queuedRelease.tokens.slice(0, queuedRelease.getFreeTokenIndex());
                 return {
                     tokens,
-                    text: tokenizer.detokenize(tokens)
+                    text: tokenizer.detokenize(tokens, false, this._LastTokens)
                 };
             }
             const freeTokenIndex = queuedRelease.getFreeTokenIndex();
             const tokens = queuedRelease.tokens.slice(0, freeTokenIndex);
-            const tokensText = tokenizer.detokenize(tokens);
+            const tokensText = tokenizer.detokenize(tokens, false, this._LastTokens);
             const freeTextIndex = queuedRelease.getFreeTextIndex();
             const text = queuedRelease.text.slice(0, freeTextIndex);
             if (text.length > tokensText.length) {
@@ -42,8 +49,10 @@ export class TokenStreamRegulator {
             else if (text.length < tokensText.length) {
                 const resTokens = [];
                 let resTokensText = "";
+                const lastTokens = this._LastTokens.slice();
                 for (const token of tokens) {
-                    const tokenText = tokenizer.detokenize([token]);
+                    const tokenText = tokenizer.detokenize([token], false, lastTokens);
+                    lastTokens.push(token);
                     if (resTokensText.length + tokenText.length > text.length) {
                         const remainingText = text.slice(resTokensText.length);
                         const remainingTokens = tokenizer(remainingText, false, "trimLeadingSpace");
@@ -71,6 +80,15 @@ export class TokenStreamRegulator {
     getAllQueuedChunkTokens() {
         return this._queue.flatMap((queuedRelease) => queuedRelease.tokens);
     }
+    getLastQueuedChunkTokens(maxTokens = maxRecentDetokenizerTokens) {
+        const res = [];
+        for (let i = this._queue.length - 1; i >= 0 && res.length < maxTokens; i--) {
+            const tokens = this._queue[i].tokens;
+            for (let j = tokens.length - 1; j >= 0 && res.length < maxTokens; j--)
+                res.unshift(tokens[j]);
+        }
+        return this._queue.flatMap((queuedRelease) => queuedRelease.tokens);
+    }
     clearQueue() {
         this._queue.length = 0;
     }

package/dist/utils/TokenStreamRegulator.js.map

@@ -1 +1 @@
-{"version":3,"file":"TokenStreamRegulator.js","sourceRoot":"","sources":["../../src/utils/TokenStreamRegulator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAErC,MAAM,OAAO,oBAAoB;IAC7B,gBAAgB,CAAkB,MAAM,GAAyB,EAAE,CAAC;IAE7D,QAAQ,CAAC,EAAC,MAAM,EAAE,IAAI,EAAkC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEhC,OAAO,aAAa,CAAC;IACzB,CAAC;IAEM,kBAAkB;QACrB,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM;YAClD,OAAO,CAAC,GAAG,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,CAAC,MAAM,CAAC,CAAC;QAE9C,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,qBAAqB,CAAC,SAAoB;QAC7C,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;YAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAErC,IAAI,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,aAAa;gBAC1D,OAAO;oBACH,MAAM,EAAE,EAAE;oBACV,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,gBAAgB,EAAE,CAAC;iBACtE,CAAC;iBACD,IAAI,aAAa,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC;gBAClE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBAChF,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC;iBACrC,CAAC;YACN,CAAC;YAED,MAAM,cAAc,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YAC7D,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;YAEhD,MAAM,aAAa,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;YAExD,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAClC,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,UAAU;iBACnB,CAAC;YACN,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAY,EAAE,CAAC;gBAC9B,IAAI,aAAa,GAAG,EAAE,CAAC;gBAEvB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;oBAEhD,IAAI,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;wBACxD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBACvD,MAAM,eAAe,GAAG,SAAS,CAAC,aAAa,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;wBAC5E,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;wBACpC,MAAM;oBACV,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtB,aAAa,IAAI,SAAS,CAAC;gBAC/B,CAAC;gBAED,OAAO;oBACH,MAAM,EAAE,SAAS;oBACjB,IAAI;iBACP,CAAC;YACN,CAAC;YAED,OAAO;gBACH,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;gBACrD,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;aACnD,CAAC;QACN,CAAC;QAED,OAAO;YACH,MAAM,EAAE,EAAoB;YAC5B,IAAI,EAAE,EAAE;SACX,CAAC;IACN,CAAC;IAEM,uBAAuB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAEM,UAAU;QACb,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC;CACJ;AAED,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,CAAkB,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjF,gBAAgB,CAAkB,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClF,gBAAgB,CAAS,OAAO,CAAmB;IACnD,gBAAgB,CAAS,KAAK,CAAS;IAEvC,YAAoB,MAAwB,EAAE,IAAY;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,aAAa;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,IAAW,eAAe;QACtB,IAAI,IAAI,CAAC,MAAM;YACX,OAAO,IAAI,CAAC;QAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAChD,OAAO,aAAa,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC;IACnD,CAAC;IAEM,gBAAgB;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAE5B,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;IAEM,iBAAiB;QACpB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAE9B,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;aACvB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IAEM,mBAAmB,CAAC,UAAkB;QACzC,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzE,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,oBAAoB,CAAC,UAAkB;QAC1C,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE1E,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,mBAAmB,CAAC,MAAwB,EAAE,IAAY;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,IAAY;QAC/C,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAkB,MAAM,CAAC;IACzC,gBAAgB,CAAkB,MAAM,CAA8B;IAEtE,YAAoB,KAAa,EAAE,KAAkC;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,SAAS;QACZ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEtB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAc,EAAE,KAAkC;QACpE,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;CACJ"}
+{"version":3,"file":"TokenStreamRegulator.js","sourceRoot":"","sources":["../../src/utils/TokenStreamRegulator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AACxD,OAAO,EAAC,OAAO,EAAC,MAAM,cAAc,CAAC;AAErC,MAAM,OAAO,oBAAoB;IAC7B,gBAAgB,CAAkB,MAAM,GAAyB,EAAE,CAAC;IACpE,gBAAgB,CAAkB,WAAW,GAAY,EAAE,CAAC;IAErD,QAAQ,CAAC,EAAC,MAAM,EAAE,IAAI,EAAkC;QAC3D,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QAE/D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAEhC,OAAO,aAAa,CAAC;IACzB,CAAC;IAEM,kBAAkB;QACrB,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;YACrD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,EAAG,CAAC,MAAM,CAAC;YAC3C,OAAO,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACrB,OAAO,CAAC,IAAI,CAAC,WAAW,EAAE,MAAM,CAAC,CAAC;QACtC,CAAC;QAED,IAAI,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,0BAA0B;YACpD,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,MAAM,GAAG,0BAA0B,CAAC,CAAC;QAErF,OAAO,GAAG,CAAC;IACf,CAAC;IAEM,qBAAqB,CAAC,SAAoB;QAC7C,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC;YAC3D,MAAM,aAAa,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;YAErC,IAAI,aAAa,CAAC,YAAY,IAAI,CAAC,aAAa,CAAC,aAAa;gBAC1D,OAAO;oBACH,MAAM,EAAE,EAAE;oBACV,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,gBAAgB,EAAE,CAAC;iBACtE,CAAC;iBACD,IAAI,aAAa,CAAC,aAAa,IAAI,CAAC,aAAa,CAAC,YAAY,EAAE,CAAC;gBAClE,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,iBAAiB,EAAE,CAAC,CAAC;gBAChF,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC;iBAC9D,CAAC;YACN,CAAC;YAED,MAAM,cAAc,GAAG,aAAa,CAAC,iBAAiB,EAAE,CAAC;YACzD,MAAM,MAAM,GAAG,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;YAC7D,MAAM,UAAU,GAAG,SAAS,CAAC,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;YAEzE,MAAM,aAAa,GAAG,aAAa,CAAC,gBAAgB,EAAE,CAAC;YACvD,MAAM,IAAI,GAAG,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;YAExD,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBAClC,OAAO;oBACH,MAAM;oBACN,IAAI,EAAE,UAAU;iBACnB,CAAC;YACN,CAAC;iBAAM,IAAI,IAAI,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC;gBACzC,MAAM,SAAS,GAAY,EAAE,CAAC;gBAC9B,IAAI,aAAa,GAAG,EAAE,CAAC;gBAEvB,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,KAAK,EAAE,CAAC;gBAC5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;oBACzB,MAAM,SAAS,GAAG,SAAS,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,EAAE,KAAK,EAAE,UAAU,CAAC,CAAC;oBACnE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBAEvB,IAAI,aAAa,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;wBACxD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;wBACvD,MAAM,eAAe,GAAG,SAAS,CAAC,aAAa,EAAE,KAAK,EAAE,kBAAkB,CAAC,CAAC;wBAC5E,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAC;wBACpC,MAAM;oBACV,CAAC;oBAED,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;oBACtB,aAAa,IAAI,SAAS,CAAC;gBAC/B,CAAC;gBAED,OAAO;oBACH,MAAM,EAAE,SAAS;oBACjB,IAAI;iBACP,CAAC;YACN,CAAC;YAED,OAAO;gBACH,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC;gBACrD,IAAI,EAAE,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;aACnD,CAAC;QACN,CAAC;QAED,OAAO;YACH,MAAM,EAAE,EAAoB;YAC5B,IAAI,EAAE,EAAE;SACX,CAAC;IACN,CAAC;IAEM,uBAAuB;QAC1B,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAEM,wBAAwB,CAAC,YAAoB,0BAA0B;QAC1E,MAAM,GAAG,GAAY,EAAE,CAAC;QAExB,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YACzE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;YACrC,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE;gBACjE,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;QAED,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,aAAa,EAAE,EAAE,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC;IACxE,CAAC;IAEM,UAAU;QACb,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC;IAC3B,CAAC;CACJ;AAED,MAAM,OAAO,kBAAkB;IAC3B,gBAAgB,CAAkB,UAAU,GAAG,IAAI,GAAG,EAA0B,CAAC;IACjF,gBAAgB,CAAkB,WAAW,GAAG,IAAI,GAAG,EAA0B,CAAC;IAClF,gBAAgB,CAAS,OAAO,CAAmB;IACnD,gBAAgB,CAAS,KAAK,CAAS;IAEvC,YAAoB,MAAwB,EAAE,IAAY;QACtD,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,OAAO,CAAC;IACxB,CAAC;IAED,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,KAAK,CAAC;IACtB,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC,CAAC;IACrE,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,UAAU,CAAC,IAAI,GAAG,CAAC,CAAC;IACpC,CAAC;IAED,IAAW,aAAa;QACpB,OAAO,IAAI,CAAC,WAAW,CAAC,IAAI,GAAG,CAAC,CAAC;IACrC,CAAC;IAED,IAAW,eAAe;QACtB,IAAI,IAAI,CAAC,MAAM;YACX,OAAO,IAAI,CAAC;QAEhB,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,EAAE,CAAC;QAC9C,MAAM,cAAc,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAChD,OAAO,aAAa,GAAG,CAAC,IAAI,cAAc,GAAG,CAAC,CAAC;IACnD,CAAC;IAEM,gBAAgB;QACnB,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC;QAE5B,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC;aACtB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAC5E,CAAC;IAEM,iBAAiB;QACpB,IAAI,IAAI,CAAC,WAAW,CAAC,IAAI,KAAK,CAAC;YAC3B,OAAO,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC;QAE9B,OAAO,CAAC,GAAG,IAAI,CAAC,WAAW,CAAC;aACvB,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9E,CAAC;IAEM,mBAAmB,CAAC,UAAkB;QACzC,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEzE,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM;YAChD,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE9B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,oBAAoB,CAAC,UAAkB;QAC1C,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,UAAU,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;QAE1E,IAAI,UAAU,IAAI,CAAC,IAAI,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,MAAM;YAClD,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAE/B,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,mBAAmB,CAAC,MAAwB,EAAE,IAAY;QAC7D,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC;IACtB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAe,EAAE,IAAY;QAC/C,OAAO,IAAI,kBAAkB,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;CACJ;AAED,MAAM,OAAO,sBAAsB;IAC/B,gBAAgB,CAAkB,MAAM,CAAC;IACzC,gBAAgB,CAAkB,MAAM,CAA8B;IAEtE,YAAoB,KAAa,EAAE,KAAkC;QACjE,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;QACpB,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC;IACxB,CAAC;IAED,IAAW,KAAK;QACZ,OAAO,IAAI,CAAC,MAAM,CAAC;IACvB,CAAC;IAEM,SAAS;QACZ,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC;YACtB,MAAM,IAAI,aAAa,EAAE,CAAC;QAE9B,MAAM,IAAI,GAAG,sBAAsB,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QAEtE,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAEtB,OAAO,IAAI,CAAC;IAChB,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAEM,CAAC,MAAM,CAAC,OAAO,CAAC;QACnB,IAAI,CAAC,OAAO,EAAE,CAAC;IACnB,CAAC;IAED,gBAAgB;IACT,MAAM,CAAC,OAAO,CAAC,MAAc,EAAE,KAAkC;QACpE,OAAO,IAAI,sBAAsB,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;IACrD,CAAC;CACJ"}

package/dist/utils/resolveLastTokens.d.ts

@@ -0,0 +1,2 @@
+import { Token } from "../types.js";
+export declare function resolveLastTokens(tokenArrays: Token[][], maxTokens?: number): Token[];

package/dist/utils/resolveLastTokens.js

@@ -0,0 +1,12 @@
+import { maxRecentDetokenizerTokens } from "../consts.js";
+export function resolveLastTokens(tokenArrays, maxTokens = maxRecentDetokenizerTokens) {
+    const lastTokens = [];
+    for (let i = tokenArrays.length - 1; i >= 0 && lastTokens.length < maxTokens; i--) {
+        const tokens = tokenArrays[i];
+        for (let j = tokens.length - 1; j >= 0 && lastTokens.length < maxTokens; j--) {
+            lastTokens.unshift(tokens[j]);
+        }
+    }
+    return lastTokens;
+}
+//# sourceMappingURL=resolveLastTokens.js.map

package/dist/utils/resolveLastTokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"resolveLastTokens.js","sourceRoot":"","sources":["../../src/utils/resolveLastTokens.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,0BAA0B,EAAC,MAAM,cAAc,CAAC;AAExD,MAAM,UAAU,iBAAiB,CAAC,WAAsB,EAAE,YAAoB,0BAA0B;IACpG,MAAM,UAAU,GAAY,EAAE,CAAC;IAC/B,KAAK,IAAI,CAAC,GAAG,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;QAChF,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3E,UAAU,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;QAClC,CAAC;IACL,CAAC;IAED,OAAO,UAAU,CAAC;AACtB,CAAC"}

package/llama/CMakeLists.txt

@@ -129,7 +129,7 @@ if (GGML_METAL)
         )
 endif()
 
-file(GLOB SOURCE_FILES "addon.cpp" ${GPU_INFO_SOURCES})
+file(GLOB SOURCE_FILES "addon/*.cpp" "addon/**/*.cpp" ${GPU_INFO_SOURCES})
 
 if(APPLE)
     set(CMAKE_SKIP_BUILD_RPATH  FALSE)