llmz 0.0.8 → 0.0.10

package/dist/chunk-SNDVQU5A.js

@@ -1,152 +0,0 @@
-import {
-  getTokenizer
-} from "./chunk-4MNIJGK6.js";
-
-// src/truncator.ts
-var DEFAULT_REMOVE_CHUNK = 250;
-var WRAP_OPEN_TAG_1 = "\u3010TRUNCATE";
-var WRAP_OPEN_TAG_2 = "\u3011";
-var WRAP_CLOSE_TAG = "\u3010/TRUNCATE\u3011";
-var getRegex = () => new RegExp(`(${WRAP_OPEN_TAG_1}(?:\\s+[\\w:]+)*\\s*${WRAP_OPEN_TAG_2})([\\s\\S]*?)(${WRAP_CLOSE_TAG})`, "g");
-var DEFAULT_TRUNCATE_OPTIONS = {
-  preserve: "top",
-  flex: 1,
-  minTokens: 0
-};
-function wrapContent(content, options) {
-  const preserve = (options == null ? void 0 : options.preserve) ?? DEFAULT_TRUNCATE_OPTIONS.preserve;
-  const flex = (options == null ? void 0 : options.flex) ?? DEFAULT_TRUNCATE_OPTIONS.flex;
-  const minTokens = (options == null ? void 0 : options.minTokens) ?? DEFAULT_TRUNCATE_OPTIONS.minTokens;
-  return `${WRAP_OPEN_TAG_1} preserve:${preserve} flex:${flex} min:${minTokens} ${WRAP_OPEN_TAG_2}${content}${WRAP_CLOSE_TAG}`;
-}
-function truncateWrappedContent({
-  messages,
-  tokenLimit,
-  throwOnFailure = true
-}) {
-  var _a, _b, _c;
-  const tokenizer = getTokenizer();
-  const parts = [];
-  for (const msg of messages) {
-    const current = [];
-    const content = typeof msg.content === "string" ? msg.content : "";
-    let match;
-    const regex = getRegex();
-    let lastIndex = 0;
-    while ((match = regex.exec(content)) !== null) {
-      const attributes = match[1].split(/\s+/).slice(1).filter((x) => x !== WRAP_OPEN_TAG_2).map((x) => x.split(":")).reduce((acc, [key, value]) => ({ ...acc, [key]: value }), {});
-      if (match.index > lastIndex) {
-        const nonTruncatableContent = content.slice(lastIndex, match.index);
-        current.push({
-          content: nonTruncatableContent,
-          tokens: tokenizer.count(nonTruncatableContent),
-          truncatable: false
-        });
-      }
-      const wrappedContent = match[2];
-      current.push({
-        content: wrappedContent,
-        tokens: tokenizer.count(wrappedContent),
-        truncatable: true,
-        attributes: {
-          preserve: attributes.preserve,
-          flex: Number(attributes.flex) || DEFAULT_TRUNCATE_OPTIONS.flex,
-          minTokens: Number(attributes.min) || DEFAULT_TRUNCATE_OPTIONS.minTokens
-        }
-      });
-      lastIndex = regex.lastIndex;
-    }
-    if (lastIndex < content.length) {
-      const remainingContent = content.slice(lastIndex);
-      current.push({
-        content: remainingContent,
-        tokens: tokenizer.count(remainingContent),
-        truncatable: false
-      });
-    }
-    parts.push(current);
-  }
-  const getCount = () => parts.reduce((acc, x) => acc + x.reduce((acc2, y) => acc2 + y.tokens, 0), 0);
-  const getTwoBiggestTruncatables = () => {
-    var _a2, _b2;
-    let biggest = null;
-    let secondBiggest = null;
-    for (const part of parts.flat()) {
-      if (part.truncatable) {
-        const flex = ((_a2 = part.attributes) == null ? void 0 : _a2.flex) ?? DEFAULT_TRUNCATE_OPTIONS.flex;
-        const tokens = part.tokens * flex;
-        if (part.tokens <= (((_b2 = part.attributes) == null ? void 0 : _b2.minTokens) ?? 0)) {
-          continue;
-        }
-        if (!biggest || tokens > biggest.tokens) {
-          secondBiggest = biggest;
-          biggest = part;
-        } else if (!secondBiggest || tokens > secondBiggest.tokens) {
-          secondBiggest = part;
-        }
-      }
-    }
-    return { biggest, secondBiggest };
-  };
-  let currentCount = getCount();
-  while (currentCount > tokenLimit) {
-    const { biggest, secondBiggest } = getTwoBiggestTruncatables();
-    if (!biggest || !biggest.truncatable || biggest.tokens <= 0) {
-      if (throwOnFailure) {
-        throw new Error(`Cannot truncate further, current count: ${getCount()}`);
-      } else {
-        break;
-      }
-    }
-    const delta = Math.max(biggest.tokens - ((secondBiggest == null ? void 0 : secondBiggest.tokens) ?? 0), DEFAULT_REMOVE_CHUNK);
-    const room = Math.min(delta, biggest.tokens);
-    let toRemove = Math.min(room, currentCount - tokenLimit);
-    if (toRemove <= 0) {
-      if (throwOnFailure) {
-        throw new Error(`Cannot truncate further, current count: ${getCount()}`);
-      } else {
-        break;
-      }
-    }
-    if (biggest.tokens - toRemove < (((_a = biggest.attributes) == null ? void 0 : _a.minTokens) ?? 0)) {
-      toRemove = biggest.tokens - (((_b = biggest.attributes) == null ? void 0 : _b.minTokens) ?? 0);
-    }
-    const preserve = ((_c = biggest.attributes) == null ? void 0 : _c.preserve) ?? DEFAULT_TRUNCATE_OPTIONS.preserve;
-    const split = tokenizer.split(biggest.content);
-    if (preserve === "bottom") {
-      biggest.content = split.slice(toRemove).join("");
-    } else if (preserve === "top") {
-      biggest.content = split.slice(0, -toRemove).join("");
-    } else {
-      const anchor = Math.ceil(split.length / 2);
-      const radius = Math.ceil(toRemove / 2);
-      const left = anchor - radius;
-      const right = anchor + radius;
-      biggest.content = split.slice(0, left).join("") + split.slice(right).join("");
-    }
-    biggest.tokens -= toRemove;
-    currentCount -= toRemove;
-  }
-  const removeRedundantWrappers = (content) => {
-    return content.replace(getRegex(), "$2");
-  };
-  return messages.map((msg, i) => {
-    const p = parts[i];
-    return {
-      ...msg,
-      content: typeof msg.content === "string" ? removeRedundantWrappers(
-        p.map((part) => {
-          if (part.truncatable) {
-            return part.content;
-          }
-          return part.content;
-        }).join("")
-      ) : msg.content
-    };
-  });
-}
-
-export {
-  wrapContent,
-  truncateWrappedContent
-};