llmz 0.6.0 → 0.6.2

package/dist/chunk-HEAQAEFZ.cjs

@@ -1,173 +0,0 @@
-"use strict";Object.defineProperty(exports, "__esModule", {value: true}); function _nullishCoalesce(lhs, rhsFn) { if (lhs != null) { return lhs; } else { return rhsFn(); } } var _class;
-
-var _chunkGOWPACSKcjs = require('./chunk-GOWPACSK.cjs');
-
-// src/truncator.ts
-var DEFAULT_REMOVE_CHUNK = 250;
-var WRAP_OPEN_TAG_1 = "\u3010TRUNCATE";
-var WRAP_OPEN_TAG_2 = "\u3011";
-var WRAP_CLOSE_TAG = "\u3010/TRUNCATE\u3011";
-var REGEXP = `(${WRAP_OPEN_TAG_1}(?:\\s+[\\w:]+)*\\s*${WRAP_OPEN_TAG_2})([\\s\\S]*?)(${WRAP_CLOSE_TAG})`;
-var DEFAULT_TRUNCATE_OPTIONS = {
-  preserve: "top",
-  flex: 1,
-  minTokens: 0
-};
-function wrapContent(content, options) {
-  const preserve = _nullishCoalesce((options == null ? void 0 : options.preserve), () => ( DEFAULT_TRUNCATE_OPTIONS.preserve));
-  const flex = _nullishCoalesce((options == null ? void 0 : options.flex), () => ( DEFAULT_TRUNCATE_OPTIONS.flex));
-  const minTokens = _nullishCoalesce((options == null ? void 0 : options.minTokens), () => ( DEFAULT_TRUNCATE_OPTIONS.minTokens));
-  return `${WRAP_OPEN_TAG_1} preserve:${preserve} flex:${flex} min:${minTokens} ${WRAP_OPEN_TAG_2}${content}${WRAP_CLOSE_TAG}`;
-}
-function truncateWrappedContent({
-  messages,
-  tokenLimit,
-  throwOnFailure = true
-}) {
-  var _a, _b, _c;
-  const tokenizer = _chunkGOWPACSKcjs.getTokenizer.call(void 0, );
-  const parts = [];
-  for (const msg of messages) {
-    const current = [];
-    const content = typeof msg.content === "string" ? msg.content : "";
-    let match;
-    const parser = new _MessageContentParser();
-    while ((match = parser.parse(content)) !== null) {
-      const { attributes, nonTruncatableContent, wrappedContent } = match;
-      if (nonTruncatableContent) {
-        current.push({
-          content: nonTruncatableContent,
-          tokens: tokenizer.count(nonTruncatableContent),
-          truncatable: false
-        });
-      }
-      current.push({
-        content: wrappedContent,
-        tokens: tokenizer.count(wrappedContent),
-        truncatable: true,
-        attributes: {
-          preserve: attributes.preserve,
-          flex: Number(attributes.flex) || DEFAULT_TRUNCATE_OPTIONS.flex,
-          minTokens: Number(attributes.min) || DEFAULT_TRUNCATE_OPTIONS.minTokens
-        }
-      });
-    }
-    const remainingContent = parser.getRemainingContent(content);
-    if (remainingContent) {
-      current.push({
-        content: remainingContent,
-        tokens: tokenizer.count(remainingContent),
-        truncatable: false
-      });
-    }
-    parts.push(current);
-  }
-  let currentCount = _countTotalTokens(parts);
-  while (currentCount > tokenLimit) {
-    const { biggest, secondBiggest } = _getTwoBiggestTruncables(parts);
-    if (!biggest || !biggest.truncatable || biggest.tokens <= 0) {
-      if (throwOnFailure) {
-        throw new Error(`Cannot truncate further, current count: ${currentCount}`);
-      } else {
-        break;
-      }
-    }
-    const delta = Math.max(biggest.tokens - (_nullishCoalesce((secondBiggest == null ? void 0 : secondBiggest.tokens), () => ( 0))), DEFAULT_REMOVE_CHUNK);
-    const room = Math.min(delta, biggest.tokens);
-    let toRemove = Math.min(room, currentCount - tokenLimit);
-    if (toRemove <= 0) {
-      if (throwOnFailure) {
-        throw new Error(`Cannot truncate further, current count: ${currentCount}`);
-      } else {
-        break;
-      }
-    }
-    if (biggest.tokens - toRemove < (_nullishCoalesce(((_a = biggest.attributes) == null ? void 0 : _a.minTokens), () => ( 0)))) {
-      toRemove = biggest.tokens - (_nullishCoalesce(((_b = biggest.attributes) == null ? void 0 : _b.minTokens), () => ( 0)));
-    }
-    const preserve = _nullishCoalesce(((_c = biggest.attributes) == null ? void 0 : _c.preserve), () => ( DEFAULT_TRUNCATE_OPTIONS.preserve));
-    const split = tokenizer.splitAndSlice(biggest.content);
-    if (preserve === "bottom") {
-      biggest.content = split.slice(toRemove).join("");
-    } else if (preserve === "top") {
-      biggest.content = split.slice(0, -toRemove).join("");
-    } else {
-      const anchor = Math.ceil(split.length / 2);
-      const radius = Math.ceil(toRemove / 2);
-      const left = anchor - radius;
-      const right = anchor + radius;
-      biggest.content = split.slice(0, left).join("") + split.slice(right).join("");
-    }
-    biggest.tokens -= toRemove;
-    currentCount -= toRemove;
-  }
-  return messages.map((msg, i) => {
-    const p = parts[i];
-    return {
-      ...msg,
-      content: typeof msg.content === "string" ? _renderRemainingWrappers(p.map((part) => part.content).join("")) : msg.content
-    };
-  });
-}
-var _MessageContentParser = (_class = class {
-  
-  __init() {this._lastIndex = 0}
-  constructor() {;_class.prototype.__init.call(this);
-    this._regex = _createRegex();
-  }
-  parse(content) {
-    const match = this._regex.exec(content);
-    if (!match) {
-      return null;
-    }
-    const attributes = match[1].split(/\s+/).slice(1).filter((x) => x !== WRAP_OPEN_TAG_2).map((x) => x.split(":")).reduce(
-      (acc, [key, value]) => ({ ...acc, [key]: value }),
-      {}
-    );
-    let nonTruncatableContent = void 0;
-    if (match.index > this._lastIndex) {
-      nonTruncatableContent = content.slice(this._lastIndex, match.index);
-    }
-    const wrappedContent = match[2];
-    this._lastIndex = this._regex.lastIndex;
-    return { attributes, nonTruncatableContent, wrappedContent };
-  }
-  getRemainingContent(content) {
-    if (this._lastIndex < content.length) {
-      const remainingContent = content.slice(this._lastIndex);
-      return remainingContent;
-    }
-    return null;
-  }
-}, _class);
-var _createRegex = () => new RegExp(REGEXP, "g");
-var _renderRemainingWrappers = (content) => content.replace(_createRegex(), "$2");
-var stripTruncationTags = (content) => _renderRemainingWrappers(content);
-var _countTotalTokens = (parts) => parts.reduce((acc, x) => acc + x.reduce((acc2, y) => acc2 + y.tokens, 0), 0);
-var _getTwoBiggestTruncables = (parts) => {
-  var _a, _b;
-  let biggest = null;
-  let secondBiggest = null;
-  for (const part of parts.flat()) {
-    if (part.truncatable) {
-      if (part.tokens <= (_nullishCoalesce(((_a = part.attributes) == null ? void 0 : _a.minTokens), () => ( 0)))) {
-        continue;
-      }
-      const flex = _nullishCoalesce(((_b = part.attributes) == null ? void 0 : _b.flex), () => ( DEFAULT_TRUNCATE_OPTIONS.flex));
-      const tokens = part.tokens * flex;
-      if (!biggest || tokens > biggest.tokens) {
-        secondBiggest = biggest;
-        biggest = part;
-      } else if (!secondBiggest || tokens > secondBiggest.tokens) {
-        secondBiggest = part;
-      }
-    }
-  }
-  return { biggest, secondBiggest };
-};
-
-
-
-
-
-exports.wrapContent = wrapContent; exports.truncateWrappedContent = truncateWrappedContent; exports.stripTruncationTags = stripTruncationTags;