@flutchai/flutch-sdk 0.6.6 → 0.6.8

package/dist/index.cjs

@@ -5113,6 +5113,49 @@ function _ts_decorate21(decorators, target, key, desc) {
   return c > 3 && r && Object.defineProperty(target, key, r), r;
 }
 __name(_ts_decorate21, "_ts_decorate");
+function joinTextSteps(parts) {
+  const kept = parts.filter(Boolean);
+  for (let i = 1; i < kept.length; i++) {
+    for (const paragraph of paragraphsOf(kept[i])) {
+      for (let j = 0; j < i; j++) {
+        kept[j] = removeNearDuplicate(kept[j], paragraph);
+      }
+    }
+  }
+  let out = "";
+  for (const part of kept) {
+    if (!part.trim()) continue;
+    if (!out || /\s$/.test(out) || /^\s/.test(part)) out += part;
+    else out += "\n\n" + part;
+  }
+  return out.replace(/\n{3,}/g, "\n\n");
+}
+__name(joinTextSteps, "joinTextSteps");
+var MIN_DUPLICATE_WORDS = 8;
+var DUPLICATE_SIMILARITY = 0.75;
+function paragraphsOf(text) {
+  return text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+}
+__name(paragraphsOf, "paragraphsOf");
+function wordsOf(text) {
+  return new Set(text.toLowerCase().match(/[\p{L}\p{N}$]+/gu) ?? []);
+}
+__name(wordsOf, "wordsOf");
+function similarity(a, b) {
+  let common = 0;
+  for (const w of a) if (b.has(w)) common++;
+  return common / (a.size + b.size - common);
+}
+__name(similarity, "similarity");
+function removeNearDuplicate(text, paragraph) {
+  const target = wordsOf(paragraph);
+  if (target.size < MIN_DUPLICATE_WORDS) return text;
+  return paragraphsOf(text).filter((p) => {
+    const words = wordsOf(p);
+    return words.size < MIN_DUPLICATE_WORDS || similarity(words, target) < DUPLICATE_SIMILARITY;
+  }).join("\n\n");
+}
+__name(removeNearDuplicate, "removeNearDuplicate");
 exports.EventProcessor = class _EventProcessor {
   static {
     __name(this, "EventProcessor");
@@ -5548,7 +5591,7 @@ exports.EventProcessor = class _EventProcessor {
       totalEvents: acc.traceEvents.length
     } : null;
     const textChain = allChains.find((c) => c.channel === "text");
-    const text = textChain ? textChain.steps.filter((step) => step.type === "text").map((step) => step.text || "").join("") : "";
+    const text = textChain ? joinTextSteps(textChain.steps.filter((step) => step.type === "text").map((step) => step.text || "")) : "";
     this.logger.log("\u{1F4CA} [EventProcessor] Final result assembled", {
       totalChains: allChains.length,
       textChains: allChains.filter((c) => c.channel === "text").length,
@@ -8683,6 +8726,7 @@ exports.isInternalMode = isInternalMode;
 exports.isReasoningBlock = isReasoningBlock;
 exports.isReasoningModel = isReasoningModel;
 exports.isSendableReasoningBlock = isSendableReasoningBlock;
+exports.joinTextSteps = joinTextSteps;
 exports.loadOAuthProviders = loadOAuthProviders;
 exports.modelAcceptsSamplingParams = modelAcceptsSamplingParams;
 exports.modelSupportsPromptCache = modelSupportsPromptCache;