@flutchai/flutch-sdk 0.6.7 → 0.6.8

package/dist/index.js

@@ -5084,15 +5084,48 @@ function _ts_decorate21(decorators, target, key, desc) {
 }
 __name(_ts_decorate21, "_ts_decorate");
 function joinTextSteps(parts) {
+  const kept = parts.filter(Boolean);
+  for (let i = 1; i < kept.length; i++) {
+    for (const paragraph of paragraphsOf(kept[i])) {
+      for (let j = 0; j < i; j++) {
+        kept[j] = removeNearDuplicate(kept[j], paragraph);
+      }
+    }
+  }
   let out = "";
-  for (const part of parts) {
-    if (!part) continue;
+  for (const part of kept) {
+    if (!part.trim()) continue;
     if (!out || /\s$/.test(out) || /^\s/.test(part)) out += part;
     else out += "\n\n" + part;
   }
-  return out;
+  return out.replace(/\n{3,}/g, "\n\n");
 }
 __name(joinTextSteps, "joinTextSteps");
+var MIN_DUPLICATE_WORDS = 8;
+var DUPLICATE_SIMILARITY = 0.75;
+function paragraphsOf(text) {
+  return text.split(/\n\s*\n/).map((p) => p.trim()).filter(Boolean);
+}
+__name(paragraphsOf, "paragraphsOf");
+function wordsOf(text) {
+  return new Set(text.toLowerCase().match(/[\p{L}\p{N}$]+/gu) ?? []);
+}
+__name(wordsOf, "wordsOf");
+function similarity(a, b) {
+  let common = 0;
+  for (const w of a) if (b.has(w)) common++;
+  return common / (a.size + b.size - common);
+}
+__name(similarity, "similarity");
+function removeNearDuplicate(text, paragraph) {
+  const target = wordsOf(paragraph);
+  if (target.size < MIN_DUPLICATE_WORDS) return text;
+  return paragraphsOf(text).filter((p) => {
+    const words = wordsOf(p);
+    return words.size < MIN_DUPLICATE_WORDS || similarity(words, target) < DUPLICATE_SIMILARITY;
+  }).join("\n\n");
+}
+__name(removeNearDuplicate, "removeNearDuplicate");
 var EventProcessor = class _EventProcessor {
   static {
     __name(this, "EventProcessor");