crawlforge-mcp-server 4.2.12 → 4.6.0

package/src/tools/tracking/trackChanges/differ.js

@@ -3,6 +3,38 @@
  * URL content fetching and history/stat helper functions.
  */
 
+/**
+ * Default Jaccard similarity threshold below which a change is considered
+ * meaningful (i.e. worth flagging). 0.85 means content must be at least 85 %
+ * similar to be treated as "no significant change".
+ */
+export const DEFAULT_CHANGE_THRESHOLD = 0.85;
+
+/**
+ * Compute token-based Jaccard similarity between two text strings.
+ * Tokenises on whitespace; returns a value in [0, 1] where 1 is identical.
+ *
+ * @param {string} text1
+ * @param {string} text2
+ * @returns {number}
+ */
+export function calculateSimilarity(text1, text2) {
+  if (!text1 && !text2) return 1;
+  if (!text1 || !text2) return 0;
+
+  const tokenise = (str) => new Set(str.toLowerCase().split(/\s+/).filter(Boolean));
+  const setA = tokenise(text1);
+  const setB = tokenise(text2);
+
+  let intersection = 0;
+  for (const token of setA) {
+    if (setB.has(token)) intersection++;
+  }
+
+  const union = setA.size + setB.size - intersection;
+  return union === 0 ? 1 : intersection / union;
+}
+
 /**
  * Fetch the HTML/text content of a URL with change-tracking headers.
  * @param {string} url
@@ -18,7 +50,7 @@ export async function fetchContent(url) {
         'Accept-Encoding': 'gzip, deflate',
         'Cache-Control': 'no-cache'
       },
-      timeout: 30000
+      signal: AbortSignal.timeout(30000)
     });
 
     if (!response.ok) {

package/src/utils/htmlToMarkdown.js

@@ -11,10 +11,11 @@
  *     headingStyle: 'atx'       -> # H1 / ## H2 instead of underline style
  *     codeBlockStyle: 'fenced'  -> triple-backtick fences
  *     bulletListMarker: '-'
- * - Tables fall back to prose (no GFM plugin loaded by default).
+ * - GFM plugin enabled for table support (turndown-plugin-gfm).
  */
 
 import TurndownService from 'turndown';
+import { gfm } from 'turndown-plugin-gfm';
 
 let _td = null;
 
@@ -30,6 +31,9 @@ function getTurndown() {
       linkStyle: 'inlined'
     });
 
+    // Enable GFM extensions (tables, strikethrough, task lists)
+    _td.use(gfm);
+
     // Remove boilerplate elements before converting
     _td.remove(['script', 'style', 'nav', 'footer', 'aside', 'noscript']);
   }