@jjlmoya/utils-developer 1.26.0 → 1.27.0

package/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@jjlmoya/utils-developer",
-    "version": "1.26.0",
+    "version": "1.27.0",
     "type": "module",
     "main": "./src/index.ts",
     "types": "./src/index.ts",

package/src/tests/spanish_leakage.test.ts

@@ -28,6 +28,8 @@ const TRANSLATABLE_KEYS = [
   'schemas',
 ] as const;
 
+const COPY_THRESHOLD = 0.9;
+
 const SPANISH_MARKERS = [
   ['sangre', /\bsangre\b/gi],
   ['molino', /\bmolino\b/gi],
@@ -111,10 +113,26 @@ function findSpanishMarkers(text: string[]): string[] {
   );
 }
 
+function similarity(left: string, right: string): number {
+  const leftTokens = left.split(/\s+/);
+  const rightCounts = new Map<string, number>();
+  right.split(/\s+/).forEach((token) => rightCounts.set(token, (rightCounts.get(token) ?? 0) + 1));
+  const matches = leftTokens.reduce((total, token) => {
+    const count = rightCounts.get(token) ?? 0;
+    if (count > 0) rightCounts.set(token, count - 1);
+    return total + (count > 0 ? 1 : 0);
+  }, 0);
+  return (2 * matches) / (leftTokens.length + right.split(/\s+/).length);
+}
+
 function findCopiedFragments(spanish: string[], translated: string[]): string[] {
-  const corpus = translated.join(' ');
   return spanish
-    .filter((fragment) => fragment.length >= 80 && corpus.includes(fragment))
+    .filter((fragment) => fragment.length >= 80)
+    .filter((fragment) => translated.some((candidate) =>
+      candidate.length >= 80 &&
+      Math.min(fragment.length, candidate.length) / Math.max(fragment.length, candidate.length) >= COPY_THRESHOLD &&
+      similarity(fragment, candidate) >= COPY_THRESHOLD,
+    ))
     .sort((a, b) => b.length - a.length)
     .slice(0, 3);
 }