@jambudipa/spider 0.3.0 → 0.3.1

package/dist/index.js

@@ -1,4 +1,4 @@
-import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, Ref, HashMap, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct } from "effect";
+import { Effect, Layer, Option, Chunk, MutableHashSet, Schema, Data, pipe, Context, DateTime, Console, Duration, MutableHashMap, Queue, HashMap, PubSub, MutableRef, Schedule, Stream, Fiber, Random, Struct, Ref } from "effect";
 import * as cheerio from "cheerio";
 import * as fs from "fs";
 import * as path from "path";
@@ -1699,71 +1699,50 @@ const normalizeUrl = (url, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.
   };
 });
 const deduplicateUrls = (urls, strategy = DEFAULT_DEDUPLICATION_STRATEGY) => Effect.gen(function* () {
-  const domainMap = yield* Ref.make(HashMap.empty());
-  const skipped = yield* Ref.make([]);
+  let domainMap = HashMap.empty();
+  const skipped = [];
   let invalidCount = 0;
-  yield* Effect.all(
-    urls.map(
-      (urlObj) => pipe(
-        normalizeUrl(urlObj.url, strategy),
-        Effect.tap(
-          (normalized) => Effect.gen(function* () {
-            const currentMap = yield* Ref.get(domainMap);
-            const key = strategy.wwwHandling === "preserve" ? normalized.normalized : normalized.domain;
-            const existingOption = HashMap.get(currentMap, key);
-            if (Option.isNone(existingOption)) {
-              yield* Ref.set(domainMap, HashMap.set(currentMap, key, urlObj));
-            } else {
-              const existing = existingOption.value;
-              let shouldReplace = false;
-              if (strategy.wwwHandling === "prefer-www") {
-                const existingHasWww = existing.url.includes("://www.");
-                const newHasWww = urlObj.url.includes("://www.");
-                shouldReplace = !existingHasWww && newHasWww;
-              } else if (strategy.wwwHandling === "prefer-non-www") {
-                const existingHasWww = existing.url.includes("://www.");
-                const newHasWww = urlObj.url.includes("://www.");
-                shouldReplace = existingHasWww && !newHasWww;
-              }
-              if (shouldReplace) {
-                yield* Ref.set(domainMap, HashMap.set(currentMap, key, urlObj));
-                yield* Ref.update(skipped, (arr) => [
-                  ...arr,
-                  { url: existing.url, reason: `Replaced by preferred variant: ${urlObj.url}` }
-                ]);
-              } else {
-                yield* Ref.update(skipped, (arr) => [
-                  ...arr,
-                  { url: urlObj.url, reason: `Duplicate of: ${existing.url}` }
-                ]);
-              }
-            }
-          })
-        ),
-        Effect.catchAll(
-          (error) => Effect.gen(function* () {
-            invalidCount++;
-            yield* Ref.update(skipped, (arr) => [
-              ...arr,
-              { url: urlObj.url, reason: `Invalid URL: ${error.message}` }
-            ]);
-            yield* Effect.logWarning(`Invalid URL skipped: ${urlObj.url}`);
-          })
-        )
-      )
-    ),
-    { concurrency: "unbounded" }
-  );
-  const finalMap = yield* Ref.get(domainMap);
-  const finalSkipped = yield* Ref.get(skipped);
-  const deduplicated = Array.from(HashMap.values(finalMap));
+  for (const urlObj of urls) {
+    const normalizeResult = yield* Effect.either(normalizeUrl(urlObj.url, strategy));
+    if (normalizeResult._tag === "Left") {
+      invalidCount++;
+      skipped.push({ url: urlObj.url, reason: `Invalid URL: ${normalizeResult.left.message}` });
+      yield* Effect.logWarning(`Invalid URL skipped: ${urlObj.url}`);
+      continue;
+    }
+    const normalized = normalizeResult.right;
+    const key = strategy.wwwHandling === "preserve" ? normalized.normalized : normalized.domain;
+    const existingOption = HashMap.get(domainMap, key);
+    if (Option.isNone(existingOption)) {
+      domainMap = HashMap.set(domainMap, key, urlObj);
+    } else {
+      const existing = existingOption.value;
+      let shouldReplace = false;
+      if (strategy.wwwHandling === "prefer-www") {
+        const existingHasWww = existing.url.includes("://www.");
+        const newHasWww = urlObj.url.includes("://www.");
+        shouldReplace = !existingHasWww && newHasWww;
+      } else if (strategy.wwwHandling === "prefer-non-www") {
+        const existingHasWww = existing.url.includes("://www.");
+        const newHasWww = urlObj.url.includes("://www.");
+        shouldReplace = existingHasWww && !newHasWww;
+      }
+      if (shouldReplace) {
+        domainMap = HashMap.set(domainMap, key, urlObj);
+        skipped.push({ url: existing.url, reason: `Replaced by preferred variant: ${urlObj.url}` });
+      } else {
+        skipped.push({ url: urlObj.url, reason: `Duplicate of: ${existing.url}` });
+      }
+    }
+  }
+  const deduplicated = Array.from(HashMap.values(domainMap));
   return {
     deduplicated,
-    skipped: finalSkipped,
+    skipped,
     stats: {
       total: urls.length,
       unique: deduplicated.length,
-      duplicates: finalSkipped.filter((s) => s.reason.startsWith("Duplicate")).length,
+      duplicates: skipped.filter((s) => s.reason.startsWith("Duplicate")).length,
       invalid: invalidCount
     }
   };