@thallylabs/migrate 0.2.29 → 0.2.30

package/dist/url-migration.js

@@ -3300,6 +3300,33 @@ async function runUrlMigration(options) {
   enqueue(sourceUrl, sourceUrl, "submitted");
   for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`${scopePath === "/" ? "" : scopePath}/${path}`, sourceUrl, "discovery-index");
   if (scopePath !== "/") for (const path of ["llms.txt", "sitemap.xml", "robots.txt"]) enqueue(`/${path}`, sourceUrl, "discovery-index");
+  const enqueueArticleLinks = (links, base, discoveredBy) => {
+    for (const link of links) {
+      try {
+        const url = new URL(link, base);
+        if (!/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js)$/i.test(url.pathname)) enqueue(link, base, discoveredBy);
+      } catch {
+      }
+    }
+  };
+  const enqueueNavigation = (nodes, base) => {
+    for (const node of nodes) {
+      if (node.url) enqueue(node.url, base, "source-navigation");
+      enqueueNavigation(node.children ?? [], base);
+    }
+  };
+  for (const record of [...records.values()]) {
+    if (!record.evidence) continue;
+    const final = canonical(record.evidence.finalUrl, sourceUrl);
+    if (!final) continue;
+    if (record.outcome === "imported") enqueueArticleLinks(record.evidence.links, final, record.url);
+    else if (record.outcome === "excluded") {
+      if (classifyMigrationResponse({ finalUrl: new URL(final), contentType: record.evidence.contentType, body: "" }) === "index") {
+        for (const link of record.evidence.links) enqueue(link, final, record.url);
+      }
+    }
+  }
+  enqueueNavigation(checkpoint.navigation, sourceUrl);
   const fetcher = options.fetcher ?? defaultMigrationFetcher;
   const capture = options.captureProvider?.capture.bind(options.captureProvider) ?? (async ({ url }) => fetcher(url, { accept: ACCEPT }));
   const assetFetcher = options.assetFetcher ?? (!options.fetcher && !options.captureProvider ? defaultAssetFetcher : void 0);
@@ -3486,20 +3513,8 @@ ${difference}`;
         checkpoint.navigation = navigation;
         navigationChanged = true;
       }
-      for (const link of evidence.links) {
-        try {
-          const url = new URL(link, final);
-          if (!/\.(?:png|jpe?g|gif|svg|webp|avif|pdf|zip|mp4|webm|css|js)$/i.test(url.pathname)) enqueue(link, final, record.url);
-        } catch {
-        }
-      }
-      const visit = (nodes) => {
-        for (const node of nodes) {
-          if (node.url) enqueue(node.url, final, "source-navigation");
-          visit(node.children ?? []);
-        }
-      };
-      visit(navigation);
+      enqueueArticleLinks(evidence.links, final, record.url);
+      enqueueNavigation(navigation, final);
     } catch (error) {
       if (options.signal?.aborted) return;
       const wasImported = record.outcome === "imported";