@mxfb/tools 1.0.3 → 1.0.5

package/node/scrapers/@tumblr/index.js

@@ -91,25 +91,35 @@ export async function readArchivePosts(dataDirPath) {
     return Array.from(postsById.values());
 }
 async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
-    const { followedKinds = ['index', 'permalink'], pages, limit = DEFAULT_LIMIT, saveRawPages = true, resume = false, useSitemap = true, onPageCrawled } = options;
+    const { followedKinds = ['index', 'permalink'], pages, limit = DEFAULT_LIMIT, saveRawPages = true, resume = false, retryFailures = false, useSitemap = true, onPageCrawled } = options;
     const restoredState = resume ? await readCrawlState(dataDirPath, baseUrl) : null;
     const robots = useSitemap ? await fetchRobots(baseUrl, fetcher) : null;
     const sitemapLocations = useSitemap
         ? await collectSitemapLocations(baseUrl, robots, fetcher)
         : new Map();
     const knownPostIds = new Set(restoredState?.knownPostIds ?? []);
-    const failures = [];
+    const failuresByUrl = new Map((restoredState?.failures ?? []).map(failure => [failure.url, failure]));
+    const urlsToRetry = retryFailures ? Array.from(failuresByUrl.keys()) : [];
     let blog = null;
     let pageCount = 0;
+    // The page being reported is moved into `processed` only once the processor returns, so
+    // it is counted by hand. `foundPageCount` dedupes because the waitlist may hold a URL
+    // twice, or one already processed, and the count must never go backwards.
+    const countPages = () => ({
+        scannedPageCount: crawler.processed.size + 1,
+        foundPageCount: new Set([...crawler.processed, ...crawler.waitlist]).size
+    });
     const reportFailure = (url, error) => {
         const report = {
             url,
             kind: null,
             postCount: 0,
             newPostCount: 0,
-            error
+            error,
+            ...countPages()
         };
-        failures.push(report);
+        const previousCount = failuresByUrl.get(url)?.failureCount ?? 0;
+        failuresByUrl.set(url, { url, failureCount: previousCount + 1, error });
         onPageCrawled?.(report);
     };
     const crawler = Crawler.create({
@@ -127,6 +137,7 @@ async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
                 return reportFailure(url, toErrorMessage(error));
             }
             const page = fillLastModifiedFromSitemap(parsedPage, sitemapLocations, baseUrl);
+            failuresByUrl.delete(url);
             pageCount = pageCount + 1;
             blog ??= page.blog;
             if (saveRawPages)
@@ -145,18 +156,24 @@ async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
                 kind: page.kind,
                 postCount: page.posts.length,
                 newPostCount,
-                error: null
+                error: null,
+                ...countPages()
             });
         }
     });
-    for (const processedUrl of restoredState?.processedUrls ?? [])
+    const retriedUrls = new Set(urlsToRetry);
+    for (const processedUrl of restoredState?.processedUrls ?? []) {
+        if (retriedUrls.has(processedUrl))
+            continue;
         crawler.processed.add(processedUrl);
+    }
     const seedUrls = pages === undefined
         ? [baseUrl]
         : buildListingPageUrls(baseUrl, pages.from, pages.to);
     if (followedKinds.includes('permalink'))
         seedUrls.push(...sitemapLocations.keys());
     seedUrls.push(...restoredState?.waitlistUrls ?? []);
+    seedUrls.push(...urlsToRetry);
     const pendingSeedUrls = Array.from(new Set(seedUrls
         .map(seedUrl => normalizeUrl(seedUrl, baseUrl))
         .filter((seedUrl) => seedUrl !== null)))
@@ -170,13 +187,14 @@ async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
         baseUrl,
         processedUrls: Array.from(crawler.processed),
         waitlistUrls: crawler.waitlist,
-        knownPostIds: Array.from(knownPostIds)
+        knownPostIds: Array.from(knownPostIds),
+        failures: Array.from(failuresByUrl.values())
     });
     return {
         dataDirPath,
         blog,
         pageCount,
         postCount: knownPostIds.size,
-        failures
+        failures: Array.from(failuresByUrl.values())
     };
 }

package/node/scrapers/@tumblr/types.d.ts

@@ -16,6 +16,8 @@ import type { TumblrBlog, TumblrPageKind } from '../../../agnostic/scrapers/@tum
  * @property saveRawPages - Keeps each page's raw HTML under `raw/`, so a later parser fix
  * can be replayed without fetching anything again. Defaults to `true`.
  * @property resume - Picks up where a previous crawl of the same directory stopped.
+ * @property retryFailures - Fetches again the URLs a previous run recorded as failed,
+ * which a plain resume skips since they were marked processed. Needs `resume`.
  * @property useSitemap - Reads `robots.txt` and `sitemap.xml` to enumerate posts without
  * walking listings. Defaults to `true`.
  * @property onPageCrawled - Called once per page, successes and failures alike.
@@ -35,6 +37,7 @@ export type CrawlBlogOptions = {
     delayMs?: number | (() => number);
     saveRawPages?: boolean;
     resume?: boolean;
+    retryFailures?: boolean;
     useSitemap?: boolean;
     onPageCrawled?: (report: CrawledPageReport) => void;
     fetcher?: (url: string) => Promise<string | null>;
@@ -48,6 +51,10 @@ export type CrawlBlogOptions = {
  * @property newPostCount - Posts the crawl had not seen before, which is what tells a
  * listing worth following from one that only repeats itself.
  * @property error - Why the page was skipped, `null` on success.
+ * @property scannedPageCount - Pages read so far, this one included, failures among them.
+ * @property foundPageCount - Pages the crawl knows of, read or still queued. It only ever
+ * grows: a listing crawl discovers as it goes, so this is what has been found rather than
+ * a total to divide by.
  */
 export type CrawledPageReport = {
     url: string;
@@ -55,6 +62,21 @@ export type CrawledPageReport = {
     postCount: number;
     newPostCount: number;
     error: string | null;
+    scannedPageCount: number;
+    foundPageCount: number;
+};
+/**
+ * A URL the crawl could not read, and how often it has resisted.
+ *
+ * @property url - The URL that failed.
+ * @property failureCount - Times it has failed, across every run against this archive. A
+ * URL that later succeeds is dropped rather than kept at its last count.
+ * @property error - Why it failed the last time it was tried.
+ */
+export type CrawlFailure = {
+    url: string;
+    failureCount: number;
+    error: string;
 };
 /**
  * What one crawl achieved.
@@ -65,14 +87,15 @@ export type CrawledPageReport = {
  * could be read.
  * @property pageCount - Pages successfully parsed.
  * @property postCount - Distinct posts seen across them.
- * @property failures - Every page that could not be fetched or parsed.
+ * @property failures - URLs still unread, this run's and those a previous run left
+ * behind, with how often each has failed. Empty when the archive holds no gap.
  */
 export type CrawlBlogResult = {
     dataDirPath: string;
     blog: TumblrBlog | null;
     pageCount: number;
     postCount: number;
-    failures: CrawledPageReport[];
+    failures: CrawlFailure[];
 };
 /**
  * A crawl's progress, as `state.json` holds it between two runs.
@@ -82,10 +105,13 @@ export type CrawlBlogResult = {
  * again.
  * @property waitlistUrls - URLs still owed a fetch when the crawl stopped.
  * @property knownPostIds - Every post id seen so far.
+ * @property failures - URLs that could not be read, kept so a later run can report them
+ * and offer to try again.
  */
 export type CrawlState = {
     baseUrl: string;
     processedUrls: string[];
     waitlistUrls: string[];
     knownPostIds: string[];
+    failures: CrawlFailure[];
 };

package/node/scrapers/@tumblr/utils.js

@@ -145,7 +145,8 @@ export async function rebuildCrawlStateFromRecords(dataDirPath, baseUrl) {
         baseUrl,
         processedUrls: Array.from(processedUrls),
         waitlistUrls: [],
-        knownPostIds: Array.from(knownPostIds)
+        knownPostIds: Array.from(knownPostIds),
+        failures: []
     };
 }
 export async function* readPageRecords(dataDirPath) {
@@ -260,9 +261,30 @@ function toCrawlStateOrNull(value) {
         baseUrl,
         processedUrls: toStringArray(record.processedUrls),
         waitlistUrls: toStringArray(record.waitlistUrls),
-        knownPostIds: toStringArray(record.knownPostIds)
+        knownPostIds: toStringArray(record.knownPostIds),
+        failures: toCrawlFailures(record.failures)
     };
 }
+function toCrawlFailures(value) {
+    if (!Array.isArray(value))
+        return [];
+    const failures = [];
+    for (const item of value) {
+        if (typeof item !== 'object' || item === null)
+            continue;
+        /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+        const record = item;
+        const url = record.url;
+        if (typeof url !== 'string')
+            continue;
+        failures.push({
+            url,
+            failureCount: typeof record.failureCount === 'number' ? record.failureCount : 1,
+            error: typeof record.error === 'string' ? record.error : 'unknown'
+        });
+    }
+    return failures;
+}
 function toPageRecordDigest(value) {
     if (typeof value !== 'object' || value === null)
         return { url: null, postIds: [] };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mxfb/tools",
-  "version": "1.0.3",
+  "version": "1.0.5",
   "description": "",
   "author": "Maxime Fabas",
   "license": "ISC",