@mxfb/tools 0.3.0 → 1.0.2

package/node/headless-browser/index.d.ts

@@ -0,0 +1,37 @@
+import type { HeadlessBrowserOptions, HeadlessReadOptions } from './types.js';
+/**
+ * A Chrome session that reads pages an HTTP client alone cannot reach.
+ *
+ * Some hosts — every `*.tumblr.com` blog among them — answer any plain request with a 403
+ * carrying a JavaScript challenge, and only serve the real page to whoever runs it. One
+ * tab is kept per origin: the first read of an origin navigates there and waits for the
+ * challenge to clear, and both read modes then reuse that cleared tab, so the cookies it
+ * was granted keep applying.
+ *
+ * The browser starts itself on first use and must be closed by the caller.
+ */
+export declare class HeadlessBrowser {
+    private readonly options;
+    private readonly pagesByOrigin;
+    private browser;
+    constructor(options?: HeadlessBrowserOptions);
+    /**
+     * Reads one URL as text, starting the browser and clearing the origin's challenge when
+     * this is the first read of that origin.
+     *
+     * @param url - Absolute URL to read.
+     * @param readOptions - Overrides this session's read defaults for this call.
+     * @returns The page's text, or `null` when the URL has no readable origin, or when a
+     * `'served'` read met a response that was not a success. A `'rendered'` read returns
+     * whatever the tab ended up holding, error pages included, since that is the document
+     * the browser was given.
+     * @throws If the browser cannot be started, if a navigation times out, or if an
+     * interstitial never clears, as `challenge-not-cleared`.
+     */
+    readText(url: string, readOptions?: HeadlessReadOptions): Promise<string | null>;
+    /** Closes every tab and the browser itself. Reading again starts a new one. */
+    close(): Promise<void>;
+    private openOrigin;
+    private start;
+    private waitForChallenge;
+}

package/node/headless-browser/index.js

@@ -0,0 +1,126 @@
+import puppeteer from 'puppeteer';
+const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
+    + ' AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36';
+const DEFAULT_LAUNCH_ARGS = [
+    '--no-sandbox',
+    '--disable-setuid-sandbox',
+    // What a challenge actually reads: the automation flag is a one-line giveaway, and
+    // Chrome's headless default user agent names itself `HeadlessChrome`.
+    '--disable-blink-features=AutomationControlled'
+];
+const DEFAULT_CHALLENGE_MARKERS = ['Checking your browser'];
+const DEFAULT_CHALLENGE_TIMEOUT_MS = 30000;
+const DEFAULT_NAVIGATION_TIMEOUT_MS = 60000;
+const DEFAULT_WAIT_UNTIL = 'networkidle2';
+const CHALLENGE_POLL_MS = 500;
+/**
+ * A Chrome session that reads pages an HTTP client alone cannot reach.
+ *
+ * Some hosts — every `*.tumblr.com` blog among them — answer any plain request with a 403
+ * carrying a JavaScript challenge, and only serve the real page to whoever runs it. One
+ * tab is kept per origin: the first read of an origin navigates there and waits for the
+ * challenge to clear, and both read modes then reuse that cleared tab, so the cookies it
+ * was granted keep applying.
+ *
+ * The browser starts itself on first use and must be closed by the caller.
+ */
+export class HeadlessBrowser {
+    options;
+    pagesByOrigin = new Map();
+    browser = null;
+    constructor(options = {}) {
+        this.options = options;
+    }
+    /**
+     * Reads one URL as text, starting the browser and clearing the origin's challenge when
+     * this is the first read of that origin.
+     *
+     * @param url - Absolute URL to read.
+     * @param readOptions - Overrides this session's read defaults for this call.
+     * @returns The page's text, or `null` when the URL has no readable origin, or when a
+     * `'served'` read met a response that was not a success. A `'rendered'` read returns
+     * whatever the tab ended up holding, error pages included, since that is the document
+     * the browser was given.
+     * @throws If the browser cannot be started, if a navigation times out, or if an
+     * interstitial never clears, as `challenge-not-cleared`.
+     */
+    async readText(url, readOptions = {}) {
+        let origin;
+        try {
+            origin = new URL(url).origin;
+        }
+        catch {
+            return null;
+        }
+        const resolved = { ...this.options.read, ...readOptions };
+        const page = await this.openOrigin(origin, url, resolved);
+        if (resolved.source !== 'rendered')
+            return await readServedText(page, url);
+        if (page.url() !== url) {
+            await page.goto(url, { waitUntil: resolved.waitUntil ?? DEFAULT_WAIT_UNTIL });
+            await this.waitForChallenge(page);
+        }
+        await wait(resolved.settleMs ?? 0);
+        return await page.content();
+    }
+    /** Closes every tab and the browser itself. Reading again starts a new one. */
+    async close() {
+        const browser = this.browser;
+        this.browser = null;
+        this.pagesByOrigin.clear();
+        await browser?.close();
+    }
+    async openOrigin(origin, entryUrl, readOptions) {
+        const known = this.pagesByOrigin.get(origin);
+        if (known !== undefined)
+            return known;
+        const browser = await this.start();
+        const page = await browser.newPage();
+        this.pagesByOrigin.set(origin, page);
+        page.setDefaultNavigationTimeout(this.options.navigationTimeoutMs ?? DEFAULT_NAVIGATION_TIMEOUT_MS);
+        // A challenge answers 403 until it is solved, so a failed navigation is expected here.
+        await page
+            .goto(entryUrl, { waitUntil: readOptions.waitUntil ?? DEFAULT_WAIT_UNTIL })
+            .catch(() => null);
+        await this.waitForChallenge(page);
+        return page;
+    }
+    async start() {
+        this.browser ??= await puppeteer.launch({
+            headless: true,
+            args: [
+                ...this.options.launchArgs ?? DEFAULT_LAUNCH_ARGS,
+                `--user-agent=${this.options.userAgent ?? DEFAULT_USER_AGENT}`
+            ]
+        });
+        return this.browser;
+    }
+    async waitForChallenge(page) {
+        const markers = this.options.challengeMarkers ?? DEFAULT_CHALLENGE_MARKERS;
+        if (markers.length === 0)
+            return;
+        const deadline = Date.now()
+            + (this.options.challengeTimeoutMs ?? DEFAULT_CHALLENGE_TIMEOUT_MS);
+        while (Date.now() < deadline) {
+            // Reading mid-navigation throws, which only means the challenge is still working.
+            const content = await page.content().catch(() => null);
+            if (content !== null && !markers.some(marker => content.includes(marker)))
+                return;
+            await wait(CHALLENGE_POLL_MS);
+        }
+        throw new Error('challenge-not-cleared');
+    }
+}
+async function readServedText(page, url) {
+    return await page.evaluate(async (target) => {
+        const response = await fetch(target, { credentials: 'include' });
+        if (!response.ok)
+            return null;
+        return await response.text();
+    }, url);
+}
+async function wait(durationMs) {
+    if (durationMs <= 0)
+        return;
+    await new Promise(resolve => { setTimeout(resolve, durationMs); });
+}

package/node/headless-browser/types.d.ts

@@ -0,0 +1,47 @@
+import type { PuppeteerLifeCycleEvent } from 'puppeteer';
+/** What a read returns: the body the server sent, or the DOM once scripts have run. */
+export declare const headlessReadSources: readonly ["served", "rendered"];
+/** Which of the two a single read asks for. */
+export type HeadlessReadSource = typeof headlessReadSources[number];
+/**
+ * How one read behaves.
+ *
+ * @property source - `'served'`, the default, returns the response body verbatim — the
+ * markup a parser should see, and the only mode that reads XML or plain text correctly.
+ * `'rendered'` navigates the tab and returns `document` as Chrome built it, which is what
+ * a page assembling its content in JavaScript requires.
+ * @property waitUntil - Which lifecycle event ends a `'rendered'` navigation. Defaults to
+ * `'networkidle2'`.
+ * @property settleMs - How long to wait after that event before reading a `'rendered'`
+ * page, for the work a lifecycle event does not cover. Defaults to 0.
+ */
+export type HeadlessReadOptions = {
+    source?: HeadlessReadSource;
+    waitUntil?: PuppeteerLifeCycleEvent;
+    settleMs?: number;
+};
+/**
+ * How a headless browser session behaves.
+ *
+ * @property userAgent - User agent to announce. The default is a current desktop Chrome
+ * string, because Chrome's own headless default names itself `HeadlessChrome`, which is
+ * enough for some edges to refuse every request.
+ * @property launchArgs - Replaces the default Chrome flags entirely, for a caller who
+ * needs a different sandbox, window size or locale.
+ * @property challengeMarkers - Strings whose presence in a page means an interstitial is
+ * still being shown rather than the page asked for. Defaults to Automattic's
+ * `Checking your browser`, which is what a Tumblr blog host answers with; an empty list
+ * turns the wait off for a host that challenges nothing.
+ * @property challengeTimeoutMs - How long to let such an interstitial clear itself before
+ * giving up. Defaults to 30000.
+ * @property navigationTimeoutMs - How long one navigation may take. Defaults to 60000.
+ * @property read - Defaults for every read of this session, each overridable per call.
+ */
+export type HeadlessBrowserOptions = {
+    userAgent?: string;
+    launchArgs?: string[];
+    challengeMarkers?: string[];
+    challengeTimeoutMs?: number;
+    navigationTimeoutMs?: number;
+    read?: HeadlessReadOptions;
+};

package/node/headless-browser/types.js

@@ -0,0 +1,5 @@
+/** What a read returns: the body the server sent, or the DOM once scripts have run. */
+export const headlessReadSources = [
+    'served',
+    'rendered'
+];

package/node/index.d.ts

@@ -1,2 +1 @@
-import * as MxfbCliConfig from './mxfb-cli-config';
-export { MxfbCliConfig };
+export {};

package/node/index.js

@@ -0,0 +1 @@
+export {};

package/node/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node/index.test.js

@@ -0,0 +1,6 @@
+import { describe, expect, it } from 'vitest';
+describe('fake-test', () => {
+    it('passes every time', () => {
+        expect(true).toBe(true);
+    });
+});

package/node/scrapers/@tumblr/index.d.ts

@@ -0,0 +1,61 @@
+import type { TumblrPage, TumblrPost } from '../../../agnostic/scrapers/@tumblr/types.js';
+import type { CrawlBlogOptions, CrawlBlogResult } from './types.js';
+/**
+ * Crawls a Tumblr blog and writes everything it reads into `dataDirPath`.
+ *
+ * The blog's sitemap is read first: it enumerates every post in a request or two, and
+ * states when each was last modified, which is the only date available on a theme that
+ * publishes none.
+ *
+ * The crawl is a frontier: it starts from the blog's sitemap and its root listing, then
+ * enqueues whatever the pages it parses reveal, keeping only the kinds named by
+ * `followedKinds`. A listing is followed to its next page for as long as it keeps
+ * yielding posts that have not been seen — which is the real stop condition, since a
+ * listing past the end of a blog answers with an empty page rather than a 404.
+ *
+ * With no `dataDirPath`, a temporary directory is created and returned, so a caller can
+ * tell its user which path to hand back to resume the crawl later.
+ *
+ * Three files are produced: `pages.ndjson` holds one parsed page per line and is the
+ * record of truth, `state.json` holds the frontier so a later run can resume, and `raw/`
+ * keeps each page's HTML so a parser fix can be replayed without fetching again.
+ *
+ * Every read goes through a headless Chrome unless a `fetcher` is given: a blog host
+ * answers a plain HTTP request with a 403 and a JavaScript challenge, so nothing — not
+ * even `robots.txt` — is readable without a browser solving it first. The browser is
+ * started for the crawl and closed with it, failures included.
+ *
+ * @param baseTumblrUrl - Any URL on the blog; only its origin is used.
+ * @param options - How the crawl should behave.
+ * @returns What the crawl achieved, and every page it could not read.
+ * @throws If `baseTumblrUrl` has no readable origin, as `invalid-base-url`. A page that
+ * cannot be fetched or parsed is reported as a failure instead, and never interrupts the
+ * crawl.
+ */
+export declare function crawlBlog(baseTumblrUrl: string, options: CrawlBlogOptions): Promise<CrawlBlogResult>;
+/**
+ * Streams the pages an archive holds, in the order the crawl wrote them.
+ *
+ * Saves a consumer knowing the archive's file name and line format, and reads the dates
+ * back as `Date` rather than the strings JSON stored. A line that is not a page record is
+ * skipped rather than thrown over, so a crawl interrupted mid-write stays readable.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns Each page in turn.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export declare function readArchivePages(dataDirPath: string): AsyncGenerator<TumblrPage>;
+/**
+ * Reads every distinct post an archive holds, one entry per post id.
+ *
+ * A post is written once per page that mentioned it — a listing identifies it, its
+ * permalink describes it — so the records are merged: the permalink's version wins, and
+ * whatever it left empty is filled from the other, which is how a post keeps both its
+ * title and the `lastModifiedOn` only the sitemap knew.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns The posts, in the order their ids were first seen. Posts with no id are left
+ * out, having nothing to be merged on.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export declare function readArchivePosts(dataDirPath: string): Promise<TumblrPost[]>;

package/node/scrapers/@tumblr/index.js

@@ -0,0 +1,182 @@
+import * as Crawler from '@design-edito/tools/agnostic/misc/crawler/index.js';
+import { fillPostDetails, parsePage } from '../../../agnostic/scrapers/@tumblr/index.js';
+import { HeadlessBrowser } from '../../headless-browser/index.js';
+import { appendPageRecord, buildListingPageUrls, collectSitemapLocations, createTemporaryDataDir, discoverUrlsToFollow, ensureDataDirectories, fetchRobots, fillLastModifiedFromSitemap, mergeArchivedPosts, normalizeUrl, readCrawlState, readPageRecords, resolveDelayMs, toErrorMessage, toOriginOrNull, writeCrawlState, writeRawPage } from './utils.js';
+const DEFAULT_LIMIT = 10000;
+const DEFAULT_DELAY_MS = 250;
+/**
+ * Crawls a Tumblr blog and writes everything it reads into `dataDirPath`.
+ *
+ * The blog's sitemap is read first: it enumerates every post in a request or two, and
+ * states when each was last modified, which is the only date available on a theme that
+ * publishes none.
+ *
+ * The crawl is a frontier: it starts from the blog's sitemap and its root listing, then
+ * enqueues whatever the pages it parses reveal, keeping only the kinds named by
+ * `followedKinds`. A listing is followed to its next page for as long as it keeps
+ * yielding posts that have not been seen — which is the real stop condition, since a
+ * listing past the end of a blog answers with an empty page rather than a 404.
+ *
+ * With no `dataDirPath`, a temporary directory is created and returned, so a caller can
+ * tell its user which path to hand back to resume the crawl later.
+ *
+ * Three files are produced: `pages.ndjson` holds one parsed page per line and is the
+ * record of truth, `state.json` holds the frontier so a later run can resume, and `raw/`
+ * keeps each page's HTML so a parser fix can be replayed without fetching again.
+ *
+ * Every read goes through a headless Chrome unless a `fetcher` is given: a blog host
+ * answers a plain HTTP request with a 403 and a JavaScript challenge, so nothing — not
+ * even `robots.txt` — is readable without a browser solving it first. The browser is
+ * started for the crawl and closed with it, failures included.
+ *
+ * @param baseTumblrUrl - Any URL on the blog; only its origin is used.
+ * @param options - How the crawl should behave.
+ * @returns What the crawl achieved, and every page it could not read.
+ * @throws If `baseTumblrUrl` has no readable origin, as `invalid-base-url`. A page that
+ * cannot be fetched or parsed is reported as a failure instead, and never interrupts the
+ * crawl.
+ */
+export async function crawlBlog(baseTumblrUrl, options) {
+    const baseUrl = toOriginOrNull(baseTumblrUrl);
+    if (baseUrl === null)
+        throw new Error('invalid-base-url');
+    const dataDirPath = options.dataDirPath ?? await createTemporaryDataDir();
+    await ensureDataDirectories(dataDirPath, options.saveRawPages ?? true);
+    const browser = options.fetcher === undefined ? new HeadlessBrowser() : null;
+    const fetcher = options.fetcher ?? (async (url) => await browser?.readText(url) ?? null);
+    try {
+        return await runCrawl(baseUrl, dataDirPath, fetcher, options);
+    }
+    finally {
+        await browser?.close();
+    }
+}
+/**
+ * Streams the pages an archive holds, in the order the crawl wrote them.
+ *
+ * Saves a consumer knowing the archive's file name and line format, and reads the dates
+ * back as `Date` rather than the strings JSON stored. A line that is not a page record is
+ * skipped rather than thrown over, so a crawl interrupted mid-write stays readable.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns Each page in turn.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export async function* readArchivePages(dataDirPath) {
+    yield* readPageRecords(dataDirPath);
+}
+/**
+ * Reads every distinct post an archive holds, one entry per post id.
+ *
+ * A post is written once per page that mentioned it — a listing identifies it, its
+ * permalink describes it — so the records are merged: the permalink's version wins, and
+ * whatever it left empty is filled from the other, which is how a post keeps both its
+ * title and the `lastModifiedOn` only the sitemap knew.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns The posts, in the order their ids were first seen. Posts with no id are left
+ * out, having nothing to be merged on.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export async function readArchivePosts(dataDirPath) {
+    const postsById = new Map();
+    for await (const page of readPageRecords(dataDirPath)) {
+        for (const post of page.posts) {
+            if (post.id === null)
+                continue;
+            const known = postsById.get(post.id);
+            postsById.set(post.id, known === undefined ? post : mergeArchivedPosts(known, post));
+        }
+    }
+    return Array.from(postsById.values());
+}
+async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
+    const { followedKinds = ['index', 'permalink'], pages, limit = DEFAULT_LIMIT, saveRawPages = true, resume = false, useSitemap = true, onPageCrawled } = options;
+    const restoredState = resume ? await readCrawlState(dataDirPath, baseUrl) : null;
+    const robots = useSitemap ? await fetchRobots(baseUrl, fetcher) : null;
+    const sitemapLocations = useSitemap
+        ? await collectSitemapLocations(baseUrl, robots, fetcher)
+        : new Map();
+    const knownPostIds = new Set(restoredState?.knownPostIds ?? []);
+    const failures = [];
+    let blog = null;
+    let pageCount = 0;
+    const reportFailure = (url, error) => {
+        const report = {
+            url,
+            kind: null,
+            postCount: 0,
+            newPostCount: 0,
+            error
+        };
+        failures.push(report);
+        onPageCrawled?.(report);
+    };
+    const crawler = Crawler.create({
+        limit,
+        delayMs: options.delayMs ?? resolveDelayMs(robots, DEFAULT_DELAY_MS),
+        fetcher,
+        processor: async (url, pageContent, { push }) => {
+            if (pageContent === null)
+                return reportFailure(url, 'fetch-failed');
+            let parsedPage;
+            try {
+                parsedPage = await fillPostDetails(parsePage(pageContent, url));
+            }
+            catch (error) {
+                return reportFailure(url, toErrorMessage(error));
+            }
+            const page = fillLastModifiedFromSitemap(parsedPage, sitemapLocations, baseUrl);
+            pageCount = pageCount + 1;
+            blog ??= page.blog;
+            if (saveRawPages)
+                await writeRawPage(dataDirPath, url, pageContent);
+            await appendPageRecord(dataDirPath, page);
+            let newPostCount = 0;
+            for (const post of page.posts) {
+                if (post.id === null || knownPostIds.has(post.id))
+                    continue;
+                knownPostIds.add(post.id);
+                newPostCount = newPostCount + 1;
+            }
+            push(...discoverUrlsToFollow(page, followedKinds, newPostCount, baseUrl));
+            onPageCrawled?.({
+                url,
+                kind: page.kind,
+                postCount: page.posts.length,
+                newPostCount,
+                error: null
+            });
+        }
+    });
+    for (const processedUrl of restoredState?.processedUrls ?? [])
+        crawler.processed.add(processedUrl);
+    const seedUrls = pages === undefined
+        ? [baseUrl]
+        : buildListingPageUrls(baseUrl, pages.from, pages.to);
+    if (followedKinds.includes('permalink'))
+        seedUrls.push(...sitemapLocations.keys());
+    seedUrls.push(...restoredState?.waitlistUrls ?? []);
+    const pendingSeedUrls = Array.from(new Set(seedUrls
+        .map(seedUrl => normalizeUrl(seedUrl, baseUrl))
+        .filter((seedUrl) => seedUrl !== null)))
+        .filter(seedUrl => !crawler.processed.has(seedUrl));
+    const lastSeedUrl = pendingSeedUrls[pendingSeedUrls.length - 1];
+    if (lastSeedUrl !== undefined) {
+        crawler.push(...pendingSeedUrls.slice(0, -1));
+        await crawler.crawl(lastSeedUrl);
+    }
+    await writeCrawlState(dataDirPath, {
+        baseUrl,
+        processedUrls: Array.from(crawler.processed),
+        waitlistUrls: crawler.waitlist,
+        knownPostIds: Array.from(knownPostIds)
+    });
+    return {
+        dataDirPath,
+        blog,
+        pageCount,
+        postCount: knownPostIds.size,
+        failures
+    };
+}

package/node/scrapers/@tumblr/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node/scrapers/@tumblr/index.test.js

@@ -0,0 +1,185 @@
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { crawlBlog, readArchivePages, readArchivePosts } from './index.js';
+const blogUrl = 'https://somebody.tumblr.com';
+function listingPage(canonicalPath, postIds, nextPath) {
+    const nextLink = nextPath === null ? '' : `<link rel="next" href="${nextPath}" />`;
+    const postLinks = postIds
+        .map(postId => `<a href="/post/${postId}/slug">${postId}</a>`)
+        .join('');
+    return `<!doctype html>
+    <html>
+      <head>
+        <link rel="canonical" href="${blogUrl}${canonicalPath}" />
+        <meta property="og:site_name" content="Somebody" />
+        <meta name="description" content="A blog about nothing" />
+        ${nextLink}
+      </head>
+      <body>${postLinks}</body>
+    </html>`;
+}
+function permalinkPage(postId) {
+    return `<!doctype html>
+    <html>
+      <head>
+        <link rel="canonical" href="${blogUrl}/post/${postId}/slug" />
+        <meta property="og:site_name" content="Somebody" />
+        <meta property="og:title" content="Post ${postId}" />
+        <meta property="article:tag" content="tagged" />
+      </head>
+      <body></body>
+    </html>`;
+}
+const responsesByUrl = {
+    [`${blogUrl}/robots.txt`]: [
+        'Sitemap: https://somebody.tumblr.com/sitemap.xml',
+        'User-agent: *',
+        'Crawl-delay: 0'
+    ].join('\n'),
+    [`${blogUrl}/sitemap.xml`]: `<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <url>
+        <loc>${blogUrl}/post/1001/slug</loc>
+        <lastmod>2019-01-31T16:39:53+00:00</lastmod>
+      </url>
+    </urlset>`,
+    [`${blogUrl}/`]: listingPage('/', ['1001', '1002'], '/page/2'),
+    [`${blogUrl}/page/2`]: listingPage('/page/2', ['1003'], '/page/3'),
+    [`${blogUrl}/page/3`]: listingPage('/page/3', [], null),
+    [`${blogUrl}/post/1001/slug`]: permalinkPage('1001'),
+    [`${blogUrl}/post/1002/slug`]: permalinkPage('1002'),
+    [`${blogUrl}/post/1003/slug`]: permalinkPage('1003')
+};
+// The crawl reads through this rather than through a headless Chrome, which is what
+// `crawlBlog` starts when no fetcher is given.
+const fetcher = async (url) => responsesByUrl[url] ?? null;
+let dataDirPath;
+beforeEach(async () => {
+    dataDirPath = await fs.mkdtemp(path.join(os.tmpdir(), 'mf-tumblr-'));
+});
+afterEach(async () => {
+    await fs.rm(dataDirPath, { recursive: true, force: true });
+});
+describe('scrapers/@tumblr/crawlBlog', () => {
+    it('walks the listings, follows the permalinks and records every post once', async () => {
+        const result = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        expect(result.blog?.name).toBe('somebody');
+        expect(result.postCount).toBe(3);
+        expect(result.pageCount).toBe(6);
+        expect(result.failures).toEqual([]);
+    });
+    it('writes one record per parsed page, plus the state and the raw pages', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const records = await fs.readFile(path.join(dataDirPath, 'pages.ndjson'), 'utf-8');
+        const lines = records.split('\n').filter(line => line.trim() !== '');
+        expect(lines).toHaveLength(6);
+        const state = await fs.readFile(path.join(dataDirPath, 'state.json'), 'utf-8');
+        expect(JSON.parse(state).knownPostIds.sort()).toEqual(['1001', '1002', '1003']);
+        const rawFiles = await fs.readdir(path.join(dataDirPath, 'raw'));
+        expect(rawFiles).toHaveLength(6);
+    });
+    it('stops walking listings once a page yields no post it has not seen', async () => {
+        const crawledUrls = [];
+        await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            onPageCrawled: report => { crawledUrls.push(report.url); }
+        });
+        expect(crawledUrls).toContain(`${blogUrl}/page/3`);
+        expect(crawledUrls).not.toContain(`${blogUrl}/page/4`);
+    });
+    it('keeps nothing but listings when permalinks are not followed', async () => {
+        const result = await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            followedKinds: ['index'],
+            useSitemap: false
+        });
+        expect(result.pageCount).toBe(3);
+        expect(result.postCount).toBe(3);
+    });
+    it('reports a page it cannot fetch as a failure without interrupting the crawl', async () => {
+        const result = await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            pages: { from: 1, to: 2 },
+            useSitemap: false,
+            followedKinds: []
+        });
+        expect(result.failures.map(failure => failure.url)).toEqual([`${blogUrl}/page/1`]);
+        expect(result.failures[0]?.error).toBe('fetch-failed');
+        expect(result.pageCount).toBe(1);
+    });
+    it('dates a post from the sitemap, which is the only date the theme offers', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const records = await fs.readFile(path.join(dataDirPath, 'pages.ndjson'), 'utf-8');
+        const posts = records
+            .split('\n')
+            .filter(line => line.trim() !== '')
+            .flatMap(line => JSON.parse(line).posts);
+        const dated = posts.filter((post) => post.lastModifiedOn !== null);
+        expect(dated).not.toHaveLength(0);
+        expect(dated[0].lastModifiedOn).toBe('2019-01-31T16:39:53.000Z');
+        expect(dated.every((post) => post.url.includes('/post/1001/'))).toBe(true);
+    });
+    it('creates a temporary directory of its own, and says which one', async () => {
+        const result = await crawlBlog(blogUrl, { delayMs: 0, fetcher });
+        try {
+            expect(result.dataDirPath.startsWith(os.tmpdir())).toBe(true);
+            expect(path.basename(result.dataDirPath)).toMatch(/^[0-9a-f]{16}$/);
+            const records = await fs.readFile(path.join(result.dataDirPath, 'pages.ndjson'), 'utf-8');
+            expect(records.split('\n').filter(line => line.trim() !== '')).toHaveLength(6);
+        }
+        finally {
+            await fs.rm(result.dataDirPath, { recursive: true, force: true });
+        }
+    });
+    it('reports back the directory it was given', async () => {
+        const result = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        expect(result.dataDirPath).toBe(dataDirPath);
+    });
+    it('fetches nothing on a resumed crawl that already covered the blog', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const resumed = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher, resume: true });
+        expect(resumed.pageCount).toBe(0);
+        expect(resumed.postCount).toBe(3);
+    });
+});
+describe('scrapers/@tumblr/readArchivePages & readArchivePosts', () => {
+    beforeEach(async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+    });
+    it('streams every page the crawl wrote', async () => {
+        const urls = [];
+        for await (const page of readArchivePages(dataDirPath))
+            urls.push(page.url);
+        expect(urls).toHaveLength(6);
+        expect(urls).toContain(`${blogUrl}/post/1001/slug`);
+    });
+    it('reads the dates back as dates rather than the strings JSON stored', async () => {
+        const posts = await readArchivePosts(dataDirPath);
+        const post = posts.find(candidate => candidate.id === '1001');
+        expect(post?.lastModifiedOn).toBeInstanceOf(Date);
+        expect(post?.lastModifiedOn?.toISOString()).toBe('2019-01-31T16:39:53.000Z');
+        for await (const page of readArchivePages(dataDirPath)) {
+            expect(page.collectedOn).toBeInstanceOf(Date);
+        }
+    });
+    it('merges a post written once per page into a single entry', async () => {
+        const posts = await readArchivePosts(dataDirPath);
+        expect(posts.map(post => post.id)).toEqual(['1001', '1002', '1003']);
+        const post = posts.find(candidate => candidate.id === '1001');
+        // Its title came from the permalink page, its lastModifiedOn from the sitemap.
+        expect(post?.title).toBe('Post 1001');
+        expect(post?.tags).toEqual(['tagged']);
+        expect(post?.lastModifiedOn).not.toBeNull();
+    });
+    it('throws when the directory holds no archive', async () => {
+        await expect(readArchivePosts(path.join(dataDirPath, 'nowhere'))).rejects.toThrow();
+    });
+});