@mxfb/tools 1.0.0 → 1.0.2

package/node/headless-browser/types.d.ts

@@ -0,0 +1,47 @@
+import type { PuppeteerLifeCycleEvent } from 'puppeteer';
+/** What a read returns: the body the server sent, or the DOM once scripts have run. */
+export declare const headlessReadSources: readonly ["served", "rendered"];
+/** Which of the two a single read asks for. */
+export type HeadlessReadSource = typeof headlessReadSources[number];
+/**
+ * How one read behaves.
+ *
+ * @property source - `'served'`, the default, returns the response body verbatim — the
+ * markup a parser should see, and the only mode that reads XML or plain text correctly.
+ * `'rendered'` navigates the tab and returns `document` as Chrome built it, which is what
+ * a page assembling its content in JavaScript requires.
+ * @property waitUntil - Which lifecycle event ends a `'rendered'` navigation. Defaults to
+ * `'networkidle2'`.
+ * @property settleMs - How long to wait after that event before reading a `'rendered'`
+ * page, for the work a lifecycle event does not cover. Defaults to 0.
+ */
+export type HeadlessReadOptions = {
+    source?: HeadlessReadSource;
+    waitUntil?: PuppeteerLifeCycleEvent;
+    settleMs?: number;
+};
+/**
+ * How a headless browser session behaves.
+ *
+ * @property userAgent - User agent to announce. The default is a current desktop Chrome
+ * string, because Chrome's own headless default names itself `HeadlessChrome`, which is
+ * enough for some edges to refuse every request.
+ * @property launchArgs - Replaces the default Chrome flags entirely, for a caller who
+ * needs a different sandbox, window size or locale.
+ * @property challengeMarkers - Strings whose presence in a page means an interstitial is
+ * still being shown rather than the page asked for. Defaults to Automattic's
+ * `Checking your browser`, which is what a Tumblr blog host answers with; an empty list
+ * turns the wait off for a host that challenges nothing.
+ * @property challengeTimeoutMs - How long to let such an interstitial clear itself before
+ * giving up. Defaults to 30000.
+ * @property navigationTimeoutMs - How long one navigation may take. Defaults to 60000.
+ * @property read - Defaults for every read of this session, each overridable per call.
+ */
+export type HeadlessBrowserOptions = {
+    userAgent?: string;
+    launchArgs?: string[];
+    challengeMarkers?: string[];
+    challengeTimeoutMs?: number;
+    navigationTimeoutMs?: number;
+    read?: HeadlessReadOptions;
+};

package/node/headless-browser/types.js

@@ -0,0 +1,5 @@
+/** What a read returns: the body the server sent, or the DOM once scripts have run. */
+export const headlessReadSources = [
+    'served',
+    'rendered'
+];

package/node/index.js

@@ -0,0 +1 @@
+export {};

package/node/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node/index.test.js

@@ -0,0 +1,6 @@
+import { describe, expect, it } from 'vitest';
+describe('fake-test', () => {
+    it('passes every time', () => {
+        expect(true).toBe(true);
+    });
+});

package/node/scrapers/@tumblr/index.d.ts

@@ -0,0 +1,61 @@
+import type { TumblrPage, TumblrPost } from '../../../agnostic/scrapers/@tumblr/types.js';
+import type { CrawlBlogOptions, CrawlBlogResult } from './types.js';
+/**
+ * Crawls a Tumblr blog and writes everything it reads into `dataDirPath`.
+ *
+ * The blog's sitemap is read first: it enumerates every post in a request or two, and
+ * states when each was last modified, which is the only date available on a theme that
+ * publishes none.
+ *
+ * The crawl is a frontier: it starts from the blog's sitemap and its root listing, then
+ * enqueues whatever the pages it parses reveal, keeping only the kinds named by
+ * `followedKinds`. A listing is followed to its next page for as long as it keeps
+ * yielding posts that have not been seen — which is the real stop condition, since a
+ * listing past the end of a blog answers with an empty page rather than a 404.
+ *
+ * With no `dataDirPath`, a temporary directory is created and returned, so a caller can
+ * tell its user which path to hand back to resume the crawl later.
+ *
+ * Three files are produced: `pages.ndjson` holds one parsed page per line and is the
+ * record of truth, `state.json` holds the frontier so a later run can resume, and `raw/`
+ * keeps each page's HTML so a parser fix can be replayed without fetching again.
+ *
+ * Every read goes through a headless Chrome unless a `fetcher` is given: a blog host
+ * answers a plain HTTP request with a 403 and a JavaScript challenge, so nothing — not
+ * even `robots.txt` — is readable without a browser solving it first. The browser is
+ * started for the crawl and closed with it, failures included.
+ *
+ * @param baseTumblrUrl - Any URL on the blog; only its origin is used.
+ * @param options - How the crawl should behave.
+ * @returns What the crawl achieved, and every page it could not read.
+ * @throws If `baseTumblrUrl` has no readable origin, as `invalid-base-url`. A page that
+ * cannot be fetched or parsed is reported as a failure instead, and never interrupts the
+ * crawl.
+ */
+export declare function crawlBlog(baseTumblrUrl: string, options: CrawlBlogOptions): Promise<CrawlBlogResult>;
+/**
+ * Streams the pages an archive holds, in the order the crawl wrote them.
+ *
+ * Saves a consumer knowing the archive's file name and line format, and reads the dates
+ * back as `Date` rather than the strings JSON stored. A line that is not a page record is
+ * skipped rather than thrown over, so a crawl interrupted mid-write stays readable.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns Each page in turn.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export declare function readArchivePages(dataDirPath: string): AsyncGenerator<TumblrPage>;
+/**
+ * Reads every distinct post an archive holds, one entry per post id.
+ *
+ * A post is written once per page that mentioned it — a listing identifies it, its
+ * permalink describes it — so the records are merged: the permalink's version wins, and
+ * whatever it left empty is filled from the other, which is how a post keeps both its
+ * title and the `lastModifiedOn` only the sitemap knew.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns The posts, in the order their ids were first seen. Posts with no id are left
+ * out, having nothing to be merged on.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export declare function readArchivePosts(dataDirPath: string): Promise<TumblrPost[]>;

package/node/scrapers/@tumblr/index.js

@@ -0,0 +1,182 @@
+import * as Crawler from '@design-edito/tools/agnostic/misc/crawler/index.js';
+import { fillPostDetails, parsePage } from '../../../agnostic/scrapers/@tumblr/index.js';
+import { HeadlessBrowser } from '../../headless-browser/index.js';
+import { appendPageRecord, buildListingPageUrls, collectSitemapLocations, createTemporaryDataDir, discoverUrlsToFollow, ensureDataDirectories, fetchRobots, fillLastModifiedFromSitemap, mergeArchivedPosts, normalizeUrl, readCrawlState, readPageRecords, resolveDelayMs, toErrorMessage, toOriginOrNull, writeCrawlState, writeRawPage } from './utils.js';
+const DEFAULT_LIMIT = 10000;
+const DEFAULT_DELAY_MS = 250;
+/**
+ * Crawls a Tumblr blog and writes everything it reads into `dataDirPath`.
+ *
+ * The blog's sitemap is read first: it enumerates every post in a request or two, and
+ * states when each was last modified, which is the only date available on a theme that
+ * publishes none.
+ *
+ * The crawl is a frontier: it starts from the blog's sitemap and its root listing, then
+ * enqueues whatever the pages it parses reveal, keeping only the kinds named by
+ * `followedKinds`. A listing is followed to its next page for as long as it keeps
+ * yielding posts that have not been seen — which is the real stop condition, since a
+ * listing past the end of a blog answers with an empty page rather than a 404.
+ *
+ * With no `dataDirPath`, a temporary directory is created and returned, so a caller can
+ * tell its user which path to hand back to resume the crawl later.
+ *
+ * Three files are produced: `pages.ndjson` holds one parsed page per line and is the
+ * record of truth, `state.json` holds the frontier so a later run can resume, and `raw/`
+ * keeps each page's HTML so a parser fix can be replayed without fetching again.
+ *
+ * Every read goes through a headless Chrome unless a `fetcher` is given: a blog host
+ * answers a plain HTTP request with a 403 and a JavaScript challenge, so nothing — not
+ * even `robots.txt` — is readable without a browser solving it first. The browser is
+ * started for the crawl and closed with it, failures included.
+ *
+ * @param baseTumblrUrl - Any URL on the blog; only its origin is used.
+ * @param options - How the crawl should behave.
+ * @returns What the crawl achieved, and every page it could not read.
+ * @throws If `baseTumblrUrl` has no readable origin, as `invalid-base-url`. A page that
+ * cannot be fetched or parsed is reported as a failure instead, and never interrupts the
+ * crawl.
+ */
+export async function crawlBlog(baseTumblrUrl, options) {
+    const baseUrl = toOriginOrNull(baseTumblrUrl);
+    if (baseUrl === null)
+        throw new Error('invalid-base-url');
+    const dataDirPath = options.dataDirPath ?? await createTemporaryDataDir();
+    await ensureDataDirectories(dataDirPath, options.saveRawPages ?? true);
+    const browser = options.fetcher === undefined ? new HeadlessBrowser() : null;
+    const fetcher = options.fetcher ?? (async (url) => await browser?.readText(url) ?? null);
+    try {
+        return await runCrawl(baseUrl, dataDirPath, fetcher, options);
+    }
+    finally {
+        await browser?.close();
+    }
+}
+/**
+ * Streams the pages an archive holds, in the order the crawl wrote them.
+ *
+ * Saves a consumer knowing the archive's file name and line format, and reads the dates
+ * back as `Date` rather than the strings JSON stored. A line that is not a page record is
+ * skipped rather than thrown over, so a crawl interrupted mid-write stays readable.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns Each page in turn.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export async function* readArchivePages(dataDirPath) {
+    yield* readPageRecords(dataDirPath);
+}
+/**
+ * Reads every distinct post an archive holds, one entry per post id.
+ *
+ * A post is written once per page that mentioned it — a listing identifies it, its
+ * permalink describes it — so the records are merged: the permalink's version wins, and
+ * whatever it left empty is filled from the other, which is how a post keeps both its
+ * title and the `lastModifiedOn` only the sitemap knew.
+ *
+ * @param dataDirPath - Directory a crawl wrote into.
+ * @returns The posts, in the order their ids were first seen. Posts with no id are left
+ * out, having nothing to be merged on.
+ * @throws If the directory holds no `pages.ndjson`.
+ */
+export async function readArchivePosts(dataDirPath) {
+    const postsById = new Map();
+    for await (const page of readPageRecords(dataDirPath)) {
+        for (const post of page.posts) {
+            if (post.id === null)
+                continue;
+            const known = postsById.get(post.id);
+            postsById.set(post.id, known === undefined ? post : mergeArchivedPosts(known, post));
+        }
+    }
+    return Array.from(postsById.values());
+}
+async function runCrawl(baseUrl, dataDirPath, fetcher, options) {
+    const { followedKinds = ['index', 'permalink'], pages, limit = DEFAULT_LIMIT, saveRawPages = true, resume = false, useSitemap = true, onPageCrawled } = options;
+    const restoredState = resume ? await readCrawlState(dataDirPath, baseUrl) : null;
+    const robots = useSitemap ? await fetchRobots(baseUrl, fetcher) : null;
+    const sitemapLocations = useSitemap
+        ? await collectSitemapLocations(baseUrl, robots, fetcher)
+        : new Map();
+    const knownPostIds = new Set(restoredState?.knownPostIds ?? []);
+    const failures = [];
+    let blog = null;
+    let pageCount = 0;
+    const reportFailure = (url, error) => {
+        const report = {
+            url,
+            kind: null,
+            postCount: 0,
+            newPostCount: 0,
+            error
+        };
+        failures.push(report);
+        onPageCrawled?.(report);
+    };
+    const crawler = Crawler.create({
+        limit,
+        delayMs: options.delayMs ?? resolveDelayMs(robots, DEFAULT_DELAY_MS),
+        fetcher,
+        processor: async (url, pageContent, { push }) => {
+            if (pageContent === null)
+                return reportFailure(url, 'fetch-failed');
+            let parsedPage;
+            try {
+                parsedPage = await fillPostDetails(parsePage(pageContent, url));
+            }
+            catch (error) {
+                return reportFailure(url, toErrorMessage(error));
+            }
+            const page = fillLastModifiedFromSitemap(parsedPage, sitemapLocations, baseUrl);
+            pageCount = pageCount + 1;
+            blog ??= page.blog;
+            if (saveRawPages)
+                await writeRawPage(dataDirPath, url, pageContent);
+            await appendPageRecord(dataDirPath, page);
+            let newPostCount = 0;
+            for (const post of page.posts) {
+                if (post.id === null || knownPostIds.has(post.id))
+                    continue;
+                knownPostIds.add(post.id);
+                newPostCount = newPostCount + 1;
+            }
+            push(...discoverUrlsToFollow(page, followedKinds, newPostCount, baseUrl));
+            onPageCrawled?.({
+                url,
+                kind: page.kind,
+                postCount: page.posts.length,
+                newPostCount,
+                error: null
+            });
+        }
+    });
+    for (const processedUrl of restoredState?.processedUrls ?? [])
+        crawler.processed.add(processedUrl);
+    const seedUrls = pages === undefined
+        ? [baseUrl]
+        : buildListingPageUrls(baseUrl, pages.from, pages.to);
+    if (followedKinds.includes('permalink'))
+        seedUrls.push(...sitemapLocations.keys());
+    seedUrls.push(...restoredState?.waitlistUrls ?? []);
+    const pendingSeedUrls = Array.from(new Set(seedUrls
+        .map(seedUrl => normalizeUrl(seedUrl, baseUrl))
+        .filter((seedUrl) => seedUrl !== null)))
+        .filter(seedUrl => !crawler.processed.has(seedUrl));
+    const lastSeedUrl = pendingSeedUrls[pendingSeedUrls.length - 1];
+    if (lastSeedUrl !== undefined) {
+        crawler.push(...pendingSeedUrls.slice(0, -1));
+        await crawler.crawl(lastSeedUrl);
+    }
+    await writeCrawlState(dataDirPath, {
+        baseUrl,
+        processedUrls: Array.from(crawler.processed),
+        waitlistUrls: crawler.waitlist,
+        knownPostIds: Array.from(knownPostIds)
+    });
+    return {
+        dataDirPath,
+        blog,
+        pageCount,
+        postCount: knownPostIds.size,
+        failures
+    };
+}

package/node/scrapers/@tumblr/index.test.d.ts

@@ -0,0 +1 @@
+export {};

package/node/scrapers/@tumblr/index.test.js

@@ -0,0 +1,185 @@
+import { promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { crawlBlog, readArchivePages, readArchivePosts } from './index.js';
+const blogUrl = 'https://somebody.tumblr.com';
+function listingPage(canonicalPath, postIds, nextPath) {
+    const nextLink = nextPath === null ? '' : `<link rel="next" href="${nextPath}" />`;
+    const postLinks = postIds
+        .map(postId => `<a href="/post/${postId}/slug">${postId}</a>`)
+        .join('');
+    return `<!doctype html>
+    <html>
+      <head>
+        <link rel="canonical" href="${blogUrl}${canonicalPath}" />
+        <meta property="og:site_name" content="Somebody" />
+        <meta name="description" content="A blog about nothing" />
+        ${nextLink}
+      </head>
+      <body>${postLinks}</body>
+    </html>`;
+}
+function permalinkPage(postId) {
+    return `<!doctype html>
+    <html>
+      <head>
+        <link rel="canonical" href="${blogUrl}/post/${postId}/slug" />
+        <meta property="og:site_name" content="Somebody" />
+        <meta property="og:title" content="Post ${postId}" />
+        <meta property="article:tag" content="tagged" />
+      </head>
+      <body></body>
+    </html>`;
+}
+const responsesByUrl = {
+    [`${blogUrl}/robots.txt`]: [
+        'Sitemap: https://somebody.tumblr.com/sitemap.xml',
+        'User-agent: *',
+        'Crawl-delay: 0'
+    ].join('\n'),
+    [`${blogUrl}/sitemap.xml`]: `<?xml version="1.0" encoding="UTF-8"?>
+    <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+      <url>
+        <loc>${blogUrl}/post/1001/slug</loc>
+        <lastmod>2019-01-31T16:39:53+00:00</lastmod>
+      </url>
+    </urlset>`,
+    [`${blogUrl}/`]: listingPage('/', ['1001', '1002'], '/page/2'),
+    [`${blogUrl}/page/2`]: listingPage('/page/2', ['1003'], '/page/3'),
+    [`${blogUrl}/page/3`]: listingPage('/page/3', [], null),
+    [`${blogUrl}/post/1001/slug`]: permalinkPage('1001'),
+    [`${blogUrl}/post/1002/slug`]: permalinkPage('1002'),
+    [`${blogUrl}/post/1003/slug`]: permalinkPage('1003')
+};
+// The crawl reads through this rather than through a headless Chrome, which is what
+// `crawlBlog` starts when no fetcher is given.
+const fetcher = async (url) => responsesByUrl[url] ?? null;
+let dataDirPath;
+beforeEach(async () => {
+    dataDirPath = await fs.mkdtemp(path.join(os.tmpdir(), 'mf-tumblr-'));
+});
+afterEach(async () => {
+    await fs.rm(dataDirPath, { recursive: true, force: true });
+});
+describe('scrapers/@tumblr/crawlBlog', () => {
+    it('walks the listings, follows the permalinks and records every post once', async () => {
+        const result = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        expect(result.blog?.name).toBe('somebody');
+        expect(result.postCount).toBe(3);
+        expect(result.pageCount).toBe(6);
+        expect(result.failures).toEqual([]);
+    });
+    it('writes one record per parsed page, plus the state and the raw pages', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const records = await fs.readFile(path.join(dataDirPath, 'pages.ndjson'), 'utf-8');
+        const lines = records.split('\n').filter(line => line.trim() !== '');
+        expect(lines).toHaveLength(6);
+        const state = await fs.readFile(path.join(dataDirPath, 'state.json'), 'utf-8');
+        expect(JSON.parse(state).knownPostIds.sort()).toEqual(['1001', '1002', '1003']);
+        const rawFiles = await fs.readdir(path.join(dataDirPath, 'raw'));
+        expect(rawFiles).toHaveLength(6);
+    });
+    it('stops walking listings once a page yields no post it has not seen', async () => {
+        const crawledUrls = [];
+        await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            onPageCrawled: report => { crawledUrls.push(report.url); }
+        });
+        expect(crawledUrls).toContain(`${blogUrl}/page/3`);
+        expect(crawledUrls).not.toContain(`${blogUrl}/page/4`);
+    });
+    it('keeps nothing but listings when permalinks are not followed', async () => {
+        const result = await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            followedKinds: ['index'],
+            useSitemap: false
+        });
+        expect(result.pageCount).toBe(3);
+        expect(result.postCount).toBe(3);
+    });
+    it('reports a page it cannot fetch as a failure without interrupting the crawl', async () => {
+        const result = await crawlBlog(blogUrl, {
+            dataDirPath,
+            delayMs: 0,
+            fetcher,
+            pages: { from: 1, to: 2 },
+            useSitemap: false,
+            followedKinds: []
+        });
+        expect(result.failures.map(failure => failure.url)).toEqual([`${blogUrl}/page/1`]);
+        expect(result.failures[0]?.error).toBe('fetch-failed');
+        expect(result.pageCount).toBe(1);
+    });
+    it('dates a post from the sitemap, which is the only date the theme offers', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const records = await fs.readFile(path.join(dataDirPath, 'pages.ndjson'), 'utf-8');
+        const posts = records
+            .split('\n')
+            .filter(line => line.trim() !== '')
+            .flatMap(line => JSON.parse(line).posts);
+        const dated = posts.filter((post) => post.lastModifiedOn !== null);
+        expect(dated).not.toHaveLength(0);
+        expect(dated[0].lastModifiedOn).toBe('2019-01-31T16:39:53.000Z');
+        expect(dated.every((post) => post.url.includes('/post/1001/'))).toBe(true);
+    });
+    it('creates a temporary directory of its own, and says which one', async () => {
+        const result = await crawlBlog(blogUrl, { delayMs: 0, fetcher });
+        try {
+            expect(result.dataDirPath.startsWith(os.tmpdir())).toBe(true);
+            expect(path.basename(result.dataDirPath)).toMatch(/^[0-9a-f]{16}$/);
+            const records = await fs.readFile(path.join(result.dataDirPath, 'pages.ndjson'), 'utf-8');
+            expect(records.split('\n').filter(line => line.trim() !== '')).toHaveLength(6);
+        }
+        finally {
+            await fs.rm(result.dataDirPath, { recursive: true, force: true });
+        }
+    });
+    it('reports back the directory it was given', async () => {
+        const result = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        expect(result.dataDirPath).toBe(dataDirPath);
+    });
+    it('fetches nothing on a resumed crawl that already covered the blog', async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+        const resumed = await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher, resume: true });
+        expect(resumed.pageCount).toBe(0);
+        expect(resumed.postCount).toBe(3);
+    });
+});
+describe('scrapers/@tumblr/readArchivePages & readArchivePosts', () => {
+    beforeEach(async () => {
+        await crawlBlog(blogUrl, { dataDirPath, delayMs: 0, fetcher });
+    });
+    it('streams every page the crawl wrote', async () => {
+        const urls = [];
+        for await (const page of readArchivePages(dataDirPath))
+            urls.push(page.url);
+        expect(urls).toHaveLength(6);
+        expect(urls).toContain(`${blogUrl}/post/1001/slug`);
+    });
+    it('reads the dates back as dates rather than the strings JSON stored', async () => {
+        const posts = await readArchivePosts(dataDirPath);
+        const post = posts.find(candidate => candidate.id === '1001');
+        expect(post?.lastModifiedOn).toBeInstanceOf(Date);
+        expect(post?.lastModifiedOn?.toISOString()).toBe('2019-01-31T16:39:53.000Z');
+        for await (const page of readArchivePages(dataDirPath)) {
+            expect(page.collectedOn).toBeInstanceOf(Date);
+        }
+    });
+    it('merges a post written once per page into a single entry', async () => {
+        const posts = await readArchivePosts(dataDirPath);
+        expect(posts.map(post => post.id)).toEqual(['1001', '1002', '1003']);
+        const post = posts.find(candidate => candidate.id === '1001');
+        // Its title came from the permalink page, its lastModifiedOn from the sitemap.
+        expect(post?.title).toBe('Post 1001');
+        expect(post?.tags).toEqual(['tagged']);
+        expect(post?.lastModifiedOn).not.toBeNull();
+    });
+    it('throws when the directory holds no archive', async () => {
+        await expect(readArchivePosts(path.join(dataDirPath, 'nowhere'))).rejects.toThrow();
+    });
+});

package/node/scrapers/@tumblr/types.d.ts

@@ -0,0 +1,91 @@
+import type { TumblrBlog, TumblrPageKind } from '../../../agnostic/scrapers/@tumblr/types.js';
+/**
+ * How one crawl of a blog should behave.
+ *
+ * @property dataDirPath - Directory the crawl writes into, created when missing. Left
+ * out, a fresh temporary directory is made and reported back as
+ * {@link CrawlBlogResult.dataDirPath}, which is the path to hand back to resume later.
+ * @property followedKinds - Which page kinds the crawl enqueues when it meets them.
+ * Defaults to `['index', 'permalink']`: listings enumerate, permalinks extract.
+ * @property pages - Seeds `/page/from` to `/page/to` instead of discovering listings from
+ * the blog root.
+ * @property limit - Hard cap on the number of pages fetched, a safety valve rather than
+ * the stop condition. Defaults to 10000.
+ * @property delayMs - Delay between two fetches. Defaults to 250, and is raised to the
+ * blog's own `Crawl-delay` when it asks for more.
+ * @property saveRawPages - Keeps each page's raw HTML under `raw/`, so a later parser fix
+ * can be replayed without fetching anything again. Defaults to `true`.
+ * @property resume - Picks up where a previous crawl of the same directory stopped.
+ * @property useSitemap - Reads `robots.txt` and `sitemap.xml` to enumerate posts without
+ * walking listings. Defaults to `true`.
+ * @property onPageCrawled - Called once per page, successes and failures alike.
+ * @property fetcher - Reads one URL as text, or answers `null` when it could not. Left
+ * out, a headless Chrome is started for the crawl and closed with it, which is what a
+ * Tumblr blog host requires: it answers a plain request with a 403 and a JavaScript
+ * challenge.
+ */
+export type CrawlBlogOptions = {
+    dataDirPath?: string;
+    followedKinds?: TumblrPageKind[];
+    pages?: {
+        from: number;
+        to: number;
+    };
+    limit?: number;
+    delayMs?: number | (() => number);
+    saveRawPages?: boolean;
+    resume?: boolean;
+    useSitemap?: boolean;
+    onPageCrawled?: (report: CrawledPageReport) => void;
+    fetcher?: (url: string) => Promise<string | null>;
+};
+/**
+ * What became of one crawled page.
+ *
+ * @property url - The URL that was fetched.
+ * @property kind - What the page turned out to be, `null` when it could not be read.
+ * @property postCount - Posts the page carried.
+ * @property newPostCount - Posts the crawl had not seen before, which is what tells a
+ * listing worth following from one that only repeats itself.
+ * @property error - Why the page was skipped, `null` on success.
+ */
+export type CrawledPageReport = {
+    url: string;
+    kind: TumblrPageKind | null;
+    postCount: number;
+    newPostCount: number;
+    error: string | null;
+};
+/**
+ * What one crawl achieved.
+ *
+ * @property dataDirPath - Where the crawl wrote, whether it was given or created. This
+ * is what a later run needs to resume.
+ * @property blog - The blog as the first readable page described it, `null` when no page
+ * could be read.
+ * @property pageCount - Pages successfully parsed.
+ * @property postCount - Distinct posts seen across them.
+ * @property failures - Every page that could not be fetched or parsed.
+ */
+export type CrawlBlogResult = {
+    dataDirPath: string;
+    blog: TumblrBlog | null;
+    pageCount: number;
+    postCount: number;
+    failures: CrawledPageReport[];
+};
+/**
+ * A crawl's progress, as `state.json` holds it between two runs.
+ *
+ * @property baseUrl - Origin the crawl was run against.
+ * @property processedUrls - URLs already fetched, which a resumed crawl will not fetch
+ * again.
+ * @property waitlistUrls - URLs still owed a fetch when the crawl stopped.
+ * @property knownPostIds - Every post id seen so far.
+ */
+export type CrawlState = {
+    baseUrl: string;
+    processedUrls: string[];
+    waitlistUrls: string[];
+    knownPostIds: string[];
+};

package/node/scrapers/@tumblr/types.js

@@ -0,0 +1 @@
+export {};

package/node/scrapers/@tumblr/utils.d.ts

@@ -0,0 +1,23 @@
+import type { TumblrPage, TumblrPageKind, TumblrPost, TumblrRobots } from '../../../agnostic/scrapers/@tumblr/types.js';
+import type { CrawlState } from './types.js';
+type PageFetcher = (url: string) => Promise<string | null>;
+export declare function toOriginOrNull(rawUrl: string): string | null;
+export declare function normalizeUrl(rawUrl: string, baseUrl: string | null): string | null;
+export declare function isListingKind(kind: TumblrPageKind): boolean;
+export declare function buildListingPageUrls(baseUrl: string, from: number, to: number): string[];
+export declare function toErrorMessage(error: unknown): string;
+export declare function fetchRobots(baseUrl: string, fetcher: PageFetcher): Promise<TumblrRobots | null>;
+export declare function collectSitemapLocations(baseUrl: string, robots: TumblrRobots | null, fetcher: PageFetcher): Promise<Map<string, Date | null>>;
+export declare function fillLastModifiedFromSitemap(page: TumblrPage, lastModifiedByUrl: Map<string, Date | null>, baseUrl: string): TumblrPage;
+export declare function createTemporaryDataDir(): Promise<string>;
+export declare function ensureDataDirectories(dataDirPath: string, saveRawPages: boolean): Promise<void>;
+export declare function appendPageRecord(dataDirPath: string, page: TumblrPage): Promise<void>;
+export declare function writeRawPage(dataDirPath: string, url: string, pageContent: string): Promise<void>;
+export declare function writeCrawlState(dataDirPath: string, state: CrawlState): Promise<void>;
+export declare function readCrawlState(dataDirPath: string, baseUrl: string): Promise<CrawlState | null>;
+export declare function rebuildCrawlStateFromRecords(dataDirPath: string, baseUrl: string): Promise<CrawlState | null>;
+export declare function readPageRecords(dataDirPath: string): AsyncGenerator<TumblrPage>;
+export declare function mergeArchivedPosts(existing: TumblrPost, incoming: TumblrPost): TumblrPost;
+export declare function resolveDelayMs(robots: TumblrRobots | null, defaultDelayMs: number): number;
+export declare function discoverUrlsToFollow(page: TumblrPage, followedKinds: TumblrPageKind[], newPostCount: number, baseUrl: string): string[];
+export {};