@mxfb/tools 1.0.0 → 1.0.3

package/agnostic/scrapers/@tumblr/index.test.js

@@ -0,0 +1,319 @@
+import { describe, it, expect, vi, afterEach } from 'vitest';
+import { fetchPage, fillPostDetails, parsePage, parseRobots, parseSitemap } from './index.js';
+const blogUrl = 'https://somebody.tumblr.com';
+const listingPage = `<!doctype html>
+<html>
+  <head>
+    <link rel="canonical" href="https://somebody.tumblr.com/page/2" />
+    <link rel="icon" href="/avatar.png" />
+    <meta property="og:site_name" content="Somebody's blog" />
+    <meta name="description" content="A blog about nothing" />
+  </head>
+  <body>
+    <nav><a href="/about">About</a></nav>
+    <a href="/post/1001/first">first</a>
+    <a href="/post/1001/first#notes">first again</a>
+    <a href="/post/1002/second">second</a>
+    <a href="https://elsewhere.tumblr.com/post/9999/foreign">foreign</a>
+    <a href="/page/1">previous</a>
+    <a href="/page/3">next</a>
+  </body>
+</html>`;
+const permalinkPage = `<!doctype html>
+<html>
+  <head>
+    <link rel="canonical" href="https://somebody.tumblr.com/post/1001/first" />
+    <meta property="og:site_name" content="Somebody's blog" />
+    <meta property="og:type" content="video.other" />
+    <meta property="og:title" content="A video post" />
+    <meta property="og:description" content="What this one post is about" />
+    <meta property="og:image" content="https://media.tumblr.com/poster.jpg" />
+    <meta property="og:image:width" content="640" />
+    <meta property="og:image:height" content="480" />
+    <meta property="article:published_time" content="2024-03-07T09:30:00Z" />
+    <meta property="article:tag" content="cinema" />
+    <meta property="article:tag" content="super 8" />
+  </head>
+  <body></body>
+</html>`;
+const listingPageWithBlocks = `<!doctype html>
+<html>
+  <head>
+    <link rel="canonical" href="https://somebody.tumblr.com/" />
+    <meta property="og:site_name" content="Somebody's blog" />
+  </head>
+  <body>
+    <header><a href="/post/1001/first">featured</a></header>
+    <main>
+      <div id="post-1001">
+        <h2><a href="/post/1001/first">First</a></h2>
+        <p>Body of the first</p>
+      </div>
+      <li>
+        <h2><a href="/post/1002/second">Second</a></h2>
+        <p>Body of the second</p>
+      </li>
+    </main>
+    <footer><a href="/archive">Archive</a></footer>
+  </body>
+</html>`;
+const permalinkPageWithBlock = `<!doctype html>
+<html>
+  <head>
+    <link rel="canonical" href="https://somebody.tumblr.com/post/1001/first" />
+    <meta property="og:site_name" content="Somebody's blog" />
+  </head>
+  <body>
+    <nav><a href="/post/1002/second">Next post</a></nav>
+    <article>
+      <h1>First</h1>
+      <p>Body of the first</p>
+      <a href="/post/1001/first">12 notes</a>
+    </article>
+  </body>
+</html>`;
+describe('scrapers/@tumblr/parsePage', () => {
+    it('throws when the content is not a Tumblr page', () => {
+        expect(() => parsePage('<html></html>', 'https://somebody.tumblr.com/'))
+            .toThrow('not-a-tumblr-page');
+    });
+    it('reads the blog off a listing page', () => {
+        const { blog } = parsePage(listingPage, `${blogUrl}/page/2`);
+        expect(blog.name).toBe('somebody');
+        expect(blog.title).toBe("Somebody's blog");
+        expect(blog.description).toBe('A blog about nothing');
+        expect(blog.baseUrl).toBe('https://somebody.tumblr.com');
+        expect(blog.avatarUrl).toBe('https://somebody.tumblr.com/avatar.png');
+        expect(blog.navLinks).toEqual([
+            { label: 'About', url: 'https://somebody.tumblr.com/about' }
+        ]);
+    });
+    it('identifies every post of a listing page once, and only its own', () => {
+        const { kind, posts } = parsePage(listingPage, `${blogUrl}/page/2`);
+        expect(kind).toBe('index');
+        expect(posts.map(post => post.id)).toEqual(['1001', '1002']);
+        expect(posts.map(post => post.url)).toEqual([
+            'https://somebody.tumblr.com/post/1001/first',
+            'https://somebody.tumblr.com/post/1002/second'
+        ]);
+        expect(posts.every(post => post.type === 'unknown')).toBe(true);
+    });
+    it('walks pagination from the page links it finds', () => {
+        const { pagination } = parsePage(listingPage, `${blogUrl}/page/2`);
+        expect(pagination.currentPageNumber).toBe(2);
+        expect(pagination.previousPageUrl).toBe('https://somebody.tumblr.com/page/1');
+        expect(pagination.nextPageUrl).toBe('https://somebody.tumblr.com/page/3');
+    });
+    it('reads the single post of a permalink page', () => {
+        const { kind, posts } = parsePage(permalinkPage, `${blogUrl}/post/1001/first`);
+        expect(kind).toBe('permalink');
+        expect(posts).toHaveLength(1);
+        const [post] = posts;
+        expect(post?.id).toBe('1001');
+        expect(post?.type).toBe('video');
+        expect(post?.title).toBe('A video post');
+        expect(post?.tags).toEqual(['cinema', 'super 8']);
+        expect(post?.publishedOn?.toISOString()).toBe('2024-03-07T09:30:00.000Z');
+        expect(post?.media).toEqual([{
+                kind: 'image',
+                url: 'https://media.tumblr.com/poster.jpg',
+                widthPx: 640,
+                heightPx: 480,
+                alternativeText: null
+            }]);
+    });
+    it('keeps the raw markup of each block a listing page carries', () => {
+        const { posts } = parsePage(listingPageWithBlocks, blogUrl);
+        const [first, second] = posts;
+        expect(first?.rawHtml).toContain('Body of the first');
+        expect(first?.rawHtml).not.toContain('Body of the second');
+        expect(first?.rawHtml?.startsWith('<div id="post-1001">')).toBe(true);
+        expect(second?.rawHtml).toContain('Body of the second');
+        expect(second?.rawHtml?.startsWith('<li>')).toBe(true);
+        expect(posts.map(post => post.rawHtmlSource)).toEqual(['listing', 'listing']);
+    });
+    it('leaves contentHtml to a blog parser rather than guessing a body', () => {
+        const { posts } = parsePage(listingPageWithBlocks, blogUrl);
+        expect(posts.every(post => post.contentHtml === null)).toBe(true);
+    });
+    it('keeps the raw markup of the post a permalink page is about', () => {
+        const { posts } = parsePage(permalinkPageWithBlock, `${blogUrl}/post/1001/first`);
+        const [post] = posts;
+        expect(post?.rawHtml?.startsWith('<article>')).toBe(true);
+        expect(post?.rawHtml).toContain('Body of the first');
+        expect(post?.rawHtml).not.toContain('Next post');
+        expect(post?.rawHtmlSource).toBe('permalink');
+    });
+    it('reports no raw markup when no block could be located', () => {
+        const { posts } = parsePage(permalinkPage, `${blogUrl}/post/1001/first`);
+        expect(posts[0]?.rawHtml).toBeNull();
+        expect(posts[0]?.rawHtmlSource).toBeNull();
+    });
+    it('never attributes a post description to the blog', () => {
+        const { blog } = parsePage(permalinkPage, `${blogUrl}/post/1001/first`);
+        expect(blog.description).toBeNull();
+    });
+});
+describe('scrapers/@tumblr/fillPostDetails', () => {
+    const loadParser = async (blogName) => blogName !== 'somebody'
+        ? null
+        : {
+            parsePostDom: postElement => ({
+                detail: {
+                    heading: postElement.querySelector('h2')?.textContent?.trim() ?? null,
+                    blockId: postElement.id
+                },
+                title: postElement.querySelector('h2')?.textContent?.trim() ?? null
+            })
+        };
+    it('fills detail with what the blog parser read out of each block', async () => {
+        const page = await fillPostDetails(parsePage(listingPageWithBlocks, blogUrl), loadParser);
+        expect(page.posts.map(post => post.detail)).toEqual([
+            { heading: 'First', blockId: 'post-1001' },
+            { heading: 'Second', blockId: '' }
+        ]);
+    });
+    it('lets the parser complete the standard fields the generic pass left empty', async () => {
+        const page = await fillPostDetails(parsePage(listingPageWithBlocks, blogUrl), loadParser);
+        expect(page.posts.map(post => post.title)).toEqual(['First', 'Second']);
+    });
+    it('leaves a page alone when no parser is registered for its blog', async () => {
+        const parsed = parsePage(listingPageWithBlocks, 'https://nobody.tumblr.com/');
+        const page = await fillPostDetails(parsed, loadParser);
+        expect(page).toBe(parsed);
+        expect(page.posts.every(post => post.detail === null)).toBe(true);
+    });
+    it('reports no detail for a post whose block was never located', async () => {
+        const page = await fillPostDetails(parsePage(permalinkPage, `${blogUrl}/post/1001/first`), loadParser);
+        expect(page.posts[0]?.detail).toBeNull();
+    });
+    it('carries no parser for any blog yet, so detail stays null by default', async () => {
+        const page = await fillPostDetails(parsePage(listingPageWithBlocks, blogUrl));
+        expect(page.posts.every(post => post.detail === null)).toBe(true);
+    });
+});
+describe('scrapers/@tumblr/fetchPage', () => {
+    const originalFetch = global.fetch;
+    afterEach(() => {
+        global.fetch = originalFetch;
+    });
+    it('returns the response body as text', async () => {
+        global.fetch = vi.fn(async () => ({
+            ok: true,
+            text: async () => '<html>ok</html>'
+        }));
+        await expect(fetchPage('https://somebody.tumblr.com')).resolves.toBe('<html>ok</html>');
+    });
+    it('throws with the status when the response is not a success', async () => {
+        global.fetch = vi.fn(async () => ({
+            ok: false,
+            status: 404,
+            text: async () => ''
+        }));
+        await expect(fetchPage('https://somebody.tumblr.com/page/999'))
+            .rejects
+            .toThrow('fetch-failed-404');
+    });
+});
+describe('scrapers/@tumblr/parseSitemap', () => {
+    it('reads a sitemap index', () => {
+        const sitemap = parseSitemap(`<?xml version="1.0" encoding="UTF-8"?>
+      <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <sitemap><loc>https://somebody.tumblr.com/sitemap-posts1.xml</loc></sitemap>
+        <sitemap><loc>https://somebody.tumblr.com/sitemap-posts2.xml</loc></sitemap>
+      </sitemapindex>`);
+        expect(sitemap.kind).toBe('index');
+        expect(sitemap.childSitemapUrls).toEqual([
+            'https://somebody.tumblr.com/sitemap-posts1.xml',
+            'https://somebody.tumblr.com/sitemap-posts2.xml'
+        ]);
+        expect(sitemap.locations).toEqual([]);
+    });
+    it('reads a location set', () => {
+        const sitemap = parseSitemap(`<?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://somebody.tumblr.com/post/1001/first</loc>
+          <changefreq>hourly</changefreq>
+          <lastmod>2019-01-31T16:39:53+00:00</lastmod>
+        </url>
+        <url><loc>https://somebody.tumblr.com/post/1002/second</loc></url>
+      </urlset>`);
+        expect(sitemap.kind).toBe('urlset');
+        expect(sitemap.childSitemapUrls).toEqual([]);
+        expect(sitemap.locations.map(location => location.url)).toEqual([
+            'https://somebody.tumblr.com/post/1001/first',
+            'https://somebody.tumblr.com/post/1002/second'
+        ]);
+    });
+    it('keeps the date a location was last modified, when it is stated', () => {
+        const sitemap = parseSitemap(`<?xml version="1.0" encoding="UTF-8"?>
+      <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+        <url>
+          <loc>https://somebody.tumblr.com/post/1001/first</loc>
+          <lastmod>2019-01-31T16:39:53+00:00</lastmod>
+        </url>
+        <url><loc>https://somebody.tumblr.com/post/1002/second</loc></url>
+      </urlset>`);
+        expect(sitemap.locations[0]?.lastModifiedOn?.toISOString()).toBe('2019-01-31T16:39:53.000Z');
+        expect(sitemap.locations[1]?.lastModifiedOn).toBeNull();
+    });
+    it('throws when the document is not a sitemap', () => {
+        expect(() => parseSitemap('<html><body>nope</body></html>')).toThrow('not-a-sitemap');
+    });
+});
+describe('scrapers/@tumblr/parseRobots', () => {
+    it('keeps the sitemaps and the rules stated for every crawler', () => {
+        const robots = parseRobots([
+            '# a comment',
+            'Sitemap: https://somebody.tumblr.com/sitemap.xml',
+            'User-agent: *',
+            'Disallow: /search',
+            'Allow: /search/public',
+            'Crawl-delay: 2'
+        ].join('\n'));
+        expect(robots.sitemapUrls).toEqual(['https://somebody.tumblr.com/sitemap.xml']);
+        expect(robots.disallowedPaths).toEqual(['/search']);
+        expect(robots.allowedPaths).toEqual(['/search/public']);
+        expect(robots.crawlDelaySeconds).toBe(2);
+    });
+    it('ignores the rules of a named crawler', () => {
+        const robots = parseRobots([
+            'User-agent: SomeBot',
+            'Disallow: /',
+            'User-agent: *',
+            'Disallow: /archive'
+        ].join('\n'));
+        expect(robots.disallowedPaths).toEqual(['/archive']);
+    });
+    it('returns empty lists for an empty file', () => {
+        const robots = parseRobots('');
+        expect(robots.sitemapUrls).toEqual([]);
+        expect(robots.disallowedPaths).toEqual([]);
+        expect(robots.allowedPaths).toEqual([]);
+        expect(robots.crawlDelaySeconds).toBeNull();
+    });
+});
+describe('scrapers/@tumblr/parsePage, against a canonical that names another host', () => {
+    const modernPage = `<!doctype html>
+    <html>
+      <head>
+        <link rel="canonical" href="https://www.tumblr.com/somebody" />
+        <meta property="og:site_name" content="Somebody's blog" />
+      </head>
+      <body><a href="/post/1001/first">first</a></body>
+    </html>`;
+    it('locates the page from the URL it was fetched from, not from the canonical', () => {
+        const page = parsePage(modernPage, `${blogUrl}/page/2`);
+        expect(page.url).toBe(`${blogUrl}/page/2`);
+        expect(page.canonicalUrl).toBe('https://www.tumblr.com/somebody');
+        expect(page.kind).toBe('index');
+        expect(page.blog.baseUrl).toBe(blogUrl);
+        expect(page.pagination.currentPageNumber).toBe(2);
+        expect(page.posts.map(post => post.id)).toEqual(['1001']);
+    });
+    it('reads the blog name from the canonical path rather than the hostname', () => {
+        const page = parsePage(modernPage, `${blogUrl}/page/2`);
+        expect(page.blog.name).toBe('somebody');
+    });
+});

package/agnostic/scrapers/@tumblr/types.d.ts

@@ -0,0 +1,238 @@
+/** Tumblr's own post taxonomy, as the platform names it. */
+export declare const tumblrPostTypes: readonly ["text", "photo", "quote", "link", "chat", "audio", "video", "answer", "unknown"];
+/** The kind of content a post carries. */
+export type TumblrPostType = typeof tumblrPostTypes[number];
+/** The kinds of page a blog serves, as told apart by their path. */
+export declare const tumblrPageKinds: readonly ["index", "permalink", "archive", "tagged", "custom", "unknown"];
+/** What a parsed page turned out to be. */
+export type TumblrPageKind = typeof tumblrPageKinds[number];
+/** The media families a post can embed. */
+export declare const tumblrMediaKinds: readonly ["image", "video", "audio", "iframe"];
+/** Which media family a single item belongs to. */
+export type TumblrMediaKind = typeof tumblrMediaKinds[number];
+/** The kinds of page a post's raw markup can have been read off. */
+export declare const tumblrRawHtmlSources: readonly ["listing", "permalink"];
+/** Which kind of page a post's raw markup came from. */
+export type TumblrRawHtmlSource = typeof tumblrRawHtmlSources[number];
+/**
+ * One media item attached to a post.
+ *
+ * @property kind - Which media family the item belongs to.
+ * @property url - Absolute URL of the media file.
+ * @property widthPx - Intrinsic width, when the page declares one.
+ * @property heightPx - Intrinsic height, when the page declares one.
+ * @property alternativeText - Textual alternative, when the page declares one.
+ */
+export type TumblrMedia = {
+    kind: TumblrMediaKind;
+    url: string;
+    widthPx: number | null;
+    heightPx: number | null;
+    alternativeText: string | null;
+};
+/**
+ * A link a theme exposes in its navigation.
+ *
+ * @property label - The link's visible text, whitespace-normalized.
+ * @property url - Absolute URL the link points at.
+ */
+export type TumblrNavLink = {
+    label: string;
+    url: string;
+};
+/**
+ * A single post, as much of it as the parsed page revealed.
+ *
+ * Every field but `type`, `tags` and `media` is `null` when the page did not carry it —
+ * which is the common case on an index page, where posts are only identified.
+ *
+ * @property id - The numeric post id, as a string.
+ * @property url - Absolute permalink.
+ * @property type - The post's content type, `'unknown'` when nothing stated it.
+ * @property publishedOn - Publication date.
+ * @property title - The post's own title, not the blog's.
+ * @property contentHtml - The post body as markup, which only a blog's own parser fills:
+ * telling a body apart from the theme chrome around it is not something that can be done
+ * across themes.
+ * @property rawHtml - The post block's markup, exactly as the page carried it, attributes
+ * and theme wrappers included. It is the raw material a parser is handed later, and URLs
+ * inside it stay relative to the page it came from.
+ * @property rawHtmlSource - Which kind of page {@link TumblrPost.rawHtml} was read off.
+ * A listing block is whatever the theme chose to show there, so it may well be a teaser of
+ * the post a permalink page carries in full.
+ * @property detail - What this blog's own parser read out of `rawHtml`, `null` when no
+ * parser is registered for the blog or when there was no block to read.
+ * @property tags - Tags attached to the post, empty when none were found.
+ * @property media - Media items attached to the post, empty when none were found.
+ * @property notesCount - Number of notes (likes and reblogs combined).
+ * @property reblogOfUrl - Permalink of the post this one reblogs.
+ * @property lastModifiedOn - When the post last changed, which a sitemap states for every
+ * post of a blog. On a post that was never edited this is also its publication date, but
+ * the two are not the same claim and are kept apart.
+ */
+export type TumblrPost = {
+    id: string | null;
+    url: string | null;
+    type: TumblrPostType;
+    publishedOn: Date | null;
+    title: string | null;
+    contentHtml: string | null;
+    rawHtml: string | null;
+    rawHtmlSource: TumblrRawHtmlSource | null;
+    detail: TumblrPostDetail | null;
+    tags: string[];
+    media: TumblrMedia[];
+    notesCount: number | null;
+    reblogOfUrl: string | null;
+    lastModifiedOn: Date | null;
+};
+/**
+ * What a blog's own parser read out of one post block.
+ *
+ * Deliberately open: a theme states things no other theme does — a track's artist, a
+ * release year, a label — and inventing a field on {@link TumblrPost} for each of them
+ * would make the shape mean less with every blog added. What a given blog puts in here is
+ * documented by that blog's own parser.
+ */
+export type TumblrPostDetail = Record<string, unknown>;
+/**
+ * What a blog's parser read out of one post block.
+ *
+ * Besides its own `detail`, a parser may complete the standard fields it reads better
+ * than the generic pass — which is most of them, a theme being the only thing that knows
+ * where it put a date or a title. A field left out, or `null`, keeps whatever the generic
+ * pass found.
+ *
+ * @property detail - What this theme states that no other does.
+ * @property title - The post's own title.
+ * @property contentHtml - The post body as markup, the one field the generic pass can
+ * never fill.
+ * @property publishedOn - Publication date.
+ * @property tags - Tags attached to the post. An empty list keeps the generic pass's.
+ * @property media - Media items attached to the post. An empty list keeps the generic
+ * pass's.
+ */
+export type TumblrParsedPost = {
+    detail: TumblrPostDetail;
+    title?: string | null;
+    contentHtml?: string | null;
+    publishedOn?: Date | null;
+    tags?: string[];
+    media?: TumblrMedia[];
+};
+/**
+ * A single blog's parser, the one thing a theme-specific module must provide.
+ *
+ * @property parsePostDom - Reads one post block, as located by the generic pass.
+ */
+export type TumblrBlogParser = {
+    parsePostDom: (postElement: Element) => TumblrParsedPost;
+};
+/**
+ * Finds the parser for a blog, by the name `extractBlog` read off the page.
+ *
+ * @returns That blog's parser, or `null` when none is registered — which is the common
+ * case, and means a post keeps only what the generic pass found.
+ */
+export type TumblrBlogParserLoader = (blogName: string) => Promise<TumblrBlogParser | null>;
+/**
+ * The blog a page belongs to, as much of it as the page revealed.
+ *
+ * @property name - The blog identifier — the `staff` of `staff.tumblr.com`, `null` on a
+ * custom domain.
+ * @property title - The blog's display title.
+ * @property description - The blog's own description, never a post's.
+ * @property baseUrl - Origin the blog is served from.
+ * @property avatarUrl - Best guess at the blog avatar, read from the page's icon links.
+ * @property navLinks - Links found in the page's navigation, empty when the theme
+ * exposes none semantically.
+ */
+export type TumblrBlog = {
+    name: string | null;
+    title: string | null;
+    description: string | null;
+    baseUrl: string | null;
+    avatarUrl: string | null;
+    navLinks: TumblrNavLink[];
+};
+/**
+ * Where a page sits in the blog's pagination, and where to go next.
+ *
+ * @property currentPageNumber - 1 for the blog root, the `N` of `/page/N`, `null`
+ * elsewhere.
+ * @property nextPageUrl - Absolute URL of the following page.
+ * @property previousPageUrl - Absolute URL of the preceding page.
+ */
+export type TumblrPagination = {
+    currentPageNumber: number | null;
+    nextPageUrl: string | null;
+    previousPageUrl: string | null;
+};
+/**
+ * Everything one Tumblr page yielded.
+ *
+ * @property kind - What the page turned out to be.
+ * @property url - The URL the page was fetched from, which is what locates it. Modern
+ * Tumblr themes declare the same canonical URL on every listing page, so this is the only
+ * field that tells two of them apart.
+ * @property canonicalUrl - The URL the page declares for itself, which often points at
+ * `www.tumblr.com` rather than the blog's own host.
+ * @property blog - The blog the page belongs to.
+ * @property posts - One post on a permalink page, every post identified on an index
+ * page, empty when none were found.
+ * @property pagination - Where the page sits among its siblings.
+ * @property collectedOn - When the parse ran.
+ */
+export type TumblrPage = {
+    kind: TumblrPageKind;
+    url: string;
+    canonicalUrl: string | null;
+    blog: TumblrBlog;
+    posts: TumblrPost[];
+    pagination: TumblrPagination;
+    collectedOn: Date;
+};
+/** The two shapes a sitemap document can take. */
+export declare const tumblrSitemapKinds: readonly ["index", "urlset"];
+/** Whether a sitemap lists other sitemaps or actual locations. */
+export type TumblrSitemapKind = typeof tumblrSitemapKinds[number];
+/**
+ * One location a sitemap lists.
+ *
+ * @property url - The location itself.
+ * @property lastModifiedOn - When it last changed, when the sitemap says so.
+ */
+export type TumblrSitemapLocation = {
+    url: string;
+    lastModifiedOn: Date | null;
+};
+/**
+ * A parsed sitemap document.
+ *
+ * @property kind - `'index'` when the document lists other sitemaps, `'urlset'` when it
+ * lists locations.
+ * @property childSitemapUrls - Sitemaps this one points at, empty on a `'urlset'`.
+ * @property locations - Locations this one lists, empty on an `'index'`.
+ */
+export type TumblrSitemap = {
+    kind: TumblrSitemapKind;
+    childSitemapUrls: string[];
+    locations: TumblrSitemapLocation[];
+};
+/**
+ * The directives a blog's `robots.txt` states for every crawler.
+ *
+ * Only the `User-agent: *` group is read — a blog singling out a named crawler is not
+ * something this parser reports on.
+ *
+ * @property sitemapUrls - Every `Sitemap:` directive, which are group-independent.
+ * @property disallowedPaths - Paths closed to every crawler.
+ * @property allowedPaths - Paths explicitly reopened inside a disallowed one.
+ * @property crawlDelaySeconds - Requested delay between two requests, when stated.
+ */
+export type TumblrRobots = {
+    sitemapUrls: string[];
+    disallowedPaths: string[];
+    allowedPaths: string[];
+    crawlDelaySeconds: number | null;
+};

package/agnostic/scrapers/@tumblr/types.js

@@ -0,0 +1,38 @@
+/** Tumblr's own post taxonomy, as the platform names it. */
+export const tumblrPostTypes = [
+    'text',
+    'photo',
+    'quote',
+    'link',
+    'chat',
+    'audio',
+    'video',
+    'answer',
+    'unknown'
+];
+/** The kinds of page a blog serves, as told apart by their path. */
+export const tumblrPageKinds = [
+    'index',
+    'permalink',
+    'archive',
+    'tagged',
+    'custom',
+    'unknown'
+];
+/** The media families a post can embed. */
+export const tumblrMediaKinds = [
+    'image',
+    'video',
+    'audio',
+    'iframe'
+];
+/** The kinds of page a post's raw markup can have been read off. */
+export const tumblrRawHtmlSources = [
+    'listing',
+    'permalink'
+];
+/** The two shapes a sitemap document can take. */
+export const tumblrSitemapKinds = [
+    'index',
+    'urlset'
+];

package/agnostic/scrapers/@tumblr/utils.d.ts

@@ -0,0 +1,12 @@
+import type { TumblrBlog, TumblrPageKind, TumblrPagination, TumblrPost, TumblrSitemapLocation } from './types.js';
+export declare function toIntegerOrNull(raw: string | null): number | null;
+export declare function readSitemapChildUrls(document: Document): string[];
+export declare function readSitemapLocations(document: Document): TumblrSitemapLocation[];
+export declare function toOrigin(url: string | null): string | null;
+export declare function readMetaProperty(document: Document, property: string): string | null;
+export declare function extractCanonicalUrl(document: Document): string | null;
+export declare function extractPageKind(pageUrl: string): TumblrPageKind;
+export declare function extractBlog(document: Document, pageUrl: string, canonicalUrl: string | null, pageKind: TumblrPageKind): TumblrBlog;
+export declare function extractPostFromPermalinkPage(document: Document, pageUrl: string): TumblrPost;
+export declare function extractPostsFromListingPage(document: Document, baseUrl: string | null): TumblrPost[];
+export declare function extractPagination(document: Document, pageUrl: string, baseUrl: string | null): TumblrPagination;