@mxfb/tools 0.3.0 → 1.0.2

package/agnostic/scrapers/@tumblr/blogs/delabienbonnemusique/index.test.js

@@ -0,0 +1,107 @@
+import { describe, it, expect } from 'vitest';
+import { JSDOM } from 'jsdom';
+import { parser } from './index.js';
+// Trimmed from a real block of the blog, keeping every part the parser reads.
+const videoPostBlock = `<article id="post-182451761040" class="post type-video">
+  <div class="post-panel">
+    <div class="media loading">
+      <iframe width="500" height="281" id="youtube_iframe"
+        src="https://www.youtube.com/embed/7UWhdL76FKI?feature=oembed"
+        title="Murcof x Vanessa Wagner - Gnossienne No. 3 (Album Version)"></iframe>
+    </div>
+    <div class="copy"><p>Murcof &amp; Vanessa Wagner — Gnossienne No. 3<br></p></div>
+    <div class="meta">
+      <ul class="meta-list">
+        <li class="date">
+          <a href="https://delabienbonnemusique.tumblr.com/post/182451761040/x"
+            title="Jeu. janvier 31, 2019 @ 5:39 pm"><span class="icon"></span>il y a 7 ans</a>
+        </li>
+      </ul>
+      <div class="share"><div class="short-link">
+        <input type="text" readonly value="https://tmblr.co/ZknzVy2fw_E_G">
+      </div></div>
+    </div>
+  </div>
+</article>`;
+const audioPostBlock = `<article id="post-170586407310" class="post type-audio">
+  <div class="post-panel">
+    <div class="copy"><p>Salvatore Adamo — Tombe la neige</p></div>
+    <div class="copy source">
+      <p>Source&nbsp;: <a href="https://open.spotify.com/track/6NVx24kOU2B8IZ0HsY2fUz">Spotify</a></p>
+    </div>
+    <div class="meta"><ul class="meta-list"><li class="date">
+      <a href="#" title="Mar. février 6, 2018 @ 10:38 pm">il y a 8 ans</a>
+    </li></ul></div>
+  </div>
+</article>`;
+function parse(block) {
+    const { document } = new JSDOM(`<body>${block}</body>`).window;
+    const postElement = document.body.firstElementChild;
+    if (postElement === null)
+        throw new Error('no-block');
+    return parser.parsePostDom(postElement);
+}
+describe('scrapers/@tumblr/blogs/delabienbonnemusique', () => {
+    it('keeps the title line whole rather than guessing an artist out of it', () => {
+        const { title } = parse(videoPostBlock);
+        expect(title).toBe('Murcof & Vanessa Wagner — Gnossienne No. 3');
+    });
+    it('reads the only publication date the blog states, as UTC', () => {
+        const { publishedOn, detail } = parse(videoPostBlock);
+        expect(publishedOn?.toISOString()).toBe('2019-01-31T17:39:00.000Z');
+        expect(detail.publishedLabel).toBe('Jeu. janvier 31, 2019 @ 5:39 pm');
+    });
+    it('reads a morning hour without pushing it into the afternoon', () => {
+        const { publishedOn } = parse(videoPostBlock.replace('5:39 pm', '11:42 am'));
+        expect(publishedOn?.toISOString()).toBe('2019-01-31T11:42:00.000Z');
+    });
+    it('reads midnight and noon the way the theme means them', () => {
+        expect(parse(videoPostBlock.replace('5:39 pm', '12:30 am')).publishedOn?.toISOString())
+            .toBe('2019-01-31T00:30:00.000Z');
+        expect(parse(videoPostBlock.replace('5:39 pm', '12:30 pm')).publishedOn?.toISOString())
+            .toBe('2019-01-31T12:30:00.000Z');
+    });
+    it('keeps the post body, the embed, and the embed title a dead video leaves behind', () => {
+        const { contentHtml, media, detail } = parse(videoPostBlock);
+        expect(contentHtml).toBe('<p>Murcof &amp; Vanessa Wagner — Gnossienne No. 3<br></p>');
+        expect(media).toEqual([{
+                kind: 'iframe',
+                url: 'https://www.youtube.com/embed/7UWhdL76FKI?feature=oembed',
+                widthPx: 500,
+                heightPx: 281,
+                alternativeText: 'Murcof x Vanessa Wagner - Gnossienne No. 3 (Album Version)'
+            }]);
+        expect(detail.postType).toBe('video');
+        expect(detail.embedTitle).toBe('Murcof x Vanessa Wagner - Gnossienne No. 3 (Album Version)');
+    });
+    it('takes the embed as the source of a video post', () => {
+        const { detail } = parse(videoPostBlock);
+        expect(detail.sourceUrl).toBe('https://www.youtube.com/embed/7UWhdL76FKI?feature=oembed');
+        expect(detail.sourceUrls).toEqual([
+            'https://www.youtube.com/embed/7UWhdL76FKI?feature=oembed'
+        ]);
+    });
+    it('never takes a share button or a permalink for a source', () => {
+        const { detail } = parse(videoPostBlock);
+        expect(JSON.stringify(detail.sourceUrls)).not.toContain('tmblr.co');
+        expect(JSON.stringify(detail.sourceUrls)).not.toContain('/post/182451761040');
+    });
+    it('reads the source link of a post that has one, and never reads it as the body', () => {
+        const { detail, contentHtml, media } = parse(audioPostBlock);
+        expect(detail.sourceUrl).toBe('https://open.spotify.com/track/6NVx24kOU2B8IZ0HsY2fUz');
+        expect(contentHtml).toBe('<p>Salvatore Adamo — Tombe la neige</p>');
+        expect(media).toEqual([]);
+    });
+    it('states what it could not find rather than inventing it', () => {
+        const { detail, publishedOn, title } = parse('<article class="post"><div></div></article>');
+        expect(title).toBeNull();
+        expect(publishedOn).toBeNull();
+        expect(detail).toEqual({
+            postType: null,
+            publishedLabel: null,
+            sourceUrl: null,
+            sourceUrls: [],
+            embedTitle: null
+        });
+    });
+});

package/agnostic/scrapers/@tumblr/blogs/index.d.ts

@@ -0,0 +1,8 @@
+import type { TumblrBlogParser } from '../types.js';
+/**
+ * Loads the parser registered for a blog, importing its module on first use.
+ *
+ * @param blogName - The blog identifier, as `TumblrBlog.name` states it.
+ * @returns That blog's parser, or `null` when none is registered.
+ */
+export declare function loadBlogParser(blogName: string): Promise<TumblrBlogParser | null>;

package/agnostic/scrapers/@tumblr/blogs/index.js

@@ -0,0 +1,23 @@
+/**
+ * Every blog this package carries a parser for, by the name its host states.
+ *
+ * Each entry is a dynamic import rather than a static one: a consumer reading one blog
+ * has no use for the parsers of the others, and a static import would put all of them in
+ * their bundle.
+ */
+const loadersByBlogName = {
+    delabienbonnemusique: async () => await import('./delabienbonnemusique/index.js')
+};
+/**
+ * Loads the parser registered for a blog, importing its module on first use.
+ *
+ * @param blogName - The blog identifier, as `TumblrBlog.name` states it.
+ * @returns That blog's parser, or `null` when none is registered.
+ */
+export async function loadBlogParser(blogName) {
+    const load = loadersByBlogName[blogName];
+    if (load === undefined)
+        return null;
+    const { parser } = await load();
+    return parser;
+}

package/agnostic/scrapers/@tumblr/index.d.ts

@@ -0,0 +1,73 @@
+import type { TumblrBlogParserLoader, TumblrPage, TumblrRobots, TumblrSitemap } from './types.js';
+/**
+ * Fetches one page and decodes it as UTF-8, which is what Tumblr serves.
+ *
+ * @param url - Absolute URL of the page to fetch.
+ * @returns The page's raw content.
+ * @throws If the response status is not a success, as `fetch-failed-<status>`, so a
+ * caller can tell a missing page from an unparseable one.
+ */
+export declare function fetchPage(url: string): Promise<string>;
+/**
+ * Extracts everything a single Tumblr page reveals about its blog and its posts.
+ *
+ * Reads only the markup Tumblr itself injects — canonical and icon links, Open Graph and
+ * `article:*` metas, and the `/post/<id>` permalink pattern — so it holds across themes
+ * rather than against one. Theme markup is never interpreted, only kept: each post
+ * carries its block's raw HTML in `rawHtml`, which is what a blog's own parser is handed
+ * to extract the fields no theme-agnostic rule can reach.
+ *
+ * Everything that locates the page is derived from `pageUrl`, never from the canonical
+ * link: a modern theme points its canonical at `www.tumblr.com/<blog>`, which names a
+ * different host than the one being read and is identical on every listing page.
+ *
+ * @param pageContent - The page's raw HTML.
+ * @param pageUrl - The URL the content was fetched from.
+ * @returns The page's blog, posts and pagination, each field `null` or empty where the
+ * page stated nothing.
+ * @throws If the content declares neither a canonical URL nor an Open Graph site name,
+ * which is taken to mean it is not a Tumblr page.
+ */
+export declare function parsePage(pageContent: string, pageUrl: string): TumblrPage;
+/**
+ * Fills each post's `detail` with what the blog's own parser reads out of its block.
+ *
+ * The generic pass stops at the markup: it locates a post's block and keeps it, but a
+ * title, a date or a track listing sit wherever a theme decided, which no cross-theme rule
+ * reaches. This is where a blog's own parser gets its turn, and it is loaded by blog name
+ * on first use — a consumer reading one blog never imports the parsers of the others.
+ *
+ * A page whose blog has no registered parser comes back untouched, which is the common
+ * case.
+ *
+ * @param page - A page as `parsePage` returned it.
+ * @param loadParser - Where to look a parser up, for a consumer registering their own.
+ * Defaults to the parsers this package carries.
+ * @returns The page, with `detail` filled on every post whose block could be read, and
+ * the standard fields the parser completed.
+ * @throws Whatever the blog's parser throws — a parser that breaks on a post is a bug in
+ * that parser, and a crawl reports the page rather than storing a post that looks complete.
+ */
+export declare function fillPostDetails(page: TumblrPage, loadParser?: TumblrBlogParserLoader): Promise<TumblrPage>;
+/**
+ * Reads a `sitemap.xml` document, whether it indexes other sitemaps or lists locations.
+ *
+ * A blog's sitemap enumerates every post without any theme involvement, which makes it
+ * the cheapest and most reliable way to discover what a blog holds.
+ *
+ * @param sitemapContent - The sitemap's raw XML.
+ * @returns The sitemaps it points at, or the locations it lists with the date each was
+ * last modified.
+ * @throws If the document's root is neither `sitemapindex` nor `urlset`, as
+ * `not-a-sitemap`.
+ */
+export declare function parseSitemap(sitemapContent: string): TumblrSitemap;
+/**
+ * Reads a `robots.txt`, keeping the `Sitemap:` directives and the rules stated for every
+ * crawler.
+ *
+ * @param robotsContent - The file's raw text.
+ * @returns The sitemaps it advertises and the `User-agent: *` rules, each list empty when
+ * the file states none.
+ */
+export declare function parseRobots(robotsContent: string): TumblrRobots;

package/agnostic/scrapers/@tumblr/index.js

@@ -0,0 +1,180 @@
+import { JSDOM } from 'jsdom';
+import { loadBlogParser } from './blogs/index.js';
+import { extractBlog, extractCanonicalUrl, extractPageKind, extractPagination, extractPostFromPermalinkPage, extractPostsFromListingPage, readMetaProperty, readSitemapChildUrls, readSitemapLocations, toIntegerOrNull, toOrigin } from './utils.js';
+/**
+ * Fetches one page and decodes it as UTF-8, which is what Tumblr serves.
+ *
+ * @param url - Absolute URL of the page to fetch.
+ * @returns The page's raw content.
+ * @throws If the response status is not a success, as `fetch-failed-<status>`, so a
+ * caller can tell a missing page from an unparseable one.
+ */
+export async function fetchPage(url) {
+    const response = await fetch(url);
+    if (!response.ok)
+        throw new Error(`fetch-failed-${response.status}`);
+    const content = await response.text();
+    return content;
+}
+/**
+ * Extracts everything a single Tumblr page reveals about its blog and its posts.
+ *
+ * Reads only the markup Tumblr itself injects — canonical and icon links, Open Graph and
+ * `article:*` metas, and the `/post/<id>` permalink pattern — so it holds across themes
+ * rather than against one. Theme markup is never interpreted, only kept: each post
+ * carries its block's raw HTML in `rawHtml`, which is what a blog's own parser is handed
+ * to extract the fields no theme-agnostic rule can reach.
+ *
+ * Everything that locates the page is derived from `pageUrl`, never from the canonical
+ * link: a modern theme points its canonical at `www.tumblr.com/<blog>`, which names a
+ * different host than the one being read and is identical on every listing page.
+ *
+ * @param pageContent - The page's raw HTML.
+ * @param pageUrl - The URL the content was fetched from.
+ * @returns The page's blog, posts and pagination, each field `null` or empty where the
+ * page stated nothing.
+ * @throws If the content declares neither a canonical URL nor an Open Graph site name,
+ * which is taken to mean it is not a Tumblr page.
+ */
+export function parsePage(pageContent, pageUrl) {
+    const collectedOn = new Date();
+    const { document } = new JSDOM(pageContent).window;
+    const canonicalUrl = extractCanonicalUrl(document);
+    if (canonicalUrl === null && readMetaProperty(document, 'og:site_name') === null) {
+        throw new Error('not-a-tumblr-page');
+    }
+    const baseUrl = toOrigin(pageUrl);
+    const kind = extractPageKind(pageUrl);
+    return {
+        kind,
+        url: pageUrl,
+        canonicalUrl,
+        blog: extractBlog(document, pageUrl, canonicalUrl, kind),
+        posts: kind === 'permalink'
+            ? [extractPostFromPermalinkPage(document, pageUrl)]
+            : extractPostsFromListingPage(document, baseUrl),
+        pagination: extractPagination(document, pageUrl, baseUrl),
+        collectedOn
+    };
+}
+/**
+ * Fills each post's `detail` with what the blog's own parser reads out of its block.
+ *
+ * The generic pass stops at the markup: it locates a post's block and keeps it, but a
+ * title, a date or a track listing sit wherever a theme decided, which no cross-theme rule
+ * reaches. This is where a blog's own parser gets its turn, and it is loaded by blog name
+ * on first use — a consumer reading one blog never imports the parsers of the others.
+ *
+ * A page whose blog has no registered parser comes back untouched, which is the common
+ * case.
+ *
+ * @param page - A page as `parsePage` returned it.
+ * @param loadParser - Where to look a parser up, for a consumer registering their own.
+ * Defaults to the parsers this package carries.
+ * @returns The page, with `detail` filled on every post whose block could be read, and
+ * the standard fields the parser completed.
+ * @throws Whatever the blog's parser throws — a parser that breaks on a post is a bug in
+ * that parser, and a crawl reports the page rather than storing a post that looks complete.
+ */
+export async function fillPostDetails(page, loadParser = loadBlogParser) {
+    const blogName = page.blog.name;
+    if (blogName === null)
+        return page;
+    const parser = await loadParser(blogName);
+    if (parser === null)
+        return page;
+    return {
+        ...page,
+        posts: page.posts.map(post => {
+            const parsed = parsePostBlock(parser, post);
+            return parsed === null ? post : applyParsedPost(post, parsed);
+        })
+    };
+}
+function parsePostBlock(parser, post) {
+    if (post.rawHtml === null)
+        return null;
+    const { document } = new JSDOM(`<body>${post.rawHtml}</body>`).window;
+    const postElement = document.body.firstElementChild;
+    if (postElement === null)
+        return null;
+    return parser.parsePostDom(postElement);
+}
+function applyParsedPost(post, parsed) {
+    const { tags = [], media = [] } = parsed;
+    return {
+        ...post,
+        detail: parsed.detail,
+        title: parsed.title ?? post.title,
+        contentHtml: parsed.contentHtml ?? post.contentHtml,
+        publishedOn: parsed.publishedOn ?? post.publishedOn,
+        tags: tags.length > 0 ? tags : post.tags,
+        media: media.length > 0 ? media : post.media
+    };
+}
+/**
+ * Reads a `sitemap.xml` document, whether it indexes other sitemaps or lists locations.
+ *
+ * A blog's sitemap enumerates every post without any theme involvement, which makes it
+ * the cheapest and most reliable way to discover what a blog holds.
+ *
+ * @param sitemapContent - The sitemap's raw XML.
+ * @returns The sitemaps it points at, or the locations it lists with the date each was
+ * last modified.
+ * @throws If the document's root is neither `sitemapindex` nor `urlset`, as
+ * `not-a-sitemap`.
+ */
+export function parseSitemap(sitemapContent) {
+    const { document } = new JSDOM(sitemapContent, { contentType: 'text/xml' }).window;
+    const rootName = document.documentElement.localName;
+    if (rootName !== 'sitemapindex' && rootName !== 'urlset')
+        throw new Error('not-a-sitemap');
+    return {
+        kind: rootName === 'sitemapindex' ? 'index' : 'urlset',
+        childSitemapUrls: readSitemapChildUrls(document),
+        locations: readSitemapLocations(document)
+    };
+}
+/**
+ * Reads a `robots.txt`, keeping the `Sitemap:` directives and the rules stated for every
+ * crawler.
+ *
+ * @param robotsContent - The file's raw text.
+ * @returns The sitemaps it advertises and the `User-agent: *` rules, each list empty when
+ * the file states none.
+ */
+export function parseRobots(robotsContent) {
+    const sitemapUrls = [];
+    const disallowedPaths = [];
+    const allowedPaths = [];
+    let crawlDelaySeconds = null;
+    let insideEveryCrawlerGroup = false;
+    for (const rawLine of robotsContent.split(/\r?\n/v)) {
+        const line = rawLine.replace(/#.*$/v, '').trim();
+        const separatorIndex = line.indexOf(':');
+        if (separatorIndex === -1)
+            continue;
+        const directive = line.slice(0, separatorIndex).trim().toLowerCase();
+        const value = line.slice(separatorIndex + 1).trim();
+        if (value === '')
+            continue;
+        if (directive === 'sitemap')
+            sitemapUrls.push(value);
+        else if (directive === 'user-agent')
+            insideEveryCrawlerGroup = value === '*';
+        else if (!insideEveryCrawlerGroup)
+            continue;
+        else if (directive === 'disallow')
+            disallowedPaths.push(value);
+        else if (directive === 'allow')
+            allowedPaths.push(value);
+        else if (directive === 'crawl-delay')
+            crawlDelaySeconds = toIntegerOrNull(value);
+    }
+    return {
+        sitemapUrls,
+        disallowedPaths,
+        allowedPaths,
+        crawlDelaySeconds
+    };
+}

package/agnostic/scrapers/@tumblr/index.test.d.ts

@@ -0,0 +1 @@
+export {};