@mxfb/tools 0.3.0 → 1.0.2

package/node/scrapers/@tumblr/types.d.ts

@@ -0,0 +1,91 @@
+import type { TumblrBlog, TumblrPageKind } from '../../../agnostic/scrapers/@tumblr/types.js';
+/**
+ * How one crawl of a blog should behave.
+ *
+ * @property dataDirPath - Directory the crawl writes into, created when missing. Left
+ * out, a fresh temporary directory is made and reported back as
+ * {@link CrawlBlogResult.dataDirPath}, which is the path to hand back to resume later.
+ * @property followedKinds - Which page kinds the crawl enqueues when it meets them.
+ * Defaults to `['index', 'permalink']`: listings enumerate, permalinks extract.
+ * @property pages - Seeds `/page/from` to `/page/to` instead of discovering listings from
+ * the blog root.
+ * @property limit - Hard cap on the number of pages fetched, a safety valve rather than
+ * the stop condition. Defaults to 10000.
+ * @property delayMs - Delay between two fetches. Defaults to 250, and is raised to the
+ * blog's own `Crawl-delay` when it asks for more.
+ * @property saveRawPages - Keeps each page's raw HTML under `raw/`, so a later parser fix
+ * can be replayed without fetching anything again. Defaults to `true`.
+ * @property resume - Picks up where a previous crawl of the same directory stopped.
+ * @property useSitemap - Reads `robots.txt` and `sitemap.xml` to enumerate posts without
+ * walking listings. Defaults to `true`.
+ * @property onPageCrawled - Called once per page, successes and failures alike.
+ * @property fetcher - Reads one URL as text, or answers `null` when it could not. Left
+ * out, a headless Chrome is started for the crawl and closed with it, which is what a
+ * Tumblr blog host requires: it answers a plain request with a 403 and a JavaScript
+ * challenge.
+ */
+export type CrawlBlogOptions = {
+    dataDirPath?: string;
+    followedKinds?: TumblrPageKind[];
+    pages?: {
+        from: number;
+        to: number;
+    };
+    limit?: number;
+    delayMs?: number | (() => number);
+    saveRawPages?: boolean;
+    resume?: boolean;
+    useSitemap?: boolean;
+    onPageCrawled?: (report: CrawledPageReport) => void;
+    fetcher?: (url: string) => Promise<string | null>;
+};
+/**
+ * What became of one crawled page.
+ *
+ * @property url - The URL that was fetched.
+ * @property kind - What the page turned out to be, `null` when it could not be read.
+ * @property postCount - Posts the page carried.
+ * @property newPostCount - Posts the crawl had not seen before, which is what tells a
+ * listing worth following from one that only repeats itself.
+ * @property error - Why the page was skipped, `null` on success.
+ */
+export type CrawledPageReport = {
+    url: string;
+    kind: TumblrPageKind | null;
+    postCount: number;
+    newPostCount: number;
+    error: string | null;
+};
+/**
+ * What one crawl achieved.
+ *
+ * @property dataDirPath - Where the crawl wrote, whether it was given or created. This
+ * is what a later run needs to resume.
+ * @property blog - The blog as the first readable page described it, `null` when no page
+ * could be read.
+ * @property pageCount - Pages successfully parsed.
+ * @property postCount - Distinct posts seen across them.
+ * @property failures - Every page that could not be fetched or parsed.
+ */
+export type CrawlBlogResult = {
+    dataDirPath: string;
+    blog: TumblrBlog | null;
+    pageCount: number;
+    postCount: number;
+    failures: CrawledPageReport[];
+};
+/**
+ * A crawl's progress, as `state.json` holds it between two runs.
+ *
+ * @property baseUrl - Origin the crawl was run against.
+ * @property processedUrls - URLs already fetched, which a resumed crawl will not fetch
+ * again.
+ * @property waitlistUrls - URLs still owed a fetch when the crawl stopped.
+ * @property knownPostIds - Every post id seen so far.
+ */
+export type CrawlState = {
+    baseUrl: string;
+    processedUrls: string[];
+    waitlistUrls: string[];
+    knownPostIds: string[];
+};

package/node/scrapers/@tumblr/types.js

@@ -0,0 +1 @@
+export {};

package/node/scrapers/@tumblr/utils.d.ts

@@ -0,0 +1,23 @@
+import type { TumblrPage, TumblrPageKind, TumblrPost, TumblrRobots } from '../../../agnostic/scrapers/@tumblr/types.js';
+import type { CrawlState } from './types.js';
+type PageFetcher = (url: string) => Promise<string | null>;
+export declare function toOriginOrNull(rawUrl: string): string | null;
+export declare function normalizeUrl(rawUrl: string, baseUrl: string | null): string | null;
+export declare function isListingKind(kind: TumblrPageKind): boolean;
+export declare function buildListingPageUrls(baseUrl: string, from: number, to: number): string[];
+export declare function toErrorMessage(error: unknown): string;
+export declare function fetchRobots(baseUrl: string, fetcher: PageFetcher): Promise<TumblrRobots | null>;
+export declare function collectSitemapLocations(baseUrl: string, robots: TumblrRobots | null, fetcher: PageFetcher): Promise<Map<string, Date | null>>;
+export declare function fillLastModifiedFromSitemap(page: TumblrPage, lastModifiedByUrl: Map<string, Date | null>, baseUrl: string): TumblrPage;
+export declare function createTemporaryDataDir(): Promise<string>;
+export declare function ensureDataDirectories(dataDirPath: string, saveRawPages: boolean): Promise<void>;
+export declare function appendPageRecord(dataDirPath: string, page: TumblrPage): Promise<void>;
+export declare function writeRawPage(dataDirPath: string, url: string, pageContent: string): Promise<void>;
+export declare function writeCrawlState(dataDirPath: string, state: CrawlState): Promise<void>;
+export declare function readCrawlState(dataDirPath: string, baseUrl: string): Promise<CrawlState | null>;
+export declare function rebuildCrawlStateFromRecords(dataDirPath: string, baseUrl: string): Promise<CrawlState | null>;
+export declare function readPageRecords(dataDirPath: string): AsyncGenerator<TumblrPage>;
+export declare function mergeArchivedPosts(existing: TumblrPost, incoming: TumblrPost): TumblrPost;
+export declare function resolveDelayMs(robots: TumblrRobots | null, defaultDelayMs: number): number;
+export declare function discoverUrlsToFollow(page: TumblrPage, followedKinds: TumblrPageKind[], newPostCount: number, baseUrl: string): string[];
+export {};

package/node/scrapers/@tumblr/utils.js

@@ -0,0 +1,328 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { createReadStream, promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createInterface } from 'node:readline';
+import { parseRobots, parseSitemap } from '../../../agnostic/scrapers/@tumblr/index.js';
+const PAGES_FILE_NAME = 'pages.ndjson';
+const STATE_FILE_NAME = 'state.json';
+const RAW_DIR_NAME = 'raw';
+const DEFAULT_SITEMAP_PATHNAME = '/sitemap.xml';
+const LISTING_PAGE_KINDS = ['index', 'archive', 'tagged'];
+export function toOriginOrNull(rawUrl) {
+    try {
+        return new URL(rawUrl).origin;
+    }
+    catch {
+        return null;
+    }
+}
+export function normalizeUrl(rawUrl, baseUrl) {
+    try {
+        const url = new URL(rawUrl, baseUrl ?? undefined);
+        url.hash = '';
+        if (url.pathname !== '/' && url.pathname.endsWith('/')) {
+            url.pathname = url.pathname.slice(0, -1);
+        }
+        return url.href;
+    }
+    catch {
+        return null;
+    }
+}
+export function isListingKind(kind) {
+    return LISTING_PAGE_KINDS.includes(kind);
+}
+export function buildListingPageUrls(baseUrl, from, to) {
+    const urls = [];
+    for (let pageNumber = from; pageNumber <= to; pageNumber = pageNumber + 1) {
+        urls.push(`${baseUrl}/page/${pageNumber}`);
+    }
+    return urls;
+}
+export function toErrorMessage(error) {
+    return error instanceof Error ? error.message : 'unknown-error';
+}
+export async function fetchRobots(baseUrl, fetcher) {
+    const content = await fetchOrNull(fetcher, `${baseUrl}/robots.txt`);
+    if (content === null)
+        return null;
+    try {
+        return parseRobots(content);
+    }
+    catch {
+        return null;
+    }
+}
+export async function collectSitemapLocations(baseUrl, robots, fetcher) {
+    const advertisedUrls = robots?.sitemapUrls ?? [];
+    const rootSitemapUrls = advertisedUrls.length > 0
+        ? advertisedUrls
+        : [`${baseUrl}${DEFAULT_SITEMAP_PATHNAME}`];
+    const lastModifiedByUrl = new Map();
+    const visitedSitemapUrls = new Set();
+    const pendingSitemapUrls = [...rootSitemapUrls];
+    while (pendingSitemapUrls.length > 0) {
+        const sitemapUrl = pendingSitemapUrls.shift();
+        if (sitemapUrl === undefined || visitedSitemapUrls.has(sitemapUrl))
+            continue;
+        visitedSitemapUrls.add(sitemapUrl);
+        const sitemap = await readSitemapOrNull(sitemapUrl, fetcher);
+        if (sitemap === null)
+            continue;
+        for (const childUrl of sitemap.childSitemapUrls)
+            pendingSitemapUrls.push(childUrl);
+        for (const location of sitemap.locations) {
+            const normalizedUrl = normalizeUrl(location.url, baseUrl);
+            if (normalizedUrl === null || lastModifiedByUrl.has(normalizedUrl))
+                continue;
+            lastModifiedByUrl.set(normalizedUrl, location.lastModifiedOn);
+        }
+    }
+    return lastModifiedByUrl;
+}
+export function fillLastModifiedFromSitemap(page, lastModifiedByUrl, baseUrl) {
+    if (lastModifiedByUrl.size === 0)
+        return page;
+    return {
+        ...page,
+        posts: page.posts.map(post => {
+            if (post.url === null || post.lastModifiedOn !== null)
+                return post;
+            const normalizedUrl = normalizeUrl(post.url, baseUrl);
+            const lastModifiedOn = normalizedUrl === null
+                ? null
+                : lastModifiedByUrl.get(normalizedUrl) ?? null;
+            return lastModifiedOn === null ? post : { ...post, lastModifiedOn };
+        })
+    };
+}
+export async function createTemporaryDataDir() {
+    const dirPath = path.join(os.tmpdir(), randomBytes(8).toString('hex'));
+    await fs.mkdir(dirPath, { recursive: true });
+    return dirPath;
+}
+export async function ensureDataDirectories(dataDirPath, saveRawPages) {
+    await fs.mkdir(dataDirPath, { recursive: true });
+    if (saveRawPages)
+        await fs.mkdir(path.join(dataDirPath, RAW_DIR_NAME), { recursive: true });
+}
+export async function appendPageRecord(dataDirPath, page) {
+    const line = `${JSON.stringify(page)}\n`;
+    await fs.appendFile(path.join(dataDirPath, PAGES_FILE_NAME), line, { encoding: 'utf-8' });
+}
+export async function writeRawPage(dataDirPath, url, pageContent) {
+    const filePath = path.join(dataDirPath, RAW_DIR_NAME, toRawFileName(url));
+    await fs.writeFile(filePath, pageContent, { encoding: 'utf-8' });
+}
+export async function writeCrawlState(dataDirPath, state) {
+    const line = JSON.stringify(state, null, 2);
+    await fs.writeFile(path.join(dataDirPath, STATE_FILE_NAME), line, { encoding: 'utf-8' });
+}
+export async function readCrawlState(dataDirPath, baseUrl) {
+    const stateContent = await readFileOrNull(path.join(dataDirPath, STATE_FILE_NAME));
+    const parsedState = stateContent === null ? null : toCrawlStateOrNull(parseJson(stateContent));
+    if (parsedState !== null)
+        return parsedState;
+    return await rebuildCrawlStateFromRecords(dataDirPath, baseUrl);
+}
+export async function rebuildCrawlStateFromRecords(dataDirPath, baseUrl) {
+    const recordsContent = await readFileOrNull(path.join(dataDirPath, PAGES_FILE_NAME));
+    if (recordsContent === null)
+        return null;
+    const processedUrls = new Set();
+    const knownPostIds = new Set();
+    for (const line of recordsContent.split('\n')) {
+        if (line.trim() === '')
+            continue;
+        const record = toPageRecordDigest(parseJson(line));
+        if (record.url !== null)
+            processedUrls.add(record.url);
+        for (const postId of record.postIds)
+            knownPostIds.add(postId);
+    }
+    return {
+        baseUrl,
+        processedUrls: Array.from(processedUrls),
+        waitlistUrls: [],
+        knownPostIds: Array.from(knownPostIds)
+    };
+}
+export async function* readPageRecords(dataDirPath) {
+    const filePath = path.join(dataDirPath, PAGES_FILE_NAME);
+    const lines = createInterface({
+        input: createReadStream(filePath, { encoding: 'utf-8' }),
+        crlfDelay: Number.POSITIVE_INFINITY
+    });
+    for await (const line of lines) {
+        if (line.trim() === '')
+            continue;
+        const page = toArchivedPage(parseJson(line));
+        if (page !== null)
+            yield page;
+    }
+}
+export function mergeArchivedPosts(existing, incoming) {
+    const incomingWins = incoming.rawHtmlSource === 'permalink'
+        && existing.rawHtmlSource !== 'permalink';
+    const [preferred, other] = incomingWins ? [incoming, existing] : [existing, incoming];
+    return {
+        id: preferred.id ?? other.id,
+        url: preferred.url ?? other.url,
+        type: preferred.type === 'unknown' ? other.type : preferred.type,
+        publishedOn: preferred.publishedOn ?? other.publishedOn,
+        title: preferred.title ?? other.title,
+        contentHtml: preferred.contentHtml ?? other.contentHtml,
+        rawHtml: preferred.rawHtml ?? other.rawHtml,
+        rawHtmlSource: preferred.rawHtml === null ? other.rawHtmlSource : preferred.rawHtmlSource,
+        detail: preferred.detail ?? other.detail,
+        tags: preferred.tags.length > 0 ? preferred.tags : other.tags,
+        media: preferred.media.length > 0 ? preferred.media : other.media,
+        notesCount: preferred.notesCount ?? other.notesCount,
+        reblogOfUrl: preferred.reblogOfUrl ?? other.reblogOfUrl,
+        lastModifiedOn: preferred.lastModifiedOn ?? other.lastModifiedOn
+    };
+}
+export function resolveDelayMs(robots, defaultDelayMs) {
+    const requestedDelayMs = (robots?.crawlDelaySeconds ?? 0) * 1000;
+    return Math.max(defaultDelayMs, requestedDelayMs);
+}
+export function discoverUrlsToFollow(page, followedKinds, newPostCount, baseUrl) {
+    const urls = [];
+    if (!isListingKind(page.kind))
+        return urls;
+    if (followedKinds.includes('permalink')) {
+        for (const post of page.posts) {
+            const postUrl = post.url === null ? null : normalizeUrl(post.url, baseUrl);
+            if (postUrl !== null)
+                urls.push(postUrl);
+        }
+    }
+    const keepsWalking = followedKinds.includes(page.kind)
+        && newPostCount > 0
+        && page.pagination.nextPageUrl !== null;
+    if (keepsWalking) {
+        const nextPageUrl = normalizeUrl(page.pagination.nextPageUrl ?? '', baseUrl);
+        if (nextPageUrl !== null)
+            urls.push(nextPageUrl);
+    }
+    return urls;
+}
+async function readSitemapOrNull(sitemapUrl, fetcher) {
+    const content = await fetchOrNull(fetcher, sitemapUrl);
+    if (content === null)
+        return null;
+    try {
+        return parseSitemap(content);
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchOrNull(fetcher, url) {
+    try {
+        return await fetcher(url);
+    }
+    catch {
+        return null;
+    }
+}
+async function readFileOrNull(filePath) {
+    try {
+        return await fs.readFile(filePath, { encoding: 'utf-8' });
+    }
+    catch {
+        return null;
+    }
+}
+function parseJson(raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+function toStringArray(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((item) => typeof item === 'string');
+}
+function toCrawlStateOrNull(value) {
+    if (typeof value !== 'object' || value === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const baseUrl = record.baseUrl;
+    if (typeof baseUrl !== 'string')
+        return null;
+    return {
+        baseUrl,
+        processedUrls: toStringArray(record.processedUrls),
+        waitlistUrls: toStringArray(record.waitlistUrls),
+        knownPostIds: toStringArray(record.knownPostIds)
+    };
+}
+function toPageRecordDigest(value) {
+    if (typeof value !== 'object' || value === null)
+        return { url: null, postIds: [] };
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const pageUrl = record.url;
+    const posts = Array.isArray(record.posts) ? record.posts : [];
+    const postIds = [];
+    for (const post of posts) {
+        if (typeof post !== 'object' || post === null)
+            continue;
+        /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+        const postRecord = post;
+        const postId = postRecord.id;
+        if (typeof postId === 'string')
+            postIds.push(postId);
+    }
+    return {
+        url: typeof pageUrl === 'string' ? pageUrl : null,
+        postIds
+    };
+}
+function toDateOrNull(value) {
+    if (typeof value !== 'string')
+        return null;
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+// JSON knows no dates, so what a record holds are the strings `JSON.stringify` wrote.
+function toArchivedPage(value) {
+    if (typeof value !== 'object' || value === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const collectedOn = toDateOrNull(record.collectedOn);
+    if (typeof record.url !== 'string' || !Array.isArray(record.posts))
+        return null;
+    if (collectedOn === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const page = value;
+    return {
+        ...page,
+        collectedOn,
+        posts: page.posts.map(post => ({
+            ...post,
+            publishedOn: toDateOrNull(post.publishedOn),
+            lastModifiedOn: toDateOrNull(post.lastModifiedOn)
+        }))
+    };
+}
+function toRawFileName(url) {
+    const digest = createHash('sha1').update(url).digest('hex').slice(0, 6);
+    let pathname;
+    try {
+        pathname = new URL(url).pathname;
+    }
+    catch {
+        pathname = url;
+    }
+    const slug = pathname.replace(/[^a-zA-Z0-9]+/gv, '-').replace(/^-+|-+$/gv, '');
+    return `${slug === '' ? 'home' : slug}-${digest}.html`;
+}

package/node/scrapers/index.d.ts

@@ -0,0 +1 @@
+export * as tumblr from './@tumblr/index.js'

package/node/scrapers/index.js

@@ -0,0 +1 @@
+export * as tumblr from './@tumblr/index.js'