@mxfb/tools 1.0.0 → 1.0.3

package/node/scrapers/@tumblr/utils.js

@@ -0,0 +1,328 @@
+import { createHash, randomBytes } from 'node:crypto';
+import { createReadStream, promises as fs } from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import { createInterface } from 'node:readline';
+import { parseRobots, parseSitemap } from '../../../agnostic/scrapers/@tumblr/index.js';
+const PAGES_FILE_NAME = 'pages.ndjson';
+const STATE_FILE_NAME = 'state.json';
+const RAW_DIR_NAME = 'raw';
+const DEFAULT_SITEMAP_PATHNAME = '/sitemap.xml';
+const LISTING_PAGE_KINDS = ['index', 'archive', 'tagged'];
+export function toOriginOrNull(rawUrl) {
+    try {
+        return new URL(rawUrl).origin;
+    }
+    catch {
+        return null;
+    }
+}
+export function normalizeUrl(rawUrl, baseUrl) {
+    try {
+        const url = new URL(rawUrl, baseUrl ?? undefined);
+        url.hash = '';
+        if (url.pathname !== '/' && url.pathname.endsWith('/')) {
+            url.pathname = url.pathname.slice(0, -1);
+        }
+        return url.href;
+    }
+    catch {
+        return null;
+    }
+}
+export function isListingKind(kind) {
+    return LISTING_PAGE_KINDS.includes(kind);
+}
+export function buildListingPageUrls(baseUrl, from, to) {
+    const urls = [];
+    for (let pageNumber = from; pageNumber <= to; pageNumber = pageNumber + 1) {
+        urls.push(`${baseUrl}/page/${pageNumber}`);
+    }
+    return urls;
+}
+export function toErrorMessage(error) {
+    return error instanceof Error ? error.message : 'unknown-error';
+}
+export async function fetchRobots(baseUrl, fetcher) {
+    const content = await fetchOrNull(fetcher, `${baseUrl}/robots.txt`);
+    if (content === null)
+        return null;
+    try {
+        return parseRobots(content);
+    }
+    catch {
+        return null;
+    }
+}
+export async function collectSitemapLocations(baseUrl, robots, fetcher) {
+    const advertisedUrls = robots?.sitemapUrls ?? [];
+    const rootSitemapUrls = advertisedUrls.length > 0
+        ? advertisedUrls
+        : [`${baseUrl}${DEFAULT_SITEMAP_PATHNAME}`];
+    const lastModifiedByUrl = new Map();
+    const visitedSitemapUrls = new Set();
+    const pendingSitemapUrls = [...rootSitemapUrls];
+    while (pendingSitemapUrls.length > 0) {
+        const sitemapUrl = pendingSitemapUrls.shift();
+        if (sitemapUrl === undefined || visitedSitemapUrls.has(sitemapUrl))
+            continue;
+        visitedSitemapUrls.add(sitemapUrl);
+        const sitemap = await readSitemapOrNull(sitemapUrl, fetcher);
+        if (sitemap === null)
+            continue;
+        for (const childUrl of sitemap.childSitemapUrls)
+            pendingSitemapUrls.push(childUrl);
+        for (const location of sitemap.locations) {
+            const normalizedUrl = normalizeUrl(location.url, baseUrl);
+            if (normalizedUrl === null || lastModifiedByUrl.has(normalizedUrl))
+                continue;
+            lastModifiedByUrl.set(normalizedUrl, location.lastModifiedOn);
+        }
+    }
+    return lastModifiedByUrl;
+}
+export function fillLastModifiedFromSitemap(page, lastModifiedByUrl, baseUrl) {
+    if (lastModifiedByUrl.size === 0)
+        return page;
+    return {
+        ...page,
+        posts: page.posts.map(post => {
+            if (post.url === null || post.lastModifiedOn !== null)
+                return post;
+            const normalizedUrl = normalizeUrl(post.url, baseUrl);
+            const lastModifiedOn = normalizedUrl === null
+                ? null
+                : lastModifiedByUrl.get(normalizedUrl) ?? null;
+            return lastModifiedOn === null ? post : { ...post, lastModifiedOn };
+        })
+    };
+}
+export async function createTemporaryDataDir() {
+    const dirPath = path.join(os.tmpdir(), randomBytes(8).toString('hex'));
+    await fs.mkdir(dirPath, { recursive: true });
+    return dirPath;
+}
+export async function ensureDataDirectories(dataDirPath, saveRawPages) {
+    await fs.mkdir(dataDirPath, { recursive: true });
+    if (saveRawPages)
+        await fs.mkdir(path.join(dataDirPath, RAW_DIR_NAME), { recursive: true });
+}
+export async function appendPageRecord(dataDirPath, page) {
+    const line = `${JSON.stringify(page)}\n`;
+    await fs.appendFile(path.join(dataDirPath, PAGES_FILE_NAME), line, { encoding: 'utf-8' });
+}
+export async function writeRawPage(dataDirPath, url, pageContent) {
+    const filePath = path.join(dataDirPath, RAW_DIR_NAME, toRawFileName(url));
+    await fs.writeFile(filePath, pageContent, { encoding: 'utf-8' });
+}
+export async function writeCrawlState(dataDirPath, state) {
+    const line = JSON.stringify(state, null, 2);
+    await fs.writeFile(path.join(dataDirPath, STATE_FILE_NAME), line, { encoding: 'utf-8' });
+}
+export async function readCrawlState(dataDirPath, baseUrl) {
+    const stateContent = await readFileOrNull(path.join(dataDirPath, STATE_FILE_NAME));
+    const parsedState = stateContent === null ? null : toCrawlStateOrNull(parseJson(stateContent));
+    if (parsedState !== null)
+        return parsedState;
+    return await rebuildCrawlStateFromRecords(dataDirPath, baseUrl);
+}
+export async function rebuildCrawlStateFromRecords(dataDirPath, baseUrl) {
+    const recordsContent = await readFileOrNull(path.join(dataDirPath, PAGES_FILE_NAME));
+    if (recordsContent === null)
+        return null;
+    const processedUrls = new Set();
+    const knownPostIds = new Set();
+    for (const line of recordsContent.split('\n')) {
+        if (line.trim() === '')
+            continue;
+        const record = toPageRecordDigest(parseJson(line));
+        if (record.url !== null)
+            processedUrls.add(record.url);
+        for (const postId of record.postIds)
+            knownPostIds.add(postId);
+    }
+    return {
+        baseUrl,
+        processedUrls: Array.from(processedUrls),
+        waitlistUrls: [],
+        knownPostIds: Array.from(knownPostIds)
+    };
+}
+export async function* readPageRecords(dataDirPath) {
+    const filePath = path.join(dataDirPath, PAGES_FILE_NAME);
+    const lines = createInterface({
+        input: createReadStream(filePath, { encoding: 'utf-8' }),
+        crlfDelay: Number.POSITIVE_INFINITY
+    });
+    for await (const line of lines) {
+        if (line.trim() === '')
+            continue;
+        const page = toArchivedPage(parseJson(line));
+        if (page !== null)
+            yield page;
+    }
+}
+export function mergeArchivedPosts(existing, incoming) {
+    const incomingWins = incoming.rawHtmlSource === 'permalink'
+        && existing.rawHtmlSource !== 'permalink';
+    const [preferred, other] = incomingWins ? [incoming, existing] : [existing, incoming];
+    return {
+        id: preferred.id ?? other.id,
+        url: preferred.url ?? other.url,
+        type: preferred.type === 'unknown' ? other.type : preferred.type,
+        publishedOn: preferred.publishedOn ?? other.publishedOn,
+        title: preferred.title ?? other.title,
+        contentHtml: preferred.contentHtml ?? other.contentHtml,
+        rawHtml: preferred.rawHtml ?? other.rawHtml,
+        rawHtmlSource: preferred.rawHtml === null ? other.rawHtmlSource : preferred.rawHtmlSource,
+        detail: preferred.detail ?? other.detail,
+        tags: preferred.tags.length > 0 ? preferred.tags : other.tags,
+        media: preferred.media.length > 0 ? preferred.media : other.media,
+        notesCount: preferred.notesCount ?? other.notesCount,
+        reblogOfUrl: preferred.reblogOfUrl ?? other.reblogOfUrl,
+        lastModifiedOn: preferred.lastModifiedOn ?? other.lastModifiedOn
+    };
+}
+export function resolveDelayMs(robots, defaultDelayMs) {
+    const requestedDelayMs = (robots?.crawlDelaySeconds ?? 0) * 1000;
+    return Math.max(defaultDelayMs, requestedDelayMs);
+}
+export function discoverUrlsToFollow(page, followedKinds, newPostCount, baseUrl) {
+    const urls = [];
+    if (!isListingKind(page.kind))
+        return urls;
+    if (followedKinds.includes('permalink')) {
+        for (const post of page.posts) {
+            const postUrl = post.url === null ? null : normalizeUrl(post.url, baseUrl);
+            if (postUrl !== null)
+                urls.push(postUrl);
+        }
+    }
+    const keepsWalking = followedKinds.includes(page.kind)
+        && newPostCount > 0
+        && page.pagination.nextPageUrl !== null;
+    if (keepsWalking) {
+        const nextPageUrl = normalizeUrl(page.pagination.nextPageUrl ?? '', baseUrl);
+        if (nextPageUrl !== null)
+            urls.push(nextPageUrl);
+    }
+    return urls;
+}
+async function readSitemapOrNull(sitemapUrl, fetcher) {
+    const content = await fetchOrNull(fetcher, sitemapUrl);
+    if (content === null)
+        return null;
+    try {
+        return parseSitemap(content);
+    }
+    catch {
+        return null;
+    }
+}
+async function fetchOrNull(fetcher, url) {
+    try {
+        return await fetcher(url);
+    }
+    catch {
+        return null;
+    }
+}
+async function readFileOrNull(filePath) {
+    try {
+        return await fs.readFile(filePath, { encoding: 'utf-8' });
+    }
+    catch {
+        return null;
+    }
+}
+function parseJson(raw) {
+    try {
+        return JSON.parse(raw);
+    }
+    catch {
+        return null;
+    }
+}
+function toStringArray(value) {
+    if (!Array.isArray(value))
+        return [];
+    return value.filter((item) => typeof item === 'string');
+}
+function toCrawlStateOrNull(value) {
+    if (typeof value !== 'object' || value === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const baseUrl = record.baseUrl;
+    if (typeof baseUrl !== 'string')
+        return null;
+    return {
+        baseUrl,
+        processedUrls: toStringArray(record.processedUrls),
+        waitlistUrls: toStringArray(record.waitlistUrls),
+        knownPostIds: toStringArray(record.knownPostIds)
+    };
+}
+function toPageRecordDigest(value) {
+    if (typeof value !== 'object' || value === null)
+        return { url: null, postIds: [] };
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const pageUrl = record.url;
+    const posts = Array.isArray(record.posts) ? record.posts : [];
+    const postIds = [];
+    for (const post of posts) {
+        if (typeof post !== 'object' || post === null)
+            continue;
+        /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+        const postRecord = post;
+        const postId = postRecord.id;
+        if (typeof postId === 'string')
+            postIds.push(postId);
+    }
+    return {
+        url: typeof pageUrl === 'string' ? pageUrl : null,
+        postIds
+    };
+}
+function toDateOrNull(value) {
+    if (typeof value !== 'string')
+        return null;
+    const parsed = new Date(value);
+    return Number.isNaN(parsed.getTime()) ? null : parsed;
+}
+// JSON knows no dates, so what a record holds are the strings `JSON.stringify` wrote.
+function toArchivedPage(value) {
+    if (typeof value !== 'object' || value === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const record = value;
+    const collectedOn = toDateOrNull(record.collectedOn);
+    if (typeof record.url !== 'string' || !Array.isArray(record.posts))
+        return null;
+    if (collectedOn === null)
+        return null;
+    /* eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion */
+    const page = value;
+    return {
+        ...page,
+        collectedOn,
+        posts: page.posts.map(post => ({
+            ...post,
+            publishedOn: toDateOrNull(post.publishedOn),
+            lastModifiedOn: toDateOrNull(post.lastModifiedOn)
+        }))
+    };
+}
+function toRawFileName(url) {
+    const digest = createHash('sha1').update(url).digest('hex').slice(0, 6);
+    let pathname;
+    try {
+        pathname = new URL(url).pathname;
+    }
+    catch {
+        pathname = url;
+    }
+    const slug = pathname.replace(/[^a-zA-Z0-9]+/gv, '-').replace(/^-+|-+$/gv, '');
+    return `${slug === '' ? 'home' : slug}-${digest}.html`;
+}

package/node/scrapers/index.d.ts

@@ -0,0 +1 @@
+export * as tumblr from './@tumblr/index.js'

package/node/scrapers/index.js

@@ -0,0 +1 @@
+export * as tumblr from './@tumblr/index.js'

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mxfb/tools",
-  "version": "1.0.0",
+  "version": "1.0.3",
   "description": "",
   "author": "Maxime Fabas",
   "license": "ISC",
@@ -11,54 +11,220 @@
   "type": "module",
   "main": "index.js",
   "module": "index.js",
+  "exports": {
+    ".": {
+      "import": "./index.js",
+      "types": "./index.d.ts"
+    },
+    "./agnostic": {
+      "import": "./agnostic/index.js",
+      "types": "./agnostic/index.d.ts"
+    },
+    "./agnostic/@publisher": {
+      "import": "./agnostic/@publisher/index.js",
+      "types": "./agnostic/@publisher/index.d.ts"
+    },
+    "./agnostic/@publisher/index.js": {
+      "import": "./agnostic/@publisher/index.js"
+    },
+    "./agnostic/elo": {
+      "import": "./agnostic/elo/index.js",
+      "types": "./agnostic/elo/index.d.ts"
+    },
+    "./agnostic/elo/index.js": {
+      "import": "./agnostic/elo/index.js"
+    },
+    "./agnostic/index.js": {
+      "import": "./agnostic/index.js"
+    },
+    "./agnostic/poetry": {
+      "import": "./agnostic/poetry/index.js",
+      "types": "./agnostic/poetry/index.d.ts"
+    },
+    "./agnostic/poetry/generate-rhymes": {
+      "import": "./agnostic/poetry/generate-rhymes/index.js",
+      "types": "./agnostic/poetry/generate-rhymes/index.d.ts"
+    },
+    "./agnostic/poetry/generate-rhymes/index.js": {
+      "import": "./agnostic/poetry/generate-rhymes/index.js"
+    },
+    "./agnostic/poetry/index.js": {
+      "import": "./agnostic/poetry/index.js"
+    },
+    "./agnostic/scrapers": {
+      "import": "./agnostic/scrapers/index.js",
+      "types": "./agnostic/scrapers/index.d.ts"
+    },
+    "./agnostic/scrapers/@meteociel": {
+      "import": "./agnostic/scrapers/@meteociel/index.js",
+      "types": "./agnostic/scrapers/@meteociel/index.d.ts"
+    },
+    "./agnostic/scrapers/@meteociel/index.js": {
+      "import": "./agnostic/scrapers/@meteociel/index.js"
+    },
+    "./agnostic/scrapers/@meteociel/locations": {
+      "import": "./agnostic/scrapers/@meteociel/locations/index.js",
+      "types": "./agnostic/scrapers/@meteociel/locations/index.d.ts"
+    },
+    "./agnostic/scrapers/@meteociel/locations/index.js": {
+      "import": "./agnostic/scrapers/@meteociel/locations/index.js"
+    },
+    "./agnostic/scrapers/@meteociel/types.js": {
+      "import": "./agnostic/scrapers/@meteociel/types.js",
+      "types": "./agnostic/scrapers/@meteociel/types.d.ts"
+    },
+    "./agnostic/scrapers/@tumblr": {
+      "import": "./agnostic/scrapers/@tumblr/index.js",
+      "types": "./agnostic/scrapers/@tumblr/index.d.ts"
+    },
+    "./agnostic/scrapers/@tumblr/blogs": {
+      "import": "./agnostic/scrapers/@tumblr/blogs/index.js",
+      "types": "./agnostic/scrapers/@tumblr/blogs/index.d.ts"
+    },
+    "./agnostic/scrapers/@tumblr/blogs/delabienbonnemusique": {
+      "import": "./agnostic/scrapers/@tumblr/blogs/delabienbonnemusique/index.js",
+      "types": "./agnostic/scrapers/@tumblr/blogs/delabienbonnemusique/index.d.ts"
+    },
+    "./agnostic/scrapers/@tumblr/blogs/delabienbonnemusique/index.js": {
+      "import": "./agnostic/scrapers/@tumblr/blogs/delabienbonnemusique/index.js"
+    },
+    "./agnostic/scrapers/@tumblr/blogs/index.js": {
+      "import": "./agnostic/scrapers/@tumblr/blogs/index.js"
+    },
+    "./agnostic/scrapers/@tumblr/index.js": {
+      "import": "./agnostic/scrapers/@tumblr/index.js"
+    },
+    "./agnostic/scrapers/@tumblr/types.js": {
+      "import": "./agnostic/scrapers/@tumblr/types.js",
+      "types": "./agnostic/scrapers/@tumblr/types.d.ts"
+    },
+    "./agnostic/scrapers/index.js": {
+      "import": "./agnostic/scrapers/index.js"
+    },
+    "./agnostic/weather": {
+      "import": "./agnostic/weather/index.js",
+      "types": "./agnostic/weather/index.d.ts"
+    },
+    "./agnostic/weather/index.js": {
+      "import": "./agnostic/weather/index.js"
+    },
+    "./agnostic/weather/types.js": {
+      "import": "./agnostic/weather/types.js",
+      "types": "./agnostic/weather/types.d.ts"
+    },
+    "./components": {
+      "import": "./components/index.js",
+      "types": "./components/index.d.ts"
+    },
+    "./components/index.js": {
+      "import": "./components/index.js"
+    },
+    "./components/utils": {
+      "import": "./components/utils/index.js",
+      "types": "./components/utils/index.d.ts"
+    },
+    "./components/utils/index.js": {
+      "import": "./components/utils/index.js"
+    },
+    "./components/utils/types.js": {
+      "import": "./components/utils/types.js",
+      "types": "./components/utils/types.d.ts"
+    },
+    "./index.js": {
+      "import": "./index.js"
+    },
+    "./node": {
+      "import": "./node/index.js",
+      "types": "./node/index.d.ts"
+    },
+    "./node/headless-browser": {
+      "import": "./node/headless-browser/index.js",
+      "types": "./node/headless-browser/index.d.ts"
+    },
+    "./node/headless-browser/index.js": {
+      "import": "./node/headless-browser/index.js"
+    },
+    "./node/headless-browser/types.js": {
+      "import": "./node/headless-browser/types.js",
+      "types": "./node/headless-browser/types.d.ts"
+    },
+    "./node/index.js": {
+      "import": "./node/index.js"
+    },
+    "./node/scrapers": {
+      "import": "./node/scrapers/index.js",
+      "types": "./node/scrapers/index.d.ts"
+    },
+    "./node/scrapers/@tumblr": {
+      "import": "./node/scrapers/@tumblr/index.js",
+      "types": "./node/scrapers/@tumblr/index.d.ts"
+    },
+    "./node/scrapers/@tumblr/index.js": {
+      "import": "./node/scrapers/@tumblr/index.js"
+    },
+    "./node/scrapers/@tumblr/types.js": {
+      "import": "./node/scrapers/@tumblr/types.js",
+      "types": "./node/scrapers/@tumblr/types.d.ts"
+    },
+    "./node/scrapers/index.js": {
+      "import": "./node/scrapers/index.js"
+    }
+  },
   "dependencies": {
-    "@aws-sdk/client-s3": "^3.907.0",
-    "@aws-sdk/lib-storage": "^3.907.0",
-    "@design-edito/tools": "^0.2.41",
-    "@google-cloud/storage": "^7.17.2",
-    "@inquirer/prompts": "^8.2.0",
+    "@aws-sdk/client-s3": "^3.999.0",
+    "@aws-sdk/lib-storage": "^3.1008.0",
+    "@design-edito/publisher": "^0.1.17",
+    "@design-edito/tools": "^0.5.39",
+    "@google-cloud/storage": "^7.19.0",
+    "@inquirer/prompts": "^8.3.0",
     "ansi-regex": "^6.2.2",
     "archiver": "^7.0.1",
-    "basic-ftp": "^5.0.5",
+    "basic-ftp": "^5.1.0",
     "chalk": "^5.6.2",
-    "express": "^5.1.0",
+    "express": "^5.2.1",
+    "jsdom": "^28.1.0",
     "multer": "^2.0.0",
+    "preact-render-to-string": "^6.6.6",
+    "puppeteer": "^24.39.0",
     "quantize": "^1.0.2",
-    "react": "^19.2.0",
-    "react-dom": "^19.2.0",
-    "sharp": "^0.34.4",
+    "react": "^19.2.4",
+    "react-dom": "^19.2.4",
+    "sharp": "^0.34.5",
     "ssh2": "^1.17.0",
-    "ssh2-sftp-client": "^12.0.0",
+    "ssh2-sftp-client": "^12.1.0",
     "xss": "^1.0.15",
-    "zod": "^4.1.12"
+    "zod": "^4.3.6"
   },
   "devDependencies": {
-    "@preact/preset-vite": "^2.10.2",
+    "@preact/preset-vite": "^2.10.3",
     "@testing-library/jest-dom": "^6.9.1",
     "@testing-library/preact": "^3.2.4",
-    "@testing-library/react": "^16.3.1",
-    "@types/archiver": "^6.0.3",
-    "@types/express": "^5.0.2",
-    "@types/jsdom": "^27.0.0",
-    "@types/multer": "^2.0.0",
-    "@types/node": "^24.7.1",
+    "@testing-library/react": "^16.3.2",
+    "@types/archiver": "^7.0.0",
+    "@types/express": "^5.0.6",
+    "@types/jsdom": "^28.0.0",
+    "@types/multer": "^2.1.0",
+    "@types/node": "^25.5.0",
     "@types/prompts": "^2.4.9",
     "@types/quantize": "^1.0.2",
-    "@types/react": "^19.2.2",
-    "@types/react-dom": "^19.2.1",
+    "@types/react": "^19.2.14",
+    "@types/react-dom": "^19.2.3",
     "@types/semver": "^7.7.1",
     "@types/sharp": "^0.31.1",
-    "@types/ssh2-sftp-client": "^9.0.4",
-    "@vitejs/plugin-react": "^5.1.2",
+    "@types/ssh2-sftp-client": "^9.0.6",
+    "@vitejs/plugin-react": "^5.1.3",
+    "change-case": "^5.4.4",
     "chokidar-cli": "^3.0.0",
     "concurrently": "^9.2.1",
-    "esbuild": "^0.25.10",
-    "jsdom": "^27.4.0",
+    "esbuild": "^0.27.4",
+    "eslint": "^9.39.5",
+    "eslint-config-love": "^154.1.0",
+    "globals": "^17.11.0",
     "prompts": "^2.4.2",
-    "puppeteer": "^24.23.1",
-    "semver": "^7.7.3",
-    "simple-git": "^3.27.0",
-    "typescript": "^5.9.3",
-    "vitest": "^4.0.17"
+    "semver": "^7.7.4",
+    "simple-git": "^3.33.0",
+    "typescript": "^5.3.3",
+    "vite": "^6.4.1",
+    "vitest": "^4.1.0"
   }
 }

package/chunks/chunk-JQLUR5LO.js

@@ -1,12 +0,0 @@
-// src/agnostic/scrapers/@meteociel/locations/index.ts
-var citiesData = {
-  // Paris 11e
-  "paris11": {
-    id: 27827,
-    urlSuffix: "paris_11eme_arrondissement"
-  }
-};
-
-export {
-  citiesData
-};