@vinikjkkj/wa-fetcher 0.1.0

package/README.md

@@ -0,0 +1,84 @@
+# @vinikjkkj/wa-fetcher
+
+Headless scraper for web.whatsapp.com bundles. Downloads every loaded JS
+chunk to disk and writes a manifest — that's all. Per-domain extractors
+(proto, mex, diff, …) consume the raw dump independently.
+
+## Install
+
+```sh
+npm i @vinikjkkj/wa-fetcher
+```
+
+## CLI
+
+```sh
+# Default: download every bundle + write manifest
+npx wa-fetcher --out dump/
+
+# Discovery only: just the URL list (no download). Useful when piping into
+# tools that have their own downloader (e.g. wa-modules-loader).
+npx wa-fetcher --urls-only --out urls.json
+npx wa-fetcher --urls-only > urls.json
+```
+
+| Flag | Default | Notes |
+|---|---|---|
+| `--out <path>` | `dump` (dir) / stdout (urls-only) | Output destination |
+| `--urls-only` | off | Skip download; emit only the discovered URL array (JSON) |
+| `--auth <file>` | none | Saved cookie state JSON for authenticated fetch (captures more lazy chunks) |
+| `--extra-wait <ms>` | `5000` | Wait this long after network-idle for lazy chunks |
+
+**Output layout:**
+
+```
+dump/
+├── manifest.json                  { waVersion, fetchedAt, bundles[] }
+└── raw/
+    └── <wa-version>/
+        ├── chunk-AAAA.js
+        ├── chunk-BBBB.js
+        └── …
+```
+
+## Library
+
+```js
+const { discoverBundleUrls, fetchBundles } = require('@vinikjkkj/wa-fetcher')
+
+// Discovery only — no download. Returns the same URL list the full fetcher
+// would have downloaded (sorted, deduped, host-filtered to static.whatsapp.net).
+const { waVersion, urls } = await discoverBundleUrls()
+//   waVersion   "2.3000.xxxxxxx" | null
+//   urls        string[]
+
+// Discovery + download.
+const dump = await fetchBundles({ out: 'dump' })
+//   dump.waVersion              "2.3000.xxxxxxx" | null
+//   dump.bundles[]              [{ url, file, bytes }, ...]
+//   dump.paths.raw              absolute path to dump/raw/<version>/
+//   dump.paths.manifest         absolute path to dump/manifest.json
+```
+
+## GitHub Action
+
+```yaml
+- uses: vinikjkkj/wa-spec/packages/fetcher@v1
+  id: fetch
+  with:
+      out: dump
+- run: npx wa-mex apply --bundles ${{ steps.fetch.outputs.raw-dir }}
+- run: npx wa-proto apply --bundles ${{ steps.fetch.outputs.raw-dir }}
+```
+
+## Caveats
+
+- **Lazy chunks** that the SPA only loads via interaction (Settings, Profile,
+  Premium) won't be in the dump. Use `--auth` with a pre-saved cookie state
+  to maximize what loads naturally.
+- **Anti-bot** — `puppeteer-real-browser` works today but Meta can tighten
+  detection. If the fetcher returns blank pages, re-evaluate the strategy.
+- **No extraction** — this package is intentionally dumb. The extractors live
+  in [`@vinikjkkj/wa-mex`](https://github.com/vinikjkkj/wa-spec/tree/main/packages/mex)
+  and [`@vinikjkkj/wa-proto`](https://github.com/vinikjkkj/wa-spec/tree/main/packages/proto)
+  so adding a new artifact never requires touching the fetcher.

package/action.yml

@@ -0,0 +1,55 @@
+name: 'wa-fetcher'
+description: 'Download every loaded web.whatsapp.com JS bundle + emit a manifest.json (no extraction)'
+author: 'vinikjkkj'
+branding:
+    icon: 'download-cloud'
+    color: 'green'
+
+inputs:
+    out:
+        description: 'Output directory for raw bundles + manifest.json'
+        required: false
+        default: 'dump'
+    auth-state:
+        description: 'Optional path to a saved cookie state JSON for authenticated fetch'
+        required: false
+    extra-wait:
+        description: 'Extra wait (ms) after network-idle, gives lazy chunks time to load'
+        required: false
+        default: '5000'
+
+outputs:
+    manifest-path:
+        description: 'Path to the generated manifest.json'
+        value: ${{ steps.run.outputs.manifest-path }}
+    raw-dir:
+        description: 'Path to the raw bundles directory (raw/<wa-version>/)'
+        value: ${{ steps.run.outputs.raw-dir }}
+    wa-version:
+        description: 'Detected WhatsApp Web version (or "unknown")'
+        value: ${{ steps.run.outputs.wa-version }}
+
+runs:
+    using: 'composite'
+    steps:
+        - name: Install fetcher
+          shell: bash
+          run: npm install --no-save @vinikjkkj/wa-fetcher@latest
+
+        - name: Run fetcher
+          id: run
+          shell: bash
+          env:
+              FETCHER_OUT: ${{ inputs.out }}
+              FETCHER_AUTH: ${{ inputs.auth-state }}
+              FETCHER_EXTRA_WAIT: ${{ inputs.extra-wait }}
+          run: |
+              ARGS="--out $FETCHER_OUT --extra-wait $FETCHER_EXTRA_WAIT"
+              if [ -n "$FETCHER_AUTH" ]; then ARGS="$ARGS --auth $FETCHER_AUTH"; fi
+              npx wa-fetcher $ARGS
+              MANIFEST="$FETCHER_OUT/manifest.json"
+              echo "manifest-path=$MANIFEST" >> "$GITHUB_OUTPUT"
+              RAW_DIR=$(node -e "console.log(JSON.parse(require('node:fs').readFileSync(process.argv[1])).rawDir)" "$MANIFEST")
+              echo "raw-dir=$FETCHER_OUT/$RAW_DIR" >> "$GITHUB_OUTPUT"
+              VERSION=$(node -e "console.log(JSON.parse(require('node:fs').readFileSync(process.argv[1])).waVersion ?? 'unknown')" "$MANIFEST")
+              echo "wa-version=$VERSION" >> "$GITHUB_OUTPUT"

package/package.json

@@ -0,0 +1,57 @@
+{
+    "name": "@vinikjkkj/wa-fetcher",
+    "version": "0.1.0",
+    "description": "Headless scraper for web.whatsapp.com bundles — downloads every loaded JS chunk + emits a manifest. Pairs with @vinikjkkj/wa-mex and @vinikjkkj/wa-proto.",
+    "keywords": [
+        "whatsapp",
+        "whatsapp-web",
+        "scraper",
+        "fetcher",
+        "puppeteer",
+        "puppeteer-real-browser",
+        "reverse-engineering",
+        "wa-spec"
+    ],
+    "license": "MIT",
+    "author": {
+        "name": "vinikjkkj",
+        "url": "https://github.com/vinikjkkj"
+    },
+    "homepage": "https://github.com/vinikjkkj/wa-spec/tree/main/packages/fetcher#readme",
+    "bugs": {
+        "url": "https://github.com/vinikjkkj/wa-spec/issues"
+    },
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/vinikjkkj/wa-spec.git",
+        "directory": "packages/fetcher"
+    },
+    "funding": [
+        {
+            "type": "github",
+            "url": "https://github.com/sponsors/vinikjkkj"
+        }
+    ],
+    "main": "src/index.cjs",
+    "bin": {
+        "wa-fetcher": "src/cli.cjs"
+    },
+    "files": [
+        "src/**/*.cjs",
+        "src/**/*.js",
+        "action.yml",
+        "README.md"
+    ],
+    "scripts": {
+        "fetch": "node src/cli.cjs --out dump/"
+    },
+    "dependencies": {
+        "puppeteer-real-browser": "^1.4.4"
+    },
+    "engines": {
+        "node": ">=20.9.0"
+    },
+    "publishConfig": {
+        "access": "public"
+    }
+}

package/src/cli.cjs

@@ -0,0 +1,82 @@
+#!/usr/bin/env node
+'use strict'
+
+const fs = require('node:fs')
+const path = require('node:path')
+const { discoverBundleUrls, fetchBundles } = require('./index.cjs')
+
+function parseArgs(argv) {
+    const opts = { out: null, urlsOnly: false }
+    for (let i = 2; i < argv.length; i++) {
+        const a = argv[i]
+        if (a === '--out') opts.out = argv[++i]
+        else if (a === '--auth') opts.authState = argv[++i]
+        else if (a === '--extra-wait') opts.extraWaitMs = Number(argv[++i])
+        else if (a === '--urls-only') opts.urlsOnly = true
+        else if (a === '--help' || a === '-h') {
+            printHelp()
+            process.exit(0)
+        } else {
+            console.error('unknown flag:', a)
+            printHelp()
+            process.exit(2)
+        }
+    }
+    return opts
+}
+
+function printHelp() {
+    console.error(
+        [
+            'usage: wa-fetcher [options]',
+            '',
+            'options:',
+            '  --out <path>          output path',
+            '                          - default mode: directory for raw bundles + manifest.json (default: dump)',
+            '                          - --urls-only mode: file to write URL array JSON (default: stdout)',
+            '  --auth <file>         optional saved cookie state JSON',
+            '  --extra-wait <ms>     extra wait after networkidle for lazy chunks (default: 5000)',
+            '  --urls-only           skip download; output just the discovered URL array as JSON',
+            '  -h, --help            show this help'
+        ].join('\n')
+    )
+}
+
+async function main() {
+    const opts = parseArgs(process.argv)
+    const start = Date.now()
+
+    if (opts.urlsOnly) {
+        console.error('[wa-fetcher] discovering URLs only (no download)…')
+        const { waVersion, urls } = await discoverBundleUrls(opts)
+        const json = JSON.stringify(urls, null, 4) + '\n'
+        if (opts.out) {
+            fs.mkdirSync(path.dirname(path.resolve(opts.out)), { recursive: true })
+            fs.writeFileSync(opts.out, json)
+            console.error(
+                `[wa-fetcher] done in ${((Date.now() - start) / 1000).toFixed(1)}s — ` +
+                    `version=${waVersion ?? 'unknown'} urls=${urls.length} → ${opts.out}`
+            )
+        } else {
+            process.stdout.write(json)
+        }
+        return
+    }
+
+    const outDir = opts.out ?? 'dump'
+    console.error('[wa-fetcher] launching headless browser…')
+    const result = await fetchBundles({ ...opts, out: outDir })
+    const elapsed = ((Date.now() - start) / 1000).toFixed(1)
+    const mb = (result.bundles.reduce((s, b) => s + b.bytes, 0) / 1024 / 1024).toFixed(1)
+    console.error(
+        `[wa-fetcher] done in ${elapsed}s — version=${result.waVersion ?? 'unknown'} ` +
+            `bundles=${result.bundles.length} (${mb}MB)`
+    )
+    console.error(`[wa-fetcher] raw → ${result.paths.raw}`)
+    console.error(`[wa-fetcher] manifest → ${result.paths.manifest}`)
+}
+
+main().catch((err) => {
+    console.error('[wa-fetcher] failed:', err.stack || err.message || err)
+    process.exit(1)
+})

package/src/index.cjs

@@ -0,0 +1,249 @@
+'use strict'
+
+/**
+ * @vinikjkkj/wa-fetcher
+ *
+ *   const { discoverBundleUrls } = require('@vinikjkkj/wa-fetcher')
+ *   const { waVersion, urls } = await discoverBundleUrls()
+ *   //   urls[]            ['https://static.whatsapp.net/.../chunk.js', ...]
+ *   //   waVersion         '2.3000.xxx' | null
+ *
+ *   const { fetchBundles } = require('@vinikjkkj/wa-fetcher')
+ *   const dump = await fetchBundles({ out: 'dump' })
+ *   //   dump.bundles[]    [{ url, file, bytes }, ...]
+ *   //   dump.paths.raw    absolute path to <out>/raw/<version>/
+ *   //   dump.paths.manifest
+ */
+
+const fs = require('node:fs')
+const path = require('node:path')
+const { connect } = require('puppeteer-real-browser')
+
+const WHATSAPP_URL = 'https://web.whatsapp.com/'
+const NAV_TIMEOUT_MS = 60_000
+const LAZY_WAIT_MS = 5_000
+
+// In-page discovery: walks `<script data-sjs>` rsrcMap manifests, preload
+// links, inline script literals, and performance entries to find every JS
+// bundle URL Meta references — not just the chunks the SPA has loaded.
+// (Technique cribbed from vinikjkkj/wa-diff/fetch.js.)
+const DISCOVERY_FN = function () {
+    const found = new Set()
+    const allowedHost = 'static.whatsapp.net'
+    const jsFileRe = /\.m?js(?:[?#]|$)/i
+    const jsInTextRe =
+        /(?:https?:)?\/\/[^\s"'`<>]+?\.m?js(?:[?#][^\s"'`<>]*)?|(?:\/|\.\/|\.\.\/)[^\s"'`<>]+?\.m?js(?:[?#][^\s"'`<>]*)?/gi
+    function addUrl(raw) {
+        if (typeof raw !== 'string') return
+        const cleaned = raw.trim().replace(/\\\//g, '/')
+        if (!cleaned || !jsFileRe.test(cleaned)) return
+        try {
+            let normalized = cleaned
+            if (
+                /^[a-z0-9.-]+\.[a-z]{2,}(?:[/?#]|$)/i.test(normalized) &&
+                !/^[a-z][a-z0-9+.-]*:/i.test(normalized)
+            ) {
+                normalized = `https://${normalized}`
+            }
+            const parsed = new URL(normalized, location.href)
+            if (parsed.hostname.toLowerCase() !== allowedHost) return
+            found.add(parsed.href)
+        } catch {}
+    }
+    function extractFromText(text) {
+        if (typeof text !== 'string' || !text) return
+        const matches = text.match(jsInTextRe)
+        if (!matches) return
+        for (const m of matches) addUrl(m)
+    }
+    function walk(obj, seen = new WeakSet()) {
+        if (!obj || typeof obj !== 'object') return
+        if (seen.has(obj)) return
+        seen.add(obj)
+        if (obj.rsrcMap && typeof obj.rsrcMap === 'object') {
+            for (const r of Object.values(obj.rsrcMap)) {
+                if (!r || typeof r !== 'object') continue
+                if (r.type === 'js') addUrl(r.src || r.url || r.href || r.uri)
+            }
+        }
+        for (const value of Object.values(obj)) {
+            if (typeof value === 'string') addUrl(value)
+            else walk(value, seen)
+        }
+    }
+    for (const s of document.querySelectorAll('script[data-sjs]')) {
+        extractFromText(s.textContent || '')
+        try {
+            walk(JSON.parse(s.textContent || ''))
+        } catch {}
+    }
+    for (const s of document.querySelectorAll('script[src]')) {
+        addUrl(s.src || s.getAttribute('src'))
+    }
+    for (const l of document.querySelectorAll(
+        'link[rel="preload"][as="script"][href], link[rel="modulepreload"][href], link[rel="prefetch"][as="script"][href], link[rel="prefetch"][href]'
+    )) {
+        addUrl(l.href || l.getAttribute('href'))
+    }
+    for (const s of document.querySelectorAll('script:not([src])')) {
+        extractFromText(s.textContent || '')
+    }
+    if (typeof performance?.getEntriesByType === 'function') {
+        for (const e of performance.getEntriesByType('resource')) {
+            if (e && typeof e.name === 'string') addUrl(e.name)
+        }
+    }
+    return Array.from(found)
+}
+
+const VERSION_FN = function () {
+    try {
+        if (typeof window !== 'undefined' && window.Debug && window.Debug.VERSION) {
+            return String(window.Debug.VERSION)
+        }
+    } catch {}
+    return null
+}
+
+async function openPage(opts) {
+    const authState = opts.authState ?? null
+    const extraWaitMs = opts.extraWaitMs ?? LAZY_WAIT_MS
+
+    const { browser, page } = await connect({
+        headless: true,
+        turnstile: true,
+        connectOption: { defaultViewport: null },
+        customConfig: {},
+        plugins: []
+    })
+
+    try {
+        if (authState && fs.existsSync(authState)) {
+            const state = JSON.parse(fs.readFileSync(authState, 'utf8'))
+            if (Array.isArray(state.cookies) && state.cookies.length > 0) {
+                await page.setCookie(...state.cookies)
+            }
+        }
+        await page.goto(WHATSAPP_URL, { waitUntil: 'networkidle2', timeout: NAV_TIMEOUT_MS })
+        await new Promise((r) => setTimeout(r, extraWaitMs))
+        return { browser, page }
+    } catch (err) {
+        await browser.close().catch(() => {})
+        throw err
+    }
+}
+
+/**
+ * Discover every JS bundle URL referenced by web.whatsapp.com.
+ * Returns `{ waVersion, urls }` — sorted, deduped, host-filtered to
+ * `static.whatsapp.net`.
+ */
+async function discoverBundleUrls(opts = {}) {
+    const { browser, page } = await openPage(opts)
+    try {
+        const [urls, waVersion] = await Promise.all([
+            page.evaluate(DISCOVERY_FN),
+            page.evaluate(VERSION_FN)
+        ])
+        return { waVersion, urls: [...new Set(urls)].sort() }
+    } finally {
+        await browser.close().catch(() => {})
+    }
+}
+
+/**
+ * Discover + download every JS bundle. Writes raw bundles to
+ * `<out>/raw/<wa-version>/*.js` and a `<out>/manifest.json` index.
+ */
+async function fetchBundles(opts = {}) {
+    const out = path.resolve(opts.out ?? 'dump')
+    fs.mkdirSync(out, { recursive: true })
+
+    const { browser, page } = await openPage(opts)
+    try {
+        const [urls, waVersion] = await Promise.all([
+            page.evaluate(DISCOVERY_FN),
+            page.evaluate(VERSION_FN)
+        ])
+        const sortedUrls = [...new Set(urls)].sort()
+
+        // CORS-safe in-page fetch (same origin as the SPA).
+        const downloaded = await page.evaluate(async (uList) => {
+            const out = []
+            await Promise.all(
+                uList.map(async (u) => {
+                    try {
+                        const r = await fetch(u)
+                        const t = await r.text()
+                        out.push({ url: u, text: t })
+                    } catch {
+                        out.push({ url: u, text: '' })
+                    }
+                })
+            )
+            return out
+        }, sortedUrls)
+
+        const versionDir = path.join(
+            out,
+            'raw',
+            waVersion ?? `unknown-${new Date().toISOString().replace(/[:.]/g, '-')}`
+        )
+        fs.mkdirSync(versionDir, { recursive: true })
+
+        const bundles = []
+        let totalBytes = 0
+        for (const b of downloaded) {
+            const file = sanitizeFilename(b.url)
+            fs.writeFileSync(path.join(versionDir, file), b.text)
+            bundles.push({ url: b.url, file, bytes: b.text.length })
+            totalBytes += b.text.length
+        }
+
+        const manifest = {
+            waVersion,
+            fetchedAt: new Date().toISOString(),
+            bundleCount: bundles.length,
+            totalBytes,
+            rawDir: path.relative(out, versionDir),
+            bundles
+        }
+        const manifestPath = path.join(out, 'manifest.json')
+        fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2) + '\n')
+
+        return {
+            waVersion,
+            bundles,
+            paths: {
+                out,
+                raw: versionDir,
+                manifest: manifestPath
+            }
+        }
+    } finally {
+        await browser.close().catch(() => {})
+    }
+}
+
+function sanitizeFilename(url) {
+    // Some WA bundle URLs have very long randomised basenames that bust the
+    // 260-char path limit on Windows. Truncate to a fixed prefix and append a
+    // short hash of the full URL to preserve uniqueness.
+    const MAX_BASE_LEN = 64
+    const { createHash } = require('node:crypto')
+    let base
+    try {
+        const u = new URL(url)
+        base = u.pathname.split('/').pop() || 'bundle'
+    } catch {
+        base = url
+    }
+    base = base.replace(/[^A-Za-z0-9._-]/g, '_')
+    if (base.length <= MAX_BASE_LEN) return base
+    const ext = base.match(/\.[A-Za-z0-9]{1,8}$/)?.[0] ?? ''
+    const stem = base.slice(0, MAX_BASE_LEN - ext.length - 9) // leave room for hash
+    const hash = createHash('sha1').update(url).digest('hex').slice(0, 8)
+    return `${stem}-${hash}${ext}`
+}
+
+module.exports = { discoverBundleUrls, fetchBundles }