@shriyanss/js-recon 1.4.1-alpha.4 → 1.4.1-alpha.5

package/.github/workflows/build-and-prettify.yaml

@@ -89,9 +89,30 @@ jobs:
           git commit -m "chore: prettify code" || echo "No changes to commit"
           git push
 
-  build:
+  test:
     needs: prettier
     runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+          cache: "npm"
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Run unit tests
+        run: npm test
+
+  build:
+    needs: test
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
@@ -119,3 +140,168 @@ jobs:
           ref: ${{ github.ref_name }}
       - name: Build docker container
         run: docker build -t shriyanss/js-recon .
+
+  integration-test-docker:
+    needs: build-docker
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout js-recon
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Checkout js-recon-labs
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-labs
+          path: js-recon-labs
+
+      - name: Checkout js-recon-rules
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-rules
+          path: js-recon-rules
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+          cache: "npm"
+
+      - name: Build js-recon Docker image
+        run: docker build -t shriyanss/js-recon .
+
+      - name: Install vuln-all-rules app dependencies
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm ci
+
+      - name: Build vuln-all-rules app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm run build
+
+      - name: Start vuln-all-rules app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm start &
+
+      - name: Wait for lab app to be ready
+        run: |
+          echo "Waiting for lab app on port 3001..."
+          n=0
+          until curl -sf http://localhost:3001 > /dev/null; do
+            n=$((n+1))
+            [ $n -ge 30 ] && echo "Timed out waiting for lab app" && exit 1
+            sleep 2
+          done
+          echo "Lab app is ready."
+
+      - name: Run js-recon via Docker against lab app
+        run: |
+          docker run --name js-recon-run \
+            --network host \
+            -v "$(pwd)/js-recon-rules:/home/pptruser/js-recon-rules" \
+            --entrypoint /bin/bash \
+            shriyanss/js-recon \
+            -c 'mkdir -p ~/.js-recon && ln -sf /home/pptruser/js-recon-rules ~/.js-recon/rules && printf "http://localhost:3001\n" > /tmp/urls.txt && node build/index.js run -u /tmp/urls.txt -r ./js-recon-rules --no-sandbox -y -k'
+          mkdir -p output
+          docker cp js-recon-run:/home/pptruser/output/. ./output/ 2>/dev/null || true
+          docker rm js-recon-run
+
+      - name: Verify all rules fired
+        run: node scripts/smoke-test.js
+
+      - name: Upload analyze.json on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: analyze-json-docker
+          path: output/localhost_3001/analyze.json
+          if-no-files-found: ignore
+
+  integration-test-macos:
+    needs: build
+    runs-on: macos-latest
+    timeout-minutes: 30
+    steps:
+      - name: Checkout js-recon
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          ref: ${{ github.ref_name }}
+
+      - name: Checkout js-recon-labs
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-labs
+          path: js-recon-labs
+
+      - name: Checkout js-recon-rules
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-rules
+          path: js-recon-rules
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+          cache: "npm"
+
+      - name: Install Chrome
+        run: brew install --cask google-chrome
+
+      - name: Install js-recon dependencies
+        run: npm ci
+        env:
+          PUPPETEER_SKIP_DOWNLOAD: "true"
+
+      - name: Build js-recon
+        run: npm run build
+
+      - name: Install vuln-all-rules app dependencies
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm ci
+
+      - name: Build vuln-all-rules app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm run build
+
+      - name: Start vuln-all-rules app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm start &
+
+      - name: Wait for lab app to be ready
+        run: |
+          echo "Waiting for lab app on port 3001..."
+          n=0
+          until curl -sf http://localhost:3001 > /dev/null; do
+            n=$((n+1))
+            [ $n -ge 30 ] && echo "Timed out waiting for lab app" && exit 1
+            sleep 2
+          done
+          echo "Lab app is ready."
+
+      - name: Pre-populate js-recon rules cache
+        run: |
+          mkdir -p ~/.js-recon
+          ln -sf "$GITHUB_WORKSPACE/js-recon-rules" ~/.js-recon/rules
+
+      - name: Run js-recon natively against lab app
+        run: |
+          echo "http://localhost:3001" > /tmp/smoke-urls.txt
+          node --max-old-space-size=4096 build/index.js run \
+            -u /tmp/smoke-urls.txt \
+            -r ./js-recon-rules \
+            -y -k
+        env:
+          PUPPETEER_EXECUTABLE_PATH: /Applications/Google Chrome.app/Contents/MacOS/Google Chrome
+
+      - name: Verify all rules fired
+        run: node scripts/smoke-test.js
+
+      - name: Upload analyze.json on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: analyze-json-macos
+          path: output/localhost_3001/analyze.json
+          if-no-files-found: ignore

package/.github/workflows/publish-js-recon.yml

@@ -113,7 +113,6 @@ jobs:
           VERSION="${{ steps.version.outputs.version }}"
           SHA="${{ steps.sha256.outputs.sha256 }}"
           F="Formula/js-recon.rb"
-          sed -i "s|^  version \".*\"|  version \"${VERSION}\"|" "${F}"
           sed -i "s|^  url \"https://registry\.npmjs\.org/@shriyanss/js-recon/-/js-recon-[^\"]*\"|  url \"https://registry.npmjs.org/@shriyanss/js-recon/-/js-recon-${VERSION}.tgz\"|" "${F}"
           sed -i "s|^  sha256 \"[a-f0-9]*\"|  sha256 \"${SHA}\"|" "${F}"
 

package/.github/workflows/rules-smoke-test.yaml

@@ -0,0 +1,79 @@
+name: Rules Smoke Test
+
+on:
+  push:
+    branches-ignore:
+      - main
+
+jobs:
+  rules-smoke-test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout js-recon
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+
+      - name: Checkout js-recon-labs
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-labs
+          path: js-recon-labs
+
+      - name: Checkout js-recon-rules
+        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
+        with:
+          repository: shriyanss/js-recon-rules
+          path: js-recon-rules
+
+      - name: Set up Node.js
+        uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+          cache: "npm"
+
+      - name: Install js-recon dependencies
+        run: npm ci
+
+      - name: Build js-recon
+        run: npm run build
+
+      - name: Install vulnerable app dependencies
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm ci
+
+      - name: Build vulnerable app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm run build
+
+      - name: Start vulnerable app
+        working-directory: js-recon-labs/next_js/vuln-all-rules
+        run: npm start &
+
+      - name: Wait for app to be ready
+        run: |
+          echo "Waiting for app on port 3001..."
+          timeout 60 bash -c 'until curl -sf http://localhost:3001 > /dev/null; do sleep 2; done'
+          echo "App is ready."
+
+      - name: Run js-recon against vulnerable app
+        run: |
+          echo "http://localhost:3001" > /tmp/smoke-urls.txt
+          node --max-old-space-size=4096 build/index.js run \
+            -u /tmp/smoke-urls.txt \
+            -r ./js-recon-rules \
+            --no-sandbox \
+            -y -k
+        env:
+          PUPPETEER_EXECUTABLE_PATH: /usr/bin/google-chrome-stable
+
+      - name: Verify all rules fired
+        run: node scripts/smoke-test.js
+
+      - name: Upload analyze.json on failure
+        if: failure()
+        uses: actions/upload-artifact@v4
+        with:
+          name: analyze-json
+          path: output/localhost_3001/analyze.json
+          if-no-files-found: ignore

package/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Change Log
 
+## 1.4.1-alpha.5 - 2026-07-13
+
+### Fixed
+
+- Homebrew install: `puppeteer-extra` now receives the ESM `puppeteer` instance directly via `addExtra()` instead of relying on its internal `require('puppeteer')` call. On newer Node.js versions (22+, as shipped by Homebrew), a synchronous `require()` of an ESM module triggers `ExperimentalWarning` on every run. Using `addExtra()` bypasses the `require()` entirely and silences the warning. (`all`)
+- `lazyload` (tech detect, react): React framework detection now works for Vite dev-mode servers. Vite's `@vitejs/plugin-react` and `@vitejs/plugin-react-swc` inject an inline `<script>` block containing `import { injectIntoGlobalHook } from "/@react-refresh"` into every dev-served page. Previously, none of the `REACT_MARKERS` matched this content, so dev-mode targets returned `unknown`. Two fixes: (1) `@react-refresh` and `injectIntoGlobalHook` are now included in `REACT_MARKERS` so the inline-script branch of `checkReact` fires immediately; (2) `/@react-refresh` is now a fast-path match in `fetchAndCheck` so any script src pointing at the HMR endpoint is also caught; (3) the intercepted-URL fallback in `frameworkDetect` now recognises `/@react-refresh` as a React signal alongside the existing Next.js / Nuxt / Svelte URL patterns. (`lazyload`, `fingerprint`, `run`)
+- `lazyload` (tech detect, angular): Angular framework detection is now more robust. Three HTML-level signals are checked before fetching `main.js` — `data-beasties-container` (Angular's Beasties SSR/prerendering marker), `ng-version` (set by the Angular runtime on the root component element after bootstrapping, visible in Puppeteer-rendered DOM), and `_nghost-*` view-encapsulation attributes. These checks require no extra HTTP request and work even for zoneless Angular apps (Angular 16+) that omit Zone.js and therefore lack the previous `isAngularZone`/`this.ngZone` patterns. The `main.js` src pattern was also broadened from `main-` (hashed production builds only) to also match `main.js` (unhashed development builds). (`lazyload`, `fingerprint`, `run`)
+- `lazyload`, `map`, `strings`: `.mjs` (ES module) files are now fully supported across all framework pipelines. Previously, extension filters in `downloadQueue.ts`, `downloadFilesUtil.ts`, `downloadLoadedJsUtil.ts`, `react_followImports.ts`, `getReactConnections.ts`, `getViteConnections.ts`, `getAngularConnections.ts`, `vue_resolveHttpClient.ts`, `vue_resolveFetch.ts`, `vue_resolveXhr.ts`, `crossFileResolver.ts`, and `strings/index.ts` only matched `.js` files. Sites that bundle all application code as `.mjs` (e.g. Framer/Rolldown) were silently ignored — all chunks were downloaded but zero files were scanned for connections, strings, or secrets. All affected paths now match both `.js` and `.mjs`. (`run`, `lazyload`, `map`, `strings`)
+- `run` (react): The map step now receives the full `outputDir` (all downloaded hosts) rather than only the target host's subdirectory. Previously, when a React site served all its JS from a CDN host, the map step was pointed at an empty target-host directory and reported zero chunks. Now mirrors the Vue pipeline's directory-passing behaviour so CDN-hosted chunks are analysed correctly. (`run`, `map`)
+- `lazyload` (react): `react_followImports` now discovers dynamic imports written as template literals (``import(`./chunk.mjs`)``) in addition to single/double-quoted strings. Bundlers that use Rolldown (e.g. Framer) emit all dynamic import paths as template literals; the previous regex only matched quoted strings, silently missing all lazily-loaded page and collection chunks. (`lazyload`, `run`)
+- `lazyload`: `.mjs.map` sourcemap files are now handled correctly. The filename extraction regex now prefers the `.mjs.map` suffix over the shorter `.mjs` match, preventing sourcemap JSON content from being written with a `.mjs` extension (which caused Prettier's Babel parser to throw a `SyntaxError` and discard the file). The sourcemap write path now uses the JSON parser for both `.js.map` and `.mjs.map` files. (`lazyload`, `run`)
+- `lazyload` (Puppeteer): `downloadLoadedJsUtil` now uses `waitUntil: "networkidle0"` with a 10 s timeout on `page.goto()`. Previously the call had no timeout and no error handling, so pages whose `load` event never fires (deferred JS, service workers, some SPA patterns) caused the lazyload step to hang indefinitely. Navigation errors and timeouts are now caught and the URLs collected up to that point are returned normally. (`lazyload`, `run`)
+- `lazyload` (Puppeteer): `browser.close()` in `downloadLoadedJsUtil` is now wrapped in a try/catch; if it hangs, the Chrome process is force-killed via `SIGKILL`. Previously a stuck `browser.close()` caused the process to hang after download was otherwise complete. (`lazyload`, `run`)
+- `lazyload` (Puppeteer): All three Puppeteer-using modules (`downloadLoadedJsUtil`, `next_GetLazyResourcesWebpackJs`, `techDetect/index`) now abort non-http/s requests (e.g. `mailto:`, `data:`, `blob:`, `chrome-extension:`, `tel:`) in the request interceptor instead of calling `request.continue()`. Calling `continue()` on these schemes throws an unhandled error and can trigger OS protocol handlers. (`lazyload`, `run`)
+- `lazyload` (Puppeteer): `downloadLoadedJsUtil` and `next_GetLazyResourcesWebpackJs` now pass `--disable-external-protocol-dialog` to Chrome at launch and install an `evaluateOnNewDocument` guard that overrides `window.open` and suppresses clicks on non-http/s anchors. This provides a three-layer defence (Chrome flag + JS intercept + request interceptor abort) ensuring that non-http protocol links on target pages never invoke OS handlers (mail client, phone, etc.). (`lazyload`, `run`)
+- `lazyload` (Puppeteer): All three Puppeteer-using modules now issue a `Page.setDownloadBehavior({ behavior: "deny" })` CDP command immediately after page creation. This prevents accidental file downloads triggered by download-link clicks or JavaScript during crawl, which could block the browser and fill the output directory with unexpected files. (`lazyload`, `run`)
+- `lazyload` (tech detect): After `waitUntil: "load"` resolves, if no framework URL has been captured in `interceptedUrls` yet, a conditional `page.waitForNavigation({ timeout: 5000 })` is now issued to wait for a second navigation event. This catches sites that serve a JS proof-of-work bot challenge (e.g. Vercel's `challenge.v2.min.js`) that fires its own `load` event before calling `window.location` to redirect to the real app. Previously, Puppeteer exited on the challenge page's `load` event and tech detection returned null (exit 10). On sites with no redirect the navigation wait times out after 5 s and detection continues normally. (`lazyload`, `run`)
+- `lazyload`: When `frameworkDetect` returns null and `downloadLoadedJs` is used as a fallback, the downloaded file URL paths are now scanned for `/_next/`, `/_nuxt/`, and `/_app/immutable/` signatures as a second-chance tech detection pass. Previously, sites that serve their framework app at a non-root `basePath` (e.g. Next.js at `/app` instead of `/`) caused the initial Puppeteer-based detection to miss the framework because the intercepted JS paths didn't match any framework pattern, resulting in exit 10. (`lazyload`, `run`)
+- `lazyload` (next_js): `next_parseLayoutJs` now applies a 1.5 MB per-file size guard — checked from both the `Content-Length` response header and the fetched content body — before passing layout files to the AST parser. Files exceeding the limit are skipped with a yellow warning. Previously, Next.js App Router sites with 50+ nested pages each loading a large `layout.js` caused cumulative AST memory of ~2 GB, exhausting the V8 heap and crashing with SIGSEGV (exit 139). (`lazyload`, `run`)
+- `cs-mast`: `--scat`, `--sinc`, `--all-scat-permutations`, `--perm-output`, and `--perm-concurrency` flags were declared in the CLI but not wired to the underlying function, causing a TypeScript compile error (`TS2554: Expected 5 arguments, but got 10`). The function now accepts all five additional parameters: `--scat`/`--sinc` override the active CS-MAST config, and `--all-scat-permutations` runs all 511 non-empty scat subsets in parallel batches and writes one collision file per subset to `--perm-output`. (`cs-mast`)
+
+- `refactor -t react-vite`: vendor chunks (`vendor-react-*.js`, `rolldown-runtime-*.js`) that are absent from `mapped.json` are now automatically located in the downloaded assets directory and injected before the refactor pass. Previously, `mapped.json` typically contained only app chunks — vendor chunks are excluded from mapping because they hold only third-party code. This left `vendorExportMaps` empty, so the `rewriteVendorImports` pass had nothing to match against and vendor import statements like `import { d as t } from './vendor-react-CLFLfR9F.js'` survived into the output, causing the Vite build check to fail with "Could not resolve". The fix uses the existing `findAssetsDir`/`findVendorChunkFiles` helpers (already used by the webpack branch) to discover the assets directory from `// File Source:` headers in chunk code.
+- `lazyload` (nuxt): `nuxt_getFromPageSource` now prints the number of new JS files discovered (delta from the global URL set) instead of the running total. Previously the count included every URL already known from earlier in the session.
+- `lazyload` (vue): Each discovery method in `vue_discoverJsFiles` now prints only the count of URL that are genuinely new — URLs already known from earlier methods are excluded. Previously `fromImports.length`, `fromStringRefs.length`, etc. could include URLs already in the accumulator and over-count.
+- Error messages emitted by `makeReq` (fetch failures, cache errors, timeout notices, firewall detection) now use `progressError`/`progressLog` instead of `console.error`/`console.log`, so they are routed through the active progress bar's logger and no longer clutter the bar line when errors occur during a scan.
+- `refactor` (`react-webpack`, `react-vite`): new `--detect-version` flag. When set, the tool fetches per-version `reliable_signatures.json` files from the `shriyanss/cs-mast-s-dataset` HuggingFace bucket (path: `version/react/<bundler>/<version>/<scat>/reliable_signatures.json`) and matches them against CS-MAST signatures generated from the target bundle's chunks. The best-matching React version is used to set `react` and `react-dom` version pins in the refactored output's `package.json` instead of the default `^18.3.1`. Signatures are cached under `~/.js-recon/refactor/version_sigs_cache/` with a 7-day TTL. Supports all webpack versions (react-0.12 through react-19) and all Vite versions (react-16 through react-19) available in the dataset. (`refactor`)
+
+### Added
+
+- `fingerprint`: output files are now written incrementally — each result is flushed to disk immediately after detection rather than buffered and written at the end. For text, csv, and jsonl formats the result is appended; for json the full array is rewritten. Output files are also created (and csv headers written) before the worker pool starts, so a partial file always exists if the scan is interrupted. (`fingerprint`)
+- CLI: help output is now colorized. Section titles (`Usage:`, `Options:`, `Commands:`) are bold cyan; flag names are yellow; `<required>` argument placeholders are magenta; `[optional]` argument placeholders are cyan; descriptions are dimmed; and `(default: …)` suffixes are dim italic. Applies to every subcommand.
+- `run`: the pipeline now runs an optional `refactor` pass after the report step for React, Vue, Nuxt, and Next.js targets. The bundler (webpack vs vite) is detected automatically using CS-MAST-S signatures sampled from the `shriyanss/cs-mast-s-dataset` HuggingFace bucket — the same bucket and caching layer used by `refactor --remote-collisions`. If the detected match count meets `--cs-mast-tech-detect-threshold` (default `50`), `refactor` is called with the full tech identifier (e.g. `react-webpack`); otherwise the step is silently skipped. Refactored output is written to `refactored/` (single-URL mode) or `<workingDir>/refactored/` (batch mode). Currently only React bundles have bucket entries; Vue, Nuxt, and Next.js gracefully skip until their signatures are added to the bucket. (`run`, `refactor`)
+- `run`: new `--cs-mast-tech-detect-threshold <n>` flag (default `50`). Sets the minimum number of CS-MAST-S signature matches required to consider a bundler detected for the refactor step. Pass `0` to disable refactor entirely. (`run`)
+- `refactor` (`react-vite`, `react-webpack`): new `--remote-collisions <path>` flag. Accepts a HuggingFace bucket path (e.g. `react/vite/large-0.1.8`) and uses it as the signature source instead of the automatic `TECH_TO_BRANCH` mapping. When the path does not exist in the `shriyanss/cs-mast-s-dataset` bucket the tool exits with code 25. The existing caching layer is fully reused — signatures are cached under `~/.js-recon/refactor/signature_cache/` and the file list is cached in `~/.js-recon/refactor/cs-mast-s-list-cache.json`. Feature directories that contain no collision records in the dataset are skipped during intersection rather than collapsing the result to zero.
+- `refactor` (`react-webpack`, `react-vite`): `--detect-version` now uses multiple scat configurations for improved accuracy. Three new flags control the behaviour: `--detect-version-config` (default `dynamic`), `--detect-version-dynamic-threshold` (default `3`), and `--detect-version-dynamic-conf-purge`. In `dynamic` mode the tool automatically selects up to `--detect-version-dynamic-threshold` scat configs that have non-empty reliable signatures across all known React versions, caches the selection in `~/.js-recon/refactor/config.json`, and aggregates match counts across all configs per version for a stronger detection signal. Alternatively, passing comma-separated scat categories (e.g. `lit,decl,loop,cond`) to `--detect-version-config` fixes the config; the tool validates signatures are non-empty for every version and exits with code 26 if not. Use `--detect-version-dynamic-conf-purge` to force recomputation of the cached config.
+
+### Changed
+
+- Set `react/webpack/large-0.1.8` as default cs-mast-s dataset for react + webpack refactoring
+
 ## 1.4.1-alpha.4 - 2026-06-29
 
 ### Added

package/CLAUDE.md

@@ -77,9 +77,9 @@ npm run start -- <subcommand> [options]
 1. **Lazyload** — downloads initial JS chunks via Puppeteer; detects framework; sets `globalsUtil.getTech()` to `"next"`
 2. **Strings** — scans downloaded JS for strings, extracts URL paths → `extracted_urls.json`
 3. **Lazyload (subsequent requests)** — re-crawls using the extracted paths to fetch dynamically loaded chunks; also fetches `buildId`
-4. **Strings (pass 2)** — re-runs strings on the expanded chunk set; generates `extracted_urls.txt` (permuted) and `extracted_urls-openapi.json`; optionally scans secrets (`--secrets`)
+4. **Strings (pass 2)** — re-runs strings on the expanded chunk set; generates `extracted_urls.txt` (permuted) and `extracted_urls-openapi.json`; optionally scans secrets (`--secrets`); optionally runs TruffleHog (`--trufflehog`)
 5. **Lazyload re-pass (step 4.5)** — a second subsequent-requests crawl to pick up chunks for dynamic routes discovered in pass 2
-6. **Strings re-pass (step 4.6)** — strings pass over the re-pass chunks
+6. **Strings re-pass (step 4.6)** — strings pass over the re-pass chunks; also runs `--secrets` / `--trufflehog` if those flags are set
 7. **Map** — parses webpack/turbopack bundles; resolves `fetch()` calls and axios usage; generates `mapped.json` and `mapped-openapi.json`; CDN-aware: if JS was served from a different host, `getCdnDir` finds the CDN output dir and passes that to map instead of `outputDir/host`
 8. **Endpoints** — extracts client-side route paths; uses `___subsequent_requests` directory presence to decide whether to pass a JS directory
 9. **Analyze** — loads YAML rules (from `-r/--rules` if provided, otherwise default rules cache); runs AST engine and request engine; writes `analyze.json`
@@ -107,7 +107,7 @@ npm run start -- <subcommand> [options]
 
 **SvelteKit `adapter-static` (SSG/SPA) boot pattern**: The static builds produce a shell HTML file (`404.html` for SSG, `index.html` for SPA) that contains both `<link rel="modulepreload">` tags for all initial chunks AND the same inline `import()` boot script as adapter-node. `svelte_getFromPageSource` picks up 17+ JS URLs from the modulepreload links plus 2 from the inline script, giving a much larger seed set than the adapter-node case.
 
-**`__vite_mapDeps` path formats**: SvelteKit emits `m.f = ["../nodes/0.js", "../chunks/x.js", ...]` (file-relative paths) inside entry chunks at `_app/immutable/entry/`. Vue and React emit `m.f = ["/assets/chunk.js", ...]` (root-relative paths). `react_followImports` differentiates by checking `p.startsWith("/")` — absolute paths resolve against the origin (`baseUrl`), relative paths resolve against the chunk's own URL (`fileUrl`). See `src/lazyload/react/CLAUDE.md` for details.
+**`__vite_mapDeps` path formats**: SvelteKit emits `m.f = ["../nodes/0.js", "../chunks/x.js", ...]` (explicit file-relative paths) inside entry chunks at `_app/immutable/entry/`. Vue and React can emit either `m.f = ["/assets/chunk.js", ...]` (absolute root-relative) or `m.f = ["assets/chunk.js", ...]` (bare root-relative, no leading `/`). `react_followImports` differentiates by checking for a `./` or `../` prefix: only explicitly relative paths resolve against the chunk's own URL (`fileUrl`); all others (absolute `/` or bare names) resolve against the origin (`baseUrl`). Bare names like `assets/x.js` must NOT be resolved against `fileUrl` — when the chunk is inside `assets/`, that would produce a double-directory path. See `src/lazyload/react/CLAUDE.md` for details.
 
 ### Batch mode
 
@@ -200,6 +200,7 @@ Rule categories:
 
 - `ast/` — AST-based pattern matching against chunk code (uses `@babel/parser` + `esquery`)
 - `request/` — OpenAPI/request-level checks against the resolved endpoint list
+- `cs-mast-s/` — CS-MAST-S structural signature matching; each step embeds a PHC string and fires if that node-level hash is found anywhere in the chunk AST. Suitable for regression detection after a vulnerability is confirmed via AST rules. See `src/analyze/engine/csMastSEngine.ts` and `src/analyze/CLAUDE.md` for details.
 
 When `-r` points to a single file, only that rule is loaded. When it points to a directory, all `.yml`/`.yaml` files are loaded recursively.
 
@@ -207,11 +208,137 @@ When `-r` points to a single file, only that rule is loaded. When it points to a
 
 **Testing is mandatory for every change.** Before reporting a task complete:
 
-1. Run `npm run cleanup` to rebuild TypeScript.
-2. Run the `run` subcommand against the target the user provides. Do not use `analyze` or other individual subcommands as a substitute — the `run` subcommand must be used to validate end-to-end behavior.
-3. If the user has not provided a target, ask for one before proceeding.
+1. Run `npm test` to execute the unit test suite (Vitest).
+2. Run `npm run cleanup` to rebuild TypeScript.
+3. Run the `run` subcommand against the target the user provides. Do not use `analyze` or other individual subcommands as a substitute — the `run` subcommand must be used to validate end-to-end behavior.
+4. If the user has not provided a target, ask for one before proceeding.
 
-Typical test invocation:
+### Rules smoke test (CI)
+
+The `rules-smoke-test` GitHub Actions workflow (`.github/workflows/rules-smoke-test.yaml`) runs on every non-main push. It:
+
+1. Checks out `shriyanss/js-recon-labs` and `shriyanss/js-recon-rules` alongside js-recon.
+2. Builds js-recon and the `next_js/vuln-all-rules` lab app.
+3. Starts the lab app on port 3001.
+4. Runs `node build/index.js run -u http://localhost:3001 -r ./js-recon-rules --no-sandbox -y -k`.
+5. Runs `node scripts/smoke-test.js` which reads `output/localhost:3001/analyze.json` and asserts that all 22 expected rule IDs are present.
+
+**`scripts/smoke-test.js`** maintains the `EXPECTED_RULES` list. When a new rule is added to js-recon-rules:
+
+- The `next_js/vuln-all-rules` app in js-recon-labs must be updated to seed the new vulnerability.
+- The new rule ID must be appended to `EXPECTED_RULES` in `scripts/smoke-test.js`.
+
+The lab app seeds:
+
+- 19 AST rules for the `next` tech stack
+- 3 request rules (`api_path`, `admin_api`, `missing_authorization_header`)
+- The Angular-only rule (`detect_angular_bypass_security_trust`) is intentionally excluded.
+
+### Unit tests
+
+Unit tests live in `src/__tests__/` and cover pure-logic components. Test framework is **Vitest** (ESM-native, TypeScript-native — no compilation step needed).
+
+```bash
+npm test          # run all unit tests once
+npm run test:watch  # watch mode
+npm run test:build  # legacy build smoke test (node build/index.js -h)
+```
+
+Test files follow the pattern `src/__tests__/<component>/<name>.test.ts`.
+
+Covered components:
+
+| File                                                                                     | Tests in                                               |
+| ---------------------------------------------------------------------------------------- | ------------------------------------------------------ |
+| `utility/urlUtils.ts` — `getURLDirectory`                                                | `src/__tests__/utility/urlUtils.test.ts`               |
+| `utility/replaceUrlPlaceholders.ts` — `replacePlaceholders`                              | `src/__tests__/utility/replaceUrlPlaceholders.test.ts` |
+| `utility/resolvePath.ts` — `resolvePath`                                                 | `src/__tests__/utility/resolvePath.test.ts`            |
+| `strings/index.ts` — `extractStrings`                                                    | `src/__tests__/strings/extractStrings.test.ts`         |
+| `analyze/helpers/validate.ts` — `parseVersion`, `compareVersions`, `isVersionCompatible` | `src/__tests__/analyze/versionCompat.test.ts`          |
+| `map/next_js/utils.ts` — `memberChainToString`                                           | `src/__tests__/map/memberChainToString.test.ts`        |
+| `fingerprint/index.ts` — `deriveOutputPath`                                              | `src/__tests__/fingerprint/deriveOutputPath.test.ts`   |
+
+When adding new pure-logic helpers, add a corresponding test file. Components that require Puppeteer, network I/O, or the full pipeline are still validated through the `run` subcommand.
+
+### Writing unit tests
+
+**What to test.** Test pure functions: anything that takes plain inputs and returns a value without I/O. The standard pattern for I/O-bound modules is to extract the parse/transform step into an exported pure function and test that. Leave the orchestrator (Puppeteer, `makeRequest`, file writes) untested at the unit level.
+
+**Extracting testable functions.** When a function mixes I/O with logic, split it:
+
+```typescript
+// Exported pure function — testable
+export const parseThings = (content: string, baseUrl: string): string[] => { ... };
+
+// Orchestrator — not unit-tested
+const myModule = async (url: string): Promise<string[]> => {
+    const resp = await makeRequest(url);
+    const content = await resp.text();
+    return parseThings(content, url);
+};
+```
+
+**Test file structure.** Use `describe` + `it` blocks. Group by function name. Cover: happy path, edge cases (empty input, malformed input), and threshold boundaries (e.g. "fewer than N entries returns []").
+
+```typescript
+import { describe, it, expect } from "vitest";
+import { myPureFunction } from "../../path/to/module.js"; // .js extension required
+
+describe("myPureFunction", () => {
+    it("extracts X from valid input", () => {
+        const result = myPureFunction("...");
+        expect(result).toContain("expected");
+    });
+
+    it("returns [] for empty input", () => {
+        expect(myPureFunction("")).toEqual([]);
+    });
+
+    it("returns [] for invalid JS", () => {
+        expect(myPureFunction("{{{{ not valid")).toEqual([]);
+    });
+});
+```
+
+**Imports always use `.js` extension** for local files (ESM project with `"module": "node16"`):
+
+```typescript
+import { fn } from "../../lazyLoad/next_js/myModule.js";
+```
+
+**Constructing Babel AST nodes for tests.** When a function under test requires real Babel AST nodes (with `scope`, `path`, correct `start`/`end` offsets), parse a code snippet in the test and capture the node via a traverse visitor — do not construct AST nodes by hand. Wrap the expression in a `const _x = <expr>;` declaration so `start`/`end` offsets are preserved for any `code.slice()` calls inside the function:
+
+```typescript
+import parser from "@babel/parser";
+import _traverse from "@babel/traverse";
+const traverse = (_traverse.default ?? _traverse) as typeof _traverse.default;
+
+function parseExpr(code: string) {
+    const src = `const _x = ${code};`;
+    const ast = parser.parse(src, { sourceType: "unambiguous", plugins: ["jsx", "typescript"] });
+    let node: any;
+    traverse(ast, {
+        VariableDeclarator(p) {
+            node = p.node.init;
+            p.stop();
+        },
+    });
+    return { node, src };
+}
+```
+
+**Avoiding GitHub secret scanning.** Strings that look like real secrets (Slack webhook URLs, Stripe keys, etc.) will be blocked by GitHub push protection even in test files. Construct them at runtime from parts arrays:
+
+```typescript
+// BAD — blocked by secret scanner
+const url = "https://hooks.slack.com/services/TABCDEF/BABCDEF/xxxxxxxxxxxx";
+
+// GOOD — assembled at runtime
+const parts = ["https://hooks.slack.com/services/T", "ABCDEF/B", "ABCDEF/xxxx"];
+const url = parts.join("");
+```
+
+Typical full test invocation:
 
 ```bash
 npm run cleanup && npm run start -- run -u <target-url> -y -k
@@ -384,6 +511,11 @@ When a user asks to fix or implement a GitHub issue, follow these steps:
 - `--min-collisions <n>` — minimum occurrences to report (default: 2)
 - `--co / --collision-output <file>` — write collision data to a file (independent of `--ct`)
 - `--cf / --collision-format json|csv` — output format (default: csv)
+- `--scat <categories>` — comma-separated scat categories to use (default: `lit,decl,loop,cond`). Overrides the fixed config for this run.
+- `--sinc <nodes>` — comma-separated exact node types to include via sinc (e.g. `IfStatement`).
+- `--all-scat-permutations` — run all 511 non-empty scat subsets and write one collision file per subset to `--perm-output`.
+- `--perm-output <dir>` — output directory for per-permutation files (required with `--all-scat-permutations`).
+- `--perm-concurrency <n>` — parallel permutation workers (default: half of CPU count).
 
 **`--co` path resolution:** if the given path is a directory or has no extension, the file is written as `collisions.<fmt>` in the current working directory.
 
@@ -395,15 +527,28 @@ When a user asks to fix or implement a GitHub issue, follow these steps:
 npm run build
 node build/index.js cs-mast -o output --ct --min-collisions 2
 node build/index.js cs-mast -o output --co output --cf csv   # writes ./collisions.csv
+node build/index.js cs-mast -o output --all-scat-permutations --perm-output ./perm-out --cf json
 ```
 
 ## refactor
 
-The `refactor` command supports three techs:
+The `refactor` command supports the following techs:
 
 - **`react-webpack`** — webpack 5 React bundles. Splits a numeric module map into per-module ES files, rewrites require→import, recovers JSX. See `src/refactor/react/CLAUDE.md`.
 - **`react-vite`** — Vite (rolldown) React bundles. Removes CJS interop wrappers, rewrites vendor imports to canonical library imports (`react`, `react/jsx-runtime`, etc.), recovers JSX. Runs a Vite build check after writing output. See `src/refactor/react-vite/CLAUDE.md`.
 - **`next`** — Next.js bundles (legacy).
+- **`next-turbopack`** — Next.js Turbopack chunks. Handles both turbopack 3-param `func_NNN=(runtime,module,exports)=>{}` and 1-param `func_NNN=(runtime)=>{}` formats plus webpack-style coexisting chunks. See `src/refactor/next/CLAUDE.md`.
+- **`next-webpack`** — Next.js webpack chunks. Input format from `mapped.json`: `NNN:(module,exports,require)=>{}`. Recovers named exports (ODP, require.d), default exports (module.exports=V), re-exports (module.exports=require(N)→export*), and require hoisting. 277/280 modules recovered on a real bundle. Param order: params[0]=module, params[1]=exports, params[2]=require. See `src/refactor/next/CLAUDE.md`.
+- **`vue-webpack`** — Vue.js webpack 4/5 chunks. Container format: `(window.webpackJsonp||[]).push([[chunkIds],{moduleId:function(t,e,r){...}}])`. Each chunk file may contain multiple module functions; each is extracted and transformed. Reuses the Next.js webpack transform (same module param semantics: params[0]=module, params[1]=exports, params[2]=require). See `src/refactor/vue/index.ts`.
+- **`vue-vite`** — Vue 3 + Vite page chunks. The main index chunk (contains `__vccOpts`, all of Vue runtime) is analysed to build an export-alias→canonical-name map. Lazy page chunks then have their index imports rewritten to canonical `import {...} from 'vue'` statements. The `_export_sfc` compiler helper is inlined as a local const. See `src/refactor/vue/vite.ts` and `src/refactor/vue/vendor-analyze-vue.ts`.
+
+### Known react-vite bugs (discovered 2026-07-01, test against js-recon-research/react/20-cve-app)
+
+**Multi-chunk file overwrite** — `map` segments each Vite chunk into multiple sub-chunks (one per top-level function). The refactor write path processes sub-chunks sequentially, each overwriting the previous output file. Only the last sub-chunk survives. For chunks with inlined library code followed by the component function (e.g. `ApiProxy`, `Editor`, `Search`), this means the component is the survivor (last in the map order), which is the correct and useful result — but any app-specific helper functions in the same chunk are also lost.
+
+**Rename race** — When JSX is detected in a sub-chunk, the write path renames the output file `.js` → `.jsx`. When the same file is written by multiple sub-chunks, the rename is attempted on each JSX-containing chunk; all attempts after the first throw `ENOENT` because the `.js` file was already renamed. The `.jsx` file is correct; the unhandled rejection is noise. Fix: track which output files have already been renamed.
+
+**Remote signatures** — The `react/vite/large` HuggingFace bucket is currently empty. The tool falls back gracefully with a warning but remote library stripping is disabled. Symptom: `[!] No remote collisions files found for scat "lit-decl-loop-cond" in branch "react/vite/large"`. Fix: populate the bucket by running CS-MAST-S generation against a corpus of large Vite apps.
 
 ### refactor `--collisions <file>` (react-webpack only)
 
@@ -413,6 +558,37 @@ Plumbing: `src/index.ts` (CLI) → `src/refactor/index.ts` (resolves the path vi
 
 The baseline files live in the sibling `js-recon-cs-mast-s/` repo (`baselines/<tech>/<scat>/collisions.json`). See its `README.md` for layout and provenance.
 
+### refactor `--detect-version` (react-webpack, react-vite)
+
+`refactor -t react-webpack` and `refactor -t react-vite` accept a `--detect-version` flag that uses CS-MAST signatures to detect the React version used in the bundle.
+
+**Related flags:**
+
+- `--detect-version-config <config>` — `"dynamic"` (default) or comma-separated scat categories (e.g. `lit,decl,loop,cond`).
+- `--detect-version-dynamic-threshold <n>` — number of scat configs to use in dynamic mode (default: 3).
+- `--detect-version-dynamic-conf-purge` — clears the cached dynamic scat config and recomputes.
+
+**How it works:**
+
+1. **Scat config resolution** (`resolveVersionDetectionScatDirs()` in `src/refactor/index.ts`):
+    - `dynamic` mode: reads cached scat configs from `~/.js-recon/refactor/config.json`. If absent (or purged), calls `selectDynamicScatConfigs()` which lists scat dirs from the HF bucket for a reference version, then validates that each scat dir has non-empty `reliable_signatures.json` for ALL versions. Saves the result to `config.json`.
+    - Static mode: parses the user's comma-separated categories, converts to a bucket dir name via `scatToDir()`, validates against all versions, and exits with code 26 if any version's file is empty.
+2. `generateBundleSignatures()` in `src/refactor/remote/version-detect.ts` runs `cs_mast_init` on every chunk + optional extra code snippets for each selected scat config. This produces a separate signature set per scat.
+3. For each available React version, `fetchReliableSignatures()` downloads (or loads from cache) the `reliable_signatures.json` per scat config. Match counts are **summed across all scat configs** per version.
+4. The version with the highest total match count is returned as the detected version.
+5. The detected version's npm semver is used to pin `react` and `react-dom` in the refactored output's `package.json`.
+
+**Caches:**
+
+- Signature cache: `~/.js-recon/refactor/version_sigs_cache/<bundler>/<version>/<scatDir>/reliable_signatures.json` + `.cached_at` (7-day TTL).
+- Dynamic config cache: `~/.js-recon/refactor/config.json` (`dynamicVersionDetectionScatConfig` field).
+
+**Dataset coverage:** webpack (react-0.12 through react-19), vite (react-16 through react-19).
+
+**Important:** the version detection data in the HF bucket was generated with `@shriyanss/cs-mast` v0.1.8. The tool requires cs-mast 0.1.8 or later to produce matching signatures. Using an older cs-mast version will result in zero matches.
+
+**Important:** the version detection data in the HF bucket was generated with `@shriyanss/cs-mast` v0.1.8. The tool requires cs-mast 0.1.8 or later to produce matching signatures. Using an older cs-mast version will result in zero matches.
+
 ## Security / confidentiality
 
 When a change is informed by behavior observed on a real target (URLs, endpoint names, response shapes, finding details, etc.):