npm - @dpopsuev/web-spider - Versions diffs - 0.10.4 → 0.10.5 - Mend

@dpopsuev/web-spider 0.10.4 → 0.10.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (67) hide show

package/dist/batch.js.map +1 -0
package/dist/cache.js.map +1 -0
package/dist/convert.js.map +1 -0
package/dist/crawl.js.map +1 -0
package/dist/disk-cache.js.map +1 -0
package/dist/graph.js.map +1 -0
package/dist/index.js.map +1 -0
package/dist/parse.js.map +1 -0
package/dist/playwright.js.map +1 -0
package/dist/ports.js.map +1 -0
package/dist/robots.js.map +1 -0
package/dist/search.js.map +1 -0
package/dist/sitemap.js.map +1 -0
package/dist/spider.js.map +1 -0
package/dist/throttle.js.map +1 -0
package/dist/tree.js.map +1 -0
package/dist/types.js.map +1 -0
package/dist/views.js.map +1 -0
package/dist/web-search.js.map +1 -0
package/package.json +2 -1
package/fixtures/article-with-images.html +0 -94
package/fixtures/gh-shell.html +0 -32
package/fixtures/guide-ai-agents-web-scraping.json +0 -552
package/fixtures/images/large.jpg +0 -0
package/fixtures/images/small.jpg +0 -0
package/fixtures/images/tiny.png +0 -0
package/fixtures/quotes-index.json +0 -40
package/scripts/fetch-guide.mjs +0 -25
package/src/cache.ts +0 -99
package/src/convert.ts +0 -161
package/src/crawl.ts +0 -186
package/src/disk-cache.ts +0 -228
package/src/graph.ts +0 -189
package/src/index.ts +0 -74
package/src/parse.ts +0 -154
package/src/playwright.ts +0 -193
package/src/ports.ts +0 -131
package/src/robots.ts +0 -121
package/src/search.ts +0 -173
package/src/sitemap.ts +0 -67
package/src/spider.ts +0 -475
package/src/throttle.ts +0 -118
package/src/tree.ts +0 -379
package/src/types.ts +0 -225
package/src/views.ts +0 -42
package/src/web-search.ts +0 -548
package/test/convert-images.test.ts +0 -69
package/test/disk-cache-images.test.ts +0 -193
package/test/engine-registry.test.ts +0 -114
package/test/exports.test.ts +0 -124
package/test/get-chunk.test.ts +0 -115
package/test/images-integration.test.ts +0 -359
package/test/improvements.test.ts +0 -279
package/test/inbound-count.test.ts +0 -111
package/test/lean.test.ts +0 -105
package/test/playwright.test.ts +0 -128
package/test/ports.test.ts +0 -161
package/test/search.test.ts +0 -219
package/test/spider-images.test.ts +0 -180
package/test/spider-unit.test.ts +0 -610
package/test/tree.test.ts +0 -272
package/test/types.test.ts +0 -169
package/test/web-search-integration.test.ts +0 -180
package/test/web-search.test.ts +0 -305
package/tsconfig.json +0 -9
package/tsconfig.test.json +0 -7
package/vitest.config.ts +0 -8

package/fixtures/article-with-images.html DELETED Viewed

@@ -1,94 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
-  <meta charset="UTF-8">
-  <title>Article With Images — Fixture</title>
-  <meta name="description" content="A fixture page for testing image scraping.">
-  <meta name="keywords" content="images, scraping, fixtures">
-  <link rel="canonical" href="https://example.com/article-with-images">
-</head>
-<body>
-  <header>
-    <nav><a href="/">Home</a></nav>
-  </header>
-  <article>
-    <h1>Understanding Web Images for AI Agents</h1>
-    <p>
-      Images are a fundamental part of the modern web. When AI agents scrape pages, they
-      often need to capture visual content — charts, product photos, diagrams — alongside
-      the text. This article covers the key considerations for image-aware scraping.
-    </p>
-    <!-- 1. Absolute JPEG src, descriptive alt -->
-    <img
-      src="https://example.com/images/hero.jpg"
-      alt="A hero image showing a web spider diagram"
-    >
-    <p>
-      The first consideration is format. JPEG is the dominant format for photographs on
-      the web, while PNG is preferred for screenshots and diagrams that require lossless
-      compression. WebP is increasingly common as a modern alternative offering superior
-      compression ratios.
-    </p>
-    <!-- 2. Absolute PNG src, descriptive alt -->
-    <img
-      src="https://example.com/images/diagram.png"
-      alt="Architecture diagram of the web spider pipeline"
-    >
-    <p>
-      For AI agents that feed images to vision-language models, the wire format that all
-      major LLM APIs accept is a base64-encoded data URL:
-      <code>data:image/jpeg;base64,…</code>. This means an agent can scrape an image,
-      encode it in-memory, and pass it directly to GPT-4o or Claude without any
-      intermediate file I/O.
-    </p>
-    <!-- 3. Absolute WebP src, descriptive alt -->
-    <img
-      src="https://example.com/images/chart.webp"
-      alt="Performance chart comparing image formats by file size"
-    >
-    <p>
-      Caching is the other key concern. Fetching the same image repeatedly wastes
-      bandwidth and risks rate limiting. A sensible strategy is to cache small images
-      (under 32 KB) inline as base64 in the page cache, and write larger images as binary
-      files to a sibling <code>images/</code> directory, storing only the file path in
-      the JSON index.
-    </p>
-    <!-- 4. Relative src — must be resolved to absolute against the page URL -->
-    <img
-      src="/images/chart.png"
-      alt="Relative URL image — should be resolved to absolute"
-    >
-    <!-- 5. data: URL — should be included without making a network request -->
-    <img
-      src="data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg=="
-      alt="Inline data URL image — 1x1 pixel PNG"
-    >
-    <!-- 6. No alt attribute — alt should default to empty string -->
-    <img src="https://example.com/images/no-alt.jpg">
-    <h2>Conclusion</h2>
-    <p>
-      Web image scraping for AI agents is straightforward when broken into three layers:
-      fetch (extend the HTTP client to return binary data), normalise (base64 + MIME type),
-      and persist (inline for small, file-backed for large). The resulting
-      <code>ImageRef</code> objects are immediately usable by any vision-capable LLM.
-    </p>
-  </article>
-  <footer>
-    <a href="/privacy">Privacy</a>
-  </footer>
-</body>
-</html>

package/fixtures/gh-shell.html DELETED Viewed

@@ -1,32 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" data-color-mode="auto" data-light-theme="light" data-dark-theme="dark">
-<head>
-  <meta charset="utf-8">
-  <title>Issues · hyprwm/aquamarine · GitHub</title>
-  <meta name="viewport" content="width=device-width, initial-scale=1">
-  <meta name="description" content="">
-  <link rel="stylesheet" href="/assets/github-abc123.css" crossorigin="anonymous">
-  <script type="module" src="/assets/wp-runtime-xyz.js"></script>
-  <script type="module" src="/assets/vendors-node-xyz.js"></script>
-  <script type="module" src="/assets/app_assets-xyz.js"></script>
-  <meta name="turbo-body-classes" content="logged-out env-production">
-</head>
-<body class="logged-out env-production">
-  <div data-target="read-only-cursor-text-area.placeholder" id="placeholder" style="display:none"></div>
-  <div class="position-relative js-header-wrapper">
-    <div class="Header" role="banner"></div>
-  </div>
-  <div id="start-of-content" class="show-on-focus"></div>
-  <div data-pjax-timeout="1000" id="js-pjax-loader-bar" class="pjax-loader-bar"></div>
-  <div hidden id="ajax-error-message" class="ajax-error-message flash flash-error"></div>
-  <div role="main" data-turbo-body class="logged-out">
-    <react-app data-ssr="false">
-      <div id="root"></div>
-    </react-app>
-  </div>
-  <footer class="footer" role="contentinfo">
-    <div class="footer-links"></div>
-  </footer>
-  <script crossorigin="anonymous" defer type="application/json" id="__PRIMER_DATA__">{"version":"1.0"}</script>
-</body>
-</html>