minifetch-api 1.4.0 → 1.5.1

package/README.md

@@ -24,7 +24,7 @@
 - ***Or*** an Ethereum or Solana private key for making USDC payments on Base or Solana networks.
 
 **Payments.** Two ways to pay:
-1. Credit card + API key. Get started free - [visit our dashboard to Sign Up](https://minifetch.com/dashboard). Create a Minifetch account and it will be auto-loaded with 25 free technical SEO page audits. Top up with your credit card later.
+1. Credit card + API key. Get started free - [visit our dashboard to Sign Up](https://minifetch.com/dashboard). Create a Minifetch account and it will be auto-loaded with credits worth up to 25 free technical SEO page audits. Top up with your credit card later.
 2. USDC on Base or Solana. Just load your wallet with USDC, you're ready. No "gas token" (ETH or SOL) required. You don't need a Minifetch account either, just load your wallet and go!
 
 ## Install
@@ -72,7 +72,7 @@ After the Quick Start, you have the following methods to use.
 
 **Wrap** these methods in a **try/catch** just like in the Quick Start example above. **Code examples** can be also found in the [Github repository /example- directories](https://github.com/Niche-Networks/minifetch-api/).
 
-The `checkAndExtract` methods check the target URL's `robots.txt` file to ensure its not blocked and tell us your preferred crawl delay (defaults to 1 second between requests to your domain). So fetching 10 URLs takes at least 10 seconds to complete by default. This is by design, so Minifetch never hammers your server or slows it down for your real users. If the URL is blocked, read [How To Unblock Minifetch](https://minifetch.com/tutorials/unblock-minifetch).
+The `checkAndExtract` methods check the target URL's `robots.txt` file to ensure its not blocked and tell us your preferred crawl delay (defaults to 1 second between requests to your domain). So fetching 10 URLs takes at least 10 seconds to complete by default. This is by design, so Minifetch never hammers your server or slows it down for your real users. If you own the site and want to allow Minifetch access or to set custom rules for it, read [How To Unblock Minifetch](https://minifetch.com/tutorials/unblock-minifetch).
 
 ```js
 await client.checkAndRunSeoPageAudit(url);
@@ -108,9 +108,8 @@ await client.checkAndExtractUrlLinks(url);
 
 await client.checkAndExtractUrlPreview(url);
 // Price: $0.002
-// For checking how your web page unfurls when shared: Extracts all
-// fields for the lightweight cards that represent your page on social
-// platforms, chat apps and AI.
+// Extracts all fields used for a page's share previews: the lightweight
+// cards that represent the page on social platforms, chat apps, and AI.
 
 await client.checkAndExtractUrlContent(url, options);
 // Price: $0.002
@@ -118,10 +117,14 @@ await client.checkAndExtractUrlContent(url, options);
 // an LLM extracts from your page after nav, ads, & scripts are stripped.
 // See what survives for AEO and AI search; respects robots.txt.
 // Options: { includeMediaUrls: true } - defaults to false.
+```
+
+For max control, you can also use the following methods directly:
+```js
+await client.preflightCheck(url, options);
+// Free: check if `minfetch` user agent can access target via robots.txt
+// Options: { "fresh": true } - bypass 24hr robots.txt cache, defaults to false
 
-// For max control, you can also use the following methods directly.
-// Free: check if `minfetch` user agent can access target via robots.txt:
-await client.preflightCheck(url);
 // Paid methods:
 await client.runSeoPageAudit(url);
 await client.extractUrlMetadata(url, options); // same options as above
@@ -129,7 +132,6 @@ await client.extractUrlLinks(url);
 await client.extractUrlPreview(url);
 await client.extractUrlContent(url, options); // same options as above
 ```
-
 ---
 
 ### Error Types
@@ -150,18 +152,15 @@ When you wrap the functions above in a try/catch, here are some of the errors yo
 
 ---
 
-### Service Limitations
-Minifetch only extracts publicly available metadata and content from pages accessible without authentication and javascript execution.
+### How We Fetch Web Pages
+Minifetch extracts publicly available metadata and content from pages accessible without authentication or javascript execution.
 
-What Minifetch does *NOT* do:
-- Ignore robots.txt directives
-- Create accounts or log into user sessions
-- Perform transactional actions (checkout, bidding, purchasing, form submissions)
-- Bypass paywalls or access restricted content
+Every response carries a `proxy` block: the `minifetch` user agent we sent and whether robots.txt was obeyed on the fetch. Proof of how the fetch happened, not just a promise, for regulated use-cases where provenance matters.
 
-What Minifetch does NOT do *currently* but may offer in the future as an add-on:
+Future add-ons:
+- Residential proxies for hard-to-reach pages
 - Javascript execution
-- Access authenticated or logged-in content
+- Access to authenticated or logged-in content
 
 ---
 

package/dist/esm/client.js

@@ -22,13 +22,18 @@ export class MinifetchClient {
      * Check if URL is allowed by robots.txt (free preflight check — no auth required)
      *
      * @param url
+     * @param options
+     * @param options.fresh
      * @throws {InvalidUrlError} if URL is invalid
      * @throws {NetworkError} if request fails
      */
-    async preflightUrlCheck(url) {
+    async preflightUrlCheck(url, options) {
         try {
             const normalizedUrl = validateAndNormalizeUrl(url);
-            const requestUrl = `${this.baseUrl}/api/v1/free/preflight/url-check?url=${encodeURIComponent(normalizedUrl)}`;
+            const params = new URLSearchParams({ url: normalizedUrl });
+            if (options?.fresh)
+                params.set("fresh", "true");
+            const requestUrl = `${this.baseUrl}/api/v1/free/preflight/url-check?${params.toString()}`;
             const response = await fetch(requestUrl);
             if (!response.ok) {
                 throw new NetworkError(`Preflight check failed: ${response.status} ${response.statusText}`);
@@ -221,9 +226,7 @@ export class MinifetchClient {
      * @param endpoint
      */
     _paidPath(endpoint) {
-        return this.config.authMode === "x402"
-            ? `/api/v1/x402${endpoint}`
-            : `/api/v1${endpoint}`;
+        return this.config.authMode === "x402" ? `/api/v1/x402${endpoint}` : `/api/v1${endpoint}`;
     }
     /**
      * Dispatch to the correct request handler based on auth mode, then

package/dist/esm/utils/payment.js

@@ -29,7 +29,7 @@ export async function handlePayment(url, config) {
         }
         else if (isSolana) {
             if (!config.privateKey)
-                throw new PaymentFailedError('privateKey is required for Solana payments');
+                throw new PaymentFailedError("privateKey is required for Solana payments");
             const privateKeyBytes = bs58.decode(config.privateKey);
             const signer = await createKeyPairSignerFromBytes(privateKeyBytes);
             const svmSigner = signer;

package/dist/types/client.d.ts

@@ -17,10 +17,14 @@ export declare class MinifetchClient {
      * Check if URL is allowed by robots.txt (free preflight check — no auth required)
      *
      * @param url
+     * @param options
+     * @param options.fresh
      * @throws {InvalidUrlError} if URL is invalid
      * @throws {NetworkError} if request fails
      */
-    preflightUrlCheck(url: string): Promise<PreflightCheckResponse>;
+    preflightUrlCheck(url: string, options?: {
+        fresh?: boolean;
+    }): Promise<PreflightCheckResponse>;
     /**
      * Run SEO page audit (paid endpoint)
      *

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "minifetch-api",
-  "version": "1.4.0",
-  "description": "Minifetch.com API Client. Pay-per-URL SEO audits. Composable toolkit for AI agents and automation pipelines.",
+  "version": "1.5.1",
+  "description": "Minifetch.com API Client. Scrape, extract and audit web pages. Pay per URL, no subscription.",
   "type": "module",
   "main": "./dist/esm/index.js",
   "types": "./dist/types/index.d.ts",
@@ -17,43 +17,43 @@
     "LICENSE"
   ],
   "keywords": [
+    "html",
+    "web page",
+    "scraper",
+    "parser",
+    "metadata",
+    "metadata extraction",
+    "content extraction",
+    "link extraction",
+    "hosted",
+    "meta tags",
+    "open graph",
+    "og tags",
+    "previews",
+    "social cards",
+    "twitter cards",
+    "pay-per-fetch",
+    "pay-per-url",
+    "indexing",
+    "extract",
+    "json-ld",
+    "JSON LD",
+    "link analysis",
     "SEO",
     "SEO page audit",
     "SEO research",
     "SEO toolkit",
     "SEO skill",
     "technical SEO",
-    "pay-per-fetch",
-    "pay-per-url",
     "GEO",
     "Generative Engine Optimization",
     "AEO",
     "Answer Engine Optimization",
-    "hosted extraction",
-    "content extraction",
-    "content indexing",
-    "extract",
-    "html",
-    "html parser",
-    "json-ld",
-    "JSON LD",
-    "link analysis",
-    "link extraction",
-    "metadata",
-    "meta tags",
-    "open graph",
-    "og tags",
     "AI",
     "AI Agents",
     "AI readability",
-    "agent skills",
     "automation",
-    "monitoring",
     "LLM",
-    "previews",
-    "social cards",
-    "twitter cards",
-    "site monitoring",
     "x402",
     "micropayments",
     "usdc"