minifetch-api 0.0.1 → 1.0.1

package/dist/esm/utils/payment.js

@@ -0,0 +1,98 @@
+import { x402Client, wrapFetchWithPayment, x402HTTPClient } from "@x402/fetch";
+import { registerExactEvmScheme } from "@x402/evm/exact/client";
+import { registerExactSvmScheme } from "@x402/svm/exact/client";
+import { privateKeyToAccount } from "viem/accounts";
+import { createKeyPairSignerFromBytes } from "@solana/kit";
+import bs58 from "bs58";
+import { PaymentFailedError, NetworkError } from "../types/errors.js";
+/**
+ * Handle x402 payment flow using Coinbase x402 client pattern
+ * 1. Initialize x402Client and register payment scheme
+ * 2. Wrap fetch with payment capabilities
+ * 3. Make request (client handles 402 detection and payment)
+ * 4. Extract payment receipt from response headers
+ *
+ * @param url
+ * @param config
+ */
+export async function handlePayment(url, config) {
+    if (!config.privateKey) {
+        throw new PaymentFailedError("Private key required for payments");
+    }
+    try {
+        // Create x402 client and signer based on network type
+        const _x402Client = new x402Client();
+        let payer;
+        const isEvm = config.network.startsWith("base");
+        const isSolana = config.network.startsWith("solana");
+        if (isEvm) {
+            // Initialize EVM signer and register scheme
+            const signer = privateKeyToAccount(config.privateKey);
+            const evmSigner = signer; // Type assertion for EVM
+            registerExactEvmScheme(_x402Client, { signer: evmSigner });
+            payer = signer.address;
+        }
+        else if (isSolana) {
+            // Initialize Solana signer and register scheme
+            const privateKeyBytes = bs58.decode(config.privateKey);
+            const signer = await createKeyPairSignerFromBytes(privateKeyBytes);
+            const svmSigner = signer; // Type assertion for Solana
+            registerExactSvmScheme(_x402Client, { signer: svmSigner });
+            payer = signer.address;
+        }
+        else {
+            throw new PaymentFailedError(`Unsupported network: ${config.network}`);
+        }
+        // Wrap fetch with payment capabilities
+        const fetchWithPayment = wrapFetchWithPayment(fetch, _x402Client);
+        // Make request with payment handling
+        // console.log("Attempting to fetch w payment:", url);
+        const response = await fetchWithPayment(url, {
+            method: "GET",
+        });
+        if (!response.ok) {
+            throw new NetworkError(`Request failed: ${response.status} ${response.statusText}`);
+        }
+        // Extract payment receipt from response headers
+        const httpClient = new x402HTTPClient(_x402Client);
+        const paymentResponse = httpClient.getPaymentSettleResponse(name => response.headers.get(name));
+        // DEBUG:
+        // console.log("Payment response:");
+        // console.log(paymentResponse);
+        // Build payment info for user
+        const payment = {
+            success: true,
+            payer,
+            network: config.network,
+            txHash: paymentResponse.transaction || "",
+            explorerLink: paymentResponse.transaction
+                ? getExplorerLink(config, paymentResponse.transaction)
+                : "",
+        };
+        return { response, payment };
+    }
+    catch (error) {
+        if (error instanceof PaymentFailedError || error instanceof NetworkError) {
+            throw error;
+        }
+        throw new PaymentFailedError(`Payment failed: ${error instanceof Error ? error.message : "Unknown error"}`);
+    }
+}
+// Helper fn to build explorer links
+/**
+ *
+ * @param config
+ * @param txHash
+ */
+function getExplorerLink(config, txHash) {
+    if (config.network === "solana-devnet") {
+        const strArray = config.explorerUrl.split("?");
+        return `${strArray[0]}/${txHash}?${strArray[1]}`;
+    }
+    else if (txHash) {
+        return `${config.explorerUrl}/${txHash}`;
+    }
+    else {
+        return "";
+    }
+}

package/dist/esm/utils/validation.js

@@ -0,0 +1,98 @@
+import { InvalidUrlError } from "../types/errors.js";
+/**
+ * Maximum allowed URL length
+ */
+const MAX_URL_LENGTH = 2048;
+/**
+ * Allowed URL protocols
+ */
+const ALLOWED_PROTOCOLS = ["http:", "https:"];
+/**
+ * Unsupported file extensions
+ */
+const UNSUPPORTED_EXTENSIONS = [
+    ".pdf",
+    ".txt",
+    ".md",
+    ".doc",
+    ".docx",
+    ".xls",
+    ".xlsx",
+    ".zip",
+    ".tar",
+    ".gz",
+];
+/**
+ * Validate and normalize a URL string
+ * Auto-normalizes by adding https:// if no protocol is present
+ *
+ * @param url
+ * @throws {InvalidUrlError} if URL is invalid
+ */
+export function validateAndNormalizeUrl(url) {
+    // Check if URL is provided
+    if (!url || typeof url !== "string") {
+        throw new InvalidUrlError(url, "URL must be a non-empty string");
+    }
+    // Normalize: add https:// if no protocol
+    let normalized = url.trim();
+    if (!normalized.match(/^https?:\/\//i)) {
+        normalized = `https://${normalized}`;
+    }
+    // Check URL length
+    if (normalized.length > MAX_URL_LENGTH) {
+        throw new InvalidUrlError(normalized, `URL exceeds maximum length of ${MAX_URL_LENGTH} characters`);
+    }
+    // Try to parse URL
+    let parsed;
+    try {
+        parsed = new URL(normalized);
+    }
+    catch (error) {
+        throw new InvalidUrlError(normalized, "Invalid URL format");
+    }
+    // Validate protocol
+    if (!ALLOWED_PROTOCOLS.includes(parsed.protocol)) {
+        throw new InvalidUrlError(normalized, `Protocol must be http: or https:, got: ${parsed.protocol}`);
+    }
+    // Validate hostname exists
+    if (!parsed.hostname) {
+        throw new InvalidUrlError(normalized, "URL must have a valid hostname");
+    }
+    // Validate hostname has at least one dot (e.g., example.com)
+    if (!parsed.hostname.includes(".")) {
+        throw new InvalidUrlError(normalized, "URL must have a valid domain with a TLD");
+    }
+    // Check for unsupported file extensions
+    const pathname = parsed.pathname.toLowerCase();
+    for (const ext of UNSUPPORTED_EXTENSIONS) {
+        if (pathname.endsWith(ext)) {
+            throw new InvalidUrlError(normalized, `Unsupported file format: ${ext}. Only HTML pages are supported.`);
+        }
+    }
+    // Basic check for localhost/private IPs
+    // API server will do deeper SSRF validation
+    if (isPrivateOrLocalhost(parsed.hostname)) {
+        throw new InvalidUrlError(normalized, "Cannot fetch from localhost or private IP addresses");
+    }
+    return normalized;
+}
+/**
+ * Check if hostname is localhost or private IP
+ *
+ * @param hostname
+ */
+function isPrivateOrLocalhost(hostname) {
+    // Localhost
+    if (hostname === "localhost" || hostname === "127.0.0.1" || hostname === "::1") {
+        return true;
+    }
+    // Private IP ranges (basic check)
+    const privateIpPatterns = [
+        /^10\./, // 10.0.0.0/8
+        /^172\.(1[6-9]|2[0-9]|3[0-1])\./, // 172.16.0.0/12
+        /^192\.168\./, // 192.168.0.0/16
+        /^169\.254\./, // 169.254.0.0/16 (link-local)
+    ];
+    return privateIpPatterns.some(pattern => pattern.test(hostname));
+}

package/dist/types/client.d.ts

@@ -0,0 +1,103 @@
+import type { ClientConfig } from "./types/config.js";
+import type { PreflightCheckResponse, PaidEndpointResponse } from "./types/responses.js";
+/**
+ * Main Minifetch API client
+ * Provides methods to check URLs and extract metadata/links/preview/content
+ */
+export declare class MinifetchClient {
+    private config;
+    private baseUrl;
+    /**
+     *
+     * @param config
+     */
+    constructor(config: ClientConfig);
+    /**
+     * Check if URL is allowed by robots.txt (free preflight check)
+     *
+     * @param url
+     * @throws {InvalidUrlError} if URL is invalid
+     * @throws {NetworkError} if request fails
+     */
+    preflightUrlCheck(url: string): Promise<PreflightCheckResponse>;
+    /**
+     * Extract URL metadata (paid, requires x402 payment)
+     *
+     * @param url
+     * @param options
+     * @param options.includeResponseBody
+     * @throws {InvalidUrlError} if URL is invalid
+     * @throws {PaymentFailedError} if payment fails
+     * @throws {ExtractionFailedError} if extraction fails
+     */
+    extractUrlMetadata(url: string, options?: {
+        includeResponseBody?: boolean;
+    }): Promise<PaidEndpointResponse>;
+    /**
+     * Extract URL links (paid, requires x402 payment)
+     *
+     * @param url
+     * @throws {InvalidUrlError} if URL is invalid
+     * @throws {PaymentFailedError} if payment fails
+     * @throws {ExtractionFailedError} if extraction fails
+     */
+    extractUrlLinks(url: string): Promise<PaidEndpointResponse>;
+    /**
+     * Extract URL preview (paid, requires x402 payment)
+     *
+     * @param url
+     * @throws {InvalidUrlError} if URL is invalid
+     * @throws {PaymentFailedError} if payment fails
+     * @throws {ExtractionFailedError} if extraction fails
+     */
+    extractUrlPreview(url: string): Promise<PaidEndpointResponse>;
+    /**
+     * Extract URL content as markdown (paid, requires x402 payment)
+     *
+     * @param url
+     * @param options
+     * @param options.includeMediaUrls
+     * @throws {InvalidUrlError} if URL is invalid
+     * @throws {PaymentFailedError} if payment fails
+     * @throws {ExtractionFailedError} if extraction fails
+     */
+    extractUrlContent(url: string, options?: {
+        includeMediaUrls?: boolean;
+    }): Promise<PaidEndpointResponse>;
+    /**
+     * Check URL and extract metadata in one call
+     * Throws RobotsBlockedError if robots.txt blocks the URL
+     *
+     * @param url
+     * @param options
+     * @param options.includeResponseBody
+     */
+    checkAndExtractUrlMetadata(url: string, options?: {
+        includeResponseBody?: boolean;
+    }): Promise<PaidEndpointResponse>;
+    /**
+     * Check URL and extract links in one call
+     * Throws RobotsBlockedError if robots.txt blocks the URL
+     *
+     * @param url
+     */
+    checkAndExtractUrlLinks(url: string): Promise<PaidEndpointResponse>;
+    /**
+     * Check URL and extract preview in one call
+     * Throws RobotsBlockedError if robots.txt blocks the URL
+     *
+     * @param url
+     */
+    checkAndExtractUrlPreview(url: string): Promise<PaidEndpointResponse>;
+    /**
+     * Check URL and extract content in one call
+     * Throws RobotsBlockedError if robots.txt blocks the URL
+     *
+     * @param url
+     * @param options
+     * @param options.includeMediaUrls
+     */
+    checkAndExtractUrlContent(url: string, options?: {
+        includeMediaUrls?: boolean;
+    }): Promise<PaidEndpointResponse>;
+}

package/dist/types/index.d.ts

@@ -0,0 +1,2 @@
+export { MinifetchClient } from "./client.js";
+export { MinifetchClient as default } from "./client.js";

package/dist/types/init.d.ts

@@ -0,0 +1,26 @@
+import type { ClientConfig, InitializedConfig, Network } from "./types/config.js";
+/**
+ * Default configuration values
+ * Exported for testing purposes
+ */
+export declare const DEFAULTS: {
+    readonly network: Network;
+    readonly apiBaseUrls: {
+        readonly "base-sepolia": "http://localhost:4021";
+        readonly "solana-devnet": "http://localhost:4021";
+        readonly base: "https://minifetch.com";
+        readonly solana: "https://minifetch.com";
+    };
+    readonly explorerUrls: {
+        readonly "base-sepolia": "https://sepolia.basescan.org/tx";
+        readonly "solana-devnet": "https://explorer.solana.com/tx?cluster=devnet";
+        readonly base: "https://basescan.org/tx";
+        readonly solana: "https://explorer.solana.com/tx";
+    };
+};
+/**
+ * Validate & initialize client configuration
+ *
+ * @param config
+ */
+export declare function initConfig(config: ClientConfig): InitializedConfig;

package/dist/types/types/config.d.ts

@@ -0,0 +1,21 @@
+export declare const VALID_NETWORKS: readonly ["base", "base-sepolia", "solana", "solana-devnet"];
+export type Network = (typeof VALID_NETWORKS)[number];
+export interface ClientConfig {
+    /**
+     * Blockchain network to use for payments
+     *
+     * @default 'base'
+     */
+    network: Network;
+    /**
+     * Private key for signing transactions -
+     * hex string for EVM, base58 for Solana
+     */
+    privateKey: string;
+}
+export interface InitializedConfig {
+    network: Network;
+    privateKey: string;
+    apiBaseUrl: string;
+    explorerUrl: string;
+}

package/dist/types/types/errors.d.ts

@@ -0,0 +1,86 @@
+/**
+ * Base error class for all Minifetch errors
+ */
+export declare class MinifetchError extends Error {
+    /**
+     *
+     * @param message
+     */
+    constructor(message: string);
+}
+/**
+ * Thrown when URL validation fails
+ */
+export declare class InvalidUrlError extends MinifetchError {
+    readonly url: string;
+    /**
+     *
+     * @param url
+     * @param message
+     */
+    constructor(url: string, message?: string);
+}
+/**
+ * Thrown when robots.txt blocks the request
+ */
+export declare class RobotsBlockedError extends MinifetchError {
+    readonly url: string;
+    /**
+     *
+     * @param url
+     * @param message
+     */
+    constructor(url: string, message?: string);
+}
+/**
+ * Thrown when payment fails
+ */
+export declare class PaymentFailedError extends MinifetchError {
+    readonly network?: string;
+    readonly originalError?: Error;
+    /**
+     *
+     * @param message
+     * @param network
+     * @param originalError
+     */
+    constructor(message: string, network?: string, originalError?: Error);
+}
+/**
+ * Thrown when extraction/fetch fails
+ */
+export declare class ExtractionFailedError extends MinifetchError {
+    readonly url: string;
+    readonly statusCode?: number;
+    readonly originalError?: Error;
+    /**
+     *
+     * @param url
+     * @param message
+     * @param statusCode
+     * @param originalError
+     */
+    constructor(url: string, message: string, statusCode?: number, originalError?: Error);
+}
+/**
+ * Thrown when configuration is invalid
+ */
+export declare class ConfigurationError extends MinifetchError {
+    /**
+     *
+     * @param message
+     */
+    constructor(message: string);
+}
+/**
+ * Thrown when network/API communication fails
+ */
+export declare class NetworkError extends MinifetchError {
+    readonly originalError?: Error;
+    /**
+     *
+     * @param message
+     * @param originalError
+     */
+    constructor(message: string, originalError?: Error);
+}

package/dist/types/types/responses.d.ts

@@ -0,0 +1,50 @@
+/**
+ * Result from preflight URL check (free endpoint)
+ * Note: This is the ONLY endpoint that doesn't require payment
+ */
+export interface PreflightCheckResponse {
+    success: boolean;
+    results: Array<{
+        data: {
+            url: string;
+            allowed: boolean;
+            message?: string;
+            crawlDelay?: number;
+            [key: string]: any;
+        };
+    }>;
+}
+/**
+ * Response structure for all paid API responses
+ */
+export interface PaidEndpointResponse {
+    /** Minifetch API success (200, ok) **/
+    success: boolean;
+    /** Data returned from the Minifetch.com API **/
+    results: Array<{
+        data: {
+            [key: string]: any;
+        };
+        error?: Record<string, any>;
+    }>;
+    /**
+     * Payment information
+     * Only present for paid endpoints when request was successful
+     * */
+    payment?: PaymentInfo;
+}
+/**
+ * Payment information included with successful paid API responses
+ */
+export interface PaymentInfo {
+    /** Whether the payment was successful **/
+    success: boolean;
+    /** Account that paid for tx **/
+    payer: string;
+    /** Network the payment was made on **/
+    network: "base" | "base-sepolia" | "solana" | "solana-devnet";
+    /** Transaction hash **/
+    txHash: string;
+    /** Link to view transaction on block explorer **/
+    explorerLink: string;
+}

package/dist/types/utils/payment.d.ts

@@ -0,0 +1,16 @@
+import type { InitializedConfig } from "../types/config.js";
+import type { PaymentInfo } from "../types/responses.js";
+/**
+ * Handle x402 payment flow using Coinbase x402 client pattern
+ * 1. Initialize x402Client and register payment scheme
+ * 2. Wrap fetch with payment capabilities
+ * 3. Make request (client handles 402 detection and payment)
+ * 4. Extract payment receipt from response headers
+ *
+ * @param url
+ * @param config
+ */
+export declare function handlePayment(url: string, config: InitializedConfig): Promise<{
+    response: Response;
+    payment?: PaymentInfo;
+}>;

package/dist/types/utils/validation.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Validate and normalize a URL string
+ * Auto-normalizes by adding https:// if no protocol is present
+ *
+ * @param url
+ * @throws {InvalidUrlError} if URL is invalid
+ */
+export declare function validateAndNormalizeUrl(url: string): string;

package/package.json

@@ -1,38 +1,87 @@
 {
   "name": "minifetch-api",
-  "version": "0.0.1",
+  "version": "1.0.1",
+  "description": "Minifetch.com API Client. Fetch & extract data from web pages. For AI Agents & SEO research.",
   "type": "module",
+  "main": "./dist/esm/index.js",
+  "types": "./dist/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/types/index.d.ts",
+      "import": "./dist/esm/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "keywords": [
+    "seo",
+    "metadata",
+    "meta-tags",
+    "web-pages",
+    "web-scraping",
+    "scraper",
+    "html-parser",
+    "ai",
+    "ai-agents",
+    "llm",
+    "x402",
+    "micropayments",
+    "usdc"
+  ],
   "bugs": {
     "url": "https://github.com/Niche-Networks/minifetch-api/issues"
   },
-  "homepage": "https://minifetch.com/ ",
+  "homepage": "https://minifetch.com",
   "repository": {
     "type": "git",
     "url": "git@github.com:Niche-Networks/minifetch-api.git"
   },
   "scripts": {
-    "dev:testnet": "tsx index.ts --env=base-testnet",
-    "dev:testnet:check": "tsx index.ts --env=base-testnet --endpoint=check",
+    "dist:clean": "rm -rf dist",
+    "build": "pnpm run dist:clean && pnpm run build:esm",
+    "build:esm": "tsc -p tsconfig.json",
     "format": "prettier -c .prettierrc --write \"**/*.{ts,js,cjs,json,md}\"",
     "format:check": "prettier -c .prettierrc --check \"**/*.{ts,js,cjs,json,md}\"",
     "lint": "eslint . --ext .ts --fix",
-    "lint:check": "eslint . --ext .ts"
+    "lint:check": "eslint . --ext .ts",
+    "test": "SKIP_DEBUG_TESTS=true vitest",
+    "test:run": "SKIP_DEBUG_TESTS=true vitest run",
+    "test:file": "vitest run",
+    "test:debug": "vitest run test/debug.test.ts",
+    "test:coverage": "vitest run --coverage",
+    "example:ts": "cd example-typescript && pnpm install && pnpm start",
+    "example:js": "cd example-javascript && pnpm install && pnpm start",
+    "prepublishOnly": "pnpm run build"
   },
   "dependencies": {
-    "dotenv": "^16.4.7",
-    "viem": "^2.23.1",
-    "x402-fetch": "0.6.0"
+    "@solana/kit": "^5.4.0",
+    "@x402/evm": "^2.2.0",
+    "@x402/fetch": "^2.2.0",
+    "@x402/svm": "^2.2.0",
+    "bs58": "^6.0.0",
+    "viem": "^2.23.1"
   },
   "devDependencies": {
     "@eslint/js": "^9.24.0",
-    "eslint": "^9.24.0",
-    "eslint-plugin-jsdoc": "^50.6.9",
-    "eslint-plugin-prettier": "^5.2.6",
+    "@types/node": "^25.2.0",
     "@typescript-eslint/eslint-plugin": "^8.29.1",
     "@typescript-eslint/parser": "^8.29.1",
+    "@vitest/coverage-v8": "^3.0.0",
+    "dotenv": "^16.4.7",
+    "eslint": "^9.24.0",
     "eslint-plugin-import": "^2.31.0",
+    "eslint-plugin-jsdoc": "^50.6.9",
+    "eslint-plugin-prettier": "^5.2.6",
     "prettier": "3.5.2",
     "tsx": "^4.7.0",
-    "typescript": "^5.3.0"
-  }
+    "typescript": "^5.3.0",
+    "vitest": "^3.0.0"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "license": "MIT"
 }

package/.env-local

@@ -1,10 +0,0 @@
-NETWORK=base-sepolia
-EXPLORER_BASE_URL=https://sepolia.basescan.org
-
-API_SERVER_URL=http://localhost:4021
-CHECK_ROBOTS_API_PATH=/api/v1/free/preflight/url-check
-METADATA_API_PATH=/api/v1/x402/extract/url-metadata
-
-METADATA_URL=
-METADATA_API_PARAM_INCLUDE_RESPONSE_BODY=
-PRIVATE_KEY=

package/.prettierignore

@@ -1,8 +0,0 @@
-docs/
-dist/
-node_modules/
-coverage/
-.github/
-src/client
-**/**/*.json
-*.md

package/.prettierrc

@@ -1,11 +0,0 @@
-{
-  "tabWidth": 2,
-  "useTabs": false,
-  "semi": true,
-  "singleQuote": false,
-  "trailingComma": "all",
-  "bracketSpacing": true,
-  "arrowParens": "avoid",
-  "printWidth": 100,
-  "proseWrap": "never"
-}

package/.tsconfig.json

@@ -1,15 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2020",
-    "module": "ES2020",
-    "moduleResolution": "bundler",
-    "esModuleInterop": true,
-    "forceConsistentCasingInFileNames": true,
-    "skipLibCheck": true,
-    "strict": true,
-    "resolveJsonModule": true,
-    "baseUrl": ".",
-    "types": ["node"]
-  },
-  "include": ["index.ts"]
-}