@xyz17711/web-scraper 0.1.0 → 0.1.2

package/README.md

@@ -18,20 +18,23 @@ npm install @xyz17711/web-scraper
 
 | Method | Description |
 | --- | --- |
-| `extractMarkdown` | Fetches a URL and returns its main article content as clean Markdown ($0.007, x402) |
+| `extractMarkdown` | URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) |
 | `getTaskStatus` | Polls for the result of a task enqueued by a *_async tool |
-| `scrapeUrl` | Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402) |
-| `scrapeUrlAsync` | Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402) |
+| `scrapeUrl` | Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) |
+| `scrapeUrlAsync` | [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) |
 
 ## Quickstart
 
 ```typescript
 import { Client } from "@xyz17711/web-scraper";
 
-const client = new Client("https://api.17711.xyz/registry/web-scraper");
+const client = new Client(); // talks to https://api.17711.xyz/registry/web-scraper by default
 // see the Methods table above for every available call and its request/response type
 ```
 
+Pass another base URL (self-hosted instance, test server) as the first argument:
+`new Client("https://my-host.example/registry/web-scraper")`.
+
 ## Paying past the free quota
 
 Every gated tool allows a small number of free calls per day; beyond that it returns a 402
@@ -41,7 +44,7 @@ network, and payee address attached:
 ```typescript
 import { Client, PaymentRequiredError } from "@xyz17711/web-scraper";
 
-const client = new Client("https://api.17711.xyz/registry/web-scraper");
+const client = new Client();
 try {
   await client.extractMarkdown(/* ... */);
 } catch (e) {
@@ -57,7 +60,7 @@ own environment/secrets manager, never a literal string in source code:
 ```typescript
 import { Client } from "@xyz17711/web-scraper";
 
-const client = new Client("https://api.17711.xyz/registry/web-scraper", {
+const client = new Client({
   x402PrivateKey: process.env.X402_PRIVATE_KEY,
   x402ExpectedNetwork: "eip155:8453", // Base mainnet; use eip155:84532 for Base Sepolia testnet
   x402MaxPaymentUsdc: 0.01,           // refuse any single call above this
@@ -76,7 +79,7 @@ with that same key in this process.
 import { Client } from "@xyz17711/web-scraper";
 import { WebScraperToolkit } from "@xyz17711/web-scraper/langchain";
 
-const client = new Client("https://api.17711.xyz/registry/web-scraper", { /* ...wallet options... */ });
+const client = new Client({ /* ...wallet options... */ });
 const tools = new WebScraperToolkit(client).getTools();
 ```
 

package/dist/index.d.ts

@@ -60,15 +60,24 @@ export interface ScrapeUrlAsyncRequest {
     /** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
     url: string;
 }
+/** Production base URL of this service; pass another to self-host or test. */
+export declare const DEFAULT_BASE_URL = "https://api.17711.xyz/registry/web-scraper";
 /** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
 export declare class Client extends BaseClient {
-    /** Fetches a URL and returns its main article content as clean Markdown ($0.007, x402) */
+    /**
+     * `new Client()` talks to the production service. Pass a base URL to target another
+     * instance, and/or wallet options (`new Client({ x402PrivateKey, ... })`) to pay past the
+     * free quota automatically.
+     */
+    constructor(options?: ClientOptions);
+    constructor(baseUrl?: string, options?: ClientOptions);
+    /** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
     extractMarkdown(request: ExtractMarkdownRequest): Promise<ExtractMarkdownResponse>;
     /** Polls for the result of a task enqueued by a *_async tool */
     getTaskStatus(request: GetTaskStatusRequest): Promise<GetTaskStatusResponse>;
-    /** Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402) */
+    /** Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) */
     scrapeUrl(request: ScrapeUrlRequest): Promise<ScrapeUrlResponse>;
-    /** Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402) */
+    /** [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) */
     scrapeUrlAsync(request: ScrapeUrlAsyncRequest): Promise<AsyncTaskResponse>;
 }
 export { ClientOptions };

package/dist/index.js

@@ -2,9 +2,17 @@
  * Typed HTTP client and request/response interfaces for web-scraper.
  */
 import { Client as BaseClient } from "@xyz17711/core";
+/** Production base URL of this service; pass another to self-host or test. */
+export const DEFAULT_BASE_URL = "https://api.17711.xyz/registry/web-scraper";
 /** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
 export class Client extends BaseClient {
-    /** Fetches a URL and returns its main article content as clean Markdown ($0.007, x402) */
+    constructor(baseUrlOrOptions, options = {}) {
+        const [url, opts] = typeof baseUrlOrOptions === "object"
+            ? [DEFAULT_BASE_URL, baseUrlOrOptions]
+            : [baseUrlOrOptions ?? DEFAULT_BASE_URL, options];
+        super(url, opts);
+    }
+    /** URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402) */
     async extractMarkdown(request) {
         return this.post("/v1/web-scraper/extract_markdown", request);
     }
@@ -12,11 +20,11 @@ export class Client extends BaseClient {
     async getTaskStatus(request) {
         return this.post("/v1/web-scraper/get_task_status", request);
     }
-    /** Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402) */
+    /** Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402) */
     async scrapeUrl(request) {
         return this.post("/v1/web-scraper/scrape_url", request);
     }
-    /** Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402) */
+    /** [Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402) */
     async scrapeUrlAsync(request) {
         return this.post("/v1/web-scraper/scrape_url_async", request);
     }

package/dist/langchain.js

@@ -11,7 +11,7 @@ export class WebScraperToolkit {
         return [
             tool(async (input) => this.client.extractMarkdown(input), {
                 name: "extractMarkdown",
-                description: "Fetches a URL and returns its main article content as clean Markdown ($0.007, x402)",
+                description: "URL to Markdown: read any web page and get its main article content as clean Markdown for agents ($0.003, x402)",
                 schema: z.object({ url: z.string() }),
             }),
             tool(async (input) => this.client.getTaskStatus(input), {
@@ -21,12 +21,12 @@ export class WebScraperToolkit {
             }),
             tool(async (input) => this.client.scrapeUrl(input), {
                 name: "scrapeUrl",
-                description: "Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402)",
+                description: "Scrape URL: fetch any web page through a proxy and get its status, content type and raw HTML ($0.002, x402)",
                 schema: z.object({ url: z.string() }),
             }),
             tool(async (input) => this.client.scrapeUrlAsync(input), {
                 name: "scrapeUrlAsync",
-                description: "Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402)",
+                description: "[Deprecated - use web_scraper_scrape_url] Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.002, x402)",
                 schema: z.object({ callback_url: z.string().nullable().optional(), url: z.string() }),
             }),
         ];

package/package.json

@@ -1,7 +1,35 @@
 {
   "name": "@xyz17711/web-scraper",
-  "version": "0.1.0",
-  "description": "Generated typed client for the web-scraper 17711.xyz microservice.",
+  "version": "0.1.2",
+  "description": "Typed TypeScript client for the 17711.xyz Web Scraper API: pay-per-call, settled in USDC via x402.",
+  "license": "MIT",
+  "author": "17711 Services",
+  "homepage": "https://api.17711.xyz",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/17711-services/clients.git",
+    "directory": "node/packages/web-scraper"
+  },
+  "bugs": {
+    "url": "https://github.com/17711-services/clients/issues"
+  },
+  "keywords": [
+    "x402",
+    "usdc",
+    "api-client",
+    "pay-per-call",
+    "ai-agents",
+    "mcp",
+    "langchain",
+    "web-scraper"
+  ],
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "publishConfig": {
+    "access": "public"
+  },
   "type": "module",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -1,97 +0,0 @@
-/**
- * Typed HTTP client and request/response interfaces for web-scraper.
- */
-
-import { Client as BaseClient, type ClientOptions } from "@xyz17711/core";
-
-export interface ExtractMarkdownResponse {
-  content_type?: string | null;
-  /** The URL actually reached, after any redirects the target sent. */
-  final_url: string;
-  /** Boilerplate-stripped main article content, as Markdown -- navigation, ads, and sidebars removed, not the raw page HTML. */
-  markdown: string;
-  /** The REAL HTTP status the underlying fetch received. */
-  status: number;
-  /** Article title, if Readability found one. */
-  title?: string | null;
-  /** `true` if the underlying fetch's HTML was truncated by web-scraper's own fixed byte ceiling (see `web_scraper_scrape_url`'s own `truncated` field) -- `markdown` may then also be incomplete. */
-  truncated: boolean;
-  /** The URL as requested, verbatim. */
-  url: string;
-}
-
-export interface ExtractMarkdownRequest {
-  /** The target URL to fetch, e.g. `"https://example.com/article"`. Must be `http://` or `https://` -- same validation as `web_scraper_scrape_url`, since this tool fetches through the exact same internal scraper. */
-  url: string;
-}
-
-/** `"pending"` (enqueued, no result yet), `"finished"` (a worker wrote a result -- see `ok`/ `result`), or `"unknown"` (this id was never valid, or both it and its result have expired -- see [`nimpoleon_tasks::TaskState`]'s own doc comment for why this is 3 states, not 2). */
-export interface TaskStatusKind {
-}
-
-export interface GetTaskStatusResponse {
-  ok?: boolean | null;
-  result?: unknown;
-  status: TaskStatusKind;
-}
-
-export interface GetTaskStatusRequest {
-  /** The `task_id` a `*_async` tool returned. */
-  task_id: string;
-}
-
-export interface ScrapeUrlResponse {
-  content_type?: string | null;
-  /** The URL actually reached, after any redirects the target sent. */
-  final_url: string;
-  /** Raw response body, decoded as UTF-8 (lossy -- see this tool's own description for why), truncated to this service's own fixed byte ceiling if needed. */
-  html: string;
-  /** The REAL HTTP status the target returned -- never masked, never itself a reason to retry. */
-  status: number;
-  /** `true` if `html` was cut short by this service's own fixed byte ceiling. */
-  truncated: boolean;
-  /** The URL as requested, verbatim. */
-  url: string;
-}
-
-export interface ScrapeUrlRequest {
-  /** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
-  url: string;
-}
-
-/** The fixed response shape of every `*_async` tool -- deliberately NOT generic over the underlying sync tool's own response type, since no work has actually happened yet at this point. */
-export interface AsyncTaskResponse {
-  task_id: string;
-}
-
-export interface ScrapeUrlAsyncRequest {
-  /** When set, the worker that processes this task POSTs the eventual result (a `TaskResult` JSON body) here, best-effort -- polling `web_scraper_get_task_status` always works too, regardless of whether this is set. */
-  callback_url?: string | null;
-  /** The target URL to fetch, e.g. `"https://example.com/page"`. Must be `http://` or `https://` -- any other scheme is rejected before any fetch is attempted. */
-  url: string;
-}
-
-/** Typed client for web-scraper, e.g. `client.extractMarkdown(...)`. */
-export class Client extends BaseClient {
-  /** Fetches a URL and returns its main article content as clean Markdown ($0.007, x402) */
-  async extractMarkdown(request: ExtractMarkdownRequest): Promise<ExtractMarkdownResponse> {
-    return this.post<ExtractMarkdownResponse>("/v1/web-scraper/extract_markdown", request);
-  }
-
-  /** Polls for the result of a task enqueued by a *_async tool */
-  async getTaskStatus(request: GetTaskStatusRequest): Promise<GetTaskStatusResponse> {
-    return this.post<GetTaskStatusResponse>("/v1/web-scraper/get_task_status", request);
-  }
-
-  /** Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402) */
-  async scrapeUrl(request: ScrapeUrlRequest): Promise<ScrapeUrlResponse> {
-    return this.post<ScrapeUrlResponse>("/v1/web-scraper/scrape_url", request);
-  }
-
-  /** Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402) */
-  async scrapeUrlAsync(request: ScrapeUrlAsyncRequest): Promise<AsyncTaskResponse> {
-    return this.post<AsyncTaskResponse>("/v1/web-scraper/scrape_url_async", request);
-  }
-}
-
-export { ClientOptions };

package/src/langchain.ts

@@ -1,52 +0,0 @@
-/**
- * LangChain.js tools for web-scraper, grouped in one Toolkit -- one tool per operation, each just an async function (JS has no separate sync form to generate). Requires the `@langchain/core`+`zod` peer dependencies declared in this package's own package.json -- imported from `web-scraper/langchain`, never from the package root, so importing the base client never requires them installed.
- */
-
-import { tool } from "@langchain/core/tools";
-import { z } from "zod";
-import { Client } from "./index.js";
-
-export class WebScraperToolkit {
-  private client: Client;
-
-  constructor(client: Client) {
-    this.client = client;
-  }
-
-  getTools() {
-    return [
-      tool(
-        async (input) => this.client.extractMarkdown(input),
-        {
-          name: "extractMarkdown",
-          description: "Fetches a URL and returns its main article content as clean Markdown ($0.007, x402)",
-          schema: z.object({ url: z.string() }),
-        },
-      ),
-      tool(
-        async (input) => this.client.getTaskStatus(input),
-        {
-          name: "getTaskStatus",
-          description: "Polls for the result of a task enqueued by a *_async tool",
-          schema: z.object({ task_id: z.string() }),
-        },
-      ),
-      tool(
-        async (input) => this.client.scrapeUrl(input),
-        {
-          name: "scrapeUrl",
-          description: "Fetches an arbitrary URL through a proxy sourced from proxy-pool, never directly ($0.005, x402)",
-          schema: z.object({ url: z.string() }),
-        },
-      ),
-      tool(
-        async (input) => this.client.scrapeUrlAsync(input),
-        {
-          name: "scrapeUrlAsync",
-          description: "Enqueues a web_scraper_scrape_url job and returns a task id immediately ($0.005, x402)",
-          schema: z.object({ callback_url: z.string().nullable().optional(), url: z.string() }),
-        },
-      ),
-    ];
-  }
-}

package/test/index.test.mjs

@@ -1,75 +0,0 @@
-/**
- * Verifies every web-scraper client method sends a request to the expected path/HTTP verb and parses a well-formed response -- one test per operation, real fetch mocked via a plain reassignment of `globalThis.fetch` (no extra test/mocking dependency). Runs against the COMPILED package (`../dist/index.js`) -- build before testing.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { Client } from "../dist/index.js";
-
-test("extractMarkdown calls the expected path", async () => {
-  const originalFetch = globalThis.fetch;
-  globalThis.fetch = async (url, init = {}) => {
-    const parsed = new URL(url);
-    assert.equal((init.method ?? "GET").toUpperCase(), "POST");
-    assert.equal(parsed.pathname, "/v1/web-scraper/extract_markdown");
-    return new Response(JSON.stringify({ content_type: null, final_url: "test", markdown: "test", status: 0, title: null, truncated: false, url: "test" }), { status: 200 });
-  };
-  try {
-    const client = new Client("https://example.test");
-    const result = await client.extractMarkdown({ url: "test" });
-    assert.ok(result !== undefined);
-  } finally {
-    globalThis.fetch = originalFetch;
-  }
-});
-
-test("getTaskStatus calls the expected path", async () => {
-  const originalFetch = globalThis.fetch;
-  globalThis.fetch = async (url, init = {}) => {
-    const parsed = new URL(url);
-    assert.equal((init.method ?? "GET").toUpperCase(), "POST");
-    assert.equal(parsed.pathname, "/v1/web-scraper/get_task_status");
-    return new Response(JSON.stringify({ ok: null, result: null, status: {  } }), { status: 200 });
-  };
-  try {
-    const client = new Client("https://example.test");
-    const result = await client.getTaskStatus({ task_id: "test" });
-    assert.ok(result !== undefined);
-  } finally {
-    globalThis.fetch = originalFetch;
-  }
-});
-
-test("scrapeUrl calls the expected path", async () => {
-  const originalFetch = globalThis.fetch;
-  globalThis.fetch = async (url, init = {}) => {
-    const parsed = new URL(url);
-    assert.equal((init.method ?? "GET").toUpperCase(), "POST");
-    assert.equal(parsed.pathname, "/v1/web-scraper/scrape_url");
-    return new Response(JSON.stringify({ content_type: null, final_url: "test", html: "test", status: 0, truncated: false, url: "test" }), { status: 200 });
-  };
-  try {
-    const client = new Client("https://example.test");
-    const result = await client.scrapeUrl({ url: "test" });
-    assert.ok(result !== undefined);
-  } finally {
-    globalThis.fetch = originalFetch;
-  }
-});
-
-test("scrapeUrlAsync calls the expected path", async () => {
-  const originalFetch = globalThis.fetch;
-  globalThis.fetch = async (url, init = {}) => {
-    const parsed = new URL(url);
-    assert.equal((init.method ?? "GET").toUpperCase(), "POST");
-    assert.equal(parsed.pathname, "/v1/web-scraper/scrape_url_async");
-    return new Response(JSON.stringify({ task_id: "test" }), { status: 200 });
-  };
-  try {
-    const client = new Client("https://example.test");
-    const result = await client.scrapeUrlAsync({ callback_url: null, url: "test" });
-    assert.ok(result !== undefined);
-  } finally {
-    globalThis.fetch = originalFetch;
-  }
-});

package/test/langchain.test.mjs

@@ -1,40 +0,0 @@
-/**
- * Verifies every web-scraper LangChain tool through the real StructuredTool (`.invoke()`), not just the underlying client method -- catches schema-shape bugs a plain client-method test would miss. Requires the `langchain`/`zod` dev dependencies.
- */
-
-import { test } from "node:test";
-import assert from "node:assert/strict";
-import { Client } from "../dist/index.js";
-import { WebScraperToolkit } from "../dist/langchain.js";
-
-test("extractMarkdown tool invoke", async () => {
-  globalThis.fetch = async () => new Response(JSON.stringify({ content_type: null, final_url: "test", markdown: "test", status: 0, title: null, truncated: false, url: "test" }), { status: 200 });
-  const toolkit = new WebScraperToolkit(new Client("https://example.test"));
-  const t = toolkit.getTools().find((x) => x.name === "extractMarkdown");
-  const result = await t.invoke({ url: "test" });
-  assert.ok(result !== undefined);
-});
-
-test("getTaskStatus tool invoke", async () => {
-  globalThis.fetch = async () => new Response(JSON.stringify({ ok: null, result: null, status: {  } }), { status: 200 });
-  const toolkit = new WebScraperToolkit(new Client("https://example.test"));
-  const t = toolkit.getTools().find((x) => x.name === "getTaskStatus");
-  const result = await t.invoke({ task_id: "test" });
-  assert.ok(result !== undefined);
-});
-
-test("scrapeUrl tool invoke", async () => {
-  globalThis.fetch = async () => new Response(JSON.stringify({ content_type: null, final_url: "test", html: "test", status: 0, truncated: false, url: "test" }), { status: 200 });
-  const toolkit = new WebScraperToolkit(new Client("https://example.test"));
-  const t = toolkit.getTools().find((x) => x.name === "scrapeUrl");
-  const result = await t.invoke({ url: "test" });
-  assert.ok(result !== undefined);
-});
-
-test("scrapeUrlAsync tool invoke", async () => {
-  globalThis.fetch = async () => new Response(JSON.stringify({ task_id: "test" }), { status: 200 });
-  const toolkit = new WebScraperToolkit(new Client("https://example.test"));
-  const t = toolkit.getTools().find((x) => x.name === "scrapeUrlAsync");
-  const result = await t.invoke({ callback_url: "test", url: "test" });
-  assert.ok(result !== undefined);
-});

package/tsconfig.json

@@ -1,16 +0,0 @@
-{
-  "compilerOptions": {
-    "target": "ES2020",
-    "module": "ES2020",
-    "moduleResolution": "bundler",
-    "declaration": true,
-    "outDir": "dist",
-    "rootDir": "src",
-    "strict": true,
-    "esModuleInterop": true,
-    "skipLibCheck": true
-  },
-  "include": [
-    "src"
-  ]
-}