@qilin-agent/web-fetch-http 0.0.0-stage → 3.1.3

package/lib/types/index.d.ts

@@ -0,0 +1,33 @@
+/**
+ * Anonymous public HTTP(S) `WebFetchProvider` plugin. It contributes to the
+ * `ctx.web` registry without owning the service.
+ *
+ * @module @qilin-agent/web-fetch-http
+ */
+import type { Context } from '@qilin-agent/kylin';
+import z from '@qilin-agent/schemastery';
+export { LOCAL_FETCH_PROVIDER_ID, HttpFetchProvider, } from './provider.ts';
+export type { HttpFetchLimits, HttpFetchResolver } from './provider.ts';
+/** Default `User-Agent`: an explicit product agent, never a browser disguise. */
+export declare const DEFAULT_USER_AGENT = "deepseek-harness/0.0.1 (+https://github.com/deepseek-ai)";
+/** Cordis plugin name used by loader diagnostics. */
+export declare const name = "web-fetch-http";
+/** The web seam this provider registers into. */
+export declare const inject: string[];
+/** Plugin config: the provider's transport and size limits plus its `User-Agent` (all defaulted). */
+export interface Config {
+    /** Maximum response body size in bytes. */
+    maxResponseBytes?: number;
+    /** Maximum decoded body length in characters. */
+    maxBodyChars?: number;
+    /** Default fetch timeout in milliseconds, within Node's timer range. */
+    timeoutMs?: number;
+    /** Maximum number of same-origin redirect hops to follow. */
+    maxRedirects?: number;
+    /** `User-Agent` header sent on every request. */
+    userAgent?: string;
+}
+export declare const Config: z<Config>;
+/** Register the local HTTP(S) fetch provider with `ctx.web`. */
+export declare function apply(ctx: Context, config: Config): void;
+//# sourceMappingURL=index.d.ts.map

package/lib/types/network.d.ts

@@ -0,0 +1,106 @@
+/**
+ * Public-network resolution and address-pinned HTTP transport for `web-fetch-http`.
+ * One DNS answer set is validated before Undici receives it through a custom lookup,
+ * so the connection cannot resolve the hostname again to a private address.
+ *
+ * @module @qilin-agent/web-fetch-http/network
+ */
+import type { LookupAddress, LookupOptions } from 'node:dns';
+import type { Dispatcher, Response } from 'undici';
+/** One address resolved and retained for the subsequent pinned connection. */
+export interface PublicAddress {
+    /** Canonical textual IPv4 or IPv6 address. */
+    readonly address: string;
+    /** Address family accepted by Node's connection lookup callback. */
+    readonly family: 4 | 6;
+}
+/** The result of one address-pinned request; closing releases its private pool. */
+export interface PinnedResponse {
+    /** HTTP response whose body remains readable until `close()` is called. */
+    readonly response: Response;
+    /** Release the request's dispatcher after the response body is consumed or cancelled. */
+    close(): Promise<void>;
+}
+/** Resolver signature used to test public-address policy without process DNS changes. */
+export type AddressResolver = (hostname: string, options: {
+    all: true;
+    order: 'verbatim';
+}) => Promise<LookupAddress[]>;
+/**
+ * Return whether an address is globally reachable unicast. IPv4-mapped IPv6 is
+ * classified by its embedded IPv4 address; transition and translation prefixes
+ * remain blocked because their eventual IPv4 destination cannot be pinned here.
+ *
+ * @param input - textual IPv4 or IPv6 address.
+ * @returns true only for a public unicast destination.
+ */
+export declare function isPublicIpAddress(input: string): boolean;
+/**
+ * Resolve a hostname once and reject the complete answer set if any destination
+ * is not public. The returned addresses are the only ones the transport may use.
+ *
+ * @param hostname - URL hostname, including brackets when it is an IPv6 literal.
+ * @param signal - aborts the wait for system resolution; an in-flight OS lookup may finish unused.
+ * @param resolver - lookup implementation, overridden only by focused tests.
+ * @returns the validated, non-empty address set.
+ */
+export declare function resolvePublicAddresses(hostname: string, signal: AbortSignal, resolver?: AddressResolver): Promise<PublicAddress[]>;
+/**
+ * Whether a hostname is an IP literal that {@link resolvePublicAddresses} would refuse.
+ *
+ * A proxied hop skips those checks because the proxy resolves the origin, but a literal needs no
+ * resolution: the address is already stated, and handing it to a proxy running on this machine
+ * would reach exactly the loopback or private service the checks exist to keep out of reach.
+ *
+ * @param hostname - a URL's hostname, bracketed or not.
+ * @returns true when the host is a literal address no request may be sent to.
+ */
+export declare function isNonPublicIpLiteral(hostname: string): boolean;
+/**
+ * Fetch through an agent whose lookup callback returns only the already validated address set. The
+ * URL hostname remains intact for HTTP Host and TLS SNI.
+ *
+ * The agent is this request's own because the address set is: pinning is how this package refuses a
+ * DNS answer that changes between validation and connection, and it may not apply process-wide —
+ * an operator-configured MCP server or model endpoint on loopback is a supported destination, and
+ * only the URLs this tool fetches are the model's to choose.
+ *
+ * @param url - validated HTTP(S) URL the policy does not route through a proxy.
+ * @param addresses - public addresses returned by {@link resolvePublicAddresses}.
+ * @param headers - request headers.
+ * @param signal - request and body-read cancellation signal.
+ * @returns a response plus the disposer its consumer must call.
+ */
+export declare function requestPinned(url: URL, addresses: readonly PublicAddress[], headers: Record<string, string>, signal: AbortSignal): Promise<PinnedResponse>;
+/**
+ * Fetch through the dispatcher the proxy policy already installed, letting the proxy resolve the
+ * origin.
+ *
+ * No address set is pinned because none exists to pin: the proxy performs the lookup, and a
+ * connection pinned to a locally resolved address would reach the origin directly and defeat the
+ * proxy. The dispatcher is the process-wide one, so hops share its connection pool and no caller
+ * closes it.
+ *
+ * @param dispatcher - the route's dispatcher, from `proxyRouteFor`.
+ * @param url - validated HTTP(S) URL the policy routes through a proxy.
+ * @param headers - request headers.
+ * @param signal - request and body-read cancellation signal.
+ * @returns a response plus a disposer that releases nothing, so both paths close alike.
+ */
+export declare function requestVia(dispatcher: Dispatcher, url: URL, headers: Record<string, string>, signal: AbortSignal): Promise<PinnedResponse>;
+/** Production network operations kept as an object so provider tests can replace resolution only. */
+export declare const publicHttpNetwork: {
+    resolve: typeof resolvePublicAddresses;
+    request: typeof requestPinned;
+    requestVia: typeof requestVia;
+};
+type LookupCallback = (error: NodeJS.ErrnoException | null, address: string | LookupAddress[], family?: number) => void;
+/**
+ * Build the connector lookup that serves a fixed validated answer set.
+ *
+ * @param addresses - public addresses retained from the preceding resolution.
+ * @returns a Node-compatible lookup callback that performs no network resolution.
+ */
+export declare function createPinnedLookup(addresses: readonly PublicAddress[]): (hostname: string, options: LookupOptions, callback: LookupCallback) => void;
+export {};
+//# sourceMappingURL=network.d.ts.map

package/lib/types/policy.d.ts

@@ -0,0 +1,72 @@
+/**
+ * URL validation and content-type classification for the local HTTP(S) fetch
+ * provider — the pure, network-free half. The provider's `fetch()` composes
+ * these with transport (redirect following, byte caps, decoding).
+ *
+ * @module @qilin-agent/web-fetch-http/policy
+ */
+/** Maximum accepted request URL length enforced by the public fetch provider. */
+export declare const WEB_FETCH_MAX_URL_LENGTH = 2048;
+/** The body kinds this provider decodes. */
+export type FetchableKind = 'html' | 'text';
+/**
+ * Parse a request URL and enforce network-independent transport restrictions:
+ * HTTP(S) only and no embedded credentials. The provider applies this before
+ * resolving a destination.
+ *
+ * @param input - the raw URL string from the fetch request.
+ * @returns the parsed `URL`.
+ */
+export declare function parseFetchUrl(input: string): URL;
+/**
+ * Validate a request URL against the provider's complete pre-network policy:
+ * bounded length plus the restrictions enforced by {@link parseFetchUrl}.
+ * Public-address resolution and connection pinning run after this check.
+ *
+ * @param input - the raw URL string from the fetch request.
+ * @returns the parsed `URL`.
+ */
+export declare function validateFetchUrl(input: string): URL;
+/**
+ * Two URLs are same-origin when scheme, hostname, and port match. A redirect
+ * that crosses origins is refused so each new origin requires a fresh tool call
+ * and public-address validation.
+ *
+ * @param a - one of the two URLs to compare.
+ * @param b - the other URL to compare.
+ * @returns true when `a` and `b` share scheme, hostname, and port.
+ */
+export declare function isSameOrigin(a: URL, b: URL): boolean;
+/**
+ * Classify a response `Content-Type` into a decodable body kind, or `undefined`
+ * for an unsupported (e.g. binary) type. `text/html` and `application/xhtml+xml`
+ * are `html`; other `text/*` plus a few structured text types are `text`.
+ *
+ * @param contentType - the raw `Content-Type` header, or `null` when the
+ *   response carries none (unsupported).
+ * @returns the decodable kind, or `undefined` for an unsupported type.
+ */
+export declare function classifyContentType(contentType: string | null): FetchableKind | undefined;
+/**
+ * Extract the `charset` parameter from a response `Content-Type`, lower-cased,
+ * or `undefined` when absent. The provider feeds this label to `TextDecoder`
+ * so a non-UTF-8 response is decoded with its declared encoding rather than
+ * silently mangled into replacement characters.
+ *
+ * @param contentType - the raw `Content-Type` header, or `null` when the
+ *   response carries none.
+ * @returns the lower-cased charset label, or `undefined` when none is declared.
+ */
+export declare function parseCharset(contentType: string | null): string | undefined;
+/**
+ * Build a `TextDecoder` for the declared charset, falling back to UTF-8 when
+ * none is declared. Throws {@link WebError} `WEB_UNSUPPORTED_CONTENT_TYPE` when
+ * the label is present but not a charset `TextDecoder` recognizes — better to
+ * fail loudly than return mojibake.
+ *
+ * @param charset - the declared charset label (from {@link parseCharset}), or
+ *   `undefined` to default to UTF-8.
+ * @returns a decoder for the declared (or defaulted) encoding.
+ */
+export declare function decoderForCharset(charset: string | undefined): TextDecoder;
+//# sourceMappingURL=policy.d.ts.map

package/lib/types/provider.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Safe HTTP(S) retrieval for `ctx.web`: validates and pins public IP destinations, follows
+ * only same-origin redirects, enforces time and size limits, classifies and decodes text,
+ * and leaves presentation to `@qilin-agent/tool-web`. Requests carry no browser cookies
+ * or ambient credentials.
+ * @module @qilin-agent/web-fetch-http/provider
+ */
+import type { WebFetchProvider, WebFetchRequest, WebFetchResult } from '@qilin-agent/web';
+import type { PublicAddress } from './network.ts';
+/** Resolved provider limits (the plugin's schemastery Config supplies defaults). */
+export interface HttpFetchLimits {
+    /** Maximum response body size in bytes (read is aborted past this). */
+    maxResponseBytes: number;
+    /** Maximum decoded body length in characters (truncated past this). */
+    maxBodyChars: number;
+    /** Default fetch timeout in milliseconds. */
+    timeoutMs: number;
+    /** Maximum number of (same-origin) redirect hops to follow. */
+    maxRedirects: number;
+    /** `User-Agent` header sent on every request. */
+    userAgent: string;
+}
+/** Resolve one hostname to an already policy-validated address set. */
+export type HttpFetchResolver = (hostname: string, signal: AbortSignal) => Promise<PublicAddress[]>;
+/** Stable id this provider registers under. */
+export declare const LOCAL_FETCH_PROVIDER_ID = "http";
+/** The anonymous public HTTP(S) fetch provider. */
+export declare class HttpFetchProvider implements WebFetchProvider {
+    private readonly limits;
+    private readonly resolveAddresses;
+    readonly id = "http";
+    /**
+     * @param limits - resolved transport and response limits.
+     * @param resolveAddresses - resolver that rejects non-public destinations before returning.
+     */
+    constructor(limits: HttpFetchLimits, resolveAddresses?: HttpFetchResolver);
+    /** No credentials to check — an anonymous public fetcher is always usable. */
+    available(): boolean;
+    fetch(request: WebFetchRequest, signal?: AbortSignal): Promise<WebFetchResult>;
+    /** Follow same-origin redirects up to the hop cap, then read the final response. */
+    private followAndRead;
+    private requestOnce;
+    /** Read, byte-cap, classify, and decode the final response body. */
+    private readBody;
+    /**
+     * Read the response stream up to `maxResponseBytes`. A `Content-Length` over
+     * the cap rejects immediately with `WEB_FETCH_TOO_LARGE`; a stream that grows
+     * past the cap is cut short (`truncatedByBytes`) rather than rejected, so a
+     * server that under-reports still yields a bounded usable body.
+     */
+    private readCapped;
+}
+//# sourceMappingURL=provider.d.ts.map

package/package.json

@@ -1,6 +1,49 @@
 {
   "name": "@qilin-agent/web-fetch-http",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
+  "description": "Anonymous public HTTP(S) fetch provider for the QiLin web capability seam (ctx.web)",
+  "version": "3.1.3",
+  "publishConfig": {
+    "access": "public"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/kkutysllb/QiLin.git",
+    "directory": "packages/web/web-fetch-http"
+  },
+  "engines": {
+    "node": "^22.19.0 || >=24.0.0"
+  },
+  "type": "module",
+  "main": "lib/index.js",
+  "types": "lib/types/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./lib/types/index.d.ts",
+      "default": "./lib/index.js"
+    },
+    "./src/*": "./src/*",
+    "./package.json": "./package.json"
+  },
+  "files": [
+    "lib/index.js",
+    "lib/types/**/*.d.ts"
+  ],
+  "license": "MIT",
+  "peerDependencies": {
+    "@qilin-agent/kylin": "~4.0.3",
+    "@qilin-agent/timeout": "3.1.3",
+    "@qilin-agent/web": "3.1.3",
+    "@qilin-agent/http-proxy": "3.1.3"
+  },
+  "dependencies": {
+    "ipaddr.js": "^2.5.0",
+    "undici": "^8.10.0",
+    "@qilin-agent/schemastery": "~3.18.3"
+  },
+  "devDependencies": {
+    "@qilin-agent/kylin": "~4.0.3",
+    "@qilin-agent/timeout": "3.1.3",
+    "@qilin-agent/web": "3.1.3",
+    "@qilin-agent/http-proxy": "3.1.3"
+  }
 }