@apifuse/provider-sdk 2.2.0-beta.62 → 2.2.0-beta.63

package/dist/runtime/browser.js

@@ -1,4 +1,19 @@
-import { readDiagnosticEnv } from "./diagnostic-env.js";
+import { readDiagnosticEnv, registerDiagnosticValue } from "./diagnostic-env.js";
+import { AsyncResource } from "node:async_hooks";
+import { observeTelemetryCallback as observeBrowserTelemetry } from "./http-telemetry-guard.js";
+import { cookieValuesFromHeaders } from "./stealth-cookies.js";
+function browserTelemetryErrorCode(error) {
+    const code = error && typeof error === "object" && "code" in error
+        ? error.code
+        : undefined;
+    if (code === "BROWSER_CDP_POOL_REQUIRED" ||
+        code === "BROWSER_PROXY_INVALID" ||
+        code === "BROWSER_RUNTIME_UNSUPPORTED")
+        return code;
+    if (typeof code === "number")
+        return "BROWSER_CDP_POOL_ERROR";
+    return "other";
+}
 import { createRequire } from "node:module";
 import { ProviderError } from "../errors.js";
 const require = createRequire(import.meta.url);
@@ -241,11 +256,9 @@ function getCdpAuthRequiredRequestId(params) {
     return params.requestId;
 }
 function isCdpProxyAuthChallenge(params) {
-    return (isRecord(params) &&
-        isRecord(params.authChallenge) &&
-        params.authChallenge.source === "Proxy");
+    return (isRecord(params) && isRecord(params.authChallenge) && params.authChallenge.source === "Proxy");
 }
-async function handleCdpAuthRequired(session, proxy, authAttempts, params) {
+async function handleCdpAuthRequired(session, proxy, authAttempts, params, telemetry) {
     const requestId = getCdpAuthRequiredRequestId(params);
     if (!requestId)
         return;
@@ -254,6 +267,7 @@ async function handleCdpAuthRequired(session, proxy, authAttempts, params) {
     const response = isCdpProxyAuthChallenge(params) && !authAttempts.has(requestId)
         ? (() => {
             authAttempts.add(requestId);
+            observeBrowserTelemetry(telemetry, () => telemetry?.recordProxyAuthChallenge());
             return {
                 authChallengeResponse: {
                     password: proxy.password,
@@ -374,6 +388,8 @@ function toPlaywrightProxy(proxy) {
             fix: "Percent-encode the proxy username and password as URL userinfo.",
         });
     }
+    registerDiagnosticValue(username);
+    registerDiagnosticValue(password);
     return {
         server: `${parsed.protocol}//${parsed.host}`,
         username,
@@ -543,14 +559,24 @@ async function loadSeleniumBase() {
 class PlaywrightBrowserPage {
     page;
     proxy;
+    telemetry;
     id = "main";
     pageId;
-    constructor(page, proxy = undefined) {
+    constructor(page, proxy = undefined, telemetry) {
         this.page = page;
         this.proxy = proxy;
+        this.telemetry = telemetry;
     }
     async goto(url, options) {
-        await this.page.goto(url, options);
+        const response = await this.page.goto(url, options);
+        const headers = await response?.headersArray().catch(() => []);
+        for (const header of headers ?? []) {
+            if (header.name.toLowerCase() === "set-cookie") {
+                registerDiagnosticValue(header.value);
+                for (const value of cookieValuesFromHeaders(header.value))
+                    registerDiagnosticValue(value);
+            }
+        }
     }
     async evaluate(fn) {
         if (typeof fn === "string") {
@@ -596,7 +622,10 @@ class PlaywrightBrowserPage {
         await this.page.close();
     }
     async cookies() {
-        return (await this.page.context().cookies()).map(toBrowserCookie);
+        const cookies = (await this.page.context().cookies()).map(toBrowserCookie);
+        for (const cookie of cookies)
+            registerDiagnosticValue(cookie.value);
+        return cookies;
     }
     async withResourcePolicy(policy, run) {
         const allowedMethods = new Set(policy.allowedMethods ?? DEFAULT_RESOURCE_METHODS);
@@ -639,7 +668,7 @@ class PlaywrightBrowserPage {
             const onAuthRequired = (params) => {
                 if (!proxy)
                     return;
-                void handleCdpAuthRequired(session, proxy, authAttempts, params);
+                void handleCdpAuthRequired(session, proxy, authAttempts, params, this.telemetry);
             };
             session.on("Fetch.requestPaused", onRequestPaused);
             if (proxy) {
@@ -713,9 +742,11 @@ class PlaywrightBrowserClient {
         return this.browser;
     }
     async newPage() {
+        if (!this.options.cdpUrl)
+            observeBrowserTelemetry(this.options.telemetry, () => this.options.telemetry?.recordPoolAcquire("not_configured"));
         const browser = await this.ensureBrowser();
         const page = await browser.newPage();
-        return new PlaywrightBrowserPage(page, this.parsedProxy);
+        return new PlaywrightBrowserPage(page, this.parsedProxy, this.options.telemetry);
     }
     async rawPage() {
         throw new ProviderError("ctx.browser.rawPage() requires a CDP pool", {
@@ -724,10 +755,12 @@ class PlaywrightBrowserClient {
         });
     }
     async withIsolatedContext(handler) {
+        if (!this.options.cdpUrl)
+            observeBrowserTelemetry(this.options.telemetry, () => this.options.telemetry?.recordPoolAcquire("not_configured"));
         const browser = await this.ensureBrowser();
         const context = await browser.newContext({ serviceWorkers: this.options.serviceWorkers });
         const page = await context.newPage();
-        const browserPage = new PlaywrightBrowserPage(page, this.parsedProxy);
+        const browserPage = new PlaywrightBrowserPage(page, this.parsedProxy, this.options.telemetry);
         try {
             return await handler(browserPage);
         }
@@ -771,6 +804,7 @@ class WebSocketCommandClient {
     pending = new Map();
     socket;
     socketPromise;
+    closed = false;
     constructor(endpoint) {
         this.endpoint = normalizeWebSocketEndpoint(endpoint);
     }
@@ -787,6 +821,8 @@ class WebSocketCommandClient {
     }
     async send(method, params = {}) {
         const socket = await this.getSocket();
+        if (this.closed)
+            throw new Error(`WebSocket closed: ${this.endpoint}`);
         const id = this.nextId++;
         return await new Promise((resolve, reject) => {
             this.pending.set(id, { resolve, reject });
@@ -794,6 +830,7 @@ class WebSocketCommandClient {
         });
     }
     async close() {
+        this.closed = true;
         for (const pending of this.pending.values()) {
             pending.reject(new Error(`WebSocket closed: ${this.endpoint}`));
         }
@@ -804,6 +841,8 @@ class WebSocketCommandClient {
         this.socketPromise = undefined;
     }
     async getSocket() {
+        if (this.closed)
+            throw new Error(`WebSocket closed: ${this.endpoint}`);
         if (this.socket?.readyState === WebSocket.OPEN) {
             return this.socket;
         }
@@ -813,6 +852,11 @@ class WebSocketCommandClient {
         this.socketPromise = new Promise((resolve, reject) => {
             const socket = new WebSocket(this.endpoint);
             socket.addEventListener("open", () => {
+                if (this.closed) {
+                    socket.close();
+                    reject(new Error(`WebSocket closed: ${this.endpoint}`));
+                    return;
+                }
                 this.socket = socket;
                 resolve(socket);
             });
@@ -954,6 +998,33 @@ function parsePoolAcquireResponse(value) {
         wsEndpoint: value.wsEndpoint,
     };
 }
+function poolAcquireOutcome(error) {
+    const code = error instanceof Error && typeof error.code === "number"
+        ? error.code
+        : undefined;
+    switch (code) {
+        case -32001:
+            return "queue_full";
+        case -32002:
+            return "timed_out";
+        case -32003:
+            return "shutting_down";
+        case -32004:
+            return "unknown_lease";
+        case -32005:
+            return "unknown_method";
+        case -32006:
+            return "missing_allowed_hosts";
+    }
+    if (error instanceof Error && /Unable to connect|WebSocket closed/.test(error.message))
+        return "transport_failure";
+    return "other";
+}
+function poolAcquireCode(error) {
+    return error instanceof Error && typeof error.code === "number"
+        ? error.code
+        : undefined;
+}
 function parseCdpFrameTreeNode(value) {
     if (!isRecord(value) || !isRecord(value.frame)) {
         return undefined;
@@ -992,9 +1063,7 @@ function getCdpDestroyedExecutionContextId(params) {
     if (!isRecord(params)) {
         return undefined;
     }
-    return typeof params.executionContextId === "number"
-        ? params.executionContextId
-        : undefined;
+    return typeof params.executionContextId === "number" ? params.executionContextId : undefined;
 }
 function isMissingExecutionContextError(error) {
     return error instanceof Error && /\b(?:cannot|failed to) find context\b/i.test(error.message);
@@ -1090,7 +1159,7 @@ class CdpPoolBrowserPage {
     browserContextId;
     pageClient;
     release;
-    closed = false;
+    closePromise;
     initialized = false;
     frameExecutionContexts = new Map();
     constructor(pageId, browserContextId, pageClient, release) {
@@ -1248,22 +1317,32 @@ class CdpPoolBrowserPage {
     async cookies() {
         await this.initialize();
         const result = await this.pageClient.send("Network.getCookies");
-        return parseCdpCookies(result.cookies);
+        const cookies = parseCdpCookies(result.cookies);
+        for (const cookie of cookies)
+            registerDiagnosticValue(cookie.value);
+        return cookies;
     }
     async close() {
-        if (this.closed) {
+        // Only the first caller observes the release outcome; repeat calls stay
+        // idempotent (as they were before the promise was memoized) instead of
+        // replaying a cached rejection, and never re-issue a release against a
+        // target the pool has already disposed.
+        if (this.closePromise) {
+            await this.closePromise.catch(() => undefined);
             return;
         }
-        this.closed = true;
-        try {
-            await this.release({
-                ...(this.browserContextId ? { browserContextId: this.browserContextId } : {}),
-                pageId: this.pageId,
-            });
-        }
-        finally {
-            await this.pageClient.close();
-        }
+        this.closePromise = (async () => {
+            try {
+                await this.release({
+                    ...(this.browserContextId ? { browserContextId: this.browserContextId } : {}),
+                    pageId: this.pageId,
+                });
+            }
+            finally {
+                await this.pageClient.close();
+            }
+        })();
+        await this.closePromise;
     }
     async withResourcePolicy(policy, run) {
         const allowedMethods = new Set(policy.allowedMethods ?? DEFAULT_RESOURCE_METHODS);
@@ -1288,8 +1367,21 @@ class CdpPoolBrowserPage {
             return await run();
         }
         finally {
-            unsubscribe();
-            await this.pageClient.send("Fetch.disable");
+            try {
+                if (this.closePromise) {
+                    // Keep interception active until release settles. The release failure
+                    // stays observable through close() itself; swallowing it here keeps
+                    // this finally from replacing run()'s own outcome, and still avoids a
+                    // reconnect or a disable against a disposed target.
+                    await this.closePromise.catch(() => undefined);
+                }
+                else {
+                    await this.pageClient.send("Fetch.disable");
+                }
+            }
+            finally {
+                unsubscribe();
+            }
         }
     }
     async handleResourcePolicyPausedRequest(params, policy, allowedMethods) {
@@ -1383,8 +1475,31 @@ class CdpPoolBrowserPage {
         this.pageClient.on("Runtime.executionContextsCleared", () => {
             this.frameExecutionContexts.clear();
         });
+        const registerResponseCookies = (params) => {
+            if (!isRecord(params))
+                return;
+            const headers = Array.isArray(params.responseHeaders)
+                ? params.responseHeaders
+                : isRecord(params.headers)
+                    ? Object.entries(params.headers).map(([name, value]) => ({ name, value }))
+                    : [];
+            for (const header of headers) {
+                if (isRecord(header) &&
+                    typeof header.name === "string" &&
+                    header.name.toLowerCase() === "set-cookie" &&
+                    typeof header.value === "string") {
+                    registerDiagnosticValue(header.value);
+                    for (const value of cookieValuesFromHeaders(header.value))
+                        registerDiagnosticValue(value);
+                }
+            }
+        };
+        const scopedCookieRegistration = AsyncResource.bind(registerResponseCookies);
+        this.pageClient.on("Network.responseReceived", scopedCookieRegistration);
+        this.pageClient.on("Network.responseReceivedExtraInfo", scopedCookieRegistration);
         await this.pageClient.send("Page.enable");
         await this.pageClient.send("Runtime.enable");
+        await this.pageClient.send("Network.enable").catch(() => undefined);
         this.initialized = true;
     }
     async getFrameExecutionContextId(frameId) {
@@ -1422,6 +1537,7 @@ class CdpPoolBrowserPage {
 class CdpPoolBrowserClient {
     allowedHosts;
     poolClient;
+    telemetry;
     engine = "playwright-stealth";
     constructor(options) {
         if (!options.cdpUrl) {
@@ -1430,16 +1546,29 @@ class CdpPoolBrowserClient {
             });
         }
         this.allowedHosts = [...new Set(options.allowedHosts ?? [])];
+        this.telemetry = options.telemetry;
         this.poolClient = new JsonRpcWebSocketClient(options.cdpUrl);
     }
     async newPage() {
         return await this.acquirePage({ isolatedContext: true });
     }
     async acquirePage(options) {
-        const acquireResult = parsePoolAcquireResponse(await this.poolClient.send("acquire", {
-            ...(this.allowedHosts.length > 0 ? { allowedHosts: this.allowedHosts } : {}),
-            ...(options?.isolatedContext ? { isolationMode: "browserContext" } : {}),
-        }));
+        let acquireResult;
+        try {
+            acquireResult = parsePoolAcquireResponse(await this.poolClient.send("acquire", {
+                ...(this.allowedHosts.length > 0 ? { allowedHosts: this.allowedHosts } : {}),
+                ...(options?.isolatedContext ? { isolationMode: "browserContext" } : {}),
+            }));
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordPoolAcquire("ok"));
+        }
+        catch (error) {
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordPoolAcquire(poolAcquireOutcome(error)));
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordError("BROWSER_CDP_POOL_ERROR"));
+            const code = poolAcquireCode(error);
+            if (code !== undefined && ![-32001, -32002, -32003, -32004, -32005, -32006].includes(code))
+                observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordPoolAcquireUnknownCode?.(code));
+            throw error;
+        }
         const pageClient = new CdpWebSocketClient(acquireResult.wsEndpoint);
         const page = new CdpPoolBrowserPage(acquireResult.pageId, acquireResult.browserContextId, pageClient, async (request) => {
             await this.poolClient.send("release", request);
@@ -1512,6 +1641,7 @@ export class BrowserClient {
     activePage;
     activePages = new Set();
     _engine;
+    telemetry;
     constructor(options = {}) {
         const resolvedOptions = {
             ...options,
@@ -1519,6 +1649,7 @@ export class BrowserClient {
         };
         const engine = resolvedOptions.engine ?? "playwright-stealth";
         this._engine = engine;
+        this.telemetry = resolvedOptions.telemetry;
         this.cdpUrl = resolvedOptions.cdpUrl;
         if (resolvedOptions.requireCdpPool && !resolvedOptions.cdpUrl) {
             throw new ProviderError("Managed CDP Pool is required for browser providers in production", {
@@ -1543,20 +1674,37 @@ export class BrowserClient {
         return this._engine;
     }
     async newPage() {
-        const page = await this.client.newPage();
-        return this.activatePage(page);
+        observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordEngine(this._engine));
+        try {
+            const page = await this.client.newPage();
+            return this.activatePage(page);
+        }
+        catch (error) {
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordError(browserTelemetryErrorCode(error)));
+            throw error;
+        }
     }
     async rawPage() {
+        observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordEngine(this._engine));
         if (!this.cdpUrl) {
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordPoolAcquire("not_configured"));
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordError("BROWSER_RUNTIME_UNSUPPORTED"));
             throw new ProviderError("ctx.browser.rawPage() requires a CDP pool", {
                 code: "BROWSER_RUNTIME_UNSUPPORTED",
                 fix: "Set APIFUSE__CDP_POOL__URL. The SDK escape hatch is CDP pool-backed only and never launches local Chromium.",
             });
         }
-        const page = await this.client.rawPage();
-        return this.activatePage(page);
+        try {
+            const page = await this.client.rawPage();
+            return this.activatePage(page);
+        }
+        catch (error) {
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordError(browserTelemetryErrorCode(error)));
+            throw error;
+        }
     }
     async withIsolatedContext(handler) {
+        observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordEngine(this._engine));
         const previousActivePage = this.activePage;
         let trackedPage;
         return await this.client.withIsolatedContext(async (page) => {
@@ -1573,24 +1721,35 @@ export class BrowserClient {
         });
     }
     activatePage(page) {
-        let closed = false;
+        let closePromise;
         const originalClose = page.close.bind(page);
         const trackedPage = new Proxy(page, {
             get: (target, property, receiver) => {
                 if (property === "close") {
                     return async () => {
-                        if (closed)
+                        // Repeat calls resolve on the first outcome instead of replaying a
+                        // cached rejection, so a failed release cannot make close() — or
+                        // BrowserClient.close(), which closes every tracked page — throw
+                        // forever.
+                        if (closePromise) {
+                            await closePromise.catch(() => undefined);
                             return;
-                        closed = true;
-                        try {
-                            await originalClose();
                         }
-                        finally {
-                            this.activePages.delete(trackedPage);
-                            if (this.activePage === trackedPage) {
-                                this.activePage = undefined;
+                        closePromise = (async () => {
+                            // Bookkeeping runs even when the release fails: the page is gone
+                            // either way, so leaving it in activePages would leak it and let
+                            // solveChallenge reuse a disposed target.
+                            try {
+                                await originalClose();
                             }
-                        }
+                            finally {
+                                this.activePages.delete(trackedPage);
+                                if (this.activePage === trackedPage) {
+                                    this.activePage = undefined;
+                                }
+                            }
+                        })();
+                        await closePromise;
                     };
                 }
                 const value = Reflect.get(target, property, receiver);
@@ -1602,7 +1761,9 @@ export class BrowserClient {
         return trackedPage;
     }
     async solveChallenge(request) {
+        observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordEngine(this._engine));
         if (request.type !== "recaptcha") {
+            observeBrowserTelemetry(this.telemetry, () => this.telemetry?.recordError("BROWSER_RUNTIME_UNSUPPORTED"));
             throw new ProviderError(`Unsupported browser challenge: ${request.type}`, {
                 code: "BROWSER_RUNTIME_UNSUPPORTED",
             });

package/dist/runtime/cache-telemetry.d.ts

@@ -0,0 +1,71 @@
+import type { ProviderCacheLookupMeta } from "../types.js";
+import { type ClosedEnum, type SpanIndex, type TelemetryContributor } from "./request-telemetry.js";
+export type CacheTelemetrySource = "memory" | "redis" | "loader";
+export type CacheTelemetrySample = {
+    key: string;
+    hit: boolean;
+    stale: boolean;
+    ageMs?: number;
+    source: CacheTelemetrySource;
+};
+export type CacheTelemetryLogPayload = {
+    telemetryFailed?: true;
+    lookups: number;
+    hits: number;
+    stale: number;
+    misses: number;
+    writes: number;
+    loaderErrors: number;
+    writeErrors: number;
+    source: {
+        memory: number;
+        redis: number;
+        loader: number;
+    };
+    redisFallbacks: number;
+    redisMode: ClosedEnum<"not_configured" | "configured" | "degraded">;
+    /** Successful Redis commands, including reads, writes, and deletes; memory-only work counts zero. */
+    redisRoundTrips: number;
+    ageMs?: {
+        p50: number;
+        max: number;
+    };
+    samples: CacheTelemetrySample[];
+    dropped: number;
+};
+export type CacheTelemetryHeaderPayload = Omit<CacheTelemetryLogPayload, "samples" | "ageMs" | "telemetryFailed"> & {
+    ageP50?: number;
+    ageMax?: number;
+};
+export interface CacheTelemetrySink {
+    markTelemetryFailed?(): void;
+    setRedisConfigured?(configured: boolean): void;
+    recordLookup(meta: ProviderCacheLookupMeta): void;
+    recordWrite(): void;
+    recordLoaderError?(): void;
+    recordWriteError?(): void;
+    recordRedisFallback(): void;
+    recordRedisRoundTrip?(): void;
+}
+/** Internal construction binding; keeps sink plumbing off ProviderCacheOptions. */
+export declare function bindCacheTelemetry<T extends object>(options: T, sink: CacheTelemetrySink | undefined): T;
+/** Internal lookup used only by the cache factory. */
+export declare function cacheTelemetryFor(options: object): CacheTelemetrySink | undefined;
+export declare function guardCacheTelemetry(sink: CacheTelemetrySink | undefined): CacheTelemetrySink;
+export declare class CacheTelemetryCollector implements TelemetryContributor<CacheTelemetryLogPayload, CacheTelemetryHeaderPayload>, CacheTelemetrySink {
+    #private;
+    readonly key: "cache";
+    constructor(options?: {
+        redact?: (text: string) => string;
+    });
+    setRedisConfigured(configured: boolean): void;
+    markTelemetryFailed(): void;
+    recordLookup(meta: ProviderCacheLookupMeta): void;
+    recordWrite(): void;
+    recordLoaderError(): void;
+    recordWriteError(): void;
+    recordRedisFallback(): void;
+    recordRedisRoundTrip(): void;
+    toLogPayload(_spans: SpanIndex): CacheTelemetryLogPayload | undefined;
+    toHeaderPayload(log: CacheTelemetryLogPayload): CacheTelemetryHeaderPayload;
+}