@demigodmode/pi-web-agent 1.9.0 → 1.11.0

package/dist/jiti-compat.js

@@ -0,0 +1,215 @@
+import { existsSync, readFileSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { createRequire } from 'node:module';
+import { dirname, join, relative, sep } from 'node:path';
+/**
+ * Works around two incompatibilities between pi's extension loader (a patched
+ * jiti) and jsdom's dependency tree:
+ *
+ *   1. jiti can't resolve the trailing-slash bare specifier require("punycode/")
+ *      used by tr46.
+ *   2. jiti wraps `module.exports = new Set(...)` (cssstyle) in a Proxy, which
+ *      breaks native Set methods on the exported value.
+ *
+ * Both files live in the shared `~/.pi/agent/npm/node_modules` tree, so
+ * installing or updating any *other* pi extension re-extracts them and reverts
+ * the patch. A postinstall hook alone therefore can't keep this healthy, which
+ * is why `ensureJitiCompat()` also runs on extension load, before jsdom is
+ * evaluated. See https://github.com/demigodmode/pi-web-agent/issues/34.
+ *
+ * scripts/patch-jiti-compat.mjs duplicates this logic for the postinstall
+ * hook. That hook runs before `npm run build`, so it cannot import dist/.
+ * Keep the two in sync.
+ */
+const requireFromHere = createRequire(import.meta.url);
+/**
+ * Write via a temp file + rename rather than in place. Two reasons, both of
+ * which bite harder now that this runs on every extension load instead of once
+ * per install:
+ *
+ *   - `writeFileSync` truncates first. A crash, a full disk, or an OOM kill
+ *     mid-write leaves a half-written file that still contains the marker
+ *     comment, so every later run would skip it as "already patched" and the
+ *     doctor would report a broken tree as healthy.
+ *   - Another Pi session can be `require`-ing the same file while we write it.
+ *     `rename` is atomic within a filesystem, so readers see the old file or
+ *     the new one, never a partial one.
+ *
+ * The rename also breaks a pnpm-style hardlink instead of writing through it
+ * into the shared content-addressable store.
+ */
+function writeFileAtomic(path, contents) {
+    const temp = `${path}.pi-web-agent-${process.pid}.tmp`;
+    try {
+        writeFileSync(temp, contents);
+        renameSync(temp, path);
+    }
+    catch (err) {
+        try {
+            if (existsSync(temp))
+                unlinkSync(temp);
+        }
+        catch {
+            // Best effort. Leaving a stray temp file is better than masking the
+            // original write failure.
+        }
+        throw err;
+    }
+}
+const defaultDeps = {
+    resolve: (specifier, fromFile) => (fromFile ? createRequire(fromFile) : requireFromHere).resolve(specifier),
+    existsSync,
+    readFileSync: (path) => readFileSync(path, 'utf8'),
+    writeFileSync: writeFileAtomic
+};
+const SET_SHIM_MARKER = 'pi/jiti workaround';
+const SET_SHIM = `
+// ${SET_SHIM_MARKER}: expose bound native Set methods as own properties so a
+// Proxy wrapper around this export does not break Set brand checks.
+for (const k of ["has", "add", "delete", "forEach", "keys", "values", "entries"]) {
+  module.exports[k] = Set.prototype[k].bind(module.exports);
+}
+module.exports[Symbol.iterator] = Set.prototype[Symbol.iterator].bind(module.exports);
+`;
+const PUNYCODE_SPECIFIER = 'require("punycode/")';
+const PUNYCODE_REPLACEMENT = 'require("punycode/punycode.js")';
+function findPackageRoot(deps, entryFile, packageName) {
+    let directory = dirname(entryFile);
+    for (;;) {
+        const manifestFile = join(directory, 'package.json');
+        if (deps.existsSync(manifestFile)) {
+            try {
+                const manifest = JSON.parse(deps.readFileSync(manifestFile));
+                if (manifest.name === packageName)
+                    return directory;
+            }
+            catch {
+                // An unreadable or malformed package.json on the way up is not our
+                // problem to report. Keep walking.
+            }
+        }
+        const parent = dirname(directory);
+        if (parent === directory) {
+            throw new Error(`could not find the ${packageName} package root from ${entryFile}`);
+        }
+        directory = parent;
+    }
+}
+/**
+ * Resolve the copies jsdom actually loads, not whichever copy happens to sit
+ * highest in the tree.
+ *
+ * `~/.pi/agent/npm/node_modules` is shared by every pi extension, so version
+ * conflicts routinely push a second copy of a package into a nested
+ * `node_modules`. Resolving `cssstyle` or `tr46` from *this* module can then
+ * find a hoisted copy that jsdom never requires: the patch lands on the wrong
+ * file, the load still fails, and the doctor reports a healthy tree because it
+ * checked the same wrong file. Walk the real import chain instead
+ * (jsdom -> cssstyle, jsdom -> whatwg-url -> tr46) and only fall back to a
+ * direct resolve when jsdom is not resolvable at all.
+ */
+function resolveFromJsdom(deps, specifier, via) {
+    try {
+        const jsdomEntry = deps.resolve('jsdom');
+        const importer = via ? deps.resolve(via, jsdomEntry) : jsdomEntry;
+        return deps.resolve(specifier, importer);
+    }
+    catch {
+        // jsdom (or the intermediate package) is not resolvable from here, e.g. a
+        // layout we do not recognize. Fall back to a plain resolve rather than
+        // giving up on the patch entirely.
+        return deps.resolve(specifier);
+    }
+}
+function tr46Entry(deps) {
+    return resolveFromJsdom(deps, 'tr46', 'whatwg-url');
+}
+function cssstyleTargets(deps) {
+    const packageRoot = findPackageRoot(deps, resolveFromJsdom(deps, 'cssstyle'), 'cssstyle');
+    return [
+        join(packageRoot, 'lib', 'allExtraProperties.js'),
+        join(packageRoot, 'lib', 'generated', 'allProperties.js'),
+        join(packageRoot, 'lib', 'generated', 'implementedProperties.js')
+    ].map((file) => ({
+        file,
+        label: `cssstyle/${relative(packageRoot, file).split(sep).join('/')}`
+    }));
+}
+/**
+ * Apply both patches if they are missing. Safe to call repeatedly: an already
+ * patched tree is a few `readFileSync` calls and no writes. Never throws.
+ */
+export function ensureJitiCompat(deps = defaultDeps) {
+    const status = { pending: [], patched: [] };
+    try {
+        const file = tr46Entry(deps);
+        const contents = deps.readFileSync(file);
+        if (contents.includes(PUNYCODE_SPECIFIER)) {
+            deps.writeFileSync(file, contents.replaceAll(PUNYCODE_SPECIFIER, PUNYCODE_REPLACEMENT));
+            status.patched.push('tr46/index.js');
+        }
+    }
+    catch {
+        // A read-only install, a missing dependency, or a future dependency bump
+        // that changes the shape. Report it rather than blocking extension load.
+        status.pending.push('tr46/index.js');
+    }
+    try {
+        for (const { file, label } of cssstyleTargets(deps)) {
+            // Guard the read too: one unreadable file (EACCES, odd permissions) must
+            // not abandon the other two, which may be perfectly writable.
+            try {
+                if (!deps.existsSync(file))
+                    continue;
+                const contents = deps.readFileSync(file);
+                if (contents.includes(SET_SHIM_MARKER))
+                    continue;
+                if (!contents.includes('module.exports = new Set('))
+                    continue;
+                deps.writeFileSync(file, contents + SET_SHIM);
+                status.patched.push(label);
+            }
+            catch {
+                status.pending.push(label);
+            }
+        }
+    }
+    catch {
+        status.pending.push('cssstyle');
+    }
+    return status;
+}
+/**
+ * Read-only view of the same checks, for `/web-agent doctor`. Never throws and
+ * never writes.
+ */
+export function checkJitiCompat(deps = defaultDeps) {
+    const status = { pending: [], patched: [] };
+    try {
+        const contents = deps.readFileSync(tr46Entry(deps));
+        if (contents.includes(PUNYCODE_SPECIFIER))
+            status.pending.push('tr46/index.js');
+    }
+    catch {
+        status.pending.push('tr46/index.js');
+    }
+    try {
+        for (const { file, label } of cssstyleTargets(deps)) {
+            try {
+                if (!deps.existsSync(file))
+                    continue;
+                const contents = deps.readFileSync(file);
+                if (contents.includes(SET_SHIM_MARKER))
+                    continue;
+                if (contents.includes('module.exports = new Set('))
+                    status.pending.push(label);
+            }
+            catch {
+                status.pending.push(label);
+            }
+        }
+    }
+    catch {
+        status.pending.push('cssstyle');
+    }
+    return status;
+}

package/dist/presentation/config-store.js

@@ -61,6 +61,10 @@ function serializeBackendConfigOverride(config) {
     if (config.headless && Object.keys(config.headless).length > 0) {
         backends.headless = { ...config.headless };
     }
+    if (config.proxy && Object.keys(config.proxy).length > 0) {
+        const { password: _password, ...proxy } = config.proxy;
+        backends.proxy = { ...proxy };
+    }
     return { backends };
 }
 async function readConfigFileForWrite(filePath) {

package/dist/readers/youtube-reader.d.ts

@@ -5,6 +5,8 @@ type Subtitle = {
     text: string;
 };
 type YoutubeReaderDeps = {
+    /** Fetch implementation used for YouTube's requests (defaults to global fetch). */
+    fetchImpl?: typeof fetch;
     fetchSubtitles?: (input: {
         videoID: string;
         lang: string;
@@ -17,5 +19,5 @@ type YoutubeReaderDeps = {
         description?: string;
     }>;
 };
-export declare function createYoutubeReader({ fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
+export declare function createYoutubeReader({ fetchImpl, fetchSubtitles, fetchDetails }?: YoutubeReaderDeps): SpecialContentReader;
 export {};

package/dist/readers/youtube-reader.js

@@ -21,7 +21,15 @@ function extractVideoId(url) {
     }
     return undefined;
 }
-export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetails = getVideoDetails } = {}) {
+export function createYoutubeReader({ fetchImpl = fetch, fetchSubtitles, fetchDetails } = {}) {
+    // Route YouTube's caption/metadata requests through the configured fetch
+    // client (e.g. the proxy) unless an explicit override is provided.
+    const doFetchSubtitles = fetchSubtitles ?? ((input) => getSubtitles({ ...input, fetch: fetchImpl }));
+    const doFetchDetails = fetchDetails ??
+        ((input) => getVideoDetails({ ...input, fetch: fetchImpl }).then((details) => ({
+            title: details.title,
+            description: details.description
+        })));
     return {
         name: 'youtube',
         canHandle(url) {
@@ -39,8 +47,8 @@ export function createYoutubeReader({ fetchSubtitles = getSubtitles, fetchDetail
             }
             try {
                 const [subtitles, details] = await Promise.all([
-                    fetchSubtitles({ videoID, lang: 'en' }),
-                    fetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
+                    doFetchSubtitles({ videoID, lang: 'en' }),
+                    doFetchDetails({ videoID, lang: 'en' }).catch(() => ({ title: undefined }))
                 ]);
                 if (!subtitles || subtitles.length === 0) {
                     return {

package/dist/search/duckduckgo.d.ts

@@ -5,5 +5,14 @@ export type ParsedDuckDuckGoResults = {
     hasResultContainers: boolean;
 };
 export declare function buildSearchUrl(query: string): string;
-export declare function fetchDuckDuckGoHtml(query: string): Promise<string>;
+export declare const DUCKDUCKGO_HEADERS: {
+    readonly 'User-Agent': "Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0";
+    readonly Accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
+    readonly 'Accept-Language': "en-US,en;q=0.9";
+};
+export declare function fetchDuckDuckGoHtml(query: string, { fetchImpl, retries, sleep }?: {
+    fetchImpl?: typeof fetch;
+    retries?: number;
+    sleep?: (ms: number) => Promise<void>;
+}): Promise<string>;
 export declare function parseDuckDuckGoResults(html: string): ParsedDuckDuckGoResults;

package/dist/search/duckduckgo.js

@@ -21,12 +21,30 @@ export function buildSearchUrl(query) {
     const params = new URLSearchParams({ q: query });
     return `https://html.duckduckgo.com/html/?${params.toString()}`;
 }
-export async function fetchDuckDuckGoHtml(query) {
-    const response = await fetch(buildSearchUrl(query));
-    if (!response.ok) {
-        throw new Error(`DuckDuckGo request failed with ${response.status}`);
+export const DUCKDUCKGO_HEADERS = {
+    'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:128.0) Gecko/20100101 Firefox/128.0',
+    Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'en-US,en;q=0.9'
+};
+const defaultSleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+export async function fetchDuckDuckGoHtml(query, { fetchImpl = fetch, retries = 1, sleep = defaultSleep } = {}) {
+    let lastError;
+    for (let attempt = 0; attempt <= retries; attempt += 1) {
+        try {
+            const response = await fetchImpl(buildSearchUrl(query), { headers: { ...DUCKDUCKGO_HEADERS } });
+            if (!response.ok) {
+                throw new Error(`DuckDuckGo request failed with ${response.status}`);
+            }
+            return response.text();
+        }
+        catch (error) {
+            lastError = error;
+            if (attempt < retries) {
+                await sleep(500);
+            }
+        }
     }
-    return response.text();
+    throw lastError instanceof Error ? lastError : new Error('DuckDuckGo request failed');
 }
 export function parseDuckDuckGoResults(html) {
     const $ = cheerio.load(html);

package/dist/search/tavily.d.ts

@@ -1,6 +1,7 @@
 import type { WebSearchResponse } from '../types.js';
-export declare function createTavilySearchTool({ apiKey, fetchImpl }: {
+export declare function createTavilySearchTool({ apiKey, keyless, fetchImpl }: {
     apiKey?: string;
+    keyless?: boolean;
     fetchImpl?: typeof fetch;
 }): ({ query }: {
     query: string;

package/dist/search/tavily.js

@@ -17,7 +17,7 @@ function normalizeResults(response) {
         ];
     });
 }
-export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
+export function createTavilySearchTool({ apiKey, keyless = false, fetchImpl = fetch }) {
     return async function tavilySearch({ query }) {
         const normalizedQuery = query.trim();
         if (!normalizedQuery) {
@@ -28,7 +28,7 @@ export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
                 error: { code: 'INVALID_QUERY', message: 'Query must not be empty.' }
             });
         }
-        if (!apiKey?.trim()) {
+        if (!apiKey?.trim() && !keyless) {
             return resultWithPresentation({
                 status: 'error',
                 results: [],
@@ -45,7 +45,9 @@ export function createTavilySearchTool({ apiKey, fetchImpl = fetch }) {
                 headers: {
                     Accept: 'application/json',
                     'Content-Type': 'application/json',
-                    Authorization: `Bearer ${apiKey}`
+                    ...(apiKey?.trim()
+                        ? { Authorization: `Bearer ${apiKey}` }
+                        : { 'X-Tavily-Access-Mode': 'keyless' })
                 },
                 body: JSON.stringify({ query: normalizedQuery, max_results: 10 })
             });

package/dist/tools/web-search.js

@@ -27,7 +27,11 @@ function htmlLooksBlocked(html) {
         normalized.includes('challenge') ||
         normalized.includes('verify you are human') ||
         normalized.includes('are you a robot') ||
-        normalized.includes('unusual traffic'));
+        normalized.includes('unusual traffic') ||
+        normalized.includes('automated requests') ||
+        normalized.includes('automated queries') ||
+        normalized.includes('detected unusual') ||
+        normalized.includes('too many requests'));
 }
 export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache = createTtlCache({ ttlMs: 30_000 }) } = {}) {
     return async function webSearch({ query }) {
@@ -57,8 +61,14 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
             };
         }
         try {
-            const html = await searchHtml(normalizedQuery);
-            const parsed = parseDuckDuckGoResults(html);
+            let html = await searchHtml(normalizedQuery);
+            let parsed = parseDuckDuckGoResults(html);
+            // A 200-OK bot-wall reads as a successful fetch, so the fetch-layer retry never sees it.
+            // Give a page that looks blocked one more shot here before we classify it.
+            if (parsed.results.length === 0 && htmlLooksBlocked(html)) {
+                html = await searchHtml(normalizedQuery);
+                parsed = parseDuckDuckGoResults(html);
+            }
             if (parsed.results.length > 0) {
                 const result = {
                     status: 'ok',
@@ -71,14 +81,16 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
                     presentation: buildSearchPresentation(result)
                 };
             }
-            if (parsed.noResults) {
+            // Check for a bot-wall before "no results": a page can carry both markers, and BLOCKED is
+            // the honest call since it routes to the fallback instead of a dead end.
+            if (htmlLooksBlocked(html)) {
                 const result = {
                     status: 'error',
                     results: [],
                     metadata: { backend: 'duckduckgo', cacheHit: false },
                     error: {
-                        code: 'NO_RESULTS',
-                        message: 'DuckDuckGo returned no usable results for this query.'
+                        code: 'BLOCKED',
+                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
                     }
                 };
                 return {
@@ -86,14 +98,14 @@ export function createWebSearchTool({ searchHtml = fetchDuckDuckGoHtml, cache =
                     presentation: buildSearchPresentation(result)
                 };
             }
-            if (htmlLooksBlocked(html)) {
+            if (parsed.noResults) {
                 const result = {
                     status: 'error',
                     results: [],
                     metadata: { backend: 'duckduckgo', cacheHit: false },
                     error: {
-                        code: 'BLOCKED',
-                        message: 'DuckDuckGo search appears to be blocked or rate limited.'
+                        code: 'NO_RESULTS',
+                        message: 'DuckDuckGo returned no usable results for this query.'
                     }
                 };
                 return {

package/dist/types.d.ts

@@ -20,7 +20,7 @@ export type ToolError = {
 export type SearchMetadata = {
     backend: 'duckduckgo' | 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
     cacheHit: boolean;
-    fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily';
+    fallbackFrom?: 'searxng' | 'brave' | 'youcom' | 'exa' | 'tavily' | 'duckduckgo';
     fallbackReason?: string;
     fanout?: FanoutMetadata;
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@demigodmode/pi-web-agent",
-  "version": "1.9.0",
+  "version": "1.11.0",
   "description": "Pi package for reliable web access with explicit search, fetch, and headless boundaries.",
   "type": "module",
   "main": "./dist/extension.js",
@@ -68,6 +68,7 @@
     "jsdom": "^26.0.0",
     "playwright": "^1.60.0",
     "typebox": "^1.1.37",
+    "undici": "^8.10.2",
     "unpdf": "^1.8.1",
     "youtube-caption-extractor": "^1.10.2"
   },

package/scripts/patch-jiti-compat.mjs

@@ -10,19 +10,63 @@
 // when this package is installed as a pi extension via `pi install npm:...`.
 // Safe to run multiple times and safe to no-op if the target files or
 // patterns are missing (e.g. a future dependency bump changes the shape).
-import { readFileSync, writeFileSync, existsSync } from "node:fs";
+//
+// src/jiti-compat.ts does the same two patches at extension load time, because
+// the shared ~/.pi/agent/npm tree means another extension's install can revert
+// them long after this hook ran (#34). The logic is deliberately duplicated
+// rather than shared: postinstall runs before `npm run build` in CI and in a
+// fresh clone, so this file cannot depend on dist/. Keep the two in sync.
+import { readFileSync, writeFileSync, existsSync, renameSync, unlinkSync } from "node:fs";
 import { createRequire } from "node:module";
 import { dirname, join, relative, sep } from "node:path";
 
-const resolveFromHere = createRequire(import.meta.url).resolve;
+const requireFromHere = createRequire(import.meta.url);
+const resolveFromHere = requireFromHere.resolve;
+
+// Resolve the copies jsdom actually loads. The shared ~/.pi/agent/npm tree
+// often holds a second, nested copy of a package when extensions disagree on
+// versions, and patching a hoisted copy jsdom never requires would fix
+// nothing. Mirrors resolveFromJsdom in src/jiti-compat.ts.
+function resolveFromJsdom(specifier, via) {
+  try {
+    const jsdomEntry = resolveFromHere("jsdom");
+    const importer = via ? createRequire(jsdomEntry).resolve(via) : jsdomEntry;
+    return createRequire(importer).resolve(specifier);
+  } catch {
+    return resolveFromHere(specifier);
+  }
+}
+
+// Temp file + rename, not an in-place write. `writeFileSync` truncates first,
+// so an interrupted write would leave a half-patched file that still contains
+// the marker comment and would be skipped as "already patched" forever.
+// Mirrors writeFileAtomic in src/jiti-compat.ts.
+function writeFileAtomic(path, contents) {
+  const temp = `${path}.pi-web-agent-${process.pid}.tmp`;
+  try {
+    writeFileSync(temp, contents);
+    renameSync(temp, path);
+  } catch (err) {
+    try {
+      if (existsSync(temp)) unlinkSync(temp);
+    } catch {
+      // Best effort.
+    }
+    throw err;
+  }
+}
 
 function findPackageRoot(entryFile, packageName) {
   let directory = dirname(entryFile);
   while (true) {
     const manifestFile = join(directory, "package.json");
     if (existsSync(manifestFile)) {
-      const manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
-      if (manifest.name === packageName) return directory;
+      try {
+        const manifest = JSON.parse(readFileSync(manifestFile, "utf8"));
+        if (manifest.name === packageName) return directory;
+      } catch {
+        // Malformed package.json on the way up. Keep walking.
+      }
     }
 
     const parent = dirname(directory);
@@ -34,7 +78,7 @@ function findPackageRoot(entryFile, packageName) {
 }
 
 function patchTr46() {
-  const file = resolveFromHere("tr46");
+  const file = resolveFromJsdom("tr46", "whatwg-url");
   if (!existsSync(file)) {
     console.debug("patch-jiti-compat: tr46/index.js not found, skipping");
     return;
@@ -44,7 +88,7 @@ function patchTr46() {
     console.debug("patch-jiti-compat: tr46/index.js does not match expected pattern, skipping");
     return;
   }
-  writeFileSync(
+  writeFileAtomic(
     file,
     contents.replaceAll('require("punycode/")', 'require("punycode/punycode.js")'),
   );
@@ -61,7 +105,7 @@ module.exports[Symbol.iterator] = Set.prototype[Symbol.iterator].bind(module.exp
 `;
 
 function patchCssstyleSetExports() {
-  const packageRoot = findPackageRoot(resolveFromHere("cssstyle"), "cssstyle");
+  const packageRoot = findPackageRoot(resolveFromJsdom("cssstyle"), "cssstyle");
   const files = [
     join(packageRoot, "lib", "allExtraProperties.js"),
     join(packageRoot, "lib", "generated", "allProperties.js"),
@@ -79,7 +123,7 @@ function patchCssstyleSetExports() {
       console.debug(`patch-jiti-compat: ${label} does not match expected pattern, skipping`);
       continue;
     }
-    writeFileSync(file, contents + SET_SHIM);
+    writeFileAtomic(file, contents + SET_SHIM);
     console.log(`patch-jiti-compat: patched ${label}`);
   }
 }