@tduyng/pi-web 0.0.0-stage → 0.0.1

package/LICENSE

@@ -0,0 +1,178 @@
+Apache License
+Version 2.0, January 2004
+http://www.apache.org/licenses/
+
+TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+1. Definitions.
+
+"License" shall mean the terms and conditions for use, reproduction,
+and distribution as defined by Sections 1 through 9 of this document.
+
+"Licensor" shall mean the copyright owner or entity authorized by
+the copyright owner that is granting the License.
+
+"Legal Entity" shall mean the union of the acting entity and all
+other entities that control, are controlled by, or are under common
+control with that entity. For the purposes of this definition,
+"control" means (i) the power, direct or indirect, to cause the
+direction or management of such entity, whether by contract or
+otherwise, or (ii) ownership of fifty percent (50%) or more of the
+outstanding shares, or (iii) beneficial ownership of such entity.
+
+"You" (or "Your") shall mean an individual or Legal Entity
+exercising permissions granted by this License.
+
+"Source" form shall mean the preferred form for making modifications,
+including but not limited to software source code, documentation
+source, and configuration files.
+
+"Object" form shall mean any form resulting from mechanical
+transformation or translation of a Source form, including but
+not limited to compiled object code, generated documentation,
+and conversions to other media types.
+
+"Work" shall mean the work of authorship, whether in Source or
+Object form, made available under the License, as indicated by a
+copyright notice that is included in or attached to the work
+(an example is provided in the Appendix below).
+
+"Derivative Works" shall mean any work, whether in Source or Object
+form, that is based on (or derived from) the Work and for which the
+editorial revisions, annotations, elaborations, or other modifications
+represent, as a whole, an original work of authorship. For the purposes
+of this License, Derivative Works shall not include works that remain
+separable from, or merely link (or bind by name) to the interfaces of,
+the Work and Derivative Works thereof.
+
+"Contribution" shall mean any work of authorship, including
+the original version of the Work and any modifications or additions
+to that Work or Derivative Works thereof, that is intentionally
+submitted to Licensor for inclusion in the Work by the copyright owner
+or by an individual or Legal Entity authorized to submit on behalf of
+the copyright owner. For the purposes of this definition, "submitted"
+means any form of electronic, verbal, or written communication sent
+to the Licensor or its representatives, including but not limited to
+communication on electronic mailing lists, source code control systems,
+and issue tracking systems that are managed by, or on behalf of, the
+Licensor for the purpose of discussing and improving the Work, but
+excluding communication that is conspicuously marked or otherwise
+designated in writing by the copyright owner as "Not a Contribution."
+
+"Contributor" shall mean Licensor and any individual or Legal Entity
+on behalf of whom a Contribution has been received by Licensor and
+subsequently incorporated within the Work.
+
+2. Grant of Copyright License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+copyright license to reproduce, prepare Derivative Works of,
+publicly display, publicly perform, sublicense, and distribute the
+Work and such Derivative Works in Source or Object form.
+
+3. Grant of Patent License. Subject to the terms and conditions of
+this License, each Contributor hereby grants to You a perpetual,
+worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+(except as stated in this section) patent license to make, have made,
+Use, offer to sell, sell, import, and otherwise transfer the Work,
+where such license applies only to those patent claims licensable
+by such Contributor that are necessarily infringed by their
+Contribution(s) alone or by combination of their Contribution(s)
+with the Work to which such Contribution(s) was submitted. If You
+institute patent litigation against any entity (including a
+cross-claim or counterclaim in a lawsuit) alleging that the Work
+or a Contribution incorporated within the Work constitutes direct
+or contributory patent infringement, then any patent licenses
+granted to You under this License for that Work shall terminate
+as of the date such litigation is filed.
+
+4. Redistribution. You may reproduce and distribute copies of the
+Work or Derivative Works thereof in any medium, with or without
+modifications, and in Source or Object form, provided that You
+meet the following conditions:
+
+(a) You must give any other recipients of the Work or
+Derivative Works a copy of this License; and
+
+(b) You must cause any modified files to carry prominent notices
+stating that You changed the files; and
+
+(c) You must retain, in the Source form of any Derivative Works
+that You distribute, all copyright, patent, trademark, and
+attribution notices from the Source form of the Work,
+excluding those notices that do not pertain to any part of
+the Derivative Works; and
+
+(d) If the Work includes a "NOTICE" text file as part of its
+distribution, then any Derivative Works that You distribute must
+include a readable copy of the attribution notices contained
+within such NOTICE file, excluding those notices that do not
+pertain to any part of the Derivative Works, in at least one
+of the following places: within a NOTICE text file distributed
+as part of the Derivative Works; within the Source form or
+documentation, if provided along with the Derivative Works; or,
+within a display generated by the Derivative Works, if and
+wherever such third-party notices normally appear. The contents
+of the NOTICE file are for informational purposes only and
+do not modify the License. You may add Your own attribution
+notices within Derivative Works that You distribute, alongside
+or as an addendum to the NOTICE text from the Work, provided
+that such additional attribution notices cannot be construed
+as modifying the License.
+
+You may add Your own copyright statement to Your modifications and
+may provide additional or different license terms and conditions
+for use, reproduction, or distribution of Your modifications, or
+for any such Derivative Works as a whole, provided Your use,
+reproduction, and distribution of the Work otherwise complies with
+the conditions stated in this License.
+
+5. Submission of Contributions. Unless You explicitly state otherwise,
+any Contribution intentionally submitted for inclusion in the Work
+by You to the Licensor shall be under the terms and conditions of
+this License, without any additional terms or conditions.
+Notwithstanding the above, nothing herein shall supersede or modify
+the terms of any separate license agreement you may have executed
+with Licensor regarding such Contributions.
+
+6. Trademarks. This License does not grant permission to use the trade
+names, trademarks, service marks, or product names of the Licensor,
+except as required for reasonable and customary use in describing the
+origin of the Work and reproducing the content of the NOTICE file.
+
+7. Disclaimer of Warranty. Unless required by applicable law or
+agreed to in writing, Licensor provides the Work (and each
+Contributor provides its Contributions) on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+implied, including, without limitation, any warranties or conditions
+of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+PARTICULAR PURPOSE. You are solely responsible for determining the
+appropriateness of using or redistributing the Work and assume any
+risks associated with Your exercise of permissions under this License.
+
+8. Limitation of Liability. In no event and under no legal theory,
+whether in tort (including negligence), contract, or otherwise,
+unless required by applicable law (such as deliberate and grossly
+negligent acts) or agreed to in writing, shall any Contributor be
+liable to You for damages, including any direct, indirect, special,
+incidental, or consequential damages of any character arising as a
+result of this License or out of the use or inability to use the
+Work (including but not limited to damages for loss of goodwill,
+work stoppage, computer failure or malfunction, or any and all
+other commercial damages or losses), even if such Contributor
+has been advised of the possibility of such damages.
+
+9. Accepting Warranty or Additional Liability. While redistributing
+the Work or Derivative Works thereof, You may choose to offer,
+and charge a fee for, acceptance of support, warranty, indemnity,
+or other liability obligations and/or rights consistent with this
+License. However, in accepting such obligations, You may act only
+on Your own behalf and on Your sole responsibility, not on behalf
+of any other Contributor, and only if You agree to indemnify,
+defend, and hold each Contributor harmless for any liability
+incurred by, or claims asserted against, such Contributor by reason
+of your accepting any such warranty or additional liability.
+
+END OF TERMS AND CONDITIONS
+
+Copyright 2026 tduyng

package/README.md

@@ -1,3 +1,39 @@
-# Temporary Holding Version
+# @tduyng/pi-web
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+Web search and fetch for pi. The extension registers `web_search` and `web_fetch`, and registers the keyless Exa MCP server through pi's own MCP client: `pi.registerMcpServer('exa', ...)`, exposure `codemode`, listed in `/mcp`. The tools call that server's two tools, `web_search_exa` and `web_fetch_exa`, through `ctx.executeTool` and carry no MCP SDK of their own. The server takes a 60 second request timeout, which covers the connect handshake, and pi retries only HTTP 408, 429 and 5xx: a handshake that overruns marks the server failed for the session and `/mcp` shows why.
+
+## Backends
+
+`web_search` takes an array of up to six queries, one call each, and groups the results under the query that produced them. The primary backend is keyless Exa MCP, with DuckDuckGo HTML behind it. The Exa leg goes through the registered `exa` server, so an unconnected or `--no-mcp` session lands in the same DuckDuckGo fallback. x.com and twitter.com URLs go to Fixupx instead, which answers a page X refuses. It returns a post together with the whole long-form Article it links. An `x.com/i/article/` URL carries an article id rather than a post id, so the reply names the post URL to fetch.
+
+`source: arxiv` searches the arXiv Atom API instead, sorted by submission date, and returns abstracts with authors and dates. A miss falls back to the open web, because arXiv indexes preprints and nothing else.
+
+`source: hn` searches Hacker News through Algolia's index, which needs no account, and returns each thread's points and comment count, the only measure of how much discussion a subject has. A comment hit is reported as the thread that holds it, with the comment's first 300 characters. A miss falls back to the open web.
+
+Search returns excerpts, so the result ends with the ranked URLs to read. Without that list the model answers from a snippet.
+
+## Typed results
+
+`web_search` declares an output schema, so a codemode script gets the answer as data in `structuredContent.groups`, one entry per query carrying `title`, `url` and `snippet`, plus `published` and `authors` where the backend has them, and `points` and `comments` on Hacker News. A script can compare values across results without parsing text. Exa's answer is prose: `Title:`, `URL:`, `Published:`, `Author:` and `Highlights:` blocks with a rule between them, which `parseExa` reads back into that shape, so the typed view holds for every backend. The model still receives the text, and the data is not sent to it a second time.
+
+A backend that answers with nothing is a miss, and the next one runs. Before, an empty Exa answer ended the search and DuckDuckGo never got a turn.
+
+## DuckDuckGo queue
+
+DuckDuckGo answers a burst of requests from one address with a bot check, and keeps that address locked out for 10 to 15 minutes, with each retry extending it. Measured on 2026-10-26: six requests at once got three pages of results and three challenge pages at 14 KB apiece, and the challenge arrives as HTTP 202 as often as a 403. Searches therefore run one at a time, two seconds apart. A bot check pauses the provider for five minutes, doubling to an hour while it keeps answering that way, and a paused search fails at once instead of waiting, so the batch reports that query as failed rather than holding the tool open.
+
+## Links in a fetched page
+
+Exa's extraction returns plain text. Measured on 2026-09-27, `arxiv.org/abs/1706.03762` came back with no links at all, so a citation cannot be read out of the page that cites it. Jina Reader returns Markdown with the links intact, 295 of them on a Hacker News thread, but it blocks a domain that hammered it: `arxiv.org` answered 403 until 2026-09-27. Exa answers first for that reason, and the links survive only when Jina does. Follow a citation by searching for the paper's title.
+
+## arXiv query form
+
+arXiv ANDs nothing on its own. Measured on the live API: `typed decision model` returns 1,470,125 rows because the words are read as alternatives, and `all:"typed decision model"` returns 3 because the phrase is too strict. So a plain query has its words ANDed one at a time, which is the only form that ranks usefully. A query already written with arXiv field prefixes (`ti:`, `cat:`, `au:`, and the rest) is passed through untouched.
+
+## Reddit and X
+
+Neither has a keyless search. Measured on 2026-09-27: `reddit.com/search.json` answers 403 on `www`, `old` and `api`, with a browser user agent and with a plain one, and X has no search without a key. DuckDuckGo's `site:` operator does return both, so a query that names the forum reaches those threads. The web backend finds them when a query names them: `Hacker News and Reddit discussion of typed decision models` returned a `news.ycombinator.com` host, while the bare query did not.
+
+## Quota
+
+The keyless Exa MCP endpoint needs no account. Set `EXA_API_KEY` to send your own key. The extension reads it at load and puts it on the registered server URL as the `exaApiKey` query parameter. Exa's free plan includes 1,000 requests per month. A batch of six queries costs six of them, so they run three at a time.

package/package.json

@@ -1,6 +1,44 @@
 {
   "name": "@tduyng/pi-web",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.0.1",
+  "description": "Keyless web search and fetch for pi: batched queries, arXiv preprints, Exa MCP with DuckDuckGo and Jina Reader fallbacks, zero dependencies.",
+  "keywords": [
+    "pi",
+    "pi-package",
+    "pi-extension",
+    "pi-coding-agent",
+    "web-search",
+    "exa"
+  ],
+  "homepage": "https://gitlab.com/tduyng/pi-agent/-/tree/main/extensions/pi-web#readme",
+  "bugs": "https://gitlab.com/tduyng/pi-agent/-/issues",
+  "license": "Apache-2.0",
+  "author": "tduyng",
+  "repository": {
+    "type": "git",
+    "url": "git+https://gitlab.com/tduyng/pi-agent.git",
+    "directory": "extensions/pi-web"
+  },
+  "type": "module",
+  "files": [
+    "src",
+    "!src/**/*.test.ts",
+    "README.md",
+    "LICENSE"
+  ],
+  "peerDependencies": {
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "pi": {
+    "extensions": [
+      "./src/index.ts"
+    ]
+  },
+  "dependencies": {
+    "ata-validator": "^1.62.1"
+  }
+}

package/src/backends.ts

@@ -0,0 +1,539 @@
+import { FIXUPX_STATUS_URL, isXUrl, X_ARTICLE_ID_ERROR, xArticleId, xPostId, xPostMarkdown } from './x-post.js'
+
+const SEARCH_OBJECTIVE =
+    'Rank pages that answer the query directly. Prefer primary sources, official documentation, and canonical pages. Skip aggregators and SEO mirrors.'
+const ARXIV_API = 'https://export.arxiv.org/api/query'
+const HN_SEARCH = 'https://hn.algolia.com/api/v1/search'
+const TIMEOUT_MS = 20_000
+export const MAX_QUERIES = 6
+const CONCURRENCY = 3
+const ABSTRACT_CHARS = 700
+const READ_LINKS = 30
+const COMMENT_CHARS = 300
+
+export const SOURCES = ['web', 'arxiv', 'hn'] as const
+export type Source = (typeof SOURCES)[number]
+
+export type Fetch = typeof fetch
+
+export interface Hit {
+    text: string
+    backend: string
+}
+
+export type SearchResult = {
+    title: string
+    url: string
+    snippet: string
+    published?: string
+    authors?: string[]
+    points?: number
+    comments?: number
+}
+
+export type SearchHit = Hit & { results: SearchResult[] }
+
+export type SearchGroup = {
+    query: string
+    backend: string
+    results: SearchResult[]
+}
+
+export type ExaArgs =
+    | { query: string; numResults: number; objective: string }
+    | { urls: string[]; maxCharacters: number }
+
+export type ExaCall = (name: string, args: ExaArgs, signal?: AbortSignal) => Promise<string>
+
+function withTimeout(signal?: AbortSignal): AbortSignal {
+    const timeout = AbortSignal.timeout(TIMEOUT_MS)
+    return signal ? AbortSignal.any([signal, timeout]) : timeout
+}
+
+const ENTITIES = new Map<string, string>([
+    ['amp', '&'],
+    ['lt', '<'],
+    ['gt', '>'],
+    ['quot', '"'],
+    ['apos', "'"],
+    ['nbsp', ' ']
+])
+
+export function decodeEntities(text: string): string {
+    return text.replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos|nbsp);/gi, (whole, entity: string) => {
+        const known = ENTITIES.get(entity.toLowerCase())
+        if (known !== undefined) return known
+        const code = entity.startsWith('#x')
+            ? Number.parseInt(entity.slice(2), 16)
+            : Number.parseInt(entity.slice(1), 10)
+        return Number.isFinite(code) && code < 0x110_000 ? String.fromCodePoint(code) : whole
+    })
+}
+
+function clean(html: string): string {
+    return decodeEntities(html.replace(/<[^>]+>/g, ''))
+        .replace(/\s+/g, ' ')
+        .trim()
+}
+
+function ddgTarget(href: string): string {
+    const raw = href.replace(/&amp;/g, '&')
+    const url = new URL(raw.startsWith('//') ? `https:${raw}` : raw)
+    return url.searchParams.get('uddg') ?? url.href
+}
+
+export interface DdgResult {
+    title: string
+    url: string
+    snippet: string
+}
+
+export function parseDdg(html: string): DdgResult[] {
+    const out: DdgResult[] = []
+    for (const chunk of html.split('class="result__a"').slice(1)) {
+        const link = /^\s*href="([^"]+)"[^>]*>([\s\S]*?)<\/a>/.exec(chunk)
+        if (!link?.[1] || link[1].includes('duckduckgo.com/y.js')) continue
+        const snippet = /class="result__snippet"[^>]*>([\s\S]*?)<\/a>/.exec(chunk)
+        out.push({ title: clean(link[2] ?? ''), url: ddgTarget(link[1]), snippet: clean(snippet?.[1] ?? '') })
+    }
+    return out
+}
+
+const DDG_GAP_MS = 2_000
+const DDG_BACKOFF_MS = 5 * 60_000
+const DDG_BACKOFF_MAX_MS = 60 * 60_000
+
+export const ddgGate = { gapMs: DDG_GAP_MS, freeAt: 0, backoffMs: 0, blockedUntil: 0 }
+
+async function sleepUntil(at: number, signal?: AbortSignal): Promise<void> {
+    const wait = at - Date.now()
+    if (wait <= 0) return
+    await new Promise<void>((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', onAbort)
+            resolve()
+        }, wait)
+        const onAbort = () => {
+            clearTimeout(timer)
+            reject(new Error('aborted while waiting its turn'))
+        }
+        signal?.addEventListener('abort', onAbort, { once: true })
+    })
+}
+
+function ddgPause(reason: string): never {
+    ddgGate.backoffMs = Math.min(ddgGate.backoffMs ? ddgGate.backoffMs * 2 : DDG_BACKOFF_MS, DDG_BACKOFF_MAX_MS)
+    ddgGate.blockedUntil = Date.now() + ddgGate.backoffMs
+    throw new Error(`bot check ${reason}, paused for ${ddgGate.backoffMs / 60_000} min`)
+}
+
+export async function ddgSearch(
+    query: string,
+    numResults: number,
+    signal?: AbortSignal,
+    f: Fetch = fetch
+): Promise<SearchHit> {
+    signal?.throwIfAborted()
+    const blocked = ddgGate.blockedUntil - Date.now()
+    if (blocked > 0) throw new Error(`paused after a bot check, ${Math.ceil(blocked / 60_000)} min left`)
+    const start = Math.max(Date.now(), ddgGate.freeAt)
+    ddgGate.freeAt = start + ddgGate.gapMs
+    await sleepUntil(start, signal)
+    const res = await f(`https://html.duckduckgo.com/html/?q=${encodeURIComponent(query)}`, {
+        headers: { 'user-agent': 'Mozilla/5.0' },
+        signal: withTimeout(signal)
+    })
+    if (res.status === 202 || res.status === 403 || res.status === 429) ddgPause(`answered HTTP ${res.status}`)
+    if (!res.ok) throw new Error(`HTTP ${res.status}`)
+    const html = await res.text()
+    if (html.includes('duckduckgo.com/anomaly.js')) ddgPause('page')
+    const results = parseDdg(html).slice(0, numResults)
+    if (!results.length) throw new Error('no results')
+    ddgGate.backoffMs = 0
+    return {
+        text: results.map((r) => `Title: ${r.title}\nURL: ${r.url}\n${r.snippet}`).join('\n\n'),
+        backend: 'duckduckgo',
+        results
+    }
+}
+
+const truncatedAt = (maxCharacters: number) => `[truncated at ${maxCharacters} chars]`
+
+function markTruncation(pages: string, maxCharacters: number): string {
+    return pages
+        .split(/^(?=# .*\nURL: \S+\n)/m)
+        .map((page) => {
+            const header = page.indexOf('\n\n')
+            const body = header < 0 ? '' : page.slice(header + 2).trimEnd()
+            return body.length < maxCharacters ? page.trimEnd() : `${page.trimEnd()}\n${truncatedAt(maxCharacters)}`
+        })
+        .filter(Boolean)
+        .join('\n\n')
+}
+
+async function fetchEach(
+    urls: string[],
+    maxCharacters: number,
+    signal: AbortSignal | undefined,
+    run: (url: string) => Promise<string>
+): Promise<string> {
+    let failed = 0
+    const pages = await Promise.all(
+        urls.map(async (url) => {
+            try {
+                const page = await run(url)
+                return page.length > maxCharacters
+                    ? `${page.slice(0, maxCharacters)}\n${truncatedAt(maxCharacters)}`
+                    : page
+            } catch (err) {
+                failed++
+                return `URL: ${url}\nError: ${err instanceof Error ? err.message : String(err)}`
+            }
+        })
+    )
+    if (failed === urls.length) throw new Error(pages.join('; '))
+    return pages.join('\n\n---\n\n')
+}
+
+export async function jinaFetch(
+    urls: string[],
+    maxCharacters: number,
+    signal?: AbortSignal,
+    f: Fetch = fetch
+): Promise<string> {
+    return fetchEach(urls, maxCharacters, signal, async (url) => {
+        const res = await f(`https://r.jina.ai/${url}`, { signal: withTimeout(signal) })
+        if (!res.ok) throw new Error(`HTTP ${res.status}`)
+        return res.text()
+    })
+}
+
+async function xFetch(
+    urls: string[],
+    maxCharacters: number,
+    signal: AbortSignal | undefined,
+    f: Fetch
+): Promise<string> {
+    return fetchEach(urls, maxCharacters, signal, async (url) => {
+        if (xArticleId(url)) throw new Error(X_ARTICLE_ID_ERROR)
+        const postId = xPostId(url)
+        if (!postId) throw new Error('no X post id in this URL')
+        const res = await f(`${FIXUPX_STATUS_URL}${postId}`, { signal: withTimeout(signal) })
+        if (!res.ok) throw new Error(`Fixupx HTTP ${res.status}`)
+        return xPostMarkdown(await res.text())
+    })
+}
+
+async function firstOf<T extends { text: string }>(
+    signal: AbortSignal | undefined,
+    backends: [string, () => Promise<T>][]
+): Promise<T & Hit> {
+    const errors: string[] = []
+    for (const [backend, run] of backends) {
+        try {
+            const hit = await run()
+            if (hit.text.trim()) return { ...hit, backend }
+            errors.push(`${backend}: empty answer`)
+        } catch (err) {
+            if (signal?.aborted) throw err
+            errors.push(`${backend}: ${err instanceof Error ? err.message : String(err)}`)
+        }
+    }
+    throw new Error(errors.join('; '))
+}
+
+function exaField(block: string, name: string): string | undefined {
+    const value = new RegExp(`^${name}: (.*)$`, 'm').exec(block)?.[1]?.trim()
+    return value && value !== 'N/A' ? value : undefined
+}
+
+export function parseExa(text: string): SearchResult[] {
+    return text.split(/\n-{3,}\n/).flatMap((block): SearchResult[] => {
+        const title = exaField(block, 'Title')
+        const url = exaField(block, 'URL')
+        if (!title || !url) return []
+        const highlights = block.indexOf('Highlights:')
+        const result: SearchResult = {
+            title,
+            url,
+            snippet: highlights < 0 ? '' : block.slice(highlights + 'Highlights:'.length).trim()
+        }
+        const published = exaField(block, 'Published')
+        if (published) result.published = published
+        const author = exaField(block, 'Author')
+        if (author) result.authors = [author]
+        return [result]
+    })
+}
+
+export async function search(
+    query: string,
+    numResults: number,
+    source: Source,
+    signal: AbortSignal | undefined,
+    exa: ExaCall,
+    f: Fetch
+): Promise<SearchHit> {
+    const special = SOURCE_SEARCH[source]
+    if (special) {
+        try {
+            return await special(query, numResults, signal, f)
+        } catch (err) {
+            if (signal?.aborted) throw err
+        }
+    }
+    return firstOf<{ text: string; results: SearchResult[] }>(signal, [
+        [
+            'exa',
+            async () => {
+                const text = await exa('web_search_exa', { query, numResults, objective: SEARCH_OBJECTIVE }, signal)
+                return { text, results: parseExa(text) }
+            }
+        ],
+        ['duckduckgo', () => ddgSearch(query, numResults, signal, f)]
+    ])
+}
+
+export interface Paper {
+    title: string
+    url: string
+    published: string
+    authors: string[]
+    abstract: string
+}
+
+function atomText(entry: string, tag: string): string {
+    const found = new RegExp(`<${tag}[^>]*>([\\s\\S]*?)</${tag}>`).exec(entry)
+    return found?.[1]
+        ? decodeEntities(found[1].replace(/<[^>]+>/g, ' '))
+              .replace(/\s+/g, ' ')
+              .trim()
+        : ''
+}
+
+export function parseArxiv(xml: string): Paper[] {
+    const papers: Paper[] = []
+    for (const entry of xml.split('<entry>').slice(1)) {
+        const title = atomText(entry, 'title')
+        const url = atomText(entry, 'id')
+        if (!title || !url) continue
+        papers.push({
+            title,
+            url: url.replace('http://', 'https://'),
+            published: atomText(entry, 'published').slice(0, 10),
+            authors: [...entry.matchAll(/<name>([\s\S]*?)<\/name>/g)].map((m) => m[1]?.trim() ?? '').filter(Boolean),
+            abstract: atomText(entry, 'summary').slice(0, ABSTRACT_CHARS)
+        })
+    }
+    return papers
+}
+
+export function arxivQuery(query: string): string {
+    if (/(?:^|\s)(?:all|ti|abs|au|co|jr|cat|rn|id|src)\s*:/i.test(query)) return query
+    return query
+        .split(/\s+/)
+        .filter(Boolean)
+        .map((term) => `all:"${term}"`)
+        .join(' AND ')
+}
+
+export async function arxivSearch(
+    query: string,
+    numResults: number,
+    signal?: AbortSignal,
+    f: Fetch = fetch
+): Promise<SearchHit> {
+    const url = new URL(ARXIV_API)
+    url.searchParams.set('search_query', arxivQuery(query))
+    url.searchParams.set('start', '0')
+    url.searchParams.set('max_results', String(numResults))
+    url.searchParams.set('sortBy', 'submittedDate')
+    url.searchParams.set('sortOrder', 'descending')
+    const res = await f(url, { headers: { 'user-agent': 'pi-web' }, signal: withTimeout(signal) })
+    if (!res.ok) throw new Error(`HTTP ${res.status}`)
+    const papers = parseArxiv(await res.text())
+    if (!papers.length) throw new Error('no papers')
+    return {
+        text: papers
+            .map(
+                (p) =>
+                    `Title: ${p.title}\nURL: ${p.url}\nPublished: ${p.published}\nAuthor: ${p.authors.join(', ')}\nAbstract: ${p.abstract}`
+            )
+            .join('\n\n'),
+        backend: 'arxiv',
+        results: papers.map((p) => ({
+            title: p.title,
+            url: p.url,
+            snippet: p.abstract,
+            published: p.published,
+            authors: p.authors
+        }))
+    }
+}
+
+interface HnHit {
+    title?: string
+    story_title?: string
+    objectID?: string
+    created_at?: string
+    points?: number
+    num_comments?: number
+    comment_text?: string
+}
+
+function parseHits(text: string): HnHit[] | undefined {
+    return (JSON.parse(text) as { hits?: HnHit[] }).hits
+}
+
+export interface HnPost {
+    title: string
+    url: string
+    published: string
+    points: number
+    comments: number
+    comment: string
+}
+
+export async function hnSearch(
+    query: string,
+    numResults: number,
+    signal?: AbortSignal,
+    f: Fetch = fetch
+): Promise<SearchHit> {
+    const url = new URL(HN_SEARCH)
+    url.searchParams.set('query', query)
+    url.searchParams.set('hitsPerPage', String(numResults))
+    const res = await f(url, { signal: withTimeout(signal) })
+    if (!res.ok) throw new Error(`HTTP ${res.status}`)
+    const posts = (parseHits(await res.text()) ?? []).flatMap((hit): HnPost[] => {
+        const title = hit.title ?? hit.story_title
+        if (!title || !hit.objectID) return []
+        return [
+            {
+                title,
+                url: `https://news.ycombinator.com/item?id=${hit.objectID}`,
+                published: (hit.created_at ?? '').slice(0, 10),
+                points: hit.points ?? 0,
+                comments: hit.num_comments ?? 0,
+                comment: clean(hit.comment_text ?? '').slice(0, COMMENT_CHARS)
+            }
+        ]
+    })
+    if (!posts.length) throw new Error('no posts')
+    return {
+        text: posts
+            .map((p) =>
+                [
+                    `Title: ${p.title}`,
+                    `URL: ${p.url}`,
+                    `Published: ${p.published}`,
+                    `Points: ${p.points}`,
+                    `Comments: ${p.comments}`,
+                    ...(p.comment ? [`Comment: ${p.comment}`] : [])
+                ].join('\n')
+            )
+            .join('\n\n'),
+        backend: 'hn',
+        results: posts.map((p) => ({
+            title: p.title,
+            url: p.url,
+            snippet: p.comment,
+            published: p.published,
+            points: p.points,
+            comments: p.comments
+        }))
+    }
+}
+
+type SourceSearch = (query: string, numResults: number, signal?: AbortSignal, f?: Fetch) => Promise<SearchHit>
+
+const SOURCE_SEARCH: Record<Source, SourceSearch | undefined> = {
+    web: undefined,
+    arxiv: arxivSearch,
+    hn: hnSearch
+}
+
+export type SearchManyHit = Hit & { groups: SearchGroup[] }
+
+export async function searchMany(
+    queries: string[],
+    numResults: number,
+    source: Source,
+    signal: AbortSignal | undefined,
+    exa: ExaCall,
+    f: Fetch
+): Promise<SearchManyHit> {
+    const sections: { query: string; hit: SearchHit }[] = []
+    for (let i = 0; i < queries.length; i += CONCURRENCY) {
+        const wave = await Promise.all(
+            queries.slice(i, i + CONCURRENCY).map((query) =>
+                search(query, numResults, source, signal, exa, f).catch((err): SearchHit => ({
+                    text: `Error: ${err instanceof Error ? err.message : String(err)}`,
+                    backend: 'failed',
+                    results: []
+                }))
+            )
+        )
+        for (const [offset, hit] of wave.entries()) sections.push({ query: queries[i + offset] ?? '', hit })
+    }
+
+    const groups: SearchGroup[] = sections.map(({ query, hit }) => ({
+        query,
+        backend: hit.backend,
+        results: hit.results
+    }))
+    const links = [...new Set(groups.flatMap((group) => group.results.map((result) => result.url)))]
+    const body = sections.map(({ query, hit }) =>
+        queries.length === 1 ? hit.text : `## ${query}\n[${hit.backend}]\n\n${hit.text}`
+    )
+    const backend = [...new Set(sections.map((s) => s.hit.backend))].join('+')
+    if (!links.length) return { text: body.join('\n\n'), backend, groups }
+
+    const read = links.slice(0, READ_LINKS)
+    const tail = `\n\n# Read these pages\nweb_fetch the promising ones among these ${read.length} URLs: ${read.join(' ')}`
+    return { text: body.join('\n\n') + tail, backend, groups }
+}
+
+export async function fetchPages(
+    urls: string[],
+    maxCharacters: number,
+    signal: AbortSignal | undefined,
+    exa: ExaCall,
+    f: Fetch = fetch
+): Promise<Hit> {
+    const xUrls = urls.filter(isXUrl)
+    const otherUrls = urls.filter((url) => !isXUrl(url))
+    if (!xUrls.length) return firstOf(signal, exaAndJina(otherUrls, maxCharacters, signal, exa, f))
+    if (!otherUrls.length) {
+        const text = await xFetch(xUrls, maxCharacters, signal, f).catch(
+            (err) => `Error: ${err instanceof Error ? err.message : String(err)}`
+        )
+        return { text, backend: 'fxtwitter' }
+    }
+
+    const [xText, other] = await Promise.all([
+        xFetch(xUrls, maxCharacters, signal, f).catch(
+            (err) => `Error: ${err instanceof Error ? err.message : String(err)}`
+        ),
+        firstOf(signal, exaAndJina(otherUrls, maxCharacters, signal, exa, f))
+    ])
+    return { text: `${xText}\n\n---\n\n${other.text}`, backend: `fxtwitter+${other.backend}` }
+}
+
+function exaAndJina(
+    urls: string[],
+    maxCharacters: number,
+    signal: AbortSignal | undefined,
+    exa: ExaCall,
+    f: Fetch
+): [string, () => Promise<{ text: string }>][] {
+    return [
+        [
+            'exa',
+            async () => ({
+                text: markTruncation(await exa('web_fetch_exa', { urls, maxCharacters }, signal), maxCharacters)
+            })
+        ],
+        ['jina', async () => ({ text: await jinaFetch(urls, maxCharacters, signal, f) })]
+    ]
+}

package/src/index.ts

@@ -0,0 +1,151 @@
+import type { ExtensionAPI, ExtensionToolContext, Theme } from '@earendil-works/pi-coding-agent'
+import { Text } from '@earendil-works/pi-tui'
+import type { Infer } from 'ata-validator'
+import { t } from 'ata-validator/t'
+import { fetchPages, MAX_QUERIES, searchMany, SOURCES, type ExaCall } from './backends.js'
+
+const EXA_MCP = 'https://mcp.exa.ai/mcp'
+
+function exaViaMcp(ctx: ExtensionToolContext): ExaCall {
+    return async (name, args, signal) => {
+        const outcome = await ctx.executeTool(`mcp__exa__${name}`, args, { signal })
+        const text = outcome.result.content
+            .map((item) => (item.type === 'text' ? item.text : ''))
+            .filter(Boolean)
+            .join('\n')
+        if (outcome.isError) throw new Error(text || 'tool error')
+        return text
+    }
+}
+
+interface ToolRenderArgs {
+    queries?: string[]
+    urls?: string[]
+}
+
+function renderCall(name: string, args: ToolRenderArgs, theme: Theme): Text {
+    const display = args.queries ? args.queries.map((q) => `"${q}"`).join(' ') : (args.urls ?? []).join(' ')
+    return new Text(theme.fg('toolTitle', theme.bold(`${name} `)) + theme.fg('muted', display), 0, 0)
+}
+
+function renderResult(
+    result: { content: { type: string; text?: string }[]; details?: { backend: string } },
+    { expanded }: { expanded: boolean },
+    theme: Theme
+): Text {
+    const text = result.content
+        .filter((item) => item.type === 'text')
+        .map((item) => item.text ?? '')
+        .join('\n')
+    if (expanded) return new Text(text, 0, 0)
+    const lines = text.split('\n').length
+    const backend = result.details?.backend ?? '?'
+    return new Text(theme.fg('muted', `${lines} lines via ${backend}`), 0, 0)
+}
+
+export default function (pi: ExtensionAPI): void {
+    const key = process.env['EXA_API_KEY']
+    pi.registerMcpServer('exa', {
+        url: key ? `${EXA_MCP}?exaApiKey=${encodeURIComponent(key)}` : EXA_MCP,
+        description: 'Exa search and page extraction, the primary backend of web_search and web_fetch',
+        exposure: 'codemode',
+        timeout: 60
+    })
+
+    const searchResult = t.object({
+        title: t.string(),
+        url: t.string(),
+        snippet: t.string(),
+        published: t.optional(t.string()),
+        authors: t.optional(t.array(t.string())),
+        points: t.optional(t.number()),
+        comments: t.optional(t.number())
+    })
+
+    const searchParams = t.object({
+        queries: t.array(t.string({ minLength: 1 }), { minItems: 1, maxItems: MAX_QUERIES }),
+        numResults: t.optional(t.integer({ minimum: 1, maximum: 10 })),
+        source: t.optional(t.enum(SOURCES))
+    })
+    type SearchParams = Infer<typeof searchParams>
+
+    pi.registerTool({
+        name: 'web_search',
+        label: 'Web Search',
+        description: [
+            'Pass several queries at once to cover a topic from several angles in one call.',
+            'Describe the ideal page, not keywords: "blog post comparing X and Y" beats "X vs Y".',
+            'source arxiv returns preprints with abstracts, authors and dates, sorted by submission date.',
+            'source hn returns Hacker News threads scored by points and comment count.',
+            'Reddit and X have no keyless search: name the forum in a query.',
+            'Results are excerpts: web_fetch the ranked links at the end before answering.'
+        ].join(' '),
+        promptSnippet: 'search the web, arxiv or Hacker News for current information, papers and discussion',
+        exposure: 'direct',
+        namespace: {
+            name: 'web',
+            description: 'The open web: search it, and read a known URL as markdown.'
+        },
+
+        annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: false, openWorldHint: true },
+        parameters: searchParams,
+        renderCall: (args, theme) => renderCall('web_search', args, theme),
+        renderResult,
+        outputSchema: t.object({
+            groups: t.array(
+                t.object({
+                    query: t.string(),
+                    backend: t.string(),
+                    results: t.array(searchResult)
+                })
+            )
+        }),
+        async execute(_toolCallId, params: SearchParams, signal, _onUpdate, ctx) {
+            const hit = await searchMany(
+                params.queries.slice(0, MAX_QUERIES),
+                params.numResults ?? 5,
+                params.source ?? 'web',
+                signal,
+                exaViaMcp(ctx),
+                fetch
+            )
+            return {
+                content: [{ type: 'text' as const, text: hit.text || 'No results' }],
+                details: { backend: hit.backend },
+                structuredContent: { groups: hit.groups }
+            }
+        }
+    })
+
+    const fetchParams = t.object({
+        urls: t.array(t.string(), { minItems: 1 }),
+        maxCharacters: t.optional(t.integer({ minimum: 1 }))
+    })
+    type FetchParams = Infer<typeof fetchParams>
+
+    pi.registerTool({
+        name: 'web_fetch',
+        label: 'Web Fetch',
+        description:
+            'Read known pages as Markdown; batch URLs, and raise maxCharacters for a long page. ' +
+            'A page cut at the limit says so at the end, and raising the limit refetches it whole.',
+        promptSnippet: 'read known URLs as markdown',
+        exposure: 'direct',
+        namespace: {
+            name: 'web',
+            description: 'The open web: search it, and read a known URL as markdown.'
+        },
+
+        annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: true },
+        parameters: fetchParams,
+        renderCall: (args, theme) => renderCall('web_fetch', args, theme),
+        renderResult,
+        async execute(_toolCallId, params: FetchParams, signal, _onUpdate, ctx) {
+            const hit = await fetchPages(params.urls, params.maxCharacters ?? 8000, signal, exaViaMcp(ctx))
+            return {
+                content: [{ type: 'text' as const, text: hit.text || 'No results' }],
+                details: { backend: hit.backend }
+            }
+        }
+    })
+}

package/src/x-post.ts

@@ -0,0 +1,164 @@
+export const FIXUPX_STATUS_URL = 'https://api.fxtwitter.com/2/status/'
+
+export const X_ARTICLE_ID_ERROR =
+    'this x.com/i/article/ URL holds an article id, not a post id, and the article body is served under the post that links it: fetch the x.com/<user>/status/<post id> URL instead'
+
+function xBlockPrefix(type: string | undefined): string {
+    switch (type) {
+        case 'header-one':
+            return '# '
+        case 'header-two':
+            return '## '
+        case 'header-three':
+            return '### '
+        case 'header-four':
+            return '#### '
+        case 'header-five':
+            return '##### '
+        case 'header-six':
+            return '###### '
+        case 'blockquote':
+            return '> '
+        case 'unordered-list-item':
+            return '- '
+        case 'ordered-list-item':
+            return '1. '
+        case 'code':
+            return '    '
+        default:
+            return ''
+    }
+}
+
+interface XRange {
+    offset: number
+    length: number
+    key?: number
+    style?: string
+}
+
+interface XBlock {
+    type?: string
+    text?: string
+    inlineStyleRanges?: XRange[]
+    entityRanges?: XRange[]
+}
+
+interface XArticle {
+    title?: string
+    preview_text?: string
+    content?: {
+        blocks?: XBlock[]
+        entityMap?:
+            | Record<string, { data?: { type?: string; url?: string } }>
+            | { data?: { type?: string; url?: string } }[]
+    }
+    media_entities?: { url?: string }[]
+}
+
+interface XPost {
+    text?: string
+    created_at?: string
+    author?: { name?: string; screen_name?: string }
+    article?: XArticle
+    media?: { photos?: { url?: string }[] }
+    quote?: XPost
+}
+
+export function isXUrl(url: string): boolean {
+    let host: string
+    try {
+        host = new URL(url).hostname.toLowerCase()
+    } catch {
+        return false
+    }
+    return host === 'x.com' || host.endsWith('.x.com') || host === 'twitter.com' || host.endsWith('.twitter.com')
+}
+
+export function xPostId(url: string): string | undefined {
+    return /\/(?:i\/web\/)?status(?:es)?\/(\d+)/.exec(url)?.[1]
+}
+
+export function xArticleId(url: string): string | undefined {
+    return /\/i\/article\/(\d+)/.exec(url)?.[1]
+}
+
+function xEntity(article: XArticle, key: number): { data?: { type?: string; url?: string } } | undefined {
+    const map = article.content?.entityMap
+    if (!map) return undefined
+    return Array.isArray(map) ? map[key] : map[String(key)]
+}
+
+function xInline(text: string, block: XBlock, article: XArticle): string {
+    const wrapped = [
+        ...(block.inlineStyleRanges ?? []).map((range) => ({
+            ...range,
+            marker: range.style === 'BOLD' ? '**' : range.style === 'ITALIC' ? '*' : ''
+        })),
+        ...(block.entityRanges ?? []).map((range) => {
+            const url = xEntity(article, range.key ?? -1)?.data?.url
+            return {
+                ...range,
+                marker: url ? `[${text.slice(range.offset, range.offset + range.length)}](${url})]` : ''
+            }
+        })
+    ]
+        .filter((range) => range.marker)
+        .toSorted((a, b) => b.offset - a.offset)
+
+    let out = text
+    const taken = new Set<number>()
+    for (const range of wrapped) {
+        const end = range.offset + range.length
+        if (Array.from({ length: range.length }, (_, i) => taken.has(range.offset + i)).some(Boolean)) continue
+        for (let i = range.offset; i < end; i++) taken.add(i)
+        out = `${out.slice(0, range.offset)}${range.marker}${out.slice(end)}`
+    }
+    return out
+}
+
+function xAtomic(block: XBlock, article: XArticle): string {
+    const key = block.entityRanges?.[0]?.key
+    if (key === undefined) return ''
+    if (xEntity(article, key)?.data?.type?.toUpperCase().includes('DIVIDER')) return '---'
+    const url = article.media_entities?.[key]?.url
+    return url ? `![${article.title ?? 'image in article'}](${url})` : ''
+}
+
+function xArticleMarkdown(article: XArticle): string {
+    const parts: string[] = []
+    if (article.title) parts.push(`# ${article.title}`)
+    const blocks = article.content?.blocks
+    if (!blocks?.length) {
+        if (article.preview_text) parts.push(article.preview_text)
+        return parts.join('\n\n')
+    }
+    for (const block of blocks) {
+        if (block.type === 'atomic') {
+            const media = xAtomic(block, article)
+            if (media) parts.push(media)
+            continue
+        }
+        const text = (block.text ?? '').trim()
+        if (!text) continue
+        parts.push(`${xBlockPrefix(block.type)}${xInline(text, block, article)}`)
+    }
+    return parts.join('\n\n')
+}
+
+export function xPostMarkdown(payload: string): string {
+    const body = JSON.parse(payload) as { status?: XPost; message?: string }
+    const post = body.status
+    if (!post) throw new Error(`Fixupx has no post for this id: ${body.message ?? 'not found'}`)
+
+    const parts: string[] = []
+    const author = post.author
+    if (author?.screen_name) parts.push(`**${author.name ?? author.screen_name}** (@${author.screen_name})`)
+    if (post.text) parts.push(post.text)
+    for (const photo of post.media?.photos ?? []) {
+        if (photo.url) parts.push(`![post image](${photo.url})`)
+    }
+    if (post.quote?.text) parts.push(`> ${post.quote.author?.screen_name ?? 'quoted post'}: ${post.quote.text}`)
+    if (post.article) parts.push(xArticleMarkdown(post.article))
+    return parts.join('\n\n')
+}