@ashx-j/lunr 0.2.25 → 0.2.27

package/dist/builtin-extensions/pi-web-access/index.js

@@ -9,10 +9,10 @@ var __rewriteRelativeImportExtension = (this && this.__rewriteRelativeImportExte
 import { Box, Text, truncateToWidth } from "@ashx-j/lunr-tui";
 import { Type } from "typebox";
 import { StringEnum, complete } from "@ashx-j/lunr-ai/compat";
+import { contentPage } from "./content-page.js";
 import { normalizeFetchContentParams } from "./fetch-params.js";
 import { getWebSearchConfigDir, getWebSearchConfigPath } from "./utils.js";
-import { clearResults, deleteResult, generateId, getAllResults, getResult, restoreFromSession, storeResult, } from "./storage.js";
-import { activityMonitor } from "./activity.js";
+import { createResultStore, generateId } from "./storage.js";
 import { randomUUID } from "node:crypto";
 import { execFileSync } from "node:child_process";
 import { createRequire } from "node:module";
@@ -21,7 +21,7 @@ import { existsSync, readFileSync, writeFileSync, mkdirSync } from "node:fs";
 import { join } from "node:path";
 import { isBrowserCookieAccessAllowed } from "./gemini-web-config.js";
 import { loadBrave, loadCuratorServer, loadExa, loadExtract, loadGeminiApi, loadGeminiSearch, loadGeminiWeb, loadOpenAISearch, loadParallel, loadPerplexity, loadSummaryReview, loadTavily, } from "./lazy.js";
-import { runSessionCleanups } from "./session-cleanup.js";
+import { createWebSession, getWebSession, runWithWebSession, runSessionCleanups } from "./session-cleanup.js";
 import { buildSearchErrorPlan } from "./render-search-error.js";
 import { collectFetchUrls, collectSearchQueries, formatFetchCallTitle, formatFetchChrome, formatFetchDetail, formatSearchCallTitle, formatSearchChrome, formatSearchDetail, } from "./render-search-chrome.js";
 import { GroupedCallText, toolGroupTree, toolStatusDotFromContext } from "../../core/tools/render-utils.js";
@@ -364,6 +364,11 @@ function formatEntryLine(entry, theme) {
     return `${typeStr.padEnd(4)} ${target.padEnd(32)} ${statusStr.padStart(5)} ${duration.padStart(5)} ${indicator}`;
 }
 export default function (pi) {
+    const { clearResults, deleteResult, getAllResults, getResult, restoreFromSession, storeResult } = createResultStore();
+    let webSession = createWebSession();
+    function registerWebTool(tool) {
+        pi.registerTool({ ...tool, execute: (...args) => runWithWebSession(webSession, () => tool.execute(...args)) });
+    }
     const initConfig = loadConfigForExtensionInit();
     const curateKey = initConfig.shortcuts?.curate || DEFAULT_SHORTCUTS.curate;
     const activityKey = initConfig.shortcuts?.activity || DEFAULT_SHORTCUTS.activity;
@@ -423,7 +428,7 @@ export default function (pi) {
     }
     function updateWidget(ctx) {
         const theme = ctx.ui.theme;
-        const entries = activityMonitor.getEntries();
+        const entries = webSession.activity.getEntries();
         const lines = [];
         lines.push(theme.fg("accent", "─── Web Search Activity " + "─".repeat(36)));
         if (entries.length === 0) {
@@ -434,7 +439,7 @@ export default function (pi) {
                 lines.push("  " + formatEntryLine(entry, theme));
         }
         lines.push(theme.fg("accent", "─".repeat(60)));
-        const rateInfo = activityMonitor.getRateLimitInfo();
+        const rateInfo = webSession.activity.getRateLimitInfo();
         const resetMs = rateInfo.oldestTimestamp
             ? Math.max(0, rateInfo.oldestTimestamp + rateInfo.windowMs - Date.now())
             : 0;
@@ -443,19 +448,21 @@ export default function (pi) {
             (resetMs > 0 ? theme.fg("dim", ` (resets in ${resetSec}s)`) : ""));
         ctx.ui.setWidget("web-activity", new Text(lines.join("\n"), 0, 0));
     }
-    function handleSessionChange(ctx) {
+    async function handleSessionChange(ctx) {
         abortPendingFetches();
         closeCurator();
-        runSessionCleanups();
+        const cleanup = runSessionCleanups(webSession);
+        webSession = createWebSession();
         sessionActive = true;
         restoreFromSession(ctx);
         widgetUnsubscribe?.();
         widgetUnsubscribe = null;
-        activityMonitor.clear();
+        webSession.activity.clear();
         if (widgetVisible) {
-            widgetUnsubscribe = activityMonitor.onUpdate(() => updateWidget(ctx));
+            widgetUnsubscribe = webSession.activity.onUpdate(() => updateWidget(ctx));
             updateWidget(ctx);
         }
+        await cleanup;
     }
     /** Populated on first summary/curator use so sync curator submit/cancel can build fallbacks. */
     let warmSummaryReview = null;
@@ -464,11 +471,11 @@ export default function (pi) {
         if (generation !== sessionGeneration)
             throw new Error("Web request cancelled: session changed");
     }
-    function startSession(ctx) {
+    async function startSession(ctx) {
         sessionGeneration++;
         sessionAbort.abort();
         sessionAbort = new AbortController();
-        handleSessionChange(ctx);
+        await handleSessionChange(ctx);
     }
     async function ensureSummaryReview() {
         if (!warmSummaryReview)
@@ -476,8 +483,9 @@ export default function (pi) {
         return warmSummaryReview;
     }
     async function runSearch(query, options) {
+        const owner = getWebSession() ?? webSession;
         const { search } = await loadGeminiSearch();
-        return search(query, options);
+        return runWithWebSession(owner, () => search(query, options));
     }
     // lunr: expose curator workflow state to core (/settings "Extensions" submenu)
     // so the /settings toggle writes through the same web-search.json saveConfig
@@ -496,7 +504,7 @@ export default function (pi) {
         const fetchId = generateId();
         const controller = new AbortController();
         pendingFetches.set(fetchId, controller);
-        void (async () => {
+        void runWithWebSession(webSession, async () => {
             try {
                 const { fetchAllContent } = await loadExtract();
                 if (!sessionActive || !pendingFetches.has(fetchId))
@@ -535,7 +543,7 @@ export default function (pi) {
             finally {
                 pendingFetches.delete(fetchId);
             }
-        })();
+        });
         return fetchId;
     }
     function storeAndPublishSearch(results) {
@@ -1114,7 +1122,7 @@ export default function (pi) {
         handler: async (ctx) => {
             widgetVisible = !widgetVisible;
             if (widgetVisible) {
-                widgetUnsubscribe = activityMonitor.onUpdate(() => updateWidget(ctx));
+                widgetUnsubscribe = webSession.activity.onUpdate(() => updateWidget(ctx));
                 updateWidget(ctx);
             }
             else {
@@ -1126,22 +1134,23 @@ export default function (pi) {
     });
     pi.on("session_start", async (_event, ctx) => startSession(ctx));
     pi.on("session_tree", async (_event, ctx) => startSession(ctx));
-    pi.on("session_shutdown", () => {
+    pi.on("session_shutdown", async () => {
         sessionGeneration++;
         sessionAbort.abort();
         sessionActive = false;
         abortPendingFetches();
         closeCurator();
-        runSessionCleanups();
+        const cleanup = runSessionCleanups(webSession);
         clearResults();
         // Unsubscribe before clear() to avoid callback with stale ctx
         widgetUnsubscribe?.();
         widgetUnsubscribe = null;
-        activityMonitor.clear();
+        webSession.activity.clear();
         widgetVisible = false;
+        await cleanup;
     });
     if (initConfig.webSearch?.enabled !== false)
-        pi.registerTool({
+        registerWebTool({
             name: "web_search",
             label: "Web Search",
             description: `Search the web using OpenAI, Brave, Parallel, Tavily, Exa, Perplexity, or Gemini. Use web_search for discovery and fetch_content for reading URLs. When the browser tool is available, use it for JavaScript-rendered inspection or explicit website interaction; never automatically fall back to browser. Returns an AI-synthesized answer with source citations. OpenAI web_search uses a Codex subscription or OpenAI API key. For comprehensive research, prefer queries (plural) with 2-4 varied angles over a single query — each query gets its own synthesized answer, so varying phrasing and scope gives much broader coverage. When includeContent is true, full page content is fetched in the background. Searches auto-open the interactive browser curator and stream results live; set workflow to "none" to skip curation or "auto-summary" for a model-generated summary without the browser curator. Provider auto-selects: OpenAI when suitable and available, then Exa, Brave, Parallel, Tavily, Perplexity, Gemini API, then Gemini Web.`,
@@ -1509,7 +1518,7 @@ export default function (pi) {
                 return new Text(chrome.result.map((line) => theme.fg(statusColor, line)).join("\n"), 0, 0);
             },
         });
-    pi.registerTool({
+    registerWebTool({
         name: "fetch_content",
         label: "Fetch Content",
         description: "Fetch URL(s) and extract readable content as markdown. Prefer fetch_content for reading URLs and web_search for discovery. The browser tool handles JavaScript-rendered inspection and explicit website interactions when enabled; it is not an automatic fallback, and interaction tasks do not require a preliminary fetch. Supports YouTube video transcripts (with thumbnail), GitHub repository contents, and local video files (with frame thumbnail). Video frames can be extracted via timestamp/range or sampled across the entire video with frames alone. Falls back to Gemini for pages that block bots or fail Readability extraction. For YouTube and video files: ALWAYS pass the user's specific question via the prompt parameter — this directs the AI to focus on that aspect of the video, producing much better results than a generic extraction. Content is always stored and can be retrieved with get_search_content.",
@@ -1587,7 +1596,7 @@ export default function (pi) {
                     : result.content;
                 if (truncated) {
                     output += `\n\n---\nShowing ${MAX_INLINE_CONTENT} of ${fullLength} chars. ` +
-                        `Use get_search_content({ responseId: "${responseId}", urlIndex: 0 }) for full content.`;
+                        `Use get_search_content({ responseId: "${responseId}", urlIndex: 0 }) for stored content. Follow nextOffset if truncated.`;
                 }
                 const content = [];
                 if (result.frames?.length) {
@@ -1630,7 +1639,7 @@ export default function (pi) {
                     output += `- ${title || url} (${content.length} chars)\n`;
                 }
             }
-            output += `\n---\nUse get_search_content({ responseId: "${responseId}", urlIndex: 0 }) to retrieve full content.`;
+            output += `\n---\nUse get_search_content({ responseId: "${responseId}", urlIndex: 0 }) to retrieve stored content. Follow nextOffset if truncated.`;
             return {
                 content: [{ type: "text", text: output }],
                 details: { urls: urlList, urlCount: urlList.length, successful, totalChars, responseId },
@@ -1696,12 +1705,13 @@ export default function (pi) {
             return new Text(chrome.result.map((line) => theme.fg(statusColor, line)).join("\n"), 0, 0);
         },
     });
-    pi.registerTool({
+    registerWebTool({
         name: "get_search_content",
         label: "Get Search Content",
-        description: "Retrieve full content from a previous web_search or fetch_content call.",
-        promptSnippet: "Use after web_search/fetch_content when full stored content is needed via responseId plus query/url selectors.",
+        description: "Retrieve stored web_search or fetch_content text in pages capped at 50 KiB and 2,000 lines. Reuse responseId and selectors with nextOffset to continue. Available-selector lists and errors are paginated too.",
+        promptSnippet: "Retrieve stored text with responseId and query/url selectors. If truncated, repeat with offset=nextOffset and the same selectors. Each page is at most 50 KiB/2,000 lines.",
         parameters: Type.Object({
+            offset: Type.Optional(Type.Integer({ minimum: 0, description: "UTF-8 byte offset, starting at 0. Use nextOffset from the previous page with unchanged selectors." })),
             responseId: Type.String({ description: "The responseId from web_search or fetch_content" }),
             query: Type.Optional(Type.String({ description: "Get content for this query (web_search)" })),
             queryIndex: Type.Optional(Type.Number({ description: "Get content for query at index" })),
@@ -1709,12 +1719,10 @@ export default function (pi) {
             urlIndex: Type.Optional(Type.Number({ description: "Get content for URL at index" })),
         }),
         async execute(_toolCallId, params) {
+            const reply = (text, details) => contentPage(text, params.offset, details);
             const data = getResult(params.responseId);
             if (!data) {
-                return {
-                    content: [{ type: "text", text: `Error: No stored results for "${params.responseId}"` }],
-                    details: { error: "Not found", responseId: params.responseId },
-                };
+                return reply(`Error: No stored results for "${params.responseId}"`, { error: "Not found", responseId: params.responseId });
             }
             if (data.type === "search" && data.queries) {
                 let queryData;
@@ -1722,38 +1730,23 @@ export default function (pi) {
                     queryData = data.queries.find((q) => q.query === params.query);
                     if (!queryData) {
                         const available = data.queries.map((q) => `"${q.query}"`).join(", ");
-                        return {
-                            content: [{ type: "text", text: `Query "${params.query}" not found. Available: ${available}` }],
-                            details: { error: "Query not found" },
-                        };
+                        return reply(`Query "${params.query}" not found. Available: ${available}`, { error: "Query not found" });
                     }
                 }
                 else if (params.queryIndex !== undefined) {
                     queryData = data.queries[params.queryIndex];
                     if (!queryData) {
-                        return {
-                            content: [{ type: "text", text: `Index ${params.queryIndex} out of range (0-${data.queries.length - 1})` }],
-                            details: { error: "Index out of range" },
-                        };
+                        return reply(`Index ${params.queryIndex} out of range (0-${data.queries.length - 1})`, { error: "Index out of range" });
                     }
                 }
                 else {
                     const available = data.queries.map((q, i) => `${i}: "${q.query}"`).join(", ");
-                    return {
-                        content: [{ type: "text", text: `Specify query or queryIndex. Available: ${available}` }],
-                        details: { error: "No query specified" },
-                    };
+                    return reply(`Specify query or queryIndex. Available: ${available}`, { error: "No query specified" });
                 }
                 if (queryData.error) {
-                    return {
-                        content: [{ type: "text", text: `Error for "${queryData.query}": ${queryData.error}` }],
-                        details: { error: queryData.error, query: queryData.query },
-                    };
+                    return reply(`Error for "${queryData.query}": ${queryData.error}`, { error: queryData.error, query: queryData.query });
                 }
-                return {
-                    content: [{ type: "text", text: formatFullResults(queryData) }],
-                    details: { query: queryData.query, resultCount: queryData.results.length },
-                };
+                return reply(formatFullResults(queryData), { query: queryData.query, resultCount: queryData.results.length });
             }
             if (data.type === "fetch" && data.urls) {
                 let urlData;
@@ -1761,43 +1754,25 @@ export default function (pi) {
                     urlData = data.urls.find((u) => u.url === params.url);
                     if (!urlData) {
                         const available = data.urls.map((u) => u.url).join("\n  ");
-                        return {
-                            content: [{ type: "text", text: `URL not found. Available:\n  ${available}` }],
-                            details: { error: "URL not found" },
-                        };
+                        return reply(`URL not found. Available:\n  ${available}`, { error: "URL not found" });
                     }
                 }
                 else if (params.urlIndex !== undefined) {
                     urlData = data.urls[params.urlIndex];
                     if (!urlData) {
-                        return {
-                            content: [{ type: "text", text: `Index ${params.urlIndex} out of range (0-${data.urls.length - 1})` }],
-                            details: { error: "Index out of range" },
-                        };
+                        return reply(`Index ${params.urlIndex} out of range (0-${data.urls.length - 1})`, { error: "Index out of range" });
                     }
                 }
                 else {
                     const available = data.urls.map((u, i) => `${i}: ${u.url}`).join("\n  ");
-                    return {
-                        content: [{ type: "text", text: `Specify url or urlIndex. Available:\n  ${available}` }],
-                        details: { error: "No URL specified" },
-                    };
+                    return reply(`Specify url or urlIndex. Available:\n  ${available}`, { error: "No URL specified" });
                 }
                 if (urlData.error) {
-                    return {
-                        content: [{ type: "text", text: `Error for ${urlData.url}: ${urlData.error}` }],
-                        details: { error: urlData.error, url: urlData.url },
-                    };
+                    return reply(`Error for ${urlData.url}: ${urlData.error}`, { error: urlData.error, url: urlData.url });
                 }
-                return {
-                    content: [{ type: "text", text: `# ${urlData.title}\n\n${urlData.content}` }],
-                    details: { url: urlData.url, title: urlData.title, contentLength: urlData.content.length },
-                };
+                return reply(`# ${urlData.title}\n\n${urlData.content}`, { url: urlData.url, title: urlData.title, contentLength: urlData.content.length });
             }
-            return {
-                content: [{ type: "text", text: "Invalid stored data format" }],
-                details: { error: "Invalid data" },
-            };
+            return reply("Invalid stored data format", { error: "Invalid data" });
         },
         renderCall(args, theme) {
             const { responseId, query, queryIndex, url, urlIndex } = args;