@herbertgao/pi-extensions 2026.8.1 → 2026.8.3

package/node_modules/pi-web-access/curator-page.ts

@@ -8,7 +8,7 @@ function safeInlineJSON(data: unknown): string {
 }
 
 function buildProviderButtons(
-	available: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; serpdive: boolean; kagi: boolean; ollama: boolean; searxng: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean },
+	available: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; jina: boolean; serpdive: boolean; kagi: boolean; bocha: boolean; ollama: boolean; searxng: boolean; duckduckgo: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean },
 	selected: string,
 	hasInitialQueries: boolean,
 ): string {
@@ -23,10 +23,13 @@ function buildProviderButtons(
 		{ value: "searchinfinity", label: "Searchinfinity", available: available.searchinfinity },
 		{ value: "querit", label: "Querit", available: available.querit },
 		{ value: "tavily", label: "Tavily", available: available.tavily },
+		{ value: "jina", label: "Jina", available: available.jina },
 		{ value: "serpdive", label: "SERPdive", available: available.serpdive },
 		{ value: "kagi", label: "Kagi", available: available.kagi },
+		{ value: "bocha", label: "Bocha", available: available.bocha },
 		{ value: "ollama", label: "Ollama", available: available.ollama },
 		{ value: "searxng", label: "SearXNG", available: available.searxng },
+		{ value: "duckduckgo", label: "DuckDuckGo", available: available.duckduckgo },
 		{ value: "perplexity", label: "Perplexity", available: available.perplexity },
 		{ value: "gemini", label: "Gemini", available: available.gemini },
 		{ value: "anysearch", label: "AnySearch", available: available.anysearch },
@@ -51,7 +54,7 @@ export function generateCuratorPage(
 	queries: string[],
 	sessionToken: string,
 	timeout: number,
-	availableProviders: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; serpdive: boolean; kagi: boolean; ollama: boolean; searxng: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean },
+	availableProviders: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; jina: boolean; serpdive: boolean; kagi: boolean; bocha: boolean; ollama: boolean; searxng: boolean; duckduckgo: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean },
 	defaultProvider: string,
 	searchProvider: string,
 	summaryModels: Array<{ value: string; label: string }>,
@@ -714,6 +717,11 @@ main {
   background: rgba(166, 227, 161, 0.14);
   border-color: rgba(166, 227, 161, 0.3);
 }
+.provider-tag.provider-jina {
+  color: #f9e2af;
+  background: rgba(249, 226, 175, 0.14);
+  border-color: rgba(249, 226, 175, 0.3);
+}
 .provider-tag.provider-serpdive {
   color: #94e2d5;
   background: rgba(148, 226, 213, 0.14);
@@ -1447,7 +1455,7 @@ const SCRIPT = `(function() {
   var token = DATA.sessionToken;
   var timeoutSec = DATA.timeout;
   var queries = Array.isArray(DATA.queries) ? DATA.queries : [];
-  var providers = ["all", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "serpdive", "kagi", "ollama", "searxng", "perplexity", "gemini", "anysearch", "xai", "brightdata", "serpbase"];
+  var providers = ["all", "openai", "exa", "brave", "parallel", "tinyfish", "search1api", "searchinfinity", "querit", "tavily", "jina", "serpdive", "kagi", "bocha", "ollama", "searxng", "duckduckgo", "perplexity", "gemini", "anysearch", "xai", "brightdata", "serpbase"];
   var availProviders = DATA.availableProviders && typeof DATA.availableProviders === "object" ? DATA.availableProviders : {};
   var workflow = "summary-review";
   var initialDefaultProvider = typeof DATA.defaultProvider === "string" ? DATA.defaultProvider : "exa";
@@ -1475,6 +1483,9 @@ const SCRIPT = `(function() {
   var lastInteraction = Date.now();
   var completedCount = 0;
   var es = null;
+  var lastLiveEventAt = Date.now();
+  var stateSyncInFlight = false;
+  var liveUpdateWarning = false;
 
   var allQueries = queries.map(function(query, slotId) { return { slotId: slotId, query: query }; });
   var nextSlotId = queries.length;
@@ -1656,10 +1667,13 @@ const SCRIPT = `(function() {
     if (provider === "searchinfinity") return "Searchinfinity";
     if (provider === "querit") return "Querit";
     if (provider === "tavily") return "Tavily";
+    if (provider === "jina") return "Jina";
     if (provider === "serpdive") return "SERPdive";
     if (provider === "kagi") return "Kagi";
+    if (provider === "bocha") return "Bocha";
     if (provider === "ollama") return "Ollama";
     if (provider === "searxng") return "SearXNG";
+    if (provider === "duckduckgo") return "DuckDuckGo";
     if (provider === "perplexity") return "Perplexity";
     if (provider === "exa") return "Exa";
     if (provider === "gemini") return "Gemini";
@@ -1942,17 +1956,30 @@ const SCRIPT = `(function() {
   }
 
   function clearError() {
+    liveUpdateWarning = false;
     if (!errorBanner) return;
     errorBanner.hidden = true;
     errorBanner.textContent = "";
   }
 
   function setError(text) {
+    liveUpdateWarning = false;
+    if (!errorBanner) return;
+    errorBanner.textContent = text;
+    errorBanner.hidden = false;
+  }
+
+  function setLiveUpdateWarning(text) {
+    liveUpdateWarning = true;
     if (!errorBanner) return;
     errorBanner.textContent = text;
     errorBanner.hidden = false;
   }
 
+  function clearLiveUpdateWarning() {
+    if (liveUpdateWarning) clearError();
+  }
+
   function updateSummaryGeneratingIndicator() {
     if (!summaryGeneratingCopy) return;
 
@@ -2749,9 +2776,10 @@ const SCRIPT = `(function() {
     }
   }
 
-  es.addEventListener("result", function(e) {
-    var data = parseSseEventData("result", e);
+  function applyResultEvent(data) {
     if (!data) return;
+    lastLiveEventAt = Date.now();
+    clearLiveUpdateWarning();
 
     var queryText = data.query || queries[data.queryIndex] || "";
     var slotId = typeof data.slotIndex === "number" ? data.slotIndex : queryIndexToSlot.get(data.queryIndex);
@@ -2761,13 +2789,16 @@ const SCRIPT = `(function() {
       card = createSearchingCard(queryText, data.provider);
       card.dataset.qi = data.queryIndex;
       insertResultCard(card, slotId, null);
+    } else if (card.dataset.completed === "true") {
+      return;
     }
     applyResponseToCard(card, data, queryText, data.provider, slotId);
-  });
+  }
 
-  es.addEventListener("search-error", function(e) {
-    var data = parseSseEventData("search-error", e);
+  function applySearchErrorEvent(data) {
     if (!data) return;
+    lastLiveEventAt = Date.now();
+    clearLiveUpdateWarning();
 
     var queryText = data.query || queries[data.queryIndex] || "";
     var slotId = typeof data.slotIndex === "number" ? data.slotIndex : queryIndexToSlot.get(data.queryIndex);
@@ -2777,6 +2808,8 @@ const SCRIPT = `(function() {
       card = createSearchingCard(queryText, data.provider);
       card.dataset.qi = data.queryIndex;
       insertResultCard(card, slotId, null);
+    } else if (card.dataset.completed === "true") {
+      return;
     }
     applyResponseToCard(card, {
       queryIndex: data.queryIndex,
@@ -2785,9 +2818,12 @@ const SCRIPT = `(function() {
       error: data.error || "Search failed",
       provider: data.provider,
     }, queryText, data.provider, slotId);
-  });
+  }
 
-  es.addEventListener("done", function() {
+  function applyDoneEvent() {
+    var wasSearchesDone = searchesDone;
+    lastLiveEventAt = Date.now();
+    clearLiveUpdateWarning();
     searchesDone = true;
     initialStreamDone = true;
     if (completedCount > 0) {
@@ -2797,13 +2833,78 @@ const SCRIPT = `(function() {
     recomputeProviderStates();
     updateStageUI();
     maybeAutoGenerateSummary();
-    resetTimer();
+    if (!wasSearchesDone) resetTimer();
+  }
+
+  function applyStoredLiveEvent(item) {
+    if (!item || typeof item !== "object") return;
+    if (item.event === "result") applyResultEvent(item.data);
+    if (item.event === "search-error") applySearchErrorEvent(item.data);
+  }
+
+  function syncStateFromServer(showWarning) {
+    if (stateSyncInFlight || submitted || timerExpired || searchesDone) return;
+    stateSyncInFlight = true;
+    fetch("/state?session=" + encodeURIComponent(token), { cache: "no-store" })
+      .then(function(res) {
+        return res.text().then(function(raw) {
+          var data = raw ? JSON.parse(raw) : null;
+          if (!res.ok || !data || data.ok === false) {
+            throw new Error(extractServerError(data) || ("HTTP " + res.status));
+          }
+          return data;
+        });
+      })
+      .then(function(data) {
+        if (Array.isArray(data.events)) {
+          data.events.forEach(applyStoredLiveEvent);
+        }
+        if (data.done) applyDoneEvent();
+        if (showWarning && !searchesDone) {
+          setLiveUpdateWarning("Live search updates disconnected. Reconnecting and polling for results.");
+        }
+      })
+      .catch(function(err) {
+        if (!showWarning || submitted || timerExpired) return;
+        var message = err instanceof Error ? err.message : String(err);
+        setLiveUpdateWarning("Live search updates disconnected. Retrying: " + (message || "unknown error"));
+      })
+      .finally(function() {
+        stateSyncInFlight = false;
+      });
+  }
+
+  es.addEventListener("open", function() {
+    lastLiveEventAt = Date.now();
+    clearLiveUpdateWarning();
+    syncStateFromServer(false);
+  });
+
+  es.addEventListener("result", function(e) {
+    var data = parseSseEventData("result", e);
+    applyResultEvent(data);
+  });
+
+  es.addEventListener("search-error", function(e) {
+    var data = parseSseEventData("search-error", e);
+    applySearchErrorEvent(data);
+  });
+
+  es.addEventListener("done", function() {
+    applyDoneEvent();
   });
 
   es.onerror = function() {
-    // EventSource reconnects automatically.
+    if (submitted || timerExpired || searchesDone) return;
+    syncStateFromServer(true);
   };
 
+  setInterval(function() {
+    if (submitted || timerExpired || searchesDone || initialStreamDone) return;
+    if (Date.now() - lastLiveEventAt <= 5000) return;
+    syncStateFromServer(false);
+  }, 5000);
+
   function setupCardInteraction(card) {
     var header = card.querySelector(".result-card-header");
     var body = card.querySelector(".result-card-body");

package/node_modules/pi-web-access/curator-server.ts

@@ -8,12 +8,17 @@ const WATCHDOG_INTERVAL_MS = 1000;
 const MAX_BODY_SIZE = 64 * 1024;
 
 type ServerState = "SEARCHING" | "RESULT_SELECTION" | "COMPLETED";
+type CuratorResultEventData = IndexedCuratorSearchEntry & { slotIndex?: number };
+type CuratorSearchErrorEventData = { queryIndex: number; query: string; error: string; provider?: string; slotIndex?: number };
+type CuratorStoredEvent =
+	| { event: "result"; data: CuratorResultEventData }
+	| { event: "search-error"; data: CuratorSearchErrorEventData };
 
 export interface CuratorServerOptions {
 	queries: string[];
 	sessionToken: string;
 	timeout: number;
-	availableProviders: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; serpdive: boolean; kagi: boolean; ollama: boolean; searxng: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean };
+	availableProviders: { all: boolean; openai: boolean; brave: boolean; parallel: boolean; tinyfish: boolean; search1api: boolean; searchinfinity: boolean; querit: boolean; tavily: boolean; jina: boolean; serpdive: boolean; kagi: boolean; bocha: boolean; ollama: boolean; searxng: boolean; duckduckgo: boolean; perplexity: boolean; exa: boolean; gemini: boolean; anysearch: boolean; xai: boolean; brightdata: boolean; serpbase: boolean };
 	defaultProvider: string;
 	searchProvider: string;
 	summaryModels: Array<{ value: string; label: string }>;
@@ -205,7 +210,8 @@ export function startCuratorServer(
 	let watchdog: NodeJS.Timeout | null = null;
 	let state: ServerState = "SEARCHING";
 	let sseResponse: ServerResponse | null = null;
-	const sseBuffer: string[] = [];
+	const streamedEventsByResultIndex = new Map<number, CuratorStoredEvent>();
+	let searchStreamDone = queries.length === 0;
 	let nextQueryIndex = queries.length;
 	let summarizeAbortController: AbortController | null = null;
 	let summarizeRequestSeq = 0;
@@ -274,10 +280,13 @@ export function startCuratorServer(
 		if (provider === "searchinfinity") return availableProviders.searchinfinity;
 		if (provider === "querit") return availableProviders.querit;
 		if (provider === "tavily") return availableProviders.tavily;
+		if (provider === "jina") return availableProviders.jina;
 		if (provider === "serpdive") return availableProviders.serpdive;
 		if (provider === "kagi") return availableProviders.kagi;
+		if (provider === "bocha") return availableProviders.bocha;
 		if (provider === "ollama") return availableProviders.ollama;
 		if (provider === "searxng") return availableProviders.searxng;
+		if (provider === "duckduckgo") return availableProviders.duckduckgo;
 		if (provider === "perplexity") return availableProviders.perplexity;
 		if (provider === "exa") return availableProviders.exa;
 		if (provider === "gemini") return availableProviders.gemini;
@@ -288,13 +297,35 @@ export function startCuratorServer(
 		return false;
 	}
 
-	function sendSSE(event: string, data: unknown): void {
+	function writeSSE(res: ServerResponse, event: string, data: unknown): boolean {
 		const payload = `event: ${event}\ndata: ${JSON.stringify(data)}\n\n`;
+		try {
+			res.write(payload);
+			return true;
+		} catch {
+			return false;
+		}
+	}
+
+	function sendSSE(event: string, data: unknown): void {
 		const res = sseResponse;
-		if (res && !res.writableEnded && res.socket && !res.socket.destroyed) {
-			try { res.write(payload); return; } catch {}
+		if (res && !res.writableEnded && res.socket && !res.socket.destroyed && writeSSE(res, event, data)) return;
+		if (sseResponse === res) sseResponse = null;
+	}
+
+	function retainStreamedEvent(event: CuratorStoredEvent): void {
+		streamedEventsByResultIndex.set(event.data.queryIndex, event);
+	}
+
+	function getStreamedEvents(): CuratorStoredEvent[] {
+		return [...streamedEventsByResultIndex.values()];
+	}
+
+	function replaySSE(res: ServerResponse): void {
+		for (const item of getStreamedEvents()) {
+			if (!writeSSE(res, item.event, item.data)) return;
 		}
-		sseBuffer.push(payload);
+		if (searchStreamDone) writeSSE(res, "done", {});
 	}
 
 	const pageHtml = generateCuratorPage(
@@ -352,18 +383,7 @@ export function startCuratorServer(
 				res.flushHeaders();
 				if (res.socket) res.socket.setNoDelay(true);
 				sseResponse = res;
-				if (sseBuffer.length > 0) {
-					const pending = sseBuffer.splice(0, sseBuffer.length);
-					for (let i = 0; i < pending.length; i++) {
-						const msg = pending[i];
-						try {
-							res.write(msg);
-						} catch {
-							sseBuffer.unshift(...pending.slice(i));
-							break;
-						}
-					}
-				}
+				replaySSE(res);
 				if (sseKeepalive) clearInterval(sseKeepalive);
 				sseKeepalive = setInterval(() => {
 					if (sseResponse) {
@@ -376,6 +396,17 @@ export function startCuratorServer(
 				return;
 			}
 
+			if (method === "GET" && url.pathname === "/state") {
+				const token = url.searchParams.get("session");
+				if (token !== sessionToken) {
+					sendJson(res, 403, { ok: false, error: "Invalid session" });
+					return;
+				}
+				touchHeartbeat();
+				sendJson(res, 200, { ok: true, events: getStreamedEvents(), done: searchStreamDone });
+				return;
+			}
+
 			if (method === "POST" && url.pathname === "/heartbeat") {
 				const body = await parseBodyOrSend(req, res);
 				if (!body) return;
@@ -684,15 +715,20 @@ export function startCuratorServer(
 				pushResult: (queryIndex, data) => {
 					if (completed) return;
 					nextQueryIndex = Math.max(nextQueryIndex, queryIndex + 1);
-					sendSSE("result", { queryIndex, query: data.query ?? queries[queryIndex] ?? "", ...data });
+					const eventData: CuratorResultEventData = { ...data, queryIndex, query: data.query ?? queries[queryIndex] ?? "" };
+					retainStreamedEvent({ event: "result", data: eventData });
+					sendSSE("result", eventData);
 				},
 				pushError: (queryIndex, error, provider, meta) => {
 					if (completed) return;
 					nextQueryIndex = Math.max(nextQueryIndex, queryIndex + 1);
-					sendSSE("search-error", { queryIndex, query: meta?.query ?? queries[queryIndex] ?? "", error, provider, slotIndex: meta?.slotIndex });
+					const eventData: CuratorSearchErrorEventData = { queryIndex, query: meta?.query ?? queries[queryIndex] ?? "", error, provider, slotIndex: meta?.slotIndex };
+					retainStreamedEvent({ event: "search-error", data: eventData });
+					sendSSE("search-error", eventData);
 				},
 				searchesDone: () => {
 					if (completed) return;
+					searchStreamDone = true;
 					sendSSE("done", {});
 					state = "RESULT_SELECTION";
 					stateChangedAt = Date.now();