@bastani/atomic 0.7.17-1 → 0.8.0-0

package/dist/builtin/web-access/rsc-extract.ts

@@ -0,0 +1,338 @@
+/**
+ * RSC Content Extractor
+ * 
+ * Extracts readable content from Next.js React Server Components (RSC) flight payloads.
+ * RSC pages embed content as JSON in <script>self.__next_f.push([...])</script> tags.
+ */
+
+export interface RSCExtractResult {
+  title: string;
+  content: string;
+}
+
+export function extractRSCContent(html: string): RSCExtractResult | null {
+  if (!html.includes("self.__next_f.push")) {
+    return null;
+  }
+
+  // Parse all RSC chunks into a map
+  const chunkMap = new Map<string, string>();
+  const scriptRegex = /<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;
+
+  for (const match of html.matchAll(scriptRegex)) {
+    let content: string;
+    try {
+      content = JSON.parse('"' + match[1] + '"');
+    } catch {
+      continue;
+    }
+
+    // Parse each line as "id:payload"
+    // Lines are separated by \n, each line is one chunk
+    // Chunk IDs are hex strings, typically 1-4 chars (supports up to 65535 chunks)
+    for (const line of content.split("\n")) {
+      if (!line.trim()) continue;
+      
+      const colonIdx = line.indexOf(":");
+      if (colonIdx <= 0 || colonIdx > 4) continue;
+      
+      const id = line.slice(0, colonIdx);
+      if (!/^[0-9a-f]+$/i.test(id)) continue;
+      
+      const payload = line.slice(colonIdx + 1);
+      if (!payload) continue;
+      
+      const existing = chunkMap.get(id);
+      if (!existing || payload.length > existing.length) {
+        chunkMap.set(id, payload);
+      }
+    }
+  }
+
+  if (chunkMap.size === 0) return null;
+
+  // Extract title
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/);
+  const title = titleMatch?.[1]?.split("|")[0]?.trim() || "";
+
+  // Parse and cache parsed chunks
+  const parsedCache = new Map<string, unknown>();
+  
+  function getParsedChunk(id: string): unknown | null {
+    if (parsedCache.has(id)) return parsedCache.get(id);
+    
+    const chunk = chunkMap.get(id);
+    if (!chunk || !chunk.startsWith("[")) {
+      parsedCache.set(id, null);
+      return null;
+    }
+    
+    try {
+      const parsed = JSON.parse(chunk);
+      parsedCache.set(id, parsed);
+      return parsed;
+    } catch {
+      parsedCache.set(id, null);
+      return null;
+    }
+  }
+
+  // Extract markdown from nodes, resolving refs on the fly
+  type Node = unknown;
+  const visitedRefs = new Set<string>();
+
+  function extractNode(node: Node, ctx = { inTable: false, inCode: false }): string {
+    if (node === null || node === undefined) return "";
+    
+    if (typeof node === "string") {
+      // Check if it's a reference like "$L30"
+      const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+      if (refMatch) {
+        const refId = refMatch[1];
+        if (visitedRefs.has(refId)) return ""; // Prevent cycles
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        const result = refNode ? extractNode(refNode, ctx) : "";
+        visitedRefs.delete(refId);
+        return result;
+      }
+      // Filter out RSC-specific artifacts, but preserve content inside code blocks
+      if (!ctx.inCode && (node === "$undefined" || node === "$" || /^\$[A-Z]/.test(node))) return "";
+      return node.trim() ? node : "";
+    }
+    
+    if (typeof node === "number") return String(node);
+    if (typeof node === "boolean") return "";
+    if (!Array.isArray(node)) return "";
+
+    // RSC element: ["$", "tag", key, props]
+    if (node[0] === "$" && typeof node[1] === "string") {
+      const tag = node[1] as string;
+      const props = (node[3] || {}) as Record<string, unknown>;
+
+      // Skip non-content
+      const skipTags = ["script", "style", "svg", "path", "circle", "link", "meta", 
+                        "template", "button", "input", "nav", "footer", "aside"];
+      if (skipTags.includes(tag)) return "";
+
+      // Component ref like $L25
+      if (tag.startsWith("$L")) {
+        const refId = tag.slice(2);
+        if (visitedRefs.has(refId)) return "";
+        
+        // Check for heading components with baseId
+        if (props.baseId && props.children) {
+          return `## ${String(props.children)}\n\n`;
+        }
+        
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        let result = "";
+        if (refNode) {
+          result = extractNode(refNode, ctx);
+        } else if (props.children) {
+          result = extractNode(props.children as Node, ctx);
+        }
+        visitedRefs.delete(refId);
+        return result;
+      }
+
+      const children = props.children;
+      const content = children ? extractNode(children as Node, ctx) : "";
+
+      switch (tag) {
+        case "h1": return `# ${content.trim()}\n\n`;
+        case "h2": return `## ${content.trim()}\n\n`;
+        case "h3": return `### ${content.trim()}\n\n`;
+        case "h4": return `#### ${content.trim()}\n\n`;
+        case "h5": return `##### ${content.trim()}\n\n`;
+        case "h6": return `###### ${content.trim()}\n\n`;
+        case "p": return ctx.inTable ? content : `${content.trim()}\n\n`;
+        case "code": {
+          const codeContent = children ? extractNode(children as Node, { ...ctx, inCode: true }) : "";
+          return ctx.inCode ? codeContent : `\`${codeContent}\``;
+        }
+        case "pre": {
+          const preContent = children ? extractNode(children as Node, { ...ctx, inCode: true }) : "";
+          return "```\n" + preContent + "\n```\n\n";
+        }
+        case "strong": case "b": return `**${content}**`;
+        case "em": case "i": return `*${content}*`;
+        case "li": return `- ${content.trim()}\n`;
+        case "ul": case "ol": return content + "\n";
+        case "blockquote": return `> ${content.trim()}\n\n`;
+        case "table": return extractTable(node as unknown[]) + "\n";
+        case "thead": case "tbody": case "tr": case "th": case "td":
+          return content;
+        case "div":
+          if (props.role === "alert" || props["data-slot"] === "alert") {
+            return `> ${content.trim()}\n\n`;
+          }
+          return content;
+        case "a": {
+          const href = props.href as string | undefined;
+          return href && !href.startsWith("#") ? `[${content}](${href})` : content;
+        }
+        default: return content;
+      }
+    }
+
+    // Array of child nodes
+    return (node as Node[]).map(n => extractNode(n, ctx)).join("");
+  }
+
+  function extractTable(tableNode: unknown[]): string {
+    const props = (tableNode[3] || {}) as Record<string, unknown>;
+    const rows: string[][] = [];
+    let headerRowCount = 0;
+
+    function walkTable(node: unknown, isHeader = false): void {
+      if (node === null || node === undefined) return;
+      
+      // Handle string refs
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode) walkTable(refNode, isHeader);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      
+      if (!Array.isArray(node)) return;
+      
+      if (node[0] === "$") {
+        const tag = node[1] as string;
+        const nodeProps = (node[3] || {}) as Record<string, unknown>;
+        
+        // Handle component refs
+        if (tag.startsWith("$L")) {
+          const refId = tag.slice(2);
+          if (!visitedRefs.has(refId)) {
+            visitedRefs.add(refId);
+            const refNode = getParsedChunk(refId);
+            if (refNode) walkTable(refNode, isHeader);
+            visitedRefs.delete(refId);
+          }
+          return;
+        }
+        
+        if (tag === "thead") walkTable(nodeProps.children, true);
+        else if (tag === "tbody") walkTable(nodeProps.children, false);
+        else if (tag === "tr") {
+          const cells: string[] = [];
+          walkCells(nodeProps.children, cells);
+          if (cells.length > 0) {
+            rows.push(cells);
+            if (isHeader) headerRowCount++;
+          }
+        } else walkTable(nodeProps.children, isHeader);
+      } else {
+        for (const child of node) walkTable(child, isHeader);
+      }
+    }
+
+    function walkCells(node: unknown, cells: string[]): void {
+      if (node === null || node === undefined) return;
+      
+      // Handle string refs
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode) walkCells(refNode, cells);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      
+      if (!Array.isArray(node)) return;
+      
+      if (node[0] === "$" && (node[1] === "td" || node[1] === "th")) {
+        const cellProps = (node[3] || {}) as Record<string, unknown>;
+        const text = extractNode(cellProps.children, { inTable: true, inCode: false })
+          .trim()
+          .replace(/\n/g, " ")
+          .replace(/\\/g, "\\\\")  // Escape backslashes first
+          .replace(/\|/g, "\\|");  // Then escape pipes
+        cells.push(text);
+      } else if (node[0] === "$" && typeof node[1] === "string" && (node[1] as string).startsWith("$L")) {
+        // Component ref for a cell
+        const refId = (node[1] as string).slice(2);
+        if (!visitedRefs.has(refId)) {
+          visitedRefs.add(refId);
+          const refNode = getParsedChunk(refId);
+          if (refNode) walkCells(refNode, cells);
+          visitedRefs.delete(refId);
+        }
+      } else {
+        for (const child of node) walkCells(child, cells);
+      }
+    }
+
+    walkTable(props.children);
+    if (rows.length === 0) return "";
+
+    const colCount = Math.max(...rows.map(r => r.length));
+    let md = "";
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i].concat(Array(colCount - rows[i].length).fill(""));
+      md += "| " + row.join(" | ") + " |\n";
+      if (i === headerRowCount - 1 || (headerRowCount === 0 && i === 0)) {
+        md += "| " + Array(colCount).fill("---").join(" | ") + " |\n";
+      }
+    }
+    return md;
+  }
+
+  // Process main content chunk (usually "23")
+  const mainChunk = getParsedChunk("23");
+  
+  if (mainChunk) {
+    const content = extractNode(mainChunk);
+    if (content.trim().length > 100) {
+      const cleaned = content
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+      return { title, content: cleaned };
+    }
+  }
+
+  // Fallback: try other chunks
+  const contentParts: { order: number; text: string }[] = [];
+
+  for (const [id] of chunkMap) {
+    if (id === "23") continue;
+    const parsed = getParsedChunk(id);
+    if (!parsed) continue;
+
+    visitedRefs.clear();
+    const text = extractNode(parsed);
+
+    if (text.trim().length > 50 && 
+        !text.includes("page was not found") && 
+        !text.includes("404")) {
+      contentParts.push({ order: parseInt(id, 16), text: text.trim() });
+    }
+  }
+
+  if (contentParts.length === 0) return null;
+
+  contentParts.sort((a, b) => a.order - b.order);
+  
+  const seen = new Set<string>();
+  const uniqueParts: string[] = [];
+  for (const part of contentParts) {
+    const key = part.text.slice(0, 150);
+    if (!seen.has(key)) {
+      seen.add(key);
+      uniqueParts.push(part.text);
+    }
+  }
+
+  const content = uniqueParts.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  return content.length > 100 ? { title, content } : null;
+}

package/dist/builtin/web-access/storage.ts

@@ -0,0 +1,72 @@
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { ExtractedContent } from "./extract.js";
+import type { SearchResult } from "./perplexity.js";
+
+const CACHE_TTL_MS = 60 * 60 * 1000;
+
+export interface QueryResultData {
+	query: string;
+	answer: string;
+	results: SearchResult[];
+	error: string | null;
+	provider?: string;
+}
+
+export interface StoredSearchData {
+	id: string;
+	type: "search" | "fetch";
+	timestamp: number;
+	queries?: QueryResultData[];
+	urls?: ExtractedContent[];
+}
+
+const storedResults = new Map<string, StoredSearchData>();
+
+export function generateId(): string {
+	return Date.now().toString(36) + Math.random().toString(36).slice(2, 8);
+}
+
+export function storeResult(id: string, data: StoredSearchData): void {
+	storedResults.set(id, data);
+}
+
+export function getResult(id: string): StoredSearchData | null {
+	return storedResults.get(id) ?? null;
+}
+
+export function getAllResults(): StoredSearchData[] {
+	return Array.from(storedResults.values());
+}
+
+export function deleteResult(id: string): boolean {
+	return storedResults.delete(id);
+}
+
+export function clearResults(): void {
+	storedResults.clear();
+}
+
+function isValidStoredData(data: unknown): data is StoredSearchData {
+	if (!data || typeof data !== "object") return false;
+	const d = data as Record<string, unknown>;
+	if (typeof d.id !== "string" || !d.id) return false;
+	if (d.type !== "search" && d.type !== "fetch") return false;
+	if (typeof d.timestamp !== "number") return false;
+	if (d.type === "search" && !Array.isArray(d.queries)) return false;
+	if (d.type === "fetch" && !Array.isArray(d.urls)) return false;
+	return true;
+}
+
+export function restoreFromSession(ctx: ExtensionContext): void {
+	storedResults.clear();
+	const now = Date.now();
+
+	for (const entry of ctx.sessionManager.getBranch()) {
+		if (entry.type === "custom" && entry.customType === "web-search-results") {
+			const data = entry.data;
+			if (isValidStoredData(data) && now - data.timestamp < CACHE_TTL_MS) {
+				storedResults.set(data.id, data);
+			}
+		}
+	}
+}

package/dist/builtin/web-access/summary-review.ts

@@ -0,0 +1,276 @@
+import { complete, getModel, type Message, type Model } from "@mariozechner/pi-ai";
+import type { ExtensionContext } from "@mariozechner/pi-coding-agent";
+import type { QueryResultData } from "./storage.js";
+
+const PREFERRED_SUMMARY_MODELS = [
+	{ provider: "anthropic", id: "claude-haiku-4-5" },
+	{ provider: "openai-codex", id: "gpt-5.3-codex-spark" },
+] as const;
+
+export interface SummaryMeta {
+	model: string | null;
+	durationMs: number;
+	tokenEstimate: number;
+	fallbackUsed: boolean;
+	fallbackReason?: string;
+	edited?: boolean;
+}
+
+export type SummaryGenerationContext = Pick<ExtensionContext, "model" | "modelRegistry">;
+
+function estimateTokens(text: string): number {
+	const trimmed = text.trim();
+	if (trimmed.length === 0) return 0;
+	return Math.max(1, Math.ceil(trimmed.length / 4));
+}
+
+function summarizeQueryResult(result: QueryResultData): string {
+	if (result.error) {
+		return `Query: ${result.query}\nStatus: Error\nError: ${result.error}`;
+	}
+
+	const lines = [
+		`Query: ${result.query}`,
+		`Provider: ${result.provider ?? "unknown"}`,
+		`Answer: ${result.answer || "(no answer text returned)"}`,
+	];
+
+	if (result.results.length === 0) {
+		lines.push("Sources: none");
+		return lines.join("\n");
+	}
+
+	lines.push("Sources:");
+	for (let i = 0; i < result.results.length; i++) {
+		const source = result.results[i];
+		lines.push(`${i + 1}. ${source.title} — ${source.url}`);
+	}
+
+	return lines.join("\n");
+}
+
+export function buildSummaryPrompt(results: QueryResultData[], feedback?: string): string {
+	const sections = [
+		"You are writing the final web search summary for a coding assistant.",
+		"Write a concise, factual summary using only the provided search results.",
+		"Requirements:",
+		"- Keep it readable and skimmable.",
+		"- Include key findings and caveats.",
+		"- Do not invent sources or claims.",
+		"- If evidence is weak or conflicting, say so explicitly.",
+		"- End with a short \"Sources\" section listing the most relevant URLs.",
+	];
+
+	if (feedback) {
+		sections.push("- Incorporate the user feedback provided below into the summary.");
+	}
+
+	sections.push("");
+	sections.push("<search_results>");
+
+	for (let i = 0; i < results.length; i++) {
+		sections.push(`\n[Result ${i + 1}]`);
+		sections.push(summarizeQueryResult(results[i]));
+	}
+
+	sections.push("\n</search_results>");
+
+	if (feedback) {
+		sections.push("");
+		sections.push("<user_feedback>");
+		sections.push(feedback);
+		sections.push("</user_feedback>");
+	}
+
+	return sections.join("\n");
+}
+
+function buildDeterministicAnswerPreview(answer: string): string {
+	let text = answer.replace(/\s+/g, " ").trim();
+	if (text.length === 0) return "";
+
+	const sourceMarker = text.search(/\bSources?\s*:/i);
+	if (sourceMarker >= 0) text = text.slice(0, sourceMarker).trim();
+	if (text.length === 0) return "";
+
+	return text.length > 240 ? `${text.slice(0, 237)}...` : text;
+}
+
+function buildDeterministicSummaryLines(results: QueryResultData[]): string[] {
+	if (results.length === 0) {
+		return [
+			"No completed search results were available when the curator session finished.",
+			"",
+			"Sources",
+			"- None",
+		];
+	}
+
+	const lines: string[] = [
+		"Summary based on the currently selected search results.",
+		"",
+	];
+
+	const sourceUrls: string[] = [];
+	let successful = 0;
+	let failed = 0;
+
+	for (const result of results) {
+		if (result.error) {
+			failed += 1;
+			lines.push(`- ${result.query}: failed (${result.error})`);
+			continue;
+		}
+
+		successful += 1;
+		const preview = buildDeterministicAnswerPreview(result.answer);
+		if (preview.length > 0) {
+			lines.push(`- ${result.query}: ${preview}`);
+		} else {
+			lines.push(`- ${result.query}: returned ${result.results.length} source${result.results.length === 1 ? "" : "s"} without answer text.`);
+		}
+
+		for (const source of result.results) {
+			if (!sourceUrls.includes(source.url)) {
+				sourceUrls.push(source.url);
+			}
+		}
+	}
+
+	lines.push("");
+	lines.push(`Completed queries: ${results.length}`);
+	lines.push(`Successful: ${successful}`);
+	lines.push(`Failed: ${failed}`);
+	lines.push("");
+	lines.push("Sources");
+
+	if (sourceUrls.length === 0) {
+		lines.push("- None");
+	} else {
+		for (const url of sourceUrls.slice(0, 12)) {
+			lines.push(`- ${url}`);
+		}
+		if (sourceUrls.length > 12) {
+			lines.push(`- ... and ${sourceUrls.length - 12} more`);
+		}
+	}
+
+	return lines;
+}
+
+export function buildDeterministicSummary(results: QueryResultData[]): { summary: string; meta: SummaryMeta } {
+	const summary = buildDeterministicSummaryLines(results).join("\n").trim();
+	const nonEmptySummary = summary.length > 0
+		? summary
+		: "No completed search results were available when the curator session finished.\n\nSources\n- None";
+
+	return {
+		summary: nonEmptySummary,
+		meta: {
+			model: null,
+			durationMs: 0,
+			tokenEstimate: estimateTokens(nonEmptySummary),
+			fallbackUsed: true,
+			fallbackReason: "deterministic-submit-fallback",
+			edited: false,
+		},
+	};
+}
+
+async function resolveSummaryModel(
+	ctx: SummaryGenerationContext,
+	modelOverride?: string,
+): Promise<{ model: Model; apiKey: string; headers?: Record<string, string> }> {
+	const normalizedOverride = typeof modelOverride === "string" ? modelOverride.trim() : "";
+	if (normalizedOverride.length > 0) {
+		const slashIndex = normalizedOverride.indexOf("/");
+		if (slashIndex <= 0 || slashIndex >= normalizedOverride.length - 1) {
+			throw new Error(`Invalid summary model: ${normalizedOverride}. Use provider/model-id.`);
+		}
+		const provider = normalizedOverride.slice(0, slashIndex);
+		const modelId = normalizedOverride.slice(slashIndex + 1);
+		const selectedModel = ctx.modelRegistry.find(provider, modelId);
+		if (!selectedModel) {
+			throw new Error(`Summary model not found: ${normalizedOverride}`);
+		}
+		const selectedAuth = await ctx.modelRegistry.getApiKeyAndHeaders(selectedModel);
+		if (!selectedAuth.ok || !selectedAuth.apiKey) {
+			throw new Error(`No API key available for summary model ${normalizedOverride}`);
+		}
+		return { model: selectedModel, apiKey: selectedAuth.apiKey, headers: selectedAuth.headers };
+	}
+
+	for (const { provider, id } of PREFERRED_SUMMARY_MODELS) {
+		const model = getModel(provider, id);
+		if (!model) continue;
+		const auth = await ctx.modelRegistry.getApiKeyAndHeaders(model);
+		if (auth.ok && auth.apiKey) return { model, apiKey: auth.apiKey, headers: auth.headers };
+	}
+
+	throw new Error(`No API key available for summary models: ${PREFERRED_SUMMARY_MODELS.map(c => `${c.provider}/${c.id}`).join(", ")}`);
+}
+
+function getTextFromContentPart(part: unknown): string {
+	if (!part || typeof part !== "object") return "";
+	const value = part as Record<string, unknown>;
+	if (typeof value.text === "string") return value.text;
+	if (typeof value.refusal === "string") return value.refusal;
+	return "";
+}
+
+function getContentPartType(part: unknown): string {
+	if (!part || typeof part !== "object") return "unknown";
+	const value = part as Record<string, unknown>;
+	return typeof value.type === "string" ? value.type : "unknown";
+}
+
+export async function generateSummaryDraft(
+	results: QueryResultData[],
+	ctx: SummaryGenerationContext,
+	signal?: AbortSignal,
+	modelOverride?: string,
+	feedback?: string,
+): Promise<{ summary: string; meta: SummaryMeta }> {
+	if (!ctx || !ctx.modelRegistry) {
+		throw new Error("Summary generation context unavailable");
+	}
+
+	const startedAt = Date.now();
+	const { model, apiKey, headers } = await resolveSummaryModel(ctx, modelOverride);
+	const prompt = buildSummaryPrompt(results, feedback);
+
+	const userMessage: Message = {
+		role: "user",
+		content: [{ type: "text", text: prompt }],
+		timestamp: Date.now(),
+	};
+
+	const response = await complete(model, { messages: [userMessage] }, { apiKey, headers, signal });
+	if (response.stopReason === "aborted") {
+		throw new Error("Aborted");
+	}
+
+	const contentParts = Array.isArray(response.content) ? response.content : [];
+	const summary = contentParts
+		.map(part => getTextFromContentPart(part))
+		.filter(text => text.trim().length > 0)
+		.join("\n")
+		.trim();
+
+	if (summary.length === 0) {
+		const partTypes = contentParts.map(part => getContentPartType(part));
+		const typesLabel = partTypes.length > 0 ? partTypes.join(", ") : "none";
+		throw new Error(`Summary model returned empty response (content parts: ${typesLabel})`);
+	}
+
+	return {
+		summary,
+		meta: {
+			model: `${model.provider}/${model.id}`,
+			durationMs: Math.max(0, Date.now() - startedAt),
+			tokenEstimate: estimateTokens(summary),
+			fallbackUsed: false,
+			edited: false,
+		},
+	};
+}