@undefineds.co/linx 0.3.5 → 0.3.8

package/vendor/pi-web-access/rsc-extract.ts

@@ -0,0 +1,338 @@
+/**
+ * RSC Content Extractor
+ * 
+ * Extracts readable content from Next.js React Server Components (RSC) flight payloads.
+ * RSC pages embed content as JSON in <script>self.__next_f.push([...])</script> tags.
+ */
+
+export interface RSCExtractResult {
+  title: string;
+  content: string;
+}
+
+export function extractRSCContent(html: string): RSCExtractResult | null {
+  if (!html.includes("self.__next_f.push")) {
+    return null;
+  }
+
+  // Parse all RSC chunks into a map
+  const chunkMap = new Map<string, string>();
+  const scriptRegex = /<script>self\.__next_f\.push\(\[1,"([\s\S]*?)"\]\)<\/script>/g;
+
+  for (const match of html.matchAll(scriptRegex)) {
+    let content: string;
+    try {
+      content = JSON.parse('"' + match[1] + '"');
+    } catch {
+      continue;
+    }
+
+    // Parse each line as "id:payload"
+    // Lines are separated by \n, each line is one chunk
+    // Chunk IDs are hex strings, typically 1-4 chars (supports up to 65535 chunks)
+    for (const line of content.split("\n")) {
+      if (!line.trim()) continue;
+      
+      const colonIdx = line.indexOf(":");
+      if (colonIdx <= 0 || colonIdx > 4) continue;
+      
+      const id = line.slice(0, colonIdx);
+      if (!/^[0-9a-f]+$/i.test(id)) continue;
+      
+      const payload = line.slice(colonIdx + 1);
+      if (!payload) continue;
+      
+      const existing = chunkMap.get(id);
+      if (!existing || payload.length > existing.length) {
+        chunkMap.set(id, payload);
+      }
+    }
+  }
+
+  if (chunkMap.size === 0) return null;
+
+  // Extract title
+  const titleMatch = html.match(/<title[^>]*>([^<]+)<\/title>/);
+  const title = titleMatch?.[1]?.split("|")[0]?.trim() || "";
+
+  // Parse and cache parsed chunks
+  const parsedCache = new Map<string, unknown>();
+  
+  function getParsedChunk(id: string): unknown | null {
+    if (parsedCache.has(id)) return parsedCache.get(id);
+    
+    const chunk = chunkMap.get(id);
+    if (!chunk || !chunk.startsWith("[")) {
+      parsedCache.set(id, null);
+      return null;
+    }
+    
+    try {
+      const parsed = JSON.parse(chunk);
+      parsedCache.set(id, parsed);
+      return parsed;
+    } catch {
+      parsedCache.set(id, null);
+      return null;
+    }
+  }
+
+  // Extract markdown from nodes, resolving refs on the fly
+  type Node = unknown;
+  const visitedRefs = new Set<string>();
+
+  function extractNode(node: Node, ctx = { inTable: false, inCode: false }): string {
+    if (node === null || node === undefined) return "";
+    
+    if (typeof node === "string") {
+      // Check if it's a reference like "$L30"
+      const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+      if (refMatch) {
+        const refId = refMatch[1];
+        if (visitedRefs.has(refId)) return ""; // Prevent cycles
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        const result = refNode ? extractNode(refNode, ctx) : "";
+        visitedRefs.delete(refId);
+        return result;
+      }
+      // Filter out RSC-specific artifacts, but preserve content inside code blocks
+      if (!ctx.inCode && (node === "$undefined" || node === "$" || /^\$[A-Z]/.test(node))) return "";
+      return node.trim() ? node : "";
+    }
+    
+    if (typeof node === "number") return String(node);
+    if (typeof node === "boolean") return "";
+    if (!Array.isArray(node)) return "";
+
+    // RSC element: ["$", "tag", key, props]
+    if (node[0] === "$" && typeof node[1] === "string") {
+      const tag = node[1] as string;
+      const props = (node[3] || {}) as Record<string, unknown>;
+
+      // Skip non-content
+      const skipTags = ["script", "style", "svg", "path", "circle", "link", "meta", 
+                        "template", "button", "input", "nav", "footer", "aside"];
+      if (skipTags.includes(tag)) return "";
+
+      // Component ref like $L25
+      if (tag.startsWith("$L")) {
+        const refId = tag.slice(2);
+        if (visitedRefs.has(refId)) return "";
+        
+        // Check for heading components with baseId
+        if (props.baseId && props.children) {
+          return `## ${String(props.children)}\n\n`;
+        }
+        
+        visitedRefs.add(refId);
+        const refNode = getParsedChunk(refId);
+        let result = "";
+        if (refNode) {
+          result = extractNode(refNode, ctx);
+        } else if (props.children) {
+          result = extractNode(props.children as Node, ctx);
+        }
+        visitedRefs.delete(refId);
+        return result;
+      }
+
+      const children = props.children;
+      const content = children ? extractNode(children as Node, ctx) : "";
+
+      switch (tag) {
+        case "h1": return `# ${content.trim()}\n\n`;
+        case "h2": return `## ${content.trim()}\n\n`;
+        case "h3": return `### ${content.trim()}\n\n`;
+        case "h4": return `#### ${content.trim()}\n\n`;
+        case "h5": return `##### ${content.trim()}\n\n`;
+        case "h6": return `###### ${content.trim()}\n\n`;
+        case "p": return ctx.inTable ? content : `${content.trim()}\n\n`;
+        case "code": {
+          const codeContent = children ? extractNode(children as Node, { ...ctx, inCode: true }) : "";
+          return ctx.inCode ? codeContent : `\`${codeContent}\``;
+        }
+        case "pre": {
+          const preContent = children ? extractNode(children as Node, { ...ctx, inCode: true }) : "";
+          return "```\n" + preContent + "\n```\n\n";
+        }
+        case "strong": case "b": return `**${content}**`;
+        case "em": case "i": return `*${content}*`;
+        case "li": return `- ${content.trim()}\n`;
+        case "ul": case "ol": return content + "\n";
+        case "blockquote": return `> ${content.trim()}\n\n`;
+        case "table": return extractTable(node as unknown[]) + "\n";
+        case "thead": case "tbody": case "tr": case "th": case "td":
+          return content;
+        case "div":
+          if (props.role === "alert" || props["data-slot"] === "alert") {
+            return `> ${content.trim()}\n\n`;
+          }
+          return content;
+        case "a": {
+          const href = props.href as string | undefined;
+          return href && !href.startsWith("#") ? `[${content}](${href})` : content;
+        }
+        default: return content;
+      }
+    }
+
+    // Array of child nodes
+    return (node as Node[]).map(n => extractNode(n, ctx)).join("");
+  }
+
+  function extractTable(tableNode: unknown[]): string {
+    const props = (tableNode[3] || {}) as Record<string, unknown>;
+    const rows: string[][] = [];
+    let headerRowCount = 0;
+
+    function walkTable(node: unknown, isHeader = false): void {
+      if (node === null || node === undefined) return;
+      
+      // Handle string refs
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode) walkTable(refNode, isHeader);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      
+      if (!Array.isArray(node)) return;
+      
+      if (node[0] === "$") {
+        const tag = node[1] as string;
+        const nodeProps = (node[3] || {}) as Record<string, unknown>;
+        
+        // Handle component refs
+        if (tag.startsWith("$L")) {
+          const refId = tag.slice(2);
+          if (!visitedRefs.has(refId)) {
+            visitedRefs.add(refId);
+            const refNode = getParsedChunk(refId);
+            if (refNode) walkTable(refNode, isHeader);
+            visitedRefs.delete(refId);
+          }
+          return;
+        }
+        
+        if (tag === "thead") walkTable(nodeProps.children, true);
+        else if (tag === "tbody") walkTable(nodeProps.children, false);
+        else if (tag === "tr") {
+          const cells: string[] = [];
+          walkCells(nodeProps.children, cells);
+          if (cells.length > 0) {
+            rows.push(cells);
+            if (isHeader) headerRowCount++;
+          }
+        } else walkTable(nodeProps.children, isHeader);
+      } else {
+        for (const child of node) walkTable(child, isHeader);
+      }
+    }
+
+    function walkCells(node: unknown, cells: string[]): void {
+      if (node === null || node === undefined) return;
+      
+      // Handle string refs
+      if (typeof node === "string") {
+        const refMatch = node.match(/^\$L([0-9a-f]+)$/i);
+        if (refMatch && !visitedRefs.has(refMatch[1])) {
+          visitedRefs.add(refMatch[1]);
+          const refNode = getParsedChunk(refMatch[1]);
+          if (refNode) walkCells(refNode, cells);
+          visitedRefs.delete(refMatch[1]);
+        }
+        return;
+      }
+      
+      if (!Array.isArray(node)) return;
+      
+      if (node[0] === "$" && (node[1] === "td" || node[1] === "th")) {
+        const cellProps = (node[3] || {}) as Record<string, unknown>;
+        const text = extractNode(cellProps.children, { inTable: true, inCode: false })
+          .trim()
+          .replace(/\n/g, " ")
+          .replace(/\\/g, "\\\\")  // Escape backslashes first
+          .replace(/\|/g, "\\|");  // Then escape pipes
+        cells.push(text);
+      } else if (node[0] === "$" && typeof node[1] === "string" && (node[1] as string).startsWith("$L")) {
+        // Component ref for a cell
+        const refId = (node[1] as string).slice(2);
+        if (!visitedRefs.has(refId)) {
+          visitedRefs.add(refId);
+          const refNode = getParsedChunk(refId);
+          if (refNode) walkCells(refNode, cells);
+          visitedRefs.delete(refId);
+        }
+      } else {
+        for (const child of node) walkCells(child, cells);
+      }
+    }
+
+    walkTable(props.children);
+    if (rows.length === 0) return "";
+
+    const colCount = Math.max(...rows.map(r => r.length));
+    let md = "";
+    for (let i = 0; i < rows.length; i++) {
+      const row = rows[i].concat(Array(colCount - rows[i].length).fill(""));
+      md += "| " + row.join(" | ") + " |\n";
+      if (i === headerRowCount - 1 || (headerRowCount === 0 && i === 0)) {
+        md += "| " + Array(colCount).fill("---").join(" | ") + " |\n";
+      }
+    }
+    return md;
+  }
+
+  // Process main content chunk (usually "23")
+  const mainChunk = getParsedChunk("23");
+  
+  if (mainChunk) {
+    const content = extractNode(mainChunk);
+    if (content.trim().length > 100) {
+      const cleaned = content
+        .replace(/\n{3,}/g, "\n\n")
+        .trim();
+      return { title, content: cleaned };
+    }
+  }
+
+  // Fallback: try other chunks
+  const contentParts: { order: number; text: string }[] = [];
+
+  for (const [id] of chunkMap) {
+    if (id === "23") continue;
+    const parsed = getParsedChunk(id);
+    if (!parsed) continue;
+
+    visitedRefs.clear();
+    const text = extractNode(parsed);
+
+    if (text.trim().length > 50 && 
+        !text.includes("page was not found") && 
+        !text.includes("404")) {
+      contentParts.push({ order: parseInt(id, 16), text: text.trim() });
+    }
+  }
+
+  if (contentParts.length === 0) return null;
+
+  contentParts.sort((a, b) => a.order - b.order);
+  
+  const seen = new Set<string>();
+  const uniqueParts: string[] = [];
+  for (const part of contentParts) {
+    const key = part.text.slice(0, 150);
+    if (!seen.has(key)) {
+      seen.add(key);
+      uniqueParts.push(part.text);
+    }
+  }
+
+  const content = uniqueParts.join("\n\n").replace(/\n{3,}/g, "\n\n").trim();
+  return content.length > 100 ? { title, content } : null;
+}

package/vendor/pi-web-access/skills/librarian/SKILL.md

@@ -0,0 +1,195 @@
+---
+name: librarian
+description: Research open-source libraries with evidence-backed answers and GitHub permalinks. Use when the user asks about library internals, needs implementation details with source code references, wants to understand why something was changed, or needs authoritative answers backed by actual code. Excels at navigating large open-source repos and providing citations to exact lines of code.
+---
+
+# Librarian
+
+Answer questions about open-source libraries by finding evidence with GitHub permalinks. Every claim backed by actual code.
+
+## Execution Model
+
+Pi executes tool calls sequentially, even when you emit multiple calls in one turn. But batching independent calls in a single turn still saves LLM round-trips (~5-10s each). Use these patterns:
+
+| Pattern | When | Actually parallel? |
+|---------|------|-------------------|
+| Batch tool calls in one turn | Independent ops (web_search + fetch_content + read) | No, but saves round-trips |
+| `fetch_content({ urls: [...] })` | Multiple URLs to fetch | Yes (3 concurrent) |
+| Bash with `&` + `wait` | Multiple git/gh commands | Yes (OS-level) |
+
+## Step 1: Classify the Request
+
+Before doing anything, classify the request to pick the right research strategy.
+
+| Type | Trigger | Primary Approach |
+|------|---------|-----------------|
+| **Conceptual** | "How do I use X?", "Best practice for Y?" | web_search + fetch_content (README/docs) |
+| **Implementation** | "How does X implement Y?", "Show me the source" | fetch_content (clone) + code search |
+| **Context/History** | "Why was this changed?", "History of X?" | git log + git blame + issue/PR search |
+| **Comprehensive** | Complex or ambiguous requests, "deep dive" | All of the above |
+
+## Step 2: Research by Type
+
+### Conceptual Questions
+
+Batch these in one turn:
+
+1. **web_search**: `"library-name topic"` via Perplexity for recent articles and discussions
+2. **fetch_content**: the library's GitHub repo URL to clone and check README, docs, or examples
+
+Synthesize web results + repo docs. Cite official documentation and link to relevant source files.
+
+### Implementation Questions
+
+The core workflow -- clone, find, permalink:
+
+1. **fetch_content** the GitHub repo URL -- this clones it locally and returns the file tree
+2. Use **bash** to search the cloned repo: `grep -rn "function_name"`, `find . -name "*.ts"`
+3. Use **read** to examine specific files once you've located them
+4. Get the commit SHA: `cd /tmp/pi-github-repos/owner/repo && git rev-parse HEAD`
+5. Construct permalink: `https://github.com/owner/repo/blob/<sha>/path/to/file#L10-L20`
+
+Batch the initial calls: fetch_content (clone) + web_search (recent discussions) in one turn. Then dig into the clone with grep/read once it's available.
+
+### Context/History Questions
+
+Use git operations on the cloned repo:
+
+```bash
+cd /tmp/pi-github-repos/owner/repo
+
+# Recent changes to a specific file
+git log --oneline -n 20 -- path/to/file.ts
+
+# Who changed what and when
+git blame -L 10,30 path/to/file.ts
+
+# Full diff for a specific commit
+git show <sha> -- path/to/file.ts
+
+# Search commit messages
+git log --oneline --grep="keyword" -n 10
+```
+
+For issues and PRs, use bash:
+
+```bash
+# Search issues
+gh search issues "keyword" --repo owner/repo --state all --limit 10
+
+# Search merged PRs
+gh search prs "keyword" --repo owner/repo --state merged --limit 10
+
+# View specific issue/PR with comments
+gh issue view <number> --repo owner/repo --comments
+gh pr view <number> --repo owner/repo --comments
+
+# Release notes
+gh api repos/owner/repo/releases --jq '.[0:5] | .[].tag_name'
+```
+
+### Comprehensive Research
+
+Combine everything. Batch these in one turn:
+
+1. **web_search**: recent articles and discussions
+2. **fetch_content**: clone the repo (or multiple repos if comparing)
+3. **bash**: `gh search issues "keyword" --repo owner/repo --limit 10 & gh search prs "keyword" --repo owner/repo --state merged --limit 10 & wait`
+
+Then dig into the clone with grep, read, git blame, git log as needed.
+
+## Step 3: Construct Permalinks
+
+Permalinks are the whole point. They make your answers citable and verifiable.
+
+```
+https://github.com/<owner>/<repo>/blob/<commit-sha>/<filepath>#L<start>-L<end>
+```
+
+Getting the SHA from a cloned repo:
+
+```bash
+cd /tmp/pi-github-repos/owner/repo && git rev-parse HEAD
+```
+
+Getting the SHA from a tag:
+
+```bash
+gh api repos/owner/repo/git/refs/tags/v1.0.0 --jq '.object.sha'
+```
+
+Always use full commit SHAs, not branch names. Branch links break when code changes. Permalinks don't.
+
+## Step 4: Cite Everything
+
+Every code-related claim needs a permalink. Format:
+
+```markdown
+The stale time check happens in [`notifyManager.ts`](https://github.com/TanStack/query/blob/abc123/packages/query-core/src/notifyManager.ts#L42-L50):
+
+\`\`\`typescript
+function isStale(query: Query, staleTime: number): boolean {
+  return query.state.dataUpdatedAt + staleTime < Date.now()
+}
+\`\`\`
+```
+
+For conceptual answers, link to official docs and relevant source files. For implementation answers, every function/class reference should have a permalink.
+
+## Video Analysis
+
+For questions about video tutorials, conference talks, or screen recordings:
+
+```typescript
+// Full extraction (transcript + visual descriptions)
+fetch_content({ url: "https://youtube.com/watch?v=abc" })
+
+// Ask a specific question about a video
+fetch_content({ url: "https://youtube.com/watch?v=abc", prompt: "What libraries are imported in this tutorial?" })
+
+// Single frame at a known moment
+fetch_content({ url: "https://youtube.com/watch?v=abc", timestamp: "23:41" })
+
+// Range scan for visual discovery
+fetch_content({ url: "https://youtube.com/watch?v=abc", timestamp: "23:41-25:00" })
+
+// Custom density across a range
+fetch_content({ url: "https://youtube.com/watch?v=abc", timestamp: "23:41-25:00", frames: 3 })
+
+// Whole-video sampling
+fetch_content({ url: "https://youtube.com/watch?v=abc", frames: 6 })
+
+// Analyze a local recording
+fetch_content({ url: "/path/to/demo.mp4", prompt: "What error message appears on screen?" })
+
+// Batch multiple videos with the same question
+fetch_content({
+  urls: ["https://youtube.com/watch?v=abc", "https://youtube.com/watch?v=def"],
+  prompt: "What packages are installed?"
+})
+```
+
+Use single timestamps for known moments, ranges for visual scanning, and frames-alone for a quick overview of the whole video.
+
+The `prompt` parameter only applies to video content (YouTube URLs and local video files). For non-video URLs, it's ignored.
+
+## Failure Recovery
+
+| Failure | Recovery |
+|---------|----------|
+| grep finds nothing | Broaden the query, try concept names instead of exact function names |
+| gh CLI rate limited | Use the already-cloned repo in /tmp/pi-github-repos/ for git operations |
+| Repo too large to clone | fetch_content returns an API-only view automatically; use that or add `forceClone: true` |
+| File not found in clone | Branch name with slashes may have misresolved; list the repo tree and navigate manually |
+| Uncertain about implementation | State your uncertainty explicitly, propose a hypothesis, show what evidence you did find |
+| Video extraction fails | Ensure Chrome is signed into gemini.google.com (free) or set GEMINI_API_KEY |
+| Page returns 403/bot block | Gemini fallback triggers automatically; no action needed if Gemini is configured |
+| web_search fails | Check provider config; try explicit `provider: "gemini"` if Perplexity key is missing |
+
+## Guidelines
+
+- Vary search queries when running multiple searches -- different angles, not the same pattern repeated
+- Prefer recent sources; filter out outdated results when they conflict with newer information
+- For version-specific questions, clone the tagged version: `fetch_content("https://github.com/owner/repo/tree/v1.0.0")`
+- When the repo is already cloned from a previous fetch_content call, reuse it -- check the path before cloning again
+- Answer directly. Skip preamble like "I'll help you with..." -- go straight to findings

package/vendor/pi-web-access/storage.ts

@@ -0,0 +1,72 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { ExtractedContent } from "./extract.js";
+import type { SearchResult } from "./perplexity.js";
+
+const CACHE_TTL_MS = 60 * 60 * 1000;
+
+export interface QueryResultData {
+	query: string;
+	answer: string;
+	results: SearchResult[];
+	error: string | null;
+	provider?: string;
+}
+
+export interface StoredSearchData {
+	id: string;
+	type: "search" | "fetch";
+	timestamp: number;
+	queries?: QueryResultData[];
+	urls?: ExtractedContent[];
+}
+
+const storedResults = new Map<string, StoredSearchData>();
+
+export function generateId(): string {
+	return Date.now().toString(36) + Math.random().toString(36).slice(2, 8);
+}
+
+export function storeResult(id: string, data: StoredSearchData): void {
+	storedResults.set(id, data);
+}
+
+export function getResult(id: string): StoredSearchData | null {
+	return storedResults.get(id) ?? null;
+}
+
+export function getAllResults(): StoredSearchData[] {
+	return Array.from(storedResults.values());
+}
+
+export function deleteResult(id: string): boolean {
+	return storedResults.delete(id);
+}
+
+export function clearResults(): void {
+	storedResults.clear();
+}
+
+function isValidStoredData(data: unknown): data is StoredSearchData {
+	if (!data || typeof data !== "object") return false;
+	const d = data as Record<string, unknown>;
+	if (typeof d.id !== "string" || !d.id) return false;
+	if (d.type !== "search" && d.type !== "fetch") return false;
+	if (typeof d.timestamp !== "number") return false;
+	if (d.type === "search" && !Array.isArray(d.queries)) return false;
+	if (d.type === "fetch" && !Array.isArray(d.urls)) return false;
+	return true;
+}
+
+export function restoreFromSession(ctx: ExtensionContext): void {
+	storedResults.clear();
+	const now = Date.now();
+
+	for (const entry of ctx.sessionManager.getBranch()) {
+		if (entry.type === "custom" && entry.customType === "web-search-results") {
+			const data = entry.data;
+			if (isValidStoredData(data) && now - data.timestamp < CACHE_TTL_MS) {
+				storedResults.set(data.id, data);
+			}
+		}
+	}
+}