@openlines/opl-pi-sht 0.1.14

package/extensions/opl-webaccess/providers/ddgs.ts

@@ -0,0 +1,72 @@
+import { isAbortError } from "../utils.js";
+import type { ProviderConfig } from "../config.js";
+import type { SearchQueryResult, SearchResult } from "../types.js";
+
+interface DdgsSearchResult {
+  title: string;
+  href: string;
+  body: string;
+}
+
+interface DdgsApiResponse {
+  results: DdgsSearchResult[];
+}
+
+export async function searchDdgs(
+  query: string,
+  cfg: ProviderConfig,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  if (signal?.aborted) return { query, answer: "", results: [], error: "Aborted" };
+
+  const apiUrl = (cfg.apiUrl ?? "http://127.0.0.1:8000").replace(/\/$/, "");
+  const maxResults = cfg.maxResults ?? 5;
+
+  const params = new URLSearchParams({
+    query,
+    max_results: String(maxResults),
+  });
+
+  let response: Response;
+  try {
+    response = await fetch(`${apiUrl}/search/text?${params.toString()}`, { signal });
+  } catch (err) {
+    if (isAbortError(err)) return { query, answer: "", results: [], error: "Aborted" };
+    const msg = err instanceof Error ? err.message : String(err);
+    if (msg.includes("ECONNREFUSED") || msg.includes("ENOTFOUND")) {
+      return {
+        query,
+        answer: "",
+        results: [],
+        error: `DDGS API server not reachable at ${apiUrl}. Check that the container is running.`,
+      };
+    }
+    return { query, answer: "", results: [], error: `DDGS API error: ${msg}` };
+  }
+
+  if (!response.ok) {
+    return { query, answer: "", results: [], error: `DDGS API error: HTTP ${response.status}` };
+  }
+
+  let data: DdgsApiResponse;
+  try {
+    data = (await response.json()) as DdgsApiResponse;
+  } catch {
+    return { query, answer: "", results: [], error: "DDGS: failed to parse JSON response" };
+  }
+
+  const raw = data.results ?? [];
+
+  const results: SearchResult[] = raw.map((r) => ({
+    title: r.title,
+    url: r.href,
+  }));
+
+  const answer = raw
+    .slice(0, 3)
+    .map((r) => r.body)
+    .filter(Boolean)
+    .join(" ");
+
+  return { query, answer, results, error: null };
+}

package/extensions/opl-webaccess/providers/exa.ts

@@ -0,0 +1,74 @@
+import { errorMessage, isAbortError } from "../utils.js";
+import type { ProviderConfig } from "../config.js";
+import type { SearchQueryResult, SearchResult } from "../types.js";
+
+interface ExaResult {
+  title: string;
+  url: string;
+  summary?: string;
+  highlights?: string[];
+  text?: string;
+}
+
+interface ExaResponse {
+  results: ExaResult[];
+  requestId?: string;
+  costDollars?: { total: number };
+}
+
+interface ExaError {
+  error?: { message: string };
+  message?: string;
+}
+
+export async function searchExa(
+  query: string,
+  cfg: ProviderConfig,
+  apiKey: string,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  let response: Response;
+  try {
+    response = await fetch("https://api.exa.ai/search", {
+      method: "POST",
+      headers: {
+        "x-api-key": apiKey,
+        "Content-Type": "application/json",
+      },
+      body: JSON.stringify({
+        query,
+        numResults: cfg.maxResults ?? 5,
+        type: cfg.searchType ?? "auto",
+        contents: cfg.includeSummary
+          ? { summary: { query } }
+          : { highlights: { numSentences: 3 } },
+      }),
+      signal,
+    });
+  } catch (err) {
+    if (isAbortError(err)) return { query, answer: "", results: [], error: "Aborted" };
+    return { query, answer: "", results: [], error: errorMessage(err) };
+  }
+
+  if (!response.ok) {
+    const err = (await response.json().catch(() => ({}))) as ExaError;
+    const msg = err.error?.message ?? err.message ?? `HTTP ${response.status}`;
+    return { query, answer: "", results: [], error: `Exa API error: ${msg}` };
+  }
+
+  const data = (await response.json()) as ExaResponse;
+
+  const results: SearchResult[] = (data.results ?? []).map((r) => ({
+    title: r.title || r.url,
+    url: r.url,
+  }));
+
+  // Prefer summary, fall back to highlights, then plain text snippets
+  const answer = (data.results ?? [])
+    .slice(0, 3)
+    .map((r) => r.summary ?? r.highlights?.join(" ") ?? r.text ?? "")
+    .filter(Boolean)
+    .join("\n\n");
+
+  return { query, answer, results, error: null };
+}

package/extensions/opl-webaccess/providers/gemini.ts

@@ -0,0 +1,60 @@
+import { errorMessage, isAbortError } from "../utils.js";
+import type { ProviderConfig } from "../config.js";
+import type { SearchQueryResult, SearchResult } from "../types.js";
+
+interface GeminiResponse {
+  candidates?: Array<{
+    content?: {
+      parts?: Array<{ text?: string }>;
+    };
+    groundingMetadata?: {
+      groundingChunks?: Array<{
+        web?: { uri: string; title: string };
+      }>;
+    };
+  }>;
+  error?: { message: string; code: number };
+}
+
+export async function searchGemini(
+  query: string,
+  cfg: ProviderConfig,
+  apiKey: string,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  const base = cfg.baseUrl ?? "https://generativelanguage.googleapis.com/v1beta";
+  const model = cfg.model ?? "gemini-2.5-flash-lite";
+
+  let response: Response;
+  try {
+    response = await fetch(`${base}/models/${model}:generateContent?key=${apiKey}`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        contents: [{ role: "user", parts: [{ text: query }] }],
+        tools: [{ google_search: {} }],
+      }),
+      signal,
+    });
+  } catch (err) {
+    if (isAbortError(err)) return { query, answer: "", results: [], error: "Aborted" };
+    return { query, answer: "", results: [], error: errorMessage(err) };
+  }
+
+  const data = (await response.json()) as GeminiResponse;
+
+  if (!response.ok) {
+    const msg = data.error?.message ?? `HTTP ${response.status}`;
+    return { query, answer: "", results: [], error: `Gemini API error: ${msg}` };
+  }
+
+  const candidate = data.candidates?.[0];
+  const answer = candidate?.content?.parts?.map((p) => p.text ?? "").join("") ?? "";
+  const chunks = candidate?.groundingMetadata?.groundingChunks ?? [];
+
+  const results: SearchResult[] = chunks
+    .filter((c): c is { web: { uri: string; title: string } } => !!c.web?.uri)
+    .map((c) => ({ title: c.web.title || c.web.uri, url: c.web.uri }));
+
+  return { query, answer, results, error: null };
+}

package/extensions/opl-webaccess/providers/searxng.ts

@@ -0,0 +1,86 @@
+import { errorMessage, isAbortError } from "../utils.js";
+import type { ProviderConfig } from "../config.js";
+import type { SearchQueryResult, SearchResult } from "../types.js";
+
+interface SearxngResult {
+  title: string;
+  url: string;
+  content?: string;
+}
+
+interface SearxngResponse {
+  results: SearxngResult[];
+  error?: string;
+}
+
+export async function searchSearxng(
+  query: string,
+  cfg: ProviderConfig,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  const instanceUrl = cfg.instanceUrl?.replace(/\/$/, "");
+  if (!instanceUrl) {
+    return {
+      query,
+      answer: "",
+      results: [],
+      error: 'SearXNG: instanceUrl not set in opl-webaccess.json. Set "instanceUrl": "https://your-searxng-instance.org" under the searxng provider.',
+    };
+  }
+
+  const params = new URLSearchParams({
+    q: query,
+    format: "json",
+    categories: cfg.categories ?? "general",
+    safesearch: String(cfg.safeSearch ?? 0),
+  });
+
+  let response: Response;
+  try {
+    response = await fetch(`${instanceUrl}/search?${params.toString()}`, {
+      headers: {
+        "Accept": "application/json",
+        "User-Agent": "Mozilla/5.0 (compatible; pi-web-access/1.0)",
+      },
+      signal,
+    });
+  } catch (err) {
+    if (isAbortError(err)) return { query, answer: "", results: [], error: "Aborted" };
+    return { query, answer: "", results: [], error: errorMessage(err) };
+  }
+
+  if (response.status === 403) {
+    return {
+      query,
+      answer: "",
+      results: [],
+      error: `SearXNG: instance at ${instanceUrl} has JSON format disabled. Try a different instance or self-host one. See: https://searx.space`,
+    };
+  }
+
+  if (!response.ok) {
+    return { query, answer: "", results: [], error: `SearXNG error: HTTP ${response.status}` };
+  }
+
+  const data = (await response.json()) as SearxngResponse;
+
+  if (data.error) {
+    return { query, answer: "", results: [], error: `SearXNG error: ${data.error}` };
+  }
+
+  const capped = (data.results ?? []).slice(0, cfg.maxResults ?? 5);
+
+  const results: SearchResult[] = capped.map((r) => ({
+    title: r.title,
+    url: r.url,
+  }));
+
+  // No native answer synthesis — join top snippets
+  const answer = capped
+    .slice(0, 3)
+    .map((r) => r.content)
+    .filter(Boolean)
+    .join(" ");
+
+  return { query, answer, results, error: null };
+}

package/extensions/opl-webaccess/providers/tavily.ts

@@ -0,0 +1,60 @@
+import { errorMessage, isAbortError } from "../utils.js";
+import type { ProviderConfig } from "../config.js";
+import type { SearchQueryResult, SearchResult } from "../types.js";
+
+interface TavilyResult {
+  title: string;
+  url: string;
+  content: string;
+  score: number;
+}
+
+interface TavilyResponse {
+  answer?: string;
+  results: TavilyResult[];
+  error?: string;
+}
+
+export async function searchTavily(
+  query: string,
+  cfg: ProviderConfig,
+  apiKey: string,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  let response: Response;
+  try {
+    response = await fetch("https://api.tavily.com/search", {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({
+        api_key: apiKey,
+        query,
+        max_results: cfg.maxResults ?? 5,
+        include_answer: true,
+      }),
+      signal,
+    });
+  } catch (err) {
+    if (isAbortError(err)) return { query, answer: "", results: [], error: "Aborted" };
+    return { query, answer: "", results: [], error: errorMessage(err) };
+  }
+
+  const data = (await response.json()) as TavilyResponse;
+
+  if (!response.ok) {
+    return { query, answer: "", results: [], error: `Tavily API error: HTTP ${response.status}` };
+  }
+
+  if (data.error) {
+    return { query, answer: "", results: [], error: `Tavily API error: ${data.error}` };
+  }
+
+  const results: SearchResult[] = (data.results ?? []).map((r) => ({
+    title: r.title,
+    url: r.url,
+  }));
+
+  const answer = data.answer ?? results.map((r) => r.title).join(". ");
+
+  return { query, answer, results, error: null };
+}

package/extensions/opl-webaccess/search.ts

@@ -0,0 +1,36 @@
+import { loadConfig, getApiKey } from "./config.js";
+import { searchGemini } from "./providers/gemini.js";
+import { searchTavily } from "./providers/tavily.js";
+import { searchDdgs } from "./providers/ddgs.js";
+import { searchSearxng } from "./providers/searxng.js";
+import { searchExa } from "./providers/exa.js";
+import type { SearchQueryResult } from "./types.js";
+
+export async function searchWeb(
+  query: string,
+  signal?: AbortSignal
+): Promise<SearchQueryResult> {
+  const config = loadConfig();
+  const providerName = config.provider;
+  const providerCfg = config.providers?.[providerName] ?? {};
+
+  switch (providerName) {
+    case "gemini":
+      return searchGemini(query, providerCfg, getApiKey(providerCfg), signal);
+    case "tavily":
+      return searchTavily(query, providerCfg, getApiKey(providerCfg), signal);
+    case "ddgs":
+      return searchDdgs(query, providerCfg, signal);
+    case "searxng":
+      return searchSearxng(query, providerCfg, signal);
+    case "exa":
+      return searchExa(query, providerCfg, getApiKey(providerCfg), signal);
+    default:
+      return {
+        query,
+        answer: "",
+        results: [],
+        error: `Unknown search provider: "${providerName}". Valid options: gemini, tavily, ddgs, searxng, exa`,
+      };
+  }
+}

package/extensions/opl-webaccess/storage.ts

@@ -0,0 +1,51 @@
+import type { ExtensionAPI, ExtensionContext } from "@earendil-works/pi-coding-agent";
+import type { StoredData } from "./types.js";
+
+const CUSTOM_TYPE = "web-access-results";
+const CACHE_TTL_MS = 60 * 60 * 1000; // 1 hour
+
+export const MAX_CONTENT_CHARS = 30_000;
+
+const store = new Map<string, StoredData>();
+
+/** Evict expired entries to prevent unbounded growth. */
+function evictExpired(): void {
+  const cutoff = Date.now() - CACHE_TTL_MS;
+  for (const [id, data] of store) {
+    if (data.timestamp < cutoff) store.delete(id);
+  }
+}
+
+export function generateId(): string {
+  return Date.now().toString(36) + Math.random().toString(36).slice(2, 8);
+}
+
+export function storeResult(id: string, data: StoredData): void {
+  store.set(id, data);
+  evictExpired();
+}
+
+export function getResult(id: string): StoredData | null {
+  return store.get(id) ?? null;
+}
+
+export function clearStore(): void {
+  store.clear();
+}
+
+export function persistResult(pi: ExtensionAPI, data: StoredData): void {
+  pi.appendEntry(CUSTOM_TYPE, data);
+}
+
+export function restoreFromSession(ctx: ExtensionContext): void {
+  store.clear();
+  const now = Date.now();
+  for (const entry of ctx.sessionManager.getBranch()) {
+    if (entry.type === "custom" && entry.customType === CUSTOM_TYPE) {
+      const data = entry.data as StoredData;
+      if (data?.id && data?.type && now - data.timestamp < CACHE_TTL_MS) {
+        store.set(data.id, data);
+      }
+    }
+  }
+}

package/extensions/opl-webaccess/turndown.d.ts

@@ -0,0 +1 @@
+declare module "turndown";

package/extensions/opl-webaccess/types.ts

@@ -0,0 +1,26 @@
+export interface SearchResult {
+  title: string;
+  url: string;
+}
+
+export interface SearchQueryResult {
+  query: string;
+  answer: string;
+  results: SearchResult[];
+  error: string | null;
+}
+
+export interface ExtractedContent {
+  url: string;
+  title: string;
+  content: string;
+  error: string | null;
+}
+
+export interface StoredData {
+  id: string;
+  type: "search" | "fetch";
+  timestamp: number;
+  queries?: SearchQueryResult[];
+  urls?: ExtractedContent[];
+}

package/extensions/opl-webaccess/utils.ts

@@ -0,0 +1,28 @@
+export function errorMessage(err: unknown): string {
+  return err instanceof Error ? err.message : String(err);
+}
+
+export function isAbortError(err: unknown): boolean {
+  const msg = errorMessage(err).toLowerCase();
+  return msg.includes("abort") || msg.includes("cancelled");
+}
+
+export function truncate(text: string, maxChars: number): string {
+  if (text.length <= maxChars) return text;
+  return (
+    text.slice(0, maxChars) +
+    "\n\n[Content truncated. Use get_search_content with the responseId to retrieve the full content.]"
+  );
+}
+
+export function isPdfUrl(url: string): boolean {
+  try {
+    return new URL(url).pathname.toLowerCase().endsWith(".pdf");
+  } catch {
+    return url.toLowerCase().endsWith(".pdf");
+  }
+}
+
+export function isPdfContentType(contentType: string): boolean {
+  return contentType.toLowerCase().includes("application/pdf");
+}

package/package.json

@@ -0,0 +1,55 @@
+{
+  "name": "@openlines/opl-pi-sht",
+  "version": "0.1.14",
+  "description": "Pi coding-agent extension kit: token-efficient context, safe modes, fingerprinted /init, model benchmarking, web access, and browser automation.",
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/linellazatin/opl-pi-sht.git"
+  },
+  "files": [
+    "extensions/",
+    "configs/"
+  ],
+  "keywords": [
+    "pi",
+    "pi-package",
+    "pi-coding-agent",
+    "pi-extension",
+    "coding-agent",
+    "extension"
+  ],
+  "license": "MIT",
+  "author": "linellazatin",
+  "pi": {
+    "extensions": [
+      "./extensions/*/index.ts"
+    ]
+  },
+  "dependencies": {
+    "@mozilla/readability": "^0.6.0",
+    "linkedom": "^0.18.13",
+    "playwright": "^1.49.0",
+    "turndown": "^7.2.4",
+    "unpdf": "^1.8.0"
+  },
+  "peerDependencies": {
+    "@earendil-works/pi-ai": "*",
+    "@earendil-works/pi-coding-agent": "*",
+    "@earendil-works/pi-tui": "*",
+    "typebox": "*"
+  },
+  "type": "module",
+  "scripts": {
+    "test": "npm run test:opl-browser && npm run test:opl-footer && npm run test:opl-init && npm run test:opl-input && npm run test:opl-modes && npm run test:opl-questionnaire && npm run test:opl-todo && npm run test:opl-webaccess && npm run test:opl-simplebench && npm run test:opl-ctxtrim",
+    "test:opl-browser": "OPL_EXTENSION=opl-browser bun test tests/extension-smoke.test.mjs",
+    "test:opl-footer": "bun test tests/opl-footer-helpers.test.mjs && OPL_EXTENSION=opl-footer bun test tests/extension-smoke.test.mjs",
+    "test:opl-init": "bun test tests/opl-init-crawl.test.mjs && OPL_EXTENSION=opl-init bun test tests/extension-smoke.test.mjs",
+    "test:opl-input": "bun test tests/opl-input-style.test.mjs && OPL_EXTENSION=opl-input bun test tests/extension-smoke.test.mjs",
+    "test:opl-modes": "bun test tests/opl-modes-helpers.test.mjs && OPL_EXTENSION=opl-modes bun test tests/extension-smoke.test.mjs",
+    "test:opl-questionnaire": "OPL_EXTENSION=opl-questionnaire bun test tests/extension-smoke.test.mjs",
+    "test:opl-todo": "bun test tests/opl-todo-config.test.mjs && OPL_EXTENSION=opl-todo bun test tests/extension-smoke.test.mjs",
+    "test:opl-webaccess": "bun test tests/opl-webaccess-helpers.test.mjs && OPL_EXTENSION=opl-webaccess bun test tests/extension-smoke.test.mjs",
+    "test:opl-simplebench": "bun test tests/opl-simplebench.test.ts && OPL_EXTENSION=opl-simplebench bun test tests/extension-smoke.test.mjs",
+    "test:opl-ctxtrim": "bun test tests/opl-ctxtrim.test.ts && OPL_EXTENSION=opl-ctxtrim bun test tests/extension-smoke.test.mjs"
+  }
+}