@danypops/pi-web-spider 0.10.4

package/test/cache-paths.test.ts

@@ -0,0 +1,213 @@
+/**
+ * Cache path tests — local materialized view behaviour.
+ *
+ * Two paths in execute() that were previously uncovered:
+ *
+ *   1. Cache hit  — when a URL has already been fetched in this session,
+ *                   web_fetch(url) returns the cached page without hitting
+ *                   the network again.
+ *
+ *   2. Cache search — web_fetch({ query }) with no url searches all cached
+ *                     pages using BM25F and returns ranked hits.
+ *
+ * Both paths use a single harness session so the cache accumulates state
+ * across calls as it would in a real Pi session.
+ */
+
+import { readFileSync } from "node:fs"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  it,
+  vi,
+} from "vitest"
+import {
+  createExtensionHarness,
+  type ExtensionHarness,
+} from "@earendil-works/pi-coding-agent/testing"
+
+const __dirname = dirname(fileURLToPath(import.meta.url))
+const FIXTURES = join(__dirname, "../../web-spider/fixtures")
+const ARTICLE_HTML = readFileSync(join(FIXTURES, "article-with-images.html"), "utf8")
+
+const URL_A = "https://example.com/article-a"
+const URL_B = "https://example.com/article-b"
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function makeOkResponse(html: string) {
+  return {
+    ok: true,
+    status: 200,
+    statusText: "OK",
+    headers: { get: () => null },
+    text:        async () => html,
+    arrayBuffer: async () => new ArrayBuffer(0),
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Single shared session — cache builds up across tests
+// ---------------------------------------------------------------------------
+
+let h: ExtensionHarness
+let fetchMock: ReturnType<typeof vi.fn>
+
+beforeAll(async () => {
+  const { default: factory } = await import("../src/index.js")
+  h = createExtensionHarness(factory, {
+    cwd: "/tmp",
+    env: { WEB_SPIDER_CACHE_PATH: "/tmp/ws-cache-paths-test.json" },
+  })
+  await h.boot()
+})
+
+afterAll(async () => {
+  await h.shutdown()
+})
+
+beforeEach(() => {
+  fetchMock = vi.fn(async (input: RequestInfo | URL) => {
+    const url = input.toString()
+    if (url.includes("robots.txt")) return makeOkResponse("User-agent: *\nAllow: /")
+    if (url.includes("sitemap"))    return { ...makeOkResponse(""), status: 404, ok: false }
+    if (url.startsWith("https://example.com")) return makeOkResponse(ARTICLE_HTML)
+    return { ...makeOkResponse(""), status: 404, ok: false }
+  })
+  vi.stubGlobal("fetch", fetchMock)
+})
+
+// ---------------------------------------------------------------------------
+// Cache listing — no url, no query
+// ---------------------------------------------------------------------------
+
+describe("cache listing path", () => {
+  it("returns total=0 and empty pages on a cold cache", async () => {
+    // Fresh harness in beforeAll, no fetches yet.
+    const result = await h.invokeTool("web_fetch", {}) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text).toHaveProperty("total")
+    expect(text).toHaveProperty("pages")
+    expect(Array.isArray(text.pages)).toBe(true)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Cache hit — second fetch for the same URL skips the network
+// ---------------------------------------------------------------------------
+
+describe("cache hit path", () => {
+  it("fetches URL_A on first call", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      url: URL_A,
+      format: "lean",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text.wordCount).toBeGreaterThan(0)
+  })
+
+  it("returns the cached page on second call without hitting the network", async () => {
+    // Reset the mock so we can assert it was NOT called again.
+    fetchMock.mockClear()
+
+    const result = await h.invokeTool("web_fetch", {
+      url: URL_A,
+      format: "lean",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text.wordCount).toBeGreaterThan(0)
+
+    // Network was not touched for the main URL — only robots.txt may fire.
+    const mainUrlCalls = fetchMock.mock.calls.filter(
+      ([input]: [RequestInfo | URL]) => input.toString().startsWith(URL_A)
+    )
+    expect(mainUrlCalls).toHaveLength(0)
+  })
+
+  it("cache listing shows URL_A after it has been fetched", async () => {
+    const result = await h.invokeTool("web_fetch", {}) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text.total).toBeGreaterThanOrEqual(1)
+    const urls = text.pages.map((p: { url: string }) => p.url)
+    expect(urls).toContain(URL_A)
+  })
+})
+
+// ---------------------------------------------------------------------------
+// Cache search — no url, with query
+// ---------------------------------------------------------------------------
+
+describe("cache search path", () => {
+  it("fetches URL_B to populate the cache", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      url: URL_B,
+      format: "lean",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+    expect(text).not.toHaveProperty("error")
+  })
+
+  it("returns BM25F hits when query matches cached content", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      query: "image scraping web spider",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text).toHaveProperty("hits")
+    expect(Array.isArray(text.hits)).toBe(true)
+    expect(text.hits.length).toBeGreaterThan(0)
+    expect(text).toHaveProperty("pagesSearched")
+    expect(text.pagesSearched).toBeGreaterThanOrEqual(1)
+  })
+
+  it("each hit has url, score, and text", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      query: "image scraping",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+    const hit = text.hits[0]
+
+    expect(hit).toHaveProperty("url")
+    expect(hit).toHaveProperty("score")
+    expect(typeof hit.score).toBe("number")
+  })
+
+  it("returns zero hits gracefully for a query with no matches", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      // Use a single token with no vowels — avoids the hyphen-splitting bug where
+    // "xyzzy-no-such-content-ever-12345" tokenises to [xyzzy, no, such, content,
+    // ever, 12345] and "content" literally matches the article fixture.
+    query: "zxqfkwjpvm",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text).toHaveProperty("hits")
+    expect(text.hits).toHaveLength(0)
+  })
+
+  it("grep= filter narrows cache listing results", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      grep: "article-a",
+    }) as { content: { text: string }[] }
+    const text = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text.pages.every((p: { url: string }) => p.url.includes("article-a"))).toBe(true)
+  })
+})

package/test/e2e-jiti.test.ts

@@ -0,0 +1,210 @@
+/**
+ * E2E tests using the production jiti load path (tryNative:true + alias map).
+ *
+ * vi.mock() operates in vitest's module registry, which the extension factory
+ * bypasses via dynamic import(). These tests run the extension in a subprocess
+ * via mock-pi-cli.mjs so the real jiti context is exercised end-to-end.
+ */
+
+import { readFileSync, mkdirSync } from "node:fs"
+import { spawn }  from "node:child_process"
+import { resolve, dirname } from "node:path"
+import { fileURLToPath } from "node:url"
+import { describe, expect, it } from "vitest"
+
+const __dirname   = dirname(fileURLToPath(import.meta.url))
+const CLI         = resolve(__dirname, "fixtures/mock-pi-cli.mjs")
+const EXTENSION   = resolve(__dirname, "../src/index.ts")
+const CACHE_DIR = "/tmp/ws-e2e-jiti"
+
+// ---------------------------------------------------------------------------
+// Helper — run mock-pi-cli and collect emitted NDJSON events
+// ---------------------------------------------------------------------------
+interface Event { type: string; [k: string]: unknown }
+
+interface RunResult {
+  events: Event[]
+  diagPath: string
+}
+
+function runCli(opts: {
+  tool?: string
+  /** Single invocation. Mutually exclusive with paramsList. */
+  params?: Record<string, unknown>
+  /** Multiple sequential invocations in the same process — same extension
+   *  instance, same cache object. Use for cache round-trip tests. */
+  paramsList?: Record<string, unknown>[]
+  env?: Record<string, string>
+  timeoutMs?: number
+}): Promise<RunResult> {
+  return new Promise((resolve, reject) => {
+    mkdirSync(CACHE_DIR, { recursive: true })
+    const tag = `${Date.now()}-${Math.random().toString(36).slice(2)}`
+    // Unique cache per run — prevents cross-test hits from masking errors.
+    const cachePath = `${CACHE_DIR}/cache-${tag}.json`
+    // Unique diag path — lets tests read and assert the boot probe.
+    const diagPath  = `${CACHE_DIR}/diag-${tag}.log`
+
+    const invocations = opts.paramsList ?? [opts.params ?? {}]
+
+    const args = [
+      "--extension", EXTENSION,
+      "--tool",      opts.tool ?? "web_fetch",
+      // Each element becomes a separate --params flag; mock-pi-cli.mjs
+      // invokes the tool once per flag in the same process.
+      ...invocations.flatMap(p => ["--params", JSON.stringify(p)]),
+      "--env",       `WEB_SPIDER_CACHE_PATH=${cachePath}`,
+      "--env",       `WEB_SPIDER_DIAG_PATH=${diagPath}`,
+      ...Object.entries(opts.env ?? {}).flatMap(([k, v]) => ["--env", `${k}=${v}`]),
+    ]
+
+    const proc = spawn("node", [CLI, ...args], {
+      env: { ...process.env, ...opts.env },
+      stdio: ["ignore", "pipe", "pipe"],
+    })
+
+    const lines: string[] = []
+    const stderr: string[] = []
+
+    proc.stdout.on("data", (chunk: Buffer) => lines.push(...chunk.toString().split("\n").filter(Boolean)))
+    proc.stderr.on("data", (chunk: Buffer) => stderr.push(chunk.toString()))
+
+    const timer = setTimeout(() => {
+      proc.kill("SIGKILL")
+      reject(new Error(`mock-pi-cli timed out after ${opts.timeoutMs ?? 15000}ms`))
+    }, opts.timeoutMs ?? 15000)
+
+    proc.on("close", () => {
+      clearTimeout(timer)
+      try {
+        resolve({ events: lines.map(l => JSON.parse(l) as Event), diagPath })
+      } catch (e) {
+        reject(new Error(`failed to parse NDJSON: ${e}\nlines: ${lines.join("\n")}\nstderr: ${stderr.join("")}`))
+      }
+    })
+  })
+}
+
+/** Read and parse the boot-probe line from a diag log written by the extension. */
+function readBootProbe(diagPath: string): Record<string, unknown> | null {
+  let raw: string
+  try { raw = readFileSync(diagPath, "utf8") } catch { return null }
+  for (const line of raw.split("\n")) {
+    try {
+      const obj = JSON.parse(line) as Record<string, unknown>
+      if (obj["tag"] === "boot-probe") return obj
+    } catch { /* skip malformed lines */ }
+  }
+  return null
+}
+
+// ---------------------------------------------------------------------------
+// Tests
+// ---------------------------------------------------------------------------
+
+describe("E2E: production jiti load (tryNative:true + alias)", () => {
+
+  it("extension loads and tool is registered", async () => {
+    const { events } = await runCli({
+      params: { url: "https://example.com", format: "lean" },
+      timeoutMs: 20000,
+    })
+
+    const start = events.find(e => e.type === "tool_execution_start")
+    const end   = events.find(e => e.type === "tool_execution_end")
+
+    expect(start).toBeDefined()
+    expect(end).toBeDefined()
+
+    const result = (end as { result: { content: { text: string }[] } }).result
+    const text   = JSON.parse(result.content[0].text)
+
+    expect(text).not.toHaveProperty("error")
+    expect(text).toHaveProperty("title")
+  }, 25000)
+
+  it("Playwright error propagates as { error } — auto-fallback path", async () => {
+    // enhanced:true forces Playwright regardless of jsRendered, exercising the
+    // same error-propagation code path as the jsRendered auto-fallback.
+    // GitHub's server-side HTML now has enough content for Readability (wordCount>0)
+    // so jsRendered stays false on plain fetch — enhanced:true is the reliable trigger.
+    const { events } = await runCli({
+      params: { url: "https://example.com", enhanced: true },
+      env:    { WEB_SPIDER_PLAYWRIGHT_EXECUTABLE: "/nonexistent" },
+      timeoutMs: 20000,
+    })
+
+    const end = events.find(e => e.type === "tool_execution_end")
+    expect(end).toBeDefined()
+
+    const result = (end as { result: { content: { text: string }[] } }).result
+    const text   = JSON.parse(result.content[0].text)
+
+    expect(text).toHaveProperty("error")
+    expect(typeof text.error).toBe("string")
+    expect(text.error).toMatch(/executable|launch|nonexistent/i)
+  }, 25000)
+
+  it("enhanced=true with missing binary returns { error } immediately", async () => {
+    const { events } = await runCli({
+      params: { url: "https://example.com", enhanced: true },
+      env:    { WEB_SPIDER_PLAYWRIGHT_EXECUTABLE: "/nonexistent" },
+      timeoutMs: 20000,
+    })
+
+    const end = events.find(e => e.type === "tool_execution_end")
+    expect(end).toBeDefined()
+
+    const result = (end as { result: { content: { text: string }[] } }).result
+    const text   = JSON.parse(result.content[0].text)
+
+    expect(text).toHaveProperty("error")
+  }, 25000)
+
+  it("process exits cleanly — does not hang", async () => {
+    const { events } = await runCli({
+      params: { url: "https://github.com/hyprwm/aquamarine/issues" },
+      env:    { WEB_SPIDER_PLAYWRIGHT_EXECUTABLE: "/nonexistent" },
+      timeoutMs: 20000,  // would hit this if the process hangs
+    })
+
+    const exit = events.find(e => e.type === "exit")
+    expect(exit).toBeDefined()
+  }, 25000)
+
+  it("cache round-trip: second fetch hits in-memory cache — no Map errors", async () => {
+    const { events } = await runCli({
+      paramsList: [
+        { url: "https://example.com", format: "lean" },
+        { url: "https://example.com", format: "lean" },
+      ],
+      timeoutMs: 25000,
+    })
+
+    const ends = events.filter(e => e.type === "tool_execution_end")
+    expect(ends).toHaveLength(2)
+
+    for (const end of ends) {
+      const text = JSON.parse((end as { result: { content: { text: string }[] } }).result.content[0].text)
+      expect(text).not.toHaveProperty("error")
+      expect(text).toHaveProperty("title")
+    }
+
+    const exit = events.find(e => e.type === "exit")
+    expect(exit).toBeDefined()
+    expect((exit as { code: number }).code).toBe(0)
+  }, 30000)
+
+  it("boot probe: storeIsMap===false and cacheClass===DiskCache (realm-safe storage)", async () => {
+    const { diagPath } = await runCli({
+      params: { url: "https://example.com", format: "lean" },
+      timeoutMs: 25000,
+    })
+
+    const probe = readBootProbe(diagPath)
+    expect(probe).not.toBeNull()
+    expect(probe!["cacheClass"]).toBe("DiskCache")
+    expect(probe!["storeIsMap"]).toBe(false)
+    expect(probe!["storeTag"]).toBe("[object Object]")
+  }, 30000)
+})

package/test/execute-contract.test.ts

@@ -0,0 +1,69 @@
+/**
+ * execute() contract: never throw, always return content.
+ * Unhandled exceptions propagate to the Pi TUI and crash the session.
+ */
+
+import { describe, it, expect, beforeAll, afterAll } from "vitest"
+import { createExtensionHarness, type ExtensionHarness } from "@earendil-works/pi-coding-agent/testing"
+import piFactory from "../src/index.js"
+
+let h: ExtensionHarness
+
+beforeAll(async () => {
+  h = createExtensionHarness(piFactory, { cwd: "/tmp" })
+  await h.boot()
+})
+
+afterAll(async () => {
+  await h.shutdown()
+})
+
+describe("stream hygiene: extension must not write to stdout or stderr", () => {
+  it("boot produces no stdout/stderr leaks", () => {
+    expect(h.leaks).toHaveLength(0)
+  })
+})
+
+describe("execute() error contract: never throws, always returns content", () => {
+  it("invalid URL scheme returns error, does not throw", async () => {
+    const result = await h.invokeTool("web_fetch", { url: "ftp://not-supported.example.com" }) as any
+    expect(result).toHaveProperty("content")
+    const text = JSON.parse(result.content[0].text)
+    expect(text).toHaveProperty("error")
+    expect(typeof text.error).toBe("string")
+  })
+
+  it("unreachable host returns error, does not throw", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      url: "http://this-host-does-not-exist-pivi-test.invalid",
+      timeoutMs: 3000,
+    }) as any
+    expect(result).toHaveProperty("content")
+    const text = JSON.parse(result.content[0].text)
+    expect(text).toHaveProperty("error")
+  })
+
+  it("missing url returns cache listing (not error), does not throw", async () => {
+    // No url → local materialized view path: returns cache listing, not an error.
+    // This is intentional: omitting url is the way to query the session cache.
+    const result = await h.invokeTool("web_fetch", {}) as any
+    expect(result).toHaveProperty("content")
+    const text = JSON.parse(result.content[0].text)
+    expect(text).not.toHaveProperty("error")
+    expect(text).toHaveProperty("total")
+    expect(typeof text.total).toBe("number")
+    expect(text).toHaveProperty("pages")
+    expect(Array.isArray(text.pages)).toBe(true)
+  })
+
+  it("highlights without query returns error, does not throw", async () => {
+    const result = await h.invokeTool("web_fetch", {
+      url: "http://127.0.0.1:1",
+      format: "highlights",
+    }) as any
+    expect(result).toHaveProperty("content")
+    const text = JSON.parse(result.content[0].text)
+    expect(text).toHaveProperty("error")
+  })
+
+})

package/test/fixtures/mock-pi-cli.mjs

@@ -0,0 +1,113 @@
+#!/usr/bin/env node
+/**
+ * mock-pi-cli.mjs — minimal Pi CLI stub for E2E integration tests.
+ *
+ * Replicates the production jiti load path (alias + tryNative:true, NOT
+ * tryNative:false as the test harness uses). This is the critical difference
+ * that lets us verify error propagation in the real runtime context.
+ *
+ * Accepts Pi CLI args subset:
+ *   --extension <path>    extension entry point to load
+ *   --env KEY=VALUE       environment overrides (repeatable)
+ *   --tool <name>         tool to invoke (default: web_fetch)
+ *   --params <json>       tool params as JSON string
+ *
+ * Emits NDJSON events on stdout matching Pi's --mode json format:
+ *   { type: "tool_execution_start", toolName, args }
+ *   { type: "tool_execution_end",   toolName, result }
+ *   { type: "exit", code }
+ */
+
+import { createJiti } from "jiti";
+import { fileURLToPath } from "node:url";
+import { resolve, dirname } from "node:path";
+import { createRequire } from "node:module";
+
+// ---------------------------------------------------------------------------
+// Parse args
+// ---------------------------------------------------------------------------
+const args = process.argv.slice(2);
+const get  = (flag) => { const i = args.indexOf(flag); return i !== -1 ? args[i + 1] : null; };
+const getAll = (flag) => {
+  const vals = [];
+  for (let i = 0; i < args.length - 1; i++) if (args[i] === flag) vals.push(args[i + 1]);
+  return vals;
+};
+
+const extensionPath = get("--extension");
+const toolName      = get("--tool") ?? "web_fetch";
+// Support multiple --params flags: each is one sequential tool invocation in the
+// same process (same extension instance, same cache object). This is essential for
+// cache round-trip tests where the second call must cross the realm boundary into
+// an already-populated in-memory cache.
+const allParamsJson = getAll("--params");
+if (allParamsJson.length === 0) allParamsJson.push("{}");
+const envOverrides  = Object.fromEntries(getAll("--env").map(s => s.split("=", 2)));
+
+if (!extensionPath) { process.stderr.write("--extension required\n"); process.exit(1); }
+
+for (const [k, v] of Object.entries(envOverrides)) process.env[k] = v;
+
+// tryNative:true (default) — matches the real Pi Node.js binary.
+// The test harness uses tryNative:false, which breaks vi.mock interception.
+const __dirname  = dirname(fileURLToPath(import.meta.url));
+const piDistRoot = resolve(__dirname, "../../../../../pi-mono/packages/coding-agent/dist");
+const require    = createRequire(import.meta.url);
+
+const alias = {
+  "@earendil-works/pi-coding-agent": resolve(piDistRoot, "index.js"),
+  "@earendil-works/pi-agent-core":   resolve(piDistRoot, "../../agent/dist/index.js"),
+  "@earendil-works/pi-tui":          resolve(piDistRoot, "../../tui/dist/index.js"),
+  "@earendil-works/pi-ai":           resolve(piDistRoot, "../../ai/dist/index.js"),
+  "typebox":                         require.resolve("typebox"),
+};
+
+const jiti = createJiti(import.meta.url, { moduleCache: false, alias });
+
+
+const tools = new Map();
+
+const pi = {
+  registerTool({ name, execute }) { tools.set(name, execute); },
+  on() {},
+  registerCommand() {},
+  ui: { notify() {} },
+};
+
+let factory;
+try {
+  factory = await jiti.import(resolve(extensionPath), { default: true });
+} catch (err) {
+  process.stderr.write(`[mock-pi-cli] load error: ${err.message}\n`);
+  process.exit(1);
+}
+
+if (typeof factory !== "function") {
+  process.stderr.write(`[mock-pi-cli] extension did not export a default function\n`);
+  process.exit(1);
+}
+
+await factory(pi);
+
+const execute = tools.get(toolName);
+if (!execute) {
+  process.stderr.write(`[mock-pi-cli] tool not found: ${toolName}\n`);
+  process.exit(1);
+}
+
+for (const paramsJson of allParamsJson) {
+  const params = JSON.parse(paramsJson);
+  emit({ type: "tool_execution_start", toolName, args: params });
+  try {
+    const result = await execute(`mock-call-${Date.now()}`, params);
+    emit({ type: "tool_execution_end", toolName, result });
+  } catch (err) {
+    emit({ type: "tool_execution_error", toolName, error: err?.message ?? String(err) });
+    emit({ type: "exit", code: 1 });
+    process.exit(1);
+  }
+}
+
+emit({ type: "exit", code: 0 });
+
+function emit(obj) { process.stdout.write(JSON.stringify(obj) + "\n"); }