@danypops/pi-web-spider 0.10.4 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/package.json +14 -4
- package/src/constants.ts +7 -0
- package/src/daemon-client.ts +190 -0
- package/src/index.ts +346 -351
- package/src/presentation.ts +430 -0
- package/test/cache-paths.test.ts +38 -91
- package/test/daemon-client.test.ts +158 -0
- package/test/daemon-isolation.ts +56 -0
- package/test/e2e-jiti.test.ts +46 -56
- package/test/execute-contract.test.ts +62 -32
- package/test/fixtures/mock-pi-cli.mjs +5 -9
- package/test/harness/extension-harness.ts +554 -0
- package/test/harness/index.ts +10 -0
- package/test/harness/jiti-native-modules.ts +13 -0
- package/test/helpers/fixture-server.ts +48 -0
- package/test/live-fetch.test.ts +10 -3
- package/test/load.test.ts +13 -3
- package/test/paths.test.ts +61 -96
- package/test/presentation.test.ts +190 -0
- package/src/format.ts +0 -142
- package/test/format.test.ts +0 -373
- package/test/playwright-fallback.test.ts +0 -248
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* daemon-client.ts is a deliberate duplication of @danypops/web-spider-daemon's
|
|
3
|
+
* client.ts/state.ts (see daemon-client.ts's own doc comment for why) — this
|
|
4
|
+
* suite mirrors that package's state.test.ts/daemon.test.ts coverage so the
|
|
5
|
+
* duplicate stays correct independently.
|
|
6
|
+
*/
|
|
7
|
+
import { afterEach, describe, expect, it } from "vitest"
|
|
8
|
+
import { mkdtempSync, rmSync } from "node:fs"
|
|
9
|
+
import { tmpdir } from "node:os"
|
|
10
|
+
import { join } from "node:path"
|
|
11
|
+
import {
|
|
12
|
+
connectOrStartWebSpiderClient,
|
|
13
|
+
connectWebSpiderClient,
|
|
14
|
+
ensureAuthToken,
|
|
15
|
+
readDaemonHandle,
|
|
16
|
+
resolveWebSpiderPaths,
|
|
17
|
+
WebSpiderClient,
|
|
18
|
+
type WebSpiderPaths,
|
|
19
|
+
} from "../src/daemon-client.js"
|
|
20
|
+
|
|
21
|
+
function tempPaths(): { root: string; paths: WebSpiderPaths; env: Record<string, string> } {
|
|
22
|
+
const root = mkdtempSync(join(tmpdir(), "pi-web-spider-daemon-client-"))
|
|
23
|
+
const env = {
|
|
24
|
+
...(process.env as Record<string, string>),
|
|
25
|
+
// HOME and WEB_SPIDER_CACHE_PATH matter too, not just the three XDG_*
|
|
26
|
+
// vars: the daemon's one-time legacy-cache importer falls back to the
|
|
27
|
+
// real home directory when WEB_SPIDER_CACHE_PATH is unset — verified
|
|
28
|
+
// happening in practice (twice) without this, importing (and renaming)
|
|
29
|
+
// the operator's real ~/.cache/web-spider/pages.json.
|
|
30
|
+
HOME: root,
|
|
31
|
+
XDG_DATA_HOME: join(root, "data"),
|
|
32
|
+
XDG_STATE_HOME: join(root, "state"),
|
|
33
|
+
XDG_RUNTIME_DIR: join(root, "run"),
|
|
34
|
+
WEB_SPIDER_CACHE_PATH: join(root, "no-legacy-cache-here.json"),
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
root,
|
|
38
|
+
// resolveWebSpiderPaths() must see the *same* XDG overrides the spawned
|
|
39
|
+
// child's env carries, or the parent polls a handle path the child never
|
|
40
|
+
// writes to — a real bug this test setup previously had.
|
|
41
|
+
paths: resolveWebSpiderPaths({ env, home: root, uid: 1000 }),
|
|
42
|
+
env,
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
describe("resolveWebSpiderPaths", () => {
|
|
47
|
+
it("places each path under the correct XDG root", () => {
|
|
48
|
+
const { paths } = tempPaths()
|
|
49
|
+
expect(paths.database).toContain(join("data", "web-spider", "web-spider.db"))
|
|
50
|
+
expect(paths.token).toContain(join("state", "web-spider", "auth-token"))
|
|
51
|
+
expect(paths.handle).toContain(join("run", "web-spider", "daemon.json"))
|
|
52
|
+
})
|
|
53
|
+
})
|
|
54
|
+
|
|
55
|
+
describe("ensureAuthToken", () => {
|
|
56
|
+
it("creates a 64-char hex token and reuses it on subsequent calls", () => {
|
|
57
|
+
const { root, paths } = tempPaths()
|
|
58
|
+
try {
|
|
59
|
+
const first = ensureAuthToken(paths)
|
|
60
|
+
expect(first).toMatch(/^[a-f0-9]{64}$/)
|
|
61
|
+
expect(ensureAuthToken(paths)).toBe(first)
|
|
62
|
+
} finally {
|
|
63
|
+
rmSync(root, { recursive: true, force: true })
|
|
64
|
+
}
|
|
65
|
+
})
|
|
66
|
+
})
|
|
67
|
+
|
|
68
|
+
describe("readDaemonHandle", () => {
|
|
69
|
+
it("returns null when no handle file exists", () => {
|
|
70
|
+
const { root, paths } = tempPaths()
|
|
71
|
+
try {
|
|
72
|
+
expect(readDaemonHandle(paths)).toBeNull()
|
|
73
|
+
} finally {
|
|
74
|
+
rmSync(root, { recursive: true, force: true })
|
|
75
|
+
}
|
|
76
|
+
})
|
|
77
|
+
})
|
|
78
|
+
|
|
79
|
+
describe("WebSpiderClient", () => {
|
|
80
|
+
it("calls the operation endpoint with a bearer token and returns the result", async () => {
|
|
81
|
+
let capturedRequest: Request | undefined
|
|
82
|
+
const client = new WebSpiderClient("http://127.0.0.1:1", "test-token", async (request) => {
|
|
83
|
+
capturedRequest = request
|
|
84
|
+
return new Response(JSON.stringify({ result: { ok: true } }), { status: 200 })
|
|
85
|
+
})
|
|
86
|
+
const result = await client.call("cache.list", {})
|
|
87
|
+
expect(result).toEqual({ ok: true })
|
|
88
|
+
expect(capturedRequest?.headers.get("authorization")).toBe("Bearer test-token")
|
|
89
|
+
expect(capturedRequest?.url).toBe("http://127.0.0.1:1/api/v1/ops")
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
it("throws the daemon's error message on a non-ok response", async () => {
|
|
93
|
+
const client = new WebSpiderClient("http://127.0.0.1:1", "test-token", async () =>
|
|
94
|
+
new Response(JSON.stringify({ error: "bad request" }), { status: 400 }))
|
|
95
|
+
await expect(client.call("fetch", { url: "https://x.test" })).rejects.toThrow("bad request")
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
it("health() validates ok:true and a string version", async () => {
|
|
99
|
+
const client = new WebSpiderClient("http://127.0.0.1:1", "test-token", async () =>
|
|
100
|
+
new Response(JSON.stringify({ ok: true, version: "0.1.0" }), { status: 200 }))
|
|
101
|
+
await expect(client.health()).resolves.toEqual({ ok: true, version: "0.1.0" })
|
|
102
|
+
})
|
|
103
|
+
})
|
|
104
|
+
|
|
105
|
+
describe("connectWebSpiderClient", () => {
|
|
106
|
+
it("fails closed with an actionable message when no daemon is running", () => {
|
|
107
|
+
const { root, paths } = tempPaths()
|
|
108
|
+
try {
|
|
109
|
+
expect(() => connectWebSpiderClient(paths)).toThrow(/daemon is not running/)
|
|
110
|
+
} finally {
|
|
111
|
+
rmSync(root, { recursive: true, force: true })
|
|
112
|
+
}
|
|
113
|
+
})
|
|
114
|
+
})
|
|
115
|
+
|
|
116
|
+
describe("connectOrStartWebSpiderClient — real subprocess integration", () => {
|
|
117
|
+
const cleanups: Array<() => void> = []
|
|
118
|
+
afterEach(() => {
|
|
119
|
+
for (const cleanup of cleanups.splice(0)) cleanup()
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
it("auto-starts the real daemon when none is running, then connects", async () => {
|
|
123
|
+
const { root, paths, env } = tempPaths()
|
|
124
|
+
cleanups.push(() => rmSync(root, { recursive: true, force: true }))
|
|
125
|
+
|
|
126
|
+
const client = await connectOrStartWebSpiderClient(paths, { env })
|
|
127
|
+
const health = await client.health()
|
|
128
|
+
expect(health.ok).toBe(true)
|
|
129
|
+
|
|
130
|
+
const listing = await client.call("cache.list", {})
|
|
131
|
+
expect(listing).toEqual({ total: 0, filtered: 0, offset: 0, limit: 20, pages: [] })
|
|
132
|
+
|
|
133
|
+
// Best-effort shutdown of the daemon we started, so the test doesn't leak a process.
|
|
134
|
+
const handle = readDaemonHandle(paths)
|
|
135
|
+
if (handle) {
|
|
136
|
+
try { process.kill(handle.pid, "SIGTERM") } catch { /* already gone */ }
|
|
137
|
+
}
|
|
138
|
+
}, 15_000)
|
|
139
|
+
|
|
140
|
+
it("reuses an already-running daemon instead of starting a second one", async () => {
|
|
141
|
+
const { root, paths, env } = tempPaths()
|
|
142
|
+
cleanups.push(() => rmSync(root, { recursive: true, force: true }))
|
|
143
|
+
|
|
144
|
+
const first = await connectOrStartWebSpiderClient(paths, { env })
|
|
145
|
+
await first.health()
|
|
146
|
+
const handleAfterFirst = readDaemonHandle(paths)
|
|
147
|
+
|
|
148
|
+
const second = await connectOrStartWebSpiderClient(paths, { env })
|
|
149
|
+
await second.health()
|
|
150
|
+
const handleAfterSecond = readDaemonHandle(paths)
|
|
151
|
+
|
|
152
|
+
expect(handleAfterSecond).toEqual(handleAfterFirst) // same daemon, not a second one started
|
|
153
|
+
|
|
154
|
+
if (handleAfterFirst) {
|
|
155
|
+
try { process.kill(handleAfterFirst.pid, "SIGTERM") } catch { /* already gone */ }
|
|
156
|
+
}
|
|
157
|
+
}, 15_000)
|
|
158
|
+
})
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared per-test daemon isolation. Every test that boots the extension can
|
|
3
|
+
* reach getClient() → connectOrStartWebSpiderClient(), which auto-starts a
|
|
4
|
+
* real `web-spider serve` subprocess using ambient XDG_DATA_HOME/
|
|
5
|
+
* XDG_STATE_HOME/XDG_RUNTIME_DIR when no override is given — verified in
|
|
6
|
+
* practice: running this suite without isolation left four real daemon
|
|
7
|
+
* processes running and real state under the operator's actual
|
|
8
|
+
* ~/.local/share/web-spider, ~/.local/state/web-spider, and
|
|
9
|
+
* $XDG_RUNTIME_DIR/web-spider (cleaned up once, by hand, when discovered).
|
|
10
|
+
*
|
|
11
|
+
* Every createExtensionHarness() call in this package's tests must pass
|
|
12
|
+
* `env: isolatedDaemonEnv().env` and call `.cleanup()` in an after hook.
|
|
13
|
+
*/
|
|
14
|
+
import { mkdtempSync, rmSync } from "node:fs"
|
|
15
|
+
import { tmpdir } from "node:os"
|
|
16
|
+
import { join } from "node:path"
|
|
17
|
+
import { readDaemonHandle, resolveWebSpiderPaths, type WebSpiderPaths } from "../src/daemon-client.js"
|
|
18
|
+
|
|
19
|
+
export interface IsolatedDaemonEnv {
|
|
20
|
+
root: string
|
|
21
|
+
env: Record<string, string>
|
|
22
|
+
paths: WebSpiderPaths
|
|
23
|
+
/** Kills any daemon this test started and removes the temp root. Call in afterAll/afterEach. */
|
|
24
|
+
cleanup(): void
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function isolatedDaemonEnv(prefix = "pi-web-spider-test-"): IsolatedDaemonEnv {
|
|
28
|
+
const root = mkdtempSync(join(tmpdir(), prefix))
|
|
29
|
+
const env: Record<string, string> = {
|
|
30
|
+
...(process.env as Record<string, string>),
|
|
31
|
+
// HOME and WEB_SPIDER_CACHE_PATH matter too, not just the three XDG_* vars:
|
|
32
|
+
// the daemon's one-time legacy-cache importer (resolveLegacyCachePath())
|
|
33
|
+
// falls back to the real home directory when WEB_SPIDER_CACHE_PATH is unset
|
|
34
|
+
// — verified happening in practice, twice, importing (and renaming) the
|
|
35
|
+
// operator's real ~/.cache/web-spider/pages.json into an "isolated" daemon
|
|
36
|
+
// that only isolated the three XDG vars.
|
|
37
|
+
HOME: root,
|
|
38
|
+
XDG_DATA_HOME: join(root, "data"),
|
|
39
|
+
XDG_STATE_HOME: join(root, "state"),
|
|
40
|
+
XDG_RUNTIME_DIR: join(root, "run"),
|
|
41
|
+
WEB_SPIDER_CACHE_PATH: join(root, "no-legacy-cache-here.json"),
|
|
42
|
+
}
|
|
43
|
+
const paths = resolveWebSpiderPaths({ env, home: root, uid: 1000 })
|
|
44
|
+
return {
|
|
45
|
+
root,
|
|
46
|
+
env,
|
|
47
|
+
paths,
|
|
48
|
+
cleanup() {
|
|
49
|
+
const handle = readDaemonHandle(paths)
|
|
50
|
+
if (handle) {
|
|
51
|
+
try { process.kill(handle.pid, "SIGTERM") } catch { /* already gone */ }
|
|
52
|
+
}
|
|
53
|
+
rmSync(root, { recursive: true, force: true })
|
|
54
|
+
},
|
|
55
|
+
}
|
|
56
|
+
}
|
package/test/e2e-jiti.test.ts
CHANGED
|
@@ -6,11 +6,12 @@
|
|
|
6
6
|
* via mock-pi-cli.mjs so the real jiti context is exercised end-to-end.
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
|
-
import {
|
|
9
|
+
import { mkdirSync } from "node:fs"
|
|
10
10
|
import { spawn } from "node:child_process"
|
|
11
|
-
import { resolve, dirname } from "node:path"
|
|
11
|
+
import { resolve, dirname, join } from "node:path"
|
|
12
12
|
import { fileURLToPath } from "node:url"
|
|
13
|
-
import { describe, expect, it } from "vitest"
|
|
13
|
+
import { afterEach, describe, expect, it } from "vitest"
|
|
14
|
+
import { readDaemonHandle, resolveWebSpiderPaths } from "../src/daemon-client.js"
|
|
14
15
|
|
|
15
16
|
const __dirname = dirname(fileURLToPath(import.meta.url))
|
|
16
17
|
const CLI = resolve(__dirname, "fixtures/mock-pi-cli.mjs")
|
|
@@ -25,8 +26,25 @@ interface Event { type: string; [k: string]: unknown }
|
|
|
25
26
|
interface RunResult {
|
|
26
27
|
events: Event[]
|
|
27
28
|
diagPath: string
|
|
29
|
+
/** XDG_RUNTIME_DIR used for this run — read the daemon handle from here for cleanup. */
|
|
30
|
+
xdgRuntimeDir: string
|
|
28
31
|
}
|
|
29
32
|
|
|
33
|
+
// Every run's spawned daemon (auto-started by the extension, detached + unref'd
|
|
34
|
+
// so it survives this test's own subprocess exit) is killed in afterEach —
|
|
35
|
+
// otherwise every single test run in this file leaks a real, permanently
|
|
36
|
+
// running `web-spider serve` process.
|
|
37
|
+
const runtimeDirsToClean: string[] = []
|
|
38
|
+
afterEach(() => {
|
|
39
|
+
for (const xdgRuntimeDir of runtimeDirsToClean.splice(0)) {
|
|
40
|
+
const paths = resolveWebSpiderPaths({ env: { XDG_RUNTIME_DIR: xdgRuntimeDir } })
|
|
41
|
+
const handle = readDaemonHandle(paths)
|
|
42
|
+
if (handle) {
|
|
43
|
+
try { process.kill(handle.pid, "SIGTERM") } catch { /* already gone */ }
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
})
|
|
47
|
+
|
|
30
48
|
function runCli(opts: {
|
|
31
49
|
tool?: string
|
|
32
50
|
/** Single invocation. Mutually exclusive with paramsList. */
|
|
@@ -40,10 +58,20 @@ function runCli(opts: {
|
|
|
40
58
|
return new Promise((resolve, reject) => {
|
|
41
59
|
mkdirSync(CACHE_DIR, { recursive: true })
|
|
42
60
|
const tag = `${Date.now()}-${Math.random().toString(36).slice(2)}`
|
|
43
|
-
// Unique
|
|
44
|
-
const cachePath = `${CACHE_DIR}/cache-${tag}.json`
|
|
45
|
-
// Unique diag path — lets tests read and assert the boot probe.
|
|
61
|
+
// Unique diag path — lets tests read diagnostics.
|
|
46
62
|
const diagPath = `${CACHE_DIR}/diag-${tag}.log`
|
|
63
|
+
// Isolated XDG roots (+ HOME + WEB_SPIDER_CACHE_PATH) per run — without
|
|
64
|
+
// ALL of these, the extension's daemon auto-start uses the operator's
|
|
65
|
+
// real ~/.local/share, ~/.local/state, $XDG_RUNTIME_DIR, and (for the
|
|
66
|
+
// one-time legacy-cache importer, which falls back to the real home
|
|
67
|
+
// directory when WEB_SPIDER_CACHE_PATH is unset) the operator's real
|
|
68
|
+
// ~/.cache/web-spider/pages.json — every one of these verified happening
|
|
69
|
+
// in practice, more than once, before this fix covered all of them.
|
|
70
|
+
const xdgRoot = `${CACHE_DIR}/xdg-${tag}`
|
|
71
|
+
const xdgDataHome = join(xdgRoot, "data")
|
|
72
|
+
const xdgStateHome = join(xdgRoot, "state")
|
|
73
|
+
const xdgRuntimeDir = join(xdgRoot, "run")
|
|
74
|
+
runtimeDirsToClean.push(xdgRuntimeDir)
|
|
47
75
|
|
|
48
76
|
const invocations = opts.paramsList ?? [opts.params ?? {}]
|
|
49
77
|
|
|
@@ -53,8 +81,12 @@ function runCli(opts: {
|
|
|
53
81
|
// Each element becomes a separate --params flag; mock-pi-cli.mjs
|
|
54
82
|
// invokes the tool once per flag in the same process.
|
|
55
83
|
...invocations.flatMap(p => ["--params", JSON.stringify(p)]),
|
|
56
|
-
"--env", `WEB_SPIDER_CACHE_PATH=${cachePath}`,
|
|
57
84
|
"--env", `WEB_SPIDER_DIAG_PATH=${diagPath}`,
|
|
85
|
+
"--env", `HOME=${xdgRoot}`,
|
|
86
|
+
"--env", `WEB_SPIDER_CACHE_PATH=${join(xdgRoot, "no-legacy-cache-here.json")}`,
|
|
87
|
+
"--env", `XDG_DATA_HOME=${xdgDataHome}`,
|
|
88
|
+
"--env", `XDG_STATE_HOME=${xdgStateHome}`,
|
|
89
|
+
"--env", `XDG_RUNTIME_DIR=${xdgRuntimeDir}`,
|
|
58
90
|
...Object.entries(opts.env ?? {}).flatMap(([k, v]) => ["--env", `${k}=${v}`]),
|
|
59
91
|
]
|
|
60
92
|
|
|
@@ -77,7 +109,7 @@ function runCli(opts: {
|
|
|
77
109
|
proc.on("close", () => {
|
|
78
110
|
clearTimeout(timer)
|
|
79
111
|
try {
|
|
80
|
-
resolve({ events: lines.map(l => JSON.parse(l) as Event), diagPath })
|
|
112
|
+
resolve({ events: lines.map(l => JSON.parse(l) as Event), diagPath, xdgRuntimeDir })
|
|
81
113
|
} catch (e) {
|
|
82
114
|
reject(new Error(`failed to parse NDJSON: ${e}\nlines: ${lines.join("\n")}\nstderr: ${stderr.join("")}`))
|
|
83
115
|
}
|
|
@@ -85,19 +117,6 @@ function runCli(opts: {
|
|
|
85
117
|
})
|
|
86
118
|
}
|
|
87
119
|
|
|
88
|
-
/** Read and parse the boot-probe line from a diag log written by the extension. */
|
|
89
|
-
function readBootProbe(diagPath: string): Record<string, unknown> | null {
|
|
90
|
-
let raw: string
|
|
91
|
-
try { raw = readFileSync(diagPath, "utf8") } catch { return null }
|
|
92
|
-
for (const line of raw.split("\n")) {
|
|
93
|
-
try {
|
|
94
|
-
const obj = JSON.parse(line) as Record<string, unknown>
|
|
95
|
-
if (obj["tag"] === "boot-probe") return obj
|
|
96
|
-
} catch { /* skip malformed lines */ }
|
|
97
|
-
}
|
|
98
|
-
return null
|
|
99
|
-
}
|
|
100
|
-
|
|
101
120
|
// ---------------------------------------------------------------------------
|
|
102
121
|
// Tests
|
|
103
122
|
// ---------------------------------------------------------------------------
|
|
@@ -123,42 +142,26 @@ describe("E2E: production jiti load (tryNative:true + alias)", () => {
|
|
|
123
142
|
expect(text).toHaveProperty("title")
|
|
124
143
|
}, 25000)
|
|
125
144
|
|
|
126
|
-
it("Playwright error propagates
|
|
127
|
-
// enhanced:true forces Playwright regardless of jsRendered, exercising the
|
|
128
|
-
// same error-propagation code path as the jsRendered auto-fallback.
|
|
129
|
-
// GitHub's server-side HTML now has enough content for Readability (wordCount>0)
|
|
130
|
-
// so jsRendered stays false on plain fetch — enhanced:true is the reliable trigger.
|
|
145
|
+
it("Playwright error propagates through the native error event", async () => {
|
|
131
146
|
const { events } = await runCli({
|
|
132
147
|
params: { url: "https://example.com", enhanced: true },
|
|
133
148
|
env: { WEB_SPIDER_PLAYWRIGHT_EXECUTABLE: "/nonexistent" },
|
|
134
149
|
timeoutMs: 20000,
|
|
135
150
|
})
|
|
136
151
|
|
|
137
|
-
const
|
|
138
|
-
expect(
|
|
139
|
-
|
|
140
|
-
const result = (end as { result: { content: { text: string }[] } }).result
|
|
141
|
-
const text = JSON.parse(result.content[0].text)
|
|
142
|
-
|
|
143
|
-
expect(text).toHaveProperty("error")
|
|
144
|
-
expect(typeof text.error).toBe("string")
|
|
145
|
-
expect(text.error).toMatch(/executable|launch|nonexistent/i)
|
|
152
|
+
const error = events.find(e => e.type === "tool_execution_error")
|
|
153
|
+
expect(error).toBeDefined()
|
|
154
|
+
expect((error as { error: string }).error).toMatch(/executable|launch|nonexistent/i)
|
|
146
155
|
}, 25000)
|
|
147
156
|
|
|
148
|
-
it("enhanced=true with missing binary
|
|
157
|
+
it("enhanced=true with missing binary emits an error immediately", async () => {
|
|
149
158
|
const { events } = await runCli({
|
|
150
159
|
params: { url: "https://example.com", enhanced: true },
|
|
151
160
|
env: { WEB_SPIDER_PLAYWRIGHT_EXECUTABLE: "/nonexistent" },
|
|
152
161
|
timeoutMs: 20000,
|
|
153
162
|
})
|
|
154
163
|
|
|
155
|
-
|
|
156
|
-
expect(end).toBeDefined()
|
|
157
|
-
|
|
158
|
-
const result = (end as { result: { content: { text: string }[] } }).result
|
|
159
|
-
const text = JSON.parse(result.content[0].text)
|
|
160
|
-
|
|
161
|
-
expect(text).toHaveProperty("error")
|
|
164
|
+
expect(events.find(e => e.type === "tool_execution_error")).toBeDefined()
|
|
162
165
|
}, 25000)
|
|
163
166
|
|
|
164
167
|
it("process exits cleanly — does not hang", async () => {
|
|
@@ -194,17 +197,4 @@ describe("E2E: production jiti load (tryNative:true + alias)", () => {
|
|
|
194
197
|
expect(exit).toBeDefined()
|
|
195
198
|
expect((exit as { code: number }).code).toBe(0)
|
|
196
199
|
}, 30000)
|
|
197
|
-
|
|
198
|
-
it("boot probe: storeIsMap===false and cacheClass===DiskCache (realm-safe storage)", async () => {
|
|
199
|
-
const { diagPath } = await runCli({
|
|
200
|
-
params: { url: "https://example.com", format: "lean" },
|
|
201
|
-
timeoutMs: 25000,
|
|
202
|
-
})
|
|
203
|
-
|
|
204
|
-
const probe = readBootProbe(diagPath)
|
|
205
|
-
expect(probe).not.toBeNull()
|
|
206
|
-
expect(probe!["cacheClass"]).toBe("DiskCache")
|
|
207
|
-
expect(probe!["storeIsMap"]).toBe(false)
|
|
208
|
-
expect(probe!["storeTag"]).toBe("[object Object]")
|
|
209
|
-
}, 30000)
|
|
210
200
|
})
|
|
@@ -1,21 +1,24 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
import { describe, it, expect, beforeAll, afterAll } from "vitest"
|
|
7
|
-
import { createExtensionHarness, type ExtensionHarness } from "@earendil-works/pi-coding-agent/testing"
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, it } from "vitest"
|
|
2
|
+
import { createExtensionHarness, type ExtensionHarness } from "./harness/index.ts"
|
|
3
|
+
import { isolatedDaemonEnv, type IsolatedDaemonEnv } from "./daemon-isolation.js"
|
|
4
|
+
import { startFixtureServer, type FixtureServer } from "./helpers/fixture-server.js"
|
|
8
5
|
import piFactory from "../src/index.js"
|
|
9
6
|
|
|
10
7
|
let h: ExtensionHarness
|
|
8
|
+
let isolated: IsolatedDaemonEnv
|
|
9
|
+
let server: FixtureServer
|
|
11
10
|
|
|
12
11
|
beforeAll(async () => {
|
|
13
|
-
|
|
12
|
+
isolated = isolatedDaemonEnv("pi-web-spider-execute-contract-test-")
|
|
13
|
+
server = await startFixtureServer()
|
|
14
|
+
h = createExtensionHarness(piFactory, { cwd: "/tmp", env: isolated.env })
|
|
14
15
|
await h.boot()
|
|
15
16
|
})
|
|
16
17
|
|
|
17
18
|
afterAll(async () => {
|
|
18
19
|
await h.shutdown()
|
|
20
|
+
await server.close()
|
|
21
|
+
isolated.cleanup()
|
|
19
22
|
})
|
|
20
23
|
|
|
21
24
|
describe("stream hygiene: extension must not write to stdout or stderr", () => {
|
|
@@ -24,46 +27,73 @@ describe("stream hygiene: extension must not write to stdout or stderr", () => {
|
|
|
24
27
|
})
|
|
25
28
|
})
|
|
26
29
|
|
|
27
|
-
describe("execute()
|
|
28
|
-
it("
|
|
29
|
-
const
|
|
30
|
-
expect(
|
|
31
|
-
|
|
32
|
-
expect(
|
|
33
|
-
|
|
30
|
+
describe("execute() result and failure channels", () => {
|
|
31
|
+
it("registers native call and result renderers", () => {
|
|
32
|
+
const definition = h.tools.get("web_fetch")?.definition
|
|
33
|
+
expect(definition).toBeDefined()
|
|
34
|
+
expect(typeof definition?.renderCall).toBe("function")
|
|
35
|
+
expect(typeof definition?.renderResult).toBe("function")
|
|
36
|
+
})
|
|
37
|
+
|
|
38
|
+
it("throws invalid URL failures through Pi's native error channel", async () => {
|
|
39
|
+
await expect(h.invokeTool("web_fetch", { url: "ftp://not-supported.example.com" }))
|
|
40
|
+
.rejects.toThrow("Unsupported protocol")
|
|
34
41
|
})
|
|
35
42
|
|
|
36
|
-
it("unreachable
|
|
37
|
-
|
|
43
|
+
it("throws unreachable-host failures through Pi's native error channel", async () => {
|
|
44
|
+
await expect(h.invokeTool("web_fetch", {
|
|
38
45
|
url: "http://this-host-does-not-exist-pivi-test.invalid",
|
|
39
46
|
timeoutMs: 3000,
|
|
40
|
-
})
|
|
41
|
-
expect(result).toHaveProperty("content")
|
|
42
|
-
const text = JSON.parse(result.content[0].text)
|
|
43
|
-
expect(text).toHaveProperty("error")
|
|
47
|
+
})).rejects.toThrow("web_fetch failed")
|
|
44
48
|
})
|
|
45
49
|
|
|
46
|
-
it("missing url returns cache listing
|
|
47
|
-
// No url → local materialized view path: returns cache listing, not an error.
|
|
48
|
-
// This is intentional: omitting url is the way to query the session cache.
|
|
50
|
+
it("missing url returns a typed cache listing", async () => {
|
|
49
51
|
const result = await h.invokeTool("web_fetch", {}) as any
|
|
50
52
|
expect(result).toHaveProperty("content")
|
|
51
53
|
const text = JSON.parse(result.content[0].text)
|
|
52
54
|
expect(text).not.toHaveProperty("error")
|
|
53
55
|
expect(text).toHaveProperty("total")
|
|
54
|
-
expect(typeof text.total).toBe("number")
|
|
55
|
-
expect(text).toHaveProperty("pages")
|
|
56
56
|
expect(Array.isArray(text.pages)).toBe(true)
|
|
57
|
+
expect(result.details).toMatchObject({ kind: "web", operation: "cache-list", cache: "listing" })
|
|
57
58
|
})
|
|
58
59
|
|
|
59
|
-
it("highlights
|
|
60
|
-
|
|
60
|
+
it("validates highlights query before attempting a fetch", async () => {
|
|
61
|
+
await expect(h.invokeTool("web_fetch", {
|
|
61
62
|
url: "http://127.0.0.1:1",
|
|
62
63
|
format: "highlights",
|
|
63
|
-
})
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
64
|
+
})).rejects.toThrow("highlights format requires a query")
|
|
65
|
+
})
|
|
66
|
+
|
|
67
|
+
it("returns robots denial as a typed blocked outcome", async () => {
|
|
68
|
+
// A real robots.txt served by the real (isolated) daemon's own RobotsCache
|
|
69
|
+
// fetch — replaces mocking globalThis.fetch, which the daemon (a separate
|
|
70
|
+
// process) never sees.
|
|
71
|
+
server.set("/robots.txt", "User-agent: *\nDisallow: /private", "text/plain")
|
|
72
|
+
server.set("/private", "<html><body><article><h1>Secret</h1><p>Should never be fetched.</p></article></body></html>")
|
|
73
|
+
|
|
74
|
+
const result = await h.invokeTool("web_fetch", { url: `${server.baseUrl}/private` }) as any
|
|
75
|
+
expect(JSON.parse(result.content[0].text)).toMatchObject({ blocked: true, reason: "robots.txt" })
|
|
76
|
+
expect(result.details).toMatchObject({ kind: "web", status: "blocked", blockedBy: "robots.txt" })
|
|
77
|
+
})
|
|
78
|
+
})
|
|
79
|
+
|
|
80
|
+
describe("ingest: explicit opt-in Papyrus wiring", () => {
|
|
81
|
+
it("never calls papyrus.ingest when ingest is omitted (default behavior unchanged)", async () => {
|
|
82
|
+
server.set("/plain", "<html><body><article><h1>Plain</h1><p>No mesh.</p></article></body></html>")
|
|
83
|
+
const result = await h.invokeTool("web_fetch", { url: `${server.baseUrl}/plain`, format: "lean" }) as any
|
|
84
|
+
expect(JSON.parse(result.content[0].text)).not.toHaveProperty("papyrus")
|
|
85
|
+
expect(result.details.papyrusDocs).toBeUndefined()
|
|
86
|
+
})
|
|
87
|
+
|
|
88
|
+
it("forwards ingest:true for a single-page fetch to the daemon's papyrus.ingest op, which fails closed with no Papyrus daemon reachable in this isolated test environment", async () => {
|
|
89
|
+
server.set("/ingest-me", "<html><body><article><h1>Ingest me</h1><p>Worth keeping.</p></article></body></html>")
|
|
90
|
+
await expect(h.invokeTool("web_fetch", { url: `${server.baseUrl}/ingest-me`, format: "lean", ingest: true }))
|
|
91
|
+
.rejects.toThrow(/Papyrus daemon is not running|Papyrus daemon state is stale/)
|
|
67
92
|
})
|
|
68
93
|
|
|
94
|
+
// The search-path wiring (maybeIngestSearch) uses the exact same call() helper and
|
|
95
|
+
// papyrus.ingest operation as the fetch path exercised above; a live-network search-
|
|
96
|
+
// engine round trip isn't repeated here to avoid a flaky, network-dependent test.
|
|
97
|
+
// Search-specific mapping/bounding is covered by web-spider-daemon's
|
|
98
|
+
// papyrus-mapping.test.ts and papyrus-ingest-service.test.ts.
|
|
69
99
|
})
|
|
@@ -19,8 +19,8 @@
|
|
|
19
19
|
*/
|
|
20
20
|
|
|
21
21
|
import { createJiti } from "jiti";
|
|
22
|
+
import { resolve } from "node:path";
|
|
22
23
|
import { fileURLToPath } from "node:url";
|
|
23
|
-
import { resolve, dirname } from "node:path";
|
|
24
24
|
import { createRequire } from "node:module";
|
|
25
25
|
|
|
26
26
|
// ---------------------------------------------------------------------------
|
|
@@ -50,16 +50,12 @@ for (const [k, v] of Object.entries(envOverrides)) process.env[k] = v;
|
|
|
50
50
|
|
|
51
51
|
// tryNative:true (default) — matches the real Pi Node.js binary.
|
|
52
52
|
// The test harness uses tryNative:false, which breaks vi.mock interception.
|
|
53
|
-
const
|
|
54
|
-
const piDistRoot = resolve(__dirname, "../../../../../pi-mono/packages/coding-agent/dist");
|
|
55
|
-
const require = createRequire(import.meta.url);
|
|
53
|
+
const require = createRequire(import.meta.url);
|
|
56
54
|
|
|
57
55
|
const alias = {
|
|
58
|
-
"@earendil-works/pi-coding-agent": resolve(
|
|
59
|
-
"@earendil-works/pi-
|
|
60
|
-
"
|
|
61
|
-
"@earendil-works/pi-ai": resolve(piDistRoot, "../../ai/dist/index.js"),
|
|
62
|
-
"typebox": require.resolve("typebox"),
|
|
56
|
+
"@earendil-works/pi-coding-agent": fileURLToPath(import.meta.resolve("@earendil-works/pi-coding-agent")),
|
|
57
|
+
"@earendil-works/pi-tui": fileURLToPath(import.meta.resolve("@earendil-works/pi-tui")),
|
|
58
|
+
"typebox": require.resolve("typebox"),
|
|
63
59
|
};
|
|
64
60
|
|
|
65
61
|
const jiti = createJiti(import.meta.url, { moduleCache: false, alias });
|