@pi-lab/webfetch 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/index.mjs +8 -2
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -14,5 +14,5 @@ pi install npm:@pi-lab/webfetch
|
|
|
14
14
|
- **Pagination** — large pages are sliced into chunks; the model reads page by page using `offset`.
|
|
15
15
|
- **Inline script index** — `<script>` tags are stripped from the Markdown body but listed as a numbered index at the end. The model can read any of them with `script=N`.
|
|
16
16
|
- **Redirect handling** — same-domain redirects are followed automatically (up to 10 hops); cross-domain redirects are surfaced to the model so it can decide whether to follow.
|
|
17
|
-
- **Binary downloads** — non-text responses (PDFs, images, etc.) are saved to
|
|
18
|
-
- **LRU cache** — processed Markdown is cached in memory
|
|
17
|
+
- **Binary downloads** — non-text responses (PDFs, images, etc.) are saved to `~/.pi/agent/pi-lab/tmp/webfetch/` and the file path is returned.
|
|
18
|
+
- **LRU cache** — processed Markdown is cached in memory so paginating the same URL doesn't re-fetch.
|
package/dist/index.mjs
CHANGED
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
import { Type } from "@sinclair/typebox";
|
|
2
|
-
import { join } from "node:path";
|
|
3
2
|
import { keyHint } from "@mariozechner/pi-coding-agent";
|
|
4
3
|
import { Text } from "@mariozechner/pi-tui";
|
|
5
4
|
import { LRUCache } from "lru-cache";
|
|
6
5
|
import { mkdir, writeFile } from "node:fs/promises";
|
|
6
|
+
import { join } from "node:path";
|
|
7
7
|
import { Readability } from "@mozilla/readability";
|
|
8
8
|
import { parseHTML } from "linkedom";
|
|
9
9
|
import TurndownService from "turndown";
|
|
10
|
+
import { homedir } from "node:os";
|
|
10
11
|
//#region src/config.ts
|
|
11
12
|
const DEFAULT_CONFIG = {
|
|
12
13
|
maxPageLength: 2e4,
|
|
@@ -233,6 +234,11 @@ function processPlainText(text) {
|
|
|
233
234
|
return text;
|
|
234
235
|
}
|
|
235
236
|
//#endregion
|
|
237
|
+
//#region src/paths.ts
|
|
238
|
+
function getBinaryTempDir(home = homedir()) {
|
|
239
|
+
return join(home, ".pi", "agent", "pi-lab", "tmp", "webfetch");
|
|
240
|
+
}
|
|
241
|
+
//#endregion
|
|
236
242
|
//#region src/tool.ts
|
|
237
243
|
function formatScriptIndex(scripts) {
|
|
238
244
|
if (scripts.length === 0) return "";
|
|
@@ -313,7 +319,7 @@ function registerWebFetchTool(pi, config) {
|
|
|
313
319
|
} catch {
|
|
314
320
|
throw new Error(`Invalid URL: ${url}`);
|
|
315
321
|
}
|
|
316
|
-
const tempDir =
|
|
322
|
+
const tempDir = getBinaryTempDir();
|
|
317
323
|
let entry = cache.get(normalizedUrl);
|
|
318
324
|
if (!entry) {
|
|
319
325
|
onUpdate?.({
|