@rahularya01/pi-essentials 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +324 -0
- package/examples/mcp.json +30 -0
- package/examples/pi-essentials.json +32 -0
- package/examples/pi-settings.json +5 -0
- package/package.json +88 -0
- package/skills/pi-essentials/SKILL.md +50 -0
- package/src/config.ts +351 -0
- package/src/errors.ts +96 -0
- package/src/index.ts +43 -0
- package/src/mcp/commands.ts +390 -0
- package/src/mcp/config.ts +157 -0
- package/src/mcp/credential-store.ts +153 -0
- package/src/mcp/index.ts +67 -0
- package/src/mcp/manager.ts +941 -0
- package/src/mcp/oauth.ts +262 -0
- package/src/mcp/proxy-tool.ts +213 -0
- package/src/mcp/render.ts +164 -0
- package/src/mcp/types.ts +63 -0
- package/src/paths.ts +48 -0
- package/src/questions/ask.ts +134 -0
- package/src/questions/index.ts +72 -0
- package/src/questions/render.ts +69 -0
- package/src/questions/validate.ts +85 -0
- package/src/security/env.ts +132 -0
- package/src/security/limits.ts +20 -0
- package/src/security/ssrf.ts +237 -0
- package/src/subagents/activity.ts +132 -0
- package/src/subagents/builtins/oracle.md +11 -0
- package/src/subagents/builtins/reviewer.md +11 -0
- package/src/subagents/builtins/scout.md +12 -0
- package/src/subagents/builtins/worker.md +11 -0
- package/src/subagents/discover.ts +54 -0
- package/src/subagents/herdr.ts +150 -0
- package/src/subagents/index.ts +642 -0
- package/src/subagents/inspector-tail.d.mts +1 -0
- package/src/subagents/inspector-tail.mjs +140 -0
- package/src/subagents/render.ts +464 -0
- package/src/subagents/runner.ts +468 -0
- package/src/subagents/schema.ts +107 -0
- package/src/subagents/types.ts +131 -0
- package/src/subagents/worktree.ts +131 -0
- package/src/todos/index.ts +170 -0
- package/src/todos/render.ts +198 -0
- package/src/todos/state.ts +310 -0
- package/src/ui/render.ts +215 -0
- package/src/web/activity.ts +91 -0
- package/src/web/cache.ts +153 -0
- package/src/web/extract.ts +75 -0
- package/src/web/fetch.ts +167 -0
- package/src/web/html-to-markdown.ts +284 -0
- package/src/web/http.ts +238 -0
- package/src/web/index.ts +214 -0
- package/src/web/providers/brave.ts +27 -0
- package/src/web/providers/duckduckgo.ts +60 -0
- package/src/web/providers/exa.ts +29 -0
- package/src/web/providers/jina.ts +25 -0
- package/src/web/providers/searxng.ts +29 -0
- package/src/web/providers/tavily.ts +31 -0
- package/src/web/providers/types.ts +75 -0
- package/src/web/render.ts +130 -0
- package/src/web/search.ts +108 -0
package/src/web/cache.ts
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { createHash } from "node:crypto";
|
|
4
|
+
import { ensureDir, writePrivateFile } from "../config.ts";
|
|
5
|
+
import { getWebCacheDir } from "../paths.ts";
|
|
6
|
+
import { WEB_CACHE_MAX_BYTES, WEB_CACHE_MAX_ENTRIES, WEB_CACHE_TTL_MS } from "../security/limits.ts";
|
|
7
|
+
|
|
8
|
+
export interface CacheEntry {
|
|
9
|
+
id: string;
|
|
10
|
+
url: string;
|
|
11
|
+
title: string;
|
|
12
|
+
markdown: string;
|
|
13
|
+
createdAt: number;
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
interface IndexEntry {
|
|
17
|
+
id: string;
|
|
18
|
+
url: string;
|
|
19
|
+
title: string;
|
|
20
|
+
createdAt: number;
|
|
21
|
+
bytes: number;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
interface IndexFile {
|
|
25
|
+
entries: IndexEntry[];
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
const ID_PATTERN = /^[0-9a-f]{16}$/;
|
|
29
|
+
|
|
30
|
+
function indexPath(): string {
|
|
31
|
+
return path.join(getWebCacheDir(), "index.json");
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function entryPath(id: string): string {
|
|
35
|
+
return path.join(getWebCacheDir(), `${id}.md`);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function isIndexEntry(value: unknown): value is IndexEntry {
|
|
39
|
+
if (!value || typeof value !== "object") return false;
|
|
40
|
+
const row = value as Partial<IndexEntry>;
|
|
41
|
+
return (
|
|
42
|
+
typeof row.id === "string" &&
|
|
43
|
+
ID_PATTERN.test(row.id) &&
|
|
44
|
+
typeof row.url === "string" &&
|
|
45
|
+
typeof row.createdAt === "number" &&
|
|
46
|
+
typeof row.bytes === "number"
|
|
47
|
+
);
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function loadIndex(): IndexFile {
|
|
51
|
+
const file = indexPath();
|
|
52
|
+
if (!fs.existsSync(file)) return { entries: [] };
|
|
53
|
+
try {
|
|
54
|
+
const parsed = JSON.parse(fs.readFileSync(file, "utf8")) as Partial<IndexFile>;
|
|
55
|
+
if (!Array.isArray(parsed.entries)) return { entries: [] };
|
|
56
|
+
return {
|
|
57
|
+
entries: parsed.entries.filter(isIndexEntry).map((row) => ({ ...row, title: String(row.title ?? row.url) })),
|
|
58
|
+
};
|
|
59
|
+
} catch {
|
|
60
|
+
return { entries: [] };
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function saveIndex(index: IndexFile): void {
|
|
65
|
+
ensureDir(getWebCacheDir());
|
|
66
|
+
writePrivateFile(indexPath(), `${JSON.stringify(index)}\n`);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export function cacheId(url: string): string {
|
|
70
|
+
return createHash("sha256").update(url).digest("hex").slice(0, 16);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export function putCache(url: string, title: string, markdown: string): string {
|
|
74
|
+
const id = cacheId(url);
|
|
75
|
+
try {
|
|
76
|
+
ensureDir(getWebCacheDir());
|
|
77
|
+
writePrivateFile(entryPath(id), markdown);
|
|
78
|
+
const index = loadIndex();
|
|
79
|
+
const entries = index.entries.filter((e) => e.id !== id);
|
|
80
|
+
entries.unshift({ id, url, title, createdAt: Date.now(), bytes: Buffer.byteLength(markdown, "utf8") });
|
|
81
|
+
prune(entries);
|
|
82
|
+
saveIndex({ entries });
|
|
83
|
+
} catch {
|
|
84
|
+
// A failing cache must never fail the fetch; the caller still gets the page.
|
|
85
|
+
}
|
|
86
|
+
return id;
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export function getCache(idOrUrl: string): CacheEntry | undefined {
|
|
90
|
+
const index = loadIndex();
|
|
91
|
+
const meta = index.entries.find((e) => e.id === idOrUrl || e.url === idOrUrl);
|
|
92
|
+
if (!meta) return undefined;
|
|
93
|
+
if (Date.now() - meta.createdAt > WEB_CACHE_TTL_MS) return undefined;
|
|
94
|
+
try {
|
|
95
|
+
const markdown = fs.readFileSync(entryPath(meta.id), "utf8");
|
|
96
|
+
return { id: meta.id, url: meta.url, title: meta.title, markdown, createdAt: meta.createdAt };
|
|
97
|
+
} catch {
|
|
98
|
+
return undefined;
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
export function sliceMarkdown(markdown: string, offset = 0, limit?: number): { text: string; total: number; start: number } {
|
|
103
|
+
const total = markdown.length;
|
|
104
|
+
const start = Math.min(Math.max(0, Math.floor(offset)), total);
|
|
105
|
+
const end = limit !== undefined && Number.isFinite(limit) ? start + Math.max(0, Math.floor(limit)) : total;
|
|
106
|
+
return { text: markdown.slice(start, end), total, start };
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
function prune(entries: IndexEntry[]): void {
|
|
110
|
+
const now = Date.now();
|
|
111
|
+
for (let i = entries.length - 1; i >= 0; i--) {
|
|
112
|
+
if (now - entries[i].createdAt > WEB_CACHE_TTL_MS) {
|
|
113
|
+
unlink(entries[i].id);
|
|
114
|
+
entries.splice(i, 1);
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
while (entries.length > WEB_CACHE_MAX_ENTRIES) {
|
|
118
|
+
const removed = entries.pop();
|
|
119
|
+
if (removed) unlink(removed.id);
|
|
120
|
+
}
|
|
121
|
+
let total = entries.reduce((sum, e) => sum + e.bytes, 0);
|
|
122
|
+
while (total > WEB_CACHE_MAX_BYTES && entries.length > 1) {
|
|
123
|
+
const removed = entries.pop();
|
|
124
|
+
if (!removed) break;
|
|
125
|
+
total -= removed.bytes;
|
|
126
|
+
unlink(removed.id);
|
|
127
|
+
}
|
|
128
|
+
removeOrphans(new Set(entries.map((entry) => entry.id)));
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/** Drop `.md` files the index no longer references (crash or lost-update leftovers). */
|
|
132
|
+
function removeOrphans(live: Set<string>): void {
|
|
133
|
+
let files: string[];
|
|
134
|
+
try {
|
|
135
|
+
files = fs.readdirSync(getWebCacheDir());
|
|
136
|
+
} catch {
|
|
137
|
+
return;
|
|
138
|
+
}
|
|
139
|
+
for (const file of files) {
|
|
140
|
+
if (!file.endsWith(".md")) continue;
|
|
141
|
+
const id = file.slice(0, -3);
|
|
142
|
+
if (!ID_PATTERN.test(id) || live.has(id)) continue;
|
|
143
|
+
unlink(id);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function unlink(id: string): void {
|
|
148
|
+
try {
|
|
149
|
+
fs.rmSync(entryPath(id), { force: true });
|
|
150
|
+
} catch {
|
|
151
|
+
// ignore
|
|
152
|
+
}
|
|
153
|
+
}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { Readability } from "@mozilla/readability";
|
|
2
|
+
import { parseHTML } from "linkedom";
|
|
3
|
+
import { htmlToMarkdown } from "./html-to-markdown.ts";
|
|
4
|
+
|
|
5
|
+
export interface ExtractedPage {
|
|
6
|
+
title: string;
|
|
7
|
+
url: string;
|
|
8
|
+
markdown: string;
|
|
9
|
+
byline?: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/** Page chrome to drop when Readability cannot identify an article. */
|
|
13
|
+
const CHROME_SELECTOR = "nav, header, footer, aside, form, [role=navigation], [role=banner], [role=contentinfo]";
|
|
14
|
+
|
|
15
|
+
const TITLE_META = [
|
|
16
|
+
'meta[property="og:title"]',
|
|
17
|
+
'meta[name="og:title"]',
|
|
18
|
+
'meta[name="twitter:title"]',
|
|
19
|
+
'meta[name="title"]',
|
|
20
|
+
];
|
|
21
|
+
|
|
22
|
+
function clean(value: string | null | undefined): string {
|
|
23
|
+
return (value ?? "").replace(/\s+/g, " ").trim();
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Resolve a page title from the most reliable source available.
|
|
28
|
+
*
|
|
29
|
+
* `document.title` is deliberately not trusted: linkedom returns an empty string
|
|
30
|
+
* for it on real-world pages whose `<head>` it parses loosely, even when a
|
|
31
|
+
* `<title>` element is present and reachable via `querySelector`.
|
|
32
|
+
*/
|
|
33
|
+
export function extractTitle(document: Document | ReturnType<typeof parseHTML>["document"]): string {
|
|
34
|
+
const tag = clean(document.querySelector?.("title")?.textContent);
|
|
35
|
+
if (tag) return tag;
|
|
36
|
+
for (const selector of TITLE_META) {
|
|
37
|
+
const meta = clean(document.querySelector?.(selector)?.getAttribute("content"));
|
|
38
|
+
if (meta) return meta;
|
|
39
|
+
}
|
|
40
|
+
return clean(document.querySelector?.("h1")?.textContent);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function extractReadable(html: string, url: string): ExtractedPage {
|
|
44
|
+
const { document } = parseHTML(html);
|
|
45
|
+
let title = extractTitle(document);
|
|
46
|
+
|
|
47
|
+
let markdown = "";
|
|
48
|
+
let byline: string | undefined;
|
|
49
|
+
try {
|
|
50
|
+
// Readability mutates the document it is given, so parse a second copy.
|
|
51
|
+
const article = new Readability(parseHTML(html).document as unknown as Document, { charThreshold: 80 }).parse();
|
|
52
|
+
if (article?.content) {
|
|
53
|
+
const parsed = parseHTML(`<div>${article.content}</div>`);
|
|
54
|
+
markdown = htmlToMarkdown(parsed.document.body);
|
|
55
|
+
// Readability's title is cleaner when it finds one, but it is often empty.
|
|
56
|
+
title = clean(article.title) || title;
|
|
57
|
+
byline = clean(article.byline) || undefined;
|
|
58
|
+
}
|
|
59
|
+
} catch {
|
|
60
|
+
markdown = "";
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
if (!markdown.trim()) {
|
|
64
|
+
for (const node of Array.from(document.querySelectorAll(CHROME_SELECTOR))) {
|
|
65
|
+
node.remove?.();
|
|
66
|
+
}
|
|
67
|
+
markdown = htmlToMarkdown(document);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
if (!markdown.trim()) {
|
|
71
|
+
markdown = clean(document.body?.textContent ?? document.textContent);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return { title: title || url, url, markdown, byline };
|
|
75
|
+
}
|
package/src/web/fetch.ts
ADDED
|
@@ -0,0 +1,167 @@
|
|
|
1
|
+
import type { ResolvedWebConfig } from "../config.ts";
|
|
2
|
+
import { capText, errorMessage, isAbortError, PiEssentialsError } from "../errors.ts";
|
|
3
|
+
import { SsrfError } from "../security/ssrf.ts";
|
|
4
|
+
import { getCache, putCache, sliceMarkdown } from "./cache.ts";
|
|
5
|
+
import { extractReadable } from "./extract.ts";
|
|
6
|
+
import { safeFetch } from "./http.ts";
|
|
7
|
+
|
|
8
|
+
export interface FetchPageResult {
|
|
9
|
+
url: string;
|
|
10
|
+
title: string;
|
|
11
|
+
markdown: string;
|
|
12
|
+
truncated: boolean;
|
|
13
|
+
cacheId: string;
|
|
14
|
+
totalChars: number;
|
|
15
|
+
/** Character offset the returned slice starts at. */
|
|
16
|
+
offset: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Content types that carry no readable text. */
|
|
20
|
+
const BINARY_TYPE = /^(?:image|audio|video|font)\/|^application\/(?:pdf|zip|gzip|octet-stream|x-|vnd\.(?!.*\+(?:json|xml)))/i;
|
|
21
|
+
|
|
22
|
+
/** Content types that are already plain text and must not go through Readability. */
|
|
23
|
+
const PLAIN_TYPE = /(?:^text\/(?:plain|markdown|csv|x-markdown)|json|yaml|javascript|typescript|\+xml$|^text\/xml)/i;
|
|
24
|
+
|
|
25
|
+
export async function fetchPage(
|
|
26
|
+
rawUrl: string,
|
|
27
|
+
config: ResolvedWebConfig["fetch"],
|
|
28
|
+
signal?: AbortSignal,
|
|
29
|
+
slice?: { offset?: number; limit?: number },
|
|
30
|
+
allowedHosts?: ReadonlySet<string>,
|
|
31
|
+
): Promise<FetchPageResult> {
|
|
32
|
+
try {
|
|
33
|
+
const response = await safeFetch(rawUrl, {
|
|
34
|
+
timeoutMs: config.timeoutMs,
|
|
35
|
+
maxBytes: config.maxBytes,
|
|
36
|
+
signal,
|
|
37
|
+
allowedHosts,
|
|
38
|
+
});
|
|
39
|
+
const contentType = response.contentType.toLowerCase();
|
|
40
|
+
if (BINARY_TYPE.test(contentType)) {
|
|
41
|
+
throw new PiEssentialsError(
|
|
42
|
+
`${response.url} returned binary content (${contentType || "unknown type"}); web_fetch only reads text pages.`,
|
|
43
|
+
"WEB_FETCH_BINARY",
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
let markdown: string;
|
|
48
|
+
let title = response.url;
|
|
49
|
+
if (PLAIN_TYPE.test(contentType)) {
|
|
50
|
+
markdown = response.text();
|
|
51
|
+
} else {
|
|
52
|
+
const extracted = extractReadable(response.text(), response.url);
|
|
53
|
+
markdown = extracted.markdown;
|
|
54
|
+
title = extracted.title;
|
|
55
|
+
if (extracted.byline) markdown = `${extracted.byline}\n\n${markdown}`;
|
|
56
|
+
}
|
|
57
|
+
if (response.truncated) {
|
|
58
|
+
markdown += `\n\n[Download stopped at ${config.maxBytes} bytes; the page may continue past this point.]`;
|
|
59
|
+
}
|
|
60
|
+
if (!markdown.trim()) {
|
|
61
|
+
markdown = "(The page returned no readable text. It may require JavaScript.)";
|
|
62
|
+
}
|
|
63
|
+
return finish(response.url, title, markdown, config, slice);
|
|
64
|
+
} catch (error) {
|
|
65
|
+
if (isAbortError(error)) {
|
|
66
|
+
throw new PiEssentialsError(`Fetching ${rawUrl} was cancelled or timed out.`, "WEB_FETCH_TIMEOUT", true);
|
|
67
|
+
}
|
|
68
|
+
if (error instanceof SsrfError) throw error;
|
|
69
|
+
if (error instanceof PiEssentialsError && error.code === "WEB_FETCH_BINARY") throw error;
|
|
70
|
+
if (config.jinaFallback) {
|
|
71
|
+
try {
|
|
72
|
+
return await fetchViaJina(rawUrl, config, signal, slice, allowedHosts);
|
|
73
|
+
} catch (fallbackError) {
|
|
74
|
+
if (signal?.aborted || isAbortError(fallbackError)) {
|
|
75
|
+
throw new PiEssentialsError(`Fetching ${rawUrl} was cancelled or timed out.`, "WEB_FETCH_TIMEOUT", true);
|
|
76
|
+
}
|
|
77
|
+
throw new PiEssentialsError(
|
|
78
|
+
`Failed to fetch ${rawUrl}: ${errorMessage(error)}. Jina fallback also failed: ${errorMessage(fallbackError)}`,
|
|
79
|
+
"WEB_FETCH_FAILED",
|
|
80
|
+
true,
|
|
81
|
+
);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
if (error instanceof PiEssentialsError) throw error;
|
|
85
|
+
throw new PiEssentialsError(`Failed to fetch ${rawUrl}: ${errorMessage(error)}`, "WEB_FETCH_FAILED", true);
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
async function fetchViaJina(
|
|
90
|
+
rawUrl: string,
|
|
91
|
+
config: ResolvedWebConfig["fetch"],
|
|
92
|
+
signal: AbortSignal | undefined,
|
|
93
|
+
slice: { offset?: number; limit?: number } | undefined,
|
|
94
|
+
allowedHosts: ReadonlySet<string> | undefined,
|
|
95
|
+
): Promise<FetchPageResult> {
|
|
96
|
+
const response = await safeFetch(`https://r.jina.ai/${rawUrl}`, {
|
|
97
|
+
timeoutMs: config.timeoutMs,
|
|
98
|
+
maxBytes: config.maxBytes,
|
|
99
|
+
signal,
|
|
100
|
+
allowedHosts,
|
|
101
|
+
headers: { accept: "text/plain" },
|
|
102
|
+
});
|
|
103
|
+
const markdown = response.text() || "(Jina returned no readable text.)";
|
|
104
|
+
return finish(rawUrl, rawUrl, markdown, config, slice);
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function finish(
|
|
108
|
+
url: string,
|
|
109
|
+
title: string,
|
|
110
|
+
markdown: string,
|
|
111
|
+
config: ResolvedWebConfig["fetch"],
|
|
112
|
+
slice: { offset?: number; limit?: number } | undefined,
|
|
113
|
+
): FetchPageResult {
|
|
114
|
+
const id = putCache(url, title, markdown);
|
|
115
|
+
const offset = Math.max(0, Math.floor(slice?.offset ?? 0));
|
|
116
|
+
// Never ask for more than maxChars, so capText cannot add a second truncation
|
|
117
|
+
// note on top of the slice note below.
|
|
118
|
+
const limit = Math.min(config.maxChars, slice?.limit !== undefined ? Math.max(1, Math.floor(slice.limit)) : config.maxChars);
|
|
119
|
+
const window = sliceMarkdown(markdown, offset, limit);
|
|
120
|
+
const capped = capText(window.text, config.maxChars);
|
|
121
|
+
const shown = capped.text.length;
|
|
122
|
+
const truncated = capped.truncated || window.start + window.text.length < window.total || window.start > 0;
|
|
123
|
+
return {
|
|
124
|
+
url,
|
|
125
|
+
title,
|
|
126
|
+
markdown: truncated ? `${capped.text}\n\n${sliceNote(window.start, shown, window.total, id)}` : capped.text,
|
|
127
|
+
truncated,
|
|
128
|
+
cacheId: id,
|
|
129
|
+
totalChars: window.total,
|
|
130
|
+
offset: window.start,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function sliceNote(start: number, shown: number, total: number, id: string): string {
|
|
135
|
+
const end = Math.min(total, start + shown);
|
|
136
|
+
if (end >= total) {
|
|
137
|
+
return start === 0 ? "" : `[Showing characters ${start}-${end} of ${total}. End of cached page.]`;
|
|
138
|
+
}
|
|
139
|
+
return `[Showing characters ${start}-${end} of ${total}. Continue with web_fetch({ cacheId: "${id}", offset: ${end} }).]`;
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
export function readCached(idOrUrl: string, offset?: number, limit?: number, maxChars = 32_000): FetchPageResult {
|
|
143
|
+
const entry = getCache(idOrUrl);
|
|
144
|
+
if (!entry) {
|
|
145
|
+
throw new PiEssentialsError(
|
|
146
|
+
`No cached page for "${idOrUrl}". It may have expired; fetch the url again.`,
|
|
147
|
+
"WEB_CACHE_MISS",
|
|
148
|
+
);
|
|
149
|
+
}
|
|
150
|
+
const window = sliceMarkdown(entry.markdown, offset ?? 0, Math.min(maxChars, limit ?? maxChars));
|
|
151
|
+
const capped = capText(window.text, maxChars);
|
|
152
|
+
const shown = capped.text.length;
|
|
153
|
+
const truncated = capped.truncated || window.start + shown < window.total || window.start > 0;
|
|
154
|
+
return {
|
|
155
|
+
url: entry.url,
|
|
156
|
+
title: entry.title,
|
|
157
|
+
markdown: truncated ? `${capped.text}\n\n${sliceNote(window.start, shown, window.total, entry.id)}` : capped.text,
|
|
158
|
+
truncated,
|
|
159
|
+
cacheId: entry.id,
|
|
160
|
+
totalChars: window.total,
|
|
161
|
+
offset: window.start,
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
export function formatPage(result: FetchPageResult): string {
|
|
166
|
+
return `# ${result.title}\nSource: ${result.url}\nCache-Id: ${result.cacheId}\n\n${result.markdown}`;
|
|
167
|
+
}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
/** Elements whose content is never useful as prose. */
|
|
2
|
+
const SKIP_TAGS = new Set([
|
|
3
|
+
"SCRIPT",
|
|
4
|
+
"STYLE",
|
|
5
|
+
"NOSCRIPT",
|
|
6
|
+
"TEMPLATE",
|
|
7
|
+
"SVG",
|
|
8
|
+
"IFRAME",
|
|
9
|
+
"CANVAS",
|
|
10
|
+
"OBJECT",
|
|
11
|
+
"EMBED",
|
|
12
|
+
"HEAD",
|
|
13
|
+
"META",
|
|
14
|
+
"LINK",
|
|
15
|
+
"AUDIO",
|
|
16
|
+
"VIDEO",
|
|
17
|
+
]);
|
|
18
|
+
|
|
19
|
+
/** Elements rendered inside the current line rather than as their own block. */
|
|
20
|
+
const INLINE_TAGS = new Set([
|
|
21
|
+
"A",
|
|
22
|
+
"ABBR",
|
|
23
|
+
"B",
|
|
24
|
+
"BDI",
|
|
25
|
+
"BDO",
|
|
26
|
+
"BIG",
|
|
27
|
+
"BR",
|
|
28
|
+
"CITE",
|
|
29
|
+
"CODE",
|
|
30
|
+
"DATA",
|
|
31
|
+
"DEL",
|
|
32
|
+
"DFN",
|
|
33
|
+
"EM",
|
|
34
|
+
"I",
|
|
35
|
+
"IMG",
|
|
36
|
+
"INS",
|
|
37
|
+
"KBD",
|
|
38
|
+
"LABEL",
|
|
39
|
+
"MARK",
|
|
40
|
+
"Q",
|
|
41
|
+
"S",
|
|
42
|
+
"SAMP",
|
|
43
|
+
"SMALL",
|
|
44
|
+
"SPAN",
|
|
45
|
+
"STRIKE",
|
|
46
|
+
"STRONG",
|
|
47
|
+
"SUB",
|
|
48
|
+
"SUP",
|
|
49
|
+
"TIME",
|
|
50
|
+
"TT",
|
|
51
|
+
"U",
|
|
52
|
+
"VAR",
|
|
53
|
+
"WBR",
|
|
54
|
+
]);
|
|
55
|
+
|
|
56
|
+
const MAX_DEPTH = 64;
|
|
57
|
+
|
|
58
|
+
interface Context {
|
|
59
|
+
out: string[];
|
|
60
|
+
pending: string[];
|
|
61
|
+
listDepth: number;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function collapse(value: string): string {
|
|
65
|
+
return value.replace(/\s+/g, " ");
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function textOf(node: { textContent?: string | null }): string {
|
|
69
|
+
return collapse(node.textContent ?? "").trim();
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
export function htmlToMarkdown(root: ParentNode | null | undefined): string {
|
|
73
|
+
if (!root) return "";
|
|
74
|
+
const ctx: Context = { out: [], pending: [], listDepth: 0 };
|
|
75
|
+
walk(pickRoot(root), ctx, 0);
|
|
76
|
+
flush(ctx);
|
|
77
|
+
return ctx.out
|
|
78
|
+
.join("\n\n")
|
|
79
|
+
.replace(/[ \t]+$/gm, "")
|
|
80
|
+
.replace(/\n{3,}/g, "\n\n")
|
|
81
|
+
.trim();
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function pickRoot(root: ParentNode): ParentNode {
|
|
85
|
+
const body = (root as ParentNode & { body?: ParentNode | null }).body;
|
|
86
|
+
if (body && (body.childNodes?.length ?? 0) > 0) return body;
|
|
87
|
+
if ((root.childNodes?.length ?? 0) > 0) return root;
|
|
88
|
+
return body ?? root;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function flush(ctx: Context): void {
|
|
92
|
+
const text = collapse(ctx.pending.join("")).trim();
|
|
93
|
+
ctx.pending.length = 0;
|
|
94
|
+
if (text) ctx.out.push(text);
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
function pushBlock(ctx: Context, block: string): void {
|
|
98
|
+
flush(ctx);
|
|
99
|
+
const trimmed = block.replace(/\s+$/, "");
|
|
100
|
+
if (trimmed.trim()) ctx.out.push(trimmed);
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
function walk(node: ParentNode, ctx: Context, depth: number): void {
|
|
104
|
+
if (depth > MAX_DEPTH) return;
|
|
105
|
+
for (const child of Array.from(node.childNodes ?? [])) {
|
|
106
|
+
handle(child as ChildNode, ctx, depth);
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function handle(node: ChildNode, ctx: Context, depth: number): void {
|
|
111
|
+
if (node.nodeType === 3) {
|
|
112
|
+
const text = collapse(node.textContent ?? "");
|
|
113
|
+
if (text.trim()) ctx.pending.push(text);
|
|
114
|
+
return;
|
|
115
|
+
}
|
|
116
|
+
if (node.nodeType !== 1) return;
|
|
117
|
+
|
|
118
|
+
const el = node as Element;
|
|
119
|
+
const tag = el.tagName.toUpperCase();
|
|
120
|
+
if (SKIP_TAGS.has(tag)) return;
|
|
121
|
+
|
|
122
|
+
if (INLINE_TAGS.has(tag)) {
|
|
123
|
+
ctx.pending.push(inline(el, depth + 1));
|
|
124
|
+
return;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
if (/^H[1-6]$/.test(tag)) {
|
|
128
|
+
const heading = textOf(el);
|
|
129
|
+
if (heading) pushBlock(ctx, `${"#".repeat(Number(tag[1]))} ${heading}`);
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
if (tag === "HR") {
|
|
133
|
+
pushBlock(ctx, "---");
|
|
134
|
+
return;
|
|
135
|
+
}
|
|
136
|
+
if (tag === "PRE") {
|
|
137
|
+
pushBlock(ctx, codeBlock(el));
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
if (tag === "UL" || tag === "OL") {
|
|
141
|
+
pushBlock(ctx, list(el, tag === "OL", ctx.listDepth, depth + 1));
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (tag === "LI") {
|
|
145
|
+
// A stray <li> outside a list still reads better as a bullet.
|
|
146
|
+
pushBlock(ctx, `- ${inline(el, depth + 1)}`);
|
|
147
|
+
return;
|
|
148
|
+
}
|
|
149
|
+
if (tag === "TABLE") {
|
|
150
|
+
pushBlock(ctx, table(el, depth + 1));
|
|
151
|
+
return;
|
|
152
|
+
}
|
|
153
|
+
if (tag === "BLOCKQUOTE") {
|
|
154
|
+
const inner = htmlToMarkdown(el);
|
|
155
|
+
if (inner.trim()) {
|
|
156
|
+
pushBlock(
|
|
157
|
+
ctx,
|
|
158
|
+
inner
|
|
159
|
+
.split("\n")
|
|
160
|
+
.map((line) => (line ? `> ${line}` : ">"))
|
|
161
|
+
.join("\n"),
|
|
162
|
+
);
|
|
163
|
+
}
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
if (tag === "DT") {
|
|
167
|
+
pushBlock(ctx, `**${inline(el, depth + 1)}**`);
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
if (tag === "DD") {
|
|
171
|
+
pushBlock(ctx, ` ${inline(el, depth + 1)}`);
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
// Any other block-level container: keep its children in separate blocks.
|
|
176
|
+
flush(ctx);
|
|
177
|
+
walk(el, ctx, depth + 1);
|
|
178
|
+
flush(ctx);
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
function codeBlock(el: Element): string {
|
|
182
|
+
const code = el.querySelector?.("code") ?? null;
|
|
183
|
+
const className = code?.getAttribute("class") ?? el.getAttribute("class") ?? "";
|
|
184
|
+
const language = /(?:language|lang)-([\w+#-]+)/i.exec(className)?.[1] ?? "";
|
|
185
|
+
const body = (el.textContent ?? "").replace(/\n+$/, "");
|
|
186
|
+
if (!body.trim()) return "";
|
|
187
|
+
return `\`\`\`${language}\n${body}\n\`\`\``;
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
function list(el: Element, ordered: boolean, listDepth: number, depth: number): string {
|
|
191
|
+
if (depth > MAX_DEPTH) return "";
|
|
192
|
+
const indent = " ".repeat(listDepth);
|
|
193
|
+
const start = Number.parseInt(el.getAttribute("start") ?? "1", 10);
|
|
194
|
+
const lines: string[] = [];
|
|
195
|
+
let index = Number.isFinite(start) ? start : 1;
|
|
196
|
+
|
|
197
|
+
for (const child of Array.from(el.children ?? [])) {
|
|
198
|
+
const item = child as Element;
|
|
199
|
+
if (item.tagName.toUpperCase() !== "LI") continue;
|
|
200
|
+
const marker = ordered ? `${index++}.` : "-";
|
|
201
|
+
const nested: string[] = [];
|
|
202
|
+
const own: string[] = [];
|
|
203
|
+
|
|
204
|
+
for (const part of Array.from(item.childNodes ?? [])) {
|
|
205
|
+
const partTag = part.nodeType === 1 ? (part as Element).tagName.toUpperCase() : "";
|
|
206
|
+
if (partTag === "UL" || partTag === "OL") {
|
|
207
|
+
nested.push(list(part as Element, partTag === "OL", listDepth + 1, depth + 1));
|
|
208
|
+
} else if (part.nodeType === 3) {
|
|
209
|
+
own.push(collapse(part.textContent ?? ""));
|
|
210
|
+
} else if (part.nodeType === 1) {
|
|
211
|
+
own.push(INLINE_TAGS.has(partTag) ? inline(part as Element, depth + 1) : `${inline(part as Element, depth + 1)} `);
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
215
|
+
const text = collapse(own.join("")).trim();
|
|
216
|
+
lines.push(`${indent}${marker} ${text}`.replace(/\s+$/, ""));
|
|
217
|
+
for (const block of nested) {
|
|
218
|
+
if (block.trim()) lines.push(block);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
return lines.join("\n");
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function table(el: Element, depth: number): string {
|
|
225
|
+
if (depth > MAX_DEPTH) return "";
|
|
226
|
+
const rows = Array.from(el.querySelectorAll?.("tr") ?? []) as Element[];
|
|
227
|
+
if (rows.length === 0) return "";
|
|
228
|
+
|
|
229
|
+
const grid = rows.map((row) =>
|
|
230
|
+
(Array.from(row.children ?? []) as Element[])
|
|
231
|
+
.filter((cell) => ["TD", "TH"].includes(cell.tagName.toUpperCase()))
|
|
232
|
+
.map((cell) => inline(cell, depth + 1).replace(/\|/g, "\\|") || " "),
|
|
233
|
+
);
|
|
234
|
+
const width = Math.max(...grid.map((row) => row.length));
|
|
235
|
+
if (width === 0) return "";
|
|
236
|
+
|
|
237
|
+
const padded = grid.map((row) => [...row, ...new Array<string>(width - row.length).fill(" ")]);
|
|
238
|
+
const headerIsLabels = rows[0].querySelector?.("th") !== null;
|
|
239
|
+
const header = headerIsLabels ? padded[0] : new Array<string>(width).fill(" ");
|
|
240
|
+
const body = headerIsLabels ? padded.slice(1) : padded;
|
|
241
|
+
|
|
242
|
+
const lines = [`| ${header.join(" | ")} |`, `| ${new Array<string>(width).fill("---").join(" | ")} |`];
|
|
243
|
+
for (const row of body) lines.push(`| ${row.join(" | ")} |`);
|
|
244
|
+
return lines.join("\n");
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function inline(el: Element, depth: number): string {
|
|
248
|
+
if (depth > MAX_DEPTH) return textOf(el);
|
|
249
|
+
const tag = el.tagName.toUpperCase();
|
|
250
|
+
|
|
251
|
+
if (tag === "BR" || tag === "WBR") return " ";
|
|
252
|
+
if (tag === "IMG") {
|
|
253
|
+
const src = el.getAttribute("src") ?? "";
|
|
254
|
+
const alt = collapse(el.getAttribute("alt") ?? "").trim();
|
|
255
|
+
return src ? `` : alt;
|
|
256
|
+
}
|
|
257
|
+
if (tag === "CODE" || tag === "KBD" || tag === "SAMP") {
|
|
258
|
+
const code = textOf(el);
|
|
259
|
+
return code ? `\`${code}\`` : "";
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const parts: string[] = [];
|
|
263
|
+
for (const child of Array.from(el.childNodes ?? [])) {
|
|
264
|
+
if (child.nodeType === 3) {
|
|
265
|
+
parts.push(collapse(child.textContent ?? ""));
|
|
266
|
+
continue;
|
|
267
|
+
}
|
|
268
|
+
if (child.nodeType !== 1) continue;
|
|
269
|
+
const nested = child as Element;
|
|
270
|
+
if (SKIP_TAGS.has(nested.tagName.toUpperCase())) continue;
|
|
271
|
+
parts.push(inline(nested, depth + 1));
|
|
272
|
+
}
|
|
273
|
+
const content = collapse(parts.join("")).trim();
|
|
274
|
+
|
|
275
|
+
if (tag === "A") {
|
|
276
|
+
const href = el.getAttribute("href") ?? "";
|
|
277
|
+
if (!href || href.startsWith("javascript:")) return content;
|
|
278
|
+
return content ? `[${content}](${href})` : href;
|
|
279
|
+
}
|
|
280
|
+
if (tag === "STRONG" || tag === "B") return content ? `**${content}**` : "";
|
|
281
|
+
if (tag === "EM" || tag === "I") return content ? `_${content}_` : "";
|
|
282
|
+
if (tag === "DEL" || tag === "S" || tag === "STRIKE") return content ? `~~${content}~~` : "";
|
|
283
|
+
return content;
|
|
284
|
+
}
|