@zosmaai/pi-llm-wiki 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/codeql/codeql-config.yml +12 -0
- package/.github/workflows/ci.yml +1 -1
- package/.github/workflows/codeql.yml +1 -1
- package/.github/workflows/release.yml +8 -0
- package/README.md +24 -0
- package/docs/configuration.md +6 -0
- package/extensions/llm-wiki/lib/source-packet.ts +104 -22
- package/package.json +1 -1
- package/prompts/wiki-digest.md +5 -1
- package/prompts/wiki-discover.md +5 -1
- package/prompts/wiki-ingest.md +5 -1
- package/prompts/wiki-init.md +5 -1
- package/prompts/wiki-lint.md +5 -1
- package/prompts/wiki-query.md +5 -1
- package/prompts/wiki-run.md +5 -1
- package/prompts/wiki-status.md +1 -1
- package/test/llm-wiki.test.ts +191 -0
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
name: "CodeQL Config"
|
|
2
|
+
|
|
3
|
+
queries:
|
|
4
|
+
- uses: security-and-quality
|
|
5
|
+
|
|
6
|
+
# Suppress false positives for temporary file creation.
|
|
7
|
+
# This is a single-user CLI tool — TOCTOU race conditions on temp directories
|
|
8
|
+
# are not a valid threat model. The alerts only fire because tests pass
|
|
9
|
+
# os.tmpdir() paths into production capture functions.
|
|
10
|
+
query-filters:
|
|
11
|
+
- exclude:
|
|
12
|
+
id: js/insecure-temporary-file
|
package/.github/workflows/ci.yml
CHANGED
|
@@ -43,6 +43,14 @@ jobs:
|
|
|
43
43
|
node -e "const p=require('./package.json'); p.version='${{ steps.version.outputs.VERSION }}'; require('fs').writeFileSync('./package.json', JSON.stringify(p, null, 2)+'\n')"
|
|
44
44
|
npx biome check --fix package.json 2>&1
|
|
45
45
|
|
|
46
|
+
- name: Commit version bump back to main
|
|
47
|
+
run: |
|
|
48
|
+
git config user.name "github-actions[bot]"
|
|
49
|
+
git config user.email "github-actions[bot]@users.noreply.github.com"
|
|
50
|
+
git add package.json
|
|
51
|
+
git commit -m "chore(release): v${{ steps.version.outputs.VERSION }}"
|
|
52
|
+
git push origin HEAD:main
|
|
53
|
+
|
|
46
54
|
- name: Generate Release Notes
|
|
47
55
|
id: notes
|
|
48
56
|
run: |
|
package/README.md
CHANGED
|
@@ -42,6 +42,14 @@ Drop sources into `raw/`, then:
|
|
|
42
42
|
| `wiki_log_event` | Record custom event |
|
|
43
43
|
| `wiki_watch` | Schedule auto-updates |
|
|
44
44
|
|
|
45
|
+
## Features
|
|
46
|
+
|
|
47
|
+
- **Configurable PDF extraction** — MarkItDown timeout adjustable via `WIKI_MARKITDOWN_TIMEOUT_MS` env var
|
|
48
|
+
- **Smart content detection** — PDF bytes sniffed even from non-`.pdf` URLs, never written as markdown
|
|
49
|
+
- **Original artifacts preserved** — URL captures save the fetched payload under `original/source.*`
|
|
50
|
+
- **Clickable source links** — Captured URLs render as clickable Markdown links in source pages
|
|
51
|
+
- **Reliable prompt forwarding** — Slash commands properly forward user arguments to the model
|
|
52
|
+
|
|
45
53
|
## Architecture
|
|
46
54
|
|
|
47
55
|
Four layers with clear ownership:
|
|
@@ -67,6 +75,22 @@ Read [docs/architecture.md](docs/architecture.md) for details.
|
|
|
67
75
|
|
|
68
76
|
See [CONTRIBUTING.md](CONTRIBUTING.md).
|
|
69
77
|
|
|
78
|
+
## Star History
|
|
79
|
+
|
|
80
|
+
[](https://star-history.com/#zosmaai/pi-llm-wiki&Date)
|
|
81
|
+
|
|
82
|
+
## Contributors
|
|
83
|
+
|
|
84
|
+
<a href="https://github.com/zosmaai/pi-llm-wiki/graphs/contributors">
|
|
85
|
+
<img src="https://contrib.rocks/image?repo=zosmaai/pi-llm-wiki" alt="Contributors" />
|
|
86
|
+
</a>
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
<div align="center">
|
|
91
|
+
<sub>Built with ❤️ by <a href="https://github.com/zosmaai">zosmaai</a></sub>
|
|
92
|
+
</div>
|
|
93
|
+
|
|
70
94
|
## License
|
|
71
95
|
|
|
72
96
|
MIT
|
package/docs/configuration.md
CHANGED
|
@@ -23,6 +23,12 @@ Wiki configuration lives in `.wiki/config.json`.
|
|
|
23
23
|
| `auto_fix_lint` | false | Auto-fix lint issues |
|
|
24
24
|
| `batch_ingest_size` | 3 | Sources processed per ingest batch |
|
|
25
25
|
|
|
26
|
+
## Environment Variables
|
|
27
|
+
|
|
28
|
+
| Variable | Default | Description |
|
|
29
|
+
| ----------------------------- | ------- | ----------------------------------------------- |
|
|
30
|
+
| `WIKI_MARKITDOWN_TIMEOUT_MS` | 180000 | Timeout (ms) for MarkItDown PDF/text extraction |
|
|
31
|
+
|
|
26
32
|
## Page Frontmatter
|
|
27
33
|
|
|
28
34
|
```yaml
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
|
-
import { join } from "node:path";
|
|
2
|
+
import { extname, join } from "node:path";
|
|
3
3
|
import type { ExtensionAPI } from "@mariozechner/pi-coding-agent";
|
|
4
4
|
import { appendEvent } from "./metadata.js";
|
|
5
5
|
import { type VaultPaths, exec, fmtDate, nextSourceId, readText, writeJson } from "./utils.js";
|
|
@@ -22,6 +22,69 @@ export interface CaptureResult {
|
|
|
22
22
|
extracted: string;
|
|
23
23
|
}
|
|
24
24
|
|
|
25
|
+
const DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000;
|
|
26
|
+
const DEFAULT_CURL_TIMEOUT_SECONDS = 30;
|
|
27
|
+
|
|
28
|
+
function markitdownTimeoutMs(): number {
|
|
29
|
+
return positiveIntegerFromEnv("WIKI_MARKITDOWN_TIMEOUT_MS", DEFAULT_MARKITDOWN_TIMEOUT_MS);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function positiveIntegerFromEnv(name: string, fallback: number): number {
|
|
33
|
+
const raw = process.env[name];
|
|
34
|
+
if (!raw) return fallback;
|
|
35
|
+
const parsed = Number.parseInt(raw, 10);
|
|
36
|
+
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function isPdfUrl(url: string): boolean {
|
|
40
|
+
try {
|
|
41
|
+
return new URL(url).pathname.toLowerCase().endsWith(".pdf");
|
|
42
|
+
} catch {
|
|
43
|
+
return url.toLowerCase().split(/[?#]/, 1)[0].endsWith(".pdf");
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function looksLikePdf(content: string): boolean {
|
|
48
|
+
return content.trimStart().startsWith("%PDF-");
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function pdfExtractionFailureMessage(source: string): string {
|
|
52
|
+
return `_PDF content could not be converted to markdown from ${source}. Try increasing WIKI_MARKITDOWN_TIMEOUT_MS._\n`;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const URL_ORIGINAL_EXTENSIONS = new Set([".html", ".htm", ".md", ".pdf", ".txt", ".xml", ".json"]);
|
|
56
|
+
|
|
57
|
+
function originalFileNameForUrl(url: string): string {
|
|
58
|
+
try {
|
|
59
|
+
const parsed = new URL(url);
|
|
60
|
+
const ext = extname(parsed.pathname).toLowerCase();
|
|
61
|
+
if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
|
|
62
|
+
} catch {
|
|
63
|
+
const path = url.split(/[?#]/, 1)[0] ?? "";
|
|
64
|
+
const ext = extname(path).toLowerCase();
|
|
65
|
+
if (URL_ORIGINAL_EXTENSIONS.has(ext)) return `source${ext}`;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return "source.html";
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
async function preserveUrlOriginal(
|
|
72
|
+
pi: ExtensionAPI,
|
|
73
|
+
packetPath: string,
|
|
74
|
+
url: string,
|
|
75
|
+
signal?: AbortSignal,
|
|
76
|
+
): Promise<void> {
|
|
77
|
+
const originalPath = join(packetPath, "original", originalFileNameForUrl(url));
|
|
78
|
+
try {
|
|
79
|
+
await exec(pi, "curl", ["-sL", "--max-time", "30", "-o", originalPath, url], {
|
|
80
|
+
signal,
|
|
81
|
+
timeout: 35_000,
|
|
82
|
+
});
|
|
83
|
+
} catch {
|
|
84
|
+
// Preserve best-effort extraction behavior even when the original artifact cannot be saved.
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
25
88
|
/** Capture a URL into a source packet. */
|
|
26
89
|
export async function captureUrl(
|
|
27
90
|
pi: ExtensionAPI,
|
|
@@ -35,11 +98,14 @@ export async function captureUrl(
|
|
|
35
98
|
mkdirSync(join(packetPath, "original"), { recursive: true });
|
|
36
99
|
mkdirSync(join(packetPath, "attachments"), { recursive: true });
|
|
37
100
|
|
|
101
|
+
await preserveUrlOriginal(pi, packetPath, url, signal);
|
|
102
|
+
|
|
38
103
|
// Try to fetch and extract content
|
|
39
104
|
let extracted = "";
|
|
40
105
|
let title = url;
|
|
106
|
+
const isPdf = isPdfUrl(url);
|
|
41
107
|
|
|
42
|
-
// Try
|
|
108
|
+
// Try MarkItDown first.
|
|
43
109
|
const markitdown = await exec(
|
|
44
110
|
pi,
|
|
45
111
|
"sh",
|
|
@@ -53,7 +119,7 @@ export async function captureUrl(
|
|
|
53
119
|
pi,
|
|
54
120
|
"sh",
|
|
55
121
|
["-c", `uvx --from 'markitdown[pdf]' markitdown "${url}" 2>/dev/null || echo ""`],
|
|
56
|
-
{ signal, timeout:
|
|
122
|
+
{ signal, timeout: markitdownTimeoutMs() },
|
|
57
123
|
);
|
|
58
124
|
if (mdResult.stdout.trim()) {
|
|
59
125
|
extracted = mdResult.stdout;
|
|
@@ -66,21 +132,35 @@ export async function captureUrl(
|
|
|
66
132
|
}
|
|
67
133
|
}
|
|
68
134
|
|
|
69
|
-
// Fallback: try fetch_content equivalent via curl
|
|
135
|
+
// Fallback: try fetch_content equivalent via curl for text/html sources.
|
|
136
|
+
// Do not write binary PDF bytes into extracted.md when PDF conversion fails.
|
|
70
137
|
if (!extracted) {
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
138
|
+
if (isPdf) {
|
|
139
|
+
extracted = pdfExtractionFailureMessage(url);
|
|
140
|
+
} else {
|
|
141
|
+
try {
|
|
142
|
+
const curlResult = await exec(
|
|
143
|
+
pi,
|
|
144
|
+
"curl",
|
|
145
|
+
["-sL", "--max-time", String(DEFAULT_CURL_TIMEOUT_SECONDS), url],
|
|
146
|
+
{
|
|
147
|
+
signal,
|
|
148
|
+
timeout: (DEFAULT_CURL_TIMEOUT_SECONDS + 5) * 1_000,
|
|
149
|
+
},
|
|
150
|
+
);
|
|
151
|
+
if (curlResult.stdout) {
|
|
152
|
+
if (looksLikePdf(curlResult.stdout)) {
|
|
153
|
+
extracted = pdfExtractionFailureMessage(url);
|
|
154
|
+
} else {
|
|
155
|
+
extracted = curlResult.stdout;
|
|
156
|
+
// Try to extract title from HTML
|
|
157
|
+
const titleMatch = extracted.match(/<title>([^<]*)<\/title>/i);
|
|
158
|
+
if (titleMatch) title = titleMatch[1].trim();
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
} catch {
|
|
162
|
+
// curl failed too
|
|
81
163
|
}
|
|
82
|
-
} catch {
|
|
83
|
-
// curl failed too
|
|
84
164
|
}
|
|
85
165
|
}
|
|
86
166
|
|
|
@@ -126,12 +206,13 @@ export async function captureFile(
|
|
|
126
206
|
mkdirSync(join(packetPath, "original"), { recursive: true });
|
|
127
207
|
mkdirSync(join(packetPath, "attachments"), { recursive: true });
|
|
128
208
|
|
|
129
|
-
const
|
|
209
|
+
const isPdf = filePath.toLowerCase().endsWith(".pdf");
|
|
210
|
+
const content = isPdf ? "" : readText(filePath);
|
|
130
211
|
const fileName = filePath.split("/").pop() || "unknown";
|
|
131
212
|
|
|
132
|
-
// Try
|
|
213
|
+
// Try MarkItDown for PDFs.
|
|
133
214
|
let extracted = content;
|
|
134
|
-
if (
|
|
215
|
+
if (isPdf) {
|
|
135
216
|
const markitdown = await exec(
|
|
136
217
|
pi,
|
|
137
218
|
"sh",
|
|
@@ -145,13 +226,14 @@ export async function captureFile(
|
|
|
145
226
|
pi,
|
|
146
227
|
"sh",
|
|
147
228
|
["-c", `uvx --from 'markitdown[pdf]' markitdown "${filePath}" 2>/dev/null || echo ""`],
|
|
148
|
-
{ signal, timeout:
|
|
229
|
+
{ signal, timeout: markitdownTimeoutMs() },
|
|
149
230
|
);
|
|
150
231
|
if (mdResult.stdout.trim()) extracted = mdResult.stdout;
|
|
151
232
|
} catch {
|
|
152
|
-
|
|
233
|
+
extracted = pdfExtractionFailureMessage(filePath);
|
|
153
234
|
}
|
|
154
235
|
}
|
|
236
|
+
if (!extracted) extracted = pdfExtractionFailureMessage(filePath);
|
|
155
237
|
}
|
|
156
238
|
|
|
157
239
|
// Copy original to packet
|
|
@@ -227,7 +309,7 @@ export function captureText(paths: VaultPaths, text: string, title?: string): Ca
|
|
|
227
309
|
function buildSourcePageSkeleton(manifest: Record<string, unknown>, extracted: string): string {
|
|
228
310
|
const id = String(manifest.id);
|
|
229
311
|
const title = String(manifest.title || id);
|
|
230
|
-
const url = manifest.url ? `\n> _Original: ${manifest.url}_` : "";
|
|
312
|
+
const url = manifest.url ? `\n> _Original: [${manifest.url}](${manifest.url})_` : "";
|
|
231
313
|
const format = String(manifest.format || "unknown");
|
|
232
314
|
const captured = String(manifest.captured || fmtDate());
|
|
233
315
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zosmaai/pi-llm-wiki",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.2",
|
|
4
4
|
"description": "LLM Wiki for Pi — self-maintaining knowledge base following Karpathy's pattern. Obsidian-friendly, auto-updating, personal & company wiki.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
package/prompts/wiki-digest.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Generate a daily or weekly digest of wiki changes — new sources, pages, insights, and gaps.
|
|
3
|
-
|
|
3
|
+
argument-hint: "[--period daily|weekly]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Generate a digest of recent wiki activity.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
## Steps
|
|
13
17
|
|
|
14
18
|
1. Read `wiki/LOG.md` — filter entries since last digest
|
package/prompts/wiki-discover.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Auto-discover new sources from the web. Searches based on config topics and known knowledge gaps.
|
|
3
|
-
|
|
3
|
+
argument-hint: "[--topic <topic>]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Find new source material for the wiki by searching the web.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first. Also read `config.yaml` for topics and feeds.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-ingest.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Process new source files in raw/ and update the wiki. Creates summaries, entities, concepts, and cross-references.
|
|
3
|
-
|
|
3
|
+
argument-hint: "[path]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Process new files in `raw/` and integrate them into the wiki.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema, page formats, and conventions.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-init.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Initialize a new LLM Wiki in the current directory. Creates the full directory structure, config, and template files.
|
|
3
|
-
|
|
3
|
+
argument-hint: "<topic> [--mode personal|company]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Initialize a new LLM Wiki in the current directory.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` (or wherever the skill is installed) first to understand the full schema and conventions.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-lint.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Health check the wiki. Detects contradictions, orphans, missing pages, stale claims, and knowledge gaps.
|
|
3
|
-
|
|
3
|
+
argument-hint: "[--fix]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Run a comprehensive health check on the wiki.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema and conventions.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-query.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Ask questions against the wiki. Synthesizes answers from wiki pages with cross-reference citations.
|
|
3
|
-
|
|
3
|
+
argument-hint: "<question>"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Ask a question and get an answer synthesized from wiki content.
|
|
11
11
|
|
|
12
|
+
## User Question
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first to understand the full schema and conventions.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-run.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
---
|
|
2
2
|
description: Run the full wiki cycle: discover → ingest → lint. Optionally schedule for auto-updates.
|
|
3
|
-
|
|
3
|
+
argument-hint: "[--schedule daily|weekly]"
|
|
4
4
|
section: LLM Wiki
|
|
5
5
|
topLevelCli: true
|
|
6
6
|
---
|
|
@@ -9,6 +9,10 @@ topLevelCli: true
|
|
|
9
9
|
|
|
10
10
|
Run the complete wiki maintenance cycle: discover new sources, ingest them, and lint for health.
|
|
11
11
|
|
|
12
|
+
## User Arguments
|
|
13
|
+
|
|
14
|
+
$ARGUMENTS
|
|
15
|
+
|
|
12
16
|
Read the LLM Wiki skill at `.pi/skills/llm-wiki/SKILL.md` first.
|
|
13
17
|
|
|
14
18
|
## Steps
|
package/prompts/wiki-status.md
CHANGED
package/test/llm-wiki.test.ts
CHANGED
|
@@ -3,6 +3,8 @@ import { tmpdir } from "node:os";
|
|
|
3
3
|
import { dirname, join, resolve } from "node:path";
|
|
4
4
|
import { fileURLToPath } from "node:url";
|
|
5
5
|
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
|
6
|
+
import { captureFile, captureText, captureUrl } from "../extensions/llm-wiki/lib/source-packet.js";
|
|
7
|
+
import { ensureVaultStructure, getVaultPaths } from "../extensions/llm-wiki/lib/utils.js";
|
|
6
8
|
|
|
7
9
|
// ─── Helpers ────────────────────────────────────────────
|
|
8
10
|
|
|
@@ -111,11 +113,33 @@ describe("package structure", () => {
|
|
|
111
113
|
expect(existsSync(path)).toBe(true);
|
|
112
114
|
const content = readFile(path);
|
|
113
115
|
expect(content).toContain("description:");
|
|
116
|
+
expect(content).toContain("argument-hint:");
|
|
114
117
|
expect(content).toContain("section: LLM Wiki");
|
|
115
118
|
expect(content).toContain("topLevelCli: true");
|
|
119
|
+
expect(content).not.toContain("\nargs:");
|
|
116
120
|
}
|
|
117
121
|
});
|
|
118
122
|
|
|
123
|
+
it("should include prompt arguments in templates that accept them", () => {
|
|
124
|
+
const promptsWithArgs = [
|
|
125
|
+
"wiki-init.md",
|
|
126
|
+
"wiki-ingest.md",
|
|
127
|
+
"wiki-query.md",
|
|
128
|
+
"wiki-lint.md",
|
|
129
|
+
"wiki-discover.md",
|
|
130
|
+
"wiki-run.md",
|
|
131
|
+
"wiki-digest.md",
|
|
132
|
+
];
|
|
133
|
+
for (const prompt of promptsWithArgs) {
|
|
134
|
+
const content = readFile(join(rootDir, "prompts", prompt));
|
|
135
|
+
expect(content).toContain("$ARGUMENTS");
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
const query = readFile(join(rootDir, "prompts", "wiki-query.md"));
|
|
139
|
+
expect(query).toContain("## User Question");
|
|
140
|
+
expect(query).toContain("$ARGUMENTS");
|
|
141
|
+
});
|
|
142
|
+
|
|
119
143
|
it("should have all wiki template files", () => {
|
|
120
144
|
const t = join(rootDir, "skills", "llm-wiki", "templates");
|
|
121
145
|
expect(existsSync(join(t, "INDEX.md"))).toBe(true);
|
|
@@ -157,6 +181,17 @@ describe("package structure", () => {
|
|
|
157
181
|
}
|
|
158
182
|
});
|
|
159
183
|
|
|
184
|
+
it("should keep MarkItDown timeout configurable and avoid PDF byte fallbacks", () => {
|
|
185
|
+
const sourcePacketPath = join(rootDir, "extensions", "llm-wiki", "lib", "source-packet.ts");
|
|
186
|
+
expect(existsSync(sourcePacketPath)).toBe(true);
|
|
187
|
+
const content = readFile(sourcePacketPath);
|
|
188
|
+
expect(content).toContain("WIKI_MARKITDOWN_TIMEOUT_MS");
|
|
189
|
+
expect(content).toContain("DEFAULT_MARKITDOWN_TIMEOUT_MS = 180_000");
|
|
190
|
+
expect(content).toContain("isPdfUrl(url)");
|
|
191
|
+
expect(content).toContain("looksLikePdf(curlResult.stdout)");
|
|
192
|
+
expect(content).toContain("pdfExtractionFailureMessage");
|
|
193
|
+
});
|
|
194
|
+
|
|
160
195
|
it("should have a comprehensive README with install instructions", () => {
|
|
161
196
|
const readme = readFile(join(rootDir, "README.md"));
|
|
162
197
|
expect(readme).toContain("@zosmaai/pi-llm-wiki");
|
|
@@ -199,6 +234,162 @@ describe("skill frontmatter validation", () => {
|
|
|
199
234
|
|
|
200
235
|
// ─── Wiki Directory Structure Tests ─────────────────────
|
|
201
236
|
|
|
237
|
+
describe("source packet capture", () => {
|
|
238
|
+
const html = "<html><head><title>Example Page</title></head><body>Hello</body></html>";
|
|
239
|
+
const pdfBytes = "%PDF-1.7\n1 0 obj\n<</Type/Catalog>>\nendobj\n";
|
|
240
|
+
|
|
241
|
+
beforeEach(() => {
|
|
242
|
+
tempDir = join(tmpdir(), `pi-llm-wiki-capture-${Date.now()}`);
|
|
243
|
+
mkdirSync(tempDir, { recursive: true });
|
|
244
|
+
});
|
|
245
|
+
|
|
246
|
+
afterEach(() => {
|
|
247
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
248
|
+
});
|
|
249
|
+
|
|
250
|
+
/** Build a mock pi.exec that optionally handles curl -o file writes. */
|
|
251
|
+
function mockPi(stdout?: string, writeOriginal = true) {
|
|
252
|
+
return {
|
|
253
|
+
exec: async (command: string, args: string[]) => {
|
|
254
|
+
if (command === "sh") return { stdout: "no\n", stderr: "", code: 0 };
|
|
255
|
+
if (command === "curl" && args.includes("-o")) {
|
|
256
|
+
if (writeOriginal) {
|
|
257
|
+
const outputPath = args[args.indexOf("-o") + 1];
|
|
258
|
+
writeFileSync(outputPath, stdout ?? html, "utf-8");
|
|
259
|
+
}
|
|
260
|
+
return { stdout: "", stderr: "", code: 0 };
|
|
261
|
+
}
|
|
262
|
+
if (command === "curl") return { stdout: stdout ?? html, stderr: "", code: 0 };
|
|
263
|
+
throw new Error(`Unexpected command: ${command}`);
|
|
264
|
+
},
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function makePaths() {
|
|
269
|
+
const p = getVaultPaths(join(tempDir, `wiki-${Math.random().toString(36).slice(2)}`));
|
|
270
|
+
ensureVaultStructure(p);
|
|
271
|
+
return p;
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
it("should preserve the original artifact for URL captures and render clickable links", async () => {
|
|
275
|
+
const paths = makePaths();
|
|
276
|
+
const pi = mockPi();
|
|
277
|
+
|
|
278
|
+
const url = "https://example.com/article";
|
|
279
|
+
const result = await captureUrl(pi as never, paths, url);
|
|
280
|
+
|
|
281
|
+
// Original artifact preserved
|
|
282
|
+
expect(existsSync(join(result.packetPath, "original", "source.html"))).toBe(true);
|
|
283
|
+
expect(readFile(join(result.packetPath, "original", "source.html"))).toBe(html);
|
|
284
|
+
expect(readFile(join(result.packetPath, "extracted.md"))).toBe(html);
|
|
285
|
+
|
|
286
|
+
// Clickable URL in source page
|
|
287
|
+
const sourcePage = readFile(result.sourcePagePath);
|
|
288
|
+
expect(sourcePage).toContain(`> _Original: [${url}](${url})_`);
|
|
289
|
+
});
|
|
290
|
+
|
|
291
|
+
it("should write PDF extraction failure message for .pdf URLs when MarkItDown is unavailable", async () => {
|
|
292
|
+
const paths = makePaths();
|
|
293
|
+
const pi = mockPi();
|
|
294
|
+
|
|
295
|
+
const url = "https://example.com/report.pdf";
|
|
296
|
+
const result = await captureUrl(pi as never, paths, url);
|
|
297
|
+
|
|
298
|
+
// Should NOT have written raw original for .pdf (no uvx)
|
|
299
|
+
expect(existsSync(join(result.packetPath, "original", "source.pdf"))).toBe(true);
|
|
300
|
+
|
|
301
|
+
// extracted.md should contain a failure message, not raw bytes
|
|
302
|
+
const extracted = readFile(join(result.packetPath, "extracted.md"));
|
|
303
|
+
expect(extracted).not.toContain("%PDF-");
|
|
304
|
+
expect(extracted).toContain("PDF content could not be converted");
|
|
305
|
+
expect(extracted).toContain(url);
|
|
306
|
+
});
|
|
307
|
+
|
|
308
|
+
it("should sniff %PDF- bytes from non-.pdf URLs and write a failure message instead", async () => {
|
|
309
|
+
const paths = makePaths();
|
|
310
|
+
// curl returns PDF bytes from a non-.pdf URL
|
|
311
|
+
const pi = mockPi(pdfBytes);
|
|
312
|
+
|
|
313
|
+
const url = "https://example.com/download?format=pdf";
|
|
314
|
+
const result = await captureUrl(pi as never, paths, url);
|
|
315
|
+
|
|
316
|
+
const extracted = readFile(join(result.packetPath, "extracted.md"));
|
|
317
|
+
expect(extracted).not.toContain("%PDF-");
|
|
318
|
+
expect(extracted).toContain("PDF content could not be converted");
|
|
319
|
+
expect(extracted).toContain(url);
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
it("should name original artifacts based on URL extension", async () => {
|
|
323
|
+
const cases: Array<{ url: string; expected: string }> = [
|
|
324
|
+
{ url: "https://example.com/article.html", expected: "source.html" },
|
|
325
|
+
{ url: "https://example.com/doc.pdf", expected: "source.pdf" },
|
|
326
|
+
{ url: "https://example.com/notes.md", expected: "source.md" },
|
|
327
|
+
{ url: "https://example.com/data.xml", expected: "source.xml" },
|
|
328
|
+
{ url: "https://example.com/readme.txt", expected: "source.txt" },
|
|
329
|
+
{ url: "https://example.com/page", expected: "source.html" },
|
|
330
|
+
{ url: "https://example.com/page?format=pdf", expected: "source.html" },
|
|
331
|
+
];
|
|
332
|
+
|
|
333
|
+
for (const { url, expected } of cases) {
|
|
334
|
+
const paths = makePaths();
|
|
335
|
+
const pi = mockPi();
|
|
336
|
+
const result = await captureUrl(pi as never, paths, url);
|
|
337
|
+
const originalFile = join(result.packetPath, "original", expected);
|
|
338
|
+
expect(existsSync(originalFile)).toBe(true);
|
|
339
|
+
}
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
it("should render source page without an Original: line for text captures", async () => {
|
|
343
|
+
const paths = makePaths();
|
|
344
|
+
const result = captureText(paths, "Some text content", "My Note");
|
|
345
|
+
|
|
346
|
+
const sourcePage = readFile(result.sourcePagePath);
|
|
347
|
+
expect(sourcePage).toContain("# My Note");
|
|
348
|
+
expect(sourcePage).not.toContain("Original:");
|
|
349
|
+
expect(sourcePage).toContain("_Auto-preview: Some text content_");
|
|
350
|
+
});
|
|
351
|
+
|
|
352
|
+
it("should truncate auto-preview to 500 characters", async () => {
|
|
353
|
+
const paths = makePaths();
|
|
354
|
+
const longText = "A".repeat(1000);
|
|
355
|
+
const result = captureText(paths, longText, "Long Note");
|
|
356
|
+
|
|
357
|
+
const sourcePage = readFile(result.sourcePagePath);
|
|
358
|
+
// Preview should be 500 chars + "..."
|
|
359
|
+
expect(sourcePage).toContain(`_Auto-preview: ${"A".repeat(500)}..._`);
|
|
360
|
+
});
|
|
361
|
+
|
|
362
|
+
it("should handle local PDF file capture failure message when MarkItDown is unavailable", async () => {
|
|
363
|
+
const paths = makePaths();
|
|
364
|
+
// Create a temporary PDF file
|
|
365
|
+
const pdfPath = join(tempDir, "test.pdf");
|
|
366
|
+
writeFileSync(pdfPath, pdfBytes, "utf-8");
|
|
367
|
+
|
|
368
|
+
const pi = mockPi();
|
|
369
|
+
const result = await captureFile(pi as never, paths, pdfPath);
|
|
370
|
+
|
|
371
|
+
const extracted = readFile(join(result.packetPath, "extracted.md"));
|
|
372
|
+
expect(extracted).not.toContain("%PDF-");
|
|
373
|
+
expect(extracted).toContain("PDF content could not be converted");
|
|
374
|
+
});
|
|
375
|
+
|
|
376
|
+
it("should copy local non-PDF file content to extracted.md", async () => {
|
|
377
|
+
const paths = makePaths();
|
|
378
|
+
const mdPath = join(tempDir, "notes.md");
|
|
379
|
+
writeFileSync(mdPath, "# My Notes\n\nHello world.", "utf-8");
|
|
380
|
+
|
|
381
|
+
const pi = mockPi();
|
|
382
|
+
const result = await captureFile(pi as never, paths, mdPath);
|
|
383
|
+
|
|
384
|
+
const extracted = readFile(join(result.packetPath, "extracted.md"));
|
|
385
|
+
expect(extracted).toContain("# My Notes");
|
|
386
|
+
expect(extracted).toContain("Hello world.");
|
|
387
|
+
|
|
388
|
+
// Original file should be preserved
|
|
389
|
+
expect(existsSync(join(result.packetPath, "original", "notes.md"))).toBe(true);
|
|
390
|
+
});
|
|
391
|
+
});
|
|
392
|
+
|
|
202
393
|
describe("wiki directory structure", () => {
|
|
203
394
|
let wikiDir: string;
|
|
204
395
|
|