@sdsrs/llm-wiki 0.7.0 → 0.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/bin/llm-wiki.mjs +7 -0
- package/package.json +1 -1
- package/src/ask.mjs +18 -8
- package/src/convert-run.mjs +1 -1
- package/src/convert.mjs +18 -5
- package/src/embed.mjs +13 -5
- package/src/indexer.mjs +7 -6
- package/src/json.mjs +12 -0
- package/src/lint.mjs +8 -5
- package/src/manifest.mjs +4 -6
- package/src/pages.mjs +12 -0
- package/src/status.mjs +2 -2
- package/src/templates.mjs +18 -2
- package/src/vector.mjs +7 -9
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,63 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.7.2 (2026-07-12)
|
|
4
|
+
|
|
5
|
+
Three defects found by continued end-to-end dogfooding — two robustness/DoS
|
|
6
|
+
guards plus one UX fix. No config schema change, no KB migration, no
|
|
7
|
+
retrieval-default change. Suite 221 → 228.
|
|
8
|
+
|
|
9
|
+
**Fixes:**
|
|
10
|
+
|
|
11
|
+
- **Malformed numeric config can no longer crash retrieval or hang scan.**
|
|
12
|
+
`wiki.config.json` is shallow-merged as-is, so a numeric key could arrive as a
|
|
13
|
+
string/0/negative/NaN and reach a scalar consumer: `bm25TitleWeight: "high"`
|
|
14
|
+
hit `Array(NaN)` → `RangeError` on *every* `ask`/`search`/MCP query (a huge int
|
|
15
|
+
→ per-page `Array(n).fill` OOM), and `batchSize: 0` made `scan`'s batch loop
|
|
16
|
+
(`i += batchSize`) spin forever. `loadKbConfig` now coerces every numeric key
|
|
17
|
+
back to a finite integer ≥ 1 (falling back to its default), and
|
|
18
|
+
`bm25TitleWeight` is additionally capped at its consumer. Legitimate overrides
|
|
19
|
+
are untouched.
|
|
20
|
+
- **Derived stores are written atomically.** `buildIndex` wrote `graph.json` /
|
|
21
|
+
`index.md` / `llms.txt` / topic files with a direct `writeFileSync`, which
|
|
22
|
+
truncates the target before writing — a reader running concurrently (a
|
|
23
|
+
long-lived MCP `graph` tool, `ask`'s `llms.txt` fallback) could read a torn
|
|
24
|
+
`graph.json` and crash on `JSON.parse`. All derived stores now write via a
|
|
25
|
+
shared temp-and-rename `writeFileAtomic` (the pattern `saveManifest` /
|
|
26
|
+
`saveVectorStore` already used), so a concurrent reader only ever sees the whole
|
|
27
|
+
old or whole new file.
|
|
28
|
+
- **A source that converts to an empty page is no longer silent.** A
|
|
29
|
+
scanned/image-only PDF, an empty DOCX, or a blank `.txt` extracts to no text;
|
|
30
|
+
`convert` still writes the (empty) raw page and counts it as converted, but now
|
|
31
|
+
prints `WARN <src>: converted to an empty page — no extractable text` so the
|
|
32
|
+
user isn't misled into thinking real content was ingested.
|
|
33
|
+
|
|
34
|
+
## 0.7.1 (2026-07-11)
|
|
35
|
+
|
|
36
|
+
Four defects found by end-to-end dogfooding — one crash-class fix plus three
|
|
37
|
+
robustness/UX fixes. No config change, no KB migration, no retrieval-default
|
|
38
|
+
change. Suite 213 → 221.
|
|
39
|
+
|
|
40
|
+
**Fixes:**
|
|
41
|
+
|
|
42
|
+
- **A single malformed page no longer breaks the whole KB.** A frontmatter list
|
|
43
|
+
field authored as a bare scalar (`tags: cache` instead of `tags: [cache]`)
|
|
44
|
+
parses as a string, which slipped past the `?? []` guards and then crashed
|
|
45
|
+
`ask`/`embed` (`.join` on a string) or was iterated character-by-character
|
|
46
|
+
(`lint` noise; `index` synthesized a bogus graph edge per character, corrupting
|
|
47
|
+
`graph.json`). List fields now route through an `Array.isArray` guard in every
|
|
48
|
+
consumer, so one bad page can no longer take down retrieval or the graph.
|
|
49
|
+
`lint` now flags `tags must be a YAML list` so the author can fix the shape.
|
|
50
|
+
- **`lint` contradiction-scan skips invalidated pages.** Retired knowledge is
|
|
51
|
+
already excluded from stale-scan and the orphan rule; the shared-tag
|
|
52
|
+
contradiction scan now matches, so you are never asked to reconcile a
|
|
53
|
+
contradiction against a dead page.
|
|
54
|
+
- **`convert` exits non-zero when every file fails.** `converted 0, failed N`
|
|
55
|
+
used to exit `0`, hiding a total failure from CI/scripts. It now exits `1` on
|
|
56
|
+
total failure; a partial success (some files converted) still exits `0`.
|
|
57
|
+
- **Network errors name the endpoint.** A mistyped `baseURL` or offline host used
|
|
58
|
+
to surface undici's opaque `fetch failed`; `ask`/`embed` now report
|
|
59
|
+
`could not reach the LLM/embedding endpoint <url> — check baseURL/network`.
|
|
60
|
+
|
|
3
61
|
## 0.7.0 (2026-07-11)
|
|
4
62
|
|
|
5
63
|
Two retrieval-default changes (hence a minor bump). Both are opt-out via
|
package/bin/llm-wiki.mjs
CHANGED
|
@@ -50,6 +50,13 @@ program.command('convert')
|
|
|
50
50
|
const r = await runConvertPlan(opts.kb)
|
|
51
51
|
console.log(`converted ${r.converted.length}, failed ${r.failed.length}`)
|
|
52
52
|
for (const f of r.failed) console.log(` FAILED ${f.src}: ${f.warnings.join('; ')}`)
|
|
53
|
+
// A converted-but-empty page (scanned PDF, empty DOCX) is not a failure, but the
|
|
54
|
+
// user must see it — an unflagged blank raw page reads as "ingested" when it isn't.
|
|
55
|
+
for (const c of r.converted) if (c.warnings) console.log(` WARN ${c.src}: ${c.warnings.join('; ')}`)
|
|
56
|
+
// Total failure (nothing converted, at least one error) is a hard error a
|
|
57
|
+
// pipeline must catch — exit 1. A partial success keeps exit 0 so a messy dir
|
|
58
|
+
// with a few unconvertible junk files doesn't break an otherwise-good run.
|
|
59
|
+
if (r.converted.length === 0 && r.failed.length > 0) process.exit(1)
|
|
53
60
|
})
|
|
54
61
|
|
|
55
62
|
program.command('index')
|
package/package.json
CHANGED
package/src/ask.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import fs from 'node:fs'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { kbPaths } from './paths.mjs'
|
|
4
4
|
import { loadKbConfig } from './templates.mjs'
|
|
5
|
-
import { listWikiPages, isInvalidated } from './pages.mjs'
|
|
5
|
+
import { listWikiPages, isInvalidated, asList } from './pages.mjs'
|
|
6
6
|
import { buildBm25Index, searchBm25 } from './bm25.mjs'
|
|
7
7
|
import { worstCaseTokens } from './scanner.mjs'
|
|
8
8
|
import { loadLlmConfig, makeTransport } from './llm-config.mjs'
|
|
@@ -15,11 +15,13 @@ export function retrievePages(kbRoot, question, k = 6) {
|
|
|
15
15
|
// title term outweighs the same term buried in the body (measured +0.04 Recall@5
|
|
16
16
|
// on the dogfood KB at ×3). Set bm25TitleWeight: 1 in wiki.config.json to restore
|
|
17
17
|
// flat single-field indexing.
|
|
18
|
-
|
|
18
|
+
// loadKbConfig guarantees a finite integer >= 1; cap the upper bound so a large
|
|
19
|
+
// value can't materialize a giant per-page array (Array(w).fill) and OOM retrieval.
|
|
20
|
+
const w = Math.min(25, loadKbConfig(kbRoot).bm25TitleWeight)
|
|
19
21
|
const pages = listWikiPages(kbRoot).filter(p => !p.error && !isInvalidated(p))
|
|
20
22
|
const idx = buildBm25Index(pages.map(p => ({
|
|
21
23
|
id: p.relPath,
|
|
22
|
-
text: [Array(w).fill(p.data.title ?? '').join('\n'), p.data.description, (p.data.tags
|
|
24
|
+
text: [Array(w).fill(p.data.title ?? '').join('\n'), p.data.description, asList(p.data.tags).join(' '), p.body].join('\n'),
|
|
23
25
|
})))
|
|
24
26
|
return searchBm25(idx, question, k).map(h => ({ relPath: h.id, score: h.score }))
|
|
25
27
|
}
|
|
@@ -84,11 +86,19 @@ export async function locatePages(kbRoot, question, { k = 6, fetchImpl, retrieva
|
|
|
84
86
|
}
|
|
85
87
|
|
|
86
88
|
export async function chatCompletion(cfg, t, messages) {
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
89
|
+
const url = `${cfg.baseURL.replace(/\/$/, '')}/chat/completions`
|
|
90
|
+
let res
|
|
91
|
+
try {
|
|
92
|
+
res = await fetchWithRetry(t.fetchImpl, url, {
|
|
93
|
+
method: 'POST',
|
|
94
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
|
|
95
|
+
body: JSON.stringify({ model: cfg.model, messages }),
|
|
96
|
+
}, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
|
|
97
|
+
} catch (err) {
|
|
98
|
+
// Node's undici surfaces a bare "fetch failed" for DNS/refused/timeout — name
|
|
99
|
+
// the endpoint so a mistyped baseURL or offline host is diagnosable, not opaque.
|
|
100
|
+
throw new Error(`could not reach the LLM endpoint ${url} — check baseURL/network in ~/.llm-wiki/config.json (${err.message})`)
|
|
101
|
+
}
|
|
92
102
|
if (!res.ok) {
|
|
93
103
|
let body = ''
|
|
94
104
|
try { body = (await res.text()).slice(0, 200) } catch { /* body unreadable; status alone */ }
|
package/src/convert-run.mjs
CHANGED
|
@@ -69,7 +69,7 @@ export async function runConvertPlan(kbRoot) {
|
|
|
69
69
|
convertedAt: new Date().toISOString().slice(0, 10),
|
|
70
70
|
...(originalRel ? { original: originalRel } : {}),
|
|
71
71
|
}
|
|
72
|
-
converted.push({ src: rel, raw: rawRel })
|
|
72
|
+
converted.push({ src: rel, raw: rawRel, ...(warnings.length ? { warnings } : {}) })
|
|
73
73
|
}
|
|
74
74
|
saveManifest(kbRoot, manifest)
|
|
75
75
|
return { converted, failed }
|
package/src/convert.mjs
CHANGED
|
@@ -19,6 +19,19 @@ function titleFrom(markdown, fallback) {
|
|
|
19
19
|
return m ? m[1].trim() : fallback
|
|
20
20
|
}
|
|
21
21
|
|
|
22
|
+
// A source can convert successfully yet yield no text — a scanned/image-only PDF, an
|
|
23
|
+
// empty DOCX, a blank .txt. That is still a converted page (markdown '' is not null, so
|
|
24
|
+
// the deliberate empty-file handling downstream is untouched), but writing a blank raw
|
|
25
|
+
// page while reporting "converted" silently misleads the user. Attach a warning the CLI
|
|
26
|
+
// can surface. `# heading`-prefixed HTML output never trims to empty, so this fires only
|
|
27
|
+
// on genuinely contentless extractions.
|
|
28
|
+
function result(markdown, title, warnings) {
|
|
29
|
+
if (markdown !== null && markdown.trim() === '') {
|
|
30
|
+
warnings.push('converted to an empty page — no extractable text (scanned/image-only PDF or empty document?)')
|
|
31
|
+
}
|
|
32
|
+
return { markdown, title, warnings }
|
|
33
|
+
}
|
|
34
|
+
|
|
22
35
|
export async function convertFile(srcPath) {
|
|
23
36
|
const ext = path.extname(srcPath).toLowerCase()
|
|
24
37
|
const base = path.basename(srcPath)
|
|
@@ -26,11 +39,11 @@ export async function convertFile(srcPath) {
|
|
|
26
39
|
try {
|
|
27
40
|
if (ext === '.md' || ext === '.markdown') {
|
|
28
41
|
const md = fs.readFileSync(srcPath, 'utf8')
|
|
29
|
-
return
|
|
42
|
+
return result(md, titleFrom(md, base), warnings)
|
|
30
43
|
}
|
|
31
44
|
if (ext === '.txt') {
|
|
32
45
|
const md = fs.readFileSync(srcPath, 'utf8')
|
|
33
|
-
return
|
|
46
|
+
return result(md, titleFrom(md, base), warnings)
|
|
34
47
|
}
|
|
35
48
|
if (ext === '.html' || ext === '.htm') {
|
|
36
49
|
const { JSDOM } = await import('jsdom')
|
|
@@ -45,17 +58,17 @@ export async function convertFile(srcPath) {
|
|
|
45
58
|
}
|
|
46
59
|
const td = new TurndownService({ headingStyle: 'atx', codeBlockStyle: 'fenced' })
|
|
47
60
|
const md = td.turndown(article.content)
|
|
48
|
-
return
|
|
61
|
+
return result(`# ${article.title || base}\n\n${md}`, article.title || base, warnings)
|
|
49
62
|
}
|
|
50
63
|
if (ext === '.pdf') {
|
|
51
64
|
const { PDFParse } = await import('pdf-parse')
|
|
52
65
|
const { text } = await new PDFParse({ data: fs.readFileSync(srcPath) }).getText()
|
|
53
|
-
return
|
|
66
|
+
return result(text, titleFrom(text, base), warnings)
|
|
54
67
|
}
|
|
55
68
|
if (ext === '.docx') {
|
|
56
69
|
const mammoth = await import('mammoth')
|
|
57
70
|
const { value } = await mammoth.extractRawText({ path: srcPath })
|
|
58
|
-
return
|
|
71
|
+
return result(value, titleFrom(value, base), warnings)
|
|
59
72
|
}
|
|
60
73
|
warnings.push(`unsupported extension: ${ext}`)
|
|
61
74
|
return { markdown: null, title: base, warnings }
|
package/src/embed.mjs
CHANGED
|
@@ -45,11 +45,19 @@ function capEmbedText(text) {
|
|
|
45
45
|
}
|
|
46
46
|
|
|
47
47
|
export async function embedTexts(cfg, t, texts) {
|
|
48
|
-
const
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
48
|
+
const url = `${cfg.baseURL.replace(/\/$/, '')}/embeddings`
|
|
49
|
+
let res
|
|
50
|
+
try {
|
|
51
|
+
res = await fetchWithRetry(t.fetchImpl, url, {
|
|
52
|
+
method: 'POST',
|
|
53
|
+
headers: { 'content-type': 'application/json', authorization: `Bearer ${cfg.apiKey}` },
|
|
54
|
+
body: JSON.stringify({ model: cfg.embeddingModel, input: texts }),
|
|
55
|
+
}, { dispatcher: t.dispatcher, ...(t.retry ?? {}) })
|
|
56
|
+
} catch (err) {
|
|
57
|
+
// Mirror chatCompletion: turn undici's opaque "fetch failed" into a diagnosable
|
|
58
|
+
// message naming the endpoint (mistyped baseURL / offline host / DNS failure).
|
|
59
|
+
throw new Error(`could not reach the embedding endpoint ${url} — check baseURL/network in ~/.llm-wiki/config.json (${err.message})`)
|
|
60
|
+
}
|
|
53
61
|
if (!res.ok) {
|
|
54
62
|
let body = ''
|
|
55
63
|
try { body = (await res.text()).slice(0, 200) } catch { /* status alone */ }
|
package/src/indexer.mjs
CHANGED
|
@@ -2,7 +2,8 @@ import fs from 'node:fs'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { kbPaths } from './paths.mjs'
|
|
4
4
|
import { loadKbConfig } from './templates.mjs'
|
|
5
|
-
import { listWikiPages, isInvalidated, RELATION_CONFIDENCES } from './pages.mjs'
|
|
5
|
+
import { listWikiPages, isInvalidated, asList, RELATION_CONFIDENCES } from './pages.mjs'
|
|
6
|
+
import { writeFileAtomic } from './json.mjs'
|
|
6
7
|
|
|
7
8
|
const WIKILINK_RE = /\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g
|
|
8
9
|
|
|
@@ -69,7 +70,7 @@ export function buildIndex(kbRoot) {
|
|
|
69
70
|
const written = new Set()
|
|
70
71
|
for (const [type, list] of Object.entries(byType)) {
|
|
71
72
|
if (!list.length) continue
|
|
72
|
-
|
|
73
|
+
writeFileAtomic(path.join(p.topics, `${type}.md`), `# ${SECTION_TITLES[type]}\n\n${list.map(line).join('\n')}\n`)
|
|
73
74
|
written.add(`${type}.md`)
|
|
74
75
|
indexBody += `\n## ${SECTION_TITLES[type]}\nSee [[topics/${type}]] (${list.length} pages)\n`
|
|
75
76
|
}
|
|
@@ -81,7 +82,7 @@ export function buildIndex(kbRoot) {
|
|
|
81
82
|
indexBody += `\n## ${SECTION_TITLES[type]}\n${list.map(line).join('\n')}\n`
|
|
82
83
|
}
|
|
83
84
|
}
|
|
84
|
-
|
|
85
|
+
writeFileAtomic(p.indexMd, `${indexBody}\n${pending}${tail}`)
|
|
85
86
|
|
|
86
87
|
const ids = new Set(pages.map(pg => pg.relPath.replace(/\.md$/, '')))
|
|
87
88
|
const nodes = pages.map(pg => ({
|
|
@@ -96,7 +97,7 @@ export function buildIndex(kbRoot) {
|
|
|
96
97
|
for (const target of extractWikilinks(pg.body)) {
|
|
97
98
|
if (ids.has(target)) edges.push({ source: id, target, type: 'wikilink', confidence: 'inferred' })
|
|
98
99
|
}
|
|
99
|
-
for (const src of pg.data.sources
|
|
100
|
+
for (const src of asList(pg.data.sources)) edges.push({ source: id, target: String(src), type: 'source', confidence: 'extracted' })
|
|
100
101
|
if (pg.data.superseded_by && ids.has(String(pg.data.superseded_by))) {
|
|
101
102
|
edges.push({ source: id, target: String(pg.data.superseded_by), type: 'superseded_by', confidence: 'extracted' })
|
|
102
103
|
}
|
|
@@ -114,12 +115,12 @@ export function buildIndex(kbRoot) {
|
|
|
114
115
|
edges.push({ source: id, target, type: String(rel.type), confidence })
|
|
115
116
|
}
|
|
116
117
|
}
|
|
117
|
-
|
|
118
|
+
writeFileAtomic(p.graphJson, JSON.stringify({ nodes, edges }, null, 2) + '\n')
|
|
118
119
|
|
|
119
120
|
const kbName = path.basename(path.resolve(kbRoot))
|
|
120
121
|
const llms = [`# ${kbName}`, '', `> llm_wiki knowledge base. Read wiki/index.md first, then open full pages.`, '', '## Pages',
|
|
121
122
|
...pages.filter(pg => !isInvalidated(pg)).map(pg => `- [${inline(pg.data.title)}](wiki/${pg.relPath}): ${inline(pg.data.description)}`)].join('\n') + '\n'
|
|
122
|
-
|
|
123
|
+
writeFileAtomic(p.llmsTxt, llms)
|
|
123
124
|
|
|
124
125
|
return { pageCount: pages.length, topicsSplit }
|
|
125
126
|
}
|
package/src/json.mjs
CHANGED
|
@@ -14,3 +14,15 @@ export function readJsonFile(file, { redactContents = false } = {}) {
|
|
|
14
14
|
throw new Error(redactContents ? `${file}: invalid JSON` : `${file}: invalid JSON (${err.message})`)
|
|
15
15
|
}
|
|
16
16
|
}
|
|
17
|
+
|
|
18
|
+
// Atomic file write: `fs.writeFileSync` truncates the target *before* writing, so a
|
|
19
|
+
// concurrent reader (a long-lived MCP server, an `ask` fallback) can observe an empty
|
|
20
|
+
// or half-written file — for JSON stores that means a `JSON.parse` crash. Write a temp
|
|
21
|
+
// sibling and rename (atomic on the POSIX filesystems this tool supports) so a reader
|
|
22
|
+
// only ever sees the whole old file or the whole new one. Single source of the pattern
|
|
23
|
+
// that manifest/vector already used and buildIndex's derived stores previously missed.
|
|
24
|
+
export function writeFileAtomic(file, data) {
|
|
25
|
+
const tmp = `${file}.tmp`
|
|
26
|
+
fs.writeFileSync(tmp, data)
|
|
27
|
+
fs.renameSync(tmp, file)
|
|
28
|
+
}
|
package/src/lint.mjs
CHANGED
|
@@ -2,7 +2,7 @@ import fs from 'node:fs'
|
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { kbPaths } from './paths.mjs'
|
|
4
4
|
import { loadKbConfig } from './templates.mjs'
|
|
5
|
-
import { listWikiPages, validatePage, isInvalidated, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
|
|
5
|
+
import { listWikiPages, validatePage, isInvalidated, asList, PAGE_STATUSES, RELATION_CONFIDENCES } from './pages.mjs'
|
|
6
6
|
import { extractWikilinks, buildIndex } from './indexer.mjs'
|
|
7
7
|
import { loadManifest } from './manifest.mjs'
|
|
8
8
|
|
|
@@ -27,7 +27,7 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
|
|
|
27
27
|
if (!ids.has(target)) mechanical.push({ rule: 'broken-wikilink', path: pg.relPath, detail: `-> ${target}` })
|
|
28
28
|
else incoming.set(target, (incoming.get(target) ?? 0) + 1)
|
|
29
29
|
}
|
|
30
|
-
for (const src of pg.data.sources
|
|
30
|
+
for (const src of asList(pg.data.sources)) {
|
|
31
31
|
if (!fs.existsSync(path.join(kbRoot, String(src)))) mechanical.push({ rule: 'missing-raw-source', path: pg.relPath, detail: String(src) })
|
|
32
32
|
}
|
|
33
33
|
if (pg.data.status !== undefined && !PAGE_STATUSES.includes(pg.data.status)) {
|
|
@@ -90,8 +90,11 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
|
|
|
90
90
|
}
|
|
91
91
|
const byTag = new Map()
|
|
92
92
|
for (const pg of pages) {
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
// Invalidated pages are retired knowledge: they must drop out of the semantic
|
|
94
|
+
// worklist just like stale-scan (below) and the orphan rule already do, so the
|
|
95
|
+
// agent is never asked to reconcile a "contradiction" against a dead page.
|
|
96
|
+
if (pg.error || isInvalidated(pg)) continue
|
|
97
|
+
for (const tag of asList(pg.data.tags)) {
|
|
95
98
|
if (!byTag.has(tag)) byTag.set(tag, [])
|
|
96
99
|
byTag.get(tag).push(pg.relPath)
|
|
97
100
|
}
|
|
@@ -115,7 +118,7 @@ export async function lintKb(kbRoot, { fix = false } = {}) {
|
|
|
115
118
|
if (pg.error || isInvalidated(pg)) continue
|
|
116
119
|
const updated = String(pg.data.updated ?? '')
|
|
117
120
|
if (!updated) continue
|
|
118
|
-
for (const src of pg.data.sources
|
|
121
|
+
for (const src of asList(pg.data.sources)) {
|
|
119
122
|
const conv = convertedAtByRaw.get(String(src))
|
|
120
123
|
if (conv && conv > updated) semantic.push({ task: 'stale-scan', detail: `${pg.relPath}: ${src} reconverted ${conv}, page updated ${updated}` })
|
|
121
124
|
}
|
package/src/manifest.mjs
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import { kbPaths } from './paths.mjs'
|
|
3
|
-
import { readJsonFile } from './json.mjs'
|
|
3
|
+
import { readJsonFile, writeFileAtomic } from './json.mjs'
|
|
4
4
|
|
|
5
5
|
export function loadManifest(kbRoot) {
|
|
6
6
|
const p = kbPaths(kbRoot)
|
|
@@ -13,11 +13,9 @@ export function loadManifest(kbRoot) {
|
|
|
13
13
|
|
|
14
14
|
export function saveManifest(kbRoot, manifest) {
|
|
15
15
|
const p = kbPaths(kbRoot)
|
|
16
|
-
// Atomic write: manifest is non-derived state (hash->raw
|
|
17
|
-
// must
|
|
18
|
-
|
|
19
|
-
fs.writeFileSync(tmp, JSON.stringify(manifest, null, 2) + '\n')
|
|
20
|
-
fs.renameSync(tmp, p.manifest)
|
|
16
|
+
// Atomic write (shared writeFileAtomic): manifest is non-derived state (hash->raw
|
|
17
|
+
// map) and a crash or concurrent reader must never see a truncated/corrupt file.
|
|
18
|
+
writeFileAtomic(p.manifest, JSON.stringify(manifest, null, 2) + '\n')
|
|
21
19
|
}
|
|
22
20
|
|
|
23
21
|
export function diffManifest(manifest, entries) {
|
package/src/pages.mjs
CHANGED
|
@@ -10,6 +10,14 @@ export const PAGE_STATUSES = ['active', 'invalidated']
|
|
|
10
10
|
|
|
11
11
|
export const RELATION_CONFIDENCES = ['extracted', 'inferred', 'ambiguous']
|
|
12
12
|
|
|
13
|
+
// A frontmatter list field (tags, sources) authored as a bare scalar — `tags: cache`
|
|
14
|
+
// instead of `tags: [cache]` — is parsed by YAML as a string. That silently passes a
|
|
15
|
+
// `?? []` guard and then crashes `.join()` (retrieval/embed) or iterates the string
|
|
16
|
+
// character-by-character (lint noise, and indexer synthesizing a bogus graph edge per
|
|
17
|
+
// char). Every consumer routes list fields through this so one malformed page cannot
|
|
18
|
+
// DoS retrieval or corrupt the graph; validatePage/lint still flag the bad shape.
|
|
19
|
+
export const asList = (v) => (Array.isArray(v) ? v : [])
|
|
20
|
+
|
|
13
21
|
export function isInvalidated(page) {
|
|
14
22
|
return page.data?.status === 'invalidated'
|
|
15
23
|
}
|
|
@@ -40,5 +48,9 @@ export function validatePage(page) {
|
|
|
40
48
|
if (page.data[k] === undefined || page.data[k] === null || page.data[k] === '') issues.push(`missing field: ${k}`)
|
|
41
49
|
}
|
|
42
50
|
if (!Array.isArray(page.data.sources)) issues.push('missing field: sources (evidence chain)')
|
|
51
|
+
// tags is caught as present by the REQUIRED loop above even when authored as a
|
|
52
|
+
// bare scalar (`tags: cache`); flag the wrong shape explicitly so the author fixes
|
|
53
|
+
// it rather than silently getting a page with no searchable tags.
|
|
54
|
+
if (page.data.tags !== undefined && !Array.isArray(page.data.tags)) issues.push('tags must be a YAML list')
|
|
43
55
|
return issues
|
|
44
56
|
}
|
package/src/status.mjs
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { kbPaths } from './paths.mjs'
|
|
4
|
-
import { listWikiPages } from './pages.mjs'
|
|
4
|
+
import { listWikiPages, asList } from './pages.mjs'
|
|
5
5
|
import { scanSource } from './scanner.mjs'
|
|
6
6
|
import { loadManifest, diffManifest } from './manifest.mjs'
|
|
7
7
|
|
|
@@ -27,7 +27,7 @@ export async function statusKb(kbRoot, srcDir) {
|
|
|
27
27
|
for (const pg of listWikiPages(kbRoot)) {
|
|
28
28
|
if (pg.error) continue
|
|
29
29
|
const id = pg.relPath.replace(/\.md$/, '')
|
|
30
|
-
for (const src of pg.data.sources
|
|
30
|
+
for (const src of asList(pg.data.sources)) {
|
|
31
31
|
const raw = String(src)
|
|
32
32
|
referenced.add(raw)
|
|
33
33
|
if (!pagesByRaw.has(raw)) pagesByRaw.set(raw, [])
|
package/src/templates.mjs
CHANGED
|
@@ -17,12 +17,28 @@ export const DEFAULT_CONFIG = {
|
|
|
17
17
|
relationTypes: ['implements', 'uses', 'depends_on', 'part_of', 'instance_of', 'derived_from', 'contrasts_with', 'causes'],
|
|
18
18
|
}
|
|
19
19
|
|
|
20
|
+
// Keys whose default is a number are consumed as scalars — an Array(n) length
|
|
21
|
+
// (bm25TitleWeight), a loop stride (batchSize), or a threshold comparison. Because
|
|
22
|
+
// wiki.config.json is shallow-merged as-is (any JSON type per key), a scalar there
|
|
23
|
+
// can be a string/object/NaN/0/negative, which crashes (`Array(NaN)` RangeError),
|
|
24
|
+
// hangs (`i += 0` never advances the batch loop), or silently disables a gate.
|
|
25
|
+
const NUMERIC_KEYS = Object.keys(DEFAULT_CONFIG).filter((k) => typeof DEFAULT_CONFIG[k] === 'number')
|
|
26
|
+
|
|
20
27
|
// Single reader for wiki.config.json (defaults merged) — every consumer used
|
|
21
28
|
// to inline this parse, each with its own bare-SyntaxError failure mode.
|
|
22
29
|
export function loadKbConfig(kbRoot) {
|
|
23
30
|
const p = kbPaths(kbRoot)
|
|
24
|
-
if (!fs.existsSync(p.config)) return DEFAULT_CONFIG
|
|
25
|
-
|
|
31
|
+
if (!fs.existsSync(p.config)) return { ...DEFAULT_CONFIG }
|
|
32
|
+
const merged = { ...DEFAULT_CONFIG, ...readJsonFile(p.config) }
|
|
33
|
+
// Coerce every numeric key back to a usable finite integer >= 1; fall back to the
|
|
34
|
+
// default on anything a user could type that isn't one. Every DEFAULT is itself a
|
|
35
|
+
// positive integer, so a legitimate override is untouched — only malformed values
|
|
36
|
+
// (which would crash/hang/mis-gate a consumer) are normalized away.
|
|
37
|
+
for (const k of NUMERIC_KEYS) {
|
|
38
|
+
const n = Math.trunc(Number(merged[k]))
|
|
39
|
+
merged[k] = Number.isFinite(n) && n >= 1 ? n : DEFAULT_CONFIG[k]
|
|
40
|
+
}
|
|
41
|
+
return merged
|
|
26
42
|
}
|
|
27
43
|
|
|
28
44
|
export function agentsMdTemplate(cfg) {
|
package/src/vector.mjs
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
import fs from 'node:fs'
|
|
2
2
|
import path from 'node:path'
|
|
3
3
|
import { kbPaths } from './paths.mjs'
|
|
4
|
-
import { readJsonFile } from './json.mjs'
|
|
4
|
+
import { readJsonFile, writeFileAtomic } from './json.mjs'
|
|
5
|
+
import { asList } from './pages.mjs'
|
|
5
6
|
|
|
6
7
|
export function normalize(vec) {
|
|
7
8
|
let s = 0
|
|
@@ -13,7 +14,7 @@ export function normalize(vec) {
|
|
|
13
14
|
|
|
14
15
|
// Same fields BM25 indexes (src/ask.mjs retrievePages) so both channels see one text.
|
|
15
16
|
export function pageEmbedText(pg) {
|
|
16
|
-
return [pg.data.title ?? '', pg.data.description ?? '', (pg.data.tags
|
|
17
|
+
return [pg.data.title ?? '', pg.data.description ?? '', asList(pg.data.tags).join(' '), pg.body].join('\n')
|
|
17
18
|
}
|
|
18
19
|
|
|
19
20
|
export function vectorStorePath(kbRoot) {
|
|
@@ -36,13 +37,10 @@ export function loadVectorStore(kbRoot) {
|
|
|
36
37
|
export function saveVectorStore(kbRoot, store) {
|
|
37
38
|
const rounded = { ...store, pages: Object.fromEntries(Object.entries(store.pages).map(([id, e]) =>
|
|
38
39
|
[id, { hash: e.hash, vec: e.vec.map(x => Number(x.toFixed(5))) }])) }
|
|
39
|
-
// Atomic write: the per-batch flush in embedKb re-saves
|
|
40
|
-
//
|
|
41
|
-
// temp+rename avoids the corruption entirely).
|
|
42
|
-
|
|
43
|
-
const tmp = `${f}.tmp`
|
|
44
|
-
fs.writeFileSync(tmp, JSON.stringify(rounded) + '\n')
|
|
45
|
-
fs.renameSync(tmp, f)
|
|
40
|
+
// Atomic write (shared writeFileAtomic): the per-batch flush in embedKb re-saves
|
|
41
|
+
// repeatedly; a crash or concurrent reader must never see a truncated store
|
|
42
|
+
// (loadVectorStore fails open, but temp+rename avoids the corruption entirely).
|
|
43
|
+
writeFileAtomic(vectorStorePath(kbRoot), JSON.stringify(rounded) + '\n')
|
|
46
44
|
}
|
|
47
45
|
|
|
48
46
|
// queryVec must already be normalized; stored vecs are normalized at save time,
|