llm-wiki-kit 0.2.14 → 0.2.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -4
- package/docs/concepts.md +14 -9
- package/docs/integrations/claude-code.md +4 -2
- package/docs/integrations/codex.md +4 -2
- package/docs/manual.md +51 -3
- package/docs/security.md +4 -0
- package/package.json +1 -1
- package/src/cli.js +52 -2
- package/src/constants.js +2 -0
- package/src/evidence.js +128 -0
- package/src/maintenance.js +127 -22
- package/src/project.js +5 -2
- package/src/templates.js +20 -0
- package/src/wiki-eval.js +110 -0
- package/src/wiki-export.js +214 -0
- package/src/wiki-lint.js +66 -1
- package/src/wiki-model.js +2 -0
- package/src/wiki-search.js +79 -13
- package/src/wiki-visibility.js +34 -4
package/src/maintenance.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { readdir, unlink } from 'fs/promises';
|
|
2
2
|
import { join, relative } from 'path';
|
|
3
|
-
import {
|
|
3
|
+
import { evidenceRefsFromEntry, normalizeEvidenceRefs, parseEvidenceRefsField } from './evidence.js';
|
|
4
|
+
import { appendText, exists, kitDataDir, readJson, readText, sha256, writeText, writeTextIfMissing } from './fs-utils.js';
|
|
4
5
|
import { classifyTurn, isMaintenanceRelatedQuery } from './capture-policy.js';
|
|
5
6
|
import { redactText, summarizeForStorage } from './redaction.js';
|
|
6
7
|
import { buildEntryFromTurnState, hasRecoverableTurnState } from './state.js';
|
|
@@ -24,7 +25,7 @@ function queueHeader() {
|
|
|
24
25
|
'',
|
|
25
26
|
'Candidates to merge into durable wiki pages. Hooks only create candidates; the active agent reviews and merges them into existing durable wiki documents.',
|
|
26
27
|
'',
|
|
27
|
-
'Status values: pending, done, skipped.',
|
|
28
|
+
'Status values: pending, approved, done, skipped.',
|
|
28
29
|
'',
|
|
29
30
|
].join('\n');
|
|
30
31
|
}
|
|
@@ -70,18 +71,51 @@ function itemId(projectRoot, source, entry) {
|
|
|
70
71
|
}
|
|
71
72
|
|
|
72
73
|
function itemBlock(item) {
|
|
73
|
-
|
|
74
|
+
const fields = {
|
|
75
|
+
...(item.fields || {}),
|
|
76
|
+
id: item.id,
|
|
77
|
+
created_at: item.created_at,
|
|
78
|
+
last_seen_at: item.last_seen_at,
|
|
79
|
+
source: item.source,
|
|
80
|
+
suggested_target: item.suggested_target,
|
|
81
|
+
target: item.target,
|
|
82
|
+
reviewed_at: item.reviewed_at,
|
|
83
|
+
review_note: item.review_note,
|
|
84
|
+
evidence_refs: normalizeEvidenceRefs(item.evidence_refs).length > 0
|
|
85
|
+
? JSON.stringify(normalizeEvidenceRefs(item.evidence_refs))
|
|
86
|
+
: item.fields?.evidence_refs,
|
|
87
|
+
reason: item.reason,
|
|
88
|
+
result_missing: item.result_missing ? 'true' : 'false',
|
|
89
|
+
};
|
|
90
|
+
const order = [
|
|
91
|
+
'id',
|
|
92
|
+
'created_at',
|
|
93
|
+
'last_seen_at',
|
|
94
|
+
'source',
|
|
95
|
+
'suggested_target',
|
|
96
|
+
'target',
|
|
97
|
+
'reviewed_at',
|
|
98
|
+
'review_note',
|
|
99
|
+
'evidence_refs',
|
|
100
|
+
'reason',
|
|
101
|
+
'result_missing',
|
|
102
|
+
];
|
|
103
|
+
const lines = [
|
|
74
104
|
`## ${item.status || 'pending'} - ${item.topic || 'maintenance item'}`,
|
|
75
105
|
'',
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
`-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
106
|
+
];
|
|
107
|
+
const emitted = new Set();
|
|
108
|
+
for (const key of order) {
|
|
109
|
+
if (fields[key] === undefined || fields[key] === '') continue;
|
|
110
|
+
lines.push(`- ${key}: ${fields[key]}`);
|
|
111
|
+
emitted.add(key);
|
|
112
|
+
}
|
|
113
|
+
for (const key of Object.keys(fields).sort()) {
|
|
114
|
+
if (emitted.has(key) || fields[key] === undefined || fields[key] === '') continue;
|
|
115
|
+
lines.push(`- ${key}: ${fields[key]}`);
|
|
116
|
+
}
|
|
117
|
+
lines.push('');
|
|
118
|
+
return lines.join('\n');
|
|
85
119
|
}
|
|
86
120
|
|
|
87
121
|
export async function readMaintenanceQueue(projectRoot) {
|
|
@@ -96,7 +130,7 @@ export async function readMaintenanceQueue(projectRoot) {
|
|
|
96
130
|
};
|
|
97
131
|
|
|
98
132
|
for (const line of text.split(/\r?\n/)) {
|
|
99
|
-
const header = line.match(/^##\s+(pending|done|skipped)\s+-\s*(.*)$/);
|
|
133
|
+
const header = line.match(/^##\s+(pending|approved|done|skipped)\s+-\s*(.*)$/);
|
|
100
134
|
if (header) {
|
|
101
135
|
pushCurrent();
|
|
102
136
|
current = {
|
|
@@ -107,7 +141,7 @@ export async function readMaintenanceQueue(projectRoot) {
|
|
|
107
141
|
continue;
|
|
108
142
|
}
|
|
109
143
|
if (!current) continue;
|
|
110
|
-
const field = line.match(/^-\s+([
|
|
144
|
+
const field = line.match(/^-\s+([A-Za-z0-9_-]+):\s*(.*)$/);
|
|
111
145
|
if (field) {
|
|
112
146
|
current.fields[field[1]] = field[2].trim();
|
|
113
147
|
}
|
|
@@ -119,19 +153,35 @@ export async function readMaintenanceQueue(projectRoot) {
|
|
|
119
153
|
exists: await exists(path),
|
|
120
154
|
items: items.map((item) => ({
|
|
121
155
|
...item.fields,
|
|
156
|
+
fields: item.fields,
|
|
122
157
|
status: item.status,
|
|
123
158
|
topic: item.topic,
|
|
159
|
+
evidence_refs: parseEvidenceRefsField(item.fields.evidence_refs),
|
|
160
|
+
evidenceRefs: parseEvidenceRefsField(item.fields.evidence_refs),
|
|
124
161
|
result_missing: String(item.fields.result_missing || '').toLowerCase() === 'true',
|
|
125
162
|
})),
|
|
126
163
|
};
|
|
127
164
|
}
|
|
128
165
|
|
|
166
|
+
async function writeMaintenanceQueue(projectRoot, items) {
|
|
167
|
+
const content = [
|
|
168
|
+
queueHeader().trimEnd(),
|
|
169
|
+
'',
|
|
170
|
+
...items.map((item) => itemBlock(item).trimEnd()),
|
|
171
|
+
'',
|
|
172
|
+
].join('\n');
|
|
173
|
+
const path = queuePath(projectRoot);
|
|
174
|
+
await writeText(path, content);
|
|
175
|
+
return path;
|
|
176
|
+
}
|
|
177
|
+
|
|
129
178
|
export function summarizeMaintenanceQueue(queue, options = {}) {
|
|
130
179
|
const staleDays = options.staleDays ?? DEFAULT_STALE_PENDING_DAYS;
|
|
131
180
|
const pendingLimit = options.pendingLimit ?? DEFAULT_PENDING_LIMIT;
|
|
132
181
|
const reviewPendingLimit = options.reviewPendingLimit ?? DEFAULT_REVIEW_PENDING_LIMIT;
|
|
133
182
|
const reviewIntervalDays = options.reviewIntervalDays ?? DEFAULT_REVIEW_INTERVAL_DAYS;
|
|
134
183
|
const pending = queue.items.filter((item) => item.status === 'pending');
|
|
184
|
+
const approved = queue.items.filter((item) => item.status === 'approved');
|
|
135
185
|
const staleCutoff = Date.now() - staleDays * 24 * 60 * 60 * 1000;
|
|
136
186
|
const stalePending = pending.filter((item) => {
|
|
137
187
|
const time = Date.parse(item.created_at || item.last_seen_at || '');
|
|
@@ -143,6 +193,7 @@ export function summarizeMaintenanceQueue(queue, options = {}) {
|
|
|
143
193
|
.filter(Number.isFinite);
|
|
144
194
|
const lastReviewMs = reviewTimes.length > 0 ? Math.max(...reviewTimes) : null;
|
|
145
195
|
const reviewReasons = [];
|
|
196
|
+
if (approved.length > 0) reviewReasons.push(`approved queue has ${approved.length} item(s) ready for durable wiki merge`);
|
|
146
197
|
if (pending.length >= reviewPendingLimit) reviewReasons.push(`pending queue has ${pending.length} items (threshold ${reviewPendingLimit})`);
|
|
147
198
|
if (stalePending.length > 0) reviewReasons.push(`${stalePending.length} pending item(s) older than ${staleDays} days`);
|
|
148
199
|
if (pending.some((item) => item.result_missing)) reviewReasons.push('pending recovered turn state needs review');
|
|
@@ -153,9 +204,11 @@ export function summarizeMaintenanceQueue(queue, options = {}) {
|
|
|
153
204
|
path: queue.path,
|
|
154
205
|
exists: queue.exists,
|
|
155
206
|
items: queue.items,
|
|
207
|
+
approved,
|
|
156
208
|
pending,
|
|
157
209
|
done: queue.items.filter((item) => item.status === 'done'),
|
|
158
210
|
skipped: queue.items.filter((item) => item.status === 'skipped'),
|
|
211
|
+
approvedCount: approved.length,
|
|
159
212
|
pendingCount: pending.length,
|
|
160
213
|
doneCount: queue.items.filter((item) => item.status === 'done').length,
|
|
161
214
|
skippedCount: queue.items.filter((item) => item.status === 'skipped').length,
|
|
@@ -246,6 +299,47 @@ export async function appendMaintenanceItem(projectRoot, item) {
|
|
|
246
299
|
return { created: true, path, id: item.id };
|
|
247
300
|
}
|
|
248
301
|
|
|
302
|
+
export async function updateMaintenanceItem(projectRoot, id, action, options = {}) {
|
|
303
|
+
const allowed = new Set(['approve', 'done', 'skip']);
|
|
304
|
+
if (!allowed.has(action)) throw new Error(`unsupported maintenance action: ${action}`);
|
|
305
|
+
const queue = await readMaintenanceQueue(projectRoot);
|
|
306
|
+
const index = queue.items.findIndex((item) => item.id === id);
|
|
307
|
+
if (index === -1) throw new Error(`maintenance item not found: ${id}`);
|
|
308
|
+
if ((action === 'approve' || action === 'done') && !options.target && !queue.items[index].target) {
|
|
309
|
+
throw new Error(`--target is required for maintenance --${action}`);
|
|
310
|
+
}
|
|
311
|
+
const now = nowIso();
|
|
312
|
+
const current = queue.items[index];
|
|
313
|
+
const next = {
|
|
314
|
+
...current,
|
|
315
|
+
fields: { ...(current.fields || {}) },
|
|
316
|
+
status: action === 'approve' ? 'approved' : (action === 'done' ? 'done' : 'skipped'),
|
|
317
|
+
last_seen_at: now,
|
|
318
|
+
reviewed_at: now,
|
|
319
|
+
target: options.target || current.target || '',
|
|
320
|
+
review_note: sanitizeField(options.note || current.review_note || '', 500),
|
|
321
|
+
};
|
|
322
|
+
if (next.target) next.fields.target = next.target;
|
|
323
|
+
next.fields.last_seen_at = next.last_seen_at;
|
|
324
|
+
next.fields.reviewed_at = next.reviewed_at;
|
|
325
|
+
if (next.review_note) next.fields.review_note = next.review_note;
|
|
326
|
+
const refs = normalizeEvidenceRefs(options.evidence_refs || current.evidence_refs || current.evidenceRefs || []);
|
|
327
|
+
if (refs.length > 0) {
|
|
328
|
+
next.evidence_refs = refs;
|
|
329
|
+
next.fields.evidence_refs = JSON.stringify(refs);
|
|
330
|
+
}
|
|
331
|
+
queue.items[index] = next;
|
|
332
|
+
await writeMaintenanceQueue(projectRoot, queue.items);
|
|
333
|
+
return {
|
|
334
|
+
workspace: projectRoot,
|
|
335
|
+
path: queue.path,
|
|
336
|
+
id,
|
|
337
|
+
status: next.status,
|
|
338
|
+
target: next.target || null,
|
|
339
|
+
reviewed_at: next.reviewed_at,
|
|
340
|
+
};
|
|
341
|
+
}
|
|
342
|
+
|
|
249
343
|
export async function recordMaintenanceForEntry(projectRoot, entry, options = {}) {
|
|
250
344
|
const source = relativeSource(projectRoot, options.source || options.queryPath || options.decisionPath || options.liveQaPath || '');
|
|
251
345
|
if (!source && !options.resultMissing) return { created: false, reason: 'missing-source' };
|
|
@@ -259,6 +353,7 @@ export async function recordMaintenanceForEntry(projectRoot, entry, options = {}
|
|
|
259
353
|
last_seen_at: created,
|
|
260
354
|
source,
|
|
261
355
|
suggested_target: options.suggestedTarget || inferSuggestedTarget(entry, source),
|
|
356
|
+
evidence_refs: evidenceRefsFromEntry(entry, { projectRoot }),
|
|
262
357
|
reason: sanitizeField(options.reason || `Captured ${options.eventName || 'turn'} needs durable wiki review.`, 300),
|
|
263
358
|
result_missing: Boolean(options.resultMissing),
|
|
264
359
|
};
|
|
@@ -313,31 +408,34 @@ export function formatMaintenanceContext(summary, options = {}) {
|
|
|
313
408
|
const eventName = options.eventName || '';
|
|
314
409
|
const defaultLimit = eventName === 'SessionStart' || eventName === 'InstructionsLoaded' ? 1 : 5;
|
|
315
410
|
const limit = options.limit || defaultLimit;
|
|
316
|
-
|
|
411
|
+
const reviewCandidates = [...(summary.approved || []), ...(summary.pending || [])];
|
|
412
|
+
let pending = reviewCandidates.slice(0, limit);
|
|
317
413
|
|
|
318
414
|
if (eventName === 'UserPromptSubmit') {
|
|
319
|
-
if (!isMaintenanceRelatedQuery(options.query || '',
|
|
320
|
-
pending =
|
|
415
|
+
if (!isMaintenanceRelatedQuery(options.query || '', reviewCandidates)) return '';
|
|
416
|
+
pending = reviewCandidates.slice(0, 1);
|
|
321
417
|
} else if (eventName !== 'SessionStart' && eventName !== 'InstructionsLoaded') {
|
|
322
418
|
return '';
|
|
323
419
|
}
|
|
324
420
|
|
|
421
|
+
const reviewCount = (summary.approvedCount || 0) + (summary.pendingCount || 0);
|
|
325
422
|
const lines = language === 'ko'
|
|
326
423
|
? [
|
|
327
424
|
'LLM Wiki maintenance status:',
|
|
328
425
|
`- review due: yes (${(summary.reviewReasons || []).slice(0, 2).join('; ') || 'periodic review threshold met'}).`,
|
|
329
|
-
`- pending review items: ${
|
|
426
|
+
`- approved/pending review items: ${reviewCount}. 현재 요청이 우선이며, 관련 있을 때만 durable wiki 정리에 사용한다.`,
|
|
330
427
|
]
|
|
331
428
|
: [
|
|
332
429
|
'LLM Wiki maintenance status:',
|
|
333
430
|
`- review due: yes (${(summary.reviewReasons || []).slice(0, 2).join('; ') || 'periodic review threshold met'}).`,
|
|
334
|
-
`- pending review items: ${
|
|
431
|
+
`- approved/pending review items: ${reviewCount}. The current request comes first; use this only when it is relevant to durable wiki cleanup.`,
|
|
335
432
|
];
|
|
336
433
|
for (const item of pending) {
|
|
337
|
-
|
|
434
|
+
const target = item.target || item.suggested_target;
|
|
435
|
+
lines.push(`- ${item.status}: ${item.topic || item.id}: ${target}; source=${item.source}${item.result_missing ? '; result missing' : ''}`);
|
|
338
436
|
}
|
|
339
|
-
if (
|
|
340
|
-
lines.push(`- ${
|
|
437
|
+
if (reviewCandidates.length > pending.length) {
|
|
438
|
+
lines.push(`- ${reviewCandidates.length - pending.length} more review item(s) available in llm-wiki/outputs/maintenance/queue.md.`);
|
|
341
439
|
}
|
|
342
440
|
return lines.join('\n');
|
|
343
441
|
}
|
|
@@ -346,6 +444,7 @@ export function formatMaintenanceResult(summary) {
|
|
|
346
444
|
const lines = [
|
|
347
445
|
'llm-wiki maintenance',
|
|
348
446
|
`- queue: ${summary.path}`,
|
|
447
|
+
`- approved: ${summary.approvedCount || 0}`,
|
|
349
448
|
`- pending: ${summary.pendingCount}`,
|
|
350
449
|
`- stale pending: ${summary.stalePendingCount || 0}`,
|
|
351
450
|
`- done: ${summary.doneCount}`,
|
|
@@ -355,6 +454,12 @@ export function formatMaintenanceResult(summary) {
|
|
|
355
454
|
if ((summary.reviewReasons || []).length > 0) {
|
|
356
455
|
lines.push(`- review reasons: ${summary.reviewReasons.join('; ')}`);
|
|
357
456
|
}
|
|
457
|
+
if ((summary.approved || []).length > 0) {
|
|
458
|
+
lines.push('', 'Approved:');
|
|
459
|
+
for (const item of summary.approved.slice(0, 10)) {
|
|
460
|
+
lines.push(`- ${item.topic || item.id}: ${item.source} -> ${item.target || item.suggested_target}`);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
358
463
|
if (summary.pending.length > 0) {
|
|
359
464
|
lines.push('', 'Pending:');
|
|
360
465
|
for (const item of summary.pending.slice(0, 10)) {
|
package/src/project.js
CHANGED
|
@@ -9,6 +9,7 @@ import {
|
|
|
9
9
|
writeTextIfMissing,
|
|
10
10
|
} from './fs-utils.js';
|
|
11
11
|
import { LLM_WIKI_DIRS } from './constants.js';
|
|
12
|
+
import { evidenceRefsFromEntry, frontmatterEvidenceRefs } from './evidence.js';
|
|
12
13
|
import { formatMaintenanceContext, maintenanceSummary } from './maintenance.js';
|
|
13
14
|
import { normalizeForStorage, redactText, summarizeForStorage } from './redaction.js';
|
|
14
15
|
import { gitignore, indexPage, llmWikiAgents, logPage, memoryPage, procedure, rootAgentsPolicy } from './templates.js';
|
|
@@ -107,7 +108,8 @@ export async function writeQueryPage(projectRoot, entry) {
|
|
|
107
108
|
const slug = slugify(entry.question, 'query');
|
|
108
109
|
const path = join(projectRoot, 'llm-wiki', 'wiki', 'queries', `${day}-${slug}.md`);
|
|
109
110
|
if (await exists(path)) return path;
|
|
110
|
-
const
|
|
111
|
+
const evidenceRefs = frontmatterEvidenceRefs(evidenceRefsFromEntry(entry, { projectRoot }));
|
|
112
|
+
const content = `---\ntitle: "${entry.topic || slug}"\ntype: "query"\nsource_ids: []\n${evidenceRefs}\nstatus: "draft"\nlast_updated: "${day}"\nconfidence: "medium"\nmemory_type: "episodic"\nimportance: 2\nlast_verified: "unknown"\nsupersedes: []\nsuperseded_by: []\n---\n\n# ${entry.topic || entry.question.slice(0, 80)}\n\n## Question\n${entry.question}\n\n## Answer Summary\n${entry.result || '(not captured)'}\n\n## Work Notes\n${entry.work || '(not captured)'}\n\n## Verification\n${entry.verification || '(not captured)'}\n\n## Related Pages\n- [[index]]\n\n## Change Log\n- ${day}: Captured automatically by llm-wiki-kit hook.\n`;
|
|
111
113
|
await writeTextIfMissing(path, redactText(content, 12000));
|
|
112
114
|
return path;
|
|
113
115
|
}
|
|
@@ -120,7 +122,8 @@ export async function writeDecisionPage(projectRoot, entry) {
|
|
|
120
122
|
const slug = slugify(entry.topic || entry.question || 'decision', 'decision');
|
|
121
123
|
const path = join(projectRoot, 'llm-wiki', 'wiki', 'decisions', `${day}-${slug}.md`);
|
|
122
124
|
if (await exists(path)) return path;
|
|
123
|
-
const
|
|
125
|
+
const evidenceRefs = frontmatterEvidenceRefs(evidenceRefsFromEntry(entry, { projectRoot }));
|
|
126
|
+
const content = `---\ntitle: "${entry.topic || slug}"\ntype: "decision"\nsource_ids: []\n${evidenceRefs}\nstatus: "draft"\nlast_updated: "${day}"\nconfidence: "medium"\nmemory_type: "semantic"\nimportance: 4\nlast_verified: "unknown"\nsupersedes: []\nsuperseded_by: []\n---\n\n# ${entry.topic || 'Decision'}\n\n## Decision\n${entry.result || '(captured from assistant response; review needed)'}\n\n## Context\n${entry.question || '(not captured)'}\n\n## Evidence\n${entry.work || '(not captured)'}\n\n## Verification\n${entry.verification || '(not captured)'}\n\n## Open Questions\n${entry.followUp || '(none captured)'}\n\n## Change Log\n- ${day}: Captured automatically by llm-wiki-kit hook.\n`;
|
|
124
127
|
await writeTextIfMissing(path, redactText(content, 12000));
|
|
125
128
|
return path;
|
|
126
129
|
}
|
package/src/templates.js
CHANGED
|
@@ -22,6 +22,15 @@ This repository uses llm-wiki-kit as a hook-first living Markdown wiki for Codex
|
|
|
22
22
|
- Record verification commands, evidence files, and uncertainty. Mark inference explicitly and preserve contradictions in Open Questions or Contradictions.
|
|
23
23
|
- Never store credentials, tokens, passwords, private keys, or raw \`.env\` contents. Store only redacted summaries when needed.
|
|
24
24
|
|
|
25
|
+
### llm-wiki-kit Implementation Plans
|
|
26
|
+
|
|
27
|
+
- When implementing an approved llm-wiki-kit plan, read the complete plan first and treat it as the implementation source of truth.
|
|
28
|
+
- A valid plan must include history/context, lifecycle intent, exact source paths, test strategy, release verification, and commit/push requirements.
|
|
29
|
+
- Do not implement feature lists in isolation. Each feature must be integrated into the hook-first, answer-first, local Markdown knowledge lifecycle.
|
|
30
|
+
- \`llms.txt\`/exports are not passive artifacts; they are agent onboarding, handoff, retrieval-eval, and external-consumption manifests.
|
|
31
|
+
- \`evidence_refs\`, maintenance review, context ranking explanations, export, and eval must share the same durable wiki visibility policy.
|
|
32
|
+
- Never store npm tokens, WinRM credentials, private keys, raw \`.env\`, or full raw transcripts in wiki, logs, generated exports, or commits.
|
|
33
|
+
|
|
25
34
|
<!-- llm-wiki-kit:end -->
|
|
26
35
|
`;
|
|
27
36
|
}
|
|
@@ -62,6 +71,15 @@ These rules replace older OMX/OMC/\`omx_wiki/\` rules for this project.
|
|
|
62
71
|
- Preserve contradictions in \`Contradictions\` or \`Open Questions\`; do not overwrite them silently.
|
|
63
72
|
- Do not store credentials, tokens, passwords, private keys, or raw \`.env\` contents in wiki.
|
|
64
73
|
|
|
74
|
+
### llm-wiki-kit Implementation Plans
|
|
75
|
+
|
|
76
|
+
- When implementing an approved llm-wiki-kit plan, read the complete plan first and treat it as the implementation source of truth.
|
|
77
|
+
- A valid plan must include history/context, lifecycle intent, exact source paths, test strategy, release verification, and commit/push requirements.
|
|
78
|
+
- Do not implement feature lists in isolation. Each feature must be integrated into the hook-first, answer-first, local Markdown knowledge lifecycle.
|
|
79
|
+
- \`llms.txt\`/exports are not passive artifacts; they are agent onboarding, handoff, retrieval-eval, and external-consumption manifests.
|
|
80
|
+
- \`evidence_refs\`, maintenance review, context ranking explanations, export, and eval must share the same durable wiki visibility policy.
|
|
81
|
+
- Never store npm tokens, WinRM credentials, private keys, raw \`.env\`, or full raw transcripts in wiki, logs, generated exports, or commits.
|
|
82
|
+
|
|
65
83
|
## Page Format
|
|
66
84
|
Use YAML frontmatter when creating wiki pages:
|
|
67
85
|
|
|
@@ -70,6 +88,7 @@ Use YAML frontmatter when creating wiki pages:
|
|
|
70
88
|
title: ""
|
|
71
89
|
type: "source | concept | entity | decision | architecture | debugging | context | query | session-log | convention"
|
|
72
90
|
source_ids: []
|
|
91
|
+
evidence_refs: []
|
|
73
92
|
status: "draft | reviewed | stale | archived"
|
|
74
93
|
last_updated: "YYYY-MM-DD"
|
|
75
94
|
confidence: "high | medium | low"
|
|
@@ -131,6 +150,7 @@ export function memoryPage() {
|
|
|
131
150
|
title: "LLM Wiki Memory"
|
|
132
151
|
type: "context"
|
|
133
152
|
source_ids: []
|
|
153
|
+
evidence_refs: []
|
|
134
154
|
status: "draft"
|
|
135
155
|
last_updated: "unknown"
|
|
136
156
|
confidence: "medium"
|
package/src/wiki-eval.js
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
import { join } from 'path';
|
|
2
|
+
import { exists, readJson } from './fs-utils.js';
|
|
3
|
+
import { buildContextPack } from './wiki-search.js';
|
|
4
|
+
import { DURABLE_VISIBILITY_POLICY } from './wiki-visibility.js';
|
|
5
|
+
|
|
6
|
+
export const DEFAULT_EVAL_FIXTURE_REL = 'llm-wiki/evals/retrieval.json';
|
|
7
|
+
|
|
8
|
+
function normalizePath(value) {
|
|
9
|
+
return String(value || '').replace(/\\/g, '/').replace(/^\.\//, '');
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function fixturePath(projectRoot, options = {}) {
|
|
13
|
+
return options.fixture || join(projectRoot, DEFAULT_EVAL_FIXTURE_REL);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function normalizeFixture(raw) {
|
|
17
|
+
const queries = Array.isArray(raw?.queries) ? raw.queries : [];
|
|
18
|
+
return queries.map((item) => ({
|
|
19
|
+
query: String(item?.query || '').trim(),
|
|
20
|
+
expected: Array.isArray(item?.expected) ? item.expected.map(normalizePath).filter(Boolean) : [],
|
|
21
|
+
unexpected: Array.isArray(item?.unexpected) ? item.unexpected.map(normalizePath).filter(Boolean) : [],
|
|
22
|
+
})).filter((item) => item.query);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export async function runEval(projectRoot, options = {}) {
|
|
26
|
+
const path = fixturePath(projectRoot, options);
|
|
27
|
+
if (!(await exists(path))) {
|
|
28
|
+
return {
|
|
29
|
+
workspace: projectRoot,
|
|
30
|
+
fixture: path,
|
|
31
|
+
ok: true,
|
|
32
|
+
status: 'missing-fixture',
|
|
33
|
+
message: 'no fixture found',
|
|
34
|
+
visibilityPolicy: DURABLE_VISIBILITY_POLICY,
|
|
35
|
+
warnings: [],
|
|
36
|
+
queries: [],
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
const fixture = await readJson(path, null);
|
|
40
|
+
if (!fixture) throw new Error(`invalid eval fixture JSON: ${path}`);
|
|
41
|
+
const limit = Number(options.limit || 5);
|
|
42
|
+
const rows = [];
|
|
43
|
+
for (const item of normalizeFixture(fixture)) {
|
|
44
|
+
const pack = await buildContextPack(projectRoot, item.query, {
|
|
45
|
+
...options,
|
|
46
|
+
limit,
|
|
47
|
+
expand: options.expand !== false,
|
|
48
|
+
});
|
|
49
|
+
const topHits = pack.hits.map((hit) => hit.path);
|
|
50
|
+
const hitSet = new Set(topHits);
|
|
51
|
+
const expectedHits = item.expected.filter((expected) => hitSet.has(expected));
|
|
52
|
+
const missedExpected = item.expected.filter((expected) => !hitSet.has(expected));
|
|
53
|
+
const unexpectedHits = item.unexpected.filter((unexpected) => hitSet.has(unexpected));
|
|
54
|
+
rows.push({
|
|
55
|
+
query: item.query,
|
|
56
|
+
ok: missedExpected.length === 0 && unexpectedHits.length === 0,
|
|
57
|
+
recall: item.expected.length === 0 ? 1 : expectedHits.length / item.expected.length,
|
|
58
|
+
expected: item.expected,
|
|
59
|
+
expectedHits,
|
|
60
|
+
missedExpected,
|
|
61
|
+
unexpected: item.unexpected,
|
|
62
|
+
unexpectedHits,
|
|
63
|
+
topHits,
|
|
64
|
+
search: pack.search,
|
|
65
|
+
});
|
|
66
|
+
}
|
|
67
|
+
const warnings = [];
|
|
68
|
+
if (fixture.visibilityPolicy && fixture.visibilityPolicy !== DURABLE_VISIBILITY_POLICY) {
|
|
69
|
+
warnings.push(`fixture visibility policy ${fixture.visibilityPolicy} differs from export/eval policy ${DURABLE_VISIBILITY_POLICY}`);
|
|
70
|
+
}
|
|
71
|
+
return {
|
|
72
|
+
workspace: projectRoot,
|
|
73
|
+
fixture: path,
|
|
74
|
+
ok: rows.every((row) => row.ok),
|
|
75
|
+
status: 'evaluated',
|
|
76
|
+
visibilityPolicy: DURABLE_VISIBILITY_POLICY,
|
|
77
|
+
warnings,
|
|
78
|
+
queries: rows,
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export function formatEvalResult(result) {
|
|
83
|
+
if (result.status === 'missing-fixture') {
|
|
84
|
+
return [
|
|
85
|
+
'llm-wiki eval',
|
|
86
|
+
`- workspace: ${result.workspace}`,
|
|
87
|
+
`- fixture: ${result.fixture}`,
|
|
88
|
+
'- result: no fixture found',
|
|
89
|
+
].join('\n');
|
|
90
|
+
}
|
|
91
|
+
const lines = [
|
|
92
|
+
'llm-wiki eval',
|
|
93
|
+
`- workspace: ${result.workspace}`,
|
|
94
|
+
`- fixture: ${result.fixture}`,
|
|
95
|
+
`- result: ${result.ok ? 'ok' : 'failed'}`,
|
|
96
|
+
`- visibility policy: ${result.visibilityPolicy}`,
|
|
97
|
+
`- queries: ${result.queries.length}`,
|
|
98
|
+
];
|
|
99
|
+
for (const warning of result.warnings || []) {
|
|
100
|
+
lines.push(`- warning: ${warning}`);
|
|
101
|
+
}
|
|
102
|
+
for (const row of result.queries) {
|
|
103
|
+
lines.push('', `## ${row.query}`);
|
|
104
|
+
lines.push(`- recall: ${row.expectedHits.length}/${row.expected.length}`);
|
|
105
|
+
lines.push(`- missed expected: ${row.missedExpected.join(', ') || 'none'}`);
|
|
106
|
+
lines.push(`- unexpected hits: ${row.unexpectedHits.join(', ') || 'none'}`);
|
|
107
|
+
lines.push(`- top hits: ${row.topHits.join(', ') || 'none'}`);
|
|
108
|
+
}
|
|
109
|
+
return lines.join('\n');
|
|
110
|
+
}
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
import { join, relative } from 'path';
|
|
2
|
+
import { ensureDir, writeText } from './fs-utils.js';
|
|
3
|
+
import { hasSecretLikeText, redactText } from './redaction.js';
|
|
4
|
+
import { collectWikiPages } from './wiki-model.js';
|
|
5
|
+
import { DURABLE_VISIBILITY_POLICY, pageImportance, visibleWikiPages } from './wiki-visibility.js';
|
|
6
|
+
|
|
7
|
+
export const DEFAULT_EXPORT_DIR_REL = 'llm-wiki/outputs/exports';
|
|
8
|
+
const DEFAULT_FULL_MAX_BYTES = 200 * 1024;
|
|
9
|
+
const EXPORT_SCHEMA_VERSION = 1;
|
|
10
|
+
|
|
11
|
+
function outputDir(projectRoot, options = {}) {
|
|
12
|
+
return options.output ? options.output : join(projectRoot, DEFAULT_EXPORT_DIR_REL);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function normalizeFormat(format) {
|
|
16
|
+
const value = String(format || 'all').toLowerCase();
|
|
17
|
+
if (!['all', 'llms', 'llms-full', 'json'].includes(value)) {
|
|
18
|
+
throw new Error('--format must be one of all, llms, llms-full, json');
|
|
19
|
+
}
|
|
20
|
+
return value;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
function selectedNames(format) {
|
|
24
|
+
if (format === 'all') return ['llms.txt', 'llms-full.txt', 'llm-wiki.json'];
|
|
25
|
+
if (format === 'llms') return ['llms.txt'];
|
|
26
|
+
if (format === 'llms-full') return ['llms-full.txt'];
|
|
27
|
+
return ['llm-wiki.json'];
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function sortPages(pages) {
|
|
31
|
+
return [...pages].sort((a, b) => {
|
|
32
|
+
const importanceDiff = pageImportance(b) - pageImportance(a);
|
|
33
|
+
if (importanceDiff !== 0) return importanceDiff;
|
|
34
|
+
const typeDiff = String(a.type || '').localeCompare(String(b.type || ''));
|
|
35
|
+
if (typeDiff !== 0) return typeDiff;
|
|
36
|
+
return a.rel.localeCompare(b.rel);
|
|
37
|
+
});
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
function pageSummary(page) {
|
|
41
|
+
const bits = [
|
|
42
|
+
page.type || 'wiki',
|
|
43
|
+
page.memoryType ? `memory:${page.memoryType}` : '',
|
|
44
|
+
page.status ? `status:${page.status}` : '',
|
|
45
|
+
page.confidence ? `confidence:${page.confidence}` : '',
|
|
46
|
+
page.frontmatter?.importance ? `importance:${page.frontmatter.importance}` : '',
|
|
47
|
+
].filter(Boolean);
|
|
48
|
+
if ((page.evidenceRefs || []).length > 0) bits.push(`evidence:${page.evidenceRefs.length}`);
|
|
49
|
+
return redactText(bits.join(', '), 500);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function linkFromExport(projectRoot, exportDir, page) {
|
|
53
|
+
const relPath = relative(exportDir, join(projectRoot, 'llm-wiki', page.rel)).split('\\').join('/');
|
|
54
|
+
return relPath.startsWith('.') ? relPath : `./${relPath}`;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function pagesByType(pages) {
|
|
58
|
+
const groups = new Map();
|
|
59
|
+
for (const page of pages) {
|
|
60
|
+
const type = page.type || 'wiki';
|
|
61
|
+
if (!groups.has(type)) groups.set(type, []);
|
|
62
|
+
groups.get(type).push(page);
|
|
63
|
+
}
|
|
64
|
+
return [...groups.entries()].sort(([a], [b]) => a.localeCompare(b));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function renderLlmsTxt(projectRoot, exportDir, pages, metadata) {
|
|
68
|
+
const lines = [
|
|
69
|
+
'# LLM Wiki',
|
|
70
|
+
'',
|
|
71
|
+
'> Durable local Markdown knowledge manifest generated by llm-wiki-kit for agent onboarding, handoff, retrieval evaluation, and external consumption.',
|
|
72
|
+
'',
|
|
73
|
+
`- workspace: ${redactText(projectRoot, 500)}`,
|
|
74
|
+
`- generated_at: ${metadata.generatedAt}`,
|
|
75
|
+
`- visibility_policy: ${metadata.visibilityPolicy}`,
|
|
76
|
+
`- durable_pages: ${pages.length}`,
|
|
77
|
+
];
|
|
78
|
+
for (const [type, group] of pagesByType(pages)) {
|
|
79
|
+
lines.push('', `## ${type}`);
|
|
80
|
+
for (const page of group) {
|
|
81
|
+
lines.push(`- [${redactText(page.title, 200)}](${linkFromExport(projectRoot, exportDir, page)}): ${pageSummary(page)}`);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return `${redactText(lines.join('\n'), 120000)}\n`;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function pageBody(page) {
|
|
88
|
+
const body = page.body.trim() || page.content.trim();
|
|
89
|
+
return redactText(body, 50000);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function renderLlmsFullTxt(projectRoot, pages, metadata, options = {}) {
|
|
93
|
+
const maxBytes = options.maxBytes || DEFAULT_FULL_MAX_BYTES;
|
|
94
|
+
const lines = [
|
|
95
|
+
'# LLM Wiki Full Context',
|
|
96
|
+
'',
|
|
97
|
+
`Generated: ${metadata.generatedAt}`,
|
|
98
|
+
`Visibility policy: ${metadata.visibilityPolicy}`,
|
|
99
|
+
`Workspace: ${redactText(projectRoot, 500)}`,
|
|
100
|
+
'',
|
|
101
|
+
];
|
|
102
|
+
for (const page of sortPages(pages)) {
|
|
103
|
+
const block = [
|
|
104
|
+
`## ${redactText(page.title, 200)}`,
|
|
105
|
+
'',
|
|
106
|
+
`Path: ${page.rel}`,
|
|
107
|
+
`Metadata: ${pageSummary(page)}`,
|
|
108
|
+
(page.evidenceRefs || []).length > 0 ? `Evidence refs: ${page.evidenceRefs.map((ref) => redactText(ref, 700)).join(', ')}` : '',
|
|
109
|
+
'',
|
|
110
|
+
pageBody(page),
|
|
111
|
+
'',
|
|
112
|
+
].filter((line) => line !== '').join('\n');
|
|
113
|
+
const candidate = `${lines.join('\n')}${block}\n`;
|
|
114
|
+
if (Buffer.byteLength(candidate, 'utf8') > maxBytes) {
|
|
115
|
+
lines.push(`[TRUNCATED: full export reached ${maxBytes} bytes]`);
|
|
116
|
+
break;
|
|
117
|
+
}
|
|
118
|
+
lines.push(block);
|
|
119
|
+
}
|
|
120
|
+
return `${redactText(lines.join('\n'), maxBytes + 2000)}\n`;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function pageManifest(page) {
|
|
124
|
+
const body = page.body.replace(/\s+/g, ' ').trim();
|
|
125
|
+
return {
|
|
126
|
+
path: page.rel,
|
|
127
|
+
title: redactText(page.title, 300),
|
|
128
|
+
type: page.type || '',
|
|
129
|
+
status: page.status || '',
|
|
130
|
+
confidence: page.confidence || '',
|
|
131
|
+
memoryType: page.memoryType || '',
|
|
132
|
+
importance: pageImportance(page),
|
|
133
|
+
lastUpdated: page.frontmatter?.last_updated || '',
|
|
134
|
+
lastVerified: page.frontmatter?.last_verified || '',
|
|
135
|
+
sourceIds: Array.isArray(page.sourceIds) ? page.sourceIds.map((item) => redactText(item, 300)) : [],
|
|
136
|
+
evidenceRefs: Array.isArray(page.evidenceRefs) ? page.evidenceRefs.map((item) => redactText(item, 700)) : [],
|
|
137
|
+
snippet: redactText(body.slice(0, 500), 700),
|
|
138
|
+
redacted: hasSecretLikeText(page.content),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function renderJson(projectRoot, pages, metadata, skipped) {
|
|
143
|
+
return `${JSON.stringify({
|
|
144
|
+
schemaVersion: EXPORT_SCHEMA_VERSION,
|
|
145
|
+
workspace: redactText(projectRoot, 500),
|
|
146
|
+
generatedAt: metadata.generatedAt,
|
|
147
|
+
visibilityPolicy: metadata.visibilityPolicy,
|
|
148
|
+
counts: {
|
|
149
|
+
exportedPages: pages.length,
|
|
150
|
+
hiddenPages: skipped.length,
|
|
151
|
+
},
|
|
152
|
+
pages: sortPages(pages).map(pageManifest),
|
|
153
|
+
}, null, 2)}\n`;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
export async function runExport(projectRoot, options = {}) {
|
|
157
|
+
const format = normalizeFormat(options.format);
|
|
158
|
+
const exportDir = outputDir(projectRoot, options);
|
|
159
|
+
const pages = await collectWikiPages(projectRoot, {
|
|
160
|
+
maxFiles: options.maxFiles || 1000,
|
|
161
|
+
maxChars: options.maxChars || 75000,
|
|
162
|
+
});
|
|
163
|
+
const visibility = visibleWikiPages(pages, options);
|
|
164
|
+
const selectedPages = sortPages(visibility.visible.map((item) => item.page));
|
|
165
|
+
const skipped = visibility.hidden.map((item) => ({ path: item.page.rel, reason: item.visibility.reason }));
|
|
166
|
+
const metadata = {
|
|
167
|
+
generatedAt: new Date().toISOString(),
|
|
168
|
+
visibilityPolicy: DURABLE_VISIBILITY_POLICY,
|
|
169
|
+
};
|
|
170
|
+
const contents = {
|
|
171
|
+
'llms.txt': renderLlmsTxt(projectRoot, exportDir, selectedPages, metadata),
|
|
172
|
+
'llms-full.txt': renderLlmsFullTxt(projectRoot, selectedPages, metadata, options),
|
|
173
|
+
'llm-wiki.json': renderJson(projectRoot, selectedPages, metadata, skipped),
|
|
174
|
+
};
|
|
175
|
+
const names = selectedNames(format);
|
|
176
|
+
const files = names.map((name) => ({
|
|
177
|
+
name,
|
|
178
|
+
path: join(exportDir, name),
|
|
179
|
+
bytes: Buffer.byteLength(contents[name], 'utf8'),
|
|
180
|
+
}));
|
|
181
|
+
if (!options.dryRun) {
|
|
182
|
+
await ensureDir(exportDir);
|
|
183
|
+
for (const file of files) {
|
|
184
|
+
await writeText(file.path, contents[file.name]);
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
return {
|
|
188
|
+
workspace: projectRoot,
|
|
189
|
+
dryRun: Boolean(options.dryRun),
|
|
190
|
+
format,
|
|
191
|
+
outputDir: exportDir,
|
|
192
|
+
visibilityPolicy: metadata.visibilityPolicy,
|
|
193
|
+
exportedPages: selectedPages.length,
|
|
194
|
+
skipped,
|
|
195
|
+
files,
|
|
196
|
+
};
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
export function formatExportResult(result) {
|
|
200
|
+
const lines = [
|
|
201
|
+
'llm-wiki export',
|
|
202
|
+
`- workspace: ${result.workspace}`,
|
|
203
|
+
`- dry-run: ${result.dryRun ? 'yes' : 'no'}`,
|
|
204
|
+
`- format: ${result.format}`,
|
|
205
|
+
`- output: ${result.outputDir}`,
|
|
206
|
+
`- visibility policy: ${result.visibilityPolicy}`,
|
|
207
|
+
`- exported pages: ${result.exportedPages}`,
|
|
208
|
+
`- skipped pages: ${result.skipped.length}`,
|
|
209
|
+
];
|
|
210
|
+
for (const file of result.files) {
|
|
211
|
+
lines.push(`- ${file.name}: ${file.path} (${file.bytes} bytes)`);
|
|
212
|
+
}
|
|
213
|
+
return lines.join('\n');
|
|
214
|
+
}
|