oh-my-llmwikimode 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +494 -0
- package/bin/llmwiki.js +1493 -0
- package/docs/INSTALLATION.md +228 -0
- package/docs/SCOPE_LOCK.md +79 -0
- package/docs/STAGE1_GUIDE.md +265 -0
- package/docs/STAGE2_AGENT_TEAM_GUIDE.md +141 -0
- package/docs/STAGE3_CONVERSATIONAL_GROWTH_GUIDE.md +50 -0
- package/docs/TEST_WORKSHEET.md +120 -0
- package/docs/github-private-bootstrap.md +53 -0
- package/docs/release.md +79 -0
- package/docs/stage4-slice1-manual-test.md +259 -0
- package/docs/stage4-slice1-user-guide.md +269 -0
- package/docs/user-guide-ko.md +452 -0
- package/package.json +76 -0
- package/scripts/install-llmwiki.ps1 +229 -0
- package/src/config.js +74 -0
- package/src/curator/browser-data.js +134 -0
- package/src/curator/queue.js +324 -0
- package/src/curator/schema.js +237 -0
- package/src/curator/scoring.js +83 -0
- package/src/hooks.js +199 -0
- package/src/librarian/schema.js +218 -0
- package/src/librarian/weekly-digest.js +478 -0
- package/src/security.js +127 -0
- package/src/server.js +860 -0
- package/src/stage4/graph-reasoning/analyzer.js +255 -0
- package/src/stage4/graph-reasoning/browser-data.js +130 -0
- package/src/stage4/graph-reasoning/index.js +35 -0
- package/src/stage4/graph-reasoning/loader.js +122 -0
- package/src/stage4/graph-reasoning/queue.js +154 -0
- package/src/stage4/graph-reasoning/schema.js +190 -0
- package/src/team/browser-data.js +142 -0
- package/src/team/capabilities.js +79 -0
- package/src/team/dispatch.js +108 -0
- package/src/team/queue.js +290 -0
- package/src/team/schema.js +225 -0
- package/src/team/shared-memory.js +183 -0
- package/src/todo/browser-data.js +71 -0
- package/src/todo/queue.js +159 -0
- package/src/todo/schema.js +90 -0
- package/src/utils/embedding-model.js +111 -0
- package/src/wiki/alias-suggestions.js +180 -0
- package/src/wiki/browser-data.js +284 -0
- package/src/wiki/doctor.js +218 -0
- package/src/wiki/entry-normalizer.js +139 -0
- package/src/wiki/ingest.js +443 -0
- package/src/wiki/lesson-proposal-analyzer.js +463 -0
- package/src/wiki/lesson-proposal-manager.js +331 -0
- package/src/wiki/lesson-template.js +182 -0
- package/src/wiki/lint.js +294 -0
- package/src/wiki/notebooklm-adapter.js +264 -0
- package/src/wiki/query.js +304 -0
- package/src/wiki/raw-manager.js +400 -0
- package/src/wiki/search-feedback.js +211 -0
- package/src/wiki/semantic-index.js +333 -0
- package/src/wiki/semantic-search.js +170 -0
- package/src/wiki/source-ledger.js +370 -0
- package/src/wiki/store.js +1329 -0
- package/src/wiki/usage-events.js +144 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import crypto from "node:crypto";
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* B2 Alias Suggestions — Review-Only Artifacts
|
|
7
|
+
*
|
|
8
|
+
* Generates JSON artifacts under .system/search-feedback/suggestions/
|
|
9
|
+
* from zero-result search feedback events.
|
|
10
|
+
*/
|
|
11
|
+
|
|
12
|
+
export const ALIAS_SUGGESTION_VERSION = 1;
|
|
13
|
+
export const SUGGESTIONS_DIR = ".system/search-feedback/suggestions";
|
|
14
|
+
|
|
15
|
+
function getSuggestionsDir(wikiRoot) {
|
|
16
|
+
return path.join(wikiRoot, SUGGESTIONS_DIR);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
function ensureSuggestionsDir(wikiRoot) {
|
|
20
|
+
const dir = getSuggestionsDir(wikiRoot);
|
|
21
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
22
|
+
return dir;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function shortHash(value) {
|
|
26
|
+
return crypto.createHash("sha256").update(String(value)).digest("hex").slice(0, 12);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function generateSuggestionId(queryHash, timestamp) {
|
|
30
|
+
return `alias_${shortHash(queryHash + timestamp)}`;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function generateSuggestionFilename(queryHash, timestamp) {
|
|
34
|
+
// Format: YYYYMMDDTHHmmssZ (ISO 8601 basic format, no separators)
|
|
35
|
+
const d = new Date(timestamp);
|
|
36
|
+
const year = d.getUTCFullYear();
|
|
37
|
+
const month = String(d.getUTCMonth() + 1).padStart(2, "0");
|
|
38
|
+
const day = String(d.getUTCDate()).padStart(2, "0");
|
|
39
|
+
const hours = String(d.getUTCHours()).padStart(2, "0");
|
|
40
|
+
const minutes = String(d.getUTCMinutes()).padStart(2, "0");
|
|
41
|
+
const seconds = String(d.getUTCSeconds()).padStart(2, "0");
|
|
42
|
+
const ts = `${year}${month}${day}T${hours}${minutes}${seconds}Z`;
|
|
43
|
+
return `${ts}-${shortHash(queryHash)}.json`;
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Check if a proposed alias already exists in the current index.
|
|
48
|
+
*
|
|
49
|
+
* @param {string} proposedAlias
|
|
50
|
+
* @param {Object} index - Build index from store.js
|
|
51
|
+
* @returns {boolean}
|
|
52
|
+
*/
|
|
53
|
+
function aliasExists(proposedAlias, index) {
|
|
54
|
+
if (!proposedAlias || !index || !Array.isArray(index.entries)) {
|
|
55
|
+
return false;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const normalized = proposedAlias.toLowerCase().trim();
|
|
59
|
+
if (!normalized) return false;
|
|
60
|
+
|
|
61
|
+
for (const entry of index.entries) {
|
|
62
|
+
const title = String(entry.title || "").toLowerCase().trim();
|
|
63
|
+
if (title === normalized) return true;
|
|
64
|
+
|
|
65
|
+
const aliases = Array.isArray(entry.aliases) ? entry.aliases : [];
|
|
66
|
+
for (const alias of aliases) {
|
|
67
|
+
if (String(alias).toLowerCase().trim() === normalized) return true;
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
return false;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Generate a review-only alias suggestion artifact from a search failure event.
|
|
76
|
+
*
|
|
77
|
+
* @param {string} wikiRoot
|
|
78
|
+
* @param {Object} failureEvent - A search failure event record
|
|
79
|
+
* @param {Object} [index] - Current wiki index (for duplicate detection)
|
|
80
|
+
* @param {Object} [options]
|
|
81
|
+
* @param {Date|string} [options.now] - Injected timestamp
|
|
82
|
+
* @returns {{success: boolean, artifact?: Object, skipped?: boolean, reason?: string, error?: string}}
|
|
83
|
+
*/
|
|
84
|
+
export function generateAliasSuggestion(wikiRoot, failureEvent, index, options = {}) {
|
|
85
|
+
try {
|
|
86
|
+
if (!failureEvent || typeof failureEvent !== "object") {
|
|
87
|
+
return { success: false, error: "Invalid failure event" };
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const proposedAlias = String(failureEvent.query || "").trim();
|
|
91
|
+
if (!proposedAlias) {
|
|
92
|
+
return { success: false, error: "failure event has no query" };
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// Skip if alias already exists
|
|
96
|
+
if (index && aliasExists(proposedAlias, index)) {
|
|
97
|
+
return {
|
|
98
|
+
success: true,
|
|
99
|
+
skipped: true,
|
|
100
|
+
reason: "Alias already exists in index",
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
const now = options.now ? new Date(options.now) : new Date();
|
|
105
|
+
const ts = now.toISOString();
|
|
106
|
+
const queryHash = failureEvent.query_hash || "";
|
|
107
|
+
|
|
108
|
+
const artifact = {
|
|
109
|
+
version: ALIAS_SUGGESTION_VERSION,
|
|
110
|
+
id: generateSuggestionId(queryHash, ts),
|
|
111
|
+
status: "review_required",
|
|
112
|
+
action: "consider_alias",
|
|
113
|
+
proposed_alias: proposedAlias,
|
|
114
|
+
target_path: Array.isArray(failureEvent.top_paths) && failureEvent.top_paths.length > 0
|
|
115
|
+
? failureEvent.top_paths[0]
|
|
116
|
+
: null,
|
|
117
|
+
evidence: {
|
|
118
|
+
failure_id: failureEvent.id || null,
|
|
119
|
+
query_hash: queryHash,
|
|
120
|
+
},
|
|
121
|
+
auto_apply: false,
|
|
122
|
+
created_at: ts,
|
|
123
|
+
};
|
|
124
|
+
|
|
125
|
+
// Write artifact
|
|
126
|
+
ensureSuggestionsDir(wikiRoot);
|
|
127
|
+
const filename = generateSuggestionFilename(queryHash, ts);
|
|
128
|
+
const filePath = path.join(getSuggestionsDir(wikiRoot), filename);
|
|
129
|
+
fs.writeFileSync(filePath, JSON.stringify(artifact, null, 2), "utf-8");
|
|
130
|
+
|
|
131
|
+
return { success: true, artifact, path: filePath };
|
|
132
|
+
} catch (error) {
|
|
133
|
+
return { success: false, error: error.message };
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* List all alias suggestion artifacts.
|
|
139
|
+
*
|
|
140
|
+
* @param {string} wikiRoot
|
|
141
|
+
* @returns {{success: boolean, suggestions?: Array, error?: string}}
|
|
142
|
+
*/
|
|
143
|
+
export function listAliasSuggestions(wikiRoot) {
|
|
144
|
+
try {
|
|
145
|
+
const dir = getSuggestionsDir(wikiRoot);
|
|
146
|
+
if (!fs.existsSync(dir)) {
|
|
147
|
+
return { success: true, suggestions: [] };
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
const files = fs
|
|
151
|
+
.readdirSync(dir)
|
|
152
|
+
.filter((f) => f.endsWith(".json"))
|
|
153
|
+
.sort();
|
|
154
|
+
|
|
155
|
+
const suggestions = [];
|
|
156
|
+
for (const file of files) {
|
|
157
|
+
try {
|
|
158
|
+
const content = fs.readFileSync(path.join(dir, file), "utf-8");
|
|
159
|
+
suggestions.push(JSON.parse(content));
|
|
160
|
+
} catch {
|
|
161
|
+
// Skip malformed files
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
return { success: true, suggestions };
|
|
166
|
+
} catch (error) {
|
|
167
|
+
return { success: false, error: error.message };
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/**
|
|
172
|
+
* Count alias suggestion artifacts.
|
|
173
|
+
*
|
|
174
|
+
* @param {string} wikiRoot
|
|
175
|
+
* @returns {number}
|
|
176
|
+
*/
|
|
177
|
+
export function countAliasSuggestions(wikiRoot) {
|
|
178
|
+
const result = listAliasSuggestions(wikiRoot);
|
|
179
|
+
return result.success ? result.suggestions.length : 0;
|
|
180
|
+
}
|
|
@@ -0,0 +1,284 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { redactSecrets } from "../security.js";
|
|
4
|
+
import { buildCuratorBrowserData } from "../curator/browser-data.js";
|
|
5
|
+
import { buildAgentTeamBrowserData } from "../team/browser-data.js";
|
|
6
|
+
import { buildTodoBrowserData } from "../todo/browser-data.js";
|
|
7
|
+
import { buildGraphReasoningBrowserData } from "../stage4/graph-reasoning/browser-data.js";
|
|
8
|
+
import {
|
|
9
|
+
detectDuplicates,
|
|
10
|
+
ensureWikiStructure,
|
|
11
|
+
getWikiPaths,
|
|
12
|
+
parseFrontmatter,
|
|
13
|
+
} from "./store.js";
|
|
14
|
+
|
|
15
|
+
export const BROWSER_DATA_RELATIVE_PATH = ".system/browser-data.json";
|
|
16
|
+
|
|
17
|
+
const AUTO_MEMORY_SOURCES = new Set(["auto-memory", "chat.message"]);
|
|
18
|
+
const EXCLUDED_BROWSER_STATUSES = new Set(["rejected", "superseded", "private", "needs-clarification"]);
|
|
19
|
+
const DEFAULT_BUILT_AT = "1970-01-01T00:00:00.000Z";
|
|
20
|
+
|
|
21
|
+
function normalizeScalar(value, maxLength = 240) {
|
|
22
|
+
const normalized = String(value ?? "")
|
|
23
|
+
.replace(/\r?\n/g, " ")
|
|
24
|
+
.replace(/\s+/g, " ")
|
|
25
|
+
.trim();
|
|
26
|
+
|
|
27
|
+
if (!normalized) return "";
|
|
28
|
+
return redactSecrets(normalized).slice(0, maxLength);
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function normalizeText(value, maxLength = 240) {
|
|
32
|
+
if (Array.isArray(value)) {
|
|
33
|
+
return value.map((item) => normalizeScalar(item, maxLength)).filter(Boolean).join(" ");
|
|
34
|
+
}
|
|
35
|
+
return normalizeScalar(value, maxLength);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function normalizeTags(value) {
|
|
39
|
+
const tags = Array.isArray(value) ? value : [value];
|
|
40
|
+
return tags.map((tag) => normalizeScalar(tag, 80)).filter(Boolean);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function normalizeStatus(value) {
|
|
44
|
+
return normalizeScalar(value || "candidate", 80).toLowerCase() || "candidate";
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function normalizeSource(value) {
|
|
48
|
+
return normalizeScalar(value, 120).toLowerCase();
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
function normalizeRelativePath(value) {
|
|
52
|
+
if (!value || typeof value !== "string") return null;
|
|
53
|
+
const normalized = value.replace(/\\/g, "/");
|
|
54
|
+
if (path.isAbsolute(normalized)) return null;
|
|
55
|
+
if (normalized.split("/").some((segment) => segment === "..")) return null;
|
|
56
|
+
return normalized;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
function toRelativePath(wikiRoot, filePath) {
|
|
60
|
+
return normalizeRelativePath(path.relative(wikiRoot, filePath));
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
function hasSafeDate(value) {
|
|
64
|
+
return typeof value === "string" && value.trim() && !Number.isNaN(Date.parse(value));
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function deterministicBuiltAt(records, graph) {
|
|
68
|
+
const dates = [];
|
|
69
|
+
|
|
70
|
+
for (const record of records) {
|
|
71
|
+
for (const value of [record.created_at, record.updated_at, record.promoted_at]) {
|
|
72
|
+
if (hasSafeDate(value)) dates.push(value);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
if (hasSafeDate(graph?.meta?.built_at)) dates.push(graph.meta.built_at);
|
|
77
|
+
if (dates.length === 0) return DEFAULT_BUILT_AT;
|
|
78
|
+
return dates.sort((a, b) => a.localeCompare(b)).at(-1);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
function addSourceMetadata(entry, frontmatter) {
|
|
82
|
+
const source = normalizeSource(frontmatter.source);
|
|
83
|
+
if (source) entry.source = source;
|
|
84
|
+
|
|
85
|
+
for (const key of ["source_session", "source_agent", "capture_reason"]) {
|
|
86
|
+
const value = normalizeText(frontmatter[key], 240);
|
|
87
|
+
if (value) entry[key] = value;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
function scanDirectory(wikiRoot, directory, category) {
|
|
92
|
+
if (!fs.existsSync(directory)) return [];
|
|
93
|
+
|
|
94
|
+
const results = [];
|
|
95
|
+
const entries = fs.readdirSync(directory, { withFileTypes: true });
|
|
96
|
+
|
|
97
|
+
for (const entry of entries) {
|
|
98
|
+
const entryPath = path.join(directory, entry.name);
|
|
99
|
+
if (entry.isDirectory()) {
|
|
100
|
+
results.push(...scanDirectory(wikiRoot, entryPath, category));
|
|
101
|
+
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
102
|
+
try {
|
|
103
|
+
const relativePath = toRelativePath(wikiRoot, entryPath);
|
|
104
|
+
if (!relativePath) continue;
|
|
105
|
+
|
|
106
|
+
const content = fs.readFileSync(entryPath, "utf-8");
|
|
107
|
+
const { frontmatter } = parseFrontmatter(content);
|
|
108
|
+
const title = normalizeText(frontmatter.title, 180);
|
|
109
|
+
if (!title) continue;
|
|
110
|
+
|
|
111
|
+
const item = {
|
|
112
|
+
path: relativePath,
|
|
113
|
+
category,
|
|
114
|
+
title,
|
|
115
|
+
tags: normalizeTags(frontmatter.tags),
|
|
116
|
+
status: normalizeStatus(frontmatter.status),
|
|
117
|
+
};
|
|
118
|
+
addSourceMetadata(item, frontmatter);
|
|
119
|
+
|
|
120
|
+
results.push({
|
|
121
|
+
entry: item,
|
|
122
|
+
created_at: normalizeScalar(frontmatter.created_at, 80),
|
|
123
|
+
updated_at: normalizeScalar(frontmatter.updated_at, 80),
|
|
124
|
+
promoted_at: normalizeScalar(frontmatter.promoted_at, 80),
|
|
125
|
+
});
|
|
126
|
+
} catch {
|
|
127
|
+
// Skip unreadable files
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
return results.sort((a, b) => a.entry.path.localeCompare(b.entry.path));
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
function scanEntries(wikiRoot) {
|
|
136
|
+
const paths = getWikiPaths(wikiRoot);
|
|
137
|
+
return [
|
|
138
|
+
...scanDirectory(wikiRoot, paths.inbox, "inbox"),
|
|
139
|
+
...scanDirectory(wikiRoot, paths.problems, "problems"),
|
|
140
|
+
...scanDirectory(wikiRoot, paths.lessons, "lessons"),
|
|
141
|
+
].sort((a, b) => a.entry.path.localeCompare(b.entry.path));
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function isBrowserVisible(entry) {
|
|
145
|
+
return !EXCLUDED_BROWSER_STATUSES.has(entry.status);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
function isAutoMemoryEntry(entry) {
|
|
149
|
+
return AUTO_MEMORY_SOURCES.has(entry.source);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function readGraph(wikiRoot) {
|
|
153
|
+
const graphPath = path.join(wikiRoot, ".system", "graph.json");
|
|
154
|
+
try {
|
|
155
|
+
return JSON.parse(fs.readFileSync(graphPath, "utf-8"));
|
|
156
|
+
} catch {
|
|
157
|
+
return { nodes: [], edges: [], communities: [], meta: {} };
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function sanitizeDuplicates(duplicates) {
|
|
162
|
+
return duplicates
|
|
163
|
+
.map((duplicate) => {
|
|
164
|
+
const warning = {
|
|
165
|
+
type: normalizeScalar(duplicate.type, 40) || "unknown",
|
|
166
|
+
entries: (duplicate.entries || [])
|
|
167
|
+
.map(normalizeRelativePath)
|
|
168
|
+
.filter(Boolean)
|
|
169
|
+
.sort((a, b) => a.localeCompare(b)),
|
|
170
|
+
};
|
|
171
|
+
|
|
172
|
+
const normalizedTitle = normalizeScalar(duplicate.normalized_title, 160);
|
|
173
|
+
const hash = normalizeScalar(duplicate.hash, 80);
|
|
174
|
+
if (normalizedTitle) warning.normalized_title = normalizedTitle;
|
|
175
|
+
if (hash) warning.hash = hash;
|
|
176
|
+
return warning;
|
|
177
|
+
})
|
|
178
|
+
.filter((warning) => warning.entries.length > 0)
|
|
179
|
+
.sort((a, b) => {
|
|
180
|
+
const left = `${a.type}:${a.normalized_title || a.hash || ""}:${a.entries.join("|")}`;
|
|
181
|
+
const right = `${b.type}:${b.normalized_title || b.hash || ""}:${b.entries.join("|")}`;
|
|
182
|
+
return left.localeCompare(right);
|
|
183
|
+
});
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
function sanitizeCommunities(graph) {
|
|
187
|
+
const communities = Array.isArray(graph?.communities) ? graph.communities : [];
|
|
188
|
+
return communities
|
|
189
|
+
.map((community) => {
|
|
190
|
+
const members = (community.members || [])
|
|
191
|
+
.map(normalizeRelativePath)
|
|
192
|
+
.filter(Boolean)
|
|
193
|
+
.sort((a, b) => a.localeCompare(b));
|
|
194
|
+
|
|
195
|
+
return {
|
|
196
|
+
id: normalizeScalar(community.id, 120),
|
|
197
|
+
label: normalizeScalar(community.label || community.id, 120),
|
|
198
|
+
members,
|
|
199
|
+
color: normalizeScalar(community.color, 40),
|
|
200
|
+
};
|
|
201
|
+
})
|
|
202
|
+
.filter((community) => community.id && community.members.length > 0)
|
|
203
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
function sanitizeGraphGaps(graph) {
|
|
207
|
+
const nodes = Array.isArray(graph?.nodes) ? graph.nodes : [];
|
|
208
|
+
return nodes
|
|
209
|
+
.map((node) => {
|
|
210
|
+
const nodePath = normalizeRelativePath(node.path || node.id);
|
|
211
|
+
if (!nodePath) return null;
|
|
212
|
+
const degree = Number.isFinite(node.degree) ? Math.max(0, Math.trunc(node.degree)) : 0;
|
|
213
|
+
return {
|
|
214
|
+
path: nodePath,
|
|
215
|
+
title: normalizeText(node.label || node.title || node.id, 180),
|
|
216
|
+
degree,
|
|
217
|
+
reason: degree === 0 ? "isolated" : "under-connected",
|
|
218
|
+
};
|
|
219
|
+
})
|
|
220
|
+
.filter((gap) => gap && gap.degree < 2)
|
|
221
|
+
.sort((a, b) => a.path.localeCompare(b.path));
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function buildDuplicateEntries(entries) {
|
|
225
|
+
return entries.map((entry) => ({
|
|
226
|
+
path: entry.path,
|
|
227
|
+
title: normalizeText(entry.title, 180),
|
|
228
|
+
}));
|
|
229
|
+
}
|
|
230
|
+
|
|
231
|
+
export function buildBrowserData(wikiRoot) {
|
|
232
|
+
const paths = getWikiPaths(wikiRoot);
|
|
233
|
+
ensureWikiStructure(paths);
|
|
234
|
+
|
|
235
|
+
const records = scanEntries(wikiRoot);
|
|
236
|
+
const visibleRecords = records.filter((record) => isBrowserVisible(record.entry));
|
|
237
|
+
const visibleEntries = visibleRecords.map((record) => record.entry);
|
|
238
|
+
const graph = readGraph(wikiRoot);
|
|
239
|
+
|
|
240
|
+
const candidates = visibleEntries.filter((entry) => entry.category === "inbox" && entry.status === "candidate");
|
|
241
|
+
const lessons = visibleEntries.filter((entry) => entry.category === "lessons");
|
|
242
|
+
const autoMemoryCaptures = visibleEntries.filter(isAutoMemoryEntry);
|
|
243
|
+
const autoMemoryCandidateCount = candidates.filter(isAutoMemoryEntry).length;
|
|
244
|
+
const duplicates = sanitizeDuplicates(detectDuplicates(buildDuplicateEntries(visibleEntries), wikiRoot));
|
|
245
|
+
|
|
246
|
+
return {
|
|
247
|
+
candidates,
|
|
248
|
+
auto_memory_captures: autoMemoryCaptures,
|
|
249
|
+
lessons,
|
|
250
|
+
duplicates,
|
|
251
|
+
communities: sanitizeCommunities(graph),
|
|
252
|
+
graph_gaps: sanitizeGraphGaps(graph),
|
|
253
|
+
todo: buildTodoBrowserData(wikiRoot),
|
|
254
|
+
agent_team: buildAgentTeamBrowserData(wikiRoot),
|
|
255
|
+
curator: buildCuratorBrowserData(wikiRoot),
|
|
256
|
+
stage4_graph_reasoning: buildGraphReasoningBrowserData(wikiRoot),
|
|
257
|
+
meta: {
|
|
258
|
+
built_at: deterministicBuiltAt(visibleRecords, graph),
|
|
259
|
+
file_count: visibleEntries.length,
|
|
260
|
+
candidate_count: candidates.length,
|
|
261
|
+
lesson_count: lessons.length,
|
|
262
|
+
auto_memory_candidate_count: autoMemoryCandidateCount,
|
|
263
|
+
},
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
export function writeBrowserData(wikiRoot, browserData = buildBrowserData(wikiRoot)) {
|
|
268
|
+
const paths = getWikiPaths(wikiRoot);
|
|
269
|
+
ensureWikiStructure(paths);
|
|
270
|
+
|
|
271
|
+
const browserDataPath = path.join(paths.system, "browser-data.json");
|
|
272
|
+
const tmpFile = `${browserDataPath}.${Date.now()}.${Math.random().toString(36).slice(2)}.tmp`;
|
|
273
|
+
fs.writeFileSync(tmpFile, JSON.stringify(browserData, null, 2));
|
|
274
|
+
fs.renameSync(tmpFile, browserDataPath);
|
|
275
|
+
|
|
276
|
+
return {
|
|
277
|
+
path: BROWSER_DATA_RELATIVE_PATH,
|
|
278
|
+
data: browserData,
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
|
|
282
|
+
export function rebuildBrowserData(wikiRoot) {
|
|
283
|
+
return writeBrowserData(wikiRoot, buildBrowserData(wikiRoot));
|
|
284
|
+
}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
import fs from "node:fs";
|
|
2
|
+
import path from "node:path";
|
|
3
|
+
import { parseFrontmatter } from "./store.js";
|
|
4
|
+
import {
|
|
5
|
+
VALID_SOURCES,
|
|
6
|
+
VALID_STATUSES,
|
|
7
|
+
normalizeEntrySource,
|
|
8
|
+
normalizeEntryStatus,
|
|
9
|
+
normalizeEntryTags,
|
|
10
|
+
normalizeEntryText,
|
|
11
|
+
normalizeWikiIndex,
|
|
12
|
+
} from "./entry-normalizer.js";
|
|
13
|
+
|
|
14
|
+
const ENTRY_DIRECTORIES = ["inbox", "problems", path.join("editorial", "lessons")];
|
|
15
|
+
|
|
16
|
+
function typeOfField(value) {
|
|
17
|
+
if (Array.isArray(value)) return "array";
|
|
18
|
+
if (value === null) return "null";
|
|
19
|
+
return typeof value;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function isRecord(value) {
|
|
23
|
+
return typeof value === "object" && value !== null && !Array.isArray(value);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function relativePath(rootDir, fullPath) {
|
|
27
|
+
return path.relative(rootDir, fullPath).replace(/\\/g, "/");
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
function collectMarkdownFiles(wikiRoot) {
|
|
31
|
+
const files = [];
|
|
32
|
+
|
|
33
|
+
function visit(directory) {
|
|
34
|
+
if (!fs.existsSync(directory)) return;
|
|
35
|
+
for (const entry of fs.readdirSync(directory, { withFileTypes: true })) {
|
|
36
|
+
const fullPath = path.join(directory, entry.name);
|
|
37
|
+
if (entry.isDirectory()) {
|
|
38
|
+
visit(fullPath);
|
|
39
|
+
} else if (entry.isFile() && entry.name.endsWith(".md")) {
|
|
40
|
+
files.push(fullPath);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
for (const entryDirectory of ENTRY_DIRECTORIES) {
|
|
46
|
+
visit(path.join(wikiRoot, entryDirectory));
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
return files.sort((left, right) => left.localeCompare(right));
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function createIssue(field, kind, value, normalizedValue, message) {
|
|
53
|
+
return {
|
|
54
|
+
field,
|
|
55
|
+
kind,
|
|
56
|
+
value_type: typeOfField(value),
|
|
57
|
+
normalized_value: normalizedValue,
|
|
58
|
+
message,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function inspectTitle(value) {
|
|
63
|
+
if (value === undefined || value === null || normalizeEntryText(value) === "") {
|
|
64
|
+
return createIssue("title", "missing", value, "", "Title is missing and needs review before indexing.");
|
|
65
|
+
}
|
|
66
|
+
if (Array.isArray(value)) {
|
|
67
|
+
return createIssue("title", "array", value, normalizeEntryText(value), "Title is an array and should be stored as a scalar string.");
|
|
68
|
+
}
|
|
69
|
+
if (isRecord(value)) {
|
|
70
|
+
return createIssue("title", "object", value, normalizeEntryText(value), "Title is an object and should be reviewed before conversion.");
|
|
71
|
+
}
|
|
72
|
+
return null;
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
function inspectTags(value) {
|
|
76
|
+
if (value === undefined || value === null) {
|
|
77
|
+
return createIssue("tags", "missing", value, [], "Tags are missing; add an explicit reviewed tag list.");
|
|
78
|
+
}
|
|
79
|
+
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
80
|
+
return createIssue("tags", "scalar", value, normalizeEntryTags(value), "Tags are scalar and should be stored as a list.");
|
|
81
|
+
}
|
|
82
|
+
if (isRecord(value)) {
|
|
83
|
+
return createIssue("tags", "object", value, normalizeEntryTags(value), "Tags are an object and should be manually mapped to a list.");
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function inspectStatus(value) {
|
|
89
|
+
if (value === undefined || value === null || normalizeEntryText(value) === "") {
|
|
90
|
+
return createIssue("status", "missing", value, normalizeEntryStatus(value), "Status is missing; candidate is the safe fallback.");
|
|
91
|
+
}
|
|
92
|
+
const normalized = normalizeEntryStatus(value);
|
|
93
|
+
if (Array.isArray(value) || isRecord(value) || !VALID_STATUSES.includes(normalizeEntryText(value).toLowerCase())) {
|
|
94
|
+
return createIssue("status", "invalid", value, normalized, "Status is invalid and will fall back to candidate until reviewed.");
|
|
95
|
+
}
|
|
96
|
+
return null;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
function inspectSource(value) {
|
|
100
|
+
if (value === undefined || value === null || normalizeEntryText(value) === "") {
|
|
101
|
+
return createIssue("source", "missing", value, normalizeEntrySource(value), "Source is missing; manual is the safe fallback.");
|
|
102
|
+
}
|
|
103
|
+
const normalized = normalizeEntrySource(value);
|
|
104
|
+
if (Array.isArray(value) || isRecord(value) || !VALID_SOURCES.includes(normalizeEntryText(value).toLowerCase())) {
|
|
105
|
+
return createIssue("source", "invalid", value, normalized, "Source is invalid and will fall back to manual until reviewed.");
|
|
106
|
+
}
|
|
107
|
+
return null;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function inspectFields(fields) {
|
|
111
|
+
return [
|
|
112
|
+
inspectTitle(fields.title),
|
|
113
|
+
inspectTags(fields.tags),
|
|
114
|
+
inspectStatus(fields.status),
|
|
115
|
+
inspectSource(fields.source),
|
|
116
|
+
].filter(Boolean);
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function addMalformedEntry(malformedEntries, scope, entryPath, issues) {
|
|
120
|
+
if (issues.length === 0) return;
|
|
121
|
+
malformedEntries.push({
|
|
122
|
+
scope,
|
|
123
|
+
path: entryPath,
|
|
124
|
+
issues,
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function readCachedIndex(wikiRoot, warnings) {
|
|
129
|
+
const indexPath = path.join(wikiRoot, ".system", "index.json");
|
|
130
|
+
if (!fs.existsSync(indexPath)) return null;
|
|
131
|
+
|
|
132
|
+
try {
|
|
133
|
+
return JSON.parse(fs.readFileSync(indexPath, "utf-8"));
|
|
134
|
+
} catch (error) {
|
|
135
|
+
warnings.push(`Unable to parse cached index: ${error.message}`);
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
function migrationActionForIssue(entry, issue) {
|
|
141
|
+
return {
|
|
142
|
+
action: "review_entry_field",
|
|
143
|
+
scope: entry.scope,
|
|
144
|
+
path: entry.path,
|
|
145
|
+
field: issue.field,
|
|
146
|
+
issue: issue.kind,
|
|
147
|
+
normalized_value: issue.normalized_value,
|
|
148
|
+
review_required: true,
|
|
149
|
+
};
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function buildMigrationPlan(malformedEntries) {
|
|
153
|
+
return {
|
|
154
|
+
review_required: true,
|
|
155
|
+
dry_run: true,
|
|
156
|
+
mutations_performed: 0,
|
|
157
|
+
actions: malformedEntries.flatMap((entry) => entry.issues.map((issue) => migrationActionForIssue(entry, issue))),
|
|
158
|
+
};
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
export function doctorWiki(wikiRoot) {
|
|
162
|
+
const normalizedRoot = path.resolve(wikiRoot);
|
|
163
|
+
const warnings = [];
|
|
164
|
+
const malformedEntries = [];
|
|
165
|
+
let entriesChecked = 0;
|
|
166
|
+
|
|
167
|
+
if (!fs.existsSync(normalizedRoot)) {
|
|
168
|
+
return {
|
|
169
|
+
success: false,
|
|
170
|
+
wiki_root: normalizedRoot,
|
|
171
|
+
entries_checked: 0,
|
|
172
|
+
malformed_entries: [],
|
|
173
|
+
warnings: [`Wiki root does not exist: ${normalizedRoot}`],
|
|
174
|
+
migration_plan: buildMigrationPlan([]),
|
|
175
|
+
};
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
for (const filePath of collectMarkdownFiles(normalizedRoot)) {
|
|
179
|
+
entriesChecked += 1;
|
|
180
|
+
try {
|
|
181
|
+
const content = fs.readFileSync(filePath, "utf-8");
|
|
182
|
+
const { frontmatter } = parseFrontmatter(content);
|
|
183
|
+
addMalformedEntry(malformedEntries, "markdown", relativePath(normalizedRoot, filePath), inspectFields(frontmatter));
|
|
184
|
+
} catch (error) {
|
|
185
|
+
warnings.push(`Unable to inspect ${relativePath(normalizedRoot, filePath)}: ${error.message}`);
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
const cachedIndex = readCachedIndex(normalizedRoot, warnings);
|
|
190
|
+
if (cachedIndex) {
|
|
191
|
+
const entries = Array.isArray(cachedIndex.entries) ? cachedIndex.entries : [];
|
|
192
|
+
const normalizedIndex = normalizeWikiIndex(cachedIndex);
|
|
193
|
+
entries.forEach((entry, entryIndex) => {
|
|
194
|
+
entriesChecked += 1;
|
|
195
|
+
const normalizedEntry = normalizedIndex.entries[entryIndex];
|
|
196
|
+
const entryPath = normalizeEntryText(entry?.path) || normalizedEntry?.path || `.system/index.json#entry-${entryIndex}`;
|
|
197
|
+
addMalformedEntry(
|
|
198
|
+
malformedEntries,
|
|
199
|
+
"cached_index",
|
|
200
|
+
`.system/index.json#${entryPath}`,
|
|
201
|
+
inspectFields(entry)
|
|
202
|
+
);
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
const migrationPlan = buildMigrationPlan(malformedEntries);
|
|
207
|
+
return {
|
|
208
|
+
success: true,
|
|
209
|
+
wiki_root: normalizedRoot,
|
|
210
|
+
entries_checked: entriesChecked,
|
|
211
|
+
malformed_entries: malformedEntries,
|
|
212
|
+
warnings,
|
|
213
|
+
migration_plan: migrationPlan,
|
|
214
|
+
message: malformedEntries.length === 0
|
|
215
|
+
? `Doctor checked ${entriesChecked} entries; no malformed entry fields found.`
|
|
216
|
+
: `Doctor checked ${entriesChecked} entries; ${malformedEntries.length} entries need review.`,
|
|
217
|
+
};
|
|
218
|
+
}
|