@tangle-network/agent-knowledge 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +57 -0
- package/dist/chunk-TXNYP4WI.js +147 -0
- package/dist/chunk-TXNYP4WI.js.map +1 -0
- package/dist/chunk-XDXIBHPS.js +703 -0
- package/dist/chunk-XDXIBHPS.js.map +1 -0
- package/dist/cli.d.ts +1 -0
- package/dist/cli.js +245 -0
- package/dist/cli.js.map +1 -0
- package/dist/index.d.ts +153 -0
- package/dist/index.js +219 -0
- package/dist/index.js.map +1 -0
- package/dist/types-C1FBNRIN.d.ts +135 -0
- package/dist/viz/index.d.ts +37 -0
- package/dist/viz/index.js +11 -0
- package/dist/viz/index.js.map +1 -0
- package/docs/architecture.md +52 -0
- package/package.json +67 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import {
|
|
2
|
+
WIKILINK_REGEX,
|
|
3
|
+
addSourcePath,
|
|
4
|
+
applyKnowledgeWriteBlocks,
|
|
5
|
+
applyKnowledgeWriteBlocksFile,
|
|
6
|
+
buildKnowledgeGraph,
|
|
7
|
+
buildKnowledgeIndex,
|
|
8
|
+
explainKnowledgeTarget,
|
|
9
|
+
extractWikilinks,
|
|
10
|
+
formatFrontmatter,
|
|
11
|
+
initKnowledgeBase,
|
|
12
|
+
inspectKnowledgeIndex,
|
|
13
|
+
isSafeKnowledgePath,
|
|
14
|
+
layoutFor,
|
|
15
|
+
lintKnowledgeIndex,
|
|
16
|
+
loadKnowledgePages,
|
|
17
|
+
loadSourceRegistry,
|
|
18
|
+
mediaTypeFor,
|
|
19
|
+
normalizeLinkTarget,
|
|
20
|
+
parseFrontmatter,
|
|
21
|
+
parseKnowledgeWriteBlocks,
|
|
22
|
+
reciprocalRankFusion,
|
|
23
|
+
searchKnowledge,
|
|
24
|
+
sha256,
|
|
25
|
+
slugify,
|
|
26
|
+
sourceRegistryPath,
|
|
27
|
+
stableId,
|
|
28
|
+
textSourceAdapter,
|
|
29
|
+
tokenizeQuery,
|
|
30
|
+
writeJson,
|
|
31
|
+
writeKnowledgeIndex,
|
|
32
|
+
writeSourceRegistry
|
|
33
|
+
} from "./chunk-XDXIBHPS.js";
|
|
34
|
+
|
|
35
|
+
// src/chunking.ts
|
|
36
|
+
var DEFAULT_OPTIONS = {
|
|
37
|
+
targetChars: 1e3,
|
|
38
|
+
maxChars: 1500,
|
|
39
|
+
minChars: 160,
|
|
40
|
+
overlapChars: 180
|
|
41
|
+
};
|
|
42
|
+
function chunkMarkdown(content, options) {
|
|
43
|
+
const opts = normalizeOptions({ ...DEFAULT_OPTIONS, ...options ?? {} });
|
|
44
|
+
const { body, bodyOffset } = stripFrontmatter(content);
|
|
45
|
+
if (body.trim() === "") return [];
|
|
46
|
+
const sections = splitSections(body, bodyOffset);
|
|
47
|
+
const chunks = [];
|
|
48
|
+
for (const section of sections) {
|
|
49
|
+
for (const part of chunkText(section.text, opts)) {
|
|
50
|
+
const start = section.start + part.start;
|
|
51
|
+
chunks.push({
|
|
52
|
+
index: chunks.length,
|
|
53
|
+
text: part.text,
|
|
54
|
+
headingPath: section.headingPath,
|
|
55
|
+
charStart: start,
|
|
56
|
+
charEnd: start + part.text.length,
|
|
57
|
+
oversized: part.text.length > opts.maxChars
|
|
58
|
+
});
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
return chunks;
|
|
62
|
+
}
|
|
63
|
+
function stripFrontmatter(content) {
|
|
64
|
+
const normalized = content.replace(/\r\n/g, "\n");
|
|
65
|
+
if (!normalized.startsWith("---\n")) return { body: normalized, bodyOffset: 0 };
|
|
66
|
+
const match = /^---\n[\s\S]*?\n---\s*\n?/.exec(normalized);
|
|
67
|
+
if (!match) return { body: normalized, bodyOffset: 0 };
|
|
68
|
+
return { body: normalized.slice(match[0].length), bodyOffset: match[0].length };
|
|
69
|
+
}
|
|
70
|
+
function normalizeOptions(opts) {
|
|
71
|
+
if (opts.maxChars < opts.targetChars) opts.maxChars = opts.targetChars;
|
|
72
|
+
if (opts.overlapChars >= opts.targetChars) opts.overlapChars = Math.floor(opts.targetChars / 2);
|
|
73
|
+
return opts;
|
|
74
|
+
}
|
|
75
|
+
function splitSections(body, bodyOffset) {
|
|
76
|
+
const lines = body.split("\n");
|
|
77
|
+
const sections = [];
|
|
78
|
+
const headings = {};
|
|
79
|
+
let current = { lines: [], start: bodyOffset, headingPath: "" };
|
|
80
|
+
let cursor = bodyOffset;
|
|
81
|
+
let fence = null;
|
|
82
|
+
const flush = () => {
|
|
83
|
+
const text = current.lines.join("\n");
|
|
84
|
+
if (text.trim() !== "") sections.push({ text, start: current.start, headingPath: current.headingPath });
|
|
85
|
+
};
|
|
86
|
+
for (let i = 0; i < lines.length; i++) {
|
|
87
|
+
const line = lines[i];
|
|
88
|
+
const lineLen = line.length + (i < lines.length - 1 ? 1 : 0);
|
|
89
|
+
const fenceMatch = /^(`{3,}|~{3,})/.exec(line);
|
|
90
|
+
if (fenceMatch) {
|
|
91
|
+
const marker = fenceMatch[1];
|
|
92
|
+
fence = fence === null ? marker : line.startsWith(fence) ? null : fence;
|
|
93
|
+
current.lines.push(line);
|
|
94
|
+
cursor += lineLen;
|
|
95
|
+
continue;
|
|
96
|
+
}
|
|
97
|
+
const heading = fence === null ? /^(#{1,6})\s+(.+?)\s*$/.exec(line) : null;
|
|
98
|
+
if (heading) {
|
|
99
|
+
flush();
|
|
100
|
+
const level = heading[1].length;
|
|
101
|
+
headings[level] = heading[2].trim();
|
|
102
|
+
for (let next = level + 1; next <= 6; next++) delete headings[next];
|
|
103
|
+
current = {
|
|
104
|
+
lines: [line],
|
|
105
|
+
start: cursor,
|
|
106
|
+
headingPath: Object.entries(headings).sort(([a], [b]) => Number(a) - Number(b)).map(([levelText, title]) => `${"#".repeat(Number(levelText))} ${title}`).join(" > ")
|
|
107
|
+
};
|
|
108
|
+
cursor += lineLen;
|
|
109
|
+
continue;
|
|
110
|
+
}
|
|
111
|
+
current.lines.push(line);
|
|
112
|
+
cursor += lineLen;
|
|
113
|
+
}
|
|
114
|
+
flush();
|
|
115
|
+
return sections;
|
|
116
|
+
}
|
|
117
|
+
function chunkText(text, opts) {
|
|
118
|
+
if (text.length <= opts.targetChars) return [{ text, start: 0 }];
|
|
119
|
+
const atoms = splitAtoms(text);
|
|
120
|
+
const chunks = [];
|
|
121
|
+
let buffer = "";
|
|
122
|
+
let bufferStart = 0;
|
|
123
|
+
for (const atom of atoms) {
|
|
124
|
+
if (buffer === "") bufferStart = atom.start;
|
|
125
|
+
if (buffer.length > 0 && buffer.length + atom.text.length > opts.targetChars) {
|
|
126
|
+
chunks.push({ text: buffer.trimEnd(), start: bufferStart });
|
|
127
|
+
const overlap = buffer.slice(Math.max(0, buffer.length - opts.overlapChars));
|
|
128
|
+
buffer = overlap + atom.text;
|
|
129
|
+
bufferStart = Math.max(bufferStart, atom.start - overlap.length);
|
|
130
|
+
} else {
|
|
131
|
+
buffer += atom.text;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
if (buffer.trim() !== "") chunks.push({ text: buffer.trimEnd(), start: bufferStart });
|
|
135
|
+
return mergeTinyChunks(chunks, opts);
|
|
136
|
+
}
|
|
137
|
+
function splitAtoms(text) {
|
|
138
|
+
const parts = [];
|
|
139
|
+
const paragraphs = text.split(/(\n\s*\n)/);
|
|
140
|
+
let cursor = 0;
|
|
141
|
+
for (let i = 0; i < paragraphs.length; i += 2) {
|
|
142
|
+
const paragraph = (paragraphs[i] ?? "") + (paragraphs[i + 1] ?? "");
|
|
143
|
+
if (paragraph === "") continue;
|
|
144
|
+
parts.push({ text: paragraph, start: cursor });
|
|
145
|
+
cursor += paragraph.length;
|
|
146
|
+
}
|
|
147
|
+
return parts;
|
|
148
|
+
}
|
|
149
|
+
function mergeTinyChunks(chunks, opts) {
|
|
150
|
+
const out = [];
|
|
151
|
+
for (const chunk of chunks) {
|
|
152
|
+
const prev = out[out.length - 1];
|
|
153
|
+
if (prev && chunk.text.length < opts.minChars && prev.text.length + chunk.text.length <= opts.maxChars) {
|
|
154
|
+
prev.text = `${prev.text}
|
|
155
|
+
|
|
156
|
+
${chunk.text}`;
|
|
157
|
+
} else {
|
|
158
|
+
out.push({ ...chunk });
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return out;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// src/optimization.ts
|
|
165
|
+
import {
|
|
166
|
+
runMultiShotOptimization
|
|
167
|
+
} from "@tangle-network/agent-eval";
|
|
168
|
+
async function runKnowledgeBaseOptimization(config) {
|
|
169
|
+
return runMultiShotOptimization({
|
|
170
|
+
...config,
|
|
171
|
+
target: config.target ?? "agent-knowledge-base"
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
function knowledgeVariantFromCandidate(candidate, options = {}) {
|
|
175
|
+
return {
|
|
176
|
+
id: options.id ?? candidate.id,
|
|
177
|
+
label: options.label ?? candidate.id,
|
|
178
|
+
generation: options.generation ?? 0,
|
|
179
|
+
payload: candidate
|
|
180
|
+
};
|
|
181
|
+
}
|
|
182
|
+
export {
|
|
183
|
+
WIKILINK_REGEX,
|
|
184
|
+
addSourcePath,
|
|
185
|
+
applyKnowledgeWriteBlocks,
|
|
186
|
+
applyKnowledgeWriteBlocksFile,
|
|
187
|
+
buildKnowledgeGraph,
|
|
188
|
+
buildKnowledgeIndex,
|
|
189
|
+
chunkMarkdown,
|
|
190
|
+
explainKnowledgeTarget,
|
|
191
|
+
extractWikilinks,
|
|
192
|
+
formatFrontmatter,
|
|
193
|
+
initKnowledgeBase,
|
|
194
|
+
inspectKnowledgeIndex,
|
|
195
|
+
isSafeKnowledgePath,
|
|
196
|
+
knowledgeVariantFromCandidate,
|
|
197
|
+
layoutFor,
|
|
198
|
+
lintKnowledgeIndex,
|
|
199
|
+
loadKnowledgePages,
|
|
200
|
+
loadSourceRegistry,
|
|
201
|
+
mediaTypeFor,
|
|
202
|
+
normalizeLinkTarget,
|
|
203
|
+
parseFrontmatter,
|
|
204
|
+
parseKnowledgeWriteBlocks,
|
|
205
|
+
reciprocalRankFusion,
|
|
206
|
+
runKnowledgeBaseOptimization,
|
|
207
|
+
searchKnowledge,
|
|
208
|
+
sha256,
|
|
209
|
+
slugify,
|
|
210
|
+
sourceRegistryPath,
|
|
211
|
+
stableId,
|
|
212
|
+
stripFrontmatter,
|
|
213
|
+
textSourceAdapter,
|
|
214
|
+
tokenizeQuery,
|
|
215
|
+
writeJson,
|
|
216
|
+
writeKnowledgeIndex,
|
|
217
|
+
writeSourceRegistry
|
|
218
|
+
};
|
|
219
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/chunking.ts","../src/optimization.ts"],"sourcesContent":["export interface ChunkingOptions {\n targetChars: number\n maxChars: number\n minChars: number\n overlapChars: number\n}\n\nexport interface KnowledgeChunk {\n index: number\n text: string\n headingPath: string\n charStart: number\n charEnd: number\n oversized: boolean\n}\n\nconst DEFAULT_OPTIONS: ChunkingOptions = {\n targetChars: 1000,\n maxChars: 1500,\n minChars: 160,\n overlapChars: 180,\n}\n\nexport function chunkMarkdown(content: string, options?: Partial<ChunkingOptions>): KnowledgeChunk[] {\n const opts = normalizeOptions({ ...DEFAULT_OPTIONS, ...(options ?? {}) })\n const { body, bodyOffset } = stripFrontmatter(content)\n if (body.trim() === '') return []\n\n const sections = splitSections(body, bodyOffset)\n const chunks: KnowledgeChunk[] = []\n for (const section of sections) {\n for (const part of chunkText(section.text, opts)) {\n const start = section.start + part.start\n chunks.push({\n index: chunks.length,\n text: part.text,\n headingPath: section.headingPath,\n charStart: start,\n charEnd: start + part.text.length,\n oversized: part.text.length > opts.maxChars,\n })\n }\n }\n return chunks\n}\n\nexport function stripFrontmatter(content: string): { body: string; bodyOffset: number } {\n const normalized = content.replace(/\\r\\n/g, '\\n')\n if (!normalized.startsWith('---\\n')) return { body: normalized, bodyOffset: 0 }\n const match = /^---\\n[\\s\\S]*?\\n---\\s*\\n?/.exec(normalized)\n if (!match) return { body: normalized, bodyOffset: 0 }\n return { body: normalized.slice(match[0].length), bodyOffset: match[0].length }\n}\n\nfunction normalizeOptions(opts: ChunkingOptions): ChunkingOptions {\n if (opts.maxChars < opts.targetChars) opts.maxChars = opts.targetChars\n if (opts.overlapChars >= opts.targetChars) opts.overlapChars = Math.floor(opts.targetChars / 2)\n return opts\n}\n\ninterface Section {\n text: string\n start: number\n headingPath: string\n}\n\nfunction splitSections(body: string, bodyOffset: number): Section[] {\n const lines = body.split('\\n')\n const sections: Section[] = []\n const headings: Record<number, string> = {}\n let current: { lines: string[]; start: number; headingPath: string } = { lines: [], start: bodyOffset, headingPath: '' }\n let cursor = bodyOffset\n let fence: string | null = null\n\n const flush = () => {\n const text = current.lines.join('\\n')\n if (text.trim() !== '') sections.push({ text, start: current.start, headingPath: current.headingPath })\n }\n\n for (let i = 0; i < lines.length; i++) {\n const line = lines[i]!\n const lineLen = line.length + (i < lines.length - 1 ? 1 : 0)\n const fenceMatch = /^(`{3,}|~{3,})/.exec(line)\n if (fenceMatch) {\n const marker = fenceMatch[1]!\n fence = fence === null ? marker : line.startsWith(fence) ? null : fence\n current.lines.push(line)\n cursor += lineLen\n continue\n }\n const heading = fence === null ? /^(#{1,6})\\s+(.+?)\\s*$/.exec(line) : null\n if (heading) {\n flush()\n const level = heading[1]!.length\n headings[level] = heading[2]!.trim()\n for (let next = level + 1; next <= 6; next++) delete headings[next]\n current = {\n lines: [line],\n start: cursor,\n headingPath: Object.entries(headings)\n .sort(([a], [b]) => Number(a) - Number(b))\n .map(([levelText, title]) => `${'#'.repeat(Number(levelText))} ${title}`)\n .join(' > '),\n }\n cursor += lineLen\n continue\n }\n current.lines.push(line)\n cursor += lineLen\n }\n flush()\n return sections\n}\n\nfunction chunkText(text: string, opts: ChunkingOptions): Array<{ text: string; start: number }> {\n if (text.length <= opts.targetChars) return [{ text, start: 0 }]\n const atoms = splitAtoms(text)\n const chunks: Array<{ text: string; start: number }> = []\n let buffer = ''\n let bufferStart = 0\n for (const atom of atoms) {\n if (buffer === '') bufferStart = atom.start\n if (buffer.length > 0 && buffer.length + atom.text.length > opts.targetChars) {\n chunks.push({ text: buffer.trimEnd(), start: bufferStart })\n const overlap = buffer.slice(Math.max(0, buffer.length - opts.overlapChars))\n buffer = overlap + atom.text\n bufferStart = Math.max(bufferStart, atom.start - overlap.length)\n } else {\n buffer += atom.text\n }\n }\n if (buffer.trim() !== '') chunks.push({ text: buffer.trimEnd(), start: bufferStart })\n return mergeTinyChunks(chunks, opts)\n}\n\nfunction splitAtoms(text: string): Array<{ text: string; start: number }> {\n const parts: Array<{ text: string; start: number }> = []\n const paragraphs = text.split(/(\\n\\s*\\n)/)\n let cursor = 0\n for (let i = 0; i < paragraphs.length; i += 2) {\n const paragraph = (paragraphs[i] ?? '') + (paragraphs[i + 1] ?? '')\n if (paragraph === '') continue\n parts.push({ text: paragraph, start: cursor })\n cursor += paragraph.length\n }\n return parts\n}\n\nfunction mergeTinyChunks(chunks: Array<{ text: string; start: number }>, opts: ChunkingOptions): Array<{ text: string; start: number }> {\n const out: Array<{ text: string; start: number }> = []\n for (const chunk of chunks) {\n const prev = out[out.length - 1]\n if (prev && chunk.text.length < opts.minChars && prev.text.length + chunk.text.length <= opts.maxChars) {\n prev.text = `${prev.text}\\n\\n${chunk.text}`\n } else {\n out.push({ ...chunk })\n }\n }\n return out\n}\n","import {\n runMultiShotOptimization,\n type MultiShotMutateAdapter,\n type MultiShotOptimizationConfig,\n type MultiShotOptimizationResult,\n type MultiShotRunner,\n type MultiShotScorer,\n type MultiShotVariant,\n} from '@tangle-network/agent-eval'\nimport type { KnowledgeBaseCandidate } from './types'\n\nexport type KnowledgeBaseVariant = MultiShotVariant<KnowledgeBaseCandidate>\n\nexport interface KnowledgeOptimizationConfig\n extends Omit<\n MultiShotOptimizationConfig<KnowledgeBaseCandidate>,\n 'runner' | 'scorer' | 'mutateAdapter' | 'target'\n > {\n target?: string\n runner: MultiShotRunner<KnowledgeBaseCandidate>\n scorer: MultiShotScorer<KnowledgeBaseCandidate>\n mutateAdapter: MultiShotMutateAdapter<KnowledgeBaseCandidate>\n}\n\nexport async function runKnowledgeBaseOptimization(\n config: KnowledgeOptimizationConfig,\n): Promise<MultiShotOptimizationResult<KnowledgeBaseCandidate>> {\n return runMultiShotOptimization<KnowledgeBaseCandidate>({\n ...config,\n target: config.target ?? 'agent-knowledge-base',\n })\n}\n\nexport function knowledgeVariantFromCandidate(\n candidate: KnowledgeBaseCandidate,\n options: { id?: string; label?: string; generation?: number } = {},\n): KnowledgeBaseVariant {\n return {\n id: options.id ?? candidate.id,\n label: options.label ?? candidate.id,\n generation: options.generation ?? 0,\n payload: candidate,\n }\n}\n"],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAgBA,IAAM,kBAAmC;AAAA,EACvC,aAAa;AAAA,EACb,UAAU;AAAA,EACV,UAAU;AAAA,EACV,cAAc;AAChB;AAEO,SAAS,cAAc,SAAiB,SAAsD;AACnG,QAAM,OAAO,iBAAiB,EAAE,GAAG,iBAAiB,GAAI,WAAW,CAAC,EAAG,CAAC;AACxE,QAAM,EAAE,MAAM,WAAW,IAAI,iBAAiB,OAAO;AACrD,MAAI,KAAK,KAAK,MAAM,GAAI,QAAO,CAAC;AAEhC,QAAM,WAAW,cAAc,MAAM,UAAU;AAC/C,QAAM,SAA2B,CAAC;AAClC,aAAW,WAAW,UAAU;AAC9B,eAAW,QAAQ,UAAU,QAAQ,MAAM,IAAI,GAAG;AAChD,YAAM,QAAQ,QAAQ,QAAQ,KAAK;AACnC,aAAO,KAAK;AAAA,QACV,OAAO,OAAO;AAAA,QACd,MAAM,KAAK;AAAA,QACX,aAAa,QAAQ;AAAA,QACrB,WAAW;AAAA,QACX,SAAS,QAAQ,KAAK,KAAK;AAAA,QAC3B,WAAW,KAAK,KAAK,SAAS,KAAK;AAAA,MACrC,CAAC;AAAA,IACH;AAAA,EACF;AACA,SAAO;AACT;AAEO,SAAS,iBAAiB,SAAuD;AACtF,QAAM,aAAa,QAAQ,QAAQ,SAAS,IAAI;AAChD,MAAI,CAAC,WAAW,WAAW,OAAO,EAAG,QAAO,EAAE,MAAM,YAAY,YAAY,EAAE;AAC9E,QAAM,QAAQ,4BAA4B,KAAK,UAAU;AACzD,MAAI,CAAC,MAAO,QAAO,EAAE,MAAM,YAAY,YAAY,EAAE;AACrD,SAAO,EAAE,MAAM,WAAW,MAAM,MAAM,CAAC,EAAE,MAAM,GAAG,YAAY,MAAM,CAAC,EAAE,OAAO;AAChF;AAEA,SAAS,iBAAiB,MAAwC;AAChE,MAAI,KAAK,WAAW,KAAK,YAAa,MAAK,WAAW,KAAK;AAC3D,MAAI,KAAK,gBAAgB,KAAK,YAAa,MAAK,eAAe,KAAK,MAAM,KAAK,cAAc,CAAC;AAC9F,SAAO;AACT;AAQA,SAAS,cAAc,MAAc,YAA+B;AAClE,QAAM,QAAQ,KAAK,MAAM,IAAI;AAC7B,QAAM,WAAsB,CAAC;AAC7B,QAAM,WAAmC,CAAC;AAC1C,MAAI,UAAmE,EAAE,OAAO,CAAC,GAAG,OAAO,YAAY,aAAa,GAAG;AACvH,MAAI,SAAS;AACb,MAAI,QAAuB;AAE3B,QAAM,QAAQ,MAAM;AAClB,UAAM,OAAO,QAAQ,MAAM,KAAK,IAAI;AACpC,QAAI,KAAK,KAAK,MAAM,GAAI,UAAS,KAAK,EAAE,MAAM,OAAO,QAAQ,OAAO,aAAa,QAAQ,YAAY,CAAC;AAAA,EACxG;AAEA,WAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK;AACrC,UAAM,OAAO,MAAM,CAAC;AACpB,UAAM,UAAU,KAAK,UAAU,IAAI,MAAM,SAAS,IAAI,IAAI;AAC1D,UAAM,aAAa,iBAAiB,KAAK,IAAI;AAC7C,QAAI,YAAY;AACd,YAAM,SAAS,WAAW,CAAC;AAC3B,cAAQ,UAAU,OAAO,SAAS,KAAK,WAAW,KAAK,IAAI,OAAO;AAClE,cAAQ,MAAM,KAAK,IAAI;AACvB,gBAAU;AACV;AAAA,IACF;AACA,UAAM,UAAU,UAAU,OAAO,wBAAwB,KAAK,IAAI,IAAI;AACtE,QAAI,SAAS;AACX,YAAM;AACN,YAAM,QAAQ,QAAQ,CAAC,EAAG;AAC1B,eAAS,KAAK,IAAI,QAAQ,CAAC,EAAG,KAAK;AACnC,eAAS,OAAO,QAAQ,GAAG,QAAQ,GAAG,OAAQ,QAAO,SAAS,IAAI;AAClE,gBAAU;AAAA,QACR,OAAO,CAAC,IAAI;AAAA,QACZ,OAAO;AAAA,QACP,aAAa,OAAO,QAAQ,QAAQ,EACjC,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,OAAO,CAAC,IAAI,OAAO,CAAC,CAAC,EACxC,IAAI,CAAC,CAAC,WAAW,KAAK,MAAM,GAAG,IAAI,OAAO,OAAO,SAAS,CAAC,CAAC,IAAI,KAAK,EAAE,EACvE,KAAK,KAAK;AAAA,MACf;AACA,gBAAU;AACV;AAAA,IACF;AACA,YAAQ,MAAM,KAAK,IAAI;AACvB,cAAU;AAAA,EACZ;AACA,QAAM;AACN,SAAO;AACT;AAEA,SAAS,UAAU,MAAc,MAA+D;AAC9F,MAAI,KAAK,UAAU,KAAK,YAAa,QAAO,CAAC,EAAE,MAAM,OAAO,EAAE,CAAC;AAC/D,QAAM,QAAQ,WAAW,IAAI;AAC7B,QAAM,SAAiD,CAAC;AACxD,MAAI,SAAS;AACb,MAAI,cAAc;AAClB,aAAW,QAAQ,OAAO;AACxB,QAAI,WAAW,GAAI,eAAc,KAAK;AACtC,QAAI,OAAO,SAAS,KAAK,OAAO,SAAS,KAAK,KAAK,SAAS,KAAK,aAAa;AAC5E,aAAO,KAAK,EAAE,MAAM,OAAO,QAAQ,GAAG,OAAO,YAAY,CAAC;AAC1D,YAAM,UAAU,OAAO,MAAM,KAAK,IAAI,GAAG,OAAO,SAAS,KAAK,YAAY,CAAC;AAC3E,eAAS,UAAU,KAAK;AACxB,oBAAc,KAAK,IAAI,aAAa,KAAK,QAAQ,QAAQ,MAAM;AAAA,IACjE,OAAO;AACL,gBAAU,KAAK;AAAA,IACjB;AAAA,EACF;AACA,MAAI,OAAO,KAAK,MAAM,GAAI,QAAO,KAAK,EAAE,MAAM,OAAO,QAAQ,GAAG,OAAO,YAAY,CAAC;AACpF,SAAO,gBAAgB,QAAQ,IAAI;AACrC;AAEA,SAAS,WAAW,MAAsD;AACxE,QAAM,QAAgD,CAAC;AACvD,QAAM,aAAa,KAAK,MAAM,WAAW;AACzC,MAAI,SAAS;AACb,WAAS,IAAI,GAAG,IAAI,WAAW,QAAQ,KAAK,GAAG;AAC7C,UAAM,aAAa,WAAW,CAAC,KAAK,OAAO,WAAW,IAAI,CAAC,KAAK;AAChE,QAAI,cAAc,GAAI;AACtB,UAAM,KAAK,EAAE,MAAM,WAAW,OAAO,OAAO,CAAC;AAC7C,cAAU,UAAU;AAAA,EACtB;AACA,SAAO;AACT;AAEA,SAAS,gBAAgB,QAAgD,MAA+D;AACtI,QAAM,MAA8C,CAAC;AACrD,aAAW,SAAS,QAAQ;AAC1B,UAAM,OAAO,IAAI,IAAI,SAAS,CAAC;AAC/B,QAAI,QAAQ,MAAM,KAAK,SAAS,KAAK,YAAY,KAAK,KAAK,SAAS,MAAM,KAAK,UAAU,KAAK,UAAU;AACtG,WAAK,OAAO,GAAG,KAAK,IAAI;AAAA;AAAA,EAAO,MAAM,IAAI;AAAA,IAC3C,OAAO;AACL,UAAI,KAAK,EAAE,GAAG,MAAM,CAAC;AAAA,IACvB;AAAA,EACF;AACA,SAAO;AACT;;;AC/JA;AAAA,EACE;AAAA,OAOK;AAgBP,eAAsB,6BACpB,QAC8D;AAC9D,SAAO,yBAAiD;AAAA,IACtD,GAAG;AAAA,IACH,QAAQ,OAAO,UAAU;AAAA,EAC3B,CAAC;AACH;AAEO,SAAS,8BACd,WACA,UAAgE,CAAC,GAC3C;AACtB,SAAO;AAAA,IACL,IAAI,QAAQ,MAAM,UAAU;AAAA,IAC5B,OAAO,QAAQ,SAAS,UAAU;AAAA,IAClC,YAAY,QAAQ,cAAc;AAAA,IAClC,SAAS;AAAA,EACX;AACF;","names":[]}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
type KnowledgeId = string;
|
|
2
|
+
interface SourceAnchor {
|
|
3
|
+
id: string;
|
|
4
|
+
sourceId: string;
|
|
5
|
+
label?: string;
|
|
6
|
+
page?: number;
|
|
7
|
+
lineStart?: number;
|
|
8
|
+
lineEnd?: number;
|
|
9
|
+
charStart?: number;
|
|
10
|
+
charEnd?: number;
|
|
11
|
+
timestampMs?: number;
|
|
12
|
+
metadata?: Record<string, unknown>;
|
|
13
|
+
}
|
|
14
|
+
interface SourceRecord {
|
|
15
|
+
id: KnowledgeId;
|
|
16
|
+
uri: string;
|
|
17
|
+
title?: string;
|
|
18
|
+
mediaType?: string;
|
|
19
|
+
contentHash: string;
|
|
20
|
+
text?: string;
|
|
21
|
+
anchors?: SourceAnchor[];
|
|
22
|
+
metadata?: Record<string, unknown>;
|
|
23
|
+
createdAt: string;
|
|
24
|
+
}
|
|
25
|
+
interface SourceRegistry {
|
|
26
|
+
generatedAt: string;
|
|
27
|
+
sources: SourceRecord[];
|
|
28
|
+
}
|
|
29
|
+
interface ClaimRef {
|
|
30
|
+
sourceId: string;
|
|
31
|
+
anchorId?: string;
|
|
32
|
+
quote?: string;
|
|
33
|
+
}
|
|
34
|
+
interface KnowledgeClaim {
|
|
35
|
+
id: KnowledgeId;
|
|
36
|
+
text: string;
|
|
37
|
+
refs: ClaimRef[];
|
|
38
|
+
confidence?: number;
|
|
39
|
+
status?: 'draft' | 'active' | 'superseded' | 'rejected';
|
|
40
|
+
metadata?: Record<string, unknown>;
|
|
41
|
+
}
|
|
42
|
+
interface KnowledgeRelation {
|
|
43
|
+
sourceId: KnowledgeId;
|
|
44
|
+
targetId: KnowledgeId;
|
|
45
|
+
predicate: string;
|
|
46
|
+
weight?: number;
|
|
47
|
+
metadata?: Record<string, unknown>;
|
|
48
|
+
}
|
|
49
|
+
interface KnowledgeUnit {
|
|
50
|
+
id: KnowledgeId;
|
|
51
|
+
title: string;
|
|
52
|
+
text: string;
|
|
53
|
+
claims?: KnowledgeClaim[];
|
|
54
|
+
relations?: KnowledgeRelation[];
|
|
55
|
+
sourceIds?: string[];
|
|
56
|
+
tags?: string[];
|
|
57
|
+
metadata?: Record<string, unknown>;
|
|
58
|
+
updatedAt?: string;
|
|
59
|
+
}
|
|
60
|
+
interface KnowledgePage {
|
|
61
|
+
id: KnowledgeId;
|
|
62
|
+
path: string;
|
|
63
|
+
title: string;
|
|
64
|
+
text: string;
|
|
65
|
+
frontmatter: Record<string, unknown>;
|
|
66
|
+
sourceIds: string[];
|
|
67
|
+
tags: string[];
|
|
68
|
+
outLinks: string[];
|
|
69
|
+
}
|
|
70
|
+
interface KnowledgeGraphNode {
|
|
71
|
+
id: KnowledgeId;
|
|
72
|
+
title: string;
|
|
73
|
+
path: string;
|
|
74
|
+
tags: string[];
|
|
75
|
+
sourceIds: string[];
|
|
76
|
+
outDegree: number;
|
|
77
|
+
inDegree: number;
|
|
78
|
+
}
|
|
79
|
+
interface KnowledgeGraphEdge {
|
|
80
|
+
source: KnowledgeId;
|
|
81
|
+
target: KnowledgeId;
|
|
82
|
+
weight: number;
|
|
83
|
+
reasons: string[];
|
|
84
|
+
}
|
|
85
|
+
interface KnowledgeGraph {
|
|
86
|
+
nodes: KnowledgeGraphNode[];
|
|
87
|
+
edges: KnowledgeGraphEdge[];
|
|
88
|
+
}
|
|
89
|
+
interface KnowledgeIndex {
|
|
90
|
+
root: string;
|
|
91
|
+
generatedAt: string;
|
|
92
|
+
sources: SourceRecord[];
|
|
93
|
+
pages: KnowledgePage[];
|
|
94
|
+
graph: KnowledgeGraph;
|
|
95
|
+
}
|
|
96
|
+
interface KnowledgeSearchResult {
|
|
97
|
+
page: KnowledgePage;
|
|
98
|
+
score: number;
|
|
99
|
+
rank: number;
|
|
100
|
+
snippet: string;
|
|
101
|
+
reasons: string[];
|
|
102
|
+
}
|
|
103
|
+
interface KnowledgeLintFinding {
|
|
104
|
+
type: 'broken-link' | 'orphan' | 'no-outlinks' | 'uncited-claim' | 'missing-source' | 'duplicate-title';
|
|
105
|
+
severity: 'info' | 'warning' | 'error';
|
|
106
|
+
page?: string;
|
|
107
|
+
message: string;
|
|
108
|
+
metadata?: Record<string, unknown>;
|
|
109
|
+
}
|
|
110
|
+
interface KnowledgePolicy {
|
|
111
|
+
id: string;
|
|
112
|
+
description?: string;
|
|
113
|
+
requiredCitationRate?: number;
|
|
114
|
+
allowedPathPrefixes?: string[];
|
|
115
|
+
metadata?: Record<string, unknown>;
|
|
116
|
+
}
|
|
117
|
+
interface KnowledgeBaseCandidate {
|
|
118
|
+
id: KnowledgeId;
|
|
119
|
+
units: KnowledgeUnit[];
|
|
120
|
+
retrievalPolicy?: string;
|
|
121
|
+
synthesisPolicy?: string;
|
|
122
|
+
questionPolicy?: string;
|
|
123
|
+
updatePolicy?: string;
|
|
124
|
+
metadata?: Record<string, unknown>;
|
|
125
|
+
}
|
|
126
|
+
interface KnowledgeWriteBlock {
|
|
127
|
+
path: string;
|
|
128
|
+
content: string;
|
|
129
|
+
}
|
|
130
|
+
interface KnowledgeWriteParseResult {
|
|
131
|
+
blocks: KnowledgeWriteBlock[];
|
|
132
|
+
warnings: string[];
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export type { ClaimRef as C, KnowledgeGraphEdge as K, SourceRecord as S, KnowledgeGraphNode as a, KnowledgeGraph as b, KnowledgeWriteParseResult as c, KnowledgePage as d, SourceRegistry as e, KnowledgeIndex as f, KnowledgeSearchResult as g, KnowledgeLintFinding as h, KnowledgeBaseCandidate as i, KnowledgeClaim as j, KnowledgeId as k, KnowledgePolicy as l, KnowledgeRelation as m, KnowledgeUnit as n, KnowledgeWriteBlock as o, SourceAnchor as p };
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { K as KnowledgeGraphEdge, a as KnowledgeGraphNode, b as KnowledgeGraph } from '../types-C1FBNRIN.js';
|
|
2
|
+
|
|
3
|
+
interface KnowledgeVizNode extends KnowledgeGraphNode {
|
|
4
|
+
degree: number;
|
|
5
|
+
community: number;
|
|
6
|
+
}
|
|
7
|
+
interface KnowledgeVizEdge extends KnowledgeGraphEdge {
|
|
8
|
+
id: string;
|
|
9
|
+
}
|
|
10
|
+
interface KnowledgeCommunity {
|
|
11
|
+
id: number;
|
|
12
|
+
nodeIds: string[];
|
|
13
|
+
topTitles: string[];
|
|
14
|
+
cohesion: number;
|
|
15
|
+
}
|
|
16
|
+
interface KnowledgeVizGraph {
|
|
17
|
+
nodes: KnowledgeVizNode[];
|
|
18
|
+
edges: KnowledgeVizEdge[];
|
|
19
|
+
communities: KnowledgeCommunity[];
|
|
20
|
+
}
|
|
21
|
+
interface KnowledgeGap {
|
|
22
|
+
type: 'isolated-node' | 'sparse-community' | 'bridge-node';
|
|
23
|
+
title: string;
|
|
24
|
+
nodeIds: string[];
|
|
25
|
+
suggestion: string;
|
|
26
|
+
}
|
|
27
|
+
interface SurprisingConnection {
|
|
28
|
+
source: KnowledgeVizNode;
|
|
29
|
+
target: KnowledgeVizNode;
|
|
30
|
+
score: number;
|
|
31
|
+
reasons: string[];
|
|
32
|
+
}
|
|
33
|
+
declare function toKnowledgeVizGraph(graph: KnowledgeGraph): KnowledgeVizGraph;
|
|
34
|
+
declare function detectKnowledgeGaps(graph: KnowledgeVizGraph, limit?: number): KnowledgeGap[];
|
|
35
|
+
declare function findSurprisingConnections(graph: KnowledgeVizGraph, limit?: number): SurprisingConnection[];
|
|
36
|
+
|
|
37
|
+
export { type KnowledgeCommunity, type KnowledgeGap, type KnowledgeVizEdge, type KnowledgeVizGraph, type KnowledgeVizNode, type SurprisingConnection, detectKnowledgeGaps, findSurprisingConnections, toKnowledgeVizGraph };
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":[],"sourcesContent":[],"mappings":"","names":[]}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# Architecture
|
|
2
|
+
|
|
3
|
+
`@tangle-network/agent-knowledge` is a domain-agnostic knowledge growth layer for agents.
|
|
4
|
+
|
|
5
|
+
It does not try to be a vector database, a RAG framework, or a product-specific wiki. It owns the small set of primitives every serious agent knowledge system needs:
|
|
6
|
+
|
|
7
|
+
- immutable source records
|
|
8
|
+
- generated knowledge pages and units
|
|
9
|
+
- claims with source references
|
|
10
|
+
- deterministic indexing, graph construction, search, and lint
|
|
11
|
+
- safe LLM write proposals
|
|
12
|
+
- eval-gated optimization through `@tangle-network/agent-eval`
|
|
13
|
+
- visualization DTOs under the `/viz` subpath
|
|
14
|
+
|
|
15
|
+
## Boundaries
|
|
16
|
+
|
|
17
|
+
`agent-eval` owns traces, ASI, multi-shot optimization, run records, and promotion gates.
|
|
18
|
+
|
|
19
|
+
`agent-knowledge` owns sources, claims, pages, graph/search/lint, and knowledge base candidates. It calls `agent-eval` instead of reimplementing evaluation.
|
|
20
|
+
|
|
21
|
+
Product apps own domain policies, source adapters, task corpora, and promotion decisions.
|
|
22
|
+
|
|
23
|
+
## Runtime Loop
|
|
24
|
+
|
|
25
|
+
1. Normalize sources into immutable source records.
|
|
26
|
+
2. Generate staged knowledge write proposals.
|
|
27
|
+
3. Parse write proposals through the safe write protocol.
|
|
28
|
+
4. Validate paths, citations, links, and schema.
|
|
29
|
+
5. Index generated knowledge pages.
|
|
30
|
+
6. Search and graph-lint the knowledge base.
|
|
31
|
+
7. Evaluate candidate KB variants with `runKnowledgeBaseOptimization`.
|
|
32
|
+
8. Promote only variants that pass downstream gates.
|
|
33
|
+
|
|
34
|
+
## CLI
|
|
35
|
+
|
|
36
|
+
The CLI is intentionally fast and local:
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
agent-knowledge init
|
|
40
|
+
agent-knowledge source-add ./source.md
|
|
41
|
+
agent-knowledge sources
|
|
42
|
+
agent-knowledge apply-write-blocks ./proposal.txt
|
|
43
|
+
agent-knowledge index
|
|
44
|
+
agent-knowledge search "query"
|
|
45
|
+
agent-knowledge inspect
|
|
46
|
+
agent-knowledge explain knowledge/concepts/example.md
|
|
47
|
+
agent-knowledge graph
|
|
48
|
+
agent-knowledge lint
|
|
49
|
+
agent-knowledge viz
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
It does not call an LLM. It operates over markdown and cached JSON indexes so fleet jobs and dev containers can use it cheaply.
|
package/package.json
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@tangle-network/agent-knowledge",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Source-grounded, eval-gated knowledge growth primitives for agents.",
|
|
5
|
+
"homepage": "https://github.com/tangle-network/agent-knowledge#readme",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/tangle-network/agent-knowledge.git"
|
|
9
|
+
},
|
|
10
|
+
"bugs": {
|
|
11
|
+
"url": "https://github.com/tangle-network/agent-knowledge/issues"
|
|
12
|
+
},
|
|
13
|
+
"type": "module",
|
|
14
|
+
"main": "./dist/index.js",
|
|
15
|
+
"types": "./dist/index.d.ts",
|
|
16
|
+
"exports": {
|
|
17
|
+
".": {
|
|
18
|
+
"types": "./dist/index.d.ts",
|
|
19
|
+
"import": "./dist/index.js",
|
|
20
|
+
"default": "./dist/index.js"
|
|
21
|
+
},
|
|
22
|
+
"./viz": {
|
|
23
|
+
"types": "./dist/viz/index.d.ts",
|
|
24
|
+
"import": "./dist/viz/index.js",
|
|
25
|
+
"default": "./dist/viz/index.js"
|
|
26
|
+
},
|
|
27
|
+
"./cli": {
|
|
28
|
+
"types": "./dist/cli.d.ts",
|
|
29
|
+
"import": "./dist/cli.js",
|
|
30
|
+
"default": "./dist/cli.js"
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
"bin": {
|
|
34
|
+
"agent-knowledge": "./dist/cli.js"
|
|
35
|
+
},
|
|
36
|
+
"files": [
|
|
37
|
+
"dist",
|
|
38
|
+
"docs",
|
|
39
|
+
"README.md"
|
|
40
|
+
],
|
|
41
|
+
"publishConfig": {
|
|
42
|
+
"access": "public"
|
|
43
|
+
},
|
|
44
|
+
"scripts": {
|
|
45
|
+
"build": "tsup",
|
|
46
|
+
"dev": "tsup --watch",
|
|
47
|
+
"prepare": "tsup",
|
|
48
|
+
"test": "vitest run",
|
|
49
|
+
"test:watch": "vitest",
|
|
50
|
+
"typecheck": "tsc --noEmit"
|
|
51
|
+
},
|
|
52
|
+
"dependencies": {
|
|
53
|
+
"@tangle-network/agent-eval": "^0.19.1",
|
|
54
|
+
"zod": "^4.3.6"
|
|
55
|
+
},
|
|
56
|
+
"devDependencies": {
|
|
57
|
+
"@types/node": "^25.6.0",
|
|
58
|
+
"tsup": "^8.0.0",
|
|
59
|
+
"typescript": "^5.7.0",
|
|
60
|
+
"vitest": "^3.0.0"
|
|
61
|
+
},
|
|
62
|
+
"engines": {
|
|
63
|
+
"node": ">=20"
|
|
64
|
+
},
|
|
65
|
+
"license": "MIT",
|
|
66
|
+
"packageManager": "pnpm@10.28.0"
|
|
67
|
+
}
|