baselineos 0.2.0-beta.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +17 -0
- package/README.md +198 -0
- package/dist/__evals__/runner.d.ts +2 -0
- package/dist/__evals__/runner.js +14687 -0
- package/dist/__evals__/runner.js.map +1 -0
- package/dist/api/server.d.ts +21 -0
- package/dist/api/server.js +1007 -0
- package/dist/api/server.js.map +1 -0
- package/dist/cli/bin.d.ts +1 -0
- package/dist/cli/bin.js +8427 -0
- package/dist/cli/bin.js.map +1 -0
- package/dist/core/agent-bus.d.ts +110 -0
- package/dist/core/agent-bus.js +242 -0
- package/dist/core/agent-bus.js.map +1 -0
- package/dist/core/cache.d.ts +66 -0
- package/dist/core/cache.js +160 -0
- package/dist/core/cache.js.map +1 -0
- package/dist/core/config.d.ts +1002 -0
- package/dist/core/config.js +429 -0
- package/dist/core/config.js.map +1 -0
- package/dist/core/indexer.d.ts +152 -0
- package/dist/core/indexer.js +481 -0
- package/dist/core/indexer.js.map +1 -0
- package/dist/core/llm-tracer.d.ts +2 -0
- package/dist/core/llm-tracer.js +241 -0
- package/dist/core/llm-tracer.js.map +1 -0
- package/dist/core/memory.d.ts +86 -0
- package/dist/core/memory.js +346 -0
- package/dist/core/memory.js.map +1 -0
- package/dist/core/opa-client.d.ts +51 -0
- package/dist/core/opa-client.js +157 -0
- package/dist/core/opa-client.js.map +1 -0
- package/dist/core/opa-policy-gate.d.ts +133 -0
- package/dist/core/opa-policy-gate.js +454 -0
- package/dist/core/opa-policy-gate.js.map +1 -0
- package/dist/core/orchestrator.d.ts +14 -0
- package/dist/core/orchestrator.js +1297 -0
- package/dist/core/orchestrator.js.map +1 -0
- package/dist/core/pii-detector.d.ts +82 -0
- package/dist/core/pii-detector.js +126 -0
- package/dist/core/pii-detector.js.map +1 -0
- package/dist/core/rag-engine.d.ts +121 -0
- package/dist/core/rag-engine.js +504 -0
- package/dist/core/rag-engine.js.map +1 -0
- package/dist/core/task-queue.d.ts +69 -0
- package/dist/core/task-queue.js +124 -0
- package/dist/core/task-queue.js.map +1 -0
- package/dist/core/telemetry.d.ts +56 -0
- package/dist/core/telemetry.js +94 -0
- package/dist/core/telemetry.js.map +1 -0
- package/dist/core/types.d.ts +328 -0
- package/dist/core/types.js +24 -0
- package/dist/core/types.js.map +1 -0
- package/dist/index.d.ts +21 -0
- package/dist/index.js +12444 -0
- package/dist/index.js.map +1 -0
- package/dist/llm-tracer-CIIujuO-.d.ts +493 -0
- package/dist/mcp/server.d.ts +2651 -0
- package/dist/mcp/server.js +676 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/orchestrator-DF89k_AK.d.ts +506 -0
- package/package.json +157 -0
- package/templates/README.md +7 -0
- package/templates/baseline.config.ts +207 -0
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
import { glob } from 'glob';
|
|
2
|
+
import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
|
|
3
|
+
import { join, extname, basename, dirname } from 'path';
|
|
4
|
+
import matter from 'gray-matter';
|
|
5
|
+
import { createHash } from 'crypto';
|
|
6
|
+
|
|
7
|
+
// src/core/indexer.ts
|
|
8
|
+
function estimateTokens(text) {
|
|
9
|
+
return Math.ceil(text.length / 4);
|
|
10
|
+
}
|
|
11
|
+
function hashContent(content) {
|
|
12
|
+
return createHash("sha256").update(content).digest("hex").slice(0, 16);
|
|
13
|
+
}
|
|
14
|
+
function determineType(filePath) {
|
|
15
|
+
const ext = extname(filePath).toLowerCase();
|
|
16
|
+
const name = basename(filePath).toLowerCase();
|
|
17
|
+
if (name.includes("spec") || name.includes("requirement")) return "spec";
|
|
18
|
+
if (name.includes("api") || name.includes("swagger") || name.includes("openapi")) return "api";
|
|
19
|
+
if ([".json", ".yaml", ".yml", ".toml"].includes(ext)) return "config";
|
|
20
|
+
if ([".md", ".mdx", ".txt", ".rst"].includes(ext)) return "doc";
|
|
21
|
+
return "code";
|
|
22
|
+
}
|
|
23
|
+
function normalizeOverlay(value, fallback) {
|
|
24
|
+
if (value === "org" || value === "repo" || value === "public") return value;
|
|
25
|
+
return fallback;
|
|
26
|
+
}
|
|
27
|
+
function normalizeSensitivity(value, fallback) {
|
|
28
|
+
if (value === "public" || value === "internal" || value === "restricted") return value;
|
|
29
|
+
return fallback;
|
|
30
|
+
}
|
|
31
|
+
function toStringArray(value) {
|
|
32
|
+
if (Array.isArray(value)) {
|
|
33
|
+
return value.map((v) => String(v)).filter(Boolean);
|
|
34
|
+
}
|
|
35
|
+
if (typeof value === "string") {
|
|
36
|
+
return [value];
|
|
37
|
+
}
|
|
38
|
+
return void 0;
|
|
39
|
+
}
|
|
40
|
+
function sensitivityRank(level) {
|
|
41
|
+
switch (level) {
|
|
42
|
+
case "public":
|
|
43
|
+
return 1;
|
|
44
|
+
case "internal":
|
|
45
|
+
return 2;
|
|
46
|
+
case "restricted":
|
|
47
|
+
return 3;
|
|
48
|
+
default:
|
|
49
|
+
return 3;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
function isDocAllowed(doc, policy) {
|
|
53
|
+
if (!policy) return true;
|
|
54
|
+
if (policy.repo && doc.repo && !policy.repo.includes(doc.repo)) return false;
|
|
55
|
+
if (policy.overlays && doc.overlay && !policy.overlays.includes(doc.overlay)) return false;
|
|
56
|
+
if (policy.sensitivity && doc.sensitivity) {
|
|
57
|
+
if (sensitivityRank(doc.sensitivity) > sensitivityRank(policy.sensitivity)) return false;
|
|
58
|
+
}
|
|
59
|
+
if (policy.scopes && doc.scopes && doc.scopes.length > 0) {
|
|
60
|
+
const ok = policy.scopes.some((s) => doc.scopes.includes(s));
|
|
61
|
+
if (!ok) return false;
|
|
62
|
+
}
|
|
63
|
+
if (policy.roles && doc.roles && doc.roles.length > 0) {
|
|
64
|
+
const ok = policy.roles.some((r) => doc.roles.includes(r));
|
|
65
|
+
if (!ok) return false;
|
|
66
|
+
}
|
|
67
|
+
return true;
|
|
68
|
+
}
|
|
69
|
+
function generateCompressions(content, title) {
|
|
70
|
+
const lines = content.split("\n").filter((l) => l.trim());
|
|
71
|
+
const firstLine = lines[0] || title;
|
|
72
|
+
const oneliner = firstLine.slice(0, 80);
|
|
73
|
+
const firstParagraph = content.split(/\n\n/)[0] || "";
|
|
74
|
+
const summary = firstParagraph.slice(0, 400);
|
|
75
|
+
const sections = content.split(/^##?\s+/m);
|
|
76
|
+
const firstSection = sections.length > 1 ? sections[0] + sections[1] : content;
|
|
77
|
+
const working = firstSection.slice(0, 2e3);
|
|
78
|
+
return { oneliner, summary, working };
|
|
79
|
+
}
|
|
80
|
+
var KnowledgeIndexer = class {
|
|
81
|
+
config;
|
|
82
|
+
documents;
|
|
83
|
+
initialized = false;
|
|
84
|
+
indexVersion = 1;
|
|
85
|
+
constructor(config = {}) {
|
|
86
|
+
this.config = {
|
|
87
|
+
projectRoot: config.projectRoot ?? process.cwd(),
|
|
88
|
+
knowledgePaths: config.knowledgePaths ?? ["./docs", "./knowledge"],
|
|
89
|
+
indexPath: config.indexPath ?? ".baseline/index",
|
|
90
|
+
includePatterns: config.includePatterns ?? [
|
|
91
|
+
"**/*.md",
|
|
92
|
+
"**/*.mdx",
|
|
93
|
+
"**/*.ts",
|
|
94
|
+
"**/*.tsx",
|
|
95
|
+
"**/*.js",
|
|
96
|
+
"**/*.py",
|
|
97
|
+
"**/*.json",
|
|
98
|
+
"**/*.yaml",
|
|
99
|
+
"**/*.yml"
|
|
100
|
+
],
|
|
101
|
+
excludePatterns: config.excludePatterns ?? [
|
|
102
|
+
"**/node_modules/**",
|
|
103
|
+
"**/dist/**",
|
|
104
|
+
"**/.git/**",
|
|
105
|
+
"**/coverage/**",
|
|
106
|
+
"**/*.min.js",
|
|
107
|
+
"**/*.map"
|
|
108
|
+
],
|
|
109
|
+
repo: config.repo,
|
|
110
|
+
defaultOverlay: config.defaultOverlay ?? "repo",
|
|
111
|
+
defaultSensitivity: config.defaultSensitivity ?? "internal"
|
|
112
|
+
};
|
|
113
|
+
this.documents = /* @__PURE__ */ new Map();
|
|
114
|
+
}
|
|
115
|
+
async initialize() {
|
|
116
|
+
if (this.initialized) return;
|
|
117
|
+
const indexFile = join(this.config.indexPath, "index.json");
|
|
118
|
+
if (existsSync(indexFile)) {
|
|
119
|
+
try {
|
|
120
|
+
const data = JSON.parse(readFileSync(indexFile, "utf-8"));
|
|
121
|
+
if (data.version === this.indexVersion) {
|
|
122
|
+
for (const doc of data.documents) {
|
|
123
|
+
this.documents.set(doc.id, doc);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
} catch (err) {
|
|
127
|
+
process.stderr.write(`[baseline:indexer] Index corrupted, rebuilding: ${err.message}
|
|
128
|
+
`);
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
this.initialized = true;
|
|
132
|
+
}
|
|
133
|
+
async indexAll() {
|
|
134
|
+
let indexed = 0;
|
|
135
|
+
let errors = 0;
|
|
136
|
+
let skipped = 0;
|
|
137
|
+
const changed = [];
|
|
138
|
+
const removed = [];
|
|
139
|
+
const seen = /* @__PURE__ */ new Set();
|
|
140
|
+
for (const knowledgePath of this.config.knowledgePaths) {
|
|
141
|
+
const fullPath = join(this.config.projectRoot, knowledgePath);
|
|
142
|
+
if (!existsSync(fullPath)) {
|
|
143
|
+
continue;
|
|
144
|
+
}
|
|
145
|
+
const stat = statSync(fullPath);
|
|
146
|
+
if (stat.isFile()) {
|
|
147
|
+
try {
|
|
148
|
+
const result = await this.indexFile(fullPath, knowledgePath);
|
|
149
|
+
seen.add(knowledgePath);
|
|
150
|
+
if (result === "indexed") indexed++;
|
|
151
|
+
else if (result === "skipped") skipped++;
|
|
152
|
+
if (result === "indexed") changed.push(knowledgePath);
|
|
153
|
+
} catch (err) {
|
|
154
|
+
process.stderr.write(`[baseline:indexer] Failed to index ${knowledgePath}: ${err.message}
|
|
155
|
+
`);
|
|
156
|
+
errors++;
|
|
157
|
+
}
|
|
158
|
+
continue;
|
|
159
|
+
}
|
|
160
|
+
const files = await glob(this.config.includePatterns, {
|
|
161
|
+
cwd: fullPath,
|
|
162
|
+
ignore: this.config.excludePatterns,
|
|
163
|
+
nodir: true
|
|
164
|
+
});
|
|
165
|
+
for (const file of files) {
|
|
166
|
+
try {
|
|
167
|
+
const filePath = join(fullPath, file);
|
|
168
|
+
const result = await this.indexFile(filePath, `${knowledgePath}/${file}`);
|
|
169
|
+
seen.add(`${knowledgePath}/${file}`);
|
|
170
|
+
if (result === "indexed") indexed++;
|
|
171
|
+
else if (result === "skipped") skipped++;
|
|
172
|
+
if (result === "indexed") changed.push(`${knowledgePath}/${file}`);
|
|
173
|
+
} catch (err) {
|
|
174
|
+
process.stderr.write(`[baseline:indexer] Failed to index ${knowledgePath}/${file}: ${err.message}
|
|
175
|
+
`);
|
|
176
|
+
errors++;
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
}
|
|
180
|
+
for (const id of this.documents.keys()) {
|
|
181
|
+
if (!seen.has(id)) {
|
|
182
|
+
this.documents.delete(id);
|
|
183
|
+
removed.push(id);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
await this.persist({ changed, removed });
|
|
187
|
+
return { indexed, errors, skipped, changed, removed };
|
|
188
|
+
}
|
|
189
|
+
async indexFile(filePath, id) {
|
|
190
|
+
const content = readFileSync(filePath, "utf-8");
|
|
191
|
+
const hash = hashContent(content);
|
|
192
|
+
const existing = this.documents.get(id);
|
|
193
|
+
if (existing && existing.hash === hash) {
|
|
194
|
+
return "skipped";
|
|
195
|
+
}
|
|
196
|
+
const ext = extname(filePath);
|
|
197
|
+
let title = basename(filePath, ext);
|
|
198
|
+
let processedContent = content;
|
|
199
|
+
let metadata = {};
|
|
200
|
+
if ([".md", ".mdx"].includes(ext)) {
|
|
201
|
+
try {
|
|
202
|
+
const parsed = matter(content);
|
|
203
|
+
title = parsed.data.title || title;
|
|
204
|
+
processedContent = parsed.content;
|
|
205
|
+
metadata = parsed.data;
|
|
206
|
+
} catch {
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
const compressions = generateCompressions(processedContent, title);
|
|
210
|
+
const repo = metadata.repo || this.config.repo || basename(this.config.projectRoot);
|
|
211
|
+
const overlay = normalizeOverlay(metadata.overlay, this.config.defaultOverlay ?? "repo");
|
|
212
|
+
const sensitivity = normalizeSensitivity(
|
|
213
|
+
metadata.sensitivity,
|
|
214
|
+
this.config.defaultSensitivity ?? "internal"
|
|
215
|
+
);
|
|
216
|
+
const scopes = toStringArray(metadata.scopes ?? metadata.scope);
|
|
217
|
+
const roles = toStringArray(metadata.roles ?? metadata.role);
|
|
218
|
+
const doc = {
|
|
219
|
+
id,
|
|
220
|
+
title,
|
|
221
|
+
content: processedContent,
|
|
222
|
+
tokenCount: estimateTokens(processedContent),
|
|
223
|
+
source: filePath,
|
|
224
|
+
type: determineType(filePath),
|
|
225
|
+
hash,
|
|
226
|
+
indexedAt: Date.now(),
|
|
227
|
+
metadata,
|
|
228
|
+
repo,
|
|
229
|
+
overlay,
|
|
230
|
+
sensitivity,
|
|
231
|
+
scopes,
|
|
232
|
+
roles,
|
|
233
|
+
compressions
|
|
234
|
+
};
|
|
235
|
+
this.documents.set(id, doc);
|
|
236
|
+
return "indexed";
|
|
237
|
+
}
|
|
238
|
+
async clear() {
|
|
239
|
+
this.documents.clear();
|
|
240
|
+
}
|
|
241
|
+
async search(query, options) {
|
|
242
|
+
const limit = options?.limit ?? 5;
|
|
243
|
+
const types = options?.types;
|
|
244
|
+
const policy = options?.policy;
|
|
245
|
+
const words = query.toLowerCase().split(/\s+/).filter((w) => w.length > 2);
|
|
246
|
+
if (words.length === 0) {
|
|
247
|
+
return [];
|
|
248
|
+
}
|
|
249
|
+
const results = [];
|
|
250
|
+
for (const doc of this.documents.values()) {
|
|
251
|
+
if (types && !types.includes(doc.type)) {
|
|
252
|
+
continue;
|
|
253
|
+
}
|
|
254
|
+
if (!isDocAllowed(doc, policy)) {
|
|
255
|
+
continue;
|
|
256
|
+
}
|
|
257
|
+
const contentLower = doc.content.toLowerCase();
|
|
258
|
+
const titleLower = doc.title.toLowerCase();
|
|
259
|
+
let score = 0;
|
|
260
|
+
for (const word of words) {
|
|
261
|
+
if (titleLower.includes(word)) score += 3;
|
|
262
|
+
const regex = new RegExp(word, "gi");
|
|
263
|
+
const matches = contentLower.match(regex);
|
|
264
|
+
if (matches) score += Math.min(matches.length, 5);
|
|
265
|
+
}
|
|
266
|
+
if (score > 0) {
|
|
267
|
+
results.push({
|
|
268
|
+
id: doc.id,
|
|
269
|
+
title: doc.title,
|
|
270
|
+
content: doc.compressions.summary,
|
|
271
|
+
score: score / words.length,
|
|
272
|
+
source: doc.source,
|
|
273
|
+
tokenCount: doc.tokenCount,
|
|
274
|
+
type: doc.type,
|
|
275
|
+
repo: doc.repo,
|
|
276
|
+
overlay: doc.overlay,
|
|
277
|
+
sensitivity: doc.sensitivity
|
|
278
|
+
});
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
return results.sort((a, b) => b.score - a.score).slice(0, limit);
|
|
282
|
+
}
|
|
283
|
+
async getContext(subject, level = "working", options) {
|
|
284
|
+
const results = await this.search(subject, { limit: 10, policy: options?.policy });
|
|
285
|
+
if (results.length === 0) {
|
|
286
|
+
return null;
|
|
287
|
+
}
|
|
288
|
+
return this.buildContextFromResults(results, level, options?.overlayOrder, options?.budget);
|
|
289
|
+
}
|
|
290
|
+
async getContextDelta(subject, level = "working", options) {
|
|
291
|
+
const results = await this.search(subject, { limit: 10, policy: options?.policy });
|
|
292
|
+
if (results.length === 0) {
|
|
293
|
+
return null;
|
|
294
|
+
}
|
|
295
|
+
const known = /* @__PURE__ */ new Map();
|
|
296
|
+
for (const item of options?.known ?? []) {
|
|
297
|
+
if (item?.id) known.set(item.id, item.hash);
|
|
298
|
+
}
|
|
299
|
+
const filtered = results.filter((result) => {
|
|
300
|
+
const doc = this.documents.get(result.id);
|
|
301
|
+
if (!doc) return false;
|
|
302
|
+
const knownHash = known.get(doc.id);
|
|
303
|
+
return !knownHash || knownHash !== doc.hash;
|
|
304
|
+
});
|
|
305
|
+
if (filtered.length === 0) {
|
|
306
|
+
return null;
|
|
307
|
+
}
|
|
308
|
+
return this.buildContextFromResults(filtered, level, options?.overlayOrder, options?.budget);
|
|
309
|
+
}
|
|
310
|
+
buildContextFromResults(results, level, overlayOrder, budget) {
|
|
311
|
+
const contextParts = [];
|
|
312
|
+
const documentIds = [];
|
|
313
|
+
const provenance = [];
|
|
314
|
+
const claims = [];
|
|
315
|
+
let confidenceTotal = 0;
|
|
316
|
+
let confidenceCount = 0;
|
|
317
|
+
let totalTokens = 0;
|
|
318
|
+
const levelMaxTokens = this.getMaxTokensForLevel(level);
|
|
319
|
+
const maxTokens = typeof budget?.maxTokens === "number" ? Math.min(levelMaxTokens, budget.maxTokens) : levelMaxTokens;
|
|
320
|
+
const costPer1k = typeof budget?.costPer1kTokensUsd === "number" ? budget.costPer1kTokensUsd : void 0;
|
|
321
|
+
const maxCostUsd = typeof budget?.maxCostUsd === "number" ? budget.maxCostUsd : void 0;
|
|
322
|
+
const order = overlayOrder ?? ["org", "repo", "public"];
|
|
323
|
+
const grouped = /* @__PURE__ */ new Map();
|
|
324
|
+
for (const result of results) {
|
|
325
|
+
const overlay = result.overlay ?? "repo";
|
|
326
|
+
const list = grouped.get(overlay) ?? [];
|
|
327
|
+
list.push(result);
|
|
328
|
+
grouped.set(overlay, list);
|
|
329
|
+
}
|
|
330
|
+
for (const overlay of order) {
|
|
331
|
+
const overlayResults = grouped.get(overlay) ?? [];
|
|
332
|
+
for (const result of overlayResults) {
|
|
333
|
+
const doc = this.documents.get(result.id);
|
|
334
|
+
if (!doc) continue;
|
|
335
|
+
let content;
|
|
336
|
+
switch (level) {
|
|
337
|
+
case "oneliner":
|
|
338
|
+
content = doc.compressions.oneliner;
|
|
339
|
+
break;
|
|
340
|
+
case "summary":
|
|
341
|
+
content = doc.compressions.summary;
|
|
342
|
+
break;
|
|
343
|
+
case "working":
|
|
344
|
+
content = doc.compressions.working;
|
|
345
|
+
break;
|
|
346
|
+
case "full":
|
|
347
|
+
content = doc.content;
|
|
348
|
+
break;
|
|
349
|
+
case "expanded":
|
|
350
|
+
content = doc.content;
|
|
351
|
+
break;
|
|
352
|
+
}
|
|
353
|
+
const tokens = estimateTokens(content);
|
|
354
|
+
if (totalTokens + tokens > maxTokens) {
|
|
355
|
+
break;
|
|
356
|
+
}
|
|
357
|
+
if (maxCostUsd !== void 0 && costPer1k !== void 0 && costPer1k > 0) {
|
|
358
|
+
const nextCost = (totalTokens + tokens) / 1e3 * costPer1k;
|
|
359
|
+
if (nextCost > maxCostUsd) {
|
|
360
|
+
break;
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
contextParts.push(`## ${doc.title}
|
|
364
|
+
|
|
365
|
+
${content}`);
|
|
366
|
+
documentIds.push(doc.id);
|
|
367
|
+
const claimConfidence = Math.max(0.1, Math.min(1, result.score ?? 0));
|
|
368
|
+
claims.push({
|
|
369
|
+
text: doc.compressions.oneliner || doc.title,
|
|
370
|
+
sourceId: doc.id,
|
|
371
|
+
confidence: claimConfidence
|
|
372
|
+
});
|
|
373
|
+
confidenceTotal += claimConfidence;
|
|
374
|
+
confidenceCount += 1;
|
|
375
|
+
provenance.push({
|
|
376
|
+
id: doc.id,
|
|
377
|
+
source: doc.source,
|
|
378
|
+
repo: doc.repo,
|
|
379
|
+
overlay: doc.overlay,
|
|
380
|
+
sensitivity: doc.sensitivity,
|
|
381
|
+
hash: doc.hash
|
|
382
|
+
});
|
|
383
|
+
totalTokens += tokens;
|
|
384
|
+
}
|
|
385
|
+
}
|
|
386
|
+
if (contextParts.length === 0) {
|
|
387
|
+
return null;
|
|
388
|
+
}
|
|
389
|
+
const consumedCostUsd = costPer1k !== void 0 && costPer1k > 0 ? totalTokens / 1e3 * costPer1k : void 0;
|
|
390
|
+
const contract = {
|
|
391
|
+
claims,
|
|
392
|
+
sources: provenance,
|
|
393
|
+
confidence: confidenceCount > 0 ? confidenceTotal / confidenceCount : 0.5
|
|
394
|
+
};
|
|
395
|
+
return {
|
|
396
|
+
content: contextParts.join("\n\n---\n\n"),
|
|
397
|
+
tokenCount: totalTokens,
|
|
398
|
+
source: results[0].source,
|
|
399
|
+
level,
|
|
400
|
+
documents: documentIds,
|
|
401
|
+
budget: budget ? {
|
|
402
|
+
maxTokens: budget.maxTokens ?? levelMaxTokens,
|
|
403
|
+
maxCostUsd,
|
|
404
|
+
costPer1kTokensUsd: costPer1k,
|
|
405
|
+
consumedTokens: totalTokens,
|
|
406
|
+
consumedCostUsd
|
|
407
|
+
} : void 0,
|
|
408
|
+
contract,
|
|
409
|
+
provenance
|
|
410
|
+
};
|
|
411
|
+
}
|
|
412
|
+
getDocument(id) {
|
|
413
|
+
return this.documents.get(id);
|
|
414
|
+
}
|
|
415
|
+
getMaxTokensForLevel(level) {
|
|
416
|
+
switch (level) {
|
|
417
|
+
case "oneliner":
|
|
418
|
+
return 20;
|
|
419
|
+
case "summary":
|
|
420
|
+
return 100;
|
|
421
|
+
case "working":
|
|
422
|
+
return 500;
|
|
423
|
+
case "full":
|
|
424
|
+
return 1e4;
|
|
425
|
+
case "expanded":
|
|
426
|
+
return 5e4;
|
|
427
|
+
}
|
|
428
|
+
}
|
|
429
|
+
async persist(changes) {
|
|
430
|
+
const indexFile = join(this.config.indexPath, "index.json");
|
|
431
|
+
const changesFile = join(this.config.indexPath, "changes.json");
|
|
432
|
+
const data = {
|
|
433
|
+
version: this.indexVersion,
|
|
434
|
+
createdAt: Date.now(),
|
|
435
|
+
documents: Array.from(this.documents.values())
|
|
436
|
+
};
|
|
437
|
+
try {
|
|
438
|
+
const { mkdirSync } = await import('fs');
|
|
439
|
+
mkdirSync(dirname(indexFile), { recursive: true });
|
|
440
|
+
writeFileSync(indexFile, JSON.stringify(data));
|
|
441
|
+
if (changes) {
|
|
442
|
+
writeFileSync(changesFile, JSON.stringify({
|
|
443
|
+
updatedAt: Date.now(),
|
|
444
|
+
changed: changes.changed,
|
|
445
|
+
removed: changes.removed
|
|
446
|
+
}));
|
|
447
|
+
}
|
|
448
|
+
} catch (err) {
|
|
449
|
+
process.stderr.write(`[baseline:indexer] Index persistence failed: ${err.message}
|
|
450
|
+
`);
|
|
451
|
+
}
|
|
452
|
+
}
|
|
453
|
+
getStats() {
|
|
454
|
+
const typeCount = {};
|
|
455
|
+
let totalCompressions = 0;
|
|
456
|
+
for (const doc of this.documents.values()) {
|
|
457
|
+
typeCount[doc.type] = (typeCount[doc.type] || 0) + 1;
|
|
458
|
+
if (doc.compressions) totalCompressions += 3;
|
|
459
|
+
}
|
|
460
|
+
return {
|
|
461
|
+
documents: this.documents.size,
|
|
462
|
+
types: Object.keys(typeCount).length,
|
|
463
|
+
compressions: totalCompressions,
|
|
464
|
+
byType: typeCount
|
|
465
|
+
};
|
|
466
|
+
}
|
|
467
|
+
};
|
|
468
|
+
/**
|
|
469
|
+
* BaselineOS Knowledge Indexer
|
|
470
|
+
*
|
|
471
|
+
* Indexes project knowledge for RAG retrieval with hierarchical compression.
|
|
472
|
+
*
|
|
473
|
+
* Key Innovation: Pre-computed compressions at 5 levels allow token-optimized
|
|
474
|
+
* context loading. Request `summary` (100 tokens) instead of `full` (10K).
|
|
475
|
+
*
|
|
476
|
+
* @license Apache-2.0
|
|
477
|
+
*/
|
|
478
|
+
|
|
479
|
+
export { KnowledgeIndexer };
|
|
480
|
+
//# sourceMappingURL=indexer.js.map
|
|
481
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/core/indexer.ts"],"names":[],"mappings":";;;;;;;AAmHA,SAAS,eAAe,IAAA,EAAsB;AAC5C,EAAA,OAAO,IAAA,CAAK,IAAA,CAAK,IAAA,CAAK,MAAA,GAAS,CAAC,CAAA;AAClC;AAGA,SAAS,YAAY,OAAA,EAAyB;AAC5C,EAAA,OAAO,UAAA,CAAW,QAAQ,CAAA,CAAE,MAAA,CAAO,OAAO,CAAA,CAAE,MAAA,CAAO,KAAK,CAAA,CAAE,KAAA,CAAM,CAAA,EAAG,EAAE,CAAA;AACvE;AAGA,SAAS,cAAc,QAAA,EAAgC;AACrD,EAAA,MAAM,GAAA,GAAM,OAAA,CAAQ,QAAQ,CAAA,CAAE,WAAA,EAAY;AAC1C,EAAA,MAAM,IAAA,GAAO,QAAA,CAAS,QAAQ,CAAA,CAAE,WAAA,EAAY;AAE5C,EAAA,IAAI,IAAA,CAAK,SAAS,MAAM,CAAA,IAAK,KAAK,QAAA,CAAS,aAAa,GAAG,OAAO,MAAA;AAClE,EAAA,IAAI,IAAA,CAAK,QAAA,CAAS,KAAK,CAAA,IAAK,IAAA,CAAK,QAAA,CAAS,SAAS,CAAA,IAAK,IAAA,CAAK,QAAA,CAAS,SAAS,CAAA,EAAG,OAAO,KAAA;AACzF,EAAA,IAAI,CAAC,SAAS,OAAA,EAAS,MAAA,EAAQ,OAAO,CAAA,CAAE,QAAA,CAAS,GAAG,CAAA,EAAG,OAAO,QAAA;AAC9D,EAAA,IAAI,CAAC,OAAO,MAAA,EAAQ,MAAA,EAAQ,MAAM,CAAA,CAAE,QAAA,CAAS,GAAG,CAAA,EAAG,OAAO,KAAA;AAC1D,EAAA,OAAO,MAAA;AACT;AAEA,SAAS,gBAAA,CAAiB,OAAgB,QAAA,EAA4B;AACpE,EAAA,IAAI,UAAU,KAAA,IAAS,KAAA,KAAU,MAAA,IAAU,KAAA,KAAU,UAAU,OAAO,KAAA;AACtE,EAAA,OAAO,QAAA;AACT;AAEA,SAAS,oBAAA,CAAqB,OAAgB,QAAA,EAAoC;AAChF,EAAA,IAAI,UAAU,QAAA,IAAY,KAAA,KAAU,UAAA,IAAc,KAAA,KAAU,cAAc,OAAO,KAAA;AACjF,EAAA,OAAO,QAAA;AACT;AAEA,SAAS,cAAc,KAAA,EAAsC;AAC3D,EAAA,IAAI,KAAA,CAAM,OAAA,CAAQ,KAAK,CAAA,EAAG;AACxB,IAAA,OAAO,KAAA,CAAM,IAAI,CAAA,CAAA,KAAK,MAAA,CAAO,CAAC,CAAC,CAAA,CAAE,OAAO,OAAO,CAAA;AAAA,EACjD;AACA,EAAA,IAAI,OAAO,UAAU,QAAA,EAAU;AAC7B,IAAA,OAAO,CAAC,KAAK,CAAA;AAAA,EACf;AACA,EAAA,OAAO,MAAA;AACT;AAUA,SAAS,gBAAgB,KAAA,EAA6B;AACpD,EAAA,QAAQ,KAAA;AAAO,IACb,KAAK,QAAA;AAAU,MAAA,OAAO,CAAA;AAAA,IACtB,KAAK,UAAA;AAAY,MAAA,OAAO,CAAA;AAAA,IACxB,KAAK,YAAA;AAAc,MAAA,OAAO,CAAA;AAAA,IAC1B;AAAS,MAAA,OAAO,CAAA;AAAA;AAEpB;AAEA,SAAS,YAAA,CAAa,KAAsB,MAAA,EAAgC;AAC1E,EAAA,IAAI,CAAC,QAAQ,OAAO,IAAA;AAEpB,EAAA,IAAI,MAAA,CAAO,IAAA,IAAQ,GAAA,CAAI,IAAA,IAAQ,CAAC,MAAA,CAAO,IAAA,CAAK,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA,EAAG,OAAO,KAAA;AACvE,EAAA,IAAI,MAAA,CAAO,QAAA,IAAY,GAAA,CAAI,OAAA,IAAW,CAAC,MAAA,CAAO,QAAA,CAAS,QAAA,CAAS,GAAA,CAAI,OAAO,CAAA,EAAG,OAAO,KAAA;AACrF,EAAA,IAAI,MAAA,CAAO,WAAA,IAAe,GAAA,CAAI,WAAA,EAAa;AACzC,IAAA,IAAI,eAAA,CAAgB,IAAI,WAAW,CAAA,GAAI,gBAAgB,MAAA,CAAO,WAAW,GAAG,OAAO,KAAA;AAAA,EACrF;AACA,EAAA,IAAI,OAAO,MAAA,IAAU,GAAA,CAAI,UAAU,GAAA,CAAI,MAAA,CAAO,SAAS,CAAA,EAAG;AACxD,IAAA,MAAM,EAAA,GAAK,OAAO,MAAA,CAAO,IAAA,CAAK,OAAK,GAAA,CAAI,MAAA,CAAQ,QAAA,CAAS,CAAC,CAAC,CAAA;AAC1D,IAAA,IAAI,CAAC,IAAI,OAAO,KAAA;AAAA,EAClB;AACA,EAAA,IAAI,OAAO,KAAA,IAAS,GAAA,CAAI,SAAS,GAAA,CAAI,KAAA,CAAM,SAAS,CAAA,EAAG;AACrD,IAAA,MAAM,EAAA,GAAK,OAAO,KAAA,CAAM,IAAA,CAAK,OAAK,GAAA,CAAI,KAAA,CAAO,QAAA,CAAS,CAAC,CAAC,CAAA;AACxD,IAAA,IAAI,CAAC,IAAI,OAAO,KAAA;AAAA,EAClB;AACA,EAAA,OAAO,IAAA;AACT;AAGA,SAAS,oBAAA,CAAqB,SAAiB,KAAA,EAI7C;AACA,EAAA,MAAM,KAAA,GAAQ,QAAQ,KAAA,CAAM,IAAI,EAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,IAAA,EAAM,CAAA;AAGtD,EAAA,MAAM,SAAA,GAAY,KAAA,CAAM,CAAC,CAAA,IAAK,KAAA;AAC9B,EAAA,MAAM,QAAA,GAAW,SAAA,CAAU,KAAA,CAAM,CAAA,EAAG,EAAE,CAAA;AAGtC,EAAA,MAAM,iBAAiB,OAAA,CAAQ,KAAA,CAAM,MAAM,CAAA,CAAE,CAAC,CAAA,IAAK,EAAA;AACnD,EAAA,MAAM,OAAA,GAAU,cAAA,CAAe,KAAA,CAAM,CAAA,EAAG,GAAG,CAAA;AAG3C,EAAA,MAAM,QAAA,GAAW,OAAA,CAAQ,KAAA,CAAM,UAAU,CAAA;AACzC,EAAA,MAAM,YAAA,GAAe,SAAS,MAAA,GAAS,CAAA,GAAI,SAAS,CAAC,CAAA,GAAI,QAAA,CAAS,CAAC,CAAA,GAAI,OAAA;AACvE,EAAA,MAAM,OAAA,GAAU,YAAA,CAAa,KAAA,CAAM,CAAA,EAAG,GAAI,CAAA;AAE1C,EAAA,OAAO,EAAE,QAAA,EAAU,OAAA,EAAS,OAAA,EAAQ;AACtC;AAEO,IAAM,mBAAN,MAAuB;AAAA,EACpB,MAAA;AAAA,EACA,SAAA;AAAA,EACA,WAAA,GAAuB,KAAA;AAAA,EACvB,YAAA,GAAuB,CAAA;AAAA,EAE/B,WAAA,CAAY,MAAA,GAAiC,EAAC,EAAG;AAC/C,IAAA,IAAA,CAAK,MAAA,GAAS;AAAA,MACZ,WAAA,EAAa,MAAA,CAAO,WAAA,IAAe,OAAA,CAAQ,GAAA,EAAI;AAAA,MAC/C,cAAA,EAAgB,MAAA,CAAO,cAAA,IAAkB,CAAC,UAAU,aAAa,CAAA;AAAA,MACjE,SAAA,EAAW,OAAO,SAAA,IAAa,iBAAA;AAAA,MAC/B,eAAA,EAAiB,OAAO,eAAA,IAAmB;AAAA,QACzC,SAAA;AAAA,QACA,UAAA;AAAA,QACA,SAAA;AAAA,QACA,UAAA;AAAA,QACA,SAAA;AAAA,QACA,SAAA;AAAA,QACA,WAAA;AAAA,QACA,WAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,eAAA,EAAiB,OAAO,eAAA,IAAmB;AAAA,QACzC,oBAAA;AAAA,QACA,YAAA;AAAA,QACA,YAAA;AAAA,QACA,gBAAA;AAAA,QACA,aAAA;AAAA,QACA;AAAA,OACF;AAAA,MACA,MAAM,MAAA,CAAO,IAAA;AAAA,MACb,cAAA,EAAgB,OAAO,cAAA,IAAkB,MAAA;AAAA,MACzC,kBAAA,EAAoB,OAAO,kBAAA,IAAsB;AAAA,KACnD;AACA,IAAA,IAAA,CAAK,SAAA,uBAAgB,GAAA,EAAI;AAAA,EAC3B;AAAA,EAEA,MAAM,UAAA,GAA4B;AAChC,IAAA,IAAI,KAAK,WAAA,EAAa;AAGtB,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,WAAW,YAAY,CAAA;AAC1D,IAAA,IAAI,UAAA,CAAW,SAAS,CAAA,EAAG;AACzB,MAAA,IAAI;AACF,QAAA,MAAM,OAAO,IAAA,CAAK,KAAA,CAAM,YAAA,CAAa,SAAA,EAAW,OAAO,CAAC,CAAA;AACxD,QAAA,IAAI,IAAA,CAAK,OAAA,KAAY,IAAA,CAAK,YAAA,EAAc;AACtC,UAAA,KAAA,MAAW,GAAA,IAAO,KAAK,SAAA,EAAW;AAChC,YAAA,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,GAAA,CAAI,EAAA,EAAI,GAAG,CAAA;AAAA,UAChC;AAAA,QACF;AAAA,MACF,SAAS,GAAA,EAAK;AACZ,QAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,gDAAA,EAAoD,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAAA,MACpG;AAAA,IACF;AAEA,IAAA,IAAA,CAAK,WAAA,GAAc,IAAA;AAAA,EACrB;AAAA,EAEA,MAAM,QAAA,GAAgH;AACpH,IAAA,IAAI,OAAA,GAAU,CAAA;AACd,IAAA,IAAI,MAAA,GAAS,CAAA;AACb,IAAA,IAAI,OAAA,GAAU,CAAA;AACd,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,MAAM,UAAoB,EAAC;AAC3B,IAAA,MAAM,IAAA,uBAAW,GAAA,EAAY;AAE7B,IAAA,KAAA,MAAW,aAAA,IAAiB,IAAA,CAAK,MAAA,CAAO,cAAA,EAAgB;AACtD,MAAA,MAAM,QAAA,GAAW,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,aAAa,aAAa,CAAA;AAE5D,MAAA,IAAI,CAAC,UAAA,CAAW,QAAQ,CAAA,EAAG;AACzB,QAAA;AAAA,MACF;AAGA,MAAA,MAAM,IAAA,GAAO,SAAS,QAAQ,CAAA;AAC9B,MAAA,IAAI,IAAA,CAAK,QAAO,EAAG;AACjB,QAAA,IAAI;AACF,UAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,SAAA,CAAU,UAAU,aAAa,CAAA;AAC3D,UAAA,IAAA,CAAK,IAAI,aAAa,CAAA;AACtB,UAAA,IAAI,WAAW,SAAA,EAAW,OAAA,EAAA;AAAA,eAAA,IACjB,WAAW,SAAA,EAAW,OAAA,EAAA;AAC/B,UAAA,IAAI,MAAA,KAAW,SAAA,EAAW,OAAA,CAAQ,IAAA,CAAK,aAAa,CAAA;AAAA,QACtD,SAAS,GAAA,EAAK;AACZ,UAAA,OAAA,CAAQ,OAAO,KAAA,CAAM,CAAA,mCAAA,EAAsC,aAAa,CAAA,EAAA,EAAM,IAAc,OAAO;AAAA,CAAI,CAAA;AACvG,UAAA,MAAA,EAAA;AAAA,QACF;AACA,QAAA;AAAA,MACF;AAGA,MAAA,MAAM,KAAA,GAAQ,MAAM,IAAA,CAAK,IAAA,CAAK,OAAO,eAAA,EAAkB;AAAA,QACrD,GAAA,EAAK,QAAA;AAAA,QACL,MAAA,EAAQ,KAAK,MAAA,CAAO,eAAA;AAAA,QACpB,KAAA,EAAO;AAAA,OACR,CAAA;AAED,MAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AACxB,QAAA,IAAI;AACF,UAAA,MAAM,QAAA,GAAW,IAAA,CAAK,QAAA,EAAU,IAAI,CAAA;AACpC,UAAA,MAAM,MAAA,GAAS,MAAM,IAAA,CAAK,SAAA,CAAU,UAAU,CAAA,EAAG,aAAa,CAAA,CAAA,EAAI,IAAI,CAAA,CAAE,CAAA;AACxE,UAAA,IAAA,CAAK,GAAA,CAAI,CAAA,EAAG,aAAa,CAAA,CAAA,EAAI,IAAI,CAAA,CAAE,CAAA;AACnC,UAAA,IAAI,WAAW,SAAA,EAAW,OAAA,EAAA;AAAA,eAAA,IACjB,WAAW,SAAA,EAAW,OAAA,EAAA;AAC/B,UAAA,IAAI,MAAA,KAAW,WAAW,OAAA,CAAQ,IAAA,CAAK,GAAG,aAAa,CAAA,CAAA,EAAI,IAAI,CAAA,CAAE,CAAA;AAAA,QACnE,SAAS,GAAA,EAAK;AACZ,UAAA,OAAA,CAAQ,MAAA,CAAO,MAAM,CAAA,mCAAA,EAAsC,aAAa,IAAI,IAAI,CAAA,EAAA,EAAM,IAAc,OAAO;AAAA,CAAI,CAAA;AAC/G,UAAA,MAAA,EAAA;AAAA,QACF;AAAA,MACF;AAAA,IACF;AAGA,IAAA,KAAA,MAAW,EAAA,IAAM,IAAA,CAAK,SAAA,CAAU,IAAA,EAAK,EAAG;AACtC,MAAA,IAAI,CAAC,IAAA,CAAK,GAAA,CAAI,EAAE,CAAA,EAAG;AACjB,QAAA,IAAA,CAAK,SAAA,CAAU,OAAO,EAAE,CAAA;AACxB,QAAA,OAAA,CAAQ,KAAK,EAAE,CAAA;AAAA,MACjB;AAAA,IACF;AAGA,IAAA,MAAM,IAAA,CAAK,OAAA,CAAQ,EAAE,OAAA,EAAS,SAAS,CAAA;AAEvC,IAAA,OAAO,EAAE,OAAA,EAAS,MAAA,EAAQ,OAAA,EAAS,SAAS,OAAA,EAAQ;AAAA,EACtD;AAAA,EAEA,MAAc,SAAA,CAAU,QAAA,EAAkB,EAAA,EAAsD;AAC9F,IAAA,MAAM,OAAA,GAAU,YAAA,CAAa,QAAA,EAAU,OAAO,CAAA;AAC9C,IAAA,MAAM,IAAA,GAAO,YAAY,OAAO,CAAA;AAGhC,IAAA,MAAM,QAAA,GAAW,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,EAAE,CAAA;AACtC,IAAA,IAAI,QAAA,IAAY,QAAA,CAAS,IAAA,KAAS,IAAA,EAAM;AACtC,MAAA,OAAO,SAAA;AAAA,IACT;AAEA,IAAA,MAAM,GAAA,GAAM,QAAQ,QAAQ,CAAA;AAC5B,IAAA,IAAI,KAAA,GAAQ,QAAA,CAAS,QAAA,EAAU,GAAG,CAAA;AAClC,IAAA,IAAI,gBAAA,GAAmB,OAAA;AACvB,IAAA,IAAI,WAAoC,EAAC;AAGzC,IAAA,IAAI,CAAC,KAAA,EAAO,MAAM,CAAA,CAAE,QAAA,CAAS,GAAG,CAAA,EAAG;AACjC,MAAA,IAAI;AACF,QAAA,MAAM,MAAA,GAAS,OAAO,OAAO,CAAA;AAC7B,QAAA,KAAA,GAAS,MAAA,CAAO,KAAK,KAAA,IAAoB,KAAA;AACzC,QAAA,gBAAA,GAAmB,MAAA,CAAO,OAAA;AAC1B,QAAA,QAAA,GAAW,MAAA,CAAO,IAAA;AAAA,MACpB,CAAA,CAAA,MAAQ;AAAA,MAER;AAAA,IACF;AAEA,IAAA,MAAM,YAAA,GAAe,oBAAA,CAAqB,gBAAA,EAAkB,KAAK,CAAA;AACjE,IAAA,MAAM,IAAA,GAAQ,SAAS,IAAA,IAAmB,IAAA,CAAK,OAAO,IAAA,IAAQ,QAAA,CAAS,IAAA,CAAK,MAAA,CAAO,WAAW,CAAA;AAC9F,IAAA,MAAM,UAAU,gBAAA,CAAiB,QAAA,CAAS,SAAS,IAAA,CAAK,MAAA,CAAO,kBAAkB,MAAM,CAAA;AACvF,IAAA,MAAM,WAAA,GAAc,oBAAA;AAAA,MAClB,QAAA,CAAS,WAAA;AAAA,MACT,IAAA,CAAK,OAAO,kBAAA,IAAsB;AAAA,KACpC;AACA,IAAA,MAAM,MAAA,GAAS,aAAA,CAAc,QAAA,CAAS,MAAA,IAAU,SAAS,KAAK,CAAA;AAC9D,IAAA,MAAM,KAAA,GAAQ,aAAA,CAAc,QAAA,CAAS,KAAA,IAAS,SAAS,IAAI,CAAA;AAE3D,IAAA,MAAM,GAAA,GAAuB;AAAA,MAC3B,EAAA;AAAA,MACA,KAAA;AAAA,MACA,OAAA,EAAS,gBAAA;AAAA,MACT,UAAA,EAAY,eAAe,gBAAgB,CAAA;AAAA,MAC3C,MAAA,EAAQ,QAAA;AAAA,MACR,IAAA,EAAM,cAAc,QAAQ,CAAA;AAAA,MAC5B,IAAA;AAAA,MACA,SAAA,EAAW,KAAK,GAAA,EAAI;AAAA,MACpB,QAAA;AAAA,MACA,IAAA;AAAA,MACA,OAAA;AAAA,MACA,WAAA;AAAA,MACA,MAAA;AAAA,MACA,KAAA;AAAA,MACA;AAAA,KACF;AAEA,IAAA,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,EAAA,EAAI,GAAG,CAAA;AAC1B,IAAA,OAAO,SAAA;AAAA,EACT;AAAA,EAEA,MAAM,KAAA,GAAuB;AAC3B,IAAA,IAAA,CAAK,UAAU,KAAA,EAAM;AAAA,EACvB;AAAA,EAEA,MAAM,MAAA,CAAO,KAAA,EAAe,OAAA,EAIA;AAC1B,IAAA,MAAM,KAAA,GAAQ,SAAS,KAAA,IAAS,CAAA;AAChC,IAAA,MAAM,QAAQ,OAAA,EAAS,KAAA;AACvB,IAAA,MAAM,SAAS,OAAA,EAAS,MAAA;AAExB,IAAA,MAAM,KAAA,GAAQ,KAAA,CAAM,WAAA,EAAY,CAAE,KAAA,CAAM,KAAK,CAAA,CAAE,MAAA,CAAO,CAAA,CAAA,KAAK,CAAA,CAAE,MAAA,GAAS,CAAC,CAAA;AAEvE,IAAA,IAAI,KAAA,CAAM,WAAW,CAAA,EAAG;AACtB,MAAA,OAAO,EAAC;AAAA,IACV;AAEA,IAAA,MAAM,UAA0B,EAAC;AAEjC,IAAA,KAAA,MAAW,GAAA,IAAO,IAAA,CAAK,SAAA,CAAU,MAAA,EAAO,EAAG;AAEzC,MAAA,IAAI,SAAS,CAAC,KAAA,CAAM,QAAA,CAAS,GAAA,CAAI,IAAI,CAAA,EAAG;AACtC,QAAA;AAAA,MACF;AACA,MAAA,IAAI,CAAC,YAAA,CAAa,GAAA,EAAK,MAAM,CAAA,EAAG;AAC9B,QAAA;AAAA,MACF;AAEA,MAAA,MAAM,YAAA,GAAe,GAAA,CAAI,OAAA,CAAQ,WAAA,EAAY;AAC7C,MAAA,MAAM,UAAA,GAAa,GAAA,CAAI,KAAA,CAAM,WAAA,EAAY;AAEzC,MAAA,IAAI,KAAA,GAAQ,CAAA;AACZ,MAAA,KAAA,MAAW,QAAQ,KAAA,EAAO;AAExB,QAAA,IAAI,UAAA,CAAW,QAAA,CAAS,IAAI,CAAA,EAAG,KAAA,IAAS,CAAA;AAGxC,QAAA,MAAM,KAAA,GAAQ,IAAI,MAAA,CAAO,IAAA,EAAM,IAAI,CAAA;AACnC,QAAA,MAAM,OAAA,GAAU,YAAA,CAAa,KAAA,CAAM,KAAK,CAAA;AACxC,QAAA,IAAI,SAAS,KAAA,IAAS,IAAA,CAAK,GAAA,CAAI,OAAA,CAAQ,QAAQ,CAAC,CAAA;AAAA,MAClD;AAEA,MAAA,IAAI,QAAQ,CAAA,EAAG;AACb,QAAA,OAAA,CAAQ,IAAA,CAAK;AAAA,UACX,IAAI,GAAA,CAAI,EAAA;AAAA,UACR,OAAO,GAAA,CAAI,KAAA;AAAA,UACX,OAAA,EAAS,IAAI,YAAA,CAAa,OAAA;AAAA,UAC1B,KAAA,EAAO,QAAQ,KAAA,CAAM,MAAA;AAAA,UACrB,QAAQ,GAAA,CAAI,MAAA;AAAA,UACZ,YAAY,GAAA,CAAI,UAAA;AAAA,UAChB,MAAM,GAAA,CAAI,IAAA;AAAA,UACV,MAAM,GAAA,CAAI,IAAA;AAAA,UACV,SAAS,GAAA,CAAI,OAAA;AAAA,UACb,aAAa,GAAA,CAAI;AAAA,SAClB,CAAA;AAAA,MACH;AAAA,IACF;AAEA,IAAA,OAAO,OAAA,CACJ,IAAA,CAAK,CAAC,CAAA,EAAG,CAAA,KAAM,CAAA,CAAE,KAAA,GAAQ,CAAA,CAAE,KAAK,CAAA,CAChC,KAAA,CAAM,CAAA,EAAG,KAAK,CAAA;AAAA,EACnB;AAAA,EAEA,MAAM,UAAA,CACJ,OAAA,EACA,KAAA,GAAsB,WACtB,OAAA,EAKyB;AAEzB,IAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,MAAA,CAAO,OAAA,EAAS,EAAE,KAAA,EAAO,EAAA,EAAI,MAAA,EAAQ,OAAA,EAAS,MAAA,EAAQ,CAAA;AAEjF,IAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxB,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAK,uBAAA,CAAwB,OAAA,EAAS,OAAO,OAAA,EAAS,YAAA,EAAc,SAAS,MAAM,CAAA;AAAA,EAC5F;AAAA,EAEA,MAAM,eAAA,CACJ,OAAA,EACA,KAAA,GAAsB,WACtB,OAAA,EAMyB;AACzB,IAAA,MAAM,OAAA,GAAU,MAAM,IAAA,CAAK,MAAA,CAAO,OAAA,EAAS,EAAE,KAAA,EAAO,EAAA,EAAI,MAAA,EAAQ,OAAA,EAAS,MAAA,EAAQ,CAAA;AACjF,IAAA,IAAI,OAAA,CAAQ,WAAW,CAAA,EAAG;AACxB,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,MAAM,KAAA,uBAAY,GAAA,EAAgC;AAClD,IAAA,KAAA,MAAW,IAAA,IAAQ,OAAA,EAAS,KAAA,IAAS,EAAC,EAAG;AACvC,MAAA,IAAI,MAAM,EAAA,EAAI,KAAA,CAAM,IAAI,IAAA,CAAK,EAAA,EAAI,KAAK,IAAI,CAAA;AAAA,IAC5C;AAEA,IAAA,MAAM,QAAA,GAAW,OAAA,CAAQ,MAAA,CAAO,CAAA,MAAA,KAAU;AACxC,MAAA,MAAM,GAAA,GAAM,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,OAAO,EAAE,CAAA;AACxC,MAAA,IAAI,CAAC,KAAK,OAAO,KAAA;AACjB,MAAA,MAAM,SAAA,GAAY,KAAA,CAAM,GAAA,CAAI,GAAA,CAAI,EAAE,CAAA;AAClC,MAAA,OAAO,CAAC,SAAA,IAAa,SAAA,KAAc,GAAA,CAAI,IAAA;AAAA,IACzC,CAAC,CAAA;AAED,IAAA,IAAI,QAAA,CAAS,WAAW,CAAA,EAAG;AACzB,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,OAAO,KAAK,uBAAA,CAAwB,QAAA,EAAU,OAAO,OAAA,EAAS,YAAA,EAAc,SAAS,MAAM,CAAA;AAAA,EAC7F;AAAA,EAEQ,uBAAA,CACN,OAAA,EACA,KAAA,EACA,YAAA,EACA,MAAA,EACgB;AAEhB,IAAA,MAAM,eAAyB,EAAC;AAChC,IAAA,MAAM,cAAwB,EAAC;AAC/B,IAAA,MAAM,aAAoC,EAAC;AAC3C,IAAA,MAAM,SAAsC,EAAC;AAC7C,IAAA,IAAI,eAAA,GAAkB,CAAA;AACtB,IAAA,IAAI,eAAA,GAAkB,CAAA;AACtB,IAAA,IAAI,WAAA,GAAc,CAAA;AAClB,IAAA,MAAM,cAAA,GAAiB,IAAA,CAAK,oBAAA,CAAqB,KAAK,CAAA;AACtD,IAAA,MAAM,SAAA,GAAY,OAAO,MAAA,EAAQ,SAAA,KAAc,QAAA,GAC3C,KAAK,GAAA,CAAI,cAAA,EAAgB,MAAA,CAAO,SAAS,CAAA,GACzC,cAAA;AACJ,IAAA,MAAM,YAAY,OAAO,MAAA,EAAQ,kBAAA,KAAuB,QAAA,GAAW,OAAO,kBAAA,GAAqB,MAAA;AAC/F,IAAA,MAAM,aAAa,OAAO,MAAA,EAAQ,UAAA,KAAe,QAAA,GAAW,OAAO,UAAA,GAAa,MAAA;AAChF,IAAA,MAAM,KAAA,GAAQ,YAAA,IAAgB,CAAC,KAAA,EAAO,QAAQ,QAAQ,CAAA;AAEtD,IAAA,MAAM,OAAA,uBAAc,GAAA,EAA6B;AACjD,IAAA,KAAA,MAAW,UAAU,OAAA,EAAS;AAC5B,MAAA,MAAM,OAAA,GAAU,OAAO,OAAA,IAAW,MAAA;AAClC,MAAA,MAAM,IAAA,GAAO,OAAA,CAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AACtC,MAAA,IAAA,CAAK,KAAK,MAAM,CAAA;AAChB,MAAA,OAAA,CAAQ,GAAA,CAAI,SAAS,IAAI,CAAA;AAAA,IAC3B;AAEA,IAAA,KAAA,MAAW,WAAW,KAAA,EAAO;AAC3B,MAAA,MAAM,cAAA,GAAiB,OAAA,CAAQ,GAAA,CAAI,OAAO,KAAK,EAAC;AAChD,MAAA,KAAA,MAAW,UAAU,cAAA,EAAgB;AACnC,QAAA,MAAM,GAAA,GAAM,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,OAAO,EAAE,CAAA;AACxC,QAAA,IAAI,CAAC,GAAA,EAAK;AAEV,QAAA,IAAI,OAAA;AAEJ,QAAA,QAAQ,KAAA;AAAO,UACb,KAAK,UAAA;AACH,YAAA,OAAA,GAAU,IAAI,YAAA,CAAa,QAAA;AAC3B,YAAA;AAAA,UACF,KAAK,SAAA;AACH,YAAA,OAAA,GAAU,IAAI,YAAA,CAAa,OAAA;AAC3B,YAAA;AAAA,UACF,KAAK,SAAA;AACH,YAAA,OAAA,GAAU,IAAI,YAAA,CAAa,OAAA;AAC3B,YAAA;AAAA,UACF,KAAK,MAAA;AACH,YAAA,OAAA,GAAU,GAAA,CAAI,OAAA;AACd,YAAA;AAAA,UACF,KAAK,UAAA;AAEH,YAAA,OAAA,GAAU,GAAA,CAAI,OAAA;AAEd,YAAA;AAAA;AAGJ,QAAA,MAAM,MAAA,GAAS,eAAe,OAAO,CAAA;AACrC,QAAA,IAAI,WAAA,GAAc,SAAS,SAAA,EAAW;AACpC,UAAA;AAAA,QACF;AACA,QAAA,IAAI,UAAA,KAAe,MAAA,IAAa,SAAA,KAAc,MAAA,IAAa,YAAY,CAAA,EAAG;AACxE,UAAA,MAAM,QAAA,GAAA,CAAa,WAAA,GAAc,MAAA,IAAU,GAAA,GAAQ,SAAA;AACnD,UAAA,IAAI,WAAW,UAAA,EAAY;AACzB,YAAA;AAAA,UACF;AAAA,QACF;AAGA,QAAA,YAAA,CAAa,IAAA,CAAK,CAAA,GAAA,EAAM,GAAA,CAAI,KAAK;;AAAA,EAAO,OAAO,CAAA,CAAE,CAAA;AACjD,QAAA,WAAA,CAAY,IAAA,CAAK,IAAI,EAAE,CAAA;AACvB,QAAA,MAAM,eAAA,GAAkB,IAAA,CAAK,GAAA,CAAI,GAAA,EAAK,IAAA,CAAK,IAAI,CAAA,EAAG,MAAA,CAAO,KAAA,IAAS,CAAC,CAAC,CAAA;AACpE,QAAA,MAAA,CAAO,IAAA,CAAK;AAAA,UACV,IAAA,EAAM,GAAA,CAAI,YAAA,CAAa,QAAA,IAAY,GAAA,CAAI,KAAA;AAAA,UACvC,UAAU,GAAA,CAAI,EAAA;AAAA,UACd,UAAA,EAAY;AAAA,SACb,CAAA;AACD,QAAA,eAAA,IAAmB,eAAA;AACnB,QAAA,eAAA,IAAmB,CAAA;AAEnB,QAAA,UAAA,CAAW,IAAA,CAAK;AAAA,UACd,IAAI,GAAA,CAAI,EAAA;AAAA,UACR,QAAQ,GAAA,CAAI,MAAA;AAAA,UACZ,MAAM,GAAA,CAAI,IAAA;AAAA,UACV,SAAS,GAAA,CAAI,OAAA;AAAA,UACb,aAAa,GAAA,CAAI,WAAA;AAAA,UACjB,MAAM,GAAA,CAAI;AAAA,SACX,CAAA;AACD,QAAA,WAAA,IAAe,MAAA;AAAA,MACjB;AAAA,IACF;AAEA,IAAA,IAAI,YAAA,CAAa,WAAW,CAAA,EAAG;AAC7B,MAAA,OAAO,IAAA;AAAA,IACT;AAEA,IAAA,MAAM,kBACJ,SAAA,KAAc,MAAA,IAAa,YAAY,CAAA,GAClC,WAAA,GAAc,MAAQ,SAAA,GACvB,MAAA;AAEN,IAAA,MAAM,QAAA,GAA8B;AAAA,MAClC,MAAA;AAAA,MACA,OAAA,EAAS,UAAA;AAAA,MACT,UAAA,EAAY,eAAA,GAAkB,CAAA,GAAI,eAAA,GAAkB,eAAA,GAAkB;AAAA,KACxE;AAEA,IAAA,OAAO;AAAA,MACL,OAAA,EAAS,YAAA,CAAa,IAAA,CAAK,aAAa,CAAA;AAAA,MACxC,UAAA,EAAY,WAAA;AAAA,MACZ,MAAA,EAAQ,OAAA,CAAQ,CAAC,CAAA,CAAE,MAAA;AAAA,MACnB,KAAA;AAAA,MACA,SAAA,EAAW,WAAA;AAAA,MACX,QAAQ,MAAA,GACJ;AAAA,QACE,SAAA,EAAW,OAAO,SAAA,IAAa,cAAA;AAAA,QAC/B,UAAA;AAAA,QACA,kBAAA,EAAoB,SAAA;AAAA,QACpB,cAAA,EAAgB,WAAA;AAAA,QAChB;AAAA,OACF,GACA,MAAA;AAAA,MACJ,QAAA;AAAA,MACA;AAAA,KACF;AAAA,EACF;AAAA,EAEA,YAAY,EAAA,EAAyC;AACnD,IAAA,OAAO,IAAA,CAAK,SAAA,CAAU,GAAA,CAAI,EAAE,CAAA;AAAA,EAC9B;AAAA,EAEQ,qBAAqB,KAAA,EAA6B;AACxD,IAAA,QAAQ,KAAA;AAAO,MACb,KAAK,UAAA;AAAY,QAAA,OAAO,EAAA;AAAA,MACxB,KAAK,SAAA;AAAW,QAAA,OAAO,GAAA;AAAA,MACvB,KAAK,SAAA;AAAW,QAAA,OAAO,GAAA;AAAA,MACvB,KAAK,MAAA;AAAQ,QAAA,OAAO,GAAA;AAAA,MACpB,KAAK,UAAA;AAAY,QAAA,OAAO,GAAA;AAAA;AAC1B,EACF;AAAA,EAEA,MAAc,QAAQ,OAAA,EAAmE;AACvF,IAAA,MAAM,SAAA,GAAY,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,WAAW,YAAY,CAAA;AAC1D,IAAA,MAAM,WAAA,GAAc,IAAA,CAAK,IAAA,CAAK,MAAA,CAAO,WAAW,cAAc,CAAA;AAC9D,IAAA,MAAM,IAAA,GAAO;AAAA,MACX,SAAS,IAAA,CAAK,YAAA;AAAA,MACd,SAAA,EAAW,KAAK,GAAA,EAAI;AAAA,MACpB,WAAW,KAAA,CAAM,IAAA,CAAK,IAAA,CAAK,SAAA,CAAU,QAAQ;AAAA,KAC/C;AAEA,IAAA,IAAI;AACF,MAAA,MAAM,EAAE,SAAA,EAAU,GAAI,MAAM,OAAO,IAAI,CAAA;AACvC,MAAA,SAAA,CAAU,QAAQ,SAAS,CAAA,EAAG,EAAE,SAAA,EAAW,MAAM,CAAA;AACjD,MAAA,aAAA,CAAc,SAAA,EAAW,IAAA,CAAK,SAAA,CAAU,IAAI,CAAC,CAAA;AAC7C,MAAA,IAAI,OAAA,EAAS;AACX,QAAA,aAAA,CAAc,WAAA,EAAa,KAAK,SAAA,CAAU;AAAA,UACxC,SAAA,EAAW,KAAK,GAAA,EAAI;AAAA,UACpB,SAAS,OAAA,CAAQ,OAAA;AAAA,UACjB,SAAS,OAAA,CAAQ;AAAA,SAClB,CAAC,CAAA;AAAA,MACJ;AAAA,IACF,SAAS,GAAA,EAAK;AACZ,MAAA,OAAA,CAAQ,MAAA,CAAO,KAAA,CAAM,CAAA,6CAAA,EAAiD,GAAA,CAAc,OAAO;AAAA,CAAI,CAAA;AAAA,IACjG;AAAA,EACF;AAAA,EAEA,QAAA,GAAW;AACT,IAAA,MAAM,YAAoC,EAAC;AAC3C,IAAA,IAAI,iBAAA,GAAoB,CAAA;AAExB,IAAA,KAAA,MAAW,GAAA,IAAO,IAAA,CAAK,SAAA,CAAU,MAAA,EAAO,EAAG;AACzC,MAAA,SAAA,CAAU,IAAI,IAAI,CAAA,GAAA,CAAK,UAAU,GAAA,CAAI,IAAI,KAAK,CAAA,IAAK,CAAA;AACnD,MAAA,IAAI,GAAA,CAAI,cAAc,iBAAA,IAAqB,CAAA;AAAA,IAC7C;AAEA,IAAA,OAAO;AAAA,MACL,SAAA,EAAW,KAAK,SAAA,CAAU,IAAA;AAAA,MAC1B,KAAA,EAAO,MAAA,CAAO,IAAA,CAAK,SAAS,CAAA,CAAE,MAAA;AAAA,MAC9B,YAAA,EAAc,iBAAA;AAAA,MACd,MAAA,EAAQ;AAAA,KACV;AAAA,EACF;AACF","file":"indexer.js","sourcesContent":["/**\n * BaselineOS Knowledge Indexer\n * \n * Indexes project knowledge for RAG retrieval with hierarchical compression.\n * \n * Key Innovation: Pre-computed compressions at 5 levels allow token-optimized\n * context loading. Request `summary` (100 tokens) instead of `full` (10K).\n * \n * @license Apache-2.0\n */\n\nimport { glob } from 'glob';\nimport { readFileSync, existsSync, statSync, writeFileSync } from 'fs';\nimport { join, extname, basename, dirname } from 'path';\nimport matter from 'gray-matter';\nimport { createHash } from 'crypto';\n\nexport type ContextLevel = 'oneliner' | 'summary' | 'working' | 'full' | 'expanded';\nexport type Overlay = 'org' | 'repo' | 'public';\nexport type Sensitivity = 'public' | 'internal' | 'restricted';\n\nexport interface RetrievalBudget {\n /** Max tokens allowed for retrieval context */\n maxTokens?: number;\n /** Max cost in USD (requires costPer1kTokensUsd) */\n maxCostUsd?: number;\n /** Cost per 1k tokens in USD */\n costPer1kTokensUsd?: number;\n}\n\nexport interface IndexerConfig {\n projectRoot: string;\n knowledgePaths: string[];\n indexPath: string;\n includePatterns?: string[];\n excludePatterns?: string[];\n repo?: string;\n defaultOverlay?: Overlay;\n defaultSensitivity?: Sensitivity;\n}\n\nexport interface SearchResult {\n id: string;\n title: string;\n content: string;\n score: number;\n source: string;\n tokenCount: number;\n type: DocumentType;\n repo?: string;\n overlay?: Overlay;\n sensitivity?: Sensitivity;\n}\n\nexport interface Context {\n content: string;\n tokenCount: number;\n source: string;\n level: ContextLevel;\n documents: string[];\n budget?: {\n maxTokens?: number;\n maxCostUsd?: number;\n costPer1kTokensUsd?: number;\n consumedTokens: number;\n consumedCostUsd?: number;\n };\n contract?: RetrievalContract;\n provenance?: Array<{\n id: string;\n source: string;\n repo?: string;\n overlay?: Overlay;\n sensitivity?: Sensitivity;\n hash?: string;\n }>;\n}\n\nexport interface RetrievalContract {\n claims: Array<{\n text: string;\n sourceId: string;\n confidence: number;\n }>;\n sources: Context['provenance'];\n confidence: number;\n}\n\ntype DocumentType = 'doc' | 'code' | 'config' | 'spec' | 'api';\n\ninterface IndexedDocument {\n id: string;\n title: string;\n content: string;\n tokenCount: number;\n source: string;\n type: DocumentType;\n hash: string;\n indexedAt: number;\n metadata: Record<string, unknown>;\n repo?: string;\n overlay?: Overlay;\n sensitivity?: Sensitivity;\n scopes?: string[];\n roles?: string[];\n \n // Pre-computed compressions\n compressions: {\n oneliner: string;\n summary: string;\n working: string;\n };\n}\n\n// Token estimation (rough approximation: 1 token ≈ 4 chars)\nfunction estimateTokens(text: string): number {\n return Math.ceil(text.length / 4);\n}\n\n// Content hashing for change detection\nfunction hashContent(content: string): string {\n return createHash('sha256').update(content).digest('hex').slice(0, 16);\n}\n\n// Determine document type from extension and path\nfunction determineType(filePath: string): DocumentType {\n const ext = extname(filePath).toLowerCase();\n const name = basename(filePath).toLowerCase();\n \n if (name.includes('spec') || name.includes('requirement')) return 'spec';\n if (name.includes('api') || name.includes('swagger') || name.includes('openapi')) return 'api';\n if (['.json', '.yaml', '.yml', '.toml'].includes(ext)) return 'config';\n if (['.md', '.mdx', '.txt', '.rst'].includes(ext)) return 'doc';\n return 'code';\n}\n\nfunction normalizeOverlay(value: unknown, fallback: Overlay): Overlay {\n if (value === 'org' || value === 'repo' || value === 'public') return value;\n return fallback;\n}\n\nfunction normalizeSensitivity(value: unknown, fallback: Sensitivity): Sensitivity {\n if (value === 'public' || value === 'internal' || value === 'restricted') return value;\n return fallback;\n}\n\nfunction toStringArray(value: unknown): string[] | undefined {\n if (Array.isArray(value)) {\n return value.map(v => String(v)).filter(Boolean);\n }\n if (typeof value === 'string') {\n return [value];\n }\n return undefined;\n}\n\nexport interface PolicyFilter {\n repo?: string[];\n overlays?: Overlay[];\n sensitivity?: Sensitivity;\n scopes?: string[];\n roles?: string[];\n}\n\nfunction sensitivityRank(level?: Sensitivity): number {\n switch (level) {\n case 'public': return 1;\n case 'internal': return 2;\n case 'restricted': return 3;\n default: return 3;\n }\n}\n\nfunction isDocAllowed(doc: IndexedDocument, policy?: PolicyFilter): boolean {\n if (!policy) return true;\n\n if (policy.repo && doc.repo && !policy.repo.includes(doc.repo)) return false;\n if (policy.overlays && doc.overlay && !policy.overlays.includes(doc.overlay)) return false;\n if (policy.sensitivity && doc.sensitivity) {\n if (sensitivityRank(doc.sensitivity) > sensitivityRank(policy.sensitivity)) return false;\n }\n if (policy.scopes && doc.scopes && doc.scopes.length > 0) {\n const ok = policy.scopes.some(s => doc.scopes!.includes(s));\n if (!ok) return false;\n }\n if (policy.roles && doc.roles && doc.roles.length > 0) {\n const ok = policy.roles.some(r => doc.roles!.includes(r));\n if (!ok) return false;\n }\n return true;\n}\n\n// Generate compressions at different levels\nfunction generateCompressions(content: string, title: string): {\n oneliner: string;\n summary: string;\n working: string;\n} {\n const lines = content.split('\\n').filter(l => l.trim());\n \n // Oneliner: First meaningful line or title (~20 tokens ≈ 80 chars)\n const firstLine = lines[0] || title;\n const oneliner = firstLine.slice(0, 80);\n\n // Summary: First paragraph or first 3 sentences (~100 tokens ≈ 400 chars)\n const firstParagraph = content.split(/\\n\\n/)[0] || '';\n const summary = firstParagraph.slice(0, 400);\n\n // Working: First section (~500 tokens ≈ 2000 chars)\n const sections = content.split(/^##?\\s+/m);\n const firstSection = sections.length > 1 ? sections[0] + sections[1] : content;\n const working = firstSection.slice(0, 2000);\n \n return { oneliner, summary, working };\n}\n\nexport class KnowledgeIndexer {\n private config: IndexerConfig;\n private documents: Map<string, IndexedDocument>;\n private initialized: boolean = false;\n private indexVersion: number = 1;\n\n constructor(config: Partial<IndexerConfig> = {}) {\n this.config = {\n projectRoot: config.projectRoot ?? process.cwd(),\n knowledgePaths: config.knowledgePaths ?? ['./docs', './knowledge'],\n indexPath: config.indexPath ?? '.baseline/index',\n includePatterns: config.includePatterns ?? [\n '**/*.md',\n '**/*.mdx',\n '**/*.ts',\n '**/*.tsx',\n '**/*.js',\n '**/*.py',\n '**/*.json',\n '**/*.yaml',\n '**/*.yml',\n ],\n excludePatterns: config.excludePatterns ?? [\n '**/node_modules/**',\n '**/dist/**',\n '**/.git/**',\n '**/coverage/**',\n '**/*.min.js',\n '**/*.map',\n ],\n repo: config.repo,\n defaultOverlay: config.defaultOverlay ?? 'repo',\n defaultSensitivity: config.defaultSensitivity ?? 'internal',\n };\n this.documents = new Map();\n }\n\n async initialize(): Promise<void> {\n if (this.initialized) return;\n \n // Try to load existing index\n const indexFile = join(this.config.indexPath, 'index.json');\n if (existsSync(indexFile)) {\n try {\n const data = JSON.parse(readFileSync(indexFile, 'utf-8'));\n if (data.version === this.indexVersion) {\n for (const doc of data.documents) {\n this.documents.set(doc.id, doc);\n }\n }\n } catch (err) {\n process.stderr.write(`[baseline:indexer] Index corrupted, rebuilding: ${(err as Error).message}\\n`);\n }\n }\n \n this.initialized = true;\n }\n\n async indexAll(): Promise<{ indexed: number; errors: number; skipped: number; changed: string[]; removed: string[] }> {\n let indexed = 0;\n let errors = 0;\n let skipped = 0;\n const changed: string[] = [];\n const removed: string[] = [];\n const seen = new Set<string>();\n\n for (const knowledgePath of this.config.knowledgePaths) {\n const fullPath = join(this.config.projectRoot, knowledgePath);\n \n if (!existsSync(fullPath)) {\n continue;\n }\n\n // Handle single file paths (like README.md)\n const stat = statSync(fullPath);\n if (stat.isFile()) {\n try {\n const result = await this.indexFile(fullPath, knowledgePath);\n seen.add(knowledgePath);\n if (result === 'indexed') indexed++;\n else if (result === 'skipped') skipped++;\n if (result === 'indexed') changed.push(knowledgePath);\n } catch (err) {\n process.stderr.write(`[baseline:indexer] Failed to index ${knowledgePath}: ${(err as Error).message}\\n`);\n errors++;\n }\n continue;\n }\n\n // Handle directories\n const files = await glob(this.config.includePatterns!, {\n cwd: fullPath,\n ignore: this.config.excludePatterns,\n nodir: true,\n });\n\n for (const file of files) {\n try {\n const filePath = join(fullPath, file);\n const result = await this.indexFile(filePath, `${knowledgePath}/${file}`);\n seen.add(`${knowledgePath}/${file}`);\n if (result === 'indexed') indexed++;\n else if (result === 'skipped') skipped++;\n if (result === 'indexed') changed.push(`${knowledgePath}/${file}`);\n } catch (err) {\n process.stderr.write(`[baseline:indexer] Failed to index ${knowledgePath}/${file}: ${(err as Error).message}\\n`);\n errors++;\n }\n }\n }\n\n // Remove documents that no longer exist\n for (const id of this.documents.keys()) {\n if (!seen.has(id)) {\n this.documents.delete(id);\n removed.push(id);\n }\n }\n\n // Persist index\n await this.persist({ changed, removed });\n\n return { indexed, errors, skipped, changed, removed };\n }\n\n private async indexFile(filePath: string, id: string): Promise<'indexed' | 'skipped' | 'error'> {\n const content = readFileSync(filePath, 'utf-8');\n const hash = hashContent(content);\n \n // Check if already indexed with same hash\n const existing = this.documents.get(id);\n if (existing && existing.hash === hash) {\n return 'skipped';\n }\n\n const ext = extname(filePath);\n let title = basename(filePath, ext);\n let processedContent = content;\n let metadata: Record<string, unknown> = {};\n \n // Extract frontmatter from markdown\n if (['.md', '.mdx'].includes(ext)) {\n try {\n const parsed = matter(content);\n title = (parsed.data.title as string) || title;\n processedContent = parsed.content;\n metadata = parsed.data;\n } catch {\n // Not valid frontmatter, use raw content\n }\n }\n\n const compressions = generateCompressions(processedContent, title);\n const repo = (metadata.repo as string) || this.config.repo || basename(this.config.projectRoot);\n const overlay = normalizeOverlay(metadata.overlay, this.config.defaultOverlay ?? 'repo');\n const sensitivity = normalizeSensitivity(\n metadata.sensitivity,\n this.config.defaultSensitivity ?? 'internal'\n );\n const scopes = toStringArray(metadata.scopes ?? metadata.scope);\n const roles = toStringArray(metadata.roles ?? metadata.role);\n\n const doc: IndexedDocument = {\n id,\n title,\n content: processedContent,\n tokenCount: estimateTokens(processedContent),\n source: filePath,\n type: determineType(filePath),\n hash,\n indexedAt: Date.now(),\n metadata,\n repo,\n overlay,\n sensitivity,\n scopes,\n roles,\n compressions,\n };\n\n this.documents.set(id, doc);\n return 'indexed';\n }\n\n async clear(): Promise<void> {\n this.documents.clear();\n }\n\n async search(query: string, options?: { \n limit?: number;\n types?: DocumentType[];\n policy?: PolicyFilter;\n }): Promise<SearchResult[]> {\n const limit = options?.limit ?? 5;\n const types = options?.types;\n const policy = options?.policy;\n \n const words = query.toLowerCase().split(/\\s+/).filter(w => w.length > 2);\n \n if (words.length === 0) {\n return [];\n }\n\n const results: SearchResult[] = [];\n\n for (const doc of this.documents.values()) {\n // Filter by type if specified\n if (types && !types.includes(doc.type)) {\n continue;\n }\n if (!isDocAllowed(doc, policy)) {\n continue;\n }\n\n const contentLower = doc.content.toLowerCase();\n const titleLower = doc.title.toLowerCase();\n \n let score = 0;\n for (const word of words) {\n // Title matches are worth more\n if (titleLower.includes(word)) score += 3;\n \n // Count occurrences in content\n const regex = new RegExp(word, 'gi');\n const matches = contentLower.match(regex);\n if (matches) score += Math.min(matches.length, 5);\n }\n\n if (score > 0) {\n results.push({\n id: doc.id,\n title: doc.title,\n content: doc.compressions.summary,\n score: score / words.length,\n source: doc.source,\n tokenCount: doc.tokenCount,\n type: doc.type,\n repo: doc.repo,\n overlay: doc.overlay,\n sensitivity: doc.sensitivity,\n });\n }\n }\n\n return results\n .sort((a, b) => b.score - a.score)\n .slice(0, limit);\n }\n\n async getContext(\n subject: string,\n level: ContextLevel = 'working',\n options?: {\n policy?: PolicyFilter;\n overlayOrder?: Overlay[];\n budget?: RetrievalBudget;\n }\n ): Promise<Context | null> {\n // Search for matching documents\n const results = await this.search(subject, { limit: 10, policy: options?.policy });\n \n if (results.length === 0) {\n return null;\n }\n\n return this.buildContextFromResults(results, level, options?.overlayOrder, options?.budget);\n }\n\n async getContextDelta(\n subject: string,\n level: ContextLevel = 'working',\n options?: {\n policy?: PolicyFilter;\n overlayOrder?: Overlay[];\n known?: Array<{ id: string; hash?: string }>;\n budget?: RetrievalBudget;\n }\n ): Promise<Context | null> {\n const results = await this.search(subject, { limit: 10, policy: options?.policy });\n if (results.length === 0) {\n return null;\n }\n\n const known = new Map<string, string | undefined>();\n for (const item of options?.known ?? []) {\n if (item?.id) known.set(item.id, item.hash);\n }\n\n const filtered = results.filter(result => {\n const doc = this.documents.get(result.id);\n if (!doc) return false;\n const knownHash = known.get(doc.id);\n return !knownHash || knownHash !== doc.hash;\n });\n\n if (filtered.length === 0) {\n return null;\n }\n\n return this.buildContextFromResults(filtered, level, options?.overlayOrder, options?.budget);\n }\n\n private buildContextFromResults(\n results: SearchResult[],\n level: ContextLevel,\n overlayOrder?: Overlay[],\n budget?: RetrievalBudget\n ): Context | null {\n // Build context based on level\n const contextParts: string[] = [];\n const documentIds: string[] = [];\n const provenance: Context['provenance'] = [];\n const claims: RetrievalContract['claims'] = [];\n let confidenceTotal = 0;\n let confidenceCount = 0;\n let totalTokens = 0;\n const levelMaxTokens = this.getMaxTokensForLevel(level);\n const maxTokens = typeof budget?.maxTokens === 'number'\n ? Math.min(levelMaxTokens, budget.maxTokens)\n : levelMaxTokens;\n const costPer1k = typeof budget?.costPer1kTokensUsd === 'number' ? budget.costPer1kTokensUsd : undefined;\n const maxCostUsd = typeof budget?.maxCostUsd === 'number' ? budget.maxCostUsd : undefined;\n const order = overlayOrder ?? ['org', 'repo', 'public'];\n\n const grouped = new Map<Overlay, SearchResult[]>();\n for (const result of results) {\n const overlay = result.overlay ?? 'repo';\n const list = grouped.get(overlay) ?? [];\n list.push(result);\n grouped.set(overlay, list);\n }\n\n for (const overlay of order) {\n const overlayResults = grouped.get(overlay) ?? [];\n for (const result of overlayResults) {\n const doc = this.documents.get(result.id);\n if (!doc) continue;\n\n let content: string;\n \n switch (level) {\n case 'oneliner':\n content = doc.compressions.oneliner;\n break;\n case 'summary':\n content = doc.compressions.summary;\n break;\n case 'working':\n content = doc.compressions.working;\n break;\n case 'full':\n content = doc.content;\n break;\n case 'expanded':\n // Include related documents\n content = doc.content;\n // TODO: Find and include related documents\n break;\n }\n\n const tokens = estimateTokens(content);\n if (totalTokens + tokens > maxTokens) {\n break;\n }\n if (maxCostUsd !== undefined && costPer1k !== undefined && costPer1k > 0) {\n const nextCost = ((totalTokens + tokens) / 1000) * costPer1k;\n if (nextCost > maxCostUsd) {\n break;\n }\n }\n\n\n contextParts.push(`## ${doc.title}\\n\\n${content}`);\n documentIds.push(doc.id);\n const claimConfidence = Math.max(0.1, Math.min(1, result.score ?? 0));\n claims.push({\n text: doc.compressions.oneliner || doc.title,\n sourceId: doc.id,\n confidence: claimConfidence,\n });\n confidenceTotal += claimConfidence;\n confidenceCount += 1;\n\n provenance.push({\n id: doc.id,\n source: doc.source,\n repo: doc.repo,\n overlay: doc.overlay,\n sensitivity: doc.sensitivity,\n hash: doc.hash,\n });\n totalTokens += tokens;\n }\n }\n\n if (contextParts.length === 0) {\n return null;\n }\n\n const consumedCostUsd =\n costPer1k !== undefined && costPer1k > 0\n ? (totalTokens / 1000) * costPer1k\n : undefined;\n\n const contract: RetrievalContract = {\n claims,\n sources: provenance,\n confidence: confidenceCount > 0 ? confidenceTotal / confidenceCount : 0.5,\n };\n\n return {\n content: contextParts.join('\\n\\n---\\n\\n'),\n tokenCount: totalTokens,\n source: results[0].source,\n level,\n documents: documentIds,\n budget: budget\n ? {\n maxTokens: budget.maxTokens ?? levelMaxTokens,\n maxCostUsd,\n costPer1kTokensUsd: costPer1k,\n consumedTokens: totalTokens,\n consumedCostUsd,\n }\n : undefined,\n contract,\n provenance,\n };\n }\n\n getDocument(id: string): IndexedDocument | undefined {\n return this.documents.get(id);\n }\n\n private getMaxTokensForLevel(level: ContextLevel): number {\n switch (level) {\n case 'oneliner': return 20;\n case 'summary': return 100;\n case 'working': return 500;\n case 'full': return 10000;\n case 'expanded': return 50000;\n }\n }\n\n private async persist(changes?: { changed: string[]; removed: string[] }): Promise<void> {\n const indexFile = join(this.config.indexPath, 'index.json');\n const changesFile = join(this.config.indexPath, 'changes.json');\n const data = {\n version: this.indexVersion,\n createdAt: Date.now(),\n documents: Array.from(this.documents.values()),\n };\n \n try {\n const { mkdirSync } = await import('fs');\n mkdirSync(dirname(indexFile), { recursive: true });\n writeFileSync(indexFile, JSON.stringify(data));\n if (changes) {\n writeFileSync(changesFile, JSON.stringify({\n updatedAt: Date.now(),\n changed: changes.changed,\n removed: changes.removed,\n }));\n }\n } catch (err) {\n process.stderr.write(`[baseline:indexer] Index persistence failed: ${(err as Error).message}\\n`);\n }\n }\n\n getStats() {\n const typeCount: Record<string, number> = {};\n let totalCompressions = 0;\n \n for (const doc of this.documents.values()) {\n typeCount[doc.type] = (typeCount[doc.type] || 0) + 1;\n if (doc.compressions) totalCompressions += 3;\n }\n\n return {\n documents: this.documents.size,\n types: Object.keys(typeCount).length,\n compressions: totalCompressions,\n byType: typeCount,\n };\n }\n}\n"]}
|