ex-brain 0.2.6 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,169 +1,21 @@
1
- import { basename, resolve } from "node:path";
2
1
  import { readFileSync } from "node:fs";
3
2
  import { Command } from "commander";
4
- import { DEFAULT_DB_NAME, inferTypeFromSlug, slugToTitle, normalizeLongSlug, slugify } from "../config";
5
- import { BrainDb } from "../db/client";
6
- import {
7
- collectMarkdownFiles,
8
- ensureDir,
9
- fileExists,
10
- pathToSlug,
11
- readMaybeStdin,
12
- readTextFile,
13
- slugToPath,
14
- writeTextFile,
15
- } from "../markdown/io";
16
- import {
17
- extractTimelineLines,
18
- extractWikiStyleLinks,
19
- parsePageMarkdown,
20
- renderPageMarkdown,
21
- } from "../markdown/parser";
22
- import { BrainRepository } from "../repositories/brain-repo";
23
- import { loadSettings, SETTINGS_PATH, DEFAULT_DB_PATH, type ResolvedLLM } from "../settings";
24
- import { extractRelations, entityToSlug, type EntityType } from "../ai/entity-link";
3
+ import { DEFAULT_DB_NAME } from "../slug-utils";
25
4
  import { registerCompileCommands } from "./compile-cmd";
26
5
  import { registerGraphCommand } from "./graph-cmd";
27
- import { createProgress, formatDuration } from "../utils/progress";
6
+ import { registerPutCommand } from "./put-cmd";
7
+ import { registerQueryCommand } from "./query-cmd";
8
+ import { registerImportCommand } from "./import-cmd";
9
+ import { registerTimelineCommand } from "./timeline-cmd";
10
+ import { registerTagCommand, registerRawCommand, registerLinkCommand } from "./misc-cmds";
28
11
  import {
29
- success,
30
- error as cliError,
31
- warning,
32
- info,
33
- step,
34
- subItem,
35
- keyValue,
36
- header,
37
- separator,
38
- createSpinner,
39
- formatCount,
40
- type ProgressSpinner,
41
- } from "../utils/cli-output";
42
-
43
- // ---------------------------------------------------------------------------
44
- // Helpers
45
- // ---------------------------------------------------------------------------
46
-
47
- function addDryRun(cmd: Command): Command {
48
- return cmd.option("--dry-run", "preview changes without executing", false);
49
- }
50
-
51
- function isDryRun(opts: Record<string, unknown>): boolean {
52
- return Boolean(opts.dryRun);
53
- }
54
-
55
- // Simple progress output to stderr (won't interfere with --json stdout).
56
- // e.g. "[3/42] import docs/api"
57
- function progress(label: string, current: number, total: number, json: boolean): void {
58
- if (json) return;
59
- process.stderr.write(`[${current}/${total}] ${label}\n`);
60
- }
61
-
62
- /**
63
- * Extract entities and create entity pages + links.
64
- * Non-blocking: failures produce warnings, not errors.
65
- */
66
- async function applyEntityLinks(
67
- repo: BrainRepository,
68
- sourceSlug: string,
69
- content: string,
70
- json: boolean,
71
- ): Promise<{ created: number; linked: number }> {
72
- if (!content.trim()) return { created: 0, linked: 0 };
73
-
74
- const settings = await loadSettings();
75
- if (!settings.llm.baseURL) {
76
- if (!json) {
77
- warning(`LLM not configured, skipping entity extraction for ${sourceSlug}`);
78
- }
79
- return { created: 0, linked: 0 };
80
- }
81
-
82
- const spinner = createSpinner();
83
- if (!json) {
84
- spinner.start(`Extracting entities from ${sourceSlug}...`);
85
- }
86
-
87
- const startTime = Date.now();
88
- let relations;
89
- try {
90
- relations = await extractRelations(content, settings.llm);
91
- } catch (err) {
92
- if (!json) {
93
- spinner.fail(`Entity extraction failed: ${err instanceof Error ? err.message : String(err)}`);
94
- }
95
- return { created: 0, linked: 0 };
96
- }
97
-
98
- // Filter by confidence
99
- const confidenceThreshold = settings.extraction.confidenceThreshold;
100
- const highConfidence = relations.filter((r) => r.confidence >= confidenceThreshold);
101
- const ignoredCount = relations.length - highConfidence.length;
102
-
103
- if (highConfidence.length === 0) {
104
- if (!json) {
105
- if (relations.length > 0) {
106
- spinner.warn(`Found ${relations.length} entities but all below confidence threshold (${confidenceThreshold})`);
107
- } else {
108
- spinner.warn(`No entities found in content`);
109
- }
110
- }
111
- return { created: 0, linked: 0 };
112
- }
113
-
114
- let created = 0;
115
- let linked = 0;
116
- const details: string[] = [];
117
-
118
- for (const r of highConfidence) {
119
- // 1. Resolve entity slugs (disambiguation)
120
- const fromCandidate = entityToSlug(r.from.name, r.from.type);
121
- const toCandidate = entityToSlug(r.to.name, r.to.type);
122
-
123
- const fromSlug = await repo.findSimilarSlug(fromCandidate, r.from.name);
124
- const toSlug = await repo.findSimilarSlug(toCandidate, r.to.name);
125
-
126
- // 2. Ensure entity pages exist
127
- const c1 = await repo.ensureEntityPage(fromSlug, r.from.type, r.from.name, r.relation, r.context, sourceSlug);
128
- const c2 = await repo.ensureEntityPage(toSlug, r.to.type, r.to.name, r.relation, r.context, sourceSlug);
129
- if (c1) { created += 1; details.push(`Created: ${r.from.name} (${r.from.type})`); }
130
- if (c2) { created += 1; details.push(`Created: ${r.to.name} (${r.to.type})`); }
131
-
132
- // 3. Link between entities (context includes relation type)
133
- await repo.link(fromSlug, toSlug, `[${r.relation}] ${r.context}`);
134
- linked += 1;
135
-
136
- // 4. Link from source document to entities (for backlinks tracing)
137
- await repo.link(sourceSlug, fromSlug, `Mentions ${r.from.name}`);
138
- linked += 1;
139
- await repo.link(sourceSlug, toSlug, `Mentions ${r.to.name}`);
140
- linked += 1;
141
- }
142
-
143
- if (!json) {
144
- const duration = formatDuration(Date.now() - startTime);
145
- const entityNames = [...new Set(highConfidence.flatMap((r) => [r.from.name, r.to.name]))];
146
- spinner.succeed(`Extracted ${entityNames.length} entities: ${entityNames.join(", ")}`);
147
-
148
- // Print detailed info
149
- subItem(`${created} entity pages created`);
150
- subItem(`${linked} links added`);
151
- if (ignoredCount > 0) {
152
- subItem(`${ignoredCount} low-confidence relations ignored`);
153
- }
154
- subItem(`Completed in ${duration}`);
155
- }
156
-
157
- return { created, linked };
158
- }
159
-
160
- async function resolveInput(
161
- fileOpt: string | undefined,
162
- stdin: boolean,
163
- ): Promise<string> {
164
- if (fileOpt) return readTextFile(resolve(fileOpt));
165
- return readMaybeStdin().then((s) => s ?? "");
166
- }
12
+ registerExportCommand,
13
+ registerEmbedCommand,
14
+ registerInitCommand,
15
+ registerStatsCommand,
16
+ registerConfigCommand,
17
+ registerServeCommand,
18
+ } from "./misc-commands";
167
19
 
168
20
  // ---------------------------------------------------------------------------
169
21
  // Build
@@ -190,1430 +42,20 @@ Examples:
190
42
  .option("--db <path>", "database path (overrides settings.json)")
191
43
  .option("--json", "output as JSON", false);
192
44
 
193
- // -- config ---------------------------------------------------------------
194
-
195
- program
196
- .command("config")
197
- .description("show resolved configuration")
198
- .action(async () => {
199
- const settings = await loadSettings();
200
- const cliDb = program.opts().db;
201
- const effectiveDb = cliDb ?? settings.dbPath;
202
- print(program, {
203
- settingsFile: SETTINGS_PATH,
204
- dbPath: effectiveDb,
205
- mode: settings.remote ? "remote" : "local",
206
- remote: settings.remote ?? null,
207
- embed: {
208
- provider: settings.embed.provider,
209
- baseURL: settings.embed.baseURL,
210
- model: settings.embed.model,
211
- dimensions: settings.embed.dimensions,
212
- hasApiKey:
213
- !!settings.embed.apiKey ||
214
- !!process.env[settings.embed.apiKeyEnv],
215
- },
216
- llm: {
217
- baseURL: settings.llm.baseURL || "(not configured)",
218
- model: settings.llm.model,
219
- hasApiKey:
220
- !!settings.llm.apiKey ||
221
- !!process.env[settings.llm.apiKeyEnv],
222
- },
223
- });
224
- });
225
-
226
- // -- page CRUD ------------------------------------------------------------
227
-
228
- addDryRun(
229
- program
230
- .command("put")
231
- .argument("[slug]", "page slug (optional; auto-generated if omitted)")
232
- .option("--file <path>", "read markdown from file")
233
- .option("--stdin", "read markdown from stdin", false)
234
- .option("--type <type>", "page type")
235
- .option("--title <title>", "page title")
236
- .description(
237
- "create or update a page (idempotent; upserts by slug). If slug is omitted, it is auto-generated from file name, title, or timestamp.",
238
- )
239
- .addHelpText(
240
- "after",
241
- `
242
- Examples:
243
- ebrain put --file api.md # auto-generate slug from file name
244
- ebrain put docs/api --file api.md # explicit slug
245
- cat note.md | ebrain put --stdin # auto-generate slug from title/timestamp
246
- ebrain put --title "My Note" --stdin # auto-generate slug from title
247
- ebrain put people/john --type person --title "John Doe"
248
- ebrain put docs/api --file api.md --dry-run
249
- `,
250
- ),
251
- ).action(
252
- async (
253
- slug: string | undefined,
254
- opts: {
255
- file?: string;
256
- stdin?: boolean;
257
- type?: string;
258
- title?: string;
259
- dryRun?: boolean;
260
- },
261
- ) => {
262
- const input = await resolveInput(opts.file, opts.stdin ?? false);
263
- if (!input.trim()) {
264
- throw new Error(
265
- "empty input — provide --file <path>, --stdin, or pipe markdown",
266
- );
267
- }
268
- const parsed = parsePageMarkdown(input);
269
-
270
- // Auto-generate slug if not provided
271
- let finalSlug = slug;
272
- if (!finalSlug) {
273
- // Priority: file name > title option > frontmatter title > timestamp
274
- if (opts.file) {
275
- const fileName = basename(opts.file).replace(/\.md$/i, "");
276
- finalSlug = normalizeLongSlug(slugify(fileName));
277
- } else if (opts.title) {
278
- finalSlug = normalizeLongSlug(slugify(opts.title));
279
- } else if (parsed.frontmatter.title) {
280
- finalSlug = normalizeLongSlug(slugify(String(parsed.frontmatter.title)));
281
- } else {
282
- // Use timestamp as fallback
283
- const timestamp = new Date().toISOString().slice(0, 19).replace(/[-:T]/g, "");
284
- finalSlug = `notes/${timestamp}`;
285
- }
286
- }
287
-
288
- const type =
289
- opts.type ??
290
- String(parsed.frontmatter.type ?? inferTypeFromSlug(finalSlug));
291
- const title =
292
- opts.title ??
293
- String(parsed.frontmatter.title ?? slugToTitle(finalSlug));
294
-
295
- if (isDryRun(opts)) {
296
- print(program, {
297
- dryRun: true,
298
- action: "put",
299
- slug: finalSlug,
300
- type,
301
- title,
302
- contentLength: parsed.compiledTruth.length,
303
- hasTimeline: !!parsed.timeline,
304
- frontmatterKeys: Object.keys(parsed.frontmatter),
305
- });
306
- return;
307
- }
308
-
309
- await withRepo(program, async (repo) => {
310
- const jsonOut = isJson(program);
311
- const spinner = createSpinner();
312
- const startTime = Date.now();
313
-
314
- if (!jsonOut) {
315
- header(`Put: ${finalSlug}`);
316
- spinner.start(`Creating/updating page...`);
317
- }
318
-
319
- const page = await repo.putPage({
320
- slug: finalSlug,
321
- type,
322
- title,
323
- compiledTruth: parsed.compiledTruth,
324
- timeline: parsed.timeline,
325
- frontmatter: parsed.frontmatter,
326
- });
327
-
328
- if (!jsonOut) {
329
- spinner.succeed(`Page saved: ${page.slug}`);
330
- keyValue("Title", title);
331
- keyValue("Type", type);
332
- keyValue("Content length", `${parsed.compiledTruth.length} chars`);
333
- }
334
-
335
- await applyEntityLinks(
336
- repo,
337
- finalSlug,
338
- parsed.compiledTruth,
339
- jsonOut,
340
- );
341
-
342
- if (!jsonOut) {
343
- const duration = formatDuration(Date.now() - startTime);
344
- success(`Operation completed in ${duration}`);
345
- }
346
-
347
- print(program, { ok: true, slug: page.slug, updatedAt: page.updatedAt });
348
- });
349
- },
350
- );
351
-
352
- program
353
- .command("get")
354
- .argument("<slug>", "page slug")
355
- .option("--json", "output as JSON (overrides global --json)")
356
- .description("read a page and render it as markdown")
357
- .addHelpText(
358
- "after",
359
- `
360
- Examples:
361
- ebrain get docs/api
362
- ebrain get docs/api --json
363
- `,
364
- )
365
- .action(async (slug: string, opts: { json?: boolean }) => {
366
- const localJson = opts.json !== undefined ? opts.json : isJson(program);
367
- await withRepo(program, async (repo) => {
368
- const page = await repo.getPage(slug);
369
- if (!page) {
370
- throw new Error(`page not found: ${slug}`);
371
- }
372
- if (localJson) {
373
- console.log(JSON.stringify(page, null, 2));
374
- return;
375
- }
376
- console.log(
377
- renderPageMarkdown(
378
- page.frontmatter,
379
- page.compiledTruth,
380
- page.timeline,
381
- ),
382
- );
383
- });
384
- });
385
-
386
- addDryRun(
387
- program
388
- .command("delete")
389
- .argument("<slug>", "page slug to delete")
390
- .description("delete a page and its related data (links, tags, timeline, raw)")
391
- .addHelpText(
392
- "after",
393
- `
394
- Examples:
395
- ebrain delete notes/old-draft
396
- ebrain delete notes/old-draft --dry-run
397
- `,
398
- ),
399
- ).action(async (slug: string, opts: { dryRun?: boolean }) => {
400
- if (isDryRun(opts)) {
401
- await withRepo(program, async (repo) => {
402
- const page = await repo.getPage(slug);
403
- if (!page) {
404
- throw new Error(`page not found: ${slug}`);
405
- }
406
- print(program, {
407
- dryRun: true,
408
- action: "delete",
409
- slug,
410
- title: page.title,
411
- });
412
- });
413
- return;
414
- }
415
- await withRepo(program, async (repo) => {
416
- const jsonOut = isJson(program);
417
- const spinner = createSpinner();
418
-
419
- if (!jsonOut) {
420
- header(`Delete: ${slug}`);
421
- spinner.start(`Deleting page and related data...`);
422
- }
423
-
424
- await repo.deletePage(slug);
425
-
426
- if (!jsonOut) {
427
- spinner.succeed(`Page deleted: ${slug}`);
428
- }
429
-
430
- print(program, { ok: true, action: "delete", slug });
431
- });
432
- });
433
-
434
- program
435
- .command("list")
436
- .option("--type <type>", "filter by page type")
437
- .option("--tag <tag>", "filter by tag")
438
- .option("-f, --fields <fields>", "comma-separated fields to display (slug,type,title,createdAt,updatedAt)")
439
- .option("--limit <number>", "max results", "50")
440
- .description("list pages")
441
- .addHelpText(
442
- "after",
443
- `
444
- Examples:
445
- ebrain list
446
- ebrain list --type person
447
- ebrain list -f slug
448
- ebrain list -f slug,title,type
449
- `,
450
- )
451
- .action(async (opts: Record<string, string | undefined>) => {
452
- await withRepo(program, async (repo) => {
453
- const rows = await repo.listPages({
454
- type: opts.type,
455
- tag: opts.tag,
456
- limit: Number(opts.limit),
457
- });
458
-
459
- // When --fields is set, show one page per line with tab-separated values
460
- if (opts.fields) {
461
- const fields = opts.fields.split(",").map((f) => f.trim());
462
- for (const row of rows) {
463
- const vals = fields.map((field) => {
464
- const val = (row as Record<string, unknown>)[field];
465
- if (val === undefined || val === null) return "";
466
- if (typeof val === "object") return JSON.stringify(val);
467
- return String(val);
468
- });
469
- console.log(vals.join("\t"));
470
- }
471
- return;
472
- }
473
-
474
- print(program, rows);
475
- });
476
- });
477
-
478
- // -- search / query -------------------------------------------------------
479
-
480
- program
481
- .command("search")
482
- .argument("<query>", "full-text search query")
483
- .option("--type <type>", "filter by page type")
484
- .option("--limit <number>", "max results", "10")
485
- .description("full-text / hybrid search")
486
- .addHelpText(
487
- "after",
488
- `
489
- Examples:
490
- ebrain search "machine learning"
491
- ebrain search "quarterly revenue" --type deal --limit 5
492
- `,
493
- )
494
- .action(async (query: string, opts: Record<string, string>) => {
495
- await withRepo(program, async (repo) => {
496
- const hits = await repo.search(
497
- query,
498
- Number(opts.limit ?? 10),
499
- opts.type,
500
- );
501
- print(program, hits);
502
- });
503
- });
504
-
505
- program
506
- .command("query")
507
- .argument("<question>", "natural language question")
508
- .option("--limit <number>", "max results", "10")
509
- .option("--llm", "use LLM to answer based on retrieved context", false)
510
- .option("--context-limit <number>", "max pages to use as context", "5")
511
- .description("semantic / vector search")
512
- .addHelpText(
513
- "after",
514
- `
515
- Examples:
516
- ebrain query "What projects did we ship in Q4?"
517
- ebrain query "Who leads the ML team?" --limit 5
518
- ebrain query "What are the key findings?" --llm
519
- `,
520
- )
521
- .action(async (question: string, opts: Record<string, string>) => {
522
- await withRepo(program, async (repo) => {
523
- const limit = Number(opts.limit ?? 10);
524
- const hits = await repo.query(question, limit);
525
-
526
- // If --llm flag, generate answer based on multi-layer context
527
- if (opts.llm) {
528
- const settings = await loadSettings();
529
- if (!settings.llm.baseURL) {
530
- print(program, { error: "LLM not configured. Set llm.baseURL in settings." });
531
- return;
532
- }
533
-
534
- const progress = createProgress();
535
- progress.start("Searching knowledge base...");
536
-
537
- const contextLimit = Number(opts.contextLimit ?? 5);
538
- const topHits = hits.slice(0, contextLimit);
539
-
540
- if (topHits.length === 0) {
541
- progress.stop();
542
- process.stderr.write("No relevant pages found.\n");
543
- print(program, { answer: "No relevant information found in the knowledge base.", sources: [] });
544
- return;
545
- }
546
-
547
- // Collect multi-layer context (primary + raw data + linked pages scored by relevance)
548
- // ~100KB char budget ≈ 25K tokens, safe for most models
549
- const MAX_CONTEXT_CHARS = 100_000;
550
- const ctxStart = Date.now();
551
- progress.update(`Loading page content...`);
552
- const { sections, totalChars, stats } = await collectContextForLLM(repo, topHits, question, MAX_CONTEXT_CHARS, (stage) => {
553
- progress.update(`Loading ${stage}...`);
554
- });
555
- const ctxDuration = formatDuration(Date.now() - ctxStart);
556
-
557
- if (sections.length === 0) {
558
- progress.stop();
559
- process.stderr.write("No content could be loaded.\n");
560
- print(program, { answer: "Failed to load page content.", sources: [] });
561
- return;
562
- }
563
-
564
- progress.succeed(`Loaded ${stats.primaryPages} page(s), ${stats.rawDocs} raw doc(s), ${stats.linkedPages} linked page(s) (${ctxDuration})`);
565
- const startTime = Date.now();
566
-
567
- const { answer, ok } = await generateAnswerWithStream(question, sections, stats, settings.llm);
568
-
569
- if (!ok) {
570
- // If streaming failed, answer contains the error message
571
- console.log(answer);
572
- return;
573
- }
574
-
575
- const duration = formatDuration(Date.now() - startTime);
576
-
577
- // Show sources breakdown
578
- console.log("\n---\n**Sources:**\n");
579
- for (let i = 0; i < sections.length; i++) {
580
- const s = sections[i];
581
- const icon = s.type === 'primary' ? '📄' : s.type === 'raw_data' ? '📎' : '🔗';
582
- console.log(`${icon} ${i + 1}. [[${s.slug}|${s.title}]] — ${s.label} (${(s.content.length / 1024).toFixed(1)}KB)`);
583
- }
584
- console.log(`\n*Context: ${stats.primaryPages} page(s), ${stats.rawDocs} raw doc(s), ${stats.linkedPages} linked page(s)*`);
585
- } else {
586
- print(program, hits);
587
- }
588
- });
589
- });
590
-
591
- // -- link -----------------------------------------------------------------
592
-
593
- addDryRun(
594
- program
595
- .command("link")
596
- .argument("<from>", "source page slug")
597
- .argument("<to>", "target page slug")
598
- .option("--context <text>", "link context", "")
599
- .description("create a cross-link between pages (idempotent)")
600
- .addHelpText(
601
- "after",
602
- `
603
- Examples:
604
- ebrain link docs/api docs/getting-started
605
- ebrain link people/john projects/alpha --context "lead"
606
- ebrain link docs/api docs/getting-started --dry-run
607
- `,
608
- ),
609
- ).action(
610
- async (
611
- from: string,
612
- to: string,
613
- opts: { context?: string; dryRun?: boolean },
614
- ) => {
615
- if (isDryRun(opts)) {
616
- print(program, {
617
- dryRun: true,
618
- action: "link",
619
- from,
620
- to,
621
- context: opts.context ?? "",
622
- });
623
- return;
624
- }
625
- await withRepo(program, async (repo) => {
626
- await repo.link(from, to, opts.context ?? "");
627
- print(program, { ok: true, from, to });
628
- });
629
- },
630
- );
631
-
632
- program
633
- .command("backlinks")
634
- .argument("<slug>", "target page slug")
635
- .description("list pages that link to this page")
636
- .addHelpText(
637
- "after",
638
- `
639
- Examples:
640
- ebrain backlinks docs/api
641
- `,
642
- )
643
- .action(async (slug: string) => {
644
- await withRepo(program, async (repo) => {
645
- const links = await repo.backlinks(slug);
646
- print(program, links);
647
- });
648
- });
649
-
650
- // -- timeline (subcommands) -----------------------------------------------
651
-
652
- const timelineCmd = program
653
- .command("timeline")
654
- .description("manage timeline entries");
655
-
656
- timelineCmd
657
- .command("list")
658
- .argument("<slug>", "page slug")
659
- .option("--limit <number>", "max results", "50")
660
- .description("list timeline entries for a page")
661
- .addHelpText(
662
- "after",
663
- `
664
- Examples:
665
- ebrain timeline list projects/alpha
666
- ebrain timeline list projects/alpha --limit 10
667
- `,
668
- )
669
- .action(async (slug: string, opts: Record<string, string>) => {
670
- await withRepo(program, async (repo) => {
671
- const rows = await repo.timeline(slug, Number(opts.limit ?? 50));
672
- print(program, rows);
673
- });
674
- });
675
-
676
- addDryRun(
677
- timelineCmd
678
- .command("add")
679
- .argument("<slug>", "page slug")
680
- .requiredOption("--date <date>", "date (YYYY-MM-DD or ISO)")
681
- .requiredOption("--summary <summary>", "one-line summary")
682
- .option("--source <source>", "event source", "manual")
683
- .option("--detail <detail>", "detail markdown", "")
684
- .description("add a timeline entry")
685
- .addHelpText(
686
- "after",
687
- `
688
- Examples:
689
- ebrain timeline add projects/alpha --date 2025-03-15 --summary "v1.0 shipped"
690
- ebrain timeline add projects/alpha --date 2025-03-15 --summary "launch" --source release
691
- ebrain timeline add projects/alpha --date 2025-03-15 --summary "launch" --dry-run
692
- `,
693
- ),
694
- ).action(
695
- async (
696
- slug: string,
697
- opts: {
698
- date: string;
699
- summary: string;
700
- source?: string;
701
- detail?: string;
702
- dryRun?: boolean;
703
- },
704
- ) => {
705
- if (isDryRun(opts)) {
706
- print(program, {
707
- dryRun: true,
708
- action: "timeline-add",
709
- slug,
710
- date: opts.date,
711
- summary: opts.summary,
712
- source: opts.source ?? "manual",
713
- });
714
- return;
715
- }
716
- await withRepo(program, async (repo) => {
717
- await repo.timelineAdd({
718
- pageSlug: slug,
719
- date: opts.date,
720
- source: opts.source ?? "manual",
721
- summary: opts.summary,
722
- detail: opts.detail ?? "",
723
- });
724
- print(program, {
725
- ok: true,
726
- action: "timeline-add",
727
- slug,
728
- date: opts.date,
729
- });
730
- });
731
- },
732
- );
733
-
734
- addDryRun(
735
- timelineCmd
736
- .command("extract")
737
- .argument("<slug>", "page slug")
738
- .option("--source <source>", "source identifier", "extracted")
739
- .option("--default-date <date>", "default date (YYYY-MM-DD)")
740
- .description("extract timeline events from page content using AI")
741
- .addHelpText(
742
- "after",
743
- `
744
- Examples:
745
- ebrain timeline extract companies/river-ai
746
- ebrain timeline extract docs/meeting --source meeting_notes --default-date 2024-03-15
747
- `,
748
- ),
749
- ).action(async (slug: string, opts: { source?: string; defaultDate?: string; dryRun?: boolean }) => {
750
- if (isDryRun(opts)) {
751
- print(program, {
752
- dryRun: true,
753
- action: "timeline-extract",
754
- slug,
755
- source: opts.source ?? "extracted",
756
- defaultDate: opts.defaultDate ?? new Date().toISOString().slice(0, 10),
757
- });
758
- return;
759
- }
760
- await withRepo(program, async (repo) => {
761
- const page = await repo.getPage(slug);
762
- if (!page) {
763
- throw new Error(`page not found: ${slug}`);
764
- }
765
- const settings = await loadSettings();
766
-
767
- const progress = createProgress();
768
- progress.start(`Extracting timeline from ${slug}...`);
769
- const startTime = Date.now();
770
-
771
- const result = await repo.extractAndAddTimeline(
772
- slug,
773
- page.compiledTruth,
774
- opts.source ?? "extracted",
775
- opts.defaultDate ?? new Date().toISOString().slice(0, 10),
776
- settings.llm,
777
- );
778
-
779
- const duration = formatDuration(Date.now() - startTime);
780
-
781
- if (result.entries.length > 0) {
782
- progress.succeed(`${result.entries.length} events extracted (${duration})`);
783
- } else {
784
- progress.stop();
785
- process.stderr.write(`No events found (${duration})\n`);
786
- }
787
-
788
- print(program, {
789
- ok: true,
790
- action: "timeline-extract",
791
- slug,
792
- entriesAdded: result.entries.length,
793
- entries: result.entries,
794
- confidence: result.confidence,
795
- });
796
- });
797
- });
798
-
799
- timelineCmd
800
- .command("global")
801
- .option("--limit <number>", "max results", "100")
802
- .description("list timeline entries across all pages")
803
- .addHelpText(
804
- "after",
805
- `
806
- Examples:
807
- ebrain timeline global
808
- ebrain timeline global --limit 20
809
- `,
810
- )
811
- .action(async (opts: Record<string, string>) => {
812
- await withRepo(program, async (repo) => {
813
- const entries = await repo.timelineGlobal(Number(opts.limit ?? 100));
814
- print(program, entries);
815
- });
816
- });
817
-
818
- // -- tag (subcommands) ----------------------------------------------------
819
-
820
- const tagCmd = program
821
- .command("tag")
822
- .description("manage tags on a page");
823
-
824
- tagCmd
825
- .command("list")
826
- .argument("<slug>", "page slug")
827
- .description("list tags on a page")
828
- .addHelpText(
829
- "after",
830
- `
831
- Examples:
832
- ebrain tag list docs/api
833
- `,
834
- )
835
- .action(async (slug: string) => {
836
- await withRepo(program, async (repo) => {
837
- const tags = await repo.tags(slug);
838
- print(program, tags);
839
- });
840
- });
841
-
842
- addDryRun(
843
- tagCmd
844
- .command("add")
845
- .argument("<slug>", "page slug")
846
- .argument("<tag>", "tag to add")
847
- .description("add a tag to a page (idempotent)")
848
- .addHelpText(
849
- "after",
850
- `
851
- Examples:
852
- ebrain tag add docs/api rest
853
- ebrain tag add docs/api rest --dry-run
854
- `,
855
- ),
856
- ).action(async (slug: string, tag: string, opts: { dryRun?: boolean }) => {
857
- if (isDryRun(opts)) {
858
- print(program, { dryRun: true, action: "tag-add", slug, tag });
859
- return;
860
- }
861
- await withRepo(program, async (repo) => {
862
- await repo.tag(slug, tag);
863
- print(program, { ok: true, action: "tag-add", slug, tag });
864
- });
865
- });
866
-
867
- addDryRun(
868
- tagCmd
869
- .command("remove")
870
- .argument("<slug>", "page slug")
871
- .argument("<tag>", "tag to remove")
872
- .description("remove a tag from a page")
873
- .addHelpText(
874
- "after",
875
- `
876
- Examples:
877
- ebrain tag remove docs/api outdated
878
- ebrain tag remove docs/api outdated --dry-run
879
- `,
880
- ),
881
- ).action(async (slug: string, tag: string, opts: { dryRun?: boolean }) => {
882
- if (isDryRun(opts)) {
883
- print(program, { dryRun: true, action: "tag-remove", slug, tag });
884
- return;
885
- }
886
- await withRepo(program, async (repo) => {
887
- await repo.untag(slug, tag);
888
- print(program, { ok: true, action: "tag-remove", slug, tag });
889
- });
890
- });
891
-
892
- // -- raw (subcommands) ----------------------------------------------------
893
-
894
- const rawCmd = program
895
- .command("raw")
896
- .description("manage raw source data for a page");
897
-
898
- rawCmd
899
- .command("get")
900
- .argument("<slug>", "page slug")
901
- .option("--source <source>", "filter by source name")
902
- .description("read raw source data for a page")
903
- .addHelpText(
904
- "after",
905
- `
906
- Examples:
907
- ebrain raw get ingest/report
908
- ebrain raw get ingest/report --source crm
909
- `,
910
- )
911
- .action(async (slug: string, opts: { source?: string }) => {
912
- await withRepo(program, async (repo) => {
913
- const rows = await repo.readRaw(slug, opts.source);
914
- print(program, rows);
915
- });
916
- });
917
-
918
- addDryRun(
919
- rawCmd
920
- .command("set")
921
- .argument("<slug>", "page slug")
922
- .requiredOption("--source <source>", "source name")
923
- .option("--data <json>", "JSON string")
924
- .option("--stdin", "read JSON from stdin", false)
925
- .description("write raw source data for a page")
926
- .addHelpText(
927
- "after",
928
- `
929
- Examples:
930
- ebrain raw set ingest/report --source crm --data '{"rev": 1000}'
931
- echo '{"rev": 1000}' | ebrain raw set ingest/report --source crm --stdin
932
- ebrain raw set ingest/report --source crm --data '{"rev": 1000}' --dry-run
933
- `,
934
- ),
935
- ).action(
936
- async (
937
- slug: string,
938
- opts: {
939
- source: string;
940
- data?: string;
941
- stdin?: boolean;
942
- dryRun?: boolean;
943
- },
944
- ) => {
945
- let data: unknown;
946
- if (opts.data) {
947
- data = JSON.parse(opts.data);
948
- } else if (opts.stdin) {
949
- const raw = await readMaybeStdin();
950
- if (!raw?.trim()) throw new Error("empty stdin — pipe JSON");
951
- data = JSON.parse(raw);
952
- } else {
953
- throw new Error("provide --data <json> or --stdin");
954
- }
955
-
956
- if (isDryRun(opts)) {
957
- print(program, {
958
- dryRun: true,
959
- action: "raw-set",
960
- slug,
961
- source: opts.source,
962
- });
963
- return;
964
- }
965
-
966
- await withRepo(program, async (repo) => {
967
- await repo.writeRaw(slug, opts.source, data);
968
- print(program, {
969
- ok: true,
970
- action: "raw-set",
971
- slug,
972
- source: opts.source,
973
- });
974
- });
975
- },
976
- );
977
-
978
- // -- import / export ------------------------------------------------------
979
-
980
- addDryRun(
981
- program
982
- .command("import")
983
- .argument("<dir>", "directory of markdown files")
984
- .description("import a directory of markdown files")
985
- .option("--skip-index", "skip vector indexing (useful if seekdb crashes)")
986
- .addHelpText(
987
- "after",
988
- `
989
- Examples:
990
- ebrain import ./docs
991
- ebrain import ./docs --dry-run
992
- ebrain import ./docs --skip-index # skip vector indexing
993
- `,
994
- ),
995
- ).action(async (dir: string, opts: { dryRun?: boolean; skipIndex?: boolean }) => {
996
- await withRepo(program, async (repo) => {
997
- const root = resolve(dir);
998
- const files = await collectMarkdownFiles(root);
999
-
1000
- if (isDryRun(opts)) {
1001
- print(program, {
1002
- dryRun: true,
1003
- action: "import",
1004
- dir: root,
1005
- filesFound: files.length,
1006
- slugs: files.map((f) => pathToSlug(f, root)),
1007
- });
1008
- return;
1009
- }
1010
-
1011
- const jsonOut = isJson(program);
1012
- const settings = await loadSettings();
1013
- const spinner = createSpinner();
1014
- const startTime = Date.now();
1015
-
1016
- if (!jsonOut) {
1017
- header(`Import: ${root}`);
1018
- }
1019
-
1020
- // Phase 1: Parse all files and collect data
1021
- if (!jsonOut) {
1022
- spinner.start(`Scanning ${files.length} files...`);
1023
- }
1024
-
1025
- const fileData: Array<{
1026
- file: string;
1027
- slug: string;
1028
- parsed: ReturnType<typeof parsePageMarkdown>;
1029
- content: string;
1030
- wikiLinks: string[];
1031
- timelineEntries: ReturnType<typeof extractTimelineLines>;
1032
- tags: string[];
1033
- }> = [];
1034
-
1035
- for (const file of files) {
1036
- const rawSlug = pathToSlug(file, root);
1037
- const slug = normalizeLongSlug(rawSlug);
1038
- const content = await readTextFile(file);
1039
- const parsed = parsePageMarkdown(content);
1040
- const wikiLinks = extractWikiStyleLinks(content).map(normalizeLinkSlug);
1041
- const timelineEntries = extractTimelineLines(parsed.timeline);
1042
- const tags = Array.isArray(parsed.frontmatter.tags)
1043
- ? parsed.frontmatter.tags.filter((t): t is string => typeof t === "string")
1044
- : [];
1045
- fileData.push({ file, slug, parsed, content, wikiLinks, timelineEntries, tags });
1046
- }
1047
-
1048
- if (!jsonOut) {
1049
- spinner.succeed(`Found ${files.length} markdown files`);
1050
- }
1051
-
1052
- // Phase 2: Write all pages first (skip embed for performance)
1053
- if (!jsonOut) {
1054
- spinner.start(`Writing ${fileData.length} pages to database...`);
1055
- }
1056
-
1057
- const allSlugs: string[] = [];
1058
- const writeErrors: string[] = [];
1059
-
1060
- for (let i = 0; i < fileData.length; i++) {
1061
- const { slug, parsed } = fileData[i]!;
1062
- if (!jsonOut && i % 20 === 0) {
1063
- spinner.update(`Writing pages... ${i + 1}/${fileData.length}`);
1064
- }
1065
- try {
1066
- await repo.putPage({
1067
- slug,
1068
- type: String(parsed.frontmatter.type ?? inferTypeFromSlug(slug)),
1069
- title: String(parsed.frontmatter.title ?? slugToTitle(slug)),
1070
- compiledTruth: parsed.compiledTruth,
1071
- timeline: parsed.timeline,
1072
- frontmatter: parsed.frontmatter,
1073
- }, true); // skipEmbed: true for performance
1074
- allSlugs.push(slug);
1075
- } catch (err) {
1076
- writeErrors.push(`${slug}: ${err instanceof Error ? err.message : String(err)}`);
1077
- }
1078
- }
1079
-
1080
- if (!jsonOut) {
1081
- spinner.succeed(`Wrote ${allSlugs.length} pages to database`);
1082
- if (writeErrors.length > 0) {
1083
- warning(`${writeErrors.length} pages failed to write`);
1084
- for (const e of writeErrors.slice(0, 3)) {
1085
- subItem(e);
1086
- }
1087
- if (writeErrors.length > 3) {
1088
- subItem(`... and ${writeErrors.length - 3} more`);
1089
- }
1090
- }
1091
- }
1092
-
1093
- // Phase 3: Parallel entity extraction (main optimization)
1094
- const BATCH_SIZE = 10;
1095
- const entityResults = new Map<string, Awaited<ReturnType<typeof extractRelations>>>();
1096
-
1097
- if (settings.llm.baseURL) {
1098
- if (!jsonOut) {
1099
- spinner.start(`Extracting entities with LLM...`);
1100
- }
1101
-
1102
- for (let i = 0; i < fileData.length; i += BATCH_SIZE) {
1103
- const batch = fileData.slice(i, i + BATCH_SIZE);
1104
- if (!jsonOut) {
1105
- spinner.update(`Extracting entities... ${Math.min(i + BATCH_SIZE, fileData.length)}/${fileData.length}`);
1106
- }
1107
- const batchPromises = batch.map(async ({ slug, content }) => {
1108
- const relations = await extractRelations(content, settings.llm);
1109
- return { slug, relations };
1110
- });
1111
- const results = await Promise.all(batchPromises);
1112
- for (const { slug, relations } of results) {
1113
- entityResults.set(slug, relations);
1114
- }
1115
- }
1116
-
1117
- if (!jsonOut) {
1118
- spinner.succeed(`Entity extraction complete`);
1119
- }
1120
- } else {
1121
- if (!jsonOut) {
1122
- warning(`LLM not configured, skipping entity extraction`);
1123
- }
1124
- }
1125
-
1126
- // Phase 4: Write links, tags, timeline, and entity pages
1127
- if (!jsonOut) {
1128
- spinner.start(`Creating links, tags, and timeline entries...`);
1129
- }
1130
-
1131
- let linkCount = 0;
1132
- let timelineCount = 0;
1133
- let entityCount = 0;
1134
- let tagCount = 0;
1135
-
1136
- // Collect timeline entries for batch insert
1137
- const allTimelineEntries: Array<{
1138
- pageSlug: string;
1139
- date: string;
1140
- source: string;
1141
- summary: string;
1142
- detail: string;
1143
- }> = [];
1144
-
1145
- for (const { slug, wikiLinks, timelineEntries, tags, content } of fileData) {
1146
- // Wiki links
1147
- for (const link of wikiLinks) {
1148
- await repo.link(slug, link, "import");
1149
- linkCount++;
1150
- }
1151
-
1152
- // Collect timeline entries for batch insert
1153
- for (const entry of timelineEntries) {
1154
- allTimelineEntries.push({
1155
- pageSlug: slug,
1156
- date: entry.date,
1157
- source: entry.source,
1158
- summary: entry.summary,
1159
- detail: "",
1160
- });
1161
- timelineCount++;
1162
- }
1163
-
1164
- // Tags
1165
- for (const tag of tags) {
1166
- await repo.tag(slug, tag);
1167
- tagCount++;
1168
- }
1169
-
1170
- // Entity links from parallel extraction
1171
- const relations = entityResults.get(slug);
1172
- if (relations && relations.length > 0) {
1173
- const highConfidence = relations.filter(r => r.confidence >= 0.6);
1174
- for (const r of highConfidence) {
1175
- const fromCandidate = entityToSlug(r.from.name, r.from.type);
1176
- const toCandidate = entityToSlug(r.to.name, r.to.type);
1177
- const fromSlug = await repo.findSimilarSlug(fromCandidate, r.from.name);
1178
- const toSlug = await repo.findSimilarSlug(toCandidate, r.to.name);
1179
-
1180
- const c1 = await repo.ensureEntityPage(fromSlug, r.from.type, r.from.name, r.relation, r.context, slug);
1181
- const c2 = await repo.ensureEntityPage(toSlug, r.to.type, r.to.name, r.relation, r.context, slug);
1182
- if (c1) entityCount++;
1183
- if (c2) entityCount++;
1184
-
1185
- await repo.link(fromSlug, toSlug, `[${r.relation}] ${r.context}`);
1186
- await repo.link(slug, fromSlug, `Mentions ${r.from.name}`);
1187
- await repo.link(slug, toSlug, `Mentions ${r.to.name}`);
1188
- linkCount += 3;
1189
- }
1190
- }
1191
- }
1192
-
1193
- // Batch insert all timeline entries
1194
- if (allTimelineEntries.length > 0) {
1195
- await repo.timelineAddBatch(allTimelineEntries);
1196
- }
1197
-
1198
- if (!jsonOut) {
1199
- spinner.succeed(`Created links, tags, and timeline`);
1200
- }
1201
-
1202
- // Phase 5: Batch sync all pages to search index
1203
- if (opts.skipIndex) {
1204
- if (!jsonOut) {
1205
- info(`Skipping vector indexing (--skip-index)`);
1206
- }
1207
- } else {
1208
- if (!jsonOut) {
1209
- spinner.start(`Indexing ${allSlugs.length} pages for search...`);
1210
- }
1211
- await repo.embedAll();
1212
-
1213
- if (!jsonOut) {
1214
- spinner.succeed(`Search indexing complete`);
1215
- }
1216
- }
1217
-
1218
- const duration = formatDuration(Date.now() - startTime);
1219
-
1220
- if (!jsonOut) {
1221
- // Print summary
1222
- header("Import Summary");
1223
- keyValue("Files imported", String(files.length));
1224
- keyValue("Pages created", String(allSlugs.length));
1225
- keyValue("Entities extracted", String(entityCount));
1226
- keyValue("Links created", String(linkCount));
1227
- keyValue("Timeline entries", String(timelineCount));
1228
- keyValue("Tags added", String(tagCount));
1229
- keyValue("Duration", duration);
1230
-
1231
- if (writeErrors.length > 0) {
1232
- warning(`${writeErrors.length} pages had errors`);
1233
- }
1234
- }
1235
-
1236
- print(program, {
1237
- ok: true,
1238
- importedFiles: files.length,
1239
- pages: allSlugs.length,
1240
- links: linkCount,
1241
- timelineEntries: timelineCount,
1242
- entities: entityCount,
1243
- });
1244
- });
1245
- });
1246
-
1247
- program
1248
- .command("export")
1249
- .option("--dir <dir>", "output directory", resolve(process.cwd(), "export"))
1250
- .description("export all pages as markdown files")
1251
- .addHelpText(
1252
- "after",
1253
- `
1254
- Examples:
1255
- ebrain export
1256
- ebrain export --dir ./backup
1257
- `,
1258
- )
1259
- .action(async (opts: { dir: string }) => {
1260
- await withRepo(program, async (repo) => {
1261
- const dir = resolve(opts.dir);
1262
- await ensureDir(dir);
1263
- const pages = await repo.listPages({ limit: 100000 });
1264
- const jsonOut = isJson(program);
1265
- for (let i = 0; i < pages.length; i += 1) {
1266
- const page = pages[i]!;
1267
- progress("export " + page.slug, i + 1, pages.length, jsonOut);
1268
- const tags = await repo.tags(page.slug);
1269
- const fm = {
1270
- ...page.frontmatter,
1271
- type: page.type,
1272
- title: page.title,
1273
- };
1274
- if (tags.length > 0)
1275
- (fm as Record<string, unknown>).tags = tags;
1276
- const md = renderPageMarkdown(fm, page.compiledTruth, page.timeline);
1277
- await writeTextFile(slugToPath(page.slug, dir), md);
1278
- }
1279
- print(program, { exported: pages.length, dir });
1280
- });
1281
- });
1282
-
1283
- // -- ingest ---------------------------------------------------------------
1284
-
1285
- addDryRun(
1286
- program
1287
- .command("ingest")
1288
- .argument("[file]", "file path to ingest (omit for stdin)")
1289
- .option("--type <type>", "source type", "doc")
1290
- .option("--stdin", "read from stdin", false)
1291
- .description("ingest a file as a new page (under ingest/<name>)")
1292
- .addHelpText(
1293
- "after",
1294
- `
1295
- Examples:
1296
- ebrain ingest report.pdf --type pdf
1297
- cat article.md | ebrain ingest --stdin --type article
1298
- ebrain ingest report.pdf --type pdf --dry-run
1299
- `,
1300
- ),
1301
- ).action(
1302
- async (
1303
- file: string | undefined,
1304
- opts: { type?: string; stdin?: boolean; dryRun?: boolean },
1305
- ) => {
1306
- let content: string;
1307
- let fileName: string;
1308
-
1309
- if (file) {
1310
- const fullPath = resolve(file);
1311
- if (!(await fileExists(fullPath))) {
1312
- throw new Error(`file not found: ${file}`);
1313
- }
1314
- content = await readTextFile(fullPath);
1315
- fileName = basename(fullPath);
1316
- } else if (opts.stdin) {
1317
- const raw = await readMaybeStdin();
1318
- if (!raw?.trim()) throw new Error("empty stdin — pipe content");
1319
- content = raw;
1320
- fileName = "stdin";
1321
- } else {
1322
- throw new Error("provide <file> or --stdin");
1323
- }
1324
-
1325
- const slug = `ingest/${fileName.replace(/\.[^.]+$/, "")}`;
1326
- const type = opts.type ?? "doc";
1327
-
1328
- if (isDryRun(opts)) {
1329
- print(program, {
1330
- dryRun: true,
1331
- action: "ingest",
1332
- slug,
1333
- type,
1334
- contentLength: content.length,
1335
- });
1336
- return;
1337
- }
1338
-
1339
- await withRepo(program, async (repo) => {
1340
- const jsonOut = isJson(program);
1341
- const spinner = createSpinner();
1342
- const startTime = Date.now();
1343
-
1344
- if (!jsonOut) {
1345
- header(`Ingest: ${fileName}`);
1346
- spinner.start(`Creating page from file...`);
1347
- }
1348
-
1349
- await repo.putPage({
1350
- slug,
1351
- type,
1352
- title: slugToTitle(slug),
1353
- compiledTruth: content,
1354
- timeline: "",
1355
- frontmatter: {
1356
- sourceFile: resolve(fileName),
1357
- sourceType: type,
1358
- },
1359
- });
1360
-
1361
- if (!jsonOut) {
1362
- spinner.succeed(`Page created: ${slug}`);
1363
- keyValue("Source file", fileName);
1364
- keyValue("Type", type);
1365
- keyValue("Content length", `${content.length} chars`);
1366
- }
1367
-
1368
- await repo.timelineAdd({
1369
- pageSlug: slug,
1370
- date: new Date().toISOString().slice(0, 10),
1371
- source: type,
1372
- summary: `Ingested file ${fileName}`,
1373
- detail: "",
1374
- });
1375
-
1376
- await applyEntityLinks(
1377
- repo,
1378
- slug,
1379
- content,
1380
- jsonOut,
1381
- );
1382
-
1383
- if (!jsonOut) {
1384
- const duration = formatDuration(Date.now() - startTime);
1385
- success(`Ingestion completed in ${duration}`);
1386
- }
1387
-
1388
- print(program, { ok: true, action: "ingest", slug });
1389
- });
1390
- },
1391
- );
1392
-
1393
- // -- embed ----------------------------------------------------------------
1394
-
1395
- addDryRun(
1396
- program
1397
- .command("embed")
1398
- .argument("[slug]", "page slug (omit with --all)")
1399
- .option("--all", "embed all pages")
1400
- .description("refresh page embedding(s)")
1401
- .addHelpText(
1402
- "after",
1403
- `
1404
- Examples:
1405
- ebrain embed docs/api
1406
- ebrain embed --all
1407
- ebrain embed --all --dry-run
1408
- `,
1409
- ),
1410
- ).action(
1411
- async (
1412
- slug: string | undefined,
1413
- opts: { all?: boolean; dryRun?: boolean },
1414
- ) => {
1415
- if (opts.all) {
1416
- if (isDryRun(opts)) {
1417
- await withRepo(program, async (repo) => {
1418
- const pages = await repo.listPages({ limit: 100000 });
1419
- print(program, {
1420
- dryRun: true,
1421
- action: "embed",
1422
- mode: "all",
1423
- pagesFound: pages.length,
1424
- });
1425
- });
1426
- return;
1427
- }
1428
- await withRepo(program, async (repo) => {
1429
- const jsonOut = isJson(program);
1430
- const spinner = createSpinner();
1431
- const startTime = Date.now();
1432
-
1433
- if (!jsonOut) {
1434
- header("Embed All Pages");
1435
- spinner.start(`Loading pages...`);
1436
- }
1437
-
1438
- const pages = await repo.listPages({ limit: 100000 });
1439
-
1440
- if (!jsonOut) {
1441
- spinner.update(`Embedding ${pages.length} pages...`);
1442
- }
1443
-
1444
- const count = await repo.embedAll();
1445
-
1446
- if (!jsonOut) {
1447
- const duration = formatDuration(Date.now() - startTime);
1448
- spinner.succeed(`Embedded ${count} pages`);
1449
- keyValue("Duration", duration);
1450
- }
1451
-
1452
- print(program, { embedded: count, mode: "all" });
1453
- });
1454
- return;
1455
- }
1456
- if (!slug) {
1457
- throw new Error("provide <slug> or --all");
1458
- }
1459
- if (isDryRun(opts)) {
1460
- print(program, { dryRun: true, action: "embed", slug });
1461
- return;
1462
- }
1463
- await withRepo(program, async (repo) => {
1464
- const jsonOut = isJson(program);
1465
- const spinner = createSpinner();
1466
-
1467
- if (!jsonOut) {
1468
- header(`Embed: ${slug}`);
1469
- spinner.start(`Generating embedding for page...`);
1470
- }
1471
-
1472
- await repo.syncPageToSearch(slug);
1473
-
1474
- if (!jsonOut) {
1475
- spinner.succeed(`Page embedded: ${slug}`);
1476
- }
1477
-
1478
- print(program, { embedded: 1, slug });
1479
- });
1480
- },
1481
- );
1482
-
1483
- // -- init / stats ---------------------------------------------------------
1484
-
1485
- program
1486
- .command("init")
1487
- .description("initialize ebrain: create config, database, and show setup guide")
1488
- .addHelpText(
1489
- "after",
1490
- `
1491
- Examples:
1492
- ebrain init
1493
- ebrain init --db ./my.db
1494
- `,
1495
- )
1496
- .action(async () => {
1497
- const jsonOut = isJson(program);
1498
- const settings = await loadSettings();
1499
- const cliDb = program.opts().db;
1500
- const dbPath = cliDb ?? settings.dbPath;
1501
-
1502
- if (!jsonOut) {
1503
- header("ebrain init");
1504
- }
1505
-
1506
- // Step 1: Create settings.json if it doesn't exist
1507
- const { createDefaultSettings } = await import("../settings");
1508
- const settingsCreated = await createDefaultSettings();
1509
-
1510
- if (!jsonOut) {
1511
- if (settingsCreated) {
1512
- success(`Created config: ${SETTINGS_PATH}`);
1513
- } else {
1514
- success(`Config already exists: ${SETTINGS_PATH}`);
1515
- }
1516
- }
1517
-
1518
- // Step 2: Check or initialize database
1519
- const dbExists = await fileExists(dbPath);
1520
- let dbInitialized = false;
1521
-
1522
- if (dbExists) {
1523
- // Database already exists, skip connection attempt to avoid
1524
- // noisy errors (e.g. embedding function key mismatch)
1525
- if (!jsonOut) {
1526
- success(`Database already exists: ${dbPath}`);
1527
- }
1528
- dbInitialized = true;
1529
- } else {
1530
- // Try to create it without collection — embedding config may not be ready
1531
- try {
1532
- const db = await BrainDb.connect(dbPath, settings, { skipCollection: true });
1533
- await db.close();
1534
- await new Promise((r) => setTimeout(r, 200));
1535
- dbInitialized = true;
1536
- if (!jsonOut) {
1537
- success(`Database initialized: ${dbPath}`);
1538
- }
1539
- } catch {
1540
- if (!jsonOut) {
1541
- warning(`Database will be auto-created on first use`);
1542
- }
1543
- }
1544
- }
1545
-
1546
- // Step 3: Show setup guide
1547
- if (!jsonOut) {
1548
- console.log("");
1549
- separator();
1550
- info("Quick Start Guide");
1551
- console.log("");
1552
-
1553
- subItem("1. Configure LLM (for AI queries):", 0);
1554
- subItem(` Edit ${SETTINGS_PATH}`, 4);
1555
- subItem(` Set llm.baseURL to your OpenAI-compatible API endpoint`, 4);
1556
- subItem(` Set llm.apiKey or export DASHSCOPE_API_KEY`, 4);
1557
- console.log("");
1558
-
1559
- subItem("2. Add your first page:", 0);
1560
- subItem(" echo '# Hello' | ebrain put hello --stdin", 4);
1561
- console.log("");
1562
-
1563
- subItem("3. Import a directory of markdown files:", 0);
1564
- subItem(" ebrain import ./docs", 4);
1565
- console.log("");
1566
-
1567
- subItem("4. Query with AI:", 0);
1568
- subItem(' ebrain query "What did we ship in Q4?" --llm', 4);
1569
- console.log("");
1570
-
1571
- subItem("5. Visualize your knowledge graph:", 0);
1572
- subItem(" ebrain graph", 4);
1573
- console.log("");
1574
-
1575
- separator();
1576
- }
1577
-
1578
- print(program, {
1579
- ok: true,
1580
- settingsPath: SETTINGS_PATH,
1581
- settingsCreated,
1582
- dbPath,
1583
- dbInitialized,
1584
- });
1585
-
1586
- process.exit(0);
1587
- });
1588
-
1589
- program
1590
- .command("stats")
1591
- .description("show knowledge base statistics")
1592
- .addHelpText(
1593
- "after",
1594
- `
1595
- Examples:
1596
- ebrain stats
1597
- ebrain stats --json
1598
- `,
1599
- )
1600
- .action(async () => {
1601
- await withRepo(program, async (repo) => {
1602
- const jsonOut = isJson(program);
1603
- const stats = await repo.stats();
1604
-
1605
- if (!jsonOut) {
1606
- header("Knowledge Base Statistics");
1607
- keyValue("Pages", String(stats.pages));
1608
- keyValue("Links", String(stats.links));
1609
- keyValue("Tags", String(stats.tags));
1610
- keyValue("Timeline entries", String(stats.timelineEntries));
1611
- keyValue("Raw data rows", String(stats.rawRows));
1612
- }
1613
-
1614
- print(program, stats);
1615
- });
1616
- });
45
+ // Register commands
46
+ registerConfigCommand(program);
47
+ registerPutCommand(program);
48
+ registerQueryCommand(program);
49
+ registerLinkCommand(program);
50
+ registerTimelineCommand(program);
51
+ registerTagCommand(program);
52
+ registerRawCommand(program);
53
+ registerImportCommand(program);
54
+ registerExportCommand(program);
55
+ registerEmbedCommand(program);
56
+ registerInitCommand(program);
57
+ registerStatsCommand(program);
58
+ registerServeCommand(program);
1617
59
 
1618
60
  // Register compile and smart-ingest commands
1619
61
  registerCompileCommands(program);
@@ -1621,621 +63,15 @@ Examples:
1621
63
  // Register graph command
1622
64
  registerGraphCommand(program);
1623
65
 
1624
- // -- serve / tools-json ---------------------------------------------------
1625
-
1626
- program
1627
- .command("serve")
1628
- .description("start MCP server over stdio (for AI tool integration)")
1629
- .addHelpText(
1630
- "after",
1631
- `
1632
- Examples:
1633
- ebrain serve
1634
- `,
1635
- )
1636
- .action(async () => {
1637
- const { startMcpServer } = await import("../mcp/server");
1638
- const dbPath = String(program.opts().db);
1639
- await startMcpServer(dbPath);
1640
- });
1641
-
66
+ // -- legacy aliases (backward compat) -------------------------------------
1642
67
  program
1643
- .command("tools-json")
1644
- .description("print MCP tools discovery JSON")
68
+ .command("tools")
69
+ .description("alias for tools-json (deprecated)")
1645
70
  .action(() => {
1646
71
  // eslint-disable-next-line @typescript-eslint/no-var-requires
1647
72
  const { TOOL_MANIFEST } = require("../mcp/server");
1648
73
  console.log(JSON.stringify({ tools: TOOL_MANIFEST }, null, 2));
1649
74
  });
1650
75
 
1651
- // -- legacy aliases (backward compat, hidden) -----------------------------
1652
-
1653
-
1654
-
1655
-
1656
-
1657
76
  return program;
1658
77
  }
1659
-
1660
- // ---------------------------------------------------------------------------
1661
- // Repo / output helpers
1662
- // ---------------------------------------------------------------------------
1663
-
1664
- async function withRepo(
1665
- program: Command,
1666
- callback: (repo: BrainRepository) => Promise<void>,
1667
- ): Promise<void> {
1668
- const settings = await loadSettings();
1669
- const cliDb = program.opts().db;
1670
- const dbPath = cliDb ?? settings.dbPath;
1671
- const db = await BrainDb.connect(dbPath, settings);
1672
- const repo = new BrainRepository(db);
1673
- await callback(repo);
1674
-
1675
- // Gracefully close database
1676
- // Note: seekdb SDK's InternalEmbeddedClient.close() is empty in embedded mode
1677
- // Data may not flush properly. Use remote seekdb server for reliability.
1678
- try {
1679
- await db.close();
1680
- } catch (e) {
1681
- // Close may fail due to seekdb native bug
1682
- }
1683
-
1684
- // Give seekdb extra time after close
1685
- await new Promise((r) => setTimeout(r, 500));
1686
-
1687
- // CLI: force exit to bypass seekdb native cleanup segfault
1688
- process.exit(0);
1689
- }
1690
-
1691
- function print(program: Command, payload: unknown): void {
1692
- if (isJson(program)) {
1693
- console.log(JSON.stringify(payload, null, 2));
1694
- return;
1695
- }
1696
- if (typeof payload === "string") {
1697
- console.log(payload);
1698
- return;
1699
- }
1700
- console.log(formatHuman(payload));
1701
- }
1702
-
1703
- function isJson(program: Command): boolean {
1704
- return Boolean(program.opts().json);
1705
- }
1706
-
1707
- function formatHuman(payload: unknown): string {
1708
- if (Array.isArray(payload)) {
1709
- return payload
1710
- .map((item) =>
1711
- typeof item === "string"
1712
- ? `- ${item}`
1713
- : `- ${JSON.stringify(item)}`,
1714
- )
1715
- .join("\n");
1716
- }
1717
- return JSON.stringify(payload, null, 2);
1718
- }
1719
-
1720
- function normalizeLinkSlug(path: string): string {
1721
- return path
1722
- .replaceAll("\\", "/")
1723
- .replace(/^\.\//, "")
1724
- .replace(/^\.\.\//g, "")
1725
- .replace(/\.md$/, "");
1726
- }
1727
-
1728
- // ---------------------------------------------------------------------------
1729
- // LLM Answer Generation — Multi-layer Context Collection
1730
- // ---------------------------------------------------------------------------
1731
-
1732
- /** A single section of context for the LLM prompt. */
1733
- interface ContextSection {
1734
- type: 'primary' | 'raw_data' | 'linked';
1735
- slug: string;
1736
- title: string;
1737
- content: string;
1738
- /** Human-readable label like "原始文档 (crm)" or "关联页面: projects/alpha". */
1739
- label: string;
1740
- }
1741
-
1742
- /**
1743
- * Collect multi-layer context for LLM answer generation.
1744
- *
1745
- * Layers (in priority order):
1746
- * 1. Primary: compiledTruth + timeline of each hit page
1747
- * 2. Raw data: original documents stored via raw.set
1748
- * 3. Linked pages: compiledTruth of pages linked to/from hit pages
1749
- *
1750
- * Budget is enforced via total character limit.
1751
- */
1752
- async function collectContextForLLM(
1753
- repo: BrainRepository,
1754
- hits: Array<{ slug: string; title: string; score: number }>,
1755
- question: string,
1756
- maxChars: number,
1757
- onProgress?: (stage: string) => void,
1758
- ): Promise<{ sections: ContextSection[]; totalChars: number; stats: ContextStats }> {
1759
- const sections: ContextSection[] = [];
1760
- let totalChars = 0;
1761
- const stats: ContextStats = {
1762
- primaryPages: 0,
1763
- rawDocs: 0,
1764
- linkedPages: 0,
1765
- skippedChars: 0,
1766
- };
1767
-
1768
- const seenSlugs = new Set<string>();
1769
-
1770
- function addSection(section: ContextSection): boolean {
1771
- if (seenSlugs.has(`${section.type}:${section.slug}:${section.label}`)) {
1772
- return false;
1773
- }
1774
- const budget = maxChars - totalChars;
1775
- if (section.content.length > budget && sections.length > 0) {
1776
- // Truncate to fit budget
1777
- section.content = section.content.slice(0, budget - 20) + '\n...[truncated]';
1778
- stats.skippedChars += section.content.length - budget;
1779
- }
1780
- if (section.content.length > 0) {
1781
- sections.push(section);
1782
- totalChars += section.content.length;
1783
- seenSlugs.add(`${section.type}:${section.slug}:${section.label}`);
1784
- return true;
1785
- }
1786
- return false;
1787
- }
1788
-
1789
- // Cache pages fetched in Layer 1 to avoid redundant DB calls in Layer 3
1790
- const pageCache = new Map<string, NonNullable<Awaited<ReturnType<typeof repo.getPage>>>>();
1791
-
1792
- // Layer 1: Primary pages (compiledTruth + timeline)
1793
- onProgress?.('page content');
1794
- for (const hit of hits) {
1795
- const page = await repo.getPage(hit.slug);
1796
- if (!page) continue;
1797
- pageCache.set(hit.slug, page);
1798
-
1799
- const parts: string[] = [];
1800
- if (page.compiledTruth?.trim()) {
1801
- parts.push(page.compiledTruth.trim());
1802
- }
1803
- const tl = page.timeline?.trim();
1804
- if (tl) {
1805
- parts.push(`## 时间线\n${tl}`);
1806
- }
1807
-
1808
- if (parts.length > 0) {
1809
- addSection({
1810
- type: 'primary',
1811
- slug: page.slug,
1812
- title: page.title,
1813
- content: parts.join('\n\n'),
1814
- label: `页面正文`,
1815
- });
1816
- stats.primaryPages++;
1817
- }
1818
- }
1819
-
1820
- // Layer 2: Raw data (original documents)
1821
- onProgress?.('raw documents');
1822
- for (const hit of hits) {
1823
- try {
1824
- const rawRows = await repo.readRaw(hit.slug) as Array<{ source: string; data: unknown; fetchedAt?: string }>;
1825
- for (const row of rawRows) {
1826
- let rawContent = '';
1827
- if (typeof row.data === 'string') {
1828
- rawContent = row.data;
1829
- } else if (typeof row.data === 'object' && row.data !== null) {
1830
- rawContent = JSON.stringify(row.data, null, 2);
1831
- }
1832
- if (rawContent.trim()) {
1833
- addSection({
1834
- type: 'raw_data',
1835
- slug: hit.slug,
1836
- title: hit.title,
1837
- content: rawContent,
1838
- label: `原始文档 (${row.source})`,
1839
- });
1840
- stats.rawDocs++;
1841
- }
1842
- }
1843
- } catch {
1844
- // Raw data fetch failure is non-fatal
1845
- }
1846
- }
1847
-
1848
- // Layer 3: Linked pages — score using cached data + keyword matching
1849
- // No second repo.query() call needed — reuse hits scores + keyword fallback
1850
- onProgress?.('linked pages');
1851
- const allLinkedSlugs = new Set<string>();
1852
- for (const hit of hits) {
1853
- try {
1854
- const outLinks = await repo.outgoingLinks(hit.slug);
1855
- outLinks.forEach(l => allLinkedSlugs.add(l.slug));
1856
- } catch { /* ignore */ }
1857
- try {
1858
- const backlinkSlugs = await repo.backlinks(hit.slug);
1859
- backlinkSlugs.forEach(s => allLinkedSlugs.add(s));
1860
- } catch { /* ignore */ }
1861
- }
1862
-
1863
- if (allLinkedSlugs.size > 0) {
1864
- // Score: use semantic scores from initial hits (already cached), keyword for rest
1865
- const semanticScoreMap = new Map(hits.map(h => [h.slug, h.score]));
1866
- const keywordScores = new Map<string, number>();
1867
- for (const linkedSlug of allLinkedSlugs) {
1868
- if (semanticScoreMap.has(linkedSlug)) continue;
1869
- // Use cached page if available, only fetch if not in cache
1870
- const cached = pageCache.get(linkedSlug);
1871
- if (cached) {
1872
- const text = `${cached.title} ${cached.compiledTruth}`.slice(0, 2000);
1873
- keywordScores.set(linkedSlug, computeKeywordRelevance(text, question));
1874
- } else {
1875
- const page = await repo.getPage(linkedSlug);
1876
- if (page) {
1877
- pageCache.set(linkedSlug, page);
1878
- const text = `${page.title} ${page.compiledTruth}`.slice(0, 2000);
1879
- keywordScores.set(linkedSlug, computeKeywordRelevance(text, question));
1880
- }
1881
- }
1882
- }
1883
-
1884
- // Combine scores
1885
- const scoredLinked = [...allLinkedSlugs].map(slug => ({
1886
- slug,
1887
- score: semanticScoreMap.get(slug) ?? keywordScores.get(slug) ?? 0,
1888
- }));
1889
-
1890
- // Filter: only include linked pages with meaningful relevance
1891
- const MIN_LINKED_SCORE = 0.02;
1892
- const relevantLinked = scoredLinked
1893
- .filter(s => s.score >= MIN_LINKED_SCORE)
1894
- .sort((a, b) => b.score - a.score);
1895
-
1896
- // Add linked pages (already cached in pageCache, no extra fetch needed)
1897
- for (const linked of relevantLinked) {
1898
- if (totalChars >= maxChars) break;
1899
-
1900
- const linkedPage = pageCache.get(linked.slug);
1901
- if (!linkedPage || !linkedPage.compiledTruth?.trim()) continue;
1902
-
1903
- const remaining = maxChars - totalChars;
1904
- let content = linkedPage.compiledTruth.trim();
1905
- if (content.length > remaining - 100) {
1906
- content = content.slice(0, remaining - 100) + '\n...[truncated]';
1907
- }
1908
-
1909
- addSection({
1910
- type: 'linked',
1911
- slug: linkedPage.slug,
1912
- title: linkedPage.title,
1913
- content,
1914
- label: `关联页面: ${linkedPage.slug} (相关度: ${(linked.score * 100).toFixed(1)}%)`,
1915
- });
1916
- stats.linkedPages++;
1917
-
1918
- // Also fetch raw data for highly relevant linked pages
1919
- if (linked.score > 0.1) {
1920
- try {
1921
- const rawRows = await repo.readRaw(linked.slug) as Array<{ source: string; data: unknown }>;
1922
- for (const row of rawRows) {
1923
- let rawContent = typeof row.data === 'string' ? row.data : JSON.stringify(row.data);
1924
- if (rawContent.trim().length > 100) {
1925
- const remaining2 = maxChars - totalChars;
1926
- if (rawContent.length > remaining2 - 100) {
1927
- rawContent = rawContent.slice(0, remaining2 - 100) + '\n...[truncated]';
1928
- }
1929
- addSection({
1930
- type: 'raw_data',
1931
- slug: linked.slug,
1932
- title: linkedPage.title,
1933
- content: rawContent,
1934
- label: `原始文档 (关联: ${row.source})`,
1935
- });
1936
- stats.rawDocs++;
1937
- }
1938
- }
1939
- } catch { /* ignore */ }
1940
- }
1941
- }
1942
- }
1943
-
1944
- return { sections, totalChars, stats };
1945
- }
1946
-
1947
- /**
1948
- * Simple keyword-based relevance scoring (fallback for pages without embeddings).
1949
- * Computes the fraction of unique meaningful characters from the question
1950
- * that appear in the text.
1951
- */
1952
- function computeKeywordRelevance(text: string, question: string): number {
1953
- const STOP_CHARS = new Set('的是了在和我有你就这不人都说上个大国为到以们年会生地要主中子自实家小对多能好可很所把当');
1954
- const questionChars = [...question]
1955
- .filter(c => !/\s|[,,。!?、;::""''()()【】\[\]{}<>\/\\|~`@#$%^&*+=_-]/.test(c) && !STOP_CHARS.has(c));
1956
- if (questionChars.length === 0) return 0;
1957
-
1958
- const uniqueChars = new Set(questionChars);
1959
- const lower = text.toLowerCase();
1960
- let matched = 0;
1961
- for (const char of uniqueChars) {
1962
- if (lower.includes(char.toLowerCase())) matched++;
1963
- }
1964
- return matched / uniqueChars.size;
1965
- }
1966
-
1967
- interface ContextStats {
1968
- primaryPages: number;
1969
- rawDocs: number;
1970
- linkedPages: number;
1971
- skippedChars: number;
1972
- }
1973
-
1974
- /**
1975
- * Build LLM prompt from collected context sections and generate answer.
1976
- */
1977
- async function generateAnswerWithStream(
1978
- question: string,
1979
- sections: ContextSection[],
1980
- stats: ContextStats,
1981
- llm: ResolvedLLM,
1982
- ): Promise<{ answer: string; ok: boolean }> {
1983
- const apiKey = llm.apiKey || process.env[llm.apiKeyEnv] || "";
1984
- if (!apiKey) {
1985
- return { answer: "Error: LLM API key not configured.", ok: false };
1986
- }
1987
-
1988
- if (sections.length === 0) {
1989
- return { answer: "知识库中没有找到相关内容。", ok: true };
1990
- }
1991
-
1992
- // Build context sections with clear labels
1993
- const contextParts: string[] = [];
1994
- let sectionIndex = 0;
1995
-
1996
- // Group by type for cleaner output
1997
- const primarySections = sections.filter(s => s.type === 'primary');
1998
- const rawSections = sections.filter(s => s.type === 'raw_data');
1999
- const linkedSections = sections.filter(s => s.type === 'linked');
2000
-
2001
- function renderSections(group: ContextSection[], header: string) {
2002
- if (group.length === 0) return;
2003
- contextParts.push(`## ${header}\n`);
2004
- for (const s of group) {
2005
- sectionIndex++;
2006
- contextParts.push(`### [${sectionIndex}] ${s.title} — ${s.label}\n**Slug:** ${s.slug}\n\n${s.content}\n`);
2007
- }
2008
- contextParts.push('');
2009
- }
2010
-
2011
- renderSections(primarySections, '页面正文');
2012
- renderSections(rawSections, '原始文档');
2013
- renderSections(linkedSections, '关联页面');
2014
-
2015
- const context = contextParts.join('\n');
2016
-
2017
- const prompt = `你是一个知识库助手,请根据提供的知识库内容回答问题。
2018
-
2019
- ## 问题
2020
- ${question}
2021
-
2022
- ## 知识库内容
2023
-
2024
- ${context}
2025
-
2026
- ## 回答要求
2027
- - 仅基于提供的知识库内容回答,不要编造信息
2028
- - 如果知识库中没有相关信息,请明确说明
2029
- - 引用来源时使用 [[slug|标题]] 的格式
2030
- - 使用清晰的 markdown 格式
2031
- - 如果涉及时间线信息,请在回答中体现
2032
- - 区分哪些信息来自「页面正文」、哪些来自「原始文档」、哪些来自「关联页面」
2033
- - 语言与提问保持一致(中文提问用中文回答,英文提问用英文回答)
2034
-
2035
- ## 回答`;
2036
-
2037
- // Disable thinking/reasoning mode to reduce latency
2038
- const disableThinking: Record<string, unknown> = {};
2039
- // OpenAI/compatible: extra_body for thinking disable
2040
- // DeepSeek: use extra_body to disable thinking
2041
- // Many providers ignore unknown fields, so this is safe to always include
2042
- const extraBody: Record<string, unknown> = {
2043
- thinking: { type: "disabled" },
2044
- };
2045
-
2046
- try {
2047
- const url = llm.baseURL.endsWith("/") ? llm.baseURL + "chat/completions" : llm.baseURL + "/chat/completions";
2048
-
2049
- // Show thinking indicator while waiting for first token
2050
- process.stderr.write(`\x1b[35m💭\x1b[0m \x1b[2mConnecting to ${llm.model}...\x1b[0m\n`);
2051
-
2052
- const resp = await fetch(
2053
- url,
2054
- {
2055
- method: "POST",
2056
- headers: {
2057
- "Content-Type": "application/json",
2058
- Authorization: `Bearer ${apiKey}`,
2059
- },
2060
- body: JSON.stringify({
2061
- model: llm.model,
2062
- stream: true,
2063
- messages: [
2064
- {
2065
- role: "system",
2066
- content: "你是一个专业的知识库助手,基于提供的知识库内容准确回答问题。引用来源时使用 [[slug|标题]] 格式。回答要条理清晰,区分信息来源。",
2067
- },
2068
- { role: "user", content: prompt },
2069
- ],
2070
- temperature: 0.3,
2071
- max_tokens: 4096,
2072
- ...disableThinking,
2073
- extra_body: extraBody,
2074
- // Also send thinking disable as top-level for providers that support it
2075
- thinking: { type: "disabled" },
2076
- }),
2077
- // Abort if no response within 30s
2078
- signal: AbortSignal.timeout(30_000),
2079
- },
2080
- );
2081
-
2082
- if (!resp.ok) {
2083
- const text = await resp.text();
2084
- // Clear the thinking indicator line
2085
- process.stderr.write("\r\x1b[K");
2086
- return { answer: `Error: LLM API failed (${resp.status}): ${text.slice(0, 200)}`, ok: false };
2087
- }
2088
-
2089
- if (!resp.body) {
2090
- process.stderr.write("\r\x1b[K");
2091
- return { answer: "Error: No response body from LLM API.", ok: false };
2092
- }
2093
-
2094
- // Clear thinking indicator, show streaming status
2095
- process.stderr.write("\r\x1b[K");
2096
- process.stderr.write(`\x1b[32m✦\x1b[0m \x1b[2mStreaming response...\x1b[0m\n`);
2097
-
2098
- // Stream the response
2099
- const reader = resp.body.getReader();
2100
- const decoder = new TextDecoder();
2101
- let fullAnswer = "";
2102
- let buffer = "";
2103
- let tokenCount = 0;
2104
-
2105
- while (true) {
2106
- const { done, value } = await reader.read();
2107
- if (done) break;
2108
-
2109
- buffer += decoder.decode(value, { stream: true });
2110
- const lines = buffer.split("\n");
2111
- // Keep the last incomplete line in buffer
2112
- buffer = lines.pop() || "";
2113
-
2114
- for (const line of lines) {
2115
- const trimmed = line.trim();
2116
- if (!trimmed || trimmed === "data: [DONE]") continue;
2117
- if (!trimmed.startsWith("data: ")) continue;
2118
-
2119
- try {
2120
- const json = JSON.parse(trimmed.slice(6));
2121
- const content = json.choices?.[0]?.delta?.content;
2122
- if (content) {
2123
- process.stdout.write(content);
2124
- fullAnswer += content;
2125
- tokenCount++;
2126
- }
2127
- } catch {
2128
- // Skip malformed SSE data
2129
- }
2130
- }
2131
- }
2132
-
2133
- // Add a newline after streaming completes
2134
- process.stdout.write("\n");
2135
-
2136
- return { answer: fullAnswer || "(No answer generated)", ok: true };
2137
- } catch (error) {
2138
- const msg = error instanceof Error ? error.message : String(error);
2139
- return { answer: `Error: ${msg}`, ok: false };
2140
- }
2141
- }
2142
-
2143
- /**
2144
- * @deprecated Use generateAnswerWithStream instead
2145
- */
2146
- async function generateAnswerWithContext(
2147
- question: string,
2148
- sections: ContextSection[],
2149
- stats: ContextStats,
2150
- llm: ResolvedLLM,
2151
- ): Promise<string> {
2152
- const apiKey = llm.apiKey || process.env[llm.apiKeyEnv] || "";
2153
- if (!apiKey) {
2154
- return "Error: LLM API key not configured.";
2155
- }
2156
-
2157
- if (sections.length === 0) {
2158
- return "知识库中没有找到相关内容。";
2159
- }
2160
-
2161
- // Build context sections with clear labels
2162
- const contextParts: string[] = [];
2163
- let sectionIndex = 0;
2164
-
2165
- // Group by type for cleaner output
2166
- const primarySections = sections.filter(s => s.type === 'primary');
2167
- const rawSections = sections.filter(s => s.type === 'raw_data');
2168
- const linkedSections = sections.filter(s => s.type === 'linked');
2169
-
2170
- function renderSections(group: ContextSection[], header: string) {
2171
- if (group.length === 0) return;
2172
- contextParts.push(`## ${header}\n`);
2173
- for (const s of group) {
2174
- sectionIndex++;
2175
- contextParts.push(`### [${sectionIndex}] ${s.title} — ${s.label}\n**Slug:** ${s.slug}\n\n${s.content}\n`);
2176
- }
2177
- contextParts.push('');
2178
- }
2179
-
2180
- renderSections(primarySections, '页面正文');
2181
- renderSections(rawSections, '原始文档');
2182
- renderSections(linkedSections, '关联页面');
2183
-
2184
- const context = contextParts.join('\n');
2185
-
2186
- const prompt = `你是一个知识库助手,请根据提供的知识库内容回答问题。
2187
-
2188
- ## 问题
2189
- ${question}
2190
-
2191
- ## 知识库内容
2192
-
2193
- ${context}
2194
-
2195
- ## 回答要求
2196
- - 仅基于提供的知识库内容回答,不要编造信息
2197
- - 如果知识库中没有相关信息,请明确说明
2198
- - 引用来源时使用 [[slug|标题]] 的格式
2199
- - 使用清晰的 markdown 格式
2200
- - 如果涉及时间线信息,请在回答中体现
2201
- - 区分哪些信息来自「页面正文」、哪些来自「原始文档」、哪些来自「关联页面」
2202
- - 语言与提问保持一致(中文提问用中文回答,英文提问用英文回答)
2203
-
2204
- ## 回答`;
2205
-
2206
- try {
2207
- const resp = await fetch(
2208
- llm.baseURL.endsWith("/") ? llm.baseURL + "chat/completions" : llm.baseURL + "/chat/completions",
2209
- {
2210
- method: "POST",
2211
- headers: {
2212
- "Content-Type": "application/json",
2213
- Authorization: `Bearer ${apiKey}`,
2214
- },
2215
- body: JSON.stringify({
2216
- model: llm.model,
2217
- messages: [
2218
- {
2219
- role: "system",
2220
- content: "你是一个专业的知识库助手,基于提供的知识库内容准确回答问题。引用来源时使用 [[slug|标题]] 格式。回答要条理清晰,区分信息来源。",
2221
- },
2222
- { role: "user", content: prompt },
2223
- ],
2224
- temperature: 0.3,
2225
- max_tokens: 4096,
2226
- }),
2227
- },
2228
- );
2229
-
2230
- if (!resp.ok) {
2231
- const text = await resp.text();
2232
- return `Error: LLM API failed (${resp.status}): ${text.slice(0, 200)}`;
2233
- }
2234
-
2235
- const data = await resp.json();
2236
- return data.choices?.[0]?.message?.content || "(No answer generated)";
2237
- } catch (error) {
2238
- const msg = error instanceof Error ? error.message : String(error);
2239
- return `Error: ${msg}`;
2240
- }
2241
- }