@jiangxiaosheng/digital-agent-platform 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/README.md +37 -0
  2. package/eslint.config.mjs +20 -0
  3. package/messages/en.json +184 -0
  4. package/messages/zh.json +184 -0
  5. package/middleware.ts +9 -0
  6. package/next-env.d.ts +5 -0
  7. package/next.config.ts +16 -0
  8. package/package.json +59 -0
  9. package/postcss.config.mjs +5 -0
  10. package/src/app/[locale]/agents/[id]/chat/page.tsx +833 -0
  11. package/src/app/[locale]/agents/[id]/edit/page.tsx +58 -0
  12. package/src/app/[locale]/agents/new/page.tsx +52 -0
  13. package/src/app/[locale]/agents/page.tsx +122 -0
  14. package/src/app/[locale]/knowledge/page.tsx +305 -0
  15. package/src/app/[locale]/layout.tsx +46 -0
  16. package/src/app/[locale]/page.tsx +6 -0
  17. package/src/app/[locale]/settings/page.tsx +1204 -0
  18. package/src/app/api/agents/[id]/route.ts +27 -0
  19. package/src/app/api/agents/route.ts +27 -0
  20. package/src/app/api/auth-config/test/route.ts +69 -0
  21. package/src/app/api/bash-output/route.ts +57 -0
  22. package/src/app/api/chat/[agentId]/route.ts +164 -0
  23. package/src/app/api/env-vars/[key]/route.ts +22 -0
  24. package/src/app/api/env-vars/route.ts +26 -0
  25. package/src/app/api/env-vars/scan/route.ts +66 -0
  26. package/src/app/api/file-serve/route.ts +58 -0
  27. package/src/app/api/fs/browse/route.ts +47 -0
  28. package/src/app/api/knowledge/[id]/documents/[docId]/route.ts +22 -0
  29. package/src/app/api/knowledge/[id]/route.ts +21 -0
  30. package/src/app/api/knowledge/[id]/upload/route.ts +90 -0
  31. package/src/app/api/knowledge/route.ts +15 -0
  32. package/src/app/api/knowledge/search/route.ts +26 -0
  33. package/src/app/api/models/route.ts +160 -0
  34. package/src/app/api/models-config/route.ts +41 -0
  35. package/src/app/api/models-config/test/route.ts +84 -0
  36. package/src/app/api/sessions/[agentId]/[sessionId]/route.ts +12 -0
  37. package/src/app/api/sessions/[agentId]/route.ts +19 -0
  38. package/src/app/api/sessions/history/route.ts +166 -0
  39. package/src/app/api/settings/route.ts +45 -0
  40. package/src/app/api/skills/[name]/route.ts +40 -0
  41. package/src/app/api/skills/install/route.ts +35 -0
  42. package/src/app/api/skills/route.ts +54 -0
  43. package/src/app/api/skills/search/route.ts +60 -0
  44. package/src/app/api/tts/route.ts +235 -0
  45. package/src/app/api/voice-provider/route.ts +98 -0
  46. package/src/app/api/workspace-files/route.ts +151 -0
  47. package/src/app/globals.css +176 -0
  48. package/src/app/layout.tsx +13 -0
  49. package/src/app/page.tsx +6 -0
  50. package/src/components/agents/agent-form.tsx +457 -0
  51. package/src/components/agents/model-selector.tsx +290 -0
  52. package/src/components/agents/skills-manager.tsx +347 -0
  53. package/src/components/chat/input-bar.tsx +561 -0
  54. package/src/components/chat/message-bubble.tsx +395 -0
  55. package/src/components/layout/sidebar.tsx +304 -0
  56. package/src/components/providers.tsx +19 -0
  57. package/src/components/ui/badge.tsx +26 -0
  58. package/src/components/ui/button.tsx +43 -0
  59. package/src/components/ui/card.tsx +46 -0
  60. package/src/components/ui/dialog.tsx +74 -0
  61. package/src/components/ui/directory-picker.tsx +185 -0
  62. package/src/components/ui/input.tsx +18 -0
  63. package/src/components/ui/label.tsx +16 -0
  64. package/src/components/ui/scroll-area.tsx +41 -0
  65. package/src/components/ui/select.tsx +93 -0
  66. package/src/components/ui/slider.tsx +35 -0
  67. package/src/components/ui/switch.tsx +35 -0
  68. package/src/components/ui/tabs.tsx +48 -0
  69. package/src/components/ui/textarea.tsx +17 -0
  70. package/src/components/ui/toaster.tsx +50 -0
  71. package/src/hooks/use-toast.ts +43 -0
  72. package/src/i18n/request.ts +13 -0
  73. package/src/i18n/routing.ts +7 -0
  74. package/src/lib/agent-factory.ts +200 -0
  75. package/src/lib/agents/store.ts +95 -0
  76. package/src/lib/db/client.ts +25 -0
  77. package/src/lib/db/schema.ts +66 -0
  78. package/src/lib/env-vars.ts +33 -0
  79. package/src/lib/knowledge/chunker.ts +34 -0
  80. package/src/lib/knowledge/embedder.ts +48 -0
  81. package/src/lib/knowledge/retriever.ts +94 -0
  82. package/src/lib/knowledge/store.ts +74 -0
  83. package/src/lib/models-config.ts +55 -0
  84. package/src/lib/settings.ts +33 -0
  85. package/src/lib/skills.ts +79 -0
  86. package/src/lib/system-prompt.ts +52 -0
  87. package/src/lib/tool-builder.ts +19 -0
  88. package/src/lib/utils.ts +6 -0
  89. package/src/lib/voice-providers-config.ts +71 -0
  90. package/src/middleware.ts +9 -0
  91. package/src/types/agent.ts +124 -0
  92. package/src/types/knowledge.ts +39 -0
  93. package/src/types/settings.ts +31 -0
  94. package/tsconfig.json +40 -0
@@ -0,0 +1,200 @@
1
+ // Core factory: dynamically builds a pi-coding-agent AgentSession
2
+ // with per-agent persona, model, tools, workspace and RAG context.
3
+ import type { DigitalAgent } from "@/types/agent";
4
+ import { buildSystemPrompt } from "./system-prompt";
5
+ import { buildToolConfig } from "./tool-builder";
6
+ import { searchChunks } from "./knowledge/retriever";
7
+ import { embedText } from "./knowledge/embedder";
8
+ import path from "path";
9
+ import os from "os";
10
+ import fs from "fs";
11
+
12
+ export function getAgentDir(): string {
13
+ return process.env.PI_AGENT_DIR ?? path.join(os.homedir(), ".pi", "agent");
14
+ }
15
+
16
+ export function getSessionDir(agentId: string): string {
17
+ return path.join(os.homedir(), ".pi", "agent", "sessions", `agent-${agentId}`);
18
+ }
19
+
20
+ /** 解析数字人工作区路径,空白时使用 agentDir */
21
+ export function resolveWorkspace(agent: DigitalAgent): string {
22
+ const wp = agent.workspacePath?.trim();
23
+ if (wp) {
24
+ const expanded = wp.replace(/^~/, os.homedir());
25
+ // 目录不存在时自动创建,而不是静默回退
26
+ fs.mkdirSync(expanded, { recursive: true });
27
+ return expanded;
28
+ }
29
+ // 未配置时使用数字人专属默认目录
30
+ const agentWorkspace = path.join(getAgentDir(), "workspaces", `agent-${agent.id}`);
31
+ fs.mkdirSync(agentWorkspace, { recursive: true });
32
+ return agentWorkspace;
33
+ }
34
+
35
+ export async function buildRagContext(
36
+ agent: DigitalAgent,
37
+ userMessage: string
38
+ ): Promise<string> {
39
+ if (!agent.knowledgeBases.length) return "";
40
+ const queryVec = await embedText(userMessage);
41
+ const results = await searchChunks(agent.knowledgeBases, queryVec, 5);
42
+ if (!results.length) return "";
43
+ return results
44
+ .map((r, i) => `[${i + 1}] (来源: ${r.chunk.metadata.filename})\n${r.chunk.content}`)
45
+ .join("\n\n---\n\n");
46
+ }
47
+
48
+ // Returns a ready-to-use AgentSession — caller must call session.dispose() when done.
49
+ export async function createSession(
50
+ agent: DigitalAgent,
51
+ userMessage: string,
52
+ existingSessionFile?: string
53
+ ) {
54
+ const {
55
+ createAgentSession,
56
+ ModelRuntime,
57
+ SessionManager,
58
+ DefaultResourceLoader,
59
+ defineTool,
60
+ } = await import("@earendil-works/pi-coding-agent");
61
+
62
+ // 1. RAG
63
+ const ragContext = await buildRagContext(agent, userMessage);
64
+
65
+ // 2. System prompt with agent identity
66
+ const systemPrompt = buildSystemPrompt(agent, ragContext);
67
+
68
+ // 3. Model runtime — load both auth.json (built-in keys) and models.json (custom providers)
69
+ const modelRuntime = await ModelRuntime.create({
70
+ authPath: path.join(getAgentDir(), "auth.json"),
71
+ modelsStorePath: path.join(getAgentDir(), "models.json"),
72
+ });
73
+
74
+ // Also inject keys saved via our settings page (SQLite store)
75
+ // This allows keys entered in Settings to override/supplement auth.json
76
+ const { getSettings } = await import("./settings");
77
+ const settings = getSettings();
78
+ const keyMap: Record<string, string | undefined> = {
79
+ anthropic: settings.apiKeys.anthropic,
80
+ openai: settings.apiKeys.openai,
81
+ google: settings.apiKeys.google,
82
+ };
83
+ for (const [provider, key] of Object.entries(keyMap)) {
84
+ if (key) await modelRuntime.setRuntimeApiKey(provider, key);
85
+ }
86
+
87
+ // Resolve model — throw with helpful message if not found
88
+ const model = modelRuntime.getModel(agent.model.provider, agent.model.modelId);
89
+ if (!model) {
90
+ const available = modelRuntime.getModels().slice(0, 5).map((m) => `${m.provider}/${m.id}`);
91
+ throw new Error(
92
+ `Model "${agent.model.provider}/${agent.model.modelId}" not found. ` +
93
+ `Check that the API key for "${agent.model.provider}" is configured in Settings. ` +
94
+ `Available models (first 5): ${available.join(", ")}`
95
+ );
96
+ }
97
+
98
+ // 4. Workspace (cwd) — use agent's own workspace, not process.cwd()
99
+ const cwd = resolveWorkspace(agent);
100
+
101
+ // 5. Load global env vars
102
+ const { readEnvVars } = await import("./env-vars");
103
+ const globalEnv = readEnvVars();
104
+
105
+ // Inject into process.env so ALL tools and skills can read them.
106
+ // spawnHook only covers the bash tool; skills (tavily-search, edge-tts, etc.)
107
+ // spawn subprocesses that inherit directly from process.env.
108
+ // These are user-configured API keys/config that are safe to set globally.
109
+ for (const [k, v] of Object.entries(globalEnv)) {
110
+ process.env[k] = v;
111
+ }
112
+
113
+ // 6. Tools
114
+ // env vars are already in process.env (injected above), so no need for spawnHook.
115
+ // Keep bash as a SDK builtin so the model recognises it correctly.
116
+ const { builtinNames, customTools: customToolDefs } = buildToolConfig(agent);
117
+ const customTools = [] as ReturnType<typeof defineTool>[];
118
+
119
+ // Calculator (custom tool)
120
+ if (customToolDefs.find((t) => t.name === "calculator")) {
121
+ const { Type } = await import("@sinclair/typebox");
122
+ customTools.push(
123
+ defineTool({
124
+ name: "calculator",
125
+ label: "Calculator",
126
+ description: "Evaluate a mathematical expression and return the result.",
127
+ parameters: Type.Object({ expression: Type.String({ description: "Math expression" }) }),
128
+ execute: async (_id: string, { expression }: { expression: string }) => {
129
+ try {
130
+ const result = Function(`"use strict"; return (${expression})`)();
131
+ return { content: [{ type: "text" as const, text: String(result) }], details: {} };
132
+ } catch (e) {
133
+ return { content: [{ type: "text" as const, text: `Error: ${String(e)}` }], details: {} };
134
+ }
135
+ },
136
+ })
137
+ );
138
+ }
139
+
140
+ // 6. Session manager — use agent-specific session directory
141
+ const sessionMgr = existingSessionFile
142
+ ? SessionManager.open(existingSessionFile)
143
+ : SessionManager.create(cwd, getSessionDir(agent.id));
144
+
145
+ // 7. Resource loader — two-phase: first discover filtered skills, then build final loader
146
+ // enabledSkills 为空 = 不限制(加载全部已安装 skills)
147
+ // enabledSkills 有值 = 白名单过滤
148
+ const enabledSkills = new Set(agent.enabledSkills ?? []);
149
+ const skillFilter = (base: { skills: { name: string }[]; diagnostics: unknown[] }) => ({
150
+ skills: enabledSkills.size > 0
151
+ ? base.skills.filter((s) => enabledSkills.has(s.name))
152
+ : base.skills, // 空列表 = 不过滤,保留全部可用 skills
153
+ diagnostics: base.diagnostics,
154
+ });
155
+
156
+ // Phase 1: discover which skills are actually available after filter
157
+ const { formatSkillsForPrompt } = await import("@earendil-works/pi-coding-agent");
158
+ const discoveryLoader = new DefaultResourceLoader({
159
+ cwd,
160
+ agentDir: getAgentDir(),
161
+ skillsOverride: skillFilter as never,
162
+ });
163
+ await discoveryLoader.reload();
164
+ const { skills: filteredSkills } = discoveryLoader.getSkills();
165
+
166
+ // Phase 2: build final loader — systemPrompt + skills appended + skill filter
167
+ const skillsSection = filteredSkills.length > 0
168
+ ? formatSkillsForPrompt(filteredSkills)
169
+ : "";
170
+
171
+ const resourceLoader = new DefaultResourceLoader({
172
+ cwd,
173
+ agentDir: getAgentDir(),
174
+ systemPromptOverride: () => systemPrompt,
175
+ appendSystemPromptOverride: skillsSection ? () => [skillsSection] : undefined,
176
+ skillsOverride: skillFilter as never,
177
+ });
178
+
179
+ // ⚠️ REQUIRED: reload() initialises internal caches incl. systemPromptOverride
180
+ await resourceLoader.reload();
181
+
182
+ // 8. Create session
183
+ // bash is a SDK builtin — env vars flow via process.env set above
184
+ const allCustomTools = customTools.length > 0 ? customTools : undefined;
185
+ const hasAnyTools = builtinNames.length > 0 || (allCustomTools?.length ?? 0) > 0;
186
+
187
+ const { session } = await createAgentSession({
188
+ cwd,
189
+ modelRuntime,
190
+ model,
191
+ thinkingLevel: agent.model.thinkingLevel as Parameters<typeof createAgentSession>[0] extends { thinkingLevel?: infer T } ? T : never,
192
+ tools: builtinNames.length > 0 ? builtinNames : undefined,
193
+ noTools: !hasAnyTools ? "all" : undefined,
194
+ customTools: allCustomTools,
195
+ sessionManager: sessionMgr,
196
+ resourceLoader,
197
+ });
198
+
199
+ return session;
200
+ }
@@ -0,0 +1,95 @@
1
+ import { getDb } from "../db/client";
2
+ import type { DigitalAgent, CreateAgentInput, UpdateAgentInput } from "@/types/agent";
3
+ import { v4 as uuidv4 } from "uuid";
4
+
5
+ type AgentRow = { id: string; config: string; created_at: number; updated_at: number };
6
+
7
+ function rowToAgent(row: AgentRow): DigitalAgent {
8
+ const cfg = JSON.parse(row.config);
9
+ return {
10
+ ...cfg,
11
+ id: row.id,
12
+ createdAt: new Date(row.created_at).toISOString(),
13
+ updatedAt: new Date(row.updated_at).toISOString(),
14
+ };
15
+ }
16
+
17
+ export const agentStore = {
18
+ list(): DigitalAgent[] {
19
+ const db = getDb();
20
+ const rows = db.prepare("SELECT * FROM agents ORDER BY created_at DESC").all() as AgentRow[];
21
+ return rows.map(rowToAgent);
22
+ },
23
+
24
+ getById(id: string): DigitalAgent | undefined {
25
+ const db = getDb();
26
+ const row = db.prepare("SELECT * FROM agents WHERE id = ?").get(id) as AgentRow | undefined;
27
+ return row ? rowToAgent(row) : undefined;
28
+ },
29
+
30
+ create(input: CreateAgentInput): DigitalAgent {
31
+ const db = getDb();
32
+ const id = uuidv4();
33
+ const now = Date.now();
34
+ const cfg = { ...input };
35
+ db.prepare("INSERT INTO agents (id, config, created_at, updated_at) VALUES (?, ?, ?, ?)").run(
36
+ id, JSON.stringify(cfg), now, now
37
+ );
38
+ return rowToAgent({ id, config: JSON.stringify(cfg), created_at: now, updated_at: now });
39
+ },
40
+
41
+ update(id: string, input: UpdateAgentInput): DigitalAgent | undefined {
42
+ const db = getDb();
43
+ const existing = this.getById(id);
44
+ if (!existing) return undefined;
45
+ const now = Date.now();
46
+ const merged = { ...existing, ...input, id: existing.id };
47
+ const { createdAt, updatedAt, ...cfg } = merged;
48
+ void createdAt; void updatedAt;
49
+ db.prepare("UPDATE agents SET config = ?, updated_at = ? WHERE id = ?").run(
50
+ JSON.stringify(cfg), now, id
51
+ );
52
+ return this.getById(id);
53
+ },
54
+
55
+ delete(id: string): boolean {
56
+ const db = getDb();
57
+ const result = db.prepare("DELETE FROM agents WHERE id = ?").run(id);
58
+ return result.changes > 0;
59
+ },
60
+ };
61
+
62
+ export const sessionStore = {
63
+ list(agentId: string) {
64
+ const db = getDb();
65
+ return db
66
+ .prepare("SELECT * FROM sessions WHERE agent_id = ? ORDER BY created_at DESC")
67
+ .all(agentId) as { id: string; agent_id: string; title: string; session_file: string; created_at: number }[];
68
+ },
69
+
70
+ create(agentId: string, sessionFile: string, title?: string): string {
71
+ const db = getDb();
72
+ const id = uuidv4();
73
+ db.prepare("INSERT INTO sessions (id, agent_id, title, session_file, created_at) VALUES (?, ?, ?, ?, ?)").run(
74
+ id, agentId, title ?? "New Session", sessionFile, Date.now()
75
+ );
76
+ return id;
77
+ },
78
+
79
+ updateFile(id: string, sessionFile: string): void {
80
+ const db = getDb();
81
+ db.prepare("UPDATE sessions SET session_file = ? WHERE id = ?").run(sessionFile, id);
82
+ },
83
+
84
+ delete(id: string): boolean {
85
+ const db = getDb();
86
+ const result = db.prepare("DELETE FROM sessions WHERE id = ?").run(id);
87
+ return result.changes > 0;
88
+ },
89
+
90
+ deleteByAgent(agentId: string): number {
91
+ const db = getDb();
92
+ const result = db.prepare("DELETE FROM sessions WHERE agent_id = ?").run(agentId);
93
+ return Number(result.changes);
94
+ },
95
+ };
@@ -0,0 +1,25 @@
1
+ import { DatabaseSync } from "node:sqlite";
2
+ import path from "path";
3
+ import os from "os";
4
+ import fs from "fs";
5
+
6
+ let _db: DatabaseSync | null = null;
7
+
8
+ export function getStorageDir(): string {
9
+ const custom = process.env.STORAGE_PATH;
10
+ if (custom) return custom;
11
+ return path.join(os.homedir(), ".digital-agents");
12
+ }
13
+
14
+ export function getDb(): DatabaseSync {
15
+ if (_db) return _db;
16
+
17
+ const dir = getStorageDir();
18
+ if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
19
+
20
+ const dbPath = path.join(dir, "agents.db");
21
+ _db = new DatabaseSync(dbPath);
22
+ _db.exec("PRAGMA journal_mode = WAL");
23
+ _db.exec("PRAGMA foreign_keys = ON");
24
+ return _db;
25
+ }
@@ -0,0 +1,66 @@
1
+ import { getDb } from "./client";
2
+
3
+ export function initSchema(): void {
4
+ const db = getDb();
5
+
6
+ db.exec(`
7
+ CREATE TABLE IF NOT EXISTS agents (
8
+ id TEXT PRIMARY KEY,
9
+ config TEXT NOT NULL,
10
+ created_at INTEGER NOT NULL,
11
+ updated_at INTEGER NOT NULL
12
+ );
13
+
14
+ CREATE TABLE IF NOT EXISTS sessions (
15
+ id TEXT PRIMARY KEY,
16
+ agent_id TEXT NOT NULL,
17
+ title TEXT,
18
+ session_file TEXT,
19
+ created_at INTEGER NOT NULL,
20
+ FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
21
+ );
22
+
23
+ CREATE TABLE IF NOT EXISTS knowledge_bases (
24
+ id TEXT PRIMARY KEY,
25
+ name TEXT NOT NULL,
26
+ description TEXT DEFAULT '',
27
+ embedding_model TEXT DEFAULT 'local',
28
+ created_at INTEGER NOT NULL,
29
+ updated_at INTEGER NOT NULL
30
+ );
31
+
32
+ CREATE TABLE IF NOT EXISTS kb_documents (
33
+ id TEXT PRIMARY KEY,
34
+ kb_id TEXT NOT NULL,
35
+ filename TEXT NOT NULL,
36
+ file_path TEXT,
37
+ mime_type TEXT,
38
+ chunk_count INTEGER DEFAULT 0,
39
+ status TEXT DEFAULT 'processing',
40
+ error TEXT,
41
+ created_at INTEGER NOT NULL,
42
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
43
+ );
44
+
45
+ CREATE TABLE IF NOT EXISTS kb_chunks (
46
+ id TEXT PRIMARY KEY,
47
+ kb_id TEXT NOT NULL,
48
+ doc_id TEXT NOT NULL,
49
+ content TEXT NOT NULL,
50
+ embedding BLOB,
51
+ metadata TEXT DEFAULT '{}',
52
+ created_at INTEGER NOT NULL,
53
+ FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE,
54
+ FOREIGN KEY (doc_id) REFERENCES kb_documents(id) ON DELETE CASCADE
55
+ );
56
+
57
+ CREATE TABLE IF NOT EXISTS settings (
58
+ key TEXT PRIMARY KEY,
59
+ value TEXT NOT NULL
60
+ );
61
+
62
+ CREATE INDEX IF NOT EXISTS idx_sessions_agent_id ON sessions(agent_id);
63
+ CREATE INDEX IF NOT EXISTS idx_kb_chunks_kb_id ON kb_chunks(kb_id);
64
+ CREATE INDEX IF NOT EXISTS idx_kb_documents_kb_id ON kb_documents(kb_id);
65
+ `);
66
+ }
@@ -0,0 +1,33 @@
1
+ import path from "path";
2
+ import os from "os";
3
+ import fs from "fs";
4
+
5
+ function getEnvPath(): string {
6
+ const dir = path.join(os.homedir(), ".digital-agents");
7
+ fs.mkdirSync(dir, { recursive: true });
8
+ return path.join(dir, "env.json");
9
+ }
10
+
11
+ export function readEnvVars(): Record<string, string> {
12
+ const p = getEnvPath();
13
+ if (!fs.existsSync(p)) return {};
14
+ try { return JSON.parse(fs.readFileSync(p, "utf8")); } catch { return {}; }
15
+ }
16
+
17
+ export function writeEnvVars(vars: Record<string, string>): void {
18
+ fs.writeFileSync(getEnvPath(), JSON.stringify(vars, null, 2), "utf8");
19
+ }
20
+
21
+ export function setEnvVar(key: string, value: string): void {
22
+ const vars = readEnvVars();
23
+ vars[key] = value;
24
+ writeEnvVars(vars);
25
+ }
26
+
27
+ export function deleteEnvVar(key: string): boolean {
28
+ const vars = readEnvVars();
29
+ if (!(key in vars)) return false;
30
+ delete vars[key];
31
+ writeEnvVars(vars);
32
+ return true;
33
+ }
@@ -0,0 +1,34 @@
1
+ // Simple sentence/paragraph-based text chunker
2
+ // Splits text into overlapping windows of ~512 tokens (≈ 1900 chars)
3
+
4
+ const CHUNK_SIZE = 1900; // chars (~512 tokens)
5
+ const OVERLAP = 200; // chars overlap
6
+
7
+ export function chunkText(
8
+ text: string,
9
+ metadata: { filename: string; pageNum?: number }
10
+ ): { content: string; metadata: object }[] {
11
+ const paragraphs = text.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
12
+ const chunks: { content: string; metadata: object }[] = [];
13
+
14
+ let current = "";
15
+ let chunkIndex = 0;
16
+
17
+ for (const para of paragraphs) {
18
+ if ((current + "\n\n" + para).length > CHUNK_SIZE && current.length > 0) {
19
+ chunks.push({ content: current.trim(), metadata: { ...metadata, chunkIndex } });
20
+ chunkIndex++;
21
+ // Overlap: keep last portion of current chunk
22
+ const overlapText = current.slice(-OVERLAP);
23
+ current = overlapText + "\n\n" + para;
24
+ } else {
25
+ current = current ? current + "\n\n" + para : para;
26
+ }
27
+ }
28
+
29
+ if (current.trim()) {
30
+ chunks.push({ content: current.trim(), metadata: { ...metadata, chunkIndex } });
31
+ }
32
+
33
+ return chunks;
34
+ }
@@ -0,0 +1,48 @@
1
+ // Embedding provider — uses OpenAI text-embedding-3-small if API key available,
2
+ // otherwise falls back to a simple TF-IDF character n-gram hash embedding.
3
+ import { getSettings } from "../settings";
4
+
5
+ const VECTOR_DIM = 1536; // OpenAI text-embedding-3-small
6
+
7
+ export async function embedText(text: string): Promise<number[]> {
8
+ const settings = getSettings();
9
+ if (settings.apiKeys.openai) {
10
+ return embedWithOpenAI(text, settings.apiKeys.openai);
11
+ }
12
+ return embedFallback(text);
13
+ }
14
+
15
+ export function getEmbeddingDim(): number {
16
+ const settings = getSettings();
17
+ return settings.apiKeys.openai ? VECTOR_DIM : 256;
18
+ }
19
+
20
+ async function embedWithOpenAI(text: string, apiKey: string): Promise<number[]> {
21
+ const res = await fetch("https://api.openai.com/v1/embeddings", {
22
+ method: "POST",
23
+ headers: {
24
+ "Content-Type": "application/json",
25
+ Authorization: `Bearer ${apiKey}`,
26
+ },
27
+ body: JSON.stringify({ model: "text-embedding-3-small", input: text }),
28
+ });
29
+ if (!res.ok) throw new Error(`OpenAI embedding error: ${res.statusText}`);
30
+ const data = (await res.json()) as { data: { embedding: number[] }[] };
31
+ return data.data[0].embedding;
32
+ }
33
+
34
+ // Lightweight 256-dim character n-gram hash embedding (no external deps)
35
+ function embedFallback(text: string): number[] {
36
+ const dim = 256;
37
+ const vec = new Float32Array(dim);
38
+ const lower = text.toLowerCase();
39
+ for (let i = 0; i < lower.length - 1; i++) {
40
+ const bigram = lower.charCodeAt(i) * 31 + lower.charCodeAt(i + 1);
41
+ vec[bigram % dim] += 1;
42
+ }
43
+ // L2 normalize
44
+ let norm = 0;
45
+ for (let i = 0; i < dim; i++) norm += vec[i] * vec[i];
46
+ norm = Math.sqrt(norm) || 1;
47
+ return Array.from(vec).map((v) => v / norm);
48
+ }
@@ -0,0 +1,94 @@
1
+ // Lightweight cosine-similarity vector search using plain SQLite BLOB storage.
2
+ // Embeddings are stored as Float32Array serialized to Buffer.
3
+
4
+ import { getDb } from "../db/client";
5
+ import { v4 as uuidv4 } from "uuid";
6
+ import type { SearchResult } from "@/types/knowledge";
7
+
8
+ export async function searchChunks(
9
+ kbIds: string[],
10
+ queryEmbedding: number[],
11
+ topK = 5
12
+ ): Promise<SearchResult[]> {
13
+ const db = getDb();
14
+ if (kbIds.length === 0) return [];
15
+
16
+ const placeholders = kbIds.map(() => "?").join(",");
17
+ const rows = db
18
+ .prepare(`SELECT * FROM kb_chunks WHERE kb_id IN (${placeholders})`)
19
+ .all(...kbIds) as {
20
+ id: string;
21
+ kb_id: string;
22
+ doc_id: string;
23
+ content: string;
24
+ embedding: Buffer | null;
25
+ metadata: string;
26
+ }[];
27
+
28
+ const results: SearchResult[] = [];
29
+
30
+ for (const row of rows) {
31
+ if (!row.embedding) continue;
32
+ const docEmbedding = bufferToFloat32Array(row.embedding);
33
+ const score = cosineSimilarity(queryEmbedding, docEmbedding);
34
+ const meta = JSON.parse(row.metadata ?? "{}");
35
+ results.push({
36
+ chunk: {
37
+ id: row.id,
38
+ kbId: row.kb_id,
39
+ docId: row.doc_id,
40
+ content: row.content,
41
+ metadata: meta,
42
+ },
43
+ score,
44
+ });
45
+ }
46
+
47
+ results.sort((a, b) => b.score - a.score);
48
+ return results.slice(0, topK);
49
+ }
50
+
51
+ export function storeChunk(
52
+ kbId: string,
53
+ docId: string,
54
+ content: string,
55
+ embedding: number[],
56
+ metadata: object
57
+ ): void {
58
+ const db = getDb();
59
+ const id = uuidv4();
60
+ const embBuf = float32ArrayToBuffer(new Float32Array(embedding));
61
+ db.prepare(
62
+ "INSERT INTO kb_chunks (id, kb_id, doc_id, content, embedding, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)"
63
+ ).run(id, kbId, docId, content, embBuf, JSON.stringify(metadata), Date.now());
64
+ }
65
+
66
+ export function deleteChunksByDoc(docId: string): void {
67
+ getDb().prepare("DELETE FROM kb_chunks WHERE doc_id = ?").run(docId);
68
+ }
69
+
70
+ export function deleteChunksByKb(kbId: string): void {
71
+ getDb().prepare("DELETE FROM kb_chunks WHERE kb_id = ?").run(kbId);
72
+ }
73
+
74
+ // --- helpers ---
75
+
76
+ function cosineSimilarity(a: number[], b: Float32Array): number {
77
+ if (a.length !== b.length) return 0;
78
+ let dot = 0, normA = 0, normB = 0;
79
+ for (let i = 0; i < a.length; i++) {
80
+ dot += a[i] * b[i];
81
+ normA += a[i] * a[i];
82
+ normB += b[i] * b[i];
83
+ }
84
+ const denom = Math.sqrt(normA) * Math.sqrt(normB);
85
+ return denom === 0 ? 0 : dot / denom;
86
+ }
87
+
88
+ function float32ArrayToBuffer(arr: Float32Array): Buffer {
89
+ return Buffer.from(arr.buffer);
90
+ }
91
+
92
+ function bufferToFloat32Array(buf: Buffer): Float32Array {
93
+ return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
94
+ }
@@ -0,0 +1,74 @@
1
+ import { getDb } from "../db/client";
2
+ import type { KnowledgeBase, KbDocument } from "@/types/knowledge";
3
+ import { v4 as uuidv4 } from "uuid";
4
+
5
+ type KbRow = {
6
+ id: string; name: string; description: string;
7
+ embedding_model: string; created_at: number; updated_at: number;
8
+ };
9
+ type DocRow = {
10
+ id: string; kb_id: string; filename: string; file_path: string;
11
+ mime_type: string; chunk_count: number; status: string; error: string | null;
12
+ created_at: number;
13
+ };
14
+
15
+ export const kbStore = {
16
+ list(): KnowledgeBase[] {
17
+ const db = getDb();
18
+ const rows = db.prepare("SELECT kb.*, (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id) as doc_count, (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) as chunk_count FROM knowledge_bases kb ORDER BY kb.created_at DESC").all() as (KbRow & { doc_count: number; chunk_count: number })[];
19
+ return rows.map((r) => ({
20
+ id: r.id, name: r.name, description: r.description,
21
+ embeddingModel: r.embedding_model,
22
+ documentCount: r.doc_count, chunkCount: r.chunk_count,
23
+ createdAt: new Date(r.created_at).toISOString(),
24
+ updatedAt: new Date(r.updated_at).toISOString(),
25
+ }));
26
+ },
27
+
28
+ getById(id: string): KnowledgeBase | undefined {
29
+ const db = getDb();
30
+ const r = db.prepare("SELECT kb.*, (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id) as doc_count, (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) as chunk_count FROM knowledge_bases kb WHERE kb.id = ?").get(id) as (KbRow & { doc_count: number; chunk_count: number }) | undefined;
31
+ if (!r) return undefined;
32
+ return { id: r.id, name: r.name, description: r.description, embeddingModel: r.embedding_model, documentCount: r.doc_count, chunkCount: r.chunk_count, createdAt: new Date(r.created_at).toISOString(), updatedAt: new Date(r.updated_at).toISOString() };
33
+ },
34
+
35
+ create(name: string, description = ""): KnowledgeBase {
36
+ const db = getDb();
37
+ const id = uuidv4();
38
+ const now = Date.now();
39
+ db.prepare("INSERT INTO knowledge_bases (id, name, description, embedding_model, created_at, updated_at) VALUES (?, ?, ?, 'auto', ?, ?)").run(id, name, description, now, now);
40
+ return this.getById(id)!;
41
+ },
42
+
43
+ delete(id: string): boolean {
44
+ const result = getDb().prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
45
+ return result.changes > 0;
46
+ },
47
+
48
+ listDocuments(kbId: string): KbDocument[] {
49
+ const db = getDb();
50
+ const rows = db.prepare("SELECT * FROM kb_documents WHERE kb_id = ? ORDER BY created_at DESC").all(kbId) as DocRow[];
51
+ return rows.map((r) => ({
52
+ id: r.id, kbId: r.kb_id, filename: r.filename, filePath: r.file_path,
53
+ mimeType: r.mime_type, chunkCount: r.chunk_count,
54
+ status: r.status as KbDocument["status"], error: r.error ?? undefined,
55
+ createdAt: new Date(r.created_at).toISOString(),
56
+ }));
57
+ },
58
+
59
+ createDocument(kbId: string, filename: string, filePath: string, mimeType: string): string {
60
+ const db = getDb();
61
+ const id = uuidv4();
62
+ db.prepare("INSERT INTO kb_documents (id, kb_id, filename, file_path, mime_type, chunk_count, status, created_at) VALUES (?, ?, ?, ?, ?, 0, 'processing', ?)").run(id, kbId, filename, filePath, mimeType, Date.now());
63
+ return id;
64
+ },
65
+
66
+ updateDocumentStatus(id: string, status: string, chunkCount = 0, error?: string): void {
67
+ getDb().prepare("UPDATE kb_documents SET status = ?, chunk_count = ?, error = ? WHERE id = ?").run(status, chunkCount, error ?? null, id);
68
+ },
69
+
70
+ deleteDocument(id: string): boolean {
71
+ const result = getDb().prepare("DELETE FROM kb_documents WHERE id = ?").run(id);
72
+ return result.changes > 0;
73
+ },
74
+ };