@jiangxiaosheng/digital-agent-platform 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +37 -0
- package/eslint.config.mjs +20 -0
- package/messages/en.json +184 -0
- package/messages/zh.json +184 -0
- package/middleware.ts +9 -0
- package/next-env.d.ts +5 -0
- package/next.config.ts +16 -0
- package/package.json +59 -0
- package/postcss.config.mjs +5 -0
- package/src/app/[locale]/agents/[id]/chat/page.tsx +833 -0
- package/src/app/[locale]/agents/[id]/edit/page.tsx +58 -0
- package/src/app/[locale]/agents/new/page.tsx +52 -0
- package/src/app/[locale]/agents/page.tsx +122 -0
- package/src/app/[locale]/knowledge/page.tsx +305 -0
- package/src/app/[locale]/layout.tsx +46 -0
- package/src/app/[locale]/page.tsx +6 -0
- package/src/app/[locale]/settings/page.tsx +1204 -0
- package/src/app/api/agents/[id]/route.ts +27 -0
- package/src/app/api/agents/route.ts +27 -0
- package/src/app/api/auth-config/test/route.ts +69 -0
- package/src/app/api/bash-output/route.ts +57 -0
- package/src/app/api/chat/[agentId]/route.ts +164 -0
- package/src/app/api/env-vars/[key]/route.ts +22 -0
- package/src/app/api/env-vars/route.ts +26 -0
- package/src/app/api/env-vars/scan/route.ts +66 -0
- package/src/app/api/file-serve/route.ts +58 -0
- package/src/app/api/fs/browse/route.ts +47 -0
- package/src/app/api/knowledge/[id]/documents/[docId]/route.ts +22 -0
- package/src/app/api/knowledge/[id]/route.ts +21 -0
- package/src/app/api/knowledge/[id]/upload/route.ts +90 -0
- package/src/app/api/knowledge/route.ts +15 -0
- package/src/app/api/knowledge/search/route.ts +26 -0
- package/src/app/api/models/route.ts +160 -0
- package/src/app/api/models-config/route.ts +41 -0
- package/src/app/api/models-config/test/route.ts +84 -0
- package/src/app/api/sessions/[agentId]/[sessionId]/route.ts +12 -0
- package/src/app/api/sessions/[agentId]/route.ts +19 -0
- package/src/app/api/sessions/history/route.ts +166 -0
- package/src/app/api/settings/route.ts +45 -0
- package/src/app/api/skills/[name]/route.ts +40 -0
- package/src/app/api/skills/install/route.ts +35 -0
- package/src/app/api/skills/route.ts +54 -0
- package/src/app/api/skills/search/route.ts +60 -0
- package/src/app/api/tts/route.ts +235 -0
- package/src/app/api/voice-provider/route.ts +98 -0
- package/src/app/api/workspace-files/route.ts +151 -0
- package/src/app/globals.css +176 -0
- package/src/app/layout.tsx +13 -0
- package/src/app/page.tsx +6 -0
- package/src/components/agents/agent-form.tsx +457 -0
- package/src/components/agents/model-selector.tsx +290 -0
- package/src/components/agents/skills-manager.tsx +347 -0
- package/src/components/chat/input-bar.tsx +561 -0
- package/src/components/chat/message-bubble.tsx +395 -0
- package/src/components/layout/sidebar.tsx +304 -0
- package/src/components/providers.tsx +19 -0
- package/src/components/ui/badge.tsx +26 -0
- package/src/components/ui/button.tsx +43 -0
- package/src/components/ui/card.tsx +46 -0
- package/src/components/ui/dialog.tsx +74 -0
- package/src/components/ui/directory-picker.tsx +185 -0
- package/src/components/ui/input.tsx +18 -0
- package/src/components/ui/label.tsx +16 -0
- package/src/components/ui/scroll-area.tsx +41 -0
- package/src/components/ui/select.tsx +93 -0
- package/src/components/ui/slider.tsx +35 -0
- package/src/components/ui/switch.tsx +35 -0
- package/src/components/ui/tabs.tsx +48 -0
- package/src/components/ui/textarea.tsx +17 -0
- package/src/components/ui/toaster.tsx +50 -0
- package/src/hooks/use-toast.ts +43 -0
- package/src/i18n/request.ts +13 -0
- package/src/i18n/routing.ts +7 -0
- package/src/lib/agent-factory.ts +200 -0
- package/src/lib/agents/store.ts +95 -0
- package/src/lib/db/client.ts +25 -0
- package/src/lib/db/schema.ts +66 -0
- package/src/lib/env-vars.ts +33 -0
- package/src/lib/knowledge/chunker.ts +34 -0
- package/src/lib/knowledge/embedder.ts +48 -0
- package/src/lib/knowledge/retriever.ts +94 -0
- package/src/lib/knowledge/store.ts +74 -0
- package/src/lib/models-config.ts +55 -0
- package/src/lib/settings.ts +33 -0
- package/src/lib/skills.ts +79 -0
- package/src/lib/system-prompt.ts +52 -0
- package/src/lib/tool-builder.ts +19 -0
- package/src/lib/utils.ts +6 -0
- package/src/lib/voice-providers-config.ts +71 -0
- package/src/middleware.ts +9 -0
- package/src/types/agent.ts +124 -0
- package/src/types/knowledge.ts +39 -0
- package/src/types/settings.ts +31 -0
- package/tsconfig.json +40 -0
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
// Core factory: dynamically builds a pi-coding-agent AgentSession
|
|
2
|
+
// with per-agent persona, model, tools, workspace and RAG context.
|
|
3
|
+
import type { DigitalAgent } from "@/types/agent";
|
|
4
|
+
import { buildSystemPrompt } from "./system-prompt";
|
|
5
|
+
import { buildToolConfig } from "./tool-builder";
|
|
6
|
+
import { searchChunks } from "./knowledge/retriever";
|
|
7
|
+
import { embedText } from "./knowledge/embedder";
|
|
8
|
+
import path from "path";
|
|
9
|
+
import os from "os";
|
|
10
|
+
import fs from "fs";
|
|
11
|
+
|
|
12
|
+
export function getAgentDir(): string {
|
|
13
|
+
return process.env.PI_AGENT_DIR ?? path.join(os.homedir(), ".pi", "agent");
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
export function getSessionDir(agentId: string): string {
|
|
17
|
+
return path.join(os.homedir(), ".pi", "agent", "sessions", `agent-${agentId}`);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/** 解析数字人工作区路径,空白时使用 agentDir */
|
|
21
|
+
export function resolveWorkspace(agent: DigitalAgent): string {
|
|
22
|
+
const wp = agent.workspacePath?.trim();
|
|
23
|
+
if (wp) {
|
|
24
|
+
const expanded = wp.replace(/^~/, os.homedir());
|
|
25
|
+
// 目录不存在时自动创建,而不是静默回退
|
|
26
|
+
fs.mkdirSync(expanded, { recursive: true });
|
|
27
|
+
return expanded;
|
|
28
|
+
}
|
|
29
|
+
// 未配置时使用数字人专属默认目录
|
|
30
|
+
const agentWorkspace = path.join(getAgentDir(), "workspaces", `agent-${agent.id}`);
|
|
31
|
+
fs.mkdirSync(agentWorkspace, { recursive: true });
|
|
32
|
+
return agentWorkspace;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export async function buildRagContext(
|
|
36
|
+
agent: DigitalAgent,
|
|
37
|
+
userMessage: string
|
|
38
|
+
): Promise<string> {
|
|
39
|
+
if (!agent.knowledgeBases.length) return "";
|
|
40
|
+
const queryVec = await embedText(userMessage);
|
|
41
|
+
const results = await searchChunks(agent.knowledgeBases, queryVec, 5);
|
|
42
|
+
if (!results.length) return "";
|
|
43
|
+
return results
|
|
44
|
+
.map((r, i) => `[${i + 1}] (来源: ${r.chunk.metadata.filename})\n${r.chunk.content}`)
|
|
45
|
+
.join("\n\n---\n\n");
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
// Returns a ready-to-use AgentSession — caller must call session.dispose() when done.
|
|
49
|
+
export async function createSession(
|
|
50
|
+
agent: DigitalAgent,
|
|
51
|
+
userMessage: string,
|
|
52
|
+
existingSessionFile?: string
|
|
53
|
+
) {
|
|
54
|
+
const {
|
|
55
|
+
createAgentSession,
|
|
56
|
+
ModelRuntime,
|
|
57
|
+
SessionManager,
|
|
58
|
+
DefaultResourceLoader,
|
|
59
|
+
defineTool,
|
|
60
|
+
} = await import("@earendil-works/pi-coding-agent");
|
|
61
|
+
|
|
62
|
+
// 1. RAG
|
|
63
|
+
const ragContext = await buildRagContext(agent, userMessage);
|
|
64
|
+
|
|
65
|
+
// 2. System prompt with agent identity
|
|
66
|
+
const systemPrompt = buildSystemPrompt(agent, ragContext);
|
|
67
|
+
|
|
68
|
+
// 3. Model runtime — load both auth.json (built-in keys) and models.json (custom providers)
|
|
69
|
+
const modelRuntime = await ModelRuntime.create({
|
|
70
|
+
authPath: path.join(getAgentDir(), "auth.json"),
|
|
71
|
+
modelsStorePath: path.join(getAgentDir(), "models.json"),
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
// Also inject keys saved via our settings page (SQLite store)
|
|
75
|
+
// This allows keys entered in Settings to override/supplement auth.json
|
|
76
|
+
const { getSettings } = await import("./settings");
|
|
77
|
+
const settings = getSettings();
|
|
78
|
+
const keyMap: Record<string, string | undefined> = {
|
|
79
|
+
anthropic: settings.apiKeys.anthropic,
|
|
80
|
+
openai: settings.apiKeys.openai,
|
|
81
|
+
google: settings.apiKeys.google,
|
|
82
|
+
};
|
|
83
|
+
for (const [provider, key] of Object.entries(keyMap)) {
|
|
84
|
+
if (key) await modelRuntime.setRuntimeApiKey(provider, key);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
// Resolve model — throw with helpful message if not found
|
|
88
|
+
const model = modelRuntime.getModel(agent.model.provider, agent.model.modelId);
|
|
89
|
+
if (!model) {
|
|
90
|
+
const available = modelRuntime.getModels().slice(0, 5).map((m) => `${m.provider}/${m.id}`);
|
|
91
|
+
throw new Error(
|
|
92
|
+
`Model "${agent.model.provider}/${agent.model.modelId}" not found. ` +
|
|
93
|
+
`Check that the API key for "${agent.model.provider}" is configured in Settings. ` +
|
|
94
|
+
`Available models (first 5): ${available.join(", ")}`
|
|
95
|
+
);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// 4. Workspace (cwd) — use agent's own workspace, not process.cwd()
|
|
99
|
+
const cwd = resolveWorkspace(agent);
|
|
100
|
+
|
|
101
|
+
// 5. Load global env vars
|
|
102
|
+
const { readEnvVars } = await import("./env-vars");
|
|
103
|
+
const globalEnv = readEnvVars();
|
|
104
|
+
|
|
105
|
+
// Inject into process.env so ALL tools and skills can read them.
|
|
106
|
+
// spawnHook only covers the bash tool; skills (tavily-search, edge-tts, etc.)
|
|
107
|
+
// spawn subprocesses that inherit directly from process.env.
|
|
108
|
+
// These are user-configured API keys/config that are safe to set globally.
|
|
109
|
+
for (const [k, v] of Object.entries(globalEnv)) {
|
|
110
|
+
process.env[k] = v;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
// 6. Tools
|
|
114
|
+
// env vars are already in process.env (injected above), so no need for spawnHook.
|
|
115
|
+
// Keep bash as a SDK builtin so the model recognises it correctly.
|
|
116
|
+
const { builtinNames, customTools: customToolDefs } = buildToolConfig(agent);
|
|
117
|
+
const customTools = [] as ReturnType<typeof defineTool>[];
|
|
118
|
+
|
|
119
|
+
// Calculator (custom tool)
|
|
120
|
+
if (customToolDefs.find((t) => t.name === "calculator")) {
|
|
121
|
+
const { Type } = await import("@sinclair/typebox");
|
|
122
|
+
customTools.push(
|
|
123
|
+
defineTool({
|
|
124
|
+
name: "calculator",
|
|
125
|
+
label: "Calculator",
|
|
126
|
+
description: "Evaluate a mathematical expression and return the result.",
|
|
127
|
+
parameters: Type.Object({ expression: Type.String({ description: "Math expression" }) }),
|
|
128
|
+
execute: async (_id: string, { expression }: { expression: string }) => {
|
|
129
|
+
try {
|
|
130
|
+
const result = Function(`"use strict"; return (${expression})`)();
|
|
131
|
+
return { content: [{ type: "text" as const, text: String(result) }], details: {} };
|
|
132
|
+
} catch (e) {
|
|
133
|
+
return { content: [{ type: "text" as const, text: `Error: ${String(e)}` }], details: {} };
|
|
134
|
+
}
|
|
135
|
+
},
|
|
136
|
+
})
|
|
137
|
+
);
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// 6. Session manager — use agent-specific session directory
|
|
141
|
+
const sessionMgr = existingSessionFile
|
|
142
|
+
? SessionManager.open(existingSessionFile)
|
|
143
|
+
: SessionManager.create(cwd, getSessionDir(agent.id));
|
|
144
|
+
|
|
145
|
+
// 7. Resource loader — two-phase: first discover filtered skills, then build final loader
|
|
146
|
+
// enabledSkills 为空 = 不限制(加载全部已安装 skills)
|
|
147
|
+
// enabledSkills 有值 = 白名单过滤
|
|
148
|
+
const enabledSkills = new Set(agent.enabledSkills ?? []);
|
|
149
|
+
const skillFilter = (base: { skills: { name: string }[]; diagnostics: unknown[] }) => ({
|
|
150
|
+
skills: enabledSkills.size > 0
|
|
151
|
+
? base.skills.filter((s) => enabledSkills.has(s.name))
|
|
152
|
+
: base.skills, // 空列表 = 不过滤,保留全部可用 skills
|
|
153
|
+
diagnostics: base.diagnostics,
|
|
154
|
+
});
|
|
155
|
+
|
|
156
|
+
// Phase 1: discover which skills are actually available after filter
|
|
157
|
+
const { formatSkillsForPrompt } = await import("@earendil-works/pi-coding-agent");
|
|
158
|
+
const discoveryLoader = new DefaultResourceLoader({
|
|
159
|
+
cwd,
|
|
160
|
+
agentDir: getAgentDir(),
|
|
161
|
+
skillsOverride: skillFilter as never,
|
|
162
|
+
});
|
|
163
|
+
await discoveryLoader.reload();
|
|
164
|
+
const { skills: filteredSkills } = discoveryLoader.getSkills();
|
|
165
|
+
|
|
166
|
+
// Phase 2: build final loader — systemPrompt + skills appended + skill filter
|
|
167
|
+
const skillsSection = filteredSkills.length > 0
|
|
168
|
+
? formatSkillsForPrompt(filteredSkills)
|
|
169
|
+
: "";
|
|
170
|
+
|
|
171
|
+
const resourceLoader = new DefaultResourceLoader({
|
|
172
|
+
cwd,
|
|
173
|
+
agentDir: getAgentDir(),
|
|
174
|
+
systemPromptOverride: () => systemPrompt,
|
|
175
|
+
appendSystemPromptOverride: skillsSection ? () => [skillsSection] : undefined,
|
|
176
|
+
skillsOverride: skillFilter as never,
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
// ⚠️ REQUIRED: reload() initialises internal caches incl. systemPromptOverride
|
|
180
|
+
await resourceLoader.reload();
|
|
181
|
+
|
|
182
|
+
// 8. Create session
|
|
183
|
+
// bash is a SDK builtin — env vars flow via process.env set above
|
|
184
|
+
const allCustomTools = customTools.length > 0 ? customTools : undefined;
|
|
185
|
+
const hasAnyTools = builtinNames.length > 0 || (allCustomTools?.length ?? 0) > 0;
|
|
186
|
+
|
|
187
|
+
const { session } = await createAgentSession({
|
|
188
|
+
cwd,
|
|
189
|
+
modelRuntime,
|
|
190
|
+
model,
|
|
191
|
+
thinkingLevel: agent.model.thinkingLevel as Parameters<typeof createAgentSession>[0] extends { thinkingLevel?: infer T } ? T : never,
|
|
192
|
+
tools: builtinNames.length > 0 ? builtinNames : undefined,
|
|
193
|
+
noTools: !hasAnyTools ? "all" : undefined,
|
|
194
|
+
customTools: allCustomTools,
|
|
195
|
+
sessionManager: sessionMgr,
|
|
196
|
+
resourceLoader,
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
return session;
|
|
200
|
+
}
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { getDb } from "../db/client";
|
|
2
|
+
import type { DigitalAgent, CreateAgentInput, UpdateAgentInput } from "@/types/agent";
|
|
3
|
+
import { v4 as uuidv4 } from "uuid";
|
|
4
|
+
|
|
5
|
+
type AgentRow = { id: string; config: string; created_at: number; updated_at: number };
|
|
6
|
+
|
|
7
|
+
function rowToAgent(row: AgentRow): DigitalAgent {
|
|
8
|
+
const cfg = JSON.parse(row.config);
|
|
9
|
+
return {
|
|
10
|
+
...cfg,
|
|
11
|
+
id: row.id,
|
|
12
|
+
createdAt: new Date(row.created_at).toISOString(),
|
|
13
|
+
updatedAt: new Date(row.updated_at).toISOString(),
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export const agentStore = {
|
|
18
|
+
list(): DigitalAgent[] {
|
|
19
|
+
const db = getDb();
|
|
20
|
+
const rows = db.prepare("SELECT * FROM agents ORDER BY created_at DESC").all() as AgentRow[];
|
|
21
|
+
return rows.map(rowToAgent);
|
|
22
|
+
},
|
|
23
|
+
|
|
24
|
+
getById(id: string): DigitalAgent | undefined {
|
|
25
|
+
const db = getDb();
|
|
26
|
+
const row = db.prepare("SELECT * FROM agents WHERE id = ?").get(id) as AgentRow | undefined;
|
|
27
|
+
return row ? rowToAgent(row) : undefined;
|
|
28
|
+
},
|
|
29
|
+
|
|
30
|
+
create(input: CreateAgentInput): DigitalAgent {
|
|
31
|
+
const db = getDb();
|
|
32
|
+
const id = uuidv4();
|
|
33
|
+
const now = Date.now();
|
|
34
|
+
const cfg = { ...input };
|
|
35
|
+
db.prepare("INSERT INTO agents (id, config, created_at, updated_at) VALUES (?, ?, ?, ?)").run(
|
|
36
|
+
id, JSON.stringify(cfg), now, now
|
|
37
|
+
);
|
|
38
|
+
return rowToAgent({ id, config: JSON.stringify(cfg), created_at: now, updated_at: now });
|
|
39
|
+
},
|
|
40
|
+
|
|
41
|
+
update(id: string, input: UpdateAgentInput): DigitalAgent | undefined {
|
|
42
|
+
const db = getDb();
|
|
43
|
+
const existing = this.getById(id);
|
|
44
|
+
if (!existing) return undefined;
|
|
45
|
+
const now = Date.now();
|
|
46
|
+
const merged = { ...existing, ...input, id: existing.id };
|
|
47
|
+
const { createdAt, updatedAt, ...cfg } = merged;
|
|
48
|
+
void createdAt; void updatedAt;
|
|
49
|
+
db.prepare("UPDATE agents SET config = ?, updated_at = ? WHERE id = ?").run(
|
|
50
|
+
JSON.stringify(cfg), now, id
|
|
51
|
+
);
|
|
52
|
+
return this.getById(id);
|
|
53
|
+
},
|
|
54
|
+
|
|
55
|
+
delete(id: string): boolean {
|
|
56
|
+
const db = getDb();
|
|
57
|
+
const result = db.prepare("DELETE FROM agents WHERE id = ?").run(id);
|
|
58
|
+
return result.changes > 0;
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
export const sessionStore = {
|
|
63
|
+
list(agentId: string) {
|
|
64
|
+
const db = getDb();
|
|
65
|
+
return db
|
|
66
|
+
.prepare("SELECT * FROM sessions WHERE agent_id = ? ORDER BY created_at DESC")
|
|
67
|
+
.all(agentId) as { id: string; agent_id: string; title: string; session_file: string; created_at: number }[];
|
|
68
|
+
},
|
|
69
|
+
|
|
70
|
+
create(agentId: string, sessionFile: string, title?: string): string {
|
|
71
|
+
const db = getDb();
|
|
72
|
+
const id = uuidv4();
|
|
73
|
+
db.prepare("INSERT INTO sessions (id, agent_id, title, session_file, created_at) VALUES (?, ?, ?, ?, ?)").run(
|
|
74
|
+
id, agentId, title ?? "New Session", sessionFile, Date.now()
|
|
75
|
+
);
|
|
76
|
+
return id;
|
|
77
|
+
},
|
|
78
|
+
|
|
79
|
+
updateFile(id: string, sessionFile: string): void {
|
|
80
|
+
const db = getDb();
|
|
81
|
+
db.prepare("UPDATE sessions SET session_file = ? WHERE id = ?").run(sessionFile, id);
|
|
82
|
+
},
|
|
83
|
+
|
|
84
|
+
delete(id: string): boolean {
|
|
85
|
+
const db = getDb();
|
|
86
|
+
const result = db.prepare("DELETE FROM sessions WHERE id = ?").run(id);
|
|
87
|
+
return result.changes > 0;
|
|
88
|
+
},
|
|
89
|
+
|
|
90
|
+
deleteByAgent(agentId: string): number {
|
|
91
|
+
const db = getDb();
|
|
92
|
+
const result = db.prepare("DELETE FROM sessions WHERE agent_id = ?").run(agentId);
|
|
93
|
+
return Number(result.changes);
|
|
94
|
+
},
|
|
95
|
+
};
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { DatabaseSync } from "node:sqlite";
|
|
2
|
+
import path from "path";
|
|
3
|
+
import os from "os";
|
|
4
|
+
import fs from "fs";
|
|
5
|
+
|
|
6
|
+
let _db: DatabaseSync | null = null;
|
|
7
|
+
|
|
8
|
+
export function getStorageDir(): string {
|
|
9
|
+
const custom = process.env.STORAGE_PATH;
|
|
10
|
+
if (custom) return custom;
|
|
11
|
+
return path.join(os.homedir(), ".digital-agents");
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export function getDb(): DatabaseSync {
|
|
15
|
+
if (_db) return _db;
|
|
16
|
+
|
|
17
|
+
const dir = getStorageDir();
|
|
18
|
+
if (!fs.existsSync(dir)) fs.mkdirSync(dir, { recursive: true });
|
|
19
|
+
|
|
20
|
+
const dbPath = path.join(dir, "agents.db");
|
|
21
|
+
_db = new DatabaseSync(dbPath);
|
|
22
|
+
_db.exec("PRAGMA journal_mode = WAL");
|
|
23
|
+
_db.exec("PRAGMA foreign_keys = ON");
|
|
24
|
+
return _db;
|
|
25
|
+
}
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
import { getDb } from "./client";
|
|
2
|
+
|
|
3
|
+
export function initSchema(): void {
|
|
4
|
+
const db = getDb();
|
|
5
|
+
|
|
6
|
+
db.exec(`
|
|
7
|
+
CREATE TABLE IF NOT EXISTS agents (
|
|
8
|
+
id TEXT PRIMARY KEY,
|
|
9
|
+
config TEXT NOT NULL,
|
|
10
|
+
created_at INTEGER NOT NULL,
|
|
11
|
+
updated_at INTEGER NOT NULL
|
|
12
|
+
);
|
|
13
|
+
|
|
14
|
+
CREATE TABLE IF NOT EXISTS sessions (
|
|
15
|
+
id TEXT PRIMARY KEY,
|
|
16
|
+
agent_id TEXT NOT NULL,
|
|
17
|
+
title TEXT,
|
|
18
|
+
session_file TEXT,
|
|
19
|
+
created_at INTEGER NOT NULL,
|
|
20
|
+
FOREIGN KEY (agent_id) REFERENCES agents(id) ON DELETE CASCADE
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
CREATE TABLE IF NOT EXISTS knowledge_bases (
|
|
24
|
+
id TEXT PRIMARY KEY,
|
|
25
|
+
name TEXT NOT NULL,
|
|
26
|
+
description TEXT DEFAULT '',
|
|
27
|
+
embedding_model TEXT DEFAULT 'local',
|
|
28
|
+
created_at INTEGER NOT NULL,
|
|
29
|
+
updated_at INTEGER NOT NULL
|
|
30
|
+
);
|
|
31
|
+
|
|
32
|
+
CREATE TABLE IF NOT EXISTS kb_documents (
|
|
33
|
+
id TEXT PRIMARY KEY,
|
|
34
|
+
kb_id TEXT NOT NULL,
|
|
35
|
+
filename TEXT NOT NULL,
|
|
36
|
+
file_path TEXT,
|
|
37
|
+
mime_type TEXT,
|
|
38
|
+
chunk_count INTEGER DEFAULT 0,
|
|
39
|
+
status TEXT DEFAULT 'processing',
|
|
40
|
+
error TEXT,
|
|
41
|
+
created_at INTEGER NOT NULL,
|
|
42
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE
|
|
43
|
+
);
|
|
44
|
+
|
|
45
|
+
CREATE TABLE IF NOT EXISTS kb_chunks (
|
|
46
|
+
id TEXT PRIMARY KEY,
|
|
47
|
+
kb_id TEXT NOT NULL,
|
|
48
|
+
doc_id TEXT NOT NULL,
|
|
49
|
+
content TEXT NOT NULL,
|
|
50
|
+
embedding BLOB,
|
|
51
|
+
metadata TEXT DEFAULT '{}',
|
|
52
|
+
created_at INTEGER NOT NULL,
|
|
53
|
+
FOREIGN KEY (kb_id) REFERENCES knowledge_bases(id) ON DELETE CASCADE,
|
|
54
|
+
FOREIGN KEY (doc_id) REFERENCES kb_documents(id) ON DELETE CASCADE
|
|
55
|
+
);
|
|
56
|
+
|
|
57
|
+
CREATE TABLE IF NOT EXISTS settings (
|
|
58
|
+
key TEXT PRIMARY KEY,
|
|
59
|
+
value TEXT NOT NULL
|
|
60
|
+
);
|
|
61
|
+
|
|
62
|
+
CREATE INDEX IF NOT EXISTS idx_sessions_agent_id ON sessions(agent_id);
|
|
63
|
+
CREATE INDEX IF NOT EXISTS idx_kb_chunks_kb_id ON kb_chunks(kb_id);
|
|
64
|
+
CREATE INDEX IF NOT EXISTS idx_kb_documents_kb_id ON kb_documents(kb_id);
|
|
65
|
+
`);
|
|
66
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import path from "path";
|
|
2
|
+
import os from "os";
|
|
3
|
+
import fs from "fs";
|
|
4
|
+
|
|
5
|
+
function getEnvPath(): string {
|
|
6
|
+
const dir = path.join(os.homedir(), ".digital-agents");
|
|
7
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
8
|
+
return path.join(dir, "env.json");
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function readEnvVars(): Record<string, string> {
|
|
12
|
+
const p = getEnvPath();
|
|
13
|
+
if (!fs.existsSync(p)) return {};
|
|
14
|
+
try { return JSON.parse(fs.readFileSync(p, "utf8")); } catch { return {}; }
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export function writeEnvVars(vars: Record<string, string>): void {
|
|
18
|
+
fs.writeFileSync(getEnvPath(), JSON.stringify(vars, null, 2), "utf8");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export function setEnvVar(key: string, value: string): void {
|
|
22
|
+
const vars = readEnvVars();
|
|
23
|
+
vars[key] = value;
|
|
24
|
+
writeEnvVars(vars);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export function deleteEnvVar(key: string): boolean {
|
|
28
|
+
const vars = readEnvVars();
|
|
29
|
+
if (!(key in vars)) return false;
|
|
30
|
+
delete vars[key];
|
|
31
|
+
writeEnvVars(vars);
|
|
32
|
+
return true;
|
|
33
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
// Simple sentence/paragraph-based text chunker
|
|
2
|
+
// Splits text into overlapping windows of ~512 tokens (≈ 1900 chars)
|
|
3
|
+
|
|
4
|
+
const CHUNK_SIZE = 1900; // chars (~512 tokens)
|
|
5
|
+
const OVERLAP = 200; // chars overlap
|
|
6
|
+
|
|
7
|
+
export function chunkText(
|
|
8
|
+
text: string,
|
|
9
|
+
metadata: { filename: string; pageNum?: number }
|
|
10
|
+
): { content: string; metadata: object }[] {
|
|
11
|
+
const paragraphs = text.split(/\n{2,}/).map((p) => p.trim()).filter(Boolean);
|
|
12
|
+
const chunks: { content: string; metadata: object }[] = [];
|
|
13
|
+
|
|
14
|
+
let current = "";
|
|
15
|
+
let chunkIndex = 0;
|
|
16
|
+
|
|
17
|
+
for (const para of paragraphs) {
|
|
18
|
+
if ((current + "\n\n" + para).length > CHUNK_SIZE && current.length > 0) {
|
|
19
|
+
chunks.push({ content: current.trim(), metadata: { ...metadata, chunkIndex } });
|
|
20
|
+
chunkIndex++;
|
|
21
|
+
// Overlap: keep last portion of current chunk
|
|
22
|
+
const overlapText = current.slice(-OVERLAP);
|
|
23
|
+
current = overlapText + "\n\n" + para;
|
|
24
|
+
} else {
|
|
25
|
+
current = current ? current + "\n\n" + para : para;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
if (current.trim()) {
|
|
30
|
+
chunks.push({ content: current.trim(), metadata: { ...metadata, chunkIndex } });
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
return chunks;
|
|
34
|
+
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Embedding provider — uses OpenAI text-embedding-3-small if API key available,
|
|
2
|
+
// otherwise falls back to a simple TF-IDF character n-gram hash embedding.
|
|
3
|
+
import { getSettings } from "../settings";
|
|
4
|
+
|
|
5
|
+
const VECTOR_DIM = 1536; // OpenAI text-embedding-3-small
|
|
6
|
+
|
|
7
|
+
export async function embedText(text: string): Promise<number[]> {
|
|
8
|
+
const settings = getSettings();
|
|
9
|
+
if (settings.apiKeys.openai) {
|
|
10
|
+
return embedWithOpenAI(text, settings.apiKeys.openai);
|
|
11
|
+
}
|
|
12
|
+
return embedFallback(text);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
export function getEmbeddingDim(): number {
|
|
16
|
+
const settings = getSettings();
|
|
17
|
+
return settings.apiKeys.openai ? VECTOR_DIM : 256;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
async function embedWithOpenAI(text: string, apiKey: string): Promise<number[]> {
|
|
21
|
+
const res = await fetch("https://api.openai.com/v1/embeddings", {
|
|
22
|
+
method: "POST",
|
|
23
|
+
headers: {
|
|
24
|
+
"Content-Type": "application/json",
|
|
25
|
+
Authorization: `Bearer ${apiKey}`,
|
|
26
|
+
},
|
|
27
|
+
body: JSON.stringify({ model: "text-embedding-3-small", input: text }),
|
|
28
|
+
});
|
|
29
|
+
if (!res.ok) throw new Error(`OpenAI embedding error: ${res.statusText}`);
|
|
30
|
+
const data = (await res.json()) as { data: { embedding: number[] }[] };
|
|
31
|
+
return data.data[0].embedding;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Lightweight 256-dim character n-gram hash embedding (no external deps)
|
|
35
|
+
function embedFallback(text: string): number[] {
|
|
36
|
+
const dim = 256;
|
|
37
|
+
const vec = new Float32Array(dim);
|
|
38
|
+
const lower = text.toLowerCase();
|
|
39
|
+
for (let i = 0; i < lower.length - 1; i++) {
|
|
40
|
+
const bigram = lower.charCodeAt(i) * 31 + lower.charCodeAt(i + 1);
|
|
41
|
+
vec[bigram % dim] += 1;
|
|
42
|
+
}
|
|
43
|
+
// L2 normalize
|
|
44
|
+
let norm = 0;
|
|
45
|
+
for (let i = 0; i < dim; i++) norm += vec[i] * vec[i];
|
|
46
|
+
norm = Math.sqrt(norm) || 1;
|
|
47
|
+
return Array.from(vec).map((v) => v / norm);
|
|
48
|
+
}
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
// Lightweight cosine-similarity vector search using plain SQLite BLOB storage.
|
|
2
|
+
// Embeddings are stored as Float32Array serialized to Buffer.
|
|
3
|
+
|
|
4
|
+
import { getDb } from "../db/client";
|
|
5
|
+
import { v4 as uuidv4 } from "uuid";
|
|
6
|
+
import type { SearchResult } from "@/types/knowledge";
|
|
7
|
+
|
|
8
|
+
export async function searchChunks(
|
|
9
|
+
kbIds: string[],
|
|
10
|
+
queryEmbedding: number[],
|
|
11
|
+
topK = 5
|
|
12
|
+
): Promise<SearchResult[]> {
|
|
13
|
+
const db = getDb();
|
|
14
|
+
if (kbIds.length === 0) return [];
|
|
15
|
+
|
|
16
|
+
const placeholders = kbIds.map(() => "?").join(",");
|
|
17
|
+
const rows = db
|
|
18
|
+
.prepare(`SELECT * FROM kb_chunks WHERE kb_id IN (${placeholders})`)
|
|
19
|
+
.all(...kbIds) as {
|
|
20
|
+
id: string;
|
|
21
|
+
kb_id: string;
|
|
22
|
+
doc_id: string;
|
|
23
|
+
content: string;
|
|
24
|
+
embedding: Buffer | null;
|
|
25
|
+
metadata: string;
|
|
26
|
+
}[];
|
|
27
|
+
|
|
28
|
+
const results: SearchResult[] = [];
|
|
29
|
+
|
|
30
|
+
for (const row of rows) {
|
|
31
|
+
if (!row.embedding) continue;
|
|
32
|
+
const docEmbedding = bufferToFloat32Array(row.embedding);
|
|
33
|
+
const score = cosineSimilarity(queryEmbedding, docEmbedding);
|
|
34
|
+
const meta = JSON.parse(row.metadata ?? "{}");
|
|
35
|
+
results.push({
|
|
36
|
+
chunk: {
|
|
37
|
+
id: row.id,
|
|
38
|
+
kbId: row.kb_id,
|
|
39
|
+
docId: row.doc_id,
|
|
40
|
+
content: row.content,
|
|
41
|
+
metadata: meta,
|
|
42
|
+
},
|
|
43
|
+
score,
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
results.sort((a, b) => b.score - a.score);
|
|
48
|
+
return results.slice(0, topK);
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
export function storeChunk(
|
|
52
|
+
kbId: string,
|
|
53
|
+
docId: string,
|
|
54
|
+
content: string,
|
|
55
|
+
embedding: number[],
|
|
56
|
+
metadata: object
|
|
57
|
+
): void {
|
|
58
|
+
const db = getDb();
|
|
59
|
+
const id = uuidv4();
|
|
60
|
+
const embBuf = float32ArrayToBuffer(new Float32Array(embedding));
|
|
61
|
+
db.prepare(
|
|
62
|
+
"INSERT INTO kb_chunks (id, kb_id, doc_id, content, embedding, metadata, created_at) VALUES (?, ?, ?, ?, ?, ?, ?)"
|
|
63
|
+
).run(id, kbId, docId, content, embBuf, JSON.stringify(metadata), Date.now());
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
export function deleteChunksByDoc(docId: string): void {
|
|
67
|
+
getDb().prepare("DELETE FROM kb_chunks WHERE doc_id = ?").run(docId);
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function deleteChunksByKb(kbId: string): void {
|
|
71
|
+
getDb().prepare("DELETE FROM kb_chunks WHERE kb_id = ?").run(kbId);
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// --- helpers ---
|
|
75
|
+
|
|
76
|
+
function cosineSimilarity(a: number[], b: Float32Array): number {
|
|
77
|
+
if (a.length !== b.length) return 0;
|
|
78
|
+
let dot = 0, normA = 0, normB = 0;
|
|
79
|
+
for (let i = 0; i < a.length; i++) {
|
|
80
|
+
dot += a[i] * b[i];
|
|
81
|
+
normA += a[i] * a[i];
|
|
82
|
+
normB += b[i] * b[i];
|
|
83
|
+
}
|
|
84
|
+
const denom = Math.sqrt(normA) * Math.sqrt(normB);
|
|
85
|
+
return denom === 0 ? 0 : dot / denom;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
function float32ArrayToBuffer(arr: Float32Array): Buffer {
|
|
89
|
+
return Buffer.from(arr.buffer);
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
function bufferToFloat32Array(buf: Buffer): Float32Array {
|
|
93
|
+
return new Float32Array(buf.buffer, buf.byteOffset, buf.byteLength / 4);
|
|
94
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { getDb } from "../db/client";
|
|
2
|
+
import type { KnowledgeBase, KbDocument } from "@/types/knowledge";
|
|
3
|
+
import { v4 as uuidv4 } from "uuid";
|
|
4
|
+
|
|
5
|
+
type KbRow = {
|
|
6
|
+
id: string; name: string; description: string;
|
|
7
|
+
embedding_model: string; created_at: number; updated_at: number;
|
|
8
|
+
};
|
|
9
|
+
type DocRow = {
|
|
10
|
+
id: string; kb_id: string; filename: string; file_path: string;
|
|
11
|
+
mime_type: string; chunk_count: number; status: string; error: string | null;
|
|
12
|
+
created_at: number;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const kbStore = {
|
|
16
|
+
list(): KnowledgeBase[] {
|
|
17
|
+
const db = getDb();
|
|
18
|
+
const rows = db.prepare("SELECT kb.*, (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id) as doc_count, (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) as chunk_count FROM knowledge_bases kb ORDER BY kb.created_at DESC").all() as (KbRow & { doc_count: number; chunk_count: number })[];
|
|
19
|
+
return rows.map((r) => ({
|
|
20
|
+
id: r.id, name: r.name, description: r.description,
|
|
21
|
+
embeddingModel: r.embedding_model,
|
|
22
|
+
documentCount: r.doc_count, chunkCount: r.chunk_count,
|
|
23
|
+
createdAt: new Date(r.created_at).toISOString(),
|
|
24
|
+
updatedAt: new Date(r.updated_at).toISOString(),
|
|
25
|
+
}));
|
|
26
|
+
},
|
|
27
|
+
|
|
28
|
+
getById(id: string): KnowledgeBase | undefined {
|
|
29
|
+
const db = getDb();
|
|
30
|
+
const r = db.prepare("SELECT kb.*, (SELECT COUNT(*) FROM kb_documents d WHERE d.kb_id = kb.id) as doc_count, (SELECT COUNT(*) FROM kb_chunks c WHERE c.kb_id = kb.id) as chunk_count FROM knowledge_bases kb WHERE kb.id = ?").get(id) as (KbRow & { doc_count: number; chunk_count: number }) | undefined;
|
|
31
|
+
if (!r) return undefined;
|
|
32
|
+
return { id: r.id, name: r.name, description: r.description, embeddingModel: r.embedding_model, documentCount: r.doc_count, chunkCount: r.chunk_count, createdAt: new Date(r.created_at).toISOString(), updatedAt: new Date(r.updated_at).toISOString() };
|
|
33
|
+
},
|
|
34
|
+
|
|
35
|
+
create(name: string, description = ""): KnowledgeBase {
|
|
36
|
+
const db = getDb();
|
|
37
|
+
const id = uuidv4();
|
|
38
|
+
const now = Date.now();
|
|
39
|
+
db.prepare("INSERT INTO knowledge_bases (id, name, description, embedding_model, created_at, updated_at) VALUES (?, ?, ?, 'auto', ?, ?)").run(id, name, description, now, now);
|
|
40
|
+
return this.getById(id)!;
|
|
41
|
+
},
|
|
42
|
+
|
|
43
|
+
delete(id: string): boolean {
|
|
44
|
+
const result = getDb().prepare("DELETE FROM knowledge_bases WHERE id = ?").run(id);
|
|
45
|
+
return result.changes > 0;
|
|
46
|
+
},
|
|
47
|
+
|
|
48
|
+
listDocuments(kbId: string): KbDocument[] {
|
|
49
|
+
const db = getDb();
|
|
50
|
+
const rows = db.prepare("SELECT * FROM kb_documents WHERE kb_id = ? ORDER BY created_at DESC").all(kbId) as DocRow[];
|
|
51
|
+
return rows.map((r) => ({
|
|
52
|
+
id: r.id, kbId: r.kb_id, filename: r.filename, filePath: r.file_path,
|
|
53
|
+
mimeType: r.mime_type, chunkCount: r.chunk_count,
|
|
54
|
+
status: r.status as KbDocument["status"], error: r.error ?? undefined,
|
|
55
|
+
createdAt: new Date(r.created_at).toISOString(),
|
|
56
|
+
}));
|
|
57
|
+
},
|
|
58
|
+
|
|
59
|
+
createDocument(kbId: string, filename: string, filePath: string, mimeType: string): string {
|
|
60
|
+
const db = getDb();
|
|
61
|
+
const id = uuidv4();
|
|
62
|
+
db.prepare("INSERT INTO kb_documents (id, kb_id, filename, file_path, mime_type, chunk_count, status, created_at) VALUES (?, ?, ?, ?, ?, 0, 'processing', ?)").run(id, kbId, filename, filePath, mimeType, Date.now());
|
|
63
|
+
return id;
|
|
64
|
+
},
|
|
65
|
+
|
|
66
|
+
updateDocumentStatus(id: string, status: string, chunkCount = 0, error?: string): void {
|
|
67
|
+
getDb().prepare("UPDATE kb_documents SET status = ?, chunk_count = ?, error = ? WHERE id = ?").run(status, chunkCount, error ?? null, id);
|
|
68
|
+
},
|
|
69
|
+
|
|
70
|
+
deleteDocument(id: string): boolean {
|
|
71
|
+
const result = getDb().prepare("DELETE FROM kb_documents WHERE id = ?").run(id);
|
|
72
|
+
return result.changes > 0;
|
|
73
|
+
},
|
|
74
|
+
};
|