@customize-agent/knowledge 1.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/chunking/text-chunker.d.ts +24 -0
- package/dist/chunking/text-chunker.js +90 -0
- package/dist/classification/classifier.d.ts +12 -0
- package/dist/classification/classifier.js +95 -0
- package/dist/constants.d.ts +8 -0
- package/dist/constants.js +40 -0
- package/dist/core/change-tracker.d.ts +10 -0
- package/dist/core/change-tracker.js +66 -0
- package/dist/core/file-scanner.d.ts +8 -0
- package/dist/core/file-scanner.js +32 -0
- package/dist/core/index-state-store.d.ts +103 -0
- package/dist/core/index-state-store.js +439 -0
- package/dist/core/knowledge-base-manager.d.ts +76 -0
- package/dist/core/knowledge-base-manager.js +300 -0
- package/dist/core/multi-project-manager.d.ts +30 -0
- package/dist/core/multi-project-manager.js +163 -0
- package/dist/core/project-config.d.ts +9 -0
- package/dist/core/project-config.js +73 -0
- package/dist/core/project-id.d.ts +1 -0
- package/dist/core/project-id.js +8 -0
- package/dist/core/project-registry.d.ts +10 -0
- package/dist/core/project-registry.js +70 -0
- package/dist/dedup/dedup-engine.d.ts +20 -0
- package/dist/dedup/dedup-engine.js +78 -0
- package/dist/dedup/relationship-detector.d.ts +10 -0
- package/dist/dedup/relationship-detector.js +84 -0
- package/dist/embedding/embedding-provider.d.ts +15 -0
- package/dist/embedding/embedding-provider.js +31 -0
- package/dist/extraction/content-extractor.d.ts +36 -0
- package/dist/extraction/content-extractor.js +655 -0
- package/dist/extraction/external-extractor.d.ts +63 -0
- package/dist/extraction/external-extractor.js +139 -0
- package/dist/index.d.ts +23 -0
- package/dist/index.js +22 -0
- package/dist/search/federation-search.d.ts +41 -0
- package/dist/search/federation-search.js +102 -0
- package/dist/server/dashboard-client.d.ts +2 -0
- package/dist/server/dashboard-client.js +396 -0
- package/dist/server/dashboard-i18n.d.ts +112 -0
- package/dist/server/dashboard-i18n.js +220 -0
- package/dist/server/dashboard-page.d.ts +6 -0
- package/dist/server/dashboard-page.js +138 -0
- package/dist/server/dashboard-server.d.ts +13 -0
- package/dist/server/dashboard-server.js +225 -0
- package/dist/server/dashboard-styles.d.ts +1 -0
- package/dist/server/dashboard-styles.js +152 -0
- package/dist/types.d.ts +82 -0
- package/dist/types.js +1 -0
- package/dist/vector/chroma-store.d.ts +39 -0
- package/dist/vector/chroma-store.js +131 -0
- package/dist/vector/collection-manager.d.ts +16 -0
- package/dist/vector/collection-manager.js +77 -0
- package/dist/vector/types.d.ts +33 -0
- package/dist/vector/types.js +1 -0
- package/dist/vector/vector-indexer.d.ts +18 -0
- package/dist/vector/vector-indexer.js +61 -0
- package/package.json +45 -0
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
export const dashboardStyles = `
|
|
2
|
+
:root {
|
|
3
|
+
color-scheme: light;
|
|
4
|
+
--bg:#f5f5f7;
|
|
5
|
+
--bg-glass:rgba(255,255,255,.72);
|
|
6
|
+
--panel:rgba(255,255,255,.86);
|
|
7
|
+
--panel-solid:#ffffff;
|
|
8
|
+
--line:rgba(60,60,67,.16);
|
|
9
|
+
--line-strong:rgba(60,60,67,.26);
|
|
10
|
+
--text:#1d1d1f;
|
|
11
|
+
--muted:#6e6e73;
|
|
12
|
+
--accent:#007aff;
|
|
13
|
+
--accent-2:#5856d6;
|
|
14
|
+
--danger:#ff3b30;
|
|
15
|
+
--ok:#34c759;
|
|
16
|
+
--warn:#ff9500;
|
|
17
|
+
--shadow:0 18px 45px rgba(0,0,0,.08);
|
|
18
|
+
--shadow-soft:0 8px 24px rgba(0,0,0,.06);
|
|
19
|
+
}
|
|
20
|
+
* { box-sizing: border-box; }
|
|
21
|
+
html { background:var(--bg); }
|
|
22
|
+
body {
|
|
23
|
+
margin:0; min-height:100vh; font-family: -apple-system, BlinkMacSystemFont, "SF Pro Display", "SF Pro Text", "Segoe UI", sans-serif;
|
|
24
|
+
background:
|
|
25
|
+
radial-gradient(circle at 15% 0%, rgba(0,122,255,.12), transparent 28rem),
|
|
26
|
+
radial-gradient(circle at 85% 8%, rgba(88,86,214,.10), transparent 26rem),
|
|
27
|
+
linear-gradient(180deg, #fbfbfd 0%, var(--bg) 100%);
|
|
28
|
+
color:var(--text);
|
|
29
|
+
}
|
|
30
|
+
header {
|
|
31
|
+
position: sticky; top:0; z-index:10; backdrop-filter: saturate(180%) blur(22px); -webkit-backdrop-filter: saturate(180%) blur(22px);
|
|
32
|
+
padding:20px max(32px, calc((100vw - 1480px) / 2 + 32px)); border-bottom:1px solid var(--line); display:flex; align-items:center; justify-content:space-between; gap:18px;
|
|
33
|
+
background:var(--bg-glass);
|
|
34
|
+
}
|
|
35
|
+
main { padding:28px 32px 40px; max-width:1480px; margin:0 auto; }
|
|
36
|
+
main, aside, main > div { min-width:0; }
|
|
37
|
+
.page { display:none; }
|
|
38
|
+
.page.active { display:block; }
|
|
39
|
+
.topbar { flex-wrap:nowrap; justify-content:flex-start; overflow-x:auto; padding-bottom:2px; scrollbar-width:none; }
|
|
40
|
+
.topbar::-webkit-scrollbar { display:none; }
|
|
41
|
+
.topbar > * { flex:0 0 auto; }
|
|
42
|
+
.tab { background:rgba(255,255,255,.72); border-radius:999px; padding:9px 15px; color:var(--muted); }
|
|
43
|
+
.tab.active { background:#1d1d1f; color:white; border-color:#1d1d1f; }
|
|
44
|
+
section {
|
|
45
|
+
position:relative; overflow:visible;
|
|
46
|
+
background:var(--panel); border:1px solid rgba(255,255,255,.7); border-radius:26px; padding:20px; box-shadow:var(--shadow-soft);
|
|
47
|
+
backdrop-filter: blur(16px); -webkit-backdrop-filter: blur(16px);
|
|
48
|
+
}
|
|
49
|
+
section:has(.custom-select.open), section.select-host-open { z-index:10000; }
|
|
50
|
+
section:hover { box-shadow:var(--shadow); transform:translateY(-1px); transition:.2s ease; }
|
|
51
|
+
h1 { font-size:24px; margin:0; letter-spacing:-.02em; font-weight:700; }
|
|
52
|
+
h2 { font-size:12px; margin:0 0 14px; color:var(--muted); text-transform:uppercase; letter-spacing:.12em; font-weight:700; }
|
|
53
|
+
h3 { font-size:15px; margin:14px 0 8px; }
|
|
54
|
+
.subtitle { margin-top:6px; color:var(--muted); font-size:14px; }
|
|
55
|
+
button, input, select {
|
|
56
|
+
min-width:0; background:rgba(255,255,255,.88); color:var(--text); border:1px solid var(--line); border-radius:14px; padding:10px 13px; outline:none;
|
|
57
|
+
font: inherit; box-shadow: inset 0 0 0 1px rgba(255,255,255,.25);
|
|
58
|
+
}
|
|
59
|
+
select {
|
|
60
|
+
min-width:132px; appearance:none; -webkit-appearance:none; padding-right:34px;
|
|
61
|
+
background-image:linear-gradient(45deg, transparent 50%, #6e6e73 50%), linear-gradient(135deg, #6e6e73 50%, transparent 50%);
|
|
62
|
+
background-position:calc(100% - 18px) 50%, calc(100% - 12px) 50%; background-size:6px 6px, 6px 6px; background-repeat:no-repeat;
|
|
63
|
+
}
|
|
64
|
+
button { cursor:pointer; white-space:nowrap; transition:.16s ease; font-weight:600; }
|
|
65
|
+
button:hover { border-color:rgba(0,122,255,.35); transform:translateY(-1px); box-shadow:0 8px 18px rgba(0,122,255,.12); }
|
|
66
|
+
button.primary { background:linear-gradient(180deg, #0a84ff, #007aff); border-color:transparent; color:white; box-shadow:0 10px 22px rgba(0,122,255,.24); }
|
|
67
|
+
input { width:100%; }
|
|
68
|
+
input::placeholder { color:#a1a1a6; }
|
|
69
|
+
input:focus, select:focus { border-color:rgba(0,122,255,.55); box-shadow:0 0 0 4px rgba(0,122,255,.12); }
|
|
70
|
+
.row { display:flex; gap:10px; margin-bottom:10px; align-items:center; }
|
|
71
|
+
.row input { flex:1; }
|
|
72
|
+
.stack { display:grid; gap:10px; }
|
|
73
|
+
.stack.compact { gap:8px; }
|
|
74
|
+
.field-row { display:grid; grid-template-columns:minmax(0, 1fr) auto; gap:8px; align-items:center; }
|
|
75
|
+
.field-row:has(input + input + button) { grid-template-columns:minmax(0, 1fr) minmax(0, 1fr) auto; }
|
|
76
|
+
.helper-title { font-size:15px; font-weight:700; color:var(--text); margin:2px 0 6px; }
|
|
77
|
+
.helper-text { color:var(--muted); font-size:13px; line-height:1.6; margin-bottom:12px; overflow-wrap:anywhere; }
|
|
78
|
+
.upload-card { display:flex; gap:14px; align-items:center; border:1.5px dashed rgba(0,122,255,.32); background:rgba(0,122,255,.06); border-radius:20px; padding:16px; cursor:pointer; margin-bottom:10px; }
|
|
79
|
+
.upload-card:hover { background:rgba(0,122,255,.10); border-color:rgba(0,122,255,.55); }
|
|
80
|
+
button.loading, .upload-card.loading { position:relative; pointer-events:none; opacity:.82; cursor:progress; }
|
|
81
|
+
button.loading::after, .upload-card.loading::after { content:''; width:14px; height:14px; border:2px solid currentColor; border-top-color:transparent; border-radius:50%; display:inline-block; margin-left:8px; vertical-align:-2px; animation:spin .8s linear infinite; }
|
|
82
|
+
.upload-card.loading::after { margin-left:auto; color:#007aff; width:18px; height:18px; }
|
|
83
|
+
button:disabled { cursor:progress; }
|
|
84
|
+
@keyframes spin { to { transform:rotate(360deg); } }
|
|
85
|
+
.upload-card input { display:none; }
|
|
86
|
+
.upload-card b { display:block; font-size:15px; }
|
|
87
|
+
.upload-card span { display:block; color:var(--muted); font-size:12px; line-height:1.45; margin-top:3px; overflow-wrap:anywhere; }
|
|
88
|
+
.upload-icon { width:38px; height:38px; flex:0 0 38px; border-radius:50%; display:grid; place-items:center; background:#007aff; color:white; font-weight:800; font-size:20px; }
|
|
89
|
+
.advanced-block { margin-top:12px; border:1px solid var(--line); border-radius:16px; padding:10px 12px; background:rgba(255,255,255,.55); }
|
|
90
|
+
.advanced-block summary { cursor:pointer; color:var(--muted); font-size:13px; font-weight:600; margin-bottom:8px; }
|
|
91
|
+
.divider { height:1px; background:var(--line); margin:16px 0; }
|
|
92
|
+
.search-bar { display:grid; grid-template-columns: 170px minmax(0, 1fr) auto; gap:10px; align-items:end; }
|
|
93
|
+
.filter-bar { display:grid; grid-template-columns: repeat(2, minmax(180px, 260px)); gap:10px; align-items:end; margin:12px 0 14px; }
|
|
94
|
+
.select-wrap { position:relative; display:grid; gap:5px; color:var(--muted); font-size:12px; font-weight:600; min-width:0; overflow:visible; }
|
|
95
|
+
.select-wrap.compact { min-width:130px; }
|
|
96
|
+
.language-select { width:96px; min-width:96px; }
|
|
97
|
+
.language-select select { min-width:96px; width:96px; cursor:pointer; }
|
|
98
|
+
.select-wrap select { width:100%; }
|
|
99
|
+
.select-native-hidden { position:absolute !important; opacity:0 !important; pointer-events:none !important; width:1px !important; height:1px !important; }
|
|
100
|
+
.custom-select { position:relative; min-width:0; z-index:1; }
|
|
101
|
+
.custom-select.open { z-index:10001; }
|
|
102
|
+
.custom-select-trigger { width:100%; justify-content:space-between; text-align:left; display:flex; align-items:center; gap:10px; background:rgba(255,255,255,.88); color:var(--text); border:1px solid var(--line); border-radius:14px; padding:10px 13px; box-shadow: inset 0 0 0 1px rgba(255,255,255,.25); }
|
|
103
|
+
.custom-select-trigger::after { content:'⌄'; color:var(--muted); font-size:13px; line-height:1; display:inline-flex; align-items:center; justify-content:center; transform:translateY(-1px); flex:0 0 auto; }
|
|
104
|
+
.custom-select.open .custom-select-trigger { border-color:rgba(0,122,255,.55); box-shadow:0 0 0 4px rgba(0,122,255,.12); }
|
|
105
|
+
.custom-select-menu { display:none; position:absolute; top:calc(100% + 6px); left:0; right:0; z-index:10002; max-height:260px; overflow:auto; padding:6px; border:1px solid var(--line); border-radius:16px; background:rgba(255,255,255,.98); box-shadow:0 18px 40px rgba(0,0,0,.16); backdrop-filter:blur(16px); }
|
|
106
|
+
.custom-select.open .custom-select-menu { display:block; }
|
|
107
|
+
.custom-select-option { display:block; width:100%; border:0; box-shadow:none; background:transparent; text-align:left; color:var(--text); border-radius:10px; padding:9px 10px; white-space:normal; }
|
|
108
|
+
.custom-select-option:hover, .custom-select-option.active { background:rgba(0,122,255,.10); color:var(--accent); transform:none; box-shadow:none; }
|
|
109
|
+
.config-row { display:grid; grid-template-columns:90px minmax(0, 1fr); gap:10px; align-items:start; padding:8px 0; border-top:1px solid var(--line); }
|
|
110
|
+
.config-row:first-child { border-top:0; }
|
|
111
|
+
.config-row b, .config-row span { min-width:0; overflow-wrap:anywhere; line-height:1.5; }
|
|
112
|
+
.file-group { border:1px solid var(--line); border-radius:18px; background:rgba(255,255,255,.58); margin-top:10px; overflow:hidden; }
|
|
113
|
+
.file-group summary { cursor:pointer; padding:12px 14px; font-weight:700; display:flex; justify-content:space-between; align-items:center; }
|
|
114
|
+
.file-group summary span { color:var(--muted); font-size:12px; }
|
|
115
|
+
.file-item { display:grid; grid-template-columns:minmax(0, 1fr) auto; gap:12px; padding:12px 14px; border-top:1px solid var(--line); align-items:center; }
|
|
116
|
+
.file-main { min-width:0; }
|
|
117
|
+
.file-actions { display:flex; gap:6px; flex-wrap:wrap; justify-content:flex-end; }
|
|
118
|
+
.file-actions button { padding:7px 9px; font-size:12px; }
|
|
119
|
+
.grid { display:grid; gap:20px; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
|
120
|
+
.panel-gap { margin-top:20px; }
|
|
121
|
+
.stat-grid { display:grid; grid-template-columns: repeat(3, 1fr); gap:10px; margin:10px 0 12px; }
|
|
122
|
+
.metric { background:linear-gradient(180deg, #fff, #f7f7fa); border:1px solid var(--line); border-radius:18px; padding:14px; }
|
|
123
|
+
.metric b { display:block; font-size:24px; margin-top:4px; letter-spacing:-.03em; }
|
|
124
|
+
.metric span { color:var(--muted); font-size:12px; }
|
|
125
|
+
.stat { display:flex; justify-content:space-between; align-items:flex-start; color:var(--muted); margin:8px 0; gap:12px; }
|
|
126
|
+
.stat b { color:var(--text); }
|
|
127
|
+
.card { border-top:1px solid var(--line); padding:14px 0; }
|
|
128
|
+
.card:first-child { border-top:0; padding-top:0; }
|
|
129
|
+
.path { color:var(--accent); font-size:13px; overflow-wrap:anywhere; word-break:normal; font-weight:600; line-height:1.45; }
|
|
130
|
+
.meta { color:var(--muted); font-size:12px; overflow-wrap:anywhere; word-break:normal; line-height:1.55; min-width:0; }
|
|
131
|
+
.badge { display:inline-flex; align-items:center; border:1px solid var(--line); border-radius:999px; padding:3px 9px; font-size:12px; color:var(--muted); margin:3px 5px 3px 0; background:rgba(255,255,255,.7); }
|
|
132
|
+
.badge.ok { color:#1d7f3a; border-color:rgba(52,199,89,.28); background:rgba(52,199,89,.10); }
|
|
133
|
+
.badge.warn { color:#9a5a00; border-color:rgba(255,149,0,.30); background:rgba(255,149,0,.10); }
|
|
134
|
+
#capabilities { max-height: 460px; overflow:auto; padding-right:4px; }
|
|
135
|
+
#results .card { background:var(--panel-solid); border-radius:20px; border:1px solid var(--line); margin-top:12px; padding:16px; box-shadow:var(--shadow-soft); }
|
|
136
|
+
.status { color:var(--muted); min-height:20px; font-size:12px; margin-top:8px; }
|
|
137
|
+
.status.ok { color:var(--ok); }
|
|
138
|
+
.status.error { color:var(--danger); }
|
|
139
|
+
.toolbar { display:flex; gap:10px; align-items:center; justify-content:flex-end; }
|
|
140
|
+
pre { white-space:pre-wrap; word-break:break-word; color:#3a3a3c; margin:10px 0 0; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size:12px; line-height:1.55; }
|
|
141
|
+
.subsection { margin-top:18px; padding-top:16px; border-top:1px solid var(--line); }
|
|
142
|
+
.modal-backdrop { position:fixed; inset:0; z-index:20000; display:grid; place-items:center; background:rgba(15,23,42,.35); backdrop-filter:blur(10px); }
|
|
143
|
+
.modal-backdrop[hidden] { display:none; }
|
|
144
|
+
.modal-card { width:min(520px, calc(100vw - 32px)); background:rgba(255,255,255,.98); border:1px solid var(--line); border-radius:24px; box-shadow:0 28px 70px rgba(0,0,0,.22); padding:22px; }
|
|
145
|
+
.modal-card h3 { margin:0 0 8px; }
|
|
146
|
+
.modal-card p { white-space:pre-wrap; color:var(--muted); margin:0 0 14px; }
|
|
147
|
+
.modal-input { width:100%; box-sizing:border-box; margin:0 0 14px; border:1px solid var(--line); border-radius:14px; padding:11px 12px; font:inherit; }
|
|
148
|
+
.modal-actions { display:flex; justify-content:flex-end; gap:10px; }
|
|
149
|
+
button.danger { background:linear-gradient(135deg,#ff3b30,#ff6b5f); color:white; border-color:rgba(255,59,48,.25); }
|
|
150
|
+
@media (max-width: 1080px) { .grid { grid-template-columns:1fr; } }
|
|
151
|
+
@media (max-width: 680px) { main { padding:16px; } header { padding:16px; align-items:flex-start; flex-direction:column; } .toolbar { width:100%; justify-content:flex-start; flex-wrap:wrap; } .row { flex-direction:column; align-items:stretch; } .search-bar, .filter-bar { grid-template-columns:1fr; } select { width:100%; } .field-row, .field-row:has(input + input + button), .file-item, .config-row { grid-template-columns:1fr; } .file-actions { justify-content:flex-start; } .stat-grid { grid-template-columns:1fr; } .upload-card { align-items:flex-start; } }
|
|
152
|
+
`;
|
package/dist/types.d.ts
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
export type KBScope = 'project' | 'global' | 'session';
|
|
2
|
+
export type FileCategory = 'document' | 'spreadsheet' | 'image' | 'cad' | 'code' | 'data' | 'web' | 'diagram' | 'archive' | 'other';
|
|
3
|
+
export type ProjectStatus = 'active' | 'idle' | 'error';
|
|
4
|
+
export interface ProjectInfo {
|
|
5
|
+
projectId: string;
|
|
6
|
+
projectRoot: string;
|
|
7
|
+
projectName?: string;
|
|
8
|
+
kbPath: string;
|
|
9
|
+
fileCount: number;
|
|
10
|
+
chunkCount: number;
|
|
11
|
+
totalSizeBytes: number;
|
|
12
|
+
lastIndexedAt: number;
|
|
13
|
+
lastOpenedAt: number;
|
|
14
|
+
status: ProjectStatus;
|
|
15
|
+
}
|
|
16
|
+
export interface ProjectConfig {
|
|
17
|
+
projectId: string;
|
|
18
|
+
projectName?: string;
|
|
19
|
+
enabled: boolean;
|
|
20
|
+
includeGlobal: boolean;
|
|
21
|
+
priorityOverGlobal: boolean;
|
|
22
|
+
watch: boolean;
|
|
23
|
+
autoIndex: boolean;
|
|
24
|
+
kbignore: string[];
|
|
25
|
+
projectTags: string[];
|
|
26
|
+
categoryDirs: Partial<Record<FileCategory, string>>;
|
|
27
|
+
createdAt: number;
|
|
28
|
+
lastOpenedAt: number;
|
|
29
|
+
}
|
|
30
|
+
export interface ClassifiedFile {
|
|
31
|
+
absolutePath: string;
|
|
32
|
+
relativePath: string;
|
|
33
|
+
category: FileCategory;
|
|
34
|
+
format: string;
|
|
35
|
+
fileSize: number;
|
|
36
|
+
mtime: number;
|
|
37
|
+
mimeType: string;
|
|
38
|
+
}
|
|
39
|
+
export interface IndexStateRecord {
|
|
40
|
+
relativePath: string;
|
|
41
|
+
category: FileCategory;
|
|
42
|
+
format: string;
|
|
43
|
+
contentHash: string;
|
|
44
|
+
fileSize: number;
|
|
45
|
+
mtime: number;
|
|
46
|
+
chunkCount: number;
|
|
47
|
+
collectionName: string;
|
|
48
|
+
indexedAt: number;
|
|
49
|
+
lastVerifiedAt: number;
|
|
50
|
+
status: 'active' | 'outdated' | 'error' | 'deleted';
|
|
51
|
+
errorMessage?: string;
|
|
52
|
+
metadataJson?: string;
|
|
53
|
+
}
|
|
54
|
+
export interface DiffResult {
|
|
55
|
+
newFiles: ClassifiedFile[];
|
|
56
|
+
modifiedFiles: ClassifiedFile[];
|
|
57
|
+
deletedFiles: IndexStateRecord[];
|
|
58
|
+
unchangedCount: number;
|
|
59
|
+
mtimeOnlyCount: number;
|
|
60
|
+
skippedFiles: Array<{
|
|
61
|
+
file: ClassifiedFile;
|
|
62
|
+
reason: string;
|
|
63
|
+
}>;
|
|
64
|
+
hasChanges: boolean;
|
|
65
|
+
diffTimeMs: number;
|
|
66
|
+
}
|
|
67
|
+
export interface KnowledgeBaseStats {
|
|
68
|
+
scope: Exclude<KBScope, 'session'>;
|
|
69
|
+
projectId?: string;
|
|
70
|
+
fileCount: number;
|
|
71
|
+
chunkCount: number;
|
|
72
|
+
totalSizeBytes: number;
|
|
73
|
+
lastIndexedAt: number;
|
|
74
|
+
}
|
|
75
|
+
export interface CrossProjectDuplicate {
|
|
76
|
+
contentHash: string;
|
|
77
|
+
files: Array<{
|
|
78
|
+
projectId: string;
|
|
79
|
+
projectRoot: string;
|
|
80
|
+
relativePath: string;
|
|
81
|
+
}>;
|
|
82
|
+
}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
import type { CollectionClient, VectorCollectionInfo, VectorDocument, VectorSearchQuery, VectorSearchResult, VectorStoreInterface } from './types.js';
|
|
2
|
+
interface ChromaQueryResponse {
|
|
3
|
+
ids?: string[][];
|
|
4
|
+
documents?: string[][];
|
|
5
|
+
metadatas?: Array<Array<Record<string, unknown>>>;
|
|
6
|
+
distances?: number[][];
|
|
7
|
+
}
|
|
8
|
+
export interface ChromaClientOptions {
|
|
9
|
+
baseUrl?: string;
|
|
10
|
+
tenant?: string;
|
|
11
|
+
database?: string;
|
|
12
|
+
}
|
|
13
|
+
export declare class ChromaHttpClient implements CollectionClient {
|
|
14
|
+
readonly baseUrl: string;
|
|
15
|
+
readonly tenant: string;
|
|
16
|
+
readonly database: string;
|
|
17
|
+
constructor(options?: ChromaClientOptions);
|
|
18
|
+
heartbeat(): Promise<boolean>;
|
|
19
|
+
getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
|
|
20
|
+
listCollections(): Promise<VectorCollectionInfo[]>;
|
|
21
|
+
deleteCollection(name: string): Promise<void>;
|
|
22
|
+
upsert(collectionName: string, documents: VectorDocument[]): Promise<void>;
|
|
23
|
+
deleteWhere(collectionName: string, where: Record<string, string | number | boolean>): Promise<void>;
|
|
24
|
+
query(collectionName: string, query: VectorSearchQuery): Promise<ChromaQueryResponse>;
|
|
25
|
+
private collectionsPath;
|
|
26
|
+
private request;
|
|
27
|
+
private toCollectionInfo;
|
|
28
|
+
}
|
|
29
|
+
export declare class ChromaVectorStore implements VectorStoreInterface {
|
|
30
|
+
private readonly client;
|
|
31
|
+
readonly collectionName: string;
|
|
32
|
+
constructor(client: ChromaHttpClient, collectionName: string);
|
|
33
|
+
ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
|
|
34
|
+
upsert(documents: VectorDocument[]): Promise<void>;
|
|
35
|
+
deleteByFilePath(filePath: string): Promise<void>;
|
|
36
|
+
search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
|
|
37
|
+
private toMetadata;
|
|
38
|
+
}
|
|
39
|
+
export {};
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
export class ChromaHttpClient {
|
|
2
|
+
baseUrl;
|
|
3
|
+
tenant;
|
|
4
|
+
database;
|
|
5
|
+
constructor(options = {}) {
|
|
6
|
+
this.baseUrl = options.baseUrl ?? 'http://localhost:8000';
|
|
7
|
+
this.tenant = options.tenant ?? 'default_tenant';
|
|
8
|
+
this.database = options.database ?? 'default_database';
|
|
9
|
+
}
|
|
10
|
+
async heartbeat() {
|
|
11
|
+
try {
|
|
12
|
+
await this.request('/api/v1/heartbeat');
|
|
13
|
+
return true;
|
|
14
|
+
}
|
|
15
|
+
catch {
|
|
16
|
+
return false;
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
async getOrCreateCollection(name, metadata = {}) {
|
|
20
|
+
const response = await this.request(this.collectionsPath(), {
|
|
21
|
+
method: 'POST',
|
|
22
|
+
body: JSON.stringify({ name, metadata, get_or_create: true }),
|
|
23
|
+
});
|
|
24
|
+
return this.toCollectionInfo(response);
|
|
25
|
+
}
|
|
26
|
+
async listCollections() {
|
|
27
|
+
const response = await this.request(this.collectionsPath());
|
|
28
|
+
return response.map(collection => this.toCollectionInfo(collection));
|
|
29
|
+
}
|
|
30
|
+
async deleteCollection(name) {
|
|
31
|
+
await this.request(`${this.collectionsPath()}/${encodeURIComponent(name)}`, { method: 'DELETE' });
|
|
32
|
+
}
|
|
33
|
+
async upsert(collectionName, documents) {
|
|
34
|
+
if (documents.length === 0)
|
|
35
|
+
return;
|
|
36
|
+
await this.getOrCreateCollection(collectionName);
|
|
37
|
+
await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/upsert`, {
|
|
38
|
+
method: 'POST',
|
|
39
|
+
body: JSON.stringify({
|
|
40
|
+
ids: documents.map(document => document.id),
|
|
41
|
+
embeddings: documents.map(document => document.embedding),
|
|
42
|
+
documents: documents.map(document => document.content),
|
|
43
|
+
metadatas: documents.map(document => document.metadata),
|
|
44
|
+
}),
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
async deleteWhere(collectionName, where) {
|
|
48
|
+
await this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/delete`, {
|
|
49
|
+
method: 'POST',
|
|
50
|
+
body: JSON.stringify({ where }),
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
async query(collectionName, query) {
|
|
54
|
+
return this.request(`${this.collectionsPath()}/${encodeURIComponent(collectionName)}/query`, {
|
|
55
|
+
method: 'POST',
|
|
56
|
+
body: JSON.stringify({
|
|
57
|
+
query_embeddings: [query.queryEmbedding],
|
|
58
|
+
n_results: query.topK,
|
|
59
|
+
where: query.where,
|
|
60
|
+
include: ['documents', 'metadatas', 'distances'],
|
|
61
|
+
}),
|
|
62
|
+
});
|
|
63
|
+
}
|
|
64
|
+
collectionsPath() {
|
|
65
|
+
return `/api/v1/tenants/${encodeURIComponent(this.tenant)}/databases/${encodeURIComponent(this.database)}/collections`;
|
|
66
|
+
}
|
|
67
|
+
async request(path, init = {}) {
|
|
68
|
+
const response = await fetch(`${this.baseUrl}${path}`, {
|
|
69
|
+
...init,
|
|
70
|
+
headers: {
|
|
71
|
+
'content-type': 'application/json',
|
|
72
|
+
...init.headers,
|
|
73
|
+
},
|
|
74
|
+
});
|
|
75
|
+
if (!response.ok) {
|
|
76
|
+
throw new Error(`ChromaDB request failed: ${response.status} ${response.statusText}`);
|
|
77
|
+
}
|
|
78
|
+
if (response.status === 204)
|
|
79
|
+
return undefined;
|
|
80
|
+
return await response.json();
|
|
81
|
+
}
|
|
82
|
+
toCollectionInfo(collection) {
|
|
83
|
+
return {
|
|
84
|
+
id: collection.id,
|
|
85
|
+
name: collection.name,
|
|
86
|
+
metadata: collection.metadata,
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
export class ChromaVectorStore {
|
|
91
|
+
client;
|
|
92
|
+
collectionName;
|
|
93
|
+
constructor(client, collectionName) {
|
|
94
|
+
this.client = client;
|
|
95
|
+
this.collectionName = collectionName;
|
|
96
|
+
}
|
|
97
|
+
async ensureCollection(metadata) {
|
|
98
|
+
await this.client.getOrCreateCollection(this.collectionName, metadata);
|
|
99
|
+
}
|
|
100
|
+
async upsert(documents) {
|
|
101
|
+
await this.client.upsert(this.collectionName, documents);
|
|
102
|
+
}
|
|
103
|
+
async deleteByFilePath(filePath) {
|
|
104
|
+
await this.client.deleteWhere(this.collectionName, { file_path: filePath });
|
|
105
|
+
}
|
|
106
|
+
async search(query) {
|
|
107
|
+
const response = await this.client.query(this.collectionName, query);
|
|
108
|
+
const ids = response.ids?.[0] ?? [];
|
|
109
|
+
const documents = response.documents?.[0] ?? [];
|
|
110
|
+
const metadatas = response.metadatas?.[0] ?? [];
|
|
111
|
+
const distances = response.distances?.[0] ?? [];
|
|
112
|
+
return ids.map((id, index) => ({
|
|
113
|
+
collection: this.collectionName,
|
|
114
|
+
score: 1 - Number(distances[index] ?? 1),
|
|
115
|
+
document: {
|
|
116
|
+
id,
|
|
117
|
+
content: documents[index] ?? '',
|
|
118
|
+
metadata: this.toMetadata(metadatas[index] ?? {}),
|
|
119
|
+
},
|
|
120
|
+
}));
|
|
121
|
+
}
|
|
122
|
+
toMetadata(metadata) {
|
|
123
|
+
const result = {};
|
|
124
|
+
for (const [key, value] of Object.entries(metadata)) {
|
|
125
|
+
if (typeof value === 'string' || typeof value === 'number' || typeof value === 'boolean' || value === null) {
|
|
126
|
+
result[key] = value;
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
return result;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import type { FileCategory } from '../types.js';
|
|
2
|
+
import type { CollectionClient } from './types.js';
|
|
3
|
+
export declare function projectCollectionName(projectId: string, category: FileCategory): string;
|
|
4
|
+
export declare function globalCollectionName(category: FileCategory): string;
|
|
5
|
+
export declare class CollectionManager {
|
|
6
|
+
private readonly client?;
|
|
7
|
+
constructor(client?: CollectionClient | undefined);
|
|
8
|
+
ensureProjectCollections(projectId: string): Promise<void>;
|
|
9
|
+
ensureGlobalCollections(): Promise<void>;
|
|
10
|
+
deleteProjectCollections(projectId: string): Promise<number>;
|
|
11
|
+
listProjectsFromClient(): Promise<string[]>;
|
|
12
|
+
getProjectCollections(projectId: string): string[];
|
|
13
|
+
getGlobalCollections(): string[];
|
|
14
|
+
getCollectionName(scope: 'project' | 'global', category: FileCategory, projectId?: string): string;
|
|
15
|
+
listProjectsFromCollectionNames(collectionNames: string[]): string[];
|
|
16
|
+
}
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
import { ALL_CATEGORIES, COLLECTION_CATEGORY_NAMES } from '../constants.js';
|
|
2
|
+
export function projectCollectionName(projectId, category) {
|
|
3
|
+
return `proj_${projectId}_kb_${COLLECTION_CATEGORY_NAMES[category]}`;
|
|
4
|
+
}
|
|
5
|
+
export function globalCollectionName(category) {
|
|
6
|
+
return `global_kb_${COLLECTION_CATEGORY_NAMES[category]}`;
|
|
7
|
+
}
|
|
8
|
+
export class CollectionManager {
|
|
9
|
+
client;
|
|
10
|
+
constructor(client) {
|
|
11
|
+
this.client = client;
|
|
12
|
+
}
|
|
13
|
+
async ensureProjectCollections(projectId) {
|
|
14
|
+
if (!this.client)
|
|
15
|
+
return;
|
|
16
|
+
for (const category of ALL_CATEGORIES) {
|
|
17
|
+
await this.client.getOrCreateCollection(projectCollectionName(projectId, category), {
|
|
18
|
+
'hnsw:space': 'cosine',
|
|
19
|
+
project_id: projectId,
|
|
20
|
+
category,
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
async ensureGlobalCollections() {
|
|
25
|
+
if (!this.client)
|
|
26
|
+
return;
|
|
27
|
+
for (const category of ALL_CATEGORIES) {
|
|
28
|
+
await this.client.getOrCreateCollection(globalCollectionName(category), {
|
|
29
|
+
'hnsw:space': 'cosine',
|
|
30
|
+
scope: 'global',
|
|
31
|
+
category,
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async deleteProjectCollections(projectId) {
|
|
36
|
+
if (!this.client)
|
|
37
|
+
return 0;
|
|
38
|
+
const prefix = `proj_${projectId}_`;
|
|
39
|
+
const collections = await this.client.listCollections();
|
|
40
|
+
let deleted = 0;
|
|
41
|
+
for (const collection of collections) {
|
|
42
|
+
if (collection.name.startsWith(prefix)) {
|
|
43
|
+
await this.client.deleteCollection(collection.name);
|
|
44
|
+
deleted += 1;
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
return deleted;
|
|
48
|
+
}
|
|
49
|
+
async listProjectsFromClient() {
|
|
50
|
+
if (!this.client)
|
|
51
|
+
return [];
|
|
52
|
+
const collections = await this.client.listCollections();
|
|
53
|
+
return this.listProjectsFromCollectionNames(collections.map(collection => collection.name));
|
|
54
|
+
}
|
|
55
|
+
getProjectCollections(projectId) {
|
|
56
|
+
return ALL_CATEGORIES.map(category => projectCollectionName(projectId, category));
|
|
57
|
+
}
|
|
58
|
+
getGlobalCollections() {
|
|
59
|
+
return ALL_CATEGORIES.map(category => globalCollectionName(category));
|
|
60
|
+
}
|
|
61
|
+
getCollectionName(scope, category, projectId) {
|
|
62
|
+
if (scope === 'global')
|
|
63
|
+
return globalCollectionName(category);
|
|
64
|
+
if (!projectId)
|
|
65
|
+
throw new Error('project scope requires projectId');
|
|
66
|
+
return projectCollectionName(projectId, category);
|
|
67
|
+
}
|
|
68
|
+
listProjectsFromCollectionNames(collectionNames) {
|
|
69
|
+
const projectIds = new Set();
|
|
70
|
+
for (const name of collectionNames) {
|
|
71
|
+
const match = name.match(/^proj_([a-f0-9]{12})_kb_/);
|
|
72
|
+
if (match?.[1])
|
|
73
|
+
projectIds.add(match[1]);
|
|
74
|
+
}
|
|
75
|
+
return [...projectIds];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export interface VectorDocument {
|
|
2
|
+
id: string;
|
|
3
|
+
content: string;
|
|
4
|
+
embedding: number[];
|
|
5
|
+
metadata: Record<string, string | number | boolean | null>;
|
|
6
|
+
}
|
|
7
|
+
export interface VectorSearchQuery {
|
|
8
|
+
queryEmbedding: number[];
|
|
9
|
+
topK: number;
|
|
10
|
+
where?: Record<string, string | number | boolean>;
|
|
11
|
+
}
|
|
12
|
+
export interface VectorSearchResult {
|
|
13
|
+
document: Omit<VectorDocument, 'embedding'>;
|
|
14
|
+
score: number;
|
|
15
|
+
collection: string;
|
|
16
|
+
}
|
|
17
|
+
export interface VectorCollectionInfo {
|
|
18
|
+
id?: string;
|
|
19
|
+
name: string;
|
|
20
|
+
metadata?: Record<string, unknown>;
|
|
21
|
+
}
|
|
22
|
+
export interface VectorStoreInterface {
|
|
23
|
+
readonly collectionName: string;
|
|
24
|
+
ensureCollection(metadata?: Record<string, unknown>): Promise<void>;
|
|
25
|
+
upsert(documents: VectorDocument[]): Promise<void>;
|
|
26
|
+
deleteByFilePath(filePath: string): Promise<void>;
|
|
27
|
+
search(query: VectorSearchQuery): Promise<VectorSearchResult[]>;
|
|
28
|
+
}
|
|
29
|
+
export interface CollectionClient {
|
|
30
|
+
getOrCreateCollection(name: string, metadata?: Record<string, unknown>): Promise<VectorCollectionInfo>;
|
|
31
|
+
listCollections(): Promise<VectorCollectionInfo[]>;
|
|
32
|
+
deleteCollection(name: string): Promise<void>;
|
|
33
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from '../embedding/embedding-provider.js';
|
|
2
|
+
import type { StoredChunk } from '../core/index-state-store.js';
|
|
3
|
+
import type { VectorStoreInterface } from './types.js';
|
|
4
|
+
export interface VectorIndexResult {
|
|
5
|
+
collectionName: string;
|
|
6
|
+
chunkCount: number;
|
|
7
|
+
embeddingModel: string;
|
|
8
|
+
embeddingDimension: number;
|
|
9
|
+
}
|
|
10
|
+
export declare class VectorIndexer {
|
|
11
|
+
private readonly embeddingProvider;
|
|
12
|
+
private readonly vectorStores;
|
|
13
|
+
constructor(embeddingProvider: EmbeddingProvider, vectorStores: Map<string, VectorStoreInterface>);
|
|
14
|
+
indexChunks(chunks: StoredChunk[]): Promise<VectorIndexResult[]>;
|
|
15
|
+
deleteFile(collectionName: string, filePath: string): Promise<void>;
|
|
16
|
+
private groupByCollection;
|
|
17
|
+
private toVectorDocument;
|
|
18
|
+
}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
export class VectorIndexer {
|
|
2
|
+
embeddingProvider;
|
|
3
|
+
vectorStores;
|
|
4
|
+
constructor(embeddingProvider, vectorStores) {
|
|
5
|
+
this.embeddingProvider = embeddingProvider;
|
|
6
|
+
this.vectorStores = vectorStores;
|
|
7
|
+
}
|
|
8
|
+
async indexChunks(chunks) {
|
|
9
|
+
const byCollection = this.groupByCollection(chunks);
|
|
10
|
+
const results = [];
|
|
11
|
+
for (const [collectionName, collectionChunks] of byCollection) {
|
|
12
|
+
const store = this.vectorStores.get(collectionName);
|
|
13
|
+
if (!store)
|
|
14
|
+
continue;
|
|
15
|
+
await store.ensureCollection({
|
|
16
|
+
embedding_model: this.embeddingProvider.model,
|
|
17
|
+
embedding_dimension: this.embeddingProvider.dimensions,
|
|
18
|
+
});
|
|
19
|
+
const embeddings = await this.embeddingProvider.embedDocuments(collectionChunks.map(chunk => chunk.content));
|
|
20
|
+
const documents = collectionChunks.map((chunk, index) => this.toVectorDocument(chunk, embeddings[index] ?? []));
|
|
21
|
+
await store.upsert(documents);
|
|
22
|
+
results.push({
|
|
23
|
+
collectionName,
|
|
24
|
+
chunkCount: documents.length,
|
|
25
|
+
embeddingModel: this.embeddingProvider.model,
|
|
26
|
+
embeddingDimension: this.embeddingProvider.dimensions,
|
|
27
|
+
});
|
|
28
|
+
}
|
|
29
|
+
return results;
|
|
30
|
+
}
|
|
31
|
+
async deleteFile(collectionName, filePath) {
|
|
32
|
+
const store = this.vectorStores.get(collectionName);
|
|
33
|
+
if (!store)
|
|
34
|
+
return;
|
|
35
|
+
await store.deleteByFilePath(filePath);
|
|
36
|
+
}
|
|
37
|
+
groupByCollection(chunks) {
|
|
38
|
+
const grouped = new Map();
|
|
39
|
+
for (const chunk of chunks) {
|
|
40
|
+
const list = grouped.get(chunk.collectionName) ?? [];
|
|
41
|
+
list.push(chunk);
|
|
42
|
+
grouped.set(chunk.collectionName, list);
|
|
43
|
+
}
|
|
44
|
+
return grouped;
|
|
45
|
+
}
|
|
46
|
+
toVectorDocument(chunk, embedding) {
|
|
47
|
+
return {
|
|
48
|
+
id: chunk.id,
|
|
49
|
+
content: chunk.content,
|
|
50
|
+
embedding,
|
|
51
|
+
metadata: {
|
|
52
|
+
file_path: chunk.relativePath,
|
|
53
|
+
chunk_index: chunk.chunkIndex,
|
|
54
|
+
category: chunk.category,
|
|
55
|
+
format: chunk.format,
|
|
56
|
+
token_count: chunk.tokenCount,
|
|
57
|
+
section_title: chunk.sectionTitle ?? null,
|
|
58
|
+
},
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@customize-agent/knowledge",
|
|
3
|
+
"version": "1.0.1",
|
|
4
|
+
"description": "Local knowledge base infrastructure for customize-agent",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./dist/index.js",
|
|
7
|
+
"types": "./dist/index.d.ts",
|
|
8
|
+
"license": "MIT",
|
|
9
|
+
"author": "Pan-jijian",
|
|
10
|
+
"repository": {
|
|
11
|
+
"type": "git",
|
|
12
|
+
"url": "https://github.com/Pan-jijian/customize-agent.git",
|
|
13
|
+
"directory": "packages/knowledge"
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"dist"
|
|
17
|
+
],
|
|
18
|
+
"publishConfig": {
|
|
19
|
+
"access": "public"
|
|
20
|
+
},
|
|
21
|
+
"keywords": [
|
|
22
|
+
"ai-agent",
|
|
23
|
+
"llm",
|
|
24
|
+
"customize-agent",
|
|
25
|
+
"knowledge-base"
|
|
26
|
+
],
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"better-sqlite3": "^12.10.0",
|
|
29
|
+
"fast-glob": "^3.3.3",
|
|
30
|
+
"jszip": "^3.10.1",
|
|
31
|
+
"mammoth": "^1.11.0",
|
|
32
|
+
"pdf-parse": "^2.4.5",
|
|
33
|
+
"tesseract.js": "^6.0.1",
|
|
34
|
+
"xlsx": "^0.18.5"
|
|
35
|
+
},
|
|
36
|
+
"devDependencies": {
|
|
37
|
+
"@types/better-sqlite3": "^7.6.13",
|
|
38
|
+
"@types/node": "^25.9.3"
|
|
39
|
+
},
|
|
40
|
+
"scripts": {
|
|
41
|
+
"build": "tsc",
|
|
42
|
+
"typecheck": "tsc --noEmit",
|
|
43
|
+
"lint": "eslint src/"
|
|
44
|
+
}
|
|
45
|
+
}
|