kiku-agent 1.2.0 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +459 -407
- package/package.json +27 -3
- package/server/agent/agentCore.js +635 -267
- package/server/agent/appPaths.js +2 -0
- package/server/agent/catalogEntityResolver.js +629 -0
- package/server/agent/citations.js +0 -21
- package/server/agent/clarificationAnalytics.js +1 -1
- package/server/agent/clarifyingQuestions.js +286 -216
- package/server/agent/cli.js +528 -73
- package/server/agent/commandFeatures.js +123 -24
- package/server/agent/commandRegistry.js +128 -62
- package/server/agent/commands.js +1410 -95
- package/server/agent/config.js +253 -163
- package/server/agent/contextCompactor.js +239 -25
- package/server/agent/dynamicCatalog.js +192 -0
- package/server/agent/envBootstrap.js +46 -0
- package/server/agent/envManager.js +317 -0
- package/server/agent/eventBus.js +3 -3
- package/server/agent/feedbackLearning.js +0 -8
- package/server/agent/gpuMetrics.js +188 -55
- package/server/agent/ingestionWorker.js +339 -0
- package/server/agent/lanDiscovery.js +369 -0
- package/server/agent/llmClient.js +111 -39
- package/server/agent/lmStudioWakeup.js +144 -0
- package/server/agent/mcpClient.js +2 -2
- package/server/agent/memory.js +635 -494
- package/server/agent/modelConfig.js +487 -0
- package/server/agent/newsIntelligence.js +219 -0
- package/server/agent/newsWatch.js +85 -0
- package/server/agent/prompts.js +146 -0
- package/server/agent/publicTools.js +7 -0
- package/server/agent/queryDetails.js +62 -1
- package/server/agent/research.js +314 -35
- package/server/agent/routingInterceptor.js +77 -0
- package/server/agent/rumoredProjects.js +15 -0
- package/server/agent/sanitizer.js +117 -17
- package/server/agent/subagent.js +52 -9
- package/server/agent/suggestionTui.js +33 -13
- package/server/agent/suggestions.js +29 -13
- package/server/agent/telemetry.js +0 -11
- package/server/agent/terminalImage.js +0 -1
- package/server/agent/textToolProtocol.js +190 -0
- package/server/agent/tokenTracker.js +220 -0
- package/server/agent/toolRouter.js +362 -216
- package/server/agent/tools.js +1155 -194
- package/server/agent/ui.js +144 -41
- package/server/agent/vramService.js +572 -0
- package/server/agent/webSearch.js +2279 -273
- package/server/agent/youtubeAuth.js +82 -73
- package/server/agent/youtubeTranscript.js +209 -0
- package/server/apiRouter.js +383 -179
- package/server/mcp/data/brands.js +5 -5
- package/server/mcp/data/discography.js +56 -45
- package/server/mcp/data/gallery.js +9 -0
- package/server/mcp/data/index.js +1 -0
- package/server/mcp/data/industryIndex.js +251 -0
- package/server/mcp/data/profile.js +5 -5
- package/server/mcp/data/timeline.js +2 -2
- package/server/mcp/data/works.js +2 -2
- package/server/mcp/handler.js +0 -2
- package/server/mcp/prompts.js +42 -2
- package/server/mcp/resources.js +5 -1
- package/server/mcp/security.js +25 -14
- package/server/mcp/server.js +92 -3
- package/server/mcp/tools.js +20 -11
- package/server/web-chat/core/connection.js +218 -194
- package/server/web-chat/core/conversationContext.js +1 -1
- package/server/web-chat/core/directCatalogEngine.js +190 -184
- package/server/web-chat/core/followUpEngine.js +88 -11
- package/server/web-chat/core/security.js +7 -4
- package/server/web-chat/router.js +54 -17
- package/server/web-chat/routes/bilibiliRoute.js +34 -0
- package/server/web-chat/routes/chatRoute.js +488 -397
- package/server/web-chat/routes/clarifyConfigRoute.js +1 -2
- package/server/web-chat/routes/exportRoute.js +2 -1
- package/server/web-chat/routes/lanDiscoveryRoute.js +90 -0
- package/server/web-chat/routes/newsRoute.js +36 -6
- package/server/web-chat/routes/providerRoute.js +87 -0
- package/server/web-chat/routes/staticRoute.js +61 -13
- package/server/web-chat/routes/statusRoute.js +23 -2
- package/server/web-chat/routes/vramRoute.js +22 -0
- package/server/web-chat/server.js +135 -106
- package/server/web-chat/ui/htmlBuilder.js +8 -8
- package/server/web-chat/ui/previews/modelSettings.preview.html +41 -0
- package/server/web-chat/ui/scripts/agentBubbleSystem.js +0 -5
- package/server/web-chat/ui/scripts/client.js +10526 -8397
- package/server/web-chat/ui/scripts/webchatI18n.js +190 -138
- package/server/web-chat/ui/styles/main.css +7471 -4246
- package/server/web-chat/ui/templates/body.html +305 -179
- package/src/data/brandsData.js +29 -6
- package/src/data/discographyData.js +918 -900
- package/src/data/galleryData.js +66 -0
- package/src/data/industryIndexData.js +292 -0
- package/src/data/musicVideoData.js +48 -5
- package/src/data/profileData.js +15 -1
- package/src/data/timelineData.js +4 -3
- package/src/data/worksData.js +536 -535
- package/src/lib/vram/estimator.js +464 -0
- package/src/lib/vram/modelRegistry.js +387 -0
- package/tokens.css +33 -2
|
@@ -0,0 +1,339 @@
|
|
|
1
|
+
import { classifySource, extractArticleDate } from './newsIntelligence.js';
|
|
2
|
+
import { simhashText, hamming64 } from './websearch/extract.js';
|
|
3
|
+
import {
|
|
4
|
+
addDynamicEntry,
|
|
5
|
+
getDynamicEntries,
|
|
6
|
+
validateCatalogEntry,
|
|
7
|
+
DEFAULT_DYNAMIC_CATALOG_PATH
|
|
8
|
+
} from './dynamicCatalog.js';
|
|
9
|
+
import { searchNews } from './webSearch.js';
|
|
10
|
+
|
|
11
|
+
export const KNOWN_OFFICIAL_PLATFORMS = new Set([
|
|
12
|
+
'weibo', 'weibo.com', 'weibo.cn',
|
|
13
|
+
'douyin', 'douyin.com',
|
|
14
|
+
'bilibili', 'bilibili.com',
|
|
15
|
+
'youtube', 'youtube.com',
|
|
16
|
+
'instagram', 'instagram.com',
|
|
17
|
+
'xiaohongshu', 'xiaohongshu.com', 'xhs',
|
|
18
|
+
'tiktok', 'tiktok.com'
|
|
19
|
+
]);
|
|
20
|
+
|
|
21
|
+
export function isTrustedSource(source) {
|
|
22
|
+
if (!source || typeof source !== 'string') {
|
|
23
|
+
return { trusted: false, tier: 3, label: 'unknown' };
|
|
24
|
+
}
|
|
25
|
+
const sLower = source.toLowerCase().trim();
|
|
26
|
+
|
|
27
|
+
if (KNOWN_OFFICIAL_PLATFORMS.has(sLower)) {
|
|
28
|
+
return { trusted: true, tier: 1, label: 'official' };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
if (/^https?:\/\//i.test(sLower)) {
|
|
32
|
+
const classified = classifySource(source);
|
|
33
|
+
return {
|
|
34
|
+
trusted: classified.tier <= 2,
|
|
35
|
+
tier: classified.tier,
|
|
36
|
+
label: classified.label
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
if (/(weibo|douyin|bilibili|youtube|instagram|xhs|xiaohongshu|tiktok|official|工作室)/i.test(sLower)) {
|
|
41
|
+
return { trusted: true, tier: 1, label: 'official' };
|
|
42
|
+
}
|
|
43
|
+
if (/(press|news|sohu|netease|sina|tencent|qq|thairath|sanook|kapook|pptv)/i.test(sLower)) {
|
|
44
|
+
return { trusted: true, tier: 2, label: 'press' };
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
return { trusted: false, tier: 3, label: 'community' };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function computeEventSimHash(text) {
|
|
51
|
+
return simhashText(String(text || ''));
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export function isDuplicateEvent(entry, existingList = [], { maxHamming = 8 } = {}) {
|
|
55
|
+
if (!entry || !Array.isArray(existingList) || existingList.length === 0) {
|
|
56
|
+
return false;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const entryText = `${entry.title || entry.brand || entry.brandNameEn || ''} ${entry.highlight || ''} ${entry.summary || ''}`.trim();
|
|
60
|
+
const entryHash = entryText ? computeEventSimHash(entryText) : null;
|
|
61
|
+
|
|
62
|
+
for (const existing of existingList) {
|
|
63
|
+
if (entry.id && existing.id && entry.id === existing.id) {
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const entryBrand = (entry.brand || entry.brandNameEn || entry.brandNameCn || '').toLowerCase().trim();
|
|
68
|
+
const existingBrand = (existing.brand || existing.brandNameEn || existing.brandNameCn || '').toLowerCase().trim();
|
|
69
|
+
if (entryBrand && existingBrand && entryBrand === existingBrand && entry.year === existing.year) {
|
|
70
|
+
return true;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
const entryTitle = (entry.title || entry.titleCn || entry.titleTh || entry.titleEn || '').toLowerCase().trim();
|
|
74
|
+
const existingTitle = (existing.title || existing.titleCn || existing.titleTh || existing.titleEn || '').toLowerCase().trim();
|
|
75
|
+
if (entryTitle && existingTitle && entryTitle === existingTitle && entry.year === existing.year) {
|
|
76
|
+
return true;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const eH = (entry.highlight || entry.summary || '').toLowerCase().trim();
|
|
80
|
+
const xH = (existing.highlight || existing.summary || '').toLowerCase().trim();
|
|
81
|
+
if (eH && xH && (eH.includes(xH) || xH.includes(eH))) {
|
|
82
|
+
return true;
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
if (entryHash != null) {
|
|
86
|
+
const existText = `${existing.title || existing.brand || existing.brandNameEn || ''} ${existing.highlight || ''} ${existing.summary || ''}`.trim();
|
|
87
|
+
if (existText) {
|
|
88
|
+
const existHash = computeEventSimHash(existText);
|
|
89
|
+
if (hamming64(entryHash, existHash) <= maxHamming) {
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
export async function extractStructuredEvent(text, { client = null } = {}) {
|
|
100
|
+
const rawText = String(text || '').trim();
|
|
101
|
+
if (!rawText) {
|
|
102
|
+
throw new Error('Event text cannot be empty');
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (client && typeof client.chatCompletion === 'function') {
|
|
106
|
+
try {
|
|
107
|
+
const prompt = `Extract a structured event for artist Ju Jingyi (鞠婧祎) from this text into JSON:
|
|
108
|
+
Text: "${rawText}"
|
|
109
|
+
|
|
110
|
+
Return ONLY a JSON object with:
|
|
111
|
+
- category: one of ["brands", "timeline", "gallery", "works", "events"]
|
|
112
|
+
- year: integer year (e.g. 2026)
|
|
113
|
+
- date: ISO date string or "YYYY-MM-DD" if present
|
|
114
|
+
- title: concise title of the event / work
|
|
115
|
+
- titleCn: Chinese title or null
|
|
116
|
+
- titleTh: Thai title or null
|
|
117
|
+
- brand: brand name (if brand endorsement) or null
|
|
118
|
+
- brandNameCn: Chinese brand name or null
|
|
119
|
+
- brandNameEn: English brand name or null
|
|
120
|
+
- role: role/title (e.g. "Brand Ambassador", "Lead Actress", "Performer")
|
|
121
|
+
- highlight: key highlight sentence
|
|
122
|
+
- summary: full description
|
|
123
|
+
- imageUrl: image URL or null
|
|
124
|
+
- source: origin platform or media outlet`;
|
|
125
|
+
|
|
126
|
+
const response = await client.chatCompletion({
|
|
127
|
+
messages: [{ role: 'user', content: prompt }],
|
|
128
|
+
temperature: 0.1,
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
const jsonStr = response?.choices?.[0]?.message?.content || '';
|
|
132
|
+
const match = jsonStr.match(/\{[\s\S]*\}/);
|
|
133
|
+
if (match) {
|
|
134
|
+
const parsed = JSON.parse(match[0]);
|
|
135
|
+
if (parsed && (parsed.title || parsed.brand || parsed.brandNameEn)) {
|
|
136
|
+
return parsed;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
} catch {
|
|
140
|
+
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const currentYear = new Date().getFullYear();
|
|
145
|
+
const dateInfo = extractArticleDate({ snippet: rawText, title: rawText });
|
|
146
|
+
const eventYear = dateInfo?.date ? dateInfo.date.getUTCFullYear() : currentYear;
|
|
147
|
+
let isoDate = dateInfo?.date ? dateInfo.date.toISOString().slice(0, 10) : null;
|
|
148
|
+
|
|
149
|
+
if (!isoDate) {
|
|
150
|
+
const THAI_MONTHS_MAP = {
|
|
151
|
+
'มกราคม': 1, 'กุมภาพันธ์': 2, 'มีนาคม': 3, 'เมษายน': 4, 'พฤษภาคม': 5, 'มิถุนายน': 6,
|
|
152
|
+
'กรกฎาคม': 7, 'สิงหาคม': 8, 'กันยายน': 9, 'ตุลาคม': 10, 'พฤศจิกายน': 11, 'ธันวาคม': 12,
|
|
153
|
+
'มค': 1, 'กพ': 2, 'มีค': 3, 'เมย': 4, 'พค': 5, 'มิย': 6, 'กค': 7, 'สค': 8, 'กย': 9, 'ตค': 10, 'พย': 11, 'ธค': 12,
|
|
154
|
+
};
|
|
155
|
+
const thMatch = rawText.match(/(?:วันที่\s*)?(\d{1,2})\s*(มกราคม|กุมภาพันธ์|มีนาคม|เมษายน|พฤษภาคม|มิถุนายน|กรกฎาคม|สิงหาคม|กันยายน|ตุลาคม|พฤศจิกายน|ธันวาคม|ม\.?ค\.?|ก\.?พ\.?|มี\.?ค\.?|เม\.?ย\.?|พ\.?ค\.?|มิ\.?ย\.?|ก\.?ค\.?|ส\.?ค\.?|ก\.?ย\.?|ต\.?ค\.?|พ\.?ย\.?|ธ\.?ค\.?)/i);
|
|
156
|
+
if (thMatch) {
|
|
157
|
+
const day = String(thMatch[1]).padStart(2, '0');
|
|
158
|
+
const mKey = thMatch[2].replace(/\./g, '');
|
|
159
|
+
const monthNum = THAI_MONTHS_MAP[mKey] || 1;
|
|
160
|
+
const month = String(monthNum).padStart(2, '0');
|
|
161
|
+
isoDate = `${eventYear}-${month}-${day}`;
|
|
162
|
+
} else {
|
|
163
|
+
isoDate = `${eventYear}-01-01`;
|
|
164
|
+
}
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
let category = 'events';
|
|
168
|
+
const isBrand = /(แบรนด์|พรีเซ็นเตอร์|โฆษณา|brand|ambassador|endorsement|CF|Commercial|代言|Lafang|拉芳|FRED|TuTuCare)/i.test(rawText);
|
|
169
|
+
const isGallery = /(ลุค|ชุดเดรส|ภาพ|รูปถ่าย|photoshoot|look|dress|gallery|fashion)/i.test(rawText);
|
|
170
|
+
const isWork = /(ซีรีส์|ละคร|เพลง|อัลบั้ม|ซิงเกิล|OST|drama|album|single|series|film)/i.test(rawText);
|
|
171
|
+
const isTimeline = /(milestone|เปิดตัว|ครบรอบ|debut|graduation|timeline)/i.test(rawText);
|
|
172
|
+
|
|
173
|
+
if (isBrand) category = 'brands';
|
|
174
|
+
else if (isWork) category = 'works';
|
|
175
|
+
else if (isGallery) category = 'gallery';
|
|
176
|
+
else if (isTimeline) category = 'timeline';
|
|
177
|
+
|
|
178
|
+
let brandNameEn = '';
|
|
179
|
+
let brandNameCn = '';
|
|
180
|
+
if (isBrand) {
|
|
181
|
+
if (/lafang|拉芳/i.test(rawText)) {
|
|
182
|
+
brandNameEn = 'Lafang';
|
|
183
|
+
brandNameCn = '拉芳';
|
|
184
|
+
} else if (/fred/i.test(rawText)) {
|
|
185
|
+
brandNameEn = 'FRED Paris';
|
|
186
|
+
brandNameCn = '斐登';
|
|
187
|
+
} else if (/tutucare/i.test(rawText)) {
|
|
188
|
+
brandNameEn = 'TuTuCare';
|
|
189
|
+
brandNameCn = '途途呵护';
|
|
190
|
+
} else {
|
|
191
|
+
const bMatch = rawText.match(/แบรนด์\s+([A-Za-z0-9\u4e00-\u9fa5]+)/i) || rawText.match(/brand\s+([A-Za-z0-9\u4e00-\u9fa5]+)/i);
|
|
192
|
+
if (bMatch) {
|
|
193
|
+
brandNameEn = bMatch[1];
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
let title = '';
|
|
199
|
+
const quoteMatch = rawText.match(/["“'「]([^"”'」]{3,50})["”'」]/);
|
|
200
|
+
if (quoteMatch) {
|
|
201
|
+
title = quoteMatch[1];
|
|
202
|
+
} else {
|
|
203
|
+
const firstSentence = rawText.split(/[.\n!?|]/)[0]?.trim();
|
|
204
|
+
title = firstSentence && firstSentence.length <= 80 ? firstSentence : `${brandNameEn || 'Ju Jingyi'} Event ${eventYear}`;
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
return {
|
|
208
|
+
category,
|
|
209
|
+
year: eventYear,
|
|
210
|
+
date: isoDate,
|
|
211
|
+
title,
|
|
212
|
+
titleCn: brandNameCn || null,
|
|
213
|
+
titleTh: title,
|
|
214
|
+
brand: brandNameEn || (isBrand ? title : null),
|
|
215
|
+
brandNameEn: brandNameEn || null,
|
|
216
|
+
brandNameCn: brandNameCn || null,
|
|
217
|
+
role: isBrand ? 'Brand Ambassador' : 'Artist',
|
|
218
|
+
highlight: rawText.slice(0, 200).trim(),
|
|
219
|
+
summary: rawText,
|
|
220
|
+
source: 'extracted_text'
|
|
221
|
+
};
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
export async function ingestRawEvent(text, {
|
|
225
|
+
category = null,
|
|
226
|
+
source = 'manual',
|
|
227
|
+
trustTier = null,
|
|
228
|
+
client = null,
|
|
229
|
+
catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH
|
|
230
|
+
} = {}) {
|
|
231
|
+
try {
|
|
232
|
+
const trustInfo = isTrustedSource(source);
|
|
233
|
+
const finalTrustTier = trustTier != null ? Number(trustTier) : trustInfo.tier;
|
|
234
|
+
|
|
235
|
+
if (finalTrustTier > 2 && !trustInfo.trusted && trustTier == null) {
|
|
236
|
+
return {
|
|
237
|
+
status: 'rejected',
|
|
238
|
+
reason: 'Untrusted source (Tier 3 community/unverified). Provide explicit trustTier to override.',
|
|
239
|
+
source,
|
|
240
|
+
tier: finalTrustTier
|
|
241
|
+
};
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
const structured = await extractStructuredEvent(text, { client });
|
|
245
|
+
const targetCategory = category || structured.category || 'events';
|
|
246
|
+
|
|
247
|
+
const entryToSave = {
|
|
248
|
+
...structured,
|
|
249
|
+
source: source || structured.source || 'manual',
|
|
250
|
+
trustTier: finalTrustTier
|
|
251
|
+
};
|
|
252
|
+
|
|
253
|
+
const existing = getDynamicEntries(targetCategory, null, catalogPath);
|
|
254
|
+
if (isDuplicateEvent(entryToSave, existing)) {
|
|
255
|
+
return {
|
|
256
|
+
status: 'duplicate',
|
|
257
|
+
message: `Event already exists in dynamic catalog category "${targetCategory}".`,
|
|
258
|
+
entry: entryToSave
|
|
259
|
+
};
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
const saved = addDynamicEntry(targetCategory, entryToSave, {
|
|
263
|
+
catalogPath,
|
|
264
|
+
trustTier: finalTrustTier,
|
|
265
|
+
source
|
|
266
|
+
});
|
|
267
|
+
|
|
268
|
+
return {
|
|
269
|
+
status: 'success',
|
|
270
|
+
message: `Successfully ingested event into category "${targetCategory}".`,
|
|
271
|
+
entry: saved
|
|
272
|
+
};
|
|
273
|
+
} catch (err) {
|
|
274
|
+
return {
|
|
275
|
+
status: 'error',
|
|
276
|
+
message: `Ingestion failed: ${err.message}`
|
|
277
|
+
};
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
export async function runIngestionScan({
|
|
282
|
+
topic = 'Ju Jingyi 鞠婧祎',
|
|
283
|
+
fetcher = null,
|
|
284
|
+
client = null,
|
|
285
|
+
catalogPath = DEFAULT_DYNAMIC_CATALOG_PATH,
|
|
286
|
+
minTrustTier = 2,
|
|
287
|
+
maxItems = 8
|
|
288
|
+
} = {}) {
|
|
289
|
+
let items = [];
|
|
290
|
+
if (typeof fetcher === 'function') {
|
|
291
|
+
items = await fetcher(topic);
|
|
292
|
+
} else {
|
|
293
|
+
const res = await searchNews(topic, maxItems);
|
|
294
|
+
items = res?.results || [];
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
let totalScanned = items.length;
|
|
298
|
+
let ingestedCount = 0;
|
|
299
|
+
let skippedDuplicates = 0;
|
|
300
|
+
let skippedUntrusted = 0;
|
|
301
|
+
const entries = [];
|
|
302
|
+
|
|
303
|
+
for (const item of items) {
|
|
304
|
+
const url = item.url || '';
|
|
305
|
+
const trust = isTrustedSource(url);
|
|
306
|
+
|
|
307
|
+
if (trust.tier > minTrustTier) {
|
|
308
|
+
skippedUntrusted++;
|
|
309
|
+
continue;
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
const combinedText = `${item.title || ''}\n${item.snippet || ''}`.trim();
|
|
313
|
+
if (!combinedText) continue;
|
|
314
|
+
|
|
315
|
+
const res = await ingestRawEvent(combinedText, {
|
|
316
|
+
source: url || item.source || 'news_scan',
|
|
317
|
+
trustTier: trust.tier,
|
|
318
|
+
client,
|
|
319
|
+
catalogPath
|
|
320
|
+
});
|
|
321
|
+
|
|
322
|
+
if (res.status === 'success') {
|
|
323
|
+
ingestedCount++;
|
|
324
|
+
entries.push(res.entry);
|
|
325
|
+
} else if (res.status === 'duplicate') {
|
|
326
|
+
skippedDuplicates++;
|
|
327
|
+
} else if (res.status === 'rejected') {
|
|
328
|
+
skippedUntrusted++;
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
return {
|
|
333
|
+
totalScanned,
|
|
334
|
+
ingestedCount,
|
|
335
|
+
skippedDuplicates,
|
|
336
|
+
skippedUntrusted,
|
|
337
|
+
entries
|
|
338
|
+
};
|
|
339
|
+
}
|
|
@@ -0,0 +1,369 @@
|
|
|
1
|
+
import net from 'node:net';
|
|
2
|
+
import os from 'node:os';
|
|
3
|
+
import { isPrivateIPv4 } from './sanitizer.js';
|
|
4
|
+
|
|
5
|
+
export const DEFAULT_LLM_PORTS = Object.freeze([1234, 8000, 11434, 8080]);
|
|
6
|
+
|
|
7
|
+
const CACHE_TTL_MS = 60_000;
|
|
8
|
+
|
|
9
|
+
const SERVER_TYPE_LABELS = Object.freeze({
|
|
10
|
+
lmstudio: 'LM Studio',
|
|
11
|
+
vllm: 'vLLM',
|
|
12
|
+
ollama: 'Ollama',
|
|
13
|
+
'openai-compatible': 'OpenAI-Compatible',
|
|
14
|
+
});
|
|
15
|
+
|
|
16
|
+
export function mapServerTypeToProvider(serverType) {
|
|
17
|
+
if (serverType === 'lmstudio') return 'lmstudio';
|
|
18
|
+
if (serverType === 'vllm') return 'vllm';
|
|
19
|
+
return 'custom';
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function parseIpv4(host) {
|
|
23
|
+
const parts = String(host || '').split('.');
|
|
24
|
+
if (parts.length !== 4) return null;
|
|
25
|
+
const octets = parts.map(p => Number(p));
|
|
26
|
+
if (octets.some(n => !Number.isInteger(n) || n < 0 || n > 255)) return null;
|
|
27
|
+
return octets;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
export function getLanCandidateSubnets(interfaces = os.networkInterfaces()) {
|
|
31
|
+
const seen = new Set();
|
|
32
|
+
const subnets = [];
|
|
33
|
+
for (const [name, addresses] of Object.entries(interfaces || {})) {
|
|
34
|
+
for (const entry of addresses || []) {
|
|
35
|
+
if (!entry || entry.internal || entry.family !== 'IPv4') continue;
|
|
36
|
+
const octets = parseIpv4(entry.address);
|
|
37
|
+
if (!octets || octets[0] === 127) continue;
|
|
38
|
+
if (octets[0] === 169 && octets[1] === 254) continue;
|
|
39
|
+
if (!isPrivateIPv4(entry.address)) continue;
|
|
40
|
+
const cidr = `${octets[0]}.${octets[1]}.${octets[2]}.0/24`;
|
|
41
|
+
if (seen.has(cidr)) continue;
|
|
42
|
+
seen.add(cidr);
|
|
43
|
+
subnets.push({ cidr, interfaceName: name, address: entry.address });
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return subnets;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function expandCidrToHosts(cidr) {
|
|
50
|
+
const base = String(cidr || '').split('/')[0];
|
|
51
|
+
const octets = parseIpv4(base);
|
|
52
|
+
if (!octets) throw new Error(`Invalid CIDR "${cidr}"`);
|
|
53
|
+
const hosts = [];
|
|
54
|
+
for (let last = 1; last <= 254; last++) {
|
|
55
|
+
hosts.push(`${octets[0]}.${octets[1]}.${octets[2]}.${last}`);
|
|
56
|
+
}
|
|
57
|
+
return hosts;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export const DEFAULT_MAX_SCAN_HOSTS = 4096;
|
|
61
|
+
|
|
62
|
+
function ipv4ToUint32(octets) {
|
|
63
|
+
return ((octets[0] << 24) | (octets[1] << 16) | (octets[2] << 8) | octets[3]) >>> 0;
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function uint32ToIp(value) {
|
|
67
|
+
return `${(value >>> 24) & 255}.${(value >>> 16) & 255}.${(value >>> 8) & 255}.${value & 255}`;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
export function expandTargets(specs, { maxHosts = DEFAULT_MAX_SCAN_HOSTS } = {}) {
|
|
71
|
+
const list = Array.isArray(specs) ? specs : String(specs || '').split(',');
|
|
72
|
+
const out = [];
|
|
73
|
+
const seen = new Set();
|
|
74
|
+
let planned = 0;
|
|
75
|
+
|
|
76
|
+
const pushIp = value => {
|
|
77
|
+
const ip = uint32ToIp(value);
|
|
78
|
+
if (!seen.has(ip)) {
|
|
79
|
+
seen.add(ip);
|
|
80
|
+
out.push(ip);
|
|
81
|
+
}
|
|
82
|
+
};
|
|
83
|
+
|
|
84
|
+
for (const raw of list) {
|
|
85
|
+
const spec = String(raw || '').trim();
|
|
86
|
+
if (!spec) continue;
|
|
87
|
+
|
|
88
|
+
if (spec.includes('/')) {
|
|
89
|
+
const [base, prefixStr] = spec.split('/');
|
|
90
|
+
const octets = parseIpv4(base);
|
|
91
|
+
const prefix = Number(prefixStr);
|
|
92
|
+
if (!octets || !Number.isInteger(prefix) || prefix < 8 || prefix > 32) {
|
|
93
|
+
throw new Error(`Invalid CIDR "${spec}" (expected a.b.c.d/8..32)`);
|
|
94
|
+
}
|
|
95
|
+
const network = prefix === 0 ? 0 : (ipv4ToUint32(octets) & (0xffffffff << (32 - prefix))) >>> 0;
|
|
96
|
+
const size = 2 ** (32 - prefix);
|
|
97
|
+
const usable = prefix <= 30 ? size - 2 : size;
|
|
98
|
+
planned += usable;
|
|
99
|
+
if (planned > maxHosts) {
|
|
100
|
+
throw new Error(`Scan target limit exceeded (>${maxHosts} hosts). Narrow the range or raise KIKU_LAN_SCAN_MAX_HOSTS.`);
|
|
101
|
+
}
|
|
102
|
+
const first = prefix <= 30 ? network + 1 : network;
|
|
103
|
+
const last = prefix <= 30 ? network + size - 2 : network + size - 1;
|
|
104
|
+
for (let ip = first; ip <= last; ip++) pushIp(ip >>> 0);
|
|
105
|
+
} else if (spec.includes('-')) {
|
|
106
|
+
const [startStr, endStr] = spec.split('-');
|
|
107
|
+
const startOctets = parseIpv4(startStr);
|
|
108
|
+
const endOctets = parseIpv4(endStr);
|
|
109
|
+
if (!startOctets || !endOctets) {
|
|
110
|
+
throw new Error(`Invalid IP range "${spec}" (expected a.b.c.d-e.f.g.h)`);
|
|
111
|
+
}
|
|
112
|
+
const start = ipv4ToUint32(startOctets);
|
|
113
|
+
const end = ipv4ToUint32(endOctets);
|
|
114
|
+
if (end < start) throw new Error(`Invalid IP range "${spec}" (start is after end)`);
|
|
115
|
+
const count = end - start + 1;
|
|
116
|
+
planned += count;
|
|
117
|
+
if (planned > maxHosts) {
|
|
118
|
+
throw new Error(`Scan target limit exceeded (>${maxHosts} hosts). Narrow the range or raise KIKU_LAN_SCAN_MAX_HOSTS.`);
|
|
119
|
+
}
|
|
120
|
+
for (let ip = start; ip <= end; ip++) pushIp(ip >>> 0);
|
|
121
|
+
} else {
|
|
122
|
+
const octets = parseIpv4(spec);
|
|
123
|
+
if (!octets) throw new Error(`Invalid IP address "${spec}"`);
|
|
124
|
+
planned += 1;
|
|
125
|
+
pushIp(ipv4ToUint32(octets));
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
function assertAllPrivate(hosts) {
|
|
133
|
+
const offenders = hosts.filter(h => {
|
|
134
|
+
const octets = parseIpv4(h);
|
|
135
|
+
if (!octets) return true;
|
|
136
|
+
if (octets[0] === 127) return false;
|
|
137
|
+
return !isPrivateIPv4(h);
|
|
138
|
+
});
|
|
139
|
+
if (offenders.length > 0) {
|
|
140
|
+
throw new Error(`LAN discovery only scans private network ranges (RFC1918). Rejected: ${offenders.slice(0, 5).join(', ')}`);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function abortError() {
|
|
145
|
+
const error = new Error('LAN scan cancelled');
|
|
146
|
+
error.name = 'AbortError';
|
|
147
|
+
return error;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function defaultTcpProbe(host, port, timeoutMs, signal) {
|
|
151
|
+
return new Promise(resolve => {
|
|
152
|
+
if (signal?.aborted) {
|
|
153
|
+
resolve(false);
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
const socket = new net.Socket();
|
|
157
|
+
let settled = false;
|
|
158
|
+
const onAbort = () => done(false);
|
|
159
|
+
const done = result => {
|
|
160
|
+
if (settled) return;
|
|
161
|
+
settled = true;
|
|
162
|
+
signal?.removeEventListener('abort', onAbort);
|
|
163
|
+
socket.destroy();
|
|
164
|
+
resolve(result);
|
|
165
|
+
};
|
|
166
|
+
socket.setTimeout(timeoutMs);
|
|
167
|
+
socket.once('connect', () => done(true));
|
|
168
|
+
socket.once('timeout', () => done(false));
|
|
169
|
+
socket.once('error', () => done(false));
|
|
170
|
+
signal?.addEventListener('abort', onAbort, { once: true });
|
|
171
|
+
socket.connect(port, host);
|
|
172
|
+
});
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
export async function classifyModelsEndpoint({ host, port, timeoutMs = 2500, signal, fetchImpl = fetch }) {
|
|
176
|
+
const started = performance.now();
|
|
177
|
+
const timeoutSignal = AbortSignal.timeout(timeoutMs);
|
|
178
|
+
const fetchSignal = signal ? AbortSignal.any([timeoutSignal, signal]) : timeoutSignal;
|
|
179
|
+
const record = {
|
|
180
|
+
host,
|
|
181
|
+
port,
|
|
182
|
+
url: `http://${host}:${port}/v1`,
|
|
183
|
+
reachable: false,
|
|
184
|
+
serverType: 'openai-compatible',
|
|
185
|
+
serverName: SERVER_TYPE_LABELS['openai-compatible'],
|
|
186
|
+
models: [],
|
|
187
|
+
modelCount: 0,
|
|
188
|
+
authRequired: false,
|
|
189
|
+
latencyMs: null,
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
let modelsPayload = null;
|
|
193
|
+
try {
|
|
194
|
+
const res = await fetchImpl(`http://${host}:${port}/v1/models`, {
|
|
195
|
+
headers: { Accept: 'application/json' },
|
|
196
|
+
signal: fetchSignal,
|
|
197
|
+
});
|
|
198
|
+
if (res.status === 401 || res.status === 403) {
|
|
199
|
+
record.reachable = true;
|
|
200
|
+
record.authRequired = true;
|
|
201
|
+
record.latencyMs = Math.round(performance.now() - started);
|
|
202
|
+
return record;
|
|
203
|
+
}
|
|
204
|
+
if (res.ok) {
|
|
205
|
+
const json = await res.json().catch(() => null);
|
|
206
|
+
if (json && Array.isArray(json.data)) {
|
|
207
|
+
record.reachable = true;
|
|
208
|
+
modelsPayload = json;
|
|
209
|
+
record.models = json.data.map(m => m?.id).filter(Boolean);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
} catch {
|
|
213
|
+
return record;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const ownedBy = (modelsPayload?.data || []).map(m => String(m?.owned_by || '').toLowerCase()).join(' ');
|
|
217
|
+
if (ownedBy.includes('lmstudio')) record.serverType = 'lmstudio';
|
|
218
|
+
else if (ownedBy.includes('vllm')) record.serverType = 'vllm';
|
|
219
|
+
else {
|
|
220
|
+
try {
|
|
221
|
+
const tagsRes = await fetchImpl(`http://${host}:${port}/api/tags`, {
|
|
222
|
+
headers: { Accept: 'application/json' },
|
|
223
|
+
signal: fetchSignal,
|
|
224
|
+
});
|
|
225
|
+
if (tagsRes.ok) {
|
|
226
|
+
const tags = await tagsRes.json().catch(() => null);
|
|
227
|
+
if (tags && Array.isArray(tags.models)) {
|
|
228
|
+
record.reachable = true;
|
|
229
|
+
record.serverType = 'ollama';
|
|
230
|
+
if (record.models.length === 0) {
|
|
231
|
+
record.models = tags.models.map(m => m?.name || m?.model).filter(Boolean);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
} catch {}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
if (record.reachable && record.serverType === 'openai-compatible') {
|
|
239
|
+
try {
|
|
240
|
+
const v0Res = await fetchImpl(`http://${host}:${port}/api/v0/models`, {
|
|
241
|
+
headers: { Accept: 'application/json' },
|
|
242
|
+
signal: fetchSignal,
|
|
243
|
+
});
|
|
244
|
+
if (v0Res.ok) {
|
|
245
|
+
const v0 = await v0Res.json().catch(() => null);
|
|
246
|
+
if (v0 && Array.isArray(v0.data)) {
|
|
247
|
+
record.serverType = 'lmstudio';
|
|
248
|
+
if (record.models.length === 0) {
|
|
249
|
+
record.models = v0.data.map(m => m?.id).filter(Boolean);
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
}
|
|
253
|
+
} catch {}
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
if (record.reachable && record.serverType === 'openai-compatible') {
|
|
257
|
+
try {
|
|
258
|
+
const versionRes = await fetchImpl(`http://${host}:${port}/version`, {
|
|
259
|
+
headers: { Accept: 'application/json' },
|
|
260
|
+
signal: fetchSignal,
|
|
261
|
+
});
|
|
262
|
+
if (versionRes.ok) {
|
|
263
|
+
const version = await versionRes.json().catch(() => null);
|
|
264
|
+
if (version && version.version) {
|
|
265
|
+
record.serverType = 'vllm';
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
} catch {}
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
record.serverName = SERVER_TYPE_LABELS[record.serverType] || record.serverType;
|
|
272
|
+
record.modelCount = record.models.length;
|
|
273
|
+
record.latencyMs = Math.round(performance.now() - started);
|
|
274
|
+
return record;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
async function runPool(items, concurrency, worker, signal) {
|
|
278
|
+
const results = [];
|
|
279
|
+
let cursor = 0;
|
|
280
|
+
const lanes = Array.from({ length: Math.max(1, Math.min(concurrency, items.length)) }, async () => {
|
|
281
|
+
while (cursor < items.length) {
|
|
282
|
+
if (signal?.aborted) throw abortError();
|
|
283
|
+
const index = cursor++;
|
|
284
|
+
results[index] = await worker(items[index], index);
|
|
285
|
+
}
|
|
286
|
+
});
|
|
287
|
+
await Promise.all(lanes);
|
|
288
|
+
return results;
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
let scanCache = null;
|
|
292
|
+
|
|
293
|
+
export function clearLanScanCache() {
|
|
294
|
+
scanCache = null;
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
export async function scanLan({
|
|
298
|
+
ports = DEFAULT_LLM_PORTS,
|
|
299
|
+
timeoutMs = 900,
|
|
300
|
+
probeTimeoutMs = 2500,
|
|
301
|
+
concurrency = 128,
|
|
302
|
+
subnets = null,
|
|
303
|
+
extraHosts = [],
|
|
304
|
+
maxHosts = Number(process.env.KIKU_LAN_SCAN_MAX_HOSTS) || DEFAULT_MAX_SCAN_HOSTS,
|
|
305
|
+
connector = defaultTcpProbe,
|
|
306
|
+
prober = classifyModelsEndpoint,
|
|
307
|
+
now = Date.now,
|
|
308
|
+
force = false,
|
|
309
|
+
signal = null,
|
|
310
|
+
} = {}) {
|
|
311
|
+
if (signal?.aborted) throw abortError();
|
|
312
|
+
const portList = [...new Set((Array.isArray(ports) ? ports : DEFAULT_LLM_PORTS).map(Number))].filter(
|
|
313
|
+
p => Number.isInteger(p) && p >= 1 && p <= 65535
|
|
314
|
+
);
|
|
315
|
+
if (portList.length === 0) throw new Error('At least one valid port is required');
|
|
316
|
+
|
|
317
|
+
const subnetList = subnets && subnets.length > 0 ? subnets : getLanCandidateSubnets().map(s => s.cidr);
|
|
318
|
+
const detectedHosts = expandTargets(subnetList, { maxHosts });
|
|
319
|
+
const hostList = [...new Set(['127.0.0.1', ...detectedHosts, ...extraHosts.map(String)])];
|
|
320
|
+
assertAllPrivate(hostList);
|
|
321
|
+
|
|
322
|
+
const cacheKey = JSON.stringify({ ports: portList, hosts: hostList });
|
|
323
|
+
if (!force && scanCache && scanCache.key === cacheKey && now() - scanCache.at < CACHE_TTL_MS) {
|
|
324
|
+
return { ...scanCache.result, cached: true };
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
const targets = [];
|
|
328
|
+
for (const host of hostList) {
|
|
329
|
+
for (const port of portList) {
|
|
330
|
+
targets.push({ host, port });
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
const started = now();
|
|
335
|
+
const openTargets = [];
|
|
336
|
+
await runPool(targets, concurrency, async ({ host, port }) => {
|
|
337
|
+
const isOpen = await connector(host, port, timeoutMs, signal);
|
|
338
|
+
if (isOpen) openTargets.push({ host, port });
|
|
339
|
+
}, signal);
|
|
340
|
+
openTargets.sort((a, b) => a.host.localeCompare(b.host, undefined, { numeric: true }) || a.port - b.port);
|
|
341
|
+
if (signal?.aborted) throw abortError();
|
|
342
|
+
|
|
343
|
+
const endpoints = await runPool(openTargets, Math.min(concurrency, 16), async ({ host, port }) => {
|
|
344
|
+
try {
|
|
345
|
+
return await prober({ host, port, timeoutMs: probeTimeoutMs, signal });
|
|
346
|
+
} catch (error) {
|
|
347
|
+
if (signal?.aborted) throw error;
|
|
348
|
+
return { host, port, url: `http://${host}:${port}/v1`, reachable: false, serverType: 'openai-compatible', serverName: 'OpenAI-Compatible', models: [], modelCount: 0, authRequired: false, latencyMs: null };
|
|
349
|
+
}
|
|
350
|
+
}, signal);
|
|
351
|
+
|
|
352
|
+
const found = endpoints
|
|
353
|
+
.filter(e => e && e.reachable)
|
|
354
|
+
.sort((a, b) => (a.latencyMs ?? 9e9) - (b.latencyMs ?? 9e9) || a.host.localeCompare(b.host, undefined, { numeric: true }));
|
|
355
|
+
|
|
356
|
+
const result = {
|
|
357
|
+
scannedAt: new Date(started).toISOString(),
|
|
358
|
+
subnets: subnetList,
|
|
359
|
+
hostsScanned: hostList.length,
|
|
360
|
+
ports: portList,
|
|
361
|
+
targetsScanned: targets.length,
|
|
362
|
+
openPorts: openTargets.length,
|
|
363
|
+
durationMs: now() - started,
|
|
364
|
+
endpoints: found,
|
|
365
|
+
};
|
|
366
|
+
|
|
367
|
+
scanCache = { key: cacheKey, at: now(), result };
|
|
368
|
+
return { ...result, cached: false };
|
|
369
|
+
}
|