kiku-agent 1.2.0 → 1.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +459 -407
- package/package.json +32 -3
- package/server/agent/agentCore.js +635 -267
- package/server/agent/appPaths.js +2 -0
- package/server/agent/catalogEntityResolver.js +637 -0
- package/server/agent/citations.js +0 -21
- package/server/agent/clarificationAnalytics.js +1 -1
- package/server/agent/clarifyingQuestions.js +286 -216
- package/server/agent/cli.js +532 -73
- package/server/agent/commandFeatures.js +123 -24
- package/server/agent/commandRegistry.js +128 -62
- package/server/agent/commands.js +1410 -95
- package/server/agent/config.js +253 -163
- package/server/agent/contextCompactor.js +239 -25
- package/server/agent/dynamicCatalog.js +192 -0
- package/server/agent/envBootstrap.js +46 -0
- package/server/agent/envManager.js +317 -0
- package/server/agent/eventBus.js +3 -3
- package/server/agent/feedbackLearning.js +0 -8
- package/server/agent/gpuMetrics.js +188 -55
- package/server/agent/ingestionWorker.js +339 -0
- package/server/agent/ink/App.jsx +348 -0
- package/server/agent/ink/Panel.jsx +54 -0
- package/server/agent/ink/PromptInput.jsx +321 -0
- package/server/agent/ink/TaskPlanPanel.jsx +88 -0
- package/server/agent/ink/ThinkingPanel.jsx +68 -0
- package/server/agent/ink/ToolResult.jsx +133 -0
- package/server/agent/ink/main.jsx +195 -0
- package/server/agent/ink/useEventBus.js +218 -0
- package/server/agent/lanDiscovery.js +369 -0
- package/server/agent/llmClient.js +111 -39
- package/server/agent/lmStudioWakeup.js +144 -0
- package/server/agent/mcpClient.js +2 -2
- package/server/agent/memory/contextAllocator.js +214 -0
- package/server/agent/memory/hierarchicalStore.js +523 -0
- package/server/agent/memory/reflectionWorker.js +110 -0
- package/server/agent/memory.js +635 -494
- package/server/agent/modelConfig.js +487 -0
- package/server/agent/newsIntelligence.js +219 -0
- package/server/agent/newsWatch.js +85 -0
- package/server/agent/prompts.js +146 -0
- package/server/agent/publicTools.js +7 -0
- package/server/agent/queryDetails.js +62 -1
- package/server/agent/research.js +314 -35
- package/server/agent/routingInterceptor.js +77 -0
- package/server/agent/rumoredProjects.js +15 -0
- package/server/agent/sanitizer.js +117 -17
- package/server/agent/subagent.js +52 -9
- package/server/agent/suggestionTui.js +33 -13
- package/server/agent/suggestions.js +29 -13
- package/server/agent/telemetry/hardwareTiers.js +332 -0
- package/server/agent/telemetry/runtimeWatchdog.js +66 -0
- package/server/agent/telemetry/techPowerUpClient.js +682 -0
- package/server/agent/telemetry/techpowerup-mcp.json +22 -0
- package/server/agent/telemetry.js +0 -11
- package/server/agent/terminalImage.js +0 -1
- package/server/agent/textToolProtocol.js +190 -0
- package/server/agent/tokenTracker.js +220 -0
- package/server/agent/toolRouter.js +362 -216
- package/server/agent/tools.js +1155 -194
- package/server/agent/ui.js +144 -41
- package/server/agent/vramService.js +572 -0
- package/server/agent/webSearch.js +2279 -273
- package/server/agent/websearch/adapters/duckduckgo.js +97 -0
- package/server/agent/websearch/adapters/jina.js +77 -0
- package/server/agent/websearch/adapters/searxng.js +57 -0
- package/server/agent/websearch/core/canonical.js +89 -0
- package/server/agent/websearch/core/fusion.js +81 -0
- package/server/agent/websearch/core/types.js +29 -0
- package/server/agent/websearch/extract.js +605 -0
- package/server/agent/websearch/index.js +43 -0
- package/server/agent/websearch/orchestrator.js +266 -0
- package/server/agent/websearch/resilience/breaker.js +97 -0
- package/server/agent/websearch/resilience/cache.js +59 -0
- package/server/agent/websearch/resilience/limiter.js +93 -0
- package/server/agent/websearch/security/filter.js +36 -0
- package/server/agent/websearch/security/ssrf.js +289 -0
- package/server/agent/websearch/telemetry.js +71 -0
- package/server/agent/youtubeAuth.js +82 -73
- package/server/agent/youtubeTranscript.js +209 -0
- package/server/apiRouter.js +390 -190
- package/server/mcp/data/brands.js +5 -5
- package/server/mcp/data/discography.js +56 -45
- package/server/mcp/data/gallery.js +9 -0
- package/server/mcp/data/index.js +1 -0
- package/server/mcp/data/industryIndex.js +251 -0
- package/server/mcp/data/profile.js +5 -5
- package/server/mcp/data/timeline.js +2 -2
- package/server/mcp/data/works.js +2 -2
- package/server/mcp/handler.js +0 -2
- package/server/mcp/prompts.js +42 -2
- package/server/mcp/resources.js +5 -1
- package/server/mcp/security.js +25 -14
- package/server/mcp/server.js +92 -3
- package/server/mcp/tools.js +20 -11
- package/server/routes/agent.js +55 -0
- package/server/web-chat/core/connection.js +218 -194
- package/server/web-chat/core/conversationContext.js +1 -1
- package/server/web-chat/core/directCatalogEngine.js +190 -184
- package/server/web-chat/core/followUpEngine.js +88 -11
- package/server/web-chat/core/security.js +7 -4
- package/server/web-chat/router.js +54 -17
- package/server/web-chat/routes/bilibiliRoute.js +34 -0
- package/server/web-chat/routes/chatRoute.js +488 -397
- package/server/web-chat/routes/clarifyConfigRoute.js +1 -2
- package/server/web-chat/routes/exportRoute.js +2 -1
- package/server/web-chat/routes/lanDiscoveryRoute.js +90 -0
- package/server/web-chat/routes/newsRoute.js +36 -6
- package/server/web-chat/routes/providerRoute.js +87 -0
- package/server/web-chat/routes/staticRoute.js +61 -13
- package/server/web-chat/routes/statusRoute.js +23 -2
- package/server/web-chat/routes/vramRoute.js +22 -0
- package/server/web-chat/server.js +135 -106
- package/server/web-chat/ui/htmlBuilder.js +8 -8
- package/server/web-chat/ui/previews/modelSettings.preview.html +41 -0
- package/server/web-chat/ui/scripts/agentBubbleSystem.js +0 -5
- package/server/web-chat/ui/scripts/client.js +10526 -8397
- package/server/web-chat/ui/scripts/webchatI18n.js +190 -138
- package/server/web-chat/ui/styles/main.css +7471 -4246
- package/server/web-chat/ui/templates/body.html +305 -179
- package/src/data/brandsData.js +29 -6
- package/src/data/discographyData.js +918 -900
- package/src/data/galleryData.js +66 -0
- package/src/data/industryIndexData.js +292 -0
- package/src/data/musicVideoData.js +48 -5
- package/src/data/profileData.js +15 -1
- package/src/data/timelineData.js +4 -3
- package/src/data/worksData.js +536 -535
- package/src/lib/vram/estimator.js +464 -0
- package/src/lib/vram/modelRegistry.js +387 -0
- package/tokens.css +33 -2
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
import http from 'node:http';
|
|
2
|
+
import https from 'node:https';
|
|
3
|
+
import { lookup as dnsLookup } from 'node:dns/promises';
|
|
4
|
+
|
|
5
|
+
export function parseIPv4(host) {
|
|
6
|
+
if (typeof host !== 'string' || !host) return null;
|
|
7
|
+
if (host.includes(':')) return null;
|
|
8
|
+
|
|
9
|
+
const parts = host.split('.');
|
|
10
|
+
if (parts.length > 4 || parts.length === 0) return null;
|
|
11
|
+
|
|
12
|
+
const numbers = [];
|
|
13
|
+
for (let i = 0; i < parts.length; i++) {
|
|
14
|
+
const part = parts[i];
|
|
15
|
+
if (!part) return null;
|
|
16
|
+
let val;
|
|
17
|
+
if (/^0x[0-9a-f]+$/i.test(part)) {
|
|
18
|
+
val = parseInt(part, 16);
|
|
19
|
+
} else if (/^0[0-7]+$/.test(part)) {
|
|
20
|
+
val = parseInt(part, 8);
|
|
21
|
+
} else if (/^\d+$/.test(part)) {
|
|
22
|
+
val = parseInt(part, 10);
|
|
23
|
+
} else {
|
|
24
|
+
return null;
|
|
25
|
+
}
|
|
26
|
+
if (isNaN(val) || val < 0) return null;
|
|
27
|
+
numbers.push(val);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
let a, b, c, d;
|
|
31
|
+
if (numbers.length === 1) {
|
|
32
|
+
const n = numbers[0];
|
|
33
|
+
if (n > 0xffffffff) return null;
|
|
34
|
+
a = (n >>> 24) & 255;
|
|
35
|
+
b = (n >>> 16) & 255;
|
|
36
|
+
c = (n >>> 8) & 255;
|
|
37
|
+
d = n & 255;
|
|
38
|
+
} else if (numbers.length === 2) {
|
|
39
|
+
if (numbers[0] > 255 || numbers[1] > 0xffffff) return null;
|
|
40
|
+
a = numbers[0];
|
|
41
|
+
b = (numbers[1] >>> 16) & 255;
|
|
42
|
+
c = (numbers[1] >>> 8) & 255;
|
|
43
|
+
d = numbers[1] & 255;
|
|
44
|
+
} else if (numbers.length === 3) {
|
|
45
|
+
if (numbers[0] > 255 || numbers[1] > 255 || numbers[2] > 0xffff) return null;
|
|
46
|
+
a = numbers[0];
|
|
47
|
+
b = numbers[1];
|
|
48
|
+
c = (numbers[2] >>> 8) & 255;
|
|
49
|
+
d = numbers[2] & 255;
|
|
50
|
+
} else if (numbers.length === 4) {
|
|
51
|
+
if (numbers.some(n => n > 255)) return null;
|
|
52
|
+
[a, b, c, d] = numbers;
|
|
53
|
+
} else {
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return `${a}.${b}.${c}.${d}`;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
export function isPrivateIPv4(ip) {
|
|
61
|
+
const parsed = parseIPv4(ip) || (typeof ip === 'string' ? ip : '');
|
|
62
|
+
const parts = parsed.split('.').map(Number);
|
|
63
|
+
if (parts.length !== 4 || parts.some(p => isNaN(p) || p < 0 || p > 255)) return false;
|
|
64
|
+
|
|
65
|
+
if (parts[0] === 127) return true;
|
|
66
|
+
if (parts[0] === 10) return true;
|
|
67
|
+
if (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31) return true;
|
|
68
|
+
if (parts[0] === 192 && parts[1] === 168) return true;
|
|
69
|
+
if (parts[0] === 169 && parts[1] === 254) return true;
|
|
70
|
+
if (parts[0] === 0) return true;
|
|
71
|
+
if (parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127) return true;
|
|
72
|
+
if (parts[0] === 198 && (parts[1] === 18 || parts[1] === 19)) return true;
|
|
73
|
+
if (parts[0] >= 224) return true;
|
|
74
|
+
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
export function isPrivateIPv6(ip) {
|
|
79
|
+
const clean = ip.toLowerCase().trim();
|
|
80
|
+
|
|
81
|
+
if (clean === '::1' || clean === '0:0:0:0:0:0:0:1' || clean === '::') return true;
|
|
82
|
+
|
|
83
|
+
if (/^fe[89ab]/i.test(clean)) return true;
|
|
84
|
+
|
|
85
|
+
if (/^f[cd]/i.test(clean)) return true;
|
|
86
|
+
|
|
87
|
+
const mapped = clean.match(/^::ffff:(\d+\.\d+\.\d+\.\d+)$/);
|
|
88
|
+
if (mapped && isPrivateIPv4(mapped[1])) return true;
|
|
89
|
+
|
|
90
|
+
return false;
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
export function isIpAddress(host) {
|
|
94
|
+
if (!host || typeof host !== 'string') return false;
|
|
95
|
+
return Boolean(parseIPv4(host)) || host.includes(':');
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const DNS_CACHE = new Map();
|
|
99
|
+
const DNS_CACHE_TTL_MS = 60_000;
|
|
100
|
+
const DNS_CACHE_FAIL_TTL_MS = 10_000;
|
|
101
|
+
const DNS_CACHE_MAX_ENTRIES = 1000;
|
|
102
|
+
|
|
103
|
+
function setDnsCacheEntry(hostname, entry) {
|
|
104
|
+
if (!DNS_CACHE.has(hostname) && DNS_CACHE.size >= DNS_CACHE_MAX_ENTRIES) {
|
|
105
|
+
const oldestKey = DNS_CACHE.keys().next().value;
|
|
106
|
+
if (oldestKey !== undefined) DNS_CACHE.delete(oldestKey);
|
|
107
|
+
}
|
|
108
|
+
DNS_CACHE.set(hostname, entry);
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function clearDnsCache() {
|
|
112
|
+
DNS_CACHE.clear();
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const httpKeepAliveAgent = new http.Agent({ keepAlive: true, maxSockets: 20 });
|
|
116
|
+
const httpsKeepAliveAgent = new https.Agent({ keepAlive: true, maxSockets: 20 });
|
|
117
|
+
|
|
118
|
+
export async function assertSafeUrlHostname(hostname) {
|
|
119
|
+
if (!hostname || typeof hostname !== 'string') {
|
|
120
|
+
return { safe: false, reason: 'Missing hostname' };
|
|
121
|
+
}
|
|
122
|
+
const clean = hostname.replace(/^\[|\]$/g, '').replace(/\.$/, '').toLowerCase();
|
|
123
|
+
|
|
124
|
+
if (clean === 'localhost' || clean.endsWith('.localhost') || clean.includes('metadata.google') || clean.includes('169.254.169.254')) {
|
|
125
|
+
return { safe: false, reason: `Blocked hostname: ${clean}` };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
const parsedIp = parseIPv4(clean);
|
|
129
|
+
if (parsedIp) {
|
|
130
|
+
if (isPrivateIPv4(parsedIp)) {
|
|
131
|
+
return { safe: false, reason: `Target is private/internal IP: ${clean}` };
|
|
132
|
+
}
|
|
133
|
+
return { safe: true, pinnedIp: parsedIp };
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
if (isIpAddress(clean)) {
|
|
137
|
+
if (isPrivateIPv6(clean)) {
|
|
138
|
+
return { safe: false, reason: `Target is private/internal IP: ${clean}` };
|
|
139
|
+
}
|
|
140
|
+
return { safe: true, pinnedIp: clean };
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const cached = DNS_CACHE.get(clean);
|
|
144
|
+
if (cached && cached.expiresAt > Date.now()) {
|
|
145
|
+
return { safe: cached.safe, pinnedIp: cached.pinnedIp, reason: cached.reason };
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
try {
|
|
149
|
+
const addresses = await dnsLookup(clean, { all: true });
|
|
150
|
+
if (!addresses || addresses.length === 0) {
|
|
151
|
+
const res = { safe: false, reason: `Unable to resolve DNS for hostname "${clean}"` };
|
|
152
|
+
setDnsCacheEntry(clean, { ...res, expiresAt: Date.now() + DNS_CACHE_FAIL_TTL_MS });
|
|
153
|
+
return res;
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
for (const { address } of addresses) {
|
|
157
|
+
if (isPrivateIPv4(address) || isPrivateIPv6(address)) {
|
|
158
|
+
const res = { safe: false, reason: `Hostname "${clean}" resolved to private IP: ${address}` };
|
|
159
|
+
setDnsCacheEntry(clean, { ...res, expiresAt: Date.now() + DNS_CACHE_FAIL_TTL_MS });
|
|
160
|
+
return res;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
const res = { safe: true, pinnedIp: addresses[0].address };
|
|
165
|
+
setDnsCacheEntry(clean, { ...res, expiresAt: Date.now() + DNS_CACHE_TTL_MS });
|
|
166
|
+
return res;
|
|
167
|
+
} catch (err) {
|
|
168
|
+
const res = { safe: false, reason: `DNS lookup failed for "${clean}": ${err.message}` };
|
|
169
|
+
setDnsCacheEntry(clean, { ...res, expiresAt: Date.now() + DNS_CACHE_FAIL_TTL_MS });
|
|
170
|
+
return res;
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
export async function safeFetchWithPinning(inputUrl, options = {}, maxRedirects = 3) {
|
|
176
|
+
let currentUrl = String(inputUrl || '');
|
|
177
|
+
let currentOptions = { ...options };
|
|
178
|
+
|
|
179
|
+
for (let redirects = 0; redirects <= maxRedirects; redirects++) {
|
|
180
|
+
let parsedUrl;
|
|
181
|
+
try {
|
|
182
|
+
parsedUrl = new URL(currentUrl);
|
|
183
|
+
} catch {
|
|
184
|
+
throw new Error(`Invalid URL format: "${currentUrl}"`);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
if (parsedUrl.protocol !== 'http:' && parsedUrl.protocol !== 'https:') {
|
|
188
|
+
throw new Error(`Unsupported protocol: ${parsedUrl.protocol}`);
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
const hostname = parsedUrl.hostname.toLowerCase();
|
|
192
|
+
const ssrfCheck = await assertSafeUrlHostname(hostname);
|
|
193
|
+
if (!ssrfCheck.safe) {
|
|
194
|
+
throw new Error(`SSRF Guard rejected: ${ssrfCheck.reason}`);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const targetIp = ssrfCheck.pinnedIp;
|
|
198
|
+
const isHttps = parsedUrl.protocol === 'https:';
|
|
199
|
+
const client = isHttps ? https : http;
|
|
200
|
+
const port = parsedUrl.port ? Number(parsedUrl.port) : (isHttps ? 443 : 80);
|
|
201
|
+
const method = String(currentOptions.method || 'GET').toUpperCase();
|
|
202
|
+
|
|
203
|
+
const headers = { ...(currentOptions.headers || {}) };
|
|
204
|
+
if (!headers.host && !headers.Host) {
|
|
205
|
+
headers.Host = parsedUrl.host;
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
const agent = currentOptions.agent || (isHttps ? httpsKeepAliveAgent : httpKeepAliveAgent);
|
|
209
|
+
|
|
210
|
+
const response = await new Promise((resolve, reject) => {
|
|
211
|
+
let req = null;
|
|
212
|
+
try {
|
|
213
|
+
req = client.request({
|
|
214
|
+
host: targetIp,
|
|
215
|
+
port,
|
|
216
|
+
path: parsedUrl.pathname + parsedUrl.search,
|
|
217
|
+
method,
|
|
218
|
+
headers,
|
|
219
|
+
agent,
|
|
220
|
+
servername: isHttps ? hostname : undefined,
|
|
221
|
+
signal: currentOptions.signal,
|
|
222
|
+
timeout: currentOptions.timeoutMs || 15000
|
|
223
|
+
}, res => {
|
|
224
|
+
const chunks = [];
|
|
225
|
+
let bytesReceived = 0;
|
|
226
|
+
const maxBytes = currentOptions.maxBytes || 5_000_000;
|
|
227
|
+
|
|
228
|
+
res.on('data', chunk => {
|
|
229
|
+
bytesReceived += chunk.length;
|
|
230
|
+
if (bytesReceived <= maxBytes) {
|
|
231
|
+
chunks.push(chunk);
|
|
232
|
+
} else {
|
|
233
|
+
res.destroy();
|
|
234
|
+
req.destroy();
|
|
235
|
+
}
|
|
236
|
+
});
|
|
237
|
+
|
|
238
|
+
res.on('end', () => {
|
|
239
|
+
const bodyBuffer = Buffer.concat(chunks);
|
|
240
|
+
const responseHeaders = new Headers();
|
|
241
|
+
for (const [k, v] of Object.entries(res.headers)) {
|
|
242
|
+
if (Array.isArray(v)) {
|
|
243
|
+
v.forEach(val => responseHeaders.append(k, val));
|
|
244
|
+
} else if (v !== undefined) {
|
|
245
|
+
responseHeaders.set(k, v);
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
resolve({
|
|
250
|
+
status: res.statusCode || 200,
|
|
251
|
+
statusText: res.statusMessage || '',
|
|
252
|
+
ok: res.statusCode >= 200 && res.statusCode < 300,
|
|
253
|
+
headers: responseHeaders,
|
|
254
|
+
url: parsedUrl.href,
|
|
255
|
+
text: async () => bodyBuffer.toString('utf8'),
|
|
256
|
+
json: async () => JSON.parse(bodyBuffer.toString('utf8')),
|
|
257
|
+
buffer: async () => bodyBuffer
|
|
258
|
+
});
|
|
259
|
+
});
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
req.on('error', reject);
|
|
263
|
+
req.on('timeout', () => {
|
|
264
|
+
req.destroy(new Error(`Request to ${hostname} (${targetIp}) timed out`));
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
if (currentOptions.body) {
|
|
268
|
+
req.write(currentOptions.body);
|
|
269
|
+
}
|
|
270
|
+
req.end();
|
|
271
|
+
} catch (err) {
|
|
272
|
+
reject(err);
|
|
273
|
+
}
|
|
274
|
+
});
|
|
275
|
+
|
|
276
|
+
const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308]);
|
|
277
|
+
if (!REDIRECT_STATUSES.has(response.status)) {
|
|
278
|
+
return response;
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const location = response.headers.get('location');
|
|
282
|
+
if (!location) return response;
|
|
283
|
+
if (redirects === maxRedirects) throw new Error('Too many redirects');
|
|
284
|
+
|
|
285
|
+
currentUrl = new URL(location, parsedUrl).href;
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
throw new Error('Too many redirects');
|
|
289
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
export class SearchTelemetry {
|
|
2
|
+
constructor() {
|
|
3
|
+
this.reset();
|
|
4
|
+
}
|
|
5
|
+
|
|
6
|
+
reset() {
|
|
7
|
+
this.totalSearches = 0;
|
|
8
|
+
this.fallbackTriggers = 0;
|
|
9
|
+
this.zeroResults = 0;
|
|
10
|
+
this.latencies = [];
|
|
11
|
+
this.circuitBreakerTrips = {
|
|
12
|
+
duckduckgo: 0,
|
|
13
|
+
searxng: 0,
|
|
14
|
+
jina: 0
|
|
15
|
+
};
|
|
16
|
+
this.searchesByEngine = {
|
|
17
|
+
duckduckgo: 0,
|
|
18
|
+
searxng: 0,
|
|
19
|
+
jina: 0
|
|
20
|
+
};
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
recordSearch({ latencyMs = 0, fallbackOccurred = false, zeroResult = false, engine = null } = {}) {
|
|
24
|
+
this.totalSearches++;
|
|
25
|
+
if (fallbackOccurred) this.fallbackTriggers++;
|
|
26
|
+
if (zeroResult) this.zeroResults++;
|
|
27
|
+
if (Number.isFinite(latencyMs) && latencyMs >= 0) {
|
|
28
|
+
this.latencies.push(latencyMs);
|
|
29
|
+
if (this.latencies.length > 2000) {
|
|
30
|
+
this.latencies = this.latencies.slice(-1000);
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
if (engine && this.searchesByEngine[engine] !== undefined) {
|
|
34
|
+
this.searchesByEngine[engine]++;
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
recordCircuitTrip(engine) {
|
|
39
|
+
const key = String(engine || '').toLowerCase();
|
|
40
|
+
if (this.circuitBreakerTrips[key] !== undefined) {
|
|
41
|
+
this.circuitBreakerTrips[key]++;
|
|
42
|
+
} else {
|
|
43
|
+
this.circuitBreakerTrips[key] = 1;
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
calculateP95Latency() {
|
|
48
|
+
if (this.latencies.length === 0) return 0;
|
|
49
|
+
const sorted = [...this.latencies].sort((a, b) => a - b);
|
|
50
|
+
const index = Math.ceil(sorted.length * 0.95) - 1;
|
|
51
|
+
return sorted[Math.max(0, index)];
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
getMetrics() {
|
|
55
|
+
const total = this.totalSearches || 1;
|
|
56
|
+
const fallbackRate = (this.fallbackTriggers / total);
|
|
57
|
+
const zeroRate = (this.zeroResults / total);
|
|
58
|
+
const p95 = this.calculateP95Latency();
|
|
59
|
+
|
|
60
|
+
return {
|
|
61
|
+
totalSearches: this.totalSearches,
|
|
62
|
+
fallbackTriggerRate: Number(fallbackRate.toFixed(4)),
|
|
63
|
+
zeroResultRate: Number(zeroRate.toFixed(4)),
|
|
64
|
+
searchLatencyP95Ms: p95,
|
|
65
|
+
circuitBreakerTrips: { ...this.circuitBreakerTrips },
|
|
66
|
+
searchesByEngine: { ...this.searchesByEngine }
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export const defaultTelemetry = new SearchTelemetry();
|
|
@@ -1,24 +1,32 @@
|
|
|
1
1
|
import fs from 'fs';
|
|
2
2
|
import path from 'path';
|
|
3
|
-
import { spawn,
|
|
3
|
+
import { spawn, execFile } from 'child_process';
|
|
4
|
+
import { promisify } from 'util';
|
|
4
5
|
import { fileURLToPath } from 'url';
|
|
5
6
|
import { KIKU_DATA_DIR } from './appPaths.js';
|
|
7
|
+
import { PACKAGE_ENV_PATH, USER_ENV_PATH, upsertEnvKey } from './envManager.js';
|
|
8
|
+
import { bootstrapEnv } from './envBootstrap.js';
|
|
6
9
|
|
|
7
10
|
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
|
8
|
-
|
|
11
|
+
|
|
12
|
+
bootstrapEnv();
|
|
13
|
+
|
|
14
|
+
const execFileAsync = promisify(execFile);
|
|
15
|
+
const getPrimaryEnvPath = () => (fs.existsSync(PACKAGE_ENV_PATH) ? PACKAGE_ENV_PATH : USER_ENV_PATH);
|
|
9
16
|
const AUTH_CONFIG_PATH = path.join(KIKU_DATA_DIR, 'yt_auth.json');
|
|
10
|
-
const
|
|
11
|
-
|
|
12
|
-
const YTDLP_CMD = process.env.KIKU_YTDLP || 'yt-dlp';
|
|
17
|
+
const COOKIE_FILE_PATH = path.join(KIKU_DATA_DIR, 'cookies.txt');
|
|
18
|
+
|
|
19
|
+
const YTDLP_CMD = () => process.env.KIKU_YTDLP || 'yt-dlp';
|
|
13
20
|
|
|
14
21
|
|
|
15
22
|
const BROWSERS = ['firefox', 'chrome', 'edge', 'brave'];
|
|
23
|
+
let browserDetectionCache = null;
|
|
16
24
|
|
|
17
|
-
function
|
|
25
|
+
async function findBrowserExeAsync(browser, { execImpl = execFileAsync } = {}) {
|
|
18
26
|
const isWin = process.platform === 'win32';
|
|
19
27
|
const probes = {
|
|
20
28
|
firefox: {
|
|
21
|
-
cmd:
|
|
29
|
+
cmd: 'firefox',
|
|
22
30
|
paths: [
|
|
23
31
|
'C:\\Program Files\\Mozilla Firefox\\firefox.exe',
|
|
24
32
|
'C:\\Program Files (x86)\\Mozilla Firefox\\firefox.exe',
|
|
@@ -53,7 +61,7 @@ function findBrowserExe(browser) {
|
|
|
53
61
|
const p = probes[browser];
|
|
54
62
|
if (!p) return null;
|
|
55
63
|
try {
|
|
56
|
-
|
|
64
|
+
await execImpl(isWin ? 'where' : 'which', [p.cmd], { windowsHide: true, timeout: 1500 });
|
|
57
65
|
return p.cmd;
|
|
58
66
|
} catch {}
|
|
59
67
|
|
|
@@ -65,12 +73,18 @@ function findBrowserExe(browser) {
|
|
|
65
73
|
return null;
|
|
66
74
|
}
|
|
67
75
|
|
|
68
|
-
function
|
|
69
|
-
|
|
76
|
+
async function detectBrowsersCached({ execImpl } = {}) {
|
|
77
|
+
if (browserDetectionCache) return browserDetectionCache;
|
|
78
|
+
const found = [];
|
|
79
|
+
for (const b of BROWSERS) {
|
|
80
|
+
if (await findBrowserExeAsync(b, { execImpl })) found.push(b);
|
|
81
|
+
}
|
|
82
|
+
browserDetectionCache = found;
|
|
83
|
+
return found;
|
|
70
84
|
}
|
|
71
85
|
|
|
72
|
-
function openBrowser(url, browser) {
|
|
73
|
-
const exe = browser ?
|
|
86
|
+
async function openBrowser(url, browser) {
|
|
87
|
+
const exe = browser ? await findBrowserExeAsync(browser) : null;
|
|
74
88
|
const isWin = process.platform === 'win32';
|
|
75
89
|
try {
|
|
76
90
|
if (exe) {
|
|
@@ -105,8 +119,7 @@ function writeAuthConfig(updates) {
|
|
|
105
119
|
} catch {}
|
|
106
120
|
}
|
|
107
121
|
|
|
108
|
-
function
|
|
109
|
-
|
|
122
|
+
function refreshYtAuthEnv() {
|
|
110
123
|
const userConfig = readAuthConfig();
|
|
111
124
|
if (userConfig.KIKU_YT_BROWSER && process.env.KIKU_YT_BROWSER === undefined) {
|
|
112
125
|
process.env.KIKU_YT_BROWSER = userConfig.KIKU_YT_BROWSER;
|
|
@@ -114,62 +127,40 @@ function loadEnv() {
|
|
|
114
127
|
if (userConfig.KIKU_YT_COOKIES && process.env.KIKU_YT_COOKIES === undefined) {
|
|
115
128
|
process.env.KIKU_YT_COOKIES = userConfig.KIKU_YT_COOKIES;
|
|
116
129
|
}
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
try {
|
|
120
|
-
if (!fs.existsSync(ENV_PATH)) return;
|
|
121
|
-
const text = fs.readFileSync(ENV_PATH, 'utf8');
|
|
122
|
-
for (const line of text.split('\n')) {
|
|
123
|
-
const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
|
|
124
|
-
if (!m) continue;
|
|
125
|
-
const key = m[1].trim();
|
|
126
|
-
const val = m[2].trim().replace(/^["']|["']$/g, '');
|
|
127
|
-
if (process.env[key] === undefined) process.env[key] = val;
|
|
128
|
-
}
|
|
129
|
-
} catch {}
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
function readEnvMap() {
|
|
133
|
-
const map = {};
|
|
134
|
-
try {
|
|
135
|
-
if (!fs.existsSync(ENV_PATH)) return map;
|
|
136
|
-
const text = fs.readFileSync(ENV_PATH, 'utf8');
|
|
137
|
-
for (const line of text.split('\n')) {
|
|
138
|
-
const m = line.match(/^\s*([\w.-]+)\s*=\s*(.*)\s*$/);
|
|
139
|
-
if (m) map[m[1]] = m[2].replace(/^["']|["']$/g, '');
|
|
140
|
-
}
|
|
141
|
-
} catch {}
|
|
142
|
-
return map;
|
|
143
130
|
}
|
|
144
131
|
|
|
145
132
|
function writeEnv(updates) {
|
|
146
|
-
|
|
147
|
-
for (const [k, v] of Object.entries(updates)) {
|
|
148
|
-
|
|
133
|
+
const cleanUpdates = {};
|
|
134
|
+
for (const [k, v] of Object.entries(updates || {})) {
|
|
135
|
+
const cleanKey = String(k || '').replace(/[\r\n]+/g, '').trim();
|
|
136
|
+
const cleanVal = String(v ?? '').replace(/[\r\n]+/g, '').trim();
|
|
137
|
+
if (/^[A-Za-z_][A-Za-z0-9_]*$/.test(cleanKey)) {
|
|
138
|
+
cleanUpdates[cleanKey] = cleanVal;
|
|
139
|
+
process.env[cleanKey] = cleanVal;
|
|
140
|
+
}
|
|
149
141
|
}
|
|
150
|
-
writeAuthConfig(
|
|
142
|
+
writeAuthConfig(cleanUpdates);
|
|
151
143
|
|
|
152
|
-
|
|
153
144
|
try {
|
|
154
|
-
const
|
|
155
|
-
Object.
|
|
156
|
-
|
|
157
|
-
|
|
145
|
+
const envFile = getPrimaryEnvPath();
|
|
146
|
+
for (const [key, value] of Object.entries(cleanUpdates)) {
|
|
147
|
+
upsertEnvKey(envFile, key, value);
|
|
148
|
+
}
|
|
158
149
|
} catch {}
|
|
159
150
|
}
|
|
160
151
|
|
|
161
|
-
function hasSession(browser) {
|
|
152
|
+
function hasSession(browser, { timeoutMs = 6000 } = {}) {
|
|
162
153
|
return new Promise((resolve) => {
|
|
163
|
-
const
|
|
154
|
+
const probeUrl = process.env.KIKU_YT_PROBE_URL || 'https://www.youtube.com/watch?v=dQw4w9WgXcQ';
|
|
164
155
|
const args = [
|
|
165
156
|
'--cookies-from-browser', browser,
|
|
166
157
|
'--print', '%(title)s',
|
|
167
158
|
'--no-warnings',
|
|
168
159
|
'--no-playlist',
|
|
169
160
|
'--no-check-certificates',
|
|
170
|
-
|
|
161
|
+
probeUrl
|
|
171
162
|
];
|
|
172
|
-
const p = spawn(YTDLP_CMD, args, {
|
|
163
|
+
const p = spawn(YTDLP_CMD(), args, {
|
|
173
164
|
windowsHide: true,
|
|
174
165
|
stdio: ['ignore', 'pipe', 'pipe']
|
|
175
166
|
});
|
|
@@ -182,17 +173,18 @@ function hasSession(browser) {
|
|
|
182
173
|
if (!resolved) {
|
|
183
174
|
resolved = true;
|
|
184
175
|
try { p.kill('SIGKILL'); } catch {}
|
|
185
|
-
resolve(false);
|
|
176
|
+
resolve({ ok: false, reason: 'timeout' });
|
|
186
177
|
}
|
|
187
178
|
}, timeoutMs);
|
|
188
179
|
|
|
189
180
|
p.stdout?.on('data', d => { out += d.toString(); });
|
|
190
181
|
p.stderr?.on('data', d => { err += d.toString(); });
|
|
191
|
-
p.on('error', () => {
|
|
182
|
+
p.on('error', (spawnErr) => {
|
|
192
183
|
if (!resolved) {
|
|
193
184
|
resolved = true;
|
|
194
185
|
clearTimeout(timer);
|
|
195
|
-
|
|
186
|
+
const missing = spawnErr?.code === 'ENOENT';
|
|
187
|
+
resolve({ ok: false, reason: missing ? 'yt-dlp-missing' : 'spawn-error', detail: spawnErr?.message });
|
|
196
188
|
}
|
|
197
189
|
});
|
|
198
190
|
p.on('close', (code) => {
|
|
@@ -201,7 +193,11 @@ function hasSession(browser) {
|
|
|
201
193
|
clearTimeout(timer);
|
|
202
194
|
const title = out.trim();
|
|
203
195
|
const blocked = /403|Forbidden|Sign in|confirm you'?re not a bot|Login Required|Private Video|database is locked/i.test(err);
|
|
204
|
-
|
|
196
|
+
if (code === 0 && title.length > 0 && !blocked) {
|
|
197
|
+
resolve({ ok: true, reason: 'session-found' });
|
|
198
|
+
} else {
|
|
199
|
+
resolve({ ok: false, reason: blocked ? 'blocked' : 'no-session', detail: err.slice(0, 200) });
|
|
200
|
+
}
|
|
205
201
|
});
|
|
206
202
|
});
|
|
207
203
|
}
|
|
@@ -248,18 +244,17 @@ export function isCookieFileValid(filePath) {
|
|
|
248
244
|
}
|
|
249
245
|
|
|
250
246
|
async function _ensureYouTubeCookiesImpl(opts = {}) {
|
|
251
|
-
|
|
247
|
+
refreshYtAuthEnv();
|
|
252
248
|
|
|
253
249
|
const cookiesCandidate = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
|
|
254
250
|
? process.env.KIKU_YT_COOKIES
|
|
255
|
-
: (isCookieFileValid(
|
|
251
|
+
: (isCookieFileValid(COOKIE_FILE_PATH) ? COOKIE_FILE_PATH : null);
|
|
256
252
|
if (cookiesCandidate && isCookieFileValid(cookiesCandidate)) {
|
|
257
253
|
_ensured = { ok: true, source: 'file', cookiesPath: cookiesCandidate };
|
|
258
254
|
return _ensured;
|
|
259
255
|
}
|
|
260
256
|
|
|
261
|
-
|
|
262
|
-
const available = detectBrowsers();
|
|
257
|
+
const available = await detectBrowsersCached();
|
|
263
258
|
if (!available.length) {
|
|
264
259
|
_ensured = { ok: false, reason: 'no-browser', detail: 'No supported browser found (Firefox, Chrome, Edge, Brave)' };
|
|
265
260
|
_lastFailTime = Date.now();
|
|
@@ -271,12 +266,21 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
|
|
|
271
266
|
? [preferred, ...available.filter(b => b !== preferred)]
|
|
272
267
|
: available;
|
|
273
268
|
|
|
269
|
+
let sawYtdlpMissing = false;
|
|
274
270
|
for (const b of candidates) {
|
|
275
|
-
|
|
271
|
+
const probe = await hasSession(b, { timeoutMs: opts.probeTimeoutMs });
|
|
272
|
+
if (probe.ok) {
|
|
276
273
|
writeEnv({ KIKU_YT_BROWSER: b });
|
|
277
274
|
_ensured = { ok: true, source: 'existing-session', browser: b };
|
|
278
275
|
return _ensured;
|
|
279
276
|
}
|
|
277
|
+
if (probe.reason === 'yt-dlp-missing') sawYtdlpMissing = true;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
if (sawYtdlpMissing) {
|
|
281
|
+
_ensured = { ok: false, reason: 'yt-dlp-missing', detail: `yt-dlp was not found (set KIKU_YTDLP in .env, e.g. ${YTDLP_CMD()})` };
|
|
282
|
+
_lastFailTime = Date.now();
|
|
283
|
+
return _ensured;
|
|
280
284
|
}
|
|
281
285
|
|
|
282
286
|
|
|
@@ -310,7 +314,8 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
|
|
|
310
314
|
}
|
|
311
315
|
|
|
312
316
|
for (const b of available) {
|
|
313
|
-
|
|
317
|
+
const probe = await hasSession(b, { timeoutMs: opts.probeTimeoutMs });
|
|
318
|
+
if (probe.ok) {
|
|
314
319
|
writeEnv({ KIKU_YT_BROWSER: b });
|
|
315
320
|
if (process.stdout.isTTY) {
|
|
316
321
|
process.stdout.write('\r' + ' '.repeat(65) + '\r');
|
|
@@ -335,9 +340,9 @@ async function _ensureYouTubeCookiesImpl(opts = {}) {
|
|
|
335
340
|
}
|
|
336
341
|
|
|
337
342
|
export async function exportCookiesToFile() {
|
|
338
|
-
|
|
343
|
+
refreshYtAuthEnv();
|
|
339
344
|
|
|
340
|
-
const available =
|
|
345
|
+
const available = await detectBrowsersCached();
|
|
341
346
|
const preferred = process.env.KIKU_YT_BROWSER;
|
|
342
347
|
const candidates = preferred && available.includes(preferred)
|
|
343
348
|
? [preferred, ...available.filter(b => b !== preferred)]
|
|
@@ -348,18 +353,18 @@ export async function exportCookiesToFile() {
|
|
|
348
353
|
fs.mkdirSync(KIKU_DATA_DIR, { recursive: true });
|
|
349
354
|
const args = [
|
|
350
355
|
'--cookies-from-browser', b,
|
|
351
|
-
'--cookies',
|
|
356
|
+
'--cookies', COOKIE_FILE_PATH,
|
|
352
357
|
'--no-warnings',
|
|
353
358
|
'--skip-download',
|
|
354
|
-
'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
|
|
359
|
+
process.env.KIKU_YT_PROBE_URL || 'https://www.youtube.com/watch?v=dQw4w9WgXcQ'
|
|
355
360
|
];
|
|
356
361
|
await new Promise((resolve, reject) => {
|
|
357
|
-
const p = spawn(YTDLP_CMD, args, { windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
362
|
+
const p = spawn(YTDLP_CMD(), args, { windowsHide: true, stdio: ['ignore', 'ignore', 'pipe'] });
|
|
358
363
|
let err = '';
|
|
359
364
|
p.stderr?.on('data', d => { err += d.toString(); });
|
|
360
365
|
p.on('error', reject);
|
|
361
366
|
p.on('close', code => {
|
|
362
|
-
if (code === 0 && fs.existsSync(
|
|
367
|
+
if (code === 0 && fs.existsSync(COOKIE_FILE_PATH)) {
|
|
363
368
|
resolve();
|
|
364
369
|
} else {
|
|
365
370
|
reject(new Error(err || `yt-dlp exited ${code}`));
|
|
@@ -367,8 +372,11 @@ export async function exportCookiesToFile() {
|
|
|
367
372
|
});
|
|
368
373
|
});
|
|
369
374
|
|
|
370
|
-
|
|
371
|
-
|
|
375
|
+
if (process.platform !== 'win32') {
|
|
376
|
+
try { fs.chmodSync(COOKIE_FILE_PATH, 0o600); } catch {}
|
|
377
|
+
}
|
|
378
|
+
writeEnv({ KIKU_YT_COOKIES: COOKIE_FILE_PATH });
|
|
379
|
+
return { ok: true, browser: b, cookiesPath: COOKIE_FILE_PATH };
|
|
372
380
|
} catch {}
|
|
373
381
|
}
|
|
374
382
|
|
|
@@ -384,12 +392,13 @@ export function resetCookieCheck() {
|
|
|
384
392
|
export function getCookieStatus() {
|
|
385
393
|
const envCookies = (process.env.KIKU_YT_COOKIES && isCookieFileValid(process.env.KIKU_YT_COOKIES))
|
|
386
394
|
? process.env.KIKU_YT_COOKIES
|
|
387
|
-
: (isCookieFileValid(
|
|
395
|
+
: (isCookieFileValid(COOKIE_FILE_PATH) ? COOKIE_FILE_PATH : null);
|
|
388
396
|
return {
|
|
389
397
|
ensured: _ensured,
|
|
390
398
|
envBrowser: process.env.KIKU_YT_BROWSER || null,
|
|
391
399
|
envCookies,
|
|
392
|
-
envPath:
|
|
400
|
+
envPath: getPrimaryEnvPath(),
|
|
401
|
+
userEnvPath: USER_ENV_PATH,
|
|
393
402
|
authConfigPath: AUTH_CONFIG_PATH
|
|
394
403
|
};
|
|
395
404
|
}
|