kanha-ai 0.1.2 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +316 -134
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +47 -6
- package/dist/index.d.ts +47 -6
- package/dist/index.js +316 -134
- package/dist/index.js.map +1 -1
- package/dist/rag.cjs +61 -0
- package/dist/rag.cjs.map +1 -0
- package/dist/rag.d.cts +16 -0
- package/dist/rag.d.ts +16 -0
- package/dist/rag.js +59 -0
- package/dist/rag.js.map +1 -0
- package/dist/widget.d.ts +48 -6
- package/dist/widget.js +336 -59
- package/dist/widget.js.map +1 -1
- package/dist/worker.js +21404 -0
- package/dist/worker.js.map +1 -0
- package/package.json +20 -5
package/dist/index.cjs
CHANGED
|
@@ -9,98 +9,153 @@ function _interopDefault (e) { return e && e.__esModule ? e : { default: e }; }
|
|
|
9
9
|
var ReactMarkdown__default = /*#__PURE__*/_interopDefault(ReactMarkdown);
|
|
10
10
|
|
|
11
11
|
// src/KanhaBot.tsx
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
12
|
+
|
|
13
|
+
// src/webllm.ts
|
|
14
|
+
var MODEL_ID = "kanha-custom-model";
|
|
15
|
+
var PREBUILT_MODEL_BY_SIZE = {
|
|
16
|
+
small: "Qwen3-0.6B-q4f16_1-MLC",
|
|
17
|
+
medium: "Qwen3-1.7B-q4f16_1-MLC",
|
|
18
|
+
large: "Qwen3-4B-q4f16_1-MLC"
|
|
19
|
+
};
|
|
20
|
+
var MIN_RAM_GB = {
|
|
21
|
+
small: 2,
|
|
22
|
+
medium: 4,
|
|
23
|
+
large: 8
|
|
17
24
|
};
|
|
18
|
-
function
|
|
25
|
+
async function checkWebGPU() {
|
|
26
|
+
try {
|
|
27
|
+
const nav = navigator;
|
|
28
|
+
return !!nav.gpu && !!await nav.gpu.requestAdapter();
|
|
29
|
+
} catch {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
function checkAvailableMemory(config) {
|
|
34
|
+
const nav = navigator;
|
|
35
|
+
const required = config.minRamGb ?? MIN_RAM_GB[config.modelSize ?? "small"] ?? MIN_RAM_GB.small;
|
|
36
|
+
if (nav.deviceMemory && nav.deviceMemory < 8 && nav.deviceMemory < required) {
|
|
37
|
+
const modelSize = config.modelSize ?? "small";
|
|
38
|
+
return `Your device has ~${nav.deviceMemory}GB RAM but the ${modelSize} model needs at least ${required}GB.`;
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
function resolveModelLib(webllm, config) {
|
|
19
43
|
if (config.modelLib) return config.modelLib;
|
|
20
44
|
const size = config.modelSize ?? "small";
|
|
21
|
-
const
|
|
22
|
-
|
|
23
|
-
|
|
45
|
+
const modelId = PREBUILT_MODEL_BY_SIZE[size];
|
|
46
|
+
const modelLib = webllm.prebuiltAppConfig.model_list.find((model) => model.model_id === modelId)?.model_lib;
|
|
47
|
+
if (!modelLib) throw new Error(`Unknown model size "${size}". Provide modelLib explicitly.`);
|
|
48
|
+
return modelLib;
|
|
24
49
|
}
|
|
25
50
|
function toWebLLMModelUrl(modelUrl) {
|
|
26
51
|
let url = modelUrl.endsWith("/") ? modelUrl.slice(0, -1) : modelUrl;
|
|
27
|
-
if (!url.includes("/resolve/"))
|
|
28
|
-
url += "/resolve/v1";
|
|
29
|
-
}
|
|
52
|
+
if (!url.includes("/resolve/")) url += "/resolve/main";
|
|
30
53
|
return url;
|
|
31
54
|
}
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
55
|
+
function createAppConfig(webllm, config) {
|
|
56
|
+
const modelLib = resolveModelLib(webllm, config);
|
|
57
|
+
const size = config.modelSize ?? "small";
|
|
58
|
+
const modelId = PREBUILT_MODEL_BY_SIZE[size];
|
|
59
|
+
const prebuiltModel = webllm.prebuiltAppConfig.model_list.find((model) => model.model_id === modelId);
|
|
60
|
+
const finalModelUrl = config.modelUrl ? toWebLLMModelUrl(config.modelUrl) : prebuiltModel?.model;
|
|
61
|
+
if (!finalModelUrl) {
|
|
62
|
+
throw new Error(`Failed to resolve model URL for size "${size}".`);
|
|
63
|
+
}
|
|
64
|
+
return {
|
|
65
|
+
model_list: [{
|
|
66
|
+
model: finalModelUrl,
|
|
67
|
+
model_id: MODEL_ID,
|
|
68
|
+
model_lib: modelLib,
|
|
69
|
+
overrides: { context_window_size: config.contextWindowSize ?? 4096 }
|
|
70
|
+
}],
|
|
71
|
+
useIndexedDBCache: (config.cacheBackend ?? "indexeddb") === "indexeddb"
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
async function createEngine(webllm, config, initProgressCallback) {
|
|
75
|
+
const appConfig = createAppConfig(webllm, config);
|
|
76
|
+
if (config.workerUrl) {
|
|
77
|
+
const worker = new Worker(config.workerUrl, { type: "module" });
|
|
78
|
+
try {
|
|
79
|
+
const engine2 = await webllm.CreateWebWorkerMLCEngine(worker, MODEL_ID, {
|
|
80
|
+
appConfig,
|
|
81
|
+
initProgressCallback,
|
|
82
|
+
logLevel: "WARN"
|
|
83
|
+
});
|
|
84
|
+
return { engine: engine2, worker };
|
|
85
|
+
} catch (error) {
|
|
86
|
+
worker.terminate();
|
|
87
|
+
throw error;
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
const engine = await webllm.CreateMLCEngine(MODEL_ID, {
|
|
91
|
+
appConfig,
|
|
92
|
+
initProgressCallback,
|
|
93
|
+
logLevel: "WARN"
|
|
94
|
+
});
|
|
95
|
+
return { engine, worker: null };
|
|
96
|
+
}
|
|
97
|
+
async function disposeEngine(engine, worker) {
|
|
38
98
|
try {
|
|
39
|
-
|
|
40
|
-
if (!nav.gpu) return false;
|
|
41
|
-
const adapter = await nav.gpu.requestAdapter();
|
|
42
|
-
if (!adapter) return false;
|
|
43
|
-
const device = await adapter.requestDevice();
|
|
44
|
-
return !!device;
|
|
99
|
+
await engine.unload();
|
|
45
100
|
} catch {
|
|
46
|
-
return false;
|
|
47
101
|
}
|
|
102
|
+
worker?.terminate();
|
|
48
103
|
}
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
104
|
+
function toMetrics(usage, config) {
|
|
105
|
+
return {
|
|
106
|
+
promptTokens: usage.prompt_tokens,
|
|
107
|
+
completionTokens: usage.completion_tokens,
|
|
108
|
+
totalTokens: usage.total_tokens,
|
|
109
|
+
e2eLatencyMs: (usage.extra?.e2e_latency_s ?? 0) * 1e3,
|
|
110
|
+
timeToFirstTokenMs: (usage.extra?.time_to_first_token_s ?? 0) * 1e3,
|
|
111
|
+
prefillTokensPerSecond: usage.extra?.prefill_tokens_per_s ?? 0,
|
|
112
|
+
decodeTokensPerSecond: usage.extra?.decode_tokens_per_s ?? 0,
|
|
113
|
+
modelSize: config.modelSize ?? "small",
|
|
114
|
+
modelId: MODEL_ID,
|
|
115
|
+
timestamp: (/* @__PURE__ */ new Date()).toISOString()
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
function emitMetrics(usage, config) {
|
|
58
119
|
try {
|
|
59
|
-
|
|
60
|
-
if (nav2.gpu) {
|
|
61
|
-
const adapter = await nav2.gpu.requestAdapter();
|
|
62
|
-
if (adapter) {
|
|
63
|
-
const maxBufferSize = adapter.limits.maxBufferSize;
|
|
64
|
-
const needed2 = MODEL_WEIGHT_BYTES[modelSize] ?? MODEL_WEIGHT_BYTES.small;
|
|
65
|
-
if (maxBufferSize < needed2) {
|
|
66
|
-
const maxMB = Math.round(maxBufferSize / (1024 * 1024));
|
|
67
|
-
const needMB = Math.round(needed2 / (1024 * 1024));
|
|
68
|
-
return `Your GPU can only allocate ~${maxMB}MB but the ${modelSize} model needs ~${needMB}MB. Try a smaller model or close other GPU-heavy tabs.`;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
}
|
|
120
|
+
config.onMetrics?.(toMetrics(usage, config));
|
|
72
121
|
} catch {
|
|
73
122
|
}
|
|
74
|
-
|
|
75
|
-
|
|
123
|
+
}
|
|
124
|
+
function resetEngine(engine) {
|
|
76
125
|
try {
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
return `Your device doesn't have enough free memory to load the ${modelSize} model. Close other tabs and try again.`;
|
|
80
|
-
}
|
|
126
|
+
void engine.resetChat().catch(() => {
|
|
127
|
+
});
|
|
81
128
|
} catch {
|
|
82
|
-
const needMB = Math.round(needed / (1024 * 1024));
|
|
83
|
-
return `Not enough free memory to load the ${modelSize} model (~${needMB}MB required). Close other tabs and try again.`;
|
|
84
|
-
}
|
|
85
|
-
const nav = navigator;
|
|
86
|
-
if (nav.deviceMemory) {
|
|
87
|
-
const required = MIN_RAM_GB[modelSize] ?? MIN_RAM_GB.small;
|
|
88
|
-
if (nav.deviceMemory < required) {
|
|
89
|
-
return `Your device has ~${nav.deviceMemory}GB RAM but the ${modelSize} model needs at least ${required}GB.`;
|
|
90
|
-
}
|
|
91
129
|
}
|
|
92
|
-
return null;
|
|
93
|
-
}
|
|
94
|
-
function isMemoryError(err) {
|
|
95
|
-
if (!(err instanceof Error)) return false;
|
|
96
|
-
const msg = err.message.toLowerCase();
|
|
97
|
-
return msg.includes("out of memory") || msg.includes("oom") || msg.includes("allocation failed") || msg.includes("err_failed") || msg.includes("arraybuffer") || msg.includes("could not allocate") || msg.includes("webgpu device lost");
|
|
98
130
|
}
|
|
99
131
|
function stripThinkTokens(text) {
|
|
100
132
|
let cleaned = text.replace(/<think>[\s\S]*?<\/think>/g, "");
|
|
101
133
|
cleaned = cleaned.replace(/<think>[\s\S]*$/g, "");
|
|
102
134
|
return cleaned.trimStart();
|
|
103
135
|
}
|
|
136
|
+
|
|
137
|
+
// src/rag-prompt.ts
|
|
138
|
+
var DEFAULT_RAG_PROMPT_TEMPLATE = "Use the following context to answer the question.\n\nContext:\n{context}\n\nQuestion:\n{query}";
|
|
139
|
+
function formatRagPrompt(context, query, template = DEFAULT_RAG_PROMPT_TEMPLATE) {
|
|
140
|
+
return template.replace(
|
|
141
|
+
/\{(context|query)\}/g,
|
|
142
|
+
(_, placeholder) => placeholder === "context" ? context : query
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// src/engine-history.ts
|
|
147
|
+
function buildEngineHistory(messages, maxHistoryMessages, currentUserContent) {
|
|
148
|
+
const history = messages.slice(-Math.max(1, maxHistoryMessages ?? 12));
|
|
149
|
+
history[history.length - 1] = { role: "user", content: currentUserContent };
|
|
150
|
+
return history;
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
// src/use-kanha-chat.ts
|
|
154
|
+
function isMemoryError(err) {
|
|
155
|
+
if (!(err instanceof Error)) return false;
|
|
156
|
+
const msg = err.message.toLowerCase();
|
|
157
|
+
return msg.includes("out of memory") || msg.includes("oom") || msg.includes("allocation failed") || msg.includes("arraybuffer") || msg.includes("could not allocate") || msg.includes("webgpu device lost");
|
|
158
|
+
}
|
|
104
159
|
function useKanhaChat(config) {
|
|
105
160
|
const [messages, setMessages] = react.useState([]);
|
|
106
161
|
const [input, setInput] = react.useState("");
|
|
@@ -110,9 +165,20 @@ function useKanhaChat(config) {
|
|
|
110
165
|
const [loadProgress, setLoadProgress] = react.useState(0);
|
|
111
166
|
const [error, setError] = react.useState(null);
|
|
112
167
|
const engineRef = react.useRef(null);
|
|
168
|
+
const workerRef = react.useRef(null);
|
|
169
|
+
const streamTimerRef = react.useRef(null);
|
|
170
|
+
const pendingTextRef = react.useRef(null);
|
|
171
|
+
const generationRef = react.useRef(0);
|
|
113
172
|
const configRef = react.useRef(config);
|
|
114
173
|
configRef.current = config;
|
|
115
|
-
const
|
|
174
|
+
const flushAssistantMessage = react.useCallback(() => {
|
|
175
|
+
if (streamTimerRef.current) {
|
|
176
|
+
clearTimeout(streamTimerRef.current);
|
|
177
|
+
streamTimerRef.current = null;
|
|
178
|
+
}
|
|
179
|
+
const text = pendingTextRef.current;
|
|
180
|
+
pendingTextRef.current = null;
|
|
181
|
+
if (text === null) return;
|
|
116
182
|
const cleaned = stripThinkTokens(text);
|
|
117
183
|
if (!cleaned) {
|
|
118
184
|
setIsThinking(true);
|
|
@@ -120,93 +186,118 @@ function useKanhaChat(config) {
|
|
|
120
186
|
}
|
|
121
187
|
setIsThinking(false);
|
|
122
188
|
setMessages((prev) => {
|
|
123
|
-
const
|
|
124
|
-
if (
|
|
125
|
-
|
|
189
|
+
const next = [...prev];
|
|
190
|
+
if (next[next.length - 1]?.role === "assistant") {
|
|
191
|
+
next[next.length - 1] = { role: "assistant", content: cleaned };
|
|
126
192
|
} else {
|
|
127
|
-
|
|
193
|
+
next.push({ role: "assistant", content: cleaned });
|
|
128
194
|
}
|
|
129
|
-
return
|
|
195
|
+
return next;
|
|
130
196
|
});
|
|
131
197
|
}, []);
|
|
198
|
+
const scheduleAssistantMessage = react.useCallback((text) => {
|
|
199
|
+
pendingTextRef.current = text;
|
|
200
|
+
if (streamTimerRef.current) return;
|
|
201
|
+
streamTimerRef.current = setTimeout(
|
|
202
|
+
flushAssistantMessage,
|
|
203
|
+
configRef.current.streamUpdateIntervalMs ?? 50
|
|
204
|
+
);
|
|
205
|
+
}, [flushAssistantMessage]);
|
|
206
|
+
const initializationKey = JSON.stringify([
|
|
207
|
+
config.modelUrl,
|
|
208
|
+
config.modelLib,
|
|
209
|
+
config.modelSize,
|
|
210
|
+
config.contextWindowSize,
|
|
211
|
+
config.minRamGb,
|
|
212
|
+
config.cacheBackend,
|
|
213
|
+
config.workerUrl?.toString()
|
|
214
|
+
]);
|
|
132
215
|
react.useEffect(() => {
|
|
133
216
|
if (typeof window === "undefined") return;
|
|
134
217
|
let cancelled = false;
|
|
218
|
+
const initConfig = configRef.current;
|
|
135
219
|
const init = async () => {
|
|
136
220
|
setMode("detecting");
|
|
221
|
+
setError(null);
|
|
137
222
|
const hasWebGPU = await checkWebGPU();
|
|
223
|
+
if (cancelled) return;
|
|
138
224
|
if (!hasWebGPU) {
|
|
139
225
|
setMode("error");
|
|
140
226
|
setError("WebGPU is not supported in this browser. Try Chrome 113+ or Edge 113+.");
|
|
141
227
|
return;
|
|
142
228
|
}
|
|
143
|
-
const
|
|
144
|
-
const memWarning = await checkAvailableMemory(modelSize);
|
|
229
|
+
const memWarning = checkAvailableMemory(initConfig);
|
|
145
230
|
if (memWarning) {
|
|
146
231
|
setMode("error");
|
|
147
232
|
setError(memWarning);
|
|
148
233
|
return;
|
|
149
234
|
}
|
|
150
|
-
|
|
235
|
+
setMode("loading");
|
|
151
236
|
setMode("loading");
|
|
152
237
|
try {
|
|
153
238
|
const webllm = await import('@mlc-ai/web-llm');
|
|
154
|
-
const
|
|
155
|
-
|
|
156
|
-
const modelUrl = toWebLLMModelUrl(configRef.current.modelUrl);
|
|
157
|
-
const engine = await webllm.CreateMLCEngine(modelId, {
|
|
158
|
-
appConfig: {
|
|
159
|
-
model_list: [
|
|
160
|
-
{
|
|
161
|
-
model: modelUrl,
|
|
162
|
-
model_id: modelId,
|
|
163
|
-
model_lib: modelLib,
|
|
164
|
-
overrides: {
|
|
165
|
-
context_window_size: configRef.current.contextWindowSize ?? 4096
|
|
166
|
-
}
|
|
167
|
-
}
|
|
168
|
-
],
|
|
169
|
-
useIndexedDBCache: true
|
|
170
|
-
},
|
|
171
|
-
initProgressCallback: (report) => {
|
|
172
|
-
if (!cancelled) setLoadProgress(Math.round(report.progress * 100));
|
|
173
|
-
},
|
|
174
|
-
logLevel: "SILENT"
|
|
239
|
+
const result = await createEngine(webllm, initConfig, (report) => {
|
|
240
|
+
if (!cancelled) setLoadProgress(Math.round(report.progress * 100));
|
|
175
241
|
});
|
|
176
|
-
if (cancelled)
|
|
177
|
-
|
|
242
|
+
if (cancelled) {
|
|
243
|
+
await disposeEngine(result.engine, result.worker);
|
|
244
|
+
return;
|
|
245
|
+
}
|
|
246
|
+
engineRef.current = result.engine;
|
|
247
|
+
workerRef.current = result.worker;
|
|
178
248
|
setMode("ready");
|
|
179
249
|
} catch (err) {
|
|
180
250
|
if (cancelled) return;
|
|
181
251
|
setMode("error");
|
|
182
252
|
if (isMemoryError(err)) {
|
|
183
|
-
const size =
|
|
253
|
+
const size = initConfig.modelSize ?? "small";
|
|
184
254
|
const ram = navigator.deviceMemory;
|
|
185
255
|
setError(
|
|
186
256
|
`Not enough memory to load the ${size} model${ram ? ` (device has ~${ram}GB RAM)` : ""}. Close other tabs or try a smaller model.`
|
|
187
257
|
);
|
|
188
258
|
} else {
|
|
189
|
-
setError(
|
|
190
|
-
err instanceof Error ? err.message : "Failed to load AI model."
|
|
191
|
-
);
|
|
259
|
+
setError(err instanceof Error ? err.message : "Failed to load AI model.");
|
|
192
260
|
}
|
|
193
261
|
}
|
|
194
262
|
};
|
|
195
263
|
init();
|
|
196
264
|
return () => {
|
|
197
265
|
cancelled = true;
|
|
198
|
-
|
|
266
|
+
generationRef.current += 1;
|
|
267
|
+
if (streamTimerRef.current) clearTimeout(streamTimerRef.current);
|
|
268
|
+
streamTimerRef.current = null;
|
|
269
|
+
pendingTextRef.current = null;
|
|
270
|
+
const engine = engineRef.current;
|
|
271
|
+
const worker = workerRef.current;
|
|
272
|
+
engineRef.current = null;
|
|
273
|
+
workerRef.current = null;
|
|
274
|
+
if (engine) {
|
|
199
275
|
try {
|
|
200
|
-
|
|
276
|
+
engine.interruptGenerate();
|
|
201
277
|
} catch {
|
|
202
278
|
}
|
|
203
|
-
|
|
279
|
+
void disposeEngine(engine, worker);
|
|
280
|
+
} else {
|
|
281
|
+
worker?.terminate();
|
|
204
282
|
}
|
|
205
283
|
};
|
|
206
|
-
}, [
|
|
284
|
+
}, [initializationKey]);
|
|
285
|
+
const stop = react.useCallback(() => {
|
|
286
|
+
generationRef.current += 1;
|
|
287
|
+
try {
|
|
288
|
+
engineRef.current?.interruptGenerate();
|
|
289
|
+
} catch {
|
|
290
|
+
}
|
|
291
|
+
flushAssistantMessage();
|
|
292
|
+
setIsLoading(false);
|
|
293
|
+
setIsThinking(false);
|
|
294
|
+
}, [flushAssistantMessage]);
|
|
207
295
|
const send = react.useCallback(async () => {
|
|
208
296
|
if (!input.trim() || mode !== "ready" || isLoading || !engineRef.current) return;
|
|
209
|
-
const
|
|
297
|
+
const userText = input.trim();
|
|
298
|
+
const generation = generationRef.current + 1;
|
|
299
|
+
generationRef.current = generation;
|
|
300
|
+
const userMsg = { role: "user", content: userText };
|
|
210
301
|
const allMessages = [...messages, userMsg];
|
|
211
302
|
setMessages(allMessages);
|
|
212
303
|
setInput("");
|
|
@@ -214,34 +305,109 @@ function useKanhaChat(config) {
|
|
|
214
305
|
setIsThinking(true);
|
|
215
306
|
setError(null);
|
|
216
307
|
try {
|
|
217
|
-
const
|
|
308
|
+
const currentConfig = configRef.current;
|
|
309
|
+
const systemMessages = currentConfig.systemPrompt ? [{ role: "system", content: currentConfig.systemPrompt }] : [];
|
|
310
|
+
let engineUserContent = userText;
|
|
311
|
+
let ragTime = 0;
|
|
312
|
+
let ragContextLength = 0;
|
|
313
|
+
if (currentConfig.onRetrieveContext) {
|
|
314
|
+
try {
|
|
315
|
+
const ragStartTime = performance.now();
|
|
316
|
+
const context = await currentConfig.onRetrieveContext(userText);
|
|
317
|
+
ragTime = (performance.now() - ragStartTime) / 1e3;
|
|
318
|
+
if (context) {
|
|
319
|
+
ragContextLength = context.length;
|
|
320
|
+
engineUserContent = formatRagPrompt(context, userText, currentConfig.ragPromptTemplate);
|
|
321
|
+
}
|
|
322
|
+
} catch (err) {
|
|
323
|
+
console.error("[KanhaBot] RAG Retrieval Failed:", err);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
if (generationRef.current !== generation) return;
|
|
327
|
+
const engineHistory = buildEngineHistory(
|
|
328
|
+
allMessages,
|
|
329
|
+
currentConfig.maxHistoryMessages,
|
|
330
|
+
engineUserContent
|
|
331
|
+
);
|
|
332
|
+
const startTime = performance.now();
|
|
333
|
+
let firstTokenTime = null;
|
|
334
|
+
let contentChunkCount = 0;
|
|
335
|
+
let completionTokens = null;
|
|
218
336
|
const completion = await engineRef.current.chat.completions.create({
|
|
219
|
-
messages: [...systemMessages, ...
|
|
220
|
-
temperature:
|
|
221
|
-
|
|
222
|
-
|
|
337
|
+
messages: [...systemMessages, ...engineHistory],
|
|
338
|
+
temperature: currentConfig.temperature ?? 0.7,
|
|
339
|
+
top_p: currentConfig.topP ?? 0.8,
|
|
340
|
+
max_tokens: currentConfig.maxTokens ?? 1024,
|
|
341
|
+
stream: true,
|
|
342
|
+
stream_options: { include_usage: true },
|
|
343
|
+
extra_body: {
|
|
344
|
+
enable_thinking: currentConfig.enableThinking ?? false,
|
|
345
|
+
enable_latency_breakdown: true
|
|
346
|
+
}
|
|
223
347
|
});
|
|
224
348
|
let text = "";
|
|
225
349
|
for await (const chunk of completion) {
|
|
350
|
+
if (generationRef.current !== generation) break;
|
|
226
351
|
const delta = chunk.choices[0]?.delta?.content || "";
|
|
227
|
-
|
|
228
|
-
|
|
352
|
+
if (delta) {
|
|
353
|
+
if (firstTokenTime === null) firstTokenTime = performance.now();
|
|
354
|
+
contentChunkCount += 1;
|
|
355
|
+
text += delta;
|
|
356
|
+
scheduleAssistantMessage(text);
|
|
357
|
+
}
|
|
358
|
+
if (chunk.usage) {
|
|
359
|
+
completionTokens = chunk.usage.completion_tokens;
|
|
360
|
+
emitMetrics(chunk.usage, currentConfig);
|
|
361
|
+
}
|
|
362
|
+
}
|
|
363
|
+
if (generationRef.current === generation) {
|
|
364
|
+
flushAssistantMessage();
|
|
365
|
+
const endTime = performance.now();
|
|
366
|
+
try {
|
|
367
|
+
if (engineRef.current.runtimeStatsText) {
|
|
368
|
+
const stats = await engineRef.current.runtimeStatsText();
|
|
369
|
+
const outputTokens = completionTokens ?? contentChunkCount;
|
|
370
|
+
const ttft = firstTokenTime ? (firstTokenTime - startTime) / 1e3 : 0;
|
|
371
|
+
const decodeTime = firstTokenTime ? (endTime - firstTokenTime) / 1e3 : 0;
|
|
372
|
+
const tps = decodeTime > 0 ? Math.max(0, outputTokens - 1) / decodeTime : 0;
|
|
373
|
+
const totalTime = (endTime - startTime) / 1e3;
|
|
374
|
+
const ragStatsLog = currentConfig.onRetrieveContext ? `RAG Retrieval Time: ${ragTime.toFixed(3)}s
|
|
375
|
+
RAG Context Inserted: ${ragContextLength} characters
|
|
376
|
+
` : "";
|
|
377
|
+
console.info(
|
|
378
|
+
`[WebLLM Metrics]
|
|
379
|
+
${stats}
|
|
380
|
+
` + ragStatsLog + `TTFT (Time To First Token): ${ttft.toFixed(3)}s
|
|
381
|
+
App-Level Decode Speed: ${tps.toFixed(2)} tok/s
|
|
382
|
+
Total Request Time: ${totalTime.toFixed(3)}s
|
|
383
|
+
Output Tokens: ${outputTokens}`
|
|
384
|
+
);
|
|
385
|
+
}
|
|
386
|
+
} catch {
|
|
387
|
+
}
|
|
229
388
|
}
|
|
230
389
|
} catch {
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
390
|
+
if (generationRef.current === generation) {
|
|
391
|
+
setError("Failed to generate response. Please try again.");
|
|
392
|
+
setMessages((prev) => [
|
|
393
|
+
...prev,
|
|
394
|
+
{ role: "assistant", content: "Sorry, I encountered an error. Please try again." }
|
|
395
|
+
]);
|
|
396
|
+
}
|
|
236
397
|
} finally {
|
|
237
|
-
|
|
238
|
-
|
|
398
|
+
if (generationRef.current === generation) {
|
|
399
|
+
flushAssistantMessage();
|
|
400
|
+
setIsLoading(false);
|
|
401
|
+
setIsThinking(false);
|
|
402
|
+
}
|
|
239
403
|
}
|
|
240
|
-
}, [input, mode, isLoading, messages,
|
|
404
|
+
}, [input, mode, isLoading, messages, scheduleAssistantMessage, flushAssistantMessage]);
|
|
241
405
|
const clear = react.useCallback(() => {
|
|
406
|
+
stop();
|
|
407
|
+
if (engineRef.current) resetEngine(engineRef.current);
|
|
242
408
|
setMessages([]);
|
|
243
409
|
setError(null);
|
|
244
|
-
}, []);
|
|
410
|
+
}, [stop]);
|
|
245
411
|
return {
|
|
246
412
|
messages,
|
|
247
413
|
input,
|
|
@@ -252,6 +418,7 @@ function useKanhaChat(config) {
|
|
|
252
418
|
loadProgress,
|
|
253
419
|
error,
|
|
254
420
|
send,
|
|
421
|
+
stop,
|
|
255
422
|
clear
|
|
256
423
|
};
|
|
257
424
|
}
|
|
@@ -421,7 +588,6 @@ function suggestionButtonStyle(primaryColor) {
|
|
|
421
588
|
textAlign: "left",
|
|
422
589
|
width: "100%",
|
|
423
590
|
transition: "border-color 0.15s ease"
|
|
424
|
-
// hover handled inline
|
|
425
591
|
};
|
|
426
592
|
}
|
|
427
593
|
var progressBarContainer = {
|
|
@@ -456,11 +622,7 @@ var TrashIcon = () => /* @__PURE__ */ jsxRuntime.jsxs("svg", { width: "16", heig
|
|
|
456
622
|
] });
|
|
457
623
|
var Spinner = ({ size = 18 }) => /* @__PURE__ */ jsxRuntime.jsx("svg", { width: size, height: size, viewBox: "0 0 24 24", fill: "none", stroke: "currentColor", strokeWidth: "2", style: { animation: "kanha-spin 1s linear infinite" }, children: /* @__PURE__ */ jsxRuntime.jsx("path", { d: "M21 12a9 9 0 1 1-6.219-8.56" }) });
|
|
458
624
|
var spinKeyframes = `@keyframes kanha-spin { to { transform: rotate(360deg); } }`;
|
|
459
|
-
var bounceKeyframes =
|
|
460
|
-
@keyframes kanha-bounce {
|
|
461
|
-
0%, 80%, 100% { transform: translateY(0); }
|
|
462
|
-
40% { transform: translateY(-4px); }
|
|
463
|
-
}`;
|
|
625
|
+
var bounceKeyframes = `@keyframes kanha-bounce { 0%, 80%, 100% { transform: translateY(0); } 40% { transform: translateY(-4px); } }`;
|
|
464
626
|
function ThinkingDots() {
|
|
465
627
|
const dotStyle = (delay) => ({
|
|
466
628
|
width: 6,
|
|
@@ -482,12 +644,22 @@ function KanhaBot({
|
|
|
482
644
|
modelSize,
|
|
483
645
|
systemPrompt,
|
|
484
646
|
temperature,
|
|
647
|
+
topP,
|
|
485
648
|
maxTokens,
|
|
649
|
+
enableThinking,
|
|
650
|
+
maxHistoryMessages,
|
|
651
|
+
streamUpdateIntervalMs,
|
|
652
|
+
cacheBackend,
|
|
653
|
+
workerUrl,
|
|
654
|
+
onMetrics,
|
|
486
655
|
contextWindowSize,
|
|
656
|
+
minRamGb,
|
|
487
657
|
botName = "AI Assistant",
|
|
488
658
|
welcomeMessage = "Ask me anything!",
|
|
489
659
|
suggestions = [],
|
|
490
|
-
theme = {}
|
|
660
|
+
theme = {},
|
|
661
|
+
onRetrieveContext,
|
|
662
|
+
ragPromptTemplate
|
|
491
663
|
}) {
|
|
492
664
|
const primaryColor = theme.primaryColor ?? "#0d9488";
|
|
493
665
|
const position = theme.position ?? "bottom-right";
|
|
@@ -508,8 +680,18 @@ function KanhaBot({
|
|
|
508
680
|
modelSize,
|
|
509
681
|
systemPrompt,
|
|
510
682
|
temperature,
|
|
683
|
+
topP,
|
|
511
684
|
maxTokens,
|
|
512
|
-
|
|
685
|
+
enableThinking,
|
|
686
|
+
maxHistoryMessages,
|
|
687
|
+
streamUpdateIntervalMs,
|
|
688
|
+
cacheBackend,
|
|
689
|
+
workerUrl,
|
|
690
|
+
onMetrics,
|
|
691
|
+
contextWindowSize,
|
|
692
|
+
minRamGb,
|
|
693
|
+
onRetrieveContext,
|
|
694
|
+
ragPromptTemplate
|
|
513
695
|
});
|
|
514
696
|
const [isOpen, setIsOpen] = react.useState(false);
|
|
515
697
|
const messagesEndRef = react.useRef(null);
|
|
@@ -547,7 +729,7 @@ function KanhaBot({
|
|
|
547
729
|
/* @__PURE__ */ jsxRuntime.jsx("button", { onClick: () => setIsOpen(false), style: iconButtonStyle, "aria-label": "Close", children: /* @__PURE__ */ jsxRuntime.jsx(CloseIcon, {}) })
|
|
548
730
|
] })
|
|
549
731
|
] }),
|
|
550
|
-
error && /* @__PURE__ */ jsxRuntime.jsx("div", { style: { padding: "10px 16px", background: "#fef2f2", borderBottom: "1px solid #fecaca", fontSize: 13, color: "#b91c1c" }, children: error }),
|
|
732
|
+
error && mode !== "error" && /* @__PURE__ */ jsxRuntime.jsx("div", { style: { padding: "10px 16px", background: "#fef2f2", borderBottom: "1px solid #fecaca", fontSize: 13, color: "#b91c1c" }, children: error }),
|
|
551
733
|
/* @__PURE__ */ jsxRuntime.jsx("div", { style: messagesContainerStyle, children: messages.length === 0 ? /* @__PURE__ */ jsxRuntime.jsx("div", { style: emptyStateStyle, children: isReady ? /* @__PURE__ */ jsxRuntime.jsxs(jsxRuntime.Fragment, { children: [
|
|
552
734
|
/* @__PURE__ */ jsxRuntime.jsx(ChatIcon, {}),
|
|
553
735
|
/* @__PURE__ */ jsxRuntime.jsx("p", { style: { marginTop: 12, fontWeight: 500, color: "#374151" }, children: welcomeMessage }),
|