kiku-agent 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/package.json +4 -12
- package/server/agent/eval/AGENTS.md +228 -0
- package/server/agent/eval/analyze.py +684 -0
- package/server/agent/eval/gates.json +15 -0
- package/server/agent/eval/modules/analysis/compare.js +198 -0
- package/server/agent/eval/modules/analysis/gates.js +72 -0
- package/server/agent/eval/modules/analysis/judge.js +70 -0
- package/server/agent/eval/modules/core/cli.js +453 -0
- package/server/agent/eval/modules/core/config.js +60 -0
- package/server/agent/eval/modules/core/store.js +236 -0
- package/server/agent/eval/modules/evaluation/cases.js +689 -0
- package/server/agent/eval/modules/evaluation/runner.js +298 -0
- package/server/agent/eval/modules/evaluation/scoring.js +351 -0
- package/server/agent/eval/modules/evaluation/worker.js +122 -0
- package/server/agent/eval/modules/reporting/benchmark.js +265 -0
- package/server/agent/eval/modules/reporting/benchmarkPng.js +487 -0
- package/server/agent/eval/modules/reporting/dashboard.js +1406 -0
- package/server/agent/eval/modules/server/evalApi.js +1099 -0
- package/server/agent/eval/modules/ui/evalUi.js +4796 -0
- package/server/agent/eval/modules/utils/smokes.js +119 -0
- package/server/agent/tests/agentBubbleSystem.test.js +65 -0
- package/server/agent/tests/agentFollowUps.test.js +127 -0
- package/server/agent/tests/apiRouter.test.js +108 -0
- package/server/agent/tests/architectureFixes.test.js +81 -0
- package/server/agent/tests/architectureRefactoring.test.js +194 -0
- package/server/agent/tests/autoResearch.test.js +64 -0
- package/server/agent/tests/bilibiliEmbed.test.js +251 -0
- package/server/agent/tests/catalogSync.test.js +130 -0
- package/server/agent/tests/citations.test.js +180 -0
- package/server/agent/tests/clarificationAnalytics.test.js +219 -0
- package/server/agent/tests/clarifyingQuestions.test.js +594 -0
- package/server/agent/tests/cliAutomation.test.js +121 -0
- package/server/agent/tests/cloudProviders.test.js +234 -0
- package/server/agent/tests/commandInput.test.js +24 -0
- package/server/agent/tests/commands.test.js +417 -0
- package/server/agent/tests/contextIntelligence.test.js +162 -0
- package/server/agent/tests/conversationSafety.test.js +53 -0
- package/server/agent/tests/decisionAndSearchEnhancements.test.js +135 -0
- package/server/agent/tests/dramaAnswerQuality.test.js +73 -0
- package/server/agent/tests/dynamicCatalogIngestion.test.js +285 -0
- package/server/agent/tests/dynamicRoutingAndSearch.test.js +92 -0
- package/server/agent/tests/edgeCasesScenarios.test.js +49 -0
- package/server/agent/tests/entertainmentMetrics.test.js +126 -0
- package/server/agent/tests/envBootstrap.test.js +87 -0
- package/server/agent/tests/envCommand.test.js +215 -0
- package/server/agent/tests/errorHandling.test.js +80 -0
- package/server/agent/tests/errors.test.js +13 -0
- package/server/agent/tests/evalAnalyze.test.js +54 -0
- package/server/agent/tests/evalCommand.test.js +54 -0
- package/server/agent/tests/evalCorpus.test.js +126 -0
- package/server/agent/tests/evalHarness.test.js +62 -0
- package/server/agent/tests/evalJudgeContext.test.js +38 -0
- package/server/agent/tests/evalMetrics.test.js +312 -0
- package/server/agent/tests/evalStore.test.js +190 -0
- package/server/agent/tests/evalStudio.test.js +179 -0
- package/server/agent/tests/feedbackLearning.test.js +186 -0
- package/server/agent/tests/firstRun.test.js +44 -0
- package/server/agent/tests/gemma4eOptimization.test.js +95 -0
- package/server/agent/tests/hardwareTiers.test.js +206 -0
- package/server/agent/tests/hybridWebSearch.test.js +192 -0
- package/server/agent/tests/judgeService.test.js +251 -0
- package/server/agent/tests/lanDiscovery.test.js +319 -0
- package/server/agent/tests/mcpClient.test.js +201 -0
- package/server/agent/tests/modelManager.test.js +109 -0
- package/server/agent/tests/multilingualScenarios.test.js +69 -0
- package/server/agent/tests/newCommands.test.js +347 -0
- package/server/agent/tests/newCommandsTui.test.js +141 -0
- package/server/agent/tests/newTools.test.js +113 -0
- package/server/agent/tests/newsIntelligence.test.js +178 -0
- package/server/agent/tests/nextGenArchitecture.test.js +99 -0
- package/server/agent/tests/offlineInput.test.js +217 -0
- package/server/agent/tests/packagingGuard.test.js +57 -0
- package/server/agent/tests/presentation.test.js +61 -0
- package/server/agent/tests/providerPersistence.test.js +39 -0
- package/server/agent/tests/queryBenchmarks.test.js +83 -0
- package/server/agent/tests/queryDetails.test.js +126 -0
- package/server/agent/tests/realWorldValidation.test.js +391 -0
- package/server/agent/tests/resilienceAndConcurrency.test.js +117 -0
- package/server/agent/tests/runtimeWatchdog.test.js +32 -0
- package/server/agent/tests/sanitizer.test.js +37 -0
- package/server/agent/tests/scenariosData.js +378 -0
- package/server/agent/tests/schemaRouting.test.js +54 -0
- package/server/agent/tests/securityAuditRemediation.test.js +214 -0
- package/server/agent/tests/securityHardening.test.js +146 -0
- package/server/agent/tests/securitySanitizer.test.js +38 -0
- package/server/agent/tests/sessionAndCopy.test.js +232 -0
- package/server/agent/tests/sidebarControls.test.js +101 -0
- package/server/agent/tests/statefulMemory.test.js +270 -0
- package/server/agent/tests/subagent.test.js +216 -0
- package/server/agent/tests/subagentProgress.test.js +109 -0
- package/server/agent/tests/subagentTextFallback.test.js +115 -0
- package/server/agent/tests/suggestions.test.js +199 -0
- package/server/agent/tests/suggestionsTui.test.js +251 -0
- package/server/agent/tests/telemetryAndProfiles.test.js +179 -0
- package/server/agent/tests/terminalImage.test.js +80 -0
- package/server/agent/tests/terminalMarkdown.test.js +166 -0
- package/server/agent/tests/terminalText.test.js +133 -0
- package/server/agent/tests/textProtocolLoop.test.js +126 -0
- package/server/agent/tests/textToolParsing.test.js +139 -0
- package/server/agent/tests/theme.test.js +26 -0
- package/server/agent/tests/toolRegistryParity.test.js +53 -0
- package/server/agent/tests/toolRoutingRegressions.test.js +119 -0
- package/server/agent/tests/tools.test.js +259 -0
- package/server/agent/tests/unifiedEndpoints.test.js +152 -0
- package/server/agent/tests/v110Features.test.js +262 -0
- package/server/agent/tests/versionBump.test.js +69 -0
- package/server/agent/tests/vramEstimator.test.js +520 -0
- package/server/agent/tests/webChatInspection.test.js +300 -0
- package/server/agent/tests/webChatMemory.test.js +112 -0
- package/server/agent/tests/webChatSecurity.test.js +169 -0
- package/server/agent/tests/webExtract.test.js +261 -0
- package/server/agent/tests/webSources.test.js +247 -0
- package/server/eval/judgeRubrics.js +110 -0
- package/server/eval/judgeService.js +290 -0
- package/server/eval/runBenchmark.js +182 -0
- package/server/mcp/examples/librechat.example.yaml +16 -0
- package/server/mcp/tests/connection.test.js +156 -0
- package/server/supabase_schema.sql +63 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
import { describe, it } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import {
|
|
4
|
+
detectAmbiguities,
|
|
5
|
+
isQueryTooVague,
|
|
6
|
+
formatQuestionsForCLI,
|
|
7
|
+
formatQuestionsForSSE,
|
|
8
|
+
parseClarifyingAnswers,
|
|
9
|
+
emitClarifyingQuestions,
|
|
10
|
+
calculateConfidence,
|
|
11
|
+
buildClarificationContext,
|
|
12
|
+
getAmbiguityPatterns,
|
|
13
|
+
generateClarifyingQuestions,
|
|
14
|
+
stripSystemDirectives,
|
|
15
|
+
cleanThinkingBlocks,
|
|
16
|
+
} from '../clarifyingQuestions.js';
|
|
17
|
+
|
|
18
|
+
describe('detectAmbiguities', () => {
|
|
19
|
+
it('returns empty array for null/undefined input', () => {
|
|
20
|
+
assert.deepEqual(detectAmbiguities(null), []);
|
|
21
|
+
assert.deepEqual(detectAmbiguities(undefined), []);
|
|
22
|
+
assert.deepEqual(detectAmbiguities(''), []);
|
|
23
|
+
});
|
|
24
|
+
|
|
25
|
+
it('detects timeframe ambiguity', () => {
|
|
26
|
+
const result = detectAmbiguities('What are her recent works?', 'en');
|
|
27
|
+
assert.ok(result.length > 0);
|
|
28
|
+
assert.equal(result[0].category, 'timeframe');
|
|
29
|
+
assert.ok(result[0].question.includes('timeframe'));
|
|
30
|
+
assert.ok(result[0].options.length > 0);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('detects scope ambiguity', () => {
|
|
34
|
+
const result = detectAmbiguities('Tell me about all her dramas', 'en');
|
|
35
|
+
assert.ok(result.length > 0);
|
|
36
|
+
assert.equal(result[0].category, 'scope');
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('detects depth ambiguity', () => {
|
|
40
|
+
const result = detectAmbiguities('Give me a detailed overview of her career', 'en');
|
|
41
|
+
assert.ok(result.length > 0);
|
|
42
|
+
const categories = result.map(r => r.category);
|
|
43
|
+
assert.ok(categories.includes('depth'));
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('detects comparison ambiguity', () => {
|
|
47
|
+
const result = detectAmbiguities('Compare her dramas vs her music', 'en');
|
|
48
|
+
assert.ok(result.length > 0);
|
|
49
|
+
assert.equal(result[0].category, 'comparison');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it('detects media_type ambiguity', () => {
|
|
53
|
+
const result = detectAmbiguities('Show me her songs', 'en');
|
|
54
|
+
assert.ok(result.length > 0);
|
|
55
|
+
const categories = result.map(r => r.category);
|
|
56
|
+
assert.ok(categories.includes('media_type'));
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('detects language ambiguity', () => {
|
|
60
|
+
const result = detectAmbiguities('Tell me about her Chinese dramas', 'en');
|
|
61
|
+
assert.ok(result.length > 0);
|
|
62
|
+
const categories = result.map(r => r.category);
|
|
63
|
+
assert.ok(categories.includes('language'));
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
it('detects multiple ambiguities', () => {
|
|
67
|
+
const result = detectAmbiguities('Show me all her recent songs in Chinese', 'en');
|
|
68
|
+
assert.ok(result.length >= 2);
|
|
69
|
+
const categories = result.map(r => r.category);
|
|
70
|
+
assert.ok(categories.includes('timeframe'));
|
|
71
|
+
assert.ok(categories.includes('media_type'));
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
it('returns Thai questions when detectedLang is th', () => {
|
|
75
|
+
const result = detectAmbiguities('recent works', 'th');
|
|
76
|
+
assert.ok(result.length > 0);
|
|
77
|
+
const timeframeQ = result.find(r => r.category === 'timeframe');
|
|
78
|
+
assert.ok(timeframeQ);
|
|
79
|
+
assert.ok(timeframeQ.question.includes('กรุณา') || timeframeQ.question.includes('ช่วงเวลา'));
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
it('returns Chinese questions when detectedLang is zh', () => {
|
|
83
|
+
const result = detectAmbiguities('recent works', 'zh');
|
|
84
|
+
assert.ok(result.length > 0);
|
|
85
|
+
const timeframeQ = result.find(r => r.category === 'timeframe');
|
|
86
|
+
assert.ok(timeframeQ);
|
|
87
|
+
assert.ok(timeframeQ.question.includes('您') || timeframeQ.question.includes('吗'));
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('returns English questions as fallback', () => {
|
|
91
|
+
const result = detectAmbiguities('recent works', 'unknown');
|
|
92
|
+
assert.ok(result.length > 0);
|
|
93
|
+
assert.ok(result[0].question.includes('timeframe') || result[0].question.includes('Could you'));
|
|
94
|
+
});
|
|
95
|
+
|
|
96
|
+
it('does not duplicate categories', () => {
|
|
97
|
+
const result = detectAmbiguities('Tell me about all her recent detailed songs in Chinese', 'en');
|
|
98
|
+
const categories = result.map(r => r.category);
|
|
99
|
+
const uniqueCategories = [...new Set(categories)];
|
|
100
|
+
assert.equal(categories.length, uniqueCategories.length);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
describe('isQueryTooVague', () => {
|
|
105
|
+
it('returns true for null/undefined', () => {
|
|
106
|
+
assert.equal(isQueryTooVague(null), true);
|
|
107
|
+
assert.equal(isQueryTooVague(undefined), true);
|
|
108
|
+
assert.equal(isQueryTooVague(''), true);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('returns true for very short queries', () => {
|
|
112
|
+
assert.equal(isQueryTooVague('a'), true);
|
|
113
|
+
assert.equal(isQueryTooVague('hi'), true);
|
|
114
|
+
assert.equal(isQueryTooVague('??'), true);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
it('returns true for punctuation-only queries', () => {
|
|
118
|
+
assert.equal(isQueryTooVague('???'), true);
|
|
119
|
+
assert.equal(isQueryTooVague('...'), true);
|
|
120
|
+
assert.equal(isQueryTooVague('!!!'), true);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('returns true for single short word', () => {
|
|
124
|
+
assert.equal(isQueryTooVague('hey'), true);
|
|
125
|
+
assert.equal(isQueryTooVague('hi'), true);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('returns false for meaningful queries', () => {
|
|
129
|
+
assert.equal(isQueryTooVague('What are her latest dramas?'), false);
|
|
130
|
+
assert.equal(isQueryTooVague('Tell me about Ju Jingyi'), false);
|
|
131
|
+
assert.equal(isQueryTooVague('ผลงานล่าสุดของเธอ'), false);
|
|
132
|
+
assert.equal(isQueryTooVague('她的最新作品'), false);
|
|
133
|
+
});
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
describe('formatQuestionsForCLI', () => {
|
|
137
|
+
it('returns empty string for empty input', () => {
|
|
138
|
+
assert.equal(formatQuestionsForCLI([]), '');
|
|
139
|
+
assert.equal(formatQuestionsForCLI(null), '');
|
|
140
|
+
assert.equal(formatQuestionsForCLI(undefined), '');
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
it('formats questions with options', () => {
|
|
144
|
+
const questions = [
|
|
145
|
+
{
|
|
146
|
+
category: 'timeframe',
|
|
147
|
+
question: 'What timeframe?',
|
|
148
|
+
options: ['2026', '2025', 'All time'],
|
|
149
|
+
},
|
|
150
|
+
];
|
|
151
|
+
const result = formatQuestionsForCLI(questions, { gold: '\x1b[33m', reset: '\x1b[0m', bold: '\x1b[1m', roseGlow: '\x1b[38;2;255;117;140m', dim: '\x1b[2m', slate: '\x1b[90m', white: '\x1b[37m' });
|
|
152
|
+
assert.ok(result.includes('TIMEFRAME'));
|
|
153
|
+
assert.ok(result.includes('What timeframe?'));
|
|
154
|
+
assert.ok(result.includes('a) 2026'));
|
|
155
|
+
assert.ok(result.includes('b) 2025'));
|
|
156
|
+
assert.ok(result.includes('c) All time'));
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('includes skip instruction', () => {
|
|
160
|
+
const questions = [{ category: 'scope', question: 'Scope?', options: ['A'] }];
|
|
161
|
+
const result = formatQuestionsForCLI(questions, {});
|
|
162
|
+
assert.ok(result.includes('skip'));
|
|
163
|
+
});
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
describe('formatQuestionsForSSE', () => {
|
|
167
|
+
it('returns null for empty input', () => {
|
|
168
|
+
assert.equal(formatQuestionsForSSE([]), null);
|
|
169
|
+
assert.equal(formatQuestionsForSSE(null), null);
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it('returns structured object', () => {
|
|
173
|
+
const questions = [
|
|
174
|
+
{ category: 'timeframe', question: 'When?', options: ['Now', 'Later'] },
|
|
175
|
+
];
|
|
176
|
+
const result = formatQuestionsForSSE(questions);
|
|
177
|
+
assert.equal(result.type, 'clarifying_questions');
|
|
178
|
+
assert.ok(result.questions.length === 1);
|
|
179
|
+
assert.equal(result.questions[0].category, 'timeframe');
|
|
180
|
+
assert.ok(result.instruction.includes('skip'));
|
|
181
|
+
});
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
describe('parseClarifyingAnswers', () => {
|
|
185
|
+
it('returns empty for skip', () => {
|
|
186
|
+
const result = parseClarifyingAnswers('skip', [
|
|
187
|
+
{ category: 'timeframe', question: 'When?', options: ['2026', '2025'] },
|
|
188
|
+
]);
|
|
189
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
190
|
+
assert.ok(result.summary.includes('skip'));
|
|
191
|
+
});
|
|
192
|
+
|
|
193
|
+
it('parses structured format "1a 2c"', () => {
|
|
194
|
+
const questions = [
|
|
195
|
+
{ category: 'timeframe', question: 'When?', options: ['2026', '2025', 'All'] },
|
|
196
|
+
{ category: 'scope', question: 'How much?', options: ['Full', 'Brief'] },
|
|
197
|
+
];
|
|
198
|
+
const result = parseClarifyingAnswers('1a 2b', questions);
|
|
199
|
+
assert.equal(result.answers.timeframe, '2026');
|
|
200
|
+
assert.equal(result.answers.scope, 'Brief');
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
it('parses single option format "a"', () => {
|
|
204
|
+
const questions = [
|
|
205
|
+
{ category: 'timeframe', question: 'When?', options: ['2026', '2025'] },
|
|
206
|
+
];
|
|
207
|
+
const result = parseClarifyingAnswers('a', questions);
|
|
208
|
+
assert.equal(result.answers.timeframe, '2026');
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
it('handles free text as first question answer', () => {
|
|
212
|
+
const questions = [
|
|
213
|
+
{ category: 'timeframe', question: 'When?', options: ['2026', '2025'] },
|
|
214
|
+
];
|
|
215
|
+
const result = parseClarifyingAnswers('the past few years', questions);
|
|
216
|
+
assert.equal(result.answers.timeframe, 'the past few years');
|
|
217
|
+
});
|
|
218
|
+
|
|
219
|
+
it('returns empty for null/undefined input', () => {
|
|
220
|
+
const result = parseClarifyingAnswers(null, []);
|
|
221
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
222
|
+
});
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
describe('emitClarifyingQuestions', () => {
|
|
226
|
+
it('returns default when no handler provided', async () => {
|
|
227
|
+
const result = await emitClarifyingQuestions(
|
|
228
|
+
[{ category: 'test', question: 'Test?', options: ['A'] }],
|
|
229
|
+
'q_test'
|
|
230
|
+
);
|
|
231
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
232
|
+
assert.ok(result.summary.includes('No response handler'));
|
|
233
|
+
});
|
|
234
|
+
|
|
235
|
+
it('returns default when questions are empty', async () => {
|
|
236
|
+
const result = await emitClarifyingQuestions([], 'q_test');
|
|
237
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
238
|
+
});
|
|
239
|
+
});
|
|
240
|
+
|
|
241
|
+
describe('calculateConfidence', () => {
|
|
242
|
+
it('returns very_low confidence for null/undefined input', () => {
|
|
243
|
+
const result = calculateConfidence(null);
|
|
244
|
+
assert.equal(result.level, 'very_low');
|
|
245
|
+
assert.equal(result.shouldAsk, true);
|
|
246
|
+
assert.ok(result.score < 0.5);
|
|
247
|
+
});
|
|
248
|
+
|
|
249
|
+
it('returns low confidence for short queries', () => {
|
|
250
|
+
const result = calculateConfidence('drama', []);
|
|
251
|
+
assert.ok(result.score < 0.8);
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
it('returns higher confidence for specific queries with year', () => {
|
|
255
|
+
const result = calculateConfidence('What dramas did Ju Jingyi star in 2024?', []);
|
|
256
|
+
assert.ok(result.score > 0.8);
|
|
257
|
+
assert.equal(result.level, 'high');
|
|
258
|
+
assert.equal(result.shouldAsk, false);
|
|
259
|
+
});
|
|
260
|
+
|
|
261
|
+
it('returns medium confidence for moderate queries', () => {
|
|
262
|
+
|
|
263
|
+
const ambiguities = [
|
|
264
|
+
{ category: 'scope', weight: 0.6 },
|
|
265
|
+
{ category: 'depth', weight: 0.5 },
|
|
266
|
+
];
|
|
267
|
+
const result = calculateConfidence('Show me something', ambiguities);
|
|
268
|
+
assert.ok(result.score >= 0.4 && result.score <= 0.9, `Expected medium confidence, got ${result.score}`);
|
|
269
|
+
});
|
|
270
|
+
|
|
271
|
+
it('deducts score for detected ambiguities', () => {
|
|
272
|
+
const ambiguities = [{ category: 'timeframe', weight: 0.7 }];
|
|
273
|
+
const withoutAmbiguity = calculateConfidence('Show me her songs', []);
|
|
274
|
+
const withAmbiguity = calculateConfidence('Show me her songs', ambiguities);
|
|
275
|
+
assert.ok(withAmbiguity.score < withoutAmbiguity.score);
|
|
276
|
+
});
|
|
277
|
+
|
|
278
|
+
it('boosts confidence for specific keywords', () => {
|
|
279
|
+
const result = calculateConfidence('Tell me specifically about In Blossom drama', []);
|
|
280
|
+
assert.ok(result.score > 0.8);
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
it('includes reasoning in result', () => {
|
|
284
|
+
const result = calculateConfidence('test query', []);
|
|
285
|
+
assert.ok(typeof result.reasoning === 'string');
|
|
286
|
+
assert.ok(result.reasoning.includes('Confidence'));
|
|
287
|
+
});
|
|
288
|
+
});
|
|
289
|
+
|
|
290
|
+
describe('buildClarificationContext', () => {
|
|
291
|
+
it('returns empty string for null result', () => {
|
|
292
|
+
assert.equal(buildClarificationContext(null, 'test'), '');
|
|
293
|
+
});
|
|
294
|
+
|
|
295
|
+
it('returns empty string for empty answers', () => {
|
|
296
|
+
assert.equal(buildClarificationContext({ answers: {}, summary: '' }, 'test'), '');
|
|
297
|
+
});
|
|
298
|
+
|
|
299
|
+
it('builds context with answers', () => {
|
|
300
|
+
const result = {
|
|
301
|
+
answers: { timeframe: '2026', scope: 'Comprehensive' },
|
|
302
|
+
summary: 'User clarified: timeframe: 2026'
|
|
303
|
+
};
|
|
304
|
+
const context = buildClarificationContext(result, 'Show me her works');
|
|
305
|
+
assert.ok(context.includes('[SYSTEM CLARIFICATION]'));
|
|
306
|
+
assert.ok(context.includes('Original query: Show me her works'));
|
|
307
|
+
assert.ok(context.includes('timeframe: 2026'));
|
|
308
|
+
assert.ok(context.includes('scope: Comprehensive'));
|
|
309
|
+
});
|
|
310
|
+
});
|
|
311
|
+
|
|
312
|
+
describe('getAmbiguityPatterns', () => {
|
|
313
|
+
it('returns array of patterns', () => {
|
|
314
|
+
const patterns = getAmbiguityPatterns();
|
|
315
|
+
assert.ok(Array.isArray(patterns));
|
|
316
|
+
assert.ok(patterns.length >= 6);
|
|
317
|
+
});
|
|
318
|
+
|
|
319
|
+
it('each pattern has required fields', () => {
|
|
320
|
+
const patterns = getAmbiguityPatterns();
|
|
321
|
+
for (const p of patterns) {
|
|
322
|
+
assert.ok(p.category, 'Pattern should have category');
|
|
323
|
+
assert.ok(p.weight !== undefined, 'Pattern should have weight');
|
|
324
|
+
assert.ok(p.detection, 'Pattern should have detection regex');
|
|
325
|
+
assert.ok(p.questions, 'Pattern should have questions');
|
|
326
|
+
assert.ok(p.questions.en, 'Pattern should have English questions');
|
|
327
|
+
assert.ok(p.options, 'Pattern should have options');
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
});
|
|
331
|
+
|
|
332
|
+
describe('parseClarifyingAnswers (enhanced)', () => {
|
|
333
|
+
it('interprets shorthand "detailed" for depth questions', () => {
|
|
334
|
+
const questions = [
|
|
335
|
+
{ category: 'depth', question: 'How detailed?', options: ['Very detailed (full analysis)', 'Moderate', 'Brief (quick summary)'] },
|
|
336
|
+
];
|
|
337
|
+
const result = parseClarifyingAnswers('detailed', questions);
|
|
338
|
+
assert.equal(result.answers.depth, 'Very detailed (full analysis)');
|
|
339
|
+
});
|
|
340
|
+
|
|
341
|
+
it('interprets shorthand "brief" for depth questions', () => {
|
|
342
|
+
const questions = [
|
|
343
|
+
{ category: 'depth', question: 'How detailed?', options: ['Very detailed (full analysis)', 'Moderate', 'Brief (quick summary)'] },
|
|
344
|
+
];
|
|
345
|
+
const result = parseClarifyingAnswers('brief', questions);
|
|
346
|
+
assert.equal(result.answers.depth, 'Brief (quick summary)');
|
|
347
|
+
});
|
|
348
|
+
|
|
349
|
+
it('interprets "2026" for timeframe questions', () => {
|
|
350
|
+
const questions = [
|
|
351
|
+
{ category: 'timeframe', question: 'When?', options: ['2026 (current year)', '2025', 'All time'] },
|
|
352
|
+
];
|
|
353
|
+
const result = parseClarifyingAnswers('2026', questions);
|
|
354
|
+
assert.equal(result.answers.timeframe, '2026 (current year)');
|
|
355
|
+
});
|
|
356
|
+
|
|
357
|
+
it('skips on "no" answer', () => {
|
|
358
|
+
const questions = [
|
|
359
|
+
{ category: 'timeframe', question: 'When?', options: ['2026', '2025'] },
|
|
360
|
+
];
|
|
361
|
+
const result = parseClarifyingAnswers('no', questions);
|
|
362
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
363
|
+
assert.ok(result.summary.includes('skip'));
|
|
364
|
+
});
|
|
365
|
+
});
|
|
366
|
+
|
|
367
|
+
describe('stripSystemDirectives', () => {
|
|
368
|
+
it('strips SYSTEM DIRECTIVE and returns clean user question', () => {
|
|
369
|
+
const raw = '我需要有關鞠靜怡的資訊。\n\n[SYSTEM DIRECTIVE (Fact Grounding & Music Playback):\n1. Do not hallucinate]';
|
|
370
|
+
const cleaned = stripSystemDirectives(raw);
|
|
371
|
+
assert.equal(cleaned, '我需要有關鞠靜怡的資訊。');
|
|
372
|
+
});
|
|
373
|
+
|
|
374
|
+
it('handles text without system directives', () => {
|
|
375
|
+
assert.equal(stripSystemDirectives('ขอเพลงหน่อย'), 'ขอเพลงหน่อย');
|
|
376
|
+
});
|
|
377
|
+
});
|
|
378
|
+
|
|
379
|
+
describe('generateClarifyingQuestions (model-driven)', () => {
|
|
380
|
+
it('uses model client to generate dynamic questions when available', async () => {
|
|
381
|
+
const mockClient = {
|
|
382
|
+
timeout: 5000,
|
|
383
|
+
async chat({ messages }) {
|
|
384
|
+
return {
|
|
385
|
+
content: JSON.stringify([
|
|
386
|
+
{
|
|
387
|
+
category: 'subject_focus',
|
|
388
|
+
question: 'สนใจเรื่องใดเกี่ยวกับจวีจิ้งอีเป็นพิเศษคะ?',
|
|
389
|
+
options: ['ผลงานซีรีส์', 'ผลงานเพลง', 'ประวัติส่วนตัว']
|
|
390
|
+
},
|
|
391
|
+
{
|
|
392
|
+
category: 'timeframe',
|
|
393
|
+
question: 'ต้องการดูข้อมูลช่วงปีใดคะ?',
|
|
394
|
+
options: ['2024-2026', '2020-2023', 'ทั้งหมด']
|
|
395
|
+
}
|
|
396
|
+
])
|
|
397
|
+
};
|
|
398
|
+
}
|
|
399
|
+
};
|
|
400
|
+
|
|
401
|
+
const questions = await generateClarifyingQuestions('ขอข้อมูลจวีจิ้งอี', 'th', { client: mockClient });
|
|
402
|
+
assert.ok(Array.isArray(questions));
|
|
403
|
+
assert.equal(questions.length, 2);
|
|
404
|
+
assert.equal(questions[0].category, 'subject_focus');
|
|
405
|
+
assert.equal(questions[0].question, 'สนใจเรื่องใดเกี่ยวกับจวีจิ้งอีเป็นพิเศษคะ?');
|
|
406
|
+
assert.deepEqual(questions[0].options, ['ผลงานซีรีส์', 'ผลงานเพลง', 'ประวัติส่วนตัว']);
|
|
407
|
+
});
|
|
408
|
+
|
|
409
|
+
it('parses markdown code block JSON from model correctly', async () => {
|
|
410
|
+
const mockClient = {
|
|
411
|
+
timeout: 5000,
|
|
412
|
+
async chat() {
|
|
413
|
+
return {
|
|
414
|
+
content: '```json\n[\n {\n "category": "career_phase",\n "question": "สนใจช่วงใดคะ?",\n "options": ["Solo", "SNH48"]\n }\n, {\n "category": "depth",\n "question": "ต้องการคำตอบละเอียดแค่ไหนคะ?",\n "options": ["สรุปกระชับ", "เจาะลึก"]\n }\n]\n```'
|
|
415
|
+
};
|
|
416
|
+
}
|
|
417
|
+
};
|
|
418
|
+
|
|
419
|
+
const questions = await generateClarifyingQuestions('เล่าเรื่องจวีจิ้งอี', 'th', { client: mockClient });
|
|
420
|
+
assert.equal(questions.length, 2);
|
|
421
|
+
assert.equal(questions[0].category, 'career_phase');
|
|
422
|
+
assert.equal(questions[1].category, 'depth');
|
|
423
|
+
});
|
|
424
|
+
|
|
425
|
+
it('falls back cleanly to synthesizeDynamicQuestions when client fails', async () => {
|
|
426
|
+
const mockClient = {
|
|
427
|
+
timeout: 5000,
|
|
428
|
+
async chat() {
|
|
429
|
+
throw new Error('Model offline');
|
|
430
|
+
}
|
|
431
|
+
};
|
|
432
|
+
|
|
433
|
+
const questions = await generateClarifyingQuestions('เพลงล่าสุด', 'th', { client: mockClient });
|
|
434
|
+
assert.ok(Array.isArray(questions));
|
|
435
|
+
assert.ok(questions.length > 0);
|
|
436
|
+
});
|
|
437
|
+
});
|
|
438
|
+
|
|
439
|
+
console.log('✅ All clarifying questions tests passed');
|
|
440
|
+
|
|
441
|
+
describe('multilingual support (ja/ko)', () => {
|
|
442
|
+
it('returns Japanese questions when detectedLang is ja', () => {
|
|
443
|
+
const result = detectAmbiguities('recent works', 'ja');
|
|
444
|
+
const timeframeQ = result.find(r => r.category === 'timeframe');
|
|
445
|
+
assert.ok(timeframeQ);
|
|
446
|
+
assert.ok(/期間/.test(timeframeQ.question));
|
|
447
|
+
assert.ok(timeframeQ.options.every(opt => !/[\u0E00-\u0E7F]/.test(opt)));
|
|
448
|
+
});
|
|
449
|
+
|
|
450
|
+
it('returns Korean questions when detectedLang is ko', () => {
|
|
451
|
+
const result = detectAmbiguities('recent works', 'ko');
|
|
452
|
+
const timeframeQ = result.find(r => r.category === 'timeframe');
|
|
453
|
+
assert.ok(timeframeQ);
|
|
454
|
+
assert.ok(/기간/.test(timeframeQ.question));
|
|
455
|
+
assert.ok(timeframeQ.options.every(opt => !/[\u0E00-\u0E7F]/.test(opt)));
|
|
456
|
+
});
|
|
457
|
+
|
|
458
|
+
it('normalizes language aliases (jp -> ja, kr -> ko, zh-TW -> zh)', () => {
|
|
459
|
+
const ja = detectAmbiguities('recent works', 'jp');
|
|
460
|
+
assert.ok(ja.length > 0 && /[\u3040-\u30FF]/.test(ja[0].question));
|
|
461
|
+
const ko = detectAmbiguities('recent works', 'kr');
|
|
462
|
+
assert.ok(ko.length > 0 && /[\uAC00-\uD7AF]/.test(ko[0].question));
|
|
463
|
+
const zh = detectAmbiguities('recent works', 'zh-TW');
|
|
464
|
+
assert.ok(zh.length > 0 && /[\u4E00-\u9FFF]/.test(zh[0].question));
|
|
465
|
+
});
|
|
466
|
+
|
|
467
|
+
it('falls back to English for unknown language tags', () => {
|
|
468
|
+
const result = detectAmbiguities('recent works', 'xx');
|
|
469
|
+
assert.ok(result.length > 0);
|
|
470
|
+
assert.ok(result[0].question.includes('timeframe') || result[0].question.includes('Could you'));
|
|
471
|
+
});
|
|
472
|
+
|
|
473
|
+
it('detects Japanese keywords in queries', () => {
|
|
474
|
+
const result = detectAmbiguities('彼女のドラマと曲を教えて', 'ja');
|
|
475
|
+
const categories = result.map(r => r.category);
|
|
476
|
+
assert.ok(categories.includes('media_type'));
|
|
477
|
+
});
|
|
478
|
+
|
|
479
|
+
it('detects Korean keywords in queries', () => {
|
|
480
|
+
const result = detectAmbiguities('그녀의 노래를 보여줘', 'ko');
|
|
481
|
+
const categories = result.map(r => r.category);
|
|
482
|
+
assert.ok(categories.includes('media_type'));
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
it('isQueryTooVague handles Japanese and Korean', () => {
|
|
486
|
+
assert.equal(isQueryTooVague('はい'), true);
|
|
487
|
+
assert.equal(isQueryTooVague('花間令について詳しく教えて'), false);
|
|
488
|
+
assert.equal(isQueryTooVague('그녀의 최신 드라마를 알려줘'), false);
|
|
489
|
+
assert.equal(isQueryTooVague('???'), true);
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
it('calculateConfidence is not over-penalized for CJK queries', () => {
|
|
493
|
+
const result = calculateConfidence('彼女の最新のドラマ作品について詳しく教えてください', []);
|
|
494
|
+
assert.ok(result.score > 0.3, `Expected score > 0.3, got ${result.score}`);
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
it('generates Japanese fallback questions without a client', async () => {
|
|
498
|
+
const questions = await generateClarifyingQuestions('彼女の作品について教えて', 'ja', {});
|
|
499
|
+
assert.ok(questions.length > 0);
|
|
500
|
+
assert.ok(questions.every(q => /[\u3040-\u30FF]/.test(q.question)));
|
|
501
|
+
});
|
|
502
|
+
|
|
503
|
+
it('generates Korean fallback questions without a client', async () => {
|
|
504
|
+
const questions = await generateClarifyingQuestions('그녀의 작품에 대해 알려줘', 'ko', {});
|
|
505
|
+
assert.ok(questions.length > 0);
|
|
506
|
+
assert.ok(questions.every(q => /[\uAC00-\uD7AF]/.test(q.question)));
|
|
507
|
+
});
|
|
508
|
+
|
|
509
|
+
it('parses localized skip answers', () => {
|
|
510
|
+
for (const word of ['ข้าม', '跳过', 'スキップ', '건너뛰기', 'pass']) {
|
|
511
|
+
const result = parseClarifyingAnswers(word, [
|
|
512
|
+
{ category: 'timeframe', question: 'When?', options: ['2026'] },
|
|
513
|
+
]);
|
|
514
|
+
assert.equal(Object.keys(result.answers).length, 0);
|
|
515
|
+
assert.ok(result.summary.includes('skip'));
|
|
516
|
+
}
|
|
517
|
+
});
|
|
518
|
+
});
|
|
519
|
+
|
|
520
|
+
describe('fan-centric dynamic clarification & thinking block cleanup', () => {
|
|
521
|
+
it('cleanThinkingBlocks strips <think>, <thought>, and markdown thought blocks', () => {
|
|
522
|
+
const withThink = '<think>User wants info on In Blossom. Brackets [1, 2] here.</think>Hello world!';
|
|
523
|
+
assert.equal(cleanThinkingBlocks(withThink), 'Hello world!');
|
|
524
|
+
|
|
525
|
+
const withThought = '<thought>Analyzing query...</thought>[{"category": "test", "question": "Q?"}]';
|
|
526
|
+
assert.equal(cleanThinkingBlocks(withThought), '[{"category": "test", "question": "Q?"}]');
|
|
527
|
+
|
|
528
|
+
const withMarkdown = '```thought\nSome internal deliberations\n```\n[{"category": "music_style"}]';
|
|
529
|
+
assert.equal(cleanThinkingBlocks(withMarkdown), '[{"category": "music_style"}]');
|
|
530
|
+
|
|
531
|
+
assert.equal(cleanThinkingBlocks(null), '');
|
|
532
|
+
assert.equal(cleanThinkingBlocks(''), '');
|
|
533
|
+
});
|
|
534
|
+
|
|
535
|
+
it('generateClarifyingQuestions parses LLM responses wrapped with reasoning thought blocks', async () => {
|
|
536
|
+
const mockClient = {
|
|
537
|
+
timeout: 5000,
|
|
538
|
+
async chat() {
|
|
539
|
+
return {
|
|
540
|
+
content: '<thought>\n' +
|
|
541
|
+
'The fan is asking about the drama In Blossom (花间令).\n' +
|
|
542
|
+
'They want to know more about the character Yang Caiwei.\n' +
|
|
543
|
+
'I should propose questions about character arc and signature scenes.\n' +
|
|
544
|
+
'[internal note: brackets should not confuse the parser]\n' +
|
|
545
|
+
'</thought>\n' +
|
|
546
|
+
'[\n' +
|
|
547
|
+
' {\n' +
|
|
548
|
+
' "category": "character_arc",\n' +
|
|
549
|
+
' "question": "เกี่ยวกับหยางไฉ่เวย สนใจเจาะลึกมิติตัวละครหรือเคมีคู่กับพานเยว่คะ?",\n' +
|
|
550
|
+
' "options": ["มิติตัวละครหยางไฉ่เวย", "เคมีคู่พานเยว่", "เบื้องหลังการถ่ายทำ"]\n' +
|
|
551
|
+
' },\n' +
|
|
552
|
+
' {\n' +
|
|
553
|
+
' "category": "scene_highlight",\n' +
|
|
554
|
+
' "question": "อยากดูการวิเคราะห์ฉากไฮไลต์ซีนไหนเป็นพิเศษไหมคะ?",\n' +
|
|
555
|
+
' "options": ["ฉากแต่งงานสลับตัว", "ฉากไขคดีแรก", "ฉากจบสุดประทับใจ"]\n' +
|
|
556
|
+
' }\n' +
|
|
557
|
+
']'
|
|
558
|
+
};
|
|
559
|
+
}
|
|
560
|
+
};
|
|
561
|
+
|
|
562
|
+
const questions = await generateClarifyingQuestions('เล่าเรื่องฮวาเจียนลิ่งหน่อย', 'th', { client: mockClient });
|
|
563
|
+
assert.ok(Array.isArray(questions));
|
|
564
|
+
assert.equal(questions.length, 2);
|
|
565
|
+
assert.equal(questions[0].category, 'character_arc');
|
|
566
|
+
assert.equal(questions[0].question, 'เกี่ยวกับหยางไฉ่เวย สนใจเจาะลึกมิติตัวละครหรือเคมีคู่กับพานเยว่คะ?');
|
|
567
|
+
assert.deepEqual(questions[0].options, ['มิติตัวละครหยางไฉ่เวย', 'เคมีคู่พานเยว่', 'เบื้องหลังการถ่ายทำ']);
|
|
568
|
+
assert.equal(questions[1].category, 'scene_highlight');
|
|
569
|
+
});
|
|
570
|
+
|
|
571
|
+
it('synthesizes music-specific dynamic questions without redundant drama vs song asking', async () => {
|
|
572
|
+
const questions = await generateClarifyingQuestions('อยากฟังเพลงของเสี่ยวจวี แนะนำหน่อย', 'th', {});
|
|
573
|
+
assert.ok(questions.length > 0);
|
|
574
|
+
const categories = questions.map(q => q.category);
|
|
575
|
+
assert.ok(categories.includes('music_aspect') || categories.includes('stage_vibe') || categories.includes('subject_focus'));
|
|
576
|
+
assert.ok(!categories.includes('media_type'), 'Should not ask if user wants drama or music when they asked for songs');
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
it('synthesizes drama-specific dynamic questions without redundant drama vs song asking', async () => {
|
|
580
|
+
const questions = await generateClarifyingQuestions('แนะนำซีรีส์น่าดูของจวีจิ้งอีหน่อย', 'th', {});
|
|
581
|
+
assert.ok(questions.length > 0);
|
|
582
|
+
const categories = questions.map(q => q.category);
|
|
583
|
+
assert.ok(categories.includes('drama_genre') || categories.includes('drama_aspect') || categories.includes('subject_focus'));
|
|
584
|
+
assert.ok(!categories.includes('media_type'), 'Should not ask if user wants drama or music when they asked for dramas');
|
|
585
|
+
});
|
|
586
|
+
|
|
587
|
+
it('synthesizes fashion-specific dynamic questions for brand queries', async () => {
|
|
588
|
+
const questions = await generateClarifyingQuestions('จวีจิ้งอีกับเครื่องประดับ FRED Paris เป็นยังไงบ้าง', 'th', {});
|
|
589
|
+
assert.ok(questions.length > 0);
|
|
590
|
+
const categories = questions.map(q => q.category);
|
|
591
|
+
assert.ok(categories.includes('fashion_aspect') || categories.includes('subject_focus'));
|
|
592
|
+
assert.ok(!categories.includes('media_type'));
|
|
593
|
+
});
|
|
594
|
+
});
|