claude-memory-layer 1.0.32 → 1.0.34
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/index.js +1114 -74
- package/dist/cli/index.js.map +4 -4
- package/dist/core/index.js +414 -25
- package/dist/core/index.js.map +2 -2
- package/dist/hooks/post-tool-use.js +416 -27
- package/dist/hooks/post-tool-use.js.map +2 -2
- package/dist/hooks/semantic-daemon.js +416 -27
- package/dist/hooks/semantic-daemon.js.map +2 -2
- package/dist/hooks/session-end.js +416 -27
- package/dist/hooks/session-end.js.map +2 -2
- package/dist/hooks/session-start.js +416 -27
- package/dist/hooks/session-start.js.map +2 -2
- package/dist/hooks/stop.js +416 -27
- package/dist/hooks/stop.js.map +2 -2
- package/dist/hooks/user-prompt-submit.js +504 -34
- package/dist/hooks/user-prompt-submit.js.map +2 -2
- package/dist/index.js +416 -27
- package/dist/index.js.map +2 -2
- package/dist/mcp/index.js +567 -51
- package/dist/mcp/index.js.map +2 -2
- package/dist/server/api/index.js +850 -44
- package/dist/server/api/index.js.map +3 -3
- package/dist/server/index.js +1073 -64
- package/dist/server/index.js.map +3 -3
- package/dist/services/memory-service.js +416 -27
- package/dist/services/memory-service.js.map +2 -2
- package/dist/ui/assets/js/bootstrap.js +2 -0
- package/dist/ui/assets/js/overview.js +166 -3
- package/dist/ui/assets/js/state.js +3 -0
- package/dist/ui/index.html +20 -0
- package/dist/ui/style.css +193 -0
- package/package.json +5 -1
package/dist/server/api/index.js
CHANGED
|
@@ -2246,10 +2246,15 @@ function sqliteClose(db) {
|
|
|
2246
2246
|
function toDateFromSQLite(value) {
|
|
2247
2247
|
if (value instanceof Date)
|
|
2248
2248
|
return value;
|
|
2249
|
-
if (typeof value === "string")
|
|
2250
|
-
return new Date(value);
|
|
2251
2249
|
if (typeof value === "number")
|
|
2252
2250
|
return new Date(value);
|
|
2251
|
+
if (typeof value === "string") {
|
|
2252
|
+
const trimmed = value.trim();
|
|
2253
|
+
if (/^\d{4}-\d{2}-\d{2}[ T]\d{2}:\d{2}:\d{2}(?:\.\d+)?$/.test(trimmed)) {
|
|
2254
|
+
return /* @__PURE__ */ new Date(trimmed.replace(" ", "T") + "Z");
|
|
2255
|
+
}
|
|
2256
|
+
return new Date(trimmed);
|
|
2257
|
+
}
|
|
2253
2258
|
return new Date(String(value));
|
|
2254
2259
|
}
|
|
2255
2260
|
function toSQLiteTimestamp(date) {
|
|
@@ -2315,6 +2320,13 @@ var MarkdownMirror2 = class {
|
|
|
2315
2320
|
};
|
|
2316
2321
|
|
|
2317
2322
|
// src/core/sqlite-event-store.ts
|
|
2323
|
+
function normalizeQueryRewriteKind(value) {
|
|
2324
|
+
const normalized = (value || "").trim().toLowerCase();
|
|
2325
|
+
if (normalized === "follow-up-context" || normalized === "intent-rewrite")
|
|
2326
|
+
return normalized;
|
|
2327
|
+
return "none";
|
|
2328
|
+
}
|
|
2329
|
+
var REWRITTEN_QUERY_REWRITE_KIND_SQL = `LOWER(TRIM(COALESCE(query_rewrite_kind, 'none'))) IN ('follow-up-context', 'intent-rewrite')`;
|
|
2318
2330
|
var SQLiteEventStore = class {
|
|
2319
2331
|
db;
|
|
2320
2332
|
initialized = false;
|
|
@@ -2575,6 +2587,8 @@ var SQLiteEventStore = class {
|
|
|
2575
2587
|
session_id TEXT,
|
|
2576
2588
|
project_hash TEXT,
|
|
2577
2589
|
query_text TEXT NOT NULL,
|
|
2590
|
+
raw_query_text TEXT,
|
|
2591
|
+
query_rewrite_kind TEXT,
|
|
2578
2592
|
strategy TEXT,
|
|
2579
2593
|
candidate_event_ids TEXT,
|
|
2580
2594
|
selected_event_ids TEXT,
|
|
@@ -2616,6 +2630,8 @@ var SQLiteEventStore = class {
|
|
|
2616
2630
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_event ON memory_helpfulness(event_id);
|
|
2617
2631
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_session ON memory_helpfulness(session_id);
|
|
2618
2632
|
CREATE INDEX IF NOT EXISTS idx_helpfulness_score ON memory_helpfulness(helpfulness_score DESC);
|
|
2633
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_created_at ON memory_helpfulness(created_at);
|
|
2634
|
+
CREATE INDEX IF NOT EXISTS idx_helpfulness_measured_at ON memory_helpfulness(measured_at);
|
|
2619
2635
|
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_created_at ON retrieval_traces(created_at DESC);
|
|
2620
2636
|
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_project_hash ON retrieval_traces(project_hash);
|
|
2621
2637
|
CREATE INDEX IF NOT EXISTS idx_retrieval_traces_session_id ON retrieval_traces(session_id);
|
|
@@ -2649,6 +2665,18 @@ var SQLiteEventStore = class {
|
|
|
2649
2665
|
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN candidate_details_json TEXT;`);
|
|
2650
2666
|
} catch {
|
|
2651
2667
|
}
|
|
2668
|
+
try {
|
|
2669
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN raw_query_text TEXT;`);
|
|
2670
|
+
} catch {
|
|
2671
|
+
}
|
|
2672
|
+
try {
|
|
2673
|
+
sqliteExec(this.db, `ALTER TABLE retrieval_traces ADD COLUMN query_rewrite_kind TEXT;`);
|
|
2674
|
+
} catch {
|
|
2675
|
+
}
|
|
2676
|
+
try {
|
|
2677
|
+
sqliteExec(this.db, `CREATE INDEX IF NOT EXISTS idx_retrieval_traces_query_rewrite_kind ON retrieval_traces(query_rewrite_kind);`);
|
|
2678
|
+
} catch {
|
|
2679
|
+
}
|
|
2652
2680
|
const tableInfo = sqliteAll(this.db, "PRAGMA table_info(events)", []);
|
|
2653
2681
|
const columnNames = tableInfo.map((col) => col.name);
|
|
2654
2682
|
if (!columnNames.includes("access_count")) {
|
|
@@ -3434,8 +3462,11 @@ var SQLiteEventStore = class {
|
|
|
3434
3462
|
/**
|
|
3435
3463
|
* Get helpfulness statistics for dashboard
|
|
3436
3464
|
*/
|
|
3437
|
-
async getHelpfulnessStats() {
|
|
3465
|
+
async getHelpfulnessStats(since) {
|
|
3438
3466
|
await this.initialize();
|
|
3467
|
+
const sinceIso = since?.toISOString();
|
|
3468
|
+
const evaluatedWhere = sinceIso ? `WHERE measured_at IS NOT NULL AND datetime(created_at) >= datetime(?)` : `WHERE measured_at IS NOT NULL`;
|
|
3469
|
+
const totalWhere = sinceIso ? `WHERE datetime(created_at) >= datetime(?)` : ``;
|
|
3439
3470
|
const stats = sqliteGet(
|
|
3440
3471
|
this.db,
|
|
3441
3472
|
`SELECT
|
|
@@ -3445,11 +3476,13 @@ var SQLiteEventStore = class {
|
|
|
3445
3476
|
SUM(CASE WHEN helpfulness_score >= 0.4 AND helpfulness_score < 0.7 THEN 1 ELSE 0 END) as neutral,
|
|
3446
3477
|
SUM(CASE WHEN helpfulness_score < 0.4 THEN 1 ELSE 0 END) as unhelpful
|
|
3447
3478
|
FROM memory_helpfulness
|
|
3448
|
-
|
|
3479
|
+
${evaluatedWhere}`,
|
|
3480
|
+
sinceIso ? [sinceIso] : []
|
|
3449
3481
|
);
|
|
3450
3482
|
const totalRow = sqliteGet(
|
|
3451
3483
|
this.db,
|
|
3452
|
-
`SELECT COUNT(*) as total FROM memory_helpfulness
|
|
3484
|
+
`SELECT COUNT(*) as total FROM memory_helpfulness ${totalWhere}`,
|
|
3485
|
+
sinceIso ? [sinceIso] : []
|
|
3453
3486
|
);
|
|
3454
3487
|
return {
|
|
3455
3488
|
avgScore: Math.round((stats?.avg_score || 0) * 100) / 100,
|
|
@@ -3548,18 +3581,21 @@ var SQLiteEventStore = class {
|
|
|
3548
3581
|
async recordRetrievalTrace(input) {
|
|
3549
3582
|
await this.initialize();
|
|
3550
3583
|
const traceId = randomUUID5();
|
|
3584
|
+
const queryRewriteKind = normalizeQueryRewriteKind(input.queryRewriteKind);
|
|
3551
3585
|
sqliteRun(
|
|
3552
3586
|
this.db,
|
|
3553
3587
|
`INSERT INTO retrieval_traces (
|
|
3554
|
-
trace_id, session_id, project_hash, query_text, strategy,
|
|
3588
|
+
trace_id, session_id, project_hash, query_text, raw_query_text, query_rewrite_kind, strategy,
|
|
3555
3589
|
candidate_event_ids, selected_event_ids, candidate_details_json, selected_details_json,
|
|
3556
3590
|
candidate_count, selected_count, confidence, fallback_trace
|
|
3557
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
3591
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
3558
3592
|
[
|
|
3559
3593
|
traceId,
|
|
3560
3594
|
input.sessionId || null,
|
|
3561
3595
|
input.projectHash || null,
|
|
3562
3596
|
input.queryText,
|
|
3597
|
+
input.rawQueryText || null,
|
|
3598
|
+
queryRewriteKind,
|
|
3563
3599
|
input.strategy || null,
|
|
3564
3600
|
JSON.stringify(input.candidateEventIds || []),
|
|
3565
3601
|
JSON.stringify(input.selectedEventIds || []),
|
|
@@ -3585,6 +3621,8 @@ var SQLiteEventStore = class {
|
|
|
3585
3621
|
sessionId: row.session_id || void 0,
|
|
3586
3622
|
projectHash: row.project_hash || void 0,
|
|
3587
3623
|
queryText: row.query_text,
|
|
3624
|
+
rawQueryText: row.raw_query_text || void 0,
|
|
3625
|
+
queryRewriteKind: normalizeQueryRewriteKind(row.query_rewrite_kind),
|
|
3588
3626
|
strategy: row.strategy || void 0,
|
|
3589
3627
|
candidateEventIds: row.candidate_event_ids ? JSON.parse(row.candidate_event_ids) : [],
|
|
3590
3628
|
selectedEventIds: row.selected_event_ids ? JSON.parse(row.selected_event_ids) : [],
|
|
@@ -3611,6 +3649,11 @@ var SQLiteEventStore = class {
|
|
|
3611
3649
|
COUNT(*) as total_queries,
|
|
3612
3650
|
AVG(candidate_count) as avg_candidate_count,
|
|
3613
3651
|
AVG(selected_count) as avg_selected_count,
|
|
3652
|
+
SUM(CASE WHEN ${REWRITTEN_QUERY_REWRITE_KIND_SQL} THEN 1 ELSE 0 END) as rewritten_queries,
|
|
3653
|
+
SUM(CASE WHEN ${REWRITTEN_QUERY_REWRITE_KIND_SQL} AND selected_count > 0 THEN 1 ELSE 0 END) as rewritten_queries_with_selection,
|
|
3654
|
+
SUM(CASE WHEN NOT (${REWRITTEN_QUERY_REWRITE_KIND_SQL}) AND selected_count > 0 THEN 1 ELSE 0 END) as raw_queries_with_selection,
|
|
3655
|
+
AVG(CASE WHEN ${REWRITTEN_QUERY_REWRITE_KIND_SQL} THEN selected_count END) as avg_selected_count_for_rewritten_queries,
|
|
3656
|
+
AVG(CASE WHEN NOT (${REWRITTEN_QUERY_REWRITE_KIND_SQL}) THEN selected_count END) as avg_selected_count_for_raw_queries,
|
|
3614
3657
|
CASE
|
|
3615
3658
|
WHEN SUM(candidate_count) > 0 THEN (SUM(selected_count) * 1.0 / SUM(candidate_count))
|
|
3616
3659
|
ELSE 0
|
|
@@ -3618,15 +3661,41 @@ var SQLiteEventStore = class {
|
|
|
3618
3661
|
FROM retrieval_traces`,
|
|
3619
3662
|
[]
|
|
3620
3663
|
);
|
|
3664
|
+
const totalQueries = Number(row?.total_queries || 0);
|
|
3665
|
+
const rewrittenQueries = Number(row?.rewritten_queries || 0);
|
|
3666
|
+
const rawQueries = Math.max(0, totalQueries - rewrittenQueries);
|
|
3667
|
+
const rewrittenQueriesWithSelection = Number(row?.rewritten_queries_with_selection || 0);
|
|
3668
|
+
const rawQueriesWithSelection = Number(row?.raw_queries_with_selection || 0);
|
|
3621
3669
|
return {
|
|
3622
|
-
totalQueries
|
|
3670
|
+
totalQueries,
|
|
3623
3671
|
avgCandidateCount: Number(row?.avg_candidate_count || 0),
|
|
3624
3672
|
avgSelectedCount: Number(row?.avg_selected_count || 0),
|
|
3625
|
-
selectionRate: Number(row?.selection_rate || 0)
|
|
3673
|
+
selectionRate: Number(row?.selection_rate || 0),
|
|
3674
|
+
rewrittenQueries,
|
|
3675
|
+
rewriteRate: totalQueries > 0 ? rewrittenQueries / totalQueries : 0,
|
|
3676
|
+
rewrittenQueriesWithSelection,
|
|
3677
|
+
rawQueriesWithSelection,
|
|
3678
|
+
rewrittenSelectionRate: rewrittenQueries > 0 ? rewrittenQueriesWithSelection / rewrittenQueries : 0,
|
|
3679
|
+
rawSelectionRate: rawQueries > 0 ? rawQueriesWithSelection / rawQueries : 0,
|
|
3680
|
+
avgSelectedCountForRewrittenQueries: Number(row?.avg_selected_count_for_rewritten_queries || 0),
|
|
3681
|
+
avgSelectedCountForRawQueries: Number(row?.avg_selected_count_for_raw_queries || 0)
|
|
3626
3682
|
};
|
|
3627
3683
|
} catch (err) {
|
|
3628
3684
|
if (err?.message?.includes("no such table")) {
|
|
3629
|
-
return {
|
|
3685
|
+
return {
|
|
3686
|
+
totalQueries: 0,
|
|
3687
|
+
avgCandidateCount: 0,
|
|
3688
|
+
avgSelectedCount: 0,
|
|
3689
|
+
selectionRate: 0,
|
|
3690
|
+
rewrittenQueries: 0,
|
|
3691
|
+
rewriteRate: 0,
|
|
3692
|
+
rewrittenQueriesWithSelection: 0,
|
|
3693
|
+
rawQueriesWithSelection: 0,
|
|
3694
|
+
rewrittenSelectionRate: 0,
|
|
3695
|
+
rawSelectionRate: 0,
|
|
3696
|
+
avgSelectedCountForRewrittenQueries: 0,
|
|
3697
|
+
avgSelectedCountForRawQueries: 0
|
|
3698
|
+
};
|
|
3630
3699
|
}
|
|
3631
3700
|
throw err;
|
|
3632
3701
|
}
|
|
@@ -4438,6 +4507,57 @@ var COMMAND_ARTIFACT_PATTERNS = [
|
|
|
4438
4507
|
/<local-command-stdout>[\s\S]*?<\/local-command-stdout>/i,
|
|
4439
4508
|
/<local-command-stderr>[\s\S]*?<\/local-command-stderr>/i
|
|
4440
4509
|
];
|
|
4510
|
+
var CONTINUATION_QUERY_PATTERNS = [
|
|
4511
|
+
/^\s*(?:continue|resume|next|what(?:'s| is)? next|next\s+(?:step|task|action)|recommended\s+(?:next\s+)?(?:step|task|action)|what should (?:we|i) do next)\??\s*$/i,
|
|
4512
|
+
/^\s*(?:응\s*)?(?:이어서(?:\s*진행(?:해줘)?)?|계속(?:\s*해줘)?|다음\s*(?:단계|작업|추천\s*작업|추천|할\s*일)?(?:은|는)?(?:\s*(?:뭐야|진행(?:해줘)?))?\??|남은\s*(?:추가(?:로)?\s*)?(?:(?:할\s*만한\s*)?(?:작업|일)|할\s*일)?(?:은|는)?\s*(?:있어|있나|있나요|뭐야)\??|추천\s*작업(?:은|는)?(?:\s*뭐야)?\??|진행해줘)\s*$/i
|
|
4513
|
+
];
|
|
4514
|
+
var SHORT_REPAIR_FOLLOW_UP_PATTERNS = [
|
|
4515
|
+
/^\s*(?:fix\s+(?:it|that)|repair\s+(?:it|that)|resolve\s+(?:it|that)|that\s+bug|same\s+issue)\s*$/i,
|
|
4516
|
+
/^\s*(?:그거|그것|이거|이것)?\s*(?:고쳐줘|수정해줘|해결해줘|처리해줘)\s*$/i
|
|
4517
|
+
];
|
|
4518
|
+
var CURRENT_STATE_QUERY_PATTERNS = [
|
|
4519
|
+
/\bcurrent\b.*\b(?:state|status|deployment|blocker|pr|pull request)\b/i,
|
|
4520
|
+
/\b(?:still|as current|current)\b.*\b(?:unresolved|open|pending|not completed)\b/i,
|
|
4521
|
+
/\b(?:old|obsolete|stale|resolved|already resolved)\b.*\b(?:current|still|unresolved|open|state|status)\b/i,
|
|
4522
|
+
/(?:현재|아직|이전|오래된|해결된).*(?:상태|미해결|열린|블로커|PR|풀리퀘스트)/i
|
|
4523
|
+
];
|
|
4524
|
+
var STALE_CONTENT_PATTERNS = [
|
|
4525
|
+
/\b(?:obsolete|superseded|outdated)\b/i,
|
|
4526
|
+
/\bstale\s+(?:operational\s+)?state\b/i,
|
|
4527
|
+
/\bstale\s+after\b/i,
|
|
4528
|
+
/\bno\s+longer\s+(?:valid|current|applies?)\b/i,
|
|
4529
|
+
/\bearlier\s+(?:pull request|pr)\b[\s\S]{0,160}\b(?:open|not completed|had not completed)\b/i,
|
|
4530
|
+
/\bshould\s+not\s+be\s+injected\s+as\s+current\s+context\b/i,
|
|
4531
|
+
/(?:오래된|더 이상 유효하지|현재 상태가 아님)/i
|
|
4532
|
+
];
|
|
4533
|
+
var CONTINUATION_EXPANSION = "current next step plan roadmap status validation replay rerank memory usefulness continuation";
|
|
4534
|
+
var REPAIR_FOLLOW_UP_EXPANSION = "review blocker fix pattern dashboard error state metrics bucket validation sanitize rerun unresolved";
|
|
4535
|
+
var RETRIEVAL_PRIVACY_DECISION_EXPANSION = "retrieval telemetry privacy public api dashboard dashboards rawQueryText queryText raw query text expose safe trace metadata trace id reason strategy rewrite kind aggregate count counts candidate selected public panel";
|
|
4536
|
+
var DECISION_RECALL_TERMS = /* @__PURE__ */ new Set([
|
|
4537
|
+
"decide",
|
|
4538
|
+
"decided",
|
|
4539
|
+
"decision",
|
|
4540
|
+
"agreed",
|
|
4541
|
+
"policy",
|
|
4542
|
+
"constraint"
|
|
4543
|
+
]);
|
|
4544
|
+
var RETRIEVAL_PRIVACY_SURFACE_TERMS = /* @__PURE__ */ new Set([
|
|
4545
|
+
"retrieval",
|
|
4546
|
+
"dashboard",
|
|
4547
|
+
"telemetry",
|
|
4548
|
+
"trace"
|
|
4549
|
+
]);
|
|
4550
|
+
var DECISION_TOPIC_WEAK_TERMS = /* @__PURE__ */ new Set([
|
|
4551
|
+
"api",
|
|
4552
|
+
"dashboard",
|
|
4553
|
+
"retrieval",
|
|
4554
|
+
"trace",
|
|
4555
|
+
"telemetry",
|
|
4556
|
+
"query",
|
|
4557
|
+
"raw",
|
|
4558
|
+
"count",
|
|
4559
|
+
"counts"
|
|
4560
|
+
]);
|
|
4441
4561
|
var GENERIC_TECHNICAL_TERMS = /* @__PURE__ */ new Set([
|
|
4442
4562
|
"api",
|
|
4443
4563
|
"cli",
|
|
@@ -4455,6 +4575,87 @@ var GENERIC_TECHNICAL_TERMS = /* @__PURE__ */ new Set([
|
|
|
4455
4575
|
"db",
|
|
4456
4576
|
"sql"
|
|
4457
4577
|
]);
|
|
4578
|
+
var LOW_INFORMATION_QUERY_TERMS = /* @__PURE__ */ new Set([
|
|
4579
|
+
"the",
|
|
4580
|
+
"and",
|
|
4581
|
+
"or",
|
|
4582
|
+
"for",
|
|
4583
|
+
"from",
|
|
4584
|
+
"with",
|
|
4585
|
+
"without",
|
|
4586
|
+
"about",
|
|
4587
|
+
"what",
|
|
4588
|
+
"when",
|
|
4589
|
+
"where",
|
|
4590
|
+
"which",
|
|
4591
|
+
"who",
|
|
4592
|
+
"why",
|
|
4593
|
+
"how",
|
|
4594
|
+
"did",
|
|
4595
|
+
"does",
|
|
4596
|
+
"do",
|
|
4597
|
+
"we",
|
|
4598
|
+
"i",
|
|
4599
|
+
"in",
|
|
4600
|
+
"to",
|
|
4601
|
+
"of",
|
|
4602
|
+
"on",
|
|
4603
|
+
"as",
|
|
4604
|
+
"be",
|
|
4605
|
+
"was",
|
|
4606
|
+
"were",
|
|
4607
|
+
"decide",
|
|
4608
|
+
"decided",
|
|
4609
|
+
"decision",
|
|
4610
|
+
"agreed",
|
|
4611
|
+
"policy",
|
|
4612
|
+
"constraint",
|
|
4613
|
+
"showing",
|
|
4614
|
+
"can",
|
|
4615
|
+
"you",
|
|
4616
|
+
"me",
|
|
4617
|
+
"show",
|
|
4618
|
+
"tell",
|
|
4619
|
+
"please",
|
|
4620
|
+
"should",
|
|
4621
|
+
"would",
|
|
4622
|
+
"could",
|
|
4623
|
+
"this",
|
|
4624
|
+
"that",
|
|
4625
|
+
"these",
|
|
4626
|
+
"those",
|
|
4627
|
+
"use",
|
|
4628
|
+
"using",
|
|
4629
|
+
"treat",
|
|
4630
|
+
"continue",
|
|
4631
|
+
"resume",
|
|
4632
|
+
"next",
|
|
4633
|
+
"step",
|
|
4634
|
+
"task",
|
|
4635
|
+
"action",
|
|
4636
|
+
"current",
|
|
4637
|
+
"state",
|
|
4638
|
+
"status",
|
|
4639
|
+
"old",
|
|
4640
|
+
"already",
|
|
4641
|
+
"still",
|
|
4642
|
+
"near",
|
|
4643
|
+
"today",
|
|
4644
|
+
"\uC751",
|
|
4645
|
+
"\uADF8\uAC70",
|
|
4646
|
+
"\uADF8\uAC83",
|
|
4647
|
+
"\uC774\uAC70",
|
|
4648
|
+
"\uC774\uAC83",
|
|
4649
|
+
"\uB2E4\uC74C",
|
|
4650
|
+
"\uB2E8\uACC4",
|
|
4651
|
+
"\uC9C4\uD589",
|
|
4652
|
+
"\uC9C4\uD589\uD574\uC918",
|
|
4653
|
+
"\uACC4\uC18D",
|
|
4654
|
+
"\uC774\uC5B4\uC11C",
|
|
4655
|
+
"\uACE0\uCCD0\uC918",
|
|
4656
|
+
"\uC218\uC815\uD574\uC918",
|
|
4657
|
+
"\uD574\uACB0\uD574\uC918"
|
|
4658
|
+
]);
|
|
4458
4659
|
function isCommandArtifactQuery(query) {
|
|
4459
4660
|
const trimmed = query.trim();
|
|
4460
4661
|
if (!trimmed)
|
|
@@ -4466,6 +4667,73 @@ function isCommandArtifactQuery(query) {
|
|
|
4466
4667
|
return true;
|
|
4467
4668
|
return COMMAND_ARTIFACT_PATTERNS.some((pattern) => pattern.test(trimmed));
|
|
4468
4669
|
}
|
|
4670
|
+
function isGenericContinuationQuery(query) {
|
|
4671
|
+
const trimmed = query.trim();
|
|
4672
|
+
if (!trimmed)
|
|
4673
|
+
return false;
|
|
4674
|
+
if (!CONTINUATION_QUERY_PATTERNS.some((pattern) => pattern.test(trimmed)))
|
|
4675
|
+
return false;
|
|
4676
|
+
if (extractTechnicalQueryTerms(trimmed).length > 0)
|
|
4677
|
+
return false;
|
|
4678
|
+
const tokens = trimmed.match(/[A-Za-z0-9가-힣#._/-]+/g) ?? [];
|
|
4679
|
+
if (tokens.length > 10)
|
|
4680
|
+
return false;
|
|
4681
|
+
return !/[A-Za-z0-9_-]+\.[A-Za-z0-9]+/.test(trimmed) && !/(?:^|\s)(?:feat|fix|chore|refactor|docs)\/[A-Za-z0-9._-]+/.test(trimmed) && !/[A-Za-z]:?[\\/]|\/Users\/|\.\/|\.\.\//.test(trimmed);
|
|
4682
|
+
}
|
|
4683
|
+
function isShortRepairFollowUpQuery(query) {
|
|
4684
|
+
const trimmed = query.trim();
|
|
4685
|
+
if (!trimmed)
|
|
4686
|
+
return false;
|
|
4687
|
+
if (extractTechnicalQueryTerms(trimmed).length > 0)
|
|
4688
|
+
return false;
|
|
4689
|
+
const tokens = trimmed.match(/[A-Za-z0-9가-힣#._/-]+/g) ?? [];
|
|
4690
|
+
if (tokens.length > 8)
|
|
4691
|
+
return false;
|
|
4692
|
+
return SHORT_REPAIR_FOLLOW_UP_PATTERNS.some((pattern) => pattern.test(trimmed));
|
|
4693
|
+
}
|
|
4694
|
+
function isCurrentStateQuery(query) {
|
|
4695
|
+
const trimmed = query.trim();
|
|
4696
|
+
if (!trimmed)
|
|
4697
|
+
return false;
|
|
4698
|
+
return CURRENT_STATE_QUERY_PATTERNS.some((pattern) => pattern.test(trimmed));
|
|
4699
|
+
}
|
|
4700
|
+
function isStaleOrSupersededContent(content) {
|
|
4701
|
+
const trimmed = content.trim();
|
|
4702
|
+
if (!trimmed)
|
|
4703
|
+
return false;
|
|
4704
|
+
return STALE_CONTENT_PATTERNS.some((pattern) => pattern.test(trimmed));
|
|
4705
|
+
}
|
|
4706
|
+
function buildRetrievalQualityQuery(query) {
|
|
4707
|
+
const trimmed = query.trim();
|
|
4708
|
+
if (!trimmed)
|
|
4709
|
+
return query;
|
|
4710
|
+
if (isRetrievalPrivacyDecisionQuery(trimmed)) {
|
|
4711
|
+
return `${trimmed} ${RETRIEVAL_PRIVACY_DECISION_EXPANSION}`;
|
|
4712
|
+
}
|
|
4713
|
+
if (isGenericContinuationQuery(trimmed)) {
|
|
4714
|
+
return `${trimmed} ${CONTINUATION_EXPANSION}`;
|
|
4715
|
+
}
|
|
4716
|
+
if (isShortRepairFollowUpQuery(trimmed)) {
|
|
4717
|
+
return `${trimmed} ${REPAIR_FOLLOW_UP_EXPANSION}`;
|
|
4718
|
+
}
|
|
4719
|
+
return query;
|
|
4720
|
+
}
|
|
4721
|
+
function isRetrievalPrivacyDecisionQuery(query) {
|
|
4722
|
+
const trimmed = query.trim();
|
|
4723
|
+
if (!trimmed)
|
|
4724
|
+
return false;
|
|
4725
|
+
const terms = new Set(tokenizeQualityText(trimmed));
|
|
4726
|
+
const hasDecisionSignal = hasAnyTerm(terms, DECISION_RECALL_TERMS) || /(?:결정|정책|원칙)/i.test(trimmed);
|
|
4727
|
+
if (!hasDecisionSignal)
|
|
4728
|
+
return false;
|
|
4729
|
+
const hasRawQuerySignal = terms.has("raw") && terms.has("query");
|
|
4730
|
+
const hasPrivacySignal = terms.has("privacy") || terms.has("expose") || terms.has("redacted");
|
|
4731
|
+
const hasRetrievalSurface = hasAnyTerm(terms, RETRIEVAL_PRIVACY_SURFACE_TERMS) || terms.has("api") && terms.has("query");
|
|
4732
|
+
const hasQuerySurface = terms.has("query") && (terms.has("dashboard") || terms.has("trace") || terms.has("telemetry") || terms.has("api"));
|
|
4733
|
+
const hasKoreanRetrievalSurface = /(?:검색|리트리벌|retrieval|대시보드|트레이스|텔레메트리|telemetry)/i.test(trimmed);
|
|
4734
|
+
const hasKoreanPrivacySurface = /(?:원문|쿼리|프라이버시|개인정보|노출|트레이스|메타데이터)/i.test(trimmed);
|
|
4735
|
+
return (hasRetrievalSurface || hasKoreanRetrievalSurface && hasKoreanPrivacySurface) && (hasRawQuerySignal || hasPrivacySignal || hasQuerySurface || hasKoreanPrivacySurface);
|
|
4736
|
+
}
|
|
4469
4737
|
function extractTechnicalQueryTerms(query) {
|
|
4470
4738
|
const matches = query.match(/[A-Za-z][A-Za-z0-9_.:-]{2,}/g) ?? [];
|
|
4471
4739
|
const terms = matches.filter((term) => {
|
|
@@ -4483,9 +4751,87 @@ function hasTechnicalTermOverlap(query, content) {
|
|
|
4483
4751
|
const normalizedContent = content.toLowerCase();
|
|
4484
4752
|
return terms.some((term) => normalizedContent.includes(term));
|
|
4485
4753
|
}
|
|
4754
|
+
function hasDiscriminativeTermOverlap(query, content) {
|
|
4755
|
+
const queryTerms = extractDiscriminativeQueryTerms(query);
|
|
4756
|
+
const contentTerms = new Set(tokenizeQualityText(content));
|
|
4757
|
+
if (isRetrievalPrivacyDecisionQuery(query) && hasRetrievalPrivacyDecisionContent(contentTerms)) {
|
|
4758
|
+
return true;
|
|
4759
|
+
}
|
|
4760
|
+
if (shouldRequireDecisionTopicOverlap(query)) {
|
|
4761
|
+
const topicTerms = queryTerms.filter((term) => !DECISION_TOPIC_WEAK_TERMS.has(term));
|
|
4762
|
+
if (topicTerms.length > 0) {
|
|
4763
|
+
return topicTerms.some((term) => contentTerms.has(term));
|
|
4764
|
+
}
|
|
4765
|
+
}
|
|
4766
|
+
if (queryTerms.length < 3)
|
|
4767
|
+
return true;
|
|
4768
|
+
const requiredHits = queryTerms.length >= 3 ? 2 : 1;
|
|
4769
|
+
let hits = 0;
|
|
4770
|
+
for (const term of queryTerms) {
|
|
4771
|
+
if (contentTerms.has(term))
|
|
4772
|
+
hits += 1;
|
|
4773
|
+
if (hits >= requiredHits)
|
|
4774
|
+
return true;
|
|
4775
|
+
}
|
|
4776
|
+
return false;
|
|
4777
|
+
}
|
|
4486
4778
|
function shouldApplyTechnicalGuard(query) {
|
|
4487
4779
|
return extractTechnicalQueryTerms(query).length > 0;
|
|
4488
4780
|
}
|
|
4781
|
+
function hasAnyTerm(terms, expectedTerms) {
|
|
4782
|
+
let found = false;
|
|
4783
|
+
expectedTerms.forEach((term) => {
|
|
4784
|
+
if (terms.has(term))
|
|
4785
|
+
found = true;
|
|
4786
|
+
});
|
|
4787
|
+
return found;
|
|
4788
|
+
}
|
|
4789
|
+
function shouldRequireDecisionTopicOverlap(query) {
|
|
4790
|
+
if (isRetrievalPrivacyDecisionQuery(query))
|
|
4791
|
+
return false;
|
|
4792
|
+
const trimmed = query.trim();
|
|
4793
|
+
if (!trimmed)
|
|
4794
|
+
return false;
|
|
4795
|
+
const terms = new Set(tokenizeQualityText(trimmed));
|
|
4796
|
+
return hasAnyTerm(terms, DECISION_RECALL_TERMS) || /(?:결정|정책|원칙)/i.test(trimmed);
|
|
4797
|
+
}
|
|
4798
|
+
function extractDiscriminativeQueryTerms(query) {
|
|
4799
|
+
const seen = /* @__PURE__ */ new Set();
|
|
4800
|
+
const terms = [];
|
|
4801
|
+
for (const token of tokenizeQualityText(query)) {
|
|
4802
|
+
if (LOW_INFORMATION_QUERY_TERMS.has(token))
|
|
4803
|
+
continue;
|
|
4804
|
+
if (GENERIC_TECHNICAL_TERMS.has(token))
|
|
4805
|
+
continue;
|
|
4806
|
+
if (seen.has(token))
|
|
4807
|
+
continue;
|
|
4808
|
+
seen.add(token);
|
|
4809
|
+
terms.push(token);
|
|
4810
|
+
}
|
|
4811
|
+
return terms;
|
|
4812
|
+
}
|
|
4813
|
+
function hasRetrievalPrivacyDecisionContent(contentTerms) {
|
|
4814
|
+
const hasDashboardTraceMetadata = contentTerms.has("dashboard") && (contentTerms.has("trace") || contentTerms.has("metadata")) && (contentTerms.has("safe") || contentTerms.has("strategy") || contentTerms.has("rewrite") || contentTerms.has("candidate") || contentTerms.has("selected") || contentTerms.has("count") || contentTerms.has("reason"));
|
|
4815
|
+
const hasRawQueryPrivacyPolicy = contentTerms.has("retrieval") && (contentTerms.has("privacy") || contentTerms.has("expose") || contentTerms.has("raw") && contentTerms.has("query") && (contentTerms.has("dashboard") || contentTerms.has("telemetry") || contentTerms.has("api") || contentTerms.has("public")));
|
|
4816
|
+
return hasDashboardTraceMetadata || hasRawQueryPrivacyPolicy;
|
|
4817
|
+
}
|
|
4818
|
+
function tokenizeQualityText(text) {
|
|
4819
|
+
return text.replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase().replace(/[^A-Za-z0-9가-힣\s_.:-]/g, " ").split(/\s+/).flatMap((token) => token.split(/(?=[._:-])|(?<=[._:-])/g)).map((token) => normalizeQualityToken(token.replace(/^[._:-]+|[._:-]+$/g, ""))).filter((token) => token.length >= 2);
|
|
4820
|
+
}
|
|
4821
|
+
function normalizeQualityToken(token) {
|
|
4822
|
+
if (token === "apis")
|
|
4823
|
+
return "api";
|
|
4824
|
+
if (token === "ids")
|
|
4825
|
+
return "id";
|
|
4826
|
+
if (LOW_INFORMATION_QUERY_TERMS.has(token) || GENERIC_TECHNICAL_TERMS.has(token))
|
|
4827
|
+
return token;
|
|
4828
|
+
if (token.length > 4 && token.endsWith("ies"))
|
|
4829
|
+
return `${token.slice(0, -3)}y`;
|
|
4830
|
+
if (token.length > 3 && token.endsWith("s") && !token.endsWith("ss") && !token.endsWith("us") && !token.endsWith("is")) {
|
|
4831
|
+
return token.slice(0, -1);
|
|
4832
|
+
}
|
|
4833
|
+
return token;
|
|
4834
|
+
}
|
|
4489
4835
|
|
|
4490
4836
|
// src/core/retriever.ts
|
|
4491
4837
|
var DEFAULT_OPTIONS = {
|
|
@@ -4538,6 +4884,7 @@ var Retriever = class {
|
|
|
4538
4884
|
const opts = { ...DEFAULT_OPTIONS, ...options };
|
|
4539
4885
|
const sessionFilter = opts.scope?.sessionId ?? opts.sessionId;
|
|
4540
4886
|
const fallbackTrace = [];
|
|
4887
|
+
const qualityQuery = buildRetrievalQualityQuery(query);
|
|
4541
4888
|
if (isCommandArtifactQuery(query)) {
|
|
4542
4889
|
fallbackTrace.push("guard:command-artifact-query");
|
|
4543
4890
|
const emptyMatch = this.matcher.matchSearchResults([], () => 0);
|
|
@@ -4554,6 +4901,7 @@ var Retriever = class {
|
|
|
4554
4901
|
const fallbackEnabled = (opts.strategy ?? "auto") === "auto";
|
|
4555
4902
|
const primaryStrategy = opts.strategy === "auto" ? "fast" : opts.strategy || "fast";
|
|
4556
4903
|
let current = await this.runStage(query, {
|
|
4904
|
+
qualityQuery,
|
|
4557
4905
|
strategy: primaryStrategy,
|
|
4558
4906
|
topK: opts.topK,
|
|
4559
4907
|
minScore: opts.minScore,
|
|
@@ -4571,6 +4919,7 @@ var Retriever = class {
|
|
|
4571
4919
|
fallbackTrace.push(`stage:primary:${primaryStrategy}`);
|
|
4572
4920
|
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results) && primaryStrategy !== "deep") {
|
|
4573
4921
|
current = await this.runStage(query, {
|
|
4922
|
+
qualityQuery,
|
|
4574
4923
|
strategy: "deep",
|
|
4575
4924
|
topK: opts.topK,
|
|
4576
4925
|
minScore: opts.minScore,
|
|
@@ -4588,6 +4937,7 @@ var Retriever = class {
|
|
|
4588
4937
|
}
|
|
4589
4938
|
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
4590
4939
|
current = await this.runStage(query, {
|
|
4940
|
+
qualityQuery,
|
|
4591
4941
|
strategy: "deep",
|
|
4592
4942
|
topK: opts.topK,
|
|
4593
4943
|
minScore: Math.max(0.5, opts.minScore - 0.15),
|
|
@@ -4604,11 +4954,21 @@ var Retriever = class {
|
|
|
4604
4954
|
fallbackTrace.push("fallback:scope-expanded");
|
|
4605
4955
|
}
|
|
4606
4956
|
if (fallbackEnabled && this.shouldFallback(current.matchResult, current.results)) {
|
|
4607
|
-
const summary = await this.buildSummaryFallback(
|
|
4957
|
+
const summary = await this.buildSummaryFallback(qualityQuery, opts.topK);
|
|
4958
|
+
const scopedSummary = await this.applyScopeFilters(summary, {
|
|
4959
|
+
scope: opts.scope,
|
|
4960
|
+
projectScopeMode: opts.projectScopeMode,
|
|
4961
|
+
projectHash: opts.projectHash,
|
|
4962
|
+
allowedProjectHashes: opts.allowedProjectHashes
|
|
4963
|
+
});
|
|
4964
|
+
const filteredSummary = this.applyQualityFilters(scopedSummary, {
|
|
4965
|
+
query,
|
|
4966
|
+
minScore: opts.minScore
|
|
4967
|
+
});
|
|
4608
4968
|
current = {
|
|
4609
|
-
results:
|
|
4610
|
-
candidateResults:
|
|
4611
|
-
matchResult: this.matcher.matchSearchResults(
|
|
4969
|
+
results: filteredSummary,
|
|
4970
|
+
candidateResults: filteredSummary,
|
|
4971
|
+
matchResult: this.matcher.matchSearchResults(filteredSummary, () => 0)
|
|
4612
4972
|
};
|
|
4613
4973
|
fallbackTrace.push("fallback:summary");
|
|
4614
4974
|
}
|
|
@@ -4633,7 +4993,10 @@ var Retriever = class {
|
|
|
4633
4993
|
semanticScore: r.semanticScore,
|
|
4634
4994
|
lexicalScore: r.lexicalScore,
|
|
4635
4995
|
recencyScore: r.recencyScore
|
|
4636
|
-
}))
|
|
4996
|
+
})),
|
|
4997
|
+
rawQueryText: current.queryRewriteKind ? query : void 0,
|
|
4998
|
+
effectiveQueryText: current.effectiveQueryText,
|
|
4999
|
+
queryRewriteKind: current.queryRewriteKind
|
|
4637
5000
|
};
|
|
4638
5001
|
}
|
|
4639
5002
|
async retrieveUnified(query, options = {}) {
|
|
@@ -4671,8 +5034,11 @@ var Retriever = class {
|
|
|
4671
5034
|
}
|
|
4672
5035
|
}
|
|
4673
5036
|
async runStage(query, input) {
|
|
4674
|
-
|
|
4675
|
-
let
|
|
5037
|
+
const searchQuery = input.qualityQuery ?? query;
|
|
5038
|
+
let rerankQuery = searchQuery;
|
|
5039
|
+
let effectiveQueryText;
|
|
5040
|
+
let queryRewriteKind;
|
|
5041
|
+
let initialResults = await this.searchByStrategy(searchQuery, {
|
|
4676
5042
|
strategy: input.strategy,
|
|
4677
5043
|
topK: input.topK,
|
|
4678
5044
|
minScore: input.minScore,
|
|
@@ -4680,9 +5046,12 @@ var Retriever = class {
|
|
|
4680
5046
|
});
|
|
4681
5047
|
if (input.intentRewrite && input.strategy === "deep" && this.queryRewriter) {
|
|
4682
5048
|
const rewritten = (await this.queryRewriter(query))?.trim();
|
|
4683
|
-
|
|
4684
|
-
|
|
4685
|
-
|
|
5049
|
+
const normalizedQuery = query.trim();
|
|
5050
|
+
if (rewritten && rewritten !== normalizedQuery) {
|
|
5051
|
+
effectiveQueryText = `${normalizedQuery} ${rewritten}`.trim();
|
|
5052
|
+
queryRewriteKind = "intent-rewrite";
|
|
5053
|
+
rerankQuery = buildRetrievalQualityQuery(effectiveQueryText);
|
|
5054
|
+
const rewrittenResults = await this.searchByStrategy(buildRetrievalQualityQuery(rewritten), {
|
|
4686
5055
|
strategy: "deep",
|
|
4687
5056
|
topK: input.topK,
|
|
4688
5057
|
minScore: Math.max(0.5, input.minScore - 0.1),
|
|
@@ -4709,10 +5078,14 @@ var Retriever = class {
|
|
|
4709
5078
|
});
|
|
4710
5079
|
const top = qualityFiltered.slice(0, input.topK);
|
|
4711
5080
|
const matchResult = this.matcher.matchSearchResults(top, () => 0);
|
|
4712
|
-
return { results: top, candidateResults: qualityFiltered, matchResult };
|
|
5081
|
+
return { results: top, candidateResults: qualityFiltered, matchResult, effectiveQueryText, queryRewriteKind };
|
|
4713
5082
|
}
|
|
4714
5083
|
applyQualityFilters(results, options) {
|
|
4715
5084
|
let filtered = [...results];
|
|
5085
|
+
if (isCurrentStateQuery(options.query)) {
|
|
5086
|
+
filtered = filtered.filter((result) => !isStaleOrSupersededContent(result.content));
|
|
5087
|
+
}
|
|
5088
|
+
filtered = filtered.filter((result) => hasDiscriminativeTermOverlap(options.query, result.content));
|
|
4716
5089
|
if (shouldApplyTechnicalGuard(options.query)) {
|
|
4717
5090
|
filtered = filtered.filter((result) => hasTechnicalTermOverlap(options.query, result.content));
|
|
4718
5091
|
}
|
|
@@ -5020,7 +5393,21 @@ _Context:_ ${sessionContext}`;
|
|
|
5020
5393
|
});
|
|
5021
5394
|
}
|
|
5022
5395
|
tokenize(text) {
|
|
5023
|
-
return text.toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).filter((t) => t.length >= 2).slice(0, 64);
|
|
5396
|
+
return text.replace(/([a-z])([A-Z])/g, "$1 $2").toLowerCase().replace(/[^\p{L}\p{N}\s]/gu, " ").split(/\s+/).map((token) => this.normalizeToken(token)).filter((t) => t.length >= 2).slice(0, 64);
|
|
5397
|
+
}
|
|
5398
|
+
normalizeToken(token) {
|
|
5399
|
+
if (token === "apis")
|
|
5400
|
+
return "api";
|
|
5401
|
+
if (token === "ids")
|
|
5402
|
+
return "id";
|
|
5403
|
+
if (token === "does")
|
|
5404
|
+
return token;
|
|
5405
|
+
if (token.length > 4 && token.endsWith("ies"))
|
|
5406
|
+
return `${token.slice(0, -3)}y`;
|
|
5407
|
+
if (token.length > 3 && token.endsWith("s") && !token.endsWith("ss") && !token.endsWith("us") && !token.endsWith("is") && !token.endsWith("ps")) {
|
|
5408
|
+
return token.slice(0, -1);
|
|
5409
|
+
}
|
|
5410
|
+
return token;
|
|
5024
5411
|
}
|
|
5025
5412
|
keywordOverlap(a, b) {
|
|
5026
5413
|
if (a.length === 0 || b.length === 0)
|
|
@@ -5083,9 +5470,9 @@ var RetrievalAnalyticsService = class {
|
|
|
5083
5470
|
await this.deps.initialize();
|
|
5084
5471
|
return this.deps.retrievalStore.getHelpfulMemories(limit);
|
|
5085
5472
|
}
|
|
5086
|
-
async getHelpfulnessStats() {
|
|
5473
|
+
async getHelpfulnessStats(since) {
|
|
5087
5474
|
await this.deps.initialize();
|
|
5088
|
-
return this.deps.retrievalStore.getHelpfulnessStats();
|
|
5475
|
+
return this.deps.retrievalStore.getHelpfulnessStats(since);
|
|
5089
5476
|
}
|
|
5090
5477
|
/**
|
|
5091
5478
|
* Extract topic keywords from event content (markdown headings and key terms).
|
|
@@ -5510,7 +5897,9 @@ var RetrievalOrchestrator = class {
|
|
|
5510
5897
|
await this.deps.traceStore.recordRetrievalTrace({
|
|
5511
5898
|
sessionId: options?.sessionId,
|
|
5512
5899
|
projectHash: projectHash || void 0,
|
|
5513
|
-
queryText: query,
|
|
5900
|
+
queryText: result.effectiveQueryText || query,
|
|
5901
|
+
rawQueryText: result.rawQueryText || (result.queryRewriteKind ? query : void 0),
|
|
5902
|
+
queryRewriteKind: result.queryRewriteKind || "none",
|
|
5514
5903
|
strategy: options?.strategy || "auto",
|
|
5515
5904
|
candidateEventIds,
|
|
5516
5905
|
selectedEventIds,
|
|
@@ -7504,8 +7893,8 @@ var MemoryService = class {
|
|
|
7504
7893
|
/**
|
|
7505
7894
|
* Get helpfulness statistics for dashboard
|
|
7506
7895
|
*/
|
|
7507
|
-
async getHelpfulnessStats() {
|
|
7508
|
-
return this.retrievalAnalyticsService.getHelpfulnessStats();
|
|
7896
|
+
async getHelpfulnessStats(since) {
|
|
7897
|
+
return this.retrievalAnalyticsService.getHelpfulnessStats(since);
|
|
7509
7898
|
}
|
|
7510
7899
|
/**
|
|
7511
7900
|
* Mark a consolidated memory as accessed
|
|
@@ -8060,6 +8449,346 @@ function computeSessionTurnCount(sessionEvents) {
|
|
|
8060
8449
|
return turnIds.size;
|
|
8061
8450
|
return sessionEvents.filter((e) => e.eventType === "user_prompt").length;
|
|
8062
8451
|
}
|
|
8452
|
+
function normalizeQueryRewriteKind2(value) {
|
|
8453
|
+
const normalized = (value || "").trim().toLowerCase();
|
|
8454
|
+
if (normalized === "follow-up-context" || normalized === "intent-rewrite")
|
|
8455
|
+
return normalized;
|
|
8456
|
+
return "none";
|
|
8457
|
+
}
|
|
8458
|
+
function normalizeMetric(value) {
|
|
8459
|
+
const numberValue = Number(value || 0);
|
|
8460
|
+
if (!Number.isFinite(numberValue))
|
|
8461
|
+
return 0;
|
|
8462
|
+
return Math.max(0, Math.min(1, numberValue));
|
|
8463
|
+
}
|
|
8464
|
+
function getTimestampMs(value) {
|
|
8465
|
+
if (value instanceof Date)
|
|
8466
|
+
return value.getTime();
|
|
8467
|
+
if (typeof value === "string") {
|
|
8468
|
+
const parsed = new Date(value).getTime();
|
|
8469
|
+
return Number.isFinite(parsed) ? parsed : 0;
|
|
8470
|
+
}
|
|
8471
|
+
return 0;
|
|
8472
|
+
}
|
|
8473
|
+
function isRewrittenRetrievalTrace(trace) {
|
|
8474
|
+
return normalizeQueryRewriteKind2(trace.queryRewriteKind) !== "none";
|
|
8475
|
+
}
|
|
8476
|
+
function getTraceSelectedCount(trace) {
|
|
8477
|
+
return Number(trace.selectedCount ?? trace.selectedEventIds?.length ?? 0);
|
|
8478
|
+
}
|
|
8479
|
+
function getTraceCandidateCount(trace) {
|
|
8480
|
+
return Number(trace.candidateCount ?? trace.candidateEventIds?.length ?? trace.candidateDetails?.length ?? 0);
|
|
8481
|
+
}
|
|
8482
|
+
function makeRetrievalReviewItem(trace) {
|
|
8483
|
+
const candidateCount = getTraceCandidateCount(trace);
|
|
8484
|
+
const selectedCount = getTraceSelectedCount(trace);
|
|
8485
|
+
const queryRewriteKind = normalizeQueryRewriteKind2(trace.queryRewriteKind);
|
|
8486
|
+
const rewritten = queryRewriteKind !== "none";
|
|
8487
|
+
const createdAtMs = getTimestampMs(trace.createdAt);
|
|
8488
|
+
const createdAt = createdAtMs > 0 ? new Date(createdAtMs).toISOString() : (/* @__PURE__ */ new Date(0)).toISOString();
|
|
8489
|
+
let reason = null;
|
|
8490
|
+
let severity = "info";
|
|
8491
|
+
let priority = 0;
|
|
8492
|
+
let title = "";
|
|
8493
|
+
let detail = "";
|
|
8494
|
+
let action = "";
|
|
8495
|
+
if (candidateCount > 0 && selectedCount === 0 && rewritten) {
|
|
8496
|
+
reason = "rewritten-query-no-selection";
|
|
8497
|
+
severity = "warn";
|
|
8498
|
+
priority = 100;
|
|
8499
|
+
title = "Rewritten query selected no memories";
|
|
8500
|
+
detail = `${candidateCount} candidates were found after query rewrite, but no memory was selected.`;
|
|
8501
|
+
action = "Review rewrite wording, rerank scores, and final selection thresholds for this trace.";
|
|
8502
|
+
} else if (candidateCount > 0 && selectedCount === 0) {
|
|
8503
|
+
reason = "candidate-no-selection";
|
|
8504
|
+
severity = "warn";
|
|
8505
|
+
priority = 90;
|
|
8506
|
+
title = "Candidates found but nothing selected";
|
|
8507
|
+
detail = `${candidateCount} candidates were available, but the final selection injected no memory.`;
|
|
8508
|
+
action = "Review rerank thresholds and candidate filtering; consider overfetching before final selection.";
|
|
8509
|
+
} else if (candidateCount === 0) {
|
|
8510
|
+
reason = "empty-candidate-set";
|
|
8511
|
+
severity = "info";
|
|
8512
|
+
priority = 70;
|
|
8513
|
+
title = "Retrieval found no candidates";
|
|
8514
|
+
detail = "The retrieval pipeline returned no candidate memories for this trace.";
|
|
8515
|
+
action = "Check trigger/query rewrite coverage and whether the project has indexed memories for this topic.";
|
|
8516
|
+
} else if (candidateCount >= 10 && safeRatio(selectedCount, candidateCount) < 0.15) {
|
|
8517
|
+
reason = "low-selection-rate";
|
|
8518
|
+
severity = "info";
|
|
8519
|
+
priority = 60;
|
|
8520
|
+
title = "Low selection ratio from many candidates";
|
|
8521
|
+
detail = `${selectedCount} of ${candidateCount} candidates were selected.`;
|
|
8522
|
+
action = "Inspect score distribution and MMR/diversity settings before lowering thresholds.";
|
|
8523
|
+
}
|
|
8524
|
+
if (!reason)
|
|
8525
|
+
return null;
|
|
8526
|
+
return {
|
|
8527
|
+
traceId: trace.traceId || "unknown-trace",
|
|
8528
|
+
reason,
|
|
8529
|
+
severity,
|
|
8530
|
+
priority,
|
|
8531
|
+
title,
|
|
8532
|
+
detail,
|
|
8533
|
+
action,
|
|
8534
|
+
queryRewriteKind,
|
|
8535
|
+
rewritten,
|
|
8536
|
+
strategy: trace.strategy || null,
|
|
8537
|
+
candidateCount,
|
|
8538
|
+
selectedCount,
|
|
8539
|
+
candidateEventIds: (trace.candidateEventIds || []).slice(0, 5),
|
|
8540
|
+
selectedEventIds: (trace.selectedEventIds || []).slice(0, 5),
|
|
8541
|
+
candidateDetails: (trace.candidateDetails || []).slice(0, 3).map((detail2) => ({
|
|
8542
|
+
eventId: detail2.eventId,
|
|
8543
|
+
score: detail2.score,
|
|
8544
|
+
semanticScore: detail2.semanticScore,
|
|
8545
|
+
lexicalScore: detail2.lexicalScore,
|
|
8546
|
+
recencyScore: detail2.recencyScore
|
|
8547
|
+
})),
|
|
8548
|
+
selectedDetails: (trace.selectedDetails || []).slice(0, 3).map((detail2) => ({
|
|
8549
|
+
eventId: detail2.eventId,
|
|
8550
|
+
score: detail2.score,
|
|
8551
|
+
semanticScore: detail2.semanticScore,
|
|
8552
|
+
lexicalScore: detail2.lexicalScore,
|
|
8553
|
+
recencyScore: detail2.recencyScore
|
|
8554
|
+
})),
|
|
8555
|
+
createdAt
|
|
8556
|
+
};
|
|
8557
|
+
}
|
|
8558
|
+
function buildRetrievalReviewQueue(traces, limit) {
|
|
8559
|
+
const reviewItems = traces.map(makeRetrievalReviewItem).filter((item) => item !== null).sort((a, b) => b.priority - a.priority || new Date(b.createdAt).getTime() - new Date(a.createdAt).getTime());
|
|
8560
|
+
return {
|
|
8561
|
+
summary: {
|
|
8562
|
+
totalTraces: traces.length,
|
|
8563
|
+
reviewItems: reviewItems.length,
|
|
8564
|
+
returnedItems: Math.min(reviewItems.length, limit),
|
|
8565
|
+
candidateNoSelection: reviewItems.filter((item) => item.reason === "candidate-no-selection").length,
|
|
8566
|
+
emptyCandidateSet: reviewItems.filter((item) => item.reason === "empty-candidate-set").length,
|
|
8567
|
+
rewrittenNoSelection: reviewItems.filter((item) => item.reason === "rewritten-query-no-selection").length,
|
|
8568
|
+
lowSelectionRate: reviewItems.filter((item) => item.reason === "low-selection-rate").length
|
|
8569
|
+
},
|
|
8570
|
+
items: reviewItems.slice(0, limit)
|
|
8571
|
+
};
|
|
8572
|
+
}
|
|
8573
|
+
function parseStatsLimit(value, fallback, max) {
|
|
8574
|
+
if (!value)
|
|
8575
|
+
return fallback;
|
|
8576
|
+
if (!/^\d+$/.test(value))
|
|
8577
|
+
return fallback;
|
|
8578
|
+
const parsed = Number(value);
|
|
8579
|
+
if (!Number.isFinite(parsed) || parsed <= 0)
|
|
8580
|
+
return fallback;
|
|
8581
|
+
return Math.min(parsed, max);
|
|
8582
|
+
}
|
|
8583
|
+
function usefulnessScoreLabel(score, confidence) {
|
|
8584
|
+
if (confidence <= 0)
|
|
8585
|
+
return "unknown";
|
|
8586
|
+
if (score >= 80)
|
|
8587
|
+
return "excellent";
|
|
8588
|
+
if (score >= 60)
|
|
8589
|
+
return "good";
|
|
8590
|
+
if (score >= 40)
|
|
8591
|
+
return "watch";
|
|
8592
|
+
return "low";
|
|
8593
|
+
}
|
|
8594
|
+
function buildMemoryUsefulnessDiagnostics(input) {
|
|
8595
|
+
const { metrics, counts } = input;
|
|
8596
|
+
const diagnostics = [];
|
|
8597
|
+
if (counts.promptCount > 0 && counts.retrievalQueries === 0) {
|
|
8598
|
+
diagnostics.push({
|
|
8599
|
+
key: "no-retrieval-traces",
|
|
8600
|
+
severity: "warn",
|
|
8601
|
+
metric: "retrievalUsageRate",
|
|
8602
|
+
value: 0,
|
|
8603
|
+
target: 0.5,
|
|
8604
|
+
title: "No retrieval traces were recorded",
|
|
8605
|
+
detail: `${counts.promptCount} prompts were seen, but none produced a retrieval trace in this window.`,
|
|
8606
|
+
action: "Confirm the prompt hook is enabled and broaden adherence triggers for continuation, write-intent, and project-specific prompts."
|
|
8607
|
+
});
|
|
8608
|
+
}
|
|
8609
|
+
if (counts.promptCount > 0 && metrics.memoryHitRate < 0.5) {
|
|
8610
|
+
diagnostics.push({
|
|
8611
|
+
key: "low-memory-hit-rate",
|
|
8612
|
+
severity: "warn",
|
|
8613
|
+
metric: "memoryHitRate",
|
|
8614
|
+
value: metrics.memoryHitRate,
|
|
8615
|
+
target: 0.5,
|
|
8616
|
+
title: "Memory checks are missing many prompts",
|
|
8617
|
+
detail: `Only ${counts.memoryCheckedPrompts} of ${counts.promptCount} prompts had an adherence check in this window.`,
|
|
8618
|
+
action: "Broaden adherence triggers for continuation, write-intent, topic-shift, and project-specific prompts."
|
|
8619
|
+
});
|
|
8620
|
+
}
|
|
8621
|
+
if (counts.retrievalQueries > 0 && metrics.queryYieldRate < 0.6) {
|
|
8622
|
+
diagnostics.push({
|
|
8623
|
+
key: "low-query-yield-rate",
|
|
8624
|
+
severity: "warn",
|
|
8625
|
+
metric: "queryYieldRate",
|
|
8626
|
+
value: metrics.queryYieldRate,
|
|
8627
|
+
target: 0.6,
|
|
8628
|
+
title: "Searches often select no memory",
|
|
8629
|
+
detail: `${counts.queriesWithSelected} of ${counts.retrievalQueries} retrieval queries injected at least one memory.`,
|
|
8630
|
+
action: "Overfetch candidates, then filter/rerank before applying the final injection threshold."
|
|
8631
|
+
});
|
|
8632
|
+
}
|
|
8633
|
+
if (counts.totalEvaluated > 0 && metrics.avgHelpfulnessScore < 0.7) {
|
|
8634
|
+
diagnostics.push({
|
|
8635
|
+
key: "low-helpfulness-score",
|
|
8636
|
+
severity: "warn",
|
|
8637
|
+
metric: "avgHelpfulnessScore",
|
|
8638
|
+
value: metrics.avgHelpfulnessScore,
|
|
8639
|
+
target: 0.7,
|
|
8640
|
+
title: "Injected memories are not translating into outcomes",
|
|
8641
|
+
detail: `${counts.totalEvaluated} evaluated retrievals averaged ${(metrics.avgHelpfulnessScore * 100).toFixed(1)}% helpfulness.`,
|
|
8642
|
+
action: "Review low-scoring retrieval samples for stale decisions, cross-project noise, or raw transcript snippets."
|
|
8643
|
+
});
|
|
8644
|
+
}
|
|
8645
|
+
if (counts.totalRetrievals > 0 && metrics.evaluationCoverage < 0.8) {
|
|
8646
|
+
diagnostics.push({
|
|
8647
|
+
key: "low-evaluation-coverage",
|
|
8648
|
+
severity: "info",
|
|
8649
|
+
metric: "evaluationCoverage",
|
|
8650
|
+
value: metrics.evaluationCoverage,
|
|
8651
|
+
target: 0.8,
|
|
8652
|
+
title: "Many retrievals are still unevaluated",
|
|
8653
|
+
detail: `${counts.totalEvaluated} of ${counts.totalRetrievals} retrievals have measured helpfulness.`,
|
|
8654
|
+
action: "Ensure Stop/session-end hooks or pending-session backfill are running so usefulness reflects real outcomes."
|
|
8655
|
+
});
|
|
8656
|
+
}
|
|
8657
|
+
if (counts.candidateMemories > 0 && counts.selectedMemories === 0) {
|
|
8658
|
+
diagnostics.push({
|
|
8659
|
+
key: "candidates-without-selection",
|
|
8660
|
+
severity: "warn",
|
|
8661
|
+
metric: "selectionRate",
|
|
8662
|
+
value: metrics.selectionRate,
|
|
8663
|
+
target: 0.2,
|
|
8664
|
+
title: "Candidates are found but none are injected",
|
|
8665
|
+
detail: `${counts.candidateMemories} candidates were retrieved, but no memories passed the injection policy.`,
|
|
8666
|
+
action: "Inspect threshold settings and prompt-injection policy before lowering filters globally."
|
|
8667
|
+
});
|
|
8668
|
+
}
|
|
8669
|
+
return diagnostics.slice(0, 3);
|
|
8670
|
+
}
|
|
8671
|
+
function computeMemoryUsefulnessSummary(events, helpfulness, traces, now, window, limits = {}) {
|
|
8672
|
+
const windowEvents = events.filter((event) => inWindow(event, now, window));
|
|
8673
|
+
const prompts = windowEvents.filter((event) => event.eventType === "user_prompt");
|
|
8674
|
+
const promptCount = prompts.length;
|
|
8675
|
+
const memoryCheckedPrompts = prompts.filter((prompt) => prompt.metadata?.adherence?.checked).length;
|
|
8676
|
+
const windowMs = windowToMs(window);
|
|
8677
|
+
const windowStart = now - windowMs;
|
|
8678
|
+
const windowTraces = traces.filter((trace) => {
|
|
8679
|
+
const ts = getTimestampMs(trace.createdAt);
|
|
8680
|
+
return ts > 0 && ts >= windowStart;
|
|
8681
|
+
});
|
|
8682
|
+
const oldestEventTimestamp = events.reduce((oldest, event) => {
|
|
8683
|
+
const timestamp = event.timestamp?.getTime?.() || 0;
|
|
8684
|
+
return timestamp > 0 ? Math.min(oldest, timestamp) : oldest;
|
|
8685
|
+
}, Number.POSITIVE_INFINITY);
|
|
8686
|
+
const oldestTraceTimestamp = traces.reduce((oldest, trace) => {
|
|
8687
|
+
const timestamp = getTimestampMs(trace.createdAt);
|
|
8688
|
+
return timestamp > 0 ? Math.min(oldest, timestamp) : oldest;
|
|
8689
|
+
}, Number.POSITIVE_INFINITY);
|
|
8690
|
+
const eventWindowTruncated = Boolean(
|
|
8691
|
+
limits.eventsLimit && events.length >= limits.eventsLimit && Number.isFinite(oldestEventTimestamp) && oldestEventTimestamp >= windowStart
|
|
8692
|
+
);
|
|
8693
|
+
const traceWindowTruncated = Boolean(
|
|
8694
|
+
limits.tracesLimit && traces.length >= limits.tracesLimit && Number.isFinite(oldestTraceTimestamp) && oldestTraceTimestamp >= windowStart
|
|
8695
|
+
);
|
|
8696
|
+
const retrievalQueries = windowTraces.length;
|
|
8697
|
+
const candidateCounts = windowTraces.map((trace) => Number(trace.candidateCount ?? trace.candidateEventIds?.length ?? 0));
|
|
8698
|
+
const selectedCounts = windowTraces.map((trace) => getTraceSelectedCount(trace));
|
|
8699
|
+
const totalCandidateCount = candidateCounts.reduce((sum, count) => sum + (Number.isFinite(count) ? count : 0), 0);
|
|
8700
|
+
const totalSelectedCount = selectedCounts.reduce((sum, count) => sum + (Number.isFinite(count) ? count : 0), 0);
|
|
8701
|
+
const queriesWithSelected = selectedCounts.filter((count) => Number.isFinite(count) && count > 0).length;
|
|
8702
|
+
const rewrittenTraces = windowTraces.filter(isRewrittenRetrievalTrace);
|
|
8703
|
+
const rawTraces = windowTraces.filter((trace) => !isRewrittenRetrievalTrace(trace));
|
|
8704
|
+
const rewrittenQueries = rewrittenTraces.length;
|
|
8705
|
+
const rawQueries = rawTraces.length;
|
|
8706
|
+
const rewrittenSelectedCount = rewrittenTraces.reduce((sum, trace) => {
|
|
8707
|
+
const selectedCount = getTraceSelectedCount(trace);
|
|
8708
|
+
return sum + (Number.isFinite(selectedCount) ? selectedCount : 0);
|
|
8709
|
+
}, 0);
|
|
8710
|
+
const rawSelectedCount = rawTraces.reduce((sum, trace) => {
|
|
8711
|
+
const selectedCount = getTraceSelectedCount(trace);
|
|
8712
|
+
return sum + (Number.isFinite(selectedCount) ? selectedCount : 0);
|
|
8713
|
+
}, 0);
|
|
8714
|
+
const rewrittenQueriesWithSelected = rewrittenTraces.filter((trace) => getTraceSelectedCount(trace) > 0).length;
|
|
8715
|
+
const rawQueriesWithSelected = rawTraces.filter((trace) => getTraceSelectedCount(trace) > 0).length;
|
|
8716
|
+
const totalEvaluated = Number(helpfulness.totalEvaluated || 0);
|
|
8717
|
+
const totalRetrievals = Number(helpfulness.totalRetrievals || 0);
|
|
8718
|
+
const helpful = Number(helpfulness.helpful || 0);
|
|
8719
|
+
const neutral = Number(helpfulness.neutral || 0);
|
|
8720
|
+
const unhelpful = Number(helpfulness.unhelpful || 0);
|
|
8721
|
+
const retrievalsPerPrompt = safeRatio(retrievalQueries, promptCount);
|
|
8722
|
+
const metrics = {
|
|
8723
|
+
avgHelpfulnessScore: round(normalizeMetric(helpfulness.avgScore)),
|
|
8724
|
+
usefulRecallRate: round(safeRatio(helpful, totalEvaluated)),
|
|
8725
|
+
memoryHitRate: round(safeRatio(memoryCheckedPrompts, promptCount)),
|
|
8726
|
+
retrievalUsageRate: round(Math.min(1, retrievalsPerPrompt)),
|
|
8727
|
+
queryYieldRate: round(safeRatio(queriesWithSelected, retrievalQueries)),
|
|
8728
|
+
evaluationCoverage: round(safeRatio(totalEvaluated, totalRetrievals)),
|
|
8729
|
+
retrievalsPerPrompt: round(retrievalsPerPrompt),
|
|
8730
|
+
avgCandidatesPerQuery: round(safeRatio(totalCandidateCount, retrievalQueries), 2),
|
|
8731
|
+
avgSelectedPerQuery: round(safeRatio(totalSelectedCount, retrievalQueries), 2),
|
|
8732
|
+
selectionRate: round(safeRatio(totalSelectedCount, totalCandidateCount)),
|
|
8733
|
+
queryRewriteRate: round(safeRatio(rewrittenQueries, retrievalQueries)),
|
|
8734
|
+
rewrittenQueryYieldRate: round(safeRatio(rewrittenQueriesWithSelected, rewrittenQueries)),
|
|
8735
|
+
rawQueryYieldRate: round(safeRatio(rawQueriesWithSelected, rawQueries)),
|
|
8736
|
+
avgSelectedPerRewrittenQuery: round(safeRatio(rewrittenSelectedCount, rewrittenQueries), 2),
|
|
8737
|
+
avgSelectedPerRawQuery: round(safeRatio(rawSelectedCount, rawQueries), 2)
|
|
8738
|
+
};
|
|
8739
|
+
const counts = {
|
|
8740
|
+
promptCount,
|
|
8741
|
+
memoryCheckedPrompts,
|
|
8742
|
+
retrievalQueries,
|
|
8743
|
+
queriesWithSelected,
|
|
8744
|
+
rewrittenQueries,
|
|
8745
|
+
rawQueries,
|
|
8746
|
+
rewrittenQueriesWithSelected,
|
|
8747
|
+
rawQueriesWithSelected,
|
|
8748
|
+
selectedMemories: totalSelectedCount,
|
|
8749
|
+
candidateMemories: totalCandidateCount,
|
|
8750
|
+
totalEvaluated,
|
|
8751
|
+
totalRetrievals,
|
|
8752
|
+
helpful,
|
|
8753
|
+
neutral,
|
|
8754
|
+
unhelpful
|
|
8755
|
+
};
|
|
8756
|
+
const componentSpecs = [
|
|
8757
|
+
{ key: "avgHelpfulnessScore", label: "Average helpfulness score", value: metrics.avgHelpfulnessScore, weight: 0.3, available: totalEvaluated > 0 },
|
|
8758
|
+
{ key: "usefulRecallRate", label: "Useful recall rate", value: metrics.usefulRecallRate, weight: 0.25, available: totalEvaluated > 0 },
|
|
8759
|
+
{ key: "memoryHitRate", label: "Memory hit rate", value: metrics.memoryHitRate, weight: 0.2, available: promptCount > 0 },
|
|
8760
|
+
{ key: "retrievalUsageRate", label: "Retrieval usage rate", value: metrics.retrievalUsageRate, weight: 0.15, available: promptCount > 0 },
|
|
8761
|
+
{ key: "queryYieldRate", label: "Query yield rate", value: metrics.queryYieldRate, weight: 0.1, available: retrievalQueries > 0 }
|
|
8762
|
+
];
|
|
8763
|
+
const totalWeight = componentSpecs.reduce((sum, component) => sum + component.weight, 0);
|
|
8764
|
+
const availableWeight = componentSpecs.filter((component) => component.available).reduce((sum, component) => sum + component.weight, 0);
|
|
8765
|
+
const weightedScore = availableWeight > 0 ? componentSpecs.reduce((sum, component) => sum + (component.available ? component.value * component.weight : 0), 0) / availableWeight : 0;
|
|
8766
|
+
const scoreValue = round(weightedScore * 100, 1);
|
|
8767
|
+
const confidence = round(safeRatio(availableWeight, totalWeight), 2);
|
|
8768
|
+
const components = componentSpecs.map((component) => ({
|
|
8769
|
+
...component,
|
|
8770
|
+
contribution: component.available ? round(component.value * component.weight * 100, 2) : 0
|
|
8771
|
+
}));
|
|
8772
|
+
return {
|
|
8773
|
+
window,
|
|
8774
|
+
score: {
|
|
8775
|
+
value: scoreValue,
|
|
8776
|
+
label: usefulnessScoreLabel(scoreValue, confidence),
|
|
8777
|
+
confidence
|
|
8778
|
+
},
|
|
8779
|
+
metrics,
|
|
8780
|
+
counts,
|
|
8781
|
+
components,
|
|
8782
|
+
diagnostics: buildMemoryUsefulnessDiagnostics({ metrics, counts }),
|
|
8783
|
+
limits: {
|
|
8784
|
+
eventsLimit: limits.eventsLimit || events.length,
|
|
8785
|
+
tracesLimit: limits.tracesLimit || traces.length,
|
|
8786
|
+
eventWindowTruncated,
|
|
8787
|
+
traceWindowTruncated
|
|
8788
|
+
},
|
|
8789
|
+
generatedAt: new Date(now).toISOString()
|
|
8790
|
+
};
|
|
8791
|
+
}
|
|
8063
8792
|
function computeKpiMetrics(events, usefulRecallRate) {
|
|
8064
8793
|
const prompts = events.filter((e) => e.eventType === "user_prompt");
|
|
8065
8794
|
const promptCount = prompts.length;
|
|
@@ -8386,6 +9115,32 @@ statsRouter.get("/helpfulness", async (c) => {
|
|
|
8386
9115
|
await memoryService.shutdown();
|
|
8387
9116
|
}
|
|
8388
9117
|
});
|
|
9118
|
+
statsRouter.get("/usefulness", async (c) => {
|
|
9119
|
+
const rawWindow = c.req.query("window") || "7d";
|
|
9120
|
+
const window = rawWindow === "24h" || rawWindow === "30d" ? rawWindow : "7d";
|
|
9121
|
+
const memoryService = getLightweightServiceFromQuery(c);
|
|
9122
|
+
try {
|
|
9123
|
+
await memoryService.initialize();
|
|
9124
|
+
const now = Date.now();
|
|
9125
|
+
const eventLimit = 2e4;
|
|
9126
|
+
const traceLimit = 5e3;
|
|
9127
|
+
const windowStart = new Date(now - windowToMs(window));
|
|
9128
|
+
const [events, helpfulness, traces] = await Promise.all([
|
|
9129
|
+
memoryService.getRecentEvents(eventLimit),
|
|
9130
|
+
memoryService.getHelpfulnessStats(windowStart),
|
|
9131
|
+
memoryService.getRecentRetrievalTraces(traceLimit)
|
|
9132
|
+
]);
|
|
9133
|
+
return c.json(computeMemoryUsefulnessSummary(events, helpfulness, traces, now, window, {
|
|
9134
|
+
eventsLimit: eventLimit,
|
|
9135
|
+
tracesLimit: traceLimit
|
|
9136
|
+
}));
|
|
9137
|
+
} catch (error) {
|
|
9138
|
+
console.error("[stats/usefulness] failed to calculate dashboard metrics", error);
|
|
9139
|
+
return c.json({ error: "Unable to calculate memory usefulness statistics" }, 500);
|
|
9140
|
+
} finally {
|
|
9141
|
+
await memoryService.shutdown();
|
|
9142
|
+
}
|
|
9143
|
+
});
|
|
8389
9144
|
statsRouter.get("/retrieval-traces", async (c) => {
|
|
8390
9145
|
const limit = parseInt(c.req.query("limit") || "50", 10);
|
|
8391
9146
|
const memoryService = getServiceFromQuery(c);
|
|
@@ -8395,26 +9150,43 @@ statsRouter.get("/retrieval-traces", async (c) => {
|
|
|
8395
9150
|
const traceStats = await memoryService.getRetrievalTraceStats();
|
|
8396
9151
|
return c.json({
|
|
8397
9152
|
stats: traceStats,
|
|
8398
|
-
traces: traces.map((t) =>
|
|
8399
|
-
|
|
8400
|
-
|
|
8401
|
-
|
|
8402
|
-
|
|
8403
|
-
|
|
8404
|
-
|
|
8405
|
-
|
|
8406
|
-
|
|
8407
|
-
|
|
8408
|
-
|
|
8409
|
-
|
|
8410
|
-
|
|
8411
|
-
|
|
8412
|
-
|
|
8413
|
-
|
|
9153
|
+
traces: traces.map((t) => {
|
|
9154
|
+
const queryRewriteKind = normalizeQueryRewriteKind2(t.queryRewriteKind);
|
|
9155
|
+
return {
|
|
9156
|
+
traceId: t.traceId,
|
|
9157
|
+
sessionId: t.sessionId || null,
|
|
9158
|
+
projectHash: t.projectHash || null,
|
|
9159
|
+
queryRewriteKind,
|
|
9160
|
+
rewritten: queryRewriteKind !== "none",
|
|
9161
|
+
strategy: t.strategy || null,
|
|
9162
|
+
candidateEventIds: t.candidateEventIds,
|
|
9163
|
+
selectedEventIds: t.selectedEventIds,
|
|
9164
|
+
candidateDetails: t.candidateDetails || [],
|
|
9165
|
+
selectedDetails: t.selectedDetails || [],
|
|
9166
|
+
candidateCount: t.candidateCount,
|
|
9167
|
+
selectedCount: t.selectedCount,
|
|
9168
|
+
confidence: t.confidence || null,
|
|
9169
|
+
fallbackTrace: t.fallbackTrace,
|
|
9170
|
+
createdAt: t.createdAt.toISOString()
|
|
9171
|
+
};
|
|
9172
|
+
})
|
|
8414
9173
|
});
|
|
8415
9174
|
} catch (error) {
|
|
8416
9175
|
return c.json({
|
|
8417
|
-
stats: {
|
|
9176
|
+
stats: {
|
|
9177
|
+
totalQueries: 0,
|
|
9178
|
+
avgCandidateCount: 0,
|
|
9179
|
+
avgSelectedCount: 0,
|
|
9180
|
+
selectionRate: 0,
|
|
9181
|
+
rewrittenQueries: 0,
|
|
9182
|
+
rewriteRate: 0,
|
|
9183
|
+
rewrittenQueriesWithSelection: 0,
|
|
9184
|
+
rawQueriesWithSelection: 0,
|
|
9185
|
+
rewrittenSelectionRate: 0,
|
|
9186
|
+
rawSelectionRate: 0,
|
|
9187
|
+
avgSelectedCountForRewrittenQueries: 0,
|
|
9188
|
+
avgSelectedCountForRawQueries: 0
|
|
9189
|
+
},
|
|
8418
9190
|
traces: [],
|
|
8419
9191
|
error: error.message
|
|
8420
9192
|
}, 500);
|
|
@@ -8422,6 +9194,40 @@ statsRouter.get("/retrieval-traces", async (c) => {
|
|
|
8422
9194
|
await memoryService.shutdown();
|
|
8423
9195
|
}
|
|
8424
9196
|
});
|
|
9197
|
+
statsRouter.get("/retrieval-review-queue", async (c) => {
|
|
9198
|
+
const limit = parseStatsLimit(c.req.query("limit"), 10, 50);
|
|
9199
|
+
const scanLimit = parseStatsLimit(c.req.query("scanLimit"), 500, 5e3);
|
|
9200
|
+
const memoryService = getServiceFromQuery(c);
|
|
9201
|
+
try {
|
|
9202
|
+
await memoryService.initialize();
|
|
9203
|
+
const traces = await memoryService.getRecentRetrievalTraces(scanLimit);
|
|
9204
|
+
return c.json({
|
|
9205
|
+
...buildRetrievalReviewQueue(traces, limit),
|
|
9206
|
+
limits: {
|
|
9207
|
+
requestedLimit: limit,
|
|
9208
|
+
scanLimit,
|
|
9209
|
+
scannedTraces: traces.length
|
|
9210
|
+
}
|
|
9211
|
+
});
|
|
9212
|
+
} catch (error) {
|
|
9213
|
+
console.error("Failed to build retrieval review queue");
|
|
9214
|
+
return c.json({
|
|
9215
|
+
summary: {
|
|
9216
|
+
totalTraces: 0,
|
|
9217
|
+
reviewItems: 0,
|
|
9218
|
+
returnedItems: 0,
|
|
9219
|
+
candidateNoSelection: 0,
|
|
9220
|
+
emptyCandidateSet: 0,
|
|
9221
|
+
rewrittenNoSelection: 0,
|
|
9222
|
+
lowSelectionRate: 0
|
|
9223
|
+
},
|
|
9224
|
+
items: [],
|
|
9225
|
+
error: "Unable to build retrieval review queue"
|
|
9226
|
+
}, 500);
|
|
9227
|
+
} finally {
|
|
9228
|
+
await memoryService.shutdown();
|
|
9229
|
+
}
|
|
9230
|
+
});
|
|
8425
9231
|
statsRouter.get("/kpi", async (c) => {
|
|
8426
9232
|
const rawWindow = c.req.query("window") || "7d";
|
|
8427
9233
|
const window = rawWindow === "24h" || rawWindow === "30d" ? rawWindow : "7d";
|