@vivantel/virage-core 0.2.23 → 0.2.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/eval/ecosystem-eval.d.ts +41 -0
- package/dist/eval/ecosystem-eval.d.ts.map +1 -0
- package/dist/eval/ecosystem-eval.js +73 -0
- package/dist/eval/ecosystem-eval.js.map +1 -0
- package/dist/eval/skill-routing-eval.d.ts +35 -0
- package/dist/eval/skill-routing-eval.d.ts.map +1 -0
- package/dist/eval/skill-routing-eval.js +76 -0
- package/dist/eval/skill-routing-eval.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/interfaces/vector-store.d.ts +10 -1
- package/dist/interfaces/vector-store.d.ts.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified ecosystem evaluator: combines retrieval metrics (precision@K, recall@K, MRR),
|
|
3
|
+
* RAGAS quality metrics (faithfulness, answerRelevance, contextRecall), and
|
|
4
|
+
* skill-routing accuracy (suggest_skill keyword matching + hook detection).
|
|
5
|
+
*/
|
|
6
|
+
import type { EmbeddingProvider, VectorStore } from "../interfaces/index.js";
|
|
7
|
+
import type { EvalDataset, RAGASResult, LLMJudge } from "../interfaces/quality.js";
|
|
8
|
+
import type { EvalResult } from "../interfaces/quality.js";
|
|
9
|
+
import { type SkillRoutingQuery, type SkillRoutingEvalResult } from "./skill-routing-eval.js";
|
|
10
|
+
export interface EcosystemEvalDataset {
|
|
11
|
+
/** Retrieval evaluation: (query → relevant chunk IDs) */
|
|
12
|
+
retrieval: EvalDataset;
|
|
13
|
+
/** RAGAS evaluation: (query + ground truth) — requires LLM judge */
|
|
14
|
+
ragas?: EvalDataset;
|
|
15
|
+
/** Skill routing evaluation: (query → expected skill + hook flag) */
|
|
16
|
+
skillRouting: SkillRoutingQuery[];
|
|
17
|
+
}
|
|
18
|
+
export interface EcosystemEvalResult {
|
|
19
|
+
retrieval: EvalResult;
|
|
20
|
+
ragas: RAGASResult | null;
|
|
21
|
+
skillRouting: SkillRoutingEvalResult;
|
|
22
|
+
timestamp: string;
|
|
23
|
+
configSnapshot: Record<string, unknown>;
|
|
24
|
+
}
|
|
25
|
+
interface SkillMetaForEval {
|
|
26
|
+
name: string;
|
|
27
|
+
when_to_use: string[];
|
|
28
|
+
estimated_tokens: number;
|
|
29
|
+
}
|
|
30
|
+
export declare class EcosystemEvaluator {
|
|
31
|
+
private readonly store;
|
|
32
|
+
private readonly embedder;
|
|
33
|
+
private readonly skills;
|
|
34
|
+
private readonly judge?;
|
|
35
|
+
private readonly topK;
|
|
36
|
+
constructor(store: VectorStore, embedder: EmbeddingProvider, skills: SkillMetaForEval[], judge?: LLMJudge | undefined, topK?: number);
|
|
37
|
+
run(dataset: EcosystemEvalDataset, configSnapshot?: Record<string, unknown>): Promise<EcosystemEvalResult>;
|
|
38
|
+
}
|
|
39
|
+
export declare function printEcosystemEvalResult(result: EcosystemEvalResult): void;
|
|
40
|
+
export {};
|
|
41
|
+
//# sourceMappingURL=ecosystem-eval.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ecosystem-eval.d.ts","sourceRoot":"","sources":["../../src/eval/ecosystem-eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,KAAK,EACV,WAAW,EACX,WAAW,EACX,QAAQ,EACT,MAAM,0BAA0B,CAAC;AAClC,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAG3D,OAAO,EAEL,KAAK,iBAAiB,EACtB,KAAK,sBAAsB,EAC5B,MAAM,yBAAyB,CAAC;AAEjC,MAAM,WAAW,oBAAoB;IACnC,yDAAyD;IACzD,SAAS,EAAE,WAAW,CAAC;IACvB,oEAAoE;IACpE,KAAK,CAAC,EAAE,WAAW,CAAC;IACpB,qEAAqE;IACrE,YAAY,EAAE,iBAAiB,EAAE,CAAC;CACnC;AAED,MAAM,WAAW,mBAAmB;IAClC,SAAS,EAAE,UAAU,CAAC;IACtB,KAAK,EAAE,WAAW,GAAG,IAAI,CAAC;IAC1B,YAAY,EAAE,sBAAsB,CAAC;IACrC,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACzC;AAED,UAAU,gBAAgB;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AAED,qBAAa,kBAAkB;IAE3B,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAJJ,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,iBAAiB,EAC3B,MAAM,EAAE,gBAAgB,EAAE,EAC1B,KAAK,CAAC,EAAE,QAAQ,YAAA,EAChB,IAAI,SAAK;IAGtB,GAAG,CACP,OAAO,EAAE,oBAAoB,EAC7B,cAAc,GAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAM,GAC3C,OAAO,CAAC,mBAAmB,CAAC;CAgChC;AAED,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,mBAAmB,GAAG,IAAI,CAmC1E"}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Unified ecosystem evaluator: combines retrieval metrics (precision@K, recall@K, MRR),
|
|
3
|
+
* RAGAS quality metrics (faithfulness, answerRelevance, contextRecall), and
|
|
4
|
+
* skill-routing accuracy (suggest_skill keyword matching + hook detection).
|
|
5
|
+
*/
|
|
6
|
+
import { EvalRunner } from "./runner.js";
|
|
7
|
+
import { RAGASRunner } from "./ragas.js";
|
|
8
|
+
import { SkillRoutingEvaluator, } from "./skill-routing-eval.js";
|
|
9
|
+
export class EcosystemEvaluator {
|
|
10
|
+
store;
|
|
11
|
+
embedder;
|
|
12
|
+
skills;
|
|
13
|
+
judge;
|
|
14
|
+
topK;
|
|
15
|
+
constructor(store, embedder, skills, judge, topK = 10) {
|
|
16
|
+
this.store = store;
|
|
17
|
+
this.embedder = embedder;
|
|
18
|
+
this.skills = skills;
|
|
19
|
+
this.judge = judge;
|
|
20
|
+
this.topK = topK;
|
|
21
|
+
}
|
|
22
|
+
async run(dataset, configSnapshot = {}) {
|
|
23
|
+
const retrievalRunner = new EvalRunner(this.store, this.embedder, dataset.retrieval, this.topK);
|
|
24
|
+
const { evalResult: retrieval } = await retrievalRunner.run();
|
|
25
|
+
let ragas = null;
|
|
26
|
+
if (this.judge && dataset.ragas) {
|
|
27
|
+
const ragasRunner = new RAGASRunner(this.judge, this.store, this.embedder, dataset.ragas, this.topK);
|
|
28
|
+
ragas = await ragasRunner.run();
|
|
29
|
+
}
|
|
30
|
+
const routingEval = new SkillRoutingEvaluator(this.skills);
|
|
31
|
+
const skillRouting = routingEval.evaluate(dataset.skillRouting);
|
|
32
|
+
return {
|
|
33
|
+
retrieval,
|
|
34
|
+
ragas,
|
|
35
|
+
skillRouting,
|
|
36
|
+
timestamp: new Date().toISOString(),
|
|
37
|
+
configSnapshot,
|
|
38
|
+
};
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
export function printEcosystemEvalResult(result) {
|
|
42
|
+
const pct = (n) => `${(n * 100).toFixed(1)}%`;
|
|
43
|
+
console.log("\n📊 Ecosystem Evaluation Results");
|
|
44
|
+
console.log("═".repeat(50));
|
|
45
|
+
console.log("\n🔍 Retrieval Metrics");
|
|
46
|
+
console.log("─".repeat(40));
|
|
47
|
+
console.log(` Queries evaluated : ${result.retrieval.queriesEvaluated}`);
|
|
48
|
+
console.log(` Precision@5 : ${pct(result.retrieval.precisionAt5)}`);
|
|
49
|
+
console.log(` Precision@10 : ${pct(result.retrieval.precisionAt10)}`);
|
|
50
|
+
console.log(` Recall@10 : ${pct(result.retrieval.recallAt10)}`);
|
|
51
|
+
console.log(` MRR : ${result.retrieval.mrr.toFixed(4)}`);
|
|
52
|
+
console.log(` HitRate@5 : ${pct(result.retrieval.hitRateAt5)}`);
|
|
53
|
+
if (result.ragas) {
|
|
54
|
+
console.log("\n⚖️ RAGAS Quality Metrics");
|
|
55
|
+
console.log("─".repeat(40));
|
|
56
|
+
console.log(` Faithfulness : ${pct(result.ragas.faithfulness)}`);
|
|
57
|
+
console.log(` Answer Relevance : ${pct(result.ragas.answerRelevance)}`);
|
|
58
|
+
console.log(` Context Recall : ${pct(result.ragas.contextRecall)}`);
|
|
59
|
+
}
|
|
60
|
+
else {
|
|
61
|
+
console.log("\n⚖️ RAGAS: skipped (no LLM judge configured)");
|
|
62
|
+
}
|
|
63
|
+
const r = result.skillRouting;
|
|
64
|
+
console.log("\n🧭 Skill Routing Metrics");
|
|
65
|
+
console.log("─".repeat(40));
|
|
66
|
+
console.log(` Queries evaluated : ${r.totalQueries}`);
|
|
67
|
+
console.log(` Routing accuracy : ${pct(r.accuracy)}`);
|
|
68
|
+
console.log(` Hook TPR : ${pct(r.hookTruePositiveRate)}`);
|
|
69
|
+
console.log(` Hook FPR : ${pct(r.hookFalsePositiveRate)}`);
|
|
70
|
+
console.log(` Avg tokens saved : ${Math.round(r.avgTokensSaved)} tok`);
|
|
71
|
+
console.log("\n═".repeat(50));
|
|
72
|
+
}
|
|
73
|
+
//# sourceMappingURL=ecosystem-eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ecosystem-eval.js","sourceRoot":"","sources":["../../src/eval/ecosystem-eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AASH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AACzC,OAAO,EACL,qBAAqB,GAGtB,MAAM,yBAAyB,CAAC;AAyBjC,MAAM,OAAO,kBAAkB;IAEV;IACA;IACA;IACA;IACA;IALnB,YACmB,KAAkB,EAClB,QAA2B,EAC3B,MAA0B,EAC1B,KAAgB,EAChB,OAAO,EAAE;QAJT,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAmB;QAC3B,WAAM,GAAN,MAAM,CAAoB;QAC1B,UAAK,GAAL,KAAK,CAAW;QAChB,SAAI,GAAJ,IAAI,CAAK;IACzB,CAAC;IAEJ,KAAK,CAAC,GAAG,CACP,OAA6B,EAC7B,iBAA0C,EAAE;QAE5C,MAAM,eAAe,GAAG,IAAI,UAAU,CACpC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,CAAC,SAAS,EACjB,IAAI,CAAC,IAAI,CACV,CAAC;QACF,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,MAAM,eAAe,CAAC,GAAG,EAAE,CAAC;QAE9D,IAAI,KAAK,GAAuB,IAAI,CAAC;QACrC,IAAI,IAAI,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;YAChC,MAAM,WAAW,GAAG,IAAI,WAAW,CACjC,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,KAAK,EACV,IAAI,CAAC,QAAQ,EACb,OAAO,CAAC,KAAK,EACb,IAAI,CAAC,IAAI,CACV,CAAC;YACF,KAAK,GAAG,MAAM,WAAW,CAAC,GAAG,EAAE,CAAC;QAClC,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,qBAAqB,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3D,MAAM,YAAY,GAAG,WAAW,CAAC,QAAQ,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAEhE,OAAO;YACL,SAAS;YACT,KAAK;YACL,YAAY;YACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,cAAc;SACf,CAAC;IACJ,CAAC;CACF;AAED,MAAM,UAAU,wBAAwB,CAAC,MAA2B;IAClE,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;IAEtD,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,wBAAwB,CAAC,CAAC;IACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAC1E,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC3E,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxE,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IAEzE,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QACjB,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;QAC3C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC1E,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,CAAC,GAAG,MAAM,CAAC,YAAY,CAAC;IAC9B,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC;IACvD,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC,oBAAoB,CAAC,EAAE,CAAC,CAAC;IACpE,OAAO,CAAC,GAAG,CAAC,yBAAyB,GAAG,CAAC,CAAC,CAAC,qBAAqB,CAAC,EAAE,CAAC,CAAC;IACrE,OAAO,CAAC,GAAG,CAAC,yBAAyB,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;IAEzE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAChC,CAAC"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluates skill routing accuracy for the suggest_skill keyword-matching logic.
|
|
3
|
+
* Tests both the MCP tool's keyword scoring and the UserPromptSubmit hook's
|
|
4
|
+
* grep-pattern detection, using a labeled dataset of (query → expected_skill) pairs.
|
|
5
|
+
*/
|
|
6
|
+
export interface SkillRoutingQuery {
|
|
7
|
+
query: string;
|
|
8
|
+
expectedSkill: string;
|
|
9
|
+
/** Whether the UserPromptSubmit hook should fire for this query */
|
|
10
|
+
shouldTriggerHook: boolean;
|
|
11
|
+
}
|
|
12
|
+
export interface SkillRoutingEvalResult {
|
|
13
|
+
totalQueries: number;
|
|
14
|
+
correctSkillSelected: number;
|
|
15
|
+
accuracy: number;
|
|
16
|
+
hookTruePositives: number;
|
|
17
|
+
hookFalsePositives: number;
|
|
18
|
+
hookTruePositiveRate: number;
|
|
19
|
+
hookFalsePositiveRate: number;
|
|
20
|
+
/** Average tokens saved by using summary path vs full skill load */
|
|
21
|
+
avgTokensSaved: number;
|
|
22
|
+
}
|
|
23
|
+
interface SkillMeta {
|
|
24
|
+
name: string;
|
|
25
|
+
when_to_use: string[];
|
|
26
|
+
estimated_tokens: number;
|
|
27
|
+
}
|
|
28
|
+
export declare class SkillRoutingEvaluator {
|
|
29
|
+
private readonly skills;
|
|
30
|
+
private readonly summaryTokens;
|
|
31
|
+
constructor(skills: SkillMeta[], summaryTokens?: number);
|
|
32
|
+
evaluate(queries: SkillRoutingQuery[]): SkillRoutingEvalResult;
|
|
33
|
+
}
|
|
34
|
+
export {};
|
|
35
|
+
//# sourceMappingURL=skill-routing-eval.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skill-routing-eval.d.ts","sourceRoot":"","sources":["../../src/eval/skill-routing-eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,iBAAiB,EAAE,OAAO,CAAC;CAC5B;AAED,MAAM,WAAW,sBAAsB;IACrC,YAAY,EAAE,MAAM,CAAC;IACrB,oBAAoB,EAAE,MAAM,CAAC;IAC7B,QAAQ,EAAE,MAAM,CAAC;IACjB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,kBAAkB,EAAE,MAAM,CAAC;IAC3B,oBAAoB,EAAE,MAAM,CAAC;IAC7B,qBAAqB,EAAE,MAAM,CAAC;IAC9B,oEAAoE;IACpE,cAAc,EAAE,MAAM,CAAC;CACxB;AAED,UAAU,SAAS;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,MAAM,EAAE,CAAC;IACtB,gBAAgB,EAAE,MAAM,CAAC;CAC1B;AA6BD,qBAAa,qBAAqB;IAE9B,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,aAAa;gBADb,MAAM,EAAE,SAAS,EAAE,EACnB,aAAa,SAAM;IAGtC,QAAQ,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,sBAAsB;CA8C/D"}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Evaluates skill routing accuracy for the suggest_skill keyword-matching logic.
|
|
3
|
+
* Tests both the MCP tool's keyword scoring and the UserPromptSubmit hook's
|
|
4
|
+
* grep-pattern detection, using a labeled dataset of (query → expected_skill) pairs.
|
|
5
|
+
*/
|
|
6
|
+
/** Mirrors the keyword-scoring logic in virage-agent-claude/src/server.ts */
|
|
7
|
+
function scoreSkill(task, skill) {
|
|
8
|
+
const taskLower = task.toLowerCase();
|
|
9
|
+
const haystack = [skill.name, ...skill.when_to_use].join(" ").toLowerCase();
|
|
10
|
+
const words = taskLower.split(/\s+/).filter((w) => w.length > 2);
|
|
11
|
+
const hits = words.filter((w) => haystack.includes(w)).length;
|
|
12
|
+
const nameBonus = taskLower.includes(skill.name.replace("-", " ")) ||
|
|
13
|
+
taskLower.includes(skill.name)
|
|
14
|
+
? 2
|
|
15
|
+
: 0;
|
|
16
|
+
return hits + nameBonus;
|
|
17
|
+
}
|
|
18
|
+
/** Mirrors the grep patterns in the UserPromptSubmit hook */
|
|
19
|
+
function hookWouldFire(query) {
|
|
20
|
+
const q = query.toLowerCase();
|
|
21
|
+
return (/\b(plan|break.?down|roadmap|sequence|implement.?steps)\b/.test(q) ||
|
|
22
|
+
/\b(adr|architect|interface.?design|system.?design|refactor.?scope)\b/.test(q) ||
|
|
23
|
+
/\b(docs?|readme|changelog|document|write.?up)\b/.test(q) ||
|
|
24
|
+
/\b(review|security|vulnerabilit|audit)\b/.test(q));
|
|
25
|
+
}
|
|
26
|
+
export class SkillRoutingEvaluator {
|
|
27
|
+
skills;
|
|
28
|
+
summaryTokens;
|
|
29
|
+
constructor(skills, summaryTokens = 150) {
|
|
30
|
+
this.skills = skills;
|
|
31
|
+
this.summaryTokens = summaryTokens;
|
|
32
|
+
}
|
|
33
|
+
evaluate(queries) {
|
|
34
|
+
let correct = 0;
|
|
35
|
+
let hookTP = 0;
|
|
36
|
+
let hookFP = 0;
|
|
37
|
+
let totalTokensSaved = 0;
|
|
38
|
+
let hookShouldFireCount = 0;
|
|
39
|
+
let hookShouldNotFireCount = 0;
|
|
40
|
+
for (const { query, expectedSkill, shouldTriggerHook } of queries) {
|
|
41
|
+
const scored = this.skills
|
|
42
|
+
.map((s) => ({ name: s.name, score: scoreSkill(query, s) }))
|
|
43
|
+
.sort((a, b) => b.score - a.score);
|
|
44
|
+
const top = scored[0];
|
|
45
|
+
if (top && top.score > 0 && top.name === expectedSkill) {
|
|
46
|
+
correct++;
|
|
47
|
+
const fullTokens = this.skills.find((s) => s.name === expectedSkill)?.estimated_tokens ??
|
|
48
|
+
0;
|
|
49
|
+
totalTokensSaved += Math.max(0, fullTokens - this.summaryTokens);
|
|
50
|
+
}
|
|
51
|
+
const fired = hookWouldFire(query);
|
|
52
|
+
if (shouldTriggerHook) {
|
|
53
|
+
hookShouldFireCount++;
|
|
54
|
+
if (fired)
|
|
55
|
+
hookTP++;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
hookShouldNotFireCount++;
|
|
59
|
+
if (fired)
|
|
60
|
+
hookFP++;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
const total = queries.length;
|
|
64
|
+
return {
|
|
65
|
+
totalQueries: total,
|
|
66
|
+
correctSkillSelected: correct,
|
|
67
|
+
accuracy: total > 0 ? correct / total : 0,
|
|
68
|
+
hookTruePositives: hookTP,
|
|
69
|
+
hookFalsePositives: hookFP,
|
|
70
|
+
hookTruePositiveRate: hookShouldFireCount > 0 ? hookTP / hookShouldFireCount : 0,
|
|
71
|
+
hookFalsePositiveRate: hookShouldNotFireCount > 0 ? hookFP / hookShouldNotFireCount : 0,
|
|
72
|
+
avgTokensSaved: correct > 0 ? totalTokensSaved / correct : 0,
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
//# sourceMappingURL=skill-routing-eval.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"skill-routing-eval.js","sourceRoot":"","sources":["../../src/eval/skill-routing-eval.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AA2BH,6EAA6E;AAC7E,SAAS,UAAU,CAAC,IAAY,EAAE,KAAgB;IAChD,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,QAAQ,GAAG,CAAC,KAAK,CAAC,IAAI,EAAE,GAAG,KAAK,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC5E,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IACjE,MAAM,IAAI,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAC9D,MAAM,SAAS,GACb,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QAChD,SAAS,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC;QAC5B,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,CAAC,CAAC;IACR,OAAO,IAAI,GAAG,SAAS,CAAC;AAC1B,CAAC;AAED,6DAA6D;AAC7D,SAAS,aAAa,CAAC,KAAa;IAClC,MAAM,CAAC,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAC9B,OAAO,CACL,0DAA0D,CAAC,IAAI,CAAC,CAAC,CAAC;QAClE,sEAAsE,CAAC,IAAI,CACzE,CAAC,CACF;QACD,iDAAiD,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,0CAA0C,CAAC,IAAI,CAAC,CAAC,CAAC,CACnD,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,qBAAqB;IAEb;IACA;IAFnB,YACmB,MAAmB,EACnB,gBAAgB,GAAG;QADnB,WAAM,GAAN,MAAM,CAAa;QACnB,kBAAa,GAAb,aAAa,CAAM;IACnC,CAAC;IAEJ,QAAQ,CAAC,OAA4B;QACnC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,gBAAgB,GAAG,CAAC,CAAC;QACzB,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,IAAI,sBAAsB,GAAG,CAAC,CAAC;QAE/B,KAAK,MAAM,EAAE,KAAK,EAAE,aAAa,EAAE,iBAAiB,EAAE,IAAI,OAAO,EAAE,CAAC;YAClE,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM;iBACvB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,UAAU,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;iBAC3D,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;YAErC,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,GAAG,IAAI,GAAG,CAAC,KAAK,GAAG,CAAC,IAAI,GAAG,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;gBACvD,OAAO,EAAE,CAAC;gBACV,MAAM,UAAU,GACd,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC,EAAE,gBAAgB;oBACnE,CAAC,CAAC;gBACJ,gBAAgB,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,UAAU,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,CAAC;YACnC,IAAI,iBAAiB,EAAE,CAAC;gBACtB,mBAAmB,EAAE,CAAC;gBACtB,IAAI,KAAK;oBAAE,MAAM,EAAE,CAAC;YACtB,CAAC;iBAAM,CAAC;gBACN,sBAAsB,EAAE,CAAC;gBACzB,IAAI,KAAK;oBAAE,MAAM,EAAE,CAAC;YACtB,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC;QAC7B,OAAO;YACL,YAAY,EAAE,KAAK;YACnB,oBAAoB,EAAE,OAAO;YAC7B,QAAQ,EAAE,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACzC,iBAAiB,EAAE,MAAM;YACzB,kBAAkB,EAAE,MAAM;YAC1B,oBAAoB,EAClB,mBAAmB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,mBAAmB,CAAC,CAAC,CAAC,CAAC;YAC5D,qBAAqB,EACnB,sBAAsB,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,sBAAsB,CAAC,CAAC,CAAC,CAAC;YAClE,cAAc,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;SAC7D,CAAC;IACJ,CAAC;CACF"}
|
package/dist/index.d.ts
CHANGED
|
@@ -41,4 +41,8 @@ export { precisionAtK, recallAtK, reciprocalRank, hitRateAtK, aggregateEvalResul
|
|
|
41
41
|
export { bootstrapPairedTest } from "./eval/statistics.js";
|
|
42
42
|
export type { StatTestResult } from "./eval/statistics.js";
|
|
43
43
|
export { loadEvalDataset, saveEvalDataset } from "./eval/dataset-io.js";
|
|
44
|
+
export { EcosystemEvaluator, printEcosystemEvalResult, } from "./eval/ecosystem-eval.js";
|
|
45
|
+
export type { EcosystemEvalDataset, EcosystemEvalResult, } from "./eval/ecosystem-eval.js";
|
|
46
|
+
export { SkillRoutingEvaluator } from "./eval/skill-routing-eval.js";
|
|
47
|
+
export type { SkillRoutingQuery, SkillRoutingEvalResult, } from "./eval/skill-routing-eval.js";
|
|
44
48
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,uBAAuB,CAAC;AAGtC,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAG/C,OAAO,EACL,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,uCAAuC,CAAC;AAGnF,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGlE,YAAY,EACV,eAAe,EACf,eAAe,EACf,eAAe,EACf,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAG3D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,uBAAuB,CAAC;AAGtC,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAG/C,OAAO,EACL,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,uCAAuC,CAAC;AAGnF,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGlE,YAAY,EACV,eAAe,EACf,eAAe,EACf,eAAe,EACf,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AAG3D,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC;AACrE,YAAY,EACV,iBAAiB,EACjB,sBAAsB,GACvB,MAAM,8BAA8B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -44,4 +44,6 @@ export { ExperimentStore, makeRunId } from "./eval/experiment-store.js";
|
|
|
44
44
|
export { precisionAtK, recallAtK, reciprocalRank, hitRateAtK, aggregateEvalResults, computeEvalResult, } from "./eval/metrics.js";
|
|
45
45
|
export { bootstrapPairedTest } from "./eval/statistics.js";
|
|
46
46
|
export { loadEvalDataset, saveEvalDataset } from "./eval/dataset-io.js";
|
|
47
|
+
export { EcosystemEvaluator, printEcosystemEvalResult, } from "./eval/ecosystem-eval.js";
|
|
48
|
+
export { SkillRoutingEvaluator } from "./eval/skill-routing-eval.js";
|
|
47
49
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,cAAc,uBAAuB,CAAC;AAEtC,SAAS;AACT,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO;AACP,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAqB,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AAGzB,mBAAmB;AACnB,OAAO,EAAE,eAAe,EAAa,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,yBAAyB;AACzB,OAAO,EACL,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAGrC,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,uCAAuC,CAAC;AAEnF,iBAAiB;AACjB,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,gBAAgB;AAChB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AASlE,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAG9B,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,uBAAuB;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,cAAc,uBAAuB,CAAC;AAEtC,SAAS;AACT,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO;AACP,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAqB,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AAGzB,mBAAmB;AACnB,OAAO,EAAE,eAAe,EAAa,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,UAAU;AACV,OAAO,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAE/C,yBAAyB;AACzB,OAAO,EACL,YAAY,EACZ,eAAe,EACf,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAGrC,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAC5D,OAAO,EAAE,0BAA0B,EAAE,MAAM,uCAAuC,CAAC;AAEnF,iBAAiB;AACjB,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,gBAAgB;AAChB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AASlE,OAAO,EACL,wBAAwB,EACxB,uBAAuB,EACvB,gBAAgB,EAChB,gBAAgB,EAChB,gBAAgB,EAChB,iBAAiB,EACjB,wBAAwB,GACzB,MAAM,sBAAsB,CAAC;AAG9B,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,uBAAuB;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AACxE,OAAO,EACL,kBAAkB,EAClB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAKlC,OAAO,EAAE,qBAAqB,EAAE,MAAM,8BAA8B,CAAC"}
|
|
@@ -33,6 +33,15 @@ export interface VectorSearchResult {
|
|
|
33
33
|
metadata: Record<string, unknown>;
|
|
34
34
|
similarity: number;
|
|
35
35
|
sourceFile?: string;
|
|
36
|
+
/** UTC timestamp when this chunk was indexed — used for recency-weighted scoring. */
|
|
37
|
+
ingestedAt?: Date;
|
|
38
|
+
}
|
|
39
|
+
/** Options controlling composite similarity + recency scoring. */
|
|
40
|
+
export interface SearchOptions {
|
|
41
|
+
/** Weight applied to vector similarity (0–1). Defaults to 0.85. */
|
|
42
|
+
alpha?: number;
|
|
43
|
+
/** Weight applied to recency score (0–1). Defaults to 0.15. Only applied when ingestedAt is available. */
|
|
44
|
+
beta?: number;
|
|
36
45
|
}
|
|
37
46
|
export interface VectorStore {
|
|
38
47
|
/** Store name */
|
|
@@ -46,7 +55,7 @@ export interface VectorStore {
|
|
|
46
55
|
/** Get current state (sourceFile → commitHash) for change detection */
|
|
47
56
|
getCurrentState(collection?: string): Promise<Map<string, string>>;
|
|
48
57
|
/** Search by embedding vector */
|
|
49
|
-
search(queryEmbedding: number[], topK: number, collection?: string): Promise<VectorSearchResult[]>;
|
|
58
|
+
search(queryEmbedding: number[], topK: number, collection?: string, options?: SearchOptions): Promise<VectorSearchResult[]>;
|
|
50
59
|
/** Batch delete by document IDs (more efficient than per-file delete when IDs are known) */
|
|
51
60
|
batchDelete?(ids: string[]): Promise<void>;
|
|
52
61
|
/** Optional: delete entire collection */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/interfaces/vector-store.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAEhB,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAElC,uBAAuB;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IAEnB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IAEnB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IAEpB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/interfaces/vector-store.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,UAAU,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEhE,MAAM,WAAW,eAAe;IAC9B,YAAY,EAAE,MAAM,CAAC;IACrB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,2DAA2D;IAC3D,EAAE,CAAC,EAAE,MAAM,CAAC;IAEZ,4BAA4B;IAC5B,OAAO,EAAE,MAAM,CAAC;IAEhB,6BAA6B;IAC7B,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAElC,uBAAuB;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IAEpB,8CAA8C;IAC9C,UAAU,EAAE,MAAM,CAAC;IAEnB,6CAA6C;IAC7C,UAAU,EAAE,MAAM,CAAC;IAEnB,0CAA0C;IAC1C,WAAW,EAAE,MAAM,CAAC;IAEpB,oDAAoD;IACpD,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,MAAM,WAAW,kBAAkB;IACjC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAClC,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,qFAAqF;IACrF,UAAU,CAAC,EAAE,IAAI,CAAC;CACnB;AAED,kEAAkE;AAClE,MAAM,WAAW,aAAa;IAC5B,mEAAmE;IACnE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,0GAA0G;IAC1G,IAAI,CAAC,EAAE,MAAM,CAAC;CACf;AAED,MAAM,WAAW,WAAW;IAC1B,iBAAiB;IACjB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB,sDAAsD;IACtD,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5B,iCAAiC;IACjC,MAAM,CAAC,SAAS,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEnD,sCAAsC;IACtC,kBAAkB,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEzD,uEAAuE;IACvE,eAAe,CAAC,UAAU,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAEnE,iCAAiC;IACjC,MAAM,CACJ,cAAc,EAAE,MAAM,EAAE,EACxB,IAAI,EAAE,MAAM,EACZ,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;IAEjC,4FAA4F;IAC5F,WAAW,CAAC,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE3C,yCAAyC;IACzC,gBAAgB,CAAC,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAErD,qCAAqC;IACrC,QAAQ,CAAC,IAAI,OAAO,CAAC;QAAE,aAAa,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,EAAE,CAAA;KAAE,CAAC,CAAC;IAEvE,iDAAiD;IACjD,aAAa,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,CAAC;IAEtC,6CAA6C;IAC7C,kBAAkB,CAAC,CAAC,cAAc,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAEtE,kEAAkE;IAClE,QAAQ,CAAC,IAAI,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC,CAAC;IAE7C,4DAA4D;IAC5D,SAAS,CAAC,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAEjD,gGAAgG;IAChG,KAAK,CAAC,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACzB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB"}
|