@aiconnect/easy-rag 0.3.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -28
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +213 -66
- package/dist/commands/init.js.map +1 -1
- package/dist/config/types.d.ts +3 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +37 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/metrics.d.ts +12 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +43 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/eval/reporter.d.ts +7 -0
- package/dist/eval/reporter.d.ts.map +1 -0
- package/dist/eval/reporter.js +105 -0
- package/dist/eval/reporter.js.map +1 -0
- package/dist/eval/runner.d.ts +8 -0
- package/dist/eval/runner.d.ts.map +1 -0
- package/dist/eval/runner.js +43 -0
- package/dist/eval/runner.js.map +1 -0
- package/dist/eval/test-cases.d.ts +3 -0
- package/dist/eval/test-cases.d.ts.map +1 -0
- package/dist/eval/test-cases.js +98 -0
- package/dist/eval/test-cases.js.map +1 -0
- package/dist/eval/types.d.ts +49 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/index.js +10 -45
- package/dist/index.js.map +1 -1
- package/dist/indexer/orchestrator.d.ts.map +1 -1
- package/dist/indexer/orchestrator.js +24 -6
- package/dist/indexer/orchestrator.js.map +1 -1
- package/dist/query/hybrid.d.ts +17 -0
- package/dist/query/hybrid.d.ts.map +1 -0
- package/dist/query/hybrid.js +116 -0
- package/dist/query/hybrid.js.map +1 -0
- package/dist/query/index.d.ts.map +1 -1
- package/dist/query/index.js +81 -16
- package/dist/query/index.js.map +1 -1
- package/dist/query/search.d.ts +7 -4
- package/dist/query/search.d.ts.map +1 -1
- package/dist/query/search.js +45 -40
- package/dist/query/search.js.map +1 -1
- package/dist/query/types.d.ts +8 -0
- package/dist/query/types.d.ts.map +1 -1
- package/dist/vector-store/index.d.ts +3 -2
- package/dist/vector-store/index.d.ts.map +1 -1
- package/dist/vector-store/index.js +2 -2
- package/dist/vector-store/index.js.map +1 -1
- package/dist/vector-store/tokenizer.d.ts +3 -0
- package/dist/vector-store/tokenizer.d.ts.map +1 -0
- package/dist/vector-store/tokenizer.js +91 -0
- package/dist/vector-store/tokenizer.js.map +1 -0
- package/dist/vector-store/utils.d.ts +4 -1
- package/dist/vector-store/utils.d.ts.map +1 -1
- package/dist/vector-store/utils.js +17 -6
- package/dist/vector-store/utils.js.map +1 -1
- package/dist/vector-store/zvec.d.ts +15 -0
- package/dist/vector-store/zvec.d.ts.map +1 -0
- package/dist/vector-store/zvec.js +136 -0
- package/dist/vector-store/zvec.js.map +1 -0
- package/package.json +6 -5
- package/skills/easy-rag/SKILL.md +40 -9
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { HybridQueryResult } from '../query/index.js';
|
|
2
|
+
export declare function computeMRR(results: HybridQueryResult[], expectedSources: string[]): number;
|
|
3
|
+
export declare function computePrecisionAtK(results: HybridQueryResult[], expectedSources: string[], k: number): number;
|
|
4
|
+
export declare function hasRelevantInTopK(results: HybridQueryResult[], expectedSources: string[], k: number): boolean;
|
|
5
|
+
export declare function getTopScore(results: HybridQueryResult[]): number;
|
|
6
|
+
export declare function buildResultDetails(results: HybridQueryResult[], expectedSources: string[]): Array<{
|
|
7
|
+
rank: number;
|
|
8
|
+
sourcePath: string;
|
|
9
|
+
score: number;
|
|
10
|
+
relevant: boolean;
|
|
11
|
+
}>;
|
|
12
|
+
//# sourceMappingURL=metrics.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAQ3D,wBAAgB,UAAU,CAAC,OAAO,EAAE,iBAAiB,EAAE,EAAE,eAAe,EAAE,MAAM,EAAE,GAAG,MAAM,CAQ1F;AAED,wBAAgB,mBAAmB,CACjC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,EACzB,CAAC,EAAE,MAAM,GACR,MAAM,CAKR;AAED,wBAAgB,iBAAiB,CAC/B,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,EACzB,CAAC,EAAE,MAAM,GACR,OAAO,CAGT;AAED,wBAAgB,WAAW,CAAC,OAAO,EAAE,iBAAiB,EAAE,GAAG,MAAM,CAIhE;AAED,wBAAgB,kBAAkB,CAChC,OAAO,EAAE,iBAAiB,EAAE,EAC5B,eAAe,EAAE,MAAM,EAAE,GACxB,KAAK,CAAC;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,OAAO,CAAA;CAAE,CAAC,CAO/E"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
function isRelevant(result, expectedSources) {
|
|
2
|
+
if (expectedSources.length === 0)
|
|
3
|
+
return false;
|
|
4
|
+
const src = result.metadata.sourcePath.toLowerCase();
|
|
5
|
+
return expectedSources.some((s) => src.includes(s.toLowerCase()));
|
|
6
|
+
}
|
|
7
|
+
export function computeMRR(results, expectedSources) {
|
|
8
|
+
if (expectedSources.length === 0)
|
|
9
|
+
return 0;
|
|
10
|
+
for (let i = 0; i < results.length; i++) {
|
|
11
|
+
if (isRelevant(results[i], expectedSources)) {
|
|
12
|
+
return 1 / (i + 1);
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return 0;
|
|
16
|
+
}
|
|
17
|
+
export function computePrecisionAtK(results, expectedSources, k) {
|
|
18
|
+
if (expectedSources.length === 0)
|
|
19
|
+
return 0;
|
|
20
|
+
const topK = results.slice(0, k);
|
|
21
|
+
const relevant = topK.filter((r) => isRelevant(r, expectedSources)).length;
|
|
22
|
+
return relevant / k;
|
|
23
|
+
}
|
|
24
|
+
export function hasRelevantInTopK(results, expectedSources, k) {
|
|
25
|
+
if (expectedSources.length === 0)
|
|
26
|
+
return false;
|
|
27
|
+
return results.slice(0, k).some((r) => isRelevant(r, expectedSources));
|
|
28
|
+
}
|
|
29
|
+
export function getTopScore(results) {
|
|
30
|
+
if (results.length === 0)
|
|
31
|
+
return 0;
|
|
32
|
+
const r = results[0];
|
|
33
|
+
return r.hybridScore ?? r.similarity ?? 0;
|
|
34
|
+
}
|
|
35
|
+
export function buildResultDetails(results, expectedSources) {
|
|
36
|
+
return results.map((r, idx) => ({
|
|
37
|
+
rank: idx + 1,
|
|
38
|
+
sourcePath: r.metadata.sourcePath,
|
|
39
|
+
score: r.hybridScore ?? r.similarity ?? 0,
|
|
40
|
+
relevant: isRelevant(r, expectedSources),
|
|
41
|
+
}));
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=metrics.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/eval/metrics.ts"],"names":[],"mappings":"AAEA,SAAS,UAAU,CAAC,MAAyB,EAAE,eAAyB;IACtE,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,MAAM,GAAG,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,WAAW,EAAE,CAAC;IACrD,OAAO,eAAe,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,UAAU,UAAU,CAAC,OAA4B,EAAE,eAAyB;IAChF,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,IAAI,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,eAAe,CAAC,EAAE,CAAC;YAC5C,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACrB,CAAC;IACH,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,OAA4B,EAC5B,eAAyB,EACzB,CAAS;IAET,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3E,OAAO,QAAQ,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,OAA4B,EAC5B,eAAyB,EACzB,CAAS;IAET,IAAI,eAAe,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,OAA4B;IACtD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IACnC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACrB,OAAO,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAChC,OAA4B,EAC5B,eAAyB;IAEzB,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;QAC9B,IAAI,EAAE,GAAG,GAAG,CAAC;QACb,UAAU,EAAE,CAAC,CAAC,QAAQ,CAAC,UAAU;QACjC,KAAK,EAAE,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC,UAAU,IAAI,CAAC;QACzC,QAAQ,EAAE,UAAU,CAAC,CAAC,EAAE,eAAe,CAAC;KACzC,CAAC,CAAC,CAAC;AACN,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { EvalSummary, EvalCaseResult } from './types.js';
|
|
2
|
+
export declare function buildSummary(collection: string, negativeThreshold: number, cases: EvalCaseResult[]): EvalSummary;
|
|
3
|
+
export declare function printSummary(summary: EvalSummary): void;
|
|
4
|
+
export declare function printComparison(current: EvalSummary, previous: EvalSummary): void;
|
|
5
|
+
export declare function saveReport(summary: EvalSummary, outputDir: string): Promise<string>;
|
|
6
|
+
export declare function loadPreviousReport(comparePath: string): Promise<EvalSummary>;
|
|
7
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAc9D,wBAAgB,YAAY,CAC1B,UAAU,EAAE,MAAM,EAClB,iBAAiB,EAAE,MAAM,EACzB,KAAK,EAAE,cAAc,EAAE,GACtB,WAAW,CAoCb;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,IAAI,CA0CvD;AAED,wBAAgB,eAAe,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,GAAG,IAAI,CAYjF;AAED,wBAAsB,UAAU,CAAC,OAAO,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAOzF;AAED,wBAAsB,kBAAkB,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAGlF"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import fs from 'fs/promises';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
function pad(s, n) {
|
|
4
|
+
return s.length >= n ? s.slice(0, n) : s + ' '.repeat(n - s.length);
|
|
5
|
+
}
|
|
6
|
+
function fmt(n, decimals = 3) {
|
|
7
|
+
return n.toFixed(decimals);
|
|
8
|
+
}
|
|
9
|
+
function passLabel(pass) {
|
|
10
|
+
return pass ? 'PASS' : 'FAIL';
|
|
11
|
+
}
|
|
12
|
+
export function buildSummary(collection, negativeThreshold, cases) {
|
|
13
|
+
const passed = cases.filter((c) => c.overallPass).length;
|
|
14
|
+
const allModes = cases.flatMap((c) => c.modes);
|
|
15
|
+
const positiveModes = allModes.filter((_, i) => {
|
|
16
|
+
const caseIdx = cases.findIndex((c) => c.modes.includes(allModes[i]));
|
|
17
|
+
return caseIdx !== -1 && !cases[caseIdx].isNegative;
|
|
18
|
+
});
|
|
19
|
+
const avgMrr = positiveModes.length > 0
|
|
20
|
+
? positiveModes.reduce((sum, m) => sum + m.mrr, 0) / positiveModes.length
|
|
21
|
+
: 0;
|
|
22
|
+
const avgP1 = positiveModes.length > 0
|
|
23
|
+
? positiveModes.reduce((sum, m) => sum + m.precisionAt1, 0) / positiveModes.length
|
|
24
|
+
: 0;
|
|
25
|
+
const avgP3 = positiveModes.length > 0
|
|
26
|
+
? positiveModes.reduce((sum, m) => sum + m.precisionAt3, 0) / positiveModes.length
|
|
27
|
+
: 0;
|
|
28
|
+
return {
|
|
29
|
+
collection,
|
|
30
|
+
timestamp: new Date().toISOString(),
|
|
31
|
+
negativeThreshold,
|
|
32
|
+
totalCases: cases.length,
|
|
33
|
+
totalPassed: passed,
|
|
34
|
+
totalFailed: cases.length - passed,
|
|
35
|
+
passRate: cases.length > 0 ? passed / cases.length : 0,
|
|
36
|
+
avgMrr,
|
|
37
|
+
avgPrecisionAt1: avgP1,
|
|
38
|
+
avgPrecisionAt3: avgP3,
|
|
39
|
+
cases,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
export function printSummary(summary) {
|
|
43
|
+
const sep = '─'.repeat(100);
|
|
44
|
+
console.log('\n' + sep);
|
|
45
|
+
console.log(` easy-rag Evaluation Report`);
|
|
46
|
+
console.log(` Collection : ${summary.collection}`);
|
|
47
|
+
console.log(` Timestamp : ${summary.timestamp}`);
|
|
48
|
+
console.log(` Neg. threshold: ${summary.negativeThreshold}`);
|
|
49
|
+
console.log(sep);
|
|
50
|
+
const header = [
|
|
51
|
+
pad('ID', 32),
|
|
52
|
+
pad('Mode', 8),
|
|
53
|
+
pad('MRR', 6),
|
|
54
|
+
pad('P@1', 5),
|
|
55
|
+
pad('P@3', 5),
|
|
56
|
+
pad('TopScore', 10),
|
|
57
|
+
pad('Result', 6),
|
|
58
|
+
].join(' ');
|
|
59
|
+
console.log(header);
|
|
60
|
+
console.log(sep);
|
|
61
|
+
for (const c of summary.cases) {
|
|
62
|
+
const prefix = c.isNegative ? '[NEG] ' : '';
|
|
63
|
+
for (const m of c.modes) {
|
|
64
|
+
const row = [
|
|
65
|
+
pad(prefix + c.id, 32),
|
|
66
|
+
pad(m.mode, 8),
|
|
67
|
+
pad(fmt(m.mrr), 6),
|
|
68
|
+
pad(fmt(m.precisionAt1, 2), 5),
|
|
69
|
+
pad(fmt(m.precisionAt3, 2), 5),
|
|
70
|
+
pad(fmt(m.topScore, 4), 10),
|
|
71
|
+
passLabel(m.pass),
|
|
72
|
+
].join(' ');
|
|
73
|
+
console.log(row);
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
console.log(sep);
|
|
77
|
+
console.log(` Total: ${summary.totalCases} cases | Passed: ${summary.totalPassed} | Failed: ${summary.totalFailed} | Pass rate: ${(summary.passRate * 100).toFixed(1)}%`);
|
|
78
|
+
console.log(` Avg MRR: ${fmt(summary.avgMrr)} | Avg P@1: ${fmt(summary.avgPrecisionAt1, 2)} | Avg P@3: ${fmt(summary.avgPrecisionAt3, 2)}`);
|
|
79
|
+
console.log(sep + '\n');
|
|
80
|
+
}
|
|
81
|
+
export function printComparison(current, previous) {
|
|
82
|
+
const delta = (a, b) => {
|
|
83
|
+
const d = a - b;
|
|
84
|
+
const sign = d >= 0 ? '+' : '';
|
|
85
|
+
return `${sign}${d.toFixed(3)}`;
|
|
86
|
+
};
|
|
87
|
+
console.log('\n Comparison vs previous run:');
|
|
88
|
+
console.log(` Pass rate : ${(previous.passRate * 100).toFixed(1)}% → ${(current.passRate * 100).toFixed(1)}% (${delta(current.passRate, previous.passRate)})`);
|
|
89
|
+
console.log(` Avg MRR : ${previous.avgMrr.toFixed(3)} → ${current.avgMrr.toFixed(3)} (${delta(current.avgMrr, previous.avgMrr)})`);
|
|
90
|
+
console.log(` Avg P@1 : ${previous.avgPrecisionAt1.toFixed(2)} → ${current.avgPrecisionAt1.toFixed(2)} (${delta(current.avgPrecisionAt1, previous.avgPrecisionAt1)})`);
|
|
91
|
+
console.log(` Avg P@3 : ${previous.avgPrecisionAt3.toFixed(2)} → ${current.avgPrecisionAt3.toFixed(2)} (${delta(current.avgPrecisionAt3, previous.avgPrecisionAt3)})\n`);
|
|
92
|
+
}
|
|
93
|
+
export async function saveReport(summary, outputDir) {
|
|
94
|
+
await fs.mkdir(outputDir, { recursive: true });
|
|
95
|
+
const ts = summary.timestamp.replace(/[:.]/g, '-');
|
|
96
|
+
const filename = `eval-results-${ts}.json`;
|
|
97
|
+
const filePath = path.join(outputDir, filename);
|
|
98
|
+
await fs.writeFile(filePath, JSON.stringify(summary, null, 2), 'utf-8');
|
|
99
|
+
return filePath;
|
|
100
|
+
}
|
|
101
|
+
export async function loadPreviousReport(comparePath) {
|
|
102
|
+
const raw = await fs.readFile(comparePath, 'utf-8');
|
|
103
|
+
return JSON.parse(raw);
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=reporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS;IAC/B,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS,EAAE,QAAQ,GAAG,CAAC;IAClC,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,SAAS,CAAC,IAAa;IAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,UAAkB,EAClB,iBAAyB,EACzB,KAAuB;IAEvB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,OAAO,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GACV,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QACzE,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,KAAK,GACT,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QAClF,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,KAAK,GACT,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QAClF,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,UAAU;QACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,iBAAiB;QACjB,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,WAAW,EAAE,MAAM;QACnB,WAAW,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM;QAClC,QAAQ,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM;QACN,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,KAAK;QACtB,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAoB;IAC/C,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QACb,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACd,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;QACnB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;KACjB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG;gBACV,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC;gBACtB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACd,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC3B,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,UAAU,oBAAoB,OAAO,CAAC,WAAW,cAAc,OAAO,CAAC,WAAW,iBAAiB,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3K,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7I,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,QAAqB;IACzE,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;QAC7C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChK,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtI,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAC1K,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;AAC9K,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAoB,EAAE,SAAiB;IACtE,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,gBAAgB,EAAE,OAAO,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,WAAmB;IAC1D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { EvalTestCase, EvalCaseResult } from './types.js';
|
|
2
|
+
export interface RunnerOptions {
|
|
3
|
+
collection: string;
|
|
4
|
+
top: number;
|
|
5
|
+
negativeThreshold: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function runTestCase(testCase: EvalTestCase, options: RunnerOptions): Promise<EvalCaseResult>;
|
|
8
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAe,MAAM,YAAY,CAAC;AAU5E,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CA4CzB"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { searchQuery } from '../query/index.js';
|
|
2
|
+
import { computeMRR, computePrecisionAtK, hasRelevantInTopK, getTopScore, buildResultDetails, } from './metrics.js';
|
|
3
|
+
export async function runTestCase(testCase, options) {
|
|
4
|
+
const modeResults = [];
|
|
5
|
+
for (const mode of testCase.modes) {
|
|
6
|
+
const results = await searchQuery(testCase.query, {
|
|
7
|
+
top: options.top,
|
|
8
|
+
includeMetadata: true,
|
|
9
|
+
collection: options.collection,
|
|
10
|
+
mode,
|
|
11
|
+
});
|
|
12
|
+
const topScore = getTopScore(results);
|
|
13
|
+
let pass;
|
|
14
|
+
if (testCase.isNegative) {
|
|
15
|
+
pass = topScore < options.negativeThreshold;
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
pass = hasRelevantInTopK(results, testCase.expectedSources, 3);
|
|
19
|
+
}
|
|
20
|
+
modeResults.push({
|
|
21
|
+
mode,
|
|
22
|
+
mrr: computeMRR(results, testCase.expectedSources),
|
|
23
|
+
precisionAt1: computePrecisionAtK(results, testCase.expectedSources, 1),
|
|
24
|
+
precisionAt3: computePrecisionAtK(results, testCase.expectedSources, 3),
|
|
25
|
+
precisionAt5: computePrecisionAtK(results, testCase.expectedSources, 5),
|
|
26
|
+
hitAt1: hasRelevantInTopK(results, testCase.expectedSources, 1),
|
|
27
|
+
hitAt3: hasRelevantInTopK(results, testCase.expectedSources, 3),
|
|
28
|
+
topScore,
|
|
29
|
+
pass,
|
|
30
|
+
results: buildResultDetails(results, testCase.expectedSources),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const overallPass = modeResults.every((m) => m.pass);
|
|
34
|
+
return {
|
|
35
|
+
id: testCase.id,
|
|
36
|
+
description: testCase.description,
|
|
37
|
+
query: testCase.query,
|
|
38
|
+
isNegative: testCase.isNegative ?? false,
|
|
39
|
+
modes: modeResults,
|
|
40
|
+
overallPass,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,UAAU,EACV,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAQtB,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAsB,EACtB,OAAsB;IAEtB,MAAM,WAAW,GAAkB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE;YAChD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,IAAI;SACL,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEtC,IAAI,IAAa,CAAC;QAClB,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,GAAG,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,IAAI;YACJ,GAAG,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,CAAC;YAClD,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,MAAM,EAAE,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YAC/D,MAAM,EAAE,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YAC/D,QAAQ;YACR,IAAI;YACJ,OAAO,EAAE,kBAAkB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,CAAC;SAC/D,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAErD,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,KAAK;QACxC,KAAK,EAAE,WAAW;QAClB,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-cases.d.ts","sourceRoot":"","sources":["../../src/eval/test-cases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,eAAO,MAAM,UAAU,EAAE,YAAY,EAgGpC,CAAC"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
export const TEST_CASES = [
|
|
2
|
+
{
|
|
3
|
+
id: 'test-01-literal-title',
|
|
4
|
+
description: 'Literal title retrieval via BM25 and hybrid',
|
|
5
|
+
query: 'How We Could Lose Control',
|
|
6
|
+
modes: ['bm25', 'hybrid'],
|
|
7
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
id: 'test-02-paraphrase-semantic',
|
|
11
|
+
description: 'Paraphrase / semantic retrieval — no exact title wording',
|
|
12
|
+
query: 'How can humans lose control over AI systems?',
|
|
13
|
+
modes: ['vector', 'hybrid'],
|
|
14
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
15
|
+
expectedContent: ['control', 'AI'],
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
id: 'test-03-transcript-detail',
|
|
19
|
+
description: 'Deep transcript content retrieval',
|
|
20
|
+
query: 'digital democracy tools like Polis in Taiwan',
|
|
21
|
+
modes: ['vector', 'hybrid'],
|
|
22
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
23
|
+
expectedContent: ['Polis', 'Taiwan', 'democracy'],
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: 'test-04-burnout-productivity',
|
|
27
|
+
description: 'Productivity / burnout with AI — thematic match',
|
|
28
|
+
query: 'using AI to automate admin work and reduce burnout',
|
|
29
|
+
modes: ['vector', 'hybrid'],
|
|
30
|
+
expectedSources: ['pp1161938-from-burnout-to-bandwidth'],
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: 'test-05-startup-no-vc',
|
|
34
|
+
description: 'Startup growth without venture capital',
|
|
35
|
+
query: 'how to grow a startup without raising venture capital',
|
|
36
|
+
modes: ['vector', 'hybrid'],
|
|
37
|
+
expectedSources: ['pp1162297-the-art-of-scaling-your-startup'],
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'test-06-niche-term',
|
|
41
|
+
description: 'Exact niche term retrieval',
|
|
42
|
+
query: 'Bootstrapping Bronto',
|
|
43
|
+
modes: ['bm25', 'hybrid'],
|
|
44
|
+
expectedSources: ['pp1162297-the-art-of-scaling-your-startup'],
|
|
45
|
+
expectedContent: ['Bootstrapping Bronto'],
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'test-07-creator-economy',
|
|
49
|
+
description: 'Creator economy semantic retrieval',
|
|
50
|
+
query: 'discovery algorithms that prioritize creator growth over ad revenue',
|
|
51
|
+
modes: ['vector', 'hybrid'],
|
|
52
|
+
expectedSources: ['pp1162453-rethinking-discovery-algorithms'],
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
id: 'test-08-named-entity',
|
|
56
|
+
description: 'Named-entity retrieval — speaker name',
|
|
57
|
+
query: 'Tristan Harris',
|
|
58
|
+
modes: ['bm25', 'vector', 'hybrid'],
|
|
59
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
60
|
+
expectedContent: ['Tristan Harris'],
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: 'test-09-cross-topic',
|
|
64
|
+
description: 'Cross-topic thematic retrieval — AI, education, workforce',
|
|
65
|
+
query: 'AI in education and future workforce',
|
|
66
|
+
modes: ['hybrid'],
|
|
67
|
+
expectedSources: [
|
|
68
|
+
'pp1161969-rethinking-higher-education',
|
|
69
|
+
'pp1162045-ai-s-rise-talent-s-fall',
|
|
70
|
+
'pp1163002-meet-your-new-work-team',
|
|
71
|
+
],
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: 'test-10-negative-out-of-domain',
|
|
75
|
+
description: 'Negative / out-of-domain query — no relevant result expected',
|
|
76
|
+
query: 'quantum battery recycling policy',
|
|
77
|
+
modes: ['vector', 'hybrid'],
|
|
78
|
+
expectedSources: [],
|
|
79
|
+
isNegative: true,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
id: 'test-11-negative-unrelated-topic',
|
|
83
|
+
description: 'Negative — unrelated topic (medieval history)',
|
|
84
|
+
query: 'medieval siege warfare and castle fortification strategies',
|
|
85
|
+
modes: ['vector', 'hybrid'],
|
|
86
|
+
expectedSources: [],
|
|
87
|
+
isNegative: true,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
id: 'test-12-negative-specific-entity',
|
|
91
|
+
description: 'Negative — specific entity not in corpus',
|
|
92
|
+
query: 'Elon Musk SpaceX Mars colonization timeline',
|
|
93
|
+
modes: ['bm25', 'hybrid'],
|
|
94
|
+
expectedSources: [],
|
|
95
|
+
isNegative: true,
|
|
96
|
+
},
|
|
97
|
+
];
|
|
98
|
+
//# sourceMappingURL=test-cases.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-cases.js","sourceRoot":"","sources":["../../src/eval/test-cases.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,UAAU,GAAmB;IACxC;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,6CAA6C;QAC1D,KAAK,EAAE,2BAA2B;QAClC,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,CAAC,qCAAqC,CAAC;KACzD;IACD;QACE,EAAE,EAAE,6BAA6B;QACjC,WAAW,EAAE,0DAA0D;QACvE,KAAK,EAAE,8CAA8C;QACrD,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC;KACnC;IACD;QACE,EAAE,EAAE,2BAA2B;QAC/B,WAAW,EAAE,mCAAmC;QAChD,KAAK,EAAE,8CAA8C;QACrD,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC;KAClD;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,WAAW,EAAE,iDAAiD;QAC9D,KAAK,EAAE,oDAAoD;QAC3D,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;KACzD;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,wCAAwC;QACrD,KAAK,EAAE,uDAAuD;QAC9D,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,2CAA2C,CAAC;KAC/D;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,4BAA4B;QACzC,KAAK,EAAE,sBAAsB;QAC7B,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,CAAC,2CAA2C,CAAC;QAC9D,eAAe,EAAE,CAAC,sBAAsB,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,oCAAoC;QACjD,KAAK,EAAE,qEAAqE;QAC5E,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,2CAA2C,CAAC;KAC/D;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,uCAAuC;QACpD,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC;QACnC,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,gBAAgB,CAAC;KACpC;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,WAAW,EAAE,2DAA2D;QACxE,KAAK,EAAE,sCAAsC;QAC7C,KAAK,EAAE,CAAC,QAAQ,CAAC;QACjB,eAAe,EAAE;YACf,uCAAuC;YACvC,mCAAmC;YACnC,mCAAmC;SACpC;KACF;IACD;QACE,EAAE,EAAE,gCAAgC;QACpC,WAAW,EAAE,8DAA8D;QAC3E,KAAK,EAAE,kCAAkC;QACzC,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;IACD;QACE,EAAE,EAAE,kCAAkC;QACtC,WAAW,EAAE,+CAA+C;QAC5D,KAAK,EAAE,4DAA4D;QACnE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;IACD;QACE,EAAE,EAAE,kCAAkC;QACtC,WAAW,EAAE,0CAA0C;QACvD,KAAK,EAAE,6CAA6C;QACpD,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;CACF,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export type SearchMode = 'auto' | 'hybrid' | 'vector' | 'bm25';
|
|
2
|
+
export interface EvalTestCase {
|
|
3
|
+
id: string;
|
|
4
|
+
description: string;
|
|
5
|
+
query: string;
|
|
6
|
+
modes: SearchMode[];
|
|
7
|
+
expectedSources: string[];
|
|
8
|
+
expectedContent?: string[];
|
|
9
|
+
isNegative?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface ModeMetrics {
|
|
12
|
+
mode: SearchMode;
|
|
13
|
+
mrr: number;
|
|
14
|
+
precisionAt1: number;
|
|
15
|
+
precisionAt3: number;
|
|
16
|
+
precisionAt5: number;
|
|
17
|
+
hitAt1: boolean;
|
|
18
|
+
hitAt3: boolean;
|
|
19
|
+
topScore: number;
|
|
20
|
+
pass: boolean;
|
|
21
|
+
results: Array<{
|
|
22
|
+
rank: number;
|
|
23
|
+
sourcePath: string;
|
|
24
|
+
score: number;
|
|
25
|
+
relevant: boolean;
|
|
26
|
+
}>;
|
|
27
|
+
}
|
|
28
|
+
export interface EvalCaseResult {
|
|
29
|
+
id: string;
|
|
30
|
+
description: string;
|
|
31
|
+
query: string;
|
|
32
|
+
isNegative: boolean;
|
|
33
|
+
modes: ModeMetrics[];
|
|
34
|
+
overallPass: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface EvalSummary {
|
|
37
|
+
collection: string;
|
|
38
|
+
timestamp: string;
|
|
39
|
+
negativeThreshold: number;
|
|
40
|
+
totalCases: number;
|
|
41
|
+
totalPassed: number;
|
|
42
|
+
totalFailed: number;
|
|
43
|
+
passRate: number;
|
|
44
|
+
avgMrr: number;
|
|
45
|
+
avgPrecisionAt1: number;
|
|
46
|
+
avgPrecisionAt3: number;
|
|
47
|
+
cases: EvalCaseResult[];
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}
|
package/dist/index.js
CHANGED
|
@@ -3,10 +3,8 @@ import { Command } from 'commander';
|
|
|
3
3
|
import { parseDocument } from './parsers/index.js';
|
|
4
4
|
import { indexFolder } from './indexer/index.js';
|
|
5
5
|
import { executeQuery } from './query/index.js';
|
|
6
|
-
import { listCollections, deleteCollection
|
|
7
|
-
import { ensureChromaRunning } from './vector-store/chroma-server.js';
|
|
6
|
+
import { listCollections, deleteCollection } from './vector-store/index.js';
|
|
8
7
|
import { init } from './commands/init.js';
|
|
9
|
-
import { serve } from './commands/serve.js';
|
|
10
8
|
import { getOpenAIKey } from './config/index.js';
|
|
11
9
|
import path from 'path';
|
|
12
10
|
import fs from 'fs/promises';
|
|
@@ -32,23 +30,6 @@ program
|
|
|
32
30
|
process.exit(1);
|
|
33
31
|
}
|
|
34
32
|
});
|
|
35
|
-
program
|
|
36
|
-
.command('serve')
|
|
37
|
-
.description('Start the ChromaDB server and keep it running in the foreground')
|
|
38
|
-
.action(async () => {
|
|
39
|
-
try {
|
|
40
|
-
await serve();
|
|
41
|
-
}
|
|
42
|
-
catch (error) {
|
|
43
|
-
if (error instanceof Error) {
|
|
44
|
-
console.error(`Error: ${error.message}`);
|
|
45
|
-
}
|
|
46
|
-
else {
|
|
47
|
-
console.error('Error: Unknown error occurred');
|
|
48
|
-
}
|
|
49
|
-
process.exit(1);
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
33
|
program
|
|
53
34
|
.command('parse')
|
|
54
35
|
.description('Parse a document file (PDF, Markdown, CSV) and extract text with metadata')
|
|
@@ -89,7 +70,7 @@ program
|
|
|
89
70
|
});
|
|
90
71
|
program
|
|
91
72
|
.command('index')
|
|
92
|
-
.description('Index documents from a folder into
|
|
73
|
+
.description('Index documents from a folder into the vector store for RAG queries')
|
|
93
74
|
.argument('<folder>', 'Path to the folder containing documents to index')
|
|
94
75
|
.action(async (folder) => {
|
|
95
76
|
const apiKey = await getOpenAIKey();
|
|
@@ -110,10 +91,6 @@ program
|
|
|
110
91
|
console.error(`\nPlease check the path and try again.`);
|
|
111
92
|
process.exit(1);
|
|
112
93
|
}
|
|
113
|
-
if (!await ensureChromaRunning()) {
|
|
114
|
-
console.error('Error: ChromaDB is required for indexing.');
|
|
115
|
-
process.exit(1);
|
|
116
|
-
}
|
|
117
94
|
const result = await indexFolder(folderPath);
|
|
118
95
|
if (!result.success) {
|
|
119
96
|
console.error('\nIndexing failed.');
|
|
@@ -126,11 +103,12 @@ program
|
|
|
126
103
|
});
|
|
127
104
|
program
|
|
128
105
|
.command('query')
|
|
129
|
-
.description('Query indexed documents using vector
|
|
106
|
+
.description('Query indexed documents using hybrid, vector, or BM25 search')
|
|
130
107
|
.argument('<question>', 'Question to search for in indexed documents')
|
|
131
108
|
.option('-t, --top <number>', 'Number of results to return', '5')
|
|
132
109
|
.option('-m, --metadata', 'Include metadata in results (source, score, chunk index)')
|
|
133
110
|
.option('-c, --collection <name>', 'Search in a specific collection (default: all collections)')
|
|
111
|
+
.option('--mode <mode>', 'Search mode: auto, hybrid, vector, or bm25 (default: auto)')
|
|
134
112
|
.action(async (question, options) => {
|
|
135
113
|
const apiKey = await getOpenAIKey();
|
|
136
114
|
if (!apiKey) {
|
|
@@ -150,11 +128,8 @@ program
|
|
|
150
128
|
top,
|
|
151
129
|
includeMetadata: !!options.metadata,
|
|
152
130
|
collection: options.collection,
|
|
131
|
+
mode: options.mode,
|
|
153
132
|
};
|
|
154
|
-
if (!await ensureChromaRunning()) {
|
|
155
|
-
console.error('Error: ChromaDB is required for querying.');
|
|
156
|
-
process.exit(1);
|
|
157
|
-
}
|
|
158
133
|
try {
|
|
159
134
|
await executeQuery(question, queryOptions);
|
|
160
135
|
}
|
|
@@ -170,15 +145,10 @@ program
|
|
|
170
145
|
});
|
|
171
146
|
program
|
|
172
147
|
.command('collections')
|
|
173
|
-
.description('List all
|
|
148
|
+
.description('List all collections with document counts')
|
|
174
149
|
.action(async () => {
|
|
175
|
-
if (!await ensureChromaRunning()) {
|
|
176
|
-
console.error('Error: ChromaDB is required to list collections.');
|
|
177
|
-
process.exit(1);
|
|
178
|
-
}
|
|
179
150
|
try {
|
|
180
|
-
|
|
181
|
-
const collections = await listCollections();
|
|
151
|
+
const collections = listCollections();
|
|
182
152
|
if (collections.length === 0) {
|
|
183
153
|
console.log('No collections found.');
|
|
184
154
|
process.exit(0);
|
|
@@ -202,17 +172,12 @@ program
|
|
|
202
172
|
});
|
|
203
173
|
program
|
|
204
174
|
.command('delete')
|
|
205
|
-
.description('Delete a
|
|
175
|
+
.description('Delete a collection')
|
|
206
176
|
.argument('<collectionName>', 'Name of the collection to delete')
|
|
207
177
|
.option('-f, --force', 'Skip confirmation prompt')
|
|
208
178
|
.action(async (collectionName, options) => {
|
|
209
|
-
if (!await ensureChromaRunning()) {
|
|
210
|
-
console.error('Error: ChromaDB is required to delete collections.');
|
|
211
|
-
process.exit(1);
|
|
212
|
-
}
|
|
213
179
|
try {
|
|
214
|
-
|
|
215
|
-
const collections = await listCollections();
|
|
180
|
+
const collections = listCollections();
|
|
216
181
|
const collectionExists = collections.some(c => c.name === collectionName);
|
|
217
182
|
if (!collectionExists) {
|
|
218
183
|
console.error(`Error: Collection '${collectionName}' not found`);
|
|
@@ -231,7 +196,7 @@ program
|
|
|
231
196
|
process.exit(0);
|
|
232
197
|
}
|
|
233
198
|
}
|
|
234
|
-
|
|
199
|
+
deleteCollection(collectionName);
|
|
235
200
|
console.log(`Collection '${collectionName}' deleted successfully.`);
|
|
236
201
|
}
|
|
237
202
|
catch (error) {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,aAAa,CAAC;AAE7B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,2DAA2D,CAAC;KACxE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,uEAAuE,CAAC;KACpF,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,2EAA2E,CAAC;KACxF,QAAQ,CAAC,YAAY,EAAE,2BAA2B,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,mBAAmB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACtF,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC5B,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;gBACrD,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YAC5D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,qEAAqE,CAAC;KAClF,QAAQ,CAAC,UAAU,EAAE,kDAAkD,CAAC;KACxE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;IACvB,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,MAAM,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;IAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,8DAA8D,CAAC;KAC3E,QAAQ,CAAC,YAAY,EAAE,6CAA6C,CAAC;KACrE,MAAM,CAAC,oBAAoB,EAAE,6BAA6B,EAAE,GAAG,CAAC;KAChE,MAAM,CAAC,gBAAgB,EAAE,0DAA0D,CAAC;KACpF,MAAM,CAAC,yBAAyB,EAAE,4DAA4D,CAAC;KAC/F,MAAM,CAAC,eAAe,EAAE,4DAA4D,CAAC;KACrF,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE;IAClC,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG;QACnB,GAAG;QACH,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ;QACnC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;KACnB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,2CAA2C,CAAC;KACxD,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;QAEtC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,KAAK,YAAY,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,qBAAqB,CAAC;KAClC,QAAQ,CAAC,kBAAkB,EAAE,kCAAkC,CAAC;KAChE,MAAM,CAAC,aAAa,EAAE,0BAA0B,CAAC;KACjD,MAAM,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE;IACxC,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;QACtC,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC;QAE1E,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,sBAAsB,cAAc,aAAa,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACxD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;gBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,cAAc,YAAY,CAAC,CAAC;YAC5G,EAAE,CAAC,KAAK,EAAE,CAAC;YAEX,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,eAAe,cAAc,yBAAyB,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAc,MAAM,YAAY,CAAC;AAa1D,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CA8H1E"}
|