@aiconnect/easy-rag 0.3.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +100 -28
- package/dist/commands/init.d.ts.map +1 -1
- package/dist/commands/init.js +105 -3
- package/dist/commands/init.js.map +1 -1
- package/dist/config/types.d.ts +3 -0
- package/dist/config/types.d.ts.map +1 -1
- package/dist/eval/index.d.ts +10 -0
- package/dist/eval/index.d.ts.map +1 -0
- package/dist/eval/index.js +37 -0
- package/dist/eval/index.js.map +1 -0
- package/dist/eval/metrics.d.ts +12 -0
- package/dist/eval/metrics.d.ts.map +1 -0
- package/dist/eval/metrics.js +43 -0
- package/dist/eval/metrics.js.map +1 -0
- package/dist/eval/reporter.d.ts +7 -0
- package/dist/eval/reporter.d.ts.map +1 -0
- package/dist/eval/reporter.js +105 -0
- package/dist/eval/reporter.js.map +1 -0
- package/dist/eval/runner.d.ts +8 -0
- package/dist/eval/runner.d.ts.map +1 -0
- package/dist/eval/runner.js +43 -0
- package/dist/eval/runner.js.map +1 -0
- package/dist/eval/test-cases.d.ts +3 -0
- package/dist/eval/test-cases.d.ts.map +1 -0
- package/dist/eval/test-cases.js +98 -0
- package/dist/eval/test-cases.js.map +1 -0
- package/dist/eval/types.d.ts +49 -0
- package/dist/eval/types.d.ts.map +1 -0
- package/dist/eval/types.js +2 -0
- package/dist/eval/types.js.map +1 -0
- package/dist/index.js +10 -45
- package/dist/index.js.map +1 -1
- package/dist/indexer/orchestrator.d.ts.map +1 -1
- package/dist/indexer/orchestrator.js +24 -6
- package/dist/indexer/orchestrator.js.map +1 -1
- package/dist/query/hybrid.d.ts +17 -0
- package/dist/query/hybrid.d.ts.map +1 -0
- package/dist/query/hybrid.js +116 -0
- package/dist/query/hybrid.js.map +1 -0
- package/dist/query/index.d.ts.map +1 -1
- package/dist/query/index.js +81 -16
- package/dist/query/index.js.map +1 -1
- package/dist/query/search.d.ts +7 -4
- package/dist/query/search.d.ts.map +1 -1
- package/dist/query/search.js +45 -40
- package/dist/query/search.js.map +1 -1
- package/dist/query/types.d.ts +8 -0
- package/dist/query/types.d.ts.map +1 -1
- package/dist/vector-store/index.d.ts +3 -2
- package/dist/vector-store/index.d.ts.map +1 -1
- package/dist/vector-store/index.js +2 -2
- package/dist/vector-store/index.js.map +1 -1
- package/dist/vector-store/tokenizer.d.ts +3 -0
- package/dist/vector-store/tokenizer.d.ts.map +1 -0
- package/dist/vector-store/tokenizer.js +91 -0
- package/dist/vector-store/tokenizer.js.map +1 -0
- package/dist/vector-store/utils.d.ts +4 -1
- package/dist/vector-store/utils.d.ts.map +1 -1
- package/dist/vector-store/utils.js +17 -6
- package/dist/vector-store/utils.js.map +1 -1
- package/dist/vector-store/zvec.d.ts +15 -0
- package/dist/vector-store/zvec.d.ts.map +1 -0
- package/dist/vector-store/zvec.js +136 -0
- package/dist/vector-store/zvec.js.map +1 -0
- package/package.json +5 -4
- package/skills/easy-rag/SKILL.md +40 -9
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/eval/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,aAAa,CAAC;AAC7B,OAAO,IAAI,MAAM,MAAM,CAAC;AAGxB,SAAS,GAAG,CAAC,CAAS,EAAE,CAAS;IAC/B,OAAO,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;AACtE,CAAC;AAED,SAAS,GAAG,CAAC,CAAS,EAAE,QAAQ,GAAG,CAAC;IAClC,OAAO,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;AAC7B,CAAC;AAED,SAAS,SAAS,CAAC,IAAa;IAC9B,OAAO,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,YAAY,CAC1B,UAAkB,EAClB,iBAAyB,EACzB,KAAuB;IAEvB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,MAAM,CAAC;IACzD,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,aAAa,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACtE,OAAO,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,UAAU,CAAC;IACtD,CAAC,CAAC,CAAC;IAEH,MAAM,MAAM,GACV,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QACzE,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,KAAK,GACT,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QAClF,CAAC,CAAC,CAAC,CAAC;IAER,MAAM,KAAK,GACT,aAAa,CAAC,MAAM,GAAG,CAAC;QACtB,CAAC,CAAC,aAAa,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,GAAG,aAAa,CAAC,MAAM;QAClF,CAAC,CAAC,CAAC,CAAC;IAER,OAAO;QACL,UAAU;QACV,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,iBAAiB;QACjB,UAAU,EAAE,KAAK,CAAC,MAAM;QACxB,WAAW,EAAE,MAAM;QACnB,WAAW,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM;QAClC,QAAQ,EAAE,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACtD,MAAM;QACN,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,KAAK;QACtB,KAAK;KACN,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,OAAoB;IAC/C,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAE5B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC,CAAC;IACxB,OAAO,CAAC,GAAG,CAAC,8BAA8B,CAAC,CAAC;IAC5C,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,UAAU,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,kBAAkB,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;IACnD,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,iBAAiB,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,MAAM,GAAG;QACb,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC;QACb,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QACd,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC;QACb,GAAG,CAAC,UAAU,EAAE,EAAE,CAAC;QACnB,GAAG,CAAC,QAAQ,EAAE,CAAC,CAAC;KACjB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACb,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACpB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjB,KAAK,MAAM,CAAC,IAAI,OAAO,CAAC,KAAK,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5C,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;YACxB,MAAM,GAAG,GAAG;gBACV,GAAG,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC;gBACtB,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACd,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBAClB,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;gBAC9B,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC3B,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;aAClB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACjB,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,UAAU,oBAAoB,OAAO,CAAC,WAAW,cAAc,OAAO,CAAC,WAAW,iBAAiB,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC3K,OAAO,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,OAAO,CAAC,MAAM,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,eAAe,GAAG,CAAC,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7I,OAAO,CAAC,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAAoB,EAAE,QAAqB;IACzE,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,CAAS,EAAU,EAAE;QAC7C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAChB,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/B,OAAO,GAAG,IAAI,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,CAAC,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,iCAAiC,CAAC,CAAC;IAC/C,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,QAAQ,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAChK,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IACtI,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC;IAC1K,OAAO,CAAC,GAAG,CAAC,iBAAiB,QAAQ,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,eAAe,EAAE,QAAQ,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC;AAC9K,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,OAAoB,EAAE,SAAiB;IACtE,MAAM,EAAE,CAAC,KAAK,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC/C,MAAM,EAAE,GAAG,OAAO,CAAC,SAAS,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,gBAAgB,EAAE,OAAO,CAAC;IAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;IAChD,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACxE,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,WAAmB;IAC1D,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;IACpD,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAgB,CAAC;AACxC,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { EvalTestCase, EvalCaseResult } from './types.js';
|
|
2
|
+
export interface RunnerOptions {
|
|
3
|
+
collection: string;
|
|
4
|
+
top: number;
|
|
5
|
+
negativeThreshold: number;
|
|
6
|
+
}
|
|
7
|
+
export declare function runTestCase(testCase: EvalTestCase, options: RunnerOptions): Promise<EvalCaseResult>;
|
|
8
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,cAAc,EAAe,MAAM,YAAY,CAAC;AAU5E,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,MAAM,CAAC;IACnB,GAAG,EAAE,MAAM,CAAC;IACZ,iBAAiB,EAAE,MAAM,CAAC;CAC3B;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,YAAY,EACtB,OAAO,EAAE,aAAa,GACrB,OAAO,CAAC,cAAc,CAAC,CA4CzB"}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
import { searchQuery } from '../query/index.js';
|
|
2
|
+
import { computeMRR, computePrecisionAtK, hasRelevantInTopK, getTopScore, buildResultDetails, } from './metrics.js';
|
|
3
|
+
export async function runTestCase(testCase, options) {
|
|
4
|
+
const modeResults = [];
|
|
5
|
+
for (const mode of testCase.modes) {
|
|
6
|
+
const results = await searchQuery(testCase.query, {
|
|
7
|
+
top: options.top,
|
|
8
|
+
includeMetadata: true,
|
|
9
|
+
collection: options.collection,
|
|
10
|
+
mode,
|
|
11
|
+
});
|
|
12
|
+
const topScore = getTopScore(results);
|
|
13
|
+
let pass;
|
|
14
|
+
if (testCase.isNegative) {
|
|
15
|
+
pass = topScore < options.negativeThreshold;
|
|
16
|
+
}
|
|
17
|
+
else {
|
|
18
|
+
pass = hasRelevantInTopK(results, testCase.expectedSources, 3);
|
|
19
|
+
}
|
|
20
|
+
modeResults.push({
|
|
21
|
+
mode,
|
|
22
|
+
mrr: computeMRR(results, testCase.expectedSources),
|
|
23
|
+
precisionAt1: computePrecisionAtK(results, testCase.expectedSources, 1),
|
|
24
|
+
precisionAt3: computePrecisionAtK(results, testCase.expectedSources, 3),
|
|
25
|
+
precisionAt5: computePrecisionAtK(results, testCase.expectedSources, 5),
|
|
26
|
+
hitAt1: hasRelevantInTopK(results, testCase.expectedSources, 1),
|
|
27
|
+
hitAt3: hasRelevantInTopK(results, testCase.expectedSources, 3),
|
|
28
|
+
topScore,
|
|
29
|
+
pass,
|
|
30
|
+
results: buildResultDetails(results, testCase.expectedSources),
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
const overallPass = modeResults.every((m) => m.pass);
|
|
34
|
+
return {
|
|
35
|
+
id: testCase.id,
|
|
36
|
+
description: testCase.description,
|
|
37
|
+
query: testCase.query,
|
|
38
|
+
isNegative: testCase.isNegative ?? false,
|
|
39
|
+
modes: modeResults,
|
|
40
|
+
overallPass,
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EACL,UAAU,EACV,mBAAmB,EACnB,iBAAiB,EACjB,WAAW,EACX,kBAAkB,GACnB,MAAM,cAAc,CAAC;AAQtB,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,QAAsB,EACtB,OAAsB;IAEtB,MAAM,WAAW,GAAkB,EAAE,CAAC;IAEtC,KAAK,MAAM,IAAI,IAAI,QAAQ,CAAC,KAAK,EAAE,CAAC;QAClC,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,QAAQ,CAAC,KAAK,EAAE;YAChD,GAAG,EAAE,OAAO,CAAC,GAAG;YAChB,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,OAAO,CAAC,UAAU;YAC9B,IAAI;SACL,CAAC,CAAC;QAEH,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC;QAEtC,IAAI,IAAa,CAAC;QAClB,IAAI,QAAQ,CAAC,UAAU,EAAE,CAAC;YACxB,IAAI,GAAG,QAAQ,GAAG,OAAO,CAAC,iBAAiB,CAAC;QAC9C,CAAC;aAAM,CAAC;YACN,IAAI,GAAG,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC;QACjE,CAAC;QAED,WAAW,CAAC,IAAI,CAAC;YACf,IAAI;YACJ,GAAG,EAAE,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,CAAC;YAClD,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,YAAY,EAAE,mBAAmB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YACvE,MAAM,EAAE,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YAC/D,MAAM,EAAE,iBAAiB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,EAAE,CAAC,CAAC;YAC/D,QAAQ;YACR,IAAI;YACJ,OAAO,EAAE,kBAAkB,CAAC,OAAO,EAAE,QAAQ,CAAC,eAAe,CAAC;SAC/D,CAAC,CAAC;IACL,CAAC;IAED,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAErD,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,WAAW,EAAE,QAAQ,CAAC,WAAW;QACjC,KAAK,EAAE,QAAQ,CAAC,KAAK;QACrB,UAAU,EAAE,QAAQ,CAAC,UAAU,IAAI,KAAK;QACxC,KAAK,EAAE,WAAW;QAClB,WAAW;KACZ,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-cases.d.ts","sourceRoot":"","sources":["../../src/eval/test-cases.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAE/C,eAAO,MAAM,UAAU,EAAE,YAAY,EAgGpC,CAAC"}
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
export const TEST_CASES = [
|
|
2
|
+
{
|
|
3
|
+
id: 'test-01-literal-title',
|
|
4
|
+
description: 'Literal title retrieval via BM25 and hybrid',
|
|
5
|
+
query: 'How We Could Lose Control',
|
|
6
|
+
modes: ['bm25', 'hybrid'],
|
|
7
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
8
|
+
},
|
|
9
|
+
{
|
|
10
|
+
id: 'test-02-paraphrase-semantic',
|
|
11
|
+
description: 'Paraphrase / semantic retrieval — no exact title wording',
|
|
12
|
+
query: 'How can humans lose control over AI systems?',
|
|
13
|
+
modes: ['vector', 'hybrid'],
|
|
14
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
15
|
+
expectedContent: ['control', 'AI'],
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
id: 'test-03-transcript-detail',
|
|
19
|
+
description: 'Deep transcript content retrieval',
|
|
20
|
+
query: 'digital democracy tools like Polis in Taiwan',
|
|
21
|
+
modes: ['vector', 'hybrid'],
|
|
22
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
23
|
+
expectedContent: ['Polis', 'Taiwan', 'democracy'],
|
|
24
|
+
},
|
|
25
|
+
{
|
|
26
|
+
id: 'test-04-burnout-productivity',
|
|
27
|
+
description: 'Productivity / burnout with AI — thematic match',
|
|
28
|
+
query: 'using AI to automate admin work and reduce burnout',
|
|
29
|
+
modes: ['vector', 'hybrid'],
|
|
30
|
+
expectedSources: ['pp1161938-from-burnout-to-bandwidth'],
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
id: 'test-05-startup-no-vc',
|
|
34
|
+
description: 'Startup growth without venture capital',
|
|
35
|
+
query: 'how to grow a startup without raising venture capital',
|
|
36
|
+
modes: ['vector', 'hybrid'],
|
|
37
|
+
expectedSources: ['pp1162297-the-art-of-scaling-your-startup'],
|
|
38
|
+
},
|
|
39
|
+
{
|
|
40
|
+
id: 'test-06-niche-term',
|
|
41
|
+
description: 'Exact niche term retrieval',
|
|
42
|
+
query: 'Bootstrapping Bronto',
|
|
43
|
+
modes: ['bm25', 'hybrid'],
|
|
44
|
+
expectedSources: ['pp1162297-the-art-of-scaling-your-startup'],
|
|
45
|
+
expectedContent: ['Bootstrapping Bronto'],
|
|
46
|
+
},
|
|
47
|
+
{
|
|
48
|
+
id: 'test-07-creator-economy',
|
|
49
|
+
description: 'Creator economy semantic retrieval',
|
|
50
|
+
query: 'discovery algorithms that prioritize creator growth over ad revenue',
|
|
51
|
+
modes: ['vector', 'hybrid'],
|
|
52
|
+
expectedSources: ['pp1162453-rethinking-discovery-algorithms'],
|
|
53
|
+
},
|
|
54
|
+
{
|
|
55
|
+
id: 'test-08-named-entity',
|
|
56
|
+
description: 'Named-entity retrieval — speaker name',
|
|
57
|
+
query: 'Tristan Harris',
|
|
58
|
+
modes: ['bm25', 'vector', 'hybrid'],
|
|
59
|
+
expectedSources: ['pp1162234-how-we-could-lose-control'],
|
|
60
|
+
expectedContent: ['Tristan Harris'],
|
|
61
|
+
},
|
|
62
|
+
{
|
|
63
|
+
id: 'test-09-cross-topic',
|
|
64
|
+
description: 'Cross-topic thematic retrieval — AI, education, workforce',
|
|
65
|
+
query: 'AI in education and future workforce',
|
|
66
|
+
modes: ['hybrid'],
|
|
67
|
+
expectedSources: [
|
|
68
|
+
'pp1161969-rethinking-higher-education',
|
|
69
|
+
'pp1162045-ai-s-rise-talent-s-fall',
|
|
70
|
+
'pp1163002-meet-your-new-work-team',
|
|
71
|
+
],
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
id: 'test-10-negative-out-of-domain',
|
|
75
|
+
description: 'Negative / out-of-domain query — no relevant result expected',
|
|
76
|
+
query: 'quantum battery recycling policy',
|
|
77
|
+
modes: ['vector', 'hybrid'],
|
|
78
|
+
expectedSources: [],
|
|
79
|
+
isNegative: true,
|
|
80
|
+
},
|
|
81
|
+
{
|
|
82
|
+
id: 'test-11-negative-unrelated-topic',
|
|
83
|
+
description: 'Negative — unrelated topic (medieval history)',
|
|
84
|
+
query: 'medieval siege warfare and castle fortification strategies',
|
|
85
|
+
modes: ['vector', 'hybrid'],
|
|
86
|
+
expectedSources: [],
|
|
87
|
+
isNegative: true,
|
|
88
|
+
},
|
|
89
|
+
{
|
|
90
|
+
id: 'test-12-negative-specific-entity',
|
|
91
|
+
description: 'Negative — specific entity not in corpus',
|
|
92
|
+
query: 'Elon Musk SpaceX Mars colonization timeline',
|
|
93
|
+
modes: ['bm25', 'hybrid'],
|
|
94
|
+
expectedSources: [],
|
|
95
|
+
isNegative: true,
|
|
96
|
+
},
|
|
97
|
+
];
|
|
98
|
+
//# sourceMappingURL=test-cases.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"test-cases.js","sourceRoot":"","sources":["../../src/eval/test-cases.ts"],"names":[],"mappings":"AAEA,MAAM,CAAC,MAAM,UAAU,GAAmB;IACxC;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,6CAA6C;QAC1D,KAAK,EAAE,2BAA2B;QAClC,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,CAAC,qCAAqC,CAAC;KACzD;IACD;QACE,EAAE,EAAE,6BAA6B;QACjC,WAAW,EAAE,0DAA0D;QACvE,KAAK,EAAE,8CAA8C;QACrD,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,SAAS,EAAE,IAAI,CAAC;KACnC;IACD;QACE,EAAE,EAAE,2BAA2B;QAC/B,WAAW,EAAE,mCAAmC;QAChD,KAAK,EAAE,8CAA8C;QACrD,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,OAAO,EAAE,QAAQ,EAAE,WAAW,CAAC;KAClD;IACD;QACE,EAAE,EAAE,8BAA8B;QAClC,WAAW,EAAE,iDAAiD;QAC9D,KAAK,EAAE,oDAAoD;QAC3D,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,qCAAqC,CAAC;KACzD;IACD;QACE,EAAE,EAAE,uBAAuB;QAC3B,WAAW,EAAE,wCAAwC;QACrD,KAAK,EAAE,uDAAuD;QAC9D,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,2CAA2C,CAAC;KAC/D;IACD;QACE,EAAE,EAAE,oBAAoB;QACxB,WAAW,EAAE,4BAA4B;QACzC,KAAK,EAAE,sBAAsB;QAC7B,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,CAAC,2CAA2C,CAAC;QAC9D,eAAe,EAAE,CAAC,sBAAsB,CAAC;KAC1C;IACD;QACE,EAAE,EAAE,yBAAyB;QAC7B,WAAW,EAAE,oCAAoC;QACjD,KAAK,EAAE,qEAAqE;QAC5E,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,CAAC,2CAA2C,CAAC;KAC/D;IACD;QACE,EAAE,EAAE,sBAAsB;QAC1B,WAAW,EAAE,uCAAuC;QACpD,KAAK,EAAE,gBAAgB;QACvB,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,EAAE,QAAQ,CAAC;QACnC,eAAe,EAAE,CAAC,qCAAqC,CAAC;QACxD,eAAe,EAAE,CAAC,gBAAgB,CAAC;KACpC;IACD;QACE,EAAE,EAAE,qBAAqB;QACzB,WAAW,EAAE,2DAA2D;QACxE,KAAK,EAAE,sCAAsC;QAC7C,KAAK,EAAE,CAAC,QAAQ,CAAC;QACjB,eAAe,EAAE;YACf,uCAAuC;YACvC,mCAAmC;YACnC,mCAAmC;SACpC;KACF;IACD;QACE,EAAE,EAAE,gCAAgC;QACpC,WAAW,EAAE,8DAA8D;QAC3E,KAAK,EAAE,kCAAkC;QACzC,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;IACD;QACE,EAAE,EAAE,kCAAkC;QACtC,WAAW,EAAE,+CAA+C;QAC5D,KAAK,EAAE,4DAA4D;QACnE,KAAK,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC;QAC3B,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;IACD;QACE,EAAE,EAAE,kCAAkC;QACtC,WAAW,EAAE,0CAA0C;QACvD,KAAK,EAAE,6CAA6C;QACpD,KAAK,EAAE,CAAC,MAAM,EAAE,QAAQ,CAAC;QACzB,eAAe,EAAE,EAAE;QACnB,UAAU,EAAE,IAAI;KACjB;CACF,CAAC"}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
export type SearchMode = 'auto' | 'hybrid' | 'vector' | 'bm25';
|
|
2
|
+
export interface EvalTestCase {
|
|
3
|
+
id: string;
|
|
4
|
+
description: string;
|
|
5
|
+
query: string;
|
|
6
|
+
modes: SearchMode[];
|
|
7
|
+
expectedSources: string[];
|
|
8
|
+
expectedContent?: string[];
|
|
9
|
+
isNegative?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export interface ModeMetrics {
|
|
12
|
+
mode: SearchMode;
|
|
13
|
+
mrr: number;
|
|
14
|
+
precisionAt1: number;
|
|
15
|
+
precisionAt3: number;
|
|
16
|
+
precisionAt5: number;
|
|
17
|
+
hitAt1: boolean;
|
|
18
|
+
hitAt3: boolean;
|
|
19
|
+
topScore: number;
|
|
20
|
+
pass: boolean;
|
|
21
|
+
results: Array<{
|
|
22
|
+
rank: number;
|
|
23
|
+
sourcePath: string;
|
|
24
|
+
score: number;
|
|
25
|
+
relevant: boolean;
|
|
26
|
+
}>;
|
|
27
|
+
}
|
|
28
|
+
export interface EvalCaseResult {
|
|
29
|
+
id: string;
|
|
30
|
+
description: string;
|
|
31
|
+
query: string;
|
|
32
|
+
isNegative: boolean;
|
|
33
|
+
modes: ModeMetrics[];
|
|
34
|
+
overallPass: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface EvalSummary {
|
|
37
|
+
collection: string;
|
|
38
|
+
timestamp: string;
|
|
39
|
+
negativeThreshold: number;
|
|
40
|
+
totalCases: number;
|
|
41
|
+
totalPassed: number;
|
|
42
|
+
totalFailed: number;
|
|
43
|
+
passRate: number;
|
|
44
|
+
avgMrr: number;
|
|
45
|
+
avgPrecisionAt1: number;
|
|
46
|
+
avgPrecisionAt3: number;
|
|
47
|
+
cases: EvalCaseResult[];
|
|
48
|
+
}
|
|
49
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,UAAU,GAAG,MAAM,GAAG,QAAQ,GAAG,QAAQ,GAAG,MAAM,CAAC;AAE/D,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,UAAU,EAAE,CAAC;IACpB,eAAe,EAAE,MAAM,EAAE,CAAC;IAC1B,eAAe,CAAC,EAAE,MAAM,EAAE,CAAC;IAC3B,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,YAAY,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,OAAO,CAAC;IAChB,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,KAAK,CAAC;QACb,IAAI,EAAE,MAAM,CAAC;QACb,UAAU,EAAE,MAAM,CAAC;QACnB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,OAAO,CAAC;KACnB,CAAC,CAAC;CACJ;AAED,MAAM,WAAW,cAAc;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,WAAW,EAAE,MAAM,CAAC;IACpB,KAAK,EAAE,MAAM,CAAC;IACd,UAAU,EAAE,OAAO,CAAC;IACpB,KAAK,EAAE,WAAW,EAAE,CAAC;IACrB,WAAW,EAAE,OAAO,CAAC;CACtB;AAED,MAAM,WAAW,WAAW;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,iBAAiB,EAAE,MAAM,CAAC;IAC1B,UAAU,EAAE,MAAM,CAAC;IACnB,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,eAAe,EAAE,MAAM,CAAC;IACxB,KAAK,EAAE,cAAc,EAAE,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/eval/types.ts"],"names":[],"mappings":""}
|
package/dist/index.js
CHANGED
|
@@ -3,10 +3,8 @@ import { Command } from 'commander';
|
|
|
3
3
|
import { parseDocument } from './parsers/index.js';
|
|
4
4
|
import { indexFolder } from './indexer/index.js';
|
|
5
5
|
import { executeQuery } from './query/index.js';
|
|
6
|
-
import { listCollections, deleteCollection
|
|
7
|
-
import { ensureChromaRunning } from './vector-store/chroma-server.js';
|
|
6
|
+
import { listCollections, deleteCollection } from './vector-store/index.js';
|
|
8
7
|
import { init } from './commands/init.js';
|
|
9
|
-
import { serve } from './commands/serve.js';
|
|
10
8
|
import { getOpenAIKey } from './config/index.js';
|
|
11
9
|
import path from 'path';
|
|
12
10
|
import fs from 'fs/promises';
|
|
@@ -32,23 +30,6 @@ program
|
|
|
32
30
|
process.exit(1);
|
|
33
31
|
}
|
|
34
32
|
});
|
|
35
|
-
program
|
|
36
|
-
.command('serve')
|
|
37
|
-
.description('Start the ChromaDB server and keep it running in the foreground')
|
|
38
|
-
.action(async () => {
|
|
39
|
-
try {
|
|
40
|
-
await serve();
|
|
41
|
-
}
|
|
42
|
-
catch (error) {
|
|
43
|
-
if (error instanceof Error) {
|
|
44
|
-
console.error(`Error: ${error.message}`);
|
|
45
|
-
}
|
|
46
|
-
else {
|
|
47
|
-
console.error('Error: Unknown error occurred');
|
|
48
|
-
}
|
|
49
|
-
process.exit(1);
|
|
50
|
-
}
|
|
51
|
-
});
|
|
52
33
|
program
|
|
53
34
|
.command('parse')
|
|
54
35
|
.description('Parse a document file (PDF, Markdown, CSV) and extract text with metadata')
|
|
@@ -89,7 +70,7 @@ program
|
|
|
89
70
|
});
|
|
90
71
|
program
|
|
91
72
|
.command('index')
|
|
92
|
-
.description('Index documents from a folder into
|
|
73
|
+
.description('Index documents from a folder into the vector store for RAG queries')
|
|
93
74
|
.argument('<folder>', 'Path to the folder containing documents to index')
|
|
94
75
|
.action(async (folder) => {
|
|
95
76
|
const apiKey = await getOpenAIKey();
|
|
@@ -110,10 +91,6 @@ program
|
|
|
110
91
|
console.error(`\nPlease check the path and try again.`);
|
|
111
92
|
process.exit(1);
|
|
112
93
|
}
|
|
113
|
-
if (!await ensureChromaRunning()) {
|
|
114
|
-
console.error('Error: ChromaDB is required for indexing.');
|
|
115
|
-
process.exit(1);
|
|
116
|
-
}
|
|
117
94
|
const result = await indexFolder(folderPath);
|
|
118
95
|
if (!result.success) {
|
|
119
96
|
console.error('\nIndexing failed.');
|
|
@@ -126,11 +103,12 @@ program
|
|
|
126
103
|
});
|
|
127
104
|
program
|
|
128
105
|
.command('query')
|
|
129
|
-
.description('Query indexed documents using vector
|
|
106
|
+
.description('Query indexed documents using hybrid, vector, or BM25 search')
|
|
130
107
|
.argument('<question>', 'Question to search for in indexed documents')
|
|
131
108
|
.option('-t, --top <number>', 'Number of results to return', '5')
|
|
132
109
|
.option('-m, --metadata', 'Include metadata in results (source, score, chunk index)')
|
|
133
110
|
.option('-c, --collection <name>', 'Search in a specific collection (default: all collections)')
|
|
111
|
+
.option('--mode <mode>', 'Search mode: auto, hybrid, vector, or bm25 (default: auto)')
|
|
134
112
|
.action(async (question, options) => {
|
|
135
113
|
const apiKey = await getOpenAIKey();
|
|
136
114
|
if (!apiKey) {
|
|
@@ -150,11 +128,8 @@ program
|
|
|
150
128
|
top,
|
|
151
129
|
includeMetadata: !!options.metadata,
|
|
152
130
|
collection: options.collection,
|
|
131
|
+
mode: options.mode,
|
|
153
132
|
};
|
|
154
|
-
if (!await ensureChromaRunning()) {
|
|
155
|
-
console.error('Error: ChromaDB is required for querying.');
|
|
156
|
-
process.exit(1);
|
|
157
|
-
}
|
|
158
133
|
try {
|
|
159
134
|
await executeQuery(question, queryOptions);
|
|
160
135
|
}
|
|
@@ -170,15 +145,10 @@ program
|
|
|
170
145
|
});
|
|
171
146
|
program
|
|
172
147
|
.command('collections')
|
|
173
|
-
.description('List all
|
|
148
|
+
.description('List all collections with document counts')
|
|
174
149
|
.action(async () => {
|
|
175
|
-
if (!await ensureChromaRunning()) {
|
|
176
|
-
console.error('Error: ChromaDB is required to list collections.');
|
|
177
|
-
process.exit(1);
|
|
178
|
-
}
|
|
179
150
|
try {
|
|
180
|
-
|
|
181
|
-
const collections = await listCollections();
|
|
151
|
+
const collections = listCollections();
|
|
182
152
|
if (collections.length === 0) {
|
|
183
153
|
console.log('No collections found.');
|
|
184
154
|
process.exit(0);
|
|
@@ -202,17 +172,12 @@ program
|
|
|
202
172
|
});
|
|
203
173
|
program
|
|
204
174
|
.command('delete')
|
|
205
|
-
.description('Delete a
|
|
175
|
+
.description('Delete a collection')
|
|
206
176
|
.argument('<collectionName>', 'Name of the collection to delete')
|
|
207
177
|
.option('-f, --force', 'Skip confirmation prompt')
|
|
208
178
|
.action(async (collectionName, options) => {
|
|
209
|
-
if (!await ensureChromaRunning()) {
|
|
210
|
-
console.error('Error: ChromaDB is required to delete collections.');
|
|
211
|
-
process.exit(1);
|
|
212
|
-
}
|
|
213
179
|
try {
|
|
214
|
-
|
|
215
|
-
const collections = await listCollections();
|
|
180
|
+
const collections = listCollections();
|
|
216
181
|
const collectionExists = collections.some(c => c.name === collectionName);
|
|
217
182
|
if (!collectionExists) {
|
|
218
183
|
console.error(`Error: Collection '${collectionName}' not found`);
|
|
@@ -231,7 +196,7 @@ program
|
|
|
231
196
|
process.exit(0);
|
|
232
197
|
}
|
|
233
198
|
}
|
|
234
|
-
|
|
199
|
+
deleteCollection(collectionName);
|
|
235
200
|
console.log(`Collection '${collectionName}' deleted successfully.`);
|
|
236
201
|
}
|
|
237
202
|
catch (error) {
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAEA,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACnD,OAAO,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AACjD,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AAChD,OAAO,EAAE,eAAe,EAAE,gBAAgB,EAAE,MAAM,yBAAyB,CAAC;AAC5E,OAAO,EAAE,IAAI,EAAE,MAAM,oBAAoB,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AACjD,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,aAAa,CAAC;AAE7B,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAE9B,OAAO;KACJ,IAAI,CAAC,UAAU,CAAC;KAChB,WAAW,CAAC,2DAA2D,CAAC;KACxE,OAAO,CAAC,OAAO,CAAC,CAAC;AAEpB,OAAO;KACJ,OAAO,CAAC,MAAM,CAAC;KACf,WAAW,CAAC,uEAAuE,CAAC;KACpF,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,IAAI,EAAE,CAAC;IACf,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,2EAA2E,CAAC;KACxF,QAAQ,CAAC,YAAY,EAAE,2BAA2B,CAAC;KACnD,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;IACzB,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;QAC1B,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;QAE7C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,OAAO,CAAC,GAAG,CAAC,YAAY,MAAM,CAAC,MAAM,mBAAmB,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACtF,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,EAAE;gBAC5B,OAAO,CAAC,GAAG,CAAC,gBAAgB,KAAK,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7C,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC,CAAC;gBACrD,OAAO,CAAC,GAAG,CAAC,WAAW,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;gBAClD,OAAO,CAAC,GAAG,CAAC,iBAAiB,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;gBACzD,OAAO,CAAC,GAAG,CAAC,eAAe,GAAG,CAAC,OAAO,IAAI,CAAC,CAAC;YAC9C,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,0BAA0B,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YACpE,OAAO,CAAC,GAAG,CAAC,WAAW,MAAM,CAAC,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,iBAAiB,MAAM,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC;YAC5D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,eAAe,MAAM,CAAC,OAAO,IAAI,CAAC,CAAC;QACjD,CAAC;IACH,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,qEAAqE,CAAC;KAClF,QAAQ,CAAC,UAAU,EAAE,kDAAkD,CAAC;KACxE,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;IACvB,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IAExC,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,KAAK,CAAC,4BAA4B,MAAM,EAAE,CAAC,CAAC;QACpD,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG,MAAM,WAAW,CAAC,UAAU,CAAC,CAAC;IAE7C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,OAAO,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACpC,IAAI,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC,CAAC;QACpE,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,OAAO,CAAC;KAChB,WAAW,CAAC,8DAA8D,CAAC;KAC3E,QAAQ,CAAC,YAAY,EAAE,6CAA6C,CAAC;KACrE,MAAM,CAAC,oBAAoB,EAAE,6BAA6B,EAAE,GAAG,CAAC;KAChE,MAAM,CAAC,gBAAgB,EAAE,0DAA0D,CAAC;KACpF,MAAM,CAAC,yBAAyB,EAAE,4DAA4D,CAAC;KAC/F,MAAM,CAAC,eAAe,EAAE,4DAA4D,CAAC;KACrF,MAAM,CAAC,KAAK,EAAE,QAAQ,EAAE,OAAO,EAAE,EAAE;IAClC,MAAM,MAAM,GAAG,MAAM,YAAY,EAAE,CAAC;IACpC,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACnD,OAAO,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;QACxC,OAAO,CAAC,KAAK,CAAC,yBAAyB,CAAC,CAAC;QACzC,OAAO,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;QAChD,OAAO,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;QAC9D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;IACtC,IAAI,KAAK,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QAC3B,OAAO,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAC;QACxD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IAED,MAAM,YAAY,GAAG;QACnB,GAAG;QACH,eAAe,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ;QACnC,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;KACnB,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;IAC7C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;QACjD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,aAAa,CAAC;KACtB,WAAW,CAAC,2CAA2C,CAAC;KACxD,MAAM,CAAC,KAAK,IAAI,EAAE;IACjB,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;QAEtC,IAAI,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC7B,OAAO,CAAC,GAAG,CAAC,uBAAuB,CAAC,CAAC;YACrC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC;QAC9B,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,GAAG,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,UAAU,CAAC,KAAK,YAAY,CAAC,CAAC;QAC7E,CAAC;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,mCAAmC,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO;KACJ,OAAO,CAAC,QAAQ,CAAC;KACjB,WAAW,CAAC,qBAAqB,CAAC;KAClC,QAAQ,CAAC,kBAAkB,EAAE,kCAAkC,CAAC;KAChE,MAAM,CAAC,aAAa,EAAE,0BAA0B,CAAC;KACjD,MAAM,CAAC,KAAK,EAAE,cAAc,EAAE,OAAO,EAAE,EAAE;IACxC,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,eAAe,EAAE,CAAC;QACtC,MAAM,gBAAgB,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,cAAc,CAAC,CAAC;QAE1E,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,OAAO,CAAC,KAAK,CAAC,sBAAsB,cAAc,aAAa,CAAC,CAAC;YACjE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAClB,CAAC;QAED,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;YACnB,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC,CAAC;YACxD,MAAM,EAAE,GAAG,QAAQ,CAAC,eAAe,CAAC;gBAClC,KAAK,EAAE,OAAO,CAAC,KAAK;gBACpB,MAAM,EAAE,OAAO,CAAC,MAAM;aACvB,CAAC,CAAC;YAEH,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,+CAA+C,cAAc,YAAY,CAAC,CAAC;YAC5G,EAAE,CAAC,KAAK,EAAE,CAAC;YAEX,IAAI,MAAM,CAAC,WAAW,EAAE,KAAK,GAAG,EAAE,CAAC;gBACjC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YAClB,CAAC;QACH,CAAC;QAED,gBAAgB,CAAC,cAAc,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,eAAe,cAAc,yBAAyB,CAAC,CAAC;IACtE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;YAC3B,OAAO,CAAC,KAAK,CAAC,UAAU,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3C,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CAAC,oCAAoC,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;AACH,CAAC,CAAC,CAAC;AAEL,OAAO,CAAC,KAAK,EAAE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAc,MAAM,YAAY,CAAC;AAa1D,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CA8H1E"}
|
|
@@ -2,7 +2,9 @@ import { scanFolder } from './scanner.js';
|
|
|
2
2
|
import { parseDocument } from '../parsers/index.js';
|
|
3
3
|
import { chunkDocument } from '../chunker/index.js';
|
|
4
4
|
import { generateEmbeddings } from '../embeddings/index.js';
|
|
5
|
-
import { getOrCreateCollection, storeEmbeddings,
|
|
5
|
+
import { getOrCreateCollection, storeEmbeddings, ensureCollectionsDirectory } from '../vector-store/index.js';
|
|
6
|
+
import { generateSparseEmbedding } from '../vector-store/tokenizer.js';
|
|
7
|
+
import { getEmbeddingModel } from '../config/index.js';
|
|
6
8
|
import path from 'path';
|
|
7
9
|
import crypto from 'crypto';
|
|
8
10
|
import fs from 'fs/promises';
|
|
@@ -25,7 +27,7 @@ export async function indexFolder(folderPath) {
|
|
|
25
27
|
}
|
|
26
28
|
const folderName = path.basename(folderPath);
|
|
27
29
|
try {
|
|
28
|
-
|
|
30
|
+
ensureCollectionsDirectory();
|
|
29
31
|
}
|
|
30
32
|
catch (error) {
|
|
31
33
|
stats.errors.push(error instanceof Error ? error.message : String(error));
|
|
@@ -35,9 +37,10 @@ export async function indexFolder(folderPath) {
|
|
|
35
37
|
collectionName: '',
|
|
36
38
|
};
|
|
37
39
|
}
|
|
40
|
+
const embeddingModel = await getEmbeddingModel();
|
|
38
41
|
let collection;
|
|
39
42
|
try {
|
|
40
|
-
collection =
|
|
43
|
+
collection = getOrCreateCollection(folderName, embeddingModel);
|
|
41
44
|
}
|
|
42
45
|
catch (error) {
|
|
43
46
|
stats.errors.push(error instanceof Error ? error.message : String(error));
|
|
@@ -47,9 +50,22 @@ export async function indexFolder(folderPath) {
|
|
|
47
50
|
collectionName: '',
|
|
48
51
|
};
|
|
49
52
|
}
|
|
50
|
-
|
|
53
|
+
let files;
|
|
54
|
+
try {
|
|
55
|
+
files = await scanFolder(folderPath);
|
|
56
|
+
}
|
|
57
|
+
catch (error) {
|
|
58
|
+
collection.closeSync();
|
|
59
|
+
stats.errors.push(error instanceof Error ? error.message : String(error));
|
|
60
|
+
return {
|
|
61
|
+
success: false,
|
|
62
|
+
stats,
|
|
63
|
+
collectionName: '',
|
|
64
|
+
};
|
|
65
|
+
}
|
|
51
66
|
if (files.length === 0) {
|
|
52
67
|
console.log('No supported files found in the folder.');
|
|
68
|
+
collection.closeSync();
|
|
53
69
|
return {
|
|
54
70
|
success: true,
|
|
55
71
|
stats,
|
|
@@ -71,13 +87,14 @@ export async function indexFolder(folderPath) {
|
|
|
71
87
|
}
|
|
72
88
|
const texts = chunks.map((c) => c.content);
|
|
73
89
|
const embeddings = await generateEmbeddings(texts);
|
|
74
|
-
const
|
|
90
|
+
const sparseChunks = chunks.map((chunk, idx) => ({
|
|
75
91
|
id: crypto.createHash('md5').update(`${filePath}:${idx}`).digest('hex'),
|
|
76
92
|
embedding: embeddings[idx],
|
|
93
|
+
sparseEmbedding: generateSparseEmbedding(chunk.content),
|
|
77
94
|
metadata: chunk.metadata,
|
|
78
95
|
content: chunk.content,
|
|
79
96
|
}));
|
|
80
|
-
|
|
97
|
+
storeEmbeddings(collection, sparseChunks);
|
|
81
98
|
stats.filesProcessed++;
|
|
82
99
|
stats.chunksIndexed += chunks.length;
|
|
83
100
|
console.log(` Indexed ${chunks.length} chunk(s)`);
|
|
@@ -89,6 +106,7 @@ export async function indexFolder(folderPath) {
|
|
|
89
106
|
stats.filesSkipped++;
|
|
90
107
|
}
|
|
91
108
|
}
|
|
109
|
+
collection.closeSync();
|
|
92
110
|
console.log(`\nIndexing complete!`);
|
|
93
111
|
console.log(`Files processed: ${stats.filesProcessed}`);
|
|
94
112
|
console.log(`Files skipped: ${stats.filesSkipped}`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/indexer/orchestrator.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,0BAA0B,EAAE,MAAM,0BAA0B,CAAC;AAE9G,OAAO,EAAE,uBAAuB,EAAE,MAAM,8BAA8B,CAAC;AACvE,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,MAAM,MAAM,QAAQ,CAAC;AAC5B,OAAO,EAAE,MAAM,aAAa,CAAC;AAE7B,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,KAAK,GAAe;QACxB,cAAc,EAAE,CAAC;QACjB,YAAY,EAAE,CAAC;QACf,aAAa,EAAE,CAAC;QAChB,MAAM,EAAE,EAAE;KACX,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IAC9B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;IAE7C,IAAI,CAAC;QACH,0BAA0B,EAAE,CAAC;IAC/B,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1E,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,MAAM,cAAc,GAAG,MAAM,iBAAiB,EAAE,CAAC;IAEjD,IAAI,UAAU,CAAC;IACf,IAAI,CAAC;QACH,UAAU,GAAG,qBAAqB,CAAC,UAAU,EAAE,cAAc,CAAC,CAAC;IACjE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1E,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,IAAI,KAAe,CAAC;IACpB,IAAI,CAAC;QACH,KAAK,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;IACvC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,UAAU,CAAC,SAAS,EAAE,CAAC;QACvB,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC1E,OAAO;YACL,OAAO,EAAE,KAAK;YACd,KAAK;YACL,cAAc,EAAE,EAAE;SACnB,CAAC;IACJ,CAAC;IAED,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,OAAO,CAAC,GAAG,CAAC,yCAAyC,CAAC,CAAC;QACvD,UAAU,CAAC,SAAS,EAAE,CAAC;QACvB,OAAO;YACL,OAAO,EAAE,IAAI;YACb,KAAK;YACL,cAAc,EAAE,UAAU;SAC3B,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,CAAC,MAAM,qBAAqB,CAAC,CAAC;IAExD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QAC1B,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QACzC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,iBAAiB,QAAQ,EAAE,CAAC,CAAC;QAEpE,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC7C,MAAM,MAAM,GAAsB,aAAa,CAAC,MAAM,CAAC,CAAC;YAExD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,OAAO,CAAC,IAAI,CAAC,uCAAuC,QAAQ,EAAE,CAAC,CAAC;gBAChE,KAAK,CAAC,YAAY,EAAE,CAAC;gBACrB,SAAS;YACX,CAAC;YAED,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;YAEnD,MAAM,YAAY,GAAkB,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC;gBAC9D,EAAE,EAAE,MAAM,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,GAAG,QAAQ,IAAI,GAAG,EAAE,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;gBACvE,SAAS,EAAE,UAAU,CAAC,GAAG,CAAC;gBAC1B,eAAe,EAAE,uBAAuB,CAAC,KAAK,CAAC,OAAO,CAAC;gBACvD,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,OAAO,EAAE,KAAK,CAAC,OAAO;aACvB,CAAC,CAAC,CAAC;YAEJ,eAAe,CAAC,UAAU,EAAE,YAAY,CAAC,CAAC;YAE1C,KAAK,CAAC,cAAc,EAAE,CAAC;YACvB,KAAK,CAAC,aAAa,IAAI,MAAM,CAAC,MAAM,CAAC;YACrC,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;QACrD,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,YAAY,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC5E,OAAO,CAAC,IAAI,CAAC,gCAAgC,QAAQ,KAAK,YAAY,EAAE,CAAC,CAAC;YAC1E,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,QAAQ,KAAK,YAAY,EAAE,CAAC,CAAC;YAClD,KAAK,CAAC,YAAY,EAAE,CAAC;QACvB,CAAC;IACH,CAAC;IAED,UAAU,CAAC,SAAS,EAAE,CAAC;IAEvB,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,oBAAoB,KAAK,CAAC,cAAc,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,CAAC,YAAY,EAAE,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,CAAC,aAAa,EAAE,CAAC,CAAC;IAC5D,OAAO,CAAC,GAAG,CAAC,eAAe,UAAU,EAAE,CAAC,CAAC;IAEzC,IAAI,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAC9D,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAI;QACb,KAAK;QACL,cAAc,EAAE,UAAU;KAC3B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { HybridQueryResult } from './types.js';
|
|
2
|
+
export type { HybridQueryResult } from './types.js';
|
|
3
|
+
export interface HybridSearchOptions {
|
|
4
|
+
top: number;
|
|
5
|
+
vectorWeight?: number;
|
|
6
|
+
bm25Weight?: number;
|
|
7
|
+
collection?: string;
|
|
8
|
+
}
|
|
9
|
+
/**
|
|
10
|
+
* Auto search: RRF fusion with equal weights (no configuration needed).
|
|
11
|
+
*/
|
|
12
|
+
export declare function autoSearch(queryEmbedding: number[], queryText: string, options: Omit<HybridSearchOptions, 'vectorWeight' | 'bm25Weight'>): HybridQueryResult[];
|
|
13
|
+
/**
|
|
14
|
+
* Weighted search: RRF fusion with configurable weights.
|
|
15
|
+
*/
|
|
16
|
+
export declare function weightedSearch(queryEmbedding: number[], queryText: string, options: HybridSearchOptions): HybridQueryResult[];
|
|
17
|
+
//# sourceMappingURL=hybrid.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hybrid.d.ts","sourceRoot":"","sources":["../../src/query/hybrid.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAe,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAIjE,YAAY,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAIpD,MAAM,WAAW,mBAAmB;IAClC,GAAG,EAAE,MAAM,CAAC;IACZ,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED;;GAEG;AACH,wBAAgB,UAAU,CACxB,cAAc,EAAE,MAAM,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,IAAI,CAAC,mBAAmB,EAAE,cAAc,GAAG,YAAY,CAAC,GAChE,iBAAiB,EAAE,CAMrB;AAED;;GAEG;AACH,wBAAgB,cAAc,CAC5B,cAAc,EAAE,MAAM,EAAE,EACxB,SAAS,EAAE,MAAM,EACjB,OAAO,EAAE,mBAAmB,GAC3B,iBAAiB,EAAE,CAMrB"}
|