@vivantel/virage-core 0.2.2 โ 0.2.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +4 -1
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/experiment.d.ts.map +1 -1
- package/dist/cli/experiment.js +4 -1
- package/dist/cli/experiment.js.map +1 -1
- package/dist/cli/file-detect.js +1 -1
- package/dist/cli/file-detect.js.map +1 -1
- package/dist/cli/validate.d.ts.map +1 -1
- package/dist/cli/validate.js.map +1 -1
- package/dist/core/chunk-processor.d.ts +1 -1
- package/dist/core/chunk-processor.d.ts.map +1 -1
- package/dist/core/chunk-processor.js +3 -1
- package/dist/core/chunk-processor.js.map +1 -1
- package/dist/core/embedder.d.ts +4 -0
- package/dist/core/embedder.d.ts.map +1 -1
- package/dist/core/embedder.js +15 -4
- package/dist/core/embedder.js.map +1 -1
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +11 -1
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/utils.d.ts +2 -0
- package/dist/core/utils.d.ts.map +1 -1
- package/dist/core/utils.js +24 -0
- package/dist/core/utils.js.map +1 -1
- package/dist/eval/runner.d.ts +1 -1
- package/dist/eval/runner.d.ts.map +1 -1
- package/dist/eval/runner.js +4 -2
- package/dist/eval/runner.js.map +1 -1
- package/dist/logger/consola-logger.d.ts.map +1 -1
- package/dist/logger/consola-logger.js +12 -2
- package/dist/logger/consola-logger.js.map +1 -1
- package/dist/plugin-registry.d.ts.map +1 -1
- package/dist/plugin-registry.js +0 -13
- package/dist/plugin-registry.js.map +1 -1
- package/dist/progress/progress-bar.d.ts +6 -0
- package/dist/progress/progress-bar.d.ts.map +1 -0
- package/dist/progress/progress-bar.js +23 -0
- package/dist/progress/progress-bar.js.map +1 -0
- package/package.json +3 -1
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"evaluate.d.ts","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAOA,MAAM,WAAW,eAAe;IAC9B,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;IAChB,YAAY,EAAE,OAAO,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,EAAE,EAAE,OAAO,CAAC;CACb;AAkBD,wBAAsB,WAAW,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,IAAI,CAAC,CAwDtE"}
|
package/dist/cli/evaluate.js
CHANGED
|
@@ -2,6 +2,7 @@ import { loadConfig } from "../config-loader.js";
|
|
|
2
2
|
import { loadEvalDataset } from "../eval/dataset-io.js";
|
|
3
3
|
import { EvalRunner } from "../eval/runner.js";
|
|
4
4
|
import { ExperimentStore, makeRunId } from "../eval/experiment-store.js";
|
|
5
|
+
import { createProgressBar } from "../progress/progress-bar.js";
|
|
5
6
|
function formatPercent(n) {
|
|
6
7
|
return `${(n * 100).toFixed(1)}%`;
|
|
7
8
|
}
|
|
@@ -25,7 +26,9 @@ export async function runEvaluate(opts) {
|
|
|
25
26
|
await cfg.vectorStore.initialize();
|
|
26
27
|
console.log("๐ Running retrieval evaluation...");
|
|
27
28
|
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
28
|
-
const
|
|
29
|
+
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
30
|
+
const { evalResult, perQueryRrScores } = await runner.run((done, total) => evalBar.update(done < total ? done : total));
|
|
31
|
+
evalBar.stop();
|
|
29
32
|
printEvalResult(evalResult);
|
|
30
33
|
if (opts.withLlmJudge) {
|
|
31
34
|
console.log("\nโ๏ธ LLM-as-judge (RAGAS) requires a judge configured in config.");
|
package/dist/cli/evaluate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;
|
|
1
|
+
{"version":3,"file":"evaluate.js","sourceRoot":"","sources":["../../src/cli/evaluate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAWhE,SAAS,aAAa,CAAC,CAAS;IAC9B,OAAO,GAAG,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACpC,CAAC;AAED,SAAS,eAAe,CAAC,MAAkB;IACzC,OAAO,CAAC,GAAG,CAAC,mCAAmC,CAAC,CAAC;IACjD,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,gBAAgB,EAAE,CAAC,CAAC;IAChE,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,YAAY,CAAC,EAAE,CAAC,CAAC;IAC3E,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,yBAAyB,MAAM,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IAC9D,OAAO,CAAC,GAAG,CAAC,yBAAyB,aAAa,CAAC,MAAM,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,IAAqB;IACrD,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpD,OAAO,CAAC,GAAG,CAAC,YAAY,OAAO,CAAC,OAAO,CAAC,MAAM,UAAU,CAAC,CAAC;IAE1D,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACxE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC5C,CAAC;IACF,OAAO,CAAC,IAAI,EAAE,CAAC;IAEf,eAAe,CAAC,UAAU,CAAC,CAAC;IAE5B,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,mEAAmE,CACpE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,iEAAiE,CAClE,CAAC;IACJ,CAAC;IAED,2BAA2B;IAC3B,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAkB;QACzB,EAAE,EAAE,SAAS,CAAC,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM;QACZ,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;QAC1D,UAAU;QACV,gBAAgB;KACjB,CAAC;IACF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,0BAA0B,SAAS,EAAE,CAAC,CAAC;IAEnD,kBAAkB;IAClB,IAAI,IAAI,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,IAAI,IAAI,CAAC,YAAY,CAAC;QACnD,IAAI,MAAM,EAAE,CAAC;YACX,OAAO,CAAC,GAAG,CACT,gCAAgC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACnF,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,KAAK,CACX,gCAAgC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,YAAY,EAAE,CACnF,CAAC;YACF,IAAI,IAAI,CAAC,EAAE;gBAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"experiment.d.ts","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAQA,MAAM,WAAW,oBAAoB;IACnC,IAAI,EAAE,MAAM,CAAC;IACb,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,WAAW,wBAAwB;IACvC,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,oBAAoB,GACzB,OAAO,CAAC,IAAI,CAAC,CAmCf;AAED,wBAAsB,iBAAiB,IAAI,OAAO,CAAC,IAAI,CAAC,CAgCvD;AAED,wBAAsB,oBAAoB,CACxC,IAAI,EAAE,wBAAwB,GAC7B,OAAO,CAAC,IAAI,CAAC,CAoEf"}
|
package/dist/cli/experiment.js
CHANGED
|
@@ -2,6 +2,7 @@ import { loadConfig } from "../config-loader.js";
|
|
|
2
2
|
import { loadEvalDataset } from "../eval/dataset-io.js";
|
|
3
3
|
import { EvalRunner } from "../eval/runner.js";
|
|
4
4
|
import { ExperimentStore, makeRunId } from "../eval/experiment-store.js";
|
|
5
|
+
import { createProgressBar } from "../progress/progress-bar.js";
|
|
5
6
|
import { bootstrapPairedTest } from "../eval/statistics.js";
|
|
6
7
|
export async function runExperimentRun(opts) {
|
|
7
8
|
console.log(`๐งช Experiment: "${opts.name}"`);
|
|
@@ -12,7 +13,9 @@ export async function runExperimentRun(opts) {
|
|
|
12
13
|
await cfg.vectorStore.initialize();
|
|
13
14
|
console.log("๐ Running evaluation...");
|
|
14
15
|
const runner = new EvalRunner(cfg.vectorStore, cfg.embedder, dataset);
|
|
15
|
-
const
|
|
16
|
+
const evalBar = createProgressBar("Evaluating", dataset.queries.length);
|
|
17
|
+
const { evalResult, perQueryRrScores } = await runner.run((done, total) => evalBar.update(done < total ? done : total));
|
|
18
|
+
evalBar.stop();
|
|
16
19
|
console.log(`\n MRR: ${evalResult.mrr.toFixed(4)}`);
|
|
17
20
|
console.log(` Precision@5: ${(evalResult.precisionAt5 * 100).toFixed(1)}%`);
|
|
18
21
|
console.log(` Recall@10: ${(evalResult.recallAt10 * 100).toFixed(1)}%`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAc5D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAA0B;IAE1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpD,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"experiment.js","sourceRoot":"","sources":["../../src/cli/experiment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAC;AAC/C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,6BAA6B,CAAC;AACzE,OAAO,EAAE,iBAAiB,EAAE,MAAM,6BAA6B,CAAC;AAChE,OAAO,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAc5D,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,IAA0B;IAE1B,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC;IAC7C,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,MAAM,GAAG,GAAG,MAAM,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,iCAAiC,IAAI,CAAC,OAAO,MAAM,CAAC,CAAC;IACjE,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAEpD,MAAM,GAAG,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;IAEnC,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,GAAG,CAAC,WAAW,EAAE,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IACtE,MAAM,OAAO,GAAG,iBAAiB,CAAC,YAAY,EAAE,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;IACxE,MAAM,EAAE,UAAU,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CACxE,OAAO,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC5C,CAAC;IACF,OAAO,CAAC,IAAI,EAAE,CAAC;IAEf,OAAO,CAAC,GAAG,CAAC,YAAY,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACrD,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,UAAU,CAAC,YAAY,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAC7E,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IACzE,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAEzE,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,GAAG,GAAkB;QACzB,EAAE,EAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;QACxB,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;QACnC,MAAM,EAAE,EAAE,UAAU,EAAE,IAAI,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE;QAC1D,UAAU;QACV,gBAAgB;KACjB,CAAC;IAEF,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACxC,OAAO,CAAC,GAAG,CAAC,oBAAoB,GAAG,CAAC,EAAE,eAAe,SAAS,EAAE,CAAC,CAAC;AACpE,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,iBAAiB;IACrC,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IACpC,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC;IAEhC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACT,sEAAsE,CACvE,CAAC;QACF,OAAO;IACT,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACpC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,KAAK,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,WAAW,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAC1E,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAE5B,eAAe;IACf,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAC3B,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAC5E,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CACT,KAAK,GAAG,CAAC,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAClG,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,MAAM,eAAe,CAAC,CAAC;AAC/C,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACxC,IAA8B;IAE9B,MAAM,KAAK,GAAG,IAAI,eAAe,EAAE,CAAC;IAEpC,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAEnD,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC1C,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,iBAAiB,QAAQ,CAAC,EAAE,UAAU,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC5E,CAAC;IACF,OAAO,CAAC,GAAG,CACT,iBAAiB,SAAS,CAAC,EAAE,UAAU,SAAS,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC9E,CAAC;IAEF,IAAI,CAAC,QAAQ,CAAC,gBAAgB,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,CAAC;QAC9D,MAAM,KAAK,GAAG,SAAS,CAAC,UAAU,CAAC,GAAG,GAAG,QAAQ,CAAC,UAAU,CAAC,GAAG,CAAC;QACjE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,kBAAkB,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QAC1E,OAAO,CAAC,GAAG,CACT,+EAA+E,CAChF,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5B,OAAO;IACT,CAAC;IAED,IAAI,QAAQ,CAAC,gBAAgB,CAAC,MAAM,KAAK,SAAS,CAAC,gBAAgB,CAAC,MAAM,EAAE,CAAC;QAC3E,OAAO,CAAC,IAAI,CACV,6BAA6B,QAAQ,CAAC,gBAAgB,CAAC,MAAM,MAAM;YACjE,GAAG,SAAS,CAAC,gBAAgB,CAAC,MAAM,uBAAuB,CAC9D,CAAC;IACJ,CAAC;IAED,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,CAChB,QAAQ,CAAC,gBAAgB,CAAC,MAAM,EAChC,SAAS,CAAC,gBAAgB,CAAC,MAAM,CAClC,CAAC;IAEF,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACV,OAAO,CAAC,GAAG,CAAC,6DAA6D,CAAC,CAAC;QAC3E,OAAO;IACT,CAAC;IAED,MAAM,IAAI,GAAG,mBAAmB,CAC9B,QAAQ,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EACrC,SAAS,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CACvC,CAAC;IAEF,MAAM,YAAY,GAChB,IAAI,CAAC,cAAc,KAAK,QAAQ;QAC9B,CAAC,CAAC,GAAG;QACL,CAAC,CAAC,IAAI,CAAC,cAAc,KAAK,QAAQ;YAChC,CAAC,CAAC,GAAG;YACL,CAAC,CAAC,KAAK,CAAC;IAEd,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5B,OAAO,CAAC,GAAG,CACT,kBAAkB,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAC7E,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,kBAAkB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;IACxD,OAAO,CAAC,GAAG,CACT,mBAAmB,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,oBAAoB,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAC1G,CAAC;IACF,OAAO,CAAC,GAAG,CACT,kBAAkB,YAAY,IAAI,IAAI,CAAC,cAAc,CAAC,WAAW,EAAE,EAAE,CACtE,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;AAC9B,CAAC"}
|
package/dist/cli/file-detect.js
CHANGED
|
@@ -36,7 +36,7 @@ export async function collectExtensions(dir, found) {
|
|
|
36
36
|
}
|
|
37
37
|
for (const entry of entries) {
|
|
38
38
|
if (entry.isDirectory()) {
|
|
39
|
-
if (!IGNORED_DIRS.has(entry.name)) {
|
|
39
|
+
if (!IGNORED_DIRS.has(entry.name) && !entry.name.startsWith(".")) {
|
|
40
40
|
await collectExtensions(join(dir, entry.name), found);
|
|
41
41
|
}
|
|
42
42
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"file-detect.js","sourceRoot":"","sources":["../../src/cli/file-detect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAQrC,MAAM,CAAC,MAAM,UAAU,GAAe;IACpC;QACE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,yBAAyB;QACrC,IAAI,EAAE,UAAU;KACjB;IACD,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE;IAC1D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE;CACjE,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAClC,cAAc;IACd,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,KAAkB;IAElB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;
|
|
1
|
+
{"version":3,"file":"file-detect.js","sourceRoot":"","sources":["../../src/cli/file-detect.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,aAAa,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAQrC,MAAM,CAAC,MAAM,UAAU,GAAe;IACpC;QACE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC;QACrB,UAAU,EAAE,yBAAyB;QACrC,IAAI,EAAE,UAAU;KACjB;IACD,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,YAAY,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,IAAI,EAAE;IAC1D,EAAE,IAAI,EAAE,CAAC,KAAK,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,QAAQ,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,CAAC,EAAE,UAAU,EAAE,eAAe,EAAE,IAAI,EAAE,MAAM,EAAE;IAC9D,EAAE,IAAI,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,EAAE,UAAU,EAAE,mBAAmB,EAAE,IAAI,EAAE,MAAM,EAAE;IAC1E,EAAE,IAAI,EAAE,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,kBAAkB,EAAE,IAAI,EAAE,MAAM,EAAE;CACjE,CAAC;AAEF,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAClC,cAAc;IACd,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,GAAW,EACX,KAAkB;IAElB,IAAI,OAAO,CAAC;IACZ,IAAI,CAAC;QACH,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,EAAE,EAAE,aAAa,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO;IACT,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;QAC5B,IAAI,KAAK,CAAC,WAAW,EAAE,EAAE,CAAC;YACxB,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjE,MAAM,iBAAiB,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,CAAC,EAAE,KAAK,CAAC,CAAC;YACxD,CAAC;QACH,CAAC;aAAM,CAAC;YACN,MAAM,GAAG,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAC9C,IAAI,GAAG;gBAAE,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAC1B,CAAC;IACH,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,oBAAoB,CAAC,GAAW;IACpD,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,iBAAiB,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IACpC,OAAO,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACpE,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAMA,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"validate.d.ts","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAMA,wBAAsB,WAAW,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CA8EnE"}
|
package/dist/cli/validate.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,+BAA+B,UAAU,IAAI,CAAC,CAAC;IAE3D,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC9C,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,WAAW,CACnB,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC5E;YACE,UAAU,EAAE,kDAAkD;YAC9D,KAAK,EAAE,GAAG;SACX,CACF,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,eAAe,eAAe,GAAG,CAAC,CAAC;QAEzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;IACvD,OAAO,CAAC,GAAG,CACT,YAAY,QAAQ,CAAC,MAAM,2BAA2B,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAC1F,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACvD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CACtC,WAAW,CAAC,IAAI,
|
|
1
|
+
{"version":3,"file":"validate.js","sourceRoot":"","sources":["../../src/cli/validate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACjD,OAAO,EAAE,UAAU,EAAE,MAAM,wBAAwB,CAAC;AACpD,OAAO,EAAE,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAChD,OAAO,EAAE,oBAAoB,EAAE,MAAM,kBAAkB,CAAC;AAExD,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,UAAkB;IAClD,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,+BAA+B,UAAU,IAAI,CAAC,CAAC;IAE3D,IAAI,MAAM,CAAC;IACX,IAAI,CAAC;QACH,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,uBAAuB,CAAC,CAAC;QAC9C,MAAM,GAAG,MAAM,UAAU,CAAC,UAAU,CAAC,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACjB,MAAM,IAAI,WAAW,CACnB,0BAA0B,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC5E;YACE,UAAU,EAAE,kDAAkD;YAC9D,KAAK,EAAE,GAAG;SACX,CACF,CAAC;IACJ,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;IACxC,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC;QACnC,MAAM,eAAe,GAAG,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACpD,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,CAAC,IAAI,eAAe,eAAe,GAAG,CAAC,CAAC;QAEzE,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,gDAAgD,CAAC,CAAC;YAC9D,WAAW,GAAG,IAAI,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,aAAa,MAAM,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAC/D,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,kBAAkB,EAAE,CAAC;IACvD,OAAO,CAAC,GAAG,CACT,YAAY,QAAQ,CAAC,MAAM,2BAA2B,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAC1F,CAAC;IAEF,OAAO,CAAC,GAAG,CAAC,+BAA+B,CAAC,CAAC;IAC7C,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QACtC,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;QAC3D,WAAW,GAAG,IAAI,CAAC;IACrB,CAAC;IAED,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;IACnD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IACvD,IAAI,cAAc,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAChC,OAAO,CAAC,GAAG,CAAC,+CAA+C,CAAC,CAAC;IAC/D,CAAC;SAAM,CAAC;QACN,MAAM,WAAW,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QAC/D,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CACtC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,GAAG,GAAG,EAAE,CAAC,CAAC,CACvE,CAAC;YACF,IAAI,OAAO,EAAE,CAAC;gBACZ,OAAO,CAAC,GAAG,CAAC,OAAO,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;YACxE,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,GAAG,CACT,SAAS,KAAK,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,gCAAgC,CAC9E,CAAC;gBACF,WAAW,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,WAAW,EAAE,CAAC;QAChB,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IACpD,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,sBAAsB,CAAC,CAAC;IACtC,CAAC;AACH,CAAC"}
|
|
@@ -8,7 +8,7 @@ export declare class ChunkProcessor {
|
|
|
8
8
|
processFiles(files: string[], fileState: Map<string, {
|
|
9
9
|
commitHash: string;
|
|
10
10
|
chunker: FileChunker;
|
|
11
|
-
}>, existingChunks?: Chunk[]): Promise<Chunk[]>;
|
|
11
|
+
}>, existingChunks?: Chunk[], onProgress?: (completed: number, total: number) => void): Promise<Chunk[]>;
|
|
12
12
|
saveChunksLocal(chunks: Chunk[], outputFile: string): Promise<void>;
|
|
13
13
|
}
|
|
14
14
|
//# sourceMappingURL=chunk-processor.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-processor.d.ts","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAQtD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAA2B;IAC3C,OAAO,CAAC,MAAM,CAAS;gBAEX,QAAQ,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM;IAK9C,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAYb,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,EACpE,cAAc,GAAE,KAAK,EAAO,
|
|
1
|
+
{"version":3,"file":"chunk-processor.d.ts","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC5D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAQtD,qBAAa,cAAc;IACzB,OAAO,CAAC,QAAQ,CAA2B;IAC3C,OAAO,CAAC,MAAM,CAAS;gBAEX,QAAQ,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,EAAE,MAAM;IAK9C,WAAW,CACf,QAAQ,EAAE,MAAM,EAChB,UAAU,EAAE,MAAM,EAClB,OAAO,EAAE,WAAW,GACnB,OAAO,CAAC,KAAK,EAAE,CAAC;IAYb,YAAY,CAChB,KAAK,EAAE,MAAM,EAAE,EACf,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE;QAAE,UAAU,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,WAAW,CAAA;KAAE,CAAC,EACpE,cAAc,GAAE,KAAK,EAAO,EAC5B,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACtD,OAAO,CAAC,KAAK,EAAE,CAAC;IA6Eb,eAAe,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;CAyB1E"}
|
|
@@ -19,7 +19,7 @@ export class ChunkProcessor {
|
|
|
19
19
|
}
|
|
20
20
|
return chunks;
|
|
21
21
|
}
|
|
22
|
-
async processFiles(files, fileState, existingChunks = []) {
|
|
22
|
+
async processFiles(files, fileState, existingChunks = [], onProgress) {
|
|
23
23
|
const allChunks = [];
|
|
24
24
|
let errorCount = 0;
|
|
25
25
|
// Build resume cache: sourceFile โ { commitHash, chunks }
|
|
@@ -48,6 +48,7 @@ export class ChunkProcessor {
|
|
|
48
48
|
if (cached && cached.commitHash === info.commitHash) {
|
|
49
49
|
this.logger.verbose(`โญ๏ธ Cached [${i + 1}/${files.length}] ${filePath} (${cached.chunks.length} chunk(s))`);
|
|
50
50
|
allChunks.push(...cached.chunks);
|
|
51
|
+
onProgress?.(i + 1, files.length);
|
|
51
52
|
continue;
|
|
52
53
|
}
|
|
53
54
|
this.logger.verbose(`[${i + 1}/${files.length}] ${filePath}`);
|
|
@@ -68,6 +69,7 @@ export class ChunkProcessor {
|
|
|
68
69
|
errorCount++;
|
|
69
70
|
this.logger.error(` โ Error: ${error instanceof Error ? error.message : String(error)}`);
|
|
70
71
|
}
|
|
72
|
+
onProgress?.(i + 1, files.length);
|
|
71
73
|
}
|
|
72
74
|
if (errorCount > 0) {
|
|
73
75
|
this.logger.warn(`โ ๏ธ ${errorCount} file(s) failed during chunking.`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IACnC,MAAM,CAAS;IAEvB,YAAY,QAAuB,EAAE,MAAe;QAClD,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE,EACpE,iBAA0B,EAAE;
|
|
1
|
+
{"version":3,"file":"chunk-processor.js","sourceRoot":"","sources":["../../src/core/chunk-processor.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAEpC,SAAS,kBAAkB,CAAC,OAAe;IACzC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,OAAO,cAAc;IACjB,QAAQ,CAA2B;IACnC,MAAM,CAAS;IAEvB,YAAY,QAAuB,EAAE,MAAe;QAClD,IAAI,CAAC,QAAQ,GAAG,IAAI,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;QAC1D,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/D,CAAC;IAED,KAAK,CAAC,WAAW,CACf,QAAgB,EAChB,UAAkB,EAClB,OAAoB;QAEpB,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;QAEzD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,KAAK,CAAC,WAAW,GAAG,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YACtD,KAAK,CAAC,UAAU,GAAG,QAAQ,CAAC;YAC5B,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC;QAChC,CAAC;QAED,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,YAAY,CAChB,KAAe,EACf,SAAoE,EACpE,iBAA0B,EAAE,EAC5B,UAAuD;QAEvD,MAAM,SAAS,GAAY,EAAE,CAAC;QAC9B,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,0DAA0D;QAC1D,MAAM,WAAW,GAAG,IAAI,GAAG,EAGxB,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAChD,IAAI,CAAC,KAAK,EAAE,CAAC;gBACX,WAAW,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE;oBAChC,UAAU,EAAE,KAAK,CAAC,UAAU;oBAC5B,MAAM,EAAE,CAAC,KAAK,CAAC;iBAChB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,QAAQ,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YAC1B,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAErC,IAAI,CAAC,IAAI,EAAE,CAAC;gBACV,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,EAAE,CAAC,CAAC;gBACnD,SAAS;YACX,CAAC;YAED,sDAAsD;YACtD,MAAM,MAAM,GAAG,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YACzC,IAAI,MAAM,IAAI,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACpD,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,KAAK,MAAM,CAAC,MAAM,CAAC,MAAM,YAAY,CACtF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;gBACjC,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;gBAClC,SAAS;YACX,CAAC;YAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,KAAK,QAAQ,EAAE,CAAC,CAAC;YAE9D,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,WAAW,CACnC,QAAQ,EACR,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,OAAO,CACb,CAAC;gBAEF,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACtB,SAAS,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAC;oBAC1B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,OAAO,MAAM,CAAC,MAAM,WAAW,CAAC,CAAC;oBACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;wBACvC,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,WAAW,CAAC,iBAAiB,MAAM,CAAC,CAAC,CAAC,CAAC,WAAW,QAAQ,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CACrF,CAAC;oBACJ,CAAC;gBACH,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;gBAC/C,CAAC;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,UAAU,EAAE,CAAC;gBACb,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,cAAc,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CACvE,CAAC;YACJ,CAAC;YACD,UAAU,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;YACnB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,UAAU,kCAAkC,CAAC,CAAC;QACvE,CAAC;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,MAAe,EAAE,UAAkB;QACvD,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,MAAM,CAAC,CAAC;QACzC,MAAM,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAEnE,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAEtD,IAAI,QAAQ,GAAY,EAAE,CAAC;QAC3B,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC1B,QAAQ,GAAG,MAAiB,CAAC;YAC/B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,wDAAwD;QAC1D,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC;QAE3E,MAAM,SAAS,GAAG,CAAC,GAAG,QAAQ,EAAE,GAAG,MAAM,CAAC,CAAC;QAE3C,MAAM,SAAS,CAAC,UAAU,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QAChE,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,SAAS,CAAC,MAAM,cAAc,UAAU,EAAE,CAAC,CAAC;IAC3E,CAAC;CACF"}
|
package/dist/core/embedder.d.ts
CHANGED
|
@@ -9,7 +9,9 @@ export declare class EmbedderProcessor {
|
|
|
9
9
|
private retryOptions;
|
|
10
10
|
private concurrency;
|
|
11
11
|
private vectorStoreName?;
|
|
12
|
+
private saveIntervalMs;
|
|
12
13
|
private logger;
|
|
14
|
+
onProgress?: (completed: number, total: number) => void;
|
|
13
15
|
constructor(provider: EmbeddingProvider, options?: {
|
|
14
16
|
rateLimitMs?: number;
|
|
15
17
|
batchSize?: number;
|
|
@@ -17,7 +19,9 @@ export declare class EmbedderProcessor {
|
|
|
17
19
|
retry?: RetryOptions;
|
|
18
20
|
concurrency?: number;
|
|
19
21
|
vectorStoreName?: string;
|
|
22
|
+
saveIntervalMs?: number;
|
|
20
23
|
logger?: Logger;
|
|
24
|
+
onProgress?: (completed: number, total: number) => void;
|
|
21
25
|
});
|
|
22
26
|
embedChunk(chunk: Chunk): Promise<EmbeddedChunk>;
|
|
23
27
|
embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,KAAK,EACN,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAKtD,OAAO,
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,KAAK,EACN,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAKtD,OAAO,EAML,YAAY,EACb,MAAM,YAAY,CAAC;AAQpB,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,MAAM,CAAS;IACvB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;gBAGtD,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACpD;IAcF,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAchD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAoDrD,gBAAgB,CACpB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;QACvB,YAAY,EAAE,cAAc,GAAG,IAAI,CAAC;QACpC,mBAAmB,EAAE,OAAO,CAAC;KAC9B,CAAC;IA4II,cAAc,CAClB,aAAa,EAAE,aAAa,EAAE,EAC9B,UAAU,EAAE,MAAM,EAClB,YAAY,EAAE,cAAc,GAAG,IAAI,EACnC,YAAY,EAAE,OAAO,EACrB,eAAe,EAAE,OAAO,GACvB,OAAO,CAAC,IAAI,CAAC;IAqDV,GAAG,CACP,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC,aAAa,EAAE,CAAC;CAkD5B"}
|
package/dist/core/embedder.js
CHANGED
|
@@ -2,7 +2,7 @@ import { NullLogger } from "../logger/null-logger.js";
|
|
|
2
2
|
import { readFile } from "fs/promises";
|
|
3
3
|
import { createHash } from "crypto";
|
|
4
4
|
import { EmbedError } from "./errors.js";
|
|
5
|
-
import { sleep, withRetry, withConcurrency, batchBySize, } from "./utils.js";
|
|
5
|
+
import { sleep, withRetry, withConcurrency, batchBySize, defaultIsRetryable, } from "./utils.js";
|
|
6
6
|
import { readEmbeddingsFile, writeEmbeddingsFile } from "./embeddings-io.js";
|
|
7
7
|
function chunkContentHash(chunk) {
|
|
8
8
|
if (chunk.contentHash)
|
|
@@ -17,7 +17,9 @@ export class EmbedderProcessor {
|
|
|
17
17
|
retryOptions;
|
|
18
18
|
concurrency;
|
|
19
19
|
vectorStoreName;
|
|
20
|
+
saveIntervalMs;
|
|
20
21
|
logger;
|
|
22
|
+
onProgress;
|
|
21
23
|
constructor(provider, options = {}) {
|
|
22
24
|
this.provider = provider;
|
|
23
25
|
this.rateLimitMs = options.rateLimitMs ?? 500;
|
|
@@ -26,10 +28,12 @@ export class EmbedderProcessor {
|
|
|
26
28
|
this.retryOptions = options.retry ?? {};
|
|
27
29
|
this.concurrency = options.concurrency ?? 1;
|
|
28
30
|
this.vectorStoreName = options.vectorStoreName;
|
|
31
|
+
this.saveIntervalMs = options.saveIntervalMs ?? 30_000;
|
|
32
|
+
this.onProgress = options.onProgress;
|
|
29
33
|
this.logger = (options.logger ?? new NullLogger()).withTag("embedder");
|
|
30
34
|
}
|
|
31
35
|
async embedChunk(chunk) {
|
|
32
|
-
const embedding = await withRetry(() => this.provider.embed(chunk.content), this.retryOptions, this.logger);
|
|
36
|
+
const embedding = await withRetry(() => this.provider.embed(chunk.content), { ...this.retryOptions, isRetryable: defaultIsRetryable }, this.logger);
|
|
33
37
|
return {
|
|
34
38
|
...chunk,
|
|
35
39
|
embedding,
|
|
@@ -39,7 +43,7 @@ export class EmbedderProcessor {
|
|
|
39
43
|
async embedBatch(chunks) {
|
|
40
44
|
if (this.provider.embedBatch && chunks.length > 0) {
|
|
41
45
|
const texts = chunks.map((c) => c.content);
|
|
42
|
-
const embeddings = await withRetry(() => this.provider.embedBatch(texts), this.retryOptions, this.logger);
|
|
46
|
+
const embeddings = await withRetry(() => this.provider.embedBatch(texts), { ...this.retryOptions, isRetryable: defaultIsRetryable }, this.logger);
|
|
43
47
|
if (embeddings.length !== chunks.length) {
|
|
44
48
|
throw new EmbedError(`embedBatch returned ${embeddings.length} embeddings for ${chunks.length} chunks`, {
|
|
45
49
|
suggestion: "Check that your EmbeddingProvider.embedBatch() returns one vector per input text.",
|
|
@@ -233,12 +237,19 @@ export class EmbedderProcessor {
|
|
|
233
237
|
this.logger.info(`๐ Need to embed ${chunksToEmbed.length} chunks`);
|
|
234
238
|
const batches = batchBySize(chunksToEmbed, this.batchSize, (c) => c.content.length, this.maxBatchChars);
|
|
235
239
|
const newEmbeddings = [];
|
|
240
|
+
let lastSaveTime = Date.now();
|
|
236
241
|
for (let i = 0; i < batches.length; i++) {
|
|
237
242
|
const totalChars = batches[i].reduce((s, c) => s + c.content.length, 0);
|
|
238
243
|
this.logger.debug(`Batch ${i + 1}/${batches.length}: ${batches[i].length} chunks, ~${totalChars} chars`);
|
|
239
244
|
const embedded = await this.embedBatch(batches[i]);
|
|
240
245
|
newEmbeddings.push(...embedded);
|
|
241
|
-
|
|
246
|
+
this.onProgress?.(newEmbeddings.length, chunksToEmbed.length);
|
|
247
|
+
const isLastBatch = i === batches.length - 1;
|
|
248
|
+
const elapsed = Date.now() - lastSaveTime;
|
|
249
|
+
if (isLastBatch || elapsed >= this.saveIntervalMs) {
|
|
250
|
+
await this.saveEmbeddings(embedded, chunksFile, existingMeta, i === 0, i === 0 && effectiveForce);
|
|
251
|
+
lastSaveTime = Date.now();
|
|
252
|
+
}
|
|
242
253
|
}
|
|
243
254
|
return newEmbeddings;
|
|
244
255
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,GAEZ,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,MAAM,CAAS;IAEvB,YACE,QAA2B,EAC3B,UAQI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,IAAI,CAAC,YAAY,EACjB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAMtB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YACxD,OAAO;gBACL,aAAa,EAAE,MAAM;gBACrB,YAAY,EAAE,IAAI;gBAClB,mBAAmB,EAAE,IAAI;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,GACxC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAE3C,oFAAoF;QACpF,gDAAgD;QAChD,mFAAmF;QACnF,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,IAAI,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,KAAK,SAAS;gBACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,iBAAiB,GACrB,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAEvD,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBACzC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,OAAO;oBACL,aAAa,EAAE,MAAM;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,IAAI;iBAC1B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,2EAA2E;QAC3E,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,oDAAoD;QACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,+EAA+E;YAC/E,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,gFAAgF;YAChF,4EAA4E;YAC5E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,YAAmC,EACnC,YAAqB,EACrB,eAAwB;QAExB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAE1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,uFAAuF;QACvF,qEAAqE;QACrE,MAAM,eAAe,GAAG,YAAY,IAAI,eAAe,CAAC;QAExD,IAAI,UAAU,GAAoB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAC9B,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAC3C,IAAI,IAAI;gBAAE,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjC,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAmB;YAC3B,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;YAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;gBAC3C,CAAC,CAAC,EAAE,CAAC;YACP,+EAA+E;YAC/E,SAAS,EACP,YAAY,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvE,SAAS,EAAE,GAAG;SACf,CAAC;QAEF,MAAM,mBAAmB,CAAC,cAAc,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,SAAS,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CACxD,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,aAAa,CAAC,MAAM,eAAe,UAAU,CAAC,MAAM,EAAE,CAClE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,mBAAmB,EAAE,GACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjD,mEAAmE;QACnE,MAAM,cAAc,GAAG,KAAK,IAAI,mBAAmB,CAAC;QAEpD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAE1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,MAAM,IAAI,CAAC,cAAc,CACvB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,CAAC,KAAK,CAAC,EACP,CAAC,KAAK,CAAC,IAAI,cAAc,CAC1B,CAAC;QACJ,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,cAAc,CAAS;IACvB,MAAM,CAAS;IACvB,UAAU,CAA8C;IAExD,YACE,QAA2B,EAC3B,UAUI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC;QACvD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAMtB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YACxD,OAAO;gBACL,aAAa,EAAE,MAAM;gBACrB,YAAY,EAAE,IAAI;gBAClB,mBAAmB,EAAE,IAAI;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,GACxC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAE3C,oFAAoF;QACpF,gDAAgD;QAChD,mFAAmF;QACnF,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,IAAI,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,KAAK,SAAS;gBACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,iBAAiB,GACrB,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAEvD,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBACzC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,OAAO;oBACL,aAAa,EAAE,MAAM;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,IAAI;iBAC1B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,2EAA2E;QAC3E,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,oDAAoD;QACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,+EAA+E;YAC/E,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,gFAAgF;YAChF,4EAA4E;YAC5E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,YAAmC,EACnC,YAAqB,EACrB,eAAwB;QAExB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAE1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,uFAAuF;QACvF,qEAAqE;QACrE,MAAM,eAAe,GAAG,YAAY,IAAI,eAAe,CAAC;QAExD,IAAI,UAAU,GAAoB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAC9B,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAC3C,IAAI,IAAI;gBAAE,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjC,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAmB;YAC3B,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;YAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;gBAC3C,CAAC,CAAC,EAAE,CAAC;YACP,+EAA+E;YAC/E,SAAS,EACP,YAAY,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvE,SAAS,EAAE,GAAG;SACf,CAAC;QAEF,MAAM,mBAAmB,CAAC,cAAc,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,SAAS,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CACxD,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,aAAa,CAAC,MAAM,eAAe,UAAU,CAAC,MAAM,EAAE,CAClE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,mBAAmB,EAAE,GACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjD,mEAAmE;QACnE,MAAM,cAAc,GAAG,KAAK,IAAI,mBAAmB,CAAC;QAEpD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAC1C,IAAI,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC1C,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClD,MAAM,IAAI,CAAC,cAAc,CACvB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,CAAC,KAAK,CAAC,EACP,CAAC,KAAK,CAAC,IAAI,cAAc,CAC1B,CAAC;gBACF,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAI1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAyCD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAkM3B"}
|
|
@@ -3,6 +3,7 @@ import { ChunkProcessor } from "./chunk-processor.js";
|
|
|
3
3
|
import { EmbedderProcessor } from "./embedder.js";
|
|
4
4
|
import { Uploader } from "./uploader.js";
|
|
5
5
|
import { TelemetryCollector } from "./telemetry.js";
|
|
6
|
+
import { createProgressBar } from "../progress/progress-bar.js";
|
|
6
7
|
import { NullLogger } from "../logger/null-logger.js";
|
|
7
8
|
import { readFile } from "fs/promises";
|
|
8
9
|
import { readEmbeddingsFile } from "./embeddings-io.js";
|
|
@@ -93,7 +94,9 @@ export class Orchestrator {
|
|
|
93
94
|
const existingChunks = opts.force
|
|
94
95
|
? []
|
|
95
96
|
: await loadExistingChunks(this.chunksFile);
|
|
96
|
-
const
|
|
97
|
+
const chunkBar = createProgressBar("Chunking", toProcess.length);
|
|
98
|
+
const chunks = await chunkProcessor.processFiles(toProcess, fileState, existingChunks, (done, total) => chunkBar.update(done < total ? done : total));
|
|
99
|
+
chunkBar.stop();
|
|
97
100
|
await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
|
|
98
101
|
const chunkDuration = Date.now() - t2;
|
|
99
102
|
logger.verbose(`Chunking done in ${chunkDuration}ms`);
|
|
@@ -110,6 +113,7 @@ export class Orchestrator {
|
|
|
110
113
|
// Step 3: Generate embeddings
|
|
111
114
|
logger.info("๐ข Step 3: Generating embeddings...");
|
|
112
115
|
const t3 = Date.now();
|
|
116
|
+
const embedBars = [];
|
|
113
117
|
const embedder = new EmbedderProcessor(this.config.embedder, {
|
|
114
118
|
rateLimitMs: opts.rateLimitMs,
|
|
115
119
|
batchSize: opts.batchSize,
|
|
@@ -118,8 +122,14 @@ export class Orchestrator {
|
|
|
118
122
|
concurrency: opts.concurrency,
|
|
119
123
|
vectorStoreName: this.config.vectorStore.name,
|
|
120
124
|
logger: opts.logger,
|
|
125
|
+
onProgress: (done, total) => {
|
|
126
|
+
if (embedBars.length === 0)
|
|
127
|
+
embedBars.push(createProgressBar("Embedding", total));
|
|
128
|
+
embedBars[0].update(done < total ? done : total);
|
|
129
|
+
},
|
|
121
130
|
});
|
|
122
131
|
const newEmbeddings = await embedder.run(this.chunksFile, opts.force || false);
|
|
132
|
+
embedBars[0]?.stop();
|
|
123
133
|
const embedDuration = Date.now() - t3;
|
|
124
134
|
logger.verbose(`Embedding done in ${embedDuration}ms`);
|
|
125
135
|
telemetry?.recordEmbedding({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EAAE,iBAAiB,EAAoB,MAAM,6BAA6B,CAAC;AAQlF,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAuBhF,KAAK,UAAU,iBAAiB,CAC9B,cAAsB;IAEtB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,UAAkB;IAClD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,MAA2B,EAC3B,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;AACH,CAAC;AAED,MAAM,OAAO,YAAY;IACf,MAAM,CAAoB;IAC1B,UAAU,CAAS;IACnB,cAAc,CAAS;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;QACpE,IAAI,CAAC,cAAc;YACjB,MAAM,CAAC,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,SAAS,EAAE,KAAK,EAAE,CAAC;QAEnB,IAAI,WAAW,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE9C,MAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,UAAU,mBAAmB,IAAI,CAAC,cAAc,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,CAC3G,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE3C,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;YAExD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK;gBAC9B,CAAC,CAAC,IAAI,GAAG,EAAkB;gBAC3B,CAAC,CAAC,MAAM,iBAAiB,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;YAEjD,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,eAAe,CAC9D,aAAa,EACb,YAAY,CACb,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,wBAAwB,WAAW,IAAI,CAAC,CAAC;YAExD,SAAS,EAAE,iBAAiB,CAAC;gBAC3B,UAAU,EAAE,WAAW;gBACvB,YAAY,EAAE,YAAY,CAAC,IAAI;gBAC/B,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnE,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACtC,OAAO;YACT,CAAC;YAED,MAAM,CAAC,IAAI,CACT,eAAe,SAAS,CAAC,MAAM,gBAAgB,QAAQ,CAAC,MAAM,YAAY,CAC3E,CAAC;YAEF,wCAAwC;YACxC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,cAAc,GAAG,IAAI,cAAc,CACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAGtB,CAAC;YACJ,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,IAAI;oBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK;gBAC/B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,CAC9C,SAAS,EACT,SAAS,EACT,cAAc,EACd,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC9D,CAAC;YACF,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,cAAc,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,oBAAoB,aAAa,IAAI,CAAC,CAAC;YAEtD,SAAS,EAAE,cAAc,CAAC;gBACxB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,SAAS,CAAC,MAAM;gBAChC,eAAe,EAAE,MAAM,CAAC,MAAM;gBAC9B,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;gBAChD,OAAO;YACT,CAAC;YAED,8BAA8B;YAC9B,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACnD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,SAAS,GAAkB,EAAE,CAAC;YACpC,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBAC3D,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI;gBAC7C,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBAC1B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;wBAAE,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;oBAClF,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACnD,CAAC;aACF,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,GAAG,CACtC,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,KAAK,IAAI,KAAK,CACpB,CAAC;YACF,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAErB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,qBAAqB,aAAa,IAAI,CAAC,CAAC;YAEvD,SAAS,EAAE,eAAe,CAAC;gBACzB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,aAAa,CAAC,MAAM;gBACpC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;aACpD,CAAC,CAAC;YAEH,iBAAiB;YACjB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;gBACvD,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAChD,MAAM,QAAQ,CAAC,gBAAgB,CAC7B,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,KAAK,IAAI,KAAK,CACpB,CAAC;gBACJ,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;oBACrD,KAAK,EAAE,IAAI,CAAC,KAAK;oBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;iBACpB,CAAC,CAAC;gBACH,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,CAC/B,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,KAAK,IAAI,KAAK,CACpB,CAAC;gBAEF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;gBAErD,SAAS,EAAE,YAAY,CAAC;oBACtB,UAAU,EAAE,cAAc;oBAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;YAE3C,IAAI,SAAS,EAAE,CAAC;gBACd,SAAS,CAAC,MAAM,EAAE,CAAC;gBACnB,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3C,aAAa,EACb,gBAAgB,CACjB,CAAC;gBACF,MAAM,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;oBAC5D,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;oBAC3C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,MAAM;oBACnC,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,EAAE,MAAM,EAAE,CAAC;YAEpB,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE;oBAC1D,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;oBACvD,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;iBAC5C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;IACH,CAAC;CACF"}
|
package/dist/core/utils.d.ts
CHANGED
|
@@ -5,7 +5,9 @@ export interface RetryOptions {
|
|
|
5
5
|
maxRetries?: number;
|
|
6
6
|
retryDelayMs?: number;
|
|
7
7
|
retryBackoffFactor?: number;
|
|
8
|
+
isRetryable?: (error: unknown) => boolean;
|
|
8
9
|
}
|
|
10
|
+
export declare function defaultIsRetryable(error: unknown): boolean;
|
|
9
11
|
export declare function withRetry<T>(fn: () => Promise<T>, opts?: RetryOptions, logger?: Logger): Promise<T>;
|
|
10
12
|
export declare function withConcurrency<T>(tasks: Array<() => Promise<T>>, limit: number): Promise<T[]>;
|
|
11
13
|
export declare function batchArray<T>(array: T[], batchSize: number): T[][];
|
package/dist/core/utils.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/core/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/core/utils.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAEtD,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE1D;AAED,wBAAgB,KAAK,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAE/C;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,kBAAkB,CAAC,EAAE,MAAM,CAAC;IAC5B,WAAW,CAAC,EAAE,CAAC,KAAK,EAAE,OAAO,KAAK,OAAO,CAAC;CAC3C;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CAyB1D;AAED,wBAAsB,SAAS,CAAC,CAAC,EAC/B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,IAAI,GAAE,YAAiB,EACvB,MAAM,CAAC,EAAE,MAAM,GACd,OAAO,CAAC,CAAC,CAAC,CA2BZ;AAED,wBAAsB,eAAe,CAAC,CAAC,EACrC,KAAK,EAAE,KAAK,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,CAAC,EAC9B,KAAK,EAAE,MAAM,GACZ,OAAO,CAAC,CAAC,EAAE,CAAC,CAed;AAED,wBAAgB,UAAU,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,EAAE,CAMlE;AAED,wBAAgB,WAAW,CAAC,CAAC,EAC3B,KAAK,EAAE,CAAC,EAAE,EACV,QAAQ,EAAE,MAAM,EAChB,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,MAAM,EAC5B,OAAO,EAAE,MAAM,GACd,CAAC,EAAE,EAAE,CAqBP;AAED,wBAAgB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAExD;AAED,wBAAgB,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,MAAM,CAIzD"}
|
package/dist/core/utils.js
CHANGED
|
@@ -5,10 +5,31 @@ export function computeContentHash(content) {
|
|
|
5
5
|
export function sleep(ms) {
|
|
6
6
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
7
7
|
}
|
|
8
|
+
export function defaultIsRetryable(error) {
|
|
9
|
+
const msg = error instanceof Error
|
|
10
|
+
? error.message
|
|
11
|
+
: typeof error === "string"
|
|
12
|
+
? error
|
|
13
|
+
: String(error);
|
|
14
|
+
const lower = msg.toLowerCase();
|
|
15
|
+
if (lower.includes("access denied") ||
|
|
16
|
+
lower.includes("forbidden") ||
|
|
17
|
+
lower.includes("unauthorized") ||
|
|
18
|
+
lower.includes("not found")) {
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
// Reject any 4xx HTTP status code except 429 (Too Many Requests)
|
|
22
|
+
const m = msg.match(/\b4(\d{2})\b/);
|
|
23
|
+
if (m) {
|
|
24
|
+
return parseInt(m[0], 10) === 429;
|
|
25
|
+
}
|
|
26
|
+
return true;
|
|
27
|
+
}
|
|
8
28
|
export async function withRetry(fn, opts = {}, logger) {
|
|
9
29
|
const maxRetries = opts.maxRetries ?? 3;
|
|
10
30
|
const delayMs = opts.retryDelayMs ?? 1000;
|
|
11
31
|
const factor = opts.retryBackoffFactor ?? 2;
|
|
32
|
+
const isRetryable = opts.isRetryable;
|
|
12
33
|
let lastErr;
|
|
13
34
|
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
14
35
|
try {
|
|
@@ -16,6 +37,9 @@ export async function withRetry(fn, opts = {}, logger) {
|
|
|
16
37
|
}
|
|
17
38
|
catch (err) {
|
|
18
39
|
lastErr = err;
|
|
40
|
+
if (isRetryable && !isRetryable(err)) {
|
|
41
|
+
throw err;
|
|
42
|
+
}
|
|
19
43
|
if (attempt < maxRetries) {
|
|
20
44
|
const wait = Math.round(delayMs * Math.pow(factor, attempt));
|
|
21
45
|
logger?.verbose(` โ ๏ธ Attempt ${attempt + 1}/${maxRetries + 1} failed` +
|
package/dist/core/utils.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/core/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGpC,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;
|
|
1
|
+
{"version":3,"file":"utils.js","sourceRoot":"","sources":["../../src/core/utils.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AAGpC,MAAM,UAAU,kBAAkB,CAAC,OAAe;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AACzE,CAAC;AAED,MAAM,UAAU,KAAK,CAAC,EAAU;IAC9B,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AASD,MAAM,UAAU,kBAAkB,CAAC,KAAc;IAC/C,MAAM,GAAG,GACP,KAAK,YAAY,KAAK;QACpB,CAAC,CAAC,KAAK,CAAC,OAAO;QACf,CAAC,CAAC,OAAO,KAAK,KAAK,QAAQ;YACzB,CAAC,CAAC,KAAK;YACP,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IAEtB,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAChC,IACE,KAAK,CAAC,QAAQ,CAAC,eAAe,CAAC;QAC/B,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC;QAC3B,KAAK,CAAC,QAAQ,CAAC,cAAc,CAAC;QAC9B,KAAK,CAAC,QAAQ,CAAC,WAAW,CAAC,EAC3B,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IAED,iEAAiE;IACjE,MAAM,CAAC,GAAG,GAAG,CAAC,KAAK,CAAC,cAAc,CAAC,CAAC;IACpC,IAAI,CAAC,EAAE,CAAC;QACN,OAAO,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,GAAG,CAAC;IACpC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAoB,EACpB,OAAqB,EAAE,EACvB,MAAe;IAEf,MAAM,UAAU,GAAG,IAAI,CAAC,UAAU,IAAI,CAAC,CAAC;IACxC,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;IAC1C,MAAM,MAAM,GAAG,IAAI,CAAC,kBAAkB,IAAI,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC;IAErC,IAAI,OAAgB,CAAC;IACrB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACvD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,OAAO,GAAG,GAAG,CAAC;YACd,IAAI,WAAW,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrC,MAAM,GAAG,CAAC;YACZ,CAAC;YACD,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;gBACzB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC;gBAC7D,MAAM,EAAE,OAAO,CACb,gBAAgB,OAAO,GAAG,CAAC,IAAI,UAAU,GAAG,CAAC,SAAS;oBACpD,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG;oBACxD,iBAAiB,IAAI,OAAO,CAC/B,CAAC;gBACF,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;IACH,CAAC;IACD,MAAM,OAAO,CAAC;AAChB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,eAAe,CACnC,KAA8B,EAC9B,KAAa;IAEb,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IAClC,MAAM,OAAO,GAAG,IAAI,KAAK,CAAI,KAAK,CAAC,MAAM,CAAC,CAAC;IAC3C,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,KAAK,UAAU,MAAM;QACnB,OAAO,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,KAAK,EAAE,CAAC;YAClB,OAAO,CAAC,CAAC,CAAC,GAAG,MAAM,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;QAChC,CAAC;IACH,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IAClD,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;IAC/D,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,UAAU,CAAI,KAAU,EAAE,SAAiB;IACzD,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;QACjD,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC,CAAC;IAC9C,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,WAAW,CACzB,KAAU,EACV,QAAgB,EAChB,OAA4B,EAC5B,OAAe;IAEf,MAAM,OAAO,GAAU,EAAE,CAAC;IAC1B,IAAI,OAAO,GAAQ,EAAE,CAAC;IACtB,IAAI,WAAW,GAAG,CAAC,CAAC;IAEpB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAC3B,IACE,OAAO,CAAC,MAAM,GAAG,CAAC;YAClB,CAAC,OAAO,CAAC,MAAM,IAAI,QAAQ,IAAI,WAAW,GAAG,IAAI,GAAG,OAAO,CAAC,EAC5D,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YACtB,OAAO,GAAG,EAAE,CAAC;YACb,WAAW,GAAG,CAAC,CAAC;QAClB,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,WAAW,IAAI,IAAI,CAAC;IACtB,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IAC9C,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,QAAgB;IAC9C,OAAO,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,QAAQ,CAAC;AAC/C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,QAAgB;IAC/C,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IAClC,KAAK,CAAC,GAAG,EAAE,CAAC;IACZ,OAAO,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACzB,CAAC"}
|
package/dist/eval/runner.d.ts
CHANGED
|
@@ -11,6 +11,6 @@ export declare class EvalRunner {
|
|
|
11
11
|
private readonly dataset;
|
|
12
12
|
private readonly topK;
|
|
13
13
|
constructor(store: VectorStore, embedder: EmbeddingProvider, dataset: EvalDataset, topK?: number);
|
|
14
|
-
run(): Promise<EvalRunResult>;
|
|
14
|
+
run(onProgress?: (completed: number, total: number) => void): Promise<EvalRunResult>;
|
|
15
15
|
}
|
|
16
16
|
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGxE,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,qBAAa,UAAU;IAEnB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAHJ,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,EAAE,WAAW,EACpB,IAAI,SAAK;IAGtB,GAAG,IAAI,OAAO,CAAC,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AAC7E,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AAGxE,MAAM,WAAW,aAAa;IAC5B,UAAU,EAAE,UAAU,CAAC;IACvB,2EAA2E;IAC3E,gBAAgB,EAAE,MAAM,EAAE,CAAC;CAC5B;AAED,qBAAa,UAAU;IAEnB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,IAAI;gBAHJ,KAAK,EAAE,WAAW,EAClB,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,EAAE,WAAW,EACpB,IAAI,SAAK;IAGtB,GAAG,CACP,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACtD,OAAO,CAAC,aAAa,CAAC;CAiD1B"}
|
package/dist/eval/runner.js
CHANGED
|
@@ -10,9 +10,10 @@ export class EvalRunner {
|
|
|
10
10
|
this.dataset = dataset;
|
|
11
11
|
this.topK = topK;
|
|
12
12
|
}
|
|
13
|
-
async run() {
|
|
13
|
+
async run(onProgress) {
|
|
14
14
|
const queryResults = [];
|
|
15
|
-
for (
|
|
15
|
+
for (let qi = 0; qi < this.dataset.queries.length; qi++) {
|
|
16
|
+
const evalQuery = this.dataset.queries[qi];
|
|
16
17
|
const embedding = await this.embedder.embed(evalQuery.query);
|
|
17
18
|
const searchResults = await this.store.search(embedding, this.topK);
|
|
18
19
|
const relevantIds = new Set();
|
|
@@ -35,6 +36,7 @@ export class EvalRunner {
|
|
|
35
36
|
}
|
|
36
37
|
}
|
|
37
38
|
queryResults.push({ retrievedIds, relevantIds });
|
|
39
|
+
onProgress?.(qi + 1, this.dataset.queries.length);
|
|
38
40
|
}
|
|
39
41
|
const evalResult = computeEvalResult(queryResults);
|
|
40
42
|
const perQueryRrScores = queryResults.map(({ retrievedIds, relevantIds }) => reciprocalRank(retrievedIds, relevantIds));
|
package/dist/eval/runner.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQjE,MAAM,OAAO,UAAU;IAEF;IACA;IACA;IACA;IAJnB,YACmB,KAAkB,EAClB,QAA2B,EAC3B,OAAoB,EACpB,OAAO,EAAE;QAHT,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAmB;QAC3B,YAAO,GAAP,OAAO,CAAa;QACpB,SAAI,GAAJ,IAAI,CAAK;IACzB,CAAC;IAEJ,KAAK,CAAC,GAAG;
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/eval/runner.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,cAAc,CAAC;AAQjE,MAAM,OAAO,UAAU;IAEF;IACA;IACA;IACA;IAJnB,YACmB,KAAkB,EAClB,QAA2B,EAC3B,OAAoB,EACpB,OAAO,EAAE;QAHT,UAAK,GAAL,KAAK,CAAa;QAClB,aAAQ,GAAR,QAAQ,CAAmB;QAC3B,YAAO,GAAP,OAAO,CAAa;QACpB,SAAI,GAAJ,IAAI,CAAK;IACzB,CAAC;IAEJ,KAAK,CAAC,GAAG,CACP,UAAuD;QAEvD,MAAM,YAAY,GAGb,EAAE,CAAC;QAER,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,EAAE,EAAE,CAAC;YACxD,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC3C,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;YAC7D,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YAEpE,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;YAEtC,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;gBAC/B,KAAK,MAAM,EAAE,IAAI,SAAS,CAAC,gBAAgB,EAAE,CAAC;oBAC5C,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC;YAED,2EAA2E;YAC3E,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CACpC,CAAC,CAAC,EAAE,EAAE,CAAE,CAAC,CAAC,QAAQ,CAAC,WAAkC,IAAI,CAAC,CAAC,EAAE,CAC9D,CAAC;YAEF,2BAA2B;YAC3B,IAAI,SAAS,CAAC,eAAe,IAAI,SAAS,CAAC,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACtE,KAAK,MAAM,QAAQ,IAAI,SAAS,CAAC,eAAe,EAAE,CAAC;oBACjD,KAAK,MAAM,MAAM,IAAI,aAAa,EAAE,CAAC;wBACnC,IAAI,MAAM,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;4BACtC,WAAW,CAAC,GAAG,CACZ,MAAM,CAAC,QAAQ,CAAC,WAAkC;gCACjD,MAAM,CAAC,EAAE,CACZ,CAAC;wBACJ,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YAED,YAAY,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC;YACjD,UAAU,EAAE,CAAC,EAAE,GAAG,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QACpD,CAAC;QAED,MAAM,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAC;QACnD,MAAM,gBAAgB,GAAG,YAAY,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,EAAE,WAAW,EAAE,EAAE,EAAE,CAC1E,cAAc,CAAC,YAAY,EAAE,WAAW,CAAC,CAC1C,CAAC;QAEF,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,CAAC;IAC1C,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consola-logger.d.ts","sourceRoot":"","sources":["../../src/logger/consola-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAatD,qBAAa,aAAc,YAAW,MAAM;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;gBAE9B,QAAQ,EAAE,eAAe;IAIrC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa;IAuC/C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAGhD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAGhD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAG/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAG/C,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIlD,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIlD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIhD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"consola-logger.d.ts","sourceRoot":"","sources":["../../src/logger/consola-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAiB,KAAK,eAAe,EAAE,MAAM,SAAS,CAAC;AAC9D,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAatD,qBAAa,aAAc,YAAW,MAAM;IAC1C,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAiB;gBAE9B,QAAQ,EAAE,eAAe;IAIrC,MAAM,CAAC,MAAM,CAAC,SAAS,EAAE,MAAM,GAAG,aAAa;IAuC/C,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAGhD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAGhD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAG/C,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAG/C,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIlD,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIlD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAIhD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAahD,KAAK,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI;IAUhD,OAAO,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM;CAG7B"}
|
|
@@ -52,12 +52,22 @@ export class ConsolaLogger {
|
|
|
52
52
|
// in args so FancyReporter renders it (formatArgs uses args, not message).
|
|
53
53
|
if (this._consola.level < 6)
|
|
54
54
|
return;
|
|
55
|
-
this._consola._log({
|
|
55
|
+
this._consola._log({
|
|
56
|
+
level: 6,
|
|
57
|
+
message: "",
|
|
58
|
+
args: [message, ...args],
|
|
59
|
+
date: new Date(),
|
|
60
|
+
});
|
|
56
61
|
}
|
|
57
62
|
silly(message, ...args) {
|
|
58
63
|
if (this._consola.level < 999)
|
|
59
64
|
return;
|
|
60
|
-
this._consola._log({
|
|
65
|
+
this._consola._log({
|
|
66
|
+
level: 999,
|
|
67
|
+
message: "",
|
|
68
|
+
args: [message, ...args],
|
|
69
|
+
date: new Date(),
|
|
70
|
+
});
|
|
61
71
|
}
|
|
62
72
|
withTag(tag) {
|
|
63
73
|
return new ConsolaLogger(this._consola.withTag(tag));
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"consola-logger.js","sourceRoot":"","sources":["../../src/logger/consola-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAwB,MAAM,SAAS,CAAC;AAG9D,MAAM,gBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,CAAU,CAAC;AAWvD,MAAM,OAAO,aAAa;IACP,QAAQ,CAAiB;IAE1C,YAAY,QAAyB;QACnC,IAAI,CAAC,QAAQ,GAAG,QAA0B,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,SAAiB;QAC7B,MAAM,KAAK,GACT,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAE1C,yEAAyE;QACzE,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,QAAQ,GACZ,QAWD,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,OAAO,QAAQ,EAAE,WAAW,KAAK,UAAU,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,QAAQ,CAAC,WAAW,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC9C,MAAM,QAAQ,GAAG,KAAK;qBACnB,KAAK,CAAC,IAAI,CAAC;qBACX,MAAM,CACL,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACtC,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CACnC;qBACA,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YACvC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,aAAa,CAAC,QAA0B,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,CAAC,OAAe,EAAE,GAAG,IAAe;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,OAAe,EAAE,GAAG,IAAe;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,CAAC,OAAe,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,OAAe,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU;IACnD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU;IACnD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,mEAAmE;QACnE,sEAAsE;QACtE,2EAA2E;QAC3E,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC;YAAE,OAAO;QACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,
|
|
1
|
+
{"version":3,"file":"consola-logger.js","sourceRoot":"","sources":["../../src/logger/consola-logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAwB,MAAM,SAAS,CAAC;AAG9D,MAAM,gBAAgB,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,GAAG,CAAU,CAAC;AAWvD,MAAM,OAAO,aAAa;IACP,QAAQ,CAAiB;IAE1C,YAAY,QAAyB;QACnC,IAAI,CAAC,QAAQ,GAAG,QAA0B,CAAC;IAC7C,CAAC;IAED,MAAM,CAAC,MAAM,CAAC,SAAiB;QAC7B,MAAM,KAAK,GACT,gBAAgB,CAAC,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,gBAAgB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,aAAa,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC;QAE1C,yEAAyE;QACzE,4EAA4E;QAC5E,gEAAgE;QAChE,MAAM,QAAQ,GACZ,QAWD,CAAC,OAAO,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1B,IAAI,OAAO,QAAQ,EAAE,WAAW,KAAK,UAAU,EAAE,CAAC;YAChD,MAAM,IAAI,GAAG,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YACjD,QAAQ,CAAC,WAAW,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,EAAE,EAAE;gBAC9C,MAAM,QAAQ,GAAG,KAAK;qBACnB,KAAK,CAAC,IAAI,CAAC;qBACX,MAAM,CACL,CAAC,IAAI,EAAE,EAAE,CACP,CAAC,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC;oBACtC,CAAC,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CACnC;qBACA,IAAI,CAAC,IAAI,CAAC,CAAC;gBACd,OAAO,IAAI,CAAC,QAAQ,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;YACvC,CAAC,CAAC;QACJ,CAAC;QAED,OAAO,IAAI,aAAa,CAAC,QAA0B,CAAC,CAAC;IACvD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACxC,CAAC;IACD,IAAI,CAAC,OAAe,EAAE,GAAG,IAAe;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,CAAC,OAAe,EAAE,GAAG,IAAe;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IACvC,CAAC;IACD,OAAO,CAAC,OAAe,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,OAAe,EAAE,GAAG,IAAe;QACzC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU;IACnD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,CAAC,CAAC,UAAU;IACnD,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,mEAAmE;QACnE,sEAAsE;QACtE,2EAA2E;QAC3E,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,CAAC;YAAE,OAAO;QACpC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACjB,KAAK,EAAE,CAAC;YACR,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YACxB,IAAI,EAAE,IAAI,IAAI,EAAE;SACjB,CAAC,CAAC;IACL,CAAC;IAED,KAAK,CAAC,OAAe,EAAE,GAAG,IAAe;QACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,KAAK,GAAG,GAAG;YAAE,OAAO;QACtC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YACjB,KAAK,EAAE,GAAG;YACV,OAAO,EAAE,EAAE;YACX,IAAI,EAAE,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC;YACxB,IAAI,EAAE,IAAI,IAAI,EAAE;SACjB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,CAAC,GAAW;QACjB,OAAO,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC;IACvD,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin-registry.d.ts","sourceRoot":"","sources":["../src/plugin-registry.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,aAAa,CAAC;AAEpD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,eAAO,MAAM,gBAAgB,EAAE,WAAW,
|
|
1
|
+
{"version":3,"file":"plugin-registry.d.ts","sourceRoot":"","sources":["../src/plugin-registry.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,UAAU,GAAG,UAAU,GAAG,aAAa,CAAC;AAEpD,MAAM,WAAW,WAAW;IAC1B,IAAI,EAAE,UAAU,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACxC;AAED,eAAO,MAAM,gBAAgB,EAAE,WAAW,EA4FzC,CAAC;AAEF,wBAAsB,uBAAuB,CAC3C,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,WAAW,EAAE,CAAC,CAkExB;AAED,MAAM,WAAW,cAAc;IAC7B,SAAS,EAAE,WAAW,EAAE,CAAC;IACzB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB;AAED,wBAAsB,YAAY,CAChC,WAAW,EAAE,MAAM,GAClB,OAAO,CAAC,cAAc,CAAC,CAgBzB"}
|
package/dist/plugin-registry.js
CHANGED
|
@@ -14,19 +14,6 @@ export const BUILT_IN_PLUGINS = [
|
|
|
14
14
|
dimensions: 1536,
|
|
15
15
|
},
|
|
16
16
|
},
|
|
17
|
-
{
|
|
18
|
-
type: "embedder",
|
|
19
|
-
label: "GitHub Models (Azure-compatible endpoint)",
|
|
20
|
-
key: "github-models",
|
|
21
|
-
package: "@vivantel/virage-embedder-openai",
|
|
22
|
-
envVars: ["GITHUB_TOKEN"],
|
|
23
|
-
defaultConfig: {
|
|
24
|
-
apiKey: "${GITHUB_TOKEN}",
|
|
25
|
-
baseURL: "https://models.github.ai/inference",
|
|
26
|
-
model: "openai/text-embedding-3-small",
|
|
27
|
-
dimensions: 1536,
|
|
28
|
-
},
|
|
29
|
-
},
|
|
30
17
|
{
|
|
31
18
|
type: "embedder",
|
|
32
19
|
label: "FastEmbed (local, no API key required)",
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"plugin-registry.js","sourceRoot":"","sources":["../src/plugin-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAa5B,MAAM,CAAC,MAAM,gBAAgB,GAAkB;IAC7C,kBAAkB;IAClB;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,iCAAiC;QACxC,GAAG,EAAE,QAAQ;QACb,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,CAAC,gBAAgB,CAAC;QAC3B,aAAa,EAAE;YACb,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,wBAAwB;YAC/B,UAAU,EAAE,IAAI;SACjB;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,
|
|
1
|
+
{"version":3,"file":"plugin-registry.js","sourceRoot":"","sources":["../src/plugin-registry.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAa5B,MAAM,CAAC,MAAM,gBAAgB,GAAkB;IAC7C,kBAAkB;IAClB;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,iCAAiC;QACxC,GAAG,EAAE,QAAQ;QACb,OAAO,EAAE,kCAAkC;QAC3C,OAAO,EAAE,CAAC,gBAAgB,CAAC;QAC3B,aAAa,EAAE;YACb,MAAM,EAAE,mBAAmB;YAC3B,KAAK,EAAE,wBAAwB;YAC/B,UAAU,EAAE,IAAI;SACjB;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,wCAAwC;QAC/C,GAAG,EAAE,WAAW;QAChB,OAAO,EAAE,qCAAqC;QAC9C,OAAO,EAAE,EAAE;QACX,aAAa,EAAE;YACb,KAAK,EAAE,wBAAwB;YAC/B,UAAU,EAAE,GAAG;SAChB;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,uDAAuD;QAC9D,GAAG,EAAE,cAAc;QACnB,OAAO,EAAE,wCAAwC;QACjD,OAAO,EAAE,EAAE;QACX,aAAa,EAAE;YACb,KAAK,EAAE,yBAAyB;YAChC,UAAU,EAAE,GAAG;SAChB;KACF;IACD;QACE,IAAI,EAAE,UAAU;QAChB,KAAK,EAAE,6DAA6D;QACpE,GAAG,EAAE,QAAQ;QACb,OAAO,EAAE,+BAA+B;QACxC,OAAO,EAAE,EAAE;QACX,aAAa,EAAE;YACb,MAAM,EAAE,iBAAiB;YACzB,KAAK,EAAE,iBAAiB;YACxB,UAAU,EAAE,IAAI;SACjB;KACF;IACD,sBAAsB;IACtB;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,yDAAyD;QAChE,GAAG,EAAE,UAAU;QACf,OAAO,EAAE,iCAAiC;QAC1C,OAAO,EAAE,CAAC,cAAc,CAAC;QACzB,aAAa,EAAE,EAAE,gBAAgB,EAAE,iBAAiB,EAAE;KACvD;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,gCAAgC;QACvC,GAAG,EAAE,QAAQ;QACb,OAAO,EAAE,+BAA+B;QACxC,OAAO,EAAE,EAAE;QACX,aAAa,EAAE;YACb,GAAG,EAAE,eAAe;YACpB,MAAM,EAAE,mBAAmB;SAC5B;KACF;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,mDAAmD;QAC1D,GAAG,EAAE,SAAS;QACd,OAAO,EAAE,gCAAgC;QACzC,OAAO,EAAE,EAAE;QACX,aAAa,EAAE,EAAE,GAAG,EAAE,iBAAiB,EAAE;KAC1C;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,4BAA4B;QACnC,GAAG,EAAE,UAAU;QACf,OAAO,EAAE,iCAAiC;QAC1C,OAAO,EAAE,EAAE;QACX,aAAa,EAAE,EAAE,IAAI,EAAE,uBAAuB,EAAE;KACjD;IACD;QACE,IAAI,EAAE,aAAa;QACnB,KAAK,EAAE,uDAAuD;QAC9D,GAAG,EAAE,QAAQ;QACb,OAAO,EAAE,4BAA4B;QACrC,OAAO,EAAE,EAAE;QACX,aAAa,EAAE,EAAE;KAClB;CACF,CAAC;AAEF,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,WAAmB;IAEnB,IAAI,OAAgC,CAAC;IACrC,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,WAAW,EAAE,cAAc,CAAC,EAAE,OAAO,CAAC,CAAC;QAC3E,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAA4B,CAAC;IAC3D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,IAAI,GAA2B;QACnC,GAAG,CAAE,OAAO,CAAC,YAAmD,IAAI,EAAE,CAAC;QACvE,GAAG,CAAE,OAAO,CAAC,eAAsD,IAAI,EAAE,CAAC;KAC3E,CAAC;IAEF,MAAM,OAAO,GAAkB,EAAE,CAAC;IAElC,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,UAAU,GAAG,IAAI,CACrB,WAAW,EACX,cAAc,EACd,OAAO,EACP,cAAc,CACf,CAAC;YACF,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAA4B,CAAC;YAEjE,MAAM,SAAS,GAAG,MAAM,CAAC,YAAY,CAAC,CAAC;YACvC,IACE,CAAC,SAAS;gBACV,OAAO,SAAS,KAAK,QAAQ;gBAC7B,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC;gBAExB,SAAS;YAEX,MAAM,MAAM,GAAG,SAAoC,CAAC;YACpD,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,EAAE,GAAG,MAAM,CAAC;YAEpC,IACE,CAAC,IAAI,KAAK,UAAU,IAAI,IAAI,KAAK,aAAa,CAAC;gBAC/C,OAAO,KAAK,KAAK,QAAQ;gBACzB,OAAO,GAAG,KAAK,QAAQ;gBAEvB,SAAS;YAEX,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE,IAAkB;gBACxB,KAAK;gBACL,GAAG;gBACH,OAAO,EAAE,OAAO;gBAChB,OAAO,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC;oBACpC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAe,EAAE,CAAC,OAAO,CAAC,KAAK,QAAQ,CAAC;oBAClE,CAAC,CAAC,EAAE;gBACN,aAAa,EACX,OAAO,MAAM,CAAC,aAAa,KAAK,QAAQ;oBACxC,MAAM,CAAC,aAAa,KAAK,IAAI;oBAC7B,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,aAAa,CAAC;oBAClC,CAAC,CAAE,MAAM,CAAC,aAAyC;oBACnD,CAAC,CAAC,EAAE;aACT,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,+CAA+C;QACjD,CAAC;IACH,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAOD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,WAAmB;IAEnB,MAAM,QAAQ,GAAG,MAAM,uBAAuB,CAAC,WAAW,CAAC,CAAC;IAE5D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAuB,CAAC;IAC9C,KAAK,MAAM,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrC,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IACD,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE,CAAC;QAC7B,MAAM,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,GAAG,EAAE,EAAE,KAAK,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IACxC,OAAO;QACL,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC;QACnD,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,aAAa,CAAC;KACpD,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress-bar.d.ts","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAEA,MAAM,WAAW,WAAW;IAC1B,MAAM,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,IAAI,CAAC,YAAY,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC;CACnC;AAED,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,WAAW,CAwB3E"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import cliProgress from "cli-progress";
|
|
2
|
+
export function createProgressBar(label, total) {
|
|
3
|
+
if (total === 0) {
|
|
4
|
+
return { update: () => { }, stop: () => { } };
|
|
5
|
+
}
|
|
6
|
+
const bar = new cliProgress.SingleBar({
|
|
7
|
+
format: `${label} [{bar}] {percentage}% | {value}/{total} | ETA: {eta_formatted}`,
|
|
8
|
+
clearOnComplete: false,
|
|
9
|
+
hideCursor: true,
|
|
10
|
+
}, cliProgress.Presets.shades_classic);
|
|
11
|
+
bar.start(total, 0);
|
|
12
|
+
return {
|
|
13
|
+
update(current) {
|
|
14
|
+
bar.update(current);
|
|
15
|
+
},
|
|
16
|
+
stop(finalMessage) {
|
|
17
|
+
bar.stop();
|
|
18
|
+
if (finalMessage)
|
|
19
|
+
console.log(finalMessage);
|
|
20
|
+
},
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=progress-bar.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"progress-bar.js","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAOvC,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,iEAAiE;QACjF,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;KACjB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAEpB,OAAO;QACL,MAAM,CAAC,OAAe;YACpB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,YAAqB;YACxB,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,IAAI,YAAY;gBAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.3",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
|
|
6
6
|
"repository": {
|
|
@@ -56,6 +56,7 @@
|
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@inquirer/prompts": "^8.5.2",
|
|
58
58
|
"chokidar": "^5.0.0",
|
|
59
|
+
"cli-progress": "^3.12.0",
|
|
59
60
|
"commander": "^15.0.0",
|
|
60
61
|
"consola": "^3.4.2",
|
|
61
62
|
"dotenv": "^17.4.2",
|
|
@@ -65,6 +66,7 @@
|
|
|
65
66
|
"devDependencies": {
|
|
66
67
|
"@eslint/js": "^10.0.1",
|
|
67
68
|
"@supabase/supabase-js": "^2.106.2",
|
|
69
|
+
"@types/cli-progress": "^3.11.6",
|
|
68
70
|
"@types/node": "^25.9.1",
|
|
69
71
|
"eslint": "^10.4.1",
|
|
70
72
|
"prettier": "^3.8.3",
|