@vivantel/virage-core 0.2.5 → 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/git-tracker.js +1 -1
- package/dist/core/git-tracker.js.map +1 -1
- package/dist/core/orchestrator.d.ts +3 -0
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +8 -33
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/virage-defaults.d.ts +1 -0
- package/dist/core/virage-defaults.d.ts.map +1 -1
- package/dist/core/virage-defaults.js +11 -0
- package/dist/core/virage-defaults.js.map +1 -1
- package/dist/index.d.ts +6 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -1
- package/dist/index.js.map +1 -1
- package/dist/strategies/chunk/markdown-headers.d.ts.map +1 -1
- package/dist/strategies/chunk/markdown-headers.js +5 -2
- package/dist/strategies/chunk/markdown-headers.js.map +1 -1
- package/dist/strategies/chunk/semantic.d.ts.map +1 -1
- package/dist/strategies/chunk/semantic.js +7 -1
- package/dist/strategies/chunk/semantic.js.map +1 -1
- package/dist/strategies/chunk/token.d.ts.map +1 -1
- package/dist/strategies/chunk/token.js +14 -4
- package/dist/strategies/chunk/token.js.map +1 -1
- package/package.json +1 -10
- package/dist/bin/virage.d.ts +0 -3
- package/dist/bin/virage.d.ts.map +0 -1
- package/dist/bin/virage.js +0 -345
- package/dist/bin/virage.js.map +0 -1
- package/dist/cli/benchmark.d.ts +0 -6
- package/dist/cli/benchmark.d.ts.map +0 -1
- package/dist/cli/benchmark.js +0 -26
- package/dist/cli/benchmark.js.map +0 -1
- package/dist/cli/chunks-report.d.ts +0 -2
- package/dist/cli/chunks-report.d.ts.map +0 -1
- package/dist/cli/chunks-report.js +0 -74
- package/dist/cli/chunks-report.js.map +0 -1
- package/dist/cli/dashboard.d.ts +0 -7
- package/dist/cli/dashboard.d.ts.map +0 -1
- package/dist/cli/dashboard.js +0 -190
- package/dist/cli/dashboard.js.map +0 -1
- package/dist/cli/evaluate.d.ts +0 -9
- package/dist/cli/evaluate.d.ts.map +0 -1
- package/dist/cli/evaluate.js +0 -67
- package/dist/cli/evaluate.js.map +0 -1
- package/dist/cli/experiment.d.ts +0 -13
- package/dist/cli/experiment.d.ts.map +0 -1
- package/dist/cli/experiment.js +0 -99
- package/dist/cli/experiment.js.map +0 -1
- package/dist/cli/file-detect.d.ts +0 -10
- package/dist/cli/file-detect.d.ts.map +0 -1
- package/dist/cli/file-detect.js +0 -55
- package/dist/cli/file-detect.js.map +0 -1
- package/dist/cli/init.d.ts +0 -2
- package/dist/cli/init.d.ts.map +0 -1
- package/dist/cli/init.js +0 -382
- package/dist/cli/init.js.map +0 -1
- package/dist/cli/report.d.ts +0 -2
- package/dist/cli/report.d.ts.map +0 -1
- package/dist/cli/report.js +0 -96
- package/dist/cli/report.js.map +0 -1
- package/dist/cli/store-cmd.d.ts +0 -10
- package/dist/cli/store-cmd.d.ts.map +0 -1
- package/dist/cli/store-cmd.js +0 -50
- package/dist/cli/store-cmd.js.map +0 -1
- package/dist/cli/validate.d.ts +0 -2
- package/dist/cli/validate.d.ts.map +0 -1
- package/dist/cli/validate.js +0 -74
- package/dist/cli/validate.js.map +0 -1
- package/dist/cli/viz.d.ts +0 -7
- package/dist/cli/viz.d.ts.map +0 -1
- package/dist/cli/viz.js +0 -163
- package/dist/cli/viz.js.map +0 -1
- package/dist/logger/consola-logger.d.ts +0 -18
- package/dist/logger/consola-logger.d.ts.map +0 -1
- package/dist/logger/consola-logger.js +0 -76
- package/dist/logger/consola-logger.js.map +0 -1
- package/dist/logger/index.d.ts +0 -5
- package/dist/logger/index.d.ts.map +0 -1
- package/dist/logger/index.js +0 -7
- package/dist/logger/index.js.map +0 -1
- package/dist/progress/progress-bar.d.ts +0 -6
- package/dist/progress/progress-bar.d.ts.map +0 -1
- package/dist/progress/progress-bar.js +0 -23
- package/dist/progress/progress-bar.js.map +0 -1
package/dist/core/git-tracker.js
CHANGED
|
@@ -3,7 +3,7 @@ import { glob } from "glob";
|
|
|
3
3
|
import { NullLogger } from "../logger/null-logger.js";
|
|
4
4
|
import { minimatch } from "minimatch";
|
|
5
5
|
import path from "path";
|
|
6
|
-
import { IGNORED_DIRS } from "
|
|
6
|
+
import { IGNORED_DIRS } from "./virage-defaults.js";
|
|
7
7
|
export class GitTracker {
|
|
8
8
|
git;
|
|
9
9
|
chunkers;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"git-tracker.js","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAG5B,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,
|
|
1
|
+
{"version":3,"file":"git-tracker.js","sourceRoot":"","sources":["../../src/core/git-tracker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAa,MAAM,YAAY,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAG5B,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AACtC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAEpD,MAAM,OAAO,UAAU;IACb,GAAG,CAAY;IACf,QAAQ,CAAgB;IACxB,WAAW,CAAW;IACtB,gBAAgB,GAAkB,IAAI,CAAC;IACvC,gBAAgB,GAAmB,IAAI,CAAC;IACxC,MAAM,CAAS;IAEvB,YAAY,QAAuB,EAAE,MAAe;QAClD,IAAI,CAAC,GAAG,GAAG,SAAS,EAAE,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,GAAG,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5D,CAAC;IAEO,KAAK,CAAC,cAAc;QAC1B,IAAI,CAAC,IAAI,CAAC,gBAAgB,EAAE,CAAC;YAC3B,IAAI,CAAC,gBAAgB,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5D,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,qBAAqB;QACjC,IAAI,IAAI,CAAC,gBAAgB,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;YACvC,IAAI,CAAC,gBAAgB,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YAChD,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;gBAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,yDAAyD,CAC1D,CAAC;YACJ,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC,gBAAgB,CAAC;IAC/B,CAAC;IAEO,iBAAiB,CAAC,QAAgB;QACxC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YACpC,KAAK,MAAM,OAAO,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;gBACvC,IAAI,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,CAAC;oBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,WAAW,QAAQ,eAAe,OAAO,CAAC,IAAI,GAAG,CAAC,CAAC;oBACrE,OAAO,OAAO,CAAC;gBACjB,CAAC;YACH,CAAC;QACH,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAEO,cAAc,CAAC,QAAgB,EAAE,OAAe;QACtD,MAAM,cAAc,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAC1D,MAAM,iBAAiB,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE5D,OAAO,SAAS,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,kBAAkB;QACtB,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC,WAAW,EAAE;YACzC,KAAK,EAAE,IAAI;YACX,MAAM,EAAE,CAAC,GAAG,YAAY,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,KAAK,CAAC;SAChD,CAAC,CAAC;QACH,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,WAAW,MAAM,CAAC,MAAM,qBAAqB,IAAI,CAAC,WAAW,CAAC,MAAM,aAAa,CAClF,CAAC;QACF,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,KAAe;QACnC,MAAM,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;QAE5C,MAAM,OAAO,CAAC,GAAG,CACf,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;YACvB,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;oBAChC,KAAK;oBACL,IAAI;oBACJ,aAAa;oBACb,IAAI;oBACJ,IAAI;iBACL,CAAC,CAAC;gBACH,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC3B,MAAM,QAAQ,GAAG,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBACvD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;gBAC9B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;YACjE,CAAC;YAAC,MAAM,CAAC;gBACP,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;gBACzC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;gBAC1B,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,YAAY,IAAI,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,aAAa,CAAC,CAAC;YACxE,CAAC;QACH,CAAC,CAAC,CACH,CAAC;QAEF,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,KAAK,CAAC,eAAe;QAGnB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACvD,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,qBAAqB,EAAE,CAAC;QACpD,MAAM,WAAW,GAAG,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;QAEhD,MAAM,KAAK,GAAG,IAAI,GAAG,EAGlB,CAAC;QAEJ,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;YAC5B,IAAI,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,WAAW,CAAC;YACpD,IAAI,QAAQ,EAAE,CAAC;gBACb,UAAU,GAAG,GAAG,UAAU,QAAQ,CAAC;YACrC,CAAC;YAED,MAAM,OAAO,GAAG,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;YAC7C,IAAI,OAAO,EAAE,CAAC;gBACZ,KAAK,CAAC,GAAG,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,OAAO,EAAE,CAAC,CAAC;YAC3C,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,aAAkC,EAClC,YAAwE;QAMxE,MAAM,OAAO,GAAG,YAAY,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;QAC/D,MAAM,SAAS,GAAa,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAAa,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAa,EAAE,CAAC;QAE/B,KAAK,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,OAAO,EAAE,CAAC;YACvC,MAAM,QAAQ,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;YAE7C,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACd,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,WAAW,QAAQ,EAAE,CAAC,CAAC;gBAC3C,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,IAAI,QAAQ,KAAK,IAAI,CAAC,UAAU,EAAE,CAAC;gBACxC,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,eAAe,QAAQ,KAAK,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CACrF,CAAC;gBACF,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;iBAAM,CAAC;gBACN,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC3B,CAAC;QACH,CAAC;QAED,KAAK,MAAM,CAAC,QAAQ,CAAC,IAAI,aAAa,EAAE,CAAC;YACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;gBAC3B,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,QAAQ,EAAE,CAAC,CAAC;gBAChD,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC;QAED,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC;IAC5C,CAAC;CACF"}
|
|
@@ -22,6 +22,9 @@ export interface RAGPipelineConfig {
|
|
|
22
22
|
webhookUrl?: string;
|
|
23
23
|
};
|
|
24
24
|
logger?: Logger;
|
|
25
|
+
onChunkProgress?: (done: number, total: number) => void;
|
|
26
|
+
onEmbedProgress?: (done: number, total: number) => void;
|
|
27
|
+
onUploadProgress?: (done: number, total: number) => void;
|
|
25
28
|
};
|
|
26
29
|
}
|
|
27
30
|
export declare class Orchestrator {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAMA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxD,eAAe,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;QACxD,gBAAgB,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KAC1D,CAAC;CACH;AAsCD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CAoM3B"}
|
|
@@ -4,7 +4,6 @@ import { EmbedderProcessor } from "./embedder.js";
|
|
|
4
4
|
import { Uploader } from "./uploader.js";
|
|
5
5
|
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
6
6
|
import { TelemetryCollector } from "./telemetry.js";
|
|
7
|
-
import { createProgressBar, } from "../progress/progress-bar.js";
|
|
8
7
|
import { NullLogger } from "../logger/null-logger.js";
|
|
9
8
|
import { readFile } from "fs/promises";
|
|
10
9
|
import { defaultChunksFile, defaultEmbeddingsFile } from "./virage-defaults.js";
|
|
@@ -96,14 +95,7 @@ export class Orchestrator {
|
|
|
96
95
|
const existingChunks = opts.force
|
|
97
96
|
? []
|
|
98
97
|
: await loadExistingChunks(this.chunksFile);
|
|
99
|
-
const
|
|
100
|
-
let chunks;
|
|
101
|
-
try {
|
|
102
|
-
chunks = await chunkProcessor.processFiles(toProcess, fileState, existingChunks, (done, total) => chunkBar.update(done < total ? done : total));
|
|
103
|
-
}
|
|
104
|
-
finally {
|
|
105
|
-
chunkBar.stop();
|
|
106
|
-
}
|
|
98
|
+
const chunks = await chunkProcessor.processFiles(toProcess, fileState, existingChunks, (done, total) => opts.onChunkProgress?.(done < total ? done : total, total));
|
|
107
99
|
await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
|
|
108
100
|
const chunkDuration = Date.now() - t2;
|
|
109
101
|
logger.verbose(`Chunking done in ${chunkDuration}ms`);
|
|
@@ -121,7 +113,6 @@ export class Orchestrator {
|
|
|
121
113
|
// Step 3: Generate embeddings
|
|
122
114
|
logger.info("🔢 Step 3: Generating embeddings...");
|
|
123
115
|
const t3 = Date.now();
|
|
124
|
-
const embedBars = [];
|
|
125
116
|
const uploader = new Uploader(this.config.vectorStore, {
|
|
126
117
|
retry: opts.retry,
|
|
127
118
|
logger: opts.logger,
|
|
@@ -135,23 +126,13 @@ export class Orchestrator {
|
|
|
135
126
|
vectorStoreName: this.config.vectorStore.name,
|
|
136
127
|
minIngestionBatchSize: opts.minIngestionBatchSize,
|
|
137
128
|
logger: opts.logger,
|
|
138
|
-
onProgress: (done, total) =>
|
|
139
|
-
if (embedBars.length === 0)
|
|
140
|
-
embedBars.push(createProgressBar("Embedding", total));
|
|
141
|
-
embedBars[0].update(done < total ? done : total);
|
|
142
|
-
},
|
|
129
|
+
onProgress: (done, total) => opts.onEmbedProgress?.(done < total ? done : total, total),
|
|
143
130
|
});
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
await uploader.uploadPending(db);
|
|
150
|
-
});
|
|
151
|
-
}
|
|
152
|
-
finally {
|
|
153
|
-
embedBars[0]?.stop();
|
|
154
|
-
}
|
|
131
|
+
const newEmbeddings = await embedder.run(db, this.chunksFile, opts.force || false, opts.skipUpload
|
|
132
|
+
? undefined
|
|
133
|
+
: async () => {
|
|
134
|
+
await uploader.uploadPending(db);
|
|
135
|
+
});
|
|
155
136
|
const embedDuration = Date.now() - t3;
|
|
156
137
|
logger.verbose(`Embedding done in ${embedDuration}ms`);
|
|
157
138
|
telemetry?.recordEmbedding({
|
|
@@ -169,13 +150,7 @@ export class Orchestrator {
|
|
|
169
150
|
else if (!opts.skipUpload) {
|
|
170
151
|
logger.info("📤 Step 4: Uploading to vector store...");
|
|
171
152
|
const t4 = Date.now();
|
|
172
|
-
|
|
173
|
-
uploadStats = await uploader.sync(db, opts.force || false, (done, total) => {
|
|
174
|
-
if (uploadBars.length === 0)
|
|
175
|
-
uploadBars.push(createProgressBar("Uploading", total));
|
|
176
|
-
uploadBars[0].update(done < total ? done : total);
|
|
177
|
-
});
|
|
178
|
-
uploadBars[0]?.stop();
|
|
153
|
+
uploadStats = await uploader.sync(db, opts.force || false, opts.onUploadProgress);
|
|
179
154
|
const uploadDuration = Date.now() - t4;
|
|
180
155
|
logger.verbose(`Upload done in ${uploadDuration}ms`);
|
|
181
156
|
telemetry?.recordUpload({
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AAQpD,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AA2BhF,SAAS,iBAAiB,CAAC,EAAgB;IACzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,UAAkB;IAClD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,MAA2B,EAC3B,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;AACH,CAAC;AAED,MAAM,OAAO,YAAY;IACf,MAAM,CAAoB;IAC1B,UAAU,CAAS;IACnB,cAAc,CAAS;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;QACpE,IAAI,CAAC,cAAc;YACjB,MAAM,CAAC,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,SAAS,EAAE,KAAK,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QAEpC,IAAI,WAAW,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE9C,MAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,UAAU,mBAAmB,IAAI,CAAC,cAAc,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,CAC3G,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE3C,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;YAExD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK;gBAC9B,CAAC,CAAC,IAAI,GAAG,EAAkB;gBAC3B,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE1B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,eAAe,CAC9D,aAAa,EACb,YAAY,CACb,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,wBAAwB,WAAW,IAAI,CAAC,CAAC;YAExD,SAAS,EAAE,iBAAiB,CAAC;gBAC3B,UAAU,EAAE,WAAW;gBACvB,YAAY,EAAE,YAAY,CAAC,IAAI;gBAC/B,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnE,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACtC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,MAAM,CAAC,IAAI,CACT,eAAe,SAAS,CAAC,MAAM,gBAAgB,QAAQ,CAAC,MAAM,YAAY,CAC3E,CAAC;YAEF,wCAAwC;YACxC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,cAAc,GAAG,IAAI,cAAc,CACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAGtB,CAAC;YACJ,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,IAAI;oBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK;gBAC/B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9C,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,CAC9C,SAAS,EACT,SAAS,EACT,cAAc,EACd,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC,CAC5E,CAAC;YACF,MAAM,cAAc,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,oBAAoB,aAAa,IAAI,CAAC,CAAC;YAEtD,SAAS,EAAE,cAAc,CAAC;gBACxB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,SAAS,CAAC,MAAM;gBAChC,eAAe,EAAE,MAAM,CAAC,MAAM;gBAC9B,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;gBAChD,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,8BAA8B;YAC9B,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACnD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAEtB,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACrD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBAC3D,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI;gBAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;gBACjD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,EAAE,KAAK,CAAC;aACxF,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,GAAG,CACtC,EAAE,EACF,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,KAAK,IAAI,KAAK,EACnB,IAAI,CAAC,UAAU;gBACb,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,IAAI,EAAE;oBACT,MAAM,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBACnC,CAAC,CACN,CAAC;YAEF,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,qBAAqB,aAAa,IAAI,CAAC,CAAC;YAEvD,SAAS,EAAE,eAAe,CAAC;gBACzB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,aAAa,CAAC,MAAM;gBACpC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;aACpD,CAAC,CAAC;YAEH,iBAAiB;YACjB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAChD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,IAAI,CAAC,gBAAgB,CAAC,CAAC;gBAElF,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;gBAErD,SAAS,EAAE,YAAY,CAAC;oBACtB,UAAU,EAAE,cAAc;oBAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;YAE3C,IAAI,SAAS,EAAE,CAAC;gBACd,SAAS,CAAC,MAAM,EAAE,CAAC;gBACnB,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3C,aAAa,EACb,gBAAgB,CACjB,CAAC;gBACF,MAAM,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;oBAC5D,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;oBAC3C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,MAAM;oBACnC,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,EAAE,MAAM,EAAE,CAAC;YAEpB,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE;oBAC1D,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;oBACvD,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;iBAC5C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"virage-defaults.d.ts","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,qBAAqB,IAAI,MAAM,CAE9C;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}
|
|
1
|
+
{"version":3,"file":"virage-defaults.d.ts","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,YAAY,aAUvB,CAAC;AAEH,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,qBAAqB,IAAI,MAAM,CAE9C;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"virage-defaults.js","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,GAAG,YAAY,EAAE,cAAc,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,OAAO,GAAG,YAAY,EAAE,kBAAkB,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,GAAG,YAAY,EAAE,gBAAgB,CAAC;AAC3C,CAAC"}
|
|
1
|
+
{"version":3,"file":"virage-defaults.js","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAClC,cAAc;IACd,MAAM;IACN,MAAM;IACN,OAAO;IACP,UAAU;IACV,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;CACT,CAAC,CAAC;AAEH,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,GAAG,YAAY,EAAE,cAAc,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,OAAO,GAAG,YAAY,EAAE,kBAAkB,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,GAAG,YAAY,EAAE,gBAAgB,CAAC;AAC3C,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -8,6 +8,10 @@ export { Orchestrator, RAGPipelineConfig } from "./core/orchestrator.js";
|
|
|
8
8
|
export { computeContentHash, sleep, batchArray, batchBySize, extractFileName, extractDirectory, withRetry, withConcurrency, } from "./core/utils.js";
|
|
9
9
|
export type { RetryOptions } from "./core/utils.js";
|
|
10
10
|
export { discoverPlugins, RagPlugin } from "./core/plugin-discovery.js";
|
|
11
|
+
export { loadRegistry } from "./plugin-registry.js";
|
|
12
|
+
export type { PluginRegistry } from "./plugin-registry.js";
|
|
13
|
+
export { EmbeddingsDb } from "./core/embeddings-db.js";
|
|
14
|
+
export { getVirageDir, defaultChunksFile, defaultEmbeddingsFile, defaultEmbeddingsDb, IGNORED_DIRS, } from "./core/virage-defaults.js";
|
|
11
15
|
/**
|
|
12
16
|
* Chunking strategies.
|
|
13
17
|
*
|
|
@@ -23,7 +27,8 @@ export { createChunker } from "./helpers/create-chunker.js";
|
|
|
23
27
|
export { readEmbeddingsFile, writeEmbeddingsFile, } from "./core/embeddings-io.js";
|
|
24
28
|
export type { EmbeddingsReadResult } from "./core/embeddings-io.js";
|
|
25
29
|
export { loadConfig, autoDetectConfig } from "./config-loader.js";
|
|
26
|
-
export {
|
|
30
|
+
export { NullLogger } from "./logger/null-logger.js";
|
|
31
|
+
export { generateEvalDataset } from "./eval/generator.js";
|
|
27
32
|
export { EvalRunner } from "./eval/runner.js";
|
|
28
33
|
export { RAGASRunner } from "./eval/ragas.js";
|
|
29
34
|
export type { EvalRunResult } from "./eval/runner.js";
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,uBAAuB,CAAC;AAGtC,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,cAAc,uBAAuB,CAAC;AAGtC,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAGpD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AACpD,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAG3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AACrC,YAAY,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAGxE,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAG5D,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AACjC,YAAY,EAAE,oBAAoB,EAAE,MAAM,yBAAyB,CAAC;AAGpE,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAGlE,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAGrD,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAC9C,YAAY,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAC3D,YAAY,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -11,6 +11,11 @@ export { Orchestrator } from "./core/orchestrator.js";
|
|
|
11
11
|
export { computeContentHash, sleep, batchArray, batchBySize, extractFileName, extractDirectory, withRetry, withConcurrency, } from "./core/utils.js";
|
|
12
12
|
// Plugin ecosystem
|
|
13
13
|
export { discoverPlugins } from "./core/plugin-discovery.js";
|
|
14
|
+
export { loadRegistry } from "./plugin-registry.js";
|
|
15
|
+
// Storage
|
|
16
|
+
export { EmbeddingsDb } from "./core/embeddings-db.js";
|
|
17
|
+
// Defaults and constants
|
|
18
|
+
export { getVirageDir, defaultChunksFile, defaultEmbeddingsFile, defaultEmbeddingsDb, IGNORED_DIRS, } from "./core/virage-defaults.js";
|
|
14
19
|
/**
|
|
15
20
|
* Chunking strategies.
|
|
16
21
|
*
|
|
@@ -28,8 +33,9 @@ export { readEmbeddingsFile, writeEmbeddingsFile, } from "./core/embeddings-io.j
|
|
|
28
33
|
// Config loader
|
|
29
34
|
export { loadConfig, autoDetectConfig } from "./config-loader.js";
|
|
30
35
|
// Logger
|
|
31
|
-
export {
|
|
36
|
+
export { NullLogger } from "./logger/null-logger.js";
|
|
32
37
|
// Evaluation framework
|
|
38
|
+
export { generateEvalDataset } from "./eval/generator.js";
|
|
33
39
|
export { EvalRunner } from "./eval/runner.js";
|
|
34
40
|
export { RAGASRunner } from "./eval/ragas.js";
|
|
35
41
|
export { ExperimentStore, makeRunId } from "./eval/experiment-store.js";
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,cAAc,uBAAuB,CAAC;AAEtC,SAAS;AACT,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO;AACP,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAqB,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AAGzB,mBAAmB;AACnB,OAAO,EAAE,eAAe,EAAa,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,aAAa;AACb,cAAc,uBAAuB,CAAC;AAEtC,SAAS;AACT,OAAO,EACL,QAAQ,EACR,WAAW,EACX,UAAU,EACV,UAAU,EACV,WAAW,GACZ,MAAM,kBAAkB,CAAC;AAE1B,OAAO;AACP,OAAO,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACnD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,iBAAiB,EAAE,MAAM,oBAAoB,CAAC;AACvD,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,YAAY,EAAqB,MAAM,wBAAwB,CAAC;AACzE,OAAO,EACL,kBAAkB,EAClB,KAAK,EACL,UAAU,EACV,WAAW,EACX,eAAe,EACf,gBAAgB,EAChB,SAAS,EACT,eAAe,GAChB,MAAM,iBAAiB,CAAC;AAGzB,mBAAmB;AACnB,OAAO,EAAE,eAAe,EAAa,MAAM,4BAA4B,CAAC;AACxE,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAC;AAGpD,UAAU;AACV,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAEvD,yBAAyB;AACzB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,qBAAqB,EACrB,mBAAmB,EACnB,YAAY,GACb,MAAM,2BAA2B,CAAC;AAEnC;;;;;;;;GAQG;AACH,OAAO,EACL,aAAa,EACb,uBAAuB,EACvB,gBAAgB,EAChB,iBAAiB,GAClB,MAAM,6BAA6B,CAAC;AAGrC,UAAU;AACV,OAAO,EAAE,aAAa,EAAE,MAAM,6BAA6B,CAAC;AAE5D,iBAAiB;AACjB,OAAO,EACL,kBAAkB,EAClB,mBAAmB,GACpB,MAAM,yBAAyB,CAAC;AAGjC,gBAAgB;AAChB,OAAO,EAAE,UAAU,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAElE,SAAS;AACT,OAAO,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAC;AAErD,uBAAuB;AACvB,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,iBAAiB,CAAC;AAE9C,OAAO,EAAE,eAAe,EAAE,SAAS,EAAE,MAAM,4BAA4B,CAAC;AACxE,OAAO,EACL,YAAY,EACZ,SAAS,EACT,cAAc,EACd,UAAU,EACV,oBAAoB,EACpB,iBAAiB,GAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAE3D,OAAO,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-headers.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/markdown-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,uBAAuB,CACrC,OAAO,GAAE,sBAA2B,GACnC,aAAa,
|
|
1
|
+
{"version":3,"file":"markdown-headers.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/markdown-headers.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,sBAAsB;IACrC,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,YAAY,CAAC,EAAE,MAAM,CAAC;CACvB;AAED,wBAAgB,uBAAuB,CACrC,OAAO,GAAE,sBAA2B,GACnC,aAAa,CAuGf"}
|
|
@@ -6,7 +6,8 @@ export function markdownHeadersStrategy(options = {}) {
|
|
|
6
6
|
name: "markdown-headers",
|
|
7
7
|
async chunk(text, filePath) {
|
|
8
8
|
const chunks = [];
|
|
9
|
-
|
|
9
|
+
// Normalise line endings so CRLF files don't leave \r on header text/content.
|
|
10
|
+
const lines = text.split("\n").map((l) => l.replace(/\r$/, ""));
|
|
10
11
|
let currentChunk = [];
|
|
11
12
|
let currentHeader = "";
|
|
12
13
|
let currentHeaderLevel = 0;
|
|
@@ -48,12 +49,14 @@ export function markdownHeadersStrategy(options = {}) {
|
|
|
48
49
|
strategy: this.name,
|
|
49
50
|
header: currentHeader,
|
|
50
51
|
header_level: currentHeaderLevel,
|
|
52
|
+
source_file: filePath,
|
|
51
53
|
truncated: true,
|
|
52
54
|
},
|
|
53
55
|
sourceFile: filePath || "unknown",
|
|
54
56
|
commitHash: "",
|
|
55
57
|
});
|
|
56
|
-
|
|
58
|
+
// Keep the header line so the continuation chunk is self-contained.
|
|
59
|
+
currentChunk = currentChunk.length > 0 ? [currentChunk[0]] : [];
|
|
57
60
|
}
|
|
58
61
|
}
|
|
59
62
|
// Last chunk
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"markdown-headers.js","sourceRoot":"","sources":["../../../src/strategies/chunk/markdown-headers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,MAAM,UAAU,uBAAuB,CACrC,UAAkC,EAAE;IAEpC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAElD,OAAO;QACL,IAAI,EAAE,kBAAkB;QAExB,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;
|
|
1
|
+
{"version":3,"file":"markdown-headers.js","sourceRoot":"","sources":["../../../src/strategies/chunk/markdown-headers.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,MAAM,UAAU,uBAAuB,CACrC,UAAkC,EAAE;IAEpC,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,GAAG,CAAC;IACjD,MAAM,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,IAAI,CAAC;IAElD,OAAO;QACL,IAAI,EAAE,kBAAkB;QAExB,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,8EAA8E;YAC9E,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,CAAC;YAEhE,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,aAAa,GAAG,EAAE,CAAC;YACvB,IAAI,kBAAkB,GAAG,CAAC,CAAC;YAE3B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC;gBAEpD,IAAI,WAAW,EAAE,CAAC;oBAChB,mCAAmC;oBACnC,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;wBAC5B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;wBAC/C,IAAI,OAAO,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;4BACnC,MAAM,CAAC,IAAI,CAAC;gCACV,OAAO;gCACP,QAAQ,EAAE;oCACR,QAAQ,EAAE,IAAI,CAAC,IAAI;oCACnB,MAAM,EAAE,aAAa;oCACrB,YAAY,EAAE,kBAAkB;oCAChC,WAAW,EAAE,QAAQ;iCACtB;gCACD,UAAU,EAAE,QAAQ,IAAI,SAAS;gCACjC,UAAU,EAAE,EAAE;6BACf,CAAC,CAAC;wBACL,CAAC;oBACH,CAAC;oBAED,kBAAkB;oBAClB,kBAAkB,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;oBAC3C,aAAa,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;oBAC/B,YAAY,GAAG,CAAC,IAAI,CAAC,CAAC;gBACxB,CAAC;qBAAM,CAAC;oBACN,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBAC1B,CAAC;gBAED,wCAAwC;gBACxC,MAAM,WAAW,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;gBACnD,IAAI,WAAW,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;oBAC3D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC/C,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO;wBACP,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,IAAI;4BACnB,MAAM,EAAE,aAAa;4BACrB,YAAY,EAAE,kBAAkB;4BAChC,WAAW,EAAE,QAAQ;4BACrB,SAAS,EAAE,IAAI;yBAChB;wBACD,UAAU,EAAE,QAAQ,IAAI,SAAS;wBACjC,UAAU,EAAE,EAAE;qBACf,CAAC,CAAC;oBACH,oEAAoE;oBACpE,YAAY,GAAG,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBAClE,CAAC;YACH,CAAC;YAED,aAAa;YACb,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC/C,IAAI,OAAO,CAAC,MAAM,IAAI,YAAY,EAAE,CAAC;oBACnC,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO;wBACP,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,IAAI;4BACnB,MAAM,EAAE,aAAa;4BACrB,YAAY,EAAE,kBAAkB;4BAChC,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,IAAI;yBACd;wBACD,UAAU,EAAE,QAAQ,IAAI,SAAS;wBACjC,UAAU,EAAE,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,eAAe,CAAC,IAAY,EAAE,SAAkB;YAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;YACrD,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,WAAW,EAAE,CAAC,CAAC,WAAW;gBAC1B,YAAY,EAAE,WAAW,EAAE,CAAC,CAAC,CAAC;gBAC9B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM;aACpC,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,MAAe;YAC/B,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,GAAE,uBAA4B,GACpC,aAAa,
|
|
1
|
+
{"version":3,"file":"semantic.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/semantic.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,uBAAuB;IACtC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,wBAAgB,gBAAgB,CAC9B,OAAO,GAAE,uBAA4B,GACpC,aAAa,CA+Ef"}
|
|
@@ -32,7 +32,7 @@ export function semanticStrategy(options = {}) {
|
|
|
32
32
|
currentChunk.push(sentence);
|
|
33
33
|
currentSize += sentenceSize;
|
|
34
34
|
}
|
|
35
|
-
// Last chunk
|
|
35
|
+
// Last chunk: append to previous if too small, otherwise save as its own chunk.
|
|
36
36
|
if (currentChunk.length > 0) {
|
|
37
37
|
const content = currentChunk.join(" ").trim();
|
|
38
38
|
if (content.length >= minChars) {
|
|
@@ -48,6 +48,12 @@ export function semanticStrategy(options = {}) {
|
|
|
48
48
|
commitHash: "",
|
|
49
49
|
});
|
|
50
50
|
}
|
|
51
|
+
else if (chunks.length > 0) {
|
|
52
|
+
// Merge into the previous chunk rather than silently dropping it.
|
|
53
|
+
const prev = chunks[chunks.length - 1];
|
|
54
|
+
prev.content = `${prev.content} ${content}`.trim();
|
|
55
|
+
prev.metadata.is_last = true;
|
|
56
|
+
}
|
|
51
57
|
}
|
|
52
58
|
return chunks;
|
|
53
59
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"semantic.js","sourceRoot":"","sources":["../../../src/strategies/chunk/semantic.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,MAAM,UAAU,gBAAgB,CAC9B,UAAmC,EAAE;IAErC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;IAEzC,OAAO;QACL,IAAI,EAAE,UAAU;QAEhB,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,MAAM,MAAM,GAAY,EAAE,CAAC;YAE3B,uCAAuC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAE9C,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAErC,IAAI,WAAW,GAAG,YAAY,GAAG,QAAQ,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;wBAC/B,MAAM,CAAC,IAAI,CAAC;4BACV,OAAO;4BACP,QAAQ,EAAE;gCACR,QAAQ,EAAE,IAAI,CAAC,IAAI;gCACnB,cAAc,EAAE,YAAY,CAAC,MAAM;gCACnC,WAAW,EAAE,QAAQ;6BACtB;4BACD,UAAU,EAAE,QAAQ,IAAI,SAAS;4BACjC,UAAU,EAAE,EAAE;yBACf,CAAC,CAAC;oBACL,CAAC;oBACD,YAAY,GAAG,EAAE,CAAC;oBAClB,WAAW,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC5B,WAAW,IAAI,YAAY,CAAC;YAC9B,CAAC;YAED,
|
|
1
|
+
{"version":3,"file":"semantic.js","sourceRoot":"","sources":["../../../src/strategies/chunk/semantic.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE,MAAM,UAAU,gBAAgB,CAC9B,UAAmC,EAAE;IAErC,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,IAAI,CAAC;IAC1C,MAAM,QAAQ,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC;IAEzC,OAAO;QACL,IAAI,EAAE,UAAU;QAEhB,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,MAAM,MAAM,GAAY,EAAE,CAAC;YAE3B,uCAAuC;YACvC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;YAE9C,IAAI,YAAY,GAAa,EAAE,CAAC;YAChC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;gBACjC,MAAM,YAAY,GAAG,QAAQ,CAAC,MAAM,CAAC;gBAErC,IAAI,WAAW,GAAG,YAAY,GAAG,QAAQ,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACrE,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;oBAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;wBAC/B,MAAM,CAAC,IAAI,CAAC;4BACV,OAAO;4BACP,QAAQ,EAAE;gCACR,QAAQ,EAAE,IAAI,CAAC,IAAI;gCACnB,cAAc,EAAE,YAAY,CAAC,MAAM;gCACnC,WAAW,EAAE,QAAQ;6BACtB;4BACD,UAAU,EAAE,QAAQ,IAAI,SAAS;4BACjC,UAAU,EAAE,EAAE;yBACf,CAAC,CAAC;oBACL,CAAC;oBACD,YAAY,GAAG,EAAE,CAAC;oBAClB,WAAW,GAAG,CAAC,CAAC;gBAClB,CAAC;gBAED,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;gBAC5B,WAAW,IAAI,YAAY,CAAC;YAC9B,CAAC;YAED,gFAAgF;YAChF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBAC5B,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,OAAO,CAAC,MAAM,IAAI,QAAQ,EAAE,CAAC;oBAC/B,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO;wBACP,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,IAAI;4BACnB,cAAc,EAAE,YAAY,CAAC,MAAM;4BACnC,WAAW,EAAE,QAAQ;4BACrB,OAAO,EAAE,IAAI;yBACd;wBACD,UAAU,EAAE,QAAQ,IAAI,SAAS;wBACjC,UAAU,EAAE,EAAE;qBACf,CAAC,CAAC;gBACL,CAAC;qBAAM,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC7B,kEAAkE;oBAClE,MAAM,IAAI,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;oBACvC,IAAI,CAAC,OAAO,GAAG,GAAG,IAAI,CAAC,OAAO,IAAI,OAAO,EAAE,CAAC,IAAI,EAAE,CAAC;oBACnD,IAAI,CAAC,QAAQ,CAAC,OAAO,GAAG,IAAI,CAAC;gBAC/B,CAAC;YACH,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,eAAe,CAAC,IAAY,EAAE,SAAkB;YAC9C,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,MAAM;gBAC3C,UAAU,EAAE,IAAI,CAAC,MAAM;aACxB,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,MAAe;YAC/B,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,OAAO,GAAE,oBAAyB,GACjC,aAAa,
|
|
1
|
+
{"version":3,"file":"token.d.ts","sourceRoot":"","sources":["../../../src/strategies/chunk/token.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,aAAa,EAAS,MAAM,2BAA2B,CAAC;AAIjE,MAAM,WAAW,oBAAoB;IACnC,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAgB,aAAa,CAC3B,OAAO,GAAE,oBAAyB,GACjC,aAAa,CAuEf"}
|
|
@@ -9,7 +9,10 @@ export function tokenStrategy(options = {}) {
|
|
|
9
9
|
const overlap = options.overlap ?? 50;
|
|
10
10
|
const charsPerToken = 4;
|
|
11
11
|
const maxChars = maxTokens * charsPerToken;
|
|
12
|
-
|
|
12
|
+
// Clamp overlap so it never equals or exceeds maxChars, guaranteeing forward progress.
|
|
13
|
+
const safeOverlap = Math.min(overlap, maxTokens - 1);
|
|
14
|
+
const overlapChars = safeOverlap * charsPerToken;
|
|
15
|
+
const stepChars = maxChars - overlapChars; // minimum advance per iteration
|
|
13
16
|
return {
|
|
14
17
|
name: `token-${maxTokens}`,
|
|
15
18
|
async chunk(text, filePath) {
|
|
@@ -17,12 +20,16 @@ export function tokenStrategy(options = {}) {
|
|
|
17
20
|
let start = 0;
|
|
18
21
|
while (start < text.length) {
|
|
19
22
|
let end = Math.min(start + maxChars, text.length);
|
|
20
|
-
//
|
|
23
|
+
// Only snap to a sentence/line boundary when it falls in the second half
|
|
24
|
+
// of the window. Searching from the full end position can land on a break
|
|
25
|
+
// point just past `start` (e.g. first newline inside a template literal),
|
|
26
|
+
// which produces tiny chunks and triggers the one-char crawl loop.
|
|
21
27
|
if (end < text.length) {
|
|
28
|
+
const minBreak = start + Math.floor(maxChars / 2);
|
|
22
29
|
const lastPeriod = text.lastIndexOf(".", end);
|
|
23
30
|
const lastNewline = text.lastIndexOf("\n", end);
|
|
24
31
|
const breakPoint = Math.max(lastPeriod, lastNewline);
|
|
25
|
-
if (breakPoint
|
|
32
|
+
if (breakPoint >= minBreak) {
|
|
26
33
|
end = breakPoint + 1;
|
|
27
34
|
}
|
|
28
35
|
}
|
|
@@ -41,7 +48,10 @@ export function tokenStrategy(options = {}) {
|
|
|
41
48
|
commitHash: "", // Will be filled by caller
|
|
42
49
|
});
|
|
43
50
|
}
|
|
44
|
-
|
|
51
|
+
// Advance by overlap window, but never less than stepChars to prevent
|
|
52
|
+
// the one-char crawl when the boundary lands too close to start.
|
|
53
|
+
const nextStart = end - overlapChars;
|
|
54
|
+
start = nextStart > start ? nextStart : start + stepChars;
|
|
45
55
|
}
|
|
46
56
|
return chunks;
|
|
47
57
|
},
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/strategies/chunk/token.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IAC3C,MAAM,
|
|
1
|
+
{"version":3,"file":"token.js","sourceRoot":"","sources":["../../../src/strategies/chunk/token.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,0BAA0B,EAAE,MAAM,sBAAsB,CAAC;AAOlE;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAC3B,UAAgC,EAAE;IAElC,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,GAAG,CAAC;IAC3C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;IACtC,MAAM,aAAa,GAAG,CAAC,CAAC;IACxB,MAAM,QAAQ,GAAG,SAAS,GAAG,aAAa,CAAC;IAC3C,uFAAuF;IACvF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IACrD,MAAM,YAAY,GAAG,WAAW,GAAG,aAAa,CAAC;IACjD,MAAM,SAAS,GAAG,QAAQ,GAAG,YAAY,CAAC,CAAC,gCAAgC;IAE3E,OAAO;QACL,IAAI,EAAE,SAAS,SAAS,EAAE;QAE1B,KAAK,CAAC,KAAK,CAAC,IAAY,EAAE,QAAiB;YACzC,MAAM,MAAM,GAAY,EAAE,CAAC;YAC3B,IAAI,KAAK,GAAG,CAAC,CAAC;YAEd,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;gBAC3B,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;gBAElD,yEAAyE;gBACzE,0EAA0E;gBAC1E,0EAA0E;gBAC1E,mEAAmE;gBACnE,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;oBACtB,MAAM,QAAQ,GAAG,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC;oBAClD,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;oBAC9C,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;oBAChD,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,CAAC;oBACrD,IAAI,UAAU,IAAI,QAAQ,EAAE,CAAC;wBAC3B,GAAG,GAAG,UAAU,GAAG,CAAC,CAAC;oBACvB,CAAC;gBACH,CAAC;gBAED,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC9C,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,CAAC,IAAI,CAAC;wBACV,OAAO;wBACP,QAAQ,EAAE;4BACR,QAAQ,EAAE,IAAI,CAAC,IAAI;4BACnB,WAAW,EAAE,MAAM,CAAC,MAAM;4BAC1B,WAAW,EAAE,QAAQ;4BACrB,UAAU,EAAE,KAAK;4BACjB,QAAQ,EAAE,GAAG;yBACd;wBACD,UAAU,EAAE,QAAQ,IAAI,SAAS;wBACjC,UAAU,EAAE,EAAE,EAAE,2BAA2B;qBAC5C,CAAC,CAAC;gBACL,CAAC;gBAED,sEAAsE;gBACtE,iEAAiE;gBACjE,MAAM,SAAS,GAAG,GAAG,GAAG,YAAY,CAAC;gBACrC,KAAK,GAAG,SAAS,GAAG,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,GAAG,SAAS,CAAC;YAC5D,CAAC;YAED,OAAO,MAAM,CAAC;QAChB,CAAC;QAED,eAAe,CAAC,IAAY,EAAE,SAAkB;YAC9C,OAAO;gBACL,QAAQ,EAAE,IAAI,CAAC,IAAI;gBACnB,UAAU,EAAE,IAAI,CAAC,MAAM;gBACvB,gBAAgB,EAAE,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,aAAa,CAAC;aACzD,CAAC;QACJ,CAAC;QAED,iBAAiB,CAAC,MAAe;YAC/B,OAAO,0BAA0B,CAAC,MAAM,CAAC,CAAC;QAC5C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.7",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
|
|
6
6
|
"repository": {
|
|
@@ -16,9 +16,6 @@
|
|
|
16
16
|
"types": "./dist/index.d.ts"
|
|
17
17
|
}
|
|
18
18
|
},
|
|
19
|
-
"bin": {
|
|
20
|
-
"virage": "./dist/bin/virage.js"
|
|
21
|
-
},
|
|
22
19
|
"files": [
|
|
23
20
|
"dist",
|
|
24
21
|
"schemas",
|
|
@@ -54,13 +51,8 @@
|
|
|
54
51
|
"author": "Vivantel",
|
|
55
52
|
"license": "MIT",
|
|
56
53
|
"dependencies": {
|
|
57
|
-
"@inquirer/prompts": "^8.5.2",
|
|
58
54
|
"better-sqlite3": "^12.10.0",
|
|
59
55
|
"chokidar": "^5.0.0",
|
|
60
|
-
"cli-progress": "^3.12.0",
|
|
61
|
-
"commander": "^15.0.0",
|
|
62
|
-
"consola": "^3.4.2",
|
|
63
|
-
"dotenv": "^17.4.2",
|
|
64
56
|
"glob": "^13.0.6",
|
|
65
57
|
"simple-git": "^3.25.0"
|
|
66
58
|
},
|
|
@@ -68,7 +60,6 @@
|
|
|
68
60
|
"@eslint/js": "^10.0.1",
|
|
69
61
|
"@supabase/supabase-js": "^2.106.2",
|
|
70
62
|
"@types/better-sqlite3": "^7.6.0",
|
|
71
|
-
"@types/cli-progress": "^3.11.6",
|
|
72
63
|
"@types/node": "^25.9.1",
|
|
73
64
|
"eslint": "^10.4.1",
|
|
74
65
|
"prettier": "^3.8.3",
|
package/dist/bin/virage.d.ts
DELETED
package/dist/bin/virage.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"virage.d.ts","sourceRoot":"","sources":["../../src/bin/virage.ts"],"names":[],"mappings":""}
|