@vivantel/virage-core 0.2.3 → 0.2.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bin/virage.js +6 -6
- package/dist/bin/virage.js.map +1 -1
- package/dist/cli/dashboard.d.ts +1 -1
- package/dist/cli/dashboard.d.ts.map +1 -1
- package/dist/cli/dashboard.js +13 -14
- package/dist/cli/dashboard.js.map +1 -1
- package/dist/cli/evaluate.d.ts.map +1 -1
- package/dist/cli/evaluate.js +7 -2
- package/dist/cli/evaluate.js.map +1 -1
- package/dist/cli/experiment.d.ts.map +1 -1
- package/dist/cli/experiment.js +7 -2
- package/dist/cli/experiment.js.map +1 -1
- package/dist/cli/init.d.ts.map +1 -1
- package/dist/cli/init.js +69 -10
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/viz.d.ts +1 -1
- package/dist/cli/viz.d.ts.map +1 -1
- package/dist/cli/viz.js +6 -4
- package/dist/cli/viz.js.map +1 -1
- package/dist/core/embedder.d.ts +7 -4
- package/dist/core/embedder.d.ts.map +1 -1
- package/dist/core/embedder.js +58 -99
- package/dist/core/embedder.js.map +1 -1
- package/dist/core/embeddings-db.d.ts +20 -0
- package/dist/core/embeddings-db.d.ts.map +1 -0
- package/dist/core/embeddings-db.js +181 -0
- package/dist/core/embeddings-db.js.map +1 -0
- package/dist/core/orchestrator.d.ts +1 -0
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +42 -16
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/uploader.d.ts +8 -2
- package/dist/core/uploader.d.ts.map +1 -1
- package/dist/core/uploader.js +47 -30
- package/dist/core/uploader.js.map +1 -1
- package/dist/core/virage-defaults.d.ts +1 -0
- package/dist/core/virage-defaults.d.ts.map +1 -1
- package/dist/core/virage-defaults.js +3 -0
- package/dist/core/virage-defaults.js.map +1 -1
- package/dist/progress/progress-bar.js +1 -1
- package/dist/progress/progress-bar.js.map +1 -1
- package/package.json +3 -1
- package/schemas/virage.config.schema.json +5 -0
|
@@ -2,16 +2,15 @@ import { GitTracker } from "./git-tracker.js";
|
|
|
2
2
|
import { ChunkProcessor } from "./chunk-processor.js";
|
|
3
3
|
import { EmbedderProcessor } from "./embedder.js";
|
|
4
4
|
import { Uploader } from "./uploader.js";
|
|
5
|
+
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
5
6
|
import { TelemetryCollector } from "./telemetry.js";
|
|
6
|
-
import { createProgressBar } from "../progress/progress-bar.js";
|
|
7
|
+
import { createProgressBar, } from "../progress/progress-bar.js";
|
|
7
8
|
import { NullLogger } from "../logger/null-logger.js";
|
|
8
9
|
import { readFile } from "fs/promises";
|
|
9
|
-
import { readEmbeddingsFile } from "./embeddings-io.js";
|
|
10
10
|
import { defaultChunksFile, defaultEmbeddingsFile } from "./virage-defaults.js";
|
|
11
|
-
|
|
12
|
-
const { chunks } = await readEmbeddingsFile(embeddingsFile);
|
|
11
|
+
function loadPreviousState(db) {
|
|
13
12
|
const state = new Map();
|
|
14
|
-
for (const chunk of
|
|
13
|
+
for (const chunk of db.getAll()) {
|
|
15
14
|
if (chunk.sourceFile && chunk.commitHash) {
|
|
16
15
|
state.set(chunk.sourceFile, chunk.commitHash);
|
|
17
16
|
}
|
|
@@ -55,6 +54,8 @@ export class Orchestrator {
|
|
|
55
54
|
const logger = (opts.logger ?? new NullLogger()).withTag("orchestrator");
|
|
56
55
|
const telemetry = opts.telemetry ? new TelemetryCollector() : null;
|
|
57
56
|
telemetry?.start();
|
|
57
|
+
const dbPath = this.embeddingsFile.replace(/\.json$/, ".db");
|
|
58
|
+
const db = new EmbeddingsDb(dbPath);
|
|
58
59
|
let uploadStats = { uploaded: 0, deleted: 0 };
|
|
59
60
|
logger.debug(`Config: chunksFile=${this.chunksFile} embeddingsFile=${this.embeddingsFile} force=${opts.force ?? false}`);
|
|
60
61
|
try {
|
|
@@ -66,7 +67,7 @@ export class Orchestrator {
|
|
|
66
67
|
const currentState = await gitTracker.getCurrentState();
|
|
67
68
|
const previousState = opts.force
|
|
68
69
|
? new Map()
|
|
69
|
-
:
|
|
70
|
+
: loadPreviousState(db);
|
|
70
71
|
const { toProcess, toDelete } = await gitTracker.getChangedFiles(previousState, currentState);
|
|
71
72
|
const gitDuration = Date.now() - t1;
|
|
72
73
|
logger.verbose(`Git tracking done in ${gitDuration}ms`);
|
|
@@ -78,6 +79,7 @@ export class Orchestrator {
|
|
|
78
79
|
});
|
|
79
80
|
if (toProcess.length === 0 && toDelete.length === 0 && !opts.force) {
|
|
80
81
|
logger.info("✨ No changes detected.");
|
|
82
|
+
db.close();
|
|
81
83
|
return;
|
|
82
84
|
}
|
|
83
85
|
logger.info(`📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete`);
|
|
@@ -95,8 +97,13 @@ export class Orchestrator {
|
|
|
95
97
|
? []
|
|
96
98
|
: await loadExistingChunks(this.chunksFile);
|
|
97
99
|
const chunkBar = createProgressBar("Chunking", toProcess.length);
|
|
98
|
-
|
|
99
|
-
|
|
100
|
+
let chunks;
|
|
101
|
+
try {
|
|
102
|
+
chunks = await chunkProcessor.processFiles(toProcess, fileState, existingChunks, (done, total) => chunkBar.update(done < total ? done : total));
|
|
103
|
+
}
|
|
104
|
+
finally {
|
|
105
|
+
chunkBar.stop();
|
|
106
|
+
}
|
|
100
107
|
await chunkProcessor.saveChunksLocal(chunks, this.chunksFile);
|
|
101
108
|
const chunkDuration = Date.now() - t2;
|
|
102
109
|
logger.verbose(`Chunking done in ${chunkDuration}ms`);
|
|
@@ -108,12 +115,17 @@ export class Orchestrator {
|
|
|
108
115
|
});
|
|
109
116
|
if (chunks.length === 0) {
|
|
110
117
|
logger.warn("⚠️ No chunks generated. Exiting.");
|
|
118
|
+
db.close();
|
|
111
119
|
return;
|
|
112
120
|
}
|
|
113
121
|
// Step 3: Generate embeddings
|
|
114
122
|
logger.info("🔢 Step 3: Generating embeddings...");
|
|
115
123
|
const t3 = Date.now();
|
|
116
124
|
const embedBars = [];
|
|
125
|
+
const uploader = new Uploader(this.config.vectorStore, {
|
|
126
|
+
retry: opts.retry,
|
|
127
|
+
logger: opts.logger,
|
|
128
|
+
});
|
|
117
129
|
const embedder = new EmbedderProcessor(this.config.embedder, {
|
|
118
130
|
rateLimitMs: opts.rateLimitMs,
|
|
119
131
|
batchSize: opts.batchSize,
|
|
@@ -121,6 +133,7 @@ export class Orchestrator {
|
|
|
121
133
|
retry: opts.retry,
|
|
122
134
|
concurrency: opts.concurrency,
|
|
123
135
|
vectorStoreName: this.config.vectorStore.name,
|
|
136
|
+
minIngestionBatchSize: opts.minIngestionBatchSize,
|
|
124
137
|
logger: opts.logger,
|
|
125
138
|
onProgress: (done, total) => {
|
|
126
139
|
if (embedBars.length === 0)
|
|
@@ -128,8 +141,17 @@ export class Orchestrator {
|
|
|
128
141
|
embedBars[0].update(done < total ? done : total);
|
|
129
142
|
},
|
|
130
143
|
});
|
|
131
|
-
|
|
132
|
-
|
|
144
|
+
let newEmbeddings;
|
|
145
|
+
try {
|
|
146
|
+
newEmbeddings = await embedder.run(db, this.chunksFile, opts.force || false, opts.skipUpload
|
|
147
|
+
? undefined
|
|
148
|
+
: async () => {
|
|
149
|
+
await uploader.uploadPending(db);
|
|
150
|
+
});
|
|
151
|
+
}
|
|
152
|
+
finally {
|
|
153
|
+
embedBars[0]?.stop();
|
|
154
|
+
}
|
|
133
155
|
const embedDuration = Date.now() - t3;
|
|
134
156
|
logger.verbose(`Embedding done in ${embedDuration}ms`);
|
|
135
157
|
telemetry?.recordEmbedding({
|
|
@@ -139,8 +161,7 @@ export class Orchestrator {
|
|
|
139
161
|
});
|
|
140
162
|
// Step 4: Upload
|
|
141
163
|
if (opts.dryRun) {
|
|
142
|
-
const
|
|
143
|
-
const { toUpload: dryUpload, toDelete: dryDelete } = await uploader.getItemsToUpload(this.embeddingsFile, opts.force || false);
|
|
164
|
+
const { toUpload: dryUpload, toDelete: dryDelete } = await uploader.getItemsToUpload(db, opts.force || false);
|
|
144
165
|
logger.info("📤 Step 4: Upload (dry-run — no changes written)");
|
|
145
166
|
logger.info(` Would upload: ${dryUpload.length} document(s)`);
|
|
146
167
|
logger.info(` Would delete: ${dryDelete.length} source file(s)`);
|
|
@@ -148,11 +169,13 @@ export class Orchestrator {
|
|
|
148
169
|
else if (!opts.skipUpload) {
|
|
149
170
|
logger.info("📤 Step 4: Uploading to vector store...");
|
|
150
171
|
const t4 = Date.now();
|
|
151
|
-
const
|
|
152
|
-
|
|
153
|
-
|
|
172
|
+
const uploadBars = [];
|
|
173
|
+
uploadStats = await uploader.sync(db, opts.force || false, (done, total) => {
|
|
174
|
+
if (uploadBars.length === 0)
|
|
175
|
+
uploadBars.push(createProgressBar("Uploading", total));
|
|
176
|
+
uploadBars[0].update(done < total ? done : total);
|
|
154
177
|
});
|
|
155
|
-
|
|
178
|
+
uploadBars[0]?.stop();
|
|
156
179
|
const uploadDuration = Date.now() - t4;
|
|
157
180
|
logger.verbose(`Upload done in ${uploadDuration}ms`);
|
|
158
181
|
telemetry?.recordUpload({
|
|
@@ -187,6 +210,9 @@ export class Orchestrator {
|
|
|
187
210
|
}
|
|
188
211
|
throw err;
|
|
189
212
|
}
|
|
213
|
+
finally {
|
|
214
|
+
db.close();
|
|
215
|
+
}
|
|
190
216
|
}
|
|
191
217
|
}
|
|
192
218
|
//# sourceMappingURL=orchestrator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EACL,iBAAiB,GAElB,MAAM,6BAA6B,CAAC;AASrC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAwBhF,SAAS,iBAAiB,CAAC,EAAgB;IACzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,UAAkB;IAClD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,MAA2B,EAC3B,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;AACH,CAAC;AAED,MAAM,OAAO,YAAY;IACf,MAAM,CAAoB;IAC1B,UAAU,CAAS;IACnB,cAAc,CAAS;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;QACpE,IAAI,CAAC,cAAc;YACjB,MAAM,CAAC,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,SAAS,EAAE,KAAK,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QAEpC,IAAI,WAAW,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE9C,MAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,UAAU,mBAAmB,IAAI,CAAC,cAAc,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,CAC3G,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE3C,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;YAExD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK;gBAC9B,CAAC,CAAC,IAAI,GAAG,EAAkB;gBAC3B,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE1B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,eAAe,CAC9D,aAAa,EACb,YAAY,CACb,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,wBAAwB,WAAW,IAAI,CAAC,CAAC;YAExD,SAAS,EAAE,iBAAiB,CAAC;gBAC3B,UAAU,EAAE,WAAW;gBACvB,YAAY,EAAE,YAAY,CAAC,IAAI;gBAC/B,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnE,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACtC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,MAAM,CAAC,IAAI,CACT,eAAe,SAAS,CAAC,MAAM,gBAAgB,QAAQ,CAAC,MAAM,YAAY,CAC3E,CAAC;YAEF,wCAAwC;YACxC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,cAAc,GAAG,IAAI,cAAc,CACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAGtB,CAAC;YACJ,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,IAAI;oBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK;gBAC/B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACjE,IAAI,MAAe,CAAC;YACpB,IAAI,CAAC;gBACH,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,CACxC,SAAS,EACT,SAAS,EACT,cAAc,EACd,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC9D,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,QAAQ,CAAC,IAAI,EAAE,CAAC;YAClB,CAAC;YACD,MAAM,cAAc,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,oBAAoB,aAAa,IAAI,CAAC,CAAC;YAEtD,SAAS,EAAE,cAAc,CAAC;gBACxB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,SAAS,CAAC,MAAM;gBAChC,eAAe,EAAE,MAAM,CAAC,MAAM;gBAC9B,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;gBAChD,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,8BAA8B;YAC9B,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACnD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,SAAS,GAAkB,EAAE,CAAC;YAEpC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACrD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBAC3D,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI;gBAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;gBACjD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBAC1B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;wBACxB,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;oBACxD,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACnD,CAAC;aACF,CAAC,CAAC;YAEH,IAAI,aAA8B,CAAC;YACnC,IAAI,CAAC;gBACH,aAAa,GAAG,MAAM,QAAQ,CAAC,GAAG,CAChC,EAAE,EACF,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,KAAK,IAAI,KAAK,EACnB,IAAI,CAAC,UAAU;oBACb,CAAC,CAAC,SAAS;oBACX,CAAC,CAAC,KAAK,IAAI,EAAE;wBACT,MAAM,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;oBACnC,CAAC,CACN,CAAC;YACJ,CAAC;oBAAS,CAAC;gBACT,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YACvB,CAAC;YAED,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,qBAAqB,aAAa,IAAI,CAAC,CAAC;YAEvD,SAAS,EAAE,eAAe,CAAC;gBACzB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,aAAa,CAAC,MAAM;gBACpC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;aACpD,CAAC,CAAC;YAEH,iBAAiB;YACjB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAChD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,MAAM,UAAU,GAAkB,EAAE,CAAC;gBACrC,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,CAC/B,EAAE,EACF,IAAI,CAAC,KAAK,IAAI,KAAK,EACnB,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBACd,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;wBACzB,UAAU,CAAC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;oBACzD,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACpD,CAAC,CACF,CAAC;gBACF,UAAU,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;gBAEtB,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;gBAErD,SAAS,EAAE,YAAY,CAAC;oBACtB,UAAU,EAAE,cAAc;oBAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;YAE3C,IAAI,SAAS,EAAE,CAAC;gBACd,SAAS,CAAC,MAAM,EAAE,CAAC;gBACnB,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3C,aAAa,EACb,gBAAgB,CACjB,CAAC;gBACF,MAAM,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;oBAC5D,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;oBAC3C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,MAAM;oBACnC,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,EAAE,MAAM,EAAE,CAAC;YAEpB,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE;oBAC1D,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;oBACvD,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;iBAC5C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|
package/dist/core/uploader.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { VectorStore, EmbeddedChunk } from "../interfaces/index.js";
|
|
2
2
|
import type { Logger } from "../interfaces/logger.js";
|
|
3
3
|
import { RetryOptions } from "./utils.js";
|
|
4
|
+
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
5
|
+
export declare function isFatalVectorStoreError(err: unknown): boolean;
|
|
4
6
|
export declare class Uploader {
|
|
5
7
|
private vectorStore;
|
|
6
8
|
private retryOptions;
|
|
@@ -10,11 +12,15 @@ export declare class Uploader {
|
|
|
10
12
|
logger?: Logger;
|
|
11
13
|
});
|
|
12
14
|
private chunkToDocument;
|
|
13
|
-
getItemsToUpload(
|
|
15
|
+
getItemsToUpload(db: EmbeddingsDb, force?: boolean): Promise<{
|
|
14
16
|
toUpload: EmbeddedChunk[];
|
|
15
17
|
toDelete: string[];
|
|
16
18
|
}>;
|
|
17
|
-
|
|
19
|
+
uploadPending(db: EmbeddingsDb, onProgress?: (done: number, total: number) => void): Promise<{
|
|
20
|
+
uploaded: number;
|
|
21
|
+
deleted: number;
|
|
22
|
+
}>;
|
|
23
|
+
sync(db: EmbeddingsDb, force?: boolean, onProgress?: (done: number, total: number) => void): Promise<{
|
|
18
24
|
uploaded: number;
|
|
19
25
|
deleted: number;
|
|
20
26
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EAEX,aAAa,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EAEX,aAAa,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAa,YAAY,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAK7D;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGrB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,YAAY,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAOzD,OAAO,CAAC,eAAe;IAejB,gBAAgB,CACpB,EAAE,EAAE,YAAY,EAChB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IA4DI,aAAa,CACjB,EAAE,EAAE,YAAY,EAChB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACjD,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IA2B3C,IAAI,CACR,EAAE,EAAE,YAAY,EAChB,KAAK,GAAE,OAAe,EACtB,UAAU,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,GACjD,OAAO,CAAC;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CA+DH"}
|
package/dist/core/uploader.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { NullLogger } from "../logger/null-logger.js";
|
|
2
2
|
import { createHash } from "crypto";
|
|
3
|
-
import { UploadError } from "./errors.js";
|
|
4
3
|
import { withRetry } from "./utils.js";
|
|
5
|
-
import { readEmbeddingsFile } from "./embeddings-io.js";
|
|
6
4
|
function contentHash(chunk) {
|
|
7
5
|
return (chunk.contentHash ??
|
|
8
6
|
createHash("sha256").update(chunk.content).digest("hex").slice(0, 16));
|
|
9
7
|
}
|
|
8
|
+
export function isFatalVectorStoreError(err) {
|
|
9
|
+
const msg = String(err instanceof Error ? err.message : err).toLowerCase();
|
|
10
|
+
return /schema error|schema mismatch|unauthorized|authentication failed/.test(msg);
|
|
11
|
+
}
|
|
10
12
|
export class Uploader {
|
|
11
13
|
vectorStore;
|
|
12
14
|
retryOptions;
|
|
@@ -27,26 +29,11 @@ export class Uploader {
|
|
|
27
29
|
collection,
|
|
28
30
|
};
|
|
29
31
|
}
|
|
30
|
-
async getItemsToUpload(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
catch (err) {
|
|
36
|
-
throw new UploadError(`Failed to load embeddings from ${embeddingsFile}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
37
|
-
suggestion: "Run the pipeline without --skip-upload to regenerate embeddings first.",
|
|
38
|
-
cause: err,
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
const { meta, chunks: embeddings } = result;
|
|
42
|
-
if (embeddings.length === 0 && !result) {
|
|
43
|
-
throw new UploadError(`No embeddings found at ${embeddingsFile}`, {
|
|
44
|
-
suggestion: "Run the pipeline without --skip-upload to regenerate embeddings first.",
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
this.logger.info(`📖 Loaded ${embeddings.length} embeddings from ${embeddingsFile}`);
|
|
48
|
-
// If the vector store changed since the last run, force a full re-upload to ensure
|
|
49
|
-
// the new store starts with consistent state.
|
|
32
|
+
async getItemsToUpload(db, force = false) {
|
|
33
|
+
const meta = db.getMeta();
|
|
34
|
+
const embeddings = db.getAll();
|
|
35
|
+
this.logger.info(`📖 Loaded ${embeddings.length} embeddings from db`);
|
|
36
|
+
// If the vector store changed since the last run, force a full re-upload.
|
|
50
37
|
let effectiveForce = force;
|
|
51
38
|
if (!force &&
|
|
52
39
|
meta?.vectorStoreName &&
|
|
@@ -85,10 +72,30 @@ export class Uploader {
|
|
|
85
72
|
toDelete: [...toDeleteSet],
|
|
86
73
|
};
|
|
87
74
|
}
|
|
88
|
-
async
|
|
75
|
+
async uploadPending(db, onProgress) {
|
|
76
|
+
const pending = db.getPending();
|
|
77
|
+
if (pending.length === 0)
|
|
78
|
+
return { uploaded: 0, deleted: 0 };
|
|
79
|
+
this.logger.verbose(`📤 Uploading ${pending.length} pending embeddings...`);
|
|
80
|
+
const batchSize = 50;
|
|
81
|
+
let uploaded = 0;
|
|
82
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
83
|
+
const batch = pending.slice(i, i + batchSize);
|
|
84
|
+
const documents = batch.map((e) => this.chunkToDocument(e));
|
|
85
|
+
await withRetry(() => this.vectorStore.upsert(documents), {
|
|
86
|
+
...this.retryOptions,
|
|
87
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
88
|
+
}, this.logger);
|
|
89
|
+
db.markUploaded(batch.map((e) => contentHash(e)));
|
|
90
|
+
uploaded += batch.length;
|
|
91
|
+
onProgress?.(uploaded, pending.length);
|
|
92
|
+
}
|
|
93
|
+
return { uploaded: pending.length, deleted: 0 };
|
|
94
|
+
}
|
|
95
|
+
async sync(db, force = false, onProgress) {
|
|
89
96
|
this.logger.info("📤 Starting incremental upload...");
|
|
90
97
|
await this.vectorStore.initialize();
|
|
91
|
-
const { toUpload, toDelete } = await this.getItemsToUpload(
|
|
98
|
+
const { toUpload, toDelete } = await this.getItemsToUpload(db, force);
|
|
92
99
|
this.logger.info(`📊 Need to upload: ${toUpload.length} documents`);
|
|
93
100
|
this.logger.info(` Need to delete: ${toDelete.length} files`);
|
|
94
101
|
if (toUpload.length === 0 && toDelete.length === 0) {
|
|
@@ -96,16 +103,26 @@ export class Uploader {
|
|
|
96
103
|
return { uploaded: 0, deleted: 0 };
|
|
97
104
|
}
|
|
98
105
|
if (toDelete.length > 0) {
|
|
99
|
-
await withRetry(() => this.vectorStore.deleteBySourceFile(toDelete),
|
|
106
|
+
await withRetry(() => this.vectorStore.deleteBySourceFile(toDelete), {
|
|
107
|
+
...this.retryOptions,
|
|
108
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
109
|
+
}, this.logger);
|
|
100
110
|
this.logger.verbose(`🗑️ Deleted ${toDelete.length} obsolete docs`);
|
|
101
111
|
}
|
|
102
112
|
if (toUpload.length > 0) {
|
|
103
|
-
const documents = toUpload.map((e) => this.chunkToDocument(e));
|
|
104
113
|
const batchSize = 50;
|
|
105
|
-
const totalBatches = Math.ceil(
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
114
|
+
const totalBatches = Math.ceil(toUpload.length / batchSize);
|
|
115
|
+
let uploaded = 0;
|
|
116
|
+
for (let i = 0; i < toUpload.length; i += batchSize) {
|
|
117
|
+
const batch = toUpload.slice(i, i + batchSize);
|
|
118
|
+
const documents = batch.map((e) => this.chunkToDocument(e));
|
|
119
|
+
await withRetry(() => this.vectorStore.upsert(documents), {
|
|
120
|
+
...this.retryOptions,
|
|
121
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
122
|
+
}, this.logger);
|
|
123
|
+
db.markUploaded(batch.map((e) => contentHash(e)));
|
|
124
|
+
uploaded += batch.length;
|
|
125
|
+
onProgress?.(uploaded, toUpload.length);
|
|
109
126
|
const batchNum = Math.floor(i / batchSize) + 1;
|
|
110
127
|
this.logger.verbose(` Batch ${batchNum}/${totalBatches}: ${batch.length} docs`);
|
|
111
128
|
this.logger.silly(` IDs: ${batch
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAgB,MAAM,YAAY,CAAC;AAGrD,SAAS,WAAW,CAAC,KAAoB;IACvC,OAAO,CACL,KAAK,CAAC,WAAW;QACjB,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,GAAY;IAClD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC3E,OAAO,iEAAiE,CAAC,IAAI,CAC3E,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,QAAQ;IACX,WAAW,CAAc;IACzB,YAAY,CAAe;IAC3B,MAAM,CAAS;IAEvB,YACE,WAAwB,EACxB,UAAqD,EAAE;QAEvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvE,CAAC;IAEO,eAAe,CACrB,KAAoB,EACpB,UAAmB;QAEnB,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC;YAC/B,UAAU;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,EAAgB,EAChB,QAAiB,KAAK;QAKtB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QAE/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,UAAU,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEtE,0EAA0E;QAC1E,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IACE,CAAC,KAAK;YACN,IAAI,EAAE,eAAe;YACrB,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,WAAW,CAAC,IAAI,EAC9C,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;YAChE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;QAC5D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,oBAAoB,aAAa,CAAC,IAAI,oBAAoB,CAC3D,CAAC;QAEF,MAAM,YAAY,GAAoB,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAEtC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAEvD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,yBAAyB,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAClF,CAAC;gBACF,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,YAAY,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,WAAW,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CACvG,CAAC;gBACF,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAChC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,WAAW,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,cAAc,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,QAAQ,EAAE,CAAC,GAAG,WAAW,CAAC;SAC3B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,EAAgB,EAChB,UAAkD;QAElD,MAAM,OAAO,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE7D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,OAAO,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE5E,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,EACxC;gBACE,GAAG,IAAI,CAAC,YAAY;gBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;aACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;YACF,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAClD,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,UAAU,EAAE,CAAC,QAAQ,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;QACzC,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAgB,EAChB,QAAiB,KAAK,EACtB,UAAkD;QAKlD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAEtD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAEpC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAEtE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;QAEhE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YAC3C,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrC,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EACnD;gBACE,GAAG,IAAI,CAAC,YAAY;gBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;aACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,QAAQ,CAAC,MAAM,gBAAgB,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,EAAE,CAAC;YACrB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;YAC5D,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5D,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,EACxC;oBACE,GAAG,IAAI,CAAC,YAAY;oBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;iBACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;gBACF,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,QAAQ,IAAI,KAAK,CAAC,MAAM,CAAC;gBACzB,UAAU,EAAE,CAAC,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,QAAQ,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,OAAO,CAC5D,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,UAAU,KAAK;qBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC;qBAC7C,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IACjE,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"virage-defaults.d.ts","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,qBAAqB,IAAI,MAAM,CAE9C"}
|
|
1
|
+
{"version":3,"file":"virage-defaults.d.ts","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,wBAAgB,YAAY,IAAI,MAAM,CAErC;AAED,wBAAgB,iBAAiB,IAAI,MAAM,CAE1C;AAED,wBAAgB,qBAAqB,IAAI,MAAM,CAE9C;AAED,wBAAgB,mBAAmB,IAAI,MAAM,CAE5C"}
|
|
@@ -7,4 +7,7 @@ export function defaultChunksFile() {
|
|
|
7
7
|
export function defaultEmbeddingsFile() {
|
|
8
8
|
return `${getVirageDir()}/embeddings.json`;
|
|
9
9
|
}
|
|
10
|
+
export function defaultEmbeddingsDb() {
|
|
11
|
+
return `${getVirageDir()}/embeddings.db`;
|
|
12
|
+
}
|
|
10
13
|
//# sourceMappingURL=virage-defaults.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"virage-defaults.js","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,GAAG,YAAY,EAAE,cAAc,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,OAAO,GAAG,YAAY,EAAE,kBAAkB,CAAC;AAC7C,CAAC"}
|
|
1
|
+
{"version":3,"file":"virage-defaults.js","sourceRoot":"","sources":["../../src/core/virage-defaults.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,YAAY;IAC1B,OAAO,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,IAAI,SAAS,CAAC;AAChD,CAAC;AAED,MAAM,UAAU,iBAAiB;IAC/B,OAAO,GAAG,YAAY,EAAE,cAAc,CAAC;AACzC,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,OAAO,GAAG,YAAY,EAAE,kBAAkB,CAAC;AAC7C,CAAC;AAED,MAAM,UAAU,mBAAmB;IACjC,OAAO,GAAG,YAAY,EAAE,gBAAgB,CAAC;AAC3C,CAAC"}
|
|
@@ -4,7 +4,7 @@ export function createProgressBar(label, total) {
|
|
|
4
4
|
return { update: () => { }, stop: () => { } };
|
|
5
5
|
}
|
|
6
6
|
const bar = new cliProgress.SingleBar({
|
|
7
|
-
format: `${label} [{bar}] {percentage}% | {value}/{total} | ETA: {eta_formatted}`,
|
|
7
|
+
format: `${label} [{bar}] {percentage}% | {value}/{total} | {duration_formatted} elapsed | ETA: {eta_formatted}`,
|
|
8
8
|
clearOnComplete: false,
|
|
9
9
|
hideCursor: true,
|
|
10
10
|
}, cliProgress.Presets.shades_classic);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progress-bar.js","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAOvC,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,
|
|
1
|
+
{"version":3,"file":"progress-bar.js","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAOvC,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,gGAAgG;QAChH,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;KACjB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAEpB,OAAO;QACL,MAAM,CAAC,OAAe;YACpB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,YAAqB;YACxB,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,IAAI,YAAY;gBAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.5",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
|
|
6
6
|
"repository": {
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"license": "MIT",
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@inquirer/prompts": "^8.5.2",
|
|
58
|
+
"better-sqlite3": "^12.10.0",
|
|
58
59
|
"chokidar": "^5.0.0",
|
|
59
60
|
"cli-progress": "^3.12.0",
|
|
60
61
|
"commander": "^15.0.0",
|
|
@@ -66,6 +67,7 @@
|
|
|
66
67
|
"devDependencies": {
|
|
67
68
|
"@eslint/js": "^10.0.1",
|
|
68
69
|
"@supabase/supabase-js": "^2.106.2",
|
|
70
|
+
"@types/better-sqlite3": "^7.6.0",
|
|
69
71
|
"@types/cli-progress": "^3.11.6",
|
|
70
72
|
"@types/node": "^25.9.1",
|
|
71
73
|
"eslint": "^10.4.1",
|
|
@@ -90,6 +90,11 @@
|
|
|
90
90
|
"batchSize": { "type": "number" },
|
|
91
91
|
"maxBatchChars": { "type": "number" },
|
|
92
92
|
"concurrency": { "type": "number" },
|
|
93
|
+
"minIngestionBatchSize": {
|
|
94
|
+
"type": "integer",
|
|
95
|
+
"minimum": 1,
|
|
96
|
+
"description": "Upload embeddings to the vector store mid-run when pending count reaches this threshold."
|
|
97
|
+
},
|
|
93
98
|
"telemetry": { "type": "boolean" },
|
|
94
99
|
"notifications": {
|
|
95
100
|
"type": "object",
|