@vivantel/virage-core 0.2.3 → 0.2.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/init.d.ts.map +1 -1
- package/dist/cli/init.js +69 -10
- package/dist/cli/init.js.map +1 -1
- package/dist/core/embedder.d.ts +7 -4
- package/dist/core/embedder.d.ts.map +1 -1
- package/dist/core/embedder.js +59 -94
- package/dist/core/embedder.js.map +1 -1
- package/dist/core/embeddings-db.d.ts +20 -0
- package/dist/core/embeddings-db.d.ts.map +1 -0
- package/dist/core/embeddings-db.js +181 -0
- package/dist/core/embeddings-db.js.map +1 -0
- package/dist/core/orchestrator.d.ts +1 -0
- package/dist/core/orchestrator.d.ts.map +1 -1
- package/dist/core/orchestrator.js +24 -14
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/uploader.d.ts +8 -2
- package/dist/core/uploader.d.ts.map +1 -1
- package/dist/core/uploader.js +41 -30
- package/dist/core/uploader.js.map +1 -1
- package/dist/progress/progress-bar.js +1 -1
- package/dist/progress/progress-bar.js.map +1 -1
- package/package.json +3 -1
- package/schemas/virage.config.schema.json +5 -0
package/dist/cli/init.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"init.d.ts","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAiPA,wBAAsB,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC,CAmP7C"}
|
package/dist/cli/init.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { checkbox, input, select } from "@inquirer/prompts";
|
|
2
2
|
import { existsSync } from "fs";
|
|
3
|
-
import { readFile, writeFile } from "fs/promises";
|
|
3
|
+
import { readFile, rename, writeFile } from "fs/promises";
|
|
4
4
|
import { join } from "path";
|
|
5
5
|
import { spawn } from "child_process";
|
|
6
6
|
import { loadRegistry } from "../plugin-registry.js";
|
|
@@ -49,6 +49,43 @@ function buildInstallCommand(pm, packages) {
|
|
|
49
49
|
return { cmd: "npm", args: ["install", ...packages] };
|
|
50
50
|
}
|
|
51
51
|
}
|
|
52
|
+
function buildGlobalInstallCommand(pm, packages) {
|
|
53
|
+
switch (pm) {
|
|
54
|
+
case "yarn":
|
|
55
|
+
return { cmd: "yarn", args: ["global", "add", ...packages] };
|
|
56
|
+
case "pnpm":
|
|
57
|
+
return { cmd: "pnpm", args: ["add", "-g", ...packages] };
|
|
58
|
+
case "bun":
|
|
59
|
+
return { cmd: "bun", args: ["add", "-g", ...packages] };
|
|
60
|
+
default:
|
|
61
|
+
return { cmd: "npm", args: ["install", "-g", ...packages] };
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
async function rotateConfigBackups(configPath) {
|
|
65
|
+
const bak = (n) => `${configPath}.bak.${n}`;
|
|
66
|
+
// Delete oldest backup if present
|
|
67
|
+
try {
|
|
68
|
+
await rename(bak(5), bak(5) + ".del");
|
|
69
|
+
const { unlink } = await import("fs/promises");
|
|
70
|
+
await unlink(bak(5) + ".del");
|
|
71
|
+
}
|
|
72
|
+
catch {
|
|
73
|
+
// No .bak.5 to remove
|
|
74
|
+
}
|
|
75
|
+
// Shift .bak.4 → .bak.5, .bak.3 → .bak.4, ... .bak.1 → .bak.2
|
|
76
|
+
for (let i = 4; i >= 1; i--) {
|
|
77
|
+
try {
|
|
78
|
+
await rename(bak(i), bak(i + 1));
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
// No backup at this slot
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
// Rename current file to .bak.1
|
|
85
|
+
if (existsSync(configPath)) {
|
|
86
|
+
await rename(configPath, bak(1));
|
|
87
|
+
}
|
|
88
|
+
}
|
|
52
89
|
function runInstall(cmd, args) {
|
|
53
90
|
return new Promise((resolve, reject) => {
|
|
54
91
|
const proc = spawn(cmd, args, {
|
|
@@ -136,6 +173,20 @@ async function writeEnvVars(envPath, vars) {
|
|
|
136
173
|
// ─── Main wizard ──────────────────────────────────────────────────────────────
|
|
137
174
|
export async function runInit() {
|
|
138
175
|
console.log("\nVirage Config Generator\n");
|
|
176
|
+
const defaultConfigPath = "./virage.config.json";
|
|
177
|
+
if (existsSync(defaultConfigPath)) {
|
|
178
|
+
const overwrite = await select({
|
|
179
|
+
message: `Config file ${defaultConfigPath} already exists. Overwrite?`,
|
|
180
|
+
choices: [
|
|
181
|
+
{ name: "Yes (backup & overwrite)", value: true },
|
|
182
|
+
{ name: "No, cancel", value: false },
|
|
183
|
+
],
|
|
184
|
+
});
|
|
185
|
+
if (!overwrite) {
|
|
186
|
+
console.log("\nCancelled.");
|
|
187
|
+
return;
|
|
188
|
+
}
|
|
189
|
+
}
|
|
139
190
|
const cwd = process.cwd();
|
|
140
191
|
console.log("Scanning project for file types...");
|
|
141
192
|
const detectedGroups = await detectFileExtensions(cwd);
|
|
@@ -231,13 +282,14 @@ export async function runInit() {
|
|
|
231
282
|
}
|
|
232
283
|
}
|
|
233
284
|
const finalState = state;
|
|
234
|
-
// ── Overwrite
|
|
235
|
-
if (
|
|
285
|
+
// ── Overwrite safety net for custom output path ──
|
|
286
|
+
if (finalState.outputPath !== defaultConfigPath &&
|
|
287
|
+
existsSync(finalState.outputPath)) {
|
|
236
288
|
const overwrite = await select({
|
|
237
289
|
message: `${finalState.outputPath} already exists. What would you like to do?`,
|
|
238
290
|
choices: [
|
|
291
|
+
{ name: "Yes (backup & overwrite)", value: true },
|
|
239
292
|
{ name: "No, cancel (keep existing file)", value: false },
|
|
240
|
-
{ name: "Yes, overwrite it", value: true },
|
|
241
293
|
],
|
|
242
294
|
});
|
|
243
295
|
if (!overwrite) {
|
|
@@ -246,6 +298,7 @@ export async function runInit() {
|
|
|
246
298
|
}
|
|
247
299
|
}
|
|
248
300
|
// ── Write config ──
|
|
301
|
+
await rotateConfigBackups(finalState.outputPath);
|
|
249
302
|
const configContent = generateJsonConfig(finalState, registry);
|
|
250
303
|
await writeFile(finalState.outputPath, configContent, "utf-8");
|
|
251
304
|
console.log(`\nCreated ${finalState.outputPath}`);
|
|
@@ -289,15 +342,21 @@ export async function runInit() {
|
|
|
289
342
|
const pkgs = getRequiredPackages(finalState, registry);
|
|
290
343
|
if (pkgs.length > 0) {
|
|
291
344
|
const pm = await detectPackageManager(cwd);
|
|
292
|
-
const
|
|
293
|
-
const
|
|
345
|
+
const localCmd = buildInstallCommand(pm, pkgs);
|
|
346
|
+
const globalCmd = buildGlobalInstallCommand(pm, pkgs);
|
|
347
|
+
const installChoice = await select({
|
|
294
348
|
message: `Install ${pkgs.join(", ")} using ${pm}?`,
|
|
295
349
|
choices: [
|
|
296
|
-
{
|
|
297
|
-
|
|
350
|
+
{
|
|
351
|
+
name: "Yes, install in current folder (recommended)",
|
|
352
|
+
value: "local",
|
|
353
|
+
},
|
|
354
|
+
{ name: "Yes, install globally", value: "global" },
|
|
355
|
+
{ name: "No, I'll install manually", value: "manual" },
|
|
298
356
|
],
|
|
299
357
|
});
|
|
300
|
-
if (
|
|
358
|
+
if (installChoice === "local" || installChoice === "global") {
|
|
359
|
+
const { cmd, args } = installChoice === "global" ? globalCmd : localCmd;
|
|
301
360
|
try {
|
|
302
361
|
await runInstall(cmd, args);
|
|
303
362
|
console.log("\nPackages installed successfully.");
|
|
@@ -307,7 +366,7 @@ export async function runInit() {
|
|
|
307
366
|
}
|
|
308
367
|
}
|
|
309
368
|
else {
|
|
310
|
-
console.log(`\nRun manually:\n ${cmd} ${args.join(" ")}`);
|
|
369
|
+
console.log(`\nRun manually:\n ${localCmd.cmd} ${localCmd.args.join(" ")}`);
|
|
311
370
|
}
|
|
312
371
|
}
|
|
313
372
|
// ── Next steps ──
|
package/dist/cli/init.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAClD,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,YAAY,EAAuB,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,YAAY,GACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,UAAU,EAEV,oBAAoB,GACrB,MAAM,kBAAkB,CAAC;AAE1B,iFAAiF;AAEjF,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,SAAS,QAAQ,CACf,OAAqC;IAErC,OAAO;QACL,GAAG,OAAO;QACV,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,UAA+B,EAAE;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,KAAK,KAAK,UAAU,CAAC;AAC9B,CAAC;AAWD,iFAAiF;AAEjF,SAAS,mBAAmB,CAC1B,KAAkB,EAClB,QAAwB;IAExB,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAChC,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,aAAa,IAAI,aAAa,CAAC,GAAG,KAAK,QAAQ;QACjD,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,UAAU,IAAI,UAAU,CAAC,GAAG,KAAK,QAAQ;QAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,WAAmB;IAEnB,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IACnE,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAC1B,EAAmC,EACnC,QAAkB;IAElB,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACrD,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACrD,KAAK,KAAK;YACR,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACpD;YACE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,IAAc;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE;YAC5B,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,KAAK,CAAC;YACR,CAAC,CAAC,OAAO,EAAE;YACX,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAC/D,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,iFAAiF;AAEjF,MAAM,mBAAmB,GAA2B;IAClD,uBAAuB,EAAE,iBAAiB;IAC1C,aAAa,EAAE,OAAO;IACtB,iBAAiB,EAAE,WAAW;IAC9B,gBAAgB,EAAE,UAAU;CAC7B,CAAC;AAEF,SAAS,kBAAkB,CACzB,KAAkB,EAClB,QAAwB;IAExB,MAAM,eAAe,GACnB,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACrB,CAAC,CAAC,KAAK,CAAC,MAAM;QACd,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAE,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,QAAQ,EAAE,mBAAmB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU;KAC5D,CAAC,CAAC,CAAC;IAEJ,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAC/B,CAAC;IACH,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,WAAW,CAAE,CAAC;IAE7E,2EAA2E;IAC3E,yEAAyE;IACzE,+CAA+C;IAC/C,MAAM,mBAAmB,GACvB,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;QACnE,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;IAE/B,MAAM,MAAM,GAAG;QACb,OAAO,EACL,wEAAwE;QAC1E,QAAQ;QACR,QAAQ,EAAE;YACR,OAAO,EAAE,aAAa,CAAC,OAAO;YAC9B,MAAM,EAAE,aAAa,CAAC,aAAa;SACpC;QACD,WAAW,EAAE;YACX,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,MAAM,EAAE,mBAAmB;SAC5B;QACD,OAAO,EAAE;YACP,UAAU,EAAE,iBAAiB,EAAE;YAC/B,cAAc,EAAE,qBAAqB,EAAE;SACxC;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAChD,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,IAA4B;IAE5B,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAa,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC7C,CAAC,CAAC,CAAC,QAAQ,CAAC;QACZ,CAAC,CAAC,QAAQ;YACR,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;YAChB,CAAC,CAAC,EAAE,CAAC;IAET,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,cAAc,GAAG,QAAQ;aAC5B,KAAK,CAAC,IAAI,CAAC;aACX,IAAI,CACH,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,IAAI,CAAC,CACpE,CAAC;QAEJ,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAE3C,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAEvD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAEzC,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC;QAChB,QAAQ,IAAI,EAAE,CAAC;YACb,kCAAkC;YAClC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;wBAC/B,OAAO,EAAE,8CAA8C;wBACvD,OAAO,EAAE;4BACP,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gCAC5B,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE;gCAC1D,KAAK,EAAE,CAAC,CAAC,IAAI;gCACb,OAAO,EAAE,IAAI;6BACd,CAAC,CAAC;4BACH,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,UAAU,EAAE;yBACpD;qBACF,CAAC,CAAC;oBACH,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;wBACnC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAC5B,OAAO;oBACT,CAAC;oBACD,KAAK,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACzC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAC3B,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CACT,2DAA2D,CAC5D,CAAC;oBACF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;wBAC5B,OAAO,EAAE,wCAAwC;wBACjD,OAAO,EAAE;4BACP,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gCACxB,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,GAAG;gCACnC,KAAK,EAAE,CAAC,CAAC,IAAI;6BACd,CAAC,CAAC;4BACH,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,UAAU,EAAE;yBACpD;qBACF,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;wBAChC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAC5B,OAAO;oBACT,CAAC;oBACD,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACnE,CAAC;gBACD,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,mCAAmC;YACnC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;oBAC1B,OAAO,EAAE,2BAA2B;oBACpC,OAAO,EAAE,QAAQ,CACf,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CACjE;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC;gBACxB,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,uCAAuC;YACvC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;oBAC1B,OAAO,EAAE,qBAAqB;oBAC9B,OAAO,EAAE,QAAQ,CACf,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAC9D;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;gBAC3B,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,4BAA4B;YAC5B,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,aAAa,GAAG,sBAAsB,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;oBAC7B,OAAO,EAAE,2DAA2D;oBACpE,OAAO,EAAE,aAAa;iBACvB,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC7B,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAoB,CAAC;IAExC,wBAAwB;IACxB,IAAI,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACtC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,GAAG,UAAU,CAAC,UAAU,6CAA6C;YAC9E,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,iCAAiC,EAAE,KAAK,EAAE,KAAK,EAAE;gBACzD,EAAE,IAAI,EAAE,mBAAmB,EAAE,KAAK,EAAE,IAAI,EAAE;aAC3C;SACF,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,aAAa,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,SAAS,CAAC,UAAU,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,aAAa,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;IAElD,qBAAqB;IACrB,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,UAAU,CAAC,QAAQ,CACrC,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,UAAU,CAAC,WAAW,CACxC,CAAC;IACF,MAAM,YAAY,GAAG;QACnB,GAAG,CAAC,aAAa,EAAE,OAAO,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;KAC/B,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,MAAM,SAAS,GAA2B,EAAE,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC;gBACxB,OAAO,EAAE,mBAAmB,OAAO,yBAAyB;gBAC5D,OAAO,EAAE,EAAE;aACZ,CAAC,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,SAAS,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YACpC,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,QAAQ,CAAC;YACzB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,6DAA6D,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC7D,CAAC;IAED,qBAAqB;IACrB,MAAM,IAAI,GAAG,mBAAmB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,mBAAmB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACpD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC;YACjC,OAAO,EAAE,WAAW,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG;YAClD,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,gCAAgC,EAAE,KAAK,EAAE,IAAI,EAAE;gBACvD,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,KAAK,EAAE;aACpD;SACF,CAAC,CAAC;QACH,IAAI,aAAa,EAAE,CAAC;YAClB,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CACT,sCAAsC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC9D,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CAAC,sBAAsB,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC7D,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,CAAC,GAAG,CACT,KAAK,QAAQ,EAAE,uDAAuD,CACvE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gFAAgF,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,sCAAsC,CAAC,CAAC;AACnE,CAAC"}
|
|
1
|
+
{"version":3,"file":"init.js","sourceRoot":"","sources":["../../src/cli/init.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,mBAAmB,CAAC;AAC5D,OAAO,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAC1D,OAAO,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAC5B,OAAO,EAAE,KAAK,EAAE,MAAM,eAAe,CAAC;AACtC,OAAO,EAAE,YAAY,EAAuB,MAAM,uBAAuB,CAAC;AAC1E,OAAO,EACL,iBAAiB,EACjB,qBAAqB,EACrB,YAAY,GACb,MAAM,4BAA4B,CAAC;AACpC,OAAO,EACL,UAAU,EAEV,oBAAoB,GACrB,MAAM,kBAAkB,CAAC;AAE1B,iFAAiF;AAEjF,MAAM,UAAU,GAAG,UAAU,CAAC;AAE9B,SAAS,QAAQ,CACf,OAAqC;IAErC,OAAO;QACL,GAAG,OAAO;QACV,EAAE,IAAI,EAAE,QAAQ,EAAE,KAAK,EAAE,UAA+B,EAAE;KAC3D,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,KAAc;IAC5B,OAAO,KAAK,KAAK,UAAU,CAAC;AAC9B,CAAC;AAWD,iFAAiF;AAEjF,SAAS,mBAAmB,CAC1B,KAAkB,EAClB,QAAwB;IAExB,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAChC,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,WAAW,CAAC,CAAC;IAC5E,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,IAAI,aAAa,IAAI,aAAa,CAAC,GAAG,KAAK,QAAQ;QACjD,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC;IAClC,IAAI,UAAU,IAAI,UAAU,CAAC,GAAG,KAAK,QAAQ;QAAE,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC5E,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,KAAK,UAAU,oBAAoB,CACjC,WAAmB;IAEnB,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7D,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,gBAAgB,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IACnE,IAAI,UAAU,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IAC9D,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,mBAAmB,CAC1B,EAAmC,EACnC,QAAkB;IAElB,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACrD,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACrD,KAAK,KAAK;YACR,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QACpD;YACE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;IAC1D,CAAC;AACH,CAAC;AAED,SAAS,yBAAyB,CAChC,EAAmC,EACnC,QAAkB;IAElB,QAAQ,EAAE,EAAE,CAAC;QACX,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,QAAQ,EAAE,KAAK,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC/D,KAAK,MAAM;YACT,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC3D,KAAK,KAAK;YACR,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;QAC1D;YACE,OAAO,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,SAAS,EAAE,IAAI,EAAE,GAAG,QAAQ,CAAC,EAAE,CAAC;IAChE,CAAC;AACH,CAAC;AAED,KAAK,UAAU,mBAAmB,CAAC,UAAkB;IACnD,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,GAAG,UAAU,QAAQ,CAAC,EAAE,CAAC;IACpD,kCAAkC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;QACtC,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,aAAa,CAAC,CAAC;QAC/C,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC;IAChC,CAAC;IAAC,MAAM,CAAC;QACP,sBAAsB;IACxB,CAAC;IACD,8DAA8D;IAC9D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,IAAI,CAAC;YACH,MAAM,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACnC,CAAC;QAAC,MAAM,CAAC;YACP,yBAAyB;QAC3B,CAAC;IACH,CAAC;IACD,gCAAgC;IAChC,IAAI,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC3B,MAAM,MAAM,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACnC,CAAC;AACH,CAAC;AAED,SAAS,UAAU,CAAC,GAAW,EAAE,IAAc;IAC7C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,IAAI,GAAG,KAAK,CAAC,GAAG,EAAE,IAAI,EAAE;YAC5B,KAAK,EAAE,SAAS;YAChB,KAAK,EAAE,OAAO,CAAC,QAAQ,KAAK,OAAO;SACpC,CAAC,CAAC;QACH,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CACxB,IAAI,KAAK,CAAC;YACR,CAAC,CAAC,OAAO,EAAE;YACX,CAAC,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,iCAAiC,IAAI,EAAE,CAAC,CAAC,CAC/D,CAAC;QACF,IAAI,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC;AAED,iFAAiF;AAEjF,MAAM,mBAAmB,GAA2B;IAClD,uBAAuB,EAAE,iBAAiB;IAC1C,aAAa,EAAE,OAAO;IACtB,iBAAiB,EAAE,WAAW;IAC9B,gBAAgB,EAAE,UAAU;CAC7B,CAAC;AAEF,SAAS,kBAAkB,CACzB,KAAkB,EAClB,QAAwB;IAExB,MAAM,eAAe,GACnB,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC;QACrB,CAAC,CAAC,KAAK,CAAC,MAAM;QACd,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,YAAY,CAAE,CAAC,CAAC;IAEzD,MAAM,QAAQ,GAAG,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;QAC3C,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,CAAC;QACvC,QAAQ,EAAE,mBAAmB,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU;KAC5D,CAAC,CAAC,CAAC;IAEJ,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,QAAQ,CAC/B,CAAC;IACH,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,KAAK,CAAC,WAAW,CAAE,CAAC;IAE7E,2EAA2E;IAC3E,yEAAyE;IACzE,+CAA+C;IAC/C,MAAM,mBAAmB,GACvB,UAAU,CAAC,GAAG,KAAK,SAAS;QAC1B,CAAC,CAAC,EAAE,GAAG,UAAU,CAAC,aAAa,EAAE,GAAG,EAAE,GAAG,YAAY,EAAE,UAAU,EAAE;QACnE,CAAC,CAAC,UAAU,CAAC,aAAa,CAAC;IAE/B,MAAM,MAAM,GAAG;QACb,OAAO,EACL,wEAAwE;QAC1E,QAAQ;QACR,QAAQ,EAAE;YACR,OAAO,EAAE,aAAa,CAAC,OAAO;YAC9B,MAAM,EAAE,aAAa,CAAC,aAAa;SACpC;QACD,WAAW,EAAE;YACX,OAAO,EAAE,UAAU,CAAC,OAAO;YAC3B,MAAM,EAAE,mBAAmB;SAC5B;QACD,OAAO,EAAE;YACP,UAAU,EAAE,iBAAiB,EAAE;YAC/B,cAAc,EAAE,qBAAqB,EAAE;SACxC;KACF,CAAC;IAEF,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;AAChD,CAAC;AAED,iFAAiF;AAEjF,KAAK,UAAU,YAAY,CACzB,OAAe,EACf,IAA4B;IAE5B,IAAI,QAAQ,GAAG,EAAE,CAAC;IAClB,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;QACxB,QAAQ,GAAG,MAAM,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,MAAM,KAAK,GAAa,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC7C,CAAC,CAAC,CAAC,QAAQ,CAAC;QACZ,CAAC,CAAC,QAAQ;YACR,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;YAChB,CAAC,CAAC,EAAE,CAAC;IAET,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC;QAChD,MAAM,cAAc,GAAG,QAAQ;aAC5B,KAAK,CAAC,IAAI,CAAC;aACX,IAAI,CACH,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,GAAG,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,GAAG,GAAG,IAAI,CAAC,CACpE,CAAC;QAEJ,IAAI,cAAc,EAAE,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;aAAM,CAAC;YACN,KAAK,CAAC,IAAI,CAAC,GAAG,GAAG,IAAI,KAAK,EAAE,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,SAAS,CAAC,OAAO,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,SAAS,EAAE,GAAG,IAAI,EAAE,OAAO,CAAC,CAAC;IACzE,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,iFAAiF;AAEjF,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAC,CAAC;IAE3C,MAAM,iBAAiB,GAAG,sBAAsB,CAAC;IACjD,IAAI,UAAU,CAAC,iBAAiB,CAAC,EAAE,CAAC;QAClC,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,eAAe,iBAAiB,6BAA6B;YACtE,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,IAAI,EAAE;gBACjD,EAAE,IAAI,EAAE,YAAY,EAAE,KAAK,EAAE,KAAK,EAAE;aACrC;SACF,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;IACH,CAAC;IAED,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;IAC1B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;IAClD,MAAM,cAAc,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;IAEvD,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,GAAG,CAAC,CAAC;IAEzC,MAAM,KAAK,GAAyB,EAAE,CAAC;IACvC,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,OAAO,IAAI,GAAG,CAAC,EAAE,CAAC;QAChB,QAAQ,IAAI,EAAE,CAAC;YACb,kCAAkC;YAClC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC9B,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;wBAC/B,OAAO,EAAE,8CAA8C;wBACvD,OAAO,EAAE;4BACP,GAAG,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gCAC5B,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE;gCAC1D,KAAK,EAAE,CAAC,CAAC,IAAI;gCACb,OAAO,EAAE,IAAI;6BACd,CAAC,CAAC;4BACH,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,UAAU,EAAE;yBACpD;qBACF,CAAC,CAAC;oBACH,IAAI,SAAS,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;wBACnC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAC5B,OAAO;oBACT,CAAC;oBACD,KAAK,CAAC,MAAM,GAAG,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CACzC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAC3B,CAAC;gBACJ,CAAC;qBAAM,CAAC;oBACN,OAAO,CAAC,GAAG,CACT,2DAA2D,CAC5D,CAAC;oBACF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;wBAC5B,OAAO,EAAE,wCAAwC;wBACjD,OAAO,EAAE;4BACP,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gCACxB,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,UAAU,GAAG;gCACnC,KAAK,EAAE,CAAC,CAAC,IAAI;6BACd,CAAC,CAAC;4BACH,EAAE,IAAI,EAAE,sBAAsB,EAAE,KAAK,EAAE,UAAU,EAAE;yBACpD;qBACF,CAAC,CAAC;oBACH,IAAI,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;wBAChC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;wBAC5B,OAAO;oBACT,CAAC;oBACD,KAAK,CAAC,MAAM,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;gBACnE,CAAC;gBACD,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,mCAAmC;YACnC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;oBAC1B,OAAO,EAAE,2BAA2B;oBACpC,OAAO,EAAE,QAAQ,CACf,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CACjE;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,QAAQ,GAAG,MAAM,CAAC;gBACxB,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,uCAAuC;YACvC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC;oBAC1B,OAAO,EAAE,qBAAqB;oBAC9B,OAAO,EAAE,QAAQ,CACf,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAC9D;iBACF,CAAC,CAAC;gBACH,IAAI,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC;oBACnB,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,WAAW,GAAG,MAAM,CAAC;gBAC3B,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;YAED,4BAA4B;YAC5B,KAAK,CAAC,CAAC,CAAC,CAAC;gBACP,MAAM,aAAa,GAAG,sBAAsB,CAAC;gBAC7C,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC;oBAC7B,OAAO,EAAE,2DAA2D;oBACpE,OAAO,EAAE,aAAa;iBACvB,CAAC,CAAC;gBACH,IAAI,UAAU,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;oBAC7B,IAAI,EAAE,CAAC;oBACP,MAAM;gBACR,CAAC;gBACD,KAAK,CAAC,UAAU,GAAG,UAAU,CAAC,IAAI,EAAE,CAAC;gBACrC,IAAI,EAAE,CAAC;gBACP,MAAM;YACR,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,UAAU,GAAG,KAAoB,CAAC;IAExC,oDAAoD;IACpD,IACE,UAAU,CAAC,UAAU,KAAK,iBAAiB;QAC3C,UAAU,CAAC,UAAU,CAAC,UAAU,CAAC,EACjC,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,MAAM,CAAC;YAC7B,OAAO,EAAE,GAAG,UAAU,CAAC,UAAU,6CAA6C;YAC9E,OAAO,EAAE;gBACP,EAAE,IAAI,EAAE,0BAA0B,EAAE,KAAK,EAAE,IAAI,EAAE;gBACjD,EAAE,IAAI,EAAE,iCAAiC,EAAE,KAAK,EAAE,KAAK,EAAE;aAC1D;SACF,CAAC,CAAC;QACH,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;IACH,CAAC;IAED,qBAAqB;IACrB,MAAM,mBAAmB,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,aAAa,GAAG,kBAAkB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/D,MAAM,SAAS,CAAC,UAAU,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,CAAC,CAAC;IAC/D,OAAO,CAAC,GAAG,CAAC,aAAa,UAAU,CAAC,UAAU,EAAE,CAAC,CAAC;IAElD,qBAAqB;IACrB,MAAM,aAAa,GAAG,QAAQ,CAAC,SAAS,CAAC,IAAI,CAC3C,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,UAAU,CAAC,QAAQ,CACrC,CAAC;IACF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CACrC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,KAAK,UAAU,CAAC,WAAW,CACxC,CAAC;IACF,MAAM,YAAY,GAAG;QACnB,GAAG,CAAC,aAAa,EAAE,OAAO,IAAI,EAAE,CAAC;QACjC,GAAG,CAAC,UAAU,EAAE,OAAO,IAAI,EAAE,CAAC;KAC/B,CAAC;IAEF,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC5B,OAAO,CAAC,GAAG,CAAC,sDAAsD,CAAC,CAAC;QACpE,MAAM,SAAS,GAA2B,EAAE,CAAC;QAE7C,KAAK,MAAM,OAAO,IAAI,YAAY,EAAE,CAAC;YACnC,MAAM,KAAK,GAAG,MAAM,KAAK,CAAC;gBACxB,OAAO,EAAE,mBAAmB,OAAO,yBAAyB;gBAC5D,OAAO,EAAE,EAAE;aACZ,CAAC,CAAC;YACH,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;gBACjB,SAAS,CAAC,OAAO,CAAC,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;YACpC,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtC,MAAM,OAAO,GAAG,QAAQ,CAAC;YACzB,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;YACpE,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,cAAc,OAAO,KAAK,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;gBACvB,OAAO,CAAC,GAAG,CAAC,8BAA8B,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YAClE,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,6DAA6D,CAC9D,CAAC;QACJ,CAAC;IACH,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,GAAG,CAAC,6CAA6C,CAAC,CAAC;IAC7D,CAAC;IAED,qBAAqB;IACrB,MAAM,IAAI,GAAG,mBAAmB,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACvD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACpB,MAAM,EAAE,GAAG,MAAM,oBAAoB,CAAC,GAAG,CAAC,CAAC;QAC3C,MAAM,QAAQ,GAAG,mBAAmB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QAC/C,MAAM,SAAS,GAAG,yBAAyB,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;QACtD,MAAM,aAAa,GAAG,MAAM,MAAM,CAAC;YACjC,OAAO,EAAE,WAAW,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,GAAG;YAClD,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,8CAA8C;oBACpD,KAAK,EAAE,OAAgB;iBACxB;gBACD,EAAE,IAAI,EAAE,uBAAuB,EAAE,KAAK,EAAE,QAAiB,EAAE;gBAC3D,EAAE,IAAI,EAAE,2BAA2B,EAAE,KAAK,EAAE,QAAiB,EAAE;aAChE;SACF,CAAC,CAAC;QACH,IAAI,aAAa,KAAK,OAAO,IAAI,aAAa,KAAK,QAAQ,EAAE,CAAC;YAC5D,MAAM,EAAE,GAAG,EAAE,IAAI,EAAE,GAAG,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC;YACxE,IAAI,CAAC;gBACH,MAAM,UAAU,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,oCAAoC,CAAC,CAAC;YACpD,CAAC;YAAC,MAAM,CAAC;gBACP,OAAO,CAAC,GAAG,CACT,sCAAsC,GAAG,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAC9D,CAAC;YACJ,CAAC;QACH,CAAC;aAAM,CAAC;YACN,OAAO,CAAC,GAAG,CACT,sBAAsB,QAAQ,CAAC,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAChE,CAAC;QACJ,CAAC;IACH,CAAC;IAED,mBAAmB;IACnB,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IAC7B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;QACxC,OAAO,CAAC,GAAG,CACT,KAAK,QAAQ,EAAE,uDAAuD,CACvE,CAAC;QACF,OAAO,CAAC,GAAG,CACT,gFAAgF,CACjF,CAAC;IACJ,CAAC;IACD,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,EAAE,+CAA+C,CAAC,CAAC;IAC5E,OAAO,CAAC,GAAG,CAAC,KAAK,QAAQ,sCAAsC,CAAC,CAAC;AACnE,CAAC"}
|
package/dist/core/embedder.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { EmbeddingProvider, EmbeddedChunk, EmbeddingsMeta, Chunk } from "../interfaces/index.js";
|
|
2
2
|
import type { Logger } from "../interfaces/logger.js";
|
|
3
3
|
import { RetryOptions } from "./utils.js";
|
|
4
|
+
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
4
5
|
export declare class EmbedderProcessor {
|
|
5
6
|
private provider;
|
|
6
7
|
private rateLimitMs;
|
|
@@ -10,6 +11,7 @@ export declare class EmbedderProcessor {
|
|
|
10
11
|
private concurrency;
|
|
11
12
|
private vectorStoreName?;
|
|
12
13
|
private saveIntervalMs;
|
|
14
|
+
private minIngestionBatchSize;
|
|
13
15
|
private logger;
|
|
14
16
|
onProgress?: (completed: number, total: number) => void;
|
|
15
17
|
constructor(provider: EmbeddingProvider, options?: {
|
|
@@ -20,17 +22,18 @@ export declare class EmbedderProcessor {
|
|
|
20
22
|
concurrency?: number;
|
|
21
23
|
vectorStoreName?: string;
|
|
22
24
|
saveIntervalMs?: number;
|
|
25
|
+
minIngestionBatchSize?: number;
|
|
23
26
|
logger?: Logger;
|
|
24
27
|
onProgress?: (completed: number, total: number) => void;
|
|
25
28
|
});
|
|
26
29
|
embedChunk(chunk: Chunk): Promise<EmbeddedChunk>;
|
|
27
30
|
embedBatch(chunks: Chunk[]): Promise<EmbeddedChunk[]>;
|
|
28
|
-
getChunksToEmbed(
|
|
31
|
+
getChunksToEmbed(db: EmbeddingsDb, chunks: Chunk[], force?: boolean): Promise<{
|
|
29
32
|
chunksToEmbed: Chunk[];
|
|
30
|
-
|
|
33
|
+
meta: EmbeddingsMeta | null;
|
|
31
34
|
isNewEmbeddingSpace: boolean;
|
|
32
35
|
}>;
|
|
33
|
-
saveEmbeddings(newEmbeddings: EmbeddedChunk[],
|
|
34
|
-
run(chunksFile: string, force?: boolean): Promise<EmbeddedChunk[]>;
|
|
36
|
+
saveEmbeddings(db: EmbeddingsDb, newEmbeddings: EmbeddedChunk[], meta: EmbeddingsMeta): Promise<void>;
|
|
37
|
+
run(db: EmbeddingsDb, chunksFile: string, force?: boolean, onIntermediateBatch?: () => Promise<void>): Promise<EmbeddedChunk[]>;
|
|
35
38
|
}
|
|
36
39
|
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,KAAK,EACN,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAKtD,OAAO,EAML,YAAY,EACb,MAAM,YAAY,CAAC;
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,iBAAiB,EACjB,aAAa,EACb,cAAc,EACd,KAAK,EACN,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAKtD,OAAO,EAML,YAAY,EACb,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAOlD,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,aAAa,CAAS;IAC9B,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,eAAe,CAAC,CAAS;IACjC,OAAO,CAAC,cAAc,CAAS;IAC/B,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,MAAM,CAAS;IACvB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;gBAGtD,QAAQ,EAAE,iBAAiB,EAC3B,OAAO,GAAE;QACP,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,CAAC,EAAE,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;KACpD;IAeF,UAAU,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,CAAC,aAAa,CAAC;IAchD,UAAU,CAAC,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,aAAa,EAAE,CAAC;IAoDrD,gBAAgB,CACpB,EAAE,EAAE,YAAY,EAChB,MAAM,EAAE,KAAK,EAAE,EACf,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,aAAa,EAAE,KAAK,EAAE,CAAC;QACvB,IAAI,EAAE,cAAc,GAAG,IAAI,CAAC;QAC5B,mBAAmB,EAAE,OAAO,CAAC;KAC9B,CAAC;IAwGI,cAAc,CAClB,EAAE,EAAE,YAAY,EAChB,aAAa,EAAE,aAAa,EAAE,EAC9B,IAAI,EAAE,cAAc,GACnB,OAAO,CAAC,IAAI,CAAC;IAMV,GAAG,CACP,EAAE,EAAE,YAAY,EAChB,UAAU,EAAE,MAAM,EAClB,KAAK,GAAE,OAAe,EACtB,mBAAmB,CAAC,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GACxC,OAAO,CAAC,aAAa,EAAE,CAAC;CAoF5B"}
|
package/dist/core/embedder.js
CHANGED
|
@@ -3,7 +3,6 @@ import { readFile } from "fs/promises";
|
|
|
3
3
|
import { createHash } from "crypto";
|
|
4
4
|
import { EmbedError } from "./errors.js";
|
|
5
5
|
import { sleep, withRetry, withConcurrency, batchBySize, defaultIsRetryable, } from "./utils.js";
|
|
6
|
-
import { readEmbeddingsFile, writeEmbeddingsFile } from "./embeddings-io.js";
|
|
7
6
|
function chunkContentHash(chunk) {
|
|
8
7
|
if (chunk.contentHash)
|
|
9
8
|
return chunk.contentHash;
|
|
@@ -18,6 +17,7 @@ export class EmbedderProcessor {
|
|
|
18
17
|
concurrency;
|
|
19
18
|
vectorStoreName;
|
|
20
19
|
saveIntervalMs;
|
|
20
|
+
minIngestionBatchSize;
|
|
21
21
|
logger;
|
|
22
22
|
onProgress;
|
|
23
23
|
constructor(provider, options = {}) {
|
|
@@ -29,6 +29,7 @@ export class EmbedderProcessor {
|
|
|
29
29
|
this.concurrency = options.concurrency ?? 1;
|
|
30
30
|
this.vectorStoreName = options.vectorStoreName;
|
|
31
31
|
this.saveIntervalMs = options.saveIntervalMs ?? 30_000;
|
|
32
|
+
this.minIngestionBatchSize = options.minIngestionBatchSize ?? Infinity;
|
|
32
33
|
this.onProgress = options.onProgress;
|
|
33
34
|
this.logger = (options.logger ?? new NullLogger()).withTag("embedder");
|
|
34
35
|
}
|
|
@@ -73,63 +74,36 @@ export class EmbedderProcessor {
|
|
|
73
74
|
});
|
|
74
75
|
return withConcurrency(tasks, this.concurrency);
|
|
75
76
|
}
|
|
76
|
-
async getChunksToEmbed(
|
|
77
|
-
|
|
78
|
-
try {
|
|
79
|
-
const content = await readFile(chunksFile, "utf-8");
|
|
80
|
-
const parsed = JSON.parse(content);
|
|
81
|
-
if (!Array.isArray(parsed)) {
|
|
82
|
-
throw new Error("chunks file does not contain a JSON array");
|
|
83
|
-
}
|
|
84
|
-
chunks = parsed;
|
|
85
|
-
}
|
|
86
|
-
catch (err) {
|
|
87
|
-
throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
88
|
-
suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
|
|
89
|
-
cause: err,
|
|
90
|
-
});
|
|
91
|
-
}
|
|
92
|
-
this.logger.info(`📖 Loaded ${chunks.length} chunks from ${chunksFile}`);
|
|
77
|
+
async getChunksToEmbed(db, chunks, force = false) {
|
|
78
|
+
this.logger.info(`📖 Loaded ${chunks.length} chunks`);
|
|
93
79
|
this.logger.debug(`Provider: ${this.provider.name}, model: ${this.provider.model ?? "(none)"}, dims: ${this.provider.dimensions}`);
|
|
80
|
+
const existingMeta = db.getMeta();
|
|
94
81
|
if (force) {
|
|
95
|
-
this.logger.verbose("Force mode: embedding all chunks");
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
existingMeta: null,
|
|
99
|
-
isNewEmbeddingSpace: true,
|
|
100
|
-
};
|
|
82
|
+
this.logger.verbose("Force mode: clearing db and embedding all chunks");
|
|
83
|
+
db.clearAll();
|
|
84
|
+
return { chunksToEmbed: chunks, meta: null, isNewEmbeddingSpace: true };
|
|
101
85
|
}
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
// are invalid for the new model's vector space.
|
|
106
|
-
// Discriminators: model + dimensions. providerName is informational only (the same
|
|
107
|
-
// model served via OpenAI, Azure, or GitHub Models produces identical vectors).
|
|
108
|
-
const isNewEmbeddingSpace = false;
|
|
109
|
-
if (meta && existingEmbeddings.length > 0) {
|
|
110
|
-
const modelChanged = meta.model !== undefined &&
|
|
86
|
+
// Check if the embedding model or dimensions changed
|
|
87
|
+
if (existingMeta) {
|
|
88
|
+
const modelChanged = existingMeta.model !== undefined &&
|
|
111
89
|
this.provider.model !== undefined &&
|
|
112
|
-
|
|
113
|
-
const dimensionsChanged =
|
|
90
|
+
existingMeta.model !== this.provider.model;
|
|
91
|
+
const dimensionsChanged = existingMeta.providerDimensions !== this.provider.dimensions;
|
|
114
92
|
if (modelChanged || dimensionsChanged) {
|
|
115
|
-
const prevModel =
|
|
93
|
+
const prevModel = existingMeta.model ?? "(unknown)";
|
|
116
94
|
const curModel = this.provider.model ?? "(unknown)";
|
|
117
|
-
const prevDims =
|
|
95
|
+
const prevDims = existingMeta.providerDimensions;
|
|
118
96
|
const curDims = this.provider.dimensions;
|
|
119
97
|
this.logger.warn(`⚠️ Embedding model changed!`);
|
|
120
98
|
this.logger.warn(` Previous: ${prevModel} (${prevDims}d)`);
|
|
121
99
|
this.logger.warn(` Current: ${curModel} (${curDims}d)`);
|
|
122
100
|
this.logger.warn(` Discarding existing embeddings — all chunks will be re-embedded.`);
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
existingMeta: null,
|
|
126
|
-
isNewEmbeddingSpace: true,
|
|
127
|
-
};
|
|
101
|
+
db.clearAll();
|
|
102
|
+
return { chunksToEmbed: chunks, meta: null, isNewEmbeddingSpace: true };
|
|
128
103
|
}
|
|
129
104
|
}
|
|
130
|
-
// Two-level index:
|
|
131
|
-
|
|
132
|
-
// contentHash (global) — for per-chunk fallback
|
|
105
|
+
// Two-level index: (sourceFile::commitHash) → Set<contentHash> and global contentHash set
|
|
106
|
+
const existingEmbeddings = db.getAll();
|
|
133
107
|
const embeddedFileVersions = new Map();
|
|
134
108
|
const embeddedByContentHash = new Set();
|
|
135
109
|
for (const emb of existingEmbeddings) {
|
|
@@ -146,7 +120,6 @@ export class EmbedderProcessor {
|
|
|
146
120
|
}
|
|
147
121
|
this.logger.verbose(`📊 Existing: ${existingEmbeddings.length} chunks across ${embeddedFileVersions.size} version(s)`);
|
|
148
122
|
this.logger.debug(`Vector store: ${this.vectorStoreName ?? "(none)"}`);
|
|
149
|
-
// Group incoming chunks by (sourceFile, commitHash)
|
|
150
123
|
const chunksByFileVersion = new Map();
|
|
151
124
|
for (const chunk of chunks) {
|
|
152
125
|
const key = `${chunk.sourceFile}::${chunk.commitHash}`;
|
|
@@ -163,7 +136,6 @@ export class EmbedderProcessor {
|
|
|
163
136
|
let skippedChunks = 0;
|
|
164
137
|
for (const [key, fileChunks] of chunksByFileVersion) {
|
|
165
138
|
const fileVersionHashes = embeddedFileVersions.get(key);
|
|
166
|
-
// Fast path: every chunk for this (sourceFile, commitHash) is already embedded
|
|
167
139
|
if (fileVersionHashes &&
|
|
168
140
|
fileVersionHashes.size === fileChunks.length &&
|
|
169
141
|
fileChunks.every((c) => fileVersionHashes.has(chunkContentHash(c)))) {
|
|
@@ -171,8 +143,6 @@ export class EmbedderProcessor {
|
|
|
171
143
|
skippedChunks += fileChunks.length;
|
|
172
144
|
continue;
|
|
173
145
|
}
|
|
174
|
-
// Per-chunk fallback: check globally by contentHash (handles partial embeddings
|
|
175
|
-
// from interrupted runs, or content-identical chunks at a different commit)
|
|
176
146
|
for (const chunk of fileChunks) {
|
|
177
147
|
if (!embeddedByContentHash.has(chunkContentHash(chunk))) {
|
|
178
148
|
chunksToEmbed.push(chunk);
|
|
@@ -182,59 +152,37 @@ export class EmbedderProcessor {
|
|
|
182
152
|
if (skippedFiles > 0) {
|
|
183
153
|
this.logger.verbose(`⏭️ Skipped ${skippedFiles} unchanged version(s) (${skippedChunks} chunks)`);
|
|
184
154
|
}
|
|
185
|
-
return { chunksToEmbed,
|
|
155
|
+
return { chunksToEmbed, meta: existingMeta, isNewEmbeddingSpace: false };
|
|
186
156
|
}
|
|
187
|
-
async saveEmbeddings(
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
157
|
+
async saveEmbeddings(db, newEmbeddings, meta) {
|
|
158
|
+
db.setMeta(meta);
|
|
159
|
+
db.insert(newEmbeddings);
|
|
160
|
+
this.logger.verbose(`Saved ${newEmbeddings.length} new embeddings to db`);
|
|
161
|
+
}
|
|
162
|
+
async run(db, chunksFile, force = false, onIntermediateBatch) {
|
|
163
|
+
this.logger.info("🔢 Starting incremental embedding generation...");
|
|
164
|
+
let chunks;
|
|
165
|
+
try {
|
|
166
|
+
const content = await readFile(chunksFile, "utf-8");
|
|
167
|
+
const parsed = JSON.parse(content);
|
|
168
|
+
if (!Array.isArray(parsed)) {
|
|
169
|
+
throw new Error("chunks file does not contain a JSON array");
|
|
170
|
+
}
|
|
171
|
+
chunks = parsed;
|
|
194
172
|
}
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
let createdAt = now;
|
|
200
|
-
if (!discardExisting) {
|
|
201
|
-
const { meta, chunks: existing } = await readEmbeddingsFile(embeddingsFile);
|
|
202
|
-
if (meta)
|
|
203
|
-
createdAt = meta.createdAt;
|
|
204
|
-
keepChunks = existing.filter((e) => {
|
|
205
|
-
const hash = e.contentHash || chunkContentHash(e);
|
|
206
|
-
return !newByHash.has(hash);
|
|
173
|
+
catch (err) {
|
|
174
|
+
throw new EmbedError(`Failed to load chunks from ${chunksFile}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
175
|
+
suggestion: "Run the pipeline without --skip-upload to regenerate chunks first.",
|
|
176
|
+
cause: err,
|
|
207
177
|
});
|
|
208
178
|
}
|
|
209
|
-
const
|
|
210
|
-
const meta = {
|
|
211
|
-
schemaVersion: 1,
|
|
212
|
-
providerName: this.provider.name,
|
|
213
|
-
providerDimensions: this.provider.dimensions,
|
|
214
|
-
...(this.provider.model !== undefined
|
|
215
|
-
? { model: this.provider.model }
|
|
216
|
-
: {}),
|
|
217
|
-
...(this.vectorStoreName !== undefined
|
|
218
|
-
? { vectorStoreName: this.vectorStoreName }
|
|
219
|
-
: {}),
|
|
220
|
-
// Preserve createdAt from the existing meta when available and not overwriting
|
|
221
|
-
createdAt: existingMeta && !discardExisting ? existingMeta.createdAt : createdAt,
|
|
222
|
-
updatedAt: now,
|
|
223
|
-
};
|
|
224
|
-
await writeEmbeddingsFile(embeddingsFile, meta, final);
|
|
225
|
-
this.logger.verbose(`Saved ${final.length} embeddings to ${embeddingsFile}`);
|
|
226
|
-
this.logger.verbose(` New: ${newEmbeddings.length}, Existing: ${keepChunks.length}`);
|
|
227
|
-
}
|
|
228
|
-
async run(chunksFile, force = false) {
|
|
229
|
-
this.logger.info("🔢 Starting incremental embedding generation...");
|
|
230
|
-
const { chunksToEmbed, existingMeta, isNewEmbeddingSpace } = await this.getChunksToEmbed(chunksFile, force);
|
|
231
|
-
// Treat a model-space change the same as --force for the save step
|
|
232
|
-
const effectiveForce = force || isNewEmbeddingSpace;
|
|
179
|
+
const { chunksToEmbed, meta: existingMeta } = await this.getChunksToEmbed(db, chunks, force);
|
|
233
180
|
if (chunksToEmbed.length === 0) {
|
|
234
181
|
this.logger.info("✨ No chunks need embedding.");
|
|
235
182
|
return [];
|
|
236
183
|
}
|
|
237
184
|
this.logger.info(`📝 Need to embed ${chunksToEmbed.length} chunks`);
|
|
185
|
+
const now = Math.floor(Date.now() / 1000);
|
|
238
186
|
const batches = batchBySize(chunksToEmbed, this.batchSize, (c) => c.content.length, this.maxBatchChars);
|
|
239
187
|
const newEmbeddings = [];
|
|
240
188
|
let lastSaveTime = Date.now();
|
|
@@ -247,8 +195,25 @@ export class EmbedderProcessor {
|
|
|
247
195
|
const isLastBatch = i === batches.length - 1;
|
|
248
196
|
const elapsed = Date.now() - lastSaveTime;
|
|
249
197
|
if (isLastBatch || elapsed >= this.saveIntervalMs) {
|
|
250
|
-
|
|
198
|
+
const meta = {
|
|
199
|
+
schemaVersion: 1,
|
|
200
|
+
providerName: this.provider.name,
|
|
201
|
+
providerDimensions: this.provider.dimensions,
|
|
202
|
+
...(this.provider.model !== undefined
|
|
203
|
+
? { model: this.provider.model }
|
|
204
|
+
: {}),
|
|
205
|
+
...(this.vectorStoreName !== undefined
|
|
206
|
+
? { vectorStoreName: this.vectorStoreName }
|
|
207
|
+
: {}),
|
|
208
|
+
createdAt: existingMeta?.createdAt ?? now,
|
|
209
|
+
updatedAt: now,
|
|
210
|
+
};
|
|
211
|
+
await this.saveEmbeddings(db, embedded, meta);
|
|
251
212
|
lastSaveTime = Date.now();
|
|
213
|
+
if (onIntermediateBatch &&
|
|
214
|
+
db.pendingCount() >= this.minIngestionBatchSize) {
|
|
215
|
+
await onIntermediateBatch();
|
|
216
|
+
}
|
|
252
217
|
}
|
|
253
218
|
}
|
|
254
219
|
return newEmbeddings;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AACpB,OAAO,EAAE,kBAAkB,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAC;AAE7E,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,cAAc,CAAS;IACvB,MAAM,CAAS;IACvB,UAAU,CAA8C;IAExD,YACE,QAA2B,EAC3B,UAUI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC;QACvD,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,UAAkB,EAClB,QAAiB,KAAK;QAMtB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,gBAAgB,UAAU,EAAE,CAAC,CAAC;QACzE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC;YACxD,OAAO;gBACL,aAAa,EAAE,MAAM;gBACrB,YAAY,EAAE,IAAI;gBAClB,mBAAmB,EAAE,IAAI;aAC1B,CAAC;QACJ,CAAC;QAED,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,kBAAkB,EAAE,GACxC,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;QAE3C,oFAAoF;QACpF,gDAAgD;QAChD,mFAAmF;QACnF,gFAAgF;QAChF,MAAM,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,IAAI,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC1C,MAAM,YAAY,GAChB,IAAI,CAAC,KAAK,KAAK,SAAS;gBACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YACrC,MAAM,iBAAiB,GACrB,IAAI,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAEvD,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,IAAI,WAAW,CAAC;gBAC5C,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,kBAAkB,CAAC;gBACzC,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,OAAO;oBACL,aAAa,EAAE,MAAM;oBACrB,YAAY,EAAE,IAAI;oBAClB,mBAAmB,EAAE,IAAI;iBAC1B,CAAC;YACJ,CAAC;QACH,CAAC;QAED,mBAAmB;QACnB,4EAA4E;QAC5E,2EAA2E;QAC3E,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,oDAAoD;QACpD,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,+EAA+E;YAC/E,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,gFAAgF;YAChF,4EAA4E;YAC5E,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,YAAY,EAAE,IAAI,EAAE,mBAAmB,EAAE,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,aAA8B,EAC9B,UAAkB,EAClB,YAAmC,EACnC,YAAqB,EACrB,eAAwB;QAExB,MAAM,cAAc,GAAG,UAAU,CAAC,OAAO,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC;QAClE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAE1C,MAAM,SAAS,GAAG,IAAI,GAAG,EAAyB,CAAC;QACnD,KAAK,MAAM,GAAG,IAAI,aAAa,EAAE,CAAC;YAChC,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;QAC3B,CAAC;QAED,uFAAuF;QACvF,qEAAqE;QACrE,MAAM,eAAe,GAAG,YAAY,IAAI,eAAe,CAAC;QAExD,IAAI,UAAU,GAAoB,EAAE,CAAC;QACrC,IAAI,SAAS,GAAG,GAAG,CAAC;QACpB,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,GAC9B,MAAM,kBAAkB,CAAC,cAAc,CAAC,CAAC;YAC3C,IAAI,IAAI;gBAAE,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;YACrC,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;gBACjC,MAAM,IAAI,GAAG,CAAC,CAAC,WAAW,IAAI,gBAAgB,CAAC,CAAC,CAAC,CAAC;gBAClD,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC9B,CAAC,CAAC,CAAC;QACL,CAAC;QAED,MAAM,KAAK,GAAG,CAAC,GAAG,UAAU,EAAE,GAAG,aAAa,CAAC,CAAC;QAEhD,MAAM,IAAI,GAAmB;YAC3B,aAAa,EAAE,CAAC;YAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;YAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;YAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAChC,CAAC,CAAC,EAAE,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;gBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;gBAC3C,CAAC,CAAC,EAAE,CAAC;YACP,+EAA+E;YAC/E,SAAS,EACP,YAAY,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;YACvE,SAAS,EAAE,GAAG;SACf,CAAC;QAEF,MAAM,mBAAmB,CAAC,cAAc,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,SAAS,KAAK,CAAC,MAAM,kBAAkB,cAAc,EAAE,CACxD,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,aAAa,CAAC,MAAM,eAAe,UAAU,CAAC,MAAM,EAAE,CAClE,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,GAAG,CACP,UAAkB,EAClB,QAAiB,KAAK;QAEtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,MAAM,EAAE,aAAa,EAAE,YAAY,EAAE,mBAAmB,EAAE,GACxD,MAAM,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QAEjD,mEAAmE;QACnE,MAAM,cAAc,GAAG,KAAK,IAAI,mBAAmB,CAAC;QAEpD,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAC1C,IAAI,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC1C,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClD,MAAM,IAAI,CAAC,cAAc,CACvB,QAAQ,EACR,UAAU,EACV,YAAY,EACZ,CAAC,KAAK,CAAC,EACP,CAAC,KAAK,CAAC,IAAI,cAAc,CAC1B,CAAC;gBACF,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/core/embedder.ts"],"names":[],"mappings":"AAOA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AACvC,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EACL,KAAK,EACL,SAAS,EACT,eAAe,EACf,WAAW,EACX,kBAAkB,GAEnB,MAAM,YAAY,CAAC;AAGpB,SAAS,gBAAgB,CAAC,KAAY;IACpC,IAAI,KAAK,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC,WAAW,CAAC;IAChD,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC/E,CAAC;AAED,MAAM,OAAO,iBAAiB;IACpB,QAAQ,CAAoB;IAC5B,WAAW,CAAS;IACpB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,YAAY,CAAe;IAC3B,WAAW,CAAS;IACpB,eAAe,CAAU;IACzB,cAAc,CAAS;IACvB,qBAAqB,CAAS;IAC9B,MAAM,CAAS;IACvB,UAAU,CAA8C;IAExD,YACE,QAA2B,EAC3B,UAWI,EAAE;QAEN,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,GAAG,CAAC;QAC9C,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,QAAQ,CAAC,kBAAkB,IAAI,EAAE,CAAC;QACxE,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,QAAQ,CAAC;QACvD,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC;QAC5C,IAAI,CAAC,eAAe,GAAG,OAAO,CAAC,eAAe,CAAC;QAC/C,IAAI,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,MAAM,CAAC;QACvD,IAAI,CAAC,qBAAqB,GAAG,OAAO,CAAC,qBAAqB,IAAI,QAAQ,CAAC;QACvE,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACrC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACzE,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAY;QAC3B,MAAM,SAAS,GAAG,MAAM,SAAS,CAC/B,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,OAAO,CAAC,EACxC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;QAEF,OAAO;YACL,GAAG,KAAK;YACR,SAAS;YACT,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;SAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,MAAe;QAC9B,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YAC3C,MAAM,UAAU,GAAG,MAAM,SAAS,CAChC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAW,CAAC,KAAK,CAAC,EACtC,EAAE,GAAG,IAAI,CAAC,YAAY,EAAE,WAAW,EAAE,kBAAkB,EAAE,EACzD,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,IAAI,UAAU,CAAC,MAAM,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC;gBACxC,MAAM,IAAI,UAAU,CAClB,uBAAuB,UAAU,CAAC,MAAM,mBAAmB,MAAM,CAAC,MAAM,SAAS,EACjF;oBACE,UAAU,EACR,mFAAmF;iBACtF,CACF,CAAC;YACJ,CAAC;YAED,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC/B,GAAG,KAAK;gBACR,SAAS,EAAE,UAAU,CAAC,CAAC,CAAC;gBACxB,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI;aAC9B,CAAC,CAAC,CAAC;QACN,CAAC;QAED,IAAI,SAAS,GAAG,CAAC,CAAC;QAClB,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,IAA4B,EAAE;YACrE,MAAM,KAAK,GACR,KAAK,CAAC,QAAQ,CAAC,UAAqB;gBACpC,KAAK,CAAC,QAAQ,CAAC,KAAgB;gBAChC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE;gBACjC,SAAS,CAAC;YAEZ,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;YAC9C,SAAS,EAAE,CAAC;YACZ,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,KAAK,EAAE,CAAC,CAAC;YAChE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,gBAAgB,CAAC,KAAK,CAAC,KAAK,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAClF,CAAC;YAEF,IAAI,IAAI,CAAC,WAAW,GAAG,CAAC,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,wBAAwB,IAAI,CAAC,WAAW,IAAI,CAAC,CAAC;gBAChE,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YAChC,CAAC;YAED,OAAO,QAAQ,CAAC;QAClB,CAAC,CAAC,CAAC;QAEH,OAAO,eAAe,CAAC,KAAK,EAAE,IAAI,CAAC,WAAW,CAAC,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,EAAgB,EAChB,MAAe,EACf,QAAiB,KAAK;QAMtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,MAAM,CAAC,MAAM,SAAS,CAAC,CAAC;QACtD,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,aAAa,IAAI,CAAC,QAAQ,CAAC,IAAI,YAAY,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,QAAQ,WAAW,IAAI,CAAC,QAAQ,CAAC,UAAU,EAAE,CAChH,CAAC;QAEF,MAAM,YAAY,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAElC,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,kDAAkD,CAAC,CAAC;YACxE,EAAE,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;QAC1E,CAAC;QAED,qDAAqD;QACrD,IAAI,YAAY,EAAE,CAAC;YACjB,MAAM,YAAY,GAChB,YAAY,CAAC,KAAK,KAAK,SAAS;gBAChC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;gBACjC,YAAY,CAAC,KAAK,KAAK,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;YAC7C,MAAM,iBAAiB,GACrB,YAAY,CAAC,kBAAkB,KAAK,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;YAE/D,IAAI,YAAY,IAAI,iBAAiB,EAAE,CAAC;gBACtC,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC,KAAK,IAAI,WAAW,CAAC;gBACpD,MAAM,QAAQ,GAAG,YAAY,CAAC,kBAAkB,CAAC;gBACjD,MAAM,OAAO,GAAG,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC;gBACzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;gBAChD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,SAAS,KAAK,QAAQ,IAAI,CAAC,CAAC;gBAC7D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,QAAQ,KAAK,OAAO,IAAI,CAAC,CAAC;gBAC3D,IAAI,CAAC,MAAM,CAAC,IAAI,CACd,qEAAqE,CACtE,CAAC;gBACF,EAAE,CAAC,QAAQ,EAAE,CAAC;gBACd,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,mBAAmB,EAAE,IAAI,EAAE,CAAC;YAC1E,CAAC;QACH,CAAC;QAED,0FAA0F;QAC1F,MAAM,kBAAkB,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QACvC,MAAM,oBAAoB,GAAG,IAAI,GAAG,EAAuB,CAAC;QAC5D,MAAM,qBAAqB,GAAG,IAAI,GAAG,EAAU,CAAC;QAChD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,GAAG,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;YACnD,MAAM,IAAI,GAAG,GAAG,CAAC,WAAW,IAAI,gBAAgB,CAAC,GAAG,CAAC,CAAC;YACtD,qBAAqB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAChC,MAAM,KAAK,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,oBAAoB,CAAC,GAAG,CAAC,GAAG,EAAE,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,gBAAgB,kBAAkB,CAAC,MAAM,kBAAkB,oBAAoB,CAAC,IAAI,aAAa,CAClG,CAAC;QACF,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,iBAAiB,IAAI,CAAC,eAAe,IAAI,QAAQ,EAAE,CAAC,CAAC;QAEvE,MAAM,mBAAmB,GAAG,IAAI,GAAG,EAAmB,CAAC;QACvD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;YAC3B,MAAM,GAAG,GAAG,GAAG,KAAK,CAAC,UAAU,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;YACvD,MAAM,KAAK,GAAG,mBAAmB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC3C,IAAI,KAAK,EAAE,CAAC;gBACV,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACpB,CAAC;iBAAM,CAAC;gBACN,mBAAmB,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,MAAM,aAAa,GAAY,EAAE,CAAC;QAClC,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QAEtB,KAAK,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,IAAI,mBAAmB,EAAE,CAAC;YACpD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAExD,IACE,iBAAiB;gBACjB,iBAAiB,CAAC,IAAI,KAAK,UAAU,CAAC,MAAM;gBAC5C,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAC,CAAC,CAAC,EACnE,CAAC;gBACD,YAAY,EAAE,CAAC;gBACf,aAAa,IAAI,UAAU,CAAC,MAAM,CAAC;gBACnC,SAAS;YACX,CAAC;YAED,KAAK,MAAM,KAAK,IAAI,UAAU,EAAE,CAAC;gBAC/B,IAAI,CAAC,qBAAqB,CAAC,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;oBACxD,aAAa,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC5B,CAAC;YACH,CAAC;QACH,CAAC;QAED,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;YACrB,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,cAAc,YAAY,0BAA0B,aAAa,UAAU,CAC5E,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,mBAAmB,EAAE,KAAK,EAAE,CAAC;IAC3E,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,EAAgB,EAChB,aAA8B,EAC9B,IAAoB;QAEpB,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACjB,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC;QACzB,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,SAAS,aAAa,CAAC,MAAM,uBAAuB,CAAC,CAAC;IAC5E,CAAC;IAED,KAAK,CAAC,GAAG,CACP,EAAgB,EAChB,UAAkB,EAClB,QAAiB,KAAK,EACtB,mBAAyC;QAEzC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,iDAAiD,CAAC,CAAC;QAEpE,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;YACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;YAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;gBAC3B,MAAM,IAAI,KAAK,CAAC,2CAA2C,CAAC,CAAC;YAC/D,CAAC;YACD,MAAM,GAAG,MAAiB,CAAC;QAC7B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,IAAI,UAAU,CAClB,8BAA8B,UAAU,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,EAC/F;gBACE,UAAU,EACR,oEAAoE;gBACtE,KAAK,EAAE,GAAG;aACX,CACF,CAAC;QACJ,CAAC;QAED,MAAM,EAAE,aAAa,EAAE,IAAI,EAAE,YAAY,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CACvE,EAAE,EACF,MAAM,EACN,KAAK,CACN,CAAC;QAEF,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAChD,OAAO,EAAE,CAAC;QACZ,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,oBAAoB,aAAa,CAAC,MAAM,SAAS,CAAC,CAAC;QAEpE,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,WAAW,CACzB,aAAa,EACb,IAAI,CAAC,SAAS,EACd,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EACvB,IAAI,CAAC,aAAa,CACnB,CAAC;QACF,MAAM,aAAa,GAAoB,EAAE,CAAC;QAC1C,IAAI,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACxC,MAAM,UAAU,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;YACxE,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,SAAS,CAAC,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,KAAK,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,aAAa,UAAU,QAAQ,CACtF,CAAC;YACF,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;YACnD,aAAa,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC,CAAC;YAChC,IAAI,CAAC,UAAU,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC,CAAC;YAE9D,MAAM,WAAW,GAAG,CAAC,KAAK,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC;YAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,YAAY,CAAC;YAC1C,IAAI,WAAW,IAAI,OAAO,IAAI,IAAI,CAAC,cAAc,EAAE,CAAC;gBAClD,MAAM,IAAI,GAAmB;oBAC3B,aAAa,EAAE,CAAC;oBAChB,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI;oBAChC,kBAAkB,EAAE,IAAI,CAAC,QAAQ,CAAC,UAAU;oBAC5C,GAAG,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,KAAK,SAAS;wBACnC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE;wBAChC,CAAC,CAAC,EAAE,CAAC;oBACP,GAAG,CAAC,IAAI,CAAC,eAAe,KAAK,SAAS;wBACpC,CAAC,CAAC,EAAE,eAAe,EAAE,IAAI,CAAC,eAAe,EAAE;wBAC3C,CAAC,CAAC,EAAE,CAAC;oBACP,SAAS,EAAE,YAAY,EAAE,SAAS,IAAI,GAAG;oBACzC,SAAS,EAAE,GAAG;iBACf,CAAC;gBACF,MAAM,IAAI,CAAC,cAAc,CAAC,EAAE,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;gBAC9C,YAAY,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAE1B,IACE,mBAAmB;oBACnB,EAAE,CAAC,YAAY,EAAE,IAAI,IAAI,CAAC,qBAAqB,EAC/C,CAAC;oBACD,MAAM,mBAAmB,EAAE,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;QAED,OAAO,aAAa,CAAC;IACvB,CAAC;CACF"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { EmbeddedChunk, EmbeddingsMeta } from "../interfaces/index.js";
|
|
2
|
+
export declare class EmbeddingsDb {
|
|
3
|
+
private db;
|
|
4
|
+
constructor(dbPath: string);
|
|
5
|
+
private isEmpty;
|
|
6
|
+
getMeta(): EmbeddingsMeta | null;
|
|
7
|
+
setMeta(meta: EmbeddingsMeta): void;
|
|
8
|
+
insert(chunks: EmbeddedChunk[]): void;
|
|
9
|
+
has(contentHash: string): boolean;
|
|
10
|
+
getAll(): EmbeddedChunk[];
|
|
11
|
+
getPending(): EmbeddedChunk[];
|
|
12
|
+
markUploaded(contentHashes: string[]): void;
|
|
13
|
+
pendingCount(): number;
|
|
14
|
+
clearAll(): void;
|
|
15
|
+
pruneUploaded(): void;
|
|
16
|
+
migrateFromJson(jsonPath: string): void;
|
|
17
|
+
close(): void;
|
|
18
|
+
private rows;
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=embeddings-db.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings-db.d.ts","sourceRoot":"","sources":["../../src/core/embeddings-db.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AA2B5E,qBAAa,YAAY;IACvB,OAAO,CAAC,EAAE,CAAoB;gBAElB,MAAM,EAAE,MAAM;IAW1B,OAAO,CAAC,OAAO;IAOf,OAAO,IAAI,cAAc,GAAG,IAAI;IAYhC,OAAO,CAAC,IAAI,EAAE,cAAc,GAAG,IAAI;IAMnC,MAAM,CAAC,MAAM,EAAE,aAAa,EAAE,GAAG,IAAI;IAsBrC,GAAG,CAAC,WAAW,EAAE,MAAM,GAAG,OAAO;IAOjC,MAAM,IAAI,aAAa,EAAE;IAIzB,UAAU,IAAI,aAAa,EAAE;IAI7B,YAAY,CAAC,aAAa,EAAE,MAAM,EAAE,GAAG,IAAI;IAU3C,YAAY,IAAI,MAAM;IAOtB,QAAQ,IAAI,IAAI;IAKhB,aAAa,IAAI,IAAI;IAIrB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,IAAI;IAyDvC,KAAK,IAAI,IAAI;IAIb,OAAO,CAAC,IAAI;CAkCb"}
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
import { createHash } from "crypto";
|
|
2
|
+
import { existsSync, readFileSync } from "fs";
|
|
3
|
+
import { rename } from "fs/promises";
|
|
4
|
+
import Database from "better-sqlite3";
|
|
5
|
+
function chunkContentHash(chunk) {
|
|
6
|
+
return (chunk.contentHash ??
|
|
7
|
+
createHash("sha256").update(chunk.content).digest("hex").slice(0, 16));
|
|
8
|
+
}
|
|
9
|
+
const SCHEMA = `
|
|
10
|
+
CREATE TABLE IF NOT EXISTS meta (
|
|
11
|
+
key TEXT PRIMARY KEY,
|
|
12
|
+
value TEXT NOT NULL
|
|
13
|
+
);
|
|
14
|
+
CREATE TABLE IF NOT EXISTS embeddings (
|
|
15
|
+
content_hash TEXT PRIMARY KEY,
|
|
16
|
+
source_file TEXT NOT NULL,
|
|
17
|
+
commit_hash TEXT NOT NULL,
|
|
18
|
+
content TEXT NOT NULL,
|
|
19
|
+
metadata_json TEXT NOT NULL,
|
|
20
|
+
embedding_json TEXT NOT NULL,
|
|
21
|
+
embedded_at INTEGER NOT NULL,
|
|
22
|
+
uploaded INTEGER NOT NULL DEFAULT 0
|
|
23
|
+
);
|
|
24
|
+
CREATE INDEX IF NOT EXISTS idx_source_file ON embeddings(source_file);
|
|
25
|
+
`;
|
|
26
|
+
export class EmbeddingsDb {
|
|
27
|
+
db;
|
|
28
|
+
constructor(dbPath) {
|
|
29
|
+
this.db = new Database(dbPath);
|
|
30
|
+
this.db.pragma("journal_mode = WAL");
|
|
31
|
+
this.db.exec(SCHEMA);
|
|
32
|
+
const jsonPath = dbPath.replace(/\.db$/, ".json");
|
|
33
|
+
if (this.isEmpty() && existsSync(jsonPath)) {
|
|
34
|
+
this.migrateFromJson(jsonPath);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
isEmpty() {
|
|
38
|
+
const row = this.db
|
|
39
|
+
.prepare("SELECT COUNT(*) as cnt FROM embeddings")
|
|
40
|
+
.get();
|
|
41
|
+
return row.cnt === 0;
|
|
42
|
+
}
|
|
43
|
+
getMeta() {
|
|
44
|
+
const row = this.db
|
|
45
|
+
.prepare("SELECT value FROM meta WHERE key = 'meta'")
|
|
46
|
+
.get();
|
|
47
|
+
if (!row)
|
|
48
|
+
return null;
|
|
49
|
+
try {
|
|
50
|
+
return JSON.parse(row.value);
|
|
51
|
+
}
|
|
52
|
+
catch {
|
|
53
|
+
return null;
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
setMeta(meta) {
|
|
57
|
+
this.db
|
|
58
|
+
.prepare("INSERT OR REPLACE INTO meta (key, value) VALUES ('meta', ?)")
|
|
59
|
+
.run(JSON.stringify(meta));
|
|
60
|
+
}
|
|
61
|
+
insert(chunks) {
|
|
62
|
+
const stmt = this.db.prepare(`
|
|
63
|
+
INSERT OR IGNORE INTO embeddings
|
|
64
|
+
(content_hash, source_file, commit_hash, content, metadata_json, embedding_json, embedded_at, uploaded)
|
|
65
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 0)
|
|
66
|
+
`);
|
|
67
|
+
const insertMany = this.db.transaction((items) => {
|
|
68
|
+
for (const chunk of items) {
|
|
69
|
+
stmt.run(chunkContentHash(chunk), chunk.sourceFile, chunk.commitHash, chunk.content, JSON.stringify(chunk.metadata), JSON.stringify(chunk.embedding), chunk.embeddedAt);
|
|
70
|
+
}
|
|
71
|
+
});
|
|
72
|
+
insertMany(chunks);
|
|
73
|
+
}
|
|
74
|
+
has(contentHash) {
|
|
75
|
+
const row = this.db
|
|
76
|
+
.prepare("SELECT 1 FROM embeddings WHERE content_hash = ?")
|
|
77
|
+
.get(contentHash);
|
|
78
|
+
return row !== undefined;
|
|
79
|
+
}
|
|
80
|
+
getAll() {
|
|
81
|
+
return this.rows("SELECT * FROM embeddings");
|
|
82
|
+
}
|
|
83
|
+
getPending() {
|
|
84
|
+
return this.rows("SELECT * FROM embeddings WHERE uploaded = 0");
|
|
85
|
+
}
|
|
86
|
+
markUploaded(contentHashes) {
|
|
87
|
+
if (contentHashes.length === 0)
|
|
88
|
+
return;
|
|
89
|
+
const placeholders = contentHashes.map(() => "?").join(", ");
|
|
90
|
+
this.db
|
|
91
|
+
.prepare(`UPDATE embeddings SET uploaded = 1 WHERE content_hash IN (${placeholders})`)
|
|
92
|
+
.run(...contentHashes);
|
|
93
|
+
}
|
|
94
|
+
pendingCount() {
|
|
95
|
+
const row = this.db
|
|
96
|
+
.prepare("SELECT COUNT(*) as cnt FROM embeddings WHERE uploaded = 0")
|
|
97
|
+
.get();
|
|
98
|
+
return row.cnt;
|
|
99
|
+
}
|
|
100
|
+
clearAll() {
|
|
101
|
+
this.db.prepare("DELETE FROM embeddings").run();
|
|
102
|
+
this.db.prepare("DELETE FROM meta").run();
|
|
103
|
+
}
|
|
104
|
+
pruneUploaded() {
|
|
105
|
+
this.db.prepare("DELETE FROM embeddings WHERE uploaded = 1").run();
|
|
106
|
+
}
|
|
107
|
+
migrateFromJson(jsonPath) {
|
|
108
|
+
let raw;
|
|
109
|
+
try {
|
|
110
|
+
raw = readFileSync(jsonPath, "utf-8");
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return;
|
|
114
|
+
}
|
|
115
|
+
let parsed;
|
|
116
|
+
try {
|
|
117
|
+
parsed = JSON.parse(raw);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
let chunks;
|
|
123
|
+
let meta = null;
|
|
124
|
+
if (Array.isArray(parsed)) {
|
|
125
|
+
chunks = parsed;
|
|
126
|
+
}
|
|
127
|
+
else {
|
|
128
|
+
const file = parsed;
|
|
129
|
+
chunks = file.chunks ?? [];
|
|
130
|
+
meta = file._meta ?? null;
|
|
131
|
+
}
|
|
132
|
+
if (meta)
|
|
133
|
+
this.setMeta(meta);
|
|
134
|
+
if (chunks.length > 0) {
|
|
135
|
+
// Mark migrated chunks as uploaded since they were previously written to JSON
|
|
136
|
+
// (implying they were already synced to the vector store)
|
|
137
|
+
const stmt = this.db.prepare(`
|
|
138
|
+
INSERT OR IGNORE INTO embeddings
|
|
139
|
+
(content_hash, source_file, commit_hash, content, metadata_json, embedding_json, embedded_at, uploaded)
|
|
140
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, 1)
|
|
141
|
+
`);
|
|
142
|
+
const insertAll = this.db.transaction((items) => {
|
|
143
|
+
for (const chunk of items) {
|
|
144
|
+
stmt.run(chunkContentHash(chunk), chunk.sourceFile, chunk.commitHash, chunk.content, JSON.stringify(chunk.metadata), JSON.stringify(chunk.embedding), chunk.embeddedAt);
|
|
145
|
+
}
|
|
146
|
+
});
|
|
147
|
+
insertAll(chunks);
|
|
148
|
+
}
|
|
149
|
+
rename(jsonPath, jsonPath + ".migrated").catch(() => { });
|
|
150
|
+
}
|
|
151
|
+
close() {
|
|
152
|
+
this.db.close();
|
|
153
|
+
}
|
|
154
|
+
rows(sql) {
|
|
155
|
+
const rows = this.db.prepare(sql).all();
|
|
156
|
+
return rows.map((row) => ({
|
|
157
|
+
contentHash: row.content_hash,
|
|
158
|
+
sourceFile: row.source_file,
|
|
159
|
+
commitHash: row.commit_hash,
|
|
160
|
+
content: row.content,
|
|
161
|
+
metadata: (() => {
|
|
162
|
+
try {
|
|
163
|
+
return JSON.parse(row.metadata_json);
|
|
164
|
+
}
|
|
165
|
+
catch {
|
|
166
|
+
return {};
|
|
167
|
+
}
|
|
168
|
+
})(),
|
|
169
|
+
embedding: (() => {
|
|
170
|
+
try {
|
|
171
|
+
return JSON.parse(row.embedding_json);
|
|
172
|
+
}
|
|
173
|
+
catch {
|
|
174
|
+
return [];
|
|
175
|
+
}
|
|
176
|
+
})(),
|
|
177
|
+
embeddedAt: row.embedded_at,
|
|
178
|
+
}));
|
|
179
|
+
}
|
|
180
|
+
}
|
|
181
|
+
//# sourceMappingURL=embeddings-db.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embeddings-db.js","sourceRoot":"","sources":["../../src/core/embeddings-db.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,MAAM,IAAI,CAAC;AAC9C,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,QAAQ,MAAM,gBAAgB,CAAC;AAGtC,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,OAAO,CACL,KAAK,CAAC,WAAW;QACjB,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;CAgBd,CAAC;AAEF,MAAM,OAAO,YAAY;IACf,EAAE,CAAoB;IAE9B,YAAY,MAAc;QACxB,IAAI,CAAC,EAAE,GAAG,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;QAC/B,IAAI,CAAC,EAAE,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC;QACrC,IAAI,CAAC,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAErB,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;QAClD,IAAI,IAAI,CAAC,OAAO,EAAE,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC3C,IAAI,CAAC,eAAe,CAAC,QAAQ,CAAC,CAAC;QACjC,CAAC;IACH,CAAC;IAEO,OAAO;QACb,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,wCAAwC,CAAC;aACjD,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC;IACvB,CAAC;IAED,OAAO;QACL,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,2CAA2C,CAAC;aACpD,GAAG,EAAmC,CAAC;QAC1C,IAAI,CAAC,GAAG;YAAE,OAAO,IAAI,CAAC;QACtB,IAAI,CAAC;YACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAmB,CAAC;QACjD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAED,OAAO,CAAC,IAAoB;QAC1B,IAAI,CAAC,EAAE;aACJ,OAAO,CAAC,6DAA6D,CAAC;aACtE,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;IAC/B,CAAC;IAED,MAAM,CAAC,MAAuB;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;KAI5B,CAAC,CAAC;QACH,MAAM,UAAU,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAsB,EAAE,EAAE;YAChE,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;gBAC1B,IAAI,CAAC,GAAG,CACN,gBAAgB,CAAC,KAAK,CAAC,EACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,OAAO,EACb,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,EAC/B,KAAK,CAAC,UAAU,CACjB,CAAC;YACJ,CAAC;QACH,CAAC,CAAC,CAAC;QACH,UAAU,CAAC,MAAM,CAAC,CAAC;IACrB,CAAC;IAED,GAAG,CAAC,WAAmB;QACrB,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,iDAAiD,CAAC;aAC1D,GAAG,CAAC,WAAW,CAAC,CAAC;QACpB,OAAO,GAAG,KAAK,SAAS,CAAC;IAC3B,CAAC;IAED,MAAM;QACJ,OAAO,IAAI,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;IAC/C,CAAC;IAED,UAAU;QACR,OAAO,IAAI,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;IAClE,CAAC;IAED,YAAY,CAAC,aAAuB;QAClC,IAAI,aAAa,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACvC,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC7D,IAAI,CAAC,EAAE;aACJ,OAAO,CACN,6DAA6D,YAAY,GAAG,CAC7E;aACA,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC;IAC3B,CAAC;IAED,YAAY;QACV,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE;aAChB,OAAO,CAAC,2DAA2D,CAAC;aACpE,GAAG,EAAqB,CAAC;QAC5B,OAAO,GAAG,CAAC,GAAG,CAAC;IACjB,CAAC;IAED,QAAQ;QACN,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,wBAAwB,CAAC,CAAC,GAAG,EAAE,CAAC;QAChD,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,kBAAkB,CAAC,CAAC,GAAG,EAAE,CAAC;IAC5C,CAAC;IAED,aAAa;QACX,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,2CAA2C,CAAC,CAAC,GAAG,EAAE,CAAC;IACrE,CAAC;IAED,eAAe,CAAC,QAAgB;QAC9B,IAAI,GAAW,CAAC;QAChB,IAAI,CAAC;YACH,GAAG,GAAG,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACxC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC3B,CAAC;QAAC,MAAM,CAAC;YACP,OAAO;QACT,CAAC;QAED,IAAI,MAAuB,CAAC;QAC5B,IAAI,IAAI,GAA0B,IAAI,CAAC;QAEvC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;YAC1B,MAAM,GAAG,MAAyB,CAAC;QACrC,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,GAAG,MAGZ,CAAC;YACF,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YAC3B,IAAI,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC;QAC5B,CAAC;QAED,IAAI,IAAI;YAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,8EAA8E;YAC9E,0DAA0D;YAC1D,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC;;;;OAI5B,CAAC,CAAC;YACH,MAAM,SAAS,GAAG,IAAI,CAAC,EAAE,CAAC,WAAW,CAAC,CAAC,KAAsB,EAAE,EAAE;gBAC/D,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;oBAC1B,IAAI,CAAC,GAAG,CACN,gBAAgB,CAAC,KAAK,CAAC,EACvB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,UAAU,EAChB,KAAK,CAAC,OAAO,EACb,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,CAAC,EAC9B,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,SAAS,CAAC,EAC/B,KAAK,CAAC,UAAU,CACjB,CAAC;gBACJ,CAAC;YACH,CAAC,CAAC,CAAC;YACH,SAAS,CAAC,MAAM,CAAC,CAAC;QACpB,CAAC;QAED,MAAM,CAAC,QAAQ,EAAE,QAAQ,GAAG,WAAW,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;IAC3D,CAAC;IAED,KAAK;QACH,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAC;IAClB,CAAC;IAEO,IAAI,CAAC,GAAW;QAWtB,MAAM,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,EAAW,CAAC;QACjD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC;YACxB,WAAW,EAAE,GAAG,CAAC,YAAY;YAC7B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,UAAU,EAAE,GAAG,CAAC,WAAW;YAC3B,OAAO,EAAE,GAAG,CAAC,OAAO;YACpB,QAAQ,EAAE,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,aAAa,CAA4B,CAAC;gBAClE,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE;YACJ,SAAS,EAAE,CAAC,GAAG,EAAE;gBACf,IAAI,CAAC;oBACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,cAAc,CAAa,CAAC;gBACpD,CAAC;gBAAC,MAAM,CAAC;oBACP,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC,CAAC,EAAE;YACJ,UAAU,EAAE,GAAG,CAAC,WAAW;SAC5B,CAAC,CAAC,CAAC;IACN,CAAC;CACF"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAUA,OAAO,EACL,WAAW,EACX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAG1C,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,WAAW,EAAE,CAAC;IACxB,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,WAAW,EAAE,WAAW,CAAC;IACzB,OAAO,CAAC,EAAE;QACR,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,KAAK,CAAC,EAAE,OAAO,CAAC;QAChB,UAAU,CAAC,EAAE,OAAO,CAAC;QACrB,MAAM,CAAC,EAAE,OAAO,CAAC;QACjB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,aAAa,CAAC,EAAE,MAAM,CAAC;QACvB,KAAK,CAAC,EAAE,YAAY,CAAC;QACrB,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,qBAAqB,CAAC,EAAE,MAAM,CAAC;QAC/B,SAAS,CAAC,EAAE,OAAO,CAAC;QACpB,aAAa,CAAC,EAAE;YAAE,UAAU,CAAC,EAAE,MAAM,CAAA;SAAE,CAAC;QACxC,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC;CACH;AAsCD,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAoB;IAClC,OAAO,CAAC,UAAU,CAAS;IAC3B,OAAO,CAAC,cAAc,CAAS;gBAEnB,MAAM,EAAE,iBAAiB;IAO/B,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC;CA4M3B"}
|
|
@@ -2,16 +2,15 @@ import { GitTracker } from "./git-tracker.js";
|
|
|
2
2
|
import { ChunkProcessor } from "./chunk-processor.js";
|
|
3
3
|
import { EmbedderProcessor } from "./embedder.js";
|
|
4
4
|
import { Uploader } from "./uploader.js";
|
|
5
|
+
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
5
6
|
import { TelemetryCollector } from "./telemetry.js";
|
|
6
|
-
import { createProgressBar } from "../progress/progress-bar.js";
|
|
7
|
+
import { createProgressBar, } from "../progress/progress-bar.js";
|
|
7
8
|
import { NullLogger } from "../logger/null-logger.js";
|
|
8
9
|
import { readFile } from "fs/promises";
|
|
9
|
-
import { readEmbeddingsFile } from "./embeddings-io.js";
|
|
10
10
|
import { defaultChunksFile, defaultEmbeddingsFile } from "./virage-defaults.js";
|
|
11
|
-
|
|
12
|
-
const { chunks } = await readEmbeddingsFile(embeddingsFile);
|
|
11
|
+
function loadPreviousState(db) {
|
|
13
12
|
const state = new Map();
|
|
14
|
-
for (const chunk of
|
|
13
|
+
for (const chunk of db.getAll()) {
|
|
15
14
|
if (chunk.sourceFile && chunk.commitHash) {
|
|
16
15
|
state.set(chunk.sourceFile, chunk.commitHash);
|
|
17
16
|
}
|
|
@@ -55,6 +54,8 @@ export class Orchestrator {
|
|
|
55
54
|
const logger = (opts.logger ?? new NullLogger()).withTag("orchestrator");
|
|
56
55
|
const telemetry = opts.telemetry ? new TelemetryCollector() : null;
|
|
57
56
|
telemetry?.start();
|
|
57
|
+
const dbPath = this.embeddingsFile.replace(/\.json$/, ".db");
|
|
58
|
+
const db = new EmbeddingsDb(dbPath);
|
|
58
59
|
let uploadStats = { uploaded: 0, deleted: 0 };
|
|
59
60
|
logger.debug(`Config: chunksFile=${this.chunksFile} embeddingsFile=${this.embeddingsFile} force=${opts.force ?? false}`);
|
|
60
61
|
try {
|
|
@@ -66,7 +67,7 @@ export class Orchestrator {
|
|
|
66
67
|
const currentState = await gitTracker.getCurrentState();
|
|
67
68
|
const previousState = opts.force
|
|
68
69
|
? new Map()
|
|
69
|
-
:
|
|
70
|
+
: loadPreviousState(db);
|
|
70
71
|
const { toProcess, toDelete } = await gitTracker.getChangedFiles(previousState, currentState);
|
|
71
72
|
const gitDuration = Date.now() - t1;
|
|
72
73
|
logger.verbose(`Git tracking done in ${gitDuration}ms`);
|
|
@@ -78,6 +79,7 @@ export class Orchestrator {
|
|
|
78
79
|
});
|
|
79
80
|
if (toProcess.length === 0 && toDelete.length === 0 && !opts.force) {
|
|
80
81
|
logger.info("✨ No changes detected.");
|
|
82
|
+
db.close();
|
|
81
83
|
return;
|
|
82
84
|
}
|
|
83
85
|
logger.info(`📊 Changes: ${toProcess.length} to process, ${toDelete.length} to delete`);
|
|
@@ -108,12 +110,17 @@ export class Orchestrator {
|
|
|
108
110
|
});
|
|
109
111
|
if (chunks.length === 0) {
|
|
110
112
|
logger.warn("⚠️ No chunks generated. Exiting.");
|
|
113
|
+
db.close();
|
|
111
114
|
return;
|
|
112
115
|
}
|
|
113
116
|
// Step 3: Generate embeddings
|
|
114
117
|
logger.info("🔢 Step 3: Generating embeddings...");
|
|
115
118
|
const t3 = Date.now();
|
|
116
119
|
const embedBars = [];
|
|
120
|
+
const uploader = new Uploader(this.config.vectorStore, {
|
|
121
|
+
retry: opts.retry,
|
|
122
|
+
logger: opts.logger,
|
|
123
|
+
});
|
|
117
124
|
const embedder = new EmbedderProcessor(this.config.embedder, {
|
|
118
125
|
rateLimitMs: opts.rateLimitMs,
|
|
119
126
|
batchSize: opts.batchSize,
|
|
@@ -121,6 +128,7 @@ export class Orchestrator {
|
|
|
121
128
|
retry: opts.retry,
|
|
122
129
|
concurrency: opts.concurrency,
|
|
123
130
|
vectorStoreName: this.config.vectorStore.name,
|
|
131
|
+
minIngestionBatchSize: opts.minIngestionBatchSize,
|
|
124
132
|
logger: opts.logger,
|
|
125
133
|
onProgress: (done, total) => {
|
|
126
134
|
if (embedBars.length === 0)
|
|
@@ -128,7 +136,11 @@ export class Orchestrator {
|
|
|
128
136
|
embedBars[0].update(done < total ? done : total);
|
|
129
137
|
},
|
|
130
138
|
});
|
|
131
|
-
const newEmbeddings = await embedder.run(this.chunksFile, opts.force || false
|
|
139
|
+
const newEmbeddings = await embedder.run(db, this.chunksFile, opts.force || false, opts.skipUpload
|
|
140
|
+
? undefined
|
|
141
|
+
: async () => {
|
|
142
|
+
await uploader.uploadPending(db);
|
|
143
|
+
});
|
|
132
144
|
embedBars[0]?.stop();
|
|
133
145
|
const embedDuration = Date.now() - t3;
|
|
134
146
|
logger.verbose(`Embedding done in ${embedDuration}ms`);
|
|
@@ -139,8 +151,7 @@ export class Orchestrator {
|
|
|
139
151
|
});
|
|
140
152
|
// Step 4: Upload
|
|
141
153
|
if (opts.dryRun) {
|
|
142
|
-
const
|
|
143
|
-
const { toUpload: dryUpload, toDelete: dryDelete } = await uploader.getItemsToUpload(this.embeddingsFile, opts.force || false);
|
|
154
|
+
const { toUpload: dryUpload, toDelete: dryDelete } = await uploader.getItemsToUpload(db, opts.force || false);
|
|
144
155
|
logger.info("📤 Step 4: Upload (dry-run — no changes written)");
|
|
145
156
|
logger.info(` Would upload: ${dryUpload.length} document(s)`);
|
|
146
157
|
logger.info(` Would delete: ${dryDelete.length} source file(s)`);
|
|
@@ -148,11 +159,7 @@ export class Orchestrator {
|
|
|
148
159
|
else if (!opts.skipUpload) {
|
|
149
160
|
logger.info("📤 Step 4: Uploading to vector store...");
|
|
150
161
|
const t4 = Date.now();
|
|
151
|
-
|
|
152
|
-
retry: opts.retry,
|
|
153
|
-
logger: opts.logger,
|
|
154
|
-
});
|
|
155
|
-
uploadStats = await uploader.sync(this.embeddingsFile, opts.force || false);
|
|
162
|
+
uploadStats = await uploader.sync(db, opts.force || false);
|
|
156
163
|
const uploadDuration = Date.now() - t4;
|
|
157
164
|
logger.verbose(`Upload done in ${uploadDuration}ms`);
|
|
158
165
|
telemetry?.recordUpload({
|
|
@@ -187,6 +194,9 @@ export class Orchestrator {
|
|
|
187
194
|
}
|
|
188
195
|
throw err;
|
|
189
196
|
}
|
|
197
|
+
finally {
|
|
198
|
+
db.close();
|
|
199
|
+
}
|
|
190
200
|
}
|
|
191
201
|
}
|
|
192
202
|
//# sourceMappingURL=orchestrator.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"orchestrator.js","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AACzC,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAClD,OAAO,EAAE,kBAAkB,EAAE,MAAM,gBAAgB,CAAC;AACpD,OAAO,EACL,iBAAiB,GAElB,MAAM,6BAA6B,CAAC;AAQrC,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,EAAE,iBAAiB,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAwBhF,SAAS,iBAAiB,CAAC,EAAgB;IACzC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAkB,CAAC;IACxC,KAAK,MAAM,KAAK,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,UAAU,IAAI,KAAK,CAAC,UAAU,EAAE,CAAC;YACzC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;QAChD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,UAAkB;IAClD,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QACpD,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC5C,OAAO,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,CAAE,MAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,aAAa,CAC1B,GAAW,EACX,MAA2B,EAC3B,OAAgC;IAEhC,IAAI,CAAC;QACH,MAAM,KAAK,CAAC,GAAG,EAAE;YACf,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,GAAG,OAAO,EAAE,CAAC;SAC7C,CAAC,CAAC;IACL,CAAC;IAAC,MAAM,CAAC;QACP,sCAAsC;IACxC,CAAC;AACH,CAAC;AAED,MAAM,OAAO,YAAY;IACf,MAAM,CAAoB;IAC1B,UAAU,CAAS;IACnB,cAAc,CAAS;IAE/B,YAAY,MAAyB;QACnC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;QACrB,IAAI,CAAC,UAAU,GAAG,MAAM,CAAC,OAAO,EAAE,UAAU,IAAI,iBAAiB,EAAE,CAAC;QACpE,IAAI,CAAC,cAAc;YACjB,MAAM,CAAC,OAAO,EAAE,cAAc,IAAI,qBAAqB,EAAE,CAAC;IAC9D,CAAC;IAED,KAAK,CAAC,GAAG;QACP,MAAM,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,EAAE,CAAC;QACvC,MAAM,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC;QACzE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,kBAAkB,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;QACnE,SAAS,EAAE,KAAK,EAAE,CAAC;QAEnB,MAAM,MAAM,GAAG,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;QAC7D,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,MAAM,CAAC,CAAC;QAEpC,IAAI,WAAW,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE9C,MAAM,CAAC,KAAK,CACV,sBAAsB,IAAI,CAAC,UAAU,mBAAmB,IAAI,CAAC,cAAc,UAAU,IAAI,CAAC,KAAK,IAAI,KAAK,EAAE,CAC3G,CAAC;QAEF,IAAI,CAAC;YACH,MAAM,CAAC,IAAI,CAAC,6BAA6B,CAAC,CAAC;YAE3C,2BAA2B;YAC3B,MAAM,CAAC,IAAI,CAAC,oCAAoC,CAAC,CAAC;YAClD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACrE,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,eAAe,EAAE,CAAC;YAExD,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK;gBAC9B,CAAC,CAAC,IAAI,GAAG,EAAkB;gBAC3B,CAAC,CAAC,iBAAiB,CAAC,EAAE,CAAC,CAAC;YAE1B,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,GAAG,MAAM,UAAU,CAAC,eAAe,CAC9D,aAAa,EACb,YAAY,CACb,CAAC;YAEF,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACpC,MAAM,CAAC,OAAO,CAAC,wBAAwB,WAAW,IAAI,CAAC,CAAC;YAExD,SAAS,EAAE,iBAAiB,CAAC;gBAC3B,UAAU,EAAE,WAAW;gBACvB,YAAY,EAAE,YAAY,CAAC,IAAI;gBAC/B,SAAS,EAAE,SAAS,CAAC,MAAM;gBAC3B,QAAQ,EAAE,QAAQ,CAAC,MAAM;aAC1B,CAAC,CAAC;YAEH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC;gBACnE,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;gBACtC,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,MAAM,CAAC,IAAI,CACT,eAAe,SAAS,CAAC,MAAM,gBAAgB,QAAQ,CAAC,MAAM,YAAY,CAC3E,CAAC;YAEF,wCAAwC;YACxC,MAAM,CAAC,IAAI,CAAC,iCAAiC,CAAC,CAAC;YAC/C,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,cAAc,GAAG,IAAI,cAAc,CACvC,IAAI,CAAC,MAAM,CAAC,QAAQ,EACpB,IAAI,CAAC,MAAM,CACZ,CAAC;YAEF,MAAM,SAAS,GAAG,IAAI,GAAG,EAGtB,CAAC;YACJ,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;gBAC7B,MAAM,IAAI,GAAG,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACpC,IAAI,IAAI;oBAAE,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;YACtC,CAAC;YAED,MAAM,cAAc,GAAG,IAAI,CAAC,KAAK;gBAC/B,CAAC,CAAC,EAAE;gBACJ,CAAC,CAAC,MAAM,kBAAkB,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9C,MAAM,QAAQ,GAAG,iBAAiB,CAAC,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,CAAC;YACjE,MAAM,MAAM,GAAG,MAAM,cAAc,CAAC,YAAY,CAC9C,SAAS,EACT,SAAS,EACT,cAAc,EACd,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAC9D,CAAC;YACF,QAAQ,CAAC,IAAI,EAAE,CAAC;YAChB,MAAM,cAAc,CAAC,eAAe,CAAC,MAAM,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;YAE9D,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,oBAAoB,aAAa,IAAI,CAAC,CAAC;YAEtD,SAAS,EAAE,cAAc,CAAC;gBACxB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,SAAS,CAAC,MAAM;gBAChC,eAAe,EAAE,MAAM,CAAC,MAAM;gBAC9B,MAAM,EAAE,CAAC;aACV,CAAC,CAAC;YAEH,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;gBAChD,EAAE,CAAC,KAAK,EAAE,CAAC;gBACX,OAAO;YACT,CAAC;YAED,8BAA8B;YAC9B,MAAM,CAAC,IAAI,CAAC,qCAAqC,CAAC,CAAC;YACnD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,MAAM,SAAS,GAAkB,EAAE,CAAC;YAEpC,MAAM,QAAQ,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,CAAC,WAAW,EAAE;gBACrD,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,MAAM,EAAE,IAAI,CAAC,MAAM;aACpB,CAAC,CAAC;YAEH,MAAM,QAAQ,GAAG,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE;gBAC3D,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,SAAS,EAAE,IAAI,CAAC,SAAS;gBACzB,aAAa,EAAE,IAAI,CAAC,aAAa;gBACjC,KAAK,EAAE,IAAI,CAAC,KAAK;gBACjB,WAAW,EAAE,IAAI,CAAC,WAAW;gBAC7B,eAAe,EAAE,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,IAAI;gBAC7C,qBAAqB,EAAE,IAAI,CAAC,qBAAqB;gBACjD,MAAM,EAAE,IAAI,CAAC,MAAM;gBACnB,UAAU,EAAE,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;oBAC1B,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC;wBACxB,SAAS,CAAC,IAAI,CAAC,iBAAiB,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;oBACxD,SAAS,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;gBACnD,CAAC;aACF,CAAC,CAAC;YAEH,MAAM,aAAa,GAAG,MAAM,QAAQ,CAAC,GAAG,CACtC,EAAE,EACF,IAAI,CAAC,UAAU,EACf,IAAI,CAAC,KAAK,IAAI,KAAK,EACnB,IAAI,CAAC,UAAU;gBACb,CAAC,CAAC,SAAS;gBACX,CAAC,CAAC,KAAK,IAAI,EAAE;oBACT,MAAM,QAAQ,CAAC,aAAa,CAAC,EAAE,CAAC,CAAC;gBACnC,CAAC,CACN,CAAC;YACF,SAAS,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;YAErB,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;YACtC,MAAM,CAAC,OAAO,CAAC,qBAAqB,aAAa,IAAI,CAAC,CAAC;YAEvD,SAAS,EAAE,eAAe,CAAC;gBACzB,UAAU,EAAE,aAAa;gBACzB,cAAc,EAAE,aAAa,CAAC,MAAM;gBACpC,aAAa,EAAE,MAAM,CAAC,MAAM,GAAG,aAAa,CAAC,MAAM;aACpD,CAAC,CAAC;YAEH,iBAAiB;YACjB,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;gBAChB,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,QAAQ,EAAE,SAAS,EAAE,GAChD,MAAM,QAAQ,CAAC,gBAAgB,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC;gBAC3D,MAAM,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,cAAc,CAAC,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,oBAAoB,SAAS,CAAC,MAAM,iBAAiB,CAAC,CAAC;YACrE,CAAC;iBAAM,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,CAAC;gBAC5B,MAAM,CAAC,IAAI,CAAC,yCAAyC,CAAC,CAAC;gBACvD,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBACtB,WAAW,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,CAAC;gBAE3D,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,CAAC;gBACvC,MAAM,CAAC,OAAO,CAAC,kBAAkB,cAAc,IAAI,CAAC,CAAC;gBAErD,SAAS,EAAE,YAAY,CAAC;oBACtB,UAAU,EAAE,cAAc;oBAC1B,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;YAED,MAAM,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;YAE3C,IAAI,SAAS,EAAE,CAAC;gBACd,SAAS,CAAC,MAAM,EAAE,CAAC;gBACnB,SAAS,CAAC,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;gBACpC,MAAM,aAAa,GAAG,IAAI,CAAC,UAAU,CAAC,OAAO,CAC3C,aAAa,EACb,gBAAgB,CACjB,CAAC;gBACF,MAAM,SAAS,CAAC,IAAI,CAAC,aAAa,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;YACnD,CAAC;YAED,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,SAAS,EAAE;oBAC5D,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;oBAC3C,MAAM,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,MAAM;oBACnC,QAAQ,EAAE,WAAW,CAAC,QAAQ;oBAC9B,OAAO,EAAE,WAAW,CAAC,OAAO;iBAC7B,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,SAAS,EAAE,MAAM,EAAE,CAAC;YAEpB,IAAI,IAAI,CAAC,aAAa,EAAE,UAAU,EAAE,CAAC;gBACnC,MAAM,aAAa,CAAC,IAAI,CAAC,aAAa,CAAC,UAAU,EAAE,OAAO,EAAE;oBAC1D,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;oBACvD,UAAU,EAAE,SAAS,EAAE,OAAO,EAAE,CAAC,UAAU;iBAC5C,CAAC,CAAC;YACL,CAAC;YAED,MAAM,GAAG,CAAC;QACZ,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,KAAK,EAAE,CAAC;QACb,CAAC;IACH,CAAC;CACF"}
|
package/dist/core/uploader.d.ts
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
import { VectorStore, EmbeddedChunk } from "../interfaces/index.js";
|
|
2
2
|
import type { Logger } from "../interfaces/logger.js";
|
|
3
3
|
import { RetryOptions } from "./utils.js";
|
|
4
|
+
import { EmbeddingsDb } from "./embeddings-db.js";
|
|
5
|
+
export declare function isFatalVectorStoreError(err: unknown): boolean;
|
|
4
6
|
export declare class Uploader {
|
|
5
7
|
private vectorStore;
|
|
6
8
|
private retryOptions;
|
|
@@ -10,11 +12,15 @@ export declare class Uploader {
|
|
|
10
12
|
logger?: Logger;
|
|
11
13
|
});
|
|
12
14
|
private chunkToDocument;
|
|
13
|
-
getItemsToUpload(
|
|
15
|
+
getItemsToUpload(db: EmbeddingsDb, force?: boolean): Promise<{
|
|
14
16
|
toUpload: EmbeddedChunk[];
|
|
15
17
|
toDelete: string[];
|
|
16
18
|
}>;
|
|
17
|
-
|
|
19
|
+
uploadPending(db: EmbeddingsDb): Promise<{
|
|
20
|
+
uploaded: number;
|
|
21
|
+
deleted: number;
|
|
22
|
+
}>;
|
|
23
|
+
sync(db: EmbeddingsDb, force?: boolean): Promise<{
|
|
18
24
|
uploaded: number;
|
|
19
25
|
deleted: number;
|
|
20
26
|
}>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EAEX,aAAa,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;
|
|
1
|
+
{"version":3,"file":"uploader.d.ts","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,WAAW,EAEX,aAAa,EACd,MAAM,wBAAwB,CAAC;AAChC,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,yBAAyB,CAAC;AAGtD,OAAO,EAAa,YAAY,EAAE,MAAM,YAAY,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AASlD,wBAAgB,uBAAuB,CAAC,GAAG,EAAE,OAAO,GAAG,OAAO,CAK7D;AAED,qBAAa,QAAQ;IACnB,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,YAAY,CAAe;IACnC,OAAO,CAAC,MAAM,CAAS;gBAGrB,WAAW,EAAE,WAAW,EACxB,OAAO,GAAE;QAAE,KAAK,CAAC,EAAE,YAAY,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAA;KAAO;IAOzD,OAAO,CAAC,eAAe;IAejB,gBAAgB,CACpB,EAAE,EAAE,YAAY,EAChB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,QAAQ,EAAE,aAAa,EAAE,CAAC;QAC1B,QAAQ,EAAE,MAAM,EAAE,CAAC;KACpB,CAAC;IA4DI,aAAa,CACjB,EAAE,EAAE,YAAY,GACf,OAAO,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAwB3C,IAAI,CACR,EAAE,EAAE,YAAY,EAChB,KAAK,GAAE,OAAe,GACrB,OAAO,CAAC;QACT,QAAQ,EAAE,MAAM,CAAC;QACjB,OAAO,EAAE,MAAM,CAAC;KACjB,CAAC;CA4DH"}
|
package/dist/core/uploader.js
CHANGED
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import { NullLogger } from "../logger/null-logger.js";
|
|
2
2
|
import { createHash } from "crypto";
|
|
3
|
-
import { UploadError } from "./errors.js";
|
|
4
3
|
import { withRetry } from "./utils.js";
|
|
5
|
-
import { readEmbeddingsFile } from "./embeddings-io.js";
|
|
6
4
|
function contentHash(chunk) {
|
|
7
5
|
return (chunk.contentHash ??
|
|
8
6
|
createHash("sha256").update(chunk.content).digest("hex").slice(0, 16));
|
|
9
7
|
}
|
|
8
|
+
export function isFatalVectorStoreError(err) {
|
|
9
|
+
const msg = String(err instanceof Error ? err.message : err).toLowerCase();
|
|
10
|
+
return /schema error|schema mismatch|unauthorized|authentication failed/.test(msg);
|
|
11
|
+
}
|
|
10
12
|
export class Uploader {
|
|
11
13
|
vectorStore;
|
|
12
14
|
retryOptions;
|
|
@@ -27,26 +29,11 @@ export class Uploader {
|
|
|
27
29
|
collection,
|
|
28
30
|
};
|
|
29
31
|
}
|
|
30
|
-
async getItemsToUpload(
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
catch (err) {
|
|
36
|
-
throw new UploadError(`Failed to load embeddings from ${embeddingsFile}: ${err instanceof Error ? err.message : String(err)}`, {
|
|
37
|
-
suggestion: "Run the pipeline without --skip-upload to regenerate embeddings first.",
|
|
38
|
-
cause: err,
|
|
39
|
-
});
|
|
40
|
-
}
|
|
41
|
-
const { meta, chunks: embeddings } = result;
|
|
42
|
-
if (embeddings.length === 0 && !result) {
|
|
43
|
-
throw new UploadError(`No embeddings found at ${embeddingsFile}`, {
|
|
44
|
-
suggestion: "Run the pipeline without --skip-upload to regenerate embeddings first.",
|
|
45
|
-
});
|
|
46
|
-
}
|
|
47
|
-
this.logger.info(`📖 Loaded ${embeddings.length} embeddings from ${embeddingsFile}`);
|
|
48
|
-
// If the vector store changed since the last run, force a full re-upload to ensure
|
|
49
|
-
// the new store starts with consistent state.
|
|
32
|
+
async getItemsToUpload(db, force = false) {
|
|
33
|
+
const meta = db.getMeta();
|
|
34
|
+
const embeddings = db.getAll();
|
|
35
|
+
this.logger.info(`📖 Loaded ${embeddings.length} embeddings from db`);
|
|
36
|
+
// If the vector store changed since the last run, force a full re-upload.
|
|
50
37
|
let effectiveForce = force;
|
|
51
38
|
if (!force &&
|
|
52
39
|
meta?.vectorStoreName &&
|
|
@@ -85,10 +72,27 @@ export class Uploader {
|
|
|
85
72
|
toDelete: [...toDeleteSet],
|
|
86
73
|
};
|
|
87
74
|
}
|
|
88
|
-
async
|
|
75
|
+
async uploadPending(db) {
|
|
76
|
+
const pending = db.getPending();
|
|
77
|
+
if (pending.length === 0)
|
|
78
|
+
return { uploaded: 0, deleted: 0 };
|
|
79
|
+
this.logger.verbose(`📤 Uploading ${pending.length} pending embeddings...`);
|
|
80
|
+
const batchSize = 50;
|
|
81
|
+
for (let i = 0; i < pending.length; i += batchSize) {
|
|
82
|
+
const batch = pending.slice(i, i + batchSize);
|
|
83
|
+
const documents = batch.map((e) => this.chunkToDocument(e));
|
|
84
|
+
await withRetry(() => this.vectorStore.upsert(documents), {
|
|
85
|
+
...this.retryOptions,
|
|
86
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
87
|
+
}, this.logger);
|
|
88
|
+
db.markUploaded(batch.map((e) => contentHash(e)));
|
|
89
|
+
}
|
|
90
|
+
return { uploaded: pending.length, deleted: 0 };
|
|
91
|
+
}
|
|
92
|
+
async sync(db, force = false) {
|
|
89
93
|
this.logger.info("📤 Starting incremental upload...");
|
|
90
94
|
await this.vectorStore.initialize();
|
|
91
|
-
const { toUpload, toDelete } = await this.getItemsToUpload(
|
|
95
|
+
const { toUpload, toDelete } = await this.getItemsToUpload(db, force);
|
|
92
96
|
this.logger.info(`📊 Need to upload: ${toUpload.length} documents`);
|
|
93
97
|
this.logger.info(` Need to delete: ${toDelete.length} files`);
|
|
94
98
|
if (toUpload.length === 0 && toDelete.length === 0) {
|
|
@@ -96,16 +100,23 @@ export class Uploader {
|
|
|
96
100
|
return { uploaded: 0, deleted: 0 };
|
|
97
101
|
}
|
|
98
102
|
if (toDelete.length > 0) {
|
|
99
|
-
await withRetry(() => this.vectorStore.deleteBySourceFile(toDelete),
|
|
103
|
+
await withRetry(() => this.vectorStore.deleteBySourceFile(toDelete), {
|
|
104
|
+
...this.retryOptions,
|
|
105
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
106
|
+
}, this.logger);
|
|
100
107
|
this.logger.verbose(`🗑️ Deleted ${toDelete.length} obsolete docs`);
|
|
101
108
|
}
|
|
102
109
|
if (toUpload.length > 0) {
|
|
103
|
-
const documents = toUpload.map((e) => this.chunkToDocument(e));
|
|
104
110
|
const batchSize = 50;
|
|
105
|
-
const totalBatches = Math.ceil(
|
|
106
|
-
for (let i = 0; i <
|
|
107
|
-
const batch =
|
|
108
|
-
|
|
111
|
+
const totalBatches = Math.ceil(toUpload.length / batchSize);
|
|
112
|
+
for (let i = 0; i < toUpload.length; i += batchSize) {
|
|
113
|
+
const batch = toUpload.slice(i, i + batchSize);
|
|
114
|
+
const documents = batch.map((e) => this.chunkToDocument(e));
|
|
115
|
+
await withRetry(() => this.vectorStore.upsert(documents), {
|
|
116
|
+
...this.retryOptions,
|
|
117
|
+
isRetryable: (err) => !isFatalVectorStoreError(err),
|
|
118
|
+
}, this.logger);
|
|
119
|
+
db.markUploaded(batch.map((e) => contentHash(e)));
|
|
109
120
|
const batchNum = Math.floor(i / batchSize) + 1;
|
|
110
121
|
this.logger.verbose(` Batch ${batchNum}/${totalBatches}: ${batch.length} docs`);
|
|
111
122
|
this.logger.silly(` IDs: ${batch
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,
|
|
1
|
+
{"version":3,"file":"uploader.js","sourceRoot":"","sources":["../../src/core/uploader.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,0BAA0B,CAAC;AACtD,OAAO,EAAE,UAAU,EAAE,MAAM,QAAQ,CAAC;AACpC,OAAO,EAAE,SAAS,EAAgB,MAAM,YAAY,CAAC;AAGrD,SAAS,WAAW,CAAC,KAAoB;IACvC,OAAO,CACL,KAAK,CAAC,WAAW;QACjB,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CACtE,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,uBAAuB,CAAC,GAAY;IAClD,MAAM,GAAG,GAAG,MAAM,CAAC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,CAAC;IAC3E,OAAO,iEAAiE,CAAC,IAAI,CAC3E,GAAG,CACJ,CAAC;AACJ,CAAC;AAED,MAAM,OAAO,QAAQ;IACX,WAAW,CAAc;IACzB,YAAY,CAAe;IAC3B,MAAM,CAAS;IAEvB,YACE,WAAwB,EACxB,UAAqD,EAAE;QAEvD,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,YAAY,GAAG,OAAO,CAAC,KAAK,IAAI,EAAE,CAAC;QACxC,IAAI,CAAC,MAAM,GAAG,CAAC,OAAO,CAAC,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;IACvE,CAAC;IAEO,eAAe,CACrB,KAAoB,EACpB,UAAmB;QAEnB,OAAO;YACL,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,SAAS,EAAE,KAAK,CAAC,SAAS;YAC1B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,UAAU,EAAE,KAAK,CAAC,UAAU;YAC5B,WAAW,EAAE,WAAW,CAAC,KAAK,CAAC;YAC/B,UAAU;SACX,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,EAAgB,EAChB,QAAiB,KAAK;QAKtB,MAAM,IAAI,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC;QAC1B,MAAM,UAAU,GAAG,EAAE,CAAC,MAAM,EAAE,CAAC;QAE/B,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,aAAa,UAAU,CAAC,MAAM,qBAAqB,CAAC,CAAC;QAEtE,0EAA0E;QAC1E,IAAI,cAAc,GAAG,KAAK,CAAC;QAC3B,IACE,CAAC,KAAK;YACN,IAAI,EAAE,eAAe;YACrB,IAAI,CAAC,eAAe,KAAK,IAAI,CAAC,WAAW,CAAC,IAAI,EAC9C,CAAC;YACD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,CAAC,CAAC;YAC7C,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC;YACzD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,gBAAgB,IAAI,CAAC,WAAW,CAAC,IAAI,EAAE,CAAC,CAAC;YAC1D,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAC;YAChE,cAAc,GAAG,IAAI,CAAC;QACxB,CAAC;QAED,IAAI,cAAc,EAAE,CAAC;YACnB,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;YACzE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;QAC5D,CAAC;QAED,MAAM,aAAa,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,eAAe,EAAE,CAAC;QAC/D,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,oBAAoB,aAAa,CAAC,IAAI,oBAAoB,CAC3D,CAAC;QAEF,MAAM,YAAY,GAAoB,EAAE,CAAC;QACzC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAU,CAAC;QAEtC,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;YAC7B,MAAM,YAAY,GAAG,aAAa,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;YAEvD,IAAI,CAAC,YAAY,EAAE,CAAC;gBAClB,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,yBAAyB,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CAClF,CAAC;gBACF,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,IAAI,YAAY,KAAK,GAAG,CAAC,UAAU,EAAE,CAAC;gBAC3C,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,WAAW,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,GAAG,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,WAAW,CACvG,CAAC;gBACF,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAChC,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;YACzB,CAAC;iBAAM,CAAC;gBACN,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,KAAK,GAAG,CAAC,UAAU,WAAW,YAAY,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,cAAc,CACrE,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO;YACL,QAAQ,EAAE,YAAY;YACtB,QAAQ,EAAE,CAAC,GAAG,WAAW,CAAC;SAC3B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,aAAa,CACjB,EAAgB;QAEhB,MAAM,OAAO,GAAG,EAAE,CAAC,UAAU,EAAE,CAAC;QAChC,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QAE7D,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,OAAO,CAAC,MAAM,wBAAwB,CAAC,CAAC;QAE5E,MAAM,SAAS,GAAG,EAAE,CAAC;QACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;YACnD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;YAC9C,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;YAC5D,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,EACxC;gBACE,GAAG,IAAI,CAAC,YAAY;gBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;aACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;YACF,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACpD,CAAC;QAED,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;IAClD,CAAC;IAED,KAAK,CAAC,IAAI,CACR,EAAgB,EAChB,QAAiB,KAAK;QAKtB,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;QAEtD,MAAM,IAAI,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC;QAEpC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,EAAE,EAAE,KAAK,CAAC,CAAC;QAEtE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,MAAM,YAAY,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,sBAAsB,QAAQ,CAAC,MAAM,QAAQ,CAAC,CAAC;QAEhE,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACnD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;YAC3C,OAAO,EAAE,QAAQ,EAAE,CAAC,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;QACrC,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,kBAAkB,CAAC,QAAQ,CAAC,EACnD;gBACE,GAAG,IAAI,CAAC,YAAY;gBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;aACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;YACF,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,QAAQ,CAAC,MAAM,gBAAgB,CAAC,CAAC;QACtE,CAAC;QAED,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,SAAS,GAAG,EAAE,CAAC;YACrB,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,GAAG,SAAS,CAAC,CAAC;YAC5D,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACpD,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC/C,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;gBAC5D,MAAM,SAAS,CACb,GAAG,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,MAAM,CAAC,SAAS,CAAC,EACxC;oBACE,GAAG,IAAI,CAAC,YAAY;oBACpB,WAAW,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,uBAAuB,CAAC,GAAG,CAAC;iBACpD,EACD,IAAI,CAAC,MAAM,CACZ,CAAC;gBACF,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC;gBAC/C,IAAI,CAAC,MAAM,CAAC,OAAO,CACjB,WAAW,QAAQ,IAAI,YAAY,KAAK,KAAK,CAAC,MAAM,OAAO,CAC5D,CAAC;gBACF,IAAI,CAAC,MAAM,CAAC,KAAK,CACf,UAAU,KAAK;qBACZ,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,GAAG,CAAC;qBAC7C,IAAI,CAAC,IAAI,CAAC,EAAE,CAChB,CAAC;YACJ,CAAC;QACH,CAAC;QAED,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,oBAAoB,CAAC,CAAC;QAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,gBAAgB,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QACvD,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,eAAe,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;QAEtD,OAAO,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IACjE,CAAC;CACF"}
|
|
@@ -4,7 +4,7 @@ export function createProgressBar(label, total) {
|
|
|
4
4
|
return { update: () => { }, stop: () => { } };
|
|
5
5
|
}
|
|
6
6
|
const bar = new cliProgress.SingleBar({
|
|
7
|
-
format: `${label} [{bar}] {percentage}% | {value}/{total} | ETA: {eta_formatted}`,
|
|
7
|
+
format: `${label} [{bar}] {percentage}% | {value}/{total} | {duration_formatted} elapsed | ETA: {eta_formatted}`,
|
|
8
8
|
clearOnComplete: false,
|
|
9
9
|
hideCursor: true,
|
|
10
10
|
}, cliProgress.Presets.shades_classic);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"progress-bar.js","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAOvC,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,
|
|
1
|
+
{"version":3,"file":"progress-bar.js","sourceRoot":"","sources":["../../src/progress/progress-bar.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,cAAc,CAAC;AAOvC,MAAM,UAAU,iBAAiB,CAAC,KAAa,EAAE,KAAa;IAC5D,IAAI,KAAK,KAAK,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,MAAM,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,GAAE,CAAC,EAAE,CAAC;IAC9C,CAAC;IAED,MAAM,GAAG,GAAG,IAAI,WAAW,CAAC,SAAS,CACnC;QACE,MAAM,EAAE,GAAG,KAAK,gGAAgG;QAChH,eAAe,EAAE,KAAK;QACtB,UAAU,EAAE,IAAI;KACjB,EACD,WAAW,CAAC,OAAO,CAAC,cAAc,CACnC,CAAC;IACF,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC;IAEpB,OAAO;QACL,MAAM,CAAC,OAAe;YACpB,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACtB,CAAC;QACD,IAAI,CAAC,YAAqB;YACxB,GAAG,CAAC,IAAI,EAAE,CAAC;YACX,IAAI,YAAY;gBAAE,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;QAC9C,CAAC;KACF,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@vivantel/virage-core",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.4",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"description": "Core RAG pipeline tools - universal chunking, embedding, vector store interfaces",
|
|
6
6
|
"repository": {
|
|
@@ -55,6 +55,7 @@
|
|
|
55
55
|
"license": "MIT",
|
|
56
56
|
"dependencies": {
|
|
57
57
|
"@inquirer/prompts": "^8.5.2",
|
|
58
|
+
"better-sqlite3": "^12.10.0",
|
|
58
59
|
"chokidar": "^5.0.0",
|
|
59
60
|
"cli-progress": "^3.12.0",
|
|
60
61
|
"commander": "^15.0.0",
|
|
@@ -66,6 +67,7 @@
|
|
|
66
67
|
"devDependencies": {
|
|
67
68
|
"@eslint/js": "^10.0.1",
|
|
68
69
|
"@supabase/supabase-js": "^2.106.2",
|
|
70
|
+
"@types/better-sqlite3": "^7.6.0",
|
|
69
71
|
"@types/cli-progress": "^3.11.6",
|
|
70
72
|
"@types/node": "^25.9.1",
|
|
71
73
|
"eslint": "^10.4.1",
|
|
@@ -90,6 +90,11 @@
|
|
|
90
90
|
"batchSize": { "type": "number" },
|
|
91
91
|
"maxBatchChars": { "type": "number" },
|
|
92
92
|
"concurrency": { "type": "number" },
|
|
93
|
+
"minIngestionBatchSize": {
|
|
94
|
+
"type": "integer",
|
|
95
|
+
"minimum": 1,
|
|
96
|
+
"description": "Upload embeddings to the vector store mid-run when pending count reaches this threshold."
|
|
97
|
+
},
|
|
93
98
|
"telemetry": { "type": "boolean" },
|
|
94
99
|
"notifications": {
|
|
95
100
|
"type": "object",
|