@backendkit-labs/agent-enterprise 0.8.3 → 0.8.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/unit/ObsidianRAGProvider.checkpoint.test.d.ts +2 -0
- package/dist/__tests__/unit/ObsidianRAGProvider.checkpoint.test.d.ts.map +1 -0
- package/dist/__tests__/unit/ObsidianRAGProvider.checkpoint.test.js +97 -0
- package/dist/__tests__/unit/ObsidianRAGProvider.checkpoint.test.js.map +1 -0
- package/dist/rag/ObsidianRAGProvider.d.ts +7 -0
- package/dist/rag/ObsidianRAGProvider.d.ts.map +1 -1
- package/dist/rag/ObsidianRAGProvider.js +11 -5
- package/dist/rag/ObsidianRAGProvider.js.map +1 -1
- package/package.json +2 -2
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObsidianRAGProvider.checkpoint.test.d.ts","sourceRoot":"","sources":["../../../src/__tests__/unit/ObsidianRAGProvider.checkpoint.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
const fs = __importStar(require("fs/promises"));
|
|
37
|
+
const os = __importStar(require("os"));
|
|
38
|
+
const path = __importStar(require("path"));
|
|
39
|
+
const ObsidianRAGProvider_1 = require("../../rag/ObsidianRAGProvider");
|
|
40
|
+
const VectorStore_1 = require("../../rag/VectorStore");
|
|
41
|
+
const FILES = 35; // 4 batches of 10 (BATCH_SIZE)
|
|
42
|
+
const embedder = {
|
|
43
|
+
model: 'fake',
|
|
44
|
+
embed: async (texts) => texts.map(() => [1, 0, 0]),
|
|
45
|
+
embedOne: async () => [1, 0, 0],
|
|
46
|
+
};
|
|
47
|
+
describe('ObsidianRAGProvider.index — checkpoints are time based, not per batch', () => {
|
|
48
|
+
let dir;
|
|
49
|
+
beforeEach(async () => {
|
|
50
|
+
dir = await fs.mkdtemp(path.join(os.tmpdir(), 'rag-'));
|
|
51
|
+
const vault = path.join(dir, 'vault');
|
|
52
|
+
await fs.mkdir(vault);
|
|
53
|
+
for (let i = 0; i < FILES; i++) {
|
|
54
|
+
await fs.writeFile(path.join(vault, `n${i}.md`), `---\ntitle: Nota ${i}\n---\n\n## Tema\n\nContenido de la nota ${i}.\n`);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
afterEach(async () => { jest.restoreAllMocks(); await fs.rm(dir, { recursive: true, force: true }); });
|
|
58
|
+
const provider = (extra = {}) => new ObsidianRAGProvider_1.ObsidianRAGProvider({
|
|
59
|
+
vaultPath: path.join(dir, 'vault'),
|
|
60
|
+
indexPath: path.join(dir, 'idx.json'),
|
|
61
|
+
embedder,
|
|
62
|
+
batchPacingMs: 0, stagePacingMs: 0,
|
|
63
|
+
...extra,
|
|
64
|
+
});
|
|
65
|
+
it('with the default interval a quick reindex writes the index once, at the end', async () => {
|
|
66
|
+
const save = jest.spyOn(VectorStore_1.VectorStore.prototype, 'save');
|
|
67
|
+
const result = await provider().index();
|
|
68
|
+
expect(result.indexed).toBeGreaterThan(0);
|
|
69
|
+
expect(save).toHaveBeenCalledTimes(1);
|
|
70
|
+
});
|
|
71
|
+
it('checkpointIntervalMs: 0 keeps the old behavior (a save per batch plus the final one)', async () => {
|
|
72
|
+
const save = jest.spyOn(VectorStore_1.VectorStore.prototype, 'save');
|
|
73
|
+
await provider({ checkpointIntervalMs: 0 }).index();
|
|
74
|
+
expect(save).toHaveBeenCalledTimes(Math.ceil(FILES / 10) + 1);
|
|
75
|
+
});
|
|
76
|
+
it('still checkpoints mid-index once the interval has passed, so a crash can resume', async () => {
|
|
77
|
+
let now = 1_000_000;
|
|
78
|
+
jest.spyOn(Date, 'now').mockImplementation(() => now);
|
|
79
|
+
const save = jest.spyOn(VectorStore_1.VectorStore.prototype, 'save');
|
|
80
|
+
const slowEmbedder = { ...embedder, embed: async (t) => { now += 20_000; return embedder.embed(t); } };
|
|
81
|
+
await new ObsidianRAGProvider_1.ObsidianRAGProvider({
|
|
82
|
+
vaultPath: path.join(dir, 'vault'), indexPath: path.join(dir, 'idx2.json'), embedder: slowEmbedder,
|
|
83
|
+
batchPacingMs: 0, stagePacingMs: 0, checkpointIntervalMs: 30_000,
|
|
84
|
+
}).index();
|
|
85
|
+
// each batch "takes" 20 s: saves after batches 2 and 4, plus the final one
|
|
86
|
+
expect(save.mock.calls.length).toBeGreaterThan(1);
|
|
87
|
+
expect(save.mock.calls.length).toBeLessThan(Math.ceil(FILES / 10) + 1);
|
|
88
|
+
});
|
|
89
|
+
it('the index on disk holds every file after the final save', async () => {
|
|
90
|
+
await provider().index();
|
|
91
|
+
const saved = JSON.parse(await fs.readFile(path.join(dir, 'idx.json'), 'utf-8'));
|
|
92
|
+
const text = JSON.stringify(saved);
|
|
93
|
+
expect(text).toContain('Nota 0');
|
|
94
|
+
expect(text).toContain(`Nota ${FILES - 1}`);
|
|
95
|
+
});
|
|
96
|
+
});
|
|
97
|
+
//# sourceMappingURL=ObsidianRAGProvider.checkpoint.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ObsidianRAGProvider.checkpoint.test.js","sourceRoot":"","sources":["../../../src/__tests__/unit/ObsidianRAGProvider.checkpoint.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,gDAAkC;AAClC,uCAAyB;AACzB,2CAA6B;AAC7B,uEAAoE;AACpE,uDAAoD;AAEpD,MAAM,KAAK,GAAG,EAAE,CAAC,CAAC,+BAA+B;AAEjD,MAAM,QAAQ,GAAG;IACb,KAAK,EAAE,MAAM;IACb,KAAK,EAAE,KAAK,EAAE,KAAe,EAAE,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,QAAQ,EAAE,KAAK,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;CAClC,CAAC;AAEF,QAAQ,CAAC,uEAAuE,EAAE,GAAG,EAAE;IACnF,IAAI,GAAW,CAAC;IAChB,UAAU,CAAC,KAAK,IAAI,EAAE;QAClB,GAAG,GAAG,MAAM,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,MAAM,CAAC,CAAC,CAAC;QACvD,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC;QACtC,MAAM,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC;YAC7B,MAAM,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,oBAAoB,CAAC,4CAA4C,CAAC,KAAK,CAAC,CAAC;QAC9H,CAAC;IACL,CAAC,CAAC,CAAC;IACH,SAAS,CAAC,KAAK,IAAI,EAAE,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvG,MAAM,QAAQ,GAAG,CAAC,QAA2C,EAAE,EAAE,EAAE,CAAC,IAAI,yCAAmB,CAAC;QACxF,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC;QAClC,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC;QACrC,QAAQ;QACR,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC;QAClC,GAAG,KAAK;KACX,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QACzF,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAW,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,MAAM,GAAG,MAAM,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QACxC,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sFAAsF,EAAE,KAAK,IAAI,EAAE;QAClG,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAW,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,QAAQ,CAAC,EAAE,oBAAoB,EAAE,CAAC,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QACpD,MAAM,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iFAAiF,EAAE,KAAK,IAAI,EAAE;QAC7F,IAAI,GAAG,GAAG,SAAS,CAAC;QACpB,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC;QACtD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,yBAAW,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;QACvD,MAAM,YAAY,GAAG,EAAE,GAAG,QAAQ,EAAE,KAAK,EAAE,KAAK,EAAE,CAAW,EAAE,EAAE,GAAG,GAAG,IAAI,MAAM,CAAC,CAAC,OAAO,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACjH,MAAM,IAAI,yCAAmB,CAAC;YAC1B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,OAAO,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,EAAE,QAAQ,EAAE,YAAY;YAClG,aAAa,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,oBAAoB,EAAE,MAAM;SACnE,CAAC,CAAC,KAAK,EAAE,CAAC;QACX,2EAA2E;QAC3E,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC3E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,QAAQ,EAAE,CAAC,KAAK,EAAE,CAAC;QACzB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,UAAU,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;QACjF,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;QACjC,MAAM,CAAC,IAAI,CAAC,CAAC,SAAS,CAAC,QAAQ,KAAK,GAAG,CAAC,EAAE,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;AACP,CAAC,CAAC,CAAC"}
|
|
@@ -39,6 +39,12 @@ export interface ObsidianRAGOptions {
|
|
|
39
39
|
stageSize?: number;
|
|
40
40
|
/** Milisegundos de pausa al completar cada etapa (stageSize archivos). Default: 3000ms. */
|
|
41
41
|
stagePacingMs?: number;
|
|
42
|
+
/**
|
|
43
|
+
* Tiempo mínimo entre checkpoints del índice durante index(). Cada checkpoint serializa el índice
|
|
44
|
+
* completo, así que guardar tras cada batch cuesta O(n²) en un vault grande. Un crash pierde como
|
|
45
|
+
* mucho este intervalo de trabajo. Default: 30000ms. 0 guarda tras cada batch (comportamiento anterior).
|
|
46
|
+
*/
|
|
47
|
+
checkpointIntervalMs?: number;
|
|
42
48
|
}
|
|
43
49
|
export declare class ObsidianRAGProvider {
|
|
44
50
|
private readonly embedder;
|
|
@@ -49,6 +55,7 @@ export declare class ObsidianRAGProvider {
|
|
|
49
55
|
private readonly batchPacingMs;
|
|
50
56
|
private readonly stageSize;
|
|
51
57
|
private readonly stagePacingMs;
|
|
58
|
+
private readonly checkpointIntervalMs;
|
|
52
59
|
private indexed;
|
|
53
60
|
constructor(opts: ObsidianRAGOptions);
|
|
54
61
|
/**
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ObsidianRAGProvider.d.ts","sourceRoot":"","sources":["../../src/rag/ObsidianRAGProvider.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAKlE,mDAAmD;AACnD,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IAC/B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2FAA2F;IAC3F,aAAa,CAAC,EAAE,MAAM,CAAC;
|
|
1
|
+
{"version":3,"file":"ObsidianRAGProvider.d.ts","sourceRoot":"","sources":["../../src/rag/ObsidianRAGProvider.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAKlE,mDAAmD;AACnD,MAAM,WAAW,QAAQ;IACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;IAC5C,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;CAC7C;AAED,MAAM,WAAW,kBAAkB;IAC/B,0CAA0C;IAC1C,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;IACnB,oFAAoF;IACpF,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,iFAAiF;IACjF,QAAQ,CAAC,EAAE,QAAQ,CAAC;IACpB,sEAAsE;IACtE,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,sDAAsD;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,kDAAkD;IAClD,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,8EAA8E;IAC9E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,0EAA0E;IAC1E,WAAW,CAAC,EAAE,MAAM,EAAE,CAAC;IACvB;;;;;OAKG;IACH,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2FAA2F;IAC3F,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB;;;;OAIG;IACH,oBAAoB,CAAC,EAAE,MAAM,CAAC;CACjC;AAYD,qBAAa,mBAAmB;IAC5B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAW;IACpC,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAe;IACvC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAS;IAC9B,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAS;IAClC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAS;IACvC,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAS;IAC9C,OAAO,CAAC,OAAO,CAAS;gBAEZ,IAAI,EAAE,kBAAkB;IA+BpC;;;OAGG;IACG,KAAK,CACP,IAAI,GAAE;QAAE,OAAO,CAAC,EAAE,OAAO,CAAC;QAAC,UAAU,CAAC,EAAE,CAAC,CAAC,EAAE;YAAE,SAAS,EAAE,MAAM,CAAC;YAAC,KAAK,EAAE,MAAM,CAAA;SAAE,KAAK,IAAI,CAAA;KAAO,GACjG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC;IAmEhD,oDAAoD;IAC9C,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;IA2B3D,8EAA8E;IAC9E,UAAU,IAAI,cAAc;IAkB5B,IAAI,UAAU,IAAI,MAAM,CAEvB;CACJ"}
|
|
@@ -44,6 +44,7 @@ const BATCH_SIZE = 10;
|
|
|
44
44
|
const DEFAULT_BATCH_PACING_MS = 300;
|
|
45
45
|
const DEFAULT_STAGE_SIZE = 50;
|
|
46
46
|
const DEFAULT_STAGE_PACING_MS = 3000;
|
|
47
|
+
const DEFAULT_CHECKPOINT_INTERVAL_MS = 30_000;
|
|
47
48
|
function sleep(ms) {
|
|
48
49
|
return new Promise(resolve => setTimeout(resolve, ms));
|
|
49
50
|
}
|
|
@@ -56,6 +57,7 @@ class ObsidianRAGProvider {
|
|
|
56
57
|
batchPacingMs;
|
|
57
58
|
stageSize;
|
|
58
59
|
stagePacingMs;
|
|
60
|
+
checkpointIntervalMs;
|
|
59
61
|
indexed = false;
|
|
60
62
|
constructor(opts) {
|
|
61
63
|
if (opts.embedder) {
|
|
@@ -78,6 +80,7 @@ class ObsidianRAGProvider {
|
|
|
78
80
|
this.batchPacingMs = opts.batchPacingMs ?? DEFAULT_BATCH_PACING_MS;
|
|
79
81
|
this.stageSize = opts.stageSize ?? DEFAULT_STAGE_SIZE;
|
|
80
82
|
this.stagePacingMs = opts.stagePacingMs ?? DEFAULT_STAGE_PACING_MS;
|
|
83
|
+
this.checkpointIntervalMs = opts.checkpointIntervalMs ?? DEFAULT_CHECKPOINT_INTERVAL_MS;
|
|
81
84
|
}
|
|
82
85
|
/**
|
|
83
86
|
* Indexa el vault de forma incremental.
|
|
@@ -91,6 +94,7 @@ class ObsidianRAGProvider {
|
|
|
91
94
|
this.store.removeFile(filePath);
|
|
92
95
|
}
|
|
93
96
|
let indexed = 0;
|
|
97
|
+
let lastCheckpoint = Date.now();
|
|
94
98
|
for (let i = 0; i < toIndex.length; i += BATCH_SIZE) {
|
|
95
99
|
const batch = toIndex.slice(i, i + BATCH_SIZE);
|
|
96
100
|
const processedSoFar = Math.min(i + BATCH_SIZE, toIndex.length);
|
|
@@ -106,11 +110,13 @@ class ObsidianRAGProvider {
|
|
|
106
110
|
this.store.removeFile(f.filePath);
|
|
107
111
|
this.store.addChunks(vectorChunks);
|
|
108
112
|
indexed += vectorChunks.length;
|
|
109
|
-
// Checkpoint
|
|
110
|
-
//
|
|
111
|
-
//
|
|
112
|
-
|
|
113
|
-
|
|
113
|
+
// Checkpoint, not just at the end — a crash mid-index (e.g. a large first-time vault
|
|
114
|
+
// reindex) must resume from here instead of re-embedding everything from scratch. By time
|
|
115
|
+
// rather than per batch: each save writes the whole index, so a save per batch is O(n²).
|
|
116
|
+
if (Date.now() - lastCheckpoint >= this.checkpointIntervalMs) {
|
|
117
|
+
await this.store.save();
|
|
118
|
+
lastCheckpoint = Date.now();
|
|
119
|
+
}
|
|
114
120
|
opts.onProgress?.({ processed: processedSoFar, total: toIndex.length });
|
|
115
121
|
if (opts.verbose) {
|
|
116
122
|
console.log(`[RAG] Indexados ${processedSoFar}/${toIndex.length} archivos`);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ObsidianRAGProvider.js","sourceRoot":"","sources":["../../src/rag/ObsidianRAGProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAC7B,uCAAyB;AACzB,4DAA4D;AAE5D,qDAAyG;AACzG,+CAA4C;AAC5C,iDAA8C;
|
|
1
|
+
{"version":3,"file":"ObsidianRAGProvider.js","sourceRoot":"","sources":["../../src/rag/ObsidianRAGProvider.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,2CAA6B;AAC7B,uCAAyB;AACzB,4DAA4D;AAE5D,qDAAyG;AACzG,+CAA4C;AAC5C,iDAA8C;AAmD9C,MAAM,UAAU,GAAG,EAAE,CAAC;AACtB,MAAM,uBAAuB,GAAG,GAAG,CAAC;AACpC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,uBAAuB,GAAG,IAAI,CAAC;AACrC,MAAM,8BAA8B,GAAG,MAAM,CAAC;AAE9C,SAAS,KAAK,CAAC,EAAU;IACrB,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,MAAa,mBAAmB;IACX,QAAQ,CAAW;IACnB,KAAK,CAAc;IACnB,OAAO,CAAe;IACtB,IAAI,CAAS;IACb,QAAQ,CAAS;IACjB,aAAa,CAAS;IACtB,SAAS,CAAS;IAClB,aAAa,CAAS;IACtB,oBAAoB,CAAS;IACtC,OAAO,GAAG,KAAK,CAAC;IAExB,YAAY,IAAwB;QAChC,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;YAChB,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,CAAC;QAClC,CAAC;aAAM,CAAC;YACJ,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,IAAI,0CAAyB,CAAC;YAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,cAAc,IAAI,2CAA0B,CAAC;YAChE,IAAI,CAAC,QAAQ,GAAG,IAAI,+BAAc,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,MAAM,gBAAgB,GAAG,IAAI,CAAC,IAAI,CAC9B,EAAE,CAAC,OAAO,EAAE,EACZ,WAAW,EACX,KAAK,EACL,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,OAAO,CAC1C,CAAC;QACF,IAAI,CAAC,KAAK,GAAG,IAAI,yBAAW,CAAC,IAAI,CAAC,SAAS,IAAI,gBAAgB,EAAE,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAEtF,IAAI,CAAC,OAAO,GAAG,IAAI,2BAAY,CAAC;YAC5B,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,WAAW,EAAE,IAAI,CAAC,WAAW,IAAI,CAAC,OAAO,CAAC;SAC7C,CAAC,CAAC;QAEH,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC;QAC3B,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,GAAG,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACnE,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,kBAAkB,CAAC;QACtD,IAAI,CAAC,aAAa,GAAG,IAAI,CAAC,aAAa,IAAI,uBAAuB,CAAC;QACnE,IAAI,CAAC,oBAAoB,GAAG,IAAI,CAAC,oBAAoB,IAAI,8BAA8B,CAAC;IAC5F,CAAC;IAED;;;OAGG;IACH,KAAK,CAAC,KAAK,CACP,OAA8F,EAAE;QAEhG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,EAAE,CAAC;QAChD,MAAM,EAAE,OAAO,EAAE,QAAQ,EAAE,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC;QAE9E,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAC9B,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACpC,CAAC;QAED,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,IAAI,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAChC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,IAAI,UAAU,EAAE,CAAC;YAClD,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,UAAU,CAAC,CAAC;YAC/C,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,UAAU,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;YAChE,MAAM,cAAc,GAAG,MAAM,OAAO,CAAC,GAAG,CACpC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAChE,CAAC;YAEF,MAAM,MAAM,GAAG,cAAc,CAAC,IAAI,EAAE,CAAC;YACrC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACtB,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;gBACxE,SAAS;YACb,CAAC;YAED,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YAEtE,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,CACzC,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,GAAG,EAAE,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,CACzD,CAAC;YAEF,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACzD,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;YACnC,OAAO,IAAI,YAAY,CAAC,MAAM,CAAC;YAE/B,qFAAqF;YACrF,0FAA0F;YAC1F,yFAAyF;YACzF,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,IAAI,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC3D,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;gBACxB,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YAChC,CAAC;YACD,IAAI,CAAC,UAAU,EAAE,CAAC,EAAE,SAAS,EAAE,cAAc,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAExE,IAAI,IAAI,CAAC,OAAO,EAAE,CAAC;gBACf,OAAO,CAAC,GAAG,CAAC,mBAAmB,cAAc,IAAI,OAAO,CAAC,MAAM,WAAW,CAAC,CAAC;YAChF,CAAC;YAED,sEAAsE;YACtE,wEAAwE;YACxE,4EAA4E;YAC5E,+EAA+E;YAC/E,MAAM,cAAc,GAAG,CAAC,GAAG,UAAU,GAAG,OAAO,CAAC,MAAM,CAAC;YACvD,IAAI,cAAc,EAAE,CAAC;gBACjB,MAAM,cAAc,GAAG,CAAC,GAAG,UAAU,CAAC;gBACtC,MAAM,oBAAoB,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,SAAS,CAAC;oBACpE,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,UAAU,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC/D,MAAM,OAAO,GAAG,oBAAoB,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;gBAC/E,IAAI,OAAO,GAAG,CAAC;oBAAE,MAAM,KAAK,CAAC,OAAO,CAAC,CAAC;YAC1C,CAAC;QACL,CAAC;QAED,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QAEpB,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,QAAQ,CAAC,MAAM,EAAE,CAAC;IACjD,CAAC;IAED,oDAAoD;IACpD,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,IAAa;QACrC,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;QAED,MAAM,cAAc,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;QAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,cAAc,EAAE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,CAAC;QACrE,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,CAAC;QAE/D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACxB,OAAO,oFAAoF,CAAC;QAChG,CAAC;QAED,OAAO,QAAQ;aACV,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACV,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;YAC3B,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,YAAY,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1E,MAAM,KAAK,GAAG,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,aAAa,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9E,OAAO;gBACH,iBAAiB,CAAC,GAAG,CAAC,iBAAiB,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,IAAI,GAAG,KAAK,OAAO;gBACtF,WAAW,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE;gBACrE,KAAK,CAAC,IAAI;aACb,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,CAAC,CAAC;aACD,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IAED,8EAA8E;IAC9E,UAAU;QACN,MAAM,IAAI,GAAG,IAAI,CAAC;QAClB,OAAO,IAAA,uBAAU,EAAC;YACd,IAAI,EAAE,kBAAkB;YACxB,WAAW,EACP,wDAAwD;gBACxD,qGAAqG;YACzG,KAAK,EAAE,cAAC,CAAC,MAAM,CAAC;gBACZ,KAAK,EAAE,cAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,iDAAiD,CAAC;gBAC7E,KAAK,EAAE,cAAC,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,QAAQ,EAAE;qBAC5C,QAAQ,CAAC,wCAAwC,CAAC;aAC1D,CAAC;YACF,OAAO,EAAE,KAAK,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE;gBAChC,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;YACrC,CAAC;SACJ,CAAC,CAAC;IACP,CAAC;IAED,IAAI,UAAU;QACV,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC;IAC3B,CAAC;CACJ;AAtKD,kDAsKC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@backendkit-labs/agent-enterprise",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.5",
|
|
4
4
|
"description": "Business agents with local LLM and RAG knowledge base for agent-core",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"prepublishOnly": "npm run build"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"@backendkit-labs/agent-core": "^0.
|
|
19
|
+
"@backendkit-labs/agent-core": "^0.36.0",
|
|
20
20
|
"@backendkit-labs/retry": "^0.1.2",
|
|
21
21
|
"yaml": "^2.4.0"
|
|
22
22
|
},
|